diff --git a/.env.example b/.env.example index bbe7a97a4..5d7f04270 100644 --- a/.env.example +++ b/.env.example @@ -16,3 +16,11 @@ HOST=0.0.0.0 # PUBLIC_BASE_URL=https://example.com # PUBLIC_HOST=127.0.0.1 # GITHUB_TOKEN= # 可选。面板查 GitHub Release 提高限额；也可用 VM2API_GITHUB_TOKEN + +# PostgreSQL 17. The installer fills a random password for a new local database. +# External PG: set VM2API_DATABASE_URL and clear COMPOSE_PROFILES. +COMPOSE_PROFILES=postgres +VM2API_POSTGRES_PORT=55432 +VM2API_POSTGRES_PASSWORD= +VM2API_DATABASE_URL= +# Preserve VM2API_DB_SECRET when importing an existing SQLite database. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4b07b3ae0..aff32a07a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,20 @@ permissions: jobs: unit: runs-on: ubuntu-latest - timeout-minutes: 8 + timeout-minutes: 20 + services: + postgres: + image: postgres:17-bookworm + env: + POSTGRES_USER: vm2api + POSTGRES_PASSWORD: vm2api-ci-test + POSTGRES_DB: vm2api_test + ports: [5432:5432] + options: >- + --health-cmd "pg_isready -U vm2api -d vm2api_test" + --health-interval 5s --health-timeout 3s --health-retries 12 + env: + VM2API_TEST_DATABASE_URL: postgresql://vm2api:vm2api-ci-test@127.0.0.1:5432/vm2api_test steps: - uses: actions/checkout@v7 - uses: actions/setup-node@v7 @@ -30,8 +43,18 @@ jobs: go-version: '1.25.x' cache-dependency-path: worker/go.sum - run: npm ci + - name: PostgreSQL 17 backup clients + run: | + sudo install -d /usr/share/postgresql-common/pgdg + curl --fail --silent --show-error https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc >/dev/null + . /etc/os-release + echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $VERSION_CODENAME-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null + sudo apt-get update + sudo apt-get install -y postgresql-client-17 + echo /usr/lib/postgresql/17/bin >> "$GITHUB_PATH" + /usr/lib/postgresql/17/bin/pg_dump --version - name: panel user and onboarding regression tests - run: node --test test/unit/panel-users.test.mjs test/unit/onboarding.test.mjs test/e2e/panel-users.e2e.test.mjs + run: node --import ./test/pg-setup.mjs --test test/unit/panel-users.test.mjs test/unit/onboarding.test.mjs test/e2e/panel-users.e2e.test.mjs - name: format-check run: npx biome format - name: syntax-check Node source and scripts @@ -42,8 +65,11 @@ jobs: npm run fetch:payload npm run build:egress - run: npm run test:unit + - run: npm run test:e2e - name: ARM64 host preparation and rollback tests run: python3 -m unittest discover -s test/deploy -v + - name: PostgreSQL application role permissions + run: python3 test/deploy/smoke_postgres_role.py arm64: runs-on: ubuntu-24.04-arm diff --git a/Dockerfile b/Dockerfile index 984611584..77d004651 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,17 @@ FROM node:22-bookworm-slim RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates iptables iproute2 python3 \ && rm -rf /var/lib/apt/lists/* +# Debian Bookworm defaults to PG15; backups require the PG17 client. +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl ca-certificates \ + && install -d /usr/share/postgresql-common/pgdg \ + && curl --fail --silent --show-error -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + && echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends postgresql-client-17 \ + && pg_dump --version | grep -E ' 17\.' \ + && pg_restore --version | grep -E ' 17\.' \ + && rm -rf /var/lib/apt/lists/* COPY --from=docker:27-cli /usr/local/bin/docker /usr/local/bin/docker WORKDIR /opt/vm2api COPY package.json package-lock.json ./ diff --git a/README.md b/README.md index a37f0b33c..a6afb4cf9 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,9 @@ chmod 600 .env docker compose pull && docker compose up -d ``` +数据库使用 PostgreSQL 17。新安装由安装脚本生成内置数据库密码；手工 Compose 安装先配置 `.env` 中的 `VM2API_POSTGRES_PASSWORD`。已有 SQLite 安装须先离线导入，保留原 `VM2API_DB_SECRET`。外部数据库、备份恢复和验证步骤见 [PostgreSQL 部署说明](docs/deploy/postgresql.md)。 + + ### 3. 访问与调用 服务启动后，系统暴露统一服务端口（默认 `8787`）： diff --git a/deploy/Dockerfile.arm64-control b/deploy/Dockerfile.arm64-control index 6e92f83f0..4742b6a72 100644 --- a/deploy/Dockerfile.arm64-control +++ b/deploy/Dockerfile.arm64-control @@ -40,6 +40,17 @@ FROM --platform=linux/arm64 ${NODE_IMAGE} RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates iptables iproute2 python3 \ && rm -rf /var/lib/apt/lists/* +# Debian Bookworm defaults to PG15; backups require the PG17 client. +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl ca-certificates \ + && install -d /usr/share/postgresql-common/pgdg \ + && curl --fail --silent --show-error -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + && echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends postgresql-client-17 \ + && pg_dump --version | grep -E ' 17\.' \ + && pg_restore --version | grep -E ' 17\.' \ + && rm -rf /var/lib/apt/lists/* COPY --from=dockercli /usr/local/bin/docker /usr/local/bin/docker WORKDIR /opt/vm2api COPY package.json package-lock.json ./ diff --git a/deploy/init-arm64-env.py b/deploy/init-arm64-env.py index bcdeb8ebb..66993bd1f 100644 --- a/deploy/init-arm64-env.py +++ b/deploy/init-arm64-env.py @@ -57,11 +57,16 @@ def main() -> int: root = Path(__file__).resolve().parents[1] env_path = root / ".env" + postgres_password = secrets.token_hex(32) values = { "VM2API_ADMIN_USER": "admin", "VM2API_ADMIN_PASSWORD": secrets.token_urlsafe(24), "VM2API_API_KEY": secrets.token_hex(32), "VM2API_DB_SECRET": secrets.token_hex(32), + "COMPOSE_PROFILES": "postgres", + "VM2API_POSTGRES_PORT": "55432", + "VM2API_POSTGRES_PASSWORD": postgres_password, + "VM2API_DATABASE_URL": f"postgresql://vm2api:{postgres_password}@127.0.0.1:55432/vm2api", "VM2API_CONTAINER_NAME": args.container_name, "VM2API_BIND_HOST": args.bind_host, "PORT": str(args.port), @@ -72,7 +77,7 @@ def main() -> int: fd = os.open(env_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) except FileExistsError: print("Existing .env preserved. Review it locally before starting this deployment.") - missing = [key for key in ("COMPOSE_FILE", "VM2API_HOST_ROOT") if not env_has(env_path, key)] + missing = [key for key in ("COMPOSE_FILE", "VM2API_HOST_ROOT", "VM2API_DATABASE_URL") if not env_has(env_path, key)] if missing: print(f"Add to .env: COMPOSE_FILE={COMPOSE_FILE} and VM2API_HOST_ROOT={root} " f"(missing: {', '.join(missing)}).") diff --git a/deploy/install.sh b/deploy/install.sh index 571661b45..ae22590a6 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -279,11 +279,19 @@ env_set() { local name="$1" local value="$2" local envf="${INSTALL_DIR}/.env" - if grep -qE "^${name}=" "$envf" 2>/dev/null; then - sed -i "s|^${name}=.*|${name}=${value}|" "$envf" - else - printf '%s=%s\n' "$name" "$value" >>"$envf" - fi + VM2API_ENV_KEY="$name" VM2API_ENV_VALUE="$value" VM2API_ENV_FILE="$envf" python3 - <<'PYENV' +import os +from pathlib import Path +p = Path(os.environ["VM2API_ENV_FILE"]) +key = os.environ["VM2API_ENV_KEY"] + "=" +value = key + os.environ["VM2API_ENV_VALUE"] + "\n" +lines = p.read_text().splitlines(keepends=True) if p.exists() else [] +if any(line.startswith(key) for line in lines): + p.write_text("".join(value if line.startswith(key) else line for line in lines)) +else: + text = "".join(lines) + p.write_text(text + ("\n" if text and not text.endswith("\n") else "") + value) +PYENV } env_set_if_empty() { @@ -380,6 +388,16 @@ ensure_env() { cp "$example" "$envf" fi chmod 600 "$envf" || true + if [ -z "$(env_get VM2API_DATABASE_URL)" ] && [ -z "$(env_get KIN_DATABASE_URL)" ]; then + if [ -f "${INSTALL_DIR}/data/kin.db" ]; then + err "SQLite 数据尚未迁移。保留运行容器，按 docs/deploy/postgresql.md 完成离线导入后再升级。" + return 1 + fi + env_set_if_empty COMPOSE_PROFILES postgres || true + env_set_if_empty VM2API_POSTGRES_PORT 55432 || true + env_set_if_empty VM2API_POSTGRES_PASSWORD "$(gen_secret)" || true + env_set_if_empty VM2API_DATABASE_URL "postgresql://vm2api:$(env_get VM2API_POSTGRES_PASSWORD)@127.0.0.1:$(env_get VM2API_POSTGRES_PORT)/vm2api" || true + fi env_set_if_empty VM2API_ADMIN_USER "$DEFAULT_ADMIN_USER" || true if env_set_if_empty VM2API_ADMIN_PASSWORD "$DEFAULT_ADMIN_PASSWORD"; then WROTE_DEFAULT_PASSWORD=1 @@ -519,9 +537,14 @@ fetch_file() { # 镜像安装：只落部署文件，不 clone 仓库，不在本机构建。 fetch_release_files() { local tag="$1" - mkdir -p "${INSTALL_DIR}/deploy" + mkdir -p "${INSTALL_DIR}/deploy" "${INSTALL_DIR}/docs/deploy" info "下载部署文件 ${tag} → ${INSTALL_DIR}" fetch_file "$tag" docker-compose.yml "${INSTALL_DIR}/docker-compose.yml" + if grep -q 'vm2api-postgres-entrypoint' "${INSTALL_DIR}/docker-compose.yml"; then + fetch_file "$tag" deploy/postgres-entrypoint.sh "${INSTALL_DIR}/deploy/postgres-entrypoint.sh" + fetch_file "$tag" deploy/postgres-init.sh "${INSTALL_DIR}/deploy/postgres-init.sh" + fetch_file "$tag" docs/deploy/postgresql.md "${INSTALL_DIR}/docs/deploy/postgresql.md" + fi fetch_file "$tag" .env.example "${INSTALL_DIR}/.env.example" fetch_file "$tag" VERSION "${INSTALL_DIR}/VERSION" fetch_file "$tag" CHANGELOG.md "${INSTALL_DIR}/CHANGELOG.md" 1 @@ -631,6 +654,11 @@ start_stack() { fi cd "${INSTALL_DIR}" auto_sync_wrap="${KIN_AUTO_SYNC_WRAP:-0}" + # An existing SQLite installation must pass the read-only import check before replacement. + if [ -f data/kin.db ] || [ -f .local/arm64/data/kin.db ]; then + if [ "$mode" = image ]; then compose pull vm2api; else compose build vm2api; fi + compose run --rm --no-deps -T --entrypoint node vm2api scripts/check-postgres-upgrade.mjs || return 1 + fi if [ "$mode" = image ]; then info "docker compose pull && up -d（拉预构建镜像，本机不构建）" if ! compose pull; then diff --git a/deploy/postgres-entrypoint.sh b/deploy/postgres-entrypoint.sh new file mode 100644 index 000000000..a177fd862 --- /dev/null +++ b/deploy/postgres-entrypoint.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +# Bootstrap postgres never shares the application's password. No password is logged or passed in argv. +if [ -z "${VM2API_POSTGRES_PASSWORD:-}" ]; then + echo 'VM2API_POSTGRES_PASSWORD is required for the application role.' >&2 + exit 1 +fi +POSTGRES_PASSWORD="$(od -An -N32 -tx1 /dev/urandom | tr -d ' \n')" +export POSTGRES_PASSWORD +exec /usr/local/bin/docker-entrypoint.sh "$@" diff --git a/deploy/postgres-init.sh b/deploy/postgres-init.sh new file mode 100644 index 000000000..2ec02d8d9 --- /dev/null +++ b/deploy/postgres-init.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +# Official entrypoint sources this only when initializing an empty data directory. +psql --no-password --set=ON_ERROR_STOP=1 --username=postgres --dbname=postgres <<'SQL' +\getenv app_password VM2API_POSTGRES_PASSWORD +CREATE ROLE vm2api LOGIN PASSWORD :'app_password' NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; +CREATE DATABASE vm2api OWNER vm2api; +SQL diff --git a/docker-compose.yml b/docker-compose.yml index efaba6264..042339942 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,10 @@ services: container_name: ${VM2API_CONTAINER_NAME:-vm2api} restart: unless-stopped init: true + depends_on: + postgres: + condition: service_healthy + required: false network_mode: host cap_add: - NET_ADMIN @@ -17,6 +21,7 @@ services: environment: KIN_PROJECT_ROOT: /opt/vm2api KIN_DATA_DIR: /opt/vm2api/data + VM2API_DATABASE_URL: ${VM2API_DATABASE_URL:-${KIN_DATABASE_URL:-postgresql://vm2api:${VM2API_POSTGRES_PASSWORD:-}@127.0.0.1:${VM2API_POSTGRES_PORT:-55432}/vm2api}} KIN_KERNEL_BIN: /opt/vm2api/bin/kin-kernel KIN_EGRESS_BIN: /opt/vm2api/bin/kin-egress KIN_WORKER_BIN: /opt/vm2api/bin/kin-worker @@ -36,3 +41,27 @@ services: - ./bin:/opt/vm2api/bin - ./share:/opt/vm2api/share - ./src/config:/opt/vm2api/src/config + + postgres: + image: postgres:17-bookworm + profiles: [postgres] + restart: unless-stopped + entrypoint: [/bin/bash, /usr/local/bin/vm2api-postgres-entrypoint] + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_INITDB_ARGS: --auth-host=scram-sha-256 + VM2API_POSTGRES_PASSWORD: ${VM2API_POSTGRES_PASSWORD:-} + ports: + - "127.0.0.1:${VM2API_POSTGRES_PORT:-55432}:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + - ./deploy/postgres-entrypoint.sh:/usr/local/bin/vm2api-postgres-entrypoint:ro + - ./deploy/postgres-init.sh:/docker-entrypoint-initdb.d/10-vm2api.sh:ro + healthcheck: + test: [CMD-SHELL, "pg_isready -U vm2api -d vm2api && psql -U vm2api -d vm2api -Atqc 'SELECT 1' >/dev/null"] + interval: 5s + timeout: 3s + retries: 12 +volumes: + postgres-data: diff --git a/docs/OAUTH.md b/docs/OAUTH.md index d8ddb0c92..402961f4b 100644 --- a/docs/OAUTH.md +++ b/docs/OAUTH.md @@ -1,6 +1,6 @@ # OAuth 凭证生命周期 -活票只写在槽位 `credentials.json`。换票由控制面 host `RefreshIfNeeded` 经槽 SOCKS5 完成；`vm.json`、SQLite 只存脱敏元数据。Go `kin-worker` 不再做 hop / refresh。 +活票只写在槽位 `credentials.json`。换票由控制面 host `RefreshIfNeeded` 经槽 SOCKS5 完成；`vm.json`、PostgreSQL 只存脱敏元数据。Go `kin-worker` 不再做 hop / refresh。 ```text sessionKey（默认 Setup Token）或授权码 diff --git a/docs/PANEL_API.md b/docs/PANEL_API.md index 78030f277..b5ce2faad 100644 --- a/docs/PANEL_API.md +++ b/docs/PANEL_API.md @@ -16,13 +16,13 @@ | `super` | 总览 / 集群 / 用量 / 日志 + 虚拟机 | 读 VM + 拨调度 / 清冷却；集群只读节点列表 | | `admin` | 全部 | `*`。可管理用户 VM，但 admin/master **未 pin** 的 `/v1` 只打未分配平台池 | -环境变量 `VM2API_ADMIN_USER` / `VM2API_ADMIN_PASSWORD` 只在库里还没有同名用户时灌进第一个 admin；之后以 SQLite `users` 为准，面板「用户」页可增删改、改密码。密码 scrypt。不能删/停用最后一个 admin。 +环境变量 `VM2API_ADMIN_USER` / `VM2API_ADMIN_PASSWORD` 只在库里还没有同名用户时灌进第一个 admin；之后以 PostgreSQL `users` 为准，面板「用户」页可增删改、改密码。密码 scrypt。不能删/停用最后一个 admin。 `vms/*.json` 的 `owner_user_id` / `origin`（`platform` \| `admin_assigned` \| `user_created`）是属主 SSOT。`PATCH /vms/:id/owner` 仅 admin。自建 VM 不能收回进平台池。 ## 引导 -所有已登录角色与 master API key 可调，不是公开接口。状态按用户名存在 SQLite `settings` 键 `onboarding_state`；master key（`apiKeyKind=master`）键为 `__master__`。 +所有已登录角色与 master API key 可调，不是公开接口。状态按用户名存在 PostgreSQL `settings` 键 `onboarding_state`；master key（`apiKeyKind=master`）键为 `__master__`。 | 方法 | 路径 | 说明 | |------|------|------| @@ -52,7 +52,7 @@ | PATCH | `/vms/:id/owner` | admin：`{ user_id }` 分配，`{ user_id: null }` 收回（仅 `admin_assigned`） | | GET | `/billing` | 用户计费汇总。`from`/`until`/`group_by=vm|key`。user 隐式只看自己；admin 可 `user_id=` | -启动时会修复旧版本遗留的已删除用户 VM 归属：仅收回 `admin_assigned`（旧记录未写 `origin` 时按已有归属规则推断），不收回租户自建 VM、停用用户或无用户记录的 VM。修改经 `vms/*.json` 写入并同步到 SQLite，不重启槽位。 +启动时会修复旧版本遗留的已删除用户 VM 归属：仅收回 `admin_assigned`（旧记录未写 `origin` 时按已有归属规则推断），不收回租户自建 VM、停用用户或无用户记录的 VM。修改经 `vms/*.json` 写入并同步到 PostgreSQL，不重启槽位。 用户名 `^[a-zA-Z][a-zA-Z0-9._-]{1,31}$`，密码 8–128。`vm_create_quota` 整数 0–100，默认 0。 @@ -155,9 +155,9 @@ Claude PATCH `max_sessions` 返回 400 `claude_max_sessions_forbidden`；OpenAI | 方法 | 路径 | 说明 | |------|------|------| -| GET | `/database/metrics` | `{ sampled_at, database, usage_cache }`；SQLite/WAL 只读快照与官方用量缓存实例统计 | +| GET | `/database/metrics` | `{ sampled_at, database, usage_cache }`；PostgreSQL 与应用连接池只读快照与官方用量缓存实例统计 | -`database` 只执行 `SELECT 1`、只读 PRAGMA、migration 摘要和文件 stat；不返回数据库路径。单项失败为 `null`，文件明确不存在时大小为 `0`，探针失败只令 `database.ok = false`。当前 `node:sqlite` 不提供 SQLite 页缓存 hit/miss，不得从这些字段推算。 +`database` 只执行 `SELECT 1`、数据库大小及 migration 摘要查询；连接池展示当前进程的 total/idle/waiting。失败字段为 `null`，探针失败令 `database.ok = false`。不返回连接地址、用户名或密码。 `usage_cache.hit_rate = (success_hits + error_hits) / requests`；`reuse_rate` 再加 `singleflight_joins`。零请求时均为 `null`。`error_hits` 是负缓存命中，不代表业务成功；累计计数随进程或缓存实例重建归零。 @@ -307,7 +307,7 @@ Claude `routing.quota` 的 5h/7d 闸分三段（`critical_band` 默认 0.05，`g | DELETE | `/refusal-guards` | 须 `{ "confirm": true }` 清空 | | DELETE | `/refusal-device-blocks` | `{ "device_id" }` 解封一条；`{ "confirm": true }` 清空。空 device、短于 8 字符的值不会入库 | -`PUT { enabled }` 写入 SQLite `settings.refusal_guard_enabled`。环境变量 `REFUSAL_GUARD=0` 仍强制关闭。与蒸馏拦截独立：0 注入跳过普通蒸馏针，本缓存仍生效。`count_tokens` 同样在 peek / worker hop 之前拦截。device 取入站 `metadata.user_id.device_id`，否则显式 `device_id` / `x-kin-device-id`，不用 IP、UA 或 API key。仅 admin。 +`PUT { enabled }` 写入 PostgreSQL `settings.refusal_guard_enabled`。环境变量 `REFUSAL_GUARD=0` 仍强制关闭。与蒸馏拦截独立：0 注入跳过普通蒸馏针，本缓存仍生效。`count_tokens` 同样在 peek / worker hop 之前拦截。device 取入站 `metadata.user_id.device_id`，否则显式 `device_id` / `x-kin-device-id`，不用 IP、UA 或 API key。仅 admin。 ## 决策模型与硬正则 @@ -315,7 +315,7 @@ Claude `routing.quota` 的 5h/7d 闸分三段（`critical_band` 默认 0.05，`g |------|------|------| | GET/PUT | `/jev-intercept` | 协议入口。顺序：蒸馏硬规则 → 去掉 reminder、按需展开 base64 后的硬正则 → 拒答缓存 → 决策模型。没配地址则跳过模型；模型失败默认放行。`fail_open: false` 时模型不可用则 403，不封设备 | -`PUT` 写入 SQLite `settings.jev_intercept`。字段：`enabled`（默认关）、`hard_regex_enabled`（默认开）、`openai_platform_enabled`（默认关）、`provider`（`jev` 默认、`laya`、`modernbert`）、`base_url`、`model`、`timeout_ms`（200–8000，默认 2000）、`patterns[]`、`rules`（`null` 与 `[]` 都用内置规则；关掉这一层用 `hard_regex_enabled: false`）、`api_key` 或 `api_keys`（最多 999 把）、`question_ids`（题库 id，缺省为全部）。省略密钥保持原值；`api_key: ""` 或 `api_keys: []` 清除。一行可以是裸 key（用 `base_url`），或 `https://host/v1/systemone ` 打到另一个 systemone。`401` / `429` / `529` 退役该 key，之后不再调度，再次保存密钥列表会重新启用。网络错误和其它 HTTP 状态只跳过这一跳的同一主机，不退役 key；失败放行的原因写在放行 keyword（`网络异常` 或 `上游 `）。另有 `safety_instruction`（空则不加自定义题）、`safety_threshold`（0–1，默认 0.5）、`block_if_below`（默认开）、`fail_open`（默认开）、`dedup_sec`（0–3600，默认 60）、`max_state_chars`（256–64000，默认 16000）、`expand_base64`、`strip_reminders`（后两个默认开）。GET 不回密钥，回 `api_key_set`、`api_key_count`、`question_bank`、`providers`、`builtin_patterns`、`rules`、`rules_customized`、`openai_platform_enabled`。 +`PUT` 写入 PostgreSQL `settings.jev_intercept`。字段：`enabled`（默认关）、`hard_regex_enabled`（默认开）、`openai_platform_enabled`（默认关）、`provider`（`jev` 默认、`laya`、`modernbert`）、`base_url`、`model`、`timeout_ms`（200–8000，默认 2000）、`patterns[]`、`rules`（`null` 与 `[]` 都用内置规则；关掉这一层用 `hard_regex_enabled: false`）、`api_key` 或 `api_keys`（最多 999 把）、`question_ids`（题库 id，缺省为全部）。省略密钥保持原值；`api_key: ""` 或 `api_keys: []` 清除。一行可以是裸 key（用 `base_url`），或 `https://host/v1/systemone ` 打到另一个 systemone。`401` / `429` / `529` 退役该 key，之后不再调度，再次保存密钥列表会重新启用。网络错误和其它 HTTP 状态只跳过这一跳的同一主机，不退役 key；失败放行的原因写在放行 keyword（`网络异常` 或 `上游 `）。另有 `safety_instruction`（空则不加自定义题）、`safety_threshold`（0–1，默认 0.5）、`block_if_below`（默认开）、`fail_open`（默认开）、`dedup_sec`（0–3600，默认 60）、`max_state_chars`（256–64000，默认 16000）、`expand_base64`、`strip_reminders`（后两个默认开）。GET 不回密钥，回 `api_key_set`、`api_key_count`、`question_bank`、`providers`、`builtin_patterns`、`rules`、`rules_customized`、`openai_platform_enabled`。 三个后端走同一套 `/v1/systemone`。题库存在 `questions`：可增删改，缺省是内置六题（综合、色情、破限、逆向、渗透、网络攻击）。每题都是「是否安全」，高分表示安全，启用的题目一起问，任一题低于阈值就拦。`safety_instruction` 非空时再多问一题自定义要求。`POST /jev-intercept/models` 用地址和 Key（留空则用已存的第一把）请求 `{base}/v1/models`，失败回 502，不会把上游 401 当成面板登出。 diff --git a/docs/deploy/postgresql.md b/docs/deploy/postgresql.md new file mode 100644 index 000000000..26248ffde --- /dev/null +++ b/docs/deploy/postgresql.md @@ -0,0 +1,57 @@ +# PostgreSQL 17 部署与 SQLite 导入 + +控制面只使用 PostgreSQL 17。`VM2API_DATABASE_URL` 优先于 `KIN_DATABASE_URL`。连接失败会阻止 HTTP 启动。`VM2API_DB_SECRET` / `KIN_DB_SECRET` 是已有业务字段的加密密钥，导入时必须保留。 + +## 新安装 + +运行 `deploy/install.sh` 会为内置 PostgreSQL 生成随机密码。`docker-compose.yml` 中的 `postgres` profile 默认由 `.env.example` 的 `COMPOSE_PROFILES=postgres` 启用。数据存放在 Compose 的 `postgres-data` 卷。初始化脚本创建 `vm2api` 业务角色，该角色拥有应用数据库，且没有超级用户、创建数据库、创建角色或复制权限。bootstrap `postgres` 密码仅在容器入口随机生成，不与业务密码共用，不写入用户配置或日志。新集群的 TCP 认证使用 SCRAM-SHA-256。端口默认仅映射到宿主 `127.0.0.1:55432`，可通过 `VM2API_POSTGRES_PORT` 修改。Compose 需要支持可选 `depends_on.required` 的版本（2.20 及以上）。 + +手工使用 Compose 时，先在权限为 `0600` 的 `.env` 中填写 `VM2API_POSTGRES_PASSWORD`。控制面的连接 URL 由 Compose 构造；不要把密码放进命令参数或复制到报告。启动命令仍为 `docker compose up -d`。保留 `vms`、配置、数据目录和数据库卷。 + +外部 PostgreSQL：在 `.env` 中填写 `VM2API_DATABASE_URL`，清空 `COMPOSE_PROFILES`。控制面使用 host 网络，数据库地址应为宿主可访问地址，不能使用 Compose 服务名 `postgres`。业务数据库应由本应用专用；应用登录角色需要管理应用 schema 的表、索引、函数和 sequence，无需超级用户或创建/删除数据库权限。应用默认 schema 为 `public`。 + +使用 systemd 直接运行 Node.js 时，在 `/etc/vm2api.env` 中配置连接 URL，并安装 PostgreSQL 17 的 `pg_dump` 与 `pg_restore`。`docs/deploy/vm2api.service` 是启动示例。两种控制面镜像均已包含本机架构的 PG 17 客户端。 + +## 已有 SQLite 安装 + +现有部署的数据迁移和停服应单独安排操作窗口。先在隔离环境演练，保留 SQLite、文件及原业务加密密钥。 + +1. 停止接收新请求，停止后台任务并排空在途请求、事务和 VM 文件镜像写入。保持源应用停止，直至切换完成。 +2. 使用 SQLite 一致性备份或现有备份归档取得源快照。不要只复制仍在写入的 `kin.db` 主文件而遗漏 WAL。保留整个 `vms` 目录（包括 CLI-home）、`src/config`、`data/backups`，以及既有运行槽位的数据和凭据存储。 +3. 准备空的专用 PostgreSQL 17 数据库。在受控进程环境中设置目标连接 URL和原业务加密密钥，执行 `npm run db:import:sqlite -- --source `。工具将检查结构、只读源文件、导入事务、内容摘要、外键和 sequence，完成后记录 `settings.sqlite_import_done`。 +4. 核对导入清单中的逐表结果与业务合计。原样带回对应的 VM、配置和备份目录。旧备份记录中的绝对文件路径需要与实际保留位置一致。 +5. 在新镜像中运行 `scripts/check-postgres-upgrade.mjs` 做只读检查，或通过安装/升级脚本执行该检查。源 `kin.db` 存在时必须查到导入完成标记；同时检查 PG 17 及已应用迁移的版本/校验和。已应用集合须为新镜像迁移序列的非空连续前缀，允许后续新迁移尚未应用。缺表、空记录、未知版本、缺口或校验不一致时不替换运行容器。 +6. 切换连接配置并启动新版本。通过登录、正常 mock 或授权业务请求、请求日志、账单、重启与备份恢复验证数据。记录运行镜像 digest 及验证的提交 SHA。 + +目标接受新写入前，可以停用目标并回到仍保留的源环境。目标接受写入后，原 SQLite 已过时。此时直接切回会丢失目标新增数据，应先设计反向数据处理方案。 + +旧 SQLite `.tar.gz` / `.db.gz` 归档使用 `npm run db:convert:sqlite-backup -- --source --output ` 转换。该命令会先把源数据导入环境连接指向的数据库，所以目标必须是预建的专用空转换库；不得指向运行库。转换库 schema 必须与最后恢复目标一致，通常为 `public`。旧 `.db.gz` 不包含文件，转换时必须通过 `--vm-dir <目录>` 和 `--config-dir <目录>` 提供对应 VM 与配置快照；旧 tar 归档缺少目录时也必须补充。仅凭脱敏数据库镜像不能恢复活凭据。生成的 v2 归档保留 VM/配置文件及摘要，随后才可在线恢复。在线恢复会返回 `legacy_backup_conversion_required`，避免将 SQLite 文件作为 PG dump 执行。 + +## PostgreSQL 备份与恢复 + +新归档仍是 `kin-backup-*.tar.gz`。清单 `version=2`，数据库文件为 `db/postgres.dump`，同时包含顶层 VM JSON、顶层配置 JSON、schema 版本和 SHA-256。文件权限为 `0600`。CLI-home、VM 磁盘和槽位凭据目录继续使用原有存储，迁移时单独保留。归档恢复替换顶层 JSON 范围，保留这些子目录和其他非 JSON 文件。 + +普通备份先短时暂停写入并排空在途任务，在专用事务中导出 PostgreSQL 快照，同时复制 VM 和配置文件。文件复制完成后恢复写入。worker 使用同一个导出快照执行 `pg_dump`，导出事务一直保留到 dump 完成。 + +恢复校验外层与内部文件摘要，再检查 `pg_restore --list` 对象目录。恢复只允许本应用 schema 中的已支持对象。数据库使用 `pg_restore --single-transaction --clean --if-exists --no-owner --no-privileges`，不创建或删除整个数据库，也不恢复其他角色的所有权或授权。 + +恢复期间拒绝新增业务写入，后台任务暂停，所有在途操作排空。恢复前自动创建 `pre_restore` 备份。数据库和文件都恢复后，等待内存 store、路由与调度状态重新加载，再退出维护状态。恢复失败会尝试恢复前快照；失败状态持续保持维护。`data/restore-maintenance.json` 在修改数据库/文件前写入，只有恢复和重新加载成功才删除。进程重启发现该文件时会在迁移或 VM 初始化前拒绝普通启动。进程仍运行时，管理员可以通过原恢复接口显式重试。进程退出后，应在停服状态下使用恢复前归档恢复对应 PostgreSQL、VM 与配置快照，验证一致性后才移除此文件并重新启动。不要在原因未查明时直接删除标记。 + +`pg_restore --list` 是对象目录预检。恢复行为还需在隔离数据库实际演练。PG dump 不含全局角色。新部署使用目标角色拥有恢复对象。 + +## 本地与 CI 验证 + +设置专用 `VM2API_TEST_DATABASE_URL`，数据库名必须以 `_test` 结束，且账号不得访问部署数据。Node 测试通过 `test/pg-setup.mjs` 分配隔离 schema。备份恢复和派生网关有各自的隔离目标，测试结束后清理。 + +```sh +npm run test:unit +npm run test:e2e +npm run test:go +pnpm -C web test +pnpm -C web build +python3 -m unittest discover -s test/deploy -v +``` + +测试缺少 PostgreSQL 或 PG 17 客户端时应报告失败，不能切回 SQLite。ARM64 镜像冒烟使用临时 Docker 网络及独立 PG 17 容器，不挂载部署凭据或宿主 Docker socket。 + +只读审计命令使用环境中的连接和应用 schema，强制只读会话。数据检查示例：`node scripts/audit-billing-contract.mjs --database --json`。另外三个 `audit-billing-gaps.mjs`、`audit-billing-openai.mjs`、`audit-fable-week.mjs` 直接查询环境指向的 PG，不再接受 SQLite 路径。 diff --git a/docs/deploy/vm2api.service b/docs/deploy/vm2api.service index 6dd51961f..89d489324 100644 --- a/docs/deploy/vm2api.service +++ b/docs/deploy/vm2api.service @@ -6,6 +6,7 @@ Wants=network-online.target [Service] Type=simple WorkingDirectory=/opt/vm2api +# /etc/vm2api.env must contain VM2API_DATABASE_URL. See postgresql.md. EnvironmentFile=/etc/vm2api.env ExecStart=/usr/bin/node src/server.mjs Restart=on-failure diff --git a/docs/plans/2026-10-21-postgresql-migration-results.md b/docs/plans/2026-10-21-postgresql-migration-results.md new file mode 100644 index 000000000..dc73878f5 --- /dev/null +++ b/docs/plans/2026-10-21-postgresql-migration-results.md @@ -0,0 +1,80 @@ +# PostgreSQL 迁移实施与验证 + +日期：2026-10-21。分支：`codex/postgresql-migration`。 + +运行数据库已改为 PostgreSQL 17。异步数据访问、业务调用链、离线导入、新备份恢复、部署配置和数据库页面已实现。未推送分支，未对现有部署执行数据迁移、配置变更、重启或发布。 + +原始提交：`eee8f6ecb6ca04455e6edef0c8e498fb3be21f21`。运行代码提交：`6aa877de`，测试隔离提交：`3054bfa0`。完整 SHA 见随附 [验证摘要](evidence/postgresql/verification.json)。后续文档提交不改变被验证的运行代码。 + +## 实现范围 + +- `pg.Pool`、原生 `$n` SQL、固定事务连接、savepoint、迁移互斥和校验和。PostgreSQL 建立独立基线；SQLite 的 41 个历史迁移保持原样。 +- repository、用户与密钥、请求日志、计费统计、调度、集群、VM 镜像和面板调用均适配异步访问。同步调度使用已初始化的工作集，写入按序排队，维护和退出前排空。面板在写入成功后发送 JSON 响应。 +- 64 位累计整数在安全范围内返回 Number，超范围返回精确十进制字符串。已失精的 Number 被拒绝。金额、平均值、比例保留原浮点语义。保留现有合法小数权重和端点优先级。 +- 旧 SQLite 通过只读连接取得一致性快照，只在私有副本升级历史版本。空目标检查、逐表摘要、历史表、运行时表、密文、sequence 高水位和完成标记在同一导入事务内处理。 +- PG custom dump 使用导出快照，VM/配置文件在相同暂停写入窗口复制。恢复验证文件摘要、路径及数据库对象范围，以单事务恢复本应用 schema，重新加载内存状态。失败保留维护状态和持久标记。 +- Compose 支持内置或外部 PostgreSQL。应用角色没有超级用户、建库、建角色或复制权限。两种控制面 Dockerfile 安装 PG 17 客户端。升级在替换旧容器前检查连接、版本、已应用迁移前缀、校验和和旧数据导入标记。 + +SQLite 只保留在离线迁移、旧备份转换和历史 fixture 中。运行入口不再依赖 SQLite，也不提供双写或降级路径。 + +VM 数据库记录保留元数据及凭据存在标记。活凭据仍位于既有槽位和 CLI-home 存储中。顶层 JSON 备份及数据库导入不能替代这些目录；切换时必须保留原目录和业务加密密钥。 + +## 验证结果 + +测试只使用本任务创建的 PostgreSQL 17.11 和合成数据。没有读取现有部署的连接或业务数据。 + +| 检查 | 被验证提交 | 实际结果 | +| --- | --- | --- | +| 全量单元测试 | `4ad444d7` | 281 文件；3325 通过、7 失败、10 跳过。2 个失败文件的 7 个断言与原始提交相同 | +| 全量端到端测试 | `3054bfa0` | 26 文件；96 通过、17 失败、1 跳过。11 个失败文件的 17 个断言与原始提交相同，无清理超时 | +| 测试隔离与恢复范围 | `3054bfa0` | 驱动、导入、备份核心/服务/E2E 共 40/40，通过实际恢复确认相邻 schema 的后续写入保留 | +| Node 22 容器兼容性 | `4ad444d7` | 93/93 通过，覆盖驱动、SQLite 导入、大整数、账号额度和重启持久化 | +| 前端测试、构建及格式 | `4ad444d7` | 60 文件、473/473 测试通过；TypeScript/Vite 生产构建和格式检查通过 | +| Go | `4ad444d7` | worker 与 api-kernel 的 race 测试通过 | +| 部署配置 | `4ad444d7` | Python 25/25；PostgreSQL 业务角色的实际认证、DDL 和权限拒绝检查通过 | +| 语法与格式 | `4ad444d7`，末次维护改动另查 | Node source/scripts 语法、Biome、Shell、前端格式及 `git diff --check` 通过 | +| amd64 控制面镜像 | `6aa877de` | 构建通过；实际容器 HTTP、PG 指标、密钥写入、备份和恢复后内存状态检查通过 | + +Node 22、前端、Go 和部署检查后的运行代码只增加普通维护提示修正；该修正由最终端到端测试、独立维护时序复现及最终镜像验证覆盖。 + +全量测试仍有原始提交的失败。独立原始 checkout 使用相同预编译 payload 和测试依赖重跑，并逐项比较测试名称和断言正文。详细清单见 [基线对照](evidence/postgresql/baseline-comparison.json)。没有把这些失败改为跳过或通过，也没有为本次数据库迁移修改相关业务行为。 + +失败输出示例（两份源码结果相同）： + +- `cli-hop-body.test.mjs:34`：实际 `ttl='1h'`，预期 `undefined`。 +- `convert-cache-shape.test.mjs:155`：实际 `{type:'ephemeral', ttl:'1h'}`，预期 `{type:'ephemeral'}`。 +- `auto-mode.e2e.test.mjs:608`：实际 thinking 包含 `display:'summarized'`，预期对象没有该字段。 +- `account-pool-failover.e2e.test.mjs:66`：`TypeError: Cannot read properties of undefined (reading 'account_id')`。 + +第一次最终 E2E 在 `4ad444d7` 有 12/26 文件失败、94 通过、19 失败、1 跳过。其中两项慢代理场景在绑定结束后立即收到 503。独立复现确认：绑定耗时约 10–16 秒，5 秒启动备份定时器进入维护并等待绑定完成；后继请求短时拒绝，约 52–54 毫秒后恢复 200。禁用自动备份时同链请求全部 200。这两项业务 fixture 已显式禁用无关后台备份；独立自动备份 E2E 仍使用默认开启设置。 + +后续全量 E2E 的 5 个备份场景全部通过，但文件清理的 `DROP DATABASE` 等待 PostgreSQL 全实例 checkpoint 超过测试文件 90 秒限制。只读采样为 `IPC/CheckpointDone`、无阻塞连接；独立复现主体 487 毫秒、进程清理总耗时约 109 秒。测试现统一使用专用 `_test` 基库内的独立 schema，移除建库/删库及 CI 的 checkpoint 参数调整。生产恢复始终限定 schema。修正后相关 40 场景 9.8 秒结束。 + +新增回归先于修复独立提交，覆盖异步集群编辑、平台持久化、日志清理、整数精度、面板处理返回值、小数导入、定时探测结果写入，以及备份调度关闭竞态。 + +测试进程的完整日志保留在本地忽略目录 `.tmp/pg-evidence/`。关键文件为 `unit-final.log`、`e2e-complete.log`、`schema-isolation-final.log`、`node22-final.log`、`image-smoke-final.log` 和 `docker-build-final.log`。摘要和页面对照图随分支提交，不包含连接信息或凭据。 + +## 镜像与页面证据 + +最终实际运行镜像 ID：`sha256:d93e36d953db42aa80acd562a2af63cf0a969789d154d063610c0e6e060dc33e`。本地标签：`vm2api:pg-migration-check`。容器内版本为 Node `v22.23.2`、`pg_dump` / `pg_restore` `17.11`。临时容器没有挂载现有数据或宿主 Docker socket，验证结束后删除。 + +本机没有 buildx。使用旧 builder 时，只在临时 Dockerfile 顶部补 `ARG BUILDPLATFORM=linux/amd64`，其余构建指令与仓库 Dockerfile 相同。首次默认网络的 Go 依赖下载超时；使用 host 构建网络重试成功，未修改依赖版本或现有网络配置。 + +数据库页面复用既有组件和样式，用合成数据在 1440×900 窗口对照。用户已授权独立 Playwright。 + +| SQLite 基线 | PostgreSQL 实现 | +| --- | --- | +| ![SQLite](evidence/postgresql/database-sqlite-1440.png) | ![PostgreSQL](evidence/postgresql/database-postgresql-1440.png) | + +## 操作入口与限制 + +部署与切换步骤见 [PostgreSQL 部署说明](../deploy/postgresql.md)。连接和原业务加密密钥通过受控环境提供，不能放到命令参数或报告中。 + +```sh +npm run db:import:sqlite -- --source +npm run db:convert:sqlite-backup -- --source --output --vm-dir --config-dir +``` + +导入目标必须为空。转换旧备份使用专用空转换库，不能指向运行库。旧 `.db.gz` 缺少 VM 和配置文件，需要对应快照。目标开始接受新写入后，原 SQLite 不再是可直接切回的最新副本。 + +ARM64 的 Dockerfile、部署脚本、配置测试和 CI 冒烟入口已更新；本机未执行原生 ARM64 镜像构建及运行。现有环境的数据规模、停服切换、实际槽位凭据、真实外部上游、生产备份恢复和发布均未验证。 diff --git a/docs/plans/2026-10-21-postgresql-migration.md b/docs/plans/2026-10-21-postgresql-migration.md new file mode 100644 index 000000000..556b5e28f --- /dev/null +++ b/docs/plans/2026-10-21-postgresql-migration.md @@ -0,0 +1,239 @@ +# SQLite 全面迁移到 PostgreSQL + +日期：2026-10-21（北京时间） +调研基线：`eee8f6ecb6ca04455e6edef0c8e498fb3be21f21` +实施分支：`codex/postgresql-migration` +状态：迁移实现已完成；全量回归与原始提交的失败对照见 [实施与验证报告](2026-10-21-postgresql-migration-results.md)。 + +## 1. 目标与边界 + +将控制面的运行数据库统一为 PostgreSQL，包括用户、API 密钥、分组、账号、调度状态、代理、集群节点、虚拟机元数据镜像、请求日志、计费、备份记录和思考签名归属。 + +应用运行、单元测试、端到端测试、运维查询和新备份都使用 PostgreSQL。SQLite 仅用于离线读取旧数据库、构造历史迁移测试数据及转换旧备份。运行时不保留双写或 SQLite 降级路径。 + +本分支完成代码、测试、迁移工具和部署说明。执行现有环境的数据迁移、停服、发布、修改现有服务配置和推送分支均不包含在本次本地实施范围内，需要相应操作的明确授权。 + +保留现有业务对象、权限边界、文件目录用途及对外 API 契约。`vms/*.json` 目前仍是 VM 运行操作的文件入口；本次不额外将整个 VM 文件模型改为数据库唯一存储。`VM2API_DB_SECRET` / `KIN_DB_SECRET` 仍是业务字段加密密钥，不能当作 PostgreSQL 登录密码，也不能在迁移时重新生成。 + +## 2. 已核实的现状 + +| 范围 | 现有实现 | 迁移影响 | +| --- | --- | --- | +| 数据库入口 | `src/lib/db/database.mjs` 使用 `node:sqlite.DatabaseSync`，默认 `/kin.db`，支持 `KIN_DB_PATH` | 打开、迁移、查询、事务和关闭均需异步化 | +| 数据访问 | `src/lib/db/repos/` 中 21 个模块，包含 20 个 repository/view 和 1 个共享 SQL 模块 | 调用者不能继续把查询结果当作同步值 | +| 版本迁移 | `src/lib/db/migrations/001_init.sql` 至 `041_usage_logs_inbound_device.sql`，逐文件事务和 SHA-256 校验 | PostgreSQL 建立独立迁移序列；原 SQLite 文件保持不变 | +| 最终表结构 | 在临时 SQLite 执行 41 个迁移后，有 29 张非 SQLite 内部表，包含 `schema_migrations` 和 3 张 `_legacy` 表 | 建立逐表清单，核对列、索引、默认值、外键及保留数据 | +| 额外表 | `src/lib/pool/thinking-owner.mjs` 在运行时创建 `thinking_signature_owners` | 纳入 PostgreSQL 正式迁移与存量导入 | +| VM 元数据镜像 | `src/lib/vm/vm-db-sync.mjs` 通过写钩子、启动校对及文件监听镜像到数据库；`VmsRepo` 移除活凭据，只保留元数据和凭据存在标记 | 异步写入必须可等待、可排空，删除顺序不能改变 | +| 备份 | `src/lib/admin/backup-core.mjs` 使用 `VACUUM INTO`，归档含 `db/kin.db`、VM 和配置文件 | 更换数据库快照格式，保留原归档的其他内容 | +| 恢复 | `backup-service.mjs` 关闭 SQLite、替换文件、重新绑定 store | PostgreSQL 恢复需维护模式、排空写入及恢复后重新加载 | +| 恢复入口隔离 | `src/server.mjs` 的恢复标志目前主要拦截推理入口 | 需要同时控制面板和后台数据库写入 | +| 监控与界面 | `database-metrics.mjs` 和数据库页面展示 SQLite/WAL/PRAGMA/页信息 | 改为 PostgreSQL 数据库、连接池和迁移指标 | +| 部署 | 控制面 Compose 使用 `network_mode: host`；存在 amd64、arm64 两种镜像构建 | PostgreSQL 地址、网络暴露、客户端版本及两种构建都需覆盖 | +| 测试 | 临时目录隔离 SQLite，`test/harness.mjs` 派生网关进程 | 改用独立 PostgreSQL 测试数据库，并隔离每个测试实例 | + +当前 SQLite 表清单： + +```text +account_allocations, account_groups, account_runtime, +account_runtime_states_legacy, accounts, accounts_legacy, +api_endpoint_keys, api_endpoint_models, api_endpoints, api_keys, +backup_records, cluster_node_direct_links, cluster_nodes, groups, +panel_users_legacy, proxies, redeem_codes, refusal_device_blocks, +refusal_guards, request_attempts, request_log_debug, schema_migrations, +settings, sticky_sessions, usage_logs, users, vm_pool_members, vm_pools, vms +``` + +`thinking_signature_owners` 为以上清单之外的运行时表。数据导入不能仅按 migrations 中的建表语句搜索结果决定范围。 + +## 3. 实现方案 + +### 3.1 驱动、生命周期与事务 + +- 使用 `pg`（node-postgres）连接池和原生异步查询。SQL 参数使用 `$1`、`$2` 等占位符；仓库不增加运行时 SQLite SQL 翻译器或同步线程代理。 +- 连接配置沿用现有 `VM2API_*` 优先、`KIN_*` 作为 fallback 的规则：`VM2API_DATABASE_URL` 优先于 `KIN_DATABASE_URL`，仅接受 PostgreSQL 连接字符串。未配置或连接失败时，在监听 HTTP 端口之前失败。`KIN_DB_PATH` 仅保留在离线源文件工具中，应用不得悄悄回退到 SQLite。 +- `await openDatabase()` 完成连接探针、迁移和必要初始化，再创建或初始化 store。构造函数不执行未等待的异步数据库写入；需要初始化时使用显式 `init()` 或工厂，并在启动阶段等待。 +- `await closeDatabase()` 排空连接池。关闭、恢复和测试清理必须等待正在执行的操作结束。 +- `withTransaction` 等待回调完成后提交。一个事务中的查询固定使用同一个连接，嵌套调用沿用同一事务。实现时选择显式传递事务对象；调用层级确有必要时再使用 `AsyncLocalStorage`，不同时维护两套事务机制。 +- 迁移使用数据库级 advisory lock 串行化并发启动。迁移锁、SQL 与迁移记录必须在同一个专用连接执行；连接在成功和异常路径都释放。 +- SQL 错误不得打印连接字符串、密码、参数值或凭据列。事务中不得沿用 SQLite 的“捕获单行错误后继续执行”方式；可预先校验的数据先校验，必须局部容错的操作使用 savepoint。 + +### 3.2 表结构和数据契约 + +- PostgreSQL 基线按当前 SQLite 的最终结构建立，不在 PostgreSQL 上重放 41 个旧迁移。新旧 `schema_migrations` 分别记录各自引擎的版本，不把 SQLite 校验和当作 PostgreSQL 校验和。 +- 基线包含 `thinking_signature_owners`。3 张 `_legacy` 表中的历史记录也纳入显式保留清单，不静默丢弃。旧 SQLite 的迁移记录写入导入清单作为源版本证据。 +- JSON 和 ISO 时间先保留为 `text`；整数布尔值保持 `0/1`。本次不混入 `jsonb`、时间输出格式或业务字段加密格式的改变。 +- 原 SQLite `REAL` 按其双精度语义映射到 `double precision`。不在同一次迁移中改变计费舍入规则。余额和费用使用固定测试数据比较迁移前后结果。 +- SQLite `INTEGER` 是 64 位。毫秒时间戳、token 计数、累计次数、字节数等不能机械改为 PostgreSQL 32 位 `integer`；逐列选择 `bigint` 或有界 `integer`。统一处理 `pg` 返回的 bigint/count/sum，超出 JavaScript 安全整数时不得静默截断。 +- 自增列使用 identity/sequence，并用 `INSERT ... RETURNING id` 取值。固定分组种子和导入的显式 ID 完成后更新 sequence。 +- 保留现有外键和级联语义。本次不额外为旧数据增加未评估的业务约束。 +- 用户名和 VM 池名称保持现有 SQLite `NOCASE` 行为。其默认大小写折叠主要针对 ASCII，不能直接用 Unicode `lower()` 宣称等价。实现可用固定 ASCII 大小写映射表达式及一致的唯一索引；登录查询与索引必须采用同一规则，并测试非 ASCII 名称。 +- PostgreSQL 新空库包含默认分组；存量导入时应在同一事务中移除基线种子并导入源分组，不能因为种子存在而绕过“目标为空”的检查。 + +### 3.3 SQL 与异步调用链 + +| SQLite 用法 | PostgreSQL 实现要求 | +| --- | --- | +| `?` | 调用处显式使用 `$n`，保持参数化 | +| `IFNULL` | 改为 `COALESCE`，核对聚合返回类型 | +| `LIKE` | 日志搜索保留默认 ASCII 大小写不敏感和反斜杠字面量语义；两侧使用一致 ASCII fold 并显式 `ESCAPE ''`，对照验证 `%` / `_` 通配行为，不直接将 `ILIKE` 当作等价替代 | +| `INSERT OR IGNORE` | 改为带实际冲突目标的 `ON CONFLICT DO NOTHING`，不吞掉其他约束错误 | +| `lastInsertRowid` | `RETURNING id` | +| `strftime` / `printf` | PostgreSQL 时间或字符串表达式；保持 UTC、上海自然日和 IANA 时区/夏令时行为 | +| `json_each` | 原生数组参数及 `ANY`，或显式 JSON 数组展开 | +| `rowid` 分页 | 稳定业务主键的 keyset pagination，不使用 `ctid` | +| 宽松 `GROUP BY` | 选择列与分组表达式一致；重点修正 statistics leaderboard | +| 浮点 `CAST(... AS INTEGER)` | 明确截断或舍入，不能依赖两种引擎的隐含差异 | +| PRAGMA / `sqlite_master` | 正式迁移、PostgreSQL catalog 或 information_schema | + +新增 `await` 后还要检查下列语义： + +1. `ApiKeyStore` 的鉴权、额度和并发准入：数据库读取完成后，最后的内存限流检查与 inflight 占位连续执行；失败路径释放占位。 +2. 账号状态、额度、余额等读改写：改为原子 UPDATE 或行锁事务，避免两个 await 路径相互覆盖。兑换码保持一次使用与余额更新的原子性。 +3. 同步配置回调：在异步边界预取，或复用已有配置的内存状态并在启动/修改/恢复时等待更新，不能把 Promise 当配置对象。 +4. 定时任务、事件监听、数组遍历和流式响应结束回调：明确等待或排队，禁止 `forEach(async ...)` 等未跟踪写入。 +5. VM 写钩子：保留现有文件入口，提供有顺序的数据库镜像写入与 flush；读取镜像、备份、删除、恢复和关闭前等待相关写入。失败不能报告镜像成功；删除后不能被迟到 upsert 或启动恢复重新创建。 + +### 3.4 存量迁移 + +提供显式离线命令，暂定 `npm run db:import:sqlite -- --source `。目标连接只从环境读取，命令行不携带密码。应用启动时不自动发现或导入旧文件。 + +1. 切换操作前冻结原应用所有写入，排空在途请求和后台任务。以 SQLite 一致性快照和文件归档作为输入，不直接复制正在写入的 `.db` 主文件而遗漏 WAL。 +2. 原 SQLite 文件只读保留。历史版本升级仅在工具创建的临时副本执行，复用原校验和及历史 001 schema 修复逻辑；不就地修改原库。 +3. 识别旧版本和未知表/列。未知版本、校验和错误或存在未映射业务数据时拒绝导入，给出不含业务行的诊断。额外历史表和运行时表按清单迁移。 +4. 校验目标为新基线且没有业务数据。导入使用一个事务并互斥执行，按依赖顺序导入；对集群节点自引用关系采用父节点优先顺序或可延迟约束，不能靠关闭约束忽略问题。 +5. 保留主键、NULL、租户归属、密文、JSON 字符串、时间文本和业务值。SQLite 源查询启用 `statement.setReadBigInts(true)`，超出 JavaScript 安全整数的值用无损十进制字符串绑定 PG、计算摘要和序列化，不能先转成 `Number`。保持原业务加密密钥，禁止将凭据解密后输出或重写。 +6. 更新 sequence 时取源 `sqlite_sequence.seq` 和最大现存 ID 的较大值，保留已经删除记录的自增高水位。采用可回滚的 sequence restart；若使用不可回滚的 `setval`，必须保存并恢复失败前序列状态并测试。逐表比较行数、主键集合和按稳定主键顺序计算的规范化内容摘要；另核对余额、费用、token 合计及外键关系。浮点聚合约定容差，逐行 double precision 应保留相同数值。 +7. 校验通过后，将源版本、源文件摘要、逐表验证结果和完成标记与数据一并提交。失败整体回滚；不能留下“已导入”标记或半成品。再次执行对已导入目标明确拒绝。 +8. 保留输入快照、文件归档和不含敏感值的结果清单。目标开始接受新写入前可以回到原 SQLite；目标接受写入后，原库已过时，不能直接切回并声称无数据丢失。 + +### 3.5 备份、恢复与运维 + +- 新备份格式使用 `pg_dump` custom archive，并继续归档 VM JSON、现有配置和清单。清单写明备份格式、数据库引擎、schema 版本和文件摘要，不包含连接凭据。 +- 数据库快照与 VM/配置文件必须有明确的一致性边界。暂停应用的数据库与相关文件写入、排空 VM 镜像队列，在专用事务中用 `pg_export_snapshot()` 建立快照，并在该冻结窗口复制文件。随后恢复写入，用 `pg_dump --snapshot` 导出；导出事务保持到 dump 完成为止。子进程已启动不代表快照已建立。不能把独立时刻的数据库 dump 与凭据文件称为一致快照。 +- 备份在异步子进程/现有 worker 执行，保证 HTTP 事件循环可响应。PostgreSQL 客户端主版本与支持的服务端版本对齐；两种 Debian Bookworm 镜像通过官方 PGDG 来源安装 `postgresql-client-17`，验证 amd64、arm64 的 `pg_dump` / `pg_restore` 主版本。不能只安装发行版默认客户端而得到 PG 15。 +- 新版恢复显式进入维护模式：拒绝新增推理和面板写入，暂停后台写任务，排空请求、事务及 VM 镜像队列，再制作恢复前备份。维护操作使用受控内部入口，避免被自身写入门禁止。 +- 恢复实现先在隔离测试数据库验证。实际恢复前检查归档校验和与 `pg_restore --list` 的对象清单，只允许应用所属 schema 和约定对象；目标恢复使用单事务的受限操作，不能通过删除整个数据库扩大权限。数据库恢复与文件替换不是跨资源原子事务，文件提前校验并准备临时目录；切换失败时使用恢复前快照回退，未恢复一致前保持维护状态。 +- 生产应用使用专属数据库和固定应用 schema（默认 `public`）。数据库查询、备份 worker 和恢复子进程均传播同一目标及 schema；`pg_dump` 显式限定应用 schema，`search_path` 不代替备份范围限制。旧归档 schema 与目标不匹配时拒绝直接恢复，通过明确转换入口处理，不用字符串替换改写 dump。 +- 恢复成功必须等待 store、分组、API 调度、路由、监控、账号运行状态和 VM 镜像重新绑定/加载，随后才退出维护模式。恢复失败不能沿用无条件 `finally` 清除维护标志的行为。 +- 保留 `_registerOrphanArchives` 的归档重新登记行为。恢复后仍可列出恢复前备份和被恢复的归档，并能从面板发起撤销恢复；数据库记录恢复不等于归档文件自动恢复。 +- 旧 `.tar.gz` 的 `db/kin.db` 及旧 `.db.gz` 提供显式转换入口，复用离线导入和校验逻辑。旧记录中的文件路径不会因为迁移到 PG 自动恢复文件；保留原备份目录并测试记录与文件关系。 +- 数据库 API 和页面改用健康探针、数据库大小、连接池 total/idle/waiting、迁移状态。字段不暴露数据库地址、用户名、密码或连接字符串。 +- Compose 提供持久化 PostgreSQL 服务及健康检查。控制面保留 host 网络时，PG 仅绑定宿主 `127.0.0.1` 的可配置端口，连接 URL 与此一致；支持外部 PostgreSQL，不能硬编码容器服务名为 host 网络下的地址。 +- 更新 `.env.example`、安装脚本、容器入口、systemd 示例、部署说明和 ARM64 测试，处理旧安装升级后缺少数据库连接的情况。`scripts/panel-upgrade.sh` 必须在替换旧容器之前预检 PG 连接、迁移完成标记及所需 Compose 服务；其现有 `--no-deps` 升级不会自动创建 PG 服务，镜像分支也不会刷新 Compose，必须显式覆盖。未迁移的 SQLite 安装在替换容器之前拒绝升级。启动也应拒绝将存在旧 SQLite 数据却没有导入标记的安装作为新空 PG 安装提供服务。 +- `deploy/init-arm64-env.py`、`test/deploy/smoke_arm64_image.py` 和调用它的 test/release workflow 一并更新。现有 ARM64 smoke 使用 `--network none`，迁移后改为专用测试网络和临时 PG 17，并清理数据库、网络和容器；不连接现有服务或扩大 Docker socket 权限。 + +## 4. 实施任务与提交顺序 + +| 任务 | 依赖 | 文件范围 | 交付与验收 | +| --- | --- | --- | --- | +| T0 结构和行为基线 | 无 | 本计划、`test/fixtures/`、必要的只读测试辅助工具 | 29 表及动态表清单、索引/约束/type 映射、当前/历史 SQLite fixture、计费与时区对照结果；fixtures 只含合成数据 | +| T1 PostgreSQL 基础 | T0 | `package.json`、lock、`src/lib/db/database.mjs`、PG 新迁移目录、数据库测试辅助工具 | 异步连接池、初始化/关闭、校验和、迁移锁、单连接事务；先在隔离 PG 测试中覆盖错误路径 | +| T2 基础 repository | T1 | `src/lib/db/repos/` 中非日志统计模块及对应测试 | 用户/分组/密钥/账号/代理/端点/集群/VM/账号池/拒绝记录/兑换码异步 SQL；唯一性、外键和并发测试通过 | +| T3 日志、统计和计费 | T1 | `usage-logs-repo.mjs`、`usage-logs-view.mjs`、`statistics-repo.mjs`、`usage-log-preds.mjs`、`admin/request-log.mjs`、`admin/error-class.mjs` 及测试 | 原生 PG SQL、稳定重定价分页、时区分桶、权限筛选、费用/余额及 retention 结果与基线一致 | +| T4 store 和业务调用链 | T2、T3 | `src/server.mjs`、`src/lib/admin/` 的相关 store/路由、`src/lib/pool/`、`src/lib/protocol/`、`src/lib/vm/` 相关调用者 | 启动到关闭全链路 await；内存并发准入、配置、思考签名、流式结束日志和 VM 镜像语义验证 | +| T5 旧数据与备份转换工具 | T0、T1、T2、T3 | `src/lib/db/legacy-import.mjs`、SQLite 专用离线模块、`scripts/` 迁移命令、迁移测试 | 源只读、历史临时升级、空目标检查、事务导入、校验、sequence、重复执行和失败回滚；包含 JSON 旧数据导入 | +| T6 备份与恢复 | T4、T5 | `backup-core.mjs`、`backup-worker.mjs`、`backup-service.mjs`、server 恢复回调和测试 | PG 备份、全写入口维护隔离、数据库和文件一致性、异步重新绑定、旧归档转换及失败回退 | +| T7 运维与界面 | T1、T4、T6 | `database-metrics.mjs`、数据库 API/类型/页面、4 个 audit 脚本、Compose、两种 Dockerfile、安装/入口/升级脚本、`deploy/init-arm64-env.py`、`test/deploy/smoke_arm64_image.py`、`test/unit/panel-upgrade-script.test.mjs`、部署文档 | PG 指标、无敏感信息的配置、amd64/arm64 构建、旧安装升级前检查；数据库页面同窗口宽度截图对照 | +| T8 全量回归与迁移演练 | T2–T7 | `test/harness.mjs`、相关单元/E2E、`.github/workflows/test.yml`、`.github/workflows/release.yml`、本计划验证记录 | CI 使用真实隔离 PG，当前/历史源导入、新请求持久化、重启、备份恢复及失败演练全部记录实际结论 | + +T2/T3 可按文件范围并行。T5 在数据库契约稳定后并行。T4 与 T6 都会修改 server，必须由同一集成负责人顺序合入。数据库基础与所有调用者异步化完成前，分支不作为可部署版本。 + +每个任务独立提交。若实施中复现现有缺陷，低成本回归测试先单独提交，再提交修复。每次验证记录实际 commit SHA；后续代码改动后重新执行受影响的检查。 + +## 5. 验证方案与完成条件 + +### 5.1 隔离环境 + +选 PostgreSQL 17 作为首次实现和 CI 的固定主版本，匹配相同主版本客户端。使用专用本地测试数据库和合成数据，不读取现有部署的凭据或业务数据。测试配置独立于生产配置，名字与连接目标都必须通过 `_test` / 专用测试库保护检查。 + +所有测试只连接预建的专用 `_test` 数据库。每个测试和派生网关使用独立 schema；测试辅助工具向所有连接和子进程传播固定 search_path，并在结束后关闭连接再删除本测试的 schema。备份恢复显式限定应用 schema，应用 schema 与归档保持一致，`pg_restore` 不承担跨 schema 重命名。通过相邻 schema 的 sentinel 恢复测试验证范围。测试辅助工具不创建或删除数据库，不需要建库权限。集成测试缺少 PG 时明确失败，不静默改用 SQLite。 + +### 5.2 必须通过的场景 + +| 范围 | 场景 | +| --- | --- | +| 初始化 | 空库、重复启动、两个进程并发迁移、校验和不匹配、失败 SQL 回滚、连接不可用、关闭释放连接 | +| 数据导入 | 当前 41 版与有覆盖的历史源、历史 001 proxy schema、旧 JSON 导入、3 张 legacy 表和动态表、NULL/密文/大整数、未知结构拒绝、失败回滚、非空目标拒绝、sequence 后续插入 | +| 用户和权限 | 登录、大小写唯一性/非 ASCII、停用与软删除、租户隔离、API 密钥权限和额度、VM 池成员隔离 | +| 并发 | 同时兑换同一码仅一次成功；余额和用量更新不丢失；同一 key 的并发准入不超限；事务失败无部分更新；VM 写入与删除不乱序 | +| 正常请求 | 通过现有 mock 上游的正常 HTTP 推理入口产生请求、日志和账单；包含流式/非流式、失败、重试、思考签名归属；重启后再查询持久化结果 | +| 统计 | UTC、Asia/Shanghai、含夏令时的 IANA 时区，零值/NULL，租户筛选，leaderboard，分页和全量重定价无遗漏重复，费用与 token 合计 | +| VM | 凭据文件变更镜像、外部文件变更、启动恢复、删除与在途写入、镜像失败、备份/恢复前 flush | +| 备份恢复 | 手动/定时备份、校验与下载、保留策略、恢复前备份、旧归档转换、损坏文件拒绝、恢复中所有写入口受控、恢复失败维护状态、恢复后业务请求 | +| 监控和部署 | PG 指标类型、连接信息脱敏、测试隔离清理、amd64/arm64 镜像和启动、Compose 内置 PG 与外部 PG 配置 | + +重点保留并迁移的现有测试包括： + +```text +test/unit/database.test.mjs +test/unit/db-migration-sub2api.test.mjs +test/unit/legacy-import.test.mjs +test/unit/database-metrics.test.mjs +test/unit/backup-service.test.mjs +test/unit/statistics-repo.test.mjs +test/unit/usage-logs-repo.test.mjs +test/unit/tenant-billing.test.mjs +test/e2e/persistence.e2e.test.mjs +test/e2e/backup.e2e.test.mjs +test/e2e/panel-users.e2e.test.mjs +``` + +### 5.3 验收门槛 + +1. `npm run test:unit` 与 `npm run test:e2e` 在隔离 PostgreSQL 上通过；记录通过/失败/跳过的实际数目和原因。只在离线源 fixture 中保留 SQLite 测试。 +2. `npm run test:go`、api-kernel 现有 Go 测试、前端测试/构建和相关格式、语法检查通过。运行所需的现有预编译 payload 需按仓库脚本准备；缺失时记录未验证，不把依赖缺失误报为数据库通过。 +3. 数据库页面与现有页面各提供一张相同窗口宽度的截图；保留现有组件和布局，只调整引擎相关信息。 +4. 当前/历史 SQLite 迁移演练、迁移回滚、PG 备份恢复演练以及正常 HTTP 请求持久化均通过，不能以连接成功代替业务验证。 +5. 检索 `node:sqlite`、`KIN_DB_PATH`、`VACUUM INTO`、`sqlite_master`、`PRAGMA`、`rowid`、SQLite 方言及旧同步调用。每个剩余命中都解释为离线兼容、历史迁移或历史文档；运行路径不得依赖它们。 +6. 报告标明被验证 SHA。若获准部署，用新行为和运行镜像 digest 验证实际版本；版本号或进程上报 SHA 不作为唯一依据。 + +## 6. 调研验证记录 + +2026-10-21，对源码基线 `eee8f6ecb6ca04455e6edef0c8e498fb3be21f21` 执行： + +```sh +node --test test/unit/database.test.mjs test/unit/db-migration-sub2api.test.mjs test/unit/database-metrics.test.mjs +``` + +结果：3 个测试文件通过，0 个失败，0 个跳过。此结果只证明现有 SQLite 基线的上述测试文件通过，不构成 PostgreSQL 验证。 + +使用临时目录执行全部现有迁移并只读查询 schema：41 个迁移，29 张非 SQLite 内部表。临时目录已删除；未读取现有业务数据库。 + +本机 Node.js 为 `v24.19.0`。调研阶段 `psql`、`postgres`、`initdb` 不在 PATH；Docker 首次查询返回 `permission denied while trying to connect to the docker API at unix:///var/run/docker.sock`。实施阶段已通过工具权限审核创建本任务专用 PostgreSQL 17 容器，并取得匹配的客户端。该容器只含合成数据，不使用现有部署的连接或业务数据。 + +参考官方资料： + +- [node-postgres：事务必须使用同一 client](https://node-postgres.com/features/transactions) +- [node-postgres：连接池](https://node-postgres.com/apis/pool) +- [PostgreSQL 17：pg_dump](https://www.postgresql.org/docs/17/app-pgdump.html) +- [PostgreSQL 17：pg_restore](https://www.postgresql.org/docs/17/app-pgrestore.html) +- [PostgreSQL：Debian PGDG 客户端安装](https://www.postgresql.org/download/linux/debian/) + +计划已由两个只读审查任务分别核对数据层和运维/测试范围。已补充 schema 备份隔离、升级前检查、导出快照同步、客户端主版本、LIKE 语义、大整数源读取和自增高水位要求。该审查仅验证计划覆盖与源码事实，不代表迁移实现已经通过验证。 + +## 7. 实施记录 + +- `fa0decc2`：PostgreSQL 驱动、独立基线和离线 SQLite 模块。 +- `3225b073`：基础 repository 与异步 store。 +- `893cb3ce`：日志、统计和计费查询。 +- `720a0474`：只读 SQLite 快照导入、事务校验和旧备份转换。 +- `1a5a7520`：导出快照备份、受控恢复和失败后维护状态。 +- `c2e56a2d`：PostgreSQL 部署、升级预检、指标与页面。 +- `38f6db6e`：启动、请求、调度、VM 镜像、恢复与关闭的异步调用链。 +- `3fbdc4bc`：独立 PG 测试 schema、CI 和重启持久化验证。 +- `6cb132b6`、`71ea797b`：整数精度及合法小数权重、端点优先级修正。 +- `4ad444d7`：关闭时等待已进入的备份调度检查。 +- `6aa877de`：区分恢复与普通维护提示，隔离慢代理测试的自动备份时序干扰。 + +具体实施时采用 `AsyncLocalStorage` 保持事务连接，savepoint 回滚同步丢弃未提交的缓存更新。保留 SQLite 字符串排序和 ASCII 大小写语义。SQLite 中声明为 INTEGER 的 `accounts.load_factor` 实际接受小数，PG 对应列使用 double precision，已用 1.25 权重导入回归验证。累计整数超出安全范围时以精确十进制字符串返回；余额、费用、平均值和比率保留原 double precision 语义。 + +VM 活凭据继续保留在既有槽位和 CLI-home 存储中。数据库导入和顶层 JSON 归档不能重建这些凭据。迁移操作必须原样保留对应目录及原业务加密密钥。 + +页面对照使用合成指标、相同组件依赖与样式、1440×900 窗口。用户已授权独立 Playwright 截图： + +| 原 SQLite 页面 | PostgreSQL 页面 | +| --- | --- | +| ![SQLite 原页面](evidence/postgresql/database-sqlite-1440.png) | ![PostgreSQL 页面](evidence/postgresql/database-postgresql-1440.png) | + +验证对象、实际结果和未验证项记录在同目录实施报告中。未对现有部署执行迁移、重启或发布，未推送分支。 diff --git a/docs/plans/evidence/postgresql/baseline-comparison.json b/docs/plans/evidence/postgresql/baseline-comparison.json new file mode 100644 index 000000000..60a841133 --- /dev/null +++ b/docs/plans/evidence/postgresql/baseline-comparison.json @@ -0,0 +1,656 @@ +{ + "verified_commit": "3054bfa0", + "production_code_commit": "6aa877de", + "verification_started_commit": "4ad444d7", + "baseline_commit": "eee8f6e", + "normalization": "Exact error/assertion body comparison after removing blank lines and stack frames; timing is outside the compared body.", + "suites": { + "unit": { + "verified_commit": "4ad444d7", + "log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/unit-final.log", + "log_complete": true, + "assertion_failed_file_count": 2, + "runner_summary": [ + "2", + "281" + ], + "infrastructure_failures": [], + "failed_file_count": 2, + "failed_test_count": 7, + "assertion_failed_test_count": 7, + "runner_failed_record_count": 7, + "same_assertion_count": 7, + "all_assertion_failures_match_baseline": true, + "all_failures_match_baseline": true, + "files": [ + { + "file": "test/unit/cli-hop-body.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-cli-hop-body.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "prepareCliHopBody drops metadata and CLI-owned system but keeps official agent leftover", + "test_line": 16, + "current_log_line": 767, + "baseline_log_line": 38, + "same_assertion": true, + "assertion_sha256": "4e6c294bbe40bd85115a154e355bc81e5b21e3e781d2e0dcd1feafcd85a57181", + "baseline_assertion_sha256": "4e6c294bbe40bd85115a154e355bc81e5b21e3e781d2e0dcd1feafcd85a57181" + }, + { + "name": "every cli-hop system becomes the identity plus the caller body", + "test_line": 41, + "current_log_line": 786, + "baseline_log_line": 57, + "same_assertion": true, + "assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "baseline_assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b" + }, + { + "name": "cli-hop normalizes mixed caller markers to the official template", + "test_line": 63, + "current_log_line": 811, + "baseline_log_line": 82, + "same_assertion": true, + "assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "baseline_assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b" + }, + { + "name": "cli-hop drops caller system/tools markers and restamps last plus penultimate user", + "test_line": 357, + "current_log_line": 836, + "baseline_log_line": 107, + "same_assertion": true, + "assertion_sha256": "e37b9c82cfabec8b2d18adaf338ef1368f30e11e38cd9e46c5f5d2d212ec2d11", + "baseline_assertion_sha256": "e37b9c82cfabec8b2d18adaf338ef1368f30e11e38cd9e46c5f5d2d212ec2d11" + }, + { + "name": "cli-hop short history only marks the last message", + "test_line": 381, + "current_log_line": 861, + "baseline_log_line": 132, + "same_assertion": true, + "assertion_sha256": "f68c71379df4d2ba7748acb814268e5f1dcc6590c86098766cf66043bb9fdf9d", + "baseline_assertion_sha256": "f68c71379df4d2ba7748acb814268e5f1dcc6590c86098766cf66043bb9fdf9d" + }, + { + "name": "cli-hop uses the inbound session TTL on both message anchors", + "test_line": 421, + "current_log_line": 891, + "baseline_log_line": 162, + "same_assertion": true, + "assertion_sha256": "5044751390e94cf06d620d10ac9c06e61d7a6a9d27b67bf37eb9711fa14b24ca", + "baseline_assertion_sha256": "5044751390e94cf06d620d10ac9c06e61d7a6a9d27b67bf37eb9711fa14b24ca" + } + ] + }, + { + "file": "test/unit/convert-cache-shape.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-convert-cache-shape.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "openai.chat multi-turn cli-hop marks last and penultimate user", + "test_line": 141, + "current_log_line": 1416, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "baseline_assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b" + } + ] + } + ] + }, + "e2e": { + "verified_commit": "3054bfa0", + "log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/e2e-complete.log", + "log_complete": true, + "assertion_failed_file_count": 11, + "runner_summary": [ + "11", + "26" + ], + "infrastructure_failures": [], + "failed_file_count": 11, + "failed_test_count": 17, + "assertion_failed_test_count": 17, + "runner_failed_record_count": 17, + "same_assertion_count": 17, + "all_assertion_failures_match_baseline": true, + "all_failures_match_baseline": true, + "files": [ + { + "file": "test/e2e/account-pool-failover.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-account-pool-failover.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "account1 quota exhaustion rotates to account2 and commits sticky final account", + "test_line": 10, + "current_log_line": 15, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "b94b33d5b07ef7a98da1199dde49d3bb8be79daed51ca7d5c01ce3002d3e4811", + "baseline_assertion_sha256": "b94b33d5b07ef7a98da1199dde49d3bb8be79daed51ca7d5c01ce3002d3e4811" + } + ] + }, + { + "file": "test/e2e/auto-mode.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-auto-mode.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "classifier terminal/error/cancel/concurrency boundaries through actual ELF", + "test_line": 497, + "current_log_line": 54, + "baseline_log_line": 18, + "same_assertion": true, + "assertion_sha256": "e736744c0afda8b9160312af46c31dfde85514afaf09a5d1a97575b09101da15", + "baseline_assertion_sha256": "e736744c0afda8b9160312af46c31dfde85514afaf09a5d1a97575b09101da15" + } + ] + }, + { + "file": "test/e2e/completions.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-completions.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "POST /v1/completions old OpenAI prompt format", + "test_line": 8, + "current_log_line": 116, + "baseline_log_line": 14, + "same_assertion": true, + "assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "baseline_assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4" + } + ] + }, + { + "file": "test/e2e/credentials.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-credentials.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "reset destroys home and recreates the same slot without oauth", + "test_line": 70, + "current_log_line": 167, + "baseline_log_line": 16, + "same_assertion": true, + "assertion_sha256": "5e6c5a7d2769e0e83c04a4a727c75808b0e1592375b77a815a02b9677de5c477", + "baseline_assertion_sha256": "5e6c5a7d2769e0e83c04a4a727c75808b0e1592375b77a815a02b9677de5c477" + } + ] + }, + { + "file": "test/e2e/crs-default.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-crs-default.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "default POST /v1/messages uses Go worker pool, not CLI", + "test_line": 8, + "current_log_line": 198, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "baseline_assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4" + } + ] + }, + { + "file": "test/e2e/errors.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-errors.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "unknown model is rejected without hop", + "test_line": 32, + "current_log_line": 231, + "baseline_log_line": 17, + "same_assertion": true, + "assertion_sha256": "6a63caf64d29521ed813c597ef288b59270dd6cd3789d4c2a11fe430f2ef95f3", + "baseline_assertion_sha256": "6a63caf64d29521ed813c597ef288b59270dd6cd3789d4c2a11fe430f2ef95f3" + } + ] + }, + { + "file": "test/e2e/health.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-health.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "harness /health 200 and honest capabilities", + "test_line": 5, + "current_log_line": 262, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "858fac7a2bd27316ba43e74a56d0ae6e0bcf45a553d8ecc3da006bfc0a768afb", + "baseline_assertion_sha256": "858fac7a2bd27316ba43e74a56d0ae6e0bcf45a553d8ecc3da006bfc0a768afb" + } + ] + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-hermes.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "Hermes unofficial: Go relay keeps persona/tools and never launches CLI", + "test_line": 15, + "current_log_line": 292, + "baseline_log_line": 14, + "same_assertion": true, + "assertion_sha256": "fc9f564f4db729f7ec444eca13379843ea4c861646e4d21c6c7fdac8b629e390", + "baseline_assertion_sha256": "fc9f564f4db729f7ec444eca13379843ea4c861646e4d21c6c7fdac8b629e390" + }, + { + "name": "Hermes system blob without UA still classifies as hermes", + "test_line": 53, + "current_log_line": 355, + "baseline_log_line": 77, + "same_assertion": true, + "assertion_sha256": "1c1ddb4f06cf94c9408f62e32d5a13a572bceccc1b0acdb5eb9f04acb2a8538b", + "baseline_assertion_sha256": "1c1ddb4f06cf94c9408f62e32d5a13a572bceccc1b0acdb5eb9f04acb2a8538b" + } + ] + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-hop-meta.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "CRS default keeps thinking + max_tokens + temperature in official body", + "test_line": 8, + "current_log_line": 431, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "baseline_assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4" + }, + { + "name": "CRS keeps a long caller system in the structured official prompt", + "test_line": 31, + "current_log_line": 449, + "baseline_log_line": 33, + "same_assertion": true, + "assertion_sha256": "d435dea7913cfa24178f66420d521003431251d6b764517887968b26f123a027", + "baseline_assertion_sha256": "d435dea7913cfa24178f66420d521003431251d6b764517887968b26f123a027" + }, + { + "name": "legacy CLI selector cannot drop HTTP request parameters", + "test_line": 61, + "current_log_line": 467, + "baseline_log_line": 51, + "same_assertion": true, + "assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "baseline_assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4" + } + ] + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-messages.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "HTTP worker preserves thinking, sampling and max_tokens", + "test_line": 114, + "current_log_line": 524, + "baseline_log_line": 20, + "same_assertion": true, + "assertion_sha256": "0af5a9fd20a0a5f8cdfa29b648678f9284a376f484b82d04a1a0230bdf6aa41b", + "baseline_assertion_sha256": "0af5a9fd20a0a5f8cdfa29b648678f9284a376f484b82d04a1a0230bdf6aa41b" + }, + { + "name": "HTTP path preserves long system without CLI argv truncation", + "test_line": 189, + "current_log_line": 548, + "baseline_log_line": 44, + "same_assertion": true, + "assertion_sha256": "5af4ac27ac1e335b2217102a235b4901fbcf18d33188d54a12497261e6e87049", + "baseline_assertion_sha256": "5af4ac27ac1e335b2217102a235b4901fbcf18d33188d54a12497261e6e87049" + } + ] + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-openai-responses.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "POST /v1/responses non-stream shape: object + output_text + usage", + "test_line": 7, + "current_log_line": 593, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "baseline_assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7" + }, + { + "name": "POST /v1/responses stream includes [DONE]", + "test_line": 32, + "current_log_line": 611, + "baseline_log_line": 33, + "same_assertion": true, + "assertion_sha256": "017b08bb23fcae3cb2a5bb334bf68a032eedf15572fce0c3f585154b3b940833", + "baseline_assertion_sha256": "017b08bb23fcae3cb2a5bb334bf68a032eedf15572fce0c3f585154b3b940833" + }, + { + "name": "POST /v1/responses tools → function_call in output", + "test_line": 52, + "current_log_line": 629, + "baseline_log_line": 51, + "same_assertion": true, + "assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "baseline_assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7" + } + ] + } + ] + } + }, + "all_assertion_failures_match_baseline": true, + "all_failures_match_baseline": true, + "previous_attempts": [ + { + "verified_commit": "4ad444d7", + "log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/e2e-final.log", + "failed_file_count": 12, + "failed_test_count": 19, + "baseline_matched_file_count": 11, + "baseline_matched_test_count": 17, + "additional_failures": [ + { + "file": "test/e2e/panel-api.e2e.test.mjs", + "name": "sessionKey import rejected when bound proxy is dead", + "actual_status": 503, + "expected_status": 200, + "classification": "scheduled_backup_maintenance_interference" + }, + { + "file": "test/e2e/panel-api.e2e.test.mjs", + "name": "local egress slot generates auth url and exchanges code", + "actual_status": 503, + "expected_status": 200, + "classification": "scheduled_backup_maintenance_interference" + } + ], + "diagnostic_logs": [ + "/tmp/vm2api-pg-implementation/panel-maintenance-diagnostic.log", + "/tmp/vm2api-pg-implementation/panel-local-maintenance-diagnostic.log" + ] + }, + { + "verified_commit": "6aa877de", + "log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/e2e-final-isolated.log", + "log_complete": true, + "assertion_failed_file_count": 11, + "runner_summary": [ + "12", + "26" + ], + "infrastructure_failures": [ + { + "file": "test/e2e/backup.e2e.test.mjs", + "name": "test/e2e/backup.e2e.test.mjs", + "current_log_line": 96, + "classification": "file_level_timeout_after_business_cases_passed", + "cause": "under_investigation", + "error": " 'Promise resolution is still pending but the event loop has already resolved'" + } + ], + "failed_file_count": 11, + "failed_test_count": 17, + "assertion_failed_test_count": 17, + "runner_failed_record_count": 18, + "same_assertion_count": 17, + "all_assertion_failures_match_baseline": true, + "all_failures_match_baseline": false, + "files": [ + { + "file": "test/e2e/account-pool-failover.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-account-pool-failover.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "account1 quota exhaustion rotates to account2 and commits sticky final account", + "test_line": 10, + "current_log_line": 15, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "b94b33d5b07ef7a98da1199dde49d3bb8be79daed51ca7d5c01ce3002d3e4811", + "baseline_assertion_sha256": "b94b33d5b07ef7a98da1199dde49d3bb8be79daed51ca7d5c01ce3002d3e4811" + } + ] + }, + { + "file": "test/e2e/auto-mode.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-auto-mode.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "classifier terminal/error/cancel/concurrency boundaries through actual ELF", + "test_line": 497, + "current_log_line": 54, + "baseline_log_line": 18, + "same_assertion": true, + "assertion_sha256": "e736744c0afda8b9160312af46c31dfde85514afaf09a5d1a97575b09101da15", + "baseline_assertion_sha256": "e736744c0afda8b9160312af46c31dfde85514afaf09a5d1a97575b09101da15" + } + ] + }, + { + "file": "test/e2e/completions.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-completions.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "POST /v1/completions old OpenAI prompt format", + "test_line": 8, + "current_log_line": 128, + "baseline_log_line": 14, + "same_assertion": true, + "assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "baseline_assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4" + } + ] + }, + { + "file": "test/e2e/credentials.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-credentials.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "reset destroys home and recreates the same slot without oauth", + "test_line": 70, + "current_log_line": 179, + "baseline_log_line": 16, + "same_assertion": true, + "assertion_sha256": "5e6c5a7d2769e0e83c04a4a727c75808b0e1592375b77a815a02b9677de5c477", + "baseline_assertion_sha256": "5e6c5a7d2769e0e83c04a4a727c75808b0e1592375b77a815a02b9677de5c477" + } + ] + }, + { + "file": "test/e2e/crs-default.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-crs-default.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "default POST /v1/messages uses Go worker pool, not CLI", + "test_line": 8, + "current_log_line": 210, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "baseline_assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4" + } + ] + }, + { + "file": "test/e2e/errors.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-errors.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "unknown model is rejected without hop", + "test_line": 32, + "current_log_line": 243, + "baseline_log_line": 17, + "same_assertion": true, + "assertion_sha256": "6a63caf64d29521ed813c597ef288b59270dd6cd3789d4c2a11fe430f2ef95f3", + "baseline_assertion_sha256": "6a63caf64d29521ed813c597ef288b59270dd6cd3789d4c2a11fe430f2ef95f3" + } + ] + }, + { + "file": "test/e2e/health.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-health.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "harness /health 200 and honest capabilities", + "test_line": 5, + "current_log_line": 274, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "858fac7a2bd27316ba43e74a56d0ae6e0bcf45a553d8ecc3da006bfc0a768afb", + "baseline_assertion_sha256": "858fac7a2bd27316ba43e74a56d0ae6e0bcf45a553d8ecc3da006bfc0a768afb" + } + ] + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-hermes.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "Hermes unofficial: Go relay keeps persona/tools and never launches CLI", + "test_line": 15, + "current_log_line": 304, + "baseline_log_line": 14, + "same_assertion": true, + "assertion_sha256": "fc9f564f4db729f7ec444eca13379843ea4c861646e4d21c6c7fdac8b629e390", + "baseline_assertion_sha256": "fc9f564f4db729f7ec444eca13379843ea4c861646e4d21c6c7fdac8b629e390" + }, + { + "name": "Hermes system blob without UA still classifies as hermes", + "test_line": 53, + "current_log_line": 367, + "baseline_log_line": 77, + "same_assertion": true, + "assertion_sha256": "1c1ddb4f06cf94c9408f62e32d5a13a572bceccc1b0acdb5eb9f04acb2a8538b", + "baseline_assertion_sha256": "1c1ddb4f06cf94c9408f62e32d5a13a572bceccc1b0acdb5eb9f04acb2a8538b" + } + ] + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-hop-meta.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "CRS default keeps thinking + max_tokens + temperature in official body", + "test_line": 8, + "current_log_line": 443, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "baseline_assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4" + }, + { + "name": "CRS keeps a long caller system in the structured official prompt", + "test_line": 31, + "current_log_line": 461, + "baseline_log_line": 33, + "same_assertion": true, + "assertion_sha256": "d435dea7913cfa24178f66420d521003431251d6b764517887968b26f123a027", + "baseline_assertion_sha256": "d435dea7913cfa24178f66420d521003431251d6b764517887968b26f123a027" + }, + { + "name": "legacy CLI selector cannot drop HTTP request parameters", + "test_line": 61, + "current_log_line": 479, + "baseline_log_line": 51, + "same_assertion": true, + "assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "baseline_assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4" + } + ] + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-messages.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "HTTP worker preserves thinking, sampling and max_tokens", + "test_line": 114, + "current_log_line": 536, + "baseline_log_line": 20, + "same_assertion": true, + "assertion_sha256": "0af5a9fd20a0a5f8cdfa29b648678f9284a376f484b82d04a1a0230bdf6aa41b", + "baseline_assertion_sha256": "0af5a9fd20a0a5f8cdfa29b648678f9284a376f484b82d04a1a0230bdf6aa41b" + }, + { + "name": "HTTP path preserves long system without CLI argv truncation", + "test_line": 189, + "current_log_line": 560, + "baseline_log_line": 44, + "same_assertion": true, + "assertion_sha256": "5af4ac27ac1e335b2217102a235b4901fbcf18d33188d54a12497261e6e87049", + "baseline_assertion_sha256": "5af4ac27ac1e335b2217102a235b4901fbcf18d33188d54a12497261e6e87049" + } + ] + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-openai-responses.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "POST /v1/responses non-stream shape: object + output_text + usage", + "test_line": 7, + "current_log_line": 605, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "baseline_assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7" + }, + { + "name": "POST /v1/responses stream includes [DONE]", + "test_line": 32, + "current_log_line": 623, + "baseline_log_line": 33, + "same_assertion": true, + "assertion_sha256": "017b08bb23fcae3cb2a5bb334bf68a032eedf15572fce0c3f585154b3b940833", + "baseline_assertion_sha256": "017b08bb23fcae3cb2a5bb334bf68a032eedf15572fce0c3f585154b3b940833" + }, + { + "name": "POST /v1/responses tools → function_call in output", + "test_line": 52, + "current_log_line": 641, + "baseline_log_line": 51, + "same_assertion": true, + "assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "baseline_assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7" + } + ] + } + ] + } + ], + "test_isolation_fix": { + "commit": "3054bfa0", + "log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/schema-isolation-final.log", + "log_sha256": "95aa62bbd19cb45c9692e7085220cfafb460b7249bfaf7d969f02282b356e3d3", + "tests": 40, + "passed": 40, + "failed": 0, + "scope": "Dedicated *_test database with per-test schemas; adjacent schema sentinel retained and writable after restore." + } +} diff --git a/docs/plans/evidence/postgresql/database-postgresql-1440.png b/docs/plans/evidence/postgresql/database-postgresql-1440.png new file mode 100644 index 000000000..771f10974 Binary files /dev/null and b/docs/plans/evidence/postgresql/database-postgresql-1440.png differ diff --git a/docs/plans/evidence/postgresql/database-sqlite-1440.png b/docs/plans/evidence/postgresql/database-sqlite-1440.png new file mode 100644 index 000000000..ea8011f06 Binary files /dev/null and b/docs/plans/evidence/postgresql/database-sqlite-1440.png differ diff --git a/docs/plans/evidence/postgresql/verification.json b/docs/plans/evidence/postgresql/verification.json new file mode 100644 index 000000000..f4b06b0f5 --- /dev/null +++ b/docs/plans/evidence/postgresql/verification.json @@ -0,0 +1,100 @@ +{ + "baseline_commit": "eee8f6ecb6ca04455e6edef0c8e498fb3be21f21", + "production_code_commit": "6aa877de792bbcaa0fd02134c821bf938c295c42", + "final_test_commit": "3054bfa08500c43f81eeb0a84804c4a63e4bd022", + "database": "PostgreSQL 17.11", + "unit": { + "verified_commit": "4ad444d71e32e938c589c647850a0a795a9b84d1", + "tests": 3342, + "pass": 3325, + "fail": 7, + "skipped": 10, + "cancelled": 0, + "failed_files": 2, + "files": 281, + "log": ".tmp/pg-evidence/unit-final.log" + }, + "e2e": { + "verified_commit": "3054bfa08500c43f81eeb0a84804c4a63e4bd022", + "tests": 114, + "pass": 96, + "fail": 17, + "skipped": 1, + "cancelled": 0, + "failed_files": 11, + "files": 26, + "log": ".tmp/pg-evidence/e2e-complete.log" + }, + "node22": { + "verified_commit": "4ad444d71e32e938c589c647850a0a795a9b84d1", + "tests": 93, + "pass": 93, + "fail": 0, + "skipped": 0, + "cancelled": 0, + "log": ".tmp/pg-evidence/node22-final.log" + }, + "schema_isolation": { + "verified_commit": "3054bfa08500c43f81eeb0a84804c4a63e4bd022", + "tests": 40, + "pass": 40, + "fail": 0, + "skipped": 0, + "cancelled": 0, + "log": ".tmp/pg-evidence/schema-isolation-final.log" + }, + "web": { + "verified_commit": "4ad444d71e32e938c589c647850a0a795a9b84d1", + "files": 60, + "tests": 473, + "pass": 473, + "fail": 0, + "build": "pass", + "format": "pass" + }, + "go": { + "verified_commit": "4ad444d71e32e938c589c647850a0a795a9b84d1", + "worker_race": "pass", + "api_kernel_race": "pass" + }, + "deployment": { + "verified_commit": "4ad444d71e32e938c589c647850a0a795a9b84d1", + "python_tests": 25, + "pass": 25, + "postgres_role_smoke": "pass" + }, + "image": { + "verified_commit": "6aa877de792bbcaa0fd02134c821bf938c295c42", + "result": "PASS", + "image": "sha256:d93e36d953db42aa80acd562a2af63cf0a969789d154d063610c0e6e060dc33e", + "versions": [ + "v22.23.2", + "pg_dump (PostgreSQL) 17.11 (Debian 17.11-1.pgdg12+2)", + "pg_restore (PostgreSQL) 17.11 (Debian 17.11-1.pgdg12+2)" + ], + "checks": [ + "HTTP health", + "PostgreSQL metrics", + "API key mutations", + "snapshot backup", + "restore and memory rebind" + ] + }, + "screenshots": { + "viewport": { + "width": 1440, + "height": 900 + }, + "baseline": "database-sqlite-1440.png", + "implementation": "database-postgresql-1440.png", + "data": "synthetic" + }, + "limitations": [ + "Existing baseline failures remain; see baseline-comparison.json", + "Native ARM64 build/runtime not performed locally", + "No existing deployment, live credentials, customer data, or external inference upstream was migrated or tested" + ], + "git": { + "pushed": false + } +} diff --git a/package-lock.json b/package-lock.json index feb8b62ab..5192f0966 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "croner": "^10.0.1", "https-proxy-agent": "^9.1.0", "node-fetch": "^2.7.0", + "pg": "^8.23.1", "socks-proxy-agent": "^10.1.0", "ssh2": "^1.17.0", "ws": "^8.22.0" @@ -322,6 +323,134 @@ } } }, + "node_modules/pg": { + "version": "8.23.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.1.tgz", + "integrity": "sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.1", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.16.1", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.1.tgz", + "integrity": "sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.1.tgz", + "integrity": "sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.1.tgz", + "integrity": "sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/proxy-agent-negotiate": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-agent-negotiate/-/proxy-agent-negotiate-1.1.0.tgz", @@ -383,6 +512,15 @@ "node": ">= 20" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/ssh2": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", @@ -448,6 +586,15 @@ "optional": true } } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } } } } diff --git a/package.json b/package.json index e226872f2..dbf054d0d 100644 --- a/package.json +++ b/package.json @@ -19,12 +19,15 @@ "fetch:payload": "node scripts/fetch-payload.mjs", "test:web": "pnpm -C web test", "format": "biome format --write && gofmt -w worker && pnpm -C web format", - "format:check": "biome format && node scripts/check-gofmt.mjs && pnpm -C web format:check" + "format:check": "biome format && node scripts/check-gofmt.mjs && pnpm -C web format:check", + "db:import:sqlite": "node scripts/import-sqlite.mjs", + "db:convert:sqlite-backup": "node scripts/convert-sqlite-backup.mjs" }, "dependencies": { "croner": "^10.0.1", "https-proxy-agent": "^9.1.0", "node-fetch": "^2.7.0", + "pg": "^8.23.1", "socks-proxy-agent": "^10.1.0", "ssh2": "^1.17.0", "ws": "^8.22.0" diff --git a/scripts/audit-billing-contract.mjs b/scripts/audit-billing-contract.mjs index f8c653d9e..660baf78e 100644 --- a/scripts/audit-billing-contract.mjs +++ b/scripts/audit-billing-contract.mjs @@ -1,14 +1,15 @@ #!/usr/bin/env node // Billing contract audit: web consumers vs backend producers, cost basis per aggregation, // and (optional) usage_logs data checks. -// node scripts/audit-billing-contract.mjs [path/to/kin.db] [--json] +// node scripts/audit-billing-contract.mjs [--database] [--json] import { readFileSync, readdirSync, statSync, existsSync } from 'node:fs' import { join, relative } from 'node:path' +import { openAuditDb } from './audit-db.mjs' const ROOT = new URL('..', import.meta.url).pathname const args = process.argv.slice(2) const asJson = args.includes('--json') -const dbPath = args.find((a) => !a.startsWith('--')) +const inspectDatabase = args.includes('--database') function walk(dir, exts, skip = /node_modules|dist|\.test\.|routeTree\.gen/) { const out = [] @@ -66,7 +67,7 @@ const unused = [...src.entries()] // 3) cost basis per aggregation SQL (total_cost = official list price, actual_cost = rate-multiplied) const repo = readFileSync(join(ROOT, 'src/lib/db/repos/usage-logs-repo.mjs'), 'utf8') const basis = [] -const methodRe = /^ {2}(_?[a-zA-Z]+)\([^)]*\)\s*\{/gm +const methodRe = /^ {2}(?:async )?(_?[a-zA-Z]+)\([^)]*\)\s*\{/gm const starts = [...repo.matchAll(methodRe)].map((m) => ({ name: m[1], at: m.index })) starts.forEach((s, i) => { const body = repo.slice(s.at, starts[i + 1]?.at ?? repo.length) @@ -95,44 +96,50 @@ const backfillRepricesUnpriced = !bf || !/pricing_model IS NULL/.test(bf[2]) // 5) optional DB checks let db = null -if (dbPath) { - if (!existsSync(dbPath)) { - console.error(`db not found: ${dbPath}`) - process.exit(2) - } - const { DatabaseSync } = await import('node:sqlite') - const d = new DatabaseSync(dbPath, { readOnly: true }) - const colNames = d - .prepare("SELECT name FROM pragma_table_info('usage_logs')") - .all() - .map((r) => r.name) - const q = (sql) => d.prepare(sql).all() - db = { - has_service_tier_col: colNames.includes('service_tier'), - has_speed_col: colNames.includes('speed'), - unpriced_rows: q("SELECT COUNT(*) n FROM usage_logs WHERE pricing_model = 'unpriced'")[0], - band_distribution: colNames.includes('service_tier') - ? q(`SELECT service_tier, speed, long_context, COUNT(*) n, ROUND(SUM(COALESCE(total_cost,0)),4) cost +if (inspectDatabase) { + const d = await openAuditDb() + try { + const colNames = ( + await d + .prepare( + "SELECT column_name AS name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'usage_logs'", + ) + .all() + ).map((r) => r.name) + const q = (sql) => d.prepare(sql).all() + db = { + has_service_tier_col: colNames.includes('service_tier'), + has_speed_col: colNames.includes('speed'), + unpriced_rows: (await q("SELECT COUNT(*) n FROM usage_logs WHERE pricing_model = 'unpriced'"))[0], + band_distribution: colNames.includes('service_tier') + ? await q(`SELECT service_tier, speed, long_context, COUNT(*) n, ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) cost FROM usage_logs GROUP BY 1, 2, 3 ORDER BY 4 DESC LIMIT 30`) - : null, - totals: q(`SELECT COUNT(*) n, - SUM(total_cost IS NULL) null_total, - SUM(actual_cost IS NULL AND total_cost IS NOT NULL) null_actual, - ROUND(SUM(COALESCE(total_cost,0)),4) sum_total, - ROUND(SUM(COALESCE(actual_cost,total_cost,0)),4) sum_actual, - SUM(COALESCE(rate_multiplier,1) <> 1) rated_rows - FROM usage_logs`)[0], - unpriced_with_tokens: q(`SELECT COALESCE(upstream_model,model,requested_model) m, COUNT(*) n + : null, + totals: ( + await q(`SELECT COUNT(*) n, + COUNT(*) FILTER (WHERE total_cost IS NULL) null_total, + COUNT(*) FILTER (WHERE actual_cost IS NULL AND total_cost IS NOT NULL) null_actual, + ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) sum_total, + ROUND((SUM(COALESCE(actual_cost,total_cost,0)))::numeric,4) sum_actual, + COUNT(*) FILTER (WHERE COALESCE(rate_multiplier,1) <> 1) rated_rows + FROM usage_logs`) + )[0], + unpriced_with_tokens: await q(`SELECT COALESCE(upstream_model,model,requested_model) m, COUNT(*) n FROM usage_logs WHERE (total_cost IS NULL OR total_cost = 0) AND COALESCE(input_tokens,0)+COALESCE(output_tokens,0) > 0 GROUP BY 1 ORDER BY 2 DESC LIMIT 20`), - pricing_model: q(`SELECT pricing_model, COUNT(*) n, ROUND(SUM(total_cost),4) cost + pricing_model: await q(`SELECT pricing_model, COUNT(*) n, ROUND((SUM(total_cost))::numeric,4) cost FROM usage_logs GROUP BY 1 ORDER BY 2 DESC LIMIT 30`), - long_context_candidates: q(`SELECT pricing_model, COUNT(*) n FROM usage_logs + long_context_candidates: await q(`SELECT pricing_model, COUNT(*) n FROM usage_logs WHERE COALESCE(input_tokens,0)+COALESCE(cache_read_tokens,0)+COALESCE(cache_creation_tokens,0) > 200000 GROUP BY 1 ORDER BY 2 DESC`), - total_vs_parts_mismatch: q(`SELECT COUNT(*) n FROM usage_logs WHERE total_cost IS NOT NULL - AND ABS(total_cost - (COALESCE(input_cost,0)+COALESCE(output_cost,0)+COALESCE(cache_read_cost,0)+COALESCE(cache_creation_cost,0))) > 1e-6`)[0], + total_vs_parts_mismatch: ( + await q(`SELECT COUNT(*) n FROM usage_logs WHERE total_cost IS NOT NULL + AND ABS(total_cost - (COALESCE(input_cost,0)+COALESCE(output_cost,0)+COALESCE(cache_read_cost,0)+COALESCE(cache_creation_cost,0))) > 1e-6`) + )[0], + } + } finally { + await d.close() } } @@ -170,5 +177,5 @@ if (asJson) { if (db) { console.log('\n== 5. DB ==') console.log(JSON.stringify(db, null, 2)) - } else console.log('\n(未给 DB 路径，跳过数据检查)') + } else console.log('\n(未传 --database，跳过数据检查)') } diff --git a/scripts/audit-billing-gaps.mjs b/scripts/audit-billing-gaps.mjs index c54f2272a..4d4f7dd44 100644 --- a/scripts/audit-billing-gaps.mjs +++ b/scripts/audit-billing-gaps.mjs @@ -1,11 +1,12 @@ -import { DatabaseSync } from 'node:sqlite' +import { openAuditDb } from './audit-db.mjs' -const db = new DatabaseSync(process.argv[2] || '/opt/kin-gateway/data/kin.db', { readOnly: true }) +const db = await openAuditDb() const rows = (sql) => db.prepare(sql).all() const one = (sql) => db.prepare(sql).get() -const out = { - totals: one(` +try { + const out = { + totals: await one(` SELECT COUNT(*) AS n, SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END) AS ok2xx, SUM(CASE WHEN input_tokens IS NULL THEN 1 ELSE 0 END) AS null_in, @@ -14,80 +15,82 @@ const out = { SUM(CASE WHEN status BETWEEN 200 AND 299 AND input_tokens IS NULL THEN 1 ELSE 0 END) AS ok_null_in, SUM(CASE WHEN status BETWEEN 200 AND 299 AND (total_cost IS NULL OR total_cost=0) AND COALESCE(input_tokens,0)+COALESCE(output_tokens,0)>0 THEN 1 ELSE 0 END) AS ok_tokens_zero_cost, - ROUND(SUM(COALESCE(total_cost,0)),4) AS cost + ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) AS cost FROM usage_logs `), - protocol: rows(` + protocol: await rows(` SELECT protocol, COUNT(*) AS n, SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END) AS ok2xx, SUM(CASE WHEN status BETWEEN 200 AND 299 AND input_tokens IS NULL THEN 1 ELSE 0 END) AS ok_null_in, SUM(CASE WHEN total_cost IS NULL THEN 1 ELSE 0 END) AS null_cost, - ROUND(SUM(COALESCE(total_cost,0)),4) AS cost, + ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) AS cost, SUM(COALESCE(input_tokens,0)) AS in_tok, SUM(COALESCE(output_tokens,0)) AS out_tok, SUM(COALESCE(cache_read_tokens,0)) AS cache_read, SUM(COALESCE(cache_creation_tokens,0)) AS cache_create FROM usage_logs GROUP BY protocol ORDER BY n DESC `), - ua: rows(` + ua: await rows(` SELECT CASE - WHEN user_agent LIKE '%kin-console-test%' THEN 'vm-test' - WHEN user_agent LIKE '%claude-cli%' OR user_agent LIKE '%claude-code%' - OR user_agent LIKE '%Claude-Code%' OR user_agent LIKE '%Claude Code%' THEN 'official-cc' - WHEN user_agent LIKE '%Rikka%' THEN 'rikka' - WHEN user_agent LIKE '%Hermes%' THEN 'hermes' - WHEN user_agent LIKE '%Cherry%' THEN 'cherry' - WHEN user_agent LIKE '%Go-http-client%' THEN 'go-http' - WHEN user_agent LIKE '%OpenAI%' OR user_agent LIKE '%openai%' THEN 'openai-sdk' + WHEN user_agent ILIKE '%kin-console-test%' THEN 'vm-test' + WHEN user_agent ILIKE '%claude-cli%' OR user_agent ILIKE '%claude-code%' + OR user_agent ILIKE '%Claude-Code%' OR user_agent ILIKE '%Claude Code%' THEN 'official-cc' + WHEN user_agent ILIKE '%Rikka%' THEN 'rikka' + WHEN user_agent ILIKE '%Hermes%' THEN 'hermes' + WHEN user_agent ILIKE '%Cherry%' THEN 'cherry' + WHEN user_agent ILIKE '%Go-http-client%' THEN 'go-http' + WHEN user_agent ILIKE '%OpenAI%' OR user_agent ILIKE '%openai%' THEN 'openai-sdk' WHEN user_agent IS NULL OR user_agent = '' THEN 'empty' ELSE 'other-third' END AS ua_class, COUNT(*) AS n, SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END) AS ok2xx, SUM(CASE WHEN status BETWEEN 200 AND 299 AND input_tokens IS NULL THEN 1 ELSE 0 END) AS ok_null_in, - ROUND(SUM(COALESCE(total_cost,0)),4) AS cost + ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) AS cost FROM usage_logs GROUP BY ua_class ORDER BY n DESC `), - path: rows('SELECT path, COUNT(*) AS n FROM usage_logs GROUP BY path ORDER BY n DESC LIMIT 20'), - other_ua: rows(` + path: await rows('SELECT path, COUNT(*) AS n FROM usage_logs GROUP BY path ORDER BY n DESC LIMIT 20'), + other_ua: await rows(` SELECT substr(user_agent,1,90) AS ua, COUNT(*) AS n, SUM(CASE WHEN status BETWEEN 200 AND 299 AND input_tokens IS NULL THEN 1 ELSE 0 END) AS ok_null_in, - ROUND(SUM(COALESCE(total_cost,0)),4) AS cost + ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) AS cost FROM usage_logs - WHERE user_agent NOT LIKE '%kin-console-test%' - AND user_agent NOT LIKE '%claude-cli%' - AND user_agent NOT LIKE '%claude-code%' - AND user_agent NOT LIKE '%Claude-Code%' - AND user_agent NOT LIKE '%Claude Code%' - AND user_agent NOT LIKE '%Rikka%' - AND user_agent NOT LIKE '%Hermes%' - AND user_agent NOT LIKE '%Cherry%' + WHERE user_agent NOT ILIKE '%kin-console-test%' + AND user_agent NOT ILIKE '%claude-cli%' + AND user_agent NOT ILIKE '%claude-code%' + AND user_agent NOT ILIKE '%Claude-Code%' + AND user_agent NOT ILIKE '%Claude Code%' + AND user_agent NOT ILIKE '%Rikka%' + AND user_agent NOT ILIKE '%Hermes%' + AND user_agent NOT ILIKE '%Cherry%' GROUP BY ua ORDER BY n DESC LIMIT 15 `), - ok_null_in_sample: rows(` + ok_null_in_sample: await rows(` SELECT protocol, status, stream, path, substr(user_agent,1,60) AS ua, model, error_code, stop_reason, final_state, via FROM usage_logs WHERE status BETWEEN 200 AND 299 AND input_tokens IS NULL ORDER BY created_at DESC LIMIT 20 `), - today_by_ua: rows(` + today_by_ua: await rows(` SELECT CASE - WHEN user_agent LIKE '%kin-console-test%' THEN 'vm-test' - WHEN user_agent LIKE '%claude-cli%' OR user_agent LIKE '%claude-code%' - OR user_agent LIKE '%Claude-Code%' OR user_agent LIKE '%Claude Code%' THEN 'official-cc' + WHEN user_agent ILIKE '%kin-console-test%' THEN 'vm-test' + WHEN user_agent ILIKE '%claude-cli%' OR user_agent ILIKE '%claude-code%' + OR user_agent ILIKE '%Claude-Code%' OR user_agent ILIKE '%Claude Code%' THEN 'official-cc' ELSE 'third-or-other' END AS ua_class, COUNT(*) AS n, SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END) AS ok2xx, - ROUND(SUM(COALESCE(total_cost,0)),4) AS cost, + ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) AS cost, SUM(COALESCE(input_tokens,0)) AS in_tok, SUM(COALESCE(output_tokens,0)) AS out_tok FROM usage_logs - WHERE created_at >= datetime('now', '+8 hours', 'start of day', '-8 hours') + WHERE created_at >= to_char(date_trunc('day', now() AT TIME ZONE 'Asia/Shanghai') AT TIME ZONE 'Asia/Shanghai' AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') GROUP BY ua_class `), -} + } -console.log(JSON.stringify(out, null, 2)) -db.close() + console.log(JSON.stringify(out, null, 2)) +} finally { + await db.close() +} diff --git a/scripts/audit-billing-openai.mjs b/scripts/audit-billing-openai.mjs index 1b5ca254e..5c5d154e5 100644 --- a/scripts/audit-billing-openai.mjs +++ b/scripts/audit-billing-openai.mjs @@ -1,51 +1,54 @@ -import { DatabaseSync } from 'node:sqlite' +import { openAuditDb } from './audit-db.mjs' -const db = new DatabaseSync(process.argv[2] || '/opt/kin-gateway/data/kin.db', { readOnly: true }) +const db = await openAuditDb() const rows = (sql) => db.prepare(sql).all() -const out = { - openai_chat: rows(` +try { + const out = { + openai_chat: await rows(` SELECT stream, status, final_state, error_code, input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens, - ROUND(total_cost,6) AS cost, substr(user_agent,1,50) AS ua, model, stop_reason + ROUND((total_cost)::numeric,6) AS cost, substr(user_agent,1,50) AS ua, model, stop_reason FROM usage_logs WHERE protocol = 'openai.chat' ORDER BY created_at DESC LIMIT 40 `), - openai_python: rows(` + openai_python: await rows(` SELECT stream, status, final_state, error_code, - input_tokens, output_tokens, ROUND(total_cost,6) AS cost, model, via - FROM usage_logs WHERE user_agent LIKE 'OpenAI/%' + input_tokens, output_tokens, ROUND((total_cost)::numeric,6) AS cost, model, via + FROM usage_logs WHERE user_agent ILIKE 'OpenAI/%' ORDER BY created_at DESC `), - openai_ok_null: rows(` + openai_ok_null: await rows(` SELECT COUNT(*) AS n, SUM(stream) AS streams, SUM(CASE WHEN final_state = 'incomplete' THEN 1 ELSE 0 END) AS incomplete, SUM(CASE WHEN final_state = 'verified' THEN 1 ELSE 0 END) AS verified, SUM(CASE WHEN error_code IS NOT NULL THEN 1 ELSE 0 END) AS with_err FROM usage_logs - WHERE protocol LIKE 'openai%' AND status BETWEEN 200 AND 299 AND input_tokens IS NULL + WHERE protocol ILIKE 'openai%' AND status BETWEEN 200 AND 299 AND input_tokens IS NULL `), - openai_ok_with_tok: rows(` + openai_ok_with_tok: await rows(` SELECT COUNT(*) AS n, SUM(stream) AS streams, SUM(COALESCE(input_tokens,0)) AS in_tok, SUM(COALESCE(output_tokens,0)) AS out_tok, SUM(COALESCE(cache_read_tokens,0)) AS cache_read, - ROUND(SUM(COALESCE(total_cost,0)),6) AS cost + ROUND((SUM(COALESCE(total_cost,0)))::numeric,6) AS cost FROM usage_logs - WHERE protocol LIKE 'openai%' AND status BETWEEN 200 AND 299 AND input_tokens IS NOT NULL + WHERE protocol ILIKE 'openai%' AND status BETWEEN 200 AND 299 AND input_tokens IS NOT NULL `), - stream_ok_null: rows(` + stream_ok_null: await rows(` SELECT protocol, COUNT(*) AS n, error_code, final_state FROM usage_logs WHERE stream = 1 AND status BETWEEN 200 AND 299 AND input_tokens IS NULL GROUP BY protocol, error_code, final_state ORDER BY n DESC `), - vm_test_anywhere: rows(` + vm_test_anywhere: await rows(` SELECT COUNT(*) AS n FROM usage_logs - WHERE user_agent LIKE '%kin-console-test%' OR path LIKE '%test-chat%' + WHERE user_agent ILIKE '%kin-console-test%' OR path ILIKE '%test-chat%' `), -} + } -console.log(JSON.stringify(out, null, 2)) -db.close() + console.log(JSON.stringify(out, null, 2)) +} finally { + await db.close() +} diff --git a/scripts/audit-db.mjs b/scripts/audit-db.mjs new file mode 100644 index 000000000..b1057d13a --- /dev/null +++ b/scripts/audit-db.mjs @@ -0,0 +1,41 @@ +/** Shared read-only PostgreSQL connection for the four audit commands. */ +import pg from 'pg' +import { quoteIdentifier, resolveDatabaseUrl } from '../src/lib/db/database.mjs' + +export async function openAuditDb() { + const schema = process.env.VM2API_DATABASE_SCHEMA || process.env.KIN_DATABASE_SCHEMA || 'public' + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(schema)) throw new Error('invalid database schema') + const client = new pg.Client({ + connectionString: resolveDatabaseUrl(), + options: `-c default_transaction_read_only=on -c search_path=${quoteIdentifier(schema)},pg_catalog -c timezone=UTC`, + types: { + getTypeParser(oid, format) { + if (format !== 'binary' && (oid === 20 || oid === 1700)) + return (value) => { + const number = Number(value) + return Number.isFinite(number) && Math.abs(number) <= Number.MAX_SAFE_INTEGER ? number : value + } + return pg.types.getTypeParser(oid, format) + }, + }, + }) + try { + await client.connect() + } catch { + throw new Error('PostgreSQL audit connection failed') + } + const query = async (sql, params) => { + try { + return await client.query(sql, params) + } catch (error) { + throw new Error(`PostgreSQL audit query failed (${error.code || 'database_error'})`) + } + } + return { + prepare: (sql) => ({ + all: async (...params) => (await query(sql, params)).rows, + get: async (...params) => (await query(sql, params)).rows[0], + }), + close: () => client.end(), + } +} diff --git a/scripts/audit-fable-week.mjs b/scripts/audit-fable-week.mjs index a9a948c00..2f088009c 100644 --- a/scripts/audit-fable-week.mjs +++ b/scripts/audit-fable-week.mjs @@ -1,38 +1,41 @@ -import { DatabaseSync } from 'node:sqlite' +import { openAuditDb } from './audit-db.mjs' -const db = new DatabaseSync(process.argv[2] || '/opt/kin-gateway/data/kin.db', { readOnly: true }) +const db = await openAuditDb() -const acc = db - .prepare(` +try { + const acc = await db + .prepare(` SELECT vm_id, - json_extract(extra, '$.unified.7d.utilization') AS u7, - json_extract(extra, '$.unified.7d.status') AS s7, - json_extract(extra, '$.unified.7d_oi.utilization') AS uoi, - json_extract(extra, '$.unified.7d_oi.status') AS soi, - json_extract(extra, '$.unified.5h.utilization') AS u5 + extra::jsonb #>> '{unified,7d,utilization}' AS u7, + extra::jsonb #>> '{unified,7d,status}' AS s7, + extra::jsonb #>> '{unified,7d_oi,utilization}' AS uoi, + extra::jsonb #>> '{unified,7d_oi,status}' AS soi, + extra::jsonb #>> '{unified,5h,utilization}' AS u5 FROM accounts `) - .all() + .all() -const logs = db - .prepare(` + const logs = await db + .prepare(` SELECT CASE - WHEN model LIKE '%fable%' OR upstream_model LIKE '%fable%' THEN 'fable' - WHEN model LIKE '%opus%' OR upstream_model LIKE '%opus%' THEN 'opus' - WHEN model LIKE '%sonnet%' OR upstream_model LIKE '%sonnet%' THEN 'sonnet' - WHEN model LIKE '%haiku%' OR upstream_model LIKE '%haiku%' THEN 'haiku' + WHEN model ILIKE '%fable%' OR upstream_model ILIKE '%fable%' THEN 'fable' + WHEN model ILIKE '%opus%' OR upstream_model ILIKE '%opus%' THEN 'opus' + WHEN model ILIKE '%sonnet%' OR upstream_model ILIKE '%sonnet%' THEN 'sonnet' + WHEN model ILIKE '%haiku%' OR upstream_model ILIKE '%haiku%' THEN 'haiku' ELSE 'other' END AS fam, COUNT(*) AS n, - ROUND(SUM(COALESCE(total_cost,0)),4) AS cost, + ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) AS cost, SUM(COALESCE(input_tokens,0)+COALESCE(output_tokens,0)+COALESCE(cache_read_tokens,0)+COALESCE(cache_creation_tokens,0)) AS toks FROM usage_logs - WHERE created_at >= datetime('now', '-7 days') + WHERE created_at >= to_char(now() AT TIME ZONE 'UTC' - interval '7 days', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') GROUP BY fam ORDER BY cost DESC `) - .all() + .all() -console.log(JSON.stringify({ quota: acc, last7d_by_family: logs }, null, 2)) -db.close() + console.log(JSON.stringify({ quota: acc, last7d_by_family: logs }, null, 2)) +} finally { + await db.close() +} diff --git a/scripts/check-postgres-upgrade.mjs b/scripts/check-postgres-upgrade.mjs new file mode 100644 index 000000000..69f4e22f9 --- /dev/null +++ b/scripts/check-postgres-upgrade.mjs @@ -0,0 +1,49 @@ +/** Read-only cutover check, executed from the new image before replacing the live container. */ +import fs from 'node:fs' +import path from 'node:path' +import crypto from 'node:crypto' +import { fileURLToPath } from 'node:url' +import { openAuditDb } from './audit-db.mjs' + +const dataDir = process.env.KIN_DATA_DIR || path.resolve('data') +let db +try { + if (fs.existsSync(path.join(dataDir, 'restore-maintenance.json'))) + throw new Error('offline restore recovery required') + db = await openAuditDb() + const server = Number((await db.prepare('SHOW server_version_num').get()).server_version_num) + if (server < 170000 || server >= 180000) throw new Error('PostgreSQL 17 required') + const directory = fileURLToPath(new URL('../src/lib/db/postgres-migrations/', import.meta.url)) + const expected = fs + .readdirSync(directory) + .filter((name) => /^\d+.*\.sql$/.test(name)) + .sort() + .map((name) => ({ + version: name.split('_')[0], + checksum: crypto + .createHash('sha256') + .update(fs.readFileSync(path.join(directory, name), 'utf8').replaceAll('\r\n', '\n')) + .digest('hex'), + })) + const applied = await db.prepare('SELECT version, checksum FROM schema_migrations ORDER BY version').all() + if ( + !applied.length || + applied.length > expected.length || + applied.some((row, index) => row.version !== expected[index]?.version || row.checksum !== expected[index]?.checksum) + ) { + throw new Error('PostgreSQL migration history is missing, unknown or inconsistent') + } + if (fs.existsSync(path.join(dataDir, 'kin.db'))) { + const row = await db.prepare("SELECT value FROM settings WHERE key = 'sqlite_import_done'").get() + if (!row?.value || !JSON.parse(row.value).source_sha256) throw new Error('missing import marker') + } else { + await db.prepare('SELECT 1 AS ok').get() + } +} catch { + console.error( + 'PostgreSQL cutover check failed. Keep the current container and verify PostgreSQL 17, migration history, import marker or offline recovery; see docs/deploy/postgresql.md.', + ) + process.exitCode = 1 +} finally { + await db?.close() +} diff --git a/scripts/convert-sqlite-backup.mjs b/scripts/convert-sqlite-backup.mjs new file mode 100644 index 000000000..16063ba45 --- /dev/null +++ b/scripts/convert-sqlite-backup.mjs @@ -0,0 +1,96 @@ +import fs from 'node:fs' +import path from 'node:path' +import os from 'node:os' +import crypto from 'node:crypto' +import { DatabaseSync } from 'node:sqlite' +import { parseArgs } from 'node:util' +import { createDatabase } from '../src/lib/db/database.mjs' +import { importSqlite } from '../src/lib/db/sqlite-import.mjs' +import { materializeSqliteBackup } from '../src/lib/db/sqlite-backup.mjs' +import { runBackupJob } from '../src/lib/admin/backup-core.mjs' +const { values } = parseArgs({ + options: { + source: { type: 'string' }, + output: { type: 'string' }, + 'vm-dir': { type: 'string' }, + 'config-dir': { type: 'string' }, + }, +}) +if (!values.source || !values.output) + throw new Error( + 'usage: npm run db:convert:sqlite-backup -- --source --output ; target PostgreSQL must be empty', + ) +const output = path.resolve(values.output) +if (fs.existsSync(output)) throw new Error('output already exists') +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-convert-backup-')) +let db, client +try { + const source = await materializeSqliteBackup(path.resolve(values.source), tmp) + for (const [option, folder] of [ + ['vm-dir', 'vms'], + ['config-dir', 'config'], + ]) { + const destination = path.join(tmp, folder) + if (values[option]) { + const directory = path.resolve(values[option]) + if (!fs.statSync(directory).isDirectory()) throw new Error(`${option} must be a directory`) + fs.mkdirSync(destination, { recursive: true, mode: 0o700 }) + for (const name of fs.readdirSync(directory)) { + const file = path.join(directory, name) + if (!name.endsWith('.json') || !fs.lstatSync(file).isFile()) continue + fs.copyFileSync(file, path.join(destination, name)) + fs.chmodSync(path.join(destination, name), 0o600) + } + } + if (!fs.existsSync(destination)) throw new Error(`legacy backup requires --${option} to restore companion files`) + } + const legacy = new DatabaseSync(source, { readOnly: true }) + try { + for (const { id } of legacy.prepare('SELECT id FROM vms').all()) { + if (typeof id !== 'string' || path.basename(id) !== id || !fs.existsSync(path.join(tmp, 'vms', id + '.json'))) + throw new Error('companion VM files do not cover the SQLite VM records') + } + } finally { + legacy.close() + } + db = await createDatabase({ allowLegacySource: true }) + await importSqlite({ source, db }) + client = await db.pool.connect() + await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY') + const snapshot = (await client.query('SELECT pg_export_snapshot() AS id')).rows[0].id + const schemaVersion = (await db.prepare('SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1').get()) + .version + const record = await runBackupJob( + { + connectionString: db.connectionString, + schema: db.schema, + schemaVersion, + snapshot, + backupDir: path.dirname(output), + tmpDir: tmp, + projectRoot: tmp, + configDir: path.join(tmp, 'config'), + }, + { + id: 'converted_' + crypto.randomBytes(6).toString('hex'), + createdAt: new Date().toISOString(), + kind: 'converted', + }, + ) + if (record.file_path !== output) { + fs.linkSync(record.file_path, output) + fs.unlinkSync(record.file_path) + } + await client.query('COMMIT') + console.log(JSON.stringify({ ok: true, output, sha256: record.sha256, includes: record.includes })) +} catch (error) { + console.error(error.message) + process.exitCode = 1 +} finally { + if (client) { + await client.query('ROLLBACK').catch(() => {}) + client.release() + } + if (db) await db.close() + fs.rmSync(tmp, { recursive: true, force: true }) +} diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh index 0b936259c..617bd966a 100644 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker-entrypoint.sh @@ -3,6 +3,10 @@ set -eu ROOT="${KIN_PROJECT_ROOT:-/opt/vm2api}" IMAGE_BIN="/opt/vm2api/image-bin" +if [ -z "${VM2API_DATABASE_URL:-${KIN_DATABASE_URL:-}}" ]; then + echo "vm2api: PostgreSQL 17 connection is required. Configure VM2API_DATABASE_URL; see docs/deploy/postgresql.md for SQLite import." >&2 + exit 1 +fi mkdir -p "$ROOT/vms" "$ROOT/data" "$ROOT/bin" # 镜像安装没有仓库 checkout：把镜像内默认配置补进被 bind-mount 覆盖的 src/config。 diff --git a/scripts/import-sqlite.mjs b/scripts/import-sqlite.mjs new file mode 100644 index 000000000..d3050c933 --- /dev/null +++ b/scripts/import-sqlite.mjs @@ -0,0 +1,15 @@ +import { parseArgs } from 'node:util' +import { createDatabase } from '../src/lib/db/database.mjs' +import { importSqlite } from '../src/lib/db/sqlite-import.mjs' +const { values } = parseArgs({ options: { source: { type: 'string' } } }) +if (!values.source) throw new Error('usage: npm run db:import:sqlite -- --source ') +const db = await createDatabase({ allowLegacySource: true }) +try { + const manifest = await importSqlite({ source: values.source, db }) + console.log(JSON.stringify({ ok: true, source_sha256: manifest.source_sha256, tables: manifest.tables })) +} catch (error) { + console.error(error.message) + process.exitCode = 1 +} finally { + await db.close() +} diff --git a/scripts/panel-upgrade.sh b/scripts/panel-upgrade.sh index 65fa1bb1c..67a948294 100755 --- a/scripts/panel-upgrade.sh +++ b/scripts/panel-upgrade.sh @@ -9,7 +9,6 @@ TAG=$5 ARCH=$6 shift 6 cd "$ROOT" - COMPOSE_FILE_LIST='' for file do if [ -n "$COMPOSE_FILE_LIST" ]; then COMPOSE_FILE_LIST="$COMPOSE_FILE_LIST:"; fi @@ -126,6 +125,8 @@ else compose pull vm2api fi +# The new image checks the import marker without starting HTTP or applying migrations. +compose run --rm --no-deps -T --entrypoint node vm2api scripts/check-postgres-upgrade.mjs compose up -d --no-deps --force-recreate vm2api CURRENT_ID=$(docker inspect --format '{{.Id}}' "$CONTAINER") if [ "$CURRENT_ID" = "$PREVIOUS_ID" ]; then diff --git a/scripts/run-isolated-tests.mjs b/scripts/run-isolated-tests.mjs index 9699a3029..8ac6c6427 100644 --- a/scripts/run-isolated-tests.mjs +++ b/scripts/run-isolated-tests.mjs @@ -3,7 +3,7 @@ * `--test-isolation=process` is Node 24+; CI stays on Node 22 to match prod. */ import { spawnSync } from 'node:child_process' -import { readdirSync } from 'node:fs' +import { readdirSync, readFileSync } from 'node:fs' import path from 'node:path' const dir = process.argv[2] @@ -25,11 +25,23 @@ if (!files.length) { let failed = 0 for (const file of files) { - const result = spawnSync(process.execPath, ['--test', file], { + const source = readFileSync(file, 'utf8') + const offline = /(?:sqlite-legacy|db-migration-sub2api)\.test\.mjs$/.test(file) + const database = + !offline && + /db\/|database\.mjs|api-keys\.mjs|panel-users\.mjs|account-quota\.mjs|sticky-router\.mjs|proxy-pool\.mjs|request-log\.mjs|api-endpoints\.mjs|thinking-owner\.mjs|harness\.mjs/.test( + source, + ) + const args = database ? ['--import', './test/pg-setup.mjs', '--test', file] : ['--test', file] + const result = spawnSync(process.execPath, args, { stdio: 'inherit', + timeout: 90_000, env: process.env, }) - if (result.status !== 0) failed += 1 + if (result.status !== 0) { + failed += 1 + if (result.error?.code === 'ETIMEDOUT') console.error(`test file timed out: ${file}`) + } } if (failed) { diff --git a/src/lib/admin/anthropic-console.mjs b/src/lib/admin/anthropic-console.mjs index 5d3f242b2..9581b2327 100644 --- a/src/lib/admin/anthropic-console.mjs +++ b/src/lib/admin/anthropic-console.mjs @@ -62,12 +62,12 @@ function headerMap(raw) { return out } -function directKeys(input = {}, store = null) { +async function directKeys(input = {}, store = null) { const listed = Array.isArray(input.keys) ? input.keys : [input] const keys = [] for (const [row, item] of listed.entries()) { const source = item && typeof item === 'object' ? item : { api_key: item } - const saved = source.key_id || source.keyId ? savedEndpointKey(store, source.key_id || source.keyId) : null + const saved = source.key_id || source.keyId ? await savedEndpointKey(store, source.key_id || source.keyId) : null const apiKey = String(source.api_key || source.apiKey || saved?.api_key || '').trim() if (!apiKey) continue const label = String(source.label || '') @@ -97,10 +97,10 @@ function chosen(source, snake, camel, fallback) { return String(fallback || '').trim() } -export function savedEndpointKey(store, keyId) { +export async function savedEndpointKey(store, keyId) { const id = String(keyId || '').trim() if (!id || !store?.listRaw) return null - for (const endpoint of store.listRaw()) { + for (const endpoint of await store.listRaw()) { const key = (endpoint.api_key_entries || []).find((item) => item.id === id) if (!key) continue return { @@ -254,7 +254,7 @@ export async function runAnthropicConsole({ return forwardConsoleRequest(request, { cfg, forward }) } - const keys = directKeys(input, store) + const keys = await directKeys(input, store) const index = Number.isInteger(Number(input?.key_index)) ? Number(input.key_index) : null const selected = index == null ? keys : keys.filter((key) => key.row === index) if (!selected.length) { diff --git a/src/lib/admin/api-endpoints.mjs b/src/lib/admin/api-endpoints.mjs index 3a33215bd..efc257f3b 100644 --- a/src/lib/admin/api-endpoints.mjs +++ b/src/lib/admin/api-endpoints.mjs @@ -1,5 +1,5 @@ import crypto from 'node:crypto' -import { resolveStoreDb } from '../db/database.mjs' +import { resolveStoreDb, withTransaction } from '../db/database.mjs' import { ApiEndpointsRepo } from '../db/repos/api-endpoints-repo.mjs' import { mergeModelLists, @@ -101,93 +101,99 @@ export class ApiEndpointStore { this.repo = new ApiEndpointsRepo(db) } - list({ reveal = false } = {}) { - return this.repo.listHydrated().map((ep) => publicEndpointView(ep, { reveal })) + async list({ reveal = false } = {}) { + return (await this.repo.listHydrated()).map((ep) => publicEndpointView(ep, { reveal })) } - listRaw() { - return this.repo.listHydrated() + async listRaw() { + return await this.repo.listHydrated() } - getRaw(id) { - return this.repo.getHydrated(id) + async getRaw(id) { + return await this.repo.getHydrated(id) } - create(input = {}) { - const preset = resolvePreset(input.kind, input) - const base_url = preset.base_url - if (!/^https?:\/\//i.test(base_url)) { - throw Object.assign(new Error('base_url must be http(s)'), { code: 'invalid_base_url' }) - } - const rec = { - id: nid('ep_'), - name: - String(preset.name || 'api') - .trim() - .slice(0, 80) || 'api', - disabled: !!input.disabled, - prefix: String(input.prefix || '').trim(), - base_url, - kind: preset.kind, - protocol: preset.protocol, - headers: headersWithAuthScheme( - input.headers && typeof input.headers === 'object' ? input.headers : {}, - input.auth_scheme || input.authScheme, - ), - disable_cooling: !!input.disable_cooling, - priority: Number(input.priority) || 0, - created_at: nowIso(), - updated_at: nowIso(), - } - const stored = this.repo.insertEndpoint(rec) - const models = Array.isArray(input.models) ? input.models : [] - if (models.length) this.replaceModels(stored.id, models) - const keys = Array.isArray(input.api_key_entries) ? input.api_key_entries : [] - for (const k of keys) this.addKey(stored.id, k) - return this.repo.getHydrated(stored.id) + async create(input = {}) { + return withTransaction(this.db, async () => { + const preset = resolvePreset(input.kind, input) + const base_url = preset.base_url + if (!/^https?:\/\//i.test(base_url)) { + throw Object.assign(new Error('base_url must be http(s)'), { code: 'invalid_base_url' }) + } + const rec = { + id: nid('ep_'), + name: + String(preset.name || 'api') + .trim() + .slice(0, 80) || 'api', + disabled: !!input.disabled, + prefix: String(input.prefix || '').trim(), + base_url, + kind: preset.kind, + protocol: preset.protocol, + headers: headersWithAuthScheme( + input.headers && typeof input.headers === 'object' ? input.headers : {}, + input.auth_scheme || input.authScheme, + ), + disable_cooling: !!input.disable_cooling, + priority: Number(input.priority) || 0, + created_at: nowIso(), + updated_at: nowIso(), + } + const stored = await this.repo.insertEndpoint(rec) + const models = Array.isArray(input.models) ? input.models : [] + if (models.length) await this.replaceModels(stored.id, models) + const keys = Array.isArray(input.api_key_entries) ? input.api_key_entries : [] + for (const k of keys) await this.addKey(stored.id, k) + return await this.repo.getHydrated(stored.id) + }) } - update(id, patch = {}) { - const rec = this.repo.getHydrated(id) - if (!rec) return null - if (patch.name != null) rec.name = String(patch.name).trim().slice(0, 80) || rec.name - if (patch.disabled != null) rec.disabled = !!patch.disabled - if (patch.prefix != null) rec.prefix = String(patch.prefix).trim() - if (patch.kind != null || patch.protocol != null || patch.base_url != null) { - const preset = resolvePreset(patch.kind ?? rec.kind, { - ...rec, - ...patch, - base_url: patch.base_url != null ? patch.base_url : rec.base_url, - }) - rec.kind = preset.kind - rec.protocol = preset.protocol - rec.base_url = preset.base_url - if (!/^https?:\/\//i.test(rec.base_url)) { - throw Object.assign(new Error('base_url must be http(s)'), { code: 'invalid_base_url' }) + async update(id, patch = {}) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT id FROM api_endpoints WHERE id = $1 FOR UPDATE').get(id) + + const rec = await this.repo.getHydrated(id) + if (!rec) return null + if (patch.name != null) rec.name = String(patch.name).trim().slice(0, 80) || rec.name + if (patch.disabled != null) rec.disabled = !!patch.disabled + if (patch.prefix != null) rec.prefix = String(patch.prefix).trim() + if (patch.kind != null || patch.protocol != null || patch.base_url != null) { + const preset = resolvePreset(patch.kind ?? rec.kind, { + ...rec, + ...patch, + base_url: patch.base_url != null ? patch.base_url : rec.base_url, + }) + rec.kind = preset.kind + rec.protocol = preset.protocol + rec.base_url = preset.base_url + if (!/^https?:\/\//i.test(rec.base_url)) { + throw Object.assign(new Error('base_url must be http(s)'), { code: 'invalid_base_url' }) + } } - } - if (patch.headers != null && typeof patch.headers === 'object') rec.headers = patch.headers - if (patch.auth_scheme != null || patch.authScheme != null) { - rec.headers = headersWithAuthScheme(rec.headers, patch.auth_scheme || patch.authScheme) - } - if (patch.disable_cooling != null) rec.disable_cooling = !!patch.disable_cooling - if (patch.priority != null) rec.priority = Number(patch.priority) || 0 - rec.updated_at = nowIso() - this.repo.updateEndpoint(rec) - if (Array.isArray(patch.models)) this.replaceModels(id, patch.models) - return this.repo.getHydrated(id) + if (patch.headers != null && typeof patch.headers === 'object') rec.headers = patch.headers + if (patch.auth_scheme != null || patch.authScheme != null) { + rec.headers = headersWithAuthScheme(rec.headers, patch.auth_scheme || patch.authScheme) + } + if (patch.disable_cooling != null) rec.disable_cooling = !!patch.disable_cooling + if (patch.priority != null) rec.priority = Number(patch.priority) || 0 + rec.updated_at = nowIso() + await this.repo.updateEndpoint(rec) + if (Array.isArray(patch.models)) await this.replaceModels(id, patch.models) + return await this.repo.getHydrated(id) + }) } - remove(id) { - return this.repo.removeEndpoint(id) + async remove(id) { + return await this.repo.removeEndpoint(id) } - addKey(endpointId, input = {}) { - const ep = this.repo.getHydrated(endpointId) + async addKey(endpointId, input = {}) { + const ep = await this.repo.getHydrated(endpointId) if (!ep) throw Object.assign(new Error('endpoint not found'), { code: 'endpoint_not_found' }) const api_key = String(input.api_key || '').trim() if (api_key.length < 4) throw Object.assign(new Error('api_key too short'), { code: 'key_too_short' }) - return this.repo.insertKey({ + return await this.repo.insertKey({ id: nid('ek_'), endpoint_id: endpointId, api_key, @@ -199,15 +205,15 @@ export class ApiEndpointStore { }) } - removeKey(id) { - return this.repo.removeKey(id) + async removeKey(id) { + return await this.repo.removeKey(id) } - updateKey(id, patch = {}) { - return this.repo.updateKey(id, patch) + async updateKey(id, patch = {}) { + return await this.repo.updateKey(id, patch) } - replaceModels(endpointId, models = []) { + async replaceModels(endpointId, models = []) { const rows = models .map((m, i) => ({ id: m.id || nid('em_'), @@ -218,17 +224,17 @@ export class ApiEndpointStore { thinking: m.thinking, })) .filter((m) => m.name && m.alias) - return this.repo.replaceModels(endpointId, rows) + return await this.repo.replaceModels(endpointId, rows) } - setKeyCooldown(id, untilIso) { - this.repo.setKeyCooldown(id, untilIso) + async setKeyCooldown(id, untilIso) { + await this.repo.setKeyCooldown(id, untilIso) } - mergeFetchedModels(endpointId, fetched = []) { - const ep = this.repo.getHydrated(endpointId) + async mergeFetchedModels(endpointId, fetched = []) { + const ep = await this.repo.getHydrated(endpointId) if (!ep) return null - return this.replaceModels(endpointId, mergeModelLists(ep.models, fetched)) + return await this.replaceModels(endpointId, mergeModelLists(ep.models, fetched)) } } diff --git a/src/lib/admin/api-keys.mjs b/src/lib/admin/api-keys.mjs index 61db2f12a..a2c35fa5c 100644 --- a/src/lib/admin/api-keys.mjs +++ b/src/lib/admin/api-keys.mjs @@ -1,5 +1,5 @@ /** - * Multi API-key store (sub2api-inspired, SQLite-backed). + * Multi API-key store (sub2api-inspired, PostgreSQL-backed). * * Fields aligned to sub2api basics: * name, status, expires_at, quota (+ used), rate windows → rpm, @@ -18,7 +18,8 @@ import { keyScopeFromRecord, normalizeKeyScope } from './key-scope.mjs' import { VmPoolsRepo } from '../db/repos/vm-pools-repo.mjs' import crypto from 'node:crypto' -import { resolveStoreDb } from '../db/database.mjs' +import { resolveStoreDb, withTransaction } from '../db/database.mjs' +import { integerValue } from '../db/integers.mjs' import { ApiKeysRepo } from '../db/repos/api-keys-repo.mjs' import { maybeEncrypt, maybeDecrypt } from '../db/secure.mjs' import { calculateCost, normalizeUsage } from './pricing.mjs' @@ -139,10 +140,10 @@ export class ApiKeyStore { } /** sub2api: an api key of a disabled/deleted user must not authenticate. */ - _ownerActive(rec) { + async _ownerActive(rec) { if (!rec?.user_id) return true // legacy unowned keys keep working try { - const owner = this.users.getById(rec.user_id) + const owner = await this.users.getById(rec.user_id) // soft-deleted owner is invisible here → key is revoked with them return !!owner && owner.status === 'active' } catch { @@ -160,14 +161,14 @@ export class ApiKeyStore { this.users = new UsersRepo(db) } - list({ reveal = false } = {}) { + async list({ reveal = false } = {}) { const names = new Map() try { - for (const pool of new VmPoolsRepo(this.db).list()) names.set(pool.id, pool.name) + for (const pool of await new VmPoolsRepo(this.db).list()) names.set(pool.id, pool.name) } catch { // A database opened before migration 034 still lists keys. } - return this.repo.list().map((k) => { + return (await this.repo.list()).map((k) => { const v = publicKeyView(k, { reveal }) v.inflight = this.inflight.get(k.id) || 0 if (v.vm_pool_id) v.vm_pool_name = names.get(v.vm_pool_id) || null @@ -175,34 +176,38 @@ export class ApiKeyStore { }) } - getById(id) { - return this.repo.getById(id) + async getById(id) { + return await this.repo.getById(id) } - authenticate(token) { + async authenticate(token) { if (!token) return { ok: false, reason: 'missing' } const hash = this._hash(token) - const indexed = this.repo.getByHash(hash) + const indexed = await this.repo.getByHash(hash) if (indexed) { - if (!this._ownerActive(indexed)) return { ok: false, reason: 'user_disabled' } + if (!(await this._ownerActive(indexed))) return { ok: false, reason: 'user_disabled' } return { ok: true, record: indexed } } // One-time migration for legacy plaintext rows. - for (const rec of this.repo.list()) { + for (const rec of await this.repo.list()) { if (String(rec.key || '').startsWith(HASH_MARKER)) continue if (timingSafeEqualStr(token, rec.key)) { - rec.key_hash = hash - rec.key_prefix = String(token).slice(0, 10) - rec.key_suffix = String(token).slice(-4) - rec.key_secret = maybeEncrypt(token) - rec.key = HASH_MARKER + hash - return { ok: true, record: this.repo.update(rec) } + return withTransaction(this.db, async () => { + const current = await this.repo.getById(rec.id, { forUpdate: true }) + if (!current || !timingSafeEqualStr(token, current.key)) return { ok: false, reason: 'invalid' } + current.key_hash = hash + current.key_prefix = String(token).slice(0, 10) + current.key_suffix = String(token).slice(-4) + current.key_secret = maybeEncrypt(token) + current.key = HASH_MARKER + hash + return { ok: true, record: await this.repo.update(current) } + }) } } return { ok: false, reason: 'invalid' } } - create(input = {}) { + async create(input = {}) { const name = String(input.name || 'default') .trim() @@ -213,7 +218,7 @@ export class ApiKeyStore { throw Object.assign(new Error('key has invalid characters'), { code: 'key_invalid' }) } const keyHash = this._hash(key) - if (this.repo.getByHash(keyHash) || this.repo.getByKey(key)) { + if ((await this.repo.getByHash(keyHash)) || (await this.repo.getByKey(key))) { throw Object.assign(new Error('key already exists'), { code: 'key_exists' }) } @@ -256,65 +261,67 @@ export class ApiKeyStore { if (rec.expires_at && Number.isNaN(Date.parse(rec.expires_at))) { throw Object.assign(new Error('invalid expires_at'), { code: 'invalid_expires_at' }) } - const stored = this.repo.insert(rec) + const stored = await this.repo.insert(rec) return { ...stored, key, _plain_key: key } } - update(id, patch = {}) { - const rec = this.repo.getById(id) - if (!rec) return null - if (patch.name != null) rec.name = String(patch.name).trim().slice(0, 100) || rec.name - if (patch.status != null) { - const st = String(patch.status).toLowerCase() - if (!['active', 'disabled'].includes(st)) { - throw Object.assign(new Error('invalid status'), { code: 'invalid_status' }) + async update(id, patch = {}) { + return withTransaction(this.db, async () => { + const rec = await this.repo.getById(id, { forUpdate: true }) + if (!rec) return null + if (patch.name != null) rec.name = String(patch.name).trim().slice(0, 100) || rec.name + if (patch.status != null) { + const st = String(patch.status).toLowerCase() + if (!['active', 'disabled'].includes(st)) { + throw Object.assign(new Error('invalid status'), { code: 'invalid_status' }) + } + rec.status = st } - rec.status = st - } - if (patch.max_concurrency != null) - rec.max_concurrency = clampInt(patch.max_concurrency, 0, 128, rec.max_concurrency) - if (patch.quota_requests != null) rec.quota_requests = clampInt(patch.quota_requests, 0, 1e9, rec.quota_requests) - if (patch.quota != null) rec.quota = Math.max(0, Number(patch.quota) || 0) - if (patch.rate_limit_5h != null) rec.rate_limit_5h = Math.max(0, Number(patch.rate_limit_5h) || 0) - if (patch.rate_limit_1d != null) rec.rate_limit_1d = Math.max(0, Number(patch.rate_limit_1d) || 0) - if (patch.rate_limit_7d != null) rec.rate_limit_7d = Math.max(0, Number(patch.rate_limit_7d) || 0) - if (patch.group_id != null) rec.group_id = Number(patch.group_id) || 1 - if (patch.user_id != null) rec.user_id = patch.user_id - if (patch.rpm != null) rec.rpm = clampInt(patch.rpm, 0, 1e6, rec.rpm) - if (patch.expires_at === null) rec.expires_at = null - else if (patch.expires_at != null) { - const iso = new Date(patch.expires_at).toISOString() - if (Number.isNaN(Date.parse(iso))) - throw Object.assign(new Error('invalid expires_at'), { code: 'invalid_expires_at' }) - rec.expires_at = iso - } - if (patch.reset_quota === true) { - rec.quota_requests_used = 0 - rec.quota_used = 0 - } - // recover from quota_exhausted whenever spend is back under the cap - if ( - rec.status === 'quota_exhausted' && - !((rec.quota ?? 0) > 0 && (rec.quota_used ?? 0) >= rec.quota) && - !(rec.quota_requests > 0 && (rec.quota_requests_used ?? 0) >= rec.quota_requests) - ) { - rec.status = 'active' - } - if (patch.category != null) { - rec.category = String(patch.category).toLowerCase() === 'api' ? 'api' : 'oauth' - } - const scope = normalizeKeyScope(patch, { partial: true, current: rec }) - if (scope) Object.assign(rec, storedKeyScope(scope)) - rec.updated_at = nowIso() - return this.repo.update(rec) + if (patch.max_concurrency != null) + rec.max_concurrency = clampInt(patch.max_concurrency, 0, 128, rec.max_concurrency) + if (patch.quota_requests != null) rec.quota_requests = clampInt(patch.quota_requests, 0, 1e9, rec.quota_requests) + if (patch.quota != null) rec.quota = Math.max(0, Number(patch.quota) || 0) + if (patch.rate_limit_5h != null) rec.rate_limit_5h = Math.max(0, Number(patch.rate_limit_5h) || 0) + if (patch.rate_limit_1d != null) rec.rate_limit_1d = Math.max(0, Number(patch.rate_limit_1d) || 0) + if (patch.rate_limit_7d != null) rec.rate_limit_7d = Math.max(0, Number(patch.rate_limit_7d) || 0) + if (patch.group_id != null) rec.group_id = Number(patch.group_id) || 1 + if (patch.user_id != null) rec.user_id = patch.user_id + if (patch.rpm != null) rec.rpm = clampInt(patch.rpm, 0, 1e6, rec.rpm) + if (patch.expires_at === null) rec.expires_at = null + else if (patch.expires_at != null) { + const iso = new Date(patch.expires_at).toISOString() + if (Number.isNaN(Date.parse(iso))) + throw Object.assign(new Error('invalid expires_at'), { code: 'invalid_expires_at' }) + rec.expires_at = iso + } + if (patch.reset_quota === true) { + rec.quota_requests_used = 0 + rec.quota_used = 0 + } + // recover from quota_exhausted whenever spend is back under the cap + if ( + rec.status === 'quota_exhausted' && + !((rec.quota ?? 0) > 0 && (rec.quota_used ?? 0) >= rec.quota) && + !(rec.quota_requests > 0 && BigInt(rec.quota_requests_used ?? 0) >= BigInt(rec.quota_requests)) + ) { + rec.status = 'active' + } + if (patch.category != null) { + rec.category = String(patch.category).toLowerCase() === 'api' ? 'api' : 'oauth' + } + const scope = normalizeKeyScope(patch, { partial: true, current: rec }) + if (scope) Object.assign(rec, storedKeyScope(scope)) + rec.updated_at = nowIso() + return await this.repo.update(rec) + }) } /** * Plaintext of a managed key for panel copy. * @returns {null | { ok: false, code: string } | { ok: true, id, name, key }} */ - reveal(id) { - const rec = this.repo.getById(id) + async reveal(id) { + const rec = await this.repo.getById(id, { forUpdate: true }) if (!rec) return null const key = plainApiKey(rec) if (!key) return { ok: false, code: 'key_not_recoverable' } @@ -322,26 +329,28 @@ export class ApiKeyStore { } /** Issue a fresh plaintext on the same record; the old value stops working. */ - rotate(id) { - const rec = this.repo.getById(id) - if (!rec) return null - const key = generateApiKey() - const keyHash = this._hash(key) - if (this.repo.getByHash(keyHash)) { - throw Object.assign(new Error('key already exists'), { code: 'key_exists' }) - } - rec.key = HASH_MARKER + keyHash - rec.key_hash = keyHash - rec.key_prefix = key.slice(0, 10) - rec.key_suffix = key.slice(-4) - rec.key_secret = maybeEncrypt(key) - rec.updated_at = nowIso() - const stored = this.repo.update(rec) - return { ...stored, key, _plain_key: key } + async rotate(id) { + return withTransaction(this.db, async () => { + const rec = await this.repo.getById(id, { forUpdate: true }) + if (!rec) return null + const key = generateApiKey() + const keyHash = this._hash(key) + if (await this.repo.getByHash(keyHash)) { + throw Object.assign(new Error('key already exists'), { code: 'key_exists' }) + } + rec.key = HASH_MARKER + keyHash + rec.key_hash = keyHash + rec.key_prefix = key.slice(0, 10) + rec.key_suffix = key.slice(-4) + rec.key_secret = maybeEncrypt(key) + rec.updated_at = nowIso() + const stored = await this.repo.update(rec) + return { ...stored, key, _plain_key: key } + }) } - remove(id) { - const removed = this.repo.remove(id) + async remove(id) { + const removed = await this.repo.remove(id) if (!removed) return false this.inflight.delete(id) this.rpmBuckets.delete(id) @@ -365,7 +374,9 @@ export class ApiKeyStore { * Pre-flight gate for a managed key. Does not acquire concurrency. * @returns {{ ok: true } | { ok: false, code, message, status }} */ - canAccept(rec, now = Date.now()) { + async canAccept(rec, now = Date.now(), { reserve = false } = {}) { + const owner = rec?.user_id ? await this.users.getById(rec.user_id) : null + const ownedKeys = rec?.user_id ? await this.repo.list() : [] if (!rec) return { ok: false, code: 'invalid_api_key', message: 'Invalid credentials', status: 401 } if (rec.status !== 'active') { if (rec.status === 'quota_exhausted') { @@ -376,7 +387,7 @@ export class ApiKeyStore { if (this._expired(rec, now)) { return { ok: false, code: 'api_key_expired', message: 'API key has expired', status: 403 } } - if (rec.quota_requests > 0 && (rec.quota_requests_used ?? 0) >= rec.quota_requests) { + if (rec.quota_requests > 0 && BigInt(rec.quota_requests_used ?? 0) >= BigInt(rec.quota_requests)) { return { ok: false, code: 'api_key_quota_exhausted', @@ -439,17 +450,10 @@ export class ApiKeyStore { // sub2api user-level cap: sum of inflight across all of the owner's keys. // The operator (admin) is never capped: its traffic is the platform's own. if (rec.user_id) { - const owner = (() => { - try { - return this.users.getById(rec.user_id) - } catch { - return null - } - })() const cap = owner?.role === 'admin' ? 0 : Number(owner?.concurrency) || 0 if (cap > 0) { let total = 0 - for (const k of this.repo.list()) { + for (const k of ownedKeys) { if (k.user_id === rec.user_id) total += this.inflight.get(k.id) || 0 } if (total >= cap) { @@ -463,18 +467,17 @@ export class ApiKeyStore { } } } + if (reserve) { + this.inflight.set(rec.id, (this.inflight.get(rec.id) || 0) + 1) + const arr = this.rpmBuckets.get(rec.id) || [] + arr.push(now) + this.rpmBuckets.set(rec.id, arr) + } return { ok: true } } - acquire(rec, now = Date.now()) { - const gate = this.canAccept(rec, now) - if (!gate.ok) return gate - const id = rec.id - this.inflight.set(id, (this.inflight.get(id) || 0) + 1) - const arr = this.rpmBuckets.get(id) || [] - arr.push(now) - this.rpmBuckets.set(id, arr) - return { ok: true } + async acquire(rec, now = Date.now()) { + return this.canAccept(rec, now, { reserve: true }) } release(recOrId) { @@ -485,9 +488,9 @@ export class ApiKeyStore { else this.inflight.set(id, n) } - recordUsage(recOrId, usage = {}, { model = null, rateMultiplier = 1, officialCost } = {}) { + async recordUsage(recOrId, usage = {}, { model = null, rateMultiplier = 1, officialCost } = {}) { const id = typeof recOrId === 'string' ? recOrId : recOrId?.id - if (!this.repo.getById(id)) return null + if (!(await this.repo.getById(id))) return null const normalized = normalizeUsage(usage) let billed = officialCost if (billed == null) billed = usage.total_cost @@ -501,22 +504,18 @@ export class ApiKeyStore { // key quota/windows track actual (group-rated) spend, sub2api-style const rate = Number.isFinite(Number(rateMultiplier)) ? Number(rateMultiplier) : 1 const cost = billed * rate - const rec = this.repo.recordUsage(id, { - tokens_in: Number(normalized.input_tokens) || Number(normalized.tokens_in) || 0, - tokens_out: Number(normalized.output_tokens) || Number(normalized.tokens_out) || 0, - cache_read_tokens: Number(normalized.cache_read_input_tokens ?? normalized.cache_read_tokens) || 0, - cache_creation_tokens: Number(normalized.cache_creation_input_tokens ?? normalized.cache_creation_tokens) || 0, + const rec = await this.repo.recordUsage(id, { + tokens_in: integerValue(normalized.input_tokens ?? normalized.tokens_in), + tokens_out: integerValue(normalized.output_tokens ?? normalized.tokens_out), + cache_read_tokens: integerValue(normalized.cache_read_input_tokens ?? normalized.cache_read_tokens), + cache_creation_tokens: integerValue(normalized.cache_creation_input_tokens ?? normalized.cache_creation_tokens), cost, }) - // sub2api: the billing write flips the key when the USD quota runs out - if (rec && rec.status === 'active' && (rec.quota ?? 0) > 0 && (rec.quota_used ?? 0) >= rec.quota) { - this.repo.update({ ...rec, status: 'quota_exhausted', updated_at: nowIso() }) - } return rec } - snapshot() { - const keys = this.list({ reveal: false }) + async snapshot() { + const keys = await this.list({ reveal: false }) return { total: keys.length, active: keys.filter((k) => k.status === 'active').length, diff --git a/src/lib/admin/backup-core.mjs b/src/lib/admin/backup-core.mjs index 2913b9b80..f78732bec 100644 --- a/src/lib/admin/backup-core.mjs +++ b/src/lib/admin/backup-core.mjs @@ -1,27 +1,9 @@ -/** - * Backup pipeline primitives. - * - * Why this module exists: the whole pipeline (VACUUM INTO of a multi-GB - * database, sha256 of the snapshot, `tar -czf`, sha256 of the archive, staging - * cleanup) is synchronous. Running it on the gateway's main thread stops the - * HTTP server from answering any request for the entire backup — a 3-5 minute - * hole that turns into Cloudflare 524s for /v1 traffic, the panel and /healthz. - * - * So the pipeline lives here and is executed by backup-worker.mjs on its own - * thread with its own database connection, while BackupService stays a thin - * orchestrator on the main thread. - * - * Fallback: when `tar` is unavailable the snapshot is gzipped in memory - * (only sane below ~2 GiB) — unchanged from the original implementation. - */ - +/** PostgreSQL backup primitives. Child processes keep dump/compression off the HTTP thread. */ import fs from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' -import zlib from 'node:zlib' -import { spawnSync } from 'node:child_process' -import { getDb, vacuumInto } from '../db/database.mjs' -import { BackupRepo } from '../db/repos/backup-repo.mjs' +import { spawn, spawnSync } from 'node:child_process' +import { pipeline } from 'node:stream/promises' export function stamp(d = new Date()) { return d.toISOString().replace(/[:.]/g, '-') @@ -63,31 +45,8 @@ function mkdtempOnDataVolume({ tmpDir }) { return fs.mkdtempSync(path.join(base, 'kin-backup-')) } -/** Drop backups past the retention window (pre_restore keeps the newest 3). */ -function applyRetention(repo, retention) { - const drop = [...repo.beyondRetention(retention), ...repo.preRestoreBeyond(3)] - for (const rec of drop) { - try { - if (rec.file_path && fs.existsSync(rec.file_path)) fs.rmSync(rec.file_path, { force: true }) - } catch {} - try { - repo.remove(rec.id) - } catch {} - } - return drop.length -} - -/** - * Run one backup job end to end. Blocking by design — call it from a worker - * thread (or the test process), never from the gateway's main thread. - * - * The caller owns the database connection: open it (openDatabase/openDatabase - * singleton) before calling and close it afterwards. - * - * @returns backup record (status 'ok') - */ -export function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null, retention = 7 }) { - const repo = new BackupRepo(getDb()) +/** The parent holds the exported snapshot until this asynchronous worker job completes. */ +export async function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null }, onFilesCopied = () => {}) { let tmp = null try { fs.mkdirSync(ctx.backupDir, { recursive: true, mode: 0o700 }) @@ -96,14 +55,14 @@ export function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null, } catch {} tmp = mkdtempOnDataVolume(ctx) - // 1. online-consistent DB snapshot (WAL-safe) - const dbSnap = path.join(tmp, 'db', 'kin.db') - vacuumInto(getDb(), dbSnap) - const dbBytes = fs.statSync(dbSnap).size + const dbSnap = path.join(tmp, 'db', 'postgres.dump') + fs.mkdirSync(path.dirname(dbSnap), { recursive: true, mode: 0o700 }) // 2. vm files + config files const includes = { db: true, vms: 0, config: 0 } - const sha256s = { 'db/kin.db': sha256File(dbSnap) } + const sha256s = {} + if (ctx.projectRoot) fs.mkdirSync(path.join(tmp, 'vms'), { recursive: true }) + if (ctx.configDir) fs.mkdirSync(path.join(tmp, 'config'), { recursive: true }) const vmsDir = ctx.projectRoot ? path.join(ctx.projectRoot, 'vms') : null if (vmsDir && fs.existsSync(vmsDir)) { const outVms = path.join(tmp, 'vms') @@ -111,13 +70,11 @@ export function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null, for (const f of fs.readdirSync(vmsDir)) { if (!/\.json$/.test(f)) continue const src = path.join(vmsDir, f) - try { - if (!fs.statSync(src).isFile()) continue - fs.copyFileSync(src, path.join(outVms, f)) - fs.chmodSync(path.join(outVms, f), 0o600) - sha256s[`vms/${f}`] = sha256File(src) - includes.vms += 1 - } catch {} + if (!fs.statSync(src).isFile()) continue + fs.copyFileSync(src, path.join(outVms, f)) + fs.chmodSync(path.join(outVms, f), 0o600) + sha256s[`vms/${f}`] = sha256File(path.join(outVms, f)) + includes.vms += 1 } } if (ctx.configDir && fs.existsSync(ctx.configDir)) { @@ -126,18 +83,37 @@ export function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null, for (const f of fs.readdirSync(ctx.configDir)) { if (!/\.json$/.test(f)) continue const src = path.join(ctx.configDir, f) - try { - if (!fs.statSync(src).isFile()) continue - fs.copyFileSync(src, path.join(outCfg, f)) - sha256s[`config/${f}`] = sha256File(src) - includes.config += 1 - } catch {} + if (!fs.statSync(src).isFile()) continue + fs.copyFileSync(src, path.join(outCfg, f)) + sha256s[`config/${f}`] = sha256File(path.join(outCfg, f)) + includes.config += 1 } } - // 3. manifest + // Files and exported DB snapshot refer to the same write-free window. + await onFilesCopied() + await runPgTool( + 'pg_dump', + [ + '--format=custom', + '--no-owner', + '--no-privileges', + `--schema=${ctx.schema}`, + `--snapshot=${ctx.snapshot}`, + '--file', + dbSnap, + ], + ctx.connectionString, + ) + fs.chmodSync(dbSnap, 0o600) + const dbBytes = fs.statSync(dbSnap).size + sha256s['db/postgres.dump'] = sha256File(dbSnap) + const manifest = { - version: 1, + version: 2, + engine: 'postgresql', + schema: ctx.schema, + schema_version: ctx.schemaVersion, tool: 'kin-gateway', created_at: createdAt, kind, @@ -146,30 +122,12 @@ export function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null, } fs.writeFileSync(path.join(tmp, 'manifest.json'), JSON.stringify(manifest, null, 2)) - // 4. archive (tar.gz preferred; gzipped db fallback) - let filePath, - fileName, - noteOut = note - if (tarAvailable()) { - fileName = `kin-backup-${stamp(new Date(createdAt))}.tar.gz` - filePath = path.join(ctx.backupDir, fileName) - const r = spawnSync('tar', ['-czf', filePath, '-C', tmp, '.'], { stdio: 'pipe' }) - if (r.status !== 0) { - throw new Error(`tar failed: ${String(r.stderr || '').slice(0, 300)}`) - } - } else { - fileName = `kin-backup-${stamp(new Date(createdAt))}.db.gz` - filePath = path.join(ctx.backupDir, fileName) - fs.writeFileSync(filePath, zlib.gzipSync(fs.readFileSync(dbSnap))) - noteOut = [note, 'tar unavailable — db-only gzip snapshot (vm files rebuilt from DB on restore)'] - .filter(Boolean) - .join(' | ') - } - try { - fs.chmodSync(filePath, 0o600) - } catch {} + const fileName = `kin-backup-${stamp(new Date(createdAt))}.tar.gz` + const filePath = path.join(ctx.backupDir, fileName) + await runProcess('tar', ['-czf', filePath, '-C', tmp, '.']) + fs.chmodSync(filePath, 0o600) - const rec = repo.insert({ + return { id, created_at: createdAt, kind, @@ -181,28 +139,8 @@ export function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null, db_bytes: dbBytes, includes, error: null, - note: noteOut, - }) - applyRetention(repo, retention) - return rec - } catch (e) { - try { - repo.insert({ - id, - created_at: createdAt, - kind, - status: 'failed', - file_path: null, - file_name: null, - size_bytes: null, - sha256: null, - db_bytes: null, - includes: null, - error: String(e.message || e).slice(0, 500), - note, - }) - } catch {} - throw e + note, + } } finally { if (tmp) { try { @@ -211,3 +149,75 @@ export function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null, } } } + +/** Secrets are passed only through the child environment; stderr may contain data and is never returned. */ +export function pgEnvironment(connectionString) { + let url + try { + url = new URL(connectionString) + } catch { + throw new Error('invalid PostgreSQL connection configuration') + } + if (!['postgres:', 'postgresql:'].includes(url.protocol)) throw new Error('PostgreSQL connection required') + const env = { + ...process.env, + PGHOST: url.searchParams.get('host') || url.hostname.replace(/^\[(.+)\]$/, '$1'), + PGPORT: url.searchParams.get('port') || url.port || '5432', + PGUSER: url.searchParams.get('user') || decodeURIComponent(url.username), + PGPASSWORD: url.searchParams.get('password') || decodeURIComponent(url.password), + PGDATABASE: decodeURIComponent(url.pathname.slice(1)), + PGCONNECT_TIMEOUT: '10', + } + for (const [key, name] of Object.entries({ + sslmode: 'PGSSLMODE', + sslcert: 'PGSSLCERT', + sslkey: 'PGSSLKEY', + sslrootcert: 'PGSSLROOTCERT', + connect_timeout: 'PGCONNECT_TIMEOUT', + application_name: 'PGAPPNAME', + options: 'PGOPTIONS', + })) { + if (url.searchParams.has(key)) env[name] = url.searchParams.get(key) + } + return env +} + +export function runProcess(command, args, env = process.env) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { env, stdio: ['ignore', 'pipe', 'ignore'] }) + let stdout = '' + child.stdout.on('data', (chunk) => { + stdout += chunk.toString() + if (stdout.length > 1024 * 1024) { + child.kill() + reject(new Error(`${command} output exceeds the archive metadata limit`)) + } + }) + child.once('error', () => reject(new Error(`${command} could not start`))) + child.once('close', (code) => + code === 0 ? resolve(stdout) : reject(new Error(`${command} failed (exit ${code})`)), + ) + }) +} + +export function runPgTool(command, args, connectionString) { + return runProcess(command, args, pgEnvironment(connectionString)) +} + +/** Hash large archives without blocking health checks on the main thread. */ +export async function sha256FileAsync(file) { + const hash = crypto.createHash('sha256') + for await (const chunk of fs.createReadStream(file)) hash.update(chunk) + return hash.digest('hex') +} + +/** Extract only a selected member's bytes; archive symlinks are never installed on disk. */ +export async function extractArchiveMember(archive, member, destination) { + fs.mkdirSync(path.dirname(destination), { recursive: true, mode: 0o700 }) + const child = spawn('tar', ['-xOzf', archive, '--', member], { stdio: ['ignore', 'pipe', 'ignore'] }) + const done = new Promise((resolve, reject) => { + child.once('error', () => reject(new Error('tar could not start'))) + child.once('close', (code) => (code === 0 ? resolve() : reject(new Error('backup archive extraction failed')))) + }) + await Promise.all([pipeline(child.stdout, fs.createWriteStream(destination, { mode: 0o600, flags: 'wx' })), done]) +} diff --git a/src/lib/admin/backup-service.mjs b/src/lib/admin/backup-service.mjs index 448ea8355..f8b19643e 100644 --- a/src/lib/admin/backup-service.mjs +++ b/src/lib/admin/backup-service.mjs @@ -1,53 +1,13 @@ -/** - * Local backup service — sub2api BackupService counterpart (no S3, per user). - * - * Backup artifact: data/backups/kin-backup-.tar.gz (0600) containing - * manifest.json {version, created_at, kind, includes, sha256s} - * db/kin.db VACUUM INTO online-consistent snapshot (credentials - * already mirrored into the `vms` table) - * vms/ slot *.json + active.json (double safety) - * config/ routing.json / intercept-rules.json … - * - * When `tar` is unavailable the service degrades to a gzipped DB snapshot - * (kin-backup-.db.gz) — still fully restorable because the DB holds - * the credential mirror (vm files are rebuilt from vms.vm_json). That fallback - * buffers the whole snapshot in memory, so it only works below ~2 GiB; the - * tar path has no such limit. - * - * Staging lives on the data volume (`/tmp`, override with - * KIN_BACKUP_TMPDIR), not os.tmpdir(): /tmp is a tmpfs sized to a fraction of - * RAM and a multi-GB `VACUUM INTO` snapshot fails there with SQLITE_FULL - * ("database or disk is full") while the real disk still has room. - * - * Scheduling (default ON, user requirement): - * settings.backup_schedule = { enabled: true, interval_hours: 24, retention: 7 } - * env overrides: KIN_BACKUP_DISABLED=1, KIN_BACKUP_INTERVAL_HOURS, KIN_BACKUP_RETENTION - * - checker runs every 10 min + a catch-up check shortly after boot - * - the job itself runs in a worker thread (backup-worker.mjs): the - * pipeline is synchronous for minutes, so running it in-process - * would stop the gateway from answering (Cloudflare 524) - * - retention prunes old ok backups (pre_restore keeps newest 3) - * - * Restore: - * verify manifest/sha256 → automatic pre_restore backup → close DB → - * swap kin.db (old one kept as kin.db.pre-restore) → restore vms/ + - * config/ files (or rebuild from DB) → reopen + migrate → rebind stores. - * `isRestoring` flag lets the server 503 protocol traffic meanwhile. - */ - +/** PostgreSQL backup scheduling, write maintenance and transactional restore. */ import fs from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' -import zlib from 'node:zlib' -import { spawnSync } from 'node:child_process' import { Worker } from 'node:worker_threads' -import { getDb, getDbPath, closeDatabase, openDatabase } from '../db/database.mjs' -import { sha256File } from './backup-core.mjs' +import { getDb, applyMigrations } from '../db/database.mjs' +import { sha256FileAsync, runPgTool, runProcess, extractArchiveMember } from './backup-core.mjs' import { SettingsRepo } from '../db/repos/settings-repo.mjs' import { BackupRepo } from '../db/repos/backup-repo.mjs' -import { reconcileVms } from '../vm/vm-db-sync.mjs' -import { repairSlotSmbios, repairVmProxyCliEnabled } from '../vm/vm-registry.mjs' -import { invalidateReadCache, isVmRecordFile } from '../vm/vm-file.mjs' +import { invalidateReadCache } from '../vm/vm-file.mjs' // tarAvailability / sha256File / the runnable pipeline live in backup-core.mjs; // keep the historical export surface for tests and callers. @@ -69,8 +29,14 @@ export class BackupService { this.backupDir = backupDir || path.join(dataDir, 'backups') this.tmpDir = tmpDir || path.join(dataDir, 'tmp') this.checkIntervalMs = checkIntervalMs - this.isRestoring = false + this.maintenanceFile = path.join(dataDir, 'restore-maintenance.json') + this.isRestoring = fs.existsSync(this.maintenanceFile) + this.isMaintaining = this.isRestoring + this._restoreRunning = false + this._maintenance = null this._running = false + this._schedulerGeneration = 0 + this._scheduledChecks = new Set() this._timer = null this._bootTimer = null this._rebindStores = null // set via onRestored() @@ -98,10 +64,27 @@ export class BackupService { this._rebindStores = typeof fn === 'function' ? fn : null } + /** The server pauses new writes, drains in-flight work, and resumes only after success. */ + onMaintenance(fn) { + this._maintenance = typeof fn === 'function' ? fn : null + } + + async _enterMaintenance(reason) { + if (this.isMaintaining) return false + this.isMaintaining = true + await this._maintenance?.(true, reason) + return true + } + + async _leaveMaintenance(reason) { + await this._maintenance?.(false, reason) + this.isMaintaining = false + } + // ---------- schedule config ---------- - getSchedule() { - let cfg = { ...DEFAULT_SCHEDULE, ...(this._settings().get(SCHEDULE_KEY, {}) || {}) } + async getSchedule() { + let cfg = { ...DEFAULT_SCHEDULE, ...((await this._settings().get(SCHEDULE_KEY, {})) || {}) } if (process.env.KIN_BACKUP_DISABLED === '1') cfg.enabled = false if (process.env.KIN_BACKUP_INTERVAL_HOURS) { const n = Number(process.env.KIN_BACKUP_INTERVAL_HOURS) @@ -114,8 +97,8 @@ export class BackupService { return cfg } - updateSchedule(patch = {}) { - const cur = { ...DEFAULT_SCHEDULE, ...(this._settings().get(SCHEDULE_KEY, {}) || {}) } + async updateSchedule(patch = {}) { + const cur = { ...DEFAULT_SCHEDULE, ...((await this._settings().get(SCHEDULE_KEY, {})) || {}) } if (patch.enabled != null) cur.enabled = !!patch.enabled if (patch.interval_hours != null) { const n = Number(patch.interval_hours) @@ -131,99 +114,149 @@ export class BackupService { } cur.retention = n } - this._settings().set(SCHEDULE_KEY, cur) + await this._settings().set(SCHEDULE_KEY, cur) return this.getSchedule() } - nextAutoAt() { - const cfg = this.getSchedule() + async nextAutoAt() { + const cfg = await this.getSchedule() if (!cfg.enabled) return null - const last = this._repo().lastSuccessful({}) + const last = await this._repo().lastSuccessful({}) const base = last ? Date.parse(last.created_at) : 0 return new Date(Math.max(Date.now(), base + cfg.interval_hours * 3600_000)).toISOString() } // ---------- create ---------- - /** - * Run one backup off the main thread. - * - * The pipeline (VACUUM INTO + sha256 + tar.gz) is synchronous for minutes on a - * multi-GB database, so it runs in a worker with its own DB connection while - * the gateway keeps serving requests. - * - * @returns {Promise} - */ - async createBackup({ kind = 'manual', note = null } = {}) { + async flush() { + await Promise.all([...this._scheduledChecks, this._pendingBackup, this._pendingRestore].filter(Boolean)) + } + + async createBackup(options = {}) { + if (this._restoreRunning && options.kind !== 'pre_restore') + throw Object.assign(new Error('restore already in progress'), { code: 'restore_in_progress' }) + if (this._running) throw Object.assign(new Error('backup already in progress'), { code: 'backup_in_progress' }) + const pending = this._createBackup(options) + this._pendingBackup = pending + try { + return await pending + } finally { + if (this._pendingBackup === pending) this._pendingBackup = null + } + } + + /** Export a PostgreSQL snapshot while files are frozen, then dump in a worker. */ + async _createBackup({ kind = 'manual', note = null } = {}) { if (this._running) throw Object.assign(new Error('backup already in progress'), { code: 'backup_in_progress' }) this._running = true const id = 'bak_' + crypto.randomBytes(6).toString('hex') const createdAt = new Date().toISOString() + let client + let entered = false try { - return await this._runBackupWorker({ - id, - createdAt, - kind, - note, - retention: this.getSchedule().retention, - }) - } catch (e) { - // The worker records its own failure; this covers the cases where it never - // got that far (module error, OOM, kill) so the panel still shows the job. + entered = await this._enterMaintenance(kind === 'pre_restore' ? 'restore' : 'backup') + const db = getDb() + client = await db.pool.connect() + await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY') + const snapshot = (await client.query('SELECT pg_export_snapshot() AS snapshot')).rows[0].snapshot + const version = (await client.query('SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1')) + .rows[0] + const rec = await this._runBackupWorker( + { id, createdAt, kind, note }, + { + connectionString: db.connectionString, + schema: db.schema, + snapshot, + schemaVersion: version?.version || null, + }, + async () => { + if (entered) { + await this._leaveMaintenance('backup') + entered = false + } + }, + ) + await client.query('COMMIT') + client.release() + client = null + const stored = await this._repo().insert(rec) + const retention = (await this.getSchedule()).retention + for (const old of [ + ...(await this._repo().beyondRetention(retention)), + ...(await this._repo().preRestoreBeyond(3)), + ]) { + if (old.file_path) fs.rmSync(old.file_path, { force: true }) + await this._repo().remove(old.id) + } + return stored + } catch (error) { try { - if (!this._repo().get(id)) { - this._repo().insert({ + if (!(await this._repo().get(id))) + await this._repo().insert({ id, created_at: createdAt, kind, status: 'failed', - file_path: null, - file_name: null, - size_bytes: null, - sha256: null, - db_bytes: null, - includes: null, - error: String(e.message || e).slice(0, 500), + error: String(error.message || error).slice(0, 500), note, }) - } } catch {} - throw e + throw error } finally { + if (client) { + try { + await client.query('ROLLBACK') + } finally { + client.release() + } + } this._running = false + if (entered) await this._leaveMaintenance('backup') } } - _runBackupWorker(job) { + _runBackupWorker(job, database, onFilesCopied) { return new Promise((resolve, reject) => { const worker = new Worker(new URL('./backup-worker.mjs', import.meta.url), { workerData: { - dataDir: this.dataDir, ctx: { projectRoot: this.projectRoot, configDir: this.configDir, backupDir: this.backupDir, tmpDir: this.tmpDir, + ...database, }, job, }, }) let settled = false + let resumed = Promise.resolve() const finish = (fn, arg) => { if (settled) return settled = true fn(arg) } - worker.once('message', (msg) => { - if (msg && msg.ok) return finish(resolve, msg.rec) - finish( - reject, - Object.assign(new Error((msg && msg.error && msg.error.message) || 'backup failed'), { - code: (msg && msg.error && msg.error.code) || 'backup_failed', - }), - ) + worker.on('message', (msg) => { + if (msg.type === 'files-copied') { + resumed = onFilesCopied() + // Keep the exported snapshot alive until the worker finishes even when resume failed. + resumed.catch(() => {}) + return + } + if (msg.ok) + resumed.then( + () => finish(resolve, msg.rec), + (error) => finish(reject, error), + ) + else + finish( + reject, + Object.assign(new Error(msg.error?.message || 'backup failed'), { + code: msg.error?.code || 'backup_failed', + }), + ) }) - worker.once('error', (e) => finish(reject, e)) + worker.once('error', (error) => finish(reject, error)) worker.once('exit', (code) => { if (code !== 0) finish(reject, new Error(`backup worker exited with code ${code}`)) }) @@ -232,21 +265,19 @@ export class BackupService { // ---------- list / delete ---------- - list({ limit = 100 } = {}) { - return this._repo() - .list({ limit }) - .map((r) => ({ - ...r, - file_exists: !!(r.file_path && fs.existsSync(r.file_path)), - })) + async list({ limit = 100 } = {}) { + return (await this._repo().list({ limit })).map((r) => ({ + ...r, + file_exists: !!(r.file_path && fs.existsSync(r.file_path)), + })) } - get(id) { + async get(id) { return this._repo().get(id) } - remove(id) { - const rec = this._repo().get(id) + async remove(id) { + const rec = await this._repo().get(id) if (!rec) return false try { if (rec.file_path && fs.existsSync(rec.file_path)) fs.rmSync(rec.file_path, { force: true }) @@ -256,139 +287,208 @@ export class BackupService { // ---------- restore ---------- - /** - * Restore from a recorded backup. Automatically snapshots the current - * state first (kind=pre_restore). Re-binds stores via onRestored callback. - */ - async restoreBackup(id) { - const rec = this._repo().get(id) - if (!rec) throw Object.assign(new Error('backup not found'), { code: 'backup_not_found' }) - if (rec.status !== 'ok') - throw Object.assign(new Error('backup record is not restorable'), { code: 'backup_not_ok' }) - if (!rec.file_path || !fs.existsSync(rec.file_path)) { - throw Object.assign(new Error('backup file missing on disk'), { code: 'backup_file_missing' }) - } - if (this.isRestoring) throw Object.assign(new Error('restore already in progress'), { code: 'restore_in_progress' }) - - // integrity: archive checksum must match the record - const actual = sha256File(rec.file_path) - if (rec.sha256 && actual !== rec.sha256) { - throw Object.assign(new Error('backup archive sha256 mismatch'), { code: 'backup_corrupt' }) - } - - this.isRestoring = true - let tmp = null + async _extractArchive(rec) { + if (!rec.file_name?.endsWith('.tar.gz')) + throw Object.assign(new Error('convert the SQLite backup with the offline import tool before restoring'), { + code: 'legacy_backup_conversion_required', + }) + const tmp = this._mkdtemp('kin-restore-') try { - // 0. safety net snapshot of the current state - const pre = await this.createBackup({ kind: 'pre_restore', note: `before restore of ${rec.id}` }) - - // 1. extract / decompress - tmp = this._mkdtemp('kin-restore-') - let dbFile - let manifest = null - if (rec.file_name?.endsWith('.tar.gz')) { - const r = spawnSync('tar', ['-xzf', rec.file_path, '-C', tmp], { stdio: 'pipe' }) - if (r.status !== 0) throw new Error(`tar extract failed: ${String(r.stderr || '').slice(0, 300)}`) - dbFile = path.join(tmp, 'db', 'kin.db') - try { - manifest = JSON.parse(fs.readFileSync(path.join(tmp, 'manifest.json'), 'utf8')) - } catch {} - if (manifest?.sha256s?.['db/kin.db']) { - if (sha256File(dbFile) !== manifest.sha256s['db/kin.db']) { - throw Object.assign(new Error('db snapshot sha256 mismatch inside archive'), { code: 'backup_corrupt' }) - } + const entries = (await runProcess('tar', ['-tzf', rec.file_path])).split('\n').filter(Boolean) + const files = new Set() + const members = new Set() + for (const entry of entries) { + const name = entry.replace(/^\.\//, '') + if (!name || name === '.') continue + if (members.has(name)) throw new Error('invalid or duplicate backup archive path') + members.add(name) + if (name === 'db/kin.db') + throw Object.assign(new Error('convert the SQLite backup with the offline import tool before restoring'), { + code: 'legacy_backup_conversion_required', + }) + if (/^(db|vms|config)\/$/.test(name)) { + fs.mkdirSync(path.join(tmp, name), { recursive: true, mode: 0o700 }) + continue } - } else { - dbFile = path.join(tmp, 'kin.db') - fs.writeFileSync(dbFile, zlib.gunzipSync(fs.readFileSync(rec.file_path))) - } - if (!fs.existsSync(dbFile)) throw new Error('no db snapshot inside backup') - - // 2. swap the live database - const dbPath = getDbPath() - closeDatabase() - const keep = `${dbPath}.pre-restore` - try { - fs.rmSync(keep, { force: true }) - } catch {} - try { - fs.renameSync(dbPath, keep) - } catch {} - for (const suffix of ['-wal', '-shm']) { - try { - fs.rmSync(dbPath + suffix, { force: true }) - } catch {} + if (!/^(manifest\.json|db\/postgres\.dump|(vms|config)\/[^/]+\.json)$/.test(name) || files.has(name)) { + throw new Error('invalid or duplicate backup archive path') + } + files.add(name) + await extractArchiveMember(rec.file_path, entry, path.join(tmp, name)) } - fs.copyFileSync(dbFile, dbPath) - try { - fs.chmodSync(dbPath, 0o600) - } catch {} - - // 3. restore vm + config files from the archive (tar flavor only) - const vmsDirSrc = path.join(tmp, 'vms') - if (this.projectRoot && fs.existsSync(vmsDirSrc)) { - const vmsDir = path.join(this.projectRoot, 'vms') - fs.mkdirSync(vmsDir, { recursive: true }) - // replace vm json records with archive state (cli-home dirs untouched) - for (const f of fs.readdirSync(vmsDir)) { - if (isVmRecordFile(f) || f === 'active.json') { - try { - fs.rmSync(path.join(vmsDir, f), { force: true }) - } catch {} - } + const manifest = JSON.parse(fs.readFileSync(path.join(tmp, 'manifest.json'), 'utf8')) + if (manifest.version !== 2 || manifest.engine !== 'postgresql') + throw Object.assign(new Error('convert the SQLite backup with the offline import tool before restoring'), { + code: 'legacy_backup_conversion_required', + }) + if (manifest.schema !== getDb().schema) throw new Error('backup schema does not match this application') + if (!manifest.sha256s?.['db/postgres.dump']) throw new Error('backup database checksum is missing') + for (const [directory, required] of [ + ['vms', !!this.projectRoot], + ['config', !!this.configDir], + ]) { + const count = [...files].filter((name) => name.startsWith(`${directory}/`)).length + if (manifest.includes?.[directory] !== count || (required && !fs.existsSync(path.join(tmp, directory)))) { + throw new Error('backup file snapshot is incomplete') } - for (const f of fs.readdirSync(vmsDirSrc)) { - fs.copyFileSync(path.join(vmsDirSrc, f), path.join(vmsDir, f)) - try { - fs.chmodSync(path.join(vmsDir, f), 0o600) - } catch {} + } + for (const name of files) { + if (name !== 'manifest.json' && !manifest.sha256s[name]) throw new Error('backup contains an unchecked file') + } + for (const [name, digest] of Object.entries(manifest.sha256s)) { + if ( + path.isAbsolute(name) || + name.split('/').includes('..') || + (await sha256FileAsync(path.join(tmp, name))) !== digest + ) { + throw Object.assign(new Error('backup file checksum mismatch'), { code: 'backup_corrupt' }) } } - const cfgDirSrc = path.join(tmp, 'config') - if (this.configDir && fs.existsSync(cfgDirSrc)) { - fs.mkdirSync(this.configDir, { recursive: true }) - for (const f of fs.readdirSync(cfgDirSrc)) { - fs.copyFileSync(path.join(cfgDirSrc, f), path.join(this.configDir, f)) + const dump = path.join(tmp, 'db', 'postgres.dump') + const listing = await runPgTool('pg_restore', ['--list', dump], getDb().connectionString) + const schema = getDb().schema + const allowed = + /^(TABLE DATA|SEQUENCE OWNED BY|SEQUENCE SET|FK CONSTRAINT|TABLE|SEQUENCE|DEFAULT|CONSTRAINT|INDEX|SCHEMA|COMMENT) (\S+) (.+)$/ + for (const line of listing.split('\n')) { + if (!line || line.startsWith(';')) continue + const entry = line.replace(/^\d+; \d+ \d+ /, '') + if (/^(ENCODING - ENCODING|STDSTRINGS - STDSTRINGS|SEARCHPATH - SEARCHPATH)( |$)/.test(entry)) continue + if (entry.startsWith(`COMMENT - SCHEMA ${schema} `)) continue + if (entry.startsWith(`FUNCTION ${schema} ascii_fold(text) `)) continue + const item = entry.match(allowed) + if ( + !item || + (item[1] === 'SCHEMA' ? item[2] !== '-' || !item[3].startsWith(`${schema} `) : item[2] !== schema) + ) { + throw new Error('backup contains objects outside the application schema allowlist') } } + return { tmp, dump, manifest } + } catch (error) { + fs.rmSync(tmp, { recursive: true, force: true }) + throw error + } + } - // 4. reopen + migrate (old backups upgrade automatically) - const db = openDatabase({ dataDir: this.dataDir, dbPath }) - - // 5. db-only backups: rebuild vm files from the credential mirror - if (this.projectRoot) { - invalidateReadCache() - try { - reconcileVms(this.projectRoot) - try { - repairVmProxyCliEnabled(this.projectRoot) - } catch {} - try { - repairSlotSmbios(this.projectRoot) - } catch {} - } catch {} + _restoreFiles(tmp) { + if (this.projectRoot && fs.existsSync(path.join(tmp, 'vms'))) { + const target = path.join(this.projectRoot, 'vms') + fs.mkdirSync(target, { recursive: true }) + for (const name of fs.readdirSync(target)) { + if (name.endsWith('.json') && !fs.lstatSync(path.join(target, name)).isDirectory()) + fs.rmSync(path.join(target, name), { force: true }) } + for (const name of fs.readdirSync(path.join(tmp, 'vms'))) { + fs.copyFileSync(path.join(tmp, 'vms', name), path.join(target, name)) + fs.chmodSync(path.join(target, name), 0o600) + } + } + if (this.configDir && fs.existsSync(path.join(tmp, 'config'))) { + fs.mkdirSync(this.configDir, { recursive: true }) + for (const name of fs.readdirSync(this.configDir)) { + if (name.endsWith('.json') && !fs.lstatSync(path.join(this.configDir, name)).isDirectory()) + fs.rmSync(path.join(this.configDir, name), { force: true }) + } + for (const name of fs.readdirSync(path.join(tmp, 'config'))) { + fs.copyFileSync(path.join(tmp, 'config', name), path.join(this.configDir, name)) + fs.chmodSync(path.join(this.configDir, name), 0o600) + } + } + invalidateReadCache() + } - // 6. the ledger itself rolled back with the DB — re-register the - // pre_restore snapshot (and any other on-disk archives) so the - // user can still see/undo from the panel. - try { - this._registerOrphanArchives([pre, rec]) - } catch {} - - // 7. re-bind stores to the fresh connection. A failed rebind leaves the - // process partially attached to the closed database, so it must make - // the restore fail instead of reporting a false success. - if (this._rebindStores) this._rebindStores(db) + async _applyArchive(archive) { + await runPgTool( + 'pg_restore', + [ + '--clean', + '--if-exists', + '--single-transaction', + '--no-owner', + '--no-privileges', + `--schema=${getDb().schema}`, + '--dbname', + decodeURIComponent(new URL(getDb().connectionString).pathname.slice(1)), + archive.dump, + ], + getDb().connectionString, + ) + this._restoreFiles(archive.tmp) + await applyMigrations(getDb()) + } - return { ok: true, restored: rec.id, pre_restore: pre.id } + async restoreBackup(id) { + if (this._restoreRunning || this._running) + throw Object.assign(new Error('restore already in progress'), { code: 'restore_in_progress' }) + const pending = this._restoreBackup(id) + this._pendingRestore = pending + try { + return await pending } finally { + if (this._pendingRestore === pending) this._pendingRestore = null + } + } + + /** Failure keeps maintenance enabled. The restore endpoint may explicitly retry. */ + async _restoreBackup(id) { + if (this._restoreRunning || this._running) + throw Object.assign(new Error('restore already in progress'), { code: 'restore_in_progress' }) + this._restoreRunning = true + let archive + let previous + let pre + let changed = false + let maintenanceStarted = false + try { + const rec = await this._repo().get(id) + if (!rec) throw Object.assign(new Error('backup not found'), { code: 'backup_not_found' }) + if (rec.status !== 'ok') throw Object.assign(new Error('backup is not restorable'), { code: 'backup_not_ok' }) + if (!rec.file_path || !fs.existsSync(rec.file_path)) + throw Object.assign(new Error('backup file missing'), { code: 'backup_file_missing' }) + if (rec.sha256 && (await sha256FileAsync(rec.file_path)) !== rec.sha256) + throw Object.assign(new Error('backup archive sha256 mismatch'), { code: 'backup_corrupt' }) + archive = await this._extractArchive(rec) + fs.mkdirSync(this.dataDir, { recursive: true, mode: 0o700 }) + fs.writeFileSync(this.maintenanceFile, JSON.stringify({ backup_id: id, started_at: new Date().toISOString() }), { + mode: 0o600, + }) + maintenanceStarted = true + this.isRestoring = true + await this._enterMaintenance('restore') + pre = await this.createBackup({ kind: 'pre_restore', note: `before restore of ${id}` }) + previous = await this._extractArchive(pre) + changed = true + await getDb().withMaintenance(async () => { + await this._applyArchive(archive) + await this._registerOrphanArchives([pre, rec]) + await this._rebindStores?.(getDb()) + }) + await this._leaveMaintenance('restore') + fs.rmSync(this.maintenanceFile) this.isRestoring = false - if (tmp) { + return { ok: true, restored: id, pre_restore: pre.id } + } catch (error) { + if (maintenanceStarted) { + this.isRestoring = true + this.isMaintaining = true + } + if (changed && previous) { try { - fs.rmSync(tmp, { recursive: true, force: true }) - } catch {} + await getDb().withMaintenance(async () => { + await this._applyArchive(previous) + await this._registerOrphanArchives([pre]) + await this._rebindStores?.(getDb()) + }) + } catch { + throw Object.assign(new Error('restore and rollback failed; maintenance remains enabled'), { + code: 'restore_rollback_failed', + }) + } } + throw error + } finally { + this._restoreRunning = false + for (const item of [archive, previous]) if (item) fs.rmSync(item.tmp, { recursive: true, force: true }) } } @@ -397,15 +497,13 @@ export class BackupService { * record in the (freshly restored) DB. `known` records are inserted * verbatim; remaining orphans get kind='imported'. */ - _registerOrphanArchives(known = []) { + async _registerOrphanArchives(known = []) { const repo = this._repo() - const existing = new Set(repo.list({ limit: 500 }).map((r) => r.file_name)) + const existing = new Set((await repo.list({ limit: 500 })).map((r) => r.file_name)) for (const rec of known) { if (rec?.file_name && !existing.has(rec.file_name) && rec.file_path && fs.existsSync(rec.file_path)) { - try { - repo.insert(rec) - existing.add(rec.file_name) - } catch {} + await repo.insert(rec) + existing.add(rec.file_name) } } if (!fs.existsSync(this.backupDir)) return @@ -414,7 +512,7 @@ export class BackupService { const fp = path.join(this.backupDir, f) try { const st = fs.statSync(fp) - repo.insert({ + await repo.insert({ id: 'bak_' + crypto.randomBytes(6).toString('hex'), created_at: st.mtime.toISOString(), kind: 'imported', @@ -422,7 +520,7 @@ export class BackupService { file_path: fp, file_name: f, size_bytes: st.size, - sha256: sha256File(fp), + sha256: await sha256FileAsync(fp), db_bytes: null, includes: null, error: null, @@ -438,15 +536,22 @@ export class BackupService { /** Auto-backup checker: every checkIntervalMs + a catch-up run at boot. */ startScheduler() { this.stopScheduler() - const check = async () => { - try { - if (this.shouldRunScheduled()) { - const rec = await this.createBackup({ kind: 'scheduled' }) - console.log(`[backup] scheduled backup completed (${Math.round((rec.size_bytes || 0) / 1048576)} MiB)`) + const generation = this._schedulerGeneration + const check = () => { + if (generation !== this._schedulerGeneration) return + const pending = (async () => { + try { + if ((await this.shouldRunScheduled()) && generation === this._schedulerGeneration) { + const rec = await this.createBackup({ kind: 'scheduled' }) + console.log(`[backup] scheduled backup completed (${Math.round((rec.size_bytes || 0) / 1048576)} MiB)`) + } + } catch (e) { + console.warn('[backup] scheduled backup failed:', String(e.message || e)) } - } catch (e) { - console.warn('[backup] scheduled backup failed:', String(e.message || e)) - } + })() + this._scheduledChecks.add(pending) + pending.finally(() => this._scheduledChecks.delete(pending)) + return pending } this._timer = setInterval(check, this.checkIntervalMs) if (typeof this._timer.unref === 'function') this._timer.unref() @@ -455,15 +560,16 @@ export class BackupService { if (typeof this._bootTimer.unref === 'function') this._bootTimer.unref() } - shouldRunScheduled(now = Date.now()) { - const cfg = this.getSchedule() - if (!cfg.enabled || this._running || this.isRestoring) return false - const last = this._repo().lastSuccessful({}) + async shouldRunScheduled(now = Date.now()) { + const cfg = await this.getSchedule() + if (!cfg.enabled || this._running || this._restoreRunning || this.isMaintaining) return false + const last = await this._repo().lastSuccessful({}) if (!last) return true return now - Date.parse(last.created_at) >= cfg.interval_hours * 3600_000 } stopScheduler() { + this._schedulerGeneration += 1 if (this._timer) { clearInterval(this._timer) this._timer = null diff --git a/src/lib/admin/backup-worker.mjs b/src/lib/admin/backup-worker.mjs index 7b5f5a12f..12e1f8c56 100644 --- a/src/lib/admin/backup-worker.mjs +++ b/src/lib/admin/backup-worker.mjs @@ -1,32 +1,15 @@ -/** - * Worker entry for the backup pipeline. - * - * The pipeline (VACUUM INTO + sha256 + tar.gz) is fully synchronous, so it must - * not run on the gateway's main thread: that is what made auto-backups freeze - * the whole gateway (HTTP/panel/healthz all stop answering → Cloudflare 524) - * for the duration of the backup. - * - * This worker opens its own SQLite connection (WAL allows the live gateway to - * keep serving while the snapshot is taken) and posts the finished record back - * to BackupService. Failures are reported as a message, never thrown across the - * thread boundary. - */ - +/** PostgreSQL archive worker; the parent owns the exported snapshot transaction. */ import { parentPort, workerData } from 'node:worker_threads' -import { openDatabase, closeDatabase } from '../db/database.mjs' import { runBackupJob } from './backup-core.mjs' try { - openDatabase({ dataDir: workerData.dataDir }) - const rec = runBackupJob(workerData.ctx, workerData.job) + const rec = await runBackupJob(workerData.ctx, workerData.job, () => { + parentPort.postMessage({ type: 'files-copied' }) + }) parentPort.postMessage({ ok: true, rec }) -} catch (e) { +} catch (error) { parentPort.postMessage({ ok: false, - error: { message: String((e && e.message) || e), code: (e && e.code) || null }, + error: { message: String(error.message || error), code: error.code || 'backup_failed' }, }) -} finally { - try { - closeDatabase() - } catch {} } diff --git a/src/lib/admin/catalog-monitor.mjs b/src/lib/admin/catalog-monitor.mjs index 3437d409c..62ee4b892 100644 --- a/src/lib/admin/catalog-monitor.mjs +++ b/src/lib/admin/catalog-monitor.mjs @@ -6,7 +6,7 @@ * Results land in settings `model_catalog_sync` so the models page can show them. */ import path from 'node:path' -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { runSlotOauth } from '../transport/slot-oauth.mjs' import { getVm, listVms, vmHasClaudeCredential } from '../vm/vm-registry.mjs' import { isCodexVm } from '../vm/vm-kind.mjs' @@ -32,10 +32,10 @@ export function readCatalogSync() { } } -function writeCatalogSync(platform, status) { +async function writeCatalogSync(platform, status) { const repo = new SettingsRepo() const cur = repo.get(CATALOG_SYNC_KEY, null) || { claude: null, codex: null } - repo.set(CATALOG_SYNC_KEY, { ...cur, [platform]: status }) + await repo.set(CATALOG_SYNC_KEY, { ...cur, [platform]: status }) return status } @@ -54,7 +54,7 @@ function claudeCatalogTargets(projectRoot) { /** Live Claude catalog → model_policy. */ export async function syncClaudeCatalog({ projectRoot, runOauth = runSlotOauth } = {}) { const targets = claudeCatalogTargets(projectRoot) - if (!targets.length) return writeCatalogSync('claude', syncStatus('skipped', { error: 'no_claude_slot' })) + if (!targets.length) return await writeCatalogSync('claude', syncStatus('skipped', { error: 'no_claude_slot' })) const ids = new Set() let successes = 0 let lastError = null @@ -70,12 +70,12 @@ export async function syncClaudeCatalog({ projectRoot, runOauth = runSlotOauth } successes++ for (const row of list) if (row?.id) ids.add(String(row.id)) } - if (!successes) return writeCatalogSync('claude', syncStatus('error', { error: lastError || 'models_failed' })) - const { added } = syncWorkerModelsIntoPolicy([...ids], { + if (!successes) return await writeCatalogSync('claude', syncStatus('error', { error: lastError || 'models_failed' })) + const { added } = await syncWorkerModelsIntoPolicy([...ids], { displayNameOf: (id) => modelsDevMeta('anthropic', id)?.name, }) seedModelCatalog() - return writeCatalogSync('claude', syncStatus('ok', { added })) + return await writeCatalogSync('claude', syncStatus('ok', { added })) } /** Live ChatGPT Codex catalog → gpt_model_policy. Timers never rotate OAuth; the panel button may. */ @@ -83,10 +83,10 @@ export async function syncGptCatalog({ projectRoot, vmId = '', catalogClientVers const result = await syncCodexCatalog({ projectRoot, vmId, rotate, catalogClientVersion }) if (!result.ok) { const skipped = result.error === 'no_codex_slot' || result.error === 'proxy_required' - writeCatalogSync('codex', syncStatus(skipped ? 'skipped' : 'error', { error: result.error })) + await writeCatalogSync('codex', syncStatus(skipped ? 'skipped' : 'error', { error: result.error })) return result } - writeCatalogSync('codex', syncStatus('ok', { added: result.added || [] })) + await writeCatalogSync('codex', syncStatus('ok', { added: result.added || [] })) return result } @@ -96,15 +96,21 @@ export async function syncGptCatalog({ projectRoot, vmId = '', catalogClientVers export function createCatalogMonitor({ projectRoot, codexCatalogVersion = () => 'auto', log = () => {} } = {}) { const timers = [] let running = false + const pending = new Set() const guard = (name, fn) => async () => { + if (!running) return + const operation = Promise.resolve().then(fn) + pending.add(operation) try { - const out = await fn() + const out = await operation log({ event: 'catalog-sync', job: name, status: out?.status ?? (out?.ok === false ? 'error' : 'ok') }) } catch (error) { log({ event: 'catalog-sync', job: name, status: 'error', error: String(error?.message || error) }) + } finally { + pending.delete(operation) } } - const modelsDev = guard('models.dev', () => syncModelsDev()) + const modelsDev = guard('models.dev', async () => await syncModelsDev()) const upstream = guard('upstream', async () => { const claude = await syncClaudeCatalog({ projectRoot }) await syncGptCatalog({ projectRoot, catalogClientVersion: codexCatalogVersion() }) @@ -129,6 +135,9 @@ export function createCatalogMonitor({ projectRoot, codexCatalogVersion = () => timers.push(t) } }, + async flush() { + while (pending.size) await Promise.allSettled([...pending]) + }, stop() { running = false for (const t of timers.splice(0)) { diff --git a/src/lib/admin/error-class.mjs b/src/lib/admin/error-class.mjs index dfef4267f..f74f56157 100644 --- a/src/lib/admin/error-class.mjs +++ b/src/lib/admin/error-class.mjs @@ -161,11 +161,11 @@ export function ingressAuthSql() { const codes = [...INGRESS_AUTH_ERROR_CODES].map((code) => `'${code}'`).join(', ') const cred = "'upstream_auth_error','oauth_need_reimport','oauth_refresh_failed'" return `( - IFNULL(error_code, '') IN (${codes}) + COALESCE(error_code, '') IN (${codes}) OR ( status IN (401, 403) - AND IFNULL(vm_id, '') = '' - AND IFNULL(error_code, '') NOT IN (${cred}) + AND COALESCE(vm_id, '') = '' + AND COALESCE(error_code, '') NOT IN (${cred}) ) )` } @@ -180,7 +180,7 @@ function mutedAuthSql() { .filter((code) => CODE_MAP[code] !== 'auth') .map((code) => `'${code}'`) .join(',') - return `(${ingressAuthSql()} AND IFNULL(error_code, '') NOT IN (${classified}))` + return `(${ingressAuthSql()} AND COALESCE(error_code, '') NOT IN (${classified}))` } export function excludeErrorClassSql(classes) { diff --git a/src/lib/admin/health-probe.mjs b/src/lib/admin/health-probe.mjs index efb810665..267417cd4 100644 --- a/src/lib/admin/health-probe.mjs +++ b/src/lib/admin/health-probe.mjs @@ -819,6 +819,9 @@ export function createHealthProbeMonitor(opts = {}) { decide, runRealProbe, start, + flush: async () => { + while (inflight) await inflight + }, stop, } } diff --git a/src/lib/admin/key-scope.mjs b/src/lib/admin/key-scope.mjs index 660124722..e83a8b831 100644 --- a/src/lib/admin/key-scope.mjs +++ b/src/lib/admin/key-scope.mjs @@ -79,15 +79,15 @@ export function keyScopeFromRecord(rec) { } } -function defaultPoolLookup(id) { +async function defaultPoolLookup(id) { if (!isDbOpen()) return null - return new VmPoolsRepo(getDb()).get(id) + return await new VmPoolsRepo(getDb()).get(id) } /** Live membership. A missing, disabled, or empty pool never widens to the global set. */ -export function applyVmPool(scope, lookup = defaultPoolLookup) { +export async function applyVmPool(scope, lookup = defaultPoolLookup) { if (!scope?.vm_pool_id) return scope - const pool = lookup?.(scope.vm_pool_id) || null + const pool = (await lookup?.(scope.vm_pool_id)) || null if (!pool) return { ...scope, allowed_vms: [], vm_pool_error: 'missing' } if (!pool.enabled) return { ...scope, allowed_vms: [], vm_pool_error: 'disabled' } const ids = Array.isArray(pool.vm_ids) ? pool.vm_ids.map((id) => String(id || '').trim()).filter(Boolean) : [] @@ -95,9 +95,9 @@ export function applyVmPool(scope, lookup = defaultPoolLookup) { return { ...scope, allowed_vms: ids, vm_pool_error: null } } -export function keyScopeFromRequest(req, lookup = defaultPoolLookup) { +export async function keyScopeFromRequest(req, lookup = defaultPoolLookup) { if (!req || req.apiKeyKind === 'master') return { group_type: 'all', allowed_vms: [], vm_pool_id: null } - return applyVmPool(keyScopeFromRecord(req.apiKeyRecord), lookup) + return await applyVmPool(keyScopeFromRecord(req.apiKeyRecord), lookup) } export function vmPoolDenial(scope) { diff --git a/src/lib/admin/notify.mjs b/src/lib/admin/notify.mjs index d59c51f4c..6acccbab1 100644 --- a/src/lib/admin/notify.mjs +++ b/src/lib/admin/notify.mjs @@ -958,6 +958,9 @@ export function createNotifyMonitor(opts = {}) { getStatus, runOnce, start, + flush: async () => { + while (inflight) await inflight + }, stop, } } diff --git a/src/lib/admin/onboarding.mjs b/src/lib/admin/onboarding.mjs index d8167854b..b9e441463 100644 --- a/src/lib/admin/onboarding.mjs +++ b/src/lib/admin/onboarding.mjs @@ -113,14 +113,14 @@ export function buildOnboardingPayload({ } } -export function loadOnboardingState(settings) { - const raw = settings?.get?.(ONBOARDING_STATE_KEY, {}) +export async function loadOnboardingState(settings) { + const raw = await settings?.get?.(ONBOARDING_STATE_KEY, {}) if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return {} return raw } -export function saveOnboardingState(settings, map) { - settings.set(ONBOARDING_STATE_KEY, map && typeof map === 'object' ? map : {}) +export async function saveOnboardingState(settings, map) { + await settings.set(ONBOARDING_STATE_KEY, map && typeof map === 'object' ? map : {}) return map } diff --git a/src/lib/admin/panel-api.mjs b/src/lib/admin/panel-api.mjs index bd406181c..5e88752ce 100644 --- a/src/lib/admin/panel-api.mjs +++ b/src/lib/admin/panel-api.mjs @@ -628,9 +628,9 @@ export async function buildDashboard({ const billing = stampVmBilling( vms, - (() => { + await (async () => { try { - return attachBillingMeta(panelBillingStats(requestLog, accountQuota, vms), snap.accounts || []) + return attachBillingMeta(await panelBillingStats(requestLog, accountQuota, vms), snap.accounts || []) } catch { return null } @@ -686,18 +686,18 @@ export async function buildDashboard({ }, gateway_stats: stats, // historic totals straight from the request_logs table (survives restarts) - db_totals: (() => { + db_totals: await (async () => { try { - return requestLog?.totals() || null + return (await requestLog?.totals()) || null } catch { return null } })(), billing, // last-hour ops cards (SLA / QPS / TTFT) — console can refetch other windows via /request-logs/stats - ops: (() => { + ops: await (async () => { try { - return requestLog?.windowStats?.({ since: new Date(Date.now() - 3600_000).toISOString() }) || null + return (await requestLog?.windowStats?.({ since: new Date(Date.now() - 3600_000).toISOString() })) || null } catch { return null } @@ -744,9 +744,9 @@ export async function buildVmList({ })() stampVmBilling( vms, - (() => { + await (async () => { try { - return attachBillingMeta(panelBillingStats(requestLog, accountQuota, vms), accounts) + return attachBillingMeta(await panelBillingStats(requestLog, accountQuota, vms), accounts) } catch { return null } @@ -803,9 +803,9 @@ export async function buildVmDetail({ role, }) const acc = findAccount(accountQuota, summary) - const billing = (() => { + const billing = await (async () => { try { - return panelBillingStats(requestLog, accountQuota, [summary]) + return await panelBillingStats(requestLog, accountQuota, [summary]) } catch { return null } @@ -813,7 +813,7 @@ export async function buildVmDetail({ const cost = lookupBilling(indexBillingAccounts(billing), acc || summary) const gpt = isCodexVm(vm) applyCostFields(summary, cost, { coveredSince: billing?.log_coverage_start }) - const detail = buildAccountBilling(cost, billing, { + const detail = await buildAccountBilling(cost, billing, { vmId: summary.id, accountId: acc?.account_id || summary.account_uuid, requestLog, @@ -1146,7 +1146,7 @@ export async function buildProbeOne({ probed_at: new Date().toISOString(), ...(headerProbe ? { ...headerProbe, error: null } : {}), } - : await cache.load(accountId, () => probeAccount({ exec, vm, includeFable }), { force: !!force }) + : await cache.load(accountId, async () => await probeAccount({ exec, vm, includeFable }), { force: !!force }) if (fatalUsageCredentialFailure(result)) { try { markVmRefreshError(path.join(cfg.paths.project, 'vms', `${id}.json`), { @@ -1306,12 +1306,12 @@ export async function buildProbeAll({ cfg, accountQuota, hop = true, force = tru return ok({ items, total: items.length }) } -export function buildUsage({ accountQuota, cfg, requestLog = null }) { +export async function buildUsage({ accountQuota, cfg, requestLog = null }) { const snap = accountQuota.snapshot() const listed = listVms(cfg?.paths?.project) - const billing = (() => { + const billing = await (async () => { try { - return panelBillingStats(requestLog, accountQuota, listed) + return await panelBillingStats(requestLog, accountQuota, listed) } catch { return null } @@ -1488,9 +1488,9 @@ export async function snapshotAccountPool({ })() const billing = stampVmBilling( vms, - (() => { + await (async () => { try { - return attachBillingMeta(panelBillingStats(requestLog, accountQuota, vms), accounts) + return attachBillingMeta(await panelBillingStats(requestLog, accountQuota, vms), accounts) } catch { return null } @@ -1498,7 +1498,7 @@ export async function snapshotAccountPool({ ) let today = null try { - today = requestLog?.windowStats?.({ since: shanghaiDayStartIso() }) || null + today = (await requestLog?.windowStats?.({ since: shanghaiDayStartIso() })) || null } catch { today = null } @@ -2318,9 +2318,9 @@ function billingWindowOf(accountId, vmId, vm, q) { } } -function panelBillingStats(requestLog, accountQuota, vms) { +async function panelBillingStats(requestLog, accountQuota, vms) { if (!requestLog?.billingStats) return null - return requestLog.billingStats({ accountWindows: accountWindowsForBilling(accountQuota, vms) }) + return await requestLog.billingStats({ accountWindows: accountWindowsForBilling(accountQuota, vms) }) } function stampVmBilling(vms, billing) { @@ -2431,7 +2431,7 @@ function windowEstimates(quota, cost, coveredSince, codex) { } } -function buildAccountBilling(cost, billing, { vmId, accountId, requestLog } = {}) { +async function buildAccountBilling(cost, billing, { vmId, accountId, requestLog } = {}) { if (!cost && !vmId && !accountId) return null const today = periodView(cost?.today) || { requests: Number(cost?.today_requests || 0), @@ -2475,14 +2475,14 @@ function buildAccountBilling(cost, billing, { vmId, accountId, requestLog } = {} const total = periodView(cost) let byModel = [] try { - byModel = requestLog?.costByModel?.({ vmId, accountId }) || [] + byModel = (await requestLog?.costByModel?.({ vmId, accountId })) || [] } catch { byModel = [] } if (!cost && !byModel.length) return null let usageStats = null try { - usageStats = vmId ? requestLog?.vmUsageStats?.({ vmId, days: 30 }) || null : null + usageStats = vmId ? (await requestLog?.vmUsageStats?.({ vmId, days: 30 })) || null : null } catch { usageStats = null } diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 4d2ab9902..5fa6a2d94 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -1,3 +1,4 @@ +import { flushPendingWrites } from '../db/pending-writes.mjs' /** * /admin and /api/panel routes. Login / logout / me and { ok, data } * envelopes are unchanged; slot start/reload go through startSlotReady. @@ -26,7 +27,7 @@ import { } from '../protocol/jev-intercept.mjs' import { RefusalGuardsRepo } from '../db/repos/refusal-guards-repo.mjs' import { RefusalDeviceBlocksRepo } from '../db/repos/refusal-device-blocks-repo.mjs' -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { parseCodexImportPayload, upsertCodexAccount, readCodexAccounts } from '../vm/codex-slot.mjs' import { generateAuthUrl, exchangeAuthCode, normalizeOauthFlavor } from '../oauth/oauth-auth-url.mjs' import { sessionKeyToOAuth, panelImportErrorPayload } from '../oauth/cookie-auth.mjs' @@ -173,7 +174,7 @@ import { GATEWAY_CAPABILITIES } from '../vm/execution-context.mjs' import { withVmLock, atomicWriteJson, isValidVmId, invalidateVmSlotReadCache } from '../vm/vm-file.mjs' import { snapshotDatabaseMetrics } from '../db/database-metrics.mjs' import { getUsageCache } from '../oauth/usage-cache.mjs' -import { getDb, getDbPath } from '../db/database.mjs' +import { getDb } from '../db/database.mjs' import { readLocalVersion, loadChangelog, @@ -182,7 +183,7 @@ import { startHostUpgrade, } from './release.mjs' import { downloadReleaseKernel, kernelReleaseHttpStatus } from './release-kernel.mjs' -import { clearActiveVmFromDb, removeVmFromDb } from '../vm/vm-db-sync.mjs' +import { clearActiveVmFromDb, removeVmFromDb, flushVmDbSync } from '../vm/vm-db-sync.mjs' import { normalizeCredentialMode } from '../oauth/credential-mode.mjs' import { reloadActiveVm } from '../core/config.mjs' import { @@ -300,7 +301,7 @@ function clusterNodesOf(ctx) { } } -function collectOnboardingFacts(req, ctx) { +async function collectOnboardingFacts(req, ctx) { const ident = panelIdentity(req) const ownerUserId = ident.role === 'user' ? req.panelUserId : null const vms = listVms(ctx.cfg.paths.project, ctx.routingConfig) @@ -311,24 +312,24 @@ function collectOnboardingFacts(req, ctx) { nodes: clusterNodesOf(ctx), }) const visible = filterVmsForPanel(vms, { role: ident.role, userId: ownerUserId }) - const keys = ctx.apiKeyStore.snapshot().keys || [] + const keys = (await ctx.apiKeyStore.snapshot()).keys || [] const mine = ident.role === 'user' ? keys.filter((k) => normalizeOwnerId(k.user_id) === normalizeOwnerId(req.panelUserId)) : keys const admin = getPanelAdmin() return { envPassword: admin.password, - envAdmin: ctx.panelUsers.getByUsername(admin.username), + envAdmin: await ctx.panelUsers.getByUsername(admin.username), proxyCount: (snap.proxies || []).length, ...vmStepFacts(visible), activeKeyCount: mine.filter((k) => k.status === 'active').length, - hasFirstRequest: ctx.requestLog.hasSuccessfulRequest({ owner_user_id: ownerUserId }), + hasFirstRequest: await ctx.requestLog.hasSuccessfulRequest({ owner_user_id: ownerUserId }), } } -function onboardingPayloadFor(req, ctx) { +async function onboardingPayloadFor(req, ctx) { const user = onboardingUserOf(req) const role = onboardingRoleOf(req) - const rec = recordOf(loadOnboardingState(new SettingsRepo()), user) + const rec = recordOf(await loadOnboardingState(new SettingsRepo()), user) return buildOnboardingPayload({ user, role, @@ -337,7 +338,7 @@ function onboardingPayloadFor(req, ctx) { guide_used: rec.guide_used, // Onboarding matches /health: base_url only when PUBLIC_* is configured; else null → browser origin. base_url: process.env.PUBLIC_BASE_URL || null, - facts: collectOnboardingFacts(req, ctx), + facts: await collectOnboardingFacts(req, ctx), }) } @@ -591,7 +592,27 @@ function slotProxyHint(px = {}, label = null) { export function createPanelHandler(ctx) { // A checkout can change under a live Node process; report its boot version. const runningVersion = readLocalVersion(ctx.cfg?.paths?.project) - const json = (...args) => ctx.json(...args) + const json = (...args) => { + const state = args[0]?._panelResponseState + if (state) { + state.response = args + return + } + return ctx.json(...args) + } + async function flushPanelWrites() { + for (const store of [ctx.accountQuota, ctx.stickyRouter, ctx.proxyPool, ctx.runtimeRepo]) { + await store?.writes?.flush?.() + } + await flushVmDbSync() + } + async function persistenceFailure(res, error) { + await ctx.onPersistenceFailure?.(error) + return ctx.json(res, 503, { + ok: false, + error: { code: 'database_persistence_failed', message: 'Database persistence failed; maintenance is required' }, + }) + } const hostUnsupported = (res, vm, cap) => { if (slotHost(vm).supports(cap)) return false json(res, 409, { @@ -603,7 +624,7 @@ export function createPanelHandler(ctx) { const readBody = (...args) => ctx.readBody(...args) const readRawBody = (...args) => (ctx.readRawBody || defaultReadRawBody)(...args) const refreshCluster = ctx.refreshClusterSlotPayloads || refreshClusterSlotPayloads - const requireAuth = (...args) => ctx.requireAuth(...args) + const requireAuth = async (...args) => await ctx.requireAuth(...args) const cfg = ctx.cfg const routingConfigPath = ctx.routingConfigPath const stickyRouter = ctx.stickyRouter @@ -617,13 +638,13 @@ export function createPanelHandler(ctx) { const vmTestSchedule = ctx.vmTestSchedule const backupService = ctx.backupService const stats = ctx.stats - const persistRoutingPatch = (...args) => ctx.persistRoutingPatch(...args) + const persistRoutingPatch = async (...args) => ctx.persistRoutingPatch(...args) const applyVmConcurrency = (...args) => ctx.applyVmConcurrency(...args) const applyVmRpm = (...args) => ctx.applyVmRpm(...args) const applyVmSessionSlots = (...args) => ctx.applyVmSessionSlots(...args) const inheritVmScheduling = (...args) => ctx.inheritVmScheduling(...args) const applyVmQuotaOverride = (...args) => ctx.applyVmQuotaOverride(...args) - const initPoolRuntime = (...args) => ctx.initPoolRuntime(...args) + const initPoolRuntime = async (...args) => ctx.initPoolRuntime(...args) const poolSchedulerConfig = (...args) => ctx.poolSchedulerConfig(...args) const commitImportedOauth = (...args) => ctx.commitImportedOauth(...args) const requireSlotProxy = (...args) => ctx.requireSlotProxy(...args) @@ -639,8 +660,8 @@ export function createPanelHandler(ctx) { setVmSchedulable(cfg.paths.project, vmId, true) } - function fillTimezoneFromExit(id) { - return fillVmTimezoneFromExit(cfg.paths.project, proxyPool, id) + async function fillTimezoneFromExit(id) { + return await fillVmTimezoneFromExit(cfg.paths.project, proxyPool, id) } /** @@ -752,7 +773,7 @@ export function createPanelHandler(ctx) { return json(res, 200, panel.ok({ schedule: vmTestSchedule.get(id) })) } if (req.method === 'DELETE') { - vmTestSchedule.remove(id) + await vmTestSchedule.remove(id) return json(res, 200, panel.ok({ schedule: null })) } let body @@ -765,7 +786,7 @@ export function createPanelHandler(ctx) { }) } try { - return json(res, 200, panel.ok({ schedule: vmTestSchedule.save(id, body) })) + return json(res, 200, panel.ok({ schedule: await vmTestSchedule.save(id, body) })) } catch (error) { const typed = error instanceof VmTestScheduleError return json(res, typed ? error.status : 500, { @@ -793,10 +814,10 @@ export function createPanelHandler(ctx) { } try { if (body == null || body.schedule == null) { - vmTestSchedule.clearDefaultTemplate() + await vmTestSchedule.clearDefaultTemplate() return json(res, 200, panel.ok({ schedule: null })) } - return json(res, 200, panel.ok({ schedule: vmTestSchedule.saveDefaultTemplate(body.schedule) })) + return json(res, 200, panel.ok({ schedule: await vmTestSchedule.saveDefaultTemplate(body.schedule) })) } catch (error) { const typed = error instanceof VmTestScheduleError return json(res, typed ? error.status : 500, { @@ -838,7 +859,7 @@ export function createPanelHandler(ctx) { body = { ...body, vm_ids: vmIds } } try { - const result = vmTestSchedule.syncSchedules({ + const result = await vmTestSchedule.syncSchedules({ scope: body?.scope, vmIds: body?.vm_ids, schedule: body?.schedule, @@ -905,17 +926,17 @@ export function createPanelHandler(ctx) { return cfg.distill } - function refusalGuardSnapshot() { + async function refusalGuardSnapshot() { const settings = new SettingsRepo() const repo = new RefusalGuardsRepo() const devices = new RefusalDeviceBlocksRepo() const read = (key, fallback) => settings.get(key, fallback) return { ...refusalGuardPolicy(read), - count: repo.count(), - items: repo.list(), - device_count: devices.count(), - devices: devices.list(), + count: await repo.count(), + items: await repo.list(), + device_count: await devices.count(), + devices: await devices.list(), } } @@ -1104,14 +1125,14 @@ export function createPanelHandler(ctx) { }) } - function restoreRoutingRuntime(previous) { + async function restoreRoutingRuntime(previous) { ctx.routingConfig = previous setManualScheduleWins(ctx.routingConfig.pool?.manual_schedule_wins) ctx.healthMonitor?.setConfig(ctx.routingConfig.health_probe) ctx.usageProbeMonitor?.setConfig(ctx.routingConfig.usage_probe) ctx.notifyMonitor?.setConfig(ctx.routingConfig.notify) if (ctx.routingConfig.logging) { - requestLog.setConfig({ + await requestLog.setConfig({ mode: ctx.routingConfig.logging.mode, retainDays: ctx.routingConfig.logging.retain_days, debugRetainDays: ctx.routingConfig.logging.debug_retain_days, @@ -1130,16 +1151,16 @@ export function createPanelHandler(ctx) { const target = normalizeInferenceConfig({ ...(ctx.routingConfig.inference || {}), ...body.inference }).engine if (previous === target) return { ok: true, changed: false, items: [] } const previousRoutingConfig = structuredClone(ctx.routingConfig) - return switchInheritedInferenceEngines({ + return await switchInheritedInferenceEngines({ projectRoot: cfg.paths.project, previousEngine: previous, targetEngine: target, - commit: () => { + commit: async () => { try { - return persistRoutingPatch(body) + return await persistRoutingPatch(body) } catch (error) { - restoreRoutingRuntime(previousRoutingConfig) - if (body.pool || body.failover) initPoolRuntime() + await restoreRoutingRuntime(previousRoutingConfig) + if (body.pool || body.failover) await initPoolRuntime() throw error } }, @@ -1161,12 +1182,13 @@ export function createPanelHandler(ctx) { } } - return async function handlePanel(req, res, url) { + async function handlePanel(req, res, url) { const p = url.pathname + if (req.method === 'GET') await flushPendingWrites() if (!p.startsWith('/admin') && !p.startsWith('/api/panel')) return false if (p.startsWith('/admin')) { - if (!requireAuth(req, res)) return true + if (!(await requireAuth(req, res))) return true if (panelIdentity(req).role !== 'admin') { json( res, @@ -1184,7 +1206,7 @@ export function createPanelHandler(ctx) { if (p === '/admin/models/refresh') return false if (p === '/admin/routing' && req.method === 'GET') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return return json(res, 200, { routing: publicRoutingNotify(ctx.routingConfig), sticky: stickyRouter.stats(), @@ -1193,11 +1215,11 @@ export function createPanelHandler(ctx) { }) } if (p === '/admin/routing' && (req.method === 'PUT' || req.method === 'POST')) { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const body = await readBody(req, cfg.limits.max_body_bytes) let applied try { - applied = persistRoutingPatch(body) + applied = await persistRoutingPatch(body) } catch (error) { if (error instanceof TypeError || error instanceof RangeError) { return json(res, 400, { @@ -1219,19 +1241,19 @@ export function createPanelHandler(ctx) { }) } if (p === '/admin/quota' && req.method === 'GET') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return return json(res, 200, accountQuota.snapshot()) } // ---- Multi-VM + usage management ---- if (req.method === 'GET' && p === '/admin/vms') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const active = getActiveVmId(cfg.paths.project) const vms = listVms(cfg.paths.project, ctx.routingConfig).map((v) => ({ ...v, active: v.id === active })) return json(res, 200, { vms, active_vm: active, total: vms.length }) } if (req.method === 'GET' && p.startsWith('/admin/vms/') && !p.includes('/probe') && !p.endsWith('/activate')) { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const id = p.split('/')[3] const vm = getVm(cfg.paths.project, id) if (!vm) return json(res, 404, { error: { message: 'vm not found' } }) @@ -1241,13 +1263,13 @@ export function createPanelHandler(ctx) { }) } if (req.method === 'POST' && /^\/admin\/vms\/[^/]+\/activate$/.test(p)) { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const id = p.split('/')[3] if (!getVm(cfg.paths.project, id)) return json(res, 404, { error: { message: 'vm not found' } }) return json(res, 200, activateVmSlot(id)) } if (req.method === 'POST' && /^\/admin\/vms\/[^/]+\/probe$/.test(p)) { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const id = p.split('/')[3] const hop = (await readBody(req, 4096).catch(() => ({})))?.hop !== false const result = await panel.buildProbeOne({ cfg, accountQuota, id, hop, routingConfig: ctx.routingConfig }) @@ -1255,13 +1277,13 @@ export function createPanelHandler(ctx) { return json(res, 200, { vm_id: id, account_uuid: result.data?.account_uuid, probe: result.data }) } if (req.method === 'POST' && p === '/admin/vms/probe-all') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const hop = (await readBody(req, 4096).catch(() => ({})))?.hop === true const result = await panel.buildProbeAll({ cfg, accountQuota, hop, routingConfig: ctx.routingConfig }) return json(res, 200, result) } if (req.method === 'GET' && p === '/admin/usage/summary') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const snap = accountQuota.snapshot() const vms = listVms(cfg.paths.project, ctx.routingConfig) const accounts = snap.accounts || [] @@ -1297,16 +1319,16 @@ export function createPanelHandler(ctx) { const body = await readBody(req, 4096) const username = body.username || body.user || body.u || '' const password = body.password || body.pass || body.p || '' - const authed = verifyPanelLogin(username, password) + const authed = await verifyPanelLogin(username, password) if (!authed) { return json(res, 401, { ok: false, error: { message: '用户名或密码错误', type: 'auth' } }) } if (authed.source === 'env') { try { - panelUsers.bootstrapFromEnv({ username: authed.username, password }) + await panelUsers.bootstrapFromEnv({ username: authed.username, password }) } catch {} } - const token = createPanelSession(authed.username, { role: authed.role }) + const token = await createPanelSession(authed.username, { role: authed.role }) const secure = panelCookieSecure(req) res.setHeader('Set-Cookie', panelSessionCookie(token, { secure })) const me = { user: authed.username, role: authed.role } @@ -1329,7 +1351,7 @@ export function createPanelHandler(ctx) { } if (p.startsWith('/api/panel')) { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return // Managed client keys may only call protocol endpoints — never panel/admin. // Only panel session or master KIN_API_KEY may administer. const isPanelOperator = !!req.panelUser || req.apiKeyKind === 'master' @@ -1386,10 +1408,10 @@ export function createPanelHandler(ctx) { try { const settings = new SettingsRepo() const user = onboardingUserOf(req) - const state = loadOnboardingState(settings) + const state = await loadOnboardingState(settings) let next if (body?.action === 'profile') { - applyUsageProfile({ + await applyUsageProfile({ mode: body?.mode, persistRouting: (patch) => persistRoutingPatch(patch), settings, @@ -1402,7 +1424,7 @@ export function createPanelHandler(ctx) { } else { next = applyOnboardingAction(state, user, body?.action) } - saveOnboardingState(settings, next) + await saveOnboardingState(settings, next) } catch (e) { if (e.code === 'invalid_action' || e.code === 'invalid_profile') { return json(res, 400, { @@ -1413,13 +1435,12 @@ export function createPanelHandler(ctx) { throw e } } - return json(res, 200, panel.ok(onboardingPayloadFor(req, ctx))) + return json(res, 200, panel.ok(await onboardingPayloadFor(req, ctx))) } - if (p.startsWith('/api/panel/cluster/')) return ctx.clusterRoutes.handle(req, res, url) + if (p.startsWith('/api/panel/cluster/')) return ctx.clusterRoutes.handle(req, res, url, json) if (req.method === 'GET' && p === '/api/panel/database/metrics') { - const snapshot = snapshotDatabaseMetrics({ + const snapshot = await snapshotDatabaseMetrics({ db: getDb(), - dbPath: getDbPath(), usageCache: getUsageCache(), }) return json(res, 200, panel.ok(snapshot)) @@ -1455,12 +1476,12 @@ export function createPanelHandler(ctx) { return json(res, result.status, panel.ok(result.data)) } if (req.method === 'GET' && p === '/api/panel/users') { - return json(res, 200, panel.ok({ items: panelUsers.list() })) + return json(res, 200, panel.ok({ items: await panelUsers.list() })) } if (req.method === 'POST' && p === '/api/panel/users') { const body = await readBody(req, 8192).catch(() => ({})) try { - const rec = panelUsers.create({ + const rec = await panelUsers.create({ username: body.username || body.user, password: body.password || body.pass, role: body.role || 'user', @@ -1480,8 +1501,8 @@ export function createPanelHandler(ctx) { const id = p.split('/').pop() const body = await readBody(req, 8192).catch(() => ({})) try { - const before = panelUsers.getById(id) - const rec = panelUsers.update(id, body || {}, { actorId: req.panelUserId }) + const before = await panelUsers.getById(id) + const rec = await panelUsers.update(id, body || {}, { actorId: req.panelUserId }) if (!rec) return json(res, 404, { ok: false, error: { message: 'user not found' } }) // Sessions cache the role from login time, so a role or enable change must log them out. if (before && (before.role !== rec.role || !!before.enabled !== !!rec.enabled)) { @@ -1500,9 +1521,9 @@ export function createPanelHandler(ctx) { } if (req.method === 'DELETE' && /^\/api\/panel\/users\/[^/]+$/.test(p)) { const id = p.split('/').pop() - const existing = panelUsers.getById(id) + const existing = await panelUsers.getById(id) try { - const ok = panelUsers.remove(id, { actorId: req.panelUserId }) + const ok = await panelUsers.remove(id, { actorId: req.panelUserId }) if (!ok) return json(res, 404, { ok: false, error: { message: 'user not found' } }) if (existing?.username) revokePanelSessionsForUser(existing.username) return json(res, 200, { ok: true, deleted: id }) @@ -1515,12 +1536,12 @@ export function createPanelHandler(ctx) { } } if (req.method === 'GET' && p === '/api/panel/vm-pools') { - return json(res, 200, panel.ok({ pools: vmPoolsFor(apiKeyStore.db).list() })) + return json(res, 200, panel.ok({ pools: await vmPoolsFor(apiKeyStore.db).list() })) } if (req.method === 'POST' && p === '/api/panel/vm-pools') { const body = await readBody(req, 64 * 1024).catch(() => ({})) try { - const pool = createVmPool(vmPoolsFor(apiKeyStore.db), body || {}, listVms(cfg.paths.project)) + const pool = await createVmPool(vmPoolsFor(apiKeyStore.db), body || {}, listVms(cfg.paths.project)) return json(res, 201, panel.ok({ pool })) } catch (e) { return json(res, vmPoolHttpStatus(e.code), { @@ -1533,7 +1554,7 @@ export function createPanelHandler(ctx) { const id = decodeURIComponent(p.split('/').pop()) const body = await readBody(req, 64 * 1024).catch(() => ({})) try { - const pool = updateVmPool(vmPoolsFor(apiKeyStore.db), id, body || {}, listVms(cfg.paths.project)) + const pool = await updateVmPool(vmPoolsFor(apiKeyStore.db), id, body || {}, listVms(cfg.paths.project)) if (!pool) return json(res, 404, { ok: false, error: { message: '账号池不存在', code: 'vm_pool_not_found' } }) return json(res, 200, panel.ok({ pool })) } catch (e) { @@ -1545,7 +1566,7 @@ export function createPanelHandler(ctx) { } if (req.method === 'DELETE' && /^\/api\/panel\/vm-pools\/[^/]+$/.test(p)) { const id = decodeURIComponent(p.split('/').pop()) - const result = deleteVmPool(vmPoolsFor(apiKeyStore.db), id) + const result = await deleteVmPool(vmPoolsFor(apiKeyStore.db), id) if (!result.ok) { return json(res, vmPoolHttpStatus(result.error), { ok: false, @@ -1559,7 +1580,7 @@ export function createPanelHandler(ctx) { return json(res, 200, { ok: true, deleted: id }) } if (req.method === 'GET' && p === '/api/panel/api-keys') { - const snap = apiKeyStore.snapshot() + const snap = await apiKeyStore.snapshot() if (panelIdentity(req).role === 'user') { const mine = (snap.keys || []).filter( (k) => normalizeOwnerId(k.user_id) === normalizeOwnerId(req.panelUserId), @@ -1575,10 +1596,10 @@ export function createPanelHandler(ctx) { } if (req.method === 'GET' && /^\/api\/panel\/api-keys\/[^/]+\/stats$/.test(p)) { const id = p.split('/')[4] - if (denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true - const rec = apiKeyStore.getById(id) + if (await denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true + const rec = await apiKeyStore.getById(id) if (!rec) return json(res, 404, { ok: false, error: { message: 'api key not found' } }) - const usage_stats = requestLog?.keyUsageStats?.({ apiKeyId: id, days: 30 }) || null + const usage_stats = (await requestLog?.keyUsageStats?.({ apiKeyId: id, days: 30 })) || null return json(res, 200, { ok: true, item: publicKeyView(rec, { reveal: false }), usage_stats }) } if (req.method === 'POST' && p === '/api/panel/api-keys') { @@ -1598,14 +1619,14 @@ export function createPanelHandler(ctx) { } const scope = normalizeKeyScope(scopeInput) const pools = vmPoolsFor(apiKeyStore.db) - if (scope.vm_pool_id) assertVmPoolExists(pools, scope.vm_pool_id) + if (scope.vm_pool_id) await assertVmPoolExists(pools, scope.vm_pool_id) else assertVmScope(keyScopeVms(req), scope) const defaultConc = Number( ctx.routingConfig?.concurrency?.default_key_concurrency ?? ctx.routingConfig?.concurrency?.default_max_per_account ?? 20, ) - const rec = apiKeyStore.create({ + const rec = await apiKeyStore.create({ name: body?.name, key: body?.key || body?.custom_key || undefined, category: body?.category, @@ -1644,7 +1665,7 @@ export function createPanelHandler(ctx) { } if (req.method === 'PATCH' && /^\/api\/panel\/api-keys\/[^/]+$/.test(p)) { const id = p.split('/').pop() - if (denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true + if (await denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true const body = await readBody(req, 8192).catch(() => ({})) const ident = panelIdentity(req) if (ident.role !== 'admin' && body?.vm_pool_id != null && String(body.vm_pool_id).trim() !== '') { @@ -1655,7 +1676,7 @@ export function createPanelHandler(ctx) { } try { const input = normalizePanelApiKeyInput(body || {}, req) - const currentRec = apiKeyStore.getById(id) + const currentRec = await apiKeyStore.getById(id) const scopeInput = { group_type: body?.group_type, allowed_vms: body?.allowed_vms } if (ident.role === 'admin' && Object.prototype.hasOwnProperty.call(body || {}, 'vm_pool_id')) { scopeInput.vm_pool_id = body.vm_pool_id @@ -1666,13 +1687,13 @@ export function createPanelHandler(ctx) { scope.vm_pool_id = currentRec.vm_pool_id } const pools = vmPoolsFor(apiKeyStore.db) - if (scope.vm_pool_id) assertVmPoolExists(pools, scope.vm_pool_id) + if (scope.vm_pool_id) await assertVmPoolExists(pools, scope.vm_pool_id) else assertVmScope(keyScopeVms(req), scope) input.group_type = scope.group_type input.allowed_vms = scope.allowed_vms input.vm_pool_id = scope.vm_pool_id } - const rec = apiKeyStore.update(id, input) + const rec = await apiKeyStore.update(id, input) if (!rec) { return json(res, 404, { ok: false, error: { message: 'api key not found' } }) } @@ -1686,8 +1707,8 @@ export function createPanelHandler(ctx) { } if (req.method === 'POST' && /^\/api\/panel\/api-keys\/[^/]+\/reveal$/.test(p)) { const id = p.split('/')[4] - if (denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true - const out = apiKeyStore.reveal(id) + if (await denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true + const out = await apiKeyStore.reveal(id) if (!out) return json(res, 404, { ok: false, error: { message: 'api key not found' } }) if (!out.ok) { return json(res, 409, { @@ -1699,9 +1720,9 @@ export function createPanelHandler(ctx) { } if (req.method === 'POST' && /^\/api\/panel\/api-keys\/[^/]+\/rotate$/.test(p)) { const id = p.split('/')[4] - if (denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true + if (await denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true try { - const rec = apiKeyStore.rotate(id) + const rec = await apiKeyStore.rotate(id) if (!rec) return json(res, 404, { ok: false, error: { message: 'api key not found' } }) return json(res, 200, { ok: true, item: publicKeyView(rec, { reveal: true }) }) } catch (e) { @@ -1713,15 +1734,15 @@ export function createPanelHandler(ctx) { } if (req.method === 'POST' && /^\/api\/panel\/api-keys\/[^/]+\/reset-quota$/.test(p)) { const id = p.split('/')[4] - if (denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true - const rec = apiKeyStore.update(id, { reset_quota: true }) + if (await denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true + const rec = await apiKeyStore.update(id, { reset_quota: true }) if (!rec) return json(res, 404, { ok: false, error: { message: 'api key not found' } }) return json(res, 200, { ok: true, item: publicKeyView(rec, { reveal: false }) }) } if (req.method === 'DELETE' && /^\/api\/panel\/api-keys\/[^/]+$/.test(p)) { const id = p.split('/').pop() - if (denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true - const ok = apiKeyStore.remove(id) + if (await denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true + const ok = await apiKeyStore.remove(id) if (!ok) return json(res, 404, { ok: false, error: { message: 'api key not found' } }) return json(res, 200, { ok: true, deleted: id }) } @@ -1729,14 +1750,14 @@ export function createPanelHandler(ctx) { if (req.method === 'GET' && p === '/api/panel/api-endpoints') { return json(res, 200, { ok: true, - items: apiEndpointStore.list({ reveal: false }), + items: await apiEndpointStore.list({ reveal: false }), presets: API_ENDPOINT_PRESETS, }) } if (req.method === 'POST' && p === '/api/panel/api-endpoints/fetch-models') { const body = await readBody(req, 8192).catch(() => ({})) try { - const saved = savedEndpointKey(apiEndpointStore, body?.key_id) + const saved = await savedEndpointKey(apiEndpointStore, body?.key_id) const proxyUrl = Object.prototype.hasOwnProperty.call(body || {}, 'proxy_id') ? resolveConsoleProxyUrl(proxyPool, body.proxy_id) : saved?.proxy_url || body?.proxy_url @@ -1769,8 +1790,8 @@ export function createPanelHandler(ctx) { if (body?.proxy_id != null) entry.proxy_id = String(body.proxy_id || '').trim() } } - const rec = apiEndpointStore.create(body || {}) - apiScheduler.reload(apiEndpointStore.listRaw()) + const rec = await apiEndpointStore.create(body || {}) + apiScheduler.reload(await apiEndpointStore.listRaw()) return json(res, 201, { ok: true, item: publicEndpointView(rec, { reveal: true }) }) } catch (e) { return json(res, 400, { @@ -1784,24 +1805,24 @@ export function createPanelHandler(ctx) { const body = await readBody(req, 1 << 20).catch(() => ({})) try { const proxyUrl = body?.proxy_id ? resolveConsoleProxyUrl(proxyPool, body.proxy_id) : '' - const rec = apiEndpointStore.update(id, body || {}) + const rec = await apiEndpointStore.update(id, body || {}) if (!rec) return json(res, 404, { ok: false, error: { message: 'endpoint not found' } }) const keyId = String(body?.key_id || '').trim() const apiKey = String(body?.api_key || '').trim() if (keyId) { - apiEndpointStore.updateKey(keyId, { + await apiEndpointStore.updateKey(keyId, { ...(apiKey ? { api_key: apiKey } : {}), ...(body?.proxy_id != null ? { proxy_url: proxyUrl, proxy_id: String(body.proxy_id || '').trim() } : {}), }) } else if (apiKey) { - apiEndpointStore.addKey(id, { + await apiEndpointStore.addKey(id, { api_key: apiKey, proxy_url: proxyUrl, proxy_id: String(body?.proxy_id || '').trim(), }) } - apiScheduler.reload(apiEndpointStore.listRaw()) - const saved = apiEndpointStore.getRaw(id) || rec + apiScheduler.reload(await apiEndpointStore.listRaw()) + const saved = (await apiEndpointStore.getRaw(id)) || rec return json(res, 200, { ok: true, item: publicEndpointView(saved) }) } catch (e) { return json(res, 400, { ok: false, error: { message: String(e.message || e), code: e.code } }) @@ -1809,15 +1830,15 @@ export function createPanelHandler(ctx) { } if (req.method === 'DELETE' && /^\/api\/panel\/api-endpoints\/[^/]+$/.test(p)) { const id = p.split('/').pop() - const ok = apiEndpointStore.remove(id) + const ok = await apiEndpointStore.remove(id) if (!ok) return json(res, 404, { ok: false, error: { message: 'endpoint not found' } }) - apiScheduler.reload(apiEndpointStore.listRaw()) + apiScheduler.reload(await apiEndpointStore.listRaw()) return json(res, 200, { ok: true, deleted: id }) } if (req.method === 'POST' && /^\/api\/panel\/api-endpoints\/[^/]+\/fetch-models$/.test(p)) { const id = p.split('/')[4] const body = await readBody(req, 8192).catch(() => ({})) - const ep = apiEndpointStore.getRaw(id) + const ep = await apiEndpointStore.getRaw(id) if (!ep) return json(res, 404, { ok: false, error: { message: 'endpoint not found' } }) const key = (ep.api_key_entries || []).find((k) => !k.disabled && (body?.key_id ? k.id === body.key_id : true)) if (!key) @@ -1833,8 +1854,8 @@ export function createPanelHandler(ctx) { headers: ep.headers, }) const apply = body?.apply !== false - const rec = apply ? apiEndpointStore.mergeFetchedModels(id, result.models) : ep - if (apply) apiScheduler.reload(apiEndpointStore.listRaw()) + const rec = apply ? await apiEndpointStore.mergeFetchedModels(id, result.models) : ep + if (apply) apiScheduler.reload(await apiEndpointStore.listRaw()) return json(res, 200, { ok: true, fetched: result.models.length, @@ -1852,9 +1873,9 @@ export function createPanelHandler(ctx) { const id = p.split('/')[4] const body = await readBody(req, 8192).catch(() => ({})) try { - const key = apiEndpointStore.addKey(id, body || {}) - apiScheduler.reload(apiEndpointStore.listRaw()) - const view = publicEndpointView(apiEndpointStore.getRaw(id), { reveal: true }) + const key = await apiEndpointStore.addKey(id, body || {}) + apiScheduler.reload(await apiEndpointStore.listRaw()) + const view = publicEndpointView(await apiEndpointStore.getRaw(id), { reveal: true }) return json(res, 201, { ok: true, item: view?.api_key_entries?.find((k) => k.id === key.id) || key }) } catch (e) { const status = e.code === 'endpoint_not_found' ? 404 : 400 @@ -1863,18 +1884,18 @@ export function createPanelHandler(ctx) { } if (req.method === 'DELETE' && /^\/api\/panel\/api-endpoints\/[^/]+\/keys\/[^/]+$/.test(p)) { const kid = p.split('/').pop() - const ok = apiEndpointStore.removeKey(kid) + const ok = await apiEndpointStore.removeKey(kid) if (!ok) return json(res, 404, { ok: false, error: { message: 'key not found' } }) - apiScheduler.reload(apiEndpointStore.listRaw()) + apiScheduler.reload(await apiEndpointStore.listRaw()) return json(res, 200, { ok: true, deleted: kid }) } if (req.method === 'PATCH' && /^\/api\/panel\/api-endpoints\/[^/]+\/keys\/[^/]+$/.test(p)) { const kid = p.split('/').pop() const body = await readBody(req, 8192).catch(() => ({})) - const rec = apiEndpointStore.updateKey(kid, body || {}) + const rec = await apiEndpointStore.updateKey(kid, body || {}) if (!rec) return json(res, 404, { ok: false, error: { message: 'key not found' } }) - apiScheduler.reload(apiEndpointStore.listRaw()) - const ep = apiEndpointStore.getRaw(rec.endpoint_id) + apiScheduler.reload(await apiEndpointStore.listRaw()) + const ep = await apiEndpointStore.getRaw(rec.endpoint_id) const item = publicEndpointView(ep)?.api_key_entries?.find((k) => k.id === kid) return json(res, 200, { ok: true, item }) } @@ -1908,8 +1929,8 @@ export function createPanelHandler(ctx) { return json(res, 200, { ok: true, mode: 'debug', - config: requestLog.snapshot(), - items: requestLog.listDebug({ + config: await requestLog.snapshot(), + items: await requestLog.listDebug({ limit, error_class: u.searchParams.get('error_class') || null, exclude_error_class: u.searchParams.get('exclude_error_class') || null, @@ -1936,11 +1957,11 @@ export function createPanelHandler(ctx) { q: u.searchParams.get('q') || null, owner_user_id: panelIdentity(req).role === 'user' ? req.panelUserId : u.searchParams.get('user_id') || null, } - const { items, total } = requestLog.queryNormal(filters) + const { items, total } = await requestLog.queryNormal(filters) return json(res, 200, { ok: true, mode: 'normal', - config: requestLog.snapshot(), + config: await requestLog.snapshot(), items, total, limit: filters.limit, @@ -1964,7 +1985,7 @@ export function createPanelHandler(ctx) { status_code: listed.status_code || (/^\d+$/.test(legacyStatus || '') ? legacyStatus : null), errors_only: legacyStatus === 'error', } - const { items, total, capped } = new UsageLogsView(getDb(), { + const { items, total, capped } = await new UsageLogsView(getDb(), { mutedErrorClasses: requestLog?.mutedErrorClasses, }).exportRows(filters) const stamp = new Date().toISOString().slice(0, 19).replace(/[:T]/g, '') @@ -1998,17 +2019,17 @@ export function createPanelHandler(ctx) { bucket, since, until, - totals: tenant ? null : requestLog.totals(), - buckets: requestLog.aggregate({ since, until, bucket, owner_user_id }), - window: requestLog.windowStats({ since, until, owner_user_id }), - muted_error_classes: requestLog.snapshot().muted_error_classes, - config: tenant ? null : requestLog.snapshot(), + totals: tenant ? null : await requestLog.totals(), + buckets: await requestLog.aggregate({ since, until, bucket, owner_user_id }), + window: await requestLog.windowStats({ since, until, owner_user_id }), + muted_error_classes: (await requestLog.snapshot()).muted_error_classes, + config: tenant ? null : await requestLog.snapshot(), }) } if (req.method === 'GET' && /^\/api\/panel\/request-logs\/[^/]+\/attempts$/.test(p)) { const requestId = p.split('/')[4] const owner_user_id = panelIdentity(req).role === 'user' ? req.panelUserId : null - if (owner_user_id && !requestLog.getDebug(requestId, { owner_user_id })) { + if (owner_user_id && !(await requestLog.getDebug(requestId, { owner_user_id }))) { return json(res, 404, { ok: false, error: { message: 'debug log not found' } }) } return json( @@ -2017,16 +2038,16 @@ export function createPanelHandler(ctx) { panel.ok({ request_id: requestId, attempts: - (typeof ctx.getAttemptsRepo === 'function' ? ctx.getAttemptsRepo() : ctx.attemptsRepo)?.list?.( + (await (typeof ctx.getAttemptsRepo === 'function' ? ctx.getAttemptsRepo() : ctx.attemptsRepo)?.list?.( requestId, - ) || [], + )) || [], }), ) } if (req.method === 'GET' && /^\/api\/panel\/request-logs\/[^/]+$/.test(p)) { const id = p.split('/').pop() const owner_user_id = panelIdentity(req).role === 'user' ? req.panelUserId : null - const rec = requestLog.getDebug(id, { owner_user_id }) + const rec = await requestLog.getDebug(id, { owner_user_id }) if (!rec) return json(res, 404, { ok: false, error: { message: 'debug log not found' } }) return json(res, 200, { ok: true, item: rec }) } @@ -2038,31 +2059,39 @@ export function createPanelHandler(ctx) { const tenantId = () => (panelIdentity(req).role === 'user' ? req.panelUserId : null) if (req.method === 'GET' && p === '/api/panel/usage-logs') { const u = new URL(req.url, 'http://x') - return json(res, 200, panel.ok(usageLogsView().listBatch(usageLogFilters(req, u)))) + return json(res, 200, panel.ok(await usageLogsView().listBatch(usageLogFilters(req, u)))) } if (req.method === 'GET' && p === '/api/panel/usage-logs/summary') { const u = new URL(req.url, 'http://x') - return json(res, 200, panel.ok(usageLogsView().summary(usageLogFilters(req, u)))) + return json(res, 200, panel.ok(await usageLogsView().summary(usageLogFilters(req, u)))) } if (req.method === 'GET' && p === '/api/panel/usage-logs/filter-options') { - return json(res, 200, panel.ok(usageLogsView().filterOptions({ owner_user_id: tenantId() }))) + return json(res, 200, panel.ok(await usageLogsView().filterOptions({ owner_user_id: tenantId() }))) } if (req.method === 'GET' && p === '/api/panel/usage-logs/session-suggestions') { const u = new URL(req.url, 'http://x') const q = u.searchParams.get('q') || '' const limit = u.searchParams.get('limit') - return json(res, 200, panel.ok(usageLogsView().sessionSuggestions({ q, limit, owner_user_id: tenantId() }))) + return json( + res, + 200, + panel.ok(await usageLogsView().sessionSuggestions({ q, limit, owner_user_id: tenantId() })), + ) } if (req.method === 'GET' && p === '/api/panel/usage-logs/active-sessions') { const u = new URL(req.url, 'http://x') const minutes = u.searchParams.get('minutes') const limit = u.searchParams.get('limit') - return json(res, 200, panel.ok(usageLogsView().activeSessions({ minutes, limit, owner_user_id: tenantId() }))) + return json( + res, + 200, + panel.ok(await usageLogsView().activeSessions({ minutes, limit, owner_user_id: tenantId() })), + ) } if (req.method === 'GET' && p === '/api/panel/usage-logs/overview') { const u = new URL(req.url, 'http://x') const tz = u.searchParams.get('tz') || 'UTC' - return json(res, 200, panel.ok(usageLogsView().overview({ tz, owner_user_id: tenantId() }))) + return json(res, 200, panel.ok(await usageLogsView().overview({ tz, owner_user_id: tenantId() }))) } // ---- Statistics page ---- @@ -2071,7 +2100,11 @@ export function createPanelHandler(ctx) { const q = statsParams(req, u, 'dimension') if (q.error) return json(res, 400, { ok: false, error: { message: q.error } }) const { range, dimension, tz, owner_user_id } = q - return json(res, 200, panel.ok(new StatisticsRepo(getDb()).statistics({ range, dimension, tz, owner_user_id }))) + return json( + res, + 200, + panel.ok(await new StatisticsRepo(getDb()).statistics({ range, dimension, tz, owner_user_id })), + ) } if (req.method === 'GET' && p === '/api/panel/statistics/leaderboard') { const u = new URL(req.url, 'http://x') @@ -2082,7 +2115,7 @@ export function createPanelHandler(ctx) { return json( res, 200, - panel.ok(new StatisticsRepo(getDb()).leaderboard({ range, scope, limit, tz, owner_user_id })), + panel.ok(await new StatisticsRepo(getDb()).leaderboard({ range, scope, limit, tz, owner_user_id })), ) } @@ -2093,16 +2126,16 @@ export function createPanelHandler(ctx) { const groupBy = u.searchParams.get('group_by') === 'key' ? 'key' : 'vm' const since = u.searchParams.get('from') || u.searchParams.get('since') || null const until = u.searchParams.get('until') || null - return json(res, 200, panel.ok(requestLog.ownerBilling({ ownerUserId, since, until, groupBy }))) + return json(res, 200, panel.ok(await requestLog.ownerBilling({ ownerUserId, since, until, groupBy }))) } // ---- Backups (local auto/manual, download, restore) ---- if (req.method === 'GET' && p === '/api/panel/backups') { return json(res, 200, { ok: true, - items: backupService.list(), - config: backupService.getSchedule(), - next_auto_at: backupService.nextAutoAt(), + items: await backupService.list(), + config: await backupService.getSchedule(), + next_auto_at: await backupService.nextAutoAt(), restoring: backupService.isRestoring, }) } @@ -2121,22 +2154,22 @@ export function createPanelHandler(ctx) { if (req.method === 'GET' && p === '/api/panel/backups/config') { return json(res, 200, { ok: true, - config: backupService.getSchedule(), - next_auto_at: backupService.nextAutoAt(), + config: await backupService.getSchedule(), + next_auto_at: await backupService.nextAutoAt(), }) } if (req.method === 'PUT' && p === '/api/panel/backups/config') { const body = await readBody(req, 8192).catch(() => ({})) try { - const config = backupService.updateSchedule(body || {}) - return json(res, 200, { ok: true, config, next_auto_at: backupService.nextAutoAt() }) + const config = await backupService.updateSchedule(body || {}) + return json(res, 200, { ok: true, config, next_auto_at: await backupService.nextAutoAt() }) } catch (e) { return json(res, 400, { ok: false, error: { message: String(e.message || e), code: e.code } }) } } if (req.method === 'GET' && /^\/api\/panel\/backups\/[^/]+\/download$/.test(p)) { const id = p.split('/')[4] - const rec = backupService.get(id) + const rec = await backupService.get(id) if (!rec || !rec.file_path || !fs.existsSync(rec.file_path)) { return json(res, 404, { ok: false, error: { message: 'backup file not found' } }) } @@ -2181,7 +2214,8 @@ export function createPanelHandler(ctx) { } if (req.method === 'DELETE' && /^\/api\/panel\/backups\/[^/]+$/.test(p)) { const id = p.split('/').pop() - if (!backupService.remove(id)) return json(res, 404, { ok: false, error: { message: 'backup not found' } }) + if (!(await backupService.remove(id))) + return json(res, 404, { ok: false, error: { message: 'backup not found' } }) return json(res, 200, { ok: true, deleted: id }) } @@ -2304,7 +2338,7 @@ export function createPanelHandler(ctx) { const previous = structuredClone(ctx.routingConfig) try { if (targets.all) { - persistRoutingPatch({ + await persistRoutingPatch({ inference: { ...(ctx.routingConfig.inference || {}), dataplane: parsed.value, @@ -2637,7 +2671,7 @@ export function createPanelHandler(ctx) { const body = await readBody(req, 8192).catch(() => ({})) const nextOwner = body.user_id === null || body.user_id === '' ? null : normalizeOwnerId(body.user_id || body.owner_user_id) - if (nextOwner && !panelUsers.getById(nextOwner)) { + if (nextOwner && !(await panelUsers.getById(nextOwner))) { return json(res, 404, { ok: false, error: { message: 'user not found' } }) } const currentOrigin = vm.origin || VM_ORIGIN.platform @@ -3332,12 +3366,12 @@ export function createPanelHandler(ctx) { fs.unlinkSync(vmPath) invalidateVmSlotReadCache(cfg.paths.project, id) try { - vmTestSchedule.remove(id) + await vmTestSchedule.remove(id) } catch {} if (active === id) { clearActiveVm(cfg.paths.project) try { - clearActiveVmFromDb() + await clearActiveVmFromDb() } catch {} try { reloadActiveVm(cfg) @@ -3345,7 +3379,7 @@ export function createPanelHandler(ctx) { } // drop the DB credential mirror row too try { - removeVmFromDb(id) + await removeVmFromDb(id) } catch {} // optional chat side files try { @@ -3512,8 +3546,8 @@ export function createPanelHandler(ctx) { } try { const repo = ctx.accountQuota?.repo - const row = repo && typeof repo.get === 'function' ? repo.get(id) : null - if (row && typeof repo.save === 'function') repo.save({ ...row, email: null, name: id }) + const row = repo && typeof repo.get === 'function' ? await repo.get(id) : null + if (row && typeof repo.save === 'function') await repo.save({ ...row, email: null, name: id }) } catch {} const saved = getVm(cfg.paths.project, id) || vm return json( @@ -3866,7 +3900,7 @@ export function createPanelHandler(ctx) { const ident = panelIdentity(req) if (parsed.ok && ident.role === 'user') { const ownerId = normalizeOwnerId(req.panelUserId) - const quota = clampVmCreateQuota(panelUsers.getById(ownerId)?.vm_create_quota, 0) + const quota = clampVmCreateQuota((await panelUsers.getById(ownerId))?.vm_create_quota, 0) const used = countUserCreatedVms(listVms(cfg.paths.project), ownerId) if (!ownerId || quota <= 0 || used >= quota) { return json(res, 403, { @@ -3930,7 +3964,7 @@ export function createPanelHandler(ctx) { const ident = panelIdentity(req) if (ident.role === 'user') { const ownerId = normalizeOwnerId(req.panelUserId) - const quota = clampVmCreateQuota(panelUsers.getById(ownerId)?.vm_create_quota, 0) + const quota = clampVmCreateQuota((await panelUsers.getById(ownerId))?.vm_create_quota, 0) const used = countUserCreatedVms(existing, ownerId) if (!ownerId || quota <= 0 || used >= quota) { return json(res, 403, { @@ -4994,7 +5028,7 @@ export function createPanelHandler(ctx) { } // GET /api/panel/usage if (req.method === 'GET' && p === '/api/panel/usage') { - return json(res, 200, panel.buildUsage({ accountQuota, cfg, requestLog })) + return json(res, 200, await panel.buildUsage({ accountQuota, cfg, requestLog })) } // GET /api/panel/models if (req.method === 'GET' && p === '/api/panel/models') { @@ -5014,7 +5048,7 @@ export function createPanelHandler(ctx) { 200, panel.ok({ policy: pol, - models: listPolicyModels(), + models: await listPolicyModels(), effective: listOfficialModels(), platform: 'anthropic', }), @@ -5024,18 +5058,18 @@ export function createPanelHandler(ctx) { const body = await readBody(req, 2 * 1024 * 1024) const platform = String(body.platform || url.searchParams.get('platform') || '').toLowerCase() if (platform === 'openai' || platform === 'gpt' || platform === 'codex') { - const saved = saveGptModelPolicy(body.policy || body) + const saved = await saveGptModelPolicy(body.policy || body) const models = listGptPolicyModels() return json(res, 200, panel.ok({ policy: saved, models, effective: models, platform: 'openai' })) } - const saved = saveModelPolicy(body.policy || body) + const saved = await saveModelPolicy(body.policy || body) seedModelCatalog() return json( res, 200, panel.ok({ policy: saved, - models: listPolicyModels(), + models: await listPolicyModels(), effective: listOfficialModels(), platform: 'anthropic', }), @@ -5046,18 +5080,18 @@ export function createPanelHandler(ctx) { const body = await readBody(req, 8192).catch(() => ({})) const platform = String(body.platform || url.searchParams.get('platform') || '').toLowerCase() if (platform === 'openai' || platform === 'gpt' || platform === 'codex') { - const saved = resetGptModelPolicy() + const saved = await resetGptModelPolicy() const models = listGptPolicyModels() return json(res, 200, panel.ok({ policy: saved, models, effective: models, platform: 'openai' })) } - const saved = resetModelPolicy() + const saved = await resetModelPolicy() seedModelCatalog() return json( res, 200, panel.ok({ policy: saved, - models: listPolicyModels(), + models: await listPolicyModels(), effective: listOfficialModels(), platform: 'anthropic', }), @@ -5078,7 +5112,7 @@ export function createPanelHandler(ctx) { 200, panel.ok({ policy: getModelPolicy(), - models: listPolicyModels(), + models: await listPolicyModels(), effective: listOfficialModels(), synced: status.added.length, added: status.added, @@ -5088,12 +5122,12 @@ export function createPanelHandler(ctx) { } // GET /api/panel/pricing — effective price catalog for the 计费 tab if (req.method === 'GET' && p === '/api/panel/pricing') { - return json(res, 200, panel.ok(buildPricing({ requestLog }))) + return json(res, 200, panel.ok(await buildPricing({ requestLog }))) } // PUT /api/panel/pricing/overrides { key, price|null } if (req.method === 'PUT' && p === '/api/panel/pricing/overrides') { const body = await readBody(req, 256 * 1024) - const saved = setPriceOverride(String(body?.key || ''), body?.price ?? null) + const saved = await setPriceOverride(String(body?.key || ''), body?.price ?? null) if (!saved.ok) { return json(res, 400, { ok: false, error: { code: 'invalid_price', message: saved.error } }) } @@ -5114,7 +5148,7 @@ export function createPanelHandler(ctx) { error: { code: 'invalid_range', message: 'since/until must be ISO times with since < until' }, }) } - const result = requestLog.repo.repriceRange({ + const result = await requestLog.repo.repriceRange({ since: new Date(since).toISOString(), until: new Date(until).toISOString(), }) @@ -5240,9 +5274,9 @@ export function createPanelHandler(ctx) { const { applied: appliedDuringSwitch, ...publicEngineRuntime } = engineRuntime let applied try { - applied = appliedDuringSwitch ?? persistRoutingPatch(body) + applied = appliedDuringSwitch ?? (await persistRoutingPatch(body)) } catch (error) { - restoreRoutingRuntime(previousRoutingConfig) + await restoreRoutingRuntime(previousRoutingConfig) return json(res, 503, { ok: false, error: { code: 'routing_persist_failed', message: String(error?.message || error) }, @@ -5282,11 +5316,11 @@ export function createPanelHandler(ctx) { return json(res, 200, panel.ok(cfg.distill)) } if (req.method === 'GET' && p === '/api/panel/refusal-guards') { - return json(res, 200, panel.ok(refusalGuardSnapshot())) + return json(res, 200, panel.ok(await refusalGuardSnapshot())) } if (req.method === 'PUT' && p === '/api/panel/refusal-guards') { const body = await readBody(req, 8 * 1024).catch(() => ({})) - const patched = applyRefusalGuardPatch(new SettingsRepo(), body || {}) + const patched = await applyRefusalGuardPatch(new SettingsRepo(), body || {}) if (!patched.ok) { return json(res, 400, { ok: false, @@ -5297,18 +5331,18 @@ export function createPanelHandler(ctx) { }, }) } - return json(res, 200, panel.ok(refusalGuardSnapshot())) + return json(res, 200, panel.ok(await refusalGuardSnapshot())) } if (req.method === 'DELETE' && /^\/api\/panel\/refusal-guards\/[0-9a-f]{64}$/i.test(p)) { const fp = p.split('/').pop() - const removed = new RefusalGuardsRepo().remove(fp) + const removed = await new RefusalGuardsRepo().remove(fp) if (!removed) { return json(res, 404, { ok: false, error: { type: 'not_found_error', code: 'refusal_guard_not_found', message: '指纹不存在' }, }) } - return json(res, 200, panel.ok(refusalGuardSnapshot())) + return json(res, 200, panel.ok(await refusalGuardSnapshot())) } if (req.method === 'DELETE' && p === '/api/panel/refusal-guards') { const body = await readBody(req, 8 * 1024).catch(() => ({})) @@ -5322,21 +5356,21 @@ export function createPanelHandler(ctx) { }, }) } - new RefusalGuardsRepo().clear() - return json(res, 200, panel.ok(refusalGuardSnapshot())) + await new RefusalGuardsRepo().clear() + return json(res, 200, panel.ok(await refusalGuardSnapshot())) } if (req.method === 'DELETE' && p === '/api/panel/refusal-device-blocks') { const body = await readBody(req, 8 * 1024).catch(() => ({})) const repo = new RefusalDeviceBlocksRepo() const deviceId = String(body?.device_id || '').trim() if (deviceId) { - if (!repo.remove(deviceId)) { + if (!(await repo.remove(deviceId))) { return json(res, 404, { ok: false, error: { type: 'not_found_error', code: 'refusal_device_not_found', message: 'device 不存在' }, }) } - return json(res, 200, panel.ok(refusalGuardSnapshot())) + return json(res, 200, panel.ok(await refusalGuardSnapshot())) } if (body?.confirm !== true) { return json(res, 400, { @@ -5348,18 +5382,18 @@ export function createPanelHandler(ctx) { }, }) } - repo.clear() - return json(res, 200, panel.ok(refusalGuardSnapshot())) + await repo.clear() + return json(res, 200, panel.ok(await refusalGuardSnapshot())) } if (req.method === 'GET' && p === '/api/panel/protocol-entry') { - return json(res, 200, panel.ok(requestLog.protocolEntryStats())) + return json(res, 200, panel.ok(await requestLog.protocolEntryStats())) } if (req.method === 'GET' && p === '/api/panel/jev-intercept') { return json(res, 200, panel.ok(jevInterceptSnapshot())) } if (req.method === 'PUT' && p === '/api/panel/jev-intercept') { const body = await readBody(req, 64 * 1024).catch(() => ({})) - const patched = applyJevPatch(new SettingsRepo(), body || {}) + const patched = await applyJevPatch(new SettingsRepo(), body || {}) if (!patched.ok) { return json(res, 400, { ok: false, @@ -5862,7 +5896,7 @@ export function createPanelHandler(ctx) { // ---- intercept rules admin ---- if (p === '/admin/intercept/rules') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return if (req.method === 'GET') { return json(res, 200, { rules: reloadRules() }) } @@ -5884,11 +5918,11 @@ export function createPanelHandler(ctx) { } if (req.method === 'GET' && p === '/admin/vm/oauth') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return return json(res, 200, await oauthStatusWithWorker()) } if (req.method === 'POST' && p === '/admin/vm/oauth/refresh') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const body = await readBody(req, 4096).catch(() => ({})) const id = String(body.vm_id || getActiveVmId(cfg.paths.project) || '').trim() const vmPath = path.join(cfg.paths.project, 'vms', `${id}.json`) @@ -5907,13 +5941,13 @@ export function createPanelHandler(ctx) { // Claude CLI inference/update was removed; workers are built and deployed separately. if (req.method === 'GET' && p === '/admin/vm/claude-code/version') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return return json(res, 410, { error: { code: 'claude_cli_removed', message: 'Claude CLI runtime was replaced by the Go slot worker' }, }) } if (req.method === 'POST' && p === '/admin/vm/claude-code/update') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return return json(res, 410, { error: { code: 'claude_cli_removed', message: 'Build and roll out the Go slot worker instead' }, }) @@ -5921,6 +5955,33 @@ export function createPanelHandler(ctx) { return false } + + return async (req, res, url) => { + if (['GET', 'HEAD', 'OPTIONS'].includes(req.method)) return handlePanel(req, res, url) + try { + await flushPanelWrites() + } catch (error) { + return persistenceFailure(res, error) + } + const state = {} + let result + res._panelResponseState = state + try { + result = await handlePanel(req, res, url) + } finally { + delete res._panelResponseState + } + try { + await flushPanelWrites() + } catch (error) { + return persistenceFailure(res, error) + } + if (state.response) { + ctx.json(...state.response) + return true + } + return result + } } function readTelemetryUploadLog(projectRoot, id) { diff --git a/src/lib/admin/panel-tenant.mjs b/src/lib/admin/panel-tenant.mjs index c6a3865c4..726774af2 100644 --- a/src/lib/admin/panel-tenant.mjs +++ b/src/lib/admin/panel-tenant.mjs @@ -64,10 +64,10 @@ export function denyIfUserCannotDeleteVm(req, res, vm, json) { return true } -export function denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId }) { +export async function denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId }) { const ident = panelIdentity(req) if (ident.role !== 'user') return false - const rec = apiKeyStore?.repo?.getById?.(keyId) || null + const rec = (await apiKeyStore?.repo?.getById?.(keyId)) || null const owner = normalizeOwnerId(rec?.user_id) if (!rec || owner !== normalizeOwnerId(req.panelUserId)) { json(res, 404, { ok: false, error: { message: 'api key not found' } }) diff --git a/src/lib/admin/panel-users.mjs b/src/lib/admin/panel-users.mjs index 90a0787a1..f71725ff0 100644 --- a/src/lib/admin/panel-users.mjs +++ b/src/lib/admin/panel-users.mjs @@ -6,7 +6,7 @@ */ import crypto from 'node:crypto' -import { resolveStoreDb } from '../db/database.mjs' +import { resolveStoreDb, withTransaction } from '../db/database.mjs' import { UsersRepo } from '../db/repos/users-repo.mjs' import { clampVmCreateQuota } from './resource-owner.mjs' import { reclaimAssignedVms } from '../vm/vm-registry.mjs' @@ -129,30 +129,30 @@ export class PanelUserStore { this.repo = new UsersRepo(db) } - list() { - return this.repo.list().map(publicUserView) + async list() { + return (await this.repo.list()).map(publicUserView) } - getById(id) { - return this.repo.getById(id) + async getById(id) { + return await this.repo.getById(id) } - getByUsername(username) { - return this.repo.getByUsername(username) + async getByUsername(username) { + return await this.repo.getByUsername(username) } - countEnabledAdmins() { - return this.repo.countEnabledRole('admin') + async countEnabledAdmins() { + return await this.repo.countEnabledRole('admin') } - bootstrapFromEnv({ username, password } = {}) { + async bootstrapFromEnv({ username, password } = {}) { const name = normalizeUsername(username) if (!name || !password) return null - const existing = this.repo.getByUsername(name) + const existing = await this.repo.getByUsername(name) if (existing) return publicUserView(existing) // Import the already accepted env credential; new passwords still go through assertPassword. return publicUserView( - this.repo.insert({ + await this.repo.insert({ id: crypto.randomUUID(), username: assertUsername(name), password_hash: hashPassword(password), @@ -166,18 +166,18 @@ export class PanelUserStore { ) } - authenticate(username, password) { - const rec = this.repo.getByUsername(username) + async authenticate(username, password) { + const rec = await this.repo.getByUsername(username) if (!rec || rec.enabled === false) return null if (!verifyPassword(password, rec.password_hash)) return null const next = { ...rec, last_login_at: nowIso(), updated_at: rec.updated_at } - this.repo.update(next) + await this.repo.touchLogin(rec.id, next.last_login_at) return publicUserView(next) } - create({ username, password, role = 'user', enabled = true, vm_create_quota = 0 } = {}) { + async create({ username, password, role = 'user', enabled = true, vm_create_quota = 0 } = {}) { const name = assertUsername(username) - if (this.repo.getByUsername(name)) { + if (await this.repo.getByUsername(name)) { const err = new Error('用户名已存在') err.code = 'username_exists' throw err @@ -193,66 +193,78 @@ export class PanelUserStore { updated_at: nowIso(), last_login_at: null, } - return this.repo.insert(rec) + return await this.repo.insert(rec) } - update(id, patch = {}, { actorId = null } = {}) { - const rec = this.repo.getById(id) - if (!rec) return null - const next = { ...rec } + async update(id, patch = {}, { actorId = null } = {}) { + return withTransaction(this.db, async () => { + await this.db + .prepare("SELECT id FROM users WHERE role = 'admin' AND deleted_at IS NULL ORDER BY id FOR UPDATE") + .all() - if (patch.role != null) { - const role = assertRole(patch.role) - if (rec.role === 'admin' && role !== 'admin' && this.countEnabledAdmins() <= 1) { - const err = new Error('不能取消最后一个管理员') - err.code = 'last_admin' - throw err - } - next.role = role - } + const rec = await this.repo.getById(id, { forUpdate: true }) + if (!rec) return null + const next = { ...rec } - if (patch.enabled != null) { - const enabled = !!patch.enabled - if (rec.role === 'admin' && rec.enabled && !enabled && this.countEnabledAdmins() <= 1) { - const err = new Error('不能停用最后一个管理员') - err.code = 'last_admin' - throw err + if (patch.role != null) { + const role = assertRole(patch.role) + if (rec.role === 'admin' && role !== 'admin' && (await this.countEnabledAdmins()) <= 1) { + const err = new Error('不能取消最后一个管理员') + err.code = 'last_admin' + throw err + } + next.role = role } - if (actorId && actorId === rec.id && !enabled) { - const err = new Error('不能停用当前登录账号') - err.code = 'self_disable' - throw err + + if (patch.enabled != null) { + const enabled = !!patch.enabled + if (rec.role === 'admin' && rec.enabled && !enabled && (await this.countEnabledAdmins()) <= 1) { + const err = new Error('不能停用最后一个管理员') + err.code = 'last_admin' + throw err + } + if (actorId && actorId === rec.id && !enabled) { + const err = new Error('不能停用当前登录账号') + err.code = 'self_disable' + throw err + } + next.enabled = enabled } - next.enabled = enabled - } - if (patch.password != null && String(patch.password) !== '') { - next.password_hash = hashPassword(assertPassword(patch.password)) - } + if (patch.password != null && String(patch.password) !== '') { + next.password_hash = hashPassword(assertPassword(patch.password)) + } - if (patch.vm_create_quota != null) { - next.vm_create_quota = clampVmCreateQuota(patch.vm_create_quota, rec.vm_create_quota || 0) - } + if (patch.vm_create_quota != null) { + next.vm_create_quota = clampVmCreateQuota(patch.vm_create_quota, rec.vm_create_quota || 0) + } - next.updated_at = nowIso() - return this.repo.update(next) + next.updated_at = nowIso() + return await this.repo.update(next) + }) } - remove(id, { actorId = null } = {}) { - const rec = this.repo.getById(id) - if (!rec) return false - if (actorId && actorId === rec.id) { - const err = new Error('不能删除当前登录账号') - err.code = 'self_delete' - throw err - } - if (rec.role === 'admin' && rec.enabled && this.countEnabledAdmins() <= 1) { - const err = new Error('不能删除最后一个管理员') - err.code = 'last_admin' - throw err - } - // Validate deletion first; a failed slot write must leave the user available for retry. - if (this.projectRoot) reclaimAssignedVms(this.projectRoot, new Set([rec.id])) - return this.repo.remove(id) + async remove(id, { actorId = null } = {}) { + return withTransaction(this.db, async () => { + await this.db + .prepare("SELECT id FROM users WHERE role = 'admin' AND deleted_at IS NULL ORDER BY id FOR UPDATE") + .all() + + const rec = await this.repo.getById(id) + if (!rec) return false + if (actorId && actorId === rec.id) { + const err = new Error('不能删除当前登录账号') + err.code = 'self_delete' + throw err + } + if (rec.role === 'admin' && rec.enabled && (await this.countEnabledAdmins()) <= 1) { + const err = new Error('不能删除最后一个管理员') + err.code = 'last_admin' + throw err + } + // Validate deletion first; a failed slot write must leave the user available for retry. + if (this.projectRoot) reclaimAssignedVms(this.projectRoot, new Set([rec.id])) + return await this.repo.remove(id) + }) } } diff --git a/src/lib/admin/price-catalog.mjs b/src/lib/admin/price-catalog.mjs index 90390c293..282fe4e84 100644 --- a/src/lib/admin/price-catalog.mjs +++ b/src/lib/admin/price-catalog.mjs @@ -2,7 +2,7 @@ * Three-layer price catalog: override > models.dev > builtin. * ID matching is exact / alias / Claude date-strip — no fuzzy family regex. */ -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { BUILTIN_ALIASES, BUILTIN_RATES, BUILTIN_SOURCE } from './builtin-rates.mjs' export const PRICE_FIELDS = ['input', 'output', 'cache_read', 'cache_5m', 'cache_1h'] @@ -27,6 +27,7 @@ let injectedFetch = null let backgroundRefresh = true let catalogMemo = null let refreshInflight = null +let refreshPaused = false export function setPriceCatalogTestHooks(hooks = null) { if (!hooks) { @@ -296,12 +297,12 @@ function readOverrides() { return raw && typeof raw === 'object' && !Array.isArray(raw) ? raw : {} } -function persistOverrides(next) { - settingsRepo().set(SETTINGS_OVERRIDES, next) +async function persistOverrides(next) { + await settingsRepo().set(SETTINGS_OVERRIDES, next) } -function persistSnapshot(next) { - settingsRepo().set(SETTINGS_SNAPSHOT, next) +async function persistSnapshot(next) { + await settingsRepo().set(SETTINGS_SNAPSHOT, next) } function statusOf(snap) { @@ -366,7 +367,7 @@ export async function syncModelsDev({ fetchImpl } = {}) { anthropic: parsed.anthropic, openai: parsed.openai, } - persistSnapshot(snap) + await persistSnapshot(snap) invalidatePriceCatalog() return statusOf(snap) } catch (e) { @@ -377,7 +378,7 @@ export async function syncModelsDev({ fetchImpl } = {}) { anthropic: prev.anthropic || {}, openai: prev.openai || {}, } - persistSnapshot(snap) + await persistSnapshot(snap) invalidatePriceCatalog() return statusOf(snap) } @@ -394,7 +395,7 @@ function canPersist() { } export function maybeRefreshModelsDev({ now } = {}) { - if (!backgroundRefresh) return + if (!backgroundRefresh || refreshPaused) return if (!canPersist()) return if (refreshInflight) return const snap = readSnapshot() @@ -402,13 +403,22 @@ export function maybeRefreshModelsDev({ now } = {}) { const t = now ?? Date.now() if (fetchedAt && Number.isFinite(fetchedAt) && t - fetchedAt < MODELS_DEV_STALE_MS) return refreshInflight = Promise.resolve() - .then(() => syncModelsDev()) + .then(async () => await syncModelsDev()) .catch(() => {}) .finally(() => { refreshInflight = null }) } +export async function pausePriceCatalog() { + refreshPaused = true + while (refreshInflight) await refreshInflight +} + +export function resumePriceCatalog() { + refreshPaused = false +} + function storedPrice(entry, platform) { if (!entry || typeof entry !== 'object') return null const { updated_at: _updated, ...rest } = entry @@ -584,7 +594,7 @@ export function listPriceOverrides() { return clone(readOverrides()) } -export function setPriceOverride(key, price) { +export async function setPriceOverride(key, price) { const k = String(key || '').trim() if (!OVERRIDE_KEY_RE.test(k) || k === '*/*') { return { ok: false, error: 'invalid override key' } @@ -592,7 +602,7 @@ export function setPriceOverride(key, price) { const overrides = { ...readOverrides() } if (price == null) { delete overrides[k] - persistOverrides(overrides) + await persistOverrides(overrides) invalidatePriceCatalog() return { ok: true } } @@ -600,7 +610,7 @@ export function setPriceOverride(key, price) { const v = validatePrice(price, { platform: plat }) if (!v.ok) return { ok: false, error: v.error } overrides[k] = { ...v.price, updated_at: new Date().toISOString() } - persistOverrides(overrides) + await persistOverrides(overrides) invalidatePriceCatalog() return { ok: true } } diff --git a/src/lib/admin/pricing-view.mjs b/src/lib/admin/pricing-view.mjs index a7a1539a9..444b994a6 100644 --- a/src/lib/admin/pricing-view.mjs +++ b/src/lib/admin/pricing-view.mjs @@ -14,12 +14,12 @@ function policyPrices() { return out } -export function buildPricing({ requestLog = null } = {}) { +export async function buildPricing({ requestLog = null } = {}) { const policy = policyPrices() const inPolicy = new Set(Object.values(policy).flatMap((row) => (row ? [row.id] : []))) let unpriced = [] try { - unpriced = requestLog?.repo?.unpricedModels?.() || [] + unpriced = (await requestLog?.repo?.unpricedModels?.()) || [] } catch { unpriced = [] } diff --git a/src/lib/admin/request-log.mjs b/src/lib/admin/request-log.mjs index 512ee62c5..2af97d1d7 100644 --- a/src/lib/admin/request-log.mjs +++ b/src/lib/admin/request-log.mjs @@ -1,5 +1,5 @@ /** - * Request logging — sub2api UsageLog + access-log inspired (SQLite-backed). + * Request logging — sub2api UsageLog + access-log inspired (PostgreSQL-backed). * * Modes: * off — no writes @@ -19,6 +19,7 @@ import path from 'node:path' import crypto from 'node:crypto' import { redactSecrets } from '../core/security.mjs' import { resolveStoreDb } from '../db/database.mjs' +import { integerValue } from '../db/integers.mjs' import { UsageLogsRepo } from '../db/repos/usage-logs-repo.mjs' import { classifyRequestError, resolveMutedErrorClasses } from './error-class.mjs' import { costColumnsFromUsage, normalizeUsage } from './pricing.mjs' @@ -114,9 +115,9 @@ function cacheCreationBreakdown(usage) { return { cache_creation_5m_tokens: null, cache_creation_1h_tokens: null } } const nested = usage.cache_creation - let five = Number(nested?.ephemeral_5m_input_tokens) || 0 - let hour = Number(nested?.ephemeral_1h_input_tokens) || 0 - const total = Number(usage.cache_creation_input_tokens ?? usage.cache_creation_tokens) || 0 + let five = integerValue(nested?.ephemeral_5m_input_tokens) + let hour = integerValue(nested?.ephemeral_1h_input_tokens) + const total = integerValue(usage.cache_creation_input_tokens ?? usage.cache_creation_tokens) if (five === 0 && hour === 0 && total > 0) { if (usage.cache_ttl === '1h') hour = total else five = total @@ -128,8 +129,8 @@ function cacheCreationBreakdown(usage) { } function webSearchRequestsOf(usage) { - const n = Number(usage?.server_tool_use?.web_search_requests ?? usage?.web_search_requests) - return Number.isFinite(n) && n > 0 ? Math.trunc(n) : 0 + const n = integerValue(usage?.server_tool_use?.web_search_requests ?? usage?.web_search_requests) + return n > 0 ? n : 0 } const PRICED_COLUMN_KEYS = [ @@ -388,9 +389,10 @@ export class RequestLogStore { this._mem = [] // recent normal summaries for panel hot path this._memMax = 200 this._cleanupTimer = null + this._cleanupPending = new Set() } - setConfig({ mode, retainDays, debugRetainDays, maxMb, mutedErrorClasses } = {}) { + async setConfig({ mode, retainDays, debugRetainDays, maxMb, mutedErrorClasses } = {}) { if (mode != null) { const m = String(mode).trim().toLowerCase() if (MODES.has(m)) this.mode = m @@ -411,7 +413,7 @@ export class RequestLogStore { if (mutedErrorClasses !== undefined) { this.mutedErrorClasses = resolveMutedErrorClasses(mutedErrorClasses) } - return this.snapshot() + return await this.snapshot() } /** Kept for API compat + post-restore hook (state lives in DB). */ @@ -499,12 +501,13 @@ export class RequestLogStore { * Write normal summary always (unless off). * Write debug full record when mode===debug. */ - finish(ctx, extra = {}) { + async finish(ctx, extra = {}) { if (!ctx || ctx.mode === 'off') return null const duration_ms = Math.max(0, Date.now() - (ctx.t0 || Date.now())) - const usage = normalizeUsage(disjointOpenaiUsage(extra.usage || {}) || {}) + const rawUsage = disjointOpenaiUsage(extra.usage || {}) || {} + const usage = normalizeUsage(rawUsage) const breakdown = cacheCreationBreakdown(usage) - const webSearch = extra.web_search_requests ?? webSearchRequestsOf(usage) + const webSearch = extra.web_search_requests ?? webSearchRequestsOf(rawUsage) const billedUsage = { ...usage, input_tokens: extra.input_tokens ?? usage.input_tokens, @@ -570,7 +573,7 @@ export class RequestLogStore { intercept: extra.intercept ? String(extra.intercept).slice(0, 400) : null, // sub2api ownership + billing columns user_id: extra.user_id ?? null, - vm_owner_user_id: this.repo.vmOwnerOf(extra.vm_id), + vm_owner_user_id: await this.repo.vmOwnerOf(extra.vm_id), group_id: extra.group_id ?? 1, rate_multiplier: Number.isFinite(Number(extra.rate_multiplier)) ? Number(extra.rate_multiplier) : 1, ...costColumnsForLog(billedUsage, extra.upstream_model || extra.model || extra.requested_model, extra), @@ -582,9 +585,12 @@ export class RequestLogStore { const classified = classifyRequestError(summary) if (classified?.error_class) summary.error_class = classified.error_class - try { - this.repo.insertSummaryIfAbsent(summary) - } catch {} + // PostgreSQL text rejects NUL; JSON debug bodies can retain the original request. + for (const [key, value] of Object.entries(summary)) { + if (typeof value === 'string') summary[key] = value.replaceAll('\0', '\\u0000') + } + + await this.repo.insertSummaryIfAbsent(summary) if (this.jsonlMirror) this._appendJsonl(summary) this._mem.push(summary) if (this._mem.length > this._memMax) this._mem = this._mem.slice(-this._memMax) @@ -615,9 +621,7 @@ export class RequestLogStore { : null, response: ctx.response ? debugResponse(ctx.response) : null, } - try { - this.repo.insertDebug(summary.request_id || summary.id, summary.ts, debugRec) - } catch {} + await this.repo.insertDebug(summary.request_id || summary.id, summary.ts, debugRec) } return summary } @@ -651,66 +655,66 @@ export class RequestLogStore { return (items || []).filter((row) => !row.error_class || !blocked.has(row.error_class)) } - listNormal({ limit = 50, ...filters } = {}) { - return this.queryNormal({ limit, ...filters }).items + async listNormal({ limit = 50, ...filters } = {}) { + return (await this.queryNormal({ limit, ...filters })).items } /** Filtered + paginated query with total (panel). Default hides muted classes. */ - queryNormal(opts = {}) { + async queryNormal(opts = {}) { const exclude = this._mutedExclude(opts) - const result = this.repo.query({ ...opts, exclude_error_class: exclude }) + const result = await this.repo.query({ ...opts, exclude_error_class: exclude }) result.items = this._dropMuted(result.items, exclude) return result } - exportRows(opts = {}) { + async exportRows(opts = {}) { const limit = Math.max(1, Math.min(5000, Number(opts.limit) || 2000)) - return this.queryNormal({ ...opts, limit, offset: 0, maxLimit: 5000 }) + return await this.queryNormal({ ...opts, limit, offset: 0, maxLimit: 5000 }) } /** Aggregated stats for charts. */ - aggregate(opts = {}) { - return this.repo.aggregate(opts) + async aggregate(opts = {}) { + return await this.repo.aggregate(opts) } - totals() { - return this.repo.totals() + async totals() { + return await this.repo.totals() } - hasSuccessfulRequest(opts = {}) { - return this.repo.hasSuccessfulRequest(opts) + async hasSuccessfulRequest(opts = {}) { + return await this.repo.hasSuccessfulRequest(opts) } - billingStats(opts = {}) { - return this.repo.billingStats(opts) + async billingStats(opts = {}) { + return await this.repo.billingStats(opts) } - ownerBilling(opts = {}) { - return this.repo.ownerBilling(opts) + async ownerBilling(opts = {}) { + return await this.repo.ownerBilling(opts) } - costByModel(opts = {}) { - return this.repo.costByModel(opts) + async costByModel(opts = {}) { + return await this.repo.costByModel(opts) } - vmUsageStats(opts = {}) { - return this.repo.vmUsageStats(opts) + async vmUsageStats(opts = {}) { + return await this.repo.vmUsageStats(opts) } - keyUsageStats(opts = {}) { - return this.repo.keyUsageStats(opts) + async keyUsageStats(opts = {}) { + return await this.repo.keyUsageStats(opts) } /** Windowed SLA / QPS / TTFT snapshot for overview + log analysis. */ - windowStats(opts = {}) { - return this.repo.windowStats(opts) + async windowStats(opts = {}) { + return await this.repo.windowStats(opts) } - protocolEntryStats(opts = {}) { - return this.repo.protocolEntryStats(opts) + async protocolEntryStats(opts = {}) { + return await this.repo.protocolEntryStats(opts) } - listDebug({ + async listDebug({ limit = 20, exclude_error_class = null, include_muted = false, @@ -719,20 +723,20 @@ export class RequestLogStore { } = {}) { const exclude = this._mutedExclude({ exclude_error_class, include_muted, error_class }) const n = Math.max(1, Math.min(100, Number(limit) || 20)) - const pool = this.repo.listDebug({ limit: exclude ? Math.min(200, n * 15) : n, owner_user_id }) + const pool = await this.repo.listDebug({ limit: exclude ? Math.min(200, n * 15) : n, owner_user_id }) const items = error_class ? pool.filter((row) => row.error_class === error_class) : this._dropMuted(pool, exclude) return items.slice(0, n) } - getDebug(requestId, { owner_user_id = null } = {}) { + async getDebug(requestId, { owner_user_id = null } = {}) { if (!requestId) return null - const rec = this.repo.getDebug(requestId) || this.repo.getByRequestId(requestId) + const rec = (await this.repo.getDebug(requestId)) || (await this.repo.getByRequestId(requestId)) if (!rec) return null - if (owner_user_id && !this.repo.belongsToOwner(requestId, owner_user_id)) return null + if (owner_user_id && !(await this.repo.belongsToOwner(requestId, owner_user_id))) return null return rec } - snapshot() { + async snapshot() { return { mode: this.mode, retain_days: this.retainDays, @@ -740,11 +744,11 @@ export class RequestLogStore { max_mb: this.maxMb, muted_error_classes: this.mutedErrorClasses.slice(), recent_normal: this._mem.length, - backend: 'sqlite', + backend: 'postgresql', jsonl_mirror: this.jsonlMirror, - total_rows: (() => { + total_rows: await (async () => { try { - return this.repo.count() + return await this.repo.count() } catch { return null } @@ -753,9 +757,9 @@ export class RequestLogStore { } /** Retention: delete rows older than retain windows, then enforce max_mb. */ - cleanup() { + async cleanup() { try { - this.repo.cleanup({ + await this.repo.cleanup({ retainDays: this.retainDays, debugRetainDays: this.debugRetainDays, maxBytes: this.maxMb > 0 ? this.maxMb * 1024 * 1024 : 0, @@ -769,9 +773,13 @@ export class RequestLogStore { this.stopCleanupScheduler() const ms = Math.max(60_000, Number(intervalMs) || CLEANUP_INTERVAL_MS) const run = () => { - try { - this.cleanup() - } catch {} + const pending = (async () => { + try { + await this.cleanup() + } catch {} + })() + this._cleanupPending.add(pending) + pending.finally(() => this._cleanupPending.delete(pending)) } this._cleanupTimer = setInterval(run, ms) if (typeof this._cleanupTimer.unref === 'function') this._cleanupTimer.unref() @@ -782,6 +790,10 @@ export class RequestLogStore { this._cleanupTimer = null } + async flush() { + while (this._cleanupPending.size) await Promise.all([...this._cleanupPending]) + } + _cleanupLegacyFiles(days, dir) { const cutoff = Date.now() - days * 86400_000 try { diff --git a/src/lib/admin/resource-owner.mjs b/src/lib/admin/resource-owner.mjs index b03bf1156..29144f45b 100644 --- a/src/lib/admin/resource-owner.mjs +++ b/src/lib/admin/resource-owner.mjs @@ -49,13 +49,13 @@ export function ownerScopeFromApiKey(record, { apiKeyKind = null, ownerRole = nu return { type: 'user', userId } } -export function ownerScopeFromRequest(req, usersRepo = null) { +export async function ownerScopeFromRequest(req, usersRepo = null) { const rec = req?.apiKeyRecord || null let ownerRole = null const userId = normalizeOwnerId(rec?.user_id) if (userId && usersRepo?.getById) { try { - ownerRole = usersRepo.getById(userId)?.role || null + ownerRole = (await usersRepo.getById(userId))?.role || null } catch { ownerRole = null } diff --git a/src/lib/admin/routing-runtime.mjs b/src/lib/admin/routing-runtime.mjs index d37618333..88f1cd5a5 100644 --- a/src/lib/admin/routing-runtime.mjs +++ b/src/lib/admin/routing-runtime.mjs @@ -26,7 +26,7 @@ import { PoolScheduler, normalizePoolRouting } from '../pool/pool-scheduler.mjs' import { FailoverRunner } from '../pool/failover-runner.mjs' import { unitCircuit } from '../pool/unit-circuit.mjs' import { RateLimitService } from '../pool/rate-limit-service.mjs' -import { AccountRuntimeRepo } from '../db/repos/account-runtime-repo.mjs' +import { SchedulingRuntime } from '../pool/scheduling-runtime.mjs' import { RequestAttemptsRepo } from '../db/repos/request-attempts-repo.mjs' import { normalizeOfficialCcConfig } from '../oauth/official-cc-bootstrap.mjs' import { normalizeHealthProbeConfig } from './health-probe.mjs' @@ -429,7 +429,7 @@ export function createRoutingRuntime(ctx) { return applied } - function persistRoutingPatch(body = {}) { + async function persistRoutingPatch(body = {}) { const hasCodexQuota = body?.codex && Object.prototype.hasOwnProperty.call(body.codex, 'quota') if (hasCodexQuota) validateOpenAIQuotaPatch(body.codex.quota) validateInferenceWebsocketPatch(body) @@ -508,7 +508,7 @@ export function createRoutingRuntime(ctx) { syncTierDefaultsIntoRouting() if (body.logging) { routingConfig.logging = normalizeLoggingConfig({ ...(routingConfig.logging || {}), ...body.logging }) - ctx.requestLog.setConfig({ + await ctx.requestLog.setConfig({ mode: routingConfig.logging.mode, retainDays: routingConfig.logging.retain_days, debugRetainDays: routingConfig.logging.debug_retain_days, @@ -682,15 +682,15 @@ export function createRoutingRuntime(ctx) { }) } - function initPoolRuntime() { + async function initPoolRuntime() { const routingConfig = getRouting() - const runtimeRepo = new AccountRuntimeRepo() + const runtimeRepo = await new SchedulingRuntime().init() const attemptsRepo = new RequestAttemptsRepo() setRuntimeRepo(runtimeRepo) setAttemptsRepo(attemptsRepo) ctx.accountQuota.attachRuntimeRepo(runtimeRepo) try { - attemptsRepo.cleanup(routingConfig?.logging?.retain_days || 7) + await attemptsRepo.cleanup(routingConfig?.logging?.retain_days || 7) } catch {} const poolScheduler = new PoolScheduler({ projectRoot: ctx.cfg.paths.project, diff --git a/src/lib/admin/usage-profile.mjs b/src/lib/admin/usage-profile.mjs index 140e65ff4..dab6d408b 100644 --- a/src/lib/admin/usage-profile.mjs +++ b/src/lib/admin/usage-profile.mjs @@ -88,24 +88,24 @@ export function riskPatchForProfile(mode) { /** * @param {{ mode: string, persistRouting: (patch: object) => unknown, settings: { get: Function, set: Function }, distillFile: string, setDistill?: (rules: object) => void }} input */ -export function applyUsageProfile({ mode, persistRouting, settings, distillFile, setDistill } = {}) { +export async function applyUsageProfile({ mode, persistRouting, settings, distillFile, setDistill } = {}) { const profile = normalizeUsageProfile(mode) if (!profile) { const err = new Error('mode 必须是 personal 或 relay') err.code = 'invalid_profile' throw err } - const routing = persistRouting(routingPatchForProfile(profile)) + const routing = await persistRouting(routingPatchForProfile(profile)) const risk = riskPatchForProfile(profile) const distill = saveDistillRules(distillFile, risk.distill) if (typeof setDistill === 'function') setDistill(distill) - const jev = applyJevPatch(settings, risk.jev) + const jev = await applyJevPatch(settings, risk.jev) if (!jev.ok) { const err = new Error(jev.problems.join('；')) err.code = 'invalid_profile' throw err } - const refusal = applyRefusalGuardPatch(settings, risk.refusal) + const refusal = await applyRefusalGuardPatch(settings, risk.refusal) if (!refusal.ok) { const err = new Error(refusal.problems.join('；')) err.code = 'invalid_profile' diff --git a/src/lib/admin/vm-pools.mjs b/src/lib/admin/vm-pools.mjs index 9307f6cb7..131b9acf8 100644 --- a/src/lib/admin/vm-pools.mjs +++ b/src/lib/admin/vm-pools.mjs @@ -33,43 +33,43 @@ export function normalizePoolMembers(value, vms) { } return ids } -export function createVmPool(repo, { name, vm_ids, enabled = true } = {}, vms = []) { +export async function createVmPool(repo, { name, vm_ids, enabled = true } = {}, vms = []) { const ids = vm_ids == null ? null : normalizePoolMembers(vm_ids, vms) - const pool = repo.insert({ name: normalizePoolName(name), enabled: enabled !== false }) + const pool = await repo.insert({ name: normalizePoolName(name), enabled: enabled !== false }) if (ids == null) return pool try { - return repo.setMembers(pool.id, ids) + return await repo.setMembers(pool.id, ids) } catch (error) { - repo.remove(pool.id) + await repo.remove(pool.id) throw error } } -export function updateVmPool(repo, id, patch = {}, vms = []) { - let pool = repo.get(id) +export async function updateVmPool(repo, id, patch = {}, vms = []) { + let pool = await repo.get(id) if (!pool) return null if (patch.name != null) { - pool = repo.rename(id, normalizePoolName(patch.name)) + pool = await repo.rename(id, normalizePoolName(patch.name)) } if (patch.enabled != null) { if (typeof patch.enabled !== 'boolean') { throw Object.assign(new Error('enabled 必须是布尔值'), { code: 'invalid_vm_pool_enabled' }) } - pool = repo.setEnabled(id, patch.enabled) + pool = await repo.setEnabled(id, patch.enabled) } if (Object.prototype.hasOwnProperty.call(patch, 'vm_ids')) { - pool = repo.setMembers(id, normalizePoolMembers(patch.vm_ids, vms)) + pool = await repo.setMembers(id, normalizePoolMembers(patch.vm_ids, vms)) } return pool } -export function deleteVmPool(repo, id) { - if (!repo.get(id)) return { ok: false, error: 'vm_pool_not_found' } - const keys = repo.countKeys(id) +export async function deleteVmPool(repo, id) { + if (!(await repo.get(id))) return { ok: false, error: 'vm_pool_not_found' } + const keys = await repo.countKeys(id) if (keys > 0) { return { ok: false, error: 'vm_pool_in_use', keys } } - repo.remove(id) + await repo.remove(id) return { ok: true } } @@ -86,9 +86,9 @@ export function vmPoolHttpStatus(code) { return 400 } -export function assertVmPoolExists(repo, id) { +export async function assertVmPoolExists(repo, id) { if (!id) return - if (!repo.get(id)) { + if (!(await repo.get(id))) { throw Object.assign(new Error('账号池不存在'), { code: 'vm_pool_unknown' }) } } diff --git a/src/lib/admin/vm-test-chat.mjs b/src/lib/admin/vm-test-chat.mjs index cba5689c7..089f4aa37 100644 --- a/src/lib/admin/vm-test-chat.mjs +++ b/src/lib/admin/vm-test-chat.mjs @@ -76,7 +76,7 @@ function gptRank(id) { return mini * 1000 - major * 10 - minor } -export function listTestableModels(platform) { +export async function listTestableModels(platform) { const kind = normalizeTestPlatform(platform) const rank = (id) => { if (kind === 'openai' || isGptSeriesId(id)) return gptRank(id) @@ -100,17 +100,15 @@ export function listTestableModels(platform) { const effective = listOfficialModels().map((m) => map(m, 'effective')) const list = effective.length ? effective - : listPolicyModels() - .filter((m) => m.enabled !== false) - .map((m) => map(m, 'policy')) + : (await listPolicyModels()).filter((m) => m.enabled !== false).map((m) => map(m, 'policy')) const filtered = list.filter((m) => !isGptTestModel(m)) return filtered.sort((a, b) => rank(a.id) - rank(b.id) || String(a.id).localeCompare(String(b.id))) } -export function testModelsView(platform) { +export async function testModelsView(platform) { const kind = normalizeTestPlatform(platform) return { - items: listTestableModels(kind), + items: await listTestableModels(kind), platform: kind, protocol: kind === 'openai' ? 'openai.responses' : kind === 'anthropic' ? 'anthropic.messages' : null, inbound_path: kind === 'openai' ? '/v1/responses' : kind === 'anthropic' ? '/v1/messages' : null, @@ -207,8 +205,8 @@ export async function syncCodexCatalog({ let access = account.access_token || target.codex?.access_token const accountId = account.chatgpt_account_id || target.codex?.chatgpt_account_id const refreshToken = account.refresh_token || target.codex?.refresh_token - const fetchOnce = () => - fetchChatgptModelCatalog({ + const fetchOnce = async () => + await fetchChatgptModelCatalog({ accessToken: access, accountId, proxyUrl, @@ -273,7 +271,7 @@ export async function syncCodexCatalog({ } } const models = [...merged.values()] - const { added } = syncGptIdsIntoPolicy(models, { displayNameOf: (id) => modelsDevMeta('openai', id)?.name }) + const { added } = await syncGptIdsIntoPolicy(models, { displayNameOf: (id) => modelsDevMeta('openai', id)?.name }) return { ok: true, vm_id: usedVm, @@ -321,7 +319,7 @@ export async function resolveTestModels({ }) if (live.ok) source = 'chatgpt' } - return { ...testModelsView(kind), source } + return { ...(await testModelsView(kind)), source } } function extractText(body) { @@ -851,11 +849,11 @@ async function postV1Loopback({ path, baseUrl, apiKey, inbound, headers, vmId, t } async function postV1Messages(opts) { - return postV1Loopback({ ...opts, path: '/v1/messages' }) + return await postV1Loopback({ ...opts, path: '/v1/messages' }) } async function postV1Responses(opts) { - return postV1Loopback({ ...opts, path: '/v1/responses' }) + return await postV1Loopback({ ...opts, path: '/v1/responses' }) } /** @@ -968,7 +966,7 @@ export async function runVmTestChat(opts = {}) { let model = String(opts.model || '').trim() if (!model) { - const models = listTestableModels(codex ? 'openai' : 'anthropic') + const models = await listTestableModels(codex ? 'openai' : 'anthropic') model = models[0]?.id || (codex ? 'gpt-5.4' : 'claude-haiku-4-5-20251001') } push('info', `模型 ${model}`) diff --git a/src/lib/admin/vm-test-schedule.mjs b/src/lib/admin/vm-test-schedule.mjs index 99f622c6c..4134a886d 100644 --- a/src/lib/admin/vm-test-schedule.mjs +++ b/src/lib/admin/vm-test-schedule.mjs @@ -166,8 +166,8 @@ function assertCron(cron, timezone) { } } -function defaultDialogParams(vm) { - const models = listTestableModels(isCodexVm(vm) ? 'openai' : 'anthropic') +async function defaultDialogParams(vm) { + const models = await listTestableModels(isCodexVm(vm) ? 'openai' : 'anthropic') const model = models[0]?.id || '' if (!model) { throw new VmTestScheduleError('no test model available for this vm', 'test_model_missing', 400) @@ -353,6 +353,7 @@ export class VmTestScheduleRunner { this.now = now this.jobs = new Map() this.running = new Set() + this.writeFailure = null this.started = false } @@ -404,15 +405,15 @@ export class VmTestScheduleRunner { } } - saveDefaultTemplate(input) { + async saveDefaultTemplate(input) { const structure = normalizeScheduleStructure(input) compileStructuredSchedule(structure) - this._settings().set(VM_TEST_SCHEDULE_DEFAULT_KEY, structure) + await this._settings().set(VM_TEST_SCHEDULE_DEFAULT_KEY, structure) return structure } - clearDefaultTemplate() { - this._settings().remove?.(VM_TEST_SCHEDULE_DEFAULT_KEY) + async clearDefaultTemplate() { + await this._settings().remove?.(VM_TEST_SCHEDULE_DEFAULT_KEY) return null } @@ -472,18 +473,18 @@ export class VmTestScheduleRunner { return null } - save(vmId, input) { + async save(vmId, input) { if (!isValidVmId(vmId)) throw new VmTestScheduleError('invalid vm id', 'invalid_vm_id') const vm = this._vm(vmId) if (!vm) throw new VmTestScheduleError('vm not found', 'vm_not_found', 404) const previous = this._settings().get(keyFor(vmId)) || {} const config = normalizeVmTestSchedule(input, vm) - this._settings().set(keyFor(vmId), { ...config, ...savedResult(previous), revision: randomUUID() }) + await this._settings().set(keyFor(vmId), { ...config, ...savedResult(previous), revision: randomUUID() }) if (this.started) this._syncJob(vmId, config) return this.get(vmId) } - applyScheduleStructure(vmId, structure, { preserveDialog = true } = {}) { + async applyScheduleStructure(vmId, structure, { preserveDialog = true } = {}) { if (!isValidVmId(vmId)) throw new VmTestScheduleError('invalid vm id', 'invalid_vm_id') const vm = this._vm(vmId) if (!vm) throw new VmTestScheduleError('vm not found', 'vm_not_found', 404) @@ -500,7 +501,7 @@ export class VmTestScheduleRunner { reasoning_effort: previous.reasoning_effort ?? null, } } else { - dialog = defaultDialogParams(vm) + dialog = await defaultDialogParams(vm) } const enabled = previous.enabled === true const config = normalizeVmTestSchedule( @@ -511,7 +512,7 @@ export class VmTestScheduleRunner { }, vm, ) - this._settings().set(keyFor(vmId), { + await this._settings().set(keyFor(vmId), { ...config, enabled, ...savedResult(previous), @@ -526,7 +527,7 @@ export class VmTestScheduleRunner { * A slot of the other kind (Claude vs Codex) keeps its own dialog params, since * a model of one kind cannot run on the other; switch and timing still apply. */ - syncSchedules({ scope, vmIds = [], schedule, test = null }) { + async syncSchedules({ scope, vmIds = [], schedule, test = null }) { const structure = normalizeScheduleStructure(schedule) const full = test == null ? null : normalizeSyncTest(test, structure) let targets = [] @@ -547,16 +548,16 @@ export class VmTestScheduleRunner { continue } if (full) { - this._applyFullTest(vmId, structure, full) + await this._applyFullTest(vmId, structure, full) updated.push(vmId) continue } const hadPlan = !!this._settings().get(keyFor(vmId)) - this.applyScheduleStructure(vmId, structure, { preserveDialog: hadPlan }) + await this.applyScheduleStructure(vmId, structure, { preserveDialog: hadPlan }) if (!hadPlan) { const stored = this._settings().get(keyFor(vmId)) if (stored) { - this._settings().set(keyFor(vmId), { ...stored, enabled: false }) + await this._settings().set(keyFor(vmId), { ...stored, enabled: false }) this.jobs.get(vmId)?.stop() this.jobs.delete(vmId) } @@ -566,7 +567,7 @@ export class VmTestScheduleRunner { return { updated, missing, schedule: structure } } - _applyFullTest(vmId, structure, full) { + async _applyFullTest(vmId, structure, full) { const vm = this._vm(vmId) const previous = this._settings().get(keyFor(vmId)) || {} let dialog @@ -578,22 +579,27 @@ export class VmTestScheduleRunner { max_tokens: previous.max_tokens ?? null, reasoning_effort: previous.reasoning_effort ?? null, } - } else dialog = defaultDialogParams(vm) + } else dialog = await defaultDialogParams(vm) const config = normalizeVmTestSchedule({ enabled: full.enabled, ...structure, ...dialog }, vm) - this._settings().set(keyFor(vmId), { ...config, ...savedResult(previous), revision: randomUUID() }) + await this._settings().set(keyFor(vmId), { ...config, ...savedResult(previous), revision: randomUUID() }) if (this.started) this._syncJob(vmId, config) } - remove(vmId) { + async remove(vmId) { this.jobs.get(vmId)?.stop() this.jobs.delete(vmId) - this._settings().remove?.(keyFor(vmId)) + await this._settings().remove?.(keyFor(vmId)) return null } - start() { + async start() { this.started = true - this.reload() + await this.reload() + } + + async flush() { + while (this.running.size) await new Promise((resolve) => setTimeout(resolve, 10)) + if (this.writeFailure) throw this.writeFailure } stop() { @@ -602,7 +608,7 @@ export class VmTestScheduleRunner { this.jobs.clear() } - reload() { + async reload() { for (const job of this.jobs.values()) job.stop() this.jobs.clear() const all = this._settings().all?.() || {} @@ -611,7 +617,7 @@ export class VmTestScheduleRunner { const vmId = key.slice(VM_TEST_SCHEDULE_PREFIX.length) const vm = this._vm(vmId) if (!vm) { - this._settings().remove?.(key) + await this._settings().remove?.(key) continue } const schedule = this._stored(vmId) @@ -623,8 +629,10 @@ export class VmTestScheduleRunner { this.jobs.get(vmId)?.stop() this.jobs.delete(vmId) if (!this.started || !schedule.enabled || !this._vm(vmId)) return - const job = new Cron(schedule.cron, { timezone: schedule.timezone, mode: '5-part', catch: true, unref: true }, () => - this._run(vmId), + const job = new Cron( + schedule.cron, + { timezone: schedule.timezone, mode: '5-part', catch: true, unref: true }, + async () => await this._run(vmId), ) this.jobs.set(vmId, job) } @@ -632,40 +640,47 @@ export class VmTestScheduleRunner { async _run(vmId) { if (!this.started || this.running.has(vmId)) return if (!this._vm(vmId)) { - this.remove(vmId) + await this.remove(vmId) return } const schedule = this._stored(vmId) if (!schedule?.enabled) return this.running.add(vmId) const startedAt = this.now() - let result try { - result = await this.runTest(vmId, { - model: schedule.model, - prompt: schedule.prompt, - max_tokens: schedule.max_tokens, - reasoning_effort: schedule.reasoning_effort, - }) - } catch (error) { - result = { ok: false, error } + let result + try { + result = await this.runTest(vmId, { + model: schedule.model, + prompt: schedule.prompt, + max_tokens: schedule.max_tokens, + reasoning_effort: schedule.reasoning_effort, + }) + } catch (error) { + result = { ok: false, error } + } + if (!this.started) return + + const current = this._settings().get(keyFor(vmId)) + if (!current || String(current.revision || '') !== schedule.revision || !this._vm(vmId)) return + const duration = Number(result?.duration_ms) + const lastDuration = + Number.isSafeInteger(duration) && duration >= 0 ? duration : Math.max(0, this.now() - startedAt) + const lastOk = result?.ok === true + try { + await this._settings().set(keyFor(vmId), { + ...current, + last_run_at: new Date(startedAt).toISOString(), + last_ok: lastOk, + last_duration_ms: lastDuration, + last_error: lastOk ? null : compactError(result?.error), + }) + } catch (error) { + this.writeFailure ||= error + throw error + } } finally { this.running.delete(vmId) } - if (!this.started) return - - const current = this._settings().get(keyFor(vmId)) - if (!current || String(current.revision || '') !== schedule.revision || !this._vm(vmId)) return - const duration = Number(result?.duration_ms) - const lastDuration = - Number.isSafeInteger(duration) && duration >= 0 ? duration : Math.max(0, this.now() - startedAt) - const lastOk = result?.ok === true - this._settings().set(keyFor(vmId), { - ...current, - last_run_at: new Date(startedAt).toISOString(), - last_ok: lastOk, - last_duration_ms: lastDuration, - last_error: lastOk ? null : compactError(result?.error), - }) } } diff --git a/src/lib/cluster/cluster-manager.mjs b/src/lib/cluster/cluster-manager.mjs index 90338f7dd..4fe95bc48 100644 --- a/src/lib/cluster/cluster-manager.mjs +++ b/src/lib/cluster/cluster-manager.mjs @@ -5,6 +5,7 @@ */ import crypto from 'node:crypto' +import { withTransaction } from '../db/database.mjs' import fs from 'node:fs' import net from 'node:net' import path from 'node:path' @@ -137,6 +138,9 @@ export class ClusterManager { this._localAt = 0 this._localInflight = null this.repo = repo + this.nodes = new Map() + this.platforms = new Map() + this.directLinksState = new Map() // Unix sockets need a native filesystem; WSL drvfs (/mnt/) refuses them. this.socketDir = socketDir || path.join(dataDir, 'cluster') this.vmsOnNode = vmsOnNode @@ -153,8 +157,27 @@ export class ClusterManager { this._healthTimer = null } - start() { - for (const node of this.repo.list()) this._attach(node) + async reload() { + const nodes = await this.repo.list() + const directLinks = this.repo.directLinks ? await this.repo.directLinks() : new Map() + const platforms = await Promise.all( + nodes.map(async (node) => [node.id, this.repo.platform ? await this.repo.platform(node.id) : 'linux']), + ) + this.nodes = new Map(nodes.map((node) => [node.id, node])) + this.directLinksState = directLinks + this.platforms = new Map(platforms) + } + + async rebind(db) { + await this.stop() + this.repo = new this.repo.constructor(db) + await this.start() + await this.restoreSlotRelays() + } + + async start() { + await this.reload() + for (const node of this.nodes.values()) this._attach(node) this._healthTimer = setInterval(() => { for (const id of this.links.keys()) this._checkHealth(id) }, HEALTH_INTERVAL_MS) @@ -216,8 +239,8 @@ export class ClusterManager { } async _targetFor(id) { - const node = this.repo.get(id) - const secrets = this.repo.getSecrets(id) + const node = this.nodes.get(id) + const secrets = await this.repo.getSecrets(id) if (!node || !secrets) throw new ClusterError(404, 'node_not_found', '节点不存在') return { host: node.host, @@ -283,7 +306,7 @@ export class ClusterManager { }) } - _view(node, directLinks = this.repo.directLinks()) { + _view(node, directLinks = this.directLinksState) { const link = this.links.get(node.id) return { ...node, @@ -297,12 +320,12 @@ export class ClusterManager { } list() { - const directLinks = this.repo.directLinks() - return this.repo.list().map((node) => this._view(node, directLinks)) + const directLinks = this.directLinksState + return [...this.nodes.values()].map((node) => this._view(node, directLinks)) } get(id) { - const node = this.repo.get(id) + const node = this.nodes.get(id) if (!node) throw new ClusterError(404, 'node_not_found', '节点不存在') return this._view(node) } @@ -328,7 +351,7 @@ export class ClusterManager { /** Ping the gate; a good answer lifts the SSH fallback early, a bad one starts it. */ async _probeDirect(id) { - const cfg = this.repo.getDirectLink(id) + const cfg = await this.repo.getDirectLink(id) if (!cfg?.certSha256) return const checked_at = new Date().toISOString() try { @@ -351,7 +374,7 @@ export class ClusterManager { * the backoff too; a missing slot socket does not, it is not a link fault. */ async _dialDirect(nodeId, vmId, kind) { - const cfg = this.repo.getDirectLink(nodeId) + const cfg = await this.repo.getDirectLink(nodeId) if (!cfg?.certSha256) return null if ((this.directState.get(nodeId)?.downUntil || 0) > Date.now()) return null const down = (err) => { @@ -372,7 +395,7 @@ export class ClusterManager { async _ensureGate(id, { strict = false } = {}) { if (this.nodePlatform(id) === 'darwin') return - const cfg = this.repo.getDirectLink(id) + const cfg = await this.repo.getDirectLink(id) const link = this.links.get(id) if (!cfg || link?.state !== 'ready' || !link.client) return const client = link.client @@ -380,7 +403,7 @@ export class ClusterManager { const host = await this.remoteHost(id) const result = await ensureNodeGate({ client, host, docker: sshDocker(client) }, cfg, this.gateBin) if (result.certSha256 !== cfg.certSha256) { - this.repo.setDirectLinkCert(id, result.certSha256) + await this.repo.setDirectLinkCert(id, result.certSha256) this.logger.log(`[cluster] slot gate on ${id}: pinned certificate ${result.certSha256.slice(0, 16)}…`) } this._directPatch(id, { gate_error: null }) @@ -394,11 +417,12 @@ export class ClusterManager { /** Enable or move the direct link. Token and certificate pin are kept across edits and never returned. */ async setDirectLink(id, body) { - if (!this.repo.get(id)) throw new ClusterError(404, 'node_not_found', '节点不存在') + if (!this.nodes.get(id)) throw new ClusterError(404, 'node_not_found', '节点不存在') this.assertLinuxNode(id, '直连通道') const { address, port } = parseDirectLinkInput(body) - const token = this.repo.getDirectLink(id)?.token || newGateToken() - this.repo.setDirectLink(id, { address, port, token }) + const token = (await this.repo.getDirectLink(id))?.token || newGateToken() + await this.repo.setDirectLink(id, { address, port, token }) + this.directLinksState = await this.repo.directLinks() this.directState.delete(id) await this._ensureGate(id).catch(() => {}) // The gate needs a moment to bind after a (re)create. @@ -411,8 +435,9 @@ export class ClusterManager { } async clearDirectLink(id) { - if (!this.repo.get(id)) throw new ClusterError(404, 'node_not_found', '节点不存在') - this.repo.clearDirectLink(id) + if (!this.nodes.get(id)) throw new ClusterError(404, 'node_not_found', '节点不存在') + await this.repo.clearDirectLink(id) + this.directLinksState.delete(id) this.directState.delete(id) const link = this.links.get(id) if (link?.state === 'ready' && link.client) { @@ -453,33 +478,35 @@ export class ClusterManager { docker, uname: unameText, platform: nodePlatformOf(unameText?.split(/\s+/)[0]), - existing_id: this.repo.findByEndpoint(spec.host, spec.port, spec.username), + existing_id: await this.repo.findByEndpoint(spec.host, spec.port, spec.username), } } finally { client.end() } } - add(input) { + async add(input) { const spec = parseNodeInput(input, { requireHostKey: true }) - if (spec.jump_node_id && !this.repo.get(spec.jump_node_id)) { + if (spec.jump_node_id && !this.nodes.get(spec.jump_node_id)) { throw new ClusterError(400, 'invalid_jump', '跳板节点不存在') } - if (this.repo.findByEndpoint(spec.host, spec.port, spec.username)) { + if (await this.repo.findByEndpoint(spec.host, spec.port, spec.username)) { throw new ClusterError(409, 'node_exists', '该主机与用户已接入') } - const node = this.repo.insert({ + const node = await this.repo.insert({ ...spec, id: `node-${crypto.randomBytes(4).toString('hex')}`, host_key_alg: spec.host_key_alg || 'unknown', }) + this.nodes.set(node.id, node) + this.platforms.set(node.id, 'linux') this._attach(node) this._nodesChanged() return this._view(node) } - _prepareEdit(id, input) { - const node = this.repo.get(id) + async _prepareEdit(id, input) { + const node = this.nodes.get(id) if (!node) throw new ClusterError(404, 'node_not_found', '节点不存在') validateNodeEditPatch(node, input) const labelOnly = Object.keys(input).every((key) => key === 'label' || key === 'expected_updated_at') @@ -488,7 +515,7 @@ export class ClusterManager { if (label.length > 64) throw new ClusterError(400, 'invalid_label', '备注过长') return { id, node, spec: { label }, connectionEdit: false, labelOnly: true, vms: this.vmsOnNode(id) } } - const next = mergeNodeEdit(node, this.repo.getSecrets(id), input) + const next = mergeNodeEdit(node, await this.repo.getSecrets(id), input) const spec = parseNodeInput(next, { requireHostKey: true }) const link = this.links.get(id) const vms = this.vmsOnNode(id) @@ -504,12 +531,12 @@ export class ClusterManager { vms, placementVersion: nodePlacementVersion(vms), } - this._assertEditCurrent(edit) + await this._assertEditCurrent(edit) return edit } - _assertEditCurrent(edit) { - const current = this.repo.get(edit.id) + async _assertEditCurrent(edit) { + const current = await this.repo.get(edit.id) if (!current) throw new ClusterError(404, 'node_not_found', '节点不存在') if (current.updated_at !== edit.node.updated_at) throw nodeEditConflict() if (!edit.connectionEdit) return @@ -524,10 +551,10 @@ export class ClusterManager { if (edit.vms.length && edit.spec.username !== edit.node.username) { throw new ClusterError(409, 'node_username_in_use', '节点上仍有虚拟机，不能更换 SSH 用户；请先迁移或删除虚拟机') } - if (this.repo.dependentsOf(edit.id).length) { + if ((await this.repo.dependentsOf(edit.id)).length) { throw new ClusterError(409, 'node_in_use', '仍有节点经由它跳转，暂不支持修改该节点的连接配置') } - const duplicate = this.repo.findByEndpoint(edit.spec.host, edit.spec.port, edit.spec.username) + const duplicate = await this.repo.findByEndpoint(edit.spec.host, edit.spec.port, edit.spec.username) if (duplicate && duplicate !== edit.id) throw new ClusterError(409, 'node_exists', '该主机与用户已接入') if (this.installs.get(edit.id)?.status === 'running') throw new ClusterError(409, 'node_busy', '节点正在安装 Docker，请稍后重试') @@ -537,7 +564,7 @@ export class ClusterManager { async _probeEdit(edit) { let previousHost = null if (edit.vms.length && edit.client && edit.link.state === 'ready') previousHost = await this.remoteHost(edit.id) - this._assertEditCurrent(edit) + await this._assertEditCurrent(edit) const sock = await this._jumpSock(edit.spec.jump_node_id, edit.spec.host, edit.spec.port) const result = await connectSsh({ host: edit.spec.host, @@ -581,7 +608,7 @@ export class ClusterManager { } else { host = await readNodeHost(result.client).catch(() => null) } - this._assertEditCurrent(edit) + await this._assertEditCurrent(edit) if (!candidate.live()) throw nodeEditConflict() return { ...candidate, host, docker } } catch (err) { @@ -592,14 +619,14 @@ export class ClusterManager { } async probeUpdate(id, input) { - const edit = this._prepareEdit(id, input) + const edit = await this._prepareEdit(id, input) if (!edit.connectionEdit) { return { host_key: { alg: edit.node.host_key_alg, sha256: edit.node.host_key_sha256 }, docker: this.health.get(id)?.docker || { ok: false, error: '未探测' }, platform: this.nodePlatform(id), vm_count: edit.vms.length, - dependent_count: this.repo.dependentsOf(id).length, + dependent_count: (await this.repo.dependentsOf(id)).length, } } const candidate = await this._probeEdit(edit) @@ -618,38 +645,60 @@ export class ClusterManager { } async update(id, input) { - const edit = this._prepareEdit(id, input) + const edit = await this._prepareEdit(id, input) const candidate = edit.connectionEdit ? await this._probeEdit(edit) : null let adopted = false + let committed = false + let adoptedGeneration try { - this._assertEditCurrent(edit) - if (candidate && !candidate.live()) throw nodeEditConflict() - let saved - try { - saved = edit.labelOnly - ? this.repo.updateLabel(id, edit.spec.label, edit.node.updated_at) - : this.repo.update(id, edit.spec, edit.node.updated_at) - } catch (err) { - if (String(err.message).includes('UNIQUE constraint failed: cluster_nodes.')) { - throw new ClusterError(409, 'node_exists', '该主机与用户已接入') + const saved = await withTransaction(this.repo.db, async () => { + await this._assertEditCurrent(edit) + const saved = edit.labelOnly + ? await this.repo.updateLabel(id, edit.spec.label, edit.node.updated_at) + : await this.repo.update(id, edit.spec, edit.node.updated_at) + if (!saved) throw nodeEditConflict() + if (candidate) { + // Reconnect and placement may change while PostgreSQL executes the CAS. + // Keep this final check and adoption synchronous; a rejection rolls back the CAS. + if ( + !candidate.live() || + this.links.get(id) !== edit.link || + edit.link.generation !== edit.generation || + edit.link.client !== edit.client || + nodePlacementVersion(this.vmsOnNode(id)) !== edit.placementVersion + ) + throw nodeEditConflict() + if (!edit.link.adopt(candidate, edit.generation)) throw nodeEditConflict() + candidate.cleanup() + adopted = true + adoptedGeneration = edit.link.generation } - throw err - } - if (!saved) throw nodeEditConflict() + return saved + }) + committed = true + this.nodes.set(id, saved) if (candidate) { - // No await between the final version check, SQLite CAS and adoption. Existing relay closures retain this link. - candidate.cleanup() - edit.link.adopt(candidate, edit.generation) - adopted = true - if (candidate.host) { + if (edit.link.generation !== adoptedGeneration || edit.link.client !== candidate.client) { + // A reconnect during COMMIT must resolve its target from the committed record. + edit.link.reconnect() + } else if (candidate.host) { this.hosts.set(candidate.client, Promise.resolve(candidate.host)) - this._learnHost(id, candidate.host) + await this._learnHost(id, candidate.host) } for (const [ticket, entry] of this.tickets) if (entry.nodeId === id) this.tickets.delete(ticket) } return this._view(saved) + } catch (error) { + if (adopted && !committed) { + const persisted = await this.repo.get(id).catch(() => null) + if (persisted) this.nodes.set(id, persisted) + else this.nodes.delete(id) + edit.link.reconnect() + } + if (error.code === '23505') throw new ClusterError(409, 'node_exists', '该主机与用户已接入') + throw error } finally { - if (candidate && !adopted) { + if (candidate && (!adopted || !committed)) { candidate.cleanup() candidate.client.end() } @@ -673,8 +722,8 @@ export class ClusterManager { } async remove(id) { - if (!this.repo.get(id)) throw new ClusterError(404, 'node_not_found', '节点不存在') - const dependents = this.repo.dependentsOf(id) + if (!this.nodes.get(id)) throw new ClusterError(404, 'node_not_found', '节点不存在') + const dependents = await this.repo.dependentsOf(id) if (dependents.length) { throw new ClusterError(409, 'node_in_use', `仍有节点经由它跳转：${dependents.join(', ')}`) } @@ -682,6 +731,7 @@ export class ClusterManager { if (placed.length) { throw new ClusterError(409, 'node_has_vms', `节点上仍有虚拟机：${placed.join(', ')}，先删除它们`) } + await this.repo.remove(id) this.links.get(id)?.stop() this.links.delete(id) await this.bridges.get(id)?.stop() @@ -690,7 +740,9 @@ export class ClusterManager { this.installs.delete(id) this.directState.delete(id) this.hostFacts.delete(id) - this.repo.remove(id) + this.nodes.delete(id) + this.platforms.delete(id) + this.directLinksState.delete(id) this._nodesChanged() } @@ -731,20 +783,25 @@ export class ClusterManager { const cached = this.hosts.get(client) if (cached) return cached const pending = readNodeHost(client) - pending.then( - (host) => this._learnHost(id, host), - () => this.hosts.delete(client), - ) + .then(async (host) => { + await this._learnHost(id, host) + return host + }) + .catch((error) => { + this.hosts.delete(client) + throw error + }) this.hosts.set(client, pending) return pending } - _learnHost(id, host) { + async _learnHost(id, host) { try { - if (!this.repo.get(id)) return + if (!this.nodes.get(id)) return this.hostFacts.set(id, host) if (host.platform && this.nodePlatform(id) !== host.platform) { - this.repo.setPlatform(id, host.platform) + await this.repo.setPlatform(id, host.platform) + this.platforms.set(id, host.platform) this.logger.log(JSON.stringify({ event: 'cluster-node-platform', node: id, platform: host.platform })) } } catch (err) { @@ -755,7 +812,7 @@ export class ClusterManager { /** 'linux' | 'darwin'. Unknown or unreadable (older schema) is Linux: the Docker runtime is the default. */ nodePlatform(id) { try { - return this.repo.platform(id) === 'darwin' ? 'darwin' : 'linux' + return this.platforms.get(id) === 'darwin' ? 'darwin' : 'linux' } catch { return 'linux' } @@ -889,7 +946,7 @@ export class ClusterManager { /** Boot: relays exist before any link is ready so the scheduler's first health dial finds a socket. */ async restoreSlotRelays() { - for (const node of this.repo.list()) { + for (const node of [...this.nodes.values()]) { for (const vm of this.vmsOnNode(node.id)) { await this.ensureSlotRelays(node.id, vm.id).catch((err) => this.logger.warn(`[cluster] slot relay ${vm.id} on ${node.id}: ${err.message}`), @@ -925,7 +982,7 @@ export class ClusterManager { /** A jumped node sees the jump host as SSH_CLIENT, so only direct ready nodes can observe our public address. */ _natObserver() { - for (const node of this.repo.list()) { + for (const node of [...this.nodes.values()]) { if (node.jump_node_id) continue const link = this.links.get(node.id) if (link?.state === 'ready' && link.client) return { node, client: link.client } diff --git a/src/lib/cluster/cluster-routes.mjs b/src/lib/cluster/cluster-routes.mjs index 4542ff2cd..aa9961ba0 100644 --- a/src/lib/cluster/cluster-routes.mjs +++ b/src/lib/cluster/cluster-routes.mjs @@ -35,27 +35,27 @@ function clampInt(v, lo, hi, dflt) { } export function createClusterRoutes({ manager, json, readBody, ok }) { - function fail(res, err) { + function fail(res, err, respond = json) { const status = err instanceof ClusterError ? err.status : 500 const code = err instanceof ClusterError ? err.code : 'cluster_error' - return json(res, status, { ok: false, error: { type: 'cluster', code, message: err.message || String(err) } }) + return respond(res, status, { ok: false, error: { type: 'cluster', code, message: err.message || String(err) } }) } - async function route(req, res, url) { + async function route(req, res, url, respond = json) { const p = url.pathname const m = req.method - if (p === `${PREFIX}/nodes` && m === 'GET') return json(res, 200, ok({ items: manager.list() })) + if (p === `${PREFIX}/nodes` && m === 'GET') return respond(res, 200, ok({ items: manager.list() })) if (p === `${PREFIX}/local` && m === 'GET') { - return json(res, 200, ok(await manager.localStatus({ refresh: url.searchParams.get('refresh') === '1' }))) + return respond(res, 200, ok(await manager.localStatus({ refresh: url.searchParams.get('refresh') === '1' }))) } if (p === `${PREFIX}/nodes` && m === 'POST') { const body = await readBody(req, MAX_BODY) - return json(res, 201, ok(manager.add(body))) + return respond(res, 201, ok(await manager.add(body))) } if (p === `${PREFIX}/nodes/probe` && m === 'POST') { const body = await readBody(req, MAX_BODY) - return json(res, 200, ok(await manager.probe(body))) + return respond(res, 200, ok(await manager.probe(body))) } const nm = NODE_RE.exec(p) @@ -63,31 +63,31 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { const id = nm[1] const sub = nm[2] || '' - if (sub === '' && m === 'GET') return json(res, 200, ok(manager.get(id))) + if (sub === '' && m === 'GET') return respond(res, 200, ok(manager.get(id))) if (sub === '' && m === 'PATCH') { const body = await readBody(req, MAX_BODY) - return json(res, 200, ok(await manager.update(id, body))) + return respond(res, 200, ok(await manager.update(id, body))) } if (sub === '/probe' && m === 'POST') { const body = await readBody(req, MAX_BODY) - return json(res, 200, ok(await manager.probeUpdate(id, body))) + return respond(res, 200, ok(await manager.probeUpdate(id, body))) } if (sub === '' && m === 'DELETE') { await manager.remove(id) - return json(res, 200, ok({ id })) + return respond(res, 200, ok({ id })) } - if (sub === '/reconnect' && m === 'POST') return json(res, 200, ok(manager.reconnect(id))) + if (sub === '/reconnect' && m === 'POST') return respond(res, 200, ok(manager.reconnect(id))) if (sub === '/direct-link' && m === 'PUT') { const body = await readBody(req, MAX_BODY) - return json(res, 200, ok(await manager.setDirectLink(id, body))) + return respond(res, 200, ok(await manager.setDirectLink(id, body))) } - if (sub === '/direct-link' && m === 'DELETE') return json(res, 200, ok(await manager.clearDirectLink(id))) - if (sub === '/shell-ticket' && m === 'POST') return json(res, 200, ok(manager.issueShellTicket(id))) + if (sub === '/direct-link' && m === 'DELETE') return respond(res, 200, ok(await manager.clearDirectLink(id))) + if (sub === '/shell-ticket' && m === 'POST') return respond(res, 200, ok(manager.issueShellTicket(id))) - if (sub === '/docker/info' && m === 'GET') return json(res, 200, ok(await dockerInfo(manager.docker(id)))) + if (sub === '/docker/info' && m === 'GET') return respond(res, 200, ok(await dockerInfo(manager.docker(id)))) if (sub === '/preflight' && m === 'POST') { const body = await readBody(req, MAX_BODY) - return json( + return respond( res, 200, ok( @@ -101,7 +101,7 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { if (sub === '/slot-image' && m === 'POST') { const body = await readBody(req, MAX_BODY) manager.get(id) - return json( + return respond( res, 202, ok( @@ -111,20 +111,20 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { } if (sub === '/slot-image' && m === 'GET') { manager.get(id) - return json( + return respond( res, 200, ok(slotImageJob(id, kernelOf(url.searchParams.get('kernel')), url.searchParams.get('runtime_type'))), ) } - if (sub === '/docker/install' && m === 'POST') return json(res, 202, ok(manager.startDockerInstall(id))) - if (sub === '/docker/install' && m === 'GET') return json(res, 200, ok(manager.get(id).install)) + if (sub === '/docker/install' && m === 'POST') return respond(res, 202, ok(manager.startDockerInstall(id))) + if (sub === '/docker/install' && m === 'GET') return respond(res, 200, ok(manager.get(id).install)) if (sub === '/docker/containers' && m === 'GET') { - return json(res, 200, ok({ items: await listContainers(manager.docker(id)) })) + return respond(res, 200, ok({ items: await listContainers(manager.docker(id)) })) } if (sub === '/docker/containers' && m === 'POST') { const body = await readBody(req, MAX_BODY) - return json(res, 201, ok(await createAndStartContainer(manager.docker(id), body))) + return respond(res, 201, ok(await createAndStartContainer(manager.docker(id), body))) } const cm = /^\/docker\/containers\/([^/]+)(?:\/(start|stop|restart|logs))?$/.exec(sub) @@ -133,35 +133,35 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { const action = cm[2] if (!action && m === 'DELETE') { await removeContainer(manager.docker(id), cid) - return json(res, 200, ok({ id: cid })) + return respond(res, 200, ok({ id: cid })) } if (action === 'logs' && m === 'GET') { const tail = clampInt(url.searchParams.get('tail'), 1, 5000, 200) - return json(res, 200, ok(await containerLogs(manager.docker(id), cid, { tail }))) + return respond(res, 200, ok(await containerLogs(manager.docker(id), cid, { tail }))) } if (action && action !== 'logs' && m === 'POST') { await containerAction(manager.docker(id), cid, action) - return json(res, 200, ok({ id: cid, action })) + return respond(res, 200, ok({ id: cid, action })) } } return false } - async function handle(req, res, url) { + async function handle(req, res, url, respond = json) { if (!url.pathname.startsWith(`${PREFIX}/`)) return false try { - const handled = await route(req, res, url) + const handled = await route(req, res, url, respond) if (handled === false) - return json(res, 404, { ok: false, error: { type: 'cluster', code: 'not_found', message: 'not found' } }) + return respond(res, 404, { ok: false, error: { type: 'cluster', code: 'not_found', message: 'not found' } }) return true } catch (err) { if (!(err instanceof ClusterError) && err?.status) { - return json(res, err.status, { + return respond(res, err.status, { ok: false, error: { type: 'cluster', code: 'bad_request', message: err.message }, }) } - return fail(res, err) + return fail(res, err, respond) } } diff --git a/src/lib/core/refusal-guard.mjs b/src/lib/core/refusal-guard.mjs index 6ece998ec..5c8092d39 100644 --- a/src/lib/core/refusal-guard.mjs +++ b/src/lib/core/refusal-guard.mjs @@ -186,7 +186,7 @@ export function normalizeRefusalSimilarity(value) { } /** Apply a panel PUT. Unknown similarity values are rejected, not coerced. */ -export function applyRefusalGuardPatch(settings, body = {}) { +export async function applyRefusalGuardPatch(settings, body = {}) { const problems = [] if (Object.hasOwn(body, 'enabled') && typeof body.enabled !== 'boolean') problems.push('enabled 必须是布尔') if (Object.hasOwn(body, 'similarity_enabled') && typeof body.similarity_enabled !== 'boolean') { @@ -199,13 +199,14 @@ export function applyRefusalGuardPatch(settings, body = {}) { problems.push('similarity 只能是 80、85、90、95') } if (problems.length) return { ok: false, problems } - if (typeof body.enabled === 'boolean') settings.set(REFUSAL_GUARD_SETTING, body.enabled) - if (typeof body.similarity_enabled === 'boolean') settings.set(REFUSAL_SIMILARITY_SETTING, body.similarity_enabled) + if (typeof body.enabled === 'boolean') await settings.set(REFUSAL_GUARD_SETTING, body.enabled) + if (typeof body.similarity_enabled === 'boolean') + await settings.set(REFUSAL_SIMILARITY_SETTING, body.similarity_enabled) if (typeof body.device_block_enabled === 'boolean') { - settings.set(REFUSAL_DEVICE_BLOCK_SETTING, body.device_block_enabled) + await settings.set(REFUSAL_DEVICE_BLOCK_SETTING, body.device_block_enabled) } if (Object.hasOwn(body, 'similarity')) { - settings.set(REFUSAL_SIMILARITY_VALUE_SETTING, normalizeRefusalSimilarity(body.similarity)) + await settings.set(REFUSAL_SIMILARITY_VALUE_SETTING, normalizeRefusalSimilarity(body.similarity)) } return { ok: true } } @@ -214,7 +215,7 @@ export function applyRefusalGuardPatch(settings, body = {}) { * Pre-hop decision. Device ban wins, then the exact fingerprint, then a * user-text signature at or above the configured similarity. Does not write. */ -export function matchStoredRefusal({ +export async function matchStoredRefusal({ inbound, body = inbound, headers, @@ -226,17 +227,17 @@ export function matchStoredRefusal({ } = {}) { const deviceId = inboundRefusalDeviceId({ inbound, body, headers }) if (deviceBlockEnabled && deviceId && typeof devices?.get === 'function') { - const banned = devices.get(deviceId) + const banned = await devices.get(deviceId) if (banned) return { kind: 'device', deviceId, row: banned } } if (!repo || typeof repo.get !== 'function') return null const fingerprint = refusalFingerprint(body, inbound) - const exact = repo.get(fingerprint) + const exact = await repo.get(fingerprint) if (exact) return { kind: 'exact', deviceId, fingerprint, row: exact } if (!similarityEnabled || typeof repo.nearest !== 'function') return null const signature = refusalSignature(refusalUserDocument(inbound, body)) if (!signature) return null - const near = repo.nearest(signature, similarity) + const near = await repo.nearest(signature, similarity) if (!near) return null return { kind: 'similar', deviceId, fingerprint: near.fingerprint, score: near.score, row: near } } diff --git a/src/lib/core/security.mjs b/src/lib/core/security.mjs index f9a368d5b..97d92d2d0 100644 --- a/src/lib/core/security.mjs +++ b/src/lib/core/security.mjs @@ -142,13 +142,13 @@ function saveSessions() { loadSessions() -export function createPanelSession(username, ttlMs = SESSION_TTL_MS, extra = {}) { +export async function createPanelSession(username, ttlMs = SESSION_TTL_MS, extra = {}) { if (ttlMs && typeof ttlMs === 'object') { extra = ttlMs ttlMs = extra.ttlMs || SESSION_TTL_MS } const token = 'kin-panel-' + crypto.randomBytes(24).toString('hex') - const role = extra.role || (_lookupRole && _lookupRole(username)) || 'admin' + const role = extra.role || (_lookupRole && (await _lookupRole(username))) || 'admin' panelSessions.set(panelSessionKey(token), { user: username, role, @@ -158,7 +158,7 @@ export function createPanelSession(username, ttlMs = SESSION_TTL_MS, extra = {}) return token } -export function verifyPanelSession(token) { +export async function verifyPanelSession(token) { if (!token || !token.startsWith('kin-panel-')) return null const key = panelSessionKey(token) let s = panelSessions.get(key) @@ -174,7 +174,7 @@ export function verifyPanelSession(token) { return null } if (!s.role && _lookupRole) { - s.role = _lookupRole(s.user) || null + s.role = (await _lookupRole(s.user)) || null } if (!s.role) { const admin = getPanelAdmin() @@ -213,12 +213,12 @@ export function revokePanelSessionsForUser(username, { exceptToken } = {}) { /** * @returns {{ username: string, role: string, id?: string } | null} */ -export function verifyPanelLogin(username, password) { +export async function verifyPanelLogin(username, password) { if (_authenticateUser) { - const rec = _authenticateUser(username, password) + const rec = await _authenticateUser(username, password) if (rec) return { username: rec.username, role: rec.role || 'user', id: rec.id } // Existing DB user (including disabled) must not fall through to env password. - if (_lookupRole && _lookupRole(username)) return null + if (_lookupRole && (await _lookupRole(username))) return null } const admin = getPanelAdmin() if (!admin.password) return null diff --git a/src/lib/db/database-metrics.mjs b/src/lib/db/database-metrics.mjs index 68925ddcf..5a637033c 100644 --- a/src/lib/db/database-metrics.mjs +++ b/src/lib/db/database-metrics.mjs @@ -1,120 +1,54 @@ -import fs from 'node:fs' - +/** Read-only PostgreSQL and application-cache metrics; never expose connection details. */ function numberOrNull(value) { if (value == null || value === '') return null - const num = Number(value) - return Number.isFinite(num) ? num : null -} - -function scalar(db, sql) { - try { - const row = db.prepare(sql).get() - if (!row || typeof row !== 'object') return null - return Object.values(row)[0] ?? null - } catch { - return null - } + const n = Number(value) + return Number.isFinite(n) && Math.abs(n) <= Number.MAX_SAFE_INTEGER ? n : null } -function probe(db, clock) { - try { - const startedAt = clock() - db.prepare('SELECT 1 AS ok').get() - const elapsed = numberOrNull(clock() - startedAt) - return { - ok: true, - latencyMs: elapsed == null ? null : Math.round(Math.max(elapsed, 0) * 1000) / 1000, - } - } catch { - return { ok: false, latencyMs: null } - } -} - -function fileSize(dbPath, suffix = '') { - if (typeof dbPath !== 'string' || !dbPath) return null +export async function snapshotDatabaseMetrics({ + db, + usageCache, + now = () => Date.now(), + clock = () => performance.now(), +}) { + let ok = false + let latency = null try { - return fs.statSync(`${dbPath}${suffix}`).size - } catch (error) { - return error?.code === 'ENOENT' ? 0 : null - } -} - -function migrationMetrics(db) { - const count = numberOrNull(scalar(db, 'SELECT COUNT(*) AS count FROM schema_migrations')) + const started = clock() + await db.prepare('SELECT 1 AS ok').get() + ok = true + latency = Math.round(Math.max(clock() - started, 0) * 1000) / 1000 + } catch {} + let size = null + let migrations = null let latest = null try { - latest = db - .prepare('SELECT version, applied_at FROM schema_migrations ORDER BY applied_at DESC, version DESC LIMIT 1') - .get() + size = numberOrNull((await db.prepare('SELECT pg_database_size(current_database()) AS bytes').get())?.bytes) } catch {} - return { - migration_count: count, - latest_migration: latest?.version == null ? null : String(latest.version), - latest_migration_at: latest?.applied_at == null ? null : String(latest.applied_at), - } -} - -function pageMetrics(pageSize, pageCount, freelistPages) { - if (pageCount == null || freelistPages == null) { - return { used_pages: null, used_bytes: null, free_ratio: null } - } - const usedPages = Math.max(pageCount - freelistPages, 0) - return { - used_pages: usedPages, - used_bytes: pageSize == null ? null : usedPages * pageSize, - free_ratio: pageCount > 0 ? freelistPages / pageCount : null, - } -} - -function safeUsageStats(usageCache) { try { - return typeof usageCache?.stats === 'function' ? usageCache.stats() : null - } catch { - return null - } -} - -function sampledAt(now) { + migrations = numberOrNull((await db.prepare('SELECT COUNT(*) AS count FROM schema_migrations').get())?.count) + } catch {} try { - return new Date(now()).toISOString() - } catch { - return null - } -} - -export function snapshotDatabaseMetrics({ - db, - dbPath, - usageCache, - now = () => Date.now(), - clock = () => performance.now(), -}) { - const health = probe(db, clock) - const journalMode = scalar(db, 'PRAGMA journal_mode') - const foreignKeys = numberOrNull(scalar(db, 'PRAGMA foreign_keys')) - const pageSize = numberOrNull(scalar(db, 'PRAGMA page_size')) - const pageCount = numberOrNull(scalar(db, 'PRAGMA page_count')) - const freelistPages = numberOrNull(scalar(db, 'PRAGMA freelist_count')) - + latest = await db.prepare('SELECT version, applied_at FROM schema_migrations ORDER BY version DESC LIMIT 1').get() + } catch {} + let usage = null + try { + usage = typeof usageCache?.stats === 'function' ? usageCache.stats() : null + } catch {} return { - sampled_at: sampledAt(now), + sampled_at: new Date(now()).toISOString(), database: { - ok: health.ok, - engine: 'sqlite', - probe_latency_ms: health.latencyMs, - journal_mode: journalMode == null ? null : String(journalMode).toLowerCase(), - synchronous: numberOrNull(scalar(db, 'PRAGMA synchronous')), - foreign_keys: foreignKeys == null ? null : foreignKeys !== 0, - busy_timeout_ms: numberOrNull(scalar(db, 'PRAGMA busy_timeout')), - file_size_bytes: fileSize(dbPath), - wal_size_bytes: fileSize(dbPath, '-wal'), - shm_size_bytes: fileSize(dbPath, '-shm'), - page_size_bytes: pageSize, - page_count: pageCount, - freelist_pages: freelistPages, - ...pageMetrics(pageSize, pageCount, freelistPages), - ...migrationMetrics(db), + ok, + engine: 'postgresql', + probe_latency_ms: latency, + size_bytes: size, + pool_total: numberOrNull(db.pool?.totalCount), + pool_idle: numberOrNull(db.pool?.idleCount), + pool_waiting: numberOrNull(db.pool?.waitingCount), + migration_count: migrations, + latest_migration: latest?.version == null ? null : String(latest.version), + latest_migration_at: latest?.applied_at == null ? null : String(latest.applied_at), }, - usage_cache: safeUsageStats(usageCache), + usage_cache: usage, } } diff --git a/src/lib/db/database.mjs b/src/lib/db/database.mjs index 4d2f5ca66..aee9eba94 100644 --- a/src/lib/db/database.mjs +++ b/src/lib/db/database.mjs @@ -1,272 +1,293 @@ -/** - * SQLite database layer — sub2api data-layer inspired, on node:sqlite. - * - * - Single-process gateway → embedded SQLite (WAL) instead of sub2api's - * PostgreSQL+Redis. Same patterns: versioned SQL migrations with SHA-256 - * checksum verification (sub2api migrations_runner), repository modules, - * settings table, backup records. - * - Zero new dependencies: uses the built-in `node:sqlite` DatabaseSync. - * - * Env: - * KIN_DB_PATH — override db file path (default /kin.db) - */ - +/** PostgreSQL persistence. SQLite is supported only by the offline importer. */ import fs from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' +import { AsyncLocalStorage } from 'node:async_hooks' import { fileURLToPath } from 'node:url' -import { DatabaseSync } from 'node:sqlite' +import pg from 'pg' -const __dirname = path.dirname(fileURLToPath(import.meta.url)) -const MIGRATIONS_DIR = path.join(__dirname, 'migrations') +const MIGRATIONS_DIR = fileURLToPath(new URL('./postgres-migrations/', import.meta.url)) +const transactions = new AsyncLocalStorage() +const maintenance = new AsyncLocalStorage() +const instances = new Set() +let singleton = null +let opening = null -function sha256Hex(s) { - return crypto.createHash('sha256').update(s).digest('hex') +export function quoteIdentifier(value) { + return '"' + String(value).replaceAll('"', '""') + '"' } -/** LF-canonical hash plus CRLF/raw variants so Windows-applied checksums still match. */ -function migrationChecksums(sql) { - const lf = sql.replace(/\r\n/g, '\n') - const crlf = lf.replace(/\n/g, '\r\n') - return { - canonical: sha256Hex(lf), - variants: new Set([sha256Hex(sql), sha256Hex(lf), sha256Hex(crlf)]), +export function resolveDatabaseUrl({ connectionString } = {}) { + const value = connectionString || process.env.VM2API_DATABASE_URL || process.env.KIN_DATABASE_URL + if (!value || !/^postgres(?:ql)?:\/\//.test(value)) { + throw new Error('PostgreSQL requires VM2API_DATABASE_URL or KIN_DATABASE_URL') } + return value } -// 001 was published briefly before the proxy pool switched to its persisted -// record shape. Its checksum cannot be treated as the current 001: migration -// 018 assumes the newer columns and otherwise fails after accepting the hash. -const HISTORICAL_001_PROXY_CHECKSUMS = new Set([ - '376c686118bbd8aaa6db6ea13fb2ff0afecc0c7464bc3d668743a55c9ccafdde', - 'd6769ea311718b5af91e47295f97c3bf2722ece6d26acee7e89df84b88599ae2', -]) - -function reconcileHistorical001ProxySchema(db, checksum) { - if (!HISTORICAL_001_PROXY_CHECKSUMS.has(checksum)) return false - const columns = new Set( - db - .prepare('PRAGMA table_info(proxies)') - .all() - .map((row) => row.name), - ) - const current = ['raw', 'enabled', 'consecutive_failures', 'last_error'].every((name) => columns.has(name)) - if (current && !columns.has('url')) return true - const historical = ['url', 'failures', 'last_probe_ok', 'disabled_reason', 'updated_at'].every((name) => - columns.has(name), - ) - if (!historical || columns.has('raw')) { - throw new Error('historical migration 001 proxy schema is not recognized') - } - - withTransaction(db, () => { - db.exec(` - ALTER TABLE proxies RENAME TO proxies_historical_001; - CREATE TABLE proxies ( - id TEXT PRIMARY KEY, - scheme TEXT DEFAULT 'socks5', - host TEXT, - port INTEGER, - username TEXT, - password TEXT, - raw TEXT, - status TEXT DEFAULT 'unknown', - enabled INTEGER DEFAULT 1, - bound_vm_id TEXT, - consecutive_failures INTEGER DEFAULT 0, - latency_ms INTEGER, - last_probe_at TEXT, - last_error TEXT, - created_at TEXT - ); - INSERT INTO proxies ( - id, scheme, host, port, username, password, raw, status, enabled, - bound_vm_id, consecutive_failures, latency_ms, last_probe_at, - last_error, created_at - ) - SELECT - id, scheme, host, port, username, password, url, - CASE - WHEN status IN ('unknown', 'ok', 'dead') THEN status - WHEN last_probe_ok = 1 THEN 'ok' - WHEN last_probe_ok = 0 THEN 'dead' - ELSE 'unknown' - END, - CASE WHEN status = 'disabled' THEN 0 ELSE 1 END, - bound_vm_id, COALESCE(failures, 0), latency_ms, last_probe_at, - disabled_reason, created_at - FROM proxies_historical_001; - DROP TABLE proxies_historical_001; - `) - }) - return true +function safeInteger(value) { + const n = Number(value) + return Number.isSafeInteger(n) ? n : value } -let _db = null -let _dbPath = null -export function resolveDbPath({ dataDir, dbPath } = {}) { - if (dbPath) return dbPath - if (process.env.KIN_DB_PATH) return process.env.KIN_DB_PATH - const dir = dataDir || path.join(process.cwd(), 'data') - return path.join(dir, 'kin.db') +function databaseError(error, operation = 'query') { + const out = new Error(`PostgreSQL ${operation} failed (${error.code || 'database_error'})`) + out.code = error.code || 'database_error' + if (error.constraint) out.constraint = error.constraint + return out } -/** - * Standalone (non-singleton) open: PRAGMAs + migrations applied. - * Used by unit tests and by stores constructed without a global db. - * @returns {DatabaseSync} - */ -export function createDatabase({ dataDir, dbPath, migrationsDir } = {}) { - const file = resolveDbPath({ dataDir, dbPath }) - fs.mkdirSync(path.dirname(file), { recursive: true }) - const db = new DatabaseSync(file) - try { - fs.chmodSync(file, 0o600) - } catch {} - db.exec('PRAGMA journal_mode = WAL') - db.exec('PRAGMA synchronous = NORMAL') - db.exec('PRAGMA busy_timeout = 5000') - db.exec('PRAGMA foreign_keys = ON') - applyMigrations(db, { migrationsDir }) - return db -} +export class PostgresDatabase { + constructor({ connectionString, schema = 'public', max = 10 } = {}) { + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(schema)) throw new Error('invalid database schema') + this.connectionString = resolveDatabaseUrl({ connectionString }) + this.schema = schema + this.pool = new pg.Pool({ + connectionString: this.connectionString, + max, + connectionTimeoutMillis: 5000, + idleTimeoutMillis: 10000, + statement_timeout: 60000, + options: `-c search_path=${quoteIdentifier(schema)},pg_catalog -c timezone=UTC`, + types: { + getTypeParser(oid, format) { + if (format !== 'binary' && (oid === 20 || oid === 1700)) { + return (value) => (/^-?\d+$/.test(value) ? safeInteger(value) : Number(value)) + } + return pg.types.getTypeParser(oid, format) + }, + }, + }) + instances.add(this) + this.closed = false + this.active = 0 + this.maintaining = false + this.waiters = [] + // Idle connection errors must not become uncaught EventEmitter errors. + this.pool.on('error', () => {}) + } -/** - * Open (or return the already-open) singleton database. - * @returns {DatabaseSync} - */ -export function openDatabase({ dataDir, dbPath, migrationsDir } = {}) { - if (_db) return _db - _db = createDatabase({ dataDir, dbPath, migrationsDir }) - _dbPath = resolveDbPath({ dataDir, dbPath }) - return _db -} + get isTransaction() { + return transactions.getStore()?.db === this + } -/** - * Resolve the db a store should use: explicit db → global singleton → - * standalone connection rooted at dataDir (test isolation path). - */ -export function resolveStoreDb({ db, dataDir } = {}) { - if (db) return db - if (_db) return _db - return createDatabase({ dataDir }) -} + async query(sql, params = []) { + if (this.closed) throw new Error('database is closed') + const tx = transactions.getStore() + if (this.maintaining && maintenance.getStore() !== this && tx?.db !== this) { + throw Object.assign(new Error('database maintenance in progress'), { code: 'database_maintenance' }) + } + this.active += 1 + try { + return await (tx?.db === this ? tx.client : this.pool).query(sql, params) + } catch (error) { + throw databaseError(error) + } finally { + this.active -= 1 + if (!this.active) for (const resolve of this.waiters.splice(0)) resolve() + } + } -/** Current db instance (throws when not opened). */ -export function getDb() { - if (!_db) throw new Error('database not opened — call openDatabase() first') - return _db -} + afterCommit(fn) { + const tx = transactions.getStore() + if (tx?.db === this) tx.commitCallbacks.push(fn) + else fn() + } -export function isDbOpen() { - return !!_db -} + prepare(sql) { + return { + get: async (...params) => (await this.query(sql, params)).rows[0], + all: async (...params) => (await this.query(sql, params)).rows, + run: async (...params) => { + const result = await this.query(sql, params) + return { changes: result.rowCount || 0, rows: result.rows } + }, + } + } -export function getDbPath() { - return _dbPath -} + async exec(sql) { + await this.query(sql) + } -export function closeDatabase() { - if (_db) { + async withMaintenance(fn) { + if (maintenance.getStore() === this) return fn() + if (this.maintaining) + throw Object.assign(new Error('database maintenance in progress'), { code: 'database_maintenance' }) + this.maintaining = true try { - _db.close() - } catch {} + if (this.active) await new Promise((resolve) => this.waiters.push(resolve)) + return await maintenance.run(this, fn) + } finally { + this.maintaining = false + } } - _db = null - _dbPath = null -} -/** - * Versioned SQL migrations with checksum verification (sub2api-style). - * Files: migrations/NNN_name.sql, applied in filename order inside a - * transaction each. `schema_migrations` records version+checksum; a - * checksum mismatch on an already-applied migration is a hard error. - */ -export function applyMigrations(db, { migrationsDir } = {}) { - const dir = migrationsDir || MIGRATIONS_DIR - db.exec(`CREATE TABLE IF NOT EXISTS schema_migrations ( - version TEXT PRIMARY KEY, - name TEXT, - checksum TEXT, - applied_at TEXT - )`) + async close() { + if (this.closing) return this.closing + if (this.closed) return + this.closing = (async () => { + let failure + try { + for (const queue of this.pendingWriteQueues || []) { + try { + await queue.flush() + } catch (error) { + failure ||= error + } + } + } finally { + this.closed = true + await this.pool.end() + instances.delete(this) + } + if (failure) throw failure + })() + return this.closing + } +} - const files = fs.existsSync(dir) - ? fs - .readdirSync(dir) - .filter((f) => /^\d+.*\.sql$/.test(f)) - .sort() - : [] +export async function withTransaction(db, fn) { + const outer = transactions.getStore() + if (outer?.db === db) return fn(db) + if (db.closed) throw new Error('database is closed') + if (db.maintaining && maintenance.getStore() !== db) { + throw Object.assign(new Error('database maintenance in progress'), { code: 'database_maintenance' }) + } + // Count the entire transaction, including time between individual queries. + db.active += 1 + let client + try { + client = await db.pool.connect() + await client.query('BEGIN') + const context = { db, client, commitCallbacks: [] } + const result = await transactions.run(context, () => fn(db)) + await client.query('COMMIT') + for (const callback of context.commitCallbacks) callback() + return result + } catch (error) { + if (client) await client.query('ROLLBACK').catch(() => {}) + throw error + } finally { + client?.release() + db.active -= 1 + if (!db.active) for (const resolve of db.waiters.splice(0)) resolve() + } +} - const appliedRows = db.prepare('SELECT version, checksum FROM schema_migrations').all() - const applied = new Map(appliedRows.map((r) => [r.version, r.checksum])) - const historical001Accepted = reconcileHistorical001ProxySchema(db, applied.get('001')) - const insert = db.prepare('INSERT INTO schema_migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)') +/** A recoverable import record must roll back both SQL and cache commit callbacks. */ +export async function withSavepoint(db, fn) { + const tx = transactions.getStore() + if (tx?.db !== db) return withTransaction(db, fn) + const mark = tx.commitCallbacks.length + const name = 'import_' + crypto.randomBytes(8).toString('hex') + await db.exec(`SAVEPOINT ${name}`) + try { + const result = await fn() + await db.exec(`RELEASE SAVEPOINT ${name}`) + return result + } catch (error) { + tx.commitCallbacks.splice(mark) + await db.exec(`ROLLBACK TO SAVEPOINT ${name}`) + await db.exec(`RELEASE SAVEPOINT ${name}`) + throw error + } +} - const results = [] - for (const file of files) { - const version = file.split('_')[0] - const sql = fs.readFileSync(path.join(dir, file), 'utf8') - const { canonical: checksum, variants } = migrationChecksums(sql) - if (applied.has(version)) { - const stored = applied.get(version) - const historicalAccepted = - version === '001' && historical001Accepted && HISTORICAL_001_PROXY_CHECKSUMS.has(stored) - if (!variants.has(stored) && !historicalAccepted) { - throw new Error(`migration checksum mismatch for ${file}: applied=${stored} current=${checksum}`) +export async function applyMigrations(db, { migrationsDir = MIGRATIONS_DIR } = {}) { + return withTransaction(db, async () => { + // Transaction-scoped lock is automatically released on connection failure. + await db.query('SELECT pg_advisory_xact_lock(hashtext(current_database()), hashtext($1))', [`vm2api:${db.schema}`]) + if (db.schema !== 'public') await db.exec(`CREATE SCHEMA IF NOT EXISTS ${quoteIdentifier(db.schema)}`) + await db.exec(`CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, name text NOT NULL, checksum text NOT NULL, applied_at text NOT NULL + )`) + const applied = new Map( + (await db.prepare('SELECT version, checksum FROM schema_migrations').all()).map((r) => [r.version, r.checksum]), + ) + const files = fs + .readdirSync(migrationsDir) + .filter((f) => /^\d+.*\.sql$/.test(f)) + .sort() + const known = new Set(files.map((f) => f.split('_')[0])) + if ([...applied.keys()].some((v) => !known.has(v))) + throw new Error('database contains an unknown PostgreSQL migration') + const results = [] + for (const file of files) { + const version = file.split('_')[0] + const sql = fs.readFileSync(path.join(migrationsDir, file), 'utf8').replaceAll('\r\n', '\n') + const checksum = crypto.createHash('sha256').update(sql).digest('hex') + if (applied.has(version)) { + if (applied.get(version) !== checksum) throw new Error(`migration checksum mismatch for ${file}`) + continue } - if (stored !== checksum) { - db.prepare('UPDATE schema_migrations SET checksum = ? WHERE version = ?').run(checksum, version) - } - continue - } - db.exec('BEGIN') - try { - db.exec(sql) - insert.run(version, file, checksum, new Date().toISOString()) - db.exec('COMMIT') - } catch (e) { try { - db.exec('ROLLBACK') - } catch {} - throw new Error(`migration ${file} failed: ${e.message}`) + await db.exec(sql) + await db + .prepare('INSERT INTO schema_migrations (version,name,checksum,applied_at) VALUES ($1,$2,$3,$4)') + .run(version, file, checksum, new Date().toISOString()) + } catch (error) { + throw new Error(`migration ${file} failed (${error.code || 'migration_error'})`) + } + results.push(file) } - results.push(file) - } - return results + return results + }) } -/** Run fn inside a transaction (nested calls just run inline). */ -export function withTransaction(db, fn) { - if (db.isTransaction) return fn() - db.exec('BEGIN') +export async function createDatabase(options = {}) { + if (options.dataDir && fs.existsSync(path.join(options.dataDir, 'restore-maintenance.json'))) { + throw new Error('restore maintenance marker exists; complete offline recovery before startup') + } + const db = new PostgresDatabase({ ...options, schema: options.schema || process.env.KIN_DATABASE_SCHEMA || 'public' }) try { - const out = fn() - db.exec('COMMIT') - return out - } catch (e) { - if (db.isTransaction) { - try { - db.exec('ROLLBACK') - } catch {} + const version = Number((await db.prepare('SHOW server_version_num').get()).server_version_num) + if (version < 170000 || version >= 180000) + throw new Error('vm2api requires PostgreSQL 17 and PostgreSQL 17 backup clients') + await applyMigrations(db, options) + if (options.dataDir && fs.existsSync(path.join(options.dataDir, 'kin.db')) && !options.allowLegacySource) { + const imported = await db.prepare("SELECT value FROM settings WHERE key = 'sqlite_import_done'").get() + if (!imported) throw new Error('existing SQLite data requires db:import:sqlite before PostgreSQL startup') } - throw e + return db + } catch (error) { + await db.close() + throw error } } -/** - * Online-consistent snapshot of the live database (WAL-safe). - * @returns {string} destination path - */ -export function vacuumInto(db, destPath) { - fs.mkdirSync(path.dirname(destPath), { recursive: true }) - try { - fs.rmSync(destPath, { force: true }) - } catch {} - const esc = String(destPath).replace(/'/g, "''") - db.exec(`VACUUM INTO '${esc}'`) +export async function openDatabase(options = {}) { + if (singleton) return singleton + if (opening) return opening + opening = createDatabase(options) try { - fs.chmodSync(destPath, 0o600) - } catch {} - return destPath + singleton = await opening + return singleton + } finally { + opening = null + } +} + +export function getDb() { + if (!singleton) throw new Error('database not opened — await openDatabase() first') + return singleton +} +export function isDbOpen() { + return !!singleton +} +export function resolveStoreDb({ db } = {}) { + return db || getDb() +} +export async function closeDatabase() { + if (opening) await opening + const db = singleton + singleton = null + if (db) await db.close() +} + +/** Drain only explicitly identified test connections before deleting their schema. */ +export async function closeDatabasesForSchema(schema, connectionString) { + const matches = [...instances].filter((db) => db.schema === schema && db.connectionString === connectionString) + const results = await Promise.allSettled(matches.map((db) => db.close())) + const failed = results.find((result) => result.status === 'rejected') + if (failed) throw failed.reason } diff --git a/src/lib/db/integers.mjs b/src/lib/db/integers.mjs new file mode 100644 index 000000000..1e21dc661 --- /dev/null +++ b/src/lib/db/integers.mjs @@ -0,0 +1,19 @@ +/** Exact arithmetic for PostgreSQL bigint counters exposed as numbers or decimal strings. */ +const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER) + +function integer(value) { + if (value == null || value === '') return 0n + if (typeof value === 'number' && !Number.isSafeInteger(value)) { + throw new RangeError('counter must be a safe integer number or an exact decimal string') + } + return BigInt(value) +} + +export function addInteger(left, right) { + const result = integer(left) + integer(right) + return result >= -MAX_SAFE && result <= MAX_SAFE ? Number(result) : result.toString() +} + +export function integerValue(value) { + return addInteger(value, 0) +} diff --git a/src/lib/db/legacy-import.mjs b/src/lib/db/legacy-import.mjs index 5a4f5b0ac..0fd4d4fa1 100644 --- a/src/lib/db/legacy-import.mjs +++ b/src/lib/db/legacy-import.mjs @@ -1,24 +1,8 @@ -/** - * One-time import of legacy JSON stores into SQLite. - * - * Sources (all optional — skipped when missing/corrupt): - * data/api-keys.json → api_keys - * data/account-stats.json → accounts + account_allocations - * data/sticky-map.json → sticky_sessions - * data/proxy-pool.json → proxies + settings.proxy_pool_config - * data/request-logs/*.jsonl → request_logs (recent retainDays) - * data/request-logs/debug/** → request_log_debug - * vms/*.json + active.json → vms + settings.active_vm - * - * Idempotent: guarded by settings.legacy_import_done. Original files are - * left untouched as a rollback safety net (they are no longer read/written, - * except vms/*.json which stay the runtime mirror surface). - */ - +/** One-time import of pre-database JSON files. Originals are never modified. */ import fs from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' -import { getDb, withTransaction } from './database.mjs' +import { getDb, withTransaction, withSavepoint } from './database.mjs' import { SettingsRepo } from './repos/settings-repo.mjs' import { ApiKeysRepo } from './repos/api-keys-repo.mjs' import { AccountsRepo } from './repos/accounts-repo.mjs' @@ -28,23 +12,10 @@ import { VmsRepo } from './repos/vms-repo.mjs' import { UsageLogsRepo } from './repos/usage-logs-repo.mjs' import { isVmRecordFile } from '../vm/vm-file.mjs' -const IMPORT_FLAG = 'legacy_import_done' - -function readJson(file) { - try { - if (!fs.existsSync(file)) return null - return JSON.parse(fs.readFileSync(file, 'utf8')) - } catch { - return undefined // corrupt (≠ missing) - } -} - -export function runLegacyImport({ dataDir, projectRoot, db = null, retainDays = 7 } = {}) { - const d = db || getDb() - const settings = new SettingsRepo(d) - const existing = settings.get(IMPORT_FLAG) +export async function runLegacyImport({ dataDir, projectRoot, db = getDb(), retainDays = 7 } = {}) { + const settings = new SettingsRepo(db) + const existing = await settings.get('legacy_import_done') if (existing) return { imported: false, already: existing } - const counts = { api_keys: 0, accounts: 0, @@ -56,161 +27,119 @@ export function runLegacyImport({ dataDir, projectRoot, db = null, retainDays = vms: 0, errors: 0, } - - withTransaction(d, () => { - // --- api keys --- - try { - const raw = readJson(path.join(dataDir, 'api-keys.json')) - if (raw === undefined) counts.errors += 1 - const repo = new ApiKeysRepo(d) - for (const k of raw?.keys || []) { - if (!k?.id || !k?.key || repo.getById(k.id) || repo.getByKey(k.key)) continue - // legacy `quota_used` was a request counter; the column is USD now - const { quota_used, ...rest } = k - repo.insert({ - ...rest, - quota_requests_used: k.quota_requests_used ?? quota_used ?? 0, - category: String(k.category || 'oauth').toLowerCase() === 'api' ? 'api' : 'oauth', - }) - counts.api_keys += 1 + return withTransaction(db, async () => { + await db.query('SELECT pg_advisory_xact_lock(hashtext(current_database()),hashtext($1))', [ + `vm2api:legacy-import:${db.schema}`, + ]) + const already = await settings.get('legacy_import_done') + if (already) return { imported: false, already } + const safe = async (fn) => { + try { + return await withSavepoint(db, fn) + } catch { + counts.errors++ + return null } - } catch { - counts.errors += 1 } - - // --- account stats --- - try { - const raw = readJson(path.join(dataDir, 'account-stats.json')) - if (raw === undefined) counts.errors += 1 - const repo = new AccountsRepo(d) - for (const acc of Object.values(raw?.accounts || {})) { - if (!acc?.account_id || repo.get(acc.account_id)) continue - repo.insert(acc) - counts.accounts += 1 - for (const alloc of acc.allocations || []) { - repo.addAllocation(acc.account_id, alloc) - counts.allocations += 1 - } - } - } catch { - counts.errors += 1 - } - - // --- sticky sessions --- - try { - const raw = readJson(path.join(dataDir, 'sticky-map.json')) - if (raw === undefined) counts.errors += 1 - const repo = new StickyRepo(d) - for (const [key, ent] of Object.entries(raw?.sessions || {})) { - if (!key || repo.get(key)) continue - repo.upsert(key, ent) - counts.sticky += 1 + const read = (file) => { + if (!fs.existsSync(file)) return null + try { + return JSON.parse(fs.readFileSync(file, 'utf8')) + } catch { + counts.errors++ + return null } - } catch { - counts.errors += 1 } - - // --- proxy pool --- - try { - const raw = readJson(path.join(dataDir, 'proxy-pool.json')) - if (raw === undefined) counts.errors += 1 - if (raw?.proxies?.length || raw?.config) { - const repo = new ProxiesRepo(d) - if (raw.config) repo.setConfig(raw.config) - const current = repo.loadAll() - if (!current.length && Array.isArray(raw.proxies)) { - repo.replaceAll(raw.proxies) - counts.proxies = raw.proxies.length + const keys = new ApiKeysRepo(db) + for (const record of read(path.join(dataDir, 'api-keys.json'))?.keys || []) + await safe(async () => { + if (!record?.id || !record?.key || (await keys.getById(record.id)) || (await keys.getByKey(record.key))) return + const { quota_used, ...rest } = record + await keys.insert({ + ...rest, + quota_requests_used: record.quota_requests_used ?? quota_used ?? 0, + category: String(record.category || 'oauth').toLowerCase() === 'api' ? 'api' : 'oauth', + }) + counts.api_keys++ + }) + const accounts = new AccountsRepo(db) + for (const account of Object.values(read(path.join(dataDir, 'account-stats.json'))?.accounts || {})) + await safe(async () => { + if (!account?.account_id || (await accounts.get(account.account_id))) return + await accounts.insert(account) + for (const allocation of account.allocations || []) await accounts.addAllocation(account.account_id, allocation) + counts.accounts++ + counts.allocations += (account.allocations || []).length + }) + const sticky = new StickyRepo(db) + for (const [key, record] of Object.entries(read(path.join(dataDir, 'sticky-map.json'))?.sessions || {})) + await safe(async () => { + if (!key || (await sticky.get(key))) return + await sticky.upsert(key, record) + counts.sticky++ + }) + const proxies = read(path.join(dataDir, 'proxy-pool.json')) + if (proxies) + await safe(async () => { + const repo = new ProxiesRepo(db) + if (proxies.config) await repo.setConfig(proxies.config) + if (!(await repo.loadAll()).length && Array.isArray(proxies.proxies)) { + await repo.replaceAll(proxies.proxies) + counts.proxies = proxies.proxies.length } - } - } catch { - counts.errors += 1 + }) + const logs = new UsageLogsRepo(db) + const root = path.join(dataDir, 'request-logs') + const cutoff = Date.now() - retainDays * 86400_000 + const recent = (day) => { + const ms = Date.parse(day + 'T00:00:00Z') + return Number.isFinite(ms) && ms >= cutoff } - - // --- request logs (normal summaries, recent days) --- - try { - const repo = new UsageLogsRepo(d) - const root = path.join(dataDir, 'request-logs') - if (fs.existsSync(root)) { - const cutoff = Date.now() - retainDays * 86400_000 - for (const f of fs.readdirSync(root)) { - if (!/^\d{4}-\d{2}-\d{2}\.jsonl$/.test(f)) continue - const dayMs = Date.parse(f.slice(0, 10) + 'T00:00:00Z') - if (Number.isFinite(dayMs) && dayMs < cutoff) continue - try { - const lines = fs.readFileSync(path.join(root, f), 'utf8').split('\n').filter(Boolean) - for (const line of lines) { - try { - const rec = JSON.parse(line) - if (!rec?.id) rec.id = 'log_' + crypto.randomBytes(6).toString('hex') - if (repo.insertSummaryIfAbsent(rec)) counts.request_logs += 1 - } catch { - counts.errors += 1 - } - } - } catch { - counts.errors += 1 - } - } - // --- debug records --- - const dbgRoot = path.join(root, 'debug') - if (fs.existsSync(dbgRoot)) { - for (const day of fs.readdirSync(dbgRoot)) { - if (!/^\d{4}-\d{2}-\d{2}$/.test(day)) continue - const dayMs = Date.parse(day + 'T00:00:00Z') - if (Number.isFinite(dayMs) && dayMs < cutoff) continue - const dir = path.join(dbgRoot, day) - let files = [] - try { - files = fs.readdirSync(dir).filter((x) => x.endsWith('.json')) - } catch { - continue - } - for (const f of files) { - try { - const rec = JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')) - const reqId = rec?.request_id || f.replace(/\.json$/, '') - if (repo.insertDebugIfAbsent(reqId, rec?.ts || new Date(dayMs).toISOString(), rec)) { - counts.request_log_debug += 1 - } - } catch { - counts.errors += 1 - } - } - } - } + if (fs.existsSync(root)) { + for (const file of fs.readdirSync(root)) { + if (!/^\d{4}-\d{2}-\d{2}\.jsonl$/.test(file) || !recent(file.slice(0, 10))) continue + for (const line of fs.readFileSync(path.join(root, file), 'utf8').split('\n').filter(Boolean)) + await safe(async () => { + const record = JSON.parse(line) + record.id ||= 'log_' + crypto.randomBytes(6).toString('hex') + if (await logs.insertSummaryIfAbsent(record)) counts.request_logs++ + }) } - } catch { - counts.errors += 1 - } - - // --- vms + active --- - try { - const repo = new VmsRepo(d) - const vmsDir = projectRoot ? path.join(projectRoot, 'vms') : null - if (vmsDir && fs.existsSync(vmsDir)) { - for (const f of fs.readdirSync(vmsDir)) { - if (!isVmRecordFile(f)) continue - try { - const file = path.join(vmsDir, f) - const vm = JSON.parse(fs.readFileSync(file, 'utf8')) - if (!vm?.id || repo.get(vm.id)) continue - const st = fs.statSync(file) - repo.upsertFromVmJson(vm, { mtimeMs: st.mtimeMs }) - counts.vms += 1 - } catch { - counts.errors += 1 - } + const debug = path.join(root, 'debug') + if (fs.existsSync(debug)) + for (const day of fs.readdirSync(debug)) { + if (!/^\d{4}-\d{2}-\d{2}$/.test(day) || !recent(day)) continue + const dir = path.join(debug, day) + if (!fs.statSync(dir).isDirectory()) continue + for (const file of fs.readdirSync(dir).filter((f) => f.endsWith('.json'))) + await safe(async () => { + const record = JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')) + if ( + await logs.insertDebugIfAbsent( + record.request_id || file.slice(0, -5), + record.ts || day + 'T00:00:00Z', + record, + ) + ) + counts.request_log_debug++ + }) } - const active = readJson(path.join(vmsDir, 'active.json')) - if (active?.active_vm) repo.setActiveVmId(active.active_vm) - } - } catch { - counts.errors += 1 } - - settings.set(IMPORT_FLAG, { at: new Date().toISOString(), counts }) + const dir = projectRoot && path.join(projectRoot, 'vms') + if (dir && fs.existsSync(dir)) { + const vms = new VmsRepo(db) + for (const file of fs.readdirSync(dir).filter(isVmRecordFile)) + await safe(async () => { + const name = path.join(dir, file) + const vm = JSON.parse(fs.readFileSync(name, 'utf8')) + if (!vm?.id || (await vms.get(vm.id))) return + await vms.upsertFromVmJson(vm, { mtimeMs: fs.statSync(name).mtimeMs }) + counts.vms++ + }) + const active = read(path.join(dir, 'active.json')) + if (active?.active_vm) await vms.setActiveVmId(active.active_vm) + } + await settings.set('legacy_import_done', { at: new Date().toISOString(), counts }) + return { imported: true, counts } }) - - return { imported: true, counts } } diff --git a/src/lib/db/pending-writes.mjs b/src/lib/db/pending-writes.mjs new file mode 100644 index 000000000..37d9a3af5 --- /dev/null +++ b/src/lib/db/pending-writes.mjs @@ -0,0 +1,20 @@ +/** Response-finish writes are tracked so maintenance and shutdown can drain them. */ +const pending = new Set() +let failure = null + +export function trackPendingWrite(operation) { + const tracked = Promise.resolve(operation) + pending.add(tracked) + tracked + .catch((error) => { + failure ||= error + console.warn(`[db] response persistence failed: ${error.code || error.name}`) + }) + .finally(() => pending.delete(tracked)) + return tracked +} + +export async function flushPendingWrites() { + while (pending.size) await Promise.allSettled([...pending]) + if (failure) throw failure +} diff --git a/src/lib/db/postgres-migrations/001_baseline.sql b/src/lib/db/postgres-migrations/001_baseline.sql new file mode 100644 index 000000000..20978b75f --- /dev/null +++ b/src/lib/db/postgres-migrations/001_baseline.sql @@ -0,0 +1,646 @@ +-- PostgreSQL baseline corresponding to SQLite migrations 001–041. +-- SQLite history remains immutable for the offline importer. + +CREATE FUNCTION ascii_fold(text) RETURNS text +LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT translate($1, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz') $$; + +CREATE TABLE "account_allocations" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "account_id" text COLLATE "C" NOT NULL, + "at" text COLLATE "C" NOT NULL, + "source" text COLLATE "C", + "util_5h" double precision, + "util_7d" double precision, + "status_5h" text COLLATE "C", + "status_7d" text COLLATE "C", + "claim" text COLLATE "C", + "tokens_in" bigint, + "tokens_out" bigint, + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_alloc_account_at" ON "account_allocations" ("account_id", "at" DESC); + +CREATE TABLE "account_groups" ( + "account_id" text COLLATE "C" NOT NULL, + "group_id" bigint NOT NULL, + "priority" bigint NOT NULL DEFAULT 50, + "created_at" text COLLATE "C", + PRIMARY KEY ("account_id", "group_id") +); + +CREATE INDEX "idx_account_groups_priority" ON "account_groups" ("priority"); +CREATE INDEX "idx_account_groups_group" ON "account_groups" ("group_id"); + +CREATE TABLE "account_runtime" ( + "account_id" text COLLATE "C", + "vm_id" text COLLATE "C" NOT NULL, + "status" text COLLATE "C" NOT NULL DEFAULT 'unknown', + "model_states_json" text COLLATE "C", + "credential_generation" bigint NOT NULL DEFAULT 0, + "refresh_status" text COLLATE "C", + "worker_heartbeat_at" bigint, + "worker_status_json" text COLLATE "C", + "updated_at" text COLLATE "C" NOT NULL, + PRIMARY KEY ("account_id") +); + +CREATE INDEX "idx_account_runtime_vm_new" ON "account_runtime" ("vm_id"); + +CREATE TABLE "account_runtime_states_legacy" ( + "account_id" text COLLATE "C", + "vm_id" text COLLATE "C" NOT NULL, + "status" text COLLATE "C" NOT NULL DEFAULT 'unknown', + "priority" bigint NOT NULL DEFAULT 0, + "weight" double precision NOT NULL DEFAULT 1, + "cooldown_until" bigint, + "cooldown_reason" text COLLATE "C", + "model_states_json" text COLLATE "C", + "last_used_at" bigint, + "credential_generation" bigint NOT NULL DEFAULT 0, + "refresh_status" text COLLATE "C", + "worker_heartbeat_at" bigint, + "worker_status_json" text COLLATE "C", + "updated_at" text COLLATE "C" NOT NULL, + "rate_limited_at" bigint, + "rate_limit_reset_at" bigint, + "overload_until" bigint, + "session_window_start" bigint, + "session_window_end" bigint, + "session_window_status" text COLLATE "C", + PRIMARY KEY ("account_id") +); + +CREATE INDEX "idx_account_runtime_cooldown" ON "account_runtime_states_legacy" ("cooldown_until"); +CREATE INDEX "idx_account_runtime_vm" ON "account_runtime_states_legacy" ("vm_id"); + +CREATE TABLE "accounts" ( + "id" text COLLATE "C", + "vm_id" text COLLATE "C", + "email" text COLLATE "C", + "concurrency" bigint DEFAULT 2, + "requests" bigint DEFAULT 0, + "tokens_in" bigint DEFAULT 0, + "tokens_out" bigint DEFAULT 0, + "updated_at" text COLLATE "C", + "cache_read_tokens" bigint DEFAULT 0, + "cache_creation_tokens" bigint DEFAULT 0, + "concurrency_override" bigint NOT NULL DEFAULT 0, + "max_rpm" bigint NOT NULL DEFAULT 0, + "rpm_override" bigint NOT NULL DEFAULT 0, + "name" text COLLATE "C" NOT NULL DEFAULT '', + "platform" text COLLATE "C" NOT NULL DEFAULT 'claude', + "type" text COLLATE "C" NOT NULL DEFAULT 'oauth', + "credentials" text COLLATE "C" NOT NULL DEFAULT '{}', + "extra" text COLLATE "C" NOT NULL DEFAULT '{}', + "proxy_id" text COLLATE "C", + "load_factor" double precision, + "priority" bigint NOT NULL DEFAULT 50, + "rate_multiplier" double precision NOT NULL DEFAULT 1.0, + "status" text COLLATE "C" NOT NULL DEFAULT 'active', + "error_message" text COLLATE "C", + "last_used_at" text COLLATE "C", + "expires_at" text COLLATE "C", + "schedulable" bigint NOT NULL DEFAULT 1, + "rate_limited_at" text COLLATE "C", + "rate_limit_reset_at" text COLLATE "C", + "overload_until" text COLLATE "C", + "temp_unschedulable_until" text COLLATE "C", + "temp_unschedulable_reason" text COLLATE "C", + "session_window_start" text COLLATE "C", + "session_window_end" text COLLATE "C", + "session_window_status" text COLLATE "C", + "created_at" text COLLATE "C", + "deleted_at" text COLLATE "C", + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_accounts_type" ON "accounts" ("type"); +CREATE INDEX "idx_accounts_deleted_at" ON "accounts" ("deleted_at"); +CREATE INDEX "idx_accounts_priority" ON "accounts" ("priority"); +CREATE INDEX "idx_accounts_schedulable" ON "accounts" ("schedulable"); +CREATE INDEX "idx_accounts_status" ON "accounts" ("status"); + +CREATE TABLE "accounts_legacy" ( + "account_id" text COLLATE "C", + "vm_id" text COLLATE "C", + "email" text COLLATE "C", + "max_concurrency" bigint, + "requests" bigint, + "tokens_in" bigint, + "tokens_out" bigint, + "unified_json" text COLLATE "C", + "last_blocked_json" text COLLATE "C", + "last_cli_rate_limit_json" text COLLATE "C", + "updated_at" text COLLATE "C", + "cache_read_tokens" bigint, + "cache_creation_tokens" bigint, + "concurrency_override" bigint, + "max_rpm" bigint, + "rpm_override" bigint +); + + +CREATE TABLE "api_endpoint_keys" ( + "id" text COLLATE "C", + "endpoint_id" text COLLATE "C" NOT NULL, + "api_key" text COLLATE "C" NOT NULL, + "proxy_url" text COLLATE "C", + "disabled" bigint NOT NULL DEFAULT 0, + "cooldown_until" text COLLATE "C", + "created_at" text COLLATE "C", + "proxy_id" text COLLATE "C" NOT NULL DEFAULT '', + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_api_endpoint_keys_ep" ON "api_endpoint_keys" ("endpoint_id"); + +CREATE TABLE "api_endpoint_models" ( + "id" text COLLATE "C", + "endpoint_id" text COLLATE "C" NOT NULL, + "sort" bigint NOT NULL DEFAULT 0, + "upstream_name" text COLLATE "C" NOT NULL, + "alias" text COLLATE "C" NOT NULL, + "image" bigint NOT NULL DEFAULT 0, + "thinking_levels_json" text COLLATE "C", + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_api_endpoint_models_ep" ON "api_endpoint_models" ("endpoint_id"); + +CREATE TABLE "api_endpoints" ( + "id" text COLLATE "C", + "name" text COLLATE "C" NOT NULL, + "disabled" bigint NOT NULL DEFAULT 0, + "prefix" text COLLATE "C", + "base_url" text COLLATE "C" NOT NULL, + "headers_json" text COLLATE "C", + "disable_cooling" bigint NOT NULL DEFAULT 0, + "priority" double precision NOT NULL DEFAULT 0, + "created_at" text COLLATE "C", + "updated_at" text COLLATE "C", + "kind" text COLLATE "C" NOT NULL DEFAULT 'custom', + "protocol" text COLLATE "C" NOT NULL DEFAULT 'anthropic', + PRIMARY KEY ("id") +); + + +CREATE TABLE "api_keys" ( + "id" text COLLATE "C", + "name" text COLLATE "C", + "key" text COLLATE "C" NOT NULL, + "status" text COLLATE "C" DEFAULT 'active', + "max_concurrency" bigint DEFAULT 2, + "quota_requests" bigint DEFAULT 0, + "quota_requests_used" bigint DEFAULT 0, + "rpm" bigint DEFAULT 0, + "expires_at" text COLLATE "C", + "created_at" text COLLATE "C", + "updated_at" text COLLATE "C", + "last_used_at" text COLLATE "C", + "requests" bigint DEFAULT 0, + "tokens_in" bigint DEFAULT 0, + "tokens_out" bigint DEFAULT 0, + "key_hash" text COLLATE "C", + "key_prefix" text COLLATE "C", + "key_suffix" text COLLATE "C", + "cache_read_tokens" bigint DEFAULT 0, + "cache_creation_tokens" bigint DEFAULT 0, + "category" text COLLATE "C" NOT NULL DEFAULT 'oauth', + "key_secret" text COLLATE "C", + "user_id" text COLLATE "C", + "group_id" bigint, + "quota" double precision NOT NULL DEFAULT 0, + "quota_used" double precision NOT NULL DEFAULT 0, + "rate_limit_5h" double precision NOT NULL DEFAULT 0, + "rate_limit_1d" double precision NOT NULL DEFAULT 0, + "rate_limit_7d" double precision NOT NULL DEFAULT 0, + "usage_5h" double precision NOT NULL DEFAULT 0, + "usage_1d" double precision NOT NULL DEFAULT 0, + "usage_7d" double precision NOT NULL DEFAULT 0, + "window_5h_start" text COLLATE "C", + "window_1d_start" text COLLATE "C", + "window_7d_start" text COLLATE "C", + "ip_whitelist" text COLLATE "C", + "ip_blacklist" text COLLATE "C", + "deleted_at" text COLLATE "C", + "group_type" text COLLATE "C" NOT NULL DEFAULT 'all', + "allowed_vms" text COLLATE "C" NOT NULL DEFAULT '[]', + "vm_pool_id" text COLLATE "C", + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_api_keys_expires" ON "api_keys" ("expires_at"); +CREATE INDEX "idx_api_keys_deleted_at" ON "api_keys" ("deleted_at"); +CREATE INDEX "idx_api_keys_status" ON "api_keys" ("status"); +CREATE INDEX "idx_api_keys_group" ON "api_keys" ("group_id"); +CREATE INDEX "idx_api_keys_user" ON "api_keys" ("user_id"); +CREATE UNIQUE INDEX "idx_api_keys_key_hash" ON "api_keys" ("key_hash") WHERE key_hash IS NOT NULL; +CREATE UNIQUE INDEX "idx_api_keys_unique_6" ON "api_keys" ("key"); + +CREATE TABLE "backup_records" ( + "id" text COLLATE "C", + "created_at" text COLLATE "C" NOT NULL, + "kind" text COLLATE "C", + "status" text COLLATE "C" NOT NULL, + "file_path" text COLLATE "C", + "file_name" text COLLATE "C", + "size_bytes" bigint, + "sha256" text COLLATE "C", + "db_bytes" bigint, + "includes_json" text COLLATE "C", + "error" text COLLATE "C", + "note" text COLLATE "C", + PRIMARY KEY ("id") +); + + +CREATE TABLE "cluster_node_direct_links" ( + "node_id" text COLLATE "C", + "address" text COLLATE "C" NOT NULL, + "port" bigint NOT NULL, + "token" text COLLATE "C" NOT NULL, + "cert_sha256" text COLLATE "C", + "updated_at" text COLLATE "C" NOT NULL, + PRIMARY KEY ("node_id") +); + + +CREATE TABLE "cluster_nodes" ( + "id" text COLLATE "C", + "label" text COLLATE "C" NOT NULL, + "host" text COLLATE "C" NOT NULL, + "port" bigint NOT NULL DEFAULT 22, + "username" text COLLATE "C" NOT NULL, + "auth_type" text COLLATE "C" NOT NULL, + "password" text COLLATE "C", + "private_key" text COLLATE "C", + "passphrase" text COLLATE "C", + "host_key_alg" text COLLATE "C" NOT NULL, + "host_key_sha256" text COLLATE "C" NOT NULL, + "jump_node_id" text COLLATE "C", + "created_at" text COLLATE "C" NOT NULL, + "updated_at" text COLLATE "C" NOT NULL, + "platform" text COLLATE "C" NOT NULL DEFAULT 'linux', + PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "idx_cluster_nodes_endpoint" ON "cluster_nodes" ("host", "port", "username"); + +CREATE TABLE "groups" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "name" text COLLATE "C" NOT NULL, + "description" text COLLATE "C", + "rate_multiplier" double precision NOT NULL DEFAULT 1.0, + "is_exclusive" bigint NOT NULL DEFAULT 0, + "platform" text COLLATE "C" NOT NULL DEFAULT 'claude', + "rpm_limit" bigint NOT NULL DEFAULT 0, + "sort_order" bigint NOT NULL DEFAULT 0, + "status" text COLLATE "C" NOT NULL DEFAULT 'active', + "created_at" text COLLATE "C", + "updated_at" text COLLATE "C", + "deleted_at" text COLLATE "C", + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_groups_deleted_at" ON "groups" ("deleted_at"); +CREATE INDEX "idx_groups_status" ON "groups" ("status"); +CREATE UNIQUE INDEX "idx_groups_name_active" ON "groups" ("name") WHERE deleted_at IS NULL; + +CREATE TABLE "panel_users_legacy" ( + "id" text COLLATE "C", + "username" text COLLATE "C" NOT NULL, + "password_hash" text COLLATE "C" NOT NULL, + "role" text COLLATE "C" NOT NULL, + "enabled" bigint NOT NULL DEFAULT 1, + "created_at" text COLLATE "C", + "updated_at" text COLLATE "C", + "last_login_at" text COLLATE "C", + PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "idx_panel_users_username" ON "panel_users_legacy" (ascii_fold("username")); + +CREATE TABLE "proxies" ( + "id" text COLLATE "C", + "protocol" text COLLATE "C" DEFAULT 'socks5', + "host" text COLLATE "C", + "port" bigint, + "username" text COLLATE "C", + "password" text COLLATE "C", + "raw" text COLLATE "C", + "status" text COLLATE "C" DEFAULT 'unknown', + "enabled" bigint DEFAULT 1, + "bound_vm_id" text COLLATE "C", + "consecutive_failures" bigint DEFAULT 0, + "latency_ms" bigint, + "last_probe_at" text COLLATE "C", + "last_error" text COLLATE "C", + "created_at" text COLLATE "C", + "name" text COLLATE "C" NOT NULL DEFAULT '', + "updated_at" text COLLATE "C", + "deleted_at" text COLLATE "C", + "owner_user_id" text COLLATE "C", + "geo_ip" text COLLATE "C", + "geo_country" text COLLATE "C", + "geo_country_code" text COLLATE "C", + "geo_region" text COLLATE "C", + "geo_city" text COLLATE "C", + "geo_isp" text COLLATE "C", + "geo_timezone" text COLLATE "C", + "geo_checked_at" text COLLATE "C", + "geo_error" text COLLATE "C", + "label" text COLLATE "C", + "geo_v6_ip" text COLLATE "C", + "geo_v6_country" text COLLATE "C", + "geo_v6_country_code" text COLLATE "C", + "geo_v6_region" text COLLATE "C", + "geo_v6_city" text COLLATE "C", + "geo_v6_isp" text COLLATE "C", + "geo_v6_timezone" text COLLATE "C", + "geo_v6_checked_at" text COLLATE "C", + "geo_v6_error" text COLLATE "C", + "domain_forward" bigint NOT NULL DEFAULT 0, + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_proxies_owner" ON "proxies" ("owner_user_id"); +CREATE INDEX "idx_proxies_deleted_at" ON "proxies" ("deleted_at"); +CREATE INDEX "idx_proxies_status" ON "proxies" ("status"); + +CREATE TABLE "redeem_codes" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "code" text COLLATE "C" NOT NULL, + "type" text COLLATE "C" NOT NULL DEFAULT 'balance', + "value" double precision NOT NULL DEFAULT 0, + "status" text COLLATE "C" NOT NULL DEFAULT 'unused', + "used_by" text COLLATE "C", + "used_at" text COLLATE "C", + "notes" text COLLATE "C", + "expires_at" text COLLATE "C", + "created_at" text COLLATE "C", + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_redeem_codes_used_by" ON "redeem_codes" ("used_by"); +CREATE INDEX "idx_redeem_codes_status" ON "redeem_codes" ("status"); +CREATE UNIQUE INDEX "idx_redeem_codes_unique_2" ON "redeem_codes" ("code"); + +CREATE TABLE "refusal_device_blocks" ( + "device_id" text COLLATE "C", + "first_seen_at" text COLLATE "C" NOT NULL, + "last_seen_at" text COLLATE "C" NOT NULL, + "hit_count" bigint NOT NULL DEFAULT 0, + "source_request_id" text COLLATE "C", + "fingerprint" text COLLATE "C", + "reason" text COLLATE "C", + PRIMARY KEY ("device_id") +); + +CREATE INDEX "idx_refusal_device_last_seen" ON "refusal_device_blocks" ("last_seen_at"); + +CREATE TABLE "refusal_guards" ( + "fingerprint" text COLLATE "C", + "model" text COLLATE "C" NOT NULL DEFAULT '', + "first_seen_at" text COLLATE "C" NOT NULL, + "last_seen_at" text COLLATE "C" NOT NULL, + "hit_count" bigint NOT NULL DEFAULT 0, + "source_request_id" text COLLATE "C", + "error_message" text COLLATE "C", + "preview" text COLLATE "C", + "expires_at" text COLLATE "C", + "signature" text COLLATE "C", + PRIMARY KEY ("fingerprint") +); + +CREATE INDEX "idx_refusal_guards_last_seen" ON "refusal_guards" ("last_seen_at"); +CREATE INDEX "idx_refusal_guards_model" ON "refusal_guards" ("model"); + +CREATE TABLE "request_attempts" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "request_id" text COLLATE "C" NOT NULL, + "attempt_no" bigint NOT NULL, + "vm_id" text COLLATE "C", + "account_id" text COLLATE "C", + "model" text COLLATE "C", + "selection_reason" text COLLATE "C", + "started_at" text COLLATE "C" NOT NULL, + "completed_at" text COLLATE "C", + "upstream_status" bigint, + "error_scope" text COLLATE "C", + "action" text COLLATE "C", + "cooldown_until" bigint, + "downstream_committed" bigint NOT NULL DEFAULT 0, + "terminal_state" text COLLATE "C", + "usage_json" text COLLATE "C", + "wait_ms" bigint, + "ttft_ms" bigint, + "latency_ms" bigint, + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_request_attempt_account" ON "request_attempts" ("account_id", "started_at" DESC); +CREATE INDEX "idx_request_attempt_request" ON "request_attempts" ("request_id", "attempt_no"); +CREATE UNIQUE INDEX "idx_request_attempt_unique" ON "request_attempts" ("request_id", "attempt_no"); + +CREATE TABLE "request_log_debug" ( + "request_id" text COLLATE "C", + "ts" text COLLATE "C" NOT NULL, + "record_json" text COLLATE "C" NOT NULL, + PRIMARY KEY ("request_id") +); + +CREATE INDEX "idx_reqdbg_ts" ON "request_log_debug" ("ts" DESC); + +CREATE TABLE "settings" ( + "key" text COLLATE "C", + "value" text COLLATE "C" NOT NULL, + "updated_at" text COLLATE "C", + PRIMARY KEY ("key") +); + + +CREATE TABLE "sticky_sessions" ( + "key" text COLLATE "C", + "account_id" text COLLATE "C", + "vm_id" text COLLATE "C", + "session_id" text COLLATE "C", + "bound_at" bigint, + "expires_at" bigint, + "hits" bigint DEFAULT 0, + "device_id" text COLLATE "C", + "generation" bigint NOT NULL DEFAULT 0, + "slot_index" bigint, + PRIMARY KEY ("key") +); + +CREATE INDEX "idx_sticky_device" ON "sticky_sessions" ("device_id"); +CREATE INDEX "idx_sticky_expires" ON "sticky_sessions" ("expires_at"); + +CREATE TABLE "usage_logs" ( + "id" text COLLATE "C", + "request_id" text COLLATE "C", + "created_at" text COLLATE "C" NOT NULL, + "log_mode" text COLLATE "C", + "method" text COLLATE "C", + "path" text COLLATE "C", + "protocol" text COLLATE "C", + "model" text COLLATE "C", + "stream" bigint, + "status" bigint, + "duration_ms" bigint, + "api_key_kind" text COLLATE "C", + "api_key_id" text COLLATE "C", + "vm_id" text COLLATE "C", + "account_id" text COLLATE "C", + "workspace" text COLLATE "C", + "input_tokens" bigint, + "output_tokens" bigint, + "error_code" text COLLATE "C", + "error_message" text COLLATE "C", + "user_agent" text COLLATE "C", + "ip_address" text COLLATE "C", + "has_tools" bigint, + "via" text COLLATE "C", + "cache_read_tokens" bigint, + "cache_creation_tokens" bigint, + "attempt_count" bigint, + "final_state" text COLLATE "C", + "final_account_id" text COLLATE "C", + "cache_creation_5m_tokens" bigint, + "cache_creation_1h_tokens" bigint, + "requested_model" text COLLATE "C", + "upstream_model" text COLLATE "C", + "model_mismatch" bigint, + "first_token_ms" bigint, + "stop_reason" text COLLATE "C", + "input_cost" double precision, + "output_cost" double precision, + "cache_read_cost" double precision, + "cache_creation_cost" double precision, + "total_cost" double precision, + "pricing_model" text COLLATE "C", + "api_key_presented" text COLLATE "C", + "user_id" text COLLATE "C", + "group_id" bigint, + "actual_cost" double precision, + "rate_multiplier" double precision, + "service_tier" text COLLATE "C", + "speed" text COLLATE "C", + "long_context" bigint, + "session_id" text COLLATE "C", + "reasoning_effort" text COLLATE "C", + "outbound_session_id" text COLLATE "C", + "intercept" text COLLATE "C", + "vm_owner_user_id" text COLLATE "C", + "web_search_requests" bigint NOT NULL DEFAULT 0, + "web_search_cost" double precision, + "price_source" text COLLATE "C", + "price_version" text COLLATE "C", + "inbound_device_id" text COLLATE "C", + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_usage_logs_inbound_device" ON "usage_logs" ("inbound_device_id", "created_at"); +CREATE INDEX "idx_usage_logs_vm_owner" ON "usage_logs" ("vm_owner_user_id", "created_at"); +CREATE INDEX "idx_usage_logs_outbound_session_created" ON "usage_logs" ("outbound_session_id", "created_at"); +CREATE INDEX "idx_usage_logs_session_created" ON "usage_logs" ("session_id", "created_at"); +CREATE INDEX "idx_usage_logs_group_created" ON "usage_logs" ("group_id", "created_at"); +CREATE INDEX "idx_usage_logs_user_created" ON "usage_logs" ("user_id", "created_at"); +CREATE INDEX "idx_reqlog_status" ON "usage_logs" ("status", "created_at" DESC); +CREATE INDEX "idx_reqlog_cost_ts" ON "usage_logs" ("created_at" DESC); +CREATE INDEX "idx_reqlog_account" ON "usage_logs" ("account_id", "created_at" DESC); +CREATE INDEX "idx_reqlog_requested_model" ON "usage_logs" ("requested_model"); +CREATE UNIQUE INDEX "idx_reqlog_reqid" ON "usage_logs" ("request_id"); +CREATE INDEX "idx_reqlog_vm" ON "usage_logs" ("vm_id", "created_at" DESC); +CREATE INDEX "idx_reqlog_key" ON "usage_logs" ("api_key_id", "created_at" DESC); +CREATE INDEX "idx_reqlog_ts" ON "usage_logs" ("created_at" DESC); + +CREATE TABLE "users" ( + "id" text COLLATE "C", + "email" text COLLATE "C" NOT NULL, + "username" text COLLATE "C" NOT NULL, + "password_hash" text COLLATE "C" NOT NULL, + "role" text COLLATE "C" NOT NULL DEFAULT 'user', + "balance" double precision NOT NULL DEFAULT 0, + "concurrency" bigint NOT NULL DEFAULT 5, + "status" text COLLATE "C" NOT NULL DEFAULT 'active', + "notes" text COLLATE "C" NOT NULL DEFAULT '', + "last_login_at" text COLLATE "C", + "last_active_at" text COLLATE "C", + "created_at" text COLLATE "C", + "updated_at" text COLLATE "C", + "deleted_at" text COLLATE "C", + "vm_create_quota" bigint NOT NULL DEFAULT 0, + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_users_deleted_at" ON "users" ("deleted_at"); +CREATE INDEX "idx_users_status" ON "users" ("status"); +CREATE UNIQUE INDEX "idx_users_email_active" ON "users" ("email") WHERE deleted_at IS NULL; +CREATE UNIQUE INDEX "idx_users_username_active" ON "users" (ascii_fold("username")) WHERE deleted_at IS NULL; + +CREATE TABLE "vm_pool_members" ( + "pool_id" text COLLATE "C" NOT NULL, + "vm_id" text COLLATE "C" NOT NULL, + "created_at" text COLLATE "C" NOT NULL, + PRIMARY KEY ("pool_id", "vm_id") +); + +CREATE UNIQUE INDEX "idx_vm_pool_members_vm" ON "vm_pool_members" ("vm_id"); + +CREATE TABLE "vm_pools" ( + "id" text COLLATE "C", + "name" text COLLATE "C" NOT NULL, + "enabled" bigint NOT NULL DEFAULT 1, + "created_at" text COLLATE "C" NOT NULL, + "updated_at" text COLLATE "C" NOT NULL, + PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "idx_vm_pools_name" ON "vm_pools" (ascii_fold("name")); + +CREATE TABLE "vms" ( + "id" text COLLATE "C", + "name" text COLLATE "C", + "status" text COLLATE "C", + "schedulable" bigint DEFAULT 1, + "email" text COLLATE "C", + "account_uuid" text COLLATE "C", + "org_uuid" text COLLATE "C", + "access_token" text COLLATE "C", + "refresh_token" text COLLATE "C", + "session_key" text COLLATE "C", + "oauth_expires_at" text COLLATE "C", + "oauth_source" text COLLATE "C", + "proxy_id" text COLLATE "C", + "claude_code_version" text COLLATE "C", + "timezone" text COLLATE "C", + "locale" text COLLATE "C", + "vm_json" text COLLATE "C" NOT NULL, + "encrypted" bigint DEFAULT 0, + "file_mtime_ms" bigint, + "synced_at" text COLLATE "C", + "created_at" text COLLATE "C", + "updated_at" text COLLATE "C", + "owner_user_id" text COLLATE "C", + "origin" text COLLATE "C" NOT NULL DEFAULT 'platform', + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_vms_origin" ON "vms" ("origin"); +CREATE INDEX "idx_vms_owner" ON "vms" ("owner_user_id"); +CREATE INDEX "idx_vms_account" ON "vms" ("account_uuid"); + +CREATE TABLE thinking_signature_owners (key text COLLATE "C" PRIMARY KEY, account_id text COLLATE "C" NOT NULL, created_at bigint NOT NULL); +CREATE INDEX idx_thinking_owner_created ON thinking_signature_owners(created_at); + +INSERT INTO groups (id,name,description,sort_order,created_at,updated_at) VALUES +(1,'coding','Coding pool (default)',0,to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'),to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')), +(2,'other','Other traffic',1,to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'),to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')); +ALTER SEQUENCE groups_id_seq RESTART WITH 3; + +ALTER TABLE "cluster_node_direct_links" ADD FOREIGN KEY ("node_id") REFERENCES "cluster_nodes" ("id") ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE "cluster_nodes" ADD FOREIGN KEY ("jump_node_id") REFERENCES "cluster_nodes" ("id") ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE "vm_pool_members" ADD FOREIGN KEY ("pool_id") REFERENCES "vm_pools" ("id") ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; diff --git a/src/lib/db/repos/account-runtime-repo.mjs b/src/lib/db/repos/account-runtime-repo.mjs index 2c08371d2..41b43eee3 100644 --- a/src/lib/db/repos/account-runtime-repo.mjs +++ b/src/lib/db/repos/account-runtime-repo.mjs @@ -10,7 +10,7 @@ * with ms-epoch numbers; conversion happens here. */ -import { getDb } from '../database.mjs' +import { getDb, withTransaction } from '../database.mjs' function parse(value, fallback = null) { if (value == null) return fallback @@ -63,14 +63,14 @@ function mergeState(rt, acc) { export class AccountRuntimeRepo { constructor(db = getDb()) { this.db = db - this._getRt = db.prepare('SELECT * FROM account_runtime WHERE account_id = ?') + this._getRt = db.prepare('SELECT * FROM account_runtime WHERE account_id = $1') this._getAcc = db.prepare(` SELECT id, vm_id, priority, load_factor, last_used_at, temp_unschedulable_until, temp_unschedulable_reason, rate_limited_at, rate_limit_reset_at, overload_until, session_window_start, session_window_end, session_window_status, updated_at - FROM accounts WHERE id = ? + FROM accounts WHERE id = $1 `) this._listIds = db.prepare(` SELECT account_id FROM account_runtime @@ -78,28 +78,30 @@ export class AccountRuntimeRepo { ORDER BY 1 `) this._ensureAcc = db.prepare(` - INSERT OR IGNORE INTO accounts (id, vm_id, name, extra, created_at, updated_at) - VALUES (?, ?, ?, '{}', ?, ?) + INSERT INTO accounts (id, vm_id, name, extra, created_at, updated_at) + VALUES ($1, $2, $3, '{}', $4, $5) + ON CONFLICT(id) DO NOTHING `) this._linkDefaultGroup = db.prepare(` - INSERT OR IGNORE INTO account_groups (account_id, group_id, priority, created_at) - VALUES (?, 1, 50, ?) + INSERT INTO account_groups (account_id, group_id, priority, created_at) + VALUES ($1, 1, 50, $2) + ON CONFLICT(account_id, group_id) DO NOTHING `) this._updateAcc = db.prepare(` UPDATE accounts SET - vm_id = ?, - priority = ?, load_factor = ?, last_used_at = ?, - temp_unschedulable_until = ?, temp_unschedulable_reason = ?, - rate_limited_at = ?, rate_limit_reset_at = ?, overload_until = ?, - session_window_start = ?, session_window_end = ?, session_window_status = ?, - updated_at = ? - WHERE id = ? + vm_id = $1, + priority = $2, load_factor = $3, last_used_at = $4, + temp_unschedulable_until = $5, temp_unschedulable_reason = $6, + rate_limited_at = $7, rate_limit_reset_at = $8, overload_until = $9, + session_window_start = $10, session_window_end = $11, session_window_status = $12, + updated_at = $13 + WHERE id = $14 `) this._upsertRt = db.prepare(` INSERT INTO account_runtime ( account_id, vm_id, status, model_states_json, credential_generation, refresh_status, worker_heartbeat_at, worker_status_json, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT(account_id) DO UPDATE SET vm_id = excluded.vm_id, status = excluded.status, @@ -110,65 +112,67 @@ export class AccountRuntimeRepo { worker_status_json = excluded.worker_status_json, updated_at = excluded.updated_at `) - this._removeRt = db.prepare('DELETE FROM account_runtime WHERE account_id = ?') + this._removeRt = db.prepare('DELETE FROM account_runtime WHERE account_id = $1') } - get(accountId) { - const rt = this._getRt.get(accountId) - const acc = this._getAcc.get(accountId) + async get(accountId) { + const rt = await this._getRt.get(accountId) + const acc = await this._getAcc.get(accountId) if (!rt && !acc) return null return mergeState(rt, acc) } - list() { - return this._listIds - .all() - .map((r) => this.get(r.account_id ?? r.id)) - .filter(Boolean) + async list() { + const ids = await this._listIds.all() + return (await Promise.all(ids.map((r) => this.get(r.account_id ?? r.id)))).filter(Boolean) } - upsert(input = {}) { - if (!input.account_id || !input.vm_id) return null - const current = this.get(input.account_id) || {} - const next = { - ...current, - ...input, - status: input.status ?? current.status ?? 'unknown', - priority: Number(input.priority ?? current.priority ?? 0), - weight: Math.max(0, Number(input.weight ?? current.weight ?? 1)), - model_states: input.model_states ?? current.model_states ?? {}, - updated_at: new Date().toISOString(), - } - this._ensureAcc.run(next.account_id, next.vm_id, next.account_id, next.updated_at, next.updated_at) - this._linkDefaultGroup.run(next.account_id, next.updated_at) - this._updateAcc.run( - next.vm_id, - next.priority, - next.weight, - toIso(next.last_used_at), - toIso(next.cooldown_until), - next.cooldown_reason ?? null, - toIso(next.rate_limited_at), - toIso(next.rate_limit_reset_at), - toIso(next.overload_until), - toIso(next.session_window_start), - toIso(next.session_window_end), - next.session_window_status ?? null, - next.updated_at, - next.account_id, - ) - this._upsertRt.run( - next.account_id, - next.vm_id, - next.status, - JSON.stringify(next.model_states || {}), - next.credential_generation ?? 0, - next.refresh_status ?? null, - next.worker_heartbeat_at ?? null, - next.worker_status != null ? JSON.stringify(next.worker_status) : null, - next.updated_at, - ) - return this.get(next.account_id) + async upsert(input = {}) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(input.account_id)) + + if (!input.account_id || !input.vm_id) return null + const current = (await this.get(input.account_id)) || {} + const next = { + ...current, + ...input, + status: input.status ?? current.status ?? 'unknown', + priority: Number(input.priority ?? current.priority ?? 0), + weight: Math.max(0, Number(input.weight ?? current.weight ?? 1)), + model_states: input.model_states ?? current.model_states ?? {}, + updated_at: new Date().toISOString(), + } + await this._ensureAcc.run(next.account_id, next.vm_id, next.account_id, next.updated_at, next.updated_at) + await this._linkDefaultGroup.run(next.account_id, next.updated_at) + await this._updateAcc.run( + next.vm_id, + next.priority, + next.weight, + toIso(next.last_used_at), + toIso(next.cooldown_until), + next.cooldown_reason ?? null, + toIso(next.rate_limited_at), + toIso(next.rate_limit_reset_at), + toIso(next.overload_until), + toIso(next.session_window_start), + toIso(next.session_window_end), + next.session_window_status ?? null, + next.updated_at, + next.account_id, + ) + await this._upsertRt.run( + next.account_id, + next.vm_id, + next.status, + JSON.stringify(next.model_states || {}), + next.credential_generation ?? 0, + next.refresh_status ?? null, + next.worker_heartbeat_at ?? null, + next.worker_status != null ? JSON.stringify(next.worker_status) : null, + next.updated_at, + ) + return await this.get(next.account_id) + }) } /** @@ -176,7 +180,7 @@ export class AccountRuntimeRepo { * (sub2api account.rate_limited_at / rate_limit_reset_at / overload_until / * session_window_* counterpart). Only provided fields are updated. */ - updateWindow( + async updateWindow( accountId, { vmId = null, @@ -188,50 +192,58 @@ export class AccountRuntimeRepo { sessionWindowStatus, } = {}, ) { - const current = this.get(accountId) || { - account_id: accountId, - vm_id: vmId, - model_states: {}, - } - if (!current.vm_id && vmId) current.vm_id = vmId - if (!current.vm_id) return null - if (rateLimitedAt !== undefined) current.rate_limited_at = rateLimitedAt - if (rateLimitResetAt !== undefined) current.rate_limit_reset_at = rateLimitResetAt - if (overloadUntil !== undefined) current.overload_until = overloadUntil - if (sessionWindowStart !== undefined) current.session_window_start = sessionWindowStart - if (sessionWindowEnd !== undefined) current.session_window_end = sessionWindowEnd - if (sessionWindowStatus !== undefined) current.session_window_status = sessionWindowStatus - return this.upsert(current) + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) + + const current = (await this.get(accountId)) || { + account_id: accountId, + vm_id: vmId, + model_states: {}, + } + if (!current.vm_id && vmId) current.vm_id = vmId + if (!current.vm_id) return null + if (rateLimitedAt !== undefined) current.rate_limited_at = rateLimitedAt + if (rateLimitResetAt !== undefined) current.rate_limit_reset_at = rateLimitResetAt + if (overloadUntil !== undefined) current.overload_until = overloadUntil + if (sessionWindowStart !== undefined) current.session_window_start = sessionWindowStart + if (sessionWindowEnd !== undefined) current.session_window_end = sessionWindowEnd + if (sessionWindowStatus !== undefined) current.session_window_status = sessionWindowStatus + return await this.upsert(current) + }) } - markCooldown(accountId, { vmId, until, reason, model = null, status = 'cooldown' } = {}) { - const current = this.get(accountId) || { - account_id: accountId, - vm_id: vmId, - model_states: {}, - } - if (model) { - current.model_states = { ...(current.model_states || {}) } - current.model_states[model] = { - status, - cooldown_until: Number(until) || null, - reason: reason || null, - updated_at: new Date().toISOString(), + async markCooldown(accountId, { vmId, until, reason, model = null, status = 'cooldown' } = {}) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) + + const current = (await this.get(accountId)) || { + account_id: accountId, + vm_id: vmId, + model_states: {}, } - } else { - current.status = status - current.cooldown_until = Number(until) || null - current.cooldown_reason = reason || null - // Structured mirror of the protocol-level limit (sub2api account columns). - // overload_until is only the 529 hard column (RateLimitService.handle529). - // A 502 failover also uses reason provider_overloaded for 15s; writing it - // here paints every hopped VM as 过载冷却 and removes them from the pool. - if (/rate_limited|quota_exhausted/i.test(String(reason || ''))) { - current.rate_limited_at = Date.now() - current.rate_limit_reset_at = Number(until) || null + if (model) { + current.model_states = { ...(current.model_states || {}) } + current.model_states[model] = { + status, + cooldown_until: Number(until) || null, + reason: reason || null, + updated_at: new Date().toISOString(), + } + } else { + current.status = status + current.cooldown_until = Number(until) || null + current.cooldown_reason = reason || null + // Structured mirror of the protocol-level limit (sub2api account columns). + // overload_until is only the 529 hard column (RateLimitService.handle529). + // A 502 failover also uses reason provider_overloaded for 15s; writing it + // here paints every hopped VM as 过载冷却 and removes them from the pool. + if (/rate_limited|quota_exhausted/i.test(String(reason || ''))) { + current.rate_limited_at = Date.now() + current.rate_limit_reset_at = Number(until) || null + } } - } - return this.upsert(current) + return await this.upsert(current) + }) } /** Stale-access 401 / 403 permission — not quota, Fable, or 429. */ @@ -253,77 +265,94 @@ export class AccountRuntimeRepo { } /** Worker fresh / refresh success: drop only auth leftover so quota cools stay. */ - clearAuthCooldown(accountId, { vmId = null } = {}) { - const current = this.get(accountId) - if (!current || !this.isAuthCooldown(current)) return false - return !!this.upsert({ - ...current, - vm_id: current.vm_id || vmId, - status: 'ready', - cooldown_until: null, - cooldown_reason: null, + async clearAuthCooldown(accountId, { vmId = null } = {}) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) + + const current = await this.get(accountId) + if (!current || !this.isAuthCooldown(current)) return false + return !!(await this.upsert({ + ...current, + vm_id: current.vm_id || vmId, + status: 'ready', + cooldown_until: null, + cooldown_reason: null, + })) }) } /** Live login / import: drop leftover oauth_revoked disable. Quota cools stay. */ - clearGrantRevokeCooldown(accountId, { vmId = null } = {}) { - const current = this.get(accountId) - if (!current || !this.isGrantRevokeCooldown(current)) return false - return !!this.upsert({ - ...current, - vm_id: current.vm_id || vmId, - status: 'ready', - cooldown_until: null, - cooldown_reason: null, + async clearGrantRevokeCooldown(accountId, { vmId = null } = {}) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) + + const current = await this.get(accountId) + if (!current || !this.isGrantRevokeCooldown(current)) return false + return !!(await this.upsert({ + ...current, + vm_id: current.vm_id || vmId, + status: 'ready', + cooldown_until: null, + cooldown_reason: null, + })) }) } /** Operator / panel: drop account + per-model cooldown leftovers. */ - clearAccountCooldown(accountId, { vmId = null } = {}) { - const current = this.get(accountId) - if (!current) return false - return !!this.upsert({ - ...current, - vm_id: current.vm_id || vmId, - status: 'ready', - cooldown_until: null, - cooldown_reason: null, - rate_limit_reset_at: null, - overload_until: null, - model_states: {}, + async clearAccountCooldown(accountId, { vmId = null } = {}) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) + + const current = await this.get(accountId) + if (!current) return false + return !!(await this.upsert({ + ...current, + vm_id: current.vm_id || vmId, + status: 'ready', + cooldown_until: null, + cooldown_reason: null, + rate_limit_reset_at: null, + overload_until: null, + model_states: {}, + })) }) } - clearExpired(now = Date.now()) { - for (const state of this.list()) { - if (!state.vm_id) continue - let changed = false - if (state.cooldown_until && state.cooldown_until <= now) { - state.status = 'ready' - state.cooldown_until = null - state.cooldown_reason = null - changed = true - } - if (state.rate_limit_reset_at && state.rate_limit_reset_at <= now) { - state.rate_limit_reset_at = null - changed = true - } - if (state.overload_until && state.overload_until <= now) { - state.overload_until = null - changed = true - } - const models = { ...(state.model_states || {}) } - for (const [model, modelState] of Object.entries(models)) { - if (modelState?.cooldown_until && Number(modelState.cooldown_until) <= now) { - delete models[model] + async clearExpired(now = Date.now()) { + for (const id of await this._listIds.all()) { + await withTransaction(this.db, async () => { + const accountId = id.account_id ?? id.id + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) + const state = await this.get(accountId) + if (!state?.vm_id) return + let changed = false + if (state.cooldown_until && state.cooldown_until <= now) { + state.status = 'ready' + state.cooldown_until = null + state.cooldown_reason = null changed = true } - } - if (changed) this.upsert({ ...state, model_states: models }) + if (state.rate_limit_reset_at && state.rate_limit_reset_at <= now) { + state.rate_limit_reset_at = null + changed = true + } + if (state.overload_until && state.overload_until <= now) { + state.overload_until = null + changed = true + } + const models = { ...(state.model_states || {}) } + for (const [model, modelState] of Object.entries(models)) { + if (modelState?.cooldown_until && Number(modelState.cooldown_until) <= now) { + delete models[model] + changed = true + } + } + if (changed) await this.upsert({ ...state, model_states: models }) + }) } } - remove(accountId) { - return this._removeRt.run(accountId).changes > 0 + async remove(accountId) { + return (await this._removeRt.run(accountId)).changes > 0 } } diff --git a/src/lib/db/repos/accounts-repo.mjs b/src/lib/db/repos/accounts-repo.mjs index e33c9df10..fd707673a 100644 --- a/src/lib/db/repos/accounts-repo.mjs +++ b/src/lib/db/repos/accounts-repo.mjs @@ -8,7 +8,8 @@ * The unified/blocked/rate-limit snapshots live inside the `extra` JSON. */ -import { getDb } from '../database.mjs' +import { getDb, withTransaction } from '../database.mjs' +import { integerValue } from '../integers.mjs' const MAX_ALLOCATIONS_PER_ACCOUNT = 50 @@ -70,7 +71,7 @@ function buildExtra(acc) { export class AccountsRepo { constructor(db = getDb()) { this.db = db - this._get = db.prepare('SELECT * FROM accounts WHERE id = ?') + this._get = db.prepare('SELECT * FROM accounts WHERE id = $1') this._list = db.prepare('SELECT * FROM accounts ORDER BY id') this._insert = db.prepare(` INSERT INTO accounts (id, vm_id, email, name, platform, type, status, @@ -79,77 +80,80 @@ export class AccountsRepo { requests, tokens_in, tokens_out, cache_read_tokens, cache_creation_tokens, extra, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19) `) this._update = db.prepare(` - UPDATE accounts SET vm_id = ?, email = ?, name = ?, type = ?, - concurrency = ?, concurrency_override = ?, - max_rpm = ?, rpm_override = ?, - requests = ?, tokens_in = ?, tokens_out = ?, - cache_read_tokens = ?, cache_creation_tokens = ?, - extra = ?, updated_at = ? - WHERE id = ? + UPDATE accounts SET vm_id = $1, email = $2, name = $3, type = $4, + concurrency = $5, concurrency_override = $6, + max_rpm = $7, rpm_override = $8, + requests = $9, tokens_in = $10, tokens_out = $11, + cache_read_tokens = $12, cache_creation_tokens = $13, + extra = $14, updated_at = $15 + WHERE id = $16 `) this._insertAlloc = db.prepare(` INSERT INTO account_allocations (account_id, at, source, util_5h, util_7d, status_5h, status_7d, claim, tokens_in, tokens_out) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) `) this._trimAlloc = db.prepare(` - DELETE FROM account_allocations WHERE account_id = ? AND id NOT IN ( - SELECT id FROM account_allocations WHERE account_id = ? ORDER BY id DESC LIMIT ? + DELETE FROM account_allocations WHERE account_id = $1 AND id NOT IN ( + SELECT id FROM account_allocations WHERE account_id = $2 ORDER BY id DESC LIMIT $3 ) `) this._recentAlloc = db.prepare(` - SELECT * FROM account_allocations WHERE account_id = ? ORDER BY id DESC LIMIT ? + SELECT * FROM account_allocations WHERE account_id = $1 ORDER BY id DESC LIMIT $2 `) - this._removeAllocs = db.prepare('DELETE FROM account_allocations WHERE account_id = ?') - this._removeGroups = db.prepare('DELETE FROM account_groups WHERE account_id = ?') - this._remove = db.prepare('DELETE FROM accounts WHERE id = ?') + this._removeAllocs = db.prepare('DELETE FROM account_allocations WHERE account_id = $1') + this._removeGroups = db.prepare('DELETE FROM account_groups WHERE account_id = $1') + this._remove = db.prepare('DELETE FROM accounts WHERE id = $1') this._linkGroup = db.prepare(` - INSERT OR IGNORE INTO account_groups (account_id, group_id, priority, created_at) - VALUES (?, ?, ?, ?) + INSERT INTO account_groups (account_id, group_id, priority, created_at) + VALUES ($1, $2, $3, $4) + ON CONFLICT(account_id, group_id) DO NOTHING `) } - get(accountId) { - return rowToAccount(this._get.get(accountId)) + async get(accountId) { + return rowToAccount(await this._get.get(accountId)) } - list() { - return this._list.all().map(rowToAccount) + async list() { + return (await this._list.all()).map(rowToAccount) } /** Insert if missing, returns stored account. New accounts join the coding group (id=1). */ - insert(acc) { - const now = new Date().toISOString() - this._insert.run( - acc.account_id, - acc.vm_id ?? null, - acc.email ?? null, - acc.name ?? acc.email ?? acc.account_id, - acc.platform ?? 'claude', - acc.type ?? 'oauth', - acc.status ?? 'active', - acc.max_concurrency ?? 2, - acc.concurrency_override ? 1 : 0, - acc.max_rpm ?? 0, - acc.rpm_override ? 1 : 0, - acc.requests ?? 0, - acc.tokens_in ?? 0, - acc.tokens_out ?? 0, - acc.cache_read_tokens ?? 0, - acc.cache_creation_tokens ?? 0, - buildExtra(acc), - acc.created_at ?? now, - acc.updated_at ?? now, - ) - this._linkGroup.run(acc.account_id, acc.group_id ?? 1, acc.group_priority ?? 50, now) - return this.get(acc.account_id) + async insert(acc) { + return withTransaction(this.db, async () => { + const now = new Date().toISOString() + await this._insert.run( + acc.account_id, + acc.vm_id ?? null, + acc.email ?? null, + acc.name ?? acc.email ?? acc.account_id, + acc.platform ?? 'claude', + acc.type ?? 'oauth', + acc.status ?? 'active', + acc.max_concurrency ?? 2, + acc.concurrency_override ? 1 : 0, + acc.max_rpm ?? 0, + acc.rpm_override ? 1 : 0, + integerValue(acc.requests), + integerValue(acc.tokens_in), + integerValue(acc.tokens_out), + integerValue(acc.cache_read_tokens), + integerValue(acc.cache_creation_tokens), + buildExtra(acc), + acc.created_at ?? now, + acc.updated_at ?? now, + ) + await this._linkGroup.run(acc.account_id, acc.group_id ?? 1, acc.group_priority ?? 50, now) + return await this.get(acc.account_id) + }) } /** Full-row save of a mutated account object. */ - save(acc) { - this._update.run( + async save(acc) { + await this._update.run( acc.vm_id ?? null, acc.email ?? null, acc.name ?? acc.email ?? acc.account_id, @@ -158,59 +162,60 @@ export class AccountsRepo { acc.concurrency_override ? 1 : 0, acc.max_rpm ?? 0, acc.rpm_override ? 1 : 0, - acc.requests ?? 0, - acc.tokens_in ?? 0, - acc.tokens_out ?? 0, - acc.cache_read_tokens ?? 0, - acc.cache_creation_tokens ?? 0, + integerValue(acc.requests), + integerValue(acc.tokens_in), + integerValue(acc.tokens_out), + integerValue(acc.cache_read_tokens), + integerValue(acc.cache_creation_tokens), buildExtra(acc), new Date().toISOString(), acc.account_id, ) - return this.get(acc.account_id) + return await this.get(acc.account_id) } - remove(accountId) { - this._removeAllocs.run(accountId) - this._removeGroups.run(accountId) - const info = this._remove.run(accountId) - return info.changes > 0 + async remove(accountId) { + return withTransaction(this.db, async () => { + await this._removeAllocs.run(accountId) + await this._removeGroups.run(accountId) + const info = await this._remove.run(accountId) + return info.changes > 0 + }) } - addAllocation(accountId, alloc, { max = MAX_ALLOCATIONS_PER_ACCOUNT } = {}) { - this._insertAlloc.run( - accountId, - alloc.at || new Date().toISOString(), - alloc.source ?? null, - alloc.util_5h ?? null, - alloc.util_7d ?? null, - alloc.status_5h ?? null, - alloc.status_7d ?? null, - alloc.claim ?? null, - alloc.tokens_in ?? null, - alloc.tokens_out ?? null, - ) - this._trimAlloc.run(accountId, accountId, max) + async addAllocation(accountId, alloc, { max = MAX_ALLOCATIONS_PER_ACCOUNT } = {}) { + return withTransaction(this.db, async () => { + await this._insertAlloc.run( + accountId, + alloc.at || new Date().toISOString(), + alloc.source ?? null, + alloc.util_5h ?? null, + alloc.util_7d ?? null, + alloc.status_5h ?? null, + alloc.status_7d ?? null, + alloc.claim ?? null, + alloc.tokens_in == null ? null : integerValue(alloc.tokens_in), + alloc.tokens_out == null ? null : integerValue(alloc.tokens_out), + ) + await this._trimAlloc.run(accountId, accountId, max) + }) } - recentAllocations(accountId, limit = 5) { - return this._recentAlloc - .all(accountId, limit) - .reverse() - .map((r) => ({ - at: r.at, - source: r.source ?? undefined, - util_5h: r.util_5h, - util_7d: r.util_7d, - status_5h: r.status_5h ?? undefined, - status_7d: r.status_7d ?? undefined, - claim: r.claim, - tokens_in: r.tokens_in, - tokens_out: r.tokens_out, - })) + async recentAllocations(accountId, limit = 5) { + return (await this._recentAlloc.all(accountId, limit)).reverse().map((r) => ({ + at: r.at, + source: r.source ?? undefined, + util_5h: r.util_5h, + util_7d: r.util_7d, + status_5h: r.status_5h ?? undefined, + status_7d: r.status_7d ?? undefined, + claim: r.claim, + tokens_in: r.tokens_in, + tokens_out: r.tokens_out, + })) } - allocationCount(accountId) { - return this.db.prepare('SELECT COUNT(*) c FROM account_allocations WHERE account_id = ?').get(accountId).c + async allocationCount(accountId) { + return (await this.db.prepare('SELECT COUNT(*) c FROM account_allocations WHERE account_id = $1').get(accountId)).c } } diff --git a/src/lib/db/repos/api-endpoints-repo.mjs b/src/lib/db/repos/api-endpoints-repo.mjs index 50c79322f..7deea21c9 100644 --- a/src/lib/db/repos/api-endpoints-repo.mjs +++ b/src/lib/db/repos/api-endpoints-repo.mjs @@ -82,43 +82,43 @@ export class ApiEndpointsRepo { constructor(db = getDb()) { this.db = db this._listEp = db.prepare('SELECT * FROM api_endpoints ORDER BY priority DESC, created_at, id') - this._getEp = db.prepare('SELECT * FROM api_endpoints WHERE id = ?') + this._getEp = db.prepare('SELECT * FROM api_endpoints WHERE id = $1') this._insEp = db.prepare( - `INSERT INTO api_endpoints (${EP_COLS.join(',')}) VALUES (${EP_COLS.map(() => '?').join(',')})`, + `INSERT INTO api_endpoints (${EP_COLS.join(',')}) VALUES (${EP_COLS.map((_, i) => '$' + (i + 1)).join(',')})`, ) - this._delEp = db.prepare('DELETE FROM api_endpoints WHERE id = ?') - this._listKeys = db.prepare('SELECT * FROM api_endpoint_keys WHERE endpoint_id = ? ORDER BY created_at, id') - this._getKey = db.prepare('SELECT * FROM api_endpoint_keys WHERE id = ?') + this._delEp = db.prepare('DELETE FROM api_endpoints WHERE id = $1') + this._listKeys = db.prepare('SELECT * FROM api_endpoint_keys WHERE endpoint_id = $1 ORDER BY created_at, id') + this._getKey = db.prepare('SELECT * FROM api_endpoint_keys WHERE id = $1') this._insKey = db.prepare( - `INSERT INTO api_endpoint_keys (${KEY_COLS.join(',')}) VALUES (${KEY_COLS.map(() => '?').join(',')})`, + `INSERT INTO api_endpoint_keys (${KEY_COLS.join(',')}) VALUES (${KEY_COLS.map((_, i) => '$' + (i + 1)).join(',')})`, ) - this._delKey = db.prepare('DELETE FROM api_endpoint_keys WHERE id = ?') - this._delKeysEp = db.prepare('DELETE FROM api_endpoint_keys WHERE endpoint_id = ?') - this._listModels = db.prepare('SELECT * FROM api_endpoint_models WHERE endpoint_id = ? ORDER BY sort, id') + this._delKey = db.prepare('DELETE FROM api_endpoint_keys WHERE id = $1') + this._delKeysEp = db.prepare('DELETE FROM api_endpoint_keys WHERE endpoint_id = $1') + this._listModels = db.prepare('SELECT * FROM api_endpoint_models WHERE endpoint_id = $1 ORDER BY sort, id') this._insModel = db.prepare( - `INSERT INTO api_endpoint_models (${MODEL_COLS.join(',')}) VALUES (${MODEL_COLS.map(() => '?').join(',')})`, + `INSERT INTO api_endpoint_models (${MODEL_COLS.join(',')}) VALUES (${MODEL_COLS.map((_, i) => '$' + (i + 1)).join(',')})`, ) - this._delModelsEp = db.prepare('DELETE FROM api_endpoint_models WHERE endpoint_id = ?') - this._cool = db.prepare('UPDATE api_endpoint_keys SET cooldown_until = ? WHERE id = ?') + this._delModelsEp = db.prepare('DELETE FROM api_endpoint_models WHERE endpoint_id = $1') + this._cool = db.prepare('UPDATE api_endpoint_keys SET cooldown_until = $1 WHERE id = $2') this._updKey = db.prepare( - 'UPDATE api_endpoint_keys SET api_key = ?, proxy_url = ?, proxy_id = ?, disabled = ?, cooldown_until = ? WHERE id = ?', + 'UPDATE api_endpoint_keys SET api_key = $1, proxy_url = $2, proxy_id = $3, disabled = $4, cooldown_until = $5 WHERE id = $6', ) } - listHydrated() { - return this._listEp.all().map((row) => this.hydrate(rowEp(row))) + async listHydrated() { + return Promise.all((await this._listEp.all()).map((row) => this.hydrate(rowEp(row)))) } - getHydrated(id) { - return this.hydrate(rowEp(this._getEp.get(id))) + async getHydrated(id) { + return await this.hydrate(rowEp(await this._getEp.get(id))) } - hydrate(ep) { + async hydrate(ep) { if (!ep) return null return { ...ep, - api_key_entries: this._listKeys.all(ep.id).map(rowKey), - models: this._listModels.all(ep.id).map(rowModel), + api_key_entries: (await this._listKeys.all(ep.id)).map(rowKey), + models: (await this._listModels.all(ep.id)).map(rowModel), } } @@ -139,31 +139,33 @@ export class ApiEndpointsRepo { } } - insertEndpoint(rec) { + async insertEndpoint(rec) { const vals = this._epValues(rec) - this._insEp.run(...EP_COLS.map((c) => vals[c] ?? null)) - return this.getHydrated(rec.id) + await this._insEp.run(...EP_COLS.map((c) => vals[c] ?? null)) + return await this.getHydrated(rec.id) } - updateEndpoint(rec) { + async updateEndpoint(rec) { const vals = this._epValues(rec) const cols = EP_COLS.filter((c) => c !== 'id') - this.db - .prepare(`UPDATE api_endpoints SET ${cols.map((c) => `${c} = ?`).join(', ')} WHERE id = ?`) + await this.db + .prepare( + `UPDATE api_endpoints SET ${cols.map((c, i) => `${c} = $${i + 1}`).join(', ')} WHERE id = $${cols.length + 1}`, + ) .run(...cols.map((c) => vals[c] ?? null), rec.id) - return this.getHydrated(rec.id) + return await this.getHydrated(rec.id) } - removeEndpoint(id) { - return withTransaction(this.db, () => { - this._delKeysEp.run(id) - this._delModelsEp.run(id) - return this._delEp.run(id).changes > 0 + async removeEndpoint(id) { + return await withTransaction(this.db, async () => { + await this._delKeysEp.run(id) + await this._delModelsEp.run(id) + return (await this._delEp.run(id)).changes > 0 }) } - insertKey(rec) { - this._insKey.run( + async insertKey(rec) { + await this._insKey.run( rec.id, rec.endpoint_id, rec.api_key, @@ -173,58 +175,68 @@ export class ApiEndpointsRepo { rec.cooldown_until || null, rec.created_at, ) - return rowKey(this._getKey.get(rec.id)) - } - - removeKey(id) { - return this._delKey.run(id).changes > 0 - } - - updateKey(id, patch = {}) { - const rec = rowKey(this._getKey.get(id)) - if (!rec) return null - if (patch.api_key != null) { - const next = String(patch.api_key).trim() - if (next.length >= 4) rec.api_key = next - } - if (patch.proxy_url != null) rec.proxy_url = String(patch.proxy_url).trim() - if (patch.proxy_id != null) rec.proxy_id = String(patch.proxy_id).trim() - if (patch.disabled != null) rec.disabled = !!patch.disabled - if (patch.cooldown_until !== undefined) rec.cooldown_until = patch.cooldown_until || null - this._updKey.run( - rec.api_key, - rec.proxy_url || '', - rec.proxy_id || '', - bool01(rec.disabled), - rec.cooldown_until, - rec.id, - ) + return rowKey(await this._getKey.get(rec.id)) + } + + async removeKey(id) { + return (await this._delKey.run(id)).changes > 0 + } + + async updateKey(id, patch = {}) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT id FROM api_endpoint_keys WHERE id = $1 FOR UPDATE').get(id) + + const rec = rowKey(await this._getKey.get(id)) + if (!rec) return null + if (patch.api_key != null) { + const next = String(patch.api_key).trim() + if (next.length >= 4) rec.api_key = next + } + if (patch.proxy_url != null) rec.proxy_url = String(patch.proxy_url).trim() + if (patch.proxy_id != null) rec.proxy_id = String(patch.proxy_id).trim() + if (patch.disabled != null) rec.disabled = !!patch.disabled + if (patch.cooldown_until !== undefined) rec.cooldown_until = patch.cooldown_until || null + await this._updKey.run( + rec.api_key, + rec.proxy_url || '', + rec.proxy_id || '', + bool01(rec.disabled), + rec.cooldown_until, + rec.id, + ) + }) } - setKeyCooldown(id, untilIso) { - this._cool.run(untilIso, id) - } - - replaceModels(endpointId, models = []) { - withTransaction(this.db, () => { - this._delModelsEp.run(endpointId) - models.forEach((m, i) => { - const levels = Array.isArray(m.thinking?.levels) - ? m.thinking.levels - : Array.isArray(m.thinking_levels) - ? m.thinking_levels - : null - this._insModel.run( - m.id, - endpointId, - Number(m.sort ?? i), - m.name || m.upstream_name, - m.alias || m.name, - bool01(m.image), - levels ? JSON.stringify(levels) : null, + async setKeyCooldown(id, untilIso) { + await this._cool.run(untilIso, id) + } + + async replaceModels(endpointId, models = []) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT id FROM api_endpoints WHERE id = $1 FOR UPDATE').get(endpointId) + + await withTransaction(this.db, async () => { + await this._delModelsEp.run(endpointId) + await Promise.all( + models.map(async (m, i) => { + const levels = Array.isArray(m.thinking?.levels) + ? m.thinking.levels + : Array.isArray(m.thinking_levels) + ? m.thinking_levels + : null + await this._insModel.run( + m.id, + endpointId, + Number(m.sort ?? i), + m.name || m.upstream_name, + m.alias || m.name, + bool01(m.image), + levels ? JSON.stringify(levels) : null, + ) + }), ) }) + return await this.getHydrated(endpointId) }) - return this.getHydrated(endpointId) } } diff --git a/src/lib/db/repos/api-keys-repo.mjs b/src/lib/db/repos/api-keys-repo.mjs index 7ef6750a1..99177b264 100644 --- a/src/lib/db/repos/api-keys-repo.mjs +++ b/src/lib/db/repos/api-keys-repo.mjs @@ -7,7 +7,8 @@ * is quota_requests / quota_requests_used. Soft delete via deleted_at. */ -import { getDb } from '../database.mjs' +import { getDb, withTransaction } from '../database.mjs' +import { integerValue } from '../integers.mjs' const COLUMNS = [ 'id', @@ -75,6 +76,16 @@ const NUMERIC_DEFAULT_0 = new Set([ 'usage_7d', ]) +const INTEGER_COUNTERS = new Set([ + 'quota_requests', + 'quota_requests_used', + 'requests', + 'tokens_in', + 'tokens_out', + 'cache_read_tokens', + 'cache_creation_tokens', +]) + function rowToRec(row) { if (!row) return null return { ...row } @@ -88,6 +99,7 @@ function toValue(rec, c) { if (c === 'allowed_vms') return '[]' return NUMERIC_DEFAULT_0.has(c) ? 0 : null } + if (INTEGER_COUNTERS.has(c)) return integerValue(v) return v } @@ -95,63 +107,69 @@ export class ApiKeysRepo { constructor(db = getDb()) { this.db = db this._list = db.prepare('SELECT * FROM api_keys WHERE deleted_at IS NULL ORDER BY created_at') - this._get = db.prepare('SELECT * FROM api_keys WHERE id = ? AND deleted_at IS NULL') - this._getByKey = db.prepare('SELECT * FROM api_keys WHERE key = ? AND deleted_at IS NULL') - this._getByHash = db.prepare('SELECT * FROM api_keys WHERE key_hash = ? AND deleted_at IS NULL') + this._get = db.prepare('SELECT * FROM api_keys WHERE id = $1 AND deleted_at IS NULL') + this._getByKey = db.prepare('SELECT * FROM api_keys WHERE key = $1 AND deleted_at IS NULL') + this._getByHash = db.prepare('SELECT * FROM api_keys WHERE key_hash = $1 AND deleted_at IS NULL') this._insert = db.prepare(` INSERT INTO api_keys (${COLUMNS.join(', ')}) - VALUES (${COLUMNS.map(() => '?').join(', ')}) + VALUES (${COLUMNS.map((_, i) => '$' + (i + 1)).join(', ')}) `) - this._delete = db.prepare('DELETE FROM api_keys WHERE id = ?') + this._delete = db.prepare('DELETE FROM api_keys WHERE id = $1') this._recordUsage = db.prepare(` UPDATE api_keys SET requests = requests + 1, quota_requests_used = quota_requests_used + 1, - tokens_in = tokens_in + ?, - tokens_out = tokens_out + ?, - cache_read_tokens = COALESCE(cache_read_tokens, 0) + ?, - cache_creation_tokens = COALESCE(cache_creation_tokens, 0) + ?, - quota_used = quota_used + ?, - usage_5h = ?, window_5h_start = ?, - usage_1d = ?, window_1d_start = ?, - usage_7d = ?, window_7d_start = ?, - last_used_at = ?, - updated_at = ? - WHERE id = ? + tokens_in = tokens_in + $1, + tokens_out = tokens_out + $2, + cache_read_tokens = COALESCE(cache_read_tokens, 0) + $3, + cache_creation_tokens = COALESCE(cache_creation_tokens, 0) + $4, + quota_used = quota_used + $5, + status = CASE WHEN status = 'active' AND quota > 0 AND quota_used + $5 >= quota THEN 'quota_exhausted' ELSE status END, + usage_5h = $6, window_5h_start = $7, + usage_1d = $8, window_1d_start = $9, + usage_7d = $10, window_7d_start = $11, + last_used_at = $12, + updated_at = $13 + WHERE id = $14 `) } - list() { - return this._list.all().map(rowToRec) + async list() { + return (await this._list.all()).map(rowToRec) } - getById(id) { - return rowToRec(this._get.get(id)) + async getById(id, { forUpdate = false } = {}) { + return rowToRec( + await (forUpdate + ? this.db.prepare('SELECT * FROM api_keys WHERE id = $1 AND deleted_at IS NULL FOR UPDATE') + : this._get + ).get(id), + ) } - getByKey(key) { - return rowToRec(this._getByKey.get(key)) + async getByKey(key) { + return rowToRec(await this._getByKey.get(key)) } - getByHash(hash) { - return rowToRec(this._getByHash.get(hash)) + async getByHash(hash) { + return rowToRec(await this._getByHash.get(hash)) } - insert(rec) { - this._insert.run(...COLUMNS.map((c) => toValue(rec, c))) - return this.getById(rec.id) + async insert(rec) { + await this._insert.run(...COLUMNS.map((c) => toValue(rec, c))) + return await this.getById(rec.id) } /** Full-row update from a record object (id immutable). */ - update(rec) { + async update(rec) { const cols = COLUMNS.filter((c) => c !== 'id') - const sql = `UPDATE api_keys SET ${cols.map((c) => `${c} = ?`).join(', ')} WHERE id = ?` - this.db.prepare(sql).run(...cols.map((c) => toValue(rec, c)), rec.id) - return this.getById(rec.id) + const sql = `UPDATE api_keys SET ${cols.map((c, i) => `${c} = $${i + 1}`).join(', ')} WHERE id = $${cols.length + 1}` + await this.db.prepare(sql).run(...cols.map((c) => toValue(rec, c)), rec.id) + return await this.getById(rec.id) } - remove(id) { - const info = this._delete.run(id) + async remove(id) { + const info = await this._delete.run(id) return info.changes > 0 } @@ -167,33 +185,38 @@ export class ApiKeysRepo { return { usage: (Number(rec[key.usage]) || 0) + cost, start: rec[key.start] } } - recordUsage(id, { tokens_in = 0, tokens_out = 0, cache_read_tokens = 0, cache_creation_tokens = 0, cost = 0 } = {}) { - const rec = this.getById(id) - if (!rec) return null - const nowMs = Date.now() - const now = new Date(nowMs).toISOString() - const usd = Number(cost) || 0 - const w = WINDOWS.map((win) => this._rollWindow(rec, win, usd, nowMs)) - this._recordUsage.run( - Number(tokens_in) || 0, - Number(tokens_out) || 0, - Number(cache_read_tokens) || 0, - Number(cache_creation_tokens) || 0, - usd, - w[0].usage, - w[0].start, - w[1].usage, - w[1].start, - w[2].usage, - w[2].start, - now, - now, - id, - ) - return this.getById(id) + async recordUsage( + id, + { tokens_in = 0, tokens_out = 0, cache_read_tokens = 0, cache_creation_tokens = 0, cost = 0 } = {}, + ) { + return withTransaction(this.db, async () => { + const rec = await this.getById(id, { forUpdate: true }) + if (!rec) return null + const nowMs = Date.now() + const now = new Date(nowMs).toISOString() + const usd = Number(cost) || 0 + const w = WINDOWS.map((win) => this._rollWindow(rec, win, usd, nowMs)) + await this._recordUsage.run( + integerValue(tokens_in), + integerValue(tokens_out), + integerValue(cache_read_tokens), + integerValue(cache_creation_tokens), + usd, + w[0].usage, + w[0].start, + w[1].usage, + w[1].start, + w[2].usage, + w[2].start, + now, + now, + id, + ) + return await this.getById(id) + }) } - count() { - return this.db.prepare('SELECT COUNT(*) c FROM api_keys WHERE deleted_at IS NULL').get().c + async count() { + return (await this.db.prepare('SELECT COUNT(*) c FROM api_keys WHERE deleted_at IS NULL').get()).c } } diff --git a/src/lib/db/repos/backup-repo.mjs b/src/lib/db/repos/backup-repo.mjs index df476a726..6b2be9502 100644 --- a/src/lib/db/repos/backup-repo.mjs +++ b/src/lib/db/repos/backup-repo.mjs @@ -34,49 +34,49 @@ export class BackupRepo { this.db = db this._insert = db.prepare(` INSERT INTO backup_records (${COLUMNS.join(', ')}) - VALUES (${COLUMNS.map(() => '?').join(', ')}) + VALUES (${COLUMNS.map((_, i) => '$' + (i + 1)).join(', ')}) `) - this._get = db.prepare('SELECT * FROM backup_records WHERE id = ?') - this._list = db.prepare('SELECT * FROM backup_records ORDER BY created_at DESC LIMIT ?') - this._remove = db.prepare('DELETE FROM backup_records WHERE id = ?') + this._get = db.prepare('SELECT * FROM backup_records WHERE id = $1') + this._list = db.prepare('SELECT * FROM backup_records ORDER BY created_at DESC LIMIT $1') + this._remove = db.prepare('DELETE FROM backup_records WHERE id = $1') } - insert(rec) { - this._insert.run( + async insert(rec) { + await this._insert.run( ...COLUMNS.map((c) => { if (c === 'includes_json') return rec.includes ? JSON.stringify(rec.includes) : (rec.includes_json ?? null) return rec[c] ?? null }), ) - return this.get(rec.id) + return await this.get(rec.id) } - get(id) { - return fromRow(this._get.get(id)) + async get(id) { + return fromRow(await this._get.get(id)) } - list({ limit = 100 } = {}) { - return this._list.all(Math.max(1, Math.min(500, Number(limit) || 100))).map(fromRow) + async list({ limit = 100 } = {}) { + return (await this._list.all(Math.max(1, Math.min(500, Number(limit) || 100)))).map(fromRow) } - remove(id) { - return this._remove.run(id).changes > 0 + async remove(id) { + return (await this._remove.run(id)).changes > 0 } - lastSuccessful({ kinds = null } = {}) { - const rows = this.list({ limit: 200 }) + async lastSuccessful({ kinds = null } = {}) { + const rows = await this.list({ limit: 200 }) return rows.find((r) => r.status === 'ok' && (!kinds || kinds.includes(r.kind))) || null } /** Successful non-pre_restore backups beyond `keep` newest. */ - beyondRetention(keep) { - const rows = this.list({ limit: 500 }).filter((r) => r.status === 'ok' && r.kind !== 'pre_restore') + async beyondRetention(keep) { + const rows = (await this.list({ limit: 500 })).filter((r) => r.status === 'ok' && r.kind !== 'pre_restore') return rows.slice(Math.max(0, keep)) } /** pre_restore backups beyond `keep` newest. */ - preRestoreBeyond(keep = 3) { - const rows = this.list({ limit: 500 }).filter((r) => r.status === 'ok' && r.kind === 'pre_restore') + async preRestoreBeyond(keep = 3) { + const rows = (await this.list({ limit: 500 })).filter((r) => r.status === 'ok' && r.kind === 'pre_restore') return rows.slice(Math.max(0, keep)) } } diff --git a/src/lib/db/repos/cluster-nodes-repo.mjs b/src/lib/db/repos/cluster-nodes-repo.mjs index 5a792166b..13966d6f4 100644 --- a/src/lib/db/repos/cluster-nodes-repo.mjs +++ b/src/lib/db/repos/cluster-nodes-repo.mjs @@ -31,49 +31,49 @@ export class ClusterNodesRepo { constructor(db = getDb()) { this.db = db this._list = db.prepare(`SELECT ${PUBLIC_COLUMNS.join(', ')} FROM cluster_nodes ORDER BY created_at, id`) - this._get = db.prepare(`SELECT ${PUBLIC_COLUMNS.join(', ')} FROM cluster_nodes WHERE id = ?`) - this._secrets = db.prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = ?') - this._findEndpoint = db.prepare('SELECT id FROM cluster_nodes WHERE host = ? AND port = ? AND username = ?') - this._dependents = db.prepare('SELECT id FROM cluster_nodes WHERE jump_node_id = ?') + this._get = db.prepare(`SELECT ${PUBLIC_COLUMNS.join(', ')} FROM cluster_nodes WHERE id = $1`) + this._secrets = db.prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = $1') + this._findEndpoint = db.prepare('SELECT id FROM cluster_nodes WHERE host = $1 AND port = $2 AND username = $3') + this._dependents = db.prepare('SELECT id FROM cluster_nodes WHERE jump_node_id = $1') this._insert = db.prepare(` INSERT INTO cluster_nodes (id, label, host, port, username, auth_type, password, private_key, passphrase, host_key_alg, host_key_sha256, jump_node_id, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) `) this._update = db.prepare(` - UPDATE cluster_nodes SET label = ?, host = ?, port = ?, username = ?, auth_type = ?, - password = ?, private_key = ?, passphrase = ?, updated_at = ? - WHERE id = ? AND updated_at = ? + UPDATE cluster_nodes SET label = $1, host = $2, port = $3, username = $4, auth_type = $5, + password = $6, private_key = $7, passphrase = $8, updated_at = $9 + WHERE id = $10 AND updated_at = $11 RETURNING ${PUBLIC_COLUMNS.join(', ')} `) this._updateLabel = db.prepare(` - UPDATE cluster_nodes SET label = ?, updated_at = ? - WHERE id = ? AND updated_at = ? + UPDATE cluster_nodes SET label = $1, updated_at = $2 + WHERE id = $3 AND updated_at = $4 RETURNING ${PUBLIC_COLUMNS.join(', ')} `) - this._remove = db.prepare('DELETE FROM cluster_nodes WHERE id = ?') + this._remove = db.prepare('DELETE FROM cluster_nodes WHERE id = $1') this._stmts = new Map() } - list() { - return this._list.all().map((row) => ({ ...row })) + async list() { + return (await this._list.all()).map((row) => ({ ...row })) } - get(id) { - const row = this._get.get(id) + async get(id) { + const row = await this._get.get(id) return row ? { ...row } : null } - findByEndpoint(host, port, username) { - return this._findEndpoint.get(host, port, username)?.id || null + async findByEndpoint(host, port, username) { + return (await this._findEndpoint.get(host, port, username))?.id || null } - dependentsOf(id) { - return this._dependents.all(id).map((row) => row.id) + async dependentsOf(id) { + return (await this._dependents.all(id)).map((row) => row.id) } - getSecrets(id) { - const row = this._secrets.get(id) + async getSecrets(id) { + const row = await this._secrets.get(id) if (!row) return null return { password: maybeDecrypt(row.password), @@ -82,8 +82,8 @@ export class ClusterNodesRepo { } } - insert(node, now = new Date().toISOString()) { - this._insert.run( + async insert(node, now = new Date().toISOString()) { + await this._insert.run( node.id, node.label, node.host, @@ -99,14 +99,14 @@ export class ClusterNodesRepo { now, now, ) - return this.get(node.id) + return await this.get(node.id) } /** Replace normalized connection fields only if the caller's version is still current. */ - update(id, node, expectedUpdatedAt) { + async update(id, node, expectedUpdatedAt) { const now = nextUpdatedAt(expectedUpdatedAt) if (!now) return null - const row = this._update.get( + const row = await this._update.get( node.label, node.host, node.port, @@ -123,15 +123,15 @@ export class ClusterNodesRepo { } /** Rename without reading, decrypting or rewriting stored credentials. */ - updateLabel(id, label, expectedUpdatedAt) { + async updateLabel(id, label, expectedUpdatedAt) { const now = nextUpdatedAt(expectedUpdatedAt) if (!now) return null - const row = this._updateLabel.get(label, now, id, expectedUpdatedAt) + const row = await this._updateLabel.get(label, now, id, expectedUpdatedAt) return row ? { ...row } : null } - remove(id) { - return this._remove.run(id).changes > 0 + async remove(id) { + return (await this._remove.run(id)).changes > 0 } // Prepared on first use: callers that only load 026 (older fixtures) never touch this table. @@ -145,44 +145,44 @@ export class ClusterNodesRepo { } /** 'linux' or 'darwin'; null when the node is gone. Detected over SSH, never taken from the form. */ - platform(id) { - return this._stmt('SELECT platform FROM cluster_nodes WHERE id = ?').get(id)?.platform ?? null + async platform(id) { + return (await this._stmt('SELECT platform FROM cluster_nodes WHERE id = $1').get(id))?.platform ?? null } /** Not a connection edit: leaves updated_at alone so a pending node edit does not conflict. */ - setPlatform(id, platform) { - return this._stmt('UPDATE cluster_nodes SET platform = ? WHERE id = ?').run(platform, id).changes > 0 + async setPlatform(id, platform) { + return (await this._stmt('UPDATE cluster_nodes SET platform = $1 WHERE id = $2').run(platform, id)).changes > 0 } /** node_id → { address, port }; the gate token stays out of listings. */ - directLinks() { - const rows = this._stmt('SELECT node_id, address, port FROM cluster_node_direct_links').all() + async directLinks() { + const rows = await this._stmt('SELECT node_id, address, port FROM cluster_node_direct_links').all() return new Map(rows.map((row) => [row.node_id, { address: row.address, port: row.port }])) } - getDirectLink(id) { - const row = this._stmt( - 'SELECT address, port, token, cert_sha256 FROM cluster_node_direct_links WHERE node_id = ?', + async getDirectLink(id) { + const row = await this._stmt( + 'SELECT address, port, token, cert_sha256 FROM cluster_node_direct_links WHERE node_id = $1', ).get(id) return row ? { address: row.address, port: row.port, token: maybeDecrypt(row.token), certSha256: row.cert_sha256 || null } : null } - setDirectLink(id, { address, port, token }, now = new Date().toISOString()) { - this._stmt(` - INSERT INTO cluster_node_direct_links (node_id, address, port, token, updated_at) VALUES (?, ?, ?, ?, ?) + async setDirectLink(id, { address, port, token }, now = new Date().toISOString()) { + await this._stmt(` + INSERT INTO cluster_node_direct_links (node_id, address, port, token, updated_at) VALUES ($1, $2, $3, $4, $5) ON CONFLICT(node_id) DO UPDATE SET address = excluded.address, port = excluded.port, token = excluded.token, updated_at = excluded.updated_at `).run(id, address, port, maybeEncrypt(token), now) } /** The pin follows the node's gate, not the address: an address edit keeps it. */ - setDirectLinkCert(id, certSha256) { - this._stmt('UPDATE cluster_node_direct_links SET cert_sha256 = ? WHERE node_id = ?').run(certSha256, id) + async setDirectLinkCert(id, certSha256) { + await this._stmt('UPDATE cluster_node_direct_links SET cert_sha256 = $1 WHERE node_id = $2').run(certSha256, id) } - clearDirectLink(id) { - return this._stmt('DELETE FROM cluster_node_direct_links WHERE node_id = ?').run(id).changes > 0 + async clearDirectLink(id) { + return (await this._stmt('DELETE FROM cluster_node_direct_links WHERE node_id = $1').run(id)).changes > 0 } } diff --git a/src/lib/db/repos/groups-repo.mjs b/src/lib/db/repos/groups-repo.mjs index 4fc65f4c3..08d1e1f25 100644 --- a/src/lib/db/repos/groups-repo.mjs +++ b/src/lib/db/repos/groups-repo.mjs @@ -32,33 +32,33 @@ export class GroupsRepo { constructor(db = getDb()) { this.db = db this._list = db.prepare('SELECT * FROM groups WHERE deleted_at IS NULL ORDER BY sort_order, id') - this._get = db.prepare('SELECT * FROM groups WHERE id = ? AND deleted_at IS NULL') - this._getByName = db.prepare('SELECT * FROM groups WHERE name = ? AND deleted_at IS NULL') + this._get = db.prepare('SELECT * FROM groups WHERE id = $1 AND deleted_at IS NULL') + this._getByName = db.prepare('SELECT * FROM groups WHERE name = $1 AND deleted_at IS NULL') } - list() { - return this._list.all().map(rowToRec) + async list() { + return (await this._list.all()).map(rowToRec) } - getById(id) { - return rowToRec(this._get.get(id)) + async getById(id) { + return rowToRec(await this._get.get(id)) } - getByName(name) { - return rowToRec(this._getByName.get(String(name || '').trim())) + async getByName(name) { + return rowToRec(await this._getByName.get(String(name || '').trim())) } /** The coding group — where keys/accounts land unless told otherwise. */ - getDefault() { - return this.getById(DEFAULT_GROUP_ID) + async getDefault() { + return await this.getById(DEFAULT_GROUP_ID) } /** Patch editable attributes only (name/platform/id are fixed). */ - update(id, patch = {}) { + async update(id, patch = {}) { const cols = EDITABLE.filter((c) => patch[c] !== undefined) - if (!cols.length) return this.getById(id) - const sql = `UPDATE groups SET ${cols.map((c) => `${c} = ?`).join(', ')}, updated_at = ? WHERE id = ? AND deleted_at IS NULL` - this.db.prepare(sql).run( + if (!cols.length) return await this.getById(id) + const sql = `UPDATE groups SET ${cols.map((c, i) => `${c} = $${i + 1}`).join(', ')}, updated_at = $${cols.length + 1} WHERE id = $${cols.length + 2} AND deleted_at IS NULL` + await this.db.prepare(sql).run( ...cols.map((c) => { if (c === 'is_exclusive') return patch[c] ? 1 : 0 return patch[c] @@ -66,12 +66,12 @@ export class GroupsRepo { new Date().toISOString(), id, ) - return this.getById(id) + return await this.getById(id) } /** Effective billing multiplier for a group id (missing group → 1). */ - rateMultiplier(id) { - const g = this.getById(id ?? DEFAULT_GROUP_ID) + async rateMultiplier(id) { + const g = await this.getById(id ?? DEFAULT_GROUP_ID) return g ? g.rate_multiplier : 1 } } diff --git a/src/lib/db/repos/proxies-repo.mjs b/src/lib/db/repos/proxies-repo.mjs index ea6fdfaaf..1af1bb305 100644 --- a/src/lib/db/repos/proxies-repo.mjs +++ b/src/lib/db/repos/proxies-repo.mjs @@ -64,19 +64,19 @@ export class ProxiesRepo { this._clear = db.prepare('DELETE FROM proxies') this._insert = db.prepare(` INSERT INTO proxies (${COLUMNS.join(', ')}) - VALUES (${COLUMNS.map(() => '?').join(', ')}) + VALUES (${COLUMNS.map((_, i) => '$' + (i + 1)).join(', ')}) `) } - loadAll() { - return this._all.all().map(rowToProxy) + async loadAll() { + return (await this._all.all()).map(rowToProxy) } - replaceAll(proxies = []) { - withTransaction(this.db, () => { - this._clear.run() + async replaceAll(proxies = []) { + await withTransaction(this.db, async () => { + await this._clear.run() for (const p of proxies) { - this._insert.run( + await this._insert.run( ...COLUMNS.map((c) => { if (c === 'enabled') return p.enabled === false ? 0 : 1 if (c === 'domain_forward') return p.domain_forward ? 1 : 0 @@ -89,11 +89,11 @@ export class ProxiesRepo { }) } - getConfig(fallback = null) { - return this.settings.get(CONFIG_KEY, fallback) + async getConfig(fallback = null) { + return await this.settings.get(CONFIG_KEY, fallback) } - setConfig(config) { - return this.settings.set(CONFIG_KEY, config) + async setConfig(config) { + return await this.settings.set(CONFIG_KEY, config) } } diff --git a/src/lib/db/repos/redeem-codes-repo.mjs b/src/lib/db/repos/redeem-codes-repo.mjs index 2daa6890c..17e7a8d76 100644 --- a/src/lib/db/repos/redeem-codes-repo.mjs +++ b/src/lib/db/repos/redeem-codes-repo.mjs @@ -33,58 +33,57 @@ export class RedeemCodesRepo { this.db = db this.users = new UsersRepo(db) this._list = db.prepare('SELECT * FROM redeem_codes ORDER BY id DESC') - this._get = db.prepare('SELECT * FROM redeem_codes WHERE id = ?') - this._getByCode = db.prepare('SELECT * FROM redeem_codes WHERE code = ?') + this._get = db.prepare('SELECT * FROM redeem_codes WHERE id = $1') + this._getByCode = db.prepare('SELECT * FROM redeem_codes WHERE code = $1') this._insert = db.prepare(` INSERT INTO redeem_codes (code, type, value, status, notes, expires_at, created_at) - VALUES (?, 'balance', ?, 'unused', ?, ?, ?) + VALUES ($1, 'balance', $2, 'unused', $3, $4, $5) RETURNING * `) this._markUsed = db.prepare(` - UPDATE redeem_codes SET status = 'used', used_by = ?, used_at = ? - WHERE id = ? AND status = 'unused' + UPDATE redeem_codes SET status = 'used', used_by = $1, used_at = $2 + WHERE id = $3 AND status = 'unused' `) - this._delete = db.prepare("DELETE FROM redeem_codes WHERE id = ? AND status = 'unused'") + this._delete = db.prepare("DELETE FROM redeem_codes WHERE id = $1 AND status = 'unused'") } - list() { - return this._list.all().map(rowToRec) + async list() { + return (await this._list.all()).map(rowToRec) } - getByCode(code) { - return rowToRec(this._getByCode.get(String(code || '').trim())) + async getByCode(code) { + return rowToRec(await this._getByCode.get(String(code || '').trim())) } - create({ value, notes = null, expires_at = null, code = null } = {}) { + async create({ value, notes = null, expires_at = null, code = null } = {}) { const v = Number(value) if (!Number.isFinite(v) || v <= 0) return null const c = code || generateRedeemCode() - const info = this._insert.run(c, v, notes, expires_at, new Date().toISOString()) - return rowToRec(this._get.get(info.lastInsertRowid)) + return rowToRec(await this._insert.get(c, v, notes, expires_at, new Date().toISOString())) } /** Unused codes only; used codes stay as the audit ledger. */ - remove(id) { - return this._delete.run(id).changes > 0 + async remove(id) { + return (await this._delete.run(id)).changes > 0 } /** * Redeem a code for a user: optimistic status flip + balance credit in one * transaction. Returns {ok, code?, error?}. */ - redeem(code, userId) { - const rec = this.getByCode(code) + async redeem(code, userId) { + const rec = await this.getByCode(code) if (!rec) return { ok: false, error: 'code_not_found' } if (rec.status !== 'unused') return { ok: false, error: 'code_used' } if (rec.expires_at && Date.parse(rec.expires_at) < Date.now()) { return { ok: false, error: 'code_expired' } } - const user = this.users.getById(userId) - if (!user) return { ok: false, error: 'user_not_found' } - return withTransaction(this.db, () => { - const info = this._markUsed.run(userId, new Date().toISOString(), rec.id) + return await withTransaction(this.db, async () => { + const user = await this.users.getById(userId, { forUpdate: true }) + if (!user) return { ok: false, error: 'user_not_found' } + const info = await this._markUsed.run(userId, new Date().toISOString(), rec.id) if (!info.changes) return { ok: false, error: 'code_used' } - this.users.addBalance(userId, rec.value) - return { ok: true, code: rowToRec(this._get.get(rec.id)) } + await this.users.addBalance(userId, rec.value) + return { ok: true, code: rowToRec(await this._get.get(rec.id)) } }) } } diff --git a/src/lib/db/repos/refusal-device-blocks-repo.mjs b/src/lib/db/repos/refusal-device-blocks-repo.mjs index 731abe728..305395b97 100644 --- a/src/lib/db/repos/refusal-device-blocks-repo.mjs +++ b/src/lib/db/repos/refusal-device-blocks-repo.mjs @@ -20,12 +20,12 @@ function rowToRec(row) { export class RefusalDeviceBlocksRepo { constructor(db = getDb()) { this.db = db - this._get = db.prepare('SELECT * FROM refusal_device_blocks WHERE device_id = ?') + this._get = db.prepare('SELECT * FROM refusal_device_blocks WHERE device_id = $1') this._insert = db.prepare(` INSERT INTO refusal_device_blocks ( device_id, first_seen_at, last_seen_at, hit_count, source_request_id, fingerprint, reason - ) VALUES (?, ?, ?, 0, ?, ?, ?) + ) VALUES ($1, $2, $3, 0, $4, $5, $6) ON CONFLICT(device_id) DO UPDATE SET last_seen_at = excluded.last_seen_at, source_request_id = COALESCE(refusal_device_blocks.source_request_id, excluded.source_request_id), @@ -33,52 +33,52 @@ export class RefusalDeviceBlocksRepo { reason = COALESCE(refusal_device_blocks.reason, excluded.reason) `) this._hit = db.prepare(` - UPDATE refusal_device_blocks SET hit_count = hit_count + 1, last_seen_at = ? - WHERE device_id = ? + UPDATE refusal_device_blocks SET hit_count = hit_count + 1, last_seen_at = $1 + WHERE device_id = $2 `) - this._list = db.prepare('SELECT * FROM refusal_device_blocks ORDER BY last_seen_at DESC, device_id LIMIT ?') + this._list = db.prepare('SELECT * FROM refusal_device_blocks ORDER BY last_seen_at DESC, device_id LIMIT $1') this._count = db.prepare('SELECT COUNT(*) AS n FROM refusal_device_blocks') - this._remove = db.prepare('DELETE FROM refusal_device_blocks WHERE device_id = ?') + this._remove = db.prepare('DELETE FROM refusal_device_blocks WHERE device_id = $1') this._clear = db.prepare('DELETE FROM refusal_device_blocks') } - get(deviceId) { + async get(deviceId) { const id = String(deviceId || '').trim() if (!id) return null - return rowToRec(this._get.get(id)) + return rowToRec(await this._get.get(id)) } - block({ deviceId, requestId = null, fingerprint = null, reason = 'refusal_guard' } = {}) { + async block({ deviceId, requestId = null, fingerprint = null, reason = 'refusal_guard' } = {}) { const id = String(deviceId || '').trim() if (!id) return null const now = new Date().toISOString() - this._insert.run(id, now, now, requestId, fingerprint, reason ? String(reason).slice(0, 80) : null) - return this.get(id) + await this._insert.run(id, now, now, requestId, fingerprint, reason ? String(reason).slice(0, 80) : null) + return await this.get(id) } - hit(deviceId) { + async hit(deviceId) { const id = String(deviceId || '').trim() if (!id) return null - this._hit.run(new Date().toISOString(), id) - return this.get(id) + await this._hit.run(new Date().toISOString(), id) + return await this.get(id) } - list(limit = 200) { + async list(limit = 200) { const n = Math.min(500, Math.max(1, Number(limit) || 200)) - return this._list.all(n).map(rowToRec) + return (await this._list.all(n)).map(rowToRec) } - count() { - return Number(this._count.get()?.n) || 0 + async count() { + return Number((await this._count.get())?.n) || 0 } - remove(deviceId) { + async remove(deviceId) { const id = String(deviceId || '').trim() if (!id) return false - return this._remove.run(id).changes > 0 + return (await this._remove.run(id)).changes > 0 } - clear() { - return this._clear.run().changes + async clear() { + return (await this._clear.run()).changes } } diff --git a/src/lib/db/repos/refusal-guards-repo.mjs b/src/lib/db/repos/refusal-guards-repo.mjs index 1b617ad78..4706af08b 100644 --- a/src/lib/db/repos/refusal-guards-repo.mjs +++ b/src/lib/db/repos/refusal-guards-repo.mjs @@ -23,12 +23,12 @@ function rowToRec(row) { export class RefusalGuardsRepo { constructor(db = getDb()) { this.db = db - this._get = db.prepare('SELECT * FROM refusal_guards WHERE fingerprint = ?') + this._get = db.prepare('SELECT * FROM refusal_guards WHERE fingerprint = $1') this._insert = db.prepare(` INSERT INTO refusal_guards ( fingerprint, model, first_seen_at, last_seen_at, hit_count, source_request_id, error_message, preview, expires_at, signature - ) VALUES (?, ?, ?, ?, 0, ?, ?, ?, ?, ?) + ) VALUES ($1, $2, $3, $4, 0, $5, $6, $7, $8, $9) ON CONFLICT(fingerprint) DO UPDATE SET last_seen_at = excluded.last_seen_at, error_message = COALESCE(excluded.error_message, refusal_guards.error_message), @@ -41,32 +41,34 @@ export class RefusalGuardsRepo { END `) this._hit = db.prepare(` - UPDATE refusal_guards SET hit_count = hit_count + 1, last_seen_at = ? - WHERE fingerprint = ? + UPDATE refusal_guards SET hit_count = hit_count + 1, last_seen_at = $1 + WHERE fingerprint = $2 `) this._list = db.prepare( `SELECT fingerprint, model, first_seen_at, last_seen_at, hit_count, source_request_id, error_message, preview, expires_at - FROM refusal_guards ORDER BY last_seen_at DESC, fingerprint LIMIT ?`, + FROM refusal_guards ORDER BY last_seen_at DESC, fingerprint LIMIT $1`, ) this._signatures = db.prepare( `SELECT fingerprint, signature FROM refusal_guards - WHERE signature IS NOT NULL AND (expires_at IS NULL OR expires_at > ?)`, + WHERE signature IS NOT NULL AND (expires_at IS NULL OR expires_at > $1)`, ) this._count = db.prepare('SELECT COUNT(*) AS n FROM refusal_guards') - this._remove = db.prepare('DELETE FROM refusal_guards WHERE fingerprint = ?') + this._remove = db.prepare('DELETE FROM refusal_guards WHERE fingerprint = $1') this._clear = db.prepare('DELETE FROM refusal_guards') } - get(fingerprint) { - const rec = rowToRec(this._get.get(String(fingerprint || ''))) + async get(fingerprint) { + const rec = rowToRec(await this._get.get(String(fingerprint || ''))) if (!rec?.expires_at) return rec if (Date.parse(rec.expires_at) > Date.now()) return rec - this.remove(rec.fingerprint) + await this.db + .prepare('DELETE FROM refusal_guards WHERE fingerprint = $1 AND expires_at = $2') + .run(rec.fingerprint, rec.expires_at) return null } - remember({ + async remember({ fingerprint, model = '', requestId = null, @@ -78,7 +80,7 @@ export class RefusalGuardsRepo { const fp = String(fingerprint || '').trim() if (!fp) return null const now = new Date().toISOString() - this._insert.run( + await this._insert.run( fp, String(model || ''), now, @@ -89,51 +91,48 @@ export class RefusalGuardsRepo { expiresAt || null, signature || null, ) - return this.get(fp) + return await this.get(fp) } - hit(fingerprint) { + async hit(fingerprint) { const fp = String(fingerprint || '').trim() if (!fp) return null - this._hit.run(new Date().toISOString(), fp) - return this.get(fp) + await this._hit.run(new Date().toISOString(), fp) + return await this.get(fp) } /** Highest stored user-text signature at or above the threshold. */ - nearest(signature, threshold = REFUSAL_SIMILARITY) { + async nearest(signature, threshold = REFUSAL_SIMILARITY) { const query = Array.isArray(signature) ? signature : decodeSignature(signature) if (!query) return null let best = null - for (const row of this._signatures.all(new Date().toISOString())) { + for (const row of await this._signatures.all(new Date().toISOString())) { const score = signatureSimilarity(query, decodeSignature(row.signature)) if (score < threshold) continue if (!best || score > best.score) best = { fingerprint: row.fingerprint, score } } if (!best) return null - const rec = this.get(best.fingerprint) + const rec = await this.get(best.fingerprint) if (!rec) return null return { ...rec, score: best.score } } - list(limit = 200) { + async list(limit = 200) { const n = Math.min(500, Math.max(1, Number(limit) || 200)) const now = Date.now() - return this._list - .all(n) - .map(rowToRec) - .filter((rec) => !rec.expires_at || Date.parse(rec.expires_at) > now) + return (await this._list.all(n)).map(rowToRec).filter((rec) => !rec.expires_at || Date.parse(rec.expires_at) > now) } - count() { - return Number(this._count.get()?.n) || 0 + async count() { + return Number((await this._count.get())?.n) || 0 } - remove(fingerprint) { + async remove(fingerprint) { const fp = String(fingerprint || '').trim() if (!fp) return false - return this._remove.run(fp).changes > 0 + return (await this._remove.run(fp)).changes > 0 } - clear() { - return this._clear.run().changes + async clear() { + return (await this._clear.run()).changes } } diff --git a/src/lib/db/repos/request-attempts-repo.mjs b/src/lib/db/repos/request-attempts-repo.mjs index 172b95e79..b704af0eb 100644 --- a/src/lib/db/repos/request-attempts-repo.mjs +++ b/src/lib/db/repos/request-attempts-repo.mjs @@ -43,26 +43,26 @@ export class RequestAttemptsRepo { started_at, completed_at, upstream_status, error_scope, action, cooldown_until, downstream_committed, terminal_state, usage_json, wait_ms, ttft_ms, latency_ms - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) `) this._complete = db.prepare(` UPDATE request_attempts SET - completed_at = ?, upstream_status = ?, error_scope = ?, action = ?, - cooldown_until = ?, downstream_committed = ?, terminal_state = ?, - usage_json = ?, ttft_ms = ?, latency_ms = ? - WHERE request_id = ? AND attempt_no = ? + completed_at = $1, upstream_status = $2, error_scope = $3, action = $4, + cooldown_until = $5, downstream_committed = $6, terminal_state = $7, + usage_json = $8, ttft_ms = $9, latency_ms = $10 + WHERE request_id = $11 AND attempt_no = $12 `) this._list = db.prepare(` - SELECT * FROM request_attempts WHERE request_id = ? ORDER BY attempt_no + SELECT * FROM request_attempts WHERE request_id = $1 ORDER BY attempt_no `) this._removeOld = db.prepare(` - DELETE FROM request_attempts WHERE started_at < ? + DELETE FROM request_attempts WHERE started_at < $1 `) } - begin({ requestId, attemptNo, vmId, accountId, model, selectionReason, waitMs = 0 }) { + async begin({ requestId, attemptNo, vmId, accountId, model, selectionReason, waitMs = 0 }) { const startedAt = new Date().toISOString() - this._insert.run( + await this._insert.run( requestId, attemptNo, vmId ?? null, @@ -82,11 +82,11 @@ export class RequestAttemptsRepo { null, null, ) - return this.list(requestId).find((item) => item.attempt_no === attemptNo) || null + return (await this.list(requestId)).find((item) => item.attempt_no === attemptNo) || null } - complete(requestId, attemptNo, result = {}) { - this._complete.run( + async complete(requestId, attemptNo, result = {}) { + await this._complete.run( new Date().toISOString(), result.upstreamStatus ?? null, result.errorScope ?? null, @@ -100,16 +100,16 @@ export class RequestAttemptsRepo { requestId, attemptNo, ) - return this.list(requestId).find((item) => item.attempt_no === attemptNo) || null + return (await this.list(requestId)).find((item) => item.attempt_no === attemptNo) || null } - list(requestId) { - return this._list.all(requestId).map(rowToAttempt) + async list(requestId) { + return (await this._list.all(requestId)).map(rowToAttempt) } - cleanup(retainDays = 7) { + async cleanup(retainDays = 7) { const days = Math.max(1, Number(retainDays) || 7) const cutoff = new Date(Date.now() - days * 86400_000).toISOString() - return this._removeOld.run(cutoff).changes + return (await this._removeOld.run(cutoff)).changes } } diff --git a/src/lib/db/repos/settings-repo.mjs b/src/lib/db/repos/settings-repo.mjs index 078de37b9..674a0776c 100644 --- a/src/lib/db/repos/settings-repo.mjs +++ b/src/lib/db/repos/settings-repo.mjs @@ -8,17 +8,17 @@ import { getDb } from '../database.mjs' export class SettingsRepo { constructor(db = getDb()) { this.db = db - this._get = db.prepare('SELECT value FROM settings WHERE key = ?') + this._get = db.prepare('SELECT value FROM settings WHERE key = $1') this._set = db.prepare(` - INSERT INTO settings (key, value, updated_at) VALUES (?, ?, ?) + INSERT INTO settings (key, value, updated_at) VALUES ($1, $2, $3) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at `) - this._del = db.prepare('DELETE FROM settings WHERE key = ?') + this._del = db.prepare('DELETE FROM settings WHERE key = $1') this._all = db.prepare('SELECT key, value, updated_at FROM settings ORDER BY key') } - get(key, fallback = null) { - const row = this._get.get(key) + async get(key, fallback = null) { + const row = await this._get.get(key) if (!row) return fallback try { return JSON.parse(row.value) @@ -27,28 +27,33 @@ export class SettingsRepo { } } - set(key, value) { - this._set.run(key, JSON.stringify(value), new Date().toISOString()) + async set(key, value) { + await this._set.run(key, JSON.stringify(value), new Date().toISOString()) + this.db.afterCommit(() => this.db.settingsState?.set(key, value)) return value } - has(key) { - return !!this._get.get(key) + async has(key) { + return !!(await this._get.get(key)) } - remove(key) { - this._del.run(key) + async remove(key) { + await this._del.run(key) + this.db.afterCommit(() => this.db.settingsState?.delete(key)) } - all() { + async all() { const out = {} - for (const row of this._all.all()) { + for (const row of await this._all.all()) { try { out[row.key] = JSON.parse(row.value) } catch { out[row.key] = null } } + this.db.afterCommit(() => { + this.db.settingsState = new Map(Object.entries(out)) + }) return out } } diff --git a/src/lib/db/repos/statistics-repo.mjs b/src/lib/db/repos/statistics-repo.mjs index 707e9127d..44ac9d1dd 100644 --- a/src/lib/db/repos/statistics-repo.mjs +++ b/src/lib/db/repos/statistics-repo.mjs @@ -11,8 +11,16 @@ */ import { getDb } from '../database.mjs' +import { addInteger, integerValue } from '../integers.mjs' import { reportTimezone, zonedDayStartMs, zonedParts, zonedWallToMs } from '../../core/timezone.mjs' -import { ERROR_PRED, PROMPT_TOKENS_SQL, SUCCESS_PRED, TOTAL_TOKENS_SQL, ownerPred } from './usage-log-preds.mjs' +import { + ERROR_PRED, + PROMPT_TOKENS_SQL, + SUCCESS_PRED, + TOTAL_TOKENS_SQL, + ownerPred, + totalTokens, +} from './usage-log-preds.mjs' import { lookupNames } from './usage-logs-view.mjs' export const STATS_RANGES = ['today', '7days', '30days', 'thisMonth'] @@ -34,7 +42,8 @@ const DIMENSION_COLUMN = { user: 'user_id', key: 'api_key_id', model: 'model', v const TOKENS_SQL = TOTAL_TOKENS_SQL /** created_at (`YYYY-MM-DDTHH:MM:SS.sssZ`) floored to its 15-minute UTC slot. */ -const SLOT_SQL = "substr(created_at, 1, 14) || printf('%02d', (CAST(substr(created_at, 15, 2) AS INTEGER) / 15) * 15)" +const SLOT_SQL = + "to_char(date_bin(interval '15 minutes', created_at::timestamptz, timestamptz '2000-01-01 00:00:00+00') AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI')" function num(v) { const n = Number(v) @@ -104,9 +113,9 @@ export class StatisticsRepo { } _scope(sinceMs, untilMs, ownerUserId) { - const where = ['created_at >= ?', 'created_at < ?'] + const where = ['created_at >= $1', 'created_at < $2'] const params = [new Date(sinceMs).toISOString(), new Date(untilMs).toISOString()] - const own = ownerPred(ownerUserId) + const own = ownerPred(ownerUserId, params.length) if (own.sql) { where.push(own.sql) params.push(...own.params) @@ -114,9 +123,9 @@ export class StatisticsRepo { return { cond: `WHERE ${where.join(' AND ')}`, params } } - _totals(sinceMs, untilMs, ownerUserId) { + async _totals(sinceMs, untilMs, ownerUserId) { const { cond, params } = this._scope(sinceMs, untilMs, ownerUserId) - const row = this.db + const row = await this.db .prepare(` SELECT COUNT(*) AS requests, COALESCE(SUM(CASE WHEN ${ERROR_PRED} THEN 1 ELSE 0 END), 0) AS errors, @@ -132,17 +141,17 @@ export class StatisticsRepo { FROM usage_logs ${cond} `) .get(...params) - const input = num(row.input_tokens) - const output = num(row.output_tokens) - const cacheRead = num(row.cache_read_tokens) - const cacheCreation = num(row.cache_creation_tokens) - const prompt = num(row.prompt_tokens) + const input = integerValue(row.input_tokens) + const output = integerValue(row.output_tokens) + const cacheRead = integerValue(row.cache_read_tokens) + const cacheCreation = integerValue(row.cache_creation_tokens) + const prompt = integerValue(row.prompt_tokens) return { - requests: num(row.requests), - errors: num(row.errors), + requests: integerValue(row.requests), + errors: integerValue(row.errors), cost: num(row.cost), actualCost: num(row.actual_cost), - tokens: input + output + cacheRead + cacheCreation, + tokens: totalTokens(row), inputTokens: input, outputTokens: output, cacheReadTokens: cacheRead, @@ -153,10 +162,10 @@ export class StatisticsRepo { } } - _names(dimension, ids) { + async _names(dimension, ids) { const real = ids.filter((id) => id !== NONE_ID) if (dimension === 'model') return new Map(real.map((id) => [id, id])) - const names = lookupNames(this.db, { + const names = await lookupNames(this.db, { userIds: dimension === 'user' ? real : [], keyIds: dimension === 'key' ? real : [], vmIds: dimension === 'vm' ? real : [], @@ -169,14 +178,14 @@ export class StatisticsRepo { } /** Stacked-area series (top 8 by cost + others), window totals, and deltas vs the previous equal window. */ - statistics({ range = 'today', dimension = 'model', tz = 'UTC', owner_user_id = null, now = Date.now() } = {}) { + async statistics({ range = 'today', dimension = 'model', tz = 'UTC', owner_user_id = null, now = Date.now() } = {}) { const r = normalizeRange(range) || 'today' const dim = normalizeDimension(dimension) || 'model' const zone = reportTimezone(tz) const { since, until, resolution, buckets } = resolveWindow(r, zone, now) const col = DIMENSION_COLUMN[dim] const { cond, params } = this._scope(since, until, owner_user_id) - const rows = this.db + const rows = await this.db .prepare(` SELECT ${col} AS sid, ${SLOT_SQL} AS slot, COUNT(*) AS requests, @@ -193,15 +202,16 @@ export class StatisticsRepo { row.id = id const cur = volume.get(id) || { id, cost: 0, requests: 0, tokens: 0 } cur.cost += num(row.cost) - cur.requests += num(row.requests) - cur.tokens += num(row.tokens) + cur.requests = addInteger(cur.requests, row.requests) + cur.tokens = addInteger(cur.tokens, row.tokens) volume.set(id, cur) } - const ranked = [...volume.values()].sort( - (a, b) => b.cost - a.cost || b.requests - a.requests || a.id.localeCompare(b.id), - ) + const ranked = [...volume.values()].sort((a, b) => { + const requests = BigInt(b.requests) - BigInt(a.requests) + return b.cost - a.cost || (requests > 0n ? 1 : requests < 0n ? -1 : 0) || a.id.localeCompare(b.id) + }) const kept = ranked.slice(0, MAX_SERIES) - const names = this._names( + const names = await this._names( dim, kept.map((s) => s.id), ) @@ -226,14 +236,18 @@ export class StatisticsRepo { if (!dataKey) continue const idx = bucketIndex(buckets, Date.parse(`${row.slot}:00.000Z`)) for (const m of metricNames) { - const v = num(row[m]) - metrics[m][idx][dataKey] += v - seriesTotals[dataKey][m] += v + if (m === 'cost') { + metrics[m][idx][dataKey] += num(row[m]) + seriesTotals[dataKey][m] += num(row[m]) + } else { + metrics[m][idx][dataKey] = addInteger(metrics[m][idx][dataKey], row[m]) + seriesTotals[dataKey][m] = addInteger(seriesTotals[dataKey][m], row[m]) + } } } - const totals = this._totals(since, until, owner_user_id) - const prev = this._totals(since - (until - since), since, owner_user_id) + const totals = await this._totals(since, until, owner_user_id) + const prev = await this._totals(since - (until - since), since, owner_user_id) return { range: r, resolution, @@ -254,7 +268,7 @@ export class StatisticsRepo { } /** Per-entity totals over the same window, by official cost. */ - leaderboard({ + async leaderboard({ range = 'today', scope = 'model', limit = LEADERBOARD_DEFAULT, @@ -269,9 +283,9 @@ export class StatisticsRepo { const { since, until } = resolveWindow(r, zone, now) const col = DIMENSION_COLUMN[s] const { cond, params } = this._scope(since, until, owner_user_id) - const rows = this.db + const rows = await this.db .prepare(` - SELECT ${col} AS sid, + SELECT COALESCE(${col}, '') AS sid, COUNT(*) AS requests, COALESCE(SUM(CASE WHEN ${SUCCESS_PRED} THEN 1 ELSE 0 END), 0) AS success, COALESCE(SUM(CASE WHEN ${ERROR_PRED} THEN 1 ELSE 0 END), 0) AS errors, @@ -285,32 +299,32 @@ export class StatisticsRepo { AVG(duration_ms) AS avg_duration_ms, AVG(first_token_ms) AS avg_ttft_ms FROM usage_logs ${cond} - GROUP BY IFNULL(${col}, '') + GROUP BY COALESCE(${col}, '') ORDER BY cost DESC, requests DESC - LIMIT ? + LIMIT $${params.length + 1} `) .all(...params, n) const ids = rows.map((row) => (row.sid == null || row.sid === '' ? NONE_ID : String(row.sid))) - const names = this._names(s, ids) + const names = await this._names(s, ids) return { scope: s, range: r, tz: zone, entries: rows.map((row, i) => { - const input = num(row.input_tokens) - const output = num(row.output_tokens) - const cacheRead = num(row.cache_read_tokens) - const cacheCreation = num(row.cache_creation_tokens) - const requests = num(row.requests) - const prompt = num(row.prompt_tokens) + const input = integerValue(row.input_tokens) + const output = integerValue(row.output_tokens) + const cacheRead = integerValue(row.cache_read_tokens) + const cacheCreation = integerValue(row.cache_creation_tokens) + const requests = integerValue(row.requests) + const prompt = integerValue(row.prompt_tokens) return { id: ids[i], name: this._nameOf(names, ids[i]), totalRequests: requests, - successRequests: num(row.success), - errorRequests: num(row.errors), - successRate: requests > 0 ? num(row.success) / requests : 0, - totalTokens: input + output + cacheRead + cacheCreation, + successRequests: integerValue(row.success), + errorRequests: integerValue(row.errors), + successRate: requests > 0 ? integerValue(row.success) / requests : 0, + totalTokens: totalTokens(row), inputTokens: input, outputTokens: output, cacheReadTokens: cacheRead, diff --git a/src/lib/db/repos/sticky-repo.mjs b/src/lib/db/repos/sticky-repo.mjs index 17def6230..032543557 100644 --- a/src/lib/db/repos/sticky-repo.mjs +++ b/src/lib/db/repos/sticky-repo.mjs @@ -22,10 +22,10 @@ function rowToEntry(row) { export class StickyRepo { constructor(db = getDb()) { this.db = db - this._get = db.prepare('SELECT * FROM sticky_sessions WHERE key = ?') + this._get = db.prepare('SELECT * FROM sticky_sessions WHERE key = $1') this._upsert = db.prepare(` INSERT INTO sticky_sessions (key, account_id, vm_id, session_id, device_id, bound_at, expires_at, hits, generation, slot_index) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT(key) DO UPDATE SET account_id = excluded.account_id, vm_id = excluded.vm_id, @@ -37,28 +37,28 @@ export class StickyRepo { generation = excluded.generation, slot_index = excluded.slot_index `) - this._delete = db.prepare('DELETE FROM sticky_sessions WHERE key = ?') - this._deleteByAccount = db.prepare('DELETE FROM sticky_sessions WHERE account_id = ? OR vm_id = ?') - this._purge = db.prepare('DELETE FROM sticky_sessions WHERE expires_at < ?') + this._delete = db.prepare('DELETE FROM sticky_sessions WHERE key = $1') + this._deleteByAccount = db.prepare('DELETE FROM sticky_sessions WHERE account_id = $1 OR vm_id = $2') + this._purge = db.prepare('DELETE FROM sticky_sessions WHERE expires_at < $1') this._dropAlias = db.prepare(` DELETE FROM sticky_sessions WHERE key = 'envelope' - OR key LIKE 'ch:%' - OR key LIKE 'dev:%' - OR key LIKE '%:envelope' - OR key LIKE '%:ch:%' - OR key LIKE '%:dev:%' + OR ascii_fold(key) LIKE 'ch:%' + OR ascii_fold(key) LIKE 'dev:%' + OR ascii_fold(key) LIKE '%:envelope' + OR ascii_fold(key) LIKE '%:ch:%' + OR ascii_fold(key) LIKE '%:dev:%' `) this._all = db.prepare('SELECT * FROM sticky_sessions') this._count = db.prepare('SELECT COUNT(*) c FROM sticky_sessions') } - get(key) { - return rowToEntry(this._get.get(key)) + async get(key) { + return rowToEntry(await this._get.get(key)) } - upsert(key, ent) { - this._upsert.run( + async upsert(key, ent) { + await this._upsert.run( key, ent.account_id ?? null, ent.vm_id ?? null, @@ -72,31 +72,31 @@ export class StickyRepo { ) } - remove(key) { - this._delete.run(key) + async remove(key) { + await this._delete.run(key) } - removeByAccount({ accountId = null, vmId = null } = {}) { + async removeByAccount({ accountId = null, vmId = null } = {}) { if (!accountId && !vmId) return 0 - return this._deleteByAccount.run(accountId || '', vmId || '').changes + return (await this._deleteByAccount.run(accountId || '', vmId || '')).changes } - purgeExpired(now = Date.now()) { - return this._purge.run(now).changes + async purgeExpired(now = Date.now()) { + return (await this._purge.run(now)).changes } /** Content fingerprints and device aliases are not conversation slots. */ - dropAliasKeys() { - return this._dropAlias.run().changes + async dropAliasKeys() { + return (await this._dropAlias.run()).changes } - all() { + async all() { const out = {} - for (const row of this._all.all()) out[row.key] = rowToEntry(row) + for (const row of await this._all.all()) out[row.key] = rowToEntry(row) return out } - count() { - return this._count.get().c + async count() { + return (await this._count.get()).c } } diff --git a/src/lib/db/repos/usage-log-preds.mjs b/src/lib/db/repos/usage-log-preds.mjs index 7b20ef315..28bc49b1e 100644 --- a/src/lib/db/repos/usage-log-preds.mjs +++ b/src/lib/db/repos/usage-log-preds.mjs @@ -5,6 +5,7 @@ */ import { ignoredErrorSqlList, slaOkErrorSqlList } from '../../admin/error-class.mjs' +import { addInteger } from '../integers.mjs' const IGNORED_CODES_SQL = ignoredErrorSqlList() const SLA_OK_CODES_SQL = slaOkErrorSqlList() @@ -18,25 +19,23 @@ export const SLA_ERROR_PRED = `(${ERROR_PRED} AND NOT (${SLA_OK_PRED}))` // After migration 037 every row stores disjoint input. Prompt (cache hit // denominator) is the input-side total; total tokens also include output. export const PROMPT_TOKENS_SQL = - 'IFNULL(input_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0)' + 'COALESCE(input_tokens, 0) + COALESCE(cache_read_tokens, 0) + COALESCE(cache_creation_tokens, 0)' export const TOTAL_TOKENS_SQL = - 'IFNULL(input_tokens, 0) + IFNULL(output_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0)' + 'COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0) + COALESCE(cache_read_tokens, 0) + COALESCE(cache_creation_tokens, 0)' export function totalTokens(row = {}) { - return ( - (Number(row.input_tokens) || 0) + - (Number(row.output_tokens) || 0) + - (Number(row.cache_read_tokens) || 0) + - (Number(row.cache_creation_tokens) || 0) + return [row.input_tokens, row.output_tokens, row.cache_read_tokens, row.cache_creation_tokens].reduce( + (sum, value) => addInteger(sum, value), + 0, ) } /** Rows a panel `user` owns: own user_id, one of their live keys, or served by a slot they owned at the time. */ -export function ownerPred(ownerUserId) { +export function ownerPred(ownerUserId, offset = 0) { const id = String(ownerUserId || '').trim() if (!id) return { sql: '', params: [] } return { - sql: `(user_id = ? OR IFNULL(api_key_id, '') IN (SELECT id FROM api_keys WHERE user_id = ? AND deleted_at IS NULL) OR vm_owner_user_id = ?)`, + sql: `(user_id = $${offset + 1} OR COALESCE(api_key_id, '') IN (SELECT id FROM api_keys WHERE user_id = $${offset + 2} AND deleted_at IS NULL) OR vm_owner_user_id = $${offset + 3})`, params: [id, id, id], } } diff --git a/src/lib/db/repos/usage-logs-repo.mjs b/src/lib/db/repos/usage-logs-repo.mjs index d5b4c5a74..381ce3d60 100644 --- a/src/lib/db/repos/usage-logs-repo.mjs +++ b/src/lib/db/repos/usage-logs-repo.mjs @@ -8,7 +8,8 @@ * may pass either name. */ -import { getDb } from '../database.mjs' +import { getDb, withTransaction } from '../database.mjs' +import { addInteger, integerValue } from '../integers.mjs' import { collectErrors, enrichLogRow, excludeErrorClassSql, ingressAuthSql } from '../../admin/error-class.mjs' import { costColumnsFromUsage, @@ -56,10 +57,24 @@ function billingStatsCacheKey(accountWindows) { // Do not re-apply a single Anthropic formula on the mixed SUM (homepage/usage // billing). Database-page PromptCacheMetricsPanel still uses input+read+write. function sumCostBuckets(rows) { - return { - ...sumCostColumns(rows), - prompt_tokens: rows.reduce((sum, row) => sum + (Number(row.prompt_tokens) || 0), 0), + const out = sumCostColumns(rows) + for (const key of [ + 'requests', + 'success', + 'errors', + 'input_tokens', + 'output_tokens', + 'cache_read_tokens', + 'cache_creation_tokens', + 'prompt_tokens', + ]) { + out[key] = rows.reduce((sum, row) => addInteger(sum, row[key]), 0) } + return out +} + +function withCacheHitRate(bucket) { + return { ...bucket, cache_hit_rate: cacheHitStats(bucket).cache_hit_rate } } const SUMMARY_COLUMNS = [ @@ -140,7 +155,7 @@ function toRow(rec) { c === 'cache_creation_1h_tokens' || c === 'web_search_requests' ) - return Number(rec[c]) || 0 + return integerValue(rec[c]) return rec[c] ?? null }) } @@ -160,11 +175,11 @@ function timeCond(since, until) { const where = [] const params = [] if (since) { - where.push('created_at >= ?') + where.push(`created_at >= $${params.length + 1}`) params.push(new Date(since).toISOString()) } if (until) { - where.push('created_at <= ?') + where.push(`created_at <= $${params.length + 1}`) params.push(new Date(until).toISOString()) } return { cond: where.length ? `WHERE ${where.join(' AND ')}` : '', params } @@ -216,13 +231,13 @@ function bucketEventsSince(rows = [], sinceMs) { ...sumCostBuckets( picked.map((row) => ({ requests: 1, - success: Number(row.success || 0), - errors: Number(row.errors || 0), - input_tokens: Number(row.input_tokens || 0), - output_tokens: Number(row.output_tokens || 0), - cache_read_tokens: Number(row.cache_read_tokens || 0), - cache_creation_tokens: Number(row.cache_creation_tokens || 0), - prompt_tokens: Number(row.prompt_tokens || 0), + success: integerValue(row.success), + errors: integerValue(row.errors), + input_tokens: integerValue(row.input_tokens), + output_tokens: integerValue(row.output_tokens), + cache_read_tokens: integerValue(row.cache_read_tokens), + cache_creation_tokens: integerValue(row.cache_creation_tokens), + prompt_tokens: integerValue(row.prompt_tokens), input_cost: Number(row.input_cost || 0), output_cost: Number(row.output_cost || 0), cache_read_cost: Number(row.cache_read_cost || 0), @@ -238,15 +253,15 @@ function filterCond({ since = null, until = null, vmId = null, accountId = null, const { cond, params } = timeCond(since, until) const parts = cond ? [cond.replace(/^WHERE /, '')] : [] if (vmId) { - parts.push('vm_id = ?') + parts.push(`vm_id = $${params.length + 1}`) params.push(vmId) } if (apiKeyId) { - parts.push('api_key_id = ?') + parts.push(`api_key_id = $${params.length + 1}`) params.push(apiKeyId) } if (accountId) { - parts.push('(account_id = ? OR final_account_id = ?)') + parts.push(`(account_id = $${params.length + 1} OR final_account_id = $${params.length + 2})`) params.push(accountId, accountId) } return { cond: parts.length ? `WHERE ${parts.join(' AND ')}` : '', params } @@ -288,86 +303,91 @@ export class UsageLogsRepo { this._billingBackfillAt = 0 this._insert = db.prepare(` INSERT INTO usage_logs (${SUMMARY_COLUMNS.join(', ')}) - VALUES (${SUMMARY_COLUMNS.map(() => '?').join(', ')}) + VALUES (${SUMMARY_COLUMNS.map((_, i) => '$' + (i + 1)).join(', ')}) `) this._insertIfAbsent = db.prepare(` - INSERT OR IGNORE INTO usage_logs (${SUMMARY_COLUMNS.join(', ')}) - VALUES (${SUMMARY_COLUMNS.map(() => '?').join(', ')}) + INSERT INTO usage_logs (${SUMMARY_COLUMNS.join(', ')}) + VALUES (${SUMMARY_COLUMNS.map((_, i) => '$' + (i + 1)).join(', ')}) + ON CONFLICT(request_id) DO NOTHING `) this._insertDebug = db.prepare(` - INSERT INTO request_log_debug (request_id, ts, record_json) VALUES (?, ?, ?) + INSERT INTO request_log_debug (request_id, ts, record_json) VALUES ($1, $2, $3) ON CONFLICT(request_id) DO UPDATE SET ts = excluded.ts, record_json = excluded.record_json `) this._insertDebugIfAbsent = db.prepare(` - INSERT OR IGNORE INTO request_log_debug (request_id, ts, record_json) VALUES (?, ?, ?) + INSERT INTO request_log_debug (request_id, ts, record_json) VALUES ($1, $2, $3) + ON CONFLICT(request_id) DO NOTHING `) - this._getDebug = db.prepare('SELECT record_json FROM request_log_debug WHERE request_id = ?') - this._vmOwner = db.prepare('SELECT owner_user_id FROM vms WHERE id = ?') - this._listDebug = db.prepare('SELECT record_json FROM request_log_debug ORDER BY ts DESC LIMIT ?') - this._getByRequestId = db.prepare('SELECT * FROM usage_logs WHERE request_id = ?') - // OCTET_LENGTH counts encoded bytes and lets SQLite read TEXT size from metadata without loading each body. + this._getDebug = db.prepare('SELECT record_json FROM request_log_debug WHERE request_id = $1') + this._vmOwner = db.prepare('SELECT owner_user_id FROM vms WHERE id = $1') + this._listDebug = db.prepare('SELECT record_json FROM request_log_debug ORDER BY ts DESC LIMIT $1') + this._getByRequestId = db.prepare('SELECT * FROM usage_logs WHERE request_id = $1') + // OCTET_LENGTH counts encoded bytes rather than characters. this._debugBytes = db.prepare('SELECT COALESCE(SUM(OCTET_LENGTH(record_json)), 0) AS n FROM request_log_debug') this._debugCount = db.prepare('SELECT COUNT(*) AS n FROM request_log_debug') - this._deleteOldUsage = db.prepare('DELETE FROM usage_logs WHERE created_at < ?') - this._deleteOldDebug = db.prepare('DELETE FROM request_log_debug WHERE ts < ?') + this._deleteOldUsage = db.prepare('DELETE FROM usage_logs WHERE created_at < $1') + this._deleteOldDebug = db.prepare('DELETE FROM request_log_debug WHERE ts < $1') this._deleteOldestDebug = db.prepare(` DELETE FROM request_log_debug WHERE request_id IN ( - SELECT request_id FROM request_log_debug ORDER BY ts ASC LIMIT ? + SELECT request_id FROM request_log_debug ORDER BY ts ASC LIMIT $1 ) `) } /** Cheap EXISTS: ≥1 2xx row visible to the caller (same ownerPred as request-log stats). */ - hasSuccessfulRequest({ owner_user_id = null } = {}) { + async hasSuccessfulRequest({ owner_user_id = null } = {}) { const own = ownerPred(owner_user_id) const where = own.sql ? `AND ${own.sql}` : '' - const row = this.db + const row = await this.db .prepare(`SELECT 1 AS ok FROM usage_logs WHERE status >= 200 AND status < 300 ${where} LIMIT 1`) .get(...own.params) return !!row } - belongsToOwner(requestId, ownerUserId) { + async belongsToOwner(requestId, ownerUserId) { if (!requestId) return false if (!ownerUserId) return true - const own = ownerPred(ownerUserId) - const row = this.db - .prepare(`SELECT 1 AS ok FROM usage_logs WHERE request_id = ? AND ${own.sql} LIMIT 1`) + const own = ownerPred(ownerUserId, 1) + const row = await this.db + .prepare(`SELECT 1 AS ok FROM usage_logs WHERE request_id = $1 AND ${own.sql} LIMIT 1`) .get(requestId, ...own.params) return !!row } - getByRequestId(requestId) { + async getByRequestId(requestId) { if (!requestId) return null - const row = this._getByRequestId.get(requestId) + const row = await this._getByRequestId.get(requestId) return row ? enrichLogRow(fromRow(row)) : null } - insertSummary(rec) { - this._insert.run(...toRow(rec)) + async insertSummary(rec) { + await this._insert.run(...toRow(rec)) return rec } - vmOwnerOf(vmId) { + async vmOwnerOf(vmId) { if (!vmId) return null - return this._vmOwner.get(String(vmId))?.owner_user_id || null + return (await this._vmOwner.get(String(vmId)))?.owner_user_id || null } - /** INSERT OR IGNORE (legacy import path). @returns true when inserted */ - insertSummaryIfAbsent(rec) { - return this._insertIfAbsent.run(...toRow(rec)).changes > 0 + /** Idempotent insert for legacy imports. @returns true when inserted */ + async insertSummaryIfAbsent(rec) { + return (await this._insertIfAbsent.run(...toRow(rec))).changes > 0 } - insertDebug(requestId, ts, record) { - this._insertDebug.run(requestId, ts || new Date().toISOString(), JSON.stringify(record)) + async insertDebug(requestId, ts, record) { + await this._insertDebug.run(requestId, ts || new Date().toISOString(), JSON.stringify(record)) } - insertDebugIfAbsent(requestId, ts, record) { - return this._insertDebugIfAbsent.run(requestId, ts || new Date().toISOString(), JSON.stringify(record)).changes > 0 + async insertDebugIfAbsent(requestId, ts, record) { + return ( + (await this._insertDebugIfAbsent.run(requestId, ts || new Date().toISOString(), JSON.stringify(record))).changes > + 0 + ) } - getDebug(requestId) { - const row = this._getDebug.get(requestId) + async getDebug(requestId) { + const row = await this._getDebug.get(requestId) if (!row) return null try { return enrichLogRow(JSON.parse(row.record_json)) @@ -376,12 +396,11 @@ export class UsageLogsRepo { } } - listDebug({ limit = 20, owner_user_id = null } = {}) { + async listDebug({ limit = 20, owner_user_id = null } = {}) { const n = Math.max(1, Math.min(100, Number(limit) || 20)) const own = ownerPred(owner_user_id) if (!own.sql) { - return this._listDebug - .all(n) + return (await this._listDebug.all(n)) .map((r) => { try { return enrichLogRow(JSON.parse(r.record_json)) @@ -391,16 +410,18 @@ export class UsageLogsRepo { }) .filter(Boolean) } - return this.db - .prepare(` + return ( + await this.db + .prepare(` SELECT d.record_json FROM request_log_debug d JOIN usage_logs u ON u.request_id = d.request_id WHERE ${own.sql} ORDER BY d.ts DESC - LIMIT ? + LIMIT $${own.params.length + 1} `) - .all(...own.params, n) + .all(...own.params, n) + ) .map((r) => { try { return enrichLogRow(JSON.parse(r.record_json)) @@ -415,7 +436,7 @@ export class UsageLogsRepo { * Filtered, paginated query over summaries (newest first). * @returns {{ items: object[], total: number }} */ - query({ + async query({ limit = 50, offset = 0, api_key_id = null, @@ -440,23 +461,23 @@ export class UsageLogsRepo { params.push(...own.params) } if (api_key_id) { - where.push('api_key_id = ?') + where.push(`api_key_id = $${params.length + 1}`) params.push(api_key_id) } if (vm_id) { - where.push('vm_id = ?') + where.push(`vm_id = $${params.length + 1}`) params.push(vm_id) } if (account_id) { - where.push('account_id = ?') + where.push(`account_id = $${params.length + 1}`) params.push(account_id) } if (model) { - where.push('model = ?') + where.push(`model = $${params.length + 1}`) params.push(model) } if (protocol) { - where.push('protocol = ?') + where.push(`protocol = $${params.length + 1}`) params.push(protocol) } if (error_class && (status == null || status === '')) status = 'error' @@ -464,7 +485,7 @@ export class UsageLogsRepo { if (String(status) === 'error') where.push(ERROR_PRED) else if (String(status) === 'ok') where.push('(status < 400 AND status IS NOT NULL)') else { - where.push('status = ?') + where.push(`status = $${params.length + 1}`) params.push(Number(status)) } } @@ -473,15 +494,17 @@ export class UsageLogsRepo { if (excludeSql) where.push(excludeSql) } if (since) { - where.push('created_at >= ?') + where.push(`created_at >= $${params.length + 1}`) params.push(new Date(since).toISOString()) } if (until) { - where.push('created_at <= ?') + where.push(`created_at <= $${params.length + 1}`) params.push(new Date(until).toISOString()) } if (q) { - where.push('(path LIKE ? OR error_code LIKE ? OR error_message LIKE ? OR request_id LIKE ?)') + where.push( + `(ascii_fold(path) LIKE ascii_fold($${params.length + 1}) ESCAPE '' OR ascii_fold(error_code) LIKE ascii_fold($${params.length + 2}) ESCAPE '' OR ascii_fold(error_message) LIKE ascii_fold($${params.length + 3}) ESCAPE '' OR ascii_fold(request_id) LIKE ascii_fold($${params.length + 4}) ESCAPE '')`, + ) const like = `%${String(q).slice(0, 100)}%` params.push(like, like, like, like) } @@ -491,18 +514,24 @@ export class UsageLogsRepo { const off = Math.max(0, Number(offset) || 0) if (error_class) { const scan = Math.min(5000, Math.max(n + off, 2000)) - const rows = this.db - .prepare(`SELECT * FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT ?`) - .all(...params, scan) + const rows = ( + await this.db + .prepare(`SELECT * FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT $${params.length + 1}`) + .all(...params, scan) + ) .map(fromRow) .map(enrichLogRow) .filter((row) => row.error_class === error_class) return { items: rows.slice(off, off + n), total: rows.length } } - const total = this.db.prepare(`SELECT COUNT(*) c FROM usage_logs ${cond}`).get(...params).c - const items = this.db - .prepare(`SELECT * FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?`) - .all(...params, n, off) + const total = (await this.db.prepare(`SELECT COUNT(*) c FROM usage_logs ${cond}`).get(...params)).c + const items = ( + await this.db + .prepare( + `SELECT * FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT $${params.length + 1} OFFSET $${params.length + 2}`, + ) + .all(...params, n, off) + ) .map(fromRow) .map(enrichLogRow) return { items, total } @@ -512,19 +541,19 @@ export class UsageLogsRepo { * Aggregate stats bucketed by day or hour (dashboard/usage charts). * @returns {Array<{bucket, requests, errors, input_tokens, output_tokens, avg_duration_ms}>} */ - aggregate({ since = null, until = null, bucket = 'day', owner_user_id = null } = {}) { - const fmt = bucket === 'hour' ? '%Y-%m-%dT%H:00' : '%Y-%m-%d' + async aggregate({ since = null, until = null, bucket = 'day', owner_user_id = null } = {}) { + const fmt = bucket === 'hour' ? 'YYYY-MM-DD\"T\"HH24:00' : 'YYYY-MM-DD' const { cond, params } = timeCond(since, until) - const own = ownerPred(owner_user_id) + const own = ownerPred(owner_user_id, params.length) let where = cond const allParams = [...params] if (own.sql) { where = where ? `${where} AND ${own.sql}` : `WHERE ${own.sql}` allParams.push(...own.params) } - return this.db + return await this.db .prepare(` - SELECT strftime('${fmt}', created_at) AS bucket, + SELECT to_char(created_at::timestamptz AT TIME ZONE 'UTC', '${fmt}') AS bucket, COUNT(*) AS requests, SUM(CASE WHEN ${ERROR_PRED} THEN 1 ELSE 0 END) AS errors, COALESCE(SUM(input_tokens), 0) AS input_tokens, @@ -533,8 +562,8 @@ export class UsageLogsRepo { COALESCE(SUM(cache_creation_tokens), 0) AS cache_creation_tokens, COALESCE(SUM(${PROMPT_TOKENS_SQL}), 0) AS prompt_tokens, COALESCE(SUM(total_cost), 0) AS total_cost, - CAST(AVG(duration_ms) AS INTEGER) AS avg_duration_ms, - CAST(AVG(first_token_ms) AS INTEGER) AS avg_first_token_ms + TRUNC(AVG(duration_ms))::bigint AS avg_duration_ms, + TRUNC(AVG(first_token_ms))::bigint AS avg_first_token_ms FROM usage_logs ${where} GROUP BY bucket ORDER BY bucket `) @@ -542,8 +571,8 @@ export class UsageLogsRepo { } /** Grand totals (dashboard db_totals). */ - totals() { - const row = this.db + async totals() { + const row = await this.db .prepare(` SELECT COUNT(*) AS requests, SUM(CASE WHEN ${ERROR_PRED} THEN 1 ELSE 0 END) AS errors, @@ -560,7 +589,7 @@ export class UsageLogsRepo { FROM usage_logs `) .get() - return { ...row, ...cacheHitStats(row) } + return withCacheHitRate(row) } _usageForPrice(row) { @@ -577,29 +606,28 @@ export class UsageLogsRepo { } } - _repriceRows(rows, { writeActualCost = false } = {}) { + async _repriceRows(rows, { writeActualCost = false } = {}) { if (!rows.length) return { rows: 0, changed: 0, still_unpriced: 0, delta_usd: 0 } const upd = writeActualCost ? this.db.prepare(` UPDATE usage_logs - SET input_cost = ?, output_cost = ?, cache_read_cost = ?, - cache_creation_cost = ?, web_search_cost = ?, total_cost = ?, - actual_cost = ?, pricing_model = ?, price_source = ?, price_version = ?, - long_context = ? - WHERE id = ? + SET input_cost = $1, output_cost = $2, cache_read_cost = $3, + cache_creation_cost = $4, web_search_cost = $5, total_cost = $6, + actual_cost = $7, pricing_model = $8, price_source = $9, price_version = $10, + long_context = $11 + WHERE id = $12 `) : this.db.prepare(` UPDATE usage_logs - SET input_cost = ?, output_cost = ?, cache_read_cost = ?, - cache_creation_cost = ?, web_search_cost = ?, total_cost = ?, - pricing_model = ?, price_source = ?, price_version = ?, long_context = ? - WHERE id = ? + SET input_cost = $1, output_cost = $2, cache_read_cost = $3, + cache_creation_cost = $4, web_search_cost = $5, total_cost = $6, + pricing_model = $7, price_source = $8, price_version = $9, long_context = $10 + WHERE id = $11 `) let changed = 0 let still_unpriced = 0 let delta_usd = 0 - this.db.exec('BEGIN') - try { + await withTransaction(this.db, async () => { for (const r of rows) { const model = r.upstream_model || r.model || r.requested_model const priced = costColumnsFromUsage(this._usageForPrice(r), model) @@ -624,7 +652,7 @@ export class UsageLogsRepo { const parsedRate = r.rate_multiplier == null ? 1 : Number(r.rate_multiplier) const rate = Number.isFinite(parsedRate) ? parsedRate : 1 const actual = priced.total_cost == null ? null : priced.total_cost * rate - upd.run( + await upd.run( priced.input_cost ?? null, priced.output_cost ?? null, priced.cache_read_cost ?? null, @@ -639,7 +667,7 @@ export class UsageLogsRepo { r.id, ) } else { - upd.run( + await upd.run( priced.input_cost ?? null, priced.output_cost ?? null, priced.cache_read_cost ?? null, @@ -654,13 +682,7 @@ export class UsageLogsRepo { ) } } - this.db.exec('COMMIT') - } catch (e) { - try { - this.db.exec('ROLLBACK') - } catch {} - throw e - } + }) return { rows: rows.length, changed, still_unpriced, delta_usd } } @@ -670,8 +692,8 @@ export class UsageLogsRepo { * (cost stays NULL) so they are never guessed at standard rates. Idempotent. * Only `pricing_model IS NULL` rows; those set actual_cost = total_cost × rate_multiplier. */ - backfillMissingCosts({ limit = 4000 } = {}) { - const rows = this.db + async backfillMissingCosts({ limit = 4000 } = {}) { + const rows = await this.db .prepare(` SELECT id, model, upstream_model, requested_model, input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens, @@ -681,53 +703,54 @@ export class UsageLogsRepo { total_cost, pricing_model, price_source, price_version, long_context FROM usage_logs WHERE pricing_model IS NULL - LIMIT ? + LIMIT $1 `) .all(Math.max(1, Math.min(20000, Number(limit) || 4000))) - return this._repriceRows(rows, { writeActualCost: true }).rows + return (await this._repriceRows(rows, { writeActualCost: true })).rows } /** Rewrite cost columns in [since, until]; never actual_cost / rate_multiplier. */ - repriceRange({ since, until } = {}) { + async repriceRange({ since, until } = {}) { const { cond, params } = timeCond(since, until) - const where = cond ? `${cond} AND rowid > ?` : 'WHERE rowid > ?' + const where = `${cond || 'WHERE 1 = 1'} AND id > $${params.length + 1}` // Keyset pages keep a month-long range from loading every row at once. const page = this.db.prepare(` - SELECT rowid AS _rowid, id, model, upstream_model, requested_model, + SELECT id, model, upstream_model, requested_model, input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens, cache_creation_5m_tokens, cache_creation_1h_tokens, web_search_requests, rate_multiplier, service_tier, speed, input_cost, output_cost, cache_read_cost, cache_creation_cost, web_search_cost, total_cost, pricing_model, price_source, price_version, long_context FROM usage_logs ${where} - ORDER BY rowid + ORDER BY id LIMIT 2000 `) const out = { rows: 0, changed: 0, still_unpriced: 0, delta_usd: 0 } - let after = 0 + let after = '' for (;;) { - const rows = page.all(...params, after) + const rows = await page.all(...params, after) if (!rows.length) break - const part = this._repriceRows(rows, { writeActualCost: false }) + const part = await this._repriceRows(rows, { writeActualCost: false }) out.rows += part.rows out.changed += part.changed out.still_unpriced += part.still_unpriced out.delta_usd += part.delta_usd - after = rows.at(-1)._rowid + after = rows.at(-1).id } out.delta_usd = Math.round(out.delta_usd * 1e8) / 1e8 return out } - unpricedModels({ since } = {}) { + async unpricedModels({ since } = {}) { const where = [`pricing_model = '${UNPRICED_MODEL}'`] const params = [] if (since) { - where.push('created_at >= ?') + where.push(`created_at >= $${params.length + 1}`) params.push(new Date(since).toISOString()) } - return this.db - .prepare(` + return ( + await this.db + .prepare(` SELECT COALESCE(NULLIF(upstream_model, ''), NULLIF(model, ''), NULLIF(requested_model, ''), '') AS model, COUNT(*) AS requests, MAX(created_at) AS last_seen @@ -736,16 +759,16 @@ export class UsageLogsRepo { GROUP BY 1 ORDER BY requests DESC, last_seen DESC `) - .all(...params) - .map((r) => ({ - model: r.model, - requests: Number(r.requests || 0), - last_seen: r.last_seen || null, - })) + .all(...params) + ).map((r) => ({ + model: r.model, + requests: integerValue(r.requests), + last_seen: r.last_seen || null, + })) } - _costSelect(cond, params) { - const row = this.db + async _costSelect(cond, params) { + const row = await this.db .prepare(` SELECT COUNT(*) AS requests, @@ -764,25 +787,25 @@ export class UsageLogsRepo { `) .get(...params) const out = { - requests: Number(row?.requests || 0), - input_tokens: Number(row?.input_tokens || 0), - output_tokens: Number(row?.output_tokens || 0), - cache_read_tokens: Number(row?.cache_read_tokens || 0), - cache_creation_tokens: Number(row?.cache_creation_tokens || 0), - prompt_tokens: Number(row?.prompt_tokens || 0), + requests: integerValue(row?.requests), + input_tokens: integerValue(row?.input_tokens), + output_tokens: integerValue(row?.output_tokens), + cache_read_tokens: integerValue(row?.cache_read_tokens), + cache_creation_tokens: integerValue(row?.cache_creation_tokens), + prompt_tokens: integerValue(row?.prompt_tokens), input_cost: Number(row?.input_cost || 0), output_cost: Number(row?.output_cost || 0), cache_read_cost: Number(row?.cache_read_cost || 0), cache_creation_cost: Number(row?.cache_creation_cost || 0), total_cost: Number(row?.total_cost || 0), - unpriced: Number(row?.unpriced || 0), + unpriced: integerValue(row?.unpriced), } - return { ...out, ...cacheHitStats(out) } + return withCacheHitRate(out) } - costByAccount({ since = null, until = null } = {}) { + async costByAccount({ since = null, until = null } = {}) { const { cond, params } = timeCond(since, until) - const rows = this.db + const rows = await this.db .prepare(` SELECT COALESCE(NULLIF(final_account_id, ''), NULLIF(account_id, ''), vm_id, '—') AS account_id, @@ -809,20 +832,20 @@ export class UsageLogsRepo { return rows.map((r) => ({ account_id: r.account_id, vm_id: r.vm_id || null, - requests: Number(r.requests || 0), - success: Number(r.success || 0), - errors: Number(r.errors || 0), - input_tokens: Number(r.input_tokens || 0), - output_tokens: Number(r.output_tokens || 0), - cache_read_tokens: Number(r.cache_read_tokens || 0), - cache_creation_tokens: Number(r.cache_creation_tokens || 0), - prompt_tokens: Number(r.prompt_tokens || 0), + requests: integerValue(r.requests), + success: integerValue(r.success), + errors: integerValue(r.errors), + input_tokens: integerValue(r.input_tokens), + output_tokens: integerValue(r.output_tokens), + cache_read_tokens: integerValue(r.cache_read_tokens), + cache_creation_tokens: integerValue(r.cache_creation_tokens), + prompt_tokens: integerValue(r.prompt_tokens), input_cost: Number(r.input_cost || 0), output_cost: Number(r.output_cost || 0), cache_read_cost: Number(r.cache_read_cost || 0), cache_creation_cost: Number(r.cache_creation_cost || 0), total_cost: Number(r.total_cost || 0), - unpriced: Number(r.unpriced || 0), + unpriced: integerValue(r.unpriced), })) } @@ -831,9 +854,9 @@ export class UsageLogsRepo { * service_tier is folded to fast / flex / other-raw (standard aliases → NULL); * speed is 'fast' or NULL. unpriced_requests counts rows left without an estimate. */ - costByModel({ vmId = null, accountId = null, since = null, until = null } = {}) { + async costByModel({ vmId = null, accountId = null, since = null, until = null } = {}) { const { cond, params } = filterCond({ since, until, vmId, accountId }) - const rows = this.db + const rows = await this.db .prepare(` SELECT COALESCE(NULLIF(upstream_model, ''), NULLIF(model, ''), NULLIF(requested_model, ''), '—') AS model, CASE @@ -866,13 +889,13 @@ export class UsageLogsRepo { service_tier: r.service_tier || null, speed: r.speed || null, long_context: Number(r.long_context || 0), - requests: Number(r.requests || 0), - unpriced_requests: Number(r.unpriced_requests || 0), - input_tokens: Number(r.input_tokens || 0), - output_tokens: Number(r.output_tokens || 0), - cache_read_tokens: Number(r.cache_read_tokens || 0), - cache_creation_tokens: Number(r.cache_creation_tokens || 0), - prompt_tokens: Number(r.prompt_tokens || 0), + requests: integerValue(r.requests), + unpriced_requests: integerValue(r.unpriced_requests), + input_tokens: integerValue(r.input_tokens), + output_tokens: integerValue(r.output_tokens), + cache_read_tokens: integerValue(r.cache_read_tokens), + cache_creation_tokens: integerValue(r.cache_creation_tokens), + prompt_tokens: integerValue(r.prompt_tokens), input_cost: Number(r.input_cost || 0), output_cost: Number(r.output_cost || 0), cache_read_cost: Number(r.cache_read_cost || 0), @@ -887,8 +910,8 @@ export class UsageLogsRepo { * `models` / `endpoints` are ranked by request count and capped; `endpoints` * is the inbound request path (usage_logs keeps no separate upstream endpoint). */ - vmUsageStats({ vmId, days = 30 } = {}) { - return this._usageWindow({ + async vmUsageStats({ vmId, days = 30 } = {}) { + return await this._usageWindow({ vmId, days, rankExpr: "COALESCE(NULLIF(path, ''), '—')", @@ -898,8 +921,8 @@ export class UsageLogsRepo { } /** Same window as vmUsageStats, isolated to one managed key. Third rank is the VM. */ - keyUsageStats({ apiKeyId, days = 30 } = {}) { - return this._usageWindow({ + async keyUsageStats({ apiKeyId, days = 30 } = {}) { + return await this._usageWindow({ apiKeyId, days, rankExpr: "COALESCE(NULLIF(vm_id, ''), '—')", @@ -908,15 +931,16 @@ export class UsageLogsRepo { }) } - _usageWindow({ vmId = null, apiKeyId = null, days = 30, rankExpr, rankKey, rankLimit = 8 } = {}) { + async _usageWindow({ vmId = null, apiKeyId = null, days = 30, rankExpr, rankKey, rankLimit = 8 } = {}) { const empty = { days: 0, since: null, history: [], models: [], [rankKey]: [] } if (!vmId && !apiKeyId) return empty const span = Math.max(1, Math.min(90, Math.floor(Number(days)) || 30)) const since = new Date(Date.parse(shanghaiDayStartIso()) - (span - 1) * 86400_000).toISOString() const { cond, params } = filterCond({ since, vmId, apiKeyId }) - const history = this.db - .prepare(` - SELECT strftime('%Y-%m-%d', created_at, '+8 hours') AS day, + const history = ( + await this.db + .prepare(` + SELECT to_char(created_at::timestamptz AT TIME ZONE 'Asia/Shanghai', 'YYYY-MM-DD') AS day, COUNT(*) AS requests, SUM(CASE WHEN ${SLA_ERROR_PRED} THEN 1 ELSE 0 END) AS errors, COALESCE(SUM(input_tokens), 0) AS input_tokens, @@ -930,23 +954,24 @@ export class UsageLogsRepo { FROM usage_logs ${cond} GROUP BY day ORDER BY day `) - .all(...params) - .map((r) => ({ - day: r.day, - requests: Number(r.requests || 0), - errors: Number(r.errors || 0), - input_tokens: Number(r.input_tokens || 0), - output_tokens: Number(r.output_tokens || 0), - cache_read_tokens: Number(r.cache_read_tokens || 0), - cache_creation_tokens: Number(r.cache_creation_tokens || 0), - prompt_tokens: Number(r.prompt_tokens || 0), - total_cost: Number(r.total_cost || 0), - duration_ms_sum: Number(r.duration_ms_sum || 0), - duration_n: Number(r.duration_n || 0), - })) - const group = (expr, limit) => - this.db - .prepare(` + .all(...params) + ).map((r) => ({ + day: r.day, + requests: integerValue(r.requests), + errors: integerValue(r.errors), + input_tokens: integerValue(r.input_tokens), + output_tokens: integerValue(r.output_tokens), + cache_read_tokens: integerValue(r.cache_read_tokens), + cache_creation_tokens: integerValue(r.cache_creation_tokens), + prompt_tokens: integerValue(r.prompt_tokens), + total_cost: Number(r.total_cost || 0), + duration_ms_sum: integerValue(r.duration_ms_sum), + duration_n: integerValue(r.duration_n), + })) + const group = async (expr, limit) => + ( + await this.db + .prepare(` SELECT ${expr} AS name, COUNT(*) AS requests, COALESCE(SUM(${TOTAL_TOKENS_SQL}), 0) AS tokens, @@ -954,19 +979,22 @@ export class UsageLogsRepo { FROM usage_logs ${cond} GROUP BY name ORDER BY requests DESC, total_cost DESC LIMIT ${limit} `) - .all(...params) - .map((r) => ({ - name: r.name, - requests: Number(r.requests || 0), - tokens: Number(r.tokens || 0), - total_cost: Number(r.total_cost || 0), - })) + .all(...params) + ).map((r) => ({ + name: r.name, + requests: integerValue(r.requests), + tokens: integerValue(r.tokens), + total_cost: Number(r.total_cost || 0), + })) return { days: span, since, history, - models: group("COALESCE(NULLIF(upstream_model, ''), NULLIF(model, ''), NULLIF(requested_model, ''), '—')", 12), - [rankKey]: group(rankExpr, rankLimit), + models: await group( + "COALESCE(NULLIF(upstream_model, ''), NULLIF(model, ''), NULLIF(requested_model, ''), '—')", + 12, + ), + [rankKey]: await group(rankExpr, rankLimit), } } @@ -975,7 +1003,7 @@ export class UsageLogsRepo { * `total` is every inbound inference request (warmup answers included) plus * gate blocks on other paths, so each stage's share uses one denominator. */ - protocolEntryStats({ since = shanghaiDayStartIso() } = {}) { + async protocolEntryStats({ since = shanghaiDayStartIso() } = {}) { // COALESCE: a bare `col = x` on NULL yields NULL, and `NOT NULL` drops the row. const block = `( COALESCE(error_code, '') = 'policy_blocked' @@ -983,7 +1011,7 @@ export class UsageLogsRepo { OR COALESCE(final_state, '') IN ('policy_blocked', 'distill_blocked', 'refusal_guard', 'refusal_similar', 'refusal_device') )` const inference = `path IN ('/v1/messages', '/v1/responses', '/v1/chat/completions')` - const row = this.db + const row = await this.db .prepare( `SELECT SUM(CASE WHEN ${inference} OR ${block} THEN 1 ELSE 0 END) AS total, @@ -993,14 +1021,14 @@ export class UsageLogsRepo { AND COALESCE(via, '') != 'warmup-intercept' THEN 1 ELSE 0 END) AS passed FROM usage_logs - WHERE created_at >= ?`, + WHERE created_at >= $1`, ) .get(since) - const keywords = this.db + const keywords = await this.db .prepare( `SELECT error_message AS message, COUNT(*) AS count FROM usage_logs - WHERE created_at >= ? + WHERE created_at >= $1 AND ${block} AND error_message IS NOT NULL AND error_message != '' @@ -1009,13 +1037,13 @@ export class UsageLogsRepo { LIMIT 50`, ) .all(since) - const grouped = this.db + const grouped = await this.db .prepare( `SELECT intercept, via, error_message AS message, CASE WHEN ${block} THEN 1 ELSE 0 END AS blocked, COUNT(*) AS count FROM usage_logs - WHERE created_at >= ? + WHERE created_at >= $1 AND ( ${block} OR (${inference} AND COALESCE(via, '') != 'warmup-intercept') @@ -1026,9 +1054,9 @@ export class UsageLogsRepo { const folded = foldInterceptRows(grouped) return { since, - total: Number(row?.total || 0), - blocked: Number(row?.blocked || 0), - passed: Number(row?.passed || 0), + total: integerValue(row?.total), + blocked: integerValue(row?.blocked), + passed: integerValue(row?.passed), keywords: collapseBlockKeywords(keywords), blocks: folded.blocks, passes: folded.passes, @@ -1036,7 +1064,7 @@ export class UsageLogsRepo { } /** Official-standard totals: all-time + Shanghai calendar today + per account. */ - billingStats(opts = {}) { + async billingStats(opts = {}) { const { accountWindows = null, now = Date.now() } = opts const cacheable = !Object.hasOwn(opts, 'now') const cacheNow = Date.now() @@ -1050,11 +1078,11 @@ export class UsageLogsRepo { if (cacheNow - this._billingBackfillAt >= BILLING_BACKFILL_INTERVAL_MS) { this._billingBackfillAt = cacheNow try { - this.backfillMissingCosts() + await this.backfillMissingCosts() } catch {} } - const value = this._computeBillingStats({ accountWindows, now }) + const value = await this._computeBillingStats({ accountWindows, now }) if (this._billingStatsCache.size > 32) { for (const [cacheKey, entry] of this._billingStatsCache) { if (entry.expires_at <= cacheNow) this._billingStatsCache.delete(cacheKey) @@ -1068,20 +1096,20 @@ export class UsageLogsRepo { return value } - _computeBillingStats({ accountWindows = null, now = Date.now() } = {}) { + async _computeBillingStats({ accountWindows = null, now = Date.now() } = {}) { const todayStart = shanghaiDayStartIso() const fiveHStart = new Date(now - WINDOW_5H_MS).toISOString() const sevenDStart = new Date(now - WINDOW_7D_MS).toISOString() - const accountsTotal = this.costByAccount() - const accountsToday = this.costByAccount({ since: todayStart }) + const accountsTotal = await this.costByAccount() + const accountsToday = await this.costByAccount({ since: todayStart }) // The per-account queries already cover every row. Re-sum them for the // global buckets instead of scanning the entire usage table two more times. // This matters on multi-GB databases where each synchronous full scan also // blocks unrelated HTTP requests on the Node event loop. const totalBucket = sumCostBuckets(accountsTotal) const todayBucket = sumCostBuckets(accountsToday) - const total = { ...totalBucket, ...cacheHitStats(totalBucket) } - const today = { ...todayBucket, ...cacheHitStats(todayBucket) } + const total = withCacheHitRate(totalBucket) + const today = withCacheHitRate(todayBucket) const keyOf = (a) => a.account_id + '\0' + (a.vm_id || '') const todayById = new Map(accountsToday.map((a) => [keyOf(a), a])) const aligned = Array.isArray(accountWindows) && accountWindows.length > 0 @@ -1100,9 +1128,13 @@ export class UsageLogsRepo { } eventsSince = new Date(earliest).toISOString() } - const events = aligned ? this._costEventsSince(eventsSince) : null - const fiveById = aligned ? null : new Map(this.costByAccount({ since: fiveHStart }).map((a) => [keyOf(a), a])) - const sevenById = aligned ? null : new Map(this.costByAccount({ since: sevenDStart }).map((a) => [keyOf(a), a])) + const events = aligned ? await this._costEventsSince(eventsSince) : null + const fiveById = aligned + ? null + : new Map((await this.costByAccount({ since: fiveHStart })).map((a) => [keyOf(a), a])) + const sevenById = aligned + ? null + : new Map((await this.costByAccount({ since: sevenDStart })).map((a) => [keyOf(a), a])) const eventsByKey = aligned ? groupCostEvents(events) : null const accounts = accountsTotal.map((a) => { const t = todayById.get(keyOf(a)) || emptyCostBucket() @@ -1122,8 +1154,8 @@ export class UsageLogsRepo { window_fable = { cost: Number(fable.total_cost || 0), tokens: totalTokens(fable), - requests: Number(fable.requests || 0), - unpriced: Number(fable.unpriced || 0), + requests: integerValue(fable.requests), + unpriced: integerValue(fable.unpriced), start: new Date(sinceFable).toISOString(), } } @@ -1133,49 +1165,43 @@ export class UsageLogsRepo { window_5h: { ...w }, window_7d: { ...w7 }, today_cost: Number(t.total_cost || 0), - today_requests: Number(t.requests || 0), - today_input_tokens: Number(t.input_tokens || 0), - today_output_tokens: Number(t.output_tokens || 0), - today_cache_read_tokens: Number(t.cache_read_tokens || 0), - today_cache_creation_tokens: Number(t.cache_creation_tokens || 0), + today_requests: integerValue(t.requests), + today_input_tokens: integerValue(t.input_tokens), + today_output_tokens: integerValue(t.output_tokens), + today_cache_read_tokens: integerValue(t.cache_read_tokens), + today_cache_creation_tokens: integerValue(t.cache_creation_tokens), today_input_cost: Number(t.input_cost || 0), today_output_cost: Number(t.output_cost || 0), today_cache_cost: Number(t.cache_read_cost || 0) + Number(t.cache_creation_cost || 0), window_5h_cost: Number(w.total_cost || 0), - window_5h_requests: Number(w.requests || 0), + window_5h_requests: integerValue(w.requests), window_5h_tokens: totalTokens(w), - window_5h_unpriced: Number(w.unpriced || 0), - window_5h_input_tokens: Number(w.input_tokens || 0), - window_5h_output_tokens: Number(w.output_tokens || 0), - window_5h_cache_read_tokens: Number(w.cache_read_tokens || 0), - window_5h_cache_creation_tokens: Number(w.cache_creation_tokens || 0), + window_5h_unpriced: integerValue(w.unpriced), + window_5h_input_tokens: integerValue(w.input_tokens), + window_5h_output_tokens: integerValue(w.output_tokens), + window_5h_cache_read_tokens: integerValue(w.cache_read_tokens), + window_5h_cache_creation_tokens: integerValue(w.cache_creation_tokens), window_5h_input_cost: Number(w.input_cost || 0), window_5h_output_cost: Number(w.output_cost || 0), window_5h_cache_cost: Number(w.cache_read_cost || 0) + Number(w.cache_creation_cost || 0), window_5h_start: aligned ? new Date(since5).toISOString() : fiveHStart, window_7d_cost: Number(w7.total_cost || 0), - window_7d_requests: Number(w7.requests || 0), - window_7d_success: Number(w7.success || 0), - window_7d_errors: Number(w7.errors || 0), + window_7d_requests: integerValue(w7.requests), + window_7d_success: integerValue(w7.success), + window_7d_errors: integerValue(w7.errors), window_7d_tokens: totalTokens(w7), - window_7d_unpriced: Number(w7.unpriced || 0), + window_7d_unpriced: integerValue(w7.unpriced), window_7d_start: aligned ? new Date(since7).toISOString() : sevenDStart, window_fable, } }) const window5h = aligned - ? { - ...sumCostBuckets(accounts.map((a) => a.window_5h)), - ...cacheHitStats(sumCostBuckets(accounts.map((a) => a.window_5h))), - } - : this._costSelect('WHERE created_at >= ?', [fiveHStart]) + ? withCacheHitRate(sumCostBuckets(accounts.map((a) => a.window_5h))) + : await this._costSelect('WHERE created_at >= $1', [fiveHStart]) const window7d = aligned - ? { - ...sumCostBuckets(accounts.map((a) => a.window_7d)), - ...cacheHitStats(sumCostBuckets(accounts.map((a) => a.window_7d))), - } - : this._costSelect('WHERE created_at >= ?', [sevenDStart]) - const coverage = this.logCoverageStart() + ? withCacheHitRate(sumCostBuckets(accounts.map((a) => a.window_7d))) + : await this._costSelect('WHERE created_at >= $1', [sevenDStart]) + const coverage = await this.logCoverageStart() return { source: 'anthropic-official', currency: 'USD', @@ -1201,17 +1227,18 @@ export class UsageLogsRepo { * surviving row (a quiet hour looks like a gap) nor the current retain_days (raising * it does not bring deleted rows back) can stand in for that boundary. */ - logCoverageStart() { + async logCoverageStart() { const created = Date.parse( - this.db.prepare("SELECT MIN(applied_at) AS t FROM schema_migrations WHERE version = '001'").get()?.t || '', + (await this.db.prepare("SELECT MIN(applied_at) AS t FROM schema_migrations WHERE version = '001'").get())?.t || + '', ) - const pruned = Date.parse(this._prunedThrough() || '') + const pruned = Date.parse((await this._prunedThrough()) || '') const start = Math.max(Number.isFinite(created) ? created : -Infinity, Number.isFinite(pruned) ? pruned : -Infinity) return Number.isFinite(start) ? new Date(start).toISOString() : null } - _prunedThrough() { - const row = this.db.prepare(`SELECT value FROM settings WHERE key = '${PRUNED_THROUGH_KEY}'`).get() + async _prunedThrough() { + const row = await this.db.prepare(`SELECT value FROM settings WHERE key = '${PRUNED_THROUGH_KEY}'`).get() try { return row ? JSON.parse(row.value) : null } catch { @@ -1219,8 +1246,8 @@ export class UsageLogsRepo { } } - _costEventsSince(since) { - return this.db + async _costEventsSince(since) { + return await this.db .prepare( ` SELECT @@ -1243,13 +1270,13 @@ export class UsageLogsRepo { COALESCE(cache_creation_cost, 0) AS cache_creation_cost, COALESCE(total_cost, 0) AS total_cost FROM usage_logs - WHERE created_at >= ? + WHERE created_at >= $1 `, ) .all(since) } - ownerBilling({ ownerUserId = null, since = null, until = null, groupBy = 'vm' } = {}) { + async ownerBilling({ ownerUserId = null, since = null, until = null, groupBy = 'vm' } = {}) { const where = [] const params = [] const own = ownerPred(ownerUserId) @@ -1258,17 +1285,17 @@ export class UsageLogsRepo { params.push(...own.params) } if (since) { - where.push('created_at >= ?') + where.push(`created_at >= $${params.length + 1}`) params.push(new Date(since).toISOString()) } if (until) { - where.push('created_at <= ?') + where.push(`created_at <= $${params.length + 1}`) params.push(new Date(until).toISOString()) } const cond = where.length ? `WHERE ${where.join(' AND ')}` : '' const grouped = groupBy === 'key' ? 'key' : 'vm' const groupCol = grouped === 'key' ? 'api_key_id' : 'vm_id' - const totals = this.db + const totals = await this.db .prepare(` SELECT COUNT(*) AS requests, SUM(CASE WHEN ${SUCCESS_PRED} THEN 1 ELSE 0 END) AS ok, @@ -1280,7 +1307,7 @@ export class UsageLogsRepo { FROM usage_logs ${cond} `) .get(...params) - const items = this.db + const items = await this.db .prepare(` SELECT ${groupCol} AS id, COUNT(*) AS requests, @@ -1301,22 +1328,22 @@ export class UsageLogsRepo { since: since ? new Date(since).toISOString() : null, until: until ? new Date(until).toISOString() : null, totals: { - requests: num(totals, 'requests'), - ok: num(totals, 'ok'), - fail: num(totals, 'fail'), - tokens_in: num(totals, 'tokens_in'), - tokens_out: num(totals, 'tokens_out'), + requests: integerValue(totals.requests), + ok: integerValue(totals.ok), + fail: integerValue(totals.fail), + tokens_in: integerValue(totals.tokens_in), + tokens_out: integerValue(totals.tokens_out), cost_usd: num(totals, 'cost_usd'), official_cost_usd: num(totals, 'official_cost_usd'), }, items: items.map((row) => ({ vm_id: grouped === 'vm' ? row.id || null : undefined, api_key_id: grouped === 'key' ? row.id || null : undefined, - requests: num(row, 'requests'), - ok: num(row, 'ok'), - fail: num(row, 'fail'), - tokens_in: num(row, 'tokens_in'), - tokens_out: num(row, 'tokens_out'), + requests: integerValue(row.requests), + ok: integerValue(row.ok), + fail: integerValue(row.fail), + tokens_in: integerValue(row.tokens_in), + tokens_out: integerValue(row.tokens_out), cost_usd: num(row, 'cost_usd'), official_cost_usd: num(row, 'official_cost_usd'), })), @@ -1327,16 +1354,16 @@ export class UsageLogsRepo { * Windowed ops snapshot aligned with sub2api overview cards: * SLA / error / 429 / 503, QPS·TPS, duration + first_token percentiles, per-model. */ - windowStats({ since = null, until = null, owner_user_id = null } = {}) { + async windowStats({ since = null, until = null, owner_user_id = null } = {}) { const { cond, params } = timeCond(since, until) - const own = ownerPred(owner_user_id) + const own = ownerPred(owner_user_id, params.length) let where = cond const allParams = [...params] if (own.sql) { where = where ? `${where} AND ${own.sql}` : `WHERE ${own.sql}` allParams.push(...own.params) } - const counts = this.db + const counts = await this.db .prepare(` SELECT COUNT(*) AS requests, @@ -1356,58 +1383,56 @@ export class UsageLogsRepo { FROM usage_logs ${where} `) .get(...allParams) - const requests = Number(counts?.requests || 0) - const success = Number(counts?.success || 0) - const errors = Number(counts?.errors || 0) - const inputTokens = Number(counts?.input_tokens || 0) - const outputTokens = Number(counts?.output_tokens || 0) - const cacheRead = Number(counts?.cache_read_tokens || 0) - const cacheWrite = Number(counts?.cache_creation_tokens || 0) - const cache = cacheHitStats({ + const requests = integerValue(counts?.requests) + const success = integerValue(counts?.success) + const errors = integerValue(counts?.errors) + const inputTokens = integerValue(counts?.input_tokens) + const outputTokens = integerValue(counts?.output_tokens) + const cacheRead = integerValue(counts?.cache_read_tokens) + const cacheWrite = integerValue(counts?.cache_creation_tokens) + const cache = withCacheHitRate({ input_tokens: inputTokens, cache_read_tokens: cacheRead, cache_creation_tokens: cacheWrite, - prompt_tokens: Number(counts?.prompt_tokens || 0), + prompt_tokens: integerValue(counts?.prompt_tokens), }) - const tokens = inputTokens + outputTokens + cacheRead + cacheWrite + const tokens = totalTokens(counts) const startMs = since ? Date.parse(since) : null const endMs = until ? Date.parse(until) : Date.now() const windowSeconds = Number.isFinite(startMs) && Number.isFinite(endMs) ? Math.max(1, (endMs - startMs) / 1000) : Math.max(1, requests) - const minuteRows = this.db + const minuteRows = await this.db .prepare(` - SELECT strftime('%Y-%m-%dT%H:%M', created_at) AS minute, + SELECT to_char(created_at::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI') AS minute, COUNT(*) AS requests, COALESCE(SUM(${TOTAL_TOKENS_SQL}), 0) AS tokens FROM usage_logs ${where} GROUP BY minute `) .all(...allParams) - const peakReq = minuteRows.reduce((m, r) => Math.max(m, Number(r.requests || 0)), 0) - const peakTok = minuteRows.reduce((m, r) => Math.max(m, Number(r.tokens || 0)), 0) + const peakReq = minuteRows.reduce((m, r) => Math.max(m, integerValue(r.requests)), 0) + const peakTok = minuteRows.reduce((m, r) => Math.max(m, integerValue(r.tokens)), 0) const lastMinute = minuteRows.length ? minuteRows[minuteRows.length - 1] : null const ttftWhere = where ? `${where} AND first_token_ms IS NOT NULL` : 'WHERE first_token_ms IS NOT NULL' const durWhere = where ? `${where} AND duration_ms IS NOT NULL` : 'WHERE duration_ms IS NOT NULL' - const ttfts = this.db - .prepare(`SELECT first_token_ms AS v FROM usage_logs ${ttftWhere}`) - .all(...allParams) - .map((r) => r.v) - const durs = this.db - .prepare(`SELECT duration_ms AS v FROM usage_logs ${durWhere}`) - .all(...allParams) - .map((r) => r.v) + const ttfts = ( + await this.db.prepare(`SELECT first_token_ms AS v FROM usage_logs ${ttftWhere}`).all(...allParams) + ).map((r) => r.v) + const durs = (await this.db.prepare(`SELECT duration_ms AS v FROM usage_logs ${durWhere}`).all(...allParams)).map( + (r) => r.v, + ) - const byModel = this.db + const byModel = await this.db .prepare(` SELECT COALESCE(NULLIF(upstream_model, ''), NULLIF(model, ''), '—') AS model, COUNT(*) AS requests, SUM(CASE WHEN ${SLA_ERROR_PRED} THEN 1 ELSE 0 END) AS errors, SUM(CASE WHEN first_token_ms IS NOT NULL THEN 1 ELSE 0 END) AS ttft_samples, - CAST(AVG(first_token_ms) AS INTEGER) AS avg_first_token_ms, - CAST(AVG(duration_ms) AS INTEGER) AS avg_duration_ms + TRUNC(AVG(first_token_ms))::bigint AS avg_first_token_ms, + TRUNC(AVG(duration_ms))::bigint AS avg_duration_ms FROM usage_logs ${where} GROUP BY 1 ORDER BY requests DESC @@ -1424,17 +1449,17 @@ export class UsageLogsRepo { errors, sla: requests ? success / requests : 1, error_rate: requests ? errors / requests : 0, - status_429: Number(counts?.status_429 || 0), - status_529: Number(counts?.status_529 || 0), - status_503: Number(counts?.status_503 || 0), - stream_requests: Number(counts?.stream_requests || 0), + status_429: integerValue(counts?.status_429), + status_529: integerValue(counts?.status_529), + status_503: integerValue(counts?.status_503), + stream_requests: integerValue(counts?.stream_requests), input_tokens: inputTokens, output_tokens: outputTokens, cache_read_tokens: cacheRead, cache_creation_tokens: cacheWrite, prompt_tokens: cache.prompt_tokens, cache_hit_rate: cache.cache_hit_rate, - sticky: this._stickyWindow({ since, until, owner_user_id }), + sticky: await this._stickyWindow({ since, until, owner_user_id }), total_cost: Number(counts?.total_cost || 0), qps: { current: lastMinute ? Number(lastMinute.requests || 0) / 60 : 0, @@ -1450,36 +1475,36 @@ export class UsageLogsRepo { ttft: summarizeLatency(ttfts), by_model: byModel.map((r) => ({ model: r.model, - requests: Number(r.requests || 0), - errors: Number(r.errors || 0), - ttft_samples: Number(r.ttft_samples || 0), + requests: integerValue(r.requests), + errors: integerValue(r.errors), + ttft_samples: integerValue(r.ttft_samples), avg_first_token_ms: r.avg_first_token_ms ?? null, avg_duration_ms: r.avg_duration_ms ?? null, })), - error_collection: this.errorCollection({ since, until, owner_user_id }), + error_collection: await this.errorCollection({ since, until, owner_user_id }), } } - _stickyWindow({ since = null, until = null, owner_user_id = null } = {}) { + async _stickyWindow({ since = null, until = null, owner_user_id = null } = {}) { const empty = { selections: 0, hits: 0, rate: null } try { const parts = [] const params = [] if (since) { - parts.push('started_at >= ?') + parts.push(`started_at >= $${params.length + 1}`) params.push(since) } if (until) { - parts.push('started_at <= ?') + parts.push(`started_at <= $${params.length + 1}`) params.push(until) } - const own = ownerPred(owner_user_id) + const own = ownerPred(owner_user_id, params.length) if (own.sql) { parts.push(`request_id IN (SELECT request_id FROM usage_logs WHERE ${own.sql})`) params.push(...own.params) } const where = parts.length ? `WHERE ${parts.join(' AND ')}` : '' - const rows = this.db + const rows = await this.db .prepare(` SELECT selection_reason, COUNT(*) AS n FROM request_attempts ${where} @@ -1489,9 +1514,9 @@ export class UsageLogsRepo { let selections = 0 let hits = 0 for (const row of rows) { - const n = Number(row.n || 0) - selections += n - if (String(row.selection_reason || '') === 'sticky') hits += n + const n = integerValue(row.n) + selections = addInteger(selections, n) + if (String(row.selection_reason || '') === 'sticky') hits = addInteger(hits, n) } return { selections, @@ -1503,11 +1528,11 @@ export class UsageLogsRepo { } } - errorCollection({ since = null, until = null, owner_user_id = null } = {}) { + async errorCollection({ since = null, until = null, owner_user_id = null } = {}) { const { cond, params } = timeCond(since, until) - const own = ownerPred(owner_user_id) + const own = ownerPred(owner_user_id, params.length) const errCond = `${cond ? `${cond} AND ` : 'WHERE '}${ERROR_PRED}${own.sql ? ` AND ${own.sql}` : ''}` - const rows = this.db + const rows = await this.db .prepare(` SELECT created_at AS ts, request_id, status, error_code, error_message, model, upstream_model, vm_id, ip_address AS ip, api_key_presented FROM usage_logs ${errCond} @@ -1517,19 +1542,20 @@ export class UsageLogsRepo { .all(...params, ...own.params) return { ...collectErrors(rows), - ingress_hits: this.ingressHits({ since, until, owner_user_id }), + ingress_hits: await this.ingressHits({ since, until, owner_user_id }), } } /** Invalid-key 401s grouped by presented token + IP (still counted when muted). */ - ingressHits({ since = null, until = null, owner_user_id = null } = {}) { + async ingressHits({ since = null, until = null, owner_user_id = null } = {}) { const { cond, params } = timeCond(since, until) - const own = ownerPred(owner_user_id) + const own = ownerPred(owner_user_id, params.length) const authPred = own.sql ? `${ingressAuthSql()} AND ${own.sql}` : ingressAuthSql() const where = cond ? `${cond} AND ${authPred}` : `WHERE ${authPred}` try { - return this.db - .prepare(` + return ( + await this.db + .prepare(` SELECT COALESCE(api_key_presented, '') AS api_key_presented, COALESCE(ip_address, '') AS ip, @@ -1542,14 +1568,14 @@ export class UsageLogsRepo { ORDER BY count DESC LIMIT 40 `) - .all(...params, ...own.params) - .map((row) => ({ - api_key_presented: row.api_key_presented || '', - ip: row.ip || '', - count: Number(row.count || 0), - last_ts: row.last_ts || null, - error_code: row.error_code || null, - })) + .all(...params, ...own.params) + ).map((row) => ({ + api_key_presented: row.api_key_presented || '', + ip: row.ip || '', + count: integerValue(row.count), + last_ts: row.last_ts || null, + error_code: row.error_code || null, + })) } catch { return [] } @@ -1559,19 +1585,11 @@ export class UsageLogsRepo { * Age out summaries/debug independently, then trim debug blobs to maxBytes. * Number form `cleanup(7)` keeps the old retainDays-only call. */ - cleanup(opts = {}) { + async cleanup(opts = {}) { const cfg = this._cleanupOpts(opts) - const usage_logs = this._pruneUsage(cfg.usageCutoff) - const debug_by_age = this._deleteOldDebug.run(cfg.debugCutoff).changes - const debug_by_size = this.trimDebugToMaxBytes(cfg.maxBytes) - if (usage_logs + debug_by_age + debug_by_size > 0) { - try { - this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)') - } catch {} - } - try { - this.db.exec('PRAGMA optimize') - } catch {} + const usage_logs = await this._pruneUsage(cfg.usageCutoff) + const debug_by_age = (await this._deleteOldDebug.run(cfg.debugCutoff)).changes + const debug_by_size = await this.trimDebugToMaxBytes(cfg.maxBytes) return { usage_logs, request_log_debug: debug_by_age + debug_by_size, @@ -1581,29 +1599,17 @@ export class UsageLogsRepo { } /** Delete summaries before `cutoff` and advance the prune boundary in one transaction. */ - _pruneUsage(cutoff) { - this.db.exec('BEGIN') - try { - const deleted = this._deleteOldUsage.run(cutoff).changes + async _pruneUsage(cutoff) { + return withTransaction(this.db, async () => { + const deleted = (await this._deleteOldUsage.run(cutoff)).changes if (deleted > 0) { - const prev = this._prunedThrough() - if (!prev || prev < cutoff) { - this.db - .prepare(` - INSERT INTO settings (key, value, updated_at) VALUES ('${PRUNED_THROUGH_KEY}', ?, ?) - ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at - `) - .run(JSON.stringify(cutoff), new Date().toISOString()) - } + await this.db + .prepare(`INSERT INTO settings (key, value, updated_at) VALUES ('${PRUNED_THROUGH_KEY}', $1, $2) + ON CONFLICT(key) DO UPDATE SET value = GREATEST(settings.value, excluded.value), updated_at = excluded.updated_at`) + .run(JSON.stringify(cutoff), new Date().toISOString()) } - this.db.exec('COMMIT') return deleted - } catch (e) { - try { - this.db.exec('ROLLBACK') - } catch {} - throw e - } + }) } _cleanupOpts(opts) { @@ -1618,35 +1624,35 @@ export class UsageLogsRepo { } } - debugBytes() { - return Number(this._debugBytes.get().n || 0) + async debugBytes() { + return integerValue((await this._debugBytes.get()).n) } - countDebug() { - return Number(this._debugCount.get().n || 0) + async countDebug() { + return integerValue((await this._debugCount.get()).n) } /** Delete oldest debug rows until the stored JSON bytes fit maxBytes. */ - trimDebugToMaxBytes(maxBytes) { + async trimDebugToMaxBytes(maxBytes) { const cap = Math.max(0, Number(maxBytes) || 0) if (cap <= 0) return 0 let deleted = 0 for (let i = 0; i < 8; i++) { - const used = this.debugBytes() + const used = await this.debugBytes() if (used <= cap) return deleted - const count = this.countDebug() + const count = await this.countDebug() if (!count) return deleted const avg = Math.max(1, Math.floor(used / count)) const need = Math.min(count, Math.max(1, Math.ceil((used - cap) / avg) + 1)) - const n = this._deleteOldestDebug.run(need).changes + const n = (await this._deleteOldestDebug.run(need)).changes if (!n) return deleted deleted += n } return deleted } - count() { - return this.db.prepare('SELECT COUNT(*) c FROM usage_logs').get().c + async count() { + return (await this.db.prepare('SELECT COUNT(*) c FROM usage_logs').get()).c } } diff --git a/src/lib/db/repos/usage-logs-view.mjs b/src/lib/db/repos/usage-logs-view.mjs index 7e7b2b90a..91a91c09c 100644 --- a/src/lib/db/repos/usage-logs-view.mjs +++ b/src/lib/db/repos/usage-logs-view.mjs @@ -10,6 +10,7 @@ */ import { getDb } from '../database.mjs' +import { addInteger, integerValue } from '../integers.mjs' import { DEFAULT_MUTED_ERROR_CLASSES, enrichLogRow, excludeErrorClassSql } from '../../admin/error-class.mjs' import { reportTimezone, zonedDayStartMs } from '../../core/timezone.mjs' import { @@ -73,30 +74,30 @@ function uniq(values) { } function placeholders(list) { - return list.map(() => '?').join(', ') + return list.map((_, i) => '$' + (i + 1)).join(', ') } /** * Batched display names for one page / one aggregate. One IN query per entity * kind so a page of 100 rows costs at most four lookups. */ -export function lookupNames(db, { userIds = [], keyIds = [], vmIds = [], accountIds = [] } = {}) { - const pick = (sql, ids, nameOf) => { +export async function lookupNames(db, { userIds = [], keyIds = [], vmIds = [], accountIds = [] } = {}) { + const pick = async (sql, ids, nameOf) => { const list = uniq(ids) const map = new Map() if (!list.length) return map - for (const r of db.prepare(sql.replace('$IN', placeholders(list))).all(...list)) map.set(r.id, nameOf(r)) + for (const r of await db.prepare(sql.replace('$IN', placeholders(list))).all(...list)) map.set(r.id, nameOf(r)) return map } return { - users: pick( + users: await pick( 'SELECT id, username, email FROM users WHERE id IN ($IN)', userIds, (r) => r.username || r.email || r.id, ), - keys: pick('SELECT id, name FROM api_keys WHERE id IN ($IN)', keyIds, (r) => r.name || r.id), - vms: pick('SELECT id, name FROM vms WHERE id IN ($IN)', vmIds, (r) => r.name || r.id), - accounts: pick( + keys: await pick('SELECT id, name FROM api_keys WHERE id IN ($IN)', keyIds, (r) => r.name || r.id), + vms: await pick('SELECT id, name FROM vms WHERE id IN ($IN)', vmIds, (r) => r.name || r.id), + accounts: await pick( 'SELECT id, email, name FROM accounts WHERE id IN ($IN)', accountIds, (r) => r.email || r.name || r.id, @@ -105,8 +106,8 @@ export function lookupNames(db, { userIds = [], keyIds = [], vmIds = [], account } function cacheTtlApplied(row) { - const short = num(row.cache_creation_5m_tokens) - const long = num(row.cache_creation_1h_tokens) + const short = integerValue(row.cache_creation_5m_tokens) + const long = integerValue(row.cache_creation_1h_tokens) if (short > 0 && long > 0) return 'mixed' if (long > 0) return '1h' if (short > 0) return '5m' @@ -155,10 +156,10 @@ function blockedBy(row) { /** Raw `usage_logs` row (already `enrichLogRow`-ed) + batched names → `UsageLogRow`. */ export function toUsageLogRow(row, names, chain = []) { const accountId = row.final_account_id || row.account_id || null - const inputTokens = num(row.input_tokens) - const outputTokens = num(row.output_tokens) - const cacheCreation = num(row.cache_creation_tokens) - const cacheRead = num(row.cache_read_tokens) + const inputTokens = integerValue(row.input_tokens) + const outputTokens = integerValue(row.output_tokens) + const cacheCreation = integerValue(row.cache_creation_tokens) + const cacheRead = integerValue(row.cache_read_tokens) const blocked = blockedBy(row) return { id: row.id, @@ -188,8 +189,8 @@ export function toUsageLogRow(row, names, chain = []) { outputTokens, cacheCreationInputTokens: cacheCreation, cacheReadInputTokens: cacheRead, - cacheCreation5mInputTokens: num(row.cache_creation_5m_tokens), - cacheCreation1hInputTokens: num(row.cache_creation_1h_tokens), + cacheCreation5mInputTokens: integerValue(row.cache_creation_5m_tokens), + cacheCreation1hInputTokens: integerValue(row.cache_creation_1h_tokens), cacheTtlApplied: cacheTtlApplied(row), totalTokens: rowTotalTokens(row), costUsd: nullableNum(row.total_cost), @@ -199,7 +200,7 @@ export function toUsageLogRow(row, names, chain = []) { durationMs: nullableNum(row.duration_ms), ttftMs: nullableNum(row.first_token_ms), stopReason: row.stop_reason ?? null, - attemptCount: nullableNum(row.attempt_count), + attemptCount: row.attempt_count == null ? null : integerValue(row.attempt_count), finalState: row.final_state ?? null, via: row.via ?? null, stream: row.stream == null ? null : !!row.stream, @@ -297,7 +298,7 @@ function buildFilter(filters, defaultMuted) { } const eq = (col, val) => { if (val == null || val === '') return - where.push(`${col} = ?`) + where.push(`${col} = $${params.length + 1}`) params.push(String(val)) } eq('user_id', filters.user_id) @@ -305,22 +306,22 @@ function buildFilter(filters, defaultMuted) { eq('vm_id', filters.vm_id) if (filters.session_id) { // Either side of the mapping finds the conversation. - where.push('(outbound_session_id = ? OR session_id = ?)') + where.push(`(outbound_session_id = $${params.length + 1} OR session_id = $${params.length + 2})`) params.push(String(filters.session_id), String(filters.session_id)) } eq('path', filters.endpoint) eq('protocol', filters.protocol) if (filters.account_id) { - where.push('(account_id = ? OR final_account_id = ?)') + where.push(`(account_id = $${params.length + 1} OR final_account_id = $${params.length + 2})`) params.push(filters.account_id, filters.account_id) } if (filters.model) { - where.push('(model = ? OR requested_model = ?)') + where.push(`(model = $${params.length + 1} OR requested_model = $${params.length + 2})`) params.push(filters.model, filters.model) } const statusCode = clampInt(filters.status_code, 0, 999, null) if (statusCode != null) { - where.push('status = ?') + where.push(`status = $${params.length + 1}`) params.push(statusCode) } else if (filters.exclude_status_200) { where.push('(status IS NULL OR status < 200 OR status >= 300)') @@ -328,7 +329,7 @@ function buildFilter(filters, defaultMuted) { if (filters.model_mismatch) where.push('model_mismatch = 1') const minAttempts = clampInt(filters.min_attempt_count, 0, 1000, null) if (minAttempts != null) { - where.push('IFNULL(attempt_count, 0) >= ?') + where.push(`COALESCE(attempt_count, 0) >= $${params.length + 1}`) params.push(minAttempts) } if (filters.debug_only) where.push("log_mode = 'debug'") @@ -336,16 +337,18 @@ function buildFilter(filters, defaultMuted) { if (filters.errors_only) where.push(ERROR_PRED) const start = isoOrNull(filters.start_time) if (start) { - where.push('created_at >= ?') + where.push(`created_at >= $${params.length + 1}`) params.push(start) } const end = isoOrNull(filters.end_time) if (end) { - where.push('created_at < ?') + where.push(`created_at < $${params.length + 1}`) params.push(end) } if (filters.q) { - where.push('(path LIKE ? OR error_code LIKE ? OR error_message LIKE ? OR request_id LIKE ?)') + where.push( + `(ascii_fold(path) LIKE ascii_fold($${params.length + 1}) ESCAPE '' OR ascii_fold(error_code) LIKE ascii_fold($${params.length + 2}) ESCAPE '' OR ascii_fold(error_message) LIKE ascii_fold($${params.length + 3}) ESCAPE '' OR ascii_fold(request_id) LIKE ascii_fold($${params.length + 4}) ESCAPE '')`, + ) const like = `%${String(filters.q).slice(0, 100)}%` params.push(like, like, like, like) } @@ -378,7 +381,7 @@ function whereSql(where) { export class UsageLogsView { /** - * @param {import('node:sqlite').DatabaseSync} db + * @param {object} db * @param {{ mutedErrorClasses?: string[] }} [opts] settings default mute list */ constructor(db = getDb(), { mutedErrorClasses = DEFAULT_MUTED_ERROR_CLASSES } = {}) { @@ -392,7 +395,7 @@ export class UsageLogsView { * that cap is hit before `limit` matches, the cursor points at the last * scanned row and `hasMore` stays true so the client keeps paging. */ - listBatch(filters = {}) { + async listBatch(filters = {}) { const limit = clampInt(filters.limit, 1, LIST_LIMIT_MAX, LIST_LIMIT_DEFAULT) const { where, params, keep } = buildFilter(filters, this.mutedErrorClasses) let cursor = @@ -408,11 +411,11 @@ export class UsageLogsView { const w = where.slice() const p = params.slice() if (cursor) { - w.push('(created_at < ? OR (created_at = ? AND id < ?))') + w.push(`(created_at < $${p.length + 1} OR (created_at = $${p.length + 2} AND id < $${p.length + 3}))`) p.push(cursor.createdAt, cursor.createdAt, cursor.id) } - const rows = this.db - .prepare(`SELECT * FROM usage_logs ${whereSql(w)} ORDER BY created_at DESC, id DESC LIMIT ?`) + const rows = await this.db + .prepare(`SELECT * FROM usage_logs ${whereSql(w)} ORDER BY created_at DESC, id DESC LIMIT $${p.length + 1}`) .all(...p, want) for (const raw of rows) { scanned++ @@ -448,24 +451,24 @@ export class UsageLogsView { hasMore = true } return { - logs: this._hydrate(page), + logs: await this._hydrate(page), nextCursor: next ? { createdAt: next.created_at, id: next.id } : null, hasMore, } } /** Names + provider chains for a whole page: one attempts query, one lookup per entity kind. */ - _hydrate(rows) { + async _hydrate(rows) { if (!rows.length) return [] const requestIds = uniq(rows.map((r) => r.request_id)) const attempts = requestIds.length - ? this.db + ? await this.db .prepare( `SELECT * FROM request_attempts WHERE request_id IN (${placeholders(requestIds)}) ORDER BY request_id, attempt_no`, ) .all(...requestIds) : [] - const names = lookupNames(this.db, { + const names = await lookupNames(this.db, { userIds: rows.map((r) => r.user_id), keyIds: rows.map((r) => r.api_key_id), vmIds: [...rows.map((r) => r.vm_id), ...attempts.map((a) => a.vm_id)], @@ -489,11 +492,11 @@ export class UsageLogsView { * SQL-expressible `auth` class; otherwise aggregates the newest * SUMMARY_SCAN_CAP rows after JS classification. */ - summary(filters = {}) { + async summary(filters = {}) { const { where, params, keep, sqlOnlySummary } = buildFilter(filters, this.mutedErrorClasses) const cond = whereSql(where) if (sqlOnlySummary) { - const row = this.db + const row = await this.db .prepare(` SELECT COUNT(*) AS requests, COALESCE(SUM(CASE WHEN ${SUCCESS_PRED} THEN 1 ELSE 0 END), 0) AS success, @@ -514,11 +517,13 @@ export class UsageLogsView { .get(...params) return toSummary(row) } - const rows = this.db - .prepare( - `SELECT *, (${PROMPT_TOKENS_SQL}) AS prompt_tokens FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT ?`, - ) - .all(...params, SUMMARY_SCAN_CAP) + const rows = ( + await this.db + .prepare( + `SELECT *, (${PROMPT_TOKENS_SQL}) AS prompt_tokens FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT $${params.length + 1}`, + ) + .all(...params, SUMMARY_SCAN_CAP) + ) .map(enrichLogRow) .filter(keep) const acc = { @@ -553,7 +558,7 @@ export class UsageLogsView { 'cache_creation_1h_tokens', 'prompt_tokens', ]) - acc[k] += num(r[k]) + acc[k] = addInteger(acc[k], r[k]) if (r.duration_ms != null) { durSum += num(r.duration_ms) durN++ @@ -576,23 +581,20 @@ export class UsageLogsView { * most EXPORT_SCAN_CAP newest rows are read and `capped` says the scan ended * before the table did. */ - exportRows(filters = {}) { + async exportRows(filters = {}) { const limit = clampInt(filters.limit, 1, EXPORT_LIMIT_MAX, EXPORT_LIMIT_DEFAULT) const { where, params, keep, sqlOnlySummary } = buildFilter(filters, this.mutedErrorClasses) const cond = whereSql(where) - const select = `SELECT * FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT ?` + const select = `SELECT * FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT $${params.length + 1}` if (!keep || sqlOnlySummary) { - const total = this.db.prepare(`SELECT COUNT(*) AS c FROM usage_logs ${cond}`).get(...params).c + const total = (await this.db.prepare(`SELECT COUNT(*) AS c FROM usage_logs ${cond}`).get(...params)).c return { - items: this.db - .prepare(select) - .all(...params, limit) - .map(exportRow), + items: (await this.db.prepare(select).all(...params, limit)).map(exportRow), total, capped: false, } } - const scanned = this.db.prepare(select).all(...params, EXPORT_SCAN_CAP) + const scanned = await this.db.prepare(select).all(...params, EXPORT_SCAN_CAP) const matched = scanned.map(exportRow).filter(keep) return { items: matched.slice(0, limit), @@ -602,17 +604,18 @@ export class UsageLogsView { } /** Lazy dropdown values. Entity lists come from their tables; free-form columns from recent rows. */ - filterOptions({ owner_user_id = null } = {}) { + async filterOptions({ owner_user_id = null } = {}) { const own = ownerPred(owner_user_id) const recentWhere = own.sql ? `WHERE ${own.sql}` : '' const recent = `(SELECT * FROM usage_logs ${recentWhere} ORDER BY created_at DESC LIMIT ${RECENT_SCAN})` - const distinct = (col) => - this.db - .prepare( - `SELECT ${col} AS v FROM ${recent} WHERE ${col} IS NOT NULL AND ${col} != '' GROUP BY ${col} ORDER BY MAX(created_at) DESC LIMIT ${OPTION_LIMIT}`, - ) - .all(...own.params) - .map((r) => r.v) + const distinct = async (col) => + ( + await this.db + .prepare( + `SELECT ${col} AS v FROM ${recent} AS recent_rows WHERE ${col} IS NOT NULL ${col === 'status' ? '' : `AND ${col} != ''`} GROUP BY ${col} ORDER BY MAX(created_at) DESC LIMIT ${OPTION_LIMIT}`, + ) + .all(...own.params) + ).map((r) => r.v) const named = (ids, map) => ids.map((id) => ({ id: String(id), name: map.get(id) ?? String(id) })) let users @@ -620,105 +623,105 @@ export class UsageLogsView { let vmIds let accountIds if (owner_user_id) { - users = this.db.prepare('SELECT id, username, email FROM users WHERE id = ?').all(String(owner_user_id)) - keys = this.db - .prepare('SELECT id, name, user_id FROM api_keys WHERE user_id = ? AND deleted_at IS NULL ORDER BY name') + users = await this.db.prepare('SELECT id, username, email FROM users WHERE id = $1').all(String(owner_user_id)) + keys = await this.db + .prepare('SELECT id, name, user_id FROM api_keys WHERE user_id = $1 AND deleted_at IS NULL ORDER BY name') .all(String(owner_user_id)) // Pool slots that served this tenant appear in its rows even when not owned. vmIds = uniq([ - ...this.db - .prepare('SELECT id FROM vms WHERE owner_user_id = ?') - .all(String(owner_user_id)) - .map((r) => r.id), - ...distinct('vm_id'), + ...(await this.db.prepare('SELECT id FROM vms WHERE owner_user_id = $1').all(String(owner_user_id))).map( + (r) => r.id, + ), + ...(await distinct('vm_id')), ]) - accountIds = uniq([...distinct('final_account_id'), ...distinct('account_id')]) + accountIds = uniq([...(await distinct('final_account_id')), ...(await distinct('account_id'))]) } else { - users = this.db.prepare('SELECT id, username, email FROM users WHERE deleted_at IS NULL ORDER BY username').all() - keys = this.db.prepare('SELECT id, name, user_id FROM api_keys WHERE deleted_at IS NULL ORDER BY name').all() - vmIds = this.db - .prepare('SELECT id FROM vms ORDER BY id') + users = await this.db + .prepare('SELECT id, username, email FROM users WHERE deleted_at IS NULL ORDER BY username') .all() - .map((r) => r.id) - accountIds = this.db - .prepare('SELECT id FROM accounts WHERE deleted_at IS NULL ORDER BY id') + keys = await this.db + .prepare('SELECT id, name, user_id FROM api_keys WHERE deleted_at IS NULL ORDER BY name') .all() - .map((r) => r.id) + vmIds = (await this.db.prepare('SELECT id FROM vms ORDER BY id').all()).map((r) => r.id) + accountIds = (await this.db.prepare('SELECT id FROM accounts WHERE deleted_at IS NULL ORDER BY id').all()).map( + (r) => r.id, + ) } - const names = lookupNames(this.db, { vmIds, accountIds }) + const names = await lookupNames(this.db, { vmIds, accountIds }) return { - users: users.map((u) => ({ id: String(u.id), name: u.username || u.email || String(u.id) })), + users: await users.map((u) => ({ id: String(u.id), name: u.username || u.email || String(u.id) })), keys: keys.map((k) => ({ id: String(k.id), name: k.name || String(k.id), userId: k.user_id ?? null })), vms: named(vmIds, names.vms), accounts: named(accountIds, names.accounts), - models: uniq([...distinct('model'), ...distinct('requested_model')]).sort(), - protocols: distinct('protocol').sort(), - endpoints: distinct('path').sort(), - statusCodes: distinct('status') - .map(Number) - .sort((a, b) => a - b), + models: uniq([...(await distinct('model')), ...(await distinct('requested_model'))]).sort(), + protocols: (await distinct('protocol')).sort(), + endpoints: (await distinct('path')).sort(), + statusCodes: (await distinct('status')).map(Number).sort((a, b) => a - b), } } /** Outbound session ids starting with `q`, most recently used first (index range scan). */ - sessionSuggestions({ q = '', limit = 20, owner_user_id = null } = {}) { + async sessionSuggestions({ q = '', limit = 20, owner_user_id = null } = {}) { const n = clampInt(limit, 1, 50, 20) - const own = ownerPred(owner_user_id) + let own = ownerPred(owner_user_id) const prefix = String(q || '').trim() if (prefix) { - const where = ['outbound_session_id >= ?', 'outbound_session_id < ?'] + const where = ['outbound_session_id COLLATE "C" >= $1', 'outbound_session_id COLLATE "C" < $2'] const params = [prefix, `${prefix}\uffff`] + own = ownerPred(owner_user_id, params.length) if (own.sql) { where.push(own.sql) params.push(...own.params) } - return this.db - .prepare( - `SELECT outbound_session_id AS sid FROM usage_logs WHERE ${where.join(' AND ')} GROUP BY outbound_session_id ORDER BY MAX(created_at) DESC LIMIT ?`, - ) - .all(...params, n) - .map((r) => r.sid) + return ( + await this.db + .prepare( + `SELECT outbound_session_id AS sid FROM usage_logs WHERE ${where.join(' AND ')} GROUP BY outbound_session_id ORDER BY MAX(created_at) DESC LIMIT $${params.length + 1}`, + ) + .all(...params, n) + ).map((r) => r.sid) } const where = ['outbound_session_id IS NOT NULL'] if (own.sql) where.push(own.sql) - return this.db - .prepare( - `SELECT sid FROM (SELECT outbound_session_id AS sid, created_at FROM usage_logs WHERE ${where.join(' AND ')} ORDER BY created_at DESC LIMIT ${RECENT_SCAN}) GROUP BY sid ORDER BY MAX(created_at) DESC LIMIT ?`, - ) - .all(...own.params, n) - .map((r) => r.sid) + return ( + await this.db + .prepare( + `SELECT sid FROM (SELECT outbound_session_id AS sid, created_at FROM usage_logs WHERE ${where.join(' AND ')} ORDER BY created_at DESC LIMIT ${RECENT_SCAN}) AS recent_sessions GROUP BY sid ORDER BY MAX(created_at) DESC LIMIT $${own.params.length + 1}`, + ) + .all(...own.params, n) + ).map((r) => r.sid) } /** Outbound sessions with a completed request in the last `minutes`. */ - activeSessions({ minutes = 5, limit = 50, owner_user_id = null, now = Date.now() } = {}) { + async activeSessions({ minutes = 5, limit = 50, owner_user_id = null, now = Date.now() } = {}) { const mins = clampInt(minutes, 1, 1440, 5) const n = clampInt(limit, 1, 200, 50) - const where = ['created_at >= ?', 'outbound_session_id IS NOT NULL'] + const where = ['created_at >= $1', 'outbound_session_id IS NOT NULL'] const params = [new Date(now - mins * 60_000).toISOString()] - const own = ownerPred(owner_user_id) + const own = ownerPred(owner_user_id, params.length) if (own.sql) { where.push(own.sql) params.push(...own.params) } const cond = `WHERE ${where.join(' AND ')}` - const total = this.db - .prepare(`SELECT COUNT(DISTINCT outbound_session_id) AS n FROM usage_logs ${cond}`) - .get(...params).n + const total = ( + await this.db.prepare(`SELECT COUNT(DISTINCT outbound_session_id) AS n FROM usage_logs ${cond}`).get(...params) + ).n // Aggregate first, then pull the newest row per session by id; SQLite's // bare-column-with-MAX shortcut is undefined once MIN() is also present. - const rows = this.db + const rows = await this.db .prepare(` WITH s AS ( SELECT outbound_session_id AS sid, MAX(created_at) AS last_at, MIN(created_at) AS first_at, COUNT(*) AS requests, - SUM(IFNULL(input_tokens, 0) + IFNULL(output_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0)) AS tokens, + SUM(COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0) + COALESCE(cache_read_tokens, 0) + COALESCE(cache_creation_tokens, 0)) AS tokens, COALESCE(SUM(total_cost), 0) AS cost FROM usage_logs ${cond} GROUP BY outbound_session_id ORDER BY last_at DESC - LIMIT ? + LIMIT $${params.length + 1} ) SELECT s.*, l.session_id AS client_session_id, l.user_id, l.api_key_id, l.vm_id, l.model, l.status, l.duration_ms, COALESCE(l.final_account_id, l.account_id) AS account @@ -730,8 +733,8 @@ export class UsageLogsView { ) ORDER BY s.last_at DESC `) - .all(...params, n, ...own.params) - const names = lookupNames(this.db, { + .all(...params, n) + const names = await lookupNames(this.db, { userIds: rows.map((r) => r.user_id), keyIds: rows.map((r) => r.api_key_id), vmIds: rows.map((r) => r.vm_id), @@ -739,7 +742,7 @@ export class UsageLogsView { }) return { minutes: mins, - total: num(total), + total: integerValue(total), sessions: rows.map((r) => ({ sessionId: r.sid, clientSessionId: r.client_session_id ?? null, @@ -751,10 +754,10 @@ export class UsageLogsView { model: r.model ?? null, firstAt: r.first_at, lastAt: r.last_at, - requests: num(r.requests), + requests: integerValue(r.requests), lastStatus: nullableNum(r.status), lastDurationMs: nullableNum(r.duration_ms), - totalTokens: num(r.tokens), + totalTokens: integerValue(r.tokens), totalCost: num(r.cost), })), } @@ -764,20 +767,20 @@ export class UsageLogsView { * Header cards. "Today" = local midnight in `tz` → now; "yesterday" = the * same elapsed span starting at yesterday's local midnight. */ - overview({ tz = 'UTC', owner_user_id = null, now = Date.now() } = {}) { + async overview({ tz = 'UTC', owner_user_id = null, now = Date.now() } = {}) { const zone = reportTimezone(tz) - const own = ownerPred(owner_user_id) + const own = ownerPred(owner_user_id, 2) const todayStart = zonedDayStartMs(now, zone) const yesterdayStart = zonedDayStartMs(now, zone, -1) const yesterdayEnd = Math.min(todayStart, yesterdayStart + (now - todayStart)) - const agg = (sinceMs, untilMs) => { - const where = ['created_at >= ?', 'created_at < ?'] + const agg = async (sinceMs, untilMs) => { + const where = ['created_at >= $1', 'created_at < $2'] const params = [new Date(sinceMs).toISOString(), new Date(untilMs).toISOString()] if (own.sql) { where.push(own.sql) params.push(...own.params) } - return this.db + return await this.db .prepare(` SELECT COUNT(*) AS requests, COALESCE(SUM(CASE WHEN ${ERROR_PRED} THEN 1 ELSE 0 END), 0) AS errors, @@ -790,20 +793,20 @@ export class UsageLogsView { .get(...params) } // +1ms so a row stamped exactly `now` still counts. - const today = agg(todayStart, now + 1) - const yesterday = agg(yesterdayStart, yesterdayEnd) - const active = agg(now - 5 * 60_000, now + 1) - const lastMinute = agg(now - 60_000, now + 1) + const today = await agg(todayStart, now + 1) + const yesterday = await agg(yesterdayStart, yesterdayEnd) + const active = await agg(now - 5 * 60_000, now + 1) + const lastMinute = await agg(now - 60_000, now + 1) const avg = (v) => (v == null ? null : Math.round(Number(v))) return { - activeSessions: num(active.sessions), - rpm: num(lastMinute.requests), - todayRequests: num(today.requests), + activeSessions: integerValue(active.sessions), + rpm: integerValue(lastMinute.requests), + todayRequests: integerValue(today.requests), todayCost: num(today.cost), todayActualCost: num(today.actual_cost), - todayErrors: num(today.errors), + todayErrors: integerValue(today.errors), todayAvgDurationMs: avg(today.avg_duration_ms), - yesterdayRequests: num(yesterday.requests), + yesterdayRequests: integerValue(yesterday.requests), yesterdayCost: num(yesterday.cost), yesterdayAvgDurationMs: avg(yesterday.avg_duration_ms), } @@ -811,15 +814,15 @@ export class UsageLogsView { } function toSummary(row) { - const input = num(row.input_tokens) - const output = num(row.output_tokens) - const cacheCreation = num(row.cache_creation_tokens) - const cacheRead = num(row.cache_read_tokens) - const prompt = num(row.prompt_tokens) + const input = integerValue(row.input_tokens) + const output = integerValue(row.output_tokens) + const cacheCreation = integerValue(row.cache_creation_tokens) + const cacheRead = integerValue(row.cache_read_tokens) + const prompt = integerValue(row.prompt_tokens) return { - totalRequests: num(row.requests), - successRequests: num(row.success), - errorRequests: num(row.errors), + totalRequests: integerValue(row.requests), + successRequests: integerValue(row.success), + errorRequests: integerValue(row.errors), totalCost: num(row.cost), totalActualCost: num(row.actual_cost), totalTokens: rowTotalTokens(row), @@ -827,8 +830,8 @@ function toSummary(row) { totalOutputTokens: output, totalCacheCreationTokens: cacheCreation, totalCacheReadTokens: cacheRead, - totalCacheCreation5mTokens: num(row.cache_creation_5m_tokens), - totalCacheCreation1hTokens: num(row.cache_creation_1h_tokens), + totalCacheCreation5mTokens: integerValue(row.cache_creation_5m_tokens), + totalCacheCreation1hTokens: integerValue(row.cache_creation_1h_tokens), cacheHitRate: prompt > 0 ? Math.min(1, cacheRead / prompt) : 0, avgDurationMs: row.avg_duration_ms == null ? null : Math.round(Number(row.avg_duration_ms)), avgTtftMs: row.avg_ttft_ms == null ? null : Math.round(Number(row.avg_ttft_ms)), diff --git a/src/lib/db/repos/users-repo.mjs b/src/lib/db/repos/users-repo.mjs index c0b29ed90..48e98d57a 100644 --- a/src/lib/db/repos/users-repo.mjs +++ b/src/lib/db/repos/users-repo.mjs @@ -56,35 +56,44 @@ export class UsersRepo { constructor(db) { this.db = db this._list = db.prepare('SELECT * FROM users WHERE deleted_at IS NULL ORDER BY created_at, username') - this._get = db.prepare('SELECT * FROM users WHERE id = ? AND deleted_at IS NULL') - this._getByName = db.prepare('SELECT * FROM users WHERE username = ? COLLATE NOCASE AND deleted_at IS NULL') + this._get = db.prepare('SELECT * FROM users WHERE id = $1 AND deleted_at IS NULL') + this._getByName = db.prepare( + 'SELECT * FROM users WHERE ascii_fold(username) = ascii_fold($1) AND deleted_at IS NULL', + ) this._insert = db.prepare(` INSERT INTO users (${COLUMNS.join(', ')}) - VALUES (${COLUMNS.map(() => '?').join(', ')}) + VALUES (${COLUMNS.map((_, i) => '$' + (i + 1)).join(', ')}) `) - this._softDelete = db.prepare('UPDATE users SET deleted_at = ?, updated_at = ? WHERE id = ? AND deleted_at IS NULL') + this._softDelete = db.prepare( + 'UPDATE users SET deleted_at = $1, updated_at = $2 WHERE id = $3 AND deleted_at IS NULL', + ) this._countRole = db.prepare( - "SELECT COUNT(*) AS c FROM users WHERE role = ? AND status = 'active' AND deleted_at IS NULL", + "SELECT COUNT(*) AS c FROM users WHERE role = $1 AND status = 'active' AND deleted_at IS NULL", ) this._addBalance = db.prepare( - 'UPDATE users SET balance = balance + ?, updated_at = ? WHERE id = ? AND deleted_at IS NULL', + 'UPDATE users SET balance = balance + $1, updated_at = $2 WHERE id = $3 AND deleted_at IS NULL', ) } - list() { - return this._list.all().map(rowToRec) + async list() { + return (await this._list.all()).map(rowToRec) } - getById(id) { - return rowToRec(this._get.get(id)) + async getById(id, { forUpdate = false } = {}) { + return rowToRec( + await (forUpdate + ? this.db.prepare('SELECT * FROM users WHERE id = $1 AND deleted_at IS NULL FOR UPDATE') + : this._get + ).get(id), + ) } - getByUsername(username) { - return rowToRec(this._getByName.get(String(username || '').trim())) + async getByUsername(username) { + return rowToRec(await this._getByName.get(String(username || '').trim())) } - insert(rec) { - this._insert.run( + async insert(rec) { + await this._insert.run( ...COLUMNS.map((c) => { if (c === 'email') return rec.email ?? `${rec.username}@panel.local` if (c === 'status') return statusOf(rec) @@ -96,13 +105,13 @@ export class UsersRepo { return rec[c] ?? null }), ) - return this.getById(rec.id) + return await this.getById(rec.id) } - update(rec) { + async update(rec) { const cols = COLUMNS.filter((c) => c !== 'id' && c !== 'deleted_at') - const sql = `UPDATE users SET ${cols.map((c) => `${c} = ?`).join(', ')} WHERE id = ? AND deleted_at IS NULL` - this.db.prepare(sql).run( + const sql = `UPDATE users SET ${cols.map((c, i) => `${c} = $${i + 1}`).join(', ')} WHERE id = $${cols.length + 1} AND deleted_at IS NULL` + await this.db.prepare(sql).run( ...cols.map((c) => { if (c === 'email') return rec.email ?? `${rec.username}@panel.local` if (c === 'status') return statusOf(rec) @@ -114,23 +123,27 @@ export class UsersRepo { }), rec.id, ) - return this.getById(rec.id) + return await this.getById(rec.id) } /** Soft delete (sub2api SoftDeleteMixin semantics). */ - remove(id) { + async remove(id) { const now = new Date().toISOString() - return this._softDelete.run(now, now, id).changes > 0 + return (await this._softDelete.run(now, now, id)).changes > 0 } - countEnabledRole(role) { - const row = this._countRole.get(role) + async countEnabledRole(role) { + const row = await this._countRole.get(role) return Number(row?.c || 0) } + async touchLogin(id, at) { + await this.db.prepare('UPDATE users SET last_login_at = $1 WHERE id = $2 AND deleted_at IS NULL').run(at, id) + } + /** Atomic balance credit/debit (redeem, billing). Negative delta = charge. */ - addBalance(id, delta) { - this._addBalance.run(Number(delta) || 0, new Date().toISOString(), id) - return this.getById(id) + async addBalance(id, delta) { + await this._addBalance.run(Number(delta) || 0, new Date().toISOString(), id) + return await this.getById(id) } } diff --git a/src/lib/db/repos/vm-pools-repo.mjs b/src/lib/db/repos/vm-pools-repo.mjs index 45f85dd22..1932c11b7 100644 --- a/src/lib/db/repos/vm-pools-repo.mjs +++ b/src/lib/db/repos/vm-pools-repo.mjs @@ -25,105 +25,115 @@ export class VmPoolsRepo { constructor(db = getDb()) { this.db = db this._list = db.prepare('SELECT * FROM vm_pools ORDER BY created_at, id') - this._get = db.prepare('SELECT * FROM vm_pools WHERE id = ?') - this._members = db.prepare('SELECT vm_id FROM vm_pool_members WHERE pool_id = ? ORDER BY vm_id') + this._get = db.prepare('SELECT * FROM vm_pools WHERE id = $1') + this._members = db.prepare('SELECT vm_id FROM vm_pool_members WHERE pool_id = $1 ORDER BY vm_id') this._allMembers = db.prepare('SELECT pool_id, vm_id FROM vm_pool_members ORDER BY vm_id') - this._insert = db.prepare('INSERT INTO vm_pools (id, name, enabled, created_at, updated_at) VALUES (?, ?, ?, ?, ?)') - this._name = db.prepare('UPDATE vm_pools SET name = ?, updated_at = ? WHERE id = ?') - this._enabled = db.prepare('UPDATE vm_pools SET enabled = ?, updated_at = ? WHERE id = ?') - this._touch = db.prepare('UPDATE vm_pools SET updated_at = ? WHERE id = ?') - this._delete = db.prepare('DELETE FROM vm_pools WHERE id = ?') - this._clearMembers = db.prepare('DELETE FROM vm_pool_members WHERE pool_id = ?') - this._addMember = db.prepare('INSERT INTO vm_pool_members (pool_id, vm_id, created_at) VALUES (?, ?, ?)') - this._other = db.prepare('SELECT pool_id FROM vm_pool_members WHERE vm_id = ? AND pool_id != ?') - this._keyCount = db.prepare('SELECT COUNT(*) AS n FROM api_keys WHERE vm_pool_id = ? AND deleted_at IS NULL') + this._insert = db.prepare( + 'INSERT INTO vm_pools (id, name, enabled, created_at, updated_at) VALUES ($1, $2, $3, $4, $5)', + ) + this._name = db.prepare('UPDATE vm_pools SET name = $1, updated_at = $2 WHERE id = $3') + this._enabled = db.prepare('UPDATE vm_pools SET enabled = $1, updated_at = $2 WHERE id = $3') + this._touch = db.prepare('UPDATE vm_pools SET updated_at = $1 WHERE id = $2') + this._delete = db.prepare('DELETE FROM vm_pools WHERE id = $1') + this._clearMembers = db.prepare('DELETE FROM vm_pool_members WHERE pool_id = $1') + this._addMember = db.prepare('INSERT INTO vm_pool_members (pool_id, vm_id, created_at) VALUES ($1, $2, $3)') + this._other = db.prepare('SELECT pool_id FROM vm_pool_members WHERE vm_id = $1 AND pool_id != $2') + this._keyCount = db.prepare('SELECT COUNT(*) AS n FROM api_keys WHERE vm_pool_id = $1 AND deleted_at IS NULL') } - _ids(poolId) { - return this._members.all(poolId).map((row) => row.vm_id) + async _ids(poolId) { + return (await this._members.all(poolId)).map((row) => row.vm_id) } - list() { + async list() { const grouped = new Map() - for (const row of this._allMembers.all()) { + for (const row of await this._allMembers.all()) { const list = grouped.get(row.pool_id) || [] list.push(row.vm_id) grouped.set(row.pool_id, list) } - return this._list.all().map((row) => rowToPool(row, grouped.get(row.id) || [])) + return (await this._list.all()).map((row) => rowToPool(row, grouped.get(row.id) || [])) } - get(id) { - const row = this._get.get(id) + async get(id) { + const row = await this._get.get(id) if (!row) return null - return rowToPool(row, this._ids(id)) + return rowToPool(row, await this._ids(id)) } - countKeys(id) { - return Number(this._keyCount.get(id)?.n) || 0 + async countKeys(id) { + return Number((await this._keyCount.get(id))?.n) || 0 } - insert({ name, enabled = true }) { + async insert({ name, enabled = true }) { const now = nowIso() const id = `pool_${crypto.randomBytes(4).toString('hex')}` try { - this._insert.run(id, name, enabled ? 1 : 0, now, now) + await this._insert.run(id, name, enabled ? 1 : 0, now, now) } catch (error) { throw mapConstraint(error) } - return this.get(id) + return await this.get(id) } - rename(id, name) { + async rename(id, name) { try { - const info = this._name.run(name, nowIso(), id) + const info = await this._name.run(name, nowIso(), id) if (!info.changes) return null } catch (error) { throw mapConstraint(error) } - return this.get(id) + return await this.get(id) } - setEnabled(id, enabled) { - const info = this._enabled.run(enabled ? 1 : 0, nowIso(), id) + async setEnabled(id, enabled) { + const info = await this._enabled.run(enabled ? 1 : 0, nowIso(), id) if (!info.changes) return null - return this.get(id) + return await this.get(id) } /** * Replace membership. A vm already in another pool is rejected and nothing * is written. Passing the same vm ids is a no-op aside from updated_at. */ - setMembers(id, vmIds) { - if (!this._get.get(id)) return null - for (const vmId of vmIds) { - const other = this._other.get(vmId, id) - if (other) { - throw Object.assign(new Error(`槽位 ${vmId} 已在其他账号池`), { - code: 'vm_in_other_pool', - vm_id: vmId, - pool_id: other.pool_id, - }) + async setMembers(id, vmIds) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT id FROM vm_pools WHERE id = $1 FOR UPDATE').get(id) + + if (!(await this._get.get(id))) return null + const lockedIds = [...new Set([...vmIds, ...(await this._ids(id))])].sort() + for (const vmId of lockedIds) { + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(`vm_pool_member:${vmId}`) } - } - const now = nowIso() - withTransaction(this.db, () => { - this._clearMembers.run(id) - for (const vmId of vmIds) this._addMember.run(id, vmId, now) - this._touch.run(now, id) + for (const vmId of vmIds) { + const other = await this._other.get(vmId, id) + if (other) { + throw Object.assign(new Error(`槽位 ${vmId} 已在其他账号池`), { + code: 'vm_in_other_pool', + vm_id: vmId, + pool_id: other.pool_id, + }) + } + } + const now = nowIso() + await withTransaction(this.db, async () => { + await this._clearMembers.run(id) + for (const vmId of vmIds) await this._addMember.run(id, vmId, now) + await this._touch.run(now, id) + }) + return await this.get(id) }) - return this.get(id) } - remove(id) { - const info = this._delete.run(id) + async remove(id) { + const info = await this._delete.run(id) return info.changes > 0 } } function mapConstraint(error) { const message = String(error?.message || error) - if (/UNIQUE/i.test(message) && /vm_pools|name/i.test(message)) { + if (error?.code === '23505' && /vm_pools.*name|name.*vm_pools/i.test(error.constraint || message)) { return Object.assign(new Error('账号池名称已存在'), { code: 'vm_pool_name_taken' }) } return error diff --git a/src/lib/db/repos/vms-repo.mjs b/src/lib/db/repos/vms-repo.mjs index 675ba9f25..0539464bf 100644 --- a/src/lib/db/repos/vms-repo.mjs +++ b/src/lib/db/repos/vms-repo.mjs @@ -18,9 +18,9 @@ export class VmsRepo { constructor(db = getDb()) { this.db = db this.settings = new SettingsRepo(db) - this._get = db.prepare('SELECT * FROM vms WHERE id = ?') + this._get = db.prepare('SELECT * FROM vms WHERE id = $1') this._list = db.prepare('SELECT * FROM vms ORDER BY id') - this._remove = db.prepare('DELETE FROM vms WHERE id = ?') + this._remove = db.prepare('DELETE FROM vms WHERE id = $1') this._upsert = db.prepare(` INSERT INTO vms ( id, name, status, schedulable, email, account_uuid, org_uuid, @@ -28,7 +28,7 @@ export class VmsRepo { proxy_id, claude_code_version, timezone, locale, owner_user_id, origin, vm_json, encrypted, file_mtime_ms, synced_at, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24) ON CONFLICT(id) DO UPDATE SET name = excluded.name, status = excluded.status, schedulable = excluded.schedulable, email = excluded.email, account_uuid = excluded.account_uuid, org_uuid = excluded.org_uuid, @@ -44,7 +44,7 @@ export class VmsRepo { } /** Mirror a parsed vm.json object into the table. */ - upsertFromVmJson(vm, { mtimeMs = null } = {}) { + async upsertFromVmJson(vm, { mtimeMs = null } = {}) { if (!vm?.id) return null const now = new Date().toISOString() const c = vm.claude || {} @@ -55,7 +55,7 @@ export class VmsRepo { }) const safeVm = { ...vm, claude: safeClaude } const enc = encryptionEnabled() - this._upsert.run( + await this._upsert.run( vm.id, vm.name ?? null, vm.status ?? null, @@ -81,43 +81,40 @@ export class VmsRepo { vm.created_at ?? now, now, ) - return this.get(vm.id) + return await this.get(vm.id) } /** Returns {row, vm} — row has decrypted credential columns, vm is the parsed vm.json. */ - get(id) { - return this._decode(this._get.get(id)) + async get(id) { + return this._decode(await this._get.get(id)) } - list() { - return this._list.all().map((r) => this._decode(r)) + async list() { + return (await this._list.all()).map((r) => this._decode(r)) } - ids() { - return this.db - .prepare('SELECT id FROM vms ORDER BY id') - .all() - .map((r) => r.id) + async ids() { + return (await this.db.prepare('SELECT id FROM vms ORDER BY id').all()).map((r) => r.id) } - remove(id) { - return this._remove.run(id).changes > 0 + async remove(id) { + return (await this._remove.run(id)).changes > 0 } - count() { - return this.db.prepare('SELECT COUNT(*) c FROM vms').get().c + async count() { + return (await this.db.prepare('SELECT COUNT(*) c FROM vms').get()).c } - getActiveVmId() { - return this.settings.get(ACTIVE_VM_KEY, null) + async getActiveVmId() { + return await this.settings.get(ACTIVE_VM_KEY, null) } - setActiveVmId(id) { - this.settings.set(ACTIVE_VM_KEY, id) + async setActiveVmId(id) { + await this.settings.set(ACTIVE_VM_KEY, id) } - clearActiveVmId() { - this.settings.remove(ACTIVE_VM_KEY) + async clearActiveVmId() { + await this.settings.remove(ACTIVE_VM_KEY) } _decode(row) { diff --git a/src/lib/db/settings-state.mjs b/src/lib/db/settings-state.mjs new file mode 100644 index 000000000..da4771201 --- /dev/null +++ b/src/lib/db/settings-state.mjs @@ -0,0 +1,40 @@ +/** Explicitly loaded settings for synchronous policy callbacks. Writes remain asynchronous. */ +import { getDb } from './database.mjs' +import { SettingsRepo } from './repos/settings-repo.mjs' + +export async function loadSettingsState(db = getDb()) { + const settings = await new SettingsRepo(db).all() + db.settingsState = new Map(Object.entries(settings)) + return db.settingsState +} + +export class RuntimeSettings { + constructor(db = getDb()) { + this.db = db + this.repo = new SettingsRepo(db) + } + + get(key, fallback = null) { + if (!this.db.settingsState) throw new Error('settings not loaded; await loadSettingsState() before startup') + return this.db.settingsState.has(key) ? structuredClone(this.db.settingsState.get(key)) : fallback + } + + has(key) { + if (!this.db.settingsState) throw new Error('settings not loaded; await loadSettingsState() before startup') + return this.db.settingsState.has(key) + } + + async set(key, value) { + const next = structuredClone(value) + return this.repo.set(key, next) + } + + async remove(key) { + await this.repo.remove(key) + } + + all() { + if (!this.db.settingsState) throw new Error('settings not loaded; await loadSettingsState() before startup') + return Object.fromEntries([...this.db.settingsState].map(([key, value]) => [key, structuredClone(value)])) + } +} diff --git a/src/lib/db/sqlite-backup.mjs b/src/lib/db/sqlite-backup.mjs new file mode 100644 index 000000000..f947a8104 --- /dev/null +++ b/src/lib/db/sqlite-backup.mjs @@ -0,0 +1,54 @@ +/** Materialize old SQLite archives without extracting archive paths or symlinks. */ +import fs from 'node:fs' +import path from 'node:path' +import { spawn } from 'node:child_process' +import { pipeline } from 'node:stream/promises' +import { createGunzip } from 'node:zlib' +import { runProcess, sha256File } from '../admin/backup-core.mjs' + +async function memberToFile(archive, member, destination) { + fs.mkdirSync(path.dirname(destination), { recursive: true, mode: 0o700 }) + const child = spawn('tar', ['-xOzf', archive, '--', member], { stdio: ['ignore', 'pipe', 'ignore'] }) + const done = new Promise((resolve, reject) => { + child.once('error', () => reject(new Error('tar could not start'))) + child.once('close', (code) => (code === 0 ? resolve() : reject(new Error('legacy archive extraction failed')))) + }) + await Promise.all([pipeline(child.stdout, fs.createWriteStream(destination, { mode: 0o600, flags: 'wx' })), done]) +} +export async function materializeSqliteBackup(source, root) { + const database = path.join(root, 'db', 'kin.db') + fs.mkdirSync(path.dirname(database), { recursive: true, mode: 0o700 }) + if (source.endsWith('.db.gz')) { + await pipeline( + fs.createReadStream(source), + createGunzip(), + fs.createWriteStream(database, { mode: 0o600, flags: 'wx' }), + ) + return database + } + if (!source.endsWith('.tar.gz')) throw new Error('legacy backup must be .tar.gz or .db.gz') + const entries = (await runProcess('tar', ['-tzf', source])).split('\n').filter(Boolean) + const seen = new Set() + for (const entry of entries) { + const name = entry.replace(/^\.\//, '') + if (name === '.' || name === '') continue + if (/^(db|vms|config)\/$/.test(name)) { + fs.mkdirSync(path.join(root, name), { recursive: true, mode: 0o700 }) + continue + } + if (!/^(db\/kin\.db|manifest\.json|(vms|config)\/[A-Za-z0-9._-]+\.json)$/.test(name) || seen.has(name)) + throw new Error('legacy archive contains an unsupported path') + seen.add(name) + await memberToFile(source, entry, path.join(root, name)) + } + if (!fs.existsSync(database)) throw new Error('legacy archive has no SQLite snapshot') + const manifest = path.join(root, 'manifest.json') + if (fs.existsSync(manifest)) { + const info = JSON.parse(fs.readFileSync(manifest, 'utf8')) + for (const [name, hash] of Object.entries(info.sha256s || {})) { + if (!seen.has(name) || sha256File(path.join(root, name)) !== hash) + throw new Error('legacy archive checksum mismatch') + } + } + return database +} diff --git a/src/lib/db/sqlite-import.mjs b/src/lib/db/sqlite-import.mjs new file mode 100644 index 000000000..f8da35827 --- /dev/null +++ b/src/lib/db/sqlite-import.mjs @@ -0,0 +1,198 @@ +/** Offline, source-preserving SQLite -> PostgreSQL import. */ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import crypto from 'node:crypto' +import { DatabaseSync, backup } from 'node:sqlite' +import { applyMigrations as upgradeSqlite, createDatabase as createSqlite } from './sqlite-legacy.mjs' +import { withTransaction, quoteIdentifier as q } from './database.mjs' + +const FLAG = 'sqlite_import_done' +const sqliteMigrations = new URL('./migrations/', import.meta.url) +function tables(db) { + return db + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name") + .all() + .map((r) => r.name) +} +function columns(db, table) { + return db.prepare(`PRAGMA table_info(${q(table)})`).all() +} +function canonical(row, cols) { + return JSON.stringify( + cols.map((c) => { + const v = row[c.name] + if (v == null) return null + if (/INT/i.test(c.type)) return String(v) + if (v instanceof Uint8Array) return Buffer.from(v).toString('base64') + return v + }), + ) +} +function digestRows(rows, cols) { + const hashes = [] + for (const row of rows) hashes.push(crypto.createHash('sha256').update(canonical(row, cols)).digest('hex')) + hashes.sort() + const digest = crypto.createHash('sha256') + for (const hash of hashes) digest.update(hash) + return { count: hashes.length, sha256: digest.digest('hex') } +} + +export async function importSqlite({ source, db }) { + if (!source || !db) throw new Error('source and PostgreSQL database required') + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-sqlite-import-')) + let original, snapshot, template + try { + original = new DatabaseSync(path.resolve(source), { readOnly: true }) + const file = path.join(tmp, 'snapshot.db') + await backup(original, file) + original.close() + original = null + fs.chmodSync(file, 0o600) + const sourceHash = crypto.createHash('sha256') + for await (const chunk of fs.createReadStream(file)) sourceHash.update(chunk) + const sourceSha = sourceHash.digest('hex') + snapshot = new DatabaseSync(file) + if (!tables(snapshot).includes('schema_migrations')) + throw new Error('source has no recognized SQLite migration history') + const sourceVersions = snapshot.prepare('SELECT version,checksum FROM schema_migrations ORDER BY version').all() + const known = new Set( + fs + .readdirSync(sqliteMigrations) + .filter((f) => /^\d+.*\.sql$/.test(f)) + .map((f) => f.split('_')[0]), + ) + if (sourceVersions.some((r) => !known.has(r.version))) + throw new Error('source contains an unknown SQLite migration') + upgradeSqlite(snapshot) + template = createSqlite({ dbPath: path.join(tmp, 'template.db') }) + template.exec( + 'CREATE TABLE thinking_signature_owners(key TEXT PRIMARY KEY,account_id TEXT NOT NULL,created_at INTEGER NOT NULL)', + ) + const expected = tables(template).filter((t) => t !== 'schema_migrations') + const actual = tables(snapshot).filter((t) => t !== 'schema_migrations') + if (actual.some((t) => !expected.includes(t))) throw new Error('source contains an unmapped SQLite table') + if (!actual.includes('thinking_signature_owners')) + snapshot.exec( + 'CREATE TABLE thinking_signature_owners(key TEXT PRIMARY KEY,account_id TEXT NOT NULL,created_at INTEGER NOT NULL)', + ) + const metadata = new Map(expected.map((t) => [t, columns(template, t)])) + for (const table of expected) { + const sourceCols = columns(snapshot, table).map((c) => c.name) + if (JSON.stringify(sourceCols) !== JSON.stringify(metadata.get(table).map((c) => c.name))) + throw new Error(`source schema mismatch for ${table}`) + } + const sequenceStatement = snapshot.prepare('SELECT name,seq FROM sqlite_sequence') + sequenceStatement.setReadBigInts(true) + const sequences = new Map(sequenceStatement.all().map((r) => [r.name, BigInt(r.seq)])) + const report = await withTransaction(db, async () => { + await db.query('SELECT pg_advisory_xact_lock(hashtext(current_database()),hashtext($1))', [ + `vm2api:sqlite-import:${db.schema}`, + ]) + await db.exec(`LOCK TABLE ${expected.map(q).join(',')} IN ACCESS EXCLUSIVE MODE`) + for (const table of expected) { + const count = (await db.prepare(`SELECT count(*) AS n FROM ${q(table)}`).get()).n + if (table === 'groups') { + const rows = await db.prepare('SELECT * FROM groups ORDER BY id').all() + const seedCols = columns(template, 'groups') + const seedRows = template.prepare('SELECT * FROM groups ORDER BY id').all() + const seedNames = seedCols.filter((c) => !['created_at', 'updated_at'].includes(c.name)) + if (digestRows(rows, seedNames).sha256 !== digestRows(seedRows, seedNames).sha256) + throw new Error('target groups are not the unchanged baseline seeds') + } else if (Number(count) !== 0) throw new Error(`target is not empty (${table})`) + } + await db.exec('DELETE FROM groups') + const results = {} + // All baseline FKs are deferred, including self-referencing jump nodes. + for (const table of expected) { + const cols = metadata.get(table) + const statement = snapshot.prepare(`SELECT * FROM ${q(table)}`) + statement.setReadBigInts(true) + let batch = [] + const sourceHashes = [] + const flush = async () => { + if (!batch.length) return + const params = [] + const values = batch.map( + (row) => + '(' + + cols + .map((c) => { + const value = row[c.name] + params.push( + typeof value === 'bigint' + ? value.toString() + : value instanceof Uint8Array + ? Buffer.from(value) + : value, + ) + return '$' + params.length + }) + .join(',') + + ')', + ) + await db.query( + `INSERT INTO ${q(table)} (${cols.map((c) => q(c.name)).join(',')}) VALUES ${values.join(',')}`, + params, + ) + batch = [] + } + for (const row of statement.iterate()) { + sourceHashes.push(crypto.createHash('sha256').update(canonical(row, cols)).digest('hex')) + batch.push(row) + if (batch.length >= 100) await flush() + } + await flush() + const sourceDigest = crypto.createHash('sha256') + for (const hash of sourceHashes.sort()) sourceDigest.update(hash) + const before = { count: sourceHashes.length, sha256: sourceDigest.digest('hex') } + sourceHashes.length = 0 + const targetHashes = [] + await db.exec(`DECLARE import_validation NO SCROLL CURSOR FOR SELECT * FROM ${q(table)}`) + try { + for (;;) { + const batch = await db.prepare('FETCH FORWARD 100 FROM import_validation').all() + if (!batch.length) break + for (const row of batch) + targetHashes.push(crypto.createHash('sha256').update(canonical(row, cols)).digest('hex')) + } + } finally { + await db.exec('CLOSE import_validation') + } + const targetDigest = crypto.createHash('sha256') + for (const hash of targetHashes.sort()) targetDigest.update(hash) + const after = { count: targetHashes.length, sha256: targetDigest.digest('hex') } + if (before.count !== after.count || before.sha256 !== after.sha256) + throw new Error(`import validation failed for ${table}`) + results[table] = before + for (const col of cols) { + const seq = await db + .prepare( + `SELECT c.relname,n.nspname FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE c.oid=pg_get_serial_sequence($1,$2)::regclass`, + ) + .get(`${q(db.schema)}.${q(table)}`, col.name) + if (!seq) continue + const maximum = await db.prepare(`SELECT max(${q(col.name)}) AS n FROM ${q(table)}`).get() + const high = [sequences.get(table) || 0n, BigInt(maximum.n || 0)].reduce((a, b) => (a > b ? a : b)) + await db.exec(`ALTER SEQUENCE ${q(seq.nspname)}.${q(seq.relname)} RESTART WITH ${high + 1n}`) + } + } + const manifest = { + at: new Date().toISOString(), + source_sha256: sourceSha, + source_versions: sourceVersions, + tables: results, + } + await db + .prepare('INSERT INTO settings(key,value,updated_at) VALUES($1,$2,$3)') + .run(FLAG, JSON.stringify(manifest), manifest.at) + return manifest + }) + return report + } finally { + original?.close() + snapshot?.close() + template?.close() + fs.rmSync(tmp, { recursive: true, force: true }) + } +} diff --git a/src/lib/db/sqlite-legacy.mjs b/src/lib/db/sqlite-legacy.mjs new file mode 100644 index 000000000..d62ae1679 --- /dev/null +++ b/src/lib/db/sqlite-legacy.mjs @@ -0,0 +1,273 @@ +/** Offline SQLite compatibility for source snapshots and historical migration fixtures only. */ +/** + * SQLite database layer — sub2api data-layer inspired, on node:sqlite. + * + * - Single-process gateway → embedded SQLite (WAL) instead of sub2api's + * PostgreSQL+Redis. Same patterns: versioned SQL migrations with SHA-256 + * checksum verification (sub2api migrations_runner), repository modules, + * settings table, backup records. + * - Zero new dependencies: uses the built-in `node:sqlite` DatabaseSync. + * + * Env: + * KIN_DB_PATH — override db file path (default /kin.db) + */ + +import fs from 'node:fs' +import path from 'node:path' +import crypto from 'node:crypto' +import { fileURLToPath } from 'node:url' +import { DatabaseSync } from 'node:sqlite' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const MIGRATIONS_DIR = path.join(__dirname, 'migrations') + +function sha256Hex(s) { + return crypto.createHash('sha256').update(s).digest('hex') +} + +/** LF-canonical hash plus CRLF/raw variants so Windows-applied checksums still match. */ +function migrationChecksums(sql) { + const lf = sql.replace(/\r\n/g, '\n') + const crlf = lf.replace(/\n/g, '\r\n') + return { + canonical: sha256Hex(lf), + variants: new Set([sha256Hex(sql), sha256Hex(lf), sha256Hex(crlf)]), + } +} + +// 001 was published briefly before the proxy pool switched to its persisted +// record shape. Its checksum cannot be treated as the current 001: migration +// 018 assumes the newer columns and otherwise fails after accepting the hash. +const HISTORICAL_001_PROXY_CHECKSUMS = new Set([ + '376c686118bbd8aaa6db6ea13fb2ff0afecc0c7464bc3d668743a55c9ccafdde', + 'd6769ea311718b5af91e47295f97c3bf2722ece6d26acee7e89df84b88599ae2', +]) + +function reconcileHistorical001ProxySchema(db, checksum) { + if (!HISTORICAL_001_PROXY_CHECKSUMS.has(checksum)) return false + const columns = new Set( + db + .prepare('PRAGMA table_info(proxies)') + .all() + .map((row) => row.name), + ) + const current = ['raw', 'enabled', 'consecutive_failures', 'last_error'].every((name) => columns.has(name)) + if (current && !columns.has('url')) return true + const historical = ['url', 'failures', 'last_probe_ok', 'disabled_reason', 'updated_at'].every((name) => + columns.has(name), + ) + if (!historical || columns.has('raw')) { + throw new Error('historical migration 001 proxy schema is not recognized') + } + + withTransaction(db, () => { + db.exec(` + ALTER TABLE proxies RENAME TO proxies_historical_001; + CREATE TABLE proxies ( + id TEXT PRIMARY KEY, + scheme TEXT DEFAULT 'socks5', + host TEXT, + port INTEGER, + username TEXT, + password TEXT, + raw TEXT, + status TEXT DEFAULT 'unknown', + enabled INTEGER DEFAULT 1, + bound_vm_id TEXT, + consecutive_failures INTEGER DEFAULT 0, + latency_ms INTEGER, + last_probe_at TEXT, + last_error TEXT, + created_at TEXT + ); + INSERT INTO proxies ( + id, scheme, host, port, username, password, raw, status, enabled, + bound_vm_id, consecutive_failures, latency_ms, last_probe_at, + last_error, created_at + ) + SELECT + id, scheme, host, port, username, password, url, + CASE + WHEN status IN ('unknown', 'ok', 'dead') THEN status + WHEN last_probe_ok = 1 THEN 'ok' + WHEN last_probe_ok = 0 THEN 'dead' + ELSE 'unknown' + END, + CASE WHEN status = 'disabled' THEN 0 ELSE 1 END, + bound_vm_id, COALESCE(failures, 0), latency_ms, last_probe_at, + disabled_reason, created_at + FROM proxies_historical_001; + DROP TABLE proxies_historical_001; + `) + }) + return true +} +let _db = null +let _dbPath = null + +export function resolveDbPath({ dataDir, dbPath } = {}) { + if (dbPath) return dbPath + if (process.env.KIN_DB_PATH) return process.env.KIN_DB_PATH + const dir = dataDir || path.join(process.cwd(), 'data') + return path.join(dir, 'kin.db') +} + +/** + * Standalone (non-singleton) open: PRAGMAs + migrations applied. + * Used by unit tests and by stores constructed without a global db. + * @returns {DatabaseSync} + */ +export function createDatabase({ dataDir, dbPath, migrationsDir } = {}) { + const file = resolveDbPath({ dataDir, dbPath }) + fs.mkdirSync(path.dirname(file), { recursive: true }) + const db = new DatabaseSync(file) + try { + fs.chmodSync(file, 0o600) + } catch {} + db.exec('PRAGMA journal_mode = WAL') + db.exec('PRAGMA synchronous = NORMAL') + db.exec('PRAGMA busy_timeout = 5000') + db.exec('PRAGMA foreign_keys = ON') + applyMigrations(db, { migrationsDir }) + return db +} + +/** + * Open (or return the already-open) singleton database. + * @returns {DatabaseSync} + */ +export function openDatabase({ dataDir, dbPath, migrationsDir } = {}) { + if (_db) return _db + _db = createDatabase({ dataDir, dbPath, migrationsDir }) + _dbPath = resolveDbPath({ dataDir, dbPath }) + return _db +} + +/** + * Resolve the db a store should use: explicit db → global singleton → + * standalone connection rooted at dataDir (test isolation path). + */ +export function resolveStoreDb({ db, dataDir } = {}) { + if (db) return db + if (_db) return _db + return createDatabase({ dataDir }) +} + +/** Current db instance (throws when not opened). */ +export function getDb() { + if (!_db) throw new Error('database not opened — call openDatabase() first') + return _db +} + +export function isDbOpen() { + return !!_db +} + +export function getDbPath() { + return _dbPath +} + +export function closeDatabase() { + if (_db) { + try { + _db.close() + } catch {} + } + _db = null + _dbPath = null +} + +/** + * Versioned SQL migrations with checksum verification (sub2api-style). + * Files: migrations/NNN_name.sql, applied in filename order inside a + * transaction each. `schema_migrations` records version+checksum; a + * checksum mismatch on an already-applied migration is a hard error. + */ +export function applyMigrations(db, { migrationsDir } = {}) { + const dir = migrationsDir || MIGRATIONS_DIR + db.exec(`CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + name TEXT, + checksum TEXT, + applied_at TEXT + )`) + + const files = fs.existsSync(dir) + ? fs + .readdirSync(dir) + .filter((f) => /^\d+.*\.sql$/.test(f)) + .sort() + : [] + + const appliedRows = db.prepare('SELECT version, checksum FROM schema_migrations').all() + const applied = new Map(appliedRows.map((r) => [r.version, r.checksum])) + const historical001Accepted = reconcileHistorical001ProxySchema(db, applied.get('001')) + const insert = db.prepare('INSERT INTO schema_migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)') + + const results = [] + for (const file of files) { + const version = file.split('_')[0] + const sql = fs.readFileSync(path.join(dir, file), 'utf8') + const { canonical: checksum, variants } = migrationChecksums(sql) + if (applied.has(version)) { + const stored = applied.get(version) + const historicalAccepted = + version === '001' && historical001Accepted && HISTORICAL_001_PROXY_CHECKSUMS.has(stored) + if (!variants.has(stored) && !historicalAccepted) { + throw new Error(`migration checksum mismatch for ${file}: applied=${stored} current=${checksum}`) + } + if (stored !== checksum) { + db.prepare('UPDATE schema_migrations SET checksum = ? WHERE version = ?').run(checksum, version) + } + continue + } + db.exec('BEGIN') + try { + db.exec(sql) + insert.run(version, file, checksum, new Date().toISOString()) + db.exec('COMMIT') + } catch (e) { + try { + db.exec('ROLLBACK') + } catch {} + throw new Error(`migration ${file} failed: ${e.message}`) + } + results.push(file) + } + return results +} + +/** Run fn inside a transaction (nested calls just run inline). */ +export function withTransaction(db, fn) { + if (db.isTransaction) return fn() + db.exec('BEGIN') + try { + const out = fn() + db.exec('COMMIT') + return out + } catch (e) { + if (db.isTransaction) { + try { + db.exec('ROLLBACK') + } catch {} + } + throw e + } +} + +/** + * Online-consistent snapshot of the live database (WAL-safe). + * @returns {string} destination path + */ +export function vacuumInto(db, destPath) { + fs.mkdirSync(path.dirname(destPath), { recursive: true }) + try { + fs.rmSync(destPath, { force: true }) + } catch {} + const esc = String(destPath).replace(/'/g, "''") + db.exec(`VACUUM INTO '${esc}'`) + try { + fs.chmodSync(destPath, 0o600) + } catch {} + return destPath +} diff --git a/src/lib/oauth/credential-refresh-monitor.mjs b/src/lib/oauth/credential-refresh-monitor.mjs index b15b1b8d4..652a236dc 100644 --- a/src/lib/oauth/credential-refresh-monitor.mjs +++ b/src/lib/oauth/credential-refresh-monitor.mjs @@ -179,6 +179,9 @@ export function createCredentialRefreshMonitor(opts = {}) { getSnapshot: () => lastRun, runOnce, start, + flush: async () => { + while (inflight) await inflight + }, stop, } } diff --git a/src/lib/oauth/usage-probe-monitor.mjs b/src/lib/oauth/usage-probe-monitor.mjs index dddcff5a6..20b457629 100644 --- a/src/lib/oauth/usage-probe-monitor.mjs +++ b/src/lib/oauth/usage-probe-monitor.mjs @@ -366,6 +366,9 @@ export function createUsageProbeMonitor(opts = {}) { runOnce, probeNow, start, + flush: async () => { + while (inflight || probing.size) await Promise.allSettled([...(inflight ? [inflight] : []), ...probing.values()]) + }, stop, } } diff --git a/src/lib/pool/account-quota.mjs b/src/lib/pool/account-quota.mjs index 4f300be1d..e685e5e31 100644 --- a/src/lib/pool/account-quota.mjs +++ b/src/lib/pool/account-quota.mjs @@ -1,5 +1,5 @@ /** - * Per-account Claude usage quota tracker (SQLite-backed). + * Per-account Claude usage quota tracker backed by PostgreSQL. * * Scheduling and list Extra come from Messages response headers. * Official GET /api/oauth/usage is a manual calibration hop that writes @@ -10,7 +10,9 @@ */ import { resolveStoreDb } from '../db/database.mjs' +import { addInteger } from '../db/integers.mjs' import { AccountsRepo } from '../db/repos/accounts-repo.mjs' +import { WriteQueue } from './write-queue.mjs' import { computeWeeklySplit, weeklySplitConfig } from './weekly-split.mjs' import { applyEffectiveWindows, @@ -60,7 +62,12 @@ export { estimateRequestRisk } export class AccountQuota { constructor({ dataDir, db, config, accounts }) { this.db = resolveStoreDb({ db, dataDir }) - this.repo = new AccountsRepo(this.db) + this.persistence = new AccountsRepo(this.db) + this.writes = this.db.schedulingWriteQueue ||= new WriteQueue(this.db) + this.rows = new Map() + this.allocations = new Map() + this.seedAccounts = accounts || [] + this.repo = this._workingSet() this.config = config?.quota || { safety_ratio: 0.85, weekly_safety_ratio: 0.8, @@ -78,17 +85,68 @@ export class AccountQuota { this.sessions = config?.sessions instanceof SessionLimitRegistry ? config.sessions : new SessionLimitRegistry() // vm_id → normalized `vm.policy.quota`; vm.json is the source of truth, this is the hot-path copy. this.vmQuota = new Map() - // seed accounts - for (const a of accounts || []) this.ensure(a) } - /** Kept for API compat + post-restore hook (no in-memory cache to refresh). */ - reload() {} + /** Load persisted counters and allocations; restore reload omits startup seed accounts. */ + async init({ seed = true } = {}) { + for (const queue of this.db.pendingWriteQueues || []) await queue.flush() + for (const row of await this.persistence.list()) { + this.rows.set(row.account_id, row) + this.allocations.set(row.account_id, await this.persistence.recentAllocations(row.account_id, 50)) + } + if (seed) for (const account of this.seedAccounts) this.ensure(account) + await this.flush() + return this + } + + async reload() { + await this.flush() + this.rows.clear() + this.allocations.clear() + return await this.init({ seed: false }) + } + + flush() { + return this.writes.flush() + } + + _workingSet() { + const save = (account, insert = false) => { + const next = structuredClone(account) + this.rows.set(next.account_id, next) + this.writes.enqueue(async () => { + const existing = insert ? await this.persistence.get(next.account_id) : true + return this.persistence[existing ? 'save' : 'insert'](next) + }) + return structuredClone(next) + } + return { + get: (id) => structuredClone(this.rows.get(id) || null), + list: () => [...this.rows.values()].map((row) => structuredClone(row)), + insert: (row) => save(row, true), + save: (row) => save(row), + remove: (id) => { + const removed = this.rows.delete(id) + this.allocations.delete(id) + this.writes.enqueue(() => this.persistence.remove(id)) + return removed + }, + addAllocation: (id, input, { max = 50 } = {}) => { + const allocation = { ...input, at: input.at || new Date().toISOString() } + this.allocations.set(id, [...(this.allocations.get(id) || []), structuredClone(allocation)].slice(-max)) + this.writes.enqueue(() => this.persistence.addAllocation(id, allocation, { max })) + }, + recentAllocations: (id, limit = 5) => structuredClone((this.allocations.get(id) || []).slice(-limit)), + allocationCount: (id) => (this.allocations.get(id) || []).length, + } + } /** Re-bind to a fresh DB connection (after backup restore). */ - rebind(db) { + async rebind(db) { + await this.flush() this.db = db - this.repo = new AccountsRepo(db) + this.persistence = new AccountsRepo(db) + return await this.reload() } /** @@ -237,12 +295,12 @@ export class AccountQuota { usage.cache_read_tokens != null || usage.cache_creation_tokens != null) ) { - acc.tokens_in += Number(usage.input_tokens) || 0 - acc.tokens_out += Number(usage.output_tokens) || 0 - acc.cache_read_tokens = (acc.cache_read_tokens || 0) + (Number(usage.cache_read_tokens) || 0) - acc.cache_creation_tokens = (acc.cache_creation_tokens || 0) + (Number(usage.cache_creation_tokens) || 0) + acc.tokens_in = addInteger(acc.tokens_in, usage.input_tokens || 0) + acc.tokens_out = addInteger(acc.tokens_out, usage.output_tokens || 0) + acc.cache_read_tokens = addInteger(acc.cache_read_tokens, usage.cache_read_tokens || 0) + acc.cache_creation_tokens = addInteger(acc.cache_creation_tokens, usage.cache_creation_tokens || 0) } - if (countRequest) acc.requests += 1 + if (countRequest) acc.requests = addInteger(acc.requests, 1) if (exhausted) applyHeaderExhausted(acc, h) @@ -486,10 +544,10 @@ export class AccountQuota { const shouldCount = countRequest == null ? usageBody?.input_tokens != null : !!countRequest if (usageBody?.input_tokens != null) { - acc.tokens_in += Number(usageBody.input_tokens) || 0 - acc.tokens_out += Number(usageBody.output_tokens) || 0 + acc.tokens_in = addInteger(acc.tokens_in, usageBody.input_tokens || 0) + acc.tokens_out = addInteger(acc.tokens_out, usageBody.output_tokens || 0) } - if (shouldCount) acc.requests += 1 + if (shouldCount) acc.requests = addInteger(acc.requests, 1) this.repo.addAllocation(accountId, { at: new Date().toISOString(), diff --git a/src/lib/pool/api-protocol.mjs b/src/lib/pool/api-protocol.mjs index 1f4ca9a8a..047a7235c 100644 --- a/src/lib/pool/api-protocol.mjs +++ b/src/lib/pool/api-protocol.mjs @@ -148,9 +148,9 @@ export async function runApiInference({ const retry = Number(upstream.headers['retry-after'] || 30) const until = new Date(Date.now() + Math.max(1, retry) * 1000).toISOString() try { - store.setKeyCooldown(picked.key.id, until) + await store.setKeyCooldown(picked.key.id, until) } catch {} - scheduler.reload(store.listRaw()) + scheduler.reload(await store.listRaw()) } const resultBase = { diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index 4984830ec..df2068a86 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -537,7 +537,7 @@ export class FailoverRunner { async run(args = {}) { const sessionKey = String(args.stickyKey || '') - if (!sessionKey) return this.runOnce(args) + if (!sessionKey) return await this.runOnce(args) const previous = this.sessionTails.get(sessionKey) || Promise.resolve() let releaseTurn const turn = new Promise((resolve) => { @@ -789,7 +789,7 @@ export class FailoverRunner { if (selected.seatMoved) seatMovedTo = selected.vmId bindAll({ accountId: selected.accountId, vmId: selected.vmId, slotIndex: pinnedSlot }, { countHit: false }) const attemptStarted = Date.now() - this.attemptsRepo?.begin?.({ + await this.attemptsRepo?.begin?.({ requestId, attemptNo, vmId: selected.vmId, @@ -858,7 +858,7 @@ export class FailoverRunner { noteHop(result) if (result) result.committed = result.committed || committed if (signal?.aborted || isClientCancelledResult(result)) { - this.attemptsRepo?.complete?.(requestId, attemptNo, { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { upstreamStatus: result?.status ?? null, errorScope: 'request', action: 'stop', @@ -908,7 +908,7 @@ export class FailoverRunner { } catch {} } const terminalState = result?.terminalState || (result?.ok ? 'unknown' : 'error') - this.attemptsRepo?.complete?.(requestId, attemptNo, { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { upstreamStatus: result?.status ?? null, errorScope: policy.scope, action: policy.action, @@ -988,7 +988,7 @@ export class FailoverRunner { const invalidCache = error?.status === 400 && error.body?.error?.code === 'invalid_cache_control' noteHop(invalidCache ? { upstreamExecutions: 0 } : null) if (error?.code === 'response_sent') { - this.attemptsRepo?.complete?.(requestId, attemptNo, { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { upstreamStatus: 403, errorScope: 'request', action: 'stop', @@ -1008,7 +1008,7 @@ export class FailoverRunner { } } if (invalidCache) { - this.attemptsRepo?.complete?.(requestId, attemptNo, { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { upstreamStatus: 0, errorScope: 'request', action: 'stop', @@ -1026,7 +1026,7 @@ export class FailoverRunner { } } if (signal?.aborted || error?.code === 'selection_cancelled' || error?.code === 'request_cancelled') { - this.attemptsRepo?.complete?.(requestId, attemptNo, { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { upstreamStatus: 0, errorScope: 'request', action: 'stop', @@ -1066,7 +1066,7 @@ export class FailoverRunner { lastPolicy = policy this.noteUnitHealth(selected, policy, result) notifyProxyFailure(this.onProxyFailure, selected, policy) - this.attemptsRepo?.complete?.(requestId, attemptNo, { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { upstreamStatus: 0, errorScope: policy.scope, action: policy.action, diff --git a/src/lib/pool/scheduling-runtime.mjs b/src/lib/pool/scheduling-runtime.mjs new file mode 100644 index 000000000..db233fac5 --- /dev/null +++ b/src/lib/pool/scheduling-runtime.mjs @@ -0,0 +1,208 @@ +/** Loaded scheduling state keeps eligibility checks and reservations synchronous. */ +import { AccountRuntimeRepo } from '../db/repos/account-runtime-repo.mjs' +import { WriteQueue } from './write-queue.mjs' + +export class SchedulingRuntime { + constructor(db) { + this.persistence = new AccountRuntimeRepo(db) + this.rows = new Map() + this.writes = this.persistence.db.schedulingWriteQueue ||= new WriteQueue(this.persistence.db) + } + + async init() { + await this.flush() + this.rows = new Map((await this.persistence.list()).map((row) => [row.account_id, row])) + return this + } + + flush() { + return this.writes.flush() + } + + get(id) { + return structuredClone(this.rows.get(id) || null) + } + + list() { + return [...this.rows.values()].map((row) => structuredClone(row)) + } + + upsert(input = {}) { + if (!input.account_id || !input.vm_id) return null + const current = this.get(input.account_id) || {} + const next = { + cooldown_until: null, + cooldown_reason: null, + last_used_at: null, + credential_generation: 0, + refresh_status: null, + worker_heartbeat_at: null, + worker_status: null, + rate_limited_at: null, + rate_limit_reset_at: null, + overload_until: null, + session_window_start: null, + session_window_end: null, + session_window_status: null, + ...current, + ...structuredClone(input), + status: input.status ?? current.status ?? 'unknown', + priority: Number(input.priority ?? current.priority ?? 0), + weight: Math.max(0, Number(input.weight ?? current.weight ?? 1)), + model_states: structuredClone(input.model_states ?? current.model_states ?? {}), + updated_at: new Date().toISOString(), + } + this.rows.set(next.account_id, next) + this.writes.enqueue(() => this.persistence.upsert(next)) + return structuredClone(next) + } + + remove(id) { + const removed = this.rows.delete(id) + this.writes.enqueue(() => this.persistence.remove(id)) + return removed + } + + updateWindow( + accountId, + { + vmId = null, + rateLimitedAt, + rateLimitResetAt, + overloadUntil, + sessionWindowStart, + sessionWindowEnd, + sessionWindowStatus, + } = {}, + ) { + const current = this.get(accountId) || { + account_id: accountId, + vm_id: vmId, + model_states: {}, + } + if (!current.vm_id && vmId) current.vm_id = vmId + if (!current.vm_id) return null + if (rateLimitedAt !== undefined) current.rate_limited_at = rateLimitedAt + if (rateLimitResetAt !== undefined) current.rate_limit_reset_at = rateLimitResetAt + if (overloadUntil !== undefined) current.overload_until = overloadUntil + if (sessionWindowStart !== undefined) current.session_window_start = sessionWindowStart + if (sessionWindowEnd !== undefined) current.session_window_end = sessionWindowEnd + if (sessionWindowStatus !== undefined) current.session_window_status = sessionWindowStatus + return this.upsert(current) + } + + markCooldown(accountId, { vmId, until, reason, model = null, status = 'cooldown' } = {}) { + const current = this.get(accountId) || { + account_id: accountId, + vm_id: vmId, + model_states: {}, + } + if (model) { + current.model_states = { ...(current.model_states || {}) } + current.model_states[model] = { + status, + cooldown_until: Number(until) || null, + reason: reason || null, + updated_at: new Date().toISOString(), + } + } else { + current.status = status + current.cooldown_until = Number(until) || null + current.cooldown_reason = reason || null + // Structured mirror of the protocol-level limit (sub2api account columns). + // overload_until is only the 529 hard column (RateLimitService.handle529). + // A 502 failover also uses reason provider_overloaded for 15s; writing it + // here paints every hopped VM as 过载冷却 and removes them from the pool. + if (/rate_limited|quota_exhausted/i.test(String(reason || ''))) { + current.rate_limited_at = Date.now() + current.rate_limit_reset_at = Number(until) || null + } + } + return this.upsert(current) + } + + static AUTH_COOLDOWN = /authentication_failed_after_refresh|permission_denied/i + + static GRANT_REVOKE_COOLDOWN = + /oauth_revoked|oauth_invalid_grant|invalid_grant|token has been revoked|oauth_no_refresh/i + + isAuthCooldown(state) { + return SchedulingRuntime.AUTH_COOLDOWN.test(String(state?.cooldown_reason || '')) + } + + isGrantRevokeCooldown(state) { + return ( + SchedulingRuntime.GRANT_REVOKE_COOLDOWN.test(String(state?.cooldown_reason || '')) || + (String(state?.status || '') === 'disabled' && + SchedulingRuntime.GRANT_REVOKE_COOLDOWN.test(String(state?.cooldown_reason || state?.status || ''))) + ) + } + + clearAuthCooldown(accountId, { vmId = null } = {}) { + const current = this.get(accountId) + if (!current || !this.isAuthCooldown(current)) return false + return !!this.upsert({ + ...current, + vm_id: current.vm_id || vmId, + status: 'ready', + cooldown_until: null, + cooldown_reason: null, + }) + } + + clearGrantRevokeCooldown(accountId, { vmId = null } = {}) { + const current = this.get(accountId) + if (!current || !this.isGrantRevokeCooldown(current)) return false + return !!this.upsert({ + ...current, + vm_id: current.vm_id || vmId, + status: 'ready', + cooldown_until: null, + cooldown_reason: null, + }) + } + + clearAccountCooldown(accountId, { vmId = null } = {}) { + const current = this.get(accountId) + if (!current) return false + return !!this.upsert({ + ...current, + vm_id: current.vm_id || vmId, + status: 'ready', + cooldown_until: null, + cooldown_reason: null, + rate_limit_reset_at: null, + overload_until: null, + model_states: {}, + }) + } + + clearExpired(now = Date.now()) { + for (const state of this.list()) { + if (!state.vm_id) continue + let changed = false + if (state.cooldown_until && state.cooldown_until <= now) { + state.status = 'ready' + state.cooldown_until = null + state.cooldown_reason = null + changed = true + } + if (state.rate_limit_reset_at && state.rate_limit_reset_at <= now) { + state.rate_limit_reset_at = null + changed = true + } + if (state.overload_until && state.overload_until <= now) { + state.overload_until = null + changed = true + } + const models = { ...(state.model_states || {}) } + for (const [model, modelState] of Object.entries(models)) { + if (modelState?.cooldown_until && Number(modelState.cooldown_until) <= now) { + delete models[model] + changed = true + } + } + if (changed) this.upsert({ ...state, model_states: models }) + } + } +} diff --git a/src/lib/pool/sticky-router.mjs b/src/lib/pool/sticky-router.mjs index ed3dcbd5b..b4e5309a7 100644 --- a/src/lib/pool/sticky-router.mjs +++ b/src/lib/pool/sticky-router.mjs @@ -7,6 +7,7 @@ import crypto from 'node:crypto' import { ENVELOPE_NEEDLES, extractPrompt } from '../core/distill-detect.mjs' import { resolveStoreDb } from '../db/database.mjs' import { StickyRepo } from '../db/repos/sticky-repo.mjs' +import { WriteQueue } from './write-queue.mjs' import { claudeCodeAgentRootSession, extractCallerSession, @@ -204,18 +205,70 @@ export function childDeclaredWithoutParent(body = {}, headers = {}) { export class StickyRouter { constructor({ dataDir, db, config }) { this.db = resolveStoreDb({ db, dataDir }) - this.repo = new StickyRepo(this.db) + this.persistence = new StickyRepo(this.db) + this.writes = new WriteQueue(this.db) + this.sessions = {} this.config = mergeStickyConfig(config) - this.repo.dropAliasKeys() + this.repo = { + get: (key) => structuredClone(this.sessions[key] || null), + all: () => structuredClone(this.sessions), + upsert: (key, entry) => { + const next = { ...entry, device_id: entry.device_id || this.sessions[key]?.device_id || null } + this.sessions[key] = structuredClone(next) + this.writes.enqueue(() => this.persistence.upsert(key, next)) + }, + remove: (key) => { + delete this.sessions[key] + this.writes.enqueue(() => this.persistence.remove(key)) + }, + removeByAccount: ({ accountId = null, vmId = null } = {}) => { + let removed = 0 + for (const [key, entry] of Object.entries(this.sessions)) { + if ((accountId && entry.account_id === accountId) || (vmId && entry.vm_id === vmId)) { + delete this.sessions[key] + removed += 1 + } + } + if (removed) this.writes.enqueue(() => this.persistence.removeByAccount({ accountId, vmId })) + return removed + }, + purgeExpired: (now) => { + let removed = 0 + for (const [key, entry] of Object.entries(this.sessions)) { + if (entry.expires_at != null && entry.expires_at < now) { + delete this.sessions[key] + removed += 1 + } + } + if (removed) this.writes.enqueue(() => this.persistence.purgeExpired(now)) + return removed + }, + } } /** Kept for API compat + post-restore hook (state lives in DB). */ - reload() {} + async init() { + for (const queue of this.db.pendingWriteQueues || []) await queue.flush() + await this.persistence.dropAliasKeys() + this.sessions = await this.persistence.all() + return this + } + + async reload() { + await this.flush() + return await this.init() + } + + flush() { + return this.writes.flush() + } /** Re-bind to a fresh DB connection (after backup restore). */ - rebind(db) { + async rebind(db) { + await this.flush() this.db = db - this.repo = new StickyRepo(db) + this.persistence = new StickyRepo(db) + return await this.init() } isolateKey(raw, req = null) { diff --git a/src/lib/pool/thinking-owner.mjs b/src/lib/pool/thinking-owner.mjs index de6485fd9..4f9454945 100644 --- a/src/lib/pool/thinking-owner.mjs +++ b/src/lib/pool/thinking-owner.mjs @@ -9,8 +9,7 @@ * block with its thinking text, and is dropped when that text is empty. * * Key: sha256 of the signature's length, first 100 and last 100 characters - * (`data` for redacted_thinking). The table lives outside the versioned - * migrations so this local addition cannot take a migration number later. + * (`data` for redacted_thinking). The table is created by the PostgreSQL baseline migration. */ import crypto from 'node:crypto' import { getDb } from '../db/database.mjs' @@ -48,23 +47,15 @@ export class ThinkingOwnerStore { return null } if (this.boundDb === db) return this.sql - db.exec(`CREATE TABLE IF NOT EXISTS thinking_signature_owners ( - key TEXT PRIMARY KEY, - account_id TEXT NOT NULL, - created_at INTEGER NOT NULL - )`) - db.exec('CREATE INDEX IF NOT EXISTS idx_thinking_owner_created ON thinking_signature_owners(created_at)') this.sql = { - owners: db.prepare( - 'SELECT key, account_id FROM thinking_signature_owners WHERE key IN (SELECT value FROM json_each(?))', - ), + owners: db.prepare('SELECT key, account_id FROM thinking_signature_owners WHERE key = ANY($1::text[])'), refresh: db.prepare( - 'UPDATE thinking_signature_owners SET created_at = ? WHERE key IN (SELECT value FROM json_each(?)) AND created_at < ?', + 'UPDATE thinking_signature_owners SET created_at = $1 WHERE key = ANY($2::text[]) AND created_at < $3', ), insert: db.prepare( - 'INSERT OR IGNORE INTO thinking_signature_owners (key, account_id, created_at) VALUES (?, ?, ?)', + 'INSERT INTO thinking_signature_owners (key, account_id, created_at) VALUES ($1, $2, $3) ON CONFLICT(key) DO NOTHING', ), - purge: db.prepare('DELETE FROM thinking_signature_owners WHERE created_at < ?'), + purge: db.prepare('DELETE FROM thinking_signature_owners WHERE created_at < $1'), } this.boundDb = db return this.sql @@ -74,26 +65,26 @@ export class ThinkingOwnerStore { * key → account for the keys that have a record; null when no database is * reachable, so the caller can leave history alone instead of guessing. */ - ownersOf(keys, now = Date.now()) { + async ownersOf(keys, now = Date.now()) { const sql = this.statements() if (!sql) return null const found = new Map() if (!keys.length) return found - const list = JSON.stringify(keys) - for (const row of sql.owners.all(list)) found.set(row.key, row.account_id) - if (found.size) sql.refresh.run(now, list, now - REFRESH_AFTER_MS) + const list = keys + for (const row of await sql.owners.all(list)) found.set(row.key, row.account_id) + if (found.size) await sql.refresh.run(now, list, now - REFRESH_AFTER_MS) return found } /** First writer wins: a signature belongs to the account that returned it. */ - record(keys, accountId, now = Date.now()) { + async record(keys, accountId, now = Date.now()) { const account = String(accountId || '') const sql = keys.length && account ? this.statements() : null if (!sql) return - for (const key of keys) sql.insert.run(key, account, now) + for (const key of keys) await sql.insert.run(key, account, now) if (now - this.lastPurge >= PURGE_EVERY_MS) { this.lastPurge = now - sql.purge.run(now - this.ttlMs) + await sql.purge.run(now - this.ttlMs) } } } @@ -118,14 +109,14 @@ function historyKeys(body) { * database the body goes unchanged. * @returns {{ body: object, kept: number, converted: number, dropped: number, unknown: number }} */ -export function filterThinkingForAccount(body, accountId, store = thinkingOwners) { +export async function filterThinkingForAccount(body, accountId, store = thinkingOwners) { const account = String(accountId || '') const counts = { kept: 0, converted: 0, dropped: 0, unknown: 0 } const keys = account ? historyKeys(body) : [] if (!keys.length) return { body, ...counts } let owners = null try { - owners = store.ownersOf([...new Set(keys)]) + owners = await store.ownersOf([...new Set(keys)]) } catch {} if (!owners) return { body, ...counts, kept: keys.length } const out = mapThinkingBlocks(body, (block, message) => { @@ -150,14 +141,14 @@ export function filterThinkingForAccount(body, accountId, store = thinkingOwners } /** Bind the thinking blocks of a served response to the account that produced them. */ -export function recordThinkingOwners(message, accountId, store = thinkingOwners) { +export async function recordThinkingOwners(message, accountId, store = thinkingOwners) { const keys = [] for (const block of Array.isArray(message?.content) ? message.content : []) { const key = thinkingSignatureKey(block) if (key) keys.push(key) } try { - store.record(keys, accountId) + await store.record(keys, accountId) } catch {} return keys.length } diff --git a/src/lib/pool/write-queue.mjs b/src/lib/pool/write-queue.mjs new file mode 100644 index 000000000..a48176ebe --- /dev/null +++ b/src/lib/pool/write-queue.mjs @@ -0,0 +1,31 @@ +/** Ordered persistence for synchronous scheduling state. Flush surfaces any failed write. */ +export class WriteQueue { + constructor(db = null) { + if (db) { + db.pendingWriteQueues ||= new Set() + db.pendingWriteQueues.add(this) + } + this.pending = Promise.resolve() + this.failure = null + } + + enqueue(write) { + const operation = this.pending.then(() => { + if (this.failure) throw this.failure + return write() + }) + this.pending = operation.catch((error) => { + this.failure ||= error + }) + return this.pending + } + + async flush() { + let pending + do { + pending = this.pending + await pending + } while (pending !== this.pending) + if (this.failure) throw this.failure + } +} diff --git a/src/lib/protocol/gpt-model-policy.mjs b/src/lib/protocol/gpt-model-policy.mjs index 7e2c7450e..ac25de0c1 100644 --- a/src/lib/protocol/gpt-model-policy.mjs +++ b/src/lib/protocol/gpt-model-policy.mjs @@ -2,7 +2,7 @@ * GPT catalog. Separate settings key from Claude model_policy. * Live ids from ChatGPT sync; seed is empty-catalog fallback. */ -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { isGptSeriesId, isSyncableGptCatalogId } from './gpt-ids.mjs' const SETTINGS_KEY = 'gpt_model_policy' @@ -56,15 +56,15 @@ export function seedGptPolicy() { let policy = seedGptPolicy() let loaded = false -function persist() { +async function persist() { try { - new SettingsRepo().set(SETTINGS_KEY, policy) + await new SettingsRepo().set(SETTINGS_KEY, policy) } catch (e) { console.warn('[gpt-model-policy] persist failed', e?.message || e) } } -export function loadGptModelPolicy({ force = false } = {}) { +export async function loadGptModelPolicy({ force = false } = {}) { if (loaded && !force) return policy try { const stored = new SettingsRepo().get(SETTINGS_KEY, null) @@ -77,7 +77,7 @@ export function loadGptModelPolicy({ force = false } = {}) { policy.source = policy.source || 'settings' } else { policy = seedGptPolicy() - persist() + await persist() } } catch { policy = seedGptPolicy() @@ -87,11 +87,10 @@ export function loadGptModelPolicy({ force = false } = {}) { } export function getGptModelPolicy() { - if (!loaded) loadGptModelPolicy() return policy } -export function saveGptModelPolicy(next) { +export async function saveGptModelPolicy(next) { const seed = seedGptPolicy() const models = { ...(seed.models || {}) } if (next?.models && typeof next.models === 'object') { @@ -108,11 +107,10 @@ export function saveGptModelPolicy(next) { updated_at: new Date().toISOString(), source: 'panel', } - persist() + await persist() return policy } export function listGptPolicyModels() { - if (!loaded) loadGptModelPolicy() return Object.entries(policy.models || {}) .filter(([id, cfg]) => isGptSeriesId(id) && cfg?.enabled !== false) .map(([id, cfg]) => ({ id, ...cfg, family: 'codex' })) @@ -120,7 +118,6 @@ export function listGptPolicyModels() { } export function getGptPolicyCatalogIds({ enabledOnly = false } = {}) { - if (!loaded) loadGptModelPolicy() return Object.entries(policy.models || {}) .filter(([id, cfg]) => isGptSeriesId(id) && (!enabledOnly || cfg?.enabled !== false)) .map(([id]) => id) @@ -128,17 +125,16 @@ export function getGptPolicyCatalogIds({ enabledOnly = false } = {}) { export function isGptModelEnabled(id) { if (!isGptSeriesId(id)) return false - if (!loaded) loadGptModelPolicy() const row = policy.models?.[id] if (row) return row.enabled !== false if (policy.source === 'codex' && Object.keys(policy.models || {}).length) return false return true } -export function resetGptModelPolicy() { +export async function resetGptModelPolicy() { policy = seedGptPolicy() loaded = true - persist() + await persist() return policy } @@ -147,8 +143,7 @@ export function resetGptModelPolicy() { * a GPT id the policy has never seen starts disabled until the admin enables it * (GPT has no worker_only mode). */ -export function syncGptIdsIntoPolicy(ids = [], { displayNameOf = null } = {}) { - if (!loaded) loadGptModelPolicy() +export async function syncGptIdsIntoPolicy(ids = [], { displayNameOf = null } = {}) { const live = [] const seen = new Set() for (const item of ids || []) { @@ -182,7 +177,7 @@ export function syncGptIdsIntoPolicy(ids = [], { displayNameOf = null } = {}) { updated_at: new Date().toISOString(), source: 'codex', } - persist() + await persist() return { ok: true, synced: live.length, diff --git a/src/lib/protocol/handle-codex.mjs b/src/lib/protocol/handle-codex.mjs index 772964936..9b604d25c 100644 --- a/src/lib/protocol/handle-codex.mjs +++ b/src/lib/protocol/handle-codex.mjs @@ -204,7 +204,7 @@ function pinnedVmId(req) { * A request that continues a stored response (`previous_response_id`) can * only run where that response lives. */ -export function pickCodexCandidates( +export async function pickCodexCandidates( projectRoot, req, { stickyRouter = null, sessions = null, body = null, excluded = null, routing = {} } = {}, @@ -212,7 +212,7 @@ export function pickCodexCandidates( const pin = pinnedVmId(req) const model = body?.model || null const routingPolicy = normalizeCodexRouting(routing.codex || routing).quota - const scope = keyScopeFromRequest(req) + const scope = await keyScopeFromRequest(req) const poolDenial = vmPoolDenial(scope) if (poolDenial) return { error: 'vm_pool_unavailable', message: poolDenial.message, ids: [], candidates: [] } if (scope.group_type === 'anthropic') return { error: 'key_group_mismatch', ids: [], candidates: [] } @@ -402,7 +402,7 @@ function clientGoneSignal(req, res) { async function admitCodexCandidate(projectRoot, req, opts, { deadline, signal }) { let woken = false for (;;) { - const picked = pickCodexCandidates(projectRoot, req, opts) + const picked = await pickCodexCandidates(projectRoot, req, opts) const livePolicy = normalizeCodexRouting(opts.routing?.codex || opts.routing).quota if (picked.error && picked.error !== 'capacity_unavailable') return { picked } for (const candidate of picked.candidates || []) { @@ -739,8 +739,8 @@ export async function handleCodexProtocol({ if (captureOutbound) logBag.outbound_body = outboundBody logBag.outbound_headers = redactHeaders(outboundHeaders) let hopEmitted = false - const runHop = () => - runCodexKernelHop({ + const runHop = async () => + await runCodexKernelHop({ hop, args: { exec: execFor(projectRoot, vm), @@ -1034,7 +1034,7 @@ function searchCredentials(projectRoot, vm) { * refresh token on its own, so first adopt a token it wrote meanwhile; otherwise refresh * once per slot, shared by concurrent callers. */ -function refreshSearchAccess(projectRoot, vm, usedAccess, { proxyUrl, fetchImpl, refresh }) { +async function refreshSearchAccess(projectRoot, vm, usedAccess, { proxyUrl, fetchImpl, refresh }) { const now = searchCredentials(projectRoot, vm) if (now.access && now.access !== usedAccess) return now.access if (!now.refresh) return '' diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index 05da003e3..04978346e 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -1,3 +1,4 @@ +import { trackPendingWrite } from '../db/pending-writes.mjs' /** * /v1 protocol handler. URLs, auth, and response envelopes stay with the * server wiring; this factory owns convert → pool → Go/Rust hop → client. @@ -16,7 +17,7 @@ import { } from '../core/refusal-guard.mjs' import { RefusalGuardsRepo } from '../db/repos/refusal-guards-repo.mjs' import { RefusalDeviceBlocksRepo } from '../db/repos/refusal-device-blocks-repo.mjs' -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { findAccount } from '../admin/panel-api.mjs' import { jevScopeWaits, readJevConfig, scopeAccountFromVm } from './jev-intercept.mjs' import { runProtocolIntercept } from './intercept-gate.mjs' @@ -171,7 +172,7 @@ export function createHandleProtocol(deps) { const json = (...args) => deps.json(...args) const writeSSEHeaders = (...args) => deps.writeSSEHeaders(...args) const readBody = (...args) => deps.readBody(...args) - const requireAuth = (...args) => deps.requireAuth(...args) + const requireAuth = async (...args) => await deps.requireAuth(...args) const cfg = deps.cfg const requestLog = deps.requestLog const stickyRouter = deps.stickyRouter @@ -198,13 +199,13 @@ export function createHandleProtocol(deps) { return disjointOpenaiUsage(next) } - function attachPricing(logBag, { managedKey = null, rateMultiplier = 1, bill = false } = {}) { + async function attachPricing(logBag, { managedKey = null, rateMultiplier = 1, bill = false } = {}) { const model = logBag.upstream_model || logBag.model const priced = costColumnsFromUsage(logBag.usage || {}, model) Object.assign(logBag, priced) if (!bill || !managedKey) return try { - apiKeyStore.recordUsage(managedKey, logBag.usage || {}, { + await apiKeyStore.recordUsage(managedKey, logBag.usage || {}, { model, rateMultiplier, officialCost: priced.total_cost, @@ -370,10 +371,10 @@ export function createHandleProtocol(deps) { } } - function banRefusalDevice(deviceId, { requestId, fingerprint, reason }) { + async function banRefusalDevice(deviceId, { requestId, fingerprint, reason }) { if (!deviceId) return try { - deviceRepo()?.block?.({ deviceId, requestId, fingerprint, reason }) + await deviceRepo()?.block?.({ deviceId, requestId, fingerprint, reason }) } catch { /* the prompt block still stands */ } @@ -388,7 +389,7 @@ export function createHandleProtocol(deps) { } } - function rememberRefusal({ inbound, body, headers, result, logBag, requestId }) { + async function rememberRefusal({ inbound, body, headers, result, logBag, requestId }) { if (!refusalEnabled()) return const contentRefusal = isUpstreamRefusal(result, logBag) const timed = result?.policy?.rememberRefusal === true @@ -398,7 +399,7 @@ export function createHandleProtocol(deps) { const fingerprint = refusalFingerprint(body, inbound) try { const ttl = Number(result?.policy?.refusalTtlMs) || 0 - repo.remember({ + await repo.remember({ fingerprint, model: body?.model || inbound?.model || '', requestId, @@ -411,7 +412,7 @@ export function createHandleProtocol(deps) { return } if (!contentRefusal || !refusalPolicy().device_block_enabled) return - banRefusalDevice(inboundRefusalDeviceId({ inbound, body, headers }), { + await banRefusalDevice(inboundRefusalDeviceId({ inbound, body, headers }), { requestId, fingerprint, reason: 'refusal_guard', @@ -451,7 +452,7 @@ export function createHandleProtocol(deps) { requestContext, slotWaitMs: candidate.slotWaitMs, noGoFallback, - ensureCredential: (exec) => ensureWorkerCredential(exec), + ensureCredential: async (exec) => await ensureWorkerCredential(exec), onEvent: async (line) => { if (/response_headers/.test(String(line))) return applyClaudeSSELineToMessage(restoreToolNamesInSSELine(line, toolNames), assembler) @@ -500,21 +501,27 @@ export function createHandleProtocol(deps) { reasoning_effort: null, outbound_session_id: null, } - res.on('finish', () => { - try { - const groupId = req.apiKeyRecord?.group_id ?? 1 - requestLog.finish(logCtx, { - status: res.statusCode || 0, - api_key_kind: req.apiKeyKind || null, - api_key_id: req.apiKeyRecord?.id || null, - user_id: req.apiKeyRecord?.user_id ?? null, - group_id: groupId, - rate_multiplier: deps.groupsRepo.rateMultiplier(groupId), - ...logBag, - }) - } catch {} - }) - if (!requireAuth(req, res)) { + res.on('finish', () => + trackPendingWrite( + (async () => { + try { + const groupId = req.apiKeyRecord?.group_id ?? 1 + await requestLog.finish(logCtx, { + status: res.statusCode || 0, + api_key_kind: req.apiKeyKind || null, + api_key_id: req.apiKeyRecord?.id || null, + user_id: req.apiKeyRecord?.user_id ?? null, + group_id: groupId, + rate_multiplier: await deps.groupsRepo.rateMultiplier(groupId), + ...logBag, + }) + } catch (error) { + throw error + } + })(), + ), + ) + if (!(await requireAuth(req, res))) { logBag.error_code = req.authError?.code || ErrorCode.INVALID_API_KEY logBag.error_message = req.authError?.message || 'Invalid credentials' logBag.api_key_presented = presentedApiKeyForLog(req.presentedApiKey) @@ -676,7 +683,7 @@ export function createHandleProtocol(deps) { logBag.error_message = errorResult.body?.error?.message || null return json(res, errorResult.status, errorResult.body) } - const keyScope = keyScopeFromRequest(req) + const keyScope = await keyScopeFromRequest(req) const poolDenial = vmPoolDenial(keyScope) if (poolDenial) { stats.errors++ @@ -740,8 +747,8 @@ export function createHandleProtocol(deps) { } // Account scope needs the slot, so the Codex hop runs the same gate again // once a candidate is claimed. Without a scope the pre-hop verdict stands. - const accountGate = (account) => - applyProtocolIntercept({ + const accountGate = async (account) => + await applyProtocolIntercept({ req, inbound, body: ctx.body, @@ -753,7 +760,7 @@ export function createHandleProtocol(deps) { }) if (protocol === 'anthropic.messages') { const codex = normalizeCodexRouting(routing.codex) - return handleCodexProtocol({ + return await handleCodexProtocol({ req, res, protocol, @@ -781,7 +788,7 @@ export function createHandleProtocol(deps) { ...codexSticky, }) } - return handleCodexProtocol({ + return await handleCodexProtocol({ req, res, protocol, @@ -1023,7 +1030,7 @@ export function createHandleProtocol(deps) { if (inferenceBackend === 'api') { const managedKey = req.apiKeyRecord || null if (managedKey) { - const gate = apiKeyStore.acquire(managedKey) + const gate = await apiKeyStore.acquire(managedKey) if (!gate.ok) { stats.errors++ return json( @@ -1099,9 +1106,9 @@ export function createHandleProtocol(deps) { ? Math.max(0, result.firstTokenAt - logCtx.t0) : (result?.ttftMs ?? null) logBag.final_state = result?.terminalState || null - attachPricing(logBag, { + await attachPricing(logBag, { managedKey, - rateMultiplier: deps.groupsRepo.rateMultiplier(managedKey?.group_id ?? 1), + rateMultiplier: await deps.groupsRepo.rateMultiplier(managedKey?.group_id ?? 1), bill: !!(result?.ok && managedKey), }) if (clientStream) { @@ -1140,7 +1147,7 @@ export function createHandleProtocol(deps) { const streamIdleTimeoutMs = resolveStreamIdleTimeoutMs(getRouting(), cfg) const managedKey = req.apiKeyRecord || null if (managedKey) { - const gate = apiKeyStore.acquire(managedKey) + const gate = await apiKeyStore.acquire(managedKey) if (!gate.ok) { stats.errors++ return json( @@ -1193,7 +1200,7 @@ export function createHandleProtocol(deps) { const pinVmRaw = String(req.headers['x-kin-vm'] || '').trim() const pinVmId = req.apiKeyKind === 'master' && isValidVmId(pinVmRaw) ? pinVmRaw : null // Pin is panel test-chat / diagnostics (manage). Unpinned /v1 is dispatch. - const ownerScope = pinVmId ? { type: 'any' } : ownerScopeFromRequest(req, apiKeyStore?.users) + const ownerScope = pinVmId ? { type: 'any' } : await ownerScopeFromRequest(req, apiKeyStore?.users) // Cache bypass is a client hint; only authenticated master-key probes may skip accounting. const healthReal = req.apiKeyKind === 'master' && isHealthRealBypass(req.headers) // Cache lives per account; a failover to another account starts cold by design. @@ -1235,7 +1242,7 @@ export function createHandleProtocol(deps) { applyAttempt: async (body, selected, extra = {}) => { // Only this account's own thinking replays as thinking. Another // account's signature, or one with no record, goes out as text. - const owned = filterThinkingForAccount(body, selected.accountId, thinkingOwnerStore) + const owned = await filterThinkingForAccount(body, selected.accountId, thinkingOwnerStore) body = owned.body if (owned.kept || owned.converted || owned.dropped) { const { kept, converted, dropped, unknown } = owned @@ -1455,7 +1462,7 @@ export function createHandleProtocol(deps) { }, callAttempt: async ({ candidate, body, attemptMeta, deliveryMode: attemptDelivery, signal, onCommit }) => { if (!clientStream) { - return streamAndAssembleClaudeMessage({ + return await streamAndAssembleClaudeMessage({ candidate, body, reqHeaders: req.headers, @@ -1516,7 +1523,7 @@ export function createHandleProtocol(deps) { routing: getRouting(), slotWaitMs: candidate.slotWaitMs, noGoFallback: !!pinVmId, - ensureCredential: (exec) => ensureWorkerCredential(exec), + ensureCredential: async (exec) => await ensureWorkerCredential(exec), onCommit: () => { if (protocol === 'anthropic.messages' && !res.headersSent) writeSSEHeaders(res) onCommit() @@ -1580,7 +1587,7 @@ export function createHandleProtocol(deps) { logBag.final_account_id = result?.accountId || null if (result?.threadBetas) logBag.hop_meta = { ...logBag.hop_meta, thread_betas: result.threadBetas } if (result?.accountId && (result.ok || result.committed)) - recordThinkingOwners(result.body, result.accountId, thinkingOwnerStore) + await recordThinkingOwners(result.body, result.accountId, thinkingOwnerStore) // Zero executions is a real 0, not unknown; the VM is the last one that ran. logBag.attempt_count = result?.attemptCount ?? 0 logBag.final_state = result?.finalState || result?.terminalState || null @@ -1600,7 +1607,14 @@ export function createHandleProtocol(deps) { logBag.error_code = logBag.error_code || 'content_filter_refusal' logBag.error_message = logBag.error_message || 'upstream stop_reason=refusal' } - rememberRefusal({ inbound, body: ctx.body, headers: req.headers, result, logBag, requestId: logCtx.request_id }) + await rememberRefusal({ + inbound, + body: ctx.body, + headers: req.headers, + result, + logBag, + requestId: logCtx.request_id, + }) if (result?.accountId) { try { @@ -1633,9 +1647,9 @@ export function createHandleProtocol(deps) { if (vm) pool.syncQuotaSchedule(vm) } } catch {} - attachPricing(logBag, { + await attachPricing(logBag, { managedKey, - rateMultiplier: deps.groupsRepo.rateMultiplier(managedKey?.group_id ?? 1), + rateMultiplier: await deps.groupsRepo.rateMultiplier(managedKey?.group_id ?? 1), bill: !!managedKey, }) } @@ -1668,7 +1682,14 @@ export function createHandleProtocol(deps) { res.write(buildAnthropicSseError(mapped.body?.error || {})) } } - rememberRefusal({ inbound, body: ctx.body, headers: req.headers, result, logBag, requestId: logCtx.request_id }) + await rememberRefusal({ + inbound, + body: ctx.body, + headers: req.headers, + result, + logBag, + requestId: logCtx.request_id, + }) return res.end() } @@ -1724,21 +1745,27 @@ export function createHandleProtocol(deps) { attempt_count: null, final_state: null, } - res.on('finish', () => { - try { - const groupId = req.apiKeyRecord?.group_id ?? 1 - requestLog.finish(logCtx, { - status: res.statusCode || 0, - api_key_kind: req.apiKeyKind || null, - api_key_id: req.apiKeyRecord?.id || null, - user_id: req.apiKeyRecord?.user_id ?? null, - group_id: groupId, - rate_multiplier: deps.groupsRepo.rateMultiplier(groupId), - ...logBag, - }) - } catch {} - }) - if (!requireAuth(req, res)) { + res.on('finish', () => + trackPendingWrite( + (async () => { + try { + const groupId = req.apiKeyRecord?.group_id ?? 1 + await requestLog.finish(logCtx, { + status: res.statusCode || 0, + api_key_kind: req.apiKeyKind || null, + api_key_id: req.apiKeyRecord?.id || null, + user_id: req.apiKeyRecord?.user_id ?? null, + group_id: groupId, + rate_multiplier: await deps.groupsRepo.rateMultiplier(groupId), + ...logBag, + }) + } catch (error) { + throw error + } + })(), + ), + ) + if (!(await requireAuth(req, res))) { logBag.error_code = req.authError?.code || ErrorCode.INVALID_API_KEY logBag.error_message = req.authError?.message || 'Invalid credentials' logBag.api_key_presented = presentedApiKeyForLog(req.presentedApiKey) @@ -1765,7 +1792,7 @@ export function createHandleProtocol(deps) { } logBag.model = typeof body?.model === 'string' ? body.model : null logBag.inbound_summary = summarizeBody(body) - return handleCodexSearch({ + return await handleCodexSearch({ req, res, body, diff --git a/src/lib/protocol/intercept-gate.mjs b/src/lib/protocol/intercept-gate.mjs index f4eb7e304..b99fa023b 100644 --- a/src/lib/protocol/intercept-gate.mjs +++ b/src/lib/protocol/intercept-gate.mjs @@ -62,11 +62,11 @@ function similarityRatio(value) { return n > 1 ? n / 100 : n } -export function commitProtocolBlock(decision, { repo, devices, policy, requestId } = {}) { +export async function commitProtocolBlock(decision, { repo, devices, policy, requestId } = {}) { if (!decision || decision.action !== 'block') return if (decision.kind === 'device') { try { - devices?.hit?.(decision.deviceId) + await devices?.hit?.(decision.deviceId) } catch { /* counter is best-effort */ } @@ -74,14 +74,14 @@ export function commitProtocolBlock(decision, { repo, devices, policy, requestId } if ((decision.kind === 'exact' || decision.kind === 'similar') && decision.fingerprint) { try { - repo?.hit?.(decision.fingerprint) + await repo?.hit?.(decision.fingerprint) } catch { /* counter is best-effort */ } } if (decision.remember && policy?.enabled && repo && decision.fingerprint) { try { - repo.remember?.({ + await repo.remember?.({ fingerprint: decision.fingerprint, model: decision.model || '', requestId: requestId || null, @@ -96,7 +96,7 @@ export function commitProtocolBlock(decision, { repo, devices, policy, requestId } if (!decision.banDevice || !policy?.enabled || !policy?.device_block_enabled || !decision.deviceId) return try { - deviceStore(devices)?.block?.({ + await deviceStore(devices)?.block?.({ deviceId: decision.deviceId, requestId: requestId || null, fingerprint: decision.fingerprint || null, @@ -233,7 +233,7 @@ export async function evaluateProtocolIntercept({ } if (policy?.enabled && repo && typeof repo.get === 'function') { - let match = matchStoredRefusal({ + let match = await matchStoredRefusal({ inbound, body, headers, @@ -317,6 +317,6 @@ export async function evaluateProtocolIntercept({ export async function runProtocolIntercept(ctx = {}) { const decision = await evaluateProtocolIntercept(ctx) if (decision.action !== 'block') return decision - commitProtocolBlock(decision, ctx) + await commitProtocolBlock(decision, ctx) return decision } diff --git a/src/lib/protocol/jev-intercept.mjs b/src/lib/protocol/jev-intercept.mjs index fa6365dde..1a014666f 100644 --- a/src/lib/protocol/jev-intercept.mjs +++ b/src/lib/protocol/jev-intercept.mjs @@ -921,10 +921,10 @@ export function validateJevPatch(body = {}) { } /** Apply a panel PUT. Omitted keys stay. An empty api_key, or api_keys: [], clears them. */ -export function applyJevPatch(settings, body = {}) { +export async function applyJevPatch(settings, body = {}) { const problems = validateJevPatch(body) if (problems.length) return { ok: false, problems } - const current = normalizeJevConfig(settings.get(JEV_SETTING, null)) + const current = normalizeJevConfig(await settings.get(JEV_SETTING, null)) const next = normalizeJevConfig({ ...current, ...body, @@ -938,7 +938,7 @@ export function applyJevPatch(settings, body = {}) { patterns: Object.hasOwn(body, 'patterns') ? body.patterns : current.patterns, rules: Object.hasOwn(body, 'rules') ? body.rules : current.rules, }) - settings.set(JEV_SETTING, next) + await settings.set(JEV_SETTING, next) if (Object.hasOwn(body, 'api_keys') || Object.hasOwn(body, 'api_key')) rearmJevKeys(next) return { ok: true, config: publicJevConfig(next) } } diff --git a/src/lib/protocol/model-policy.mjs b/src/lib/protocol/model-policy.mjs index ca0287809..72f8c5cb4 100644 --- a/src/lib/protocol/model-policy.mjs +++ b/src/lib/protocol/model-policy.mjs @@ -8,7 +8,7 @@ * worker_only — ignore policy enabled flags for listing */ -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { CONTEXT_1M_BETA, DEFAULT_CONTEXT_1M_WHITELIST, @@ -537,18 +537,18 @@ function heuristicCapabilities(modelId = '') { return { ...CAP_46 } } -export function loadModelPolicy({ force = false } = {}) { +export async function loadModelPolicy({ force = false } = {}) { if (loaded && !force) return policy try { const repo = new SettingsRepo() const stored = repo.get(SETTINGS_KEY, null) if (stored) { - const persisted = persistHaiku55Model(repo, persistOpus55Model(repo, stored)) + const persisted = await persistHaiku55Model(repo, await persistOpus55Model(repo, stored)) policy = normalizePolicy(persisted) policy.source = policy.source || 'settings' } else { policy = seedDefaultPolicy() - repo.set(SETTINGS_KEY, policy) + await repo.set(SETTINGS_KEY, policy) } } catch { policy = seedDefaultPolicy() @@ -558,7 +558,7 @@ export function loadModelPolicy({ force = false } = {}) { } /** Append Opus 5.5 onto an existing settings row. Does not reset other models. */ -function persistOpus55Model(repo, stored) { +async function persistOpus55Model(repo, stored) { if (!stored || typeof stored !== 'object') return stored const seed = seedDefaultPolicy() const models = stored.models && typeof stored.models === 'object' ? { ...stored.models } : {} @@ -580,7 +580,7 @@ function persistOpus55Model(repo, stored) { if (!dirty) return stored const next = { ...stored, models, aliases } try { - repo.set(SETTINGS_KEY, next) + await repo.set(SETTINGS_KEY, next) } catch { return next } @@ -588,7 +588,7 @@ function persistOpus55Model(repo, stored) { } /** Append or migrate Haiku 5.5 onto an existing settings row. Does not reset other models. */ -function persistHaiku55Model(repo, stored) { +async function persistHaiku55Model(repo, stored) { if (!stored || typeof stored !== 'object') return stored const seed = seedDefaultPolicy() const desired = seed.models[HAIKU_55_ID] @@ -617,7 +617,7 @@ function persistHaiku55Model(repo, stored) { if (!dirty) return stored const next = { ...stored, models } try { - repo.set(SETTINGS_KEY, next) + await repo.set(SETTINGS_KEY, next) } catch { return next } @@ -625,17 +625,16 @@ function persistHaiku55Model(repo, stored) { } export function getModelPolicy() { - if (!loaded) loadModelPolicy() return policy } -export function saveModelPolicy(next) { +export async function saveModelPolicy(next) { policy = normalizePolicy(next) policy.updated_at = new Date().toISOString() policy.source = 'panel' try { const repo = new SettingsRepo() - repo.set(SETTINGS_KEY, policy) + await repo.set(SETTINGS_KEY, policy) } catch (e) { console.warn('[model-policy] persist failed', e?.message || e) } @@ -643,18 +642,17 @@ export function saveModelPolicy(next) { return policy } -export function resetModelPolicy() { +export async function resetModelPolicy() { policy = seedDefaultPolicy() try { const repo = new SettingsRepo() - repo.set(SETTINGS_KEY, policy) + await repo.set(SETTINGS_KEY, policy) } catch {} loaded = true return policy } export function resolvePolicyModelId(raw = '') { - if (!loaded) loadModelPolicy() const m = String(raw || '').trim() if (!m) return null const bare = m.split('/').filter(Boolean).pop() || m @@ -678,7 +676,6 @@ export function resolvePolicyModelId(raw = '') { } export function getModelEntry(modelId = '') { - if (!loaded) loadModelPolicy() const key = resolvePolicyModelId(modelId) if (key && policy.models[key]) return { id: key, ...policy.models[key] } const id = key || String(modelId || '') @@ -720,7 +717,6 @@ export function getBetaPolicy(modelId = '') { } export function getContext1mWhitelist() { - if (!loaded) loadModelPolicy() return normalizeContext1mWhitelist(policy.defaults?.context_1m_whitelist, DEFAULT_CONTEXT_1M_WHITELIST) } @@ -738,7 +734,6 @@ export function resolveContext1mPass( } export function shouldPassContext1mByPolicy(modelId = '') { - if (!loaded) loadModelPolicy() return resolveContext1mPass(modelId, { entry: getModelEntry(modelId), whitelist: getContext1mWhitelist(), @@ -775,7 +770,6 @@ function adaptiveOnlyThinking(thinking) { } export function normalizeThinkingByPolicy(body = {}) { - if (!loaded) loadModelPolicy() if (!body || typeof body !== 'object') return body if (policy.defaults.normalize_thinking === false) return body const thinking = body.thinking @@ -912,7 +906,6 @@ export function applyBetaPolicyToHeader( modelId = '', { isOfficial = false, whitelist, want1m = false } = {}, ) { - if (!loaded) loadModelPolicy() const pass = whitelist !== undefined ? resolveContext1mPass(modelId, { @@ -946,14 +939,12 @@ export function unofficialMimicryBetaHeader(modelId = '') { /** All policy model ids that look like Claude catalog ids (disabled included). */ export function getPolicyCatalogIds({ enabledOnly = false } = {}) { - if (!loaded) loadModelPolicy() return Object.entries(policy.models || {}) .filter(([id, cfg]) => isCatalogModelId(id) && (!enabledOnly || cfg.enabled !== false)) .map(([id]) => id) } export function filterPublicModelIds(workerIds = []) { - if (!loaded) loadModelPolicy() const mode = policy.catalog_mode || 'policy_only' const ids = [ ...new Set( @@ -972,7 +963,6 @@ export function filterPublicModelIds(workerIds = []) { } export function listPolicyModels() { - if (!loaded) loadModelPolicy() return Object.entries(policy.models) .map(([id, cfg]) => ({ id, ...cfg })) .sort((a, b) => (a.sort ?? 100) - (b.sort ?? 100) || a.id.localeCompare(b.id)) @@ -983,8 +973,7 @@ export function listPolicyModels() { * public only under worker_only; other modes add it disabled for the admin to enable. * `displayNameOf(id)` may supply a display name (models.dev) for new entries. */ -export function syncWorkerModelsIntoPolicy(workerIds = [], { displayNameOf = null } = {}) { - if (!loaded) loadModelPolicy() +export async function syncWorkerModelsIntoPolicy(workerIds = [], { displayNameOf = null } = {}) { const added = [] const enabled = policy.catalog_mode === 'worker_only' for (const workerId of workerIds || []) { @@ -1021,7 +1010,7 @@ export function syncWorkerModelsIntoPolicy(workerIds = [], { displayNameOf = nul policy.updated_at = new Date().toISOString() policy.source = 'worker_merge' try { - new SettingsRepo().set(SETTINGS_KEY, policy) + await new SettingsRepo().set(SETTINGS_KEY, policy) } catch {} } return { policy, added } @@ -1032,7 +1021,6 @@ export const FALLBACK_MAX_TOKENS = 4096 /** Official per-model default for a missing caller max_tokens. Never overwrites inbound. */ export function defaultMaxTokensForModel(modelId = '') { - if (!loaded) loadModelPolicy() const params = getModelParams(modelId) const n = Number(params.max_tokens_default) const cap = Number(params.max_tokens_cap) diff --git a/src/lib/protocol/models.mjs b/src/lib/protocol/models.mjs index 5dd98c189..cf1e8b7d2 100644 --- a/src/lib/protocol/models.mjs +++ b/src/lib/protocol/models.mjs @@ -10,7 +10,6 @@ import { getModelEntry, getModelPolicy, getPolicyCatalogIds, - loadModelPolicy, seedDefaultPolicy, } from './model-policy.mjs' import { hasClaudeCode1mSuffix, stripClaudeCode1mSuffix } from './context-1m.mjs' @@ -35,7 +34,6 @@ export function seedModelCatalog() { if (catalogIsTestPinned()) return cache let ids = [...SEED_MODEL_IDS] try { - loadModelPolicy() const fromPolicy = getPolicyCatalogIds() if (fromPolicy.length) ids = fromPolicy } catch {} @@ -65,9 +63,6 @@ function listGptPublicModels() { */ export function gatewayModelCatalog({ groupType = 'all' } = {}) { seedModelCatalog() - try { - loadModelPolicy() - } catch {} const gpt = groupType === 'anthropic' ? [] : listGptPublicModels() const claude = groupType === 'openai' ? [] : listOfficialModels() return { @@ -142,7 +137,6 @@ export function latestIdForUndatedAlias(raw, ids) { function resolvePolicyAliasToCatalog(raw, ids) { try { - loadModelPolicy() const policy = getModelPolicy() const lower = String(raw || '').toLowerCase() const candidates = [] @@ -234,9 +228,6 @@ export function clearModelsCache() { } export function listOfficialModels() { - try { - loadModelPolicy() - } catch {} const ids = filterPublicModelIds(cache.ids || []) return ids.map((id) => { let display = id diff --git a/src/lib/protocol/responses-ws.mjs b/src/lib/protocol/responses-ws.mjs index 24d046140..f0b44a2ce 100644 --- a/src/lib/protocol/responses-ws.mjs +++ b/src/lib/protocol/responses-ws.mjs @@ -637,7 +637,7 @@ export function createResponsesWsHandler({ } } - ws.on('message', (data, isBinary) => { + ws.on('message', async (data, isBinary) => { if (closed) return if (isBinary) { closed = true @@ -724,7 +724,7 @@ export function createResponsesWsHandler({ }) } - function handleUpgrade(req, socket, head) { + async function handleUpgrade(req, socket, head) { let pathname = '/v1/responses' try { pathname = new URL(req.url || '/', 'http://local').pathname @@ -735,7 +735,7 @@ export function createResponsesWsHandler({ const routing = typeof getRouting === 'function' ? getRouting() || {} : {} if (rejectNewWs(socket, routing)) return true const res = handshakeRes(socket) - if (typeof requireAuth === 'function' && !requireAuth(req, res)) return true + if (typeof requireAuth === 'function' && !(await requireAuth(req, res))) return true wss.handleUpgrade(req, socket, head, (ws) => attach(ws, req)) return true } diff --git a/src/lib/protocol/user-count-tokens.mjs b/src/lib/protocol/user-count-tokens.mjs index 51e25dad7..516d946f2 100644 --- a/src/lib/protocol/user-count-tokens.mjs +++ b/src/lib/protocol/user-count-tokens.mjs @@ -6,7 +6,7 @@ import { makeError, mapUpstreamError, rewritePoolErrorForClient, ErrorType, Erro import { refusalGuardPolicy } from '../core/refusal-guard.mjs' import { RefusalGuardsRepo } from '../db/repos/refusal-guards-repo.mjs' import { RefusalDeviceBlocksRepo } from '../db/repos/refusal-device-blocks-repo.mjs' -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { findAccount } from '../admin/panel-api.mjs' import { readJevConfig, scopeAccountFromVm } from './jev-intercept.mjs' import { runProtocolIntercept } from './intercept-gate.mjs' @@ -115,7 +115,7 @@ export async function peekCurrentAccount({ if (!poolScheduler?.peekAccount) { return { ok: false, code: 'no_eligible_accounts' } } - const keyScope = keyScopeFromRequest(req) + const keyScope = await keyScopeFromRequest(req) const poolDenial = vmPoolDenial(keyScope) if (poolDenial) return { ok: false, code: ErrorCode.VM_POOL_UNAVAILABLE, message: poolDenial.message } // Same gate as Messages: a platform-scoped key must not hop to the other platform's VMs. @@ -126,7 +126,7 @@ export async function peekCurrentAccount({ model, stickyKey, signal, - ownerScope: ownerScopeFromRequest(req, usersRepo), + ownerScope: await ownerScopeFromRequest(req, usersRepo), keyScope, }) } @@ -245,7 +245,7 @@ export async function blockCountTokensBeforeHop(req, inbound, deps = {}) { export async function handleUserCountTokens(req, res, deps) { const json = (...args) => deps.json(...args) - if (!deps.requireAuth(req, res)) return + if (!(await deps.requireAuth(req, res))) return let inbound try { inbound = await deps.readBody(req, inboundBodyLimit(deps.getRouting?.(), deps.cfg.limits.max_body_bytes)) @@ -372,7 +372,7 @@ export async function resolveUsageWindows({ export async function handleUserUsage(req, res, deps) { const json = (...args) => deps.json(...args) - if (!deps.requireAuth(req, res)) return + if (!(await deps.requireAuth(req, res))) return const peeked = await peekCurrentAccount({ poolScheduler: typeof deps.getPoolScheduler === 'function' ? deps.getPoolScheduler() : deps.poolScheduler, stickyRouter: deps.stickyRouter, diff --git a/src/lib/transport/kernel-watchdog.mjs b/src/lib/transport/kernel-watchdog.mjs index d5d38a5be..6fe2888bc 100644 --- a/src/lib/transport/kernel-watchdog.mjs +++ b/src/lib/transport/kernel-watchdog.mjs @@ -332,6 +332,9 @@ export function createKernelWatchdog({ timer = setInterval(() => void tick(), config.interval_sec * 1000) timer.unref?.() }, + async flush() { + while (inTick) await new Promise((resolve) => setTimeout(resolve, 10)) + }, stop() { running = false clearInterval(timer) diff --git a/src/lib/vm/cli-node-guard.mjs b/src/lib/vm/cli-node-guard.mjs index 2dc1371e1..d747773fb 100644 --- a/src/lib/vm/cli-node-guard.mjs +++ b/src/lib/vm/cli-node-guard.mjs @@ -113,6 +113,9 @@ export function createCliNodeGuard({ timer = setInterval(() => void tick(), intervalMs) timer.unref?.() }, + async flush() { + while (inTick) await new Promise((resolve) => setTimeout(resolve, 10)) + }, stop() { if (timer) clearInterval(timer) timer = null diff --git a/src/lib/vm/egress.mjs b/src/lib/vm/egress.mjs index 9334c88a3..546b15a31 100644 --- a/src/lib/vm/egress.mjs +++ b/src/lib/vm/egress.mjs @@ -9,7 +9,7 @@ import { execFileSync, spawn } from 'node:child_process' import fs from 'node:fs' import path from 'node:path' import { getDb, isDbOpen } from '../db/database.mjs' -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { socksProxyUrl } from './socks-address.mjs' import { assertProxyAllowed, proxyBlockedReason } from './proxy-policy.mjs' @@ -556,7 +556,7 @@ async function stopOwnedEgress(pid) { } catch (error) { if (error.code !== 'ESRCH') return { ok: false, error: `egress_stop_failed: ${error.code || error.message}` } } - return waitEgressExit(pid) + return await waitEgressExit(pid) } export function inspectEgressProcess(projectRoot, proxyId) { @@ -653,7 +653,7 @@ export async function waitEgressExit( try { process.kill(pid, 'SIGKILL') } catch (error) { - if (error.code === 'ESRCH') return waitEgressReap(pid, killGraceMs) + if (error.code === 'ESRCH') return await waitEgressReap(pid, killGraceMs) return { ok: false, error: `egress_kill_failed: ${error.code || error.message}`, pid } } const reaped = await waitEgressReap(pid, killGraceMs) @@ -783,7 +783,11 @@ export async function runExclusiveProxyEgress(projectRoot, proxyId, fn) { export async function ensureProxyEgress(projectRoot, proxy, opts = {}) { if (isLocalEgressProxy(proxy)) return ensureLocalProxyEgress(proxy, { runDocker: opts.runDocker }) - return runExclusiveProxyEgress(projectRoot, proxy?.id, () => startProxyEgress(projectRoot, proxy, opts)) + return await runExclusiveProxyEgress( + projectRoot, + proxy?.id, + async () => await startProxyEgress(projectRoot, proxy, opts), + ) } export async function startProxyEgress( diff --git a/src/lib/vm/proxy-policy.mjs b/src/lib/vm/proxy-policy.mjs index 850b690e6..7a6aaba98 100644 --- a/src/lib/vm/proxy-policy.mjs +++ b/src/lib/vm/proxy-policy.mjs @@ -1,5 +1,5 @@ import { getDb, isDbOpen } from '../db/database.mjs' -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { socksProxyFamily } from './socks-address.mjs' export function configuredIpv6Enabled() { diff --git a/src/lib/vm/proxy-pool.mjs b/src/lib/vm/proxy-pool.mjs index 003d3f076..a3c465113 100644 --- a/src/lib/vm/proxy-pool.mjs +++ b/src/lib/vm/proxy-pool.mjs @@ -14,6 +14,7 @@ import net from 'node:net' import crypto from 'node:crypto' import { resolveStoreDb } from '../db/database.mjs' import { ProxiesRepo } from '../db/repos/proxies-repo.mjs' +import { WriteQueue } from '../pool/write-queue.mjs' import { canBindProxyToVm, normalizeOwnerId, proxyOwnerId, vmOwnerId } from '../admin/resource-owner.mjs' import { validTimezone } from '../core/timezone.mjs' import { lookupProxyGeo, lookupProxyGeoV6 } from './proxy-geo.mjs' @@ -341,43 +342,53 @@ export class ProxyPool extends EventEmitter { this._probeSockets = new Map() // Node exits are not pool rows: px-local's geo is the control plane's, a node slot leaves elsewhere. this._exitGeo = new Map() - this.load() + this.writes = new WriteQueue(this.db) + this.db.pendingWriteQueues.add(this) } /** Re-read working set from DB (startup + post-restore). */ - load() { - try { - this.state.config = { ...DEFAULT_CONFIG, ...(this.repo.getConfig({}) || {}) } - this.state.proxies = this.repo.loadAll().map(hydrateProxy) - } catch { - this.state = { config: { ...DEFAULT_CONFIG }, proxies: [] } - } + async load() { + for (const queue of this.db.pendingWriteQueues || []) await queue.flush() + await this.flush() + this.state.config = { ...DEFAULT_CONFIG, ...((await this.repo.getConfig({})) || {}) } + this.state.proxies = (await this.repo.loadAll()).map(hydrateProxy) + return this + } + + async init() { + return await this.load() } - reload() { - this.load() + async reload() { + return await this.load() } - /** Re-bind to a fresh DB connection (after backup restore) and re-read. */ - rebind(db) { + async rebind(db) { + await this.flush() this.db = db this.repo = new ProxiesRepo(db) - this.load() + return await this.load() + } + + async flush() { + while (this._probing) await new Promise((resolve) => setTimeout(resolve, 10)) + await this.writes.flush() } save() { - // Batched rewrites (local-row migration) persist once at the end. if (this._saveDeferred) { this._savePending = true return } - this.repo.setConfig(this.state.config) - this.repo.replaceAll( - this.state.proxies.map((p) => ({ - ...p, - bound_vm_id: encodeBoundVmIds(boundVmIdsOf(p)), - })), - ) + const config = structuredClone(this.state.config) + const proxies = this.state.proxies.map((p) => ({ + ...structuredClone(p), + bound_vm_id: encodeBoundVmIds(boundVmIdsOf(p)), + })) + this.writes.enqueue(async () => { + await this.repo.setConfig(config) + await this.repo.replaceAll(proxies) + }) this.emit('change') } @@ -1382,7 +1393,7 @@ export class ProxyPool extends EventEmitter { const min = this.state.config.probe_interval_min || 10 const ms = min * 60 * 1000 void this.probeAll({ onlyEnabled: true }).catch(() => {}) - this._timer = setInterval(() => { + this._timer = setInterval(async () => { this.probeAll({ onlyEnabled: true }).catch(() => {}) }, ms) this._timer.unref?.() diff --git a/src/lib/vm/vm-db-sync.mjs b/src/lib/vm/vm-db-sync.mjs index 694a3e2a1..484715d7c 100644 --- a/src/lib/vm/vm-db-sync.mjs +++ b/src/lib/vm/vm-db-sync.mjs @@ -1,7 +1,7 @@ /** - * VM/credential DB mirror — write-through + reconcile + fs.watch. + * VM metadata DB mirror — write-through + reconcile + fs.watch. * - * Policy (user requirement: credentials must live in the database): + * Policy (VmsRepo omits active credentials and retains their presence flags): * - vms/*.json stay the runtime single-writer surface (persistOauthToVm, * CLI seed, execution context all read files — no behavior drift). * - Every vm.json write is mirrored into the `vms` table: @@ -21,6 +21,7 @@ import fs from 'node:fs' import path from 'node:path' import { isDbOpen, getDb } from '../db/database.mjs' import { VmsRepo } from '../db/repos/vms-repo.mjs' +import { WriteQueue } from '../pool/write-queue.mjs' import { setVmWriteHook, atomicWriteJson, isVmRecordFile } from './vm-file.mjs' import { reclaimAssignedVms, repairSlotSmbios, repairVmProxyCliEnabled } from './vm-registry.mjs' @@ -28,6 +29,9 @@ let _repo = null let _watcher = null let _watchTimer = null let _projectRoot = null +let _writes = new WriteQueue() +let _paused = false +const _deleted = new Set() function repo() { if (!isDbOpen()) return null @@ -42,103 +46,148 @@ export function isVmJsonPath(filePath) { return isVmRecordFile(base) || base === 'active.json' } -/** Mirror one vms/*.json file into the DB (safe no-op without DB). */ +/** Mirror a captured file snapshot in write order. Flush reports persistence errors. */ export function syncVmFile(filePath) { const r = repo() - if (!r) return false + if (!r || _paused) return Promise.resolve(false) + let value + let mtimeMs try { - const base = path.basename(filePath) - if (base === 'active.json') { - const a = JSON.parse(fs.readFileSync(filePath, 'utf8')) - if (a?.active_vm) r.setActiveVmId(a.active_vm) - return true - } - const vm = JSON.parse(fs.readFileSync(filePath, 'utf8')) - if (!vm?.id) return false - const st = fs.statSync(filePath) - r.upsertFromVmJson(vm, { mtimeMs: st.mtimeMs }) - return true - } catch { - return false + value = JSON.parse(fs.readFileSync(filePath, 'utf8')) + mtimeMs = fs.statSync(filePath).mtimeMs + } catch (error) { + if (error.code === 'ENOENT') return Promise.resolve(false) + _writes.enqueue(() => { + throw error + }) + return _writes.flush().then(() => false) } + const active = path.basename(filePath) === 'active.json' + if (!active && value?.id) _deleted.delete(value.id) + if (active ? !value?.active_vm : !value?.id) return Promise.resolve(false) + _writes.enqueue(() => (active ? r.setActiveVmId(value.active_vm) : r.upsertFromVmJson(value, { mtimeMs }))) + return _writes.flush().then(() => true) } -/** Remove a VM row (called when the vm.json file is deleted). */ -export function removeVmFromDb(id) { +/** Deletes follow every previously captured upsert. */ +export async function removeVmFromDb(id) { const r = repo() if (!r || !id) return false - return r.remove(id) + _deleted.add(id) + let removed = false + _writes.enqueue(async () => { + removed = await r.remove(id) + }) + await _writes.flush() + return removed } -/** Forget settings.active_vm so startup reconcile does not rewrite active.json. */ -export function clearActiveVmFromDb() { +export async function clearActiveVmFromDb() { const r = repo() if (!r) return false - r.clearActiveVmId() + _writes.enqueue(() => r.clearActiveVmId()) + await _writes.flush() return true } +export function flushVmDbSync() { + return _writes.flush() +} + +export async function pauseVmDbSync() { + _paused = true + stopVmWatch() + await flushVmDbSync() +} + +export function resumeVmDbSync() { + _paused = false + if (_projectRoot) startVmWatch(_projectRoot) +} + /** * Startup reconcile: * - file newer than DB row (or row missing) → upsert from file * - DB row exists but file missing → rebuild file from vm_json * @returns {{ upserted: number, rebuilt: number, active: string|null }} */ -export function reconcileVms(projectRoot) { +export async function reconcileVms(projectRoot) { const r = repo() if (!r) return { upserted: 0, rebuilt: 0, active: null } const dir = path.join(projectRoot, 'vms') const out = { upserted: 0, rebuilt: 0, active: null } const seen = new Set() + // The file snapshot and its conditional database write share the same queue + // as write hooks. A newer hook always executes after this snapshot. if (fs.existsSync(dir)) { - for (const f of fs.readdirSync(dir)) { - if (!isVmRecordFile(f)) continue - const file = path.join(dir, f) - try { - const vm = JSON.parse(fs.readFileSync(file, 'utf8')) - if (!vm?.id) continue + for (const name of fs.readdirSync(dir)) { + if (!isVmRecordFile(name)) continue + _writes.enqueue(async () => { + const file = path.join(dir, name) + let vm + let stat + try { + vm = JSON.parse(fs.readFileSync(file, 'utf8')) + stat = fs.statSync(file) + } catch (error) { + if (error.code === 'ENOENT' || error instanceof SyntaxError) return + throw error + } + if (!vm?.id || _deleted.has(vm.id)) return seen.add(vm.id) - const st = fs.statSync(file) - const row = r.get(vm.id) - if (!row || (row.file_mtime_ms || 0) < Math.floor(st.mtimeMs)) { - r.upsertFromVmJson(vm, { mtimeMs: st.mtimeMs }) + const row = await r.get(vm.id) + if (!row || (row.file_mtime_ms || 0) < Math.floor(stat.mtimeMs)) { + await r.upsertFromVmJson(vm, { mtimeMs: stat.mtimeMs }) out.upserted += 1 } - } catch {} + }) } } + await flushVmDbSync() - // DB rows without files → rebuild files (restore / disaster pull-up) - for (const row of r.list()) { - if (seen.has(row.id) || !row.vm) continue - try { - fs.mkdirSync(dir, { recursive: true }) + let rows = [] + _writes.enqueue(async () => { + rows = await r.list() + }) + await flushVmDbSync() + for (const row of rows) { + if (seen.has(row.id)) continue + _writes.enqueue(async () => { + const current = await r.get(row.id) const file = path.join(dir, `${row.id}.json`) - atomicWriteJson(file, row.vm, { mode: 0o600 }) + if (!current?.vm || _deleted.has(row.id) || fs.existsSync(file)) return + fs.mkdirSync(dir, { recursive: true }) + atomicWriteJson(file, current.vm, { mode: 0o600 }) out.rebuilt += 1 - } catch {} + }) } + await flushVmDbSync() - // active.json ↔ settings.active_vm - const activeFile = path.join(dir, 'active.json') - try { + _writes.enqueue(async () => { + const activeFile = path.join(dir, 'active.json') if (fs.existsSync(activeFile)) { - const a = JSON.parse(fs.readFileSync(activeFile, 'utf8')) - if (a?.active_vm) { - r.setActiveVmId(a.active_vm) - out.active = a.active_vm + let active + try { + active = JSON.parse(fs.readFileSync(activeFile, 'utf8')) + } catch (error) { + if (error instanceof SyntaxError) return + throw error + } + if (active?.active_vm && !_deleted.has(active.active_vm)) { + await r.setActiveVmId(active.active_vm) + out.active = active.active_vm } } else { - const dbActive = r.getActiveVmId() - if (dbActive) { + const active = await r.getActiveVmId() + if (active && !_deleted.has(active)) { fs.mkdirSync(dir, { recursive: true }) - atomicWriteJson(activeFile, { active_vm: dbActive, updated_at: new Date().toISOString() }) - out.active = dbActive + atomicWriteJson(activeFile, { active_vm: active, updated_at: new Date().toISOString() }) + out.active = active } } - } catch {} - + }) + await flushVmDbSync() return out } @@ -146,21 +195,20 @@ export function reconcileVms(projectRoot) { * Install the write-through hook + directory watcher. * Call once at server startup, after openDatabase(). */ -export function initVmDbSync(projectRoot, { watch = true } = {}) { +export async function initVmDbSync(projectRoot, { watch = true } = {}) { _projectRoot = projectRoot - setVmWriteHook((filePath) => { - if (isVmJsonPath(filePath)) syncVmFile(filePath) + await flushVmDbSync() + _writes = new WriteQueue(isDbOpen() ? getDb() : null) + _paused = false + _deleted.clear() + setVmWriteHook(async (filePath) => { + if (isVmJsonPath(filePath)) syncVmFile(filePath).catch(() => {}) }) - const result = reconcileVms(projectRoot) + const result = await reconcileVms(projectRoot) // Older releases soft-deleted tenants without releasing their assigned slots. // Only confirmed deletions qualify; disabled/missing users and private slots stay isolated. const deletedOwners = isDbOpen() - ? new Set( - getDb() - .prepare('SELECT id FROM users WHERE deleted_at IS NOT NULL') - .all() - .map((row) => row.id), - ) + ? new Set((await getDb().prepare('SELECT id FROM users WHERE deleted_at IS NOT NULL').all()).map((row) => row.id)) : new Set() result.reclaimed_vm_ids = reclaimAssignedVms(projectRoot, deletedOwners, { onError: (error, name) => { @@ -177,6 +225,7 @@ export function initVmDbSync(projectRoot, { watch = true } = {}) { } catch { result.smbios_repaired = 0 } + await flushVmDbSync() if (watch) startVmWatch(projectRoot) return result } @@ -189,10 +238,12 @@ export function startVmWatch(projectRoot) { try { _watcher = fs.watch(dir, () => { if (_watchTimer) clearTimeout(_watchTimer) - _watchTimer = setTimeout(() => { + _watchTimer = setTimeout(async () => { _watchTimer = null try { - reconcileDeletionsAndChanges(projectRoot) + reconcileDeletionsAndChanges(projectRoot).catch((error) => { + console.warn(`[vm] database mirror failed: ${error.code || error.name}`) + }) } catch {} }, 500) if (typeof _watchTimer.unref === 'function') _watchTimer.unref() @@ -205,37 +256,21 @@ export function startVmWatch(projectRoot) { } } -function reconcileDeletionsAndChanges(projectRoot) { +async function reconcileDeletionsAndChanges(projectRoot) { const r = repo() if (!r) return - reconcileVmsChangesOnly(projectRoot, r) + if (!_paused) await reconcileVmsChangesOnly(projectRoot, r) } /** Watch-triggered pass: upsert changed files; DO NOT rebuild deleted files * (a panel-initiated VM delete must stay deleted — server removes the row). */ -function reconcileVmsChangesOnly(projectRoot, r) { +async function reconcileVmsChangesOnly(projectRoot) { const dir = path.join(projectRoot, 'vms') if (!fs.existsSync(dir)) return - for (const f of fs.readdirSync(dir)) { - if (!isVmRecordFile(f)) continue - const file = path.join(dir, f) - try { - const st = fs.statSync(file) - const vm = JSON.parse(fs.readFileSync(file, 'utf8')) - if (!vm?.id) continue - const row = r.get(vm.id) - if (!row || (row.file_mtime_ms || 0) < Math.floor(st.mtimeMs)) { - r.upsertFromVmJson(vm, { mtimeMs: st.mtimeMs }) - } - } catch {} + for (const file of fs.readdirSync(dir)) { + if (_paused) return + if (isVmRecordFile(file) || file === 'active.json') await syncVmFile(path.join(dir, file)) } - const activeFile = path.join(dir, 'active.json') - try { - if (fs.existsSync(activeFile)) { - const a = JSON.parse(fs.readFileSync(activeFile, 'utf8')) - if (a?.active_vm && a.active_vm !== r.getActiveVmId()) r.setActiveVmId(a.active_vm) - } - } catch {} } export function stopVmWatch() { diff --git a/src/server.mjs b/src/server.mjs index a4a97d8fb..1bb90387d 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -34,6 +34,7 @@ import { ensureTelemetrySidecar } from './lib/vm/slot-process-status.mjs' import { createSlotBridgeHealer, powerOnSlot, reviveDownSlots } from './lib/vm/slot-recovery.mjs' import { createUsageProbeMonitor, normalizeUsageProbeConfig } from './lib/oauth/usage-probe-monitor.mjs' import { createCatalogMonitor } from './lib/admin/catalog-monitor.mjs' +import { pausePriceCatalog, resumePriceCatalog } from './lib/admin/price-catalog.mjs' import { normalizeOfficialCcConfig } from './lib/oauth/official-cc-bootstrap.mjs' import { invalidateLiveCredentialCache } from './lib/admin/panel-live-credentials.mjs' import { normalizeHealthProbeConfig, createHealthProbeMonitor, HEALTH_REAL_HEADER } from './lib/admin/health-probe.mjs' @@ -47,7 +48,7 @@ import { normalizeTiers } from './lib/pool/quota-tiers.mjs' import { ApiKeyStore } from './lib/admin/api-keys.mjs' import { keyScopeFromRequest } from './lib/admin/key-scope.mjs' import { GroupsRepo } from './lib/db/repos/groups-repo.mjs' -import { SettingsRepo } from './lib/db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo, loadSettingsState } from './lib/db/settings-state.mjs' import { ApiEndpointStore } from './lib/admin/api-endpoints.mjs' import { ApiScheduler } from './lib/pool/api-scheduler.mjs' import { resolveInferenceBackend } from './lib/pool/api-protocol.mjs' @@ -72,7 +73,9 @@ import { GATEWAY_CAPABILITIES } from './lib/vm/execution-context.mjs' import { isTelemetryPath, telemetryInterceptResponse } from './lib/identity/telemetry-rewrite.mjs' import { openDatabase, closeDatabase } from './lib/db/database.mjs' import { runLegacyImport } from './lib/db/legacy-import.mjs' -import { initVmDbSync, stopVmWatch } from './lib/vm/vm-db-sync.mjs' +import { initVmDbSync, stopVmWatch, flushVmDbSync, pauseVmDbSync, resumeVmDbSync } from './lib/vm/vm-db-sync.mjs' +import { flushPendingWrites } from './lib/db/pending-writes.mjs' +import { loadGptModelPolicy } from './lib/protocol/gpt-model-policy.mjs' import { BackupService } from './lib/admin/backup-service.mjs' import { ClusterNodesRepo } from './lib/db/repos/cluster-nodes-repo.mjs' import { ClusterManager } from './lib/cluster/cluster-manager.mjs' @@ -139,21 +142,20 @@ const allowRate = createRateLimiter({ refillPerSec: cfg.limits.rate_refill, }) -// --- Persistent store (SQLite, sub2api-inspired) --- +// --- Persistent PostgreSQL store --- +const activeRequests = new Set() +let _shuttingDown = false const dataDir = cfg.paths.data || path.join(cfg.paths.root, 'data') -openDatabase({ dataDir }) -try { - loadModelPolicy() -} catch (e) { - console.warn('[model-policy] boot load failed', e?.message || e) -} -// one-time migration of legacy JSON files (data/*.json + request-logs) into the DB -const legacyImport = runLegacyImport({ dataDir, projectRoot: cfg.paths.project }) +await openDatabase({ dataDir }) +const legacyImport = await runLegacyImport({ dataDir, projectRoot: cfg.paths.project }) if (legacyImport?.imported) { console.log('[db] legacy JSON import done:', JSON.stringify(legacyImport.counts || {})) } +await loadSettingsState() +await loadModelPolicy() +await loadGptModelPolicy() // VM/credential mirror: write-through hook + startup reconcile + fs.watch -const vmSync = initVmDbSync(cfg.paths.project) +const vmSync = await initVmDbSync(cfg.paths.project) if (vmSync.upserted || vmSync.rebuilt) { console.log(`[db] vm mirror reconciled: upserted=${vmSync.upserted} rebuilt=${vmSync.rebuilt}`) } @@ -287,7 +289,7 @@ routingConfig.usage_probe = normalizeUsageProbeConfig(routingConfig.usage_probe) routingConfig.notify = normalizeNotifyConfig(routingConfig.notify) routingConfig.tiers = normalizeTiers(routingConfig.tiers, routingConfig.quota, routingConfig.concurrency) syncTierDefaultsIntoRouting() -stickyRouter = new StickyRouter({ dataDir, config: routingConfig }) +stickyRouter = await new StickyRouter({ dataDir, config: routingConfig }).init() accountQuota = new AccountQuota({ dataDir, config: routingConfig, @@ -300,29 +302,30 @@ accountQuota = new AccountQuota({ max_rpm: v.policy?.maxRpm ?? routingConfig?.concurrency?.default_max_rpm ?? 0, })), }) +await accountQuota.init() accountQuota.loadVmQuotaOverrides(listVms(cfg.paths.project)) const apiKeyStore = new ApiKeyStore({ dataDir: cfg.paths.data }) groupsRepo = new GroupsRepo() const apiEndpointStore = new ApiEndpointStore({ dataDir: cfg.paths.data }) const apiScheduler = new ApiScheduler() -apiScheduler.reload(apiEndpointStore.listRaw()) +apiScheduler.reload(await apiEndpointStore.listRaw()) const panelUsers = new PanelUserStore({ dataDir: cfg.paths.data, projectRoot: cfg.paths.project }) try { - panelUsers.bootstrapFromEnv(getPanelAdmin()) + await panelUsers.bootstrapFromEnv(getPanelAdmin()) } catch (e) { console.warn('[panel-users] bootstrap failed', e?.message || e) } attachPanelAuth({ - authenticate: (username, password) => panelUsers.authenticate(username, password), - lookupRole: (username) => panelUsers.getByUsername(username)?.role || null, + authenticate: async (username, password) => await panelUsers.authenticate(username, password), + lookupRole: async (username) => (await panelUsers.getByUsername(username))?.role || null, }) requestLog = new RequestLogStore({ dataDir: cfg.paths.data, mode: process.env.KIN_REQUEST_LOG_MODE || routingConfig?.logging?.mode || 'normal', }) if (routingConfig?.logging) { - requestLog.setConfig({ + await requestLog.setConfig({ // Env wins at boot so e2e / ops can force off|debug. Panel PUT still hot-updates. mode: process.env.KIN_REQUEST_LOG_MODE || routingConfig.logging.mode, retainDays: routingConfig.logging.retain_days, @@ -332,7 +335,7 @@ if (routingConfig?.logging) { }) } try { - requestLog.cleanup() + await requestLog.cleanup() } catch {} requestLog.startCleanupScheduler() @@ -369,8 +372,9 @@ proxyPool = new ProxyPool({ setVmSchedulable(cfg.paths.project, vmId, true) }, egressCheck: (proxy) => proxyEgressReady(proxy, cfg.paths.project), - repairEgress: (proxy) => ensureProxyEgress(cfg.paths.project, proxy), + repairEgress: async (proxy) => await ensureProxyEgress(cfg.paths.project, proxy), }) +await proxyPool.init() // An upgrade replaces bin/ files; slots mounting data/run/slot-guest-bin must see the new ones. try { refreshSlotGuestBinDir(cfg.paths.project) @@ -384,7 +388,7 @@ try { if (!proxyPool.snapshot().proxies?.length) { const seeded = proxyPool.ensureLocal() if (seeded.created) { - void ensureProxyEgress(cfg.paths.project, proxyPool.getProxyByIdWithAuth(seeded.proxy.id)) + void (await ensureProxyEgress(cfg.paths.project, proxyPool.getProxyByIdWithAuth(seeded.proxy.id))) console.log('[bootstrap] seeded local egress px-local') } } @@ -392,7 +396,7 @@ try { console.warn('[bootstrap] local egress seed failed', e?.message || e) } -initPoolRuntime() +await initPoolRuntime() try { applyRoutingTierConcurrency(routingConfig.tiers) } catch (e) { @@ -446,8 +450,8 @@ function attachSlotRuntime(vm) { healthMonitor = createHealthProbeMonitor({ config: routingConfig.health_probe, listTargets: () => listVms(cfg.paths.project).map(attachSlotRuntime), - runChat: (vm, real) => - runVmTestChat({ + runChat: async (vm, real) => + await runVmTestChat({ projectRoot: cfg.paths.project, vmId: vm.id, model: real.model, @@ -465,8 +469,8 @@ const vmTestSchedule = new VmTestScheduleRunner({ projectRoot: cfg.paths.project, settings: () => new SettingsRepo(), list: () => listVms(cfg.paths.project), - runTest: (vmId, params) => - runVmTestChat({ + runTest: async (vmId, params) => + await runVmTestChat({ projectRoot: cfg.paths.project, vmId, ...params, @@ -486,8 +490,8 @@ credentialRefreshMonitor = createCredentialRefreshMonitor({ } catch {} return { ...vm, last_probe: lastProbe } }), - refreshOne: (vm) => - refreshWorkerCredentialForVm({ + refreshOne: async (vm) => + await refreshWorkerCredentialForVm({ vmId: vm.id, vmPath: path.join(cfg.paths.project, 'vms', `${vm.id}.json`), homeDir: path.join(cfg.paths.project, 'vms', vm.id, 'cli-home'), @@ -495,20 +499,20 @@ credentialRefreshMonitor = createCredentialRefreshMonitor({ force: false, }), }) -function bootSlot(id) { - return powerOnSlot({ projectRoot: cfg.paths.project, id, proxyPool, routing: routingConfig }) +async function bootSlot(id) { + return await powerOnSlot({ projectRoot: cfg.paths.project, id, proxyPool, routing: routingConfig }) } kernelWatchdog = createKernelWatchdog({ config: routingConfig.kernel_watchdog, listTargets: () => listVms(cfg.paths.project), getRouting: () => routingConfig, homeDirFor: (vm) => path.join(cfg.paths.project, 'vms', vm.id, 'cli-home'), - ensureTelemetry: (vm) => ensureTelemetrySidecar({ projectRoot: cfg.paths.project, vm }), - revive: (vm) => bootSlot(vm.id), + ensureTelemetry: async (vm) => await ensureTelemetrySidecar({ projectRoot: cfg.paths.project, vm }), + revive: async (vm) => await bootSlot(vm.id), healBridges: createSlotBridgeHealer({ projectRoot: cfg.paths.project }), - onFault: (vm, reason) => { + onFault: async (vm, reason) => { const title = `槽内核故障 ${vm.id}` - dispatchNotify(routingConfig.notify, { + await dispatchNotify(routingConfig.notify, { title, text: `【KIN】${title}\n\n${reason}\n已停止调度该槽；内核恢复健康后自动恢复调度。`, subject: `KIN · ${title}`, @@ -538,7 +542,7 @@ usageProbeMonitor = createUsageProbeMonitor({ poolScheduler.syncQuotaSchedule(vm, account) } catch {} }, - probeOne: (vm) => panel.buildProbeOne({ cfg, accountQuota, id: vm.id, routingConfig }), + probeOne: async (vm) => await panel.buildProbeOne({ cfg, accountQuota, id: vm.id, routingConfig }), }) catalogMonitor = createCatalogMonitor({ projectRoot: cfg.paths.project, @@ -548,8 +552,8 @@ catalogMonitor = createCatalogMonitor({ notifyMonitor = createNotifyMonitor({ config: routingConfig.notify, baseUrl: cfg.base_url, - snapshot: () => - panel.snapshotAccountPool({ + snapshot: async () => + await panel.snapshotAccountPool({ cfg, accountQuota, routingConfig, @@ -565,15 +569,18 @@ const backupService = new BackupService({ projectRoot: cfg.paths.project, configDir: path.dirname(routingConfigPath), }) -backupService.onRestored((db) => { - // A restore swaps the DatabaseSync instance and routing files. Rebuild every +backupService.onRestored(async (db) => { + await loadSettingsState(db) + await loadModelPolicy({ force: true }) + await loadGptModelPolicy({ force: true }) + // A restore swaps the database pool and routing files. Rebuild every // holder before reporting success; otherwise requests keep stale snapshots // or silently write through statements owned by the closed connection. for (const store of [apiKeyStore, apiEndpointStore, panelUsers, accountQuota, stickyRouter, proxyPool, requestLog]) { - store.rebind(db) + await store.rebind(db) } groupsRepo = new GroupsRepo(db) - apiScheduler.reload(apiEndpointStore.listRaw()) + apiScheduler.reload(await apiEndpointStore.listRaw()) routingConfig = loadRoutingConfig() migrateLegacyOpenAIPolicies(routingConfig) @@ -590,7 +597,7 @@ backupService.onRestored((db) => { stickyRouter.reloadConfig(routingConfig) accountQuota.reloadConfig(routingConfig) accountQuota.loadVmQuotaOverrides(listVms(cfg.paths.project)) - requestLog.setConfig({ + await requestLog.setConfig({ mode: process.env.KIN_REQUEST_LOG_MODE || routingConfig.logging?.mode, retainDays: routingConfig.logging?.retain_days, debugRetainDays: routingConfig.logging?.debug_retain_days, @@ -603,11 +610,36 @@ backupService.onRestored((db) => { usageProbeMonitor?.setConfig(routingConfig.usage_probe) notifyMonitor?.setConfig(routingConfig.notify) - initPoolRuntime() + await initPoolRuntime() + await clusterManager.rebind(db) reloadActiveVm(cfg) - vmTestSchedule.reload() + await vmTestSchedule.reload() +}) +backupService.onMaintenance(async (enabled) => { + if (enabled) { + vmTestSchedule.stop() + proxyPool.stopScheduler() + requestLog.stopCleanupScheduler() + for (const monitor of maintenanceMonitors()) await monitor?.stop?.() + while ([...activeRequests].some((request) => !/\/(?:backups)(?:\/|$)/.test(String(request.url || '')))) { + await new Promise((resolve) => setTimeout(resolve, 10)) + } + for (const monitor of maintenanceMonitors()) await monitor?.flush?.() + await vmTestSchedule.flush() + await pausePriceCatalog() + await flushRuntimeWrites() + await pauseVmDbSync() + } else { + if (_shuttingDown) return + resumeVmDbSync() + resumePriceCatalog() + for (const monitor of maintenanceMonitors()) await monitor?.start?.({ immediate: false }) + proxyPool.startScheduler() + requestLog.startCleanupScheduler() + await vmTestSchedule.start() + } }) -backupService.startScheduler() +await backupService.startScheduler() const stats = { requests: 0, @@ -687,10 +719,23 @@ async function refreshWorkerCredentialForVm({ vmId, vmPath, homeDir, vm, force = return published } -const { json, writeSSEHeaders, readBody, readRawBody } = createRespond(cfg, { +const { + json: sendJson, + writeSSEHeaders, + readBody, + readRawBody, +} = createRespond(cfg, { tcpNodelay: () => routingConfig?.inference?.tcp_nodelay !== false, }) +function json(res, ...args) { + if (res._panelResponseState) { + res._panelResponseState.response = [res, ...args] + return + } + return sendJson(res, ...args) +} + function rejectAuth(req, res, error) { req.authError = { code: error?.body?.error?.code || ErrorCode.INVALID_API_KEY, @@ -701,7 +746,7 @@ function rejectAuth(req, res, error) { return false } -function requireAuth(req, res) { +async function requireAuth(req, res) { req.presentedApiKey = extractApiKey(req) const token = extractPanelToken(req) || req.presentedApiKey if (!token) { @@ -717,7 +762,7 @@ function requireAuth(req, res) { ) } // Panel session - const session = verifyPanelSession(token) + const session = await verifyPanelSession(token) if (session) { if (!allowRate('panel:' + session.user)) { return rejectAuth( @@ -732,8 +777,8 @@ function requireAuth(req, res) { ) } req.panelUser = session.user - req.panelRole = session.role || panelUsers.getByUsername(session.user)?.role || 'user' - const rec = panelUsers.getByUsername(session.user) + req.panelRole = session.role || (await panelUsers.getByUsername(session.user))?.role || 'user' + const rec = await panelUsers.getByUsername(session.user) req.panelUserId = rec?.id || null req.panelVmCreateQuota = rec?.vm_create_quota ?? 0 return true @@ -759,7 +804,7 @@ function requireAuth(req, res) { } // Managed multi-keys (sub2api-style) - const managed = apiKeyStore.authenticate(token) + const managed = await apiKeyStore.authenticate(token) if (!managed.ok) { return rejectAuth( req, @@ -772,7 +817,7 @@ function requireAuth(req, res) { }), ) } - const gate = apiKeyStore.canAccept(managed.record) + const gate = await apiKeyStore.canAccept(managed.record) if (!gate.ok) { const type = gate.status === 429 @@ -834,7 +879,7 @@ const importCommit = createImportCommit({ const { commitImportedOauth, requireSlotProxy, officialCcStatsHandler } = importCommit -const { handleProtocol, handleSearch } = createHandleProtocol({ +const protocolHandlers = createHandleProtocol({ json, writeSSEHeaders, readBody, @@ -869,6 +914,23 @@ const { handleProtocol, handleSearch } = createHandleProtocol({ }, }) +async function dispatchProtocol(name, req, res, ...args) { + if (backupService.isMaintaining) { + return json(res, 503, { + error: { type: 'overloaded_error', code: 'maintenance_in_progress', message: 'Database maintenance in progress' }, + }) + } + activeRequests.add(req) + try { + return await protocolHandlers[name](req, res, ...args) + } finally { + activeRequests.delete(req) + } +} + +const handleProtocol = async (...args) => await dispatchProtocol('handleProtocol', ...args) +const handleSearch = async (...args) => await dispatchProtocol('handleSearch', ...args) + const responsesWs = createResponsesWsHandler({ handleProtocol, requireAuth, @@ -911,10 +973,12 @@ const clusterManager = new ClusterManager({ throw new ClusterError(409, 'node_in_use', '节点仍有进行中的请求、虚拟机操作或镜像构建，请完成后重试') } }, - onNodesChanged: () => syncNodeLocalEgress(), + onNodesChanged: () => { + if (!backupService.isMaintaining) syncNodeLocalEgress() + }, // A node's local row follows its link: re-probe as soon as the link settles. onLinkState: (nodeId, state) => { - if (state === 'ready' || state === 'error' || state === 'backoff') { + if (!backupService.isMaintaining && (state === 'ready' || state === 'error' || state === 'backoff')) { proxyPool.probeLocalNode(nodeId).catch(() => {}) } }, @@ -922,7 +986,7 @@ const clusterManager = new ClusterManager({ localEgressCluster = clusterManager bindPlacement({ manager: clusterManager, projectRoot: cfg.paths.project }) await startNodeEgressSocks({ clientFor: (nodeId) => clusterManager.client(nodeId) }) -clusterManager.start() +await clusterManager.start() clusterManager.restoreSlotRelays() syncNodeLocalEgress() if (proxyPool.snapshot().config.ipv6_enabled !== true) { @@ -947,6 +1011,14 @@ cliNodeGuard = createCliNodeGuard({ }) const handlePanel = createPanelHandler({ + onPersistenceFailure: async () => { + backupService.isMaintaining = true + vmTestSchedule.stop() + proxyPool.stopScheduler() + requestLog.stopCleanupScheduler() + stopVmWatch() + for (const monitor of maintenanceMonitors()) await monitor?.stop?.() + }, json, writeSSEHeaders, readBody, @@ -1018,6 +1090,7 @@ const handlePanel = createPanelHandler({ }) const server = http.createServer(async (req, res) => { + activeRequests.add(req) try { res.req = req if (req.method === 'OPTIONS') { @@ -1036,16 +1109,18 @@ const server = http.createServer(async (req, res) => { } if ( - backupService.isRestoring && - (p.startsWith('/v1/') || p === '/messages' || p === '/chat/completions' || p === '/responses') + backupService.isMaintaining && + p !== '/healthz' && + p !== '/health' && + !(req.method === 'POST' && /^\/(?:api\/panel|admin)\/backups\/[^/]+\/restore$/.test(p)) ) { return json( res, 503, makeError({ type: ErrorType.OVERLOADED, - code: 'restore_in_progress', - message: 'Gateway is restoring from backup; retry shortly', + code: backupService.isRestoring ? 'restore_in_progress' : 'maintenance_in_progress', + message: 'Database maintenance in progress; retry shortly', status: 503, }).body, ) @@ -1085,7 +1160,7 @@ const server = http.createServer(async (req, res) => { health_probe: healthMonitor?.getSnapshot?.() || null, api_kernel: { ...apiKernelSnapshot(cfg), - endpoints: apiEndpointStore.listRaw().filter((ep) => !ep.disabled).length, + endpoints: (await apiEndpointStore.listRaw()).filter((ep) => !ep.disabled).length, }, }) } @@ -1111,15 +1186,15 @@ const server = http.createServer(async (req, res) => { } if (req.method === 'GET' && p === '/v1/models') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return if (resolveInferenceBackend(req) === 'api') { return json(res, 200, apiScheduler.catalog()) } - const result = await fetchWorkerModels({ groupType: keyScopeFromRequest(req).group_type }) + const result = await fetchWorkerModels({ groupType: (await keyScopeFromRequest(req)).group_type }) return json(res, 200, result) } if (req.method === 'POST' && p === '/admin/models/refresh') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const result = await fetchWorkerModels() return json(res, 200, result) } @@ -1181,11 +1256,17 @@ const server = http.createServer(async (req, res) => { res.end() } catch {} } + } finally { + activeRequests.delete(req) } }) -server.on('upgrade', (req, socket, head) => { - if (responsesWs.handleUpgrade(req, socket, head)) return +server.on('upgrade', async (req, socket, head) => { + if (backupService.isMaintaining) { + socket.end('HTTP/1.1 503 Service Unavailable\r\nConnection: close\r\n\r\n') + return + } + if (await responsesWs.handleUpgrade(req, socket, head)) return if (clusterRoutes.handleUpgrade(req, socket, head)) return if (!slotShell.handleUpgrade(req, socket, head)) socket.destroy() }) @@ -1238,9 +1319,8 @@ fs.writeFileSync(path.join(cfg.paths.root, 'config', 'gateway-v2.public.json'), process.on('uncaughtException', (e) => console.error('[uncaught]', e)) process.on('unhandledRejection', (e) => console.error('[unhandled]', e)) -// Graceful shutdown: stop schedulers/watchers, close the DB (WAL checkpoint). -let _shuttingDown = false -function shutdown(signal) { +// Graceful shutdown: stop schedulers/watchers, drain persistence, close the pool. +async function shutdown(signal) { if (_shuttingDown) return _shuttingDown = true console.log(`[shutdown] ${signal} — closing`) @@ -1285,21 +1365,54 @@ function shutdown(signal) { clusterManager.stop() } catch {} try { - server.close(() => {}) + await new Promise((resolve) => server.close(resolve)) } catch {} try { - closeDatabase() - } catch {} + await backupService.flush() + await pausePriceCatalog() + for (const monitor of maintenanceMonitors()) await monitor?.flush?.() + await vmTestSchedule.flush() + await flushRuntimeWrites() + await closeDatabase() + } catch (error) { + console.error(`[shutdown] persistence failed: ${error.code || error.name}`) + try { + await closeDatabase() + } catch {} + process.exit(1) + } process.exit(0) } -process.on('SIGTERM', () => shutdown('SIGTERM')) -process.on('SIGINT', () => shutdown('SIGINT')) +process.on('SIGTERM', async () => await shutdown('SIGTERM')) +process.on('SIGINT', async () => await shutdown('SIGINT')) + +function maintenanceMonitors() { + return [ + healthMonitor, + credentialRefreshMonitor, + kernelWatchdog, + cliNodeGuard, + usageProbeMonitor, + catalogMonitor, + notifyMonitor, + ] +} + +async function flushRuntimeWrites() { + await requestLog.flush?.() + await flushPendingWrites() + await accountQuota.flush() + await stickyRouter.flush() + await proxyPool.flush() + await runtimeRepo?.flush?.() + await flushVmDbSync() +} await bootApiKernel(cfg) -server.listen(cfg.port, cfg.host, () => { +server.listen(cfg.port, cfg.host, async () => { try { - vmTestSchedule.start() + await vmTestSchedule.start() } catch (e) { console.warn('[vm-test-schedule] start failed', e?.message || e) } diff --git a/test/deploy/smoke_arm64_image.py b/test/deploy/smoke_arm64_image.py index 062bd8d50..64d048553 100644 --- a/test/deploy/smoke_arm64_image.py +++ b/test/deploy/smoke_arm64_image.py @@ -3,6 +3,9 @@ import argparse import json +import os +import secrets +import time from pathlib import Path import subprocess import uuid @@ -41,6 +44,11 @@ def elf(path, machine, static=False): elf(root / 'image-bin' / 'kin-api-kernel', 183, static=True) elf(Path('/usr/local/bin/docker'), 183, static=True) subprocess.run(['docker', '--version'], check=True, timeout=10) +for name in ('pg_dump', 'pg_restore'): + binary = Path('/usr/lib/postgresql/17/bin') / name + elf(binary, 183) + assert ' 17.' in subprocess.check_output([str(binary), '--version'], text=True) + for name in ('kin-kernel', 'kin-codex-kernel', 'kin-oauth-auth', 'kin-computer'): elf(root / 'image-bin' / name, 62) for name in ('kin-kernel', 'kin-codex-kernel'): @@ -113,18 +121,30 @@ def main(): args = parser.parse_args() expected = {"version": (ROOT / "VERSION").read_text().strip()} name = "vm2api-arm64-smoke-" + uuid.uuid4().hex[:12] + network = name + '-net' + postgres = name + '-pg' + password = secrets.token_hex(32) + db_env = {**os.environ, 'POSTGRES_PASSWORD': password, 'POSTGRES_USER': 'vm2api', 'POSTGRES_DB': 'vm2api_smoke_test', + 'VM2API_DATABASE_URL': f'postgresql://vm2api:{password}@{postgres}:5432/vm2api_smoke_test'} try: + subprocess.run(['docker', 'network', 'create', '--internal', network], check=True, capture_output=True) + subprocess.run(['docker', 'run', '-d', '--name', postgres, '--network', network, + '-e', 'POSTGRES_PASSWORD', '-e', 'POSTGRES_USER', '-e', 'POSTGRES_DB', + 'postgres:17-bookworm'], env=db_env, check=True, capture_output=True) + deadline = time.monotonic() + 60 + while subprocess.run(['docker', 'exec', postgres, 'psql', '-U', 'vm2api', '-d', 'vm2api_smoke_test', '-Atqc', 'SELECT 1'], + capture_output=True).returncode: + if time.monotonic() > deadline: + raise RuntimeError('isolated PostgreSQL did not become ready') + time.sleep(0.5) subprocess.run(["docker", "run", "--name", name, "--platform", "linux/arm64", - "--network", "none", "-i", "--entrypoint", "python3", args.image, - "-", json.dumps(expected)], input=PROGRAM, text=True, check=True, timeout=210) + "--network", network, '-e', 'VM2API_DATABASE_URL', "-i", "--entrypoint", "python3", args.image, + "-", json.dumps(expected)], env=db_env, input=PROGRAM, text=True, check=True, timeout=210) finally: - result = subprocess.run(["docker", "inspect", "--format", "{{.Id}} {{.State.Running}}", name], - capture_output=True, text=True) - if result.returncode == 0: - container_id, running = result.stdout.strip().split() - if running == "true": - subprocess.run(["docker", "stop", "--time", "5", container_id], check=True) - subprocess.run(["docker", "container", "rm", container_id], check=True) + for container in (name, postgres): + subprocess.run(['docker', 'rm', '-fv', container], capture_output=True) + subprocess.run(['docker', 'network', 'rm', network], capture_output=True) + if __name__ == "__main__": diff --git a/test/deploy/smoke_postgres_role.py b/test/deploy/smoke_postgres_role.py new file mode 100644 index 000000000..4b835c062 --- /dev/null +++ b/test/deploy/smoke_postgres_role.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Verify the Compose bootstrap role in a disposable PostgreSQL container.""" +import os +from pathlib import Path +import secrets +import subprocess +import time +import uuid + +name = 'vm2api-pg-role-test-' + uuid.uuid4().hex[:10] +root = Path(__file__).resolve().parents[2] +env = os.environ.copy() +env['VM2API_POSTGRES_PASSWORD'] = secrets.token_hex(32) +env['PGPASSWORD'] = env['VM2API_POSTGRES_PASSWORD'] +env['POSTGRES_INITDB_ARGS'] = '--auth-host=scram-sha-256' + + +def run(args, check=True): + return subprocess.run(['docker', *args], env=env, capture_output=True, text=True, check=check) + + +try: + run(['run', '-d', '--name', name, '--network', 'none', '-e', 'VM2API_POSTGRES_PASSWORD', '-e', 'POSTGRES_INITDB_ARGS', + '-v', str(root / 'deploy/postgres-entrypoint.sh') + ':/usr/local/bin/vm2api-postgres-entrypoint:ro', + '-v', str(root / 'deploy/postgres-init.sh') + ':/docker-entrypoint-initdb.d/10-vm2api.sh:ro', + '--entrypoint', '/bin/bash', 'postgres:17-bookworm', + '/usr/local/bin/vm2api-postgres-entrypoint', 'postgres']) + deadline = time.monotonic() + 60 + while run(['exec', name, 'psql', '-U', 'vm2api', '-d', 'vm2api', '-Atqc', 'SELECT 1'], False).returncode: + if time.monotonic() > deadline: + raise RuntimeError('isolated role test PostgreSQL did not become ready') + time.sleep(0.25) + flags = run(['exec', name, 'psql', '-U', 'postgres', '-d', 'postgres', '-Atqc', + "SELECT rolsuper,rolcreatedb,rolcreaterole,rolreplication FROM pg_roles WHERE rolname='vm2api'"]).stdout.strip() + owner = run(['exec', name, 'psql', '-U', 'postgres', '-d', 'postgres', '-Atqc', + "SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname='vm2api'"]).stdout.strip() + assert flags == 'f|f|f|f', 'application role permissions differ' + assert owner == 'vm2api', 'application database has a different owner' + run(['exec', '-e', 'PGPASSWORD', name, 'psql', '-h', '127.0.0.1', '-U', 'vm2api', '-d', 'vm2api', + '--no-password', '-v', 'ON_ERROR_STOP=1', '-c', 'CREATE TABLE role_check (id integer); DROP TABLE role_check;']) + bootstrap = run(['exec', '-e', 'PGPASSWORD', name, 'psql', '-h', '127.0.0.1', '-U', 'postgres', '-d', 'postgres', + '--no-password', '-Atqc', 'SELECT 1'], False) + assert bootstrap.returncode != 0 and 'password authentication failed' in bootstrap.stderr, 'bootstrap must use a different password' + denied = run(['exec', '-e', 'PGPASSWORD', name, 'psql', '-h', '127.0.0.1', '-U', 'vm2api', '-d', 'vm2api', + '--no-password', '-v', 'ON_ERROR_STOP=1', '-c', 'CREATE DATABASE forbidden_role_test;'], False) + assert denied.returncode != 0 and 'permission denied' in denied.stderr, 'application role must not create databases' + print('PASS: app role has no superuser/createdb/createrole/replication; owns database; ' + 'password auth and DDL work; bootstrap password differs; CREATE DATABASE denied') +finally: + run(['rm', '-fv', name], False) diff --git a/test/deploy/test_postgres_env.py b/test/deploy/test_postgres_env.py new file mode 100644 index 000000000..79cb4a27f --- /dev/null +++ b/test/deploy/test_postgres_env.py @@ -0,0 +1,36 @@ +"""Validate generated PostgreSQL deployment config without exposing its credentials.""" +from pathlib import Path +import shutil +import stat +import subprocess +import tempfile +import unittest +from urllib.parse import urlparse + + +class PostgreSqlEnvironmentTests(unittest.TestCase): + def test_arm64_initializer_writes_matching_private_database_configuration(self): + with tempfile.TemporaryDirectory(prefix='vm2api-env-') as directory: + root = Path(directory) + (root / 'deploy').mkdir() + script = root / 'deploy' / 'init-arm64-env.py' + shutil.copyfile(Path(__file__).resolve().parents[2] / 'deploy' / 'init-arm64-env.py', script) + result = subprocess.run(['python3', str(script), '--port', '18787'], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + values = dict(line.split('=', 1) for line in (root / '.env').read_text().splitlines() if line and not line.startswith('#')) + url = urlparse(values['VM2API_DATABASE_URL']) + self.assertEqual(url.password, values['VM2API_POSTGRES_PASSWORD']) + self.assertEqual(url.hostname, '127.0.0.1') + self.assertEqual(url.port, 55432) + self.assertEqual(url.path, '/vm2api') + self.assertEqual(values['COMPOSE_PROFILES'], 'postgres') + self.assertEqual(stat.S_IMODE((root / '.env').stat().st_mode), 0o600) + self.assertNotIn(values['VM2API_POSTGRES_PASSWORD'], result.stdout + result.stderr) + original = (root / '.env').read_bytes() + result = subprocess.run(['python3', str(script)], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((root / '.env').read_bytes(), original) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/e2e/backup.e2e.test.mjs b/test/e2e/backup.e2e.test.mjs index d9eac8f4f..eca175bdd 100644 --- a/test/e2e/backup.e2e.test.mjs +++ b/test/e2e/backup.e2e.test.mjs @@ -9,7 +9,6 @@ import os from 'node:os' import path from 'node:path' import zlib from 'node:zlib' import { spawnSync } from 'node:child_process' -import { DatabaseSync } from 'node:sqlite' import { startGateway, api } from '../harness.mjs' test('manual backup → download → mutate → restore rolls back', async () => { @@ -51,10 +50,10 @@ test('manual backup → download → mutate → restore rolls back', async () => fs.writeFileSync(path.join(tmp, 'a.tar.gz'), buf) const tar = spawnSync('tar', ['-xzf', path.join(tmp, 'a.tar.gz'), '-C', tmp]) assert.equal(tar.status, 0) - assert.ok(fs.existsSync(path.join(tmp, 'db', 'kin.db'))) + assert.ok(fs.existsSync(path.join(tmp, 'db', 'postgres.dump'))) assert.ok(fs.existsSync(path.join(tmp, 'manifest.json'))) const manifest = JSON.parse(fs.readFileSync(path.join(tmp, 'manifest.json'), 'utf8')) - assert.equal(manifest.version, 1) + assert.equal(manifest.version, 2) fs.rmSync(tmp, { recursive: true, force: true }) // 4. mutate state after the backup @@ -87,9 +86,6 @@ test('manual backup → download → mutate → restore rolls back', async () => }) assert.equal(m.status, 200, m.text) - // old db kept on disk as a safety net - assert.ok(fs.existsSync(path.join(gw.project, 'data', 'kin.db.pre-restore'))) - // ledger after restore still shows the pre_restore snapshot (undo path) const after = await api(gw, 'GET', '/api/panel/backups') assert.ok( @@ -129,18 +125,13 @@ test('restore rebinds billing, API scheduler, and routing config without restart assert.equal(keyResponse.status, 201, keyResponse.text) const apiKey = keyResponse.json.item.key - const dbPath = path.join(gw.project, 'data', 'kin.db') - const beforeDb = new DatabaseSync(dbPath) - beforeDb.prepare('UPDATE groups SET rate_multiplier = 0 WHERE id = 1').run() - beforeDb.close() + await gw.db.exec('UPDATE groups SET rate_multiplier = 0 WHERE id = 1') const backup = await api(gw, 'POST', '/api/panel/backups') assert.equal(backup.status, 201, backup.text) const backupId = backup.json.item.id - const afterDb = new DatabaseSync(dbPath) - afterDb.prepare('UPDATE groups SET rate_multiplier = 2 WHERE id = 1').run() - afterDb.close() + await gw.db.exec('UPDATE groups SET rate_multiplier = 2 WHERE id = 1') assert.equal((await api(gw, 'DELETE', `/api/panel/api-endpoints/${endpointBeforeId}`)).status, 200) const endpointAfter = await api(gw, 'POST', '/api/panel/api-endpoints', { body: { diff --git a/test/e2e/long-session.e2e.test.mjs b/test/e2e/long-session.e2e.test.mjs index cc0e8cc73..28121ae7e 100644 --- a/test/e2e/long-session.e2e.test.mjs +++ b/test/e2e/long-session.e2e.test.mjs @@ -4,7 +4,7 @@ */ import test from 'node:test' import assert from 'node:assert/strict' -import path from 'node:path' +import { createDatabase } from '../../src/lib/db/database.mjs' import { startGateway, api, takeTrace } from '../harness.mjs' const MODEL = 'claude-haiku-4-5-20251001' @@ -60,7 +60,7 @@ test('same x-session-id keeps native full history without CLI resume', async () } }) -test('one sticky session holds across three protocols and stream', async () => { +test('one sticky session holds across allowed protocols and stream', async () => { const gw = await startGateway({ mockText: 'ack' }) try { const h = { 'x-session-id': 'conv-multi-proto-1', 'x-kin-forward': 'cli' } @@ -86,10 +86,9 @@ test('one sticky session holds across three protocols and stream', async () => { headers: h, body: { model: MODEL, input: 'p-openai-responses' }, }) - assert.equal(c.status, 200, c.text) - const trResp = takeTrace(gw) - assert.equal(trResp.argv, undefined) - assert.match(JSON.stringify(trResp.body.messages), /p-openai-responses/) + assert.equal(c.status, 400, c.text) + assert.equal(c.json.error.code, 'protocol_not_allowed') + assert.equal(takeTrace(gw), null) const res = await fetch(gw.baseUrl + '/v1/messages', { method: 'POST', @@ -117,10 +116,10 @@ test('one sticky session holds across three protocols and stream', async () => { const snap = await api(gw, 'GET', '/admin/routing') assert.equal(snap.status, 200, snap.text) const sessions = snap.json?.sticky?.sessions || {} - const hit = sessions['conv-multi-proto-1'] + const hit = sessions['p:anthropic:conv-multi-proto-1'] assert.ok(hit, `missing sticky bind: ${JSON.stringify(snap.json?.sticky)}`) - assert.equal(hit.session_id, null) - assert.ok(hit.hits >= 4, `hits=${hit.hits}`) + assert.equal(typeof hit.session_id, 'string') + assert.ok(hit.hits >= 3, `hits=${hit.hits}`) assert.equal(hit.vm_id, 'vm-sim-01') assert.equal(hit.account_id, 'acct-seed') } finally { @@ -145,14 +144,14 @@ test('different x-session-id creates independent sticky bindings', async () => { assert.equal(trB.argv, undefined) assert.ok(msgText(trB.body.messages[0]).endsWith('only-B')) const snap = await api(gw, 'GET', '/admin/routing') - assert.ok(snap.json.sticky.sessions['conv-A']) - assert.ok(snap.json.sticky.sessions['conv-B']) + assert.ok(snap.json.sticky.sessions['p:anthropic:conv-A']) + assert.ok(snap.json.sticky.sessions['p:anthropic:conv-B']) } finally { await gw.stop() } }) -test('sticky bindings persist in sqlite after a long sequential session', async () => { +test('sticky bindings persist in PostgreSQL after a long sequential session', async () => { const gw = await startGateway({ mockText: 'n' }) try { const h = { 'x-session-id': 'conv-long-n', 'x-kin-forward': 'cli' } @@ -166,16 +165,16 @@ test('sticky bindings persist in sqlite after a long sequential session', async assert.equal(tr.argv, undefined) assert.ok(msgText(tr.body.messages[0]).endsWith(`n-${i}`)) } - // sticky bindings now live in the SQLite store (data/kin.db) - const { DatabaseSync } = await import('node:sqlite') - const db = new DatabaseSync(path.join(gw.project, 'data', 'kin.db'), { readOnly: true }) + // Shutdown must drain queued writes before reporting the gateway stopped. + await gw.stop() + const db = await createDatabase({ connectionString: gw.connectionString, schema: gw.schema }) try { - const row = db.prepare('SELECT * FROM sticky_sessions WHERE key = ?').get('conv-long-n') + const row = await db.prepare('SELECT * FROM sticky_sessions WHERE key = $1').get('p:anthropic:conv-long-n') assert.ok(row, 'sticky binding row should exist in DB') - assert.equal(row.session_id, null) + assert.equal(typeof row.session_id, 'string') assert.ok(row.hits >= 6) } finally { - db.close() + await db.close() } } finally { await gw.stop() diff --git a/test/e2e/panel-api.e2e.test.mjs b/test/e2e/panel-api.e2e.test.mjs index a807cbc8f..2f5174323 100644 --- a/test/e2e/panel-api.e2e.test.mjs +++ b/test/e2e/panel-api.e2e.test.mjs @@ -33,18 +33,18 @@ test('panel login → cookie → /api/panel/me', async () => { } }) -test('admin database metrics endpoint returns a safe SQLite snapshot', async () => { +test('admin database metrics endpoint returns a safe PostgreSQL snapshot', async () => { const gw = await startGateway() try { const response = await api(gw, 'GET', '/api/panel/database/metrics') assert.equal(response.status, 200, response.text) const data = response.json.data || response.json - assert.equal(data.database.engine, 'sqlite') + assert.equal(data.database.engine, 'postgresql') assert.equal(data.database.ok, true) - assert.equal(data.database.journal_mode, 'wal') + assert.equal('wal_size_bytes' in data.database, false) assert.equal(typeof data.database.probe_latency_ms, 'number') - assert.equal(typeof data.database.file_size_bytes, 'number') - assert.equal(typeof data.database.page_count, 'number') + assert.equal(typeof data.database.size_bytes, 'number') + assert.equal(typeof data.database.pool_total, 'number') assert.equal(typeof data.usage_cache.requests, 'number') assert.ok(data.sampled_at) @@ -202,7 +202,8 @@ test('sessionKey import without SOCKS5 is rejected', async () => { }) test('sessionKey import rejected when bound proxy is dead', async () => { - const gw = await startGateway() + // Slow proxy probing must not overlap the independent boot backup timer. + const gw = await startGateway({ env: { KIN_BACKUP_DISABLED: '1' } }) try { const added = await api(gw, 'POST', '/api/panel/proxies/import', { body: { text: '10.9.9.9:1080' }, @@ -304,7 +305,7 @@ test('generate-auth-url then exchange-code writes fake oauth', async () => { }) test('local egress slot generates auth url and exchanges code', async () => { - const gw = await startGateway({ oauth: false }) + const gw = await startGateway({ oauth: false, env: { KIN_BACKUP_DISABLED: '1' } }) try { const bind = await api(gw, 'POST', '/api/panel/proxies/px-local/bind', { body: { vm_id: 'vm-sim-01' } }) assert.equal(bind.status, 200, bind.text) diff --git a/test/e2e/panel-users.e2e.test.mjs b/test/e2e/panel-users.e2e.test.mjs index 6ecf7d019..41e2d2ced 100644 --- a/test/e2e/panel-users.e2e.test.mjs +++ b/test/e2e/panel-users.e2e.test.mjs @@ -2,7 +2,6 @@ import test from 'node:test' import assert from 'node:assert/strict' import fs from 'node:fs' import path from 'node:path' -import { DatabaseSync } from 'node:sqlite' import { startGateway, api } from '../harness.mjs' async function login(gw, username, password) { @@ -167,13 +166,13 @@ test('deleting a tenant reclaims its assigned VM in the runtime file and SQLite assert.equal(vm.owner_user_id, null) assert.equal(vm.origin, 'platform') assert.equal(vm.schedulable, true) - const db = new DatabaseSync(path.join(gw.project, 'data', 'kin.db'), { readOnly: true }) + const db = gw.db try { - const row = db.prepare('SELECT owner_user_id, origin FROM vms WHERE id = ?').get(vm.id) + const row = await db.prepare('SELECT owner_user_id, origin FROM vms WHERE id = $1').get(vm.id) assert.equal(row.owner_user_id, null) assert.equal(row.origin, 'platform') } finally { - db.close() + // The gateway harness owns this connection. } assert.equal((await panel(gw, 'GET', '/api/panel/me', { cookie: session.cookie })).status, 401) } finally { diff --git a/test/e2e/persistence.e2e.test.mjs b/test/e2e/persistence.e2e.test.mjs index 5c7dd21df..a4dc6d7c8 100644 --- a/test/e2e/persistence.e2e.test.mjs +++ b/test/e2e/persistence.e2e.test.mjs @@ -1,18 +1,17 @@ /** - * E2E: SQLite persistence across gateway restarts + legacy JSON import. + * E2E: PostgreSQL persistence across gateway restarts + legacy JSON import. */ import test from 'node:test' import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { DatabaseSync } from 'node:sqlite' import { startGateway, api } from '../harness.mjs' const MODEL = 'claude-haiku-4-5-20251001' function openDb(gw) { - return new DatabaseSync(path.join(gw.project, 'data', 'kin.db'), { readOnly: true }) + return gw.db } test('data survives a full gateway restart on the same data dir', async () => { @@ -72,7 +71,7 @@ test('vm credentials are mirrored into the vms table (入库)', async () => { // seeded VM has oauth in vms/vm-sim-01.json — reconcile mirrors it at boot const db = openDb(gw) try { - const row = db.prepare('SELECT * FROM vms WHERE id = ?').get('vm-sim-01') + const row = await db.prepare('SELECT * FROM vms WHERE id = $1').get('vm-sim-01') assert.ok(row, 'vms row must exist') assert.equal(row.access_token, null) assert.equal(row.refresh_token, null) @@ -81,10 +80,10 @@ test('vm credentials are mirrored into the vms table (入库)', async () => { assert.equal(vmJson.claude.access_token, undefined) assert.equal(vmJson.claude.has_access, true) assert.equal(vmJson.claude.has_refresh, true) - const active = db.prepare("SELECT value FROM settings WHERE key = 'active_vm'").get() + const active = await db.prepare("SELECT value FROM settings WHERE key = 'active_vm'").get() assert.equal(JSON.parse(active.value), 'vm-sim-01') } finally { - db.close() + // The gateway harness owns this connection. } // sessionKey import (fake) writes through persistOauthToVm → DB mirror updates @@ -94,14 +93,14 @@ test('vm credentials are mirrored into the vms table (入库)', async () => { assert.equal(imp.status, 200, imp.text) const db2 = openDb(gw) try { - const row = db2.prepare('SELECT access_token, session_key, vm_json FROM vms WHERE id = ?').get('vm-sim-01') + const row = await db2.prepare('SELECT access_token, session_key, vm_json FROM vms WHERE id = $1').get('vm-sim-01') assert.equal(row.access_token, null) assert.equal(row.session_key, null) const vmJson = JSON.parse(row.vm_json) assert.equal(vmJson.claude.session_key, undefined) assert.equal(vmJson.claude.has_access, true) } finally { - db2.close() + // The gateway harness owns this connection. } } finally { await gw.stop() @@ -172,11 +171,13 @@ test('legacy JSON files are imported once at first boot', async () => { // import flag recorded; original files untouched const db = openDb(gw) try { - const flag = JSON.parse(db.prepare("SELECT value FROM settings WHERE key = 'legacy_import_done'").get().value) + const flag = JSON.parse( + (await db.prepare("SELECT value FROM settings WHERE key = 'legacy_import_done'").get()).value, + ) assert.equal(flag.counts.api_keys, 1) assert.equal(flag.counts.request_logs, 1) } finally { - db.close() + // The gateway harness owns this connection. } assert.ok(fs.existsSync(path.join(dataDir, 'api-keys.json'))) } finally { diff --git a/test/e2e/request-log.e2e.test.mjs b/test/e2e/request-log.e2e.test.mjs index bf2864cf4..1abd7bf22 100644 --- a/test/e2e/request-log.e2e.test.mjs +++ b/test/e2e/request-log.e2e.test.mjs @@ -4,6 +4,47 @@ import { startGateway, api } from '../harness.mjs' const MODEL = 'claude-haiku-4-5-20251001' +test('NUL-containing rejected requests preserve panel queries and backups', async () => { + const gw = await startGateway({ env: { KIN_REQUEST_LOG_MODE: 'normal', KIN_BACKUP_DISABLED: '1' } }) + try { + for (const mode of ['normal', 'debug']) { + const model = 'invalid\0model' + const rejected = await api(gw, 'POST', '/v1/messages', { + headers: { 'x-kin-log': mode }, + body: { model, max_tokens: 8, messages: [{ role: 'user', content: 'hi' }] }, + }) + assert.equal(rejected.status, 400, rejected.text) + const rid = rejected.headers.get('x-request-id') + const logs = await api(gw, 'GET', '/api/panel/request-logs?mode=normal&limit=10') + assert.equal(logs.status, 200, logs.text) + const row = logs.json.items.find((item) => item.request_id === rid) + assert.ok(row, `rejected ${mode} request must remain logged`) + assert.equal(row.status, 400) + assert.equal(row.model, 'invalid\\u0000model') + assert.equal(row.requested_model, 'invalid\\u0000model') + if (mode === 'debug') { + const debug = await api(gw, 'GET', `/api/panel/request-logs/${rid}`) + assert.equal(debug.status, 200, debug.text) + assert.equal(debug.json.item.inbound_body.model, model) + } + const keys = await api(gw, 'GET', '/api/panel/api-keys') + assert.equal(keys.status, 200, keys.text) + } + const backup = await api(gw, 'POST', '/api/panel/backups') + assert.equal(backup.status, 201, backup.text) + assert.equal(backup.json.item.status, 'ok') + const valid = await api(gw, 'POST', '/v1/messages', { + body: { model: MODEL, max_tokens: 8, messages: [{ role: 'user', content: 'after rejected requests' }] }, + }) + assert.equal(valid.status, 200, valid.text) + const logs = await api(gw, 'GET', '/api/panel/request-logs?mode=normal&limit=10') + assert.equal(logs.status, 200, logs.text) + assert.ok(logs.json.items.some((item) => item.request_id === valid.headers.get('x-request-id'))) + } finally { + await gw.stop() + } +}) + test('normal mode writes summary log with X-Request-ID', async () => { const gw = await startGateway({ mockText: 'logged', env: { KIN_REQUEST_LOG_MODE: 'normal' } }) try { diff --git a/test/harness.mjs b/test/harness.mjs index fd85f1232..82609f8d4 100644 --- a/test/harness.mjs +++ b/test/harness.mjs @@ -9,6 +9,8 @@ import os from 'node:os' import path from 'node:path' import { fileURLToPath } from 'node:url' import { createRequire } from 'node:module' +import crypto from 'node:crypto' +import { createDatabase } from '../src/lib/db/database.mjs' const __dirname = path.dirname(fileURLToPath(import.meta.url)) export const GATEWAY_ROOT = path.resolve(__dirname, '../src') @@ -81,9 +83,20 @@ export async function startGateway(opts = {}) { } const port = opts.port || (await pickFreePort()) const traceFile = opts.traceFile || path.join(project, 'mock-trace.json') + const connectionString = process.env.VM2API_TEST_DATABASE_URL + if (!connectionString || !/_test$/.test(new URL(connectionString).pathname)) { + throw new Error('gateway tests require VM2API_TEST_DATABASE_URL for a dedicated *_test database') + } + const schema = + (process.env.KIN_DATABASE_SCHEMA || 'gateway_test') + + '_gw_' + + crypto.createHash('sha256').update(project).digest('hex').slice(0, 12) const env = { ...process.env, KIN_API_KEY: opts.apiKey || TEST_API_KEY, + VM2API_DATABASE_URL: connectionString, + KIN_DATABASE_URL: connectionString, + KIN_DATABASE_SCHEMA: schema, KIN_ADMIN_USER: 'admin', KIN_ADMIN_PASSWORD: 'testpass', PORT: String(port), @@ -102,6 +115,7 @@ export async function startGateway(opts = {}) { KIN_MOCK_TRACE_FILE: traceFile, KIN_MOCK_TEXT: opts.mockText || 'pong', KIN_CRS_MOCK: '1', + KIN_KERNEL_BIN: path.join(REPO_ROOT, 'bin', 'kin-kernel'), } if (opts.env) Object.assign(env, opts.env) @@ -140,7 +154,11 @@ export async function startGateway(opts = {}) { throw new Error(`gateway failed to start\nstdout:\n${stdout}\nstderr:\n${stderr}`) } + const db = await createDatabase({ connectionString, schema: env.KIN_DATABASE_SCHEMA }) return { + db, + schema: env.KIN_DATABASE_SCHEMA, + connectionString, baseUrl, port, project, @@ -150,10 +168,18 @@ export async function startGateway(opts = {}) { stdout: () => stdout, stderr: () => stderr, async stop() { + await db.close() if (child.exitCode == null) { child.kill('SIGTERM') - await new Promise((r) => setTimeout(r, 150)) - if (child.exitCode == null) child.kill('SIGKILL') + await new Promise((resolve) => { + const timer = setTimeout(() => { + child.kill('SIGKILL') + }, 5000) + child.once('exit', () => { + clearTimeout(timer) + resolve() + }) + }) } }, } diff --git a/test/pg-setup.mjs b/test/pg-setup.mjs new file mode 100644 index 000000000..2f561fdf9 --- /dev/null +++ b/test/pg-setup.mjs @@ -0,0 +1,38 @@ +/** Per-test PostgreSQL isolation for the Node test runner. Never use a deployment URL. */ +import { beforeEach, afterEach } from 'node:test' +import crypto from 'node:crypto' +import pg from 'pg' +import { openDatabase, closeDatabase, closeDatabasesForSchema } from '../src/lib/db/database.mjs' + +const url = process.env.VM2API_TEST_DATABASE_URL +if (!url || !/_test$/.test(new URL(url).pathname)) { + throw new Error('tests require VM2API_TEST_DATABASE_URL pointing to a dedicated *_test database') +} +let schema +beforeEach(async () => { + await closeDatabase() + schema = `test_${process.pid}_${crypto.randomBytes(8).toString('hex')}` + // Restore is restricted to this schema. Avoid DROP DATABASE's global checkpoint. + process.env.VM2API_TEST_DATABASE_URL = url + process.env.KIN_DATABASE_SCHEMA = schema + process.env.KIN_DATABASE_URL = url + process.env.VM2API_DATABASE_URL = url + const db = await openDatabase({ connectionString: url, schema }) + const { loadSettingsState } = await import('../src/lib/db/settings-state.mjs') + await loadSettingsState(db) +}) +afterEach(async () => { + await closeDatabase() + if (!schema) return + await closeDatabasesForSchema(schema, url) + const client = new pg.Client({ connectionString: url }) + await client.connect() + try { + const schemas = (await client.query('SELECT nspname FROM pg_namespace WHERE left(nspname,length($1))=$1', [schema])) + .rows + for (const row of schemas) await client.query(`DROP SCHEMA IF EXISTS "${row.nspname}" CASCADE`) + } finally { + await client.end() + } + delete process.env.KIN_DATABASE_SCHEMA +}) diff --git a/test/unit/account-quota.test.mjs b/test/unit/account-quota.test.mjs index 744dead6d..c6c5b6e98 100644 --- a/test/unit/account-quota.test.mjs +++ b/test/unit/account-quota.test.mjs @@ -10,12 +10,12 @@ function tmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), 'kin-quota-')) } -test('ensure seeds account and is idempotent', () => { - const q = new AccountQuota({ +test('ensure seeds account and is idempotent', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {}, accounts: [{ account_id: 'a1', vm_id: 'vm-1', email: 'a@x' }], - }) + }).init() const acc = q.ensure({ account_id: 'a1' }) assert.equal(acc.vm_id, 'vm-1') assert.equal(acc.email, 'a@x') @@ -23,8 +23,8 @@ test('ensure seeds account and is idempotent', () => { assert.equal(acc.unified['5h'].status, 'active') }) -test('a model entitlement rejection is persisted for that account and model', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('a model entitlement rejection is persisted for that account and model', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'a-denied' }) q.markModelUnsupported('a-denied', 'claude-fable-5') const until = q.repo.get('a-denied').unified.model_denied_until['claude-fable-5'] @@ -38,8 +38,8 @@ function futureReset(msFromNow) { return new Date(Date.now() + msFromNow).toISOString() } -test('ingestHeaders updates unified windows + counters + allocations', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('ingestHeaders updates unified windows + counters + allocations', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestHeaders( 'a2', { @@ -61,8 +61,8 @@ test('ingestHeaders updates unified windows + counters + allocations', () => { assert.ok(acc.unified.headers.sampled_at) }) -test('ingestHeaders countRequest:false does not overwrite Extra with hello 0%', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('ingestHeaders countRequest:false does not overwrite Extra with hello 0%', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestHeaders('a-health', { 'anthropic-ratelimit-unified-5h-utilization': '0.91', 'anthropic-ratelimit-unified-5h-status': 'rejected', @@ -86,8 +86,8 @@ test('ingestHeaders countRequest:false does not overwrite Extra with hello 0%', assert.equal(acc.recent_allocations.length, 1) }) -test('ingestHeaders accumulates cache tokens and mirrors the session window', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('ingestHeaders accumulates cache tokens and mirrors the session window', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() const windows = [] q.attachRuntimeRepo({ updateWindow: (accountId, patch) => windows.push({ accountId, ...patch }) }) q.ingestHeaders( @@ -115,8 +115,8 @@ test('ingestHeaders accumulates cache tokens and mirrors the session window', () assert.equal(windows[0].sessionWindowStatus, 'allowed') }) -test('ingestHeaders reads OpenAI prompt_tokens_details cache fields', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('ingestHeaders reads OpenAI prompt_tokens_details cache fields', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestHeaders( 'a-oa', {}, @@ -133,8 +133,8 @@ test('ingestHeaders reads OpenAI prompt_tokens_details cache fields', () => { assert.equal(acc.cache_creation_tokens, 3) }) -test('test-chat last_probe does not overwrite a successful usage probe', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('test-chat last_probe does not overwrite a successful usage probe', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.recordLastProbe('a-keep', { ok: true, source: 'vm-oauth-usage' }) q.recordLastProbe('a-keep', { ok: false, @@ -146,11 +146,11 @@ test('test-chat last_probe does not overwrite a successful usage probe', () => { assert.equal(acc.unified.last_probe.ok, true) }) -test('canAccept blocks at safety ratio and records last_blocked', () => { - const q = new AccountQuota({ +test('canAccept blocks at safety ratio and records last_blocked', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() q.ingestHeaders('a3', { 'anthropic-ratelimit-unified-5h-utilization': '0.96' }) const gate = q.canAccept('a3') assert.equal(gate.ok, false) @@ -159,19 +159,19 @@ test('canAccept blocks at safety ratio and records last_blocked', () => { assert.equal(acc.last_blocked.window, '5h') }) -test('cli rate_limit_event blocks via status', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('cli rate_limit_event blocks via status', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestCliRateLimit('a4', { rateLimitType: 'five_hour', status: 'rejected', resetsAt: 1893456000 }) const gate = q.canAccept('a4') assert.equal(gate.ok, false) assert.equal(gate.reason, 'quota_5h_cli') }) -test('official 26 percent does not trip the quota gate', () => { - const q = new AccountQuota({ +test('official 26 percent does not trip the quota gate', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true, block_on_7d: true } }, - }) + }).init() q.ingestHeaders('pct-26', { 'anthropic-ratelimit-unified-5h-utilization': '26', 'anthropic-ratelimit-unified-5h-status': 'rejected', @@ -200,8 +200,8 @@ test('extraHeadersFromLimitError fills Extra 5h + reset from the text when wrap assert.equal(kept['anthropic-ratelimit-unified-5h-status'], 'allowed_warning') }) -test('limit error without a fresh reset does not inherit the elapsed reset', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('limit error without a fresh reset does not inherit the elapsed reset', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() const elapsed = String(Math.floor((Date.now() - 3_600_000) / 1000)) q.ingestHeaders('elapsed-5h', { 'anthropic-ratelimit-unified-5h-status': 'allowed', @@ -218,8 +218,8 @@ test('limit error without a fresh reset does not inherit the elapsed reset', () assert.ok(acc.unified.headers.exhausted_at) }) -test('cli rate_limit_event copies unifiedWindows utilization', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('cli rate_limit_event copies unifiedWindows utilization', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestCliRateLimit('a4-util', { rateLimitType: 'five_hour', status: 'allowed', @@ -231,11 +231,11 @@ test('cli rate_limit_event copies unifiedWindows utilization', () => { assert.equal(acc.unified.headers['5h'].status, 'allowed') }) -test('tryAcquire enforces and releases the configured concurrency cap', () => { - const q = new AccountQuota({ +test('tryAcquire enforces and releases the configured concurrency cap', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), accounts: [{ account_id: 'acc-reserve', max_concurrency: 1, concurrency_override: 1 }], - }) + }).init() assert.equal(q.tryAcquire('acc-reserve').ok, true) assert.equal(q.tryAcquire('acc-reserve').reason, 'concurrency_limit') q.release('acc-reserve') @@ -243,12 +243,12 @@ test('tryAcquire enforces and releases the configured concurrency cap', () => { q.release('acc-reserve') }) -test('state persists across re-open (same dataDir)', () => { +test('state persists across re-open (same dataDir)', async () => { const dir = tmpDir() - const q1 = new AccountQuota({ dataDir: dir, config: {} }) + const q1 = await new AccountQuota({ dataDir: dir, config: {} }).init() q1.ingestHeaders('a5', { 'anthropic-ratelimit-unified-7d-utilization': '0.5' }, { input_tokens: 7, output_tokens: 3 }) - const q2 = new AccountQuota({ dataDir: dir, config: {} }) + const q2 = await new AccountQuota({ dataDir: dir, config: {} }).init() const acc = q2.snapshot().accounts.find((a) => a.account_id === 'a5') assert.ok(acc) assert.equal(acc.unified['7d'].utilization, 0.5) @@ -257,8 +257,8 @@ test('state persists across re-open (same dataDir)', () => { assert.equal(acc.recent_allocations.length, 1) }) -test('allocations trimmed to 50 per account', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('allocations trimmed to 50 per account', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() for (let i = 0; i < 60; i++) { q.ingestHeaders('a6', { 'anthropic-ratelimit-unified-5h-utilization': String(i / 100) }) } @@ -268,8 +268,11 @@ test('allocations trimmed to 50 per account', () => { assert.equal(recent[4].util_5h, 0.59) }) -test('max_concurrency 0 blocks instead of treating as unlimited', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: { concurrency: { default_max_per_account: 20 } } }) +test('max_concurrency 0 blocks instead of treating as unlimited', async () => { + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: { concurrency: { default_max_per_account: 20 } }, + }).init() q.ensure({ account_id: 'zero', max_concurrency: 0 }) q.setMaxConcurrency('zero', 0, { override: true }) const gate = q.canAccept('zero') @@ -278,8 +281,8 @@ test('max_concurrency 0 blocks instead of treating as unlimited', () => { assert.equal(gate.detail.max, 0) }) -test('rebindToVm moves the UUID row onto the new slot', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('rebindToVm moves the UUID row onto the new slot', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: '1c5a7a73', vm_id: 'vm-02', email: 'old@x' }) q.rebindToVm('1c5a7a73', 'vm-04', { email: 'new@x' }) const acc = q.snapshot().accounts.find((a) => a.account_id === '1c5a7a73') @@ -287,8 +290,11 @@ test('rebindToVm moves the UUID row onto the new slot', () => { assert.equal(acc.email, 'new@x') }) -test('concurrency inflight gate stays in memory', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: { concurrency: { default_max_per_account: 1 } } }) +test('concurrency inflight gate stays in memory', async () => { + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: { concurrency: { default_max_per_account: 1 } }, + }).init() q.ensure({ account_id: 'a7' }) q.setMaxConcurrency('a7', 1, { override: true }) q.acquire('a7') @@ -299,8 +305,8 @@ test('concurrency inflight gate stays in memory', () => { assert.equal(q.canAccept('a7').ok, true) }) -test('rpm window blocks then recovers', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('rpm window blocks then recovers', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'rpm-acc', max_concurrency: 8 }) q.setMaxRpm('rpm-acc', 2, { override: true }) assert.equal(q.tryAcquire('rpm-acc').ok, true) @@ -314,8 +320,8 @@ test('rpm window blocks then recovers', () => { assert.ok(gate.detail.reset_at > Date.now()) }) -test('max_rpm 0 is unlimited', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('max_rpm 0 is unlimited', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'rpm-off', max_concurrency: 8, max_rpm: 0 }) for (let i = 0; i < 5; i++) { assert.equal(q.tryAcquire('rpm-off').ok, true) @@ -324,11 +330,11 @@ test('max_rpm 0 is unlimited', () => { assert.equal(q.canAccept('rpm-off').ok, true) }) -test('quota gate still blocks before RPM wait', () => { - const q = new AccountQuota({ +test('quota gate still blocks before RPM wait', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true, block_on_7d: true } }, - }) + }).init() q.ensure({ account_id: 'rpm-quota', max_concurrency: 8, max_rpm: 60 }) q.ingestHeaders('rpm-quota', { 'anthropic-ratelimit-unified-5h-utilization': '1', @@ -340,11 +346,11 @@ test('quota gate still blocks before RPM wait', () => { assert.equal(gate.reason, 'quota_5h_cli') }) -test('ingestOAuthUsage stores 5h/7d and isolated fable limit', () => { - const q = new AccountQuota({ +test('ingestOAuthUsage stores 5h/7d and isolated fable limit', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() q.ingestOAuthUsage('acc-fable', { ok: true, five_hour: { utilization: 0.2, resets_at: '2026-08-18T20:00:00Z', status: 'allowed' }, @@ -374,11 +380,11 @@ test('ingestOAuthUsage stores 5h/7d and isolated fable limit', () => { assert.equal(gate.ok, true, 'fable weekly limit must not block the whole account') }) -test('official /usage 429 keeps the last successful probe', () => { - const q = new AccountQuota({ +test('official /usage 429 keeps the last successful probe', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() q.ingestOAuthUsage('acc-rl', { ok: true, source: 'official-cc-usage', @@ -404,11 +410,11 @@ test('official /usage 429 keeps the last successful probe', () => { assert.ok(Date.parse(acc.unified.usage_rate_limited_until) > Date.now()) }) -test('usage ok plus fable 401 does not poison last_probe with revoke text', () => { - const q = new AccountQuota({ +test('usage ok plus fable 401 does not poison last_probe with revoke text', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() q.ingestOAuthUsage('acc-pro-401', { ok: true, usage_status: 200, @@ -436,8 +442,8 @@ test('usage ok plus fable 401 does not poison last_probe with revoke text', () = assert.equal(after.unified.account_tier, undefined) }) -test('Fable 403 does not classify Pro when official usage cannot verify the grant', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('Fable 403 does not classify Pro when official usage cannot verify the grant', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestOAuthUsage('acc-uncertain-403', { ok: false, usage_status: 500, @@ -449,8 +455,8 @@ test('Fable 403 does not classify Pro when official usage cannot verify the gran assert.equal(acc.unified.fable.plan_denied, false) }) -test('official usage without Fable evidence does not classify an unprobed account as Pro', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('official usage without Fable evidence does not classify an unprobed account as Pro', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestOAuthUsage('acc-unprobed', { ok: true, usage_status: 200, @@ -463,8 +469,8 @@ test('official usage without Fable evidence does not classify an unprobed accoun assert.equal(acc.unified.fable, undefined) }) -test('complete official usage without Fable classifies Pro', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('complete official usage without Fable classifies Pro', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestOAuthUsage('acc-complete-pro', { ok: true, usage_status: 200, @@ -479,8 +485,8 @@ test('complete official usage without Fable classifies Pro', () => { assert.equal(acc.unified.account_tier, 'pro') }) -test('complete official usage overrides a stale profile tier', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('complete official usage overrides a stale profile tier', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'acc-stale-profile' }) q.setAccountTier('acc-stale-profile', 'max', { source: 'profile' }) q.ingestOAuthUsage('acc-stale-profile', { @@ -497,8 +503,8 @@ test('complete official usage overrides a stale profile tier', () => { assert.equal(acc.unified.account_tier_source, 'usage') }) -test('setup-token usage scope failure preserves historical tier and does not mark grant dead', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('setup-token usage scope failure preserves historical tier and does not mark grant dead', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'acc-setup-scope' }) q.setAccountTier('acc-setup-scope', 'max', { source: 'usage' }) q.ingestOAuthUsage('acc-setup-scope', { @@ -515,8 +521,8 @@ test('setup-token usage scope failure preserves historical tier and does not mar assert.match(acc.unified.last_probe.error, /user:profile/) }) -test('incomplete official usage without Fable does not downgrade a stored Max', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('incomplete official usage without Fable does not downgrade a stored Max', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'acc-incomplete-max' }) q.setAccountTier('acc-incomplete-max', 'max', { source: 'usage' }) q.ingestOAuthUsage('acc-incomplete-max', { @@ -531,11 +537,11 @@ test('incomplete official usage without Fable does not downgrade a stored Max', assert.equal(acc.unified.account_tier, 'max') }) -test('clearGrantRevokeLeftover drops stale revoke after refresh', () => { - const q = new AccountQuota({ +test('clearGrantRevokeLeftover drops stale revoke after refresh', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() q.ingestOAuthUsage('acc-stale', { ok: false, usage_status: 401, @@ -552,11 +558,11 @@ test('clearGrantRevokeLeftover drops stale revoke after refresh', () => { assert.equal(acc.unified.account_tier, undefined) }) -test('clearGrantRevokeLeftover keeps stored Max off leftover Fable revoke', () => { - const q = new AccountQuota({ +test('clearGrantRevokeLeftover keeps stored Max off leftover Fable revoke', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() q.ingestOAuthUsage('acc-max', { ok: true, usage_status: 200, @@ -583,11 +589,11 @@ test('clearGrantRevokeLeftover keeps stored Max off leftover Fable revoke', () = assert.equal(acc.unified.fable.plan_denied, false) }) -test('fable 429 without 7d_oi window does not invent a full Fable quota', () => { - const q = new AccountQuota({ +test('fable 429 without 7d_oi window does not invent a full Fable quota', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() q.ingestOAuthUsage('acc-pro', { ok: true, five_hour: { utilization: 0.1, status: 'allowed' }, @@ -604,11 +610,11 @@ test('fable 429 without 7d_oi window does not invent a full Fable quota', () => assert.equal(q.fableWindowLimited('acc-pro'), false) }) -test('official usage without Fable hop does not turn leftover 429 into Pro', () => { - const q = new AccountQuota({ +test('official usage without Fable hop does not turn leftover 429 into Pro', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() q.ingestOAuthUsage('acc-pro-skip', { ok: true, five_hour: { utilization: 0.1, status: 'allowed' }, @@ -640,11 +646,11 @@ test('official usage without Fable hop does not turn leftover 429 into Pro', () assert.equal(acc.unified.account_tier, undefined) }) -test('Fable hop alone does not classify Pro; complete usage can flip stored Pro to Max', () => { - const q = new AccountQuota({ +test('Fable hop alone does not classify Pro; complete usage can flip stored Pro to Max', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() q.ingestOAuthUsage('acc-mispro', { ok: true, usage_status: 200, @@ -673,11 +679,11 @@ test('Fable hop alone does not classify Pro; complete usage can flip stored Pro assert.equal(acc.unified.fable.ok, true) }) -test('7d_oi window is Fable-only and does not block the account', () => { - const q = new AccountQuota({ +test('7d_oi window is Fable-only and does not block the account', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() const reset = futureReset(7 * 24 * 3600_000) q.ingestHeaders('acc-oi', { 'anthropic-ratelimit-unified-7d_oi-utilization': '1', @@ -690,11 +696,11 @@ test('7d_oi window is Fable-only and does not block the account', () => { assert.equal(q.fableWindowResetAt('acc-oi'), Date.parse(reset)) }) -test('7d_oi rejected with elapsed second-level reset is not Fable-limited', () => { - const q = new AccountQuota({ +test('7d_oi rejected with elapsed second-level reset is not Fable-limited', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() const now = Date.parse('2026-10-19T09:00:00Z') const resetSec = '1792206000' const acc = q.ensure({ account_id: 'acc-oi-elapsed' }) @@ -710,8 +716,8 @@ test('7d_oi rejected with elapsed second-level reset is not Fable-limited', () = assert.equal(q.fableWindowResetAt('acc-oi-elapsed'), parseResetMs(resetSec)) }) -test('7d_oi rejected without reset stays Fable-limited', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('7d_oi rejected without reset stays Fable-limited', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() const acc = q.ensure({ account_id: 'acc-oi-no-reset' }) acc.unified['7d_oi'] = { utilization: 1, status: 'rejected' } q.repo.save(acc) @@ -719,8 +725,8 @@ test('7d_oi rejected without reset stays Fable-limited', () => { assert.equal(q.fableWindowResetAt('acc-oi-no-reset'), null) }) -test('7d_oi rejected with future second-level reset stays Fable-limited', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('7d_oi rejected with future second-level reset stays Fable-limited', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() const now = Date.parse('2026-10-19T09:00:00Z') const resetSec = '1792810800' const acc = q.ensure({ account_id: 'acc-oi-future' }) @@ -731,8 +737,8 @@ test('7d_oi rejected with future second-level reset stays Fable-limited', () => assert.equal(q.fableWindowResetAt('acc-oi-future'), parseResetMs(resetSec)) }) -test('persistEffectiveWindows clears elapsed unified 7d_oi', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('persistEffectiveWindows clears elapsed unified 7d_oi', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() const now = Date.parse('2026-10-19T09:00:00Z') const acc = q.ensure({ account_id: 'acc-wipe-oi' }) acc.unified['7d_oi'] = { utilization: 1, reset: '1792206000', status: 'rejected' } @@ -745,11 +751,11 @@ test('persistEffectiveWindows clears elapsed unified 7d_oi', () => { assert.equal(oi.stale_reason, 'reset_elapsed') }) -test('probe 5h rejected with no last_used stays a hard block until reset', () => { - const q = new AccountQuota({ +test('probe 5h rejected with no last_used stays a hard block until reset', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() const reset = new Date(Date.now() + 4 * 3600_000).toISOString() q.ingestOAuthUsage('acc-sticky', { ok: true, @@ -763,8 +769,8 @@ test('probe 5h rejected with no last_used stays a hard block until reset', () => assert.equal(acc.unified['5h'].status, 'rejected') }) -test('official probe fills Extra reset when Extra has usage but no reset', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('official probe fills Extra reset when Extra has usage but no reset', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestHeaders('acc-no-reset', { 'anthropic-ratelimit-unified-5h-utilization': '0.44', 'anthropic-ratelimit-unified-5h-status': 'allowed', @@ -783,11 +789,11 @@ test('official probe fills Extra reset when Extra has usage but no reset', () => assert.equal(acc.unified.headers['5h'].reset, reset) }) -test('official leftover 100% does not overwrite live Extra allowed', () => { - const q = new AccountQuota({ +test('official leftover 100% does not overwrite live Extra allowed', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() q.ingestHeaders('acc-live', { 'anthropic-ratelimit-unified-5h-utilization': '0.36', 'anthropic-ratelimit-unified-5h-status': 'allowed', @@ -807,8 +813,8 @@ test('official leftover 100% does not overwrite live Extra allowed', () => { assert.equal(q.canAccept('acc-live').ok, true) }) -test('successful probe overwrites leftover rejected when official percent is low', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('successful probe overwrites leftover rejected when official percent is low', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestHeaders( 'acc-stale-status', { @@ -831,11 +837,11 @@ test('successful probe overwrites leftover rejected when official percent is low assert.equal(q.canAccept('acc-stale-status').ok, true) }) -test('probe 5h rejected stays blocked when last_used is missing', () => { - const q = new AccountQuota({ +test('probe 5h rejected stays blocked when last_used is missing', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() q.ingestOAuthUsage('acc-no-used', { ok: true, source: 'official-cc-usage', @@ -852,11 +858,11 @@ test('probe 5h rejected stays blocked when last_used is missing', () => { assert.equal(gate.reason, 'quota_5h_cli') }) -test('probe 5h rejected stays blocked when the slot was used in this window', () => { - const q = new AccountQuota({ +test('probe 5h rejected stays blocked when the slot was used in this window', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() const reset = new Date(Date.now() + 4 * 3600_000).toISOString() q.attachRuntimeRepo({ get: () => ({ last_used_at: Date.now() - 60_000 }), @@ -872,8 +878,8 @@ test('probe 5h rejected stays blocked when the slot was used in this window', () assert.equal(gate.reason, 'quota_5h_cli') }) -test('persistEffectiveWindows writes a cleared 5h after reset elapsed', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('persistEffectiveWindows writes a cleared 5h after reset elapsed', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestOAuthUsage('acc-persist', { ok: true, source: 'vm-oauth-usage', @@ -892,11 +898,11 @@ test('persistEffectiveWindows writes a cleared 5h after reset elapsed', () => { assert.equal(q.canAccept('acc-persist').ok, true) }) -test('elapsed 5h reset is not a hard block even after a live header write', () => { - const q = new AccountQuota({ +test('elapsed 5h reset is not a hard block even after a live header write', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init() q.ingestHeaders('acc-elapsed', { 'anthropic-ratelimit-unified-5h-utilization': '1', 'anthropic-ratelimit-unified-5h-status': 'rejected', @@ -905,8 +911,8 @@ test('elapsed 5h reset is not a hard block even after a live header write', () = assert.equal(q.canAccept('acc-elapsed').ok, true) }) -test('429 unified exhausted marks headers rejected without inventing official 100%', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('429 unified exhausted marks headers rejected without inventing official 100%', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestHeaders( 'acc-429', { @@ -923,8 +929,8 @@ test('429 unified exhausted marks headers rejected without inventing official 10 assert.equal(q.canAccept('acc-429').ok, false) }) -test('local 429 without quota evidence does not reject the 5h window', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('local 429 without quota evidence does not reject the 5h window', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestHeaders('acc-local-429', { 'anthropic-ratelimit-unified-5h-utilization': '0.23', 'anthropic-ratelimit-unified-5h-status': 'allowed', @@ -956,11 +962,11 @@ test('local 429 without quota evidence does not reject the 5h window', () => { assert.equal(q.canAccept('acc-header-429').ok, false) }) -test('stuck 5h header with growing tokens does not lift official utilization', () => { - const q = new AccountQuota({ +test('stuck 5h header with growing tokens does not lift official utilization', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { block_on_5h: true }, tiers: { default: { limit_5h: 0.85 } } }, - }) + }).init() q.ingestOAuthUsage('acc-stall', { ok: true, source: 'official-cc-usage', @@ -985,11 +991,11 @@ test('stuck 5h header with growing tokens does not lift official utilization', ( assert.equal(q.canAccept('acc-stall').ok, true) }) -test('official /usage 40 percent is 0.40 and stays under the 80 percent gate', () => { - const q = new AccountQuota({ +test('official /usage 40 percent is 0.40 and stays under the 80 percent gate', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, weekly_safety_ratio: 0.8, block_on_5h: true, block_on_7d: true } }, - }) + }).init() q.ingestOAuthUsage('acc-40', { ok: true, source: 'official-cc-usage', @@ -1006,11 +1012,11 @@ test('official /usage 40 percent is 0.40 and stays under the 80 percent gate', ( assert.equal(q.canAccept('acc-40').ok, true) }) -test('persistEffectiveWindows clears leftover quota cooldown when official is under the gate', () => { - const q = new AccountQuota({ +test('persistEffectiveWindows clears leftover quota cooldown when official is under the gate', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true, block_on_7d: true } }, - }) + }).init() const runtime = new Map() q.attachRuntimeRepo({ get: (id) => runtime.get(id) || null, @@ -1043,11 +1049,11 @@ test('persistEffectiveWindows clears leftover quota cooldown when official is un assert.equal(q.canAccept('acc-reconcile').ok, true) }) -test('official 0.44 allowed overwrites local 0.9 and clears quota cooldown', () => { - const q = new AccountQuota({ +test('official 0.44 allowed overwrites local 0.9 and clears quota cooldown', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true, block_on_7d: true } }, - }) + }).init() const runtime = new Map() q.attachRuntimeRepo({ get: (id) => runtime.get(id) || null, @@ -1085,8 +1091,11 @@ test('official 0.44 allowed overwrites local 0.9 and clears quota cooldown', () assert.equal(runtime.get('acc-clear').cooldown_reason, null) }) -test('/usage 40 writes Extra 0.40; 1 percent writes 0.01 not 1.0', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }) +test('/usage 40 writes Extra 0.40; 1 percent writes 0.01 not 1.0', async () => { + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: { quota: { safety_ratio: 0.85, block_on_5h: true } }, + }).init() q.ingestOAuthUsage('acc-scale', { ok: true, source: 'official-cc-usage', @@ -1104,11 +1113,11 @@ test('/usage 40 writes Extra 0.40; 1 percent writes 0.01 not 1.0', () => { assert.notEqual(acc.unified.headers['7d'].utilization, 1) }) -test('official 0.90 plus header 0.40 is schedulable', () => { - const q = new AccountQuota({ +test('official 0.90 plus header 0.40 is schedulable', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, weekly_safety_ratio: 0.8, block_on_5h: true, block_on_7d: true } }, - }) + }).init() const acc = q.ensure({ account_id: 'acc-mix-open' }) acc.unified.official = { '5h': { utilization: 0.9, status: 'allowed', reset: futureReset(4 * 3600_000) }, @@ -1125,11 +1134,11 @@ test('official 0.90 plus header 0.40 is schedulable', () => { assert.equal(q.canAccept('acc-mix-open').ok, true) }) -test('header 0.90 plus official 0.40 hits the 5h gate', () => { - const q = new AccountQuota({ +test('header 0.90 plus official 0.40 hits the 5h gate', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, weekly_safety_ratio: 0.8, block_on_5h: true, block_on_7d: true } }, - }) + }).init() const acc = q.ensure({ account_id: 'acc-mix-shut' }) acc.unified.official = { '5h': { utilization: 0.4, status: 'allowed', reset: futureReset(4 * 3600_000) }, @@ -1144,8 +1153,11 @@ test('header 0.90 plus official 0.40 hits the 5h gate', () => { assert.equal(gate.reason, 'quota_5h_safety') }) -test('elapsed Extra is not restored from leftover official 100%', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }) +test('elapsed Extra is not restored from leftover official 100%', async () => { + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: { quota: { safety_ratio: 0.85, block_on_5h: true } }, + }).init() const nextReset = new Date(Date.now() + 4 * 3600_000).toISOString() q.ingestOAuthUsage('acc-restore', { ok: true, @@ -1167,8 +1179,11 @@ test('elapsed Extra is not restored from leftover official 100%', () => { assert.equal(q.canAccept('acc-restore').ok, true) }) -test('already-wiped Extra stays empty when official leftover is still 100%', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }) +test('already-wiped Extra stays empty when official leftover is still 100%', async () => { + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: { quota: { safety_ratio: 0.85, block_on_5h: true } }, + }).init() const nextReset = new Date(Date.now() + 4 * 3600_000).toISOString() q.ingestOAuthUsage('acc-stale-wipe', { ok: true, @@ -1192,8 +1207,11 @@ test('already-wiped Extra stays empty when official leftover is still 100%', () assert.equal(q.canAccept('acc-stale-wipe').ok, true) }) -test('persistEffectiveWindows heals Extra overwritten by leftover official 100%', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }) +test('persistEffectiveWindows heals Extra overwritten by leftover official 100%', async () => { + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: { quota: { safety_ratio: 0.85, block_on_5h: true } }, + }).init() const reset = futureReset(4 * 3600_000) q.ingestHeaders('acc-heal', { 'anthropic-ratelimit-unified-5h-utilization': '0.36', @@ -1217,8 +1235,11 @@ test('persistEffectiveWindows heals Extra overwritten by leftover official 100%' assert.equal(q.canAccept('acc-heal').ok, true) }) -test('elapsed header reset wipes Extra and becomes schedulable', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }) +test('elapsed header reset wipes Extra and becomes schedulable', async () => { + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: { quota: { safety_ratio: 0.85, block_on_5h: true } }, + }).init() q.ingestHeaders('acc-wipe', { 'anthropic-ratelimit-unified-5h-utilization': '0.91', 'anthropic-ratelimit-unified-5h-status': 'rejected', @@ -1231,7 +1252,7 @@ test('elapsed header reset wipes Extra and becomes schedulable', () => { }) test('complete usage may replace a profile-sourced tier', async () => { - const quota = new AccountQuota({ dataDir: tmpDir(), config: {} }) + const quota = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() quota.ensure({ account_id: 'acct-profile', vm_id: 'vm-01' }) quota.setAccountTier('acct-profile', 'pro', { source: 'profile' }) quota.setAccountTier('acct-profile', 'max') @@ -1248,3 +1269,83 @@ test('complete usage may replace a profile-sourced tier', async () => { quota.setAccountTier('acct-profile', 'max', { source: 'profile' }) assert.equal(quota.repo.get('acct-profile').unified.account_tier, 'max') }) + +test('concurrent completed hops persist every quota counter update', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() + q.ensure({ account_id: 'parallel-account' }) + await Promise.all( + Array.from({ length: 20 }, () => + Promise.resolve().then(() => { + q.ingestHeaders('parallel-account', {}, { input_tokens: 3, output_tokens: 2 }) + }), + ), + ) + await q.flush() + const stored = await q.persistence.get('parallel-account') + assert.equal(stored.requests, 20) + assert.equal(stored.tokens_in, 60) + assert.equal(stored.tokens_out, 40) +}) + +test('restore reload does not recreate accounts from the original startup seed', async () => { + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: {}, + accounts: [{ account_id: 'old-startup-account' }], + }).init() + await q.persistence.remove('old-startup-account') + await q.rebind(q.db) + assert.equal(q.repo.get('old-startup-account'), null) + assert.equal(await q.persistence.get('old-startup-account'), null) +}) + +for (const source of ['headers', 'cli']) { + test(`${source} quota counters add exactly beyond JavaScript safe integers`, async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() + q.ensure({ account_id: 'large-counters' }) + await q.flush() + await q.db + .prepare(`UPDATE accounts SET requests = $1, tokens_in = $1, tokens_out = $1, + cache_read_tokens = $1, cache_creation_tokens = $1 WHERE id = $2`) + .run('9007199254740993', 'large-counters') + await q.reload() + if (source === 'headers') + q.ingestHeaders( + 'large-counters', + {}, + { + input_tokens: 2, + output_tokens: 3, + cache_read_input_tokens: 4, + cache_creation_input_tokens: 5, + }, + ) + else q.ingestCliRateLimit('large-counters', null, { input_tokens: 2, output_tokens: 3 }) + await q.flush() + const account = await q.persistence.get('large-counters') + assert.equal(account.requests, '9007199254740994') + assert.equal(account.tokens_in, '9007199254740995') + assert.equal(account.tokens_out, '9007199254740996') + assert.equal(account.cache_read_tokens, source === 'headers' ? '9007199254740997' : '9007199254740993') + assert.equal(account.cache_creation_tokens, source === 'headers' ? '9007199254740998' : '9007199254740993') + q.ensure({ account_id: 'large-delta' }) + if (source === 'headers') + q.ingestHeaders( + 'large-delta', + {}, + { + input_tokens: '9007199254740993', + output_tokens: '9007199254740995', + }, + ) + else + q.ingestCliRateLimit('large-delta', null, { + input_tokens: '9007199254740993', + output_tokens: '9007199254740995', + }) + await q.flush() + const delta = await q.persistence.get('large-delta') + assert.equal(delta.tokens_in, '9007199254740993') + assert.equal(delta.tokens_out, '9007199254740995') + }) +} diff --git a/test/unit/anthropic-console.test.mjs b/test/unit/anthropic-console.test.mjs index 26a04b4e8..17d76649e 100644 --- a/test/unit/anthropic-console.test.mjs +++ b/test/unit/anthropic-console.test.mjs @@ -462,7 +462,7 @@ test('saved Anthropic row reloads its proxy and can test or fetch models without const savedKey = response.body.item.api_key_entries[0] assert.equal(savedKey.proxy_id, 'px-1') - const reloaded = new ApiEndpointStore({ dataDir: dir }).list() + const reloaded = await new ApiEndpointStore({ dataDir: dir }).list() assert.equal(reloaded[0].api_key_entries[0].proxy_id, 'px-1') assert.equal(reloaded[0].api_key_entries[0].api_key.includes('sk-ant-saved'), false) diff --git a/test/unit/api-backend.test.mjs b/test/unit/api-backend.test.mjs index 587be2c9d..bec7682b0 100644 --- a/test/unit/api-backend.test.mjs +++ b/test/unit/api-backend.test.mjs @@ -1,3 +1,4 @@ +import { flushPendingWrites } from '../../src/lib/db/pending-writes.mjs' import test from 'node:test' import assert from 'node:assert/strict' import fs from 'node:fs' @@ -295,6 +296,7 @@ test('logs the session id actually sent upstream, not the caller one', async () try { await handler.handleProtocol(req, response, 'anthropic.messages', '/v1/messages') response.finish?.() + await flushPendingWrites() const sent = sessionIdFromOutboundBody(prepared.body) || prepared.meta.sessionId assert.ok(sent) assert.equal(logged.session_id, 'caller-sess') @@ -310,7 +312,10 @@ for (const previousVm of [null, 'vm-01']) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-queued-session-')) const routingFile = path.join(root, 'routing.json') fs.writeFileSync(routingFile, JSON.stringify({ compatibility: {} })) - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() const key = sticky.canonicalSessionKey('queued-caller') if (previousVm) { sticky.bind(key, { accountId: 'account-1', vmId: previousVm, sessionId: 'previous-outbound' }) @@ -339,9 +344,9 @@ for (const previousVm of [null, 'vm-01']) { const secondReady = new Promise((resolve) => { enteredSecond = resolve }) - t.after(() => { + t.after(async () => { releaseFirst() - sticky.db.close() + await sticky.db.close() fs.rmSync(root, { recursive: true, force: true }) }) const sent = [] @@ -833,13 +838,16 @@ function sessionBody(userId, extra = {}) { } } -function realStickyRouter() { +async function realStickyRouter() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-identity-sticky-')) - return { dir, router: new StickyRouter({ dataDir: dir, config: { sticky: { enabled: true, ttl_seconds: 600 } } }) } + return { + dir, + router: await new StickyRouter({ dataDir: dir, config: { sticky: { enabled: true, ttl_seconds: 600 } } }).init(), + } } test('inbound session and device keys match across API keys at the protocol entry', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const body = sessionBody({ device_id: 'dev-cross', account_uuid: 'acct-a', session_id: 'sess-cross' }) const a = await captureRunOpts({ body, stickyRouter: router, apiKeyRecord: { id: 'key-a', group_id: 1 } }) @@ -864,7 +872,7 @@ test('inbound session and device keys match across API keys at the protocol entr }) test('the protocol entry hands the device seat key to the pool, independent of the API key', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const turn = { messages: [{ role: 'user', content: 'please refactor the scheduler module and explain every single change' }], @@ -888,7 +896,7 @@ test('the protocol entry hands the device seat key to the pool, independent of t }) test('shared API key keeps different devices apart at the protocol entry', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const apiKeyRecord = { id: 'shared-key', group_id: 1 } const a = await captureRunOpts({ @@ -909,7 +917,7 @@ test('shared API key keeps different devices apart at the protocol entry', async }) test('explicit body device_id is the fallback when metadata has no device', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const opts = await captureRunOpts({ body: sessionBody({ session_id: 'sess-fallback' }, { device_id: 'dev-explicit' }), @@ -925,7 +933,7 @@ test('explicit body device_id is the fallback when metadata has no device', asyn }) test('legacy API-key-scoped session row migrates lazily at the protocol entry', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const apiKeyRecord = { id: 'key-old', group_id: 1 } const legacyKey = 'p:anthropic:kkey-old:sess-legacy' @@ -959,7 +967,7 @@ test('legacy API-key-scoped session row migrates lazily at the protocol entry', }) test('parent and child family key is shared across API keys and inherits a live legacy family', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { router.bind('p:anthropic:kkey-p:family:parent-sess', { accountId: 'acc-fam', vmId: 'vm-fam' }) const parent = await captureRunOpts({ @@ -983,15 +991,15 @@ test('parent and child family key is shared across API keys and inherits a live }) test('Claude Code sub-agent hops run as child sessions of the main session', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const apiKeyRecord = { id: 'key-cc', group_id: 1 } const body = sessionBody({ device_id: 'dev-cc', session_id: 'main-sess' }) const main = await captureRunOpts({ body, stickyRouter: router, apiKeyRecord }) router.bind(main.stickyKey, { accountId: 'acc-cc', vmId: 'vm-cc', slotIndex: 0 }) router.bind(main.familyKey, { accountId: 'acc-cc', vmId: 'vm-cc' }) - const agentHop = (agentId, parentAgentId) => - captureRunOpts({ + const agentHop = async (agentId, parentAgentId) => + await captureRunOpts({ body, stickyRouter: router, apiKeyRecord, @@ -1027,7 +1035,7 @@ test('Claude Code sub-agent hops run as child sessions of the main session', asy }) test('request without trusted session id keeps the legacy scoped sticky key', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const body = { model: 'claude-sonnet-4-5', @@ -1044,7 +1052,7 @@ test('request without trusted session id keeps the legacy scoped sticky key', as }) test('one-shot test call skips the session seat but keeps its sticky key', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const probe = await captureRunOpts({ body: sessionBody({ device_id: 'dev-probe', session_id: 'sess-probe' }, { max_tokens: 1024 }), diff --git a/test/unit/api-endpoints.test.mjs b/test/unit/api-endpoints.test.mjs index 9ce59bfeb..1d96decb6 100644 --- a/test/unit/api-endpoints.test.mjs +++ b/test/unit/api-endpoints.test.mjs @@ -10,9 +10,9 @@ function tmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), 'kin-api-ep-')) } -test('create endpoint with keys and models, list masks keys', () => { +test('create endpoint with keys and models, list masks keys', async () => { const store = new ApiEndpointStore({ dataDir: tmpDir() }) - const rec = store.create({ + const rec = await store.create({ name: 'openrouter', base_url: 'https://example.com', prefix: 'or', @@ -22,14 +22,14 @@ test('create endpoint with keys and models, list masks keys', () => { assert.equal(rec.base_url, 'https://example.com') assert.equal(rec.api_key_entries.length, 1) assert.equal(rec.models[0].alias, 'kimi') - const listed = store.list() + const listed = await store.list() assert.ok(listed[0].api_key_entries[0].api_key.includes('…')) assert.ok(!listed[0].api_key_entries[0].api_key.includes('secret-key')) }) -test('create strips trailing /v1 from base_url', () => { +test('create strips trailing /v1 from base_url', async () => { const store = new ApiEndpointStore({ dataDir: tmpDir() }) - const rec = store.create({ + const rec = await store.create({ name: 'a', base_url: 'https://example.com/v1/', api_key_entries: [{ api_key: 'sk-or-xxxx' }], @@ -38,23 +38,23 @@ test('create strips trailing /v1 from base_url', () => { assert.equal(rec.base_url, 'https://example.com') }) -test('official kinds lock base_url', () => { +test('official kinds lock base_url', async () => { const store = new ApiEndpointStore({ dataDir: tmpDir() }) - const claude = store.create({ kind: 'claude', api_key_entries: [{ api_key: 'sk-ant-xxxx' }] }) + const claude = await store.create({ kind: 'claude', api_key_entries: [{ api_key: 'sk-ant-xxxx' }] }) assert.equal(claude.kind, 'claude') assert.equal(claude.base_url, 'https://api.anthropic.com') assert.equal(claude.protocol, 'anthropic') - const openai = store.create({ kind: 'openai', name: 'oai', api_key_entries: [{ api_key: 'sk-oai-xxxx' }] }) + const openai = await store.create({ kind: 'openai', name: 'oai', api_key_entries: [{ api_key: 'sk-oai-xxxx' }] }) assert.equal(openai.kind, 'openai') assert.equal(openai.base_url, 'https://api.openai.com') assert.equal(openai.protocol, 'openai') }) -test('sk-kin category defaults oauth and can switch to api', () => { +test('sk-kin category defaults oauth and can switch to api', async () => { const keys = new ApiKeyStore({ dataDir: tmpDir() }) - const rec = keys.create({ name: 'c' }) + const rec = await keys.create({ name: 'c' }) assert.equal(rec.category, 'oauth') - const updated = keys.update(rec.id, { category: 'api' }) + const updated = await keys.update(rec.id, { category: 'api' }) assert.equal(updated.category, 'api') - assert.equal(keys.list()[0].category, 'api') + assert.equal((await keys.list())[0].category, 'api') }) diff --git a/test/unit/api-keys.test.mjs b/test/unit/api-keys.test.mjs index 35e7647a4..fa2c16524 100644 --- a/test/unit/api-keys.test.mjs +++ b/test/unit/api-keys.test.mjs @@ -23,203 +23,255 @@ test('maskApiKey hides middle', () => { assert.ok(!m.includes('a'.repeat(20))) }) -test('create + authenticate + list mask', () => { +test('create + authenticate + list mask', async () => { const store = tmpStore() - const rec = store.create({ name: 'demo', max_concurrency: 3, quota_requests: 10, rpm: 60 }) + const rec = await store.create({ name: 'demo', max_concurrency: 3, quota_requests: 10, rpm: 60 }) assert.equal(rec.name, 'demo') assert.equal(rec.max_concurrency, 3) assert.equal(rec.quota_requests, 10) - const auth = store.authenticate(rec.key) + const auth = await store.authenticate(rec.key) assert.equal(auth.ok, true) assert.equal(auth.record.id, rec.id) - const stored = store.getById(rec.id) + const stored = await store.getById(rec.id) assert.notEqual(stored.key, rec.key) assert.match(stored.key, /^hmac:/) assert.ok(stored.key_hash) - const listed = store.list() + const listed = await store.list() assert.equal(listed.length, 1) assert.ok(listed[0].key.includes('…')) assert.notEqual(listed[0].key, rec.key) }) -test('reveal returns the plaintext for copy; rotate swaps it in place', () => { +test('reveal returns the plaintext for copy; rotate swaps it in place', async () => { const store = tmpStore() - const rec = store.create({ name: 'copyable', max_concurrency: 4 }) - const listed = store.list()[0] + const rec = await store.create({ name: 'copyable', max_concurrency: 4 }) + const listed = (await store.list())[0] assert.equal(listed.revealable, true) assert.ok(listed.key.includes('…'), 'list stays masked') - const shown = store.reveal(rec.id) + const shown = await store.reveal(rec.id) assert.equal(shown.ok, true) assert.equal(shown.key, rec.key) - const rotated = store.rotate(rec.id) + const rotated = await store.rotate(rec.id) assert.notEqual(rotated.key, rec.key) - assert.equal(store.authenticate(rec.key).ok, false, 'old value dies') - assert.equal(store.authenticate(rotated.key).ok, true) - assert.equal(store.reveal(rec.id).key, rotated.key) - const kept = store.getById(rec.id) + assert.equal((await store.authenticate(rec.key)).ok, false, 'old value dies') + assert.equal((await store.authenticate(rotated.key)).ok, true) + assert.equal((await store.reveal(rec.id)).key, rotated.key) + const kept = await store.getById(rec.id) assert.equal(kept.name, 'copyable') assert.equal(kept.max_concurrency, 4) }) -test('reveal reports hash-only rows as unrecoverable', () => { +test('reveal reports hash-only rows as unrecoverable', async () => { const store = tmpStore() - const rec = store.create({ name: 'legacy' }) - const row = store.getById(rec.id) + const rec = await store.create({ name: 'legacy' }) + const row = await store.getById(rec.id) row.key_secret = null - store.repo.update(row) + await store.repo.update(row) - const out = store.reveal(rec.id) + const out = await store.reveal(rec.id) assert.equal(out.ok, false) assert.equal(out.code, 'key_not_recoverable') - assert.equal(store.list()[0].revealable, false) - assert.equal(store.reveal('key_missing'), null) - assert.equal(store.authenticate(rec.key).ok, true, 'auth still works off the hash') + assert.equal((await store.list())[0].revealable, false) + assert.equal(await store.reveal('key_missing'), null) + assert.equal((await store.authenticate(rec.key)).ok, true, 'auth still works off the hash') }) -test('key_secret is encrypted at rest when KIN_DB_SECRET is set', () => { +test('key_secret is encrypted at rest when KIN_DB_SECRET is set', async () => { const prev = process.env.KIN_DB_SECRET process.env.KIN_DB_SECRET = 'unit-test-secret' try { const store = tmpStore() - const rec = store.create({ name: 'enc' }) - const row = store.getById(rec.id) + const rec = await store.create({ name: 'enc' }) + const row = await store.getById(rec.id) assert.match(row.key_secret, /^enc:v1:/) assert.ok(!row.key_secret.includes(rec.key)) - assert.equal(store.reveal(rec.id).key, rec.key) + assert.equal((await store.reveal(rec.id)).key, rec.key) } finally { if (prev === undefined) delete process.env.KIN_DB_SECRET else process.env.KIN_DB_SECRET = prev } }) -test('canAccept rejects disabled / expired / quota / concurrency', () => { +test('canAccept rejects disabled / expired / quota / concurrency', async () => { const store = tmpStore() - const rec = store.create({ name: 'lim', max_concurrency: 1, quota_requests: 2, rpm: 0 }) - assert.equal(store.canAccept(rec).ok, true) + const rec = await store.create({ name: 'lim', max_concurrency: 1, quota_requests: 2, rpm: 0 }) + assert.equal((await store.canAccept(rec)).ok, true) - store.update(rec.id, { status: 'disabled' }) - assert.equal(store.canAccept(store.getById(rec.id)).code, 'api_key_disabled') - store.update(rec.id, { status: 'active' }) + await store.update(rec.id, { status: 'disabled' }) + assert.equal((await store.canAccept(await store.getById(rec.id))).code, 'api_key_disabled') + await store.update(rec.id, { status: 'active' }) - store.update(rec.id, { expires_at: new Date(Date.now() - 1000).toISOString() }) - assert.equal(store.canAccept(store.getById(rec.id)).code, 'api_key_expired') - store.update(rec.id, { expires_at: null }) + await store.update(rec.id, { expires_at: new Date(Date.now() - 1000).toISOString() }) + assert.equal((await store.canAccept(await store.getById(rec.id))).code, 'api_key_expired') + await store.update(rec.id, { expires_at: null }) - const a = store.acquire(store.getById(rec.id)) + const a = await store.acquire(await store.getById(rec.id)) assert.equal(a.ok, true) - const blocked = store.canAccept(store.getById(rec.id)) + const blocked = await store.canAccept(await store.getById(rec.id)) assert.equal(blocked.code, 'api_key_concurrency_limit') store.release(rec.id) - store.recordUsage(rec.id) - store.recordUsage(rec.id) - assert.equal(store.canAccept(store.getById(rec.id)).code, 'api_key_quota_exhausted') + await store.recordUsage(rec.id) + await store.recordUsage(rec.id) + assert.equal((await store.canAccept(await store.getById(rec.id))).code, 'api_key_quota_exhausted') }) -test('rpm window blocks then recovers', () => { +test('request quotas compare imported bigint counters exactly', async () => { const store = tmpStore() - const rec = store.create({ name: 'rpm', max_concurrency: 0, quota_requests: 0, rpm: 2 }) - assert.equal(store.acquire(rec).ok, true) + const rec = await store.create({ name: 'large quota' }) + await store.db + .prepare('UPDATE api_keys SET quota_requests = $1, quota_requests_used = $2 WHERE id = $3') + .run('10000000000000000', '9007199254740993', rec.id) + assert.equal((await store.canAccept(await store.getById(rec.id))).ok, true) + await store.db.prepare('UPDATE api_keys SET status = $1 WHERE id = $2').run('quota_exhausted', rec.id) + assert.equal((await store.update(rec.id, { name: 'renamed' })).status, 'active') + await store.db.prepare('UPDATE api_keys SET quota_requests_used = quota_requests WHERE id = $1').run(rec.id) + assert.equal((await store.canAccept(await store.getById(rec.id))).code, 'api_key_quota_exhausted') +}) + +test('rpm window blocks then recovers', async () => { + const store = tmpStore() + const rec = await store.create({ name: 'rpm', max_concurrency: 0, quota_requests: 0, rpm: 2 }) + assert.equal((await store.acquire(rec)).ok, true) store.release(rec) - assert.equal(store.acquire(rec).ok, true) + assert.equal((await store.acquire(rec)).ok, true) store.release(rec) - const gate = store.canAccept(store.getById(rec.id)) + const gate = await store.canAccept(await store.getById(rec.id)) assert.equal(gate.code, 'api_key_rate_limit') }) -test('user concurrency caps regular users but never the admin operator', () => { +test('user concurrency caps regular users but never the admin operator', async () => { const store = tmpStore() const now = new Date().toISOString() for (const [id, role] of [ ['u-admin', 'admin'], ['u-user', 'user'], ]) { - store.users.insert({ id, username: id, password_hash: 'x', role, concurrency: 1, created_at: now, updated_at: now }) + await store.users.insert({ + id, + username: id, + password_hash: 'x', + role, + concurrency: 1, + created_at: now, + updated_at: now, + }) } - const admin = store.create({ name: 'admin-key', max_concurrency: 0, user_id: 'u-admin' }) - const user = store.create({ name: 'user-key', max_concurrency: 0, user_id: 'u-user' }) + const admin = await store.create({ name: 'admin-key', max_concurrency: 0, user_id: 'u-admin' }) + const user = await store.create({ name: 'user-key', max_concurrency: 0, user_id: 'u-user' }) - for (let i = 0; i < 3; i++) assert.equal(store.acquire(admin).ok, true) + for (let i = 0; i < 3; i++) assert.equal((await store.acquire(admin)).ok, true) - assert.equal(store.acquire(user).ok, true) - const blocked = store.canAccept(user) + assert.equal((await store.acquire(user)).ok, true) + const blocked = await store.canAccept(user) assert.equal(blocked.code, 'user_concurrency_limit') assert.deepEqual(blocked.detail, { inflight: 1, max: 1 }) }) -test('update reset_quota and remove', () => { +test('update reset_quota and remove', async () => { const store = tmpStore() - const rec = store.create({ name: 'x', quota_requests: 5 }) - store.recordUsage(rec.id) - store.recordUsage(rec.id) - assert.equal(store.getById(rec.id).quota_requests_used, 2) - store.update(rec.id, { reset_quota: true }) - assert.equal(store.getById(rec.id).quota_requests_used, 0) - assert.equal(store.remove(rec.id), true) - assert.equal(store.getById(rec.id), null) - assert.equal(store.authenticate(rec.key).ok, false) -}) - -test('recordUsage accumulates cache token columns', () => { + const rec = await store.create({ name: 'x', quota_requests: 5 }) + await store.recordUsage(rec.id) + await store.recordUsage(rec.id) + assert.equal((await store.getById(rec.id)).quota_requests_used, 2) + await store.update(rec.id, { reset_quota: true }) + assert.equal((await store.getById(rec.id)).quota_requests_used, 0) + assert.equal(await store.remove(rec.id), true) + assert.equal(await store.getById(rec.id), null) + assert.equal((await store.authenticate(rec.key)).ok, false) +}) + +test('recordUsage accumulates cache token columns', async () => { const store = tmpStore() - const rec = store.create({ name: 'cache' }) - store.recordUsage(rec.id, { + const rec = await store.create({ name: 'cache' }) + await store.recordUsage(rec.id, { input_tokens: 10, output_tokens: 5, cache_read_input_tokens: 3, cache_creation_input_tokens: 7, }) - store.recordUsage(rec.id, { input_tokens: 1, output_tokens: 1, cache_read_tokens: 2 }) - const saved = store.getById(rec.id) + await store.recordUsage(rec.id, { input_tokens: 1, output_tokens: 1, cache_read_tokens: 2 }) + const saved = await store.getById(rec.id) assert.equal(saved.tokens_in, 11) assert.equal(saved.tokens_out, 6) assert.equal(saved.cache_read_tokens, 5) assert.equal(saved.cache_creation_tokens, 7) }) -test('custom key rejected when duplicate', () => { +test('custom key rejected when duplicate', async () => { const store = tmpStore() - store.create({ name: 'a', key: 'sk-kin-customkey0001' }) - assert.throws(() => store.create({ name: 'b', key: 'sk-kin-customkey0001' }), /exists/) + await store.create({ name: 'a', key: 'sk-kin-customkey0001' }) + await assert.rejects(async () => await store.create({ name: 'b', key: 'sk-kin-customkey0001' }), /exists/) }) -test('keys persist in sqlite across store re-open (same dataDir)', () => { +test('keys persist in PostgreSQL across store recreation', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-keys-')) const store = new ApiKeyStore({ dataDir: dir }) - const rec = store.create({ name: 'persist', quota_requests: 9 }) - store.recordUsage(rec.id, { input_tokens: 11, output_tokens: 22 }) + const rec = await store.create({ name: 'persist', quota_requests: 9 }) + await store.recordUsage(rec.id, { input_tokens: 11, output_tokens: 22 }) const store2 = new ApiKeyStore({ dataDir: dir }) - const again = store2.getById(rec.id) + const again = await store2.getById(rec.id) assert.ok(again, 'record should survive re-open') assert.equal(again.name, 'persist') assert.equal(again.quota_requests_used, 1) assert.equal(again.tokens_in, 11) assert.equal(again.tokens_out, 22) - assert.equal(store2.authenticate(rec.key).ok, true) - assert.ok(fs.existsSync(path.join(dir, 'kin.db')), 'kin.db file exists') + assert.equal((await store2.authenticate(rec.key)).ok, true) }) -test('group_type all is the default and a platform group keeps its VM list', () => { +test('group_type all is the default and a platform group keeps its VM list', async () => { const store = tmpStore() - const open = store.create({ name: 'global' }) - assert.equal(store.list()[0].group_type, 'all') - assert.deepEqual(store.list()[0].allowed_vms, []) - const scoped = store.create({ name: 'claude', group_type: 'anthropic', allowed_vms: ['vm-01', 'vm-03'] }) - assert.equal(store.getById(scoped.id).group_type, 'anthropic') - assert.equal( - store - .list() - .find((k) => k.id === scoped.id) - .allowed_vms.join(','), - 'vm-01,vm-03', - ) - const widened = store.update(scoped.id, { group_type: 'all' }) + const open = await store.create({ name: 'global' }) + assert.equal((await store.list())[0].group_type, 'all') + assert.deepEqual((await store.list())[0].allowed_vms, []) + const scoped = await store.create({ name: 'claude', group_type: 'anthropic', allowed_vms: ['vm-01', 'vm-03'] }) + assert.equal((await store.getById(scoped.id)).group_type, 'anthropic') + assert.equal((await store.list()).find((k) => k.id === scoped.id).allowed_vms.join(','), 'vm-01,vm-03') + const widened = await store.update(scoped.id, { group_type: 'all' }) assert.equal(widened.group_type, 'all') assert.equal(widened.allowed_vms, '[]') - assert.throws(() => store.create({ name: 'bare', group_type: 'openai' }), /至少勾选一台 VM/) - store.update(open.id, { status: 'disabled' }) - assert.equal(store.getById(open.id).group_type, 'all') + await assert.rejects(async () => await store.create({ name: 'bare', group_type: 'openai' }), /至少勾选一台 VM/) + await store.update(open.id, { status: 'disabled' }) + assert.equal((await store.getById(open.id)).group_type, 'all') +}) + +test('simultaneous owned-key acquires reserve one concurrency slot', async () => { + const store = tmpStore() + await store.users.insert({ + id: 'parallel-owner', + username: 'parallel-owner', + password_hash: 'x', + role: 'user', + concurrency: 1, + }) + const key = await store.create({ name: 'parallel-key', user_id: 'parallel-owner', max_concurrency: 1 }) + const results = await Promise.all(Array.from({ length: 12 }, () => store.acquire(key))) + assert.equal(results.filter((r) => r.ok).length, 1) + assert.equal(store.inflight.get(key.id), 1) + store.release(key) + assert.equal((await store.acquire(key)).ok, true) + store.release(key) +}) + +test('simultaneous usage writes retain every counter and spending window', async () => { + const store = tmpStore() + const key = await store.create({ name: 'parallel-billing', quota: 2 }) + await Promise.all( + Array.from({ length: 12 }, () => + store.recordUsage(key.id, { input_tokens: 3, output_tokens: 2, total_cost: 0.25 }), + ), + ) + const row = await store.getById(key.id) + assert.equal(row.requests, 12) + assert.equal(row.quota_requests_used, 12) + assert.equal(row.tokens_in, 36) + assert.equal(row.tokens_out, 24) + assert.equal(row.quota_used, 3) + assert.equal(row.usage_5h, 3) + assert.equal(row.usage_1d, 3) + assert.equal(row.usage_7d, 3) + assert.equal(row.status, 'quota_exhausted') }) diff --git a/test/unit/audit-postgres.test.mjs b/test/unit/audit-postgres.test.mjs new file mode 100644 index 000000000..e022255c2 --- /dev/null +++ b/test/unit/audit-postgres.test.mjs @@ -0,0 +1,89 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { getDb } from '../../src/lib/db/database.mjs' +const exec = promisify(execFile) + +test('audit commands execute PostgreSQL aggregates using a read-only connection', async () => { + const db = getDb() + for (const [script, args] of [ + ['audit-billing-gaps.mjs', []], + ['audit-billing-openai.mjs', []], + ['audit-fable-week.mjs', []], + ['audit-billing-contract.mjs', ['--database', '--json']], + ]) { + const { stdout } = await exec(process.execPath, [`scripts/${script}`, ...args], { + env: { ...process.env, VM2API_DATABASE_URL: db.connectionString, VM2API_DATABASE_SCHEMA: db.schema }, + }) + const report = JSON.parse(stdout) + assert.equal(typeof report, 'object', script) + if (script === 'audit-billing-gaps.mjs') assert.equal(typeof report.totals.n, 'number') + if (script === 'audit-billing-contract.mjs') assert.equal(typeof report.db.totals.n, 'number') + assert.equal(stdout.includes('postgresql://'), false, script) + } +}) + +async function preflight() { + const db = getDb() + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-upgrade-preflight-')) + try { + return await exec(process.execPath, ['scripts/check-postgres-upgrade.mjs'], { + env: { + ...process.env, + KIN_DATA_DIR: directory, + VM2API_DATABASE_URL: db.connectionString, + VM2API_DATABASE_SCHEMA: db.schema, + }, + }) + } finally { + fs.rmSync(directory, { recursive: true, force: true }) + } +} + +test('upgrade preflight reads valid source checksums without modifying migration history', async () => { + const before = await getDb().prepare('SELECT * FROM schema_migrations ORDER BY version').all() + await preflight() + assert.deepEqual(await getDb().prepare('SELECT * FROM schema_migrations ORDER BY version').all(), before) +}) + +test('upgrade preflight refuses an applied checksum mismatch without repairing it', async () => { + const row = await getDb().prepare('SELECT version FROM schema_migrations ORDER BY version LIMIT 1').get() + await getDb() + .prepare('UPDATE schema_migrations SET checksum = $1 WHERE version = $2') + .run('synthetic-mismatch', row.version) + await assert.rejects(preflight(), (error) => error.code === 1 && /preflight|cutover/i.test(error.stderr)) + assert.equal( + (await getDb().prepare('SELECT checksum FROM schema_migrations WHERE version = $1').get(row.version)).checksum, + 'synthetic-mismatch', + ) +}) + +test('upgrade preflight refuses unknown migrations and an empty history', async () => { + await getDb() + .prepare('INSERT INTO schema_migrations (version,name,checksum,applied_at) VALUES ($1,$2,$3,$4)') + .run('999-unknown', 'unknown', 'unknown', new Date().toISOString()) + await assert.rejects(preflight(), (error) => error.code === 1) + await getDb().exec('DELETE FROM schema_migrations') + await assert.rejects(preflight(), (error) => error.code === 1) +}) + +test('upgrade preflight refuses missing migration history and persisted restore maintenance', async () => { + await getDb().exec('DROP TABLE schema_migrations') + await assert.rejects(preflight(), (error) => error.code === 1) + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-preflight-marker-')) + try { + fs.writeFileSync(path.join(directory, 'restore-maintenance.json'), '{}') + await assert.rejects( + exec(process.execPath, ['scripts/check-postgres-upgrade.mjs'], { + env: { ...process.env, KIN_DATA_DIR: directory }, + }), + (error) => error.code === 1, + ) + } finally { + fs.rmSync(directory, { recursive: true, force: true }) + } +}) diff --git a/test/unit/backup-core.test.mjs b/test/unit/backup-core.test.mjs new file mode 100644 index 000000000..30e9ba626 --- /dev/null +++ b/test/unit/backup-core.test.mjs @@ -0,0 +1,27 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { pgEnvironment, runPgTool } from '../../src/lib/admin/backup-core.mjs' + +test('PostgreSQL child environment preserves encoded credentials and IPv6 without command arguments', () => { + const env = pgEnvironment('postgresql://test:p%40ss%3Aword@[::1]:55432/example_test?sslmode=verify-full') + assert.equal(env.PGPASSWORD, 'p@ss:word') + assert.equal(env.PGHOST, '::1') + assert.equal(env.PGPORT, '55432') + assert.equal(env.PGDATABASE, 'example_test') + assert.equal(env.PGSSLMODE, 'verify-full') +}) + +test('child process errors do not return stderr that can contain row values', async () => { + await assert.rejects( + runPgTool( + process.execPath, + ['-e', "process.stderr.write('synthetic-sensitive-row');process.exit(2)"], + 'postgresql://test:synthetic-password@127.0.0.1/example_test', + ), + (error) => { + assert.equal(error.message.includes('synthetic'), false) + assert.match(error.message, /failed \(exit 2\)/) + return true + }, + ) +}) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index bba99bf36..28709cb93 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -4,7 +4,9 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { spawnSync } from 'node:child_process' -import { openDatabase, closeDatabase, getDb } from '../../src/lib/db/database.mjs' +import { openDatabase, closeDatabase, getDb, createDatabase } from '../../src/lib/db/database.mjs' +import { BackupRepo } from '../../src/lib/db/repos/backup-repo.mjs' +import { sha256FileAsync } from '../../src/lib/admin/backup-core.mjs' import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' import { BackupService, tarAvailable } from '../../src/lib/admin/backup-service.mjs' @@ -33,21 +35,21 @@ function seedVm(id = 'vm-b1') { fs.writeFileSync(path.join(vmsDir, 'active.json'), JSON.stringify({ active_vm: id })) } -beforeEach(() => { +beforeEach(async () => { root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-bak-')) dataDir = path.join(root, 'data') configDir = path.join(root, 'config') fs.mkdirSync(configDir, { recursive: true }) fs.writeFileSync(path.join(configDir, 'routing.json'), JSON.stringify({ sticky: { enabled: true } })) - openDatabase({ dataDir }) + await openDatabase({ dataDir }) svc = new BackupService({ dataDir, projectRoot: root, configDir }) }) -afterEach(() => { +afterEach(async () => { svc.stopScheduler() stopVmWatch() setVmWriteHook(null) - closeDatabase() + await closeDatabase() fs.rmSync(root, { recursive: true, force: true }) delete process.env.KIN_BACKUP_DISABLED delete process.env.KIN_BACKUP_INTERVAL_HOURS @@ -58,10 +60,26 @@ test('tar is available in this environment', () => { assert.equal(tarAvailable(), true) }) +test('restoring the application schema preserves later writes in a neighboring schema', async () => { + const db = getDb() + const neighbor = `"${db.schema}_neighbor"` + await db.exec( + `CREATE SCHEMA ${neighbor}; CREATE TABLE ${neighbor}.sentinel(value text); INSERT INTO ${neighbor}.sentinel VALUES('before')`, + ) + try { + const rec = await svc.createBackup() + await db.exec(`UPDATE ${neighbor}.sentinel SET value='after'`) + await svc.restoreBackup(rec.id) + assert.equal((await db.prepare(`SELECT value FROM ${neighbor}.sentinel`).get()).value, 'after') + } finally { + await db.exec(`DROP SCHEMA ${neighbor} CASCADE`) + } +}) + test('createBackup produces tar.gz with manifest + db + vms + config, records ok', async () => { seedVm() - initVmDbSync(root, { watch: false }) - new SettingsRepo(getDb()).set('marker', 'before-backup') + await initVmDbSync(root, { watch: false }) + await new SettingsRepo(getDb()).set('marker', 'before-backup') const rec = await svc.createBackup({ kind: 'manual' }) assert.equal(rec.status, 'ok') @@ -77,18 +95,18 @@ test('createBackup produces tar.gz with manifest + db + vms + config, records ok const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-bak-x-')) spawnSync('tar', ['-xzf', rec.file_path, '-C', tmp]) assert.ok(fs.existsSync(path.join(tmp, 'manifest.json'))) - assert.ok(fs.existsSync(path.join(tmp, 'db', 'kin.db'))) + assert.ok(fs.existsSync(path.join(tmp, 'db', 'postgres.dump'))) assert.ok(fs.existsSync(path.join(tmp, 'vms', 'vm-b1.json'))) assert.ok(fs.existsSync(path.join(tmp, 'config', 'routing.json'))) const manifest = JSON.parse(fs.readFileSync(path.join(tmp, 'manifest.json'), 'utf8')) - assert.equal(manifest.version, 1) - assert.ok(manifest.sha256s['db/kin.db']) + assert.equal(manifest.version, 2) + assert.ok(manifest.sha256s['db/postgres.dump']) fs.rmSync(tmp, { recursive: true, force: true }) }) test('createBackup does not block the main thread (gateway keeps ticking)', async () => { seedVm() - initVmDbSync(root, { watch: false }) + await initVmDbSync(root, { watch: false }) let ticks = 0 const timer = setInterval(() => { ticks += 1 @@ -106,38 +124,37 @@ test('createBackup does not block the main thread (gateway keeps ticking)', asyn test('restoreBackup rolls data back and creates pre_restore record', async () => { seedVm() - initVmDbSync(root, { watch: false }) + await initVmDbSync(root, { watch: false }) const keys = new ApiKeyStore({ db: getDb() }) - const kept = keys.create({ name: 'kept-key' }) + const kept = await keys.create({ name: 'kept-key' }) const rec = await svc.createBackup({ kind: 'manual' }) // mutate after the backup - const lost = keys.create({ name: 'lost-key' }) - new SettingsRepo(getDb()).set('marker', 'after-backup') + const lost = await keys.create({ name: 'lost-key' }) + await new SettingsRepo(getDb()).set('marker', 'after-backup') // wire the rebind like the server does let reboundDb = null - svc.onRestored((db) => { + svc.onRestored(async (db) => { reboundDb = db - keys.rebind(db) + await keys.rebind(db) }) const out = await svc.restoreBackup(rec.id) assert.equal(out.ok, true) + assert.equal(fs.existsSync(svc.maintenanceFile), false) assert.ok(reboundDb, 'rebind callback should fire') // state rolled back - assert.ok(keys.getById(kept.id), 'pre-backup key survives') - assert.equal(keys.getById(lost.id), null, 'post-backup key rolled back') - assert.equal(new SettingsRepo(getDb()).get('marker'), null) + assert.ok(await keys.getById(kept.id), 'pre-backup key survives') + assert.equal(await keys.getById(lost.id), null, 'post-backup key rolled back') + assert.equal(await new SettingsRepo(getDb()).get('marker'), null) - // old db file kept on disk as safety net - assert.ok(fs.existsSync(path.join(dataDir, 'kin.db.pre-restore'))) // pre_restore archive file kept on disk for manual recovery const preFiles = fs.readdirSync(path.join(dataDir, 'backups')) assert.ok(preFiles.length >= 2, 'manual + pre_restore archives on disk') // ledger re-registered after restore: pre_restore (and restored manual) visible - const ledger = svc.list() + const ledger = await svc.list() assert.ok( ledger.some((r) => r.id === out.pre_restore && r.kind === 'pre_restore'), 'pre_restore record re-registered in restored DB', @@ -152,21 +169,24 @@ test('restoreBackup rolls data back and creates pre_restore record', async () => ) }) -test('restore fails when stores cannot rebind to the reopened database', async () => { +test('restore failure keeps maintenance enabled and exposes the recovery archive', async () => { seedVm() const rec = await svc.createBackup({ kind: 'manual' }) svc.onRestored(() => { throw new Error('store rebind failed') }) - await assert.rejects(svc.restoreBackup(rec.id), /store rebind failed/) - assert.equal(svc.isRestoring, false) - assert.doesNotThrow(() => getDb().prepare('SELECT 1').get()) + await assert.rejects(svc.restoreBackup(rec.id), /restore and rollback failed/) + assert.equal(svc.isRestoring, true) + assert.equal(svc.isMaintaining, true) + assert.equal(fs.existsSync(svc.maintenanceFile), true) + await assert.rejects(createDatabase({ dataDir, connectionString: 'invalid' }), /restore maintenance marker/) + assert.equal((await getDb().prepare('SELECT 1 AS ok').get()).ok, 1) }) -test('restore rebuilds vm files from DB mirror for db-only path', async () => { +test('restore restores VM JSON files together with PostgreSQL metadata', async () => { seedVm('vm-b2') - initVmDbSync(root, { watch: false }) + await initVmDbSync(root, { watch: false }) const rec = await svc.createBackup({ kind: 'manual' }) // delete vm file + its row is still in backup db; also delete local file post-backup @@ -176,7 +196,7 @@ test('restore rebuilds vm files from DB mirror for db-only path', async () => { const back = JSON.parse(fs.readFileSync(path.join(root, 'vms', 'vm-b2.json'), 'utf8')) // tar snapshot still has the on-disk file from backup time; DB is metadata only. assert.equal(back.claude.access_token, 'sk-ant-oat01-BAK') - const row = new VmsRepo(getDb()).get('vm-b2') + const row = await new VmsRepo(getDb()).get('vm-b2') assert.equal(row.access_token, null) assert.equal(row.vm.claude.has_access, true) assert.equal(row.vm.claude.access_token, undefined) @@ -194,7 +214,7 @@ test('retention prunes old backups', async () => { seedVm() const recs = [] for (let i = 0; i < 4; i++) recs.push(await svc.createBackup({ kind: 'manual' })) - const remaining = svc.list().filter((r) => r.kind === 'manual') + const remaining = (await svc.list()).filter((r) => r.kind === 'manual') assert.equal(remaining.length, 2) // newest two kept assert.ok(remaining.some((r) => r.id === recs[3].id)) @@ -204,34 +224,219 @@ test('retention prunes old backups', async () => { assert.equal(fs.existsSync(recs[1].file_path), false) }) -test('schedule config defaults on, env override off, updateSchedule persists', () => { - const cfg = svc.getSchedule() +test('schedule config defaults on, env override off, updateSchedule persists', async () => { + const cfg = await svc.getSchedule() assert.equal(cfg.enabled, true) assert.equal(cfg.interval_hours, 24) assert.equal(cfg.retention, 7) - svc.updateSchedule({ interval_hours: 6, retention: 3 }) - assert.equal(svc.getSchedule().interval_hours, 6) - assert.equal(svc.getSchedule().retention, 3) - assert.throws(() => svc.updateSchedule({ interval_hours: 0 }), /interval_hours/) + await svc.updateSchedule({ interval_hours: 6, retention: 3 }) + assert.equal((await svc.getSchedule()).interval_hours, 6) + assert.equal((await svc.getSchedule()).retention, 3) + await assert.rejects(() => svc.updateSchedule({ interval_hours: 0 }), /interval_hours/) process.env.KIN_BACKUP_DISABLED = '1' - assert.equal(svc.getSchedule().enabled, false) + assert.equal((await svc.getSchedule()).enabled, false) }) test('shouldRunScheduled: true when never run, false right after a backup', async () => { seedVm() - assert.equal(svc.shouldRunScheduled(), true) + assert.equal(await svc.shouldRunScheduled(), true) await svc.createBackup({ kind: 'scheduled' }) - assert.equal(svc.shouldRunScheduled(), false) + assert.equal(await svc.shouldRunScheduled(), false) // pretend 25h elapsed - assert.equal(svc.shouldRunScheduled(Date.now() + 25 * 3600_000), true) + assert.equal(await svc.shouldRunScheduled(Date.now() + 25 * 3600_000), true) +}) + +test('stopping scheduling drains an in-flight check without starting a backup afterward', async (t) => { + let check + t.mock.method(globalThis, 'setInterval', (callback) => { + check = callback + return { unref() {} } + }) + t.mock.method(globalThis, 'clearInterval', () => {}) + let releaseCheck, signalCheck + const gate = new Promise((resolve) => { + releaseCheck = resolve + }) + const started = new Promise((resolve) => { + signalCheck = resolve + }) + svc.shouldRunScheduled = async () => { + signalCheck() + await gate + return true + } + let backups = 0 + svc.createBackup = async () => { + backups += 1 + return { size_bytes: 0 } + } + svc.startScheduler() + const checking = check() + await started + svc.stopScheduler() + let flushed = false + const pending = svc.flush().then(() => { + flushed = true + }) + let flushedDuringCheck + try { + await new Promise((resolve) => setImmediate(resolve)) + flushedDuringCheck = flushed + } finally { + releaseCheck() + await checking + await pending + } + assert.deepEqual({ flushedDuringCheck, backups }, { flushedDuringCheck: false, backups: 0 }) +}) + +test('restarting scheduling does not reactivate a check from the previous schedule', async (t) => { + let check + t.mock.method(globalThis, 'setInterval', (callback) => { + check = callback + return { unref() {} } + }) + t.mock.method(globalThis, 'clearInterval', () => {}) + let releaseCheck + const gate = new Promise((resolve) => { + releaseCheck = resolve + }) + svc.shouldRunScheduled = async () => { + await gate + return true + } + let backups = 0 + svc.createBackup = async () => { + backups += 1 + return { size_bytes: 0 } + } + svc.startScheduler() + const previous = check() + svc.startScheduler() + releaseCheck() + await previous + assert.equal(backups, 0) + await check() + await svc.flush() + assert.equal(backups, 1) }) test('remove deletes record + file', async () => { seedVm() const rec = await svc.createBackup({ kind: 'manual' }) - assert.equal(svc.remove(rec.id), true) + assert.equal(await svc.remove(rec.id), true) assert.equal(fs.existsSync(rec.file_path), false) - assert.equal(svc.get(rec.id), null) + assert.equal(await svc.get(rec.id), null) +}) + +test('backup snapshot and copied files precede resumed writes', async () => { + seedVm() + await new SettingsRepo(getDb()).set('snapshot-boundary', 'before') + const transitions = [] + svc.onMaintenance(async (enabled, reason) => { + transitions.push([enabled, reason]) + if (!enabled) { + await new SettingsRepo(getDb()).set('snapshot-boundary', 'after') + fs.writeFileSync(path.join(configDir, 'routing.json'), '{"after":true}') + } + }) + const rec = await svc.createBackup() + svc.onMaintenance(null) + await svc.restoreBackup(rec.id) + assert.equal(await new SettingsRepo(getDb()).get('snapshot-boundary'), 'before') + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(configDir, 'routing.json'))), { sticky: { enabled: true } }) + assert.deepEqual(transitions, [ + [true, 'backup'], + [false, 'backup'], + ]) +}) + +test('restore rejects an archive directory symlink before entering maintenance', async () => { + const source = path.join(root, 'symlink-source') + const outside = path.join(root, 'outside-extraction') + fs.mkdirSync(source) + fs.mkdirSync(outside) + fs.writeFileSync(path.join(outside, 'sentinel.json'), '{"unchanged":true}') + fs.writeFileSync(path.join(source, 'manifest.json'), '{}') + fs.symlinkSync(outside, path.join(source, 'vms')) + const archive = path.join(root, 'symlink.tar.gz') + assert.equal(spawnSync('tar', ['-czf', archive, '-C', source, 'manifest.json', 'vms']).status, 0) + // Reproduce the old extraction behavior using only this test's temporary paths. + const oldExtraction = path.join(root, 'old-extraction') + fs.mkdirSync(oldExtraction) + assert.equal(spawnSync('tar', ['-xzf', archive, '-C', oldExtraction]).status, 0) + assert.equal(fs.lstatSync(path.join(oldExtraction, 'vms')).isSymbolicLink(), true) + const rec = await new BackupRepo(getDb()).insert({ + id: 'bak_symlink', + created_at: new Date().toISOString(), + status: 'ok', + kind: 'manual', + file_name: 'symlink.tar.gz', + file_path: archive, + sha256: await sha256FileAsync(archive), + }) + await assert.rejects(svc.restoreBackup(rec.id), /invalid or duplicate backup archive path/) + assert.equal(fs.readFileSync(path.join(outside, 'sentinel.json'), 'utf8'), '{"unchanged":true}') + assert.equal(svc.isMaintaining, false) +}) + +test('restore rejects file counts that disagree with the manifest', async () => { + seedVm() + const rec = await svc.createBackup() + const source = path.join(root, 'count-mismatch') + fs.mkdirSync(source) + assert.equal(spawnSync('tar', ['-xzf', rec.file_path, '-C', source]).status, 0) + const file = path.join(source, 'manifest.json') + const manifest = JSON.parse(fs.readFileSync(file)) + manifest.includes.vms += 1 + fs.writeFileSync(file, JSON.stringify(manifest)) + assert.equal(spawnSync('tar', ['-czf', rec.file_path, '-C', source, '.']).status, 0) + await getDb() + .prepare('UPDATE backup_records SET sha256 = $1 WHERE id = $2') + .run(await sha256FileAsync(rec.file_path), rec.id) + await assert.rejects(svc.restoreBackup(rec.id), /backup file snapshot is incomplete/) + assert.equal(svc.isMaintaining, false) +}) + +test('restore replaces all top-level JSON while preserving CLI homes and non-JSON files', async () => { + seedVm() + const vmsDir = path.join(root, 'vms') + const cliHome = path.join(vmsDir, 'vm-b1', 'cli-home') + fs.mkdirSync(cliHome, { recursive: true }) + fs.writeFileSync(path.join(cliHome, 'keep.txt'), 'keep') + fs.writeFileSync(path.join(configDir, 'keep.txt'), 'keep') + const rec = await svc.createBackup() + fs.writeFileSync(path.join(vmsDir, 'vm-b1-chat.json'), '{}') + fs.writeFileSync(path.join(configDir, 'after-backup.json'), '{}') + await svc.restoreBackup(rec.id) + assert.equal(fs.existsSync(path.join(vmsDir, 'vm-b1-chat.json')), false) + assert.equal(fs.existsSync(path.join(configDir, 'after-backup.json')), false) + assert.equal(fs.readFileSync(path.join(cliHome, 'keep.txt'), 'utf8'), 'keep') + assert.equal(fs.readFileSync(path.join(configDir, 'keep.txt'), 'utf8'), 'keep') +}) + +test('failed restore rolls back target-only JSON and persists maintenance until an explicit successful retry', async () => { + seedVm() + const targetOnly = path.join(configDir, 'target-only.json') + const currentOnly = path.join(configDir, 'current-only.json') + fs.writeFileSync(targetOnly, '{}') + const rec = await svc.createBackup() + fs.rmSync(targetOnly) + fs.writeFileSync(currentOnly, '{}') + let calls = 0 + svc.onRestored(async () => { + if (++calls === 1) throw new Error('first rebind failed') + }) + await assert.rejects(svc.restoreBackup(rec.id), /first rebind failed/) + assert.equal(fs.existsSync(targetOnly), false) + assert.equal(fs.existsSync(currentOnly), true) + assert.equal(fs.existsSync(svc.maintenanceFile), true) + assert.equal(svc.isMaintaining, true) + await svc.restoreBackup(rec.id) + assert.equal(fs.existsSync(svc.maintenanceFile), false) + assert.equal(svc.isMaintaining, false) + assert.equal(fs.existsSync(targetOnly), true) + assert.equal(fs.existsSync(currentOnly), false) }) diff --git a/test/unit/bigint-counters.test.mjs b/test/unit/bigint-counters.test.mjs new file mode 100644 index 000000000..7efcfd76f --- /dev/null +++ b/test/unit/bigint-counters.test.mjs @@ -0,0 +1,229 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { getDb } from '../../src/lib/db/database.mjs' +import { runLegacyImport } from '../../src/lib/db/legacy-import.mjs' +import { AccountsRepo } from '../../src/lib/db/repos/accounts-repo.mjs' +import { ApiKeysRepo } from '../../src/lib/db/repos/api-keys-repo.mjs' +import { UsageLogsRepo } from '../../src/lib/db/repos/usage-logs-repo.mjs' +import { UsageLogsView } from '../../src/lib/db/repos/usage-logs-view.mjs' +import { StatisticsRepo } from '../../src/lib/db/repos/statistics-repo.mjs' +import { ApiKeyStore } from '../../src/lib/admin/api-keys.mjs' +import { RequestLogStore } from '../../src/lib/admin/request-log.mjs' + +const LARGE = '9007199254740993' +const INPUT_TOTAL = '9007199254740995' +const PROMPT_TOTAL = '9007199254741006' +const TOKEN_TOTAL = '9007199254741011' + +async function seedLogs(db) { + const now = Date.now() + for (const [id, input, output, read, write] of [ + ['large', LARGE, 4, 5, 6], + ['small', 2, 1, 0, 0], + ]) { + await db + .prepare(` + INSERT INTO usage_logs (id, request_id, created_at, model, status, input_tokens, output_tokens, + cache_read_tokens, cache_creation_tokens, vm_id, account_id, api_key_id, outbound_session_id, + protocol, path, total_cost, actual_cost, pricing_model) + VALUES ($1,$2,$3,$4,200,$5,$6,$7,$8,'vm-a','account-a','key-a','session-a', + 'anthropic.messages','/v1/messages',1,1,'synthetic-price') + `) + .run(id, 'request-' + id, new Date(now - 1).toISOString(), 'model-a', input, output, read, write) + } + return now + 1 +} + +test('legacy JSONL and summary inserts preserve exact bigint token fields', async (t) => { + const db = getDb() + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-bigint-import-')) + t.after(() => fs.rmSync(dir, { recursive: true, force: true })) + const ts = new Date().toISOString() + const record = { + id: 'legacy-large', + request_id: 'legacy-large', + ts, + status: 200, + input_tokens: LARGE, + output_tokens: LARGE, + cache_read_tokens: LARGE, + cache_creation_tokens: LARGE, + cache_creation_5m_tokens: LARGE, + cache_creation_1h_tokens: LARGE, + web_search_requests: LARGE, + } + const logDir = path.join(dir, 'request-logs') + fs.mkdirSync(logDir) + fs.writeFileSync(path.join(logDir, ts.slice(0, 10) + '.jsonl'), JSON.stringify(record) + '\n') + assert.equal((await runLegacyImport({ dataDir: dir, projectRoot: dir, db })).counts.request_logs, 1) + const logs = new UsageLogsRepo(db) + const imported = await logs.getByRequestId('legacy-large') + for (const field of Object.keys(record).filter((key) => key.endsWith('_tokens') || key === 'web_search_requests')) { + assert.equal(imported[field], LARGE, field) + } + await logs.insertSummary({ ...imported, id: 'copied-large', request_id: 'copied-large' }) + assert.equal((await logs.getByRequestId('copied-large')).input_tokens, LARGE) +}) + +test('bigint counter writers reject an already rounded unsafe Number', async () => { + const db = getDb() + const logs = new UsageLogsRepo(db) + await assert.rejects( + logs.insertSummary({ + id: 'unsafe', + request_id: 'unsafe', + ts: new Date().toISOString(), + input_tokens: Number(LARGE), + }), + RangeError, + ) + const keys = new ApiKeysRepo(db) + await assert.rejects( + keys.insert({ id: 'unsafe-key', key: 'synthetic-unsafe-key', requests: Number(LARGE) }), + RangeError, + ) + await keys.insert({ id: 'key-a', key: 'synthetic-key-a', status: 'active' }) + await assert.rejects(keys.recordUsage('key-a', { tokens_in: Number(LARGE) }), RangeError) + assert.equal((await keys.getById('key-a')).tokens_in, 0) + await assert.rejects(keys.update({ ...(await keys.getById('key-a')), requests: Number(LARGE) }), RangeError) + const accounts = new AccountsRepo(db) + await assert.rejects(accounts.insert({ account_id: 'unsafe-account', requests: Number(LARGE) }), RangeError) + await accounts.insert({ account_id: 'account-a', requests: 0 }) + await assert.rejects(accounts.save({ ...(await accounts.get('account-a')), tokens_in: Number(LARGE) }), RangeError) + await assert.rejects(accounts.addAllocation('account-a', { tokens_in: Number(LARGE) }), RangeError) +}) + +test('API key usage preserves both large stored counters and exact string deltas', async () => { + const keys = new ApiKeysRepo(getDb()) + await keys.insert({ id: 'key-a', key: 'synthetic-key-a', status: 'active', requests: LARGE, tokens_in: LARGE }) + const smallDelta = await keys.recordUsage('key-a', { tokens_in: 2 }) + assert.equal(smallDelta.requests, '9007199254740994') + assert.equal(smallDelta.tokens_in, INPUT_TOTAL) + await keys.insert({ id: 'key-b', key: 'synthetic-key-b', status: 'active' }) + const largeDelta = await keys.recordUsage('key-b', { + tokens_in: LARGE, + tokens_out: LARGE, + cache_read_tokens: LARGE, + cache_creation_tokens: LARGE, + }) + for (const field of ['tokens_in', 'tokens_out', 'cache_read_tokens', 'cache_creation_tokens']) { + assert.equal(largeDelta[field], LARGE, field) + } +}) + +test('account updates and allocation history preserve bigint counters', async () => { + const accounts = new AccountsRepo(getDb()) + await accounts.insert({ account_id: 'account-a', vm_id: 'vm-a', requests: LARGE, tokens_in: LARGE }) + const account = await accounts.get('account-a') + account.name = 'renamed' + await accounts.save(account) + await accounts.addAllocation('account-a', { tokens_in: LARGE }) + assert.equal((await accounts.get('account-a')).tokens_in, LARGE) + assert.equal((await accounts.get('account-a')).requests, LARGE) + assert.equal((await accounts.recentAllocations('account-a'))[0].tokens_in, LARGE) +}) + +test('API key store forwards exact string token deltas to persistence', async () => { + const store = new ApiKeyStore({ db: getDb() }) + const key = await store.create({ name: 'large usage' }) + await store.recordUsage(key.id, { input_tokens: LARGE, cache_creation_input_tokens: LARGE }) + const saved = await store.getById(key.id) + assert.equal(saved.tokens_in, LARGE) + assert.equal(saved.cache_creation_tokens, LARGE) +}) + +test('request logging preserves exact cache subdivisions and web search counts', async (t) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-bigint-request-')) + t.after(() => fs.rmSync(dir, { recursive: true, force: true })) + const store = new RequestLogStore({ db: getDb(), dataDir: dir }) + const ctx = store.start({ method: 'POST', headers: {} }) + await store.finish(ctx, { + status: 200, + usage: { + input_tokens: LARGE, + cache_creation_input_tokens: LARGE, + cache_creation: { ephemeral_5m_input_tokens: LARGE }, + web_search_requests: LARGE, + }, + }) + const saved = await store.repo.getByRequestId(ctx.request_id) + assert.equal(saved.input_tokens, LARGE) + assert.equal(saved.cache_creation_5m_tokens, LARGE) + assert.equal(saved.web_search_requests, LARGE) +}) + +test('usage totals and billing windows return exact integer sums', async () => { + const db = getDb() + const now = await seedLogs(db) + const logs = new UsageLogsRepo(db) + const totals = await logs.totals() + assert.equal(totals.input_tokens, INPUT_TOTAL) + assert.equal(totals.prompt_tokens, PROMPT_TOTAL) + assert.equal(totals.requests, 2) + assert.equal((await logs.aggregate())[0].input_tokens, INPUT_TOTAL) + const owner = await logs.ownerBilling() + assert.equal(owner.totals.tokens_in, PROMPT_TOTAL) + assert.equal(owner.items[0].tokens_in, PROMPT_TOTAL) + assert.equal((await logs.costByAccount())[0].input_tokens, INPUT_TOTAL) + assert.equal((await logs.costByModel())[0].input_tokens, INPUT_TOTAL) + const rolling = await logs.billingStats({ now }) + assert.equal(rolling.total.input_tokens, INPUT_TOTAL) + assert.equal(rolling.window_5h.input_tokens, INPUT_TOTAL) + assert.equal(rolling.accounts[0].window_5h_tokens, TOKEN_TOTAL) + const aligned = await logs.billingStats({ + now, + accountWindows: [ + { + account_id: 'account-a', + vm_id: 'vm-a', + reset_5h: new Date(now + 60_000).toISOString(), + reset_7d: new Date(now + 60_000).toISOString(), + }, + ], + }) + assert.equal(aligned.window_5h.input_tokens, INPUT_TOTAL) + assert.equal(aligned.accounts[0].window_5h_tokens, TOKEN_TOTAL) + assert.equal((await logs.vmUsageStats({ vmId: 'vm-a' })).history[0].input_tokens, INPUT_TOTAL) + assert.equal((await logs.keyUsageStats({ apiKeyId: 'key-a' })).models[0].tokens, TOKEN_TOTAL) + const window = await logs.windowStats() + assert.equal(window.input_tokens, INPUT_TOTAL) + assert.equal(window.prompt_tokens, PROMPT_TOTAL) +}) + +test('SQL and classified usage summaries return the same exact integers', async () => { + const db = getDb() + const now = await seedLogs(db) + for (const mutedErrorClasses of [[], ['distill']]) { + const view = new UsageLogsView(db, { mutedErrorClasses }) + const summary = await view.summary() + assert.equal(summary.totalInputTokens, INPUT_TOTAL) + assert.equal(summary.totalTokens, TOKEN_TOTAL) + assert.equal(summary.totalRequests, 2) + assert.equal((await view.activeSessions({ now })).sessions[0].totalTokens, TOKEN_TOTAL) + assert.equal( + (await view.exportRows({ include_muted: true })).items.find((row) => row.id === 'large').input_tokens, + LARGE, + ) + } +}) + +test('statistics totals, series, and leaderboards preserve exact integer sums', async () => { + const db = getDb() + const now = await seedLogs(db) + const repo = new StatisticsRepo(db) + const stats = await repo.statistics({ range: 'today', now }) + assert.equal(stats.totals.inputTokens, INPUT_TOTAL) + assert.equal(stats.totals.tokens, TOKEN_TOTAL) + assert.equal(stats.seriesTotals.s0.tokens, TOKEN_TOTAL) + assert.equal( + stats.metrics.tokens.flatMap((point) => point.s0).find((value) => value !== 0), + TOKEN_TOTAL, + ) + const entry = (await repo.leaderboard({ range: 'today', now })).entries[0] + assert.equal(entry.inputTokens, INPUT_TOTAL) + assert.equal(entry.totalTokens, TOKEN_TOTAL) + assert.equal(entry.totalRequests, 2) +}) diff --git a/test/unit/catalog-sync.test.mjs b/test/unit/catalog-sync.test.mjs index f8c280709..032d0d54d 100644 --- a/test/unit/catalog-sync.test.mjs +++ b/test/unit/catalog-sync.test.mjs @@ -27,17 +27,17 @@ function seedClaudeVm(root, id) { ) } -function withDb(t, catalogMode) { +async function withDb(t, catalogMode) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-catalog-sync-')) - openDatabase({ dbPath: ':memory:' }) - if (catalogMode) new SettingsRepo().set('model_policy', { catalog_mode: catalogMode, models: {} }) - loadModelPolicy({ force: true }) - loadGptModelPolicy({ force: true }) + await openDatabase({ dbPath: ':memory:' }) + if (catalogMode) await new SettingsRepo().set('model_policy', { catalog_mode: catalogMode, models: {} }) + await loadModelPolicy({ force: true }) + await loadGptModelPolicy({ force: true }) clearModelsCache() - t.after(() => { - closeDatabase() - loadModelPolicy({ force: true }) - loadGptModelPolicy({ force: true }) + t.after(async () => { + await closeDatabase() + await loadModelPolicy({ force: true }) + await loadGptModelPolicy({ force: true }) clearModelsCache() fs.rmSync(root, { recursive: true, force: true }) }) @@ -47,7 +47,7 @@ function withDb(t, catalogMode) { const upstream = (ids) => async () => ({ ok: true, status: 200, body: { data: ids.map((id) => ({ id })) } }) test('a Claude id new to the policy is added disabled outside worker_only', async (t) => { - const root = withDb(t, 'policy_only') + const root = await withDb(t, 'policy_only') seedClaudeVm(root, 'vm-1') const before = getModelPolicy().models['claude-opus-5'] const status = await syncClaudeCatalog({ @@ -63,14 +63,14 @@ test('a Claude id new to the policy is added disabled outside worker_only', asyn }) test('worker_only publishes a new Claude id immediately', async (t) => { - const root = withDb(t, 'worker_only') + const root = await withDb(t, 'worker_only') seedClaudeVm(root, 'vm-1') await syncClaudeCatalog({ projectRoot: root, runOauth: upstream(['claude-opus-6-preview']) }) assert.equal(getModelPolicy().models['claude-opus-6-preview'].enabled, true) }) test('Claude sync without a reachable slot records the failure and changes nothing', async (t) => { - const root = withDb(t, 'policy_only') + const root = await withDb(t, 'policy_only') seedClaudeVm(root, 'vm-1') const before = structuredClone(getModelPolicy().models) const status = await syncClaudeCatalog({ @@ -82,9 +82,9 @@ test('Claude sync without a reachable slot records the failure and changes nothi assert.deepEqual(getModelPolicy().models, before) }) -test('a GPT id new to the policy starts disabled; known ids keep their switch', (t) => { - withDb(t) - const result = syncGptIdsIntoPolicy([{ id: 'gpt-5.6' }, { id: 'gpt-7-preview' }]) +test('a GPT id new to the policy starts disabled; known ids keep their switch', async (t) => { + await withDb(t) + const result = await syncGptIdsIntoPolicy([{ id: 'gpt-5.6' }, { id: 'gpt-7-preview' }]) assert.deepEqual(result.added, ['gpt-7-preview']) const models = getGptModelPolicy().models assert.equal(models['gpt-5.6'].enabled, true) diff --git a/test/unit/cluster-direct-link.test.mjs b/test/unit/cluster-direct-link.test.mjs index 3ff849c12..48815ae60 100644 --- a/test/unit/cluster-direct-link.test.mjs +++ b/test/unit/cluster-direct-link.test.mjs @@ -6,7 +6,7 @@ import net from 'node:net' import os from 'node:os' import path from 'node:path' import tls from 'node:tls' -import { DatabaseSync } from 'node:sqlite' +import { getDb } from '../../src/lib/db/database.mjs' import { ClusterManager, DIRECT_LINK_BACKOFF_MS } from '../../src/lib/cluster/cluster-manager.mjs' import { certSha256, @@ -61,11 +61,11 @@ async function fakeGate(t, { token = TOKEN, slots = new Set(['vm-1']), extra = ' const pair = selfSigned(t) const seen = [] const held = new Set() - const server = tls.createServer({ ...pair, minVersion: 'TLSv1.3', allowHalfOpen: true }, (sock) => { + const server = tls.createServer({ ...pair, minVersion: 'TLSv1.3', allowHalfOpen: true }, async (sock) => { held.add(sock) sock.on('close', () => held.delete(sock)) let buf = Buffer.alloc(0) - const onData = (chunk) => { + const onData = async (chunk) => { buf = Buffer.concat([buf, chunk]) const nl = buf.indexOf(10) if (nl < 0) return @@ -75,7 +75,7 @@ async function fakeGate(t, { token = TOKEN, slots = new Set(['vm-1']), extra = ' seen.push({ vm, kind, early: parts[0].toString() }) if (magic !== 'KSG1' || tok !== token) return sock.end('ERR auth\n') if (kind === 'ping') return sock.end('OK\n') - if (!slots.has(vm)) return sock.end('ERR no_socket\n') + if (!(await slots.has(vm))) return sock.end('ERR no_socket\n') sock.write(`OK\n${extra}`) sock.on('data', (c) => parts.push(c)) sock.on('end', () => sock.end(Buffer.concat(parts).toString().toUpperCase())) @@ -84,9 +84,9 @@ async function fakeGate(t, { token = TOKEN, slots = new Set(['vm-1']), extra = ' sock.on('error', () => {}) }) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) - t.after(() => { + t.after(async () => { for (const s of held) s.destroy() - server.close() + await server.close() }) return { address: '127.0.0.1', port: server.address().port, token, certSha256: certSha256(pair.cert), seen } } @@ -144,9 +144,9 @@ test('dialGate reports refusals through ready and silence as unreachable', async const held = [] const silent = net.createServer((sock) => held.push(sock)) await new Promise((resolve) => silent.listen(0, '127.0.0.1', resolve)) - t.after(() => { + t.after(async () => { for (const sock of held) sock.destroy() - silent.close() + await silent.close() }) await assert.rejects(dialGate({ ...gate, port: silent.address().port, kind: 'ping', timeoutMs: 150 }), { code: 'gate_unreachable', @@ -217,15 +217,11 @@ test('SocketRelay: direct dial, SSH fallback on null, local drop when the slot s assert.deepEqual(sshAsked, ['f.sock']) }) -function managerWithRepo(t) { - const db = new DatabaseSync(':memory:') - for (const name of ['026_cluster_nodes.sql', '039_cluster_node_direct_links.sql']) { - db.exec(fs.readFileSync(new URL(`../../src/lib/db/migrations/${name}`, import.meta.url), 'utf8')) - } - db.exec('PRAGMA foreign_keys = ON') - t.after(() => db.close()) +async function managerWithRepo(t) { + const db = getDb() + t.after(async () => await db.close()) const repo = new ClusterNodesRepo(db) - repo.insert({ + await repo.insert({ id: 'node-a', label: 'A', host: '203.0.113.5', @@ -242,26 +238,27 @@ function managerWithRepo(t) { dataDir: os.tmpdir(), logger: { log() {}, warn: (m) => warnings.push(m) }, }) + await manager.reload() return { db, repo, manager, warnings } } test('direct link: token and pin survive edits and never leave; failures fall back to SSH with backoff', async (t) => { - const { db, repo, manager, warnings } = managerWithRepo(t) + const { db, repo, manager, warnings } = await managerWithRepo(t) const gate = await fakeGate(t) await manager.setDirectLink('node-a', { address: '127.0.0.1', port: 1025 }) - const token = repo.getDirectLink('node-a').token + const token = (await repo.getDirectLink('node-a')).token assert.equal(await manager._dialDirect('node-a', 'vm-1', 'kernel'), null, 'no pin yet: SSH') - assert.equal(manager.get('node-a').direct_link.state, 'pending') + assert.equal((await manager.get('node-a')).direct_link.state, 'pending') - repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: TOKEN }) - repo.setDirectLinkCert('node-a', gate.certSha256) + await repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: TOKEN }) + await repo.setDirectLinkCert('node-a', gate.certSha256) await manager.setDirectLink('node-a', { address: gate.address, port: gate.port }) - assert.equal(repo.getDirectLink('node-a').token, TOKEN, 'edit keeps the token') + assert.equal((await repo.getDirectLink('node-a')).token, TOKEN, 'edit keeps the token') assert.notEqual(token, TOKEN) - assert.equal(repo.getDirectLink('node-a').certSha256, gate.certSha256, 'edit keeps the pin') - assert.equal(manager.get('node-a').direct_link.state, 'ok') - const listed = JSON.stringify(manager.list()) + assert.equal((await repo.getDirectLink('node-a')).certSha256, gate.certSha256, 'edit keeps the pin') + assert.equal((await manager.get('node-a')).direct_link.state, 'ok') + const listed = JSON.stringify(await manager.list()) assert.equal(listed.includes(TOKEN), false) assert.equal(listed.includes(gate.certSha256), false) @@ -271,28 +268,28 @@ test('direct link: token and pin survive edits and never leave; failures fall ba conn.sock.destroy() const missing = await manager._dialDirect('node-a', 'vm-2', 'kernel') await assert.rejects(missing.ready, { code: 'gate_no_socket' }) - assert.equal(manager.get('node-a').direct_link.state, 'ok', 'a missing slot socket is not a link fault') + assert.equal((await manager.get('node-a')).direct_link.state, 'ok', 'a missing slot socket is not a link fault') - repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: 'c'.repeat(64) }) + await repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: 'c'.repeat(64) }) const refused = await manager._dialDirect('node-a', 'vm-1', 'kernel') await assert.rejects(refused.ready, { code: 'gate_auth' }) assert.match(warnings.at(-1), /gate_auth/) - repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: TOKEN }) + await repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: TOKEN }) const before = gate.seen.length assert.equal(await manager._dialDirect('node-a', 'vm-1', 'kernel'), null, 'still in backoff') assert.equal(gate.seen.length, before) - assert.equal(manager.get('node-a').direct_link.state, 'down') - assert.ok(manager.directState.get('node-a').downUntil <= Date.now() + DIRECT_LINK_BACKOFF_MS) + assert.equal((await manager.get('node-a')).direct_link.state, 'down') + assert.ok((await manager.directState.get('node-a')).downUntil <= Date.now() + DIRECT_LINK_BACKOFF_MS) await manager._probeDirect('node-a') - assert.equal(manager.get('node-a').direct_link.state, 'ok', 'a good ping lifts the backoff') + assert.equal((await manager.get('node-a')).direct_link.state, 'ok', 'a good ping lifts the backoff') - repo.setDirectLink('node-a', { address: '127.0.0.1', port: 1, token: TOKEN }) + await repo.setDirectLink('node-a', { address: '127.0.0.1', port: 1, token: TOKEN }) assert.equal(await manager._dialDirect('node-a', 'vm-1', 'kernel'), null, 'unreachable: SSH') assert.match(warnings.at(-1), /gate_unreachable/) await manager.clearDirectLink('node-a') - assert.equal(manager.get('node-a').direct_link, null) - repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: TOKEN }) - db.prepare('DELETE FROM cluster_nodes WHERE id = ?').run('node-a') - assert.equal(repo.getDirectLink('node-a'), null, 'removing the node drops its link') + assert.equal((await manager.get('node-a')).direct_link, null) + await repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: TOKEN }) + await db.prepare('DELETE FROM cluster_nodes WHERE id = $1').run('node-a') + assert.equal(await repo.getDirectLink('node-a'), null, 'removing the node drops its link') }) diff --git a/test/unit/cluster-native-node.test.mjs b/test/unit/cluster-native-node.test.mjs index 289fd1909..4f9110f7c 100644 --- a/test/unit/cluster-native-node.test.mjs +++ b/test/unit/cluster-native-node.test.mjs @@ -3,7 +3,9 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { DatabaseSync } from 'node:sqlite' +import { EventEmitter } from 'node:events' +import { getDb } from '../../src/lib/db/database.mjs' +import { ClusterManager } from '../../src/lib/cluster/cluster-manager.mjs' import { describeNativePayload, isDarwinArm64MachO, @@ -183,13 +185,10 @@ test('node platform comes from uname -s and the darwin host stats parse like /pr assert.equal(stats.containers_mem, 1048576) }) -test('platform is stored per node without touching the edit version', () => { - const db = new DatabaseSync(':memory:') - for (const name of ['026_cluster_nodes.sql', '040_cluster_node_platform.sql']) { - db.exec(fs.readFileSync(new URL(`../../src/lib/db/migrations/${name}`, import.meta.url), 'utf8')) - } +test('platform is stored per node without touching the edit version', async () => { + const db = getDb() const repo = new ClusterNodesRepo(db) - const node = repo.insert({ + const node = await repo.insert({ id: 'node-m', label: 'mini', host: 'mini.example.test', @@ -200,12 +199,64 @@ test('platform is stored per node without touching the edit version', () => { host_key_alg: 'ssh-ed25519', host_key_sha256: 'SHA256:x', }) - assert.equal(repo.platform('node-m'), 'linux') - assert.equal(repo.setPlatform('node-m', 'darwin'), true) - assert.equal(repo.platform('node-m'), 'darwin') - assert.equal(repo.get('node-m').updated_at, node.updated_at) - assert.equal(repo.platform('node-gone'), null) - db.close() + assert.equal(await repo.platform('node-m'), 'linux') + assert.equal(await repo.setPlatform('node-m', 'darwin'), true) + assert.equal(await repo.platform('node-m'), 'darwin') + assert.equal((await repo.get('node-m')).updated_at, node.updated_at) + assert.equal(await repo.platform('node-gone'), null) + await db.close() +}) + +test('remote host discovery waits for the persisted platform and preserves one in-flight read', async () => { + let releaseSave + const savePending = new Promise((resolve) => { + releaseSave = resolve + }) + let signalSave + const saveStarted = new Promise((resolve) => { + signalSave = resolve + }) + const manager = new ClusterManager({ + repo: { + setPlatform: async () => { + signalSave() + await savePending + }, + }, + dataDir: '/unused', + logger: { log() {}, warn() {} }, + }) + manager.nodes.set('node-mac', { id: 'node-mac' }) + manager.platforms.set('node-mac', 'linux') + const client = { + exec(_command, callback) { + const stream = new EventEmitter() + stream.stderr = new EventEmitter() + stream.close = () => {} + callback(null, stream) + queueMicrotask(() => { + stream.emit('data', Buffer.from('501\n20\n/Users/kin\nsudo\nDarwin\n')) + stream.emit('exit', 0) + stream.emit('close') + }) + }, + } + manager.links.set('node-mac', { state: 'ready', client }) + const pending = manager.remoteHost('node-mac') + assert.equal(manager.remoteHost('node-mac'), pending) + let settled = false + pending.then(() => { + settled = true + }) + try { + await saveStarted + await Promise.resolve() + assert.equal(settled, false) + } finally { + releaseSave() + } + assert.equal((await pending).platform, 'darwin') + assert.equal(manager.nodePlatform('node-mac'), 'darwin') }) function bindMac(t, { ready = true, root = '/Users/kin/.vm2api', projectRoot = '/unused' } = {}) { diff --git a/test/unit/cluster-node-edit-repo.test.mjs b/test/unit/cluster-node-edit-repo.test.mjs index ef4bc6c49..3fd740725 100644 --- a/test/unit/cluster-node-edit-repo.test.mjs +++ b/test/unit/cluster-node-edit-repo.test.mjs @@ -1,7 +1,6 @@ import test from 'node:test' import assert from 'node:assert/strict' -import fs from 'node:fs' -import { DatabaseSync } from 'node:sqlite' +import { getDb } from '../../src/lib/db/database.mjs' import { ClusterNodesRepo } from '../../src/lib/db/repos/cluster-nodes-repo.mjs' const NODE = { @@ -19,7 +18,7 @@ const NODE = { jump_node_id: null, } -function setup(t, secret) { +async function setup(t, secret) { const env = { VM2API_DB_SECRET: process.env.VM2API_DB_SECRET, KIN_DB_SECRET: process.env.KIN_DB_SECRET } delete process.env.VM2API_DB_SECRET delete process.env.KIN_DB_SECRET @@ -30,9 +29,8 @@ function setup(t, secret) { else process.env[key] = value } }) - const db = new DatabaseSync(':memory:') - db.exec(fs.readFileSync(new URL('../../src/lib/db/migrations/026_cluster_nodes.sql', import.meta.url), 'utf8')) - t.after(() => db.close()) + const db = getDb() + t.after(async () => await db.close()) return { db, repo: new ClusterNodesRepo(db) } } @@ -42,12 +40,12 @@ function assertPublic(node) { } } -test('node rename preserves identity, pinned host key, jump node and credentials without exposing secrets', (t) => { - const { repo } = setup(t) - repo.insert({ ...NODE, id: 'node-jump', host: 'jump.example.test' }) +test('node rename preserves identity, pinned host key, jump node and credentials without exposing secrets', async (t) => { + const { repo } = await setup(t) + await repo.insert({ ...NODE, id: 'node-jump', host: 'jump.example.test' }) const spec = { ...NODE, jump_node_id: 'node-jump' } - const original = repo.insert(spec, '2020-01-01T00:00:00.000Z') - const saved = repo.update( + const original = await repo.insert(spec, '2020-01-01T00:00:00.000Z') + const saved = await repo.update( original.id, { ...spec, @@ -65,110 +63,112 @@ test('node rename preserves identity, pinned host key, jump node and credentials assert.equal(saved[key], original[key]) } assert.ok(Date.parse(saved.updated_at) > Date.parse(original.updated_at)) - assert.deepEqual(repo.getSecrets(original.id), { + assert.deepEqual(await repo.getSecrets(original.id), { password: NODE.password, privateKey: null, passphrase: null, }) assertPublic(saved) - assertPublic(repo.get(original.id)) - for (const node of repo.list()) assertPublic(node) + assertPublic(await repo.get(original.id)) + for (const node of await repo.list()) assertPublic(node) }) -test('label-only edits preserve encrypted credential bytes even when the current database key cannot decrypt them', (t) => { - const { db, repo } = setup(t, 'original-test-database-key') - const original = repo.insert({ ...NODE, private_key: 'saved-private-key', passphrase: 'saved-passphrase' }) - const credentials = db.prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = ?') - const stored = credentials.get(NODE.id) +test('label-only edits preserve encrypted credential bytes even when the current database key cannot decrypt them', async (t) => { + const { db, repo } = await setup(t, 'original-test-database-key') + const original = await repo.insert({ ...NODE, private_key: 'saved-private-key', passphrase: 'saved-passphrase' }) + const credentials = db.prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = $1') + const stored = await credentials.get(NODE.id) for (const value of Object.values(stored)) assert.match(value, /^enc:v1:/) process.env.VM2API_DB_SECRET = 'different-test-database-key' - assert.throws(() => repo.getSecrets(NODE.id)) - const saved = repo.updateLabel(NODE.id, 'Renamed without credentials', original.updated_at) + await assert.rejects(async () => await repo.getSecrets(NODE.id)) + const saved = await repo.updateLabel(NODE.id, 'Renamed without credentials', original.updated_at) assert.deepEqual(saved, { ...original, label: 'Renamed without credentials', updated_at: saved.updated_at }) assert.ok(Date.parse(saved.updated_at) > Date.parse(original.updated_at)) assertPublic(saved) - assert.deepEqual(credentials.get(NODE.id), stored) - assert.throws(() => repo.getSecrets(NODE.id)) + assert.deepEqual(await credentials.get(NODE.id), stored) + await assert.rejects(async () => await repo.getSecrets(NODE.id)) }) -test('label-only updates enforce CAS and advance versions when the clock moves backwards', (t) => { - const { repo } = setup(t) +test('label-only updates enforce CAS and advance versions when the clock moves backwards', async (t) => { + const { repo } = await setup(t) const timestamp = '2026-10-18T00:00:00.000Z' const initialMs = Date.parse(timestamp) - const original = repo.insert(NODE, timestamp) + const original = await repo.insert(NODE, timestamp) const clock = t.mock.method(Date, 'now', () => initialMs) - const first = repo.updateLabel(NODE.id, 'First rename', original.updated_at) + const first = await repo.updateLabel(NODE.id, 'First rename', original.updated_at) assert.equal(Date.parse(first.updated_at), initialMs + 1) - assert.equal(repo.updateLabel(NODE.id, 'Stale rename', original.updated_at), null) - assert.equal(repo.updateLabel('node-missing', 'Missing node', first.updated_at), null) - assert.equal(repo.updateLabel(NODE.id, 'Missing version', undefined), null) - assert.deepEqual(repo.get(NODE.id), first) + assert.equal(await repo.updateLabel(NODE.id, 'Stale rename', original.updated_at), null) + assert.equal(await repo.updateLabel('node-missing', 'Missing node', first.updated_at), null) + assert.equal(await repo.updateLabel(NODE.id, 'Missing version', undefined), null) + assert.deepEqual(await repo.get(NODE.id), first) clock.mock.mockImplementation(() => initialMs - 1000) - const second = repo.updateLabel(NODE.id, 'Second rename', first.updated_at) + const second = await repo.updateLabel(NODE.id, 'Second rename', first.updated_at) assert.equal(Date.parse(second.updated_at), initialMs + 2) - assert.equal(repo.getSecrets(NODE.id).password, NODE.password) + assert.equal((await repo.getSecrets(NODE.id)).password, NODE.password) }) -test('stale and missing node updates return null without overwriting a successful credential change', (t) => { - const { repo } = setup(t) - const original = repo.insert(NODE) - const saved = repo.update(original.id, { ...NODE, password: 'new-password' }, original.updated_at) +test('stale and missing node updates return null without overwriting a successful credential change', async (t) => { + const { repo } = await setup(t) + const original = await repo.insert(NODE) + const saved = await repo.update(original.id, { ...NODE, password: 'new-password' }, original.updated_at) assert.ok(saved) - assert.equal(repo.update(original.id, { ...NODE, label: 'Stale edit' }, original.updated_at), null) - assert.equal(repo.update('node-missing', NODE, original.updated_at), null) - assert.equal(repo.update(original.id, NODE, undefined), null) - assert.deepEqual(repo.get(original.id), saved) - assert.equal(repo.getSecrets(original.id).password, 'new-password') + assert.equal(await repo.update(original.id, { ...NODE, label: 'Stale edit' }, original.updated_at), null) + assert.equal(await repo.update('node-missing', NODE, original.updated_at), null) + assert.equal(await repo.update(original.id, NODE, undefined), null) + assert.deepEqual(await repo.get(original.id), saved) + assert.equal((await repo.getSecrets(original.id)).password, 'new-password') }) -test('each successful update advances its version even when the clock has not advanced or moves backwards', (t) => { - const { repo } = setup(t) +test('each successful update advances its version even when the clock has not advanced or moves backwards', async (t) => { + const { repo } = await setup(t) const timestamp = '2026-10-18T00:00:00.000Z' const initialMs = Date.parse(timestamp) - const original = repo.insert(NODE, timestamp) + const original = await repo.insert(NODE, timestamp) const clock = t.mock.method(Date, 'now', () => initialMs) - const first = repo.update(original.id, { ...NODE, label: 'First' }, original.updated_at) + const first = await repo.update(original.id, { ...NODE, label: 'First' }, original.updated_at) assert.equal(Date.parse(first.updated_at), initialMs + 1) clock.mock.mockImplementation(() => initialMs - 1000) - const second = repo.update(original.id, { ...NODE, label: 'Second' }, first.updated_at) + const second = await repo.update(original.id, { ...NODE, label: 'Second' }, first.updated_at) assert.equal(Date.parse(second.updated_at), initialMs + 2) clock.mock.mockImplementation(() => initialMs + 10_000) - const third = repo.update(original.id, { ...NODE, label: 'Third' }, second.updated_at) + const third = await repo.update(original.id, { ...NODE, label: 'Third' }, second.updated_at) assert.equal(Date.parse(third.updated_at), initialMs + 10_000) }) -test('switching to key authentication encrypts new secrets and removes the old password', (t) => { - const { db, repo } = setup(t, 'test-only-database-secret') - const original = repo.insert(NODE) - const saved = repo.update( +test('switching to key authentication encrypts new secrets and removes the old password', async (t) => { + const { db, repo } = await setup(t, 'test-only-database-secret') + const original = await repo.insert(NODE) + const saved = await repo.update( original.id, { ...NODE, auth_type: 'key', password: null, private_key: 'new-private-key', passphrase: 'new-passphrase' }, original.updated_at, ) assert.equal(saved.auth_type, 'key') assertPublic(saved) - const stored = db.prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = ?').get(NODE.id) + const stored = await db + .prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = $1') + .get(NODE.id) assert.equal(stored.password, null) assert.match(stored.private_key, /^enc:v1:/) assert.match(stored.passphrase, /^enc:v1:/) - assert.deepEqual(repo.getSecrets(NODE.id), { + assert.deepEqual(await repo.getSecrets(NODE.id), { password: null, privateKey: 'new-private-key', passphrase: 'new-passphrase', }) }) -test('switching to password authentication clears old key secrets without requiring database encryption', (t) => { - const { db, repo } = setup(t) - const original = repo.insert({ +test('switching to password authentication clears old key secrets without requiring database encryption', async (t) => { + const { db, repo } = await setup(t) + const original = await repo.insert({ ...NODE, auth_type: 'key', password: null, private_key: 'old-private-key', passphrase: 'old-passphrase', }) - const saved = repo.update( + const saved = await repo.update( original.id, { ...NODE, host: 'new.example.test', port: 2222, username: 'new-operator', password: 'new-password' }, original.updated_at, @@ -178,23 +178,25 @@ test('switching to password authentication clears old key secrets without requir ['new.example.test', 2222, 'new-operator', 'password'], ) assertPublic(saved) - const stored = db.prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = ?').get(NODE.id) + const stored = await db + .prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = $1') + .get(NODE.id) assert.deepEqual({ ...stored }, { password: 'new-password', private_key: null, passphrase: null }) }) -test('a duplicate endpoint rejects the whole update and preserves the previous version and credentials', (t) => { - const { repo } = setup(t) - const original = repo.insert(NODE) - repo.insert({ ...NODE, id: 'node-b', host: 'node-b.example.test' }) - assert.throws( - () => - repo.update( +test('a duplicate endpoint rejects the whole update and preserves the previous version and credentials', async (t) => { + const { repo } = await setup(t) + const original = await repo.insert(NODE) + await repo.insert({ ...NODE, id: 'node-b', host: 'node-b.example.test' }) + await assert.rejects( + async () => + await repo.update( NODE.id, { ...NODE, host: 'node-b.example.test', label: 'Conflict', password: 'uncommitted-password' }, original.updated_at, ), - /UNIQUE constraint failed/, + { code: '23505' }, ) - assert.deepEqual(repo.get(NODE.id), original) - assert.equal(repo.getSecrets(NODE.id).password, NODE.password) + assert.deepEqual(await repo.get(NODE.id), original) + assert.equal((await repo.getSecrets(NODE.id)).password, NODE.password) }) diff --git a/test/unit/cluster-node-edit.test.mjs b/test/unit/cluster-node-edit.test.mjs index 3a1af006a..342be9757 100644 --- a/test/unit/cluster-node-edit.test.mjs +++ b/test/unit/cluster-node-edit.test.mjs @@ -1,9 +1,8 @@ import test from 'node:test' import assert from 'node:assert/strict' -import fs from 'node:fs' import http from 'node:http' import net from 'node:net' -import { DatabaseSync } from 'node:sqlite' +import { getDb } from '../../src/lib/db/database.mjs' import ssh2 from 'ssh2' import { ClusterManager } from '../../src/lib/cluster/cluster-manager.mjs' import { ClusterError, connectSsh, hostKeyDigest, SshLink } from '../../src/lib/cluster/ssh-link.mjs' @@ -11,9 +10,6 @@ import { dockerPing, sshDocker } from '../../src/lib/cluster/docker-remote.mjs' import { ClusterNodesRepo } from '../../src/lib/db/repos/cluster-nodes-repo.mjs' const { Server, utils } = ssh2 -const schema = ['026_cluster_nodes.sql', '039_cluster_node_direct_links.sql'] - .map((name) => fs.readFileSync(new URL(`../../src/lib/db/migrations/${name}`, import.meta.url), 'utf8')) - .join('\n') const key = () => utils.generateKeyPairSync('ecdsa', { bits: 256 }) const defer = () => { let resolve @@ -109,10 +105,9 @@ async function sshHost(t, options = {}) { async function fixture(t, opts = {}) { const remote = await sshHost(t) - const db = new DatabaseSync(':memory:') - db.exec(schema) + const db = getDb() const repo = new ClusterNodesRepo(db) - repo.insert( + await repo.insert( { id: 'node-a', label: 'A', @@ -134,12 +129,13 @@ async function fixture(t, opts = {}) { assertNodeEditable: opts.assertNodeEditable, logger: { log() {}, warn() {} }, }) - const link = new SshLink({ id: 'node-a', resolveTarget: () => manager._targetFor('node-a') }) + await manager.reload() + const link = new SshLink({ id: 'node-a', resolveTarget: async () => await manager._targetFor('node-a') }) manager.links.set('node-a', link) if (opts.ready !== false) link.adopt(await connectSsh(await manager._targetFor('node-a'))) t.after(async () => { await manager.stop() - db.close() + await db.close() }) return { ...remote, @@ -148,7 +144,7 @@ async function fixture(t, opts = {}) { manager, link, vms, - patch: (change) => ({ expected_updated_at: repo.get('node-a').updated_at, ...change }), + patch: async (change) => ({ expected_updated_at: (await repo.get('node-a')).updated_at, ...change }), } } @@ -198,18 +194,24 @@ test('rename changes only metadata, even when the node has VMs or jump dependent }, }) slot(f) - f.repo.insert({ ...f.repo.get('node-a'), id: 'node-b', port: f.port + 1, jump_node_id: 'node-a', password: 'pw' }) + await f.repo.insert({ + ...(await f.repo.get('node-a')), + id: 'node-b', + port: f.port + 1, + jump_node_id: 'node-a', + password: 'pw', + }) const client = f.link.client const authCount = f.state.auth.length - const before = f.repo.get('node-a') + const before = await f.repo.get('node-a') const getSecrets = f.repo.getSecrets.bind(f.repo) f.repo.getSecrets = () => { throw new Error('stored credentials cannot be decrypted') } - const probe = await f.manager.probeUpdate('node-a', f.patch({ label: 'Renamed' })) + const probe = await f.manager.probeUpdate('node-a', await f.patch({ label: 'Renamed' })) assert.equal(probe.vm_count, 1) assert.equal(probe.dependent_count, 1) - const result = await f.manager.update('node-a', f.patch({ label: 'Renamed' })) + const result = await f.manager.update('node-a', await f.patch({ label: 'Renamed' })) assert.equal(result.label, 'Renamed') assert.equal(result.id, before.id) assert.equal(result.created_at, before.created_at) @@ -217,7 +219,7 @@ test('rename changes only metadata, even when the node has VMs or jump dependent assert.equal(f.state.auth.length, authCount) assert.deepEqual(f.state.requests, []) assert.equal(JSON.stringify(result).includes('old-password'), false) - assert.equal(getSecrets('node-a').password, 'old-password') + assert.equal((await getSecrets('node-a')).password, 'old-password') }) test('successful credential rotation revalidates on save, reuses the link and clears old shell tickets', async (t) => { @@ -225,7 +227,7 @@ test('successful credential rotation revalidates on save, reuses the link and cl const client = f.link.client const relayClient = () => (f.link.state === 'ready' ? f.link.client : null) const ticket = f.manager.issueShellTicket('node-a').ticket - const patch = f.patch({ password: 'new-password' }) + const patch = await f.patch({ password: 'new-password' }) const preview = await f.manager.probeUpdate('node-a', patch) assert.deepEqual(preview.docker, { ok: true, error: null }) assert.equal(preview.host_key.sha256, f.sha256) @@ -236,7 +238,7 @@ test('successful credential rotation revalidates on save, reuses the link and cl assert.ok(f.state.auth.length > authCount, 'save opens its own verified candidate') assert.equal(f.manager.links.get('node-a'), f.link) assert.notEqual(relayClient(), client) - assert.equal(f.repo.getSecrets('node-a').password, 'new-password') + assert.equal((await f.repo.getSecrets('node-a')).password, 'new-password') assert.equal(result.link.state, 'ready') assert.equal(f.manager.consumeShellTicket(ticket, 'node-a'), false) assert.equal(await dockerPing(sshDocker(relayClient())), true) @@ -245,13 +247,13 @@ test('successful credential rotation revalidates on save, reuses the link and cl test('a successful preview is not a credential authorization for a later save', async (t) => { const f = await fixture(t) - const patch = f.patch({ password: 'new-password' }) + const patch = await f.patch({ password: 'new-password' }) await f.manager.probeUpdate('node-a', patch) f.state.passwords.delete('new-password') const client = f.link.client await assert.rejects(f.manager.update('node-a', patch), { code: 'auth_failed' }) assert.equal(f.link.client, client) - assert.equal(f.repo.getSecrets('node-a').password, 'old-password') + assert.equal((await f.repo.getSecrets('node-a')).password, 'old-password') }) test('resubmitted saved credentials or equivalent endpoints are authenticated instead of trusting cached health', async (t) => { @@ -262,14 +264,14 @@ test('resubmitted saved credentials or equivalent endpoints are authenticated in f.state.passwords.delete('old-password') // A previously successful observation must not authorize a new request. f.manager.health.set('node-a', { docker: { ok: true, error: null } }) - const before = f.repo.get('node-a') + const before = await f.repo.get('node-a') const client = f.link.client for (const change of [{ password: 'old-password' }, { host: ' 127.0.0.1 ' }, { port: String(f.port) }]) { const count = f.state.auth.length - await assert.rejects(f.manager.probeUpdate('node-a', f.patch(change)), { code: 'auth_failed' }) - await assert.rejects(f.manager.update('node-a', f.patch(change)), { code: 'auth_failed' }) + await assert.rejects(f.manager.probeUpdate('node-a', await f.patch(change)), { code: 'auth_failed' }) + await assert.rejects(f.manager.update('node-a', await f.patch(change)), { code: 'auth_failed' }) assert.ok(f.state.auth.length > count) - assert.deepEqual(f.repo.get('node-a'), before) + assert.deepEqual(await f.repo.get('node-a'), before) assert.equal(f.link.client, client) } }) @@ -280,20 +282,20 @@ test('password to key rotation clears the saved password and remains usable', as const f = await fixture(t) const userKey = key() f.state.publicKey = Buffer.from(userKey.public.split(' ')[1], 'base64') - await f.manager.update('node-a', f.patch({ auth_type: 'key', private_key: userKey.private })) - assert.equal(f.repo.get('node-a').auth_type, 'key') - assert.equal(f.repo.getSecrets('node-a').password, null) - assert.equal(f.repo.getSecrets('node-a').privateKey, userKey.private.trim()) + await f.manager.update('node-a', await f.patch({ auth_type: 'key', private_key: userKey.private })) + assert.equal((await f.repo.get('node-a')).auth_type, 'key') + assert.equal((await f.repo.getSecrets('node-a')).password, null) + assert.equal((await f.repo.getSecrets('node-a')).privateKey, userKey.private.trim()) assert.equal(f.link.state, 'ready') }) test('candidate authentication failure leaves saved credentials and old connection intact', async (t) => { const f = await fixture(t) - const before = f.repo.get('node-a') + const before = await f.repo.get('node-a') const client = f.link.client - await assert.rejects(f.manager.update('node-a', f.patch({ password: 'wrong' })), { code: 'auth_failed' }) - assert.deepEqual(f.repo.get('node-a'), before) - assert.equal(f.repo.getSecrets('node-a').password, 'old-password') + await assert.rejects(f.manager.update('node-a', await f.patch({ password: 'wrong' })), { code: 'auth_failed' }) + assert.deepEqual(await f.repo.get('node-a'), before) + assert.equal((await f.repo.getSecrets('node-a')).password, 'old-password') assert.equal(f.link.client, client) await dockerPing(sshDocker(client)) }) @@ -302,30 +304,34 @@ test('changed endpoints must have the saved host key before credentials are offe const f = await fixture(t) const impostor = await sshHost(t) const client = f.link.client - await assert.rejects(f.manager.update('node-a', f.patch({ port: impostor.port })), { code: 'host_key_mismatch' }) + await assert.rejects(f.manager.update('node-a', await f.patch({ port: impostor.port })), { + code: 'host_key_mismatch', + }) assert.deepEqual(impostor.state.auth, []) - assert.equal(f.repo.get('node-a').port, f.port) + assert.equal((await f.repo.get('node-a')).port, f.port) assert.equal(f.link.client, client) const sameHost = await sshHost(t, { hostKey: f.hostKey }) - await f.manager.update('node-a', f.patch({ port: sameHost.port })) - assert.equal(f.repo.get('node-a').port, sameHost.port) + await f.manager.update('node-a', await f.patch({ port: sameHost.port })) + assert.equal((await f.repo.get('node-a')).port, sameHost.port) }) test('empty nodes can change username and save without Docker, but existing slots require Docker', async (t) => { const f = await fixture(t) f.state.username = 'nextuser' f.state.docker = false - const patch = f.patch({ username: 'nextuser' }) + const patch = await f.patch({ username: 'nextuser' }) const probe = await f.manager.probeUpdate('node-a', patch) assert.equal(probe.docker.ok, false) await f.manager.update('node-a', patch) - assert.equal(f.repo.get('node-a').username, 'nextuser') + assert.equal((await f.repo.get('node-a')).username, 'nextuser') slot(f) const client = f.link.client - await assert.rejects(f.manager.update('node-a', f.patch({ password: 'new-password' })), { + await assert.rejects(f.manager.update('node-a', await f.patch({ password: 'new-password' })), { code: 'node_runtime_mismatch', }) - await assert.rejects(f.manager.update('node-a', f.patch({ username: 'third' })), { code: 'node_username_in_use' }) + await assert.rejects(f.manager.update('node-a', await f.patch({ username: 'third' })), { + code: 'node_username_in_use', + }) assert.equal(f.link.client, client) }) @@ -335,13 +341,13 @@ test('VM identity and Docker/KVM mount roots are verified without changing conta const f = await fixture(t) const vm = slot(f, runtime) const snapshot = JSON.stringify(vm) - await f.manager.update('node-a', f.patch({ password: 'new-password' })) + await f.manager.update('node-a', await f.patch({ password: 'new-password' })) assert.equal(JSON.stringify(vm), snapshot) assert.ok(f.state.requests.every(([method]) => method === 'GET')) assert.ok(f.state.commands.every((command) => command.startsWith('sh -c ') || command.startsWith('test -d '))) f.state.containers.get('container-a').Mounts[0].Source = '/elsewhere/run' const client = f.link.client - await assert.rejects(f.manager.update('node-a', f.patch({ password: 'old-password' })), { + await assert.rejects(f.manager.update('node-a', await f.patch({ password: 'old-password' })), { code: 'node_runtime_mismatch', }) assert.equal(f.link.client, client) @@ -360,12 +366,12 @@ test('changed HOME, UID/GID, missing container or directories reject an attached if (change === 'gid') f.state.gid = 999 if (change === 'container') f.state.containers.clear() if (change === 'directories') f.state.directories = false - const before = f.repo.get('node-a') + const before = await f.repo.get('node-a') const client = f.link.client - await assert.rejects(f.manager.update('node-a', f.patch({ password: 'new-password' })), { + await assert.rejects(f.manager.update('node-a', await f.patch({ password: 'new-password' })), { code: 'node_runtime_mismatch', }) - assert.deepEqual(f.repo.get('node-a'), before) + assert.deepEqual(await f.repo.get('node-a'), before) assert.equal(f.link.client, client) }) } @@ -374,38 +380,38 @@ test('changed HOME, UID/GID, missing container or directories reject an attached test('a disconnected node can recover using its pinned identity and recorded runtime', async (t) => { const f = await fixture(t, { ready: false }) slot(f) - await f.manager.update('node-a', f.patch({ password: 'new-password' })) + await f.manager.update('node-a', await f.patch({ password: 'new-password' })) assert.equal(f.link.state, 'ready') - assert.equal(f.repo.getSecrets('node-a').password, 'new-password') + assert.equal((await f.repo.getSecrets('node-a')).password, 'new-password') }) test('connection updates reject jump dependents, duplicate endpoints and busy installs before connecting', async (t) => { const f = await fixture(t) - const next = { ...f.repo.get('node-a'), id: 'node-b', port: f.port + 1, password: 'pw' } - f.repo.insert({ ...next, jump_node_id: 'node-a' }) + const next = { ...(await f.repo.get('node-a')), id: 'node-b', port: f.port + 1, password: 'pw' } + await f.repo.insert({ ...next, jump_node_id: 'node-a' }) const count = f.state.auth.length - await assert.rejects(f.manager.update('node-a', f.patch({ password: 'new-password' })), { code: 'node_in_use' }) - f.repo.remove('node-b') - f.repo.insert(next) - await assert.rejects(f.manager.update('node-a', f.patch({ port: next.port })), { code: 'node_exists' }) + await assert.rejects(f.manager.update('node-a', await f.patch({ password: 'new-password' })), { code: 'node_in_use' }) + await f.repo.remove('node-b') + await f.repo.insert(next) + await assert.rejects(f.manager.update('node-a', await f.patch({ port: next.port })), { code: 'node_exists' }) f.manager.installs.set('node-a', { status: 'running' }) - await assert.rejects(f.manager.update('node-a', f.patch({ password: 'new-password' })), { code: 'node_busy' }) + await assert.rejects(f.manager.update('node-a', await f.patch({ password: 'new-password' })), { code: 'node_busy' }) assert.equal(f.state.auth.length, count) }) test('stale edits and database failures do not replace a working link', async (t) => { const f = await fixture(t) - const stale = f.patch({ password: 'new-password' }) - await f.manager.update('node-a', f.patch({ label: 'new label' })) + const stale = await f.patch({ password: 'new-password' }) + await f.manager.update('node-a', await f.patch({ label: 'new label' })) await assert.rejects(f.manager.probeUpdate('node-a', stale), { code: 'node_edit_conflict' }) await assert.rejects(f.manager.update('node-a', stale), { code: 'node_edit_conflict' }) const client = f.link.client - const before = f.repo.get('node-a') + const before = await f.repo.get('node-a') f.repo.update = () => { throw new Error('write failed') } - await assert.rejects(f.manager.update('node-a', f.patch({ password: 'new-password' })), /write failed/) - assert.deepEqual(f.repo.get('node-a'), before) + await assert.rejects(f.manager.update('node-a', await f.patch({ password: 'new-password' })), /write failed/) + assert.deepEqual(await f.repo.get('node-a'), before) assert.equal(f.link.client, client) await dockerPing(sshDocker(client)) }) @@ -424,14 +430,14 @@ test('the final guard rejects a node that becomes busy while its candidate is ch await release.promise } const client = f.link.client - const pending = f.manager.update('node-a', f.patch({ password: 'new-password' })) + const pending = f.manager.update('node-a', await f.patch({ password: 'new-password' })) const rejected = assert.rejects(pending, { code: 'node_busy' }) await entered.promise busy = true release.resolve() await rejected assert.equal(f.link.client, client) - assert.equal(f.repo.getSecrets('node-a').password, 'old-password') + assert.equal((await f.repo.getSecrets('node-a')).password, 'old-password') }) test('reconnect, removal and another save during verification cannot resurrect stale configuration', async (t) => { @@ -444,17 +450,53 @@ test('reconnect, removal and another save during verification cannot resurrect s entered.resolve() await release.promise } - const pending = f.manager.update('node-a', f.patch({ password: 'new-password' })) + const pending = f.manager.update('node-a', await f.patch({ password: 'new-password' })) const rejected = assert.rejects(pending, { code: action === 'remove' ? 'node_not_found' : 'node_edit_conflict' }) await entered.promise if (action === 'reconnect') f.manager.reconnect('node-a') if (action === 'remove') await f.manager.remove('node-a') - if (action === 'rename') await f.manager.update('node-a', f.patch({ label: 'concurrent' })) + if (action === 'rename') await f.manager.update('node-a', await f.patch({ label: 'concurrent' })) if (action === 'placement') f.vms.push({ id: 'new-vm' }) release.resolve() await rejected - if (action !== 'remove') assert.equal(f.repo.getSecrets('node-a').password, 'old-password') - else assert.equal(f.repo.get('node-a'), null) + if (action !== 'remove') assert.equal((await f.repo.getSecrets('node-a')).password, 'old-password') + else assert.equal(await f.repo.get('node-a'), null) }) } }) + +test('reconnect during the database compare-and-swap rolls back credentials and closes the candidate', async (t) => { + const f = await fixture(t) + const before = await f.repo.get('node-a') + const entered = defer(), + release = defer(), + closed = defer() + const update = f.repo.update.bind(f.repo) + f.repo.update = async (...args) => { + const saved = await update(...args) + entered.resolve() + await release.promise + return saved + } + const probe = f.manager._probeEdit.bind(f.manager) + f.manager._probeEdit = async (...args) => { + const candidate = await probe(...args) + candidate.client.once('close', closed.resolve) + return candidate + } + const pending = f.manager.update('node-a', await f.patch({ password: 'new-password' })) + const rejected = assert.rejects(pending, { code: 'node_edit_conflict' }) + await entered.promise + f.manager.reconnect('node-a') + release.resolve() + await rejected + await Promise.race([ + closed.promise, + new Promise((_resolve, reject) => { + const timer = setTimeout(() => reject(new Error('candidate was not closed')), 1500) + timer.unref?.() + }), + ]) + assert.deepEqual(await f.repo.get('node-a'), before) + assert.equal((await f.repo.getSecrets('node-a')).password, 'old-password') +}) diff --git a/test/unit/cluster-placement.test.mjs b/test/unit/cluster-placement.test.mjs index d7e331317..038790105 100644 --- a/test/unit/cluster-placement.test.mjs +++ b/test/unit/cluster-placement.test.mjs @@ -595,10 +595,11 @@ test('socket relay resolves the remote path lazily and reaches a unix socket ove test('a node with placed VMs cannot be removed', async () => { const manager = new ClusterManager({ - repo: { get: () => ({ id: 'n1' }), dependentsOf: () => [] }, + repo: { list: () => [{ id: 'n1' }], get: () => ({ id: 'n1' }), dependentsOf: () => [] }, dataDir: '/tmp', vmsOnNode: () => [{ id: 'vm-02' }], }) + await manager.reload() await assert.rejects(manager.remove('n1'), { code: 'node_has_vms' }) }) diff --git a/test/unit/database-metrics.test.mjs b/test/unit/database-metrics.test.mjs index 63b1a8b9f..a3fc79d62 100644 --- a/test/unit/database-metrics.test.mjs +++ b/test/unit/database-metrics.test.mjs @@ -1,138 +1,61 @@ import test from 'node:test' import assert from 'node:assert/strict' -import fs from 'node:fs' -import os from 'node:os' -import path from 'node:path' -import { DatabaseSync } from 'node:sqlite' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' import { snapshotDatabaseMetrics } from '../../src/lib/db/database-metrics.mjs' -function fakeDb(values = {}, failures = new Set()) { - const statements = [] - return { - statements, - prepare(sql) { - statements.push(sql) - if (failures.has(sql)) throw new Error('read failed') - return { - get() { - if (failures.has(`${sql}:get`)) throw new Error('read failed') - if (sql === 'SELECT 1 AS ok') return { ok: 1 } - if (sql.startsWith('SELECT COUNT(*)')) return { count: values.migration_count ?? 0 } - if (sql.startsWith('SELECT version')) return values.latest ?? undefined - return { value: values[sql] } - }, - } - }, - } -} - -test('snapshot reports the live SQLite/WAL state without exposing its path', () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-db-metrics-')) - const dbPath = path.join(tmp, 'private-name.db') - const db = createDatabase({ dbPath }) - try { - const snapshot = snapshotDatabaseMetrics({ - db, - dbPath, - usageCache: { stats: () => ({ requests: 0, hit_rate: null }) }, - now: () => Date.parse('2026-08-31T00:00:00.000Z'), - clock: (() => { - const samples = [10, 10.375] - return () => samples.shift() - })(), - }) - - assert.equal(snapshot.sampled_at, '2026-08-31T00:00:00.000Z') - assert.equal(snapshot.database.ok, true) - assert.equal(snapshot.database.engine, 'sqlite') - assert.equal(snapshot.database.probe_latency_ms, 0.375) - assert.equal(snapshot.database.journal_mode, 'wal') - assert.equal(snapshot.database.synchronous, 1) - assert.equal(snapshot.database.foreign_keys, true) - assert.equal(snapshot.database.busy_timeout_ms, 5000) - assert.ok(snapshot.database.file_size_bytes > 0) - assert.ok(snapshot.database.page_size_bytes > 0) - assert.ok(snapshot.database.page_count > 0) - assert.ok(snapshot.database.migration_count > 0) - assert.match(snapshot.database.latest_migration, /^\d+$/) - assert.equal(snapshot.usage_cache.requests, 0) - assert.equal(JSON.stringify(snapshot).includes(tmp), false) - assert.equal(JSON.stringify(snapshot).includes('private-name.db'), false) - } finally { - db.close() - fs.rmSync(tmp, { recursive: true, force: true }) - } +const fakeDb = (failures = new Set()) => ({ + pool: { totalCount: 3, idleCount: 2, waitingCount: 1 }, + prepare(sql) { + assert.match(sql, /^SELECT /) + return { + async get() { + if (failures.has(sql)) throw new Error('private connection failure') + if (sql.includes('pg_database_size')) return { bytes: '4096' } + if (sql.includes('COUNT(*)')) return { count: '2' } + if (sql.includes('version,')) return { version: '001', applied_at: '2026-10-21T00:00:00.000Z' } + return { ok: 1 } + }, + } + }, }) -test('snapshot derives page usage and treats absent database files as real zeroes', () => { - const db = fakeDb({ - 'PRAGMA journal_mode': 'wal', - 'PRAGMA synchronous': 1, - 'PRAGMA foreign_keys': 1, - 'PRAGMA busy_timeout': 5000, - 'PRAGMA page_size': 4096, - 'PRAGMA page_count': 10, - 'PRAGMA freelist_count': 3, - migration_count: 2, - latest: { version: '002', applied_at: '2026-08-30T00:00:00.000Z' }, - }) - const missing = path.join(os.tmpdir(), `missing-kin-${process.pid}-${Date.now()}.db`) - const snapshot = snapshotDatabaseMetrics({ db, dbPath: missing, usageCache: null, clock: () => 1 }) - - assert.equal(snapshot.database.used_pages, 7) - assert.equal(snapshot.database.used_bytes, 28_672) - assert.equal(snapshot.database.free_ratio, 0.3) - assert.equal(snapshot.database.file_size_bytes, 0) - assert.equal(snapshot.database.wal_size_bytes, 0) - assert.equal(snapshot.database.shm_size_bytes, 0) - assert.equal(snapshot.usage_cache, null) - assert.ok(db.statements.every((sql) => /^(SELECT 1|SELECT (COUNT|version)|PRAGMA )/.test(sql))) - assert.ok(db.statements.every((sql) => !/(checkpoint|vacuum|optimize|integrity|quick_check)/i.test(sql))) +test('PostgreSQL metrics report database size, migration state and this process pool', async () => { + const snapshot = await snapshotDatabaseMetrics({ db: getDb(), usageCache: { stats: () => ({ requests: 4 }) } }) + assert.equal(snapshot.database.engine, 'postgresql') + assert.equal(snapshot.database.ok, true) + assert.ok(snapshot.database.size_bytes > 0) + assert.ok(snapshot.database.migration_count > 0) + assert.equal(typeof snapshot.database.pool_total, 'number') + assert.equal(snapshot.usage_cache.requests, 4) + assert.equal(JSON.stringify(snapshot).includes('postgresql://'), false) + assert.equal('wal_size_bytes' in snapshot.database, false) }) -test('snapshot preserves partial results when the probe or a metric read fails', () => { - const db = fakeDb( - { - 'PRAGMA journal_mode': 'wal', - 'PRAGMA synchronous': 1, - 'PRAGMA foreign_keys': 1, - 'PRAGMA busy_timeout': 5000, - 'PRAGMA page_size': 4096, - 'PRAGMA page_count': 0, - 'PRAGMA freelist_count': 0, - }, - new Set(['SELECT 1 AS ok', 'PRAGMA synchronous']), - ) - const snapshot = snapshotDatabaseMetrics({ db, dbPath: null, usageCache: { stats: () => null } }) - +test('read-only snapshot retains independent metrics when the probe fails', async () => { + const snapshot = await snapshotDatabaseMetrics({ + db: fakeDb(new Set(['SELECT 1 AS ok'])), + now: () => Date.parse('2026-10-21T00:00:00.000Z'), + usageCache: null, + }) + assert.equal(snapshot.sampled_at, '2026-10-21T00:00:00.000Z') assert.equal(snapshot.database.ok, false) assert.equal(snapshot.database.probe_latency_ms, null) - assert.equal(snapshot.database.journal_mode, 'wal') - assert.equal(snapshot.database.synchronous, null) - assert.equal(snapshot.database.page_count, 0) - assert.equal(snapshot.database.used_pages, 0) - assert.equal(snapshot.database.used_bytes, 0) - assert.equal(snapshot.database.free_ratio, null) - assert.equal(snapshot.database.file_size_bytes, null) + assert.equal(snapshot.database.size_bytes, 4096) + assert.equal(snapshot.database.pool_total, 3) + assert.equal(snapshot.database.pool_idle, 2) + assert.equal(snapshot.database.pool_waiting, 1) + assert.equal(JSON.stringify(snapshot).includes('private'), false) }) -test('snapshot returns null for unavailable page inputs and cache stats failures', () => { - const db = new DatabaseSync(':memory:') - try { - const snapshot = snapshotDatabaseMetrics({ - db, - dbPath: null, - usageCache: { - stats() { - throw new Error('cache unavailable') - }, +test('cache-stat failure does not hide PostgreSQL health', async () => { + const snapshot = await snapshotDatabaseMetrics({ + db: fakeDb(), + usageCache: { + stats() { + throw new Error('cache failed') }, - }) - assert.equal(snapshot.database.migration_count, null) - assert.equal(snapshot.database.latest_migration, null) - assert.equal(snapshot.usage_cache, null) - } finally { - db.close() - } + }, + }) + assert.equal(snapshot.database.ok, true) + assert.equal(snapshot.usage_cache, null) }) diff --git a/test/unit/database.test.mjs b/test/unit/database.test.mjs index a2cd03de7..7453be3f6 100644 --- a/test/unit/database.test.mjs +++ b/test/unit/database.test.mjs @@ -1,417 +1,176 @@ -import { test, beforeEach, afterEach } from 'node:test' +import { test, after } from 'node:test' import assert from 'node:assert/strict' import fs from 'node:fs' -import os from 'node:os' -import path from 'node:path' import crypto from 'node:crypto' -import { DatabaseSync } from 'node:sqlite' import { + createDatabase, openDatabase, closeDatabase, getDb, isDbOpen, - getDbPath, - applyMigrations, - resolveDbPath, - vacuumInto, withTransaction, + applyMigrations, } from '../../src/lib/db/database.mjs' -let tmp - -beforeEach(() => { - tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-db-test-')) -}) - -afterEach(() => { - closeDatabase() - fs.rmSync(tmp, { recursive: true, force: true }) - delete process.env.KIN_DB_PATH +const url = process.env.VM2API_TEST_DATABASE_URL +function options() { + assert.ok(url && /_test$/.test(new URL(url).pathname), 'dedicated PostgreSQL *_test database required') + return { connectionString: url, schema: 'db_' + crypto.randomBytes(8).toString('hex') } +} +const dbs = [] +async function database(extra = {}) { + const db = await createDatabase({ ...options(), ...extra }) + dbs.push(db) + return db +} +after(async () => { + await closeDatabase() + for (const db of dbs) { + if (!db.closed) await db.exec(`DROP SCHEMA IF EXISTS "${db.schema}" CASCADE`) + await db.close() + } }) -test('openDatabase creates db, applies migrations, WAL enabled', () => { - const db = openDatabase({ dataDir: tmp }) - assert.ok(isDbOpen()) - assert.equal(getDbPath(), path.join(tmp, 'kin.db')) - const mode = db.prepare('PRAGMA journal_mode').get() - assert.equal(String(Object.values(mode)[0]).toLowerCase(), 'wal') - const tables = db - .prepare("SELECT name FROM sqlite_master WHERE type='table'") - .all() - .map((r) => r.name) - for (const t of [ +test('PostgreSQL baseline includes business and historical tables and default groups', async () => { + const db = await database() + const rows = await db.prepare('SELECT table_name FROM information_schema.tables WHERE table_schema=$1').all(db.schema) + assert.equal(rows.length, 30) + for (const name of [ 'settings', + 'users', 'api_keys', 'accounts', - 'account_allocations', - 'sticky_sessions', - 'proxies', - 'vms', 'usage_logs', - 'request_log_debug', - 'backup_records', - 'account_runtime', - 'request_attempts', - 'schema_migrations', - 'api_endpoints', - 'api_endpoint_keys', - 'api_endpoint_models', - 'users', - 'groups', - 'account_groups', - 'redeem_codes', + 'thinking_signature_owners', + 'accounts_legacy', + 'panel_users_legacy', ]) { - assert.ok(tables.includes(t), `missing table ${t}`) + assert.ok( + rows.some((r) => r.table_name === name), + name, + ) } - const keyCols = db - .prepare('PRAGMA table_info(api_keys)') - .all() - .map((r) => r.name) - assert.ok(keyCols.includes('category'), 'api_keys missing category') -}) - -test('migration 005 adds protocol usage alignment columns', () => { - const db = openDatabase({ dataDir: tmp }) - const cols = (table) => - db - .prepare(`PRAGMA table_info(${table})`) - .all() - .map((r) => r.name) - const reqCols = cols('usage_logs') - for (const c of [ - 'cache_creation_5m_tokens', - 'cache_creation_1h_tokens', - 'requested_model', - 'upstream_model', - 'model_mismatch', - 'first_token_ms', - 'stop_reason', - 'input_cost', - 'output_cost', - 'cache_read_cost', - 'cache_creation_cost', - 'total_cost', - 'pricing_model', - 'service_tier', - 'speed', - 'long_context', - ]) - assert.ok(reqCols.includes(c), `usage_logs missing ${c}`) - for (const c of ['cache_read_tokens', 'cache_creation_tokens']) { - assert.ok(cols('accounts').includes(c), `accounts missing ${c}`) - assert.ok(cols('api_keys').includes(c), `api_keys missing ${c}`) - } - // scheduling window columns now live on accounts (sub2api SSOT merge) - const accCols = cols('accounts') - for (const c of [ - 'rate_limited_at', - 'rate_limit_reset_at', - 'overload_until', - 'session_window_start', - 'session_window_end', - 'session_window_status', - 'schedulable', - 'priority', - ]) - assert.ok(accCols.includes(c), `accounts missing ${c}`) -}) - -test('pre-005 database upgrades in place and keeps old rows readable', () => { - // Build a DB with migrations 001–004 only, insert a legacy row, then upgrade. - const migSrc = path.resolve('src/lib/db/migrations') - const oldDir = path.join(tmp, 'old-migs') - fs.mkdirSync(oldDir, { recursive: true }) - for (const f of fs.readdirSync(migSrc)) { - if (f < '005') fs.copyFileSync(path.join(migSrc, f), path.join(oldDir, f)) - } - const dbPath = path.join(tmp, 'upgrade.db') - const legacy = new DatabaseSync(dbPath) - applyMigrations(legacy, { migrationsDir: oldDir }) - legacy - .prepare(` - INSERT INTO request_logs (id, request_id, ts, protocol, model, status, input_tokens, output_tokens) - VALUES ('log_old', 'rid_old', ?, 'anthropic.messages', 'claude-sonnet-5', 200, 7, 2) - `) - .run(new Date().toISOString()) - legacy.close() - - process.env.KIN_DB_PATH = dbPath - const db = openDatabase({ dbPath }) - const row = db.prepare('SELECT * FROM usage_logs WHERE request_id = ?').get('rid_old') - assert.equal(row.input_tokens, 7) - // migration 017 normalizes NULL token counters to 0 - assert.equal(row.cache_creation_5m_tokens, 0) - assert.equal(row.stop_reason, null) - // new rows can use the new columns - db.prepare(` - INSERT INTO usage_logs (id, request_id, created_at, protocol, model, status, upstream_model, stop_reason, first_token_ms) - VALUES ('log_new', 'rid_new', ?, 'anthropic.messages', 'claude-sonnet-5', 200, 'claude-sonnet-5', 'end_turn', 21) - `).run(new Date().toISOString()) - assert.equal( - db.prepare('SELECT stop_reason FROM usage_logs WHERE request_id = ?').get('rid_new').stop_reason, - 'end_turn', + assert.deepEqual( + (await db.prepare('SELECT id,name FROM groups ORDER BY id').all()).map((r) => [r.id, r.name]), + [ + [1, 'coding'], + [2, 'other'], + ], ) + assert.deepEqual(await applyMigrations(db), []) }) -test('openDatabase is idempotent singleton; getDb throws when closed', () => { - const a = openDatabase({ dataDir: tmp }) - const b = openDatabase({ dataDir: path.join(tmp, 'other') }) - assert.equal(a, b) - closeDatabase() - assert.throws(() => getDb(), /not opened/) -}) - -test('migrations are idempotent across reopen', () => { - openDatabase({ dataDir: tmp }) - closeDatabase() - const db = openDatabase({ dataDir: tmp }) - const rows = db.prepare('SELECT version FROM schema_migrations').all() - assert.equal(rows.length, new Set(rows.map((r) => r.version)).size) -}) - -test('migration 037 normalizes OpenAI input including NULL route fields; SUM(prompt) holds', () => { - const migSrc = path.resolve('src/lib/db/migrations') - const oldDir = path.join(tmp, 'pre-037') - fs.mkdirSync(oldDir, { recursive: true }) - for (const f of fs.readdirSync(migSrc)) { - if (f < '037') fs.copyFileSync(path.join(migSrc, f), path.join(oldDir, f)) - } - const dbPath = path.join(tmp, 'bill037.db') - const db = new DatabaseSync(dbPath) - applyMigrations(db, { migrationsDir: oldDir }) - - const cacheModel = `LOWER(TRIM(COALESCE( - NULLIF(NULLIF(pricing_model, ''), 'unpriced'), NULLIF(upstream_model, ''), - NULLIF(model, ''), requested_model, '' - )))` - const openaiIds = [ - 'gpt-6-astra', - 'gpt-6.1-sol', - 'gpt-6-sol', - 'gpt-6-luna', - 'gpt-5.6-sol', - 'gpt-5.6-terra', - 'gpt-5.6-luna', - 'gpt-5.6', - 'gpt-5.5-pro', - 'gpt-5.5', - 'gpt-5.4-mini', - 'gpt-5.4-nano', - 'gpt-5.4-pro', - 'gpt-5.4', - 'gpt-5.3-codex', - 'gpt-5.2-pro', - 'gpt-5.2', - 'gpt-5.1', - 'gpt-5-mini', - 'gpt-5-nano', - 'gpt-5-pro', - 'gpt-5', - 'gpt-4.1-mini', - 'gpt-4.1-nano', - 'gpt-4.1', - 'gpt-4o-2024-05-13', - 'gpt-4o-mini', - 'gpt-4o', - 'o1-pro', - 'o1', - 'o3-pro', - 'o3-mini', - 'o3', - 'o4-mini', - 'gpt-4-turbo', - 'gpt-4', - 'gpt-3.5-turbo-instruct', - 'gpt-3.5-turbo-1106', - 'gpt-3.5-turbo', - 'gpt-5.1-codex-mini', - 'gpt-5.3-chat-latest', - 'gpt-5.6-cyber', - 'gpt-5.5-cyber', - 'chat-latest', - 'gpt-5-codex', - 'gpt-5.1-codex', - 'gpt-5.1-codex-max', - 'gpt-5-chat-latest', - 'gpt-5.1-chat-latest', - 'gpt-5.2-codex', - 'gpt-5.2-chat-latest', - ] - .map((id) => `'${id}'`) - .join(', ') - const oldPrompt = `CASE - WHEN via = 'api-kernel' AND protocol = 'anthropic.messages' AND stream = 0 - THEN IFNULL(input_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0) - WHEN via LIKE 'codex-%' - OR (${cacheModel} LIKE 'gpt%' AND ${cacheModel} NOT LIKE 'gpt-image%') - OR ${cacheModel} LIKE 'codex-%' - OR ${cacheModel} IN (${openaiIds}) - THEN IFNULL(input_tokens, 0) - ELSE IFNULL(input_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0) - END` - const newPrompt = 'IFNULL(input_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0)' - - const ins = db.prepare(` - INSERT INTO usage_logs ( - id, request_id, created_at, model, protocol, stream, via, - input_tokens, cache_read_tokens, cache_creation_tokens, output_tokens, upstream_model - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `) - const at = '2026-09-01T00:00:00.000Z' - ins.run('n-via', 'r-via', at, 'gpt-5.4', 'anthropic.messages', 0, null, 100, 80, 0, 2, null) - ins.run('n-proto', 'r-proto', at, 'gpt-5.4', null, 0, 'go-worker-pool', 100, 80, 0, 2, null) - ins.run('n-stream', 'r-stream', at, 'gpt-5.4', 'anthropic.messages', null, 'go-worker-pool', 100, 80, 0, 2, null) - ins.run('n-all', 'r-all', at, 'gpt-5.4', null, null, null, 100, 80, 0, 2, null) - ins.run('keep-api', 'r-api', at, 'gpt-5.4', 'anthropic.messages', 0, 'api-kernel', 20, 80, 0, 2, 'gpt-5.4') - ins.run('keep-claude', 'r-claude', at, 'claude-sonnet-5', 'anthropic.messages', 0, null, 20, 80, 0, 2, null) - ins.run('codex', 'r-codex', at, 'gpt-5.4', 'openai.responses', 1, 'codex-kernel', 100000, 80000, 0, 2000, null) - - const before = db.prepare(`SELECT SUM(${oldPrompt}) AS n FROM usage_logs`).get().n - applyMigrations(db, { migrationsDir: migSrc }) - const after = db.prepare(`SELECT SUM(${newPrompt}) AS n FROM usage_logs`).get().n - assert.equal(after, before) - - const cols = db - .prepare('PRAGMA table_info(usage_logs)') - .all() - .map((r) => r.name) - for (const c of ['web_search_requests', 'web_search_cost', 'price_source', 'price_version']) { - assert.ok(cols.includes(c), `missing ${c}`) - } - - const get = (id) => db.prepare('SELECT input_tokens, cache_read_tokens FROM usage_logs WHERE id = ?').get(id) - assert.equal(get('n-via').input_tokens, 20) - assert.equal(get('n-proto').input_tokens, 20) - assert.equal(get('n-stream').input_tokens, 20) - assert.equal(get('n-all').input_tokens, 20) - assert.equal(get('keep-api').input_tokens, 20) - assert.equal(get('keep-claude').input_tokens, 20) - assert.equal(get('codex').input_tokens, 20000) - // Earlier prunes were never recorded; existing rows only vouch for history from the oldest one. - const pruned = db.prepare("SELECT value FROM settings WHERE key = 'usage_logs_pruned_through'").get() - assert.equal(JSON.parse(pruned.value), at) - db.close() -}) - -test('checksum mismatch on applied migration throws', () => { - const migDir = path.join(tmp, 'migs') - fs.mkdirSync(migDir, { recursive: true }) - fs.writeFileSync(path.join(migDir, '001_a.sql'), 'CREATE TABLE t1 (a TEXT);') - const db = new DatabaseSync(path.join(tmp, 'x.db')) - applyMigrations(db, { migrationsDir: migDir }) - fs.writeFileSync(path.join(migDir, '001_a.sql'), 'CREATE TABLE t1 (a TEXT, b TEXT);') - assert.throws(() => applyMigrations(db, { migrationsDir: migDir }), /checksum mismatch/) - db.close() -}) - -test('CRLF-applied checksum of the same SQL is accepted and rewritten', () => { - const migDir = path.join(tmp, 'migs') - fs.mkdirSync(migDir, { recursive: true }) - const lf = 'CREATE TABLE t1 (a TEXT);\n' - const crlf = lf.replace(/\n/g, '\r\n') - fs.writeFileSync(path.join(migDir, '001_a.sql'), lf) - const db = new DatabaseSync(path.join(tmp, 'x.db')) - applyMigrations(db, { migrationsDir: migDir }) - const crlfHash = crypto.createHash('sha256').update(crlf).digest('hex') - db.prepare('UPDATE schema_migrations SET checksum = ? WHERE version = ?').run(crlfHash, '001') - assert.doesNotThrow(() => applyMigrations(db, { migrationsDir: migDir })) - const stored = db.prepare('SELECT checksum FROM schema_migrations WHERE version = ?').get('001').checksum - assert.notEqual(stored, crlfHash) - assert.equal(stored.length, 64) - db.close() -}) - -test('new migration files apply in order and record checksums', () => { - const migDir = path.join(tmp, 'migs') - fs.mkdirSync(migDir, { recursive: true }) - fs.writeFileSync(path.join(migDir, '001_a.sql'), 'CREATE TABLE t1 (a TEXT);') - const db = new DatabaseSync(path.join(tmp, 'x.db')) - let applied = applyMigrations(db, { migrationsDir: migDir }) - assert.deepEqual(applied, ['001_a.sql']) - fs.writeFileSync(path.join(migDir, '002_b.sql'), 'ALTER TABLE t1 ADD COLUMN b TEXT;') - applied = applyMigrations(db, { migrationsDir: migDir }) - assert.deepEqual(applied, ['002_b.sql']) - const rows = db.prepare('SELECT version, checksum FROM schema_migrations ORDER BY version').all() - assert.equal(rows.length, 2) - assert.ok(rows.every((r) => /^[0-9a-f]{64}$/.test(r.checksum))) - db.close() -}) - -test('failed migration rolls back and is not recorded', () => { - const migDir = path.join(tmp, 'migs') - fs.mkdirSync(migDir, { recursive: true }) - fs.writeFileSync(path.join(migDir, '001_bad.sql'), 'CREATE TABLE ok1 (a TEXT); THIS IS NOT SQL;') - const db = new DatabaseSync(path.join(tmp, 'x.db')) - assert.throws(() => applyMigrations(db, { migrationsDir: migDir }), /001_bad\.sql failed/) - const tables = db - .prepare("SELECT name FROM sqlite_master WHERE type='table'") - .all() - .map((r) => r.name) - assert.ok(!tables.includes('ok1'), 'rollback should drop partial table') - assert.equal(db.prepare('SELECT COUNT(*) c FROM schema_migrations').get().c, 0) - db.close() -}) - -test('KIN_DB_PATH env overrides path', () => { - const custom = path.join(tmp, 'custom', 'my.db') - process.env.KIN_DB_PATH = custom - assert.equal(resolveDbPath({ dataDir: tmp }), custom) - openDatabase({ dataDir: tmp }) - assert.equal(getDbPath(), custom) - assert.ok(fs.existsSync(custom)) -}) - -test('vacuumInto produces a consistent snapshot', () => { - const db = openDatabase({ dataDir: tmp }) - db.prepare('INSERT INTO settings (key, value, updated_at) VALUES (?, ?, ?)').run( - 'k1', - '"v1"', - new Date().toISOString(), +test('transactions await callbacks and rollback writes on the same connection', async () => { + const db = await database() + await withTransaction(db, async () => { + await db.prepare('INSERT INTO settings(key,value) VALUES($1,$2)').run('outer', '1') + await withTransaction(db, async () => { + await db.prepare('INSERT INTO settings(key,value) VALUES($1,$2)').run('nested', '2') + }) + }) + await assert.rejects( + withTransaction(db, async () => { + await db.prepare('INSERT INTO settings(key,value) VALUES($1,$2)').run('rolled', '3') + await new Promise((r) => setImmediate(r)) + throw new Error('rollback marker') + }), + /rollback marker/, + ) + assert.deepEqual( + (await db.prepare('SELECT key FROM settings ORDER BY key').all()).map((r) => r.key), + ['nested', 'outer'], ) - const dest = path.join(tmp, 'backup', 'snap.db') - vacuumInto(db, dest) - const snap = new DatabaseSync(dest) - const row = snap.prepare('SELECT value FROM settings WHERE key = ?').get('k1') - assert.equal(row.value, '"v1"') - snap.close() }) -test('withTransaction commits and rolls back', () => { - const db = openDatabase({ dataDir: tmp }) - withTransaction(db, () => { - db.prepare("INSERT INTO settings (key, value) VALUES ('a', '1')").run() - }) - assert.ok(db.prepare("SELECT * FROM settings WHERE key='a'").get()) - assert.throws( - () => - withTransaction(db, () => { - db.prepare("INSERT INTO settings (key, value) VALUES ('b', '2')").run() - throw new Error('boom') +test('concurrent transactions do not share a connection or rollback each other', async () => { + const db = await database() + await Promise.all([ + withTransaction(db, async () => { + await db.prepare("INSERT INTO settings(key,value) VALUES('keep','1')").run() + }), + assert.rejects( + withTransaction(db, async () => { + await db.prepare("INSERT INTO settings(key,value) VALUES('drop','2')").run() + throw new Error('fail') }), - /boom/, + /fail/, + ), + ]) + assert.ok(await db.prepare("SELECT * FROM settings WHERE key='keep'").get()) + assert.equal(await db.prepare("SELECT * FROM settings WHERE key='drop'").get(), undefined) +}) + +test('concurrent initialization serializes migrations and checksums are enforced', async () => { + const opts = options() + const [a, b] = await Promise.all([createDatabase(opts), createDatabase(opts)]) + dbs.push(a, b) + assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 1) + await a.prepare("UPDATE schema_migrations SET checksum='invalid'").run() + await assert.rejects(applyMigrations(b), /checksum mismatch/) +}) + +test('failed migrations rollback DDL and migration records', async () => { + const dir = fs.mkdtempSync('/tmp/vm2api-pg-migrations-') + fs.writeFileSync(dir + '/002_bad.sql', 'CREATE TABLE partial_write(id text); INVALID SQL;') + const db = await database() + await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /unknown PostgreSQL migration/) + fs.copyFileSync( + new URL('../../src/lib/db/postgres-migrations/001_baseline.sql', import.meta.url), + dir + '/001_baseline.sql', + ) + await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /002_bad.sql failed/) + assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(db.schema + '.partial_write')).name, null) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 1) + fs.rmSync(dir, { recursive: true, force: true }) +}) + +test('big integers preserve precision and errors exclude SQL parameter values', async () => { + const db = await database() + const row = await db.prepare('SELECT $1::bigint AS large, $2::bigint AS small').get('9007199254740993', 42) + assert.equal(row.large, '9007199254740993') + assert.equal(row.small, 42) + await db.prepare('INSERT INTO settings(key,value) VALUES($1,$2)').run('sensitive-key', 'sensitive-value') + await assert.rejects( + db.prepare('INSERT INTO settings(key,value) VALUES($1,$2)').run('sensitive-key', 'sensitive-value'), + (error) => error.code === '23505' && !error.message.includes('sensitive'), ) - assert.equal(db.prepare("SELECT * FROM settings WHERE key='b'").get(), undefined) }) -test('withTransaction only runs inline for a real outer transaction', () => { - const db = openDatabase({ dataDir: tmp }) - db.exec('BEGIN') - withTransaction(db, () => { - db.prepare("INSERT INTO settings (key, value) VALUES ('nested', '1')").run() +test('singleton opens once and closes explicitly', async () => { + await closeDatabase() + assert.throws(getDb, /not opened/) + const opts = options() + const [a, b] = await Promise.all([openDatabase(opts), openDatabase(opts)]) + assert.equal(a, b) + assert.equal(getDb(), a) + assert.equal(isDbOpen(), true) + await a.exec(`DROP SCHEMA "${a.schema}" CASCADE`) + await closeDatabase() + assert.equal(isDbOpen(), false) +}) + +test('maintenance drains transactions and rejects unrelated writes', async () => { + const db = await database() + let release, started + const ready = new Promise((r) => { + started = r }) - assert.equal(db.isTransaction, true) - db.exec('ROLLBACK') - assert.equal(db.prepare("SELECT * FROM settings WHERE key='nested'").get(), undefined) - - const closed = new DatabaseSync(':memory:') - closed.close() - let called = false - assert.throws( - () => - withTransaction(closed, () => { - called = true - }), - /not open|closed/i, - ) - assert.equal(called, false) + const transaction = withTransaction(db, async () => { + started() + await new Promise((r) => { + release = r + }) + await db.prepare("INSERT INTO settings(key,value) VALUES('before','1')").run() + }) + await ready + const maintaining = db.withMaintenance(async () => { + await db.prepare("INSERT INTO settings(key,value) VALUES('maintenance','1')").run() + }) + await assert.rejects(db.prepare("INSERT INTO settings(key,value) VALUES('blocked','1')").run(), { + code: 'database_maintenance', + }) + release() + await transaction + await maintaining + assert.equal((await db.prepare('SELECT count(*) AS n FROM settings').get()).n, 2) }) diff --git a/test/unit/db-migration-sub2api.test.mjs b/test/unit/db-migration-sub2api.test.mjs index 9d3338698..a31e7d007 100644 --- a/test/unit/db-migration-sub2api.test.mjs +++ b/test/unit/db-migration-sub2api.test.mjs @@ -4,7 +4,7 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { fileURLToPath } from 'node:url' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { createDatabase } from '../../src/lib/db/sqlite-legacy.mjs' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const MIGRATIONS = path.join(__dirname, '../../src/lib/db/migrations') diff --git a/test/unit/distill-detect.test.mjs b/test/unit/distill-detect.test.mjs index 8aaca2e65..ee873a0c7 100644 --- a/test/unit/distill-detect.test.mjs +++ b/test/unit/distill-detect.test.mjs @@ -608,10 +608,10 @@ test('normalize keeps default fingerprints when omitted', () => { test('handleProtocol intercepts before credential hop', () => { const src = fs.readFileSync(path.join(root, 'src/lib/protocol/handle-protocol.mjs'), 'utf8') const early = src.indexOf('await applyProtocolIntercept(') - const codex = src.indexOf('return handleCodexProtocol(') + const codex = src.indexOf('handleCodexProtocol({') assert.ok(early > 0 && early < codex) const before = src.indexOf("applyIntercept(cfg.intercept.rules, 'before_upstream'") - const late = src.indexOf('await applyProtocolIntercept(', early + 1) + const late = src.indexOf('await applyProtocolIntercept(', before) const api = src.indexOf("inferenceBackend === 'api'") assert.ok(before > 0) assert.ok(late > before) diff --git a/test/unit/error-class.test.mjs b/test/unit/error-class.test.mjs index 7a8260e93..2e822ac92 100644 --- a/test/unit/error-class.test.mjs +++ b/test/unit/error-class.test.mjs @@ -1,5 +1,6 @@ import test from 'node:test' import assert from 'node:assert/strict' +import { getDb } from '../../src/lib/db/database.mjs' import { classifyRequestError, collectErrors, @@ -119,15 +120,14 @@ test('excludeErrorClassSql hides ingress auth without dropping oauth credential const sql = excludeErrorClassSql('auth') assert.match(sql, /invalid_api_key/) assert.match(sql, /NOT \(/) - assert.match(sql, /IFNULL\(error_code/) + assert.match(sql, /COALESCE\(error_code/) assert.match(sql, /upstream_auth_error/) }) test('default auth mute keeps policy 403s that already name a non-auth class', async () => { - const { DatabaseSync } = await import('node:sqlite') - const db = new DatabaseSync(':memory:') - db.exec('CREATE TABLE logs (status INTEGER, error_code TEXT, vm_id TEXT)') - const insert = db.prepare('INSERT INTO logs VALUES (?, ?, ?)') + const db = getDb() + await db.exec('CREATE TABLE logs (status INTEGER, error_code TEXT, vm_id TEXT)') + const insert = db.prepare('INSERT INTO logs VALUES ($1, $2, $3)') for (const row of [ [401, 'invalid_api_key', null], [403, null, null], @@ -136,11 +136,10 @@ test('default auth mute keeps policy 403s that already name a non-auth class', a [403, 'distill_blocked', null], [401, 'upstream_auth_error', null], ]) { - insert.run(...row) + await insert.run(...row) } - const kept = db - .prepare(`SELECT error_code FROM logs WHERE ${excludeErrorClassSql('auth')}`) - .all() - .map((r) => r.error_code) + const kept = (await db.prepare(`SELECT error_code FROM logs WHERE ${excludeErrorClassSql('auth')}`).all()).map( + (r) => r.error_code, + ) assert.deepEqual(kept.sort(), ['claude_code_only', 'distill_blocked', 'key_group_mismatch', 'upstream_auth_error']) }) diff --git a/test/unit/failover-runner.test.mjs b/test/unit/failover-runner.test.mjs index b5c342e55..e465acc5b 100644 --- a/test/unit/failover-runner.test.mjs +++ b/test/unit/failover-runner.test.mjs @@ -909,9 +909,9 @@ test('aborted signal stops before another account is tried', async () => { for (const cancellation of ['result', 'throw']) { test(`client cancellation via ${cancellation} preserves outbound session beyond fifty turns`, async (t) => { const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-cancel-session-')) - const sticky = new StickyRouter({ dataDir, config: { sticky: { enabled: true } } }) - t.after(() => { - sticky.db.close() + const sticky = await new StickyRouter({ dataDir, config: { sticky: { enabled: true } } }).init() + t.after(async () => { + await sticky.db.close() fs.rmSync(dataDir, { recursive: true, force: true }) }) const scheduler = new Scheduler([{ ...candidate(1), slotIndex: 3 }]) diff --git a/test/unit/haiku-55-policy.test.mjs b/test/unit/haiku-55-policy.test.mjs index c613ead76..0037f0130 100644 --- a/test/unit/haiku-55-policy.test.mjs +++ b/test/unit/haiku-55-policy.test.mjs @@ -38,11 +38,11 @@ test('haiku 5.5 is in the panel seed and keeps forced tool choice', () => { assert.deepEqual(blocked.thinking, { type: 'adaptive' }) }) -test('loading old Haiku 5.5 seed migrates only the old 1M defaults to 100K', () => { - openDatabase({ dbPath: ':memory:' }) +test('loading old Haiku 5.5 seed migrates only the old 1M defaults to 100K', async () => { + await openDatabase({ dbPath: ':memory:' }) try { const repo = new SettingsRepo() - repo.set('model_policy', { + await repo.set('model_policy', { source: 'panel', models: { 'claude-haiku-5-5': { @@ -56,14 +56,14 @@ test('loading old Haiku 5.5 seed migrates only the old 1M defaults to 100K', () }, }, }) - loadModelPolicy({ force: true }) - const stored = repo.get('model_policy') + await loadModelPolicy({ force: true }) + const stored = await repo.get('model_policy') const row = stored.models['claude-haiku-5-5'] assert.equal(row.capabilities.context_window, 100000) assert.equal(row.params.max_tokens_default, 100000) assert.equal(row.params.max_tokens_cap, 100000) assert.equal(row.params.default_effort, 'medium') } finally { - closeDatabase() + await closeDatabase() } }) diff --git a/test/unit/handle-codex-search.test.mjs b/test/unit/handle-codex-search.test.mjs index 5b5cecac4..61d867529 100644 --- a/test/unit/handle-codex-search.test.mjs +++ b/test/unit/handle-codex-search.test.mjs @@ -1,3 +1,4 @@ +import { flushPendingWrites } from '../../src/lib/db/pending-writes.mjs' import { test } from 'node:test' import assert from 'node:assert/strict' import fs from 'node:fs' @@ -234,6 +235,7 @@ test('the alpha/search route authenticates, logs and reaches the Codex search ho } await handleSearch(req, res, '/v1/alpha/search') res.finish() + await flushPendingWrites() assert.equal(res.statusCode, 502) assert.equal(res.body.error.code, 'proxy_required') assert.equal(rows.length, 1) diff --git a/test/unit/handle-codex.test.mjs b/test/unit/handle-codex.test.mjs index dc6fe9040..42f0d7ceb 100644 --- a/test/unit/handle-codex.test.mjs +++ b/test/unit/handle-codex.test.mjs @@ -1077,7 +1077,10 @@ async function hopCodex({ root, stickyRouter, routing = {}, headers = {}, body, test('codex rebuild outbound session is not the inbound session', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-codex-sess-')) - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() writeGptVm(root, 'vm-gpt-a') const inbound = '11111111-1111-4111-8111-111111111111' const { envelopes } = await hopCodex({ @@ -1101,13 +1104,16 @@ test('codex rebuild outbound session is not the inbound session', async () => { assert.equal(envelopes[0].body.prompt_cache_key, session.session_id) assert.equal(envelopes[0].body.conversation_id, undefined) assert.equal(envelopes[0].body.session_id, undefined) - sticky.db?.close?.() + await sticky.db?.close?.() fs.rmSync(root, { recursive: true, force: true }) }) test('codex kernel envelope preserves official routing headers without client auth', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-codex-routing-headers-')) - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() writeGptVm(root, 'vm-gpt-a') const { envelopes } = await hopCodex({ root, @@ -1143,13 +1149,16 @@ test('codex kernel envelope preserves official routing headers without client au assert.equal(envelopes[0].headers['session-id'], session.session_id) assert.equal(envelopes[0].headers.authorization, undefined) assert.equal(envelopes[0].body.service_tier, 'priority') - sticky.db?.close?.() + await sticky.db?.close?.() fs.rmSync(root, { recursive: true, force: true }) }) test('codex rebuild outbound session is stable on the same slot', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-codex-stable-')) - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() writeGptVm(root, 'vm-gpt-a') const inbound = 'sess-stable' const first = await hopCodex({ @@ -1165,7 +1174,7 @@ test('codex rebuild outbound session is stable on the same slot', async () => { body: { model: 'gpt-5.4', input: 'hi again', stream: false }, }) assert.equal(first.envelopes[0].session.session_id, second.envelopes[0].session.session_id) - sticky.db?.close?.() + await sticky.db?.close?.() fs.rmSync(root, { recursive: true, force: true }) }) @@ -1173,7 +1182,10 @@ test('codex rebuild remints session after failover to another slot', async () => const rt = await import('../../src/lib/pool/openai-account-runtime.mjs') rt.resetOpenAIAccountRuntime() const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-codex-fail-')) - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() writeGptVm(root, 'vm-gpt-a') writeGptVm(root, 'vm-gpt-b') const inbound = 'sess-fail' @@ -1227,13 +1239,16 @@ test('codex rebuild remints session after failover to another slot', async () => assert.notEqual(envelopes[1].session.session_id, inbound) assert.notEqual(envelopes[0].session.session_id, envelopes[1].session.session_id) rt.resetOpenAIAccountRuntime() - sticky.db?.close?.() + await sticky.db?.close?.() fs.rmSync(root, { recursive: true, force: true }) }) test('codex passthrough keeps the inbound session and cache key', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-codex-pass-')) - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() writeGptVm(root, 'vm-gpt-a') const inbound = 'sess-pass' const { envelopes } = await hopCodex({ @@ -1252,13 +1267,16 @@ test('codex passthrough keeps the inbound session and cache key', async () => { assert.equal(envelopes[0].session.session_id, inbound) assert.equal(envelopes[0].body.prompt_cache_key, 'cache-pass') assert.equal(envelopes[0].body.conversation_id, inbound) - sticky.db?.close?.() + await sticky.db?.close?.() fs.rmSync(root, { recursive: true, force: true }) }) test('codex passthrough without an inbound session derives a stable fallback session', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-codex-pass-fallback-')) - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() writeGptVm(root, 'vm-gpt-a') const opener = (question) => ({ model: 'gpt-5.4', @@ -1282,7 +1300,7 @@ test('codex passthrough without an inbound session derives a stable fallback ses assert.match(sessionId, /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/) assert.equal(again.envelopes[0].session.session_id, sessionId) assert.notEqual(other.envelopes[0].session.session_id, sessionId) - sticky.db?.close?.() + await sticky.db?.close?.() fs.rmSync(root, { recursive: true, force: true }) }) diff --git a/test/unit/jev-account-scope.test.mjs b/test/unit/jev-account-scope.test.mjs index c1a7fce38..15972088f 100644 --- a/test/unit/jev-account-scope.test.mjs +++ b/test/unit/jev-account-scope.test.mjs @@ -186,13 +186,13 @@ test('without a scope the GPT switch still decides and Claude still scans', asyn assert.equal(claude.action, 'block') }) -test('the panel PUT round-trips platform-plan pairs and rejects junk', () => { +test('the panel PUT round-trips platform-plan pairs and rejects junk', async () => { const stored = new Map() const settings = { get: (key, fallback) => (stored.has(key) ? stored.get(key) : fallback), set: (key, value) => stored.set(key, value), } - const saved = applyJevPatch(settings, { + const saved = await applyJevPatch(settings, { account_platform_plans: ['Claude:Max', 'gpt:plus'], account_ids: ['vm-02', 'ops@example.com'], account_types: [], @@ -202,7 +202,7 @@ test('the panel PUT round-trips platform-plan pairs and rejects junk', () => { assert.deepEqual(saved.config.account_platform_plans, ['claude:max', 'gpt:plus']) assert.equal(jevScopeConfigured(stored.get('jev_intercept')), true) - const cleared = applyJevPatch(settings, { account_platform_plans: [], account_ids: [] }) + const cleared = await applyJevPatch(settings, { account_platform_plans: [], account_ids: [] }) assert.equal(cleared.ok, true) assert.equal(jevScopeConfigured(stored.get('jev_intercept')), false) assert.deepEqual(publicJevConfig(null).account_platform_plans, []) diff --git a/test/unit/jev-intercept.test.mjs b/test/unit/jev-intercept.test.mjs index ddc522319..683fb9ded 100644 --- a/test/unit/jev-intercept.test.mjs +++ b/test/unit/jev-intercept.test.mjs @@ -128,9 +128,9 @@ test('hard regex blocks jailbreak and crack, not ordinary debugging', () => { assert.equal(matchHardPolicy('explain the difference between phishing and spam for a security class'), null) }) -test('saved rules replace the builtins and a bad regex is rejected', () => { +test('saved rules replace the builtins and a bad regex is rejected', async () => { let saved = null - const patched = applyJevPatch( + const patched = await applyJevPatch( { get: () => null, set: (_key, value) => { @@ -143,11 +143,11 @@ test('saved rules replace the builtins and a bad regex is rejected', () => { assert.equal(patched.config.rules_customized, true) assert.equal(matchHardPolicy('feat vmpanel-keygen', [], saved.rules).keyword, 'vmpanel') assert.equal(matchHardPolicy('write a keygen', [], saved.rules), null) - const bad = applyJevPatch({ get: () => null, set() {} }, { rules: [{ category: 'custom', source: '(' }] }) + const bad = await applyJevPatch({ get: () => null, set() {} }, { rules: [{ category: 'custom', source: '(' }] }) assert.equal(bad.ok, false) }) -test('an empty rule list falls back to the builtins instead of clearing the layer', () => { +test('an empty rule list falls back to the builtins instead of clearing the layer', async () => { let saved = null const settings = { get: () => null, @@ -156,7 +156,7 @@ test('an empty rule list falls back to the builtins instead of clearing the laye }, } assert.equal(normalizeJevConfig({ rules: [] }).rules, null) - const patched = applyJevPatch(settings, { rules: [] }) + const patched = await applyJevPatch(settings, { rules: [] }) assert.equal(patched.ok, true) assert.equal(patched.config.rules_customized, false) assert.equal(patched.config.rules.length, builtinHardRules().length) @@ -165,7 +165,7 @@ test('an empty rule list falls back to the builtins instead of clearing the laye 'jailbreak', ) // The panel can still turn the layer off with its own switch. - const off = applyJevPatch(settings, { hard_regex_enabled: false }) + const off = await applyJevPatch(settings, { hard_regex_enabled: false }) assert.equal(off.config.hard_regex_enabled, false) assert.equal(saved.rules, null) }) @@ -339,23 +339,30 @@ test('content-policy error codes are refusals; generic api errors are not', () = assert.equal(isUpstreamRefusal({ body: { error: { code: 'distill_blocked', message: '不允许蒸馏' } } }), false) }) -test('panel patch keeps a stored key unless cleared', () => { +test('panel patch keeps a stored key unless cleared', async () => { const saved = new Map() const settings = { get: (key, fallback) => (saved.has(key) ? saved.get(key) : fallback), set: (key, value) => saved.set(key, value), } assert.equal( - applyJevPatch(settings, { enabled: true, base_url: 'http://127.0.0.1:9/v1', model: 'jev', api_key: 'secret' }).ok, + ( + await applyJevPatch(settings, { + enabled: true, + base_url: 'http://127.0.0.1:9/v1', + model: 'jev', + api_key: 'secret', + }) + ).ok, true, ) - const kept = applyJevPatch(settings, { timeout_ms: 1500 }) + const kept = await applyJevPatch(settings, { timeout_ms: 1500 }) assert.equal(kept.ok, true) assert.equal(kept.config.api_key_set, true) const stored = saved.get('jev_intercept') assert.equal(publicJevConfig(stored).api_key_set, true) assert.equal(stored.api_key, 'secret') - assert.equal(applyJevPatch(settings, { api_key: '' }).config.api_key_set, false) + assert.equal((await applyJevPatch(settings, { api_key: '' })).config.api_key_set, false) assert.equal(validateJevPatch({ timeout_ms: 10 }).length, 1) }) @@ -763,7 +770,10 @@ test('401 retires that key until the key list is saved again', async () => { settings.value = value }, } - assert.equal(applyJevPatch(settings, { api_keys: ['k-auth-retire'], base_url: cfg.base_url, enabled: true }).ok, true) + assert.equal( + (await applyJevPatch(settings, { api_keys: ['k-auth-retire'], base_url: cfg.base_url, enabled: true })).ok, + true, + ) let retried = 0 await classifyJev('retire-auth-3', { ...cfg, api_keys: ['k-auth-retire'] }, async () => { retried += 1 diff --git a/test/unit/key-scope.test.mjs b/test/unit/key-scope.test.mjs index 45c6be6af..2949aa612 100644 --- a/test/unit/key-scope.test.mjs +++ b/test/unit/key-scope.test.mjs @@ -13,12 +13,12 @@ import { const claude = { id: 'vm-01', platform: 'anthropic' } const codex = { id: 'vm-02', platform: 'openai', family: 'codex' } -test('all is global and ignores a stored VM list', () => { +test('all is global and ignores a stored VM list', async () => { const scope = keyScopeFromRecord({ group_type: 'all', allowed_vms: '["vm-01"]' }) assert.deepEqual(scope, { group_type: 'all', allowed_vms: [], vm_pool_id: null }) assert.equal(keyAllowsVm(scope, claude), true) assert.equal(keyAllowsVm(scope, codex), true) - assert.deepEqual(keyScopeFromRequest({ apiKeyKind: 'master', apiKeyRecord: { group_type: 'openai' } }), { + assert.deepEqual(await keyScopeFromRequest({ apiKeyKind: 'master', apiKeyRecord: { group_type: 'openai' } }), { group_type: 'all', allowed_vms: [], vm_pool_id: null, @@ -52,17 +52,17 @@ test('a status-only patch does not widen or clear an existing scope', () => { }) }) -test('a named pool is the only schedule set and a dead pool does not widen', () => { +test('a named pool is the only schedule set and a dead pool does not widen', async () => { const bound = normalizeKeyScope({ vm_pool_id: 'pool_abcd1234', group_type: 'all' }) assert.deepEqual(bound, { group_type: 'all', allowed_vms: [], vm_pool_id: 'pool_abcd1234' }) - const live = applyVmPool(bound, () => ({ id: 'pool_abcd1234', enabled: true, vm_ids: ['vm-02'] })) + const live = await applyVmPool(bound, () => ({ id: 'pool_abcd1234', enabled: true, vm_ids: ['vm-02'] })) assert.equal(keyAllowsVm(live, codex), true) assert.equal(keyAllowsVm(live, claude), false) - const moved = applyVmPool(bound, () => ({ id: 'pool_abcd1234', enabled: true, vm_ids: ['vm-01'] })) + const moved = await applyVmPool(bound, () => ({ id: 'pool_abcd1234', enabled: true, vm_ids: ['vm-01'] })) assert.equal(keyAllowsVm(moved, codex), false) assert.equal(keyAllowsVm(moved, claude), true) for (const vm_pool_error of ['missing', 'disabled', 'empty']) { - const dead = applyVmPool(bound, () => + const dead = await applyVmPool(bound, () => vm_pool_error === 'missing' ? null : { enabled: vm_pool_error !== 'disabled', vm_ids: [] }, ) assert.equal(dead.vm_pool_error, vm_pool_error) diff --git a/test/unit/legacy-import.test.mjs b/test/unit/legacy-import.test.mjs index 94196db76..d27f5f77f 100644 --- a/test/unit/legacy-import.test.mjs +++ b/test/unit/legacy-import.test.mjs @@ -149,20 +149,20 @@ function seedLegacyFiles() { fs.writeFileSync(path.join(vmsDir, 'active.json'), JSON.stringify({ active_vm: 'vm-l1' })) } -beforeEach(() => { +beforeEach(async () => { root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-import-')) dataDir = path.join(root, 'data') - openDatabase({ dataDir }) + await openDatabase({ dataDir }) }) -afterEach(() => { - closeDatabase() +afterEach(async () => { + await closeDatabase() fs.rmSync(root, { recursive: true, force: true }) }) -test('imports every legacy store and is idempotent', () => { +test('imports every legacy store and is idempotent', async () => { seedLegacyFiles() - const res = runLegacyImport({ dataDir, projectRoot: root }) + const res = await runLegacyImport({ dataDir, projectRoot: root }) assert.equal(res.imported, true) assert.deepEqual( { ...res.counts }, @@ -179,33 +179,33 @@ test('imports every legacy store and is idempotent', () => { }, ) const db = getDb() - assert.equal(new ApiKeysRepo(db).getById('key_legacy1').tokens_in, 300) - assert.equal(new AccountsRepo(db).get('acct-legacy').requests, 7) - assert.equal(new AccountsRepo(db).allocationCount('acct-legacy'), 2) - assert.equal(new StickyRepo(db).get('conv-legacy').hits, 4) - assert.equal(new ProxiesRepo(db).loadAll()[0].bound_vm_id, 'vm-l1') - assert.equal(new ProxiesRepo(db).getConfig().probe_interval_min, 30) - assert.equal(new UsageLogsRepo(db).count(), 2) - assert.ok(new UsageLogsRepo(db).getDebug('rid-l1')) - const vmRow = new VmsRepo(db).get('vm-l1') + assert.equal((await new ApiKeysRepo(db).getById('key_legacy1')).tokens_in, 300) + assert.equal((await new AccountsRepo(db).get('acct-legacy')).requests, 7) + assert.equal(await new AccountsRepo(db).allocationCount('acct-legacy'), 2) + assert.equal((await new StickyRepo(db).get('conv-legacy')).hits, 4) + assert.equal((await new ProxiesRepo(db).loadAll())[0].bound_vm_id, 'vm-l1') + assert.equal((await new ProxiesRepo(db).getConfig()).probe_interval_min, 30) + assert.equal(await new UsageLogsRepo(db).count(), 2) + assert.ok(await new UsageLogsRepo(db).getDebug('rid-l1')) + const vmRow = await new VmsRepo(db).get('vm-l1') assert.equal(vmRow.access_token, null) assert.equal(vmRow.vm.claude.has_access, true) assert.equal(vmRow.vm.claude.access_token, undefined) - assert.equal(new VmsRepo(db).getActiveVmId(), 'vm-l1') + assert.equal(await new VmsRepo(db).getActiveVmId(), 'vm-l1') // second run skips - const res2 = runLegacyImport({ dataDir, projectRoot: root }) + const res2 = await runLegacyImport({ dataDir, projectRoot: root }) assert.equal(res2.imported, false) assert.ok(res2.already?.at) - assert.equal(new ApiKeysRepo(db).count(), 1) + assert.equal(await new ApiKeysRepo(db).count(), 1) // original files untouched assert.ok(fs.existsSync(path.join(dataDir, 'api-keys.json'))) assert.ok(fs.existsSync(path.join(dataDir, 'proxy-pool.json'))) }) -test('missing files import cleanly with zero counts', () => { - const res = runLegacyImport({ dataDir, projectRoot: root }) +test('missing files import cleanly with zero counts', async () => { + const res = await runLegacyImport({ dataDir, projectRoot: root }) assert.equal(res.imported, true) assert.equal( Object.values(res.counts).reduce((a, b) => a + b, 0), @@ -213,14 +213,14 @@ test('missing files import cleanly with zero counts', () => { ) }) -test('corrupt file is skipped, counted as error, others still import', () => { +test('corrupt file is skipped, counted as error, others still import', async () => { seedLegacyFiles() fs.writeFileSync(path.join(dataDir, 'api-keys.json'), '{ not json') - const res = runLegacyImport({ dataDir, projectRoot: root }) + const res = await runLegacyImport({ dataDir, projectRoot: root }) assert.equal(res.imported, true) assert.equal(res.counts.api_keys, 0) assert.ok(res.counts.errors >= 1) assert.equal(res.counts.accounts, 1) assert.equal(res.counts.vms, 1) - assert.ok(new SettingsRepo(getDb()).get('legacy_import_done')) + assert.ok(await new SettingsRepo(getDb()).get('legacy_import_done')) }) diff --git a/test/unit/local-egress-per-node.test.mjs b/test/unit/local-egress-per-node.test.mjs index 5d3d8b9fe..c7a3e44ed 100644 --- a/test/unit/local-egress-per-node.test.mjs +++ b/test/unit/local-egress-per-node.test.mjs @@ -17,9 +17,9 @@ import { const NODES = { 'vm-ctrl': null, 'vm-a1': 'node-a', 'vm-a2': 'node-a', 'vm-b1': 'node-b' } -function makePool(t, opts = {}) { +async function makePool(t, opts = {}) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-local-node-')) - const pool = new ProxyPool({ dataDir: dir, vmNodeOf: (vmId) => NODES[vmId] ?? null, ...opts }) + const pool = await new ProxyPool({ dataDir: dir, vmNodeOf: (vmId) => NODES[vmId] ?? null, ...opts }).init() pool.stopScheduler() t.after(() => { pool.stopScheduler() @@ -43,8 +43,8 @@ test('local egress ids name their VPS and share one bridge per host', () => { assert.deepEqual(portsForProxy('px-local-node-a'), portsForProxy('px-local')) }) -test('a generic px-local bind lands on the slot’s own VPS row, with its own capacity', (t) => { - const pool = makePool(t) +test('a generic px-local bind lands on the slot’s own VPS row, with its own capacity', async (t) => { + const pool = await makePool(t) pool.ensureLocal() pool.updateConfig({ bind_limit: 1 }) assert.equal(pool.bind('px-local', 'vm-ctrl').proxy.id, 'px-local') @@ -57,8 +57,8 @@ test('a generic px-local bind lands on the slot’s own VPS row, with its own ca assert.equal(pool.getProxyForVm('vm-a1').id, 'px-local-node-a') }) -test('another machine’s local row is refused', (t) => { - const pool = makePool(t) +test('another machine’s local row is refused', async (t) => { + const pool = await makePool(t) pool.ensureLocal() pool.ensureLocalForNode('node-a') const wrong = pool.bind('px-local-node-a', 'vm-b1') @@ -68,16 +68,16 @@ test('another machine’s local row is refused', (t) => { assert.equal(pool.bind('px-local-node-a', 'vm-ctrl').error, 'local_proxy_node_mismatch') }) -test('auto-allocation only offers the slot’s own local row', (t) => { - const pool = makePool(t) +test('auto-allocation only offers the slot’s own local row', async (t) => { + const pool = await makePool(t) pool.ensureLocal() const picked = pool.allocateForVm('vm-b1') assert.equal(picked.id, 'px-local-node-b') assert.equal(pool.allocateForVm('vm-ctrl').id, 'px-local') }) -test('node rows follow the cluster and keep rows that still hold slots', (t) => { - const pool = makePool(t) +test('node rows follow the cluster and keep rows that still hold slots', async (t) => { + const pool = await makePool(t) // No local egress in use: nothing is created for nodes. assert.deepEqual(pool.syncLocalNodes(['node-a']), { created: [], removed: [] }) pool.ensureLocal() @@ -90,8 +90,8 @@ test('node rows follow the cluster and keep rows that still hold slots', (t) => assert.ok(pool.state.proxies.slice(0, 2).every((p) => isLocalEgressProxy(p))) }) -test('slots left on the shared row move to their node row, past a full limit', (t) => { - const pool = makePool(t) +test('slots left on the shared row move to their node row, past a full limit', async (t) => { + const pool = await makePool(t) pool.ensureLocal() pool.updateConfig({ bind_limit: 1 }) // Pre-split data: every local slot sits on px-local. @@ -108,9 +108,10 @@ test('slots left on the shared row move to their node row, past a full limit', ( assert.deepEqual(pool.adoptNodeLocalBindings(), []) }) -test('migration leaves ids of deleted slots alone', (t) => { - const pool = makePool(t, { vmNodeOf: (vmId) => (vmId === 'vm-gone' ? undefined : (NODES[vmId] ?? null)) }) +test('migration leaves ids of deleted slots alone', async (t) => { + const pool = await makePool(t, { vmNodeOf: (vmId) => (vmId === 'vm-gone' ? undefined : (NODES[vmId] ?? null)) }) pool.ensureLocal() + await pool.flush() pool.state.proxies.find((p) => p.id === 'px-local').bound_vm_ids = ['vm-gone', 'vm-a1'] assert.deepEqual( pool.adoptNodeLocalBindings().map((m) => m.vm_id), @@ -121,7 +122,7 @@ test('migration leaves ids of deleted slots alone', (t) => { test('a node row probes its node link; a down link marks it failed without parking slots', async (t) => { let healthy = true const disabled = [] - const pool = makePool(t, { + const pool = await makePool(t, { localExitHealth: (nodeId) => healthy ? { ok: true, latency_ms: 42 } : { ok: false, error: `node_link_down:${nodeId}` }, onDisableVm: (vmId) => disabled.push(vmId), @@ -144,7 +145,7 @@ test('a node row probes its node link; a down link marks it failed without parki test('a node link still dialling leaves the row alone until it settles', async (t) => { let health = { pending: true, error: 'node_link_connecting' } - const pool = makePool(t, { localExitHealth: () => health }) + const pool = await makePool(t, { localExitHealth: () => health }) pool.ensureLocal() pool.bind('px-local', 'vm-a1') const row = pool.state.proxies.find((p) => p.id === 'px-local-node-a') @@ -164,7 +165,7 @@ test('a node link still dialling leaves the row alone until it settles', async ( test('a node row looks up its geo through that node’s route', async (t) => { const seen = [] const geo = { ok: true, geo: { ip: '203.0.113.7', country: 'Japan', timezone: 'Asia/Tokyo' } } - const pool = makePool(t, { + const pool = await makePool(t, { localExitUrl: (nodeId) => `socks5h://${nodeId}@127.0.0.1:1080`, geoLookup: async (url) => { seen.push(url) @@ -199,7 +200,7 @@ test('tearing down a node row never touches the control plane bridge', () => { test('an operator-disabled node row stays disabled through link changes', async (t) => { let health = { ok: false, error: 'node_link_backoff' } - const pool = makePool(t, { localExitHealth: () => health }) + const pool = await makePool(t, { localExitHealth: () => health }) pool.ensureLocal() pool.bind('px-local', 'vm-a1') const row = pool.state.proxies.find((p) => p.id === 'px-local-node-a') @@ -213,9 +214,10 @@ test('an operator-disabled node row stays disabled through link changes', async assert.equal(row.status, 'dead') }) -test('migration persists once for the whole batch', (t) => { - const pool = makePool(t) +test('migration persists once for the whole batch', async (t) => { + const pool = await makePool(t) pool.ensureLocal() + await pool.flush() pool.state.proxies.find((p) => p.id === 'px-local').bound_vm_ids = ['vm-a1', 'vm-a2', 'vm-b1'] let writes = 0 const replaceAll = pool.repo.replaceAll.bind(pool.repo) @@ -224,11 +226,12 @@ test('migration persists once for the whole batch', (t) => { return replaceAll(rows) } assert.equal(pool.adoptNodeLocalBindings().length, 3) + await pool.flush() assert.equal(writes, 1) }) -test('resolveLocalTarget lands a generic pick on the target VPS', (t) => { - const pool = makePool(t) +test('resolveLocalTarget lands a generic pick on the target VPS', async (t) => { + const pool = await makePool(t) pool.ensureLocal() assert.deepEqual(pool.resolveLocalTarget('px-local', 'node-a'), { ok: true, id: 'px-local-node-a' }) assert.ok(pool.state.proxies.some((p) => p.id === 'px-local-node-a')) diff --git a/test/unit/model-policy-fable51.test.mjs b/test/unit/model-policy-fable51.test.mjs index af6a7c342..e7b06f9c0 100644 --- a/test/unit/model-policy-fable51.test.mjs +++ b/test/unit/model-policy-fable51.test.mjs @@ -82,15 +82,15 @@ test('explicit canonical overrides win over legacy fields in either insertion or } }) -test('loading a persisted disabled legacy model keeps both request spellings disabled', () => { - openDatabase({ dbPath: ':memory:' }) +test('loading a persisted disabled legacy model keeps both request spellings disabled', async () => { + await openDatabase({ dbPath: ':memory:' }) try { - new SettingsRepo().set('model_policy', { + await new SettingsRepo().set('model_policy', { catalog_mode: 'worker_only', models: { [legacy]: { enabled: false } }, aliases: { private: legacy }, }) - loadModelPolicy() + await loadModelPolicy() clearModelsCache() const ids = getPolicyCatalogIds() assert.ok(ids.includes(canonical)) @@ -101,11 +101,11 @@ test('loading a persisted disabled legacy model keeps both request spellings dis assert.equal(result.error.code, 'model_disabled', model) } assert.deepEqual(filterPublicModelIds([legacy, canonical]), [canonical]) - syncWorkerModelsIntoPolicy([legacy]) + await syncWorkerModelsIntoPolicy([legacy]) assert.ok(!getPolicyCatalogIds().includes(legacy)) } finally { - closeDatabase() - loadModelPolicy() + await closeDatabase() + await loadModelPolicy() clearModelsCache() } }) diff --git a/test/unit/onboarding.test.mjs b/test/unit/onboarding.test.mjs index 00a90edd5..f492edaf8 100644 --- a/test/unit/onboarding.test.mjs +++ b/test/unit/onboarding.test.mjs @@ -102,14 +102,14 @@ test('guide pops until skip or finish, even when setup steps are already done', assert.equal(legacy.guide_used, false) }) -test('PUT complete / dismiss / reset round-trip through settings; invalid action rejected', () => { +test('PUT complete / dismiss / reset round-trip through settings; invalid action rejected', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-onboarding-')) - const db = createDatabase({ dataDir: dir }) + const db = await createDatabase({ dataDir: dir }) const settings = new SettingsRepo(db) - assert.equal(recordOf(loadOnboardingState(settings), 'admin').status, 'pending') + assert.equal(recordOf(await loadOnboardingState(settings), 'admin').status, 'pending') - saveOnboardingState(settings, applyOnboardingAction(loadOnboardingState(settings), 'admin', 'complete')) - const completed = recordOf(loadOnboardingState(settings), 'admin') + await saveOnboardingState(settings, applyOnboardingAction(await loadOnboardingState(settings), 'admin', 'complete')) + const completed = recordOf(await loadOnboardingState(settings), 'admin') assert.equal(completed.status, 'completed') assert.equal(completed.guide_used, true) assert.ok(completed.updated_at) @@ -121,17 +121,17 @@ test('PUT complete / dismiss / reset round-trip through settings; invalid action }) assert.equal(afterComplete.needs_welcome, false) - saveOnboardingState(settings, applyOnboardingAction(loadOnboardingState(settings), 'admin', 'dismiss')) - assert.equal(recordOf(loadOnboardingState(settings), 'admin').status, 'dismissed') - assert.equal(recordOf(loadOnboardingState(settings), 'admin').guide_used, true) + await saveOnboardingState(settings, applyOnboardingAction(await loadOnboardingState(settings), 'admin', 'dismiss')) + assert.equal(recordOf(await loadOnboardingState(settings), 'admin').status, 'dismissed') + assert.equal(recordOf(await loadOnboardingState(settings), 'admin').guide_used, true) const legacy = recordOf({ admin: { status: 'completed', updated_at: '2026-01-01T00:00:00.000Z' } }, 'admin') assert.equal(legacy.guide_used, false) assert.equal(buildOnboardingPayload({ user: 'admin', role: 'admin', ...legacy }).needs_welcome, true) - assert.equal(recordOf(loadOnboardingState(settings), 'bob').status, 'pending') + assert.equal(recordOf(await loadOnboardingState(settings), 'bob').status, 'pending') - saveOnboardingState(settings, applyOnboardingAction(loadOnboardingState(settings), 'admin', 'reset')) - assert.equal(recordOf(loadOnboardingState(settings), 'admin').status, 'pending') - assert.equal(loadOnboardingState(settings).admin, undefined) + await saveOnboardingState(settings, applyOnboardingAction(await loadOnboardingState(settings), 'admin', 'reset')) + assert.equal(recordOf(await loadOnboardingState(settings), 'admin').status, 'pending') + assert.equal((await loadOnboardingState(settings)).admin, undefined) assert.throws( () => applyOnboardingAction({}, 'admin', 'skip'), @@ -159,48 +159,48 @@ test('only VMs visible to the user drive slot and schedulable steps', () => { assert.equal(steps('u2').find((s) => s.id === 'schedulable').done, true) }) -test('first_request EXISTS uses the same ownerPred as request-log stats', () => { +test('first_request EXISTS uses the same ownerPred as request-log stats', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-onboarding-logs-')) - const db = createDatabase({ dbPath: path.join(dir, 'logs.db') }) - db.exec(` + const db = await createDatabase({ dbPath: path.join(dir, 'logs.db') }) + await db.exec(` INSERT INTO users (id, email, username, password_hash) VALUES ('u1', 'a@x', 'alice', 'x'), ('u2', 'b@x', 'bob', 'x'); INSERT INTO api_keys (id, name, key, user_id) VALUES ('k1', 'alice-key', 'sk-1', 'u1'), ('k2', 'bob-key', 'sk-2', 'u2'); `) const repo = new UsageLogsRepo(db) - repo.insertSummary({ + await repo.insertSummary({ id: 'log_fail', request_id: 'req-fail', created_at: '2026-01-01T00:00:00.000Z', status: 401, user_id: 'u1', }) - assert.equal(repo.hasSuccessfulRequest({ owner_user_id: 'u1' }), false) - repo.insertSummary({ + assert.equal(await repo.hasSuccessfulRequest({ owner_user_id: 'u1' }), false) + await repo.insertSummary({ id: 'log_other', request_id: 'req-other', created_at: '2026-01-01T00:00:01.000Z', status: 200, user_id: 'u2', }) - assert.equal(repo.hasSuccessfulRequest({ owner_user_id: 'u1' }), false) - assert.equal(repo.hasSuccessfulRequest({ owner_user_id: 'u2' }), true) - repo.insertSummary({ + assert.equal(await repo.hasSuccessfulRequest({ owner_user_id: 'u1' }), false) + assert.equal(await repo.hasSuccessfulRequest({ owner_user_id: 'u2' }), true) + await repo.insertSummary({ id: 'log_ok', request_id: 'req-ok', created_at: '2026-01-01T00:00:02.000Z', status: 200, user_id: 'u1', }) - assert.equal(repo.hasSuccessfulRequest({ owner_user_id: 'u1' }), true) - assert.equal(repo.hasSuccessfulRequest({}), true) - repo.insertSummary({ + assert.equal(await repo.hasSuccessfulRequest({ owner_user_id: 'u1' }), true) + assert.equal(await repo.hasSuccessfulRequest({}), true) + await repo.insertSummary({ id: 'log_redirect', request_id: 'req-3xx', created_at: '2026-01-01T00:00:03.000Z', status: 301, user_id: 'u3', }) - assert.equal(repo.hasSuccessfulRequest({ owner_user_id: 'u3' }), false) + assert.equal(await repo.hasSuccessfulRequest({ owner_user_id: 'u3' }), false) }) test('master API key identity is __master__ / admin; base_url strips trailing slash', () => { diff --git a/test/unit/opus-55-policy.test.mjs b/test/unit/opus-55-policy.test.mjs index edcda7196..86dcbecab 100644 --- a/test/unit/opus-55-policy.test.mjs +++ b/test/unit/opus-55-policy.test.mjs @@ -48,17 +48,17 @@ test('a saved dotted Opus 5.5 row keeps administrator overrides on the hyphen id assert.equal(migrated.aliases[legacy], canonical) }) -test('loading settings that predate Opus 5.5 appends the model without resetting Opus 5', () => { - openDatabase({ dbPath: ':memory:' }) +test('loading settings that predate Opus 5.5 appends the model without resetting Opus 5', async () => { + await openDatabase({ dbPath: ':memory:' }) try { const repo = new SettingsRepo() - repo.set('model_policy', { + await repo.set('model_policy', { source: 'panel', models: { 'claude-opus-5': { enabled: false, display_name: 'kept' } }, aliases: { opus: 'claude-opus-5' }, }) - loadModelPolicy({ force: true }) - const stored = repo.get('model_policy') + await loadModelPolicy({ force: true }) + const stored = await repo.get('model_policy') assert.equal(stored.models['claude-opus-5'].enabled, false) assert.equal(stored.models['claude-opus-5'].display_name, 'kept') assert.equal(stored.models[canonical].display_name, 'Opus 5.5') @@ -72,7 +72,7 @@ test('loading settings that predate Opus 5.5 appends the model without resetting assert.deepEqual(thinking.thinking, { type: 'adaptive' }) assert.equal(thinking.thinking.budget_tokens, undefined) } finally { - closeDatabase() + await closeDatabase() clearModelsCache() } }) diff --git a/test/unit/panel-api-proxy.test.mjs b/test/unit/panel-api-proxy.test.mjs index f37780ae7..594c92187 100644 --- a/test/unit/panel-api-proxy.test.mjs +++ b/test/unit/panel-api-proxy.test.mjs @@ -29,7 +29,7 @@ function writeVm(project, rec) { fs.writeFileSync(path.join(dir, `${rec.id}.json`), JSON.stringify(rec, null, 2)) } -test('summarizeProxyPool includes disconnect_on_error', () => { +test('summarizeProxyPool includes disconnect_on_error', async () => { assert.deepEqual(summarizeProxyPool(null), { total: 0, free: 0, @@ -40,7 +40,7 @@ test('summarizeProxyPool includes disconnect_on_error', () => { disconnect_on_error: false, }) const dir = tmpDir() - const pool = new ProxyPool({ dataDir: dir }) + const pool = await new ProxyPool({ dataDir: dir }).init() pool.importLines('10.0.0.9:1080') pool.updateConfig({ disconnect_on_error: true }) pool.stopScheduler() @@ -63,7 +63,7 @@ test('dashboard and vm detail merge pool health onto bound VM', async () => { claude: {}, policy: { maxConcurrency: 4, weight: 1, priority: 8 }, }) - const pool = new ProxyPool({ dataDir }) + const pool = await new ProxyPool({ dataDir }).init() pool.importLines('10.8.8.8:1080') const px = pool.snapshot().proxies[0] pool.bind(px.id, 'vm-02') @@ -157,7 +157,7 @@ test('dead bound proxy blocks credential import', async () => { claude: {}, policy: {}, }) - const pool = new ProxyPool({ dataDir }) + const pool = await new ProxyPool({ dataDir }).init() pool.importLines('10.7.7.7:1080') const id = pool.snapshot().proxies[0].id pool.bind(id, 'vm-03') @@ -358,10 +358,10 @@ test('local proxy node labels are present for admin and absent for tenant', asyn { id: 'vm-tokyo', node_id: 'node-a' }, { id: 'vm-tenant', node_id: 'node-a', owner_user_id: 'user-1' }, ] - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir, vmNodeOf: (vmId) => vms.find((vm) => vm.id === vmId)?.node_id || null, - }) + }).init() t.after(() => pool.stopScheduler()) pool.ensureLocal() // A generic px-local bind lands each slot on its own VPS's local row. diff --git a/test/unit/panel-billing-lookup.test.mjs b/test/unit/panel-billing-lookup.test.mjs index ac849a5bc..80faaa516 100644 --- a/test/unit/panel-billing-lookup.test.mjs +++ b/test/unit/panel-billing-lookup.test.mjs @@ -30,7 +30,7 @@ test('billing index keeps UUID row when leftover vm-id account is empty', () => assert.equal(lookupBilling(index, { id: 'vm-50' }), real) }) -test('billing rows of an older account on a reused slot keep their own email', (t) => { +test('billing rows of an older account on a reused slot keep their own email', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-billing-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) const billing = { @@ -48,7 +48,7 @@ test('billing rows of an older account on a reused slot keep their own email', ( ], }), } - const usage = buildUsage({ + const usage = await buildUsage({ accountQuota, cfg: { paths: { project: root } }, requestLog: { billingStats: () => billing }, @@ -95,12 +95,12 @@ test('VM list carries lifetime tokens from the same billing row as total_cost', assert.equal(byId['vm-08'].total_tokens, 0) }) -test('usage rows of a deleted VM are not listed', (t) => { +test('usage rows of a deleted VM are not listed', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-usage-deleted-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) fs.mkdirSync(path.join(root, 'vms'), { recursive: true }) fs.writeFileSync(path.join(root, 'vms', 'vm-02.json'), JSON.stringify({ id: 'vm-02' })) - const usage = buildUsage({ + const usage = await buildUsage({ accountQuota: { snapshot: () => ({ accounts: [ @@ -122,7 +122,7 @@ test('usage rows of a deleted VM are not listed', (t) => { assert.equal(usage.data.billing.accounts[0].email, 'three@example.com') }) -test('GPT usage row without pool email takes slot email', (t) => { +test('GPT usage row without pool email takes slot email', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-gpt-usage-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) fs.mkdirSync(path.join(root, 'vms'), { recursive: true }) @@ -137,7 +137,7 @@ test('GPT usage row without pool email takes slot email', (t) => { codex: { email: 'lemeryvorhees@gmail.com' }, }), ) - const usage = buildUsage({ + const usage = await buildUsage({ accountQuota: { snapshot: () => ({ accounts: [{ account_id: 'vm-codex-01', vm_id: 'vm-codex-01', email: null }], diff --git a/test/unit/panel-persistence.test.mjs b/test/unit/panel-persistence.test.mjs new file mode 100644 index 000000000..64221fcee --- /dev/null +++ b/test/unit/panel-persistence.test.mjs @@ -0,0 +1,86 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' + +function setup({ failBefore = false } = {}) { + const sent = [] + let writes = 0 + let changed = 0 + let maintenance = 0 + const handler = createPanelHandler({ + cfg: { paths: { project: '/missing-project' } }, + json: (_res, status, body) => sent.push({ status, body }), + readBody: async () => ({ enabled: false }), + requireAuth: () => true, + clusterRoutes: { + handle: (_req, res, _url, respond) => { + changed += 1 + return respond(res, 201, { ok: true }) + }, + }, + onPersistenceFailure: () => { + maintenance += 1 + }, + proxyPool: { + writes: { + flush: async () => { + writes += 1 + if (failBefore || writes > 1) throw new Error('synthetic persistence failure') + }, + }, + snapshot: () => ({ config: {}, proxies: [] }), + updateConfig: () => { + changed += 1 + return { ok: true, config: { enabled: false } } + }, + }, + }) + return { handler, sent, changed: () => changed, maintenance: () => maintenance } +} + +test('panel cache mutations send failure when persistence fails before a success response', async () => { + const state = setup() + await state.handler({ method: 'PUT', apiKeyKind: 'master' }, {}, new URL('http://test/api/panel/proxies/config')) + assert.equal(state.changed(), 1) + assert.equal(state.maintenance(), 1) + assert.equal(state.sent.length, 1) + assert.equal(state.sent[0].status, 503) + assert.equal(state.sent[0].body.error.code, 'database_persistence_failed') +}) + +test('an already failed cache queue rejects a new panel mutation before changing state', async () => { + const state = setup({ failBefore: true }) + await state.handler({ method: 'PUT', apiKeyKind: 'master' }, {}, new URL('http://test/api/panel/proxies/config')) + assert.equal(state.changed(), 0) + assert.equal(state.maintenance(), 1) + assert.equal(state.sent[0].status, 503) +}) + +test('delegated cluster responses wait for the same persistence boundary', async () => { + const state = setup() + await state.handler({ method: 'POST', apiKeyKind: 'master' }, {}, new URL('http://test/api/panel/cluster/nodes')) + assert.equal(state.changed(), 1) + assert.equal(state.maintenance(), 1) + assert.equal(state.sent.length, 1) + assert.equal(state.sent[0].status, 503) +}) + +test('a staged panel response remains handled after persistence succeeds', async () => { + const sent = [] + const handler = createPanelHandler({ + cfg: {}, + requireAuth: () => true, + json: (_res, status, body) => { + sent.push({ status, body }) + }, + }) + + const handled = await handler( + { method: 'POST', apiKeyKind: 'managed' }, + {}, + new URL('http://test/admin/models/refresh'), + ) + assert.equal(handled, true) + assert.equal(sent.length, 1) + assert.equal(sent[0].status, 403) +}) diff --git a/test/unit/panel-tenant-guards.test.mjs b/test/unit/panel-tenant-guards.test.mjs index 3e418fe45..f684d2992 100644 --- a/test/unit/panel-tenant-guards.test.mjs +++ b/test/unit/panel-tenant-guards.test.mjs @@ -140,17 +140,17 @@ test('tenant proxy import cannot bind a slot it does not own', async (t) => { ) }) -function keyFixture(t) { +async function keyFixture(t) { const root = makeProject(t) claudeVm(root, 'vm-mine', 'u-1') claudeVm(root, 'vm-other', 'u-2') const store = new ApiKeyStore({ dataDir: fs.mkdtempSync(path.join(root, 'keys-')) }) - const key = store.create({ name: 'mine', user_id: 'u-1', group_id: 1 }) + const key = await store.create({ name: 'mine', user_id: 'u-1', group_id: 1 }) return { root, store, key } } test('tenant key PATCH drops user_id and group_id', async (t) => { - const { root, store, key } = keyFixture(t) + const { root, store, key } = await keyFixture(t) const out = await makeHandler( root, TENANT, @@ -158,7 +158,7 @@ test('tenant key PATCH drops user_id and group_id', async (t) => { { apiKeyStore: store }, ).call('PATCH', `/api/panel/api-keys/${key.id}`) assert.equal(out.status, 200, JSON.stringify(out.body)) - const saved = store.getById(key.id) + const saved = await store.getById(key.id) assert.equal(saved.name, 'renamed') assert.equal(saved.user_id, 'u-1') assert.equal(saved.group_id, 1) @@ -168,11 +168,11 @@ test('tenant key PATCH drops user_id and group_id', async (t) => { `/api/panel/api-keys/${key.id}`, ) assert.equal(cleared.status, 200) - assert.equal(store.getById(key.id).user_id, 'u-1') + assert.equal((await store.getById(key.id)).user_id, 'u-1') }) test('tenant key scope only accepts its own slots, on PATCH and create', async (t) => { - const { root, store, key } = keyFixture(t) + const { root, store, key } = await keyFixture(t) const foreign = { group_type: 'anthropic', allowed_vms: ['vm-other'] } const missing = { group_type: 'anthropic', allowed_vms: ['vm-nope'] } @@ -187,7 +187,7 @@ test('tenant key scope only accepts its own slots, on PATCH and create', async ( `/api/panel/api-keys/${key.id}`, ) assert.deepEqual(patchForeign.body, patchMissing.body) - assert.equal(store.getById(key.id).group_type, 'all') + assert.equal((await store.getById(key.id)).group_type, 'all') const patchOwn = await makeHandler( root, @@ -196,7 +196,7 @@ test('tenant key scope only accepts its own slots, on PATCH and create', async ( { apiKeyStore: store }, ).call('PATCH', `/api/panel/api-keys/${key.id}`) assert.equal(patchOwn.status, 200, JSON.stringify(patchOwn.body)) - assert.deepEqual(parseAllowedVms(store.getById(key.id).allowed_vms), ['vm-mine']) + assert.deepEqual(parseAllowedVms((await store.getById(key.id)).allowed_vms), ['vm-mine']) const before = store.list().length const createForeign = await makeHandler(root, TENANT, { name: 'x', ...foreign }, { apiKeyStore: store }).call( @@ -214,13 +214,13 @@ test('tenant key scope only accepts its own slots, on PATCH and create', async ( { apiKeyStore: store }, ).call('POST', '/api/panel/api-keys') assert.equal(created.status, 201, JSON.stringify(created.body)) - const rec = store.getById(created.body.item.id) + const rec = await store.getById(created.body.item.id) assert.equal(rec.user_id, 'u-1') assert.equal(rec.group_id, 1) }) test('admin key edits keep user_id, group_id and any slot', async (t) => { - const { root, store, key } = keyFixture(t) + const { root, store, key } = await keyFixture(t) const out = await makeHandler( root, ADMIN, @@ -228,7 +228,7 @@ test('admin key edits keep user_id, group_id and any slot', async (t) => { { apiKeyStore: store }, ).call('PATCH', `/api/panel/api-keys/${key.id}`) assert.equal(out.status, 200, JSON.stringify(out.body)) - const saved = store.getById(key.id) + const saved = await store.getById(key.id) assert.equal(saved.user_id, 'u-2') assert.equal(saved.group_id, 7) assert.deepEqual(parseAllowedVms(saved.allowed_vms), ['vm-other']) @@ -240,15 +240,15 @@ test('admin key edits keep user_id, group_id and any slot', async (t) => { { apiKeyStore: store }, ).call('POST', '/api/panel/api-keys') assert.equal(created.status, 201, JSON.stringify(created.body)) - const rec = store.getById(created.body.item.id) + const rec = await store.getById(created.body.item.id) assert.equal(rec.user_id, 'u-2') assert.equal(rec.group_id, 7) }) test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) => { - const { root, store, key } = keyFixture(t) + const { root, store, key } = await keyFixture(t) const poolRepo = new VmPoolsRepo(store.db) - const pool = createVmPool(poolRepo, { name: 'Shared Pool', vm_ids: [] }) + const pool = await createVmPool(poolRepo, { name: 'Shared Pool', vm_ids: [] }) // 1. Tenant POST /api/panel/api-keys with non-empty vm_pool_id -> 403 key_pool_forbidden const countBefore = store.list().length @@ -273,7 +273,7 @@ test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) = { apiKeyStore: store }, ).call('POST', '/api/panel/api-keys') assert.equal(tenantCreateEmpty.status, 201, JSON.stringify(tenantCreateEmpty.body)) - const emptyKey = store.getById(tenantCreateEmpty.body.item.id) + const emptyKey = await store.getById(tenantCreateEmpty.body.item.id) assert.equal(emptyKey.vm_pool_id, null) const tenantCreateNull = await makeHandler( @@ -283,7 +283,7 @@ test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) = { apiKeyStore: store }, ).call('POST', '/api/panel/api-keys') assert.equal(tenantCreateNull.status, 201, JSON.stringify(tenantCreateNull.body)) - const nullKey = store.getById(tenantCreateNull.body.item.id) + const nullKey = await store.getById(tenantCreateNull.body.item.id) assert.equal(nullKey.vm_pool_id, null) // 3. Admin POST with vm_pool_id -> 201 success @@ -294,7 +294,7 @@ test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) = { apiKeyStore: store }, ).call('POST', '/api/panel/api-keys') assert.equal(adminCreate.status, 201, JSON.stringify(adminCreate.body)) - const adminKey = store.getById(adminCreate.body.item.id) + const adminKey = await store.getById(adminCreate.body.item.id) assert.equal(adminKey.vm_pool_id, pool.id) // 4. Admin assigns pool to tenant key @@ -303,7 +303,7 @@ test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) = `/api/panel/api-keys/${key.id}`, ) assert.equal(adminPatchPool.status, 200, JSON.stringify(adminPatchPool.body)) - assert.equal(store.getById(key.id).vm_pool_id, pool.id) + assert.equal((await store.getById(key.id)).vm_pool_id, pool.id) // 5. Tenant PATCH with non-empty vm_pool_id -> 403 key_pool_forbidden const tenantPatchWithPool = await makeHandler( @@ -317,7 +317,7 @@ test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) = ok: false, error: { code: 'key_pool_forbidden', message: '只有管理员可以设置账号池' }, }) - assert.equal(store.getById(key.id).vm_pool_id, pool.id, 'retained existing pool') + assert.equal((await store.getById(key.id)).vm_pool_id, pool.id, 'retained existing pool') // 6. Tenant PATCH with null or empty string vm_pool_id -> retains existing pool without error const tenantPatchNull = await makeHandler( @@ -327,7 +327,7 @@ test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) = { apiKeyStore: store }, ).call('PATCH', `/api/panel/api-keys/${key.id}`) assert.equal(tenantPatchNull.status, 200, JSON.stringify(tenantPatchNull.body)) - const keyAfterTenantNull = store.getById(key.id) + const keyAfterTenantNull = await store.getById(key.id) assert.equal(keyAfterTenantNull.name, 'renamed-by-tenant') assert.equal(keyAfterTenantNull.vm_pool_id, pool.id, 'pool was not wiped by tenant') @@ -337,46 +337,46 @@ test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) = `/api/panel/api-keys/${key.id}`, ) assert.equal(tenantPatchOmit.status, 200, JSON.stringify(tenantPatchOmit.body)) - assert.equal(store.getById(key.id).vm_pool_id, pool.id) + assert.equal((await store.getById(key.id)).vm_pool_id, pool.id) }) -function usersFixture(t) { +async function usersFixture(t) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-tenant-guards-users-')) - const db = createDatabase({ dataDir: dir }) - t.after(() => { - db.close() + const db = await createDatabase({ dataDir: dir }) + t.after(async () => { + await db.close() fs.rmSync(dir, { recursive: true, force: true }) }) const panelUsers = new PanelUserStore({ db }) - panelUsers.create({ username: 'boss', password: 'boss-pass-1', role: 'admin' }) + await panelUsers.create({ username: 'boss', password: 'boss-pass-1', role: 'admin' }) return panelUsers } test('changing a user role or disabling it ends that user panel sessions', async (t) => { - const panelUsers = usersFixture(t) + const panelUsers = await usersFixture(t) const root = makeProject(t) for (const [i, patch] of [{ role: 'super' }, { enabled: false }].entries()) { - const user = panelUsers.create({ username: `tenant${i}`, password: 'tenant-pass', role: 'user' }) - const token = createPanelSession(user.username, { role: 'user' }) - const other = createPanelSession('boss', { role: 'admin' }) + const user = await panelUsers.create({ username: `tenant${i}`, password: 'tenant-pass', role: 'user' }) + const token = await createPanelSession(user.username, { role: 'user' }) + const other = await createPanelSession('boss', { role: 'admin' }) const out = await makeHandler(root, ADMIN, patch, { panelUsers }).call('PATCH', `/api/panel/users/${user.id}`) assert.equal(out.status, 200, JSON.stringify(out.body)) - assert.equal(verifyPanelSession(token), null, JSON.stringify(patch)) - assert.equal(verifyPanelSession(other)?.user, 'boss') + assert.equal(await verifyPanelSession(token), null, JSON.stringify(patch)) + assert.equal((await verifyPanelSession(other))?.user, 'boss') } }) test('unrelated user edits keep sessions alive', async (t) => { - const panelUsers = usersFixture(t) + const panelUsers = await usersFixture(t) const root = makeProject(t) - const user = panelUsers.create({ username: 'quota', password: 'tenant-pass', role: 'user' }) - const token = createPanelSession(user.username, { role: 'user' }) + const user = await panelUsers.create({ username: 'quota', password: 'tenant-pass', role: 'user' }) + const token = await createPanelSession(user.username, { role: 'user' }) const out = await makeHandler(root, ADMIN, { vm_create_quota: 3, role: 'user', enabled: true }, { panelUsers }).call( 'PATCH', `/api/panel/users/${user.id}`, ) assert.equal(out.status, 200, JSON.stringify(out.body)) - assert.equal(verifyPanelSession(token)?.user, 'quota') + assert.equal((await verifyPanelSession(token))?.user, 'quota') }) test('test-models is readable by user and super; other routes stay closed', () => { diff --git a/test/unit/panel-upgrade-script.test.mjs b/test/unit/panel-upgrade-script.test.mjs index 3728bed73..715515d0c 100644 --- a/test/unit/panel-upgrade-script.test.mjs +++ b/test/unit/panel-upgrade-script.test.mjs @@ -17,6 +17,7 @@ const args = process.argv.slice(2) args.push('--compose-file-env', process.env.COMPOSE_FILE || '') fs.appendFileSync(process.env.TEST_LOG, JSON.stringify(args) + '\n') if (args[0] === 'compose') { + if (args.includes('run') && args.includes('scripts/check-postgres-upgrade.mjs') && process.env.FAIL_CUTOVER === '1') process.exit(1) if (args.includes('build') && process.env.FAIL_BUILD === '1') process.exit(1) if (args.includes('up') && process.env.STALE_BACKEND !== '1') { fs.writeFileSync(process.env.BOOT_VERSION, 'v1.2.7') @@ -227,3 +228,10 @@ test('ARM64 source upgrade keeps the native override and rejects a missing one b assert.match(missing.stderr, /lacks docker-compose.arm64.yml/) assert.ok(!missing.calls.some((args) => args[0] === 'compose')) }) + +test('failed PostgreSQL cutover preflight leaves the running control plane intact', async () => { + const result = await runUpgrade({ source: false, env: { FAIL_CUTOVER: '1' } }) + assert.equal(result.code, 1) + assert.ok(result.calls.some((args) => args.includes('scripts/check-postgres-upgrade.mjs'))) + assert.ok(!result.calls.some((args) => args[0] === 'compose' && args.includes('up'))) +}) diff --git a/test/unit/panel-users.test.mjs b/test/unit/panel-users.test.mjs index e6de6afaa..7619a2127 100644 --- a/test/unit/panel-users.test.mjs +++ b/test/unit/panel-users.test.mjs @@ -3,12 +3,14 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb, withTransaction } from '../../src/lib/db/database.mjs' +import { RedeemCodesRepo } from '../../src/lib/db/repos/redeem-codes-repo.mjs' +import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' import { PanelUserStore, hashPassword, verifyPassword, publicUserView } from '../../src/lib/admin/panel-users.mjs' -function tmpStore() { +async function tmpStore() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-panel-users-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() return { dir, db, store: new PanelUserStore({ db, projectRoot: dir }) } } @@ -20,82 +22,88 @@ test('scrypt hash verifies and is not reversible', () => { assert.doesNotMatch(hashed, /correct-horse/) }) -test('bootstrap seeds env admin once', () => { - const { store } = tmpStore() - const first = store.bootstrapFromEnv({ username: 'admin', password: 'testpass1' }) - const second = store.bootstrapFromEnv({ username: 'admin', password: 'otherpass' }) +test('bootstrap seeds env admin once', async () => { + const { store } = await tmpStore() + const first = await store.bootstrapFromEnv({ username: 'admin', password: 'testpass1' }) + const second = await store.bootstrapFromEnv({ username: 'admin', password: 'otherpass' }) assert.equal(first.username, 'admin') assert.equal(first.role, 'admin') assert.equal(second.id, first.id) - assert.ok(store.authenticate('admin', 'testpass1')) - assert.equal(store.authenticate('admin', 'otherpass'), null) + assert.ok(await store.authenticate('admin', 'testpass1')) + assert.equal(await store.authenticate('admin', 'otherpass'), null) }) -test('bootstrap imports legacy env passwords without relaxing user password validation', () => { - const { store } = tmpStore() +test('bootstrap imports legacy env passwords without relaxing user password validation', async () => { + const { store } = await tmpStore() for (const [username, password] of [ ['admin', '123456'], ['legacy', 'short'], ]) { - const admin = store.bootstrapFromEnv({ username, password }) + const admin = await store.bootstrapFromEnv({ username, password }) assert.equal(admin.role, 'admin') assert.equal(admin.enabled, true) assert.equal(admin.password_hash, undefined) - assert.ok(store.list().some((user) => user.id === admin.id)) - assert.ok(store.authenticate(username, password)) - assert.notEqual(store.getById(admin.id).password_hash, password) - assert.throws(() => store.update(admin.id, { password: 'short' }), /密码至少 8 位/) + assert.ok((await store.list()).some((user) => user.id === admin.id)) + assert.ok(await store.authenticate(username, password)) + assert.notEqual((await store.getById(admin.id)).password_hash, password) + await assert.rejects(async () => await store.update(admin.id, { password: 'short' }), /密码至少 8 位/) - store.update(admin.id, { password: 'changed-admin-pass' }) - const again = store.bootstrapFromEnv({ username, password }) + await store.update(admin.id, { password: 'changed-admin-pass' }) + const again = await store.bootstrapFromEnv({ username, password }) assert.equal(again.id, admin.id) - assert.equal(store.authenticate(username, password), null) - assert.ok(store.authenticate(username, 'changed-admin-pass')) + assert.equal(await store.authenticate(username, password), null) + assert.ok(await store.authenticate(username, 'changed-admin-pass')) } - assert.throws(() => store.create({ username: 'reader', password: '123456' }), /密码至少 8 位/) + await assert.rejects(async () => await store.create({ username: 'reader', password: '123456' }), /密码至少 8 位/) }) -test('bootstrap preserves disabled admins and validates usernames', () => { - const { store } = tmpStore() - store.create({ username: 'owner', password: 'owner-pass', role: 'admin' }) - const admin = store.create({ username: 'admin', password: 'admin-pass', role: 'admin', enabled: false }) - const imported = store.bootstrapFromEnv({ username: 'admin', password: '123456' }) +test('bootstrap preserves disabled admins and validates usernames', async () => { + const { store } = await tmpStore() + await store.create({ username: 'owner', password: 'owner-pass', role: 'admin' }) + const admin = await store.create({ username: 'admin', password: 'admin-pass', role: 'admin', enabled: false }) + const imported = await store.bootstrapFromEnv({ username: 'admin', password: '123456' }) assert.equal(imported.id, admin.id) assert.equal(imported.enabled, false) - assert.equal(store.authenticate('admin', '123456'), null) - assert.equal(store.bootstrapFromEnv({ username: 'empty', password: '' }), null) - assert.throws(() => store.bootstrapFromEnv({ username: '1bad', password: '123456' }), /用户名/) + assert.equal(await store.authenticate('admin', '123456'), null) + assert.equal(await store.bootstrapFromEnv({ username: 'empty', password: '' }), null) + await assert.rejects(async () => await store.bootstrapFromEnv({ username: '1bad', password: '123456' }), /用户名/) }) -test('create / authenticate / change password / roles', () => { - const { store } = tmpStore() - store.bootstrapFromEnv({ username: 'admin', password: 'adminpass' }) - const viewer = store.create({ username: 'reader', password: 'reader-pass', role: 'user' }) - const superu = store.create({ username: 'ops', password: 'super-pass', role: 'super' }) +test('create / authenticate / change password / roles', async () => { + const { store } = await tmpStore() + await store.bootstrapFromEnv({ username: 'admin', password: 'adminpass' }) + const viewer = await store.create({ username: 'reader', password: 'reader-pass', role: 'user' }) + const superu = await store.create({ username: 'ops', password: 'super-pass', role: 'super' }) assert.equal(viewer.role, 'user') assert.equal(superu.role, 'super') - assert.equal(store.authenticate('reader', 'reader-pass').role, 'user') - assert.equal(store.authenticate('ops', 'wrong-pass'), null) + assert.equal((await store.authenticate('reader', 'reader-pass')).role, 'user') + assert.equal(await store.authenticate('ops', 'wrong-pass'), null) - const updated = store.update(viewer.id, { password: 'reader-next' }) - assert.equal(store.authenticate('reader', 'reader-pass'), null) - assert.ok(store.authenticate('reader', 'reader-next')) + const updated = await store.update(viewer.id, { password: 'reader-next' }) + assert.equal(await store.authenticate('reader', 'reader-pass'), null) + assert.ok(await store.authenticate('reader', 'reader-next')) assert.equal(publicUserView(updated).password_hash, undefined) }) -test('cannot remove or demote the last admin', () => { - const { store } = tmpStore() - const admin = store.create({ username: 'admin', password: 'adminpass', role: 'admin' }) - assert.throws(() => store.update(admin.id, { role: 'user' }), /最后一个管理员/) - assert.throws(() => store.update(admin.id, { enabled: false }), /最后一个管理员/) - assert.throws(() => store.remove(admin.id), /最后一个管理员/) +test('cannot remove or demote the last admin', async () => { + const { store } = await tmpStore() + const admin = await store.create({ username: 'admin', password: 'adminpass', role: 'admin' }) + await assert.rejects(async () => await store.update(admin.id, { role: 'user' }), /最后一个管理员/) + await assert.rejects(async () => await store.update(admin.id, { enabled: false }), /最后一个管理员/) + await assert.rejects(async () => await store.remove(admin.id), /最后一个管理员/) }) -test('rejects bad username / short password / unknown role', () => { - const { store } = tmpStore() - assert.throws(() => store.create({ username: '1bad', password: 'longenough', role: 'user' }), /用户名/) - assert.throws(() => store.create({ username: 'okuser', password: 'short', role: 'user' }), /密码/) - assert.throws(() => store.create({ username: 'okuser', password: 'longenough', role: 'root' }), /角色/) +test('rejects bad username / short password / unknown role', async () => { + const { store } = await tmpStore() + await assert.rejects( + async () => await store.create({ username: '1bad', password: 'longenough', role: 'user' }), + /用户名/, + ) + await assert.rejects(async () => await store.create({ username: 'okuser', password: 'short', role: 'user' }), /密码/) + await assert.rejects( + async () => await store.create({ username: 'okuser', password: 'longenough', role: 'root' }), + /角色/, + ) }) function seedOwnedVm(project, id, owner, extra = {}) { @@ -115,21 +123,21 @@ function seedOwnedVm(project, id, owner, extra = {}) { return vm } -test('deleting a user reclaims assigned VMs while preserving private and unrelated slots', (t) => { - const { dir, db, store } = tmpStore() - t.after(() => { - db.close() +test('deleting a user reclaims assigned VMs while preserving private and unrelated slots', async (t) => { + const { dir, db, store } = await tmpStore() + t.after(async () => { + await db.close() fs.rmSync(dir, { recursive: true, force: true }) }) - const user = store.create({ username: 'tenant', password: 'tenant-pass' }) + const user = await store.create({ username: 'tenant', password: 'tenant-pass' }) const assigned = seedOwnedVm(dir, 'assigned', user.id) seedOwnedVm(dir, 'legacy', user.id, { origin: undefined }) const privateVm = seedOwnedVm(dir, 'private', user.id, { origin: 'user_created' }) const otherVm = seedOwnedVm(dir, 'other', 'another-tenant') const publicVm = seedOwnedVm(dir, 'public', null, { origin: 'platform' }) - assert.equal(store.remove(user.id), true) - assert.equal(store.getById(user.id), null) + assert.equal(await store.remove(user.id), true) + assert.equal(await store.getById(user.id), null) const read = (id) => JSON.parse(fs.readFileSync(path.join(dir, 'vms', `${id}.json`), 'utf8')) const reclaimed = read('assigned') assert.equal(reclaimed.owner_user_id, null) @@ -144,37 +152,83 @@ test('deleting a user reclaims assigned VMs while preserving private and unrelat assert.deepEqual(read('public'), publicVm) }) -test('rejected user deletions leave assigned VMs with their owners', (t) => { - const { dir, db, store } = tmpStore() - t.after(() => { - db.close() +test('rejected user deletions leave assigned VMs with their owners', async (t) => { + const { dir, db, store } = await tmpStore() + t.after(async () => { + await db.close() fs.rmSync(dir, { recursive: true, force: true }) }) - const admin = store.create({ username: 'admin', password: 'admin-pass', role: 'admin' }) - const user = store.create({ username: 'tenant', password: 'tenant-pass' }) + const admin = await store.create({ username: 'admin', password: 'admin-pass', role: 'admin' }) + const user = await store.create({ username: 'tenant', password: 'tenant-pass' }) const assigned = seedOwnedVm(dir, 'assigned', user.id) const adminVm = seedOwnedVm(dir, 'admin-vm', admin.id) - assert.throws(() => store.remove(user.id, { actorId: user.id }), /不能删除当前登录账号/) - assert.throws(() => store.remove(admin.id), /最后一个管理员/) - assert.equal(store.remove('missing-user'), false) + await assert.rejects(async () => await store.remove(user.id, { actorId: user.id }), /不能删除当前登录账号/) + await assert.rejects(async () => await store.remove(admin.id), /最后一个管理员/) + assert.equal(await store.remove('missing-user'), false) assert.deepEqual(JSON.parse(fs.readFileSync(path.join(dir, 'vms', 'assigned.json'), 'utf8')), assigned) assert.deepEqual(JSON.parse(fs.readFileSync(path.join(dir, 'vms', 'admin-vm.json'), 'utf8')), adminVm) }) -test('a VM write failure does not soft-delete the user', (t) => { - const { dir, db, store } = tmpStore() - t.after(() => { - db.close() +test('a VM write failure does not soft-delete the user', async (t) => { + const { dir, db, store } = await tmpStore() + t.after(async () => { + await db.close() fs.rmSync(dir, { recursive: true, force: true }) }) - const user = store.create({ username: 'tenant', password: 'tenant-pass' }) + const user = await store.create({ username: 'tenant', password: 'tenant-pass' }) const assigned = seedOwnedVm(dir, 'assigned', user.id) t.mock.method(fs, 'renameSync', () => { throw new Error('simulated VM write failure') }) - assert.throws(() => store.remove(user.id), /simulated VM write failure/) - assert.ok(store.getById(user.id)) + await assert.rejects(async () => await store.remove(user.id), /simulated VM write failure/) + assert.ok(await store.getById(user.id)) assert.deepEqual(JSON.parse(fs.readFileSync(path.join(dir, 'vms', 'assigned.json'), 'utf8')), assigned) }) + +test('concurrent redemption credits one balance and rolls back on a failed credit', async () => { + const { store, db } = await tmpStore() + const user = await store.create({ username: 'redeemer', password: 'long-password' }) + const codes = new RedeemCodesRepo(db) + const code = await codes.create({ value: 5 }) + const results = await Promise.all(Array.from({ length: 8 }, () => codes.redeem(code.code, user.id))) + assert.equal(results.filter((r) => r.ok).length, 1) + assert.equal((await store.getById(user.id)).balance, 5) + const failed = await codes.create({ value: 7 }) + codes.users.addBalance = async () => { + throw new Error('credit failed') + } + await assert.rejects(codes.redeem(failed.code, user.id), /credit failed/) + assert.equal((await codes.getByCode(failed.code)).status, 'unused') + assert.equal((await store.getById(user.id)).balance, 5) +}) + +test('concurrent admin demotions keep one enabled administrator', async () => { + const { store } = await tmpStore() + const a = await store.create({ username: 'admin_a', password: 'long-password', role: 'admin' }) + const b = await store.create({ username: 'admin_b', password: 'long-password', role: 'admin' }) + const results = await Promise.allSettled([store.update(a.id, { role: 'user' }), store.update(b.id, { role: 'user' })]) + assert.equal(results.filter((r) => r.status === 'fulfilled').length, 1) + assert.equal(await store.countEnabledAdmins(), 1) +}) + +test('settings cache changes only after a successful transaction commit', async () => { + const db = getDb() + const settings = new SettingsRepo(db) + await settings.set('transaction_setting', 'before') + await assert.rejects( + withTransaction(db, async () => { + await settings.set('transaction_setting', 'after') + assert.equal(db.settingsState.get('transaction_setting'), 'before') + throw new Error('rollback') + }), + /rollback/, + ) + assert.equal(await settings.get('transaction_setting'), 'before') + assert.equal(db.settingsState.get('transaction_setting'), 'before') + await withTransaction(db, async () => { + await settings.set('transaction_setting', 'committed') + }) + assert.equal(db.settingsState.get('transaction_setting'), 'committed') +}) diff --git a/test/unit/panel-vm-create.test.mjs b/test/unit/panel-vm-create.test.mjs index 62e53a58a..0f2865893 100644 --- a/test/unit/panel-vm-create.test.mjs +++ b/test/unit/panel-vm-create.test.mjs @@ -248,7 +248,7 @@ test('auto-numbering skips ids a deleted VM left behind in account history', asy test('create binds an explicitly chosen exit instead of auto-allocating', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-create-exit-')) - const pool = new ProxyPool({ dataDir: root, geoLookup: async () => ({ ok: false, error: 'offline' }) }) + const pool = await new ProxyPool({ dataDir: root, geoLookup: async () => ({ ok: false, error: 'offline' }) }).init() pool.stopScheduler() try { const socks = pool.importLines('1.2.3.4:1080').items[0].id @@ -295,7 +295,7 @@ test('create stamps docker machine from routing defaults', async () => { test('create refuses an unknown or full exit before writing the slot', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-create-exit-bad-')) - const pool = new ProxyPool({ dataDir: root, geoLookup: async () => ({ ok: false, error: 'offline' }) }) + const pool = await new ProxyPool({ dataDir: root, geoLookup: async () => ({ ok: false, error: 'offline' }) }).init() pool.stopScheduler() try { const missing = makeCreateHandler(root, { id: 'vm-x', start: false, proxy_id: 'px-nope' }, pool) diff --git a/test/unit/panel-vm-delete.test.mjs b/test/unit/panel-vm-delete.test.mjs index 474129f86..48d604f90 100644 --- a/test/unit/panel-vm-delete.test.mjs +++ b/test/unit/panel-vm-delete.test.mjs @@ -95,13 +95,13 @@ test('startup reconcile does not resurrect a deleted sole active VM', async () = const { setVmWriteHook } = await import('../../src/lib/vm/vm-file.mjs') const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-del-db-')) try { - openDatabase({ dataDir: path.join(root, 'data') }) + await openDatabase({ dataDir: path.join(root, 'data') }) seed(root, 'vm-01', { active: true }) - initVmDbSync(root, { watch: false }) + await initVmDbSync(root, { watch: false }) const { handlePanel, response } = handler(root) await handlePanel({ method: 'DELETE' }, {}, new URL('http://localhost/api/panel/vms/vm-01')) assert.equal(response.status, 200, JSON.stringify(response.body)) - const again = reconcileVms(root) + const again = await reconcileVms(root) assert.equal(again.rebuilt, 0) assert.equal(again.active, null) assert.equal(fs.existsSync(path.join(root, 'vms', 'vm-01.json')), false) @@ -109,7 +109,7 @@ test('startup reconcile does not resurrect a deleted sole active VM', async () = } finally { stopVmWatch() setVmWriteHook(null) - closeDatabase() + await closeDatabase() fs.rmSync(root, { recursive: true, force: true }) } }) diff --git a/test/unit/panel-vm-exit-probe.test.mjs b/test/unit/panel-vm-exit-probe.test.mjs index 113969465..c6f66e643 100644 --- a/test/unit/panel-vm-exit-probe.test.mjs +++ b/test/unit/panel-vm-exit-probe.test.mjs @@ -12,10 +12,10 @@ function seed(root, vm) { fs.writeFileSync(path.join(dir, `${vm.id}.json`), JSON.stringify({ status: 'stopped', policy: {}, ...vm })) } -function setup(t) { +async function setup(t) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-exit-probe-')) const lookups = [] - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir: path.join(root, 'data'), vmNodeOf: (vmId) => JSON.parse(fs.readFileSync(path.join(root, 'vms', `${vmId}.json`), 'utf8')).node_id || null, localExitUrl: (nodeId) => `socks5h://${nodeId}@127.0.0.1:1080`, @@ -24,7 +24,7 @@ function setup(t) { return { ok: true, geo: { ip: '203.0.113.9', country: 'Japan', city: 'Tokyo', timezone: 'Asia/Tokyo' } } }, geoV6Lookup: async () => ({ ok: false, error: 'no_v6' }), - }) + }).init() pool.stopScheduler() t.after(() => { pool.stopScheduler() @@ -52,7 +52,7 @@ function setup(t) { } test('an unbound slot has no exit to probe', async (t) => { - const { root, probe, lookups } = setup(t) + const { root, probe, lookups } = await setup(t) seed(root, { id: 'vm-free', proxy: null }) const res = await probe('vm-free') assert.equal(res.status, 502) @@ -61,7 +61,7 @@ test('an unbound slot has no exit to probe', async (t) => { }) test('a node slot on local egress probes its own node row and refreshes it', async (t) => { - const { root, pool, probe, lookups } = setup(t) + const { root, pool, probe, lookups } = await setup(t) seed(root, { id: 'vm-tokyo', node_id: 'node-a', proxy: { id: 'px-local', host: 'local', scheme: 'local' } }) pool.ensureLocal() assert.equal(pool.bind('px-local', 'vm-tokyo').proxy.id, 'px-local-node-a') @@ -76,7 +76,7 @@ test('a node slot on local egress probes its own node row and refreshes it', asy }) test('a control-plane Codex slot is probed through the deployment proxy', async (t) => { - const { root, pool, probe, lookups } = setup(t) + const { root, pool, probe, lookups } = await setup(t) const saved = process.env.HTTPS_PROXY process.env.HTTPS_PROXY = 'socks5://deploy.example:1080' t.after(() => { diff --git a/test/unit/pool-runtime-repos.test.mjs b/test/unit/pool-runtime-repos.test.mjs index 942e00e58..bcbf0caa0 100644 --- a/test/unit/pool-runtime-repos.test.mjs +++ b/test/unit/pool-runtime-repos.test.mjs @@ -3,16 +3,16 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' import { AccountRuntimeRepo } from '../../src/lib/db/repos/account-runtime-repo.mjs' import { RequestAttemptsRepo } from '../../src/lib/db/repos/request-attempts-repo.mjs' -test('account runtime state persists account and model cooldowns', () => { +test('account runtime state persists account and model cooldowns', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-runtime-repo-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new AccountRuntimeRepo(db) - repo.upsert({ + await repo.upsert({ account_id: 'account-1', vm_id: 'vm-01', status: 'ready', @@ -22,131 +22,131 @@ test('account runtime state persists account and model cooldowns', () => { worker_status: { ok: true }, }) const until = Date.now() + 60_000 - repo.markCooldown('account-1', { + await repo.markCooldown('account-1', { vmId: 'vm-01', model: 'claude-test', until, reason: 'model_rate_limit', }) - const state = repo.get('account-1') + const state = await repo.get('account-1') assert.equal(state.priority, 3) assert.equal(state.weight, 5) assert.equal(state.model_states['claude-test'].reason, 'model_rate_limit') assert.equal(state.model_states['claude-test'].cooldown_until, until) } finally { - db.close() + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } }) -test('account cooldown mirrors structured rate-limit columns (sub2api style)', () => { +test('account cooldown mirrors structured rate-limit columns (sub2api style)', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-runtime-repo-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new AccountRuntimeRepo(db) const until = Date.now() + 300_000 - repo.markCooldown('account-rl', { + await repo.markCooldown('account-rl', { vmId: 'vm-01', until, reason: 'account_quota_exhausted', }) - let state = repo.get('account-rl') + let state = await repo.get('account-rl') assert.equal(state.rate_limit_reset_at, until) assert.ok(state.rate_limited_at <= Date.now()) assert.equal(state.overload_until, null) - repo.markCooldown('account-rl', { vmId: 'vm-01', until: until + 1000, reason: 'provider_overloaded' }) - state = repo.get('account-rl') + await repo.markCooldown('account-rl', { vmId: 'vm-01', until: until + 1000, reason: 'provider_overloaded' }) + state = await repo.get('account-rl') assert.equal(state.overload_until, null) assert.equal(state.cooldown_until, until + 1000) assert.equal(state.cooldown_reason, 'provider_overloaded') assert.equal(state.rate_limit_reset_at, until) } finally { - db.close() + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } }) -test('updateWindow persists session window without touching cooldowns', () => { +test('updateWindow persists session window without touching cooldowns', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-runtime-repo-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new AccountRuntimeRepo(db) const end = Date.now() + 3600_000 - repo.updateWindow('account-w', { + await repo.updateWindow('account-w', { vmId: 'vm-02', sessionWindowStart: end - 5 * 3600_000, sessionWindowEnd: end, sessionWindowStatus: 'allowed_warning', }) - const state = repo.get('account-w') + const state = await repo.get('account-w') assert.equal(state.session_window_end, end) assert.equal(state.session_window_start, end - 5 * 3600_000) assert.equal(state.session_window_status, 'allowed_warning') assert.equal(state.cooldown_until, null) // partial update keeps prior fields - repo.updateWindow('account-w', { sessionWindowStatus: 'rejected' }) - assert.equal(repo.get('account-w').session_window_end, end) - assert.equal(repo.get('account-w').session_window_status, 'rejected') + await repo.updateWindow('account-w', { sessionWindowStatus: 'rejected' }) + assert.equal((await repo.get('account-w')).session_window_end, end) + assert.equal((await repo.get('account-w')).session_window_status, 'rejected') } finally { - db.close() + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } }) -test('clearAuthCooldown drops 401 leftover and keeps quota cools', () => { +test('clearAuthCooldown drops 401 leftover and keeps quota cools', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-runtime-auth-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new AccountRuntimeRepo(db) const until = Date.now() + 600_000 - repo.markCooldown('auth-1', { + await repo.markCooldown('auth-1', { vmId: 'vm-11', until, reason: 'authentication_failed_after_refresh', }) - repo.markCooldown('quota-1', { + await repo.markCooldown('quota-1', { vmId: 'vm-12', until, reason: 'account_quota_exhausted', }) - assert.equal(repo.clearAuthCooldown('auth-1', { vmId: 'vm-11' }), true) - assert.equal(repo.get('auth-1').cooldown_reason, null) - assert.equal(repo.get('auth-1').cooldown_until, null) - assert.equal(repo.clearAuthCooldown('quota-1', { vmId: 'vm-12' }), false) - assert.equal(repo.get('quota-1').cooldown_reason, 'account_quota_exhausted') - repo.markCooldown('revoked-1', { + assert.equal(await repo.clearAuthCooldown('auth-1', { vmId: 'vm-11' }), true) + assert.equal((await repo.get('auth-1')).cooldown_reason, null) + assert.equal((await repo.get('auth-1')).cooldown_until, null) + assert.equal(await repo.clearAuthCooldown('quota-1', { vmId: 'vm-12' }), false) + assert.equal((await repo.get('quota-1')).cooldown_reason, 'account_quota_exhausted') + await repo.markCooldown('revoked-1', { vmId: 'vm-100', until: Number.MAX_SAFE_INTEGER, reason: 'oauth_revoked', status: 'disabled', }) - assert.equal(repo.clearAuthCooldown('revoked-1', { vmId: 'vm-100' }), false) - assert.equal(repo.get('revoked-1').cooldown_reason, 'oauth_revoked') - assert.equal(repo.clearGrantRevokeCooldown('revoked-1', { vmId: 'vm-100' }), true) - assert.equal(repo.get('revoked-1').cooldown_reason, null) - assert.equal(repo.get('revoked-1').status, 'ready') - repo.markCooldown('setup-1', { + assert.equal(await repo.clearAuthCooldown('revoked-1', { vmId: 'vm-100' }), false) + assert.equal((await repo.get('revoked-1')).cooldown_reason, 'oauth_revoked') + assert.equal(await repo.clearGrantRevokeCooldown('revoked-1', { vmId: 'vm-100' }), true) + assert.equal((await repo.get('revoked-1')).cooldown_reason, null) + assert.equal((await repo.get('revoked-1')).status, 'ready') + await repo.markCooldown('setup-1', { vmId: 'vm-05', until: Number.MAX_SAFE_INTEGER, reason: 'oauth_no_refresh', status: 'disabled', }) - assert.equal(repo.clearGrantRevokeCooldown('setup-1', { vmId: 'vm-05' }), true) - assert.equal(repo.get('setup-1').cooldown_reason, null) - assert.equal(repo.get('setup-1').status, 'ready') + assert.equal(await repo.clearGrantRevokeCooldown('setup-1', { vmId: 'vm-05' }), true) + assert.equal((await repo.get('setup-1')).cooldown_reason, null) + assert.equal((await repo.get('setup-1')).status, 'ready') } finally { - db.close() + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } }) -test('request attempt ledger records final state and commit boundary', () => { +test('request attempt ledger records final state and commit boundary', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-attempt-repo-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new RequestAttemptsRepo(db) - repo.begin({ + await repo.begin({ requestId: 'req-1', attemptNo: 1, vmId: 'vm-01', @@ -155,7 +155,7 @@ test('request attempt ledger records final state and commit boundary', () => { selectionReason: 'balanced', waitMs: 4, }) - repo.complete('req-1', 1, { + await repo.complete('req-1', 1, { upstreamStatus: 429, errorScope: 'account', action: 'continue-and-cooldown', @@ -163,7 +163,7 @@ test('request attempt ledger records final state and commit boundary', () => { terminalState: 'rejected', latencyMs: 12, }) - repo.begin({ + await repo.begin({ requestId: 'req-1', attemptNo: 2, vmId: 'vm-02', @@ -171,7 +171,7 @@ test('request attempt ledger records final state and commit boundary', () => { model: 'claude-test', selectionReason: 'failover', }) - repo.complete('req-1', 2, { + await repo.complete('req-1', 2, { upstreamStatus: 200, downstreamCommitted: true, terminalState: 'verified', @@ -179,14 +179,14 @@ test('request attempt ledger records final state and commit boundary', () => { ttftMs: 20, latencyMs: 40, }) - const attempts = repo.list('req-1') + const attempts = await repo.list('req-1') assert.equal(attempts.length, 2) assert.equal(attempts[0].action, 'continue-and-cooldown') assert.equal(attempts[1].terminal_state, 'verified') assert.equal(attempts[1].downstream_committed, true) assert.deepEqual(attempts[1].usage, { input_tokens: 2, output_tokens: 3 }) } finally { - db.close() + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } }) diff --git a/test/unit/pool-scheduler.test.mjs b/test/unit/pool-scheduler.test.mjs index 2cfc63ab8..1f9fad486 100644 --- a/test/unit/pool-scheduler.test.mjs +++ b/test/unit/pool-scheduler.test.mjs @@ -675,11 +675,11 @@ test('unpinned slot follows live tier concurrency after reloadConfig', async (t) vm.policy.maxConcurrency = 2 vm.policy.concurrencyOverride = false fs.writeFileSync(file, JSON.stringify(vm)) - const accountQuota = new AccountQuota({ + const accountQuota = await new AccountQuota({ dataDir: path.join(root, 'data'), config: { tiers: { max: { max_concurrency: 2, max_rpm: 0 } } }, accounts: [{ account_id: 'account-1', vm_id: 'vm-01', max_concurrency: 2 }], - }) + }).init() accountQuota.setAccountTier('account-1', 'max') const pool = scheduler(root, { accountQuota }) accountQuota.reloadConfig({ @@ -1333,7 +1333,7 @@ test('syncQuotaSchedule keeps Extra 5h exhausted schedulable and skips the slot' const root = project() let pool t.after(() => cleanupProject(root, pool)) - const q = new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }) + const q = await new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }).init() q.ensure({ account_id: 'account-1', vm_id: 'vm-01' }) q.ensure({ account_id: 'account-2', vm_id: 'vm-02' }) const reset = new Date(Date.now() + 4 * 3600_000).toISOString() @@ -1369,7 +1369,7 @@ test('syncQuotaSchedule auto-picks Extra 5h again after the window opens', async const root = project() let pool t.after(() => cleanupProject(root, pool)) - const q = new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }) + const q = await new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }).init() q.ensure({ account_id: 'account-1', vm_id: 'vm-01' }) const reset = new Date(Date.now() + 4 * 3600_000).toISOString() q.ingestHeaders('account-1', { @@ -1411,7 +1411,7 @@ test('syncQuotaSchedule does not turn a manual-off slot back on', async (t) => { const root = project() let pool t.after(() => cleanupProject(root, pool)) - const q = new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }) + const q = await new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }).init() q.ensure({ account_id: 'account-1', vm_id: 'vm-01' }) q.ensure({ account_id: 'account-2', vm_id: 'vm-02' }) q.ingestHeaders('account-1', { @@ -1443,11 +1443,11 @@ test('syncQuotaSchedule does not turn a manual-off slot back on', async (t) => { picked.release() }) -test('leftover quota 调度关 is restored to restricted without flipping the operator off', (t) => { +test('leftover quota 调度关 is restored to restricted without flipping the operator off', async (t) => { const root = project() let pool t.after(() => cleanupProject(root, pool)) - const q = new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }) + const q = await new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }).init() q.ensure({ account_id: 'account-1', vm_id: 'vm-01' }) q.ingestHeaders('account-1', { 'anthropic-ratelimit-unified-5h-utilization': '1', @@ -1539,14 +1539,14 @@ test('platform scope skips tenant-owned VMs', async (t) => { test('deleting a tenant makes its assigned spare selectable while the primary quota is exhausted', async (t) => { const root = project() - const quota = new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }) + const quota = await new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }).init() let pool - t.after(() => { - quota.db.close() + t.after(async () => { + await quota.db.close() cleanupProject(root, pool) }) const users = new PanelUserStore({ db: quota.db, projectRoot: root }) - const user = users.create({ username: 'tenant', password: 'tenant-pass' }) + const user = await users.create({ username: 'tenant', password: 'tenant-pass' }) persistVmOwner(root, 'vm-02', { ownerUserId: user.id, origin: 'admin_assigned' }) quota.ensure({ account_id: 'account-1', vm_id: 'vm-01' }) quota.ensure({ account_id: 'account-2', vm_id: 'vm-02' }) @@ -1559,7 +1559,7 @@ test('deleting a tenant makes its assigned spare selectable while the primary qu const exhausted = await pool.selectAndReserve({ model: 'claude-test', allowWait: false }) assert.equal(exhausted.ok, false) - assert.equal(users.remove(user.id), true) + assert.equal(await users.remove(user.id), true) const recovered = await pool.selectAndReserve({ model: 'claude-test', allowWait: false }) assert.equal(recovered.ok, true) assert.equal(recovered.vmId, 'vm-02') @@ -2067,10 +2067,13 @@ test('#A08: a refused reservation hands back exactly what it took; release is id test('#A15: a hard account exit moves every pinned conversation but not live leases', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2206,10 +2209,13 @@ function verifiedHop(text) { test('family without pin schedules parent and child on one VM', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2305,10 +2311,13 @@ test('family without pin schedules parent and child on one VM', async (t) => { test('family whose VM is gated moves as a whole and releases its sticky pins', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2363,10 +2372,13 @@ test('family whose VM is gated moves as a whole and releases its sticky pins', a test('family VM left only for kernel slot_busy keeps the family pin', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2438,10 +2450,13 @@ function applyRebuildAttempt(callerSession, extra = {}) { test('rebuild outbound session stays on one VM and changes after failover', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2488,10 +2503,13 @@ test('rebuild outbound session stays on one VM and changes after failover', asyn test('consecutive turns without an inbound session keep one outbound session', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2521,10 +2539,13 @@ test('consecutive turns without an inbound session keep one outbound session', a test('family migration remints parent and child outbound sessions', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2604,10 +2625,13 @@ test('family migration remints parent and child outbound sessions', async (t) => test('intra-request failover remints and persists the second outbound session', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2652,10 +2676,13 @@ test('intra-request failover remints and persists the second outbound session', test('returning to a previously used VM mints a new outbound session', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) diff --git a/test/unit/price-catalog.test.mjs b/test/unit/price-catalog.test.mjs index 8848c8d34..e9740dbc4 100644 --- a/test/unit/price-catalog.test.mjs +++ b/test/unit/price-catalog.test.mjs @@ -94,20 +94,20 @@ test('layer precedence: override provider/model > provider/* > */model > models. assert.equal(fromDev.source, 'models.dev') assert.equal(fromDev.price.cache_read, 9) - assert.equal(setPriceOverride(`*/${id}`, { ...FULL, cache_read: 3 }).ok, true) + assert.equal((await setPriceOverride(`*/${id}`, { ...FULL, cache_read: 3 })).ok, true) assert.equal(resolvePrice(id).source, 'override') assert.equal(resolvePrice(id).price.cache_read, 3) - assert.equal(setPriceOverride('anthropic/*', { ...FULL, cache_read: 2 }).ok, true) + assert.equal((await setPriceOverride('anthropic/*', { ...FULL, cache_read: 2 })).ok, true) assert.equal(resolvePrice(id).price.cache_read, 2) - assert.equal(setPriceOverride(`anthropic/${id}`, { ...FULL, cache_read: 1 }).ok, true) + assert.equal((await setPriceOverride(`anthropic/${id}`, { ...FULL, cache_read: 1 })).ok, true) assert.equal(resolvePrice(id).price.cache_read, 1) assert.equal(resolvePrice(id).source, 'override') }) -test('a provider/* override on a dated id keeps the canonical model builtin fast mode', () => { - assert.equal(setPriceOverride('anthropic/*', FULL).ok, true) +test('a provider/* override on a dated id keeps the canonical model builtin fast mode', async () => { + assert.equal((await setPriceOverride('anthropic/*', FULL)).ok, true) const hit = resolvePrice('claude-opus-5-5-20261001') assert.equal(hit.id, 'claude-opus-5-5') assert.equal(hit.source, 'override') @@ -123,19 +123,19 @@ test('an override on the canonical id also prices its dated snapshot', async () jsonResponse({ anthropic: { models: { [dated]: { cost: { input: 1, output: 5, cache_read: 0.1 } } } } }), }) assert.equal(resolvePrice(dated).source, 'models.dev') - assert.equal(setPriceOverride('anthropic/claude-haiku-4-5', FULL).ok, true) + assert.equal((await setPriceOverride('anthropic/claude-haiku-4-5', FULL)).ok, true) const hit = resolvePrice(dated) assert.equal(hit.source, 'override') assert.equal(hit.price.input, FULL.input) }) -test('partial override is rejected', () => { - const missingRead = setPriceOverride('anthropic/claude-sonnet-5-5', { input: 1, output: 2 }) +test('partial override is rejected', async () => { + const missingRead = await setPriceOverride('anthropic/claude-sonnet-5-5', { input: 1, output: 2 }) assert.equal(missingRead.ok, false) assert.match(missingRead.error, /cache_read/) assert.equal(resolvePrice('claude-sonnet-5-5').source, 'builtin') - const missingInput = setPriceOverride('openai/gpt-5.5', { output: 1, cache_read: 0 }) + const missingInput = await setPriceOverride('openai/gpt-5.5', { output: 1, cache_read: 0 }) assert.equal(missingInput.ok, false) assert.match(missingInput.error, /input/) }) @@ -259,10 +259,10 @@ test('validatePrice fills optional cache writes', () => { assert.equal(o.price.cache_1h, 0) }) -test('saving an override invalidates the memoized catalog', () => { +test('saving an override invalidates the memoized catalog', async () => { const before = listPriceCatalog().find((e) => e.id === 'claude-sonnet-5-5') assert.equal(before.override, null) - assert.equal(setPriceOverride('anthropic/claude-sonnet-5-5', FULL).ok, true) + assert.equal((await setPriceOverride('anthropic/claude-sonnet-5-5', FULL)).ok, true) const after = listPriceCatalog().find((e) => e.id === 'claude-sonnet-5-5') assert.equal(after.source, 'override') assert.equal(after.override.input, 9) diff --git a/test/unit/probe-status.test.mjs b/test/unit/probe-status.test.mjs index 0d1228473..24cc7a0a8 100644 --- a/test/unit/probe-status.test.mjs +++ b/test/unit/probe-status.test.mjs @@ -16,7 +16,7 @@ function usageCache(probe) { } } -function fixture(t, { mode = 'setup-token', headers = true, failed = false } = {}) { +async function fixture(t, { mode = 'setup-token', headers = true, failed = false } = {}) { const project = fs.mkdtempSync(path.join(os.tmpdir(), 'probe-card-')) fs.mkdirSync(path.join(project, 'vms')) fs.writeFileSync( @@ -29,8 +29,8 @@ function fixture(t, { mode = 'setup-token', headers = true, failed = false } = { }), ) const dataDir = path.join(project, 'data') - let db = createDatabase({ dataDir }) - let quota = new AccountQuota({ db }) + let db = await createDatabase({ dataDir }) + let quota = await new AccountQuota({ db }).init() const acc = quota.ensure({ account_id: 'account-test', vm_id: 'vm-02', max_concurrency: 20 }) acc.unified.headers = headers ? { '5h': { utilization: 0.15, status: 'allowed' }, sampled_at: '2026-09-20T00:00:00Z' } @@ -45,18 +45,18 @@ function fixture(t, { mode = 'setup-token', headers = true, failed = false } = { acc.unified.usage_rate_limited_until = '2099-01-01T00:00:00Z' } quota.repo.save(acc) - t.after(() => { - db.close() + t.after(async () => { + await db.close() fs.rmSync(project, { recursive: true, force: true }) }) const args = { cfg: { paths: { project }, rewrite: {} }, id: 'vm-02', force: true, hop: true } return { quota, args, - reopen() { - db.close() - db = createDatabase({ dataDir }) - quota = new AccountQuota({ db }) + async reopen() { + await db.close() + db = await createDatabase({ dataDir }) + quota = await new AccountQuota({ db }).init() return quota }, async detail(q = quota) { @@ -66,7 +66,7 @@ function fixture(t, { mode = 'setup-token', headers = true, failed = false } = { } test('Setup Token probe persists the card check time/source across DB reopen and detail reload', async (t) => { - const f = fixture(t) + const f = await fixture(t) const probe = { ok: true, source: 'official-cc-usage', @@ -77,7 +77,7 @@ test('Setup Token probe persists the card check time/source across DB reopen and const result = (await buildProbeOne({ ...f.args, accountQuota: f.quota, usageCache: usageCache(probe) })).data assert.equal(result.ok, true) assert.equal(result.source, 'official-cc-usage') - const detail = await f.detail(f.reopen()) + const detail = await f.detail(await f.reopen()) assert.equal(detail.account.last_probe_check?.at, result.probed_at) assert.equal(detail.account.last_probe_check.source, result.source) assert.equal(detail.account.last_probe_check.via, 'worker') @@ -86,7 +86,7 @@ test('Setup Token probe persists the card check time/source across DB reopen and }) test('cached-header check preserves active-probe failures, backoff, quota and counters', async (t) => { - const f = fixture(t, { failed: true }) + const f = await fixture(t, { failed: true }) const before = f.quota.repo.get('account-test') await buildProbeOne({ ...f.args, @@ -109,7 +109,7 @@ test('cached-header check preserves active-probe failures, backoff, quota and co }) test('no header samples reports unavailable and persists the attempt without fake success', async (t) => { - const f = fixture(t, { headers: false }) + const f = await fixture(t, { headers: false }) const probe = { ok: false, source: 'official-cc-usage', @@ -125,13 +125,13 @@ test('no header samples reports unavailable and persists the attempt without fak assert.equal(check.ok, false) assert.equal(check.data_at, result.probed_at) assert.equal(f.quota.repo.get('account-test').unified.last_probe?.ok, false) - const detail = await f.detail(f.reopen()) + const detail = await f.detail(await f.reopen()) assert.equal(detail.vm.account_tier, 'pro') assert.equal(detail.vm.usage_has_fable, null) }) test('Setup Token Fable hop does not classify the plan without complete official usage', async (t) => { - const f = fixture(t) + const f = await fixture(t) f.quota.setAccountTier('account-test', 'pro') const result = ( await buildProbeOne({ @@ -148,12 +148,12 @@ test('Setup Token Fable hop does not classify the plan without complete official assert.equal(f.quota.repo.get('account-test').unified.usage_has_fable, undefined) assert.equal(f.quota.repo.get('account-test').unified.fable.ok, false) assert.equal(f.quota.repo.get('account-test').unified.fable.plan_denied, false) - const detail = await f.detail(f.reopen()) + const detail = await f.detail(await f.reopen()) assert.equal(detail.vm.account_tier, 'pro') }) test('Setup Token usage scope failure reports auth scope and preserves history', async (t) => { - const f = fixture(t) + const f = await fixture(t) f.quota.setAccountTier('account-test', 'max', { source: 'usage' }) let fableCalled = false const result = ( @@ -189,7 +189,7 @@ test('Setup Token usage scope failure reports auth scope and preserves history', }) test('official OAuth probe still ingests real usage and exposes its actual result to the card', async (t) => { - const f = fixture(t, { mode: 'oauth' }) + const f = await fixture(t, { mode: 'oauth' }) const probe = { ok: false, source: 'official-cc-usage', @@ -210,7 +210,7 @@ test('official OAuth probe still ingests real usage and exposes its actual resul }) ).data assert.equal(result.ok, false) - const detail = await f.detail(f.reopen()) + const detail = await f.detail(await f.reopen()) assert.equal(detail.account.last_probe_check?.at, probe.probed_at) assert.equal(detail.account.last_probe_check.ok, false) assert.equal(detail.account.last_probe_check.error, 'probe_failed') @@ -227,7 +227,7 @@ test('probe errors remain text across the response, persisted checks and detail ] for (const { expected, ...errors } of cases) { await t.test(JSON.stringify(errors), async (t) => { - const f = fixture(t, { mode: 'oauth' }) + const f = await fixture(t, { mode: 'oauth' }) const result = ( await buildProbeOne({ ...f.args, @@ -244,7 +244,7 @@ test('probe errors remain text across the response, persisted checks and detail ).data assert.equal(result.ok, false) assert.equal(result.error, expected) - const detail = await f.detail(f.reopen()) + const detail = await f.detail(await f.reopen()) for (const record of [detail.vm, detail.account]) { assert.equal(record.last_probe.ok, false) assert.equal(record.last_probe.error, expected) @@ -256,7 +256,7 @@ test('probe errors remain text across the response, persisted checks and detail }) test('confirmed usage revoke parks the VM and reports revoked status', async (t) => { - const f = fixture(t, { mode: 'oauth' }) + const f = await fixture(t, { mode: 'oauth' }) const result = ( await buildProbeOne({ ...f.args, @@ -275,13 +275,13 @@ test('confirmed usage revoke parks the VM and reports revoked status', async (t) const vm = JSON.parse(fs.readFileSync(path.join(f.args.cfg.paths.project, 'vms/vm-02.json'), 'utf8')) assert.equal(vm.schedulable, false) assert.equal(vm.schedule_disabled_reason, 'oauth_revoked') - const detail = await f.detail(f.reopen()) + const detail = await f.detail(await f.reopen()) assert.equal(detail.vm.cred_status.key, 'bad') assert.equal(detail.vm.cred_status.text, 'revoke') }) test('complete Pro usage replaces stale Max windows in persisted panel data', async (t) => { - const f = fixture(t, { mode: 'oauth' }) + const f = await fixture(t, { mode: 'oauth' }) f.quota.ingestOAuthUsage('account-test', { ok: true, usage_status: 200, @@ -307,7 +307,7 @@ test('complete Pro usage replaces stale Max windows in persisted panel data', as ).data assert.equal(result.account_tier, 'pro') assert.equal(result.quota.account_tier, 'pro') - const detail = await f.detail(f.reopen()) + const detail = await f.detail(await f.reopen()) assert.equal(detail.vm.account_tier, 'pro') assert.equal(detail.vm.usage_has_fable, false) assert.equal(detail.vm.utilization_7d_oi, null) diff --git a/test/unit/proxy-cli-enabled.test.mjs b/test/unit/proxy-cli-enabled.test.mjs index 164f675b8..42906a772 100644 --- a/test/unit/proxy-cli-enabled.test.mjs +++ b/test/unit/proxy-cli-enabled.test.mjs @@ -166,11 +166,11 @@ test('OAuth commit after bind does not reset proxy_cli_enabled', async () => { test('package import with a proxy keeps proxy_cli_enabled even if the spec says false', async (t) => { const root = tmpRoot() t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const db = createDatabase({ dbPath: ':memory:' }) - const pool = new ProxyPool({ db }) - t.after(() => { + const db = await createDatabase({ dbPath: ':memory:' }) + const pool = await new ProxyPool({ db }).init() + t.after(async () => { pool.stopScheduler() - db.close() + await db.close() }) const doc = buildVmPackage({ vm: { diff --git a/test/unit/proxy-geo.test.mjs b/test/unit/proxy-geo.test.mjs index 32ac8c0e3..e6fe1353f 100644 --- a/test/unit/proxy-geo.test.mjs +++ b/test/unit/proxy-geo.test.mjs @@ -18,8 +18,8 @@ function tmpDir(prefix = 'kin-geo-') { return fs.mkdtempSync(path.join(os.tmpdir(), prefix)) } -function makePool(geoLookup) { - const pool = new ProxyPool({ dataDir: tmpDir(), geoLookup }) +async function makePool(geoLookup) { + const pool = await new ProxyPool({ dataDir: tmpDir(), geoLookup }).init() pool.stopScheduler() return pool } @@ -138,7 +138,7 @@ test('geo lookup surfaces transport and HTTP failures instead of throwing', asyn test('detectGeo caches per proxy and publishes location on the snapshot', async () => { let calls = 0 - const pool = makePool(async () => { + const pool = await makePool(async () => { calls += 1 return { ok: true, geo: { ip: '203.0.113.9', country: 'Japan', country_code: 'JP', timezone: 'Asia/Tokyo' } } }) @@ -170,7 +170,7 @@ test('detectGeo caches per proxy and publishes location on the snapshot', async }) test('detectGeo keeps IPv4 and IPv6 results separate and records IPv6 unavailability', async () => { - const pool = makePool(async () => ({ + const pool = await makePool(async () => ({ ok: true, geo: { ip: '203.0.113.9', country: 'Japan', timezone: 'Asia/Tokyo' }, })) @@ -184,7 +184,7 @@ test('detectGeo keeps IPv4 and IPv6 results separate and records IPv6 unavailabi }) test('detectGeo rejects IPv4 masquerading as IPv6 on the v6 path', async () => { - const pool = makePool(async () => ({ + const pool = await makePool(async () => ({ ok: true, geo: { ip: '203.0.113.9', country: 'Japan', timezone: 'Asia/Tokyo' }, })) @@ -199,7 +199,7 @@ test('detectGeo rejects IPv4 masquerading as IPv6 on the v6 path', async () => { test('a failed lookup keeps the last known location and records the error', async () => { let ok = true - const pool = makePool(async () => + const pool = await makePool(async () => ok ? { ok: true, geo: { ip: '203.0.113.9', country: 'Japan', timezone: 'Asia/Tokyo' } } : { ok: false, error: 'geo_http_500' }, @@ -216,7 +216,7 @@ test('a failed lookup keeps the last known location and records the error', asyn }) test('binding a proxy moves the slot onto the exit node timezone', async () => { - const pool = makePool(async () => ({ + const pool = await makePool(async () => ({ ok: true, geo: { ip: '203.0.113.9', country: 'Japan', timezone: 'Asia/Tokyo' }, })) @@ -238,7 +238,7 @@ test('binding a proxy moves the slot onto the exit node timezone', async () => { }) test('a hand-pinned slot timezone survives a proxy bind unless forced', async () => { - const pool = makePool(async () => ({ ok: true, geo: { timezone: 'Asia/Tokyo' } })) + const pool = await makePool(async () => ({ ok: true, geo: { timezone: 'Asia/Tokyo' } })) const id = pool.importLines('1.2.3.4:1080').items[0].id pool.bind(id, 'vm-02') const root = seedProject({ @@ -259,7 +259,7 @@ test('a hand-pinned slot timezone survives a proxy bind unless forced', async () }) test('follow_proxy_timezone=false stops the bind-time default', async () => { - const pool = makePool(async () => ({ ok: true, geo: { timezone: 'Asia/Tokyo' } })) + const pool = await makePool(async () => ({ ok: true, geo: { timezone: 'Asia/Tokyo' } })) pool.updateConfig({ follow_proxy_timezone: false }) // updateConfig restarts the probe loop; left running it keeps the test // process alive forever. @@ -275,7 +275,7 @@ test('follow_proxy_timezone=false stops the bind-time default', async () => { }) test('an unresolvable exit zone leaves the slot timezone alone', async () => { - const pool = makePool(async () => ({ ok: true, geo: { ip: '203.0.113.9', country: 'Japan', timezone: null } })) + const pool = await makePool(async () => ({ ok: true, geo: { ip: '203.0.113.9', country: 'Japan', timezone: null } })) const id = pool.importLines('1.2.3.4:1080').items[0].id pool.bind(id, 'vm-04') const root = seedProject({ id: 'vm-04', timezone: 'America/Denver', proxy: { id } }) @@ -287,7 +287,7 @@ test('an unresolvable exit zone leaves the slot timezone alone', async () => { test('a node slot on local egress follows its node exit, not the control plane row', async (t) => { const lookups = [] - const pool = makePool(async (url) => { + const pool = await makePool(async (url) => { lookups.push(url) // '' = the control plane's own route (px-local row); a SOCKS URL = the node forwarder. return { ok: true, geo: { timezone: url ? 'Europe/Berlin' : 'Asia/Tokyo' } } diff --git a/test/unit/proxy-ipv6.test.mjs b/test/unit/proxy-ipv6.test.mjs index fb5565c99..cabeb9cf8 100644 --- a/test/unit/proxy-ipv6.test.mjs +++ b/test/unit/proxy-ipv6.test.mjs @@ -10,18 +10,18 @@ import { resolveImportProxy } from '../../src/lib/vm/proxy-resolve.mjs' import { evaluateSlotGate } from '../../src/lib/pool/schedule-eligibility.mjs' import { isCodexSlotReady, pickCodexSlots } from '../../src/lib/pool/codex-slot-pool.mjs' -function makePool(t) { - const pool = new ProxyPool({ db: openDatabase({ dbPath: ':memory:' }) }) +async function makePool(t) { + const pool = await new ProxyPool({ db: await openDatabase({ dbPath: ':memory:' }) }).init() pool.updateConfig({ enabled: false }) - t.after(() => { + t.after(async () => { pool.stopScheduler() - closeDatabase() + await closeDatabase() }) return pool } -test('IPv6 aliases normalize and deduplicate after loading bracketed historical hosts', (t) => { - const pool = makePool(t) +test('IPv6 aliases normalize and deduplicate after loading bracketed historical hosts', async (t) => { + const pool = await makePool(t) const first = pool.importLines('socks5h://a%3Ab:p%40%3A%2F%25@[2001:0db8:0:0:0:0:0:1]:1080') const id = first.items[0].id const full = pool.getProxyByIdWithAuth(id) @@ -30,8 +30,9 @@ test('IPv6 aliases normalize and deduplicate after loading bracketed historical assert.equal(full.password, 'p@:/%') assert.equal(full.url, 'socks5://a%3Ab:p%40%3A%2F%25@[2001:db8::1]:1080') assert.equal(parseSocks5Line(full.url).password, full.password) - pool.db.prepare('UPDATE proxies SET host = ? WHERE id = ?').run('[2001:0db8::1]', id) - pool.reload() + await pool.flush() + await pool.db.prepare('UPDATE proxies SET host = $1 WHERE id = $2').run('[2001:0db8::1]', id) + await pool.reload() assert.equal(pool.state.proxies[0].host, '2001:db8::1') const result = pool.importLines('', { fields: [{ host: '2001:db8::1', port: 1080, username: 'a:b', password: 'p@:/%' }], @@ -59,7 +60,7 @@ test('bracketed IPv6 input works without changing hostname vendor credentials', }) test('malformed historical IPv6 hosts cannot silently probe localhost', async (t) => { - const pool = makePool(t) + const pool = await makePool(t) let connections = 0 const sockets = new Set() const server = net.createServer((socket) => { @@ -77,15 +78,16 @@ test('malformed historical IPv6 hosts cannot silently probe localhost', async (t const { items: [proxy], } = pool.importLines(`127.0.0.1:${server.address().port}`) - pool.db.prepare('UPDATE proxies SET host = ? WHERE id = ?').run('[::1', proxy.id) - pool.reload() + await pool.flush() + await pool.db.prepare('UPDATE proxies SET host = $1 WHERE id = $2').run('[::1', proxy.id) + await pool.reload() const result = await pool.probeById(proxy.id) assert.equal(result.probe.error, 'invalid_proxy_host') assert.equal(connections, 0) }) test('IPv6 policy survives reload and preserves binding and health instead of reallocating', async (t) => { - const pool = makePool(t) + const pool = await makePool(t) const { items: [ipv6], } = pool.importLines('[2001:db8::1]:1080\n127.0.0.1:1080') @@ -99,7 +101,7 @@ test('IPv6 policy survives reload and preserves binding and health instead of re Object.assign(proxy, { status: 'ok', latency_ms: 20, last_probe_at: '2026-10-01T00:00:00Z', consecutive_failures: 2 }) const history = { status: proxy.status, last_probe_at: proxy.last_probe_at, failures: proxy.consecutive_failures } pool.updateConfig({ ipv6_enabled: false, disconnect_on_error: true }) - pool.reload() + await pool.reload() assert.equal(pool.snapshot().config.ipv6_enabled, false) assert.equal(pool.getProxyForVm('vm-01'), null) assert.equal(pool.ensureBoundToVm('vm-01', ipv6.id), null) @@ -128,13 +130,13 @@ test('IPv6 policy survives reload and preserves binding and health instead of re assert.equal(isCodexSlotReady(codex), false) assert.equal(pickCodexSlots([codex], { pin: codex.id }).error, 'ipv6_disabled') pool.updateConfig({ ipv6_enabled: true }) - pool.reload() + await pool.reload() assert.equal(pool.getProxyForVm('vm-01').id, ipv6.id) assert.equal(boundProxyUrl(vm.proxy), 'socks5h://[2001:db8::1]:1080') }) test('URL-imported IPv6 probes connect only when enabled and closing cancels in-flight sockets', async (t) => { - const pool = makePool(t) + const pool = await makePool(t) const sockets = new Set() let connections = 0 const server = net.createServer((socket) => { diff --git a/test/unit/proxy-pool-bind.test.mjs b/test/unit/proxy-pool-bind.test.mjs index 3afb27ba5..3cdc6db97 100644 --- a/test/unit/proxy-pool-bind.test.mjs +++ b/test/unit/proxy-pool-bind.test.mjs @@ -10,16 +10,16 @@ function tmpDir(prefix = 'kin-px-') { return fs.mkdtempSync(path.join(os.tmpdir(), prefix)) } -function makePool() { - const pool = new ProxyPool({ dataDir: tmpDir() }) +async function makePool() { + const pool = await new ProxyPool({ dataDir: tmpDir() }).init() pool.stopScheduler() return pool } -test('proxy pool persists custom dns_primary and rejects invalid URLs without changing it', (t) => { +test('proxy pool persists custom dns_primary and rejects invalid URLs without changing it', async (t) => { const dir = tmpDir() t.after(() => fs.rmSync(dir, { recursive: true, force: true })) - const pool = new ProxyPool({ dataDir: dir }) + const pool = await new ProxyPool({ dataDir: dir }).init() pool.stopScheduler() assert.equal(pool.updateConfig({ dns_primary: '8.8.8.8:53' }).ok, true) const primary = 'https://cloudflare-dns.com:8443/dns-query?key=a%2Cb' @@ -37,15 +37,15 @@ test('proxy pool persists custom dns_primary and rejects invalid URLs without ch assert.equal(result.error, 'invalid_dns_primary', invalid) assert.equal(pool.snapshot().config.dns_primary, primary) } - const reopened = new ProxyPool({ dataDir: dir }) + const reopened = await new ProxyPool({ dataDir: dir }).init() reopened.stopScheduler() assert.equal(reopened.snapshot().config.dns_primary, primary) }) -test('DNS type suppression persists both toggle states and rejects non-booleans', (t) => { +test('DNS type suppression persists both toggle states and rejects non-booleans', async (t) => { const dir = tmpDir() t.after(() => fs.rmSync(dir, { recursive: true, force: true })) - const pool = new ProxyPool({ dataDir: dir }) + const pool = await new ProxyPool({ dataDir: dir }).init() pool.stopScheduler() assert.equal(pool.snapshot().config.dns_disable_svcb_https, false) assert.equal(pool.updateConfig({ dns_disable_svcb_https: true }).ok, true) @@ -55,11 +55,11 @@ test('DNS type suppression persists both toggle states and rejects non-booleans' assert.equal(pool.snapshot().config.dns_disable_svcb_https, true) assert.equal(pool.snapshot().config.dns_primary, 'auto') } - const reopened = new ProxyPool({ dataDir: dir }) + const reopened = await new ProxyPool({ dataDir: dir }).init() reopened.stopScheduler() assert.equal(reopened.snapshot().config.dns_disable_svcb_https, true) assert.equal(reopened.updateConfig({ dns_disable_svcb_https: false }).ok, true) - const disabled = new ProxyPool({ dataDir: dir }) + const disabled = await new ProxyPool({ dataDir: dir }).init() disabled.stopScheduler() assert.equal(disabled.snapshot().config.dns_disable_svcb_https, false) }) @@ -74,8 +74,8 @@ test('clampBindLimit defaults to 5 and stays in 1..32', () => { assert.equal(clampBindLimit('x'), MAX_VMS_PER_PROXY) }) -test('bind_limit defaults to 5 and is configurable', () => { - const pool = makePool() +test('bind_limit defaults to 5 and is configurable', async () => { + const pool = await makePool() pool.importLines('10.0.0.1:1080') const id = pool.snapshot().proxies[0].id assert.equal(pool.bindLimit(), 5) @@ -106,8 +106,8 @@ test('bind_limit defaults to 5 and is configurable', () => { assert.equal(pool.bind(id, 'vm-03').max, 2) }) -test('remove deletes a proxy and updateConfig rejects bad bind_limit', () => { - const pool = makePool() +test('remove deletes a proxy and updateConfig rejects bad bind_limit', async () => { + const pool = await makePool() pool.importLines('10.0.0.2:1080') const id = pool.snapshot().proxies[0].id const removed = pool.remove(id) @@ -131,10 +131,10 @@ async function listenSocks() { test('probeOne marks egress_down after SOCKS greeting if helper is dead', async () => { const socks = await listenSocks() - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir: tmpDir(), egressCheck: () => ({ ok: false, reason: 'not_running' }), - }) + }).init() pool.stopScheduler() try { pool.importLines(`${socks.host}:${socks.port}`) @@ -150,13 +150,13 @@ test('probeOne marks egress_down after SOCKS greeting if helper is dead', async test('probeOne repairs egress then passes', async () => { const socks = await listenSocks() let healthy = false - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir: tmpDir(), egressCheck: () => ({ ok: healthy }), repairEgress: () => { healthy = true }, - }) + }).init() pool.stopScheduler() try { pool.importLines(`${socks.host}:${socks.port}`) @@ -173,14 +173,14 @@ test('startScheduler immediately repairs enabled egress', async () => { const socks = await listenSocks() let healthy = false let repairs = 0 - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir: tmpDir(), egressCheck: () => ({ ok: healthy, reason: healthy ? null : 'not_running' }), repairEgress: () => { repairs += 1 healthy = true }, - }) + }).init() try { pool.importLines(`${socks.host}:${socks.port}`) pool.startScheduler() @@ -196,10 +196,10 @@ test('startScheduler immediately repairs enabled egress', async () => { test('egress_down does not mark a live SOCKS proxy dead', async () => { const socks = await listenSocks() - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir: tmpDir(), egressCheck: () => ({ ok: false, reason: 'not_running' }), - }) + }).init() pool.stopScheduler() try { pool.importLines(`${socks.host}:${socks.port}`) @@ -220,7 +220,7 @@ test('TCP connect then close is not a SOCKS death', async () => { const server = net.createServer((socket) => socket.destroy()) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const addr = server.address() - const pool = new ProxyPool({ dataDir: tmpDir() }) + const pool = await new ProxyPool({ dataDir: tmpDir() }).init() pool.stopScheduler() try { pool.importLines(`${addr.address}:${addr.port}`) @@ -234,7 +234,7 @@ test('TCP connect then close is not a SOCKS death', async () => { test('probeAll resurrects a probe-killed proxy', async () => { const socks = await listenSocks() - const pool = new ProxyPool({ dataDir: tmpDir() }) + const pool = await new ProxyPool({ dataDir: tmpDir() }).init() pool.stopScheduler() try { pool.importLines(`${socks.host}:${socks.port}`) @@ -253,8 +253,8 @@ test('probeAll resurrects a probe-killed proxy', async () => { } }) -test('ensureLocal adds a single local egress row', () => { - const pool = makePool() +test('ensureLocal adds a single local egress row', async () => { + const pool = await makePool() const first = pool.ensureLocal() const second = pool.ensureLocal() assert.equal(first.ok, true) @@ -272,11 +272,11 @@ test('ensureLocal adds a single local egress row', () => { test('local egress probe stays direct when kin-egress is down', async () => { const disabled = [] - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir: tmpDir(), egressCheck: () => ({ ok: false, reason: 'local_network_missing' }), onDisableVm: (vmId) => disabled.push(vmId), - }) + }).init() pool.stopScheduler() pool.ensureLocal() pool.bind('px-local', 'vm-01') diff --git a/test/unit/proxy-pool-import.test.mjs b/test/unit/proxy-pool-import.test.mjs index 57750094b..fb6d6c0f3 100644 --- a/test/unit/proxy-pool-import.test.mjs +++ b/test/unit/proxy-pool-import.test.mjs @@ -3,25 +3,25 @@ import assert from 'node:assert/strict' import { ProxyPool } from '../../src/lib/vm/proxy-pool.mjs' import { createDatabase } from '../../src/lib/db/database.mjs' -function makePool(t) { - const db = createDatabase({ dbPath: ':memory:' }) - const pool = new ProxyPool({ db }) - t.after(() => { +async function makePool(t) { + const db = await createDatabase({ dbPath: ':memory:' }) + const pool = await new ProxyPool({ db }).init() + t.after(async () => { pool.stopScheduler() - db.close() + await db.close() }) return pool } -test('import: different passwords at the same endpoint are distinct within a batch and after reload', (t) => { - const pool = makePool(t) +test('import: different passwords at the same endpoint are distinct within a batch and after reload', async (t) => { + const pool = await makePool(t) const first = pool.importLines('1.2.3.4:1080:alice:first\n1.2.3.4:1080:alice:second\n1.2.3.4:1080:alice:first') assert.equal(first.added, 2) assert.equal(first.skipped, 1) assert.equal(first.skip_details[0].reason, 'duplicate') assert.notEqual(first.items[0].id, first.items[1].id) - pool.reload() + await pool.reload() assert.deepEqual(pool.state.proxies.map((p) => p.password).sort(), ['first', 'second']) const next = pool.importLines('1.2.3.4:1080:alice:second\n1.2.3.4:1080:alice:third') assert.equal(next.added, 1) @@ -29,8 +29,8 @@ test('import: different passwords at the same endpoint are distinct within a bat assert.equal(pool.state.proxies.length, 3) }) -test('import: structured credentials containing colons do not collide, equivalent URLs still deduplicate', (t) => { - const pool = makePool(t) +test('import: structured credentials containing colons do not collide, equivalent URLs still deduplicate', async (t) => { + const pool = await makePool(t) const result = pool.importLines('', { fields: [ { host: '1.2.3.4', port: 1080, username: 'alice:region', password: 'secret' }, diff --git a/test/unit/proxy-pool-update.test.mjs b/test/unit/proxy-pool-update.test.mjs index 2e762ec64..a0cc5579c 100644 --- a/test/unit/proxy-pool-update.test.mjs +++ b/test/unit/proxy-pool-update.test.mjs @@ -5,8 +5,8 @@ import os from 'node:os' import path from 'node:path' import { ProxyPool } from '../../src/lib/vm/proxy-pool.mjs' -function makePool() { - const pool = new ProxyPool({ dataDir: fs.mkdtempSync(path.join(os.tmpdir(), 'kin-px-upd-')) }) +async function makePool() { + const pool = await new ProxyPool({ dataDir: fs.mkdtempSync(path.join(os.tmpdir(), 'kin-px-upd-')) }).init() pool.stopScheduler() return pool } @@ -15,8 +15,8 @@ function seed(pool, line = 'socks5://alice:s3cret@1.2.3.4:1080') { return pool.state.proxies[0] } -test('update: 只改 port，账密保留；raw 明文密码被清除', () => { - const pool = makePool() +test('update: 只改 port，账密保留；raw 明文密码被清除', async () => { + const pool = await makePool() const p = seed(pool) assert.match(p.raw, /s3cret/, '前提：导入后 raw 含明文密码') const r = pool.update(p.id, { port: 1081 }) @@ -28,16 +28,16 @@ test('update: 只改 port，账密保留；raw 明文密码被清除', () => { assert.doesNotMatch(p.raw, /s3cret/, 'raw 不应再含明文密码') }) -test('update: username 传空串清除账密（password 一并清）', () => { - const pool = makePool() +test('update: username 传空串清除账密（password 一并清）', async () => { + const pool = await makePool() const p = seed(pool) assert.equal(pool.update(p.id, { username: '' }).ok, true) assert.equal(p.username, null) assert.equal(p.password, null) }) -test('update: 不传的字段保持不动', () => { - const pool = makePool() +test('update: 不传的字段保持不动', async () => { + const pool = await makePool() const p = seed(pool) pool.update(p.id, { password: 'newpw' }) assert.equal(p.host, '1.2.3.4') @@ -46,8 +46,8 @@ test('update: 不传的字段保持不动', () => { assert.equal(p.password, 'newpw') }) -test('update: 只给密码不给用户名被拒，不静默存成「仍无账密」', () => { - const pool = makePool() +test('update: 只给密码不给用户名被拒，不静默存成「仍无账密」', async () => { + const pool = await makePool() pool.importLines('9.9.9.9:1080') const p = pool.state.proxies[0] assert.equal(p.username, null, '前提：这条本来没有账密') @@ -57,16 +57,16 @@ test('update: 只给密码不给用户名被拒，不静默存成「仍无账密 assert.equal(pool.update(p.id, { username: '', password: 'x' }).error, 'password_without_username') }) -test('update: 只换用户名时沿用原密码（不传的键就是不动）', () => { - const pool = makePool() +test('update: 只换用户名时沿用原密码（不传的键就是不动）', async () => { + const pool = await makePool() const p = seed(pool) assert.equal(pool.update(p.id, { username: 'bob' }).ok, true) assert.equal(p.username, 'bob') assert.equal(p.password, 's3cret', 'password 没传，按语义保持原值') }) -test('update: 非法值与空 patch 被拒，且不改动原记录', () => { - const pool = makePool() +test('update: 非法值与空 patch 被拒，且不改动原记录', async () => { + const pool = await makePool() const p = seed(pool) assert.equal(pool.update(p.id, { port: 99999 }).error, 'invalid_proxy') assert.equal(pool.update(p.id, { host: '' }).error, 'invalid_proxy') @@ -75,8 +75,8 @@ test('update: 非法值与空 patch 被拒，且不改动原记录', () => { assert.equal(p.port, 1080, '被拒后原值不变') }) -test('update: 响应里的 publicProxy 不含账密', () => { - const pool = makePool() +test('update: 响应里的 publicProxy 不含账密', async () => { + const pool = await makePool() const p = seed(pool) const r = pool.update(p.id, { port: 1081 }) const s = JSON.stringify(r.proxy) @@ -84,8 +84,8 @@ test('update: 响应里的 publicProxy 不含账密', () => { assert.equal(r.proxy.has_auth, true, '但要能看出有账密') }) -test('getProxyByIdWithAuth: 拼出完整 URI，特殊字符转义', () => { - const pool = makePool() +test('getProxyByIdWithAuth: 拼出完整 URI，特殊字符转义', async () => { + const pool = await makePool() const p = seed(pool) assert.equal(pool.getProxyByIdWithAuth(p.id).url, 'socks5://alice:s3cret@1.2.3.4:1080') pool.update(p.id, { username: 'u@x', password: 'p:w/d' }) @@ -94,8 +94,8 @@ test('getProxyByIdWithAuth: 拼出完整 URI，特殊字符转义', () => { assert.equal(pool.getProxyByIdWithAuth(p.id).url, 'socks5://1.2.3.4:1080', '无账密时不带 @') }) -test('getProxyByIdWithAuth: 禁用/失效的代理仍可读回（getProxyForVm 做不到）', () => { - const pool = makePool() +test('getProxyByIdWithAuth: 禁用/失效的代理仍可读回（getProxyForVm 做不到）', async () => { + const pool = await makePool() const p = seed(pool) pool.bind(p.id, 'vm-01') assert.ok(pool.getProxyForVm('vm-01'), '前提：启用时 getProxyForVm 能取到') @@ -104,28 +104,28 @@ test('getProxyByIdWithAuth: 禁用/失效的代理仍可读回（getProxyForVm assert.equal(pool.getProxyByIdWithAuth(p.id).url, 'socks5://alice:s3cret@1.2.3.4:1080', '新方法不受影响') }) -test('snapshot 回归：列表接口永不返回账密', () => { - const pool = makePool() +test('snapshot 回归：列表接口永不返回账密', async () => { + const pool = await makePool() seed(pool) const s = JSON.stringify(pool.snapshot()) assert.doesNotMatch(s, /alice|s3cret|socks5:\/\//, 'GET /proxies 的载荷不得含账密') }) -test('update: 只改代理名称不动连接字段，也不要求重载槽位；重开后名称仍在', () => { - const pool = makePool() +test('update: 只改代理名称不动连接字段，也不要求重载槽位；重开后名称仍在', async () => { + const pool = await makePool() const p = seed(pool) const r = pool.update(p.id, { label: ' 东京-2 号机 ' }) assert.equal(r.ok, true) assert.equal(r.connection_changed, false) assert.equal(r.proxy.label, '东京-2 号机') assert.equal(p.raw, 'socks5://alice:s3cret@1.2.3.4:1080', '只改名称不应改写 raw') - pool.load() + await pool.load() assert.equal(pool.state.proxies[0].label, '东京-2 号机') assert.equal(pool.snapshot().proxies[0].label, '东京-2 号机') }) -test('update: 代理名称传空串清除；超长或非字符串被拒', () => { - const pool = makePool() +test('update: 代理名称传空串清除；超长或非字符串被拒', async () => { + const pool = await makePool() const p = seed(pool) pool.update(p.id, { label: 'a' }) assert.equal(pool.update(p.id, { label: '' }).proxy.label, null) @@ -133,8 +133,8 @@ test('update: 代理名称传空串清除；超长或非字符串被拒', () => assert.equal(pool.update(p.id, { label: 123 }).error, 'invalid_label') }) -test('update: 连接字段与名称同改时报告 connection_changed', () => { - const pool = makePool() +test('update: 连接字段与名称同改时报告 connection_changed', async () => { + const pool = await makePool() const p = seed(pool) const r = pool.update(p.id, { port: 1082, label: 'box' }) assert.equal(r.connection_changed, true) @@ -142,8 +142,8 @@ test('update: 连接字段与名称同改时报告 connection_changed', () => { assert.equal(p.label, 'box') }) -test('labelOf 只给名称，不带账密；未知代理返回 null', () => { - const pool = makePool() +test('labelOf 只给名称，不带账密；未知代理返回 null', async () => { + const pool = await makePool() const p = seed(pool) assert.equal(pool.labelOf(p.id), null) pool.update(p.id, { label: 'tw-box' }) diff --git a/test/unit/proxy-resolve.test.mjs b/test/unit/proxy-resolve.test.mjs index a70b39682..a7888755f 100644 --- a/test/unit/proxy-resolve.test.mjs +++ b/test/unit/proxy-resolve.test.mjs @@ -12,8 +12,8 @@ function tmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), 'kin-resolve-')) } -test('prefers pool credentials over host-only vm.proxy', () => { - const pool = new ProxyPool({ dataDir: tmpDir() }) +test('prefers pool credentials over host-only vm.proxy', async () => { + const pool = await new ProxyPool({ dataDir: tmpDir() }).init() pool.importLines('socks5://user:s3cret@10.0.0.5:1080') const id = pool.snapshot().proxies[0].id pool.bind(id, 'vm-02') @@ -26,8 +26,8 @@ test('prefers pool credentials over host-only vm.proxy', () => { assert.equal(resolved.proxyUrl, 'socks5h://user:s3cret@10.0.0.5:1080') }) -test('blocks import when bound proxy is fail or dead', () => { - const pool = new ProxyPool({ dataDir: tmpDir() }) +test('blocks import when bound proxy is fail or dead', async () => { + const pool = await new ProxyPool({ dataDir: tmpDir() }).init() pool.importLines('10.0.0.6:1080') const id = pool.snapshot().proxies[0].id pool.bind(id, 'vm-03') diff --git a/test/unit/quota-critical-band.test.mjs b/test/unit/quota-critical-band.test.mjs index ae2a89988..3568d43ad 100644 --- a/test/unit/quota-critical-band.test.mjs +++ b/test/unit/quota-critical-band.test.mjs @@ -28,8 +28,8 @@ function tmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), 'kin-quota-band-')) } -function quota(extra = {}) { - return new AccountQuota({ +async function quota(extra = {}) { + return await new AccountQuota({ dataDir: tmpDir(), config: { quota: { @@ -48,7 +48,7 @@ function quota(extra = {}) { max: { limit_5h: 0.8, limit_7d: 0.8, max_concurrency: 2 }, }, }, - }) + }).init() } const smallRisk = { @@ -77,8 +77,8 @@ function setUtil(q, id, { u5 = null, u7 = null } = {}) { q.ingestHeaders(id, headers) } -test('79% admits a small request and rejects a single large cold with inflight=0', () => { - const q = quota() +test('79% admits a small request and rejects a single large cold with inflight=0', async () => { + const q = await quota() setUtil(q, 'a', { u5: 0.79 }) assert.equal(q.canAccept('a', { requestRisk: smallRisk }).ok, true) const large = q.canAccept('a', { requestRisk: largeColdRisk }) @@ -88,8 +88,8 @@ test('79% admits a small request and rejects a single large cold with inflight=0 assert.equal(q.inflight.get('a') || 0, 0) }) -test('5h and 7d bands are independent', () => { - const q = quota() +test('5h and 7d bands are independent', async () => { + const q = await quota() setUtil(q, 'five', { u5: 0.79, u7: 0.2 }) assert.equal(q.canAccept('five', { requestRisk: largeColdRisk }).reason, 'quota_guard_request_too_large') assert.equal(q.canAccept('five', { requestRisk: smallRisk }).ok, true) @@ -102,8 +102,8 @@ test('5h and 7d bands are independent', () => { assert.equal(q.canAccept('open', { requestRisk: largeColdRisk }).ok, true) }) -test('past the line plus overshoot still blocks ordinary requests', () => { - const q = quota() +test('past the line plus overshoot still blocks ordinary requests', async () => { + const q = await quota() setUtil(q, 'line', { u5: 0.8 }) assert.equal(q.canAccept('line', { requestRisk: smallRisk }).ok, true, 'at the line, inside the 5% overshoot') setUtil(q, 'full', { u5: 0.85 }) @@ -112,8 +112,8 @@ test('past the line plus overshoot still blocks ordinary requests', () => { assert.equal(q.canAccept('over').reason, 'quota_5h_safety') }) -test('failed or cancelled release does not leave a reservation or stale band debt', () => { - const q = quota() +test('failed or cancelled release does not leave a reservation or stale band debt', async () => { + const q = await quota() setUtil(q, 'rel', { u5: 0.79 }) assert.equal(q.tryAcquire('rel', { requestRisk: smallRisk }).ok, true) assert.equal(q.inflight.get('rel'), 1) @@ -123,8 +123,8 @@ test('failed or cancelled release does not leave a reservation or stale band deb assert.equal(q.canAccept('rel', { requestRisk: smallRisk }).ok, true) }) -test('stale reading after an in-band settle blocks the next large request', () => { - const q = quota() +test('stale reading after an in-band settle blocks the next large request', async () => { + const q = await quota() setUtil(q, 'stale', { u5: 0.79 }) assert.equal(q.tryAcquire('stale', { requestRisk: smallRisk }).ok, true) q.noteBandSettled('stale') @@ -135,8 +135,8 @@ test('stale reading after an in-band settle blocks the next large request', () = assert.equal(q.canAccept('stale', { requestRisk: smallRisk }).reason, 'quota_5h_safety') }) -test('a same-percent header after settle keeps the stale large-cold block', () => { - const q = quota() +test('a same-percent header after settle keeps the stale large-cold block', async () => { + const q = await quota() setUtil(q, 'lag', { u5: 0.78 }) assert.equal(q.tryAcquire('lag', { requestRisk: smallRisk }).ok, true) q.noteBandSettled('lag') @@ -145,8 +145,8 @@ test('a same-percent header after settle keeps the stale large-cold block', () = assert.equal(q.canAccept('lag', { requestRisk: largeColdRisk }).reason, 'quota_guard_request_too_large') }) -test('official usage refresh clears stale band debt', () => { - const q = quota() +test('official usage refresh clears stale band debt', async () => { + const q = await quota() setUtil(q, 'off', { u5: 0.79 }) assert.equal(q.tryAcquire('off', { requestRisk: smallRisk }).ok, true) q.noteBandSettled('off') @@ -207,7 +207,7 @@ class RuntimeRepo { test('scheduler switches using requestRisk on the quotaRequest body estimate', async (t) => { const root = schedulerProject() t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const accountQuota = quota() + const accountQuota = await quota() setUtil(accountQuota, 'account-1', { u5: 0.79 }) setUtil(accountQuota, 'account-2', { u5: 0.2 }) const orig = accountQuota.canAccept.bind(accountQuota) @@ -234,7 +234,7 @@ test('scheduler switches using requestRisk on the quotaRequest body estimate', a test('all in-band accounts yield quota_guard_request_too_large', async (t) => { const root = schedulerProject() t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const accountQuota = quota() + const accountQuota = await quota() setUtil(accountQuota, 'account-1', { u5: 0.79 }) setUtil(accountQuota, 'account-2', { u5: 0.77 }) const orig = accountQuota.canAccept.bind(accountQuota) @@ -314,7 +314,7 @@ test('token estimate counts a long prompt as large cold on first turn', () => { assert.equal(risk.largeCold, true) }) -test('output ceiling does not make a short prompt large in the critical band', () => { +test('output ceiling does not make a short prompt large in the critical band', async () => { clearCachePrefixSessions() const account = 'band' const session = 'sess-demo' @@ -358,7 +358,7 @@ test('output ceiling does not make a short prompt large in the critical band', ( assert.equal(omitted.largeCold, false) assert.ok(omitted.uncachedTokens < DEFAULT_LARGE_COLD_UNCACHED_TOKENS) - const q = quota() + const q = await quota() setUtil(q, account, { u5: 0.79 }) const warmReq = { body: turn(2), sessionId: session } assert.equal(q.canAccept(account, { quotaRequest: warmReq }).ok, true) @@ -381,7 +381,7 @@ test('output ceiling does not make a short prompt large in the critical band', ( ) }) -test('stale band still blocks a warm turn whose new user text is itself large', () => { +test('stale band still blocks a warm turn whose new user text is itself large', async () => { clearCachePrefixSessions() const base = { model: 'claude-opus-5-5', @@ -402,7 +402,7 @@ test('stale band still blocks a warm turn whose new user text is itself large', assert.equal(risk.cold, false) assert.equal(risk.largeCold, false) assert.ok(risk.uncachedTokens >= DEFAULT_LARGE_COLD_UNCACHED_TOKENS) - const q = quota() + const q = await quota() setUtil(q, 'band', { u5: 0.79 }) const quotaRequest = { body: extended, sessionId: 'sess' } assert.equal(q.canAccept('band', { quotaRequest }).ok, true) @@ -423,9 +423,9 @@ test('quota_guard_request_too_large stays a 429 and is not rewritten to pool_una assert.equal(rewritten.body.error.code, 'quota_guard_request_too_large') }) -test('prompt estimate is skipped below the band and walked once for in-band accounts', () => { +test('prompt estimate is skipped below the band and walked once for in-band accounts', async () => { resetPromptEstimateStats() - const q = quota() + const q = await quota() setUtil(q, 'low', { u5: 0.5, u7: 0.2 }) setUtil(q, 'band-a', { u5: 0.79 }) setUtil(q, 'band-b', { u5: 0.76 }) @@ -445,15 +445,15 @@ test('prompt estimate is skipped below the band and walked once for in-band acco q.release('band-a') }) -test('admission and the panel agree near the gate (vm-10: pro, 7d 90% of a 95% line)', () => { +test('admission and the panel agree near the gate (vm-10: pro, 7d 90% of a 95% line)', async () => { const tier = { limit_5h: 0.95, limit_7d: 0.95, max_concurrency: 2 } - const q = new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { block_on_5h: true, block_on_7d: true, critical_band: 0.05, gate_overshoot: 0.05 }, tiers: { default: tier, pro: tier, max: tier }, }, - }) + }).init() const panel = (u7) => { q.ingestHeaders('vm10', { 'anthropic-ratelimit-unified-5h-utilization': '0.12', diff --git a/test/unit/quota-tiers.test.mjs b/test/unit/quota-tiers.test.mjs index bf4a33147..84167be72 100644 --- a/test/unit/quota-tiers.test.mjs +++ b/test/unit/quota-tiers.test.mjs @@ -109,8 +109,8 @@ test('buildRouting exposes effective Codex routing to the panel', () => { assert.equal(data.codex.protocols['openai.responses'].enabled, true) }) -test('canAccept uses per-tier 5h and weekly lines', () => { - const q = new AccountQuota({ +test('canAccept uses per-tier 5h and weekly lines', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { block_on_5h: true, block_on_7d: true, gate_overshoot: 0 }, @@ -120,7 +120,7 @@ test('canAccept uses per-tier 5h and weekly lines', () => { max: { max_concurrency: 4, safety_ratio: 0.95, weekly_safety_ratio: 0.95, warn_ratio: 0.85 }, }, }, - }) + }).init() q.ensure({ account_id: 'pro-1' }) q.setAccountTier('pro-1', 'pro') q.ingestHeaders('pro-1', { 'anthropic-ratelimit-unified-5h-utilization': '0.86' }) @@ -138,8 +138,8 @@ test('canAccept uses per-tier 5h and weekly lines', () => { assert.equal(q.canAccept('max-1').reason, 'quota_7d_safety') }) -test('reloadConfig picks up saved weekly line without process restart shape', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('reloadConfig picks up saved weekly line without process restart shape', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'p' }) q.setAccountTier('p', 'pro') // Pro default 7d line 0.80 + default 0.05 overshoot. @@ -152,15 +152,15 @@ test('reloadConfig picks up saved weekly line without process restart shape', () assert.equal(q.canAccept('p').ok, true) }) -test('applyTierConcurrency writes only the matching tier', () => { - const q = new AccountQuota({ +test('applyTierConcurrency writes only the matching tier', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { concurrency: { default_max_per_account: 20 } }, accounts: [ { account_id: 'a-pro', vm_id: 'vm-1', max_concurrency: 20 }, { account_id: 'a-max', vm_id: 'vm-2', max_concurrency: 20 }, ], - }) + }).init() q.setAccountTier('a-pro', 'pro') q.setAccountTier('a-max', 'max') q.applyTierConcurrency({ @@ -172,8 +172,8 @@ test('applyTierConcurrency writes only the matching tier', () => { assert.equal(q.repo.get('a-max').max_concurrency, 4) }) -test('applyTierRpm writes only the matching tier and skips override', () => { - const q = new AccountQuota({ +test('applyTierRpm writes only the matching tier and skips override', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {}, accounts: [ @@ -181,7 +181,7 @@ test('applyTierRpm writes only the matching tier and skips override', () => { { account_id: 'r-max', vm_id: 'vm-2', max_rpm: 0 }, { account_id: 'r-pin', vm_id: 'vm-3', max_rpm: 99 }, ], - }) + }).init() q.setAccountTier('r-pro', 'pro') q.setAccountTier('r-max', 'max') q.setMaxRpm('r-pin', 99, { override: true }) @@ -207,8 +207,8 @@ test('resolveTierPolicy reads live config', () => { assert.equal(p.weekly_safety_ratio, 0.7) }) -test('saving Pro 5h=80% lands on policyFor limit_5h', () => { - const q = new AccountQuota({ +test('saving Pro 5h=80% lands on policyFor limit_5h', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { block_on_5h: true, block_on_7d: true, gate_overshoot: 0 }, @@ -216,7 +216,7 @@ test('saving Pro 5h=80% lands on policyFor limit_5h', () => { pro: { limit_5h: 0.8, limit_7d: 0.8, max_concurrency: 2 }, }), }, - }) + }).init() q.ensure({ account_id: 'pro-80' }) q.setAccountTier('pro-80', 'pro') assert.equal(q.policyFor(q.repo.get('pro-80')).limit_5h, 0.8) @@ -224,8 +224,8 @@ test('saving Pro 5h=80% lands on policyFor limit_5h', () => { assert.equal(q.canAccept('pro-80').reason, 'quota_5h_safety') }) -test('Pro 95% runs one call at a time from 90% and blocks at line + overshoot', () => { - const q = new AccountQuota({ +test('Pro 95% runs one call at a time from 90% and blocks at line + overshoot', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { block_on_5h: true, block_on_7d: true }, @@ -233,7 +233,7 @@ test('Pro 95% runs one call at a time from 90% and blocks at line + overshoot', pro: { limit_5h: 0.95, limit_7d: 0.95, max_concurrency: 4 }, }), }, - }) + }).init() q.ensure({ account_id: 'pro-95' }) q.setAccountTier('pro-95', 'pro') q.ingestHeaders('pro-95', { @@ -256,14 +256,14 @@ test('Pro 95% runs one call at a time from 90% and blocks at line + overshoot', assert.equal(q.canAccept('pro-95').reason, 'quota_5h_safety') }) -test('reloadConfig applies conc and rpm without rewriting account rows', () => { - const q = new AccountQuota({ +test('reloadConfig applies conc and rpm without rewriting account rows', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { block_on_5h: true, block_on_7d: true }, tiers: { default: { max_concurrency: 2, max_rpm: 0 } }, }, - }) + }).init() q.ensure({ account_id: 'live' }) const acc = q.repo.get('live') assert.equal(q.limitFor(acc), 2) @@ -276,8 +276,8 @@ test('reloadConfig applies conc and rpm without rewriting account rows', () => { assert.equal(q.rpmLimitFor(q.repo.get('live')), 12) }) -test('manual conc and rpm pins ignore live tier', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('manual conc and rpm pins ignore live tier', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'pin' }) q.setMaxConcurrency('pin', 1, { override: true }) q.setMaxRpm('pin', 2, { override: true }) diff --git a/test/unit/rate-limit-service.test.mjs b/test/unit/rate-limit-service.test.mjs index 9b5ae05ee..f9a2b9992 100755 --- a/test/unit/rate-limit-service.test.mjs +++ b/test/unit/rate-limit-service.test.mjs @@ -4,15 +4,15 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { createDatabase } from '../../src/lib/db/database.mjs' -import { AccountRuntimeRepo } from '../../src/lib/db/repos/account-runtime-repo.mjs' +import { SchedulingRuntime as AccountRuntimeRepo } from '../../src/lib/pool/scheduling-runtime.mjs' import { AccountQuota } from '../../src/lib/pool/account-quota.mjs' import { RateLimitService, anthropic429Reset, hardBlockOf } from '../../src/lib/pool/rate-limit-service.mjs' -function setup(config = {}) { +async function setup(config = {}) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-rate-limit-')) - const db = createDatabase({ dataDir: dir }) - const runtimeRepo = new AccountRuntimeRepo(db) - const accountQuota = new AccountQuota({ db, config: {} }) + const db = await createDatabase({ dataDir: dir }) + const runtimeRepo = await new AccountRuntimeRepo(db).init() + const accountQuota = await new AccountQuota({ db, config: {} }).init() accountQuota.attachRuntimeRepo(runtimeRepo) runtimeRepo.upsert({ account_id: 'acc-1', vm_id: 'vm-01', status: 'ready' }) const probes = [] @@ -22,8 +22,8 @@ function setup(config = {}) { config, onUsageProbe: (item) => probes.push(item), }) - const close = () => { - db.close() + const close = async () => { + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } return { service, runtimeRepo, accountQuota, probes, close } @@ -41,8 +41,8 @@ function limitResult(message) { } } -test('plan limit writes rate_limit_reset_at from the text and survives a passive Extra re-read', () => { - const { service, runtimeRepo, accountQuota, probes, close } = setup() +test('plan limit writes rate_limit_reset_at from the text and survives a passive Extra re-read', async () => { + const { service, runtimeRepo, accountQuota, probes, close } = await setup() try { const now = Date.now() const block = service.handleUpstreamError({ @@ -64,12 +64,12 @@ test('plan limit writes rate_limit_reset_at from the text and survives a passive accountQuota.clearQuotaExhaustedCooldown('acc-1') assert.equal(runtimeRepo.get('acc-1').rate_limit_reset_at, state.rate_limit_reset_at) } finally { - close() + await close() } }) -test('plan limit without any reset falls back to the configured cooldown and asks for one /usage hop', () => { - const { service, runtimeRepo, probes, close } = setup({ fallback_cooldown_min: 30 }) +test('plan limit without any reset falls back to the configured cooldown and asks for one /usage hop', async () => { + const { service, runtimeRepo, probes, close } = await setup({ fallback_cooldown_min: 30 }) try { const now = Date.now() const block = service.handleUpstreamError({ @@ -84,12 +84,12 @@ test('plan limit without any reset falls back to the configured cooldown and ask assert.equal(Math.abs(runtimeRepo.get('acc-1').rate_limit_reset_at - block.until) < 1000, true) assert.deepEqual(probes, [{ accountId: 'acc-1', vmId: 'vm-01' }]) } finally { - close() + await close() } }) -test('model-scoped 429 does not write an account hard block', () => { - const { service, runtimeRepo, close } = setup() +test('model-scoped 429 does not write an account hard block', async () => { + const { service, runtimeRepo, close } = await setup() try { const block = service.handleUpstreamError({ accountId: 'acc-1', @@ -100,12 +100,12 @@ test('model-scoped 429 does not write an account hard block', () => { assert.equal(block, null) assert.equal(runtimeRepo.get('acc-1').rate_limit_reset_at, null) } finally { - close() + await close() } }) -test('#163: an unknown bare 429 asks for one usage probe and fabricates no 5h block', () => { - const { service, runtimeRepo, probes, close } = setup() +test('#163: an unknown bare 429 asks for one usage probe and fabricates no 5h block', async () => { + const { service, runtimeRepo, probes, close } = await setup() try { const block = service.handleUpstreamError({ accountId: 'acc-1', @@ -117,12 +117,12 @@ test('#163: an unknown bare 429 asks for one usage probe and fabricates no 5h bl assert.equal(runtimeRepo.get('acc-1').rate_limit_reset_at, null) assert.deepEqual(probes, [{ accountId: 'acc-1', vmId: 'vm-01' }]) } finally { - close() + await close() } }) -test('529 writes overload_until for the configured minutes', () => { - const { service, runtimeRepo, close } = setup({ overload_cooldown_min: 10 }) +test('529 writes overload_until for the configured minutes', async () => { + const { service, runtimeRepo, close } = await setup({ overload_cooldown_min: 10 }) try { const now = Date.now() const block = service.handleUpstreamError({ @@ -135,12 +135,12 @@ test('529 writes overload_until for the configured minutes', () => { assert.equal(block.until, now + 10 * 60_000) assert.equal(hardBlockOf(runtimeRepo.get('acc-1'), now).reason, 'overloaded') } finally { - close() + await close() } }) -test('only a live allowed header lifts a rate limit early', () => { - const { service, runtimeRepo, close } = setup() +test('only a live allowed header lifts a rate limit early', async () => { + const { service, runtimeRepo, close } = await setup() try { const now = Date.now() service.handleUpstreamError({ @@ -170,12 +170,12 @@ test('only a live allowed header lifts a rate limit early', () => { assert.equal(hardBlockOf(state, now), null) assert.equal(state.session_window_end, Number(reset) * 1000) } finally { - close() + await close() } }) -test('an out-of-range 5h reset header is ignored for the session window', () => { - const { service, runtimeRepo, close } = setup() +test('an out-of-range 5h reset header is ignored for the session window', async () => { + const { service, runtimeRepo, close } = await setup() try { const now = Date.now() service.updateSessionWindow({ @@ -188,12 +188,12 @@ test('an out-of-range 5h reset header is ignored for the session window', () => }) assert.equal(runtimeRepo.get('acc-1').session_window_end, null) } finally { - close() + await close() } }) -test('clearExpired drops elapsed rate limit and overload columns', () => { - const { runtimeRepo, close } = setup() +test('clearExpired drops elapsed rate limit and overload columns', async () => { + const { runtimeRepo, close } = await setup() try { const now = Date.now() runtimeRepo.updateWindow('acc-1', { rateLimitResetAt: now - 1000, overloadUntil: now - 1000 }) @@ -202,12 +202,12 @@ test('clearExpired drops elapsed rate limit and overload columns', () => { assert.equal(state.rate_limit_reset_at, null) assert.equal(state.overload_until, null) } finally { - close() + await close() } }) -test('an incomplete hop does not write an empty_response cooldown', () => { - const { service, runtimeRepo, close } = setup({ empty_response_cooldown_sec: 60 }) +test('an incomplete hop does not write an empty_response cooldown', async () => { + const { service, runtimeRepo, close } = await setup({ empty_response_cooldown_sec: 60 }) try { const block = service.handleUpstreamError({ accountId: 'acc-1', @@ -220,7 +220,7 @@ test('an incomplete hop does not write an empty_response cooldown', () => { assert.equal(runtimeRepo.get('acc-1').cooldown_until, null) assert.equal(typeof service.noteDistinctEmptyHop, 'undefined') } finally { - close() + await close() } }) diff --git a/test/unit/refusal-guard.test.mjs b/test/unit/refusal-guard.test.mjs index c7e85aefe..402825467 100644 --- a/test/unit/refusal-guard.test.mjs +++ b/test/unit/refusal-guard.test.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' import { RefusalGuardsRepo } from '../../src/lib/db/repos/refusal-guards-repo.mjs' import { REFUSAL_GUARD_MESSAGE, @@ -88,41 +88,41 @@ test('zero inject still allows distill skip while refusal fingerprint is indepen assert.ok(refusalFingerprint(inbound)) }) -test('remember then lookup hits the same prompt', () => { +test('remember then lookup hits the same prompt', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-refusal-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new RefusalGuardsRepo(db) const req = body('Return exactly CACHE_30_5m_OK.') const fp = refusalFingerprint(req) - assert.equal(repo.get(fp), null) - repo.remember({ + assert.equal(await repo.get(fp), null) + await repo.remember({ fingerprint: fp, model: req.model, requestId: 'req-1', errorMessage: 'violate our Usage Policy', preview: 'Return exactly CACHE_30_5m_OK.', }) - const hit = repo.get(fp) + const hit = await repo.get(fp) assert.ok(hit) assert.equal(hit.hit_count, 0) - const bumped = repo.hit(fp) + const bumped = await repo.hit(fp) assert.equal(bumped.hit_count, 1) const other = refusalFingerprint(body('Return exactly CACHE_30_5m_OK.', { model: 'claude-opus-4-8' })) - assert.equal(repo.get(other), null) + assert.equal(await repo.get(other), null) } finally { - db.close() + await db.close() } }) -test('timed refusal expires and no longer blocks', () => { +test('timed refusal expires and no longer blocks', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-refusal-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new RefusalGuardsRepo(db) const req = body('pause this 500') const fp = refusalFingerprint(req) - repo.remember({ + await repo.remember({ fingerprint: fp, model: req.model, requestId: 'req-500', @@ -130,10 +130,10 @@ test('timed refusal expires and no longer blocks', () => { preview: 'pause this 500', expiresAt: new Date(Date.now() - 1000).toISOString(), }) - assert.equal(repo.get(fp), null) - assert.equal(repo.list().length, 0) + assert.equal(await repo.get(fp), null) + assert.equal((await repo.list()).length, 0) } finally { - db.close() + await db.close() } }) test('guard error is a non-retryable HTTP 400 request rejection, not a login error', () => { @@ -170,27 +170,27 @@ test('guard error is a non-retryable HTTP 400 request rejection, not a login err }) }) -test('list remove and clear', () => { +test('list remove and clear', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-refusal-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new RefusalGuardsRepo(db) const a = refusalFingerprint(body('A')) const b = refusalFingerprint(body('B')) - repo.remember({ fingerprint: a, model: 'claude-opus-5', preview: 'A' }) - repo.remember({ fingerprint: b, model: 'claude-opus-5', preview: 'B' }) - assert.equal(repo.count(), 2) - assert.equal(repo.list().length, 2) - assert.equal(repo.remove(a), true) - assert.equal(repo.count(), 1) - assert.equal(repo.clear(), 1) - assert.equal(repo.count(), 0) + await repo.remember({ fingerprint: a, model: 'claude-opus-5', preview: 'A' }) + await repo.remember({ fingerprint: b, model: 'claude-opus-5', preview: 'B' }) + assert.equal(await repo.count(), 2) + assert.equal((await repo.list()).length, 2) + assert.equal(await repo.remove(a), true) + assert.equal(await repo.count(), 1) + assert.equal(await repo.clear(), 1) + assert.equal(await repo.count(), 0) } finally { - db.close() + await db.close() } }) -test('panel setting and env can disable the guard', () => { +test('panel setting and env can disable the guard', async () => { const settings = new Map([[REFUSAL_GUARD_SETTING, true]]) const read = (key, fb) => (settings.has(key) ? settings.get(key) : fb) assert.equal(isRefusalGuardEnabled(read), true) diff --git a/test/unit/refusal-similarity.test.mjs b/test/unit/refusal-similarity.test.mjs index 8295bd6c4..dee7e5c23 100644 --- a/test/unit/refusal-similarity.test.mjs +++ b/test/unit/refusal-similarity.test.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' import { RefusalGuardsRepo } from '../../src/lib/db/repos/refusal-guards-repo.mjs' import { RefusalDeviceBlocksRepo } from '../../src/lib/db/repos/refusal-device-blocks-repo.mjs' import { @@ -99,9 +99,9 @@ test('a Responses prompt with a small tail edit stays above the similarity line' assert.ok(nearScore >= REFUSAL_SIMILARITY, `near ${nearScore}`) }) -test('stored near-duplicate blocks and then the device blocks any later prompt', () => { +test('stored near-duplicate blocks and then the device blocks any later prompt', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-near-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const guards = new RefusalGuardsRepo(db) const devices = new RefusalDeviceBlocksRepo(db) @@ -109,7 +109,7 @@ test('stored near-duplicate blocks and then the device blocks any later prompt', metadata: { user_id: { device_id: 'device-aaaabbbbcccc', session_id: 'sess-1' } }, }) const fp = refusalFingerprint(original) - guards.remember({ + await guards.remember({ fingerprint: fp, model: original.model, signature: refusalPromptSignature(original), @@ -119,41 +119,41 @@ test('stored near-duplicate blocks and then the device blocks any later prompt', metadata: { user_id: { device_id: 'device-aaaabbbbcccc', session_id: 'sess-1' } }, }) assert.notEqual(refusalFingerprint(edited), fp) - const similar = matchStoredRefusal({ inbound: edited, body: edited, repo: guards, devices }) + const similar = await matchStoredRefusal({ inbound: edited, body: edited, repo: guards, devices }) assert.equal(similar.kind, 'similar') assert.ok(similar.score >= REFUSAL_SIMILARITY) - devices.block({ deviceId: similar.deviceId, fingerprint: similar.fingerprint, reason: 'refusal_similar' }) + await devices.block({ deviceId: similar.deviceId, fingerprint: similar.fingerprint, reason: 'refusal_similar' }) const other = body('completely different short ask', { metadata: { user_id: { device_id: 'device-aaaabbbbcccc', session_id: 'sess-2' } }, }) - const banned = matchStoredRefusal({ inbound: other, body: other, repo: guards, devices }) + const banned = await matchStoredRefusal({ inbound: other, body: other, repo: guards, devices }) assert.equal(banned.kind, 'device') assert.equal(inboundRefusalDeviceId({ inbound: { device_id: 'short' } }), '') - assert.equal(devices.block({ deviceId: '' }), null) + assert.equal(await devices.block({ deviceId: '' }), null) } finally { - db.close() + await db.close() } }) -test('different user text under the same system is not a near match', () => { +test('different user text under the same system is not a near match', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-near-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const guards = new RefusalGuardsRepo(db) const stored = body(longUser('payload-a')) - guards.remember({ + await guards.remember({ fingerprint: refusalFingerprint(stored), model: stored.model, signature: refusalPromptSignature(stored), }) const other = body(longUser('unrelated-question-about-refactoring-the-scheduler')) - assert.equal(matchStoredRefusal({ inbound: other, body: other, repo: guards, devices: null }), null) + assert.equal(await matchStoredRefusal({ inbound: other, body: other, repo: guards, devices: null }), null) } finally { - db.close() + await db.close() } }) -test('panel policy defaults on at 90 and rejects an unknown threshold', () => { +test('panel policy defaults on at 90 and rejects an unknown threshold', async () => { assert.deepEqual(refusalGuardPolicy(), { enabled: true, similarity_enabled: true, @@ -165,8 +165,8 @@ test('panel policy defaults on at 90 and rejects an unknown threshold', () => { get: (key, fallback) => (settings.has(key) ? settings.get(key) : fallback), set: (key, value) => settings.set(key, value), } - assert.equal(applyRefusalGuardPatch(store, { similarity: 70 }).ok, false) - assert.equal(applyRefusalGuardPatch(store, { similarity: 85, similarity_enabled: false }).ok, true) + assert.equal((await applyRefusalGuardPatch(store, { similarity: 70 })).ok, false) + assert.equal((await applyRefusalGuardPatch(store, { similarity: 85, similarity_enabled: false })).ok, true) assert.equal(normalizeRefusalSimilarity(85), 85) assert.deepEqual( refusalGuardPolicy((key, fallback) => store.get(key, fallback)), @@ -179,26 +179,26 @@ test('panel policy defaults on at 90 and rejects an unknown threshold', () => { ) }) -test('turning similarity or device block off skips that check', () => { +test('turning similarity or device block off skips that check', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-near-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const guards = new RefusalGuardsRepo(db) const devices = new RefusalDeviceBlocksRepo(db) const stored = body(longUser('payload'), { metadata: { user_id: { device_id: 'device-aaaabbbbcccc', session_id: 'sess-1' } }, }) - guards.remember({ + await guards.remember({ fingerprint: refusalFingerprint(stored), model: stored.model, signature: refusalPromptSignature(stored), }) - devices.block({ deviceId: 'device-aaaabbbbcccc', reason: 'refusal_guard' }) + await devices.block({ deviceId: 'device-aaaabbbbcccc', reason: 'refusal_guard' }) const edited = body(longUser('payload').slice(0, -20) + 'tail-edit-xx', { metadata: { user_id: { device_id: 'device-aaaabbbbcccc', session_id: 'sess-9' } }, }) assert.equal( - matchStoredRefusal({ + await matchStoredRefusal({ inbound: edited, body: edited, repo: guards, @@ -209,6 +209,6 @@ test('turning similarity or device block off skips that check', () => { null, ) } finally { - db.close() + await db.close() } }) diff --git a/test/unit/request-log.test.mjs b/test/unit/request-log.test.mjs index 333e4ddc7..3f6f53c6f 100644 --- a/test/unit/request-log.test.mjs +++ b/test/unit/request-log.test.mjs @@ -14,6 +14,7 @@ import { } from '../../src/lib/admin/request-log.mjs' import { ApiKeyStore } from '../../src/lib/admin/api-keys.mjs' import { GroupsRepo } from '../../src/lib/db/repos/groups-repo.mjs' +import { trackPendingWrite, flushPendingWrites } from '../../src/lib/db/pending-writes.mjs' import { gateVerdict } from '../../src/lib/protocol/intercept-stats.mjs' function tmpStore(mode = 'normal') { @@ -48,14 +49,14 @@ test('summarizeBody counts messages and tools', () => { assert.equal(s.system_len, 10) }) -test('normal mode writes jsonl without body', () => { +test('normal mode writes jsonl without body', async () => { const store = tmpStore('normal') const ctx = store.start( { method: 'POST', headers: { 'user-agent': 't' }, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, ) assert.equal(ctx.mode, 'normal') - const sum = store.finish(ctx, { + const sum = await store.finish(ctx, { status: 200, model: 'claude-haiku-4-5-20251001', stream: false, @@ -66,20 +67,62 @@ test('normal mode writes jsonl without body', () => { assert.equal(sum.status, 200) assert.equal(sum.api_key_id, 'key_1') assert.equal(sum.inbound_body, undefined) - const listed = store.listNormal({ limit: 5 }) + const listed = await store.listNormal({ limit: 5 }) assert.equal(listed.length, 1) assert.equal(listed[0].request_id, ctx.request_id) // no debug file - assert.equal(store.listDebug({ limit: 5 }).length, 0) + assert.equal((await store.listDebug({ limit: 5 })).length, 0) }) -test('summary persists protocol usage detail (cache/model/ttft/stop_reason)', () => { +for (const mode of ['normal', 'debug']) { + test(`${mode} logs escape NUL fields and leave response writes drainable`, async () => { + const store = tmpStore(mode) + const ctx = store.start( + { method: 'POST', headers: {}, socket: {} }, + { protocol: 'anthropic.messages', pathName: '/v1/messages' }, + ) + const model = 'invalid\0model' + const inbound = { model, messages: [{ role: 'user', content: '文本\0content' }] } + const summary = await trackPendingWrite( + store.finish(ctx, { + status: 400, + model, + requested_model: model, + upstream_model: 'upstream\0model', + error_message: 'invalid model: ' + model, + session_id: 'session\0id', + inbound_body: inbound, + }), + ) + await flushPendingWrites() + await flushPendingWrites() + const row = await store.repo.getByRequestId(ctx.request_id) + for (const [field, expected] of Object.entries({ + model: 'invalid\\u0000model', + requested_model: 'invalid\\u0000model', + upstream_model: 'upstream\\u0000model', + error_message: 'invalid model: invalid\\u0000model', + session_id: 'session\\u0000id', + })) { + assert.equal(row[field], expected, field) + assert.equal(summary[field], expected, field) + } + if (mode === 'debug') { + const debug = await store.getDebug(ctx.request_id) + assert.equal(debug.model, summary.model) + assert.deepEqual(debug.inbound_body, inbound) + } + assert.equal(inbound.model, model, 'logging must not mutate the request') + }) +} + +test('summary persists protocol usage detail (cache/model/ttft/stop_reason)', async () => { const store = tmpStore('normal') const ctx = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, ) - const sum = store.finish(ctx, { + const sum = await store.finish(ctx, { status: 200, model: 'sonnet', requested_model: 'sonnet', @@ -106,28 +149,28 @@ test('summary persists protocol usage detail (cache/model/ttft/stop_reason)', () // Sonnet 5 official: $2/$10/cache 5m $2.50/1h $4/read $0.20 per MTok assert.equal(sum.pricing_model, 'claude-sonnet-5') assert.ok(Math.abs(sum.total_cost - 0.0000926) < 1e-12) - const row = store.queryNormal({ limit: 1 }).items[0] + const row = (await store.queryNormal({ limit: 1 })).items[0] assert.equal(row.cache_creation_5m_tokens, 4) assert.equal(row.upstream_model, 'claude-sonnet-5') assert.equal(row.stop_reason, 'end_turn') assert.equal(row.total_cost, 0.0000926) }) -test('backfillMissingCosts restores actual cost with the stored group multiplier', () => { +test('backfillMissingCosts restores actual cost with the stored group multiplier', async () => { const store = tmpStore('normal') const insert = store.db.prepare(` INSERT INTO usage_logs ( id, request_id, created_at, model, upstream_model, status, input_tokens, output_tokens, rate_multiplier - ) VALUES (?, ?, ?, 'claude-opus-5', 'claude-opus-5', 200, 1000000, 0, ?) + ) VALUES ($1, $2, $3, 'claude-opus-5', 'claude-opus-5', 200, 1000000, 0, $4) `) const now = new Date().toISOString() - insert.run('log_old_bill_default', 'rid_old_bill_default', now, null) - insert.run('log_old_bill_free', 'rid_old_bill_free', now, 0) - insert.run('log_old_bill_double', 'rid_old_bill_double', now, 2) + await insert.run('log_old_bill_default', 'rid_old_bill_default', now, null) + await insert.run('log_old_bill_free', 'rid_old_bill_free', now, 0) + await insert.run('log_old_bill_double', 'rid_old_bill_double', now, 2) - assert.equal(store.repo.backfillMissingCosts(), 3) - const rows = store.db + assert.equal(await store.repo.backfillMissingCosts(), 3) + const rows = await store.db .prepare( "SELECT id, total_cost, actual_cost, pricing_model FROM usage_logs WHERE id LIKE 'log_old_bill_%' ORDER BY id", ) @@ -140,25 +183,32 @@ test('backfillMissingCosts restores actual cost with the stored group multiplier { id: 'log_old_bill_free', total_cost: 5, actual_cost: 0, pricing_model: 'claude-opus-5' }, ], ) - assert.equal(store.repo.backfillMissingCosts(), 0) + assert.equal(await store.repo.backfillMissingCosts(), 0) }) -function finishWith(store, model, usage) { +async function finishWith(store, model, usage) { const ctx = store.start({ method: 'POST', headers: {}, socket: {} }, { protocol: 'x', pathName: '/v1' }) - return store.finish(ctx, { status: 200, model, upstream_model: model, vm_id: 'vm-t', account_id: 'acc-t', usage }) + return await store.finish(ctx, { + status: 200, + model, + upstream_model: model, + vm_id: 'vm-t', + account_id: 'acc-t', + usage, + }) } -test('tier-unpriced rows stay unpriced after backfill', () => { +test('tier-unpriced rows stay unpriced after backfill', async () => { const store = tmpStore('normal') // gpt-5.5 publishes no >272K fast column; gpt-5.3-codex has no flex band. - const a = finishWith(store, 'gpt-5.5', { input_tokens: 1_000_000, output_tokens: 0, service_tier: 'priority' }) - const b = finishWith(store, 'gpt-5.3-codex', { input_tokens: 1000, output_tokens: 0, service_tier: 'flex' }) + const a = await finishWith(store, 'gpt-5.5', { input_tokens: 1_000_000, output_tokens: 0, service_tier: 'priority' }) + const b = await finishWith(store, 'gpt-5.3-codex', { input_tokens: 1000, output_tokens: 0, service_tier: 'flex' }) assert.equal(a.total_cost, null) assert.equal(a.pricing_model, 'unpriced') assert.equal(a.service_tier, 'priority') assert.equal(b.pricing_model, 'unpriced') - store.repo.billingStats() - const rows = store.db.prepare('SELECT total_cost, actual_cost, pricing_model FROM usage_logs ORDER BY id').all() + await store.repo.billingStats() + const rows = await store.db.prepare('SELECT total_cost, actual_cost, pricing_model FROM usage_logs ORDER BY id').all() for (const r of rows) { assert.equal(r.total_cost, null) assert.equal(r.actual_cost, null) @@ -166,36 +216,35 @@ test('tier-unpriced rows stay unpriced after backfill', () => { } }) -test('backfill reprices with the stored service tier and speed', () => { +test('backfill reprices with the stored service tier and speed', async () => { const store = tmpStore('normal') const insert = store.db.prepare(` INSERT INTO usage_logs (id, request_id, created_at, model, upstream_model, status, input_tokens, output_tokens, service_tier, speed) - VALUES (?, ?, ?, ?, ?, 200, ?, 0, ?, ?) + VALUES ($1, $2, $3, $4, $5, 200, $6, 0, $7, $8) `) const now = new Date().toISOString() // 100K stays under the 272K long-context threshold → plain flex band ($2.5/MTok input). - insert.run('log_bf_flex', 'rid_bf_flex', now, 'gpt-5.5', 'gpt-5.5', 100_000, 'flex', null) - insert.run('log_bf_fast', 'rid_bf_fast', now, 'claude-opus-4-8', 'claude-opus-4-8', 1_000_000, null, 'fast') - insert.run('log_bf_weird', 'rid_bf_weird', now, 'gpt-5.5', 'gpt-5.5', 1000, 'weird', null) - store.repo.backfillMissingCosts() + await insert.run('log_bf_flex', 'rid_bf_flex', now, 'gpt-5.5', 'gpt-5.5', 100_000, 'flex', null) + await insert.run('log_bf_fast', 'rid_bf_fast', now, 'claude-opus-4-8', 'claude-opus-4-8', 1_000_000, null, 'fast') + await insert.run('log_bf_weird', 'rid_bf_weird', now, 'gpt-5.5', 'gpt-5.5', 1000, 'weird', null) + await store.repo.backfillMissingCosts() const got = Object.fromEntries( - store.db - .prepare("SELECT id, total_cost, pricing_model FROM usage_logs WHERE id LIKE 'log_bf_%'") - .all() - .map((r) => [r.id, [r.total_cost, r.pricing_model]]), + (await store.db.prepare("SELECT id, total_cost, pricing_model FROM usage_logs WHERE id LIKE 'log_bf_%'").all()).map( + (r) => [r.id, [r.total_cost, r.pricing_model]], + ), ) assert.deepEqual(got.log_bf_flex, [0.25, 'gpt-5.5']) assert.deepEqual(got.log_bf_fast, [10, 'claude-opus-4-8']) assert.deepEqual(got.log_bf_weird, [null, 'unpriced']) - assert.equal(store.repo.backfillMissingCosts(), 0) + assert.equal(await store.repo.backfillMissingCosts(), 0) }) -test('costByModel splits rows by billing band', () => { +test('costByModel splits rows by billing band', async () => { const store = tmpStore('normal') - finishWith(store, 'claude-opus-4-8', { input_tokens: 1_000_000, output_tokens: 0, speed: 'fast' }) - finishWith(store, 'claude-opus-4-8', { input_tokens: 1_000_000, output_tokens: 0 }) - const rows = store.repo.costByModel({ vmId: 'vm-t' }) + await finishWith(store, 'claude-opus-4-8', { input_tokens: 1_000_000, output_tokens: 0, speed: 'fast' }) + await finishWith(store, 'claude-opus-4-8', { input_tokens: 1_000_000, output_tokens: 0 }) + const rows = await store.repo.costByModel({ vmId: 'vm-t' }) assert.equal(rows.length, 2) const fast = rows.find((r) => r.speed === 'fast') const std = rows.find((r) => r.speed !== 'fast') @@ -205,23 +254,23 @@ test('costByModel splits rows by billing band', () => { assert.equal(std.long_context, 0) }) -test('vmUsageStats buckets by Shanghai day, isolates the slot and ranks models and paths', () => { +test('vmUsageStats buckets by Shanghai day, isolates the slot and ranks models and paths', async () => { const store = tmpStore('normal') const insert = store.db.prepare(` INSERT INTO usage_logs (id, request_id, created_at, path, model, upstream_model, status, vm_id, input_tokens, output_tokens, total_cost, duration_ms) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 100, 50, ?, ?) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 100, 50, $9, $10) `) const shanghaiDay = (offsetDays) => new Date(Date.now() + 8 * 3600_000 - offsetDays * 86400_000).toISOString().slice(0, 10) // 16:30Z is 00:30 the next Shanghai day: a UTC bucket would file it under the wrong date. const at = (offsetDays) => `${shanghaiDay(offsetDays + 1)}T16:30:00.000Z` - insert.run('u1', 'r1', at(0), '/v1/messages', 'm-a', 'm-a', 200, 'vm-x', 1, 1000) - insert.run('u2', 'r2', at(0), '/v1/messages', 'm-a', 'm-a', 200, 'vm-x', 1, 3000) - insert.run('u3', 'r3', at(2), '/v1/chat/completions', 'm-b', 'm-b', 200, 'vm-x', 4, null) - insert.run('u4', 'r4', at(0), '/v1/messages', 'm-a', 'm-a', 200, 'vm-other', 99, 1) - insert.run('u5', 'r5', at(40), '/v1/messages', 'm-a', 'm-a', 200, 'vm-x', 99, 1) - const out = store.repo.vmUsageStats({ vmId: 'vm-x', days: 30 }) + await insert.run('u1', 'r1', at(0), '/v1/messages', 'm-a', 'm-a', 200, 'vm-x', 1, 1000) + await insert.run('u2', 'r2', at(0), '/v1/messages', 'm-a', 'm-a', 200, 'vm-x', 1, 3000) + await insert.run('u3', 'r3', at(2), '/v1/chat/completions', 'm-b', 'm-b', 200, 'vm-x', 4, null) + await insert.run('u4', 'r4', at(0), '/v1/messages', 'm-a', 'm-a', 200, 'vm-other', 99, 1) + await insert.run('u5', 'r5', at(40), '/v1/messages', 'm-a', 'm-a', 200, 'vm-x', 99, 1) + const out = await store.repo.vmUsageStats({ vmId: 'vm-x', days: 30 }) assert.deepEqual( out.history.map((h) => [h.day, h.requests, h.total_cost]), [ @@ -245,21 +294,21 @@ test('vmUsageStats buckets by Shanghai day, isolates the slot and ranks models a ['/v1/messages', '/v1/chat/completions'], ) assert.equal(out.endpoints[0].tokens, 300) - assert.deepEqual(store.repo.vmUsageStats({ vmId: null }).history, []) + assert.deepEqual((await store.repo.vmUsageStats({ vmId: null })).history, []) }) -test('keyUsageStats counts only that key and ranks VMs', () => { +test('keyUsageStats counts only that key and ranks VMs', async () => { const store = tmpStore('normal') const insert = store.db.prepare(` INSERT INTO usage_logs (id, request_id, created_at, path, model, upstream_model, status, vm_id, api_key_id, input_tokens, output_tokens, total_cost, duration_ms) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 100, 50, ?, ?) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 100, 50, $10, $11) `) const now = new Date().toISOString() - insert.run('k1', 'rk1', now, '/v1/messages', 'm-a', 'm-a', 200, 'vm-a', 'key_a', 1, 10) - insert.run('k2', 'rk2', now, '/v1/messages', 'm-b', 'm-b', 200, 'vm-b', 'key_a', 3, 20) - insert.run('k3', 'rk3', now, '/v1/messages', 'm-a', 'm-a', 200, 'vm-a', 'key_b', 99, 1) - const out = store.repo.keyUsageStats({ apiKeyId: 'key_a', days: 30 }) + await insert.run('k1', 'rk1', now, '/v1/messages', 'm-a', 'm-a', 200, 'vm-a', 'key_a', 1, 10) + await insert.run('k2', 'rk2', now, '/v1/messages', 'm-b', 'm-b', 200, 'vm-b', 'key_a', 3, 20) + await insert.run('k3', 'rk3', now, '/v1/messages', 'm-a', 'm-a', 200, 'vm-a', 'key_b', 99, 1) + const out = await store.repo.keyUsageStats({ apiKeyId: 'key_a', days: 30 }) assert.equal( out.history.reduce((n, row) => n + row.requests, 0), 2, @@ -276,22 +325,22 @@ test('keyUsageStats counts only that key and ranks VMs', () => { ], ) assert.equal(out.endpoints, undefined) - assert.deepEqual(store.repo.keyUsageStats({ apiKeyId: null }).history, []) + assert.deepEqual((await store.repo.keyUsageStats({ apiKeyId: null })).history, []) }) -test('zero group multiplier keeps usage and key counters but charges no USD', () => { +test('zero group multiplier keeps usage and key counters but charges no USD', async () => { const store = tmpStore('normal') const groups = new GroupsRepo(store.db) - groups.update(1, { rate_multiplier: 0 }) - const rateMultiplier = groups.rateMultiplier(1) - assert.equal(groups.getById(1).rate_multiplier, 0) + await groups.update(1, { rate_multiplier: 0 }) + const rateMultiplier = await groups.rateMultiplier(1) + assert.equal((await groups.getById(1)).rate_multiplier, 0) assert.equal(rateMultiplier, 0) const ctx = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, ) - const summary = store.finish(ctx, { + const summary = await store.finish(ctx, { status: 200, model: 'claude-opus-5', upstream_model: 'claude-opus-5', @@ -302,8 +351,8 @@ test('zero group multiplier keeps usage and key counters but charges no USD', () assert.equal(summary.actual_cost, 0) const keys = new ApiKeyStore({ db: store.db, hashSecret: 'group-zero-test' }) - const key = keys.create({ name: 'free-group', quota: 100 }) - keys.recordUsage( + const key = await keys.create({ name: 'free-group', quota: 100 }) + await keys.recordUsage( key.id, { input_tokens: 1_000_000 }, { @@ -311,21 +360,21 @@ test('zero group multiplier keeps usage and key counters but charges no USD', () rateMultiplier, }, ) - const stored = keys.getById(key.id) + const stored = await keys.getById(key.id) assert.equal(stored.requests, 1) assert.equal(stored.tokens_in, 1_000_000) assert.equal(stored.quota_used, 0) assert.equal(stored.usage_5h, 0) }) -test('billingStats Extra window excludes older logs outside reset-5h', () => { +test('billingStats Extra window excludes older logs outside reset-5h', async () => { const store = tmpStore('normal') const now = Date.parse('2026-09-26T12:00:00.000Z') const ctxOld = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, ) - store.finish(ctxOld, { + await store.finish(ctxOld, { status: 200, model: 'claude-opus-5', upstream_model: 'claude-opus-5', @@ -333,12 +382,14 @@ test('billingStats Extra window excludes older logs outside reset-5h', () => { account_id: 'acc-1', usage: { input_tokens: 1_000_000, output_tokens: 0 }, }) - store.repo.db.prepare('UPDATE usage_logs SET created_at = ? WHERE vm_id = ?').run('2026-09-26T08:00:00.000Z', 'vm-01') + await store.repo.db + .prepare('UPDATE usage_logs SET created_at = $1 WHERE vm_id = $2') + .run('2026-09-26T08:00:00.000Z', 'vm-01') const ctxNew = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, ) - store.finish(ctxNew, { + await store.finish(ctxNew, { status: 200, model: 'claude-opus-5', upstream_model: 'claude-opus-5', @@ -346,30 +397,30 @@ test('billingStats Extra window excludes older logs outside reset-5h', () => { account_id: 'acc-1', usage: { input_tokens: 200_000, output_tokens: 0 }, }) - store.repo.db + await store.repo.db .prepare( "UPDATE usage_logs SET created_at = '2026-09-26T11:30:00.000Z' WHERE created_at > '2026-09-26T10:00:00.000Z'", ) .run() const reset5h = '2026-09-26T16:00:00.000Z' - const bill = store.billingStats({ + const bill = await store.billingStats({ now, accountWindows: [{ account_id: 'acc-1', vm_id: 'vm-01', reset_5h: reset5h }], }) assert.equal(bill.accounts[0].window_5h_requests, 1) assert.ok(bill.accounts[0].window_5h_cost < 2) assert.equal(bill.accounts[0].window_5h_cost, bill.window_5h.total_cost) - const rolling = store.billingStats({ now }) + const rolling = await store.billingStats({ now }) assert.equal(rolling.accounts[0].window_5h_requests, 2) }) -test('billingStats aggregates official cost per account and today', () => { +test('billingStats aggregates official cost per account and today', async () => { const store = tmpStore('normal') const ctx = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, ) - store.finish(ctx, { + await store.finish(ctx, { status: 200, model: 'claude-opus-5', upstream_model: 'claude-opus-5', @@ -377,7 +428,7 @@ test('billingStats aggregates official cost per account and today', () => { account_id: 'acc-1', usage: { input_tokens: 1_000_000, output_tokens: 0 }, }) - const bill = store.billingStats() + const bill = await store.billingStats() assert.equal(bill.currency, 'USD') assert.equal(bill.total.total_cost, 5) assert.equal(bill.today.total_cost, 5) @@ -393,20 +444,20 @@ test('billingStats aggregates official cost per account and today', () => { assert.equal(bill.accounts[0].window_7d_success, 1) assert.equal(bill.accounts[0].window_7d_errors, 0) assert.ok(bill.window_7d) - const models = store.costByModel({ vmId: 'vm-01' }) + const models = await store.costByModel({ vmId: 'vm-01' }) assert.equal(models.length, 1) assert.equal(models[0].model, 'claude-opus-5') assert.equal(models[0].total_cost, 5) assert.equal(models[0].input_tokens, 1_000_000) }) -test('finish writes OpenAI Responses cache read and cache write', () => { +test('finish writes OpenAI Responses cache read and cache write', async () => { const store = tmpStore('normal') const ctx = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'openai.responses', pathName: '/v1/responses' }, ) - const sum = store.finish(ctx, { + const sum = await store.finish(ctx, { status: 200, protocol: 'openai.responses', model: 'gpt-5.6-sol', @@ -422,13 +473,13 @@ test('finish writes OpenAI Responses cache read and cache write', () => { assert.equal(sum.cache_creation_tokens, 10) }) -test('finish prices OpenAI usage with top-level cached_tokens', () => { +test('finish prices OpenAI usage with top-level cached_tokens', async () => { const store = tmpStore('normal') const ctx = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'openai.chat', pathName: '/v1/chat/completions' }, ) - const sum = store.finish(ctx, { + const sum = await store.finish(ctx, { status: 200, protocol: 'openai.chat', model: 'gpt-5.5', @@ -439,13 +490,13 @@ test('finish prices OpenAI usage with top-level cached_tokens', () => { assert.equal(sum.cache_read_tokens, 200_000) }) -test('finish prices OpenAI-shaped usage from third-party clients', () => { +test('finish prices OpenAI-shaped usage from third-party clients', async () => { const store = tmpStore('normal') const ctx = store.start( { method: 'POST', headers: { 'user-agent': 'OpenAI/Python 1.70.0' }, socket: {} }, { protocol: 'openai.chat', pathName: '/v1/chat/completions' }, ) - const sum = store.finish(ctx, { + const sum = await store.finish(ctx, { status: 200, protocol: 'openai.chat', model: 'claude-sonnet-5', @@ -460,9 +511,9 @@ test('finish prices OpenAI-shaped usage from third-party clients', () => { assert.equal(sum.cache_read_tokens, 1_000_000) }) -test('cache breakdown without a split bills 5m unless cache_ttl is explicitly 1h', () => { +test('cache breakdown without a split bills 5m unless cache_ttl is explicitly 1h', async () => { const store = tmpStore('normal') - const five = store.finish( + const five = await store.finish( store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, @@ -477,7 +528,7 @@ test('cache breakdown without a split bills 5m unless cache_ttl is explicitly 1h assert.equal(five.cache_creation_5m_tokens, 9) assert.equal(five.cache_creation_1h_tokens, 0) assert.equal(five.model_mismatch, 0) - const hour = store.finish( + const hour = await store.finish( store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, @@ -493,9 +544,9 @@ test('cache breakdown without a split bills 5m unless cache_ttl is explicitly 1h assert.equal(hour.cache_creation_1h_tokens, 9) }) -test('finish persists web_search_requests from server_tool_use', () => { +test('finish persists web_search_requests from server_tool_use', async () => { const store = tmpStore('normal') - const sum = store.finish( + const sum = await store.finish( store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, @@ -512,31 +563,32 @@ test('finish persists web_search_requests from server_tool_use', () => { }, ) assert.equal(sum.web_search_requests, 3) - const row = store.db.prepare('SELECT web_search_requests FROM usage_logs WHERE id = ?').get(sum.id) + const row = await store.db.prepare('SELECT web_search_requests FROM usage_logs WHERE id = $1').get(sum.id) assert.equal(row.web_search_requests, 3) }) -test('repriceRange rewrites total_cost and fills unpriced rows while actual_cost stays put', () => { +test('repriceRange rewrites total_cost and fills unpriced rows while actual_cost stays put', async () => { const store = tmpStore('normal') const now = new Date().toISOString() - store.db + await store.db .prepare(` INSERT INTO usage_logs ( id, request_id, created_at, model, upstream_model, status, input_tokens, output_tokens, total_cost, actual_cost, pricing_model, rate_multiplier ) VALUES - ('log_rp_old', 'rid_rp_old', ?, 'claude-opus-5', 'claude-opus-5', 200, 1000000, 0, 1, 9, 'opus-5', 1), - ('log_rp_null', 'rid_rp_null', ?, 'claude-opus-5', 'claude-opus-5', 200, 1000000, 0, NULL, 4, NULL, 2) + ('log_rp_old', 'rid_rp_old', $1, 'claude-opus-5', 'claude-opus-5', 200, 1000000, 0, 1, 9, 'opus-5', 1), + ('log_rp_null', 'rid_rp_null', $2, 'claude-opus-5', 'claude-opus-5', 200, 1000000, 0, NULL, 4, NULL, 2) `) .run(now, now) - const out = store.repo.repriceRange({ since: now, until: now }) + const out = await store.repo.repriceRange({ since: now, until: now }) assert.equal(out.rows, 2) assert.ok(out.changed >= 1) const rows = Object.fromEntries( - store.db - .prepare("SELECT id, total_cost, actual_cost, pricing_model FROM usage_logs WHERE id LIKE 'log_rp_%'") - .all() - .map((r) => [r.id, r]), + ( + await store.db + .prepare("SELECT id, total_cost, actual_cost, pricing_model FROM usage_logs WHERE id LIKE 'log_rp_%'") + .all() + ).map((r) => [r.id, r]), ) assert.equal(rows.log_rp_old.actual_cost, 9) assert.notEqual(rows.log_rp_old.total_cost, 1) @@ -545,13 +597,13 @@ test('repriceRange rewrites total_cost and fills unpriced rows while actual_cost assert.equal(rows.log_rp_null.total_cost, 5) }) -test('finish records washed Responses path over inbound chat path', () => { +test('finish records washed Responses path over inbound chat path', async () => { const store = tmpStore('normal') const ctx = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'openai.chat', pathName: '/v1/chat/completions' }, ) - const sum = store.finish(ctx, { + const sum = await store.finish(ctx, { status: 200, protocol: 'openai.responses', path: '/v1/responses', @@ -561,18 +613,18 @@ test('finish records washed Responses path over inbound chat path', () => { assert.equal(sum.protocol, 'openai.responses') }) -test('debug mode stores full redacted body', () => { +test('debug mode stores full redacted body', async () => { const store = tmpStore('normal') const req = { method: 'POST', headers: { 'x-kin-debug': '1', authorization: 'Bearer sk-kin-abc' }, socket: {} } const ctx = store.start(req, { protocol: 'openai.chat', pathName: '/v1/chat/completions' }) assert.equal(ctx.mode, 'debug') - store.finish(ctx, { + await store.finish(ctx, { status: 200, model: 'm', inbound_body: { model: 'm', messages: [{ role: 'user', content: 'hi sk-ant-oat01-ABCDEFGH12345678' }] }, hop_meta: { params: { dropped: ['max_tokens'] } }, }) - const dbg = store.listDebug({ limit: 5 }) + const dbg = await store.listDebug({ limit: 5 }) assert.equal(dbg.length, 1) assert.equal(dbg[0].request_id, ctx.request_id) assert.ok(dbg[0].headers.authorization === '***REDACTED***') @@ -582,38 +634,38 @@ test('debug mode stores full redacted body', () => { assert.deepEqual(dbg[0].hop_meta.params.dropped, ['max_tokens']) }) -test('outbound bodies are stored only when the operator configured debug, not via caller headers', () => { +test('outbound bodies are stored only when the operator configured debug, not via caller headers', async () => { const outbound = { body: { system: 'gateway persona overlay' }, headers: { 'x-app': 'cli' } } - const record = (store, headers) => { + const record = async (store, headers) => { const ctx = store.start({ method: 'POST', headers, socket: {} }, { pathName: '/v1/messages' }) - store.finish(ctx, { + await store.finish(ctx, { status: 200, inbound_body: { model: 'm' }, outbound_body: outbound.body, outbound_headers: outbound.headers, }) - return store.getDebug(ctx.request_id) + return await store.getDebug(ctx.request_id) } for (const headers of [{ 'x-kin-debug': '1' }, { 'x-kin-log': 'debug' }]) { - const forced = record(tmpStore('normal'), headers) + const forced = await record(tmpStore('normal'), headers) assert.deepEqual(forced.inbound_body, { model: 'm' }) assert.equal(forced.outbound_body, null) assert.equal(forced.outbound_headers, null) } - const operator = record(tmpStore('debug'), {}) + const operator = await record(tmpStore('debug'), {}) assert.deepEqual(operator.outbound_body, outbound.body) assert.deepEqual(operator.outbound_headers, outbound.headers) }) -test('off mode writes nothing', () => { +test('off mode writes nothing', async () => { const store = tmpStore('off') const ctx = store.start({ method: 'POST', headers: {}, socket: {} }, { pathName: '/v1/messages' }) assert.equal(ctx.mode, 'off') - assert.equal(store.finish(ctx, { status: 200 }), null) - assert.equal(store.listNormal({ limit: 5 }).length, 0) + assert.equal(await store.finish(ctx, { status: 200 }), null) + assert.equal((await store.listNormal({ limit: 5 })).length, 0) }) -test('classifier debug diagnostics omit system and transcript bodies', () => { +test('classifier debug diagnostics omit system and transcript bodies', async () => { const store = tmpStore('debug') const ctx = store.start( { method: 'POST', headers: {}, socket: {} }, @@ -631,8 +683,8 @@ test('classifier debug diagnostics omit system and transcript bodies', () => { wire_observed: false, before: { max_tokens: 64 }, } - store.finish(ctx, { status: 200, classifier, inbound_body: body, outbound_body: body }) - const [debug] = store.listDebug({ limit: 1 }) + await store.finish(ctx, { status: 200, classifier, inbound_body: body, outbound_body: body }) + const [debug] = await store.listDebug({ limit: 1 }) assert.deepEqual(debug.classifier, classifier) assert.equal(debug.request_body_snapshot, null) assert.equal(debug.inbound_body, null) @@ -640,38 +692,38 @@ test('classifier debug diagnostics omit system and transcript bodies', () => { assert.equal(JSON.stringify(debug).includes('sensitive transcript'), false) }) -function logOne(store, extra = {}) { +async function logOne(store, extra = {}) { const ctx = store.start({ method: 'POST', headers: {}, socket: {} }, { pathName: extra.path || '/v1/messages' }) - return store.finish(ctx, { status: 200, ...extra }) + return await store.finish(ctx, { status: 200, ...extra }) } -test('summaries persist in sqlite across store re-open', () => { +test('summaries persist in PostgreSQL across store recreation', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-rlog-')) const s1 = new RequestLogStore({ dataDir: dir, mode: 'normal' }) - logOne(s1, { model: 'claude-haiku-4-5', api_key_id: 'key_p' }) + await logOne(s1, { model: 'claude-haiku-4-5', api_key_id: 'key_p' }) const s2 = new RequestLogStore({ dataDir: dir, mode: 'normal' }) - const listed = s2.listNormal({ limit: 10 }) + const listed = await s2.listNormal({ limit: 10 }) assert.equal(listed.length, 1) assert.equal(listed[0].api_key_id, 'key_p') - assert.equal(s2.snapshot().total_rows, 1) + assert.equal((await s2.snapshot()).total_rows, 1) }) -test('gate stats count every inbound request and keep the stored verdict', () => { +test('gate stats count every inbound request and keep the stored verdict', async () => { const store = tmpStore() // Plain successes: error_code and final_state stay NULL. - logOne(store, { intercept: gateVerdict({ kind: 'pass', by: 'jev' }) }) - logOne(store, { intercept: gateVerdict({ kind: 'pass', by: 'regex' }) }) - logOne(store, { path: '/v1/responses', intercept: gateVerdict({ kind: 'pass', by: 'regex' }) }) - logOne(store, { status: 401, error_code: 'invalid_api_key' }) - logOne(store, { + await logOne(store, { intercept: gateVerdict({ kind: 'pass', by: 'jev' }) }) + await logOne(store, { intercept: gateVerdict({ kind: 'pass', by: 'regex' }) }) + await logOne(store, { path: '/v1/responses', intercept: gateVerdict({ kind: 'pass', by: 'regex' }) }) + await logOne(store, { status: 401, error_code: 'invalid_api_key' }) + await logOne(store, { status: 403, via: 'distill-detect', final_state: 'distill_blocked', error_code: 'distill_blocked', intercept: gateVerdict({ kind: 'block', by: 'distill', keyword: 'Memory-stage-one extractor' }), }) - logOne(store, { + await logOne(store, { status: 503, via: 'refusal-guard', final_state: 'refusal_device', @@ -679,7 +731,7 @@ test('gate stats count every inbound request and keep the stored verdict', () => intercept: gateVerdict({ kind: 'block', by: 'refusal', keyword: 'device' }), }) - const stats = store.protocolEntryStats({ since: '1970-01-01T00:00:00.000Z' }) + const stats = await store.protocolEntryStats({ since: '1970-01-01T00:00:00.000Z' }) assert.equal(stats.total, 6) assert.equal(stats.blocked, 2) assert.equal(stats.passed, 4) @@ -689,27 +741,27 @@ test('gate stats count every inbound request and keep the stored verdict', () => assert.deepEqual(blockBy, { 'distill:Memory-stage-one extractor': 1, 'refusal:device': 1 }) }) -test('queryNormal filters + pagination + total', () => { +test('queryNormal filters + pagination + total', async () => { const store = tmpStore('normal') - logOne(store, { api_key_id: 'key_a', model: 'm1', input_tokens: 10, output_tokens: 5 }) - logOne(store, { api_key_id: 'key_a', model: 'm2', status: 500, error_code: 'upstream_error' }) - logOne(store, { api_key_id: 'key_b', model: 'm1', vm_id: 'vm-9' }) + await logOne(store, { api_key_id: 'key_a', model: 'm1', input_tokens: 10, output_tokens: 5 }) + await logOne(store, { api_key_id: 'key_a', model: 'm2', status: 500, error_code: 'upstream_error' }) + await logOne(store, { api_key_id: 'key_b', model: 'm1', vm_id: 'vm-9' }) - assert.equal(store.queryNormal({ api_key_id: 'key_a' }).total, 2) - assert.equal(store.queryNormal({ vm_id: 'vm-9' }).total, 1) - assert.equal(store.queryNormal({ model: 'm1' }).total, 2) - assert.equal(store.queryNormal({ status: 'error' }).total, 1) - assert.equal(store.queryNormal({ status: 'ok' }).total, 2) - assert.equal(store.queryNormal({ status: 'error', include_muted: true }).total, 1) - assert.equal(store.queryNormal({ q: 'upstream' }).total, 1) - const page = store.queryNormal({ limit: 2, offset: 2 }) + assert.equal((await store.queryNormal({ api_key_id: 'key_a' })).total, 2) + assert.equal((await store.queryNormal({ vm_id: 'vm-9' })).total, 1) + assert.equal((await store.queryNormal({ model: 'm1' })).total, 2) + assert.equal((await store.queryNormal({ status: 'error' })).total, 1) + assert.equal((await store.queryNormal({ status: 'ok' })).total, 2) + assert.equal((await store.queryNormal({ status: 'error', include_muted: true })).total, 1) + assert.equal((await store.queryNormal({ q: 'upstream' })).total, 1) + const page = await store.queryNormal({ limit: 2, offset: 2 }) assert.equal(page.total, 3) assert.equal(page.items.length, 1) }) -test('windowStats computes ttft percentiles, sla and qps', () => { +test('windowStats computes ttft percentiles, sla and qps', async () => { const store = tmpStore('normal') - logOne(store, { + await logOne(store, { status: 200, first_token_ms: 100, duration_ms: 200, @@ -719,7 +771,7 @@ test('windowStats computes ttft percentiles, sla and qps', () => { cache_creation_tokens: 10, model: 'claude-sonnet-5', }) - logOne(store, { + await logOne(store, { status: 200, first_token_ms: 300, duration_ms: 400, @@ -729,8 +781,8 @@ test('windowStats computes ttft percentiles, sla and qps', () => { cache_creation_tokens: 0, model: 'claude-sonnet-5', }) - logOne(store, { status: 503, error_code: 'overloaded', duration_ms: 50, model: 'claude-fable-5' }) - const w = store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) + await logOne(store, { status: 503, error_code: 'overloaded', duration_ms: 50, model: 'claude-fable-5' }) + const w = await store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) assert.equal(w.cache_read_tokens, 80) assert.equal(w.prompt_tokens, 110) assert.ok(Math.abs(w.cache_hit_rate - 80 / 110) < 1e-9) @@ -747,21 +799,21 @@ test('windowStats computes ttft percentiles, sla and qps', () => { assert.equal(w.by_model.length, 2) assert.equal(w.error_collection.total, 1) assert.equal(w.error_collection.by_class[0].id, 'overloaded') - const filtered = store.queryNormal({ error_class: 'overloaded' }) + const filtered = await store.queryNormal({ error_class: 'overloaded' }) assert.equal(filtered.total, 1) assert.equal(filtered.items[0].error_label, '过载排队') }) -test('windowStats treats 429 quota/rate-limit as SLA success', () => { +test('windowStats treats 429 quota/rate-limit as SLA success', async () => { const store = tmpStore('normal') - logOne(store, { status: 200, first_token_ms: 80, duration_ms: 100, model: 'claude-sonnet-5' }) - logOne(store, { + await logOne(store, { status: 200, first_token_ms: 80, duration_ms: 100, model: 'claude-sonnet-5' }) + await logOne(store, { status: 429, error_code: 'upstream_rate_limit', error_message: '5h extra usage', model: 'claude-opus-5', }) - const w = store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) + const w = await store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) assert.equal(w.requests, 2) assert.equal(w.success, 2) assert.equal(w.errors, 0) @@ -771,22 +823,22 @@ test('windowStats treats 429 quota/rate-limit as SLA success', () => { assert.equal(w.error_collection.by_class[0].id, 'rate_limit') }) -test('windowStats treats ingress auth failures as SLA success like sub2api', () => { +test('windowStats treats ingress auth failures as SLA success like sub2api', async () => { const store = tmpStore('normal') - logOne(store, { status: 200, first_token_ms: 80, duration_ms: 100, model: 'claude-sonnet-5' }) - logOne(store, { + await logOne(store, { status: 200, first_token_ms: 80, duration_ms: 100, model: 'claude-sonnet-5' }) + await logOne(store, { status: 401, error_code: 'auth_failed', error_message: 'missing or invalid credentials', model: 'claude-sonnet-5', }) - logOne(store, { + await logOne(store, { status: 401, error_code: 'invalid_api_key', error_message: 'Invalid credentials', model: 'claude-opus-5', }) - const w = store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) + const w = await store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) assert.equal(w.requests, 3) assert.equal(w.success, 3) assert.equal(w.errors, 0) @@ -796,54 +848,59 @@ test('windowStats treats ingress auth failures as SLA success like sub2api', () assert.equal(w.error_collection.by_class[0].owner, 'client') }) -test('windowStats ignores client abort like sub2api ignore_context_canceled', () => { +test('windowStats ignores client abort like sub2api ignore_context_canceled', async () => { const store = tmpStore('normal') - logOne(store, { status: 200, first_token_ms: 80, duration_ms: 100, model: 'claude-sonnet-5' }) - logOne(store, { status: 200, error_code: 'client_cancelled', error_message: 'ECONNRESET', model: 'claude-sonnet-5' }) - const w = store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) + await logOne(store, { status: 200, first_token_ms: 80, duration_ms: 100, model: 'claude-sonnet-5' }) + await logOne(store, { + status: 200, + error_code: 'client_cancelled', + error_message: 'ECONNRESET', + model: 'claude-sonnet-5', + }) + const w = await store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) assert.equal(w.requests, 2) assert.equal(w.success, 2) assert.equal(w.errors, 0) assert.equal(w.error_collection.total, 0) - assert.equal(store.queryNormal({ status: 'error' }).total, 0) + assert.equal((await store.queryNormal({ status: 'error' })).total, 0) }) -test('aggregate buckets by day with token sums', () => { +test('aggregate buckets by day with token sums', async () => { const store = tmpStore('normal') - logOne(store, { input_tokens: 10, output_tokens: 5 }) - logOne(store, { input_tokens: 20, output_tokens: 15, status: 500, error_code: 'x' }) - const rows = store.aggregate({ bucket: 'day' }) + await logOne(store, { input_tokens: 10, output_tokens: 5 }) + await logOne(store, { input_tokens: 20, output_tokens: 15, status: 500, error_code: 'x' }) + const rows = await store.aggregate({ bucket: 'day' }) assert.equal(rows.length, 1) assert.equal(rows[0].requests, 2) assert.equal(rows[0].errors, 1) assert.equal(rows[0].input_tokens, 30) assert.equal(rows[0].output_tokens, 20) - const totals = store.totals() + const totals = await store.totals() assert.equal(totals.requests, 2) assert.equal(totals.input_tokens, 30) }) -test('cleanup removes rows older than retainDays', () => { +test('cleanup removes rows older than retainDays', async () => { const store = tmpStore('normal') // one fresh row - logOne(store, {}) + await logOne(store, {}) // one stale row injected directly - store.repo.insertSummary({ + await store.repo.insertSummary({ id: 'log_old', request_id: 'rid-old', ts: new Date(Date.now() - 30 * 86400_000).toISOString(), }) - store.repo.insertDebug('rid-old', new Date(Date.now() - 30 * 86400_000).toISOString(), { old: true }) - assert.equal(store.repo.count(), 2) - store.cleanup() - assert.equal(store.repo.count(), 1) - assert.equal(store.getDebug('rid-old'), null) + await store.repo.insertDebug('rid-old', new Date(Date.now() - 30 * 86400_000).toISOString(), { old: true }) + assert.equal(await store.repo.count(), 2) + await store.cleanup() + assert.equal(await store.repo.count(), 1) + assert.equal(await store.getDebug('rid-old'), null) }) -test('jsonl mirror writes legacy format when enabled', () => { +test('jsonl mirror writes legacy format when enabled', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-rlog-')) const store = new RequestLogStore({ dataDir: dir, mode: 'normal', jsonlMirror: true }) - logOne(store, { model: 'mm' }) + await logOne(store, { model: 'mm' }) const day = new Date().toISOString().slice(0, 10) const file = path.join(dir, 'request-logs', `${day}.jsonl`) assert.ok(fs.existsSync(file)) @@ -880,46 +937,46 @@ test('sanitizeRequestBodySnapshot omits image bytes', async () => { assert.equal(JSON.stringify(snap).includes(payload), false) }) -test('setConfig hot-updates mode', () => { +test('setConfig hot-updates mode', async () => { const s = tmpStore('normal') - s.setConfig({ mode: 'debug', retainDays: 3 }) - const snap = s.snapshot() + await s.setConfig({ mode: 'debug', retainDays: 3 }) + const snap = await s.snapshot() assert.equal(snap.mode, 'debug') assert.equal(snap.retain_days, 3) assert.equal(snap.debug_retain_days, 3) assert.equal(snap.max_mb, 2048) assert.deepEqual(snap.muted_error_classes, ['auth']) - s.setConfig({ mutedErrorClasses: [] }) - assert.deepEqual(s.snapshot().muted_error_classes, []) + await s.setConfig({ mutedErrorClasses: [] }) + assert.deepEqual((await s.snapshot()).muted_error_classes, []) }) -test('cleanup keeps 7-day summaries and drops 3-day-old debug', () => { +test('cleanup keeps 7-day summaries and drops 3-day-old debug', async () => { const store = tmpStore('normal') - store.setConfig({ retainDays: 7, debugRetainDays: 3, maxMb: 0 }) + await store.setConfig({ retainDays: 7, debugRetainDays: 3, maxMb: 0 }) const fiveDaysAgo = new Date(Date.now() - 5 * 86400_000).toISOString() - store.repo.insertSummary({ + await store.repo.insertSummary({ id: 'log_mid', request_id: 'rid-mid', ts: fiveDaysAgo, }) - store.repo.insertDebug('rid-mid', fiveDaysAgo, { old: true }) - store.cleanup() - assert.equal(store.repo.count(), 1) - assert.equal(store.repo.getDebug('rid-mid'), null) + await store.repo.insertDebug('rid-mid', fiveDaysAgo, { old: true }) + await store.cleanup() + assert.equal(await store.repo.count(), 1) + assert.equal(await store.repo.getDebug('rid-mid'), null) }) -test('cleanup trims oldest debug rows when over max_mb', () => { +test('cleanup trims oldest debug rows when over max_mb', async () => { const store = tmpStore('debug') - store.setConfig({ retainDays: 7, debugRetainDays: 7, maxMb: 1 }) + await store.setConfig({ retainDays: 7, debugRetainDays: 7, maxMb: 1 }) const blob = 'x'.repeat(400_000) for (let i = 0; i < 4; i++) { - store.repo.insertDebug(`rid-${i}`, new Date(Date.now() - (4 - i) * 1000).toISOString(), { blob }) + await store.repo.insertDebug(`rid-${i}`, new Date(Date.now() - (4 - i) * 1000).toISOString(), { blob }) } - assert.ok(store.repo.debugBytes() > 1024 * 1024) - store.cleanup() - assert.ok(store.repo.debugBytes() <= 1024 * 1024) - assert.equal(store.getDebug('rid-0'), null) - assert.ok(store.getDebug('rid-3')) + assert.ok((await store.repo.debugBytes()) > 1024 * 1024) + await store.cleanup() + assert.ok((await store.repo.debugBytes()) <= 1024 * 1024) + assert.equal(await store.getDebug('rid-0'), null) + assert.ok(await store.getDebug('rid-3')) }) test('normalizeLoggingConfig defaults debug retain to 3 and caps at retain_days', () => { @@ -933,7 +990,7 @@ test('normalizeLoggingConfig defaults debug retain to 3 and caps at retain_days' assert.equal(b.max_mb, 512) }) -test('ingress 401 stores presented key + ip and can be excluded from the list', () => { +test('ingress 401 stores presented key + ip and can be excluded from the list', async () => { const store = tmpStore('normal') const ctx = store.start( { @@ -943,28 +1000,28 @@ test('ingress 401 stores presented key + ip and can be excluded from the list', }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, ) - store.finish(ctx, { + await store.finish(ctx, { status: 401, error_code: 'invalid_api_key', error_message: 'Invalid credentials', api_key_presented: 'sk-bad-key-plain', }) - assert.equal(store.queryNormal({ status: 'error' }).total, 0) - const listed = store.queryNormal({ status: 'error', include_muted: true }) + assert.equal((await store.queryNormal({ status: 'error' })).total, 0) + const listed = await store.queryNormal({ status: 'error', include_muted: true }) assert.equal(listed.total, 1) assert.equal(listed.items[0].api_key_presented, 'sk-bad-key-plain') assert.equal(listed.items[0].ip, '203.0.113.9') assert.equal(listed.items[0].error_class, 'auth') - assert.equal(store.queryNormal({ status: 'error', error_class: 'auth' }).total, 1) - assert.equal(store.queryNormal({ status: 'error', exclude_error_class: 'auth' }).total, 0) - const hits = store.windowStats().error_collection.ingress_hits + assert.equal((await store.queryNormal({ status: 'error', error_class: 'auth' })).total, 1) + assert.equal((await store.queryNormal({ status: 'error', exclude_error_class: 'auth' })).total, 0) + const hits = (await store.windowStats()).error_collection.ingress_hits assert.equal(hits.length, 1) assert.equal(hits[0].api_key_presented, 'sk-bad-key-plain') assert.equal(hits[0].ip, '203.0.113.9') assert.equal(hits[0].count, 1) }) -test('exportRows + csv/jsonl keep presented key and ip', () => { +test('exportRows + csv/jsonl keep presented key and ip', async () => { const store = tmpStore('normal') const ctx = store.start( { @@ -974,13 +1031,13 @@ test('exportRows + csv/jsonl keep presented key and ip', () => { }, { pathName: '/v1/messages' }, ) - store.finish(ctx, { + await store.finish(ctx, { status: 401, error_code: 'invalid_api_key', error_message: 'Invalid credentials', api_key_presented: 'sk-export-plain', }) - const { items, total } = store.exportRows({ status: 'error', include_muted: true, limit: 100 }) + const { items, total } = await store.exportRows({ status: 'error', include_muted: true, limit: 100 }) assert.equal(total, 1) assert.equal(items[0].ip, '198.51.100.4') const csv = logsToCsv(items) @@ -991,14 +1048,14 @@ test('exportRows + csv/jsonl keep presented key and ip', () => { assert.match(jsonl, /sk-export-plain/) }) -test('finish persists caller and outbound sessions (capped) and only known reasoning efforts', () => { +test('finish persists caller and outbound sessions (capped) and only known reasoning efforts', async () => { const store = tmpStore('normal') - const write = (extra) => { + const write = async (extra) => { const ctx = store.start({ method: 'POST', headers: {}, socket: {} }, { pathName: '/v1/messages' }) - store.finish(ctx, { status: 200, ...extra }) - return store.repo.getByRequestId(ctx.request_id) + await store.finish(ctx, { status: 200, ...extra }) + return await store.repo.getByRequestId(ctx.request_id) } - const kept = write({ + const kept = await write({ session_id: ` ${'s'.repeat(250)} `, outbound_session_id: '11111111-2222-4333-8444-555555555555', reasoning_effort: 'xhigh', @@ -1006,7 +1063,7 @@ test('finish persists caller and outbound sessions (capped) and only known reaso assert.equal(kept.session_id, 's'.repeat(200)) assert.equal(kept.outbound_session_id, '11111111-2222-4333-8444-555555555555') assert.equal(kept.reasoning_effort, 'xhigh') - const dropped = write({ session_id: ' ', outbound_session_id: '', reasoning_effort: 'turbo' }) + const dropped = await write({ session_id: ' ', outbound_session_id: '', reasoning_effort: 'turbo' }) assert.equal(dropped.session_id, null) assert.equal(dropped.outbound_session_id, null) assert.equal(dropped.reasoning_effort, null) @@ -1037,7 +1094,7 @@ function fakeRes() { return res } -test('debug mode records the exact client response; normal mode leaves res untouched', () => { +test('debug mode records the exact client response; normal mode leaves res untouched', async () => { const store = new RequestLogStore({ dataDir: fs.mkdtempSync(path.join(os.tmpdir(), 'kin-rlog-')), mode: 'debug', @@ -1051,8 +1108,8 @@ test('debug mode records the exact client response; normal mode leaves res untou res.write('event: message_start\ndata: {"a":1}\n\n') res.write(Buffer.from('data: {"type":"content_block_delta","delta":{"text":"hello"}}\n\n')) res.end('data: [DONE]\n\n') - store.finish(ctx, { status: 200 }) - const rec = store.getDebug(ctx.request_id) + await store.finish(ctx, { status: 200 }) + const rec = await store.getDebug(ctx.request_id) assert.equal(rec.response.status, 200) assert.equal(rec.response.headers['content-type'], 'text/event-stream') assert.equal(rec.response.headers['set-cookie'], '***REDACTED***') @@ -1070,36 +1127,103 @@ test('debug mode records the exact client response; normal mode leaves res untou assert.equal(nctx.response, undefined) }) -test('debug response body masks credentials that fit inside the capture window', () => { +test('debug response body masks credentials that fit inside the capture window', async () => { const store = tmpStore('debug') const ctx = store.start({ method: 'POST', headers: {}, socket: {} }, { pathName: '/v1/messages' }) const res = fakeRes() store.tapResponse(ctx, res) res.end('{"echo":"sk-ant-oat01-ABCDEFGH12345678"}') - store.finish(ctx, { status: 200 }) - const body = store.getDebug(ctx.request_id).response.body + await store.finish(ctx, { status: 200 }) + const body = (await store.getDebug(ctx.request_id)).response.body assert.doesNotMatch(body, /ABCDEFGH12345678/) assert.match(body, /REDACTED/) }) -test('tenant log scope follows the slot owner at write time, not the current owner', () => { +test('tenant log scope follows the slot owner at write time, not the current owner', async () => { const store = tmpStore() - store.db.exec(` + await store.db.exec(` INSERT INTO users (id, email, username, password_hash) VALUES ('alice', 'a@x', 'alice', 'x'), ('bob', 'b@x', 'bob', 'x'); INSERT INTO vms (id, name, vm_json) VALUES ('vm-a', 'Slot A', '{}'); `) - logOne(store, { user_id: 'bob', vm_id: 'vm-a', error_code: 'upstream_error', status: 500 }) - store.db.exec(`UPDATE vms SET owner_user_id = 'alice', origin = 'admin_assigned' WHERE id = 'vm-a'`) - const testChat = logOne(store, { api_key_kind: 'master', vm_id: 'vm-a', error_code: 'upstream_error', status: 500 }) + await logOne(store, { user_id: 'bob', vm_id: 'vm-a', error_code: 'upstream_error', status: 500 }) + await store.db.exec(`UPDATE vms SET owner_user_id = 'alice', origin = 'admin_assigned' WHERE id = 'vm-a'`) + const testChat = await logOne(store, { + api_key_kind: 'master', + vm_id: 'vm-a', + error_code: 'upstream_error', + status: 500, + }) - const mine = store.queryNormal({ owner_user_id: 'alice' }) + const mine = await store.queryNormal({ owner_user_id: 'alice' }) assert.deepEqual( mine.items.map((r) => r.request_id), [testChat.request_id], ) assert.deepEqual( - store.windowStats({ owner_user_id: 'alice' }).error_collection.recent.map((r) => r.request_id), + (await store.windowStats({ owner_user_id: 'alice' })).error_collection.recent.map((r) => r.request_id), [testChat.request_id], ) - assert.equal(store.queryNormal({}).total, 2) + assert.equal((await store.queryNormal({})).total, 2) +}) + +test('stopping retention scheduling still waits for its running cleanup during flush', async (t) => { + const store = tmpStore() + let run + t.mock.method(globalThis, 'setInterval', (callback) => { + run = callback + return { unref() {} } + }) + t.mock.method(globalThis, 'clearInterval', () => {}) + let releaseCleanup + const cleanupPending = new Promise((resolve) => { + releaseCleanup = resolve + }) + let signalCleanup + const cleanupStarted = new Promise((resolve) => { + signalCleanup = resolve + }) + store.cleanup = async () => { + signalCleanup() + await cleanupPending + } + store.startCleanupScheduler() + run() + await cleanupStarted + store.stopCleanupScheduler() + let settled = false + let pending + try { + pending = store.flush().then(() => { + settled = true + }) + await Promise.resolve() + assert.equal(settled, false) + } finally { + releaseCleanup() + } + await pending + assert.equal(settled, true) +}) + +test('summary and debug write failures reject finish and response flush', async () => { + const store = tmpStore('debug') + const query = store.db.query.bind(store.db) + const failure = Object.assign(new Error('simulated persistence failure'), { code: 'TEST_PERSISTENCE_FAILED' }) + try { + store.db.query = (sql, params) => + sql.includes('INSERT INTO request_log_debug') ? Promise.reject(failure) : query(sql, params) + const debug = store.start({ method: 'POST', url: '/v1/messages', headers: {} }) + await assert.rejects(store.finish(debug, { status: 200 }), { code: 'TEST_PERSISTENCE_FAILED' }) + assert.ok(await store.repo.getByRequestId(debug.request_id), 'summary persists even when debug persistence fails') + + store.db.query = (sql, params) => + sql.includes('INSERT INTO usage_logs') ? Promise.reject(failure) : query(sql, params) + const { trackPendingWrite, flushPendingWrites } = await import('../../src/lib/db/pending-writes.mjs') + const normal = store.start({ method: 'POST', url: '/v1/messages', headers: {} }) + const pending = trackPendingWrite(store.finish(normal, { status: 200 })) + await assert.rejects(pending, { code: 'TEST_PERSISTENCE_FAILED' }) + await assert.rejects(flushPendingWrites(), { code: 'TEST_PERSISTENCE_FAILED' }) + } finally { + store.db.query = query + } }) diff --git a/test/unit/request-logs-export.test.mjs b/test/unit/request-logs-export.test.mjs index 86540db13..95b5f6461 100644 --- a/test/unit/request-logs-export.test.mjs +++ b/test/unit/request-logs-export.test.mjs @@ -9,23 +9,18 @@ import { UsageLogsView } from '../../src/lib/db/repos/usage-logs-view.mjs' import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-log-export-')) -const db = openDatabase({ dbPath: path.join(root, 'logs.db') }) -test.after(() => { - closeDatabase() +let db +test.after(async () => { + await closeDatabase() fs.rmSync(root, { recursive: true, force: true }) }) -db.exec(` - INSERT INTO users (id, email, username, password_hash) VALUES ('u1', 'a@x', 'alice', 'x'), ('u2', 'b@x', 'bob', 'x'); - INSERT INTO api_keys (id, name, key, user_id) VALUES ('k1', 'alice-key', 'sk-1', 'u1'), ('k2', 'bob-key', 'sk-2', 'u2'); -`) - let seq = 0 -function seed(rows) { +async function seed(rows) { const repo = new UsageLogsRepo(db) for (const r of rows) { seq++ - repo.insertSummary({ + await repo.insertSummary({ id: `log_${String(seq).padStart(8, '0')}`, request_id: `req-${seq}`, created_at: new Date(Date.UTC(2026, 0, 1) + seq * 1000).toISOString(), @@ -43,15 +38,24 @@ function seed(rows) { } } -seed([ - { session_id: 's1', status: 200 }, - { session_id: 's1', status: 500, error_code: 'upstream_error' }, - { session_id: 's1', status: 500, error_code: 'upstream_error', path: '/v1/chat/completions' }, - { session_id: 's2', status: 500, error_code: 'upstream_error' }, - { session_id: 's1', status: 200, attempt_count: 3, log_mode: 'debug' }, - { session_id: 's1', status: 401, error_code: 'invalid_api_key', vm_id: null }, - { session_id: 's9', user_id: 'u2', api_key_id: 'k2', status: 500, error_code: 'upstream_error' }, -]) +test.beforeEach(async () => { + db = await openDatabase() + seq = 0 + await db.exec(` + INSERT INTO users (id, email, username, password_hash) VALUES ('u1', 'a@x', 'alice', 'x'), ('u2', 'b@x', 'bob', 'x'); + INSERT INTO api_keys (id, name, key, user_id) VALUES ('k1', 'alice-key', 'sk-1', 'u1'), ('k2', 'bob-key', 'sk-2', 'u2'); +`) + + await seed([ + { session_id: 's1', status: 200 }, + { session_id: 's1', status: 500, error_code: 'upstream_error' }, + { session_id: 's1', status: 500, error_code: 'upstream_error', path: '/v1/chat/completions' }, + { session_id: 's2', status: 500, error_code: 'upstream_error' }, + { session_id: 's1', status: 200, attempt_count: 3, log_mode: 'debug' }, + { session_id: 's1', status: 401, error_code: 'invalid_api_key', vm_id: null }, + { session_id: 's9', user_id: 'u2', api_key_id: 'k2', status: 500, error_code: 'upstream_error' }, + ]) +}) async function exportLogs(query, { role = 'admin', userId = null, headers = {} } = {}) { const out = {} @@ -97,9 +101,9 @@ test('export honours the list filters the page sends (session, status code, endp const { rows, headers } = await exportLogs(query) assert.deepEqual(ids(rows), ['req-2']) assert.equal(headers['x-kin-export-total'], '1') - const listed = new UsageLogsView(db) - .listBatch({ session_id: 's1', status_code: '500', endpoint: '/v1/messages' }) - .logs.map((r) => r.requestId) + const listed = ( + await new UsageLogsView(db).listBatch({ session_id: 's1', status_code: '500', endpoint: '/v1/messages' }) + ).logs.map((r) => r.requestId) assert.deepEqual(ids(rows), listed) }) diff --git a/test/unit/resource-owner.test.mjs b/test/unit/resource-owner.test.mjs index cf61089c2..68c0860ca 100644 --- a/test/unit/resource-owner.test.mjs +++ b/test/unit/resource-owner.test.mjs @@ -10,6 +10,7 @@ import { countUserCreatedVms, filterVmsForPanel, ownerScopeFromApiKey, + ownerScopeFromRequest, vmMatchesOwnerScope, } from '../../src/lib/admin/resource-owner.mjs' @@ -63,3 +64,12 @@ test('unassign origin and proxy bind rules', () => { assert.equal(canBindProxyToVm({ owner_user_id: 'u1' }, {}, { role: 'admin' }), false) assert.equal(canBindProxyToVm({}, { owner_user_id: 'u1' }, { role: 'admin' }), true) }) + +test('request ownership waits for the asynchronous owner role lookup', async () => { + const request = { apiKeyKind: 'managed', apiKeyRecord: { user_id: 'owner' } } + assert.deepEqual(await ownerScopeFromRequest(request, { getById: async () => ({ role: 'admin' }) }), PLATFORM_SCOPE) + assert.deepEqual(await ownerScopeFromRequest(request, { getById: async () => ({ role: 'user' }) }), { + type: 'user', + userId: 'owner', + }) +}) diff --git a/test/unit/routing-concurrency-overrides.test.mjs b/test/unit/routing-concurrency-overrides.test.mjs index 4671d701f..3032763af 100644 --- a/test/unit/routing-concurrency-overrides.test.mjs +++ b/test/unit/routing-concurrency-overrides.test.mjs @@ -6,10 +6,10 @@ import path from 'node:path' import { AccountQuota } from '../../src/lib/pool/account-quota.mjs' import { createRoutingRuntime } from '../../src/lib/admin/routing-runtime.mjs' -function fixture(t, cap, { override = true, tier = 'default' } = {}) { +async function fixture(t, cap, { override = true, tier = 'default' } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-concurrency-')) const accountId = 'test-account' - const quota = new AccountQuota({ + const quota = await new AccountQuota({ dataDir: path.join(root, 'data'), accounts: [ { @@ -19,7 +19,7 @@ function fixture(t, cap, { override = true, tier = 'default' } = {}) { concurrency_override: override, }, ], - }) + }).init() quota.setAccountTier(accountId, tier) const vmPath = path.join(root, 'vms/vm-02.json') fs.mkdirSync(path.dirname(vmPath), { recursive: true }) @@ -36,8 +36,8 @@ function fixture(t, cap, { override = true, tier = 'default' } = {}) { accountQuota: quota, routingConfig: { tiers: {}, quota: {}, concurrency: {} }, }) - t.after(() => { - quota.db.close() + t.after(async () => { + await quota.db.close() fs.rmSync(root, { recursive: true, force: true }) }) return { @@ -49,8 +49,8 @@ function fixture(t, cap, { override = true, tier = 'default' } = {}) { } for (const cap of [0, 2, 16, 20]) { - test(`tier synchronization preserves a manual concurrency value of ${cap}`, (t) => { - const f = fixture(t, cap) + test(`tier synchronization preserves a manual concurrency value of ${cap}`, async (t) => { + const f = await fixture(t, cap) const applied = f.runtime.applyRoutingTierConcurrency({ default: { max_concurrency: 4 } }) assert.equal(applied.skipped, 1) assert.equal(f.policy().maxConcurrency, cap) @@ -59,8 +59,8 @@ for (const cap of [0, 2, 16, 20]) { assert.equal(Boolean(f.account().concurrency_override), true) }) - test(`global default updates preserve a manual concurrency value of ${cap}`, (t) => { - const f = fixture(t, cap) + test(`global default updates preserve a manual concurrency value of ${cap}`, async (t) => { + const f = await fixture(t, cap) f.runtime.applyRoutingConcurrency(4) assert.equal(f.policy().maxConcurrency, cap) assert.equal(f.policy().concurrencyOverride, true) @@ -70,8 +70,8 @@ for (const cap of [0, 2, 16, 20]) { } for (const tier of ['default', 'pro', 'max']) { - test(`inherited ${tier} concurrency receives the tier default`, (t) => { - const f = fixture(t, 2, { override: false, tier }) + test(`inherited ${tier} concurrency receives the tier default`, async (t) => { + const f = await fixture(t, 2, { override: false, tier }) const applied = f.runtime.applyRoutingTierConcurrency({ [tier]: { max_concurrency: 6 } }) assert.equal(applied[tier], 1) assert.equal(f.policy().maxConcurrency, 6) @@ -80,29 +80,29 @@ for (const tier of ['default', 'pro', 'max']) { }) } -test('global default updates still reach inherited concurrency', (t) => { - const f = fixture(t, 2, { override: false }) +test('global default updates still reach inherited concurrency', async (t) => { + const f = await fixture(t, 2, { override: false }) f.runtime.applyRoutingConcurrency(4) assert.equal(f.policy().maxConcurrency, 4) assert.equal(f.policy().concurrencyOverride, false) assert.equal(f.account().max_concurrency, 4) }) -test('tier synchronization does not rewrite an unchanged inherited VM record', (t) => { - const f = fixture(t, 4, { override: false }) +test('tier synchronization does not rewrite an unchanged inherited VM record', async (t) => { + const f = await fixture(t, 4, { override: false }) const before = fs.readFileSync(f.vmPath, 'utf8') const applied = f.runtime.applyRoutingTierConcurrency({ default: { max_concurrency: 4 } }) assert.equal(applied.default, 0) assert.equal(fs.readFileSync(f.vmPath, 'utf8'), before) }) -function rpmFixture(t, rpm, { override = true } = {}) { +async function rpmFixture(t, rpm, { override = true } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-rpm-')) const accountId = 'test-account' - const quota = new AccountQuota({ + const quota = await new AccountQuota({ dataDir: path.join(root, 'data'), accounts: [{ account_id: accountId, vm_id: 'vm-02', max_rpm: rpm }], - }) + }).init() const vmPath = path.join(root, 'vms/vm-02.json') fs.mkdirSync(path.dirname(vmPath), { recursive: true }) fs.writeFileSync( @@ -118,8 +118,8 @@ function rpmFixture(t, rpm, { override = true } = {}) { accountQuota: quota, routingConfig: { tiers: {}, quota: {}, concurrency: {} }, }) - t.after(() => { - quota.db.close() + t.after(async () => { + await quota.db.close() fs.rmSync(root, { recursive: true, force: true }) }) return { @@ -130,8 +130,8 @@ function rpmFixture(t, rpm, { override = true } = {}) { } } -test('tier RPM synchronization preserves a manual RPM value', (t) => { - const f = rpmFixture(t, 30) +test('tier RPM synchronization preserves a manual RPM value', async (t) => { + const f = await rpmFixture(t, 30) const applied = f.runtime.applyRoutingTierRpm({ default: { max_rpm: 10 } }) assert.equal(applied.skipped, 1) assert.equal(f.policy().maxRpm, 30) @@ -139,8 +139,8 @@ test('tier RPM synchronization preserves a manual RPM value', (t) => { assert.equal(f.account().max_rpm, 30) }) -test('tier RPM synchronization updates inherited RPM and skips unchanged records', (t) => { - const f = rpmFixture(t, 10, { override: false }) +test('tier RPM synchronization updates inherited RPM and skips unchanged records', async (t) => { + const f = await rpmFixture(t, 10, { override: false }) const before = fs.readFileSync(f.vmPath, 'utf8') assert.equal(f.runtime.applyRoutingTierRpm({ default: { max_rpm: 10 } }).default, 0) assert.equal(fs.readFileSync(f.vmPath, 'utf8'), before) diff --git a/test/unit/routing-runtime-load.test.mjs b/test/unit/routing-runtime-load.test.mjs index 82e08040f..414c5f9c1 100644 --- a/test/unit/routing-runtime-load.test.mjs +++ b/test/unit/routing-runtime-load.test.mjs @@ -50,7 +50,7 @@ test('applyVmSessionSlots updates only native admission policy', () => { } }) -test('persistRoutingPatch reconciles inherited Claude session slots and preserves overrides', () => { +test('persistRoutingPatch reconciles inherited Claude session slots and preserves overrides', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-routing-session-default-')) const vms = path.join(root, 'vms') const routingFile = path.join(root, 'routing.json') @@ -104,7 +104,7 @@ test('persistRoutingPatch reconciles inherited Claude session slots and preserve requestLog: { setConfig() {} }, }) - const applied = runtime.persistRoutingPatch({ inference: { session_slots: 4 } }) + const applied = await runtime.persistRoutingPatch({ inference: { session_slots: 4 } }) const inherited = JSON.parse(fs.readFileSync(path.join(vms, 'vm-inherited.json'), 'utf8')) const overridden = JSON.parse(fs.readFileSync(path.join(vms, 'vm-override.json'), 'utf8')) @@ -120,7 +120,7 @@ test('persistRoutingPatch reconciles inherited Claude session slots and preserve } }) -test('persistRoutingPatch writes compatibility cache_ttl into Claude kernel configs', () => { +test('persistRoutingPatch writes compatibility cache_ttl into Claude kernel configs', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-routing-cache-ttl-')) const vms = path.join(root, 'vms') const routingFile = path.join(root, 'routing.json') @@ -147,7 +147,7 @@ test('persistRoutingPatch writes compatibility cache_ttl into Claude kernel conf requestLog: { setConfig() {} }, }) - runtime.persistRoutingPatch({ compatibility: { cache_ttl: '5m' } }) + await runtime.persistRoutingPatch({ compatibility: { cache_ttl: '5m' } }) const kernel = JSON.parse(fs.readFileSync(path.join(vms, 'vm-claude', 'run', 'kernel.json'), 'utf8')) assert.equal(kernel.default_cache_ttl, '5m') @@ -157,7 +157,7 @@ test('persistRoutingPatch writes compatibility cache_ttl into Claude kernel conf } }) -test('persistRoutingPatch writes a changed stream idle timeout into Claude kernel configs', () => { +test('persistRoutingPatch writes a changed stream idle timeout into Claude kernel configs', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-routing-idle-')) const vms = path.join(root, 'vms') const routingFile = path.join(root, 'routing.json') @@ -182,11 +182,11 @@ test('persistRoutingPatch writes a changed stream idle timeout into Claude kerne requestLog: { setConfig() {} }, }) - const same = runtime.persistRoutingPatch({ failover: { stream_idle_timeout_ms: 600000 } }) + const same = await runtime.persistRoutingPatch({ failover: { stream_idle_timeout_ms: 600000 } }) assert.equal(same.kernel_persona, null) assert.equal(fs.existsSync(kernelFile), false) - const changed = runtime.persistRoutingPatch({ failover: { stream_idle_timeout_ms: 900000 } }) + const changed = await runtime.persistRoutingPatch({ failover: { stream_idle_timeout_ms: 900000 } }) assert.equal(changed.kernel_reload, 0) assert.equal(JSON.parse(fs.readFileSync(kernelFile, 'utf8')).idle_timeout_seconds, 900) } finally { @@ -194,7 +194,7 @@ test('persistRoutingPatch writes a changed stream idle timeout into Claude kerne } }) -test('persistRoutingPatch writes a changed request body limit into Claude kernel configs', () => { +test('persistRoutingPatch writes a changed request body limit into Claude kernel configs', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-routing-body-')) const vms = path.join(root, 'vms') const routingFile = path.join(root, 'routing.json') @@ -217,10 +217,10 @@ test('persistRoutingPatch writes a changed request body limit into Claude kernel }, requestLog: { setConfig() {} }, }) - const same = runtime.persistRoutingPatch({ failover: { stream_idle_timeout_ms: 180000 } }) + const same = await runtime.persistRoutingPatch({ failover: { stream_idle_timeout_ms: 180000 } }) assert.equal(same.kernel_persona, null) assert.equal(fs.existsSync(kernelFile), false) - const changed = runtime.persistRoutingPatch({ failover: { max_request_mib: 64 } }) + const changed = await runtime.persistRoutingPatch({ failover: { max_request_mib: 64 } }) assert.equal(changed.kernel_reload, 0) assert.equal(JSON.parse(fs.readFileSync(kernelFile, 'utf8')).max_request_bytes, 64 * 1024 * 1024) } finally { @@ -228,7 +228,7 @@ test('persistRoutingPatch writes a changed request body limit into Claude kernel } }) -test('persistRoutingPatch writes persona_preset into Claude kernel system_layout', () => { +test('persistRoutingPatch writes persona_preset into Claude kernel system_layout', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-routing-persona-kernel-')) const vms = path.join(root, 'vms') const routingFile = path.join(root, 'routing.json') @@ -259,7 +259,7 @@ test('persistRoutingPatch writes persona_preset into Claude kernel system_layout requestLog: { setConfig() {} }, }) - const applied = runtime.persistRoutingPatch({ compatibility: { persona_preset: 'zero' } }) + const applied = await runtime.persistRoutingPatch({ compatibility: { persona_preset: 'zero' } }) assert.equal(applied.kernel_persona.updated, 1) const kernel = JSON.parse(fs.readFileSync(path.join(vms, 'vm-claude', 'run', 'kernel.json'), 'utf8')) @@ -427,7 +427,7 @@ test('loadRoutingConfig normalizes a missing or invalid vm section', () => { } }) -test('persistRoutingPatch merges a valid vm patch', () => { +test('persistRoutingPatch merges a valid vm patch', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-routing-vm-persist-')) const routingFile = path.join(root, 'routing.json') const routingConfig = { concurrency: {}, tiers: {}, vm: { memory: '1g', vcpus: 2 } } @@ -435,7 +435,7 @@ test('persistRoutingPatch merges a valid vm patch', () => { fs.writeFileSync(routingFile, JSON.stringify(routingConfig)) try { const runtime = persistHarness(root, routingFile, routingConfig) - runtime.persistRoutingPatch({ vm: { memory: '4g', vcpus: 4, smbios: { product: 'Precision 5860' } } }) + await runtime.persistRoutingPatch({ vm: { memory: '4g', vcpus: 4, smbios: { product: 'Precision 5860' } } }) const saved = JSON.parse(fs.readFileSync(routingFile, 'utf8')) assert.equal(saved.vm.memory, '4g') assert.equal(saved.vm.vcpus, 4) @@ -471,7 +471,7 @@ test('persistRoutingPatch rewrites Claude worker.json when the upload interval c t.mock.method(console, 'warn', (line) => warned.push(String(line))) try { const runtime = persistHarness(root, routingFile, routingConfig) - const changed = runtime.persistRoutingPatch({ telemetry: { upload_interval_seconds: 3600 } }) + const changed = await runtime.persistRoutingPatch({ telemetry: { upload_interval_seconds: 3600 } }) assert.deepEqual(changed.telemetry_sync, { updated: 1, queued: 1, skipped: 0, failed: [] }) assert.equal(JSON.parse(fs.readFileSync(routingFile, 'utf8')).telemetry.upload_interval_seconds, 3600) assert.equal(worker('vm-claude').upload_interval_seconds, 3600) @@ -482,8 +482,11 @@ test('persistRoutingPatch rewrites Claude worker.json when the upload interval c const push = JSON.parse(warned.find((line) => line.includes('remote_slot_push_failed'))) assert.equal(push.vm_id, 'vm-node') assert.deepEqual(push.sets, ['run']) - assert.equal(runtime.persistRoutingPatch({ telemetry: { upload_interval_seconds: 3600 } }).telemetry_sync, null) - assert.equal(runtime.persistRoutingPatch({ sticky: {} }).telemetry_sync, null) + assert.equal( + (await runtime.persistRoutingPatch({ telemetry: { upload_interval_seconds: 3600 } })).telemetry_sync, + null, + ) + assert.equal((await runtime.persistRoutingPatch({ sticky: {} })).telemetry_sync, null) assert.equal(JSON.parse(fs.readFileSync(routingFile, 'utf8')).telemetry.upload_interval_seconds, 3600) } finally { fs.rmSync(root, { recursive: true, force: true }) @@ -517,7 +520,7 @@ test('PUT /api/panel/routing rejects an invalid vm patch before persist', async assert.match(response.body?.error?.message, /memory/) }) -test('persistRoutingPatch round-trips websocket gates and analytics', () => { +test('persistRoutingPatch round-trips websocket gates and analytics', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-routing-ws-persist-')) const routingFile = path.join(root, 'routing.json') const routingConfig = { concurrency: {}, tiers: {}, codex: { enabled: true } } @@ -525,7 +528,7 @@ test('persistRoutingPatch round-trips websocket gates and analytics', () => { fs.writeFileSync(routingFile, JSON.stringify(routingConfig)) try { const runtime = persistHarness(root, routingFile, routingConfig) - runtime.persistRoutingPatch({ + await runtime.persistRoutingPatch({ websocket: { enabled: false }, codex: { websocket: { enabled: false }, diff --git a/test/unit/seat-planner.test.mjs b/test/unit/seat-planner.test.mjs index 819e12bc1..66d94a6b1 100644 --- a/test/unit/seat-planner.test.mjs +++ b/test/unit/seat-planner.test.mjs @@ -473,21 +473,21 @@ test('seat stream frames carry per-VM seats, global queue and queue_max', async req.emit('close') }) -function gatedQuota(root) { +async function gatedQuota(root) { const tier = { limit_5h: 0.8, limit_7d: 0.8, max_concurrency: 4 } - return new AccountQuota({ + return await new AccountQuota({ dataDir: fs.mkdtempSync(path.join(root, 'quota-')), config: { quota: { block_on_5h: true, block_on_7d: true, critical_band: 0.05, gate_overshoot: 0.05 }, tiers: { default: tier, pro: tier, max: tier }, }, - }) + }).init() } -function gatedSetup(t, specs) { +async function gatedSetup(t, specs) { const root = project(specs) t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const quota = gatedQuota(root) + const quota = await gatedQuota(root) const pool = new PoolScheduler({ projectRoot: root, runtimeRepo: new RuntimeRepo(), @@ -508,7 +508,7 @@ function setUtil5h(quota, accountId, value) { } test('near the gate a seated device waits on its home, one call at a time, and never moves', async (t) => { - const { pool, quota } = gatedSetup(t, [ + const { pool, quota } = await gatedSetup(t, [ { id: 'vm-01', policy: { sessionSlots: 4 } }, { id: 'vm-02', policy: { sessionSlots: 4 } }, ]) @@ -534,7 +534,7 @@ test('near the gate a seated device waits on its home, one call at a time, and n }) test('at line plus overshoot a seated device leaves and its old seat frees at once', async (t) => { - const { pool, quota } = gatedSetup(t, [ + const { pool, quota } = await gatedSetup(t, [ { id: 'vm-01', policy: { sessionSlots: 4 } }, { id: 'vm-02', policy: { sessionSlots: 4 } }, ]) @@ -550,7 +550,7 @@ test('at line plus overshoot a seated device leaves and its old seat frees at on }) test('a new device takes an open VM before a draining one, and a draining one when nothing else is open', async (t) => { - const { pool, quota } = gatedSetup(t, [ + const { pool, quota } = await gatedSetup(t, [ { id: 'vm-01', policy: { sessionSlots: 1 } }, { id: 'vm-02', policy: { sessionSlots: 1 } }, ]) diff --git a/test/unit/security.test.mjs b/test/unit/security.test.mjs index 791912823..0e5eed3a0 100644 --- a/test/unit/security.test.mjs +++ b/test/unit/security.test.mjs @@ -38,9 +38,9 @@ test('panel session persistence stores only token hashes', async () => { process.env.KIN_DATA_DIR = dir try { const security = await import(`../../src/lib/core/security.mjs?session=${Date.now()}`) - const token = security.createPanelSession('admin', 60_000) + const token = await security.createPanelSession('admin', 60_000) assert.match(token, /^kin-panel-/) - assert.equal(security.verifyPanelSession(token).user, 'admin') + assert.equal((await security.verifyPanelSession(token)).user, 'admin') const file = path.join(dir, 'panel-sessions.json') const stored = fs.readFileSync(file, 'utf8') assert.doesNotMatch(stored, /kin-panel-/) @@ -51,7 +51,7 @@ test('panel session persistence stores only token hashes', async () => { [], ) security.revokePanelSession(token) - assert.equal(security.verifyPanelSession(token), null) + assert.equal(await security.verifyPanelSession(token), null) } finally { if (previous == null) delete process.env.KIN_DATA_DIR else process.env.KIN_DATA_DIR = previous @@ -65,14 +65,14 @@ test('verifyPanelSession sliding-renews exp when remaining TTL is under half', a process.env.KIN_DATA_DIR = dir try { const security = await import(`../../src/lib/core/security.mjs?slide=${Date.now()}`) - const token = security.createPanelSession('admin', 60_000) + const token = await security.createPanelSession('admin', 60_000) const file = path.join(dir, 'panel-sessions.json') const before = JSON.parse(fs.readFileSync(file, 'utf8')) const storedKey = Object.keys(before)[0] const expBeforeVerify = before[storedKey].exp const security2 = await import(`../../src/lib/core/security.mjs?slide2=${Date.now()}`) - assert.equal(security2.verifyPanelSession(token)?.user, 'admin') + assert.equal((await security2.verifyPanelSession(token))?.user, 'admin') const after = JSON.parse(fs.readFileSync(file, 'utf8')) const expAfterVerify = after[storedKey].exp assert.ok(expBeforeVerify <= Date.now() + 61_000) @@ -91,7 +91,7 @@ test('verifyPanelSession does not renew when more than half of TTL remains', asy process.env.KIN_DATA_DIR = dir try { const security = await import(`../../src/lib/core/security.mjs?norel=${Date.now()}`) - const token = security.createPanelSession('admin') + const token = await security.createPanelSession('admin') const file = path.join(dir, 'panel-sessions.json') const raw = JSON.parse(fs.readFileSync(file, 'utf8')) const storedKey = Object.keys(raw)[0] @@ -100,7 +100,7 @@ test('verifyPanelSession does not renew when more than half of TTL remains', asy fs.writeFileSync(file, JSON.stringify(raw)) const security2 = await import(`../../src/lib/core/security.mjs?norel2=${Date.now()}`) - assert.equal(security2.verifyPanelSession(token)?.user, 'admin') + assert.equal((await security2.verifyPanelSession(token))?.user, 'admin') const after = JSON.parse(fs.readFileSync(file, 'utf8')) assert.equal(after[storedKey].exp, raw[storedKey].exp) } finally { @@ -116,17 +116,17 @@ test('verifyPanelSession skips a second disk write when TTL was just renewed', a process.env.KIN_DATA_DIR = dir try { const security = await import(`../../src/lib/core/security.mjs?twice=${Date.now()}`) - const token = security.createPanelSession('admin', 60_000) + const token = await security.createPanelSession('admin', 60_000) const file = path.join(dir, 'panel-sessions.json') const security2 = await import(`../../src/lib/core/security.mjs?twice2=${Date.now()}`) - assert.ok(security2.verifyPanelSession(token)) + assert.ok(await security2.verifyPanelSession(token)) const afterFirst = JSON.parse(fs.readFileSync(file, 'utf8')) const storedKey = Object.keys(afterFirst)[0] const expAfterFirst = afterFirst[storedKey].exp const mtimeAfterFirst = fs.statSync(file).mtimeMs - assert.ok(security2.verifyPanelSession(token)) + assert.ok(await security2.verifyPanelSession(token)) const afterSecond = JSON.parse(fs.readFileSync(file, 'utf8')) assert.equal(afterSecond[storedKey].exp, expAfterFirst) assert.equal(fs.statSync(file).mtimeMs, mtimeAfterFirst) @@ -150,7 +150,7 @@ test('existing loose panel session file is tightened on load', async () => { try { const security = await import(`../../src/lib/core/security.mjs?tighten=${Date.now()}`) assert.equal(fs.statSync(file).mode & 0o777, 0o600) - assert.equal(security.verifyPanelSession('kin-panel-load').user, 'admin') + assert.equal((await security.verifyPanelSession('kin-panel-load')).user, 'admin') } finally { if (previous == null) delete process.env.KIN_DATA_DIR else process.env.KIN_DATA_DIR = previous diff --git a/test/unit/settings-state.test.mjs b/test/unit/settings-state.test.mjs new file mode 100644 index 000000000..f845911ca --- /dev/null +++ b/test/unit/settings-state.test.mjs @@ -0,0 +1,30 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb, withTransaction } from '../../src/lib/db/database.mjs' +import { RuntimeSettings } from '../../src/lib/db/settings-state.mjs' + +test('settings cache changes only when its transaction commits', async () => { + const db = getDb() + const settings = new RuntimeSettings(db) + await settings.set('runtime_cache_rollback', { version: 1 }) + await assert.rejects( + withTransaction(db, async () => { + await settings.set('runtime_cache_rollback', { version: 2 }) + throw new Error('synthetic rollback') + }), + /synthetic rollback/, + ) + assert.deepEqual(settings.get('runtime_cache_rollback'), { version: 1 }) + await assert.rejects( + withTransaction(db, async () => { + await settings.remove('runtime_cache_rollback') + throw new Error('synthetic rollback') + }), + /synthetic rollback/, + ) + assert.deepEqual(settings.get('runtime_cache_rollback'), { version: 1 }) + await withTransaction(db, async () => { + await settings.set('runtime_cache_rollback', { version: 3 }) + }) + assert.deepEqual(settings.get('runtime_cache_rollback'), { version: 3 }) +}) diff --git a/test/unit/sqlite-backup-conversion.test.mjs b/test/unit/sqlite-backup-conversion.test.mjs new file mode 100644 index 000000000..19671790a --- /dev/null +++ b/test/unit/sqlite-backup-conversion.test.mjs @@ -0,0 +1,55 @@ +import { test, afterEach } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import zlib from 'node:zlib' +import { spawnSync } from 'node:child_process' +import { createDatabase as sqlite } from '../../src/lib/db/sqlite-legacy.mjs' +let root +afterEach(() => { + if (root) fs.rmSync(root, { recursive: true, force: true }) +}) +function fixture() { + root = fs.mkdtempSync('/tmp/vm2api-convert-test-') + const old = sqlite({ dbPath: root + '/source.db' }) + old.prepare('INSERT INTO vms(id,vm_json) VALUES(?,?)').run('vm-fixture', '{"id":"vm-fixture"}') + old.close() + fs.writeFileSync(root + '/source.db.gz', zlib.gzipSync(fs.readFileSync(root + '/source.db'))) + fs.mkdirSync(root + '/vms') + fs.mkdirSync(root + '/config') + fs.writeFileSync(root + '/vms/vm-fixture.json', '{"id":"vm-fixture","name":"backup fixture"}') + fs.writeFileSync(root + '/config/routing.json', '{}') +} +test('old SQLite db-only backup converts to PG v2 only with companion files', async () => { + fixture() + const output = root + '/converted.tar.gz' + const missing = spawnSync( + process.execPath, + ['scripts/convert-sqlite-backup.mjs', '--source', root + '/source.db.gz', '--output', output], + { encoding: 'utf8', env: process.env }, + ) + assert.equal(missing.status, 1) + assert.match(missing.stderr, /requires --vm-dir/) + assert.equal(fs.existsSync(output), false) + const converted = spawnSync( + process.execPath, + [ + 'scripts/convert-sqlite-backup.mjs', + '--source', + root + '/source.db.gz', + '--output', + output, + '--vm-dir', + root + '/vms', + '--config-dir', + root + '/config', + ], + { encoding: 'utf8', env: process.env }, + ) + assert.equal(converted.status, 0, converted.stderr) + const manifest = JSON.parse(spawnSync('tar', ['-xOzf', output, './manifest.json'], { encoding: 'utf8' }).stdout) + assert.equal(manifest.engine, 'postgresql') + assert.equal(manifest.version, 2) + assert.deepEqual(manifest.includes, { db: true, vms: 1, config: 1 }) + assert.ok(manifest.sha256s['db/postgres.dump']) + assert.ok(manifest.sha256s['vms/vm-fixture.json']) +}) diff --git a/test/unit/sqlite-import.test.mjs b/test/unit/sqlite-import.test.mjs new file mode 100644 index 000000000..bc61d5f36 --- /dev/null +++ b/test/unit/sqlite-import.test.mjs @@ -0,0 +1,103 @@ +import { test, beforeEach, afterEach } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import crypto from 'node:crypto' +import { createDatabase as sqlite } from '../../src/lib/db/sqlite-legacy.mjs' +import { importSqlite } from '../../src/lib/db/sqlite-import.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +let dir, source +beforeEach(() => { + dir = fs.mkdtempSync('/tmp/vm2api-import-test-') + source = dir + '/source.db' +}) +afterEach(() => fs.rmSync(dir, { recursive: true, force: true })) +function digest() { + return crypto.createHash('sha256').update(fs.readFileSync(source)).digest('hex') +} + +test('offline import preserves source, credentials, bigint, runtime and legacy data, and sequence high-water', async () => { + const old = sqlite({ dbPath: source }) + old.exec( + `CREATE TABLE thinking_signature_owners(key TEXT PRIMARY KEY,account_id TEXT NOT NULL,created_at INTEGER NOT NULL)`, + ) + old.prepare('INSERT INTO settings(key,value) VALUES(?,?)').run('encrypted_fixture', '"enc:synthetic-ciphertext"') + old.prepare('INSERT INTO thinking_signature_owners VALUES(?,?,?)').run('signature', 'account', 9007199254740993n) + old.prepare("INSERT INTO redeem_codes(id,code,value) VALUES(100,'deleted-code',1)").run() + old.prepare('DELETE FROM redeem_codes WHERE id=100').run() + old.close() + const before = digest() + const db = getDb() + const report = await importSqlite({ source, db }) + assert.equal(digest(), before) + assert.equal(report.tables.thinking_signature_owners.count, 1) + assert.equal( + (await db.prepare('SELECT created_at FROM thinking_signature_owners').get()).created_at, + '9007199254740993', + ) + assert.equal( + (await db.prepare("SELECT value FROM settings WHERE key='encrypted_fixture'").get()).value, + '"enc:synthetic-ciphertext"', + ) + const next = await db.prepare("INSERT INTO redeem_codes(code,value) VALUES('new-code',1) RETURNING id").get() + assert.equal(next.id, 101) + await assert.rejects(importSqlite({ source, db }), /target is not empty/) +}) + +test('unmapped source schema is rejected without writing target', async () => { + const old = sqlite({ dbPath: source }) + old.exec('CREATE TABLE unknown_business(id TEXT)') + old.close() + await assert.rejects(importSqlite({ source, db: getDb() }), /unmapped SQLite table/) + assert.equal((await getDb().prepare('SELECT count(*) AS n FROM settings').get()).n, 0) +}) + +test('invalid source row rolls back all target data and seeds', async () => { + const old = sqlite({ dbPath: source }) + old.exec("INSERT INTO settings(key,value) VALUES(NULL,'bad')") + old.close() + await assert.rejects(importSqlite({ source, db: getDb() }), { code: '23502' }) + assert.equal((await getDb().prepare('SELECT count(*) AS n FROM settings').get()).n, 0) + assert.equal((await getDb().prepare('SELECT count(*) AS n FROM groups').get()).n, 2) +}) + +test('fractional scheduling weights survive SQLite import and PostgreSQL updates', async () => { + const old = sqlite({ dbPath: source }) + old.prepare('INSERT INTO accounts(id,load_factor) VALUES(?,?)').run('fractional', 1.25) + old.close() + const db = getDb() + await importSqlite({ source, db }) + assert.equal((await db.prepare("SELECT load_factor FROM accounts WHERE id='fractional'").get()).load_factor, 1.25) + const { AccountRuntimeRepo } = await import('../../src/lib/db/repos/account-runtime-repo.mjs') + const record = await new AccountRuntimeRepo(db).upsert({ account_id: 'fractional', vm_id: 'vm-01', weight: 2.75 }) + assert.equal(record.weight, 2.75) +}) + +test('fractional weights in the preserved legacy runtime table survive import', async () => { + const old = sqlite({ dbPath: source }) + old + .prepare('INSERT INTO account_runtime_states_legacy(account_id,vm_id,weight,updated_at) VALUES(?,?,?,?)') + .run('legacy-fractional', 'vm-01', 1.25, '2026-10-21T00:00:00.000Z') + old.close() + const db = getDb() + await importSqlite({ source, db }) + const row = await db + .prepare('SELECT weight FROM account_runtime_states_legacy WHERE account_id=$1') + .get('legacy-fractional') + assert.equal(row.weight, 1.25) +}) + +test('fractional API endpoint priorities survive import and store mutations', async () => { + const old = sqlite({ dbPath: source }) + old + .prepare('INSERT INTO api_endpoints(id,name,base_url,priority) VALUES(?,?,?,?)') + .run('fractional-endpoint', 'fractional', 'https://example.invalid', 1.25) + old.close() + const db = getDb() + await importSqlite({ source, db }) + const { ApiEndpointStore } = await import('../../src/lib/admin/api-endpoints.mjs') + const store = new ApiEndpointStore({ db }) + assert.equal((await store.getRaw('fractional-endpoint')).priority, 1.25) + const created = await store.create({ base_url: 'https://example.invalid', priority: 2.75 }) + assert.equal(created.priority, 2.75) + assert.equal((await store.update(created.id, { priority: 0.25 })).priority, 0.25) +}) diff --git a/test/unit/sqlite-legacy.test.mjs b/test/unit/sqlite-legacy.test.mjs new file mode 100644 index 000000000..8258be4df --- /dev/null +++ b/test/unit/sqlite-legacy.test.mjs @@ -0,0 +1,417 @@ +import { test, beforeEach, afterEach } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import crypto from 'node:crypto' +import { DatabaseSync } from 'node:sqlite' +import { + openDatabase, + closeDatabase, + getDb, + isDbOpen, + getDbPath, + applyMigrations, + resolveDbPath, + vacuumInto, + withTransaction, +} from '../../src/lib/db/sqlite-legacy.mjs' + +let tmp + +beforeEach(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-db-test-')) +}) + +afterEach(() => { + closeDatabase() + fs.rmSync(tmp, { recursive: true, force: true }) + delete process.env.KIN_DB_PATH +}) + +test('openDatabase creates db, applies migrations, WAL enabled', () => { + const db = openDatabase({ dataDir: tmp }) + assert.ok(isDbOpen()) + assert.equal(getDbPath(), path.join(tmp, 'kin.db')) + const mode = db.prepare('PRAGMA journal_mode').get() + assert.equal(String(Object.values(mode)[0]).toLowerCase(), 'wal') + const tables = db + .prepare("SELECT name FROM sqlite_master WHERE type='table'") + .all() + .map((r) => r.name) + for (const t of [ + 'settings', + 'api_keys', + 'accounts', + 'account_allocations', + 'sticky_sessions', + 'proxies', + 'vms', + 'usage_logs', + 'request_log_debug', + 'backup_records', + 'account_runtime', + 'request_attempts', + 'schema_migrations', + 'api_endpoints', + 'api_endpoint_keys', + 'api_endpoint_models', + 'users', + 'groups', + 'account_groups', + 'redeem_codes', + ]) { + assert.ok(tables.includes(t), `missing table ${t}`) + } + const keyCols = db + .prepare('PRAGMA table_info(api_keys)') + .all() + .map((r) => r.name) + assert.ok(keyCols.includes('category'), 'api_keys missing category') +}) + +test('migration 005 adds protocol usage alignment columns', () => { + const db = openDatabase({ dataDir: tmp }) + const cols = (table) => + db + .prepare(`PRAGMA table_info(${table})`) + .all() + .map((r) => r.name) + const reqCols = cols('usage_logs') + for (const c of [ + 'cache_creation_5m_tokens', + 'cache_creation_1h_tokens', + 'requested_model', + 'upstream_model', + 'model_mismatch', + 'first_token_ms', + 'stop_reason', + 'input_cost', + 'output_cost', + 'cache_read_cost', + 'cache_creation_cost', + 'total_cost', + 'pricing_model', + 'service_tier', + 'speed', + 'long_context', + ]) + assert.ok(reqCols.includes(c), `usage_logs missing ${c}`) + for (const c of ['cache_read_tokens', 'cache_creation_tokens']) { + assert.ok(cols('accounts').includes(c), `accounts missing ${c}`) + assert.ok(cols('api_keys').includes(c), `api_keys missing ${c}`) + } + // scheduling window columns now live on accounts (sub2api SSOT merge) + const accCols = cols('accounts') + for (const c of [ + 'rate_limited_at', + 'rate_limit_reset_at', + 'overload_until', + 'session_window_start', + 'session_window_end', + 'session_window_status', + 'schedulable', + 'priority', + ]) + assert.ok(accCols.includes(c), `accounts missing ${c}`) +}) + +test('pre-005 database upgrades in place and keeps old rows readable', () => { + // Build a DB with migrations 001–004 only, insert a legacy row, then upgrade. + const migSrc = path.resolve('src/lib/db/migrations') + const oldDir = path.join(tmp, 'old-migs') + fs.mkdirSync(oldDir, { recursive: true }) + for (const f of fs.readdirSync(migSrc)) { + if (f < '005') fs.copyFileSync(path.join(migSrc, f), path.join(oldDir, f)) + } + const dbPath = path.join(tmp, 'upgrade.db') + const legacy = new DatabaseSync(dbPath) + applyMigrations(legacy, { migrationsDir: oldDir }) + legacy + .prepare(` + INSERT INTO request_logs (id, request_id, ts, protocol, model, status, input_tokens, output_tokens) + VALUES ('log_old', 'rid_old', ?, 'anthropic.messages', 'claude-sonnet-5', 200, 7, 2) + `) + .run(new Date().toISOString()) + legacy.close() + + process.env.KIN_DB_PATH = dbPath + const db = openDatabase({ dbPath }) + const row = db.prepare('SELECT * FROM usage_logs WHERE request_id = ?').get('rid_old') + assert.equal(row.input_tokens, 7) + // migration 017 normalizes NULL token counters to 0 + assert.equal(row.cache_creation_5m_tokens, 0) + assert.equal(row.stop_reason, null) + // new rows can use the new columns + db.prepare(` + INSERT INTO usage_logs (id, request_id, created_at, protocol, model, status, upstream_model, stop_reason, first_token_ms) + VALUES ('log_new', 'rid_new', ?, 'anthropic.messages', 'claude-sonnet-5', 200, 'claude-sonnet-5', 'end_turn', 21) + `).run(new Date().toISOString()) + assert.equal( + db.prepare('SELECT stop_reason FROM usage_logs WHERE request_id = ?').get('rid_new').stop_reason, + 'end_turn', + ) +}) + +test('openDatabase is idempotent singleton; getDb throws when closed', () => { + const a = openDatabase({ dataDir: tmp }) + const b = openDatabase({ dataDir: path.join(tmp, 'other') }) + assert.equal(a, b) + closeDatabase() + assert.throws(() => getDb(), /not opened/) +}) + +test('migrations are idempotent across reopen', () => { + openDatabase({ dataDir: tmp }) + closeDatabase() + const db = openDatabase({ dataDir: tmp }) + const rows = db.prepare('SELECT version FROM schema_migrations').all() + assert.equal(rows.length, new Set(rows.map((r) => r.version)).size) +}) + +test('migration 037 normalizes OpenAI input including NULL route fields; SUM(prompt) holds', () => { + const migSrc = path.resolve('src/lib/db/migrations') + const oldDir = path.join(tmp, 'pre-037') + fs.mkdirSync(oldDir, { recursive: true }) + for (const f of fs.readdirSync(migSrc)) { + if (f < '037') fs.copyFileSync(path.join(migSrc, f), path.join(oldDir, f)) + } + const dbPath = path.join(tmp, 'bill037.db') + const db = new DatabaseSync(dbPath) + applyMigrations(db, { migrationsDir: oldDir }) + + const cacheModel = `LOWER(TRIM(COALESCE( + NULLIF(NULLIF(pricing_model, ''), 'unpriced'), NULLIF(upstream_model, ''), + NULLIF(model, ''), requested_model, '' + )))` + const openaiIds = [ + 'gpt-6-astra', + 'gpt-6.1-sol', + 'gpt-6-sol', + 'gpt-6-luna', + 'gpt-5.6-sol', + 'gpt-5.6-terra', + 'gpt-5.6-luna', + 'gpt-5.6', + 'gpt-5.5-pro', + 'gpt-5.5', + 'gpt-5.4-mini', + 'gpt-5.4-nano', + 'gpt-5.4-pro', + 'gpt-5.4', + 'gpt-5.3-codex', + 'gpt-5.2-pro', + 'gpt-5.2', + 'gpt-5.1', + 'gpt-5-mini', + 'gpt-5-nano', + 'gpt-5-pro', + 'gpt-5', + 'gpt-4.1-mini', + 'gpt-4.1-nano', + 'gpt-4.1', + 'gpt-4o-2024-05-13', + 'gpt-4o-mini', + 'gpt-4o', + 'o1-pro', + 'o1', + 'o3-pro', + 'o3-mini', + 'o3', + 'o4-mini', + 'gpt-4-turbo', + 'gpt-4', + 'gpt-3.5-turbo-instruct', + 'gpt-3.5-turbo-1106', + 'gpt-3.5-turbo', + 'gpt-5.1-codex-mini', + 'gpt-5.3-chat-latest', + 'gpt-5.6-cyber', + 'gpt-5.5-cyber', + 'chat-latest', + 'gpt-5-codex', + 'gpt-5.1-codex', + 'gpt-5.1-codex-max', + 'gpt-5-chat-latest', + 'gpt-5.1-chat-latest', + 'gpt-5.2-codex', + 'gpt-5.2-chat-latest', + ] + .map((id) => `'${id}'`) + .join(', ') + const oldPrompt = `CASE + WHEN via = 'api-kernel' AND protocol = 'anthropic.messages' AND stream = 0 + THEN IFNULL(input_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0) + WHEN via LIKE 'codex-%' + OR (${cacheModel} LIKE 'gpt%' AND ${cacheModel} NOT LIKE 'gpt-image%') + OR ${cacheModel} LIKE 'codex-%' + OR ${cacheModel} IN (${openaiIds}) + THEN IFNULL(input_tokens, 0) + ELSE IFNULL(input_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0) + END` + const newPrompt = 'IFNULL(input_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0)' + + const ins = db.prepare(` + INSERT INTO usage_logs ( + id, request_id, created_at, model, protocol, stream, via, + input_tokens, cache_read_tokens, cache_creation_tokens, output_tokens, upstream_model + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `) + const at = '2026-09-01T00:00:00.000Z' + ins.run('n-via', 'r-via', at, 'gpt-5.4', 'anthropic.messages', 0, null, 100, 80, 0, 2, null) + ins.run('n-proto', 'r-proto', at, 'gpt-5.4', null, 0, 'go-worker-pool', 100, 80, 0, 2, null) + ins.run('n-stream', 'r-stream', at, 'gpt-5.4', 'anthropic.messages', null, 'go-worker-pool', 100, 80, 0, 2, null) + ins.run('n-all', 'r-all', at, 'gpt-5.4', null, null, null, 100, 80, 0, 2, null) + ins.run('keep-api', 'r-api', at, 'gpt-5.4', 'anthropic.messages', 0, 'api-kernel', 20, 80, 0, 2, 'gpt-5.4') + ins.run('keep-claude', 'r-claude', at, 'claude-sonnet-5', 'anthropic.messages', 0, null, 20, 80, 0, 2, null) + ins.run('codex', 'r-codex', at, 'gpt-5.4', 'openai.responses', 1, 'codex-kernel', 100000, 80000, 0, 2000, null) + + const before = db.prepare(`SELECT SUM(${oldPrompt}) AS n FROM usage_logs`).get().n + applyMigrations(db, { migrationsDir: migSrc }) + const after = db.prepare(`SELECT SUM(${newPrompt}) AS n FROM usage_logs`).get().n + assert.equal(after, before) + + const cols = db + .prepare('PRAGMA table_info(usage_logs)') + .all() + .map((r) => r.name) + for (const c of ['web_search_requests', 'web_search_cost', 'price_source', 'price_version']) { + assert.ok(cols.includes(c), `missing ${c}`) + } + + const get = (id) => db.prepare('SELECT input_tokens, cache_read_tokens FROM usage_logs WHERE id = ?').get(id) + assert.equal(get('n-via').input_tokens, 20) + assert.equal(get('n-proto').input_tokens, 20) + assert.equal(get('n-stream').input_tokens, 20) + assert.equal(get('n-all').input_tokens, 20) + assert.equal(get('keep-api').input_tokens, 20) + assert.equal(get('keep-claude').input_tokens, 20) + assert.equal(get('codex').input_tokens, 20000) + // Earlier prunes were never recorded; existing rows only vouch for history from the oldest one. + const pruned = db.prepare("SELECT value FROM settings WHERE key = 'usage_logs_pruned_through'").get() + assert.equal(JSON.parse(pruned.value), at) + db.close() +}) + +test('checksum mismatch on applied migration throws', () => { + const migDir = path.join(tmp, 'migs') + fs.mkdirSync(migDir, { recursive: true }) + fs.writeFileSync(path.join(migDir, '001_a.sql'), 'CREATE TABLE t1 (a TEXT);') + const db = new DatabaseSync(path.join(tmp, 'x.db')) + applyMigrations(db, { migrationsDir: migDir }) + fs.writeFileSync(path.join(migDir, '001_a.sql'), 'CREATE TABLE t1 (a TEXT, b TEXT);') + assert.throws(() => applyMigrations(db, { migrationsDir: migDir }), /checksum mismatch/) + db.close() +}) + +test('CRLF-applied checksum of the same SQL is accepted and rewritten', () => { + const migDir = path.join(tmp, 'migs') + fs.mkdirSync(migDir, { recursive: true }) + const lf = 'CREATE TABLE t1 (a TEXT);\n' + const crlf = lf.replace(/\n/g, '\r\n') + fs.writeFileSync(path.join(migDir, '001_a.sql'), lf) + const db = new DatabaseSync(path.join(tmp, 'x.db')) + applyMigrations(db, { migrationsDir: migDir }) + const crlfHash = crypto.createHash('sha256').update(crlf).digest('hex') + db.prepare('UPDATE schema_migrations SET checksum = ? WHERE version = ?').run(crlfHash, '001') + assert.doesNotThrow(() => applyMigrations(db, { migrationsDir: migDir })) + const stored = db.prepare('SELECT checksum FROM schema_migrations WHERE version = ?').get('001').checksum + assert.notEqual(stored, crlfHash) + assert.equal(stored.length, 64) + db.close() +}) + +test('new migration files apply in order and record checksums', () => { + const migDir = path.join(tmp, 'migs') + fs.mkdirSync(migDir, { recursive: true }) + fs.writeFileSync(path.join(migDir, '001_a.sql'), 'CREATE TABLE t1 (a TEXT);') + const db = new DatabaseSync(path.join(tmp, 'x.db')) + let applied = applyMigrations(db, { migrationsDir: migDir }) + assert.deepEqual(applied, ['001_a.sql']) + fs.writeFileSync(path.join(migDir, '002_b.sql'), 'ALTER TABLE t1 ADD COLUMN b TEXT;') + applied = applyMigrations(db, { migrationsDir: migDir }) + assert.deepEqual(applied, ['002_b.sql']) + const rows = db.prepare('SELECT version, checksum FROM schema_migrations ORDER BY version').all() + assert.equal(rows.length, 2) + assert.ok(rows.every((r) => /^[0-9a-f]{64}$/.test(r.checksum))) + db.close() +}) + +test('failed migration rolls back and is not recorded', () => { + const migDir = path.join(tmp, 'migs') + fs.mkdirSync(migDir, { recursive: true }) + fs.writeFileSync(path.join(migDir, '001_bad.sql'), 'CREATE TABLE ok1 (a TEXT); THIS IS NOT SQL;') + const db = new DatabaseSync(path.join(tmp, 'x.db')) + assert.throws(() => applyMigrations(db, { migrationsDir: migDir }), /001_bad\.sql failed/) + const tables = db + .prepare("SELECT name FROM sqlite_master WHERE type='table'") + .all() + .map((r) => r.name) + assert.ok(!tables.includes('ok1'), 'rollback should drop partial table') + assert.equal(db.prepare('SELECT COUNT(*) c FROM schema_migrations').get().c, 0) + db.close() +}) + +test('KIN_DB_PATH env overrides path', () => { + const custom = path.join(tmp, 'custom', 'my.db') + process.env.KIN_DB_PATH = custom + assert.equal(resolveDbPath({ dataDir: tmp }), custom) + openDatabase({ dataDir: tmp }) + assert.equal(getDbPath(), custom) + assert.ok(fs.existsSync(custom)) +}) + +test('vacuumInto produces a consistent snapshot', () => { + const db = openDatabase({ dataDir: tmp }) + db.prepare('INSERT INTO settings (key, value, updated_at) VALUES (?, ?, ?)').run( + 'k1', + '"v1"', + new Date().toISOString(), + ) + const dest = path.join(tmp, 'backup', 'snap.db') + vacuumInto(db, dest) + const snap = new DatabaseSync(dest) + const row = snap.prepare('SELECT value FROM settings WHERE key = ?').get('k1') + assert.equal(row.value, '"v1"') + snap.close() +}) + +test('withTransaction commits and rolls back', () => { + const db = openDatabase({ dataDir: tmp }) + withTransaction(db, () => { + db.prepare("INSERT INTO settings (key, value) VALUES ('a', '1')").run() + }) + assert.ok(db.prepare("SELECT * FROM settings WHERE key='a'").get()) + assert.throws( + () => + withTransaction(db, () => { + db.prepare("INSERT INTO settings (key, value) VALUES ('b', '2')").run() + throw new Error('boom') + }), + /boom/, + ) + assert.equal(db.prepare("SELECT * FROM settings WHERE key='b'").get(), undefined) +}) + +test('withTransaction only runs inline for a real outer transaction', () => { + const db = openDatabase({ dataDir: tmp }) + db.exec('BEGIN') + withTransaction(db, () => { + db.prepare("INSERT INTO settings (key, value) VALUES ('nested', '1')").run() + }) + assert.equal(db.isTransaction, true) + db.exec('ROLLBACK') + assert.equal(db.prepare("SELECT * FROM settings WHERE key='nested'").get(), undefined) + + const closed = new DatabaseSync(':memory:') + closed.close() + let called = false + assert.throws( + () => + withTransaction(closed, () => { + called = true + }), + /not open|closed/i, + ) + assert.equal(called, false) +}) diff --git a/test/unit/statistics-repo.test.mjs b/test/unit/statistics-repo.test.mjs index fca5d355b..25883fc85 100644 --- a/test/unit/statistics-repo.test.mjs +++ b/test/unit/statistics-repo.test.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { closeDatabase, createDatabase, openDatabase } from '../../src/lib/db/database.mjs' +import { closeDatabase, getDb, openDatabase } from '../../src/lib/db/database.mjs' import { UsageLogsRepo } from '../../src/lib/db/repos/usage-logs-repo.mjs' import { StatisticsRepo, resolveWindow } from '../../src/lib/db/repos/statistics-repo.mjs' import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' @@ -11,26 +11,27 @@ import { codexBodyToAnthropicMessage } from '../../src/lib/protocol/codex-conver let seq = 0 -function seedUsers(db) { - db.exec(` +async function seedUsers(db) { + await db.exec(` INSERT INTO users (id, email, username, password_hash) VALUES ('u1', 'a@x', 'alice', 'x'), ('u2', 'b@x', 'bob', 'x'); INSERT INTO api_keys (id, name, key, user_id) VALUES ('k1', 'alice-key', 'sk-1', 'u1'), ('k2', 'bob-key', 'sk-2', 'u2'); INSERT INTO vms (id, name, vm_json) VALUES ('vm-a', 'Slot A', '{}'); `) } -function freshDb() { +async function freshDb() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-stats-')) - const db = createDatabase({ dbPath: path.join(dir, 'stats.db') }) - seedUsers(db) + const db = await getDb() + await db.exec('TRUNCATE users, api_keys, vms, usage_logs, request_attempts') + await seedUsers(db) return db } -function seed(db, rows) { +async function seed(db, rows) { const repo = new UsageLogsRepo(db) for (const r of rows) { seq++ - repo.insertSummary({ + await repo.insertSummary({ id: `log_${seq}`, request_id: `req-${seq}`, log_mode: 'normal', @@ -43,7 +44,7 @@ function seed(db, rows) { } } -test('day buckets follow local midnights across a DST change', () => { +test('day buckets follow local midnights across a DST change', async () => { const tz = 'America/New_York' const now = Date.parse('2026-03-09T15:00:00Z') // 11:00 EDT, the day after spring-forward const { buckets, resolution } = resolveWindow('7days', tz, now) @@ -53,12 +54,12 @@ test('day buckets follow local midnights across a DST change', () => { assert.equal(new Date(buckets[5]).toISOString(), '2026-03-08T05:00:00.000Z') assert.equal(new Date(buckets[6]).toISOString(), '2026-03-09T04:00:00.000Z') - const db = freshDb() - seed(db, [ + const db = await freshDb() + await seed(db, [ { created_at: '2026-03-09T03:30:00.000Z', total_cost: 1 }, // 23:30 EDT on Mar 8 { created_at: '2026-03-09T04:30:00.000Z', total_cost: 2 }, // 00:30 EDT on Mar 9 ]) - const data = new StatisticsRepo(db).statistics({ range: '7days', dimension: 'model', tz, now }) + const data = await new StatisticsRepo(db).statistics({ range: '7days', dimension: 'model', tz, now }) const cost = data.metrics.cost assert.equal(cost[5].date, '2026-03-08T05:00:00.000Z') assert.equal(cost[5].s0, 1) @@ -67,36 +68,36 @@ test('day buckets follow local midnights across a DST change', () => { assert.equal(data.since, '2026-03-03T05:00:00.000Z') }) -test('hourly buckets start at local midnight in half-hour zones; bad tz falls back to UTC', () => { - const db = freshDb() +test('hourly buckets start at local midnight in half-hour zones; bad tz falls back to UTC', async () => { + const db = await freshDb() const now = Date.parse('2026-05-01T00:00:00Z') // 05:30 in Kolkata - seed(db, [ + await seed(db, [ { created_at: '2026-04-30T18:29:59.000Z', total_cost: 100 }, // previous local day { created_at: '2026-04-30T19:29:00.000Z', total_cost: 1 }, // 00:59 local { created_at: '2026-04-30T19:30:00.000Z', total_cost: 2 }, // 01:00 local ]) const repo = new StatisticsRepo(db) - const data = repo.statistics({ range: 'today', dimension: 'model', tz: 'Asia/Kolkata', now }) + const data = await repo.statistics({ range: 'today', dimension: 'model', tz: 'Asia/Kolkata', now }) assert.equal(data.resolution, 'hour') assert.equal(data.metrics.cost.length, 6) assert.equal(data.metrics.cost[0].date, '2026-04-30T18:30:00.000Z') assert.equal(data.metrics.cost[0].s0, 1) assert.equal(data.metrics.cost[1].s0, 2) assert.equal(data.totals.cost, 3) - const utc = repo.statistics({ range: 'today', tz: 'Mars/Base', now }) + const utc = await repo.statistics({ range: 'today', tz: 'Mars/Base', now }) assert.equal(utc.tz, 'UTC') assert.equal(utc.since, '2026-05-01T00:00:00.000Z') }) -test('series keep the top 8 by cost, fold the rest into __others__, and name empty values', () => { - const db = freshDb() +test('series keep the top 8 by cost, fold the rest into __others__, and name empty values', async () => { + const db = await freshDb() const now = Date.parse('2026-05-01T12:00:00Z') const at = '2026-05-01T10:00:00.000Z' const rows = [] for (let i = 1; i <= 10; i++) rows.push({ created_at: at, model: `m${i}`, total_cost: i, input_tokens: 10 }) rows.push({ created_at: at, model: null, total_cost: 20 }) - seed(db, rows) - const data = new StatisticsRepo(db).statistics({ range: 'today', dimension: 'model', tz: 'UTC', now }) + await seed(db, rows) + const data = await new StatisticsRepo(db).statistics({ range: 'today', dimension: 'model', tz: 'UTC', now }) assert.deepEqual( data.series.map((s) => s.id), ['__none__', 'm10', 'm9', 'm8', 'm7', 'm6', 'm5', 'm4', '__others__'], @@ -114,28 +115,28 @@ test('series keep the top 8 by cost, fold the rest into __others__, and name emp assert.equal(data.totals.requests, 11) }) -test('deltas compare with the previous equal-length window', () => { - const db = freshDb() +test('deltas compare with the previous equal-length window', async () => { + const db = await freshDb() const now = Date.parse('2026-05-08T12:00:00Z') // 7days window = [2026-05-02, now); previous = same span ending 2026-05-02. - seed(db, [ + await seed(db, [ { created_at: '2026-05-05T00:00:00.000Z', total_cost: 3, input_tokens: 30 }, { created_at: '2026-05-06T00:00:00.000Z', total_cost: 3, input_tokens: 30 }, { created_at: '2026-04-28T00:00:00.000Z', total_cost: 2, input_tokens: 40 }, ]) - const data = new StatisticsRepo(db).statistics({ range: '7days', tz: 'UTC', now }) + const data = await new StatisticsRepo(db).statistics({ range: '7days', tz: 'UTC', now }) assert.equal(data.deltas.requests, 1) assert.equal(data.deltas.cost, 2) assert.equal(data.deltas.tokens, 0.5) - const empty = new StatisticsRepo(db).statistics({ range: 'today', tz: 'UTC', now }) + const empty = await new StatisticsRepo(db).statistics({ range: 'today', tz: 'UTC', now }) assert.equal(empty.deltas.requests, null) }) -test('leaderboard sorts by official cost, names entities, and caps the limit', () => { - const db = freshDb() +test('leaderboard sorts by official cost, names entities, and caps the limit', async () => { + const db = await freshDb() const now = Date.parse('2026-05-01T12:00:00Z') const at = '2026-05-01T09:00:00.000Z' - seed(db, [ + await seed(db, [ { created_at: at, user_id: 'u1', @@ -150,7 +151,7 @@ test('leaderboard sorts by official cost, names entities, and caps the limit', ( { created_at: at, user_id: null, api_key_id: null, total_cost: 0.5 }, ]) const repo = new StatisticsRepo(db) - const board = repo.leaderboard({ range: 'today', scope: 'user', tz: 'UTC', now }) + const board = await repo.leaderboard({ range: 'today', scope: 'user', tz: 'UTC', now }) assert.deepEqual( board.entries.map((e) => [e.id, e.name]), [ @@ -166,11 +167,14 @@ test('leaderboard sorts by official cost, names entities, and caps the limit', ( assert.equal(alice.successRate, 0.5) assert.equal(alice.totalActualCost, 2) assert.equal(alice.cacheHitRate, 0.2) - assert.equal(repo.leaderboard({ range: 'today', scope: 'key', limit: 1, tz: 'UTC', now }).entries[0].name, 'bob-key') - assert.equal(repo.leaderboard({ range: 'today', scope: 'key', limit: 500, tz: 'UTC', now }).entries.length, 3) + assert.equal( + (await repo.leaderboard({ range: 'today', scope: 'key', limit: 1, tz: 'UTC', now })).entries[0].name, + 'bob-key', + ) + assert.equal((await repo.leaderboard({ range: 'today', scope: 'key', limit: 500, tz: 'UTC', now })).entries.length, 3) }) -test('cache hit rate uses disjoint input plus cache buckets for every route', () => { +test('cache hit rate uses disjoint input plus cache buckets for every route', async () => { const now = Date.parse('2026-05-01T12:00:00Z') const at = '2026-05-01T09:00:00.000Z' const disjoint = { input_tokens: 10, cache_read_tokens: 80, cache_creation_tokens: 10 } @@ -197,27 +201,26 @@ test('cache hit rate uses disjoint input plus cache buckets for every route', () { name: 'missing input counts', row: { input_tokens: null, output_tokens: 50 }, rate: 0 }, ] for (const { name, row, rate, display } of cases) { - const db = freshDb() + const db = await freshDb() try { - seed(db, [{ created_at: at, user_id: 'u1', ...row }]) + await seed(db, [{ created_at: at, user_id: 'u1', ...row }]) const repo = new StatisticsRepo(db) - const totals = repo.statistics({ range: 'today', tz: 'UTC', now }).totals - const entry = repo.leaderboard({ range: 'today', scope: 'user', tz: 'UTC', now }).entries[0] + const totals = (await repo.statistics({ range: 'today', tz: 'UTC', now })).totals + const entry = (await repo.leaderboard({ range: 'today', scope: 'user', tz: 'UTC', now })).entries[0] assert.equal(totals.cacheHitRate, rate, `totals: ${name}`) assert.equal(entry.cacheHitRate, rate, `leaderboard: ${name}`) if (display) assert.equal((entry.cacheHitRate * 100).toFixed(1), display) } finally { - db.close() } } }) -test('Codex 100k input / 80k cached / 2k out stores 20k input and totals 102k tokens', () => { +test('Codex 100k input / 80k cached / 2k out stores 20k input and totals 102k tokens', async () => { const now = Date.parse('2026-05-01T12:00:00Z') const at = '2026-05-01T09:00:00.000Z' - const db = freshDb() + const db = await freshDb() try { - seed(db, [ + await seed(db, [ { created_at: at, user_id: 'u1', @@ -229,20 +232,19 @@ test('Codex 100k input / 80k cached / 2k out stores 20k input and totals 102k to cache_creation_tokens: 0, }, ]) - const totals = new StatisticsRepo(db).statistics({ range: 'today', tz: 'UTC', now }).totals + const totals = (await new StatisticsRepo(db).statistics({ range: 'today', tz: 'UTC', now })).totals assert.equal(totals.inputTokens, 20_000) assert.equal(totals.tokens, 102_000) assert.notEqual(totals.tokens, 182_000) } finally { - db.close() } }) -test('API OpenAI replies converted to non-streaming Anthropic retain disjoint historical cache counts', () => { +test('API OpenAI replies converted to non-streaming Anthropic retain disjoint historical cache counts', async () => { const now = Date.parse('2026-05-01T12:00:00Z') const at = '2026-05-01T09:00:00.000Z' for (const cacheWrite of [0, 10]) { - const db = freshDb() + const db = await freshDb() try { const upstream = { model: 'gpt-5.4', @@ -256,7 +258,7 @@ test('API OpenAI replies converted to non-streaming Anthropic retain disjoint hi const message = codexBodyToAnthropicMessage(upstream) // The API backend persists result.body.usage first. For an assembled // Anthropic reply it has already subtracted cache reads and writes. - seed(db, [ + await seed(db, [ { created_at: at, user_id: 'u1', @@ -276,25 +278,25 @@ test('API OpenAI replies converted to non-streaming Anthropic retain disjoint hi actual_cost: 0.2, }, ]) - const stored = db.prepare('SELECT * FROM usage_logs').all() + const stored = await db.prepare('SELECT * FROM usage_logs').all() assert.equal(stored[0].input_tokens, 20 - cacheWrite) assert.equal(stored[0].cache_read_tokens, 80) assert.equal(stored[0].cache_creation_tokens, cacheWrite) const repo = new StatisticsRepo(db) const options = { range: 'today', tz: 'UTC', now, owner_user_id: 'u1' } - const totals = repo.statistics(options).totals + const totals = (await repo.statistics(options)).totals assert.equal(totals.cacheHitRate, 0.8) assert.equal(totals.inputTokens, 20 - cacheWrite) assert.equal(totals.cost, 0.25) for (const scope of ['user', 'key', 'model', 'vm']) { - const entry = repo.leaderboard({ ...options, scope }).entries[0] + const entry = (await repo.leaderboard({ ...options, scope })).entries[0] assert.equal(entry.cacheHitRate, 0.8, scope) assert.equal((entry.cacheHitRate * 100).toFixed(1), '80.0', scope) assert.equal(entry.totalActualCost, 0.2, scope) } // Statistics fix the denominator without rewriting historical usage or billing. - assert.deepEqual(db.prepare('SELECT * FROM usage_logs').all(), stored) - seed(db, [ + assert.deepEqual(await db.prepare('SELECT * FROM usage_logs').all(), stored) + await seed(db, [ { created_at: at, user_id: 'u1', @@ -306,21 +308,20 @@ test('API OpenAI replies converted to non-streaming Anthropic retain disjoint hi cache_read_tokens: 400, }, ]) - assert.equal(repo.statistics(options).totals.cacheHitRate, 480 / 1100) - assert.equal(repo.leaderboard({ ...options, scope: 'user' }).entries[0].cacheHitRate, 480 / 1100) + assert.equal((await repo.statistics(options)).totals.cacheHitRate, 480 / 1100) + assert.equal((await repo.leaderboard({ ...options, scope: 'user' })).entries[0].cacheHitRate, 480 / 1100) } finally { - db.close() } } }) -test('mixed-provider cache rates weight each row by prompt tokens for every scope and owner', () => { - const db = freshDb() +test('mixed-provider cache rates weight each row by prompt tokens for every scope and owner', async () => { + const db = await freshDb() try { const now = Date.parse('2026-05-01T12:00:00Z') const at = '2026-05-01T09:00:00.000Z' const shared = { created_at: at, user_id: 'u1', api_key_id: 'k1', vm_id: 'vm-a', model: 'shared-alias' } - seed(db, [ + await seed(db, [ { ...shared, upstream_model: 'claude-sonnet-4', cache_read_tokens: 80, cache_creation_tokens: 20 }, { ...shared, via: 'codex-kernel', input_tokens: 500, cache_read_tokens: 400, cache_creation_tokens: 100 }, { created_at: at, user_id: 'u2', api_key_id: 'k2', input_tokens: 50, cache_read_tokens: 50 }, @@ -331,31 +332,31 @@ test('mixed-provider cache rates weight each row by prompt tokens for every scop const options = { range: 'today', tz: 'UTC', now } const expected = 480 / 1100 for (const scope of ['user', 'key', 'model', 'vm']) { - const entries = repo.leaderboard({ ...options, scope, owner_user_id: 'u1' }).entries + const entries = (await repo.leaderboard({ ...options, scope, owner_user_id: 'u1' })).entries assert.equal(entries.length, 1, scope) assert.equal(entries[0].cacheHitRate, expected, scope) } - const mine = repo.statistics({ ...options, owner_user_id: 'u1' }).totals + const mine = (await repo.statistics({ ...options, owner_user_id: 'u1' })).totals assert.equal(mine.cacheHitRate, expected) assert.equal(mine.cacheReadTokens, 480) assert.equal(mine.cacheCreationTokens, 120) // The existing total-token metric is intentionally separate from this fix. assert.equal(mine.tokens, 1100) - assert.equal(repo.statistics(options).totals.cacheHitRate, 530 / 1200) - const empty = repo.statistics({ ...options, owner_user_id: 'missing-user' }).totals + assert.equal((await repo.statistics(options)).totals.cacheHitRate, 530 / 1200) + const empty = (await repo.statistics({ ...options, owner_user_id: 'missing-user' })).totals assert.equal(empty.cacheHitRate, 0) } finally { - db.close() } }) test('panel routes owner-scope statistics for users and reject user/vm groupings', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-stats-route-')) - const db = openDatabase({ dbPath: path.join(dir, 'panel.db') }) + const db = await openDatabase({ dbPath: path.join(dir, 'panel.db') }) try { - seedUsers(db) + await db.exec('TRUNCATE users, api_keys, vms, usage_logs, request_attempts') + await seedUsers(db) const at = new Date(Date.now() - 1000).toISOString() - seed(db, [ + await seed(db, [ { created_at: at, user_id: 'u1', api_key_id: 'k1', total_cost: 1 }, { created_at: at, user_id: 'u2', api_key_id: 'k2', total_cost: 7 }, ]) @@ -392,6 +393,6 @@ test('panel routes owner-scope statistics for users and reject user/vm groupings assert.equal(all.status, 200) assert.equal(all.body.data.totals.cost, 8) } finally { - closeDatabase() + await closeDatabase() } }) diff --git a/test/unit/sticky-router.test.mjs b/test/unit/sticky-router.test.mjs index 75903d2c8..0da51bd73 100644 --- a/test/unit/sticky-router.test.mjs +++ b/test/unit/sticky-router.test.mjs @@ -16,8 +16,8 @@ function tmpDir(prefix = 'kin-sticky-') { return fs.mkdtempSync(path.join(os.tmpdir(), prefix)) } -test('string and header parent ids stay on one family and do not cross API keys', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('string and header parent ids stay on one family and do not cross API keys', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const req = { apiKeyRecord: { id: 'key-a' }, headers: {} } const other = { apiKeyRecord: { id: 'key-b' }, headers: {} } const child = { @@ -56,8 +56,8 @@ test('a declared child without parent or root is rejected as a relation', () => assert.equal(childDeclaredWithoutParent({ metadata: { user_id: { device_id: 'dev' } } }, {}), false) }) -test('bind + resolve + hits increment', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('bind + resolve + hits increment', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() r.bind('conv-1', { accountId: 'acc-1', vmId: 'vm-1', sessionId: 'sess-9' }) const hit = r.resolve('conv-1') assert.equal(hit.accountId, 'acc-1') @@ -69,8 +69,8 @@ test('bind + resolve + hits increment', () => { assert.equal(r.stats().sessions['conv-1'].session_id, 'sess-9') }) -test('bind keeps outbound session on the same VM and overwrites after a VM change', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('bind keeps outbound session on the same VM and overwrites after a VM change', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() r.bind('conv-s', { accountId: 'acc-1', vmId: 'vm-1', sessionId: 'S1' }) r.bind('conv-s', { accountId: 'acc-1', vmId: 'vm-1', sessionId: 'S2' }) assert.equal(r.resolve('conv-s').sessionId, 'S1') @@ -86,8 +86,8 @@ test('bind keeps outbound session on the same VM and overwrites after a VM chang assert.equal(r.resolve('conv-s').sessionId, 'S4') }) -test('bind stays locked when both VM and account differ', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('bind stays locked when both VM and account differ', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() r.bind('conv-lock', { accountId: 'acc-1', vmId: 'vm-1', sessionId: 'S1' }) r.bind('conv-lock', { accountId: 'acc-2', vmId: 'vm-2', sessionId: 'S2' }) assert.equal(r.resolve('conv-lock').accountId, 'acc-1') @@ -95,51 +95,54 @@ test('bind stays locked when both VM and account differ', () => { assert.equal(r.resolve('conv-lock').sessionId, 'S1') }) -test('mergeStickyConfig normalizes outbound_session to rebuild or passthrough', () => { +test('mergeStickyConfig normalizes outbound_session to rebuild or passthrough', async () => { assert.equal(mergeStickyConfig({}).outbound_session, 'rebuild') assert.equal(mergeStickyConfig({ sticky: {} }).outbound_session, 'rebuild') assert.equal(mergeStickyConfig({ sticky: { outbound_session: 'rebuild' } }).outbound_session, 'rebuild') assert.equal(mergeStickyConfig({ sticky: { outbound_session: 'passthrough' } }).outbound_session, 'passthrough') assert.equal(mergeStickyConfig({ sticky: { outbound_session: 'other' } }).outbound_session, 'rebuild') - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() assert.equal(r.config.outbound_session, 'rebuild') r.reloadConfig({ sticky: { enabled: true, outbound_session: 'passthrough' } }) assert.equal(r.config.outbound_session, 'passthrough') }) -test('expired sessions purge on resolve/stats', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: -1 } } }) +test('expired sessions purge on resolve/stats', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: -1 } } }).init() r.bind('conv-2', { accountId: 'a', vmId: 'v' }) assert.equal(r.resolve('conv-2'), null) assert.equal(r.stats().active_sessions, 0) }) -test('disabled sticky returns null and binds nothing', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: false } } }) +test('disabled sticky returns null and binds nothing', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: false } } }).init() r.bind('conv-3', { accountId: 'a', vmId: 'v' }) assert.equal(r.resolve('conv-3'), null) }) -test('sticky sessions persist across re-open', () => { +test('sticky sessions persist across re-open', async () => { const dir = tmpDir() const cfg = { sticky: { enabled: true, ttl_seconds: 3600 } } - const r1 = new StickyRouter({ dataDir: dir, config: cfg }) + const r1 = await new StickyRouter({ dataDir: dir, config: cfg }).init() r1.bind('conv-4', { accountId: 'acc-4', vmId: 'vm-4' }) - const r2 = new StickyRouter({ dataDir: dir, config: cfg }) + const r2 = await new StickyRouter({ dataDir: dir, config: cfg }).init() const hit = r2.resolve('conv-4') assert.ok(hit) assert.equal(hit.accountId, 'acc-4') }) -test('extractKey uses default header_keys when config omits them', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'conversation' } } }) +test('extractKey uses default header_keys when config omits them', async () => { + const r = await new StickyRouter({ + dataDir: tmpDir(), + config: { sticky: { enabled: true, mode: 'conversation' } }, + }).init() const key = r.extractKey({ headers: { 'x-session-id': 'sess-default' } }, {}) assert.equal(key, 'sess-default') }) -test('extractKey prefers metadata.user_id session over headers', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('extractKey prefers metadata.user_id session over headers', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const key = r.extractKey( { headers: { 'x-session-id': 'header-sess' } }, { metadata: { user_id: { session_id: 'meta-sess' } } }, @@ -147,8 +150,11 @@ test('extractKey prefers metadata.user_id session over headers', () => { assert.equal(key, 'meta-sess') }) -test('caller session wins over a persistable envelope', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'conversation' } } }) +test('caller session wins over a persistable envelope', async () => { + const r = await new StickyRouter({ + dataDir: tmpDir(), + config: { sticky: { enabled: true, mode: 'conversation' } }, + }).init() const a = r.extractPoolKey( { apiKeyRecord: { id: 'key_f041' }, headers: { 'thread-id': '01a0b947-aaaa' } }, { @@ -180,8 +186,11 @@ test('caller session wins over a persistable envelope', () => { assert.notEqual(a, b) }) -test('envelope without a session id stays one key per API key', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'conversation' } } }) +test('envelope without a session id stays one key per API key', async () => { + const r = await new StickyRouter({ + dataDir: tmpDir(), + config: { sticky: { enabled: true, mode: 'conversation' } }, + }).init() const body = { messages: [{ role: 'user', content: 'Persistable response items (JSON):\n[{"text":"任务"}]' }], } @@ -198,8 +207,11 @@ test('envelope without a session id stays one key per API key', () => { ) }) -test('persistable envelope keys stay isolated per API key', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'conversation' } } }) +test('persistable envelope keys stay isolated per API key', async () => { + const r = await new StickyRouter({ + dataDir: tmpDir(), + config: { sticky: { enabled: true, mode: 'conversation' } }, + }).init() const body = { messages: [{ role: 'user', content: 'Persistable response items (JSON):\n[{"text":"任务"}]' }], } @@ -207,8 +219,8 @@ test('persistable envelope keys stay isolated per API key', () => { assert.equal(r.extractKey({ apiKeyRecord: { id: 'key_b' } }, body), 'kkey_b:envelope') }) -test('ip mode does not switch envelope traffic onto the API-key envelope key', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'ip' } } }) +test('ip mode does not switch envelope traffic onto the API-key envelope key', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'ip' } } }).init() const key = r.extractKey( { apiKeyRecord: { id: 'key_f041' }, headers: { 'x-forwarded-for': '203.0.113.9' } }, { messages: [{ role: 'user', content: 'Persistable response items (JSON):\n[{"text":"任务"}]' }] }, @@ -216,8 +228,8 @@ test('ip mode does not switch envelope traffic onto the API-key envelope key', ( assert.equal(key, 'kkey_f041:ip:203.0.113.9') }) -test('plain first-user hash is unchanged without envelope', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('plain first-user hash is unchanged without envelope', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const a = r.extractKey( { apiKeyRecord: { id: 'key_f041' } }, { messages: [{ role: 'user', content: '同一段会话的第一句' }] }, @@ -227,8 +239,8 @@ test('plain first-user hash is unchanged without envelope', () => { assert.notEqual(a, b) }) -test('extractKey ignores x-client-request-id and hashes first user', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('extractKey ignores x-client-request-id and hashes first user', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const a = r.extractKey( { headers: { 'x-client-request-id': 'req-aaaa' } }, { messages: [{ role: 'user', content: '同一段会话的第一句' }] }, @@ -247,8 +259,8 @@ test('extractKey ignores x-client-request-id and hashes first user', () => { assert.equal(a, b) }) -test('later blocks of the first user message stay on one session slot', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('later blocks of the first user message stay on one session slot', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const req = { apiKeyRecord: { id: 'key_f041' }, headers: {} } const first = { messages: [ @@ -284,8 +296,8 @@ test('later blocks of the first user message stay on one session slot', () => { assert.notEqual(a, r.extractPoolKey(req, other, { platform: 'anthropic' })) }) -test('a shared system-reminder opener does not merge different conversations', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('a shared system-reminder opener does not merge different conversations', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const req = { apiKeyRecord: { id: 'key_f041' }, headers: {} } const opener = (question, extra = []) => ({ messages: [ @@ -310,19 +322,19 @@ test('a shared system-reminder opener does not merge different conversations', ( assert.equal(r.extractPoolKey(req, next, { platform: 'anthropic' }), a) }) -test('extractKey mode=ip uses forwarded address', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'ip' } } }) +test('extractKey mode=ip uses forwarded address', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'ip' } } }).init() const key = r.extractKey({ headers: { 'x-forwarded-for': '203.0.113.9, 10.0.0.1' } }, {}) assert.equal(key, 'ip:203.0.113.9') }) -test('extractKey mode=session isolates by API key', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'session' } } }) +test('extractKey mode=session isolates by API key', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'session' } } }).init() assert.equal(r.extractKey({ apiKeyRecord: { id: 4 } }, {}), 'k4:login') }) -test('extractOfficialFamilyKey binds parent and child hops by device_id', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('extractOfficialFamilyKey binds parent and child hops by device_id', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const parent = r.extractOfficialFamilyKey( { headers: { 'x-claude-code-session-id': 'parent-sess' } }, { metadata: { user_id: { device_id: 'aabbcc', session_id: 'parent-sess' } } }, @@ -342,8 +354,8 @@ test('extractOfficialFamilyKey binds parent and child hops by device_id', () => ) }) -test('parent and child session ids stay on separate slots', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('parent and child session ids stay on separate slots', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const parentReq = { headers: { 'user-agent': 'claude-cli/2.1.241 (external, sdk-cli)', 'x-claude-code-session-id': 'parent-sess' }, } @@ -388,8 +400,8 @@ function companionHaiku(sessionId, deviceId = '998dad9c1e3eccf11cd192c3919d4d1f' } } -test('haiku subagent keeps its own session and does not reuse the parent key', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('haiku subagent keeps its own session and does not reuse the parent key', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const device = '8cd2bdff61fcd056' const req = { apiKeyRecord: { id: 'key_f0419454c00d' }, headers: { 'user-agent': 'Go-http-client/1.1' } } const parentBody = { @@ -409,8 +421,8 @@ test('haiku subagent keeps its own session and does not reuse the parent key', ( assert.match(String(childKey), /1334bab2-94bb-4429-a694-b2fa3434b1f5/) }) -test('explicit parent session shares a family VM and keeps a separate session key', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('explicit parent session shares a family VM and keeps a separate session key', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const req = { apiKeyRecord: { id: 'key_family' }, headers: {} } const parentBody = { model: 'claude-opus-5-5', @@ -436,8 +448,8 @@ test('explicit parent session shares a family VM and keeps a separate session ke assert.notEqual(r.resolve(parentKey).sessionId, r.resolve(childKey).sessionId) }) -test('companion haiku sessions stay independent without an explicit parent', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('companion haiku sessions stay independent without an explicit parent', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const req = { apiKeyRecord: { id: 'key_side' }, headers: {} } const a = r.extractPoolKey(req, companionHaiku('sess-a'), { platform: 'anthropic' }) const b = r.extractPoolKey(req, companionHaiku('sess-b'), { platform: 'anthropic' }) @@ -446,8 +458,11 @@ test('companion haiku sessions stay independent without an explicit parent', () assert.doesNotMatch(String(b), /fam:/) }) -test('a stale parent and a real haiku chat do not absorb the companion rule', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 3600 } } }) +test('a stale parent and a real haiku chat do not absorb the companion rule', async () => { + const r = await new StickyRouter({ + dataDir: tmpDir(), + config: { sticky: { enabled: true, ttl_seconds: 3600 } }, + }).init() const device = 'dev-old' const req = { apiKeyRecord: { id: 'key_stale' }, headers: {} } const parentBody = { @@ -486,8 +501,8 @@ test('a stale parent and a real haiku chat do not absorb the companion rule', () assert.match(r.extractPoolKey(req, chat, { platform: 'anthropic' }), /chat-sess$/) }) -test('anthropic and openai sticky keys do not share a session', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('anthropic and openai sticky keys do not share a session', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const req = { headers: { 'x-session-id': 'same-session' } } const body = { metadata: { user_id: { session_id: 'same-session' } } } const claude = r.extractPoolKey(req, body, { platform: 'anthropic' }) @@ -500,8 +515,8 @@ test('anthropic and openai sticky keys do not share a session', () => { assert.equal(r.resolve(r.extractPoolKey(req, body, { platform: 'anthropic' })).vmId, 'vm-claude') }) -test('a caller session does not stick through a content fingerprint', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('a caller session does not stick through a content fingerprint', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const firstUser = [{ role: 'user', content: '同一段跨协议会话的首条消息' }] const anthropic = { metadata: { user_id: { session_id: 'anthropic-session' } }, @@ -521,8 +536,8 @@ test('a caller session does not stick through a content fingerprint', () => { assert.equal(r.resolve(r.extractPoolKey(req, openai)), null) }) -test('explicit session is the stable lock when concurrent turns have different first messages', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('explicit session is the stable lock when concurrent turns have different first messages', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const req = { apiKeyRecord: { id: 'key_lock' }, headers: { 'x-session-id': 'shared-session' } } const first = { messages: [{ role: 'user', content: 'first visible turn' }] } const second = { messages: [{ role: 'user', content: 'trimmed current turn' }] } @@ -532,23 +547,23 @@ test('explicit session is the stable lock when concurrent turns have different f assert.deepEqual(r.collectPoolKeys(req, second), ['kkey_lock:shared-session']) }) -test('provisional bind does not increment hits', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('provisional bind does not increment hits', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() r.bind('conv-p', { accountId: 'acc', vmId: 'vm-1' }, { countHit: false }) assert.equal(r.stats().sessions['conv-p'].hits, 0) r.bind('conv-p', { accountId: 'acc', vmId: 'vm-1' }) assert.equal(r.stats().sessions['conv-p'].hits, 1) }) -test('extractKey isolates the same session per API key', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('extractKey isolates the same session per API key', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const raw = { headers: { 'x-session-id': 'same-session' } } assert.equal(r.extractKey(raw, {}), 'same-session') assert.equal(r.extractKey({ ...raw, apiKeyRecord: { id: 7 } }, {}), 'k7:same-session') }) -test('canonical identity keys are not scoped to API keys', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('canonical identity keys are not scoped to API keys', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const reqA = { apiKeyRecord: { id: 'key-a' } } const reqB = { apiKeyRecord: { id: 'key-b' } } @@ -566,8 +581,8 @@ test('canonical identity keys are not scoped to API keys', () => { assert.equal(r.canonicalDeviceKey(''), null) }) -test('canonical identity keys ignore account_uuid changes', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('canonical identity keys ignore account_uuid changes', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const first = { device_id: 'device-same', account_uuid: 'account-a', session_id: 'session-same' } const second = { device_id: 'device-same', account_uuid: 'account-b', session_id: 'session-same' } @@ -575,8 +590,8 @@ test('canonical identity keys ignore account_uuid changes', () => { assert.equal(r.canonicalDeviceKey(first.device_id), r.canonicalDeviceKey(second.device_id)) }) -test('device affinity bind carries no session seat and can move to another VM', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('device affinity bind carries no session seat and can move to another VM', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() const key = r.canonicalDeviceKey('device-affinity') const first = r.bindDeviceAffinity(key, { accountId: 'acc-1', vmId: 'vm-01' }) assert.equal(first.generation, 1) @@ -596,8 +611,8 @@ test('device affinity bind carries no session seat and can move to another VM', assert.equal(hit.slotIndex, null) }) -test('migrateLegacyIdentity lazily writes canonical keys on a legacy hit', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('migrateLegacyIdentity lazily writes canonical keys on a legacy hit', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() const req = { apiKeyRecord: { id: 'key_legacy' }, headers: {} } const legacyKey = r.isolateKey('legacy-session', req) r.bind(legacyKey, { accountId: 'acc-legacy', vmId: 'vm-legacy', sessionId: 'out-legacy' }) @@ -623,8 +638,8 @@ test('migrateLegacyIdentity lazily writes canonical keys on a legacy hit', () => assert.equal(legacyHit.vmId, 'vm-legacy') }) -test('migrateLegacyIdentity does not overwrite an existing canonical session binding on conflict', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('migrateLegacyIdentity does not overwrite an existing canonical session binding on conflict', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() const req = { apiKeyRecord: { id: 'key_conflict' }, headers: {} } const sessionKey = r.canonicalSessionKey('conflict-session') // A prior request already bound the canonical session key to vm-a. @@ -649,8 +664,8 @@ test('migrateLegacyIdentity does not overwrite an existing canonical session bin assert.equal(legacyHit.accountId, 'acc-b') }) -test('migrateLegacyIdentity is a no-op without a resolvable legacy hit and deletes nothing', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('migrateLegacyIdentity is a no-op without a resolvable legacy hit and deletes nothing', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() r.bind('unrelated-conv', { accountId: 'acc-unrelated', vmId: 'vm-unrelated' }) const before = r.stats().active_sessions @@ -668,8 +683,8 @@ test('migrateLegacyIdentity is a no-op without a resolvable legacy hit and delet assert.equal(r.resolve('unrelated-conv').vmId, 'vm-unrelated') }) -test('migrateLegacyIdentity without a trusted device/session identity mints nothing', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('migrateLegacyIdentity without a trusted device/session identity mints nothing', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() const req = { apiKeyRecord: { id: 'key_notrust' }, headers: {} } const legacyKey = r.isolateKey('untrusted-session', req) r.bind(legacyKey, { accountId: 'acc-untrusted', vmId: 'vm-untrusted' }) @@ -679,8 +694,8 @@ test('migrateLegacyIdentity without a trusted device/session identity mints noth assert.equal(result.deviceKey, null) }) -test('migrateLegacyIdentity keeps a live device home VM', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('migrateLegacyIdentity keeps a live device home VM', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() const req = { apiKeyRecord: { id: 'key_home' }, headers: {} } r.bindDeviceAffinity(r.canonicalDeviceKey('device-home'), { accountId: 'acc-home', vmId: 'vm-home' }) const legacyKey = r.isolateKey('old-session', req) @@ -691,8 +706,8 @@ test('migrateLegacyIdentity keeps a live device home VM', () => { assert.equal(r.resolve('dev2:device-home').vmId, 'vm-home') }) -test('sessionPoolKeys uses one canonical key across API keys and keeps legacy aliases', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('sessionPoolKeys uses one canonical key across API keys and keeps legacy aliases', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() const body = { metadata: { user_id: JSON.stringify({ device_id: 'dev-s', session_id: 'sess-s' }) } } const reqA = { apiKeyRecord: { id: 'a' }, headers: {} } const reqB = { apiKeyRecord: { id: 'b' }, headers: {} } @@ -712,8 +727,8 @@ test('sessionPoolKeys uses one canonical key across API keys and keeps legacy al assert.match(r.familyPoolKey(reqA, 'sess-s'), /ka:family:sess-s$/) }) -test('unbind and unbindByAccount drop dead bindings', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('unbind and unbindByAccount drop dead bindings', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() r.bind('conv-dead', { accountId: 'acc-x', vmId: 'vm-02' }) r.bind('conv-other', { accountId: 'acc-y', vmId: 'vm-04' }) r.unbind('conv-dead') @@ -724,9 +739,9 @@ test('unbind and unbindByAccount drop dead bindings', () => { assert.equal(r.resolve('conv-other').vmId, 'vm-04') }) -test('proxy pool import/bind/config persist across re-open', () => { +test('proxy pool import/bind/config persist across re-open', async () => { const dir = tmpDir('kin-proxy-') - const pool = new ProxyPool({ dataDir: dir }) + const pool = await new ProxyPool({ dataDir: dir }).init() const res = pool.importLines('socks5://user:pass@10.0.0.1:1080\n10.0.0.2:1080\nbadline:xx\n10.0.0.1:1080:user:pass') assert.equal(res.added, 2) const id = pool.snapshot().proxies[0].id @@ -734,7 +749,7 @@ test('proxy pool import/bind/config persist across re-open', () => { pool.updateConfig({ probe_interval_min: 30, max_failures: 3 }) pool.stopScheduler() - const pool2 = new ProxyPool({ dataDir: dir }) + const pool2 = await new ProxyPool({ dataDir: dir }).init() const snap = pool2.snapshot() assert.equal(snap.totals.total, 2) assert.equal(snap.config.probe_interval_min, 30) @@ -745,9 +760,9 @@ test('proxy pool import/bind/config persist across re-open', () => { pool2.stopScheduler() }) -test('proxy remove + unbindVm persist', () => { +test('proxy remove + unbindVm persist', async () => { const dir = tmpDir('kin-proxy-') - const pool = new ProxyPool({ dataDir: dir }) + const pool = await new ProxyPool({ dataDir: dir }).init() pool.importLines('10.1.1.1:1080\n10.1.1.2:1080') const [a, b] = pool.snapshot().proxies.map((p) => p.id) pool.bind(a, 'vm-z') @@ -755,22 +770,22 @@ test('proxy remove + unbindVm persist', () => { pool.remove(b) pool.stopScheduler() - const pool2 = new ProxyPool({ dataDir: dir }) + const pool2 = await new ProxyPool({ dataDir: dir }).init() const snap = pool2.snapshot() assert.equal(snap.totals.total, 1) assert.equal(snap.proxies[0].bound_vm_id, null) pool2.stopScheduler() }) -test('disconnect_on_error config persists and runtime failure disables slot', () => { +test('disconnect_on_error config persists and runtime failure disables slot', async () => { const dir = tmpDir('kin-proxy-') const disabled = [] const disconnected = [] - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir: dir, onDisableVm: (vmId, reason, proxyId) => disabled.push({ vmId, reason, proxyId }), onDisconnectVm: (vmId, reason, proxyId) => disconnected.push({ vmId, reason, proxyId }), - }) + }).init() pool.importLines('10.2.2.2:1080') const id = pool.snapshot().proxies[0].id pool.bind(id, 'vm-err') @@ -779,16 +794,16 @@ test('disconnect_on_error config persists and runtime failure disables slot', () assert.equal(skipped.skipped, true) assert.equal(disconnected.length, 0) - const updated = pool.updateConfig({ disconnect_on_error: true, max_failures: 1 }) + const updated = pool.updateConfig({ enabled: false, disconnect_on_error: true, max_failures: 1 }) assert.equal(updated.ok, true) assert.equal(updated.config.disconnect_on_error, true) pool.stopScheduler() - const pool2 = new ProxyPool({ + const pool2 = await new ProxyPool({ dataDir: dir, onDisableVm: (vmId, reason, proxyId) => disabled.push({ vmId, reason, proxyId }), onDisconnectVm: (vmId, reason, proxyId) => disconnected.push({ vmId, reason, proxyId }), - }) + }).init() assert.equal(pool2.snapshot().config.disconnect_on_error, true) const reported = pool2.reportRuntimeFailure('vm-err', 'proxy_transport_failure') assert.equal(reported.ok, true) @@ -842,8 +857,8 @@ test('isShortProbeRequest leaves conversations alone', () => { assert.equal(isShortProbeRequest({ messages: [{ role: 'user', content: ' ' }] }), false) }) -test('replaceSession remints the outbound session on the same VM', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('replaceSession remints the outbound session on the same VM', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() r.bind('sess:a', { accountId: 'acc', vmId: 'vm-05', sessionId: 'old-session', slotIndex: 0 }) r.bind( 'sess:a', @@ -855,5 +870,5 @@ test('replaceSession remints the outbound session on the same VM', () => { r.bind('sess:a', { accountId: 'acc', vmId: 'vm-05' }, { countHit: false, replaceSession: true, clearSlot: true }) assert.equal(r.resolve('sess:a').sessionId, null) assert.equal(r.resolve('sess:a').slotIndex, null) - r.db?.close?.() + await r.db?.close?.() }) diff --git a/test/unit/tenant-billing.test.mjs b/test/unit/tenant-billing.test.mjs index 96b15ea29..3f2f0b32c 100644 --- a/test/unit/tenant-billing.test.mjs +++ b/test/unit/tenant-billing.test.mjs @@ -3,12 +3,12 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' import { UsageLogsRepo } from '../../src/lib/db/repos/usage-logs-repo.mjs' -function tmpRepo() { +async function tmpRepo() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-billing-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() return { dir, db, logs: new UsageLogsRepo(db) } } @@ -28,40 +28,44 @@ function row(partial) { } } -test('ownerBilling and query hide other tenants', () => { - const { logs, db } = tmpRepo() - db.prepare( - "INSERT INTO vms (id, name, vm_json, owner_user_id, origin) VALUES ('vm-a','a','{}','user-a','user_created')", - ).run() - db.prepare( - "INSERT INTO vms (id, name, vm_json, owner_user_id, origin) VALUES ('vm-b','b','{}','user-b','user_created')", - ).run() - logs.insertSummary( +test('ownerBilling and query hide other tenants', async () => { + const { logs, db } = await tmpRepo() + await db + .prepare( + "INSERT INTO vms (id, name, vm_json, owner_user_id, origin) VALUES ('vm-a','a','{}','user-a','user_created')", + ) + .run() + await db + .prepare( + "INSERT INTO vms (id, name, vm_json, owner_user_id, origin) VALUES ('vm-b','b','{}','user-b','user_created')", + ) + .run() + await logs.insertSummary( row({ id: 'l1', request_id: 'r1', vm_id: 'vm-a', user_id: 'user-a', total_cost: 1, actual_cost: 1 }), ) - logs.insertSummary( + await logs.insertSummary( row({ id: 'l2', request_id: 'r2', vm_id: 'vm-b', user_id: 'user-b', total_cost: 9, actual_cost: 9, status: 500 }), ) - const billed = logs.ownerBilling({ ownerUserId: 'user-a', groupBy: 'vm' }) + const billed = await logs.ownerBilling({ ownerUserId: 'user-a', groupBy: 'vm' }) assert.equal(billed.totals.requests, 1) assert.equal(billed.items.length, 1) assert.equal(billed.items[0].vm_id, 'vm-a') assert.equal(billed.items[0].cost_usd, 1) assert.equal(billed.items[0].official_cost_usd, 1) - const listed = logs.query({ owner_user_id: 'user-a' }) + const listed = await logs.query({ owner_user_id: 'user-a' }) assert.equal(listed.total, 1) assert.equal(listed.items[0].request_id, 'r1') - assert.equal(logs.belongsToOwner('r1', 'user-a'), true) - assert.equal(logs.belongsToOwner('r2', 'user-a'), false) + assert.equal(await logs.belongsToOwner('r1', 'user-a'), true) + assert.equal(await logs.belongsToOwner('r2', 'user-a'), false) }) -test('ownerBilling exposes official cost next to rate-multiplied cost', () => { - const { logs } = tmpRepo() - logs.insertSummary(row({ id: 'm1', request_id: 'rm1', vm_id: 'vm-x', total_cost: 1, actual_cost: 2 })) - const billed = logs.ownerBilling({ groupBy: 'vm' }) +test('ownerBilling exposes official cost next to rate-multiplied cost', async () => { + const { logs } = await tmpRepo() + await logs.insertSummary(row({ id: 'm1', request_id: 'rm1', vm_id: 'vm-x', total_cost: 1, actual_cost: 2 })) + const billed = await logs.ownerBilling({ groupBy: 'vm' }) assert.equal(billed.totals.cost_usd, 2) assert.equal(billed.totals.official_cost_usd, 1) assert.equal(billed.items[0].cost_usd, 2) diff --git a/test/unit/thinking-owner.test.mjs b/test/unit/thinking-owner.test.mjs index ff2faccc0..3384c2966 100644 --- a/test/unit/thinking-owner.test.mjs +++ b/test/unit/thinking-owner.test.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { DatabaseSync } from 'node:sqlite' +import { getDb } from '../../src/lib/db/database.mjs' import { createHandleProtocol } from '../../src/lib/protocol/handle-protocol.mjs' import { ThinkingOwnerStore, @@ -18,7 +18,7 @@ const UNKNOWN = 'sig-never-recorded-0123456789abcd' const SHORT = 'too-short' function store() { - return new ThinkingOwnerStore({ db: new DatabaseSync(':memory:') }) + return new ThinkingOwnerStore({ db: getDb() }) } function history() { @@ -40,12 +40,12 @@ function history() { } } -test('a signature stays with the account that first returned it', () => { +test('a signature stays with the account that first returned it', async () => { const owners = store() const response = { content: [{ type: 'thinking', thinking: 'own chain', signature: OWN }] } - assert.equal(recordThinkingOwners(response, 'account-a', owners), 1) - recordThinkingOwners(response, 'account-b', owners) - const out = filterThinkingForAccount(history(), 'account-a', owners) + assert.equal(await recordThinkingOwners(response, 'account-a', owners), 1) + await recordThinkingOwners(response, 'account-b', owners) + const out = await filterThinkingForAccount(history(), 'account-a', owners) const blocks = out.body.messages[1].content assert.equal(blocks[0].type, 'thinking') assert.equal(blocks[0].signature, OWN) @@ -62,15 +62,15 @@ test('a signature stays with the account that first returned it', () => { ) }) -test('the same foreign signature converts to the same text on a later hop', () => { +test('the same foreign signature converts to the same text on a later hop', async () => { const owners = store() - recordThinkingOwners( + await recordThinkingOwners( { content: [{ type: 'thinking', thinking: 'foreign chain', signature: FOREIGN }] }, 'account-b', owners, ) - const first = filterThinkingForAccount(history(), 'account-a', owners) - const second = filterThinkingForAccount(history(), 'account-a', owners) + const first = await filterThinkingForAccount(history(), 'account-a', owners) + const second = await filterThinkingForAccount(history(), 'account-a', owners) assert.deepEqual(first.body.messages[1].content[1], second.body.messages[1].content[1]) assert.equal( thinkingSignatureKey({ type: 'thinking', signature: FOREIGN }), @@ -78,23 +78,23 @@ test('the same foreign signature converts to the same text on a later hop', () = ) }) -test('an unreachable database leaves the history untouched', () => { +test('an unreachable database leaves the history untouched', async () => { const body = history() - const closed = new DatabaseSync(':memory:') - closed.close() + const closed = getDb() + await closed.close() const owners = new ThinkingOwnerStore({ db: closed }) - const out = filterThinkingForAccount(body, 'account-a', owners) + const out = await filterThinkingForAccount(body, 'account-a', owners) assert.equal(out.body, body) assert.equal(out.kept, 4) assert.equal(out.converted, 0) - assert.equal(recordThinkingOwners(body.messages[1], 'account-a', owners), 4) + assert.equal(await recordThinkingOwners(body.messages[1], 'account-a', owners), 4) }) const SERVED = 'sig-served-by-account-a-0123456789' test('a handler records the served signature and the next account replays it as text', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-thinking-owner-')) - const owners = new ThinkingOwnerStore({ db: new DatabaseSync(':memory:') }) + const owners = new ThinkingOwnerStore({ db: getDb() }) const seen = [] const handler = createHandleProtocol({ json: (_res, status) => status, @@ -182,10 +182,10 @@ test('a handler records the served signature and the next account replays it as fs.rmSync(root, { recursive: true, force: true }) } }) -test('dropping the only signed block merges the neighbouring turns', () => { +test('dropping the only signed block merges the neighbouring turns', async () => { const owners = store() - recordThinkingOwners({ content: [{ type: 'thinking', thinking: '', signature: FOREIGN }] }, 'account-b', owners) - const out = filterThinkingForAccount( + await recordThinkingOwners({ content: [{ type: 'thinking', thinking: '', signature: FOREIGN }] }, 'account-b', owners) + const out = await filterThinkingForAccount( { messages: [ { role: 'user', content: 'before' }, diff --git a/test/unit/usage-logs-repo.test.mjs b/test/unit/usage-logs-repo.test.mjs index 67bc83f76..9464f1ff2 100644 --- a/test/unit/usage-logs-repo.test.mjs +++ b/test/unit/usage-logs-repo.test.mjs @@ -3,17 +3,17 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' import { UsageLogsRepo } from '../../src/lib/db/repos/usage-logs-repo.mjs' import { codexBodyToAnthropicMessage } from '../../src/lib/protocol/codex-convert.mjs' import { estimateWindow } from '../../src/lib/admin/window-estimate.mjs' let seq = 0 -function freshDb() { +async function freshDb() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-ulrepo-')) - const db = createDatabase({ dbPath: path.join(dir, 'logs.db') }) - db.exec(` + const db = await getDb() + await db.exec(` INSERT INTO users (id, email, username, password_hash) VALUES ('u1', 'a@x', 'alice', 'x'), ('u2', 'b@x', 'bob', 'x'); INSERT INTO api_keys (id, name, key, user_id) VALUES ('k1', 'alice-key', 'sk-1', 'u1'), ('k2', 'bob-key', 'sk-2', 'u2'); INSERT INTO vms (id, name, vm_json) VALUES ('vm-a', 'Slot A', '{}'); @@ -21,17 +21,17 @@ function freshDb() { return { dir, db, repo: new UsageLogsRepo(db) } } -function storedDebugBytes(db) { - return db - .prepare('SELECT record_json FROM request_log_debug') - .all() - .reduce((sum, row) => sum + Buffer.byteLength(row.record_json, 'utf8'), 0) +async function storedDebugBytes(db) { + return (await db.prepare('SELECT record_json FROM request_log_debug').all()).reduce( + (sum, row) => sum + Buffer.byteLength(row.record_json, 'utf8'), + 0, + ) } -function seed(repo, rows) { +async function seed(repo, rows) { for (const r of rows) { seq++ - repo.insertSummary({ + await repo.insertSummary({ id: `log_${seq}`, request_id: `req-${seq}`, created_at: new Date().toISOString(), @@ -51,65 +51,66 @@ function assertHit(stats, prompt, read, label) { assert.equal(stats.cache_hit_rate, prompt > 0 ? read / prompt : null, `${label} cache_hit_rate`) } -test('debugBytes counts stored JSON bytes for empty, ASCII, and UTF-8 records', () => { - const { dir, db, repo } = freshDb() +test('debugBytes counts stored JSON bytes for empty, ASCII, and UTF-8 records', async () => { + const { dir, db, repo } = await freshDb() try { - assert.equal(storedDebugBytes(db), 0) - assert.equal(repo.debugBytes(), 0) + assert.equal(await storedDebugBytes(db), 0) + assert.equal(await repo.debugBytes(), 0) const records = [{ body: 'ascii' }, { body: '中🙂' }, { body: 'line\nquote"' }] - records.forEach((record, i) => { - repo.insertDebug(`debug-${i}`, `2026-10-15T21:00:0${i}.000Z`, record) - }) + await Promise.all( + records.map(async (record, i) => { + await repo.insertDebug(`debug-${i}`, `2026-10-15T21:00:0${i}.000Z`, record) + }), + ) const expected = records.reduce((sum, record) => sum + Buffer.byteLength(JSON.stringify(record), 'utf8'), 0) - assert.equal(storedDebugBytes(db), expected) - assert.equal(repo.debugBytes(), expected) + assert.equal(await storedDebugBytes(db), expected) + assert.equal(await repo.debugBytes(), expected) } finally { - db.close() + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } }) -test('trimDebugToMaxBytes removes oldest UTF-8 records until actual bytes fit', () => { - const { dir, db, repo } = freshDb() +test('trimDebugToMaxBytes removes oldest UTF-8 records until actual bytes fit', async () => { + const { dir, db, repo } = await freshDb() try { const record = { body: '中'.repeat(10) } const serialized = JSON.stringify(record) const maxBytes = 164 for (let i = 0; i < 5; i++) { - repo.insertDebug(`debug-${i}`, `2026-10-15T21:00:0${i}.000Z`, record) + await repo.insertDebug(`debug-${i}`, `2026-10-15T21:00:0${i}.000Z`, record) } - const before = db.prepare('SELECT record_json FROM request_log_debug').all() + const before = await db.prepare('SELECT record_json FROM request_log_debug').all() const characterCount = before.reduce((sum, row) => sum + row.record_json.length, 0) assert.equal(characterCount, serialized.length * 5) assert.ok(characterCount < maxBytes) - assert.ok(storedDebugBytes(db) > maxBytes) + assert.ok((await storedDebugBytes(db)) > maxBytes) - const originalIds = db - .prepare('SELECT request_id FROM request_log_debug ORDER BY ts ASC') - .all() - .map((row) => row.request_id) - const deleted = repo.trimDebugToMaxBytes(maxBytes) + const originalIds = (await db.prepare('SELECT request_id FROM request_log_debug ORDER BY ts ASC').all()).map( + (row) => row.request_id, + ) + const deleted = await repo.trimDebugToMaxBytes(maxBytes) assert.ok(deleted > 0 && deleted < originalIds.length) - const remaining = db.prepare('SELECT request_id FROM request_log_debug ORDER BY ts ASC').all() + const remaining = await db.prepare('SELECT request_id FROM request_log_debug ORDER BY ts ASC').all() const remainingIds = remaining.map((row) => row.request_id) assert.equal(remainingIds.length, originalIds.length - deleted) assert.deepEqual(remainingIds, originalIds.slice(deleted)) assert.equal(remainingIds.at(-1), originalIds.at(-1)) - assert.ok(storedDebugBytes(db) <= maxBytes) - assert.equal(repo.debugBytes(), storedDebugBytes(db)) + assert.ok((await storedDebugBytes(db)) <= maxBytes) + assert.equal(await repo.debugBytes(), await storedDebugBytes(db)) } finally { - db.close() + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } }) -test('mixed Claude/Codex usage keeps a row-wise prompt denominator on totals and windows', () => { - const { db, repo } = freshDb() +test('mixed Claude/Codex usage keeps a row-wise prompt denominator on totals and windows', async () => { + const { db, repo } = await freshDb() try { - seed(repo, [ + await seed(repo, [ { user_id: 'u1', api_key_id: 'k1', @@ -135,32 +136,32 @@ test('mixed Claude/Codex usage keeps a row-wise prompt denominator on totals and cache_read_tokens: 50, }, ]) - assertHit(repo.totals(), 300, 240, 'totals') - assertHit(repo.billingStats().total, 300, 240, 'billing.total') + assertHit(await repo.totals(), 300, 240, 'totals') + assertHit((await repo.billingStats()).total, 300, 240, 'billing.total') const since = new Date(Date.now() - 60_000).toISOString() - assertHit(repo.windowStats({ since }), 300, 240, 'window') - assertHit(repo.windowStats({ since, owner_user_id: 'u1' }), 200, 190, 'owner window') - assertHit(repo.windowStats({ since, owner_user_id: 'u2' }), 100, 50, 'other owner') + assertHit(await repo.windowStats({ since }), 300, 240, 'window') + assertHit(await repo.windowStats({ since, owner_user_id: 'u1' }), 200, 190, 'owner window') + assertHit(await repo.windowStats({ since, owner_user_id: 'u2' }), 100, 50, 'other owner') } finally { - db.close() + await db.close() } }) -test('empty usage windows keep a null cache hit rate', () => { - const { db, repo } = freshDb() +test('empty usage windows keep a null cache hit rate', async () => { + const { db, repo } = await freshDb() try { - assertHit(repo.totals(), 0, 0, 'empty totals') - assertHit(repo.billingStats().total, 0, 0, 'empty billing') - const empty = repo.windowStats({ since: '2099-01-01T00:00:00.000Z' }) + assertHit(await repo.totals(), 0, 0, 'empty totals') + assertHit((await repo.billingStats()).total, 0, 0, 'empty billing') + const empty = await repo.windowStats({ since: '2099-01-01T00:00:00.000Z' }) assert.equal(empty.requests, 0) assertHit(empty, 0, 0, 'empty window') } finally { - db.close() + await db.close() } }) -test('billingStats reuses identical panel snapshots but separates account windows', () => { - const { db, repo } = freshDb() +test('billingStats reuses identical panel snapshots but separates account windows', async () => { + const { db, repo } = await freshDb() try { let computes = 0 const compute = repo._computeBillingStats.bind(repo) @@ -173,23 +174,26 @@ test('billingStats reuses identical panel snapshots but separates account window const winAReordered = [{ vm_id: 'vm-a', reset_5h: '2026-10-19T05:00:00.000Z', account_id: 'acc-1' }] const winB = [{ account_id: 'acc-1', vm_id: 'vm-a', reset_5h: '2026-10-19T06:00:00.000Z' }] - assert.equal(repo.billingStats({ accountWindows: winA }), repo.billingStats({ accountWindows: winAReordered })) + assert.equal( + await repo.billingStats({ accountWindows: winA }), + await repo.billingStats({ accountWindows: winAReordered }), + ) assert.equal(computes, 1) - repo.billingStats({ accountWindows: winB }) + await repo.billingStats({ accountWindows: winB }) assert.equal(computes, 2) // Explicit historical snapshots are deterministic test/report calls and must // not be served from the live panel cache. - repo.billingStats({ accountWindows: winA, now: Date.parse('2026-10-19T00:00:00.000Z') }) - repo.billingStats({ accountWindows: winA, now: Date.parse('2026-10-19T00:00:00.000Z') }) + await repo.billingStats({ accountWindows: winA, now: Date.parse('2026-10-19T00:00:00.000Z') }) + await repo.billingStats({ accountWindows: winA, now: Date.parse('2026-10-19T00:00:00.000Z') }) assert.equal(computes, 4) } finally { - db.close() + await db.close() } }) -test('billingStats TTL starts after a slow aggregation finishes', () => { - const { db, repo } = freshDb() +test('billingStats TTL starts after a slow aggregation finishes', async () => { + const { db, repo } = await freshDb() const realNow = Date.now try { let clock = 1_000_000 @@ -201,39 +205,39 @@ test('billingStats TTL starts after a slow aggregation finishes', () => { clock += 20_000 // Simulate a scan slower than the 10-second cache TTL. return compute(opts) } - const first = repo.billingStats() - assert.equal(repo.billingStats(), first) + const first = await repo.billingStats() + assert.equal(await repo.billingStats(), first) assert.equal(computes, 1) clock += 10_001 - repo.billingStats() + await repo.billingStats() assert.equal(computes, 2) } finally { Date.now = realNow - db.close() + await db.close() } }) -test('billingStats rate-limits maintenance backfill independently of cache keys', () => { - const { db, repo } = freshDb() +test('billingStats rate-limits maintenance backfill independently of cache keys', async () => { + const { db, repo } = await freshDb() try { let backfills = 0 repo.backfillMissingCosts = () => { backfills++ return 0 } - repo.billingStats() - repo.billingStats({ accountWindows: [{ account_id: 'acc-1', vm_id: 'vm-a' }] }) - repo.billingStats({ accountWindows: [{ account_id: 'acc-2', vm_id: 'vm-a' }] }) + await repo.billingStats() + await repo.billingStats({ accountWindows: [{ account_id: 'acc-1', vm_id: 'vm-a' }] }) + await repo.billingStats({ accountWindows: [{ account_id: 'acc-2', vm_id: 'vm-a' }] }) assert.equal(backfills, 1) } finally { - db.close() + await db.close() } }) -test('billingStats derives global totals from account aggregates without duplicate full scans', () => { - const { db, repo } = freshDb() +test('billingStats derives global totals from account aggregates without duplicate full scans', async () => { + const { db, repo } = await freshDb() try { - seed(repo, [ + await seed(repo, [ { account_id: 'acc-1', vm_id: 'vm-a', input_tokens: 20, cache_read_tokens: 80, total_cost: 1 }, { account_id: 'acc-2', vm_id: null, input_tokens: 40, output_tokens: 5, total_cost: 2 }, ]) @@ -243,7 +247,7 @@ test('billingStats derives global totals from account aggregates without duplica if (!cond) fullScans++ return costSelect(cond, params) } - const bill = repo.billingStats({ now: Date.now() }) + const bill = await repo.billingStats({ now: Date.now() }) assert.equal(fullScans, 0) assert.equal(bill.total.requests, 2) assert.equal(bill.total.input_tokens, 60) @@ -255,15 +259,15 @@ test('billingStats derives global totals from account aggregates without duplica bill.accounts.reduce((sum, row) => sum + row.total_cost, 0), ) } finally { - db.close() + await db.close() } }) -test('aligned account windows re-sum per-row prompt tokens instead of mixed Anthropic formula', () => { - const { db, repo } = freshDb() +test('aligned account windows re-sum per-row prompt tokens instead of mixed Anthropic formula', async () => { + const { db, repo } = await freshDb() try { const now = Date.parse('2026-09-26T12:00:00.000Z') - seed(repo, [ + await seed(repo, [ { created_at: '2026-09-26T10:00:00.000Z', vm_id: 'vm-a', @@ -289,22 +293,22 @@ test('aligned account windows re-sum per-row prompt tokens instead of mixed Anth cache_read_tokens: 95, }, ]) - const aligned = repo.billingStats({ + const aligned = await repo.billingStats({ now, accountWindows: [{ account_id: 'acc-1', vm_id: 'vm-a', reset_5h: '2026-09-26T16:00:00.000Z' }], }) assertHit(aligned.window_5h, 200, 190, 'aligned 5h') assert.equal(Number(aligned.accounts[0].window_5h.prompt_tokens), 200) assert.equal(Number(aligned.accounts[0].window_5h.cache_read_tokens), 190) - const rolling = repo.billingStats({ now }) + const rolling = await repo.billingStats({ now }) assertHit(rolling.window_5h, 10_205, 190, 'rolling 5h') } finally { - db.close() + await db.close() } }) -test('converted non-streaming Anthropic API usage stays disjoint in UsageLogsRepo totals', () => { - const { db, repo } = freshDb() +test('converted non-streaming Anthropic API usage stays disjoint in UsageLogsRepo totals', async () => { + const { db, repo } = await freshDb() try { const upstream = { model: 'gpt-5.4', @@ -316,7 +320,7 @@ test('converted non-streaming Anthropic API usage stays disjoint in UsageLogsRep }, } const message = codexBodyToAnthropicMessage(upstream) - seed(repo, [ + await seed(repo, [ { user_id: 'u1', via: 'api-kernel', @@ -340,17 +344,17 @@ test('converted non-streaming Anthropic API usage stays disjoint in UsageLogsRep ]) assert.equal(message.usage.input_tokens, 20) assert.equal(message.usage.cache_read_input_tokens, 80) - assertHit(repo.totals(), 200, 160, 'converted+codex totals') + assertHit(await repo.totals(), 200, 160, 'converted+codex totals') } finally { - db.close() + await db.close() } }) -test('billingStats window tokens are total including cache', () => { - const { db, repo } = freshDb() +test('billingStats window tokens are total including cache', async () => { + const { db, repo } = await freshDb() try { const now = Date.parse('2026-09-26T12:00:00.000Z') - seed(repo, [ + await seed(repo, [ { created_at: '2026-09-26T11:00:00.000Z', vm_id: 'vm-a', @@ -360,53 +364,53 @@ test('billingStats window tokens are total including cache', () => { cache_read_tokens: 80, }, ]) - const bill = repo.billingStats({ now }) + const bill = await repo.billingStats({ now }) assert.equal(bill.accounts[0].window_5h_tokens, 102) - const window = repo.windowStats({ since: '2026-09-26T00:00:00.000Z', until: '2026-09-26T12:00:00.000Z' }) + const window = await repo.windowStats({ since: '2026-09-26T00:00:00.000Z', until: '2026-09-26T12:00:00.000Z' }) assert.equal(window.input_tokens + window.output_tokens + window.cache_read_tokens, 102) } finally { - db.close() + await db.close() } }) -test('log coverage follows recorded prunes, not the oldest row or the current retention', () => { - const { db, repo } = freshDb() +test('log coverage follows recorded prunes, not the oldest row or the current retention', async () => { + const { db, repo } = await freshDb() try { const now = Date.now() const iso = (msAgo) => new Date(now - msAgo).toISOString() - db.prepare("UPDATE schema_migrations SET applied_at = ? WHERE version = '001'").run(iso(30 * 86400_000)) - seed(repo, [ + await db.prepare("UPDATE schema_migrations SET applied_at = $1 WHERE version = '001'").run(iso(30 * 86400_000)) + await seed(repo, [ { created_at: iso(3 * 86400_000), vm_id: 'vm-a', account_id: 'acc-1', input_tokens: 1 }, { created_at: iso(3600_000), vm_id: 'vm-a', account_id: 'acc-1', input_tokens: 1 }, ]) // Never pruned: a quiet stretch before the oldest row is still complete history. - assert.equal(repo.billingStats().log_coverage_start, iso(30 * 86400_000)) + assert.equal((await repo.billingStats()).log_coverage_start, iso(30 * 86400_000)) - assert.equal(repo.cleanup({ retainDays: 1 }).usage_logs, 1) - const afterPrune = Date.parse(repo.logCoverageStart()) + assert.equal((await repo.cleanup({ retainDays: 1 })).usage_logs, 1) + const afterPrune = Date.parse(await repo.logCoverageStart()) assert.ok(afterPrune >= now - 86400_000 && afterPrune <= Date.now() - 86400_000 + 1000) // Raising retention later does not bring the deleted day back. - assert.equal(repo.cleanup({ retainDays: 7 }).usage_logs, 0) - assert.equal(Date.parse(repo.logCoverageStart()), afterPrune) + assert.equal((await repo.cleanup({ retainDays: 7 })).usage_logs, 0) + assert.equal(Date.parse(await repo.logCoverageStart()), afterPrune) const week = estimateWindow({ spentUsd: 5, usedRatio: 0.5, reset: iso(-86400_000), windowStart: iso(6 * 86400_000), - coveredSince: repo.logCoverageStart(), + coveredSince: await repo.logCoverageStart(), }) assert.equal(week.status, 'log_gap') } finally { - db.close() + await db.close() } }) -test('window_fable only counts fable rows since reset_7d_oi', () => { - const { db, repo } = freshDb() +test('window_fable only counts fable rows since reset_7d_oi', async () => { + const { db, repo } = await freshDb() try { const now = Date.parse('2026-09-26T12:00:00.000Z') - seed(repo, [ + await seed(repo, [ { created_at: '2026-09-20T11:00:00.000Z', vm_id: 'vm-a', @@ -439,7 +443,7 @@ test('window_fable only counts fable rows since reset_7d_oi', () => { total_cost: 8, }, ]) - const bill = repo.billingStats({ + const bill = await repo.billingStats({ now, accountWindows: [ { @@ -455,29 +459,29 @@ test('window_fable only counts fable rows since reset_7d_oi', () => { assert.equal(fable.cost, 3) assert.equal(fable.tokens, 22) assert.equal(bill.accounts[0].window_fable, fable) - const none = repo.billingStats({ + const none = await repo.billingStats({ now, accountWindows: [{ account_id: 'acc-1', vm_id: 'vm-a' }], }) assert.equal(none.accounts[0].window_fable, null) } finally { - db.close() + await db.close() } }) -test('unpricedModels lists unpriced rows', () => { - const { db, repo } = freshDb() +test('unpricedModels lists unpriced rows', async () => { + const { db, repo } = await freshDb() try { - seed(repo, [ + await seed(repo, [ { model: 'mystery', pricing_model: 'unpriced', input_tokens: 10 }, { model: 'mystery', pricing_model: 'unpriced', input_tokens: 4 }, { model: 'claude-sonnet-5', pricing_model: 'sonnet-5', input_tokens: 8 }, ]) - const rows = repo.unpricedModels() + const rows = await repo.unpricedModels() assert.equal(rows.length, 1) assert.equal(rows[0].model, 'mystery') assert.equal(rows[0].requests, 2) } finally { - db.close() + await db.close() } }) diff --git a/test/unit/usage-logs-view.test.mjs b/test/unit/usage-logs-view.test.mjs index db6491c4c..2896927fb 100644 --- a/test/unit/usage-logs-view.test.mjs +++ b/test/unit/usage-logs-view.test.mjs @@ -3,16 +3,16 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb, withTransaction } from '../../src/lib/db/database.mjs' import { UsageLogsRepo } from '../../src/lib/db/repos/usage-logs-repo.mjs' import { UsageLogsView } from '../../src/lib/db/repos/usage-logs-view.mjs' let seq = 0 -function freshDb() { +async function freshDb() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-ulview-')) - const db = createDatabase({ dbPath: path.join(dir, 'logs.db') }) - db.exec(` + const db = await getDb() + await db.exec(` INSERT INTO users (id, email, username, password_hash) VALUES ('u1', 'a@x', 'alice', 'x'), ('u2', 'b@x', 'bob', 'x'); INSERT INTO api_keys (id, name, key, user_id) VALUES ('k1', 'alice-key', 'sk-1', 'u1'), ('k2', 'bob-key', 'sk-2', 'u2'); INSERT INTO vms (id, name, vm_json) VALUES ('vm-a', 'Slot A', '{}'), ('vm-b', 'Slot B', '{}'); @@ -22,37 +22,37 @@ function freshDb() { } /** Insert rows newest-last; created_at defaults to a strictly increasing clock. */ -function seed(db, rows) { +async function seed(db, rows) { const repo = new UsageLogsRepo(db) const out = [] - db.exec('BEGIN') - for (const r of rows) { - seq++ - const rec = { - id: `log_${String(seq).padStart(8, '0')}`, - request_id: `req-${seq}`, - created_at: new Date(Date.UTC(2026, 0, 1) + seq * 1000).toISOString(), - log_mode: 'normal', - method: 'POST', - path: '/v1/messages', - protocol: 'anthropic.messages', - model: 'claude-sonnet-4', - status: 200, - vm_id: 'vm-a', - ...r, + await withTransaction(db, async () => { + for (const r of rows) { + seq++ + const rec = { + id: `log_${String(seq).padStart(8, '0')}`, + request_id: `req-${seq}`, + created_at: new Date(Date.UTC(2026, 0, 1) + seq * 1000).toISOString(), + log_mode: 'normal', + method: 'POST', + path: '/v1/messages', + protocol: 'anthropic.messages', + model: 'claude-sonnet-4', + status: 200, + vm_id: 'vm-a', + ...r, + } + await repo.insertSummary(rec) + out.push(rec) } - repo.insertSummary(rec) - out.push(rec) - } - db.exec('COMMIT') + }) return out } -function pageAll(view, filters) { +async function pageAll(view, filters) { const seen = [] let cursor = null for (let i = 0; i < 50; i++) { - const page = view.listBatch({ + const page = await view.listBatch({ ...filters, cursor_created_at: cursor?.createdAt, cursor_id: cursor?.id, @@ -65,20 +65,20 @@ function pageAll(view, filters) { throw new Error('paging did not terminate') } -test('keyset paging returns every row once, newest first', () => { - const db = freshDb() - const rows = seed( +test('keyset paging returns every row once, newest first', async () => { + const db = await freshDb() + const rows = await seed( db, Array.from({ length: 5 }, () => ({})), ) const view = new UsageLogsView(db) - const first = view.listBatch({ limit: 2 }) + const first = await view.listBatch({ limit: 2 }) assert.deepEqual( first.logs.map((r) => r.id), [rows[4].id, rows[3].id], ) assert.equal(first.hasMore, true) - const { seen, pages } = pageAll(view, { limit: 2 }) + const { seen, pages } = await pageAll(view, { limit: 2 }) assert.equal(pages, 3) assert.deepEqual( seen.map((r) => r.id), @@ -86,24 +86,24 @@ test('keyset paging returns every row once, newest first', () => { ) }) -test('JS-filtered paging hands back the scan cursor instead of ending early', () => { - const db = freshDb() - const old = seed(db, [ +test('JS-filtered paging hands back the scan cursor instead of ending early', async () => { + const db = await freshDb() + const old = await seed(db, [ { status: 504, error_code: 'upstream_timeout' }, { status: 504, error_code: 'upstream_timeout' }, { status: 504, error_code: 'upstream_timeout' }, ]) // 2100 newer errors of another class: more than one call's scan budget. - seed( + await seed( db, Array.from({ length: 2100 }, () => ({ status: 500, error_code: 'upstream_error' })), ) const view = new UsageLogsView(db) - const first = view.listBatch({ error_class: 'timeout', limit: 50 }) + const first = await view.listBatch({ error_class: 'timeout', limit: 50 }) assert.equal(first.logs.length, 0) assert.equal(first.hasMore, true) assert.ok(first.nextCursor) - const { seen } = pageAll(view, { error_class: 'timeout', limit: 50 }) + const { seen } = await pageAll(view, { error_class: 'timeout', limit: 50 }) assert.deepEqual( seen.map((r) => r.id), old.map((r) => r.id).reverse(), @@ -111,79 +111,75 @@ test('JS-filtered paging hands back the scan cursor instead of ending early', () assert.ok(seen.every((r) => r.errorClass === 'timeout')) }) -test('user owner scope keeps only own key / own user rows and ignores other tenants', () => { - const db = freshDb() - seed(db, [ +test('user owner scope keeps only own key / own user rows and ignores other tenants', async () => { + const db = await freshDb() + await seed(db, [ { user_id: 'u1', api_key_id: 'k1' }, { user_id: null, api_key_id: 'k1' }, { user_id: 'u2', api_key_id: 'k2' }, ]) const view = new UsageLogsView(db) - const mine = view.listBatch({ owner_user_id: 'u1' }).logs + const mine = (await view.listBatch({ owner_user_id: 'u1' })).logs assert.equal(mine.length, 2) assert.ok(mine.every((r) => r.keyId === 'k1')) - assert.equal(view.summary({ owner_user_id: 'u1' }).totalRequests, 2) - const opts = view.filterOptions({ owner_user_id: 'u1' }) + assert.equal((await view.summary({ owner_user_id: 'u1' })).totalRequests, 2) + const opts = await view.filterOptions({ owner_user_id: 'u1' }) assert.deepEqual(opts.users, [{ id: 'u1', name: 'alice' }]) assert.deepEqual(opts.keys, [{ id: 'k1', name: 'alice-key', userId: 'u1' }]) - assert.equal(view.listBatch({ user_id: 'u2' }).logs.length, 1) + assert.equal((await view.listBatch({ user_id: 'u2' })).logs.length, 1) }) -test('muting follows RequestLogStore: settings default, explicit list, include_muted, error_class override', () => { - const db = freshDb() - seed(db, [ +test('muting follows RequestLogStore: settings default, explicit list, include_muted, error_class override', async () => { + const db = await freshDb() + await seed(db, [ { status: 401, error_code: 'invalid_api_key', vm_id: null }, { status: 429, error_code: 'quota_5h_safety' }, { status: 200 }, ]) - const classes = (view, f = {}) => - view - .listBatch(f) - .logs.map((r) => r.errorClass ?? 'ok') - .sort() + const classes = async (view, f = {}) => (await view.listBatch(f)).logs.map((r) => r.errorClass ?? 'ok').sort() const byDefault = new UsageLogsView(db) - assert.deepEqual(classes(byDefault), ['ok', 'quota']) + assert.deepEqual(await classes(byDefault), ['ok', 'quota']) const settings = new UsageLogsView(db, { mutedErrorClasses: ['auth', 'quota'] }) - assert.deepEqual(classes(settings), ['ok']) - assert.equal(settings.summary({}).totalRequests, 1) - assert.deepEqual(classes(settings, { include_muted: true }), ['auth', 'ok', 'quota']) - assert.deepEqual(classes(settings, { error_class: 'auth' }), ['auth']) - assert.deepEqual(classes(settings, { exclude_error_class: 'quota' }), ['auth', 'ok']) + assert.deepEqual(await classes(settings), ['ok']) + assert.equal((await settings.summary({})).totalRequests, 1) + assert.deepEqual(await classes(settings, { include_muted: true }), ['auth', 'ok', 'quota']) + assert.deepEqual(await classes(settings, { error_class: 'auth' }), ['auth']) + assert.deepEqual(await classes(settings, { exclude_error_class: 'quota' }), ['auth', 'ok']) }) -test('summary cache hit rate counts Claude writes and Codex disjoint input on both summary paths', () => { - const db = freshDb() - seed(db, [ +test('summary cache hit rate counts Claude writes and Codex disjoint input on both summary paths', async () => { + const db = await freshDb() + await seed(db, [ { input_tokens: 20, cache_read_tokens: 80_000, cache_creation_tokens: 20_000 }, { via: 'codex-kernel', model: 'gpt-5.4', input_tokens: 20, cache_read_tokens: 80 }, ]) const want = 80_080 / 100_120 - const sqlPath = new UsageLogsView(db).summary({}) - const scanPath = new UsageLogsView(db, { mutedErrorClasses: ['auth', 'quota'] }).summary({}) + const sqlPath = await new UsageLogsView(db).summary({}) + const scanPath = await new UsageLogsView(db, { mutedErrorClasses: ['auth', 'quota'] }).summary({}) for (const summary of [sqlPath, scanPath]) { assert.equal(summary.totalRequests, 2) assert.ok(Math.abs(summary.cacheHitRate - want) < 1e-9, String(summary.cacheHitRate)) } }) -test('provider chains for a page come from one request_attempts query, ordered by attempt', () => { - const db = freshDb() - const [a, b, c] = seed(db, [ +test('provider chains for a page come from one request_attempts query, ordered by attempt', async () => { + const db = await freshDb() + const [a, b, c] = await seed(db, [ { account_id: 'acc-a' }, { final_account_id: 'acc-b', vm_id: 'vm-b', duration_ms: 2000 }, {}, ]) const before = (row, ms) => new Date(Date.parse(row.created_at) - ms).toISOString() const ins = db.prepare( - 'INSERT INTO request_attempts (request_id, attempt_no, vm_id, account_id, started_at, selection_reason, downstream_committed) VALUES (?, ?, ?, ?, ?, ?, ?)', + 'INSERT INTO request_attempts (request_id, attempt_no, vm_id, account_id, started_at, selection_reason, downstream_committed) VALUES ($1, $2, $3, $4, $5, $6, $7)', ) - ins.run(a.request_id, 2, 'vm-b', 'acc-b', before(a, 200), 'failover', 1) - ins.run(a.request_id, 1, 'vm-a', 'acc-a', before(a, 500), 'sticky', 0) - ins.run(b.request_id, 1, 'vm-b', 'acc-b', before(b, 1500), 'roundrobin', 1) + await ins.run(a.request_id, 2, 'vm-b', 'acc-b', before(a, 200), 'failover', 1) + await ins.run(a.request_id, 1, 'vm-a', 'acc-a', before(a, 500), 'sticky', 0) + await ins.run(b.request_id, 1, 'vm-b', 'acc-b', before(b, 1500), 'roundrobin', 1) // Client-chosen x-request-id reused by later / earlier requests: their attempts // fall outside the row's own lifetime and must not be attached. - ins.run(b.request_id, 2, 'vm-a', 'acc-a', before(b, 60_000), 'sticky', 1) - ins.run(c.request_id, 1, 'vm-b', 'acc-b', before(c, -3600_000), 'sticky', 1) + await ins.run(b.request_id, 2, 'vm-a', 'acc-a', before(b, 60_000), 'sticky', 1) + await ins.run(c.request_id, 1, 'vm-b', 'acc-b', before(c, -3600_000), 'sticky', 1) const attemptQueries = [] const spy = { @@ -192,7 +188,7 @@ test('provider chains for a page come from one request_attempts query, ordered b return db.prepare(sql) }, } - const logs = new UsageLogsView(spy).listBatch({}).logs + const logs = (await new UsageLogsView(spy).listBatch({})).logs assert.equal(attemptQueries.length, 1) const rowA = logs.find((r) => r.requestId === a.request_id) assert.deepEqual( @@ -219,40 +215,39 @@ test('provider chains for a page come from one request_attempts query, ordered b assert.deepEqual(logs.find((r) => r.requestId === c.request_id).providerChain, []) }) -test('filters: model/requested_model, status, attempts, session, debug, time, q', () => { - const db = freshDb() - const rows = seed(db, [ +test('filters: model/requested_model, status, attempts, session, debug, time, q', async () => { + const db = await freshDb() + const rows = await seed(db, [ { model: 'claude-opus-4', requested_model: 'opus-alias', session_id: 's-1', outbound_session_id: 'out-1' }, { status: 500, error_code: 'upstream_error', attempt_count: 3, session_id: 's-2', outbound_session_id: 'out-2' }, { status: 204, log_mode: 'debug', path: '/v1/chat/completions' }, { status: 200, error_message: 'needle here', attempt_count: 1 }, ]) const view = new UsageLogsView(db) - const ids = (f) => - view - .listBatch(f) - .logs.map((r) => r.id) - .sort() - assert.deepEqual(ids({ model: 'opus-alias' }), [rows[0].id]) - assert.deepEqual(ids({ status_code: '500' }), [rows[1].id]) - assert.deepEqual(ids({ exclude_status_200: true }), [rows[1].id]) - assert.deepEqual(ids({ exclude_status_200: true, status_code: '204' }), [rows[2].id]) - assert.deepEqual(ids({ min_attempt_count: '2' }), [rows[1].id]) + const ids = async (f) => (await view.listBatch(f)).logs.map((r) => r.id).sort() + assert.deepEqual(await ids({ model: 'opus-alias' }), [rows[0].id]) + assert.deepEqual(await ids({ status_code: '500' }), [rows[1].id]) + assert.deepEqual(await ids({ exclude_status_200: true }), [rows[1].id]) + assert.deepEqual(await ids({ exclude_status_200: true, status_code: '204' }), [rows[2].id]) + assert.deepEqual(await ids({ min_attempt_count: '2' }), [rows[1].id]) // Either the caller's id or the id the gateway sent upstream finds the row. - assert.deepEqual(ids({ session_id: 's-2' }), [rows[1].id]) - assert.deepEqual(ids({ session_id: 'out-2' }), [rows[1].id]) - const [first] = view.listBatch({ session_id: 'out-1' }).logs + assert.deepEqual(await ids({ session_id: 's-2' }), [rows[1].id]) + assert.deepEqual(await ids({ session_id: 'out-2' }), [rows[1].id]) + const [first] = (await view.listBatch({ session_id: 'out-1' })).logs assert.equal(first.sessionId, 'out-1') assert.equal(first.clientSessionId, 's-1') - assert.deepEqual(ids({ debug_only: true }), [rows[2].id]) - assert.deepEqual(ids({ endpoint: '/v1/chat/completions' }), [rows[2].id]) - assert.deepEqual(ids({ q: 'needle' }), [rows[3].id]) - assert.deepEqual(ids({ start_time: rows[1].created_at, end_time: rows[3].created_at }), [rows[1].id, rows[2].id]) + assert.deepEqual(await ids({ debug_only: true }), [rows[2].id]) + assert.deepEqual(await ids({ endpoint: '/v1/chat/completions' }), [rows[2].id]) + assert.deepEqual(await ids({ q: 'needle' }), [rows[3].id]) + assert.deepEqual(await ids({ start_time: rows[1].created_at, end_time: rows[3].created_at }), [ + rows[1].id, + rows[2].id, + ]) }) -test('row badges: via only when non-default, context1m from long_context, fast/priority', () => { - const db = freshDb() - const [plain, special] = seed(db, [ +test('row badges: via only when non-default, context1m from long_context, fast/priority', async () => { + const db = await freshDb() + const [plain, special] = await seed(db, [ { via: 'go-worker-pool', stream: true }, { via: 'api-kernel', @@ -265,7 +260,7 @@ test('row badges: via only when non-default, context1m from long_context, fast/p workspace: 'client', }, ]) - const logs = new UsageLogsView(db).listBatch({}).logs + const logs = (await new UsageLogsView(db).listBatch({})).logs const keys = (id) => logs .find((r) => r.id === id) @@ -277,17 +272,17 @@ test('row badges: via only when non-default, context1m from long_context, fast/p assert.equal(logs.find((r) => r.id === plain.id).context1mApplied, null) }) -test('overview splits today/yesterday-same-time in the requested zone', () => { - const db = freshDb() +test('overview splits today/yesterday-same-time in the requested zone', async () => { + const db = await freshDb() const now = Date.parse('2026-03-10T04:00:00Z') // 12:00 in Shanghai - seed(db, [ + await seed(db, [ { created_at: '2026-03-09T02:00:00.000Z', total_cost: 1 }, // yesterday 10:00 local { created_at: '2026-03-09T06:00:00.000Z', total_cost: 2 }, // yesterday 14:00 local, after same-time { created_at: '2026-03-09T17:00:00.000Z', total_cost: 4, status: 500, error_code: 'upstream_error' }, { created_at: '2026-03-10T03:59:30.000Z', total_cost: 8, session_id: 's-live', outbound_session_id: 'out-live' }, ]) const view = new UsageLogsView(db) - const sh = view.overview({ tz: 'Asia/Shanghai', now }) + const sh = await view.overview({ tz: 'Asia/Shanghai', now }) assert.equal(sh.todayRequests, 2) assert.equal(sh.todayCost, 12) assert.equal(sh.todayErrors, 1) @@ -296,17 +291,17 @@ test('overview splits today/yesterday-same-time in the requested zone', () => { assert.equal(sh.rpm, 1) assert.equal(sh.activeSessions, 1) // Unknown zone falls back to UTC: today starts 2026-03-10T00:00Z. - const utc = view.overview({ tz: 'Not/AZone', now }) + const utc = await view.overview({ tz: 'Not/AZone', now }) assert.equal(utc.todayRequests, 1) assert.equal(utc.yesterdayRequests, 1) }) -test('active sessions and session suggestions group by outbound session, owner-scoped, newest first', () => { - const db = freshDb() +test('active sessions and session suggestions group by outbound session, owner-scoped, newest first', async () => { + const db = await freshDb() const now = Date.now() const iso = (ago) => new Date(now - ago).toISOString() // One client session that the gateway rebuilt into two upstream sessions (abc-1 / abc-1b). - seed(db, [ + await seed(db, [ { created_at: iso(60_000), session_id: 'client-1', @@ -329,7 +324,7 @@ test('active sessions and session suggestions group by outbound session, owner-s { created_at: iso(5_000), session_id: 'client-legacy', api_key_id: 'k1' }, ]) const view = new UsageLogsView(db) - const active = view.activeSessions({ owner_user_id: 'u1', now }) + const active = await view.activeSessions({ owner_user_id: 'u1', now }) assert.equal(active.total, 2) assert.deepEqual( active.sessions.map((s) => [s.sessionId, s.clientSessionId, s.requests]), @@ -340,19 +335,19 @@ test('active sessions and session suggestions group by outbound session, owner-s ) assert.equal(active.sessions[1].lastStatus, 200) assert.equal(active.sessions[1].keyName, 'alice-key') - assert.equal(view.activeSessions({ now }).total, 3) - assert.deepEqual(view.sessionSuggestions({ q: 'abc', owner_user_id: 'u1' }), ['abc-1b', 'abc-1', 'abc-3']) - assert.deepEqual(view.sessionSuggestions({ q: 'abc-2' }), ['abc-2']) - assert.deepEqual(view.sessionSuggestions({ q: 'client' }), []) - assert.equal(view.overview({ now }).activeSessions, 3) + assert.equal((await view.activeSessions({ now })).total, 3) + assert.deepEqual(await view.sessionSuggestions({ q: 'abc', owner_user_id: 'u1' }), ['abc-1b', 'abc-1', 'abc-3']) + assert.deepEqual(await view.sessionSuggestions({ q: 'abc-2' }), ['abc-2']) + assert.deepEqual(await view.sessionSuggestions({ q: 'client' }), []) + assert.equal((await view.overview({ now })).activeSessions, 3) }) -test('active sessions never resolve the last row from another tenant sharing the session id', () => { - const db = freshDb() +test('active sessions never resolve the last row from another tenant sharing the session id', async () => { + const db = await freshDb() const now = Date.now() const at = new Date(now - 30_000).toISOString() // Same client-chosen session id and completion instant; bob's row sorts later by id. - seed(db, [ + await seed(db, [ { created_at: at, session_id: 'shared', @@ -371,15 +366,15 @@ test('active sessions never resolve the last row from another tenant sharing the error_code: 'upstream_error', }, ]) - const [s] = new UsageLogsView(db).activeSessions({ owner_user_id: 'u1', now }).sessions + const [s] = (await new UsageLogsView(db).activeSessions({ owner_user_id: 'u1', now })).sessions assert.equal(s.keyName, 'alice-key') assert.equal(s.vmId, 'vm-a') assert.equal(s.lastStatus, 200) }) -test('tenant error collection hides other users and anonymous ingress failures', () => { - const db = freshDb() - seed(db, [ +test('tenant error collection hides other users and anonymous ingress failures', async () => { + const db = await freshDb() + await seed(db, [ { user_id: 'u2', api_key_id: 'k2', @@ -393,13 +388,27 @@ test('tenant error collection hides other users and anonymous ingress failures', { user_id: 'u1', api_key_id: 'k1', status: 500, error_code: 'upstream_error', error_message: 'boom' }, ]) const repo = new UsageLogsRepo(db) - const mine = repo.windowStats({ owner_user_id: 'u1' }).error_collection + const mine = (await repo.windowStats({ owner_user_id: 'u1' })).error_collection assert.deepEqual( mine.recent.map((r) => r.error_code), ['upstream_error'], ) assert.deepEqual(mine.ingress_hits, []) - const all = repo.windowStats().error_collection + const all = (await repo.windowStats()).error_collection assert.equal(all.recent.length, 3) assert.deepEqual(all.ingress_hits.map((h) => h.ip).sort(), ['198.51.100.7', '203.0.113.9']) }) + +test('search keeps ASCII case folding and literal backslashes', async () => { + const db = await freshDb() + await seed(db, [ + { id: 'search-upper', request_id: 'SearchABC\\literal', path: '/v1/messages', status: 200 }, + { id: 'search-nonascii', request_id: 'search-Ä', path: '/v1/messages', status: 200 }, + ]) + const view = new UsageLogsView(db, { mutedErrorClasses: [] }) + assert.deepEqual( + (await view.listBatch({ q: 'searchabc\\lit' })).logs.map((r) => r.id), + ['search-upper'], + ) + assert.deepEqual((await view.listBatch({ q: 'search-ä' })).logs, []) +}) diff --git a/test/unit/usage-profile.test.mjs b/test/unit/usage-profile.test.mjs index 7c37a1719..609b664cd 100644 --- a/test/unit/usage-profile.test.mjs +++ b/test/unit/usage-profile.test.mjs @@ -75,13 +75,13 @@ test('relay restores the shipped quota and risk defaults only', () => { assert.equal(risk.refusal.enabled, true) }) -test('applyUsageProfile writes routing, distill, and risk settings', () => { +test('applyUsageProfile writes routing, distill, and risk settings', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-profile-')) const distillFile = path.join(dir, 'distill-rules.json') const settings = memorySettings() const seen = [] try { - const applied = applyUsageProfile({ + const applied = await applyUsageProfile({ mode: 'personal', persistRouting: (patch) => { seen.push(patch) @@ -97,7 +97,7 @@ test('applyUsageProfile writes routing, distill, and risk settings', () => { assert.equal(readJevConfig((key, fallback) => settings.get(key, fallback)).hard_regex_enabled, false) assert.equal(refusalGuardPolicy((key, fallback) => settings.get(key, fallback)).enabled, false) - applyUsageProfile({ + await applyUsageProfile({ mode: 'relay', persistRouting: (patch) => { seen.push(patch) @@ -115,11 +115,16 @@ test('applyUsageProfile writes routing, distill, and risk settings', () => { } }) -test('unknown profile is rejected and a saved profile survives complete', () => { +test('unknown profile is rejected and a saved profile survives complete', async () => { assert.equal(routingPatchForProfile('nope'), null) - assert.throws( - () => - applyUsageProfile({ mode: 'station', persistRouting() {}, settings: memorySettings(), distillFile: '/tmp/x' }), + await assert.rejects( + async () => + await applyUsageProfile({ + mode: 'station', + persistRouting() {}, + settings: memorySettings(), + distillFile: '/tmp/x', + }), (error) => error.code === 'invalid_profile', ) const saved = setOnboardingProfile({}, 'admin', 'personal', '2026-10-13T00:00:00.000Z') diff --git a/test/unit/vm-db-sync.test.mjs b/test/unit/vm-db-sync.test.mjs index 4d5bbadf9..d9d91d94d 100644 --- a/test/unit/vm-db-sync.test.mjs +++ b/test/unit/vm-db-sync.test.mjs @@ -7,7 +7,14 @@ import { openDatabase, closeDatabase, getDb } from '../../src/lib/db/database.mj import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' import { UsersRepo } from '../../src/lib/db/repos/users-repo.mjs' import { encryptString, decryptString, isEncrypted } from '../../src/lib/db/secure.mjs' -import { initVmDbSync, stopVmWatch, syncVmFile, removeVmFromDb, reconcileVms } from '../../src/lib/vm/vm-db-sync.mjs' +import { + initVmDbSync, + stopVmWatch, + syncVmFile, + removeVmFromDb, + reconcileVms, + flushVmDbSync, +} from '../../src/lib/vm/vm-db-sync.mjs' import { setVmWriteHook, atomicWriteJson } from '../../src/lib/vm/vm-file.mjs' import { persistOauthToVm } from '../../src/lib/oauth/oauth-credentials.mjs' import { setVmSchedulable, setActiveVm } from '../../src/lib/vm/vm-registry.mjs' @@ -37,26 +44,27 @@ function seedVmFile(id = 'vm-t1', extra = {}) { return vm } -beforeEach(() => { +beforeEach(async () => { project = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vmsync-')) - openDatabase({ dataDir: path.join(project, 'data') }) + await openDatabase({ dataDir: path.join(project, 'data') }) }) -afterEach(() => { +afterEach(async () => { stopVmWatch() setVmWriteHook(null) - closeDatabase() + await closeDatabase() fs.rmSync(project, { recursive: true, force: true }) delete process.env.KIN_DB_SECRET }) -test('initVmDbSync reconciles existing vm files into DB (credentials included)', () => { +test('initVmDbSync reconciles existing vm files into DB (credentials included)', async () => { seedVmFile('vm-t1') setActiveVm(project, 'vm-t1') - const res = initVmDbSync(project, { watch: false }) + const res = await initVmDbSync(project, { watch: false }) assert.equal(res.upserted, 1) const repo = new VmsRepo(getDb()) - const row = repo.get('vm-t1') + await flushVmDbSync() + const row = await repo.get('vm-t1') assert.equal(row.access_token, null) assert.equal(row.refresh_token, null) assert.equal(row.session_key, null) @@ -64,16 +72,16 @@ test('initVmDbSync reconciles existing vm files into DB (credentials included)', assert.equal(row.vm.claude.access_token, undefined) assert.equal(row.vm.claude.has_access, true) assert.equal(row.vm.claude.has_refresh, true) - assert.equal(repo.getActiveVmId(), 'vm-t1') + assert.equal(await repo.getActiveVmId(), 'vm-t1') }) -test('startup reclaims deleted users assigned VMs, including DB-only restored records', () => { +test('startup reclaims deleted users assigned VMs, including DB-only restored records', async () => { process.env.KIN_DB_SECRET = 'reclaim-test-secret' const users = new UsersRepo(getDb()) for (const id of ['deleted-owner', 'live-owner', 'disabled-owner']) { - users.insert({ id, username: id, password_hash: 'unused', role: 'user', enabled: id !== 'disabled-owner' }) + await users.insert({ id, username: id, password_hash: 'unused', role: 'user', enabled: id !== 'disabled-owner' }) } - users.remove('deleted-owner') + await users.remove('deleted-owner') seedVmFile('assigned', { owner_user_id: 'deleted-owner', origin: 'admin_assigned' }) seedVmFile('legacy', { owner_user_id: 'deleted-owner' }) seedVmFile('private', { owner_user_id: 'deleted-owner', origin: 'user_created' }) @@ -82,18 +90,19 @@ test('startup reclaims deleted users assigned VMs, including DB-only restored re seedVmFile('unknown', { owner_user_id: 'unknown-owner', origin: 'admin_assigned' }) const restored = seedVmFile('restored', { owner_user_id: 'deleted-owner', origin: 'admin_assigned' }) const repo = new VmsRepo(getDb()) - repo.upsertFromVmJson(restored) + await repo.upsertFromVmJson(restored) fs.rmSync(path.join(project, 'vms', 'restored.json')) - const result = initVmDbSync(project, { watch: false }) + const result = await initVmDbSync(project, { watch: false }) assert.deepEqual(result.reclaimed_vm_ids.sort(), ['assigned', 'legacy', 'restored']) for (const id of result.reclaimed_vm_ids) { const vm = JSON.parse(fs.readFileSync(path.join(project, 'vms', `${id}.json`), 'utf8')) assert.equal(vm.owner_user_id, null) assert.equal(vm.origin, 'platform') - const row = repo.get(id) + await flushVmDbSync() + const row = await repo.get(id) assert.equal(row.vm.owner_user_id, null) - const raw = getDb().prepare('SELECT owner_user_id, origin FROM vms WHERE id = ?').get(id) + const raw = await getDb().prepare('SELECT owner_user_id, origin FROM vms WHERE id = $1').get(id) assert.equal(raw.owner_user_id, null) assert.equal(raw.origin, 'platform') assert.equal(row.vm.claude.has_access, true) @@ -105,31 +114,31 @@ test('startup reclaims deleted users assigned VMs, including DB-only restored re ['disabled', 'disabled-owner'], ['unknown', 'unknown-owner'], ]) { - assert.equal(repo.get(id).vm.owner_user_id, owner) + assert.equal((await repo.get(id)).vm.owner_user_id, owner) } const before = fs.readFileSync(path.join(project, 'vms', 'assigned.json'), 'utf8') - assert.deepEqual(initVmDbSync(project, { watch: false }).reclaimed_vm_ids, []) + assert.deepEqual((await initVmDbSync(project, { watch: false })).reclaimed_vm_ids, []) assert.equal(fs.readFileSync(path.join(project, 'vms', 'assigned.json'), 'utf8'), before) }) -test('startup ownership recovery skips an unreadable slot and continues with valid assignments', (t) => { +test('startup ownership recovery skips an unreadable slot and continues with valid assignments', async (t) => { const users = new UsersRepo(getDb()) - users.insert({ id: 'deleted-owner', username: 'deleted', password_hash: 'unused', role: 'user' }) - users.remove('deleted-owner') + await users.insert({ id: 'deleted-owner', username: 'deleted', password_hash: 'unused', role: 'user' }) + await users.remove('deleted-owner') seedVmFile('assigned', { owner_user_id: 'deleted-owner', origin: 'admin_assigned' }) fs.writeFileSync(path.join(project, 'vms', 'broken.json'), '{') const warnings = [] t.mock.method(console, 'warn', (message) => warnings.push(message)) - const result = initVmDbSync(project, { watch: false }) + const result = await initVmDbSync(project, { watch: false }) assert.deepEqual(result.reclaimed_vm_ids, ['assigned']) assert.ok(warnings.some((message) => /ownership recovery skipped broken.json/.test(message))) - assert.equal(new VmsRepo(getDb()).get('assigned').vm.owner_user_id, null) + assert.equal((await new VmsRepo(getDb()).get('assigned')).vm.owner_user_id, null) }) -test('write-through hook mirrors persistOauthToVm into DB', () => { +test('write-through hook mirrors persistOauthToVm into DB', async () => { seedVmFile('vm-t2') - initVmDbSync(project, { watch: false }) + await initVmDbSync(project, { watch: false }) const vmPath = path.join(project, 'vms', 'vm-t2.json') persistOauthToVm(vmPath, { access_token: 'sk-ant-oat01-NEW', @@ -137,7 +146,8 @@ test('write-through hook mirrors persistOauthToVm into DB', () => { expires_at: 1893460000, }) const repo = new VmsRepo(getDb()) - const row = repo.get('vm-t2') + await flushVmDbSync() + const row = await repo.get('vm-t2') assert.equal(row.access_token, null) assert.equal(row.refresh_token, null) assert.equal(row.vm.claude.has_access, true) @@ -145,18 +155,19 @@ test('write-through hook mirrors persistOauthToVm into DB', () => { assert.equal(row.vm.claude.access_token, undefined) }) -test('vm-registry writes are mirrored (setVmSchedulable)', () => { +test('vm-registry writes are mirrored (setVmSchedulable)', async () => { seedVmFile('vm-t3') - initVmDbSync(project, { watch: false }) + await initVmDbSync(project, { watch: false }) setVmSchedulable(project, 'vm-t3', false, 'test-reason') - const row = new VmsRepo(getDb()).get('vm-t3') + await flushVmDbSync() + const row = await new VmsRepo(getDb()).get('vm-t3') assert.equal(row.schedulable, false) assert.equal(row.vm.schedule_disabled_reason, 'test-reason') }) -test('setVmSchedulable preserveStatus keeps running when operator disables', () => { +test('setVmSchedulable preserveStatus keeps running when operator disables', async () => { seedVmFile('vm-t3c') - initVmDbSync(project, { watch: false }) + await initVmDbSync(project, { watch: false }) setVmSchedulable(project, 'vm-t3c', false, 'disabled', { preserveStatus: true }) const off = JSON.parse(fs.readFileSync(path.join(project, 'vms', 'vm-t3c.json'), 'utf8')) assert.equal(off.status, 'running') @@ -164,9 +175,9 @@ test('setVmSchedulable preserveStatus keeps running when operator disables', () assert.equal(off.schedule_disabled_reason, 'disabled') }) -test('setVmSchedulable(true) restores soft-paused status to running', () => { +test('setVmSchedulable(true) restores soft-paused status to running', async () => { seedVmFile('vm-t3b') - initVmDbSync(project, { watch: false }) + await initVmDbSync(project, { watch: false }) setVmSchedulable(project, 'vm-t3b', false, 'proxy_rebind_worker_restart') const paused = JSON.parse(fs.readFileSync(path.join(project, 'vms', 'vm-t3b.json'), 'utf8')) assert.equal(paused.status, 'paused') @@ -178,12 +189,12 @@ test('setVmSchedulable(true) restores soft-paused status to running', () => { assert.equal(ready.schedule_disabled_reason, null) }) -test('reconcile rebuilds missing vm file from DB (restore pull-up)', () => { +test('reconcile rebuilds missing vm file from DB (restore pull-up)', async () => { const vm = seedVmFile('vm-t4') - initVmDbSync(project, { watch: false }) + await initVmDbSync(project, { watch: false }) const vmPath = path.join(project, 'vms', 'vm-t4.json') fs.rmSync(vmPath) - const res = reconcileVms(project) + const res = await reconcileVms(project) assert.equal(res.rebuilt, 1) assert.ok(fs.existsSync(vmPath)) const back = JSON.parse(fs.readFileSync(vmPath, 'utf8')) @@ -193,24 +204,24 @@ test('reconcile rebuilds missing vm file from DB (restore pull-up)', () => { assert.equal(back.claude.email, vm.claude.email) }) -test('removeVmFromDb drops the row', () => { +test('removeVmFromDb drops the row', async () => { seedVmFile('vm-t5') - initVmDbSync(project, { watch: false }) - assert.equal(removeVmFromDb('vm-t5'), true) - assert.equal(new VmsRepo(getDb()).get('vm-t5'), null) + await initVmDbSync(project, { watch: false }) + assert.equal(await removeVmFromDb('vm-t5'), true) + assert.equal(await new VmsRepo(getDb()).get('vm-t5'), null) }) -test('KIN_DB_SECRET encrypts credentials at rest, decrypts on read', () => { +test('KIN_DB_SECRET encrypts credentials at rest, decrypts on read', async () => { process.env.KIN_DB_SECRET = 'test-secret-123' seedVmFile('vm-t6') - initVmDbSync(project, { watch: false }) + await initVmDbSync(project, { watch: false }) // raw row must be ciphertext - const raw = getDb().prepare('SELECT access_token, vm_json, encrypted FROM vms WHERE id = ?').get('vm-t6') + const raw = await getDb().prepare('SELECT access_token, vm_json, encrypted FROM vms WHERE id = $1').get('vm-t6') assert.equal(raw.encrypted, 1) assert.equal(raw.access_token, null) assert.ok(isEncrypted(raw.vm_json), 'vm_json should be encrypted') assert.doesNotMatch(String(raw.vm_json), /sk-ant-oat01/) - const row = new VmsRepo(getDb()).get('vm-t6') + const row = await new VmsRepo(getDb()).get('vm-t6') assert.equal(row.access_token, null) assert.equal(row.vm.claude.has_access, true) assert.equal(row.vm.claude.refresh_token, undefined) @@ -224,28 +235,89 @@ test('secure round-trip + tamper detection', () => { assert.equal(decryptString('plain-value', 's3cret'), 'plain-value') }) -test('syncVmFile no-ops safely when DB is closed', () => { +test('syncVmFile no-ops safely when DB is closed', async () => { seedVmFile('vm-t7') - closeDatabase() - assert.equal(syncVmFile(path.join(project, 'vms', 'vm-t7.json')), false) - assert.equal(removeVmFromDb('vm-t7'), false) + await closeDatabase() + assert.equal(await syncVmFile(path.join(project, 'vms', 'vm-t7.json')), false) + assert.equal(await removeVmFromDb('vm-t7'), false) // atomicWriteJson still works without a hook/DB atomicWriteJson(path.join(project, 'vms', 'vm-t7.json'), { id: 'vm-t7' }) }) test('file newer than DB wins on reconcile (mtime compare)', async () => { seedVmFile('vm-t8') - initVmDbSync(project, { watch: false }) + await initVmDbSync(project, { watch: false }) // simulate external edit with a newer mtime await new Promise((r) => setTimeout(r, 20)) const vmPath = path.join(project, 'vms', 'vm-t8.json') const vm = JSON.parse(fs.readFileSync(vmPath, 'utf8')) vm.claude.access_token = 'sk-ant-oat01-EXTERNAL' fs.writeFileSync(vmPath, JSON.stringify(vm, null, 2)) - const res = reconcileVms(project) + const res = await reconcileVms(project) assert.equal(res.upserted, 1) - const row = new VmsRepo(getDb()).get('vm-t8') + const row = await new VmsRepo(getDb()).get('vm-t8') assert.equal(row.access_token, null) assert.equal(row.vm.claude.has_access, true) assert.equal(row.vm.claude.access_token, undefined) }) + +test('queued VM upserts finish before deletion and cannot rebuild the deleted VM', async () => { + seedVmFile('vm-ordered') + await initVmDbSync(project, { watch: false }) + const file = path.join(project, 'vms', 'vm-ordered.json') + const vm = JSON.parse(fs.readFileSync(file, 'utf8')) + atomicWriteJson(file, { ...vm, name: 'first update' }) + atomicWriteJson(file, { ...vm, name: 'last update' }) + fs.unlinkSync(file) + assert.equal(await removeVmFromDb('vm-ordered'), true) + await flushVmDbSync() + const restored = await reconcileVms(project) + assert.equal(restored.rebuilt, 0) + assert.equal(await new VmsRepo(getDb()).get('vm-ordered'), null) + assert.equal(fs.existsSync(file), false) +}) + +test('reconciliation cannot overwrite a newer queued VM snapshot', async () => { + seedVmFile('vm-race', { name: 'old snapshot' }) + await initVmDbSync(project, { watch: false }) + const db = getDb() + await db.prepare('UPDATE vms SET file_mtime_ms = 0 WHERE id = $1').run('vm-race') + const query = db.query.bind(db) + let enteredResolve + let releaseResolve + const entered = new Promise((resolve) => { + enteredResolve = resolve + }) + const release = new Promise((resolve) => { + releaseResolve = resolve + }) + let intercepted = false + db.query = async (sql, values) => { + const result = await query(sql, values) + if (!intercepted && sql === 'SELECT * FROM vms WHERE id = $1') { + intercepted = true + enteredResolve() + await release + } + return result + } + const reconcile = reconcileVms(project) + let flush + try { + await entered + atomicWriteJson(path.join(project, 'vms', 'vm-race.json'), { id: 'vm-race', name: 'new snapshot' }) + let flushed = false + flush = flushVmDbSync().then(() => { + flushed = true + }) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(flushed, false, 'flush must wait for the reconciliation preceding the new snapshot') + } finally { + releaseResolve() + await reconcile + await flush + db.query = query + } + const row = await new VmsRepo(db).get('vm-race') + assert.equal(row.vm.name, 'new snapshot') +}) diff --git a/test/unit/vm-package.test.mjs b/test/unit/vm-package.test.mjs index 7f7dc1bef..3217b9a32 100644 --- a/test/unit/vm-package.test.mjs +++ b/test/unit/vm-package.test.mjs @@ -83,12 +83,12 @@ function makeHandler(project, body, { pool, role = 'admin', panelUsers = null } return { handlePanel, response } } -function makePool(t) { - const db = createDatabase({ dbPath: ':memory:' }) - const pool = new ProxyPool({ db }) - t.after(() => { +async function makePool(t) { + const db = await createDatabase({ dbPath: ':memory:' }) + const pool = await new ProxyPool({ db }).init() + t.after(async () => { pool.stopScheduler() - db.close() + await db.close() }) return pool } @@ -134,7 +134,7 @@ test('package export keeps a window whose stored fields are partly invalid', () test('import creates the slot and rolls a colliding id forward', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vm-pkg-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const pool = makePool(t) + const pool = await makePool(t) const doc = claudePackage('vm-pkg') const first = makeHandler(root, doc, { pool }) await first.handlePanel({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/package')) @@ -174,7 +174,7 @@ test('import creates the slot and rolls a colliding id forward', async (t) => { test('write-back updates the current slot and leaves runtime counters', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vm-pkg-put-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const pool = makePool(t) + const pool = await makePool(t) const created = makeHandler(root, claudePackage('vm-pkg'), { pool }) await created.handlePanel({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/package')) assert.equal(created.response.status, 201, created.response.body?.error?.message) @@ -204,7 +204,7 @@ test('write-back updates the current slot and leaves runtime counters', async (t test('local egress imports as px-local and codex tokens stay out of vm.json', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vm-pkg-kind-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const pool = makePool(t) + const pool = await makePool(t) const local = claudePackage('vm-local') local.proxy = { kind: 'local' } const localRes = makeHandler(root, local, { pool }) @@ -249,7 +249,7 @@ test('local egress imports as px-local and codex tokens stay out of vm.json', as test('user cannot download a package and cannot import past the create quota', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vm-pkg-user-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const pool = makePool(t) + const pool = await makePool(t) const admin = makeHandler(root, claudePackage('vm-pkg'), { pool }) await admin.handlePanel({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/package')) assert.equal(admin.response.status, 201, admin.response.body?.error?.message) @@ -275,7 +275,7 @@ test('user cannot download a package and cannot import past the create quota', a test('tenant import cannot bind px-local or another owner SOCKS and drops seed_policy', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vm-pkg-tenant-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const pool = makePool(t) + const pool = await makePool(t) const panelUsers = { getById: () => ({ vm_create_quota: 5 }) } const importAs = async (doc) => { const h = makeHandler(root, doc, { pool, role: 'user', panelUsers }) @@ -315,7 +315,7 @@ test('tenant import cannot bind px-local or another owner SOCKS and drops seed_p test('admin export returns the SOCKS password', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vm-pkg-get-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const pool = makePool(t) + const pool = await makePool(t) const created = makeHandler(root, claudePackage('vm-pkg'), { pool }) await created.handlePanel({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/package')) const got = makeHandler(root, null, { pool }) diff --git a/test/unit/vm-pools.test.mjs b/test/unit/vm-pools.test.mjs index 48641dcd2..622b415d9 100644 --- a/test/unit/vm-pools.test.mjs +++ b/test/unit/vm-pools.test.mjs @@ -16,49 +16,55 @@ function open() { return { dir, store, repo: new VmPoolsRepo(store.db) } } -test('pool membership is shared by keys and applied on the next read', () => { +test('pool membership is shared by keys and applied on the next read', async () => { const { dir, store, repo } = open() try { - const pool = createVmPool(repo, { name: 'Pool A', vm_ids: ['vm-01', 'vm-02'] }, vms) - const key = store.create({ name: 'a', vm_pool_id: pool.id }) - const other = store.create({ name: 'b', vm_pool_id: pool.id }) + const pool = await createVmPool(repo, { name: 'Pool A', vm_ids: ['vm-01', 'vm-02'] }, vms) + const key = await store.create({ name: 'a', vm_pool_id: pool.id }) + const other = await store.create({ name: 'b', vm_pool_id: pool.id }) assert.equal(key.vm_pool_id, pool.id) assert.equal(other.vm_pool_id, pool.id) - const before = applyVmPool(keyScopeFromRecord(store.getById(key.id)), (id) => repo.get(id)) + const before = await applyVmPool(keyScopeFromRecord(await store.getById(key.id)), async (id) => await repo.get(id)) assert.deepEqual(before.allowed_vms, ['vm-01', 'vm-02']) - updateVmPool(repo, pool.id, { vm_ids: ['vm-03'] }, vms) - const after = applyVmPool(keyScopeFromRecord(store.getById(other.id)), (id) => repo.get(id)) + await updateVmPool(repo, pool.id, { vm_ids: ['vm-03'] }, vms) + const after = await applyVmPool(keyScopeFromRecord(await store.getById(other.id)), async (id) => await repo.get(id)) assert.deepEqual(after.allowed_vms, ['vm-03']) assert.equal(after.vm_pool_error, null) - assert.throws(() => createVmPool(repo, { name: 'Pool B', vm_ids: ['vm-03'] }, vms), /其他账号池/) + await assert.rejects(async () => await createVmPool(repo, { name: 'Pool B', vm_ids: ['vm-03'] }, vms), /其他账号池/) assert.equal( - repo.list().some((item) => item.name === 'Pool B'), + (await repo.list()).some((item) => item.name === 'Pool B'), false, ) - updateVmPool(repo, pool.id, { enabled: false }, vms) - const disabled = applyVmPool(keyScopeFromRecord(store.getById(key.id)), (id) => repo.get(id)) + await updateVmPool(repo, pool.id, { enabled: false }, vms) + const disabled = await applyVmPool( + keyScopeFromRecord(await store.getById(key.id)), + async (id) => await repo.get(id), + ) assert.equal(disabled.vm_pool_error, 'disabled') assert.deepEqual(disabled.allowed_vms, []) - assert.equal(deleteVmPool(repo, pool.id).error, 'vm_pool_in_use') - store.update(key.id, { vm_pool_id: null }) - store.update(other.id, { vm_pool_id: null }) - assert.equal(deleteVmPool(repo, pool.id).ok, true) - const gone = applyVmPool(keyScopeFromRecord({ vm_pool_id: pool.id, group_type: 'all' }), (id) => repo.get(id)) + assert.equal((await deleteVmPool(repo, pool.id)).error, 'vm_pool_in_use') + await store.update(key.id, { vm_pool_id: null }) + await store.update(other.id, { vm_pool_id: null }) + assert.equal((await deleteVmPool(repo, pool.id)).ok, true) + const gone = await applyVmPool( + keyScopeFromRecord({ vm_pool_id: pool.id, group_type: 'all' }), + async (id) => await repo.get(id), + ) assert.equal(gone.vm_pool_error, 'missing') } finally { fs.rmSync(dir, { recursive: true, force: true }) } }) -test('an empty pool stays empty instead of meaning every VM', () => { +test('an empty pool stays empty instead of meaning every VM', async () => { const { dir, store, repo } = open() try { - const pool = createVmPool(repo, { name: 'Empty' }, vms) - const key = store.create({ name: 'e', vm_pool_id: pool.id }) - const scope = applyVmPool(keyScopeFromRecord(store.getById(key.id)), (id) => repo.get(id)) + const pool = await createVmPool(repo, { name: 'Empty' }, vms) + const key = await store.create({ name: 'e', vm_pool_id: pool.id }) + const scope = await applyVmPool(keyScopeFromRecord(await store.getById(key.id)), async (id) => await repo.get(id)) assert.equal(scope.vm_pool_error, 'empty') assert.deepEqual(scope.allowed_vms, []) - assert.equal(store.list()[0].vm_pool_name, 'Empty') + assert.equal((await store.list())[0].vm_pool_name, 'Empty') } finally { fs.rmSync(dir, { recursive: true, force: true }) } diff --git a/test/unit/vm-quota-override.test.mjs b/test/unit/vm-quota-override.test.mjs index 781f6bd33..2b71f6ac0 100644 --- a/test/unit/vm-quota-override.test.mjs +++ b/test/unit/vm-quota-override.test.mjs @@ -6,8 +6,8 @@ import path from 'node:path' import { AccountQuota } from '../../src/lib/pool/account-quota.mjs' import { parseVmQuotaOverride, vmQuotaOverrideOf } from '../../src/lib/pool/vm-quota-override.mjs' -function quota() { - return new AccountQuota({ +async function quota() { + return await new AccountQuota({ dataDir: fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vm-quota-')), config: { quota: { block_on_5h: true, block_on_7d: true }, @@ -17,7 +17,7 @@ function quota() { { account_id: 'acc-a', vm_id: 'vm-a' }, { account_id: 'acc-b', vm_id: 'vm-b' }, ], - }) + }).init() } function setUtil(q, accountId, u5, u7 = 0.1) { @@ -32,8 +32,8 @@ function setUtil(q, accountId, u5, u7 = 0.1) { ) } -test('VM 5h threshold override only moves that VM gate', () => { - const q = quota() +test('VM 5h threshold override only moves that VM gate', async () => { + const q = await quota() q.setVmQuotaOverride('vm-a', { limit_5h: 0.95 }) setUtil(q, 'acc-a', 0.9) setUtil(q, 'acc-b', 0.9) @@ -41,8 +41,8 @@ test('VM 5h threshold override only moves that VM gate', () => { assert.equal(q.canAccept('acc-b').reason, 'quota_5h_safety') }) -test('VM block switch off lets the slot run past its threshold; clearing restores global', () => { - const q = quota() +test('VM block switch off lets the slot run past its threshold; clearing restores global', async () => { + const q = await quota() setUtil(q, 'acc-a', 0.9) q.setVmQuotaOverride('vm-a', { block_on_5h: false }) assert.equal(q.canAccept('acc-a').ok, true) diff --git a/test/unit/vm-test-chat-billing.test.mjs b/test/unit/vm-test-chat-billing.test.mjs index 40d979103..23e7189cc 100644 --- a/test/unit/vm-test-chat-billing.test.mjs +++ b/test/unit/vm-test-chat-billing.test.mjs @@ -70,7 +70,7 @@ test('runVmTestChat without project/vm does not invent a panel request_log', asy const result = await runVmTestChat({ vmId: 'vm-01', requestLog: store }) assert.equal(result.ok, false) assert.equal(result.error?.code, 'invalid_request') - assert.equal(store.listNormal({ limit: 5 }).length, 0) + assert.equal((await store.listNormal({ limit: 5 })).length, 0) }) test('runVmTestChat loopbacks /v1/messages with x-kin-vm pin and never calls a worker', async (t) => { diff --git a/test/unit/vm-test-chat-codex.test.mjs b/test/unit/vm-test-chat-codex.test.mjs index b29995a6a..9777f891b 100644 --- a/test/unit/vm-test-chat-codex.test.mjs +++ b/test/unit/vm-test-chat-codex.test.mjs @@ -66,9 +66,9 @@ async function startLoopbackServer(t, respond) { return { baseUrl: `http://127.0.0.1:${server.address().port}`, calls } } -test('listTestableModels(openai) hides Claude ids', () => { - const gpt = listTestableModels('openai') - const claude = listTestableModels('anthropic') +test('listTestableModels(openai) hides Claude ids', async () => { + const gpt = await listTestableModels('openai') + const claude = await listTestableModels('anthropic') assert.ok(gpt.length > 0) assert.ok(claude.length > 0) assert.ok(gpt.every((m) => /^(gpt-|codex-)/i.test(m.id) || m.family === 'codex')) @@ -294,7 +294,7 @@ test('runVmTestChat Codex ENOENT is not rewritten as wrap cli-hop', async (t) => t.after(() => fs.rmSync(root, { recursive: true, force: true })) seedCodexVm(root) seedRouting(root) - const model = listTestableModels('openai')[0]?.id || 'gpt-5.5' + const model = (await listTestableModels('openai'))[0]?.id || 'gpt-5.5' const { baseUrl } = await startLoopbackServer(t, () => ({ status: 503, body: { diff --git a/test/unit/vm-test-schedule.test.mjs b/test/unit/vm-test-schedule.test.mjs index 21249af08..bb231a2ae 100644 --- a/test/unit/vm-test-schedule.test.mjs +++ b/test/unit/vm-test-schedule.test.mjs @@ -3,8 +3,8 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { DatabaseSync } from 'node:sqlite' -import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' +import { createDatabase } from '../../src/lib/db/database.mjs' +import { loadSettingsState, RuntimeSettings as SettingsRepo } from '../../src/lib/db/settings-state.mjs' import { compileStructuredSchedule, normalizeScheduleStructure, @@ -94,36 +94,36 @@ test('legacy cron input must match structured patterns', () => { ) }) -test('SettingsRepo schedules reload after restart without an immediate catch-up run', (t) => { +test('SettingsRepo schedules reload after restart without an immediate catch-up run', async (t) => { const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-test-schedule-')) - const dbPath = path.join(dataDir, 'settings.db') - let db = new DatabaseSync(dbPath) - db.exec('CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at TEXT NOT NULL)') + let db = await createDatabase() + await loadSettingsState(db) let settings = new SettingsRepo(db) let first let restored - t.after(() => { + t.after(async () => { first?.stop() restored?.stop() - db.close() + await db.close() fs.rmSync(dataDir, { recursive: true, force: true }) }) first = runner({ settings }).schedule - first.save('vm-01', claudeConfig) + await first.save('vm-01', claudeConfig) assert.deepEqual( Object.keys(settings.all()).filter((key) => key.startsWith(VM_TEST_SCHEDULE_PREFIX)), [`${VM_TEST_SCHEDULE_PREFIX}vm-01`], ) - first.start() + await first.start() assert.equal(first.jobs.has('vm-01'), true) assert.equal(first.get('vm-01').running, false) assert.ok(first.get('vm-01').next_run_at) assert.equal(first.get('vm-01').cron, undefined) first.stop() - db.close() - db = new DatabaseSync(dbPath) + await db.close() + db = await createDatabase() + await loadSettingsState(db) settings = new SettingsRepo(db) restored = new VmTestScheduleRunner({ projectRoot: '/missing-project', @@ -131,7 +131,7 @@ test('SettingsRepo schedules reload after restart without an immediate catch-up list: () => [claudeVm], runTest: async () => ({ ok: true }), }) - restored.start() + await restored.start() assert.equal(restored.jobs.has('vm-01'), true) assert.equal(restored.get('vm-01').enabled, true) restored.stop() @@ -148,8 +148,8 @@ test('runs do not overlap per VM and failures persist only compact result fields }) }, }) - schedule.save('vm-01', claudeConfig) - schedule.start() + await schedule.save('vm-01', claudeConfig) + await schedule.start() const job = schedule.jobs.get('vm-01') const first = job.trigger() @@ -175,11 +175,11 @@ test('runs do not overlap per VM and failures persist only compact result fields schedule.stop() }) -test('sync overwrites schedule only and creates disabled plans with default dialog params', () => { +test('sync overwrites schedule only and creates disabled plans with default dialog params', async () => { const codexVm = { id: 'vm-02', platform: 'openai' } const { settings, schedule } = runner({ list: () => [claudeVm, codexVm] }) - schedule.save('vm-01', { ...claudeConfig, prompt: 'keep-me', enabled: true }) - const result = schedule.syncSchedules({ + await schedule.save('vm-01', { ...claudeConfig, prompt: 'keep-me', enabled: true }) + const result = await schedule.syncSchedules({ scope: 'all', schedule: normalizeScheduleStructure({ frequency: 'hourly', @@ -202,9 +202,9 @@ test('sync overwrites schedule only and creates disabled plans with default dial assert.equal(created.reasoning_effort, 'medium') }) -test('default template persists schedule structure only', () => { +test('default template persists schedule structure only', async () => { const { schedule } = runner() - const saved = schedule.saveDefaultTemplate({ + const saved = await schedule.saveDefaultTemplate({ frequency: 'weekly', time: '10:15', weekdays: [5], @@ -212,7 +212,7 @@ test('default template persists schedule structure only', () => { }) assert.equal(saved.frequency, 'weekly') assert.deepEqual(schedule.getDefaultTemplate(), saved) - schedule.clearDefaultTemplate() + await schedule.clearDefaultTemplate() assert.equal(schedule.getDefaultTemplate(), null) }) @@ -231,12 +231,12 @@ test('unparseable stored cron surfaces parse_error without exposing cron string' assert.equal(view.cron, undefined) }) -test('sync with test copies switch and dialog to every slot of the same kind', () => { +test('sync with test copies switch and dialog to every slot of the same kind', async () => { const codexVm = { id: 'vm-02', platform: 'openai' } const otherClaude = { id: 'vm-03', platform: 'anthropic' } const { settings, schedule } = runner({ list: () => [claudeVm, codexVm, otherClaude] }) - schedule.save('vm-01', { ...claudeConfig, enabled: false }) - schedule.save('vm-02', { + await schedule.save('vm-01', { ...claudeConfig, enabled: false }) + await schedule.save('vm-02', { enabled: false, frequency: 'daily', time: '08:00', @@ -246,7 +246,7 @@ test('sync with test copies switch and dialog to every slot of the same kind', ( max_tokens: null, reasoning_effort: 'low', }) - const result = schedule.syncSchedules({ + const result = await schedule.syncSchedules({ scope: 'all', schedule: normalizeScheduleStructure({ frequency: 'hourly', minute: 0, timezone: 'Asia/Shanghai' }), test: { enabled: true, model: 'claude-haiku-5-5', prompt: 'ping', max_tokens: 32 }, @@ -267,7 +267,7 @@ test('sync with test copies switch and dialog to every slot of the same kind', ( assert.equal(codex.reasoning_effort, 'low') assert.equal(codex.max_tokens, null) - schedule.syncSchedules({ + await schedule.syncSchedules({ scope: 'all', schedule: normalizeScheduleStructure({ frequency: 'hourly', minute: 0, timezone: 'Asia/Shanghai' }), test: { enabled: false, model: 'claude-haiku-5-5', prompt: 'ping', max_tokens: 32 }, @@ -277,22 +277,86 @@ test('sync with test copies switch and dialog to every slot of the same kind', ( } }) -test('sync with an invalid test writes nothing', () => { +test('sync with an invalid test writes nothing', async () => { const { settings, schedule } = runner({ list: () => [claudeVm, { id: 'vm-03', platform: 'anthropic' }] }) - schedule.save('vm-01', { ...claudeConfig, enabled: false }) + await schedule.save('vm-01', { ...claudeConfig, enabled: false }) const before = settings.all() for (const test of [ { enabled: 'yes', model: 'm', prompt: 'p', max_tokens: 1 }, { enabled: true, model: '', prompt: 'p', max_tokens: 1 }, { enabled: true, model: 'm', prompt: 'p' }, ]) { - assert.throws(() => - schedule.syncSchedules({ - scope: 'all', - schedule: normalizeScheduleStructure({ frequency: 'hourly', minute: 0, timezone: 'Asia/Shanghai' }), - test, - }), + await assert.rejects( + async () => + await schedule.syncSchedules({ + scope: 'all', + schedule: normalizeScheduleStructure({ frequency: 'hourly', minute: 0, timezone: 'Asia/Shanghai' }), + test, + }), ) } assert.deepEqual(settings.all(), before) }) + +test('flush waits until a scheduled result finishes persisting', async () => { + const { settings, schedule } = runner() + await schedule.save('vm-01', claudeConfig) + schedule.started = true + let release + let entered + const gate = new Promise((resolve) => { + release = resolve + }) + const writing = new Promise((resolve) => { + entered = resolve + }) + const originalSet = settings.set.bind(settings) + settings.set = async (...args) => { + entered() + await gate + return originalSet(...args) + } + const run = schedule._run('vm-01') + await writing + let flushed = false + const flush = schedule.flush().then(() => { + flushed = true + }) + try { + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(flushed, false, 'the scheduled result is still being written') + } finally { + release() + await run + await flush + schedule.stop() + } +}) + +test('flush reports a failed scheduled result write', async () => { + const { settings, schedule } = runner() + await schedule.save('vm-01', claudeConfig) + schedule.started = true + const failure = new Error('synthetic scheduled result write failure') + let release + let entered + const gate = new Promise((resolve) => { + release = resolve + }) + const writing = new Promise((resolve) => { + entered = resolve + }) + settings.set = async () => { + entered() + await gate + throw failure + } + const run = schedule._run('vm-01') + run.catch(() => {}) + await writing + const flush = schedule.flush() + release() + await assert.rejects(run, (error) => error === failure) + await assert.rejects(flush, (error) => error === failure) + schedule.stop() +}) diff --git a/test/unit/weekly-split.test.mjs b/test/unit/weekly-split.test.mjs index 41c894357..2998eb3fc 100644 --- a/test/unit/weekly-split.test.mjs +++ b/test/unit/weekly-split.test.mjs @@ -97,12 +97,12 @@ test('misaligned 7d vs 7d_oi uses the difference as regular usage', () => { assert.equal(s.mode, 'open') }) -test('AccountQuota.weeklySplitOf reads unified windows and config', () => { +test('AccountQuota.weeklySplitOf reads unified windows and config', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-wsplit-')) - const q = new AccountQuota({ + const q = await new AccountQuota({ dataDir: dir, config: { quota: { weekly_split: { enabled: true, fable_share: 0.5 } } }, - }) + }).init() q.ingestHeaders('acc-1', { 'anthropic-ratelimit-unified-7d-utilization': '0.50', 'anthropic-ratelimit-unified-7d_oi-utilization': '0', diff --git a/test/unit/write-queue.test.mjs b/test/unit/write-queue.test.mjs new file mode 100644 index 000000000..3db293572 --- /dev/null +++ b/test/unit/write-queue.test.mjs @@ -0,0 +1,38 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { WriteQueue } from '../../src/lib/pool/write-queue.mjs' + +test('flush waits for ordered writes even when the first write is delayed', async () => { + const queue = new WriteQueue() + const writes = [] + let release + const gate = new Promise((resolve) => { + release = resolve + }) + queue.enqueue(async () => { + await gate + writes.push('first') + }) + queue.enqueue(() => writes.push('second')) + const flushed = queue.flush() + await Promise.resolve() + assert.deepEqual(writes, []) + release() + await flushed + assert.deepEqual(writes, ['first', 'second']) +}) + +test('failed persistence is reported by flush and later writes cannot report success', async () => { + const queue = new WriteQueue() + const failure = Object.assign(new Error('synthetic persistence failure'), { code: 'test_failure' }) + let wroteAfterFailure = false + queue.enqueue(() => { + throw failure + }) + queue.enqueue(() => { + wroteAfterFailure = true + }) + await assert.rejects(queue.flush(), (error) => error === failure) + assert.equal(wroteAfterFailure, false) + await assert.rejects(queue.flush(), (error) => error === failure) +}) diff --git a/web/src/features/database/database-overview.tsx b/web/src/features/database/database-overview.tsx index 584bd148c..29f8dbfec 100644 --- a/web/src/features/database/database-overview.tsx +++ b/web/src/features/database/database-overview.tsx @@ -4,7 +4,6 @@ import { PanelCard, StatCell } from '@/features/overview/panel-card' import { formatMetricBytes, formatMetricNumber, - formatMetricPercent, formatMetricTime, } from './metrics' @@ -14,17 +13,6 @@ function latency(value: number | null): string { return `${Math.round(value)}ms` } -function synchronousLabel(value: number | null): string { - if (value == null) return '—' - return ['OFF', 'NORMAL', 'FULL', 'EXTRA'][value] || String(value) -} - -function timeout(value: number | null): string { - if (value == null || !Number.isFinite(value)) return '—' - if (value >= 1000 && value % 1000 === 0) return `${value / 1000}s` - return `${value.toLocaleString()}ms` -} - function Detail({ label, value }: { label: string; value: string }) { return (
@@ -43,11 +31,6 @@ export function DatabaseOverview({ metrics: DatabaseRuntimeMetrics sampledAt: string | null }) { - const pageUse = - metrics.used_pages == null && metrics.page_count == null - ? '—' - : `${formatMetricNumber(metrics.used_pages)} / ${formatMetricNumber(metrics.page_count)}` - return ( - - - - -

- 当前 node:sqlite 接口不提供 SQLite 页缓存 - hit/miss；这里展示的是文件、页使用和只读探针，不把应用缓存冒充数据库缓存。 + 连接池指标属于当前控制面进程。数据库大小和健康探针通过只读查询获取。

) diff --git a/web/src/types/panel-database.ts b/web/src/types/panel-database.ts index 0f827f90e..ada58f63a 100644 --- a/web/src/types/panel-database.ts +++ b/web/src/types/panel-database.ts @@ -1,20 +1,11 @@ export type DatabaseRuntimeMetrics = { ok: boolean - engine: 'sqlite' + engine: 'postgresql' probe_latency_ms: number | null - journal_mode: string | null - synchronous: number | null - foreign_keys: boolean | null - busy_timeout_ms: number | null - file_size_bytes: number | null - wal_size_bytes: number | null - shm_size_bytes: number | null - page_size_bytes: number | null - page_count: number | null - freelist_pages: number | null - used_pages: number | null - used_bytes: number | null - free_ratio: number | null + size_bytes: number | null + pool_total: number | null + pool_idle: number | null + pool_waiting: number | null migration_count: number | null latest_migration: string | null latest_migration_at: string | null