From ad921b0573d853a07237f2b3e6ca9f213fa7a05c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 21 Oct 2026 01:59:39 +0000 Subject: [PATCH 1/5] fix(fleet): skip telemetry refresh when slot container is absent (#628) REQ-0628: install.sh upgrade and fleet action=telemetry no longer fail the whole batch when a DB-registered VM has no kin-NN container on the host. Log an explicit skip, continue other VMs, and still fail on real refresh errors. Co-authored-by: ted --- CHANGELOG.md | 1 + scripts/fleet-services.mjs | 9 +++++++ src/lib/cluster/remote-slot.mjs | 6 +++++ src/lib/vm/docker-container.mjs | 21 +++++++++++++++ src/lib/vm/fleet-update.mjs | 43 ++++++++++++++++++++++++------ src/lib/vm/slot-process-status.mjs | 20 ++++++++++++-- test/unit/fleet-update.test.mjs | 31 +++++++++++++++++++++ 7 files changed, 121 insertions(+), 10 deletions(-) create mode 100644 src/lib/vm/docker-container.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b4cc3007..80870f4c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ ## 1.3.206 — 2026-10-21 +- 一键升级与 fleet `action=telemetry` 刷新 Go 槽位服务时,数据库仍登记但本机已无 Docker 容器(`No such container: kin-NN`)的 VM 记为跳过并继续处理其余槽位,不再因此整批失败;真实容器上的刷新失败仍会硬失败。(#628,REQ-0628) - 官方登录终端把授权链接显式提取出来,跨终端帧也不会漏。复制链接、打开授权页和回填授权码与普通 OAuth 相同。 - VM 页 OAuth 方式顺序为 Claude Code、授权链接、Cookie。设置里的「同步遥测」按默认关闭显示。 diff --git a/scripts/fleet-services.mjs b/scripts/fleet-services.mjs index adcfa2128..0ffbb5bf3 100644 --- a/scripts/fleet-services.mjs +++ b/scripts/fleet-services.mjs @@ -7,4 +7,13 @@ const report = await runFleetUpdate(root, { concurrency, }) console.log(JSON.stringify(report)) +if (report.skipped_count) { + for (const item of report.items) { + if (!item.skipped) continue + const container = item.telemetry?.container || item.id + console.warn( + `[fleet-services] skipped ${item.id}: container unavailable (${container})`, + ) + } +} if (report.failed_count) process.exit(1) diff --git a/src/lib/cluster/remote-slot.mjs b/src/lib/cluster/remote-slot.mjs index 10d057985..d4d4b1490 100644 --- a/src/lib/cluster/remote-slot.mjs +++ b/src/lib/cluster/remote-slot.mjs @@ -33,6 +33,7 @@ import { isCodexVm } from '../vm/vm-kind.mjs' import { proxyBlockedReason } from '../vm/proxy-policy.mjs' import { isKvmRuntime, runtimeKind } from '../vm/runtime-kind.mjs' import { slotMemory, slotDockerCpus, memoryMiB, normalizeVmConfig, VM_CONFIG_DEFAULTS } from '../vm/machine-spec.mjs' +import { logTelemetryContainerSkip, telemetryContainerSkip } from '../vm/docker-container.mjs' import { slotExecArgv, slotExecCmd } from '../vm/slot-exec.mjs' import { CODEX_SLOT_ROLE, @@ -1201,6 +1202,11 @@ export async function restartRemoteServices(vm, projectRoot, { telemetryEnabled const remoteDir = remoteSlotDir(session.host, vm.id) const slotDir = path.join(projectRoot, 'vms', vm.id) const name = vm.runtime?.container || containerName(vm.id) + const slotInfo = await inspectContainerOrNull(session.docker, name) + if (!slotInfo) { + logTelemetryContainerSkip('fleet', vm.id, name) + return telemetryContainerSkip(name) + } const migration = await migrateRemoteServiceContainers(vm, projectRoot) if (!migration.ok) return migration await pushSlotFiles(vm, slotDir, ['run', 'seed'], session) diff --git a/src/lib/vm/docker-container.mjs b/src/lib/vm/docker-container.mjs new file mode 100644 index 000000000..bb9489860 --- /dev/null +++ b/src/lib/vm/docker-container.mjs @@ -0,0 +1,21 @@ +/** Shared helpers for docker slot containers on the control plane / node host. */ + +export function isMissingContainerMessage(message) { + return /no such container/i.test(String(message || '')) +} + +export function telemetryContainerSkip(container, detail) { + return { + ok: true, + skipped: true, + action: 'skipped', + reason: 'container_unavailable', + container: String(container || ''), + ...(detail ? { detail: String(detail).slice(0, 300) } : {}), + } +} + +export function logTelemetryContainerSkip(scope, vmId, container, detail) { + const suffix = detail ? `: ${detail}` : '' + console.warn(`[${scope}] Go slot service refresh skipped ${vmId}: no container ${container} on this host${suffix}`) +} diff --git a/src/lib/vm/fleet-update.mjs b/src/lib/vm/fleet-update.mjs index fb303f288..32c0b3708 100644 --- a/src/lib/vm/fleet-update.mjs +++ b/src/lib/vm/fleet-update.mjs @@ -12,6 +12,11 @@ import { restartTelemetrySidecar } from './slot-process-status.mjs' import { isNativeNodeVm, vmNodeId } from '../cluster/placement.mjs' import { slotHost } from './slot-host.mjs' import { restartRemoteServices } from '../cluster/remote-slot.mjs' +import { + isMissingContainerMessage, + logTelemetryContainerSkip, + telemetryContainerSkip, +} from './docker-container.mjs' import { drainEgressForProxyConfig, egressPidsForProxyConfig, @@ -62,7 +67,18 @@ async function restartLocalServices({ projectRoot, vm, enabled, egressOpts = {} if (!telemetry.ok) return telemetry const egress = await restartLocalEgress(projectRoot, vm, egressOpts) if (!egress.ok) return egress - return { ...telemetry, egress: egress.egress } + return { ...telemetry, egress: egress.egress ?? (telemetry.skipped ? 'unchanged' : undefined) } +} + +function normalizeTelemetryRefresh(changed, vm) { + if (changed?.ok || changed?.skipped) return changed + const message = String(changed?.error || changed?.code || '') + if (!isMissingContainerMessage(message)) return changed + const container = + String(vm?.runtime?.container || '').trim() || + (vm?.id ? `kin-${String(vm.id).replace(/^vm-/, '')}` : '') + logTelemetryContainerSkip('fleet', vm.id, container, message) + return telemetryContainerSkip(container, message) } function sleep(ms) { @@ -149,14 +165,23 @@ export async function updateOneSlot( const enabled = telemetryEnabled === undefined ? vm.seed_policy?.telemetry_disabled === false : telemetryEnabled === true const vmRef = getVm(projectRoot, id) || vm - const changed = telemetryFn - ? await telemetryFn({ projectRoot, vm: vmRef, telemetryEnabled: enabled, enabled }) - : isNativeNodeVm(vmRef) - ? await slotHost(vmRef).restartTelemetry(vmRef, { enabled }) - : vmNodeId(vm) - ? await restartRemoteServices(vmRef, projectRoot, { telemetryEnabled: enabled }) - : await restartLocalServices({ projectRoot, vm: vmRef, enabled }) + const changed = normalizeTelemetryRefresh( + telemetryFn + ? await telemetryFn({ projectRoot, vm: vmRef, telemetryEnabled: enabled, enabled }) + : isNativeNodeVm(vmRef) + ? await slotHost(vmRef).restartTelemetry(vmRef, { enabled }) + : vmNodeId(vm) + ? await restartRemoteServices(vmRef, projectRoot, { telemetryEnabled: enabled }) + : await restartLocalServices({ projectRoot, vm: vmRef, enabled }), + vmRef, + ) result.telemetry = changed + if (changed?.skipped) { + result.skipped = true + result.skip_reason = changed.reason || 'container_unavailable' + result.ok = true + return result + } if (!changed?.ok) { result.error = changed?.error || 'telemetry service update failed' result.code = changed?.code || 'telemetry_update_failed' @@ -211,11 +236,13 @@ export async function runFleetUpdate( }), ) const okCount = items.filter((row) => row.ok).length + const skippedCount = items.filter((row) => row.skipped).length return { action: nextAction, concurrency: conc, total: items.length, ok_count: okCount, + skipped_count: skippedCount, failed_count: items.length - okCount, items, } diff --git a/src/lib/vm/slot-process-status.mjs b/src/lib/vm/slot-process-status.mjs index 59e62a29f..2c8dde647 100644 --- a/src/lib/vm/slot-process-status.mjs +++ b/src/lib/vm/slot-process-status.mjs @@ -4,6 +4,12 @@ import { readFile } from 'node:fs/promises' import path from 'node:path' import { promisify } from 'node:util' import { slotContainerName } from '../transport/rust-kernel-supervisor.mjs' +import { + isMissingContainerMessage, + logTelemetryContainerSkip, + telemetryContainerSkip, +} from './docker-container.mjs' +import { inspectContainer } from './vm-runtime.mjs' import { slotHost } from './slot-host.mjs' import { runtimeKind } from './runtime-kind.mjs' import { slotExecArgv } from './slot-exec.mjs' @@ -115,7 +121,12 @@ export async function restartTelemetrySidecar({ projectRoot, vm, enabled, run = const host = slotHost(vm) if (host.native) return host.restartTelemetry(vm, { enabled }) const env = host.dockerEnv() - const target = { ...vm, runtime: { ...(vm.runtime || {}), container: slotContainerName({ vm }), type: kind } } + const container = slotContainerName({ vm }) + const target = { ...vm, runtime: { ...(vm.runtime || {}), container, type: kind } } + if (!inspectContainer(container)) { + logTelemetryContainerSkip('fleet', vm.id, container) + return telemetryContainerSkip(container) + } const stop = 'pkill -TERM -x kin-worker 2>/dev/null || true' try { await run('docker', slotExecArgv(target, ['sh', '-c', stop]), { @@ -137,6 +148,11 @@ export async function restartTelemetrySidecar({ projectRoot, vm, enabled, run = ) return { ok: true, action: 'restarted' } } catch (error) { - return { ok: false, action: 'restart_failed', error: String(error?.message || error) } + const message = String(error?.message || error) + if (isMissingContainerMessage(message)) { + logTelemetryContainerSkip('fleet', vm.id, container, message) + return telemetryContainerSkip(container, message) + } + return { ok: false, action: 'restart_failed', error: message } } } diff --git a/test/unit/fleet-update.test.mjs b/test/unit/fleet-update.test.mjs index 14aafff1d..dba398068 100644 --- a/test/unit/fleet-update.test.mjs +++ b/test/unit/fleet-update.test.mjs @@ -83,6 +83,37 @@ test('degraded worker without OAuth still counts as up', () => { assert.equal(isWorkerProcessUp({ code: 'worker_unavailable' }), false) }) +test('fleet telemetry skips db rows whose container is absent on this host', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-fleet-skip-')) + writeSlot(root, 'vm-07') + const report = await runFleetUpdate(root, { + action: 'telemetry', + telemetryFn: async () => ({ ok: false, error: 'Error: No such container: kin-07' }), + }) + assert.equal(report.total, 1) + assert.equal(report.ok_count, 1) + assert.equal(report.skipped_count, 1) + assert.equal(report.failed_count, 0) + assert.equal(report.items[0].skipped, true) + assert.equal(report.items[0].skip_reason, 'container_unavailable') + assert.equal(report.items[0].telemetry.skipped, true) + fs.rmSync(root, { recursive: true, force: true }) +}) + +test('fleet telemetry still fails when a present container refresh errors', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-fleet-hard-')) + writeSlot(root, 'vm-08') + const report = await runFleetUpdate(root, { + action: 'telemetry', + telemetryFn: async () => ({ ok: false, error: 'docker exec failed: permission denied' }), + }) + assert.equal(report.failed_count, 1) + assert.equal(report.ok_count, 0) + assert.equal(report.items[0].skipped, undefined) + assert.equal(report.items[0].code, 'telemetry_update_failed') + fs.rmSync(root, { recursive: true, force: true }) +}) + test('fleet telemetry uses injected telemetryFn with options object', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-fleet-tel-')) writeSlot(root, 'vm-01', { node_id: 'node-a' }) From 8acdbbd7944b92eb7f8c86783e8038bd02c6591f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 21 Oct 2026 02:01:59 +0000 Subject: [PATCH 2/5] style: biome format fleet skip changes Co-authored-by: ted --- scripts/fleet-services.mjs | 4 +--- src/lib/vm/fleet-update.mjs | 9 ++------- src/lib/vm/slot-process-status.mjs | 6 +----- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/scripts/fleet-services.mjs b/scripts/fleet-services.mjs index 0ffbb5bf3..5b9f36345 100644 --- a/scripts/fleet-services.mjs +++ b/scripts/fleet-services.mjs @@ -11,9 +11,7 @@ if (report.skipped_count) { for (const item of report.items) { if (!item.skipped) continue const container = item.telemetry?.container || item.id - console.warn( - `[fleet-services] skipped ${item.id}: container unavailable (${container})`, - ) + console.warn(`[fleet-services] skipped ${item.id}: container unavailable (${container})`) } } if (report.failed_count) process.exit(1) diff --git a/src/lib/vm/fleet-update.mjs b/src/lib/vm/fleet-update.mjs index 32c0b3708..ad36fe746 100644 --- a/src/lib/vm/fleet-update.mjs +++ b/src/lib/vm/fleet-update.mjs @@ -12,11 +12,7 @@ import { restartTelemetrySidecar } from './slot-process-status.mjs' import { isNativeNodeVm, vmNodeId } from '../cluster/placement.mjs' import { slotHost } from './slot-host.mjs' import { restartRemoteServices } from '../cluster/remote-slot.mjs' -import { - isMissingContainerMessage, - logTelemetryContainerSkip, - telemetryContainerSkip, -} from './docker-container.mjs' +import { isMissingContainerMessage, logTelemetryContainerSkip, telemetryContainerSkip } from './docker-container.mjs' import { drainEgressForProxyConfig, egressPidsForProxyConfig, @@ -75,8 +71,7 @@ function normalizeTelemetryRefresh(changed, vm) { const message = String(changed?.error || changed?.code || '') if (!isMissingContainerMessage(message)) return changed const container = - String(vm?.runtime?.container || '').trim() || - (vm?.id ? `kin-${String(vm.id).replace(/^vm-/, '')}` : '') + String(vm?.runtime?.container || '').trim() || (vm?.id ? `kin-${String(vm.id).replace(/^vm-/, '')}` : '') logTelemetryContainerSkip('fleet', vm.id, container, message) return telemetryContainerSkip(container, message) } diff --git a/src/lib/vm/slot-process-status.mjs b/src/lib/vm/slot-process-status.mjs index 2c8dde647..4db3ac805 100644 --- a/src/lib/vm/slot-process-status.mjs +++ b/src/lib/vm/slot-process-status.mjs @@ -4,11 +4,7 @@ import { readFile } from 'node:fs/promises' import path from 'node:path' import { promisify } from 'node:util' import { slotContainerName } from '../transport/rust-kernel-supervisor.mjs' -import { - isMissingContainerMessage, - logTelemetryContainerSkip, - telemetryContainerSkip, -} from './docker-container.mjs' +import { isMissingContainerMessage, logTelemetryContainerSkip, telemetryContainerSkip } from './docker-container.mjs' import { inspectContainer } from './vm-runtime.mjs' import { slotHost } from './slot-host.mjs' import { runtimeKind } from './runtime-kind.mjs' From 59f102837f63ea834416b4f9feed2c3d0a423693 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 21 Oct 2026 02:09:19 +0000 Subject: [PATCH 3/5] test: stub inspectContainerOrNull in remote service refresh sandbox Co-authored-by: ted --- test/unit/remote-slot-migration.test.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/unit/remote-slot-migration.test.mjs b/test/unit/remote-slot-migration.test.mjs index 0b8220555..7ae4e355e 100644 --- a/test/unit/remote-slot-migration.test.mjs +++ b/test/unit/remote-slot-migration.test.mjs @@ -185,6 +185,9 @@ test('remote service refresh forwards the migrated binary digest to egress', asy nodeSession: async () => ({ host: {}, docker: {} }), remoteSlotDir: () => '/slots/kin-05', path, + inspectContainerOrNull: async () => ({ State: { Running: true } }), + logTelemetryContainerSkip: () => {}, + telemetryContainerSkip: () => ({ ok: true, skipped: true }), migrateRemoteServiceContainers: async () => ({ ok: true, migrated: [], binary_digest: 'new-binary-digest' }), pushSlotFiles: async () => {}, runtimeKind: () => 'docker', From 19e44174ba313982269730d81e2084e6aa1f9d56 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 22 Oct 2026 06:34:44 +0000 Subject: [PATCH 4/5] =?UTF-8?q?docs(changelog):=20move=20#628=20entry=20un?= =?UTF-8?q?der=20=E6=9C=AA=E5=8F=91=E5=B8=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: ted --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80870f4c8..c6dce32f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - 槽位「采集特征」和终端「系统特征」不再读到宿主机:Docker 槽的 CPU 型号、DMI、MAC 读槽位计算机(`/etc/kin-hardware`、`/etc/kin-dmi`),核数与内存读容器 cgroup 配额;没有配额时读计算机文件,都没有才回落 `/proc`、`/sys`。KVM 槽不变。(#673) - `kin-computer` 默认造 Linux PC 计算机(Docker 槽原为 macOS 人设),`--os` 可选 `windows` / `macos`,KVM 支持 linux / windows。payload 改指 `payload-2026.10.22-computer-linux`,只换 `kin-computer`。已有槽位身份不重算。(#673) +- 一键升级与 fleet `action=telemetry` 刷新 Go 槽位服务时,数据库仍登记但本机已无 Docker 容器(`No such container: kin-NN`)的 VM 记为跳过并继续处理其余槽位,不再因此整批失败;真实容器上的刷新失败仍会硬失败。(#642,#628,REQ-0628) 已部署机升级:更新 Node 控制面并执行 `npm run fetch:payload`(或更新控制面镜像)换上新 `bin/kin-computer`,重启一次 Node。无迁移、无新依赖,不必 `wrap-cli/sync`,不必重建槽位容器。不要覆盖 `routing.json`、`vms/`、`data/`、`.env`。 @@ -30,7 +31,6 @@ ## 1.3.206 — 2026-10-21 -- 一键升级与 fleet `action=telemetry` 刷新 Go 槽位服务时,数据库仍登记但本机已无 Docker 容器(`No such container: kin-NN`)的 VM 记为跳过并继续处理其余槽位,不再因此整批失败;真实容器上的刷新失败仍会硬失败。(#628,REQ-0628) - 官方登录终端把授权链接显式提取出来,跨终端帧也不会漏。复制链接、打开授权页和回填授权码与普通 OAuth 相同。 - VM 页 OAuth 方式顺序为 Claude Code、授权链接、Cookie。设置里的「同步遥测」按默认关闭显示。 From 871f448e98a7d90aee9e9798936c33ad8edee61d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 22 Oct 2026 06:43:44 +0000 Subject: [PATCH 5/5] fix(fleet): only skip telemetry refresh on confirmed missing container Distinguish docker inspect not-found from daemon/permission failures via inspectContainerOutcome. Pre-check skips only when absent; other inspect errors fail the slot like a real refresh failure. (#628, REQ-0628) Co-authored-by: ted --- src/lib/vm/docker-container.mjs | 6 ++ src/lib/vm/slot-process-status.mjs | 22 +++++- src/lib/vm/vm-runtime.mjs | 35 ++++++--- test/unit/docker-inspect-outcome.test.mjs | 48 ++++++++++++ test/unit/fleet-telemetry-inspect.test.mjs | 89 ++++++++++++++++++++++ 5 files changed, 187 insertions(+), 13 deletions(-) create mode 100644 test/unit/docker-inspect-outcome.test.mjs create mode 100644 test/unit/fleet-telemetry-inspect.test.mjs diff --git a/src/lib/vm/docker-container.mjs b/src/lib/vm/docker-container.mjs index bb9489860..3b55d0ffb 100644 --- a/src/lib/vm/docker-container.mjs +++ b/src/lib/vm/docker-container.mjs @@ -4,6 +4,12 @@ export function isMissingContainerMessage(message) { return /no such container/i.test(String(message || '')) } +/** Local `docker inspect` / Engine 404: container id or name is gone. */ +export function isDockerInspectNotFound({ stderr = '', stdout = '' } = {}) { + const hay = `${stderr}\n${stdout}` + return /no such container|no such object/i.test(hay) +} + export function telemetryContainerSkip(container, detail) { return { ok: true, diff --git a/src/lib/vm/slot-process-status.mjs b/src/lib/vm/slot-process-status.mjs index 4db3ac805..ce92f1100 100644 --- a/src/lib/vm/slot-process-status.mjs +++ b/src/lib/vm/slot-process-status.mjs @@ -5,7 +5,7 @@ import path from 'node:path' import { promisify } from 'node:util' import { slotContainerName } from '../transport/rust-kernel-supervisor.mjs' import { isMissingContainerMessage, logTelemetryContainerSkip, telemetryContainerSkip } from './docker-container.mjs' -import { inspectContainer } from './vm-runtime.mjs' +import { inspectContainerOutcome } from './vm-runtime.mjs' import { slotHost } from './slot-host.mjs' import { runtimeKind } from './runtime-kind.mjs' import { slotExecArgv } from './slot-exec.mjs' @@ -110,7 +110,14 @@ export async function ensureTelemetrySidecar({ projectRoot, vm, run = exec, time } /** Replace only the Go telemetry child; the slot container and Rust process stay up. */ -export async function restartTelemetrySidecar({ projectRoot, vm, enabled, run = exec, timeoutMs = 15_000 } = {}) { +export async function restartTelemetrySidecar({ + projectRoot, + vm, + enabled, + run = exec, + timeoutMs = 15_000, + inspectContainerOutcome: inspectOutcome = inspectContainerOutcome, +} = {}) { if (!projectRoot || !isSafeSlotId(vm?.id)) return { ok: false, action: 'invalid_vm' } const kind = runtimeKind(vm) if (kind !== 'docker' && kind !== 'kvm') return { ok: false, action: 'unsupported_runtime' } @@ -119,10 +126,19 @@ export async function restartTelemetrySidecar({ projectRoot, vm, enabled, run = const env = host.dockerEnv() const container = slotContainerName({ vm }) const target = { ...vm, runtime: { ...(vm.runtime || {}), container, type: kind } } - if (!inspectContainer(container)) { + const inspected = inspectOutcome(container) + if (inspected.status === 'missing') { logTelemetryContainerSkip('fleet', vm.id, container) return telemetryContainerSkip(container) } + if (inspected.status === 'error') { + return { + ok: false, + action: 'inspect_failed', + code: 'docker_inspect_failed', + error: inspected.error, + } + } const stop = 'pkill -TERM -x kin-worker 2>/dev/null || true' try { await run('docker', slotExecArgv(target, ['sh', '-c', stop]), { diff --git a/src/lib/vm/vm-runtime.mjs b/src/lib/vm/vm-runtime.mjs index 7f430b6d1..0f4b478ae 100644 --- a/src/lib/vm/vm-runtime.mjs +++ b/src/lib/vm/vm-runtime.mjs @@ -37,6 +37,7 @@ import { import { slotMemory, slotDockerCpus, memoryMiB, VM_CONFIG_DEFAULTS } from './machine-spec.mjs' import { probeLocalKvm } from './kvm-host.mjs' import { slotExecArgv } from './slot-exec.mjs' +import { isDockerInspectNotFound } from './docker-container.mjs' export const RUNTIME = 'docker' const WORKER_BIN = process.env.KIN_WORKER_BIN || '/opt/kin-gateway/bin/kin-worker' @@ -148,16 +149,8 @@ export function displayName(vmId) { return String(vmId || '').replace(/^vm-/, '') } -export function inspectContainer(name) { - const r = sh([ - 'docker', - 'inspect', - '--format', - '{{.State.Running}}|{{.State.Pid}}|{{.HostConfig.NetworkMode}}|{{.State.StartedAt}}|{{.Config.Image}}|{{.Config.Hostname}}|{{index .Config.Labels "kin.vm.runtime"}}|{{index .Config.Labels "kin.vm.role"}}|{{.State.Restarting}}', - name, - ]) - if (!r.ok) return null - const [running, pid, networkMode, startedAt, image, hostname, runtime, role, restarting] = r.stdout.split('|') +function parseInspectStdout(name, stdout) { + const [running, pid, networkMode, startedAt, image, hostname, runtime, role, restarting] = stdout.split('|') // Docker reports Running=true during a restart-policy backoff; nothing can be exec'd there. const crashLooping = restarting === 'true' return { @@ -175,6 +168,28 @@ export function inspectContainer(name) { } } +const INSPECT_FORMAT = + '{{.State.Running}}|{{.State.Pid}}|{{.HostConfig.NetworkMode}}|{{.State.StartedAt}}|{{.Config.Image}}|{{.Config.Hostname}}|{{index .Config.Labels "kin.vm.runtime"}}|{{index .Config.Labels "kin.vm.role"}}|{{.State.Restarting}}' + +/** + * @returns {{ status: 'found', container: object } | { status: 'missing' } | { status: 'error', error: string, code?: number }} + */ +export function inspectContainerOutcome(name, { run = sh } = {}) { + const r = run(['docker', 'inspect', '--format', INSPECT_FORMAT, name]) + if (r.ok) return { status: 'found', container: parseInspectStdout(name, r.stdout) } + if (isDockerInspectNotFound(r)) return { status: 'missing' } + return { + status: 'error', + error: (r.stderr || r.stdout || 'docker inspect failed').trim(), + code: r.code, + } +} + +export function inspectContainer(name, opts) { + const outcome = inspectContainerOutcome(name, opts) + return outcome.status === 'found' ? outcome.container : null +} + /** Label on GPT slot containers: their PID 1 idles; the Codex kernel runs on the host. */ export const CODEX_SLOT_ROLE = 'codex' diff --git a/test/unit/docker-inspect-outcome.test.mjs b/test/unit/docker-inspect-outcome.test.mjs new file mode 100644 index 000000000..37d3e8d1e --- /dev/null +++ b/test/unit/docker-inspect-outcome.test.mjs @@ -0,0 +1,48 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { inspectContainerOutcome } from '../../src/lib/vm/vm-runtime.mjs' +import { isDockerInspectNotFound } from '../../src/lib/vm/docker-container.mjs' + +test('isDockerInspectNotFound matches docker inspect stderr for absent names', () => { + assert.equal(isDockerInspectNotFound({ stderr: 'Error: No such object: kin-05' }), true) + assert.equal(isDockerInspectNotFound({ stderr: 'Error response from daemon: No such container: kin-05' }), true) + assert.equal(isDockerInspectNotFound({ stderr: 'Cannot connect to the Docker daemon' }), false) +}) + +test('inspectContainerOutcome: missing container', () => { + const outcome = inspectContainerOutcome('kin-ghost', { + run: () => ({ + ok: false, + stderr: 'Error: No such object: kin-ghost', + stdout: '', + code: 1, + }), + }) + assert.deepEqual(outcome, { status: 'missing' }) +}) + +test('inspectContainerOutcome: daemon down is an error, not missing', () => { + const outcome = inspectContainerOutcome('kin-01', { + run: () => ({ + ok: false, + stderr: 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock', + stdout: '', + code: 1, + }), + }) + assert.equal(outcome.status, 'error') + assert.match(outcome.error, /Cannot connect/) +}) + +test('inspectContainerOutcome: found parses running state', () => { + const outcome = inspectContainerOutcome('kin-01', { + run: () => ({ + ok: true, + stdout: 'true|12345|kin-px-net|2026-01-01T00:00:00Z|img|host|docker||false', + stderr: '', + }), + }) + assert.equal(outcome.status, 'found') + assert.equal(outcome.container.running, true) + assert.equal(outcome.container.pid, 12345) +}) diff --git a/test/unit/fleet-telemetry-inspect.test.mjs b/test/unit/fleet-telemetry-inspect.test.mjs new file mode 100644 index 000000000..db41bd1d3 --- /dev/null +++ b/test/unit/fleet-telemetry-inspect.test.mjs @@ -0,0 +1,89 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { runFleetUpdate } from '../../src/lib/vm/fleet-update.mjs' +import { restartTelemetrySidecar } from '../../src/lib/vm/slot-process-status.mjs' + +function writeSlot(root, id) { + const dir = path.join(root, 'vms') + fs.mkdirSync(dir, { recursive: true }) + fs.writeFileSync( + path.join(dir, `${id}.json`), + JSON.stringify({ + id, + name: id.replace('vm-', ''), + status: 'running', + runtime: { type: 'docker', worker: 'go' }, + }), + ) +} + +test('restartTelemetrySidecar skips only when inspect confirms the container is missing', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-tel-miss-')) + const skipped = await restartTelemetrySidecar({ + projectRoot: root, + vm: { id: 'vm-11', runtime: { type: 'docker' } }, + enabled: true, + inspectContainerOutcome: () => ({ status: 'missing' }), + run: async () => { + throw new Error('docker exec must not run when the container is absent') + }, + }) + assert.equal(skipped.skipped, true) + fs.rmSync(root, { recursive: true, force: true }) +}) + +test('restartTelemetrySidecar fails when inspect cannot reach docker', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-tel-daemon-')) + const failed = await restartTelemetrySidecar({ + projectRoot: root, + vm: { id: 'vm-12', runtime: { type: 'docker' } }, + enabled: true, + inspectContainerOutcome: () => ({ + status: 'error', + error: 'Cannot connect to the Docker daemon', + }), + }) + assert.equal(failed.ok, false) + assert.equal(failed.code, 'docker_inspect_failed') + assert.equal(failed.skipped, undefined) + fs.rmSync(root, { recursive: true, force: true }) +}) + +test('restartTelemetrySidecar fails when the container exists but exec fails', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-tel-exec-')) + const failed = await restartTelemetrySidecar({ + projectRoot: root, + vm: { id: 'vm-13', runtime: { type: 'docker' } }, + enabled: true, + inspectContainerOutcome: () => ({ + status: 'found', + container: { name: 'kin-13', running: true }, + }), + run: async () => { + throw new Error('permission denied') + }, + }) + assert.equal(failed.ok, false) + assert.equal(failed.action, 'restart_failed') + assert.match(failed.error, /permission denied/) + fs.rmSync(root, { recursive: true, force: true }) +}) + +test('fleet telemetry fails the batch when local inspect errors', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-fleet-inspect-')) + writeSlot(root, 'vm-14') + const report = await runFleetUpdate(root, { + action: 'telemetry', + telemetryFn: async () => ({ + ok: false, + code: 'docker_inspect_failed', + error: 'Cannot connect to the Docker daemon', + }), + }) + assert.equal(report.failed_count, 1) + assert.equal(report.skipped_count, 0) + fs.rmSync(root, { recursive: true, force: true }) +})