fix(sandbox): wait for guest agent, verify exec user provisioning, pin rebuilt image

Dockerfile.sandbox

@@ -73,15 +73,18 @@ RUN echo "Installing playwright=${PLAYWRIGHT_VERSION} chromium (cachebust=${TOOL
 # /usr/local/lib/node_modules for the NODE_PATH fallback above to work.
 ENV npm_config_prefix=/opt/agent-tools
 
-# The exec user is an unknown numeric uid, so verify as uid 1000 (also absent
-# from /etc/passwd) instead of root. A root-only check would not catch the
-# EACCES failures this image exists to prevent.
-RUN echo "Verifying guest toolchain as an unprivileged uid (cachebust=${TOOLS_CACHEBUST})" && \
+# The runtime exec user is a numeric host uid with no /etc/passwd entry, so the
+# toolchain is verified as uid 4242, which is absent from this image's passwd
+# database. uid 1000 would not do: it is the base image's `node` user, so it
+# would exercise the known-user path and pass even if the real one were broken.
+# sudo is checked separately as that known user, because sudo refuses unknown
+# uids by design and the Manager provisions their passwd entry at runtime.
+RUN echo "Verifying guest toolchain as an unknown uid (cachebust=${TOOLS_CACHEBUST})" && \
     rm -rf /home/ocm-agent && \
     mkdir -p /home/ocm-agent && \
     chmod 1777 /home/ocm-agent && \
-    setpriv --reuid=1000 --regid=1000 --clear-groups sh -c "set -e; pnpm --version; bun --version; bunx --version; uv --version; uvx --version; playwright --version; gh --version; jq --version; rg --version; npm install -g --silent cowsay && cowsay -t verify-ok | head -1; sudo -n apt-get update -qq && sudo -n apt-get install -y -qq --no-install-recommends bc && bc --version | head -1; uv venv /tmp/uv-check >/dev/null; uv pip install --python /tmp/uv-check/bin/python idna >/dev/null; pip3 install --user --force-reinstall --no-deps --quiet idna; rm -rf /tmp/uv-check" && \
-    npm uninstall -g --silent cowsay && \
+    setpriv --reuid=4242 --regid=4242 --clear-groups sh -c "set -e; pnpm --version; bun --version; bunx --version; uv --version; uvx --version; playwright --version; gh --version; jq --version; rg --version; npm install -g --silent cowsay && cowsay -t verify-ok | head -1; git config --global user.email verify@ocm.local; uv venv /tmp/uv-check >/dev/null; uv pip install --python /tmp/uv-check/bin/python idna >/dev/null; pip3 install --user --force-reinstall --no-deps --quiet idna; rm -rf /tmp/uv-check" && \
+    setpriv --reuid=1000 --regid=1000 --clear-groups sh -c "set -e; sudo -n apt-get update -qq; sudo -n apt-get install -y -qq --no-install-recommends bc; bc --version | head -1" && \
     rm -rf /home/ocm-agent /opt/agent-tools /var/lib/apt/lists/* && \
     mkdir -p /home/ocm-agent /opt/agent-tools/bin && \
     chmod 1777 /home/ocm-agent /opt/agent-tools /opt/agent-tools/bin

backend/src/services/sandbox/command.ts

@@ -124,6 +124,27 @@ export function buildSandboxPullArgs(): string[] {
   return ['pull', ENV.SANDBOX.IMAGE]
 }
 
+export function buildSandboxPingArgs(): string[] {
+  return ['ping', WORKSPACE_SANDBOX_NAME, '-q']
+}
+
+export function buildSandboxVerifyProvisionArgs(): string[] {
+  const uid = resolveSandboxExecUserUid()
+  if (uid === null) return []
+  return [
+    'exec',
+    WORKSPACE_SANDBOX_NAME,
+    '--no-tty',
+    '-q',
+    '-u',
+    '0:0',
+    '--',
+    '/bin/sh',
+    '-c',
+    `getent passwd ${uid} >/dev/null`,
+  ]
+}
+
 export function sandboxMountRoots(): string[] {
   return [getReposPath(), getScheduleWorktreesPath()]
 }
@@ -155,6 +176,8 @@ export function buildSandboxCreateArgs(): string[] {
     getReposPath(),
     '--entrypoint',
     '/usr/bin/env',
+    '--shell',
+    '/bin/sh',
     ENV.SANDBOX.IMAGE,
     '--',
     'sleep',
@@ -314,6 +337,7 @@ function parseSandboxCreateArgs(args: string[]): {
   mountDirs: string[]
   workdir: string
   entrypoint: string[]
+  shell: string
   image: string
   cmd: string[]
 } {
@@ -325,6 +349,7 @@ function parseSandboxCreateArgs(args: string[]): {
   let user = ''
   let workdir = ''
   let entrypoint: string[] = []
+  let shell = ''
   let image = ''
   let cmd: string[] = []
   for (let i = 1; i < args.length; i++) {
@@ -351,12 +376,13 @@ function parseSandboxCreateArgs(args: string[]): {
       case '--mount-dir': if (value !== undefined) mountDirs.push(value); i += 1; break
       case '-w': workdir = value ?? ''; i += 1; break
       case '--entrypoint': if (value !== undefined) entrypoint = [value]; i += 1; break
+      case '--shell': if (value !== undefined) shell = value; i += 1; break
       case '-d': break
       default:
         if (image === '' && !token.startsWith('-')) image = token
     }
   }
-  return { name, labels, memory, cpus, user, mountDirs, workdir, entrypoint, image, cmd }
+  return { name, labels, memory, cpus, user, mountDirs, workdir, entrypoint, shell, image, cmd }
 }
 
 export function buildCanonicalSandboxSpec(): Record<string, unknown> {
@@ -392,7 +418,7 @@ export function buildCanonicalSandboxSpec(): Record<string, unknown> {
     },
     runtime: {
       workdir: args.workdir,
-      shell: null,
+      shell: args.shell,
       scripts: {},
       entrypoint: args.entrypoint,
       cmd: args.cmd,
@@ -480,8 +506,11 @@ export function buildSandboxProvisionArgs(): string[] {
     '/bin/sh',
     '-c',
     `getent group ${gid} >/dev/null 2>&1 || echo 'ocm-exec:x:${gid}:' >> /etc/group; ` +
-      `getent passwd ${uid} >/dev/null 2>&1 || echo 'ocm-exec:x:${uid}:${gid}:Manager sandbox exec user:/home/ocm-agent:/bin/sh' >> /etc/passwd; ` +
-      `grep -q '^ocm-exec:' /etc/shadow || echo 'ocm-exec:*:19000:0:99999:7:::' >> /etc/shadow`,
+      `getent passwd ${uid} >/dev/null 2>&1 || { ` +
+      `echo 'ocm-exec:x:${uid}:${gid}:Manager sandbox exec user:/home/ocm-agent:/bin/sh' >> /etc/passwd; ` +
+      `grep -q '^ocm-exec:' /etc/shadow || echo 'ocm-exec:*:19000:0:99999:7:::' >> /etc/shadow; ` +
+      `}; ` +
+      `getent passwd ${uid} >/dev/null || exit 1`,
   ]
 }
 

backend/src/services/sandbox/runtime.ts

@@ -15,11 +15,13 @@ import {
   buildSandboxCreateArgs,
   buildSandboxInspectArgs,
   buildSandboxListArgs,
+  buildSandboxPingArgs,
   buildSandboxProvisionArgs,
   buildSandboxPullArgs,
   buildSandboxRemoveArgs,
   buildSandboxStartArgs,
   buildSandboxStopManagedArgs,
+  buildSandboxVerifyProvisionArgs,
   resolveExpectedSandboxNetworkPolicy,
   resolveSandboxRuntimeTmpfsSizeMib,
   resolveSandboxWorkDirectory,
@@ -33,6 +35,9 @@ const SANDBOX_LS_TIMEOUT_MS = 15000
 const SANDBOX_STOP_TIMEOUT_MS = 30000
 const SANDBOX_PROVISION_ATTEMPTS = 3
 const SANDBOX_PROVISION_RETRY_DELAY_MS = 2000
+const SANDBOX_PROVISION_RETRY_ATTEMPTS = 5
+const SANDBOX_AGENT_PING_TIMEOUT_MS = 10000
+const SANDBOX_AGENT_POLL_DELAY_MS = 1000
 const SANDBOX_RUNTIME_TMPFS_GUEST = path.resolve('/tmp')
 
 export type SandboxShellPlan =
@@ -59,6 +64,9 @@ export function resetSandboxRuntimeState(): void {
   shutdownRequested = false
   stopInProgress = false
   canonicalSandboxSpecMemo = null
+  provisionRetryGeneration += 1
+  backgroundProvisionRetry = null
+  inFlightProvision = null
 }
 
 function memoizedCanonicalSandboxSpec(): Record<string, unknown> {
@@ -133,7 +141,7 @@ async function bootWorkspaceSandbox(): Promise<void> {
           await createWorkspaceSandbox()
         } else {
           await startWorkspaceSandbox()
-          await provisionSandboxExecUserPasswd()
+          await provisionSandboxExecUser()
           const runningAttestation = await attestWorkspaceSandbox(true)
           if (!runningAttestation.trusted) {
             logger.warn(
@@ -166,7 +174,7 @@ async function bootWorkspaceSandboxFromListing(): Promise<void> {
       await createWorkspaceSandbox()
     } else if (!entry.running) {
       await startWorkspaceSandbox()
-      await provisionSandboxExecUserPasswd()
+      await provisionSandboxExecUser()
       const runningAttestation = await attestWorkspaceSandbox(true)
       if (!runningAttestation.trusted) {
         logger.warn(
@@ -551,35 +559,139 @@ async function cacheSandboxImage(): Promise<void> {
   logger.info(`Sandbox guest image ${ENV.SANDBOX.IMAGE} is cached`)
 }
 
+async function pingSandboxAgent(): Promise<boolean> {
+  try {
+    const result = (await executeCommand([sandboxExecutablePath(), ...buildSandboxPingArgs()], {
+      ignoreExitCode: true,
+      silent: true,
+      timeout: SANDBOX_AGENT_PING_TIMEOUT_MS,
+    })) as string | { exitCode: number; stdout: string; stderr: string }
+    return typeof result === 'string' || result.exitCode === 0
+  } catch {
+    return false
+  }
+}
+
+async function waitForSandboxAgent(): Promise<boolean> {
+  const deadline = Date.now() + ENV.SANDBOX.START_TIMEOUT_MS
+  for (;;) {
+    if (await pingSandboxAgent()) return true
+    if (Date.now() >= deadline) return false
+    await new Promise((resolve) => setTimeout(resolve, SANDBOX_AGENT_POLL_DELAY_MS))
+  }
+}
+
+async function provisionSandboxExecUser(): Promise<void> {
+  try {
+    await ensureSandboxExecUserProvisioned()
+    return
+  } catch (error) {
+    logger.warn(
+      `Sandbox exec user provisioning failed, so sudo will not work inside the guest yet; retrying in the background: ${error instanceof Error ? error.message : String(error)}`,
+    )
+  }
+  scheduleBackgroundProvisionRetry()
+}
+
 async function provisionSandboxExecUserPasswd(): Promise<void> {
   const provisionArgs = buildSandboxProvisionArgs()
   if (provisionArgs.length === 0) {
     return
   }
+  if (!(await waitForSandboxAgent())) {
+    throw new Error('the sandbox agent did not become reachable before the exec user could be provisioned')
+  }
   let lastError: unknown = null
+  const deadline = Date.now() + ENV.SANDBOX.START_TIMEOUT_MS
   for (let attempt = 0; attempt < SANDBOX_PROVISION_ATTEMPTS; attempt++) {
     if (attempt > 0) {
+      if (Date.now() >= deadline) break
       await new Promise((resolve) => setTimeout(resolve, SANDBOX_PROVISION_RETRY_DELAY_MS))
     }
     try {
       await executeCommand([sandboxExecutablePath(), ...provisionArgs], {
-        timeout: SANDBOX_LS_TIMEOUT_MS,
+        timeout: ENV.SANDBOX.START_TIMEOUT_MS,
       })
+      const verifyArgs = buildSandboxVerifyProvisionArgs()
+      if (verifyArgs.length > 0) {
+        await executeCommand([sandboxExecutablePath(), ...verifyArgs], {
+          timeout: SANDBOX_LS_TIMEOUT_MS,
+        })
+      }
       return
     } catch (error) {
       lastError = error
+      if (Date.now() >= deadline) break
     }
   }
   throw new Error(
     `provisioning the sandbox exec user passwd entry failed: ${lastError instanceof Error ? lastError.message : String(lastError)}`,
   )
 }
 
+let backgroundProvisionRetry: Promise<void> | null = null
+let provisionRetryGeneration = 0
+let inFlightProvision: Promise<void> | null = null
+
+function ensureSandboxExecUserProvisioned(): Promise<void> {
+  if (inFlightProvision) {
+    return inFlightProvision
+  }
+  inFlightProvision = provisionSandboxExecUserPasswd().finally(() => {
+    inFlightProvision = null
+  })
+  return inFlightProvision
+}
+
+export function backgroundProvisionRetryForTests(): Promise<void> | null {
+  return backgroundProvisionRetry
+}
+
+export function provisionSandboxExecUserForTests(): Promise<void> {
+  return ensureSandboxExecUserProvisioned()
+}
+
+function scheduleBackgroundProvisionRetry(): void {
+  const generation = provisionRetryGeneration
+  if (backgroundProvisionRetry !== null) return
+  logger.info('Sandbox exec user provisioning background retry scheduled')
+  backgroundProvisionRetry = (async () => {
+    for (let attempt = 1; attempt <= SANDBOX_PROVISION_RETRY_ATTEMPTS; attempt++) {
+      await new Promise((resolve) => setTimeout(resolve, SANDBOX_PROVISION_RETRY_DELAY_MS))
+      if (generation !== provisionRetryGeneration) {
+        logger.info('Sandbox exec user provisioning background retry cancelled by a runtime state reset')
+        return
+      }
+      if (shutdownRequested) {
+        logger.info('Sandbox exec user provisioning background retry cancelled by shutdown')
+        return
+      }
+      try {
+        await ensureSandboxExecUserProvisioned()
+        logger.info(`Sandbox exec user provisioned by the background retry (attempt ${attempt})`)
+        return
+      } catch (error) {
+        logger.warn(
+          `Sandbox exec user provisioning background retry attempt ${attempt} failed: ${error instanceof Error ? error.message : String(error)}`,
+        )
+        if (generation !== provisionRetryGeneration || shutdownRequested) return
+      }
+    }
+    logger.warn(
+      'Sandbox exec user provisioning retries were exhausted; sudo will not work inside the guest until the next boot cycle',
+    )
+  })().finally(() => {
+    if (generation === provisionRetryGeneration) {
+      backgroundProvisionRetry = null
+    }
+  })
+}
+
 async function createWorkspaceSandbox(): Promise<void> {
   await executeCommand([sandboxExecutablePath(), ...buildSandboxCreateArgs()], {
     timeout: ENV.SANDBOX.START_TIMEOUT_MS,
   })
-  await provisionSandboxExecUserPasswd()
+  await provisionSandboxExecUser()
   const attestation = await attestWorkspaceSandbox(true)
   if (!attestation.trusted) {
     throw new Error(`newly created sandbox ${WORKSPACE_SANDBOX_NAME} failed attestation: ${attestation.reason}`)
@@ -665,8 +777,15 @@ export class SandboxRuntimeService {
     if (getProcessIdentityAttestationError() !== null) return
     await cacheSandboxImage()
     await ensureWorkspaceSandbox()
-    await provisionSandboxExecUserPasswd()
-    logger.info('Workspace sandbox is running and its exec user is provisioned')
+    try {
+      await ensureSandboxExecUserProvisioned()
+      logger.info('Workspace sandbox is running and its exec user is provisioned')
+    } catch (error) {
+      logger.warn(
+        `Workspace sandbox is running but its exec user is not provisioned yet; retrying in the background: ${error instanceof Error ? error.message : String(error)}`,
+      )
+      scheduleBackgroundProvisionRetry()
+    }
   }
 
   isEnabled(): boolean {
@@ -727,6 +846,8 @@ export class SandboxRuntimeService {
 
   private async stopManagedSandbox(): Promise<void> {
     stopInProgress = true
+    provisionRetryGeneration += 1
+    backgroundProvisionRetry = null
     try {
       await this.runManagedSandboxStop()
     } finally {

backend/src/utils/process.ts

@@ -43,7 +43,8 @@ export async function executeCommand(
     const proc: ChildProcess = spawn(command || '', cmdArgs, {
       cwd: options.cwd,
       shell: false,
-      env: effectiveEnv
+      env: effectiveEnv,
+      stdio: ['ignore', 'pipe', 'pipe'],
     })
 
     let stdout = ''