From 2b9e65ca0168f90d5da1169391100f008f093c48 Mon Sep 17 00:00:00 2001 From: mintaka Date: Sun, 11 Oct 2026 15:59:59 -0400 Subject: [PATCH 1/2] fix(runtime): report unstartable exec programs as exit 127/126 on every backend (RIG-5121) guestd and the host backend now match podman. A missing program is exit 127 and a present program that cannot be invoked (no exec bit, not a regular file, ENOEXEC, execve EACCES/EPERM after the pre-check) is exit 126, with a reason on stderr. A missing or non-directory workdir, a missing script interpreter, and runtime faults stay Go errors. Adds one-shot and streaming contract-suite rows, rejects an empty guest exec_id, and amends the V2b and V8 records. Refs RIG-5121, RIG-2922 Co-authored-by: Matt Wilkinson --- .../microvm-v2b-guest-supervisor-exec.md | 5 + .../microvm-v8-acceptance-suite-benchmarks.md | 71 +-- .../guest_control.connect.go | 46 +- .../gen/compass/v1/guest_control.pb.go | 15 +- go/internal/guestd/supervisor.go | 212 ++++++-- go/internal/guestd/supervisor_test.go | 455 +++++++++++++++++- go/internal/runtime/contract_suite_test.go | 116 ++++- go/internal/runtime/host_backend.go | 209 ++++++-- go/internal/runtime/host_backend_test.go | 351 ++++++++++++++ go/internal/runtime/microvm/exec.go | 16 +- go/internal/runtime/microvm/exec_test.go | 17 + go/internal/runtime/microvm_lifecycle.go | 20 +- go/internal/runtime/podman.go | 20 +- proto/compass/v1/guest_control.proto | 34 +- 14 files changed, 1374 insertions(+), 213 deletions(-) diff --git a/docs/designs/infra/runtime/compass-elastic-session-runtime/microvm-v2b-guest-supervisor-exec.md b/docs/designs/infra/runtime/compass-elastic-session-runtime/microvm-v2b-guest-supervisor-exec.md index 851f51f6e..e0c18fbe9 100644 --- a/docs/designs/infra/runtime/compass-elastic-session-runtime/microvm-v2b-guest-supervisor-exec.md +++ b/docs/designs/infra/runtime/compass-elastic-session-runtime/microvm-v2b-guest-supervisor-exec.md @@ -96,6 +96,11 @@ error from `Exec` means the exec could not be *attempted* (spawn failure, refused uid, unprovisioned gate) — exactly the cases `PodmanCLI.Exec` folds into its non-nil error return (`podman.go:517-520`). +> Amended 2026-10-11 by Matt's ruling: a missing program returns +> `exit_code` 127; a present program that cannot be invoked (no exec bit, +> ENOEXEC, execve EACCES/EPERM) returns 126, not a Connect error. A missing or +> non-directory workdir and a missing script interpreter stay errors. + The streaming pair carries the whole `StreamingExec` lifecycle over one bidi stream: diff --git a/docs/designs/infra/runtime/compass-elastic-session-runtime/microvm-v8-acceptance-suite-benchmarks.md b/docs/designs/infra/runtime/compass-elastic-session-runtime/microvm-v8-acceptance-suite-benchmarks.md index 8d6b204ea..078723927 100644 --- a/docs/designs/infra/runtime/compass-elastic-session-runtime/microvm-v8-acceptance-suite-benchmarks.md +++ b/docs/designs/infra/runtime/compass-elastic-session-runtime/microvm-v8-acceptance-suite-benchmarks.md @@ -354,27 +354,11 @@ is a wall on first run: before `Start`** — it rides the share the guest already mounts. An executor reaching for a dedicated read-only mount for a test-only binary gets `UnsupportedMountError` at `Create`. -- **A missing or unstartable probe is an `Exec` ERROR, not exit 126/127.** - guestd does not surface an unresolvable program as an exit code: - `resolveProgram`'s error is wrapped into - `connect.NewError(connect.CodeInternal, …)` by `buildChild` - (`internal/guestd/supervisor.go:600-614`), and a path-form argv like - `/workspace/vsockprobe` bypasses PATH resolution entirely - (`resolveProgram`: a name containing a slash "is used as given", - `supervisor.go:643-646`) and then fails at `cmd.Start()` — also - `CodeInternal`. Host-side that is a returned error, not an - `ExecOutput.ExitCode`: `Exec` maps a transport/refusal failure to - `ExecOutput{}, err` (`microvm_lifecycle.go:486-496`, whose doc states "a - guest refusal or transport failure is an error" while "a non-zero exit is - a SUCCESSFUL call", `:470-475`). W2 therefore asserts **`err == nil` - first** — a transport/spawn error is a harness fault and must `t.Fatal`, - matching V6's `guestSh` posture ("A transport/refusal error is fatal; a - NON-ZERO EXIT IS NOT", PR #912 - `microvm_isolation_microvm_test.go:111-123`) — and only then - discriminates `0`/`2`/`3` on `ExecOutput.ExitCode`. The exit-code - contract accordingly drops its 126/127 arm: those codes are not - observable on this path, and a missing probe is caught by the `err == nil` - assertion instead. +- **Amended 2026-10-11 by Matt's ruling: a missing probe exits 127; a present + but non-executable probe exits 126.** Both are successful `Exec` results with + stderr diagnostics. A missing or non-directory workdir, a missing + interpreter, and transport failures remain errors. The harness checks `err == nil`, rejects 126/127, then + discriminates 0/2/3 on `ExecOutput.ExitCode`. The test asserts, in order: @@ -404,14 +388,13 @@ Vacuity analysis: "dial fails" alone would pass if the vsock device were missing, the port wrong, or the probe binary broken — all caught by the positive control against CID 2, and further separated by the exit-code discrimination: `3` for absent transport or refused connect, `2` for -accepted-then-closed, `0` for connected-and-served. A missing or unstartable -probe is deliberately **not** in that table: it never reaches an exit code, -surfacing instead as an `Exec` error (guestd's `CodeInternal`, see above), -which W2's `err == nil` assertion catches as a harness fault before any -exit-code comparison runs. Proving mutation: use the test-only initrd built -with `bootModules ++ [ "vsock_loopback" ]` and selected through -`COMPASS_TEST_GUEST_INITRD`; the probe MUST flip from exit 3 to exit 2, making -the production-image exit-3 assertion go red. Production boots are unchanged. +accepted-then-closed, and `0` for connected-and-served. A missing probe exits +127 and a present but non-executable probe exits 126; `err == nil` is checked +first, then both setup codes are rejected before the transport comparison. +Use the test-only initrd built with `bootModules ++ [ "vsock_loopback" ]` and +selected through `COMPASS_TEST_GUEST_INITRD`; the probe MUST flip from exit 3 to +exit 2, making the production-image exit-3 assertion go red. Production boots +are unchanged. ### (e) Cycles 2-5 — contract and failure modes at the assembled layer @@ -1057,24 +1040,20 @@ leg. spawn failure is a harness fault and `t.Fatal`s, matching V6's `guestSh` posture ("A transport/refusal error is fatal; a NON-ZERO EXIT IS NOT", PR #912 `microvm_isolation_microvm_test.go:111-123`) — and only then is an - exit code compared. Then: CID 2:1025 dial to the test listener exits 0 (positive control); the CID-1 - dial MUST exit 3 on the production image because `vsock_loopback` is - absent from the initrd. Exit 3 proves transport-level unreachability, + exit code compared. The CID-2 control dials the test listener and exits 0; + the CID-1 dial MUST exit 3 on the production image because `vsock_loopback` + is absent from the initrd. Exit 3 proves transport-level unreachability, strictly stronger than the parent's "refused" wording. The CID-2 control plus exit-code discrimination rules out a missing device, wrong port, or broken probe: `3` means no transport or refused connect, `2` means - accepted-then-closed, and `0` means connected-and-served. **A missing or - unstaged probe is NOT in that table:** guestd converts an unresolvable or - unstartable program into `connect.NewError(connect.CodeInternal, …)` - (`internal/guestd/supervisor.go:600-614`; a slash-bearing argv bypasses - PATH entirely, `:643-646`), which `Exec` returns as - `ExecOutput{}, err` (`microvm_lifecycle.go:486-496`) — so 126/127 can - never be observed as an exit code on this path, and the `err == nil` - assertion above catches broken staging. Proving mutation: build a TEST-ONLY - initrd with `bootModules ++ [ "vsock_loopback" ]`, select it through - `COMPASS_TEST_GUEST_INITRD`, and rerun the probe. It MUST flip from exit 3 - to exit 2, making the production-image assertion go red; production boots - remain unchanged. + accepted-then-closed, and `0` means connected-and-served. A missing probe + exits 127 and a present but non-executable probe exits 126. Check `err == nil` + first, then reject both setup codes before the `0`/`2`/`3` transport comparison. + Proving mutation: build a TEST-ONLY initrd with + `bootModules ++ [ "vsock_loopback" ]`, select it through + `COMPASS_TEST_GUEST_INITRD`, and rerun the probe. It MUST flip from exit 3 to + exit 2, making the production-image assertion go red; production boots remain + unchanged. - **Depends:** none beyond main (the guestd gate is on main, `go/internal/guestd/vsock.go:25-29`); independent of W1 and of V6/V7 — one of the Plan preamble's **two** full exemptions (with W5), and W2's @@ -1657,8 +1636,8 @@ Extends the existing `microvm` job (`ci.yml:624-850`) per § Approach (h). `./...`, so an untagged compile guard catches rot in the default lane), staged by writing into the existing `/workspace` host dir before `Start` (never a second mount, `microvm_lifecycle.go:313-330`); - `Exec` `err == nil` asserted BEFORE any exit code (a missing probe is - a `CodeInternal` error, never 126/127); CID 2:1025 positive control against a test listener; CID-1 + `Exec` `err == nil` asserted BEFORE exit-code validation; missing probes + yield 127 and present but non-executable probes 126; reject both before CID 2:1025 exits 3 on the production initrd; the test-only initrd mutation adds `vsock_loopback` and flips the probe to exit 2; independent of V6/V7 — may land first diff --git a/go/internal/gen/compass/v1/compassv1internalconnect/guest_control.connect.go b/go/internal/gen/compass/v1/compassv1internalconnect/guest_control.connect.go index be6264f42..446aecda5 100644 --- a/go/internal/gen/compass/v1/compassv1internalconnect/guest_control.connect.go +++ b/go/internal/gen/compass/v1/compassv1internalconnect/guest_control.connect.go @@ -77,22 +77,17 @@ const ( // GuestControlClient is a client for the compass.v1.GuestControl service. type GuestControlClient interface { Health(context.Context, *connect.Request[v1.HealthRequest]) (*connect.Response[v1.HealthResponse], error) - // Exec runs one command to completion and returns its captured output. A - // non-zero exit is a SUCCESSFUL call whose ExecResponse.exit_code is - // non-zero — never a Connect error. A Connect error means the exec could not - // be attempted or could not complete: a spawn failure, a refused uid (0), a - // timeout, or an unprovisioned gate. + // Exec captures one command. A missing program returns 127; a present one + // that cannot be invoked (no exec bit, ENOEXEC, execve EACCES/EPERM) returns + // 126. A missing or non-directory workdir or missing interpreter is an error. Exec(context.Context, *connect.Request[v1.ExecRequest]) (*connect.Response[v1.ExecResponse], error) - // ExecStream runs a long-lived command over one bidi stream: the request - // stream carries the start frame, then stdin bytes and an optional - // half-close; the response stream carries the exec_id, then interleaved - // stdout/stderr, then exactly one terminal exit frame. Kill rides Signal - // (below), not this stream, so a teardown never contends with an in-flight - // stdin write. + // ExecStream: one start frame, stdin, optional half-close. Start failures + // follow Exec's 127/126 rule as ExecStarted, stderr, ExecExit; a missing or + // non-directory workdir or missing interpreter errors before ExecStarted. ExecStream(context.Context) *connect.BidiStreamForClient[v1.ExecStreamRequest, v1.ExecStreamResponse] - // Signal delivers a signal to a running exec by exec_id; an empty exec_id - // targets the guest itself (graceful Stop, record §(d)). Signalling an - // already-exited exec is a no-op success. + // Signal delivers a signal to a running exec by exec_id (exec kill rides + // here, not ExecStream); an empty exec_id targets the guest itself (graceful + // Stop, record §(d)). Signalling an already-exited exec is a no-op success. Signal(context.Context, *connect.Request[v1.SignalRequest]) (*connect.Response[v1.SignalResponse], error) // Provision transitions the supervisor ready -> provisioned: it records the // session's default exec uid and base env and, when nft_script is non-empty, @@ -184,22 +179,17 @@ func (c *guestControlClient) Provision(ctx context.Context, req *connect.Request // GuestControlHandler is an implementation of the compass.v1.GuestControl service. type GuestControlHandler interface { Health(context.Context, *connect.Request[v1.HealthRequest]) (*connect.Response[v1.HealthResponse], error) - // Exec runs one command to completion and returns its captured output. A - // non-zero exit is a SUCCESSFUL call whose ExecResponse.exit_code is - // non-zero — never a Connect error. A Connect error means the exec could not - // be attempted or could not complete: a spawn failure, a refused uid (0), a - // timeout, or an unprovisioned gate. + // Exec captures one command. A missing program returns 127; a present one + // that cannot be invoked (no exec bit, ENOEXEC, execve EACCES/EPERM) returns + // 126. A missing or non-directory workdir or missing interpreter is an error. Exec(context.Context, *connect.Request[v1.ExecRequest]) (*connect.Response[v1.ExecResponse], error) - // ExecStream runs a long-lived command over one bidi stream: the request - // stream carries the start frame, then stdin bytes and an optional - // half-close; the response stream carries the exec_id, then interleaved - // stdout/stderr, then exactly one terminal exit frame. Kill rides Signal - // (below), not this stream, so a teardown never contends with an in-flight - // stdin write. + // ExecStream: one start frame, stdin, optional half-close. Start failures + // follow Exec's 127/126 rule as ExecStarted, stderr, ExecExit; a missing or + // non-directory workdir or missing interpreter errors before ExecStarted. ExecStream(context.Context, *connect.BidiStream[v1.ExecStreamRequest, v1.ExecStreamResponse]) error - // Signal delivers a signal to a running exec by exec_id; an empty exec_id - // targets the guest itself (graceful Stop, record §(d)). Signalling an - // already-exited exec is a no-op success. + // Signal delivers a signal to a running exec by exec_id (exec kill rides + // here, not ExecStream); an empty exec_id targets the guest itself (graceful + // Stop, record §(d)). Signalling an already-exited exec is a no-op success. Signal(context.Context, *connect.Request[v1.SignalRequest]) (*connect.Response[v1.SignalResponse], error) // Provision transitions the supervisor ready -> provisioned: it records the // session's default exec uid and base env and, when nft_script is non-empty, diff --git a/go/internal/gen/compass/v1/guest_control.pb.go b/go/internal/gen/compass/v1/guest_control.pb.go index e078744e5..3fe44d9f6 100644 --- a/go/internal/gen/compass/v1/guest_control.pb.go +++ b/go/internal/gen/compass/v1/guest_control.pb.go @@ -157,11 +157,10 @@ func (x *HealthResponse) GetBootNonce() []byte { return nil } -// ExecRequest is a one-shot exec. It mirrors runtime.ExecSpec field for field. +// ExecRequest mirrors runtime.ExecSpec. See GuestControl.Exec for the 127/126 rule. type ExecRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - // command is argv; command[0] is the program. - Command []string `protobuf:"bytes,1,rep,name=command,proto3" json:"command,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Command []string `protobuf:"bytes,1,rep,name=command,proto3" json:"command,omitempty"` // uid is the exec's user. Absent = the session default uid set by Provision // (the baked agent uid). uid 0 is REFUSED with a failed-precondition error // before any spawn — the guest supervisor never runs an exec as root. @@ -256,8 +255,8 @@ type ExecResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Stdout []byte `protobuf:"bytes,1,opt,name=stdout,proto3" json:"stdout,omitempty"` Stderr []byte `protobuf:"bytes,2,opt,name=stderr,proto3" json:"stderr,omitempty"` - // exit_code is the command's exit status. Non-zero is a successful call with - // a failed command, never a Connect error (see the Exec doc-comment). + // Non-zero statuses, including 126/127, are successful calls. A missing or + // non-directory workdir, a missing interpreter, or a transport failure is an error. ExitCode int32 `protobuf:"varint,3,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -520,8 +519,8 @@ func (*StdinClose) Descriptor() ([]byte, []int) { return file_compass_v1_guest_control_proto_rawDescGZIP(), []int{6} } -// ExecStreamResponse frames flow guest -> host: exactly one ExecStarted first, -// then interleaved stdout/stderr, then exactly one ExecExit. +// Exactly one ExecStarted, then interleaved stdout/stderr, then one ExecExit. +// A start failure classified as 127/126 uses the same order. type ExecStreamResponse struct { state protoimpl.MessageState `protogen:"open.v1"` // Types that are valid to be assigned to Frame: diff --git a/go/internal/guestd/supervisor.go b/go/internal/guestd/supervisor.go index cfe8aeaf1..e0f041571 100644 --- a/go/internal/guestd/supervisor.go +++ b/go/internal/guestd/supervisor.go @@ -111,7 +111,7 @@ type childExec struct { // "signal on an exited exec is a no-op success" case — not actionable, so the // error is intentionally dropped. func (c *childExec) signalGroup(sig syscall.Signal) { - if c.cmd.Process == nil { + if c == nil || c.cmd == nil || c.cmd.Process == nil { return } // ESRCH (already-reaped group) is the documented no-op-success case; any @@ -280,12 +280,7 @@ func (s *supervisor) Provision( return connect.NewResponse(&compassv1internal.ProvisionResponse{}), nil } -// Exec runs one command to completion and returns its captured output (§(b)). -// A non-zero child exit is a SUCCESSFUL response with a non-zero ExitCode, never -// a handler error; a handler error means the exec could not be attempted or -// completed (gate closed, uid 0, spawn failure, timeout, or capture overflow). -// stdin bytes are fed to the child's stdin pipe, never argv, so a script's body -// never appears in the guest process list. +// Exec returns 127 for missing programs and 126 when a program cannot execute. func (s *supervisor) Exec( ctx context.Context, req *connect.Request[compassv1internal.ExecRequest], @@ -307,7 +302,17 @@ func (s *supervisor) Exec( defer cancel() } - cmd, err := s.buildChild(ctx, m.GetCommand(), uid, m.Workdir, m.GetEnv()) + command := m.GetCommand() + if len(command) == 0 { + return nil, connect.NewError(connect.CodeInvalidArgument, errors.New("exec command is empty: command[0] is the program")) + } + mergedEnv := s.mergeEnv(m.GetEnv()) + if code, stderr, ok := guestExecPrecheck(command, m.Workdir, mergedEnv); ok { + return connect.NewResponse(&compassv1internal.ExecResponse{ + Stderr: []byte(stderr), ExitCode: int32(code), //nolint:gosec // G115: code is the bounded 126/127 exec-start status + }), nil + } + cmd, err := s.buildChild(ctx, command, uid, m.Workdir, m.GetEnv()) if err != nil { return nil, err } @@ -319,6 +324,11 @@ func (s *supervisor) Exec( cmd.Stderr = stderr if err := cmd.Start(); err != nil { + if code, stderr, ok := guestExecStartFailure(command[0], m.Workdir, cmd.Path, err); ok { + return connect.NewResponse(&compassv1internal.ExecResponse{ + Stderr: []byte(stderr), ExitCode: int32(code), //nolint:gosec // G115: code is the bounded 126/127 exec-start status + }), nil + } return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("spawning exec: %w", err)) } @@ -355,12 +365,8 @@ func (s *supervisor) Exec( } } -// ExecStream runs a long-lived command over one bidi stream (§(b)): the first -// frame MUST be StartExec; the response stream is ExecStarted first, interleaved -// stdout/stderr, then exactly one terminal ExecExit emitted from guestd's own -// reap. stdin frames feed the child's stdin pipe; StdinClose half-closes it. -// The child is bound to the stream context: if the stream breaks (host -// disconnect / cancel), guestd SIGKILLs and reaps it so no orphan survives. +// ExecStream sends 126/127 start failures after ExecStarted. Workdir and +// interpreter faults are Connect errors before ExecStarted. func (s *supervisor) ExecStream( ctx context.Context, stream *connect.BidiStream[compassv1internal.ExecStreamRequest, compassv1internal.ExecStreamResponse], @@ -383,7 +389,16 @@ func (s *supervisor) ExecStream( if err != nil { return err } - cmd, err := s.buildChild(ctx, start.GetCommand(), uid, start.Workdir, start.GetEnv()) + sendMu := &sync.Mutex{} + command := start.GetCommand() + if len(command) == 0 { + return connect.NewError(connect.CodeInvalidArgument, errors.New("exec command is empty: command[0] is the program")) + } + mergedEnv := s.mergeEnv(start.GetEnv()) + if code, stderr, ok := guestExecPrecheck(command, start.Workdir, mergedEnv); ok { + return s.sendStartFailure(stream, sendMu, nil, code, stderr) + } + cmd, err := s.buildChild(ctx, command, uid, start.Workdir, start.GetEnv()) if err != nil { return err } @@ -392,17 +407,16 @@ func (s *supervisor) ExecStream( if err != nil { return connect.NewError(connect.CodeInternal, fmt.Errorf("opening stdin pipe: %w", err)) } - var sendMu sync.Mutex - cmd.Stdout = &streamWriter{stream: stream, mu: &sendMu, stdout: true} - cmd.Stderr = &streamWriter{stream: stream, mu: &sendMu, stdout: false} + cmd.Stdout = &streamWriter{stream: stream, mu: sendMu, stdout: true} + cmd.Stderr = &streamWriter{stream: stream, mu: sendMu, stdout: false} - // Acquire the shared send mutex BEFORE cmd.Start so the child's stdout/stderr - // copier goroutines cannot emit an output frame ahead of the mandatory first - // ExecStarted frame (§(b)). A fast child (e.g. `echo`) can otherwise win the - // mutex first; streamWriter.Write blocks on it until Started has gone out. + // The send lock keeps a fast child's output behind its mandatory ExecStarted frame. sendMu.Lock() if err := cmd.Start(); err != nil { sendMu.Unlock() + if code, stderr, ok := guestExecStartFailure(command[0], start.Workdir, cmd.Path, err); ok { + return s.sendStartFailure(stream, sendMu, cmd, code, stderr) + } return connect.NewError(connect.CodeInternal, fmt.Errorf("spawning exec: %w", err)) } @@ -591,18 +605,17 @@ func (s *supervisor) buildChild(ctx context.Context, argv []string, uid uint32, return nil, connect.NewError(connect.CodeInvalidArgument, errors.New("exec command is empty: command[0] is the program")) } + // The exec belongs to the provisioned guest session and is stopped explicitly, + // so it must survive the RPC context that starts it. + cmd := exec.CommandContext(context.WithoutCancel(ctx), argv[0], argv[1:]...) //nolint:gosec // argv is the caller's command by design — this IS the exec surface merged := s.mergeEnv(env) - prog, resolveErr := resolveProgram(argv[0], merged) + prog, resolveErr := resolveProgram(argv[0], merged, workdir) if resolveErr != nil { - return nil, connect.NewError(connect.CodeInternal, + return cmd, connect.NewError(connect.CodeInternal, fmt.Errorf("resolving exec program %q: %w", argv[0], resolveErr)) } - // exec.Command resolves a bare argv[0] against guestd's OWN process $PATH, - // but guestd is PID 1 with no PATH; override Path with the session-env - // resolution above and clear the (now-irrelevant) process-PATH lookup error. - // The exec belongs to the provisioned guest session and is stopped explicitly, - // so it must survive the RPC context that starts it. - cmd := exec.CommandContext(context.WithoutCancel(ctx), argv[0], argv[1:]...) //nolint:gosec // argv is the caller's command by design — this IS the exec surface + // guestd is PID 1 with no PATH, so use the session-env resolution above and + // clear exec.Command's process-PATH lookup error. cmd.Path = prog cmd.Err = nil if workdir != nil { @@ -624,14 +637,8 @@ func (s *supervisor) buildChild(ctx context.Context, argv []string, uid uint32, // without every caller having to spell out a PATH. const defaultGuestPATH = "/bin:/usr/bin:/sbin:/usr/sbin" -// resolveProgram finds the executable for a bare argv[0] using the search PATH -// carried in the merged session env, NOT guestd's own process env. guestd runs -// as PID 1 with no PATH, so Go's exec.LookPath (which reads os.Environ) can -// never resolve a bare command — resolution must honor the session env, the -// same PATH the child runs with (§(b), the container-image-PATH analog). A name -// containing a slash is used as given. Returns the resolved path, or an error -// if no executable is found on the session PATH. -func resolveProgram(file string, env []string) (string, error) { +// resolveProgram selects a regular executable from PATH, matching shell lookup. +func resolveProgram(file string, env []string, workdir *string) (string, error) { if strings.ContainsRune(file, '/') { return file, nil } @@ -642,21 +649,136 @@ func resolveProgram(file string, env []string) (string, error) { break } } + var notExecutable bool for _, dir := range filepath.SplitList(pathEnv) { if dir == "" { dir = "." } - if candidate := filepath.Join(dir, file); isExecutable(candidate) { - return candidate, nil + candidate := filepath.Join(dir, file) + if candidate == file { + candidate = "." + string(filepath.Separator) + candidate + } + lookupPath := pathAtGuestWorkdir(candidate, workdir) + info, err := os.Stat(lookupPath) + // Like a shell PATH search: unreadable entries and directories are skipped. + if err != nil || !info.Mode().IsRegular() { + continue + } + if info.Mode().Perm()&0o111 == 0 { + notExecutable = true + continue + } + return candidate, nil + } + if notExecutable { + return "", &os.PathError{Op: "exec", Path: file, Err: syscall.EACCES} + } + return "", &exec.Error{Name: file, Err: exec.ErrNotFound} +} + +func pathAtGuestWorkdir(path string, workdir *string) string { + if workdir != nil && !filepath.IsAbs(path) { + return filepath.Join(*workdir, path) + } + return path +} + +func guestExecPrecheck(argv []string, workdir *string, env []string) (int, string, bool) { + if len(argv) == 0 { + return 0, "", false + } + if workdir != nil { + if info, err := os.Stat(*workdir); err != nil || !info.IsDir() { + return 0, "", false + } + } + program := argv[0] + explicit := strings.ContainsRune(program, '/') + if explicit { + info, err := os.Stat(pathAtGuestWorkdir(program, workdir)) + if errors.Is(err, syscall.ENOENT) { + return guestMissingProgram(program, true) + } + if err != nil { + return 0, "", false + } + if !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 { + return guestNonExecutableProgram(program) + } + return 0, "", false + } + if _, err := resolveProgram(program, env, workdir); err != nil { + if errors.Is(err, exec.ErrNotFound) { + return guestMissingProgram(program, false) + } + if errors.Is(err, syscall.EACCES) || errors.Is(err, os.ErrPermission) { + return guestNonExecutableProgram(program) } } - return "", fmt.Errorf("executable file %q not found in $PATH", file) + return 0, "", false +} + +func guestMissingProgram(program string, explicit bool) (int, string, bool) { + if explicit { + return 127, fmt.Sprintf("%q: no such file or directory\n", program), true + } + return 127, fmt.Sprintf("executable file %q not found in $PATH\n", program), true } -// isExecutable reports whether path is a regular file with an execute bit set. -func isExecutable(path string) bool { - info, err := os.Stat(path) - return err == nil && info != nil && info.Mode().IsRegular() && info.Mode().Perm()&0o111 != 0 +func guestNonExecutableProgram(program string) (int, string, bool) { + return 126, fmt.Sprintf("%q: permission denied / not executable\n", program), true +} + +func guestExecStartFailure(program string, workdir *string, executablePath string, err error) (int, string, bool) { + path, ok := errors.AsType[*os.PathError](err) + if !ok || path.Op != "fork/exec" || path.Path != executablePath { + return 0, "", false + } + switch { + case errors.Is(path.Err, syscall.ENOENT): + if workdir != nil { + if _, err := os.Stat(*workdir); err != nil { + return 0, "", false + } + } + if _, err := os.Stat(pathAtGuestWorkdir(executablePath, workdir)); errors.Is(err, syscall.ENOENT) { + return guestMissingProgram(program, strings.ContainsRune(program, '/')) + } + return 0, "", false + case errors.Is(path.Err, syscall.ENOEXEC), errors.Is(path.Err, syscall.EACCES), errors.Is(path.Err, syscall.EPERM): + return guestNonExecutableProgram(program) + default: + return 0, "", false + } +} + +func (s *supervisor) sendStartFailure( + stream *connect.BidiStream[compassv1internal.ExecStreamRequest, compassv1internal.ExecStreamResponse], + sendMu *sync.Mutex, + cmd *exec.Cmd, + code int, + stderr string, +) error { + id := s.register(cmd) + defer s.unregister(id) + sendMu.Lock() + defer sendMu.Unlock() + if err := stream.Send(&compassv1internal.ExecStreamResponse{ + Frame: &compassv1internal.ExecStreamResponse_Started{Started: &compassv1internal.ExecStarted{ExecId: id}}, + }); err != nil { + return connect.NewError(connect.CodeInternal, fmt.Errorf("sending started frame: %w", err)) + } + if err := stream.Send(&compassv1internal.ExecStreamResponse{ + Frame: &compassv1internal.ExecStreamResponse_Stderr{Stderr: []byte(stderr)}, + }); err != nil { + return connect.NewError(connect.CodeInternal, fmt.Errorf("sending start failure stderr: %w", err)) + } + if err := stream.Send(&compassv1internal.ExecStreamResponse{ + Frame: &compassv1internal.ExecStreamResponse_Exit{Exit: &compassv1internal.ExecExit{ExitCode: int32(code)}}, //nolint:gosec // G115: code is the bounded 126/127 exec-start status + }); err != nil { + return connect.NewError(connect.CodeInternal, fmt.Errorf("sending start failure exit: %w", err)) + } + return nil } // initiateStop is the RPC-driven shutdown trigger (§(d)): it SIGTERMs every diff --git a/go/internal/guestd/supervisor_test.go b/go/internal/guestd/supervisor_test.go index ceb0ac05a..5bdcc553d 100644 --- a/go/internal/guestd/supervisor_test.go +++ b/go/internal/guestd/supervisor_test.go @@ -90,6 +90,30 @@ func TestExecRefusedBeforeProvision(t *testing.T) { } } +func TestExecEmptyCommandIsInvalidArgument(t *testing.T) { + client, _ := newTestSupervisor(t, true, 1000) + _, err := client.Exec(t.Context(), connect.NewRequest(&compassv1internal.ExecRequest{})) + if connect.CodeOf(err) != connect.CodeInvalidArgument { + t.Fatalf("Exec with empty command = %v, want InvalidArgument", err) + } +} + +func TestExecStreamEmptyCommandIsInvalidArgument(t *testing.T) { + client, _ := newTestSupervisor(t, true, 1000) + stream := client.ExecStream(t.Context()) + if err := stream.Send(&compassv1internal.ExecStreamRequest{ + Frame: &compassv1internal.ExecStreamRequest_Start{Start: &compassv1internal.StartExec{}}, + }); err != nil { + t.Fatalf("send start: %v", err) + } + if err := stream.CloseRequest(); err != nil { + t.Fatalf("close request: %v", err) + } + if _, err := stream.Receive(); connect.CodeOf(err) != connect.CodeInvalidArgument { + t.Fatalf("ExecStream with empty command = %v, want InvalidArgument", err) + } +} + func TestProvisionOpensGateAndRejectsRootAndNft(t *testing.T) { client, svc := newTestSupervisor(t, false, 0) @@ -178,11 +202,9 @@ func TestProvisionArmFailureLeavesGateClosed(t *testing.T) { if state != stateReady { t.Fatalf("state after failed arm = %d, want stateReady", state) } - // The exec gate is still closed. if gerr := svc.requireProvisioned(); gerr == nil { t.Fatal("requireProvisioned after failed arm = nil, want refused") } - // A retried Provision may run (state still stateReady). svc.armFunc = func(_ context.Context, _ string) error { return nil } if _, rerr := svc.Provision(t.Context(), connect.NewRequest(&compassv1internal.ProvisionRequest{ DefaultExecUid: 1000, @@ -364,6 +386,339 @@ func TestExecEnvMergedExecKeysWin(t *testing.T) { t.Fatalf("env merge = %q, want base,exec (exec key wins)", got) } } +func TestExecUnstartableProgramReturnsExitCode(t *testing.T) { + dir := t.TempDir() + nonExecutable := dir + "/noexec" + if err := os.WriteFile(nonExecutable, []byte("not executable"), 0o644); err != nil { + t.Fatal(err) + } + pathFile := dir + "/path-entry-file" + if err := os.WriteFile(pathFile, []byte("not a directory"), 0o644); err != nil { + t.Fatal(err) + } + executableButInvalid := dir + "/invalid-executable" + if err := os.WriteFile(executableButInvalid, []byte("not an executable format"), 0o755); err != nil { + t.Fatal(err) + } + tests := []struct { + name string + command string + path string + code int32 + }{ + {name: "missing from path", command: "compass-missing-exec-test", path: dir, code: 127}, + {name: "missing after unusable path entry", command: "compass-missing-exec-test", path: pathFile, code: 127}, + {name: "missing explicit path", command: dir + "/missing", code: 127}, + {name: "directory is not executable", command: "/", code: 126}, + {name: "non-executable from path", command: "noexec", path: dir, code: 126}, + {name: "non-executable explicit path", command: nonExecutable, code: 126}, + {name: "exec format failure", command: executableButInvalid, code: 126}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client, svc := newTestSupervisor(t, true, 1000) + if tt.path != "" { + svc.mu.Lock() + svc.baseEnv = map[string]string{"PATH": tt.path} + svc.mu.Unlock() + } + resp, err := client.Exec(t.Context(), connect.NewRequest(&compassv1internal.ExecRequest{ + Command: []string{tt.command}, + })) + if err != nil { + t.Fatalf("Exec(%q) = %v, want exit %d with no handler error", tt.command, err, tt.code) + } + if got := resp.Msg.GetExitCode(); got != tt.code { + t.Fatalf("Exec(%q) exit_code = %d, want %d (stderr %q)", tt.command, got, tt.code, resp.Msg.GetStderr()) + } + if got := resp.Msg.GetStderr(); len(got) == 0 { + t.Fatalf("Exec(%q) stderr = %q, want non-empty stderr", tt.command, got) + } + }) + } +} + +func TestExecInvalidWorkdirTakesPrecedenceOverMissingProgram(t *testing.T) { + for _, streaming := range []bool{false, true} { + t.Run(map[bool]string{false: "one-shot", true: "streaming"}[streaming], func(t *testing.T) { + client, _ := newTestSupervisor(t, true, 1000) + missing := t.TempDir() + "/missing-workdir" + program := missing + "/missing-program" + if streaming { + stream := client.ExecStream(t.Context()) + if err := stream.Send(&compassv1internal.ExecStreamRequest{Frame: &compassv1internal.ExecStreamRequest_Start{Start: &compassv1internal.StartExec{Command: []string{program}, Workdir: &missing}}}); err != nil { + t.Fatalf("send start: %v", err) + } + if err := stream.CloseRequest(); err != nil { + t.Fatalf("close request: %v", err) + } + if _, err := stream.Receive(); err == nil { + t.Fatal("ExecStream with invalid workdir and missing program returned a response") + } + return + } + _, err := client.Exec(t.Context(), connect.NewRequest(&compassv1internal.ExecRequest{Command: []string{program}, Workdir: &missing})) + if err == nil { + t.Fatal("Exec with invalid workdir and missing program returned nil error") + } + }) + } +} + +func TestExecNonDirectoryWorkdirRemainsError(t *testing.T) { + client, _ := newTestSupervisor(t, true, 1000) + file := t.TempDir() + "/not-a-directory" + if err := os.WriteFile(file, []byte("file"), 0o600); err != nil { + t.Fatal(err) + } + for _, program := range []string{"/bin/true", "compass-missing-workdir-probe"} { + _, err := client.Exec(t.Context(), connect.NewRequest(&compassv1internal.ExecRequest{Command: []string{program}, Workdir: &file})) + if err == nil { + t.Fatalf("Exec %q with non-directory workdir returned nil error", program) + } + } +} + +func TestExecStreamNonDirectoryWorkdirRemainsError(t *testing.T) { + client, _ := newTestSupervisor(t, true, 1000) + file := t.TempDir() + "/not-a-directory" + if err := os.WriteFile(file, []byte("file"), 0o600); err != nil { + t.Fatal(err) + } + for _, program := range []string{"/bin/true", "compass-missing-workdir-probe"} { + stream := client.ExecStream(t.Context()) + if err := stream.Send(&compassv1internal.ExecStreamRequest{Frame: &compassv1internal.ExecStreamRequest_Start{Start: &compassv1internal.StartExec{Command: []string{program}, Workdir: &file}}}); err != nil { + t.Fatalf("send start: %v", err) + } + if err := stream.CloseRequest(); err != nil { + t.Fatalf("close request: %v", err) + } + if _, err := stream.Receive(); err == nil { + t.Fatalf("ExecStream %q with non-directory workdir returned a response", program) + } + } +} + +func TestGuestExecStartFailureClassifiesPermissionErrors(t *testing.T) { + for _, errno := range []error{syscall.EACCES, syscall.EPERM, syscall.ENOEXEC} { + t.Run(errno.Error(), func(t *testing.T) { + path := &os.PathError{Op: "fork/exec", Path: "/probe", Err: errno} + code, stderr, ok := guestExecStartFailure("/probe", nil, "/probe", path) + if !ok || code != 126 || !strings.Contains(stderr, "/probe") { + t.Fatalf("guestExecStartFailure() = (%d, %q, %v), want 126, diagnostic, true", code, stderr, ok) + } + }) + } +} + +func TestExecBareInvalidFormatProgramReturns126(t *testing.T) { + dir := t.TempDir() + program := dir + "/invalid-format" + if err := os.WriteFile(program, []byte("not an executable format"), 0o755); err != nil { + t.Fatal(err) + } + client, svc := newTestSupervisor(t, true, 1000) + svc.mu.Lock() + svc.baseEnv = map[string]string{"PATH": dir} + svc.mu.Unlock() + resp, err := client.Exec(t.Context(), connect.NewRequest(&compassv1internal.ExecRequest{Command: []string{"invalid-format"}})) + if err != nil { + t.Fatalf("Exec bare invalid-format program: %v", err) + } + if resp.Msg.GetExitCode() != 126 || len(resp.Msg.GetStderr()) == 0 { + t.Fatalf("Exec result = %+v, want exit 126 with stderr", resp.Msg) + } +} + +func TestExecSkipsUnsearchablePATHEntry(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory search permission") + } + locked := t.TempDir() + "/locked" + if err := os.Mkdir(locked, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(locked, 0o700) }) + client, svc := newTestSupervisor(t, true, 1000) + svc.mu.Lock() + svc.baseEnv = map[string]string{"PATH": locked} + svc.mu.Unlock() + resp, err := client.Exec(t.Context(), connect.NewRequest(&compassv1internal.ExecRequest{Command: []string{"compass-unsearchable-probe"}})) + if err != nil || resp.Msg.GetExitCode() != 127 { + t.Fatalf("Exec with unsearchable PATH entry = (%v, %v), want exit 127", resp, err) + } +} + +func TestExecStreamBareInvalidFormatProgramReturns126(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(dir+"/invalid-format", []byte("not an executable format"), 0o755); err != nil { + t.Fatal(err) + } + client, svc := newTestSupervisor(t, true, 1000) + svc.mu.Lock() + svc.baseEnv = map[string]string{"PATH": dir} + svc.mu.Unlock() + stream := client.ExecStream(t.Context()) + if err := stream.Send(&compassv1internal.ExecStreamRequest{Frame: &compassv1internal.ExecStreamRequest_Start{Start: &compassv1internal.StartExec{Command: []string{"invalid-format"}}}}); err != nil { + t.Fatalf("send start: %v", err) + } + var frames []string + var code int32 = -1 + for { + msg, err := stream.Receive() + if err != nil { + t.Fatalf("receive after %v: %v", frames, err) + } + switch f := msg.GetFrame().(type) { + case *compassv1internal.ExecStreamResponse_Started: + frames = append(frames, "started") + case *compassv1internal.ExecStreamResponse_Stderr: + frames = append(frames, "stderr") + case *compassv1internal.ExecStreamResponse_Exit: + frames = append(frames, "exit") + code = f.Exit.GetExitCode() + } + if code >= 0 { + break + } + } + if strings.Join(frames, ",") != "started,stderr,exit" || code != 126 { + t.Fatalf("frames = %v exit = %d, want started,stderr,exit and 126", frames, code) + } +} + +func TestExecMissingWorkdirRemainsError(t *testing.T) { + client, _ := newTestSupervisor(t, true, 1000) + missing := t.TempDir() + "/missing" + _, err := client.Exec(t.Context(), connect.NewRequest(&compassv1internal.ExecRequest{ + Command: []string{"/bin/true"}, + Workdir: &missing, + })) + if err == nil { + t.Fatal("Exec with missing workdir returned nil error") + } +} + +func TestExecStreamMissingWorkdirRemainsError(t *testing.T) { + client, _ := newTestSupervisor(t, true, 1000) + missing := t.TempDir() + "/missing" + stream := client.ExecStream(t.Context()) + if err := stream.Send(&compassv1internal.ExecStreamRequest{ + Frame: &compassv1internal.ExecStreamRequest_Start{ + Start: &compassv1internal.StartExec{Command: []string{"/bin/true"}, Workdir: &missing}, + }, + }); err != nil { + t.Fatalf("send start: %v", err) + } + if err := stream.CloseRequest(); err != nil { + t.Fatalf("close request: %v", err) + } + if _, err := stream.Receive(); err == nil { + t.Fatal("ExecStream with missing workdir returned a response, want spawn error") + } +} + +func TestExecMissingInterpreterRemainsError(t *testing.T) { + client, _ := newTestSupervisor(t, true, 1000) + program := t.TempDir() + "/missing-interpreter" + if err := os.WriteFile(program, []byte("#!/compass-missing-interpreter\necho never\n"), 0o755); err != nil { + t.Fatal(err) + } + _, err := client.Exec(t.Context(), connect.NewRequest(&compassv1internal.ExecRequest{ + Command: []string{program}, + })) + if err == nil { + t.Fatal("Exec with missing script interpreter returned nil error") + } +} + +func TestExecStreamMissingInterpreterRemainsError(t *testing.T) { + client, _ := newTestSupervisor(t, true, 1000) + program := t.TempDir() + "/missing-interpreter" + if err := os.WriteFile(program, []byte("#!/compass-missing-interpreter\necho never\n"), 0o755); err != nil { + t.Fatal(err) + } + stream := client.ExecStream(t.Context()) + if err := stream.Send(&compassv1internal.ExecStreamRequest{ + Frame: &compassv1internal.ExecStreamRequest_Start{ + Start: &compassv1internal.StartExec{Command: []string{program}}, + }, + }); err != nil { + t.Fatalf("send start: %v", err) + } + if err := stream.CloseRequest(); err != nil { + t.Fatalf("close request: %v", err) + } + if _, err := stream.Receive(); err == nil { + t.Fatal("ExecStream with missing script interpreter returned a response") + } +} + +func TestExecStreamUnstartableProgramReturnsExit(t *testing.T) { + dir := t.TempDir() + nonExecutable := dir + "/noexec" + if err := os.WriteFile(nonExecutable, []byte("not executable"), 0o644); err != nil { + t.Fatal(err) + } + pathFile := dir + "/path-entry-file" + if err := os.WriteFile(pathFile, []byte("not a directory"), 0o644); err != nil { + t.Fatal(err) + } + executableButInvalid := dir + "/invalid-executable" + if err := os.WriteFile(executableButInvalid, []byte("not an executable format"), 0o755); err != nil { + t.Fatal(err) + } + tests := []struct { + name string + command string + path string + code int32 + }{ + {name: "missing", command: "/compass-missing-stream-test", code: 127}, + {name: "directory", command: "/", code: 126}, + {name: "non-executable from path", command: "noexec", path: dir, code: 126}, + {name: "non-executable explicit path", command: nonExecutable, code: 126}, + {name: "missing after unusable PATH entry", command: "compass-missing-stream-test", path: pathFile, code: 127}, + {name: "exec format failure", command: executableButInvalid, code: 126}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client, svc := newTestSupervisor(t, true, 1000) + if tt.path != "" { + svc.mu.Lock() + svc.baseEnv = map[string]string{"PATH": tt.path} + svc.mu.Unlock() + } + stream := client.ExecStream(t.Context()) + if err := stream.Send(&compassv1internal.ExecStreamRequest{ + Frame: &compassv1internal.ExecStreamRequest_Start{ + Start: &compassv1internal.StartExec{Command: []string{tt.command}}, + }, + }); err != nil { + t.Fatalf("send start: %v", err) + } + if err := stream.CloseRequest(); err != nil { + t.Fatalf("close request: %v", err) + } + + started, err := stream.Receive() + if err != nil || started.GetStarted() == nil { + t.Fatalf("first frame = %v, err = %v; want ExecStarted", started, err) + } + stderr, err := stream.Receive() + if err != nil || len(stderr.GetStderr()) == 0 { + t.Fatalf("stderr frame = %v, err = %v; want non-empty stderr", stderr, err) + } + exit, err := stream.Receive() + if err != nil || exit.GetExit() == nil || exit.GetExit().GetExitCode() != tt.code { + t.Fatalf("terminal frame = %v, err = %v; want exit %d", exit, err, tt.code) + } + if _, err := stream.Receive(); err == nil { + t.Fatal("stream continued after terminal ExecExit frame") + } + }) + } +} func TestCappedBufferFlagsOverflow(t *testing.T) { // The cap unit itself: writes up to limit are retained, and the first write @@ -585,6 +940,17 @@ func TestSignalKillsLiveChildAndExitCarriesSignal(t *testing.T) { } } +func TestSignalOnNilCommandExecIsNoOpSuccess(t *testing.T) { + client, svc := newTestSupervisor(t, true, 1000) + execID := svc.register(nil) + _, err := client.Signal(t.Context(), connect.NewRequest(&compassv1internal.SignalRequest{ + ExecId: execID, + Signal: int32(syscall.SIGTERM), + })) + if err != nil { + t.Fatalf("Signal on start-failed exec = %v, want no-op success", err) + } +} func TestSignalOnExitedExecIsNoOpSuccess(t *testing.T) { client, _ := newTestSupervisor(t, true, 1000) // Signal an exec_id that never existed (equivalent to already-reaped): the @@ -745,6 +1111,91 @@ func TestExecResolvesBareCommandViaSessionPATH(t *testing.T) { } } +func TestExecResolvesRelativePATHFromWorkdir(t *testing.T) { + workdir, err := os.MkdirTemp(".", "guest-relative-path-") //nolint:usetesting // The relative directory is required to test child workdir PATH lookup. + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(workdir) }) + bin := workdir + "/bin" + if err := os.Mkdir(bin, 0o755); err != nil { + t.Fatal(err) + } + probe := bin + "/probe" + if err := os.WriteFile(probe, []byte("#!/bin/sh\nprintf relative-path\n"), 0o755); err != nil { + t.Fatalf("writing probe: %v", err) + } + client, _ := newTestSupervisor(t, false, 0) + if _, err := client.Provision(t.Context(), connect.NewRequest(&compassv1internal.ProvisionRequest{ + DefaultExecUid: 1000, + BaseEnv: map[string]string{"PATH": "bin"}, + })); err != nil { + t.Fatalf("Provision: %v", err) + } + resp, err := client.Exec(t.Context(), connect.NewRequest(&compassv1internal.ExecRequest{ + Command: []string{"probe"}, + Workdir: &workdir, + })) + if err != nil { + t.Fatalf("Exec from relative PATH and workdir: %v", err) + } + if resp.Msg.GetExitCode() != 0 || string(resp.Msg.GetStdout()) != "relative-path" { + t.Fatalf("Exec result = %+v, want relative-path with exit 0", resp.Msg) + } +} + +func TestExecResolvesDotPATHFromWorkdir(t *testing.T) { + workdir, err := os.MkdirTemp(".", "guest-dot-path-") //nolint:usetesting // Relative workdir is needed to test a dot PATH entry. + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(workdir) }) + program := workdir + "/dot-path-probe" + if err := os.WriteFile(program, []byte("#!/bin/sh\nprintf dot-path\n"), 0o755); err != nil { + t.Fatal(err) + } + client, _ := newTestSupervisor(t, false, 0) + if _, err := client.Provision(t.Context(), connect.NewRequest(&compassv1internal.ProvisionRequest{DefaultExecUid: 1000, BaseEnv: map[string]string{"PATH": "."}})); err != nil { + t.Fatalf("Provision: %v", err) + } + resp, err := client.Exec(t.Context(), connect.NewRequest(&compassv1internal.ExecRequest{Command: []string{"dot-path-probe"}, Workdir: &workdir})) + if err != nil { + t.Fatalf("Exec with dot PATH: %v", err) + } + if resp.Msg.GetExitCode() != 0 || string(resp.Msg.GetStdout()) != "dot-path" { + t.Fatalf("Exec result = %+v, want dot-path with exit 0", resp.Msg) + } +} + +func TestExecForeignOwnedExecutableReturns126(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires root to run the child as another uid") + } + dir := t.TempDir() + if err := os.Chmod(dir, 0o755); err != nil { + t.Fatal(err) + } + program := dir + "/foreign-owned" + if err := os.WriteFile(program, []byte("not an executable format"), 0o700); err != nil { + t.Fatal(err) + } + client, svc := newTestSupervisor(t, false, 0) + svc.newCredential = func(uint32) *syscall.Credential { + return &syscall.Credential{Uid: 65534, Gid: 65534} + } + if _, err := client.Provision(t.Context(), connect.NewRequest(&compassv1internal.ProvisionRequest{DefaultExecUid: 1000})); err != nil { + t.Fatalf("Provision: %v", err) + } + uid := uint32(65534) + resp, err := client.Exec(t.Context(), connect.NewRequest(&compassv1internal.ExecRequest{Command: []string{program}, Uid: &uid})) + if err != nil { + t.Fatalf("Exec foreign-owned program: %v", err) + } + if resp.Msg.GetExitCode() != 126 || len(resp.Msg.GetStderr()) == 0 { + t.Fatalf("Exec result = %+v, want exit 126 with stderr", resp.Msg) + } +} + func TestMergeEnvFloorsDefaultPATH(t *testing.T) { // mergeEnv floors defaultGuestPATH only when neither the base nor the exec // env carries a PATH (the container-baked-PATH analog), and keeps a diff --git a/go/internal/runtime/contract_suite_test.go b/go/internal/runtime/contract_suite_test.go index 77a7dbff0..ac289da58 100644 --- a/go/internal/runtime/contract_suite_test.go +++ b/go/internal/runtime/contract_suite_test.go @@ -197,8 +197,10 @@ func runContractSuite(t *testing.T, newRuntime func(t *testing.T) WorkloadRuntim t.Run("exec_exit_codes", func(t *testing.T) { rowExecExitCodes(t, rt, caps, primary) }) t.Run("exec_stdin", func(t *testing.T) { rowExecStdin(t, rt, caps, primary) }) + t.Run("exec_unstartable_programs", func(t *testing.T) { rowExecUnstartablePrograms(t, rt, caps, primary) }) + t.Run("exec_stream_unstartable_programs", func(t *testing.T) { rowExecStreamingUnstartablePrograms(t, rt, caps, primary) }) t.Run("streaming_stdio", func(t *testing.T) { rowStreamingStdio(t, rt, caps, primary) }) - t.Run("kill_wait_deliberate", func(t *testing.T) { rowKillWait(t, rt, caps, primary) }) + t.Run("kill_wait", func(t *testing.T) { rowKillWait(t, rt, caps, primary) }) t.Run("ctx_cancel_reaps", func(t *testing.T) { rowCtxCancelReaps(t, rt, caps, primary) }) t.Run("uid_enforcement", func(t *testing.T) { rowUIDEnforcement(t, rt, caps, primary) }) t.Run("resize_not_implemented", func(t *testing.T) { rowResize(t, rt, caps, primary) }) @@ -252,6 +254,78 @@ func rowExecExitCodes(t *testing.T, rt WorkloadRuntime, caps backendCaps, primar } } +// hasCannotInvokeReason accepts each backend's 126 wording, podman's included. +func hasCannotInvokeReason(stderr string) bool { + lower := strings.ToLower(stderr) + for _, reason := range []string{"not executable", "permission denied", "not a regular file"} { + if strings.Contains(lower, reason) { + return true + } + } + return false +} + +func rowExecUnstartablePrograms(t *testing.T, rt WorkloadRuntime, caps backendCaps, primary WorkloadID) { + t.Helper() + program := "/etc/passwd" + for _, tc := range []struct { + name string + command string + code int + }{ + {name: "missing program", command: "/compass-missing-contract-probe", code: 127}, + {name: "present but non-executable program", command: program, code: 126}, + } { + t.Run(tc.name, func(t *testing.T) { + out, err := rt.Exec(t.Context(), primary, NewExecSpec(tc.command).AsUser(caps.execUser())) + if err != nil { + t.Fatalf("Exec(%q) = %v, want exit %d and nil error", tc.command, err, tc.code) + } + if out.ExitCode != tc.code || out.Stderr == "" { + t.Fatalf("Exec(%q) = %+v, want exit %d and non-empty stderr", tc.command, out, tc.code) + } + if tc.code == 126 && (!strings.Contains(out.Stderr, tc.command) || !hasCannotInvokeReason(out.Stderr)) { + t.Fatalf("Exec(%q) stderr = %q, want program path and a cannot-invoke reason", tc.command, out.Stderr) + } + }) + } +} + +func rowExecStreamingUnstartablePrograms(t *testing.T, rt WorkloadRuntime, caps backendCaps, primary WorkloadID) { + t.Helper() + program := "/etc/passwd" + for _, tc := range []struct { + name string + command string + code int + }{ + {name: "missing program", command: "/compass-missing-contract-probe", code: 127}, + {name: "present but non-executable program", command: program, code: 126}, + } { + t.Run(tc.name, func(t *testing.T) { + stream, err := rt.ExecStreaming(t.Context(), primary, NewStreamingExecSpec(tc.command).AsUser(caps.execUser())) + if err != nil { + t.Fatalf("ExecStreaming(%q) = %v, want an exited stream", tc.command, err) + } + defer stream.IO.Stdin.Close() + stdout, err := io.ReadAll(stream.IO.Stdout) + if err != nil || len(stdout) != 0 { + t.Fatalf("stdout = %q, err = %v; want empty stdout", stdout, err) + } + stderr, err := io.ReadAll(stream.IO.Stderr) + if err != nil || len(stderr) == 0 { + t.Fatalf("stderr = %q, err = %v; want non-empty stderr", stderr, err) + } + if tc.code == 126 && (!strings.Contains(string(stderr), tc.command) || !hasCannotInvokeReason(string(stderr))) { + t.Fatalf("stderr = %q, want program path %q and a cannot-invoke reason", stderr, tc.command) + } + if waitErr := stream.Process.Wait(); exitCodeOf(waitErr, caps) != tc.code { + t.Fatalf("Process.Wait() = %v; want exit %d without signal", waitErr, tc.code) + } + }) + } +} + // rowExecStdin — row 2 (record 567-569, agent.go:238-246): the script-over-stdin // shape end to end (the secret-safe channel). `sh -s` reads the script from // stdin, so the body never appears in the argv / process list. @@ -630,6 +704,46 @@ func registerRemove(t *testing.T, rt WorkloadRuntime, id WorkloadID, label strin }) } +// exitCodeOf reads a plain exit code from a Wait error: the portable +// *ExitStatusError, or *exec.ExitError only for backends without one; -1 otherwise. +func exitCodeOf(err error, caps backendCaps) int { + if e, ok := errors.AsType[*ExitStatusError](err); ok && e.Signal == 0 { + return e.Code + } + if caps.portableKillError { + return -1 + } + if e, ok := errors.AsType[*exec.ExitError](err); ok { + return e.ExitCode() + } + return -1 +} + +func TestExitCodeOf(t *testing.T) { + cmd := exec.CommandContext(t.Context(), "sh", "-c", "exit 3") + runErr := cmd.Run() + if _, ok := errors.AsType[*exec.ExitError](runErr); !ok { + t.Fatalf("sh exit 3 = %v, want *exec.ExitError", runErr) + } + for _, tc := range []struct { + name string + err error + portable bool + want int + }{ + {name: "exec.ExitError without portable exit error", err: runErr, want: 3}, + {name: "exec.ExitError with portable exit error", err: runErr, portable: true, want: -1}, + {name: "portable exit", err: &ExitStatusError{Code: 126}, portable: true, want: 126}, + {name: "signalled portable exit", err: &ExitStatusError{Signal: 9}, portable: true, want: -1}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := exitCodeOf(tc.err, backendCaps{portableKillError: tc.portable}); got != tc.want { + t.Fatalf("exitCodeOf = %d, want %d", got, tc.want) + } + }) + } +} + // deliberateKill reports whether err is the deliberate-kill signal error the // backend produces after a Kill+Wait (divergence 6): the portable // *ExitStatusError with a non-zero Signal on microVM, or the byte-identical diff --git a/go/internal/runtime/host_backend.go b/go/internal/runtime/host_backend.go index 88d1b1a89..59f510cd9 100644 --- a/go/internal/runtime/host_backend.go +++ b/go/internal/runtime/host_backend.go @@ -19,6 +19,7 @@ import ( "context" "errors" "fmt" + "io" "maps" "os" "os/exec" //nolint:depguard // host backend seam: *exec.Cmd/*exec.ExitError drive the direct host subprocess this backend exists to run @@ -210,13 +211,9 @@ func (h *HostRuntime) Start(_ context.Context, id WorkloadID) error { return nil } -// Exec runs the command as a direct host subprocess of the Runner, under the -// Runner's own uid, honoring ExecSpec's command/env/workdir/stdin under the -// per-command timeout. A non-zero exit is a successful call returning -// ExecOutput.ExitCode; only a spawn failure or a timeout is a Go error. The -// child inherits the Runner's environment with ExecSpec.Env overriding per key -// — there is no image here to supply a baseline. See checkUser for the AsUser -// rule. +// Exec runs a command as a direct host child under the Runner's uid and +// timeout. Missing programs return 127; programs that cannot execute return +// 126. A missing or non-directory workdir or missing interpreter is an error. func (h *HostRuntime) Exec(ctx context.Context, id WorkloadID, spec ExecSpec) (ExecOutput, error) { if err := h.checkUser(spec.User); err != nil { return ExecOutput{}, err @@ -228,13 +225,19 @@ func (h *HostRuntime) Exec(ctx context.Context, id WorkloadID, spec ExecSpec) (E return ExecOutput{}, errors.New("runtime: host exec requires a command") } + if err := ctx.Err(); err != nil { + return ExecOutput{}, err + } + childEnv := envSlice(spec.Env) + if code, stderr, ok := hostExecPrecheck(spec.Command[0], spec.Workdir, childEnv); ok { + return ExecOutput{Stderr: stderr, ExitCode: code}, nil + } cctx, cancel := context.WithTimeout(ctx, h.timeout) defer cancel() - //nolint:gosec // G204: the host backend seam spawns the Runner-assembled agent command; the argv is not attacker-controlled. - cmd := exec.CommandContext(cctx, spec.Command[0], spec.Command[1:]...) + cmd := hostExecCommand(cctx, spec.Command, spec.Workdir, childEnv) cmd.WaitDelay = 10 * time.Second - cmd.Env = envSlice(spec.Env) + cmd.Env = childEnv if spec.Workdir != nil { cmd.Dir = *spec.Workdir } @@ -254,14 +257,14 @@ func (h *HostRuntime) Exec(ctx context.Context, id WorkloadID, spec ExecSpec) (E case ctx.Err() != nil: return ExecOutput{}, ctx.Err() default: + executablePath := cmd.Path + if code, stderr, ok := hostExecStartFailure(spec.Command[0], spec.Workdir, executablePath, runErr); ok { + return ExecOutput{Stderr: stderr, ExitCode: code}, nil + } if exitErr, ok := errors.AsType[*exec.ExitError](runErr); ok { - // Ran to completion but exited non-zero: a successful call. return ExecOutput{Stdout: out.String(), Stderr: errBuf.String(), ExitCode: exitErr.ExitCode()}, nil } if errors.Is(runErr, exec.ErrWaitDelay) { - // The command itself exited; a leaked grandchild held the output - // pipe past that exit. The exit status is real, so report it with - // what was captured rather than losing it to a spawn error. return ExecOutput{Stdout: out.String(), Stderr: errBuf.String(), ExitCode: cmd.ProcessState.ExitCode()}, nil } return ExecOutput{}, &SpawnError{Program: spec.Command[0], Err: runErr} @@ -270,11 +273,9 @@ func (h *HostRuntime) Exec(ctx context.Context, id WorkloadID, spec ExecSpec) (E return ExecOutput{Stdout: out.String(), Stderr: errBuf.String(), ExitCode: 0}, nil } -// ExecStreaming spawns the agent as a host child in its own process group, -// stdio piped, bound to ctx so cancelling it terminates the process. This is -// the one clean mapping — where the host-tier agent comes to life. No -// wall-clock timeout: the process is meant to run indefinitely. Same AsUser -// rule as Exec. +// ExecStreaming starts a host child in its own process group and returns its live +// pipes. Missing programs return 127; programs that cannot execute return 126. +// A missing or non-directory workdir or missing interpreter is an error. func (h *HostRuntime) ExecStreaming(ctx context.Context, id WorkloadID, spec StreamingExecSpec) (*StreamingExec, error) { if err := h.checkUser(spec.User); err != nil { return nil, err @@ -287,15 +288,19 @@ func (h *HostRuntime) ExecStreaming(ctx context.Context, id WorkloadID, spec Str return nil, errors.New("runtime: host streaming exec requires a command") } + if err := ctx.Err(); err != nil { + return nil, err + } + childEnv := envSlice(spec.Env) + if code, stderr, ok := hostExecPrecheck(spec.Command[0], spec.Workdir, childEnv); ok { + return hostStartFailureStream(code, stderr), nil + } execCtx, cancel := context.WithCancel(ctx) - //nolint:gosec // G204: the host backend seam spawns the Runner-assembled agent command; the argv is not attacker-controlled. - cmd := exec.CommandContext(execCtx, spec.Command[0], spec.Command[1:]...) - // Own process group so Stop/Remove can signal the whole tree, and ctx - // cancellation SIGKILLs that group rather than the leader alone. + cmd := hostExecCommand(execCtx, spec.Command, spec.Workdir, childEnv) cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} cmd.Cancel = func() error { return killGroup(cmd.Process.Pid, syscall.SIGKILL) } cmd.WaitDelay = 10 * time.Second - cmd.Env = envSlice(spec.Env) + cmd.Env = childEnv if spec.Workdir != nil { cmd.Dir = *spec.Workdir } @@ -324,7 +329,16 @@ func (h *HostRuntime) ExecStreaming(ctx context.Context, id WorkloadID, spec Str if startErr := cmd.Start(); startErr != nil { cancel() - return nil, errors.Join(&SpawnError{Program: spec.Command[0], Err: startErr}, closePipes(stdinR, stdinW, stdoutR, stdoutW, stderrR, stderrW)) + cleanupErr := closePipes(stdinR, stdinW, stdoutR, stdoutW, stderrR, stderrW) + if code, stderr, ok := hostExecStartFailure(spec.Command[0], spec.Workdir, cmd.Path, startErr); ok { + stream := hostStartFailureStream(code, stderr) + if cleanupErr != nil { + _ = errors.Join(stream.IO.Stdin.Close(), stream.IO.Stdout.Close(), stream.IO.Stderr.Close()) + return nil, &SpawnError{Program: spec.Command[0], Err: cleanupErr} + } + return stream, nil + } + return nil, errors.Join(&SpawnError{Program: spec.Command[0], Err: startErr}, cleanupErr) } // Close the child-side ends in the parent so the caller sees EOF when the // child exits and closes its own copies. @@ -376,10 +390,147 @@ func (h *HostRuntime) ExecStreaming(ctx context.Context, id WorkloadID, spec Str }, nil } -// Stop signals the handle's process group: SIGTERM, wait up to timeout, then -// SIGKILL. Scope is the group the backend spawned — a process the agent -// double-forked out of that group is NOT reliably stopped (no cgroup freezer in -// v1). Stopping a handle with no live process is a no-op. +func hostExecCommand(ctx context.Context, command []string, workdir *string, env []string) *exec.Cmd { + path := command[0] + if !strings.ContainsRune(path, '/') { + pathEnv := "" + for _, kv := range env { + if value, ok := strings.CutPrefix(kv, "PATH="); ok { + pathEnv = value + break + } + } + for _, dir := range filepath.SplitList(pathEnv) { + if dir == "" { + dir = "." + } + candidate := filepath.Join(dir, path) + if candidate == path { + candidate = "." + string(filepath.Separator) + candidate + } + resolved := pathAtWorkdir(candidate, workdir) + if info, err := os.Stat(resolved); err == nil && info.Mode().IsRegular() && info.Mode().Perm()&0o111 != 0 { + path, err = filepath.Abs(resolved) + if err != nil { + path = candidate + } + break + } + } + } + cmd := exec.CommandContext(ctx, path, command[1:]...) //nolint:gosec // path is resolved from the exec environment PATH; argv remains an argument vector + cmd.Path = path + cmd.Err = nil + return cmd +} + +func hostExecPrecheck(program string, workdir *string, env []string) (int, string, bool) { + explicit := strings.ContainsRune(program, '/') + if workdir != nil { + if info, err := os.Stat(*workdir); err != nil || !info.IsDir() { + return 0, "", false + } + } + if explicit { + info, err := os.Stat(pathAtWorkdir(program, workdir)) + if errors.Is(err, syscall.ENOENT) { + return hostMissingProgram(program, true) + } + if err != nil { + return 0, "", false + } + if !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 { + return hostNonExecutableProgram(program) + } + return 0, "", false + } + pathEnv := "" + for _, kv := range env { + if value, ok := strings.CutPrefix(kv, "PATH="); ok { + pathEnv = value + break + } + } + notExecutable := false + for _, dir := range filepath.SplitList(pathEnv) { + if dir == "" { + dir = "." + } + candidate := filepath.Join(dir, program) + info, err := os.Stat(pathAtWorkdir(candidate, workdir)) + // Like a shell PATH search: unreadable entries and directories are skipped. + if err != nil || !info.Mode().IsRegular() { + continue + } + if info.Mode().Perm()&0o111 == 0 { + notExecutable = true + continue + } + return 0, "", false + } + if notExecutable { + return hostNonExecutableProgram(program) + } + return hostMissingProgram(program, false) +} +func hostMissingProgram(program string, explicit bool) (int, string, bool) { + if explicit { + return 127, fmt.Sprintf("%q: no such file or directory\n", program), true + } + return 127, fmt.Sprintf("executable file %q not found in $PATH\n", program), true +} + +func pathAtWorkdir(path string, workdir *string) string { + if workdir != nil && !filepath.IsAbs(path) { + return filepath.Join(*workdir, path) + } + return path +} + +func hostExecStartFailure(program string, workdir *string, executablePath string, err error) (int, string, bool) { + path, ok := errors.AsType[*os.PathError](err) + if !ok || path.Op != "fork/exec" || path.Path != executablePath { + return 0, "", false + } + switch { + case errors.Is(path.Err, syscall.ENOENT): + if workdir != nil { + if _, err := os.Stat(*workdir); err != nil { + return 0, "", false + } + } + if _, err := os.Stat(pathAtWorkdir(executablePath, workdir)); errors.Is(err, syscall.ENOENT) { + return hostMissingProgram(program, strings.ContainsRune(program, '/')) + } + return 0, "", false + case errors.Is(path.Err, syscall.ENOEXEC), errors.Is(path.Err, syscall.EACCES), errors.Is(path.Err, syscall.EPERM): + return hostNonExecutableProgram(program) + default: + return 0, "", false + } +} + +func hostNonExecutableProgram(program string) (int, string, bool) { + return 126, fmt.Sprintf("%q: permission denied / not executable\n", program), true +} + +func hostStartFailureStream(code int, stderr string) *StreamingExec { + stdinR, stdinW := io.Pipe() + _ = stdinR.Close() + return &StreamingExec{ + IO: StreamingIO{ + Stdin: stdinW, + Stdout: io.NopCloser(strings.NewReader("")), + Stderr: io.NopCloser(strings.NewReader(stderr)), + }, + Process: newChildHandleFuncs(func() error { return nil }, func() error { + return &ExitStatusError{Code: code} + }), + } +} + +// Stop SIGTERMs the spawned process group, then SIGKILLs after timeout. A process +// double-forked out of that group is not reliably stopped (no cgroup freezer). func (h *HostRuntime) Stop(ctx context.Context, id WorkloadID, timeout time.Duration) error { stopped := h.liveProcess(id) if stopped != nil { diff --git a/go/internal/runtime/host_backend_test.go b/go/internal/runtime/host_backend_test.go index d898d16cf..56ec361b6 100644 --- a/go/internal/runtime/host_backend_test.go +++ b/go/internal/runtime/host_backend_test.go @@ -15,6 +15,7 @@ import ( "strconv" "strings" "sync" + "syscall" "testing" "time" ) @@ -109,6 +110,284 @@ func TestHostExecNonZeroExitIsNotError(t *testing.T) { } } +func TestHostExecUnstartableProgramReturnsExitCode(t *testing.T) { + h := newHostRuntime(t) + id := createStarted(t, h, "agent-unstartable") + dir := t.TempDir() + nonExecutable := filepath.Join(dir, "noexec") + if err := os.WriteFile(nonExecutable, []byte("not executable"), 0o644); err != nil { + t.Fatal(err) + } + pathFile := filepath.Join(dir, "path-entry-file") + if err := os.WriteFile(pathFile, []byte("not a directory"), 0o644); err != nil { + t.Fatal(err) + } + invalidExecutable := filepath.Join(dir, "invalid-executable") + if err := os.WriteFile(invalidExecutable, []byte("not an executable format"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "bare-noexec"), []byte("not executable"), 0o644); err != nil { + t.Fatal(err) + } + tests := []struct { + name string + command string + code int + }{ + {name: "missing", command: "/compass-missing-exec-test", code: 127}, + {name: "missing after regular-file PATH entry", command: "missing-after-path-entry", code: 127}, + {name: "directory", command: "/", code: 126}, + {name: "non-executable explicit path", command: nonExecutable, code: 126}, + {name: "non-executable from PATH", command: "bare-noexec", code: 126}, + {name: "non-executable after PATH file", command: "bare-noexec", code: 126}, + {name: "exec format failure", command: invalidExecutable, code: 126}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := NewExecSpec(tt.command) + spec.Env["PATH"] = dir + string(os.PathListSeparator) + os.Getenv("PATH") + if tt.name == "missing after regular-file PATH entry" { + spec.Env["PATH"] = pathFile + } + if tt.name == "non-executable after PATH file" { + spec.Env["PATH"] = pathFile + string(os.PathListSeparator) + dir + } + out, err := h.Exec(t.Context(), id, spec) + if err != nil { + t.Fatalf("Exec(%q) = %v, want exit %d with no error", tt.command, err, tt.code) + } + if out.ExitCode != tt.code || out.Stderr == "" { + t.Fatalf("Exec(%q) = %+v, want exit %d and non-empty stderr", tt.command, out, tt.code) + } + }) + } +} + +func TestHostExecStartFailureClassifiesPermissionErrors(t *testing.T) { + for _, errno := range []error{syscall.EACCES, syscall.EPERM, syscall.ENOEXEC} { + t.Run(errno.Error(), func(t *testing.T) { + path := &os.PathError{Op: "fork/exec", Path: "/probe", Err: errno} + code, stderr, ok := hostExecStartFailure("/probe", nil, "/probe", path) + if !ok || code != 126 || !strings.Contains(stderr, "/probe") { + t.Fatalf("hostExecStartFailure() = (%d, %q, %v), want 126, diagnostic, true", code, stderr, ok) + } + }) + } +} +func TestHostExecSkipsUnsearchablePATHEntry(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory search permission") + } + h := newHostRuntime(t) + id := createStarted(t, h, "agent-unsearchable-path") + locked := filepath.Join(t.TempDir(), "locked") + if err := os.Mkdir(locked, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(locked, 0o700) }) + spec := NewExecSpec("compass-unsearchable-probe") + spec.Env = map[string]string{"PATH": locked} + out, err := h.Exec(t.Context(), id, spec) + if err != nil || out.ExitCode != 127 { + t.Fatalf("Exec with unsearchable PATH entry = (%+v, %v), want exit 127", out, err) + } +} + +func TestHostExecCanceledContextBeforePrecheck(t *testing.T) { + h := newHostRuntime(t) + id := createStarted(t, h, "agent-canceled-precheck") + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if out, err := h.Exec(ctx, id, NewExecSpec("/compass-missing-canceled-probe")); !errors.Is(err, context.Canceled) { + t.Fatalf("Exec with canceled ctx = (%+v, %v), want context.Canceled", out, err) + } + if _, err := h.ExecStreaming(ctx, id, NewStreamingExecSpec("/compass-missing-canceled-probe")); !errors.Is(err, context.Canceled) { + t.Fatalf("ExecStreaming with canceled ctx = %v, want context.Canceled", err) + } +} + +func TestHostExecNonDirectoryWorkdirTakesPrecedenceOverMissingBareProgram(t *testing.T) { + h := newHostRuntime(t) + id := createStarted(t, h, "agent-nondir-workdir-bare-missing") + file := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(file, []byte("file"), 0o600); err != nil { + t.Fatal(err) + } + for _, streaming := range []bool{false, true} { + t.Run(map[bool]string{false: "one-shot", true: "streaming"}[streaming], func(t *testing.T) { + if streaming { + spec := NewStreamingExecSpec("compass-missing-workdir-probe") + spec.Workdir = &file + if _, err := h.ExecStreaming(t.Context(), id, spec); err == nil { + t.Fatal("ExecStreaming with non-directory workdir and missing bare program returned nil error") + } + return + } + spec := NewExecSpec("compass-missing-workdir-probe") + spec.Workdir = &file + if _, err := h.Exec(t.Context(), id, spec); err == nil { + t.Fatal("Exec with non-directory workdir and missing bare program returned nil error") + } + }) + } +} + +func TestHostExecMissingWorkdirRemainsError(t *testing.T) { + h := newHostRuntime(t) + id := createStarted(t, h, "agent-missing-workdir") + missing := filepath.Join(t.TempDir(), "missing") + spec := NewExecSpec("/bin/true") + spec.Workdir = &missing + if _, err := h.Exec(t.Context(), id, spec); err == nil { + t.Fatal("Exec with missing workdir returned nil error") + } +} + +func TestHostExecNonDirectoryWorkdirRemainsError(t *testing.T) { + h := newHostRuntime(t) + id := createStarted(t, h, "agent-nondir-workdir") + file := filepath.Join(t.TempDir(), "file") + if err := os.WriteFile(file, []byte("not a directory"), 0o600); err != nil { + t.Fatal(err) + } + spec := NewExecSpec("/bin/true") + spec.Workdir = &file + if _, err := h.Exec(t.Context(), id, spec); err == nil { + t.Fatal("Exec with non-directory workdir returned nil error") + } +} + +func TestHostExecInvalidWorkdirTakesPrecedenceOverMissingProgram(t *testing.T) { + h := newHostRuntime(t) + id := createStarted(t, h, "agent-workdir-precedence") + missing := filepath.Join(t.TempDir(), "missing-workdir") + for _, streaming := range []bool{false, true} { + t.Run(map[bool]string{false: "one-shot", true: "streaming"}[streaming], func(t *testing.T) { + workdir := missing + if streaming { + spec := NewStreamingExecSpec(filepath.Join(missing, "missing-program")) + spec.Workdir = &workdir + if _, err := h.ExecStreaming(t.Context(), id, spec); err == nil { + t.Fatal("ExecStreaming with invalid workdir and missing program returned nil error") + } + return + } + spec := NewExecSpec(filepath.Join(missing, "missing-program")) + spec.Workdir = &workdir + if _, err := h.Exec(t.Context(), id, spec); err == nil { + t.Fatal("Exec with invalid workdir and missing program returned nil error") + } + }) + } +} + +func TestHostExecStreamingInvalidWorkdirRemainsError(t *testing.T) { + h := newHostRuntime(t) + id := createStarted(t, h, "agent-stream-invalid-workdir") + file := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(file, []byte("file"), 0o600); err != nil { + t.Fatal(err) + } + spec := NewStreamingExecSpec("/bin/true") + spec.Workdir = &file + if _, err := h.ExecStreaming(t.Context(), id, spec); err == nil { + t.Fatal("ExecStreaming with non-directory workdir returned nil error") + } +} + +func TestHostExecStreamingFailedStartClosesAllPipeDescriptors(t *testing.T) { + h := newHostRuntime(t) + id := createStarted(t, h, "agent-stream-failed-start-fds") + program := filepath.Join(t.TempDir(), "missing-interpreter") + if err := os.WriteFile(program, []byte("#!/compass-missing-interpreter\n"), 0o755); err != nil { + t.Fatal(err) + } + countFDs := func() int { + t.Helper() + entries, err := os.ReadDir("/proc/self/fd") + if err != nil { + t.Fatalf("reading open descriptors: %v", err) + } + return len(entries) + } + before := countFDs() + for range 10 { + if _, err := h.ExecStreaming(t.Context(), id, NewStreamingExecSpec(program)); err == nil { + t.Fatal("ExecStreaming with missing interpreter returned nil error") + } + } + if after := countFDs(); after != before { + t.Fatalf("open file descriptors = %d after failed starts, want %d before", after, before) + } +} + +func TestHostExecMissingInterpreterRemainsError(t *testing.T) { + h := newHostRuntime(t) + id := createStarted(t, h, "agent-missing-interpreter") + program := filepath.Join(t.TempDir(), "missing-interpreter") + if err := os.WriteFile(program, []byte("#!/compass-missing-interpreter\necho never\n"), 0o755); err != nil { + t.Fatal(err) + } + if _, err := h.Exec(t.Context(), id, NewExecSpec(program)); err == nil { + t.Fatal("Exec with missing script interpreter returned nil error") + } + if _, err := h.ExecStreaming(t.Context(), id, NewStreamingExecSpec(program)); err == nil { + t.Fatal("ExecStreaming with missing script interpreter returned nil error") + } +} + +func TestHostExecStreamingUnstartableProgramReturnsExit(t *testing.T) { + h := newHostRuntime(t) + id := createStarted(t, h, "agent-stream-unstartable") + dir := t.TempDir() + nonExecutable := filepath.Join(dir, "noexec") + if err := os.WriteFile(nonExecutable, []byte("not executable"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "bare-noexec"), []byte("not executable"), 0o644); err != nil { + t.Fatal(err) + } + invalidExecutable := filepath.Join(dir, "invalid-executable") + if err := os.WriteFile(invalidExecutable, []byte("not an executable format"), 0o755); err != nil { + t.Fatal(err) + } + tests := []struct { + name string + command string + code int + }{ + {name: "missing", command: "/compass-missing-stream-test", code: 127}, + {name: "directory", command: "/", code: 126}, + {name: "non-executable explicit path", command: nonExecutable, code: 126}, + {name: "non-executable from PATH", command: "bare-noexec", code: 126}, + {name: "exec format failure", command: invalidExecutable, code: 126}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := NewStreamingExecSpec(tt.command) + spec.Env["PATH"] = dir + string(os.PathListSeparator) + os.Getenv("PATH") + stream, err := h.ExecStreaming(t.Context(), id, spec) + if err != nil { + t.Fatalf("ExecStreaming(%q) = %v, want an exited stream", tt.command, err) + } + defer stream.IO.Stdin.Close() + stdout, err := io.ReadAll(stream.IO.Stdout) + if err != nil || len(stdout) != 0 { + t.Fatalf("stdout = %q, err = %v; want empty stdout", stdout, err) + } + stderr, err := io.ReadAll(stream.IO.Stderr) + if err != nil || len(stderr) == 0 { + t.Fatalf("stderr = %q, err = %v; want non-empty stderr", stderr, err) + } + waitErr := stream.Process.Wait() + status, ok := errors.AsType[*ExitStatusError](waitErr) + if !ok || status.Code != tt.code || status.Signal != 0 { + t.Fatalf("Process.Wait() = %v; want exit %d without signal", waitErr, tt.code) + } + }) + } +} + // TestHostExecLeakedChildKeepsExitStatus: a command that SUCCEEDS but leaves a // background child holding the output pipe must still report success. Go's // WaitDelay fires on the orphan's inherited pipe, not on the command. A @@ -199,6 +478,78 @@ func TestHostExecInheritsRunnerEnv(t *testing.T) { } } +func TestHostExecUsesExecEnvironmentPATH(t *testing.T) { + h := newHostRuntime(t) + id := createStarted(t, h, "agent-custom-path") + dir := t.TempDir() + program := filepath.Join(dir, "host-path-probe") + if err := os.WriteFile(program, []byte("#!/bin/sh\nprintf custom-path\n"), 0o755); err != nil { + t.Fatal(err) + } + spec := NewExecSpec("host-path-probe") + spec.Env["PATH"] = dir + out, err := h.Exec(t.Context(), id, spec) + if err != nil { + t.Fatalf("Exec with custom PATH: %v", err) + } + if out.ExitCode != 0 || out.Stdout != "custom-path" { + t.Fatalf("Exec output = %+v, want custom-path with exit 0", out) + } +} + +func TestHostExecRelativePATHWithRelativeWorkdir(t *testing.T) { + h := newHostRuntime(t) + id := createStarted(t, h, "agent-relative-path") + dir, err := os.MkdirTemp(".", "host-relative-path-") //nolint:usetesting // The relative directory is required to test child workdir PATH lookup. + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + workdir := dir + bin := filepath.Join(workdir, "bin") + if err := os.Mkdir(bin, 0o755); err != nil { + t.Fatal(err) + } + program := filepath.Join(bin, "host-relative-path-probe") + if err := os.WriteFile(program, []byte("#!/bin/sh\nprintf relative-path\n"), 0o755); err != nil { + t.Fatal(err) + } + spec := NewExecSpec("host-relative-path-probe") + spec.Env["PATH"] = "bin" + spec.Workdir = &workdir + out, err := h.Exec(t.Context(), id, spec) + if err != nil { + t.Fatalf("Exec with relative PATH and workdir: %v", err) + } + if out.ExitCode != 0 || out.Stdout != "relative-path" { + t.Fatalf("Exec output = %+v, want relative-path with exit 0", out) + } +} + +func TestHostExecDotPATHWithWorkdir(t *testing.T) { + h := newHostRuntime(t) + id := createStarted(t, h, "agent-dot-path") + workdir, err := os.MkdirTemp(".", "host-dot-path-") //nolint:usetesting // Relative workdir is needed to test a dot PATH entry. + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(workdir) }) + program := filepath.Join(workdir, "dot-path-probe") + if err := os.WriteFile(program, []byte("#!/bin/sh\nprintf dot-path\n"), 0o755); err != nil { + t.Fatal(err) + } + spec := NewExecSpec("dot-path-probe") + spec.Env["PATH"] = "." + spec.Workdir = &workdir + out, err := h.Exec(t.Context(), id, spec) + if err != nil { + t.Fatalf("Exec with dot PATH: %v", err) + } + if out.ExitCode != 0 || out.Stdout != "dot-path" { + t.Fatalf("Exec output = %+v, want dot-path with exit 0", out) + } +} + // TestHostExecTimeout: a command that overruns the per-command cap is killed // and surfaced as a TimeoutError, not a ctx error. A short WithTimeout makes // the internal deadline fire deterministically against a command that would diff --git a/go/internal/runtime/microvm/exec.go b/go/internal/runtime/microvm/exec.go index c2f380631..5bc3122cc 100644 --- a/go/internal/runtime/microvm/exec.go +++ b/go/internal/runtime/microvm/exec.go @@ -170,13 +170,12 @@ func isDeadline(err error) bool { connect.CodeOf(err) == connect.CodeDeadlineExceeded } -// ExecStream opens the bidi stream, sends the StartExec frame, and AWAITS the -// ExecStarted frame before returning — so a spawn failure surfaces as the -// returned error rather than a broken pipe on first read. It returns a -// GuestStream whose io.Pipe stdio is pumped by a per-exec goroutine: the Stdin -// pipe is framed as stdin frames (Close -> stdin_close), and stdout/stderr -// response frames are demuxed onto the read pipes, which close when the -// ExecExit frame arrives. +// ExecStream opens the bidi stream, sends the StartExec frame, and awaits +// ExecStarted before returning. An unstartable program instead returns an +// exited GuestStream with a stderr diagnostic and terminal 126/127 status. +// It returns a GuestStream whose io.Pipe stdio is pumped by a per-exec +// goroutine: stdin is framed as stdin frames and stdout/stderr are demuxed +// onto the read pipes, which close when ExecExit arrives. func (g *GuestExec) ExecStream(ctx context.Context, call StreamCall) (_ *GuestStream, retErr error) { stream := g.openBidi(ctx) // Until the pumps take ownership, any early-error return must reap both @@ -214,6 +213,9 @@ func (g *GuestExec) ExecStream(ctx context.Context, call StreamCall) (_ *GuestSt if started == nil { return nil, fmt.Errorf("guest exec stream: first frame was not started: %T", first.GetFrame()) } + if started.GetExecId() == "" { + return nil, errors.New("guest exec stream: ExecStarted has empty exec_id") + } stdinR, stdinW := io.Pipe() stdoutR, stdoutW := io.Pipe() diff --git a/go/internal/runtime/microvm/exec_test.go b/go/internal/runtime/microvm/exec_test.go index cd0f6a26b..1f88a4bcb 100644 --- a/go/internal/runtime/microvm/exec_test.go +++ b/go/internal/runtime/microvm/exec_test.go @@ -457,6 +457,23 @@ func TestGuestExec_Stream_FirstFrameNotStartedIsError(t *testing.T) { } } +func TestGuestExec_Stream_EmptyExecIDIsProtocolError(t *testing.T) { + ge := serveFakeGuest(t, &fakeGuest{ + execStreamFn: func(_ context.Context, stream *connect.BidiStream[compassv1.ExecStreamRequest, compassv1.ExecStreamResponse]) error { + if _, err := stream.Receive(); err != nil { + return err + } + return stream.Send(&compassv1.ExecStreamResponse{ + Frame: &compassv1.ExecStreamResponse_Started{Started: &compassv1.ExecStarted{}}, + }) + }, + }) + gs, err := ge.ExecStream(context.Background(), StreamCall{Command: []string{"true"}}) + if err == nil || gs != nil { + t.Fatalf("ExecStream with empty exec_id returned stream %v and error %v, want protocol error", gs, err) + } +} + // blockingBidiStream is a fake bidiStream whose Receive returns one ExecStarted // frame and then blocks until CloseResponse is called. It models the reap-hang // preconditions exactly: a guest child that emits nothing (so no exit frame diff --git a/go/internal/runtime/microvm_lifecycle.go b/go/internal/runtime/microvm_lifecycle.go index 6de7e99da..ee25c3aa8 100644 --- a/go/internal/runtime/microvm_lifecycle.go +++ b/go/internal/runtime/microvm_lifecycle.go @@ -461,12 +461,9 @@ func bootPollContext(ctx context.Context) (context.Context, context.CancelFunc) return context.WithTimeout(ctx, bootDeadline) } -// Exec runs one command to completion in the session guest, mapping the spec -// onto microvm.ExecCall and the result back onto ExecOutput. A non-zero exit is -// a SUCCESSFUL call (captured in ExecOutput.ExitCode), never an error; a guest -// refusal or transport failure is an error, and a host-side timeout is mapped -// to a *runtime.TimeoutError so requireSuccess/atStage callers behave -// identically to the podman path (record §(c)). +// Exec runs one command in the session guest. Missing programs return 127 and +// programs that cannot execute return 126 in ExecOutput; runtime and transport +// faults remain errors, with timeouts mapped to *runtime.TimeoutError. func (m *MicroVMRuntime) Exec(ctx context.Context, id WorkloadID, spec ExecSpec) (ExecOutput, error) { guestExec, err := m.startedExec(id) if err != nil { @@ -549,14 +546,9 @@ func exitError(st microvm.ExitStatus) error { } } -// ExecStreaming starts a long-lived streaming exec in the session guest, -// mapping the spec onto microvm.StreamCall and the GuestStream onto a -// *StreamingExec. ExecStream awaits the ExecStarted frame, so a spawn failure -// surfaces as the returned error. The ChildHandle is built over the remote -// exec's kill/wait pair: killFunc issues a bounded SIGKILL Signal (never -// blocking teardown), and waitFunc maps the guest exit onto nil / a -// *runtime.ExitStatusError so the runner's isDeliberateKill recognizes a -// signalled exit as a deliberate kill (OQ-G/U3b, record §(c)). +// ExecStreaming starts a streaming exec in the session guest. Missing programs +// return exit 127; programs that cannot execute return 126. Other spawn failures +// and transport faults remain errors. func (m *MicroVMRuntime) ExecStreaming(ctx context.Context, id WorkloadID, spec StreamingExecSpec) (*StreamingExec, error) { guestExec, err := m.startedExec(id) if err != nil { diff --git a/go/internal/runtime/podman.go b/go/internal/runtime/podman.go index cb8e7a12d..b9ad886ec 100644 --- a/go/internal/runtime/podman.go +++ b/go/internal/runtime/podman.go @@ -360,22 +360,16 @@ type WorkloadRuntime interface { // Start starts a created workload. Start(ctx context.Context, id WorkloadID) error - // Exec runs a command in a running workload, capturing its output. A - // non-zero exit is a successful runtime call returning a failed command - // (ExecOutput.ExitCode), not an error; only a spawn failure or timeout is an - // error. + // Missing programs return exit 127; present programs that cannot execute + // return 126. Workdir and interpreter faults are backend-specific. Exec(ctx context.Context, id WorkloadID, spec ExecSpec) (ExecOutput, error) - // ExecStreaming starts a long-lived streaming command in a running - // workload, returning its live stdio pipes plus a kill/wait handle rather - // than awaiting completion. The transport for the long-running agent - // process: its stdout/stderr stay open and are drained as diagnostics for - // the process's life (the agent's protocol rides the per-workload socket). - // Unlike Exec there is no wall-clock timeout — the process is meant to run - // indefinitely — but the exec is still bound to ctx, so cancelling it - // terminates the process. + // ExecStreaming starts a long-lived streaming command in a running workload, + // returning its live stdio pipes and kill/wait handle. Missing programs and + // non-executable programs return an exited stream with code 127 or 126. + // Other runtime faults return errors. + // Cancelling ctx terminates the process. ExecStreaming(ctx context.Context, id WorkloadID, spec StreamingExecSpec) (*StreamingExec, error) - // Stop stops a running workload, allowing timeout for graceful exit before // the backend kills it. Stop(ctx context.Context, id WorkloadID, timeout time.Duration) error diff --git a/proto/compass/v1/guest_control.proto b/proto/compass/v1/guest_control.proto index 40ea5fb55..17bf97393 100644 --- a/proto/compass/v1/guest_control.proto +++ b/proto/compass/v1/guest_control.proto @@ -53,24 +53,19 @@ package compass.v1; service GuestControl { rpc Health(HealthRequest) returns (HealthResponse); - // Exec runs one command to completion and returns its captured output. A - // non-zero exit is a SUCCESSFUL call whose ExecResponse.exit_code is - // non-zero — never a Connect error. A Connect error means the exec could not - // be attempted or could not complete: a spawn failure, a refused uid (0), a - // timeout, or an unprovisioned gate. + // Exec captures one command. A missing program returns 127; a present one + // that cannot be invoked (no exec bit, ENOEXEC, execve EACCES/EPERM) returns + // 126. A missing or non-directory workdir or missing interpreter is an error. rpc Exec(ExecRequest) returns (ExecResponse); - // ExecStream runs a long-lived command over one bidi stream: the request - // stream carries the start frame, then stdin bytes and an optional - // half-close; the response stream carries the exec_id, then interleaved - // stdout/stderr, then exactly one terminal exit frame. Kill rides Signal - // (below), not this stream, so a teardown never contends with an in-flight - // stdin write. + // ExecStream: one start frame, stdin, optional half-close. Start failures + // follow Exec's 127/126 rule as ExecStarted, stderr, ExecExit; a missing or + // non-directory workdir or missing interpreter errors before ExecStarted. rpc ExecStream(stream ExecStreamRequest) returns (stream ExecStreamResponse); - // Signal delivers a signal to a running exec by exec_id; an empty exec_id - // targets the guest itself (graceful Stop, record §(d)). Signalling an - // already-exited exec is a no-op success. + // Signal delivers a signal to a running exec by exec_id (exec kill rides + // here, not ExecStream); an empty exec_id targets the guest itself (graceful + // Stop, record §(d)). Signalling an already-exited exec is a no-op success. rpc Signal(SignalRequest) returns (SignalResponse); // Provision transitions the supervisor ready -> provisioned: it records the @@ -94,9 +89,8 @@ message HealthResponse { bytes boot_nonce = 4; } -// ExecRequest is a one-shot exec. It mirrors runtime.ExecSpec field for field. +// ExecRequest mirrors runtime.ExecSpec. See GuestControl.Exec for the 127/126 rule. message ExecRequest { - // command is argv; command[0] is the program. repeated string command = 1; // uid is the exec's user. Absent = the session default uid set by Provision // (the baked agent uid). uid 0 is REFUSED with a failed-precondition error @@ -117,8 +111,8 @@ message ExecRequest { message ExecResponse { bytes stdout = 1; bytes stderr = 2; - // exit_code is the command's exit status. Non-zero is a successful call with - // a failed command, never a Connect error (see the Exec doc-comment). + // Non-zero statuses, including 126/127, are successful calls. A missing or + // non-directory workdir, a missing interpreter, or a transport failure is an error. int32 exit_code = 3; } @@ -143,8 +137,8 @@ message StartExec { // StdinClose half-closes the child's stdin pipe without ending the stream. message StdinClose {} -// ExecStreamResponse frames flow guest -> host: exactly one ExecStarted first, -// then interleaved stdout/stderr, then exactly one ExecExit. +// Exactly one ExecStarted, then interleaved stdout/stderr, then one ExecExit. +// A start failure classified as 127/126 uses the same order. message ExecStreamResponse { oneof frame { ExecStarted started = 1; From 41e9c2640652827d16c008baee3a561b706fe63a Mon Sep 17 00:00:00 2001 From: mintaka Date: Sun, 11 Oct 2026 23:39:56 -0400 Subject: [PATCH 2/2] test(runtime): drain stdout and stderr concurrently in the streaming contract row (RIG-5121) microVM stdio is an unbuffered io.Pipe, so reading stdout to EOF before stderr deadlocked the response pump on the stderr frame and timed out the microvm job. Refs RIG-5121 Co-authored-by: Matt Wilkinson --- go/internal/runtime/contract_suite_test.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/go/internal/runtime/contract_suite_test.go b/go/internal/runtime/contract_suite_test.go index ac289da58..6c9b6e3b4 100644 --- a/go/internal/runtime/contract_suite_test.go +++ b/go/internal/runtime/contract_suite_test.go @@ -308,13 +308,20 @@ func rowExecStreamingUnstartablePrograms(t *testing.T, rt WorkloadRuntime, caps t.Fatalf("ExecStreaming(%q) = %v, want an exited stream", tc.command, err) } defer stream.IO.Stdin.Close() + // Drain both concurrently: microVM stdio is unbuffered io.Pipe, so + // reading stdout to EOF first blocks the pump on the stderr frame. + stderrCh := make(chan []byte, 1) + go func() { + b, _ := io.ReadAll(stream.IO.Stderr) + stderrCh <- b + }() stdout, err := io.ReadAll(stream.IO.Stdout) if err != nil || len(stdout) != 0 { t.Fatalf("stdout = %q, err = %v; want empty stdout", stdout, err) } - stderr, err := io.ReadAll(stream.IO.Stderr) - if err != nil || len(stderr) == 0 { - t.Fatalf("stderr = %q, err = %v; want non-empty stderr", stderr, err) + stderr := <-stderrCh + if len(stderr) == 0 { + t.Fatalf("stderr is empty; want a diagnostic") } if tc.code == 126 && (!strings.Contains(string(stderr), tc.command) || !hasCannotInvokeReason(string(stderr))) { t.Fatalf("stderr = %q, want program path %q and a cannot-invoke reason", stderr, tc.command)