diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ca527a25f..ce03db6a3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -91,6 +91,10 @@ env: # fork image, so unrelated image changes must not trigger this publish. GATEWAY_IMAGE_CLOSURE_PATHS: | tools/gateway-image/** + go/cmd/compass-gen-cert/** + go/internal/certgen/** + go/go.mod + go/go.sum .github/workflows/release.yml # Same toolchain contract as the devenv shell: moon runs the nix tools on # PATH, never a ~/.proto shim, even if a runner image ever ships one. @@ -1803,7 +1807,8 @@ jobs: publish-gateway-image: name: publish-gateway-image - runs-on: ubuntu-latest + # passt on noble predates --map-host-loopback, which needs 2024-08-21 or later. + runs-on: ubuntu-26.04 permissions: contents: read packages: write @@ -1891,6 +1896,18 @@ jobs: nix build --no-link "$store" echo "$store/bin" >>"$GITHUB_PATH" + - name: Put the pinned go toolchain on PATH + if: steps.gate.outputs.should_publish == 'true' + run: | + set -euo pipefail + store=$(nix eval --json -f tools/toolchain/gate-tools.nix langs.go | jq -r '.store') + if [ -z "$store" ] || [ "$store" = null ]; then + echo "::error::gate-tools.nix langs.go produced no store path" >&2 + exit 1 + fi + nix build --no-link "$store" + echo "$store/bin" >>"$GITHUB_PATH" + - name: Put the pinned skopeo on PATH if: steps.gate.outputs.should_publish == 'true' run: | diff --git a/tools/gateway-image/README.md b/tools/gateway-image/README.md index 6105121ca..016a9109b 100644 --- a/tools/gateway-image/README.md +++ b/tools/gateway-image/README.md @@ -15,7 +15,8 @@ bun tools/gateway-image/smoke.ts Both commands accept `--out `; the default is `tools/gateway-image/out`. The build needs a reachable rootless BuildKit daemon and `BUILDKIT_HOST`. The -smoke needs `skopeo` and `podman`. +smoke needs `skopeo`, `podman`, Go, and Bun. It starts a local TLS fake Compass +Server and checks the gateway's health and agent-token handling. `moon run gateway-image:ci` runs the lane's typecheck and tests. The build and smoke tasks are not part of CI. @@ -31,8 +32,8 @@ It refuses to overwrite an existing tag with different content. ## Credentials Build and smoke need no registry credentials: the fork is public, and the smoke -mints a throwaway gateway token per run and deletes it on exit. Publish needs -GHCR write access through `REGISTRY_AUTH_FILE`. +uses throwaway service and agent tokens per run, then removes the mounted files. +Publish needs GHCR write access through `REGISTRY_AUTH_FILE`. ## Rotate diff --git a/tools/gateway-image/core.test.ts b/tools/gateway-image/core.test.ts index e77360bab..acd5cbffd 100644 --- a/tools/gateway-image/core.test.ts +++ b/tools/gateway-image/core.test.ts @@ -6,6 +6,7 @@ import { digestRef, FORK_BRANCH, FORK_REPO, + fakeCompassDoorResponse, gatewayBuildArgs, healthzOk, imageConfigEntries, @@ -234,14 +235,17 @@ describe("secretConfigViolations", () => { const scan = (env: string[], labels: Record = {}) => secretConfigViolations({ env, labels }); - test("passes the allowlisted names the gateway config ships", () => { + test("passes the benign names the gateway config ships", () => { + expect(scan(["GPG_KEY=A035C8C1", "PATH=/bin"])).toEqual([]); + }); + + test("flags the removed gateway token file name", () => { expect( scan([ - "GPG_KEY=A035C8C1", "COMPASS_GATEWAY_TOKEN_FILE=/run/token", - "PATH=/bin", + "COMPASS_GATEWAY_TOKEN_FILE_2=/x", ]), - ).toEqual([]); + ).toEqual(["COMPASS_GATEWAY_TOKEN_FILE", "COMPASS_GATEWAY_TOKEN_FILE_2"]); }); test("still flags a real secret name", () => { @@ -250,7 +254,7 @@ describe("secretConfigViolations", () => { ]); }); - test("allows only an exact allowlist match", () => { + test("does not allowlist credential-shaped names", () => { expect(scan(["GPG_KEY_X=1", "COMPASS_GATEWAY_TOKEN_FILE_2=/x"])).toEqual([ "GPG_KEY_X", "COMPASS_GATEWAY_TOKEN_FILE_2", @@ -265,7 +269,7 @@ describe("secretConfigViolations", () => { test("applies the exact-match allowlist to label keys and values", () => { expect( scan([], { GPG_KEY: "x", note: "COMPASS_GATEWAY_TOKEN_FILE" }), - ).toEqual([]); + ).toEqual(["note"]); expect(scan([], { GPG_KEY_X: "x" })).toEqual(["GPG_KEY_X"]); }); @@ -278,7 +282,7 @@ describe("secretConfigViolations", () => { ).toEqual(["org.example.registry-token", "org.example.api.key"]); }); - test("passes every env name of the built gateway image", () => { + test("flags the removed token env if it is put back on the image", () => { const env = [ "LANG=C.UTF-8", "GPG_KEY=7169605F62C751356D054A26A821E680E5FA6305", @@ -295,11 +299,13 @@ describe("secretConfigViolations", () => { "PATH=/opt/bun/bin:/usr/local/cargo/bin:/usr/local/bin:/usr/bin:/bin", "PI_ROOT=/pi", "HOME=/tmp", - "COMPASS_GATEWAY_TOKEN_FILE=/run/compass/gateway.token", "COMPASS_GATEWAY_BIND=0.0.0.0:4000", "COMPASS_GATEWAY_DRAIN_MS=20000", ]; expect(scan(env, { "io.buildah.version": "1.43.2" })).toEqual([]); + expect( + scan([...env, "COMPASS_GATEWAY_TOKEN_FILE=/run/compass/gateway.token"]), + ).toEqual(["COMPASS_GATEWAY_TOKEN_FILE"]); }); test("flags a label whose value names a secret", () => { @@ -369,3 +375,61 @@ describe("smoke predicates", () => { expect(modelsListOk(200, '{"error":"unauthorized"}')).toBe(false); }); }); + +describe("fake Compass door", () => { + const options = { + serviceToken: "service-token", + agentToken: "good-agent-token", + }; + const makeRequest = (token: string, serviceToken = options.serviceToken) => + new Request( + "https://host.containers.internal/compass.v1.GatewayAuth/VerifyAgentToken", + { + method: "POST", + headers: { + "Content-Type": "application/json", + "Connect-Protocol-Version": "1", + Authorization: `Bearer ${serviceToken}`, + }, + body: JSON.stringify({ token }), + }, + ); + + test("returns the verified agent and owner", async () => { + const response = await fakeCompassDoorResponse( + makeRequest(options.agentToken), + options, + ); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + agentAccountId: "agent-smoke", + ownerUserId: "owner-smoke", + }); + }); + + test("rejects empty and incorrect agent tokens with the verifier contract", async () => { + for (const token of ["", "wrong-agent-token"]) { + const response = await fakeCompassDoorResponse( + makeRequest(token), + options, + ); + expect(response.status).toBe(401); + expect(await response.json()).toEqual({ + code: "unauthenticated", + message: "invalid agent token", + }); + } + }); + + test("rejects an invalid service bearer", async () => { + const response = await fakeCompassDoorResponse( + makeRequest(options.agentToken, "wrong-service-token"), + options, + ); + expect(response.status).toBe(401); + expect(await response.json()).toEqual({ + code: "unauthenticated", + message: "unauthenticated", + }); + }); +}); diff --git a/tools/gateway-image/core.ts b/tools/gateway-image/core.ts index 0916bb099..40071e924 100644 --- a/tools/gateway-image/core.ts +++ b/tools/gateway-image/core.ts @@ -278,12 +278,8 @@ export function blobPath(layoutDir: string, digest: string): string { return join(layoutDir, "blobs", "sha256", digest.slice("sha256:".length)); } -/** Benign names the gateway config ships: a public-key fingerprint the - * `python` base sets, and a path. Exempt on an exact match only. */ -export const SECRET_NAME_ALLOWLIST: readonly string[] = [ - "GPG_KEY", - "COMPASS_GATEWAY_TOKEN_FILE", -]; +/** Benign name shipped by the `python` base. Exempt on an exact match only. */ +export const SECRET_NAME_ALLOWLIST: readonly string[] = ["GPG_KEY"]; /** Credential-shaped name segments. The boundary spans `_`, `.` and `-` so it * also reads label keys (`org.example.registry-token`), and keeps `PAT` from @@ -368,7 +364,72 @@ export function tagDisposition( }; } -/** The gateway's `/healthz`: 200 with `{"ok": true}`. */ +const VERIFY_AGENT_TOKEN_PATH = "/compass.v1.GatewayAuth/VerifyAgentToken"; + +export interface FakeCompassDoorOptions { + readonly serviceToken: string; + readonly agentToken: string; +} + +/** The fake network door used by the gateway-image smoke. */ +export async function fakeCompassDoorResponse( + request: Request, + options: FakeCompassDoorOptions, +): Promise { + if (new URL(request.url).pathname !== VERIFY_AGENT_TOKEN_PATH) { + return new Response(null, { status: 404 }); + } + if (request.method !== "POST") return new Response(null, { status: 405 }); + if ( + request.headers.get("connect-protocol-version") !== "1" || + request.headers.get("content-type")?.split(";")[0]?.trim() !== + "application/json" + ) { + return new Response(null, { status: 415 }); + } + if ( + request.headers.get("authorization") !== `Bearer ${options.serviceToken}` + ) { + return Response.json( + { code: "unauthenticated", message: "unauthenticated" }, + { status: 401 }, + ); + } + + let body: unknown; + try { + body = await request.json(); + } catch { + return Response.json( + { code: "invalid_argument", message: "invalid JSON" }, + { status: 400 }, + ); + } + const token = + typeof body === "object" && + body !== null && + "token" in body && + typeof body.token === "string" + ? body.token + : undefined; + if (token === undefined) { + return Response.json( + { code: "invalid_argument", message: "token is required" }, + { status: 400 }, + ); + } + if (token !== options.agentToken) { + return Response.json( + { code: "unauthenticated", message: "invalid agent token" }, + { status: 401 }, + ); + } + return Response.json({ + agentAccountId: "agent-smoke", + ownerUserId: "owner-smoke", + }); +} + export function healthzOk(status: number, body: string): boolean { return status === 200 && jsonField(body, "ok") === true; } diff --git a/tools/gateway-image/smoke.ts b/tools/gateway-image/smoke.ts index 925a63368..2ac439217 100755 --- a/tools/gateway-image/smoke.ts +++ b/tools/gateway-image/smoke.ts @@ -2,10 +2,15 @@ import { spawnSync } from "node:child_process"; import { randomBytes, randomUUID } from "node:crypto"; -import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { EXIT, healthzOk, modelsListOk } from "./core.ts"; +import { + EXIT, + fakeCompassDoorResponse, + healthzOk, + modelsListOk, +} from "./core.ts"; const workspaceRoot = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const laneDir = dirname(fileURLToPath(import.meta.url)); @@ -14,18 +19,29 @@ const laneDir = dirname(fileURLToPath(import.meta.url)); const COMMAND_TIMEOUT_MS = 300_000; type CommandResult = { status: number | null; stdout: string; stderr: string }; +type SmokeServer = { + port: number | undefined; + stop(closeActiveConnections?: boolean): void; +}; function failUsage(): never { console.error( - "usage: smoke.ts [--out ] [--fault no-broker-env|wrong-token]", + "usage: smoke.ts [--out ] [--fault no-server-env|wrong-token]", ); process.exit(EXIT.usage); } /** A deliberate fault proves the smoke can fail: each must exit 7. */ -type Fault = "no-broker-env" | "wrong-token"; +type Fault = "no-server-env" | "wrong-token"; function parseArgs(argv: readonly string[]): { out: string; fault?: Fault } { + if (argv.length === 1 && argv[0] === "--help") { + console.log( + "usage: smoke.ts [--out ] [--fault no-server-env|wrong-token]", + ); + process.exit(0); + } + const values = new Map(); for (let index = 0; index < argv.length; index += 2) { const name = argv[index]; @@ -43,7 +59,7 @@ function parseArgs(argv: readonly string[]): { out: string; fault?: Fault } { const fault = values.get("--fault"); if ( fault !== undefined && - fault !== "no-broker-env" && + fault !== "no-server-env" && fault !== "wrong-token" ) { failUsage(); @@ -57,11 +73,11 @@ function parseArgs(argv: readonly string[]): { out: string; fault?: Fault } { function run( command: string, args: readonly string[], - options: { env?: NodeJS.ProcessEnv; timeoutMs?: number } = {}, + options: { env?: NodeJS.ProcessEnv; timeoutMs?: number; cwd?: string } = {}, ): CommandResult { // A stalled podman client must not hang the smoke past its budget. const result = spawnSync(command, [...args], { - cwd: workspaceRoot, + cwd: options.cwd ?? workspaceRoot, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env: options.env, @@ -114,12 +130,15 @@ const { out: outDir, fault } = parseArgs(process.argv.slice(2)); const suffix = randomUUID().slice(0, 8); const smokeDir = join(outDir, `smoke-${suffix}`); const tarPath = join(outDir, `smoke-${suffix}.tar`); -const tokenPath = join(smokeDir, "gateway.token"); +const serviceTokenPath = join(smokeDir, "service.token"); +const certPath = join(smokeDir, "server-ca.crt"); +const keyPath = join(smokeDir, "server-ca.key"); const image = `compass-gateway-smoke:${suffix}`; -const network = `gw-smoke-${suffix}`; -const broker = `gw-broker-${suffix}`; const gateway = `gw-gateway-${suffix}`; +const serviceToken = randomBytes(32).toString("hex"); +const agentToken = randomBytes(32).toString("hex"); let failed = false; +let activeFakeServer: SmokeServer | undefined; function check(name: string, passed: boolean): void { console.error(`gateway-image smoke: ${passed ? "PASS" : "FAIL"} ${name}`); @@ -128,6 +147,48 @@ function check(name: string, passed: boolean): void { try { mkdirSync(outDir, { recursive: true }); + mkdirSync(smokeDir, { recursive: true }); + writeFileSync(serviceTokenPath, `${serviceToken}\n`, { mode: 0o600 }); + const certResult = run( + "go", + [ + "run", + "./cmd/compass-gen-cert", + "--cert-out", + certPath, + "--key-out", + keyPath, + "--hosts", + "host.containers.internal,127.0.0.1", + ], + { cwd: join(workspaceRoot, "go") }, + ); + if (certResult.status !== 0) { + throw new Error( + `compass-gen-cert failed (exit ${certResult.status ?? 1}): ${certResult.stderr.trim()}`, + ); + } + if (certResult.stderr !== "") console.error(certResult.stderr.trimEnd()); + + const fakeServer = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + tls: { cert: readFileSync(certPath), key: readFileSync(keyPath) }, + // The wrong-token fault makes the door reject the mounted service token. + fetch: (request) => + fakeCompassDoorResponse(request, { + serviceToken: + fault === "wrong-token" + ? randomBytes(32).toString("hex") + : serviceToken, + agentToken, + }), + }); + const fakeServerPort = fakeServer.port; + if (fakeServerPort === undefined) + throw new Error("fake Compass Server did not bind a port"); + activeFakeServer = fakeServer; + const copied = requireCommand("skopeo", [ "copy", `oci:${join(outDir, "oci")}`, @@ -135,83 +196,62 @@ try { ]); if (copied.stderr !== "") console.error(copied.stderr.trimEnd()); requireCommand("podman", ["load", "-i", tarPath]); - requireCommand("podman", ["network", "create", network]); - requireCommand("podman", [ + + const gatewayRunArgs = [ "run", "-d", "--name", - broker, + gateway, "--network", - network, - "--entrypoint", - "/usr/local/bin/omp", + "pasta:--map-host-loopback,169.254.1.2", + "-p", + "127.0.0.1::4000", + "-v", + `${serviceTokenPath}:/run/compass/service.token:ro,Z`, + "-v", + `${certPath}:/run/compass/server-ca.crt:ro,z`, + "-e", + "COMPASS_SERVICE_TOKEN_FILE=/run/compass/service.token", + "-e", + `COMPASS_SERVER_CA_FILE=/run/compass/server-ca.crt`, + "-e", + `COMPASS_GATEWAY_BIND=0.0.0.0:4000`, + "-e", + "COMPASS_GATEWAY_DRAIN_MS=20000", + ...(fault === "no-server-env" + ? [] + : [ + "-e", + `COMPASS_SERVER_URL=https://host.containers.internal:${fakeServerPort}`, + ]), image, - "auth-broker", - "serve", - "--bind=0.0.0.0:8765", - ]); - - const brokerReady = await waitFor(async (remainingMs) => { - const probe = run( - "podman", - [ - "exec", - broker, - "bun", - "-e", - "fetch('http://127.0.0.1:8765/v1/healthz', { signal: AbortSignal.timeout(2000) }).then(r => process.exit(r.status === 200 ? 0 : 1)).catch(() => process.exit(1))", - ], - { timeoutMs: Math.max(remainingMs, 1) }, + ]; + const gatewayRun = run("podman", gatewayRunArgs); + if (gatewayRun.status !== 0) { + throw new Error( + `podman run gateway failed (exit ${gatewayRun.status ?? 1}): ${gatewayRun.stderr.trim()}`, ); - return probe.status === 0; - }, 30_000); - if (!brokerReady) { - throw new Error("broker health check timed out after 30 seconds"); } - const brokerToken = requireCommand("podman", [ - "exec", - broker, - "cat", - "/tmp/.omp/auth-broker.token", - ]).stdout.trim(); - if (brokerToken === "") throw new Error("broker returned an empty token"); - mkdirSync(smokeDir, { recursive: true }); - writeFileSync(tokenPath, randomBytes(32).toString("hex"), { mode: 0o600 }); - const gatewayToken = - fault === "wrong-token" - ? randomBytes(32).toString("hex") - : (await Bun.file(tokenPath).text()).trim(); - const gatewayRun = run( - "podman", - [ - "run", - "-d", - "--name", + if (fault === "no-server-env") { + const waited = run("podman", ["wait", gateway], { timeoutMs: 30_000 }); + const state = run("podman", [ + "inspect", + "--format", + "{{.State.ExitCode}} {{.State.Status}}", gateway, - "--network", - network, - "-p", - "127.0.0.1::4000", - "-v", - `${tokenPath}:/run/compass/gateway.token:ro,Z`, - ...(fault === "no-broker-env" - ? [] - : [ - "-e", - `OMP_AUTH_BROKER_URL=http://gw-broker-${suffix}:8765`, - "-e", - "OMP_AUTH_BROKER_TOKEN", - ]), - image, - ], - { env: { ...process.env, OMP_AUTH_BROKER_TOKEN: brokerToken } }, - ); - if (gatewayRun.status !== 0) { - throw new Error( - `podman run gateway failed (exit ${gatewayRun.status ?? 1}): ${gatewayRun.stderr.trim()}`, + ]); + const exitedBeforeBind = + waited.status === 0 && + state.status === 0 && + /^[1-9][0-9]* exited$/.test(state.stdout.trim()); + check( + "unset COMPASS_SERVER_URL exits non-zero before the gateway binds", + exitedBeforeBind, ); + throw new Error("fault --fault no-server-env verified the pre-bind exit"); } + const portText = requireCommand("podman", [ "port", gateway, @@ -231,28 +271,29 @@ try { }, 30_000); check("/healthz returns ok within 30 seconds", healthy); - let unauthorized = false; + let authorized = false; try { const response = await fetch(`${baseUrl}/v1/models`, { + headers: { Authorization: `Bearer ${agentToken}` }, signal: AbortSignal.timeout(5000), }); - unauthorized = response.status === 401; + authorized = modelsListOk(response.status, await response.text()); } catch { - unauthorized = false; + authorized = false; } - check("/v1/models rejects missing authorization", unauthorized); + check("/v1/models returns a model list for a valid agent token", authorized); - let authorized = false; + let unauthorized = false; try { const response = await fetch(`${baseUrl}/v1/models`, { - headers: { Authorization: `Bearer ${gatewayToken}` }, + headers: { Authorization: `Bearer ${randomBytes(32).toString("hex")}` }, signal: AbortSignal.timeout(5000), }); - authorized = modelsListOk(response.status, await response.text()); + unauthorized = response.status === 401; } catch { - authorized = false; + unauthorized = false; } - check("/v1/models accepts the mounted gateway token", authorized); + check("/v1/models rejects a bad agent token with 401", unauthorized); const stopped = run("podman", ["stop", "-t", "25", gateway]); const exitCode = @@ -271,18 +312,17 @@ try { `gateway-image smoke: FAIL ${error instanceof Error ? error.message : String(error)}`, ); } finally { + activeFakeServer?.stop(true); if (failed) { - for (const name of [broker, gateway]) { - const logs = run("podman", ["logs", name]); - if (logs.status !== 0 && logs.stderr !== "") continue; - console.error(`gateway-image smoke: podman logs ${name}`); + const logs = run("podman", ["logs", gateway]); + if (logs.status === 0 || logs.stderr === "") { + console.error(`gateway-image smoke: podman logs ${gateway}`); if (logs.stdout !== "") console.error(logs.stdout.trimEnd()); if (logs.stderr !== "") console.error(logs.stderr.trimEnd()); } } for (const args of [ - ["rm", "-f", broker, gateway], - ["network", "rm", network], + ["rm", "-f", gateway], ["rmi", image], ]) { const result = run("podman", args);