diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml index 8aaf87afb..204402d1c 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml @@ -352,6 +352,61 @@ steps = [ ], ] +[[e2e]] +name = "expired_after_fetch" +cfg = 'not(target_env = "musl")' +ignore = true +steps = [ + [ + "remote-cache-server", + "start", + ], + { argv = [ + "remote-cache-server", + "run", + "--github-actions", + "vt", + "run", + "build", + ], envs = [ + [ + "VP_REMOTE_CACHE", + "read-write", + ], + [ + "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS", + "1", + ], + ] }, + [ + "vt", + "cache", + "clean", + ], + { argv = [ + "remote-cache-server", + "expire-entry", + "--before-download", + "1", + ], comment = "The entry expires after the next fetch returns it, before its archive is downloaded." }, + { argv = [ + "remote-cache-server", + "run", + "vt", + "run", + "build", + ], comment = "The fetch finds the entry, but its archive is gone, so the task reruns." }, + { argv = [ + "vt", + "run", + "--last-details", + ], comment = "The details include the underlying error." }, + [ + "remote-cache-server", + "stop", + ], +] + [[e2e]] name = "restore_failure" cfg = 'not(target_env = "musl")' diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/expired_after_fetch.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/expired_after_fetch.md new file mode 100644 index 000000000..750cb81e4 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/expired_after_fetch.md @@ -0,0 +1,65 @@ +# expired_after_fetch + +## `remote-cache-server start` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` + +``` +$ vtt write-file dist/output.txt built + +[remote-cache] POST /fetch 404 +[remote-cache] POST /store 200 +``` + +## `vt cache clean` + +``` +``` + +## `remote-cache-server expire-entry --before-download 1` + +The entry expires after the next fetch returns it, before its archive is downloaded. + +``` +``` + +## `remote-cache-server run vt run build` + +The fetch finds the entry, but its archive is gone, so the task reruns. + +``` +$ vtt write-file dist/output.txt built ○ cache miss: remote cache download failed, executing + +[remote-cache] POST /fetch 200 exact [1] +[remote-cache] GET /blob/1 404 +``` + +## `vt run --last-details` + +The details include the underlying error. + +``` + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Vite+ Task Runner • Execution Summary +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Statistics: 1 task • 0 cache hits • 1 cache miss +Performance: 0% cache hit rate + +Task Details: +──────────────────────────────────────────────── + [1] remote-cache#build: $ vtt write-file dist/output.txt built ✓ + → Cache miss: remote cache download failed + ↳ HTTP status 404 + ↳ Not found +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +``` + +## `remote-cache-server stop` + +``` +``` diff --git a/packages/tools/README.md b/packages/tools/README.md index 31d3775f1..354a82bb3 100644 --- a/packages/tools/README.md +++ b/packages/tools/README.md @@ -15,7 +15,7 @@ An E2E case starts its own backend in its first step and stops it in its last, s - `remote-cache-server start [--entries-per-key N]` starts the backend in the background on free loopback ports and returns once it's ready. The namespace keeps the newest `N` entries of each key, or the Worker's default of 8. The backend runs in its own session and doesn't use the terminal, so the step can finish and Ctrl-C in later steps doesn't reach it. It writes its endpoint, `http://127.0.0.1:/projects/test`, to `remote-cache/server.json`, and its output to `remote-cache/server.log`. - `remote-cache-server run [--github-actions] COMMAND [ARGS...]` runs the command with `VP_REMOTE_CACHE_URL` set to the endpoint. The fixed base path gives the endpoint a namespace path. With `--github-actions`, the command runs as if in a GitHub Actions job with `id-token: write`: `ACTIONS_ID_TOKEN_REQUEST_URL` points to the backend's stand-in for GitHub's token service, and `ACTIONS_ID_TOKEN_REQUEST_TOKEN` defaults to `main-push`. The command inherits stdio and handles Ctrl-C, which `run` ignores. `run` exits with the command's exit code. - `remote-cache-server corrupt-blob NUMBER` overwrites a stored blob, numbered as in the request lines, with other bytes. -- `remote-cache-server expire-entry NUMBER` makes the entry whose blob has that number expire now, as if its retention had passed. Nothing removes it, as before the Worker's scheduled cleanup. +- `remote-cache-server expire-entry [--before-download] NUMBER` makes the entry whose blob has that number expire now, as if its retention had passed. Nothing removes it, as before the Worker's scheduled cleanup. With `--before-download`, the entry expires when its blob is next requested instead, before the tap forwards the request, as if its retention passed between a fetch and the download. - `remote-cache-server stop` stops the backend. It fails if the backend answered a request with a 5xx status or couldn't be reached. A backend whose case directory disappears stops by itself, as does one that has had no requests for ten minutes, e.g. because its case timed out before the stop step. diff --git a/packages/tools/src/remote-cache/cli.ts b/packages/tools/src/remote-cache/cli.ts index 1a8692c03..dcafd0a6e 100755 --- a/packages/tools/src/remote-cache/cli.ts +++ b/packages/tools/src/remote-cache/cli.ts @@ -11,7 +11,7 @@ import { defaultRequestToken } from './github.ts'; import { logFile, serverFile, stateDirectory, type ServerInfo } from './state.ts'; const usage = - 'Usage: remote-cache-server start [--entries-per-key N] | run [--github-actions] COMMAND [ARGS...] | corrupt-blob NUMBER | expire-entry NUMBER | stop'; + 'Usage: remote-cache-server start [--entries-per-key N] | run [--github-actions] COMMAND [ARGS...] | corrupt-blob NUMBER | expire-entry [--before-download] NUMBER | stop'; const directory = resolve(stateDirectory); function fail(message: string): never { @@ -128,7 +128,9 @@ async function stop(): Promise { const [subcommand, ...args] = process.argv.slice(2); if (subcommand === 'start') await start(args); else if (subcommand === 'run') await run(args); -else if (subcommand === 'corrupt-blob' || subcommand === 'expire-entry') { +else if (subcommand === 'expire-entry' && args[0] === '--before-download') { + await changeBlob('expire-before-download', args.slice(1)); +} else if (subcommand === 'corrupt-blob' || subcommand === 'expire-entry') { await changeBlob(subcommand, args); } else if (subcommand === 'stop') await stop(); else fail(usage); diff --git a/packages/tools/src/remote-cache/daemon.ts b/packages/tools/src/remote-cache/daemon.ts index 16d8bfe17..876c50fa5 100644 --- a/packages/tools/src/remote-cache/daemon.ts +++ b/packages/tools/src/remote-cache/daemon.ts @@ -7,6 +7,7 @@ import { type IncomingHttpHeaders, type IncomingMessage, type Server, + type ServerResponse, } from 'node:http'; import type { AddressInfo } from 'node:net'; import { startBackend } from './backend.ts'; @@ -42,6 +43,8 @@ const requests: string[] = []; const anomalies: string[] = []; /** The ID of each stored blob, in upload order. Blob number `n` is at `n - 1`. */ const blobs: string[] = []; +/** The blobs whose entries expire when the blob is next requested. */ +const expireBeforeDownload = new Set(); let lastUse = Date.now(); let stopping = false; @@ -90,6 +93,29 @@ const tap = createServer((request, response) => { const path = new URL(request.url ?? '/', 'http://localhost').pathname; const route = path.startsWith(basePath) ? path.slice(basePath.length) : path; const label = `${request.method} ${numbered(route)}`; + const id = /^\/blob\/(.+)$/.exec(route)?.[1]; + if (id !== undefined && expireBeforeDownload.delete(id)) { + // The client sent this request after the fetch's response, so the entry + // expires between the two. + backend.expireEntry(id).then( + () => pass(request, response, route, label), + (error: unknown) => { + anomalies.push(`${label} failed to expire its entry: ${String(error)}`); + response.writeHead(502).end(); + }, + ); + } else { + pass(request, response, route, label); + } +}); + +/** Forward a request to the backend and its response back, recording the response's line. */ +function pass( + request: IncomingMessage, + response: ServerResponse, + route: string, + label: string, +): void { let clientGone = false; let backendReply: IncomingMessage | undefined; const upstream = forward( @@ -134,7 +160,7 @@ const tap = createServer((request, response) => { upstream.destroy(); }); request.pipe(upstream); -}); +} const control = createServer((request, response) => { lastUse = Date.now(); @@ -156,7 +182,8 @@ const control = createServer((request, response) => { else reply({ count: 1, value }); return; } - const [, change, number] = /^\/(corrupt-blob|expire-entry)\/(\d+)$/.exec(request.url ?? '') ?? []; + const [, change, number] = + /^\/(corrupt-blob|expire-entry|expire-before-download)\/(\d+)$/.exec(request.url ?? '') ?? []; if (request.method === 'POST' && request.url === '/take') { reply(requests.splice(0)); } else if (request.method === 'POST' && number !== undefined) { @@ -165,6 +192,11 @@ const control = createServer((request, response) => { response.writeHead(404).end(); return; } + if (change === 'expire-before-download') { + expireBeforeDownload.add(id); + reply(null); + return; + } const changed = change === 'corrupt-blob' ? backend.writeBlob(id, Buffer.from('corrupt'))