test(cache): cover a remote cache entry expiring between fetch and download

crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml

@@ -352,6 +352,61 @@ steps = [
   ],
 ]
 
+[[e2e]]
+name = "expired_after_fetch"
+cfg = 'not(target_env = "musl")'
+ignore = true
+steps = [
+  [
+    "remote-cache-server",
+    "start",
+  ],
+  { argv = [
+    "remote-cache-server",
+    "run",
+    "--github-actions",
+    "vt",
+    "run",
+    "build",
+  ], envs = [
+    [
+      "VP_REMOTE_CACHE",
+      "read-write",
+    ],
+    [
+      "VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS",
+      "1",
+    ],
+  ] },
+  [
+    "vt",
+    "cache",
+    "clean",
+  ],
+  { argv = [
+    "remote-cache-server",
+    "expire-entry",
+    "--before-download",
+    "1",
+  ], comment = "The entry expires after the next fetch returns it, before its archive is downloaded." },
+  { argv = [
+    "remote-cache-server",
+    "run",
+    "vt",
+    "run",
+    "build",
+  ], comment = "The fetch finds the entry, but its archive is gone, so the task reruns." },
+  { argv = [
+    "vt",
+    "run",
+    "--last-details",
+  ], comment = "The details include the underlying error." },
+  [
+    "remote-cache-server",
+    "stop",
+  ],
+]
+
 [[e2e]]
 name = "restore_failure"
 cfg = 'not(target_env = "musl")'

crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/expired_after_fetch.md

@@ -0,0 +1,65 @@
+# expired_after_fetch
+
+## `remote-cache-server start`
+
+```
+```
+
+## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build`
+
+```
+$ vtt write-file dist/output.txt built
+
+[remote-cache] POST /fetch 404
+[remote-cache] POST /store 200
+```
+
+## `vt cache clean`
+
+```
+```
+
+## `remote-cache-server expire-entry --before-download 1`
+
+The entry expires after the next fetch returns it, before its archive is downloaded.
+
+```
+```
+
+## `remote-cache-server run vt run build`
+
+The fetch finds the entry, but its archive is gone, so the task reruns.
+
+```
+$ vtt write-file dist/output.txt built ○ cache miss: remote cache download failed, executing
+
+[remote-cache] POST /fetch 200 exact [1]
+[remote-cache] GET /blob/1 404
+```
+
+## `vt run --last-details`
+
+The details include the underlying error.
+
+```
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+    Vite+ Task Runner • Execution Summary
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+Statistics:   1 task • 0 cache hits • 1 cache miss
+Performance:  0% cache hit rate
+
+Task Details:
+────────────────────────────────────────────────
+  [1] remote-cache#build: $ vtt write-file dist/output.txt built ✓
+      → Cache miss: remote cache download failed
+        ↳ HTTP status 404
+        ↳ Not found
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+```
+
+## `remote-cache-server stop`
+
+```
+```

packages/tools/README.md

@@ -15,7 +15,7 @@ An E2E case starts its own backend in its first step and stops it in its last, s
 - `remote-cache-server start [--entries-per-key N]` starts the backend in the background on free loopback ports and returns once it's ready. The namespace keeps the newest `N` entries of each key, or the Worker's default of 8. The backend runs in its own session and doesn't use the terminal, so the step can finish and Ctrl-C in later steps doesn't reach it. It writes its endpoint, `http://127.0.0.1:<port>/projects/test`, to `remote-cache/server.json`, and its output to `remote-cache/server.log`.
 - `remote-cache-server run [--github-actions] COMMAND [ARGS...]` runs the command with `VP_REMOTE_CACHE_URL` set to the endpoint. The fixed base path gives the endpoint a namespace path. With `--github-actions`, the command runs as if in a GitHub Actions job with `id-token: write`: `ACTIONS_ID_TOKEN_REQUEST_URL` points to the backend's stand-in for GitHub's token service, and `ACTIONS_ID_TOKEN_REQUEST_TOKEN` defaults to `main-push`. The command inherits stdio and handles Ctrl-C, which `run` ignores. `run` exits with the command's exit code.
 - `remote-cache-server corrupt-blob NUMBER` overwrites a stored blob, numbered as in the request lines, with other bytes.
-- `remote-cache-server expire-entry NUMBER` makes the entry whose blob has that number expire now, as if its retention had passed. Nothing removes it, as before the Worker's scheduled cleanup.
+- `remote-cache-server expire-entry [--before-download] NUMBER` makes the entry whose blob has that number expire now, as if its retention had passed. Nothing removes it, as before the Worker's scheduled cleanup. With `--before-download`, the entry expires when its blob is next requested instead, before the tap forwards the request, as if its retention passed between a fetch and the download.
 - `remote-cache-server stop` stops the backend. It fails if the backend answered a request with a 5xx status or couldn't be reached.
 
 A backend whose case directory disappears stops by itself, as does one that has had no requests for ten minutes, e.g. because its case timed out before the stop step.

packages/tools/src/remote-cache/cli.ts

@@ -11,7 +11,7 @@ import { defaultRequestToken } from './github.ts';
 import { logFile, serverFile, stateDirectory, type ServerInfo } from './state.ts';
 
 const usage =
-  'Usage: remote-cache-server start [--entries-per-key N] | run [--github-actions] COMMAND [ARGS...] | corrupt-blob NUMBER | expire-entry NUMBER | stop';
+  'Usage: remote-cache-server start [--entries-per-key N] | run [--github-actions] COMMAND [ARGS...] | corrupt-blob NUMBER | expire-entry [--before-download] NUMBER | stop';
 const directory = resolve(stateDirectory);
 
 function fail(message: string): never {
@@ -128,7 +128,9 @@ async function stop(): Promise<void> {
 const [subcommand, ...args] = process.argv.slice(2);
 if (subcommand === 'start') await start(args);
 else if (subcommand === 'run') await run(args);
-else if (subcommand === 'corrupt-blob' || subcommand === 'expire-entry') {
+else if (subcommand === 'expire-entry' && args[0] === '--before-download') {
+  await changeBlob('expire-before-download', args.slice(1));
+} else if (subcommand === 'corrupt-blob' || subcommand === 'expire-entry') {
   await changeBlob(subcommand, args);
 } else if (subcommand === 'stop') await stop();
 else fail(usage);

packages/tools/src/remote-cache/daemon.ts

@@ -7,6 +7,7 @@ import {
   type IncomingHttpHeaders,
   type IncomingMessage,
   type Server,
+  type ServerResponse,
 } from 'node:http';
 import type { AddressInfo } from 'node:net';
 import { startBackend } from './backend.ts';
@@ -42,6 +43,8 @@ const requests: string[] = [];
 const anomalies: string[] = [];
 /** The ID of each stored blob, in upload order. Blob number `n` is at `n - 1`. */
 const blobs: string[] = [];
+/** The blobs whose entries expire when the blob is next requested. */
+const expireBeforeDownload = new Set<string>();
 let lastUse = Date.now();
 let stopping = false;
 
@@ -90,6 +93,29 @@ const tap = createServer((request, response) => {
   const path = new URL(request.url ?? '/', 'http://localhost').pathname;
   const route = path.startsWith(basePath) ? path.slice(basePath.length) : path;
   const label = `${request.method} ${numbered(route)}`;
+  const id = /^\/blob\/(.+)$/.exec(route)?.[1];
+  if (id !== undefined && expireBeforeDownload.delete(id)) {
+    // The client sent this request after the fetch's response, so the entry
+    // expires between the two.
+    backend.expireEntry(id).then(
+      () => pass(request, response, route, label),
+      (error: unknown) => {
+        anomalies.push(`${label} failed to expire its entry: ${String(error)}`);
+        response.writeHead(502).end();
+      },
+    );
+  } else {
+    pass(request, response, route, label);
+  }
+});
+
+/** Forward a request to the backend and its response back, recording the response's line. */
+function pass(
+  request: IncomingMessage,
+  response: ServerResponse,
+  route: string,
+  label: string,
+): void {
   let clientGone = false;
   let backendReply: IncomingMessage | undefined;
   const upstream = forward(
@@ -134,7 +160,7 @@ const tap = createServer((request, response) => {
     upstream.destroy();
   });
   request.pipe(upstream);
-});
+}
 
 const control = createServer((request, response) => {
   lastUse = Date.now();
@@ -156,7 +182,8 @@ const control = createServer((request, response) => {
     else reply({ count: 1, value });
     return;
   }
-  const [, change, number] = /^\/(corrupt-blob|expire-entry)\/(\d+)$/.exec(request.url ?? '') ?? [];
+  const [, change, number] =
+    /^\/(corrupt-blob|expire-entry|expire-before-download)\/(\d+)$/.exec(request.url ?? '') ?? [];
   if (request.method === 'POST' && request.url === '/take') {
     reply(requests.splice(0));
   } else if (request.method === 'POST' && number !== undefined) {
@@ -165,6 +192,11 @@ const control = createServer((request, response) => {
       response.writeHead(404).end();
       return;
     }
+    if (change === 'expire-before-download') {
+      expireBeforeDownload.add(id);
+      reply(null);
+      return;
+    }
     const changed =
       change === 'corrupt-blob'
         ? backend.writeBlob(id, Buffer.from('corrupt'))