fix(gemini): tighten reactive CachedContent NOT_FOUND matcher

packages/test/src/test/ai-provider/Gemini_CachedContentFallback.test.ts

@@ -0,0 +1,160 @@
+/**
+ * @license
+ * Copyright 2026 Steven Roussey <sroussey@gmail.com>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { _testOnly } from "@workglow/google-gemini/ai";
+import { describe, expect, it, vi } from "vitest";
+
+const { isGeminiCachedContentNotFoundError, generateGeminiStreamWithCacheFallback } = _testOnly;
+
+describe("isGeminiCachedContentNotFoundError", () => {
+  describe("true — scoped CachedContent NOT_FOUND", () => {
+    it.each([
+      [
+        "status 404 with cachedContents/… mention",
+        { status: 404, message: "CachedContent 'cachedContents/abc' not found" },
+      ],
+      [
+        "code NOT_FOUND with cachedContent mention",
+        { code: "NOT_FOUND", message: "cachedContent handle expired" },
+      ],
+      [
+        "nested error.status NOT_FOUND with resource path",
+        { error: { status: "NOT_FOUND" }, message: "cachedContents/xyz does not exist" },
+      ],
+      [
+        "status 404 with resource wording",
+        { status: 404, message: "The requested cachedContent resource was not found." },
+      ],
+      [
+        "nested error.code 404 with cachedContents/ path",
+        { error: { code: 404 }, message: "resource cachedContents/abc-123 not found" },
+      ],
+      [
+        "status NOT_FOUND string with cachedContent",
+        { status: "NOT_FOUND", message: "cachedContent NOT_FOUND" },
+      ],
+    ])("returns true for %s", (_label, err) => {
+      expect(isGeminiCachedContentNotFoundError(err)).toBe(true);
+    });
+  });
+
+  describe("false — regression cases the old matcher over-triggered on", () => {
+    it.each([
+      [
+        "model-not-found (status 400)",
+        {
+          status: 400,
+          message: "The requested model 'gemini-x-nope' was not found or is not available.",
+        },
+      ],
+      ["file-not-found part", { status: 400, message: "File 'files/abc' was not found." }],
+      [
+        "tokenizer / function-declaration not-found",
+        { message: "Function name 'foo' not found in declarations." },
+      ],
+      [
+        "404 on unrelated URL (no cache mention)",
+        {
+          status: 404,
+          message: "The requested URL /v1beta/models/gemini-2.5-pro:generateContent was not found.",
+        },
+      ],
+      ["bare NOT_FOUND with no scope", new Error("NOT_FOUND")],
+      ["null", null],
+      ["undefined", undefined],
+      ["plain string", "not found"],
+      ["message-only 'not found' with no cache mention", { message: "resource not found" }],
+    ])("returns false for %s", (_label, err) => {
+      expect(isGeminiCachedContentNotFoundError(err)).toBe(false);
+    });
+  });
+});
+
+describe("generateGeminiStreamWithCacheFallback", () => {
+  it("evicts + retries once on a scoped CachedContent NOT_FOUND", async () => {
+    // deleteGeminiCachedContent is a no-op when the store has no entry (we
+    // never seed one), so no need to mock the cache-store seam here — the
+    // test's contract is on the buildRequest / runStream call pattern.
+    const runStream = vi
+      .fn<[Record<string, unknown>], Promise<string>>()
+      .mockImplementationOnce(async () => {
+        throw { status: 404, message: "cachedContents/chk-A not found" };
+      })
+      .mockImplementationOnce(async () => "retry-ok");
+    const buildRequest = vi
+      .fn<[boolean], Record<string, unknown>>()
+      .mockImplementation((useCache) => ({ useCache }));
+
+    const result = await generateGeminiStreamWithCacheFallback({
+      useCachedContent: true,
+      checkpointId: "chk-A",
+      buildRequest,
+      runStream,
+    });
+
+    expect(result).toBe("retry-ok");
+    expect(buildRequest).toHaveBeenCalledTimes(2);
+    expect(buildRequest).toHaveBeenNthCalledWith(1, true);
+    expect(buildRequest).toHaveBeenNthCalledWith(2, false);
+    expect(runStream).toHaveBeenCalledTimes(2);
+  });
+
+  it("does NOT evict or retry on a non-CachedContent NOT_FOUND (model-not-found)", async () => {
+    const runStream = vi.fn(async () => {
+      throw { status: 400, message: "The requested model 'gemini-x-nope' was not found." };
+    });
+    const buildRequest = vi.fn((useCache: boolean) => ({ useCache }));
+
+    await expect(
+      generateGeminiStreamWithCacheFallback({
+        useCachedContent: true,
+        checkpointId: "chk-A",
+        buildRequest,
+        runStream,
+      })
+    ).rejects.toMatchObject({ status: 400 });
+
+    expect(buildRequest).toHaveBeenCalledTimes(1);
+    expect(runStream).toHaveBeenCalledTimes(1);
+  });
+
+  it("does NOT evict or retry when useCachedContent is false, even on a cache-scoped NOT_FOUND", async () => {
+    const runStream = vi.fn(async () => {
+      throw { status: 404, message: "cachedContents/chk-A not found" };
+    });
+    const buildRequest = vi.fn((useCache: boolean) => ({ useCache }));
+
+    await expect(
+      generateGeminiStreamWithCacheFallback({
+        useCachedContent: false,
+        checkpointId: "chk-A",
+        buildRequest,
+        runStream,
+      })
+    ).rejects.toMatchObject({ status: 404 });
+
+    expect(buildRequest).toHaveBeenCalledTimes(1);
+    expect(runStream).toHaveBeenCalledTimes(1);
+  });
+
+  it("does NOT evict or retry when no checkpointId is supplied", async () => {
+    const runStream = vi.fn(async () => {
+      throw { status: 404, message: "cachedContents/mystery not found" };
+    });
+    const buildRequest = vi.fn((useCache: boolean) => ({ useCache }));
+
+    await expect(
+      generateGeminiStreamWithCacheFallback({
+        useCachedContent: true,
+        checkpointId: undefined,
+        buildRequest,
+        runStream,
+      })
+    ).rejects.toMatchObject({ status: 404 });
+
+    expect(runStream).toHaveBeenCalledTimes(1);
+  });
+});

providers/google-gemini/src/ai/common/Gemini_CachedContentFallback.ts

@@ -8,19 +8,64 @@ import { getLogger } from "@workglow/util/worker";
 import { deleteGeminiCachedContent } from "./Gemini_CacheStore";
 
 /**
- * Reactive NOT_FOUND signature. A CachedContent that TTL-expires (or was
- * disposed elsewhere) between the consumer's proactive check and the API call
- * surfaces as a 404 / `NOT_FOUND` / "not found" from `generateContentStream`.
- * When the caller was referencing that entry, the fallback is the same as the
- * stale path: evict, rebuild the request without `cachedContent`, and retry
- * once. Any other error propagates untouched.
+ * Match ONLY the reactive "the referenced CachedContent no longer exists"
+ * signal. A cachedContent that TTL-expires or is disposed elsewhere between
+ * the consumer's proactive check and the API call surfaces as a NOT_FOUND
+ * from `generateContentStream`; when the pending request references that
+ * entry, the fallback is to evict and retry inline (see
+ * {@link generateGeminiStreamWithCacheFallback}). ANY other error — a model
+ * misconfiguration ("model not found"), a missing File part, a tokenizer /
+ * function-declaration message, a 404 on an unrelated URL — must NOT trigger
+ * that fallback, or the caller's still-valid CachedContent entry will be
+ * destroyed (and every OTHER consumer of the same checkpoint will silently
+ * pay the full re-encode cost on their next call) while the request retries
+ * doomed to fail the same way.
+ *
+ * A hit therefore requires BOTH signals:
+ *  1. a structured NOT_FOUND (`status === 404` OR `status === "NOT_FOUND"` OR
+ *     `code === 404` OR `code === "NOT_FOUND"`, on the top-level error or the
+ *     nested `.error` the Google GenAI SDK sometimes wraps GAPI errors in),
+ *  2. AND a scoped mention of `cachedContent` (or a `cachedContents/…`
+ *     resource name) in the message.
+ * If only a message signal is available, the pattern must additionally scope
+ * the NOT_FOUND wording to a nearby `cachedContent` mention — a bare
+ * "NOT_FOUND" is not enough.
  */
 export function isGeminiCachedContentNotFoundError(err: unknown): boolean {
-  const anyErr = err as { status?: unknown; code?: unknown; message?: unknown };
-  if (anyErr?.status === 404) return true;
-  if (anyErr?.code === "NOT_FOUND") return true;
-  const message = String(anyErr?.message ?? err ?? "");
-  return /NOT_FOUND|not.*found/i.test(message);
+  if (err === null || err === undefined) return false;
+
+  const anyErr = err as {
+    status?: unknown;
+    code?: unknown;
+    message?: unknown;
+    error?: { status?: unknown; code?: unknown } | undefined;
+  };
+  const nested = anyErr.error;
+
+  const hasStructuredNotFound =
+    anyErr.status === 404 ||
+    anyErr.status === "NOT_FOUND" ||
+    anyErr.code === 404 ||
+    anyErr.code === "NOT_FOUND" ||
+    nested?.status === 404 ||
+    nested?.status === "NOT_FOUND" ||
+    nested?.code === 404 ||
+    nested?.code === "NOT_FOUND";
+
+  const message = String(anyErr.message ?? err ?? "");
+  // `cached[_ ]?content` also matches `cachedContent` inside a resource path
+  // like `cachedContents/abc-123`, so the resource-form is covered by this
+  // single pattern.
+  const messageMentionsCache = /cached[_ ]?content/i.test(message);
+
+  if (hasStructuredNotFound) return messageMentionsCache;
+
+  // No structured signal: require a scoped `cachedContent … NOT_FOUND` pattern
+  // in the message. A bare "NOT_FOUND" (e.g. a top-level Error("NOT_FOUND"))
+  // is deliberately NOT enough — it fires on too many unrelated code paths.
+  return /cached[_ ]?content(?:s\/[\w-]+)?[^.\n]{0,120}(?:NOT_FOUND|not\s+found|does\s+not\s+exist)/i.test(
+    message
+  );
 }
 
 interface ExecuteWithFallbackParams<TStream> {
@@ -37,9 +82,13 @@ interface ExecuteWithFallbackParams<TStream> {
 /**
  * Runs `generateContentStream` with the reactive NOT_FOUND fallback: if the
  * initial request references a `cachedContent` handle and the API returns a
- * NOT_FOUND, evict the store entry (locally + best-effort server delete),
- * rebuild the request inline, and retry **once**. All other errors — including
- * a NOT_FOUND on a request that was never using cached content — propagate.
+ * NOT_FOUND SCOPED TO CACHEDCONTENT, evict the store entry (locally + best-
+ * effort server delete), rebuild the request inline, and retry **once**. All
+ * other errors — including a NOT_FOUND for a model / file / tokenizer /
+ * unrelated URL, or a NOT_FOUND on a request that was never using cached
+ * content — propagate untouched; if a debug-level logger is installed a
+ * one-liner records that the propagating error was NOT treated as a cache
+ * miss, so a genuine cache regression stays diagnosable.
  *
  * The proactive stale check lives in the callers (they own the request-shape
  * choice and want to log a different debug line); this helper covers only the
@@ -53,7 +102,13 @@ export async function generateGeminiStreamWithCacheFallback<TStream>(
   try {
     return await runStream(request);
   } catch (err) {
-    if (!useCachedContent || !checkpointId || !isGeminiCachedContentNotFoundError(err)) {
+    if (!useCachedContent || !checkpointId) throw err;
+    if (!isGeminiCachedContentNotFoundError(err)) {
+      const anyErr = err as { status?: unknown; code?: unknown };
+      getLogger().debug(
+        "Gemini stream failed on cached-content request; not a CachedContent NOT_FOUND, propagating",
+        { status: anyErr?.status, code: anyErr?.code }
+      );
       throw err;
     }
     getLogger().debug("Gemini cachedContent NOT_FOUND; replaying inline");

providers/google-gemini/src/ai/index.ts

@@ -13,6 +13,10 @@ export * from "./common/Gemini_ModelSearch";
 export * from "./registerGemini";
 
 import { GEMINI_RUN_FN_SPECS } from "./common/Gemini_Capabilities";
+import {
+  generateGeminiStreamWithCacheFallback,
+  isGeminiCachedContentNotFoundError,
+} from "./common/Gemini_CachedContentFallback";
 import { _testOnly as clientTestOnly } from "./common/Gemini_Client";
 import {
   _cacheStoreTestOnly,
@@ -41,4 +45,6 @@ export const _testOnly = {
   setGeminiCachedContent,
   getGeminiCachedContent,
   cacheStoreTestOnly: _cacheStoreTestOnly,
+  isGeminiCachedContentNotFoundError,
+  generateGeminiStreamWithCacheFallback,
 } as const;