The model listing answers from a cache and refreshes behind it

server/src/chat/routes.ts

@@ -411,25 +411,28 @@ function templateKwargsFor(model: string): Record<string, unknown> {
 }
 
 export async function chatRoutes(app: FastifyInstance): Promise<void> {
-  app.get('/api/chat/models', async (req, reply) => {
-    if (!gatewayConfigured() && !resolveUserKey(req.user?.id) && !sidecarConfigured()) {
-      return reply.send({ models: [], unreachableSessions: [], error: 'no gateway credential: set PW_API_KEY or authenticate the pw CLI' })
-    }
-    const eff0 = effectiveSettings()
-    const cred = resolveUserCred(req.user?.id)
-    // Settings asks with config=1: the deployment's own model choices (the
-    // vision model, the RAG default) have to be configurable even where
-    // chat requires each user to bring a key, so that listing falls back to
-    // the deployment credential instead of coming back empty.
-    const forConfig = String((req.query as { config?: string }).config ?? '') === '1'
-    if (!forConfig && eff0.requirePersonalKey && authEnabled() && !personalKeysDisabled() && !cred && !getSharedKeyStatus().active) {
-      return reply.send({ models: [], unreachableSessions: [], error: 'This deployment requires your own model credential: add your API key or platform token in Settings, Model access.' })
-    }
-    // The platform catalog and the model served on this node are listed
-    // independently. Either can be missing: the gateway credential may be
-    // absent on a deployment whose whole point is the private model beside
-    // it, and the serve may still be loading. Neither absence should empty
-    // the picker of the other.
+  // The expensive part of a listing is the gateway catalog plus one probe
+  // per provider family, and both were paid on every load: about a second
+  // here, several on a deployment whose gateway and providers are further
+  // away, and per user, since each credential probes on its own. The
+  // computed listing is now kept per credential. A load inside the fresh
+  // window is answered from it; an older one is answered from it too and
+  // refreshed in the background, so the next load is current; only the
+  // very first load per credential waits. ?refresh=1 is the user saying
+  // "look again": it drops the probes and recomputes before answering.
+  // Per-request decorations (a model whose last call failed) are applied
+  // after the cache, so they stay live.
+  type Listing = { models: any[]; impaired: { id: string; locked: boolean; unlock_url: string | null }[]; unreachableSessions: unknown[] }
+  const LISTING_FRESH_MS = 60_000
+  const LISTING_KEEP_MS = 30 * 60_000
+  const listings = new Map<string, { at: number; v: Listing }>()
+  const refreshing = new Map<string, Promise<Listing>>()
+  const listingKey = (cred: { key?: string | null; baseUrl?: string | null } | null, viewer: string) =>
+    `${viewer}|${(cred?.key ?? '').slice(-8)}|${cred?.baseUrl ?? ''}`
+
+  // Everything up to the cache: catalog, provider probes, availability marks.
+  class ListingReply extends Error { constructor(public payload: unknown) { super('listing reply') } }
+  async function computeListing(cred: ReturnType<typeof resolveUserCred>, req: any): Promise<Listing> {
     const sidecar = await listSidecarModels()
     let wire: any = {}
     try {
@@ -441,7 +444,7 @@ export async function chatRoutes(app: FastifyInstance): Promise<void> {
       const msg = String((e as Error).message ?? e)
       const unlockUrl = extractUnlockUrl(msg)
       if (cred?.baseUrl && (unlockUrl || /401|403|locked|unauthorized/i.test(msg))) {
-        return reply.send({
+        throw new ListingReply({
           models: sidecar, unreachableSessions: [],
           error: `Your provider key is locked or rejected by ${new URL(cred.baseUrl).host}.${unlockUrl ? ` Unlock it here and reload: ${unlockUrl}` : ''} The Settings, Model access page shows the same link and a Re-check.`,
         })
@@ -452,7 +455,7 @@ export async function chatRoutes(app: FastifyInstance): Promise<void> {
       const rejected = /\b(401|403)\b|unauthorized|forbidden/i.test(msg)
       if (rejected) {
         const st = cred && req.user ? getUserKeyStatus(req.user.id) : null
-        return reply.send({
+        throw new ListingReply({
           models: sidecar, unreachableSessions: [], credential: 'rejected',
           error: credentialRejectionMessage(cred ? 'personal' : 'deployment', st?.kind ?? null, st?.credExpiresAt ?? null),
         })
@@ -575,13 +578,58 @@ export async function chatRoutes(app: FastifyInstance): Promise<void> {
     // A model whose last call failed is still offered (the provider may
     // recover, or the key may be renewed), but the picker gets told, so
     // choosing it is informed rather than a surprise at reply time.
+    return { models, impaired, unreachableSessions: wire.unreachable_sessions ?? [] } as Listing
+  }
+
+  app.get('/api/chat/models', async (req, reply) => {
+    if (!gatewayConfigured() && !resolveUserKey(req.user?.id) && !sidecarConfigured()) {
+      return reply.send({ models: [], unreachableSessions: [], error: 'no gateway credential: set PW_API_KEY or authenticate the pw CLI' })
+    }
+    const eff0 = effectiveSettings()
+    const cred = resolveUserCred(req.user?.id)
+    // Settings asks with config=1: the deployment's own model choices (the
+    // vision model, the RAG default) have to be configurable even where
+    // chat requires each user to bring a key, so that listing falls back to
+    // the deployment credential instead of coming back empty.
+    const forConfig = String((req.query as { config?: string }).config ?? '') === '1'
+    if (!forConfig && eff0.requirePersonalKey && authEnabled() && !personalKeysDisabled() && !cred && !getSharedKeyStatus().active) {
+      return reply.send({ models: [], unreachableSessions: [], error: 'This deployment requires your own model credential: add your API key or platform token in Settings, Model access.' })
+    }
+    // The platform catalog and the model served on this node are listed
+    // independently. Either can be missing: the gateway credential may be
+    // absent on a deployment whose whole point is the private model beside
+    // it, and the serve may still be loading. Neither absence should empty
+    // the picker of the other.
+    const key = listingKey(cred, (req.user?.username ?? '').toLowerCase())
+    const wantFresh = String((req.query as { refresh?: string }).refresh ?? '') === '1'
+    if (wantFresh) { invalidateProviderProbes(); listings.delete(key) }
+    const compute = () => {
+      const p = computeListing(cred, req).then(v => { listings.set(key, { at: Date.now(), v }); return v }).finally(() => refreshing.delete(key))
+      refreshing.set(key, p)
+      return p
+    }
+    const hit = listings.get(key)
+    let listing: Listing
+    try {
+      if (hit && Date.now() - hit.at < LISTING_KEEP_MS) {
+        listing = hit.v
+        if (Date.now() - hit.at > LISTING_FRESH_MS && !refreshing.has(key)) compute().catch(() => { /* the next load retries */ })
+      } else {
+        listing = await (refreshing.get(key) ?? compute())
+      }
+    } catch (e) {
+      if (e instanceof ListingReply) return reply.send(e.payload)
+      throw e
+    }
+    let models = listing.models
+    const impaired = listing.impaired
     models = models.map((m: any) => {
       const f = modelFailure(String(m.id))
       return f ? { ...m, callable: false, last_error: { at: f.at, auth: f.auth, status: f.status } } : m
     })
     models.sort((a: any, b: any) =>
       Number(String(a.id).startsWith('org:')) - Number(String(b.id).startsWith('org:')))
-    return reply.send({ models, impaired, unreachableSessions: wire.unreachable_sessions ?? [] })
+    return reply.send({ models, impaired, unreachableSessions: listing.unreachableSessions })
   })
 
   // Live model-callability check for the footer status line.
@@ -736,7 +784,7 @@ export async function chatRoutes(app: FastifyInstance): Promise<void> {
   app.post('/api/me/model-key/test', async (req, reply) => {
     // A re-check is the user saying "I changed something, look again":
     // the provider probe cache must not outlive that.
-    invalidateProviderProbes()
+    invalidateProviderProbes(); listings.clear()
     if (!req.user) return reply.status(403).send({ error: 'not verified' })
     const body = req.body as { key?: string; baseUrl?: string } | null
     const stored = resolveUserCred(req.user.id)

server/test/modelsListingCache.test.mjs

@@ -0,0 +1,70 @@
+// The model listing is cached per viewer and credential and refreshed in
+// the background: the first load pays for the catalog and the probes, the
+// next loads answer at once, and a stale answer triggers a refresh so the
+// following load is current. ?refresh=1 recomputes before answering.
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+
+process.env.KB_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'kb-'))
+process.env.INDEX_BASE = fs.mkdtempSync(path.join(os.tmpdir(), 'ix-'))
+process.env.PW_GATEWAY_URL = 'https://gw.test/api/openai/v1'
+process.env.PW_API_KEY = 'deployment-key'
+
+let catalogCalls = 0
+let probeCalls = 0
+let catalogVersion = 1
+globalThis.fetch = async (url, init) => {
+  const u = String(url)
+  if (u.endsWith('/models')) {
+    catalogCalls++
+    return { ok: true, status: 200, json: async () => ({ data: [{ id: `me:genaimil/gemini-v${catalogVersion}`, object: 'model', owned_by: 'x' }] }), text: async () => '' }
+  }
+  if (u.endsWith('/chat/completions')) {
+    probeCalls++
+    const text = 'data: {"choices":[{"delta":{"content":"pong"}}]}\n\ndata: [DONE]\n'
+    return { ok: true, status: 200, text: async () => text, body: null }
+  }
+  return { ok: false, status: 404, text: async () => 'nope', json: async () => ({}) }
+}
+
+const { default: Fastify } = await import('fastify')
+const { chatRoutes } = await import('../dist/chat/routes.js')
+const { sanitizedErrorHandler } = await import('../dist/routes.js')
+const app = Fastify()
+app.setErrorHandler(sanitizedErrorHandler(app))
+await app.register(chatRoutes)
+await app.ready()
+const list = async (q = '') => (await app.inject({ method: 'GET', url: `/api/chat/models${q}` })).json()
+const sleep = ms => new Promise(r => setTimeout(r, ms))
+
+test('the first load computes; the second answers from the cache without asking again', async () => {
+  const a = await list()
+  assert.equal(a.models.length, 1)
+  const [c1, p1] = [catalogCalls, probeCalls]
+  assert.ok(c1 >= 1 && p1 >= 1, 'first load fetched the catalog and probed')
+  const b = await list()
+  assert.equal(b.models[0].id, a.models[0].id)
+  assert.equal(catalogCalls, c1, 'no catalog fetch on a cached load')
+  assert.equal(probeCalls, p1, 'no probe on a cached load')
+})
+
+test('refresh=1 recomputes before answering, and sees a changed catalog', async () => {
+  catalogVersion = 2
+  const [c1] = [catalogCalls]
+  const r = await list('?refresh=1')
+  assert.equal(r.models[0].id, 'me:genaimil/gemini-v2')
+  assert.ok(catalogCalls > c1, 'the catalog was fetched again')
+})
+
+test('the per-request decoration stays live on a cached listing', async () => {
+  const { noteModelFailure } = await import('../dist/chat/gateway.js').catch(() => ({}))
+  if (!noteModelFailure) return // decoration helper not exported here; covered by the route reading modelFailure per request
+  noteModelFailure('me:genaimil/gemini-v2', { status: 401, auth: true })
+  const r = await list()
+  assert.equal(r.models[0].callable, false)
+})
+
+test.after(async () => { await app.close() })

web/src/adapter.ts

@@ -1,16 +1,16 @@
 import type { ChatAdapter, ModelsList, StreamCompletion } from '@parallelworks/ai-chat'
 import { getLabelScope, getPersona } from './labelScope'
+import { fetchModels } from './api'
 
 async function listModels(): Promise<ModelsList> {
-  const res = await fetch('/api/chat/models')
-  if (!res.ok) throw new Error(`models: ${res.status}`)
-  const data = await res.json()
+  // The shared request is typed loosely; the package's own types apply here.
+  const data = (await fetchModels()) as unknown as { models?: ModelsList['models']; unreachableSessions?: ModelsList['unreachableSessions'] }
   // The server marks models whose most recent call failed (an expired
   // provider key fails at call time while listing fine). The picker
   // component is upstream, so the signal rides the display name; it
   // disappears on the first successful call.
-  const models = (data.models ?? []).map((m: { id: string; name?: string; callable?: boolean }) =>
-    m?.callable === false ? { ...m, name: `${m.name || m.id} · last call failed` } : m)
+  const models = (data.models ?? []).map(m =>
+    (m as { callable?: boolean }).callable === false ? { ...m, name: `${m.name || m.id} · last call failed` } : m)
   return { models, unreachableSessions: data.unreachableSessions ?? [] }
 }
 

web/src/api.ts

@@ -219,3 +219,37 @@ export interface IndexJob {
   ms: number | null
   error: string | null
 }
+
+// ---- the model list ----
+// Several places ask for it on the same mount: the chat package's picker,
+// the availability banner, the credential notice, the fleet page. One
+// in-flight request serves all of them, and a listing is reused for a
+// short window so a remount does not refetch. The server keeps its own
+// longer cache and refreshes in the background; this only stops the
+// client asking three times for one answer.
+export interface ModelsResponse {
+  models: { id: string; name?: string; callable?: boolean; [k: string]: unknown }[]
+  impaired?: { id: string; locked: boolean; unlock_url: string | null }[]
+  unreachableSessions?: unknown[]
+  error?: string
+  credential?: string
+}
+const MODELS_REUSE_MS = 15_000
+let modelsAt = 0
+let modelsLast: ModelsResponse | null = null
+let modelsInFlight: Promise<ModelsResponse> | null = null
+export function fetchModels(opts: { refresh?: boolean } = {}): Promise<ModelsResponse> {
+  if (!opts.refresh && modelsLast && Date.now() - modelsAt < MODELS_REUSE_MS) return Promise.resolve(modelsLast)
+  if (modelsInFlight) return modelsInFlight
+  modelsInFlight = fetch(opts.refresh ? '/api/chat/models?refresh=1' : '/api/chat/models')
+    .then(async r => {
+      if (!r.ok) throw new Error(`models: ${r.status}`)
+      const d = (await r.json()) as ModelsResponse
+      modelsLast = d; modelsAt = Date.now()
+      return d
+    })
+    .finally(() => { modelsInFlight = null })
+  return modelsInFlight
+}
+/** Forget the reused listing, so the next ask goes to the server. */
+export function forgetModels(): void { modelsLast = null; modelsAt = 0 }

web/src/views/ChatView.tsx

@@ -9,7 +9,7 @@ import {
 } from '@parallelworks/ai-chat'
 import { createStudioAdapter, getChatListFilter, setChatListFilter, setViewerUsername } from '../adapter'
 import { useAppConfig } from '../config'
-import { api } from '../api'
+import { api, fetchModels, forgetModels, type ModelsResponse } from '../api'
 import { setLabelScope, setPersona } from '../labelScope'
 import { PersonaIcon } from '../components/PersonaIcon'
 import { ConversationScrubber } from '../components/ConversationScrubber'
@@ -126,9 +126,8 @@ export function ChatView() {
   useEffect(() => {
     const onAccessChange = () => {
       setCredNote(null)
-      fetch('/api/chat/models').then(r => r.json())
-        .then(d => { if (!d.models?.length && d.error) setCredNote(String(d.error)) })
-        .catch(() => { /* the remount below refetches regardless */ })
+      forgetModels()
+      fetchModels({ refresh: true }).then(applyModels).catch(() => { /* the remount below refetches regardless */ })
       setChatEpoch(e => e + 1)
     }
     window.addEventListener('ade:model-access-changed', onAccessChange)
@@ -161,23 +160,48 @@ export function ChatView() {
   }, [])
   const [scopeFilter, setScopeFilter] = useState('')
 
+  // What the listing says, applied wherever it came from: the banner above
+  // the thread, and a remount of the picker when the set of marked models
+  // changes, since the picker is the package's and only re-reads on mount.
+  // A remount mid-reply would drop the visible stream, so it waits for the
+  // next idle moment.
+  const marksRef = useRef<string>('')
+  const applyModels = (d: ModelsResponse) => {
+    if (d.error && !(d.models ?? []).length) { setCredNote(String(d.error)); return }
+    const impaired = (d.impaired ?? []) as { id: string; locked: boolean }[]
+    const marks = impaired.map(m => `${m.id}:${m.locked ? 'L' : 'U'}`).sort().join(',')
+    if (impaired.length) {
+      const locked = impaired.some(m => m.locked)
+      setCredNote(`${impaired.length} model${impaired.length === 1 ? ' is' : 's are'} marked [${locked ? 'locked' : 'unavailable'}] in the model list${locked
+        ? ': the provider reports the key is locked'
+        : ' (for GenAI this usually means the key is locked on its 8-hour schedule)'}. Unlock and Re-check under Settings, Model access; the marks clear on the next listing.`)
+    } else if (marksRef.current) {
+      setCredNote(null)
+    }
+    if (marksRef.current && marks !== marksRef.current) setChatEpoch(e => e + 1)
+    marksRef.current = marks
+  }
+  // Keep the status fresh without a reload: ask again when the tab comes
+  // back into view and every five minutes while it stays there. The
+  // server answers from its cache and refreshes behind the answer, so
+  // these are cheap.
+  useEffect(() => {
+    const tick = () => { if (document.visibilityState === 'visible') fetchModels().then(applyModels).catch(() => {}) }
+    const onVisible = () => { if (document.visibilityState === 'visible') { forgetModels(); tick() } }
+    document.addEventListener('visibilitychange', onVisible)
+    const id = window.setInterval(() => { forgetModels(); tick() }, 5 * 60_000)
+    return () => { document.removeEventListener('visibilitychange', onVisible); window.clearInterval(id) }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [])
+
   useEffect(() => {
     migrateRememberedModel()
     api.tagVocabulary().then(v => setVocab(v.tags)).catch(() => {})
     // The model picker is the package's and renders labels from the id
     // alone, so a locked provider cannot be marked inside it (upstream
     // issue filed). The warning therefore lives here, above the thread,
     // where it cannot be missed or truncated.
-    fetch('/api/chat/models').then(r => r.json()).then(d => {
-      if (d.error && !(d.models ?? []).length) { setCredNote(String(d.error)); return }
-      const impaired = (d.impaired ?? []) as { id: string; locked: boolean }[]
-      if (impaired.length) {
-        const locked = impaired.some(m => m.locked)
-        setCredNote(`${impaired.length} model${impaired.length === 1 ? ' is' : 's are'} marked [${locked ? 'locked' : 'unavailable'}] in the model list${locked
-          ? ': the provider reports the key is locked'
-          : ' (for GenAI this usually means the key is locked on its 8-hour schedule)'}. Unlock and Re-check under Settings, Model access; the marks clear on the next listing.`)
-      }
-    }).catch(() => {})
+    fetchModels().then(applyModels).catch(() => {})
     fetch('/api/me/model-key').then(r => r.json())
       .then(d => {
         setMultiUser(!!d.authEnabled)
@@ -195,9 +219,6 @@ export function ChatView() {
     // Both the no-models state and mid-conversation credential failures
     // surface as a toast with a click path to Settings; the empty state
     // text explains, the toast provides the button.
-    fetch('/api/chat/models').then(r => r.json())
-      .then(d => { if (!d.models?.length && d.error) setCredNote(String(d.error)) })
-      .catch(() => {})
     const onCredError = (e: Event) => setCredNote(String((e as CustomEvent).detail ?? 'Model credential needed.'))
     window.addEventListener('ade-credential-error', onCredError)
     // Back and forward between conversations.