diff --git a/package-lock.json b/package-lock.json index 2c53c38..e686ac0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "flujo-factory", - "version": "0.7.1", + "version": "0.7.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "flujo-factory", - "version": "0.7.1", + "version": "0.7.2", "license": "MIT", "dependencies": { "@modelcontextprotocol/sdk": "1.32.1" diff --git a/package.json b/package.json index 0792fdd..7fb1cb5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flujo-factory", - "version": "0.7.1", + "version": "0.7.2", "description": "Durable FLUJO swarm engine and Factory SDK, CLI, and MCP server", "license": "MIT", "repository": { diff --git a/selfhost/README.md b/selfhost/README.md index c9aade5..acad9c0 100644 --- a/selfhost/README.md +++ b/selfhost/README.md @@ -94,6 +94,13 @@ Overrides are explicit settings fields, for example `{ replicas: 2 }`, `{ varian `{ enable_thinking: false }` or `{ scaledown_minutes: 2 }`. Unknown fields, unpinned images and settings that would not fit the GPU are refused before anything is created. +- `request_timeout_minutes` (default 30) is the longest single request. Modal's starter plan never + schedules a GPU function whose limit is 45 minutes or more: the deployment and every request + then wait without an error. Raise it only on a plan that allows it. +- `weights_volume` / `compile_volume` name an existing volume that already holds the weights (a + Hugging Face cache) or vLLM's compile caches, for example one an earlier deployment filled. The + app uses it instead of creating its own; `retire` never deletes it. + ## Things to know - vLLM's `--api-key` protects `/v1/*`; `/health` and `/metrics` answer without it (counters only). diff --git a/selfhost/modal/app.py b/selfhost/modal/app.py index 53f99dc..36e71b3 100644 --- a/selfhost/modal/app.py +++ b/selfhost/modal/app.py @@ -38,8 +38,11 @@ ) app = modal.App(S.app_name) -hf_cache = modal.Volume.from_name(S.app_name + '-hf-cache', create_if_missing=True) -vllm_cache = modal.Volume.from_name(S.app_name + '-vllm-cache', create_if_missing=True) +# The app's own volumes are created on first use; a reused one (weights_volume, compile_volume) +# must already exist and is never created here. +_V = S.volumes() +hf_cache = modal.Volume.from_name(_V['weights']['name'], create_if_missing=_V['weights']['owned']) +vllm_cache = modal.Volume.from_name(_V['compile']['name'], create_if_missing=_V['compile']['owned']) auth = modal.Secret.from_name(S.app_name + '-auth', required_keys=['VLLM_API_KEY']) VOLUMES = {HF_CACHE: hf_cache, '/root/.cache/vllm': vllm_cache} diff --git a/selfhost/modal/settings.py b/selfhost/modal/settings.py index 60c5f8f..bcdc915 100644 --- a/selfhost/modal/settings.py +++ b/selfhost/modal/settings.py @@ -23,6 +23,8 @@ # --<12 hex>; Modal names stay below 64 characters with the volume suffixes. APP_NAME = re.compile(r'[a-z][a-z0-9]{1,15}-[a-z][a-z0-9]{1,15}-[a-f0-9]{12}') PROFILE_NAME = re.compile(r'[a-z][a-z0-9]{1,15}') +# Modal volume names: letters, digits, dashes, dots and underscores, below 64 characters. +VOLUME_NAME = re.compile(r'[A-Za-z0-9][A-Za-z0-9._-]{0,62}') SETTINGS_ENV = 'FACTORY_SELFHOST_SETTINGS_B64' # Profile keys that describe the model for clients, not the server. CLIENT_KEYS = ('comment', 'compact_tokens', 'efforts') @@ -68,8 +70,16 @@ class Settings: replicas: int = 1 # Idle minutes before a GPU container stops (Modal allows 1..20). scaledown_minutes: int = 10 - request_timeout_minutes: int = 60 + # Longest single request. Modal's starter plan never schedules a GPU function whose limit is + # 45 minutes or more (it waits without an error; measured 2026-11-11: 10 to 30 start in + # seconds, 45 and 60 never). One model call of an agent stays far below 30. + request_timeout_minutes: int = 30 startup_timeout_minutes: int = 40 + # An existing volume to use for the weights / the compile caches instead of the app's own + # (for example one an earlier deployment filled). Such a volume is never created or deleted + # here; empty = the owned `-hf-cache` / `-vllm-cache`. + weights_volume: str = '' + compile_volume: str = '' port: int = 8000 def __post_init__(self): @@ -93,6 +103,12 @@ def __post_init__(self): raise ValueError(f'replicas must be within 1..{MAX_REPLICAS}') if not 1 <= self.scaledown_minutes <= 20: raise ValueError('scaledown_minutes must be within 1..20 (Modal limit)') + if not 1 <= self.request_timeout_minutes <= 1440: + raise ValueError('request_timeout_minutes must be within 1..1440 (Modal limit)') + for name in ('weights_volume', 'compile_volume'): + value = getattr(self, name) + if value and (not VOLUME_NAME.fullmatch(value) or value.startswith(self.app_name)): + raise ValueError(f'{name} must name an existing volume outside this app') if self.variant and self.variant not in self.variants: raise ValueError(f'unknown variant {self.variant!r}; expected one of {sorted(self.variants)}') if self.image and not re.fullmatch(r'[\w./-]+@sha256:[a-f0-9]{64}', self.image): @@ -110,6 +126,11 @@ def endpoint_functions(self): """The web functions, one per replica; each has its own URL.""" return list(REPLICA_FUNCTIONS[:self.replicas]) + def volumes(self): + """Weights and compile-cache volume names, and whether this app owns (creates, deletes) each.""" + return {'weights': {'name': self.weights_volume or self.app_name + '-hf-cache', 'owned': not self.weights_volume}, + 'compile': {'name': self.compile_volume or self.app_name + '-vllm-cache', 'owned': not self.compile_volume}} + def usd_per_second(self): """List price of one running replica: GPU plus its reserved CPU and memory.""" return (PRICES['gpuPerSecond'][self.gpu] * self.gpu_count + self.cpu * PRICES['cpuCorePerSecond'] @@ -157,7 +178,7 @@ def describe(self): 'gpu': self.gpu_spec, 'replicas': self.replicas, 'functions': self.endpoint_functions(), 'usdPerHourPerReplica': self.usd_per_hour(), 'usdPerSecondPerReplica': self.usd_per_second(), 'scaledownMinutes': self.scaledown_minutes, 'startupTimeoutMinutes': self.startup_timeout_minutes, - 'requestTimeoutMinutes': self.request_timeout_minutes, + 'requestTimeoutMinutes': self.request_timeout_minutes, 'volumes': self.volumes(), 'contextWindow': self.max_model_len, 'variant': self.variant or None, 'inputModalities': list(self.input_modalities), 'estimateGib': self.estimate_gib(), 'vllmCommand': self.vllm_command()} diff --git a/selfhost/modal/test_settings.py b/selfhost/modal/test_settings.py index 65826c1..cc26bf9 100644 --- a/selfhost/modal/test_settings.py +++ b/selfhost/modal/test_settings.py @@ -81,6 +81,28 @@ def test_overrides_are_explicit_and_validated(self): with self.assertRaises(ValueError): resolve(QWEN, 'missing') + def test_request_limit_is_one_the_starter_plan_schedules(self): + # Modal's starter plan never gives a GPU to a function with a request limit of 45 minutes or more. + for name, app in (('qwen', QWEN), ('nemotron', NEMOTRON)): + s = resolve(app, name) + self.assertLess(s.request_timeout_minutes, 45, name) + self.assertEqual(s.describe()['requestTimeoutMinutes'], s.request_timeout_minutes) + self.assertEqual(resolve(QWEN, 'qwen', {'request_timeout_minutes': 60}).request_timeout_minutes, 60) + for bad in (0, 1441): + with self.assertRaises(ValueError): + resolve(QWEN, 'qwen', {'request_timeout_minutes': bad}) + + def test_an_earlier_volume_can_hold_the_weights_and_is_never_owned(self): + own = resolve(NEMOTRON, 'nemotron').volumes() + self.assertEqual(own, {'weights': {'name': NEMOTRON + '-hf-cache', 'owned': True}, + 'compile': {'name': NEMOTRON + '-vllm-cache', 'owned': True}}) + reused = resolve(NEMOTRON, 'nemotron', {'weights_volume': 'earlier-hf-cache', 'compile_volume': 'earlier-vllm-cache'}) + self.assertEqual(reused.describe()['volumes'], {'weights': {'name': 'earlier-hf-cache', 'owned': False}, + 'compile': {'name': 'earlier-vllm-cache', 'owned': False}}) + for bad in (' x', 'a/b', 'x' * 64, NEMOTRON + '-hf-cache'): + with self.assertRaises(ValueError, msg=bad): + resolve(NEMOTRON, 'nemotron', {'weights_volume': bad}) + def test_frozen_settings_round_trip_through_the_image_environment(self): s = resolve(NEMOTRON, 'nemotron', {'replicas': 2}) restored = from_env({SETTINGS_ENV: s.to_b64()}) diff --git a/src/cloud-lite/claude-driver.mjs b/src/cloud-lite/claude-driver.mjs index 8394f40..d70c6a1 100644 --- a/src/cloud-lite/claude-driver.mjs +++ b/src/cloud-lite/claude-driver.mjs @@ -25,7 +25,8 @@ export function claudeEnvironment({ plan, index = 0, home, commandWrapper, githu ENABLE_TOOL_SEARCH: 'false', CLAUDE_CODE_DISABLE_AUTO_MEMORY: '1', CLAUDE_CODE_DISABLE_BACKGROUND_TASKS: '1', CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1', DISABLE_TELEMETRY: '1', DISABLE_ERROR_REPORTING: '1', DISABLE_AUTOUPDATER: '1', ...(githubToken && !deployKey ? { GH_TOKEN: githubToken } : {}) }; - if (plan.modelProvider) return applyProviderEnvironment(base, claudeProviderEnvironment(plan.modelProvider, { index, key: providerKey, model: plan.model })); + // The base is built from scratch: it carries no Claude session variables to clear. + if (plan.modelProvider) return applyProviderEnvironment(base, claudeProviderEnvironment(plan.modelProvider, { index, key: providerKey, model: plan.model, inherited: {} })); return { ...base, ...(oauthToken ? { CLAUDE_CODE_OAUTH_TOKEN: oauthToken } : {}) }; } diff --git a/src/lite-worker.mjs b/src/lite-worker.mjs index 532500f..791fc4d 100644 --- a/src/lite-worker.mjs +++ b/src/lite-worker.mjs @@ -35,7 +35,7 @@ export function liteWorkerCommand({ provider, database, prompt, agentId, cwd = p }; if (provider === 'claude') return { command: 'claude', cwd, - ...(modelProvider ? { env: applyProviderEnvironment(env, claudeProviderEnvironment(modelProvider, { index: providerIndex, key: providerKey })) } : {}), + ...(modelProvider ? { env: applyProviderEnvironment(env, claudeProviderEnvironment(modelProvider, { index: providerIndex, key: providerKey, inherited: env })) } : {}), args: ['--dangerously-skip-permissions', '--strict-mcp-config', '--mcp-config', JSON.stringify({ mcpServers: { factory: server } }), ...(prompt === undefined ? [] : [prompt])], }; diff --git a/src/model-provider.mjs b/src/model-provider.mjs index 3e2f83c..850c663 100644 --- a/src/model-provider.mjs +++ b/src/model-provider.mjs @@ -122,17 +122,25 @@ export function codexProviderOverrides(provider, options = {}) { ...tables.flatMap(([table, entries]) => entries.map(([key, value]) => `${table}.${key}=${tomlValue(value)}`))]; } +// A caller that itself runs inside a Claude session (a desktop or SDK session) passes that +// session's variables on; a Claude Code child then offers the session's own login to the +// endpoint (HTTP 401) instead of the endpoint key. By name pattern, not by a list: such sessions +// add variables with every version. +const CLAUDE_SESSION_VARIABLE = /^(CLAUDE|ANTHROPIC_)|OAUTH/i; + /** * Claude Code environment for a self-hosted provider (Anthropic Messages API at the server * root). Every model alias maps to the served model; nonessential traffic is off. Keys whose - * value is null must be removed from the inherited environment. + * value is null must be removed from the inherited environment: every Claude session variable + * of `inherited` (default: this process) is. */ -export function claudeProviderEnvironment(provider, { index = 0, key, model } = {}) { +export function claudeProviderEnvironment(provider, { index = 0, key, model, inherited = process.env } = {}) { requireApi(provider, 'claude'); if (typeof key !== 'string' || key.length < 8) throw new TypeError('A model provider needs its API key.'); const served = provider.model ?? model; if (typeof served !== 'string' || !served) throw new TypeError('Select the served model name.'); return { + ...Object.fromEntries(Object.keys(inherited ?? {}).filter(name => CLAUDE_SESSION_VARIABLE.test(name)).map(name => [name, null])), ANTHROPIC_BASE_URL: providerBaseUrl(provider, index).replace(/\/+$/, '').replace(/\/v1$/, ''), ANTHROPIC_AUTH_TOKEN: key, ANTHROPIC_API_KEY: null, diff --git a/test/model-provider.test.mjs b/test/model-provider.test.mjs index febc171..afbb439 100644 --- a/test/model-provider.test.mjs +++ b/test/model-provider.test.mjs @@ -7,6 +7,7 @@ import { fileURLToPath } from 'node:url'; import { claudeProviderEnvironment, codexProviderConfig, codexProviderOverrides, liteWorkerCommand, providerPrograms, validateModelProvider } from '../src/public-sdk.mjs'; import { validateCloudLitePlan, secret } from '../src/cloud-lite/protocol.mjs'; import { providerConfigToml } from '../src/cloud-lite/runtime.mjs'; +import { applyProviderEnvironment } from '../src/model-provider.mjs'; import { tempDir } from './support/temp-dir.mjs'; // One description of a self-hosted vLLM endpoint that serves every wire API. @@ -56,6 +57,17 @@ test('the Claude Code mapping uses the Anthropic API at the server root with eve assert.throws(() => claudeProviderEnvironment(endpoint(), { key: 'short' }), /API key/); }); +test('a Claude Code child started from inside a Claude session inherits none of that session\'s variables', () => { + const key = secret(); + const inherited = { PATH: '/bin', CLAUDECODE: '1', CLAUDE_CODE_ENTRYPOINT: 'sdk-ts', CLAUDE_CODE_SDK_HAS_HOST_AUTH_REFRESH: '1', USE_STAGING_OAUTH: '1', ANTHROPIC_CUSTOM_HEADERS: 'x' }; + const env = claudeProviderEnvironment(endpoint(), { key, inherited }); + for (const name of ['CLAUDECODE', 'CLAUDE_CODE_ENTRYPOINT', 'CLAUDE_CODE_SDK_HAS_HOST_AUTH_REFRESH', 'USE_STAGING_OAUTH', 'ANTHROPIC_CUSTOM_HEADERS']) assert.equal(env[name], null, name); + assert.equal('PATH' in env, false); + // its own settings still win over a cleared name + assert.equal(claudeProviderEnvironment(endpoint(), { key, inherited: { CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '0' } }).CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC, '1'); + assert.equal(applyProviderEnvironment(inherited, env).CLAUDE_CODE_ENTRYPOINT, undefined); +}); + test('local lite workers launch Codex and Claude Code against a self-hosted model', () => { const key = secret(); const inherited = { PATH: '/bin', ANTHROPIC_API_KEY: 'vendor-key-value', CLAUDE_CODE_OAUTH_TOKEN: 'vendor-login' }; const codex = liteWorkerCommand({ provider: 'codex', database: 'board.sqlite', modelProvider: endpoint(), providerKey: key, providerIndex: 1, env: inherited }); diff --git a/test/selfhost-modal.test.mjs b/test/selfhost-modal.test.mjs index ca7c485..2d3a42f 100644 --- a/test/selfhost-modal.test.mjs +++ b/test/selfhost-modal.test.mjs @@ -32,7 +32,11 @@ function fixture({ deployError, describeError, holdDescribe, workspace = 'fixtur if (request.action === 'deploy') { if (deployError) return { code: 1, stdout: JSON.stringify({ error: deployError }) }; if (!request.reuseSecret) { assert.equal(secrets.has(`${appName}-auth`), false); secrets.add(`${appName}-auth`); } - key = request.key; volumes.add(`${appName}-hf-cache`); volumes.add(`${appName}-vllm-cache`); + key = request.key; + // a reused volume must exist already; only the app's own ones are created + for (const [name, own] of [[request.overrides.weights_volume, `${appName}-hf-cache`], [request.overrides.compile_volume, `${appName}-vllm-cache`]]) { + if (name) assert.ok(volumes.has(name), `${name} exists before the deployment`); else volumes.add(own); + } apps.set(appName, 'deployed'); const replicas = request.overrides.replicas ?? 1; return { code: 0, stdout: JSON.stringify({ endpoints: ['serve', 'serve-b'].slice(0, replicas).map(name => `https://fixture--${appName}-${name}.modal.run`) }) }; @@ -137,6 +141,18 @@ test('deploy, verify, describe, stop, redeploy and retire touch only the owned r assert.ok(named.every(name => name.startsWith(f.appName())), named.join(',')); }); +test('volumes an earlier deployment filled are used and survive the retire', async () => { + const f = fixture(); + f.volumes.add('earlier-hf-cache'); f.volumes.add('earlier-vllm-cache'); + const ready = await f.model.deploy({ profile: 'nemotron', overrides: { weights_volume: 'earlier-hf-cache', compile_volume: 'earlier-vllm-cache' } }); + assert.equal(ready.phase, 'ready'); + assert.deepEqual([...f.volumes].sort(), ['earlier-hf-cache', 'earlier-vllm-cache'], 'no volume of its own is created'); + assert.equal((await f.model.retire()).phase, 'retired'); + assert.deepEqual([...f.volumes].sort(), ['earlier-hf-cache', 'earlier-vllm-cache'], 'the reused volumes are kept'); + const named = f.calls.filter(call => call.args.includes('delete')).map(call => call.args.at(-1)); + assert.ok(named.every(name => name.startsWith(f.appName())), named.join(',')); +}); + test('an uncertain deployment is kept for reconciliation and never replayed', async () => { const f = fixture({ deployError: 'The Modal deployment did not complete.' }); await assert.rejects(f.model.deploy({ profile: 'nemotron' }), error => error.outcome === 'unknown');