diff --git a/.github/workflows/public-sdk.yml b/.github/workflows/public-sdk.yml index a081a7c..5ee5e8e 100644 --- a/.github/workflows/public-sdk.yml +++ b/.github/workflows/public-sdk.yml @@ -23,7 +23,9 @@ jobs: cache: npm - run: npm ci - run: npm audit --omit=dev --audit-level=high - - run: node --test test/cloud-lite.test.mjs test/cloud-lite-provider.test.mjs test/model-provider.test.mjs test/public-sdk.test.mjs test/template-catalog.test.mjs test/factory-team-template.test.mjs test/flujo-workspace.test.mjs test/savia-case-plan.test.mjs test/savia-topology.test.mjs test/managed-fleet.test.mjs test/managed-fleet-real-module.test.mjs test/managed-fleet-provider-closure.test.mjs test/provider-cell-retirement.test.mjs test/presentation.test.mjs test/presentation-modal.test.mjs test/presentation-closure.test.mjs test/observatory-client.test.mjs test/workflows.test.mjs test/workflow-native-wake.test.mjs test/workflow-m0-lifecycle.test.mjs test/workflow-m1-portable-artifacts.test.mjs test/workflow-m2-ack-recovery.test.mjs test/workflow-m2-authority-recovery.test.mjs + - run: node --test test/cloud-lite.test.mjs test/cloud-lite-provider.test.mjs test/model-provider.test.mjs test/selfhost-modal.test.mjs test/public-sdk.test.mjs test/template-catalog.test.mjs test/factory-team-template.test.mjs test/flujo-workspace.test.mjs test/savia-case-plan.test.mjs test/savia-topology.test.mjs test/managed-fleet.test.mjs test/managed-fleet-real-module.test.mjs test/managed-fleet-provider-closure.test.mjs test/provider-cell-retirement.test.mjs test/presentation.test.mjs test/presentation-modal.test.mjs test/presentation-closure.test.mjs test/observatory-client.test.mjs test/workflows.test.mjs test/workflow-native-wake.test.mjs test/workflow-m0-lifecycle.test.mjs test/workflow-m1-portable-artifacts.test.mjs test/workflow-m2-ack-recovery.test.mjs test/workflow-m2-authority-recovery.test.mjs + - name: Self-hosted model settings (Python, offline) + run: python3 -B -m unittest discover -s selfhost/modal -p "test_*.py" - name: Install and exercise the packed public package shell: bash run: | @@ -34,6 +36,7 @@ jobs: node "$RUNNER_TEMP/consumer/node_modules/flujo-factory/bin/factory-public.mjs" --help node "$RUNNER_TEMP/consumer/node_modules/flujo-factory/bin/factory-lite.mjs" --help node "$RUNNER_TEMP/consumer/node_modules/flujo-factory/bin/factory-evidence.mjs" --help + node "$RUNNER_TEMP/consumer/node_modules/flujo-factory/bin/factory-selfhost.mjs" profiles FACTORY_TEST_MCP="$RUNNER_TEMP/consumer/node_modules/flujo-factory/bin/factory-mcp.mjs" node --test --test-name-pattern="public MCP stdio" test/public-sdk.test.mjs - name: Generic workflow and evidence recovery regressions run: node --test test/generic-swarm-evidence.test.mjs diff --git a/PUBLIC_SDK.md b/PUBLIC_SDK.md index 1af8ff8..afc7499 100644 --- a/PUBLIC_SDK.md +++ b/PUBLIC_SDK.md @@ -1,18 +1,21 @@ # Public local Factory SDK, CLI, and MCP -Version 0.4.0 is distributed as a versioned package in -[GitHub Releases](https://github.com/flujo-app/FACTORY/releases/tag/v0.4.0): +Version 0.5.0 is distributed as a versioned package in +[GitHub Releases](https://github.com/flujo-app/FACTORY/releases/tag/v0.5.0): ```sh -npm install https://github.com/flujo-app/FACTORY/releases/download/v0.4.0/flujo-factory-0.4.0.tgz +npm install https://github.com/flujo-app/FACTORY/releases/download/v0.5.0/flujo-factory-0.5.0.tgz ``` -0.4.0 lets cloud lite workers run on a self-hosted OpenAI-compatible model, -clone with a read-only deploy key, prepare agent folders, take operator-driven -jobs in a narrower sandbox, and record far less trace volume. See -[Self-hosted models and operator-driven jobs](#self-hosted-models-and-operator-driven-jobs). -All new plan settings are optional; existing plans behave as before, except -that streaming deltas are no longer traced unless `traceDeltas` is set. +FACTORY can serve an open-weight model on your own Modal GPUs and run workers on it. +`ModalSelfHostedModel` and `factory-selfhost` deploy, verify, stop and retire a vLLM endpoint +with Qwen3.8-27B (H100) or Nemotron 3.5 Super (B300) profiles; see +[selfhost/README.md](selfhost/README.md). Its endpoint description is a model provider that +cloud lite workers (Codex) and local lite workers (Codex or Claude Code) use directly. See +[Self-hosted models and operator-driven jobs](#self-hosted-models-and-operator-driven-jobs) +for the provider, read-only deploy keys, prepare hooks, operator-driven jobs and quieter +traces added in 0.4.0. All of it is optional; existing plans behave as before, except that +streaming deltas are no longer traced unless `traceDeltas` is set. `FactoryControl` and `executeEffect` are public exports for integrations that retain FACTORY's durable admission while using an existing fleet executor. @@ -259,6 +262,10 @@ public journal, source commit or trace. Store the fleet directory privately. #### Self-hosted models and operator-driven jobs +To run the model yourself, `ModalSelfHostedModel` (CLI `factory-selfhost`) deploys a +bundled vLLM profile on your Modal account and returns this description from `endpoint()`, +with its key from `key()`; see [selfhost/README.md](selfhost/README.md). + Lite workers can run on a self-hosted, OpenAI-compatible model instead of the subscription login. Codex reaches it through the Responses API (`/v1/responses`), which vLLM serves for every model. Add `modelProvider` to the diff --git a/bin/factory-selfhost.mjs b/bin/factory-selfhost.mjs new file mode 100644 index 0000000..e136b4f --- /dev/null +++ b/bin/factory-selfhost.mjs @@ -0,0 +1,68 @@ +#!/usr/bin/env node +import { writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { parseArgs } from 'node:util'; +import { ModalSelfHostedModel, selfHostedProfiles } from '../src/selfhost/modal.mjs'; + +const usage = `Usage: factory-selfhost [options] + +A self-hosted model (vLLM) on your Modal account, owned and journaled in a private folder. +Uses the active Modal profile and a Python that has the modal package. + + profiles list model profiles with list-price estimates + deploy --dir [--profile qwen|nemotron] [--set key=value ...] [--prefetch] [--no-verify] + create the key, deploy, and verify the served model + status --dir phase, model, GPU and price (no key or URL) + endpoint --dir [--out f] model-provider description for workers (no key) + key --dir --out write the endpoint key to a private file + verify --dir check the served model again + stop --dir GPU off now; deploy again to resume + retire --dir delete the app, secret, volumes and key, and check it + +Options: --python (default python3, or python on Windows), --prefix (default factory). +--set values are JSON when they parse (--set replicas=2 --set variant="fast"), text otherwise. +Running replicas cost money; idle ones stop after the profile's scale-down time. +`; + +const { values, positionals } = parseArgs({ allowPositionals: true, options: { + dir: { type: 'string' }, profile: { type: 'string' }, set: { type: 'string', multiple: true }, prefetch: { type: 'boolean' }, + 'no-verify': { type: 'boolean' }, out: { type: 'string' }, python: { type: 'string' }, prefix: { type: 'string' }, help: { type: 'boolean' } } }); +const [command] = positionals; +const print = value => process.stdout.write(`${JSON.stringify(value, null, 2)}\n`); + +async function main() { + if (!command || values.help || command === 'help') return process.stdout.write(usage); + if (command === 'profiles') return print(selfHostedProfiles()); + if (!values.dir) throw new Error('--dir is required.'); + const model = new ModalSelfHostedModel({ directory: resolve(values.dir), ...(values.python ? { python: values.python } : {}), ...(values.prefix ? { namePrefix: values.prefix } : {}) }); + const controller = new AbortController(); + process.once('SIGINT', () => controller.abort()); + const signal = controller.signal; + if (command === 'status') return print(model.status()); + if (command === 'endpoint') { + const description = model.endpoint(); + if (values.out) { writeFileSync(values.out, JSON.stringify(description, null, 2)); return print({ written: values.out }); } + return print(description); + } + if (command === 'key') { + if (!values.out) throw new Error('--out is required; the key is never printed.'); + writeFileSync(values.out, await model.key(), { mode: 0o600 }); + return print({ written: values.out }); + } + if (command === 'deploy') { + const overrides = {}; + for (const entry of values.set ?? []) { + const at = entry.indexOf('='); + if (at < 1) throw new Error(`--set needs key=value: ${entry}`); + const raw = entry.slice(at + 1); + try { overrides[entry.slice(0, at)] = JSON.parse(raw); } catch { overrides[entry.slice(0, at)] = raw; } + } + return print(await model.deploy({ profile: values.profile ?? 'qwen', overrides, prefetch: values.prefetch === true, verify: values['no-verify'] !== true, signal })); + } + if (command === 'verify') return print(await model.verify({ signal })); + if (command === 'stop') return print(await model.stop({ signal })); + if (command === 'retire') return print(await model.retire({ signal })); + throw new Error(`Unknown command: ${command}`); +} + +main().catch(error => { process.stderr.write(`${error.message}\n`); process.exitCode = 1; }); diff --git a/package-lock.json b/package-lock.json index 9e835d2..2aa52bc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "flujo-factory", - "version": "0.4.0", + "version": "0.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "flujo-factory", - "version": "0.4.0", + "version": "0.5.0", "license": "MIT", "dependencies": { "@modelcontextprotocol/sdk": "1.32.1" @@ -15,7 +15,8 @@ "factory": "bin/factory-public.mjs", "factory-evidence": "bin/factory-evidence.mjs", "factory-lite": "bin/factory-lite.mjs", - "factory-mcp": "bin/factory-mcp.mjs" + "factory-mcp": "bin/factory-mcp.mjs", + "factory-selfhost": "bin/factory-selfhost.mjs" }, "engines": { "node": "^22.17.0 || >=24" diff --git a/package.json b/package.json index fad3419..fec264c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flujo-factory", - "version": "0.4.0", + "version": "0.5.0", "description": "Durable FLUJO swarm engine and Factory SDK, CLI, and MCP server", "license": "MIT", "repository": { @@ -19,12 +19,18 @@ "bin/factory-lite.mjs", "bin/factory-cloud-mcp.mjs", "bin/factory-cloud-runtime.mjs", + "bin/factory-selfhost.mjs", "bin/factory.mjs", "scripts/watch-pilot.mjs", "scripts/pilot.mjs", "scripts/run-tests.mjs", "scripts/local-flujo-live-smoke.mjs", "scripts/savia-local-live-smoke.mjs", + "selfhost/README.md", + "selfhost/modal/*.py", + "selfhost/modal/prices.json", + "selfhost/modal/profiles/*.json", + "!selfhost/modal/test_*.py", "README.md", "PUBLIC_SDK.md", "WORKFLOWS.md", @@ -45,7 +51,8 @@ "factory-evidence": "./bin/factory-evidence.mjs", "factory": "./bin/factory-public.mjs", "factory-mcp": "./bin/factory-mcp.mjs", - "factory-lite": "./bin/factory-lite.mjs" + "factory-lite": "./bin/factory-lite.mjs", + "factory-selfhost": "./bin/factory-selfhost.mjs" }, "type": "module", "engines": { diff --git a/scripts/verify-public-sdk.mjs b/scripts/verify-public-sdk.mjs index a205c64..f1a4108 100644 --- a/scripts/verify-public-sdk.mjs +++ b/scripts/verify-public-sdk.mjs @@ -98,3 +98,10 @@ const endpoint = { ...providerPlan.modelProvider, model: 'served-model', apis: [ assert.deepEqual(sdk.providerPrograms(endpoint), ['codex', 'claude']); assert.match(sdk.codexProviderConfig(endpoint), /wire_api = "responses"/); assert.equal(sdk.claudeProviderEnvironment(endpoint, { key: 'package-smoke-key' }).ANTHROPIC_BASE_URL, 'https://model.example'); +// Self-hosted model serving ships its Python sources and profiles, but not its tests. +assert.deepEqual(sdk.selfHostedProfiles().map(profile => profile.id), ['nemotron', 'qwen']); +for (const file of ['app.py', 'settings.py', 'discovery.py', 'deploy.py', 'prices.json', 'profiles/qwen.json', 'profiles/nemotron.json']) { + assert.equal(existsSync(join(sdk.SELFHOST_MODAL_DIR, file)), true, `${file} is missing`); +} +assert.equal(existsSync(join(sdk.SELFHOST_MODAL_DIR, 'test_settings.py')), false); +assert.equal(typeof sdk.ModalSelfHostedModel, 'function'); diff --git a/selfhost/README.md b/selfhost/README.md new file mode 100644 index 0000000..f8d9594 --- /dev/null +++ b/selfhost/README.md @@ -0,0 +1,104 @@ +# Self-hosted models on Modal + +FACTORY can serve an open-weight model on your own Modal GPUs with vLLM, so workers run on it +instead of a vendor API: source code and prompts stay in your account. The served endpoint +speaks the OpenAI Responses API (Codex), the Anthropic Messages API (Claude Code) and Chat +Completions, and plugs straight into `plan.modelProvider` and the local lite workers. + +This folder is separate from `modal/`, which holds the earlier pilot definitions. + +## Profiles + +| Profile | Model (served name) | GPU | Price while running | Context | Notes | +|---|---|---|---|---|---| +| `qwen` (default) | Qwen3.8-27B FP8 (`qwen3.8-27b`) | 1 × H100 80 GB | ~$4.84/h | 262,144 | vLLM 0.30.0; parsers `qwen3_xml` / `qwen3`; thinking on | +| `nemotron` | Nemotron 3.5 Super VL 120B-A12B NVFP4 (`nemotron-3.5-super-vl`) | 1 × B300 288 GB | ~$8.43/h | 262,144 | vLLM 0.32.0 release image (pinned by digest); parsers `qwen3_coder` / `nemotron_v3`; text only | + +Prices are Modal list prices (GPU plus the reserved CPU and memory, `prices.json`, checked +2026-11-01), per running replica. They are estimates, not invoices; storage, egress and image +builds are extra. A replica with no requests stops after the profile's scale-down time (10 or +8 minutes) and costs nothing while stopped. Each replica is its own web function with at most +one GPU container (`replicas` 1–4), so the GPU count cannot grow by itself. + +The Nemotron profile has named flag sets (`variant`), all checked against vLLM 0.32.0, the +current stable release: + +| Variant | Adds | Use | +|---|---|---| +| `base` | FP32 Mamba state | reference | +| `fast` | async scheduling, FlashInfer Mamba kernel | 128–200 long conversations per GPU, full precision | +| `mtp1` (default) | `fast` + multi-token prediction, 1 draft token | up to ~128 conversations; each agent much faster | +| `mtp3` | `fast` + 3 draft tokens | up to ~32 conversations | +| `mtp1-fp16` | `mtp1` + FP16 Mamba state with stochastic rounding (lower precision) | MTP speed at ~190 conversations | + +All Nemotron variants use prefix caching for the hybrid model (`--mamba-cache-mode align`, +`--enable-mamba-shared-prefix-checkpoint` with MTP) and load the language model only. + +## Measured (long agent conversations, 300-token answers, histories 11k–100k tokens) + +| Setup | Comfortable agents (turn ≤ 10 s, cache hits ≥ 85%) | Each agent | $ per agent-hour | +|---|---|---|---| +| Qwen, 1 × H100 | 8 | 39 tok/s | 0.61 | +| Nemotron `mtp1`, 1 × B300 | 64 (128 busy) | 57 tok/s (32 busy) | 0.13 (0.066 busy) | +| Nemotron `fast` / `mtp1-fp16`, 1 × B300 | 192 busy | 14 / 23 tok/s | 0.044 | + +On one H100 the Qwen model's GPU cache (about 660k tokens) cannot hold more than about ten such +histories: cache hits fall from about 90% to 15% and answers wait 40–140 s. The B300 holds +23 million cache tokens, about 35 times more. In a real 190-agent coding run (10 lite-worker +machines × 19 Codex sessions, `fast`), one B300 carried about 170 requests at once with none +queued. The GPU cache was at most 37% full and 84% of 464 million input tokens came from the +prompt cache. The run wrote 12.5 million output tokens in 2 h 18 min for about $20 of GPU time. +Cold starts: Qwen 4–5 min (first 7.5 min); Nemotron about 9.5 min (weights from the volume plus +kernel compilation). + +## Use it + +Needs Python 3.10+ with `pip install modal` and a signed-in Modal profile +(`modal profile current`). The private folder holds the journal and, by default, the key; keep +it out of Git. + +```sh +factory-selfhost profiles +factory-selfhost deploy --dir ./private/model --profile nemotron --prefetch # CPU weight download first, then deploy and verify +factory-selfhost endpoint --dir ./private/model --out endpoint.json # provider description, no key +factory-selfhost key --dir ./private/model --out ./private/model-key # backend only +factory-selfhost stop --dir ./private/model # GPU off now; deploy again to resume +factory-selfhost retire --dir ./private/model # delete app, secret, volumes and key +``` + +```js +import { ModalSelfHostedModel, FactoryCloudLiteFleet } from 'flujo-factory'; + +const model = new ModalSelfHostedModel({ directory: '/absolute/private/model' }); +await model.deploy({ profile: 'qwen', overrides: { replicas: 2 } }); +const modelProvider = model.endpoint(); // validated provider description +await fleet.deploy({ ...plan, model: modelProvider.model, subagentModel: modelProvider.model, modelProvider }, + { image, bundlePath, providerKey: await model.key() }); +// ... +await model.retire(); +``` + +`deploy` creates an owned app name (`--<12 hex>`), checks the active Modal +profile and workspace, and generates a fresh key that only Modal's secret and your credential +store hold. It then deploys and proves each replica serves the expected model: the catalogue +lists it, and one non-thinking completion returns an exact challenge. Modal's 303 result links +are followed on the same origin only, so the POST is never repeated and the key never goes to +another host. Any uncertain remote outcome leaves the model `unknown`. It is never retried +automatically; `retire` reconciles it. `stop` and `retire` refuse a different Modal workspace, +and `retire` checks that the secret and both volumes are gone. +`adopt({ appName, profileName, workspaceName })` takes over an owned app that an earlier +version of your app created, in the `unknown` phase, so `retire` can clean it up. + +Overrides are explicit settings fields, for example `{ replicas: 2 }`, `{ variant: 'fast' }`, +`{ enable_thinking: false }` or `{ scaledown_minutes: 2 }`. Unknown fields, unpinned images and +settings that would not fit the GPU are refused before anything is created. + +## Things to know + +- vLLM's `--api-key` protects `/v1/*`; `/health` and `/metrics` answer without it (counters only). +- A request to a cold replica is held, not refused, until the container is up. +- Qwen3.8's chat template accepts reasoning efforts `low`, `medium` and `xhigh` and rejects + `high`. Nemotron's template ignores the effort; `enable_thinking` switches thinking. +- Spread workers over replicas (`baseUrls[i % n]`), so each worker keeps its prompt cache on one GPU. +- Offline checks: `python -B -m unittest discover -s selfhost/modal -p "test_*.py"` and + `node --test test/selfhost-modal.test.mjs`. Neither makes a cloud call. diff --git a/selfhost/modal/app.py b/selfhost/modal/app.py new file mode 100644 index 0000000..2390726 --- /dev/null +++ b/selfhost/modal/app.py @@ -0,0 +1,121 @@ +"""A self-hosted model on Modal, served by vLLM itself. + +vLLM answers /v1/responses (Codex), /v1/messages (Claude Code) and /v1/chat/completions +directly; there is no translation layer. Access is the bearer key in the `-auth` Modal +secret, enforced by vLLM's --api-key. Each replica is its own web function with at most one +GPU container, so the GPU count cannot run away; idle containers stop after the profile's +scale-down time and cost nothing while stopped. +""" +import os + +import modal + +from settings import from_env + +S = from_env() +MINUTES = 60 +HF_CACHE = '/root/.cache/huggingface' + +if S.image: + # A ready-made vLLM release image pinned by digest. Modal looks for `python`; the vLLM + # images only have `python3`. + base = (modal.Image.from_registry(S.image) + .dockerfile_commands(['RUN ln -sf "$(command -v python3)" /usr/local/bin/python && python --version']) + .entrypoint([])) +else: + base = (modal.Image.from_registry('nvidia/cuda:12.9.0-devel-ubuntu22.04', add_python='3.12') + .entrypoint([]) + .uv_pip_install(f'vllm=={S.vllm_version}')) +image = ( + base + .env({'HF_XET_HIGH_PERFORMANCE': '1', 'VLLM_ALLOW_LONG_MAX_MODEL_LEN': '1', + # Triton kernels compiled at start-up are kept in the vLLM cache volume. + 'TRITON_CACHE_DIR': '/root/.cache/vllm/triton', + # Frozen settings: the container serves what was deployed. + 'FACTORY_SELFHOST_SETTINGS_B64': S.to_b64()}) + .add_local_python_source('settings', 'discovery') + .add_local_file(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'prices.json'), '/root/prices.json') +) + +app = modal.App(S.app_name) +hf_cache = modal.Volume.from_name(S.app_name + '-hf-cache', create_if_missing=True) +vllm_cache = modal.Volume.from_name(S.app_name + '-vllm-cache', create_if_missing=True) +auth = modal.Secret.from_name(S.app_name + '-auth', required_keys=['VLLM_API_KEY']) +VOLUMES = {HF_CACHE: hf_cache, '/root/.cache/vllm': vllm_cache} + +# CPU helpers exist only in the short-lived app that the deploy script starts for a weight +# prefetch or a flag check (FACTORY_SELFHOST_HELPERS=1); the deployed app leaves them out, so +# nothing idle can fail or alert. +HELPERS = os.environ.get('FACTORY_SELFHOST_HELPERS') == '1' + + +def _helper(**kwargs): + return app.function(**kwargs) if HELPERS else (lambda f: f) + + +@_helper(image=image, cpu=4, memory=8192, volumes=VOLUMES, timeout=60 * MINUTES) +def download(): + """Fill the weights volume on a cheap CPU container (no GPU billed while downloading).""" + import time + from huggingface_hub import snapshot_download + began = time.monotonic() + snapshot_download(S.model, revision=S.revision) + hf_cache.commit() + return {'seconds': round(time.monotonic() - began)} + + +@_helper(image=image, cpu=2, memory=4096, timeout=10 * MINUTES) +def vllm_flags(words: list): + """Which of these flags does this vLLM know? Reads vLLM's source (its CLI needs a GPU).""" + import pathlib + import vllm + source = ' '.join(p.read_text(errors='ignore') for p in pathlib.Path(vllm.__file__).parent.rglob('*.py')) + return {word: word.lstrip('-').replace('-', '_') in source for word in words} + + +def _start_vllm(): + import subprocess + if len(os.environ['VLLM_API_KEY']) < 32: + raise RuntimeError('INFERENCE_AUTH_NOT_CONFIGURED') + # /root holds settings.py and discovery.py; vLLM imports the middleware by name. The + # profile's environment wins over what Modal sets (Modal sets OMP_NUM_THREADS to the CPUs). + subprocess.Popen(S.vllm_command(), env={**os.environ, **S.env, 'PYTHONPATH': '/root', + 'FACTORY_SELFHOST_INPUT_MODALITIES': ','.join(S.input_modalities)}) + + +GPU_FUNCTION = dict(image=image, gpu=S.gpu_spec, cpu=S.cpu, memory=S.memory_mib, secrets=[auth], + volumes=VOLUMES, scaledown_window=S.scaledown_minutes * MINUTES, + min_containers=0, max_containers=1, timeout=S.request_timeout_minutes * MINUTES) +# A web_server endpoint holds requests while a cold container starts. Modal's own auth stays +# off; vLLM checks the bearer key (VLLM_API_KEY). +WEB = dict(port=S.port, startup_timeout=S.startup_timeout_minutes * MINUTES) +CONCURRENT = dict(max_inputs=min(1000, S.max_num_seqs * 4)) # Modal's limit is 1000 + + +@app.function(**GPU_FUNCTION) +@modal.concurrent(**CONCURRENT) +@modal.web_server(**WEB) +def serve(): + _start_vllm() + + +if S.replicas >= 2: + @app.function(**GPU_FUNCTION) + @modal.concurrent(**CONCURRENT) + @modal.web_server(**WEB) + def serve_b(): + _start_vllm() + +if S.replicas >= 3: + @app.function(**GPU_FUNCTION) + @modal.concurrent(**CONCURRENT) + @modal.web_server(**WEB) + def serve_c(): + _start_vllm() + +if S.replicas >= 4: + @app.function(**GPU_FUNCTION) + @modal.concurrent(**CONCURRENT) + @modal.web_server(**WEB) + def serve_d(): + _start_vllm() diff --git a/selfhost/modal/deploy.py b/selfhost/modal/deploy.py new file mode 100644 index 0000000..9d61336 --- /dev/null +++ b/selfhost/modal/deploy.py @@ -0,0 +1,119 @@ +"""Deploy, prefetch, check or describe one owned self-hosted model app on Modal. + +Reads one JSON request on stdin and prints exactly one JSON line; Modal's own output is +captured and discarded, so neither the key nor a transcript reaches the caller: + + {"action": "describe", "appName": "...", "profile": "qwen", "overrides": {...}} + {"action": "prefetch", "appName": "...", "profile": "...", "overrides": {...}} CPU only: weights into the volume + {"action": "check-flags", "appName": "...", "profile": "...", "flags": ["--a", ...]} CPU only + {"action": "deploy", "appName": "...", "profile": "...", "overrides": {...}, "key": "...", "reuseSecret": false} + +`deploy` creates the `-auth` secret (never replacing an existing one unless +`reuseSecret` is set for a redeploy), runs `modal deploy app.py` and returns the replica URLs. +The active Modal profile selects the workspace. +""" +import contextlib +import io +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +from settings import SETTINGS_ENV, client_hints, resolve # noqa: E402 + +MAX_INPUT = 65536 + + +class Refused(Exception): + """A safe, caller-facing reason.""" + + +def read_request(): + raw = sys.stdin.read(MAX_INPUT + 1) + if len(raw) > MAX_INPUT: + raise Refused('The request is too large.') + request = json.loads(raw) + if not isinstance(request, dict) or request.get('action') not in ('describe', 'prefetch', 'check-flags', 'deploy'): + raise Refused('Unknown action.') + return request + + +def settings_for(request): + try: + return resolve(request.get('appName', ''), request.get('profile', ''), request.get('overrides') or {}) + except (ValueError, TypeError) as error: + raise Refused(str(error)) from None + + +def quiet_import_app(settings, helpers): + os.environ[SETTINGS_ENV] = settings.to_b64() + if helpers: + os.environ['FACTORY_SELFHOST_HELPERS'] = '1' + else: + os.environ.pop('FACTORY_SELFHOST_HELPERS', None) + import app # noqa: PLC0415 - reads the settings above at import + return app + + +def run_helper(settings, call): + app = quiet_import_app(settings, helpers=True) + sink = io.StringIO() + with contextlib.redirect_stdout(sink), contextlib.redirect_stderr(sink), app.app.run(): + return call(app) + + +def deploy(settings, request): + import modal + key = request.get('key') + if not isinstance(key, str) or not 32 <= len(key) <= 256: + raise Refused('The endpoint key is invalid.') + if not settings.estimate_gib()['fits']: + raise Refused('The model does not fit the selected GPU.') + secret = settings.app_name + '-auth' + if request.get('reuseSecret') is True: + modal.Secret.from_name(secret, required_keys=['VLLM_API_KEY']).hydrate() + else: + modal.Secret.objects.create(secret, {'VLLM_API_KEY': key}, allow_existing=False) + env = {**os.environ, 'PYTHONIOENCODING': 'utf-8', SETTINGS_ENV: settings.to_b64()} + env.pop('FACTORY_SELFHOST_HELPERS', None) + with tempfile.TemporaryFile() as out: + result = subprocess.run([sys.executable, '-B', '-m', 'modal', 'deploy', str(HERE / 'app.py')], + cwd=HERE, stdout=out, stderr=out, env=env, check=False) + if result.returncode: + raise Refused('The Modal deployment did not complete.') + urls = [modal.Function.from_name(settings.app_name, name).get_web_url() for name in settings.endpoint_functions()] + if len(urls) != settings.replicas or not all(isinstance(url, str) and url.startswith('https://') for url in urls): + raise Refused('The deployed endpoints could not be identified.') + return {'endpoints': urls, 'settings': settings.describe()} + + +def main(): + request = read_request() + settings = settings_for(request) + action = request['action'] + if action == 'describe': + return {'settings': settings.describe(), 'hints': client_hints(settings.profile)} + if action == 'prefetch': + return run_helper(settings, lambda app: app.download.remote()) + if action == 'check-flags': + flags = request.get('flags') + if not isinstance(flags, list) or not flags or not all(isinstance(flag, str) and flag.startswith('--') for flag in flags): + raise Refused('List the flags to check.') + return run_helper(settings, lambda app: app.vllm_flags.remote(flags)) + return deploy(settings, request) + + +if __name__ == '__main__': + try: + print(json.dumps(main())) + except Refused as error: + print(json.dumps({'error': str(error)})) + sys.exit(1) + except Exception as error: # noqa: BLE001 - never echo provider details that may hold names or keys + print(json.dumps({'error': f'The Modal operation did not complete ({type(error).__name__}).'})) + sys.exit(1) diff --git a/selfhost/modal/discovery.py b/selfhost/modal/discovery.py new file mode 100644 index 0000000..f8c95d4 --- /dev/null +++ b/selfhost/modal/discovery.py @@ -0,0 +1,51 @@ +"""Model-catalogue metadata for client auto-discovery. + +vLLM reports a model's size as `max_model_len`. Clients that discover models (OpenRouter- +style catalogues) read `context_length`, `max_completion_tokens`, `supported_parameters` and +`architecture`. This vLLM `--middleware` adds those fields to GET /v1/models; every other +request passes through untouched and unbuffered. +""" +import json +import os + +SUPPORTED_PARAMETERS = ['max_tokens', 'max_completion_tokens', 'temperature', 'top_p', 'top_k', + 'min_p', 'seed', 'stop', 'frequency_penalty', 'presence_penalty', + 'repetition_penalty', 'logit_bias', 'logprobs', 'top_logprobs', + 'response_format', 'structured_outputs', 'tools', 'tool_choice', + 'reasoning', 'include_reasoning'] + + +def input_modalities(env=None): + """What the deployment accepts (a text-only deployment must not advertise images).""" + env = os.environ if env is None else env + values = [value for value in env.get('FACTORY_SELFHOST_INPUT_MODALITIES', 'text').split(',') if value] + return tuple(values) or ('text',) + + +def enrich(catalogue, modalities=('text',)): + """Add discovery fields to an OpenAI model list. Existing fields are kept.""" + for model in catalogue.get('data', []): + limit = model.get('max_model_len') + if not isinstance(limit, int): + continue + model.setdefault('name', model.get('id')) + model.setdefault('context_length', limit) + model.setdefault('max_completion_tokens', limit) + model.setdefault('top_provider', {'context_length': limit, 'max_completion_tokens': limit}) + model.setdefault('supported_parameters', list(SUPPORTED_PARAMETERS)) + model.setdefault('architecture', {'input_modalities': list(modalities), 'output_modalities': ['text']}) + return catalogue + + +async def models_metadata(request, call_next): + response = await call_next(request) + if request.method != 'GET' or request.url.path.rstrip('/') != '/v1/models' or response.status_code != 200: + return response + from starlette.responses import Response + raw = b''.join([chunk async for chunk in response.body_iterator]) + try: + raw = json.dumps(enrich(json.loads(raw), input_modalities())).encode() + except ValueError: + pass + headers = {k: v for k, v in response.headers.items() if k.lower() != 'content-length'} + return Response(raw, status_code=200, headers=headers, media_type='application/json') diff --git a/selfhost/modal/prices.json b/selfhost/modal/prices.json new file mode 100644 index 0000000..66e22b2 --- /dev/null +++ b/selfhost/modal/prices.json @@ -0,0 +1,17 @@ +{ + "checkedAt": "2026-11-01", + "source": "https://modal.com/pricing", + "currency": "USD", + "note": "Modal list prices per second. Estimates only: storage, egress, image builds, credits and invoice adjustments are not included.", + "gpuPerSecond": { + "L40S": 0.000542, + "A100-80GB": 0.000694, + "RTX-PRO-6000": 0.000842, + "H100": 0.001097, + "H200": 0.001261, + "B200": 0.001736, + "B300": 0.001972 + }, + "cpuCorePerSecond": 0.0000131, + "memoryGibPerSecond": 0.00000222 +} diff --git a/selfhost/modal/profiles/nemotron.json b/selfhost/modal/profiles/nemotron.json new file mode 100644 index 0000000..98534a4 --- /dev/null +++ b/selfhost/modal/profiles/nemotron.json @@ -0,0 +1,47 @@ +{ + "comment": "NVIDIA Nemotron 3.5 Super VL 120B-A12B (Mamba-2 + attention hybrid MoE), NVIDIA's NVFP4 weights, text only, on one B300 (288 GB) with the vLLM 0.32.0 release image. Prefix caching for the hybrid model needs --mamba-cache-mode align. Variants: base (none), fast (async scheduling + FlashInfer Mamba kernel), mtp1 / mtp3 (fast + the model's multi-token prediction head), mtp1-fp16 (mtp1 + FP16 Mamba state with stochastic rounding, lower precision). Up to about 128 long agent conversations use mtp1; 128 to 200 use fast (full precision) or mtp1-fp16.", + "label": "Nemotron 3.5 Super VL NVFP4 on B300", + "model": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4", + "revision": "2e5cc3953bb1ea66b2fb7c831fee3aa317427e69", + "served_name": "nemotron-3.5-super-vl", + "weights_gib": 70, + "gpu": "B300", + "cpu": 12, + "memory_mib": 98304, + "max_model_len": 262144, + "kv_bytes_per_token": 4096, + "state_gib_per_sequence": 0.16, + "gpu_memory_utilization": 0.9, + "max_num_seqs": 256, + "enable_thinking": true, + "tool_call_parser": "qwen3_coder", + "reasoning_parser": "nemotron_v3", + "trust_remote_code": true, + "image": "vllm/vllm-openai@sha256:6ee133356d49ed24cee03dd0cda3d3980279a11c107ce3e752c89e76540f41c9", + "vllm_version": "0.32.0", + "env": {"OMP_NUM_THREADS": "1"}, + "input_modalities": ["text"], + "compact_tokens": 80000, + "efforts": ["low", "medium", "high"], + "extra_args": [ + "--max-num-batched-tokens", "16384", + "--block-size", "64", + "--prefix-match-unit", "64", + "--mamba-cache-mode", "align", + "--language-model-only", + "--enable-prompt-tokens-details" + ], + "variants": { + "base": ["--mamba-ssm-cache-dtype", "float32"], + "fast": ["--mamba-ssm-cache-dtype", "float32", "--async-scheduling", "--mamba-backend", "flashinfer"], + "mtp1": ["--mamba-ssm-cache-dtype", "float32", "--async-scheduling", "--mamba-backend", "flashinfer", + "--speculative-config", "{\"method\": \"mtp\", \"num_speculative_tokens\": 1}", "--enable-mamba-shared-prefix-checkpoint"], + "mtp3": ["--mamba-ssm-cache-dtype", "float32", "--async-scheduling", "--mamba-backend", "flashinfer", + "--speculative-config", "{\"method\": \"mtp\", \"num_speculative_tokens\": 3}", "--enable-mamba-shared-prefix-checkpoint"], + "mtp1-fp16": ["--mamba-ssm-cache-dtype", "float16", "--enable-mamba-cache-stochastic-rounding", "--mamba-cache-philox-rounds", "5", + "--async-scheduling", "--mamba-backend", "flashinfer", + "--speculative-config", "{\"method\": \"mtp\", \"num_speculative_tokens\": 1}", "--enable-mamba-shared-prefix-checkpoint"] + }, + "variant": "mtp1", + "scaledown_minutes": 8 +} diff --git a/selfhost/modal/profiles/qwen.json b/selfhost/modal/profiles/qwen.json new file mode 100644 index 0000000..cd0cb34 --- /dev/null +++ b/selfhost/modal/profiles/qwen.json @@ -0,0 +1,23 @@ +{ + "comment": "Qwen3.8-27B FP8 on one H100 80 GB, vLLM 0.30.0. Native 262k context (no YaRN stretching). Thinking on: the model card evaluates agentic coding with it. One H100 keeps about 8 long agent conversations at full speed; use more replicas for more agents.", + "label": "Qwen3.8-27B FP8 on H100", + "model": "Qwen/Qwen3.8-27B-FP8", + "revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a", + "served_name": "qwen3.8-27b", + "weights_gib": 30.9, + "gpu": "H100", + "cpu": 8, + "memory_mib": 65536, + "max_model_len": 262144, + "kv_bytes_per_token": 65536, + "gpu_memory_utilization": 0.92, + "max_num_seqs": 48, + "enable_thinking": true, + "tool_call_parser": "qwen3_xml", + "reasoning_parser": "qwen3", + "vllm_version": "0.30.0", + "input_modalities": ["text", "image"], + "compact_tokens": 80000, + "efforts": ["low", "medium", "xhigh"], + "scaledown_minutes": 10 +} diff --git a/selfhost/modal/settings.py b/selfhost/modal/settings.py new file mode 100644 index 0000000..60c5f8f --- /dev/null +++ b/selfhost/modal/settings.py @@ -0,0 +1,222 @@ +"""Settings for a self-hosted model served by vLLM on Modal. No Modal import: unit-testable. + +A deployment is a profile from `profiles/.json` plus explicit overrides, bound to one +owned Modal app name. At deploy time the resolved settings travel as base64 JSON in +FACTORY_SELFHOST_SETTINGS_B64 and are frozen into the image, so the container serves exactly +what was deployed. +""" +import base64 +import json +import os +import re +from dataclasses import asdict, dataclass, field, fields +from pathlib import Path + +HERE = Path(__file__).resolve().parent +PROFILES_DIR = HERE / 'profiles' +PRICES = json.loads((HERE / 'prices.json').read_text(encoding='utf-8')) +GPU_MEMORY_GIB = {'L40S': 48, 'A100-80GB': 80, 'RTX-PRO-6000': 96, 'H100': 80, 'H200': 141, 'B200': 180, 'B300': 288} +KV_DTYPES = ('auto', 'bf16', 'fp8') +# Each replica is its own web function (and URL) with at most one GPU container. +MAX_REPLICAS = 4 +REPLICA_FUNCTIONS = ('serve', 'serve_b', 'serve_c', 'serve_d') +# --<12 hex>; Modal names stay below 64 characters with the volume suffixes. +APP_NAME = re.compile(r'[a-z][a-z0-9]{1,15}-[a-z][a-z0-9]{1,15}-[a-f0-9]{12}') +PROFILE_NAME = re.compile(r'[a-z][a-z0-9]{1,15}') +SETTINGS_ENV = 'FACTORY_SELFHOST_SETTINGS_B64' +# Profile keys that describe the model for clients, not the server. +CLIENT_KEYS = ('comment', 'compact_tokens', 'efforts') + + +@dataclass(frozen=True) +class Settings: + app_name: str = 'factory-unconfigured-000000000000' + profile: str = 'qwen' + label: str = 'Qwen3.8-27B FP8 on H100' + model: str = 'Qwen/Qwen3.8-27B-FP8' + revision: str = '017b9c7af6b5689d5dd426a76e0bc077eb5ca20a' + served_name: str = 'qwen3.8-27b' + weights_gib: float = 30.9 + gpu: str = 'H100' + gpu_count: int = 1 + cpu: int = 8 + memory_mib: int = 65536 + max_model_len: int = 262_144 + kv_cache_dtype: str = 'auto' + # Attention cache bytes per token at the checkpoint's own cache precision ('auto'). + kv_bytes_per_token: int = 65536 + # Constant recurrent (Mamba / DeltaNet) state per running sequence, when it matters. + state_gib_per_sequence: float = 0.0 + gpu_memory_utilization: float = 0.92 + max_num_seqs: int = 48 + enable_thinking: bool = True + tool_call_parser: str = 'qwen3_xml' + reasoning_parser: str = 'qwen3' + extra_args: tuple = () + # More chat-template switches sent by default. + template_kwargs: dict = field(default_factory=dict) + # Named sets of extra vLLM flags and the one deployed. + variants: dict = field(default_factory=dict) + variant: str = '' + trust_remote_code: bool = False + # A ready-made vLLM image pinned by digest; empty = CUDA image + `pip install vllm`. + image: str = '' + vllm_version: str = '0.30.0' + # Extra environment variables for the vLLM process. + env: dict = field(default_factory=dict) + input_modalities: tuple = ('text',) + replicas: int = 1 + # Idle minutes before a GPU container stops (Modal allows 1..20). + scaledown_minutes: int = 10 + request_timeout_minutes: int = 60 + startup_timeout_minutes: int = 40 + port: int = 8000 + + def __post_init__(self): + if not APP_NAME.fullmatch(self.app_name): + raise ValueError('app_name must be --<12 hex>') + if not PROFILE_NAME.fullmatch(self.profile): + raise ValueError('profile must be a short lowercase name') + if self.gpu not in GPU_MEMORY_GIB or self.gpu not in PRICES['gpuPerSecond']: + raise ValueError(f'unknown gpu {self.gpu!r}; expected one of {sorted(GPU_MEMORY_GIB)}') + if not 1 <= self.gpu_count <= 8: + raise ValueError('gpu_count must be within 1..8') + if not 1024 <= self.max_model_len <= 1_000_000: + raise ValueError('max_model_len must be within 1024..1000000') + if self.kv_cache_dtype not in KV_DTYPES: + raise ValueError(f'kv_cache_dtype must be one of {KV_DTYPES}') + if not 0.5 <= self.gpu_memory_utilization <= 0.98: + raise ValueError('gpu_memory_utilization must be within 0.5..0.98') + if not 1 <= self.max_num_seqs <= 1024: + raise ValueError('max_num_seqs must be within 1..1024') + if not 1 <= self.replicas <= MAX_REPLICAS: + raise ValueError(f'replicas must be within 1..{MAX_REPLICAS}') + if not 1 <= self.scaledown_minutes <= 20: + raise ValueError('scaledown_minutes must be within 1..20 (Modal limit)') + if self.variant and self.variant not in self.variants: + raise ValueError(f'unknown variant {self.variant!r}; expected one of {sorted(self.variants)}') + if self.image and not re.fullmatch(r'[\w./-]+@sha256:[a-f0-9]{64}', self.image): + raise ValueError('image must be pinned by digest') + if not set(self.input_modalities) <= {'text', 'image'} or 'text' not in self.input_modalities: + raise ValueError('input_modalities must include text and may add image') + if not all(isinstance(arg, str) for arg in self.extra_args): + raise ValueError('extra_args must be strings') + + @property + def gpu_spec(self): + return self.gpu if self.gpu_count == 1 else f'{self.gpu}:{self.gpu_count}' + + def endpoint_functions(self): + """The web functions, one per replica; each has its own URL.""" + return list(REPLICA_FUNCTIONS[:self.replicas]) + + def usd_per_second(self): + """List price of one running replica: GPU plus its reserved CPU and memory.""" + return (PRICES['gpuPerSecond'][self.gpu] * self.gpu_count + self.cpu * PRICES['cpuCorePerSecond'] + + self.memory_mib / 1024 * PRICES['memoryGibPerSecond']) + + def usd_per_hour(self): + return round(self.usd_per_second() * 3600, 2) + + def estimate_gib(self): + """Memory for one request at the full context. A calculation, not a measurement.""" + per_token = self.kv_bytes_per_token / (2 if self.kv_cache_dtype == 'fp8' else 1) + kv = self.max_model_len * per_token / 2**30 + usable = GPU_MEMORY_GIB[self.gpu] * self.gpu_count * self.gpu_memory_utilization + total = self.weights_gib + kv + self.state_gib_per_sequence + return {'weights': self.weights_gib, 'kv_cache': round(kv, 1), 'state': self.state_gib_per_sequence, + 'total': round(total, 1), 'usable': round(usable, 1), 'fits': total <= usable} + + def chat_template_kwargs(self): + return {'enable_thinking': self.enable_thinking, **self.template_kwargs} + + def variant_args(self): + return list(self.variants.get(self.variant, ())) if self.variant else [] + + def vllm_command(self): + command = ['vllm', 'serve', self.model, '--revision', self.revision, + '--served-model-name', self.served_name, + '--host', '0.0.0.0', '--port', str(self.port), + '--max-model-len', str(self.max_model_len), + '--gpu-memory-utilization', str(self.gpu_memory_utilization), + '--kv-cache-dtype', self.kv_cache_dtype, + '--max-num-seqs', str(self.max_num_seqs), + '--enable-prefix-caching', '--middleware', 'discovery.models_metadata', + '--enable-auto-tool-choice', '--tool-call-parser', self.tool_call_parser, + '--reasoning-parser', self.reasoning_parser, + '--default-chat-template-kwargs', json.dumps(self.chat_template_kwargs())] + if self.trust_remote_code: + command += ['--trust-remote-code'] + if self.gpu_count > 1: + command += ['--tensor-parallel-size', str(self.gpu_count)] + return command + list(self.extra_args) + self.variant_args() + + def describe(self): + """Public description: no key, no URL.""" + return {'appName': self.app_name, 'profile': self.profile, 'label': self.label, 'model': self.served_name, + 'gpu': self.gpu_spec, 'replicas': self.replicas, 'functions': self.endpoint_functions(), + 'usdPerHourPerReplica': self.usd_per_hour(), 'usdPerSecondPerReplica': self.usd_per_second(), + 'scaledownMinutes': self.scaledown_minutes, 'startupTimeoutMinutes': self.startup_timeout_minutes, + 'requestTimeoutMinutes': self.request_timeout_minutes, + 'contextWindow': self.max_model_len, 'variant': self.variant or None, + 'inputModalities': list(self.input_modalities), 'estimateGib': self.estimate_gib(), + 'vllmCommand': self.vllm_command()} + + def to_json(self): + data = asdict(self) + data['extra_args'] = list(self.extra_args) + data['input_modalities'] = list(self.input_modalities) + return json.dumps(data, sort_keys=True) + + def to_b64(self): + # base64: flag values with quotes survive the image's ENV line. + return base64.b64encode(self.to_json().encode('utf-8')).decode('ascii') + + +def profile_data(name): + if not PROFILE_NAME.fullmatch(name or '') or not (PROFILES_DIR / f'{name}.json').is_file(): + raise ValueError(f'unknown profile {name!r}; expected one of {profile_names()}') + return json.loads((PROFILES_DIR / f'{name}.json').read_text(encoding='utf-8')) + + +def profile_names(): + return sorted(path.stem for path in PROFILES_DIR.glob('*.json')) + + +def client_hints(name): + """Profile hints for worker programs (compaction point, accepted effort levels).""" + data = profile_data(name) + return {key: data[key] for key in ('compact_tokens', 'efforts') if key in data} + + +def from_dict(data): + names = {f.name for f in fields(Settings)} + unknown = set(data) - names + if unknown: + raise ValueError(f'unknown settings: {sorted(unknown)}') + data = dict(data) + for key in ('extra_args', 'input_modalities'): + if key in data: + data[key] = tuple(data[key]) + return Settings(**data) + + +def resolve(app_name, profile, overrides=None): + """The profile's settings with explicit overrides, bound to an owned app name.""" + data = {key: value for key, value in profile_data(profile).items() if key not in CLIENT_KEYS} + overrides = dict(overrides or {}) + for key in ('app_name', 'profile'): + if key in overrides: + raise ValueError(f'{key} cannot be overridden') + if not app_name.split('-')[1] == profile: + raise ValueError('the owned app name must carry its profile') + return from_dict({**data, **overrides, 'app_name': app_name, 'profile': profile}) + + +def from_env(env=None): + """Inside the container (and during `modal deploy`): the frozen settings.""" + env = os.environ if env is None else env + raw = env.get(SETTINGS_ENV) + if not raw: + raise ValueError(f'{SETTINGS_ENV} is required') + return from_dict(json.loads(base64.b64decode(raw).decode('utf-8'))) diff --git a/selfhost/modal/test_settings.py b/selfhost/modal/test_settings.py new file mode 100644 index 0000000..65826c1 --- /dev/null +++ b/selfhost/modal/test_settings.py @@ -0,0 +1,127 @@ +"""Offline checks for the self-hosted model settings. No Modal import, no cloud call. + + python -B -m unittest discover -s selfhost/modal -p "test_*.py" +""" +import base64 +import json +import subprocess +import sys +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import discovery # noqa: E402 +from settings import (MAX_REPLICAS, SETTINGS_ENV, client_hints, from_env, profile_names, # noqa: E402 + resolve) + +QWEN = 'factory-qwen-0123456789ab' +NEMOTRON = 'factory-nemotron-0123456789ab' + + +class Profiles(unittest.TestCase): + def test_both_profiles_exist_and_fit_their_gpu(self): + self.assertEqual(profile_names(), ['nemotron', 'qwen']) + for name, app in (('qwen', QWEN), ('nemotron', NEMOTRON)): + estimate = resolve(app, name).estimate_gib() + self.assertTrue(estimate['fits'], (name, estimate)) + + def test_qwen_command_uses_its_parsers_without_remote_code(self): + s = resolve(QWEN, 'qwen') + command = s.vllm_command() + self.assertEqual(command[:3], ['vllm', 'serve', 'Qwen/Qwen3.8-27B-FP8']) + for pair in (('--tool-call-parser', 'qwen3_xml'), ('--reasoning-parser', 'qwen3'), ('--max-model-len', '262144'), + ('--served-model-name', 'qwen3.8-27b')): + self.assertEqual(command[command.index(pair[0]) + 1], pair[1]) + self.assertIn('--enable-prefix-caching', command) + self.assertNotIn('--trust-remote-code', command) + self.assertEqual(json.loads(command[command.index('--default-chat-template-kwargs') + 1]), {'enable_thinking': True}) + self.assertEqual(s.gpu, 'H100') + self.assertEqual(s.usd_per_hour(), 4.84) + self.assertEqual(client_hints('qwen'), {'compact_tokens': 80000, 'efforts': ['low', 'medium', 'xhigh']}) + + def test_nemotron_command_uses_the_release_image_mtp_and_hybrid_prefix_caching(self): + s = resolve(NEMOTRON, 'nemotron') + command = s.vllm_command() + self.assertEqual(s.gpu, 'B300') + self.assertRegex(s.image, r'^vllm/vllm-openai@sha256:[a-f0-9]{64}$') + self.assertEqual(s.variant, 'mtp1') + for pair in (('--tool-call-parser', 'qwen3_coder'), ('--reasoning-parser', 'nemotron_v3'), + ('--mamba-cache-mode', 'align'), ('--mamba-backend', 'flashinfer'), ('--mamba-ssm-cache-dtype', 'float32')): + self.assertEqual(command[command.index(pair[0]) + 1], pair[1]) + for flag in ('--trust-remote-code', '--language-model-only', '--async-scheduling', '--enable-mamba-shared-prefix-checkpoint'): + self.assertIn(flag, command) + self.assertEqual(json.loads(command[command.index('--speculative-config') + 1]), {'method': 'mtp', 'num_speculative_tokens': 1}) + self.assertEqual(s.env, {'OMP_NUM_THREADS': '1'}) + self.assertEqual(s.input_modalities, ('text',)) + self.assertEqual(s.usd_per_hour(), 8.43) + + def test_variants_change_only_the_speed_flags(self): + fast = resolve(NEMOTRON, 'nemotron', {'variant': 'fast'}).vllm_command() + self.assertNotIn('--speculative-config', fast) + self.assertIn('--async-scheduling', fast) + fp16 = resolve(NEMOTRON, 'nemotron', {'variant': 'mtp1-fp16'}).vllm_command() + self.assertEqual(fp16[fp16.index('--mamba-ssm-cache-dtype') + 1], 'float16') + self.assertIn('--enable-mamba-cache-stochastic-rounding', fp16) + + def test_overrides_are_explicit_and_validated(self): + s = resolve(QWEN, 'qwen', {'enable_thinking': False, 'replicas': 2, 'scaledown_minutes': 1}) + self.assertEqual(s.endpoint_functions(), ['serve', 'serve_b']) + self.assertEqual(json.loads(s.vllm_command()[s.vllm_command().index('--default-chat-template-kwargs') + 1]), {'enable_thinking': False}) + bad = [{'gpu': 'TPU'}, {'replicas': MAX_REPLICAS + 1}, {'scaledown_minutes': 30}, {'variant': 'turbo'}, + {'unknown': 1}, {'app_name': NEMOTRON}, {'image': 'vllm/vllm-openai:latest'}, {'input_modalities': ['image']}] + for override in bad: + with self.assertRaises(ValueError, msg=override): + resolve(QWEN, 'qwen', override) + with self.assertRaises(ValueError): + resolve(NEMOTRON, 'qwen') + with self.assertRaises(ValueError): + resolve('Bad-Name', 'qwen') + with self.assertRaises(ValueError): + resolve(QWEN, 'missing') + + def test_frozen_settings_round_trip_through_the_image_environment(self): + s = resolve(NEMOTRON, 'nemotron', {'replicas': 2}) + restored = from_env({SETTINGS_ENV: s.to_b64()}) + self.assertEqual(restored, s) + self.assertNotIn('"', s.to_b64()) + self.assertIn('num_speculative_tokens', base64.b64decode(s.to_b64()).decode()) + with self.assertRaises(ValueError): + from_env({}) + + +class Discovery(unittest.TestCase): + def test_catalogue_gets_context_length_and_honest_modalities(self): + catalogue = discovery.enrich({'data': [{'id': 'm', 'max_model_len': 262144}]}, discovery.input_modalities({'FACTORY_SELFHOST_INPUT_MODALITIES': 'text'})) + model = catalogue['data'][0] + self.assertEqual(model['context_length'], 262144) + self.assertEqual(model['architecture']['input_modalities'], ['text']) + self.assertIn('tools', model['supported_parameters']) + + +class DeployScript(unittest.TestCase): + def run_script(self, request): + return subprocess.run([sys.executable, '-B', str(HERE / 'deploy.py')], input=json.dumps(request), capture_output=True, + text=True, check=False) + + def test_describe_needs_no_modal_and_prints_one_json_line(self): + result = self.run_script({'action': 'describe', 'appName': NEMOTRON, 'profile': 'nemotron', 'overrides': {'variant': 'fast'}}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + lines = result.stdout.strip().splitlines() + self.assertEqual(len(lines), 1) + described = json.loads(lines[0]) + self.assertEqual(described['settings']['model'], 'nemotron-3.5-super-vl') + self.assertEqual(described['settings']['variant'], 'fast') + self.assertEqual(described['hints']['compact_tokens'], 80000) + + def test_refusals_are_one_safe_json_line(self): + result = self.run_script({'action': 'describe', 'appName': QWEN, 'profile': 'qwen', 'overrides': {'gpu': 'TPU'}}) + self.assertEqual(result.returncode, 1) + self.assertIn('unknown gpu', json.loads(result.stdout)['error']) + result = self.run_script({'action': 'erase'}) + self.assertEqual(json.loads(result.stdout), {'error': 'Unknown action.'}) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/public-sdk.mjs b/src/public-sdk.mjs index c75ef1a..cc052a8 100644 --- a/src/public-sdk.mjs +++ b/src/public-sdk.mjs @@ -97,6 +97,7 @@ export { observeSaviaCaseTopology } from './flujo-swarm/template/savia-topology. export { createPresentationServer, startPresentationServer } from './presentation.mjs'; export { createObservatoryClient } from './observatory-client.mjs'; export { liteWorkerCommand, startLiteWorker } from './lite-worker.mjs'; +export { ModalSelfHostedModel, selfHostedProfiles, probeSelfHostedModel, modalResult, readBoundedJson, modalEndpointOrigin, runCommand, SELFHOST_MODAL_DIR, SELFHOSTED_KEY_ENV } from './selfhost/modal.mjs'; export { MODEL_PROVIDER_APIS, WORKER_PROGRAM_APIS, validateModelProvider, providerPrograms, codexProviderConfig, codexProviderOverrides, claudeProviderEnvironment } from './model-provider.mjs'; export { FactoryCloudLiteFleet, CodexSubscriptionBroker } from './cloud-lite/fleet.mjs'; export { validateCloudLitePlan, agentInventory, machineConfig as cloudLiteMachineConfig } from './cloud-lite/protocol.mjs'; diff --git a/src/selfhost/modal.mjs b/src/selfhost/modal.mjs new file mode 100644 index 0000000..8b24d6c --- /dev/null +++ b/src/selfhost/modal.mjs @@ -0,0 +1,421 @@ +// A self-hosted model on Modal (vLLM), owned and journaled by FACTORY. The Python sources in +// selfhost/modal/ define the Modal app; this class runs them with the caller's Python and +// Modal profile, verifies the served model, and stops or retires exactly the owned resources. +import { randomBytes, randomUUID } from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { validateModelProvider } from '../model-provider.mjs'; + +export const SELFHOST_MODAL_DIR = fileURLToPath(new URL('../../selfhost/modal/', import.meta.url)); +export const SELFHOSTED_KEY_ENV = 'FACTORY_SELFHOSTED_MODEL_KEY'; +const SOURCES = ['app.py', 'settings.py', 'discovery.py', 'deploy.py', 'prices.json']; +const OWNED_NAME = /^[a-z][a-z0-9]{1,15}-[a-z][a-z0-9]{1,15}-[a-f0-9]{12}$/; +const MINUTE = 60_000; + +const notApplied = message => Object.assign(new Error(message), { outcome: 'not_applied' }); +const unknown = message => Object.assign(new Error(message), { code: 'UNKNOWN', outcome: 'unknown', unknown: true }); +const uncertain = () => unknown('The self-hosted model has an uncertain outcome. Its owned resources are kept for reconciliation; retire them before deploying again.'); +const json = raw => { try { return JSON.parse(raw); } catch { return null; } }; +const lastJsonLine = raw => json(String(raw ?? '').trim().split(/\r?\n/).reverse().find(line => line.trim().startsWith('{')) ?? ''); + +/** The bundled model profiles with list-price estimates per running replica. */ +export function selfHostedProfiles() { + const prices = JSON.parse(readFileSync(join(SELFHOST_MODAL_DIR, 'prices.json'), 'utf8')); + return readdirSync(join(SELFHOST_MODAL_DIR, 'profiles')).filter(name => name.endsWith('.json')).sort().map(name => { + const profile = JSON.parse(readFileSync(join(SELFHOST_MODAL_DIR, 'profiles', name), 'utf8')); + const cpu = profile.cpu ?? 8, memoryMib = profile.memory_mib ?? 65536, gpuCount = profile.gpu_count ?? 1; + const usdPerSecond = prices.gpuPerSecond[profile.gpu] * gpuCount + cpu * prices.cpuCorePerSecond + memoryMib / 1024 * prices.memoryGibPerSecond; + return { id: name.slice(0, -5), label: profile.label, model: profile.served_name, source: profile.model, gpu: profile.gpu, gpuCount, cpu, memoryMib, + contextWindow: profile.max_model_len, autoCompactTokenLimit: profile.compact_tokens ?? null, efforts: profile.efforts ?? null, + variant: profile.variant ?? null, variants: Object.keys(profile.variants ?? {}), scaledownMinutes: profile.scaledown_minutes ?? 10, + usdPerSecond, usdPerHour: Math.round(usdPerSecond * 3600 * 100) / 100, pricesCheckedAt: prices.checkedAt, estimate: 'list price, not an invoice' }; + }); +} + +/** An owned Modal web endpoint: https, Modal's domains, origin only. */ +export function modalEndpointOrigin(raw) { + try { + const url = new URL(raw); + return url.protocol === 'https:' && !url.username && !url.password && !url.port + && (url.hostname.endsWith('.modal.run') || url.hostname.endsWith('.modal.direct')) + && (url.pathname === '/' || url.pathname === '') && !url.search && !url.hash ? url.origin : null; + } catch { return null; } +} + +/** + * Fetch from a Modal web endpoint. Modal answers a long request with 303 and a result URL on + * the same origin and path; follow at most 20 of them with GET, never replaying a POST and + * never sending the key to another origin. + */ +export async function modalResult(fetchImpl, url, options, controller) { + const original = new URL(url); + let current = original.href; + let request = { ...options, signal: controller.signal, redirect: 'manual' }; + for (let redirects = 0; redirects <= 20; redirects++) { + if (controller.signal.aborted) throw uncertain(); + const response = await fetchImpl(current, request); + if (response.status !== 303) return response; + let next; + try { next = new URL(response.headers?.get('location'), current); } catch { controller.abort(); throw uncertain(); } + if (redirects === 20 || next.origin !== original.origin || next.pathname !== original.pathname || !next.search || next.hash || next.username || next.password) { + controller.abort(); throw uncertain(); + } + response.body?.cancel?.().catch(() => {}); + current = next.href; + request = { method: 'GET', headers: { authorization: options.headers.authorization }, signal: controller.signal, redirect: 'manual' }; + } + controller.abort(); throw uncertain(); +} + +/** Read a JSON body up to maxBytes; a larger or broken body aborts the request. */ +export async function readBoundedJson(response, maxBytes, controller) { + if (!response?.body || typeof response.body.getReader !== 'function') throw uncertain(); + const reader = response.body.getReader(); + const chunks = []; let bytes = 0, complete = false; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) { complete = true; break; } + if (!(value instanceof Uint8Array)) throw uncertain(); + bytes += value.byteLength; + if (bytes > maxBytes) throw uncertain(); + chunks.push(value); + } + return json(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks, bytes))); + } catch { controller.abort(); throw uncertain(); } + finally { + if (!complete) { try { reader.cancel().catch(() => {}); } catch { /* already aborted */ } } + reader.releaseLock(); + } +} + +/** + * Prove an endpoint serves the expected model: its catalogue lists it and one short, non- + * thinking completion returns an exact challenge with finish_reason "stop". + */ +export async function probeSelfHostedModel({ fetchImpl = fetch, endpoint, key, model, challenge, signal, timeoutMs = 45 * MINUTE }) { + const controller = new AbortController(); + const abort = () => controller.abort(); + if (signal?.aborted) controller.abort(); + signal?.addEventListener('abort', abort, { once: true }); + const timer = setTimeout(abort, timeoutMs); + try { + const catalogue = await modalResult(fetchImpl, `${endpoint}/v1/models`, { headers: { authorization: `Bearer ${key}` } }, controller); + if (!catalogue.ok) { controller.abort(); throw uncertain(); } + const models = (await readBoundedJson(catalogue, 64_000, controller))?.data; + if (!Array.isArray(models) || !models.some(item => item?.id === model)) throw uncertain(); + const catalogVerifiedAt = new Date().toISOString(); + const completion = await modalResult(fetchImpl, `${endpoint}/v1/chat/completions`, { method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` }, + body: JSON.stringify({ model, temperature: 0, max_tokens: 48, stream: false, chat_template_kwargs: { enable_thinking: false }, + messages: [{ role: 'user', content: `Reply with exactly this text and nothing else: ${challenge}` }] }) }, controller); + if (!completion.ok) { controller.abort(); throw uncertain(); } + const result = await readBoundedJson(completion, 64_000, controller); + if (controller.signal.aborted) throw uncertain(); + const choice = result?.choices?.[0]; + if (result?.model !== model || !Array.isArray(result?.choices) || result.choices.length !== 1 || choice?.message?.role !== 'assistant' + || typeof choice.message.content !== 'string' || choice.message.content.trim() !== challenge || choice.finish_reason !== 'stop') throw uncertain(); + return { model, catalogVerifiedAt, completionVerifiedAt: new Date().toISOString() }; + } finally { clearTimeout(timer); signal?.removeEventListener('abort', abort); } +} + +/** Run a command with stdin input and bounded output; Stop and timeouts end its process tree. */ +export function runCommand(command, args, { input = '', env = process.env, cwd, timeoutMs = 10 * MINUTE, signal, maxBytes = 1_000_000 } = {}) { + return new Promise((resolve, reject) => { + const posix = process.platform !== 'win32'; + const child = spawn(command, args, { cwd, env, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true, detached: posix }); + let stdout = '', stderr = '', size = 0, settled = false; + const killTree = () => { + try { + if (posix) process.kill(-child.pid, 'SIGKILL'); + else spawn(join(process.env.SystemRoot ?? 'C:\\Windows', 'System32', 'taskkill.exe'), ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, stdio: 'ignore' }); + } catch { child.kill('SIGKILL'); } + }; + const finish = action => { if (settled) return; settled = true; clearTimeout(timer); signal?.removeEventListener('abort', onAbort); action(); }; + const fail = message => { killTree(); finish(() => reject(new Error(message))); }; + const onAbort = () => fail('The command was stopped.'); + const timer = setTimeout(() => fail('The command timed out.'), timeoutMs); + if (signal?.aborted) { onAbort(); return; } + signal?.addEventListener('abort', onAbort, { once: true }); + const collect = target => chunk => { + size += chunk.length; + if (size > maxBytes) return fail('The command output exceeded its limit.'); + if (target === 'out') stdout += chunk; else stderr += chunk; + }; + child.stdout.on('data', collect('out')); child.stderr.on('data', collect('err')); + child.on('error', error => finish(() => reject(error))); + child.on('close', code => finish(() => resolve({ code, stdout, stderr }))); + child.stdin.on('error', () => {}); + child.stdin.end(input); + }); +} + +function fileCredentialStore(directory) { + const file = name => join(directory, `${name}.key`); + return { + get: async name => existsSync(file(name)) ? readFileSync(file(name), 'utf8') : null, + set: async (name, value) => { mkdirSync(directory, { recursive: true, mode: 0o700 }); writeFileSync(file(name), value, { mode: 0o600 }); }, + delete: async name => rmSync(file(name), { force: true }), + }; +} + +/** + * One owned self-hosted model on Modal. Phases: prepared → deploying → unverified → ready, + * stopped (GPU off, resources kept), retired (resources deleted and checked), unknown (an + * uncertain remote outcome; retire to reconcile). Uncertain operations are never replayed. + */ +export class ModalSelfHostedModel { + constructor({ directory, python = process.platform === 'win32' ? 'python' : 'python3', pythonArgs = [], env = process.env, + commandRunner = runCommand, fetchImpl = fetch, credentialStore, keyName = 'selfhosted-model', namePrefix = 'factory', + clock = Date.now, deployTimeoutMs = 45 * MINUTE, prefetchTimeoutMs = 70 * MINUTE } = {}) { + if (typeof directory !== 'string' || !directory) throw new TypeError('Select a private state directory.'); + if (!/^[a-z][a-z0-9]{1,15}$/.test(namePrefix)) throw new TypeError('namePrefix must be a short lowercase name.'); + Object.assign(this, { directory, python, pythonArgs: [...pythonArgs], commandRunner, fetch: fetchImpl, keyName, namePrefix, clock, deployTimeoutMs, prefetchTimeoutMs }); + this.environment = typeof env === 'function' ? env : () => env; + this.store = credentialStore ?? fileCredentialStore(directory); + this.path = join(directory, 'model.json'); + this.busy = false; + this.record = null; + if (existsSync(this.path)) { + const record = json(readFileSync(this.path, 'utf8')); + this.record = record?.version === 1 && OWNED_NAME.test(record.appName ?? '') ? record : { version: 1, phase: 'unknown' }; + } + } + + #save(record) { + mkdirSync(this.directory, { recursive: true, mode: 0o700 }); + const next = { ...record, updatedAt: new Date(this.clock()).toISOString() }; + const temporary = `${this.path}.${randomUUID()}.tmp`; + writeFileSync(temporary, JSON.stringify(next), { mode: 0o600 }); + renameSync(temporary, this.path); + this.record = next; + return next; + } + + #env() { + const env = { ...this.environment() }; + for (const name of Object.keys(env)) if (/^FACTORY_SELFHOST_/i.test(name)) delete env[name]; + if (this.record?.profileName) env.MODAL_PROFILE = this.record.profileName; + return env; + } + + #cli(args, { signal, timeoutMs = MINUTE } = {}) { + return this.commandRunner(this.python, [...this.pythonArgs, '-m', 'modal', ...args], { signal, timeoutMs, maxBytes: 1_000_000, env: this.#env(), killTree: true }); + } + + async #script(request, { signal, timeoutMs }) { + const source = join(this.directory, 'source'); + mkdirSync(source, { recursive: true, mode: 0o700 }); + for (const name of SOURCES) copyFileSync(join(SELFHOST_MODAL_DIR, name), join(source, name)); + const result = await this.commandRunner(this.python, [...this.pythonArgs, '-B', join(source, 'deploy.py')], + { signal, timeoutMs, input: JSON.stringify(request), maxBytes: 1_000_000, env: this.#env(), killTree: true, cwd: source }); + const answer = lastJsonLine(result?.stdout); + if (result?.code !== 0 || !answer || answer.error) return { error: answer?.error ?? 'The self-hosted model script did not complete.' }; + return answer; + } + + async #account(signal) { + const result = await this.#cli(['profile', 'list', '--json'], { signal, timeoutMs: 20_000 }); + const profiles = json(result?.stdout); + const active = Array.isArray(profiles) ? profiles.filter(item => item?.active === true) : []; + if (result?.code !== 0 || active.length !== 1 || typeof active[0].name !== 'string' || !active[0].name + || typeof active[0].workspace !== 'string' || !active[0].workspace || /^unknown/i.test(active[0].workspace)) { + throw notApplied('The Modal profile and workspace could not be verified.'); + } + return { profileName: active[0].name, workspaceName: active[0].workspace }; + } + + /** Safe state: no key, URL or app name. */ + status() { + const r = this.record, s = r?.settings; + const phase = r?.phase ?? 'off'; + return { phase, ready: phase === 'ready' && !this.busy, busy: this.busy, profile: r?.profile ?? null, label: s?.label ?? null, model: s?.model ?? null, + gpu: s?.gpu ?? null, replicas: s?.replicas ?? null, endpoints: r?.endpoints?.length ?? 0, contextWindow: s?.contextWindow ?? null, + usdPerHourPerReplica: s?.usdPerHourPerReplica ?? null, scaledownMinutes: s?.scaledownMinutes ?? null, + verified: Boolean(r?.verification && phase === 'ready'), cleanupVerified: r?.cleanupVerified === true, updatedAt: r?.updatedAt ?? null }; + } + + /** Deploy a profile (or redeploy a stopped model) and, by default, verify it serves the model. */ + async deploy({ profile = 'qwen', overrides = {}, signal, verify = true, prefetch = false } = {}) { + if (this.busy) throw notApplied('The self-hosted model is already changing.'); + if (signal?.aborted) throw notApplied('Deployment was cancelled before it started.'); + if (!/^[a-z][a-z0-9]{1,15}$/.test(profile) || !overrides || typeof overrides !== 'object' || Array.isArray(overrides)) throw notApplied('Select a profile and an overrides object.'); + this.busy = true; + try { + let record = this.record; + const redeploy = record?.phase === 'stopped'; + if (record && !['retired', 'stopped'].includes(record.phase)) { + throw notApplied(record.phase === 'unknown' ? 'The owned model has an uncertain outcome; retire it to reconcile first.' : `The owned model is ${record.phase}.`); + } + if (redeploy && (record.profile !== profile || JSON.stringify(record.overrides ?? {}) !== JSON.stringify(overrides))) { + throw notApplied('A stopped model keeps its profile and settings; retire it to change them.'); + } + if (!redeploy) { + const appName = `${this.namePrefix}-${profile}-${randomBytes(6).toString('hex')}`; + record = this.#save({ version: 1, attemptId: randomUUID(), appName, profile, overrides, phase: 'prepared', createdAt: new Date(this.clock()).toISOString() }); + try { + const described = await this.#script({ action: 'describe', appName, profile, overrides }, { signal, timeoutMs: MINUTE }); + if (described.error) throw notApplied(described.error); + const account = await this.#account(signal); + await this.store.set(this.keyName, randomBytes(48).toString('base64url')); + record = this.#save({ ...this.record, ...account, settings: described.settings, hints: described.hints ?? {} }); + if (signal?.aborted) throw notApplied('Deployment was cancelled before remote work started.'); + } catch (error) { + // Nothing remote exists yet: retire the local preparation. + await this.store.delete(this.keyName)?.catch?.(() => {}); + this.#save({ ...this.record, phase: 'retired', cleanupVerified: true }); + throw error.outcome ? error : notApplied('The self-hosted model could not be prepared.'); + } + } + this.#save({ ...record, phase: 'deploying', verification: undefined }); + try { + const key = await this.store.get(this.keyName); + if (typeof key !== 'string' || key.length < 32) throw uncertain(); + const request = { appName: record.appName, profile: record.profile, overrides: record.overrides ?? {} }; + if (prefetch && !redeploy) { + const fetched = await this.#script({ ...request, action: 'prefetch' }, { signal, timeoutMs: this.prefetchTimeoutMs }); + if (fetched.error) throw uncertain(); + } + const result = await this.#script({ ...request, action: 'deploy', key, reuseSecret: redeploy }, { signal, timeoutMs: this.deployTimeoutMs }); + const endpoints = Array.isArray(result.endpoints) ? result.endpoints.map(modalEndpointOrigin) : []; + if (result.error || !endpoints.length || endpoints.some(value => !value) || endpoints.length !== record.settings.replicas) throw uncertain(); + this.#save({ ...this.record, phase: 'unverified', endpoints, deployedAt: new Date(this.clock()).toISOString() }); + } catch { this.#save({ ...this.record, phase: 'unknown' }); throw uncertain(); } + if (verify) await this.#verify(signal); + return this.status(); + } finally { this.busy = false; } + } + + async #verify(signal) { + const record = this.record; + try { + const key = await this.store.get(this.keyName); + const timeoutMs = ((record.settings?.startupTimeoutMinutes ?? 40) + 5) * MINUTE; + let verification; + for (const endpoint of record.endpoints) { + verification = await probeSelfHostedModel({ fetchImpl: this.fetch, endpoint, key, model: record.settings.model, + challenge: `FACTORY-${randomBytes(4).toString('hex').toUpperCase()}`, signal, timeoutMs }); + } + this.#save({ ...this.record, phase: 'ready', verification: { ...verification, version: 1 } }); + } catch { this.#save({ ...this.record, phase: 'unknown' }); throw uncertain(); } + } + + /** Re-check an unverified deployment. */ + async verify({ signal } = {}) { + if (this.busy) throw notApplied('The self-hosted model is already changing.'); + if (!['unverified', 'ready'].includes(this.record?.phase)) throw notApplied('Deploy the model before verifying it.'); + this.busy = true; + try { await this.#verify(signal); return this.status(); } finally { this.busy = false; } + } + + /** A model-provider description for workers (see validateModelProvider); never the key. */ + endpoint() { + const r = this.record; + if (r?.phase !== 'ready' || !r.verification) throw notApplied('The self-hosted model is not verified and ready.'); + const s = r.settings; + return validateModelProvider({ id: `selfhosted-${r.profile}`, name: s.label, model: s.model, envKey: SELFHOSTED_KEY_ENV, + baseUrls: r.endpoints.map(origin => `${origin}/v1`), apis: ['responses', 'chat', 'anthropic'], contextWindow: s.contextWindow, + ...(r.hints?.compact_tokens ? { autoCompactTokenLimit: r.hints.compact_tokens } : {}), + profile: r.profile, gpu: s.gpu, replicas: s.replicas, usdPerHourPerReplica: s.usdPerHourPerReplica, scaledownMinutes: s.scaledownMinutes }); + } + + /** The endpoint key (backend only). */ + async key() { + if (!['ready', 'unverified', 'stopped'].includes(this.record?.phase)) throw notApplied('No owned self-hosted model is deployed.'); + return this.store.get(this.keyName); + } + + async #confirmAccount(signal) { + const record = this.record; + const account = await this.#account(signal); + if (account.profileName !== record.profileName || account.workspaceName !== record.workspaceName) { + throw notApplied('The active Modal profile or workspace changed; select the original one first.'); + } + } + + // app list --json: the owned app's state, or null when Modal no longer lists it. + async #appState(signal) { + const listed = await this.#cli(['app', 'list', '--json'], { signal }); + const apps = json(listed?.stdout); + if (listed?.code !== 0 || !Array.isArray(apps)) throw uncertain(); + const own = apps.filter(item => item?.description === this.record.appName); + return own.length ? own.map(item => String(item.state ?? '')) : null; + } + + async #stopApp(signal) { + const stopped = await this.#cli(['app', 'stop', '--yes', this.record.appName], { signal }); + const states = await this.#appState(signal); + // Stopped (now or earlier), or never created after an early deployment failure, is a + // confirmed GPU-off state; anything else stays uncertain. + if (states && !states.every(state => /^stopped$/i.test(state))) throw uncertain(); + return { stopCommand: stopped?.code === 0 ? 'ok' : 'refused', listed: Boolean(states) }; + } + + /** Stop the GPU containers now; key, secret and cached weights stay for a later redeploy. */ + async stop({ signal } = {}) { + if (this.busy) throw notApplied('The self-hosted model is already changing.'); + if (!['ready', 'unverified', 'unknown'].includes(this.record?.phase) || !OWNED_NAME.test(this.record?.appName ?? '')) throw notApplied('No owned self-hosted model is running.'); + this.busy = true; + try { + await this.#confirmAccount(signal); + try { await this.#stopApp(signal); } catch { this.#save({ ...this.record, phase: 'unknown' }); throw uncertain(); } + this.#save({ ...this.record, phase: 'stopped', verification: undefined, stoppedAt: new Date(this.clock()).toISOString() }); + return this.status(); + } finally { this.busy = false; } + } + + /** + * Take over an owned app that another version of the caller created, so it can be retired. + * The model starts as unknown: it is never trusted for work without a fresh deployment. + */ + adopt({ appName, profileName, workspaceName, endpoints = [] } = {}) { + if (this.busy) throw notApplied('The self-hosted model is already changing.'); + if (this.record && this.record.phase !== 'retired') throw notApplied('An owned model is already journaled here.'); + if (!OWNED_NAME.test(appName ?? '') || !appName.startsWith(`${this.namePrefix}-`)) throw notApplied('Only an app with this owner prefix can be adopted.'); + if (typeof profileName !== 'string' || !profileName || typeof workspaceName !== 'string' || !workspaceName) { + throw notApplied('Adopting needs the original Modal profile and workspace.'); + } + const origins = Array.isArray(endpoints) ? endpoints.map(modalEndpointOrigin) : [null]; + if (origins.some(origin => !origin)) throw notApplied('Adopted endpoints must be owned Modal origins.'); + const at = new Date(this.clock()).toISOString(); + this.#save({ version: 1, attemptId: randomUUID(), appName, profile: appName.split('-')[1], overrides: {}, phase: 'unknown', + profileName, workspaceName, endpoints: origins, createdAt: at, adoptedAt: at }); + return this.status(); + } + + /** Stop and delete every owned resource (app, secret, both volumes, local key) and check it. */ + async retire({ signal } = {}) { + if (this.busy) throw notApplied('The self-hosted model is already changing.'); + const record = this.record; + if (!record || record.phase === 'retired') return this.status(); + if (!OWNED_NAME.test(record.appName ?? '')) throw uncertain(); + this.busy = true; + try { + if (record.phase === 'prepared') { + await this.store.delete(this.keyName); + this.#save({ ...record, phase: 'retired', cleanupVerified: true }); + return this.status(); + } + await this.#confirmAccount(signal); + this.#save({ ...record, phase: 'retiring' }); + try { + await this.#stopApp(signal); + const app = record.appName; + const owned = [['secret', `${app}-auth`], ['volume', `${app}-hf-cache`], ['volume', `${app}-vllm-cache`]]; + for (const [kind, name] of owned) { + const deleted = await this.#cli([kind, 'delete', '--allow-missing', '--yes', name], { signal }); + if (deleted?.code !== 0) throw uncertain(); + } + const [secrets, volumes] = await Promise.all([this.#cli(['secret', 'list', '--json'], { signal }), this.#cli(['volume', 'list', '--json'], { signal })]); + const remaining = [json(secrets?.stdout), json(volumes?.stdout)]; + if (secrets?.code !== 0 || volumes?.code !== 0 || remaining.some(items => !Array.isArray(items)) + || remaining.some(items => items.some(item => owned.some(([, name]) => item?.name === name)))) throw uncertain(); + await this.store.delete(this.keyName); + } catch { this.#save({ ...this.record, phase: 'unknown' }); throw uncertain(); } + this.#save({ ...this.record, phase: 'retired', cleanupVerified: true, endpoints: undefined, verification: undefined, retiredAt: new Date(this.clock()).toISOString() }); + return this.status(); + } finally { this.busy = false; } + } +} diff --git a/test/selfhost-modal.test.mjs b/test/selfhost-modal.test.mjs new file mode 100644 index 0000000..8d99e93 --- /dev/null +++ b/test/selfhost-modal.test.mjs @@ -0,0 +1,186 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { ModalSelfHostedModel, selfHostedProfiles, codexProviderConfig, claudeProviderEnvironment, validateCloudLitePlan } from '../src/public-sdk.mjs'; + +const fresh = () => mkdtempSync(join(tmpdir(), 'factory-selfhost-')); +const read = (path) => readFileSync(path, 'utf8'); + +// A fake Modal account: the deploy script, the Modal CLI and the served model. +function fixture({ deployError, describeError, holdDescribe, workspace = 'fixture-workspace', completion, redirect, catalogueBytes } = {}) { + const calls = [], requests = [], apps = new Map(), secrets = new Set(), volumes = new Set(); + let key, appName, account = workspace; + const runner = async (command, args, options) => { + calls.push({ command, args, input: options.input, env: options.env }); + if (args.at(-1)?.endsWith('deploy.py')) { + const request = JSON.parse(options.input); + appName = request.appName; + assert.match(appName, /^factory-(qwen|nemotron)-[a-f0-9]{12}$/); + if (request.action === 'describe') { + if (holdDescribe) return new Promise((_, reject) => options.signal.addEventListener('abort', () => reject(new Error('stopped')), { once: true })); + if (describeError) return { code: 1, stdout: JSON.stringify({ error: describeError }) }; + const nemotron = request.profile === 'nemotron'; + return { code: 0, stdout: `noise\n${JSON.stringify({ settings: { appName, profile: request.profile, label: nemotron ? 'Nemotron on B300' : 'Qwen on H100', + model: nemotron ? 'nemotron-3.5-super-vl' : 'qwen3.8-27b', gpu: nemotron ? 'B300' : 'H100', replicas: request.overrides.replicas ?? 1, + contextWindow: 262144, usdPerHourPerReplica: nemotron ? 8.43 : 4.84, scaledownMinutes: 10, startupTimeoutMinutes: 40 }, hints: { compact_tokens: 80000 } })}` }; + } + if (request.action === 'deploy') { + if (deployError) return { code: 1, stdout: JSON.stringify({ error: deployError }) }; + if (!request.reuseSecret) { assert.equal(secrets.has(`${appName}-auth`), false); secrets.add(`${appName}-auth`); } + key = request.key; volumes.add(`${appName}-hf-cache`); volumes.add(`${appName}-vllm-cache`); + apps.set(appName, 'deployed'); + const replicas = request.overrides.replicas ?? 1; + return { code: 0, stdout: JSON.stringify({ endpoints: ['serve', 'serve-b'].slice(0, replicas).map(name => `https://fixture--${appName}-${name}.modal.run`) }) }; + } + return { code: 0, stdout: JSON.stringify({ seconds: 1 }) }; + } + const cli = args.slice(args.indexOf('modal') + 1); + if (cli[0] === 'profile') return { code: 0, stdout: JSON.stringify([{ name: 'fixture', workspace: account, active: true }]) }; + if (cli[0] === 'app' && cli[1] === 'stop') { const known = apps.has(cli.at(-1)); if (known) apps.set(cli.at(-1), 'stopped'); return { code: known ? 0 : 1, stdout: '' }; } + if (cli[0] === 'app' && cli[1] === 'list') return { code: 0, stdout: JSON.stringify([...apps].map(([description, state]) => ({ description, state }))) }; + if (cli[1] === 'delete') { (cli[0] === 'secret' ? secrets : volumes).delete(cli.at(-1)); return { code: 0, stdout: '' }; } + if (cli[1] === 'list') return { code: 0, stdout: JSON.stringify([...(cli[0] === 'secret' ? secrets : volumes)].map(name => ({ name }))) }; + throw new Error(`Unexpected command ${cli.join(' ')}`); + }; + const fetchImpl = async (url, options) => { + requests.push({ url, authorization: options.headers?.authorization }); + const target = new URL(url); + if (redirect && options.method === 'POST') return new Response(null, { status: 303, headers: { location: redirect(target) } }); + if (target.pathname === '/v1/models') return new Response(catalogueBytes ? ' '.repeat(catalogueBytes) : JSON.stringify({ data: [{ id: 'qwen3.8-27b' }, { id: 'nemotron-3.5-super-vl' }] })); + const body = options.method === 'POST' ? JSON.parse(options.body) : { model: 'qwen3.8-27b', messages: [{ content: target.searchParams.get('challenge') }] }; + const challenge = body.messages[0].content.match(/FACTORY-[A-F0-9]{8}/)?.[0]; + return new Response(JSON.stringify(completion?.(body, challenge) ?? { model: body.model, choices: [{ message: { role: 'assistant', content: challenge }, finish_reason: 'stop' }] })); + }; + const model = new ModalSelfHostedModel({ directory: fresh(), python: 'fixture-python', commandRunner: runner, fetchImpl, + env: { PATH: 'fixture-path', FACTORY_SELFHOST_SETTINGS_B64: 'inherited-must-be-dropped' } }); + return { model, calls, requests, apps, secrets, volumes, key: () => key, appName: () => appName, setWorkspace: value => { account = value; } }; +} + +test('bundled profiles carry list-price estimates per running replica', () => { + const profiles = Object.fromEntries(selfHostedProfiles().map(profile => [profile.id, profile])); + assert.deepEqual(Object.keys(profiles), ['nemotron', 'qwen']); + assert.equal(profiles.qwen.gpu, 'H100'); assert.equal(profiles.qwen.usdPerHour, 4.84); assert.equal(profiles.qwen.model, 'qwen3.8-27b'); + assert.equal(profiles.nemotron.gpu, 'B300'); assert.equal(profiles.nemotron.usdPerHour, 8.43); assert.equal(profiles.nemotron.variant, 'mtp1'); + assert.ok(profiles.nemotron.variants.includes('fast')); + for (const profile of Object.values(profiles)) { assert.equal(profile.contextWindow, 262144); assert.equal(profile.autoCompactTokenLimit, 80000); } +}); + +test('deploy, verify, describe, stop, redeploy and retire touch only the owned resources', async () => { + const f = fixture(); + const ready = await f.model.deploy({ profile: 'qwen', overrides: { replicas: 2 } }); + assert.equal(ready.phase, 'ready'); assert.equal(ready.verified, true); assert.equal(ready.endpoints, 2); + const key = await f.model.key(); + assert.equal(key, f.key()); assert.ok(key.length >= 48); + const journal = read(join(f.model.directory, 'model.json')); + for (const text of [JSON.stringify(ready), journal]) assert.equal(text.includes(key), false); + assert.doesNotMatch(JSON.stringify(ready), /modal\.run|factory-qwen-/); + assert.ok(f.calls.every(call => call.env.FACTORY_SELFHOST_SETTINGS_B64 === undefined)); + assert.ok(f.calls.filter(call => !call.args.at(-1)?.endsWith('deploy.py')).every(call => call.env.MODAL_PROFILE === 'fixture' || call.args.includes('profile'))); + assert.equal(f.calls.filter(call => call.input?.includes(key)).length, 1, 'the key reaches only the deploy request'); + assert.ok(f.requests.every(request => request.authorization === `Bearer ${key}`)); + const provider = f.model.endpoint(); + assert.deepEqual(provider.baseUrls, [`https://fixture--${f.appName()}-serve.modal.run/v1`, `https://fixture--${f.appName()}-serve-b.modal.run/v1`]); + assert.equal(provider.envKey, 'FACTORY_SELFHOSTED_MODEL_KEY'); assert.equal(provider.autoCompactTokenLimit, 80000); + assert.equal(validateCloudLitePlan({ runId: 'selfhost-run', app: 'selfhost-app', goal: 'x', modelProvider: provider }).modelProvider.model, 'qwen3.8-27b'); + assert.match(codexProviderConfig(provider, { index: 1 }), /serve-b\.modal\.run\/v1/); + assert.match(claudeProviderEnvironment(provider, { key }).ANTHROPIC_BASE_URL, /-serve\.modal\.run$/); + assert.equal((await f.model.stop()).phase, 'stopped'); + assert.equal(f.apps.get(f.appName()), 'stopped'); + assert.throws(() => f.model.endpoint(), /not verified/); + await assert.rejects(f.model.deploy({ profile: 'nemotron' }), /keeps its profile/); + const resumed = await f.model.deploy({ profile: 'qwen', overrides: { replicas: 2 } }); + assert.equal(resumed.phase, 'ready'); assert.equal(await f.model.key(), key, 'a redeploy keeps its key and secret'); + const retired = await f.model.retire(); + assert.equal(retired.phase, 'retired'); assert.equal(retired.cleanupVerified, true); + assert.equal(f.secrets.size, 0); assert.equal(f.volumes.size, 0); + assert.equal(existsSync(join(f.model.directory, 'selfhosted-model.key')), false); + const named = f.calls.filter(call => call.args.includes('stop') || call.args.includes('delete')).map(call => call.args.at(-1)); + assert.ok(named.every(name => name.startsWith(f.appName())), named.join(',')); +}); + +test('an uncertain deployment is kept for reconciliation and never replayed', async () => { + const f = fixture({ deployError: 'The Modal deployment did not complete.' }); + await assert.rejects(f.model.deploy({ profile: 'nemotron' }), error => error.outcome === 'unknown'); + assert.equal(f.model.status().phase, 'unknown'); + await assert.rejects(f.model.deploy({ profile: 'nemotron' }), /retire it to reconcile/); + assert.equal(f.calls.filter(call => call.input?.includes('"deploy"')).length, 1); + const retired = await f.model.retire(); + assert.equal(retired.phase, 'retired', 'a never-created app counts as stopped; secret and volumes are deleted and checked'); +}); + +test('a wrong model or a broken challenge never becomes ready', async () => { + for (const completion of [(body, challenge) => ({ model: 'other-model', choices: [{ message: { role: 'assistant', content: challenge }, finish_reason: 'stop' }] }), + () => ({ model: 'qwen3.8-27b', choices: [{ message: { role: 'assistant', content: 'something else' }, finish_reason: 'stop' }] }), + (body, challenge) => ({ model: 'qwen3.8-27b', choices: [{ message: { role: 'assistant', content: challenge }, finish_reason: 'length' }] })]) { + const f = fixture({ completion }); + await assert.rejects(f.model.deploy(), error => error.outcome === 'unknown'); + assert.equal(f.model.status().phase, 'unknown'); + assert.throws(() => f.model.endpoint(), /not verified/); + } + const oversized = fixture({ catalogueBytes: 70_000 }); + await assert.rejects(oversized.model.deploy(), error => error.outcome === 'unknown'); +}); + +test('Modal result redirects are followed on the same origin only, without replaying the POST', async () => { + const same = fixture({ redirect: target => `${target.origin}${target.pathname}?__modal_function_call_id=fixture` }); + await assert.rejects(same.model.deploy(), error => error.outcome === 'unknown', 'a GET result without the original body cannot echo a challenge'); + assert.equal(same.requests.filter(request => request.url.endsWith('/v1/chat/completions')).length, 1); + assert.ok(same.requests.some(request => request.url.includes('__modal_function_call_id=fixture'))); + const away = fixture({ redirect: () => 'https://elsewhere.example.test/v1/chat/completions?x=1' }); + await assert.rejects(away.model.deploy(), error => error.outcome === 'unknown'); + assert.equal(away.requests.some(request => request.url.startsWith('https://elsewhere')), false, 'the key never goes to another origin'); +}); + +test('cleanup refuses a changed Modal workspace and cancellation before remote work leaves nothing behind', async () => { + const f = fixture(); + await f.model.deploy(); + f.setWorkspace('another-workspace'); + await assert.rejects(f.model.retire(), /workspace changed/); + await assert.rejects(f.model.stop(), /workspace changed/); + assert.equal(f.calls.some(call => call.args.includes('stop')), false); + f.setWorkspace('fixture-workspace'); + assert.equal((await f.model.retire()).phase, 'retired'); + + const cancelled = fixture({ holdDescribe: true }); const controller = new AbortController(); + const pending = cancelled.model.deploy({ signal: controller.signal }); + setTimeout(() => controller.abort(), 10); + await assert.rejects(pending, error => error.outcome === 'not_applied'); + assert.equal(cancelled.model.status().phase, 'retired'); + assert.equal(cancelled.calls.some(call => call.input?.includes('"deploy"')), false); + const refused = fixture({ describeError: 'unknown gpu' }); + await assert.rejects(refused.model.deploy({ overrides: { gpu: 'TPU' } }), /unknown gpu/); + assert.equal(refused.model.status().phase, 'retired'); +}); + +test('an owned app from an earlier version can be adopted for verified retirement only', async () => { + const f = fixture(); + const appName = 'factory-qwen-00112233aabb'; + f.apps.set(appName, 'deployed'); f.secrets.add(`${appName}-auth`); f.volumes.add(`${appName}-hf-cache`); + assert.throws(() => f.model.adopt({ appName: 'other-qwen-00112233aabb', profileName: 'fixture', workspaceName: 'fixture-workspace' }), /owner prefix/); + assert.throws(() => f.model.adopt({ appName, profileName: 'fixture' }), /profile and workspace/); + assert.throws(() => f.model.adopt({ appName, profileName: 'fixture', workspaceName: 'fixture-workspace', endpoints: ['https://elsewhere.example.test'] }), /owned Modal origins/); + const adopted = f.model.adopt({ appName, profileName: 'fixture', workspaceName: 'fixture-workspace', endpoints: [`https://fixture--${appName}-inference.modal.run`] }); + assert.equal(adopted.phase, 'unknown'); + assert.throws(() => f.model.endpoint(), /not verified/); + await assert.rejects(f.model.deploy(), /retire it to reconcile/); + assert.throws(() => f.model.adopt({ appName, profileName: 'fixture', workspaceName: 'fixture-workspace' }), /already journaled/); + const retired = await f.model.retire(); + assert.equal(retired.phase, 'retired'); assert.equal(retired.cleanupVerified, true); + assert.equal(f.apps.get(appName), 'stopped'); assert.equal(f.secrets.size, 0); assert.equal(f.volumes.size, 0); +}); + +test('factory-selfhost lists profiles and never prints the key', () => { + const cli = fileURLToPath(new URL('../bin/factory-selfhost.mjs', import.meta.url)); + const listed = spawnSync(process.execPath, [cli, 'profiles'], { encoding: 'utf8' }); + assert.equal(listed.status, 0, listed.stderr); + assert.deepEqual(JSON.parse(listed.stdout).map(profile => profile.id), ['nemotron', 'qwen']); + const dir = fresh(); + assert.equal(JSON.parse(spawnSync(process.execPath, [cli, 'status', '--dir', dir], { encoding: 'utf8' }).stdout).phase, 'off'); + const key = spawnSync(process.execPath, [cli, 'key', '--dir', dir], { encoding: 'utf8' }); + assert.equal(key.status, 1); assert.match(key.stderr, /never printed/); + assert.match(spawnSync(process.execPath, [cli, '--help'], { encoding: 'utf8' }).stdout, /retire/); +});