From 903006530fb7772bb668bbf192ee4278e273e583 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 24 Oct 2026 16:18:45 +0000 Subject: [PATCH 1/9] clef: serve pinned flash and 27B text decisions Add a Python reference worker for Cloudflare Clef-flash and Clef on /v1/systemone. It hashes the public joint-schema loader, refuses over-long text instead of slicing it, and leaves image, video and native CUDA unsupported. Public231 concurrency-1 timing on L20X is an operator checklist; no latency was measured. Co-authored-by: Hongsheng Liu --- README.md | 1 + docs/architecture.md | 4 +- docs/supported-models.md | 6 + mkdocs.yml | 1 + recipe/README.md | 4 + recipe/clef/example-request.json | 24 + recipe/clef/export_prompt_fixture.py | 231 ++++++++ recipe/clef/reference.md | 198 +++++++ recipe/clef/requirements.txt | 11 + src/frontend/clef.py | 163 ++++++ src/models/clef/README.md | 145 +++++ src/models/clef/__init__.py | 1 + src/models/clef/engine.py | 270 +++++++++ src/models/clef/postprocess.py | 46 ++ src/models/clef/preprocess.py | 70 +++ src/models/clef/prompt.py | 103 ++++ src/models/clef/protocol.py | 109 ++++ tests/clef/__init__.py | 0 tests/clef/fixtures/text_contract.json | 760 +++++++++++++++++++++++++ tests/clef/test_engine.py | 91 +++ tests/clef/test_postprocess.py | 40 ++ tests/clef/test_prompt.py | 55 ++ tests/clef/test_protocol.py | 109 ++++ tests/clef/test_worker.py | 210 +++++++ 24 files changed, 2651 insertions(+), 1 deletion(-) create mode 100644 recipe/clef/example-request.json create mode 100644 recipe/clef/export_prompt_fixture.py create mode 100644 recipe/clef/reference.md create mode 100644 recipe/clef/requirements.txt create mode 100644 src/frontend/clef.py create mode 100644 src/models/clef/README.md create mode 100644 src/models/clef/__init__.py create mode 100644 src/models/clef/engine.py create mode 100644 src/models/clef/postprocess.py create mode 100644 src/models/clef/preprocess.py create mode 100644 src/models/clef/prompt.py create mode 100644 src/models/clef/protocol.py create mode 100644 tests/clef/__init__.py create mode 100644 tests/clef/fixtures/text_contract.json create mode 100644 tests/clef/test_engine.py create mode 100644 tests/clef/test_postprocess.py create mode 100644 tests/clef/test_prompt.py create mode 100644 tests/clef/test_protocol.py create mode 100644 tests/clef/test_worker.py diff --git a/README.md b/README.md index a1b55e87..1190bce6 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,7 @@ pinned runtime for text, image and video state: | LFM2.5-350M | [Transformers worker](recipe/lfm2/README.md); text state and independent choice questions | | OmniJev-4B v1.1 | [Checkpoint export and CPU contract](recipe/omnijev/README.md) only; the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | | D3 (Decision 3.0) | [Python reference worker](recipe/d3/reference.md) on the checkpoint's pinned runtime; `choice`/`noul`/`score` over text, image and video state; author-run on A100 for text and media serving (see the integration and media PRs) | +| Clef-flash (9B) and Clef (27B) | [Python reference worker](recipe/clef/reference.md) for text `choice`/`noul`/`score`; flash is the bring-up variant. Image, video and native CUDA are not served. No GPU run is recorded | [Supported models and hardware](docs/supported-models.md) lists the devices and where each worker has been run. diff --git a/docs/architecture.md b/docs/architecture.md index 33ec4043..ea7c134c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -37,7 +37,9 @@ before FP32 calibration and CPU response assembly. Laya computes its scorer and action head on CUDA. Its fixed-shape Graph captures Encoder/Decision; gather, scorer/action head and synchronized readback remain outside capture. Native Metal remains planned; Python workers -retain their documented reference/serving roles. +retain their documented reference/serving roles. The Clef reference worker +is one of those Python paths: it loads the checkpoint's joint schema head +on a Qwen3.5 backbone and does not use the shared Qwen CUDA executor. The Qwen executor's `forward_shared` takes a request prefix, group prefixes and branches, ends each prefix at its last multiple of 64 tokens, and returns one final diff --git a/docs/supported-models.md b/docs/supported-models.md index 4afd2514..39a4f1d6 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -27,6 +27,8 @@ Models that are being added are also tracked in issues labeled [new model](https | CLM-v0.1-8B | [Native Rust engine](../src/models/clm/README.md), `clm-run`, in front of a Qwen3-8B `/v1/embeddings` server | Engine-only agreement **4.5e-06** on identical vectors ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)); contract checks documented ([#23](https://github.com/ThinkFlowLab/system1-omni/pull/23)) | **Validated on compute capability 8.9** ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)) | Unverified | Rust to build and a separate Qwen3-8B `/v1/embeddings` server for the encoder; the engine itself has no device-specific code | | OmniJev-4B v1.1 | None yet: [checkpoint export and CPU contract](../recipe/omnijev/README.md); the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | Request contract, preparation, heads and finishing checked against the reference ([#120](https://github.com/ThinkFlowLab/system1-omni/pull/120)); no inference | Not yet | Not supported | Python 3.10 reference environment for the export and fixtures | | D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) on the checkpoint's pinned runtime — text, image and video state over `choice`/`noul`/`score`, with the deliberate transport differences documented in the recipe; the native worker is in progress ([#136](https://github.com/ThinkFlowLab/system1-omni/issues/136)) | Unverified | Author-run on NVIDIA A100 (sm_80): serving smoke and reference parity for text and media (exact decisions, max delta 0.0); evidence in the integration and media PRs. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) | +| Clef-flash (9B) | [Python reference worker](../recipe/clef/reference.md); text `choice`/`noul`/`score`; non-thinking prompt; 16384-token refusal | Contract tests only; no inference | Not run. The operator checklist targets lab NVIDIA L20X at compute capability 8.9 and does not record a result | Not supported | Python 3.12, [card-tested torch 2.11 and transformers 5.10.2](../recipe/clef/requirements.txt), pinned `Cloudflare/clef-flash` snapshot (~19.1 GB) | +| Clef (27B) | Same worker and text contract as Clef-flash, selected with `--variant clef` | Contract tests only; no inference | Not run | Not supported | Same stack; pinned `Cloudflare/clef` snapshot (~55.0 GB). Weights and joint-head config differ from flash | - **Validated:** covered by the recipe on `main` or by the checks in the linked merged pull request. - **Author-run:** recorded by the recipe author on the stated hardware and not independently reproduced; the row names its owner. @@ -38,6 +40,10 @@ LAYA's English worker, Open-Jev and CLM support `choice`, `score`, and `noul` te D3's reference worker serves the same three kinds for text, image and video state (inline base64 media under the pinned runtime's per-item budgets); its media serving evidence is author-run on the A100 above. +Clef-flash and Clef serve the same three kinds for text only. The worker +refuses image and video inputs. No CUDA timing is recorded; the recipe's +Public231 concurrency-1 checklist is for a lab NVIDIA L20X at compute +capability 8.9 and is not a result. CLM has a row per path: the merged `clm-serve` recipe exercises those answer shapes with stub embeddings and says nothing about decision quality, while the native engine's CUDA row is the path through a real Qwen3-8B encoder. Its engine-only figure comes from the same comparison run diff --git a/mkdocs.yml b/mkdocs.yml index ec30436b..40875098 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,4 +81,5 @@ nav: - CLM stub-encoder contract: recipe/clm/README.md - LFM2.5-350M choice worker: recipe/lfm2/README.md - D3 reference worker: recipe/d3/reference.md + - Clef reference worker: recipe/clef/reference.md - Contributing: CONTRIBUTING.md diff --git a/recipe/README.md b/recipe/README.md index 27778139..ac5424a6 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -38,6 +38,10 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting verify its runtime manifest, and serve `choice`/`noul`/`score` decisions over text, image and video state through the Python worker and the Rust frontend (text serving and parity validated on A100; media budgets and caps in the recipe). +- [Clef reference worker](clef/reference.md): load pinned Clef-flash (9B) or + Clef (27B) and serve text `choice`/`noul`/`score`. Image, video and native + CUDA are not served. Public231 concurrency-1 timing on L20X is an operator + checklist; no latency number is recorded here. - [LFM2.5-350M choice worker](lfm2/README.md): score text decisions with independent questions, hybrid cache branching and configurable candidate batches. diff --git a/recipe/clef/example-request.json b/recipe/clef/example-request.json new file mode 100644 index 00000000..da535b23 --- /dev/null +++ b/recipe/clef/example-request.json @@ -0,0 +1,24 @@ +{ + "model": "clef-flash", + "state": "Checkout has been failing for every customer for the last hour.", + "questions": { + "urgent": { + "type": "noul", + "instructions": "Is this support request urgent?" + }, + "team": { + "type": "choice", + "instructions": "Which team should handle this request?", + "criteria": { + "billing": "Payments, invoices, and refunds", + "technical": "Outages, errors, and configuration", + "sales": "Plans and upgrades" + } + }, + "severity": { + "type": "score", + "instructions": "How severe is the customer impact?", + "criteria": ["No impact", "Minor", "Major", "Critical"] + } + } +} diff --git a/recipe/clef/export_prompt_fixture.py b/recipe/clef/export_prompt_fixture.py new file mode 100644 index 00000000..28c7ea9e --- /dev/null +++ b/recipe/clef/export_prompt_fixture.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Regenerate the text prompt fixture from the pinned Clef loader and tokenizer. + +Run from the repository root. The loader is imported with a torch stub; this +does not download weights or run the joint head. + + python recipe/clef/export_prompt_fixture.py \ + --loader /path/to/joint_schema_model.py \ + --tokenizer /path/to/tokenizer.json \ + --output tests/clef/fixtures/text_contract.json + +``--check`` compares an existing fixture and exits non-zero on drift. +The tokenizer must be the pinned ``tokenizer.json`` +(SHA-256 06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523). +Token ids come from the ``tokenizers`` library +(``Tokenizer.encode(..., add_special_tokens=False)``), which is the backend +named in the pinned ``tokenizer_config.json``. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import sys +import types +from pathlib import Path + +from models.clef.engine import LOADER_SHA256, TOKENIZER_SHA256 +from models.clef.prompt import prompt_pieces + +CASES = [ + { + "name": "choice_noul_score", + "request": { + "model": "clef-flash", + "state": {"invoice": {"currency": "USD", "total": 1250, "vendor": "Acme"}}, + "questions": { + "team": { + "type": "choice", + "instructions": "Which team should handle the invoice?", + "criteria": { + "technical": "Bugs or outages", + "billing": "Payments or invoices", + }, + }, + "large": { + "type": "noul", + "instructions": "Is the total above 1000 USD?", + "criteria": {"true": "The total is greater than 1000 USD."}, + }, + "severity": { + "type": "score", + "instructions": "How severe is the customer impact?", + "criteria": ["No impact", "Minor", "Major", "Critical"], + }, + }, + }, + }, + { + "name": "string_state_and_default_instruction", + "request": { + "model": "Cloudflare/clef", + "state": "Checkout has been failing for every customer.", + "questions": { + "urgent": {"type": "noul"}, + "team": { + "type": "choice", + "criteria": {"sales": "Plans and upgrades", "billing": "Payments"}, + }, + }, + }, + }, +] + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _install_torch_stub() -> None: + """Let ``joint_schema_model.py`` import without installing torch.""" + + def module(name: str) -> types.ModuleType: + created = types.ModuleType(name) + sys.modules[name] = created + return created + + torch = module("torch") + torch.bfloat16 = "bfloat16" + torch.dtype = object + torch.device = object + torch.nn = module("torch.nn") + torch.nn.functional = module("torch.nn.functional") + torch.nn.Module = object + torch.nn.LayerNorm = object + torch.nn.Linear = object + torch.nn.MultiheadAttention = object + torch.nn.Dropout = object + torch.nn.Sequential = object + torch.nn.GELU = object + torch.nn.Embedding = object + torch.nn.TransformerDecoderLayer = object + torch.nn.Parameter = lambda *args, **kwargs: None + + def inference_mode(*args, **kwargs): + def decorator(fn): + return fn + + return decorator + + torch.inference_mode = inference_mode + + +def _load_encoder(path: Path): + if _sha256(path) != LOADER_SHA256: + raise SystemExit(f"{path} is not the pinned joint_schema_model.py") + _install_torch_stub() + spec = importlib.util.spec_from_file_location("pinned_clef_encoder", path) + if spec is None or spec.loader is None: + raise SystemExit(f"cannot import {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class _PieceTokenizer: + """Adapt a ``tokenizers.Tokenizer`` to the pinned ``_tokens`` call.""" + + def __init__(self, tokenizer) -> None: + self.tokenizer = tokenizer + + def __call__(self, text: str, add_special_tokens: bool = False): + if add_special_tokens: + raise AssertionError("the pinned encoder must not add special tokens") + ids = self.tokenizer.encode(text, add_special_tokens=False).ids + + class Batch: + input_ids = ids + + return Batch() + + +def _ids(tokenizer, text: str) -> list[int]: + return tokenizer.encode(text, add_special_tokens=False).ids + + +def build_fixture(loader_path: Path, tokenizer_path: Path) -> dict: + if _sha256(tokenizer_path) != TOKENIZER_SHA256: + raise SystemExit(f"{tokenizer_path} is not the pinned tokenizer.json") + from tokenizers import Tokenizer + + encoder = _load_encoder(loader_path) + tokenizer = Tokenizer.from_file(str(tokenizer_path)) + adapted = _PieceTokenizer(tokenizer) + cases = [] + for case in CASES: + request = case["request"] + pieces = prompt_pieces(request) + encoded = encoder.encode_record(adapted, request, max_length=10**9) + ids: list[int] = [] + piece_token_lengths = [] + for piece in pieces: + piece_ids = _ids(tokenizer, piece) + piece_token_lengths.append(len(piece_ids)) + ids.extend(piece_ids) + if tuple(ids) != encoded.input_ids: + raise SystemExit(f"{case['name']}: prompt pieces do not match encode_record") + state_tokens = piece_token_lengths[1] + cases.append( + { + "name": case["name"], + "request": request, + "pieces": pieces, + "piece_token_lengths": piece_token_lengths, + "state_tokens": state_tokens, + "fixed_length": len(ids) - state_tokens, + "input_ids": ids, + } + ) + return { + "provenance": { + "loader_sha256": LOADER_SHA256, + "tokenizer_sha256": TOKENIZER_SHA256, + "tokenizer_api": "tokenizers.Tokenizer.encode add_special_tokens=False", + "repos": [ + "Cloudflare/clef-flash@f0b7d3cd65296abfe459e6c228b618f5d8b26d2f", + "Cloudflare/clef@a30162b99d68a481d8bd75d38df069a0cf83df40", + ], + "note": ( + "Both revisions ship the same joint_schema_model.py and tokenizer.json. " + "Ids are the untruncated concatenation. The encoder's max_length slice " + "was not applied." + ), + }, + "cases": cases, + } + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--loader", type=Path, required=True) + parser.add_argument("--tokenizer", type=Path, required=True) + parser.add_argument( + "--output", + type=Path, + default=Path("tests/clef/fixtures/text_contract.json"), + ) + parser.add_argument("--check", action="store_true") + args = parser.parse_args(argv) + fixture = build_fixture(args.loader, args.tokenizer) + encoded = json.dumps(fixture, ensure_ascii=False, indent=2) + "\n" + if args.check: + current = args.output.read_text(encoding="utf-8") + if current != encoded: + raise SystemExit(f"{args.output} does not match the pinned loader") + print(f"checked {args.output}") + return + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(encoded, encoding="utf-8") + print(f"wrote {args.output}") + + +if __name__ == "__main__": + main() diff --git a/recipe/clef/reference.md b/recipe/clef/reference.md new file mode 100644 index 00000000..0afab678 --- /dev/null +++ b/recipe/clef/reference.md @@ -0,0 +1,198 @@ +# Clef reference worker + +Serve Cloudflare Clef-flash or Clef on `POST /v1/systemone` by loading the +pinned snapshot's joint schema model. The contract, hashes and the +unsupported image, video and native-CUDA paths are in the +[model README](../../src/models/clef/README.md). Commands below run from the +repository root. + +Clef-flash is the bring-up variant: about 19.1 GB of safetensors versus +about 55.0 GB for Clef. Both use the same text contract. This recipe does +not record a GPU run. + +## Pins + +| Variant | Snapshot | Revision | +| --- | --- | --- | +| `clef-flash` (default) | `Cloudflare/clef-flash` | `f0b7d3cd65296abfe459e6c228b618f5d8b26d2f` | +| `clef` | `Cloudflare/clef` | `a30162b99d68a481d8bd75d38df069a0cf83df40` | + +The card names torch 2.11 and transformers 5.10.2. Install +[requirements.txt](requirements.txt) in a Python 3.12 environment. A CUDA +build of that torch is required for the worker process. CPU inference is +not a supported path. Storage for the flash snapshot is about 19.1 GB plus +the download cache; the 27B snapshot is about 55.0 GB. Device memory has +to hold the bf16 weights and one prefill. No specific GPU size was measured +here. + +## Start + +```sh +python -m pip install -r recipe/clef/requirements.txt +``` + +```sh +python - <<'PY' +from huggingface_hub import snapshot_download +print(snapshot_download( + "Cloudflare/clef-flash", + revision="f0b7d3cd65296abfe459e6c228b618f5d8b26d2f", +)) +PY +``` + +Use the directory that command prints as ``. Do not point the +worker at a different revision; the hash gate refuses it. + +```sh +PYTHONPATH=src python -m frontend.clef \ + --model \ + --variant clef-flash \ + --device cuda \ + --host 127.0.0.1 \ + --port 8000 +``` + +Readiness is the process log line after `load_release_model` returns. +`GET http://127.0.0.1:8000/health` then returns `status: ready`, +`variant: clef-flash`, `prompt_mode: non-thinking`, `modalities: ["text"]` +and `max_input_tokens: 16384`. The first `POST` is the first forward. That +forward was not run in this change. + +For the 27B checkpoint, download `Cloudflare/clef` at +`a30162b99d68a481d8bd75d38df069a0cf83df40` and pass `--variant clef`. + +## Example + +```sh +curl --fail-with-body http://127.0.0.1:8000/v1/systemone \ + -H 'Content-Type: application/json' \ + --data-binary @recipe/clef/example-request.json +``` + +The Rust frontend can forward the same body. From the repository root, +after `cargo build --release --locked`: + +```sh +OMNI_JEV_BIND=127.0.0.1:8080 \ +OMNI_JEV_BACKEND_URL=http://127.0.0.1:8000 \ + ./target/release/omni-jev +``` + +```sh +curl --fail-with-body http://127.0.0.1:8080/v1/systemone \ + -H 'Content-Type: application/json' \ + --data-binary @recipe/clef/example-request.json +``` + +A non-empty `images` or `videos` field returns HTTP 422. A prompt longer +than 16384 tokens returns HTTP 422 and the state is not shortened. The +worker echoes the request's `model` string; it does not have to equal the +loaded repo id. + +## Checks that do not load weights + +```sh +PYTHONPATH=src python -m pip install pytest +PYTHONPATH=src python -m pytest tests/clef -q +PYTHONPATH=src python -m compileall -q src/models/clef src/frontend/clef.py tests/clef +``` + +To confirm the checked-in token fixture against a downloaded loader and +tokenizer, without weights: + +```sh +PYTHONPATH=src python recipe/clef/export_prompt_fixture.py \ + --loader /joint_schema_model.py \ + --tokenizer /tokenizer.json \ + --output tests/clef/fixtures/text_contract.json \ + --check +``` + +## Public231 C1 on lab L20X (sm_89) + +This is the operator checklist for a concurrency-1 latency measurement of +the flash worker. It uses the existing +[benchmark runner](../../benchmarks/README.md). The client does not reserve +a GPU and does not prove that CUDA ran. This repository does not contain +the Public231 JSONL. No p50 or p95 from that workload is known for Clef. + +Record results outside the repository. Do not put hostnames, account names, +home directories, or the name of a reservation system in the metadata or +in a commit. + +1. Confirm the GPU the process will use. `nvidia-smi` must show the device + name `NVIDIA L20X` and compute capability 8.9. If either differs, stop. + Do not report the run as H200, H800, or as the Public231 leaderboard. +2. Start `frontend.clef` as above, on loopback port 8000, variant + `clef-flash`, device `cuda`. Leave that process up for every traversal. +3. Obtain the frozen Public231 request manifest whose file SHA-256 is + `00d1f10ca1142499a598b5f1c6602d329cce1cef04c961c46fc0318daa2c67d3`. + That is the JSONL byte hash cited for the JevBench `7ce310c7` export in + [the latency leaderboard](../../benchmarks/latency-leaderboard.md). + The canonical task hash + `dc3995d8ae1e2fc8e81ce38431add509eb8bb39b85aadfd0c7c32079382dde51` + is a different encoding and must not be substituted. Check the file + with `sha256sum` before `validate`. +4. Install `benchmarks/requirements.txt` (`httpx==0.28.1`) into the client + environment. The client and the worker may use different environments. +5. Write a metadata JSON file. Required keys are `gpu`, `gpu_ids`, + `driver`, `cuda`, `precision`, `model_revision`, `runtime_revision`, + `cache_policy`, `cuda_evidence` and `reservation`. Use these values + unless a measured fact differs: + + ```json + { + "gpu": "NVIDIA L20X", + "gpu_ids": ["0"], + "driver": "", + "cuda": "", + "precision": "bfloat16", + "model_revision": "Cloudflare/clef-flash@f0b7d3cd65296abfe459e6c228b618f5d8b26d2f", + "runtime_revision": "", + "cache_policy": "one request at a time; no prefix cache; no CUDA graph; transformers cache left at its default", + "cuda_evidence": "nvidia-smi showed NVIDIA L20X compute capability 8.9 while the worker was running with --device cuda", + "reservation": "" + } + ``` + +6. Predeclare the budget before the first measured request: concurrency 1, + five warmups, three separate measured traversals, score decimals 4, + and one timeout. The runner default timeout is 60 seconds and may be + too small for a long prompt. Pick the timeout from a feasibility run + and do not raise it after a measured traversal fails. A 422 from the + 16384-token limit is a failed request. A traversal with failures is not + a latency result. +7. Run, with a new output directory each time: + + ```sh + python benchmarks/bench.py validate + python benchmarks/bench.py run \ + --endpoint http://127.0.0.1:8000/v1/systemone \ + --model clef-flash \ + --metadata \ + --phase feasibility \ + --concurrency 1 --warmup 5 --timeout \ + --output /feasibility + ``` + + Repeat `run` three times with `--phase measured` and the same manifest, + metadata, timeout, warmup and concurrency. Exclude feasibility from the + pooled latency. +8. For each measured directory, recompute + `python benchmarks/bench.py summary `. Pool only the three + rounds if each summary has `evidence_complete` true and zero failures. + Concatenate successful `latency_ms` values. p50 is the median. p95 is + the nearest rank `x[ceil(0.95 * n) - 1]` on the sorted samples. Report + per-round and pooled counts. Keep failure latencies separate. +9. The clock is the runner's + `post_to_body_json_and_schema_validation` clock. It is not the H800 + leaderboard clock. Do not compute a speedup against that board, against + H200, or against an unmeasured hypothesis. + +## What remains unverified + +GPU startup, the first forward, image and video inputs, native CUDA, and +every latency number. Hypotheses for later serve-path work are listed in +the model README. H1, one encode per request, is already how this worker +calls the pinned model. Its effect on p50 and p95 is not measured. diff --git a/recipe/clef/requirements.txt b/recipe/clef/requirements.txt new file mode 100644 index 00000000..ce46d539 --- /dev/null +++ b/recipe/clef/requirements.txt @@ -0,0 +1,11 @@ +# Card-tested stack for Cloudflare/clef-flash and Cloudflare/clef: +# torch 2.11 and transformers 5.10.2. This file is not a resolved lock; +# this environment did not install or import that stack. +torch>=2.11 +transformers==5.10.2 +safetensors>=0.6 +huggingface_hub>=1.0 +tokenizers>=0.22 +# The processor loads with the backbone. Image and video requests are rejected +# by this worker; pillow is still imported by the pinned processor. +pillow>=10 diff --git a/src/frontend/clef.py b/src/frontend/clef.py new file mode 100644 index 00000000..ae1fd4f0 --- /dev/null +++ b/src/frontend/clef.py @@ -0,0 +1,163 @@ +"""Python reference worker for Cloudflare Clef and Clef-flash. + +The Rust frontend remains the public transport layer. This worker owns HTTP +lifecycle and request orchestration. Wire admission lives in +``models.clef.protocol``, question and token admission in +``models.clef.preprocess``, and the pinned joint-schema checkpoint in +``models.clef.engine``. + +Over-long state is rejected. The checkpoint encoder would otherwise drop the +tail of the state without an error. Image and video fields are rejected; +this slice does not serve them. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import socket +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + +from models.clef.engine import DEFAULT_VARIANT, VARIANTS, ClefExecutor +from models.clef.preprocess import validate_questions +from models.clef.protocol import MAX_BODY, RequestError, parse_body, parse_request + +LOG = logging.getLogger(__name__) + + +def decide(raw: bytes, executor: Any) -> dict[str, Any]: + """Admit one complete text request and run it if it fits intact.""" + + request = parse_request(parse_body(raw)) + validate_questions(request) + return executor.serve(request.payload) + + +class WorkerServer(ThreadingHTTPServer): + daemon_threads = False + + def __init__(self, address, executor: Any): + self.executor = executor + self.inference_lock = threading.Lock() + self._executor_closed = False + super().__init__(address, Handler) + + def server_close(self): + super().server_close() + if not self._executor_closed: + self.executor.close() + self._executor_closed = True + + +class Handler(BaseHTTPRequestHandler): + def setup(self): + super().setup() + self.connection.settimeout(15) + + def log_message(self, format, *args): + # Do not log paths, prompts, or arbitrary request headers. + pass + + def send_json(self, status: int, value: dict): + raw = json.dumps(value, ensure_ascii=False, allow_nan=False).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + try: + self.wfile.write(raw) + except (BrokenPipeError, ConnectionResetError): + pass + + def _send_request_error(self, exc: RequestError): + if exc.body is not None: + self.send_json(exc.status, exc.body) + else: + self.send_json(exc.status, {"detail": str(exc)}) + + def do_GET(self): + if self.path == "/health": + self.send_json(200, self.server.executor.health()) + else: + self.send_json(404, {"detail": "unknown route"}) + + def do_POST(self): + if self.path != "/v1/systemone": + self.send_json(404, {"detail": "unknown route"}) + return + if self.headers.get("Transfer-Encoding"): + self.send_json(411, {"detail": "Content-Length is required; chunked requests are unsupported"}) + return + lengths = self.headers.get_all("Content-Length", []) + if len(lengths) != 1: + self.send_json(411, {"detail": "one Content-Length is required"}) + return + try: + length = int(lengths[0]) + except ValueError: + self.send_json(400, {"detail": "invalid Content-Length"}) + return + if length < 0 or length > MAX_BODY: + self.send_json(413, {"detail": "request exceeds the 8 MiB body limit"}) + return + if self.headers.get_content_type() != "application/json": + self.send_json(415, {"detail": "Content-Type must be application/json"}) + return + if not self.server.inference_lock.acquire(blocking=False): + self.send_json(503, {"detail": "worker busy"}) + return + try: + raw = self.rfile.read(length) + if len(raw) != length: + self.send_json(400, {"detail": "incomplete body"}) + return + self.send_json(200, decide(raw, self.server.executor)) + except RequestError as exc: + self._send_request_error(exc) + except (TimeoutError, socket.timeout): + self.send_json(408, {"detail": "request body timed out"}) + except Exception as exc: + LOG.error("clef inference failed: %s", type(exc).__name__) + self.send_json(500, {"detail": "inference failed"}) + finally: + self.server.inference_lock.release() + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--model", required=True, help="snapshot directory of the pinned revision") + parser.add_argument("--variant", default=DEFAULT_VARIANT, choices=sorted(VARIANTS)) + parser.add_argument("--device", default="cuda") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8000) + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + logging.basicConfig(level=logging.INFO) + executor = ClefExecutor(args.model, variant=args.variant, device=args.device) + server = None + try: + server = WorkerServer((args.host, args.port), executor) + LOG.info( + "%s worker ready on %s:%s", + executor.variant.repo_id, + args.host, + args.port, + ) + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + if server is not None: + server.server_close() + else: + executor.close() + + +if __name__ == "__main__": + main() diff --git a/src/models/clef/README.md b/src/models/clef/README.md new file mode 100644 index 00000000..38a36c3c --- /dev/null +++ b/src/models/clef/README.md @@ -0,0 +1,145 @@ +# Clef and Clef-flash + +Python reference worker for the Cloudflare Clef family on `POST /v1/systemone`. +The default bring-up is Clef-flash (9B). Clef (27B) uses the same loader, +tokenizer and request contract with its own backbone and joint-head config. +Native CUDA is not implemented. Image and video inputs are rejected. + +The checkpoint is not an Open-Jev export. `joint_schema_model.py` loads +`Qwen3_5ForConditionalGeneration` and a separate `JointSchemaHead`. The +safetensors index contains the Qwen language model and vision tower and has +no LoRA keys; the joint head is `joint_head.safetensors`. Shared Qwen CUDA +kernels in this repository score a different head and are not used here. + +## Pinned artifacts + +Both public snapshots were read on 2026-10-24. The loader and tokenizer bytes +match across the two revisions. Config and joint-head bytes do not. + +| | Clef-flash | Clef | +| --- | --- | --- | +| Repo | [Cloudflare/clef-flash](https://huggingface.co/Cloudflare/clef-flash) | [Cloudflare/clef](https://huggingface.co/Cloudflare/clef) | +| Revision | `f0b7d3cd65296abfe459e6c228b618f5d8b26d2f` | `a30162b99d68a481d8bd75d38df069a0cf83df40` | +| Base model | `Qwen/Qwen3.5-9B` | `Qwen/Qwen3.8-27B` | +| License | Apache-2.0 | Apache-2.0 | +| Backbone | hidden 4096, 32 layers | hidden 5120, 64 layers | +| Joint head | hidden 4096, width 1024, 2 routing layers, 4 decoder layers | hidden 5120, same width and depth | +| Safetensors, including the joint head | about 19.1 GB | about 55.0 GB | +| `config.json` SHA-256 | `66f87f6fb2616b46604daf2a9c67ddc87938296d07156efa34d59b5be49e3238` | `c42e88892bd3fd84e8276b2ad90df58c1c3b797676ea161035006a72ad468c58` | +| `joint_head_config.json` SHA-256 | `77efe959a38b5b17b241543e129e695f3c77465ece55a25985279bd8176279a0` | `890be585d75b981201eb96a35f98a9967afe37bc8d220cfdea2e72d56507534f` | + +Shared file SHA-256 values, identical at both revisions: + +| File | SHA-256 | +| --- | --- | +| `joint_schema_model.py` | `0e304cf7c6500e8bb59bef7e2afd2c6373f82596dfb3b57d1aa93c175e2dc3a3` | +| `tokenizer.json` | `06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523` | +| `tokenizer_config.json` | `91a08f825d370d085d692e04cf117cdd7faad7bf18e996f1e6031b6dab03db72` | +| `processor_config.json` | `d89ef49ce9cd37fbf510158e13c1ef063d9286411c1ec9049932dbe0487143b1` | + +Weight shards are not hashed by the worker. Download the snapshot at the +revision above so the shards belong to the same commit as the hashed files. +The card names torch 2.11 and transformers 5.10.2. Those packages were not +imported in this change. + +Workers AI model names (`@cf/cloudflare/clef`, `@cf/cloudflare/clef-flash`) +are a hosted API. This worker does not call them. + +## Request and response + +`choice`, `noul` and `score` are served for text `state`. `state` may be any +JSON value. Strings are inserted as text; every other value is compact JSON +with sorted keys. The response echoes `model`, returns one answer per question +in request order, and sets `usage.output_tokens` to 0. `usage.input_tokens` +is the untruncated prompt length. + +The encoder's decision suffix is the empty think block +`\n\n\n\nJOINT SCHEMA DECISIONS:`. That is the non-thinking +path. The chat templates differ between the two revisions and can open a +think block; `systemone` does not apply those templates. There is no +thinking switch on this worker. `/health` reports `prompt_mode: non-thinking`. + +Choice options are sorted by id inside the prompt. Answer `probabilities` +keep the request's criteria order. Ties use that same request order. The +chosen id is selected from the unrounded probabilities; the wire values are +then rounded to four decimal places. `noul` is the rounded probability of +`true` only. `score` is the rounded expected index, with `legend` holding +the original criteria values. Confidence for `choice` and `score` is the +rounded maximum probability, not an entropy. + +Noul criteria may mention `true` and `false`. Any other noul key remains in +the request object and is not an option. A score criterion list must be +non-empty; a one-level list is accepted because the pinned encoder accepts +it. The Public231 runner separately requires at least two score levels. + +## Limits + +The shipped `systemone` default is 16384 tokens. This worker uses that limit +and rejects a longer prompt. The pinned encoder would otherwise drop the tail +of the state. The blog's 64k window and the backbone +`max_position_embeddings` of 262144 are not admission limits here. + +HTTP bodies larger than 8 MiB are rejected. That cap is larger than a 16384 +token text request and smaller than an unbounded upload. Unknown request +fields are rejected. Non-empty `images` or `videos` are rejected with a +message that those modalities are not served. Empty lists are accepted and +add no media tokens. + +One request is scored at a time. The executor holds the loaded weights and +the joint head for the process lifetime. The handler takes a process-wide +lock for the request, so a second in-flight request receives HTTP 503. +There is no cross-request batch and no prefix cache. + +## Processor and executor + +[prompt.py](prompt.py) builds the text pieces in concatenation order. +[preprocess.py](preprocess.py) checks question shape and the token budget. +[postprocess.py](postprocess.py) mirrors `systemone_answer` for CPU tests. +The live worker calls the checkpoint's `systemone_answer` after one +`encode_record`. It does not call the checkpoint's `systemone` wrapper, +because that wrapper slices state to the token limit. + +The forward is the pinned backbone, then the pinned joint head, in bf16 as +loaded, with a float32 softmax. The head reads mean-pooled question and +option spans plus a lexical prior from the output embedding. Those layouts +belong to the checkpoint file. This repository does not reimplement them. + +## Validation + +CPU tests cover the envelope, the three answer formulas, the 16384-token +refusal, media refusal, the hash gate, and the checked-in prompt fixture. +The fixture's token ids were produced from the pinned tokenizer with +`tokenizers.Tokenizer.encode(..., add_special_tokens=False)` and checked +against `encode_record` from the pinned loader. Regenerate it from the +repository root: + +```sh +PYTHONPATH=src python recipe/clef/export_prompt_fixture.py \ + --loader /path/to/joint_schema_model.py \ + --tokenizer /path/to/tokenizer.json \ + --check +``` + +No weight was loaded. No GPU forward was run. Public231 concurrency-1 +latency on the lab L20X (compute capability 8.9) was not measured. The +measurement steps are in the [recipe](../../../recipe/clef/reference.md). + +## Latency hypotheses + +No p50 or p95 is reported. Do not read the H800 Public231 board as a result +for this worker, and do not label an L20X run as H200. + +| Id | Change | Expected effect | Delta | +| --- | --- | --- | --- | +| H1 | One `encode_record` per request, then the pinned forward. The handler does not tokenize the schema again before that encode. | Removes a second BPE pass from the critical path. Prefill and the joint head still dominate. | Not measured | +| H2 | Fuse the joint head's per-question Python loop. | Fewer small launches after prefill. Needs a forked head and numerical parity against the pinned module. | Not implemented | +| H3 | `torch.compile` or CUDA graphs on the Qwen3.5 backbone. | Uncertain on the linear-attention layers. | Not implemented | + +## Status + +| Path | Clef-flash | Clef | +| --- | --- | --- | +| Text `choice` / `noul` / `score`, non-thinking prompt | Contract tested; GPU inference unverified | Same loader pin; GPU inference unverified | +| Image and video | Rejected | Rejected | +| Native CUDA | Not implemented | Not implemented | +| Workers AI proxy | Out of scope | Out of scope | diff --git a/src/models/clef/__init__.py b/src/models/clef/__init__.py new file mode 100644 index 00000000..e54ed120 --- /dev/null +++ b/src/models/clef/__init__.py @@ -0,0 +1 @@ +"""Cloudflare Clef reference contract and serving adapters.""" diff --git a/src/models/clef/engine.py b/src/models/clef/engine.py new file mode 100644 index 00000000..2fb0a431 --- /dev/null +++ b/src/models/clef/engine.py @@ -0,0 +1,270 @@ +"""Clef executor for one pinned Cloudflare snapshot. + +The executor owns checkpoint identity, the hash gate, and the loaded joint +schema model. It does not admit HTTP requests. ``load_release_model`` and +``systemone`` are the checkpoint's own functions, imported only after every +executed and tokenized file matches the pin. + +Clef-flash (9B, Qwen3.5-9B) and Clef (27B, Qwen3.8-27B) share one loader and +one tokenizer. They do not share ``config.json`` or ``joint_head_config.json``. +Neither snapshot is an Open-Jev export: the decision head is +``joint_head.safetensors``, and the backbone index has no separate LoRA keys. +""" + +from __future__ import annotations + +import hashlib +import importlib.util +import json +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from models.clef.preprocess import MAX_INPUT_TOKENS, enforce_limit +from models.clef.prompt import render + +DEFAULT_VARIANT = "clef-flash" +LOADER_SHA256 = "0e304cf7c6500e8bb59bef7e2afd2c6373f82596dfb3b57d1aa93c175e2dc3a3" +TOKENIZER_SHA256 = "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523" +TOKENIZER_CONFIG_SHA256 = "91a08f825d370d085d692e04cf117cdd7faad7bf18e996f1e6031b6dab03db72" +PROCESSOR_CONFIG_SHA256 = "d89ef49ce9cd37fbf510158e13c1ef063d9286411c1ec9049932dbe0487143b1" +SHARED_FILES = { + "joint_schema_model.py": LOADER_SHA256, + "tokenizer.json": TOKENIZER_SHA256, + "tokenizer_config.json": TOKENIZER_CONFIG_SHA256, + "processor_config.json": PROCESSOR_CONFIG_SHA256, +} +EXECUTABLE = "joint_schema_model.py" + + +@dataclass(frozen=True) +class Variant: + """One public Clef release. Hashes are of the file bytes at ``revision``.""" + + name: str + repo_id: str + revision: str + base_model: str + hidden_size: int + num_hidden_layers: int + config_sha256: str + joint_head_config_sha256: str + + +VARIANTS = { + "clef-flash": Variant( + name="clef-flash", + repo_id="Cloudflare/clef-flash", + revision="f0b7d3cd65296abfe459e6c228b618f5d8b26d2f", + base_model="Qwen/Qwen3.5-9B", + hidden_size=4096, + num_hidden_layers=32, + config_sha256="66f87f6fb2616b46604daf2a9c67ddc87938296d07156efa34d59b5be49e3238", + joint_head_config_sha256="77efe959a38b5b17b241543e129e695f3c77465ece55a25985279bd8176279a0", + ), + "clef": Variant( + name="clef", + repo_id="Cloudflare/clef", + revision="a30162b99d68a481d8bd75d38df069a0cf83df40", + base_model="Qwen/Qwen3.8-27B", + hidden_size=5120, + num_hidden_layers=64, + config_sha256="c42e88892bd3fd84e8276b2ad90df58c1c3b797676ea161035006a72ad468c58", + joint_head_config_sha256="890be585d75b981201eb96a35f98a9967afe37bc8d220cfdea2e72d56507534f", + ), +} + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _require_hash(root: Path, name: str, expected: str) -> None: + path = root / name + if not path.is_file(): + raise ValueError(f"missing {name}") + actual = sha256_file(path) + if actual != expected: + raise ValueError(f"{name} SHA-256 {actual} is not the pinned digest") + + +def verify_snapshot( + root: Path, variant: Variant, shared_files: dict[str, str] | None = None +) -> None: + """Hash the loader, tokenizer and variant configs before importing them. + + Weight shards are not hashed here. Download the snapshot at + ``variant.revision`` so those shards belong to the same commit as the + hashed files. An extra ``.py`` or a ``__pycache__`` in the snapshot root + is refused so the import cannot pick up a different module. + """ + + root = Path(root) + if not root.is_dir(): + raise ValueError(f"the Clef snapshot directory does not exist: {root}") + for name, digest in (SHARED_FILES if shared_files is None else shared_files).items(): + _require_hash(root, name, digest) + _require_hash(root, "config.json", variant.config_sha256) + _require_hash(root, "joint_head_config.json", variant.joint_head_config_sha256) + for entry in root.iterdir(): + if entry.name == "__pycache__": + raise ValueError("unexpected bytecode cache __pycache__/ in the snapshot root") + if entry.is_file() and entry.suffix == ".py" and entry.name != EXECUTABLE: + raise ValueError(f"unexpected Python file {entry.name} in the snapshot root") + config = json.loads((root / "config.json").read_text(encoding="utf-8")) + text = config.get("text_config") or {} + if config.get("architectures") != ["Qwen3_5ForConditionalGeneration"]: + raise ValueError("config.json is not a Qwen3_5ForConditionalGeneration backbone") + if text.get("hidden_size") != variant.hidden_size: + raise ValueError( + f"config hidden_size {text.get('hidden_size')} is not {variant.hidden_size}" + ) + if text.get("num_hidden_layers") != variant.num_hidden_layers: + raise ValueError( + f"config num_hidden_layers {text.get('num_hidden_layers')} " + f"is not {variant.num_hidden_layers}" + ) + head = json.loads((root / "joint_head_config.json").read_text(encoding="utf-8")) + if head.get("hidden_size") != variant.hidden_size: + raise ValueError( + f"joint head hidden_size {head.get('hidden_size')} is not {variant.hidden_size}" + ) + + +def untruncated_budget(total: int, state_tokens: int, max_length: int) -> None: + """Reject a prompt whose full encoding does not fit, without slicing state. + + ``state_tokens`` is the rendered state counted by the same tokenizer the + encoder uses for that piece. A negative fixed length means the two counts + disagree; the request is not served. + """ + + if state_tokens < 0 or state_tokens > total: + raise RuntimeError( + f"token accounting mismatch: encoded {total}, state piece {state_tokens}" + ) + enforce_limit(total - state_tokens, total, max_length) + + +def require_token_agreement(admitted: int, response: dict[str, Any]) -> dict[str, Any]: + """Refuse a response whose prompt length is not the untruncated admission count. + + The pinned ``systemone`` slices state to fit. Admission already rejected + that case; a shorter encoder count means the slice still happened. + """ + + usage = response.get("usage") if isinstance(response, dict) else None + got = usage.get("input_tokens") if isinstance(usage, dict) else None + if got != admitted: + raise RuntimeError( + f"token accounting mismatch: admitted {admitted} untruncated tokens, " + f"encoder returned {got}" + ) + return response + + +def load_joint_schema_module(path: Path) -> Any: + """Import one verified ``joint_schema_model.py`` without putting its directory on ``sys.path``.""" + + spec = importlib.util.spec_from_file_location("clef_joint_schema_model", path) + if spec is None or spec.loader is None: + raise ValueError(f"cannot import {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class ClefExecutor: + """Load and run one immutable pinned Clef snapshot.""" + + def __init__(self, snapshot: str | Path, variant: str = DEFAULT_VARIANT, device: str = "cuda") -> None: + if variant not in VARIANTS: + raise ValueError(f"variant must be one of {sorted(VARIANTS)}") + self.variant = VARIANTS[variant] + self.root = Path(snapshot) + self.device = device + self.max_length = MAX_INPUT_TOKENS + verify_snapshot(self.root, self.variant) + self.module = load_joint_schema_module(self.root / EXECUTABLE) + self.model, self.processor = self.module.load_release_model(self.root, device=device) + if self.processor.tokenizer.pad_token_id is None: + raise ValueError("tokenizer pad_token_id is required") + + def count_tokens(self, text: str) -> int: + return len(self.processor.tokenizer(text, add_special_tokens=False).input_ids) + + def serve(self, payload: dict[str, Any]) -> dict[str, Any]: + """Encode once, refuse an over-long prompt, then score that encoding. + + The pinned ``systemone`` helper tokenizes and then slices state to + ``max_length``. Calling it after a separate admission pass would + tokenize the schema twice and could still slice if the counts drifted. + This path keeps the full ``encode_record`` result and runs the joint + head on it. The duplicate BPE pass is the only serve-path change; the + head and backbone are the pinned modules. No latency delta is claimed. + """ + + encoded = self.module.encode_record( + self.processor.tokenizer, + payload, + max_length=10**9, + processor=self.processor, + ) + total = len(encoded.input_ids) + state_tokens = self.count_tokens(render(payload["state"])) + untruncated_budget(total, state_tokens, self.max_length) + response = self._forward(payload, encoded) + return require_token_agreement(total, response) + + def _forward(self, payload: dict[str, Any], encoded: Any) -> dict[str, Any]: + import torch + + pad = self.processor.tokenizer.pad_token_id + if pad is None: + raise RuntimeError("tokenizer pad_token_id is required") + device = next(self.model.parameters()).device + with torch.inference_mode(): + logits = self.model( + self.module.collate_records([encoded], pad, device) + )[0] + questions = payload["questions"] + answers = { + question.question_id: self.module.systemone_answer( + questions[question.question_id], + dict( + zip( + question.option_ids, + question_logits.float().softmax(-1).tolist(), + ) + ), + ) + for question, question_logits in zip(encoded.questions, logits) + } + return { + "model": payload["model"], + "answers": answers, + "usage": {"input_tokens": len(encoded.input_ids), "output_tokens": 0}, + } + + def health(self) -> dict[str, Any]: + return { + "status": "ready", + "model": self.variant.repo_id, + "variant": self.variant.name, + "revision": self.variant.revision, + "base_model": self.variant.base_model, + "max_input_tokens": self.max_length, + "modalities": ["text"], + "prompt_mode": "non-thinking", + } + + def close(self) -> None: + self.model = None + self.processor = None + self.module = None diff --git a/src/models/clef/postprocess.py b/src/models/clef/postprocess.py new file mode 100644 index 00000000..83d974e2 --- /dev/null +++ b/src/models/clef/postprocess.py @@ -0,0 +1,46 @@ +"""SystemOne answers for one Clef question. + +This is the pinned ``systemone_answer`` formula from +``joint_schema_model.py``: four-decimal rounding, choice ties broken by the +request's criteria order, and score as the expected index. The live worker +still calls the hashed checkpoint function. These mirrors exist so the +contract can be tested without the checkpoint. +""" + +from __future__ import annotations + +from typing import Any + + +def choice_answer(question: dict[str, Any], probabilities: dict[str, float]) -> dict[str, Any]: + options = [str(option) for option in question["criteria"]] + choice = max(options, key=probabilities.__getitem__) + return { + "type": "choice", + "choice": choice, + "confidence": round(probabilities[choice], 4), + "probabilities": {option: round(probabilities[option], 4) for option in options}, + } + + +def noul_answer(probabilities: dict[str, float]) -> dict[str, Any]: + return {"type": "noul", "noul": round(probabilities["true"], 4)} + + +def score_answer(question: dict[str, Any], probabilities: dict[str, float]) -> dict[str, Any]: + levels = [str(index) for index in range(len(question["criteria"]))] + return { + "type": "score", + "score": round(sum(index * probabilities[level] for index, level in enumerate(levels)), 4), + "confidence": round(max(probabilities[level] for level in levels), 4), + "legend": dict(zip(levels, question["criteria"])), + "probabilities": {level: round(probabilities[level], 4) for level in levels}, + } + + +def systemone_answer(question: dict[str, Any], probabilities: dict[str, float]) -> dict[str, Any]: + if question["type"] == "noul": + return noul_answer(probabilities) + if question["type"] == "choice": + return choice_answer(question, probabilities) + return score_answer(question, probabilities) diff --git a/src/models/clef/preprocess.py b/src/models/clef/preprocess.py new file mode 100644 index 00000000..f19a582e --- /dev/null +++ b/src/models/clef/preprocess.py @@ -0,0 +1,70 @@ +"""Question admission and the untruncated token limit. + +The pinned encoder accepts any non-empty criteria container and then slices +the state to fit. This module rejects a question shape the encoder would +misread (a score given as an object, a noul criterion other than true/false) +and rejects a prompt that would not fit with its state intact. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from models.clef.prompt import QUESTION_TYPES, token_lengths +from models.clef.protocol import Request, RequestError + +# ``systemone(..., max_length=16384)`` in the pinned loader. The blog describes +# a 64k window and the backbone config allows 262144 positions; this worker +# does not raise the shipped encoder default. +MAX_INPUT_TOKENS = 16384 + + +def validate_questions(request: Request) -> None: + """Reject question bodies the text encoder cannot score unambiguously.""" + + for question_id, question in request.questions.items(): + if not isinstance(question, dict): + raise RequestError(f"{question_id}: question must be an object") + question_type = question.get("type") + if question_type not in QUESTION_TYPES: + raise RequestError(f"{question_id}: type must be noul, choice, or score") + criteria = question.get("criteria") + if question_type == "noul": + if criteria is None: + continue + if not isinstance(criteria, dict): + raise RequestError(f"{question_id}: noul criteria must be an object") + # The encoder scores only true and false. Other keys stay in the + # request object and do not become options, matching the pinned + # ``question_options`` update. They are not dropped from the JSON + # body; they are absent from the prompt. + continue + if question_type == "choice": + if not isinstance(criteria, dict) or not criteria: + raise RequestError(f"{question_id}: criteria must not be empty") + if any(not isinstance(option_id, str) or not option_id for option_id in criteria): + raise RequestError(f"{question_id}: choice option ids must be nonempty strings") + continue + if not isinstance(criteria, list) or not criteria: + raise RequestError(f"{question_id}: criteria must not be empty") + + +def measure(count: Callable[[str], int], request: Request) -> tuple[int, int]: + """Token lengths of the untruncated text prompt.""" + + return token_lengths(count, request.payload) + + +def enforce_limit(fixed: int, total: int, max_length: int = MAX_INPUT_TOKENS) -> None: + """Refuse a prompt that the pinned encoder would reject or truncate.""" + + if fixed > max_length: + raise RequestError( + f"schema requires {fixed} tokens before state; maximum is {max_length}" + ) + if total > max_length: + raise RequestError( + f"request is {total} tokens, which exceeds the {max_length} token limit; " + "the state was not truncated" + ) diff --git a/src/models/clef/prompt.py b/src/models/clef/prompt.py new file mode 100644 index 00000000..886a03fc --- /dev/null +++ b/src/models/clef/prompt.py @@ -0,0 +1,103 @@ +"""Text prompt pieces for the pinned Clef ``encode_record`` path. + +``joint_schema_model.py`` tokenizes each piece on its own and concatenates the +ids. The strings here follow that concatenation order (prefix, rendered state, +schema, non-thinking suffix). Image and video placeholders are not emitted; +this integration rejects media before encoding. + +The shipped encoder silently slices ``state`` once the schema and fixed +affixes fill ``max_length``. Callers that need a limit must compare +``token_lengths`` with the full state and refuse the request. This module +does not slice. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +SYSTEM_PROMPT = ( + "Read the complete state and schema. Decide every field jointly. Each answer " + "must be exactly one of that field's allowed options." +) +PREFIX = f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n<|im_start|>user\nSTATE:\n" +# The decision encoder hardcodes the empty think block. That is the non-thinking +# path of the Qwen chat template (``enable_thinking`` false). The template's +# default open ```` is not used by ``systemone``. +SUFFIX = "\n<|im_end|>\n<|im_start|>assistant\n\n\n\n\nJOINT SCHEMA DECISIONS:" +NOUL_DEFAULT_CRITERIA = { + "true": "The proposition is true or the answer is yes.", + "false": "The proposition is false or the answer is no.", +} +QUESTION_TYPES = ("noul", "choice", "score") + + +def render(value: Any) -> str: + """Render one JSON value the way the pinned encoder does. + + Strings are inserted unchanged. Every other value is compact JSON with + sorted keys, so object key order in the request does not change the prompt. + """ + + if isinstance(value, str): + return value + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def question_options(question: dict[str, Any]) -> list[tuple[str, Any]]: + """Return ``(option_id, description)`` pairs in encoder order. + + Choice ids are sorted by their string form. That order is the order the + joint head scores, which is not the order of keys in the response + ``probabilities`` object. Score ids are the decimal indices of ``criteria``. + Noul ids are always ``true`` then ``false``. + """ + + question_type = question["type"] + if question_type == "noul": + criteria = dict(NOUL_DEFAULT_CRITERIA) + criteria.update(question.get("criteria") or {}) + return [(key, criteria[key]) for key in ("true", "false")] + if question_type == "choice": + return sorted((str(key), value) for key, value in question["criteria"].items()) + return [(str(index), value) for index, value in enumerate(question["criteria"])] + + +def prompt_pieces(record: dict[str, Any]) -> list[str]: + """Return the text pieces whose token ids are concatenated for one record.""" + + schema: list[str] = ["\n\nSCHEMA FIELDS:\n"] + for question_index, (question_id, question) in enumerate(record["questions"].items()): + schema.append( + f"\nFIELD {question_index + 1}\nID: {question_id}\nTYPE: {question['type']}\nINSTRUCTION: " + ) + instructions = question.get("instructions") + if instructions is None or instructions == "": + instructions = str(question_id) + schema.append(render(instructions)) + schema.append("\nALLOWED OPTIONS:\n") + for option_index, (option_id, description) in enumerate(question_options(question)): + schema.append(f"OPTION {option_index + 1}: ") + semantics: dict[str, Any] = {"option_id": option_id} + if description is not None: + semantics["description"] = description + schema.append(render(semantics)) + schema.append("\n") + schema.append("END FIELD\n") + return [PREFIX, render(record["state"]), *schema, SUFFIX] + + +def token_lengths(count: Callable[[str], int], record: dict[str, Any]) -> tuple[int, int]: + """Return ``(fixed_length, total_length)`` for an untruncated record. + + ``fixed_length`` excludes the rendered state, matching the pinned + encoder's overflow check. ``total_length`` counts every state token. + ``count`` must be the length of one piece tokenized alone + (``add_special_tokens=False``); the pieces are not retokenized as one string. + """ + + lengths = [count(piece) for piece in prompt_pieces(record)] + state_tokens = lengths[1] + fixed = sum(lengths) - state_tokens + return fixed, fixed + state_tokens diff --git a/src/models/clef/protocol.py b/src/models/clef/protocol.py new file mode 100644 index 00000000..9610c0ea --- /dev/null +++ b/src/models/clef/protocol.py @@ -0,0 +1,109 @@ +"""Wire contract for the Clef reference worker. + +Validation here is the HTTP envelope only: a JSON object, the SystemOne +fields, and rejection of image or video payloads. Question shape and the +token limit live in ``preprocess`` so a prompt change stays next to the +encoder pieces. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any + +# Text-only cap. The pinned ``systemone`` function has no HTTP server and no +# body limit; 8 MiB is enough for a request that still has to be refused at +# the 16384-token admission limit, and it refuses larger bodies explicitly. +MAX_BODY = 8 * 1024 * 1024 +REQUEST_FIELDS = {"model", "state", "questions", "images", "videos"} +MEDIA_UNSUPPORTED = ( + "{field} are not served by this integration; image and video inputs remain unsupported" +) + + +class RequestError(ValueError): + """A request cannot be evaluated under the reference contract.""" + + def __init__(self, message: str, status: int = 422, body: dict | None = None) -> None: + super().__init__(message) + self.status = status + self.body = body + + +class MalformedJSON(RequestError): + """The body is not a usable UTF-8 JSON object.""" + + def __init__(self, message: str) -> None: + super().__init__(message, 400) + + +@dataclass(frozen=True) +class Request: + """One admitted envelope. ``payload`` keeps the parsed object and its key order.""" + + payload: dict[str, Any] + + @property + def questions(self) -> dict[str, Any]: + return self.payload["questions"] + + +def _object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise MalformedJSON(f"duplicate JSON key {key!r}") + result[key] = value + return result + + +def _nonfinite(value: str) -> None: + raise MalformedJSON(f"non-finite JSON number {value} is not supported") + + +def parse_body(raw: bytes) -> dict[str, Any]: + """Decode a bounded JSON body and reject ambiguous JSON values.""" + + if len(raw) > MAX_BODY: + raise RequestError("request body exceeds 8 MiB", 413) + try: + value = json.loads( + raw.decode("utf-8"), + object_pairs_hook=_object, + parse_constant=_nonfinite, + ) + json.dumps(value, ensure_ascii=False, allow_nan=False).encode("utf-8") + except MalformedJSON: + raise + except (UnicodeError, ValueError, RecursionError) as exc: + raise MalformedJSON("request body must contain valid JSON and UTF-8 text") from exc + if not isinstance(value, dict): + raise MalformedJSON("request must be a JSON object") + return value + + +def _reject_media(value: dict[str, Any], field: str) -> None: + if field not in value or value[field] is None or value[field] == []: + return + if not isinstance(value[field], list): + raise RequestError(f"{field} must be a list") + raise RequestError(MEDIA_UNSUPPORTED.format(field=field)) + + +def parse_request(value: dict[str, Any]) -> Request: + """Validate the systemone envelope and refuse media inputs.""" + + unknown = sorted(set(value) - REQUEST_FIELDS) + if unknown: + raise RequestError(f"Unknown request fields: {unknown}") + _reject_media(value, "images") + _reject_media(value, "videos") + if not isinstance(value.get("model"), str) or "state" not in value: + raise RequestError("model and state are required") + questions = value.get("questions") + if not isinstance(questions, dict) or not questions: + raise RequestError("at least one question is required") + if any(not isinstance(name, str) or not name for name in questions): + raise RequestError("question IDs must be nonempty strings") + return Request(value) diff --git a/tests/clef/__init__.py b/tests/clef/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/clef/fixtures/text_contract.json b/tests/clef/fixtures/text_contract.json new file mode 100644 index 00000000..83c4f0ef --- /dev/null +++ b/tests/clef/fixtures/text_contract.json @@ -0,0 +1,760 @@ +{ + "provenance": { + "loader_sha256": "0e304cf7c6500e8bb59bef7e2afd2c6373f82596dfb3b57d1aa93c175e2dc3a3", + "tokenizer_sha256": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523", + "tokenizer_api": "tokenizers.Tokenizer.encode add_special_tokens=False", + "repos": [ + "Cloudflare/clef-flash@f0b7d3cd65296abfe459e6c228b618f5d8b26d2f", + "Cloudflare/clef@a30162b99d68a481d8bd75d38df069a0cf83df40" + ], + "note": "Both revisions ship the same joint_schema_model.py and tokenizer.json. Ids are the untruncated concatenation. The encoder's max_length slice was not applied." + }, + "cases": [ + { + "name": "choice_noul_score", + "request": { + "model": "clef-flash", + "state": { + "invoice": { + "currency": "USD", + "total": 1250, + "vendor": "Acme" + } + }, + "questions": { + "team": { + "type": "choice", + "instructions": "Which team should handle the invoice?", + "criteria": { + "technical": "Bugs or outages", + "billing": "Payments or invoices" + } + }, + "large": { + "type": "noul", + "instructions": "Is the total above 1000 USD?", + "criteria": { + "true": "The total is greater than 1000 USD." + } + }, + "severity": { + "type": "score", + "instructions": "How severe is the customer impact?", + "criteria": [ + "No impact", + "Minor", + "Major", + "Critical" + ] + } + } + }, + "pieces": [ + "<|im_start|>system\nRead the complete state and schema. Decide every field jointly. Each answer must be exactly one of that field's allowed options.<|im_end|>\n<|im_start|>user\nSTATE:\n", + "{\"invoice\":{\"currency\":\"USD\",\"total\":1250,\"vendor\":\"Acme\"}}", + "\n\nSCHEMA FIELDS:\n", + "\nFIELD 1\nID: team\nTYPE: choice\nINSTRUCTION: ", + "Which team should handle the invoice?", + "\nALLOWED OPTIONS:\n", + "OPTION 1: ", + "{\"description\":\"Payments or invoices\",\"option_id\":\"billing\"}", + "\n", + "OPTION 2: ", + "{\"description\":\"Bugs or outages\",\"option_id\":\"technical\"}", + "\n", + "END FIELD\n", + "\nFIELD 2\nID: large\nTYPE: noul\nINSTRUCTION: ", + "Is the total above 1000 USD?", + "\nALLOWED OPTIONS:\n", + "OPTION 1: ", + "{\"description\":\"The total is greater than 1000 USD.\",\"option_id\":\"true\"}", + "\n", + "OPTION 2: ", + "{\"description\":\"The proposition is false or the answer is no.\",\"option_id\":\"false\"}", + "\n", + "END FIELD\n", + "\nFIELD 3\nID: severity\nTYPE: score\nINSTRUCTION: ", + "How severe is the customer impact?", + "\nALLOWED OPTIONS:\n", + "OPTION 1: ", + "{\"description\":\"No impact\",\"option_id\":\"0\"}", + "\n", + "OPTION 2: ", + "{\"description\":\"Minor\",\"option_id\":\"1\"}", + "\n", + "OPTION 3: ", + "{\"description\":\"Major\",\"option_id\":\"2\"}", + "\n", + "OPTION 4: ", + "{\"description\":\"Critical\",\"option_id\":\"3\"}", + "\n", + "END FIELD\n", + "\n<|im_end|>\n<|im_start|>assistant\n\n\n\n\nJOINT SCHEMA DECISIONS:" + ], + "piece_token_lengths": [ + 36, + 19, + 7, + 17, + 7, + 6, + 5, + 12, + 1, + 5, + 14, + 1, + 3, + 18, + 11, + 6, + 5, + 20, + 1, + 5, + 18, + 1, + 3, + 17, + 7, + 6, + 5, + 11, + 1, + 5, + 10, + 1, + 5, + 10, + 1, + 5, + 10, + 1, + 3, + 18 + ], + "state_tokens": 19, + "fixed_length": 318, + "input_ids": [ + 248045, + 8678, + 198, + 4274, + 279, + 4434, + 1528, + 321, + 10485, + 13, + 95462, + 1396, + 2002, + 50407, + 13, + 8618, + 4087, + 1902, + 381, + 6681, + 799, + 314, + 421, + 2002, + 579, + 5252, + 2519, + 13, + 248046, + 198, + 248045, + 846, + 198, + 23852, + 25, + 198, + 4754, + 21529, + 21624, + 15501, + 3147, + 25895, + 2129, + 4873, + 763, + 16, + 17, + 20, + 15, + 1288, + 18633, + 3147, + 11318, + 2573, + 29958, + 271, + 3423, + 34651, + 426, + 32349, + 25, + 198, + 198, + 27806, + 220, + 16, + 198, + 887, + 25, + 2015, + 198, + 9222, + 25, + 5576, + 198, + 669, + 55530, + 25, + 220, + 22365, + 2015, + 1220, + 3579, + 279, + 23839, + 30, + 198, + 51535, + 1425, + 49555, + 25, + 198, + 32929, + 220, + 16, + 25, + 220, + 4754, + 4532, + 3147, + 84624, + 466, + 61893, + 2129, + 1979, + 816, + 3147, + 37355, + 8934, + 198, + 32929, + 220, + 17, + 25, + 220, + 4754, + 4532, + 3147, + 33, + 12914, + 466, + 680, + 1098, + 2129, + 1979, + 816, + 3147, + 69685, + 8934, + 198, + 4537, + 38945, + 198, + 198, + 27806, + 220, + 17, + 198, + 887, + 25, + 3349, + 198, + 9222, + 25, + 307, + 10644, + 198, + 669, + 55530, + 25, + 220, + 3742, + 279, + 2702, + 3294, + 220, + 16, + 15, + 15, + 15, + 18954, + 30, + 198, + 51535, + 1425, + 49555, + 25, + 198, + 32929, + 220, + 16, + 25, + 220, + 4754, + 4532, + 3147, + 760, + 2702, + 369, + 6826, + 1056, + 220, + 16, + 15, + 15, + 15, + 18954, + 46292, + 1979, + 816, + 3147, + 1802, + 8934, + 198, + 32929, + 220, + 17, + 25, + 220, + 4754, + 4532, + 3147, + 760, + 38749, + 369, + 867, + 466, + 279, + 4087, + 369, + 874, + 46292, + 1979, + 816, + 3147, + 3721, + 8934, + 198, + 4537, + 38945, + 198, + 198, + 27806, + 220, + 18, + 198, + 887, + 25, + 28943, + 198, + 9222, + 25, + 5284, + 198, + 669, + 55530, + 25, + 220, + 4199, + 14938, + 369, + 279, + 5813, + 5253, + 30, + 198, + 51535, + 1425, + 49555, + 25, + 198, + 32929, + 220, + 16, + 25, + 220, + 4754, + 4532, + 3147, + 2665, + 5253, + 2129, + 1979, + 816, + 3147, + 15, + 8934, + 198, + 32929, + 220, + 17, + 25, + 220, + 4754, + 4532, + 3147, + 55116, + 2129, + 1979, + 816, + 3147, + 16, + 8934, + 198, + 32929, + 220, + 18, + 25, + 220, + 4754, + 4532, + 3147, + 33331, + 2129, + 1979, + 816, + 3147, + 17, + 8934, + 198, + 32929, + 220, + 19, + 25, + 220, + 4754, + 4532, + 3147, + 40621, + 2129, + 1979, + 816, + 3147, + 18, + 8934, + 198, + 4537, + 38945, + 198, + 198, + 248046, + 198, + 248045, + 74455, + 198, + 248068, + 271, + 248069, + 271, + 25310, + 3118, + 7295, + 34651, + 41402, + 23451, + 50, + 25 + ] + }, + { + "name": "string_state_and_default_instruction", + "request": { + "model": "Cloudflare/clef", + "state": "Checkout has been failing for every customer.", + "questions": { + "urgent": { + "type": "noul" + }, + "team": { + "type": "choice", + "criteria": { + "sales": "Plans and upgrades", + "billing": "Payments" + } + } + } + }, + "pieces": [ + "<|im_start|>system\nRead the complete state and schema. Decide every field jointly. Each answer must be exactly one of that field's allowed options.<|im_end|>\n<|im_start|>user\nSTATE:\n", + "Checkout has been failing for every customer.", + "\n\nSCHEMA FIELDS:\n", + "\nFIELD 1\nID: urgent\nTYPE: noul\nINSTRUCTION: ", + "urgent", + "\nALLOWED OPTIONS:\n", + "OPTION 1: ", + "{\"description\":\"The proposition is true or the answer is yes.\",\"option_id\":\"true\"}", + "\n", + "OPTION 2: ", + "{\"description\":\"The proposition is false or the answer is no.\",\"option_id\":\"false\"}", + "\n", + "END FIELD\n", + "\nFIELD 2\nID: team\nTYPE: choice\nINSTRUCTION: ", + "team", + "\nALLOWED OPTIONS:\n", + "OPTION 1: ", + "{\"description\":\"Payments\",\"option_id\":\"billing\"}", + "\n", + "OPTION 2: ", + "{\"description\":\"Plans and upgrades\",\"option_id\":\"sales\"}", + "\n", + "END FIELD\n", + "\n<|im_end|>\n<|im_start|>assistant\n\n\n\n\nJOINT SCHEMA DECISIONS:" + ], + "piece_token_lengths": [ + 36, + 8, + 7, + 18, + 1, + 6, + 5, + 18, + 1, + 5, + 18, + 1, + 3, + 17, + 1, + 6, + 5, + 10, + 1, + 5, + 12, + 1, + 3, + 18 + ], + "state_tokens": 8, + "fixed_length": 198, + "input_ids": [ + 248045, + 8678, + 198, + 4274, + 279, + 4434, + 1528, + 321, + 10485, + 13, + 95462, + 1396, + 2002, + 50407, + 13, + 8618, + 4087, + 1902, + 381, + 6681, + 799, + 314, + 421, + 2002, + 579, + 5252, + 2519, + 13, + 248046, + 198, + 248045, + 846, + 198, + 23852, + 25, + 198, + 53291, + 682, + 978, + 20738, + 364, + 1396, + 5813, + 13, + 271, + 3423, + 34651, + 426, + 32349, + 25, + 198, + 198, + 27806, + 220, + 16, + 198, + 887, + 25, + 32556, + 198, + 9222, + 25, + 307, + 10644, + 198, + 669, + 55530, + 25, + 220, + 82118, + 198, + 51535, + 1425, + 49555, + 25, + 198, + 32929, + 220, + 16, + 25, + 220, + 4754, + 4532, + 3147, + 760, + 38749, + 369, + 804, + 466, + 279, + 4087, + 369, + 9542, + 46292, + 1979, + 816, + 3147, + 1802, + 8934, + 198, + 32929, + 220, + 17, + 25, + 220, + 4754, + 4532, + 3147, + 760, + 38749, + 369, + 867, + 466, + 279, + 4087, + 369, + 874, + 46292, + 1979, + 816, + 3147, + 3721, + 8934, + 198, + 4537, + 38945, + 198, + 198, + 27806, + 220, + 17, + 198, + 887, + 25, + 2015, + 198, + 9222, + 25, + 5576, + 198, + 669, + 55530, + 25, + 220, + 8923, + 198, + 51535, + 1425, + 49555, + 25, + 198, + 32929, + 220, + 16, + 25, + 220, + 4754, + 4532, + 3147, + 84624, + 2129, + 1979, + 816, + 3147, + 37355, + 8934, + 198, + 32929, + 220, + 17, + 25, + 220, + 4754, + 4532, + 3147, + 94355, + 321, + 30575, + 2129, + 1979, + 816, + 3147, + 28094, + 8934, + 198, + 4537, + 38945, + 198, + 198, + 248046, + 198, + 248045, + 74455, + 198, + 248068, + 271, + 248069, + 271, + 25310, + 3118, + 7295, + 34651, + 41402, + 23451, + 50, + 25 + ] + } + ] +} diff --git a/tests/clef/test_engine.py b/tests/clef/test_engine.py new file mode 100644 index 00000000..5a55cef9 --- /dev/null +++ b/tests/clef/test_engine.py @@ -0,0 +1,91 @@ +import json +from pathlib import Path + +import pytest + +from models.clef.engine import ( + DEFAULT_VARIANT, + LOADER_SHA256, + TOKENIZER_SHA256, + VARIANTS, + require_token_agreement, + sha256_file, + untruncated_budget, + verify_snapshot, +) +from models.clef.protocol import RequestError + + +def test_pins_name_both_public_revisions_and_the_shared_loader(): + flash = VARIANTS["clef-flash"] + clef = VARIANTS["clef"] + assert DEFAULT_VARIANT == "clef-flash" + assert flash.repo_id == "Cloudflare/clef-flash" + assert flash.revision == "f0b7d3cd65296abfe459e6c228b618f5d8b26d2f" + assert flash.base_model == "Qwen/Qwen3.5-9B" + assert flash.hidden_size == 4096 + assert flash.num_hidden_layers == 32 + assert clef.repo_id == "Cloudflare/clef" + assert clef.revision == "a30162b99d68a481d8bd75d38df069a0cf83df40" + assert clef.base_model == "Qwen/Qwen3.8-27B" + assert clef.hidden_size == 5120 + assert clef.num_hidden_layers == 64 + assert LOADER_SHA256 == "0e304cf7c6500e8bb59bef7e2afd2c6373f82596dfb3b57d1aa93c175e2dc3a3" + assert TOKENIZER_SHA256 == "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523" + assert flash.config_sha256 != clef.config_sha256 + assert flash.joint_head_config_sha256 != clef.joint_head_config_sha256 + + +def _snapshot(tmp_path: Path, *, extra: str | None = None) -> tuple[Path, object]: + shared = {"notes.txt": b"loader-bytes"} + config = { + "architectures": ["Qwen3_5ForConditionalGeneration"], + "text_config": {"hidden_size": 4096, "num_hidden_layers": 32}, + } + head = {"hidden_size": 4096, "width": 1024} + (tmp_path / "notes.txt").write_bytes(shared["notes.txt"]) + (tmp_path / "config.json").write_text(json.dumps(config), encoding="utf-8") + (tmp_path / "joint_head_config.json").write_text(json.dumps(head), encoding="utf-8") + if extra: + (tmp_path / extra).write_text("print('no')\n", encoding="utf-8") + variant = VARIANTS["clef-flash"] + forged = type(variant)( + **{ + **variant.__dict__, + "config_sha256": sha256_file(tmp_path / "config.json"), + "joint_head_config_sha256": sha256_file(tmp_path / "joint_head_config.json"), + } + ) + shared_hashes = {name: sha256_file(tmp_path / name) for name in shared} + return forged, shared_hashes + + +def test_verify_snapshot_accepts_a_matching_tree_and_rejects_drift(tmp_path): + variant, shared = _snapshot(tmp_path) + verify_snapshot(tmp_path, variant, shared) + (tmp_path / "notes.txt").write_text("changed", encoding="utf-8") + with pytest.raises(ValueError, match="notes.txt"): + verify_snapshot(tmp_path, variant, shared) + + +def test_verify_snapshot_rejects_extra_python(tmp_path): + variant, shared = _snapshot(tmp_path, extra="extra.py") + with pytest.raises(ValueError, match="unexpected Python file"): + verify_snapshot(tmp_path, variant, shared) + + +def test_budget_refuses_overflow_and_disagreement(): + untruncated_budget(10, 4, 10) + with pytest.raises(RequestError, match="was not truncated"): + untruncated_budget(11, 4, 10) + with pytest.raises(RequestError, match="schema requires"): + untruncated_budget(11, 0, 10) + with pytest.raises(RuntimeError, match="token accounting mismatch"): + untruncated_budget(3, 4, 10) + + +def test_response_token_count_must_match_admission(): + response = {"usage": {"input_tokens": 12, "output_tokens": 0}} + assert require_token_agreement(12, response) is response + with pytest.raises(RuntimeError, match="encoder returned 9"): + require_token_agreement(12, {"usage": {"input_tokens": 9}}) diff --git a/tests/clef/test_postprocess.py b/tests/clef/test_postprocess.py new file mode 100644 index 00000000..0e106574 --- /dev/null +++ b/tests/clef/test_postprocess.py @@ -0,0 +1,40 @@ +from models.clef.postprocess import systemone_answer + + +def test_choice_keeps_request_order_and_breaks_ties_that_way(): + question = { + "type": "choice", + "criteria": {"technical": "Bugs", "billing": "Payments"}, + } + tied = systemone_answer(question, {"technical": 0.5, "billing": 0.5}) + assert list(tied["probabilities"]) == ["technical", "billing"] + assert tied["choice"] == "technical" + assert tied["confidence"] == 0.5 + + later = systemone_answer(question, {"technical": 0.25, "billing": 0.75}) + assert later["choice"] == "billing" + assert later["probabilities"] == {"technical": 0.25, "billing": 0.75} + + +def test_choice_is_selected_before_four_decimal_rounding(): + question = {"type": "choice", "criteria": {"a": "A", "b": "B"}} + answer = systemone_answer(question, {"a": 0.50004, "b": 0.49996}) + assert answer["choice"] == "a" + assert answer["probabilities"] == {"a": 0.5, "b": 0.5} + assert answer["confidence"] == 0.5 + + +def test_noul_reports_only_rounded_true_probability(): + assert systemone_answer({"type": "noul"}, {"true": 1 / 3, "false": 2 / 3}) == { + "type": "noul", + "noul": round(1 / 3, 4), + } + + +def test_score_is_the_rounded_expected_index(): + question = {"type": "score", "criteria": ["low", {"kind": "high"}]} + answer = systemone_answer(question, {"0": 0.25, "1": 0.75}) + assert answer["score"] == round(0.75, 4) + assert answer["confidence"] == 0.75 + assert answer["legend"] == {"0": "low", "1": {"kind": "high"}} + assert list(answer["probabilities"]) == ["0", "1"] diff --git a/tests/clef/test_prompt.py b/tests/clef/test_prompt.py new file mode 100644 index 00000000..db8302e8 --- /dev/null +++ b/tests/clef/test_prompt.py @@ -0,0 +1,55 @@ +import json +from pathlib import Path + +from models.clef.prompt import SUFFIX, prompt_pieces, render, token_lengths + +FIXTURE = Path(__file__).parent / "fixtures" / "text_contract.json" + + +def test_render_keeps_strings_and_sorts_object_keys(): + assert render("Checkout") == "Checkout" + assert render({"b": 1, "a": 2}) == '{"a":2,"b":1}' + assert render(None) == "null" + + +def test_fixture_pieces_match_the_pinned_encoder_trace(): + fixture = json.loads(FIXTURE.read_text(encoding="utf-8")) + assert fixture["provenance"]["loader_sha256"] + assert fixture["provenance"]["tokenizer_sha256"] + for case in fixture["cases"]: + pieces = prompt_pieces(case["request"]) + assert pieces == case["pieces"] + assert pieces[-1] == SUFFIX + assert "\n\n" in pieces[-1] + assert sum(case["piece_token_lengths"]) == len(case["input_ids"]) + assert case["fixed_length"] + case["state_tokens"] == len(case["input_ids"]) + assert token_lengths(lambda text: len(text.encode("utf-8")), case["request"])[1] > 0 + + +def test_choice_options_are_sorted_in_the_prompt_only(): + fixture = json.loads(FIXTURE.read_text(encoding="utf-8")) + pieces = prompt_pieces(fixture["cases"][0]["request"]) + billing = pieces.index('{"description":"Payments or invoices","option_id":"billing"}') + technical = pieces.index('{"description":"Bugs or outages","option_id":"technical"}') + assert billing < technical + omitted = prompt_pieces(fixture["cases"][1]["request"]) + assert "urgent" in omitted + assert omitted[omitted.index("\nFIELD 1\nID: urgent\nTYPE: noul\nINSTRUCTION: ") + 1] == "urgent" + + +def test_extra_noul_keys_do_not_become_options(): + pieces = prompt_pieces( + { + "state": "note", + "questions": { + "urgent": { + "type": "noul", + "criteria": {"true": "Yes.", "maybe": "Ignored by the encoder."}, + } + }, + } + ) + rendered = "\n".join(pieces) + assert "maybe" not in rendered + assert '"option_id":"true"' in rendered + assert '"option_id":"false"' in rendered diff --git a/tests/clef/test_protocol.py b/tests/clef/test_protocol.py new file mode 100644 index 00000000..65efbf8e --- /dev/null +++ b/tests/clef/test_protocol.py @@ -0,0 +1,109 @@ +import json +from pathlib import Path + +import pytest + +from models.clef.preprocess import MAX_INPUT_TOKENS, enforce_limit, measure, validate_questions +from models.clef.protocol import MAX_BODY, MalformedJSON, RequestError, parse_body, parse_request + + +def body(**overrides): + value = { + "model": "clef-flash", + "state": "Checkout has been failing.", + "questions": { + "team": { + "type": "choice", + "instructions": "Which team?", + "criteria": {"billing": "Payments", "technical": "Bugs"}, + } + }, + } + value.update(overrides) + return value + + +def test_parse_body_rejects_ambiguous_json_and_the_body_cap(): + with pytest.raises(MalformedJSON, match="duplicate"): + parse_body(b'{"model":"a","model":"b"}') + with pytest.raises(MalformedJSON, match="non-finite"): + parse_body(b'{"state":NaN}') + with pytest.raises(MalformedJSON): + parse_body(b"[1]") + with pytest.raises(RequestError) as excinfo: + parse_body(b'{"state":"' + b"a" * MAX_BODY + b'"}') + assert excinfo.value.status == 413 + + +def test_parse_requires_model_and_state_and_rejects_unknown_fields(): + with pytest.raises(RequestError, match="Unknown request fields"): + parse_request(body(media_kwargs={"fps": 1})) + with pytest.raises(RequestError, match="model and state are required"): + parse_request(body(model=None)) + parsed = parse_request(body(state={"b": 1, "a": 2})) + assert parsed.payload["state"] == {"b": 1, "a": 2} + + +def test_media_lists_are_refused_and_empty_lists_are_absent(): + assert parse_request(body(images=[], videos=[])).payload["images"] == [] + with pytest.raises(RequestError, match="images are not served"): + parse_request(body(images=["data:image/png;base64,AAAA"])) + with pytest.raises(RequestError, match="videos must be a list"): + parse_request(body(videos="clip.mp4")) + + +def test_question_shapes_match_the_three_decision_types(): + payload = body() + payload["questions"] = { + "team": payload["questions"]["team"], + "urgent": {"type": "noul"}, + "severity": {"type": "score", "criteria": ["No impact", "Critical"]}, + } + validate_questions(parse_request(payload)) + with pytest.raises(RequestError, match="type must be noul, choice, or score"): + validate_questions(parse_request(body(questions={"x": {"type": "rank"}}))) + with pytest.raises(RequestError, match="criteria must not be empty"): + validate_questions( + parse_request(body(questions={"x": {"type": "choice", "criteria": {}}})) + ) + with pytest.raises(RequestError, match="criteria must not be empty"): + validate_questions( + parse_request(body(questions={"x": {"type": "score", "criteria": {"0": "low"}}})) + ) + + +def test_limits_reject_schema_overflow_and_state_overflow_without_slicing(): + request = parse_request(body()) + fixed, total = measure(lambda text: len(text), request) + assert total > fixed > 0 + enforce_limit(fixed, total, total) + with pytest.raises(RequestError, match="schema requires") as schema_error: + enforce_limit(fixed, total, fixed - 1) + assert schema_error.value.status == 422 + with pytest.raises(RequestError, match="was not truncated") as state_error: + enforce_limit(fixed, total, total - 1) + assert state_error.value.status == 422 + assert MAX_INPUT_TOKENS == 16384 + + +def test_example_request_is_admitted(): + path = Path(__file__).resolve().parents[2] / "recipe" / "clef" / "example-request.json" + request = parse_request(json.loads(path.read_text(encoding="utf-8"))) + validate_questions(request) + fixed, total = measure(lambda text: 1, request) + enforce_limit(fixed, total) + + +def test_smoke_manifest_requests_are_admitted(): + path = Path_smoke() + for line in path.read_text(encoding="utf-8").splitlines(): + case = json.loads(line) + payload = {**case["request"], "model": "clef-flash"} + request = parse_request(payload) + validate_questions(request) + fixed, total = measure(lambda text: 1, request) + enforce_limit(fixed, total) + + +def Path_smoke(): + return Path(__file__).resolve().parents[2] / "benchmarks" / "smoke.jsonl" diff --git a/tests/clef/test_worker.py b/tests/clef/test_worker.py new file mode 100644 index 00000000..c96596f4 --- /dev/null +++ b/tests/clef/test_worker.py @@ -0,0 +1,210 @@ +import json +from http.client import HTTPConnection +from threading import Thread + +import pytest + +from frontend.clef import WorkerServer, decide +from models.clef.postprocess import systemone_answer +from models.clef.preprocess import enforce_limit, measure +from models.clef.protocol import MAX_BODY, Request, RequestError + + +def three_kinds(): + return { + "model": "clef-flash", + "state": "Checkout has been failing for every customer.", + "questions": { + "team": { + "type": "choice", + "instructions": "Which team should handle this?", + "criteria": {"technical": "Bugs", "billing": "Payments"}, + }, + "urgent": {"type": "noul", "instructions": "Is this urgent?"}, + "severity": { + "type": "score", + "instructions": "How severe is the impact?", + "criteria": ["No impact", "Minor", "Major"], + }, + }, + } + + +class FakeClef: + def __init__(self, max_length=16384, fail=False): + self.max_length = max_length + self.fail = fail + self.served = 0 + + def count_tokens(self, text: str) -> int: + return len(text) + + def serve(self, payload: dict): + self.served += 1 + if self.fail: + raise RuntimeError("boom") + fixed, total = measure(self.count_tokens, Request(payload)) + enforce_limit(fixed, total, self.max_length) + answers = { + name: systemone_answer( + question, + _uniform(question), + ) + for name, question in payload["questions"].items() + } + return { + "model": payload["model"], + "answers": answers, + "usage": {"input_tokens": total, "output_tokens": 0}, + } + + def health(self): + return { + "status": "ready", + "model": "Cloudflare/clef-flash", + "variant": "clef-flash", + "revision": "f0b7d3cd65296abfe459e6c228b618f5d8b26d2f", + "max_input_tokens": self.max_length, + "modalities": ["text"], + "prompt_mode": "non-thinking", + } + + def close(self): + pass + + +def _uniform(question): + if question["type"] == "noul": + return {"true": 0.25, "false": 0.75} + if question["type"] == "choice": + keys = list(question["criteria"]) + else: + keys = [str(index) for index in range(len(question["criteria"]))] + share = 1 / len(keys) + return {key: share for key in keys} + + +def test_decide_returns_three_answer_types_in_request_order(): + raw = json.dumps(three_kinds()).encode("utf-8") + response = decide(raw, FakeClef()) + assert list(response["answers"]) == ["team", "urgent", "severity"] + assert response["answers"]["team"]["choice"] == "technical" + assert response["answers"]["urgent"] == {"type": "noul", "noul": 0.25} + assert response["answers"]["severity"]["legend"]["0"] == "No impact" + assert response["usage"]["output_tokens"] == 0 + assert response["usage"]["input_tokens"] > 0 + + +def test_decide_rejects_overflow_before_a_second_serve_step(): + payload = three_kinds() + fixed, total = measure(lambda text: len(text), Request(payload)) + assert total > fixed + executor = FakeClef(max_length=total - 1) + with pytest.raises(RequestError, match="was not truncated") as excinfo: + decide(json.dumps(payload).encode("utf-8"), executor) + assert excinfo.value.status == 422 + assert executor.served == 1 + + +def test_decide_rejects_images(): + payload = three_kinds() + payload["images"] = ["data:image/png;base64,AAAA"] + with pytest.raises(RequestError, match="images are not served"): + decide(json.dumps(payload).encode("utf-8"), FakeClef()) + + +def start(executor): + server = WorkerServer(("127.0.0.1", 0), executor) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + return server + + +def stop(server): + server.shutdown() + server.server_close() + + +def post(server, payload, headers=()): + defaults = {"Content-Type": "application/json", "Content-Length": str(len(payload))} + extra = [] + for name, value in headers: + defaults.pop(name, None) + extra.append((name, value)) + conn = HTTPConnection("127.0.0.1", server.server_address[1], timeout=10) + conn.putrequest("POST", "/v1/systemone", skip_accept_encoding=True) + for name, value in (*defaults.items(), *extra): + conn.putheader(name, value) + conn.endheaders(payload) + response = conn.getresponse() + raw = response.read() + status = response.status + conn.close() + return status, raw + + +@pytest.fixture() +def server(): + instance = start(FakeClef()) + yield instance + stop(instance) + + +def test_health_and_post(server): + conn = HTTPConnection("127.0.0.1", server.server_address[1], timeout=10) + conn.request("GET", "/health") + response = conn.getresponse() + health = json.loads(response.read()) + conn.close() + assert response.status == 200 + assert health["variant"] == "clef-flash" + assert health["prompt_mode"] == "non-thinking" + assert health["modalities"] == ["text"] + assert health["max_input_tokens"] == 16384 + + status, raw = post(server, json.dumps(three_kinds()).encode("utf-8")) + body = json.loads(raw) + assert status == 200 + assert body["answers"]["urgent"]["noul"] == 0.25 + + +def test_transport_errors(server): + status, raw = post(server, b"{") + assert status == 400 and "detail" in json.loads(raw) + status, raw = post( + server, + json.dumps(three_kinds()).encode("utf-8"), + headers=(("Content-Type", "text/plain"),), + ) + assert status == 415 + status, raw = post( + server, + b"{}", + headers=(("Content-Length", str(MAX_BODY + 1)),), + ) + assert status == 413 and "8 MiB" in json.loads(raw)["detail"] + status, raw = post(server, b"", headers=(("Transfer-Encoding", "chunked"),)) + assert status == 411 + conn = HTTPConnection("127.0.0.1", server.server_address[1], timeout=10) + conn.request("POST", "/missing", body=b"{}") + response = conn.getresponse() + assert response.status == 404 + response.read() + conn.close() + + +def test_busy_and_inference_failure(server): + server.inference_lock.acquire() + try: + status, raw = post(server, json.dumps(three_kinds()).encode("utf-8")) + finally: + server.inference_lock.release() + assert status == 503 + + failed = start(FakeClef(fail=True)) + try: + status, raw = post(failed, json.dumps(three_kinds()).encode("utf-8")) + finally: + stop(failed) + assert status == 500 + assert json.loads(raw) == {"detail": "inference failed"} From aa43d0f3300f891af457efdac107d63b7a3bb031 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 25 Oct 2026 00:16:35 +0000 Subject: [PATCH 2/9] clef: admit thinking.enabled false and record the L20X C1 baseline Stock Public231 sends a thinking object. HEAD 9030065 returned HTTP 422 for that field, so the measured campaign used a stripped manifest. enabled false is now removed before encoding; enabled true stays rejected. The operator-reported flash Public231 C1 primary is run1 p50 247.518 ms and p95 413.025 ms. No unmeasured serve-path change is included. Co-authored-by: Hongsheng Liu --- README.md | 2 +- docs/supported-models.md | 10 ++++--- recipe/README.md | 4 +-- recipe/clef/reference.md | 39 +++++++++++++++++++++---- src/models/clef/README.md | 57 ++++++++++++++++++++++++++----------- src/models/clef/protocol.py | 24 ++++++++++++++-- tests/clef/test_protocol.py | 16 +++++++++++ tests/clef/test_worker.py | 9 ++++++ 8 files changed, 130 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 1190bce6..5e9f5b00 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ pinned runtime for text, image and video state: | LFM2.5-350M | [Transformers worker](recipe/lfm2/README.md); text state and independent choice questions | | OmniJev-4B v1.1 | [Checkpoint export and CPU contract](recipe/omnijev/README.md) only; the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | | D3 (Decision 3.0) | [Python reference worker](recipe/d3/reference.md) on the checkpoint's pinned runtime; `choice`/`noul`/`score` over text, image and video state; author-run on A100 for text and media serving (see the integration and media PRs) | -| Clef-flash (9B) and Clef (27B) | [Python reference worker](recipe/clef/reference.md) for text `choice`/`noul`/`score`; flash is the bring-up variant. Image, video and native CUDA are not served. No GPU run is recorded | +| Clef-flash (9B) and Clef (27B) | [Python reference worker](recipe/clef/reference.md) for text `choice`/`noul`/`score`; flash is the bring-up variant. Image, video and native CUDA are not served. Operator-reported flash Public231 C1 on L20X sm_89: run1 p50 247.518 ms, p95 413.025 ms | [Supported models and hardware](docs/supported-models.md) lists the devices and where each worker has been run. diff --git a/docs/supported-models.md b/docs/supported-models.md index 39a4f1d6..80541058 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -27,7 +27,7 @@ Models that are being added are also tracked in issues labeled [new model](https | CLM-v0.1-8B | [Native Rust engine](../src/models/clm/README.md), `clm-run`, in front of a Qwen3-8B `/v1/embeddings` server | Engine-only agreement **4.5e-06** on identical vectors ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)); contract checks documented ([#23](https://github.com/ThinkFlowLab/system1-omni/pull/23)) | **Validated on compute capability 8.9** ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)) | Unverified | Rust to build and a separate Qwen3-8B `/v1/embeddings` server for the encoder; the engine itself has no device-specific code | | OmniJev-4B v1.1 | None yet: [checkpoint export and CPU contract](../recipe/omnijev/README.md); the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | Request contract, preparation, heads and finishing checked against the reference ([#120](https://github.com/ThinkFlowLab/system1-omni/pull/120)); no inference | Not yet | Not supported | Python 3.10 reference environment for the export and fixtures | | D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) on the checkpoint's pinned runtime — text, image and video state over `choice`/`noul`/`score`, with the deliberate transport differences documented in the recipe; the native worker is in progress ([#136](https://github.com/ThinkFlowLab/system1-omni/issues/136)) | Unverified | Author-run on NVIDIA A100 (sm_80): serving smoke and reference parity for text and media (exact decisions, max delta 0.0); evidence in the integration and media PRs. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) | -| Clef-flash (9B) | [Python reference worker](../recipe/clef/reference.md); text `choice`/`noul`/`score`; non-thinking prompt; 16384-token refusal | Contract tests only; no inference | Not run. The operator checklist targets lab NVIDIA L20X at compute capability 8.9 and does not record a result | Not supported | Python 3.12, [card-tested torch 2.11 and transformers 5.10.2](../recipe/clef/requirements.txt), pinned `Cloudflare/clef-flash` snapshot (~19.1 GB) | +| Clef-flash (9B) | [Python reference worker](../recipe/clef/reference.md); text `choice`/`noul`/`score`; non-thinking prompt; `thinking.enabled: false` admitted and omitted from the prompt; 16384-token refusal | Contract tests only; no inference | Operator-reported Public231 C1 on NVIDIA L20X (sm_89), HEAD `903006530fb7772bb668bbf192ee4278e273e583`, warmup 5, concurrency 1. Primary run1 p50 247.518 ms, p95 413.025 ms, 231/231. Manifest `public231-no-thinking.jsonl` (`thinking` key stripped). Not re-run here | Not supported | Python 3.12, [card-tested torch 2.11 and transformers 5.10.2](../recipe/clef/requirements.txt), pinned `Cloudflare/clef-flash` snapshot (~19.1 GB) | | Clef (27B) | Same worker and text contract as Clef-flash, selected with `--variant clef` | Contract tests only; no inference | Not run | Not supported | Same stack; pinned `Cloudflare/clef` snapshot (~55.0 GB). Weights and joint-head config differ from flash | - **Validated:** covered by the recipe on `main` or by the checks in the linked merged pull request. @@ -41,9 +41,11 @@ D3's reference worker serves the same three kinds for text, image and video state (inline base64 media under the pinned runtime's per-item budgets); its media serving evidence is author-run on the A100 above. Clef-flash and Clef serve the same three kinds for text only. The worker -refuses image and video inputs. No CUDA timing is recorded; the recipe's -Public231 concurrency-1 checklist is for a lab NVIDIA L20X at compute -capability 8.9 and is not a result. +refuses image and video inputs. The flash CUDA cell is an operator-reported +Public231 concurrency-1 result on a lab NVIDIA L20X at compute capability +8.9 (primary run1 p50 247.518 ms, p95 413.025 ms). It is not an H200 or +H800 figure, and it was not re-run for this document. The 27B variant has +no recorded timing. CLM has a row per path: the merged `clm-serve` recipe exercises those answer shapes with stub embeddings and says nothing about decision quality, while the native engine's CUDA row is the path through a real Qwen3-8B encoder. Its engine-only figure comes from the same comparison run diff --git a/recipe/README.md b/recipe/README.md index ac5424a6..06010674 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -40,8 +40,8 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting (text serving and parity validated on A100; media budgets and caps in the recipe). - [Clef reference worker](clef/reference.md): load pinned Clef-flash (9B) or Clef (27B) and serve text `choice`/`noul`/`score`. Image, video and native - CUDA are not served. Public231 concurrency-1 timing on L20X is an operator - checklist; no latency number is recorded here. + CUDA are not served. Operator-reported flash Public231 concurrency-1 + timing on L20X sm_89 is in the recipe (run1 p50 247.518 ms, p95 413.025 ms). - [LFM2.5-350M choice worker](lfm2/README.md): score text decisions with independent questions, hybrid cache branching and configurable candidate batches. diff --git a/recipe/clef/reference.md b/recipe/clef/reference.md index 0afab678..53faf333 100644 --- a/recipe/clef/reference.md +++ b/recipe/clef/reference.md @@ -111,11 +111,31 @@ PYTHONPATH=src python recipe/clef/export_prompt_fixture.py \ ## Public231 C1 on lab L20X (sm_89) +Operator-reported, not re-run in the environment that edited this recipe. +Clef-flash, worker HEAD `903006530fb7772bb668bbf192ee4278e273e583`, NVIDIA +L20X at compute capability 8.9, warmup 5, concurrency 1. Stock Public231 +includes a `thinking` field. That HEAD rejected it with HTTP 422 +`Unknown request fields`. The measured manifest was +`public231-no-thinking.jsonl`, with the `thinking` key stripped. The +stripped file's SHA-256 was not reported. Primary for later deltas is run1. +Do not average the percentiles. + +| Round | p50 (ms) | p95 (ms) | Successful | +| --- | --- | --- | --- | +| run1 (primary) | 247.518 | 413.025 | 231/231 | +| run2 | 247.404 | 426.231 | 231/231 | +| run3 | 242.421 | 410.369 | 231/231 | + +The worker now admits `thinking` when it is an object with `enabled` set to +the JSON boolean `false`, then drops the field before encoding. A stock +Public231 request in that shape is the next comparison against run1. That +admission has no measured latency delta. `enabled: true` stays a 422. + This is the operator checklist for a concurrency-1 latency measurement of the flash worker. It uses the existing [benchmark runner](../../benchmarks/README.md). The client does not reserve a GPU and does not prove that CUDA ran. This repository does not contain -the Public231 JSONL. No p50 or p95 from that workload is known for Clef. +the Public231 JSONL. Record results outside the repository. Do not put hostnames, account names, home directories, or the name of a reservation system in the metadata or @@ -133,7 +153,11 @@ in a commit. The canonical task hash `dc3995d8ae1e2fc8e81ce38431add509eb8bb39b85aadfd0c7c32079382dde51` is a different encoding and must not be substituted. Check the file - with `sha256sum` before `validate`. + with `sha256sum` before `validate`. The recorded campaign above did not + use that byte string: the `thinking` key had been removed, and the + resulting file hash was not reported. Record `sha256sum` of whichever + file the next run actually sends. Keep `thinking` when its `enabled` + value is false; stripping it is no longer required for admission. 4. Install `benchmarks/requirements.txt` (`httpx==0.28.1`) into the client environment. The client and the worker may use different environments. 5. Write a metadata JSON file. Required keys are `gpu`, `gpu_ids`, @@ -192,7 +216,10 @@ in a commit. ## What remains unverified -GPU startup, the first forward, image and video inputs, native CUDA, and -every latency number. Hypotheses for later serve-path work are listed in -the model README. H1, one encode per request, is already how this worker -calls the pinned model. Its effect on p50 and p95 is not measured. +Image and video inputs, native CUDA, answer parity, and any latency delta +after HEAD `903006530fb7772bb668bbf192ee4278e273e583`. The operator-reported +C1 numbers above are the baseline. H1, one encode per request, is already +how that HEAD calls the pinned model, so run1 includes it and its isolated +effect is unknown. Further serve-path changes stay out until an L20X run +can cite a p50 and p95 delta against run1. This environment has no GPU and +did not repeat the campaign. diff --git a/src/models/clef/README.md b/src/models/clef/README.md index 38a36c3c..fee94bac 100644 --- a/src/models/clef/README.md +++ b/src/models/clef/README.md @@ -56,8 +56,14 @@ is the untruncated prompt length. The encoder's decision suffix is the empty think block `\n\n\n\nJOINT SCHEMA DECISIONS:`. That is the non-thinking path. The chat templates differ between the two revisions and can open a -think block; `systemone` does not apply those templates. There is no -thinking switch on this worker. `/health` reports `prompt_mode: non-thinking`. +think block; `systemone` does not apply those templates. `/health` reports +`prompt_mode: non-thinking`. + +A request may include `thinking` only as an object whose `enabled` value is +the JSON boolean `false`. That object is removed before encoding, including +any other keys on it, so the prompt matches a request that never had the +field. `enabled: true`, a non-object, and an object without `enabled: false` +are rejected. This worker does not serve a thinking prompt. Choice options are sorted by id inside the prompt. Answer `probabilities` keep the request's criteria order. Ties use that same request order. The @@ -81,9 +87,10 @@ of the state. The blog's 64k window and the backbone HTTP bodies larger than 8 MiB are rejected. That cap is larger than a 16384 token text request and smaller than an unbounded upload. Unknown request -fields are rejected. Non-empty `images` or `videos` are rejected with a -message that those modalities are not served. Empty lists are accepted and -add no media tokens. +fields are rejected. `thinking` is a known field and is admitted only by the +rule above. Non-empty `images` or `videos` are rejected with a message that +those modalities are not served. Empty lists are accepted and add no media +tokens. One request is scored at a time. The executor holds the loaded weights and the joint head for the process lifetime. The handler takes a process-wide @@ -120,26 +127,44 @@ PYTHONPATH=src python recipe/clef/export_prompt_fixture.py \ --check ``` -No weight was loaded. No GPU forward was run. Public231 concurrency-1 -latency on the lab L20X (compute capability 8.9) was not measured. The -measurement steps are in the [recipe](../../../recipe/clef/reference.md). +No weight was loaded in this environment. The operator-reported Public231 +concurrency-1 campaign below was not re-run here. The measurement steps are +in the [recipe](../../../recipe/clef/reference.md). + +## Latency -## Latency hypotheses +Operator-reported Public231 concurrency 1 on NVIDIA L20X, compute capability +8.9, Clef-flash, worker HEAD +`903006530fb7772bb668bbf192ee4278e273e583`, warmup 5. Stock Public231 carries +a `thinking` field and that HEAD returned HTTP 422 `Unknown request fields`. +The measured file was `public231-no-thinking.jsonl` (the `thinking` key +stripped). Its SHA-256 was not reported. Do not cite the leaderboard export +hash `00d1f10ca1142499a598b5f1c6602d329cce1cef04c961c46fc0318daa2c67d3` as +the hash of that file. Driver, CUDA version and hostname were not reported. +Do not read the H800 Public231 board as a result for this worker, and do not +label this run as H200. Percentiles are not averaged across rounds. + +| Round | p50 (ms) | p95 (ms) | Successful | +| --- | --- | --- | --- | +| run1, primary for later deltas | 247.518 | 413.025 | 231/231 | +| run2 | 247.404 | 426.231 | 231/231 | +| run3 | 242.421 | 410.369 | 231/231 | -No p50 or p95 is reported. Do not read the H800 Public231 board as a result -for this worker, and do not label an L20X run as H200. +The `thinking.enabled: false` admission in the current tree strips that +field before encoding. It has no measured p50 or p95 delta against run1. +Answer parity against another implementation was not part of this report. -| Id | Change | Expected effect | Delta | +| Id | Change | Expected effect | Delta vs run1 | | --- | --- | --- | --- | -| H1 | One `encode_record` per request, then the pinned forward. The handler does not tokenize the schema again before that encode. | Removes a second BPE pass from the critical path. Prefill and the joint head still dominate. | Not measured | -| H2 | Fuse the joint head's per-question Python loop. | Fewer small launches after prefill. Needs a forked head and numerical parity against the pinned module. | Not implemented | -| H3 | `torch.compile` or CUDA graphs on the Qwen3.5 backbone. | Uncertain on the linear-attention layers. | Not implemented | +| H1 | One `encode_record` per request, then the pinned forward. The handler does not tokenize the schema again before that encode. | Removes a second BPE pass from the critical path. Prefill and the joint head still dominate. | Included in HEAD `9030065`. Isolated delta unknown | +| H2 | Fuse the joint head's per-question Python loop. | Fewer small launches after prefill. Needs a forked head and numerical parity against the pinned module. | Not implemented. Held until an L20X measurement can cite a delta | +| H3 | `torch.compile` or CUDA graphs on the Qwen3.5 backbone. | Uncertain on the linear-attention layers. | Not implemented. Held until an L20X measurement can cite a delta | ## Status | Path | Clef-flash | Clef | | --- | --- | --- | -| Text `choice` / `noul` / `score`, non-thinking prompt | Contract tested; GPU inference unverified | Same loader pin; GPU inference unverified | +| Text `choice` / `noul` / `score`, non-thinking prompt | Contract tested. Operator-reported Public231 C1 latency on L20X sm_89 at HEAD `9030065`; answer parity not checked | Same loader pin; that latency campaign was flash only | | Image and video | Rejected | Rejected | | Native CUDA | Not implemented | Not implemented | | Workers AI proxy | Out of scope | Out of scope | diff --git a/src/models/clef/protocol.py b/src/models/clef/protocol.py index 9610c0ea..c44874f2 100644 --- a/src/models/clef/protocol.py +++ b/src/models/clef/protocol.py @@ -16,7 +16,7 @@ # body limit; 8 MiB is enough for a request that still has to be refused at # the 16384-token admission limit, and it refuses larger bodies explicitly. MAX_BODY = 8 * 1024 * 1024 -REQUEST_FIELDS = {"model", "state", "questions", "images", "videos"} +REQUEST_FIELDS = {"model", "state", "questions", "images", "videos", "thinking"} MEDIA_UNSUPPORTED = ( "{field} are not served by this integration; image and video inputs remain unsupported" ) @@ -91,6 +91,24 @@ def _reject_media(value: dict[str, Any], field: str) -> None: raise RequestError(MEDIA_UNSUPPORTED.format(field=field)) +def _accept_thinking(value: dict[str, Any]) -> None: + """Allow the stock Public231 ``thinking`` object only when it is off. + + The pinned encoder always writes the empty think block. ``enabled: true`` + is rejected so a thinking request is not scored on that prompt. Other keys + on the object are not interpreted. The key is removed before encoding, so + the text matches a request that never had the field. + """ + + if "thinking" not in value: + return + thinking = value["thinking"] + if not isinstance(thinking, dict) or thinking.get("enabled") is not False: + raise RequestError( + "thinking.enabled must be false; this worker only serves the non-thinking prompt" + ) + + def parse_request(value: dict[str, Any]) -> Request: """Validate the systemone envelope and refuse media inputs.""" @@ -99,6 +117,7 @@ def parse_request(value: dict[str, Any]) -> Request: raise RequestError(f"Unknown request fields: {unknown}") _reject_media(value, "images") _reject_media(value, "videos") + _accept_thinking(value) if not isinstance(value.get("model"), str) or "state" not in value: raise RequestError("model and state are required") questions = value.get("questions") @@ -106,4 +125,5 @@ def parse_request(value: dict[str, Any]) -> Request: raise RequestError("at least one question is required") if any(not isinstance(name, str) or not name for name in questions): raise RequestError("question IDs must be nonempty strings") - return Request(value) + payload = {key: item for key, item in value.items() if key != "thinking"} + return Request(payload) diff --git a/tests/clef/test_protocol.py b/tests/clef/test_protocol.py index 65efbf8e..ff276619 100644 --- a/tests/clef/test_protocol.py +++ b/tests/clef/test_protocol.py @@ -35,6 +35,22 @@ def test_parse_body_rejects_ambiguous_json_and_the_body_cap(): assert excinfo.value.status == 413 +def test_thinking_enabled_false_is_admitted_and_omitted_from_the_prompt(): + from models.clef.prompt import prompt_pieces + + plain = body() + marked = body(thinking={"enabled": False, "ignored": 1}) + parsed = parse_request(marked) + assert "thinking" not in parsed.payload + assert prompt_pieces(parsed.payload) == prompt_pieces(plain) + with pytest.raises(RequestError, match="thinking.enabled must be false"): + parse_request(body(thinking={"enabled": True})) + with pytest.raises(RequestError, match="thinking.enabled must be false"): + parse_request(body(thinking="off")) + with pytest.raises(RequestError, match="thinking.enabled must be false"): + parse_request(body(thinking={"budget": 0})) + + def test_parse_requires_model_and_state_and_rejects_unknown_fields(): with pytest.raises(RequestError, match="Unknown request fields"): parse_request(body(media_kwargs={"fps": 1})) diff --git a/tests/clef/test_worker.py b/tests/clef/test_worker.py index c96596f4..47b62bb4 100644 --- a/tests/clef/test_worker.py +++ b/tests/clef/test_worker.py @@ -106,6 +106,15 @@ def test_decide_rejects_overflow_before_a_second_serve_step(): assert executor.served == 1 +def test_decide_accepts_thinking_enabled_false(): + payload = three_kinds() + payload["thinking"] = {"enabled": False} + response = decide(json.dumps(payload).encode("utf-8"), FakeClef()) + assert response["answers"]["urgent"]["noul"] == 0.25 + bare = decide(json.dumps(three_kinds()).encode("utf-8"), FakeClef()) + assert response["usage"]["input_tokens"] == bare["usage"]["input_tokens"] + + def test_decide_rejects_images(): payload = three_kinds() payload["images"] = ["data:image/png;base64,AAAA"] From 033685b7e9511eff5672855b212535303ae25dab Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 25 Oct 2026 00:25:13 +0000 Subject: [PATCH 3/9] clef: install the fast linear-attention kernels for the next L20X run Transformers 5.10.2 keeps the PyTorch gated-delta fallback unless flash-linear-attention and causal-conv1d import first. The recipe now installs those builds, and /health reports which binding loaded. The Public231 C1 primary is unchanged at run1 p50 247.518 ms and p95 413.025 ms. No new latency or answer-parity delta was measured. Co-authored-by: Hongsheng Liu --- docs/supported-models.md | 9 ++++--- recipe/clef/reference.md | 29 ++++++++++++++++------ recipe/clef/requirements.txt | 7 ++++++ src/models/clef/README.md | 20 +++++++++++---- src/models/clef/engine.py | 47 ++++++++++++++++++++++++++++++++++++ tests/clef/test_engine.py | 22 +++++++++++++++++ 6 files changed, 119 insertions(+), 15 deletions(-) diff --git a/docs/supported-models.md b/docs/supported-models.md index 80541058..e6854759 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -43,9 +43,12 @@ its media serving evidence is author-run on the A100 above. Clef-flash and Clef serve the same three kinds for text only. The worker refuses image and video inputs. The flash CUDA cell is an operator-reported Public231 concurrency-1 result on a lab NVIDIA L20X at compute capability -8.9 (primary run1 p50 247.518 ms, p95 413.025 ms). It is not an H200 or -H800 figure, and it was not re-run for this document. The 27B variant has -no recorded timing. +8.9 (primary run1 p50 247.518 ms, p95 413.025 ms, metric +`successful_latency`, clock `post_to_body_json_and_schema_validation`). +It is not an H200 or H800 figure, and it was not re-run for this document. +The recipe installs flash-linear-attention and causal-conv1d for the next +run; that binding has no measured latency or answer-parity delta. The 27B +variant has no recorded timing. CLM has a row per path: the merged `clm-serve` recipe exercises those answer shapes with stub embeddings and says nothing about decision quality, while the native engine's CUDA row is the path through a real Qwen3-8B encoder. Its engine-only figure comes from the same comparison run diff --git a/recipe/clef/reference.md b/recipe/clef/reference.md index 53faf333..0c28e1fe 100644 --- a/recipe/clef/reference.md +++ b/recipe/clef/reference.md @@ -7,8 +7,8 @@ unsupported image, video and native-CUDA paths are in the repository root. Clef-flash is the bring-up variant: about 19.1 GB of safetensors versus -about 55.0 GB for Clef. Both use the same text contract. This recipe does -not record a GPU run. +about 55.0 GB for Clef. Both use the same text contract. The recorded +Public231 concurrency-1 result is later in this recipe. ## Pins @@ -131,6 +131,19 @@ the JSON boolean `false`, then drops the field before encoding. A stock Public231 request in that shape is the next comparison against run1. That admission has no measured latency delta. `enabled: true` stays a 422. +The operator named the metric `successful_latency` and the clock +`post_to_body_json_and_schema_validation`. These three rounds are that +measurement. They are not a new campaign. + +`recipe/clef/requirements.txt` now installs `flash-linear-attention==0.5.2` +and `causal-conv1d>=1.5`. Transformers 5.10.2 uses those packages, when +they import before `modeling_qwen3_5`, in place of +`torch_chunk_gated_delta_rule` and the conv1d fallback. `/health` reports +the binding in `linear_attention` (`torch_chunk` when neither bound). +Restart the worker after installing them. The p50 and p95 delta of that +binding against run1 is not measured. Answer parity against the torch +fallback is not measured. Do not publish a speedup from an unmeasured run. + This is the operator checklist for a concurrency-1 latency measurement of the flash worker. It uses the existing [benchmark runner](../../benchmarks/README.md). The client does not reserve @@ -218,8 +231,10 @@ in a commit. Image and video inputs, native CUDA, answer parity, and any latency delta after HEAD `903006530fb7772bb668bbf192ee4278e273e583`. The operator-reported -C1 numbers above are the baseline. H1, one encode per request, is already -how that HEAD calls the pinned model, so run1 includes it and its isolated -effect is unknown. Further serve-path changes stay out until an L20X run -can cite a p50 and p95 delta against run1. This environment has no GPU and -did not repeat the campaign. +C1 numbers above are the baseline, on the clock +`post_to_body_json_and_schema_validation`, metric `successful_latency`. +H1, one encode per request, is already how that HEAD calls the pinned +model, so run1 includes it and its isolated effect is unknown. The recipe +now installs the fast linear-attention packages. That binding has no +measured p50, p95, or answer-parity delta against run1. This environment +has no GPU and did not repeat the campaign. diff --git a/recipe/clef/requirements.txt b/recipe/clef/requirements.txt index ce46d539..0d83d4fc 100644 --- a/recipe/clef/requirements.txt +++ b/recipe/clef/requirements.txt @@ -9,3 +9,10 @@ tokenizers>=0.22 # The processor loads with the backbone. Image and video requests are rejected # by this worker; pillow is still imported by the pinned processor. pillow>=10 +# Transformers 5.10.2 binds these at the first import of modeling_qwen3_5. +# Installed, they replace torch_chunk_gated_delta_rule and the conv1d +# fallback. They are CUDA builds. No L20X latency delta and no answer-parity +# delta against the torch fallback have been measured. If either import +# fails, the worker stays on the torch path and /health says so. +flash-linear-attention==0.5.2 +causal-conv1d>=1.5 diff --git a/src/models/clef/README.md b/src/models/clef/README.md index fee94bac..53892ec6 100644 --- a/src/models/clef/README.md +++ b/src/models/clef/README.md @@ -150,15 +150,25 @@ label this run as H200. Percentiles are not averaged across rounds. | run2 | 247.404 | 426.231 | 231/231 | | run3 | 242.421 | 410.369 | 231/231 | -The `thinking.enabled: false` admission in the current tree strips that -field before encoding. It has no measured p50 or p95 delta against run1. -Answer parity against another implementation was not part of this report. +The `thinking.enabled: false` admission strips that field before encoding. +It has no measured p50 or p95 delta against run1. The operator named the +metric `successful_latency` and the clock +`post_to_body_json_and_schema_validation`. Answer parity against another +implementation was not part of this report. + +`flash-linear-attention==0.5.2` and `causal-conv1d>=1.5` are installed by +the recipe. Transformers 5.10.2 binds them, if they import first, instead +of the PyTorch chunked gated-delta rule and the conv1d fallback. `/health` +field `linear_attention` is that binding. Their p50 and p95 delta against +run1 is not measured, and their answer parity against `torch_chunk` is not +measured. | Id | Change | Expected effect | Delta vs run1 | | --- | --- | --- | --- | | H1 | One `encode_record` per request, then the pinned forward. The handler does not tokenize the schema again before that encode. | Removes a second BPE pass from the critical path. Prefill and the joint head still dominate. | Included in HEAD `9030065`. Isolated delta unknown | -| H2 | Fuse the joint head's per-question Python loop. | Fewer small launches after prefill. Needs a forked head and numerical parity against the pinned module. | Not implemented. Held until an L20X measurement can cite a delta | -| H3 | `torch.compile` or CUDA graphs on the Qwen3.5 backbone. | Uncertain on the linear-attention layers. | Not implemented. Held until an L20X measurement can cite a delta | +| H2 | Fuse the joint head's per-question Python loop. | Fewer small launches after prefill. Needs a forked head and numerical parity against the pinned module. | Not implemented. No measured L20X delta | +| H3 | `torch.compile` or CUDA graphs on the Qwen3.5 backbone. | Uncertain on the linear-attention layers. | Not implemented. No measured L20X delta | +| H4 | Let Transformers bind `flash-linear-attention` 0.5.2 and `causal-conv1d` for the 24 linear-attention layers. | Replaces the PyTorch chunk loop. The recipe installs the packages. A process that already imported the modeling module does not rebind. | Not measured | ## Status diff --git a/src/models/clef/engine.py b/src/models/clef/engine.py index 2fb0a431..adb28c41 100644 --- a/src/models/clef/engine.py +++ b/src/models/clef/engine.py @@ -14,8 +14,10 @@ from __future__ import annotations import hashlib +import importlib import importlib.util import json +import logging import sys from dataclasses import dataclass from pathlib import Path @@ -36,6 +38,8 @@ "processor_config.json": PROCESSOR_CONFIG_SHA256, } EXECUTABLE = "joint_schema_model.py" +MODELING = "transformers.models.qwen3_5.modeling_qwen3_5" +LOG = logging.getLogger(__name__) @dataclass(frozen=True) @@ -168,6 +172,46 @@ def require_token_agreement(admitted: int, response: dict[str, Any]) -> dict[str return response +def bound_linear_attention(modeling: Any) -> str: + """Name the linear-attention implementation Transformers actually bound. + + ``modeling_qwen3_5`` keeps the fast callables as module globals when the + optional packages imported, and ``None`` when it kept the PyTorch + fallback. The report is that binding, not an install attempt. + """ + + if modeling is None: + return "unloaded" + + fla = getattr(modeling, "chunk_gated_delta_rule", None) is not None + conv = getattr(modeling, "causal_conv1d_fn", None) is not None + if fla and conv: + return "fla+causal_conv1d" + if fla: + return "fla" + if conv: + return "causal_conv1d" + return "torch_chunk" + + +def activate_linear_attention() -> None: + """Import the optional kernels before Transformers binds Qwen3.5. + + The modeling module chooses ``chunk_gated_delta_rule`` and + ``causal_conv1d_fn`` once, at import. A later install does not move a + process that already imported the module. A failed import leaves the + PyTorch fallback in place. + """ + + if MODELING in sys.modules: + return + for name in ("causal_conv1d", "fla.ops.gated_delta_rule"): + try: + importlib.import_module(name) + except (ImportError, OSError) as exc: + LOG.info("clef linear-attention package %s is not active: %s", name, type(exc).__name__) + + def load_joint_schema_module(path: Path) -> Any: """Import one verified ``joint_schema_model.py`` without putting its directory on ``sys.path``.""" @@ -191,8 +235,10 @@ def __init__(self, snapshot: str | Path, variant: str = DEFAULT_VARIANT, device: self.device = device self.max_length = MAX_INPUT_TOKENS verify_snapshot(self.root, self.variant) + activate_linear_attention() self.module = load_joint_schema_module(self.root / EXECUTABLE) self.model, self.processor = self.module.load_release_model(self.root, device=device) + self.linear_attention = bound_linear_attention(sys.modules.get(MODELING)) if self.processor.tokenizer.pad_token_id is None: raise ValueError("tokenizer pad_token_id is required") @@ -262,6 +308,7 @@ def health(self) -> dict[str, Any]: "max_input_tokens": self.max_length, "modalities": ["text"], "prompt_mode": "non-thinking", + "linear_attention": self.linear_attention, } def close(self) -> None: diff --git a/tests/clef/test_engine.py b/tests/clef/test_engine.py index 5a55cef9..8c1a4590 100644 --- a/tests/clef/test_engine.py +++ b/tests/clef/test_engine.py @@ -89,3 +89,25 @@ def test_response_token_count_must_match_admission(): assert require_token_agreement(12, response) is response with pytest.raises(RuntimeError, match="encoder returned 9"): require_token_agreement(12, {"usage": {"input_tokens": 9}}) + + +def test_linear_attention_name_follows_the_bound_callables(): + from models.clef.engine import activate_linear_attention, bound_linear_attention + + assert bound_linear_attention(None) == "unloaded" + assert bound_linear_attention(object()) == "torch_chunk" + assert ( + bound_linear_attention(type("M", (), {"chunk_gated_delta_rule": object()})()) + == "fla" + ) + assert ( + bound_linear_attention(type("M", (), {"causal_conv1d_fn": object()})()) + == "causal_conv1d" + ) + both = type( + "M", + (), + {"chunk_gated_delta_rule": object(), "causal_conv1d_fn": object()}, + )() + assert bound_linear_attention(both) == "fla+causal_conv1d" + activate_linear_attention() From 4e430e18a26197c6fa70b8733852020f10e78a27 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 25 Oct 2026 01:30:37 +0000 Subject: [PATCH 4/9] clef: wire FlashInfer GDN prefill for H200 Ready now requires flashinfer-python 0.6.16.post3, flash-linear-attention 0.5.2, and causal-conv1d >= 1.5, with the chunk global replaced before Qwen3.5 layers copy it. Startup fails closed on torch_chunk, F.conv1d, an FLA-only chunk, and any CUDA device that is not compute capability 9.0. The recorded L20X Public231 rounds stay labeled L20X. No H200 C1 is claimed. Co-authored-by: Hongsheng Liu --- README.md | 2 +- docs/supported-models.md | 9 +- recipe/README.md | 2 + recipe/clef/reference.md | 108 ++++++++++----- recipe/clef/requirements.txt | 18 ++- src/models/clef/README.md | 35 +++-- src/models/clef/engine.py | 258 ++++++++++++++++++++++++++++++++--- tests/clef/test_engine.py | 69 ++++++++-- 8 files changed, 413 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 5e9f5b00..d850ddd6 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ pinned runtime for text, image and video state: | LFM2.5-350M | [Transformers worker](recipe/lfm2/README.md); text state and independent choice questions | | OmniJev-4B v1.1 | [Checkpoint export and CPU contract](recipe/omnijev/README.md) only; the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | | D3 (Decision 3.0) | [Python reference worker](recipe/d3/reference.md) on the checkpoint's pinned runtime; `choice`/`noul`/`score` over text, image and video state; author-run on A100 for text and media serving (see the integration and media PRs) | -| Clef-flash (9B) and Clef (27B) | [Python reference worker](recipe/clef/reference.md) for text `choice`/`noul`/`score`; flash is the bring-up variant. Image, video and native CUDA are not served. Operator-reported flash Public231 C1 on L20X sm_89: run1 p50 247.518 ms, p95 413.025 ms | +| Clef-flash (9B) and Clef (27B) | [Python reference worker](recipe/clef/reference.md) for text `choice`/`noul`/`score`; flash is the bring-up variant. Image, video and native CUDA are not served. Operator-reported flash Public231 C1 on L20X sm_89: run1 p50 247.518 ms, p95 413.025 ms. That figure is not an H200 result. The forward prefill target is H200 (sm_90) and has no recorded C1 | [Supported models and hardware](docs/supported-models.md) lists the devices and where each worker has been run. diff --git a/docs/supported-models.md b/docs/supported-models.md index e6854759..4feef834 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -45,10 +45,11 @@ refuses image and video inputs. The flash CUDA cell is an operator-reported Public231 concurrency-1 result on a lab NVIDIA L20X at compute capability 8.9 (primary run1 p50 247.518 ms, p95 413.025 ms, metric `successful_latency`, clock `post_to_body_json_and_schema_validation`). -It is not an H200 or H800 figure, and it was not re-run for this document. -The recipe installs flash-linear-attention and causal-conv1d for the next -run; that binding has no measured latency or answer-parity delta. The 27B -variant has no recorded timing. +That cell stays an L20X figure. It is not an H200 or H800 result, and it +was not re-run for this document. The forward path wires FlashInfer GDN +prefill with FLA and causal-conv1d for compute capability 9.0. No H200 C1 +latency or answer-parity delta is recorded. The 27B variant has no +recorded timing. CLM has a row per path: the merged `clm-serve` recipe exercises those answer shapes with stub embeddings and says nothing about decision quality, while the native engine's CUDA row is the path through a real Qwen3-8B encoder. Its engine-only figure comes from the same comparison run diff --git a/recipe/README.md b/recipe/README.md index 06010674..f9556cd5 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -42,6 +42,8 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting Clef (27B) and serve text `choice`/`noul`/`score`. Image, video and native CUDA are not served. Operator-reported flash Public231 concurrency-1 timing on L20X sm_89 is in the recipe (run1 p50 247.518 ms, p95 413.025 ms). + That figure is not an H200 result. The forward prefill target is H200 + (sm_90); no H200 C1 is recorded. - [LFM2.5-350M choice worker](lfm2/README.md): score text decisions with independent questions, hybrid cache branching and configurable candidate batches. diff --git a/recipe/clef/reference.md b/recipe/clef/reference.md index 0c28e1fe..58c42ae2 100644 --- a/recipe/clef/reference.md +++ b/recipe/clef/reference.md @@ -31,6 +31,36 @@ here. python -m pip install -r recipe/clef/requirements.txt ``` +The requirements pin `causal-conv1d>=1.5`, `flash-linear-attention==0.5.2`, +and `flashinfer-python==0.6.16.post3`. The worker imports `causal_conv1d`, +then `fla.ops.gated_delta_rule`, then `flashinfer.gdn_prefill`, and only +then `transformers.models.qwen3_5.modeling_qwen3_5`. That order binds +`FusedRMSNormGated` and `causal_conv1d_fn`. Startup then replaces the +modeling module's `chunk_gated_delta_rule` with the FlashInfer GDN prefill +wrapper before `load_release_model` builds each `Qwen3_5GatedDeltaNet`, +because the layer copies that global at init. + +Startup raises when any of the three distributions is missing, when +flash-linear-attention is not exactly `0.5.2`, when flashinfer-python is +not exactly `0.6.16.post3`, when causal-conv1d is older than 1.5, when an +import fails, when `causal_conv1d_fn` did not bind, or when the chunk +global is not the FlashInfer wrapper. A ready process has `/health` +`linear_attention` equal to `flashinfer+fla+causal_conv1d`. The PyTorch +`torch_chunk_gated_delta_rule` and `F.conv1d` paths are not served. An FLA +chunk binding without the FlashInfer wrapper is not ready. + +Prefill calls `flashinfer.gdn_prefill.chunk_gated_delta_rule` with +`use_cp=False`, so Hopper uses the fused SM90 kernel. Q and K L2 +normalization uses FLA `l2norm_fwd`, and FlashInfer's +`use_qk_l2norm_in_kernel` flag stays false. Log-space `g` is +exponentiated. The call is one sequence and does not continue a recurrent +state. vLLM's separate CuteDSL prefill backend, CUDA graphs, and +`torch.compile` are not used. The device must be CUDA compute capability +9.0. A device named `NVIDIA L20X` at capability 8.9 is the wrong machine. +The same name at capability 9.0 is the known driver mislabel of an H200 +and passes the capability check. No H200 Public231 C1 latency, and no +answer parity against `torch_chunk` or the FLA chunk, has been measured. + ```sh python - <<'PY' from huggingface_hub import snapshot_download @@ -117,8 +147,8 @@ L20X at compute capability 8.9, warmup 5, concurrency 1. Stock Public231 includes a `thinking` field. That HEAD rejected it with HTTP 422 `Unknown request fields`. The measured manifest was `public231-no-thinking.jsonl`, with the `thinking` key stripped. The -stripped file's SHA-256 was not reported. Primary for later deltas is run1. -Do not average the percentiles. +stripped file's SHA-256 was not reported. Run1 is the primary L20X round. +Do not average the percentiles. Run1 is not a baseline for an H200 delta. | Round | p50 (ms) | p95 (ms) | Successful | | --- | --- | --- | --- | @@ -127,25 +157,30 @@ Do not average the percentiles. | run3 | 242.421 | 410.369 | 231/231 | The worker now admits `thinking` when it is an object with `enabled` set to -the JSON boolean `false`, then drops the field before encoding. A stock -Public231 request in that shape is the next comparison against run1. That -admission has no measured latency delta. `enabled: true` stays a 422. +the JSON boolean `false`, then drops the field before encoding. The next +H200 campaign should send stock Public231 in that shape. That admission +has no measured latency delta on these L20X rounds. `enabled: true` stays +a 422. The operator named the metric `successful_latency` and the clock `post_to_body_json_and_schema_validation`. These three rounds are that measurement. They are not a new campaign. -`recipe/clef/requirements.txt` now installs `flash-linear-attention==0.5.2` -and `causal-conv1d>=1.5`. Transformers 5.10.2 uses those packages, when -they import before `modeling_qwen3_5`, in place of -`torch_chunk_gated_delta_rule` and the conv1d fallback. `/health` reports -the binding in `linear_attention` (`torch_chunk` when neither bound). -Restart the worker after installing them. The p50 and p95 delta of that -binding against run1 is not measured. Answer parity against the torch -fallback is not measured. Do not publish a speedup from an unmeasured run. - -This is the operator checklist for a concurrency-1 latency measurement of -the flash worker. It uses the existing +Those three rounds are the L20X report at HEAD `9030065`. They are not an +H200 baseline. Do not relabel them as H200, and do not treat a later H200 +p50 or p95 as a same-device speedup against 247.518 / 413.025. + +## Next measurement: H200 (sm_90) + +The forward operator path is FlashInfer GDN prefill plus `causal_conv1d` +and FLA, on compute capability 9.0. No H200 Public231 C1 has been recorded. +Answer parity of that prefill against `torch_chunk` and against the FLA +chunk is not measured. The standalone GDN timings in +[benchmarks/gdn/README.md](../../benchmarks/gdn/README.md) are a different +experiment and are not a Clef HTTP result. + +This is the operator checklist for the next concurrency-1 latency +measurement of the flash worker. It uses the existing [benchmark runner](../../benchmarks/README.md). The client does not reserve a GPU and does not prove that CUDA ran. This repository does not contain the Public231 JSONL. @@ -154,11 +189,19 @@ Record results outside the repository. Do not put hostnames, account names, home directories, or the name of a reservation system in the metadata or in a commit. -1. Confirm the GPU the process will use. `nvidia-smi` must show the device - name `NVIDIA L20X` and compute capability 8.9. If either differs, stop. - Do not report the run as H200, H800, or as the Public231 leaderboard. +1. Confirm the GPU the process will use. `nvidia-smi` must show compute + capability 9.0. Record the device name it prints. Capability 9.0 is the + H200 target, including when the name is the known `NVIDIA L20X` + mislabel. Capability 8.9 is Ada; stop, and do not report that machine + as this H200 run. Do not report the run as H800 or as the Public231 + leaderboard. 2. Start `frontend.clef` as above, on loopback port 8000, variant `clef-flash`, device `cuda`. Leave that process up for every traversal. + `GET /health` must show `status` `ready` and `linear_attention` + `flashinfer+fla+causal_conv1d`. A process that exits at startup did not + bind FlashInfer GDN prefill and `causal_conv1d_fn`. Do not measure a + process that is still on the PyTorch chunk, `F.conv1d`, or an FLA-only + chunk path. 3. Obtain the frozen Public231 request manifest whose file SHA-256 is `00d1f10ca1142499a598b5f1c6602d329cce1cef04c961c46fc0318daa2c67d3`. That is the JSONL byte hash cited for the JevBench `7ce310c7` export in @@ -180,7 +223,7 @@ in a commit. ```json { - "gpu": "NVIDIA L20X", + "gpu": "NVIDIA H200", "gpu_ids": ["0"], "driver": "", "cuda": "", @@ -188,7 +231,7 @@ in a commit. "model_revision": "Cloudflare/clef-flash@f0b7d3cd65296abfe459e6c228b618f5d8b26d2f", "runtime_revision": "", "cache_policy": "one request at a time; no prefix cache; no CUDA graph; transformers cache left at its default", - "cuda_evidence": "nvidia-smi showed NVIDIA L20X compute capability 8.9 while the worker was running with --device cuda", + "cuda_evidence": "nvidia-smi showed compute capability 9.0 while the worker was running with --device cuda. Record the printed device name. If that name is NVIDIA L20X and the capability is 9.0, say so; that pair is the known H200 mislabel.", "reservation": "" } ``` @@ -224,17 +267,18 @@ in a commit. per-round and pooled counts. Keep failure latencies separate. 9. The clock is the runner's `post_to_body_json_and_schema_validation` clock. It is not the H800 - leaderboard clock. Do not compute a speedup against that board, against - H200, or against an unmeasured hypothesis. + leaderboard clock. Report this campaign as the H200 C1. Do not relabel + the L20X table above as this result, and do not compute a same-device + speedup against 247.518 / 413.025. ## What remains unverified -Image and video inputs, native CUDA, answer parity, and any latency delta -after HEAD `903006530fb7772bb668bbf192ee4278e273e583`. The operator-reported -C1 numbers above are the baseline, on the clock -`post_to_body_json_and_schema_validation`, metric `successful_latency`. -H1, one encode per request, is already how that HEAD calls the pinned -model, so run1 includes it and its isolated effect is unknown. The recipe -now installs the fast linear-attention packages. That binding has no -measured p50, p95, or answer-parity delta against run1. This environment -has no GPU and did not repeat the campaign. +Image and video inputs, native CUDA, answer parity, and any H200 latency. +The operator-reported L20X numbers above stay on the clock +`post_to_body_json_and_schema_validation`, metric `successful_latency`, at +HEAD `903006530fb7772bb668bbf192ee4278e273e583`. H1, one encode per request, +is already how that HEAD calls the pinned model, so run1 includes it and +its isolated effect is unknown. The H200 path wires FlashInfer GDN prefill, +FLA, and causal-conv1d. That path has no measured p50, p95, or +answer-parity delta. This environment has no GPU and did not repeat the +campaign. diff --git a/recipe/clef/requirements.txt b/recipe/clef/requirements.txt index 0d83d4fc..3f3986b6 100644 --- a/recipe/clef/requirements.txt +++ b/recipe/clef/requirements.txt @@ -9,10 +9,16 @@ tokenizers>=0.22 # The processor loads with the backbone. Image and video requests are rejected # by this worker; pillow is still imported by the pinned processor. pillow>=10 -# Transformers 5.10.2 binds these at the first import of modeling_qwen3_5. -# Installed, they replace torch_chunk_gated_delta_rule and the conv1d -# fallback. They are CUDA builds. No L20X latency delta and no answer-parity -# delta against the torch fallback have been measured. If either import -# fails, the worker stays on the torch path and /health says so. -flash-linear-attention==0.5.2 +# Import order is causal_conv1d, then fla.ops.gated_delta_rule, then +# flashinfer.gdn_prefill, and only then modeling_qwen3_5. FLA must be +# importable first so Transformers binds FusedRMSNormGated instead of the +# PyTorch norm. The worker then replaces the chunk global with FlashInfer +# GDN prefill (use_cp false, fused SM90) before layers copy it. +# causal_conv1d_fn stays the convolution. Startup refuses torch_chunk and +# F.conv1d, and refuses an FLA chunk binding that was not replaced. +# The target device is H200, compute capability 9.0. vLLM's separate CuteDSL +# prefill backend, CUDA graphs, and torch.compile are not selected. +# No H200 Public231 latency or answer-parity delta has been measured. causal-conv1d>=1.5 +flash-linear-attention==0.5.2 +flashinfer-python==0.6.16.post3 diff --git a/src/models/clef/README.md b/src/models/clef/README.md index 53892ec6..b4c10050 100644 --- a/src/models/clef/README.md +++ b/src/models/clef/README.md @@ -146,35 +146,42 @@ label this run as H200. Percentiles are not averaged across rounds. | Round | p50 (ms) | p95 (ms) | Successful | | --- | --- | --- | --- | -| run1, primary for later deltas | 247.518 | 413.025 | 231/231 | +| run1, primary L20X round | 247.518 | 413.025 | 231/231 | | run2 | 247.404 | 426.231 | 231/231 | | run3 | 242.421 | 410.369 | 231/231 | The `thinking.enabled: false` admission strips that field before encoding. -It has no measured p50 or p95 delta against run1. The operator named the +These L20X rounds do not include a measured p50 or p95 for that admission. +The operator named the metric `successful_latency` and the clock `post_to_body_json_and_schema_validation`. Answer parity against another implementation was not part of this report. -`flash-linear-attention==0.5.2` and `causal-conv1d>=1.5` are installed by -the recipe. Transformers 5.10.2 binds them, if they import first, instead -of the PyTorch chunked gated-delta rule and the conv1d fallback. `/health` -field `linear_attention` is that binding. Their p50 and p95 delta against -run1 is not measured, and their answer parity against `torch_chunk` is not -measured. - -| Id | Change | Expected effect | Delta vs run1 | +`causal-conv1d>=1.5`, `flash-linear-attention==0.5.2`, and +`flashinfer-python==0.6.16.post3` are installed by the recipe. The worker +imports `causal_conv1d`, then `fla.ops.gated_delta_rule`, then +`flashinfer.gdn_prefill`, before Transformers binds Qwen3.5, and replaces +the chunk global with FlashInfer GDN prefill (`use_cp` false) before +layers copy it. Startup raises unless that wrapper and `causal_conv1d_fn` +are bound, so a ready `/health` reports +`linear_attention: flashinfer+fla+causal_conv1d`. The device gate is +compute capability 9.0. `torch_chunk_gated_delta_rule` and `F.conv1d` are +not served. No H200 p50 or p95 has been measured, and answer parity +against `torch_chunk` and against the FLA chunk is not measured. The L20X +table above is not an H200 baseline. + +| Id | Change | Expected effect | Measured result | | --- | --- | --- | --- | | H1 | One `encode_record` per request, then the pinned forward. The handler does not tokenize the schema again before that encode. | Removes a second BPE pass from the critical path. Prefill and the joint head still dominate. | Included in HEAD `9030065`. Isolated delta unknown | -| H2 | Fuse the joint head's per-question Python loop. | Fewer small launches after prefill. Needs a forked head and numerical parity against the pinned module. | Not implemented. No measured L20X delta | -| H3 | `torch.compile` or CUDA graphs on the Qwen3.5 backbone. | Uncertain on the linear-attention layers. | Not implemented. No measured L20X delta | -| H4 | Let Transformers bind `flash-linear-attention` 0.5.2 and `causal-conv1d` for the 24 linear-attention layers. | Replaces the PyTorch chunk loop. The recipe installs the packages. A process that already imported the modeling module does not rebind. | Not measured | +| H2 | Fuse the joint head's per-question Python loop. | Fewer small launches after prefill. Needs a forked head and numerical parity against the pinned module. | Not implemented. No measured H200 delta | +| H3 | `torch.compile` or CUDA graphs on the Qwen3.5 backbone. | Uncertain on the linear-attention layers. | Not implemented. No measured H200 delta | +| H4 | Bind FlashInfer GDN prefill 0.6.16.post3, `flash-linear-attention` 0.5.2, and `causal-conv1d` >= 1.5 before the Qwen3.5 layers copy the chunk global. Startup is not ready on `torch_chunk`, `F.conv1d`, or FLA-only chunk. Prefill uses FlashInfer with `use_cp` false on compute capability 9.0. vLLM CuteDSL, CUDA graphs, and `torch.compile` are not used. | Replaces the PyTorch chunk loop on H200. | Not measured. No H200 C1 | ## Status | Path | Clef-flash | Clef | | --- | --- | --- | -| Text `choice` / `noul` / `score`, non-thinking prompt | Contract tested. Operator-reported Public231 C1 latency on L20X sm_89 at HEAD `9030065`; answer parity not checked | Same loader pin; that latency campaign was flash only | +| Text `choice` / `noul` / `score`, non-thinking prompt | Contract tested. Operator-reported Public231 C1 latency on L20X sm_89 at HEAD `9030065` (not an H200 figure); H200 FlashInfer path has no C1 yet; answer parity not checked | Same loader pin; that latency campaign was flash only | | Image and video | Rejected | Rejected | | Native CUDA | Not implemented | Not implemented | | Workers AI proxy | Out of scope | Out of scope | diff --git a/src/models/clef/engine.py b/src/models/clef/engine.py index adb28c41..d4c4bb11 100644 --- a/src/models/clef/engine.py +++ b/src/models/clef/engine.py @@ -15,9 +15,9 @@ import hashlib import importlib +import importlib.metadata import importlib.util import json -import logging import sys from dataclasses import dataclass from pathlib import Path @@ -39,7 +39,18 @@ } EXECUTABLE = "joint_schema_model.py" MODELING = "transformers.models.qwen3_5.modeling_qwen3_5" -LOG = logging.getLogger(__name__) +FLA_DISTRIBUTION = "flash-linear-attention" +FLA_VERSION = "0.5.2" +FLA_MODULE = "fla.ops.gated_delta_rule" +CONV_DISTRIBUTION = "causal-conv1d" +CONV_MODULE = "causal_conv1d" +CONV_MINIMUM = (1, 5) +FI_DISTRIBUTION = "flashinfer-python" +FI_VERSION = "0.6.16.post3" +FI_MODULE = "flashinfer.gdn_prefill" +GDN_PREFILL = "flashinfer" +HOPPER = (9, 0) +REQUIRED_LINEAR_ATTENTION = "flashinfer+fla+causal_conv1d" @dataclass(frozen=True) @@ -173,43 +184,247 @@ def require_token_agreement(admitted: int, response: dict[str, Any]) -> dict[str def bound_linear_attention(modeling: Any) -> str: - """Name the linear-attention implementation Transformers actually bound. + """Name the linear-attention implementation the modeling module exposes. - ``modeling_qwen3_5`` keeps the fast callables as module globals when the - optional packages imported, and ``None`` when it kept the PyTorch - fallback. The report is that binding, not an install attempt. + Transformers copies ``chunk_gated_delta_rule`` and ``causal_conv1d_fn`` + onto each ``Qwen3_5GatedDeltaNet`` at ``__init__``. Ready means this + module global is the FlashInfer wrapper, ``causal_conv1d_fn`` is bound, + and ``clef_gdn_prefill`` is ``flashinfer``. An FLA chunk global without + that wrapper is not the H200 path. """ if modeling is None: return "unloaded" - fla = getattr(modeling, "chunk_gated_delta_rule", None) is not None + chunk = getattr(modeling, "chunk_gated_delta_rule", None) conv = getattr(modeling, "causal_conv1d_fn", None) is not None - if fla and conv: + prefill = getattr(modeling, "clef_gdn_prefill", None) + if prefill == GDN_PREFILL and chunk is flashinfer_chunk_gated_delta_rule and conv: + return REQUIRED_LINEAR_ATTENTION + if chunk is not None and conv: return "fla+causal_conv1d" - if fla: + if chunk is not None: return "fla" if conv: return "causal_conv1d" return "torch_chunk" -def activate_linear_attention() -> None: - """Import the optional kernels before Transformers binds Qwen3.5. +def require_sm90(capability: tuple[int, int], name: str) -> None: + """Accept compute capability 9.0. The device name is not the gate. - The modeling module chooses ``chunk_gated_delta_rule`` and - ``causal_conv1d_fn`` once, at import. A later install does not move a - process that already imported the module. A failed import leaves the - PyTorch fallback in place. + A driver can label an H200 ``NVIDIA L20X``. Capability 9.0 is that + machine. Capability 8.9 is Ada and is not the FlashInfer prefill target. """ - if MODELING in sys.modules: + if capability == HOPPER: return - for name in ("causal_conv1d", "fla.ops.gated_delta_rule"): + major, minor = capability + raise RuntimeError( + "Clef FlashInfer GDN prefill requires compute capability 9.0 (H200). " + f"This device is {name} sm_{major}{minor}." + ) + + +def require_hopper_device(device: str) -> None: + """Refuse startup unless ``device`` is a CUDA device at capability 9.0.""" + + import torch + + requested = torch.device(device) + if requested.type != "cuda" or not torch.cuda.is_available(): + raise RuntimeError( + "Clef FlashInfer GDN prefill requires --device cuda on compute " + "capability 9.0 (H200)." + ) + index = requested.index if requested.index is not None else torch.cuda.current_device() + require_sm90(torch.cuda.get_device_capability(index), torch.cuda.get_device_name(index)) + + +def flashinfer_chunk_gated_delta_rule( + q: Any, + k: Any, + v: Any, + g: Any = None, + beta: Any = None, + scale: float | None = None, + initial_state: Any = None, + output_final_state: bool = False, + use_qk_l2norm_in_kernel: bool = False, + cu_seqlens: Any = None, + **kwargs: Any, +) -> tuple[Any, None]: + """Run one-sequence GDN prefill through FlashInfer's fused SM90 kernel. + + Transformers calls this with FLA's ``[B, T, H, D]`` layout and a + log-space decay ``g``. FlashInfer 0.6.16 wants ``[T, H, D]``, a float32 + multiplicative gate, and a non-null ``cu_seqlens``. ``use_cp`` is false + so SM90 does not take the context-parallel path. Q/K L2 norm is FLA's + ``l2norm_fwd``; FlashInfer's own flag stays false so the vectors are not + normalized twice. vLLM's SM90 adapter is the source of that split. + FLA state is ``[N, H, K, V]`` and FlashInfer state is ``[N, H, V, K]``, + so a recurrent state is refused. Clef prefill passes neither. + """ + + del kwargs + if initial_state is not None or output_final_state: + raise RuntimeError( + "Clef FlashInfer GDN prefill does not continue or return a recurrent " + "state. This worker serves one full-prompt prefill." + ) + if getattr(q, "shape", (0,))[0] != 1: + raise RuntimeError( + "Clef FlashInfer GDN prefill serves one sequence, " + f"got batch {getattr(q, 'shape', None)}" + ) + + import torch + from flashinfer.gdn_prefill import chunk_gated_delta_rule as chunk_gated_delta_rule_fi + from fla.modules.l2norm import l2norm_fwd + + if use_qk_l2norm_in_kernel: + q, _ = l2norm_fwd(q) + k, _ = l2norm_fwd(k) + q = q.squeeze(0).contiguous() + k = k.squeeze(0).contiguous() + v = v.squeeze(0).contiguous() + fi_g = None if g is None else torch.exp(g.squeeze(0).contiguous().to(torch.float32)) + fi_beta = None if beta is None else beta.squeeze(0).contiguous().to(torch.float32) + if cu_seqlens is None: + cu_seqlens = torch.tensor([0, q.shape[0]], dtype=torch.int64, device=q.device) + else: + cu_seqlens = cu_seqlens.to(device=q.device, dtype=torch.int64) + result = chunk_gated_delta_rule_fi( + q=q, + k=k, + v=v, + g=fi_g, + beta=fi_beta, + scale=scale, + initial_state=None, + output_final_state=False, + cu_seqlens=cu_seqlens, + use_qk_l2norm_in_kernel=False, + use_cp=False, + ) + return result.unsqueeze(0), None + + +def install_flashinfer_gdn_prefill(modeling: Any) -> None: + """Replace the modeling chunk global before layers copy it. + + ``Qwen3_5GatedDeltaNet.__init__`` binds ``self.chunk_gated_delta_rule`` + from this module global. Patching after ``load_release_model`` does not + change layers that already copied the FLA function. + """ + + if getattr(modeling, "causal_conv1d_fn", None) is None: + raise RuntimeError( + "causal_conv1d_fn is not bound. Import causal_conv1d before " + f"{MODELING}. F.conv1d is not served." + ) + if getattr(modeling, "chunk_gated_delta_rule", None) is None: + raise RuntimeError( + "chunk_gated_delta_rule is not bound. Import " + f"{FLA_DISTRIBUTION}=={FLA_VERSION} before {MODELING}. " + "torch_chunk_gated_delta_rule is not served." + ) + modeling.chunk_gated_delta_rule = flashinfer_chunk_gated_delta_rule + modeling.clef_gdn_prefill = GDN_PREFILL + + +def version_tuple(text: str) -> tuple[int, ...]: + """Return the leading numeric components of a distribution version.""" + + parts: list[int] = [] + for piece in text.split("."): + digits = "" + for char in piece: + if not char.isdigit(): + break + digits += char + if not digits: + break + parts.append(int(digits)) + return tuple(parts) + + +def check_distribution_version( + installed: str, *, exact: str | None = None, minimum: tuple[int, ...] | None = None +) -> None: + """Reject a pin outside the recipe's exact version or minimum range.""" + + if exact is not None and installed != exact: + raise RuntimeError(f"required {exact}, found {installed}") + if minimum is not None and version_tuple(installed) < minimum: + joined = ".".join(str(part) for part in minimum) + raise RuntimeError(f"required >={joined}, found {installed}") + + +def require_fast_linear_attention() -> None: + """Import conv, FLA, and FlashInfer before Transformers binds Qwen3.5. + + The modeling module chooses ``FusedRMSNormGated``, ``causal_conv1d_fn``, + and the chunk global once, at import. Packages are imported first, the + chunk global is then replaced with FlashInfer, and only after that may + ``load_release_model`` construct the layers. A missing package, a wrong + pin, or an import that fails leaves ``torch_chunk_gated_delta_rule`` or + ``F.conv1d`` in place, so startup raises instead of becoming ready. + """ + + if MODELING in sys.modules: + raise RuntimeError( + f"{MODELING} is already imported. Start a new process so " + f"{CONV_DISTRIBUTION}>={'.'.join(str(part) for part in CONV_MINIMUM)}, " + f"{FLA_DISTRIBUTION}=={FLA_VERSION}, and " + f"{FI_DISTRIBUTION}=={FI_VERSION} import first. " + "torch_chunk_gated_delta_rule and F.conv1d are not served." + ) + for distribution, exact, minimum in ( + (CONV_DISTRIBUTION, None, CONV_MINIMUM), + (FLA_DISTRIBUTION, FLA_VERSION, None), + (FI_DISTRIBUTION, FI_VERSION, None), + ): try: - importlib.import_module(name) + installed = importlib.metadata.version(distribution) + except importlib.metadata.PackageNotFoundError as exc: + raise RuntimeError( + f"{distribution} is not installed. The PyTorch linear-attention " + "fallback is not served." + ) from exc + try: + check_distribution_version(installed, exact=exact, minimum=minimum) + except RuntimeError as exc: + raise RuntimeError(f"{distribution} {exc}") from exc + for distribution, module in ( + (CONV_DISTRIBUTION, CONV_MODULE), + (FLA_DISTRIBUTION, FLA_MODULE), + (FI_DISTRIBUTION, FI_MODULE), + ): + try: + importlib.import_module(module) except (ImportError, OSError) as exc: - LOG.info("clef linear-attention package %s is not active: %s", name, type(exc).__name__) + raise RuntimeError( + f"{distribution} did not import {module} ({type(exc).__name__}). " + "torch_chunk_gated_delta_rule and F.conv1d are not served." + ) from exc + install_flashinfer_gdn_prefill(importlib.import_module(MODELING)) + + +def require_bound_linear_attention(modeling: Any) -> str: + """Refuse ready unless FlashInfer prefill and causal_conv1d are bound.""" + + bound = bound_linear_attention(modeling) + if bound == REQUIRED_LINEAR_ATTENTION: + return bound + raise RuntimeError( + f"Clef linear attention is {bound}, so the worker is not ready. " + f"Ready requires {REQUIRED_LINEAR_ATTENTION} " + f"({FI_DISTRIBUTION}=={FI_VERSION} GDN prefill, " + f"{FLA_DISTRIBUTION}=={FLA_VERSION}, and " + f"{CONV_DISTRIBUTION}>={'.'.join(str(part) for part in CONV_MINIMUM)} " + "causal_conv1d_fn). torch_chunk_gated_delta_rule and F.conv1d are not served." + ) def load_joint_schema_module(path: Path) -> Any: @@ -235,10 +450,11 @@ def __init__(self, snapshot: str | Path, variant: str = DEFAULT_VARIANT, device: self.device = device self.max_length = MAX_INPUT_TOKENS verify_snapshot(self.root, self.variant) - activate_linear_attention() + require_fast_linear_attention() + require_hopper_device(device) self.module = load_joint_schema_module(self.root / EXECUTABLE) self.model, self.processor = self.module.load_release_model(self.root, device=device) - self.linear_attention = bound_linear_attention(sys.modules.get(MODELING)) + self.linear_attention = require_bound_linear_attention(sys.modules.get(MODELING)) if self.processor.tokenizer.pad_token_id is None: raise ValueError("tokenizer pad_token_id is required") diff --git a/tests/clef/test_engine.py b/tests/clef/test_engine.py index 8c1a4590..652ecb71 100644 --- a/tests/clef/test_engine.py +++ b/tests/clef/test_engine.py @@ -92,22 +92,71 @@ def test_response_token_count_must_match_admission(): def test_linear_attention_name_follows_the_bound_callables(): - from models.clef.engine import activate_linear_attention, bound_linear_attention + from models.clef.engine import ( + flashinfer_chunk_gated_delta_rule, + install_flashinfer_gdn_prefill, + bound_linear_attention, + require_bound_linear_attention, + require_sm90, + ) assert bound_linear_attention(None) == "unloaded" assert bound_linear_attention(object()) == "torch_chunk" - assert ( - bound_linear_attention(type("M", (), {"chunk_gated_delta_rule": object()})()) - == "fla" - ) - assert ( - bound_linear_attention(type("M", (), {"causal_conv1d_fn": object()})()) - == "causal_conv1d" - ) + fla_only = type("M", (), {"chunk_gated_delta_rule": object()})() + conv_only = type("M", (), {"causal_conv1d_fn": object()})() + assert bound_linear_attention(fla_only) == "fla" + assert bound_linear_attention(conv_only) == "causal_conv1d" both = type( "M", (), {"chunk_gated_delta_rule": object(), "causal_conv1d_fn": object()}, )() assert bound_linear_attention(both) == "fla+causal_conv1d" - activate_linear_attention() + with pytest.raises(RuntimeError, match="not ready"): + require_bound_linear_attention(both) + install_flashinfer_gdn_prefill(both) + assert both.chunk_gated_delta_rule is flashinfer_chunk_gated_delta_rule + assert both.clef_gdn_prefill == "flashinfer" + assert bound_linear_attention(both) == "flashinfer+fla+causal_conv1d" + assert require_bound_linear_attention(both) == "flashinfer+fla+causal_conv1d" + for modeling in (None, object(), fla_only, conv_only): + with pytest.raises(RuntimeError, match="not ready"): + require_bound_linear_attention(modeling) + with pytest.raises(RuntimeError, match="causal_conv1d_fn"): + install_flashinfer_gdn_prefill(fla_only) + with pytest.raises(RuntimeError, match="chunk_gated_delta_rule"): + install_flashinfer_gdn_prefill(conv_only) + require_sm90((9, 0), "NVIDIA L20X") + require_sm90((9, 0), "NVIDIA H200") + with pytest.raises(RuntimeError, match="sm_89"): + require_sm90((8, 9), "NVIDIA L20X") + with pytest.raises(RuntimeError, match="one sequence"): + flashinfer_chunk_gated_delta_rule(object(), object(), object()) + with pytest.raises(RuntimeError, match="recurrent state"): + flashinfer_chunk_gated_delta_rule(object(), object(), object(), output_final_state=True) + + +def test_linear_attention_pins_and_startup_fail_closed(): + from models.clef.engine import ( + FI_VERSION, + FLA_VERSION, + check_distribution_version, + require_fast_linear_attention, + version_tuple, + ) + + assert version_tuple("1.5.0.post8") == (1, 5, 0) + assert version_tuple("0.5.2") == (0, 5, 2) + assert version_tuple("0.6.16.post3") == (0, 6, 16) + check_distribution_version("0.5.2", exact=FLA_VERSION) + check_distribution_version(FI_VERSION, exact=FI_VERSION) + check_distribution_version("1.5.0.post8", minimum=(1, 5)) + check_distribution_version("1.6", minimum=(1, 5)) + with pytest.raises(RuntimeError, match="required 0.5.2"): + check_distribution_version("0.5.1", exact=FLA_VERSION) + with pytest.raises(RuntimeError, match="required 0.6.16.post3"): + check_distribution_version("0.6.16", exact=FI_VERSION) + with pytest.raises(RuntimeError, match="required >=1.5"): + check_distribution_version("1.4.9", minimum=(1, 5)) + with pytest.raises(RuntimeError, match="not installed"): + require_fast_linear_attention() From 594528a9f9c1ca38ce0dbb58a549191dfbde5bf3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 25 Oct 2026 01:39:56 +0000 Subject: [PATCH 5/9] clef: start on the L20X-named lab host at capability 8.9 Ready no longer depends on compute capability 9.0 or on the name H200. FlashInfer GDN prefill is used when the reported major is 9, 10, or 12 and the pinned package loads. A device that reports 8.9, including nvidia-smi name NVIDIA L20X, stays on the FLA chunk plus causal_conv1d. /health reports the bind that actually started. torch_chunk and F.conv1d still refuse ready. No new C1 number is claimed. Co-authored-by: Hongsheng Liu --- README.md | 2 +- docs/supported-models.md | 12 +-- recipe/README.md | 6 +- recipe/clef/reference.md | 117 +++++++++++++++-------------- recipe/clef/requirements.txt | 10 ++- src/models/clef/README.md | 23 +++--- src/models/clef/engine.py | 140 +++++++++++++++++++++++------------ tests/clef/test_engine.py | 54 +++++++++----- 8 files changed, 219 insertions(+), 145 deletions(-) diff --git a/README.md b/README.md index d850ddd6..51a2400b 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ pinned runtime for text, image and video state: | LFM2.5-350M | [Transformers worker](recipe/lfm2/README.md); text state and independent choice questions | | OmniJev-4B v1.1 | [Checkpoint export and CPU contract](recipe/omnijev/README.md) only; the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | | D3 (Decision 3.0) | [Python reference worker](recipe/d3/reference.md) on the checkpoint's pinned runtime; `choice`/`noul`/`score` over text, image and video state; author-run on A100 for text and media serving (see the integration and media PRs) | -| Clef-flash (9B) and Clef (27B) | [Python reference worker](recipe/clef/reference.md) for text `choice`/`noul`/`score`; flash is the bring-up variant. Image, video and native CUDA are not served. Operator-reported flash Public231 C1 on L20X sm_89: run1 p50 247.518 ms, p95 413.025 ms. That figure is not an H200 result. The forward prefill target is H200 (sm_90) and has no recorded C1 | +| Clef-flash (9B) and Clef (27B) | [Python reference worker](recipe/clef/reference.md) for text `choice`/`noul`/`score`; flash is the bring-up variant. Image, video and native CUDA are not served. Operator-reported flash Public231 C1 on the lab host named NVIDIA L20X, at HEAD `9030065`: run1 p50 247.518 ms, p95 413.025 ms. The current tip has no recorded C1. That host stays eligible when `nvidia-smi` reports capability 8.9 | [Supported models and hardware](docs/supported-models.md) lists the devices and where each worker has been run. diff --git a/docs/supported-models.md b/docs/supported-models.md index 4feef834..db0a2bbf 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -45,11 +45,13 @@ refuses image and video inputs. The flash CUDA cell is an operator-reported Public231 concurrency-1 result on a lab NVIDIA L20X at compute capability 8.9 (primary run1 p50 247.518 ms, p95 413.025 ms, metric `successful_latency`, clock `post_to_body_json_and_schema_validation`). -That cell stays an L20X figure. It is not an H200 or H800 result, and it -was not re-run for this document. The forward path wires FlashInfer GDN -prefill with FLA and causal-conv1d for compute capability 9.0. No H200 C1 -latency or answer-parity delta is recorded. The 27B variant has no -recorded timing. +That cell is the earlier L20X report at that HEAD, and it was not re-run +for this document. The worker starts on the lab host `nvidia-smi` names +`NVIDIA L20X`, including when the capability prints 8.9. FlashInfer GDN +prefill is used when the reported major is 9, 10, or 12 and the package +loads; otherwise the ready bind is the FLA chunk plus causal-conv1d. +`/health` reports which one started. No C1 of the current tip is recorded. +The 27B variant has no recorded timing. CLM has a row per path: the merged `clm-serve` recipe exercises those answer shapes with stub embeddings and says nothing about decision quality, while the native engine's CUDA row is the path through a real Qwen3-8B encoder. Its engine-only figure comes from the same comparison run diff --git a/recipe/README.md b/recipe/README.md index f9556cd5..e17e4e25 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -41,9 +41,9 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting - [Clef reference worker](clef/reference.md): load pinned Clef-flash (9B) or Clef (27B) and serve text `choice`/`noul`/`score`. Image, video and native CUDA are not served. Operator-reported flash Public231 concurrency-1 - timing on L20X sm_89 is in the recipe (run1 p50 247.518 ms, p95 413.025 ms). - That figure is not an H200 result. The forward prefill target is H200 - (sm_90); no H200 C1 is recorded. + timing on the lab host named NVIDIA L20X is in the recipe (HEAD `9030065`, + run1 p50 247.518 ms, p95 413.025 ms). The current tip has no recorded C1. + That host stays eligible when `nvidia-smi` reports capability 8.9. - [LFM2.5-350M choice worker](lfm2/README.md): score text decisions with independent questions, hybrid cache branching and configurable candidate batches. diff --git a/recipe/clef/reference.md b/recipe/clef/reference.md index 58c42ae2..7cbf41b5 100644 --- a/recipe/clef/reference.md +++ b/recipe/clef/reference.md @@ -33,33 +33,36 @@ python -m pip install -r recipe/clef/requirements.txt The requirements pin `causal-conv1d>=1.5`, `flash-linear-attention==0.5.2`, and `flashinfer-python==0.6.16.post3`. The worker imports `causal_conv1d`, -then `fla.ops.gated_delta_rule`, then `flashinfer.gdn_prefill`, and only -then `transformers.models.qwen3_5.modeling_qwen3_5`. That order binds -`FusedRMSNormGated` and `causal_conv1d_fn`. Startup then replaces the -modeling module's `chunk_gated_delta_rule` with the FlashInfer GDN prefill -wrapper before `load_release_model` builds each `Qwen3_5GatedDeltaNet`, -because the layer copies that global at init. - -Startup raises when any of the three distributions is missing, when -flash-linear-attention is not exactly `0.5.2`, when flashinfer-python is -not exactly `0.6.16.post3`, when causal-conv1d is older than 1.5, when an -import fails, when `causal_conv1d_fn` did not bind, or when the chunk -global is not the FlashInfer wrapper. A ready process has `/health` -`linear_attention` equal to `flashinfer+fla+causal_conv1d`. The PyTorch -`torch_chunk_gated_delta_rule` and `F.conv1d` paths are not served. An FLA -chunk binding without the FlashInfer wrapper is not ready. - -Prefill calls `flashinfer.gdn_prefill.chunk_gated_delta_rule` with -`use_cp=False`, so Hopper uses the fused SM90 kernel. Q and K L2 -normalization uses FLA `l2norm_fwd`, and FlashInfer's -`use_qk_l2norm_in_kernel` flag stays false. Log-space `g` is +then `fla.ops.gated_delta_rule`, then tries `flashinfer.gdn_prefill`, and +only then imports `transformers.models.qwen3_5.modeling_qwen3_5`. That +order binds `FusedRMSNormGated` and `causal_conv1d_fn`. When the reported +compute capability major is 9, 10, or 12 and FlashInfer loaded, startup +replaces the modeling module's `chunk_gated_delta_rule` with the FlashInfer +GDN prefill wrapper before `load_release_model` builds each +`Qwen3_5GatedDeltaNet`, because the layer copies that global at init. + +`causal-conv1d` and `flash-linear-attention` are required. Startup raises +when either is missing, when flash-linear-attention is not exactly `0.5.2`, +when causal-conv1d is older than 1.5, or when `causal_conv1d_fn` and a +chunk kernel did not bind. A missing or different FlashInfer install does +not, by itself, refuse ready. `/health` `linear_attention` is the bind that +started: `flashinfer+fla+causal_conv1d` or `fla+causal_conv1d`. The PyTorch +`torch_chunk_gated_delta_rule` and `F.conv1d` paths are not served. + +FlashInfer prefill calls `flashinfer.gdn_prefill.chunk_gated_delta_rule` +with `use_cp=False`. Q and K L2 normalization uses FLA `l2norm_fwd`, and +FlashInfer's `use_qk_l2norm_in_kernel` flag stays false. Log-space `g` is exponentiated. The call is one sequence and does not continue a recurrent -state. vLLM's separate CuteDSL prefill backend, CUDA graphs, and -`torch.compile` are not used. The device must be CUDA compute capability -9.0. A device named `NVIDIA L20X` at capability 8.9 is the wrong machine. -The same name at capability 9.0 is the known driver mislabel of an H200 -and passes the capability check. No H200 Public231 C1 latency, and no -answer parity against `torch_chunk` or the FLA chunk, has been measured. +state. FlashInfer 0.6.16 does not dispatch that kernel when the device +reports capability 8.9, so those processes keep the FLA chunk. vLLM's +separate CuteDSL prefill backend, CUDA graphs, and `torch.compile` are +not used. + +The lab host is the machine `nvidia-smi` names `NVIDIA L20X`, including +when it reports compute capability 8.9. Startup does not require the name +`NVIDIA H200` and does not refuse ready because the capability is not 9.0. +No Public231 C1 of this tip has been measured. Answer parity against +`torch_chunk` is not measured. ```sh python - <<'PY' @@ -166,16 +169,19 @@ The operator named the metric `successful_latency` and the clock `post_to_body_json_and_schema_validation`. These three rounds are that measurement. They are not a new campaign. -Those three rounds are the L20X report at HEAD `9030065`. They are not an -H200 baseline. Do not relabel them as H200, and do not treat a later H200 -p50 or p95 as a same-device speedup against 247.518 / 413.025. +Those three rounds are the earlier report at HEAD `9030065` on the lab +host `nvidia-smi` names `NVIDIA L20X`. This tip has no new p50 or p95. +Do not publish a delta until a remeasure of this tip is pasted. -## Next measurement: H200 (sm_90) +## Next measurement: lab host named NVIDIA L20X -The forward operator path is FlashInfer GDN prefill plus `causal_conv1d` -and FLA, on compute capability 9.0. No H200 Public231 C1 has been recorded. -Answer parity of that prefill against `torch_chunk` and against the FLA -chunk is not measured. The standalone GDN timings in +That host is the measurement machine, including when `nvidia-smi` reports +compute capability 8.9. Do not require a card whose name is `NVIDIA H200`. +On a reported 8.9 the ready bind is `fla+causal_conv1d`. When the same +process sees reported major 9, 10, or 12 and FlashInfer loaded, the ready +bind is `flashinfer+fla+causal_conv1d`. Record the `/health` string the +process returns. No Public231 C1 of this tip has been recorded. Answer +parity against `torch_chunk` is not measured. The standalone GDN timings in [benchmarks/gdn/README.md](../../benchmarks/gdn/README.md) are a different experiment and are not a Clef HTTP result. @@ -189,19 +195,18 @@ Record results outside the repository. Do not put hostnames, account names, home directories, or the name of a reservation system in the metadata or in a commit. -1. Confirm the GPU the process will use. `nvidia-smi` must show compute - capability 9.0. Record the device name it prints. Capability 9.0 is the - H200 target, including when the name is the known `NVIDIA L20X` - mislabel. Capability 8.9 is Ada; stop, and do not report that machine - as this H200 run. Do not report the run as H800 or as the Public231 - leaderboard. +1. Confirm the GPU the process will use. The lab host is the one + `nvidia-smi` names `NVIDIA L20X`. Record that name and the compute + capability it prints, including 8.9. Do not stop because the name is + not `NVIDIA H200`, and do not stop because the capability is not 9.0. + Do not report the run as H800 or as the Public231 leaderboard. 2. Start `frontend.clef` as above, on loopback port 8000, variant `clef-flash`, device `cuda`. Leave that process up for every traversal. - `GET /health` must show `status` `ready` and `linear_attention` - `flashinfer+fla+causal_conv1d`. A process that exits at startup did not - bind FlashInfer GDN prefill and `causal_conv1d_fn`. Do not measure a - process that is still on the PyTorch chunk, `F.conv1d`, or an FLA-only - chunk path. + `GET /health` must show `status` `ready`. `linear_attention` is + `fla+causal_conv1d` or `flashinfer+fla+causal_conv1d`, whichever the + process bound. Record that string. A process that exits at startup did + not bind `causal_conv1d_fn` and a chunk kernel. Do not measure a + process that is still on the PyTorch chunk or `F.conv1d` path. 3. Obtain the frozen Public231 request manifest whose file SHA-256 is `00d1f10ca1142499a598b5f1c6602d329cce1cef04c961c46fc0318daa2c67d3`. That is the JSONL byte hash cited for the JevBench `7ce310c7` export in @@ -223,7 +228,7 @@ in a commit. ```json { - "gpu": "NVIDIA H200", + "gpu": "NVIDIA L20X", "gpu_ids": ["0"], "driver": "", "cuda": "", @@ -231,7 +236,7 @@ in a commit. "model_revision": "Cloudflare/clef-flash@f0b7d3cd65296abfe459e6c228b618f5d8b26d2f", "runtime_revision": "", "cache_policy": "one request at a time; no prefix cache; no CUDA graph; transformers cache left at its default", - "cuda_evidence": "nvidia-smi showed compute capability 9.0 while the worker was running with --device cuda. Record the printed device name. If that name is NVIDIA L20X and the capability is 9.0, say so; that pair is the known H200 mislabel.", + "cuda_evidence": "nvidia-smi name NVIDIA L20X, compute capability as printed (8.9 on the lab host), while the worker was running with --device cuda. Copy /health linear_attention.", "reservation": "" } ``` @@ -267,18 +272,20 @@ in a commit. per-round and pooled counts. Keep failure latencies separate. 9. The clock is the runner's `post_to_body_json_and_schema_validation` clock. It is not the H800 - leaderboard clock. Report this campaign as the H200 C1. Do not relabel - the L20X table above as this result, and do not compute a same-device - speedup against 247.518 / 413.025. + leaderboard clock. Record the `nvidia-smi` name, the printed capability, + and `/health` `linear_attention` with the p50 and p95. The table above + is the earlier `9030065` report. This tip has no measured p50 or p95 + until that paste exists. ## What remains unverified -Image and video inputs, native CUDA, answer parity, and any H200 latency. -The operator-reported L20X numbers above stay on the clock +Image and video inputs, native CUDA, answer parity, and any Public231 C1 +of the current tip. The operator-reported numbers above stay on the clock `post_to_body_json_and_schema_validation`, metric `successful_latency`, at HEAD `903006530fb7772bb668bbf192ee4278e273e583`. H1, one encode per request, is already how that HEAD calls the pinned model, so run1 includes it and -its isolated effect is unknown. The H200 path wires FlashInfer GDN prefill, -FLA, and causal-conv1d. That path has no measured p50, p95, or -answer-parity delta. This environment has no GPU and did not repeat the +its isolated effect is unknown. This tip selects FlashInfer when that +kernel can dispatch and otherwise serves the FLA chunk with +`causal_conv1d`. Neither bind has a measured p50, p95, or answer-parity +result on this tip. This environment has no GPU and did not repeat the campaign. diff --git a/recipe/clef/requirements.txt b/recipe/clef/requirements.txt index 3f3986b6..45757c40 100644 --- a/recipe/clef/requirements.txt +++ b/recipe/clef/requirements.txt @@ -15,10 +15,12 @@ pillow>=10 # PyTorch norm. The worker then replaces the chunk global with FlashInfer # GDN prefill (use_cp false, fused SM90) before layers copy it. # causal_conv1d_fn stays the convolution. Startup refuses torch_chunk and -# F.conv1d, and refuses an FLA chunk binding that was not replaced. -# The target device is H200, compute capability 9.0. vLLM's separate CuteDSL -# prefill backend, CUDA graphs, and torch.compile are not selected. -# No H200 Public231 latency or answer-parity delta has been measured. +# F.conv1d. FlashInfer replaces the chunk only when the reported capability +# major is 9, 10, or 12 and this package imports. A device that reports 8.9, +# including nvidia-smi name NVIDIA L20X, stays on the FLA chunk and still +# becomes ready. The device name is not required to be NVIDIA H200. +# vLLM's separate CuteDSL prefill backend, CUDA graphs, and torch.compile +# are not selected. No Public231 C1 of this tip has been measured. causal-conv1d>=1.5 flash-linear-attention==0.5.2 flashinfer-python==0.6.16.post3 diff --git a/src/models/clef/README.md b/src/models/clef/README.md index b4c10050..c951dff7 100644 --- a/src/models/clef/README.md +++ b/src/models/clef/README.md @@ -159,29 +159,28 @@ implementation was not part of this report. `causal-conv1d>=1.5`, `flash-linear-attention==0.5.2`, and `flashinfer-python==0.6.16.post3` are installed by the recipe. The worker -imports `causal_conv1d`, then `fla.ops.gated_delta_rule`, then -`flashinfer.gdn_prefill`, before Transformers binds Qwen3.5, and replaces -the chunk global with FlashInfer GDN prefill (`use_cp` false) before -layers copy it. Startup raises unless that wrapper and `causal_conv1d_fn` -are bound, so a ready `/health` reports -`linear_attention: flashinfer+fla+causal_conv1d`. The device gate is -compute capability 9.0. `torch_chunk_gated_delta_rule` and `F.conv1d` are -not served. No H200 p50 or p95 has been measured, and answer parity -against `torch_chunk` and against the FLA chunk is not measured. The L20X -table above is not an H200 baseline. +imports `causal_conv1d` and `fla.ops.gated_delta_rule` before Transformers +binds Qwen3.5. FlashInfer replaces the chunk global, with `use_cp` false, +only when the reported capability major is 9, 10, or 12 and that package +loads. A reported 8.9, including the lab name `NVIDIA L20X`, keeps the FLA +chunk and still becomes ready. Startup does not require the name +`NVIDIA H200`. `/health` `linear_attention` is +`flashinfer+fla+causal_conv1d` or `fla+causal_conv1d`, matching the bind. +`torch_chunk_gated_delta_rule` and `F.conv1d` are not served. This tip has +no measured p50 or p95. Answer parity against `torch_chunk` is not measured. | Id | Change | Expected effect | Measured result | | --- | --- | --- | --- | | H1 | One `encode_record` per request, then the pinned forward. The handler does not tokenize the schema again before that encode. | Removes a second BPE pass from the critical path. Prefill and the joint head still dominate. | Included in HEAD `9030065`. Isolated delta unknown | | H2 | Fuse the joint head's per-question Python loop. | Fewer small launches after prefill. Needs a forked head and numerical parity against the pinned module. | Not implemented. No measured H200 delta | | H3 | `torch.compile` or CUDA graphs on the Qwen3.5 backbone. | Uncertain on the linear-attention layers. | Not implemented. No measured H200 delta | -| H4 | Bind FlashInfer GDN prefill 0.6.16.post3, `flash-linear-attention` 0.5.2, and `causal-conv1d` >= 1.5 before the Qwen3.5 layers copy the chunk global. Startup is not ready on `torch_chunk`, `F.conv1d`, or FLA-only chunk. Prefill uses FlashInfer with `use_cp` false on compute capability 9.0. vLLM CuteDSL, CUDA graphs, and `torch.compile` are not used. | Replaces the PyTorch chunk loop on H200. | Not measured. No H200 C1 | +| H4 | Bind `flash-linear-attention` 0.5.2 and `causal-conv1d` >= 1.5 before the Qwen3.5 layers copy the chunk global. Use FlashInfer GDN prefill 0.6.16.post3 with `use_cp` false when the reported capability major is 9, 10, or 12 and the package loads. Reported 8.9, including nvidia-smi name `NVIDIA L20X`, stays on the FLA chunk and still starts. Startup is not ready on `torch_chunk` or `F.conv1d`. vLLM CuteDSL, CUDA graphs, and `torch.compile` are not used. | Replaces the PyTorch chunk loop. `/health` names the bind that started. | Not measured on this tip | ## Status | Path | Clef-flash | Clef | | --- | --- | --- | -| Text `choice` / `noul` / `score`, non-thinking prompt | Contract tested. Operator-reported Public231 C1 latency on L20X sm_89 at HEAD `9030065` (not an H200 figure); H200 FlashInfer path has no C1 yet; answer parity not checked | Same loader pin; that latency campaign was flash only | +| Text `choice` / `noul` / `score`, non-thinking prompt | Contract tested. Operator-reported Public231 C1 latency on L20X at HEAD `9030065`. This tip has no C1 yet. A reported capability of 8.9 on the `NVIDIA L20X` host is ready. Answer parity not checked | Same loader pin; that latency campaign was flash only | | Image and video | Rejected | Rejected | | Native CUDA | Not implemented | Not implemented | | Workers AI proxy | Out of scope | Out of scope | diff --git a/src/models/clef/engine.py b/src/models/clef/engine.py index d4c4bb11..6e0e6eac 100644 --- a/src/models/clef/engine.py +++ b/src/models/clef/engine.py @@ -49,8 +49,12 @@ FI_VERSION = "0.6.16.post3" FI_MODULE = "flashinfer.gdn_prefill" GDN_PREFILL = "flashinfer" -HOPPER = (9, 0) +FLA_PREFILL = "fla" +# FlashInfer 0.6.16 GDN prefill dispatches reported majors 9, 10, and 12. +FLASHINFER_ARCH_MAJORS = (9, 10, 12) REQUIRED_LINEAR_ATTENTION = "flashinfer+fla+causal_conv1d" +FLA_LINEAR_ATTENTION = "fla+causal_conv1d" +READY_LINEAR_ATTENTION = frozenset({REQUIRED_LINEAR_ATTENTION, FLA_LINEAR_ATTENTION}) @dataclass(frozen=True) @@ -187,10 +191,9 @@ def bound_linear_attention(modeling: Any) -> str: """Name the linear-attention implementation the modeling module exposes. Transformers copies ``chunk_gated_delta_rule`` and ``causal_conv1d_fn`` - onto each ``Qwen3_5GatedDeltaNet`` at ``__init__``. Ready means this - module global is the FlashInfer wrapper, ``causal_conv1d_fn`` is bound, - and ``clef_gdn_prefill`` is ``flashinfer``. An FLA chunk global without - that wrapper is not the H200 path. + onto each ``Qwen3_5GatedDeltaNet`` at ``__init__``. Ready is either the + FlashInfer wrapper or the FLA chunk, together with ``causal_conv1d_fn``. + ``/health`` reports whichever global is actually bound. """ if modeling is None: @@ -202,7 +205,7 @@ def bound_linear_attention(modeling: Any) -> str: if prefill == GDN_PREFILL and chunk is flashinfer_chunk_gated_delta_rule and conv: return REQUIRED_LINEAR_ATTENTION if chunk is not None and conv: - return "fla+causal_conv1d" + return FLA_LINEAR_ATTENTION if chunk is not None: return "fla" if conv: @@ -210,35 +213,55 @@ def bound_linear_attention(modeling: Any) -> str: return "torch_chunk" -def require_sm90(capability: tuple[int, int], name: str) -> None: - """Accept compute capability 9.0. The device name is not the gate. +def select_gdn_prefill(capability: tuple[int, int] | None) -> str: + """Choose FlashInfer only when its GDN prefill kernel can dispatch. - A driver can label an H200 ``NVIDIA L20X``. Capability 9.0 is that - machine. Capability 8.9 is Ada and is not the FlashInfer prefill target. + Reported majors 9, 10, and 12 take FlashInfer. A device that reports + 8.9 stays on the FLA chunk, including the lab cards ``nvidia-smi`` names + ``NVIDIA L20X``. The printed name is not a gate, and a capability other + than 9.0 is not a reason to refuse ready. """ - if capability == HOPPER: - return - major, minor = capability - raise RuntimeError( - "Clef FlashInfer GDN prefill requires compute capability 9.0 (H200). " - f"This device is {name} sm_{major}{minor}." - ) + if capability is not None and capability[0] in FLASHINFER_ARCH_MAJORS: + return GDN_PREFILL + return FLA_PREFILL -def require_hopper_device(device: str) -> None: - """Refuse startup unless ``device`` is a CUDA device at capability 9.0.""" +def cuda_capability(device: str) -> tuple[int, int]: + """Return the capability of a CUDA device. The name is not consulted.""" import torch requested = torch.device(device) if requested.type != "cuda" or not torch.cuda.is_available(): - raise RuntimeError( - "Clef FlashInfer GDN prefill requires --device cuda on compute " - "capability 9.0 (H200)." - ) + raise RuntimeError("Clef prefill requires --device cuda.") index = requested.index if requested.index is not None else torch.cuda.current_device() - require_sm90(torch.cuda.get_device_capability(index), torch.cuda.get_device_name(index)) + return torch.cuda.get_device_capability(index) + + +def bind_selected_prefill(modeling: Any, backend: str, flashinfer_loaded: bool) -> None: + """Install FlashInfer when it was selected and the package loaded. + + Otherwise the FLA chunk global that Transformers bound at import stays + in place. Layers copy that global, so this runs before + ``load_release_model``. + """ + + if backend == GDN_PREFILL and flashinfer_loaded: + install_flashinfer_gdn_prefill(modeling) + + +def activate_gdn_prefill(device: str, flashinfer_loaded: bool) -> None: + """Bind the prefill that this CUDA device can run.""" + + modeling = sys.modules.get(MODELING) + if modeling is None: + raise RuntimeError(f"{MODELING} is not imported.") + bind_selected_prefill( + modeling, + select_gdn_prefill(cuda_capability(device)), + flashinfer_loaded, + ) def flashinfer_chunk_gated_delta_rule( @@ -361,29 +384,50 @@ def check_distribution_version( raise RuntimeError(f"required >={joined}, found {installed}") -def require_fast_linear_attention() -> None: - """Import conv, FLA, and FlashInfer before Transformers binds Qwen3.5. +def try_import_flashinfer() -> bool: + """Return whether the pinned FlashInfer GDN prefill module imported. + + A missing package, a different version, or an import error means the + FLA chunk remains the prefill. That is not a readiness failure. + """ + + try: + installed = importlib.metadata.version(FI_DISTRIBUTION) + except importlib.metadata.PackageNotFoundError: + return False + try: + check_distribution_version(installed, exact=FI_VERSION) + except RuntimeError: + return False + try: + importlib.import_module(FI_MODULE) + except (ImportError, OSError): + return False + return True + + +def require_fast_linear_attention() -> bool: + """Import conv and FLA before Transformers binds Qwen3.5. The modeling module chooses ``FusedRMSNormGated``, ``causal_conv1d_fn``, - and the chunk global once, at import. Packages are imported first, the - chunk global is then replaced with FlashInfer, and only after that may - ``load_release_model`` construct the layers. A missing package, a wrong - pin, or an import that fails leaves ``torch_chunk_gated_delta_rule`` or - ``F.conv1d`` in place, so startup raises instead of becoming ready. + and the FLA chunk global once, at import. ``causal-conv1d`` and + ``flash-linear-attention`` are required. A missing copy leaves + ``torch_chunk_gated_delta_rule`` or ``F.conv1d``, so startup raises. + FlashInfer is attempted and the return value says whether it loaded. + The device choice patches the chunk global later, before + ``load_release_model``. """ if MODELING in sys.modules: raise RuntimeError( f"{MODELING} is already imported. Start a new process so " - f"{CONV_DISTRIBUTION}>={'.'.join(str(part) for part in CONV_MINIMUM)}, " - f"{FLA_DISTRIBUTION}=={FLA_VERSION}, and " - f"{FI_DISTRIBUTION}=={FI_VERSION} import first. " + f"{CONV_DISTRIBUTION}>={'.'.join(str(part) for part in CONV_MINIMUM)} " + f"and {FLA_DISTRIBUTION}=={FLA_VERSION} import first. " "torch_chunk_gated_delta_rule and F.conv1d are not served." ) for distribution, exact, minimum in ( (CONV_DISTRIBUTION, None, CONV_MINIMUM), (FLA_DISTRIBUTION, FLA_VERSION, None), - (FI_DISTRIBUTION, FI_VERSION, None), ): try: installed = importlib.metadata.version(distribution) @@ -399,7 +443,6 @@ def require_fast_linear_attention() -> None: for distribution, module in ( (CONV_DISTRIBUTION, CONV_MODULE), (FLA_DISTRIBUTION, FLA_MODULE), - (FI_DISTRIBUTION, FI_MODULE), ): try: importlib.import_module(module) @@ -408,22 +451,27 @@ def require_fast_linear_attention() -> None: f"{distribution} did not import {module} ({type(exc).__name__}). " "torch_chunk_gated_delta_rule and F.conv1d are not served." ) from exc - install_flashinfer_gdn_prefill(importlib.import_module(MODELING)) + flashinfer_loaded = try_import_flashinfer() + importlib.import_module(MODELING) + return flashinfer_loaded def require_bound_linear_attention(modeling: Any) -> str: - """Refuse ready unless FlashInfer prefill and causal_conv1d are bound.""" + """Refuse ready on ``torch_chunk`` or ``F.conv1d``. + + Both ``flashinfer+fla+causal_conv1d`` and ``fla+causal_conv1d`` are + ready. The string is the bind the process actually installed. + """ bound = bound_linear_attention(modeling) - if bound == REQUIRED_LINEAR_ATTENTION: + if bound in READY_LINEAR_ATTENTION: return bound raise RuntimeError( f"Clef linear attention is {bound}, so the worker is not ready. " - f"Ready requires {REQUIRED_LINEAR_ATTENTION} " - f"({FI_DISTRIBUTION}=={FI_VERSION} GDN prefill, " - f"{FLA_DISTRIBUTION}=={FLA_VERSION}, and " - f"{CONV_DISTRIBUTION}>={'.'.join(str(part) for part in CONV_MINIMUM)} " - "causal_conv1d_fn). torch_chunk_gated_delta_rule and F.conv1d are not served." + f"Ready is {FLA_LINEAR_ATTENTION} or {REQUIRED_LINEAR_ATTENTION} " + f"({FLA_DISTRIBUTION}=={FLA_VERSION} and " + f"{CONV_DISTRIBUTION}>={'.'.join(str(part) for part in CONV_MINIMUM)}). " + "torch_chunk_gated_delta_rule and F.conv1d are not served." ) @@ -450,8 +498,8 @@ def __init__(self, snapshot: str | Path, variant: str = DEFAULT_VARIANT, device: self.device = device self.max_length = MAX_INPUT_TOKENS verify_snapshot(self.root, self.variant) - require_fast_linear_attention() - require_hopper_device(device) + flashinfer_loaded = require_fast_linear_attention() + activate_gdn_prefill(device, flashinfer_loaded) self.module = load_joint_schema_module(self.root / EXECUTABLE) self.model, self.processor = self.module.load_release_model(self.root, device=device) self.linear_attention = require_bound_linear_attention(sys.modules.get(MODELING)) diff --git a/tests/clef/test_engine.py b/tests/clef/test_engine.py index 652ecb71..481a3709 100644 --- a/tests/clef/test_engine.py +++ b/tests/clef/test_engine.py @@ -91,13 +91,22 @@ def test_response_token_count_must_match_admission(): require_token_agreement(12, {"usage": {"input_tokens": 9}}) +def _bound_pair(): + return type( + "M", + (), + {"chunk_gated_delta_rule": object(), "causal_conv1d_fn": object()}, + )() + + def test_linear_attention_name_follows_the_bound_callables(): from models.clef.engine import ( + bind_selected_prefill, + bound_linear_attention, flashinfer_chunk_gated_delta_rule, install_flashinfer_gdn_prefill, - bound_linear_attention, require_bound_linear_attention, - require_sm90, + select_gdn_prefill, ) assert bound_linear_attention(None) == "unloaded" @@ -106,19 +115,14 @@ def test_linear_attention_name_follows_the_bound_callables(): conv_only = type("M", (), {"causal_conv1d_fn": object()})() assert bound_linear_attention(fla_only) == "fla" assert bound_linear_attention(conv_only) == "causal_conv1d" - both = type( - "M", - (), - {"chunk_gated_delta_rule": object(), "causal_conv1d_fn": object()}, - )() - assert bound_linear_attention(both) == "fla+causal_conv1d" - with pytest.raises(RuntimeError, match="not ready"): - require_bound_linear_attention(both) - install_flashinfer_gdn_prefill(both) - assert both.chunk_gated_delta_rule is flashinfer_chunk_gated_delta_rule - assert both.clef_gdn_prefill == "flashinfer" - assert bound_linear_attention(both) == "flashinfer+fla+causal_conv1d" - assert require_bound_linear_attention(both) == "flashinfer+fla+causal_conv1d" + fla_ready = _bound_pair() + assert bound_linear_attention(fla_ready) == "fla+causal_conv1d" + assert require_bound_linear_attention(fla_ready) == "fla+causal_conv1d" + install_flashinfer_gdn_prefill(fla_ready) + assert fla_ready.chunk_gated_delta_rule is flashinfer_chunk_gated_delta_rule + assert fla_ready.clef_gdn_prefill == "flashinfer" + assert bound_linear_attention(fla_ready) == "flashinfer+fla+causal_conv1d" + assert require_bound_linear_attention(fla_ready) == "flashinfer+fla+causal_conv1d" for modeling in (None, object(), fla_only, conv_only): with pytest.raises(RuntimeError, match="not ready"): require_bound_linear_attention(modeling) @@ -126,10 +130,22 @@ def test_linear_attention_name_follows_the_bound_callables(): install_flashinfer_gdn_prefill(fla_only) with pytest.raises(RuntimeError, match="chunk_gated_delta_rule"): install_flashinfer_gdn_prefill(conv_only) - require_sm90((9, 0), "NVIDIA L20X") - require_sm90((9, 0), "NVIDIA H200") - with pytest.raises(RuntimeError, match="sm_89"): - require_sm90((8, 9), "NVIDIA L20X") + assert select_gdn_prefill((8, 9)) == "fla" + assert select_gdn_prefill((9, 0)) == "flashinfer" + assert select_gdn_prefill((10, 0)) == "flashinfer" + assert select_gdn_prefill((12, 0)) == "flashinfer" + assert select_gdn_prefill(None) == "fla" + reported_89 = _bound_pair() + original_chunk = reported_89.chunk_gated_delta_rule + bind_selected_prefill(reported_89, select_gdn_prefill((8, 9)), True) + assert reported_89.chunk_gated_delta_rule is original_chunk + assert require_bound_linear_attention(reported_89) == "fla+causal_conv1d" + reported_90 = _bound_pair() + bind_selected_prefill(reported_90, select_gdn_prefill((9, 0)), True) + assert require_bound_linear_attention(reported_90) == "flashinfer+fla+causal_conv1d" + unloaded_fi = _bound_pair() + bind_selected_prefill(unloaded_fi, select_gdn_prefill((9, 0)), False) + assert require_bound_linear_attention(unloaded_fi) == "fla+causal_conv1d" with pytest.raises(RuntimeError, match="one sequence"): flashinfer_chunk_gated_delta_rule(object(), object(), object()) with pytest.raises(RuntimeError, match="recurrent state"): From 24941dabc9ec69a03418a24f996c00ad264c560f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 25 Oct 2026 02:19:19 +0000 Subject: [PATCH 6/9] clef: record the verified L20X C1 for 594528a9 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The L20X-labeled lab Public231 C1 of that tip is 231/231, p50 164.969 ms, p95 216.755 ms, with /health linear_attention fla+causal_conv1d. Against 9030065 run1 that is Δp50 −33.35% (−82.5 ms) and Δp95 −47.52% (−196.3 ms). The serve path is unchanged. A later code tip needs its own paste. Co-authored-by: Hongsheng Liu --- README.md | 2 +- docs/supported-models.md | 21 +++++++------ recipe/README.md | 6 ++-- recipe/clef/reference.md | 63 +++++++++++++++++++++++---------------- src/models/clef/README.md | 15 +++++++--- 5 files changed, 63 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 51a2400b..b3e75b40 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ pinned runtime for text, image and video state: | LFM2.5-350M | [Transformers worker](recipe/lfm2/README.md); text state and independent choice questions | | OmniJev-4B v1.1 | [Checkpoint export and CPU contract](recipe/omnijev/README.md) only; the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | | D3 (Decision 3.0) | [Python reference worker](recipe/d3/reference.md) on the checkpoint's pinned runtime; `choice`/`noul`/`score` over text, image and video state; author-run on A100 for text and media serving (see the integration and media PRs) | -| Clef-flash (9B) and Clef (27B) | [Python reference worker](recipe/clef/reference.md) for text `choice`/`noul`/`score`; flash is the bring-up variant. Image, video and native CUDA are not served. Operator-reported flash Public231 C1 on the lab host named NVIDIA L20X, at HEAD `9030065`: run1 p50 247.518 ms, p95 413.025 ms. The current tip has no recorded C1. That host stays eligible when `nvidia-smi` reports capability 8.9 | +| Clef-flash (9B) and Clef (27B) | [Python reference worker](recipe/clef/reference.md) for text `choice`/`noul`/`score`; flash is the bring-up variant. Image, video and native CUDA are not served. L20X-labeled lab Public231 C1: `9030065` run1 p50 247.518 ms, p95 413.025 ms; tip `594528a9` p50 164.969 ms, p95 216.755 ms, `/health` `fla+causal_conv1d` | [Supported models and hardware](docs/supported-models.md) lists the devices and where each worker has been run. diff --git a/docs/supported-models.md b/docs/supported-models.md index db0a2bbf..01189585 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -27,7 +27,7 @@ Models that are being added are also tracked in issues labeled [new model](https | CLM-v0.1-8B | [Native Rust engine](../src/models/clm/README.md), `clm-run`, in front of a Qwen3-8B `/v1/embeddings` server | Engine-only agreement **4.5e-06** on identical vectors ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)); contract checks documented ([#23](https://github.com/ThinkFlowLab/system1-omni/pull/23)) | **Validated on compute capability 8.9** ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)) | Unverified | Rust to build and a separate Qwen3-8B `/v1/embeddings` server for the encoder; the engine itself has no device-specific code | | OmniJev-4B v1.1 | None yet: [checkpoint export and CPU contract](../recipe/omnijev/README.md); the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | Request contract, preparation, heads and finishing checked against the reference ([#120](https://github.com/ThinkFlowLab/system1-omni/pull/120)); no inference | Not yet | Not supported | Python 3.10 reference environment for the export and fixtures | | D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) on the checkpoint's pinned runtime — text, image and video state over `choice`/`noul`/`score`, with the deliberate transport differences documented in the recipe; the native worker is in progress ([#136](https://github.com/ThinkFlowLab/system1-omni/issues/136)) | Unverified | Author-run on NVIDIA A100 (sm_80): serving smoke and reference parity for text and media (exact decisions, max delta 0.0); evidence in the integration and media PRs. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) | -| Clef-flash (9B) | [Python reference worker](../recipe/clef/reference.md); text `choice`/`noul`/`score`; non-thinking prompt; `thinking.enabled: false` admitted and omitted from the prompt; 16384-token refusal | Contract tests only; no inference | Operator-reported Public231 C1 on NVIDIA L20X (sm_89), HEAD `903006530fb7772bb668bbf192ee4278e273e583`, warmup 5, concurrency 1. Primary run1 p50 247.518 ms, p95 413.025 ms, 231/231. Manifest `public231-no-thinking.jsonl` (`thinking` key stripped). Not re-run here | Not supported | Python 3.12, [card-tested torch 2.11 and transformers 5.10.2](../recipe/clef/requirements.txt), pinned `Cloudflare/clef-flash` snapshot (~19.1 GB) | +| Clef-flash (9B) | [Python reference worker](../recipe/clef/reference.md); text `choice`/`noul`/`score`; non-thinking prompt; `thinking.enabled: false` admitted and omitted from the prompt; 16384-token refusal | Contract tests only; no inference | Operator-reported Public231 C1 on the L20X-labeled lab host. Baseline `9030065` run1 p50 247.518 ms, p95 413.025 ms, 231/231. Tip `594528a9`: 231/231, p50 164.969 ms, p95 216.755 ms, `/health` `fla+causal_conv1d`. Versus run1: Δp50 −33.35% (−82.5 ms), Δp95 −47.52% (−196.3 ms). Not re-run here | Not supported | Python 3.12, [card-tested torch 2.11 and transformers 5.10.2](../recipe/clef/requirements.txt), pinned `Cloudflare/clef-flash` snapshot (~19.1 GB) | | Clef (27B) | Same worker and text contract as Clef-flash, selected with `--variant clef` | Contract tests only; no inference | Not run | Not supported | Same stack; pinned `Cloudflare/clef` snapshot (~55.0 GB). Weights and joint-head config differ from flash | - **Validated:** covered by the recipe on `main` or by the checks in the linked merged pull request. @@ -41,17 +41,16 @@ D3's reference worker serves the same three kinds for text, image and video state (inline base64 media under the pinned runtime's per-item budgets); its media serving evidence is author-run on the A100 above. Clef-flash and Clef serve the same three kinds for text only. The worker -refuses image and video inputs. The flash CUDA cell is an operator-reported -Public231 concurrency-1 result on a lab NVIDIA L20X at compute capability -8.9 (primary run1 p50 247.518 ms, p95 413.025 ms, metric -`successful_latency`, clock `post_to_body_json_and_schema_validation`). -That cell is the earlier L20X report at that HEAD, and it was not re-run -for this document. The worker starts on the lab host `nvidia-smi` names -`NVIDIA L20X`, including when the capability prints 8.9. FlashInfer GDN +refuses image and video inputs. The flash CUDA cell records two +operator-reported Public231 concurrency-1 results on the L20X-labeled lab +host. Baseline `9030065` run1 is p50 247.518 ms, p95 413.025 ms, metric +`successful_latency`, clock `post_to_body_json_and_schema_validation`. +Tip `594528a9` is p50 164.969 ms, p95 216.755 ms, 231/231, `/health` +`linear_attention` `fla+causal_conv1d`, Δp50 −33.35% (−82.5 ms), Δp95 +−47.52% (−196.3 ms). Neither campaign was re-run for this document. The +worker starts on that host when the capability prints 8.9. FlashInfer GDN prefill is used when the reported major is 9, 10, or 12 and the package -loads; otherwise the ready bind is the FLA chunk plus causal-conv1d. -`/health` reports which one started. No C1 of the current tip is recorded. -The 27B variant has no recorded timing. +loads; that path has no C1. The 27B variant has no recorded timing. CLM has a row per path: the merged `clm-serve` recipe exercises those answer shapes with stub embeddings and says nothing about decision quality, while the native engine's CUDA row is the path through a real Qwen3-8B encoder. Its engine-only figure comes from the same comparison run diff --git a/recipe/README.md b/recipe/README.md index e17e4e25..b04ad4eb 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -41,9 +41,9 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting - [Clef reference worker](clef/reference.md): load pinned Clef-flash (9B) or Clef (27B) and serve text `choice`/`noul`/`score`. Image, video and native CUDA are not served. Operator-reported flash Public231 concurrency-1 - timing on the lab host named NVIDIA L20X is in the recipe (HEAD `9030065`, - run1 p50 247.518 ms, p95 413.025 ms). The current tip has no recorded C1. - That host stays eligible when `nvidia-smi` reports capability 8.9. + timing on the L20X-labeled lab host is in the recipe: `9030065` run1 + p50 247.518 ms, p95 413.025 ms; tip `594528a9` p50 164.969 ms, p95 + 216.755 ms, `/health` `fla+causal_conv1d`. - [LFM2.5-350M choice worker](lfm2/README.md): score text decisions with independent questions, hybrid cache branching and configurable candidate batches. diff --git a/recipe/clef/reference.md b/recipe/clef/reference.md index 7cbf41b5..5b17a0ba 100644 --- a/recipe/clef/reference.md +++ b/recipe/clef/reference.md @@ -150,8 +150,8 @@ L20X at compute capability 8.9, warmup 5, concurrency 1. Stock Public231 includes a `thinking` field. That HEAD rejected it with HTTP 422 `Unknown request fields`. The measured manifest was `public231-no-thinking.jsonl`, with the `thinking` key stripped. The -stripped file's SHA-256 was not reported. Run1 is the primary L20X round. -Do not average the percentiles. Run1 is not a baseline for an H200 delta. +stripped file's SHA-256 was not reported. Run1 is the baseline for the +tip `594528a9` comparison below. Do not average the percentiles. | Round | p50 (ms) | p95 (ms) | Successful | | --- | --- | --- | --- | @@ -160,28 +160,41 @@ Do not average the percentiles. Run1 is not a baseline for an H200 delta. | run3 | 242.421 | 410.369 | 231/231 | The worker now admits `thinking` when it is an object with `enabled` set to -the JSON boolean `false`, then drops the field before encoding. The next -H200 campaign should send stock Public231 in that shape. That admission -has no measured latency delta on these L20X rounds. `enabled: true` stays -a 422. +the JSON boolean `false`, then drops the field before encoding. That +admission has no separate measured latency delta on these `9030065` rounds. +`enabled: true` stays a 422. The operator named the metric `successful_latency` and the clock `post_to_body_json_and_schema_validation`. These three rounds are that measurement. They are not a new campaign. -Those three rounds are the earlier report at HEAD `9030065` on the lab -host `nvidia-smi` names `NVIDIA L20X`. This tip has no new p50 or p95. -Do not publish a delta until a remeasure of this tip is pasted. +Those three rounds are the baseline at HEAD `9030065` on the lab host +`nvidia-smi` names `NVIDIA L20X`. -## Next measurement: lab host named NVIDIA L20X +## Public231 C1 of tip `594528a9` + +Operator-reported on that same L20X-labeled lab host. Not re-run in the +environment that edited this recipe. Worker +`594528a9f9c1ca38ce0dbb58a549191dfbde5bf3`. `/health` `linear_attention` +was `fla+causal_conv1d`. Successful requests 231/231. p50 164.969 ms. p95 +216.755 ms. Against baseline `9030065` run1 (p50 247.518 ms, p95 413.025 ms): +Δp50 −33.35% (−82.5 ms), Δp95 −47.52% (−196.3 ms). This run's bind is the +FLA chunk, not FlashInfer. Answer parity was not part of the paste. +Warmup count, clock name, manifest hash, driver, and CUDA version were +not restated in the paste, so they are not copied from the `9030065` +campaign. + +This code tip stays. A later prefill change is a new tip and needs its own +Public231 C1 paste before another delta is claimed. + +## Next code tip on the L20X-labeled host That host is the measurement machine, including when `nvidia-smi` reports compute capability 8.9. Do not require a card whose name is `NVIDIA H200`. On a reported 8.9 the ready bind is `fla+causal_conv1d`. When the same process sees reported major 9, 10, or 12 and FlashInfer loaded, the ready bind is `flashinfer+fla+causal_conv1d`. Record the `/health` string the -process returns. No Public231 C1 of this tip has been recorded. Answer -parity against `torch_chunk` is not measured. The standalone GDN timings in +process returns. The standalone GDN timings in [benchmarks/gdn/README.md](../../benchmarks/gdn/README.md) are a different experiment and are not a Clef HTTP result. @@ -273,19 +286,19 @@ in a commit. 9. The clock is the runner's `post_to_body_json_and_schema_validation` clock. It is not the H800 leaderboard clock. Record the `nvidia-smi` name, the printed capability, - and `/health` `linear_attention` with the p50 and p95. The table above - is the earlier `9030065` report. This tip has no measured p50 or p95 - until that paste exists. + and `/health` `linear_attention` with the p50 and p95. Compare a new + code tip with the recorded tip `594528a9` only after that new tip has + its own paste. The `9030065` run1 figures remain the baseline of the + recorded comparison above. ## What remains unverified -Image and video inputs, native CUDA, answer parity, and any Public231 C1 -of the current tip. The operator-reported numbers above stay on the clock -`post_to_body_json_and_schema_validation`, metric `successful_latency`, at -HEAD `903006530fb7772bb668bbf192ee4278e273e583`. H1, one encode per request, -is already how that HEAD calls the pinned model, so run1 includes it and -its isolated effect is unknown. This tip selects FlashInfer when that -kernel can dispatch and otherwise serves the FLA chunk with -`causal_conv1d`. Neither bind has a measured p50, p95, or answer-parity -result on this tip. This environment has no GPU and did not repeat the -campaign. +Image and video inputs, native CUDA, and answer parity. The `9030065` +rounds use metric `successful_latency` and clock +`post_to_body_json_and_schema_validation`. The tip `594528a9` p50 and p95 +are the operator paste in the section above; this environment has no GPU +and did not repeat either campaign. H1, one encode per request, is already +inside the `9030065` baseline, so its isolated effect is unknown. The +recorded win is the FLA chunk plus `causal_conv1d` on the reported-8.9 +host. FlashInfer on a reported major 9, 10, or 12 has no C1. A later code +change is unmeasured until its own paste. diff --git a/src/models/clef/README.md b/src/models/clef/README.md index c951dff7..605c664d 100644 --- a/src/models/clef/README.md +++ b/src/models/clef/README.md @@ -166,21 +166,28 @@ loads. A reported 8.9, including the lab name `NVIDIA L20X`, keeps the FLA chunk and still becomes ready. Startup does not require the name `NVIDIA H200`. `/health` `linear_attention` is `flashinfer+fla+causal_conv1d` or `fla+causal_conv1d`, matching the bind. -`torch_chunk_gated_delta_rule` and `F.conv1d` are not served. This tip has -no measured p50 or p95. Answer parity against `torch_chunk` is not measured. +`torch_chunk_gated_delta_rule` and `F.conv1d` are not served. + +Operator-reported Public231 C1 of worker +`594528a9f9c1ca38ce0dbb58a549191dfbde5bf3` on the L20X-labeled lab host: +231/231, p50 164.969 ms, p95 216.755 ms, `/health` `linear_attention` +`fla+causal_conv1d`. Against `9030065` run1: Δp50 −33.35% (−82.5 ms), +Δp95 −47.52% (−196.3 ms). Not re-run here. Warmup, clock, manifest hash, +driver, and CUDA version were not restated in that paste. Answer parity +was not reported. The measured bind is the FLA chunk. | Id | Change | Expected effect | Measured result | | --- | --- | --- | --- | | H1 | One `encode_record` per request, then the pinned forward. The handler does not tokenize the schema again before that encode. | Removes a second BPE pass from the critical path. Prefill and the joint head still dominate. | Included in HEAD `9030065`. Isolated delta unknown | | H2 | Fuse the joint head's per-question Python loop. | Fewer small launches after prefill. Needs a forked head and numerical parity against the pinned module. | Not implemented. No measured H200 delta | | H3 | `torch.compile` or CUDA graphs on the Qwen3.5 backbone. | Uncertain on the linear-attention layers. | Not implemented. No measured H200 delta | -| H4 | Bind `flash-linear-attention` 0.5.2 and `causal-conv1d` >= 1.5 before the Qwen3.5 layers copy the chunk global. Use FlashInfer GDN prefill 0.6.16.post3 with `use_cp` false when the reported capability major is 9, 10, or 12 and the package loads. Reported 8.9, including nvidia-smi name `NVIDIA L20X`, stays on the FLA chunk and still starts. Startup is not ready on `torch_chunk` or `F.conv1d`. vLLM CuteDSL, CUDA graphs, and `torch.compile` are not used. | Replaces the PyTorch chunk loop. `/health` names the bind that started. | Not measured on this tip | +| H4 | Bind `flash-linear-attention` 0.5.2 and `causal-conv1d` >= 1.5 before the Qwen3.5 layers copy the chunk global. Use FlashInfer GDN prefill 0.6.16.post3 with `use_cp` false when the reported capability major is 9, 10, or 12 and the package loads. Reported 8.9, including nvidia-smi name `NVIDIA L20X`, stays on the FLA chunk and still starts. Startup is not ready on `torch_chunk` or `F.conv1d`. vLLM CuteDSL, CUDA graphs, and `torch.compile` are not used. | Replaces the PyTorch chunk loop. `/health` names the bind that started. | Tip `594528a9`, L20X-labeled host, `/health` `fla+causal_conv1d`: p50 164.969 ms, p95 216.755 ms, 231/231. Versus `9030065` run1: Δp50 −33.35% (−82.5 ms), Δp95 −47.52% (−196.3 ms) | ## Status | Path | Clef-flash | Clef | | --- | --- | --- | -| Text `choice` / `noul` / `score`, non-thinking prompt | Contract tested. Operator-reported Public231 C1 latency on L20X at HEAD `9030065`. This tip has no C1 yet. A reported capability of 8.9 on the `NVIDIA L20X` host is ready. Answer parity not checked | Same loader pin; that latency campaign was flash only | +| Text `choice` / `noul` / `score`, non-thinking prompt | Contract tested. `9030065` run1 p50 247.518 ms, p95 413.025 ms. Tip `594528a9` on the L20X-labeled host: p50 164.969 ms, p95 216.755 ms, 231/231, `/health` `fla+causal_conv1d`. Answer parity not checked | Same loader pin; both latency campaigns were flash only | | Image and video | Rejected | Rejected | | Native CUDA | Not implemented | Not implemented | | Workers AI proxy | Out of scope | Out of scope | From 302dfa4f644a6d3578999c05acc6536af9fa8b28 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 25 Oct 2026 02:19:28 +0000 Subject: [PATCH 7/9] clef: note the recorded 594528a9 C1 in the recipe pin comment The requirements comment still said this tip had no Public231 C1. Co-authored-by: Hongsheng Liu --- recipe/clef/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/recipe/clef/requirements.txt b/recipe/clef/requirements.txt index 45757c40..a2b83e12 100644 --- a/recipe/clef/requirements.txt +++ b/recipe/clef/requirements.txt @@ -20,7 +20,8 @@ pillow>=10 # including nvidia-smi name NVIDIA L20X, stays on the FLA chunk and still # becomes ready. The device name is not required to be NVIDIA H200. # vLLM's separate CuteDSL prefill backend, CUDA graphs, and torch.compile -# are not selected. No Public231 C1 of this tip has been measured. +# are not selected. The recorded L20X-labeled C1 is tip 594528a9 on the +# FLA chunk: p50 164.969 ms, p95 216.755 ms, versus 9030065 run1. causal-conv1d>=1.5 flash-linear-attention==0.5.2 flashinfer-python==0.6.16.post3 From 8d5852409bd709c2db42f108baac8bfa2b168589 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 25 Oct 2026 02:36:46 +0000 Subject: [PATCH 8/9] clef: record the 594528a9 remeasure as the Public231 C1 primary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The L20X-labeled paste is warmup 5, concurrency 1, stock Public231 with thinking.enabled false, and metric successful_latency. Tip p50 164.969 ms and p95 216.755 ms, 231/231, health fla+causal_conv1d, versus 9030065 run1: Δp50 −82.549 ms (−33.35%), Δp95 −196.270 ms (−47.52%). The serve path is unchanged. Co-authored-by: Hongsheng Liu --- README.md | 2 +- docs/supported-models.md | 10 +++-- recipe/README.md | 7 ++-- recipe/clef/reference.md | 73 +++++++++++++++++++++++------------- recipe/clef/requirements.txt | 7 +++- src/models/clef/README.md | 21 ++++++----- 6 files changed, 75 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index b3e75b40..782ef0df 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ pinned runtime for text, image and video state: | LFM2.5-350M | [Transformers worker](recipe/lfm2/README.md); text state and independent choice questions | | OmniJev-4B v1.1 | [Checkpoint export and CPU contract](recipe/omnijev/README.md) only; the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | | D3 (Decision 3.0) | [Python reference worker](recipe/d3/reference.md) on the checkpoint's pinned runtime; `choice`/`noul`/`score` over text, image and video state; author-run on A100 for text and media serving (see the integration and media PRs) | -| Clef-flash (9B) and Clef (27B) | [Python reference worker](recipe/clef/reference.md) for text `choice`/`noul`/`score`; flash is the bring-up variant. Image, video and native CUDA are not served. L20X-labeled lab Public231 C1: `9030065` run1 p50 247.518 ms, p95 413.025 ms; tip `594528a9` p50 164.969 ms, p95 216.755 ms, `/health` `fla+causal_conv1d` | +| Clef-flash (9B) and Clef (27B) | [Python reference worker](recipe/clef/reference.md) for text `choice`/`noul`/`score`; flash is the bring-up variant. Image, video and native CUDA are not served. L20X-labeled lab Public231 C1 primary is tip `594528a9`: p50 164.969 ms, p95 216.755 ms, `/health` `fla+causal_conv1d`, versus `9030065` run1 p50 247.518 ms, p95 413.025 ms (Δp50 −82.549 ms, Δp95 −196.270 ms) | [Supported models and hardware](docs/supported-models.md) lists the devices and where each worker has been run. diff --git a/docs/supported-models.md b/docs/supported-models.md index 01189585..495c7c14 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -27,7 +27,7 @@ Models that are being added are also tracked in issues labeled [new model](https | CLM-v0.1-8B | [Native Rust engine](../src/models/clm/README.md), `clm-run`, in front of a Qwen3-8B `/v1/embeddings` server | Engine-only agreement **4.5e-06** on identical vectors ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)); contract checks documented ([#23](https://github.com/ThinkFlowLab/system1-omni/pull/23)) | **Validated on compute capability 8.9** ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)) | Unverified | Rust to build and a separate Qwen3-8B `/v1/embeddings` server for the encoder; the engine itself has no device-specific code | | OmniJev-4B v1.1 | None yet: [checkpoint export and CPU contract](../recipe/omnijev/README.md); the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | Request contract, preparation, heads and finishing checked against the reference ([#120](https://github.com/ThinkFlowLab/system1-omni/pull/120)); no inference | Not yet | Not supported | Python 3.10 reference environment for the export and fixtures | | D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) on the checkpoint's pinned runtime — text, image and video state over `choice`/`noul`/`score`, with the deliberate transport differences documented in the recipe; the native worker is in progress ([#136](https://github.com/ThinkFlowLab/system1-omni/issues/136)) | Unverified | Author-run on NVIDIA A100 (sm_80): serving smoke and reference parity for text and media (exact decisions, max delta 0.0); evidence in the integration and media PRs. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) | -| Clef-flash (9B) | [Python reference worker](../recipe/clef/reference.md); text `choice`/`noul`/`score`; non-thinking prompt; `thinking.enabled: false` admitted and omitted from the prompt; 16384-token refusal | Contract tests only; no inference | Operator-reported Public231 C1 on the L20X-labeled lab host. Baseline `9030065` run1 p50 247.518 ms, p95 413.025 ms, 231/231. Tip `594528a9`: 231/231, p50 164.969 ms, p95 216.755 ms, `/health` `fla+causal_conv1d`. Versus run1: Δp50 −33.35% (−82.5 ms), Δp95 −47.52% (−196.3 ms). Not re-run here | Not supported | Python 3.12, [card-tested torch 2.11 and transformers 5.10.2](../recipe/clef/requirements.txt), pinned `Cloudflare/clef-flash` snapshot (~19.1 GB) | +| Clef-flash (9B) | [Python reference worker](../recipe/clef/reference.md); text `choice`/`noul`/`score`; non-thinking prompt; `thinking.enabled: false` admitted and omitted from the prompt; 16384-token refusal | Contract tests only; no inference | Operator-reported Public231 C1 on the L20X-labeled lab host. Baseline `9030065` run1 p50 247.518 ms, p95 413.025 ms, 231/231. Tip `594528a9` remeasure, warmup 5, concurrency 1, stock Public231 with `thinking.enabled` false, metric `successful_latency`: 231/231, p50 164.969 ms, p95 216.755 ms, `/health` `fla+causal_conv1d`. Versus run1: Δp50 −82.549 ms (−33.35%), Δp95 −196.270 ms (−47.52%). Not re-run here | Not supported | Python 3.12, [card-tested torch 2.11 and transformers 5.10.2](../recipe/clef/requirements.txt), pinned `Cloudflare/clef-flash` snapshot (~19.1 GB) | | Clef (27B) | Same worker and text contract as Clef-flash, selected with `--variant clef` | Contract tests only; no inference | Not run | Not supported | Same stack; pinned `Cloudflare/clef` snapshot (~55.0 GB). Weights and joint-head config differ from flash | - **Validated:** covered by the recipe on `main` or by the checks in the linked merged pull request. @@ -45,9 +45,11 @@ refuses image and video inputs. The flash CUDA cell records two operator-reported Public231 concurrency-1 results on the L20X-labeled lab host. Baseline `9030065` run1 is p50 247.518 ms, p95 413.025 ms, metric `successful_latency`, clock `post_to_body_json_and_schema_validation`. -Tip `594528a9` is p50 164.969 ms, p95 216.755 ms, 231/231, `/health` -`linear_attention` `fla+causal_conv1d`, Δp50 −33.35% (−82.5 ms), Δp95 -−47.52% (−196.3 ms). Neither campaign was re-run for this document. The +Tip `594528a9` is the primary: warmup 5, concurrency 1, stock Public231 +with `thinking.enabled` false, metric `successful_latency`, p50 164.969 ms, +p95 216.755 ms, 231/231, `/health` `linear_attention` `fla+causal_conv1d`, +Δp50 −82.549 ms (−33.35%), Δp95 −196.270 ms (−47.52%). The tip paste does +not restate the clock name. Neither campaign was re-run for this document. The worker starts on that host when the capability prints 8.9. FlashInfer GDN prefill is used when the reported major is 9, 10, or 12 and the package loads; that path has no C1. The 27B variant has no recorded timing. diff --git a/recipe/README.md b/recipe/README.md index b04ad4eb..cf27c361 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -41,9 +41,10 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting - [Clef reference worker](clef/reference.md): load pinned Clef-flash (9B) or Clef (27B) and serve text `choice`/`noul`/`score`. Image, video and native CUDA are not served. Operator-reported flash Public231 concurrency-1 - timing on the L20X-labeled lab host is in the recipe: `9030065` run1 - p50 247.518 ms, p95 413.025 ms; tip `594528a9` p50 164.969 ms, p95 - 216.755 ms, `/health` `fla+causal_conv1d`. + timing on the L20X-labeled lab host is in the recipe. The primary is + tip `594528a9`: p50 164.969 ms, p95 216.755 ms, `/health` + `fla+causal_conv1d`, versus `9030065` run1 p50 247.518 ms, p95 413.025 ms + (Δp50 −82.549 ms, Δp95 −196.270 ms). - [LFM2.5-350M choice worker](lfm2/README.md): score text decisions with independent questions, hybrid cache branching and configurable candidate batches. diff --git a/recipe/clef/reference.md b/recipe/clef/reference.md index 5b17a0ba..cbbbcdeb 100644 --- a/recipe/clef/reference.md +++ b/recipe/clef/reference.md @@ -173,19 +173,28 @@ Those three rounds are the baseline at HEAD `9030065` on the lab host ## Public231 C1 of tip `594528a9` -Operator-reported on that same L20X-labeled lab host. Not re-run in the -environment that edited this recipe. Worker -`594528a9f9c1ca38ce0dbb58a549191dfbde5bf3`. `/health` `linear_attention` -was `fla+causal_conv1d`. Successful requests 231/231. p50 164.969 ms. p95 -216.755 ms. Against baseline `9030065` run1 (p50 247.518 ms, p95 413.025 ms): -Δp50 −33.35% (−82.5 ms), Δp95 −47.52% (−196.3 ms). This run's bind is the -FLA chunk, not FlashInfer. Answer parity was not part of the paste. -Warmup count, clock name, manifest hash, driver, and CUDA version were -not restated in the paste, so they are not copied from the `9030065` -campaign. +Operator-reported remeasure on that same L20X-labeled lab host. Not re-run +in the environment that edited this recipe. This paste is the primary a +later tip has to beat. + +Worker `594528a9f9c1ca38ce0dbb58a549191dfbde5bf3`. The paste states warmup +5, concurrency 1, stock Public231 with `thinking.enabled` false, and metric +`successful_latency`. `/health` `linear_attention` was `fla+causal_conv1d`. +Successful requests 231/231. p50 164.969 ms. p95 216.755 ms. Against +baseline `9030065` run1 (p50 247.518 ms, p95 413.025 ms, 231/231): Δp50 +−82.549 ms (−33.35%), Δp95 −196.270 ms (−47.52%). This run's bind is the +FLA chunk, not FlashInfer. + +The paste does not restate the clock name, the manifest SHA-256, the +driver, the CUDA version, or answer parity. Those fields are not copied +from the `9030065` rounds. Those rounds used `public231-no-thinking.jsonl` +because that HEAD returned HTTP 422 on `thinking`. This remeasure names +stock Public231 with `thinking.enabled` false. Neither paste includes a +`sha256sum` of the file that was sent. This code tip stays. A later prefill change is a new tip and needs its own -Public231 C1 paste before another delta is claimed. +Public231 C1 paste before another delta is claimed. Compare that paste +with p50 164.969 ms and p95 216.755 ms. ## Next code tip on the L20X-labeled host @@ -198,6 +207,10 @@ process returns. The standalone GDN timings in [benchmarks/gdn/README.md](../../benchmarks/gdn/README.md) are a different experiment and are not a Clef HTTP result. +The primary to beat is the `594528a9` remeasure: p50 164.969 ms, p95 +216.755 ms, 231/231, `/health` `fla+causal_conv1d`. A new tip needs its +own paste against those two figures. + This is the operator checklist for the next concurrency-1 latency measurement of the flash worker. It uses the existing [benchmark runner](../../benchmarks/README.md). The client does not reserve @@ -227,11 +240,13 @@ in a commit. The canonical task hash `dc3995d8ae1e2fc8e81ce38431add509eb8bb39b85aadfd0c7c32079382dde51` is a different encoding and must not be substituted. Check the file - with `sha256sum` before `validate`. The recorded campaign above did not - use that byte string: the `thinking` key had been removed, and the - resulting file hash was not reported. Record `sha256sum` of whichever - file the next run actually sends. Keep `thinking` when its `enabled` - value is false; stripping it is no longer required for admission. + with `sha256sum` before `validate`. The `9030065` rounds did not use + that byte string: the `thinking` key had been removed, and the resulting + file hash was not reported. The `594528a9` remeasure names stock + Public231 with `thinking.enabled` false and also does not report a file + hash. Record `sha256sum` of whichever file the next run actually sends. + Keep `thinking` when its `enabled` value is false; stripping it is no + longer required for admission. 4. Install `benchmarks/requirements.txt` (`httpx==0.28.1`) into the client environment. The client and the worker may use different environments. 5. Write a metadata JSON file. Required keys are `gpu`, `gpu_ids`, @@ -287,18 +302,24 @@ in a commit. `post_to_body_json_and_schema_validation` clock. It is not the H800 leaderboard clock. Record the `nvidia-smi` name, the printed capability, and `/health` `linear_attention` with the p50 and p95. Compare a new - code tip with the recorded tip `594528a9` only after that new tip has - its own paste. The `9030065` run1 figures remain the baseline of the - recorded comparison above. + code tip with the recorded primary, tip `594528a9` at p50 164.969 ms + and p95 216.755 ms, only after that new tip has its own paste. The + `9030065` run1 figures remain the baseline of that recorded comparison. ## What remains unverified Image and video inputs, native CUDA, and answer parity. The `9030065` rounds use metric `successful_latency` and clock -`post_to_body_json_and_schema_validation`. The tip `594528a9` p50 and p95 -are the operator paste in the section above; this environment has no GPU -and did not repeat either campaign. H1, one encode per request, is already -inside the `9030065` baseline, so its isolated effect is unknown. The -recorded win is the FLA chunk plus `causal_conv1d` on the reported-8.9 -host. FlashInfer on a reported major 9, 10, or 12 has no C1. A later code -change is unmeasured until its own paste. +`post_to_body_json_and_schema_validation`. The tip `594528a9` remeasure +states metric `successful_latency`, warmup 5, concurrency 1, and stock +Public231 with `thinking.enabled` false. It does not restate the clock +name, manifest hash, driver, CUDA version, or answer parity. This +environment has no GPU and did not repeat either campaign. H1, one encode +per request, is already inside the `9030065` baseline, so its isolated +effect is unknown. The recorded win, and the primary a later tip has to +beat, is the FLA chunk plus `causal_conv1d` on the reported-8.9 host: +p50 164.969 ms, p95 216.755 ms. FlashInfer on a reported major 9, 10, or +12 has no C1. FlashInfer's GDN prefill does not dispatch when the reported +major is 8, so this primary stays on the FLA chunk. Full attention is +already Transformers SDPA at head dimension 256. No further kernel change +is in this tree. A later code change is unmeasured until its own paste. diff --git a/recipe/clef/requirements.txt b/recipe/clef/requirements.txt index a2b83e12..2591473f 100644 --- a/recipe/clef/requirements.txt +++ b/recipe/clef/requirements.txt @@ -20,8 +20,11 @@ pillow>=10 # including nvidia-smi name NVIDIA L20X, stays on the FLA chunk and still # becomes ready. The device name is not required to be NVIDIA H200. # vLLM's separate CuteDSL prefill backend, CUDA graphs, and torch.compile -# are not selected. The recorded L20X-labeled C1 is tip 594528a9 on the -# FLA chunk: p50 164.969 ms, p95 216.755 ms, versus 9030065 run1. +# are not selected. The recorded L20X-labeled C1 primary is tip 594528a9 +# on the FLA chunk: warmup 5, concurrency 1, stock Public231 with +# thinking.enabled false, metric successful_latency, p50 164.969 ms, +# p95 216.755 ms, 231/231. Versus 9030065 run1: Δp50 −82.549 ms +# (−33.35%), Δp95 −196.270 ms (−47.52%). causal-conv1d>=1.5 flash-linear-attention==0.5.2 flashinfer-python==0.6.16.post3 diff --git a/src/models/clef/README.md b/src/models/clef/README.md index 605c664d..dcf9e2eb 100644 --- a/src/models/clef/README.md +++ b/src/models/clef/README.md @@ -168,26 +168,29 @@ chunk and still becomes ready. Startup does not require the name `flashinfer+fla+causal_conv1d` or `fla+causal_conv1d`, matching the bind. `torch_chunk_gated_delta_rule` and `F.conv1d` are not served. -Operator-reported Public231 C1 of worker -`594528a9f9c1ca38ce0dbb58a549191dfbde5bf3` on the L20X-labeled lab host: -231/231, p50 164.969 ms, p95 216.755 ms, `/health` `linear_attention` -`fla+causal_conv1d`. Against `9030065` run1: Δp50 −33.35% (−82.5 ms), -Δp95 −47.52% (−196.3 ms). Not re-run here. Warmup, clock, manifest hash, -driver, and CUDA version were not restated in that paste. Answer parity -was not reported. The measured bind is the FLA chunk. +Operator-reported Public231 C1 remeasure of worker +`594528a9f9c1ca38ce0dbb58a549191dfbde5bf3` on the L20X-labeled lab host. +The paste states warmup 5, concurrency 1, stock Public231 with +`thinking.enabled` false, and metric `successful_latency`. 231/231, p50 +164.969 ms, p95 216.755 ms, `/health` `linear_attention` +`fla+causal_conv1d`. Against `9030065` run1: Δp50 −82.549 ms (−33.35%), +Δp95 −196.270 ms (−47.52%). Not re-run here. The paste does not restate +the clock name, manifest hash, driver, or CUDA version. Answer parity +was not reported. The measured bind is the FLA chunk. This pair is the +primary a later tip has to beat. | Id | Change | Expected effect | Measured result | | --- | --- | --- | --- | | H1 | One `encode_record` per request, then the pinned forward. The handler does not tokenize the schema again before that encode. | Removes a second BPE pass from the critical path. Prefill and the joint head still dominate. | Included in HEAD `9030065`. Isolated delta unknown | | H2 | Fuse the joint head's per-question Python loop. | Fewer small launches after prefill. Needs a forked head and numerical parity against the pinned module. | Not implemented. No measured H200 delta | | H3 | `torch.compile` or CUDA graphs on the Qwen3.5 backbone. | Uncertain on the linear-attention layers. | Not implemented. No measured H200 delta | -| H4 | Bind `flash-linear-attention` 0.5.2 and `causal-conv1d` >= 1.5 before the Qwen3.5 layers copy the chunk global. Use FlashInfer GDN prefill 0.6.16.post3 with `use_cp` false when the reported capability major is 9, 10, or 12 and the package loads. Reported 8.9, including nvidia-smi name `NVIDIA L20X`, stays on the FLA chunk and still starts. Startup is not ready on `torch_chunk` or `F.conv1d`. vLLM CuteDSL, CUDA graphs, and `torch.compile` are not used. | Replaces the PyTorch chunk loop. `/health` names the bind that started. | Tip `594528a9`, L20X-labeled host, `/health` `fla+causal_conv1d`: p50 164.969 ms, p95 216.755 ms, 231/231. Versus `9030065` run1: Δp50 −33.35% (−82.5 ms), Δp95 −47.52% (−196.3 ms) | +| H4 | Bind `flash-linear-attention` 0.5.2 and `causal-conv1d` >= 1.5 before the Qwen3.5 layers copy the chunk global. Use FlashInfer GDN prefill 0.6.16.post3 with `use_cp` false when the reported capability major is 9, 10, or 12 and the package loads. Reported 8.9, including nvidia-smi name `NVIDIA L20X`, stays on the FLA chunk and still starts. Startup is not ready on `torch_chunk` or `F.conv1d`. vLLM CuteDSL, CUDA graphs, and `torch.compile` are not used. | Replaces the PyTorch chunk loop. `/health` names the bind that started. | Tip `594528a9`, L20X-labeled host, warmup 5, concurrency 1, stock Public231 with `thinking.enabled` false, metric `successful_latency`, `/health` `fla+causal_conv1d`: p50 164.969 ms, p95 216.755 ms, 231/231. Versus `9030065` run1: Δp50 −82.549 ms (−33.35%), Δp95 −196.270 ms (−47.52%). This is the primary | ## Status | Path | Clef-flash | Clef | | --- | --- | --- | -| Text `choice` / `noul` / `score`, non-thinking prompt | Contract tested. `9030065` run1 p50 247.518 ms, p95 413.025 ms. Tip `594528a9` on the L20X-labeled host: p50 164.969 ms, p95 216.755 ms, 231/231, `/health` `fla+causal_conv1d`. Answer parity not checked | Same loader pin; both latency campaigns were flash only | +| Text `choice` / `noul` / `score`, non-thinking prompt | Contract tested. `9030065` run1 p50 247.518 ms, p95 413.025 ms. Tip `594528a9` primary on the L20X-labeled host: p50 164.969 ms, p95 216.755 ms, 231/231, `/health` `fla+causal_conv1d`, Δp50 −82.549 ms, Δp95 −196.270 ms. Answer parity not checked | Same loader pin; both latency campaigns were flash only | | Image and video | Rejected | Rejected | | Native CUDA | Not implemented | Not implemented | | Workers AI proxy | Out of scope | Out of scope | From 980b4c148eff7b497be424a907767d0316636820 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 25 Oct 2026 07:16:55 +0000 Subject: [PATCH 9/9] clef: warm up before ready and keep FlashInfer opt-in One text forward runs before the port binds, so CUDA init fails at startup. FlashInfer GDN prefill stays behind --gdn-prefill flashinfer until a forward and answer parity against the FLA path exist. Clef pytest runs in CI, the pin test ignores installed packages, and the inference lock is taken only after the body is read. Co-authored-by: Hongsheng Liu --- .github/workflows/ci.yml | 4 ++ docs/supported-models.md | 7 +-- recipe/clef/reference.md | 78 +++++++++++++++++----------- recipe/clef/requirements.txt | 18 ++++--- src/frontend/clef.py | 47 +++++++++++++---- src/models/clef/README.md | 17 ++++--- src/models/clef/engine.py | 99 +++++++++++++++++++++++++++++++----- tests/clef/test_engine.py | 39 +++++++++++--- tests/clef/test_worker.py | 61 +++++++++++++++++++++- 9 files changed, 291 insertions(+), 79 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 306f3e39..97e4945b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,10 @@ jobs: - run: python -m unittest discover -s tests/jev_vl -p 'test_export_vision.py' -v - name: Decider validation helpers run: python -m unittest discover -s tests/decider -p 'test_*.py' -v + - name: Clef reference worker tests + run: | + python -m pip install pytest + PYTHONPATH=src python -m pytest tests/clef -q cuda-contract: # The CUDA backend contract and its checker. No CUDA toolkit and no GPU: both diff --git a/docs/supported-models.md b/docs/supported-models.md index b1b8adda..bf726d7a 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -50,9 +50,10 @@ with `thinking.enabled` false, metric `successful_latency`, p50 164.969 ms, p95 216.755 ms, 231/231, `/health` `linear_attention` `fla+causal_conv1d`, Δp50 −82.549 ms (−33.35%), Δp95 −196.270 ms (−47.52%). The tip paste does not restate the clock name. Neither campaign was re-run for this document. The -worker starts on that host when the capability prints 8.9. FlashInfer GDN -prefill is used when the reported major is 9, 10, or 12 and the package -loads; that path has no C1. The 27B variant has no recorded timing. +worker starts on that host when the capability prints 8.9. The default +prefill is the FLA chunk. `--gdn-prefill flashinfer` is opt-in and is the +named follow-up blocker: it has no measured forward and no answer parity +against `fla+causal_conv1d`. The 27B variant has no recorded timing. CLM has a row per path: the merged `clm-serve` recipe exercises those answer shapes with stub embeddings and says nothing about decision quality, while the native engine's CUDA row is the path through a real Qwen3-8B encoder. Its engine-only figure comes from the same comparison run diff --git a/recipe/clef/reference.md b/recipe/clef/reference.md index cbbbcdeb..b04cda73 100644 --- a/recipe/clef/reference.md +++ b/recipe/clef/reference.md @@ -35,34 +35,41 @@ The requirements pin `causal-conv1d>=1.5`, `flash-linear-attention==0.5.2`, and `flashinfer-python==0.6.16.post3`. The worker imports `causal_conv1d`, then `fla.ops.gated_delta_rule`, then tries `flashinfer.gdn_prefill`, and only then imports `transformers.models.qwen3_5.modeling_qwen3_5`. That -order binds `FusedRMSNormGated` and `causal_conv1d_fn`. When the reported -compute capability major is 9, 10, or 12 and FlashInfer loaded, startup +order binds `FusedRMSNormGated` and `causal_conv1d_fn`. The default +prefill is the FLA chunk. `--gdn-prefill flashinfer` is opt-in: startup replaces the modeling module's `chunk_gated_delta_rule` with the FlashInfer GDN prefill wrapper before `load_release_model` builds each -`Qwen3_5GatedDeltaNet`, because the layer copies that global at init. +`Qwen3_5GatedDeltaNet` only when that flag is set, the package imports, +and the reported capability major is 9, 10, or 12. The layer copies that +global at init. `causal-conv1d` and `flash-linear-attention` are required. Startup raises when either is missing, when flash-linear-attention is not exactly `0.5.2`, when causal-conv1d is older than 1.5, or when `causal_conv1d_fn` and a chunk kernel did not bind. A missing or different FlashInfer install does -not, by itself, refuse ready. `/health` `linear_attention` is the bind that -started: `flashinfer+fla+causal_conv1d` or `fla+causal_conv1d`. The PyTorch -`torch_chunk_gated_delta_rule` and `F.conv1d` paths are not served. +not, by itself, refuse the default FLA path. Requesting +`--gdn-prefill flashinfer` without that package, or on a device whose +reported major is not 9, 10, or 12, refuses startup. `/health` +`linear_attention` is the bind that started: +`fla+causal_conv1d` by default, or `flashinfer+fla+causal_conv1d` when the +opt-in flag bound. The PyTorch `torch_chunk_gated_delta_rule` and +`F.conv1d` paths are not served. FlashInfer prefill calls `flashinfer.gdn_prefill.chunk_gated_delta_rule` with `use_cp=False`. Q and K L2 normalization uses FLA `l2norm_fwd`, and FlashInfer's `use_qk_l2norm_in_kernel` flag stays false. Log-space `g` is exponentiated. The call is one sequence and does not continue a recurrent state. FlashInfer 0.6.16 does not dispatch that kernel when the device -reports capability 8.9, so those processes keep the FLA chunk. vLLM's -separate CuteDSL prefill backend, CUDA graphs, and `torch.compile` are -not used. +reports capability 8.9. vLLM's separate CuteDSL prefill backend, CUDA +graphs, and `torch.compile` are not used. The lab host is the machine `nvidia-smi` names `NVIDIA L20X`, including when it reports compute capability 8.9. Startup does not require the name -`NVIDIA H200` and does not refuse ready because the capability is not 9.0. -No Public231 C1 of this tip has been measured. Answer parity against -`torch_chunk` is not measured. +`NVIDIA H200` and does not refuse the default FLA path because the +capability is not 9.0. The measured primary is that FLA path. FlashInfer +GDN prefill has no measured forward and no answer parity against +`fla+causal_conv1d`. That parity run is the named follow-up blocker +before `--gdn-prefill flashinfer` can become the default. ```sh python - <<'PY' @@ -82,15 +89,18 @@ PYTHONPATH=src python -m frontend.clef \ --model \ --variant clef-flash \ --device cuda \ + --gdn-prefill fla \ --host 127.0.0.1 \ --port 8000 ``` -Readiness is the process log line after `load_release_model` returns. -`GET http://127.0.0.1:8000/health` then returns `status: ready`, -`variant: clef-flash`, `prompt_mode: non-thinking`, `modalities: ["text"]` -and `max_input_tokens: 16384`. The first `POST` is the first forward. That -forward was not run in this change. +The process runs one text forward, then binds. `GET +http://127.0.0.1:8000/health` is not served until that forward returns. +It then returns `status: ready`, `variant: clef-flash`, +`prompt_mode: non-thinking`, `modalities: ["text"]`, +`max_input_tokens: 16384`, and `linear_attention: fla+causal_conv1d` on +the default flag. A CUDA init failure exits before the port opens. That +startup forward was not run in this editing environment. For the 27B checkpoint, download `Cloudflare/clef` at `a30162b99d68a481d8bd75d38df069a0cf83df40` and pass `--variant clef`. @@ -200,10 +210,11 @@ with p50 164.969 ms and p95 216.755 ms. That host is the measurement machine, including when `nvidia-smi` reports compute capability 8.9. Do not require a card whose name is `NVIDIA H200`. -On a reported 8.9 the ready bind is `fla+causal_conv1d`. When the same -process sees reported major 9, 10, or 12 and FlashInfer loaded, the ready -bind is `flashinfer+fla+causal_conv1d`. Record the `/health` string the -process returns. The standalone GDN timings in +On a reported 8.9 the ready bind is `fla+causal_conv1d`. That is also the +default on reported majors 9, 10, and 12. `flashinfer+fla+causal_conv1d` +is ready only when the process was started with `--gdn-prefill flashinfer` +and that kernel could bind. Record the `/health` string the process +returns. The standalone GDN timings in [benchmarks/gdn/README.md](../../benchmarks/gdn/README.md) are a different experiment and are not a Clef HTTP result. @@ -228,11 +239,13 @@ in a commit. Do not report the run as H800 or as the Public231 leaderboard. 2. Start `frontend.clef` as above, on loopback port 8000, variant `clef-flash`, device `cuda`. Leave that process up for every traversal. - `GET /health` must show `status` `ready`. `linear_attention` is - `fla+causal_conv1d` or `flashinfer+fla+causal_conv1d`, whichever the - process bound. Record that string. A process that exits at startup did - not bind `causal_conv1d_fn` and a chunk kernel. Do not measure a - process that is still on the PyTorch chunk or `F.conv1d` path. + `GET /health` is absent until the startup forward returns, then must + show `status` `ready`. The default `linear_attention` is + `fla+causal_conv1d`. `flashinfer+fla+causal_conv1d` means the process + was started with `--gdn-prefill flashinfer`. Record that string. A + process that exits at startup did not bind `causal_conv1d_fn` and a + chunk kernel. Do not measure a process that is still on the PyTorch + chunk or `F.conv1d` path. 3. Obtain the frozen Public231 request manifest whose file SHA-256 is `00d1f10ca1142499a598b5f1c6602d329cce1cef04c961c46fc0318daa2c67d3`. That is the JSONL byte hash cited for the JevBench `7ce310c7` export in @@ -318,8 +331,11 @@ environment has no GPU and did not repeat either campaign. H1, one encode per request, is already inside the `9030065` baseline, so its isolated effect is unknown. The recorded win, and the primary a later tip has to beat, is the FLA chunk plus `causal_conv1d` on the reported-8.9 host: -p50 164.969 ms, p95 216.755 ms. FlashInfer on a reported major 9, 10, or -12 has no C1. FlashInfer's GDN prefill does not dispatch when the reported -major is 8, so this primary stays on the FLA chunk. Full attention is -already Transformers SDPA at head dimension 256. No further kernel change -is in this tree. A later code change is unmeasured until its own paste. +p50 164.969 ms, p95 216.755 ms. The default prefill remains that FLA +chunk, including on reported majors 9, 10, and 12. `--gdn-prefill +flashinfer` is opt-in. It has no measured forward and no answer parity +against `fla+causal_conv1d`; that parity run is the named follow-up +blocker before the flag becomes the default. The startup forward that +runs before `/health` is separate from the client warmup in the C1 +protocol and has no latency claim. A later code change is unmeasured +until its own paste. diff --git a/recipe/clef/requirements.txt b/recipe/clef/requirements.txt index 2591473f..015c59fb 100644 --- a/recipe/clef/requirements.txt +++ b/recipe/clef/requirements.txt @@ -9,16 +9,20 @@ tokenizers>=0.22 # The processor loads with the backbone. Image and video requests are rejected # by this worker; pillow is still imported by the pinned processor. pillow>=10 -# Import order is causal_conv1d, then fla.ops.gated_delta_rule, then -# flashinfer.gdn_prefill, and only then modeling_qwen3_5. FLA must be -# importable first so Transformers binds FusedRMSNormGated instead of the -# PyTorch norm. The worker then replaces the chunk global with FlashInfer -# GDN prefill (use_cp false, fused SM90) before layers copy it. +# Import order is causal_conv1d, then fla.ops.gated_delta_rule, then an +# attempted flashinfer.gdn_prefill import, and only then modeling_qwen3_5. +# FLA must be importable first so Transformers binds FusedRMSNormGated +# instead of the PyTorch norm. The default leaves the FLA chunk in place. +# --gdn-prefill flashinfer replaces that global with FlashInfer GDN prefill +# (use_cp false) before layers copy it. # causal_conv1d_fn stays the convolution. Startup refuses torch_chunk and -# F.conv1d. FlashInfer replaces the chunk only when the reported capability -# major is 9, 10, or 12 and this package imports. A device that reports 8.9, +# F.conv1d. The default prefill is the FLA chunk. --gdn-prefill flashinfer +# replaces the chunk only when that flag is set, this package imports, and +# the reported capability major is 9, 10, or 12. A device that reports 8.9, # including nvidia-smi name NVIDIA L20X, stays on the FLA chunk and still # becomes ready. The device name is not required to be NVIDIA H200. +# FlashInfer has no measured forward and no answer parity against the FLA +# path. That parity run is the follow-up blocker before the flag is default. # vLLM's separate CuteDSL prefill backend, CUDA graphs, and torch.compile # are not selected. The recorded L20X-labeled C1 primary is tip 594528a9 # on the FLA chunk: warmup 5, concurrency 1, stock Public231 with diff --git a/src/frontend/clef.py b/src/frontend/clef.py index ae1fd4f0..7ad98720 100644 --- a/src/frontend/clef.py +++ b/src/frontend/clef.py @@ -21,7 +21,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any -from models.clef.engine import DEFAULT_VARIANT, VARIANTS, ClefExecutor +from models.clef.engine import DEFAULT_VARIANT, FLA_PREFILL, GDN_PREFILL, VARIANTS, ClefExecutor from models.clef.preprocess import validate_questions from models.clef.protocol import MAX_BODY, RequestError, parse_body, parse_request @@ -106,19 +106,23 @@ def do_POST(self): if self.headers.get_content_type() != "application/json": self.send_json(415, {"detail": "Content-Type must be application/json"}) return + try: + raw = self.rfile.read(length) + except (TimeoutError, socket.timeout): + self.send_json(408, {"detail": "request body timed out"}) + return + except OSError: + return + if len(raw) != length: + self.send_json(400, {"detail": "incomplete body"}) + return if not self.server.inference_lock.acquire(blocking=False): self.send_json(503, {"detail": "worker busy"}) return try: - raw = self.rfile.read(length) - if len(raw) != length: - self.send_json(400, {"detail": "incomplete body"}) - return self.send_json(200, decide(raw, self.server.executor)) except RequestError as exc: self._send_request_error(exc) - except (TimeoutError, socket.timeout): - self.send_json(408, {"detail": "request body timed out"}) except Exception as exc: LOG.error("clef inference failed: %s", type(exc).__name__) self.send_json(500, {"detail": "inference failed"}) @@ -131,18 +135,43 @@ def parse_args(argv=None): parser.add_argument("--model", required=True, help="snapshot directory of the pinned revision") parser.add_argument("--variant", default=DEFAULT_VARIANT, choices=sorted(VARIANTS)) parser.add_argument("--device", default="cuda") + parser.add_argument( + "--gdn-prefill", + default=FLA_PREFILL, + choices=(FLA_PREFILL, GDN_PREFILL), + help=( + "Linear-attention prefill. fla is the measured default. " + "flashinfer is opt-in and has no answer parity against that path." + ), + ) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=8000) return parser.parse_args(argv) +def listen(executor: Any, address: tuple[str, int]) -> WorkerServer: + """Bind only after one forward warmup succeeds. + + A failure here leaves the port closed, so ``/health`` does not report + ready until CUDA init and one text forward have returned. + """ + + executor.warmup() + return WorkerServer(address, executor) + + def main(argv=None): args = parse_args(argv) logging.basicConfig(level=logging.INFO) - executor = ClefExecutor(args.model, variant=args.variant, device=args.device) + executor = ClefExecutor( + args.model, + variant=args.variant, + device=args.device, + gdn_prefill=args.gdn_prefill, + ) server = None try: - server = WorkerServer((args.host, args.port), executor) + server = listen(executor, (args.host, args.port)) LOG.info( "%s worker ready on %s:%s", executor.variant.repo_id, diff --git a/src/models/clef/README.md b/src/models/clef/README.md index dcf9e2eb..194c422f 100644 --- a/src/models/clef/README.md +++ b/src/models/clef/README.md @@ -160,12 +160,15 @@ implementation was not part of this report. `causal-conv1d>=1.5`, `flash-linear-attention==0.5.2`, and `flashinfer-python==0.6.16.post3` are installed by the recipe. The worker imports `causal_conv1d` and `fla.ops.gated_delta_rule` before Transformers -binds Qwen3.5. FlashInfer replaces the chunk global, with `use_cp` false, -only when the reported capability major is 9, 10, or 12 and that package -loads. A reported 8.9, including the lab name `NVIDIA L20X`, keeps the FLA -chunk and still becomes ready. Startup does not require the name -`NVIDIA H200`. `/health` `linear_attention` is -`flashinfer+fla+causal_conv1d` or `fla+causal_conv1d`, matching the bind. +binds Qwen3.5. The default prefill is the FLA chunk. `--gdn-prefill +flashinfer` replaces the chunk global, with `use_cp` false, only when that +flag is set, the reported capability major is 9, 10, or 12, and that +package loads. A reported 8.9, including the lab name `NVIDIA L20X`, keeps +the FLA chunk and still becomes ready. Startup does not require the name +`NVIDIA H200`. `/health` `linear_attention` is `fla+causal_conv1d` by +default, or `flashinfer+fla+causal_conv1d` when the opt-in flag bound. +The process runs one text forward before it binds, so `/health` is not +served until that forward returns. `torch_chunk_gated_delta_rule` and `F.conv1d` are not served. Operator-reported Public231 C1 remeasure of worker @@ -184,7 +187,7 @@ primary a later tip has to beat. | H1 | One `encode_record` per request, then the pinned forward. The handler does not tokenize the schema again before that encode. | Removes a second BPE pass from the critical path. Prefill and the joint head still dominate. | Included in HEAD `9030065`. Isolated delta unknown | | H2 | Fuse the joint head's per-question Python loop. | Fewer small launches after prefill. Needs a forked head and numerical parity against the pinned module. | Not implemented. No measured H200 delta | | H3 | `torch.compile` or CUDA graphs on the Qwen3.5 backbone. | Uncertain on the linear-attention layers. | Not implemented. No measured H200 delta | -| H4 | Bind `flash-linear-attention` 0.5.2 and `causal-conv1d` >= 1.5 before the Qwen3.5 layers copy the chunk global. Use FlashInfer GDN prefill 0.6.16.post3 with `use_cp` false when the reported capability major is 9, 10, or 12 and the package loads. Reported 8.9, including nvidia-smi name `NVIDIA L20X`, stays on the FLA chunk and still starts. Startup is not ready on `torch_chunk` or `F.conv1d`. vLLM CuteDSL, CUDA graphs, and `torch.compile` are not used. | Replaces the PyTorch chunk loop. `/health` names the bind that started. | Tip `594528a9`, L20X-labeled host, warmup 5, concurrency 1, stock Public231 with `thinking.enabled` false, metric `successful_latency`, `/health` `fla+causal_conv1d`: p50 164.969 ms, p95 216.755 ms, 231/231. Versus `9030065` run1: Δp50 −82.549 ms (−33.35%), Δp95 −196.270 ms (−47.52%). This is the primary | +| H4 | Bind `flash-linear-attention` 0.5.2 and `causal-conv1d` >= 1.5 before the Qwen3.5 layers copy the chunk global. The default prefill is that FLA chunk. `--gdn-prefill flashinfer` uses FlashInfer GDN prefill 0.6.16.post3 with `use_cp` false only when the flag is set, the package loads, and the reported capability major is 9, 10, or 12. Reported 8.9, including nvidia-smi name `NVIDIA L20X`, stays on the FLA chunk and still starts. Startup is not ready on `torch_chunk` or `F.conv1d`. vLLM CuteDSL, CUDA graphs, and `torch.compile` are not used. | Replaces the PyTorch chunk loop. `/health` names the bind that started. | Tip `594528a9`, L20X-labeled host, warmup 5, concurrency 1, stock Public231 with `thinking.enabled` false, metric `successful_latency`, `/health` `fla+causal_conv1d`: p50 164.969 ms, p95 216.755 ms, 231/231. Versus `9030065` run1: Δp50 −82.549 ms (−33.35%), Δp95 −196.270 ms (−47.52%). This is the primary. FlashInfer has no forward and no answer parity against it | ## Status diff --git a/src/models/clef/engine.py b/src/models/clef/engine.py index 6e0e6eac..c5ca3738 100644 --- a/src/models/clef/engine.py +++ b/src/models/clef/engine.py @@ -213,20 +213,57 @@ def bound_linear_attention(modeling: Any) -> str: return "torch_chunk" -def select_gdn_prefill(capability: tuple[int, int] | None) -> str: - """Choose FlashInfer only when its GDN prefill kernel can dispatch. - - Reported majors 9, 10, and 12 take FlashInfer. A device that reports - 8.9 stays on the FLA chunk, including the lab cards ``nvidia-smi`` names - ``NVIDIA L20X``. The printed name is not a gate, and a capability other - than 9.0 is not a reason to refuse ready. +def select_gdn_prefill( + capability: tuple[int, int] | None, requested: str = FLA_PREFILL +) -> str: + """Choose the prefill bind. The default is the measured FLA chunk. + + FlashInfer is opt-in. It is returned only when ``requested`` is + ``flashinfer`` and the reported major is 9, 10, or 12. A device that + reports 8.9 stays on the FLA chunk, including the lab cards + ``nvidia-smi`` names ``NVIDIA L20X``. The printed name is not a gate, + and a capability other than 9.0 is not a reason to refuse the FLA path. """ - if capability is not None and capability[0] in FLASHINFER_ARCH_MAJORS: + if requested not in (FLA_PREFILL, GDN_PREFILL): + raise ValueError(f"gdn prefill must be {FLA_PREFILL!r} or {GDN_PREFILL!r}") + if ( + requested == GDN_PREFILL + and capability is not None + and capability[0] in FLASHINFER_ARCH_MAJORS + ): return GDN_PREFILL return FLA_PREFILL +def resolve_gdn_prefill( + capability: tuple[int, int] | None, requested: str, flashinfer_loaded: bool +) -> str: + """Return the bind, or refuse an opt-in FlashInfer request that cannot run. + + The default request stays on FLA even when FlashInfer imported and the + reported major is 9, 10, or 12. That path has no forward and no answer + parity against ``fla+causal_conv1d``, so it is not selected by itself. + """ + + backend = select_gdn_prefill(capability, requested) + if requested == GDN_PREFILL and backend != GDN_PREFILL: + printed = "unavailable" if capability is None else f"{capability[0]}.{capability[1]}" + raise RuntimeError( + "Clef FlashInfer GDN prefill was requested with --gdn-prefill " + f"flashinfer, but this device reports capability {printed}. " + "That kernel is selected only for reported majors 9, 10, and 12. " + "The measured path is --gdn-prefill fla." + ) + if requested == GDN_PREFILL and not flashinfer_loaded: + raise RuntimeError( + "Clef FlashInfer GDN prefill was requested, but " + f"{FI_DISTRIBUTION}=={FI_VERSION} did not import. " + "The measured path is --gdn-prefill fla." + ) + return backend + + def cuda_capability(device: str) -> tuple[int, int]: """Return the capability of a CUDA device. The name is not consulted.""" @@ -251,15 +288,17 @@ def bind_selected_prefill(modeling: Any, backend: str, flashinfer_loaded: bool) install_flashinfer_gdn_prefill(modeling) -def activate_gdn_prefill(device: str, flashinfer_loaded: bool) -> None: - """Bind the prefill that this CUDA device can run.""" +def activate_gdn_prefill( + device: str, flashinfer_loaded: bool, requested: str = FLA_PREFILL +) -> None: + """Bind the requested prefill. The default is the FLA chunk.""" modeling = sys.modules.get(MODELING) if modeling is None: raise RuntimeError(f"{MODELING} is not imported.") bind_selected_prefill( modeling, - select_gdn_prefill(cuda_capability(device)), + resolve_gdn_prefill(cuda_capability(device), requested, flashinfer_loaded), flashinfer_loaded, ) @@ -490,16 +529,26 @@ def load_joint_schema_module(path: Path) -> Any: class ClefExecutor: """Load and run one immutable pinned Clef snapshot.""" - def __init__(self, snapshot: str | Path, variant: str = DEFAULT_VARIANT, device: str = "cuda") -> None: + def __init__( + self, + snapshot: str | Path, + variant: str = DEFAULT_VARIANT, + device: str = "cuda", + gdn_prefill: str = FLA_PREFILL, + ) -> None: if variant not in VARIANTS: raise ValueError(f"variant must be one of {sorted(VARIANTS)}") + if gdn_prefill not in (FLA_PREFILL, GDN_PREFILL): + raise ValueError(f"gdn_prefill must be {FLA_PREFILL!r} or {GDN_PREFILL!r}") self.variant = VARIANTS[variant] self.root = Path(snapshot) self.device = device + self.gdn_prefill_request = gdn_prefill self.max_length = MAX_INPUT_TOKENS + self._warmed = False verify_snapshot(self.root, self.variant) flashinfer_loaded = require_fast_linear_attention() - activate_gdn_prefill(device, flashinfer_loaded) + activate_gdn_prefill(device, flashinfer_loaded, gdn_prefill) self.module = load_joint_schema_module(self.root / EXECUTABLE) self.model, self.processor = self.module.load_release_model(self.root, device=device) self.linear_attention = require_bound_linear_attention(sys.modules.get(MODELING)) @@ -562,9 +611,31 @@ def _forward(self, payload: dict[str, Any], encoded: Any) -> dict[str, Any]: "usage": {"input_tokens": len(encoded.input_ids), "output_tokens": 0}, } + def warmup(self) -> None: + """Run one text forward before this process reports ready. + + The HTTP server is not bound until this returns. A CUDA context or + kernel failure exits startup instead of the first live request. + The answer is discarded and is not a latency sample. + """ + + self.serve( + { + "model": self.variant.name, + "state": "warmup", + "questions": { + "ready": { + "type": "noul", + "instructions": "Is warmup complete?", + } + }, + } + ) + self._warmed = True + def health(self) -> dict[str, Any]: return { - "status": "ready", + "status": "ready" if self._warmed else "loading", "model": self.variant.repo_id, "variant": self.variant.name, "revision": self.variant.revision, diff --git a/tests/clef/test_engine.py b/tests/clef/test_engine.py index 481a3709..e50814b2 100644 --- a/tests/clef/test_engine.py +++ b/tests/clef/test_engine.py @@ -1,4 +1,5 @@ import json +import sys from pathlib import Path import pytest @@ -106,6 +107,7 @@ def test_linear_attention_name_follows_the_bound_callables(): flashinfer_chunk_gated_delta_rule, install_flashinfer_gdn_prefill, require_bound_linear_attention, + resolve_gdn_prefill, select_gdn_prefill, ) @@ -131,20 +133,34 @@ def test_linear_attention_name_follows_the_bound_callables(): with pytest.raises(RuntimeError, match="chunk_gated_delta_rule"): install_flashinfer_gdn_prefill(conv_only) assert select_gdn_prefill((8, 9)) == "fla" - assert select_gdn_prefill((9, 0)) == "flashinfer" - assert select_gdn_prefill((10, 0)) == "flashinfer" - assert select_gdn_prefill((12, 0)) == "flashinfer" + assert select_gdn_prefill((9, 0)) == "fla" + assert select_gdn_prefill((10, 0)) == "fla" + assert select_gdn_prefill((12, 0)) == "fla" assert select_gdn_prefill(None) == "fla" + assert select_gdn_prefill((9, 0), "flashinfer") == "flashinfer" + assert select_gdn_prefill((10, 0), "flashinfer") == "flashinfer" + assert select_gdn_prefill((12, 0), "flashinfer") == "flashinfer" + assert select_gdn_prefill((8, 9), "flashinfer") == "fla" + assert select_gdn_prefill(None, "flashinfer") == "fla" + assert resolve_gdn_prefill((9, 0), "fla", True) == "fla" + assert resolve_gdn_prefill((9, 0), "flashinfer", True) == "flashinfer" + with pytest.raises(RuntimeError, match="reports capability 8.9"): + resolve_gdn_prefill((8, 9), "flashinfer", True) + with pytest.raises(RuntimeError, match="did not import"): + resolve_gdn_prefill((9, 0), "flashinfer", False) reported_89 = _bound_pair() original_chunk = reported_89.chunk_gated_delta_rule - bind_selected_prefill(reported_89, select_gdn_prefill((8, 9)), True) + bind_selected_prefill(reported_89, select_gdn_prefill((8, 9), "flashinfer"), True) assert reported_89.chunk_gated_delta_rule is original_chunk assert require_bound_linear_attention(reported_89) == "fla+causal_conv1d" + default_90 = _bound_pair() + bind_selected_prefill(default_90, select_gdn_prefill((9, 0)), True) + assert require_bound_linear_attention(default_90) == "fla+causal_conv1d" reported_90 = _bound_pair() - bind_selected_prefill(reported_90, select_gdn_prefill((9, 0)), True) + bind_selected_prefill(reported_90, select_gdn_prefill((9, 0), "flashinfer"), True) assert require_bound_linear_attention(reported_90) == "flashinfer+fla+causal_conv1d" unloaded_fi = _bound_pair() - bind_selected_prefill(unloaded_fi, select_gdn_prefill((9, 0)), False) + bind_selected_prefill(unloaded_fi, select_gdn_prefill((9, 0), "flashinfer"), False) assert require_bound_linear_attention(unloaded_fi) == "fla+causal_conv1d" with pytest.raises(RuntimeError, match="one sequence"): flashinfer_chunk_gated_delta_rule(object(), object(), object()) @@ -152,7 +168,10 @@ def test_linear_attention_name_follows_the_bound_callables(): flashinfer_chunk_gated_delta_rule(object(), object(), object(), output_final_state=True) -def test_linear_attention_pins_and_startup_fail_closed(): +def test_linear_attention_pins_and_startup_fail_closed(monkeypatch): + import importlib.metadata + + from models.clef import engine from models.clef.engine import ( FI_VERSION, FLA_VERSION, @@ -174,5 +193,11 @@ def test_linear_attention_pins_and_startup_fail_closed(): check_distribution_version("0.6.16", exact=FI_VERSION) with pytest.raises(RuntimeError, match="required >=1.5"): check_distribution_version("1.4.9", minimum=(1, 5)) + + def missing(name): + raise importlib.metadata.PackageNotFoundError(name) + + monkeypatch.delitem(sys.modules, engine.MODELING, raising=False) + monkeypatch.setattr(engine.importlib.metadata, "version", missing) with pytest.raises(RuntimeError, match="not installed"): require_fast_linear_attention() diff --git a/tests/clef/test_worker.py b/tests/clef/test_worker.py index 47b62bb4..8c55be57 100644 --- a/tests/clef/test_worker.py +++ b/tests/clef/test_worker.py @@ -1,10 +1,11 @@ import json +import time from http.client import HTTPConnection from threading import Thread import pytest -from frontend.clef import WorkerServer, decide +from frontend.clef import WorkerServer, decide, listen from models.clef.postprocess import systemone_answer from models.clef.preprocess import enforce_limit, measure from models.clef.protocol import MAX_BODY, Request, RequestError @@ -202,6 +203,64 @@ def test_transport_errors(server): conn.close() +def test_listen_binds_only_after_warmup(monkeypatch): + order = [] + + class Executor: + def __init__(self): + self.warmed = False + + def warmup(self): + order.append("warmup") + self.warmed = True + + def close(self): + order.append("close") + + original = WorkerServer.__init__ + + def init(self, address, executor): + assert executor.warmed + order.append("bind") + original(self, address, executor) + + monkeypatch.setattr(WorkerServer, "__init__", init) + server = listen(Executor(), ("127.0.0.1", 0)) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + assert order == ["warmup", "bind"] + finally: + stop(server) + + class Failing: + def warmup(self): + order.append("fail") + raise RuntimeError("cuda init") + + def close(self): + pass + + with pytest.raises(RuntimeError, match="cuda init"): + listen(Failing(), ("127.0.0.1", 0)) + # stop() closes the first executor before the failing warmup runs. + assert order == ["warmup", "bind", "close", "fail"] + + +def test_stalled_body_does_not_hold_the_inference_lock(server): + payload = json.dumps(three_kinds()).encode("utf-8") + stalled = HTTPConnection("127.0.0.1", server.server_address[1], timeout=5) + stalled.putrequest("POST", "/v1/systemone", skip_accept_encoding=True) + stalled.putheader("Content-Type", "application/json") + stalled.putheader("Content-Length", str(len(payload))) + stalled.endheaders() + time.sleep(0.2) + status, raw = post(server, payload) + stalled.close() + assert status == 200 + assert json.loads(raw)["answers"]["urgent"]["noul"] == 0.25 + + def test_busy_and_inference_failure(server): server.inference_lock.acquire() try: