From 30e87f51a4820c6c20b678d9c7979f42b3b5e360 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 24 Oct 2026 16:13:47 +0000 Subject: [PATCH 1/3] Add the StartLux Instant 4B native worker Serve the pinned Instant 4B checkpoint on /v1/systemone for choice, noul, and score, with bidirectional full attention and a tied letter readout. Document the Ada sm_89 build and the L20X closed-loop latency recipe. Weights stay CC BY-NC 4.0 and are not included. Co-authored-by: Hongsheng Liu --- Cargo.lock | 16 + Cargo.toml | 2 +- README.md | 8 +- docs/architecture.md | 12 +- docs/supported-models.md | 4 +- mkdocs.yml | 2 + recipe/README.md | 4 + recipe/startlux/README.md | 124 ++ recipe/startlux/latency.md | 150 ++ src/backends/cuda/qwen3_5/README.md | 15 +- src/backends/cuda/qwen3_5/attention.cu | 32 +- src/backends/cuda/qwen3_5/ops.h | 7 + .../cuda/qwen3_5/qwen3_5.backend.json | 3 +- src/models/qwen3_5/native/src/cuda.rs | 30 + src/models/qwen3_5/native/src/lib.rs | 2 +- src/models/qwen3_5/native/src/model.rs | 137 +- src/models/startlux/NOTICE | 10 + src/models/startlux/README.md | 99 + src/models/startlux/native/Cargo.toml | 26 + src/models/startlux/native/src/contract.rs | 1670 +++++++++++++++++ src/models/startlux/native/src/engine.rs | 92 + src/models/startlux/native/src/executor.rs | 155 ++ src/models/startlux/native/src/lib.rs | 8 + src/models/startlux/native/src/main.rs | 117 ++ src/models/startlux/native/src/processing.rs | 89 + tests/startlux/contract.rs | 397 ++++ .../startlux/fixtures/choice_noul_score.json | 17 + tests/startlux/fixtures/decision_config.json | 8 + .../startlux/fixtures/instant_4b_config.json | 34 + 29 files changed, 3232 insertions(+), 38 deletions(-) create mode 100644 recipe/startlux/README.md create mode 100644 recipe/startlux/latency.md create mode 100644 src/models/startlux/NOTICE create mode 100644 src/models/startlux/README.md create mode 100644 src/models/startlux/native/Cargo.toml create mode 100644 src/models/startlux/native/src/contract.rs create mode 100644 src/models/startlux/native/src/engine.rs create mode 100644 src/models/startlux/native/src/executor.rs create mode 100644 src/models/startlux/native/src/lib.rs create mode 100644 src/models/startlux/native/src/main.rs create mode 100644 src/models/startlux/native/src/processing.rs create mode 100644 tests/startlux/contract.rs create mode 100644 tests/startlux/fixtures/choice_noul_score.json create mode 100644 tests/startlux/fixtures/decision_config.json create mode 100644 tests/startlux/fixtures/instant_4b_config.json diff --git a/Cargo.lock b/Cargo.lock index 4affb469..16664f21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1321,6 +1321,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "omni-startlux-native" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "half", + "memmap2", + "omni-qwen3-5-native", + "omni-runtime", + "safetensors 0.8.0", + "serde_json", + "tokenizers 0.22.2", + "tokio", +] + [[package]] name = "once_cell" version = "1.21.4" diff --git a/Cargo.toml b/Cargo.toml index 6c49c1f7..d4d00a3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["src/frontend", "src/runtime", "src/models/clm", "src/models/cua_s1/native", "src/models/qwen3_5/native", "src/models/open_jev/native", "src/models/jev_vl/native", "src/models/omnijev/native", "src/models/laya", "src/backends/cuda", "src/models/decider/native", "src/models/jemm/native"] +members = ["src/frontend", "src/runtime", "src/models/clm", "src/models/cua_s1/native", "src/models/qwen3_5/native", "src/models/open_jev/native", "src/models/jev_vl/native", "src/models/omnijev/native", "src/models/laya", "src/backends/cuda", "src/models/decider/native", "src/models/jemm/native", "src/models/startlux/native"] resolver = "3" diff --git a/README.md b/README.md index a1b55e87..9240e4d1 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,11 @@ The LFM2.5-350M Transformers worker owns candidate scoring and hybrid cache bran processing and executor modules, with shared FIFO admission and blocking dispatch in the [native runtime](src/runtime/README.md). - **Native CUDA workers.** The Cua-S1 4B 0.2 `text` adapter, - Open-Jev-27B-v1.1 and Open-Jev-9B run as native workers with shared CUDA - kernels. Cua-S1 also has a Python worker that serves as the correctness reference. + Open-Jev-27B-v1.1, Open-Jev-9B, and StartLux-Decision v1.1 4B Instant run + as native workers with shared CUDA kernels. Cua-S1 also has a Python + worker that serves as the correctness reference. The StartLux worker is + text-only Instant; its weights are CC BY-NC 4.0 and its GPU parity is + unverified. AutoThink is not served. - **LAYA text serving.** LAYA runs as an external CPU Python worker, the in-repository Python MPS/CPU worker, or a native Rust/CUDA worker on Hopper. - **CUDA backend and planned Metal backend.** High-performance GPU operations @@ -176,6 +179,7 @@ pinned runtime for text, image and video state: | Valen-Preview-0923 | [Python reference worker](recipe/valen/reference.md); text state or one inline PNG/JPEG image, `choice`; author-run on RTX 5060 Laptop GPU (sm_120, WSL2), owner: community | | Open-Jev-27B-v1.1 | [Native Rust/CUDA worker](recipe/open_jev/native.md); eager independent text candidates; [H200 validation](recipe/open_jev/validation.md) | | Open-Jev-9B | The same [native Rust/CUDA worker](recipe/open_jev/native.md); eager independent text candidates; [reference comparison on sm_89](recipe/open_jev/validation-9b.md) | +| StartLux-Decision v1.1 4B Instant | [Native Rust/CUDA text worker](recipe/startlux/README.md); `choice` / `noul` / `score`; weights CC BY-NC 4.0; Instant only (no AutoThink, no 27B); GPU parity unverified; [L20X latency recipe](recipe/startlux/latency.md) | | autotrust/JEV-27B-VL | [Experimental Rust/CUDA worker](recipe/jev_vl/README.md); single-question text, online inline PNG/JPEG and offline-preencoded image inputs; [validation scope and limits](recipe/jev_vl/validation.md) | | CLM-v0.1-8B | [External worker with a CPU stub encoder](recipe/clm/README.md); contract checks only, real Qwen3-8B decisions unverified by this recipe | | LFM2.5-350M | [Transformers worker](recipe/lfm2/README.md); text state and independent choice questions | diff --git a/docs/architecture.md b/docs/architecture.md index 33ec4043..8cdbc91a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -7,9 +7,12 @@ design. Concrete input/output types follow each executor's supported layout. ## Implementation status The [Rust frontend](../src/frontend/README.md) currently forwards HTTP requests -to separately running workers. Cua-S1, Open-Jev and Decider have native Rust/CUDA workers -that share the [Qwen3.5/3.8 executor](../src/models/qwen3_5/native/), which accepts -single prompts and bounded packed prefill. Cua-S1 and Open-Jev-9B use +to separately running workers. Cua-S1, Open-Jev, Decider, and StartLux Instant +4B have native Rust/CUDA workers that share the +[Qwen3.5/3.8 executor](../src/models/qwen3_5/native/), which accepts +single prompts and bounded packed prefill. StartLux Instant uses a +bidirectional full-attention call on each prompt; its GDN stays causal, and +it does not pack prompts or cache prefixes. Cua-S1 and Open-Jev-9B use single-prompt calls; Open-Jev-27B-v1.1 packs candidates within one request for input and gate/up GEMMs while preserving per-sequence mixers and output/down GEMM shapes. Decider optionally packs up to four complete rows and 4096 tokens within @@ -72,6 +75,7 @@ and candidate identity, usage, and response metadata outside the executor. | Decider | Complete independent token-ID rows in question order; Score expands one no/yes row per level. | One BF16-projection-rounded FP32 selected-label logit vector per row. | Per-type calibration, whole-Score normalization, ordered typed answers and unique-prefix token usage. | | Laya | One padded request: token IDs, true lengths, question types and ordered option markers; at most 16 questions, 512 tokens per row and 2048 markers. | Per-question FP32 option logits and two action logits copied back after GPU heads. | Calibrate and decode ordered `choice`, `score` and `noul` answers, usage and metadata. | | JEMM | Ordered token IDs, three-axis positions, candidate counts and optional shared image patches for one complete request. | Per-question selected BF16 label logits copied back as FP32. | Apply checksum-covered text/image calibration and reconstruct ordered `choice`, `score` and `noul` answers and usage. | +| StartLux Instant 4B | One unpadded token-ID prompt per question, or per knockout group and then a final prompt, in request order. | FP32 letter logits from a CPU dot of the tied embedding rows at the last position. | Per-type temperature, none-check split, wide-choice residual, and ordered `choice`, `noul`, and `score` answers with token usage. | Cua-S1 and Open-Jev-9B input collections are serial work. For Open-Jev-27B-v1.1, Open-Jev's model-specific batch adapter packs up to 16 independent candidates and @@ -79,7 +83,7 @@ Open-Jev's model-specific batch adapter packs up to 16 independent candidates an execute alone. It restores question/candidate grouping before normalization. Laya batches questions within one request. Shared runtime admission precedes blocking dispatch: Cua-S1 admits one question forward at a -time; Open-Jev, Decider and Laya admit one complete request. Cua-S1's CPU letter projection +time; Open-Jev, Decider, Laya, and StartLux Instant admit one complete request. StartLux runs each prompt inside that admission one at a time. Cua-S1's CPU letter projection stays outside admission; Open-Jev's scalar heads and Laya's GPU heads and synchronized readback remain inside their request unit. Decider includes every backbone row and CUDA head projection, synchronizes both streams and retires diff --git a/docs/supported-models.md b/docs/supported-models.md index 4afd2514..bfe67bc7 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -19,6 +19,7 @@ Models that are being added are also tracked in issues labeled [new model](https | Valen-Preview-0923 | [Python reference worker](../recipe/valen/reference.md); text state or one inline PNG/JPEG image; `choice` questions | Not supported | Author-run on RTX 5060 Laptop (sm_120, WSL2): one text and one 2×2 PNG request; input and readout parity ≤1e-6 against reference (model outputs not compared). Owner: community. | Not supported | Python 3.12, pinned Valen source/checkpoint, Qwen3.5-2B base, BF16 CUDA | | Open-Jev-27B-v1.1 | [Native Rust/CUDA worker](../recipe/open_jev/native.md) on the shared Qwen3.5/3.8 executor | Not supported | Validated on H200 (sm_90) for the [74 single-candidate workload](../recipe/open_jev/validation.md) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, exported merged weights and trained head | | Open-Jev-9B | The same [native Rust/CUDA worker](../recipe/open_jev/native.md) | Not supported | Validated on compute capability 8.9 against the reference for [253 requests](../recipe/open_jev/validation-9b.md) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, exported merged weights and trained head | +| StartLux-Decision v1.1 4B Instant | [Native Rust/CUDA worker](../recipe/startlux/README.md) | Contract checks only; no CPU inference | Unverified. Build target is L20X / Ada sm_89. The [latency recipe](../recipe/startlux/latency.md) is not a result | Not supported | Pinned CC BY-NC 4.0 Instant 4B checkpoint; rebuilt ABI 9 library with `cs1_attention_gated_bidirectional`. Instant only: AutoThink and 27B are refused | | Decider-2B v11 | [Native Rust/CUDA worker](../recipe/decider/README.md) | Contract checks only; no CPU inference | RTX 4090 (`sm_89`) synthetic parity corpus; [limits and provenance](../recipe/decider/validation.md) | Not supported | Pinned original BF16 checkpoint; ABI 9 library and consumers rebuilt together | | JEMM / Qwen3.8-27B | [Native Rust/CUDA worker](../recipe/jemm/README.md) | Host checks only; no CPU inference | A800 (`sm_80`) [text and 1–4-image frozen corpus](../src/models/jemm/README.md#validation) | Not supported | Pinned `jemm-native/2` export with BF16 base and unmerged FP32 LoRA, selected untied LM head, cuDNN 9 and additive reference CUDA symbols | | autotrust/JEV-27B-VL | [Experimental native worker](../recipe/jev_vl/README.md), single-question API | Contract tests only; no CPU inference | H800 frozen-corpus check; [experimental scope and limits](../recipe/jev_vl/validation.md) | Not supported | Pinned merged checkpoint, rebuilt ABI 9 CUDA library; image inputs require offline Transformers preencoding | @@ -34,7 +35,8 @@ Models that are being added are also tracked in issues labeled [new model](https - **Planned:** not implemented yet; the linked issue tracks it. The Cua-S1 workers answer `choice` questions only. -LAYA's English worker, Open-Jev and CLM support `choice`, `score`, and `noul` text questions. +LAYA's English worker, Open-Jev, CLM, and StartLux Instant 4B support `choice`, `score`, and `noul` text questions. +StartLux serves Instant only; AutoThink is not implemented, and the weights are CC BY-NC 4.0. D3's reference worker serves the same three kinds for text, image and video state (inline base64 media under the pinned runtime's per-item budgets); its media serving evidence is author-run on the A100 above. diff --git a/mkdocs.yml b/mkdocs.yml index ec30436b..b6cc1fb0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -70,6 +70,8 @@ nav: - Cua-S1 native screenshots: recipe/cua_s1/native_screenshot.md - Cua-S1 vision CUDA Graph: recipe/cua_s1/native_vision_graph.md - Open-Jev native text worker: recipe/open_jev/native.md + - StartLux Instant 4B native worker: recipe/startlux/README.md + - StartLux Instant 4B L20X latency: recipe/startlux/latency.md - Open-Jev-27B H200 validation: recipe/open_jev/validation.md - Open-Jev-9B validation: recipe/open_jev/validation-9b.md - Decider native text worker: recipe/decider/README.md diff --git a/recipe/README.md b/recipe/README.md index 27778139..3958289d 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -26,6 +26,10 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting backbone and trained decision head of Open-Jev-27B-v1.1 or Open-Jev-9B, then serve with Rust and CUDA. [Open-Jev-9B validation](open_jev/validation-9b.md) compares the 9B worker with the reference. +- [StartLux Instant 4B native worker](startlux/README.md): build the Ada sm_89 + CUDA library, serve the pinned CC BY-NC Instant checkpoint, and run the + [L20X closed-loop latency recipe](startlux/latency.md). AutoThink and 27B + are not in this recipe. GPU parity is unverified. - [JEMM native worker](jemm/README.md): pinned text/image decisions and export; known multi-image numerical limitation is documented in the model contract. - [JEV-27B-VL experimental worker](jev_vl/README.md): native text and preencoded diff --git a/recipe/startlux/README.md b/recipe/startlux/README.md new file mode 100644 index 00000000..44c1b1b7 --- /dev/null +++ b/recipe/startlux/README.md @@ -0,0 +1,124 @@ +# StartLux-Decision v1.1 4B Instant + +Serve text `choice`, `noul`, and `score` from the pinned Instant 4B +checkpoint with the native worker `omni-startlux-native`. AutoThink and +Instant 27B are not part of this recipe. + +Weights are **CC BY-NC 4.0** +(`startlux-models/StartLux-Decision-v1.1-4B-Instant`, revision +`416088f77cfce09633906a3675c7cd2e5dabcf10`, about 9.4 GB of BF16 files). +Download them on the machine that will serve. This repository does not +contain them. Inference code adapted from StartLux-Decision +`a99977d9332decc9fb27a2488b4a83b533b6cd5f` is Apache-2.0; see +[NOTICE](../../src/models/startlux/NOTICE). + +GPU parity and latency are unverified here. The intended device is one +L20X (Ada, sm_89), not an H200. [latency.md](latency.md) is the closed-loop +measurement to run after this worker starts on that GPU. Reserve the GPU +with the host's usual scheduler and record the reservation id. This recipe +does not add scheduler flags. + +Commands below run from the repository root unless a path says otherwise. + +## Build + +The CUDA library and the worker must be built together. ABI stays 9. +`cs1_attention_gated_bidirectional` is an extra symbol; a library built +before it loads for causal workers and is refused by this one. + +```sh +src/backends/cuda/qwen3_5/build.sh target/release 89 +cargo build --release --locked -p omni-startlux-native -p omni-jev +``` + +`build.sh` needs the CUDA toolkit (`nvcc`) and writes +`target/release/libqwen3_5_cuda.so`. The second command builds the worker +and the Rust frontend. There is no merge/export step. + +## Weights + +```sh +hf download startlux-models/StartLux-Decision-v1.1-4B-Instant \ + --revision 416088f77cfce09633906a3675c7cd2e5dabcf10 \ + --local-dir "$STARTLUX_MODEL" +``` + +`STARTLUX_MODEL` is the folder that contains `config.json`, +`decision_config.json`, `tokenizer.json`, and the safetensors index. The +worker checks the Instant 4B shapes and refuses an AutoThink or 27B folder. + +## Serve + +```sh +STARTLUX_MODEL=/path/to/StartLux-Decision-v1.1-4B-Instant \ +STARTLUX_CUDA_LIB=$PWD/target/release/libqwen3_5_cuda.so \ +STARTLUX_HOST=127.0.0.1 \ +STARTLUX_PORT=8000 \ + ./target/release/omni-startlux-native +``` + +The process loads the checkpoint, runs one warmup decision, then listens. +Startup prints the CC BY-NC line. `GET /health` reports `ready`, the model +id, `prompt_attention: bidirectional`, the license, and the weights pin. + +| Variable | Default | Meaning | +| --- | --- | --- | +| `STARTLUX_MODEL` | required | Instant 4B folder | +| `STARTLUX_CUDA_LIB` | `libqwen3_5_cuda.so` beside the binary | Shared CUDA library | +| `STARTLUX_HOST` | `127.0.0.1` | Listen address | +| `STARTLUX_PORT` | `8000` | Listen port, so `OMNI_JEV_BACKEND_URL=http://127.0.0.1:8000` matches | +| `STARTLUX_MAX_MODEL_LEN` | `65536` | Reject longer prompts. `1..=262144` | +| `STARTLUX_NONE_CHECK` | on | `0` turns off the extra "None of the above" option | +| `STARTLUX_ALLOW_NONE` | off | `1` allows `choice: null` when that extra option wins | + +The body limit is 32 MiB. A larger body is 413. The content type must be +`application/json`. Bad JSON is 400. Contract errors are 422 with +`error` and `detail`. The worker does not truncate. + +Optional frontend, from the same root, against the worker above: + +```sh +OMNI_JEV_BIND=127.0.0.1:8080 \ +OMNI_JEV_BACKEND_URL=http://127.0.0.1:8000 \ + ./target/release/omni-jev +``` + +## Requests + +```sh +curl -sS http://127.0.0.1:8000/health + +curl -sS http://127.0.0.1:8000/v1/systemone \ + -H 'content-type: application/json' \ + -d '{"state":"I was charged twice.","questions":{"route":{"type":"choice","instructions":"What should happen?","criteria":{"refund":"Refund the charge","keep":"Keep the charge"}}}}' + +curl -sS http://127.0.0.1:8000/v1/systemone \ + -H 'content-type: application/json' \ + -d '{"state":"I was charged twice.","questions":{"ask":{"type":"noul","instructions":"Is a refund requested?"}}}' + +curl -sS http://127.0.0.1:8000/v1/systemone \ + -H 'content-type: application/json' \ + -d '{"state":"I was charged twice.","questions":{"severity":{"type":"score","instructions":"How severe is it?","criteria":["low","high"]}}}' +``` + +With the defaults, the choice answer includes `none_probability` and the +choice stays one of the listed options. `think_mode` omitted or `never` is +the Instant path. `think_mode` `auto` or `always` is 422. + +## Checks without a GPU + +```sh +cargo test --locked -p omni-startlux-native --test contract +``` + +That test does not load weights or CUDA. It does not establish numerical +parity. + +## Limits + +Text only. No AutoThink thought, no 27B, no images, no prefix cache, no +CUDA graph, no packed prompts. Two GDN parameters are stored FP32 upstream +and rounded to BF16 at load. Letter logits are a CPU dot of the tied +embedding and are unverified against vLLM. A causal vLLM server without the +StartLux bidirectional plugin is a different model; do not use it as the +Instant baseline. The baseline command is in [latency.md](latency.md). diff --git a/recipe/startlux/latency.md b/recipe/startlux/latency.md new file mode 100644 index 00000000..2f6ab867 --- /dev/null +++ b/recipe/startlux/latency.md @@ -0,0 +1,150 @@ +# StartLux Instant 4B latency on L20X + +This is the measurement to run on hk2 after the native worker is serving. +No latency number is recorded in this change. Do not compare the result +with the H800 or H200 rows in +[the Public231 leaderboard](../../benchmarks/latency-leaderboard.md): those +used different hardware, clocks, and a different client clock. + +Label the hardware **L20X** (Ada, compute capability **sm_89**). Do not +label it H200. + +## What is measured + +Use `benchmarks/bench.py`. Its clock includes the HTTP body, JSON decoding, +and schema checks. It is the same runner as the Open-Jev-9B C1 path in the +new-runner notes, not the historical H800 body-receipt clock. + +Closed loop, concurrency 1. Five warmup requests, then three full +traversals of the same manifest. Each traversal gets its own output +directory. Warmups and process startup stay out of the pooled samples. +Report per-round and pooled p50/p95 from `bench.py summary`, with +planned/attempted/successful/failed counts. + +Both arms: + +- the same Instant 4B folder, revision + `416088f77cfce09633906a3675c7cd2e5dabcf10` +- BF16 +- thinking off (`think_mode` omitted or `never`) +- `none_check` on and `allow_none` off (native defaults; the upstream + `serve` default is the same) +- one reserved L20X +- no retries + +`--score-decimals 4` (the bench default) accepts StartLux full-precision +scores when the scalar equals the expectation of the returned +probabilities. + +## Manifest + +Public231 is not in this repository. The tasks are JevBench `7ce310c7` +(`https://github.com/fstandhartinger/jevbench/tree/7ce310c7262ed49cc85853339a8a42459298e3f3`). +The five-model campaign records canonical dataset SHA-256 +`dc3995d8ae1e2fc8e81ce38431add509eb8bb39b85aadfd0c7c32079382dde51`. +The exported public231 JSONL used by the two-model run is SHA-256 +`00d1f10ca1142499a598b5f1c6602d329cce1cef04c961c46fc0318daa2c67d3` +([issue #130](https://github.com/ThinkFlowLab/system1-omni/issues/130)). +Those hashes are different representations. Record which file you replayed. + +```sh +python -m pip install -r benchmarks/requirements.txt +python benchmarks/bench.py validate /path/to/public231.jsonl +``` + +## Metadata + +Write `run-metadata.json` before measuring. Required fields are `gpu`, +`gpu_ids`, `driver`, `cuda`, `precision`, `model_revision`, +`runtime_revision`, `cache_policy`, `cuda_evidence`, and `reservation`. +Set `gpu` to `L20X` and record sm_89 in the notes. `model_revision` is +`416088f77cfce09633906a3675c7cd2e5dabcf10`. `runtime_revision` is the worker +git revision. On the reference arm also record StartLux-Decision +`a99977d9332decc9fb27a2488b4a83b533b6cd5f` and +`python -c 'import vllm; print(vllm.__version__)'`. Upstream documents vLLM +0.19.1 and 0.31. `cache_policy` for both arms is no prefix cache. +`reservation` is the id from the host scheduler. + +## Native arm + +Build and serve with [README.md](README.md). Leave `STARTLUX_NONE_CHECK` +unset and do not set `STARTLUX_ALLOW_NONE`. Then, from the repository root, +three traversals: + +```sh +python benchmarks/bench.py run /path/to/public231.jsonl \ + --endpoint http://127.0.0.1:8000/v1/systemone \ + --model StartLux-Decision-v1.1-4B-Instant \ + --metadata run-metadata.json \ + --phase feasibility \ + --concurrency 1 --warmup 5 --score-decimals 4 \ + --output /tmp/startlux-l20x-feasibility + +python benchmarks/bench.py run /path/to/public231.jsonl \ + --endpoint http://127.0.0.1:8000/v1/systemone \ + --model StartLux-Decision-v1.1-4B-Instant \ + --metadata run-metadata.json \ + --phase measured \ + --concurrency 1 --warmup 5 --score-decimals 4 \ + --output /tmp/startlux-l20x-c1-run1 + +python benchmarks/bench.py run /path/to/public231.jsonl \ + --endpoint http://127.0.0.1:8000/v1/systemone \ + --model StartLux-Decision-v1.1-4B-Instant \ + --metadata run-metadata.json \ + --phase measured \ + --concurrency 1 --warmup 5 --score-decimals 4 \ + --output /tmp/startlux-l20x-c1-run2 + +python benchmarks/bench.py run /path/to/public231.jsonl \ + --endpoint http://127.0.0.1:8000/v1/systemone \ + --model StartLux-Decision-v1.1-4B-Instant \ + --metadata run-metadata.json \ + --phase measured \ + --concurrency 1 --warmup 5 --score-decimals 4 \ + --output /tmp/startlux-l20x-c1-run3 +``` + +Feasibility is not a performance sample. Reuse the same server for the +three measured traversals. Existing output directories are not overwritten. + +```sh +python benchmarks/bench.py summary /tmp/startlux-l20x-c1-run1 +``` + +Repeat for run2 and run3. Pool the three traversals the way +`benchmarks/README.md` describes; do not average the three p95 figures. + +## Reference arm + +The baseline is the upstream Instant server on the **same** GPU and the +**same** checkpoint, with the bidirectional plugin enabled. A causal vLLM +process without that plugin is not Instant. + +From a checkout of StartLux-Decision `a99977d9332decc9fb27a2488b4a83b533b6cd5f`: + +```sh +startlux-decide serve \ + --model /path/to/StartLux-Decision-v1.1-4B-Instant \ + --launch-vllm \ + --gpus 0 +``` + +`--launch-vllm` starts vLLM with the folder's bidirectional plugin. `--gpus` +selects the device (`0` is the process default; point it at the reserved +L20X). `serve` listens on `127.0.0.1:8090` unless `--port` is set. The vLLM +process is the engine behind that server, not the bench endpoint. Keep +none-check on and allow-none off (the `serve` defaults; do not pass +`--no-none-check`). + +Run the same three measured commands against +`http://127.0.0.1:8090/v1/systemone`, with a metadata file whose +`runtime_revision` names the vLLM and StartLux pins. Use output directories +`/tmp/startlux-l20x-vllm-c1-run1` through `run3`. Compare native and +reference only inside this L20X campaign. + +## What not to claim + +This file is the recipe, not a result. Until both arms finish on the +reserved L20X, p50/p95 are unknown. A feasibility run, a single traversal, +or a different GPU does not fill them in. diff --git a/src/backends/cuda/qwen3_5/README.md b/src/backends/cuda/qwen3_5/README.md index d4096f2a..ee2dbeba 100644 --- a/src/backends/cuda/qwen3_5/README.md +++ b/src/backends/cuda/qwen3_5/README.md @@ -16,7 +16,20 @@ The norm, elementwise and q/k preparation kernels round to bfloat16 where Transf `cs1_attention_gated` fuses the sigmoid gate into the attention epilogue, preserving the BF16 rounding of both attention and sigmoid before multiplication. The native workers use this entry point; the separate operations remain available for kernel -comparisons. Rebuild the library and all native Rust consumers together for +comparisons. `cs1_attention_gated_bidirectional` is an optional ABI 9 symbol with +the same argument list. It is the Instant prompt mask: every query reads every +key in that prompt, and a non-zero `q_base` is rejected. Causal +`cs1_attention_gated` and prefix continuation stay causal. The second template +parameter defaults to false, so existing causal launches keep that mask. Causal +callers do not resolve the new symbol. StartLux Instant fails closed when it is +missing. Rebuild this library before starting that worker. This tree does not +claim that the causal specialization's SASS is bit-identical to the previous +build; GPU revalidation of Cua-S1 and Open-Jev was not run with this change. + +HF Qwen3.5 folders store `linear_attn.A_log` and `linear_attn.norm.weight` as +FP32. The shared loader rounds those two suffixes to BF16 at upload because the +kernels consume BF16. Other non-BF16 language tensors still fail. That rounding +is a parity gap versus an FP32 `exp(A_log)` and GDN norm in Transformers. Rebuild the library and all native Rust consumers together for **ABI version 9**, even with resident features disabled. ABI 9 retains the vision, CUDA Graph, gated attention and prefix-continuation entry points and ABI 8's `cs1_get_device` and `cs1_bf16_finite` for device feature validation, and adds the diff --git a/src/backends/cuda/qwen3_5/attention.cu b/src/backends/cuda/qwen3_5/attention.cu index 71cf6e83..24a03909 100644 --- a/src/backends/cuda/qwen3_5/attention.cu +++ b/src/backends/cuda/qwen3_5/attention.cu @@ -1,4 +1,8 @@ // Full-attention layers of Qwen3.5: q/k preparation and causal attention. +// `flash_kernel` is the StartLux Instant prompt mask: every query +// reads every key in the same prompt. The `` specialization is the +// causal kernel used by Cua-S1, Open-Jev, Decider and JEMM; its mask and early +// key cutoff stay compile-time constants. // // cs1_attention is a FlashAttention-2 style kernel on tensor cores (mma.sync // m16n8k16, bfloat16 in, float32 accumulation): a block takes 64 queries of one head, @@ -81,7 +85,7 @@ constexpr int SMEM_BYTES = (BM + 2 * BN) * LDS * 2; // q_base offsets row 0 of the q/gate/out buffers: this launch covers the window // [q_base, T_total) while k/v cover [0, T_total). Used by the cached-prefix // continuation; q_base = 0 reduces to the plain full pass. -template +template __global__ void __launch_bounds__(THREADS) flash_kernel(const bf16* __restrict__ q, const bf16* __restrict__ k, const bf16* __restrict__ v, int ldv, const bf16* __restrict__ gate, bf16* __restrict__ out, int T, int Hq, int Hk, @@ -107,7 +111,8 @@ __global__ void __launch_bounds__(THREADS) for (int n = 0; n < D / 8; n++) o[n][0] = o[n][1] = o[n][2] = o[n][3] = 0.f; float m[2] = {-INFINITY, -INFINITY}, l[2] = {0.f, 0.f}; - const int kv_end = min(T, q0 + BM); + // Causal blocks stop at their last query. A bidirectional prompt reads every key. + const int kv_end = Bidirectional ? T : min(T, q0 + BM); for (int k0 = 0; k0 < kv_end; k0 += BN) { for (int c = tid; c < BN * (D / 8); c += THREADS) { const int r = c / (D / 8), col = (c % (D / 8)) * 8, s = k0 + r; @@ -122,8 +127,8 @@ __global__ void __launch_bounds__(THREADS) cp_async_wait<1>(); // Q and K __syncthreads(); - // keys past every query of this warp contribute nothing - const bool active = k0 <= row0 + 15; + // Causal warps skip keys past their queries. Bidirectional warps cannot. + const bool active = Bidirectional || k0 <= row0 + 15; float sc[BN / 8][4]; #pragma unroll for (int n = 0; n < BN / 8; n++) sc[n][0] = sc[n][1] = sc[n][2] = sc[n][3] = 0.f; @@ -150,7 +155,8 @@ __global__ void __launch_bounds__(THREADS) #pragma unroll for (int e = 0; e < 4; e++) { const int key = k0 + n * 8 + 2 * t + (e & 1), row = row0 + g + (e >> 1) * 8; - sc[n][e] = (key <= row && key < T) ? sc[n][e] * scale_log2 : -INFINITY; + const bool visible = Bidirectional ? (key < T) : (key <= row && key < T); + sc[n][e] = visible ? sc[n][e] * scale_log2 : -INFINITY; mx[e >> 1] = fmaxf(mx[e >> 1], sc[n][e]); } } @@ -233,20 +239,22 @@ __global__ void __launch_bounds__(THREADS) } } -template +template int launch(const void* q, const void* k, const void* v, int ldv, const void* gate, void* out, int T, int Hq, int Hk, int Dh, float scale, int q_base, void* stream) { if (Dh != D || Hk <= 0 || Hq <= 0 || Hq % Hk != 0 || ldv % 8 != 0 || ldv < Hk * Dh || T < 0) return cudaErrorInvalidValue; if (q_base < 0 || q_base % BM || q_base > T) return cudaErrorInvalidValue; + // Prefix continuation is causal. Instant reads each prompt whole from row 0. + if (Bidirectional && q_base != 0) return cudaErrorInvalidValue; if (T == q_base) return cudaSuccess; if (Gated && gate == nullptr) return cudaErrorInvalidValue; // Once per specialization (for the device current at the first call). static const cudaError_t configured = cudaFuncSetAttribute( - flash_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, SMEM_BYTES); + flash_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, SMEM_BYTES); if (configured != cudaSuccess) return configured; constexpr float LOG2E = 1.4426950408889634f; - flash_kernel<<<<(stream)>>>( static_cast(q), static_cast(k), static_cast(v), ldv, static_cast(gate), static_cast(out), T, Hq, Hk, scale * LOG2E, q_base); @@ -286,6 +294,14 @@ extern "C" int cs1_attention_gated(const void* q, const void* k, const void* v, return flash::launch(q, k, v, ldv, gate, out, T, Hq, Hk, Dh, scale, 0, stream); } +// Additive ABI 9 symbol. Full-attention prompt mask for StartLux Instant: +// query q reads key k when both are inside the prompt. GDN stays on the causal path. +extern "C" int cs1_attention_gated_bidirectional(const void* q, const void* k, const void* v, int ldv, + const void* gate, void* out, int T, int Hq, int Hk, int Dh, + float scale, void* stream) { + return flash::launch(q, k, v, ldv, gate, out, T, Hq, Hk, Dh, scale, 0, stream); +} + // Same layout contract as the full pass; only the covered query rows differ: // q_base is a multiple of 64 and rows below it keep whatever was already in k/v. extern "C" int cs1_attention_gated_prefix(const void* q, const void* k, const void* v, int ldv, const void* gate, diff --git a/src/backends/cuda/qwen3_5/ops.h b/src/backends/cuda/qwen3_5/ops.h index 1a4e6695..73224df5 100644 --- a/src/backends/cuda/qwen3_5/ops.h +++ b/src/backends/cuda/qwen3_5/ops.h @@ -113,6 +113,13 @@ int cs1_attention(const void* q, const void* k, const void* v, int ldv, void* ou int cs1_attention_gated(const void* q, const void* k, const void* v, int ldv, const void* gate, void* out, int T, int Hq, int Hk, int Dh, float scale, void* stream); +// Additive, optional at ABI 9. Same tensors as cs1_attention_gated, but every query +// reads every key in the prompt. Missing on libraries built before this entry point; +// causal callers do not resolve it. q_base-style prefix continuation is not supported. +int cs1_attention_gated_bidirectional(const void* q, const void* k, const void* v, int ldv, + const void* gate, void* out, int T, int Hq, int Hk, int Dh, + float scale, void* stream); + // Windowed variant for the cached prefix: k and v cover rows [0, T) (the cached // prefix must already be in place); q, gate and out cover rows [q_base, T) only. // q_base must be a multiple of 64; q_base = 0 reduces to cs1_attention_gated. diff --git a/src/backends/cuda/qwen3_5/qwen3_5.backend.json b/src/backends/cuda/qwen3_5/qwen3_5.backend.json index e49e1457..35e44c8e 100644 --- a/src/backends/cuda/qwen3_5/qwen3_5.backend.json +++ b/src/backends/cuda/qwen3_5/qwen3_5.backend.json @@ -28,7 +28,8 @@ "src/models/open_jev/native/", "src/models/jev_vl/native/", "src/models/decider/native/", - "src/models/jemm/native/" + "src/models/jemm/native/", + "src/models/startlux/native/" ], "build": { "script": "build.sh", diff --git a/src/models/qwen3_5/native/src/cuda.rs b/src/models/qwen3_5/native/src/cuda.rs index e47bf6ec..feac0e8f 100644 --- a/src/models/qwen3_5/native/src/cuda.rs +++ b/src/models/qwen3_5/native/src/cuda.rs @@ -278,7 +278,37 @@ impl Drop for Fp32Gdn { unsafe { (self.destroy)(self.handle) }; } } +/// `cs1_attention_gated_bidirectional`: same arguments as `cs1_attention_gated`. +pub type GatedAttention = unsafe extern "C" fn( + *const c_void, + *const c_void, + *const c_void, + c_int, + *const c_void, + *mut c_void, + c_int, + c_int, + c_int, + c_int, + f32, + Stream, +) -> c_int; + impl Api { + /// Prompt-bidirectional full attention. Absent on CUDA libraries built before + /// this additive ABI 9 symbol; causal workers never resolve it. + pub fn attention_gated_bidirectional(&self) -> Result { + // SAFETY: the signature matches ops.h. The library lives in this Api. + unsafe { + self._lib + .get::(b"cs1_attention_gated_bidirectional\0") + .map(|symbol| *symbol) + .context( + "libqwen3_5_cuda.so has no cs1_attention_gated_bidirectional; rebuild src/backends/cuda/qwen3_5 (ABI 9 additive symbol required by StartLux Instant)", + ) + } + } + pub fn fp32_gdn(&self) -> Result { // SAFETY: these optional signatures match ops.h; the library lives in static API. let (create, destroy, run, size) = unsafe { diff --git a/src/models/qwen3_5/native/src/lib.rs b/src/models/qwen3_5/native/src/lib.rs index 2e47adb9..a92d1b25 100644 --- a/src/models/qwen3_5/native/src/lib.rs +++ b/src/models/qwen3_5/native/src/lib.rs @@ -1,5 +1,5 @@ //! Qwen3.5/3.8 prefill, runtime-loaded CUDA operations, and request JSON helpers -//! shared by the Cua-S1 and Open-Jev native workers. +//! shared by the Cua-S1, Open-Jev, and StartLux Instant native workers. pub mod cuda; pub mod json; pub mod model; diff --git a/src/models/qwen3_5/native/src/model.rs b/src/models/qwen3_5/native/src/model.rs index 279bca89..f1f246d3 100644 --- a/src/models/qwen3_5/native/src/model.rs +++ b/src/models/qwen3_5/native/src/model.rs @@ -223,6 +223,35 @@ const GROUPS: &[&str] = &[ /// Upload order: by layer, and inside a layer the GROUPS members first and in order, /// so each group's matrices sit back to back and form one [sum N, K] matrix. +/// HF Qwen3.5 folders store these two GDN parameters in FP32. Every other +/// language tensor is BF16. The kernels read both as BF16. +fn fp32_gdn_param(name: &str) -> bool { + name.ends_with("linear_attn.A_log") || name.ends_with("linear_attn.norm.weight") +} + +fn bf16_payload_len(name: &str, dtype: safetensors::Dtype, nbytes: usize) -> Result { + match dtype { + safetensors::Dtype::BF16 => Ok(nbytes), + safetensors::Dtype::F32 if fp32_gdn_param(name) => { + ensure!(nbytes.is_multiple_of(4), "{name} has a short FP32 payload"); + Ok(nbytes / 2) + } + other => bail!("{name} is {other:?}, not bfloat16"), + } +} + +fn round_f32_param_to_bf16(data: &[u8]) -> Result> { + ensure!(data.len().is_multiple_of(4), "FP32 parameter length"); + let (chunks, _) = data.as_chunks::<4>(); + let mut out = Vec::with_capacity(chunks.len() * 2); + for chunk in chunks { + let value = f32::from_le_bytes(*chunk); + ensure!(value.is_finite(), "non-finite FP32 GDN parameter"); + out.extend_from_slice(&half::bf16::from_f32(value).to_le_bytes()); + } + Ok(out) +} + fn upload_order(name: &str) -> (usize, usize, String) { if let Some(tail) = name.strip_prefix("layers.") && let Some((layer, rest)) = tail.split_once('.') @@ -295,20 +324,24 @@ impl Weights { let mut total = 0usize; for (i, name) in ours { let view = sts[i].tensor(&name)?; - ensure!( - view.dtype() == safetensors::Dtype::BF16, - "{name} is {:?}, not bfloat16", - view.dtype() - ); + let bytes = bf16_payload_len(&name, view.dtype(), view.data().len())?; plan.push((i, name, total)); - total = (total + view.data().len()).next_multiple_of(ALIGN); + total = (total + bytes).next_multiple_of(ALIGN); } let arena = DeviceBuffer::new(total)?; let mut tensors = HashMap::new(); for (i, name, offset) in plan { let view = sts[i].tensor(&name)?; + // A_log and the GDN norm are FP32 in current HF Qwen3.5 folders. + // The kernels consume BF16, so round each finite value once at load. + let converted = if view.dtype() == safetensors::Dtype::F32 { + Some(round_f32_param_to_bf16(view.data())?) + } else { + None + }; + let bytes = converted.as_deref().unwrap_or_else(|| view.data()); // SAFETY: the arena has room for every planned tensor at its offset. - unsafe { cuda::upload(arena.at(offset), view.data(), stream)? }; + unsafe { cuda::upload(arena.at(offset), bytes, stream)? }; tensors.insert( name[prefix.len()..].to_string(), Tensor { @@ -1116,6 +1149,39 @@ impl Model { Ok(self.forward_batch(&[ids])?.pop().unwrap()) } + /// One prompt whose full-attention layers read the whole prompt in both + /// directions. Linear attention stays causal. CUDA graphs, LoRA and the + /// reference numerics are refused: those paths are causal-only. + pub fn forward_prompt_bidirectional(&mut self, ids: &[u32]) -> Result> { + ensure!( + !self.graph_enabled && self.reference.is_none() && self.lora.is_none(), + "prompt-bidirectional prefill is eager, merged and uses the default kernels" + ); + ensure!( + !ids.is_empty() && ids.len() <= self.cfg.max_positions, + "empty or oversized prompt" + ); + ensure!( + ids.iter().all(|&id| (id as usize) < self.embed.shape[0]), + "token id outside the vocabulary" + ); + self.prepare_scratch(ids.len())?; + let s = self.scratch.as_ref().unwrap(); + self.embed_tokens(s, ids)?; + self.run(s, &[ids.len()], false, true)?; + self.last_hidden(s, ids.len()) + } + + /// Fail at load when this library cannot run an Instant prompt. + pub fn require_prompt_bidirectional(&self) -> Result<()> { + ensure!( + !self.graph_enabled && self.reference.is_none() && self.lora.is_none(), + "prompt-bidirectional prefill is eager, merged and uses the default kernels" + ); + cuda::api().attention_gated_bidirectional()?; + Ok(()) + } + /// Pack independent text prompts for input and gate/up GEMMs. /// Mixers reset at each boundary; final-position hidden states retain input order. pub fn forward_batch(&mut self, inputs: &[&[u32]]) -> Result>> { @@ -1156,13 +1222,14 @@ impl Model { } else { // Warm plans and keep the eager result: run() advances the residual // in place, so replaying on this cache miss would advance it twice. - self.run(s, &lengths, false)?; + self.run(s, &lengths, false, false)?; cuda::synchronize(self.stream)?; - let captured = cuda::Graph::capture(self.stream, || self.run(s, &lengths, false)); + let captured = + cuda::Graph::capture(self.stream, || self.run(s, &lengths, false, false)); self.cache_graph(&lengths, false, captured)?; } } else { - self.run(s, &lengths, false)?; + self.run(s, &lengths, false, false)?; } let s = self.scratch.as_ref().unwrap(); let mut end = 0; @@ -1209,13 +1276,13 @@ impl Model { } else { // Uploads and embedding are outside capture. Keep the warmed eager // result; launching now would advance the residual a second time. - self.run(s, &[t], true)?; + self.run(s, &[t], true, false)?; cuda::synchronize(self.stream)?; - let captured = cuda::Graph::capture(self.stream, || self.run(s, &[t], true)); + let captured = cuda::Graph::capture(self.stream, || self.run(s, &[t], true, false)); self.cache_graph(&[t], true, captured)?; } } else { - self.run(s, &[t], true)?; + self.run(s, &[t], true, false)?; } self.last_hidden(self.scratch.as_ref().unwrap(), t) } @@ -1305,7 +1372,7 @@ impl Model { } begin = end; } - self.run(s, &[t], true)?; + self.run(s, &[t], true, false)?; self.last_hidden(s, t) })(); let synced = self.synchronize(); @@ -1717,8 +1784,24 @@ impl Model { } /// Queue language layers over packed embeddings, resetting sequence positions. - /// Final-norm hidden states end up in `s.x`. - fn run(&self, s: &Scratch, lengths: &[usize], custom_positions: bool) -> Result<()> { + /// Final-norm hidden states end up in `s.x`. `bidirectional` changes only the + /// full-attention mask; Gated DeltaNet stays causal and sequence-local. + fn run( + &self, + s: &Scratch, + lengths: &[usize], + custom_positions: bool, + bidirectional: bool, + ) -> Result<()> { + let bidir = if bidirectional { + ensure!( + self.reference.is_none() && self.lora.is_none(), + "prompt-bidirectional prefill does not use reference numerics or an adapter" + ); + Some(cuda::api().attention_gated_bidirectional()?) + } else { + None + }; let t: usize = lengths.iter().sum(); let mut start = 0; let sequences: Vec<(usize, i32)> = lengths @@ -1906,8 +1989,24 @@ impl Model { ), "attention prep", )?; + let scale = (cfg.head_dim as f32).powf(-0.5); check( - if let Some(reference) = self.reference { + if let Some(bidir) = bidir { + (bidir)( + p(s.aq + offset * cfg.heads * cfg.head_dim * BF16), + p(s.ak + offset * cfg.kv_heads * cfg.head_dim * BF16), + p(v + offset * w.attn_in * BF16), + ld, + p(s.agate + offset * cfg.heads * cfg.head_dim * BF16), + p(s.ao + offset * cfg.heads * cfg.head_dim * BF16), + length, + hq, + hk, + hd, + scale, + st, + ) + } else if let Some(reference) = self.reference { (reference.attention)( p(s.aq + offset * cfg.heads * cfg.head_dim * BF16), p(s.ak + offset * cfg.kv_heads * cfg.head_dim * BF16), @@ -1918,7 +2017,7 @@ impl Model { length, hq, hk, - (cfg.head_dim as f32).powf(-0.5), + scale, p(s.workspace), st, ) @@ -1934,7 +2033,7 @@ impl Model { hq, hk, hd, - (cfg.head_dim as f32).powf(-0.5), + scale, st, ) }, diff --git a/src/models/startlux/NOTICE b/src/models/startlux/NOTICE new file mode 100644 index 00000000..19b4dbce --- /dev/null +++ b/src/models/startlux/NOTICE @@ -0,0 +1,10 @@ +Prompt text, option lines, temperatures, none-check, and wide-choice +combination follow StartLuxLabs/StartLux-Decision commit +a99977d9332decc9fb27a2488b4a83b533b6cd5f (`prompt.py`, `readout.py`, +`pipeline.py`, `controls.py`). That source is Apache-2.0. + +The Instant 4B checkpoint +`startlux-models/StartLux-Decision-v1.1-4B-Instant` revision +`416088f77cfce09633906a3675c7cd2e5dabcf10` is CC BY-NC 4.0. This +repository does not distribute those weights. Commercial use of the +weights needs a separate license from the weight owner. diff --git a/src/models/startlux/README.md b/src/models/startlux/README.md new file mode 100644 index 00000000..b9623e12 --- /dev/null +++ b/src/models/startlux/README.md @@ -0,0 +1,99 @@ +# StartLux-Decision v1.1 4B Instant + +Native text worker for one pinned checkpoint. It answers `choice`, `noul`, +and `score` on `POST /v1/systemone`. AutoThink, Instant 27B, and images are +out of scope. + +The weights are **CC BY-NC 4.0**. They are not in this repository. The +adapted prompt and readout rules come from Apache-2.0 code; see +[NOTICE](NOTICE). + +| Pin | Value | +| --- | --- | +| Weights | `startlux-models/StartLux-Decision-v1.1-4B-Instant` | +| Weights revision | `416088f77cfce09633906a3675c7cd2e5dabcf10` | +| Prompt/readout source | `StartLuxLabs/StartLux-Decision` `a99977d9332decc9fb27a2488b4a83b533b6cd5f` | +| Served model id | `StartLux-Decision-v1.1-4B-Instant` | +| Decision format | `lodestar-letter-readout-v1` | +| Prompt attention | bidirectional full attention; linear attention stays causal | + +Build and serve from [the recipe](../../../recipe/startlux/README.md). The +L20X latency recipe is [latency.md](../../../recipe/startlux/latency.md). +Contract tests live in `tests/startlux/contract.rs`. + +## What this worker is + +Instant is Qwen3.5-4B with bidirectional full attention on one of every four +layers and a letter readout. It is not an Open-Jev LoRA plus a scalar head. +The worker reuses the shared Qwen3.5 loader, GDN, MLP, norms, RoPE, and +GEMMs. Full-attention layers call the additive +`cs1_attention_gated_bidirectional` symbol. Gated DeltaNet stays causal. +There is no prefix cache and no CUDA graph on this path. + +Processing (`processing.rs`) loads the tokenizer, checks that A–Z are the +pinned token ids 32..=57 immediately after the thinking-off prompt, and +compiles the request. The executor loads the checkpoint, requires the +bidirectional symbol, and returns FP32 letter logits from a CPU dot of the +tied `model.language_model.embed_tokens.weight` rows. Softmax, per-type +temperature, none-check, and wide-choice combination stay in the contract +module. One request is one `SerialScheduler` admission. Prompts inside that +admission run one at a time. There is no candidate packing. + +## Request contract + +Defaults match `startlux-decide serve`: `none_check` on, `allow_none` off, +context cap 65536 tokens (the vLLM release `max-model-len`). The model +context is 262144; `STARTLUX_MAX_MODEL_LEN` can raise the cap up to that. +Prompts over the cap return 422. Nothing is truncated. + +`think_mode` `never` and `plain` are accepted. `auto`, `always`, `hybrid`, +and `think` return 422. This folder has no `auto_think` block. Images, +`return_details`, `return_thought`, and `thought_generation: stream` return +422. Bad JSON is 400. The request `model` field is ignored; the response +model id is the pin above. `/v1/decisions` is not served. + +Choice lists keep request order. A single listed option with none-check off +skips the forward and returns probability 1 with `input_tokens` 0. Lists +that do not fit in 26 letters (25 when the extra "None of the above" option +is present) use the pinned knockout groups (25 / keep 3 / residual 0.001). +Group rounds do not include None. The final round does. Wide-choice +probability keys follow the original option order. Upstream's combined dict +is finalists then the rest; the values are the same combination, and a tie +goes to the earlier listed option. Choice entries that are not strings or +numbers are rejected. + +Score and choice probabilities are full precision. `benchmarks/bench.py` +`--score-decimals 4` accepts a score that equals the expectation of those +probabilities. Noul is P(true), shown in the prompt as yes/no. + +`input_tokens` is the sum of prompt tokens, including knockout group prompts +and the final round. `output_tokens` is 0. + +## Checkpoint load + +Language tensors are BF16 except `linear_attn.A_log` and +`linear_attn.norm.weight`, which are FP32 in this folder (48 tensors). The +existing kernels read BF16, so those two suffixes are rounded to BF16 at +upload. Any other non-BF16 language tensor still fails the load. Vision and +`mtp.*` tensors are not uploaded. The letter head is the tied embedding. +That CPU dot is not yet compared with vLLM or Transformers. + +## Status + +CPU contract tests cover prompts, option order, none-check, wide knockout, +score limits, and 422/400 rejections. GPU numerical parity and HTTP latency +are **unverified**. The build target for hk2 is Ada sm_89 / L20X. Do not +read an H200 or H800 leaderboard number as a result for this model. + +## Gaps versus Open-Jev native + +- Bidirectional full attention, not causal full attention. +- Letter logits and a temperature per question type, not one trained scalar + head and one temperature. +- No candidate packing. Prompts are serial, as in Open-Jev-9B rather than + the 27B packed path. +- No prefix cache and no CUDA graph. +- Text only. None-check adds an extra option. +- FP32 `A_log` and GDN norm are rounded to BF16. +- No GPU parity yet. AutoThink and 27B are not implemented. +- Wide-choice probability keys follow request order. diff --git a/src/models/startlux/native/Cargo.toml b/src/models/startlux/native/Cargo.toml new file mode 100644 index 00000000..7460172a --- /dev/null +++ b/src/models/startlux/native/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "omni-startlux-native" +version = "0.1.0" +edition = "2024" +publish = false +description = "Native Rust/CUDA worker for StartLux-Decision v1.1 4B Instant" + +[[bin]] +name = "omni-startlux-native" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.100" +axum = "0.8.8" +half = "2.7.1" +memmap2 = "0.9.9" +omni-qwen3-5-native = { path = "../../qwen3_5/native" } +omni-runtime = { path = "../../../runtime" } +safetensors = "0.8.0" +serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] } +tokenizers = { version = "=0.22.2", default-features = false, features = ["onig"] } +tokio = { version = "1.49.0", features = ["macros", "net", "rt-multi-thread", "sync"] } + +[[test]] +name = "contract" +path = "../../../../tests/startlux/contract.rs" diff --git a/src/models/startlux/native/src/contract.rs b/src/models/startlux/native/src/contract.rs new file mode 100644 index 00000000..9a203f94 --- /dev/null +++ b/src/models/startlux/native/src/contract.rs @@ -0,0 +1,1670 @@ +//! Instant prompt, letter readout and `/v1/systemone` answers. +//! +//! The strings and formulas follow StartLux-Decision +//! `a99977d9332decc9fb27a2488b4a83b533b6cd5f` (`prompt.py`, `readout.py`, +//! `pipeline.py`, `controls.py`). This module does not generate text. + +use serde_json::{Map, Number, Value}; + +pub const MODEL_ID: &str = "StartLux-Decision-v1.1-4B-Instant"; +pub const WEIGHTS_REPO: &str = "startlux-models/StartLux-Decision-v1.1-4B-Instant"; +pub const WEIGHTS_REVISION: &str = "416088f77cfce09633906a3675c7cd2e5dabcf10"; +pub const CODE_REVISION: &str = "a99977d9332decc9fb27a2488b4a83b533b6cd5f"; +pub const LICENSE: &str = "CC-BY-NC-4.0"; + +pub const SYSTEM: &str = "Apply the criterion to the evidence. Choose exactly one listed option. Answer with its letter only."; +pub const THINK_OFF: &str = "\n\n\n\n"; +pub const NONE_OPTION: &str = "None of the above"; +pub const NOUL_INSTRUCTIONS: &str = "Which answer fits the evidence?"; + +pub const MAX_OPTIONS: usize = 26; +pub const MAX_LEVELS: usize = 10; +pub const ANNOTATE_MIN: usize = 8; +pub const WIDE_GROUP: usize = 25; +pub const WIDE_KEEP: usize = 3; +pub const WIDE_RESIDUAL: f64 = 0.001; +pub const DEFAULT_MAX_MODEL_LEN: usize = 65_536; +pub const NATIVE_CONTEXT: usize = 262_144; + +pub const TEMP_CHOICE: f64 = 1.1103; +pub const TEMP_NOUL: f64 = 1.2662; +pub const TEMP_SCORE: f64 = 1.1344; + +/// A–Z as pinned in `decision_config.json` for this checkpoint. +pub const LETTER_IDS: [u32; 26] = [ + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, +]; + +const LETTERS: &[u8; 26] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + Choice, + Noul, + Score, +} + +impl Kind { + pub fn as_str(self) -> &'static str { + match self { + Self::Choice => "choice", + Self::Noul => "noul", + Self::Score => "score", + } + } + + pub fn temperature(self) -> f64 { + match self { + Self::Choice => TEMP_CHOICE, + Self::Noul => TEMP_NOUL, + Self::Score => TEMP_SCORE, + } + } +} + +#[derive(Debug, Clone)] +pub struct Settings { + pub max_model_len: usize, + /// `startlux-decide serve` turns this on unless `--no-none-check` is set. + pub none_check: bool, + pub allow_none: bool, +} + +impl Default for Settings { + fn default() -> Self { + Self { + max_model_len: DEFAULT_MAX_MODEL_LEN, + none_check: true, + allow_none: false, + } + } +} + +#[derive(Debug)] +pub struct RequestError { + pub status: u16, + pub message: String, + pub loc: Vec, +} + +impl RequestError { + fn new(status: u16, loc: Vec, message: impl Into) -> Self { + Self { + status, + message: message.into(), + loc, + } + } + + pub fn body(&self) -> Value { + json_map([ + ("error", Value::String(self.message.clone())), + ( + "detail", + Value::Array(vec![json_map([ + ( + "loc", + Value::Array(self.loc.iter().cloned().map(Value::String).collect()), + ), + ("msg", Value::String(self.message.clone())), + ("type", Value::String("value_error".into())), + ])]), + ), + ]) + } +} + +impl std::fmt::Display for RequestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} + +impl std::error::Error for RequestError {} + +#[derive(Debug, Clone)] +struct OptionSpec { + id: String, + criterion: Option, +} + +#[derive(Debug, Clone)] +struct Row { + kind: Kind, + state: String, + instructions: String, + options: Vec, + none_option: bool, +} + +#[derive(Debug)] +pub struct Prompt { + pub ids: Vec, + pub letters: usize, +} + +#[derive(Debug)] +enum Body { + Only { + id: String, + }, + Direct { + kind: Kind, + prompt: usize, + readout: Vec, + legend: Vec, + none: bool, + null_ok: bool, + tokens: usize, + }, + Wide { + groups: Vec>, + group_prompts: Vec, + criteria: Map, + state: String, + instructions: String, + none: bool, + null_ok: bool, + group_tokens: usize, + }, +} + +#[derive(Debug)] +struct Question { + key: String, + body: Body, +} + +#[derive(Debug)] +pub struct Prepared { + pub prompts: Vec, + questions: Vec, +} + +#[derive(Debug)] +pub struct FinalRound { + pub prompts: Vec, + slots: Vec, +} + +#[derive(Debug)] +struct FinalSlot { + question: usize, + prompt: usize, + finalists: Vec, + first: Vec<(String, f64)>, + tokens: usize, + none: bool, + null_ok: bool, +} + +pub trait PromptEncoder { + fn encode(&mut self, text: &str) -> Result, String>; +} + +impl PromptEncoder for F +where + F: FnMut(&str) -> Result, String>, +{ + fn encode(&mut self, text: &str) -> Result, String> { + self(text) + } +} + +pub fn chat_prompt(user: &str) -> String { + let user = user.trim(); + format!( + "<|im_start|>system\n{SYSTEM}<|im_end|>\n<|im_start|>user\n{user}<|im_end|>\n<|im_start|>assistant\n{THINK_OFF}" + ) +} + +pub fn probabilities(logits: &[f64], temperature: f64) -> Vec { + let max = logits.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let weights: Vec = logits + .iter() + .map(|logit| ((logit - max) / temperature).exp()) + .collect(); + let sum: f64 = weights.iter().sum(); + weights.into_iter().map(|weight| weight / sum).collect() +} + +pub fn choice_confidence(probs: &[f64]) -> f64 { + let n = probs.len(); + if n < 2 { + return 1.0; + } + let top = probs.iter().copied().fold(f64::NEG_INFINITY, f64::max); + ((n as f64 * top - 1.0) / (n as f64 - 1.0)).clamp(0.0, 1.0) +} + +pub fn score_confidence(probs: &[f64]) -> f64 { + let n = probs.len(); + let mut mode = 0; + for (i, prob) in probs.iter().enumerate().skip(1) { + if *prob > probs[mode] { + mode = i; + } + } + let spread: f64 = probs + .iter() + .enumerate() + .map(|(i, prob)| prob * (i as f64 - mode as f64).abs()) + .sum(); + let even = (0..n) + .map(|i| (i as f64 - (n as f64 - 1.0) / 2.0).abs()) + .sum::() + / n as f64; + if even == 0.0 { + 1.0 + } else { + (1.0 - spread / even).max(0.0) + } +} + +pub fn prepare_request( + raw: &[u8], + settings: &Settings, + encode: &mut dyn PromptEncoder, +) -> Result { + let body: Value = serde_json::from_slice(raw).map_err(|_| { + RequestError::new( + 400, + vec!["body".into()], + "the request body is not valid JSON", + ) + })?; + let root = body.as_object().ok_or_else(|| { + RequestError::new( + 400, + vec!["body".into()], + "the request body must be an object", + ) + })?; + reject_thinking(root)?; + reject_images(root)?; + let (none_check, allow_none) = controls(root, settings)?; + let state = root.get("state").ok_or_else(|| { + RequestError::new( + 422, + vec!["body".into(), "state".into()], + "state is required", + ) + })?; + let questions = root + .get("questions") + .and_then(Value::as_object) + .ok_or_else(|| { + RequestError::new( + 422, + vec!["body".into(), "questions".into()], + "questions must be a non-empty object", + ) + })?; + if questions.is_empty() { + return Err(RequestError::new( + 422, + vec!["body".into(), "questions".into()], + "questions must be a non-empty object", + )); + } + let mut prompts = Vec::new(); + let mut planned = Vec::new(); + for (key, spec) in questions { + let loc = vec!["body".into(), "questions".into(), key.clone()]; + let spec = spec + .as_object() + .ok_or_else(|| RequestError::new(422, loc.clone(), "a question must be an object"))?; + let question = plan_question( + key.clone(), + state, + spec, + none_check, + allow_none, + settings, + encode, + &mut prompts, + ) + .map_err(|err| { + let mut full = loc; + full.extend(err.loc); + RequestError::new(err.status, full, err.message) + })?; + planned.push(question); + } + Ok(Prepared { + prompts, + questions: planned, + }) +} + +pub fn plan_finals( + prepared: &Prepared, + logits: &[Vec], + settings: &Settings, + encode: &mut dyn PromptEncoder, +) -> Result { + ensure_logits(logits, &prepared.prompts)?; + let mut prompts = Vec::new(); + let mut slots = Vec::new(); + for (index, question) in prepared.questions.iter().enumerate() { + let Body::Wide { + groups, + group_prompts, + criteria, + state, + instructions, + none, + null_ok, + .. + } = &question.body + else { + continue; + }; + let loc = vec!["body".into(), "questions".into(), question.key.clone()]; + let reads: Vec> = group_prompts + .iter() + .map(|&prompt| probabilities(&logits[prompt], TEMP_CHOICE)) + .collect(); + if reads + .iter() + .any(|row| row.iter().any(|prob| !prob.is_finite())) + { + return Err(RequestError::new( + 422, + loc, + "non-finite choice probabilities", + )); + } + let (first, finalists) = + knockout_finalists(groups, &reads, WIDE_KEEP, MAX_OPTIONS - usize::from(*none)); + let row = choice_row(state, instructions, &finalists, criteria, *none) + .map_err(|err| RequestError::new(err.status, loc.clone(), err.message))?; + let text = chat_prompt(&user_content(&row)); + let ids = encode.encode(&text).map_err(|err| { + RequestError::new(422, loc.clone(), format!("tokenizer failed: {err}")) + })?; + if ids.is_empty() || ids.len() > settings.max_model_len || ids.len() > NATIVE_CONTEXT { + return Err(RequestError::new( + 422, + loc, + format!( + "prompt has {} tokens, above the {} token limit; this worker does not truncate", + ids.len(), + settings.max_model_len.min(NATIVE_CONTEXT) + ), + )); + } + let prompt = prompts.len(); + let tokens = ids.len(); + prompts.push(Prompt { + letters: row.options.len(), + ids, + }); + slots.push(FinalSlot { + question: index, + prompt, + finalists, + first, + tokens, + none: *none, + null_ok: *null_ok, + }); + } + Ok(FinalRound { prompts, slots }) +} + +pub fn finish( + prepared: &Prepared, + first_logits: &[Vec], + finals: &FinalRound, + final_logits: &[Vec], +) -> Result { + ensure_logits(first_logits, &prepared.prompts)?; + ensure_logits(final_logits, &finals.prompts)?; + let mut answers = Map::new(); + let mut input_tokens = 0usize; + for (index, question) in prepared.questions.iter().enumerate() { + let answer = match &question.body { + Body::Only { id } => choice_value(id, &[(id.clone(), 1.0)], None, false), + Body::Direct { + kind, + prompt, + readout, + legend, + none, + null_ok, + tokens, + } => { + input_tokens += tokens; + let probs = probabilities(&first_logits[*prompt], kind.temperature()); + typed_answer(*kind, readout, legend, &probs, *none, *null_ok, None, None)? + } + Body::Wide { group_tokens, .. } => { + input_tokens += group_tokens; + let slot = finals + .slots + .iter() + .find(|slot| slot.question == index) + .expect("wide question has a final round"); + input_tokens += slot.tokens; + let probs = probabilities(&final_logits[slot.prompt], TEMP_CHOICE); + let mut readout = slot.finalists.clone(); + if slot.none { + readout.push(NONE_OPTION.to_string()); + } + typed_answer( + Kind::Choice, + &readout, + &[], + &probs, + slot.none, + slot.null_ok, + Some(&slot.first), + Some(WIDE_RESIDUAL), + )? + } + }; + answers.insert(question.key.clone(), answer); + } + Ok(json_map([ + ("answers", Value::Object(answers)), + ( + "usage", + json_map([ + ("input_tokens", Value::from(input_tokens)), + ("output_tokens", Value::from(0)), + ]), + ), + ("model", Value::String(MODEL_ID.into())), + ])) +} + +#[allow(clippy::too_many_arguments)] +fn plan_question( + key: String, + state: &Value, + spec: &Map, + none_check: bool, + allow_none: bool, + settings: &Settings, + encode: &mut dyn PromptEncoder, + prompts: &mut Vec, +) -> Result { + let kind = question_kind(spec.get("type"))?; + let mut allow = if kind == Kind::Choice { + allow_none + } else { + false + }; + if kind == Kind::Choice + && let Some(flag) = spec.get("allow_none") + { + allow = flag.as_bool().ok_or_else(|| { + RequestError::new( + 422, + vec!["allow_none".into()], + format!("allow_none must be true or false, not {flag}"), + ) + })?; + } + let none = kind == Kind::Choice && (allow || none_check); + let null_ok = kind == Kind::Choice && allow; + if kind == Kind::Choice { + let criteria = choice_criteria(spec)?; + if criteria.len() == 1 && !none { + let id = criteria.keys().next().expect("len checked").clone(); + validate_id(&id)?; + return Ok(Question { + key, + body: Body::Only { id }, + }); + } + if criteria.len() > MAX_OPTIONS - usize::from(none) { + let order: Vec = criteria.keys().cloned().collect(); + let groups = knockout_groups(&order, WIDE_GROUP); + let state_text = state_text(state); + let instructions = required_instructions(spec)?; + let mut group_prompts = Vec::new(); + let mut group_tokens = 0; + for group in &groups { + let row = choice_row(&state_text, &instructions, group, &criteria, false)?; + let (ids, tokens) = encode_row(&row, settings, encode)?; + group_tokens += tokens; + group_prompts.push(prompts.len()); + prompts.push(Prompt { + letters: row.options.len(), + ids, + }); + } + return Ok(Question { + key, + body: Body::Wide { + groups, + group_prompts, + criteria, + state: state_text, + instructions, + none, + null_ok, + group_tokens, + }, + }); + } + } + let (row, legend) = render_row(state, spec, kind, none)?; + let (ids, tokens) = encode_row(&row, settings, encode)?; + let prompt = prompts.len(); + prompts.push(Prompt { + letters: row.options.len(), + ids, + }); + Ok(Question { + key, + body: Body::Direct { + kind, + prompt, + readout: row.options.into_iter().map(|option| option.id).collect(), + legend, + none: row.none_option, + null_ok, + tokens, + }, + }) +} + +fn encode_row( + row: &Row, + settings: &Settings, + encode: &mut dyn PromptEncoder, +) -> Result<(Vec, usize), RequestError> { + let text = chat_prompt(&user_content(row)); + let ids = encode + .encode(&text) + .map_err(|err| RequestError::new(422, vec![], format!("tokenizer failed: {err}")))?; + if ids.is_empty() || ids.len() > settings.max_model_len || ids.len() > NATIVE_CONTEXT { + return Err(RequestError::new( + 422, + vec![], + format!( + "prompt has {} tokens, above the {} token limit; this worker does not truncate", + ids.len(), + settings.max_model_len.min(NATIVE_CONTEXT) + ), + )); + } + let tokens = ids.len(); + Ok((ids, tokens)) +} + +fn render_row( + state: &Value, + spec: &Map, + kind: Kind, + none: bool, +) -> Result<(Row, Vec), RequestError> { + let state = state_text(state); + match kind { + Kind::Choice => { + let criteria = choice_criteria(spec)?; + let ids: Vec = criteria.keys().cloned().collect(); + let row = choice_row(&state, &required_instructions(spec)?, &ids, &criteria, none)?; + Ok((row, Vec::new())) + } + Kind::Noul => { + let criteria = spec + .get("criteria") + .or_else(|| spec.get("options")) + .and_then(Value::as_object); + let description = + |key: &str| criteria.and_then(|map| map.get(key)).and_then(value_text); + let mut instructions = optional_instructions(spec)?; + if instructions.is_empty() { + instructions = NOUL_INSTRUCTIONS.into(); + } + let row = Row { + kind, + state, + instructions, + options: vec![ + OptionSpec { + id: "true".into(), + criterion: description("true"), + }, + OptionSpec { + id: "false".into(), + criterion: description("false"), + }, + ], + none_option: false, + }; + validate_row(&row)?; + Ok((row, Vec::new())) + } + Kind::Score => { + let (levels, legend) = score_levels(spec)?; + if !(2..=MAX_LEVELS).contains(&levels.len()) { + return Err(RequestError::new( + 422, + vec![], + format!( + "a score question needs 2 to {MAX_LEVELS} levels, got {}", + levels.len() + ), + )); + } + let options = levels + .into_iter() + .enumerate() + .map(|(i, criterion)| OptionSpec { + id: i.to_string(), + criterion, + }) + .collect(); + let row = Row { + kind, + state, + instructions: required_instructions(spec)?, + options, + none_option: false, + }; + validate_row(&row)?; + Ok((row, legend)) + } + } +} + +fn choice_row( + state: &str, + instructions: &str, + ids: &[String], + criteria: &Map, + none: bool, +) -> Result { + if none && ids.iter().any(|id| norm(id) == norm(NONE_OPTION)) { + return Err(RequestError::new( + 422, + vec![], + format!( + "allow_none adds the option \"{NONE_OPTION}\" itself: remove the option of that name or leave allow_none out" + ), + )); + } + let mut options: Vec = ids + .iter() + .map(|id| OptionSpec { + id: id.clone(), + criterion: criteria.get(id).and_then(value_text), + }) + .collect(); + if none { + options.push(OptionSpec { + id: NONE_OPTION.into(), + criterion: None, + }); + } + let row = Row { + kind: Kind::Choice, + state: state.to_string(), + instructions: instructions.to_string(), + options, + none_option: none, + }; + validate_row(&row)?; + Ok(row) +} + +fn choice_criteria(spec: &Map) -> Result, RequestError> { + let raw = spec.get("criteria").or_else(|| spec.get("options")); + match raw { + Some(Value::Object(map)) => Ok(map.clone()), + Some(Value::Array(items)) => { + let mut map = Map::new(); + for item in items { + let id = match item { + Value::String(text) => text.clone(), + Value::Number(number) => number.to_string(), + _ => { + return Err(RequestError::new( + 422, + vec![], + "choice list entries must be strings or numbers", + )); + } + }; + map.insert(id, Value::Null); + } + Ok(map) + } + _ => Err(RequestError::new( + 422, + vec![], + "a choice question needs criteria: {option: description or null} or a list of options", + )), + } +} + +fn score_levels( + spec: &Map, +) -> Result<(Vec>, Vec), RequestError> { + let raw = spec + .get("criteria") + .or_else(|| spec.get("options")) + .ok_or_else(|| { + RequestError::new( + 422, + vec![], + "a score question needs criteria: a list of levels, lowest first", + ) + })?; + let values: Vec = match raw { + Value::Array(items) => items.clone(), + Value::Object(map) => score_keys(map)? + .into_iter() + .map(|key| map.get(&key).expect("key from map").clone()) + .collect(), + _ => { + return Err(RequestError::new( + 422, + vec![], + "a score question needs criteria: a list of levels, lowest first", + )); + } + }; + let text = values.iter().map(value_text).collect(); + Ok((text, values)) +} + +fn score_keys(map: &Map) -> Result, RequestError> { + let keys: Vec<&String> = map.keys().collect(); + let numbers: Option> = keys + .iter() + .map(|key| key.parse::().ok().filter(|value| value.is_finite())) + .collect(); + let mut order: Vec = (0..keys.len()).collect(); + if let Some(numbers) = numbers { + order.sort_by(|&left, &right| { + numbers[left] + .partial_cmp(&numbers[right]) + .unwrap_or(std::cmp::Ordering::Equal) + }); + } + Ok(order.into_iter().map(|index| keys[index].clone()).collect()) +} + +fn question_kind(value: Option<&Value>) -> Result { + let Some(value) = value else { + return Ok(Kind::Choice); + }; + let Some(text) = value.as_str() else { + return Err(RequestError::new( + 422, + vec!["type".into()], + format!("unknown question type {value}"), + )); + }; + match text { + "choice" => Ok(Kind::Choice), + "noul" | "bool" => Ok(Kind::Noul), + "score" => Ok(Kind::Score), + other => Err(RequestError::new( + 422, + vec!["type".into()], + format!("unknown question type {other:?}: use choice, noul or score"), + )), + } +} + +fn required_instructions(spec: &Map) -> Result { + let text = optional_instructions(spec)?; + if text.is_empty() { + return Err(RequestError::new( + 422, + vec![], + "instructions are required for choice and score questions", + )); + } + Ok(text) +} + +fn optional_instructions(spec: &Map) -> Result { + let raw = spec.get("instructions").or_else(|| spec.get("question")); + Ok(raw.and_then(value_text).unwrap_or_default()) +} + +fn validate_row(row: &Row) -> Result<(), RequestError> { + if row.instructions.trim().is_empty() { + return Err(RequestError::new( + 422, + vec![], + "instructions are required for choice and score questions", + )); + } + if !(2..=MAX_OPTIONS).contains(&row.options.len()) { + return Err(RequestError::new( + 422, + vec![], + format!( + "a question needs 2 to {MAX_OPTIONS} options in one pass, got {}", + row.options.len() + ), + )); + } + let mut seen = Vec::new(); + for option in &row.options { + validate_id(&option.id)?; + if seen.iter().any(|id| id == &option.id) { + return Err(RequestError::new(422, vec![], "duplicate option id")); + } + seen.push(option.id.clone()); + } + if row.kind == Kind::Noul { + let mut ids = seen.clone(); + ids.sort(); + if ids != ["false".to_string(), "true".to_string()] { + return Err(RequestError::new( + 422, + vec![], + "yes/no options must be 'true' and 'false'", + )); + } + } + if row.kind == Kind::Score && !(2..=MAX_LEVELS).contains(&row.options.len()) { + return Err(RequestError::new( + 422, + vec![], + format!( + "a score question needs 2 to {MAX_LEVELS} levels, got {}", + row.options.len() + ), + )); + } + Ok(()) +} + +fn validate_id(id: &str) -> Result<(), RequestError> { + if id.trim().is_empty() || id.contains('\n') || id.contains('\r') || id.chars().count() > 200 { + return Err(RequestError::new( + 422, + vec![], + format!( + "bad option id {id:?}: ids must be non-empty single-line strings of at most 200 characters" + ), + )); + } + Ok(()) +} + +fn user_content(row: &Row) -> String { + let state = if row.state.trim().is_empty() { + "(none)" + } else { + row.state.as_str() + }; + format!( + "Evidence:\n{state}\n\nQuestion: {}\nOptions:\n{}", + row.instructions.trim(), + option_lines(row).join("\n") + ) +} + +fn option_lines(row: &Row) -> Vec { + let extra = row.none_option.then(|| row.options.len() - 1); + let listed = match extra { + Some(index) => &row.options[..index], + None => &row.options, + }; + let hide = row.kind == Kind::Choice + && listed.iter().all(|option| is_bare(&option.id)) + && listed.iter().all(|option| { + option + .criterion + .as_ref() + .is_some_and(|text| !text.is_empty()) + }); + row.options + .iter() + .enumerate() + .map(|(index, option)| { + let name = if row.kind == Kind::Noul { + if option.id == "true" { "yes" } else { "no" } + } else { + option.id.as_str() + }; + let body = if hide && Some(index) != extra { + option.criterion.as_deref().unwrap_or(name) + } else { + match option.criterion.as_deref() { + Some(text) if !text.trim().is_empty() && norm(text) != norm(name) => { + return format!("{}) {name}: {text}", letter(index)); + } + _ => name, + } + }; + format!("{}) {body}", letter(index)) + }) + .collect() +} + +fn letter(index: usize) -> char { + LETTERS[index] as char +} + +fn is_bare(id: &str) -> bool { + let lower = id.to_ascii_lowercase(); + if let Some(rest) = lower + .strip_prefix("option") + .or_else(|| lower.strip_prefix("opt")) + { + let rest = rest.strip_prefix(['_', ' ']).unwrap_or(rest); + if (1..=3).contains(&rest.len()) && rest.bytes().all(|byte| byte.is_ascii_digit()) { + return true; + } + } + let bytes = lower.as_bytes(); + let bytes = bytes.strip_prefix(b"(").unwrap_or(bytes); + let bytes = bytes + .strip_suffix(b")") + .or_else(|| bytes.strip_suffix(b".")) + .unwrap_or(bytes); + (bytes.len() == 1 && bytes[0].is_ascii_alphabetic()) + || ((1..=3).contains(&bytes.len()) && bytes.iter().all(|byte| byte.is_ascii_digit())) +} + +fn norm(text: &str) -> String { + let mut out = String::new(); + let mut separator = false; + for ch in text.chars() { + if ch.is_whitespace() || ch == '_' || ch == '-' { + if !out.is_empty() { + separator = true; + } + } else if separator { + out.push(' '); + separator = false; + out.extend(ch.to_lowercase()); + } else { + out.extend(ch.to_lowercase()); + } + } + out +} + +#[allow(clippy::too_many_arguments)] +fn typed_answer( + kind: Kind, + readout: &[String], + legend: &[Value], + probs: &[f64], + none: bool, + null_ok: bool, + first: Option<&[(String, f64)]>, + residual: Option, +) -> Result { + if probs.len() != readout.len() || probs.iter().any(|prob| !prob.is_finite()) { + return Err(RequestError::new( + 422, + vec![], + "non-finite or mismatched option probabilities", + )); + } + match kind { + Kind::Noul => { + let index = readout.iter().position(|id| id == "true").ok_or_else(|| { + RequestError::new(422, vec![], "yes/no options must be 'true' and 'false'") + })?; + Ok(json_map([ + ("type", Value::String("noul".into())), + ("noul", json_f64(probs[index])), + ])) + } + Kind::Score => { + let score = probs + .iter() + .enumerate() + .map(|(index, prob)| index as f64 * prob) + .sum::(); + let mut probabilities = Map::new(); + let mut legend_map = Map::new(); + for (index, prob) in probs.iter().enumerate() { + let key = index.to_string(); + probabilities.insert(key.clone(), json_f64(*prob)); + let label = legend + .get(index) + .cloned() + .unwrap_or(Value::String(key.clone())); + legend_map.insert(key, label); + } + Ok(json_map([ + ("type", Value::String("score".into())), + ("score", json_f64(score)), + ("confidence", json_f64(score_confidence(probs))), + ("legend", Value::Object(legend_map)), + ("probabilities", Value::Object(probabilities)), + ])) + } + Kind::Choice => { + let (listed, none_prob, abstain) = if none { + split_none(probs) + } else { + (probs.to_vec(), None, false) + }; + let ids: Vec = if none { + readout[..readout.len() - 1].to_vec() + } else { + readout.to_vec() + }; + let combined = if let (Some(first), Some(residual)) = (first, residual) { + knockout_combine(&ids, &listed, first, residual) + } else { + ids.iter().cloned().zip(listed).collect() + }; + // Response keys follow the caller's listed order when there is no + // knock-out, and the original option order carried in `first` otherwise. + let ordered = if let Some(first) = first { + first + .iter() + .map(|(id, _)| { + let prob = combined + .iter() + .find(|(key, _)| key == id) + .map(|(_, prob)| *prob) + .unwrap_or(0.0); + (id.clone(), prob) + }) + .collect() + } else { + combined + }; + Ok(choice_value( + &argmax(&ordered), + &ordered, + none_prob, + abstain && null_ok, + )) + } + } +} + +fn choice_value(id: &str, probs: &[(String, f64)], none: Option, abstain: bool) -> Value { + let mut probabilities = Map::new(); + for (key, prob) in probs { + probabilities.insert(key.clone(), json_f64(*prob)); + } + let mut answer = Map::new(); + answer.insert("type".into(), Value::String("choice".into())); + answer.insert( + "choice".into(), + if abstain { + Value::Null + } else { + Value::String(id.into()) + }, + ); + answer.insert( + "confidence".into(), + json_f64(choice_confidence( + &probs.iter().map(|(_, prob)| *prob).collect::>(), + )), + ); + answer.insert("probabilities".into(), Value::Object(probabilities)); + if let Some(none) = none { + answer.insert("none_probability".into(), json_f64(none)); + } + Value::Object(answer) +} + +fn split_none(probs: &[f64]) -> (Vec, Option, bool) { + let (listed, none) = probs.split_at(probs.len() - 1); + let none = none[0]; + let total: f64 = listed.iter().sum(); + let renormalised = if total > 0.0 { + listed.iter().map(|prob| prob / total).collect() + } else { + vec![1.0 / listed.len() as f64; listed.len()] + }; + let top = listed.iter().copied().fold(f64::NEG_INFINITY, f64::max); + (renormalised, Some(none), none > top) +} + +fn knockout_groups(keys: &[String], group: usize) -> Vec> { + let groups = keys.len().div_ceil(group); + let (size, extra) = (keys.len() / groups, keys.len() % groups); + let mut out = Vec::new(); + let mut start = 0; + for index in 0..groups { + let end = start + size + usize::from(index < extra); + out.push(keys[start..end].to_vec()); + start = end; + } + out +} + +fn knockout_finalists( + groups: &[Vec], + reads: &[Vec], + keep: usize, + max_options: usize, +) -> (Vec<(String, f64)>, Vec) { + let mut first = Vec::new(); + let mut finalists = Vec::new(); + for (group, probs) in groups.iter().zip(reads) { + for (id, prob) in group.iter().zip(probs) { + first.push((id.clone(), *prob)); + } + let mut ranked: Vec = (0..group.len()).collect(); + ranked.sort_by(|&left, &right| { + probs[right] + .partial_cmp(&probs[left]) + .unwrap_or(std::cmp::Ordering::Equal) + }); + for index in ranked.into_iter().take(keep) { + finalists.push(group[index].clone()); + } + } + if finalists.len() > max_options { + finalists.sort_by(|left, right| { + let lp = lookup(&first, left); + let rp = lookup(&first, right); + rp.partial_cmp(&lp).unwrap_or(std::cmp::Ordering::Equal) + }); + finalists.truncate(max_options); + } + (first, finalists) +} + +fn knockout_combine( + finalists: &[String], + final_probs: &[f64], + first: &[(String, f64)], + residual: f64, +) -> Vec<(String, f64)> { + let rest: Vec<&str> = first + .iter() + .map(|(id, _)| id.as_str()) + .filter(|id| !finalists.iter().any(|finalist| finalist == id)) + .collect(); + let mass = rest.iter().map(|id| lookup(first, id)).sum::(); + let mass = if mass == 0.0 { 1.0 } else { mass }; + let mut dist: Vec<(String, f64)> = finalists + .iter() + .zip(final_probs) + .map(|(id, prob)| (id.clone(), prob * (1.0 - residual))) + .collect(); + for id in rest { + dist.push((id.to_string(), residual * lookup(first, id) / mass)); + } + let sum: f64 = dist.iter().map(|(_, prob)| *prob).sum(); + if sum > 0.0 { + for (_, prob) in &mut dist { + *prob /= sum; + } + } + dist +} + +fn lookup(pairs: &[(String, f64)], id: &str) -> f64 { + pairs + .iter() + .find(|(key, _)| key == id) + .map(|(_, prob)| *prob) + .unwrap_or(0.0) +} + +fn argmax(pairs: &[(String, f64)]) -> String { + let mut best = 0; + for (index, (_, prob)) in pairs.iter().enumerate().skip(1) { + if *prob > pairs[best].1 { + best = index; + } + } + pairs[best].0.clone() +} + +fn ensure_logits(logits: &[Vec], prompts: &[Prompt]) -> Result<(), RequestError> { + if logits.len() != prompts.len() + || logits + .iter() + .zip(prompts) + .any(|(row, prompt)| row.len() != prompt.letters) + { + return Err(RequestError::new( + 422, + vec![], + "the readout does not match the prepared prompts", + )); + } + Ok(()) +} + +fn reject_thinking(root: &Map) -> Result<(), RequestError> { + if let Some(mode) = root.get("think_mode") { + let canonical = match mode.as_str() { + Some("auto" | "hybrid") => "auto", + Some("always" | "think") => "always", + Some("never" | "plain") => "never", + _ => { + return Err(RequestError::new( + 422, + vec!["body".into(), "think_mode".into()], + "think_mode must be one of auto, never, always (or hybrid, plain, think)", + )); + } + }; + if canonical != "never" { + return Err(RequestError::new( + 422, + vec!["body".into(), "think_mode".into()], + "this is an Instant model: it answers in one pass and does not think; leave think_mode out or set it to \"never\" (thinking needs an AutoThink model)", + )); + } + } + if let Some(generation) = root.get("thought_generation") { + match generation.as_str() { + Some("auto" | "chunked") => {} + Some("stream") => { + return Err(RequestError::new( + 422, + vec!["body".into(), "thought_generation".into()], + "this engine does not stream thoughts (vLLM 0.31 or later does): leave thought_generation out or set it to \"chunked\"", + )); + } + _ => { + return Err(RequestError::new( + 422, + vec!["body".into(), "thought_generation".into()], + "thought_generation must be one of auto, stream, chunked", + )); + } + } + } + for name in ["return_details", "return_thought"] { + if let Some(value) = root.get(name) { + match value.as_bool() { + Some(false) => {} + Some(true) => { + return Err(RequestError::new( + 422, + vec!["body".into(), name.into()], + "return_details and return_thought are not served by this native worker", + )); + } + None => { + return Err(RequestError::new( + 422, + vec!["body".into(), name.into()], + format!("{name} must be true or false"), + )); + } + } + } + } + Ok(()) +} + +fn reject_images(root: &Map) -> Result<(), RequestError> { + match root.get("images") { + None | Some(Value::Null) => Ok(()), + Some(Value::Array(items)) if items.is_empty() => Ok(()), + Some(Value::Array(_)) => Err(RequestError::new( + 422, + vec!["body".into(), "images".into()], + "this native worker reads text only: send the evidence without images", + )), + Some(_) => Err(RequestError::new( + 422, + vec!["body".into(), "images".into()], + "images must be a list of base64 strings or data URIs", + )), + } +} + +fn controls(root: &Map, settings: &Settings) -> Result<(bool, bool), RequestError> { + let mut none_check = settings.none_check; + let mut allow_none = settings.allow_none; + if let Some(value) = root.get("none_check") { + none_check = value.as_bool().ok_or_else(|| { + RequestError::new( + 422, + vec!["body".into(), "none_check".into()], + "none_check must be true or false", + ) + })?; + } + if let Some(value) = root.get("allow_none") { + allow_none = value.as_bool().ok_or_else(|| { + RequestError::new( + 422, + vec!["body".into(), "allow_none".into()], + "allow_none must be true or false", + ) + })?; + } + if let Some(value) = root.get("gate") { + fraction(value, "gate")?; + } + if let Some(value) = root.get("mix") { + fraction(value, "mix")?; + } + if let Some(value) = root.get("think_budget") { + let Some(budget) = value.as_u64() else { + return Err(RequestError::new( + 422, + vec!["body".into(), "think_budget".into()], + "think_budget must be an integer of at least 1", + )); + }; + if budget < 1 { + return Err(RequestError::new( + 422, + vec!["body".into(), "think_budget".into()], + "think_budget must be an integer of at least 1", + )); + } + } + Ok((none_check, allow_none)) +} + +fn fraction(value: &Value, name: &str) -> Result { + let Some(number) = value.as_f64() else { + return Err(RequestError::new( + 422, + vec!["body".into(), name.into()], + format!("{name} must be a number from 0 to 1"), + )); + }; + if !(0.0..=1.0).contains(&number) { + return Err(RequestError::new( + 422, + vec!["body".into(), name.into()], + format!("{name} must be a number from 0 to 1"), + )); + } + Ok(number) +} + +pub fn state_text(state: &Value) -> String { + match state { + Value::String(text) => text.clone(), + Value::Null => String::new(), + Value::Object(map) if map.is_empty() => String::new(), + Value::Array(items) if items.is_empty() => String::new(), + other => py_dumps(&annotate(other)), + } +} + +fn annotate(value: &Value) -> Value { + match value { + Value::Array(items) if items.len() >= ANNOTATE_MIN => Value::Array( + items + .iter() + .enumerate() + .map(|(index, item)| { + let mut object = Map::new(); + object.insert("_index".into(), Value::from(index)); + match annotate(item) { + Value::Object(inner) => { + for (key, value) in inner { + object.insert(key, value); + } + } + other => { + object.insert("value".into(), other); + } + } + Value::Object(object) + }) + .collect(), + ), + Value::Array(items) => Value::Array(items.iter().map(annotate).collect()), + Value::Object(map) => { + let mut object = Map::new(); + for (key, value) in map { + object.insert(key.clone(), annotate(value)); + } + Value::Object(object) + } + other => other.clone(), + } +} + +fn py_dumps(value: &Value) -> String { + let mut out = String::new(); + write_py(&mut out, value); + out +} + +fn write_py(out: &mut String, value: &Value) { + match value { + Value::Null => out.push_str("null"), + Value::Bool(flag) => out.push_str(if *flag { "true" } else { "false" }), + Value::Number(number) => out.push_str(&py_number(number)), + Value::String(text) => push_py_string(out, text), + Value::Array(items) => { + out.push('['); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + write_py(out, item); + } + out.push(']'); + } + Value::Object(map) => { + out.push('{'); + for (index, (key, item)) in map.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + push_py_string(out, key); + out.push_str(": "); + write_py(out, item); + } + out.push('}'); + } + } +} + +fn py_number(number: &Number) -> String { + if let Some(value) = number.as_i64() { + return value.to_string(); + } + if let Some(value) = number.as_u64() { + return value.to_string(); + } + serde_json::to_string(number).unwrap_or_else(|_| "null".into()) +} + +fn push_py_string(out: &mut String, text: &str) { + out.push('"'); + for ch in text.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\u{0008}' => out.push_str("\\b"), + '\u{000c}' => out.push_str("\\f"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + ch if (ch as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", ch as u32)), + ch => out.push(ch), + } + } + out.push('"'); +} + +fn value_text(value: &Value) -> Option { + if value.is_null() { + return None; + } + let text = match value { + Value::String(text) => text.clone(), + other => py_dumps(other), + }; + let text = text.trim(); + if text.is_empty() { + None + } else { + Some(text.to_string()) + } +} + +fn json_f64(value: f64) -> Value { + Number::from_f64(value) + .map(Value::Number) + .unwrap_or(Value::Null) +} + +fn json_map(pairs: [(&str, Value); N]) -> Value { + let mut map = Map::new(); + for (key, value) in pairs { + map.insert(key.to_string(), value); + } + Value::Object(map) +} + +pub fn probe_prompt() -> String { + chat_prompt("Evidence:\ns\n\nQuestion: q\nOptions:\nA) yes\nB) no") +} + +/// Refuse AutoThink, 27B and any other folder. The numeric pins are the Instant 4B +/// `config.json` / `decision_config.json` at `WEIGHTS_REVISION`. +pub fn require_instant_configs(config: &Value, decision: &Value) -> Result<(), String> { + let text = config.get("text_config").unwrap_or(config); + let arch = config + .get("architectures") + .and_then(Value::as_array) + .and_then(|items| { + items + .iter() + .any(|item| item.as_str() == Some("Qwen3_5ForConditionalGeneration")) + .then_some(()) + }); + if arch.is_none() { + return Err( + "config.json architectures must include Qwen3_5ForConditionalGeneration".into(), + ); + } + if config.get("tie_word_embeddings").and_then(Value::as_bool) != Some(true) { + return Err( + "Instant 4B ties the letter head to embed_tokens; tie_word_embeddings must be true" + .into(), + ); + } + if config.get("jev_prompt_attention").and_then(Value::as_str) != Some("bidirectional") { + return Err("config.json jev_prompt_attention must be bidirectional; this worker does not serve causal AutoThink".into()); + } + expect_u64(text, "hidden_size", 2560)?; + expect_u64(text, "intermediate_size", 9216)?; + expect_u64(text, "num_hidden_layers", 32)?; + expect_u64(text, "num_attention_heads", 16)?; + expect_u64(text, "num_key_value_heads", 4)?; + expect_u64(text, "head_dim", 256)?; + expect_u64(text, "vocab_size", 248_320)?; + expect_u64(text, "max_position_embeddings", 262_144)?; + expect_u64(text, "linear_num_key_heads", 16)?; + expect_u64(text, "linear_num_value_heads", 32)?; + expect_u64(text, "linear_key_head_dim", 128)?; + expect_u64(text, "linear_value_head_dim", 128)?; + let layers = text + .get("layer_types") + .and_then(Value::as_array) + .ok_or("config.json text_config.layer_types is missing")?; + if layers.len() != 32 + || layers.iter().enumerate().any(|(index, layer)| { + layer.as_str() + != Some(if index % 4 == 3 { + "full_attention" + } else { + "linear_attention" + }) + }) + { + return Err( + "Instant 4B full attention is every fourth layer; this folder is not that model".into(), + ); + } + let rope = text + .get("rope_parameters") + .ok_or("config.json rope_parameters is missing")?; + if rope.get("partial_rotary_factor").and_then(Value::as_f64) != Some(0.25) { + return Err("partial_rotary_factor must be 0.25".into()); + } + if rope.get("rope_theta").and_then(Value::as_f64) != Some(10_000_000.0) { + return Err("rope_theta must be 10000000".into()); + } + let section = rope.get("mrope_section").and_then(Value::as_array); + let section_ok = section.is_some_and(|items| { + items.len() == 3 + && items[0].as_u64() == Some(11) + && items[1].as_u64() == Some(11) + && items[2].as_u64() == Some(10) + }); + if !section_ok { + return Err("mrope_section must be [11, 11, 10]".into()); + } + if decision.get("auto_think").is_some() { + return Err("decision_config.json has auto_think; this worker serves Instant only".into()); + } + if decision.get("format").and_then(Value::as_str) != Some("lodestar-letter-readout-v1") { + return Err("decision_config.json format must be lodestar-letter-readout-v1".into()); + } + if decision.get("prompt_attention").and_then(Value::as_str) != Some("bidirectional") { + return Err("decision_config.json prompt_attention must be bidirectional".into()); + } + if decision.get("max_options_per_pass").and_then(Value::as_u64) != Some(26) { + return Err("max_options_per_pass must be 26".into()); + } + let letters = decision + .get("letter_token_ids") + .and_then(Value::as_array) + .ok_or("letter_token_ids is missing")?; + if letters.len() != LETTER_IDS.len() + || letters + .iter() + .enumerate() + .any(|(index, id)| id.as_u64() != Some(u64::from(LETTER_IDS[index]))) + { + return Err("letter_token_ids must be 32..=57".into()); + } + let temps = decision + .get("temperature_by_type") + .and_then(Value::as_object) + .ok_or("temperature_by_type is missing")?; + expect_temp(temps, "choice", TEMP_CHOICE)?; + expect_temp(temps, "noul", TEMP_NOUL)?; + expect_temp(temps, "score", TEMP_SCORE)?; + let wide = decision + .get("wide_choice") + .and_then(Value::as_object) + .ok_or("wide_choice is missing")?; + if wide.get("group").and_then(Value::as_u64) != Some(25) + || wide.get("keep").and_then(Value::as_u64) != Some(3) + { + return Err("wide_choice group/keep must be 25/3".into()); + } + let residual = wide.get("residual").and_then(Value::as_f64).unwrap_or(-1.0); + if (residual - WIDE_RESIDUAL).abs() > 1e-12 { + return Err("wide_choice residual must be 0.001".into()); + } + Ok(()) +} + +fn expect_u64(value: &Value, key: &str, expected: u64) -> Result<(), String> { + if value.get(key).and_then(Value::as_u64) == Some(expected) { + Ok(()) + } else { + Err(format!( + "config.json {key} must be {expected}; this worker serves Instant 4B only" + )) + } +} + +fn expect_temp(temps: &Map, key: &str, expected: f64) -> Result<(), String> { + let value = temps.get(key).and_then(Value::as_f64).unwrap_or(f64::NAN); + if (value - expected).abs() <= 1e-9 { + Ok(()) + } else { + Err(format!("temperature_by_type.{key} must be {expected}")) + } +} diff --git a/src/models/startlux/native/src/engine.rs b/src/models/startlux/native/src/engine.rs new file mode 100644 index 00000000..b5f911ea --- /dev/null +++ b/src/models/startlux/native/src/engine.rs @@ -0,0 +1,92 @@ +//! One Instant folder: processor, serial admission, bidirectional executor. + +use std::path::Path; + +use anyhow::{Context, Result, bail}; +use omni_runtime::SerialScheduler; +use serde_json::{Value, json}; + +use crate::contract::{self, Settings}; +use crate::executor::Executor; +use crate::processing::Processor; + +pub struct Engine { + processor: Processor, + scheduler: SerialScheduler, + executor: Executor, +} + +impl Engine { + pub async fn load(dir: &Path, library: &Path, settings: Settings) -> Result { + let config = read_json(&dir.join("config.json"))?; + let decision = read_json(&dir.join("decision_config.json"))?; + contract::require_instant_configs(&config, &decision) + .map_err(|err| anyhow::anyhow!("{err}"))?; + let processor = Processor::load(dir, settings)?; + let executor = Executor::load(dir, library).await?; + Ok(Self { + processor, + scheduler: SerialScheduler::default(), + executor, + }) + } + + /// Prepare on the caller, then run every prompt of the request under one admission. + pub async fn decide(&self, raw: &[u8]) -> Result { + let prepared = self.processor.prepare(raw)?; + let settings = self.processor.settings().clone(); + let mut encoder = self.processor.encoder(); + let executor = self.executor.clone(); + self.scheduler + .run(move || { + let first = executor.logits(&prepared.prompts)?; + let finals = contract::plan_finals(&prepared, &first, &settings, &mut encoder) + .map_err(anyhow::Error::from)?; + let second = executor.logits(&finals.prompts)?; + contract::finish(&prepared, &first, &finals, &second).map_err(anyhow::Error::from) + }) + .await + } +} + +pub fn settings_from_env() -> Result { + let mut settings = Settings::default(); + if let Ok(value) = std::env::var("STARTLUX_MAX_MODEL_LEN") { + let parsed: usize = value.parse().context("STARTLUX_MAX_MODEL_LEN")?; + if !(1..=contract::NATIVE_CONTEXT).contains(&parsed) { + bail!("STARTLUX_MAX_MODEL_LEN must be 1..=262144"); + } + settings.max_model_len = parsed; + } + if let Ok(value) = std::env::var("STARTLUX_NONE_CHECK") { + settings.none_check = flag(&value, "STARTLUX_NONE_CHECK")?; + } + if let Ok(value) = std::env::var("STARTLUX_ALLOW_NONE") { + settings.allow_none = flag(&value, "STARTLUX_ALLOW_NONE")?; + } + Ok(settings) +} + +fn flag(value: &str, name: &str) -> Result { + match value { + "1" | "true" | "TRUE" => Ok(true), + "0" | "false" | "FALSE" => Ok(false), + _ => bail!("{name} must be 0 or 1"), + } +} + +fn read_json(path: &Path) -> Result { + let bytes = std::fs::read(path).with_context(|| format!("reading {}", path.display()))?; + serde_json::from_slice(&bytes).with_context(|| format!("parsing {}", path.display())) +} + +pub fn health() -> Value { + json!({ + "status": "ready", + "model": contract::MODEL_ID, + "prompt_attention": "bidirectional", + "license": contract::LICENSE, + "weights": contract::WEIGHTS_REPO, + "revision": contract::WEIGHTS_REVISION, + }) +} diff --git a/src/models/startlux/native/src/executor.rs b/src/models/startlux/native/src/executor.rs new file mode 100644 index 00000000..d8aacb87 --- /dev/null +++ b/src/models/startlux/native/src/executor.rs @@ -0,0 +1,155 @@ +//! Bidirectional Qwen prefill and the tied letter rows. No request semantics. + +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use anyhow::{Context, Result, ensure}; +use omni_qwen3_5_native::model::Model; +use serde_json::Value as Json; + +use crate::contract::{self, Prompt}; + +const EMBEDDING: &str = "model.language_model.embed_tokens.weight"; +const HIDDEN: usize = 2560; +const VOCAB: usize = 248_320; + +#[derive(Clone)] +pub struct Executor { + model: Arc>, + letters: Arc>, +} + +impl Executor { + pub async fn load(dir: &Path, library: &Path) -> Result { + let (folder, lib) = (dir.to_path_buf(), library.to_path_buf()); + let model = tokio::task::spawn_blocking(move || { + let model = Model::load_with_graph(&folder, &lib, false)?; + model.require_prompt_bidirectional()?; + pin_model(&model)?; + Ok::<_, anyhow::Error>(model) + }) + .await??; + let letters = Arc::new(letter_rows(dir)?); + Ok(Self { + model: Arc::new(Mutex::new(model)), + letters, + }) + } + + /// Last-position letter logits, one row per prompt, in prompt order. + /// The caller admits the whole request once; this does not admit again. + pub fn logits(&self, prompts: &[Prompt]) -> Result>> { + if prompts.is_empty() { + return Ok(Vec::new()); + } + let mut model = self.model.lock().map_err(|_| anyhow::anyhow!("poisoned"))?; + let mut rows = Vec::with_capacity(prompts.len()); + for prompt in prompts { + let last = model.forward_prompt_bidirectional(&prompt.ids)?; + rows.push(letter_logits(&self.letters, &last, prompt.letters)?); + } + Ok(rows) + } +} + +fn pin_model(model: &Model) -> Result<()> { + let cfg = &model.cfg; + ensure!(cfg.hidden == HIDDEN, "hidden size is not Instant 4B"); + ensure!( + cfg.intermediate == 9216, + "intermediate size is not Instant 4B" + ); + ensure!( + cfg.full_attention.len() == 32 + && cfg + .full_attention + .iter() + .enumerate() + .all(|(index, full)| *full == (index % 4 == 3)), + "layer pattern is not Instant 4B" + ); + ensure!( + cfg.heads == 16 && cfg.kv_heads == 4 && cfg.head_dim == 256, + "attention shape is not Instant 4B" + ); + ensure!( + cfg.lin_k_heads == 16 + && cfg.lin_v_heads == 32 + && cfg.lin_k_dim == 128 + && cfg.lin_v_dim == 128, + "linear-attention shape is not Instant 4B" + ); + ensure!( + cfg.max_positions == contract::NATIVE_CONTEXT, + "native context is not 262144" + ); + ensure!(cfg.rotary_half == 32, "rotary half is not 32"); + ensure!( + cfg.mrope_section == [11, 11, 10], + "mrope section is not Instant 4B" + ); + ensure!( + cfg.rope_theta == 10_000_000.0, + "rope theta is not Instant 4B" + ); + Ok(()) +} + +fn letter_logits(letters: &[f32], last: &[f32], n_options: usize) -> Result> { + ensure!(last.len() == HIDDEN, "last hidden state is not {HIDDEN}"); + ensure!(n_options <= contract::MAX_OPTIONS, "more than 26 letters"); + let (rows, _) = letters.as_chunks::(); + let logits: Vec = rows + .iter() + .take(n_options) + .map(|row| { + let dot = row + .iter() + .zip(last) + .map(|(&weight, &hidden)| f64::from(weight) * f64::from(hidden)) + .sum::() as f32; + f64::from(dot) + }) + .collect(); + ensure!(logits.len() == n_options, "letter head is short"); + ensure!( + logits.iter().all(|logit| logit.is_finite()), + "non-finite letter logit" + ); + Ok(logits) +} + +fn letter_rows(dir: &Path) -> Result> { + let index: Json = serde_json::from_str(&std::fs::read_to_string( + dir.join("model.safetensors.index.json"), + )?)?; + let file = index + .get("weight_map") + .and_then(|map| map.get(EMBEDDING)) + .and_then(Json::as_str) + .context(EMBEDDING)?; + let file = std::fs::File::open(dir.join(file))?; + // SAFETY: the checkpoint is not modified while the worker runs. + let mmap = unsafe { memmap2::Mmap::map(&file)? }; + let tensors = safetensors::SafeTensors::deserialize(&mmap)?; + let view = tensors.tensor(EMBEDDING)?; + ensure!( + view.dtype() == safetensors::Dtype::BF16 && view.shape() == [VOCAB, HIDDEN], + "{EMBEDDING}: {:?} {:?}", + view.dtype(), + view.shape() + ); + let mut rows = Vec::with_capacity(contract::LETTER_IDS.len() * HIDDEN); + for &id in &contract::LETTER_IDS { + let start = id as usize * HIDDEN * 2; + let row = &view.data()[start..start + HIDDEN * 2]; + let (pairs, rest) = row.as_chunks::<2>(); + ensure!(rest.is_empty(), "short embedding row"); + rows.extend( + pairs + .iter() + .map(|&bytes| half::bf16::from_le_bytes(bytes).to_f32()), + ); + } + Ok(rows) +} diff --git a/src/models/startlux/native/src/lib.rs b/src/models/startlux/native/src/lib.rs new file mode 100644 index 00000000..0db23bd3 --- /dev/null +++ b/src/models/startlux/native/src/lib.rs @@ -0,0 +1,8 @@ +//! StartLux-Decision v1.1 4B Instant: text decisions on the shared Qwen3.5 prefill. +//! Full-attention layers read each prompt bidirectionally. Linear attention stays causal. +//! Weights are CC BY-NC 4.0 and are not distributed here. + +pub mod contract; +pub mod engine; +pub mod executor; +pub mod processing; diff --git a/src/models/startlux/native/src/main.rs b/src/models/startlux/native/src/main.rs new file mode 100644 index 00000000..18df1683 --- /dev/null +++ b/src/models/startlux/native/src/main.rs @@ -0,0 +1,117 @@ +//! STARTLUX_MODEL= omni-startlux-native +//! +//! Weights are CC BY-NC 4.0 and are not in this repository. + +use std::sync::Arc; + +use anyhow::{Context, Result, ensure}; +use axum::{ + Json, Router, + body::Bytes, + extract::{DefaultBodyLimit, State, rejection::BytesRejection}, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::{get, post}, +}; +use omni_qwen3_5_native::cuda; +use omni_startlux_native::contract::RequestError; +use omni_startlux_native::engine::{self, Engine}; +use serde_json::json; + +const WARMUP: &[u8] = br#"{"state":"Dialog: service is up.","questions":{"q":{"type":"choice","instructions":"Continue.","criteria":{"ok":"OK","wait":"Wait"}}}}"#; +const BODY_LIMIT: usize = 32 << 20; + +fn contract_error(err: &RequestError) -> Response { + let status = StatusCode::from_u16(err.status).unwrap_or(StatusCode::UNPROCESSABLE_ENTITY); + (status, Json(err.body())).into_response() +} + +fn failure(err: anyhow::Error) -> Response { + if let Some(request) = err.downcast_ref::() { + return contract_error(request); + } + eprintln!("inference failed: {err:#}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": "model inference failed"})), + ) + .into_response() +} + +async fn systemone( + State(engine): State>, + headers: HeaderMap, + body: Result, +) -> Response { + let content_type = headers + .get("content-type") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .split(';') + .next() + .unwrap_or("") + .trim(); + if !content_type.eq_ignore_ascii_case("application/json") { + return ( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + Json(json!({"error": "Content-Type must be application/json"})), + ) + .into_response(); + } + let raw = match body { + Ok(raw) => raw, + Err(err) => { + return (err.status(), Json(json!({"error": err.body_text()}))).into_response(); + } + }; + match engine.decide(&raw).await { + Ok(body) => Json(body).into_response(), + Err(err) => failure(err), + } +} + +#[tokio::main] +async fn main() -> Result<()> { + let model = std::env::var_os("STARTLUX_MODEL") + .context("set STARTLUX_MODEL to the Instant 4B folder")?; + let library = std::env::var_os("STARTLUX_CUDA_LIB") + .map(Into::into) + .map_or_else(cuda::default_library, Ok)?; + let settings = engine::settings_from_env()?; + eprintln!( + "StartLux-Decision Instant 4B weights are {} ({}); this binary does not distribute them", + omni_startlux_native::contract::LICENSE, + omni_startlux_native::contract::WEIGHTS_REPO + ); + let engine = Arc::new(Engine::load(model.as_ref(), &library, settings).await?); + ensure!( + systemone_status(&engine, WARMUP).await == StatusCode::OK, + "warmup failed" + ); + let host = std::env::var("STARTLUX_HOST").unwrap_or_else(|_| "127.0.0.1".into()); + let port: u16 = std::env::var("STARTLUX_PORT") + .ok() + .map(|value| value.parse()) + .transpose() + .context("STARTLUX_PORT")? + .unwrap_or(8000); + let app = Router::new() + .route("/health", get(|| async { Json(engine::health()) })) + .route("/v1/systemone", post(systemone)) + .layer(DefaultBodyLimit::max(BODY_LIMIT)) + .with_state(engine); + let listener = tokio::net::TcpListener::bind((host.as_str(), port)).await?; + println!("listening on {host}:{port}"); + axum::serve(listener, app).await?; + Ok(()) +} + +async fn systemone_status(engine: &Engine, raw: &[u8]) -> StatusCode { + match engine.decide(raw).await { + Ok(_) => StatusCode::OK, + Err(err) => { + eprintln!("warmup failed: {err:#}"); + StatusCode::INTERNAL_SERVER_ERROR + } + } +} diff --git a/src/models/startlux/native/src/processing.rs b/src/models/startlux/native/src/processing.rs new file mode 100644 index 00000000..f03d1eb6 --- /dev/null +++ b/src/models/startlux/native/src/processing.rs @@ -0,0 +1,89 @@ +//! Tokenizer probe and request preparation. No device work. + +use std::path::Path; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use tokenizers::Tokenizer; + +use crate::contract::{self, LETTER_IDS, Prepared, RequestError, Settings, THINK_OFF}; + +pub struct Processor { + tokenizer: Tokenizer, + settings: Settings, +} + +pub struct Encoder { + tokenizer: Tokenizer, +} + +impl contract::PromptEncoder for Encoder { + fn encode(&mut self, text: &str) -> Result, String> { + encode_text(&self.tokenizer, text) + } +} + +impl Processor { + pub fn load(dir: &Path, settings: Settings) -> Result { + ensure!( + (1..=contract::NATIVE_CONTEXT).contains(&settings.max_model_len), + "STARTLUX_MAX_MODEL_LEN must be 1..=262144" + ); + let path = dir.join("tokenizer.json"); + let tokenizer = Tokenizer::from_file(&path) + .map_err(anyhow::Error::msg) + .with_context(|| format!("reading {}", path.display()))?; + probe(&tokenizer)?; + Ok(Self { + tokenizer, + settings, + }) + } + + pub fn prepare(&self, raw: &[u8]) -> Result { + let mut encoder = self.encoder(); + contract::prepare_request(raw, &self.settings, &mut encoder) + } + + pub fn encoder(&self) -> Encoder { + Encoder { + tokenizer: self.tokenizer.clone(), + } + } + + pub fn settings(&self) -> &Settings { + &self.settings + } +} + +fn encode_text(tokenizer: &Tokenizer, text: &str) -> Result, String> { + tokenizer + .encode(text, false) + .map(|encoding| encoding.get_ids().to_vec()) + .map_err(|err| err.to_string()) +} + +/// Letters A–Z are one token each and stay one token when appended to the probe. +fn probe(tokenizer: &Tokenizer) -> Result<()> { + let text = contract::probe_prompt(); + ensure!( + text.ends_with(THINK_OFF), + "the Instant prompt must end with the empty think block" + ); + let base = encode_text(tokenizer, &text).map_err(|err| anyhow!(err))?; + for (index, letter) in b"ABCDEFGHIJKLMNOPQRSTUVWXYZ".iter().enumerate() { + let letter = char::from(*letter).to_string(); + let alone = encode_text(tokenizer, &letter).map_err(|err| anyhow!(err))?; + let extended = + encode_text(tokenizer, &format!("{text}{letter}")).map_err(|err| anyhow!(err))?; + if alone.len() != 1 + || alone[0] != LETTER_IDS[index] + || extended != [&base[..], &alone[..]].concat() + { + bail!( + "the letter {letter} is not token {} immediately after the Instant prompt", + LETTER_IDS[index] + ); + } + } + Ok(()) +} diff --git a/tests/startlux/contract.rs b/tests/startlux/contract.rs new file mode 100644 index 00000000..7d5c0bd8 --- /dev/null +++ b/tests/startlux/contract.rs @@ -0,0 +1,397 @@ +use omni_startlux_native::contract::{ + self, DEFAULT_MAX_MODEL_LEN, Kind, Prepared, RequestError, Settings, WIDE_RESIDUAL, +}; +use serde_json::{Value, json}; + +fn chars(text: &str) -> Result, String> { + Ok(text.chars().map(u32::from).collect()) +} + +fn decode(ids: &[u32]) -> String { + ids.iter().map(|id| char::from_u32(*id).unwrap()).collect() +} + +fn prepare(raw: &Value, settings: &Settings) -> Result { + let bytes = serde_json::to_vec(raw).unwrap(); + let mut encode = chars; + contract::prepare_request(&bytes, settings, &mut encode) +} + +fn listed() -> Settings { + Settings { + max_model_len: DEFAULT_MAX_MODEL_LEN, + none_check: false, + allow_none: false, + } +} + +fn prompt_text(prepared: &Prepared, index: usize) -> String { + decode(&prepared.prompts[index].ids) +} + +fn zeros(prepared: &Prepared) -> Vec> { + prepared + .prompts + .iter() + .map(|prompt| vec![0.0; prompt.letters]) + .collect() +} + +#[test] +fn pinned_instant_folder_accepts_4b_and_refuses_autothink_and_27b() { + let config: Value = + serde_json::from_str(include_str!("fixtures/instant_4b_config.json")).unwrap(); + let decision: Value = + serde_json::from_str(include_str!("fixtures/decision_config.json")).unwrap(); + contract::require_instant_configs(&config, &decision).unwrap(); + + let mut autothink = decision.clone(); + autothink["auto_think"] = json!({"think_mode": "auto"}); + let err = contract::require_instant_configs(&config, &autothink).unwrap_err(); + assert!(err.contains("auto_think"), "{err}"); + + let mut bigger = config.clone(); + bigger["text_config"]["hidden_size"] = json!(5120); + let err = contract::require_instant_configs(&bigger, &decision).unwrap_err(); + assert!(err.contains("Instant 4B"), "{err}"); + + let mut causal = config.clone(); + causal["jev_prompt_attention"] = json!("causal"); + let err = contract::require_instant_configs(&causal, &decision).unwrap_err(); + assert!( + err.contains("AutoThink") || err.contains("bidirectional"), + "{err}" + ); +} + +#[test] +fn choice_noul_and_score_prompts_keep_order_and_full_precision() { + let request: Value = + serde_json::from_str(include_str!("fixtures/choice_noul_score.json")).unwrap(); + let mut request = request; + request["none_check"] = json!(false); + let prepared = prepare(&request, &Settings::default()).unwrap(); + assert_eq!(prepared.prompts.len(), 3); + + let choice = prompt_text(&prepared, 0); + let expected_choice = contract::chat_prompt( + "Evidence:\n{\"ticket\": \"I was charged twice\"}\n\nQuestion: What should happen?\nOptions:\nA) refund: Refund the charge\nB) keep: Keep the charge", + ); + assert_eq!(choice, expected_choice); + assert!(choice.ends_with(contract::THINK_OFF)); + assert!(!choice.contains("None of the above")); + + let noul = prompt_text(&prepared, 1); + assert!(noul.contains("A) yes\nB) no"), "{noul}"); + assert!(!noul.contains("true")); + + let score = prompt_text(&prepared, 2); + assert!(score.contains("A) 0: low\nB) 1: high"), "{score}"); + + let answer = contract::finish( + &prepared, + &zeros(&prepared), + &contract::plan_finals( + &prepared, + &zeros(&prepared), + &Settings::default(), + &mut chars, + ) + .unwrap(), + &[], + ) + .unwrap(); + assert_eq!(answer["model"], contract::MODEL_ID); + assert_eq!(answer["usage"]["output_tokens"], 0); + let tokens: usize = prepared.prompts.iter().map(|prompt| prompt.ids.len()).sum(); + assert_eq!(answer["usage"]["input_tokens"], tokens); + assert_eq!(answer["answers"]["route"]["choice"], "refund"); + assert_eq!( + answer["answers"]["route"]["probabilities"]["refund"].as_f64(), + Some(0.5) + ); + assert_eq!( + answer["answers"]["route"]["probabilities"]["keep"].as_f64(), + Some(0.5) + ); + assert!(answer["answers"]["route"].get("none_probability").is_none()); + assert_eq!(answer["answers"]["ask"]["type"], "noul"); + assert_eq!(answer["answers"]["ask"]["noul"].as_f64(), Some(0.5)); + assert_eq!(answer["answers"]["severity"]["score"].as_f64(), Some(0.5)); + let keys: Vec<_> = answer["answers"]["severity"]["probabilities"] + .as_object() + .unwrap() + .keys() + .cloned() + .collect(); + assert_eq!(keys, ["0".to_string(), "1".to_string()]); + assert_eq!(answer["answers"]["severity"]["legend"]["0"], "low"); +} + +#[test] +fn default_none_check_adds_the_extra_option_without_abstaining() { + let request: Value = + serde_json::from_str(include_str!("fixtures/choice_noul_score.json")).unwrap(); + let prepared = prepare(&request, &Settings::default()).unwrap(); + assert_eq!(prepared.prompts[0].letters, 3); + assert!(prompt_text(&prepared, 0).contains("C) None of the above")); + let finals = contract::plan_finals( + &prepared, + &zeros(&prepared), + &Settings::default(), + &mut chars, + ) + .unwrap(); + let answer = contract::finish(&prepared, &zeros(&prepared), &finals, &[]).unwrap(); + assert_eq!(answer["answers"]["route"]["choice"], "refund"); + assert!( + answer["answers"]["route"]["none_probability"] + .as_f64() + .unwrap() + > 0.0 + ); + let probs = answer["answers"]["route"]["probabilities"] + .as_object() + .unwrap(); + assert_eq!(probs.len(), 2); + let sum: f64 = probs.values().map(|value| value.as_f64().unwrap()).sum(); + assert!((sum - 1.0).abs() < 1e-12, "{sum}"); +} + +#[test] +fn bare_letter_ids_are_hidden_and_json_state_matches_python_spacing() { + assert_eq!( + contract::state_text(&json!({"ticket": "I was charged twice"})), + r#"{"ticket": "I was charged twice"}"# + ); + let long = json!(["a", "b", "c", "d", "e", "f", "g", "h"]); + let text = contract::state_text(&long); + assert!(text.contains(r#""_index": 0"#), "{text}"); + assert!(text.contains(", "), "{text}"); + assert!(!text.contains(r"\u")); + assert_eq!(contract::state_text(&json!("café")), "café"); + + let prepared = prepare( + &json!({ + "state": "s", + "none_check": false, + "questions": {"q": {"type": "choice", "instructions": "City?", "criteria": {"A": "Paris", "B": "London"}}} + }), + &listed(), + ) + .unwrap(); + let prompt = prompt_text(&prepared, 0); + assert!(prompt.contains("A) Paris\nB) London"), "{prompt}"); + assert!(!prompt.contains("A) A:")); +} + +#[test] +fn single_option_skips_the_forward_and_wide_lists_are_not_dropped() { + let prepared = prepare( + &json!({ + "state": "s", + "questions": {"q": {"type": "choice", "instructions": "Only one.", "criteria": {"only": "Take it"}}} + }), + &listed(), + ) + .unwrap(); + assert!(prepared.prompts.is_empty()); + let finals = contract::plan_finals(&prepared, &[], &listed(), &mut chars).unwrap(); + let answer = contract::finish(&prepared, &[], &finals, &[]).unwrap(); + assert_eq!(answer["answers"]["q"]["choice"], "only"); + assert_eq!(answer["answers"]["q"]["probabilities"]["only"], 1.0); + assert_eq!(answer["usage"]["input_tokens"], 0); + + let mut criteria = serde_json::Map::new(); + for index in 0..27 { + criteria.insert(format!("o{index:02}"), json!(format!("c{index:02}"))); + } + let prepared = prepare( + &json!({"state": "s", "questions": {"q": {"type": "choice", "instructions": "Pick.", "criteria": criteria}}}), + &Settings::default(), + ) + .unwrap(); + assert_eq!( + prepared.prompts.len(), + 2, + "27 options with none_check use two groups" + ); + assert_eq!(prepared.prompts[0].letters, 14); + assert_eq!(prepared.prompts[1].letters, 13); + assert!(!prompt_text(&prepared, 0).contains("None of the above")); + let finals = contract::plan_finals( + &prepared, + &zeros(&prepared), + &Settings::default(), + &mut chars, + ) + .unwrap(); + assert_eq!(finals.prompts.len(), 1); + assert_eq!(finals.prompts[0].letters, 7); + assert!(decode(&finals.prompts[0].ids).contains("None of the above")); + let answer = + contract::finish(&prepared, &zeros(&prepared), &finals, &zeros_final(&finals)).unwrap(); + let probs = answer["answers"]["q"]["probabilities"].as_object().unwrap(); + let keys: Vec<_> = probs.keys().cloned().collect(); + let expected: Vec<_> = (0..27).map(|index| format!("o{index:02}")).collect(); + assert_eq!(keys, expected); + assert_eq!(answer["answers"]["q"]["choice"], "o00"); + let sum: f64 = probs.values().map(|value| value.as_f64().unwrap()).sum(); + assert!((sum - 1.0).abs() < 1e-9, "{sum}"); + let group_tokens: usize = prepared.prompts.iter().map(|prompt| prompt.ids.len()).sum(); + let total = group_tokens + finals.prompts[0].ids.len(); + assert_eq!(answer["usage"]["input_tokens"], total); + assert!((WIDE_RESIDUAL - 0.001).abs() < 1e-15); +} + +fn zeros_final(finals: &contract::FinalRound) -> Vec> { + finals + .prompts + .iter() + .map(|prompt| vec![0.0; prompt.letters]) + .collect() +} + +#[test] +fn allow_none_abstains_only_when_the_extra_option_wins() { + let request = json!({ + "state": "s", + "allow_none": true, + "none_check": false, + "questions": {"q": {"type": "choice", "instructions": "Pick.", "criteria": {"a": "A", "b": "B"}}} + }); + let prepared = prepare(&request, &listed()).unwrap(); + assert_eq!(prepared.prompts[0].letters, 3); + let logits = vec![vec![0.0, 0.0, 10.0]]; + let finals = contract::plan_finals(&prepared, &logits, &listed(), &mut chars).unwrap(); + let answer = contract::finish(&prepared, &logits, &finals, &[]).unwrap(); + assert_eq!(answer["answers"]["q"]["choice"], Value::Null); + + let staying = json!({ + "state": "s", + "allow_none": false, + "none_check": true, + "questions": {"q": {"type": "choice", "instructions": "Pick.", "criteria": {"a": "A", "b": "B"}}} + }); + let prepared = prepare(&staying, &listed()).unwrap(); + let finals = contract::plan_finals(&prepared, &logits, &listed(), &mut chars).unwrap(); + let answer = contract::finish(&prepared, &logits, &finals, &[]).unwrap(); + assert_eq!(answer["answers"]["q"]["choice"], "a"); + assert!(answer["answers"]["q"]["none_probability"].as_f64().unwrap() > 0.5); +} + +#[test] +fn rejections_match_the_instant_contract() { + let bad = contract::prepare_request(b"not-json", &Settings::default(), &mut chars).unwrap_err(); + assert_eq!(bad.status, 400); + assert_eq!(bad.body()["detail"][0]["type"], "value_error"); + + let empty = prepare( + &json!({"state": "s", "questions": {}}), + &Settings::default(), + ) + .unwrap_err(); + assert_eq!(empty.status, 422); + + let missing = prepare( + &json!({"questions": {"q": {"type": "choice"}}}), + &Settings::default(), + ) + .unwrap_err(); + assert_eq!(missing.status, 422); + assert!(missing.loc.iter().any(|part| part == "state")); + + let images = prepare( + &json!({"state": "s", "images": ["abcd"], "questions": {"q": {"type": "noul"}}}), + &Settings::default(), + ) + .unwrap_err(); + assert_eq!(images.status, 422); + assert!(images.message.contains("text only"), "{}", images.message); + + let think = prepare( + &json!({"state": "s", "think_mode": "always", "questions": {"q": {"type": "noul"}}}), + &Settings::default(), + ) + .unwrap_err(); + assert_eq!(think.status, 422); + assert!(think.message.contains("AutoThink"), "{}", think.message); + + let plain = prepare( + &json!({"state": "s", "think_mode": "plain", "questions": {"q": {"type": "bool"}}}), + &Settings::default(), + ) + .unwrap(); + assert_eq!(plain.prompts.len(), 1); + + let stream = prepare( + &json!({"state": "s", "think_mode": "never", "thought_generation": "stream", "questions": {"q": {"type": "noul"}}}), + &Settings::default(), + ) + .unwrap_err(); + assert_eq!(stream.status, 422); + assert!(stream.message.contains("stream"), "{}", stream.message); + + let details = prepare( + &json!({"state": "s", "return_details": true, "questions": {"q": {"type": "noul"}}}), + &Settings::default(), + ) + .unwrap_err(); + assert_eq!(details.status, 422); + + let levels: Vec<_> = (0..11).map(|index| format!("level {index}")).collect(); + let score = prepare( + &json!({"state": "s", "questions": {"q": {"type": "score", "instructions": "Rate.", "criteria": levels}}}), + &Settings::default(), + ) + .unwrap_err(); + assert_eq!(score.status, 422); + assert!(score.message.contains("10"), "{}", score.message); + + let odd = prepare( + &json!({"state": "s", "questions": {"q": {"type": "choice", "instructions": "Pick.", "criteria": [true]}}}), + &Settings::default(), + ) + .unwrap_err(); + assert_eq!(odd.status, 422); + assert!( + odd.message.contains("strings or numbers"), + "{}", + odd.message + ); + + let tiny = Settings { + max_model_len: 8, + ..Settings::default() + }; + let limited = prepare( + &json!({"state": "s", "questions": {"q": {"type": "noul"}}}), + &tiny, + ) + .unwrap_err(); + assert_eq!(limited.status, 422); + assert!( + limited.message.contains("does not truncate"), + "{}", + limited.message + ); + + let named = prepare( + &json!({"state": "s", "questions": {"q": {"type": "choice", "instructions": "Pick.", "criteria": {"None of the above": "extra"}}}}), + &Settings::default(), + ) + .unwrap_err(); + assert_eq!(named.status, 422); +} + +#[test] +fn temperatures_and_confidence_follow_the_pinned_readout() { + let even = contract::probabilities(&[0.0, 0.0], Kind::Choice.temperature()); + assert!((even[0] - 0.5).abs() < 1e-12); + assert!((contract::choice_confidence(&[1.0, 0.0]) - 1.0).abs() < 1e-12); + assert!(contract::choice_confidence(&[0.5, 0.5]).abs() < 1e-12); + assert!((contract::score_confidence(&[1.0, 0.0]) - 1.0).abs() < 1e-12); + assert_eq!(Kind::Noul.temperature(), contract::TEMP_NOUL); + assert_eq!(contract::WEIGHTS_REVISION.len(), 40); + assert_eq!(contract::CODE_REVISION.len(), 40); +} diff --git a/tests/startlux/fixtures/choice_noul_score.json b/tests/startlux/fixtures/choice_noul_score.json new file mode 100644 index 00000000..db48f44f --- /dev/null +++ b/tests/startlux/fixtures/choice_noul_score.json @@ -0,0 +1,17 @@ +{ + "model": "ignored-alias", + "state": {"ticket": "I was charged twice"}, + "questions": { + "route": { + "type": "choice", + "instructions": "What should happen?", + "criteria": {"refund": "Refund the charge", "keep": "Keep the charge"} + }, + "ask": {"type": "noul", "instructions": "Is a refund requested?"}, + "severity": { + "type": "score", + "instructions": "How severe is it?", + "criteria": ["low", "high"] + } + } +} diff --git a/tests/startlux/fixtures/decision_config.json b/tests/startlux/fixtures/decision_config.json new file mode 100644 index 00000000..4dd7e4e3 --- /dev/null +++ b/tests/startlux/fixtures/decision_config.json @@ -0,0 +1,8 @@ +{ + "format": "lodestar-letter-readout-v1", + "letter_token_ids": [32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57], + "temperature_by_type": {"noul": 1.2662, "choice": 1.1103, "score": 1.1344}, + "max_options_per_pass": 26, + "wide_choice": {"group": 25, "keep": 3, "residual": 0.001}, + "prompt_attention": "bidirectional" +} diff --git a/tests/startlux/fixtures/instant_4b_config.json b/tests/startlux/fixtures/instant_4b_config.json new file mode 100644 index 00000000..1509e098 --- /dev/null +++ b/tests/startlux/fixtures/instant_4b_config.json @@ -0,0 +1,34 @@ +{ + "architectures": ["Qwen3_5ForConditionalGeneration"], + "tie_word_embeddings": true, + "jev_prompt_attention": "bidirectional", + "text_config": { + "hidden_size": 2560, + "intermediate_size": 9216, + "num_hidden_layers": 32, + "num_attention_heads": 16, + "num_key_value_heads": 4, + "head_dim": 256, + "vocab_size": 248320, + "max_position_embeddings": 262144, + "layer_types": [ + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention" + ], + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_key_head_dim": 128, + "linear_value_head_dim": 128, + "rope_parameters": { + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "mrope_section": [11, 11, 10] + } + } +} From 5bb25beb1c23f72497be189156d82f62ffb974da Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 24 Oct 2026 16:20:57 +0000 Subject: [PATCH 2/3] Share the Instant tokenizer across requests. Tokenizer::clone copies the BPE tables onto the bench.py HTTP clock. encode only needs a shared reference, so each request now takes an Arc clone. Device scratch is unchanged: the static 65536-token layout is about 15 GiB before weights, and that buffer was not measured on L20X. Co-authored-by: Hongsheng Liu --- recipe/startlux/README.md | 8 +-- recipe/startlux/latency.md | 56 +++++++++++++++++--- src/models/startlux/README.md | 9 ++-- src/models/startlux/native/src/processing.rs | 17 +++--- 4 files changed, 70 insertions(+), 20 deletions(-) diff --git a/recipe/startlux/README.md b/recipe/startlux/README.md index 44c1b1b7..6c6326c8 100644 --- a/recipe/startlux/README.md +++ b/recipe/startlux/README.md @@ -13,10 +13,10 @@ contain them. Inference code adapted from StartLux-Decision [NOTICE](../../src/models/startlux/NOTICE). GPU parity and latency are unverified here. The intended device is one -L20X (Ada, sm_89), not an H200. [latency.md](latency.md) is the closed-loop -measurement to run after this worker starts on that GPU. Reserve the GPU -with the host's usual scheduler and record the reservation id. This recipe -does not add scheduler flags. +L20X (Ada, sm_89). [latency.md](latency.md) is the closed-loop measurement +to run after this worker starts on that GPU, including the operator +checklist. Record the bench metadata field `reservation` for the device +hold. This recipe does not add scheduler flags. Commands below run from the repository root unless a path says otherwise. diff --git a/recipe/startlux/latency.md b/recipe/startlux/latency.md index 2f6ab867..90b56830 100644 --- a/recipe/startlux/latency.md +++ b/recipe/startlux/latency.md @@ -1,14 +1,39 @@ # StartLux Instant 4B latency on L20X -This is the measurement to run on hk2 after the native worker is serving. -No latency number is recorded in this change. Do not compare the result -with the H800 or H200 rows in +This is the measurement to run on one L20X (Ada, sm_89) after the native +worker is serving. No latency number is recorded in this change. Do not +compare the result with the H800 or H200 rows in [the Public231 leaderboard](../../benchmarks/latency-leaderboard.md): those used different hardware, clocks, and a different client clock. Label the hardware **L20X** (Ada, compute capability **sm_89**). Do not label it H200. +## Operator checklist + +From the repository root, on the L20X: + +```sh +src/backends/cuda/qwen3_5/build.sh target/release 89 +cargo build --release --locked -p omni-startlux-native -p omni-jev +``` + +Serve Instant with thinking off (omit `think_mode`, or send `never`). Leave +`STARTLUX_NONE_CHECK` unset and do not set `STARTLUX_ALLOW_NONE`. + +```sh +STARTLUX_MODEL=/path/to/StartLux-Decision-v1.1-4B-Instant \ +STARTLUX_CUDA_LIB=$PWD/target/release/libqwen3_5_cuda.so \ +STARTLUX_HOST=127.0.0.1 \ +STARTLUX_PORT=8000 \ + ./target/release/omni-startlux-native +``` + +Bench endpoint: `http://127.0.0.1:8000/v1/systemone`. Model alias: +`StartLux-Decision-v1.1-4B-Instant`. The three measured `bench.py` invocations +are in [Native arm](#native-arm). The reference arm listens on +`http://127.0.0.1:8090/v1/systemone`. + ## What is measured Use `benchmarks/bench.py`. Its clock includes the HTTP body, JSON decoding, @@ -63,7 +88,7 @@ git revision. On the reference arm also record StartLux-Decision `a99977d9332decc9fb27a2488b4a83b533b6cd5f` and `python -c 'import vllm; print(vllm.__version__)'`. Upstream documents vLLM 0.19.1 and 0.31. `cache_policy` for both arms is no prefix cache. -`reservation` is the id from the host scheduler. +`reservation` is the bench metadata field for this run's device hold. ## Native arm @@ -146,5 +171,24 @@ reference only inside this L20X campaign. ## What not to claim This file is the recipe, not a result. Until both arms finish on the -reserved L20X, p50/p95 are unknown. A feasibility run, a single traversal, -or a different GPU does not fill them in. +L20X, p50/p95 are unknown. A feasibility run, a single traversal, or a +different GPU does not fill them in. + +## Unmeasured costs + +Code review, not a profile. No millisecond claim. + +The worker shares one tokenizer. `Tokenizer::clone` would copy the BPE +tables on the bench clock (`prepare` and the knockout encoder each used to +clone). `encode` takes `&self`, so those handles are `Arc` clones. Hypothesis: +that copy was large next to a short L20X prefill. Measured delta: none. + +Device scratch grows to the next multiple of 1024 tokens on the first prompt +that exceeds the current cap, with a stream sync inside the request. Process +warmup is one short choice, and `bench.py` warms five manifest rows, so a +later longer Public231 prompt can allocate during a measured traversal. +A static count of the Instant 4B scratch layout (the byte offsets in +`Scratch::new`, including `cs1_gdn_workspace_floats`) is about 240 MiB at +1024 tokens and about 15 GiB at 65536 tokens, before the weight arena. +That 65536-token buffer is not reserved. L20X free memory was not measured. +Hypothesis: the in-request realloc can move p95. Measured delta: none. diff --git a/src/models/startlux/README.md b/src/models/startlux/README.md index b9623e12..334cf02d 100644 --- a/src/models/startlux/README.md +++ b/src/models/startlux/README.md @@ -30,9 +30,10 @@ GEMMs. Full-attention layers call the additive `cs1_attention_gated_bidirectional` symbol. Gated DeltaNet stays causal. There is no prefix cache and no CUDA graph on this path. -Processing (`processing.rs`) loads the tokenizer, checks that A–Z are the -pinned token ids 32..=57 immediately after the thinking-off prompt, and -compiles the request. The executor loads the checkpoint, requires the +Processing (`processing.rs`) loads the tokenizer once and shares it. +A request encodes through that share. +The load checks that A–Z are the pinned token ids 32..=57 immediately +after the thinking-off prompt, and compiles the request. The executor loads the checkpoint, requires the bidirectional symbol, and returns FP32 letter logits from a CPU dot of the tied `model.language_model.embed_tokens.weight` rows. Softmax, per-type temperature, none-check, and wide-choice combination stay in the contract @@ -82,7 +83,7 @@ That CPU dot is not yet compared with vLLM or Transformers. CPU contract tests cover prompts, option order, none-check, wide knockout, score limits, and 422/400 rejections. GPU numerical parity and HTTP latency -are **unverified**. The build target for hk2 is Ada sm_89 / L20X. Do not +are **unverified**. The build target is Ada sm_89 / L20X. Do not read an H200 or H800 leaderboard number as a result for this model. ## Gaps versus Open-Jev native diff --git a/src/models/startlux/native/src/processing.rs b/src/models/startlux/native/src/processing.rs index f03d1eb6..87f9124d 100644 --- a/src/models/startlux/native/src/processing.rs +++ b/src/models/startlux/native/src/processing.rs @@ -1,6 +1,7 @@ //! Tokenizer probe and request preparation. No device work. use std::path::Path; +use std::sync::Arc; use anyhow::{Context, Result, anyhow, bail, ensure}; use tokenizers::Tokenizer; @@ -8,12 +9,12 @@ use tokenizers::Tokenizer; use crate::contract::{self, LETTER_IDS, Prepared, RequestError, Settings, THINK_OFF}; pub struct Processor { - tokenizer: Tokenizer, + tokenizer: Arc, settings: Settings, } pub struct Encoder { - tokenizer: Tokenizer, + tokenizer: Arc, } impl contract::PromptEncoder for Encoder { @@ -29,9 +30,11 @@ impl Processor { "STARTLUX_MAX_MODEL_LEN must be 1..=262144" ); let path = dir.join("tokenizer.json"); - let tokenizer = Tokenizer::from_file(&path) - .map_err(anyhow::Error::msg) - .with_context(|| format!("reading {}", path.display()))?; + let tokenizer = Arc::new( + Tokenizer::from_file(&path) + .map_err(anyhow::Error::msg) + .with_context(|| format!("reading {}", path.display()))?, + ); probe(&tokenizer)?; Ok(Self { tokenizer, @@ -44,9 +47,11 @@ impl Processor { contract::prepare_request(raw, &self.settings, &mut encoder) } + /// Shared encode handle. `Tokenizer::clone` copies the BPE tables, and + /// `encode` only needs `&self`, so a request must not pay that copy. pub fn encoder(&self) -> Encoder { Encoder { - tokenizer: self.tokenizer.clone(), + tokenizer: Arc::clone(&self.tokenizer), } } From 0950a3648f7cc57a2288a1aeb018395ab0c5bc3b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 25 Oct 2026 07:05:27 +0000 Subject: [PATCH 3/3] Match Instant evidence JSON to pinned prompt.py dumps. Floats use Python repr (1e-05, 1e+16). Non-ASCII stays UTF-8 because the pinned prompt calls json.dumps with ensure_ascii=False. Contract tests lock both cases. Co-authored-by: Hongsheng Liu --- recipe/startlux/README.md | 10 ++++ src/models/startlux/README.md | 5 +- src/models/startlux/native/src/contract.rs | 69 +++------------------- tests/startlux/contract.rs | 56 ++++++++++++++++++ 4 files changed, 77 insertions(+), 63 deletions(-) diff --git a/recipe/startlux/README.md b/recipe/startlux/README.md index 6c6326c8..cd0e624a 100644 --- a/recipe/startlux/README.md +++ b/recipe/startlux/README.md @@ -105,6 +105,16 @@ With the defaults, the choice answer includes `none_probability` and the choice stays one of the listed options. `think_mode` omitted or `never` is the Instant path. `think_mode` `auto` or `always` is 422. +A string `state` is copied unchanged. Any other JSON state follows pinned +`startlux_decide/prompt.py` (`a99977d`): +`json.dumps(annotate_indices(state), ensure_ascii=False)`. Separators are +`, ` and `: `. Lists of eight or more items gain `_index`. Non-ASCII stays +UTF-8 (`café`, `東京`); Python's default `ensure_ascii=True` escapes are not +the pin. Floats follow Python `repr`, so `1e-5` is `1e-05` and `1e16` is +`1e+16`, not serde_json's `0.00001` and `1e16`. Control characters still use +`\u00XX`. `cargo test --locked -p omni-startlux-native --test contract` locks +both the exponent and the non-ASCII cases. + ## Checks without a GPU ```sh diff --git a/src/models/startlux/README.md b/src/models/startlux/README.md index 334cf02d..96e43252 100644 --- a/src/models/startlux/README.md +++ b/src/models/startlux/README.md @@ -48,7 +48,10 @@ context is 262144; `STARTLUX_MAX_MODEL_LEN` can raise the cap up to that. Prompts over the cap return 422. Nothing is truncated. `think_mode` `never` and `plain` are accepted. `auto`, `always`, `hybrid`, -and `think` return 422. This folder has no `auto_think` block. Images, +and `think` return 422. A string `state` is copied unchanged. Other JSON +states use pinned `prompt.py` `json.dumps(..., ensure_ascii=False)`: UTF-8 +for non-ASCII, and Python float `repr` (`1e-05`, `1e+16`) rather than +serde_json's default. This folder has no `auto_think` block. Images, `return_details`, `return_thought`, and `thought_generation: stream` return 422. Bad JSON is 400. The request `model` field is ignored; the response model id is the pin above. `/v1/decisions` is not served. diff --git a/src/models/startlux/native/src/contract.rs b/src/models/startlux/native/src/contract.rs index 9a203f94..ec1196d6 100644 --- a/src/models/startlux/native/src/contract.rs +++ b/src/models/startlux/native/src/contract.rs @@ -1428,69 +1428,14 @@ fn annotate(value: &Value) -> Value { } } +/// Pinned `startlux_decide/prompt.py` (`a99977d`) calls +/// `json.dumps(..., ensure_ascii=False)`. +/// +/// Non-ASCII stays UTF-8. Python's default `ensure_ascii=True` (`\u00e9`) is +/// not the pin. Floats use Python `repr` (`1e-05`, `1e+16`), not serde_json's +/// default (`0.00001`, `1e16`). Control characters still use `\u00XX`. fn py_dumps(value: &Value) -> String { - let mut out = String::new(); - write_py(&mut out, value); - out -} - -fn write_py(out: &mut String, value: &Value) { - match value { - Value::Null => out.push_str("null"), - Value::Bool(flag) => out.push_str(if *flag { "true" } else { "false" }), - Value::Number(number) => out.push_str(&py_number(number)), - Value::String(text) => push_py_string(out, text), - Value::Array(items) => { - out.push('['); - for (index, item) in items.iter().enumerate() { - if index > 0 { - out.push_str(", "); - } - write_py(out, item); - } - out.push(']'); - } - Value::Object(map) => { - out.push('{'); - for (index, (key, item)) in map.iter().enumerate() { - if index > 0 { - out.push_str(", "); - } - push_py_string(out, key); - out.push_str(": "); - write_py(out, item); - } - out.push('}'); - } - } -} - -fn py_number(number: &Number) -> String { - if let Some(value) = number.as_i64() { - return value.to_string(); - } - if let Some(value) = number.as_u64() { - return value.to_string(); - } - serde_json::to_string(number).unwrap_or_else(|_| "null".into()) -} - -fn push_py_string(out: &mut String, text: &str) { - out.push('"'); - for ch in text.chars() { - match ch { - '"' => out.push_str("\\\""), - '\\' => out.push_str("\\\\"), - '\u{0008}' => out.push_str("\\b"), - '\u{000c}' => out.push_str("\\f"), - '\n' => out.push_str("\\n"), - '\r' => out.push_str("\\r"), - '\t' => out.push_str("\\t"), - ch if (ch as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", ch as u32)), - ch => out.push(ch), - } - } - out.push('"'); + omni_qwen3_5_native::json::dumps(value) } fn value_text(value: &Value) -> Option { diff --git a/tests/startlux/contract.rs b/tests/startlux/contract.rs index 7d5c0bd8..ae7f172a 100644 --- a/tests/startlux/contract.rs +++ b/tests/startlux/contract.rs @@ -170,6 +170,62 @@ fn bare_letter_ids_are_hidden_and_json_state_matches_python_spacing() { assert!(text.contains(", "), "{text}"); assert!(!text.contains(r"\u")); assert_eq!(contract::state_text(&json!("café")), "café"); + let mixed = json!({ + "ticket": "I was charged twice", + "note": "café", + "city": "東京", + "tiny": 1e-5, + "huge": 1e16, + "edge": 1e-4, + "plain": 1e15 + }); + assert_eq!( + contract::state_text(&mixed), + r#"{"ticket": "I was charged twice", "note": "café", "city": "東京", "tiny": 1e-05, "huge": 1e+16, "edge": 0.0001, "plain": 1000000000000000.0}"# + ); + assert_eq!( + contract::state_text(&json!({"bell": "a\u{0001}b"})), + "{\"bell\": \"a\\u0001b\"}" + ); + let indexed = json!(["a", "b", "c", "d", "e", "f", "g", "café"]); + let indexed_text = contract::state_text(&indexed); + assert!( + indexed_text.contains(r#""value": "café""#), + "{indexed_text}" + ); + assert!(!indexed_text.contains('\\'), "{indexed_text}"); + + let dumped = prepare( + &json!({ + "state": mixed, + "none_check": false, + "questions": {"q": {"type": "choice", "instructions": "City?", "criteria": {"A": 1e-5, "B": "東京"}}} + }), + &listed(), + ) + .unwrap(); + let dumped_prompt = prompt_text(&dumped, 0); + assert!( + dumped_prompt.contains(r#""note": "café""#), + "{dumped_prompt}" + ); + assert!( + dumped_prompt.contains(r#""city": "東京""#), + "{dumped_prompt}" + ); + assert!( + dumped_prompt.contains(r#""tiny": 1e-05"#), + "{dumped_prompt}" + ); + assert!( + dumped_prompt.contains(r#""huge": 1e+16"#), + "{dumped_prompt}" + ); + assert!( + dumped_prompt.contains("A) 1e-05\nB) 東京"), + "{dumped_prompt}" + ); + assert!(!dumped_prompt.contains('\\'), "{dumped_prompt}"); let prepared = prepare( &json!({