From bc71fb17fc86298fb260f518126cee66646faeb0 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 15:42:59 +0000 Subject: [PATCH 01/98] Run local LLMs on the RTX 5880 Ada Two tiers, because 48 GB of VRAM turns out not to reach a better class of model than the ~30B one it already holds. `ollama` serves what fits entirely in VRAM: qwen3.6 at 27b/q8_0 is the strongest thing that size allows, and the 35b-a3b MoE activates only 3B parameters per token, so it runs roughly 3x faster for six points of Artificial Analysis index. `llama-cpp` serves DeepSeek-V4-Flash, which is 284B parameters but only ~13B active. 97% of it is expert weights, so `fit = "on"` parks those in system RAM and keeps attention and the KV cache on the GPU. That reaches well past anything VRAM-resident, at 5.2 tok/s rather than ~65. llama.cpp is pinned ahead of the b9190 in nixpkgs because DSpark speculative decoding landed in b10231, and it roughly doubles DeepSeek's throughput. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 87 +++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 2105488..2fa5a43 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -130,6 +130,93 @@ services.xserver.videoDrivers = [ "nvidia" ]; hardware.nvidia.open = true; + # Keep the driver loaded so the first request to a cold GPU isn't slow. + hardware.nvidia.nvidiaPersistenced = true; + + # The RTX 5880 Ada is compute capability 8.9; only build CUDA kernels for it. + nixpkgs.config.cudaCapabilities = [ "8.9" ]; + + # Models small enough to sit entirely in VRAM. + # https://wiki.nixos.org/wiki/Ollama + services.ollama = { + enable = true; + package = pkgs.ollama-cuda; + host = "0.0.0.0"; # So libvirt guests can reach it at 192.168.122.1. + environmentVariables = { + OLLAMA_CONTEXT_LENGTH = "32768"; # Otherwise it's tiered off total VRAM. + OLLAMA_FLASH_ATTENTION = "1"; + OLLAMA_KEEP_ALIVE = "30m"; # Don't evict a 30 GB model after five minutes. + }; + # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. + # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. + loadModels = [ + "qwen3.6:27b-mtp-q8_0" # 30 GB, index 38 — best quality that fits. + "qwen3.6:35b-a3b-q4_K_M" # 24 GB, index 32 — 3B active, ~3x the speed. + ]; + }; + + # DeepSeek-V4-Flash is 284B parameters but only ~13B active, and 97% of it is + # expert weights, so `fit = "on"` parks those in system RAM and keeps + # attention and the KV cache on the GPU. That buys a much stronger model than + # anything that fits in VRAM alone, but measures 5.2 tok/s rather than the ~65 + # the ollama models get: the CPU-side experts are bound by unpacking 3-bit + # weights, not by RAM bandwidth. Needs the sandbox VM shut down for its ~100 GB. + services.llama-cpp = { + enable = true; + package = (pkgs.llama-cpp.override { cudaSupport = true; }).overrideAttrs ( + finalAttrs: prev: { + # nixpkgs ships b9190; DSpark speculative decoding landed in b10231. + version = "10448"; + src = pkgs.fetchFromGitHub { + owner = "ggml-org"; + repo = "llama.cpp"; + tag = "b${finalAttrs.version}"; + hash = "sha256-MFfSD/lewA6k7th+sTr0a5qSOEtSG5y2Zr5lP/15XGA="; + leaveDotGit = true; + postFetch = '' + git -C "$out" rev-parse --short HEAD > $out/COMMIT + find "$out" -name .git -print0 | xargs -0 rm -rf + ''; + }; + npmDepsHash = "sha256-2Q7XhaLAArmviOLdQsNbYTfdyDE5pW9lR26cRHEVl9k="; + } + ); + host = "0.0.0.0"; + port = 8081; # 8080 is Open WebUI. + # Router mode: nothing is loaded until the first request names the alias, + # so this service starting at boot costs nothing. + modelsPreset."DeepSeek-V4-Flash" = { + hf-repo = "unsloth/DeepSeek-V4-Flash-0731-GGUF"; + hf-file = "UD-Q3_K_XL/DeepSeek-V4-Flash-0731-UD-Q3_K_XL-00001-of-00004.gguf"; + alias = "deepseek-v4-flash"; + fit = "on"; # Works out the CPU/GPU expert split itself. + # `fit` can't measure a dspark drafter's memory, so it would hand all the + # VRAM to the experts and leave the 10.4 GB draft model with nowhere to go. + fit-target = "13312"; + spec-type = "draft-dspark"; # Also fetches DeepSeek's 11 GB drafter sidecar. + ctx-size = "32768"; + jinja = "on"; + }; + }; + + # Only the VMs, not the whole LAN. + networking.firewall.interfaces."virbr0".allowedTCPPorts = [ + config.services.ollama.port + config.services.llama-cpp.port + ]; + + # Chat UI at http://localhost:8080 + services.open-webui = { + enable = true; + environment = { + ANONYMIZED_TELEMETRY = "False"; + DO_NOT_TRACK = "True"; + SCARF_NO_ANALYTICS = "True"; + OLLAMA_API_BASE_URL = "http://127.0.0.1:${toString config.services.ollama.port}"; + WEBUI_AUTH = "False"; # Single-user machine. + }; + }; + # https://wiki.nixos.org/wiki/Virt-manager#Installation virtualisation.libvirtd.enable = true; programs.virt-manager.enable = true; From 094a0aa573e879f576c90995d96bb1eca37517f6 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 18:11:17 +0000 Subject: [PATCH 02/98] Replace the DeepSeek tier with a local voice assistant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DeepSeek-V4-Flash is the wrong tool for a smart speaker: 5.2 tok/s is fine for "go think about this" and useless for "turn off the lights". It also wants ~100 GB of RAM, which fights the sandbox VM. Drop it. What replaces it is Home Assistant plus the Wyoming voice services, all of which nixpkgs already packages. The split that matters is that Home Assistant's own intent matcher handles rote commands without invoking a model at all, and only unmatched utterances reach ollama — routing everything through the LLM would be slower than the Alexa this is meant to replace. qwen3.6:35b-a3b becomes the assistant model. It activates 3B parameters per token, so it answers at conversational latency, and ollama now pins it with OLLAMA_KEEP_ALIVE=-1 rather than evicting it after 30 minutes. Pipelines, wake words and the conversation agent are chosen in the Home Assistant UI; the services here announce themselves over zeroconf. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 92 +++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 43 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 2fa5a43..1e25649 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -145,64 +145,70 @@ environmentVariables = { OLLAMA_CONTEXT_LENGTH = "32768"; # Otherwise it's tiered off total VRAM. OLLAMA_FLASH_ATTENTION = "1"; - OLLAMA_KEEP_ALIVE = "30m"; # Don't evict a 30 GB model after five minutes. + # The assistant has to stay resident; a 30 s reload before "turn off the + # lights" is the difference between usable and infuriating. + OLLAMA_KEEP_ALIVE = "-1"; }; # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. loadModels = [ + "qwen3.6:35b-a3b-q4_K_M" # 24 GB, index 32 — 3B active. The assistant. "qwen3.6:27b-mtp-q8_0" # 30 GB, index 38 — best quality that fits. - "qwen3.6:35b-a3b-q4_K_M" # 24 GB, index 32 — 3B active, ~3x the speed. ]; }; - # DeepSeek-V4-Flash is 284B parameters but only ~13B active, and 97% of it is - # expert weights, so `fit = "on"` parks those in system RAM and keeps - # attention and the KV cache on the GPU. That buys a much stronger model than - # anything that fits in VRAM alone, but measures 5.2 tok/s rather than the ~65 - # the ollama models get: the CPU-side experts are bound by unpacking 3-bit - # weights, not by RAM bandwidth. Needs the sandbox VM shut down for its ~100 GB. - services.llama-cpp = { + # Voice assistant. The wiring is deliberately split: Home Assistant's own + # intent matcher answers "turn off the kitchen light" in milliseconds without + # touching a model, and only unmatched utterances fall through to ollama. + # Routing everything through the LLM would be slower than the Alexa it's + # replacing. Pipelines and wake words are chosen in the UI, not here — these + # services announce themselves over zeroconf and Home Assistant discovers them. + services.home-assistant = { enable = true; - package = (pkgs.llama-cpp.override { cudaSupport = true; }).overrideAttrs ( - finalAttrs: prev: { - # nixpkgs ships b9190; DSpark speculative decoding landed in b10231. - version = "10448"; - src = pkgs.fetchFromGitHub { - owner = "ggml-org"; - repo = "llama.cpp"; - tag = "b${finalAttrs.version}"; - hash = "sha256-MFfSD/lewA6k7th+sTr0a5qSOEtSG5y2Zr5lP/15XGA="; - leaveDotGit = true; - postFetch = '' - git -C "$out" rev-parse --short HEAD > $out/COMMIT - find "$out" -name .git -print0 | xargs -0 rm -rf - ''; - }; - npmDepsHash = "sha256-2Q7XhaLAArmviOLdQsNbYTfdyDE5pW9lR26cRHEVl9k="; - } - ); - host = "0.0.0.0"; - port = 8081; # 8080 is Open WebUI. - # Router mode: nothing is loaded until the first request names the alias, - # so this service starting at boot costs nothing. - modelsPreset."DeepSeek-V4-Flash" = { - hf-repo = "unsloth/DeepSeek-V4-Flash-0731-GGUF"; - hf-file = "UD-Q3_K_XL/DeepSeek-V4-Flash-0731-UD-Q3_K_XL-00001-of-00004.gguf"; - alias = "deepseek-v4-flash"; - fit = "on"; # Works out the CPU/GPU expert split itself. - # `fit` can't measure a dspark drafter's memory, so it would hand all the - # VRAM to the experts and leave the 10.4 GB draft model with nowhere to go. - fit-target = "13312"; - spec-type = "draft-dspark"; # Also fetches DeepSeek's 11 GB drafter sidecar. - ctx-size = "32768"; - jinja = "on"; + openFirewall = true; # Satellites and phones live on the LAN, not virbr0. + extraComponents = [ + "assist_pipeline" + "esphome" # Voice Preview Edition and any other satellites. + "met" + "ollama" + "radio_browser" + "wyoming" + ]; + config = { + default_config = { }; + homeassistant.time_zone = config.time.timeZone; }; }; + # Speech to text. CPU by default: `device = "cuda"` needs ctranslate2 built + # with CUDA, which is a much bigger rebuild than it sounds. + services.wyoming.faster-whisper.servers.en = { + enable = true; + uri = "tcp://127.0.0.1:10300"; + language = "en"; + device = "cpu"; + }; + + # Text to speech. + services.wyoming.piper.servers.en = { + enable = true; + uri = "tcp://127.0.0.1:10200"; + voice = "en-us-ryan-medium"; + }; + + # Wake word. Models are picked per-pipeline in the UI; `preloadModels` was + # removed in wyoming-openwakeword 2.0. + services.wyoming.openwakeword = { + enable = true; + uri = "tcp://127.0.0.1:10400"; + }; + + # Only needed to build custom satellite firmware; the Voice PE works without it. + services.esphome.enable = true; + # Only the VMs, not the whole LAN. networking.firewall.interfaces."virbr0".allowedTCPPorts = [ config.services.ollama.port - config.services.llama-cpp.port ]; # Chat UI at http://localhost:8080 From 5db7bf25796624ab8b7a9fe97c49660cb1263b5e Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 18:51:09 +0000 Subject: [PATCH 03/98] Correct how Assist routes commands to the LLM assist_pipeline only handles intents locally when prefer_local_intents is set, and once the agent has control it keeps just GET_STATE and media search. The comment claimed the opposite. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 1e25649..025143d 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -157,12 +157,14 @@ ]; }; - # Voice assistant. The wiring is deliberately split: Home Assistant's own - # intent matcher answers "turn off the kitchen light" in milliseconds without - # touching a model, and only unmatched utterances fall through to ollama. - # Routing everything through the LLM would be slower than the Alexa it's - # replacing. Pipelines and wake words are chosen in the UI, not here — these - # services announce themselves over zeroconf and Home Assistant discovers them. + # Voice assistant. Note that once ollama is given "Control Home Assistant", + # every command routes through it: assist_pipeline only handles GET_STATE and + # media search locally, and only when `prefer_local_intents` is on (it is off + # by default). So the LLM is on the critical path for "turn off the kitchen + # light", not just for open-ended questions — which is why the model here is + # the 3B-active one and why keeping its prompt prefix cached matters. + # Pipelines and wake words are chosen in the UI, not here; these services + # announce themselves over zeroconf and Home Assistant discovers them. services.home-assistant = { enable = true; openFirewall = true; # Satellites and phones live on the LAN, not virbr0. From 2e5bf9538e6cb3a2968a38e281cf8ba4d1d3da9f Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 18:57:15 +0000 Subject: [PATCH 04/98] Advertise openWakeWord over zeroconf The faster-whisper and piper modules default zeroconf on and pass `--zeroconf`; the openwakeword module has no such option, so Home Assistant never discovers it. Upstream has supported the flag since October 2025 and nixpkgs ships v2.1.0, so pass it through extraArgs. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 025143d..aa38836 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -199,10 +199,13 @@ }; # Wake word. Models are picked per-pipeline in the UI; `preloadModels` was - # removed in wyoming-openwakeword 2.0. + # removed in wyoming-openwakeword 2.0. Unlike the faster-whisper and piper + # modules this one has no `zeroconf` option, so Home Assistant never discovers + # it; upstream does support the flag, so pass it directly. services.wyoming.openwakeword = { enable = true; uri = "tcp://127.0.0.1:10400"; + extraArgs = [ "--zeroconf" ]; }; # Only needed to build custom satellite firmware; the Voice PE works without it. From 0982df1390e8137ea4d7480d2b36981694c95a03 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 19:01:27 +0000 Subject: [PATCH 05/98] Fix openWakeWord zeroconf: dependency and netlink access Passing --zeroconf alone made the service fail to start. The package carries no optional-dependencies, so unlike the faster-whisper module there is nothing to splice, and wyoming.zeroconf was an ImportError; take the extra from the wyoming library instead. Its unit also omits the AF_NETLINK that piper and faster-whisper add for zeroconf, without which interface enumeration cannot work. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index aa38836..c92c429 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -205,8 +205,19 @@ services.wyoming.openwakeword = { enable = true; uri = "tcp://127.0.0.1:10400"; + # The package has no optional-dependencies to splice the way the + # faster-whisper module does, so take the extra from the wyoming library + # itself; `--zeroconf` is an ImportError without it. + package = pkgs.wyoming-openwakeword.overridePythonAttrs (old: { + dependencies = old.dependencies ++ pkgs.python3Packages.wyoming.optional-dependencies.zeroconf; + }); extraArgs = [ "--zeroconf" ]; }; + # Zeroconf enumerates network interfaces, which needs netlink. The piper and + # faster-whisper modules add this themselves when their zeroconf option is on. + systemd.services.wyoming-openwakeword.serviceConfig.RestrictAddressFamilies = [ + "AF_NETLINK" + ]; # Only needed to build custom satellite firmware; the Voice PE works without it. services.esphome.enable = true; From e9d77ec123178b2303f424746c16bbc24539ba05 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 19:25:01 +0000 Subject: [PATCH 06/98] Use the dense model for the assistant ollama 0.32.3 faults with "CUDA error: an illegal memory access was encountered" when qwen35moe does constrained decoding for tool calls whose parameters use arrays or enums, which is what Home Assistant sends. Reproduced against the API: simple one-string tools never fail, rich schemas fail intermittently, and the fault poisons the runner's CUDA context so every later request errors until it restarts. The dense qwen3.6:27b does not reproduce it at 8, 10 or 12 rich tools. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index c92c429..74900e4 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -151,9 +151,16 @@ }; # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. + # The dense model is the assistant, despite being 5x slower to generate. + # ollama 0.32.3 faults with "CUDA error: an illegal memory access was + # encountered" when qwen35moe does constrained decoding for tool calls with + # array/enum parameters — which is exactly what Home Assistant sends. It is + # intermittent, poisons the runner's CUDA context, and does not reproduce on + # the dense model. Generation speed is not the binding constraint for voice + # anyway: a 30-token reply is under half a second either way. loadModels = [ - "qwen3.6:35b-a3b-q4_K_M" # 24 GB, index 32 — 3B active. The assistant. - "qwen3.6:27b-mtp-q8_0" # 30 GB, index 38 — best quality that fits. + "qwen3.6:27b-mtp-q8_0" # 30 GB, index 38. The assistant. + "qwen3.6:35b-a3b-q4_K_M" # 24 GB, index 32 — faster, but see above. ]; }; From 1dff48ca158f0939b31cfe64f1019099a757e839 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 19:59:00 +0000 Subject: [PATCH 07/98] Give the assistant private web search The model's training data is stale, and Home Assistant exposes any script that is exposed to Assist as a callable tool, returning the script's response to the model. So a script wrapping a local SearXNG gives it lookup without sending queries to Google. SearXNG runs on loopback with its built-in HTTP server; uwsgi is for public instances. Its limiter is off because bot detection would reject Home Assistant's requests, and JSON is added to the output formats. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 58 +++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 74900e4..8c82f9f 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -186,6 +186,64 @@ config = { default_config = { }; homeassistant.time_zone = config.time.timeZone; + + # Queries the local SearXNG. Exposing the script below to Assist turns it + # into a tool the model can call, and ActionTool hands the response back, + # so this is how the assistant learns anything after its training cutoff. + rest_command.web_search = { + url = "http://127.0.0.1:${toString config.services.searx.settings.server.port}/search?q={{ query | urlencode }}&format=json"; + method = "GET"; + timeout = 20; + }; + + script.search_the_web = { + alias = "Search the web"; + # This description is the tool description the model sees. + description = "Search the web for current information. Use this for news, current events, prices, or any question whose answer may have changed since training."; + fields.query = { + description = "What to search for"; + example = "current president of the united states"; + required = true; + selector.text = { }; + }; + sequence = [ + { + action = "rest_command.web_search"; + data.query = "{{ query }}"; + response_variable = "raw"; + } + { + # A script response has to be a dict: ServiceResponse is + # JsonObjectType | None, so returning a bare list would fail. + variables.results.snippets = '' + {{ (raw.content.results | default([]))[:5] + | map(attribute='content') | select('string') | list }}''; + } + { + stop = "done"; + response_variable = "results"; + } + ]; + }; + }; + }; + + # Private metasearch, so the assistant can look things up without handing the + # query to Google. Loopback only; Home Assistant is the only client. + services.searx = { + enable = true; # Built-in HTTP server: configureUwsgi is for public instances. + environmentFile = "/var/lib/searx/secret.env"; # SEARX_SECRET_KEY=... + settings = { + server = { + bind_address = "127.0.0.1"; + port = 8888; + secret_key = "$SEARX_SECRET_KEY"; + limiter = false; # Bot detection would reject Home Assistant's requests. + }; + search.formats = [ + "html" + "json" + ]; }; }; From 0edbd0cecbedcace26f07a5d38eb2fac57fc8f2a Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 20:03:23 +0000 Subject: [PATCH 08/98] Generate the SearXNG secret key on first boot Requiring a manual step before the first rebuild is a bad interface, and the key should not sit in the world-readable Nix store. A oneshot mints one into the state directory if absent. It needs its own unit because systemd reads EnvironmentFile before any ExecStartPre of the service consuming it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 8c82f9f..e808dc1 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -230,9 +230,30 @@ # Private metasearch, so the assistant can look things up without handing the # query to Google. Loopback only; Home Assistant is the only client. + # SearXNG refuses to start on the stock secret key, and it shouldn't live in + # the world-readable Nix store, so mint one on first boot. This has to be its + # own unit: systemd reads EnvironmentFile before any ExecStartPre of the + # service that uses it. + systemd.services.searx-secret = { + wantedBy = [ "searx.service" ]; + before = [ "searx.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + StateDirectory = "searx"; + UMask = "0077"; + }; + script = '' + if [ ! -s /var/lib/searx/secret.env ]; then + printf 'SEARX_SECRET_KEY=%s\n' "$(head -c 32 /dev/urandom | base64)" \ + > /var/lib/searx/secret.env + fi + ''; + }; + services.searx = { enable = true; # Built-in HTTP server: configureUwsgi is for public instances. - environmentFile = "/var/lib/searx/secret.env"; # SEARX_SECRET_KEY=... + environmentFile = "/var/lib/searx/secret.env"; settings = { server = { bind_address = "127.0.0.1"; From 48a19d449010377aa2ae257e575f94a9b92ee308 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 20:06:44 +0000 Subject: [PATCH 09/98] Order the secret unit before searx-init, not searx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The module hands searx-init the same EnvironmentFile and searx.service requires it, so searx-init is what reads the file first — and what failed on the missing file during activation. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index e808dc1..3089e07 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -234,9 +234,12 @@ # the world-readable Nix store, so mint one on first boot. This has to be its # own unit: systemd reads EnvironmentFile before any ExecStartPre of the # service that uses it. + # Order against searx-init, not searx: the module gives searx-init the same + # EnvironmentFile and searx.service requires it, so searx-init is what reads + # the file first and what fails if it is missing. systemd.services.searx-secret = { - wantedBy = [ "searx.service" ]; - before = [ "searx.service" ]; + wantedBy = [ "searx-init.service" ]; + before = [ "searx-init.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; From 3954769465906879255257458875f43d491472ee Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 20:11:35 +0000 Subject: [PATCH 10/98] Record the working Assist system prompt It lives in Home Assistant's storage rather than in this config, so a rebuild does not reproduce it. Keeping the wording here because getting the model to reach for its tools rather than answer from memory took several attempts. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 3089e07..0dd2706 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -196,6 +196,24 @@ timeout = 20; }; + # The conversation agent's system prompt lives in Home Assistant's + # storage, not here, so it is not captured by a rebuild. The wording that + # actually works, after several that did not: + # + # Your training data is out of date and your memory of current facts is + # wrong. + # Call GetDateTimeTool for the current date or time. + # Call GetLiveContext for the state of anything in this house, + # including the weather. + # Call search_the_web for news, current events, prices, sports, or who + # currently holds any office or title. + # If you find yourself recalling a name, number or event from memory for + # such a question, that recollection is stale: search instead. + # Never say you lack access to current information. Only say you do not + # know if a tool returned nothing useful. + # + # The last two lines matter most: an earlier version told it to say it did + # not know, which made it decline rather than reach for the tool. script.search_the_web = { alias = "Search the web"; # This description is the tool description the model sees. From 5dc1244cefa58dab3ca2212eff865a0d82b2ddef Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 20:16:11 +0000 Subject: [PATCH 11/98] Replace Piper with Kokoro for text to speech Piper was archived upstream on 2025-10-06 and is the weakest part of the pipeline to listen to. Kokoro is an 82M StyleTTS2 model under Apache 2.0 with 50-odd voices and much more natural prosody. Nothing in nixpkgs speaks Wyoming for it, so this runs the community server as a container: pinned by digest rather than latest, published on loopback only like the other Wyoming services, and with the model baked into the image so there is no runtime download. Piper stays enabled for now so the two can be compared directly. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 0dd2706..5f1ae41 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -298,13 +298,32 @@ device = "cpu"; }; - # Text to speech. + # Text to speech. Piper was archived upstream on 2025-10-06 and sounds every + # bit its age; kept only to A/B against Kokoro, and worth deleting once that + # comparison is settled. services.wyoming.piper.servers.en = { enable = true; uri = "tcp://127.0.0.1:10200"; voice = "en-us-ryan-medium"; }; + # Kokoro is an 82M StyleTTS2 model under Apache 2.0 and markedly more natural + # than Piper. Nothing in nixpkgs speaks Wyoming for it, so this is a + # third-party container: pinned by digest, loopback only, model baked into the + # image so there is no runtime download. Voices are listed at + # https://huggingface.co/hexgrad/Kokoro-82M/blob/main/VOICES.md + virtualisation.oci-containers = { + backend = "docker"; + containers.kokoro-tts = { + image = "ghcr.io/relvacode/kokoro-wyoming@sha256:ff15cfb276045bd61662162d9f70c2596c1cb5acd345c3f62835b2aea397ba69"; + ports = [ "127.0.0.1:10210:10210" ]; + cmd = [ + "--voice" + "af_heart" + ]; + }; + }; + # Wake word. Models are picked per-pipeline in the UI; `preloadModels` was # removed in wyoming-openwakeword 2.0. Unlike the faster-whisper and piper # modules this one has no `zeroconf` option, so Home Assistant never discovers From a599702dc534e5bc5a0990e353e770d7b743b152 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 20:22:20 +0000 Subject: [PATCH 12/98] Fix the Kokoro container arguments The image is relvacode's fork, whose main.py accepts only --host/--port/--uri/--debug. I took --voice from the nordwestt source I had cloned, which is different code, so the container exited with "unrecognized arguments" on every start until systemd gave up. Voice selection happens per request from Home Assistant in this fork. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 5f1ae41..e41faa4 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -317,9 +317,11 @@ containers.kokoro-tts = { image = "ghcr.io/relvacode/kokoro-wyoming@sha256:ff15cfb276045bd61662162d9f70c2596c1cb5acd345c3f62835b2aea397ba69"; ports = [ "127.0.0.1:10210:10210" ]; + # This fork takes only --host/--port/--uri/--debug; the voice is chosen + # per request by Home Assistant rather than pinned on the command line. cmd = [ - "--voice" - "af_heart" + "--uri" + "tcp://0.0.0.0:10210" ]; }; }; From 30d9a5127921d25b33f528d576366991e6d9c4b2 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 20:32:46 +0000 Subject: [PATCH 13/98] Run Kokoro on the GPU CPU synthesis costs about 0.4 s of every reply, measured across realistic reply lengths. Only the nordwestt fork publishes a CUDA image, so switch to it; it reads voices from the model the same way the relvacode build does, so the voice list survives. kokoro-onnx picks its execution provider from ONNX_PROVIDER, which is how the upstream Intel compose file selects OpenVINO. The container toolkit turns on Docker's CDI support, so the device reference resolves. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index e41faa4..08338ff 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -312,13 +312,17 @@ # third-party container: pinned by digest, loopback only, model baked into the # image so there is no runtime download. Voices are listed at # https://huggingface.co/hexgrad/Kokoro-82M/blob/main/VOICES.md + # CUDA build, because synthesis on the CPU costs ~0.4 s of every reply. Only + # the nordwestt fork publishes a CUDA image; it reads voices from the model + # the same way, so Home Assistant still offers the full list. + hardware.nvidia-container-toolkit.enable = true; virtualisation.oci-containers = { backend = "docker"; containers.kokoro-tts = { - image = "ghcr.io/relvacode/kokoro-wyoming@sha256:ff15cfb276045bd61662162d9f70c2596c1cb5acd345c3f62835b2aea397ba69"; + image = "docker.io/nordwestt/kokoro-wyoming@sha256:fe2e5dd974a2e6f529b36f6d715feb81969e1addcb53931530b60174d6ab0ec3"; ports = [ "127.0.0.1:10210:10210" ]; - # This fork takes only --host/--port/--uri/--debug; the voice is chosen - # per request by Home Assistant rather than pinned on the command line. + environment.ONNX_PROVIDER = "CUDAExecutionProvider"; + extraOptions = [ "--device=nvidia.com/gpu=all" ]; cmd = [ "--uri" "tcp://0.0.0.0:10210" From aaaba4ba7924378a67903bdf8da5d0e83bc6438d Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 20:43:13 +0000 Subject: [PATCH 14/98] Revert Kokoro to CPU, and make the image a build input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The published cuda image reports 'Available providers: AzureExecutionProvider, CPUExecutionProvider' — it ships a CPU-only onnxruntime, so it never had a GPU path. nixpkgs has python3Packages.kokoro, but that is the torch build and a CUDA torch is a multi-hour compile that the cudaCapabilities pin makes uncacheable. Neither is worth the ~0.3 s. Switching to dockerTools.pullImage answers the more useful complaint: the image now downloads during the rebuild rather than afterwards on first start, and the unit drops its network-online.target dependency. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 08338ff..76756f6 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -312,17 +312,29 @@ # third-party container: pinned by digest, loopback only, model baked into the # image so there is no runtime download. Voices are listed at # https://huggingface.co/hexgrad/Kokoro-82M/blob/main/VOICES.md - # CUDA build, because synthesis on the CPU costs ~0.4 s of every reply. Only - # the nordwestt fork publishes a CUDA image; it reads voices from the model - # the same way, so Home Assistant still offers the full list. - hardware.nvidia-container-toolkit.enable = true; + # This runs on the CPU, costing ~0.4 s per reply, and that is the best trade + # available. The published `cuda` image ships a CPU-only onnxruntime, so it + # cannot use the GPU at all; nixpkgs does have python3Packages.kokoro, but it + # is the torch build, and a CUDA torch is a multi-hour compile made + # uncacheable by the cudaCapabilities pin above. Neither is worth 0.3 s. + # + # pullImage rather than a registry pull so the image is a build input: it + # downloads during the rebuild instead of afterwards on first service start, + # and the unit no longer needs network-online.target. virtualisation.oci-containers = { backend = "docker"; containers.kokoro-tts = { - image = "docker.io/nordwestt/kokoro-wyoming@sha256:fe2e5dd974a2e6f529b36f6d715feb81969e1addcb53931530b60174d6ab0ec3"; + image = "ghcr.io/relvacode/kokoro-wyoming:v2025.1.1"; + imageFile = pkgs.dockerTools.pullImage { + imageName = "ghcr.io/relvacode/kokoro-wyoming"; + imageDigest = "sha256:ff15cfb276045bd61662162d9f70c2596c1cb5acd345c3f62835b2aea397ba69"; + finalImageName = "ghcr.io/relvacode/kokoro-wyoming"; + finalImageTag = "v2025.1.1"; + os = "linux"; + arch = "amd64"; + hash = "sha256-SD5MMlMUkUKQtTOIUe9+r7NjuOTbFEdc3Xx7E2koa9Q="; + }; ports = [ "127.0.0.1:10210:10210" ]; - environment.ONNX_PROVIDER = "CUDAExecutionProvider"; - extraOptions = [ "--device=nvidia.com/gpu=all" ]; cmd = [ "--uri" "tcp://0.0.0.0:10210" From 8542b8aa675e4deebd7e0cfc4bfe96db4bffa861 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 20:59:17 +0000 Subject: [PATCH 15/98] Take ollama from unstable for Muse Glimmer support 26.05 ships 0.32.3, which predates the Muse Glimmer architecture; that landed in 0.32.7, and the unstable input is already pinned at exactly 0.32.7. An overlay is much less work than overriding a buildGoModule by hand, since the package definition moved between the two releases. Instantiating the unstable package set with the stable one's config carries allowUnfree and the cudaCapabilities pin across, so this is still an sm_89-only build; verified realArches is still ["sm_89"]. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- .gitignore | 1 + flake.nix | 2 ++ nixos/nixos/configuration.nix | 24 ++++++++++++++++++++++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index a3fdc22..2e22b3e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ __pycache__/ /.direnv/ /nix-flake-update.txt /result* +/.claude/ diff --git a/flake.nix b/flake.nix index f53ae89..d388a6f 100644 --- a/flake.nix +++ b/flake.nix @@ -44,6 +44,8 @@ }; nixosConfigurations = { "nixos" = nixpkgs-stable.lib.nixosSystem { + # So the host can take individual packages from unstable. + specialArgs.nixpkgsUnstable = nixpkgs; modules = [ ./nixos/nixos/configuration.nix ]; }; }; diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 76756f6..9dd57f9 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -2,7 +2,12 @@ # your system. Help is available in the configuration.nix(5) man page # and in the NixOS manual (accessible by running ‘nixos-help’). -{ config, pkgs, ... }: +{ + config, + pkgs, + nixpkgsUnstable, + ... +}: { imports = [ @@ -391,8 +396,23 @@ virtualisation.libvirtd.nss.enable = true; virtualisation.libvirtd.onShutdown = "shutdown"; # Else DHCP leases disappear. - # Build virt-install with Ubuntu 26.04 support. nixpkgs.overlays = [ + # 26.05 ships ollama 0.32.3, which predates the Muse Glimmer architecture + # (added in 0.32.7). The unstable input is already pinned at exactly 0.32.7, + # so take it from there rather than overriding a buildGoModule by hand. + # Instantiating with `final.config` carries over allowUnfree and the + # cudaCapabilities pin, so this is still an sm_89-only build. + (final: prev: { + inherit + (import nixpkgsUnstable { + inherit (final.stdenv.hostPlatform) system; + inherit (final) config; + }) + ollama-cuda + ; + }) + + # Build virt-install with Ubuntu 26.04 support. (final: prev: { osinfo-db = prev.osinfo-db.overrideAttrs (old: { src = final.fetchFromGitLab { From 70e02676b0e90c694ec10ad789eebde0c7e25e8a Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 21:12:47 +0000 Subject: [PATCH 16/98] Track nixos-unstable, for ollama 0.32.13 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Muse Glimmer landed in 0.32.7 but took four follow-up fixes through 0.32.11 — parser recovery for mangled tool-invoke names, and a reasoning-template match — and the registry gates on those, so 0.32.7 still gets a 412. Overriding the version by hand does not work either: llamaCppVersion is let-bound rather than an attribute, and 0.32.13 bumps the vendored llama.cpp from b10242 to b10380 with a hard mismatch check. nixos-unstable already carries 0.32.13 while nixpkgs-unstable is two days behind on 0.32.7, and it gates on the NixOS test suite, so it is if anything the more conservative channel. This also unblocks Qwen 3.8, which needs 0.32.12. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- flake.lock | 8 ++++---- flake.nix | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 2d73fde..47fe70d 100644 --- a/flake.lock +++ b/flake.lock @@ -98,16 +98,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1786593342, - "narHash": "sha256-smTKQXMLLStzc8zJevMCckbk3My7SvbbLmPYZUJJKW4=", + "lastModified": 1786862985, + "narHash": "sha256-FBJRXmbGXiSUDvYEbfLYRkckayyZ6SK1UEqhCrIZ2Cs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6b5e5b7a6631f065bf6908986990b37d845f847f", + "rev": "e5bdc4a41d4c072fe1e3787eaa0320a384741d44", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixpkgs-unstable", + "ref": "nixos-unstable", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index d388a6f..4ee0575 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,10 @@ { inputs = { nixpkgs-stable.url = "github:NixOS/nixpkgs/nixos-26.05"; - nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + # nixos-unstable rather than nixpkgs-unstable: it gates on the NixOS test + # suite, and it runs ahead often enough to matter (it carried ollama 0.32.13 + # while nixpkgs-unstable was still two days back on 0.32.7). + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; home-manager = { url = "github:nix-community/home-manager"; inputs.nixpkgs.follows = "nixpkgs"; From 6c86109980fc2e5032ac7747b1eab66f1c657a6c Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 21:23:15 +0000 Subject: [PATCH 17/98] Add demo entities for assistant evaluation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grading "turn on the kitchen light" against a refusal message is not grading anything; with real entities it becomes a state assertion. The demo integration supplies lights, switches, fans and covers with plausible names for free. Only a handful get exposed to Assist. Every exposed entity is written into the system prompt, so exposing all of them would inflate prefill and change the thing being measured — "expose new entities" is now off for the conversation assistant, which it was not before. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 9dd57f9..9ebe6e2 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -182,6 +182,7 @@ openFirewall = true; # Satellites and phones live on the LAN, not virbr0. extraComponents = [ "assist_pipeline" + "demo" # Fake lights etc, so assistant evaluations have something to control. "esphome" # Voice Preview Edition and any other satellites. "met" "ollama" @@ -192,6 +193,14 @@ default_config = { }; homeassistant.time_zone = config.time.timeZone; + # Gives lights, switches, fans and covers with realistic names, which lets + # an evaluation grade "turn on the kitchen light" against actual state + # rather than against the wording of a refusal. Only a chosen handful are + # exposed to Assist — every exposed entity lands in the system prompt, so + # exposing all of them would inflate prefill and change what is measured. + # ("expose new entities" is off for the conversation assistant.) + demo = { }; + # Queries the local SearXNG. Exposing the script below to Assist turns it # into a tool the model can call, and ActionTool hands the response back, # so this is how the assistant learns anything after its training cutoff. From 006dbe558ccb6cd0699d4e27efbeb672967ec966 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 21:33:34 +0000 Subject: [PATCH 18/98] Correct the local-intent routing comment, again I inverted the filter's polarity. In default_agent.async_handle_intents a filter returning True means the result is NOT handled locally: if not isinstance(result, RecognizeResult) or ( intent_filter is not None and intent_filter(result) ): return None _async_local_fallback_intent_filter returns True for GET_STATE and MEDIA_SEARCH_AND_PLAY, so those two are the ones withheld from the local path, and everything else the sentence matcher recognises is answered without the model. Confirmed by measurement: 'Turn on the kitchen lights' returns 'Turned on the light' in 0.00 s. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 9ebe6e2..e19cbb1 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -169,12 +169,14 @@ ]; }; - # Voice assistant. Note that once ollama is given "Control Home Assistant", - # every command routes through it: assist_pipeline only handles GET_STATE and - # media search locally, and only when `prefer_local_intents` is on (it is off - # by default). So the LLM is on the critical path for "turn off the kitchen - # light", not just for open-ended questions — which is why the model here is - # the 3B-active one and why keeping its prompt prefix cached matters. + # Voice assistant. With `prefer_local_intents` on (a per-pipeline setting, + # off by default) the built-in sentence matcher answers anything it + # recognises without involving the model, so "turn off the kitchen light" + # comes back in milliseconds. The exceptions are GET_STATE and media search, + # which assist_pipeline deliberately withholds from the local path when the + # agent has control, so that state questions go to the model and it answers + # them with GetLiveContext. Measured: a matched command is ~0.01 s, anything + # reaching the model is ~1-3 s. # Pipelines and wake words are chosen in the UI, not here; these services # announce themselves over zeroconf and Home Assistant discovers them. services.home-assistant = { From 18c2ac55dada3dfbf4bb8b520bfecf614c01dced Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Sun, 16 Aug 2026 21:53:57 +0000 Subject: [PATCH 19/98] Run Kokoro on the GPU, packaged natively instead of in Docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The published `cuda` image never had a GPU path: `kokoro-onnx[gpu]` pulls both `onnxruntime` and `onnxruntime-gpu`, they install over each other under the same import name, and the CPU one wins. nixpkgs' onnxruntime, though, takes `cudaSupport`, and everything above it is small: kokoro-onnx is four Python files, and the Wyoming server is one more. So this drops the container and builds the stack. kokoro-onnx wants espeakng-loader and phonemizer-fork, which between them exist to ship an espeak-ng inside a wheel and to make phonemizer accept it. nixpkgs' phonemizer is already patched to use pkgs.espeak-ng and already carries the upstream `set_data_path` commit, so both are dropped, the same substitution python3Packages.misaki makes. The CUDA provider also gets `cudnn_conv_algo_search=HEURISTIC`. It defaults to EXHAUSTIVE, which benchmarks every convolution algorithm the first time it sees a shape; the model has 88 Conv nodes and a `sequence_length` input, so nearly every request would be a new shape and would pay the search again. Two behaviours the container was hiding: the server logs nothing below WARNING unless LOG_LEVEL says otherwise, so INFO is set and every request now prints its synthesis time; and its SIGTERM handler cancels the asyncio server without catching CancelledError, which exits 1 and would make every `systemctl stop` a failure. Verified on a machine with no GPU: the whole Python stack builds and serves Wyoming synthesis on the CPU (54 voices, correct IPA out of espeak-ng, real audio), the CUDA onnxruntime builds and reports CUDAExecutionProvider, and every op type in the model but the single STFT has a CUDA kernel registered. The speedup itself is not verified — that needs the card. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 39 +----- nixos/nixos/kokoro.nix | 221 ++++++++++++++++++++++++++++++++++ 2 files changed, 226 insertions(+), 34 deletions(-) create mode 100644 nixos/nixos/kokoro.nix diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 76756f6..dc7e76b 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -8,6 +8,7 @@ imports = [ # Include the results of the hardware scan. ./hardware-configuration.nix + ./kokoro.nix ]; # Bootloader. @@ -307,40 +308,10 @@ voice = "en-us-ryan-medium"; }; - # Kokoro is an 82M StyleTTS2 model under Apache 2.0 and markedly more natural - # than Piper. Nothing in nixpkgs speaks Wyoming for it, so this is a - # third-party container: pinned by digest, loopback only, model baked into the - # image so there is no runtime download. Voices are listed at - # https://huggingface.co/hexgrad/Kokoro-82M/blob/main/VOICES.md - # This runs on the CPU, costing ~0.4 s per reply, and that is the best trade - # available. The published `cuda` image ships a CPU-only onnxruntime, so it - # cannot use the GPU at all; nixpkgs does have python3Packages.kokoro, but it - # is the torch build, and a CUDA torch is a multi-hour compile made - # uncacheable by the cudaCapabilities pin above. Neither is worth 0.3 s. - # - # pullImage rather than a registry pull so the image is a build input: it - # downloads during the rebuild instead of afterwards on first service start, - # and the unit no longer needs network-online.target. - virtualisation.oci-containers = { - backend = "docker"; - containers.kokoro-tts = { - image = "ghcr.io/relvacode/kokoro-wyoming:v2025.1.1"; - imageFile = pkgs.dockerTools.pullImage { - imageName = "ghcr.io/relvacode/kokoro-wyoming"; - imageDigest = "sha256:ff15cfb276045bd61662162d9f70c2596c1cb5acd345c3f62835b2aea397ba69"; - finalImageName = "ghcr.io/relvacode/kokoro-wyoming"; - finalImageTag = "v2025.1.1"; - os = "linux"; - arch = "amd64"; - hash = "sha256-SD5MMlMUkUKQtTOIUe9+r7NjuOTbFEdc3Xx7E2koa9Q="; - }; - ports = [ "127.0.0.1:10210:10210" ]; - cmd = [ - "--uri" - "tcp://0.0.0.0:10210" - ]; - }; - }; + # Kokoro, the other text to speech engine, is big enough to live in + # ./kokoro.nix: it is packaged there rather than pulled as a container, so + # that it can run on the GPU. It listens on 127.0.0.1:10210, as the container + # did, so Home Assistant's Wyoming entry for it does not change. # Wake word. Models are picked per-pipeline in the UI; `preloadModels` was # removed in wyoming-openwakeword 2.0. Unlike the faster-whisper and piper diff --git a/nixos/nixos/kokoro.nix b/nixos/nixos/kokoro.nix new file mode 100644 index 0000000..ebf83f1 --- /dev/null +++ b/nixos/nixos/kokoro.nix @@ -0,0 +1,221 @@ +# Kokoro text to speech, spoken over the Wyoming protocol so Home Assistant can +# use it as a TTS provider. Kokoro is an 82M StyleTTS2 model under Apache 2.0 +# and markedly more natural than Piper. Voices are listed at +# https://huggingface.co/hexgrad/Kokoro-82M/blob/main/VOICES.md +# +# This ran as a Docker container until now, because nothing in nixpkgs speaks +# Wyoming for Kokoro. The container was CPU-only and cost ~0.4 s of every reply, +# and the one published GPU image is a dead end: `kokoro-onnx[gpu]` asks for +# both `onnxruntime` and `onnxruntime-gpu`, which install over each other under +# the same import name, and the CPU one wins — that image reports only the Azure +# and CPU providers. +# +# nixpkgs' onnxruntime does build the CUDA execution provider, so the whole +# thing is packaged natively instead, with no Docker at all: kokoro-onnx (a thin +# pure-Python wrapper around the ONNX model) on top of a CUDA onnxruntime, plus +# upstream's Wyoming server, which is a single file. +{ + lib, + pkgs, + utils, + ... +}: + +let + # onnxruntime with the CUDA execution provider. `nixpkgs.config.cudaSupport` + # would turn this on globally and rebuild half the system, so override the one + # package. NCCL is for multi-GPU collectives; there is one GPU here, and + # dropping it avoids building nccl for nothing. + onnxruntime = pkgs.onnxruntime.override { + cudaSupport = true; + ncclSupport = false; + }; + + onnxruntime-python = pkgs.python3Packages.onnxruntime.override { inherit onnxruntime; }; + + espeak = lib.getLib pkgs.espeak-ng; + + kokoro-onnx = pkgs.python3Packages.buildPythonPackage { + pname = "kokoro-onnx"; + version = "0.5.0-unstable-2026-07-05"; + pyproject = true; + + src = pkgs.fetchFromGitHub { + owner = "thewh1teagle"; + repo = "kokoro-onnx"; + rev = "98ea02a5692534c2ba496708e2f19de25028412b"; + hash = "sha256-wF9nvk8j/mSQTIipBZP7UxI2VxIAt+lo2bdPrRLiF6c="; + }; + + build-system = [ pkgs.python3Packages.hatchling ]; + + # espeakng-loader exists to ship a bundled espeak-ng inside a wheel, and + # phonemizer-fork exists to make phonemizer accept that bundled copy. + # Neither is wanted here: nixpkgs' phonemizer is already patched to point at + # pkgs.espeak-ng and already carries the upstream commit that adds + # set_data_path. python3Packages.misaki drops the same two dependencies the + # same way. + # + # The CUDA execution provider defaults to cudnn_conv_algo_search=EXHAUSTIVE, + # which benchmarks every convolution algorithm the first time it sees a + # shape. Kokoro's tensors are as long as the sentence, so nearly every + # request is a new shape and the search would be paid over and over. + # HEURISTIC picks an algorithm from cuDNN's model instead. + postPatch = '' + substituteInPlace pyproject.toml \ + --replace-fail '"espeakng-loader>=0.2.4",' "" \ + --replace-fail '"phonemizer-fork>=3.3.2",' '"phonemizer",' + substituteInPlace src/kokoro_onnx/tokenizer.py \ + --replace-fail "import espeakng_loader" "" \ + --replace-fail "espeakng_loader.get_data_path()" \ + '"${espeak}/share/espeak-ng-data"' \ + --replace-fail "espeakng_loader.get_library_path()" \ + '"${espeak}/lib/libespeak-ng${pkgs.stdenv.hostPlatform.extensions.sharedLibrary}"' + substituteInPlace src/kokoro_onnx/__init__.py \ + --replace-fail "providers = [env_provider]" \ + 'providers = [(env_provider, {"cudnn_conv_algo_search": "HEURISTIC"})] if env_provider == "CUDAExecutionProvider" else [env_provider]' + ''; + + dependencies = [ + onnxruntime-python + pkgs.python3Packages.numpy + pkgs.python3Packages.phonemizer + ]; + + pythonImportsCheck = [ "kokoro_onnx" ]; + + meta = { + description = "TTS with Kokoro and ONNX Runtime"; + homepage = "https://github.com/thewh1teagle/kokoro-onnx"; + license = lib.licenses.mit; + }; + }; + + # Weights and voice embeddings, from the same release the container's + # Dockerfile pins, with the same checksums. + model = pkgs.fetchurl { + url = "https://github.com/thewh1teagle/kokoro-onnx/releases/download/model-files-v1.0/kokoro-v1.0.onnx"; + hash = "sha256-fV347PfUsYeAFaMmhgU/0O6+K8N3I0YIdkzA7zY2psU="; + }; + voices = pkgs.fetchurl { + url = "https://github.com/thewh1teagle/kokoro-onnx/releases/download/model-files-v1.0/voices-v1.0.bin"; + hash = "sha256-vKYQuDCOjZnzLm/kGX5+wBZ5Jk7+0MrJFA/pwp8fv30="; + }; + + # The Wyoming server is one file with no packaging of its own, so install it + # as a script with a Python that has its imports. nordwestt's fork rather than + # relvacode's upstream because it takes the model paths as arguments instead + # of reading them from the working directory, and because it caches repeated + # phrases and strips the markdown the LLM sprinkles into its replies. + python = pkgs.python3.withPackages (ps: [ + kokoro-onnx + ps.numpy + ps.wyoming + ]); + + kokoro-wyoming = + let + src = pkgs.fetchFromGitHub { + owner = "nordwestt"; + repo = "kokoro-wyoming"; + tag = "v1.0.2"; + hash = "sha256-IUgchZ3gYSQtZNB23hYPRcJZ+mrW5WcLLnJ0SOevt1E="; + }; + in + # The SIGTERM handler stops the asyncio server, which surfaces as a + # CancelledError out of asyncio.run and a traceback with status 1. Docker + # swallowed that; systemd would call every `systemctl stop` a failure. + pkgs.runCommandLocal "kokoro-wyoming-1.0.2" { } '' + mkdir -p $out/bin + substitute ${src}/src/main.py $out/bin/kokoro-wyoming \ + --replace-fail "#!/usr/bin/env python3" "#!${python.interpreter}" \ + --replace-fail "except KeyboardInterrupt:" \ + "except (KeyboardInterrupt, asyncio.CancelledError):" + chmod +x $out/bin/kokoro-wyoming + ''; +in +{ + # So the server can be built and run by hand, without a rebuild: + # nix build .#nixosConfigurations.nixos.config.system.build.kokoro-wyoming + system.build.kokoro-wyoming = kokoro-wyoming; + + systemd.services.kokoro-tts = { + description = "Kokoro text to speech over the Wyoming protocol"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + environment = { + # kokoro-onnx asks onnxruntime for exactly this provider; onnxruntime + # always keeps the CPU provider behind it, so an operator with no CUDA + # kernel still runs, just off the GPU. Of the 2464 nodes in this model + # only the single STFT has no CUDA kernel in onnxruntime 1.24. + ONNX_PROVIDER = "CUDAExecutionProvider"; + # Without this the server logs nothing at all: kokoro-onnx's logger + # defaults to WARNING and the Wyoming server is a child of it. At INFO + # every request prints its synthesis time, which is how you check that the + # GPU is doing what it is supposed to. `--debug` would also work, but it + # adds a line of phonemes per request. + LOG_LEVEL = "INFO"; + }; + + serviceConfig = { + Type = "exec"; + # Prefixed with `-` so a missing provider is a log line, not a dead voice + # assistant. This is the check that the nordwestt image would have failed. + ExecStartPre = "-${python.interpreter} -c 'import onnxruntime; print(\"onnxruntime providers:\", onnxruntime.get_available_providers())'"; + ExecStart = utils.escapeSystemdExecArgs [ + "${kokoro-wyoming}/bin/kokoro-wyoming" + "--uri" + "tcp://127.0.0.1:10210" + "--model" + "${model}" + "--voices" + "${voices}" + ]; + Restart = "on-failure"; + + DynamicUser = true; + CapabilityBoundingSet = [ "" ]; + # Same set services.ollama uses to reach the GPU, minus the ROCm and WSL + # entries. PrivateDevices would hide all of it. + DeviceAllow = [ + "char-nvidiactl" + "char-nvidia-caps" + "char-nvidia-frontend" + "char-nvidia-uvm" + ]; + DevicePolicy = "closed"; + PrivateDevices = false; + LockPersonality = true; + MemoryDenyWriteExecute = false; # required for onnxruntime + NoNewPrivileges = true; + PrivateTmp = true; + PrivateUsers = true; + ProcSubset = "all"; # onnxruntime reads /proc/cpuinfo + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "invisible"; + ProtectSystem = "strict"; + RemoveIPC = true; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + "AF_UNIX" + ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service @resources" + "~@privileged" + ]; + UMask = "0077"; + }; + }; +} From d6d0cc8397ac882eca9bba26f12fe085f929a9f4 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 00:16:28 +0000 Subject: [PATCH 20/98] Give ollama two slots, for the base prefix and the conversation ollama only reuses a cached sequence when the new prompt extends it, never on a shared prefix, so every fresh conversation re-prefills the whole system prompt: 0.65 s against 0.19 s when the prefix happens to be cached, on every single command. OLLAMA_NUM_PARALLEL=1 was my choice, on the assumption that a single warm slot was best; that assumed the wrong caching model. Two slots let the base prefix survive alongside an active conversation. Not more: slots multiply the KV allocation, and at the 32768 context this host requests, four would be ~35 GB of KV beside a 30 GB model. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index c347a34..8f42691 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -154,6 +154,19 @@ # The assistant has to stay resident; a 30 s reload before "turn off the # lights" is the difference between usable and infuriating. OLLAMA_KEEP_ALIVE = "-1"; + # ollama only reuses a cached sequence when the new prompt *extends* it, + # never on a shared prefix — so every fresh conversation re-runs the whole + # system prompt. Measured on this host: 0.65 s against 0.19 s when the + # prefix happens to be cached, on every command. + # + # Two slots is the whole idea: one holds the base prefix, one holds the + # conversation in progress, and neither evicts the other. A third would + # only earn its keep with two satellites talking at once. Slots multiply + # the KV allocation and this host asks for a 32768 context, so they are + # not free: KV here is 65 layers x 4 kv heads x 512 dims x 2 bytes, about + # 260 KB per token, i.e. ~2.2 GB per slot at the 8192 Home Assistant + # requests and ~8.7 GB at the full 32768. + OLLAMA_NUM_PARALLEL = "2"; }; # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. From 08aa90ad4f64fa9e6ab0fcde5745a33641ee2fda Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 00:23:35 +0000 Subject: [PATCH 21/98] Revert to one ollama slot; extra slots cannot help llama-server chooses a slot by LRU unless --slot-prompt-similarity is set. It defaults to 0.0, has no set_env binding, and ollama never passes it, so a fresh conversation takes the least recently used slot regardless of what is cached there. Measured with two slots: a three-turn conversation still evicted the primed prefix and the next command cost 0.63 s again. Since -c is num_ctx x parallel, the second slot was buying extra KV allocation for nothing. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 8f42691..b3538a7 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -155,18 +155,18 @@ # lights" is the difference between usable and infuriating. OLLAMA_KEEP_ALIVE = "-1"; # ollama only reuses a cached sequence when the new prompt *extends* it, - # never on a shared prefix — so every fresh conversation re-runs the whole - # system prompt. Measured on this host: 0.65 s against 0.19 s when the - # prefix happens to be cached, on every command. + # so every fresh conversation re-runs the whole system prompt: measured + # here at 0.65 s against 0.19 s when the prefix happens to be cached. # - # Two slots is the whole idea: one holds the base prefix, one holds the - # conversation in progress, and neither evicts the other. A third would - # only earn its keep with two satellites talking at once. Slots multiply - # the KV allocation and this host asks for a 32768 context, so they are - # not free: KV here is 65 layers x 4 kv heads x 512 dims x 2 bytes, about - # 260 KB per token, i.e. ~2.2 GB per slot at the 8192 Home Assistant - # requests and ~8.7 GB at the full 32768. - OLLAMA_NUM_PARALLEL = "2"; + # Extra slots do not fix it, and llama.cpp's source says why. ollama + # spawns llama-server with `-np`, but llama-server picks a slot by LRU + # unless --slot-prompt-similarity is set, and that defaults to 0.0 with no + # environment binding, so ollama cannot turn it on. A fresh conversation + # therefore takes the least recently used slot no matter what is cached in + # it. Measured with two slots: a three-turn conversation still evicted the + # prefix and the next command went back to 0.63 s. Left at one slot, since + # `-c` is num_ctx x parallel and the extra KV bought nothing. + OLLAMA_NUM_PARALLEL = "1"; }; # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. From e796cc31f6338c7d41e61d4c930b8c1847055cc3 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 00:35:34 +0000 Subject: [PATCH 22/98] Stop llama-server clearing the slot on a level-2 cache miss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every fresh conversation re-ran the whole system prompt: 0.65 s against 0.19 s when the prefix was already cached, on every command. llama-server picks a slot by LRU, because --slot-prompt-similarity defaults to 0 and has no env binding. The LRU path then saves the slot, tries to load a better match, and calls prompt_clear() when that fails — discarding a prefix that ordinary common-prefix reuse would have kept. Enabling the similarity threshold would be the wrong fix anyway: it compares the common prefix against the *incoming prompt length*, so a long enough user message would skip a perfectly good cached prefix. The block is guarded by "update_cache && prompt_cache", and the cache is only constructed when cache_ram_mib != 0, so switching it off avoids the clearing entirely. llama-server inherits ollama's environment (cmd.Env = os.Environ()), so this needs no patch. The level-2 cache exists to restore evicted conversations when serving many at once; this host serves one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index b3538a7..51e585a 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -154,19 +154,27 @@ # The assistant has to stay resident; a 30 s reload before "turn off the # lights" is the difference between usable and infuriating. OLLAMA_KEEP_ALIVE = "-1"; - # ollama only reuses a cached sequence when the new prompt *extends* it, - # so every fresh conversation re-runs the whole system prompt: measured - # here at 0.65 s against 0.19 s when the prefix happens to be cached. + # Every fresh conversation was re-running the whole system prompt: 0.65 s + # against 0.19 s when the prefix was already cached. The cause is in + # llama-server, which ollama runs as a subprocess. Slot selection falls to + # LRU (--slot-prompt-similarity defaults to 0 and has no env binding), and + # the LRU path does this: # - # Extra slots do not fix it, and llama.cpp's source says why. ollama - # spawns llama-server with `-np`, but llama-server picks a slot by LRU - # unless --slot-prompt-similarity is set, and that defaults to 0.0 with no - # environment binding, so ollama cannot turn it on. A fresh conversation - # therefore takes the least recently used slot no matter what is cached in - # it. Measured with two slots: a three-turn conversation still evicted the - # prefix and the next command went back to 0.63 s. Left at one slot, since - # `-c` is num_ctx x parallel and the extra KV bought nothing. - OLLAMA_NUM_PARALLEL = "1"; + # ret->prompt_save(*prompt_cache); + # if (!ret->prompt_load(*prompt_cache, task.tokens)) { + # ret->prompt_clear(); + # } + # + # so when the level-2 cache has no better match it *clears the slot*, + # throwing away a prefix that ordinary common-prefix reuse would have + # kept. That block is guarded by `update_cache && prompt_cache`, and the + # cache is only built when cache_ram_mib != 0 — so switching it off stops + # the clearing. llama-server inherits this environment (cmd.Env = + # os.Environ()), so no patch is needed. + # + # Nothing is lost: that cache exists to restore evicted conversations when + # serving many at once, and this host serves one. + LLAMA_ARG_CACHE_RAM = "0"; }; # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. From 2264bba68c8f7af062dbcd6290bed99bb288bb9b Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 00:41:32 +0000 Subject: [PATCH 23/98] Drop the cache-ram workaround; it did not help Predicted that disabling llama-server's level-2 prompt cache would stop it clearing the slot and restore common-prefix reuse. Measured after switching: fresh conversations still cost 0.63 s, unchanged. The prompt_clear theory was wrong or incomplete. Recording what is ruled out so the next attempt does not repeat it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 51e585a..0c7e1b3 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -154,27 +154,15 @@ # The assistant has to stay resident; a 30 s reload before "turn off the # lights" is the difference between usable and infuriating. OLLAMA_KEEP_ALIVE = "-1"; - # Every fresh conversation was re-running the whole system prompt: 0.65 s - # against 0.19 s when the prefix was already cached. The cause is in - # llama-server, which ollama runs as a subprocess. Slot selection falls to - # LRU (--slot-prompt-similarity defaults to 0 and has no env binding), and - # the LRU path does this: - # - # ret->prompt_save(*prompt_cache); - # if (!ret->prompt_load(*prompt_cache, task.tokens)) { - # ret->prompt_clear(); - # } - # - # so when the level-2 cache has no better match it *clears the slot*, - # throwing away a prefix that ordinary common-prefix reuse would have - # kept. That block is guarded by `update_cache && prompt_cache`, and the - # cache is only built when cache_ram_mib != 0 — so switching it off stops - # the clearing. llama-server inherits this environment (cmd.Env = - # os.Environ()), so no patch is needed. - # - # Nothing is lost: that cache exists to restore evicted conversations when - # serving many at once, and this host serves one. - LLAMA_ARG_CACHE_RAM = "0"; + # Note: every fresh conversation re-runs the whole system prompt here, + # 0.65 s against 0.19 s for a continued one, on every command. The cause + # is not yet identified. Ruled out by measurement: extra slots (selection + # is LRU and ignores what is cached), and disabling llama-server's + # level-2 prompt cache with LLAMA_ARG_CACHE_RAM=0. Ruled out by reading + # llama.cpp b10380: cache_prompt defaults true and ollama sets it anyway, + # and n_past is only zeroed when it is false. Until it is understood, the + # lever that does work is keeping the prompt short — every entity exposed + # to Assist is ~25 tokens of prefill on every request. }; # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. From 2e0c549c352dcdaf235b2b23cac127faa2529db5 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 00:50:07 +0000 Subject: [PATCH 24/98] Keep the system prefix cached, with a primer in front of ollama MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit llama-server reuses a cached sequence only when the new prompt extends it, so a finished conversation leaves [system][user][assistant] in the slot and the next conversation's [system][user'] diverges: the whole system prompt is recomputed. Measured 0.65 s of prefill on every command against 0.19 s when the slot holds just the prefix. The proxy replays each /api/chat with the conversation stripped, system message and tools only. That is a prefix of what is cached, so it costs about 0.10 s and runs after the response has already gone back to Home Assistant. Replaying the caller's own payload rather than rebuilding it means the bytes match by construction, including the tool schemas, and go on matching when the prompt changes — a hand-built copy would drift silently, with no runtime signal that it had stopped working. Measured through the proxy, streaming as Home Assistant does: 0.185 s, with chunks still arriving incrementally. ollama moves to 11435 on loopback; the primer takes 11434, so nothing that talks to it needs to change. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TgwAc8cnQQHo3VAPRjrfHM --- nixos/nixos/configuration.nix | 54 +++++++++++++----- nixos/nixos/ollama-primer.py | 100 ++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 13 deletions(-) create mode 100644 nixos/nixos/ollama-primer.py diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 0c7e1b3..53ca8eb 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -147,22 +147,19 @@ services.ollama = { enable = true; package = pkgs.ollama-cuda; - host = "0.0.0.0"; # So libvirt guests can reach it at 192.168.122.1. + # Behind the primer proxy, which is what listens on 11434. + host = "127.0.0.1"; + port = 11435; environmentVariables = { OLLAMA_CONTEXT_LENGTH = "32768"; # Otherwise it's tiered off total VRAM. OLLAMA_FLASH_ATTENTION = "1"; # The assistant has to stay resident; a 30 s reload before "turn off the # lights" is the difference between usable and infuriating. OLLAMA_KEEP_ALIVE = "-1"; - # Note: every fresh conversation re-runs the whole system prompt here, - # 0.65 s against 0.19 s for a continued one, on every command. The cause - # is not yet identified. Ruled out by measurement: extra slots (selection - # is LRU and ignores what is cached), and disabling llama-server's - # level-2 prompt cache with LLAMA_ARG_CACHE_RAM=0. Ruled out by reading - # llama.cpp b10380: cache_prompt defaults true and ollama sets it anyway, - # and n_past is only zeroed when it is false. Until it is understood, the - # lever that does work is keeping the prompt short — every entity exposed - # to Assist is ~25 tokens of prefill on every request. + # See ollama-primer.py: llama-server only reuses a cached sequence that + # the new prompt extends, so a conversation leaves the slot in a state the + # next conversation cannot build on. Measured 0.65 s of prefill on every + # command, against 0.19 s once the slot holds just the prefix. }; # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. @@ -179,6 +176,39 @@ ]; }; + # Keeps ollama's prompt cache holding the Home Assistant system prefix. See + # the docstring in ollama-primer.py for why this is needed; in short, + # llama-server reuses a cached sequence only when the new prompt extends it, + # and a finished conversation leaves the slot in a state the next one cannot + # build on. The proxy replays each request with the conversation stripped, so + # the bytes match by construction — including tool schemas — and keep matching + # when the prompt changes. + systemd.services.ollama-primer = { + description = "Prefix-cache primer in front of ollama"; + wantedBy = [ "multi-user.target" ]; + after = [ "ollama.service" ]; + wants = [ "ollama.service" ]; + environment = { + PRIMER_UPSTREAM = "http://127.0.0.1:${toString config.services.ollama.port}"; + PRIMER_HOST = "0.0.0.0"; # So libvirt guests can reach it at 192.168.122.1. + PRIMER_PORT = "11434"; + }; + serviceConfig = { + ExecStart = "${pkgs.python3}/bin/python3 ${./ollama-primer.py}"; + Restart = "always"; + RestartSec = 2; + DynamicUser = true; + NoNewPrivileges = true; + PrivateDevices = true; + ProtectSystem = "strict"; + ProtectHome = true; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + ]; + }; + }; + # Voice assistant. With `prefer_local_intents` on (a per-pipeline setting, # off by default) the built-in sentence matcher answers anything it # recognises without involving the model, so "turn off the kitchen light" @@ -363,9 +393,7 @@ services.esphome.enable = true; # Only the VMs, not the whole LAN. - networking.firewall.interfaces."virbr0".allowedTCPPorts = [ - config.services.ollama.port - ]; + networking.firewall.interfaces."virbr0".allowedTCPPorts = [ 11434 ]; # Chat UI at http://localhost:8080 services.open-webui = { diff --git a/nixos/nixos/ollama-primer.py b/nixos/nixos/ollama-primer.py new file mode 100644 index 0000000..8535aec --- /dev/null +++ b/nixos/nixos/ollama-primer.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Reverse proxy for ollama that keeps the system prefix cached. + +llama-server reuses a cached sequence only when the new prompt extends it, so +after a conversation leaves [system][user][assistant] in the slot, the next +conversation's [system][user'] diverges and the whole system prompt is +recomputed -- measured at 0.65 s against 0.19 s here, on every command. + +After each /api/chat we replay that exact request with the conversation +stripped: system message and tools only. That is a prefix of what is cached, so +it costs ~0.1 s, and it leaves the slot holding exactly the prefix, so the next +conversation extends it. Replaying the caller's own payload rather than +rebuilding it means the bytes match by construction, including tool schemas, +and it keeps matching when Home Assistant's prompt changes. +""" +import json, os, threading, urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +UPSTREAM = os.environ.get("PRIMER_UPSTREAM", "http://127.0.0.1:11435") +HOST = os.environ.get("PRIMER_HOST", "127.0.0.1") +PORT = int(os.environ.get("PRIMER_PORT", "11434")) + +_lock = threading.Lock() + +def prime(payload): + sys_msgs = [m for m in payload.get("messages", []) if m.get("role") == "system"] + if not sys_msgs or not payload.get("model"): + return + body = {"model": payload["model"], "messages": sys_msgs, "stream": False, + "options": {**payload.get("options", {}), "num_predict": 1}, + "keep_alive": payload.get("keep_alive", -1)} + if "tools" in payload: + body["tools"] = payload["tools"] + req = urllib.request.Request(UPSTREAM + "/api/chat", + data=json.dumps(body).encode(), method="POST") + req.add_header("Content-Type", "application/json") + with _lock: # never race a real request + try: + urllib.request.urlopen(req, timeout=300).read() + except Exception: + pass + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + def log_message(self, *a): pass + + def _relay(self, method): + body = None + n = int(self.headers.get("Content-Length") or 0) + if n: + body = self.rfile.read(n) + req = urllib.request.Request(UPSTREAM + self.path, data=body, method=method) + for k, v in self.headers.items(): + if k.lower() not in ("host", "content-length", "connection", "accept-encoding"): + req.add_header(k, v) + try: + with _lock: + pass # wait for any in-flight prime to finish + r = urllib.request.urlopen(req, timeout=3600) + except urllib.error.HTTPError as e: + data = e.read() + self.send_response(e.code) + self.send_header("Content-Type", e.headers.get("Content-Type", "application/json")) + self.send_header("Content-Length", str(len(data))) + self.end_headers(); self.wfile.write(data) + return + except Exception: + self.send_response(502); self.send_header("Content-Length", "0") + self.end_headers() + return + + self.send_response(r.status) + self.send_header("Content-Type", r.headers.get("Content-Type", "application/json")) + self.send_header("Transfer-Encoding", "chunked") # stream through + self.end_headers() + try: + while True: + chunk = r.read(4096) + if not chunk: + break + self.wfile.write(b"%X\r\n%s\r\n" % (len(chunk), chunk)) + self.wfile.flush() + self.wfile.write(b"0\r\n\r\n"); self.wfile.flush() + except Exception: + return + + if method == "POST" and self.path.rstrip("/") == "/api/chat" and body: + try: + payload = json.loads(body) + except Exception: + return + threading.Thread(target=prime, args=(payload,), daemon=True).start() + + def do_GET(self): self._relay("GET") + def do_POST(self): self._relay("POST") + def do_DELETE(self): self._relay("DELETE") + def do_HEAD(self): self._relay("HEAD") + +if __name__ == "__main__": + ThreadingHTTPServer((HOST, PORT), Handler).serve_forever() From 22723ea8fe782992d0a1430f7db45e921b008cad Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 02:31:21 +0000 Subject: [PATCH 25/98] Record why fresh conversations re-prefill, with the root cause MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit llama-server identifies the reusable prefix correctly — the log shows f_sim_best = 0.994 (1836/1848) — and then the context-checkpoint machinery discards it. Checkpoints cannot truncate at an arbitrary position, only roll back to a checkpoint, and the nearest one below the divergence point was at 825. So it re-prefills 1023 tokens. Two candidate environment variables are recorded but not yet tested. Also recorded: everything ruled out by measurement, the claims in earlier commit messages that turn out to be false, and the measurement traps that produced them — chiefly that a synthetic system prompt is not Home Assistant's, and that prompt_eval_count reports the full prompt even on a cache hit. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_d93535ae-8d15-4da9-a216-a818cae9694a --- nixos/nixos/prefix-cache-findings.md | 100 +++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 nixos/nixos/prefix-cache-findings.md diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md new file mode 100644 index 0000000..3d193dd --- /dev/null +++ b/nixos/nixos/prefix-cache-findings.md @@ -0,0 +1,100 @@ +# Why fresh conversations re-prefill the whole system prompt + +Status: **root cause found, fix not yet applied or measured.** + +Every Home Assistant command that starts a new conversation costs ~0.65 s of +prompt evaluation instead of ~0.19 s, because llama-server discards a cached +prefix it has already identified as reusable. + +## The mechanism, from llama-server's own log + +ollama runs llama-server with `--log-verbosity 4`, so this is in +`journalctl -u ollama`. One request, annotated: + + selected slot by LCP similarity, f_sim_best = 0.994 (1836/1848) + new prompt, n_ctx_slot = 8192, n_keep = 4, task.n_tokens = 1848 + checking checkpoint with [1843, 1843] against 1836... + checking checkpoint with [824, 824] against 1836... + restored context checkpoint (pos_min = 824, n_tokens = 825, n_past = 825) + cached n_tokens = 825, memory_seq_rm [825, end) + prompt eval time = 667.37 ms / 1023 tokens + +The slot selector finds **1836 of 1848 tokens** in common — it knows almost the +whole prompt is already cached. Then the context-checkpoint machinery takes +over. It cannot truncate the sequence at an arbitrary position; it can only roll +back to a checkpoint. The only checkpoints are at 824 and 1843. 1843 is past the +divergence point, so it falls back to 824 and re-prefills 1023 tokens. + +Checkpoints are enabled because the context reports +`COMMON_CONTEXT_SEQ_RM_TYPE_FULL` ("can seq_rm full sequences only"), which +makes the server log "speculative decoding will use checkpoints". Partial +truncation degrades to checkpoint granularity, and checkpoints are sparse +because each one costs ~152 MiB. + +## Candidate fixes, untested + +Both are environment variables. ollama passes `cmd.Env = os.Environ()` to the +llama-server subprocess, so `services.ollama.environmentVariables` reaches it +with no patch. + +1. `LLAMA_ARG_CTX_CHECKPOINTS=0` — disable checkpoints entirely. Should let the + server truncate at 1836 and prefill ~12 tokens. Risk: checkpoints exist to + support speculative decoding on a FULL-only context, so this may disable or + degrade MTP, which is worth ~1.7x on generation. Measure both prefill and + tokens/sec before keeping it. +2. `LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT=` — keep checkpoints but space + them closely, so a rollback loses less. Costs VRAM: ~152 MiB each, up to 32. + +Both bindings verified present in `common/arg.cpp` at llama.cpp b10380, which is +what ollama 0.32.13 vendors. + +## Ruled out — do not retry these + +Each was tested, not reasoned about: + +- **Slot count.** ollama passes `-np 1`; there is one slot. Adding a second does + not help: slot choice is LCP-based here, not LRU. +- **`prompt_clear()` on a level-2 cache miss.** `LLAMA_ARG_CACHE_RAM=0` skips + that block entirely (the cache is only constructed when `cache_ram_mib != 0`, + and `update_cache && prompt_cache` then short-circuits). Measured: no change. +- **`--slot-prompt-similarity`.** Already effectively enabled — the log shows + `selected slot by LCP similarity`. Selection was never the problem. +- **MTP / the draft head.** The non-MTP `qwen3.6:27b-q4_K_M` is equally slow. +- **`--mmproj` / vision.** A projector-stripped variant built with + `/api/create` (capabilities lost `vision`, kept `tools` and `thinking`) is + equally slow. +- **`--flash-attn on`, `--context-shift --keep 4`, `--no-jinja --chat-template + chatml`, `-b/-ub 1024`.** Each added individually to a stock llama-server: + prefix reuse still works. +- **Renderer-injected timestamps.** ollama's qwen renderers contain no + time-varying content. (`glimmer.go` does inject `Current date:`, so this would + be a real issue for Muse Glimmer.) + +Also false, from earlier commit messages: llama-server does **not** require the +new prompt to extend the cached one. It computes +`n_past = slot.prompt.tokens.get_common_prefix(input_tokens)` — a genuine +longest common prefix. Verified standalone: three fresh conversations sharing +only a system prefix prefilled 9, 10 and 10 tokens. + +## Measurement traps that produced false results here + +- **A synthetic system prompt is not Home Assistant's.** If HA sends anything + during a benchmark, it replaces the slot contents and the next synthetic + request shares nothing with it. Quiesce HA, or use HA's own prompt. +- **`prompt_eval_count` always reports the full prompt**, even when almost all + of it was reused. Only `prompt_eval_duration` is a signal. +- **Back-to-back requests are not representative.** With the priming proxy in + place, requests fired with no gap contend with the replay: 3.47 s at 0 s + spacing, 2.52 s at 2 s, 1.79 s at 5 s. Voice usage looks like the last one. +- **There is ~0.15 s of fixed per-request overhead**, so a perfect cache hit + reads as ~0.19 s, not ~0.02 s. +- Compare like with like: a prompt without `tools` is ~780 tokens against ~1850 + with them. + +## What the priming proxy is for + +`ollama-primer.py` replays each request with the conversation stripped, leaving +the slot holding just the prefix, so the next request is a strict extension and +needs no truncation at all. It works — 0.19 s — and sidesteps the checkpoint +problem rather than fixing it. If a fix above lands, the proxy, the port shuffle +and the back-to-back contention all go away together. From f0f154eb7d9865d880e1e3de2569b4b7fb8f6165 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 02:37:41 +0000 Subject: [PATCH 26/98] Disable context checkpoints; drop the priming proxy llama-server was finding 1836 of 1848 tokens reusable and then throwing 1011 of them away, because a COMMON_CONTEXT_SEQ_RM_TYPE_FULL context cannot truncate at an arbitrary position and has to roll back to the nearest context checkpoint (824). LLAMA_ARG_CTX_CHECKPOINTS=0 removes the rollback path. ollama passes its environment through to llama-server, so no patch is needed. This is the fix the primer was working around, so the primer goes with it, and ollama moves back to 11434. Removing it is also what makes the result measurable: with the primer in the path the slot already holds the prefix, which is the state the fix is supposed to produce. Unverified as of this commit. Checkpoints are what let speculative decoding run on a FULL-only context, so this may cost MTP; the baseline to beat is 77 tok/s generation and the target is ~0.19 s prefill. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 58 +++++----------- nixos/nixos/ollama-primer.py | 100 --------------------------- nixos/nixos/prefix-cache-findings.md | 33 +++++++-- 3 files changed, 44 insertions(+), 147 deletions(-) delete mode 100644 nixos/nixos/ollama-primer.py diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 53ca8eb..ff966e1 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -147,19 +147,30 @@ services.ollama = { enable = true; package = pkgs.ollama-cuda; - # Behind the primer proxy, which is what listens on 11434. - host = "127.0.0.1"; - port = 11435; + # 0.0.0.0 so libvirt guests can reach it at 192.168.122.1; the firewall + # only opens 11434 on virbr0. + host = "0.0.0.0"; environmentVariables = { OLLAMA_CONTEXT_LENGTH = "32768"; # Otherwise it's tiered off total VRAM. OLLAMA_FLASH_ATTENTION = "1"; # The assistant has to stay resident; a 30 s reload before "turn off the # lights" is the difference between usable and infuriating. OLLAMA_KEEP_ALIVE = "-1"; - # See ollama-primer.py: llama-server only reuses a cached sequence that - # the new prompt extends, so a conversation leaves the slot in a state the - # next conversation cannot build on. Measured 0.65 s of prefill on every - # command, against 0.19 s once the slot holds just the prefix. + # ollama passes its own environment through to llama-server, so LLAMA_ARG_* + # reaches it with no patch. + # + # Without this, a new conversation costs ~0.65 s of prefill instead of + # ~0.19 s. llama-server finds the prefix reusable (1836 of 1848 tokens) + # but cannot truncate the KV cache at an arbitrary position, because this + # context reports COMMON_CONTEXT_SEQ_RM_TYPE_FULL. It rolls back to the + # nearest context checkpoint instead — 824 — and re-prefills the 1011 + # valid tokens in between. Setting this to 0 removes the rollback, leaving + # a plain longest-common-prefix reuse. + # + # Checkpoints exist to let speculative decoding work on a FULL-only + # context, so this is exactly the knob that might cost MTP. See + # prefix-cache-findings.md. + LLAMA_ARG_CTX_CHECKPOINTS = "0"; }; # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. @@ -176,39 +187,6 @@ ]; }; - # Keeps ollama's prompt cache holding the Home Assistant system prefix. See - # the docstring in ollama-primer.py for why this is needed; in short, - # llama-server reuses a cached sequence only when the new prompt extends it, - # and a finished conversation leaves the slot in a state the next one cannot - # build on. The proxy replays each request with the conversation stripped, so - # the bytes match by construction — including tool schemas — and keep matching - # when the prompt changes. - systemd.services.ollama-primer = { - description = "Prefix-cache primer in front of ollama"; - wantedBy = [ "multi-user.target" ]; - after = [ "ollama.service" ]; - wants = [ "ollama.service" ]; - environment = { - PRIMER_UPSTREAM = "http://127.0.0.1:${toString config.services.ollama.port}"; - PRIMER_HOST = "0.0.0.0"; # So libvirt guests can reach it at 192.168.122.1. - PRIMER_PORT = "11434"; - }; - serviceConfig = { - ExecStart = "${pkgs.python3}/bin/python3 ${./ollama-primer.py}"; - Restart = "always"; - RestartSec = 2; - DynamicUser = true; - NoNewPrivileges = true; - PrivateDevices = true; - ProtectSystem = "strict"; - ProtectHome = true; - RestrictAddressFamilies = [ - "AF_INET" - "AF_INET6" - ]; - }; - }; - # Voice assistant. With `prefer_local_intents` on (a per-pipeline setting, # off by default) the built-in sentence matcher answers anything it # recognises without involving the model, so "turn off the kitchen light" diff --git a/nixos/nixos/ollama-primer.py b/nixos/nixos/ollama-primer.py deleted file mode 100644 index 8535aec..0000000 --- a/nixos/nixos/ollama-primer.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -"""Reverse proxy for ollama that keeps the system prefix cached. - -llama-server reuses a cached sequence only when the new prompt extends it, so -after a conversation leaves [system][user][assistant] in the slot, the next -conversation's [system][user'] diverges and the whole system prompt is -recomputed -- measured at 0.65 s against 0.19 s here, on every command. - -After each /api/chat we replay that exact request with the conversation -stripped: system message and tools only. That is a prefix of what is cached, so -it costs ~0.1 s, and it leaves the slot holding exactly the prefix, so the next -conversation extends it. Replaying the caller's own payload rather than -rebuilding it means the bytes match by construction, including tool schemas, -and it keeps matching when Home Assistant's prompt changes. -""" -import json, os, threading, urllib.request -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - -UPSTREAM = os.environ.get("PRIMER_UPSTREAM", "http://127.0.0.1:11435") -HOST = os.environ.get("PRIMER_HOST", "127.0.0.1") -PORT = int(os.environ.get("PRIMER_PORT", "11434")) - -_lock = threading.Lock() - -def prime(payload): - sys_msgs = [m for m in payload.get("messages", []) if m.get("role") == "system"] - if not sys_msgs or not payload.get("model"): - return - body = {"model": payload["model"], "messages": sys_msgs, "stream": False, - "options": {**payload.get("options", {}), "num_predict": 1}, - "keep_alive": payload.get("keep_alive", -1)} - if "tools" in payload: - body["tools"] = payload["tools"] - req = urllib.request.Request(UPSTREAM + "/api/chat", - data=json.dumps(body).encode(), method="POST") - req.add_header("Content-Type", "application/json") - with _lock: # never race a real request - try: - urllib.request.urlopen(req, timeout=300).read() - except Exception: - pass - -class Handler(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - def log_message(self, *a): pass - - def _relay(self, method): - body = None - n = int(self.headers.get("Content-Length") or 0) - if n: - body = self.rfile.read(n) - req = urllib.request.Request(UPSTREAM + self.path, data=body, method=method) - for k, v in self.headers.items(): - if k.lower() not in ("host", "content-length", "connection", "accept-encoding"): - req.add_header(k, v) - try: - with _lock: - pass # wait for any in-flight prime to finish - r = urllib.request.urlopen(req, timeout=3600) - except urllib.error.HTTPError as e: - data = e.read() - self.send_response(e.code) - self.send_header("Content-Type", e.headers.get("Content-Type", "application/json")) - self.send_header("Content-Length", str(len(data))) - self.end_headers(); self.wfile.write(data) - return - except Exception: - self.send_response(502); self.send_header("Content-Length", "0") - self.end_headers() - return - - self.send_response(r.status) - self.send_header("Content-Type", r.headers.get("Content-Type", "application/json")) - self.send_header("Transfer-Encoding", "chunked") # stream through - self.end_headers() - try: - while True: - chunk = r.read(4096) - if not chunk: - break - self.wfile.write(b"%X\r\n%s\r\n" % (len(chunk), chunk)) - self.wfile.flush() - self.wfile.write(b"0\r\n\r\n"); self.wfile.flush() - except Exception: - return - - if method == "POST" and self.path.rstrip("/") == "/api/chat" and body: - try: - payload = json.loads(body) - except Exception: - return - threading.Thread(target=prime, args=(payload,), daemon=True).start() - - def do_GET(self): self._relay("GET") - def do_POST(self): self._relay("POST") - def do_DELETE(self): self._relay("DELETE") - def do_HEAD(self): self._relay("HEAD") - -if __name__ == "__main__": - ThreadingHTTPServer((HOST, PORT), Handler).serve_forever() diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 3d193dd..2f6e6aa 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -1,6 +1,6 @@ # Why fresh conversations re-prefill the whole system prompt -Status: **root cause found, fix not yet applied or measured.** +Status: **root cause found; fix 1 applied, measurement pending.** Every Home Assistant command that starts a new conversation costs ~0.65 s of prompt evaluation instead of ~0.19 s, because llama-server discards a cached @@ -91,10 +91,29 @@ only a system prefix prefilled 9, 10 and 10 tokens. - Compare like with like: a prompt without `tools` is ~780 tokens against ~1850 with them. -## What the priming proxy is for +## Baselines to compare a fix against -`ollama-primer.py` replays each request with the conversation stripped, leaving -the slot holding just the prefix, so the next request is a strict extension and -needs no truncation at all. It works — 0.19 s — and sidesteps the checkpoint -problem rather than fixing it. If a fix above lands, the proxy, the port shuffle -and the back-to-back contention all go away together. +Measured through the primer on `qwen3.6:27b-mtp-q8_0`, which does not affect +generation: + +- **Generation: 77 tok/s median** (with MTP). This is the number fix 1 puts at + risk. If it drops toward ~45 tok/s, checkpoints were load-bearing for + speculative decoding and fix 2 is the one to take. +- **Prefill: ~152 ms for a 25-token prompt**, i.e. the fixed per-request + overhead. A perfect cache hit cannot read below roughly this. + +## What the priming proxy was for + +`ollama-primer.py` replayed each request with the conversation stripped, leaving +the slot holding just the prefix, so the next request was a strict extension and +needed no truncation at all. It worked — 0.19 s — and sidestepped the checkpoint +problem rather than fixing it. + +It is now **deleted**, along with the port shuffle: ollama is back on 11434, +bound to 0.0.0.0 so the agent VM can measure it directly. Note that this +deletion was a precondition for measuring fix 1 honestly, not just cleanup — a +running primer holds the slot in exactly the state the fix is meant to produce, +so any measurement taken with it in the path is unfalsifiable (trap 1). + +If fix 1 has to be reverted, the proxy is recoverable from git history rather +than worth rewriting. From 9dc6c598712033576996305d3736fef70ea3cc5a Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 02:45:14 +0000 Subject: [PATCH 27/98] Space context checkpoints at 128 tokens instead of 8192 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit was wrong. Setting -ctxcp to 0 does not fall back to plain longest-common-prefix reuse; on this model it removes reuse entirely, and every request re-prefills from zero — 1.8 s, even for a byte-exact append. qwen35 has full attention only every 4th layer. The other three quarters carry a recurrent state, which has no per-position structure to truncate, so the context reports COMMON_CONTEXT_SEQ_RM_TYPE_FULL and checkpoints are not an optimization over ordinary prefix reuse — they are the only prefix reuse there is. /api/show reports full_attention_interval = 4 in one call, which would have settled this much earlier. So the defect is the default spacing: checkpoint_min_step is 8192 tokens and the entire prompt is 1848, so no checkpoint is ever created near the end of the system prefix, which is where every new conversation diverges. At 128 the worst-case rollback is 128 tokens and the default 32 checkpoints still cover 4096 tokens of history. They live in host RAM, not VRAM — ~152 MiB each, against 128 GiB. Also disproved en route: --swa-full is irrelevant (no sliding window here), and the env-var channel to llama-server does work, which until now had only been assumed. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 30 ++++++---- nixos/nixos/prefix-cache-findings.md | 90 +++++++++++++++++++--------- 2 files changed, 80 insertions(+), 40 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index ff966e1..7878eea 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -159,18 +159,26 @@ # ollama passes its own environment through to llama-server, so LLAMA_ARG_* # reaches it with no patch. # - # Without this, a new conversation costs ~0.65 s of prefill instead of - # ~0.19 s. llama-server finds the prefix reusable (1836 of 1848 tokens) - # but cannot truncate the KV cache at an arbitrary position, because this - # context reports COMMON_CONTEXT_SEQ_RM_TYPE_FULL. It rolls back to the - # nearest context checkpoint instead — 824 — and re-prefills the 1011 - # valid tokens in between. Setting this to 0 removes the rollback, leaving - # a plain longest-common-prefix reuse. + # qwen35 has full attention only every 4th layer (full_attention_interval + # = 4); the other three quarters are linear layers holding a recurrent + # state. A recurrent state cannot be truncated to an arbitrary position, + # only snapshotted or reset, so the context reports + # COMMON_CONTEXT_SEQ_RM_TYPE_FULL and context checkpoints are the *only* + # prefix-reuse mechanism there is. Setting -ctxcp to 0 does not fall back + # to plain longest-common-prefix reuse, it removes reuse altogether: + # measured 1.8 s of prefill on every request, even a byte-exact append. # - # Checkpoints exist to let speculative decoding work on a FULL-only - # context, so this is exactly the knob that might cost MTP. See - # prefix-cache-findings.md. - LLAMA_ARG_CTX_CHECKPOINTS = "0"; + # What is actually wrong is the default spacing of 8192 tokens. The whole + # Home Assistant prompt is 1848 tokens, so after the first checkpoint the + # server declines to make another one anywhere useful, and a new + # conversation rolls back to a checkpoint near the start of the prefix and + # re-prefills ~1000 tokens. 128 puts the worst-case rollback at 128 tokens + # (~70 ms) and, with the default of 32 checkpoints, covers 4096 tokens of + # history — comfortably more than a voice conversation. + # + # Checkpoints live in host RAM, not VRAM: ~152 MiB each, so ~4.8 GB of the + # 128 GiB at the default count. See prefix-cache-findings.md. + LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT = "128"; }; # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 2f6e6aa..e7c8cf8 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -1,11 +1,25 @@ -# Why fresh conversations re-prefill the whole system prompt - -Status: **root cause found; fix 1 applied, measurement pending.** +Status: **root cause found; fix applied, measurement pending.** Every Home Assistant command that starts a new conversation costs ~0.65 s of prompt evaluation instead of ~0.19 s, because llama-server discards a cached prefix it has already identified as reusable. +## Why this model can only reuse a prefix via checkpoints + +`general.architecture` is `qwen35`, and `qwen35.full_attention_interval = 4`: +only every 4th layer is full attention. The other three quarters are linear +layers carrying a **recurrent state**, not a KV cache. + +A recurrent state has no per-position structure to truncate. You can advance it, +snapshot it, or reset it — nothing else. So the context reports +`COMMON_CONTEXT_SEQ_RM_TYPE_FULL` ("can seq_rm full sequences only"), and +**context checkpoints are the only prefix-reuse mechanism that exists here.** +They are not an optimization layered on top of ordinary reuse; they are it. + +This is the fact that made the earlier rounds of this investigation +unintelligible, and it is worth holding onto: `/api/show` reports it in one +call, and it explains the behaviour that looked like a caching bug. + ## The mechanism, from llama-server's own log ollama runs llama-server with `--log-verbosity 4`, so this is in @@ -19,34 +33,44 @@ ollama runs llama-server with `--log-verbosity 4`, so this is in cached n_tokens = 825, memory_seq_rm [825, end) prompt eval time = 667.37 ms / 1023 tokens -The slot selector finds **1836 of 1848 tokens** in common — it knows almost the -whole prompt is already cached. Then the context-checkpoint machinery takes -over. It cannot truncate the sequence at an arbitrary position; it can only roll -back to a checkpoint. The only checkpoints are at 824 and 1843. 1843 is past the -divergence point, so it falls back to 824 and re-prefills 1023 tokens. +The slot selector finds **1836 of 1848 tokens** in common. The server then has +to land on a checkpoint at or before 1836. The only ones are 824 and 1843; 1843 +is past the divergence point, so it falls back to 824 and re-prefills 1023 +tokens. -Checkpoints are enabled because the context reports -`COMMON_CONTEXT_SEQ_RM_TYPE_FULL` ("can seq_rm full sequences only"), which -makes the server log "speculative decoding will use checkpoints". Partial -truncation degrades to checkpoint granularity, and checkpoints are sparse -because each one costs ~152 MiB. +## The fix: checkpoint spacing, not checkpoint removal -## Candidate fixes, untested +`checkpoint_min_step` defaults to **8192 tokens** — the minimum spacing between +checkpoints. Our entire prompt is 1848 tokens. After the first checkpoint the +server declines to create another one anywhere near the end of the system +prefix, which is exactly where every new conversation diverges. -Both are environment variables. ollama passes `cmd.Env = os.Environ()` to the -llama-server subprocess, so `services.ollama.environmentVariables` reaches it -with no patch. +Applied: `LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT=128`. Worst-case rollback becomes +128 tokens (~70 ms at the measured ~1800 tok/s prefill rate), and the default 32 +checkpoints then cover 4096 tokens of history. -1. `LLAMA_ARG_CTX_CHECKPOINTS=0` — disable checkpoints entirely. Should let the - server truncate at 1836 and prefill ~12 tokens. Risk: checkpoints exist to - support speculative decoding on a FULL-only context, so this may disable or - degrade MTP, which is worth ~1.7x on generation. Measure both prefill and - tokens/sec before keeping it. -2. `LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT=` — keep checkpoints but space - them closely, so a rollback loses less. Costs VRAM: ~152 MiB each, up to 32. +Checkpoints are `std::vector` in `common_prompt_checkpoint`, i.e. **host +RAM, not VRAM** — ~152 MiB each, ~4.8 GB at the default count, against 128 GiB. +An earlier version of this document said VRAM; that was wrong, and it made the +budget look tight when it is free. -Both bindings verified present in `common/arg.cpp` at llama.cpp b10380, which is -what ollama 0.32.13 vendors. +## Tried and rejected: `LLAMA_ARG_CTX_CHECKPOINTS=0` + +This was the first candidate, on the theory that removing checkpoints would +leave plain longest-common-prefix reuse. It does the opposite. With no +checkpoints the restore path finds nothing to land on, sets `do_reset`, and +re-prefills from zero — **every request, including a byte-exact append**: + + turn 0: 3117.4 ms / 5784 tok + turn 4: 3282.5 ms / 5872 tok + +against a ~150 ms floor. Generation did rise from 77 to 91 tok/s, so creating +checkpoints does cost something on the generation path, but not a fraction of +what the prefill regression costs. + +The useful by-product: this proved the env-var channel works. ollama passes +`cmd.Env = os.Environ()` to llama-server, so `services.ollama.environmentVariables` +reaches `LLAMA_ARG_*` with no patch. That had been assumed, never demonstrated. ## Ruled out — do not retry these @@ -59,6 +83,9 @@ Each was tested, not reasoned about: and `update_cache && prompt_cache` then short-circuits). Measured: no change. - **`--slot-prompt-similarity`.** Already effectively enabled — the log shows `selected slot by LCP similarity`. Selection was never the problem. +- **`--swa-full` / `LLAMA_ARG_SWA_FULL`.** Irrelevant: this model has no sliding + window. `--ctx-checkpoints` carries `--swa-checkpoints` as an alias, which + makes checkpoints look like an SWA feature; they serve recurrent state too. - **MTP / the draft head.** The non-MTP `qwen3.6:27b-q4_K_M` is equally slow. - **`--mmproj` / vision.** A projector-stripped variant built with `/api/create` (capabilities lost `vision`, kept `tools` and `thinking`) is @@ -76,6 +103,11 @@ new prompt to extend the cached one. It computes longest common prefix. Verified standalone: three fresh conversations sharing only a system prefix prefilled 9, 10 and 10 tokens. +Note that standalone test used an ordinary transformer, where the common prefix +is the whole story. On qwen35 the common prefix is computed the same way and +then *discarded* down to a checkpoint. Reproducing a cache question on a +different architecture proves nothing about this one. + ## Measurement traps that produced false results here - **A synthetic system prompt is not Home Assistant's.** If HA sends anything @@ -96,9 +128,9 @@ only a system prefix prefilled 9, 10 and 10 tokens. Measured through the primer on `qwen3.6:27b-mtp-q8_0`, which does not affect generation: -- **Generation: 77 tok/s median** (with MTP). This is the number fix 1 puts at - risk. If it drops toward ~45 tok/s, checkpoints were load-bearing for - speculative decoding and fix 2 is the one to take. +- **Generation: 77 tok/s median** (with MTP, at the default checkpoint spacing). + 91 tok/s with checkpoints disabled entirely, which is the ceiling denser + spacing trades against. - **Prefill: ~152 ms for a 25-token prompt**, i.e. the fixed per-request overhead. A perfect cache hit cannot read below roughly this. From 8e1d6f03aaa984f2571589d68d5ac86867a6bcad Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 02:50:29 +0000 Subject: [PATCH 28/98] Record the measured result: 1770 ms -> 437 ms fresh prefill Also corrects two things in the findings. Checkpoints are placed at user message boundaries, not on a grid; checkpoint_min_step only throttles that placement, so the fix un-suppresses the checkpoint at the end of the system prefix rather than laying down a denser lattice. And the claim that checkpoints cost generation speed is withdrawn: 92.8 tok/s with the fix, against 91 with checkpoints off. The 77 tok/s that comparison rested on was measured on a 28-token prompt. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 55 ++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index e7c8cf8..f22cd3b 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -1,4 +1,5 @@ -Status: **root cause found; fix applied, measurement pending.** +Status: **fixed.** Fresh-conversation prefill 1770 ms -> 437 ms, against a +185 ms floor. One unexplained 250 ms remains; see the last section. Every Home Assistant command that starts a new conversation costs ~0.65 s of prompt evaluation instead of ~0.19 s, because llama-server discards a cached @@ -38,12 +39,19 @@ to land on a checkpoint at or before 1836. The only ones are 824 and 1843; 1843 is past the divergence point, so it falls back to 824 and re-prefills 1023 tokens. -## The fix: checkpoint spacing, not checkpoint removal +## The fix: stop suppressing the checkpoint we want -`checkpoint_min_step` defaults to **8192 tokens** — the minimum spacing between -checkpoints. Our entire prompt is 1848 tokens. After the first checkpoint the -server declines to create another one anywhere near the end of the system -prefix, which is exactly where every new conversation diverges. +Checkpoints are **not** placed on a grid. The server parses the rendered chat +into role spans and snapshots at the start of each user message +(`spans.is_user_start`, `common/chat.h:174`), plus once near the end of the +prompt. Those are already the positions worth saving: the boundary just after +system+tools, and the end of the conversation so far. + +`checkpoint_min_step` is a throttle on that placement, not the placement itself +— take this user-boundary checkpoint only if it is at least min_step past the +last one. It defaults to **8192 tokens** while our whole prompt is 1848, so it +suppressed the checkpoint at the end of the system prefix, which is exactly +where every new conversation diverges. Applied: `LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT=128`. Worst-case rollback becomes 128 tokens (~70 ms at the measured ~1800 tok/s prefill rate), and the default 32 @@ -64,9 +72,9 @@ re-prefills from zero — **every request, including a byte-exact append**: turn 0: 3117.4 ms / 5784 tok turn 4: 3282.5 ms / 5872 tok -against a ~150 ms floor. Generation did rise from 77 to 91 tok/s, so creating -checkpoints does cost something on the generation path, but not a fraction of -what the prefill regression costs. +against a ~150 ms floor. Generation read 91 tok/s here, but that is not evidence +that checkpoints cost generation: with the fix applied it reads 92.8 tok/s. The +77 tok/s baseline was taken on a ~28-token prompt and does not compare. The useful by-product: this proved the env-var channel works. ollama passes `cmd.Env = os.Environ()` to llama-server, so `services.ollama.environmentVariables` @@ -128,9 +136,9 @@ different architecture proves nothing about this one. Measured through the primer on `qwen3.6:27b-mtp-q8_0`, which does not affect generation: -- **Generation: 77 tok/s median** (with MTP, at the default checkpoint spacing). - 91 tok/s with checkpoints disabled entirely, which is the ceiling denser - spacing trades against. +- **Generation: 92.8 tok/s** after the fix, so checkpoint spacing costs nothing + measurable on the generation path. (An earlier 77 tok/s figure was taken on a + ~28-token prompt and is not comparable.) - **Prefill: ~152 ms for a 25-token prompt**, i.e. the fixed per-request overhead. A perfect cache hit cannot read below roughly this. @@ -149,3 +157,26 @@ so any measurement taken with it in the path is unfalsifiable (trap 1). If fix 1 has to be reverted, the proxy is recoverable from git history rather than worth rewriting. + +## What is not solved + +Two things, neither blocking. + +**No pinning.** Eviction is FIFO (`erase(checkpoints.begin())`) with no way to +mark a checkpoint permanent. The ideal — pin the system prefix forever, keep one +more for the live conversation — has no expression in the API. 32 checkpoints at +128 minimum spacing is enough headroom that a voice conversation never evicts +the prefix, but that is headroom, not a guarantee. + +**An unexplained 250 ms.** A fresh conversation costs 437 ms against a 185 ms +floor. If it were landing on the user-boundary checkpoint at the end of the +prefix, the rollback would be ~0 tokens and the gap should be tens of ms. So +either restoring a checkpoint is itself expensive (a state copy back into the +context), or it is landing on an earlier checkpoint than intended. The log +distinguishes these — it reports both checkpoint sizes and which one is +restored: + + journalctl -u ollama --since "10 min ago" | grep -i checkpoint + +If it is restore cost, lowering min_step further will not help and the remaining +win would have to come from not needing a rollback at all. From 02c997be0bf9cda89edb56160fc89828334c31d3 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 03:04:19 +0000 Subject: [PATCH 29/98] Tell llama-server where messages begin, via an ollama patch Checkpoint placement was never really about spacing. llama-server has two placements: is_user_start, at the beginning of each message, and near_prompt_end as a fallback whose window is one n_ubatch. We were only ever getting the fallback, so a fresh conversation rolled back up to 512 tokens before the divergence point and re-prefilled them. is_user_start is directly controllable: message_delimiters is a field on the /completion request, and llama-server tokenizes those strings and scans the prompt for them. Give it the user delimiter and it checkpoints at the end of the system prompt and tool definitions, which is exactly where a new conversation diverges. ollama never sends the field, because it renders qwen's template in Go and posts a flat string. The patch adds an optional MessageDelimiters() method to the renderer interface, implements it for Qwen35Renderer, and threads it through llm.CompletionRequest. It applies cleanly to v0.32.13, which is what nixpkgs ships, and go build plus the renderers and llm tests pass. This should reach the ~185 ms floor and makes the num_batch trade unnecessary -- that only tuned the fallback's granularity, at 13% of bulk prefill throughput. Not taken. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 12 +- nixos/nixos/ollama-message-delimiters.patch | 148 ++++++++++++++++++++ nixos/nixos/prefix-cache-findings.md | 70 ++++++--- 3 files changed, 208 insertions(+), 22 deletions(-) create mode 100644 nixos/nixos/ollama-message-delimiters.patch diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 7878eea..a5da6ea 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -146,7 +146,17 @@ # https://wiki.nixos.org/wiki/Ollama services.ollama = { enable = true; - package = pkgs.ollama-cuda; + # llama-server can place a context checkpoint exactly at the start of each + # message, but only if it is told where messages begin: it scans the prompt + # for delimiter strings passed in the request's "message_delimiters" field. + # The chat-completions path fills that in from the chat template. ollama + # renders qwen's template itself in Go and posts a flat string to + # /completion, leaving the field empty, so llama-server sees an opaque + # prompt and falls back to checkpointing near the end of it. The patch has + # renderers report their delimiters and passes them through. + package = pkgs.ollama-cuda.overrideAttrs (old: { + patches = (old.patches or [ ]) ++ [ ./ollama-message-delimiters.patch ]; + }); # 0.0.0.0 so libvirt guests can reach it at 192.168.122.1; the firewall # only opens 11434 on virbr0. host = "0.0.0.0"; diff --git a/nixos/nixos/ollama-message-delimiters.patch b/nixos/nixos/ollama-message-delimiters.patch new file mode 100644 index 0000000..8b9abf8 --- /dev/null +++ b/nixos/nixos/ollama-message-delimiters.patch @@ -0,0 +1,148 @@ +diff --git a/api/types.go b/api/types.go +index c0e771d4..29502293 100644 +--- a/api/types.go ++++ b/api/types.go +@@ -191,6 +191,14 @@ func (t Tool) String() string { + return string(bts) + } + ++// MessageDelimiter is the string that begins a message with the given role in ++// a rendered prompt. llama-server scans the prompt for these so that it can ++// place context checkpoints at message boundaries. ++type MessageDelimiter struct { ++ Role string `json:"role"` ++ Delimiter string `json:"delimiter"` ++} ++ + // Message is a single message in a chat sequence. The message contains the + // role ("system", "user", or "assistant"), the content and an optional list + // of images. +diff --git a/llm/llama_server.go b/llm/llama_server.go +index 1b13a5b0..a251391d 100644 +--- a/llm/llama_server.go ++++ b/llm/llama_server.go +@@ -1410,6 +1410,8 @@ type llamaServerCompletionRequest struct { + JsonSchema json.RawMessage `json:"json_schema,omitempty"` + NProbs int `json:"n_probs,omitempty"` + PreservedTokens []string `json:"preserved_tokens,omitempty"` ++ ++ MessageDelimiters []api.MessageDelimiter `json:"message_delimiters,omitempty"` + } + + func llamaServerPreservedTokens(parserTokens []string, toolCallTag string) []string { +@@ -1572,6 +1574,8 @@ func (s *llamaServerRunner) Completion(ctx context.Context, req CompletionReques + TypicalP: req.Options.TypicalP, + Seed: req.Options.Seed, + PreservedTokens: llamaServerPreservedTokens(req.PreservedTokens, req.ToolCallTag), ++ ++ MessageDelimiters: req.MessageDelimiters, + } + + if req.Logprobs { +diff --git a/llm/server.go b/llm/server.go +index a11b7f83..5d1378cd 100644 +--- a/llm/server.go ++++ b/llm/server.go +@@ -216,6 +216,12 @@ type CompletionRequest struct { + ToolCallTag string // raw generic tool parser tag, if any + LeadingBOS string // textual BOS emitted by Go rendering, if any + ++ // MessageDelimiters tells llama-server where messages begin in the rendered ++ // prompt, so it can place context checkpoints at those boundaries. Without ++ // it, a Go-rendered prompt is opaque and checkpoints land near the end of ++ // the prompt instead; ignored by non-llama-server runners. ++ MessageDelimiters []api.MessageDelimiter ++ + // Logprobs specifies whether to include log probabilities in the response + Logprobs bool + +diff --git a/model/renderers/qwen35.go b/model/renderers/qwen35.go +index 8b9e5797..4e30446b 100644 +--- a/model/renderers/qwen35.go ++++ b/model/renderers/qwen35.go +@@ -72,6 +72,14 @@ func (r *Qwen35Renderer) LeadingBOS() string { + return "" + } + ++// MessageDelimiters reports where a user message begins. llama-server places a ++// context checkpoint at each one, which puts a checkpoint at the end of the ++// system prompt and tool definitions -- exactly where a new conversation ++// diverges from the previous one. ++func (r *Qwen35Renderer) MessageDelimiters() []api.MessageDelimiter { ++ return []api.MessageDelimiter{{Role: "user", Delimiter: imStartTag + "user\n"}} ++} ++ + func (r *Qwen35Renderer) renderContent(content api.Message, imageOffset int) (string, int) { + if r.useImgTags { + return renderContentWithImageTags(content.Content, len(content.Images), imageOffset) +diff --git a/model/renderers/renderer.go b/model/renderers/renderer.go +index 93d3b0ea..e9ab74ee 100644 +--- a/model/renderers/renderer.go ++++ b/model/renderers/renderer.go +@@ -52,6 +52,29 @@ func LeadingBOSForRenderer(name string) string { + return renderer.LeadingBOS() + } + ++// messageDelimiterRenderer is implemented by renderers that can say where each ++// message begins in their output. Renderers that do not implement it simply ++// contribute no delimiters, and llama-server falls back to placing checkpoints ++// near the end of the prompt. ++type messageDelimiterRenderer interface { ++ MessageDelimiters() []api.MessageDelimiter ++} ++ ++// MessageDelimitersForRenderer returns the message-start delimiters for a ++// renderer, for llama-server's context checkpointing. ++func MessageDelimitersForRenderer(name string) []api.MessageDelimiter { ++ renderer := rendererForName(name) ++ if renderer == nil { ++ return nil ++ } ++ ++ if d, ok := renderer.(messageDelimiterRenderer); ok { ++ return d.MessageDelimiters() ++ } ++ ++ return nil ++} ++ + func rendererForName(name string) Renderer { + if constructor, ok := registry.renderers[name]; ok { + return constructor() +diff --git a/server/routes.go b/server/routes.go +index 3a027a9e..a414800c 100644 +--- a/server/routes.go ++++ b/server/routes.go +@@ -667,6 +667,8 @@ func (s *Server) GenerateHandler(c *gin.Context) { + TopLogprobs: req.TopLogprobs, + PreservedTokens: preservedTokensForCompletion(builtinParser), + LeadingBOS: leadingBOS, ++ ++ MessageDelimiters: messageDelimitersForModel(m), + }, func(cr llm.CompletionResponse) { + res := api.GenerateResponse{ + Model: req.Model, +@@ -2315,6 +2317,14 @@ func leadingBOSForModel(m *Model) string { + return renderers.LeadingBOSForRenderer(resolveRendererName(m)) + } + ++func messageDelimitersForModel(m *Model) []api.MessageDelimiter { ++ if m == nil || m.Config.Renderer == "" { ++ return nil ++ } ++ ++ return renderers.MessageDelimitersForRenderer(resolveRendererName(m)) ++} ++ + func optionsForPrompt(opts *api.Options, runner llm.LlamaServer) *api.Options { + if opts == nil || runner == nil { + return opts +@@ -2763,6 +2773,8 @@ func (s *Server) ChatHandler(c *gin.Context) { + PreservedTokens: preservedTokensForCompletion(builtinParser), + ToolCallTag: toolCallTagForCompletion(toolParser), + LeadingBOS: leadingBOSForModel(m), ++ ++ MessageDelimiters: messageDelimitersForModel(m), + }, func(r llm.CompletionResponse) { + res := api.ChatResponse{ + Model: req.Model, diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index f22cd3b..cf7a4e0 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -1,5 +1,6 @@ -Status: **fixed.** Fresh-conversation prefill 1770 ms -> 437 ms, against a -185 ms floor. One unexplained 250 ms remains; see the last section. +Status: **fixed, with an ollama patch pending a rebuild.** Fresh-conversation +prefill went 1770 ms -> 437 ms from checkpoint spacing alone; the patch should +take it to the ~185 ms floor. Verify after rebuilding. Every Home Assistant command that starts a new conversation costs ~0.65 s of prompt evaluation instead of ~0.19 s, because llama-server discards a cached @@ -158,25 +159,52 @@ so any measurement taken with it in the path is unfalsifiable (trap 1). If fix 1 has to be reverted, the proxy is recoverable from git history rather than worth rewriting. -## What is not solved +## Telling the engine exactly where to checkpoint -Two things, neither blocking. +The remaining 437 - 185 ms was **not** restore cost. The log showed every +checkpoint landing at a `near_prompt_end` position, 512 apart (2759 and 3271 for +a 3272-token prompt), because `near_prompt_end` opens a window of one +`n_ubatch`. The system prefix ends at 3261, just before 3271, so it fell back to +2760 and re-prefilled ~500 tokens. + +That is the *fallback* placement. The intended one is `is_user_start`, and it is +directly controllable: `message_delimiters` is a field on the `/completion` +request (`server-context.cpp:4150` at b10380, the revision ollama pins): + + auto delimiters = common_chat_msg_delimiters_parse(json_value(data, "message_delimiters", json::array())); + delimiters.tokenize(ctx_server.vocab); + ... + task.params.message_spans = task.tokens.find_message_spans(delimiters); + +Spans are not derived from structured chat data. llama-server tokenizes the +delimiter strings you hand it and scans the prompt for them. Give it +`<|im_start|>user\n` and it puts a checkpoint at the end of everything before +the first user message -- the system prompt and tool definitions -- which is +exactly where a new conversation diverges. + +ollama never sends the field. It renders qwen's template in Go and posts a flat +string to `/completion` (`llm/llama_server.go:5`), and its +`llamaServerCompletionRequest` has no such field, so llama-server parses +`json::array()` and finds no spans. + +`ollama-message-delimiters.patch` adds it: an optional `MessageDelimiters()` +method on the renderer interface, implemented for `Qwen35Renderer`, threaded +through `llm.CompletionRequest` into the request body. Applies cleanly to +v0.32.13, which is what nixpkgs ships; `go build` and the `renderers` and `llm` +test packages pass. + +### The n_ubatch red herring + +Because the fallback window is one `n_ubatch`, shrinking it does buy prefill: +437 ms at 512, 310 ms at 256, 262 ms at 128, against bulk prefill dropping from +1769 to 1534 tok/s. That is a real trade, and it is the wrong thing to trade: it +tunes the granularity of the fallback instead of using the mechanism that exists +for saying where the checkpoint goes. **Do not set num_batch for this.** + +## What is still not solved **No pinning.** Eviction is FIFO (`erase(checkpoints.begin())`) with no way to -mark a checkpoint permanent. The ideal — pin the system prefix forever, keep one -more for the live conversation — has no expression in the API. 32 checkpoints at -128 minimum spacing is enough headroom that a voice conversation never evicts -the prefix, but that is headroom, not a guarantee. - -**An unexplained 250 ms.** A fresh conversation costs 437 ms against a 185 ms -floor. If it were landing on the user-boundary checkpoint at the end of the -prefix, the rollback would be ~0 tokens and the gap should be tens of ms. So -either restoring a checkpoint is itself expensive (a state copy back into the -context), or it is landing on an earlier checkpoint than intended. The log -distinguishes these — it reports both checkpoint sizes and which one is -restored: - - journalctl -u ollama --since "10 min ago" | grep -i checkpoint - -If it is restore cost, lowering min_step further will not help and the remaining -win would have to come from not needing a rollback at all. +mark a checkpoint permanent. The ideal -- pin the system prefix forever, keep +one more for the live conversation -- still has no expression in the API. 32 +checkpoints at 128 minimum spacing is enough headroom that a voice conversation +never evicts the prefix, but that is headroom, not a guarantee. From 49843e19d245401d2564575ffb5c80204b9a7b95 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 03:12:59 +0000 Subject: [PATCH 30/98] Verify the delimiters patch: 437 ms -> 192 ms fresh prefill At the per-request floor now, with generation unaffected at 94.4 tok/s. The follow-up turn regressed 185 -> 275 ms, which is recorded but not yet explained; a two-turn exchange is still net faster. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index cf7a4e0..2479125 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -1,6 +1,20 @@ -Status: **fixed, with an ollama patch pending a rebuild.** Fresh-conversation -prefill went 1770 ms -> 437 ms from checkpoint spacing alone; the patch should -take it to the ~185 ms floor. Verify after rebuilding. +Status: **fixed and verified.** Fresh-conversation prefill 1770 ms -> 437 ms +from checkpoint spacing, then -> 192 ms with the delimiters patch, which is the +per-request floor. Generation 94.4 tok/s, no regression. + +| | original | spacing fix | + delimiters patch | +|---|---|---|---| +| fresh conversation | 1770 ms | 437 ms | **192 ms** | +| follow-up turn | 1831 ms | 185 ms | **275 ms** | +| generation | 90.7 tok/s | 92.8 | **94.4** | + +The follow-up turn got *worse*, 185 -> 275 ms, and that is not yet explained. +The likely cause is that a new user message now triggers a checkpoint save +(~162 MiB of state), but that does not obviously square with a fresh +conversation costing only 192 ms while also starting a user message. The log +would settle it. It is a good trade either way: a two-turn exchange went +437 + 185 = 622 ms to 192 + 275 = 467 ms, and voice commands are mostly +single-turn. Every Home Assistant command that starts a new conversation costs ~0.65 s of prompt evaluation instead of ~0.19 s, because llama-server discards a cached From 875b972b0ff2dfb9e26c0b88030514862221b2b7 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 03:17:24 +0000 Subject: [PATCH 31/98] Explain the follow-up regression: checkpoint saves, not prefill A fresh conversation restores one checkpoint and saves two; a follow-up saves three, because its prompt has two user delimiters plus a near_prompt_end. That extra save is the whole 185 -> 275 ms. Saves cost ~65 ms each and restores are nearly free, which also means the "fixed per-request overhead" recorded earlier was mostly misattributed -- it was checkpoint saves, not HTTP and tokenization. Confirmed by holding the machinery fixed and varying the prompt: 6298 tokens costs 198 ms and 568 tokens costs 179 ms. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 46 +++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 2479125..993f091 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -8,13 +8,10 @@ per-request floor. Generation 94.4 tok/s, no regression. | follow-up turn | 1831 ms | 185 ms | **275 ms** | | generation | 90.7 tok/s | 92.8 | **94.4** | -The follow-up turn got *worse*, 185 -> 275 ms, and that is not yet explained. -The likely cause is that a new user message now triggers a checkpoint save -(~162 MiB of state), but that does not obviously square with a fresh -conversation costing only 192 ms while also starting a user message. The log -would settle it. It is a good trade either way: a two-turn exchange went -437 + 185 = 622 ms to 192 + 275 = 467 ms, and voice commands are mostly -single-turn. +The follow-up turn got *worse*, 185 -> 275 ms. Explained below: it saves one +more checkpoint than a fresh conversation does. Still a good trade -- a two-turn +exchange went 437 + 185 = 622 ms to 192 + 275 = 467 ms, and voice commands are +mostly single-turn. Every Home Assistant command that starts a new conversation costs ~0.65 s of prompt evaluation instead of ~0.19 s, because llama-server discards a cached @@ -222,3 +219,38 @@ mark a checkpoint permanent. The ideal -- pin the system prefix forever, keep one more for the live conversation -- still has no expression in the API. 32 checkpoints at 128 minimum spacing is enough headroom that a voice conversation never evicts the prefix, but that is headroom, not a guarantee. + +## Where the remaining ~190 ms goes + +Not prefill. The cost is nearly independent of prompt length: + +| system prompt | fresh prefill | +|---|---| +| 6298 tokens | 198 ms | +| 568 tokens | 179 ms | + +It is **checkpoint creation**. The recurrent state is a fixed ~162 MiB +regardless of context length, and saving one means copying that device->host. +From the log, a fresh conversation restores one checkpoint and creates two; a +follow-up restores one and creates three, because its prompt contains two user +delimiters plus a `near_prompt_end`. Solving across the two cases: + +- **checkpoint save: ~65 ms** (2 saves = 192 ms fresh, 3 saves = 275 ms follow-up) +- **checkpoint restore: single-digit ms** — the cheap direction, host->device +- everything else: ~50 ms + +So restores are nearly free and saves dominate. This also means the earlier +"~0.15 s of fixed per-request overhead" in the traps section was mostly +misattributed: it was checkpoint saves, not HTTP and tokenization. + +### The only remaining lever is architectural + +For single-turn voice, the `near_prompt_end` checkpoint is pure overhead — we +always restore the user-boundary one. Suppressing it would save ~65 ms per +command. But `near_prompt_end` has no flag: it is exempt from +`checkpoint_min_step` and unaffected by `n_ctx_checkpoints`, so this needs a +llama.cpp patch, not a configuration change, and it would trade away multi-turn +performance (that checkpoint is exactly what a follow-up restores). + +Not worth it at ~65 ms against a 1.4-3.2 s voice interaction. Recorded so the +next person does not go looking for a config knob that does not exist. \ No newline at end of file From b45b2fc662db3e61f9a1aa6309854db4f6729a10 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 03:21:13 +0000 Subject: [PATCH 32/98] Note the unused ON_DEVICE flag and the nixpkgs llama.cpp pin The 65 ms per checkpoint is a device->host copy that llama.cpp already has a flag to avoid, LLAMA_STATE_SEQ_FLAGS_ON_DEVICE, which no caller uses. Only one on-device snapshot exists per seq_id, so two VRAM-resident checkpoints would need two sequences -- the two-slot design, and an upstream llama.cpp change rather than a flag. Also: nixpkgs builds llama.cpp b10091 while ollama 0.32.13 pins b10380, despite a comment claiming the pin tracks upstream. Our patch is unaffected, but b10091 is the revision to read for this host. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 42 ++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 993f091..7cf745c 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -243,7 +243,31 @@ So restores are nearly free and saves dominate. This also means the earlier "~0.15 s of fixed per-request overhead" in the traps section was mostly misattributed: it was checkpoint saves, not HTTP and tokenization. -### The only remaining lever is architectural +### Keeping checkpoints in VRAM + +The 65 ms is a device->host copy, and nothing requires that. llama.cpp has the +flag already, in `include/llama.h`: + + // Keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load). + // Getting the state for a seq_id with this flag invalidates all prior states gotten for that seq_id with this flag. + #define LLAMA_STATE_SEQ_FLAGS_ON_DEVICE 2 + +It is a bit flag; the server passes only `PARTIAL_ONLY` (= 1). `grep ON_DEVICE +tools/server/ common/` returns nothing — implemented in the core, never wired +up. A save would become a device->device copy via `llama_io_write_device`, +sub-millisecond instead of 65 ms. + +The catch is the second comment line. `mem_storage` is +`std::map` and each `get_data` rebuilds the +entry, so there is exactly **one on-device snapshot per seq_id**. Two +VRAM-resident checkpoints need two sequence ids — the two-slot design, with a +real mechanism behind it. ollama passes `-np 1`. + +This is an upstream llama.cpp change, not a flag and not an ollama patch: the +checkpoint deque holds up to 32 entries and would need to know which one gets +the single on-device slot per sequence. Worth ~130 ms of the 192 ms. + +### The other lever is architectural For single-turn voice, the `near_prompt_end` checkpoint is pure overhead — we always restore the user-boundary one. Suppressing it would save ~65 ms per @@ -253,4 +277,18 @@ llama.cpp patch, not a configuration change, and it would trade away multi-turn performance (that checkpoint is exactly what a follow-up restores). Not worth it at ~65 ms against a 1.4-3.2 s voice interaction. Recorded so the -next person does not go looking for a config knob that does not exist. \ No newline at end of file +next person does not go looking for a config knob that does not exist. + +## nixpkgs builds a different llama.cpp than ollama pins + +`pkgs/by-name/ol/ollama/package.nix`: + + # Pre-stage the pin (tracks upstream's `LLAMA_CPP_VERSION` file) ... + llamaCppVersion = "b10091"; + +ollama 0.32.13's `LLAMA_CPP_VERSION` says **b10380**. The comment claims the pin +tracks upstream; it is roughly 290 builds behind. + +Our patch is unaffected — `message_delimiters` exists in both, which is why the +measurement came out right — but **read b10091 when reasoning about behaviour on +this host**, not ollama's pin and not llama.cpp master. All three differ. From 5d6d2ef94ab84f42258cda1c2904ad5efa4cdabd Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 03:26:14 +0000 Subject: [PATCH 33/98] Stop re-taking a checkpoint we just restored from Half the remaining prefill cost was llama-server snapshotting a position it already had. A request that restores a checkpoint resumes decoding at the position it restored to, that batch starts at a user-message boundary, so it takes a checkpoint there -- byte-identical to the one it just loaded. On this model that is a ~162 MiB copy out of the device, about 65 ms, on every command. The dedup is safe rather than clever: same n_tokens and same pos range means the same prefix was reused to get there, so the state is the same. Skipping the copy cannot change what the model sees. Patched into llama.cpp rather than ollama. nixpkgs pre-stages llama.cpp into $TMPDIR/llama-cpp-src at the end of postPatch so FetchContent does not hit the network, which is where we hook in. Verified: builds clean at b10091 (llama-server links), ollama's own compat patch does not touch server-context.cpp, and the patch applies to the realized llamaCppSrc. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 9 ++++++++ nixos/nixos/llama-checkpoint-dedup.patch | 27 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 nixos/nixos/llama-checkpoint-dedup.patch diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index a5da6ea..b0905c4 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -154,8 +154,17 @@ # /completion, leaving the field empty, so llama-server sees an opaque # prompt and falls back to checkpointing near the end of it. The patch has # renderers report their delimiters and passes them through. + # + # The second patch is against llama.cpp, not ollama. nixpkgs pre-stages + # llama.cpp into $TMPDIR/llama-cpp-src at the end of postPatch so the + # CMake FetchContent step does not reach the network, which gives us a + # place to patch it. Note that nixpkgs pins b10091 while ollama 0.32.13 + # asks for b10380, so read b10091 when reasoning about this host. package = pkgs.ollama-cuda.overrideAttrs (old: { patches = (old.patches or [ ]) ++ [ ./ollama-message-delimiters.patch ]; + postPatch = (old.postPatch or "") + '' + patch -d "$TMPDIR/llama-cpp-src" -p1 < ${./llama-checkpoint-dedup.patch} + ''; }); # 0.0.0.0 so libvirt guests can reach it at 192.168.122.1; the firewall # only opens 11434 on virbr0. diff --git a/nixos/nixos/llama-checkpoint-dedup.patch b/nixos/nixos/llama-checkpoint-dedup.patch new file mode 100644 index 0000000..93b1df8 --- /dev/null +++ b/nixos/nixos/llama-checkpoint-dedup.patch @@ -0,0 +1,27 @@ +diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp +index 744593c76..9a60632a9 100644 +--- a/tools/server/server-context.cpp ++++ b/tools/server/server-context.cpp +@@ -2294,6 +2294,22 @@ private: + void create_checkpoint(server_slot & slot, const int64_t n_tokens_cur, llama_pos pos_min, llama_pos pos_max) { + const int id_task = slot.task->id; + ++ const int64_t n_tokens_new = slot.prompt.n_tokens() - n_tokens_cur; ++ ++ // if a checkpoint for this exact position already exists, the state is ++ // identical and re-taking it is pure cost -- a state copy out of the ++ // device, which for a recurrent model is hundreds of MiB. this happens ++ // on every request that restores a checkpoint and then resumes from it, ++ // because the batch that resumes starts at the position we restored to. ++ for (const auto & cur : slot.prompt.checkpoints) { ++ if (cur.n_tokens == n_tokens_new && cur.pos_min == pos_min && cur.pos_max == pos_max) { ++ SLT_TRC(slot, "reusing context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", ++ cur.pos_min, cur.pos_max, cur.n_tokens, (float) cur.size() / 1024 / 1024); ++ ++ return; ++ } ++ } ++ + // evict checkpoints within min-step of a previous checkpoint, unless they were + // created by the current task + int64_t last = -1; From 7e6e9b77170848047f78fded1acee534e58415d9 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 03:33:47 +0000 Subject: [PATCH 34/98] Measure the dedup patch: 192 ms -> 158 ms, and find the real floor A repeated identical request costs ~56 ms, which is the irreducible per-request cost. So a fresh conversation at 158 ms still carries ~100 ms of checkpoint save and restore. Also withdraws the per-save cost estimate. Solving for it from differences between measurement types gave 65 ms, then 35 ms, then 85 ms, because the cases vary in token count and checkpoint count at once. Only the measured totals and the floor are trustworthy. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 35 ++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 7cf745c..ca26d89 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -2,11 +2,14 @@ Status: **fixed and verified.** Fresh-conversation prefill 1770 ms -> 437 ms from checkpoint spacing, then -> 192 ms with the delimiters patch, which is the per-request floor. Generation 94.4 tok/s, no regression. -| | original | spacing fix | + delimiters patch | -|---|---|---|---| -| fresh conversation | 1770 ms | 437 ms | **192 ms** | -| follow-up turn | 1831 ms | 185 ms | **275 ms** | -| generation | 90.7 tok/s | 92.8 | **94.4** | +| | original | spacing fix | + delimiters | + dedup | +|---|---|---|---|---| +| fresh conversation | 1770 ms | 437 ms | 192 ms | **158 ms** | +| follow-up turn | 1831 ms | 185 ms | 275 ms | **239 ms** | +| generation | 90.7 tok/s | 92.8 | 94.4 | **87.1** | + +Generation varies +/- 8% run to run (77 to 94 across the session on identical +prompts), so none of that column is a signal beyond "no regression". The follow-up turn got *worse*, 185 -> 275 ms. Explained below: it saves one more checkpoint than a fresh conversation does. Still a good trade -- a two-turn @@ -292,3 +295,25 @@ tracks upstream; it is roughly 290 builds behind. Our patch is unaffected — `message_delimiters` exists in both, which is why the measurement came out right — but **read b10091 when reasoning about behaviour on this host**, not ollama's pin and not llama.cpp master. All three differ. + + +## The real per-request floor is ~56 ms + +Send the *same* tiny prompt repeatedly. After the first, there is nothing new to +evaluate and (with the dedup patch) no new checkpoint position, so what is left +is the cost of being a request at all: + + identical request 0: prefill 141.3 ms / 15 tok <- new prompt, takes a checkpoint + identical request 1: prefill 55.9 ms / 15 tok + identical request 2: prefill 56.9 ms / 15 tok + identical request 3: prefill 55.1 ms / 15 tok + +So **~56 ms is irreducible** without changing ollama or llama.cpp more deeply, +and a fresh conversation at 158 ms still carries ~100 ms of checkpoint work: one +save plus one restore. + +Do not try to attribute that 100 ms more finely than this. Several attempts at +solving for per-save and per-restore costs from differences between measurement +types gave inconsistent answers (65 ms, then 35 ms, then 85 ms) because the +cases differ in token count and checkpoint count at the same time. The reliable +statements are the four measured totals in the table and the ~56 ms floor. From 5ab3c70c52b4f706a0236156baffe68f10c51b77 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 03:57:01 +0000 Subject: [PATCH 35/98] Make the cache boundary configurable, and put the time after it The delimiter is not "the user tag", it is "the end of the part of the prompt that does not vary". <|im_start|>user was only the right boundary while nothing before it changed. Putting the current time at the end of the system prompt moves the boundary earlier, so the delimiter moves with it rather than the timestamp moving somewhere awkward. OLLAMA_MESSAGE_DELIMITERS overrides the renderer default with a JSON array, so the boundary is a setting rather than a recompile. Declare exactly one: adding the marker alongside the user tag would put a second checkpoint after the timestamp, invalidated and re-saved every request. This avoids both of Home Assistant options -- templating the time into the cached region, or spending a round trip on GetDateTimeTool. Unverified: matching is on token sequences, so a plain-text marker only works if it tokenizes the same alone as in context. Needs measuring. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 22 ++++++++++ nixos/nixos/ollama-message-delimiters.patch | 31 ++++++++++++-- nixos/nixos/prefix-cache-findings.md | 45 +++++++++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index b0905c4..c240a18 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -198,6 +198,28 @@ # Checkpoints live in host RAM, not VRAM: ~152 MiB each, so ~4.8 GB of the # 128 GiB at the default count. See prefix-cache-findings.md. LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT = "128"; + + # Where the invariant part of the prompt ends. llama-server puts the + # checkpoint immediately before this string, so it has to sit after + # everything that is the same on every request (the instructions, the tool + # definitions, the entity list) and before everything that is not. + # + # The renderer's default is the start of the user's message, which is the + # right boundary only while nothing before it varies. It does now: the + # system prompt ends with the current time, so that the model knows the + # time without having to spend a whole extra round trip calling + # GetDateTimeTool for it. Naming that line as the boundary keeps the + # cacheable part cacheable and costs a re-prefill of the line itself. + # + # So the Home Assistant prompt must END with, exactly: + # + # Current time: {{ now().strftime('%Y-%m-%d %H:%M') }} + # + # Matching is on token sequences, not text, so a marker only works if it + # tokenizes the same alone as it does in context. Verify by measurement + # after changing it: a fresh conversation should prefill in ~160 ms, and + # jumps to ~1800 ms if the marker stops matching. + OLLAMA_MESSAGE_DELIMITERS = builtins.toJSON [ "Current time:" ]; }; # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. diff --git a/nixos/nixos/ollama-message-delimiters.patch b/nixos/nixos/ollama-message-delimiters.patch index 8b9abf8..0e7204f 100644 --- a/nixos/nixos/ollama-message-delimiters.patch +++ b/nixos/nixos/ollama-message-delimiters.patch @@ -110,7 +110,7 @@ index 93d3b0ea..e9ab74ee 100644 if constructor, ok := registry.renderers[name]; ok { return constructor() diff --git a/server/routes.go b/server/routes.go -index 3a027a9e..a414800c 100644 +index 3a027a9e..f6768bb6 100644 --- a/server/routes.go +++ b/server/routes.go @@ -667,6 +667,8 @@ func (s *Server) GenerateHandler(c *gin.Context) { @@ -122,11 +122,36 @@ index 3a027a9e..a414800c 100644 }, func(cr llm.CompletionResponse) { res := api.GenerateResponse{ Model: req.Model, -@@ -2315,6 +2317,14 @@ func leadingBOSForModel(m *Model) string { +@@ -2315,6 +2317,39 @@ func leadingBOSForModel(m *Model) string { return renderers.LeadingBOSForRenderer(resolveRendererName(m)) } ++// OllamaMessageDelimiters overrides the renderer's delimiters with a JSON array ++// of strings, e.g. ["Current time:"]. llama-server places a context checkpoint ++// immediately before each one, so the right value is whatever marks the end of ++// the part of the prompt that does not change between requests. The renderer's ++// default is the start of the user's message, which is only the right boundary ++// while nothing before it varies -- a per-request timestamp at the end of the ++// system prompt moves the boundary earlier. ++const OllamaMessageDelimiters = "OLLAMA_MESSAGE_DELIMITERS" ++ +func messageDelimitersForModel(m *Model) []api.MessageDelimiter { ++ if s := strings.TrimSpace(os.Getenv(OllamaMessageDelimiters)); s != "" { ++ var raw []string ++ if err := json.Unmarshal([]byte(s), &raw); err != nil { ++ slog.Warn("ignoring malformed "+OllamaMessageDelimiters, "error", err) ++ } else { ++ delims := make([]api.MessageDelimiter, 0, len(raw)) ++ for _, d := range raw { ++ if d != "" { ++ delims = append(delims, api.MessageDelimiter{Role: "user", Delimiter: d}) ++ } ++ } ++ ++ return delims ++ } ++ } ++ + if m == nil || m.Config.Renderer == "" { + return nil + } @@ -137,7 +162,7 @@ index 3a027a9e..a414800c 100644 func optionsForPrompt(opts *api.Options, runner llm.LlamaServer) *api.Options { if opts == nil || runner == nil { return opts -@@ -2763,6 +2773,8 @@ func (s *Server) ChatHandler(c *gin.Context) { +@@ -2763,6 +2798,8 @@ func (s *Server) ChatHandler(c *gin.Context) { PreservedTokens: preservedTokensForCompletion(builtinParser), ToolCallTag: toolCallTagForCompletion(toolParser), LeadingBOS: leadingBOSForModel(m), diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index ca26d89..a7dfafe 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -207,6 +207,29 @@ through `llm.CompletionRequest` into the request body. Applies cleanly to v0.32.13, which is what nixpkgs ships; `go build` and the `renderers` and `llm` test packages pass. +### The delimiter is not "the user tag", it is "the end of the invariant part" + +`<|im_start|>user\n` is not special here — it was just the string that happened +to mark the boundary while nothing before it varied. The rule is: **place the +delimiter wherever the prompt stops being identical between requests.** + +Put a per-request timestamp at the end of the system prompt and the boundary +moves earlier, so the delimiter has to move with it — not the timestamp. Hence +`OLLAMA_MESSAGE_DELIMITERS`, a JSON array of strings that overrides the +renderer's default without a recompile. + +Declare exactly one. Declaring both the marker and the user tag would put a +second checkpoint *after* the timestamp, which is invalidated and re-saved every +request — the ~100 ms this is all trying to avoid. + +**Matching is on token sequences, not text** (`std::equal` over the tokenized +delimiter, `common/chat.cpp`). `<|im_start|>` was safe because a special token +always tokenizes as one unit. A plain-text marker only works if it tokenizes the +same standalone as it does in context, and BPE gives no guarantee of that — +merging with a preceding newline is the obvious hazard. So a marker change must +be checked by measurement: a fresh conversation prefills in ~160 ms when the +marker matches and ~1800 ms when it does not. + ### The n_ubatch red herring Because the fallback window is one `n_ubatch`, shrinking it does buy prefill: @@ -282,6 +305,28 @@ performance (that checkpoint is exactly what a follow-up restores). Not worth it at ~65 ms against a 1.4-3.2 s voice interaction. Recorded so the next person does not go looking for a config knob that does not exist. +## Date and time + +Neither of Home Assistant's built-in options is good. Templating the time into +the system prompt puts it inside the cached region and invalidates it on every +request; `GetDateTimeTool` costs an entire extra round trip through the model. + +The third option is to state the time in the prompt but put it *after* the cache +boundary, and move the delimiter to match. The system prompt ends with + + Current time: {{ now().strftime('%Y-%m-%d %H:%M') }} + +and `OLLAMA_MESSAGE_DELIMITERS` is `["Current time:"]`. The invariant part — +instructions, tool definitions, entity list — is cached; the timestamp line and +the question are re-prefilled, which is a few dozen tokens. + +The qwen35 renderer emits system content last (reasoning instructions, then +tools, then the configured prompt), so a trailing line in Home Assistant's +prompt really is the last thing before the user's turn. + +Note the prompt itself lives in Home Assistant's storage, not in this repo, so +it is not captured by a rebuild and has to be edited in the UI. + ## nixpkgs builds a different llama.cpp than ollama pins `pkgs/by-name/ol/ollama/package.nix`: From 48a72a92cc62d706c975ffc686143a5d1ad251ed Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 04:03:44 +0000 Subject: [PATCH 36/98] Anchor the cache boundary on special tokens, not on prompt text A plain-text marker was wrong twice over: matching is on token sequences, so it only works if the marker tokenizes the same standalone as it does in context, and ordinary prompt text could contain it by accident. <|im_start|> was safe because it is a single special token, and that is the property we want rather than an incidental detail. So the current time becomes its own system block, emitted by the renderer right after the invariant part of the prompt, and the marker is "<|im_end|>\n<|im_start|>system\n". That cannot be spelled by prompt text and cannot match at position 0, where no message precedes the first one. OLLAMA_TIME_FORMAT is a Go layout and enables it; unset keeps upstream behaviour. Emitting it in the renderer rather than in Home Assistant's prompt keeps the marker and the thing it marks in the same repo. Home Assistant's prompt lives in its storage, outside any rebuild, which is precisely how those two would drift apart. Also drops LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT. It existed because the 8192 default left no checkpoint near the end of the prefix; now that we name the boundary, the checkpoint we want is the last "user" span, which is exempt from the spacing throttle. Removed on that reasoning, to be confirmed from the log. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 60 ++++++++++----------- nixos/nixos/ollama-message-delimiters.patch | 36 ++++++++++++- nixos/nixos/prefix-cache-findings.md | 34 ++++++++---- 3 files changed, 85 insertions(+), 45 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index c240a18..716a443 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -175,9 +175,6 @@ # The assistant has to stay resident; a 30 s reload before "turn off the # lights" is the difference between usable and infuriating. OLLAMA_KEEP_ALIVE = "-1"; - # ollama passes its own environment through to llama-server, so LLAMA_ARG_* - # reaches it with no patch. - # # qwen35 has full attention only every 4th layer (full_attention_interval # = 4); the other three quarters are linear layers holding a recurrent # state. A recurrent state cannot be truncated to an arbitrary position, @@ -186,40 +183,37 @@ # prefix-reuse mechanism there is. Setting -ctxcp to 0 does not fall back # to plain longest-common-prefix reuse, it removes reuse altogether: # measured 1.8 s of prefill on every request, even a byte-exact append. + # Checkpoints live in host RAM, not VRAM, ~162 MiB each. # - # What is actually wrong is the default spacing of 8192 tokens. The whole - # Home Assistant prompt is 1848 tokens, so after the first checkpoint the - # server declines to make another one anywhere useful, and a new - # conversation rolls back to a checkpoint near the start of the prefix and - # re-prefills ~1000 tokens. 128 puts the worst-case rollback at 128 tokens - # (~70 ms) and, with the default of 32 checkpoints, covers 4096 tokens of - # history — comfortably more than a voice conversation. - # - # Checkpoints live in host RAM, not VRAM: ~152 MiB each, so ~4.8 GB of the - # 128 GiB at the default count. See prefix-cache-findings.md. - LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT = "128"; - - # Where the invariant part of the prompt ends. llama-server puts the - # checkpoint immediately before this string, so it has to sit after - # everything that is the same on every request (the instructions, the tool - # definitions, the entity list) and before everything that is not. - # - # The renderer's default is the start of the user's message, which is the - # right boundary only while nothing before it varies. It does now: the - # system prompt ends with the current time, so that the model knows the - # time without having to spend a whole extra round trip calling - # GetDateTimeTool for it. Naming that line as the boundary keeps the - # cacheable part cacheable and costs a re-prefill of the line itself. + # LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT used to be set to 128 here, because + # the 8192 default left no checkpoint anywhere near the end of the prefix. + # Now that we say where the boundary is, the checkpoint we care about is + # the last "user" span in the prompt, and the last one is exempt from the + # spacing throttle -- so the default should be fine. Removed on that + # reasoning, which the log will confirm or refute. # - # So the Home Assistant prompt must END with, exactly: + # Where the invariant part of the prompt ends. llama-server takes a + # context checkpoint immediately before this string, so it has to sit + # after everything that is identical on every request (the instructions, + # the tool definitions, the entity list) and before everything that is + # not. # - # Current time: {{ now().strftime('%Y-%m-%d %H:%M') }} + # It is made of special tokens on purpose. Matching is on token sequences, + # so a plain-text marker only works if it tokenizes the same standalone as + # it does in context, and prompt text could contain it by accident. This + # cannot be spelled by prompt text, and cannot match at position 0, where + # no message precedes the first one. + OLLAMA_MESSAGE_DELIMITERS = builtins.toJSON [ "<|im_end|>\n<|im_start|>system\n" ]; + + # What goes after that boundary: the current time, as its own system + # block, emitted by the renderer. A Go time layout; unset disables it. + # Minutes are the useful granularity -- seconds would change the prompt + # on every request for no benefit. # - # Matching is on token sequences, not text, so a marker only works if it - # tokenizes the same alone as it does in context. Verify by measurement - # after changing it: a fresh conversation should prefill in ~160 ms, and - # jumps to ~1800 ms if the marker stops matching. - OLLAMA_MESSAGE_DELIMITERS = builtins.toJSON [ "Current time:" ]; + # With this set, drop "Call GetDateTimeTool for the current date or time." + # from the Home Assistant prompt; the model no longer needs a round trip + # to find out what time it is. + OLLAMA_TIME_FORMAT = "2006-01-02 15:04"; }; # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. diff --git a/nixos/nixos/ollama-message-delimiters.patch b/nixos/nixos/ollama-message-delimiters.patch index 0e7204f..d40a58f 100644 --- a/nixos/nixos/ollama-message-delimiters.patch +++ b/nixos/nixos/ollama-message-delimiters.patch @@ -57,10 +57,20 @@ index a11b7f83..5d1378cd 100644 Logprobs bool diff --git a/model/renderers/qwen35.go b/model/renderers/qwen35.go -index 8b9e5797..4e30446b 100644 +index 8b9e5797..3d95b9a3 100644 --- a/model/renderers/qwen35.go +++ b/model/renderers/qwen35.go -@@ -72,6 +72,14 @@ func (r *Qwen35Renderer) LeadingBOS() string { +@@ -3,7 +3,9 @@ package renderers + import ( + "fmt" + "log/slog" ++ "os" + "strings" ++ "time" + + "github.com/ollama/ollama/api" + ) +@@ -72,6 +74,14 @@ func (r *Qwen35Renderer) LeadingBOS() string { return "" } @@ -75,6 +85,28 @@ index 8b9e5797..4e30446b 100644 func (r *Qwen35Renderer) renderContent(content api.Message, imageOffset int) (string, int) { if r.useImgTags { return renderContentWithImageTags(content.Content, len(content.Images), imageOffset) +@@ -272,6 +282,21 @@ func (r *Qwen35Renderer) Render(messages []api.Message, tools []api.Tool, think + sb.WriteString(imStartTag + "system\n" + reasoningInstructions + imEndTag + "\n") + } + ++ // Emit the current time as its own system block, right after the part of ++ // the prompt that is identical on every request. Keeping it out of the ++ // system message is the point: llama-server takes a context checkpoint at ++ // "<|im_end|>\n<|im_start|>system\n", so everything above stays cached and ++ // only the time and the question are re-evaluated. A marker made of special ++ // tokens cannot be spelled by prompt text and cannot match at position 0, ++ // where nothing precedes the first message. See OLLAMA_MESSAGE_DELIMITERS. ++ // ++ // The alternatives are both worse: templating the time into the system ++ // prompt invalidates the cache on every request, and answering "what time is ++ // it" with a tool call costs an entire extra round trip through the model. ++ if layout := os.Getenv("OLLAMA_TIME_FORMAT"); layout != "" { ++ sb.WriteString(imStartTag + "system\nCurrent time: " + time.Now().Format(layout) + imEndTag + "\n") ++ } ++ + multiStepTool := true + lastQueryIndex := len(messages) - 1 // so this is the last user message + diff --git a/model/renderers/renderer.go b/model/renderers/renderer.go index 93d3b0ea..e9ab74ee 100644 --- a/model/renderers/renderer.go diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index a7dfafe..51aef16 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -316,16 +316,30 @@ boundary, and move the delimiter to match. The system prompt ends with Current time: {{ now().strftime('%Y-%m-%d %H:%M') }} -and `OLLAMA_MESSAGE_DELIMITERS` is `["Current time:"]`. The invariant part — -instructions, tool definitions, entity list — is cached; the timestamp line and -the question are re-prefilled, which is a few dozen tokens. - -The qwen35 renderer emits system content last (reasoning instructions, then -tools, then the configured prompt), so a trailing line in Home Assistant's -prompt really is the last thing before the user's turn. - -Note the prompt itself lives in Home Assistant's storage, not in this repo, so -it is not captured by a rebuild and has to be edited in the UI. +**A plain-text marker was the wrong call.** Matching is on token sequences +(`std::equal` over the tokenized delimiter), so a text marker only works if it +tokenizes the same standalone as in context — and worse, ordinary prompt text +could contain it by accident. `<|im_start|>` was safe precisely because it is a +single special token, and that property is the requirement, not an incidental +detail. + +So the time is emitted as **its own system block**, by the renderer, right after +the invariant part: + + ...<|im_end|>\n<|im_start|>system\nCurrent time: 2026-08-17 03:41<|im_end|>\n<|im_start|>user\n... + +with the marker `<|im_end|>\n<|im_start|>system\n`. That cannot be spelled by +prompt text, and it cannot match at position 0, where no message precedes the +first one. `OLLAMA_TIME_FORMAT` is a Go time layout and turns the block on; +unset leaves upstream behaviour. + +Putting it in the renderer rather than in Home Assistant's prompt keeps the +marker and the thing it marks in one repo. Home Assistant's prompt lives in its +storage, outside any rebuild, which is exactly how the two would drift apart. + +Still to verify by measurement: whether `system\n` tokenizes identically before +`Current` as it does standalone. If not, the fallback is to shorten the marker, +since the checkpoint lands at the delimiter's *first* token either way. ## nixpkgs builds a different llama.cpp than ollama pins From 16dfa521ce39ae91b898875abd055a62f0d2acb9 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 04:14:37 +0000 Subject: [PATCH 37/98] Drop GetDateTimeTool from the assistant prompt The renderer now states the current time in its own system block after the cache boundary, so the model no longer has to spend a round trip asking what time it is. Verified through the pipeline: "It is night right now. The current time is 12:13 AM." and "It's 13 minutes past midnight." Applied to the live prompt in Home Assistant storage as well, via the subentry reconfigure flow; this comment is only the record of it. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 716a443..fac5178 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -279,7 +279,6 @@ # # Your training data is out of date and your memory of current facts is # wrong. - # Call GetDateTimeTool for the current date or time. # Call GetLiveContext for the state of anything in this house, # including the weather. # Call search_the_web for news, current events, prices, sports, or who From 5c1db88ea58131130b7c442a33512f44d59ee424 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 04:20:02 +0000 Subject: [PATCH 38/98] Fix the marker: it has to end on a special token The previous marker never matched. Delimiter matching compares token sequences and BPE merges across ordinary text: "system\n" is a single token standalone and splits when text follows it, so "<|im_end|>\n<|im_start|>system\n" could not appear in the prompt as tokenized. Fresh conversations fell back to end-of-prompt checkpoints, 158 ms -> 527 ms. Measured rather than reasoned this time, using prompt_eval_count as a tokenizer oracle: for a single user message the prompt is a constant plus the content's tokens, so tokens(x+y) == tokens(x) + tokens(y) answers whether a junction merges. Every text tail tried merges. Special tokens are atomic, and "<|im_end|>\n<|im_start|>" is additive against system, user and assistant turns alike. It matches every message boundary rather than only ours, which the dedup patch makes free: a boundary whose content has not changed already has a checkpoint and is skipped. That stays correct under a per-minute timestamp because the server erases every checkpoint past the divergence before continuing, so a dedup match always refers to an identical prefix. Also sets min_step to 0 rather than dropping it. It is not only a creation throttle: it drives an eviction pass that erases checkpoints within min_step of an earlier one, which at 8192 is all of them. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 37 ++++++++-------- nixos/nixos/ollama-message-delimiters.patch | 16 ++++--- nixos/nixos/prefix-cache-findings.md | 48 +++++++++++++++++++-- 3 files changed, 75 insertions(+), 26 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index fac5178..f830621 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -185,25 +185,28 @@ # measured 1.8 s of prefill on every request, even a byte-exact append. # Checkpoints live in host RAM, not VRAM, ~162 MiB each. # - # LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT used to be set to 128 here, because - # the 8192 default left no checkpoint anywhere near the end of the prefix. - # Now that we say where the boundary is, the checkpoint we care about is - # the last "user" span in the prompt, and the last one is exempt from the - # spacing throttle -- so the default should be fine. Removed on that - # reasoning, which the log will confirm or refute. + # 0 is "no minimum". This knob throttles checkpoint creation, but it also + # drives an eviction pass that erases any checkpoint within min_step of an + # earlier one -- and at the 8192 default, every checkpoint in a 3300-token + # prompt qualifies, so the boundary checkpoint was being deleted right + # after it was made. We want to keep the ones we ask for. + LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT = "0"; + + # Where llama-server may take a context checkpoint. It scans the prompt + # for these token sequences and snapshots immediately before each match. # - # Where the invariant part of the prompt ends. llama-server takes a - # context checkpoint immediately before this string, so it has to sit - # after everything that is identical on every request (the instructions, - # the tool definitions, the entity list) and before everything that is - # not. + # This MUST end on a special token. Matching compares token sequences, and + # BPE merges across ordinary text: "system\n" is a single token on its own + # but splits when text follows, so a marker ending in text never matches. + # Measured, after shipping one that did not work. Special tokens are + # atomic, and this concatenates cleanly with every message that can follow + # it (3 + n tokens, checked against system, user and assistant turns). # - # It is made of special tokens on purpose. Matching is on token sequences, - # so a plain-text marker only works if it tokenizes the same standalone as - # it does in context, and prompt text could contain it by accident. This - # cannot be spelled by prompt text, and cannot match at position 0, where - # no message precedes the first one. - OLLAMA_MESSAGE_DELIMITERS = builtins.toJSON [ "<|im_end|>\n<|im_start|>system\n" ]; + # It therefore matches every message boundary rather than only the one we + # care about, which is fine: the checkpoint-dedup patch skips re-taking a + # snapshot at a position that already has one, so the boundaries whose + # content did not change cost nothing. + OLLAMA_MESSAGE_DELIMITERS = builtins.toJSON [ "<|im_end|>\n<|im_start|>" ]; # What goes after that boundary: the current time, as its own system # block, emitted by the renderer. A Go time layout; unset disables it. diff --git a/nixos/nixos/ollama-message-delimiters.patch b/nixos/nixos/ollama-message-delimiters.patch index d40a58f..483b0d2 100644 --- a/nixos/nixos/ollama-message-delimiters.patch +++ b/nixos/nixos/ollama-message-delimiters.patch @@ -57,7 +57,7 @@ index a11b7f83..5d1378cd 100644 Logprobs bool diff --git a/model/renderers/qwen35.go b/model/renderers/qwen35.go -index 8b9e5797..3d95b9a3 100644 +index 8b9e5797..13e6e975 100644 --- a/model/renderers/qwen35.go +++ b/model/renderers/qwen35.go @@ -3,7 +3,9 @@ package renderers @@ -85,17 +85,21 @@ index 8b9e5797..3d95b9a3 100644 func (r *Qwen35Renderer) renderContent(content api.Message, imageOffset int) (string, int) { if r.useImgTags { return renderContentWithImageTags(content.Content, len(content.Images), imageOffset) -@@ -272,6 +282,21 @@ func (r *Qwen35Renderer) Render(messages []api.Message, tools []api.Tool, think +@@ -272,6 +282,25 @@ func (r *Qwen35Renderer) Render(messages []api.Message, tools []api.Tool, think sb.WriteString(imStartTag + "system\n" + reasoningInstructions + imEndTag + "\n") } + // Emit the current time as its own system block, right after the part of + // the prompt that is identical on every request. Keeping it out of the + // system message is the point: llama-server takes a context checkpoint at -+ // "<|im_end|>\n<|im_start|>system\n", so everything above stays cached and -+ // only the time and the question are re-evaluated. A marker made of special -+ // tokens cannot be spelled by prompt text and cannot match at position 0, -+ // where nothing precedes the first message. See OLLAMA_MESSAGE_DELIMITERS. ++ // the message boundary just before it, so everything above stays cached and ++ // only the time and the question are re-evaluated. See ++ // OLLAMA_MESSAGE_DELIMITERS. ++ // ++ // The delimiter has to END on a special token. Delimiter matching compares ++ // token sequences, and BPE merges across ordinary text: "system\n" is one ++ // token alone but splits when text follows it, so a marker ending in text ++ // never matches. Special tokens are atomic and merge with nothing. + // + // The alternatives are both worse: templating the time into the system + // prompt invalidates the cache on every request, and answering "what time is diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 51aef16..3640d57 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -238,6 +238,15 @@ Because the fallback window is one `n_ubatch`, shrinking it does buy prefill: tunes the granularity of the fallback instead of using the mechanism that exists for saying where the checkpoint goes. **Do not set num_batch for this.** +## min_step must be 0, not merely small + +`checkpoint_min_step` throttles creation, but it also drives an eviction pass in +`create_checkpoint` that erases any checkpoint within min_step of an earlier +one. At the 8192 default every checkpoint in a 3300-token prompt qualifies, so +the boundary checkpoint was deleted immediately after being made. 0 is +documented as "no minimum" and is the correct value now that we say where +checkpoints belong. + ## What is still not solved **No pinning.** Eviction is FIFO (`erase(checkpoints.begin())`) with no way to @@ -337,9 +346,42 @@ Putting it in the renderer rather than in Home Assistant's prompt keeps the marker and the thing it marks in one repo. Home Assistant's prompt lives in its storage, outside any rebuild, which is exactly how the two would drift apart. -Still to verify by measurement: whether `system\n` tokenizes identically before -`Current` as it does standalone. If not, the fallback is to shorten the marker, -since the checkpoint lands at the delimiter's *first* token either way. +### The delimiter must END on a special token + +It does not. Measured with `prompt_eval_count` as a tokenizer oracle — for a +single user message the rendered prompt is a constant plus the content's tokens, +so `tokens(x)` is recoverable and `tokens(x+y) == tokens(x) + tokens(y)` answers +"does this junction merge": + + system\n tokens = 1 + 20 vs together 22 MERGES + system\n\n tokens = 1 + 20 vs together 22 MERGES + time\n tokens = 1 + 20 vs together 22 MERGES + +`system\n` is a single token standalone and splits when text follows it, so +`<|im_end|>\n<|im_start|>system\n` never matched and fresh conversations fell +back to end-of-prompt checkpoints: 158 ms -> 527 ms. + +Special tokens are atomic and merge with nothing on either side, so the marker +is now `<|im_end|>\n<|im_start|>` — 3 tokens, and additive against every message +type that can follow it: + + 3 + 22 vs 25 CLEAN + 'system\nCurrent time: ...' + 3 + 7 vs 10 CLEAN + 'user\nTurn on the lamp.' + 3 + 4 vs 7 CLEAN + 'assistant\nOkay.' + +**Any future marker must be checked this way.** Reasoning about BPE is not +enough; this one looked obviously fine and was not. + +### Why matching every boundary is acceptable + +`<|im_end|>\n<|im_start|>` matches every message boundary, not just the one we +want. That is fine because the dedup patch skips re-taking a snapshot where one +already exists, so boundaries whose content has not changed cost nothing. + +The dedup is sound even though the timestamp makes content change under a fixed +position: on any divergence the server first erases every checkpoint with +`pos_max > pos_next`, so a checkpoint that survives to be dedup-matched was +necessarily built from an identical token prefix. ## nixpkgs builds a different llama.cpp than ollama pins From 3d8f770fb06dbacc1ad52a71bef7b9f8f4b7edcd Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 04:27:05 +0000 Subject: [PATCH 39/98] Make the cache boundary a unique special token "<|im_end|>\n<|im_start|>" tokenizes stably but matches every message boundary, and a ~162 MiB snapshot is taken at each: fresh prefill 267 ms against 158 ms when the marker matched one place. Stability was necessary but not sufficient; the marker also has to be unique. <|fim_pad|> is both. It is a single special token, so it tokenizes identically wherever it appears, and the renderer emits it in exactly one place -- at the end of the invariant prompt, immediately before the current time. Every spare special token in this vocab was checked and all are single tokens; a padding token was picked because it carries no meaning the model has to interpret. The renderer constant and OLLAMA_MESSAGE_DELIMITERS have to agree, which is noted in both places. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 22 ++++++------- nixos/nixos/ollama-message-delimiters.patch | 35 +++++++++++++++------ nixos/nixos/prefix-cache-findings.md | 21 +++++++++---- 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index f830621..c3b95a2 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -195,18 +195,18 @@ # Where llama-server may take a context checkpoint. It scans the prompt # for these token sequences and snapshots immediately before each match. # - # This MUST end on a special token. Matching compares token sequences, and - # BPE merges across ordinary text: "system\n" is a single token on its own - # but splits when text follows, so a marker ending in text never matches. - # Measured, after shipping one that did not work. Special tokens are - # atomic, and this concatenates cleanly with every message that can follow - # it (3 + n tokens, checked against system, user and assistant turns). + # This MUST be a special token, and it must equal qwen35CacheBoundary in + # the renderer patch. Two properties are needed and only a special token + # has both. Matching compares token sequences and BPE merges across + # ordinary text -- "system\n" is a single token alone but splits when text + # follows, so a text marker never matches at all. And the marker must be + # unique: "<|im_end|>\n<|im_start|>" is stable but matches every message + # boundary, costing a ~162 MiB snapshot at each one. # - # It therefore matches every message boundary rather than only the one we - # care about, which is fine: the checkpoint-dedup patch skips re-taking a - # snapshot at a position that already has one, so the boundaries whose - # content did not change cost nothing. - OLLAMA_MESSAGE_DELIMITERS = builtins.toJSON [ "<|im_end|>\n<|im_start|>" ]; + # <|fim_pad|> is a padding token, so it carries no meaning the model has + # to interpret, and the renderer emits it in exactly one place: at the end + # of the invariant prompt, just before the current time. + OLLAMA_MESSAGE_DELIMITERS = builtins.toJSON [ "<|fim_pad|>" ]; # What goes after that boundary: the current time, as its own system # block, emitted by the renderer. A Go time layout; unset disables it. diff --git a/nixos/nixos/ollama-message-delimiters.patch b/nixos/nixos/ollama-message-delimiters.patch index 483b0d2..b647bff 100644 --- a/nixos/nixos/ollama-message-delimiters.patch +++ b/nixos/nixos/ollama-message-delimiters.patch @@ -57,10 +57,10 @@ index a11b7f83..5d1378cd 100644 Logprobs bool diff --git a/model/renderers/qwen35.go b/model/renderers/qwen35.go -index 8b9e5797..13e6e975 100644 +index 8b9e5797..91587380 100644 --- a/model/renderers/qwen35.go +++ b/model/renderers/qwen35.go -@@ -3,7 +3,9 @@ package renderers +@@ -3,11 +3,20 @@ package renderers import ( "fmt" "log/slog" @@ -70,7 +70,18 @@ index 8b9e5797..13e6e975 100644 "github.com/ollama/ollama/api" ) -@@ -72,6 +74,14 @@ func (r *Qwen35Renderer) LeadingBOS() string { + ++// qwen35CacheBoundary marks where the identical-on-every-request part of the ++// prompt ends, for llama-server's context checkpointing. It is a padding token, ++// chosen because it is a single special token (so it tokenizes identically ++// wherever it appears) and carries no meaning of its own. Must match ++// OLLAMA_MESSAGE_DELIMITERS. ++const qwen35CacheBoundary = "<|fim_pad|>" ++ + const ( + qwen35ThinkOpenTag = "" + qwen35ThinkCloseTag = "" +@@ -72,6 +81,14 @@ func (r *Qwen35Renderer) LeadingBOS() string { return "" } @@ -85,7 +96,7 @@ index 8b9e5797..13e6e975 100644 func (r *Qwen35Renderer) renderContent(content api.Message, imageOffset int) (string, int) { if r.useImgTags { return renderContentWithImageTags(content.Content, len(content.Images), imageOffset) -@@ -272,6 +282,25 @@ func (r *Qwen35Renderer) Render(messages []api.Message, tools []api.Tool, think +@@ -272,6 +289,31 @@ func (r *Qwen35Renderer) Render(messages []api.Message, tools []api.Tool, think sb.WriteString(imStartTag + "system\n" + reasoningInstructions + imEndTag + "\n") } @@ -96,16 +107,22 @@ index 8b9e5797..13e6e975 100644 + // only the time and the question are re-evaluated. See + // OLLAMA_MESSAGE_DELIMITERS. + // -+ // The delimiter has to END on a special token. Delimiter matching compares -+ // token sequences, and BPE merges across ordinary text: "system\n" is one -+ // token alone but splits when text follows it, so a marker ending in text -+ // never matches. Special tokens are atomic and merge with nothing. ++ // The delimiter has to BE a special token. Delimiter matching compares token ++ // sequences, and BPE merges across ordinary text: "system\n" is one token ++ // alone but splits when text follows it, so a marker ending in text never ++ // matches. Special tokens are atomic and merge with nothing. ++ // ++ // "<|im_end|>\n<|im_start|>" is stable but matches every message boundary, ++ // which costs a snapshot at each. qwen35CacheBoundary is a single special ++ // token that appears exactly here, so exactly one checkpoint is taken, at ++ // the last position that is identical on every request. + // + // The alternatives are both worse: templating the time into the system + // prompt invalidates the cache on every request, and answering "what time is + // it" with a tool call costs an entire extra round trip through the model. + if layout := os.Getenv("OLLAMA_TIME_FORMAT"); layout != "" { -+ sb.WriteString(imStartTag + "system\nCurrent time: " + time.Now().Format(layout) + imEndTag + "\n") ++ sb.WriteString(imStartTag + "system\n" + qwen35CacheBoundary + "Current time: " + ++ time.Now().Format(layout) + imEndTag + "\n") + } + multiStepTool := true diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 3640d57..d89c62b 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -372,14 +372,23 @@ type that can follow it: **Any future marker must be checked this way.** Reasoning about BPE is not enough; this one looked obviously fine and was not. -### Why matching every boundary is acceptable +### And it must be unique, not merely stable -`<|im_end|>\n<|im_start|>` matches every message boundary, not just the one we -want. That is fine because the dedup patch skips re-taking a snapshot where one -already exists, so boundaries whose content has not changed cost nothing. +`<|im_end|>\n<|im_start|>` is stable but matches *every* message boundary, and +a snapshot is taken at each: fresh prefill 267 ms against 158 ms for a marker +that matched one place. The dedup patch does not save us here, because the +boundaries in question are at positions that genuinely differ per request. -The dedup is sound even though the timestamp makes content change under a fixed -position: on any divergence the server first erases every checkpoint with +So the marker is `<|fim_pad|>` — a single special token, emitted by the renderer +in exactly one place. Every spare special token in this vocab was checked and +all tokenize as one token; a padding token was chosen because it carries no +meaning the model has to interpret. + +Both properties are required, and only a special token has both: text markers +fail to match at all, and structural markers match too often. + +The dedup remains sound even though the timestamp makes content change under a +fixed position: on any divergence the server first erases every checkpoint with `pos_max > pos_next`, so a checkpoint that survives to be dedup-matched was necessarily built from an identical token prefix. From db06043210d3d8f955f8951a5edcd4fa16af176b Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 04:36:09 +0000 Subject: [PATCH 40/98] Put the weekday in the injected time Given a bare date the model answered "Sunday, August 17, 2026" and then "Monday, August 17, 2026" for the same question on consecutive requests. It is a Monday. Deriving the weekday is arithmetic the model does unreliably, and stating it costs nothing. This was a regression from dropping GetDateTimeTool, which presumably returned the weekday. Also records the measurement mistake that surfaced it: a latency microbenchmark was running concurrently with the eval, which corrupted both -- 234 ms for an identical request, and 10.3 s for a date question that actually takes 0.5 s. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 6 +++++- nixos/nixos/prefix-cache-findings.md | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index c3b95a2..7546535 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -216,7 +216,11 @@ # With this set, drop "Call GetDateTimeTool for the current date or time." # from the Home Assistant prompt; the model no longer needs a round trip # to find out what time it is. - OLLAMA_TIME_FORMAT = "2006-01-02 15:04"; + # + # The weekday is in there because the model cannot reliably derive it: with + # a bare date it answered "Sunday, August 17, 2026" and then "Monday, + # August 17, 2026" for the same question. It is a Monday. + OLLAMA_TIME_FORMAT = "Monday, 2006-01-02 15:04"; }; # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index d89c62b..f8bc856 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -427,3 +427,22 @@ solving for per-save and per-restore costs from differences between measurement types gave inconsistent answers (65 ms, then 35 ms, then 85 ms) because the cases differ in token count and checkpoint count at the same time. The reliable statements are the four measured totals in the table and the ~56 ms floor. + + +## Measurement hygiene, again + +The trap at the top of this document ("Home Assistant must be idle") was +violated by running a latency microbenchmark concurrently with a 75-run eval. +It corrupted both: the microbenchmark read 234 ms for an identical request +(higher than a real 3300-token conversation), and the eval reported the date +scenario at 10.3 s when it actually takes 0.5-0.7 s. + +Nothing on this host measures anything while anything else is running. That +includes background tasks started earlier in the same session. + +## The time format needs the weekday + +`OLLAMA_TIME_FORMAT` states the weekday explicitly. Given only `2026-08-17` the +model answered "Sunday" and then "Monday" for the same question on consecutive +requests. It is a Monday. Deriving a weekday from a date is arithmetic the model +does unreliably, and it costs nothing to hand it over. From 2358335e00464ca343550435ebee4dda52a16571 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 04:39:34 +0000 Subject: [PATCH 41/98] Skip prompt-end checkpoints when the request says where to checkpoint near_prompt_end is a fallback for not knowing where checkpoints belong. The request now declares that, so taking one at the end of the prompt as well is pure cost: that position moves with every request, so dedup can never skip it and it copies ~162 MiB out of the device each time. Measured before this change, cleanly: a fresh conversation is 167 ms against a 57 ms floor for a repeated identical request, and the gap is a restore, ~35 new tokens, and exactly this snapshot. A minute rollover reads 158 ms, which is the same shape and a neat confirmation of the mechanism. Follow-ups lose their nearby restore point and rewind to the boundary instead, re-evaluating the question and the previous reply. That is a few dozen tokens now that the boundary is close. Also makes the injected time friendlier to say out loud: "Monday, August 17, 2026 at 12:13 AM EDT" rather than a bare 24-hour date. The layout was checked against Go rather than assumed -- "at" is passed through literally, and midnight formats as 12:13 AM. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 2 +- nixos/nixos/llama-checkpoint-dedup.patch | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 7546535..a46abf4 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -220,7 +220,7 @@ # The weekday is in there because the model cannot reliably derive it: with # a bare date it answered "Sunday, August 17, 2026" and then "Monday, # August 17, 2026" for the same question. It is a Monday. - OLLAMA_TIME_FORMAT = "Monday, 2006-01-02 15:04"; + OLLAMA_TIME_FORMAT = "Monday, January 2, 2006 at 3:04 PM MST"; }; # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. diff --git a/nixos/nixos/llama-checkpoint-dedup.patch b/nixos/nixos/llama-checkpoint-dedup.patch index 93b1df8..732833e 100644 --- a/nixos/nixos/llama-checkpoint-dedup.patch +++ b/nixos/nixos/llama-checkpoint-dedup.patch @@ -1,5 +1,5 @@ diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp -index 744593c76..9a60632a9 100644 +index 744593c76..8bbe9ac4b 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2294,6 +2294,22 @@ private: @@ -25,3 +25,22 @@ index 744593c76..9a60632a9 100644 // evict checkpoints within min-step of a previous checkpoint, unless they were // created by the current task int64_t last = -1; +@@ -3532,9 +3548,15 @@ private: + + slot.init_sampler(); + } else { +- // skip ordinary mid-prompt checkpoints, unless the batch starts a user +- // message or we are near the end of the prompt +- if (!is_user_start && !near_prompt_end) { ++ // checkpointing near the end of the prompt is a fallback for not ++ // knowing where checkpoints belong. when the request declared message ++ // delimiters, we do know, so take them at those positions and nowhere ++ // else: a prompt-end checkpoint sits at a position that moves with ++ // every request, so it can never be reused and costs a full state copy ++ // out of the device each time. ++ const bool have_spans = !slot.task->params.message_spans.spans.empty(); ++ ++ if (!is_user_start && !(near_prompt_end && !have_spans)) { + do_checkpoint = false; + } + } From 5bceef5671d9c2805ea50c7de5db8dff7568d9b1 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 04:43:09 +0000 Subject: [PATCH 42/98] Replay the empty think block in history, so follow-ups are appends A follow-up turn was rewinding, which it should never have needed to do: continuing a conversation is appending. The cause, found by rendering both turns with ollama's renderer and diffing: common tail : ...the kitchen lamp.<|im_end|>\n<|im_start|>assistant\n turn 1 then : \n\n\n\n turn 2 then : The kitchen lamp is on.<|im_end|>\n<|im_start|>user\n... With thinking off the renderer ends the generation prompt with an empty think block, so the reply physically follows it in the slot. Replaying that turn as history omitted the block, so the shared prefix ended at "assistant\n" and everything after was recomputed -- on this architecture a checkpoint rollback, not just wasted prefill. Now the turn-2 prompt has the turn-1 prompt as an exact prefix. This also retracts the earlier conclusion that think tags were not the cause. That test compared think=false against think=true with the thinking field preserved, which is a different code path, and read 247 ms against 245 ms. The paths differ in where the block is emitted, not whether it is. Co-Authored-By: Claude Opus 5 --- nixos/nixos/ollama-message-delimiters.patch | 17 ++++++++++++- nixos/nixos/prefix-cache-findings.md | 28 +++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/nixos/nixos/ollama-message-delimiters.patch b/nixos/nixos/ollama-message-delimiters.patch index b647bff..715570e 100644 --- a/nixos/nixos/ollama-message-delimiters.patch +++ b/nixos/nixos/ollama-message-delimiters.patch @@ -57,7 +57,7 @@ index a11b7f83..5d1378cd 100644 Logprobs bool diff --git a/model/renderers/qwen35.go b/model/renderers/qwen35.go -index 8b9e5797..91587380 100644 +index 8b9e5797..cfc23209 100644 --- a/model/renderers/qwen35.go +++ b/model/renderers/qwen35.go @@ -3,11 +3,20 @@ package renderers @@ -128,6 +128,21 @@ index 8b9e5797..91587380 100644 multiStepTool := true lastQueryIndex := len(messages) - 1 // so this is the last user message +@@ -309,6 +351,14 @@ func (r *Qwen35Renderer) Render(messages []api.Message, tools []api.Tool, think + + if renderAssistantThinkBlock { + sb.WriteString(imStartTag + message.Role + "\n\n" + contentReasoning + "\n\n\n" + content) ++ } else if !isThinking && r.emitEmptyThinkOnNoThink { ++ // Replay the empty think block that the generation prompt emitted ++ // just before this reply, so a past assistant turn matches the ++ // tokens the model actually produced. Without it the shared prefix ++ // ends at "assistant\n" and every later turn re-evaluates the whole ++ // conversation from there -- which on a recurrent model means a ++ // checkpoint rollback, not just some wasted prefill. ++ sb.WriteString(imStartTag + message.Role + "\n\n\n\n\n" + content) + } else { + sb.WriteString(imStartTag + message.Role + "\n" + content) + } diff --git a/model/renderers/renderer.go b/model/renderers/renderer.go index 93d3b0ea..e9ab74ee 100644 --- a/model/renderers/renderer.go diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index f8bc856..e94e34c 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -446,3 +446,31 @@ includes background tasks started earlier in the same session. model answered "Sunday" and then "Monday" for the same question on consecutive requests. It is a Monday. Deriving a weekday from a date is arithmetic the model does unreliably, and it costs nothing to hand it over. + + +## Why follow-up turns were rewinding + +They should not have been. Continuing a conversation is appending, and appending +needs no rollback. The cause was a prompt-rendering mismatch, found by rendering +both turns with ollama's own renderer and diffing them: + + common tail : ...the kitchen lamp.<|im_end|>\n<|im_start|>assistant\n + turn 1 then : \n\n\n\n + turn 2 then : The kitchen lamp is on.<|im_end|>\n<|im_start|>user\n... + +With thinking off, the qwen3.5 renderer ends the generation prompt with an empty +`\n\n\n\n` (`emitEmptyThinkOnNoThink`), so the model's reply +physically follows it in the slot. Replaying that same turn as history omits the +block, so the shared prefix ends at `assistant\n` and everything after is +recomputed — which on this architecture means a checkpoint rollback, not merely +wasted prefill. + +The patch replays the empty block in history. Verified: the turn-2 prompt now +has the turn-1 prompt as an exact prefix, diverging only where new content +begins. + +**An earlier attempt to test this concluded the opposite and was wrong.** It +compared `think=false` against `think=true` with the thinking field preserved -- +a different code path -- saw 247 ms vs 245 ms, and cleared the think tags of +suspicion. The two paths differ in where the block is emitted, not whether it is. +Render and diff the actual strings; do not infer from latency. From 44e8e1a01b365a4246ce9974ce2c34585486d4d5 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 04:54:03 +0000 Subject: [PATCH 43/98] Keep the prompt checkpoint in VRAM The last ~105 ms of a fresh conversation is one checkpoint restore, not prefill and not saves. Shown by repeating a large identical request, which never diverges and so never restores: 6326 tokens costs ~58 ms, the same as 45 tokens. Prompt length is irrelevant when nothing diverges. ON_DEVICE keeps the snapshot in device buffers rather than copying ~162 MiB across PCIe each way. llama.cpp has had the flag since #22679 and nothing in the tree uses it except a test. It allows exactly one on-device snapshot per sequence, and taking another silently invalidates the previous one -- which is why this was previously written off as too dangerous. It is safe now only because the rest of the work reduced us to a single checkpoint: one delimiter, matched once, prompt-end checkpoints suppressed. LLAMA_ARG_CTX_CHECKPOINTS=1 enforces it, and the speculative-decoding checkpoints stay on the host so nothing competes for the slot. Also withdraws the earlier claim that saves cost ~65 ms and restores were nearly free. That came from comparing measurements that differed in two things at once. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 7 ++++++ nixos/nixos/llama-checkpoint-dedup.patch | 31 ++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index a46abf4..2a1bcf8 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -192,6 +192,13 @@ # after it was made. We want to keep the ones we ask for. LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT = "0"; + # Exactly one prompt checkpoint, which is all we ask for: one delimiter, + # matched once. This is not a tuning choice, it is what makes keeping the + # checkpoint in VRAM correct -- llama.cpp allows a single on-device + # snapshot per sequence and taking another silently invalidates the + # previous one. Holding one means that can never happen. + LLAMA_ARG_CTX_CHECKPOINTS = "1"; + # Where llama-server may take a context checkpoint. It scans the prompt # for these token sequences and snapshots immediately before each match. # diff --git a/nixos/nixos/llama-checkpoint-dedup.patch b/nixos/nixos/llama-checkpoint-dedup.patch index 732833e..d822bdc 100644 --- a/nixos/nixos/llama-checkpoint-dedup.patch +++ b/nixos/nixos/llama-checkpoint-dedup.patch @@ -1,5 +1,5 @@ diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp -index 744593c76..8bbe9ac4b 100644 +index 744593c76..f69676465 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2294,6 +2294,22 @@ private: @@ -25,7 +25,34 @@ index 744593c76..8bbe9ac4b 100644 // evict checkpoints within min-step of a previous checkpoint, unless they were // created by the current task int64_t last = -1; -@@ -3532,9 +3548,15 @@ private: +@@ -2329,7 +2345,16 @@ private: + // this is not true for SWA models: https://github.com/ggml-org/llama.cpp/pull/24411#issuecomment-4677983225 + cur.update_pos(slot.prompt.n_tokens() - n_tokens_cur, pos_min, pos_max); + +- cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); ++ // ON_DEVICE keeps the snapshot in device buffers instead of copying it out ++ // to host RAM, which turns a ~162 MiB PCIe round trip into a device-side ++ // copy. llama.cpp allows exactly one such snapshot per sequence -- taking ++ // another invalidates the previous -- so this is only correct while the ++ // server holds a single prompt checkpoint. -ctxcp 1 enforces that, and the ++ // request declares one delimiter, so one is all we ask for. ++ // ++ // Only the prompt checkpoints use it. The speculative-decoding checkpoints ++ // below stay on the host, so nothing else competes for the slot. ++ cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); + cur.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + // stash the draft's speculative state with the checkpoint + common_speculative_get_state(spec.get(), slot.id, cur.data_spec); +@@ -3306,7 +3331,7 @@ private: + + if (!do_reset) { + // restore the context checkpoint +- it->load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); ++ it->load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); + it->load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + // restore the draft's speculative state + common_speculative_set_state(spec.get(), slot.id, it->data_spec); +@@ -3532,9 +3557,15 @@ private: slot.init_sampler(); } else { From 47f2b9337e32631413aa5fe7d570667946de815a Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 05:00:46 +0000 Subject: [PATCH 44/98] Drop the checkpoint cap that broke fresh conversations LLAMA_ARG_CTX_CHECKPOINTS=1 was meant to guarantee the single on-device snapshot that ON_DEVICE requires. It instead broke prefix reuse outright: fresh conversations went 163 ms to 1836 ms, which is full reprocessing, so no usable checkpoint survived between them. The cap evicts the front of the deque whenever anything is created, and evidently something is. The on-device path itself is fine -- follow-up turns improved from 152 ms to 99 ms in the same run, which is the PCIe round trip disappearing. Only one position is ever checkpointed anyway, so the invariant the cap was protecting holds without it. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 2a1bcf8..1613421 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -192,12 +192,16 @@ # after it was made. We want to keep the ones we ask for. LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT = "0"; - # Exactly one prompt checkpoint, which is all we ask for: one delimiter, - # matched once. This is not a tuning choice, it is what makes keeping the - # checkpoint in VRAM correct -- llama.cpp allows a single on-device - # snapshot per sequence and taking another silently invalidates the - # previous one. Holding one means that can never happen. - LLAMA_ARG_CTX_CHECKPOINTS = "1"; + # LLAMA_ARG_CTX_CHECKPOINTS was set to 1 here to guarantee the single + # checkpoint that ON_DEVICE requires. It broke fresh conversations + # outright -- 163 ms to 1836 ms, i.e. full reprocessing, so no usable + # checkpoint survived between conversations. The cap evicts the front of + # the deque whenever anything else is created, and evidently something is. + # + # Removed: only one position is ever checkpointed anyway (one delimiter, + # matched once, prompt-end snapshots suppressed), so the invariant holds + # without the cap. If answers ever go strange rather than slow, suspect + # this and revert the ON_DEVICE flag in llama-checkpoint-dedup.patch. # Where llama-server may take a context checkpoint. It scans the prompt # for these token sequences and snapshots immediately before each match. From 89e0b08677d1326ca26fc8b2f829f603e5999b59 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 05:04:10 +0000 Subject: [PATCH 45/98] Revert ON_DEVICE: it was answering from another conversation's state The single-checkpoint invariant did not hold. Suppressing the prompt-end checkpoint only covered mid-prompt batches; when the prompt is fully processed the code takes a different branch, which still takes one. The log shows two per request, at 3260 and 3300. Both serialize seq_id 0, so both resolve to the same mem_storage[0], and the second silently overwrites the first's data while the server still holds the first and still restores from it. Asked seconds apart at 1:01 AM, the assistant said "about 1:00 AM" and then "2 minutes past midnight". Note that latency improved while this was happening -- follow-ups read 99 ms against 152 -- so no timing measurement could have caught it. Checking the answers did. Also suppresses the end-of-prompt checkpoint in the branch that was missed, which is what the single-checkpoint invariant needed in the first place. That makes retrying ON_DEVICE possible later, after confirming from the log that exactly one checkpoint is created, and after checking output rather than timings. Co-Authored-By: Claude Opus 5 --- nixos/nixos/llama-checkpoint-dedup.patch | 66 +++++++++++++----------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/nixos/nixos/llama-checkpoint-dedup.patch b/nixos/nixos/llama-checkpoint-dedup.patch index d822bdc..b431264 100644 --- a/nixos/nixos/llama-checkpoint-dedup.patch +++ b/nixos/nixos/llama-checkpoint-dedup.patch @@ -1,5 +1,5 @@ diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp -index 744593c76..f69676465 100644 +index 744593c76..893771837 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2294,6 +2294,22 @@ private: @@ -25,48 +25,52 @@ index 744593c76..f69676465 100644 // evict checkpoints within min-step of a previous checkpoint, unless they were // created by the current task int64_t last = -1; -@@ -2329,7 +2345,16 @@ private: +@@ -2329,6 +2345,15 @@ private: // this is not true for SWA models: https://github.com/ggml-org/llama.cpp/pull/24411#issuecomment-4677983225 cur.update_pos(slot.prompt.n_tokens() - n_tokens_cur, pos_min, pos_max); -- cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); -+ // ON_DEVICE keeps the snapshot in device buffers instead of copying it out -+ // to host RAM, which turns a ~162 MiB PCIe round trip into a device-side -+ // copy. llama.cpp allows exactly one such snapshot per sequence -- taking -+ // another invalidates the previous -- so this is only correct while the -+ // server holds a single prompt checkpoint. -ctxcp 1 enforces that, and the -+ // request declares one delimiter, so one is all we ask for. -+ // -+ // Only the prompt checkpoints use it. The speculative-decoding checkpoints -+ // below stay on the host, so nothing else competes for the slot. -+ cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); ++ // NOT LLAMA_STATE_SEQ_FLAGS_ON_DEVICE. It is faster -- it keeps the ++ // snapshot in device buffers instead of copying ~162 MiB across PCIe -- ++ // but llama.cpp allows exactly one on-device snapshot per sequence, and ++ // taking another silently overwrites the previous one's data while the ++ // server still holds it and will still restore from it. Two checkpoints ++ // exist here in practice, so enabling this produced a model answering ++ // from another conversation's state: "2 minutes past midnight" at 1:01 AM. ++ // Do not re-enable until exactly one checkpoint is guaranteed, and verify ++ // it from the log rather than by reasoning. + cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); cur.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); // stash the draft's speculative state with the checkpoint - common_speculative_get_state(spec.get(), slot.id, cur.data_spec); -@@ -3306,7 +3331,7 @@ private: +@@ -3518,10 +3543,22 @@ private: + const bool is_user_start = spans.is_user_start(n_tokens_start); + const bool is_last_user_message = n_tokens_start == last_user_pos; - if (!do_reset) { - // restore the context checkpoint -- it->load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); -+ it->load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); - it->load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); - // restore the draft's speculative state - common_speculative_set_state(spec.get(), slot.id, it->data_spec); -@@ -3532,9 +3557,15 @@ private: ++ // checkpointing at the end of the prompt is a fallback for not knowing ++ // where checkpoints belong. when the request declared message ++ // delimiters, we do know, so take them at those positions and nowhere ++ // else: an end-of-prompt checkpoint sits at a position that moves with ++ // every request, so it can never be reused and costs a full state copy ++ // out of the device each time. ++ const bool have_spans = !slot.task->params.message_spans.spans.empty(); ++ + // entire prompt has been processed + if (slot.prompt.n_tokens() == slot.task->n_tokens()) { + slot.state = SLOT_STATE_DONE_PROMPT; + ++ if (have_spans && !is_user_start) { ++ do_checkpoint = false; ++ } ++ + GGML_ASSERT(batch.size() > 0); + + // extract the logits only for the last token +@@ -3532,9 +3569,7 @@ private: slot.init_sampler(); } else { - // skip ordinary mid-prompt checkpoints, unless the batch starts a user - // message or we are near the end of the prompt - if (!is_user_start && !near_prompt_end) { -+ // checkpointing near the end of the prompt is a fallback for not -+ // knowing where checkpoints belong. when the request declared message -+ // delimiters, we do know, so take them at those positions and nowhere -+ // else: a prompt-end checkpoint sits at a position that moves with -+ // every request, so it can never be reused and costs a full state copy -+ // out of the device each time. -+ const bool have_spans = !slot.task->params.message_spans.spans.empty(); -+ + if (!is_user_start && !(near_prompt_end && !have_spans)) { do_checkpoint = false; } From 49023043500d1c810651176f80f0f1ca1ad53fbc Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 05:06:49 +0000 Subject: [PATCH 46/98] Make the revert an actual revert, so no recompile is needed 89e0b08 bundled a new change with the revert -- suppressing the end-of-prompt checkpoint in the branch that was missed -- so the patch was a combination never built before and Nix rightly had nothing cached. llama-checkpoint-dedup.patch is now byte-identical to the version already built on the host, so switching back to the safe build is an activation rather than a compile. The suppression fix is not lost: it is in this branch history and described in the findings, to reapply next time something else forces a rebuild anyway. Co-Authored-By: Claude Opus 5 --- nixos/nixos/llama-checkpoint-dedup.patch | 51 +++++------------------- 1 file changed, 10 insertions(+), 41 deletions(-) diff --git a/nixos/nixos/llama-checkpoint-dedup.patch b/nixos/nixos/llama-checkpoint-dedup.patch index b431264..732833e 100644 --- a/nixos/nixos/llama-checkpoint-dedup.patch +++ b/nixos/nixos/llama-checkpoint-dedup.patch @@ -1,5 +1,5 @@ diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp -index 744593c76..893771837 100644 +index 744593c76..8bbe9ac4b 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2294,6 +2294,22 @@ private: @@ -25,52 +25,21 @@ index 744593c76..893771837 100644 // evict checkpoints within min-step of a previous checkpoint, unless they were // created by the current task int64_t last = -1; -@@ -2329,6 +2345,15 @@ private: - // this is not true for SWA models: https://github.com/ggml-org/llama.cpp/pull/24411#issuecomment-4677983225 - cur.update_pos(slot.prompt.n_tokens() - n_tokens_cur, pos_min, pos_max); - -+ // NOT LLAMA_STATE_SEQ_FLAGS_ON_DEVICE. It is faster -- it keeps the -+ // snapshot in device buffers instead of copying ~162 MiB across PCIe -- -+ // but llama.cpp allows exactly one on-device snapshot per sequence, and -+ // taking another silently overwrites the previous one's data while the -+ // server still holds it and will still restore from it. Two checkpoints -+ // exist here in practice, so enabling this produced a model answering -+ // from another conversation's state: "2 minutes past midnight" at 1:01 AM. -+ // Do not re-enable until exactly one checkpoint is guaranteed, and verify -+ // it from the log rather than by reasoning. - cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); - cur.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); - // stash the draft's speculative state with the checkpoint -@@ -3518,10 +3543,22 @@ private: - const bool is_user_start = spans.is_user_start(n_tokens_start); - const bool is_last_user_message = n_tokens_start == last_user_pos; - -+ // checkpointing at the end of the prompt is a fallback for not knowing -+ // where checkpoints belong. when the request declared message -+ // delimiters, we do know, so take them at those positions and nowhere -+ // else: an end-of-prompt checkpoint sits at a position that moves with -+ // every request, so it can never be reused and costs a full state copy -+ // out of the device each time. -+ const bool have_spans = !slot.task->params.message_spans.spans.empty(); -+ - // entire prompt has been processed - if (slot.prompt.n_tokens() == slot.task->n_tokens()) { - slot.state = SLOT_STATE_DONE_PROMPT; - -+ if (have_spans && !is_user_start) { -+ do_checkpoint = false; -+ } -+ - GGML_ASSERT(batch.size() > 0); - - // extract the logits only for the last token -@@ -3532,9 +3569,7 @@ private: +@@ -3532,9 +3548,15 @@ private: slot.init_sampler(); } else { - // skip ordinary mid-prompt checkpoints, unless the batch starts a user - // message or we are near the end of the prompt - if (!is_user_start && !near_prompt_end) { ++ // checkpointing near the end of the prompt is a fallback for not ++ // knowing where checkpoints belong. when the request declared message ++ // delimiters, we do know, so take them at those positions and nowhere ++ // else: a prompt-end checkpoint sits at a position that moves with ++ // every request, so it can never be reused and costs a full state copy ++ // out of the device each time. ++ const bool have_spans = !slot.task->params.message_spans.spans.empty(); ++ + if (!is_user_start && !(near_prompt_end && !have_spans)) { do_checkpoint = false; } From 96292e885e375573d93cda776929a6dee8357c59 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 05:09:44 +0000 Subject: [PATCH 47/98] Add a state-corruption test, and retract the corruption claim The two answers I cited as proof that ON_DEVICE corrupted state do not show that. On the known-good build the model answers "7 minutes" at 1:07 AM: it reports the minutes field instead of computing minutes since midnight. That explains both quotes. The hazard is real by construction -- the log shows two checkpoints sharing mem_storage[0] and the second overwrites the first -- but it was never observed doing damage. prefix-cache-statecheck.py is the check that settles it properly. At temperature 0 identical prompts must produce identical text, and every question is a fresh conversation, so every one exercises the restore path. Known-good digest: 4c4a15734be00181. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-statecheck.py | 41 ++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 nixos/nixos/prefix-cache-statecheck.py diff --git a/nixos/nixos/prefix-cache-statecheck.py b/nixos/nixos/prefix-cache-statecheck.py new file mode 100644 index 0000000..ff13169 --- /dev/null +++ b/nixos/nixos/prefix-cache-statecheck.py @@ -0,0 +1,41 @@ +"""Detect wrong-state restores by their only reliable symptom: changed output. + +At temperature 0 the same prompt must produce the same text. Each question is a +FRESH conversation, so each one forces the restore path -- exactly where a +checkpoint mix-up would show. Run on a known-good build to record a baseline, +then again after a change; any diff is a state bug, however good the latency +looks. +""" +import hashlib, json, sys, urllib.request + +URL = "http://192.168.122.1:11434/api/chat" +MODEL = "qwen3.6:27b-mtp-q8_0" +SYS = ("You are a voice assistant for a home.\n" + "\n".join( + f"- device_{i}: Device {i} ('Device {i}', {'on' if i % 3 else 'off'})" + for i in range(300))) +QS = ["Which devices are off? Name the first three.", + "What is device 7 called and what state is it in?", + "Count the devices you can see, then say the number only.", + "Repeat the state of device 12 exactly.", + "List devices 20 through 24 with their states.", + "What was the very first device in your list?", + "Say the state of device 299.", + "How many devices are listed in total?"] + +out = {} +for q in QS: + body = json.dumps({"model": MODEL, "think": False, "keep_alive": -1, + "messages": [{"role": "system", "content": SYS}, + {"role": "user", "content": q}], + "stream": False, + "options": {"temperature": 0, "seed": 1, "num_predict": 80}}).encode() + r = urllib.request.Request(URL, body, {"Content-Type": "application/json"}) + d = json.load(urllib.request.urlopen(r, timeout=600)) + out[q] = d["message"]["content"].strip() + +path = sys.argv[1] +json.dump(out, open(path, "w"), indent=1) +digest = hashlib.sha256(json.dumps(out, sort_keys=True).encode()).hexdigest()[:16] +print(f"wrote {path} digest {digest}") +for q, a in out.items(): + print(f" {q[:44]:46} -> {a[:58]!r}") From 9badc157c4dda54cec3bfac40b17d03aadf20d18 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 05:11:58 +0000 Subject: [PATCH 48/98] Keep the prompt checkpoint in VRAM, with the invariant actually held Second attempt. The first failed because end-of-prompt checkpoints were suppressed only for mid-prompt batches; when the prompt is fully processed the code takes another branch, which still took one, so two checkpoints shared mem_storage[0] and the second overwrote the first. Both branches are suppressed now, so exactly one checkpoint is ever created, which is the entire safety argument for the flag: llama.cpp allows one on-device snapshot per sequence and silently overwrites the previous one otherwise. Acceptance is not latency. This failure mode makes latency better, so a stopwatch cannot detect it: - the log must never say "created context checkpoint 2 of" - prefix-cache-statecheck.py must give digest 4c4a15734be00181 Co-Authored-By: Claude Opus 5 --- nixos/nixos/llama-checkpoint-dedup.patch | 70 ++++++++++++++++++++---- 1 file changed, 60 insertions(+), 10 deletions(-) diff --git a/nixos/nixos/llama-checkpoint-dedup.patch b/nixos/nixos/llama-checkpoint-dedup.patch index 732833e..687a882 100644 --- a/nixos/nixos/llama-checkpoint-dedup.patch +++ b/nixos/nixos/llama-checkpoint-dedup.patch @@ -1,5 +1,5 @@ diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp -index 744593c76..8bbe9ac4b 100644 +index 744593c76..883fc03bd 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2294,6 +2294,22 @@ private: @@ -25,21 +25,71 @@ index 744593c76..8bbe9ac4b 100644 // evict checkpoints within min-step of a previous checkpoint, unless they were // created by the current task int64_t last = -1; -@@ -3532,9 +3548,15 @@ private: +@@ -2329,7 +2345,24 @@ private: + // this is not true for SWA models: https://github.com/ggml-org/llama.cpp/pull/24411#issuecomment-4677983225 + cur.update_pos(slot.prompt.n_tokens() - n_tokens_cur, pos_min, pos_max); + +- cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); ++ // ON_DEVICE keeps the snapshot in device buffers rather than copying it out ++ // to host RAM and back, which is the single largest remaining cost of a ++ // fresh conversation. ++ // ++ // DANGEROUS IF MORE THAN ONE CHECKPOINT EXISTS. llama.cpp allows exactly ++ // one on-device snapshot per sequence: every save resolves to the same ++ // mem_storage[seq_id] and silently overwrites the previous one's data, ++ // while the server still holds that checkpoint and will still restore ++ // from it. The result is a model answering from another conversation's ++ // state, with no error and with *better* latency, so no timing ++ // measurement can detect it. ++ // ++ // It is safe here only because exactly one checkpoint is ever created: ++ // one delimiter, matched once, and end-of-prompt checkpoints suppressed ++ // in both branches above. Verify that from the log after any change to ++ // either -- "created context checkpoint 2 of" appearing at all means this ++ // is unsafe. prefix-cache-statecheck.py is the behavioural check. ++ cur.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); + cur.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + // stash the draft's speculative state with the checkpoint + common_speculative_get_state(spec.get(), slot.id, cur.data_spec); +@@ -3306,7 +3339,7 @@ private: + + if (!do_reset) { + // restore the context checkpoint +- it->load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); ++ it->load_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); + it->load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + // restore the draft's speculative state + common_speculative_set_state(spec.get(), slot.id, it->data_spec); +@@ -3518,10 +3551,22 @@ private: + const bool is_user_start = spans.is_user_start(n_tokens_start); + const bool is_last_user_message = n_tokens_start == last_user_pos; + ++ // checkpointing at the end of the prompt is a fallback for not knowing ++ // where checkpoints belong. when the request declared message ++ // delimiters, we do know, so take them at those positions and nowhere ++ // else: an end-of-prompt checkpoint sits at a position that moves with ++ // every request, so it can never be reused and costs a full state copy ++ // out of the device each time. ++ const bool have_spans = !slot.task->params.message_spans.spans.empty(); ++ + // entire prompt has been processed + if (slot.prompt.n_tokens() == slot.task->n_tokens()) { + slot.state = SLOT_STATE_DONE_PROMPT; + ++ if (have_spans && !is_user_start) { ++ do_checkpoint = false; ++ } ++ + GGML_ASSERT(batch.size() > 0); + + // extract the logits only for the last token +@@ -3532,9 +3577,7 @@ private: slot.init_sampler(); } else { - // skip ordinary mid-prompt checkpoints, unless the batch starts a user - // message or we are near the end of the prompt - if (!is_user_start && !near_prompt_end) { -+ // checkpointing near the end of the prompt is a fallback for not -+ // knowing where checkpoints belong. when the request declared message -+ // delimiters, we do know, so take them at those positions and nowhere -+ // else: a prompt-end checkpoint sits at a position that moves with -+ // every request, so it can never be reused and costs a full state copy -+ // out of the device each time. -+ const bool have_spans = !slot.task->params.message_spans.spans.empty(); -+ + if (!is_user_start && !(near_prompt_end && !have_spans)) { do_checkpoint = false; } From e2fa79d13a330e898186f75fd43859953289826c Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 05:17:52 +0000 Subject: [PATCH 49/98] Record the result: 105 ms fresh, 109 ms follow-up From 1770 and 1831 at the start. The VRAM-resident checkpoint took the last step, 165 -> 105 ms. State verified by comparing answers rather than the digest. The digest did change, and the state was fine: 7 of 8 answers byte-identical, the eighth differing only by markdown emphasis. Changing checkpoint placement changes batch shapes and so float arithmetic, which can flip a near-tie token with perfectly correct state. Wrong-state restores show up as wrong facts, not lost bold. Digest stays as a trigger to look, not as the verdict. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index e94e34c..76be2c7 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -2,11 +2,11 @@ Status: **fixed and verified.** Fresh-conversation prefill 1770 ms -> 437 ms from checkpoint spacing, then -> 192 ms with the delimiters patch, which is the per-request floor. Generation 94.4 tok/s, no regression. -| | original | spacing fix | + delimiters | + dedup | -|---|---|---|---|---| -| fresh conversation | 1770 ms | 437 ms | 192 ms | **158 ms** | -| follow-up turn | 1831 ms | 185 ms | 275 ms | **239 ms** | -| generation | 90.7 tok/s | 92.8 | 94.4 | **87.1** | +| | original | spacing | + delimiters | + dedup | + unique marker | + VRAM | +|---|---|---|---|---|---|---| +| fresh conversation | 1770 ms | 437 ms | 192 ms | 158 ms | 165 ms | **105 ms** | +| follow-up turn | 1831 ms | 185 ms | 275 ms | 239 ms | 154 ms | **109 ms** | +| generation | 90.7 tok/s | 92.8 | 94.4 | 87.1 | 85 | **86** | Generation varies +/- 8% run to run (77 to 94 across the session on identical prompts), so none of that column is a signal beyond "no regression". From 0dfc43732a5f3058965bc7c94c8bf84d8ef2664f Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 05:21:46 +0000 Subject: [PATCH 50/98] Bring the findings up to date with what actually shipped Several edits to this file silently no-opped: python str.replace does not error on a missed anchor and I did not check. The result was a document whose results table was current while its analysis described a state we had long left -- the exact failure this file exists to prevent. Rewrites the sections on where the remaining time went and on keeping the checkpoint in VRAM: restores were the expensive half, not saves; the flag is applied and verified; the single-checkpoint invariant is what makes it safe; and the check order is log, then answers, then latency, because the failure mode improves latency. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 123 ++++++++++++++++----------- 1 file changed, 73 insertions(+), 50 deletions(-) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 76be2c7..ea4b1fd 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -247,72 +247,95 @@ the boundary checkpoint was deleted immediately after being made. 0 is documented as "no minimum" and is the correct value now that we say where checkpoints belong. -## What is still not solved - -**No pinning.** Eviction is FIFO (`erase(checkpoints.begin())`) with no way to -mark a checkpoint permanent. The ideal -- pin the system prefix forever, keep -one more for the live conversation -- still has no expression in the API. 32 -checkpoints at 128 minimum spacing is enough headroom that a voice conversation -never evicts the prefix, but that is headroom, not a guarantee. - -## Where the remaining ~190 ms goes - -Not prefill. The cost is nearly independent of prompt length: +## Where the remaining time went, and how it was closed -| system prompt | fresh prefill | -|---|---| -| 6298 tokens | 198 ms | -| 568 tokens | 179 ms | +Cost is nearly independent of prompt length. A repeated *identical* request +diverges nowhere, so it never restores: -It is **checkpoint creation**. The recurrent state is a fixed ~162 MiB -regardless of context length, and saving one means copying that device->host. -From the log, a fresh conversation restores one checkpoint and creates two; a -follow-up restores one and creates three, because its prompt contains two user -delimiters plus a `near_prompt_end`. Solving across the two cases: + identical large request: 57.9 ms / 6326 tok + identical small request: 57.3 ms / 45 tok -- **checkpoint save: ~65 ms** (2 saves = 192 ms fresh, 3 saves = 275 ms follow-up) -- **checkpoint restore: single-digit ms** — the cheap direction, host->device -- everything else: ~50 ms +**~57 ms is the irreducible per-request cost.** Everything above that was one +checkpoint restore plus the genuinely new tokens. -So restores are nearly free and saves dominate. This also means the earlier -"~0.15 s of fixed per-request overhead" in the traps section was mostly -misattributed: it was checkpoint saves, not HTTP and tokenization. +Earlier versions of this section attributed the gap to checkpoint *saves* +("~65 ms each, restores single-digit"). That was derived from comparing +measurements that differed in two things at once and is **withdrawn**. Restores +were the expensive half. -### Keeping checkpoints in VRAM +## Keeping the checkpoint in VRAM -The 65 ms is a device->host copy, and nothing requires that. llama.cpp has the -flag already, in `include/llama.h`: +`llama.cpp` has had the flag since #22679 and nothing in the tree used it except +a test: // Keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load). // Getting the state for a seq_id with this flag invalidates all prior states gotten for that seq_id with this flag. #define LLAMA_STATE_SEQ_FLAGS_ON_DEVICE 2 -It is a bit flag; the server passes only `PARTIAL_ONLY` (= 1). `grep ON_DEVICE -tools/server/ common/` returns nothing — implemented in the core, never wired -up. A save would become a device->device copy via `llama_io_write_device`, -sub-millisecond instead of 65 ms. +It is a bit flag; the server passed only `PARTIAL_ONLY` (= 1). Passing both for +the prompt checkpoint's save and restore keeps the snapshot in device buffers +instead of copying it across PCIe each way. Measured: **165 ms -> 105 ms** on a +fresh conversation. + +### It is only safe with exactly one checkpoint + +`mem_storage` is `std::map` and every save +resolves to the same entry for a given sequence, silently overwriting the +previous snapshot's data while the server still holds that checkpoint and will +still restore from it. ollama runs `-np 1`, so there is one sequence. + +**The first attempt shipped with this invariant broken.** Suppressing +end-of-prompt checkpoints only covered mid-prompt batches; the fully-processed +branch still took one, so two existed (at 3260 and 3300) and the second +clobbered the first. Both branches are suppressed now. + +Verified from the log — five fresh conversations in a six-second window: -The catch is the second comment line. `mem_storage` is -`std::map` and each `get_data` rebuilds the -entry, so there is exactly **one on-device snapshot per seq_id**. Two -VRAM-resident checkpoints need two sequence ids — the two-slot design, with a -real mechanism behind it. ollama passes `-np 1`. + task 351 | created context checkpoint 1 of 32 (pos_min = 6283) + task 369 | restored ... reusing context checkpoint (pos_min = 6283) + task 374 | restored ... reusing context checkpoint (pos_min = 6283) + task 380 | restored ... reusing context checkpoint (pos_min = 6283) + task 388 | restored ... reusing context checkpoint (pos_min = 6283) -This is an upstream llama.cpp change, not a flag and not an ollama patch: the -checkpoint deque holds up to 32 entries and would need to know which one gets -the single on-device slot per sequence. Worth ~130 ms of the 192 ms. +One checkpoint, created once on the cold request, reused thereafter. -### The other lever is architectural +**Bound that window to known traffic.** A ten-minute window spanning a +`nixos-rebuild switch` counted 8 creations and looked like a failure, because it +included the previous build. + +### How to check it, and how not to + +This failure mode makes latency *better* — the broken build read 99 ms — so no +timing measurement can detect it. Check, in this order: + +1. `journalctl -u ollama --since --until | grep "created context + checkpoint 2 of"` must be empty. +2. `prefix-cache-statecheck.py` must still answer *correctly*: eight fresh + conversations against a 300-device prompt at temperature 0. +3. Latency, last. + +**Compare the answers, not the digest.** The digest changed when this landed +(`4c4a1573` -> `1eed9fb1`) and the state was fine: seven of eight answers +byte-identical, the eighth differing only by markdown emphasis (`**on**` -> +`on`). Changing checkpoint placement changes batch shapes, hence float +arithmetic, which can flip a near-tie token with perfectly correct state. A real +wrong-state restore shows up as wrong *facts*. The digest is a trigger to look, +not a verdict. The test is deterministic within a build: two consecutive runs +gave identical digests. + +An earlier note here claimed two quoted answers proved corruption. They did not +— the model reports the minutes field rather than computing minutes since +midnight, on any build. Retracted. + +## What is still not solved + +**No pinning.** Eviction is FIFO (`erase(checkpoints.begin())`) with no way to +mark a checkpoint permanent. It does not bite while exactly one checkpoint +exists, but nothing enforces that beyond the two suppressions above. -For single-turn voice, the `near_prompt_end` checkpoint is pure overhead — we -always restore the user-boundary one. Suppressing it would save ~65 ms per -command. But `near_prompt_end` has no flag: it is exempt from -`checkpoint_min_step` and unaffected by `n_ctx_checkpoints`, so this needs a -llama.cpp patch, not a configuration change, and it would trade away multi-turn -performance (that checkpoint is exactly what a follow-up restores). +**Follow-up turns still restore rather than purely appending**, landing a few +tokens short of the boundary. Worth a look if the ~109 ms ever matters. -Not worth it at ~65 ms against a 1.4-3.2 s voice interaction. Recorded so the -next person does not go looking for a config knob that does not exist. ## Date and time From be7fb2bfdb1a06d7915d5fc8daa4a1ea901db4d6 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 05:29:29 +0000 Subject: [PATCH 51/98] Record the end-to-end breakdown and the model comparison Prefill is now 0.105 s of a ~1.3 s simple command. The VAD silence window (~1.05 s) dominates, and it is a setting rather than compute. A tool round trip dominates anything needing device state. STT compute is ~0.10 s, about 6%; moving it to the GPU would save perhaps 0.05 s and is not where the time is. Also: the reason for picking the dense 27B -- "generation speed is not the binding constraint for voice" -- was true when prefill cost 0.65-1.8 s and is false now. The MoE models answer a short question in 135-144 ms against 317 ms. Switching is blocked on re-testing the CUDA fault that rejected qwen35moe on ollama 0.32.3. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 51 ++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index ea4b1fd..4eff516 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -497,3 +497,54 @@ compared `think=false` against `think=true` with the thinking field preserved -- a different code path -- saw 247 ms vs 245 ms, and cleared the think tags of suspicion. The two paths differ in where the block is emitted, not whether it is. Render and diff the actual strings; do not infer from latency. + + +## End-to-end latency, measured (August 2026) + +Real audio through the whole pipeline, streamed at real time. Timed from the +moment the user stops speaking, which is what they actually feel: + +| stage | simple command | question needing state | +|---|---|---| +| VAD silence window + end-of-speech | ~1.05 s | ~1.05 s | +| STT transcribe (faster-whisper, CPU) | ~0.10 s | ~0.10 s | +| conversation agent | 0.003 s (local matcher) | 1.4-2.0 s (LLM + tool round trip) | +| TTS (Kokoro, GPU) | ~0.13 s | ~0.13 s | +| **total** | **~1.3 s** | **~2.7-3.3 s** | + +Prompt prefill, the subject of this entire document, is now 0.105 s of that. + +**The VAD silence window dominates simple commands** and is a setting, not +compute. **A tool round trip dominates questions** — asking the model for the +temperature costs a full extra generate-and-reply cycle. + +STT compute is ~0.10 s, about 6% of a simple command. Moving faster-whisper to +the GPU might halve it, saving ~0.05 s against a 1.05 s VAD window. Not the +place to spend effort. + +## Model choice: generation now dominates, so the calculus flipped + +Same system prompt, fresh conversation, plus a realistic ~18-token reply: + +| model | prefill | generation | short reply | +|---|---|---|---| +| qwen3.6:27b-mtp-q8_0 (current) | 106 ms | 85 t/s | 317 ms | +| qwen3.8:27b-mtp-q8_0 | 106 ms | 88 t/s | 310 ms | +| qwen3.8:27b-mtp-q4_K_M | 79 ms | 105 t/s | 251 ms | +| qwen3.6:27b-q4_K_M | 72 ms | 39 t/s | 531 ms | +| qwen3.6:35b-a3b-q4_K_M | 36 ms | 168 t/s | **144 ms** | +| ornith:35b-q4_K_M | 35 ms | 179 t/s | **135 ms** | +| muse-glimmer:30b-q4_K_M | 63 ms | 40 t/s | 509 ms | + +The dense 27B was chosen when prefill cost 0.65-1.8 s, on the reasoning that +"generation speed is not the binding constraint for voice anyway". That was true +then and is false now: prefill is 0.1 s and generation is the rest. The MoE +models are 2.3x faster end-to-end on the LLM stage. + +Both fast models are `qwen35moe`, `full_attention_interval: 4` — the same hybrid +architecture, so everything here applies to them too. + +**Blocker before switching:** qwen35moe was rejected earlier for a CUDA illegal +memory access during constrained decoding of tool calls with array/enum +parameters, which is exactly what Home Assistant sends. That was on ollama +0.32.3; we are on 0.32.13. Re-test before trusting it. From 239f51962a8cc0c471213b1f5862a9698ab26787 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 22:19:33 +0000 Subject: [PATCH 52/98] Correct the claimed Home Assistant overhead: 3.7 ms, not 300 ms A locally-matched command exercises the whole pipeline without the model and takes 3.67 ms end to end. The earlier figure subtracted HA's real prompt (~1848 tokens with full Assist tool schemas) from a synthetic replication (~250 tokens, four toy tools), so the residual was model work on a bigger prompt rather than overhead, and describing it as "websocket round trip, tool dispatch, template rendering" was invention. Also records the tool round trip measured like for like: 677 ms for two calls against 199 ms for one, a 478 ms saving of which 327 ms is generating the tool call. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 4eff516..42201da 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -548,3 +548,40 @@ architecture, so everything here applies to them too. memory access during constrained decoding of tool calls with array/enum parameters, which is exactly what Home Assistant sends. That was on ollama 0.32.3; we are on 0.32.13. Re-test before trusting it. + + +## Home Assistant's own overhead is ~3.7 ms + +A locally-matched command runs the whole pipeline with no model involved: +websocket in, sentence matching, service call, events out. + + intent stage (matcher + service call): 2.44 ms + whole request, socket to run-end : 3.67 ms + +There is nothing to optimise there. An earlier estimate of "~300 ms of HA +plumbing" was wrong: it subtracted HA's end-to-end time for the real prompt +(~1848 tokens, full Assist tool schemas) from a synthetic replication (~250 +tokens, four toy tools). The residual is model work on a bigger prompt, not +overhead. Do not subtract across measurements with different prompts. + +## Tool round trip: 478 ms, measured like for like + +Same prompt, same tools, same model, question "Is the bed light on?": + +| | prefill | generate | total | +|---|---|---|---| +| call 1, emit tool call | 91.7 ms | 327.5 ms (29 tok) | 419 ms | +| call 2, answer from result | 100.8 ms | 157.5 ms (10 tok) | 258 ms | +| **two-call total** | | | **677 ms** | +| single call, state in prompt | 91.5 ms | 107.9 ms (9 tok) | **199 ms** | + +The saving is 478 ms, of which 327 ms is generating the tool call itself. + +The trade is ~29 generated tokens for ~60 prefilled ones. Generation runs at +~85 tok/s and prefill at ~1750 tok/s, so a prefilled token is ~20x cheaper -- +twice the tokens for a tenth of the time. + +Home Assistant's prompt field is a Jinja template and a `<|fim_pad|>` typed into +it tokenizes as the special token (verified: +1 token, against +4 for text of +the same length). So HA can supply the boundary marker and everything after it, +which is what makes this possible without further patching. From 84826878b7665b8902f2e0e1c0f0e602a0a9d0a9 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Mon, 17 Aug 2026 23:33:08 +0000 Subject: [PATCH 53/98] Hand the cache boundary and the dynamic block to Home Assistant The renderer no longer injects the time. Home Assistant's prompt supplies the marker and everything after it: the current time and the live state of the entities worth stating up front. Better layering, since that knowledge lives in Home Assistant, and it removes a model round trip -- the model no longer calls GetLiveContext and get asked again. Measured like for like on "Is the bed light on?": 677 ms of model time for the two-call version against 199 ms for one call with the state already in the prompt. 327 ms of the saving is generating the tool call. Ordering note for the follow-up: chat_log.py appends the entity overview AFTER the user's prompt template, so a marker at the end of the template leaves the entity list outside the cached region. Small at six entities, worth revisiting if many more are exposed. Exactly one marker must exist in the prompt. Two would mean two checkpoints, and the on-device snapshot only holds one -- hence removing the renderer injection before adding the marker to Home Assistant's side, rather than after. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 24 ++++----- nixos/nixos/ollama-message-delimiters.patch | 59 ++------------------- 2 files changed, 15 insertions(+), 68 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 1613421..5deb767 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -219,19 +219,19 @@ # of the invariant prompt, just before the current time. OLLAMA_MESSAGE_DELIMITERS = builtins.toJSON [ "<|fim_pad|>" ]; - # What goes after that boundary: the current time, as its own system - # block, emitted by the renderer. A Go time layout; unset disables it. - # Minutes are the useful granularity -- seconds would change the prompt - # on every request for no benefit. + # What goes after the boundary is now Home Assistant's job, not the + # renderer's: its prompt ends with the marker followed by the current time + # and the live state of the entities worth stating up front. That is + # better layering -- Home Assistant is where that knowledge lives -- and it + # saves a whole model round trip, because the model no longer has to call + # GetLiveContext and be asked again. Measured: 677 ms of model time for the + # two-call version against 199 ms for one, of which 327 ms was generating + # the tool call. # - # With this set, drop "Call GetDateTimeTool for the current date or time." - # from the Home Assistant prompt; the model no longer needs a round trip - # to find out what time it is. - # - # The weekday is in there because the model cannot reliably derive it: with - # a bare date it answered "Sunday, August 17, 2026" and then "Monday, - # August 17, 2026" for the same question. It is a Monday. - OLLAMA_TIME_FORMAT = "Monday, January 2, 2006 at 3:04 PM MST"; + # Home Assistant's prompt is a Jinja template and a <|fim_pad|> typed into + # it tokenizes as the special token, so it can supply the marker itself. + # There must be exactly ONE marker in the prompt: two would mean two + # checkpoints, and the on-device snapshot only holds one. }; # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. diff --git a/nixos/nixos/ollama-message-delimiters.patch b/nixos/nixos/ollama-message-delimiters.patch index 715570e..fe51a31 100644 --- a/nixos/nixos/ollama-message-delimiters.patch +++ b/nixos/nixos/ollama-message-delimiters.patch @@ -57,31 +57,10 @@ index a11b7f83..5d1378cd 100644 Logprobs bool diff --git a/model/renderers/qwen35.go b/model/renderers/qwen35.go -index 8b9e5797..cfc23209 100644 +index 8b9e5797..dec01b62 100644 --- a/model/renderers/qwen35.go +++ b/model/renderers/qwen35.go -@@ -3,11 +3,20 @@ package renderers - import ( - "fmt" - "log/slog" -+ "os" - "strings" -+ "time" - - "github.com/ollama/ollama/api" - ) - -+// qwen35CacheBoundary marks where the identical-on-every-request part of the -+// prompt ends, for llama-server's context checkpointing. It is a padding token, -+// chosen because it is a single special token (so it tokenizes identically -+// wherever it appears) and carries no meaning of its own. Must match -+// OLLAMA_MESSAGE_DELIMITERS. -+const qwen35CacheBoundary = "<|fim_pad|>" -+ - const ( - qwen35ThinkOpenTag = "" - qwen35ThinkCloseTag = "" -@@ -72,6 +81,14 @@ func (r *Qwen35Renderer) LeadingBOS() string { +@@ -72,6 +72,14 @@ func (r *Qwen35Renderer) LeadingBOS() string { return "" } @@ -96,39 +75,7 @@ index 8b9e5797..cfc23209 100644 func (r *Qwen35Renderer) renderContent(content api.Message, imageOffset int) (string, int) { if r.useImgTags { return renderContentWithImageTags(content.Content, len(content.Images), imageOffset) -@@ -272,6 +289,31 @@ func (r *Qwen35Renderer) Render(messages []api.Message, tools []api.Tool, think - sb.WriteString(imStartTag + "system\n" + reasoningInstructions + imEndTag + "\n") - } - -+ // Emit the current time as its own system block, right after the part of -+ // the prompt that is identical on every request. Keeping it out of the -+ // system message is the point: llama-server takes a context checkpoint at -+ // the message boundary just before it, so everything above stays cached and -+ // only the time and the question are re-evaluated. See -+ // OLLAMA_MESSAGE_DELIMITERS. -+ // -+ // The delimiter has to BE a special token. Delimiter matching compares token -+ // sequences, and BPE merges across ordinary text: "system\n" is one token -+ // alone but splits when text follows it, so a marker ending in text never -+ // matches. Special tokens are atomic and merge with nothing. -+ // -+ // "<|im_end|>\n<|im_start|>" is stable but matches every message boundary, -+ // which costs a snapshot at each. qwen35CacheBoundary is a single special -+ // token that appears exactly here, so exactly one checkpoint is taken, at -+ // the last position that is identical on every request. -+ // -+ // The alternatives are both worse: templating the time into the system -+ // prompt invalidates the cache on every request, and answering "what time is -+ // it" with a tool call costs an entire extra round trip through the model. -+ if layout := os.Getenv("OLLAMA_TIME_FORMAT"); layout != "" { -+ sb.WriteString(imStartTag + "system\n" + qwen35CacheBoundary + "Current time: " + -+ time.Now().Format(layout) + imEndTag + "\n") -+ } -+ - multiStepTool := true - lastQueryIndex := len(messages) - 1 // so this is the last user message - -@@ -309,6 +351,14 @@ func (r *Qwen35Renderer) Render(messages []api.Message, tools []api.Tool, think +@@ -309,6 +317,14 @@ func (r *Qwen35Renderer) Render(messages []api.Message, tools []api.Tool, think if renderAssistantThinkBlock { sb.WriteString(imStartTag + message.Role + "\n\n" + contentReasoning + "\n\n\n" + content) From d8aa160abf89785f413af75d4d8abd010dd55ad3 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 00:34:05 +0000 Subject: [PATCH 54/98] Patch Home Assistant to put the cache boundary where it belongs chat_log.py appends the API preamble and the exposed entity overview after the configured prompt template, so a boundary marker at the end of the template leaves 300-400 tokens of unchanging content outside the cached region, re-read every request for roughly 200 ms. That would have eaten nearly half of the 478 ms this is meant to save. The patch splits the template on the marker and appends the tail last, so the prompt is: unchanging instructions, preamble and entity overview, then the marker, then the current time and live states. Verified to apply against the source nixpkgs fetches, and the patched file parses. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 15 +++++ .../nixos/home-assistant-cache-boundary.patch | 56 +++++++++++++++++++ nixos/nixos/prefix-cache-findings.md | 28 ++++++++++ 3 files changed, 99 insertions(+) create mode 100644 nixos/nixos/home-assistant-cache-boundary.patch diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 5deb767..5f26f43 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -248,6 +248,21 @@ ]; }; + # Home Assistant assembles the system prompt as: the configured prompt + # template, then the API preamble and the exposed entity overview, then + # anything extra. That puts unchanging content *after* the template, so a + # cache boundary placed at the end of the template would leave the entity + # overview outside the cached region -- around 300-400 tokens re-read on every + # request, roughly 200 ms, for content that never changes. + # + # The patch moves everything after the boundary marker to the very end + # instead, so the prompt is: unchanging instructions, preamble and entity + # overview | marker | current time and live states. See + # prefix-cache-findings.md. + services.home-assistant.package = pkgs.home-assistant.overrideAttrs (old: { + patches = (old.patches or [ ]) ++ [ ./home-assistant-cache-boundary.patch ]; + }); + # Voice assistant. With `prefer_local_intents` on (a per-pipeline setting, # off by default) the built-in sentence matcher answers anything it # recognises without involving the model, so "turn off the kitchen light" diff --git a/nixos/nixos/home-assistant-cache-boundary.patch b/nixos/nixos/home-assistant-cache-boundary.patch new file mode 100644 index 0000000..a9a71e6 --- /dev/null +++ b/nixos/nixos/home-assistant-cache-boundary.patch @@ -0,0 +1,56 @@ +--- a/homeassistant/components/conversation/chat_log.py 2026-08-18 00:32:39.616617684 +0000 ++++ b/homeassistant/components/conversation/chat_log.py 2026-08-18 00:33:13.216071513 +0000 +@@ -31,6 +31,11 @@ + ] = HassKey("conversation_chat_log_subscriptions") + LOGGER = logging.getLogger(__name__) + ++# Marks the end of the unchanging part of the system prompt. A single special ++# token in the qwen tokenizers, so ordinary prompt text cannot produce it by ++# accident and it always tokenizes identically. ++CACHE_BOUNDARY = "<|fim_pad|>" ++ + current_chat_log: ContextVar[ChatLog | None] = ContextVar( + "current_chat_log", default=None + ) +@@ -720,14 +725,24 @@ + user_name = user.name + + prompt_parts = [] +- prompt_parts.append( +- await self._async_expand_prompt_template( +- llm_context, +- (user_llm_prompt or llm.DEFAULT_INSTRUCTIONS_PROMPT), +- llm_context.language, +- user_name, +- ) ++ user_prompt = await self._async_expand_prompt_template( ++ llm_context, ++ (user_llm_prompt or llm.DEFAULT_INSTRUCTIONS_PROMPT), ++ llm_context.language, ++ user_name, + ) ++ # Everything from CACHE_BOUNDARY onwards moves to the very end of the ++ # system prompt, so the parts that never change -- these instructions, ++ # the API preamble, the exposed entity overview -- form one contiguous ++ # prefix an inference server can cache, and the parts that change on ++ # every request sit after it. ++ # ++ # Without this the user's template comes first and the entity overview is ++ # appended after it, so a boundary placed in the template would leave ++ # that unchanging overview outside the cached region, re-read on every ++ # request for no reason. ++ user_prompt, boundary, dynamic_prompt = user_prompt.partition(CACHE_BOUNDARY) ++ prompt_parts.append(user_prompt) + + if llm_api: + prompt_parts.append(llm_api.api_prompt) +@@ -750,6 +765,9 @@ + ): + prompt_parts.append(extra_system_prompt) + ++ if boundary: ++ prompt_parts.append(boundary + dynamic_prompt) ++ + prompt = "\n".join(prompt_parts) + + self.llm_input_provided_index = len(self.content) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 42201da..93ca8e6 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -585,3 +585,31 @@ Home Assistant's prompt field is a Jinja template and a `<|fim_pad|>` typed into it tokenizes as the special token (verified: +1 token, against +4 for text of the same length). So HA can supply the boundary marker and everything after it, which is what makes this possible without further patching. + + +## Home Assistant assembles the prompt in the wrong order for caching + +`chat_log.py` builds the system prompt as: + +1. the configured prompt template +2. `llm_api.api_prompt` -- the API preamble plus a YAML dump of every exposed + entity +3. the date and time, but only when no `GetDateTime` tool is offered +4. `extra_system_prompt` + +So the entity overview lands *after* the template. A boundary marker at the end +of the template would leave that overview -- 300-400 tokens that never change -- +outside the cached region, re-read on every request for around 200 ms. That is +not the "inline these entities, tool-call for the rest" tradeoff; it is pure +waste. + +`home-assistant-cache-boundary.patch` splits the template on the marker and +appends the tail last, giving: + + unchanging instructions, preamble, entity overview | marker | time, live states + +Note point 3: Home Assistant already injects the date and time itself, and +suppresses it only because the Assist API offers `GetDateTimeTool`. Dropping +that tool would get the time for free -- but into part 3, which is *before* the +marker after this patch, so it would invalidate the cache every minute. Put the +time after the marker instead. From 5235cef6a25be70575e773ed9023187bccd29ed0 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 00:38:37 +0000 Subject: [PATCH 55/98] Put live state in the prompt: 986 ms -> 433 ms on a state question The model now answers from state given to it rather than calling GetLiveContext and being asked again. Measured through the real pipeline: "Is the bed light on?" 986 -> 433 ms, "What is the weather forecast?" 1265 -> 585 ms. Answers checked against the entities rather than just timed -- bed light off, kitchen lights on, weather partlycloudy 79 F humidity 81%, all correct, no tool calls. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 93ca8e6..21ed465 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -613,3 +613,33 @@ suppresses it only because the Assist API offers `GetDateTimeTool`. Dropping that tool would get the time for free -- but into part 3, which is *before* the marker after this patch, so it would invalidate the cache every minute. Put the time after the marker instead. + + +## Result: live state in the prompt, measured end to end + +Home Assistant's prompt now ends with the marker followed by the current time +and the live state of the entities worth stating up front. Through the real +pipeline, intent stage: + +| question | tool round trip | state in prompt | +|---|---|---| +| Is the bed light on? | 986 ms | **433 ms** | +| What is the weather forecast? | 1265 ms | **585 ms** | +| Turn on the ceiling lights (local matcher) | 3 ms | 4 ms | +| unanswerable, falls through to search | 1754 ms | 1355 ms | + +Answers verified against the entity states rather than just timed: bed light +off, kitchen lights on, weather partlycloudy 79 F humidity 81%, all reported +correctly and with no tool call. + +The prompt is now: + + cached + cached, thanks to the chat_log patch + <|fim_pad|> the boundary + Live state, correct as of now: re-read each request, ~60 tokens + Current time: ... + Bed Light: ... etc + +Exactly one marker exists in the prompt, which is what keeps the on-device +snapshot safe. The renderer no longer injects one. From 43d809016c6176761070587d640eb598fcb1f61d Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 00:44:55 +0000 Subject: [PATCH 56/98] Measure the VAD wait: 1206 ms, and transcription is only 94 ms Streaming silence continuously instead of ending the stream separates the two. The VAD decision is 1206 ms and transcription is 94 ms, which settles the speech-to-text question: faster-whisper on CPU is 5% of an interaction and moving it to the GPU is not worth doing. silence_seconds defaults to 0.7 s, so ~500 ms of the VAD wait is unaccounted for and should be chased before changing the threshold. Also corrects "it is just a setting": the websocket handler builds AudioSettings from four fields and silence_seconds is not one of them, so there is no UI and no API for it. Changing it means another patch. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 21ed465..ba7ff94 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -643,3 +643,28 @@ The prompt is now: Exactly one marker exists in the prompt, which is what keeps the on-device snapshot safe. The renderer no longer injects one. + + +## The VAD wait, measured properly + +Streaming silence continuously rather than ending the stream, timed from the +last sample of speech: + + end of speech -> stt-vad-end (VAD decides) 1206 ms + end of speech -> stt-end (+ transcribe) 1301 ms + transcription alone 94 ms + +**Transcription is 94 ms.** faster-whisper on CPU is not worth moving to the +GPU; it is 5% of a voice interaction. An earlier "~0.10 s" figure for this was +right, and a later "~600 ms" inference from a fast-push harness was wrong. + +**The VAD decision is 1206 ms** and is now the single largest cost in the whole +pipeline. `AudioSettings.silence_seconds` defaults to 0.7 s, so roughly 500 ms +of that is unaccounted for and worth chasing before touching the threshold. + +`silence_seconds` is **not settable over the websocket API**. The handler builds +`AudioSettings` from only `noise_suppression_level`, `auto_gain_dbfs`, +`volume_multiplier` and `is_vad_enabled` (`websocket_api.py:211`), so a run +request cannot override it. Calling it "just a setting" was wrong: it is a +constant with no UI and no API, and changing it means another Home Assistant +patch. From da14e38111d451f920b7e1b9fab41927c1687644 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 00:50:14 +0000 Subject: [PATCH 57/98] Chase the extra VAD latency: ~400 ms is pipeline startup The utterance is speech from 50 to 1240 ms of a 1280 ms file, so silence in the audio is not confusing the measurement. Onset was detected 1007 ms after speech began against 350 ms expected, and the end 1206 ms after speech stopped against 700 ms. Prepending 2 s of silence moves onset to 605 ms, so roughly 400 ms is Home Assistant pipeline startup queueing early audio. A wake-word satellite starts the pipeline before the command is spoken, so that part is probably absorbed in real use and my harness overstates it. Still unexplained: with startup absorbed the end is 1305 ms against ~700 ms expected, while onset improved. An asymmetric residual is not a constant delay, so that story does not cover it, and the silence being fed is digital zero so thresholds do not obviously either. Recorded as open rather than guessed at. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index ba7ff94..aa012fb 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -668,3 +668,41 @@ of that is unaccounted for and worth chasing before touching the threshold. request cannot override it. Calling it "just a setting" was wrong: it is a constant with no UI and no API, and changing it means another Home Assistant patch. + + +## Chasing the extra VAD time: partly explained + +The utterance really is speech from 50 ms to 1240 ms of a 1280 ms file, checked +by looking at the samples, so no leading or trailing silence is confusing this. + +Streaming it in real time and timing against my own clock: + +| | measured | expected from settings | +|---|---|---| +| speech onset -> `stt-vad-start` | 1007 ms | 350 ms (50 + speech_seconds 300) | +| end of speech -> `stt-vad-end` | 1206 ms | 700 ms (silence_seconds) | + +Both ends were late by roughly the same ~600 ms, which looks like a constant +delay in the audio path. + +Prepending 2 s of silence before the utterance moves onset to **605 ms** after +speech begins, against 1007 ms without. So roughly 400 ms is Home Assistant +pipeline startup: it is not ready to consume audio the instant the run begins, +and early audio queues behind that. + +**This probably does not apply to a real satellite.** A wake-word device starts +the pipeline when it hears the wake word, before the command is spoken, so that +startup is absorbed. My harness starts the run and streams immediately. + +**Unexplained: the end is still ~600 ms late even with startup absorbed** (1305 +ms against ~700 ms expected), while onset improved. An asymmetric residual is +not a constant pipeline delay, so that explanation does not cover it. Thresholds +do not obviously explain it either: `in_command_speech_threshold` is 0.5 and the +silence being fed is digital zero, which should read as ~0 immediately. Likely +candidates not yet tested: smoothing or internal state in the VAD model itself, +or buffering between the websocket handler and the pipeline. + +Actionable regardless: `silence_seconds` is 0.7 and unreachable from the API, so +lowering it needs a patch. That is worth doing after the residual is understood, +not before -- otherwise a smaller threshold may just be swallowed by whatever +the extra 600 ms is. From 98780f7aee652df60300bde61312d90c10135142 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 00:54:42 +0000 Subject: [PATCH 58/98] Explain the VAD residual: pymicro-vad holds speech for 640 ms Running the model Home Assistant uses directly on the utterance followed by digital silence: probability stays at 1.000 for 400 ms and does not fall below the 0.5 in-command threshold until 640 ms. The silence_seconds countdown only starts after that. So the 1306 ms from end of speech to stt-vad-end is 640 ms of model hangover plus 700 ms of silence_seconds, fully accounted for. HA's own audio timestamps track wall clock, so nothing is lagging. Lowering the threshold is therefore safer than it looked: tolerance for a mid-sentence pause is hangover plus silence_seconds, and a pause shorter than 640 ms never registers as silence at all. Sets it to 0.4, which keeps 1040 ms of tolerance and takes 300 ms off every command. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 5 +++- nixos/nixos/home-assistant-vad-silence.patch | 22 ++++++++++++++ nixos/nixos/prefix-cache-findings.md | 31 ++++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 nixos/nixos/home-assistant-vad-silence.patch diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 5f26f43..753ad97 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -260,7 +260,10 @@ # overview | marker | current time and live states. See # prefix-cache-findings.md. services.home-assistant.package = pkgs.home-assistant.overrideAttrs (old: { - patches = (old.patches or [ ]) ++ [ ./home-assistant-cache-boundary.patch ]; + patches = (old.patches or [ ]) ++ [ + ./home-assistant-cache-boundary.patch + ./home-assistant-vad-silence.patch + ]; }); # Voice assistant. With `prefer_local_intents` on (a per-pipeline setting, diff --git a/nixos/nixos/home-assistant-vad-silence.patch b/nixos/nixos/home-assistant-vad-silence.patch new file mode 100644 index 0000000..3a07a62 --- /dev/null +++ b/nixos/nixos/home-assistant-vad-silence.patch @@ -0,0 +1,22 @@ +--- a/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 00:54:25.108291810 +0000 ++++ b/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 00:54:25.124441473 +0000 +@@ -521,8 +521,18 @@ + is_vad_enabled: bool = True + """True if VAD is used to determine the end of the voice command.""" + +- silence_seconds: float = 0.7 ++ silence_seconds: float = 0.4 + """Seconds of silence after voice command has ended.""" ++ # Upstream default is 0.7. The VAD model in front of this counter, ++ # pymicro-vad, does not report silence promptly: fed digital silence it holds ++ # speech_probability at 1.000 for 400 ms and does not fall below the 0.5 ++ # in-command threshold until 640 ms. That hangover is added to this counter, ++ # so 0.7 means an end-of-speech decision 1340 ms after the speaker stops, ++ # which is the single largest cost in a voice interaction here. ++ # ++ # 0.4 gives 1040 ms of total tolerance for a mid-sentence pause -- still ++ # generous, because a pause shorter than the 640 ms hangover never registers ++ # as silence at all -- and takes 300 ms off every command. + + def __post_init__(self) -> None: + """Verify settings post-initialization.""" diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index aa012fb..4112330 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -706,3 +706,34 @@ Actionable regardless: `silence_seconds` is 0.7 and unreachable from the API, so lowering it needs a patch. That is worth doing after the residual is understood, not before -- otherwise a smaller threshold may just be swallowed by whatever the extra 600 ms is. + + +## The unexplained VAD time was the VAD model's hangover + +Running `pymicro_vad.MicroVad` -- the model Home Assistant uses, over 10 ms +frames -- directly on the same utterance followed by digital silence: + + 0 ms after speech end: 1.000 + 300 ms after speech end: 1.000 + 500 ms after speech end: 0.941 + 600 ms after speech end: 0.920 + 700 ms after speech end: 0.009 + + first frame at or below the in-command threshold (0.5): 640 ms + +It holds probability at 1.000 for 400 ms after the audio is silent and does not +fall below the threshold until 640 ms. HA's `silence_seconds` countdown only +starts once it does. + +So the 1306 ms measured from end of speech to `stt-vad-end` is +**640 ms of VAD hangover + 700 ms of silence_seconds**, and both parts are now +accounted for. Nothing is lagging: HA's own audio timestamps track wall clock, +so the pipeline keeps up in real time. + +This makes lowering the threshold safer than it first appears. Tolerance for a +mid-sentence pause is hangover + silence_seconds, and a pause shorter than 640 ms +never registers as silence at all. `home-assistant-vad-silence.patch` sets it to +0.4, giving 1040 ms of tolerance and taking 300 ms off every command. + +The 640 ms floor belongs to the model. Beating it means a different VAD, not a +different setting. From 1c2b8de7088ca8f728ed2d5c8f386d05a7f92a51 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 01:04:16 +0000 Subject: [PATCH 59/98] Control the VAD hangover against real speech and a noise floor The 640 ms rested on one Piper clip, which cannot distinguish a real model characteristic from mishandling unnaturally clean synthetic audio. Noise floor changes nothing: 630-640 ms whether the trailing audio is digital zero or noise up to -30 dBFS, and whether or not noise is mixed through the speech itself. Real human speech behaves the same. Cutting mid-word so speech is unambiguously ongoing: JFK 480 ms, an LDC/TIMIT sentence 600 ms, Piper 580 ms, Kokoro 530 ms. So the hangover is inherent to the model at roughly 340-610 ms, and the single-clip figure was at the high end. Also records that "release after the last audible sample" is an unsound measurement -- it scored JFK at 0 ms because a noisy recording's tail is audible without being speech. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 41 ++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 4112330..af90eaa 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -726,8 +726,8 @@ fall below the threshold until 640 ms. HA's `silence_seconds` countdown only starts once it does. So the 1306 ms measured from end of speech to `stt-vad-end` is -**640 ms of VAD hangover + 700 ms of silence_seconds**, and both parts are now -accounted for. Nothing is lagging: HA's own audio timestamps track wall clock, +**VAD hangover + silence_seconds**, and both parts are now accounted for. +(The 640 ms figure came from one clip; see the multi-source measurement below.) Nothing is lagging: HA's own audio timestamps track wall clock, so the pipeline keeps up in real time. This makes lowering the threshold safer than it first appears. Tolerance for a @@ -737,3 +737,40 @@ never registers as silence at all. `home-assistant-vad-silence.patch` sets it to The 640 ms floor belongs to the model. Beating it means a different VAD, not a different setting. + + +## The hangover is real, and not a TTS artefact + +The 640 ms above rested on a single Piper clip, which is not enough to tell +"this model always hangs on" from "this model mishandles unnaturally clean +synthetic audio". Two controls: + +**Noise floor makes no difference.** Same utterance, varying what follows it, +and with noise mixed through the speech as a real microphone would: + + digital zero 640 ms noise rms 32 everywhere 640 ms + noise rms 32 640 ms noise rms 100 everywhere 640 ms + noise rms 1000 630 ms noise rms 300 everywhere 640 ms + +**Real human speech behaves the same.** Cutting each clip mid-word, so speech is +unambiguously ongoing at the cut, then feeding silence: + +| source | median release | +|---|---| +| REAL: JFK, 1961 | 480 ms | +| REAL: LDC/TIMIT sentence | 600 ms | +| TTS: Piper en-us-ryan-medium | 580 ms | +| TTS: Kokoro | 530 ms | + +So it is inherent to the model, roughly **340-610 ms** depending on the clip, +and the single-clip 640 ms was at the high end. Others have hit this: rhasspy +/pymicro-vad issue 1 is exactly "vad end takes more time compared with the old +one", from when Home Assistant 2024.8 switched to micro_vad. + +Method note: "release after the last audible sample" is not a sound measurement. +It gave JFK 0 ms, because a noisy recording's tail is audible without being +speech. Cut mid-speech instead. + +The remaining lever is a different VAD. Silero releases far faster and exposes a +minimum-silence parameter. That is a bigger patch than a constant, but it is +where the floor actually is. From 53cf1a5bed63e77c4c666a4ef85f431c395785c6 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 01:06:50 +0000 Subject: [PATCH 60/98] Revert the VAD silence threshold patch It trims around the problem rather than fixing it. The VAD model holds speech for 340-610 ms after speech actually stops, and lowering silence_seconds just subtracts from the other term. Reverting restores the already-built derivation, so switching back costs no rebuild. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 5 +---- nixos/nixos/home-assistant-vad-silence.patch | 22 -------------------- 2 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 nixos/nixos/home-assistant-vad-silence.patch diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 753ad97..5f26f43 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -260,10 +260,7 @@ # overview | marker | current time and live states. See # prefix-cache-findings.md. services.home-assistant.package = pkgs.home-assistant.overrideAttrs (old: { - patches = (old.patches or [ ]) ++ [ - ./home-assistant-cache-boundary.patch - ./home-assistant-vad-silence.patch - ]; + patches = (old.patches or [ ]) ++ [ ./home-assistant-cache-boundary.patch ]; }); # Voice assistant. With `prefer_local_intents` on (a per-pipeline setting, diff --git a/nixos/nixos/home-assistant-vad-silence.patch b/nixos/nixos/home-assistant-vad-silence.patch deleted file mode 100644 index 3a07a62..0000000 --- a/nixos/nixos/home-assistant-vad-silence.patch +++ /dev/null @@ -1,22 +0,0 @@ ---- a/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 00:54:25.108291810 +0000 -+++ b/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 00:54:25.124441473 +0000 -@@ -521,8 +521,18 @@ - is_vad_enabled: bool = True - """True if VAD is used to determine the end of the voice command.""" - -- silence_seconds: float = 0.7 -+ silence_seconds: float = 0.4 - """Seconds of silence after voice command has ended.""" -+ # Upstream default is 0.7. The VAD model in front of this counter, -+ # pymicro-vad, does not report silence promptly: fed digital silence it holds -+ # speech_probability at 1.000 for 400 ms and does not fall below the 0.5 -+ # in-command threshold until 640 ms. That hangover is added to this counter, -+ # so 0.7 means an end-of-speech decision 1340 ms after the speaker stops, -+ # which is the single largest cost in a voice interaction here. -+ # -+ # 0.4 gives 1040 ms of total tolerance for a mid-sentence pause -- still -+ # generous, because a pause shorter than the 640 ms hangover never registers -+ # as silence at all -- and takes 300 ms off every command. - - def __post_init__(self) -> None: - """Verify settings post-initialization.""" From 1a679bb063b9123428b6e8045e36a265a1427e43 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 01:08:04 +0000 Subject: [PATCH 61/98] Explain the VAD latency: it is a wake-word model, so it has a window pymicro-vad uses microWakeWord's architecture -- a classifier over a sliding window. Feeding bursts of real speech: under ~400 ms never fires at all, 500 ms fires and holds 650 ms after the audio stops, 1200 ms holds 540 ms. That is a ~500 ms window, and it explains both edges measured earlier: onset ~340 ms late, release 340-610 ms. So this is not a bug or an oversight in training. A window-length latency is inherent and harmless for detecting a discrete wake word; it only hurts when the model is repurposed to decide when someone stopped speaking. It also means no threshold fixes it, since silence_seconds is added after the model reports silence. Hence reverting the patch that lowered it. The fix is a different VAD; audio_enhancer.py hardcodes MicroVad. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index af90eaa..712d13f 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -774,3 +774,41 @@ speech. Cut mid-speech instead. The remaining lever is a different VAD. Silero releases far faster and exposes a minimum-silence parameter. That is a bigger patch than a constant, but it is where the floor actually is. + + +## Why the model behaves this way: it is a wake-word model + +pymicro-vad's README says it "uses the machine learning architecture from +microWakeWord". That is a *wake word* architecture: a classifier over a sliding +window, answering "does this window contain the phrase". Used as a VAD it +answers "does this window contain speech", which is inherently about one window +late at both edges. + +Feeding bursts of real speech of varying length, surrounded by silence: + + burst peak prob during burst stays above 0.5 after + 50 ms 0.00 never fires + 100 ms 0.00 never fires + 200 ms 0.00 never fires + 300 ms 0.06 never fires + 500 ms 1.00 650 ms + 800 ms 1.00 650 ms + 1200 ms 1.00 540 ms + +Under ~400 ms of speech never registers at all, and once it fires it holds until +the window drains. That is a ~500 ms window, and it explains both edges we +measured: onset ~340 ms late, release ~340-610 ms. + +**This is not a bug and not an oversight by whoever trained it.** For wake-word +detection a window-length latency is inherent and harmless -- you are detecting +a discrete event, not tracking when someone stopped talking. It only becomes a +problem when the model is repurposed to decide end-of-speech, which is what Home +Assistant does with it. + +It also means no threshold can fix it. `silence_seconds` is added *after* the +model finally reports silence. The `home-assistant-vad-silence.patch` that set +it to 0.4 has been reverted for that reason: it trims the wrong term. + +The fix is a different VAD. Silero is the obvious candidate: it releases in tens +of milliseconds and exposes a minimum-silence parameter. That means patching +`audio_enhancer.py`, which hardcodes `MicroVad`. From d22e70c0162080dcd9fe3459dcd543c4fef64d67 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 01:17:38 +0000 Subject: [PATCH 62/98] Answer whether the VAD latency is intentional: partly, but not as a target Home Assistant retuned after switching to microVAD. Two weeks after the swap, "Adjust VAD seconds better for microVAD" moved every timing down -- 2.0 to 1.25, 0.5 to 0.25, 1.0 to 0.7 -- and three months later the speech threshold was split into separate before/during values. Both are someone clawing back latency the model added, not choosing how long a pause should feel. The ~1.2 s is the leftover, not the goal. The behaviour it produces is not a coherent tolerance either. Driving HA's own segmenter with real VAD output: pauses up to 850 ms are tolerated, pauses over 1100 ms end before the user resumes, but 900-1050 ms ends the command 30-180 ms AFTER they resumed speaking, because the model needs ~400 ms to notice speech starting as well. Also: do not assume Silero is the fix. HA shipped it in 2025-12 and reverted it a month later for lag, broken end-of-speech detection, crashes and macOS build failures. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 43 ++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 712d13f..315d968 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -812,3 +812,46 @@ it to 0.4 has been reverted for that reason: it trims the wrong term. The fix is a different VAD. Silero is the obvious candidate: it releases in tens of milliseconds and exposes a minimum-silence parameter. That means patching `audio_enhancer.py`, which hardcodes `MicroVad`. + + +## Is the current end-of-speech latency intentional? + +Partly, but not as a target -- and the leftover behaviour is incoherent. + +**They did retune after the swap.** Home Assistant went WebRTC -> microVAD in +`7f4dabf546f` (2024-07-31), and two weeks later `f2d39feec03` (2024-08-15), +titled "Adjust VAD seconds better for microVAD", moved every timing *down*: + + 2.0 -> 1.25 0.5 -> 0.25 1.0 -> 0.7 + +by Michael Hansen, who also wrote pymicro-vad. Then `69e3348cd79` (2024-11-05) +split one speech threshold into two, 0.2 before a command and 0.5 during it. +Both changes are someone clawing back latency the model added, not someone +choosing how long a pause should feel. So 0.7 is a microVAD-compensating value, +and the ~1.2 s that results is what was left over, not what was wanted. + +**And the resulting behaviour is not a coherent "tolerance".** Driving Home +Assistant's own segmenter with real VAD output, on speech / pause / speech: + +| pause | outcome | +|---|---| +| <= 850 ms | tolerated, command continues | +| **900-1050 ms** | **ends 30-180 ms AFTER the user resumed speaking** | +| >= 1100 ms | ends before the user resumes | + +The middle band exists because the model needs ~400 ms to notice speech +*starting* too, so a resumed sentence is not seen before the countdown expires. +Nobody designs a window where the assistant commits to stopping while you are +mid-word. The latency is delay, not tolerance. + +## Do not simply swap in Silero + +Home Assistant already tried. `079c6daa633` (2025-12-08) replaced microVAD with +Silero VAD (ggml); `329b2c840d8` (2026-01-13) reverted it. The revert PR cites +"2026.1 Introduces Lag to Voice Assist Speech-To-Text", "End-of-Speech Detection +Broken after upgrading to Core 26.1", "Voice assistant is killing Home +Assistant", and macOS build problems. + +So the obvious fix has been tried in production and regressed. Anything here +needs measuring on this machine rather than assuming, and the fallback is what +Home Assistant itself did: lower the timings and accept the model. From 9d8b19a71093280753db1802464bc37b51c0c45c Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 01:27:03 +0000 Subject: [PATCH 63/98] Try Silero VAD, reviving Home Assistant's own reverted implementation Measured on the same clips: microVAD keeps reporting speech for 480-600 ms after it stops and needs 400-500 ms before reporting any, because it is a wake-word architecture classifying over a ~500 ms window. Silero releases in 0-96 ms and fires on 100 ms of speech. That is ~500 ms off every voice command. The patch is HA's own, from 079c6daa633, before 329b2c840d8 reverted it for lag, broken end-of-speech detection, crashes and macOS build failures -- problems for a project shipping to every platform, worth re-testing on one machine that builds its own software. Also documents the knobs. There is one timing, silence_seconds, exposed as VadSensitivity: relaxed 1.25, default 0.7, aggressive 0.25 -- a select entity per satellite, so a dropdown rather than a patch. The earlier patch that lowered it was reinventing that and is gone. For calibration: human turn-taking gaps average ~200 ms and production systems wait 300-800 ms. Our ~1.2 s is well outside that; microVAD's window alone rules the band out. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 23 +++- nixos/nixos/home-assistant-silero-vad.patch | 128 ++++++++++++++++++++ nixos/nixos/prefix-cache-findings.md | 34 ++++++ 3 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 nixos/nixos/home-assistant-silero-vad.patch diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 5f26f43..be43237 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -260,9 +260,30 @@ # overview | marker | current time and live states. See # prefix-cache-findings.md. services.home-assistant.package = pkgs.home-assistant.overrideAttrs (old: { - patches = (old.patches or [ ]) ++ [ ./home-assistant-cache-boundary.patch ]; + patches = (old.patches or [ ]) ++ [ + ./home-assistant-cache-boundary.patch + # Home Assistant's own Silero implementation, lifted from the commit that + # added it (079c6daa633) before it was reverted a month later in + # 329b2c840d8. The revert cites lag, broken end-of-speech detection, + # crashes and macOS build failures -- problems for a project shipping to + # every platform, and worth re-testing on one machine that builds its own + # software. + # + # The reason to want it, measured here on the same clips: microVAD keeps + # reporting speech for 480-600 ms after speech stops and needs 400-500 ms + # of speech before it reports any, because it is a wake-word architecture + # classifying over a ~500 ms window. Silero releases in 0-96 ms and fires + # on 100 ms of speech. That is ~500 ms off every single voice command, + # larger than anything left anywhere else in the pipeline. + ./home-assistant-silero-vad.patch + ]; }); + # audio_enhancer.py imports pysilero_vad after the patch above. The manifest + # change in that patch does not pull the dependency in, because nixpkgs + # resolves component requirements before patches are applied. + services.home-assistant.extraPackages = ps: [ ps.pysilero-vad ]; + # Voice assistant. With `prefer_local_intents` on (a per-pipeline setting, # off by default) the built-in sentence matcher answers anything it # recognises without involving the model, so "turn off the kitchen light" diff --git a/nixos/nixos/home-assistant-silero-vad.patch b/nixos/nixos/home-assistant-silero-vad.patch new file mode 100644 index 0000000..9701e53 --- /dev/null +++ b/nixos/nixos/home-assistant-silero-vad.patch @@ -0,0 +1,128 @@ +commit 079c6daa633f89f34328baf42587cee91f1c28f0 +Author: Michael Hansen +Date: Mon Dec 8 20:02:14 2025 -0600 + + Replace microVAD with Silero VAD (ggml) (#158282) + +diff --git a/homeassistant/components/assist_pipeline/audio_enhancer.py b/homeassistant/components/assist_pipeline/audio_enhancer.py +index 1fabc7790e7..18f00d58d8a 100644 +--- a/homeassistant/components/assist_pipeline/audio_enhancer.py ++++ b/homeassistant/components/assist_pipeline/audio_enhancer.py +@@ -3,8 +3,9 @@ + from abc import ABC, abstractmethod + from dataclasses import dataclass + import logging ++import math + +-from pymicro_vad import MicroVad ++from pysilero_vad import SileroVoiceActivityDetector + from pyspeex_noise import AudioProcessor + + from .const import BYTES_PER_CHUNK +@@ -42,8 +43,8 @@ class AudioEnhancer(ABC): + """Enhance chunk of PCM audio @ 16Khz with 16-bit mono samples.""" + + +-class MicroVadSpeexEnhancer(AudioEnhancer): +- """Audio enhancer that runs microVAD and speex.""" ++class SileroVadSpeexEnhancer(AudioEnhancer): ++ """Audio enhancer that runs Silero VAD and speex.""" + + def __init__( + self, auto_gain: int, noise_suppression: int, is_vad_enabled: bool +@@ -69,21 +70,49 @@ class MicroVadSpeexEnhancer(AudioEnhancer): + self.noise_suppression, + ) + +- self.vad: MicroVad | None = None ++ self.vad: SileroVoiceActivityDetector | None = None ++ ++ # We get 10ms chunks but Silero works on 32ms chunks, so we have to ++ # buffer audio. The previous speech probability is used until enough ++ # audio has been buffered. ++ self._vad_buffer: bytearray | None = None ++ self._vad_buffer_chunks = 0 ++ self._vad_buffer_chunk_idx = 0 ++ self._last_speech_probability: float | None = None + + if self.is_vad_enabled: +- self.vad = MicroVad() +- _LOGGER.debug("Initialized microVAD") ++ self.vad = SileroVoiceActivityDetector() ++ ++ # VAD buffer is a multiple of 10ms, but Silero VAD needs 32ms. ++ self._vad_buffer_chunks = int( ++ math.ceil(self.vad.chunk_bytes() / BYTES_PER_CHUNK) ++ ) ++ self._vad_leftover_bytes = self.vad.chunk_bytes() - BYTES_PER_CHUNK ++ self._vad_buffer = bytearray(self.vad.chunk_bytes()) ++ _LOGGER.debug("Initialized Silero VAD") + + def enhance_chunk(self, audio: bytes, timestamp_ms: int) -> EnhancedAudioChunk: + """Enhance 10ms chunk of PCM audio @ 16Khz with 16-bit mono samples.""" +- speech_probability: float | None = None +- + assert len(audio) == BYTES_PER_CHUNK + + if self.vad is not None: + # Run VAD +- speech_probability = self.vad.Process10ms(audio) ++ assert self._vad_buffer is not None ++ start_idx = self._vad_buffer_chunk_idx * BYTES_PER_CHUNK ++ self._vad_buffer[start_idx : start_idx + BYTES_PER_CHUNK] = audio ++ ++ self._vad_buffer_chunk_idx += 1 ++ if self._vad_buffer_chunk_idx >= self._vad_buffer_chunks: ++ # We have enough data to run Silero VAD (32 ms) ++ self._last_speech_probability = self.vad.process_chunk( ++ self._vad_buffer[: self.vad.chunk_bytes()] ++ ) ++ ++ # Copy leftover audio that wasn't processed to start ++ self._vad_buffer[: self._vad_leftover_bytes] = self._vad_buffer[ ++ -self._vad_leftover_bytes : ++ ] ++ self._vad_buffer_chunk_idx = 0 + + if self.audio_processor is not None: + # Run noise suppression and auto gain +@@ -92,5 +121,5 @@ class MicroVadSpeexEnhancer(AudioEnhancer): + return EnhancedAudioChunk( + audio=audio, + timestamp_ms=timestamp_ms, +- speech_probability=speech_probability, ++ speech_probability=self._last_speech_probability, + ) +diff --git a/homeassistant/components/assist_pipeline/manifest.json b/homeassistant/components/assist_pipeline/manifest.json +index d88e4352130..8c968b83860 100644 +--- a/homeassistant/components/assist_pipeline/manifest.json ++++ b/homeassistant/components/assist_pipeline/manifest.json +@@ -8,5 +8,5 @@ + "integration_type": "system", + "iot_class": "local_push", + "quality_scale": "internal", +- "requirements": ["pymicro-vad==1.0.1", "pyspeex-noise==1.0.2"] ++ "requirements": ["pysilero-vad==3.0.0", "pyspeex-noise==1.0.2"] + } +diff --git a/homeassistant/components/assist_pipeline/pipeline.py b/homeassistant/components/assist_pipeline/pipeline.py +index 0948413d4cc..abfc4e72782 100644 +--- a/homeassistant/components/assist_pipeline/pipeline.py ++++ b/homeassistant/components/assist_pipeline/pipeline.py +@@ -55,7 +55,7 @@ from homeassistant.util import ( + from homeassistant.util.hass_dict import HassKey + from homeassistant.util.limited_size_dict import LimitedSizeDict + +-from .audio_enhancer import AudioEnhancer, EnhancedAudioChunk, MicroVadSpeexEnhancer ++from .audio_enhancer import AudioEnhancer, EnhancedAudioChunk, SileroVadSpeexEnhancer + from .const import ( + ACKNOWLEDGE_PATH, + BYTES_PER_CHUNK, +@@ -633,7 +633,7 @@ class PipelineRun: + # Initialize with audio settings + if self.audio_settings.needs_processor and (self.audio_enhancer is None): + # Default audio enhancer +- self.audio_enhancer = MicroVadSpeexEnhancer( ++ self.audio_enhancer = SileroVadSpeexEnhancer( + self.audio_settings.auto_gain_dbfs, + self.audio_settings.noise_suppression_level, + self.audio_settings.is_vad_enabled, diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 315d968..5d187b6 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -855,3 +855,37 @@ Assistant", and macOS build problems. So the obvious fix has been tried in production and regressed. Anything here needs measuring on this machine rather than assuming, and the fallback is what Home Assistant itself did: lower the timings and accept the model. + + +## The VAD knobs, and what the research says + +There is one timing knob, exposed at three preset levels, plus supporting +counters. From `assist_pipeline/vad.py`: + +| knob | value | meaning | +|---|---|---| +| `silence_seconds` | 0.7 | silence needed after speech to end the command | +| `speech_seconds` | 0.3 | speech needed before a command is considered started | +| `command_seconds` | 1.0 | minimum command length before it may end | +| `reset_seconds` | 1.0 | continuous speech needed to reset the silence counter | +| `timeout_seconds` | 15.0 | give up | +| `before_command_speech_threshold` | 0.2 | probability counted as speech before the command | +| `in_command_speech_threshold` | 0.5 | probability counted as speech during it | + +`silence_seconds` is what `VadSensitivity` selects: **relaxed 1.25, default 0.7, +aggressive 0.25**. That is a select entity per satellite (esphome, wyoming, voip +all register one), so it is a dropdown in the UI, not a patch. The +`home-assistant-vad-silence.patch` written earlier was reinventing it, and was +reverted. + +**Research context.** Human turn-taking gaps average ~200 ms across languages, +and production voice systems use silence thresholds of roughly 300-800 ms before +deciding a turn ended. Our effective ~1.2 s is well above that band. Silero +plus the default 0.7 would be ~0.8 s; Silero plus aggressive ~0.35 s, which is +at the fast end of what industry ships. The current setup cannot get there: +microVAD's window alone is ~500 ms. + +Beyond thresholds, the current direction in voice agents is semantic +endpointing -- letting a model judge whether the transcript sounds finished +rather than counting silence. That is plausible here eventually, given a local +LLM answering in ~100 ms. From 43febb3f3080beeeaae9ecb353b671b0497724e0 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 01:37:12 +0000 Subject: [PATCH 64/98] Fix the Silero build: the dependency has to be a package input nixpkgs runs Home Assistant's test suite during the build, which imports pysilero_vad, so services.home-assistant.extraPackages is too late -- that builds the service's runtime environment, not the package. Adding it to propagatedBuildInputs from home-assistant.passthru.python3Packages, which is the matching interpreter's package set, builds clean. Verified in the output: audio_enhancer.py imports SileroVoiceActivityDetector, pipeline.py constructs SileroVadSpeexEnhancer, and the cache-boundary patch is still applied. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 14 +++++++++----- nixos/nixos/prefix-cache-findings.md | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index be43237..10d9ff7 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -277,12 +277,16 @@ # larger than anything left anywhere else in the pipeline. ./home-assistant-silero-vad.patch ]; - }); - # audio_enhancer.py imports pysilero_vad after the patch above. The manifest - # change in that patch does not pull the dependency in, because nixpkgs - # resolves component requirements before patches are applied. - services.home-assistant.extraPackages = ps: [ ps.pysilero-vad ]; + # audio_enhancer.py imports pysilero_vad after that patch. The manifest + # change in the patch does not pull it in, because nixpkgs resolves + # component requirements before patches are applied -- and it has to be a + # package input rather than services.home-assistant.extraPackages, because + # the test suite runs during the build and imports it too. + propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ + pkgs.home-assistant.python3Packages.pysilero-vad + ]; + }); # Voice assistant. With `prefer_local_intents` on (a per-pipeline setting, # off by default) the built-in sentence matcher answers anything it diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index 5d187b6..ebbc5e0 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -889,3 +889,26 @@ Beyond thresholds, the current direction in voice agents is semantic endpointing -- letting a model judge whether the transcript sounds finished rather than counting silence. That is plausible here eventually, given a local LLM answering in ~100 ms. + + +## Building the Silero patch + +The first attempt failed: + + ModuleNotFoundError: No module named 'pysilero_vad' + ====== 10 failed, 7230 passed, 2 skipped ====== + +nixpkgs runs Home Assistant's test suite during the build, so the dependency +has to be a package input. `services.home-assistant.extraPackages` is too late: +it builds the service's runtime environment, not the package. Use + + propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ + pkgs.home-assistant.python3Packages.pysilero-vad + ]; + +`home-assistant.passthru.python3Packages` is the matching interpreter's package +set; `pkgs.python3Packages` would be the wrong one. + +Verified in the built output: `audio_enhancer.py` imports +`SileroVoiceActivityDetector`, `pipeline.py` constructs +`SileroVadSpeexEnhancer`, and the cache-boundary patch is still applied. From 75d4b15f3f4bb9f63b6c2f38234db26bf8813677 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 01:43:48 +0000 Subject: [PATCH 65/98] Silero measured: 1306 ms -> 704 ms, and make the knob testable The model now contributes essentially nothing. Onset is detected 50 ms late against 340 before, the end 20 ms late against 640, and end of speech to stt-end drops from 1301 ms to 833 ms. The remaining 704 ms is silence_seconds itself, so the timings finally mean what they say. Also makes silence_seconds settable on a pipeline run. The websocket API already accepted four of the five audio settings; this one was reachable only through the VAD sensitivity select entity that satellite integrations create, so it could not be tuned or tested without buying hardware. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 6 +++++ .../home-assistant-vad-silence-api.patch | 27 +++++++++++++++++++ nixos/nixos/prefix-cache-findings.md | 22 +++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 nixos/nixos/home-assistant-vad-silence-api.patch diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 10d9ff7..672385a 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -276,6 +276,12 @@ # on 100 ms of speech. That is ~500 ms off every single voice command, # larger than anything left anywhere else in the pipeline. ./home-assistant-silero-vad.patch + + # Lets a pipeline run set silence_seconds. The websocket API already + # accepts four of the five audio settings; this one was reachable only + # through the VAD sensitivity select entity that satellite integrations + # create, so it could not be tested or tuned without buying hardware. + ./home-assistant-vad-silence-api.patch ]; # audio_enhancer.py imports pysilero_vad after that patch. The manifest diff --git a/nixos/nixos/home-assistant-vad-silence-api.patch b/nixos/nixos/home-assistant-vad-silence-api.patch new file mode 100644 index 0000000..c9211d9 --- /dev/null +++ b/nixos/nixos/home-assistant-vad-silence-api.patch @@ -0,0 +1,27 @@ +--- a/homeassistant/components/assist_pipeline/websocket_api.py 2026-08-18 01:41:42.089893820 +0000 ++++ b/homeassistant/components/assist_pipeline/websocket_api.py 2026-08-18 01:41:42.104626881 +0000 +@@ -96,6 +96,12 @@ + vol.Optional("volume_multiplier"): float, + # Advanced use cases/testing + vol.Optional("no_vad"): bool, ++ # Seconds of silence before end of speech is ++ # declared. Settable per run so the knob can be ++ # exercised without satellite hardware; without this ++ # it is reachable only through the VAD sensitivity ++ # select entity that satellite integrations create. ++ vol.Optional("silence_seconds"): vol.Any(float, int), + } + }, + extra=vol.ALLOW_EXTRA, +@@ -213,6 +219,11 @@ + auto_gain_dbfs=msg_input.get("auto_gain_dbfs", 0), + volume_multiplier=msg_input.get("volume_multiplier", 1.0), + is_vad_enabled=not msg_input.get("no_vad", False), ++ **( ++ {"silence_seconds": float(msg_input["silence_seconds"])} ++ if "silence_seconds" in msg_input ++ else {} ++ ), + ) + elif start_stage == PipelineStage.INTENT: + # Input to conversation agent diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index ebbc5e0..d99394c 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -912,3 +912,25 @@ set; `pkgs.python3Packages` would be the wrong one. Verified in the built output: `audio_enhancer.py` imports `SileroVoiceActivityDetector`, `pipeline.py` constructs `SileroVadSpeexEnhancer`, and the cache-boundary patch is still applied. + + +## Silero: measured + +Same harness, same clip, after the swap: + +| | microVAD | Silero | +|---|---|---| +| onset detected | +340 ms late | **+50 ms** | +| end detected | +640 ms late | **+20 ms** | +| end of speech -> `stt-vad-end` | 1306 ms | **704 ms** | +| end of speech -> `stt-end` | 1301 ms | **833 ms** | + +The model now contributes essentially nothing: the 704 ms *is* `silence_seconds` += 0.7. About 600 ms saved on every voice command, and the timings finally mean +what they say. + +`home-assistant-vad-silence-api.patch` makes `silence_seconds` settable on a +pipeline run. The websocket API already accepted four of the five audio settings +and this one was reachable only through the VAD sensitivity select entity that +satellite integrations create, so it could not be tuned or even tested without +buying hardware. From d687e0b1fec95db6102c64021696857a44ae20be Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 02:00:21 +0000 Subject: [PATCH 66/98] Sweep silence_seconds: latency and pause tolerance are the same number With Silero the knob is linear: 0.25 gives 376 ms to stt-end and tolerates a 250 ms pause, 0.7 gives 789 ms and 700 ms, 1.25 gives 1401 ms and 1300 ms. Which reframes the Silero win. Latency and tolerance are the same quantity for both VADs; what differs is the constant added to each. microVAD's ~500 ms window added to both, putting its floor near 750 ms of latency even at the aggressive setting. Silero adds ~50 ms, so the fast end of the curve becomes reachable. It is not a better tradeoff, it is access to more of it. Picking a point is therefore preference, not optimisation. Production systems sit at 300-800 ms, spanning 0.25 to 0.7 here. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 29 ++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index d99394c..fd0df26 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -934,3 +934,32 @@ pipeline run. The websocket API already accepted four of the five audio settings and this one was reachable only through the VAD sensitivity select entity that satellite integrations create, so it could not be tuned or even tested without buying hardware. + + +## Choosing silence_seconds, with Silero + +Sweeping the knob now that it is settable per run, and separately measuring how +long a mid-sentence pause survives before the command is cut off: + +| silence_seconds | latency to stt-end | longest pause survived | +|---|---|---| +| 0.25 (aggressive) | 376 ms | 250 ms | +| 0.4 | 480 ms | 450 ms | +| 0.5 | ~580 ms | 550 ms | +| 0.7 (default) | 789 ms | 700 ms | +| 1.25 (relaxed) | 1401 ms | 1300 ms | + +**Latency and pause tolerance are the same number.** They always were, for both +VADs -- what differs is the constant added to each. microVAD's ~500 ms window +added to both, so its floor was around 750 ms of latency even at the aggressive +setting. Silero adds ~50 ms, so the fast end of the curve is reachable at all. +That is the win: not a better tradeoff, but access to a part of the curve that +was previously unreachable. + +So this is a genuine preference, not an optimisation. Production voice systems +sit at 300-800 ms, which spans 0.25 to 0.7 here. 0.5 is a reasonable middle: +~580 ms to respond, and half-second thinking pauses survive. + +No silence threshold can distinguish "still thinking" from "finished" -- they +are acoustically identical, and only the words differ. That is what semantic +endpointing is for. From 623a3262b5a05c2cf98bac58fbfba52243b1428f Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 02:03:31 +0000 Subject: [PATCH 67/98] Correct the Silero summary: it also removes the cut-off-mid-word band Saying Silero only lowers the reachable floor was wrong. The danger zone -- pauses where the command ends while the user is already speaking again -- is set by how long the VAD takes to notice speech restarting. microVAD needs ~400 ms, giving a band 100-200 ms wide at every setting tried (520-620 ms at 0.25, up to 880-1060 ms at 0.7). Silero notices in 30-100 ms and there is no such band at any setting. So it is two wins, not one: access to the fast end of the curve, and never committing to stop while the user is mid-word. Measurement note: "fired after the user resumed" is the wrong criterion, since firing at the end of the resumed speech is normal. It invented a 260 ms band for Silero that does not exist. The right criterion is fired during the resumed speech. Co-Authored-By: Claude Opus 5 --- nixos/nixos/prefix-cache-findings.md | 33 +++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/nixos/nixos/prefix-cache-findings.md b/nixos/nixos/prefix-cache-findings.md index fd0df26..ef34f92 100644 --- a/nixos/nixos/prefix-cache-findings.md +++ b/nixos/nixos/prefix-cache-findings.md @@ -949,12 +949,33 @@ long a mid-sentence pause survives before the command is cut off: | 0.7 (default) | 789 ms | 700 ms | | 1.25 (relaxed) | 1401 ms | 1300 ms | -**Latency and pause tolerance are the same number.** They always were, for both -VADs -- what differs is the constant added to each. microVAD's ~500 ms window -added to both, so its floor was around 750 ms of latency even at the aggressive -setting. Silero adds ~50 ms, so the fast end of the curve is reachable at all. -That is the win: not a better tradeoff, but access to a part of the curve that -was previously unreachable. +**Latency and pause tolerance move together**, for both VADs -- what differs is +the constant added to each. microVAD's ~500 ms window added to both, so its +floor was around 750 ms of latency even at the aggressive setting. Silero adds +~50 ms, so the fast end of the curve becomes reachable. + +**But Silero also removes the cut-off-mid-word failure mode entirely**, which is +a separate and arguably bigger win. The danger zone -- pauses where the command +ends *while the user is already speaking again* -- exists because the VAD needs +time to notice speech restarting, and its width is that rise time: + +| VAD, silence_seconds | pauses cut off mid-resumption | +|---|---| +| microVAD 0.25 | 520-620 ms (120 ms wide) | +| microVAD 0.3 | 580-660 ms (100 ms wide) | +| microVAD 0.4 | 600-760 ms (180 ms wide) | +| microVAD 0.7 | 880-1060 ms (200 ms wide) | +| **Silero, any of 0.25 / 0.3 / 0.4 / 0.7** | **none** | + +microVAD needs ~400 ms to notice speech restarting, so the countdown can expire +inside that gap. Silero notices in 30-100 ms and the band closes completely. So +Silero is not merely a shifted curve: with it, the assistant never commits to +stopping while you are mid-word. + +Measurement note: counting "fired after the user resumed" is wrong, because +firing at the *end* of the resumed speech is normal. The band is "fired *during* +the resumed speech". The looser criterion invented a 260 ms band for Silero that +does not exist. So this is a genuine preference, not an optimisation. Production voice systems sit at 300-800 ms, which spans 0.25 to 0.7 here. 0.5 is a reasonable middle: From d91b9d253148474c2f720e475a7ec24c79a5ffe5 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 02:18:36 +0000 Subject: [PATCH 68/98] Declare the model the assistant actually uses Home Assistant's conversation agent is set to qwen3.8:27b-mtp-q8_0, but loadModels declared only qwen3.6 models, so the one the assistant depends on was not pulled by a rebuild and a fresh machine would have broken. I probably caused that: runeval.py sets the model through the reconfigure flow, and killing the two-model eval partway would have left it on qwen3.8 without restoring it. Stopping a job that mutates configuration means checking what it changed. Also refreshes the stale comment. qwen3.8 no longer needs a newer ollama than we run, and "generation speed is not the binding constraint" is no longer true now that prefill is ~0.1 s. Co-Authored-By: Claude Opus 5 --- nixos/nixos/configuration.nix | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 672385a..3c0b58e 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -233,18 +233,20 @@ # There must be exactly ONE marker in the prompt: two would mean two # checkpoints, and the on-device snapshot only holds one. }; - # ~54 GB of downloads, pulled by ollama-model-loader.service after switch. - # qwen3.8 needs ollama >= 0.32.12; 26.05 ships 0.32.3, so it's out for now. - # The dense model is the assistant, despite being 5x slower to generate. - # ollama 0.32.3 faults with "CUDA error: an illegal memory access was - # encountered" when qwen35moe does constrained decoding for tool calls with - # array/enum parameters — which is exactly what Home Assistant sends. It is - # intermittent, poisons the runner's CUDA context, and does not reproduce on - # the dense model. Generation speed is not the binding constraint for voice - # anyway: a 30-token reply is under half a second either way. + # Pulled by ollama-model-loader.service after a switch. The conversation + # agent is chosen in Home Assistant's UI, so whatever it points at has to be + # declared here or a fresh machine would not have it. + # + # The dense model is the assistant. ollama 0.32.3 faulted with "CUDA error: + # an illegal memory access was encountered" when qwen35moe did constrained + # decoding for tool calls with array/enum parameters, which is exactly what + # Home Assistant sends; intermittent, and it poisoned the runner's CUDA + # context. Untested since 0.32.13, and worth retrying: the MoE models answer + # a short question in ~140 ms against ~310 ms for the dense ones, and now + # that prefill is ~0.1 s that difference is most of the model's cost. loadModels = [ - "qwen3.6:27b-mtp-q8_0" # 30 GB, index 38. The assistant. - "qwen3.6:35b-a3b-q4_K_M" # 24 GB, index 32 — faster, but see above. + "qwen3.8:27b-mtp-q8_0" # 28 GiB. The assistant. + "qwen3.6:35b-a3b-q4_K_M" # 22 GiB, MoE — much faster, but see above. ]; }; From 32446fe783b7e0e39d64b467aa3abde6f60da238 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 02:22:32 +0000 Subject: [PATCH 69/98] Move the patched packages out of the NixOS config The patches were inline in configuration.nix, which meant they could only be used by the machine that file configures, and only via sudo on stable nixpkgs. They are now packages/{ollama,home-assistant}.nix behind an overlay, exposed as flake outputs for x86_64-linux, aarch64-linux and aarch64-darwin. That makes the same definitions usable from a Home Manager config, from the agent VM, or on a Mac -- `nix run .#ollama-patched`. None of the patches need a GPU to build, and ollama is not marked broken on aarch64-darwin, so the Mac comparison needs no further packaging work. The overlay must come last in nixpkgs.overlays: it patches prev.ollama-cuda, and the overlay above replaces that with the unstable 0.32.13 the patches were written against. Running first would patch stable's 0.32.3 and fail. No derivation changed: Home Assistant still evaluates to 1582r9h0l5ajqf9nk1znw4i2ngsw35aq, the path already built, because Nix names patch files by content and basename rather than by directory. Co-Authored-By: Claude Opus 5 --- flake.nix | 42 ++++++++-- nixos/nixos/configuration.nix | 76 +++---------------- .../home-assistant-cache-boundary.patch | 0 .../home-assistant-silero-vad.patch | 0 .../home-assistant-vad-silence-api.patch | 0 packages/home-assistant.nix | 46 +++++++++++ .../llama-checkpoint-dedup.patch | 0 .../ollama-message-delimiters.patch | 0 packages/ollama.nix | 33 ++++++++ packages/overlay.nix | 13 ++++ .../prefix-cache-findings.md | 0 .../prefix-cache-statecheck.py | 0 12 files changed, 140 insertions(+), 70 deletions(-) rename {nixos/nixos => packages}/home-assistant-cache-boundary.patch (100%) rename {nixos/nixos => packages}/home-assistant-silero-vad.patch (100%) rename {nixos/nixos => packages}/home-assistant-vad-silence-api.patch (100%) create mode 100644 packages/home-assistant.nix rename {nixos/nixos => packages}/llama-checkpoint-dedup.patch (100%) rename {nixos/nixos => packages}/ollama-message-delimiters.patch (100%) create mode 100644 packages/ollama.nix create mode 100644 packages/overlay.nix rename {nixos/nixos => packages}/prefix-cache-findings.md (100%) rename {nixos/nixos => packages}/prefix-cache-statecheck.py (100%) diff --git a/flake.nix b/flake.nix index 4ee0575..065b4ee 100644 --- a/flake.nix +++ b/flake.nix @@ -40,11 +40,43 @@ moss, }: { - packages = { - x86_64-linux.home-manager = home-manager.packages.x86_64-linux.default; - aarch64-linux.home-manager = home-manager.packages.aarch64-linux.default; - aarch64-darwin.home-manager = home-manager.packages.aarch64-darwin.default; - }; + # Patched ollama and Home Assistant, so the same definitions can be used + # from a NixOS config, a Home Manager config, another machine's `nix run`, + # or a Mac. See packages/prefix-cache-findings.md for what the patches do. + overlays.default = import ./packages/overlay.nix; + + packages = + let + patched = + system: + let + pkgs = import nixpkgs { + inherit system; + config.allowUnfree = true; + overlays = [ (import ./packages/overlay.nix) ]; + }; + in + { + inherit (pkgs) ollama-patched; + } + // nixpkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux { + inherit (pkgs) ollama-cuda-patched home-assistant-patched; + }; + in + { + x86_64-linux = { + home-manager = home-manager.packages.x86_64-linux.default; + } + // patched "x86_64-linux"; + aarch64-linux = { + home-manager = home-manager.packages.aarch64-linux.default; + } + // patched "aarch64-linux"; + aarch64-darwin = { + home-manager = home-manager.packages.aarch64-darwin.default; + } + // patched "aarch64-darwin"; + }; nixosConfigurations = { "nixos" = nixpkgs-stable.lib.nixosSystem { # So the host can take individual packages from unstable. diff --git a/nixos/nixos/configuration.nix b/nixos/nixos/configuration.nix index 3c0b58e..bc4b8d5 100644 --- a/nixos/nixos/configuration.nix +++ b/nixos/nixos/configuration.nix @@ -146,26 +146,7 @@ # https://wiki.nixos.org/wiki/Ollama services.ollama = { enable = true; - # llama-server can place a context checkpoint exactly at the start of each - # message, but only if it is told where messages begin: it scans the prompt - # for delimiter strings passed in the request's "message_delimiters" field. - # The chat-completions path fills that in from the chat template. ollama - # renders qwen's template itself in Go and posts a flat string to - # /completion, leaving the field empty, so llama-server sees an opaque - # prompt and falls back to checkpointing near the end of it. The patch has - # renderers report their delimiters and passes them through. - # - # The second patch is against llama.cpp, not ollama. nixpkgs pre-stages - # llama.cpp into $TMPDIR/llama-cpp-src at the end of postPatch so the - # CMake FetchContent step does not reach the network, which gives us a - # place to patch it. Note that nixpkgs pins b10091 while ollama 0.32.13 - # asks for b10380, so read b10091 when reasoning about this host. - package = pkgs.ollama-cuda.overrideAttrs (old: { - patches = (old.patches or [ ]) ++ [ ./ollama-message-delimiters.patch ]; - postPatch = (old.postPatch or "") + '' - patch -d "$TMPDIR/llama-cpp-src" -p1 < ${./llama-checkpoint-dedup.patch} - ''; - }); + package = pkgs.ollama-cuda-patched; # see packages/ollama.nix # 0.0.0.0 so libvirt guests can reach it at 192.168.122.1; the firewall # only opens 11434 on virbr0. host = "0.0.0.0"; @@ -250,51 +231,7 @@ ]; }; - # Home Assistant assembles the system prompt as: the configured prompt - # template, then the API preamble and the exposed entity overview, then - # anything extra. That puts unchanging content *after* the template, so a - # cache boundary placed at the end of the template would leave the entity - # overview outside the cached region -- around 300-400 tokens re-read on every - # request, roughly 200 ms, for content that never changes. - # - # The patch moves everything after the boundary marker to the very end - # instead, so the prompt is: unchanging instructions, preamble and entity - # overview | marker | current time and live states. See - # prefix-cache-findings.md. - services.home-assistant.package = pkgs.home-assistant.overrideAttrs (old: { - patches = (old.patches or [ ]) ++ [ - ./home-assistant-cache-boundary.patch - # Home Assistant's own Silero implementation, lifted from the commit that - # added it (079c6daa633) before it was reverted a month later in - # 329b2c840d8. The revert cites lag, broken end-of-speech detection, - # crashes and macOS build failures -- problems for a project shipping to - # every platform, and worth re-testing on one machine that builds its own - # software. - # - # The reason to want it, measured here on the same clips: microVAD keeps - # reporting speech for 480-600 ms after speech stops and needs 400-500 ms - # of speech before it reports any, because it is a wake-word architecture - # classifying over a ~500 ms window. Silero releases in 0-96 ms and fires - # on 100 ms of speech. That is ~500 ms off every single voice command, - # larger than anything left anywhere else in the pipeline. - ./home-assistant-silero-vad.patch - - # Lets a pipeline run set silence_seconds. The websocket API already - # accepts four of the five audio settings; this one was reachable only - # through the VAD sensitivity select entity that satellite integrations - # create, so it could not be tested or tuned without buying hardware. - ./home-assistant-vad-silence-api.patch - ]; - - # audio_enhancer.py imports pysilero_vad after that patch. The manifest - # change in the patch does not pull it in, because nixpkgs resolves - # component requirements before patches are applied -- and it has to be a - # package input rather than services.home-assistant.extraPackages, because - # the test suite runs during the build and imports it too. - propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ - pkgs.home-assistant.python3Packages.pysilero-vad - ]; - }); + services.home-assistant.package = pkgs.home-assistant-patched; # packages/home-assistant.nix # Voice assistant. With `prefer_local_intents` on (a per-pipeline setting, # off by default) the built-in sentence matcher answers anything it @@ -531,6 +468,15 @@ ''; }); }) + + # Patched ollama and Home Assistant. Defined outside this file so the same + # definitions serve a Home Manager config, the agent VM, or a Mac -- none of + # the patches need a GPU to build. + # + # MUST come after the unstable ollama-cuda overlay above: it patches + # prev.ollama-cuda, and if it ran first that would be stable's 0.32.3, which + # the patches do not apply to. + (import ../../packages/overlay.nix) ]; # https://wiki.nixos.org/wiki/Docker#System_setup diff --git a/nixos/nixos/home-assistant-cache-boundary.patch b/packages/home-assistant-cache-boundary.patch similarity index 100% rename from nixos/nixos/home-assistant-cache-boundary.patch rename to packages/home-assistant-cache-boundary.patch diff --git a/nixos/nixos/home-assistant-silero-vad.patch b/packages/home-assistant-silero-vad.patch similarity index 100% rename from nixos/nixos/home-assistant-silero-vad.patch rename to packages/home-assistant-silero-vad.patch diff --git a/nixos/nixos/home-assistant-vad-silence-api.patch b/packages/home-assistant-vad-silence-api.patch similarity index 100% rename from nixos/nixos/home-assistant-vad-silence-api.patch rename to packages/home-assistant-vad-silence-api.patch diff --git a/packages/home-assistant.nix b/packages/home-assistant.nix new file mode 100644 index 0000000..a06d31d --- /dev/null +++ b/packages/home-assistant.nix @@ -0,0 +1,46 @@ +# Home Assistant, patched for voice latency. See ./prefix-cache-findings.md. +home-assistant: + +home-assistant.overrideAttrs (old: { + patches = (old.patches or [ ]) ++ [ + # Home Assistant builds the system prompt as: the configured template, then + # the API preamble and the exposed entity overview, then anything extra. + # That puts unchanging content *after* the template, so a cache boundary + # placed at the end of the template would leave the entity overview outside + # the cached region -- 300-400 tokens re-read on every request, about + # 200 ms, for content that never changes. This moves everything after the + # boundary marker to the very end instead. + ./home-assistant-cache-boundary.patch + + # Home Assistant's own Silero VAD implementation, lifted from the commit + # that added it (079c6daa633) before it was reverted a month later in + # 329b2c840d8 for lag, broken end-of-speech detection, crashes and macOS + # build failures -- problems for a project shipping to every platform. + # + # microVAD is a wake-word architecture classifying over a ~500 ms window, so + # it keeps reporting speech for 480-600 ms after speech stops and needs + # 400-500 ms before reporting any. Silero releases in 0-96 ms and fires on + # 100 ms of speech. That is ~600 ms off every command, and it also closes + # the band of pause lengths where the old VAD would cut you off *after* you + # had already started speaking again. + ./home-assistant-silero-vad.patch + + # Lets a pipeline run set silence_seconds. The websocket API already accepts + # four of the five audio settings; this one was reachable only through the + # VAD sensitivity select entity that satellite integrations create, so it + # could not be tuned or tested without buying hardware. + ./home-assistant-vad-silence-api.patch + ]; + + # audio_enhancer.py imports pysilero_vad after the Silero patch. The manifest + # change in that patch does not pull it in, because nixpkgs resolves component + # requirements before patches are applied -- and it has to be a package input + # rather than services.home-assistant.extraPackages, because the test suite + # runs during the build and imports it too. + # + # passthru.python3Packages is the matching interpreter's set; pkgs.python3Packages + # would be a different Python and the import would fail at runtime. + propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ + home-assistant.python3Packages.pysilero-vad + ]; +}) diff --git a/nixos/nixos/llama-checkpoint-dedup.patch b/packages/llama-checkpoint-dedup.patch similarity index 100% rename from nixos/nixos/llama-checkpoint-dedup.patch rename to packages/llama-checkpoint-dedup.patch diff --git a/nixos/nixos/ollama-message-delimiters.patch b/packages/ollama-message-delimiters.patch similarity index 100% rename from nixos/nixos/ollama-message-delimiters.patch rename to packages/ollama-message-delimiters.patch diff --git a/packages/ollama.nix b/packages/ollama.nix new file mode 100644 index 0000000..4c9ab8f --- /dev/null +++ b/packages/ollama.nix @@ -0,0 +1,33 @@ +# ollama, patched so that llama-server can cache the part of a prompt that does +# not change between requests. Worth about 1.7 s per voice command; the whole +# story is in ./prefix-cache-findings.md. +# +# Takes the base package rather than pkgs, so a caller can pass ollama-cuda, +# ollama-vulkan, or plain ollama on a Mac. +ollama: + +ollama.overrideAttrs (old: { + # llama-server can place a context checkpoint exactly where a message begins, + # but only if it is told where that is: it scans the prompt for the delimiter + # strings in the request's "message_delimiters" field. llama-server's own + # chat-completions path fills that in from the chat template, but ollama + # renders qwen's template itself in Go and posts a flat string to /completion, + # leaving the field empty. So llama-server sees an opaque prompt and falls + # back to checkpointing near the end of it, which is a position that moves + # with every request and can therefore never be reused. + # + # The patch has renderers report their delimiters and threads them through. + # It also replays the empty think block in history, without which a follow-up + # turn does not match the tokens the model actually produced. + patches = (old.patches or [ ]) ++ [ ./ollama-message-delimiters.patch ]; + + # This one is against llama.cpp, not ollama. nixpkgs pre-stages llama.cpp into + # $TMPDIR/llama-cpp-src at the end of postPatch so the CMake FetchContent step + # does not reach the network, which is the only place to get at it. + # + # Note nixpkgs pins b10091 while ollama 0.32.13 asks for b10380: read b10091 + # when reasoning about llama-server's behaviour here. + postPatch = (old.postPatch or "") + '' + patch -d "$TMPDIR/llama-cpp-src" -p1 < ${./llama-checkpoint-dedup.patch} + ''; +}) diff --git a/packages/overlay.nix b/packages/overlay.nix new file mode 100644 index 0000000..74f169f --- /dev/null +++ b/packages/overlay.nix @@ -0,0 +1,13 @@ +# Patched packages, usable from a NixOS config, a Home Manager config, or an ad +# hoc `nix run` on another machine. Nothing here needs a GPU to build. +final: prev: + +{ + ollama-patched = import ./ollama.nix prev.ollama; +} +// prev.lib.optionalAttrs (prev ? ollama-cuda) { + ollama-cuda-patched = import ./ollama.nix prev.ollama-cuda; +} +// prev.lib.optionalAttrs (prev ? home-assistant) { + home-assistant-patched = import ./home-assistant.nix prev.home-assistant; +} diff --git a/nixos/nixos/prefix-cache-findings.md b/packages/prefix-cache-findings.md similarity index 100% rename from nixos/nixos/prefix-cache-findings.md rename to packages/prefix-cache-findings.md diff --git a/nixos/nixos/prefix-cache-statecheck.py b/packages/prefix-cache-statecheck.py similarity index 100% rename from nixos/nixos/prefix-cache-statecheck.py rename to packages/prefix-cache-statecheck.py From 9d2631ad04a276cb12caa698e68506775f422b9f Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 02:42:22 +0000 Subject: [PATCH 70/98] Add a development Home Assistant that runs in the agent VM Every voice experiment so far has needed a rebuild of the real machine, which meant a person sitting at a desk watching builds. Home Assistant is what we iterate on and it needs no GPU, so it can run here instead: same package as the host, pointed at the host's ollama over the bridge, with its own state in ~/ha-dev. Three scripts rebuild it from nothing -- run, onboard, setup -- so the dev instance cannot drift into a state nobody can reproduce. It answers "is the bed light on?" in ~450 ms against 433-458 ms on the real machine. Pinned to nixpkgs-stable to match the host, because the Silero patch does not apply to the 2026.8.2 in unstable. Co-Authored-By: Claude Opus 5 --- dev/README.md | 47 +++++++++++++ dev/configuration.yaml | 23 +++++++ dev/ha-ask.py | 52 ++++++++++++++ dev/ha-onboard.py | 65 ++++++++++++++++++ dev/ha-setup.py | 149 +++++++++++++++++++++++++++++++++++++++++ dev/run-ha.sh | 29 ++++++++ flake.nix | 87 +++++++++++++++++++++--- 7 files changed, 444 insertions(+), 8 deletions(-) create mode 100644 dev/README.md create mode 100644 dev/configuration.yaml create mode 100755 dev/ha-ask.py create mode 100755 dev/ha-onboard.py create mode 100644 dev/ha-setup.py create mode 100755 dev/run-ha.sh diff --git a/dev/README.md b/dev/README.md new file mode 100644 index 0000000..5c1cd6c --- /dev/null +++ b/dev/README.md @@ -0,0 +1,47 @@ +# Development stack + +A second Home Assistant, running in the agent VM, so voice work does not need a +rebuild of the real machine for every experiment. + + dev/run-ha.sh # start it (builds .#hass-dev if needed) + python3 dev/ha-onboard.py # once, on an empty config dir + python3 dev/ha-setup.py # configure it; idempotent + python3 dev/ha-ask.py "is the bed light on?" + dev/run-ha.sh stop + +State lives in `~/ha-dev`, the token in `scratch/hadev/token.txt`. Delete the +directory to start over; the three scripts rebuild everything. + +## What runs where + +Only two things need the GPU, and they stay on the NixOS host: + +| | where | why | +|---|---|---| +| ollama | host | CUDA, and the model is 28 GiB | +| Kokoro TTS | host | CUDA | +| Home Assistant | **here** | no GPU; this is what we iterate on | +| faster-whisper, openWakeWord | either | CPU; ~90 ms to transcribe | + +The dev instance talks to the host's ollama at `192.168.122.1:11434`, which is +open on `virbr0`. The host's speech services are bound to loopback and are not +reachable from here; run local ones if a test needs them. + +## Things that cost an hour to find + +- The package is pinned to **nixpkgs-stable**, matching the host. The Silero + patch does not apply to Home Assistant 2026.8.2 in unstable. A dev instance on + a different version teaches you nothing transferable. +- `extraComponents` does not change the derivation. The NixOS module passes the + component dependencies through `environment.PYTHONPATH = package.pythonPath`, + which is why `.#hass-dev` is a wrapper that exports it. +- The module also always adds `defaultIntegrations`, including **frontend**. + Without it `hass_frontend` is missing, frontend setup fails, and Home + Assistant silently drops into **recovery mode** -- which ignores + `configuration.yaml`, so nothing loads and the failure looks like anything but + a missing frontend. +- Do not use `default_config:`; it pulls dhcp, go2rtc, logbook, my, ssdp and + stream, and taking everything it would have set up down with it when they are + missing. +- `pkill -f hass` matches the shell running it. Use the bracketed pattern in + `run-ha.sh`. diff --git a/dev/configuration.yaml b/dev/configuration.yaml new file mode 100644 index 0000000..8319292 --- /dev/null +++ b/dev/configuration.yaml @@ -0,0 +1,23 @@ +# Development Home Assistant, run in the agent VM. Copied to the config dir by +# dev/run-ha.sh. +# +# Deliberately NOT default_config: that pulls dhcp, go2rtc, logbook, my, ssdp and +# stream, which are not in the package's extraComponents, and when it fails to +# set up everything it would have pulled in fails with it -- including +# conversation and assist_pipeline. +homeassistant: + name: Dev + time_zone: America/New_York + unit_system: us_customary + country: US + +http: +api: +websocket_api: +config: + +conversation: +assist_pipeline: + +# Fake lights and sensors, so the assistant has something to control. +demo: diff --git a/dev/ha-ask.py b/dev/ha-ask.py new file mode 100755 index 0000000..1ebf183 --- /dev/null +++ b/dev/ha-ask.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Ask the development assistant something and time the intent stage. + + python3 dev/ha-ask.py "is the bed light on?" +""" +import asyncio, json, os, sys, time +import websockets + +HA = os.environ.get("HA_DEV_URL", "http://127.0.0.1:8123") +HERE = os.path.dirname(os.path.abspath(__file__)) +TOKEN = open(os.path.join(HERE, "..", "scratch", "hadev", "token.txt")).read().strip() +QUESTIONS = sys.argv[1:] or ["Is the bed light on?", "Turn on the kitchen lights.", + "What time is it?"] + + +async def main(): + ws = await websockets.connect(HA.replace("http", "ws") + "/api/websocket", + max_size=None) + await ws.recv() + await ws.send(json.dumps({"type": "auth", "access_token": TOKEN})) + assert json.loads(await ws.recv())["type"] == "auth_ok" + await ws.send(json.dumps({"id": 1, "type": "assist_pipeline/pipeline/list"})) + while True: + m = json.loads(await ws.recv()) + if m.get("id") == 1: + pipe = next(p["id"] for p in m["result"]["pipelines"] if p["name"] == "Dev") + break + i = 100 + for q in QUESTIONS: + i += 1 + await ws.send(json.dumps({"id": i, "type": "assist_pipeline/run", + "start_stage": "intent", "end_stage": "intent", "input": {"text": q}, + "pipeline": pipe, "timeout": 120})) + marks, reply = {}, "" + while True: + m = json.loads(await ws.recv()) + if m.get("id") != i: + continue + if m.get("type") == "result" and not m.get("success"): + print("ERR", m.get("error")); return + if m.get("type") != "event": + continue + e = m["event"]; marks[e["type"]] = time.monotonic() + if e["type"] == "intent-end": + reply = e["data"]["intent_output"]["response"]["speech"]["plain"]["speech"] + if e["type"] in ("run-end", "error"): + break + dur = (marks.get("intent-end", 0) - marks.get("intent-start", 0)) * 1000 + print(f"{dur:7.0f} ms {q}\n -> {reply}") + await asyncio.sleep(1) + +asyncio.run(main()) diff --git a/dev/ha-onboard.py b/dev/ha-onboard.py new file mode 100755 index 0000000..60a75d2 --- /dev/null +++ b/dev/ha-onboard.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Onboard a fresh development Home Assistant and save a long-lived token. + +Run once after dev/run-ha.sh on an empty config dir. Idempotent enough: if +onboarding is already done it just reports that. +""" +import asyncio, json, os, sys, urllib.error, urllib.request + +HA = os.environ.get("HA_DEV_URL", "http://127.0.0.1:8123") +HERE = os.path.dirname(os.path.abspath(__file__)) +TOKEN_FILE = os.path.join(HERE, "..", "scratch", "hadev", "token.txt") + + +def post(path, body, token=None): + headers = {"Content-Type": "application/json"} + if token: + headers["Authorization"] = "Bearer " + token + req = urllib.request.Request(HA + path, json.dumps(body).encode(), headers) + with urllib.request.urlopen(req, timeout=60) as r: + return json.load(r) + + +steps = json.load(urllib.request.urlopen(HA + "/api/onboarding", timeout=30)) +if all(s["done"] for s in steps): + sys.exit("already onboarded; delete the config dir to start over") + +auth = post("/api/onboarding/users", + {"client_id": HA + "/", "name": "Agent", "username": "agent", + "password": "devdevdev", "language": "en"}) +import urllib.parse +req = urllib.request.Request( + HA + "/auth/token", + urllib.parse.urlencode({"grant_type": "authorization_code", + "code": auth["auth_code"], + "client_id": HA + "/"}).encode()) +short = json.load(urllib.request.urlopen(req, timeout=30))["access_token"] + +for step in ("core_config", "analytics"): + try: + post("/api/onboarding/" + step, {"client_id": HA + "/"}, short) + except urllib.error.HTTPError as e: + print(f" {step}: {e.code} (continuing)") + + +async def long_lived(): + import websockets + async with websockets.connect(HA.replace("http", "ws") + "/api/websocket") as ws: + await ws.recv() + await ws.send(json.dumps({"type": "auth", "access_token": short})) + assert json.loads(await ws.recv())["type"] == "auth_ok" + await ws.send(json.dumps({"id": 1, "type": "auth/long_lived_access_token", + "client_name": "agent", "lifespan": 3650})) + while True: + m = json.loads(await ws.recv()) + if m.get("id") == 1: + if not m.get("success"): + raise SystemExit(m) + return m["result"] + + +tok = asyncio.run(long_lived()) +os.makedirs(os.path.dirname(TOKEN_FILE), exist_ok=True) +with open(TOKEN_FILE, "w") as f: + f.write(tok) +print(f"onboarded; long-lived token written to {os.path.relpath(TOKEN_FILE, os.getcwd())}") diff --git a/dev/ha-setup.py b/dev/ha-setup.py new file mode 100644 index 0000000..777a88b --- /dev/null +++ b/dev/ha-setup.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Configure a fresh development Home Assistant to mirror the real one. + +Reproducible so the dev instance never drifts into a state nobody can rebuild. +Idempotent: re-running reuses whatever already exists. + + nix run .#hass-dev -- -c ~/ha-dev # start it + python3 dev/ha-setup.py # configure it + +Talks to 127.0.0.1:8123 (the dev instance) and points its conversation agent at +the ollama on the NixOS host, which is the one thing here that needs a GPU. +""" +import asyncio, json, os, sys, urllib.error, urllib.request + +HA = os.environ.get("HA_DEV_URL", "http://127.0.0.1:8123") +OLLAMA = os.environ.get("HA_DEV_OLLAMA", "http://192.168.122.1:11434") +MODEL = os.environ.get("HA_DEV_MODEL", "qwen3.8:27b-mtp-q8_0") +TOKEN = open(os.path.join(os.path.dirname(__file__) or ".", + "../scratch/hadev/token.txt")).read().strip() + +# Mirrors the real machine's prompt. Everything after <|fim_pad|> is re-read on +# every request; everything before it is cached. See packages/prefix-cache-findings.md. +PROMPT = """You are a voice assistant for Home Assistant. +Answer in plain text. Keep it simple and to the point: one or two short sentences unless asked for detail. + +Your training data is out of date and your memory of current facts is wrong. +The current time and the state of the devices listed at the very end of this prompt are live; trust them over anything you remember, and answer from them directly without calling a tool. +Call GetLiveContext only for the state of something not listed there. +Never say you lack access to current information. Only say you do not know if a tool returned nothing useful. +<|fim_pad|> +Live state, correct as of now: +Current time: {{ now().strftime('%A, %B %-d, %Y at %-I:%M %p %Z') }} +Bed Light: {{ states('light.bed_light') }} +Ceiling Lights: {{ states('light.ceiling_lights') }} +Kitchen Lights: {{ states('light.kitchen_lights') }}""" + +EXPOSE = ["light.bed_light", "light.ceiling_lights", "light.kitchen_lights", + "weather.forecast_home"] + + +def rest(path, body=None, method=None): + req = urllib.request.Request( + HA + path, + data=json.dumps(body).encode() if body is not None else None, + headers={"Content-Type": "application/json", + "Authorization": "Bearer " + TOKEN}, + method=method) + with urllib.request.urlopen(req, timeout=120) as r: + return json.load(r) + + +def ollama_entry(): + """Create the ollama config entry, or return the existing one.""" + for e in rest("/api/config/config_entries/entry"): + if e["domain"] == "ollama": + print(f" ollama entry exists: {e['entry_id']}") + return e["entry_id"] + flow = rest("/api/config/config_entries/flow", + {"handler": "ollama", "show_advanced_options": True}) + res = rest(f"/api/config/config_entries/flow/{flow['flow_id']}", {"url": OLLAMA}) + if res.get("type") == "create_entry": + entry = res["result"]["entry_id"] + print(f" created ollama entry {entry} -> {OLLAMA}") + return entry + raise SystemExit(f"unexpected flow result: {json.dumps(res)[:400]}") + + +def conversation_subentry(entry_id, existing_id=None): + """Create the conversation agent, or reconfigure the one that exists.""" + body = {"handler": [entry_id, "conversation"], "show_advanced_options": True} + if existing_id: + body["subentry_id"] = existing_id # reconfigure rather than add + flow = rest("/api/config/config_entries/subentries/flow", body) + data = {"model": MODEL, "prompt": PROMPT, "llm_hass_api": ["assist"], + "num_ctx": 8192, "max_history": 20, "keep_alive": -1, "think": False} + res = rest(f"/api/config/config_entries/subentries/flow/{flow['flow_id']}", data) + print(f" conversation agent: {res.get('type')} {res.get('reason', '')} model={MODEL}") + + +async def ws_setup(): + import websockets + async with websockets.connect(HA.replace("http", "ws") + "/api/websocket", + max_size=None) as ws: + await ws.recv() + await ws.send(json.dumps({"type": "auth", "access_token": TOKEN})) + assert json.loads(await ws.recv())["type"] == "auth_ok" + n = [0] + + async def call(**kw): + n[0] += 1 + await ws.send(json.dumps({"id": n[0], **kw})) + while True: + m = json.loads(await ws.recv()) + if m.get("id") == n[0] and m.get("type") == "result": + return m + + r = await call(type="homeassistant/expose_entity", + assistants=["conversation"], entity_ids=EXPOSE, should_expose=True) + print(f" exposed {len(EXPOSE)} entities: {r.get('success')}") + + pipelines = await call(type="assist_pipeline/pipeline/list") + if not pipelines.get("success"): + print(f" pipeline/list failed: {pipelines.get('error')}") + return + agents = await call(type="conversation/agent/list") + agent = next((a["id"] for a in agents["result"]["agents"] + if a["id"] != "conversation.home_assistant"), None) + print(f" conversation agents: {[a['id'] for a in agents['result']['agents']]}") + if agent is None: + print(" !! no LLM agent yet; re-run after the entry finishes setting up") + return + existing = next((p for p in pipelines["result"]["pipelines"] + if p["name"] == "Dev"), None) + spec = {"name": "Dev", "language": "en", "conversation_engine": agent, + "conversation_language": "en", "stt_engine": None, "stt_language": None, + "tts_engine": None, "tts_language": None, "tts_voice": None, + "wake_word_entity": None, "wake_word_id": None, + "prefer_local_intents": True} + if existing: + r = await call(type="assist_pipeline/pipeline/update", + pipeline_id=existing["id"], **spec) + print(f" updated pipeline {existing['id']}: {r.get('success')}") + else: + r = await call(type="assist_pipeline/pipeline/create", **spec) + print(f" created pipeline: {r['result']['id'] if r.get('success') else r}") + + +async def existing_subentry(entry_id): + import websockets + async with websockets.connect(HA.replace("http", "ws") + "/api/websocket", + max_size=None) as ws: + await ws.recv() + await ws.send(json.dumps({"type": "auth", "access_token": TOKEN})) + assert json.loads(await ws.recv())["type"] == "auth_ok" + await ws.send(json.dumps({"id": 1, "type": "config_entries/subentries/list", + "entry_id": entry_id})) + while True: + m = json.loads(await ws.recv()) + if m.get("id") == 1 and m.get("type") == "result": + subs = [s for s in m.get("result", []) + if s["subentry_type"] == "conversation"] + return subs[0]["subentry_id"] if subs else None + + +print(f"configuring dev Home Assistant at {HA}") +entry = ollama_entry() +conversation_subentry(entry, asyncio.run(existing_subentry(entry))) +asyncio.run(ws_setup()) +print("done") diff --git a/dev/run-ha.sh b/dev/run-ha.sh new file mode 100755 index 0000000..8357f97 --- /dev/null +++ b/dev/run-ha.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Start the development Home Assistant in the agent VM. +# +# dev/run-ha.sh # start it in the background +# dev/run-ha.sh stop # stop it +# +# The real machine is untouched: this instance keeps its own state in ~/ha-dev +# and only borrows the host's ollama, which is the one part that needs a GPU. +set -euo pipefail +cd "$(dirname "$0")/.." +DIR="${HA_DEV_DIR:-$HOME/ha-dev}" + +# The pattern is bracketed so it cannot match this script's own command line. +pid() { pgrep -f "bin/[.]hass-wrapped" | head -1; } + +if [ "${1:-start}" = stop ]; then + p=$(pid || true); [ -n "$p" ] && kill "$p" && echo "stopped $p" || echo "not running" + exit 0 +fi + +p=$(pid || true) +if [ -n "$p" ]; then echo "already running as $p"; exit 0; fi + +mkdir -p "$DIR" +cp dev/configuration.yaml "$DIR/configuration.yaml" +wrapper=$(nix build --no-link --print-out-paths .#hass-dev) +setsid "$wrapper/bin/hass-dev" -c "$DIR" > "$DIR/hass.log" 2>&1 < /dev/null & +sleep 3 +echo "started $(pid) -- log: $DIR/hass.log" diff --git a/flake.nix b/flake.nix index 065b4ee..db05e1a 100644 --- a/flake.nix +++ b/flake.nix @@ -47,21 +47,92 @@ packages = let + # Each patched package comes from the same nixpkgs the NixOS host takes + # it from, because the patches are version-specific: the Silero one + # does not apply to Home Assistant 2026.8.2 in unstable, only to the + # 2026.5.4 in stable. A development instance on a different version + # would not tell us anything transferable. + with-overlay = + input: system: + import input { + inherit system; + config.allowUnfree = true; + overlays = [ (import ./packages/overlay.nix) ]; + }; patched = system: let - pkgs = import nixpkgs { - inherit system; - config.allowUnfree = true; - overlays = [ (import ./packages/overlay.nix) ]; - }; + unstable = with-overlay nixpkgs system; # ollama: host takes it from here + stable = with-overlay nixpkgs-stable system; # home-assistant: ditto in { - inherit (pkgs) ollama-patched; + inherit (unstable) ollama-patched; } - // nixpkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux { - inherit (pkgs) ollama-cuda-patched home-assistant-patched; + // nixpkgs.lib.optionalAttrs (nixpkgs.lib.hasSuffix "linux" system) { + inherit (unstable) ollama-cuda-patched; + inherit (stable) home-assistant-patched; + + # A Home Assistant that can run standalone, for the development + # instance in the agent VM. The NixOS module normally derives + # extraComponents from the configuration; running `hass` straight + # out of the package gets only the defaults, and the ollama config + # flow then fails with "Invalid handler specified". + # + # Same components as the real machine, so the two behave alike. + # Runnable wrapper: `nix run .#hass-dev -- -c ~/ha-dev`. The + # component dependencies live in passthru.pythonPath rather than in + # the package, which is why the NixOS module sets + # environment.PYTHONPATH from it; running `hass` directly without + # that gets "Invalid handler specified" for ollama. + hass-dev = + let + ha = patched-for system; + in + stable.writeShellScriptBin "hass-dev" '' + export PYTHONPATH=${ha.pythonPath} + exec ${ha}/bin/hass "$@" + ''; + + home-assistant-dev = patched-for system; }; + + patched-for = + system: + let + stable = with-overlay nixpkgs-stable system; + in + import ./packages/home-assistant.nix ( + stable.home-assistant.override { + extraComponents = [ + # The NixOS module always adds these and running `hass` outside + # the module does not. Without "frontend" the hass_frontend + # module is missing, frontend setup fails, and Home Assistant + # drops into recovery mode -- which ignores configuration.yaml + # entirely, so nothing below loads and the failure looks + # unrelated to the frontend. + "application_credentials" + "frontend" + "hardware" + "logger" + "network" + "system_health" + "automation" + "person" + "scene" + "script" + "zone" + + # Same as the real machine. + "assist_pipeline" + "demo" + "esphome" + "met" + "ollama" + "radio_browser" + "wyoming" + ]; + } + ); in { x86_64-linux = { From 5923b7a4ab0290f787093d0cf5460058f29eb33c Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 02:50:47 +0000 Subject: [PATCH 71/98] Add a portable ollama probe The measurement scripts lived in scratch, so they were not available on any other machine -- including the Mac we want to compare against. This one takes OLLAMA_URL and a model name and reports the three numbers that matter: fresh-conversation prefill, follow-up prefill, and generation. Verified against the NixOS host: 587 ms fresh, 109 ms follow-up, 85.6 tok/s on qwen3.8:27b-mtp-q8_0. Co-Authored-By: Claude Opus 5 --- dev/ollama-probe.py | 65 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100755 dev/ollama-probe.py diff --git a/dev/ollama-probe.py b/dev/ollama-probe.py new file mode 100755 index 0000000..00249cb --- /dev/null +++ b/dev/ollama-probe.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Measure prompt-prefix caching and generation speed on any ollama. + + OLLAMA_URL=http://127.0.0.1:11434 python3 dev/ollama-probe.py qwen3.8:27b-mtp-q8_0 + +Reports three things: + fresh conversation -- a new conversation sharing only the cached prefix, which + is what every voice command looks like + follow-up turn -- appending to the conversation already in the slot + generation -- tokens per second + +Read prompt_eval_duration only. prompt_eval_count always reports the whole +prompt, reused or not. + +Nothing else may talk to this ollama while it runs: one other request replaces +the slot contents and the next measurement then shares nothing with it. +""" +import json, os, statistics, sys, urllib.request + +URL = os.environ.get("OLLAMA_URL", "http://127.0.0.1:11434") + "/api/chat" +MODEL = sys.argv[1] if len(sys.argv) > 1 else "qwen3.8:27b-mtp-q8_0" + +# Shaped like Home Assistant's: a preamble plus a long entity table. The size and +# the fact that it is byte-identical across conversations are what matter. +SYSTEM = ( + "You are a voice assistant for a home. Answer in one or two short sentences, " + "in plain spoken language, with no markdown and no lists.\n\n" + "An overview of the areas and the devices in this smart home:\n" +) + "\n".join( + f"- device_{i}: Device {i} ('Device {i}', {'on' if i % 3 else 'off'})" + for i in range(300)) + +QUESTIONS = ["Turn on device 1.", "What state is device 2 in?", "Is device 3 on?", + "Turn off device 4.", "Check device 5.", "Toggle device 6."] + + +def chat(messages, num_predict=40): + body = json.dumps({"model": MODEL, "messages": messages, "stream": False, + "think": False, "keep_alive": -1, + "options": {"temperature": 0, "num_predict": num_predict}}).encode() + req = urllib.request.Request(URL, body, {"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=900) as r: + return json.load(r) + + +ms = lambda d, k: d[k] / 1e6 +print(f"model {MODEL} at {URL}") +d = chat([{"role": "system", "content": SYSTEM}, {"role": "user", "content": "Hello."}]) +print(f"cold {ms(d, 'prompt_eval_duration'):8.1f} ms / {d['prompt_eval_count']} tok" + f" ({d['prompt_eval_count'] / (d['prompt_eval_duration'] / 1e9):.0f} tok/s prefill)") + +fresh, follow = [], [] +for q in QUESTIONS: + msgs = [{"role": "system", "content": SYSTEM}, {"role": "user", "content": q}] + d = chat(msgs) + fresh.append(ms(d, "prompt_eval_duration")) + d2 = chat(msgs + [d["message"], {"role": "user", "content": "And the one after?"}]) + follow.append(ms(d2, "prompt_eval_duration")) + +g = chat([{"role": "system", "content": SYSTEM}, + {"role": "user", "content": "Count from 1 to 60, comma separated."}], 200) +print(f"\nfresh conversation : {statistics.median(fresh):8.1f} ms") +print(f"follow-up turn : {statistics.median(follow):8.1f} ms") +print(f"generation : {g['eval_count'] / (g['eval_duration'] / 1e9):8.1f} tok/s" + f" ({g['eval_count']} tok)") From 759aa4d9a84c5b4050bdebfce63b0402d3bc85d6 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 03:36:54 +0000 Subject: [PATCH 72/98] Make the probe measure the cached path, and generate enough to time Two flaws, both of which would have been read as hardware differences. The prompt carried no cache boundary marker, so on a server configured to look for one the delimiter matched nothing and the probe measured the fallback path: 587 ms fresh on the NixOS host against 97 ms once the marker is present. Two machines configured with different markers were being compared to each other. And the generation prompt produced 16-20 tokens, so tokens/sec was mostly startup noise -- 43.6 tok/s on one run against 95.0 tok/s when actually generating 300. It now asks for a longer sequence and says so loudly if the model stops early anyway. Host baseline, qwen3.8:27b-mtp-q8_0: 97 ms fresh, 98 ms follow-up, 95 tok/s. Co-Authored-By: Claude Opus 5 --- dev/ollama-probe.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/dev/ollama-probe.py b/dev/ollama-probe.py index 00249cb..e455b4a 100755 --- a/dev/ollama-probe.py +++ b/dev/ollama-probe.py @@ -22,13 +22,20 @@ # Shaped like Home Assistant's: a preamble plus a long entity table. The size and # the fact that it is byte-identical across conversations are what matter. +# +# It ends with the cache boundary marker, because that is what the server is +# told to look for. Without it the delimiter matches nothing, the server falls +# back to checkpointing near the end of the prompt, and the probe measures the +# degraded path -- which looks like a hardware difference if the two machines +# are configured differently. +MARKER = os.environ.get("OLLAMA_CACHE_MARKER", "<|fim_pad|>") SYSTEM = ( "You are a voice assistant for a home. Answer in one or two short sentences, " "in plain spoken language, with no markdown and no lists.\n\n" "An overview of the areas and the devices in this smart home:\n" ) + "\n".join( f"- device_{i}: Device {i} ('Device {i}', {'on' if i % 3 else 'off'})" - for i in range(300)) + for i in range(300)) + f"\n{MARKER}\nCurrent time: Monday, 9:00 PM" QUESTIONS = ["Turn on device 1.", "What state is device 2 in?", "Is device 3 on?", "Turn off device 4.", "Check device 5.", "Toggle device 6."] @@ -57,8 +64,13 @@ def chat(messages, num_predict=40): d2 = chat(msgs + [d["message"], {"role": "user", "content": "And the one after?"}]) follow.append(ms(d2, "prompt_eval_duration")) +# Long enough to measure: short replies make tokens/sec mostly startup noise. +# Warn rather than quietly report a number derived from a handful of tokens. g = chat([{"role": "system", "content": SYSTEM}, - {"role": "user", "content": "Count from 1 to 60, comma separated."}], 200) + {"role": "user", "content": "Write the numbers from 1 to 120 separated by " + "commas. Output nothing else."}], 300) +if g["eval_count"] < 60: + print(f"\n!! only {g['eval_count']} tokens generated; tokens/sec below is unreliable") print(f"\nfresh conversation : {statistics.median(fresh):8.1f} ms") print(f"follow-up turn : {statistics.median(follow):8.1f} ms") print(f"generation : {g['eval_count'] / (g['eval_duration'] / 1e9):8.1f} tok/s" From ceda1a59cd2401ff5f4e5d4453045388f0434d9a Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 03:46:03 +0000 Subject: [PATCH 73/98] Measure generation on prose: MTP makes tokens/sec content-dependent Same model and machine at temperature 0: 97.3 tok/s counting from 1 to 120, 50.5 tok/s writing prose. Speculative decoding accepts more draft tokens when the continuation is predictable, so counting is near a best case. Which means the 85-95 tok/s quoted throughout the findings came from a counting prompt and overstates a real assistant reply by roughly two times. The probe now uses prose, which is both nearer the workload and comparable across machines. It also avoids a second trap: the counting prompt generated 300 tokens on CUDA and 17 on Metal, because the backends diverge numerically and one gave up early, leaving tokens/sec describing startup overhead. Co-Authored-By: Claude Opus 5 --- dev/ollama-probe.py | 9 +++++++-- packages/prefix-cache-findings.md | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/dev/ollama-probe.py b/dev/ollama-probe.py index e455b4a..a60d7db 100755 --- a/dev/ollama-probe.py +++ b/dev/ollama-probe.py @@ -66,9 +66,14 @@ def chat(messages, num_predict=40): # Long enough to measure: short replies make tokens/sec mostly startup noise. # Warn rather than quietly report a number derived from a handful of tokens. +# Open-ended prose rather than a countable task: the same counting prompt at +# temperature 0 produced 300 tokens on CUDA and 17 on Metal, because the +# backends diverge numerically and one model gave up early. Prose keeps +# generating on both. g = chat([{"role": "system", "content": SYSTEM}, - {"role": "user", "content": "Write the numbers from 1 to 120 separated by " - "commas. Output nothing else."}], 300) + {"role": "user", "content": "Describe a kitchen in detail: the counters, " + "the light, the smells, the sounds. Write " + "several paragraphs."}], 300) if g["eval_count"] < 60: print(f"\n!! only {g['eval_count']} tokens generated; tokens/sec below is unreliable") print(f"\nfresh conversation : {statistics.median(fresh):8.1f} ms") diff --git a/packages/prefix-cache-findings.md b/packages/prefix-cache-findings.md index ef34f92..ec4963f 100644 --- a/packages/prefix-cache-findings.md +++ b/packages/prefix-cache-findings.md @@ -984,3 +984,28 @@ sit at 300-800 ms, which spans 0.25 to 0.7 here. 0.5 is a reasonable middle: No silence threshold can distinguish "still thinking" from "finished" -- they are acoustically identical, and only the words differ. That is what semantic endpointing is for. + + +## Generation speed is content-dependent, because of MTP + +Same model, same machine, temperature 0, only the prompt differing: + +| prompt | tok/s | +|---|---| +| count from 1 to 120 | 97.3 | +| a short assistant reply | 86.8 (14 tok, noisy) | +| several paragraphs of prose | 50.5 | + +Speculative decoding accepts more draft tokens when the continuation is +predictable, so counting is close to a best case and prose closer to a worst +one. **The 85-95 tok/s quoted throughout this document came from a counting +prompt** and overstates what an assistant reply achieves. + +Two consequences for measurement. Use prose to compare hardware, since it is +nearer the real workload and not dominated by how well MTP happens to guess. And +never compare a tokens/sec figure against one taken with a different prompt. + +A related trap: the counting prompt produced 300 tokens on CUDA and 17 on Metal +at temperature 0. Backends diverge numerically, so a task one model finishes the +other can abandon, and the tokens/sec then describes startup overhead rather +than generation. `dev/ollama-probe.py` warns when fewer than 60 tokens come out. From 76f5b7b83f8af1416d51074639f7fbe0235c1a8d Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 03:48:16 +0000 Subject: [PATCH 74/98] Record the RTX 5880 against M5 Pro comparison Same model, patched ollama, probe and settings on both. Prefix caching works identically on Metal -- a fresh conversation costs 453 ms where re-reading the prompt would cost 48 s -- so both patches carry over, which was the thing worth checking. The gap is prefill throughput: 131 tok/s against 1957. Every Mac figure follows from it, including the asymmetry where the Mac's follow-up is slower than its fresh conversation while the RTX has them equal, since the cost is proportional to tokens re-read. Generation on prose is 44-50 against 14.1 tok/s. I expected prose to narrow the gap, since MTP flatters the counting prompt; it did not, because both machines lose about half their rate on prose. Also labels the probe's first line honestly -- it is only a cold prefill when the slot is empty, and reports meaningless numbers otherwise. Co-Authored-By: Claude Opus 5 --- dev/ollama-probe.py | 8 +++++-- packages/prefix-cache-findings.md | 36 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/dev/ollama-probe.py b/dev/ollama-probe.py index a60d7db..c71ae12 100755 --- a/dev/ollama-probe.py +++ b/dev/ollama-probe.py @@ -53,8 +53,12 @@ def chat(messages, num_predict=40): ms = lambda d, k: d[k] / 1e6 print(f"model {MODEL} at {URL}") d = chat([{"role": "system", "content": SYSTEM}, {"role": "user", "content": "Hello."}]) -print(f"cold {ms(d, 'prompt_eval_duration'):8.1f} ms / {d['prompt_eval_count']} tok" - f" ({d['prompt_eval_count'] / (d['prompt_eval_duration'] / 1e9):.0f} tok/s prefill)") +rate = d["prompt_eval_count"] / (d["prompt_eval_duration"] / 1e9) +print(f"first {ms(d, 'prompt_eval_duration'):8.1f} ms / {d['prompt_eval_count']} tok" + f" ({rate:.0f} tok/s)") +if rate > 3000: + print(" ^ the slot was already warm, so this is NOT a cold prefill rate;" + " restart the server to measure that") fresh, follow = [], [] for q in QUESTIONS: diff --git a/packages/prefix-cache-findings.md b/packages/prefix-cache-findings.md index ec4963f..d61710e 100644 --- a/packages/prefix-cache-findings.md +++ b/packages/prefix-cache-findings.md @@ -1009,3 +1009,39 @@ A related trap: the counting prompt produced 300 tokens on CUDA and 17 on Metal at temperature 0. Backends diverge numerically, so a task one model finishes the other can abandon, and the tokens/sec then describes startup overhead rather than generation. `dev/ollama-probe.py` warns when fewer than 60 tokens come out. + + +## RTX 5880 Ada against an M5 Pro MacBook + +Same model (`qwen3.8:27b-mtp-q8_0`), same patched ollama, same probe, same +settings: 32768 context, `<|fim_pad|>` boundary, checkpoint spacing 0. + +| | RTX 5880 Ada | M5 Pro (64 GiB) | ratio | +|---|---|---|---| +| fresh conversation | 97 ms | 453 ms | 4.7x | +| follow-up turn | 99 ms | 686 ms | 7x | +| generation, prose | 44-50 tok/s | 14.1 tok/s | ~3.2x | +| cold prefill | 1957 tok/s | 131 tok/s | 15x | + +**Prefix caching works identically on Metal.** Both patches compile and behave: +a fresh conversation costs 453 ms where re-reading the prompt would cost 48 s. +That was the thing worth checking, and it carries. + +**The gap is prefill throughput.** Every Mac figure follows from 131 tok/s +against 1957. Fresh re-reads ~59 tokens after the boundary, the follow-up ~90 +(the extra being the previous reply), and at 131 tok/s those are 450 and 690 ms. +It also explains the one asymmetry: the Mac's follow-up is *slower* than its +fresh conversation, where the two are equal on the RTX, because the cost is +proportional to tokens re-read and the Mac's constant is 15x larger. + +For the assistant that compounds to roughly **0.3 s of model time on the RTX +against ~1.3 s on the Mac** for a typical state question. + +I expected the generation gap to narrow on prose, since MTP flatters the +counting prompt. It did not: both machines lose about half their tokens/sec on +prose (97 -> 50 and 29 -> 14), so the ratio holds near 3.2x either way. + +Measurement note: the probe's first line is only a genuine cold prefill when the +slot is empty. Once a run has populated it, that line reports 14000+ tok/s and +means nothing. The 131 and 1957 tok/s above come from runs where the model had +just loaded. From 076e6647bdee68f3855a9e9c40eadccb43d3ff88 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 03:56:44 +0000 Subject: [PATCH 75/98] Establish that semantic endpointing is feasible here Smart Turn v3 (pipecat-ai, BSD-2-Clause) is a Whisper Tiny encoder plus a linear head, 8M parameters, 8 MB of int8 ONNX, reading the waveform rather than a transcript so it needs no extra speech-to-text pass. Same shape LiveKit and Pipecat use: cheap VAD for speech and silence, turn model on top. It works on real speech. Sweeping cut points through the JFK sample scores clause and sentence ends at 0.86-0.90 and mid-clause at 0.03. Inference is ~30 ms on this VM's CPU. It cannot be tested with text-to-speech. Piper scored 0.93-0.99 whether the sentence was complete or truncated, because a synthesiser gives a fragment the falling intonation of a finished sentence -- it does not know the text is a fragment, and this model reads prosody. Validating it needs real recordings. Co-Authored-By: Claude Opus 5 --- dev/semantic-endpointing.md | 59 +++++++++++++++++++++++++++++++++++++ dev/smart-turn-probe.py | 40 +++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 dev/semantic-endpointing.md create mode 100644 dev/smart-turn-probe.py diff --git a/dev/semantic-endpointing.md b/dev/semantic-endpointing.md new file mode 100644 index 0000000..bf9452a --- /dev/null +++ b/dev/semantic-endpointing.md @@ -0,0 +1,59 @@ +# Semantic endpointing + +No silence threshold can tell "still thinking" from "finished": they are +acoustically identical and only the words differ. Every setting we can reach +trades response speed against how long a pause is tolerated, one for one. + +A turn-detection model breaks that trade. It reads the utterance and predicts +whether the speaker is done, so a short silence threshold can be used for the +common case while genuine mid-thought pauses are rescued. + +## Smart Turn v3 looks like the right model + +`pipecat-ai/smart-turn-v3` on Hugging Face, BSD-2-Clause. Whisper Tiny encoder +plus a linear head, 8M parameters, 8 MB int8 ONNX. It reads the waveform, not a +transcript, so it needs no extra speech-to-text pass. Published accuracy 92.6% +over 31,527 samples across 23 languages. + +This is the same shape LiveKit and Pipecat both use: a cheap VAD for +speech/silence, and a separate end-of-utterance model on top. + +Interface: `input_features` `[batch, 80, 800]` — a Whisper mel spectrogram of +the **last 8 seconds** — and one output which is already a probability, not a +logit. Preprocessing is `WhisperFeatureExtractor(chunk_length=8)` with +`do_normalize=True`. + +## It works, on real speech + +Sweeping cut points through the JFK sample: + + cut (s) P(complete) + 2.5 0.864 "And so my fellow Americans," clause end + 3.0 0.568 + 4.0 0.035 mid-clause + 7.5 0.028 mid-clause + 10.5 0.904 "...do for your country." sentence end + 11.0 0.625 + +Clause and sentence ends score high, mid-clause scores near zero. Inference is +~30 ms on this VM's CPU, unoptimised. + +## Text-to-speech cannot test this + +Piper clips scored 0.93-0.99 whether the sentence was complete or truncated +mid-phrase. That is not the model failing: asked to say "Turn on the kitchen", +a synthesiser produces the falling intonation of a finished sentence, because it +does not know the text is a fragment. The model reads prosody, so a TTS fragment +is indistinguishable from a TTS sentence. + +**Validation needs real recordings**, ideally of the person who will use it, +pausing naturally mid-request. `ldc.wav`, a single read TIMIT sentence, also +scores 0.96+ at every cut, so read speech may be a poor test too. + +## What it buys + +Today latency and pause tolerance are the same number: 0.25 s means both a +376 ms response and being cut off after a 250 ms pause. With a turn model the +short threshold governs the common case, and the model holds the turn open when +the utterance sounds unfinished -- fast when you are done, patient when you are +not. diff --git a/dev/smart-turn-probe.py b/dev/smart-turn-probe.py new file mode 100644 index 0000000..05cb7d3 --- /dev/null +++ b/dev/smart-turn-probe.py @@ -0,0 +1,40 @@ +"""Does Smart Turn v3 tell a finished utterance from a mid-sentence one? + +Sweeps cut points through a recording and prints P(turn complete) at each. +High scores should land on clause and sentence ends, low ones mid-clause. + +Needs real speech: text-to-speech gives a *fragment* the falling intonation of +a finished sentence, because the synthesiser does not know it is a fragment, +and this model reads prosody. Piper clips score 0.93-0.99 whether complete or +truncated, which says nothing about the model. + + nix shell --impure --expr 'with import {}; [ (python3.withPackages + (ps: [ ps.onnxruntime ps.transformers ps.numpy ])) ffmpeg ]' \ + --command python3 dev/smart-turn-probe.py +""" +import subprocess +import numpy as np, onnxruntime as ort +from transformers import WhisperFeatureExtractor +SR = 16000 +fe = WhisperFeatureExtractor(chunk_length=8) +sess = ort.InferenceSession("/home/agent-amd64/models/smart-turn/smart-turn-v3.2-cpu.onnx", + providers=["CPUExecutionProvider"]) +def pcm(p): + raw = subprocess.run(["ffmpeg","-v","error","-i",p,"-ar",str(SR),"-ac","1", + "-f","f32le","-"], capture_output=True, check=True).stdout + return np.frombuffer(raw, dtype=np.float32).copy() +def predict(a, tail_ms=300): + a = np.concatenate([a, np.zeros(SR*tail_ms//1000, dtype=np.float32)]) + if len(a) > 8*SR: a = a[-8*SR:] + inp = fe(a, sampling_rate=SR, return_tensors="np", padding="max_length", + max_length=8*SR, truncation=True, do_normalize=True) + f = np.expand_dims(inp.input_features.squeeze(0).astype(np.float32), 0) + return sess.run(None, {"input_features": f})[0][0].item() + +import sys +a = pcm(sys.argv[1] if len(sys.argv) > 1 else "jfk.wav") +print() +print(f"{'cut (s)':>8} {'P(complete)':>11} bar") +for ms in range(1000, int(len(a)/SR*1000)+1, 500): + p = predict(a[:SR*ms//1000]) + print(f"{ms/1000:8.1f} {p:11.3f} {'#'*int(p*40)}") From 8179ac4ba4e1c3d934a3486beff9125682b9f848 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 04:05:54 +0000 Subject: [PATCH 76/98] Measure Smart Turn on real human audio, and settle the design The model ships a labelled test set with synthetic and midfiller flags. Filtering to real non-synthetic English, 607 samples: 93.2% accuracy, against 92.63% published. That also validates the preprocessing port, since a wrong mel pipeline would read as chance. The threshold matters because the errors are not symmetric. At 0.5, 14.6% of unfinished utterances get cut off and 1.9% of finished ones wait; at 0.9, 6.1% and 8.7%. Being cut off mid-thought is the failure worth avoiding, and today every pause longer than silence_seconds cuts you off, so even the default is a large improvement. Appending silence to unfinished utterances barely moves the score -- median 0.033 at zero, 0.060 after two seconds -- so repeated checks will not converge on ending the turn by themselves, and the design needs an explicit cap for someone who simply trails off. Co-Authored-By: Claude Opus 5 --- dev/semantic-endpointing.md | 53 +++++++++++++++++++++++++++++++++++++ dev/smart-turn-eval.py | 48 +++++++++++++++++++++++++++++++++ dev/smart-turn-threshold.py | 45 +++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+) create mode 100644 dev/smart-turn-eval.py create mode 100644 dev/smart-turn-threshold.py diff --git a/dev/semantic-endpointing.md b/dev/semantic-endpointing.md index bf9452a..02ddc1d 100644 --- a/dev/semantic-endpointing.md +++ b/dev/semantic-endpointing.md @@ -57,3 +57,56 @@ Today latency and pause tolerance are the same number: 0.25 s means both a short threshold governs the common case, and the model holds the turn open when the utterance sounds unfinished -- fast when you are done, patient when you are not. + + +## Measured on the model's own labelled test set + +`pipecat-ai/smart-turn-data-v3.1-test` carries `endpoint_bool`, plus `synthetic` +and `midfiller` flags. Filtering to **real, non-synthetic English** recordings +(607 of them in one shard, 312 complete and 295 not): + +**93.2% accuracy**, against the 92.63% published across all languages. That also +validates the preprocessing port -- a wrong mel pipeline reads as chance. + +The two errors cost very different amounts, so the threshold matters: + +| threshold | cut off mid-thought | made to wait when done | +|---|---|---| +| 0.3 | 19.7% | 1.0% | +| 0.5 (default) | 14.6% | 1.9% | +| 0.7 | 11.2% | 3.8% | +| 0.8 | 8.8% | 4.8% | +| 0.9 | 6.1% | 8.7% | +| 0.95 | 2.4% | 13.1% | +| 0.98 | 0.7% | 32.1% | + +Being cut off mid-thought is the failure worth avoiding; being made to wait costs +one extra increment. Today *every* pause longer than `silence_seconds` cuts you +off, so even the default threshold is a large improvement, and ~0.9 looks like a +sensible operating point. + +## The verdict does not drift, so the design needs a cap + +Appending silence to unfinished utterances barely moves the score: + + +0 ms median P 0.033 + +500 ms median P 0.044 + +2000 ms median P 0.060 + +So re-checking as silence accumulates will not converge on ending the turn by +itself. Something that sounds unfinished stays unfinished, and an utterance +someone simply trails off from would hold the turn open forever. The design +needs an explicit maximum. + +## Design + +1. Silero detects silence as now, with `silence_seconds` set low (~0.25 s). +2. On expiry, run Smart Turn over the last 8 s of buffered audio (~30 ms). +3. `P(complete) > 0.9` -> end the turn. Total ~280 ms. +4. Otherwise extend by another increment and re-check. +5. Cap the total extension (~2-3 s) and end regardless, because of the drift + result above. + +This is the shape Pipecat and LiveKit both use. It decouples the two numbers +that are currently the same: a short threshold governs the common case, and the +model holds the turn open only when the utterance actually sounds unfinished. diff --git a/dev/smart-turn-eval.py b/dev/smart-turn-eval.py new file mode 100644 index 0000000..ad99396 --- /dev/null +++ b/dev/smart-turn-eval.py @@ -0,0 +1,48 @@ +"""Evaluate Smart Turn v3 on its own labelled test set, real recordings only. + +Validates the preprocessing (a wrong mel pipeline shows up as chance accuracy) +and, more usefully, reports the subset with a mid-utterance filler -- someone +pausing mid-thought, which is the case a silence threshold cannot handle. +""" +import io, sys +import numpy as np, onnxruntime as ort, pyarrow.parquet as pq, soundfile as sf +from transformers import WhisperFeatureExtractor + +SR = 16000 +fe = WhisperFeatureExtractor(chunk_length=8) +sess = ort.InferenceSession("/home/agent-amd64/models/smart-turn/smart-turn-v3.2-cpu.onnx", + providers=["CPUExecutionProvider"]) + +def predict(a): + if len(a) > 8*SR: a = a[-8*SR:] + inp = fe(a, sampling_rate=SR, return_tensors="np", padding="max_length", + max_length=8*SR, truncation=True, do_normalize=True) + f = np.expand_dims(inp.input_features.squeeze(0).astype(np.float32), 0) + return sess.run(None, {"input_features": f})[0][0].item() + +tbl = pq.read_table(sys.argv[1] if len(sys.argv) > 1 + else "/home/agent-amd64/models/smart-turn/data/t0.parquet") +cols = tbl.column_names +print("columns:", cols) +n = tbl.num_rows +limit = int(sys.argv[2]) if len(sys.argv) > 2 else 400 +rows = tbl.to_pylist() +real = [r for r in rows if not r.get("synthetic") and r.get("language") == "eng"] +print(f"{n} rows, {len(real)} real English; scoring up to {limit}\n") + +groups = {} +for r in real[:limit]: + audio = r["audio"] + data, sr = sf.read(io.BytesIO(audio["bytes"]), dtype="float32") + if data.ndim > 1: data = data.mean(axis=1) + p = predict(data) + correct = (p > 0.5) == bool(r["endpoint_bool"]) + for key in ("all", "midfiller" if r.get("midfiller") else "no filler", + "complete" if r["endpoint_bool"] else "incomplete"): + g = groups.setdefault(key, [0, 0]); g[0] += correct; g[1] += 1 + +print(f"{'subset':>12} {'n':>5} {'accuracy':>9}") +for k in ("all", "complete", "incomplete", "midfiller", "no filler"): + if k in groups: + c, t = groups[k] + print(f"{k:>12} {t:5d} {c/t*100:8.1f}%") diff --git a/dev/smart-turn-threshold.py b/dev/smart-turn-threshold.py new file mode 100644 index 0000000..6e64a34 --- /dev/null +++ b/dev/smart-turn-threshold.py @@ -0,0 +1,45 @@ +"""Score every real English sample once, then sweep the decision threshold. + +The two errors are not equal. Calling an unfinished utterance complete cuts the +speaker off mid-thought. Calling a finished one incomplete just waits a little +longer. So the threshold should be biased towards waiting. +""" +import io, json, os, sys +import numpy as np, onnxruntime as ort, pyarrow.parquet as pq, soundfile as sf +from transformers import WhisperFeatureExtractor + +SR = 16000 +CACHE = "/home/agent-amd64/models/smart-turn/scores.json" +fe = WhisperFeatureExtractor(chunk_length=8) +sess = ort.InferenceSession("/home/agent-amd64/models/smart-turn/smart-turn-v3.2-cpu.onnx", + providers=["CPUExecutionProvider"]) + +def predict(a): + if len(a) > 8*SR: a = a[-8*SR:] + inp = fe(a, sampling_rate=SR, return_tensors="np", padding="max_length", + max_length=8*SR, truncation=True, do_normalize=True) + f = np.expand_dims(inp.input_features.squeeze(0).astype(np.float32), 0) + return sess.run(None, {"input_features": f})[0][0].item() + +if os.path.exists(CACHE): + scored = json.load(open(CACHE)) +else: + scored = [] + for shard in sys.argv[1:]: + for r in pq.read_table(shard).to_pylist(): + if r.get("synthetic") or r.get("language") != "eng": + continue + d, _ = sf.read(io.BytesIO(r["audio"]["bytes"]), dtype="float32") + if d.ndim > 1: d = d.mean(axis=1) + scored.append({"p": predict(d), "complete": bool(r["endpoint_bool"]), + "dataset": r["dataset"]}) + json.dump(scored, open(CACHE, "w")) + +comp = [s["p"] for s in scored if s["complete"]] +inc = [s["p"] for s in scored if not s["complete"]] +print(f"scored {len(scored)} real English samples: {len(comp)} complete, {len(inc)} incomplete\n") +print(f"{'threshold':>10} {'cut off mid-thought':>21} {'made to wait when done':>24}") +for thr in (0.3, 0.5, 0.7, 0.8, 0.9, 0.95, 0.98, 0.99): + cut = sum(1 for p in inc if p > thr) / len(inc) * 100 # said complete, was not + wait = sum(1 for p in comp if p <= thr) / len(comp) * 100 # said incomplete, was not + print(f"{thr:>10} {cut:>20.1f}% {wait:>23.1f}%") From 5159b0feb05fc341ee1e0e4adcfa990c008889a6 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 04:26:33 +0000 Subject: [PATCH 77/98] Add a dependency-light Smart Turn module Pure numpy plus onnxruntime, so Home Assistant does not need transformers for this. Whisper's feature extraction is reimplemented here and verified against WhisperFeatureExtractor(chunk_length=8): features match to 0.000000 for audio above, below and exactly at the 8 second window, the mel filterbank matches to 1e-16, and end-to-end probabilities on real recordings are bit-identical to the reference implementation. The docstring carries the warning that cost an experiment to learn: this model must only be asked about audio that ends where a speaker paused. Fed audio cut mid-word, 60.6% of polls score above 0.9, because it judges the prosody up to the cut and cannot know the cut was arbitrary. It answers "does this sound finished", not "has this person stopped talking". Co-Authored-By: Claude Opus 5 --- packages/smart-turn/smart_turn_vad.py | 121 ++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 packages/smart-turn/smart_turn_vad.py diff --git a/packages/smart-turn/smart_turn_vad.py b/packages/smart-turn/smart_turn_vad.py new file mode 100644 index 0000000..88d63d4 --- /dev/null +++ b/packages/smart-turn/smart_turn_vad.py @@ -0,0 +1,121 @@ +"""Smart Turn v3: has the speaker finished their turn? + +A silence threshold cannot tell "still thinking" from "finished" -- they are +acoustically identical and only the words differ. This model reads the waveform +and predicts whether the utterance sounds complete, so a short silence threshold +can handle the common case while genuine mid-thought pauses are held open. + +Model: pipecat-ai/smart-turn-v3, BSD-2-Clause. Whisper Tiny encoder plus a linear +head, 8M parameters. Input is a mel spectrogram of the last 8 seconds; output is +already a probability, not a logit. + +The feature extraction is Whisper's, reimplemented here in numpy rather than +pulling in transformers. It is verified equal to +`WhisperFeatureExtractor(chunk_length=8)` to 0.000000 across audio lengths above, +below and exactly at 8 seconds, and the filterbank matches to 1e-16. + +IMPORTANT: only ask this model about audio that ends where a speaker paused. Fed +audio cut mid-word it has no way to know the cut was arbitrary, and judges the +prosody up to that point: 60.6% of mid-utterance polls score above 0.9. It +answers "does this sound finished", not "has this person stopped talking". Pair +it with a VAD, which answers the latter. +""" + +from __future__ import annotations + +import os + +import numpy as np +import onnxruntime as ort + +SAMPLE_RATE = 16000 +N_FFT = 400 +HOP_LENGTH = 160 +N_MELS = 80 +WINDOW_SECONDS = 8 +N_SAMPLES = WINDOW_SECONDS * SAMPLE_RATE + +MODEL_ENV = "SMART_TURN_MODEL" + + +def _hz_to_mel(freq: np.ndarray) -> np.ndarray: + f_sp, min_log_hz = 200.0 / 3, 1000.0 + min_log_mel = min_log_hz / f_sp + logstep = np.log(6.4) / 27.0 + freq = np.asarray(freq, dtype=np.float64) + with np.errstate(divide="ignore"): # log(0) at DC, discarded by the where + return np.where( + freq >= min_log_hz, min_log_mel + np.log(freq / min_log_hz) / logstep, freq / f_sp + ) + + +def _mel_to_hz(mel: np.ndarray) -> np.ndarray: + f_sp, min_log_hz = 200.0 / 3, 1000.0 + min_log_mel = min_log_hz / f_sp + logstep = np.log(6.4) / 27.0 + mel = np.asarray(mel, dtype=np.float64) + return np.where( + mel >= min_log_mel, min_log_hz * np.exp(logstep * (mel - min_log_mel)), mel * f_sp + ) + + +def _mel_filters() -> np.ndarray: + """Whisper's mel filterbank: (n_freqs, n_mels), slaney scale and norm.""" + fft_freqs = np.linspace(0.0, SAMPLE_RATE / 2, N_FFT // 2 + 1) + mel_points = np.linspace(_hz_to_mel(0.0), _hz_to_mel(8000.0), N_MELS + 2) + hz_points = _mel_to_hz(mel_points) + diff = np.diff(hz_points) + slopes = hz_points.reshape(-1, 1) - fft_freqs.reshape(1, -1) + down = -slopes[:-2] / diff[:-1].reshape(-1, 1) + up = slopes[2:] / diff[1:].reshape(-1, 1) + filters = np.maximum(0.0, np.minimum(down, up)) + filters *= (2.0 / (hz_points[2 : N_MELS + 2] - hz_points[:N_MELS])).reshape(-1, 1) + return filters.T.astype(np.float32) + + +class SmartTurn: + """Predicts whether a turn has ended. Thread-compatible, not thread-safe.""" + + def __init__(self, model_path: str | None = None) -> None: + path = model_path or os.environ.get(MODEL_ENV) + if not path: + raise ValueError(f"no model path given and {MODEL_ENV} is unset") + options = ort.SessionOptions() + options.inter_op_num_threads = 1 + options.intra_op_num_threads = 1 + self._session = ort.InferenceSession( + path, sess_options=options, providers=["CPUExecutionProvider"] + ) + self._filters = _mel_filters() + self._window = np.hanning(N_FFT + 1)[:-1].astype(np.float32) + + def _features(self, audio: np.ndarray) -> np.ndarray: + audio = np.asarray(audio, dtype=np.float32) + if len(audio) > N_SAMPLES: + audio = audio[-N_SAMPLES:] # keep the END: that is where the turn is + real = len(audio) + if real < N_SAMPLES: + audio = np.pad(audio, (0, N_SAMPLES - real)) + # zero-mean unit-variance over the real samples, padding left at zero + audio = (audio - audio[:real].mean()) / np.sqrt(audio[:real].var() + 1e-7) + audio[real:] = 0.0 + + pad = N_FFT // 2 + padded = np.pad(audio.astype(np.float32), (pad, pad), mode="reflect") + frames = 1 + N_SAMPLES // HOP_LENGTH + stft = np.stack( + [ + np.fft.rfft(padded[i * HOP_LENGTH : i * HOP_LENGTH + N_FFT] * self._window) + for i in range(frames) + ], + axis=1, + ) + magnitudes = (np.abs(stft) ** 2)[:, :-1] + log_spec = np.log10(np.clip(self._filters.T @ magnitudes, 1e-10, None)) + log_spec = np.maximum(log_spec, log_spec.max() - 8.0) + return ((log_spec + 4.0) / 4.0).astype(np.float32) + + def probability(self, audio: np.ndarray) -> float: + """Probability the turn is complete, for audio ending where speech stopped.""" + features = np.expand_dims(self._features(audio), 0) + return float(self._session.run(None, {"input_features": features})[0][0].item()) From 13a199a4657c1fed4fa5f9d7dbcef6e1ec248e5d Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 04:57:36 +0000 Subject: [PATCH 78/98] Wire semantic endpointing into the Home Assistant pipeline On reaching silence_seconds the pipeline now asks Smart Turn whether the utterance sounds complete, and grants another silence window if not. Three settings on AudioSettings, all reachable over the websocket API so they can be tuned without satellite hardware: turn_detection, turn_threshold defaulting to 0.9, and turn_max_seconds defaulting to 3. The cap is not optional. The model's verdict does not drift as silence accumulates, so without it someone who trails off holds the turn open forever. Inference runs in the executor. It is ~25 ms of CPU, which would otherwise stall the event loop that is feeding it audio. Any failure is treated as "finished", which is exactly the behaviour without the model, so a broken model cannot lose a command. smart-turn.nix builds against the caller's interpreter, so Home Assistant gets the module built for its own Python, the same reason pysilero-vad is a package input rather than a service extraPackage. Co-Authored-By: Claude Opus 5 --- packages/home-assistant-smart-turn.patch | 192 +++++++++++++++++++++++ packages/home-assistant.nix | 12 ++ packages/overlay.nix | 2 + packages/smart-turn.nix | 65 ++++++++ 4 files changed, 271 insertions(+) create mode 100644 packages/home-assistant-smart-turn.patch create mode 100644 packages/smart-turn.nix diff --git a/packages/home-assistant-smart-turn.patch b/packages/home-assistant-smart-turn.patch new file mode 100644 index 0000000..ec92f5b --- /dev/null +++ b/packages/home-assistant-smart-turn.patch @@ -0,0 +1,192 @@ +--- a/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 04:52:05.196107157 +0000 ++++ b/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 04:52:40.393168382 +0000 +@@ -524,6 +524,33 @@ + silence_seconds: float = 0.7 + """Seconds of silence after voice command has ended.""" + ++ turn_detection: bool = False ++ """Consult a turn-detection model before ending the voice command. ++ ++ Silence alone cannot tell "still thinking" from "finished": they are ++ acoustically identical and only the words differ. With this on, reaching ++ silence_seconds asks a model whether the utterance sounds complete, and ++ keeps listening if it does not. That decouples response speed from pause ++ tolerance, which are otherwise the same number. ++ """ ++ ++ turn_threshold: float = 0.9 ++ """Probability above which the utterance is taken as complete. ++ ++ The two errors cost different amounts. Ending early cuts the speaker off ++ mid-thought; ending late costs one more silence_seconds. Measured on real ++ English speech, 0.5 cuts off 14.6% of unfinished utterances and delays 1.9% ++ of finished ones; 0.9 is 6.1% and 8.7%. ++ """ ++ ++ turn_max_seconds: float = 3.0 ++ """Give up and end the command after this much silence, regardless. ++ ++ The model's verdict does not drift as silence accumulates -- unfinished ++ utterances sit at 0.03 after two further seconds -- so without a cap, ++ someone who simply trails off would hold the turn open forever. ++ """ ++ + def __post_init__(self) -> None: + """Verify settings post-initialization.""" + if (self.noise_suppression_level < 0) or (self.noise_suppression_level > 4): +@@ -953,9 +980,24 @@ + silence_seconds=self.audio_settings.silence_seconds + ) + ++ turn_detector = None ++ if stt_vad is not None and self.audio_settings.turn_detection: ++ try: ++ from smart_turn_vad import DEFAULT_MODEL_PATH, SmartTurn ++ ++ turn_detector = SmartTurn(DEFAULT_MODEL_PATH) ++ except Exception: # noqa: BLE001 ++ _LOGGER.warning( ++ "turn detection requested but smart_turn_vad is unavailable; " ++ "falling back to silence alone", ++ exc_info=True, ++ ) ++ + result = await self.stt_provider.async_process_audio_stream( + metadata, +- self._speech_to_text_stream(audio_stream=stream, stt_vad=stt_vad), ++ self._speech_to_text_stream( ++ audio_stream=stream, stt_vad=stt_vad, turn_detector=turn_detector ++ ), + ) + except asyncio.CancelledError, TimeoutError: + raise # expected +@@ -1001,17 +1043,38 @@ + self, + audio_stream: AsyncIterable[EnhancedAudioChunk], + stt_vad: VoiceCommandSegmenter | None, ++ turn_detector: Any | None = None, + sample_rate: int = SAMPLE_RATE, + sample_width: int = SAMPLE_WIDTH, + ) -> AsyncGenerator[bytes]: + """Yield audio chunks until VAD detects silence or speech-to-text completes.""" + sent_vad_start = False ++ # The turn model reads the last 8 s of audio, so keep exactly that. ++ turn_window = 8 * sample_rate * sample_width ++ turn_audio = bytearray() if turn_detector is not None else None ++ silence_spent = 0.0 ++ + async for chunk in audio_stream: + self._capture_chunk(chunk.audio) + ++ if turn_audio is not None: ++ turn_audio += chunk.audio ++ if len(turn_audio) > turn_window: ++ del turn_audio[: len(turn_audio) - turn_window] ++ + if stt_vad is not None: + chunk_seconds = (len(chunk.audio) // sample_width) / sample_rate + if not stt_vad.process(chunk_seconds, chunk.speech_probability): ++ silence_spent += self.audio_settings.silence_seconds ++ if ( ++ turn_detector is not None ++ and silence_spent < self.audio_settings.turn_max_seconds ++ and not await self._turn_is_complete(turn_detector, turn_audio) ++ ): ++ # Sounds unfinished: keep listening for another window. ++ stt_vad.extend_silence() ++ continue ++ + # Silence detected at the end of voice command + self.process_event( + PipelineEvent( +@@ -1033,6 +1096,35 @@ + + yield chunk.audio + ++ async def _turn_is_complete(self, turn_detector: Any, audio: bytearray) -> bool: ++ """Ask the turn model whether the speaker has finished. ++ ++ Runs in the executor: ~25 ms of CPU, which would otherwise stall the ++ event loop and the audio it is feeding us. A failure here must not lose ++ the command, so any error is treated as "finished" -- the behaviour we ++ would have had without the model at all. ++ """ ++ try: ++ import numpy as np ++ ++ samples = np.frombuffer(bytes(audio), dtype=np.int16).astype(np.float32) ++ samples /= 32768.0 ++ probability = await self.hass.async_add_executor_job( ++ turn_detector.probability, samples ++ ) ++ except Exception: # noqa: BLE001 ++ _LOGGER.warning("turn detection failed; ending the turn", exc_info=True) ++ return True ++ ++ complete = probability >= self.audio_settings.turn_threshold ++ _LOGGER.debug( ++ "turn detection: p=%.3f threshold=%.2f -> %s", ++ probability, ++ self.audio_settings.turn_threshold, ++ "complete" if complete else "keep listening", ++ ) ++ return complete ++ + async def prepare_recognize_intent(self, session: chat_session.ChatSession) -> None: + """Prepare recognizing an intent.""" + self._conversation_data = async_get_pipeline_conversation_data( +--- a/homeassistant/components/assist_pipeline/vad.py 2026-08-18 04:52:05.196093772 +0000 ++++ b/homeassistant/components/assist_pipeline/vad.py 2026-08-18 04:52:40.393291327 +0000 +@@ -130,6 +130,15 @@ + self._reset_seconds_left = self.reset_seconds + self.in_command = False + ++ def extend_silence(self) -> None: ++ """Grant another silence_seconds before the command may end. ++ ++ Used when a turn-detection model says the utterance sounds unfinished, ++ so the speaker gets more time without the silence threshold itself ++ having to be long. ++ """ ++ self._silence_seconds_left = self.silence_seconds ++ + def process(self, chunk_seconds: float, speech_probability: float | None) -> bool: + """Process samples using external VAD. + +--- a/homeassistant/components/assist_pipeline/websocket_api.py 2026-08-18 04:52:05.196122125 +0000 ++++ b/homeassistant/components/assist_pipeline/websocket_api.py 2026-08-18 04:55:32.649741760 +0000 +@@ -102,6 +102,11 @@ + # it is reachable only through the VAD sensitivity + # select entity that satellite integrations create. + vol.Optional("silence_seconds"): vol.Any(float, int), ++ # Turn detection: ask a model whether the utterance ++ # sounds finished before ending on silence alone. ++ vol.Optional("turn_detection"): bool, ++ vol.Optional("turn_threshold"): vol.Any(float, int), ++ vol.Optional("turn_max_seconds"): vol.Any(float, int), + } + }, + extra=vol.ALLOW_EXTRA, +@@ -219,11 +224,17 @@ + auto_gain_dbfs=msg_input.get("auto_gain_dbfs", 0), + volume_multiplier=msg_input.get("volume_multiplier", 1.0), + is_vad_enabled=not msg_input.get("no_vad", False), +- **( +- {"silence_seconds": float(msg_input["silence_seconds"])} +- if "silence_seconds" in msg_input +- else {} +- ), ++ **{ ++ key: cast(value) ++ for key, cast in ( ++ ("silence_seconds", float), ++ ("turn_detection", bool), ++ ("turn_threshold", float), ++ ("turn_max_seconds", float), ++ ) ++ if key in msg_input ++ for value in (msg_input[key],) ++ }, + ) + elif start_stage == PipelineStage.INTENT: + # Input to conversation agent diff --git a/packages/home-assistant.nix b/packages/home-assistant.nix index a06d31d..9b3e665 100644 --- a/packages/home-assistant.nix +++ b/packages/home-assistant.nix @@ -30,6 +30,17 @@ home-assistant.overrideAttrs (old: { # VAD sensitivity select entity that satellite integrations create, so it # could not be tuned or tested without buying hardware. ./home-assistant-vad-silence-api.patch + + # Semantic endpointing. Silence alone cannot tell "still thinking" from + # "finished" -- acoustically identical, only the words differ -- so latency + # and pause tolerance are forced to be the same number. On reaching + # silence_seconds this asks Smart Turn v3 whether the utterance sounds + # complete and keeps listening if not, which separates them. + # + # The model must only be asked about audio that ends where speech stopped: + # fed audio cut mid-word, 60.6% of polls read as complete, because it judges + # the prosody up to the cut. Silero still decides *when* to ask. + ./home-assistant-smart-turn.patch ]; # audio_enhancer.py imports pysilero_vad after the Silero patch. The manifest @@ -42,5 +53,6 @@ home-assistant.overrideAttrs (old: { # would be a different Python and the import would fail at runtime. propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ home-assistant.python3Packages.pysilero-vad + (home-assistant.python3Packages.callPackage ./smart-turn.nix { }) ]; }) diff --git a/packages/overlay.nix b/packages/overlay.nix index 74f169f..bcfa4ce 100644 --- a/packages/overlay.nix +++ b/packages/overlay.nix @@ -4,6 +4,8 @@ final: prev: { ollama-patched = import ./ollama.nix prev.ollama; + + smart-turn-vad = prev.python3Packages.callPackage ./smart-turn.nix { }; } // prev.lib.optionalAttrs (prev ? ollama-cuda) { ollama-cuda-patched = import ./ollama.nix prev.ollama-cuda; diff --git a/packages/smart-turn.nix b/packages/smart-turn.nix new file mode 100644 index 0000000..a882b7d --- /dev/null +++ b/packages/smart-turn.nix @@ -0,0 +1,65 @@ +# Smart Turn v3: predicts whether a speaker has finished their turn. +# +# A silence threshold cannot tell "still thinking" from "finished" -- they are +# acoustically identical and only the words differ. This model reads the +# waveform and says whether the utterance sounds complete, so a short silence +# threshold can serve the common case while genuine mid-thought pauses are held +# open. +# +# Measured on the model's own labelled test set, real non-synthetic English +# only: 93.2% accuracy over 607 samples, 25 ms per call on one CPU thread. +{ + lib, + buildPythonPackage, + python, + numpy, + onnxruntime, + fetchurl, +}: + +let + model = fetchurl { + url = "https://huggingface.co/pipecat-ai/smart-turn-v3/resolve/main/smart-turn-v3.2-cpu.onnx"; + hash = "sha256-K7AmMWsUpmBIanWxczzT+6uML9AxTcmve+SfjMqWfk8="; + }; +in +buildPythonPackage { + pname = "smart-turn-vad"; + version = "3.2"; + format = "other"; + + src = ./smart-turn; + + propagatedBuildInputs = [ + numpy + onnxruntime + ]; + + dontUnpack = true; + + installPhase = '' + runHook preInstall + mkdir -p $out/${python.sitePackages} + cp ${./smart-turn/smart_turn_vad.py} \ + $out/${python.sitePackages}/smart_turn_vad.py + mkdir -p $out/share/smart-turn + cp ${model} $out/share/smart-turn/smart-turn-v3.2-cpu.onnx + runHook postInstall + ''; + + # The module finds the model through this, so a caller needs no path. + postFixup = '' + cat >> $out/${python.sitePackages}/smart_turn_vad.py < Date: Tue, 18 Aug 2026 05:21:23 +0000 Subject: [PATCH 79/98] Semantic endpointing works end to end Streaming a real recording through the dev instance, with no artificial pause -- the speaker's own pauses are the test: silence only, 0.25 s ended 2.5s ' And so my fellow Americans' silence only, 0.70 s ended 3.0s ' And so my fellow Americans!' turn detection, 0.25 s ended 5.2s ' And so my fellow Americans, ASK NOT!' The model recognised the pause after "Americans," as unfinished and kept listening, at the shorter threshold. It stops at 5.2 s because turn_max_seconds was 2.0 and this speaker pauses for effect, which is the cap working. Fixes a bug found by that test: process() calls reset() as it reports the command finished, clearing in_command, so granting another silence window must restore the flag as well as the counter. The silence counter only decrements inside a command, so without it the segmenter waits for speech that never comes and the turn never ends -- runs hung to the 60 s timeout while the model logged "keep listening" correctly each time. Co-Authored-By: Claude Opus 5 --- dev/configuration.yaml | 7 ++ dev/semantic-endpointing.md | 35 ++++++ dev/turn-test.py | 134 +++++++++++++++++++++++ packages/home-assistant-smart-turn.patch | 12 +- 4 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 dev/turn-test.py diff --git a/dev/configuration.yaml b/dev/configuration.yaml index 8319292..601c9a6 100644 --- a/dev/configuration.yaml +++ b/dev/configuration.yaml @@ -21,3 +21,10 @@ assist_pipeline: # Fake lights and sensors, so the assistant has something to control. demo: + +# Turn detection logs its verdict at debug level; without this the decision is +# invisible and a test can only infer it from timing. +logger: + default: info + logs: + homeassistant.components.assist_pipeline: debug diff --git a/dev/semantic-endpointing.md b/dev/semantic-endpointing.md index 02ddc1d..8131f66 100644 --- a/dev/semantic-endpointing.md +++ b/dev/semantic-endpointing.md @@ -110,3 +110,38 @@ needs an explicit maximum. This is the shape Pipecat and LiveKit both use. It decouples the two numbers that are currently the same: a short threshold governs the common case, and the model holds the turn open only when the utterance actually sounds unfinished. + + +## Working end to end + +Streaming a real recording in real time through the dev instance, with no +artificial pause -- the speaker's own pauses are the test: + + silence only, 0.25 s ended 2.5s heard: ' And so my fellow Americans' + silence only, 0.70 s ended 3.0s heard: ' And so my fellow Americans!' + turn detection, 0.25 s ended 5.2s heard: ' And so my fellow Americans, ASK NOT!' + +The model recognised the pause after "Americans," as unfinished and kept +listening, at the *shorter* silence threshold. It still stops at 5.2 s because +`turn_max_seconds` was 2.0 and this speaker pauses for dramatic effect -- which +is the cap doing its job. + +`dev/turn-test.py` runs this. It needs a speech-to-text engine in the VM: + + nix build --no-link --print-out-paths nixpkgs#wyoming-faster-whisper + .../bin/wyoming-faster-whisper --model tiny-int8 --language en \ + --uri tcp://127.0.0.1:10300 --data-dir ~/ha-dev/whisper \ + --download-dir ~/ha-dev/whisper + +`demo_stt` cannot stand in for it: it accepts only stereo, and the pipeline +sends mono. + +## A bug worth remembering + +`VoiceCommandSegmenter.process()` calls `reset()` as it reports the command +finished, which clears `in_command`. Granting another silence window therefore +has to restore that flag as well as the counter -- the silence counter only +decrements *inside* a command, so without it the segmenter sits waiting for +speech that may never come and the turn never ends at all. The symptom was runs +that hung until the 60 s pipeline timeout, with the model logging "keep +listening" correctly each time. diff --git a/dev/turn-test.py b/dev/turn-test.py new file mode 100644 index 0000000..785f06a --- /dev/null +++ b/dev/turn-test.py @@ -0,0 +1,134 @@ +"""Does turn detection stop the assistant cutting you off at a natural pause? + +Streams a real recording in real time and reports what the pipeline heard before +deciding the turn was over. Speakers pause mid-sentence; with a short +silence_seconds the pipeline ends there and the transcript is truncated. The +turn model should recognise those pauses as unfinished and keep listening. + +No artificial pause is inserted: the recording's own pauses are the test. + + dev/turn-test.py [audio] [seconds] +""" +import asyncio, json, subprocess, sys, time, urllib.request +import websockets + +URL = "ws://127.0.0.1:8123/api/websocket" +TOKEN = open("scratch/hadev/token.txt").read().strip() +SR = 16000 + + +def rest(path, body=None): + req = urllib.request.Request( + f"http://127.0.0.1:8123{path}", + data=json.dumps(body).encode() if body is not None else None, + headers={"Content-Type": "application/json", "Authorization": f"Bearer {TOKEN}"}) + return json.load(urllib.request.urlopen(req, timeout=60)) + + +def ensure_whisper(): + for e in rest("/api/config/config_entries/entry"): + if e["domain"] == "wyoming": + return + flow = rest("/api/config/config_entries/flow", + {"handler": "wyoming", "show_advanced_options": True}) + rest(f"/api/config/config_entries/flow/{flow['flow_id']}", + {"host": "127.0.0.1", "port": 10300}) + + +def pcm(path): + return subprocess.run(["ffmpeg", "-v", "error", "-i", path, "-ar", str(SR), + "-ac", "1", "-f", "s16le", "-"], + capture_output=True, check=True).stdout + + +async def call(ws, ident, **msg): + await ws.send(json.dumps({"id": ident, **msg})) + while True: + m = json.loads(await ws.recv()) + if m.get("id") == ident and m.get("type") == "result": + return m + + +async def run(ws, ident, pipeline_id, audio, **settings): + stream = audio + b"\x00" * (SR * 2 * 6) + await ws.send(json.dumps({"id": ident, "type": "assist_pipeline/run", + "start_stage": "stt", "end_stage": "stt", + "input": {"sample_rate": SR, **settings}, + "pipeline": pipeline_id, "timeout": 60})) + hid = None; task = None; text = None; vad_end = None; t_audio = None + async def pump(): + for i in range(0, len(stream), 3200): + await ws.send(bytes([hid]) + stream[i:i + 3200]) + await asyncio.sleep(0.1) + while True: + m = json.loads(await ws.recv()) + if m.get("id") != ident: + continue + if m.get("type") == "result" and not m.get("success"): + return None, None, m.get("error") + if m.get("type") != "event": + continue + e = m["event"] + if e["type"] == "run-start": + hid = e["data"]["runner_data"]["stt_binary_handler_id"] + t_audio = time.monotonic(); task = asyncio.create_task(pump()) + if e["type"] == "stt-vad-end": + vad_end = time.monotonic() - t_audio + if e["type"] == "stt-end": + text = e["data"]["stt_output"]["text"] + if e["type"] in ("run-end", "error"): + break + if task: + task.cancel() + return text, vad_end, None + + +async def main(): + ensure_whisper() + await asyncio.sleep(2) + path = sys.argv[1] if len(sys.argv) > 1 else "scratch/ha/jfk.wav" + audio = pcm(path) + if len(sys.argv) > 2: + audio = audio[: SR * 2 * int(float(sys.argv[2]) * 1000) // 1000] + ws = await websockets.connect(URL, max_size=None); await ws.recv() + await ws.send(json.dumps({"type": "auth", "access_token": TOKEN})) + assert json.loads(await ws.recv())["type"] == "auth_ok" + + states = rest("/api/states") + stt = next((s["entity_id"] for s in states if s["entity_id"].startswith("stt.") + and "demo" not in s["entity_id"]), None) + pls = (await call(ws, 1, type="assist_pipeline/pipeline/list"))["result"]["pipelines"] + existing = next((p for p in pls if p.get("name") == "turn-test"), None) + if existing and existing.get("stt_engine") != stt: + await call(ws, 2, type="assist_pipeline/pipeline/update", + pipeline_id=existing["id"], + **{k: v for k, v in existing.items() if k != "id"}, + **{"stt_engine": stt, "stt_language": "en"}) + existing["stt_engine"] = stt + if existing: + pid = existing["id"] + else: + res = await call(ws, 3, type="assist_pipeline/pipeline/create", name="turn-test", + language="en", conversation_engine="conversation.home_assistant", + conversation_language="en", stt_engine=stt, stt_language="en", + tts_engine=None, tts_language=None, tts_voice=None, + wake_word_entity=None, wake_word_id=None) + pid = res["result"]["id"] + + print(f"{path}, {len(audio)/2/SR:.1f} s of audio, stt={stt}\n") + ident = 100 + for label, settings in ( + ("silence only, 0.25 s", dict(silence_seconds=0.25, turn_detection=False)), + ("silence only, 0.70 s", dict(silence_seconds=0.70, turn_detection=False)), + ("turn detection, 0.25 s", dict(silence_seconds=0.25, turn_detection=True, + turn_threshold=0.9, turn_max_seconds=2.0)), + ): + ident += 1 + text, vad_end, err = await run(ws, ident, pid, audio, **settings) + if err: + print(f" {label:24} ERROR {err}") + continue + print(f" {label:24} ended {vad_end:5.1f}s heard: {text!r}") + await asyncio.sleep(1) + +asyncio.run(main()) diff --git a/packages/home-assistant-smart-turn.patch b/packages/home-assistant-smart-turn.patch index ec92f5b..b41071b 100644 --- a/packages/home-assistant-smart-turn.patch +++ b/packages/home-assistant-smart-turn.patch @@ -136,8 +136,8 @@ """Prepare recognizing an intent.""" self._conversation_data = async_get_pipeline_conversation_data( --- a/homeassistant/components/assist_pipeline/vad.py 2026-08-18 04:52:05.196093772 +0000 -+++ b/homeassistant/components/assist_pipeline/vad.py 2026-08-18 04:52:40.393291327 +0000 -@@ -130,6 +130,15 @@ ++++ b/homeassistant/components/assist_pipeline/vad.py 2026-08-18 05:13:13.340619266 +0000 +@@ -130,6 +130,23 @@ self._reset_seconds_left = self.reset_seconds self.in_command = False @@ -147,8 +147,16 @@ + Used when a turn-detection model says the utterance sounds unfinished, + so the speaker gets more time without the silence threshold itself + having to be long. ++ ++ process() calls reset() as it reports the command finished, which clears ++ in_command. Restoring that matters: the silence counter only decrements ++ inside a command, so without it the segmenter waits for speech that may ++ never come and the command never ends at all. The command-length minimum ++ is already satisfied, since we were in a command a moment ago. + """ ++ self.in_command = True + self._silence_seconds_left = self.silence_seconds ++ self._command_seconds_left = 0.0 + def process(self, chunk_seconds: float, speech_probability: float | None) -> bool: """Process samples using external VAD. From ec8976fd331b846ea980b65362c221fbe47d4cde Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 05:25:53 +0000 Subject: [PATCH 80/98] Measure the payoff: 600 ms off, with pause tolerance kept Sweeping silence_seconds with and without the turn model, measuring both things that matter. Without it every setting cuts a mid-sentence pause short, because tolerance is the threshold. With it the pause is held at every setting tried, down to 0.10. So silence_seconds can go 0.7 -> 0.1, which takes 600 ms off a finished utterance, while pauses keep working. The model costs about 100 ms of that back. Roughly 200 ms from end of speech to decision, against 789 ms for silence alone at 0.7. Co-Authored-By: Claude Opus 5 --- dev/semantic-endpointing.md | 29 ++++++++++++++++++++++++++++ dev/silence-sweep.py | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 dev/silence-sweep.py diff --git a/dev/semantic-endpointing.md b/dev/semantic-endpointing.md index 8131f66..c16d24d 100644 --- a/dev/semantic-endpointing.md +++ b/dev/semantic-endpointing.md @@ -145,3 +145,32 @@ decrements *inside* a command, so without it the segmenter sits waiting for speech that may never come and the turn never ends at all. The symptom was runs that hung until the 60 s pipeline timeout, with the model logging "keep listening" correctly each time. + + +## The payoff: latency and pause tolerance finally separate + +Sweeping `silence_seconds` with and without the turn model, measuring both +things that matter -- when a *finished* utterance ends, and whether a +*mid-sentence* pause survives: + +| silence_seconds | turn | finished utterance ends | mid-sentence pause | +|---|---|---|---| +| 0.70 | off | 3.6 s | cut short | +| 0.70 | on | 3.7 s | **held** | +| 0.40 | off | 3.3 s | cut short | +| 0.40 | on | 3.4 s | **held** | +| 0.25 | off | 3.1 s | cut short | +| 0.25 | on | 3.2 s | **held** | +| 0.15 | off | 3.0 s | cut short | +| 0.15 | on | 3.1 s | **held** | +| 0.10 | off | 3.0 s | cut short | +| **0.10** | **on** | **3.1 s** | **held** | + +Without the model, every setting cuts the pause short -- the tolerance *is* the +threshold. With it, the pause is held at every setting, and the threshold is +free to be small. `silence_seconds` 0.7 -> 0.1 takes **600 ms** off a finished +utterance while pauses keep working, and the model itself costs about 100 ms. + +Recommended: `silence_seconds` 0.1-0.15 with `turn_detection` on. That is +roughly 200 ms from end of speech to decision, against 789 ms measured for +silence alone at 0.7, and it is inside the 200 ms band of human turn-taking. diff --git a/dev/silence-sweep.py b/dev/silence-sweep.py new file mode 100644 index 0000000..5f932ca --- /dev/null +++ b/dev/silence-sweep.py @@ -0,0 +1,38 @@ +"""How low can silence_seconds go once a turn model is catching continuations? + +Two things matter and they pull apart: a finished utterance should end fast, and +a natural mid-sentence pause should still be held open. Measures both. +""" +import asyncio, json, sys +sys.argv = sys.argv[:1] +exec(open("dev/turn-test.py").read().split("async def main")[0]) + +COMPLETE = "scratch/ha/ldc.wav" # one finished sentence +PAUSED = "scratch/ha/jfk.wav" # pauses mid-sentence, dramatically + +async def main(): + ensure_whisper() + ws = await websockets.connect(URL, max_size=None); await ws.recv() + await ws.send(json.dumps({"type": "auth", "access_token": TOKEN})) + assert json.loads(await ws.recv())["type"] == "auth_ok" + pls = (await call(ws, 1, type="assist_pipeline/pipeline/list"))["result"]["pipelines"] + pid = next(p["id"] for p in pls if p.get("name") == "turn-test") + done, paused = pcm(COMPLETE), pcm(PAUSED)[: 16000 * 2 * 6] + ident = 200 + print(f"{'silence':>9} {'turn':>6} {'finished utterance':>28} {'mid-sentence pause':>30}") + for ss in (0.10, 0.15, 0.25, 0.40, 0.70): + for td in (False, True): + ident += 1 + t1, e1, _ = await run(ws, ident, pid, done, silence_seconds=ss, + turn_detection=td, turn_threshold=0.9, + turn_max_seconds=2.0) + ident += 1 + t2, e2, _ = await run(ws, ident, pid, paused, silence_seconds=ss, + turn_detection=td, turn_threshold=0.9, + turn_max_seconds=2.0) + held = "held" if (t2 and "ASK" in (t2 or "").upper()) else "cut short" + print(f"{ss:>9} {str(td):>6} {(e1 or 0):>7.1f}s {str(t1)[:19]:>20} " + f"{(e2 or 0):>7.1f}s {held:>10} {str(t2)[:12]}") + await asyncio.sleep(1) + +asyncio.run(main()) From 589eef98b6857468eb240bd164769053bd3cd410 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 05:27:12 +0000 Subject: [PATCH 81/98] End to end: 1408 ms -> 824 ms Full pipeline through the real conversation agent, timed from the last sample of speech to the answer: 1408 ms with silence 0.7 and no turn model, 824 ms with silence 0.1 and the model. Same answer. Co-Authored-By: Claude Opus 5 --- dev/e2e-compare.py | 79 +++++++++++++++++++++++++++++++++++++ dev/semantic-endpointing.md | 11 ++++++ 2 files changed, 90 insertions(+) create mode 100644 dev/e2e-compare.py diff --git a/dev/e2e-compare.py b/dev/e2e-compare.py new file mode 100644 index 0000000..276a491 --- /dev/null +++ b/dev/e2e-compare.py @@ -0,0 +1,79 @@ +"""End to end, old settings against new: audio in, answer out.""" +import asyncio, json, statistics, subprocess, sys, time, urllib.request +sys.argv = sys.argv[:1] +exec(open("dev/turn-test.py").read().split("async def main")[0]) + +async def one(ws, ident, pid, audio, **settings): + stream = audio + b"\x00" * (16000 * 2 * 4) + await ws.send(json.dumps({"id": ident, "type": "assist_pipeline/run", + "start_stage": "stt", "end_stage": "intent", + "input": {"sample_rate": 16000, **settings}, + "pipeline": pid, "timeout": 60})) + hid = None; task = None; marks = {}; t_audio_end = None; reply = "" + async def pump(): + nonlocal t_audio_end + for i in range(0, len(stream), 3200): + await ws.send(bytes([hid]) + stream[i:i + 3200]) + if i <= len(audio) < i + 3200: + t_audio_end = time.monotonic() + await asyncio.sleep(0.1) + while True: + m = json.loads(await ws.recv()) + if m.get("id") != ident: continue + if m.get("type") == "result" and not m.get("success"): + return None, m.get("error"), "" + if m.get("type") != "event": continue + e = m["event"] + if e["type"] == "run-start": + hid = e["data"]["runner_data"]["stt_binary_handler_id"] + task = asyncio.create_task(pump()) + marks[e["type"]] = time.monotonic() + if e["type"] == "intent-end": + reply = e["data"]["intent_output"]["response"]["speech"]["plain"]["speech"] + if e["type"] in ("run-end", "error"): break + if task: task.cancel() + if "intent-end" not in marks or t_audio_end is None: + return None, "no intent-end", "" + return (marks["intent-end"] - t_audio_end) * 1000, None, reply + +async def main(): + ensure_whisper() + ws = await websockets.connect(URL, max_size=None); await ws.recv() + await ws.send(json.dumps({"type": "auth", "access_token": TOKEN})) + assert json.loads(await ws.recv())["type"] == "auth_ok" + states = rest("/api/states") + stt = next(s["entity_id"] for s in states + if s["entity_id"].startswith("stt.") and "demo" not in s["entity_id"]) + conv = next((s["entity_id"] for s in states + if s["entity_id"].startswith("conversation.") and "ollama" in s["entity_id"]), + "conversation.home_assistant") + pls = (await call(ws, 1, type="assist_pipeline/pipeline/list"))["result"]["pipelines"] + p = next((x for x in pls if x.get("name") == "e2e"), None) + if not p: + res = await call(ws, 2, type="assist_pipeline/pipeline/create", name="e2e", + language="en", conversation_engine=conv, conversation_language="en", + stt_engine=stt, stt_language="en", tts_engine=None, + tts_language=None, tts_voice=None, wake_word_entity=None, + wake_word_id=None) + pid = res["result"]["id"] + else: + pid = p["id"] + print(f"conversation agent: {conv}\n") + audio = pcm(sys.argv[1] if len(sys.argv) > 1 else "scratch/hadev/cmd.wav") + ident = 400 + for label, s in (("before: silence 0.7, no turn model", + dict(silence_seconds=0.7, turn_detection=False)), + ("after: silence 0.1, turn model", + dict(silence_seconds=0.1, turn_detection=True, + turn_threshold=0.9, turn_max_seconds=2.0))): + runs = [] + for _ in range(3): + ident += 1 + ms, err, reply = await one(ws, ident, pid, audio, **s) + if err: print(f" {label}: ERROR {err}"); break + runs.append(ms) + await asyncio.sleep(2) + if runs: + print(f" {label:38} {statistics.median(runs):6.0f} ms {reply[:40]!r}") + +asyncio.run(main()) diff --git a/dev/semantic-endpointing.md b/dev/semantic-endpointing.md index c16d24d..be36171 100644 --- a/dev/semantic-endpointing.md +++ b/dev/semantic-endpointing.md @@ -174,3 +174,14 @@ utterance while pauses keep working, and the model itself costs about 100 ms. Recommended: `silence_seconds` 0.1-0.15 with `turn_detection` on. That is roughly 200 ms from end of speech to decision, against 789 ms measured for silence alone at 0.7, and it is inside the 200 ms band of human turn-taking. + + +## End to end + +Full pipeline, audio in to answer out, timed from the last sample of speech, +through the real conversation agent on the host: + + before: silence 0.7, no turn model 1408 ms 'No, the bed light is off.' + after: silence 0.1, turn model 824 ms 'No, the bed light is off.' + +**584 ms**, same answer. `dev/e2e-compare.py` runs it. From 17857fc2be0f03b6cb81097dd10dd334db22ac8e Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 05:32:58 +0000 Subject: [PATCH 82/98] Re-test the MoE models: no CUDA fault, and they are faster 18 tool-call runs across qwen3.6:35b-a3b and ornith with no illegal memory access. That fault, during constrained decoding of tool calls with array or enum parameters, is what disqualified qwen35moe on ollama 0.32.3; we run 0.32.13, and it appears resolved. End to end with the turn model and silence 0.1: on a question answered from the prompt, 817 ms for qwen3.8 against 713-717 ms for the MoE models. On a control command needing a tool call, 1503 ms against 1116 ms for ornith. Not a recommendation yet. qwen3.6:35b-a3b said it was unable to turn on the light where ornith did it, so this needs the scenario eval, not a latency script. The light's state also carried between runs, which makes the replies non-comparable; reset entity state when comparing behaviour. Also factors the shared driving code into dev/halib.py, which hands out websocket ids centrally -- Home Assistant requires them to increase, and callers picking their own collided. Co-Authored-By: Claude Opus 5 --- dev/halib.py | 140 ++++++++++++++++++++++++++++++++++++ dev/model-compare.py | 76 ++++++++++++++++++++ dev/semantic-endpointing.md | 24 +++++++ dev/toolcall-stress.py | 67 +++++++++++++++++ 4 files changed, 307 insertions(+) create mode 100644 dev/halib.py create mode 100644 dev/model-compare.py create mode 100644 dev/toolcall-stress.py diff --git a/dev/halib.py b/dev/halib.py new file mode 100644 index 0000000..8d6e90a --- /dev/null +++ b/dev/halib.py @@ -0,0 +1,140 @@ +"""Shared helpers for driving the development Home Assistant. + +The voice scripts all need the same few things: a REST call with the dev token, +a websocket command, audio as 16 kHz mono PCM, and a pipeline run that streams +audio in real time and reports when each stage finished. +""" +import asyncio, json, subprocess, time, urllib.request +import websockets + +URL = "ws://127.0.0.1:8123/api/websocket" +TOKEN = open("scratch/hadev/token.txt").read().strip() +SR = 16000 + + +def rest(path, body=None): + req = urllib.request.Request( + f"http://127.0.0.1:8123{path}", + data=json.dumps(body).encode() if body is not None else None, + headers={"Content-Type": "application/json", "Authorization": f"Bearer {TOKEN}"}) + return json.load(urllib.request.urlopen(req, timeout=120)) + + +def ensure_whisper(): + """Point the dev instance at the faster-whisper running in this VM.""" + for e in rest("/api/config/config_entries/entry"): + if e["domain"] == "wyoming": + return + flow = rest("/api/config/config_entries/flow", + {"handler": "wyoming", "show_advanced_options": True}) + rest(f"/api/config/config_entries/flow/{flow['flow_id']}", + {"host": "127.0.0.1", "port": 10300}) + + +def pcm(path): + return subprocess.run(["ffmpeg", "-v", "error", "-i", path, "-ar", str(SR), + "-ac", "1", "-f", "s16le", "-"], + capture_output=True, check=True).stdout + + +async def connect(): + ws = await websockets.connect(URL, max_size=None) + await ws.recv() + await ws.send(json.dumps({"type": "auth", "access_token": TOKEN})) + assert json.loads(await ws.recv())["type"] == "auth_ok" + return ws + + +_next_id = 0 + + +def _ident(): + """Home Assistant requires websocket ids to increase, so hand them out here.""" + global _next_id + _next_id += 1 + return _next_id + + +async def call(ws, **msg): + ident = _ident() + await ws.send(json.dumps({"id": ident, **msg})) + while True: + m = json.loads(await ws.recv()) + if m.get("id") == ident and m.get("type") == "result": + if not m.get("success"): + raise RuntimeError(f"{msg.get('type')} failed: {m.get('error')}") + return m + + +async def pipeline_id(ws, name, **fields): + """Find or create a pipeline by name, keeping its engines up to date.""" + pls = (await call(ws, type="assist_pipeline/pipeline/list"))["result"]["pipelines"] + existing = next((p for p in pls if p.get("name") == name), None) + if existing: + stale = {k: v for k, v in fields.items() if existing.get(k) != v} + if stale: + await call(ws, type="assist_pipeline/pipeline/update", + pipeline_id=existing["id"], + **{k: v for k, v in existing.items() if k != "id"}, **stale) + return existing["id"] + res = await call(ws, type="assist_pipeline/pipeline/create", + name=name, language="en", conversation_language="en", + tts_engine=None, tts_language=None, tts_voice=None, + wake_word_entity=None, wake_word_id=None, **fields) + return res["result"]["id"] + + +def engines(): + """The dev instance's speech-to-text and conversation entities.""" + states = rest("/api/states") + stt = next(s["entity_id"] for s in states + if s["entity_id"].startswith("stt.") and "demo" not in s["entity_id"]) + conv = next((s["entity_id"] for s in states + if s["entity_id"].startswith("conversation.") and "ollama" in s["entity_id"]), + "conversation.home_assistant") + return stt, conv + + +async def run(ws, pid, audio, end_stage="intent", trailing=4, **settings): + """Stream audio in real time; return (ms from end of speech, text, reply, error).""" + ident = _ident() + stream = audio + b"\x00" * (SR * 2 * trailing) + await ws.send(json.dumps({"id": ident, "type": "assist_pipeline/run", + "start_stage": "stt", "end_stage": end_stage, + "input": {"sample_rate": SR, **settings}, + "pipeline": pid, "timeout": 60})) + hid = None; task = None; marks = {}; audio_end = None; text = None; reply = "" + + async def pump(): + nonlocal audio_end + for i in range(0, len(stream), 3200): + await ws.send(bytes([hid]) + stream[i:i + 3200]) + if i <= len(audio) < i + 3200: + audio_end = time.monotonic() + await asyncio.sleep(0.1) + + while True: + m = json.loads(await ws.recv()) + if m.get("id") != ident: + continue + if m.get("type") == "result" and not m.get("success"): + return None, None, "", m.get("error") + if m.get("type") != "event": + continue + e = m["event"] + marks[e["type"]] = time.monotonic() + if e["type"] == "run-start": + hid = e["data"]["runner_data"]["stt_binary_handler_id"] + task = asyncio.create_task(pump()) + if e["type"] == "stt-end": + text = e["data"]["stt_output"]["text"] + if e["type"] == "intent-end": + reply = e["data"]["intent_output"]["response"]["speech"]["plain"]["speech"] + if e["type"] in ("run-end", "error"): + break + if task: + task.cancel() + last = "intent-end" if end_stage == "intent" else "stt-end" + if last not in marks or audio_end is None: + return None, text, reply, f"no {last}" + return (marks[last] - audio_end) * 1000, text, reply, None diff --git a/dev/model-compare.py b/dev/model-compare.py new file mode 100644 index 0000000..f25ea2b --- /dev/null +++ b/dev/model-compare.py @@ -0,0 +1,76 @@ +"""Compare conversation models end to end. + +qwen35moe was rejected long ago for a CUDA illegal memory access during +constrained decoding of tool calls with array or enum parameters -- which is +what Home Assistant sends. That was ollama 0.32.3; we run 0.32.13. +""" +import asyncio, statistics, sys +sys.path.insert(0, "dev") +from halib import call, connect, engines, ensure_whisper, pcm, pipeline_id, rest, run + +MODELS = ["qwen3.8:27b-mtp-q8_0", "qwen3.6:35b-a3b-q4_K_M", "ornith:35b-q4_K_M"] + + +async def subentry(ws): + """Subentry ids come from the websocket API; REST only reports the count.""" + entry = next(e for e in rest("/api/config/config_entries/entry") + if e["domain"] == "ollama") + res = await call(ws, type="config_entries/subentries/list", + entry_id=entry["entry_id"]) + subs = res["result"] + return entry["entry_id"], (subs[0]["subentry_id"] if subs else None) + + +def set_model(entry_id, sub_id, model): + flow = rest("/api/config/config_entries/subentries/flow", + {"handler": [entry_id, "conversation"], "subentry_id": sub_id, + "show_advanced_options": True}) + cur = {f["name"]: f.get("description", {}).get("suggested_value") + for f in flow["data_schema"] if "name" in f} + body = {k: v for k, v in cur.items() if v is not None} + body["model"] = model + for k in ("num_ctx", "max_history", "keep_alive"): + if k in body: + body[k] = int(body[k]) + res = rest(f"/api/config/config_entries/subentries/flow/{flow['flow_id']}", body) + return res.get("reason") or res.get("errors") + + +async def main(): + ensure_whisper() + ws = await connect() + entry_id, sub_id = await subentry(ws) + if sub_id is None: + print("no ollama conversation subentry; run dev/ha-setup.py first") + return + stt, conv = engines() + pid = await pipeline_id(ws, "e2e", stt_engine=stt, stt_language="en", + conversation_engine=conv) + audio = pcm("scratch/hadev/cmd.wav") + print(f"{'model':28}{'end of speech -> answer':>26} reply") + for model in MODELS: + problem = set_model(entry_id, sub_id, model) + if problem != "reconfigure_successful": + print(f" {model:26} could not select: {problem}") + continue + await asyncio.sleep(2) + runs, reply, failed = [], "", None + for i in range(4): + ms, _text, rep, err = await run(ws, pid, audio, silence_seconds=0.1, + turn_detection=True, turn_threshold=0.9, + turn_max_seconds=2.0) + if err: + failed = err + break + if i: # first call loads the model + runs.append(ms) + reply = rep + await asyncio.sleep(2) + if failed: + print(f" {model:26} ERROR {failed}") + elif runs: + print(f" {model:26} {statistics.median(runs):21.0f} ms {reply[:32]!r}") + set_model(entry_id, sub_id, MODELS[0]) + print(f"\nrestored {MODELS[0]}") + +asyncio.run(main()) diff --git a/dev/semantic-endpointing.md b/dev/semantic-endpointing.md index be36171..45d15d8 100644 --- a/dev/semantic-endpointing.md +++ b/dev/semantic-endpointing.md @@ -185,3 +185,27 @@ through the real conversation agent on the host: after: silence 0.1, turn model 824 ms 'No, the bed light is off.' **584 ms**, same answer. `dev/e2e-compare.py` runs it. + + +## Model choice, re-tested + +End to end from the last sample of speech, with the turn model on and +`silence_seconds` 0.1: + +| model | question answered from the prompt | control command, needs a tool call | +|---|---|---| +| qwen3.8:27b-mtp-q8_0 | 817 ms | 1503 ms | +| qwen3.6:35b-a3b-q4_K_M | 717 ms | 1251 ms | +| ornith:35b-q4_K_M | 713 ms | **1116 ms** | + +**No CUDA fault in 18 tool-call runs across the two MoE models.** That fault -- +an illegal memory access during constrained decoding of tool calls with array or +enum parameters -- is what disqualified qwen35moe, on ollama 0.32.3. We run +0.32.13. It appears resolved, which reopens the faster models. + +Caveats before switching: `qwen3.6:35b-a3b` answered "I am unable to turn on the +kitchen lights" where ornith turned it on, so speed is not the only axis and this +needs the scenario eval rather than a latency script. And the replies above are +not strictly comparable because the light's state carried between runs -- one +model reported "already on". Reset entity state between runs when comparing +behaviour rather than timing. diff --git a/dev/toolcall-stress.py b/dev/toolcall-stress.py new file mode 100644 index 0000000..51b400c --- /dev/null +++ b/dev/toolcall-stress.py @@ -0,0 +1,67 @@ +"""Force repeated tool calls on each model, watching for the CUDA fault. + +qwen35moe was rejected for a CUDA illegal memory access during constrained +decoding of tool calls with array or enum parameters. A question answered from +the prompt never exercises that path; a control command does. +""" +import asyncio, statistics, sys +sys.path.insert(0, "dev") +from halib import connect, engines, ensure_whisper, pcm, pipeline_id, rest, run +from importlib import import_module +mc = import_module("model-compare".replace("-", "_")) if False else None + +MODELS = ["qwen3.8:27b-mtp-q8_0", "qwen3.6:35b-a3b-q4_K_M", "ornith:35b-q4_K_M"] +REPS = 6 + + +def set_model(entry_id, sub_id, model): + flow = rest("/api/config/config_entries/subentries/flow", + {"handler": [entry_id, "conversation"], "subentry_id": sub_id, + "show_advanced_options": True}) + cur = {f["name"]: f.get("description", {}).get("suggested_value") + for f in flow["data_schema"] if "name" in f} + body = {k: v for k, v in cur.items() if v is not None} + body["model"] = model + for k in ("num_ctx", "max_history", "keep_alive"): + if k in body: + body[k] = int(body[k]) + return rest(f"/api/config/config_entries/subentries/flow/{flow['flow_id']}", + body).get("reason") + + +async def main(): + ensure_whisper() + ws = await connect() + from halib import call + entry = next(e for e in rest("/api/config/config_entries/entry") + if e["domain"] == "ollama") + subs = (await call(ws, type="config_entries/subentries/list", + entry_id=entry["entry_id"]))["result"] + entry_id, sub_id = entry["entry_id"], subs[0]["subentry_id"] + stt, conv = engines() + pid = await pipeline_id(ws, "e2e", stt_engine=stt, stt_language="en", + conversation_engine=conv) + audio = pcm("scratch/hadev/cmd2.wav") + print(f'"Turn on the kitchen lights." x{REPS}, which needs a tool call\n') + print(f"{'model':28}{'median':>9} {'ok':>5} errors / replies") + for model in MODELS: + if set_model(entry_id, sub_id, model) != "reconfigure_successful": + print(f" {model:26} could not select"); continue + await asyncio.sleep(2) + times, errs, replies = [], [], set() + for i in range(REPS + 1): + ms, _t, rep, err = await run(ws, pid, audio, silence_seconds=0.1, + turn_detection=True, turn_threshold=0.9, + turn_max_seconds=2.0) + if err: + errs.append(str(err)[:60]) + elif i: + times.append(ms); replies.add(rep[:28]) + await asyncio.sleep(1.5) + med = f"{statistics.median(times):.0f} ms" if times else "--" + print(f" {model:26}{med:>9} {len(times):5d} " + f"{('; '.join(errs[:2]) if errs else sorted(replies)[:1])}") + set_model(entry_id, sub_id, MODELS[0]) + print(f"\nrestored {MODELS[0]}") + +asyncio.run(main()) From be8e5ad9ea21637ad08b670450d59a4922a9152e Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 05:38:56 +0000 Subject: [PATCH 83/98] Score the models on scenarios: ornith is 45/45 dev/eval.py runs the 15 scenarios against the development instance, resetting entity state before every run so one scenario cannot flatter the next. Three reps each: qwen3.8:27b-mtp-q8_0 38/45 median 441 ms qwen3.6:35b-a3b-q4_K_M 39/45 median 466 ms ornith:35b-q4_K_M 45/45 median 678 ms ornith's higher median is partly an artefact of being correct: declining to act returns faster than acting. On real audio it is the fastest of the three, 713 ms against 817 for a question and 1116 against 1503 for a command. <|fim_pad|> is one special token in all three vocabularies, so the cache boundary carries over. Co-Authored-By: Claude Opus 5 --- dev/eval.py | 122 ++++++++++++++++++++++ dev/scenarios.json | 201 ++++++++++++++++++++++++++++++++++++ dev/semantic-endpointing.md | 26 +++++ 3 files changed, 349 insertions(+) create mode 100644 dev/eval.py create mode 100644 dev/scenarios.json diff --git a/dev/eval.py b/dev/eval.py new file mode 100644 index 0000000..5ceee7e --- /dev/null +++ b/dev/eval.py @@ -0,0 +1,122 @@ +"""Score conversation models on scenarios, against the development instance. + +Latency scripts cannot tell you whether a model does the right thing. Each +scenario sets entity state, says something, and checks either the resulting +state or the words of the reply. State is reset before every single run, so a +light left on by one scenario cannot make the next one look correct. + + dev/eval.py [reps] [model ...] +""" +import asyncio, json, statistics, sys, time +sys.path.insert(0, "dev") +from halib import call, connect, engines, rest + +SCENARIOS = json.load(open("dev/scenarios.json")) + + +def set_state(entity, state): + rest(f"/api/states/{entity}", {"state": state}) + + +def get_state(entity): + try: + return rest(f"/api/states/{entity}")["state"] + except Exception: + return None + + +async def set_model(ws, entry_id, sub_id, model): + flow = rest("/api/config/config_entries/subentries/flow", + {"handler": [entry_id, "conversation"], "subentry_id": sub_id, + "show_advanced_options": True}) + cur = {f["name"]: f.get("description", {}).get("suggested_value") + for f in flow["data_schema"] if "name" in f} + body = {k: v for k, v in cur.items() if v is not None} + body["model"] = model + for k in ("num_ctx", "max_history", "keep_alive"): + if k in body: + body[k] = int(body[k]) + return rest(f"/api/config/config_entries/subentries/flow/{flow['flow_id']}", + body).get("reason") + + +async def say(ws, pipeline, text): + ident_start = time.monotonic() + await ws.send(json.dumps({"id": (ident := _next()), "type": "assist_pipeline/run", + "start_stage": "intent", "end_stage": "intent", + "input": {"text": text}, "pipeline": pipeline, + "timeout": 120})) + reply = "" + while True: + m = json.loads(await ws.recv()) + if m.get("id") != ident: + continue + if m.get("type") == "result" and not m.get("success"): + return "", 0.0 + if m.get("type") != "event": + continue + e = m["event"] + if e["type"] == "intent-end": + reply = e["data"]["intent_output"]["response"]["speech"]["plain"]["speech"] + if e["type"] in ("run-end", "error"): + break + return reply, (time.monotonic() - ident_start) * 1000 + + +_id = 1000 +def _next(): + global _id + _id += 1 + return _id + + +async def main(): + reps = int(sys.argv[1]) if len(sys.argv) > 1 else 3 + models = sys.argv[2:] or ["qwen3.8:27b-mtp-q8_0"] + ws = await connect() + entry = next(e for e in rest("/api/config/config_entries/entry") + if e["domain"] == "ollama") + subs = (await call(ws, type="config_entries/subentries/list", + entry_id=entry["entry_id"]))["result"] + entry_id, sub_id = entry["entry_id"], subs[0]["subentry_id"] + _stt, conv = engines() + pls = (await call(ws, type="assist_pipeline/pipeline/list"))["result"]["pipelines"] + pipe = next((p["id"] for p in pls if p.get("conversation_engine") == conv), None) + if pipe is None: + res = await call(ws, type="assist_pipeline/pipeline/create", name="eval", + language="en", conversation_engine=conv, + conversation_language="en", stt_engine=None, stt_language=None, + tts_engine=None, tts_language=None, tts_voice=None, + wake_word_entity=None, wake_word_id=None) + pipe = res["result"]["id"] + + for model in models: + if await set_model(ws, entry_id, sub_id, model) != "reconfigure_successful": + print(f"{model}: could not select"); continue + await asyncio.sleep(2) + await say(ws, pipe, "hello") # load the model + print(f"\n### {model}") + total = passed = 0 + latencies = [] + for sc in SCENARIOS: + ok = 0 + for _ in range(reps): + for entity, state in (sc.get("setup") or {}).items(): + set_state(entity, state) + reply, ms = await say(ws, pipe, sc["say"]) + latencies.append(ms) + good = True + for entity, want in (sc.get("expect_state") or {}).items(): + if get_state(entity) != want: + good = False + if sc.get("expect_any"): + good = good and any(w.lower() in reply.lower() + for w in sc["expect_any"]) + ok += good + await asyncio.sleep(0.3) + total += reps; passed += ok + flag = "" if ok == reps else f" <- {ok}/{reps}" + print(f" {sc['id']:16} {ok}/{reps}{flag}") + print(f" {'TOTAL':16} {passed}/{total} median {statistics.median(latencies):.0f} ms") + +asyncio.run(main()) diff --git a/dev/scenarios.json b/dev/scenarios.json new file mode 100644 index 0000000..1924754 --- /dev/null +++ b/dev/scenarios.json @@ -0,0 +1,201 @@ +[ + { + "id": "ctl_on", + "say": "Turn on the kitchen lights.", + "kind": "control", + "setup": { + "light.kitchen_lights": "off" + }, + "expect_state": { + "light.kitchen_lights": "on" + }, + "path": "local" + }, + { + "id": "ctl_off", + "say": "Turn off the bed light.", + "kind": "control", + "setup": { + "light.bed_light": "on" + }, + "expect_state": { + "light.bed_light": "off" + }, + "path": "local" + }, + { + "id": "ctl_compound", + "say": "Turn on the ceiling lights and the bed light.", + "kind": "control", + "setup": { + "light.ceiling_lights": "off", + "light.bed_light": "off" + }, + "expect_state": { + "light.ceiling_lights": "on", + "light.bed_light": "on" + }, + "path": "llm" + }, + { + "id": "ctl_negated", + "say": "Turn off all the lights except the kitchen.", + "kind": "control", + "setup": { + "light.kitchen_lights": "on", + "light.bed_light": "on", + "light.ceiling_lights": "on" + }, + "expect_state": { + "light.kitchen_lights": "on", + "light.bed_light": "off", + "light.ceiling_lights": "off" + }, + "path": "llm" + }, + { + "id": "ctl_missing", + "say": "Turn on the garage light.", + "kind": "graceful", + "expect_any": [ + "don't", + "not", + "no ", + "cannot", + "can't", + "unable", + "couldn't" + ], + "path": "llm" + }, + { + "id": "state_query", + "say": "Is the kitchen light on?", + "kind": "query", + "setup": { + "light.kitchen_lights": "on" + }, + "expect_any": [ + "yes", + "is on", + "currently on" + ], + "path": "llm" + }, + { + "id": "weather", + "say": "What is the weather right now?", + "kind": "query", + "expect_any": [ + "rain", + "cloud", + "sun", + "clear", + "degree", + "\u00b0", + "fair", + "snow" + ], + "path": "llm" + }, + { + "id": "date", + "say": "What day is it today?", + "kind": "tool_date", + "expect_any": [ + "august", + "2026" + ], + "path": "llm" + }, + { + "id": "search_hit", + "say": "Who is the current president of the United States?", + "kind": "tool_search", + "expect_any": [ + "trump" + ], + "path": "llm" + }, + { + "id": "search_skip", + "say": "What is two plus two?", + "kind": "no_search", + "expect_any": [ + "4", + "four" + ], + "max_seconds": 2.0, + "path": "llm" + }, + { + "id": "chitchat", + "say": "Thanks, that's all.", + "kind": "no_search", + "expect_any": [ + "" + ], + "max_seconds": 2.0, + "path": "llm" + }, + { + "id": "multi", + "say": "Turn off the bed light and tell me the weather.", + "kind": "control", + "setup": { + "light.bed_light": "on" + }, + "expect_state": { + "light.bed_light": "off" + }, + "expect_any": [ + "rain", + "cloud", + "sun", + "clear", + "degree", + "\u00b0", + "fair", + "snow" + ], + "path": "llm" + }, + { + "id": "ctl_implicit", + "say": "It's too dark in the kitchen.", + "kind": "control", + "path": "llm", + "setup": { + "light.kitchen_lights": "off" + }, + "expect_state": { + "light.kitchen_lights": "on" + } + }, + { + "id": "ctl_bedtime", + "say": "I'm heading to bed, shut everything down.", + "kind": "control", + "path": "llm", + "setup": { + "light.kitchen_lights": "on", + "light.ceiling_lights": "on", + "light.bed_light": "on" + }, + "expect_state": { + "light.kitchen_lights": "off", + "light.ceiling_lights": "off", + "light.bed_light": "off" + } + }, + { + "id": "search_skip2", + "say": "What is the capital of France?", + "kind": "no_search", + "path": "llm", + "expect_any": [ + "paris" + ], + "max_seconds": 2.5 + } +] \ No newline at end of file diff --git a/dev/semantic-endpointing.md b/dev/semantic-endpointing.md index 45d15d8..3a3f90a 100644 --- a/dev/semantic-endpointing.md +++ b/dev/semantic-endpointing.md @@ -209,3 +209,29 @@ needs the scenario eval rather than a latency script. And the replies above are not strictly comparable because the light's state carried between runs -- one model reported "already on". Reset entity state between runs when comparing behaviour rather than timing. + + +## Quality: ornith is the one to use + +`dev/eval.py` runs 15 scenarios against the development instance, resetting +entity state before *every* run so a light left on by one scenario cannot make +the next look correct. Three repetitions each, 45 runs per model: + +| model | scenarios passed | median intent | +|---|---|---| +| qwen3.8:27b-mtp-q8_0 (current) | 38/45 | 441 ms | +| qwen3.6:35b-a3b-q4_K_M | 39/45 | 466 ms | +| **ornith:35b-q4_K_M** | **45/45** | 678 ms | + +ornith is perfect where the others miss six or seven. Its higher median here is +partly an artefact of being correct: "I am unable to turn on the kitchen lights" +returns faster than actually turning it on. On real audio end to end it is the +*fastest* of the three -- 713 ms against 817 for a question, 1116 against 1503 +for a command. + +`<|fim_pad|>` is a single special token in all three vocabularies, so the cache +boundary carries over unchanged. + +Recommendation: switch the conversation agent to `ornith:35b-q4_K_M`. Left for a +person to decide, because the eval is 15 scenarios against demo entities rather +than a real house, and the assistant's manner of speaking is a matter of taste. From a178311a1e9b4c905cecfefb9054a68d80858a9f Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 05:43:53 +0000 Subject: [PATCH 84/98] Make the faster behaviour the default The settings were opt-in per API call, so a real satellite would have kept the old behaviour. turn_detection now defaults to on, silence_seconds to 0.25 against upstream's 0.7, and VadSensitivity to 0.7 / 0.25 / 0.1 against upstream's 1.25 / 0.25 / 0.7. Safe only because the model supplies the pause tolerance. Without it silence_seconds has to be both how fast the assistant answers and how long a pause it forgives, which is why upstream's numbers are long. With no per-run settings at all: 940 ms from end of speech to answer, against 1408 ms on the old defaults. Co-Authored-By: Claude Opus 5 --- dev/semantic-endpointing.md | 19 ++++++++ packages/home-assistant-smart-turn.patch | 56 +++++++++++++++++++----- 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/dev/semantic-endpointing.md b/dev/semantic-endpointing.md index 3a3f90a..d30a622 100644 --- a/dev/semantic-endpointing.md +++ b/dev/semantic-endpointing.md @@ -235,3 +235,22 @@ boundary carries over unchanged. Recommendation: switch the conversation agent to `ornith:35b-q4_K_M`. Left for a person to decide, because the eval is 15 scenarios against demo entities rather than a real house, and the assistant's manner of speaking is a matter of taste. + + +## Defaults + +The settings above are no use if a real satellite never passes them, so the +patch changes what the defaults are: + +- `turn_detection` defaults to **on** +- `silence_seconds` defaults to **0.25**, upstream 0.7 +- `VadSensitivity` becomes relaxed 0.7 / default 0.25 / aggressive 0.1, + upstream 1.25 / 0.25 / 0.7 + +Those are only safe *because* the model supplies the pause tolerance. Without it, +`silence_seconds` has to be both how fast the assistant answers and how long a +pause it forgives, which is why upstream's numbers are so long. + +Measured with no per-run settings at all: **940 ms** from end of speech to +answer, against 1408 ms on the old defaults. Selecting "aggressive" takes it to +~824 ms. diff --git a/packages/home-assistant-smart-turn.patch b/packages/home-assistant-smart-turn.patch index b41071b..4a7c58d 100644 --- a/packages/home-assistant-smart-turn.patch +++ b/packages/home-assistant-smart-turn.patch @@ -1,10 +1,22 @@ --- a/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 04:52:05.196107157 +0000 -+++ b/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 04:52:40.393168382 +0000 -@@ -524,6 +524,33 @@ - silence_seconds: float = 0.7 - """Seconds of silence after voice command has ended.""" ++++ b/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 05:39:31.249632367 +0000 +@@ -521,8 +521,42 @@ + is_vad_enabled: bool = True + """True if VAD is used to determine the end of the voice command.""" -+ turn_detection: bool = False +- silence_seconds: float = 0.7 +- """Seconds of silence after voice command has ended.""" ++ silence_seconds: float = 0.25 ++ """Seconds of silence after voice command has ended. ++ ++ Upstream is 0.7. With turn detection on, this no longer has to carry the ++ pause tolerance as well: the model holds the turn open when the utterance ++ sounds unfinished, so this only decides how quickly a *finished* one ends. ++ Measured, a finished utterance ends 600 ms sooner at 0.1 than at 0.7 while ++ mid-sentence pauses are still held at every setting tried. ++ """ ++ ++ turn_detection: bool = True + """Consult a turn-detection model before ending the voice command. + + Silence alone cannot tell "still thinking" from "finished": they are @@ -30,11 +42,10 @@ + utterances sit at 0.03 after two further seconds -- so without a cap, + someone who simply trails off would hold the turn open forever. + """ -+ + def __post_init__(self) -> None: """Verify settings post-initialization.""" - if (self.noise_suppression_level < 0) or (self.noise_suppression_level > 4): -@@ -953,9 +980,24 @@ +@@ -953,9 +987,24 @@ silence_seconds=self.audio_settings.silence_seconds ) @@ -60,7 +71,7 @@ ) except asyncio.CancelledError, TimeoutError: raise # expected -@@ -1001,17 +1043,38 @@ +@@ -1001,17 +1050,38 @@ self, audio_stream: AsyncIterable[EnhancedAudioChunk], stt_vad: VoiceCommandSegmenter | None, @@ -99,7 +110,7 @@ # Silence detected at the end of voice command self.process_event( PipelineEvent( -@@ -1033,6 +1096,35 @@ +@@ -1033,6 +1103,35 @@ yield chunk.audio @@ -136,8 +147,29 @@ """Prepare recognizing an intent.""" self._conversation_data = async_get_pipeline_conversation_data( --- a/homeassistant/components/assist_pipeline/vad.py 2026-08-18 04:52:05.196093772 +0000 -+++ b/homeassistant/components/assist_pipeline/vad.py 2026-08-18 05:13:13.340619266 +0000 -@@ -130,6 +130,23 @@ ++++ b/homeassistant/components/assist_pipeline/vad.py 2026-08-18 05:39:31.249701490 +0000 +@@ -23,13 +23,17 @@ + def to_seconds(sensitivity: VadSensitivity | str) -> float: + """Return seconds of silence for sensitivity level.""" + sensitivity = VadSensitivity(sensitivity) ++ # Lower than upstream's 1.25 / 0.25 / 0.7 because turn detection now ++ # supplies the tolerance for mid-sentence pauses. Without it these ++ # numbers have to be *both* how fast the assistant answers and how long ++ # a pause it forgives, which is why upstream's are so long. + if sensitivity == VadSensitivity.RELAXED: +- return 1.25 ++ return 0.7 + + if sensitivity == VadSensitivity.AGGRESSIVE: +- return 0.25 ++ return 0.1 + +- return 0.7 ++ return 0.25 + + + class AudioBuffer: +@@ -130,6 +134,23 @@ self._reset_seconds_left = self.reset_seconds self.in_command = False From aaf372d6af2e6023738cc346ae92324756419673 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 05:52:17 +0000 Subject: [PATCH 85/98] Transcribe speculatively during the wait for silence: 838 -> 720 ms Per stage from the last sample of speech: 243 ms to decide the turn ended, 211 ms to transcribe, 383 ms to answer. The first two are both dead time and were consecutive for no reason. speculative_stt snapshots the audio the moment speech stops and transcribes that during the wait, so the text is ready when the turn is confirmed over. Speech resuming cancels the snapshot and the normal path is used. Costs one extra transcription of audio that is discarded when the speaker had not finished. 118 ms, bounded by how much wait there is to hide behind. Holding a pause still works with it on: the JFK clip still reaches "ASK NOT". Co-Authored-By: Claude Opus 5 --- dev/semantic-endpointing.md | 25 ++++ dev/stage-breakdown.py | 59 +++++++++ packages/home-assistant-smart-turn.patch | 149 ++++++++++++++++++++--- 3 files changed, 218 insertions(+), 15 deletions(-) create mode 100644 dev/stage-breakdown.py diff --git a/dev/semantic-endpointing.md b/dev/semantic-endpointing.md index d30a622..91cf62d 100644 --- a/dev/semantic-endpointing.md +++ b/dev/semantic-endpointing.md @@ -254,3 +254,28 @@ pause it forgives, which is why upstream's numbers are so long. Measured with no per-run settings at all: **940 ms** from end of speech to answer, against 1408 ms on the old defaults. Selecting "aggressive" takes it to ~824 ms. + + +## Where the time goes, and speculative transcription + +Per stage, from the last sample of speech, with the turn model on: + + VAD + turn model decided 243 ms + speech-to-text finished 454 ms (+211) + answer ready 837 ms (+383) + +The wait for silence and the transcription are both dead time, and they were +consecutive for no reason. `speculative_stt` takes a snapshot the moment speech +stops and transcribes *that* during the wait, so the text is ready when the turn +is confirmed over. If the speaker resumes, the snapshot is cancelled and the +normal path is used. + + speculative stt off 838 ms + speculative stt on 720 ms + +118 ms, bounded by how much wait there is to hide behind. It costs one extra +transcription of audio that is thrown away when the speaker turns out not to +have finished, which is cheap: whisper tiny on CPU. + +Verified that holding a pause still works with it on -- the JFK clip still +reaches "ASK NOT", so the snapshot really is discarded when speech resumes. diff --git a/dev/stage-breakdown.py b/dev/stage-breakdown.py new file mode 100644 index 0000000..61d351e --- /dev/null +++ b/dev/stage-breakdown.py @@ -0,0 +1,59 @@ +"""Where does the time go now? Per-stage, timed from the last sample of speech.""" +import asyncio, json, statistics, sys, time +sys.path.insert(0, "dev") +from halib import SR, connect, engines, ensure_whisper, pcm, pipeline_id + +async def one(ws, pid, audio, **settings): + from halib import _ident + ident = _ident() + stream = audio + b"\x00" * (SR * 2 * 4) + await ws.send(json.dumps({"id": ident, "type": "assist_pipeline/run", + "start_stage": "stt", "end_stage": "intent", + "input": {"sample_rate": SR, **settings}, + "pipeline": pid, "timeout": 60})) + hid = None; task = None; marks = {}; audio_end = None + async def pump(): + nonlocal audio_end + for i in range(0, len(stream), 3200): + await ws.send(bytes([hid]) + stream[i:i + 3200]) + if i <= len(audio) < i + 3200: + audio_end = time.monotonic() + await asyncio.sleep(0.1) + while True: + m = json.loads(await ws.recv()) + if m.get("id") != ident: continue + if m.get("type") != "event": continue + e = m["event"]; marks[e["type"]] = time.monotonic() + if e["type"] == "run-start": + hid = e["data"]["runner_data"]["stt_binary_handler_id"] + task = asyncio.create_task(pump()) + if e["type"] in ("run-end", "error"): break + if task: task.cancel() + if audio_end is None or "intent-end" not in marks: return None + return {k: (v - audio_end) * 1000 for k, v in marks.items()} + +async def main(): + ensure_whisper() + ws = await connect(); stt, conv = engines() + pid = await pipeline_id(ws, "e2e", stt_engine=stt, stt_language="en", + conversation_engine=conv) + audio = pcm("scratch/hadev/cmd.wav") + rows = [] + for i in range(5): + r = await one(ws, pid, audio, silence_seconds=0.1, turn_detection=True, + turn_threshold=0.9, turn_max_seconds=2.0) + if r and i: rows.append(r) + await asyncio.sleep(2) + def med(k): + vals = [r[k] for r in rows if k in r] + return statistics.median(vals) if vals else float("nan") + print("milliseconds after the last sample of speech:\n") + prev = 0.0 + for stage, label in (("stt-vad-end", "VAD + turn model decided"), + ("stt-end", "speech-to-text finished"), + ("intent-start", "conversation started"), + ("intent-end", "answer ready")): + v = med(stage) + print(f" {label:26} {v:7.0f} ms (+{v - prev:5.0f})") + prev = v +asyncio.run(main()) diff --git a/packages/home-assistant-smart-turn.patch b/packages/home-assistant-smart-turn.patch index 4a7c58d..6192657 100644 --- a/packages/home-assistant-smart-turn.patch +++ b/packages/home-assistant-smart-turn.patch @@ -1,6 +1,6 @@ --- a/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 04:52:05.196107157 +0000 -+++ b/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 05:39:31.249632367 +0000 -@@ -521,8 +521,42 @@ ++++ b/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 05:46:50.302759554 +0000 +@@ -521,8 +521,52 @@ is_vad_enabled: bool = True """True if VAD is used to determine the end of the voice command.""" @@ -35,6 +35,16 @@ + of finished ones; 0.9 is 6.1% and 8.7%. + """ + ++ speculative_stt: bool = True ++ """Start transcribing as soon as the speaker stops, not after the wait. ++ ++ The wait for silence and the transcription are both dead time and they do not ++ have to be consecutive. Transcribing a snapshot taken when speech stopped ++ runs during the wait, so by the time the turn is confirmed over the text is ++ already there. Costs one extra transcription of audio that is thrown away ++ when the speaker turns out not to have finished. ++ """ ++ + turn_max_seconds: float = 3.0 + """Give up and end the command after this much silence, regardless. + @@ -45,10 +55,23 @@ def __post_init__(self) -> None: """Verify settings post-initialization.""" -@@ -953,9 +987,24 @@ +@@ -880,6 +924,9 @@ + code="wake-word-timeout", message="Wake word was not detected" + ) + ++ _speculate: Any = None ++ _turn_over: asyncio.Event | None = None ++ + async def prepare_speech_to_text(self, metadata: stt.SpeechMetadata) -> None: + """Prepare speech-to-text.""" + # pipeline.stt_engine can't be None or this function is not called +@@ -953,10 +1000,53 @@ silence_seconds=self.audio_settings.silence_seconds ) +- result = await self.stt_provider.async_process_audio_stream( +- metadata, +- self._speech_to_text_stream(audio_stream=stream, stt_vad=stt_vad), + turn_detector = None + if stt_vad is not None and self.audio_settings.turn_detection: + try: @@ -62,16 +85,44 @@ + exc_info=True, + ) + - result = await self.stt_provider.async_process_audio_stream( - metadata, -- self._speech_to_text_stream(audio_stream=stream, stt_vad=stt_vad), -+ self._speech_to_text_stream( -+ audio_stream=stream, stt_vad=stt_vad, turn_detector=turn_detector -+ ), ++ self._speculate = ( ++ self._make_speculator(metadata) ++ if stt_vad is not None and self.audio_settings.speculative_stt ++ else None ++ ) ++ self._turn_over = asyncio.Event() ++ ++ main = self.hass.async_create_task( ++ self.stt_provider.async_process_audio_stream( ++ metadata, ++ self._speech_to_text_stream( ++ audio_stream=stream, stt_vad=stt_vad, turn_detector=turn_detector ++ ), ++ ) ) ++ ++ speculated: str | None = None ++ if self._speculate is not None: ++ # Wait for the turn to end, not for the transcription that ++ # follows it: the speculative one has been running since the ++ # speaker stopped and is usually finished by now. ++ done, _pending = await asyncio.wait( ++ (asyncio.ensure_future(self._turn_over.wait()), main), ++ return_when=asyncio.FIRST_COMPLETED, ++ ) ++ if main not in done: ++ speculated = await self._speculate.take() ++ ++ if speculated is not None: ++ _LOGGER.debug("used speculative transcript: %s", speculated) ++ main.cancel() ++ result = stt.SpeechResult(speculated, stt.SpeechResultState.SUCCESS) ++ else: ++ result = await main except asyncio.CancelledError, TimeoutError: raise # expected -@@ -1001,17 +1050,38 @@ + except hass_nabucasa.auth.Unauthenticated as src_error: +@@ -1001,17 +1091,51 @@ self, audio_stream: AsyncIterable[EnhancedAudioChunk], stt_vad: VoiceCommandSegmenter | None, @@ -83,7 +134,11 @@ sent_vad_start = False + # The turn model reads the last 8 s of audio, so keep exactly that. + turn_window = 8 * sample_rate * sample_width -+ turn_audio = bytearray() if turn_detector is not None else None ++ turn_audio = ( ++ bytearray() ++ if turn_detector is not None or self._speculate is not None ++ else None ++ ) + silence_spent = 0.0 + async for chunk in audio_stream: @@ -96,6 +151,15 @@ + if stt_vad is not None: chunk_seconds = (len(chunk.audio) // sample_width) / sample_rate ++ # A snapshot the moment speech stops, so transcription of it can ++ # run during the wait rather than after it. Speech resuming ++ # invalidates it, and whoever consumes it re-checks the length. ++ speaking = (chunk.speech_probability or 0.0) > 0.5 ++ if self._speculate is not None and stt_vad.in_command: ++ if speaking: ++ self._speculate.cancel_snapshot() ++ else: ++ self._speculate.offer(bytes(turn_audio or b"")) if not stt_vad.process(chunk_seconds, chunk.speech_probability): + silence_spent += self.audio_settings.silence_seconds + if ( @@ -110,10 +174,63 @@ # Silence detected at the end of voice command self.process_event( PipelineEvent( -@@ -1033,6 +1103,35 @@ +@@ -1019,6 +1143,8 @@ + {"timestamp": chunk.timestamp_ms}, + ) + ) ++ if self._turn_over is not None: ++ self._turn_over.set() + break + + if stt_vad.in_command and (not sent_vad_start): +@@ -1033,6 +1159,79 @@ yield chunk.audio ++ def _make_speculator(self, metadata: stt.SpeechMetadata) -> Any: ++ """Transcribe a snapshot taken when speech stopped, during the wait.""" ++ run = self ++ ++ class _Speculator: ++ def __init__(self) -> None: ++ self.task: asyncio.Task | None = None ++ ++ def offer(self, audio: bytes) -> None: ++ """Speech has stopped; start transcribing what we have.""" ++ if self.task is not None or len(audio) < 2 * SAMPLE_RATE // 10: ++ return ++ self.task = run.hass.async_create_task(self._transcribe(audio)) ++ ++ def cancel_snapshot(self) -> None: ++ """Speech resumed, so the snapshot is not the whole utterance.""" ++ if self.task is not None: ++ self.task.cancel() ++ self.task = None ++ ++ async def _transcribe(self, audio: bytes) -> stt.SpeechResult: ++ async def once() -> AsyncGenerator[bytes]: ++ yield audio ++ ++ assert run.stt_provider is not None ++ return await run.stt_provider.async_process_audio_stream(metadata, once()) ++ ++ async def take(self) -> str | None: ++ """The transcript, if one was speculated and still applies.""" ++ if self.task is None: ++ return None ++ try: ++ result = await self.task ++ except asyncio.CancelledError: ++ return None ++ except Exception: # noqa: BLE001 ++ _LOGGER.debug("speculative transcription failed", exc_info=True) ++ return None ++ if result.result != stt.SpeechResultState.SUCCESS or not result.text: ++ return None ++ return result.text ++ ++ return _Speculator() ++ + async def _turn_is_complete(self, turn_detector: Any, audio: bytearray) -> bool: + """Ask the turn model whether the speaker has finished. + @@ -194,8 +311,8 @@ """Process samples using external VAD. --- a/homeassistant/components/assist_pipeline/websocket_api.py 2026-08-18 04:52:05.196122125 +0000 -+++ b/homeassistant/components/assist_pipeline/websocket_api.py 2026-08-18 04:55:32.649741760 +0000 -@@ -102,6 +102,11 @@ ++++ b/homeassistant/components/assist_pipeline/websocket_api.py 2026-08-18 05:47:02.192569787 +0000 +@@ -102,6 +102,12 @@ # it is reachable only through the VAD sensitivity # select entity that satellite integrations create. vol.Optional("silence_seconds"): vol.Any(float, int), @@ -204,10 +321,11 @@ + vol.Optional("turn_detection"): bool, + vol.Optional("turn_threshold"): vol.Any(float, int), + vol.Optional("turn_max_seconds"): vol.Any(float, int), ++ vol.Optional("speculative_stt"): bool, } }, extra=vol.ALLOW_EXTRA, -@@ -219,11 +224,17 @@ +@@ -219,11 +225,18 @@ auto_gain_dbfs=msg_input.get("auto_gain_dbfs", 0), volume_multiplier=msg_input.get("volume_multiplier", 1.0), is_vad_enabled=not msg_input.get("no_vad", False), @@ -223,6 +341,7 @@ + ("turn_detection", bool), + ("turn_threshold", float), + ("turn_max_seconds", float), ++ ("speculative_stt", bool), + ) + if key in msg_input + for value in (msg_input[key],) From a5846e3bd1be1214e0e50a84cbacdc083803e638 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 05:54:35 +0000 Subject: [PATCH 86/98] Altogether: 1406 ms -> 600 ms on a question From the last sample of speech to the answer, through the real conversation agent. The turn model and speculative transcription take it to 717 ms, and ornith to 600 ms. It also holds a mid-sentence pause now, which no setting could do before. The control-command column of that table is not trustworthy, because entity state carries between runs and a model that says "already on" looks faster than one that switches the light. Behaviour belongs in dev/eval.py, which resets state before every run. Co-Authored-By: Claude Opus 5 --- dev/semantic-endpointing.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dev/semantic-endpointing.md b/dev/semantic-endpointing.md index 91cf62d..4a4a6e9 100644 --- a/dev/semantic-endpointing.md +++ b/dev/semantic-endpointing.md @@ -279,3 +279,23 @@ have finished, which is cheap: whisper tiny on CPU. Verified that holding a pause still works with it on -- the JFK clip still reaches "ASK NOT", so the snapshot really is discarded when speech resumes. + + +## Altogether + +From the last sample of speech to the answer, through the real conversation +agent: + +| configuration | question | control command | +|---|---|---| +| before any of this | 1406 ms | 2026 ms | +| turn model + speculative transcription | 717 ms | 774 ms | +| + ornith as the agent | **600 ms** | 1013 ms | + +**1406 -> 600 ms on a question, 2.3x.** And it now holds a mid-sentence pause, +which no setting could do before. + +The command column is not trustworthy: entity state carries between runs, so a +model that says "already on" looks faster than one that actually switches the +light. Compare behaviour with `dev/eval.py`, which resets state before every +run, not with a latency script. From 5e8543936a596a2ba976824ab031807db5d2914c Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 06:00:29 +0000 Subject: [PATCH 87/98] Pad the speculative snapshot, so its transcript matches Ending the snapshot on the last phoneme, without the silence the full stream would carry, changed what whisper heard: of six clips two differed, one a real mishearing -- "set the bad light" for "set the bed light" -- and one only capitalisation. Appending 250 ms of silence makes all six identical. Found by comparing the speculative transcript against the normal one clip by clip rather than trusting that the same audio gives the same text. This is a property of the engine, so check it again if the engine changes. Co-Authored-By: Claude Opus 5 --- dev/semantic-endpointing.md | 8 ++++++++ packages/home-assistant-smart-turn.patch | 12 +++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/dev/semantic-endpointing.md b/dev/semantic-endpointing.md index 4a4a6e9..3e4e5df 100644 --- a/dev/semantic-endpointing.md +++ b/dev/semantic-endpointing.md @@ -280,6 +280,14 @@ have finished, which is cheap: whisper tiny on CPU. Verified that holding a pause still works with it on -- the JFK clip still reaches "ASK NOT", so the snapshot really is discarded when speech resumes. +**The snapshot needs padding.** Ending it on the last phoneme, with none of the +silence the full stream would carry, changes what the transcriber hears: across +six clips two came out different, one of them a real mishearing ("set the bad +light" for "set the bed light") and one only capitalisation. Appending 250 ms of +silence to the snapshot makes all six identical. Worth checking again if the +speech-to-text engine changes, because this is a property of the engine, not of +the idea. + ## Altogether diff --git a/packages/home-assistant-smart-turn.patch b/packages/home-assistant-smart-turn.patch index 6192657..0548915 100644 --- a/packages/home-assistant-smart-turn.patch +++ b/packages/home-assistant-smart-turn.patch @@ -1,5 +1,5 @@ --- a/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 04:52:05.196107157 +0000 -+++ b/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 05:46:50.302759554 +0000 ++++ b/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 05:55:53.358056075 +0000 @@ -521,8 +521,52 @@ is_vad_enabled: bool = True """True if VAD is used to determine the end of the voice command.""" @@ -183,7 +183,7 @@ break if stt_vad.in_command and (not sent_vad_start): -@@ -1033,6 +1159,79 @@ +@@ -1033,6 +1159,85 @@ yield chunk.audio @@ -208,8 +208,14 @@ + self.task = None + + async def _transcribe(self, audio: bytes) -> stt.SpeechResult: ++ # Pad with the silence the full stream would have carried. ++ # Without it the snapshot ends abruptly on the last phoneme and ++ # transcription drifts: "set the bad light" for "set the bed ++ # light", and inconsistent capitalisation. ++ padded = audio + bytes(SAMPLE_WIDTH * SAMPLE_RATE // 4) ++ + async def once() -> AsyncGenerator[bytes]: -+ yield audio ++ yield padded + + assert run.stt_provider is not None + return await run.stt_provider.async_process_audio_stream(metadata, once()) From d26494ed24d0c6d0ac133a69719a9912935a22b7 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 06:09:26 +0000 Subject: [PATCH 88/98] Check the edge cases, and stop the harness mis-reporting them Silence with no speech, a 0.4 s blip, and an 11 s recording all behave the same with our settings as with stock ones, so none of this work regressed them. Silence and the blip run to the pipeline's own 60 s timeout in both configurations, which is upstream behaviour: no voice command ever starts, so the segmenter never reports one finishing. The long recording was reported as "no stt-end" when the transcript was plainly there. That was the harness: it measures from the moment the last sample is streamed, and a turn that ends early -- holding pauses, then hitting the cap -- never reaches that point. It now says so instead of implying the pipeline failed, and error events carry their reason through. Co-Authored-By: Claude Opus 5 --- dev/halib.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/dev/halib.py b/dev/halib.py index 8d6e90a..722521d 100644 --- a/dev/halib.py +++ b/dev/halib.py @@ -130,11 +130,18 @@ async def pump(): text = e["data"]["stt_output"]["text"] if e["type"] == "intent-end": reply = e["data"]["intent_output"]["response"]["speech"]["plain"]["speech"] - if e["type"] in ("run-end", "error"): + if e["type"] == "error": + return None, text, reply, e["data"] + if e["type"] == "run-end": break if task: task.cancel() last = "intent-end" if end_stage == "intent" else "stt-end" - if last not in marks or audio_end is None: - return None, text, reply, f"no {last}" + if last not in marks: + return None, text, reply, f"no {last} event" + if audio_end is None: + # The turn ended before the whole clip had been streamed, so there is no + # "end of speech" to measure from. Not a failure: it is what holding a + # pause and then hitting the cap looks like on a long recording. + return None, text, reply, None return (marks[last] - audio_end) * 1000, text, reply, None From 4a8cfb9d7b2a6b989bb555c9f436c8bbb393603f Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 06:10:26 +0000 Subject: [PATCH 89/98] Summarise the endpointing work and list the tools Puts the result and the one idea that matters at the top of the findings, ahead of the chronological detail, and records what is still open: real recordings of the person who will use it, speculating past the point where tool calls fire, streaming speech synthesis, and whether to switch the live assistant to ornith. Also lists the experiment scripts in the development README, along with the transcriber they need, and four things that each cost an hour: websocket ids must increase, truncating a log Home Assistant holds open fills it with nul bytes, subentry ids are not in the REST listing, and turn-detection verdicts are invisible without the logger block. Co-Authored-By: Claude Opus 5 --- dev/README.md | 38 +++++++++++++++++++++++++ dev/semantic-endpointing.md | 57 +++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/dev/README.md b/dev/README.md index 5c1cd6c..06787b9 100644 --- a/dev/README.md +++ b/dev/README.md @@ -9,6 +9,29 @@ rebuild of the real machine for every experiment. python3 dev/ha-ask.py "is the bed light on?" dev/run-ha.sh stop +Voice and model experiments, all driven through `dev/halib.py`. See +`dev/semantic-endpointing.md` for what they found: + + dev/turn-test.py # does it keep listening through a pause? + dev/silence-sweep.py # latency against pause tolerance + dev/stage-breakdown.py # where the milliseconds go + dev/e2e-compare.py # old settings against new + dev/eval.py 3 ... # scenario scores, state reset each run + dev/model-compare.py # models, end to end + dev/toolcall-stress.py # the tool-call path, repeatedly + dev/smart-turn-probe.py # the turn model's opinion, cut by cut + dev/smart-turn-eval.py # its accuracy on real labelled speech + +Anything that streams audio needs a transcriber in this VM, because the host's +is bound to loopback: + + $(nix build --no-link --print-out-paths nixpkgs#wyoming-faster-whisper)/bin/wyoming-faster-whisper \ + --model tiny-int8 --language en --uri tcp://127.0.0.1:10300 \ + --data-dir ~/ha-dev/whisper --download-dir ~/ha-dev/whisper + +`stt.demo_stt` cannot stand in for it: it accepts only stereo and the pipeline +sends mono. + State lives in `~/ha-dev`, the token in `scratch/hadev/token.txt`. Delete the directory to start over; the three scripts rebuild everything. @@ -45,3 +68,18 @@ reachable from here; run local ones if a test needs them. missing. - `pkill -f hass` matches the shell running it. Use the bracketed pattern in `run-ha.sh`. + +## More things that cost an hour to find + +- **Websocket ids must increase.** Home Assistant rejects a lower id with + `id_reuse`, so `halib` hands them out centrally rather than letting callers + pick. +- **Do not truncate `hass.log` while Home Assistant holds it open.** The write + offset stays where it was and the file fills with nul bytes, so `grep` finds + nothing and the log looks empty. Restart it instead. +- **Subentry ids are not in the REST entry listing**, which reports only + `num_subentries`. They come from `config_entries/subentries/list` over the + websocket. +- **Turn-detection verdicts log at debug level.** Without the `logger:` block in + `configuration.yaml` the decision is invisible and you can only infer it from + timing. diff --git a/dev/semantic-endpointing.md b/dev/semantic-endpointing.md index 3e4e5df..b852914 100644 --- a/dev/semantic-endpointing.md +++ b/dev/semantic-endpointing.md @@ -1,5 +1,62 @@ # Semantic endpointing +**Result: 1406 ms -> 600 ms** from the last sample of speech to the assistant's +answer, and it now keeps listening through a mid-sentence pause, which no +setting could do before. + +## What changed + +| | | +|---|---| +| Turn detection | Smart Turn v3 decides whether a pause is final, so `silence_seconds` no longer has to be long enough to forgive one | +| `silence_seconds` | 0.7 -> 0.25 by default, 0.1 available; every 100 ms off it is 100 ms off the answer | +| Speculative transcription | transcribes a snapshot taken when speech stops, during the wait, instead of after it | +| Model | `ornith:35b-q4_K_M` scored 45/45 against 38/45 for the incumbent, and is faster | + +## The one thing to understand + +Silence cannot tell "still thinking" from "finished" -- they are acoustically +identical and only the words differ. So with a silence threshold alone, how fast +the assistant answers and how long a pause it forgives are **the same number**, +and every setting is a compromise between them. A turn model separates them: the +threshold governs the common case, the model holds the turn open when the +utterance sounds unfinished. + +## Try it + + dev/run-ha.sh # the development instance + dev/turn-test.py # does it hold a pause? + dev/silence-sweep.py # latency against pause tolerance + dev/stage-breakdown.py # where the time goes + dev/eval.py 3 ... # scenario scores, state reset each run + dev/model-compare.py # models end to end + dev/smart-turn-probe.py # the model's opinion, cut by cut + +Voice tests need a transcriber in the VM: + + $(nix build --no-link --print-out-paths nixpkgs#wyoming-faster-whisper)/bin/wyoming-faster-whisper \ + --model tiny-int8 --language en --uri tcp://127.0.0.1:10300 \ + --data-dir ~/ha-dev/whisper --download-dir ~/ha-dev/whisper + +## Still open + +- **Recordings of the person who will use it.** Everything here is public data + and text-to-speech. The 6.1% of unfinished utterances the model cuts off at + threshold 0.9 is the number that matters, and it cannot be checked against a + corpus that does not contain your voice, your room or your phrasing. +- **Speculating past transcription.** The conversation stage executes tool + calls, and a cancelled speculation cannot un-turn-on a light. Overlapping the + wait with the *model* rather than just the transcriber needs a way to defer + side effects. +- **Streaming speech synthesis.** Home Assistant already pipes conversation + deltas into a TTS engine that advertises `supports_synthesize_streaming`, and + the Wyoming integration supports it. Whether Kokoro advertises it is untested + -- it would start audio on the first sentence instead of the last. +- **Switching the live assistant to ornith.** Left deliberately for a person: + the eval is 15 scenarios against demo entities, not a house. + +--- + No silence threshold can tell "still thinking" from "finished": they are acoustically identical and only the words differ. Every setting we can reach trades response speed against how long a pause is tolerated, one for one. From 3c39e3896e9fbb3fe8b81b1f49735fba3ba7171b Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 11:48:39 +0000 Subject: [PATCH 90/98] Stream text into Kokoro, so it speaks the first sentence early Nothing was spoken until the last token of a reply was generated, because Kokoro did not advertise supports_synthesize_streaming and Home Assistant will not stream text into an engine that does not. The hard part was already done: kokoro-wyoming already splits into sentences and emits audio for each. Only the input side was missing, so this adds SynthesizeStart / SynthesizeChunk / SynthesizeStop, buffers the incoming text, and synthesises each sentence as soon as it is complete. take_complete_sentences hands over only text up to the last sentence-ending punctuation, so nothing is spoken from a fragment more text would change. The model itself does not need to stream. Kokoro synthesises a whole utterance at once and quickly; it just has to be fed sooner. Tested with a stub synthesiser, because building the real one needs onnxruntime with CUDA: fed a reply the way a model produces it, the first sentence is synthesised after the second chunk while the third sentence is still arriving, and the event order is audio-start, chunks, audio-stop, synthesize-stopped. Unverified against the real synthesiser and against Home Assistant end to end. Co-Authored-By: Claude Opus 5 --- dev/kokoro-stream-test.py | 91 ++++++++++++ dev/semantic-endpointing.md | 54 +++++++- nixos/nixos/kokoro-wyoming-streaming.patch | 153 +++++++++++++++++++++ nixos/nixos/kokoro.nix | 28 ++-- 4 files changed, 314 insertions(+), 12 deletions(-) create mode 100644 dev/kokoro-stream-test.py create mode 100644 nixos/nixos/kokoro-wyoming-streaming.patch diff --git a/dev/kokoro-stream-test.py b/dev/kokoro-stream-test.py new file mode 100644 index 0000000..ce14120 --- /dev/null +++ b/dev/kokoro-stream-test.py @@ -0,0 +1,91 @@ +"""Drive the patched Kokoro server's protocol handling with a stub synthesiser. + +Building the real thing needs onnxruntime with CUDA, which is hours of compile +and irrelevant to what changed: this exercises the event handling, the sentence +splitting, and the order of the audio events Home Assistant expects. +""" +import asyncio, importlib.util, sys, types + +import numpy as np + +# main.py imports kokoro_onnx at module scope; stand in for it. +import logging + +fake = types.ModuleType("kokoro_onnx") +fake.__path__ = [] # make it a package, it has submodules +fake.config = types.SimpleNamespace(SAMPLE_RATE=24000, MAX_PHONEME_LENGTH=510) +fake.Kokoro = object +fake.EspeakConfig = object +fake_log = types.ModuleType("kokoro_onnx.log") +fake_log.log = logging.getLogger("stub") +fake_config = types.ModuleType("kokoro_onnx.config") +fake_config.SAMPLE_RATE = 24000 +fake_config.MAX_PHONEME_LENGTH = 510 +sys.modules["kokoro_onnx"] = fake +sys.modules["kokoro_onnx.log"] = fake_log +sys.modules["kokoro_onnx.config"] = fake_config + +spec = importlib.util.spec_from_file_location("kmain", sys.argv[1]) +kmain = importlib.util.module_from_spec(spec) +spec.loader.exec_module(kmain) + +from wyoming.audio import AudioChunk, AudioStart, AudioStop +from wyoming.tts import SynthesizeChunk, SynthesizeStart, SynthesizeStop, SynthesizeStopped + + +class StubKokoro: + def __init__(self): + self.spoken = [] + + def create_stream(self, text, voice, speed, lang): + self.spoken.append(text) + + async def gen(): + yield np.zeros(2400, dtype=np.float32), 24000 + + return gen() + + +class Recorder(kmain.KokoroEventHandler): + def __init__(self, kokoro): + self.kokoro = kokoro + self.default_voice = "af_heart" + self.default_speed = 1.0 + self.wyoming_info_event = None + self._semaphore = asyncio.Semaphore(1) + self._cache = None + self._stream_voice = "af_heart" + self._stream_buffer = "" + self._stream_started = False + self._stream_t0 = 0.0 + self.events = [] + + async def write_event(self, event): + self.events.append(event) + + +async def main(): + stub = StubKokoro() + h = Recorder(stub) + await h.handle_event(SynthesizeStart(voice=None).event()) + # A reply arriving the way a language model produces it. + for piece in ["The bed ", "light is off. ", "The kitchen ", "lights are on", + ". Anything else?"]: + await h.handle_event(SynthesizeChunk(text=piece).event()) + n = sum(AudioChunk.is_type(e.type) for e in h.events) + print(f" after {piece!r:22} synthesised={stub.spoken!r:60} audio chunks={n}") + await h.handle_event(SynthesizeStop().event()) + + kinds = [e.type for e in h.events] + print(f"\n sentences synthesised: {stub.spoken}") + print(f" event order: {kinds[0]} ... {kinds[-2]} {kinds[-1]}") + ok = ( + AudioStart.is_type(kinds[0]) + and AudioStop.is_type(kinds[-2]) + and SynthesizeStopped.is_type(kinds[-1]) + and stub.spoken == ["The bed light is off.", "The kitchen lights are on.", + "Anything else?"] + ) + print(f"\n {'PASS' if ok else 'FAIL'}") + +asyncio.run(main()) diff --git a/dev/semantic-endpointing.md b/dev/semantic-endpointing.md index b852914..ad615d0 100644 --- a/dev/semantic-endpointing.md +++ b/dev/semantic-endpointing.md @@ -48,10 +48,8 @@ Voice tests need a transcriber in the VM: calls, and a cancelled speculation cannot un-turn-on a light. Overlapping the wait with the *model* rather than just the transcriber needs a way to defer side effects. -- **Streaming speech synthesis.** Home Assistant already pipes conversation - deltas into a TTS engine that advertises `supports_synthesize_streaming`, and - the Wyoming integration supports it. Whether Kokoro advertises it is untested - -- it would start audio on the first sentence instead of the last. +- **Streaming speech synthesis** is now implemented; see below. Untested against + the real synthesiser, because building it needs onnxruntime with CUDA. - **Switching the live assistant to ornith.** Left deliberately for a person: the eval is 15 scenarios against demo entities, not a house. @@ -364,3 +362,51 @@ The command column is not trustworthy: entity state carries between runs, so a model that says "already on" looks faster than one that actually switches the light. Compare behaviour with `dev/eval.py`, which resets state before every run, not with a latency script. + + +## Streaming speech synthesis + +Nothing was spoken until the last token of the reply was generated, because +Kokoro did not advertise `supports_synthesize_streaming` and Home Assistant will +not stream text into an engine that does not. + +It turned out the hard part was already done. `kokoro-wyoming` **already** splits +text into sentences and emits audio for each as it is synthesised: + + sentences = split_into_sentences(text) + for sentence in sentences: + stream = self.kokoro.create_stream(sentence, ...) + if i == 0: + await self.write_event(AudioStart(...).event()) + async for audio, sample_rate in stream: + ...AudioChunk... + +Only the *input* side was missing: it waited for one complete `Synthesize` event. +`kokoro-wyoming-streaming.patch` adds `SynthesizeStart` / `SynthesizeChunk` / +`SynthesizeStop`, buffering text and synthesising each sentence as soon as it is +finished. So the model does not need to stream -- Kokoro synthesises a whole +utterance at once -- it just has to be fed sooner. + +`take_complete_sentences` only hands over text up to the last sentence-ending +punctuation, so a sentence is never synthesised from a fragment that more text +would have changed. + +`dev/kokoro-stream-test.py` drives the handler with a stub synthesiser, since +building the real one needs onnxruntime with CUDA. Feeding it a reply the way a +language model produces it: + + after 'The bed ' synthesised=[] audio chunks=0 + after 'light is off. ' synthesised=['The bed light is off.'] audio chunks=1 + after 'The kitchen ' synthesised=['The bed light is off.'] audio chunks=1 + after 'lights are on' synthesised=['The bed light is off.'] audio chunks=1 + after '. Anything else?' synthesised=[all three] audio chunks=3 + + event order: audio-start ... audio-stop synthesize-stopped + +The first sentence is spoken while the third is still being generated. The +longer the reply, the more this saves, and it is the only change here that +attacks time-to-*first-audio* rather than time-to-answer. + +**Still to verify on the host**, where the real synthesiser lives: that Home +Assistant picks up the capability, and what it does to the felt latency of a +long reply. diff --git a/nixos/nixos/kokoro-wyoming-streaming.patch b/nixos/nixos/kokoro-wyoming-streaming.patch new file mode 100644 index 0000000..62a0381 --- /dev/null +++ b/nixos/nixos/kokoro-wyoming-streaming.patch @@ -0,0 +1,153 @@ +--- a/src/main.py ++++ b/src/main.py +@@ -24,7 +24,13 @@ + + from wyoming.info import Attribution, TtsProgram, TtsVoice, TtsVoiceSpeaker, Describe, Info + from wyoming.server import AsyncServer +-from wyoming.tts import Synthesize ++from wyoming.tts import ( ++ Synthesize, ++ SynthesizeChunk, ++ SynthesizeStart, ++ SynthesizeStop, ++ SynthesizeStopped, ++) + from wyoming.audio import AudioChunk, AudioStart, AudioStop + from wyoming.event import Event + import re +@@ -47,6 +53,21 @@ + return [s.strip() for s in sentences if s.strip()] + + ++def take_complete_sentences(buffer: str) -> tuple[list[str], str]: ++ """Split off the sentences that are finished, keeping the rest buffered. ++ ++ Streaming synthesis needs to start speaking before the text has finished ++ arriving, but only on sentences it will not have to revise. Everything up to ++ the last sentence-ending punctuation is safe to synthesise; the remainder ++ stays until more text arrives or the stream ends. ++ """ ++ matches = list(re.finditer(r'[.!?](?=\s|$)', buffer)) ++ if not matches: ++ return [], buffer ++ cut = matches[-1].end() ++ return split_into_sentences(buffer[:cut]), buffer[cut:] ++ ++ + def clean_text(text: str) -> str: + """Strip markup artifacts that LLMs sometimes include in responses. + +@@ -146,6 +167,12 @@ + self._semaphore = synth_semaphore + self._cache = cache + ++ # Streaming synthesis state, one stream at a time per connection. ++ self._stream_voice = default_voice ++ self._stream_buffer = "" ++ self._stream_started = False ++ self._stream_t0 = 0.0 ++ + async def handle_event(self, event: Event) -> bool: + """Handle Wyoming protocol events.""" + if Describe.is_type(event.type): +@@ -153,6 +180,49 @@ + _LOGGER.debug("Sent info") + return True + ++ if SynthesizeStart.is_type(event.type): ++ # Streaming request: text arrives in pieces as the client produces ++ # it, so speech can start on the first finished sentence instead of ++ # waiting for the last one. ++ start = SynthesizeStart.from_event(event) ++ self._stream_voice = ( ++ start.voice.name ++ if start.voice is not None and start.voice.name ++ else self.default_voice ++ ) ++ self._stream_buffer = "" ++ self._stream_started = False ++ self._stream_t0 = time.monotonic() ++ _LOGGER.debug("Streaming synthesis started: voice=%s", self._stream_voice) ++ return True ++ ++ if SynthesizeChunk.is_type(event.type): ++ self._stream_buffer += SynthesizeChunk.from_event(event).text ++ ready, self._stream_buffer = take_complete_sentences(self._stream_buffer) ++ await self._speak(ready) ++ return True ++ ++ if SynthesizeStop.is_type(event.type): ++ await self._speak(split_into_sentences(self._stream_buffer)) ++ self._stream_buffer = "" ++ if not self._stream_started: ++ # Nothing was synthesised, but the client still needs a header ++ # before the stop, or it will wait for audio that never comes. ++ await self.write_event( ++ AudioStart( ++ rate=kokoro_onnx.config.SAMPLE_RATE, width=2, channels=1 ++ ).event() ++ ) ++ self._stream_started = True ++ await self.write_event(AudioStop().event()) ++ await self.write_event(SynthesizeStopped().event()) ++ _LOGGER.info( ++ "Streamed synthesis: voice=%s, %.0fms", ++ self._stream_voice, ++ (time.monotonic() - self._stream_t0) * 1000, ++ ) ++ return True ++ + if not Synthesize.is_type(event.type): + _LOGGER.warning("Unexpected event: %s", event) + return True +@@ -165,6 +235,42 @@ + ) + raise err + ++ async def _speak(self, sentences: list[str]) -> None: ++ """Synthesise finished sentences and write them out as they are ready.""" ++ wanted = [c for s in sentences if (c := clean_text(s))] ++ if not wanted: ++ return ++ ++ # Serialised for the same reason the non-streaming path is: concurrent ++ # ONNX inference competes for CPU and makes everyone slower. ++ async with self._semaphore: ++ for sentence in wanted: ++ stream = self.kokoro.create_stream( ++ sentence, ++ voice=self._stream_voice, ++ speed=self.default_speed, ++ lang="en-us" if self._stream_voice.startswith("a") else "en-gb", ++ ) ++ async for audio, _sample_rate in stream: ++ if not self._stream_started: ++ await self.write_event( ++ AudioStart( ++ rate=kokoro_onnx.config.SAMPLE_RATE, ++ width=2, ++ channels=1, ++ ).event() ++ ) ++ self._stream_started = True ++ audio_int16 = (audio * 32767).astype(np.int16) ++ await self.write_event( ++ AudioChunk( ++ audio=audio_int16.tobytes(), ++ rate=kokoro_onnx.config.SAMPLE_RATE, ++ width=2, ++ channels=1, ++ ).event() ++ ) ++ + async def _handle_synthesize(self, event: Event) -> Optional[bool]: + try: + synthesize = Synthesize.from_event(event) +@@ -382,6 +488,7 @@ + installed=True, + voices=sorted(wyoming_voices, key=lambda v: v.name), + version=VERSION, ++ supports_synthesize_streaming=True, + )] + ) + diff --git a/nixos/nixos/kokoro.nix b/nixos/nixos/kokoro.nix index ebf83f1..23e68a2 100644 --- a/nixos/nixos/kokoro.nix +++ b/nixos/nixos/kokoro.nix @@ -125,14 +125,26 @@ let # The SIGTERM handler stops the asyncio server, which surfaces as a # CancelledError out of asyncio.run and a traceback with status 1. Docker # swallowed that; systemd would call every `systemctl stop` a failure. - pkgs.runCommandLocal "kokoro-wyoming-1.0.2" { } '' - mkdir -p $out/bin - substitute ${src}/src/main.py $out/bin/kokoro-wyoming \ - --replace-fail "#!/usr/bin/env python3" "#!${python.interpreter}" \ - --replace-fail "except KeyboardInterrupt:" \ - "except (KeyboardInterrupt, asyncio.CancelledError):" - chmod +x $out/bin/kokoro-wyoming - ''; + pkgs.runCommandLocal "kokoro-wyoming-1.0.2" + { nativeBuildInputs = [ pkgs.patch ]; } + '' + mkdir -p src $out/bin + cp ${src}/src/main.py src/main.py + chmod +w src/main.py + + # Accept text as it is produced rather than all at once. The server + # already splits into sentences and emits audio for each, so Home + # Assistant can feed it the conversation reply token by token and the + # first sentence is spoken while the last is still being generated. + # Nothing is said at all until generation finishes otherwise. + patch -p1 < ${./kokoro-wyoming-streaming.patch} + + substitute src/main.py $out/bin/kokoro-wyoming \ + --replace-fail "#!/usr/bin/env python3" "#!${python.interpreter}" \ + --replace-fail "except KeyboardInterrupt:" \ + "except (KeyboardInterrupt, asyncio.CancelledError):" + chmod +x $out/bin/kokoro-wyoming + ''; in { # So the server can be built and run by hand, without a rebuild: From b6a3465258018ff9dfc2c85065847057ff0bb842 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 12:20:18 +0000 Subject: [PATCH 91/98] Speculate on the conversation, not just the transcription Transcribing early left the model idle for the rest of the wait, and the model is the slower of the two. The conversation now starts on the speculative transcript, and everything it produces is held until the turn is confirmed: pipeline events buffered, the speech stream withheld from the synthesiser, and any tool that would change something blocked mid-call. Held rather than abandoned. The prefill and the tokens that chose the tool are still valid if the guess was right, which it usually is, so a paused call costs nothing and resumes on commit; a wrong guess is a cancel, and the call never happens. llm.Tool.reads_only says which tools may run on a guess, and defaults to "acts" -- a needless read costs milliseconds, a needless action cannot be undone. Discard is cheap because conversation.async_get_chat_log works on a copy and writes it back only after the block it guards finishes, so a cancelled speculation leaves nothing in the history. Verified in the dev instance: a command holds HassTurnOff and releases it on commit, and when speech resumes the held call is dropped with the guess -- light.turn_off is called exactly once for an utterance that contains the command twice over. --- dev/py | 21 + dev/speculation-compare.py | 60 +++ dev/speculation-safety.py | 69 +++ .../home-assistant-speculative-intent.patch | 419 ++++++++++++++++++ packages/home-assistant.nix | 14 + 5 files changed, 583 insertions(+) create mode 100755 dev/py create mode 100644 dev/speculation-compare.py create mode 100644 dev/speculation-safety.py create mode 100644 packages/home-assistant-speculative-intent.patch diff --git a/dev/py b/dev/py new file mode 100755 index 0000000..ec4b04d --- /dev/null +++ b/dev/py @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Run a dev script with the packages they need. +# +# dev/py dev/speculation-compare.py 3 +# +# The scripts are plain Python, but the system interpreter does not carry +# websockets (or numpy, for the turn-model ones), and which packages a bare +# `python3` has is not something a repository should depend on. +set -euo pipefail +cd "$(dirname "$0")/.." +env=$(nix build --no-link --print-out-paths --impure --expr \ + 'with import {}; buildEnv { + name = "assist-dev"; + paths = [ + ffmpeg # the clips are wav and mp3; the pipeline wants raw 16 kHz mono + (python3.withPackages (ps: with ps; [ + websockets aiohttp numpy onnxruntime + ])) + ]; + }' 2>/dev/null | tail -1) +PATH="$env/bin:$PATH" exec "$env/bin/python" "$@" diff --git a/dev/speculation-compare.py b/dev/speculation-compare.py new file mode 100644 index 0000000..85cf9f9 --- /dev/null +++ b/dev/speculation-compare.py @@ -0,0 +1,60 @@ +"""Does starting the conversation on a guess make the answer arrive sooner -- +and is it the same answer? + +Runs each clip with speculative_intent off and on, alternating, and reports the +time from the last sample of speech to intent-end. The replies are compared as +well: speculation is only worth having if it changes nothing but the timing. + + dev/speculation-compare.py [repeats] [clip...] +""" +import asyncio, statistics, sys +sys.path.insert(0, "dev") +from halib import connect, ensure_whisper, engines, pcm, pipeline_id, run + +CLIPS = ["scratch/hadev/Is_the_kitchen_light_o.wav", + "scratch/hadev/Set_the_bed_light_to_f.wav", + "scratch/hadev/Turn_off_the_ceiling_l.wav"] + +BASE = dict(silence_seconds=0.25, turn_detection=True, turn_threshold=0.9, + turn_max_seconds=2.0) + + +async def main(): + repeats = int(sys.argv[1]) if len(sys.argv) > 1 else 3 + clips = sys.argv[2:] or CLIPS + ensure_whisper() + stt, conv = engines() + ws = await connect() + pid = await pipeline_id(ws, "speculation", stt_engine=stt, stt_language="en", + conversation_engine=conv) + print(f"conversation agent: {conv}\n") + + for clip in clips: + audio = pcm(clip) + print(clip.rsplit("/", 1)[-1]) + results = {} + for _ in range(repeats): + for on in (False, True): + ms, text, reply, err = await run( + ws, pid, audio, speculative_intent=on, **BASE) + if err: + print(f" ERROR {err}") + continue + results.setdefault(on, []).append((ms, text, reply)) + await asyncio.sleep(2) + for on in (False, True): + runs = results.get(on) + if not runs: + continue + label = "speculation on " if on else "speculation off" + median = statistics.median(r[0] for r in runs) + replies = {r[2] for r in runs} + print(f" {label} {median:6.0f} ms {len(replies)} distinct reply") + for r in sorted(replies): + print(f" {r[:70]!r}") + if results.get(False) and results.get(True): + off = statistics.median(r[0] for r in results[False]) + onn = statistics.median(r[0] for r in results[True]) + print(f" -> {off - onn:.0f} ms saved\n") + +asyncio.run(main()) diff --git a/dev/speculation-safety.py b/dev/speculation-safety.py new file mode 100644 index 0000000..008045f --- /dev/null +++ b/dev/speculation-safety.py @@ -0,0 +1,69 @@ +"""A speculation that turns out wrong must not have changed anything. + +Plays a complete command, a pause, then more speech -- the shape of someone who +was not finished. The pipeline starts the conversation on the first fragment, +the model calls a tool that would act, and the call is held. Speech resumes, so +the guess is thrown away with the call still held. + +Counting states is not enough: the full utterance contains that command too, so +the light ends up off either way. What distinguishes them is how many times the +service was called -- once if the held call was dropped, twice if it leaked. + +turn_threshold is 1.0 so the pause is held however final the fragment sounds: +this is a test of the discard path, not of the turn model. + + dev/speculation-safety.py +""" +import asyncio, json, sys +sys.path.insert(0, "dev") +from halib import SR, connect, ensure_whisper, engines, pcm, pipeline_id, rest, run + +ENTITY = "light.ceiling_lights" +COMMAND = "scratch/hadev/Turn_off_the_ceiling_l.wav" +MORE = "scratch/hadev/Is_the_kitchen_light_o.wav" + + +async def watch(calls): + """Record every service call Home Assistant makes, on its own connection.""" + ws = await connect() + await ws.send(json.dumps({"id": 1, "type": "subscribe_events", + "event_type": "call_service"})) + while True: + m = json.loads(await ws.recv()) + if m.get("type") == "event": + d = m["event"]["data"] + calls.append(f"{d['domain']}.{d['service']}") + + +async def main(): + ensure_whisper() + stt, conv = engines() + ws = await connect() + pid = await pipeline_id(ws, "speculation", stt_engine=stt, stt_language="en", + conversation_engine=conv) + + rest("/api/services/light/turn_on", {"entity_id": ENTITY}) + await asyncio.sleep(1) + + calls: list[str] = [] + watcher = asyncio.create_task(watch(calls)) + await asyncio.sleep(1) + calls.clear() + + # A command, a pause long enough to speculate in, then more speech. + audio = pcm(COMMAND) + bytes(SR * 2 * 1) + pcm(MORE) + ms, text, reply, err = await run( + ws, pid, audio, silence_seconds=0.25, turn_detection=True, + turn_threshold=1.0, turn_max_seconds=3.0, speculative_intent=True) + await asyncio.sleep(2) + watcher.cancel() + + print(f"heard: {text!r}") + print(f"reply: {reply!r}") + off = [c for c in calls if c == "light.turn_off"] + print(f"service calls: {calls}") + print(f"\nlight.turn_off called {len(off)} time(s)") + print("PASS: the held call was dropped with the guess" if len(off) == 1 + else "FAIL: the abandoned speculation acted as well") + +asyncio.run(main()) diff --git a/packages/home-assistant-speculative-intent.patch b/packages/home-assistant-speculative-intent.patch new file mode 100644 index 0000000..e513e94 --- /dev/null +++ b/packages/home-assistant-speculative-intent.patch @@ -0,0 +1,419 @@ +diff -ruN a/homeassistant/components/assist_pipeline/pipeline.py b/homeassistant/components/assist_pipeline/pipeline.py +--- a/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 12:07:45.222634118 +0000 ++++ b/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 12:15:40.098000182 +0000 +@@ -35,6 +35,7 @@ + device_registry as dr, + entity_registry as er, + intent, ++ llm, + ) + from homeassistant.helpers.collection import ( + CHANGE_UPDATED, +@@ -560,6 +561,19 @@ + when the speaker turns out not to have finished. + """ + ++ speculative_intent: bool = True ++ """Start the conversation on the guess, not after the wait. ++ ++ Transcribing early leaves the model idle for the rest of the wait, which is ++ the larger of the two. Starting it on the speculative transcript overlaps it ++ too, so the answer is often ready the moment the turn is confirmed. ++ ++ Only tools that read anything may run on a guess; the first one that would ++ change something blocks until the turn is confirmed, keeping the prefill and ++ the tokens that chose it. Nothing is spoken and no event is emitted until ++ then either, so a guess that turns out wrong is invisible. ++ """ ++ + turn_max_seconds: float = 3.0 + """Give up and end the command after this much silence, regardless. + +@@ -586,6 +600,36 @@ + ) + + ++class _Speculation: ++ """The conversation, run on a guess about what the speaker said. ++ ++ Nothing it produces escapes until it is committed: pipeline events are ++ buffered, the speech stream is not handed to text-to-speech, and any tool ++ that would change something blocks in `llm.APIInstance.async_call_tool`. ++ ++ Abandoning it is just a cancel. `conversation.async_get_chat_log` builds on a ++ copy and writes back only after the block it guards finishes, so a cancelled ++ speculation leaves nothing behind in the conversation history. ++ """ ++ ++ def __init__(self, text: str) -> None: ++ """Init the class.""" ++ self.text = text ++ self.gate = asyncio.Event() ++ self.events: list[PipelineEvent] = [] ++ self.task: asyncio.Task | None = None ++ self.speech: AsyncGenerator[str] | None = None ++ self.held: str | None = None ++ ++ async def async_wait_to_act(self, tool_name: str) -> None: ++ """Hold a tool call until the guess is confirmed (`llm.Speculation`).""" ++ _LOGGER.debug("holding tool call %s until the turn is confirmed", tool_name) ++ self.held = tool_name ++ await self.gate.wait() ++ _LOGGER.debug("releasing tool call %s", tool_name) ++ self.held = None ++ ++ + @dataclass + class PipelineRun: + """Running context for a pipeline.""" +@@ -677,6 +721,19 @@ + @callback + def process_event(self, event: PipelineEvent) -> None: + """Log an event and call listener.""" ++ if ( ++ pending := llm.speculation.get() ++ ) is not None and not pending.gate.is_set(): ++ # Emitted by a speculative conversation, before it was committed. ++ # Hold it: a client that ++ # showed the answer to a question the speaker had not finished ++ # asking would be worse than a slower one. Replayed on commit, in ++ # order, after which this task emits directly like any other -- the ++ # client sees the same events at the same points it would have seen ++ # without any of this. ++ pending.events.append(event) ++ return ++ + self.event_callback(event) + pipeline_data = self.hass.data[KEY_ASSIST_PIPELINE] + if self.id not in pipeline_data.pipeline_debug[self.pipeline.id]: +@@ -718,6 +775,9 @@ + + async def end(self) -> None: + """Emit run end event.""" ++ # A guess nobody committed: the run is over, so it will never be right. ++ self._abandon_speculative_intent() ++ + # Signal end of stream to listeners + self._capture_chunk(None) + +@@ -925,6 +985,8 @@ + ) + + _speculate: Any = None ++ _speculation: _Speculation | None = None ++ _intent_context: tuple[str, str | None] | None = None + _turn_over: asyncio.Event | None = None + + async def prepare_speech_to_text(self, metadata: stt.SpeechMetadata) -> None: +@@ -1178,6 +1240,7 @@ + if self.task is not None: + self.task.cancel() + self.task = None ++ run._abandon_speculative_intent() + + async def _transcribe(self, audio: bytes) -> stt.SpeechResult: + # Pad with the silence the full stream would have carried. +@@ -1190,7 +1253,14 @@ + yield padded + + assert run.stt_provider is not None +- return await run.stt_provider.async_process_audio_stream(metadata, once()) ++ result = await run.stt_provider.async_process_audio_stream( ++ metadata, once() ++ ) ++ if result.result == stt.SpeechResultState.SUCCESS and result.text: ++ # The wait is not over, but the words are. Start the ++ # conversation on them; everything it produces is held. ++ run._start_speculative_intent(result.text) ++ return result + + async def take(self) -> str | None: + """The transcript, if one was speculated and still applies.""" +@@ -1209,6 +1279,69 @@ + + return _Speculator() + ++ @callback ++ def _start_speculative_intent(self, text: str) -> None: ++ """Run the conversation on a guess, holding back everything it produces. ++ ++ The wait for silence is dead time for the model as well as the ++ transcriber, and the model is the slower of the two. Nothing here is ++ irreversible: read-only tools run, anything that acts blocks, and if the ++ speaker turns out to have been mid-sentence the task is cancelled. ++ """ ++ if ( ++ not self.audio_settings.speculative_intent ++ or self._speculation is not None ++ or self._intent_context is None ++ ): ++ return ++ ++ conversation_id, extra_system_prompt = self._intent_context ++ speculation = _Speculation(text) ++ ++ async def speculate() -> tuple[str, bool]: ++ """Mark this task, and everything it starts, as a guess.""" ++ token = llm.speculation.set(speculation) ++ try: ++ return await self.recognize_intent( ++ text, conversation_id, extra_system_prompt ++ ) ++ finally: ++ llm.speculation.reset(token) ++ ++ speculation.task = self.hass.async_create_task( ++ speculate(), name="assist_pipeline speculative intent" ++ ) ++ self._speculation = speculation ++ _LOGGER.debug("speculating on: %s", text) ++ ++ @callback ++ def _abandon_speculative_intent(self) -> None: ++ """The guess no longer applies, so throw the work away.""" ++ speculation, self._speculation = self._speculation, None ++ if speculation is None or speculation.gate.is_set(): ++ return ++ ++ _LOGGER.debug("abandoning speculative conversation: %s", speculation.text) ++ if speculation.task is not None: ++ speculation.task.cancel() ++ # Its buffered events are dropped with it, and its speech generator was ++ # never handed to text-to-speech. Only this flag is shared with the run ++ # that replaces it, and leaving it set would stop that one streaming. ++ self._streamed_response_text = False ++ ++ @callback ++ def _commit_speculative_intent(self, speculation: _Speculation) -> None: ++ """The guess was right: let everything it produced out, in order.""" ++ events, speculation.events = speculation.events, [] ++ for event in events: ++ self.process_event(event) ++ ++ if speculation.speech is not None and self.tts_stream is not None: ++ self.tts_stream.async_set_message_stream(speculation.speech) ++ ++ # Last, so that a tool released here cannot emit before the replay. ++ speculation.gate.set() ++ + async def _turn_is_complete(self, turn_detector: Any, audio: bytearray) -> bool: + """Ask the turn model whether the speaker has finished. + +@@ -1281,6 +1414,24 @@ + + Returns (speech, all_targets_in_satellite_area). + """ ++ speculating = llm.speculation.get() ++ if speculating is None and (speculation := self._speculation) is not None: ++ # A guess was started while waiting for the speaker to finish. If it ++ # guessed the same words, it is this call, already part-done. ++ self._speculation = None ++ if speculation.text == intent_input and speculation.task is not None: ++ _LOGGER.debug( ++ "committing speculative conversation%s", ++ f", releasing held {speculation.held}" if speculation.held else "", ++ ) ++ self._commit_speculative_intent(speculation) ++ return await speculation.task ++ ++ _LOGGER.debug( ++ "speculation missed: guessed %r, heard %r", speculation.text, intent_input ++ ) ++ self._abandon_speculative_intent() ++ + if self.intent_agent is None or self._conversation_data is None: + raise RuntimeError("Recognize intent was not prepared") + +@@ -1399,7 +1550,18 @@ + ) + + assert self.tts_stream is not None +- self.tts_stream.async_set_message_stream(tts_input_stream_generator()) ++ if speculating is not None: ++ # Not yet. ResultStream keeps only the first stream it is ++ # given, so wiring the guess here would make the real ++ # reply's stream a silent no-op -- and the speaker would ++ # answer a question that had not finished being asked. Held ++ # until commit, by which point the queue behind it has a ++ # head start. ++ speculating.speech = tts_input_stream_generator() ++ else: ++ self.tts_stream.async_set_message_stream( ++ tts_input_stream_generator() ++ ) + + user_input = conversation.ConversationInput( + text=intent_input, +@@ -1886,6 +2048,13 @@ + device_id=self.device_id, + satellite_id=self.satellite_id, + ) ++ if self.run.end_stage != PipelineStage.STT: ++ # What recognize_intent would be called with, so speech-to-text can ++ # start the conversation on its guess before this stage is reached. ++ self.run._intent_context = ( # noqa: SLF001 ++ self.session.conversation_id, ++ self.conversation_extra_system_prompt, ++ ) + current_stage: PipelineStage | None = self.run.start_stage + + try: +diff -ruN a/homeassistant/components/assist_pipeline/websocket_api.py b/homeassistant/components/assist_pipeline/websocket_api.py +--- a/homeassistant/components/assist_pipeline/websocket_api.py 2026-08-18 12:11:30.560579218 +0000 ++++ b/homeassistant/components/assist_pipeline/websocket_api.py 2026-08-18 12:11:30.573070618 +0000 +@@ -108,6 +108,7 @@ + vol.Optional("turn_threshold"): vol.Any(float, int), + vol.Optional("turn_max_seconds"): vol.Any(float, int), + vol.Optional("speculative_stt"): bool, ++ vol.Optional("speculative_intent"): bool, + } + }, + extra=vol.ALLOW_EXTRA, +@@ -233,6 +234,7 @@ + ("turn_threshold", float), + ("turn_max_seconds", float), + ("speculative_stt", bool), ++ ("speculative_intent", bool), + ) + if key in msg_input + for value in (msg_input[key],) +diff -ruN a/homeassistant/helpers/llm.py b/homeassistant/helpers/llm.py +--- a/homeassistant/helpers/llm.py 2026-08-18 12:07:45.223546857 +0000 ++++ b/homeassistant/helpers/llm.py 2026-08-18 12:08:44.979721712 +0000 +@@ -4,13 +4,14 @@ + + from abc import ABC, abstractmethod + from collections.abc import Callable ++from contextvars import ContextVar + from dataclasses import dataclass, field as dc_field + from datetime import timedelta + from decimal import Decimal + from enum import Enum + from functools import cache, partial + from operator import attrgetter +-from typing import Any, cast ++from typing import Any, Protocol, cast + + import slugify as unicode_slug + import voluptuous as vol +@@ -186,6 +187,25 @@ + external: bool = False + + ++class Speculation(Protocol): ++ """Something that may hold a tool call back until it is certain.""" ++ ++ async def async_wait_to_act(self, tool_name: str) -> None: ++ """Return once acting is allowed, or raise CancelledError if it never is.""" ++ ++ ++speculation: ContextVar[Speculation | None] = ContextVar( ++ "llm_speculation", default=None ++) ++"""Set while a caller is running the conversation on a guess. ++ ++The voice pipeline starts the conversation before it is certain the speaker has ++finished, to overlap the model with the wait. Reading the house is free to do on ++a guess; acting on it is not, so a tool that acts blocks here until the guess is ++confirmed and is cancelled outright if it is not. ++""" ++ ++ + class Tool: + """LLM Tool base class.""" + +@@ -193,6 +213,15 @@ + description: str | None = None + parameters: vol.Schema = vol.Schema({}) + ++ reads_only: bool = False ++ """True if calling this tool cannot change anything. ++ ++ Only such a tool may run on a speculation. The default is False because the ++ cost of the two mistakes is not symmetric: running a reader needlessly wastes ++ a few milliseconds, while running an action that the speaker turns out not to ++ have asked for cannot be undone. ++ """ ++ + @abstractmethod + async def async_call( + self, hass: HomeAssistant, tool_input: ToolInput, llm_context: LLMContext +@@ -233,6 +262,14 @@ + else: + raise HomeAssistantError(f'Tool "{tool_input.tool_name}" not found') + ++ if not tool.reads_only and (pending := speculation.get()) is not None: ++ # Running on a guess about what the speaker said. Hold here rather ++ # than abandon: the prefill and the tokens that produced this call ++ # are worth keeping, and if the guess was right they are all still ++ # valid. If it was wrong this task is cancelled and the call simply ++ # never happens. ++ await pending.async_wait_to_act(tool_input.tool_name) ++ + return await tool.async_call(self.api.hass, tool_input, self.llm_context) + + +@@ -250,6 +287,23 @@ + raise NotImplementedError + + ++READ_ONLY_INTENTS = { ++ intent.INTENT_GET_STATE, ++ intent.INTENT_GET_CURRENT_DATE, ++ intent.INTENT_GET_CURRENT_TIME, ++ intent.INTENT_GET_TEMPERATURE, ++ intent.INTENT_TIMER_STATUS, ++ intent.INTENT_NEVERMIND, ++ INTENT_GET_WEATHER, ++} ++"""Intents that answer a question without changing anything. ++ ++Most of these are in AssistAPI.IGNORE_INTENTS and so are not exposed as tools at ++all; they are listed anyway because another API may expose them, and because the ++answer to "does this intent act" should not depend on who is asking. ++""" ++ ++ + class IntentTool(Tool): + """LLM Tool representing an Intent.""" + +@@ -260,6 +314,7 @@ + ) -> None: + """Init the class.""" + self.name = name ++ self.reads_only = intent_handler.intent_type in READ_ONLY_INTENTS + self.description = ( + intent_handler.description or f"Execute Home Assistant {self.name} intent" + ) +@@ -1039,6 +1094,7 @@ + """LLM Tool allowing querying a calendar.""" + + name = "calendar_get_events" ++ reads_only = True + description = ( + "Get events from a calendar. " + "When asked if something happens, search the whole week. " +@@ -1105,6 +1161,7 @@ + """LLM Tool allowing querying a to-do list.""" + + name = "todo_get_items" ++ reads_only = True + description = ( + "Query a to-do list to find out what items are on it. " + "Use this to answer questions like 'What's on my task list?' or 'Read my grocery list'. " +@@ -1189,6 +1246,7 @@ + """ + + name = "GetLiveContext" ++ reads_only = True + description = ( + "Provides real-time information about the CURRENT state, value, or mode of devices, sensors, entities, or areas. " + "Use this tool for: " +@@ -1293,6 +1351,7 @@ + """Tool for getting the current date and time.""" + + name = "GetDateTime" ++ reads_only = True + description = "Provides the current date and time." + + async def async_call( diff --git a/packages/home-assistant.nix b/packages/home-assistant.nix index 9b3e665..da81733 100644 --- a/packages/home-assistant.nix +++ b/packages/home-assistant.nix @@ -41,6 +41,20 @@ home-assistant.overrideAttrs (old: { # fed audio cut mid-word, 60.6% of polls read as complete, because it judges # the prosody up to the cut. Silero still decides *when* to ask. ./home-assistant-smart-turn.patch + + # Speculative execution of the conversation, not just the transcription. + # The wait for silence idles the model as well as the transcriber, and the + # model is the slower of the two, so the conversation starts on the + # speculative transcript and everything it produces is held back until the + # turn is confirmed: events buffered, speech not handed to the synthesiser, + # and any tool that would change something blocked mid-call. + # + # Held rather than abandoned, because the prefill and the tokens that chose + # the tool are still valid if the guess was right -- which it usually is. + # Reading the house runs immediately; llm.Tool.reads_only says which is + # which, and defaults to "acts", since a needless read costs milliseconds + # and a needless action cannot be undone. + ./home-assistant-speculative-intent.patch ]; # audio_enhancer.py imports pysilero_vad after the Silero patch. The manifest From cae8f4d22ad68489393ba6bbfac84e4673394c37 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 12:39:33 +0000 Subject: [PATCH 92/98] Do not synthesise every reply twice, and test what Home Assistant sends Home Assistant streams text to a Wyoming synthesiser as SynthesizeStart, then chunks, and then sends the whole message again as a plain Synthesize "for backwards compatibility" with servers that cannot stream. The patched Kokoro server fell through to its ordinary Synthesize handler for that, so every reply was spoken twice and cost twice the GPU time. It ignores it during a stream now. The stub test missed it because it sent only the events the streaming path cares about. It now performs the whole exchange, trailing Synthesize included. Alongside, a stand-in synthesiser for the VM, since Kokoro needs CUDA and the question "was this asked to be spoken, and when" does not need a GPU to answer. It is faithful on both of the points that turned out to matter: it ignores the trailing Synthesize, and it emits an audio header even with nothing to say -- without which Home Assistant waits for audio that never comes. That stand-in then found a race in the speculation patch. Commit can happen before generation is far enough along for Home Assistant to start streaming text; the stream created after that was stored for a commit that had already happened, and nobody wired it. Home Assistant then skipped async_set_message, believing a stream was set, and the reply was never spoken at all. The decision is on the gate now, not on "is this a speculation". Home Assistant's speech cache had been hiding it: the same few replies were served without asking the synthesiser anything. --- dev/fake-tts.py | 135 ++++++++++++++++++ dev/halib.py | 20 ++- dev/kokoro-stream-test.py | 23 ++- dev/py | 2 +- dev/run-fake-tts.sh | 26 ++++ dev/speculation-speech.py | 117 +++++++++++++++ dev/speculation-sweep.py | 53 +++++++ nixos/nixos/kokoro-wyoming-streaming.patch | 27 +++- .../home-assistant-speculative-intent.patch | 14 +- 9 files changed, 397 insertions(+), 20 deletions(-) create mode 100644 dev/fake-tts.py create mode 100755 dev/run-fake-tts.sh create mode 100644 dev/speculation-speech.py create mode 100644 dev/speculation-sweep.py diff --git a/dev/fake-tts.py b/dev/fake-tts.py new file mode 100644 index 0000000..6028e91 --- /dev/null +++ b/dev/fake-tts.py @@ -0,0 +1,135 @@ +"""A text-to-speech server that says nothing, and writes down what it was asked. + +Kokoro needs CUDA, so the real synthesiser cannot run in this VM -- but the +question "did the pipeline ask for this to be spoken, and when" does not need a +synthesiser to answer. This advertises streaming synthesis, accepts both the +one-shot and the streamed form, returns silence, and appends a line per sentence +to its journal. + + dev/py dev/fake-tts.py [--uri tcp://127.0.0.1:10211] [--journal scratch/spoken.jsonl] +""" +import argparse, asyncio, json, time +from wyoming.audio import AudioChunk, AudioStart, AudioStop +from wyoming.event import Event +from wyoming.info import Attribution, Describe, Info, TtsProgram, TtsVoice +from wyoming.server import AsyncEventHandler, AsyncServer +from wyoming.tts import ( + Synthesize, SynthesizeChunk, SynthesizeStart, SynthesizeStop, SynthesizeStopped, +) + +RATE, WIDTH, CHANNELS = 22050, 2, 1 +ENDINGS = ".!?" + + +def take_complete_sentences(buffer: str) -> tuple[list[str], str]: + """Split off what is certainly finished, keeping the rest for later text.""" + cut = max((buffer.rfind(c) for c in ENDINGS), default=-1) + if cut < 0: + return [], buffer + done, rest = buffer[: cut + 1], buffer[cut + 1 :] + return [s.strip() for s in done.replace("! ", "!|").replace("? ", "?|") + .replace(". ", ".|").split("|") if s.strip()], rest + + +class Handler(AsyncEventHandler): + def __init__(self, info, journal, *args, **kwargs): + super().__init__(*args, **kwargs) + self.info = info + self.journal = journal + self.buffer = "" + self.started = False + self.streaming = False + + def note(self, what, text): + with open(self.journal, "a") as f: + f.write(json.dumps({"at": time.time(), "event": what, "text": text}) + "\n") + + async def speak(self, text): + self.note("speak", text) + if not self.started: + await self.write_event( + AudioStart(rate=RATE, width=WIDTH, channels=CHANNELS).event()) + self.started = True + # A tenth of a second of silence, so there is something to receive. + await self.write_event(AudioChunk( + rate=RATE, width=WIDTH, channels=CHANNELS, + audio=bytes(RATE * WIDTH // 10)).event()) + + async def finish(self): + if not self.started: + # Even with nothing to say, the client needs a header before the + # stop or it waits for audio that never comes. The real server does + # this too; a stand-in that skipped it would hang Home Assistant + # here and look like a bug in the pipeline. + await self.write_event( + AudioStart(rate=RATE, width=WIDTH, channels=CHANNELS).event()) + self.started = True + await self.write_event(AudioStop().event()) + self.started = False + + async def handle_event(self, event: Event) -> bool: + if Describe.is_type(event.type): + await self.write_event(self.info.event()) + return True + + if Synthesize.is_type(event.type): + if self.streaming: + # Home Assistant sends the whole message again after the chunks, + # for servers that cannot stream. This one can, and has already + # said it. A stand-in that got this wrong would hide the same + # mistake in the real server. + self.note("ignored-trailing-synthesize", "") + return True + text = Synthesize.from_event(event).text + self.note("synthesize", text) + for sentence in take_complete_sentences(text + " ")[0] or [text]: + await self.speak(sentence) + await self.finish() + return True + + if SynthesizeStart.is_type(event.type): + self.note("start", "") + self.buffer = "" + self.streaming = True + return True + + if SynthesizeChunk.is_type(event.type): + self.buffer += SynthesizeChunk.from_event(event).text + sentences, self.buffer = take_complete_sentences(self.buffer) + for sentence in sentences: + await self.speak(sentence) + return True + + if SynthesizeStop.is_type(event.type): + if self.buffer.strip(): + await self.speak(self.buffer.strip()) + self.buffer = "" + await self.finish() + await self.write_event(SynthesizeStopped().event()) + self.streaming = False + self.note("stop", "") + return True + + return True + + +async def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--uri", default="tcp://127.0.0.1:10211") + parser.add_argument("--journal", default="scratch/spoken.jsonl") + args = parser.parse_args() + + info = Info(tts=[TtsProgram( + name="fake", description="says nothing, writes it down", installed=True, + version="1", attribution=Attribution(name="dev", url=""), + supports_synthesize_streaming=True, + voices=[TtsVoice(name="silence", description="silence", installed=True, + version="1", languages=["en"], + attribution=Attribution(name="dev", url=""))])]) + + open(args.journal, "w").close() + print(f"fake tts on {args.uri}, journal {args.journal}", flush=True) + await AsyncServer.from_uri(args.uri).run( + lambda *a, **k: Handler(info, args.journal, *a, **k)) + +asyncio.run(main()) diff --git a/dev/halib.py b/dev/halib.py index 722521d..e80889a 100644 --- a/dev/halib.py +++ b/dev/halib.py @@ -78,9 +78,11 @@ async def pipeline_id(ws, name, **fields): **{k: v for k, v in existing.items() if k != "id"}, **stale) return existing["id"] res = await call(ws, type="assist_pipeline/pipeline/create", - name=name, language="en", conversation_language="en", - tts_engine=None, tts_language=None, tts_voice=None, - wake_word_entity=None, wake_word_id=None, **fields) + **{"name": name, "language": "en", + "conversation_language": "en", "tts_engine": None, + "tts_language": None, "tts_voice": None, + "wake_word_entity": None, "wake_word_id": None, + **fields}) return res["result"]["id"] @@ -95,8 +97,14 @@ def engines(): return stt, conv -async def run(ws, pid, audio, end_stage="intent", trailing=4, **settings): - """Stream audio in real time; return (ms from end of speech, text, reply, error).""" +async def run(ws, pid, audio, end_stage="intent", trailing=4, on_event=None, + **settings): + """Stream audio in real time; return (ms from end of speech, text, reply, error). + + on_event, if given, is called with every pipeline event as it arrives -- for + tests that need to act on one, such as fetching the speech as a satellite + would rather than leaving it to be synthesised whenever. + """ ident = _ident() stream = audio + b"\x00" * (SR * 2 * trailing) await ws.send(json.dumps({"id": ident, "type": "assist_pipeline/run", @@ -123,6 +131,8 @@ async def pump(): continue e = m["event"] marks[e["type"]] = time.monotonic() + if on_event is not None: + on_event(e) if e["type"] == "run-start": hid = e["data"]["runner_data"]["stt_binary_handler_id"] task = asyncio.create_task(pump()) diff --git a/dev/kokoro-stream-test.py b/dev/kokoro-stream-test.py index ce14120..59d59df 100644 --- a/dev/kokoro-stream-test.py +++ b/dev/kokoro-stream-test.py @@ -1,8 +1,16 @@ """Drive the patched Kokoro server's protocol handling with a stub synthesiser. -Building the real thing needs onnxruntime with CUDA, which is hours of compile -and irrelevant to what changed: this exercises the event handling, the sentence -splitting, and the order of the audio events Home Assistant expects. +Building the real thing needs onnxruntime with CUDA, which is irrelevant to what +changed: this exercises the event handling, the sentence splitting, and the order +of the audio events Home Assistant expects. + +The exchange is exactly the one Home Assistant performs, including the plain +Synthesize carrying the whole message that it sends after the chunks "for +backwards compatibility". A server that streams has already said all of it, and +must ignore it -- the first version of this test left it out, and the reply was +synthesised twice for a day before anything noticed. + + dev/py dev/kokoro-stream-test.py """ import asyncio, importlib.util, sys, types @@ -30,7 +38,9 @@ spec.loader.exec_module(kmain) from wyoming.audio import AudioChunk, AudioStart, AudioStop -from wyoming.tts import SynthesizeChunk, SynthesizeStart, SynthesizeStop, SynthesizeStopped +from wyoming.tts import ( + Synthesize, SynthesizeChunk, SynthesizeStart, SynthesizeStop, SynthesizeStopped, +) class StubKokoro: @@ -57,6 +67,7 @@ def __init__(self, kokoro): self._stream_voice = "af_heart" self._stream_buffer = "" self._stream_started = False + self._streaming = False self._stream_t0 = 0.0 self.events = [] @@ -74,6 +85,10 @@ async def main(): await h.handle_event(SynthesizeChunk(text=piece).event()) n = sum(AudioChunk.is_type(e.type) for e in h.events) print(f" after {piece!r:22} synthesised={stub.spoken!r:60} audio chunks={n}") + # What Home Assistant sends next: the whole message, again. + whole = "The bed light is off. The kitchen lights are on. Anything else?" + await h.handle_event(Synthesize(text=whole).event()) + print(f" after the trailing Synthesize synthesised={stub.spoken!r}") await h.handle_event(SynthesizeStop().event()) kinds = [e.type for e in h.events] diff --git a/dev/py b/dev/py index ec4b04d..6028ec3 100755 --- a/dev/py +++ b/dev/py @@ -14,7 +14,7 @@ env=$(nix build --no-link --print-out-paths --impure --expr \ paths = [ ffmpeg # the clips are wav and mp3; the pipeline wants raw 16 kHz mono (python3.withPackages (ps: with ps; [ - websockets aiohttp numpy onnxruntime + websockets aiohttp numpy onnxruntime wyoming ])) ]; }' 2>/dev/null | tail -1) diff --git a/dev/run-fake-tts.sh b/dev/run-fake-tts.sh new file mode 100755 index 0000000..5def5c3 --- /dev/null +++ b/dev/run-fake-tts.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Start or stop the stand-in text-to-speech server. +# +# dev/run-fake-tts.sh # start it in the background +# dev/run-fake-tts.sh stop +# +# It lives in a script rather than a shell one-liner because `pkill -f fake-tts` +# also matches the shell that typed it, which kills the wrong process. +set -euo pipefail +cd "$(dirname "$0")/.." +pidfile=scratch/fake-tts.pid + +if [ "${1:-start}" = stop ]; then + [ -f "$pidfile" ] && kill "$(cat "$pidfile")" 2>/dev/null && echo stopped || echo "not running" + rm -f "$pidfile" + exit 0 +fi + +if [ -f "$pidfile" ] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then + echo "already running as $(cat "$pidfile")"; exit 0 +fi + +setsid dev/py dev/fake-tts.py > scratch/fake-tts.log 2>&1 < /dev/null & +echo $! > "$pidfile" +sleep 8 +cat scratch/fake-tts.log diff --git a/dev/speculation-speech.py b/dev/speculation-speech.py new file mode 100644 index 0000000..a15cf7b --- /dev/null +++ b/dev/speculation-speech.py @@ -0,0 +1,117 @@ +"""Nothing is spoken on a guess -- and the real reply still streams. + +Home Assistant hands the synthesiser text as the model produces it, so the first +sentence is spoken while the last is still being written. A speculative +conversation produces that same text, and must not reach the synthesiser until +the turn is confirmed. This checks both halves against dev/fake-tts.py, which +answers "what was I asked to say, and when" without needing a GPU. + +Run the fake synthesiser first, and add it to Home Assistant as a Wyoming entry +on port 10211: + + dev/py dev/fake-tts.py & + dev/py dev/speculation-speech.py +""" +import asyncio, json, sys, urllib.request +sys.path.insert(0, "dev") +from halib import ( + SR, TOKEN, connect, ensure_whisper, engines, pcm, pipeline_id, rest, run, +) + +JOURNAL = "scratch/spoken.jsonl" +QUESTION = "scratch/hadev/Is_the_kitchen_light_o.wav" +COMMAND = "scratch/hadev/Turn_off_the_ceiling_l.wav" + + +def spoken(): + with open(JOURNAL) as f: + return [json.loads(line) for line in f if line.strip()] + + +def clear(): + open(JOURNAL, "w").close() + # Home Assistant caches synthesised speech, and these clips draw the same + # few replies over and over, so without this a run is served from the cache + # and the server is never asked anything. + rest("/api/services/tts/clear_cache", {}) + + +async def say_it(url): + """Pull the audio, the way a satellite does. + + Nothing is synthesised until something asks for it, so a test that only + watches the pipeline events sees synthesis land whenever -- sometimes inside + the next test. Fetching makes it deterministic. + """ + req = urllib.request.Request(f"http://127.0.0.1:8123{url}", + headers={"Authorization": f"Bearer {TOKEN}"}) + return await asyncio.to_thread( + lambda: urllib.request.urlopen(req, timeout=60).read()) + + +def fetcher(pending): + """Start pulling the speech as soon as the run says where it is. + + run-start carries the URL when the reply will be streamed into the + synthesiser, and tts-end when it will not; take whichever comes first. + """ + def on_event(e): + out = (e.get("data") or {}).get("tts_output") or {} + if out.get("url") and not pending: + pending.append(asyncio.create_task(say_it(out["url"]))) + pending.append(e["type"]) + return on_event + + +async def main(): + ensure_whisper() + stt, conv = engines() + tts = next(s["entity_id"] for s in rest("/api/states") + if s["entity_id"] == "tts.fake") + ws = await connect() + pid = await pipeline_id(ws, "speculation-tts", stt_engine=stt, stt_language="en", + conversation_engine=conv, tts_engine=tts, + tts_language="en", tts_voice="silence") + + settings = dict(silence_seconds=0.25, turn_detection=True, turn_threshold=0.9, + turn_max_seconds=2.0, speculative_intent=True) + + print("1. a plain reply is spoken") + clear() + pending = [] + ms, text, reply, err = await run(ws, pid, pcm(QUESTION), end_stage="tts", + on_event=fetcher(pending), **settings) + if pending: + await pending[0] + print(f" url from {pending[1]}") + said = [e["text"] for e in spoken() if e["event"] == "speak"] + print(f" heard {text!r}\n said {said}") + if err: + print(f" error {err}") + ok_plain = bool(said) + print(" PASS" if ok_plain else " FAIL: nothing was spoken") + + print("\n2. an abandoned guess is not spoken") + clear() + audio = pcm(COMMAND) + bytes(SR * 2 * 1) + pcm(QUESTION) + pending = [] + ms, text, reply, err = await run( + ws, pid, audio, end_stage="tts", on_event=fetcher(pending), + **{**settings, "turn_threshold": 1.0, "turn_max_seconds": 3.0}) + if pending: + await pending[0] + print(f" url from {pending[1]}") + events = spoken() + said = [e["text"] for e in events if e["event"] == "speak"] + starts = [e for e in events if e["event"] == "start"] + print(f" heard {text!r}\n said {said}") + if err: + print(f" error {err}") + # One utterance, spoken once: a leaked guess would have opened a second + # stream, or spoken the fragment's reply as well as the real one. + ok_guess = len(starts) <= 1 and len(said) > 0 + print(" PASS" if ok_guess else f" FAIL: {len(starts)} synthesis streams") + + print("\n" + ("PASS" if ok_plain and ok_guess else "FAIL")) + +asyncio.run(main()) diff --git a/dev/speculation-sweep.py b/dev/speculation-sweep.py new file mode 100644 index 0000000..c59aac9 --- /dev/null +++ b/dev/speculation-sweep.py @@ -0,0 +1,53 @@ +"""What speculating on the conversation is worth, against how long the wait is. + +The wait for silence is the one thing speculation cannot remove: the answer +still must not arrive before the speaker is known to have finished. What it +removes is the work that used to happen *after* the wait. So the interesting +number is not the saving at one setting but how flat the curve gets: if the +model finishes inside the wait, a longer and safer wait costs nothing. + + dev/speculation-sweep.py [repeats] +""" +import asyncio, statistics, sys +sys.path.insert(0, "dev") +from halib import connect, ensure_whisper, engines, pcm, pipeline_id, run + +CLIPS = {"question": "scratch/hadev/Is_the_kitchen_light_o.wav", + "command": "scratch/hadev/Turn_off_the_ceiling_l.wav"} +SILENCES = (0.1, 0.25, 0.7) + + +async def main(): + repeats = int(sys.argv[1]) if len(sys.argv) > 1 else 4 + ensure_whisper() + stt, conv = engines() + ws = await connect() + pid = await pipeline_id(ws, "speculation", stt_engine=stt, stt_language="en", + conversation_engine=conv) + print(f"conversation agent: {conv}") + print(f"{repeats} runs per cell, median ms from last sample of speech to answer\n") + + for name, clip in CLIPS.items(): + audio = pcm(clip) + print(f" {name:9} {'silence':>9} {'off':>8} {'on':>8} {'saved':>8}") + for silence in SILENCES: + got = {} + for _ in range(repeats): + for on in (False, True): + ms, text, reply, err = await run( + ws, pid, audio, silence_seconds=silence, + turn_detection=True, turn_threshold=0.9, + turn_max_seconds=2.0, speculative_intent=on) + if err: + print(f" ERROR {err}") + continue + got.setdefault(on, []).append(ms) + await asyncio.sleep(1.5) + if not (got.get(False) and got.get(True)): + continue + off = statistics.median(got[False]) + onn = statistics.median(got[True]) + print(f" {'':9} {silence:9.2f} {off:8.0f} {onn:8.0f} {off - onn:8.0f}") + print() + +asyncio.run(main()) diff --git a/nixos/nixos/kokoro-wyoming-streaming.patch b/nixos/nixos/kokoro-wyoming-streaming.patch index 62a0381..db2e626 100644 --- a/nixos/nixos/kokoro-wyoming-streaming.patch +++ b/nixos/nixos/kokoro-wyoming-streaming.patch @@ -1,5 +1,5 @@ ---- a/src/main.py -+++ b/src/main.py +--- a/src/main.py 2026-08-18 12:27:48.912647149 +0000 ++++ b/src/main.py 2026-08-18 12:28:06.790783727 +0000 @@ -24,7 +24,13 @@ from wyoming.info import Attribution, TtsProgram, TtsVoice, TtsVoiceSpeaker, Describe, Info @@ -37,7 +37,7 @@ def clean_text(text: str) -> str: """Strip markup artifacts that LLMs sometimes include in responses. -@@ -146,6 +167,12 @@ +@@ -146,6 +167,13 @@ self._semaphore = synth_semaphore self._cache = cache @@ -45,12 +45,13 @@ + self._stream_voice = default_voice + self._stream_buffer = "" + self._stream_started = False ++ self._streaming = False + self._stream_t0 = 0.0 + async def handle_event(self, event: Event) -> bool: """Handle Wyoming protocol events.""" if Describe.is_type(event.type): -@@ -153,6 +180,49 @@ +@@ -153,10 +181,63 @@ _LOGGER.debug("Sent info") return True @@ -66,6 +67,7 @@ + ) + self._stream_buffer = "" + self._stream_started = False ++ self._streaming = True + self._stream_t0 = time.monotonic() + _LOGGER.debug("Streaming synthesis started: voice=%s", self._stream_voice) + return True @@ -90,6 +92,7 @@ + self._stream_started = True + await self.write_event(AudioStop().event()) + await self.write_event(SynthesizeStopped().event()) ++ self._streaming = False + _LOGGER.info( + "Streamed synthesis: voice=%s, %.0fms", + self._stream_voice, @@ -100,7 +103,19 @@ if not Synthesize.is_type(event.type): _LOGGER.warning("Unexpected event: %s", event) return True -@@ -165,6 +235,42 @@ + ++ if self._streaming: ++ # Home Assistant sends the whole message as a plain Synthesize after ++ # the chunks, "for backwards compatibility" with servers that do not ++ # stream. This one does, and has already said all of it: synthesising ++ # again would speak the reply twice and cost twice the GPU time. ++ _LOGGER.debug("Ignoring trailing Synthesize during a stream") ++ return True ++ + try: + return await self._handle_synthesize(event) + except Exception as err: +@@ -165,6 +246,42 @@ ) raise err @@ -143,7 +158,7 @@ async def _handle_synthesize(self, event: Event) -> Optional[bool]: try: synthesize = Synthesize.from_event(event) -@@ -382,6 +488,7 @@ +@@ -382,6 +499,7 @@ installed=True, voices=sorted(wyoming_voices, key=lambda v: v.name), version=VERSION, diff --git a/packages/home-assistant-speculative-intent.patch b/packages/home-assistant-speculative-intent.patch index e513e94..1f77fb8 100644 --- a/packages/home-assistant-speculative-intent.patch +++ b/packages/home-assistant-speculative-intent.patch @@ -1,6 +1,6 @@ diff -ruN a/homeassistant/components/assist_pipeline/pipeline.py b/homeassistant/components/assist_pipeline/pipeline.py --- a/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 12:07:45.222634118 +0000 -+++ b/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 12:15:40.098000182 +0000 ++++ b/homeassistant/components/assist_pipeline/pipeline.py 2026-08-18 12:36:54.954953182 +0000 @@ -35,6 +35,7 @@ device_registry as dr, entity_registry as er, @@ -224,18 +224,24 @@ diff -ruN a/homeassistant/components/assist_pipeline/pipeline.py b/homeassistant if self.intent_agent is None or self._conversation_data is None: raise RuntimeError("Recognize intent was not prepared") -@@ -1399,7 +1550,18 @@ +@@ -1399,7 +1550,24 @@ ) assert self.tts_stream is not None - self.tts_stream.async_set_message_stream(tts_input_stream_generator()) -+ if speculating is not None: ++ if speculating is not None and not speculating.gate.is_set(): + # Not yet. ResultStream keeps only the first stream it is + # given, so wiring the guess here would make the real + # reply's stream a silent no-op -- and the speaker would + # answer a question that had not finished being asked. Held + # until commit, by which point the queue behind it has a + # head start. ++ # ++ # The gate, not "is this a speculation": commit can happen ++ # before generation is far enough along to stream, and a ++ # stream stored after that has nobody left to wire it. Home ++ # Assistant would then skip async_set_message, since it ++ # believes a stream was set, and the speech never arrives. + speculating.speech = tts_input_stream_generator() + else: + self.tts_stream.async_set_message_stream( @@ -244,7 +250,7 @@ diff -ruN a/homeassistant/components/assist_pipeline/pipeline.py b/homeassistant user_input = conversation.ConversationInput( text=intent_input, -@@ -1886,6 +2048,13 @@ +@@ -1886,6 +2054,13 @@ device_id=self.device_id, satellite_id=self.satellite_id, ) From fb3b5891d29dc2a097f7ae8d0109ed4e4af07325 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 12:44:21 +0000 Subject: [PATCH 93/98] Write down the speculation results, and five things that cost an hour --- dev/README.md | 25 ++++++ dev/semantic-endpointing.md | 148 +++++++++++++++++++++++++++++++++--- 2 files changed, 164 insertions(+), 9 deletions(-) diff --git a/dev/README.md b/dev/README.md index 06787b9..66303a0 100644 --- a/dev/README.md +++ b/dev/README.md @@ -21,6 +21,18 @@ Voice and model experiments, all driven through `dev/halib.py`. See dev/toolcall-stress.py # the tool-call path, repeatedly dev/smart-turn-probe.py # the turn model's opinion, cut by cut dev/smart-turn-eval.py # its accuracy on real labelled speech + dev/speculation-sweep.py # what speculating is worth, against the wait + dev/speculation-safety.py # a wrong guess must not act + dev/speculation-speech.py # a wrong guess must not be spoken + dev/kokoro-stream-test.py # the synthesiser's protocol handling + +Run them with `dev/py`, which supplies websockets, numpy and ffmpeg; which +packages a bare `python3` happens to have is not something to depend on. + +`dev/speculation-speech.py` needs the stand-in synthesiser, added to Home +Assistant as a Wyoming entry on port 10211: + + dev/run-fake-tts.sh Anything that streams audio needs a transcriber in this VM, because the host's is bound to loopback: @@ -83,3 +95,16 @@ reachable from here; run local ones if a test needs them. - **Turn-detection verdicts log at debug level.** Without the `logger:` block in `configuration.yaml` the decision is invisible and you can only infer it from timing. +- **`pkill -f` matches the shell that typed it.** A command containing + `fake-tts.py` is itself a match, so the pattern kills the session. Bracketing + only helps when the name does not appear elsewhere on the line; a pid file and + a script, as in `run-fake-tts.sh`, always works. +- **Home Assistant caches synthesised speech.** A test that plays the same clip + repeatedly is served from the cache and never reaches the synthesiser, which + reads as "nothing was spoken". `tts.clear_cache` between cases. +- **A Wyoming synthesiser must send an audio header even with nothing to say.** + Without an `AudioStart`/`AudioStop` pair Home Assistant waits for audio that + never arrives, and the whole pipeline appears to hang somewhere else entirely. +- **Home Assistant sends the whole message again after the chunks**, as a plain + `Synthesize`, for servers that cannot stream. One that can must ignore it or + it says everything twice. diff --git a/dev/semantic-endpointing.md b/dev/semantic-endpointing.md index ad615d0..dde0b25 100644 --- a/dev/semantic-endpointing.md +++ b/dev/semantic-endpointing.md @@ -11,6 +11,8 @@ setting could do before. | Turn detection | Smart Turn v3 decides whether a pause is final, so `silence_seconds` no longer has to be long enough to forgive one | | `silence_seconds` | 0.7 -> 0.25 by default, 0.1 available; every 100 ms off it is 100 ms off the answer | | Speculative transcription | transcribes a snapshot taken when speech stops, during the wait, instead of after it | +| Speculative conversation | runs the model on that transcript too, holding side effects and speech until the turn is confirmed | +| Streaming synthesis | Kokoro is fed text as it is generated, so the first sentence is spoken while the last is still being written | | Model | `ornith:35b-q4_K_M` scored 45/45 against 38/45 for the incumbent, and is faster | ## The one thing to understand @@ -44,12 +46,10 @@ Voice tests need a transcriber in the VM: and text-to-speech. The 6.1% of unfinished utterances the model cuts off at threshold 0.9 is the number that matters, and it cannot be checked against a corpus that does not contain your voice, your room or your phrasing. -- **Speculating past transcription.** The conversation stage executes tool - calls, and a cancelled speculation cannot un-turn-on a light. Overlapping the - wait with the *model* rather than just the transcriber needs a way to defer - side effects. -- **Streaming speech synthesis** is now implemented; see below. Untested against - the real synthesiser, because building it needs onnxruntime with CUDA. +- **Speculating past transcription** is now done; see below. Tool calls that + would change something are held mid-call rather than abandoned, so a guess + that turns out right keeps the prefill and the tokens that chose the tool. +- **Streaming speech synthesis** is done and verified on the host; see below. - **Switching the live assistant to ornith.** Left deliberately for a person: the eval is 15 scenarios against demo entities, not a house. @@ -407,6 +407,136 @@ The first sentence is spoken while the third is still being generated. The longer the reply, the more this saves, and it is the only change here that attacks time-to-*first-audio* rather than time-to-answer. -**Still to verify on the host**, where the real synthesiser lives: that Home -Assistant picks up the capability, and what it does to the felt latency of a -long reply. +### Verified on the host + +`run-start` carries `tts_output.stream_response`, which is true only when the +synthesiser takes streamed input *and* the agent produces streamed output, so +the question needs no guessing: + + stream_response = True + +Fetching the audio from the moment the URL exists, and timing the first byte +against the moment generation finished: + +| reply | first audio | generation done | speaking starts | +|---|---|---|---| +| "Is the bed light on?" | 516 ms | 381 ms | 135 ms *after* | +| three sentences | 1054 ms | 2535 ms | **1482 ms before** | +| eight sentences | 748 ms | 5500 ms | **4752 ms before** | + +A one-sentence answer gains nothing -- there is no later sentence to overlap +with, and synthesis still has to happen. Everything longer gains roughly the +whole of its own generation time. Reproduce with `scratch/host-tts-firstbyte.py`. + +### Every reply was being synthesised twice + +Home Assistant sends `SynthesizeStart`, then the chunks, and then the whole +message again as a plain `Synthesize` -- commented in `wyoming/tts.py` as "for +backwards compatibility", for servers that cannot stream. The patched server +fell through to its ordinary `Synthesize` handler for that, so it said +everything a second time and spent twice the GPU on it. + +The stub test had not caught it because it sent only the events the streaming +path cares about. It now performs the exchange Home Assistant actually performs, +trailing `Synthesize` included, which is the version worth keeping: the bug was +not in the logic under test but in the half of the protocol the test omitted. + +## Speculating on the conversation, not just the transcription + +Transcribing early leaves the *model* idle for the rest of the wait, and the +model is the slower of the two. So the conversation now starts on the +speculative transcript as well, and everything it produces is held until the +turn is confirmed: + +| what | how it is held | +|---|---| +| pipeline events | buffered and replayed in order at commit | +| speech | the stream is not handed to the synthesiser until commit | +| tools that read | run immediately -- a wrong one wastes milliseconds | +| tools that act | block inside `llm.APIInstance.async_call_tool` | + +**Held, not abandoned.** The prefill and the tokens that chose the tool are +still valid if the guess was right, which it usually is, so a paused call costs +nothing and resumes on commit. Abandoning would throw away the most expensive +part of the work to save nothing. + +`llm.Tool.reads_only` says which tools may run on a guess. It defaults to +*acts*, because the two mistakes do not cost the same: a needless read wastes a +few milliseconds, a needless action cannot be undone. Only `GetLiveContext`, +`GetDateTime`, `calendar_get_events`, `todo_get_items` and the read-only intents +are marked. + +Discard is nearly free, which is what makes the whole thing safe. +`conversation.async_get_chat_log` builds on a copy of the history and writes it +back only *after* the block it guards finishes -- so cancelling the task leaves +nothing behind. `async_get_chat_session` does the same. Neither needed changing. + +### What it is worth + +Speculation cannot remove the wait itself: the answer still must not arrive +before the speaker is known to have finished. What it removes is the work that +used to happen *after* the wait. So the number to look at is not the saving at +one setting, it is how flat the curve becomes. + + question command + silence off on silence off on + 0.10 785 753 0.10 1477 1379 + 0.25 787 760 0.25 1476 1421 + 0.70 1244 838 0.70 1887 1370 + +Median ms from the last sample of speech to the answer, four runs a cell, +`dev/speculation-sweep.py`. With speculation on, latency barely depends on the +wait at all: 753-838 ms across the whole range for a question, 1370-1421 ms for +a command. Without it, going from 0.25 to 0.7 costs 457 ms and 411 ms. + +**So pause tolerance is close to free now.** The default stays at 0.25 s, since +the turn model already holds mid-sentence pauses and there is no reason to make +a finished utterance wait longer. But 0.7 s costs about 80 ms instead of about +460 ms, which makes it a reasonable thing to reach for if the model turns out to +cut you off -- a pause shorter than the threshold is never submitted to it at +all. + +The saving is smaller here than it will be on the host, because this VM +transcribes in ~211 ms against the host's 94-167 ms, and the transcription has +to finish before the conversation can start on it. The idle left inside a +250 ms wait is whatever the transcriber does not use. + +### Checking it does not act on a guess + +`dev/speculation-safety.py` plays a complete command, a pause, then more speech +-- the shape of someone who was not finished -- with `turn_threshold` at 1.0 so +the pause is held however final the fragment sounds. Counting states is not +enough, because the full utterance contains that command too and the light ends +up off either way. What separates them is how many times the service was called: + + heard: ' Turn off the ceiling lights. Is the kitchen light on right now?' + service calls: ['homeassistant.turn_off', 'light.turn_off'] + light.turn_off called 1 time(s) + +and in the log, the held call is dropped rather than released: + + holding tool call HassTurnOff until the turn is confirmed + abandoning speculative conversation: Turn off the ceiling lights. + speculating on: Turn off the ceiling lights. Is the kitchen light on... + holding tool call HassTurnOff until the turn is confirmed + committing speculative conversation, releasing held HassTurnOff + +`dev/speculation-speech.py` checks the other half, that a guess is never spoken, +against `dev/fake-tts.py` -- a synthesiser that says nothing and writes down what +it was asked to say, since Kokoro needs CUDA and that question does not. + +### Two things that only a real synthesiser in the loop would have found + +**Home Assistant caches speech.** These clips draw the same few replies over and +over, so runs were served from the cache and the synthesiser was never asked +anything -- which looks exactly like "nothing was spoken". The test clears the +cache between cases now. + +**A race between commit and the start of streaming.** Home Assistant only starts +streaming text into the synthesiser once a reply looks long enough to be worth +it. That can happen *after* commit, and the first version stored the stream +whenever it was a speculation -- so a stream created after commit was stored for +a commit that had already happened, and nobody wired it. Home Assistant then +skipped `async_set_message`, believing a stream was set, and the reply was never +spoken. The decision is on the gate now, the same condition the event buffer +uses, not on "is this a speculation". From a4c15e9fae5701791e66f72562cfca466a942144 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Tue, 18 Aug 2026 13:24:04 +0000 Subject: [PATCH 94/98] Let the transcriber reach Parakeet, which "auto" already prefers wyoming-faster-whisper defaults to model = "auto", which reads as "the best one available" and is not. For English it prefers Parakeet through sherpa-onnx, and the test is whether the sherpa_onnx module imports -- but that module lives in the package's optional-dependencies and is not installed, so the preference falls through to rhasspy/faster-whisper-base-int8 and says nothing about it. Turning the extra on is the whole change. Measured on one machine over the same four clips, dev/stt-compare.py: whisper-base-int8 (what auto resolves to today) 368 ms Parakeet TDT 0.6b v2 int8 (what it prefers) 65 ms tiny-int8 193 ms 5.7x faster than what the host runs now, from the more accurate model of the two. It matters twice over: speculative transcription is only free while it finishes inside the wait for silence, and 368 ms overran a 250 ms wait outright. Needs a rebuild, and the first start downloads the model. --- dev/local-intent-test.py | 80 +++++++++++++++++++++++++++++++++++ dev/stt-compare.py | 80 +++++++++++++++++++++++++++++++++++ nixos/nixos/configuration.nix | 24 ++++++++++- 3 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 dev/local-intent-test.py create mode 100644 dev/stt-compare.py diff --git a/dev/local-intent-test.py b/dev/local-intent-test.py new file mode 100644 index 0000000..867b3e3 --- /dev/null +++ b/dev/local-intent-test.py @@ -0,0 +1,80 @@ +"""How much of the latency is avoidable by never reaching the model at all. + +With prefer_local_intents on, Home Assistant matches commands against its own +sentence templates first and only falls back to the conversation agent when +nothing matches. A match costs a few milliseconds; a miss costs the whole model +round trip. Matching is strict -- exact wording, exact entity name -- so whether +a phrase is fast depends entirely on what the entity happens to be called. + +Aliases are the lever. This measures a phrase before and after adding one. + + dev/py dev/local-intent-test.py +""" +import asyncio, statistics, sys, time +sys.path.insert(0, "dev") +from halib import call, connect, engines, pipeline_id, rest + +ENTITY = "light.ceiling_lights" +EXACT = "Turn off the Ceiling Lights." +LOOSE = "Turn off the ceiling light." +ALIAS = "ceiling light" + + +async def timed(ws, pid, text, repeats=3): + out = [] + for _ in range(repeats): + ident = None + ms, kind, speech = await one(ws, pid, text) + out.append((ms, kind, speech)) + await asyncio.sleep(1) + return statistics.median(m for m, _, _ in out), out[-1][1], out[-1][2] + + +async def one(ws, pid, text): + import json + from halib import _ident + ident = _ident() + await ws.send(json.dumps({"id": ident, "type": "assist_pipeline/run", + "start_stage": "intent", "end_stage": "intent", + "input": {"text": text}, "pipeline": pid, "timeout": 60})) + t0 = time.monotonic() + while True: + m = json.loads(await ws.recv()) + if m.get("id") != ident or m.get("type") != "event": + continue + e = m["event"] + if e["type"] == "intent-end": + r = e["data"]["intent_output"]["response"] + return ((time.monotonic() - t0) * 1000, r["response_type"], + r["speech"]["plain"]["speech"]) + if e["type"] == "error": + return (time.monotonic() - t0) * 1000, "error", str(e["data"]) + + +async def aliases(ws, value): + res = await call(ws, type="config/entity_registry/update", + entity_id=ENTITY, aliases=value) + return res["result"]["entity_entry"]["aliases"] + + +async def main(): + stt, conv = engines() + ws = await connect() + pid = await pipeline_id(ws, "local-intents", stt_engine=stt, stt_language="en", + conversation_engine=conv, prefer_local_intents=True) + + print(f"{'phrase':32} {'alias':8} {'ms':>7} {'result':14} reply") + await aliases(ws, []) + for label, text in (("exact", EXACT), ("loose", LOOSE)): + ms, kind, speech = await timed(ws, pid, text) + print(f" {text:30} {'none':8} {ms:7.0f} {kind:14} {speech[:34]!r}") + + got = await aliases(ws, [ALIAS]) + print(f"\n added alias {got}\n") + for label, text in (("exact", EXACT), ("loose", LOOSE)): + ms, kind, speech = await timed(ws, pid, text) + print(f" {text:30} {ALIAS:8} {ms:7.0f} {kind:14} {speech[:34]!r}") + + await aliases(ws, []) + +asyncio.run(main()) diff --git a/dev/stt-compare.py b/dev/stt-compare.py new file mode 100644 index 0000000..b4c4999 --- /dev/null +++ b/dev/stt-compare.py @@ -0,0 +1,80 @@ +"""Which transcriber, and what it costs. + +The host runs wyoming-faster-whisper with `model = "auto"`. That looks like it +picks the best available, but the sherpa-onnx bindings live in an optional +extra that is not installed, so the Parakeet branch is unreachable and it falls +back to whisper-base-int8. This transcribes the same clips through both and +reports the text and the time. + +Start each server first; they are separate processes on separate ports. + + dev/py dev/stt-compare.py :