From edab12701c35257ea888459ed6970251873e6022 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 20 Sep 2026 18:10:21 +0800 Subject: [PATCH 1/6] fix: scale decode budget with audio length instead of a fixed cap --- docs/input-limits.md | 50 ++++++++++++--- docs/models/qwen3-asr.md | 8 ++- src/arch/canary/model.cpp | 27 +++++--- src/arch/canary_qwen/model.cpp | 36 +++++++---- src/arch/cohere/model.cpp | 23 ++++--- src/arch/funasr_nano/model.cpp | 46 +++++++++----- src/arch/granite/model.cpp | 43 ++++++++----- src/arch/moss/model.cpp | 11 ++-- src/arch/qwen3_asr/model.cpp | 42 +++++++++---- src/arch/voxtral/model.cpp | 17 +++-- src/transcribe-decode-budget.h | 58 ++++++++++++++++++ tests/CMakeLists.txt | 16 +++++ tests/decode_budget_unit.cpp | 92 ++++++++++++++++++++++++++++ tests/qwen3_asr_batch_truncation.cpp | 55 +++++++++++++---- 14 files changed, 414 insertions(+), 110 deletions(-) create mode 100644 src/transcribe-decode-budget.h create mode 100644 tests/decode_budget_unit.cpp diff --git a/docs/input-limits.md b/docs/input-limits.md index fb7a6ae0..5d7e81eb 100644 --- a/docs/input-limits.md +++ b/docs/input-limits.md @@ -82,14 +82,43 @@ over-length clip immediately — the caller never pays for a compute pass that cannot fit. The rejection goes through the log callback, not raw stderr. The one case that cannot be predicted up front is the transcript itself running -long enough to exhaust the remaining budget mid-decode (rare — the output would -have to be very large for the audio length). There, the run returns the hard -status `TRANSCRIBE_ERR_OUTPUT_TRUNCATED` while keeping the partial transcript -readable (exactly like an aborted run); `transcribe_was_truncated(session)` is -also set, and a `WARN` is logged. A truncated transcript is never returned as -`TRANSCRIBE_OK` — a caller cannot mistake it for complete — and the partial -output is never discarded. In `transcribe_run_batch` this is a per-utterance -status (the whole-batch call still returns `TRANSCRIBE_OK`). +long enough to exhaust the remaining budget mid-decode. There, the run returns +the hard status `TRANSCRIBE_ERR_OUTPUT_TRUNCATED` while keeping the partial +transcript readable (exactly like an aborted run); +`transcribe_was_truncated(session)` is also set, and a `WARN` is logged. A +truncated transcript is never returned as `TRANSCRIBE_OK` — a caller cannot +mistake it for complete — and the partial output is never discarded. In +`transcribe_run_batch` this is a per-utterance status (the whole-batch call +still returns `TRANSCRIBE_OK`). + +### The decode budget + +How much output an accepted clip may produce is **derived from the clip**, not +fixed. Each autoregressive family resolves a per-run decode budget as: + +```text +budget = clamp(max(generation_reserve, predicted_transcript_tokens), + 0, ceiling - prompt_tokens) +``` + +`predicted_transcript_tokens` is the encoder's audio-token count (speech never +yields more text tokens than the encoder yields audio tokens, so it is a safe +upper bound; moss scales it up because its output also carries speaker +markers). `generation_reserve` is the per-family floor — the same constant the +up-front gate reserves and `max_audio_ms` subtracts — so a short clip decodes +exactly as it always has. `ceiling` is the decoder context, which +`transcribe_session_params::n_ctx` lowers. + +**`n_ctx` is the only caller-facing control over output length.** There is +deliberately no per-run "max tokens" parameter: an ASR transcript's length is a +property of the audio, so the library derives it rather than asking. Lowering +`n_ctx` to bound memory also lowers the budget, and can turn a run that would +have completed into `OUTPUT_TRUNCATED`. + +Historically these budgets were flat per-family constants (256 or 512 tokens) +that ignored audio length entirely, so a clip well inside `max_audio_ms` could +still truncate with most of the context unused. That is fixed; the reserve +constants remain only as the floor. ### 3. Soft window — warn and proceed @@ -192,6 +221,11 @@ detect truncation should check `transcribe_was_truncated()` after finalize. ## Design notes (for maintainers) +- `generation_reserve` is a floor, not a cap. The gate and `max_audio_ms` + reserve it so an accepted clip is guaranteed at least that much output room; + the per-run budget then scales up with the audio (see "The decode budget"). + Changing a family's reserve moves its published `max_audio_ms`, so it is not + a free knob — the budget rule is the thing to tune. - The upfront gate and `max_audio_ms` share a shape for decoder-context-bound families but differ in precision: `max_audio_ms ≈ (ceiling − representative_prompt − generation_reserve) / tokens_per_ms`, diff --git a/docs/models/qwen3-asr.md b/docs/models/qwen3-asr.md index ed816492..963ddbed 100644 --- a/docs/models/qwen3-asr.md +++ b/docs/models/qwen3-asr.md @@ -47,7 +47,13 @@ family. That ceiling is there to bound memory and sits far beyond any normal clip; audio past it is rejected up front with `TRANSCRIBE_ERR_INPUT_TOO_LONG` rather than silently truncated. Lowering `--n-ctx` lowers the limit (and the KV-cache footprint), and `transcribe_session_get_limits()` reports the exact -per-session value. See the [input-length contract](../input-limits.md). +per-session value. + +Output length is not separately capped: the decode budget scales with the audio +and is bounded only by the context left after the prompt, so a clip inside the +input limit transcribes in full. `--n-ctx` bounds both — lowering it far enough +will truncate a long transcript (`TRANSCRIBE_ERR_OUTPUT_TRUNCATED`, partial text +retained). See the [input-length contract](../input-limits.md). ## Quick start diff --git a/src/arch/canary/model.cpp b/src/arch/canary/model.cpp index 6b556357..403dfc85 100644 --- a/src/arch/canary/model.cpp +++ b/src/arch/canary/model.cpp @@ -14,6 +14,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-env.h" #include "transcribe-flash-policy.h" #include "transcribe-load-common.h" @@ -214,10 +215,16 @@ constexpr float kBnEps = 1e-5f; // (a) INPUT — the encoder rel-pos table (enc_pos_emb_max_len, ~400 s). // T_enc must stay within it or the runtime table aliases past the // trained range; gated up front. Drives max_audio_ms. -// (b) DECODER self-KV (dec_max_position) + 512 max-new cap bound the -// OUTPUT length; an overrun is kept as a partial and flagged via +// (b) DECODER self-KV (dec_max_position) bounds the OUTPUT length; an +// overrun is kept as a partial and flagged via // transcribe_was_truncated(), not rejected. +// Generation reserve, in tokens: the floor under the per-run decode budget. +// The budget itself scales with the audio and is clamped to the decoder +// self-KV ceiling (see transcribe-decode-budget.h), so a short clip decodes +// exactly as it always has while a long one is no longer cut at a flat 512. +constexpr int k_gen_reserve = 512; + // Predicted encoder frame count T_enc for a given mel frame count. The // FastConformer pre-encode downsamples time via stride-2, kernel-3, pad-1 // convs; each stage maps T_in -> floor((T_in-1)/2)+1. We fold that exact @@ -391,7 +398,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->limits.has_context_cap = true; m->limits.audio_from_caps = true; m->limits.model_max_ctx = m->hparams.dec_max_position; - m->limits.gen_reserve = 512; // run()'s max-new-tokens cap + m->limits.gen_reserve = k_gen_reserve; // Whisper-style decoder self-KV: dec_d_model per layer, K and V, no GQA. m->limits.kv_elems_per_ctx_token = (int64_t) m->hparams.dec_d_model * m->hparams.dec_n_layers * 2; } @@ -1090,7 +1097,7 @@ transcribe_status run(transcribe_session * session, cc->clear_result(); const int eos_id = cm->hparams.eos_token_id; - const int max_tokens = std::min(512, cc->kv_cache.n_ctx - prompt_len); + const int max_tokens = transcribe::pick_decode_budget(T_enc, k_gen_reserve, prompt_len, cc->kv_cache.n_ctx); int next_token = 0; if (prompt_skip_softmax && db.argmax_out != nullptr) { @@ -1608,15 +1615,17 @@ transcribe_status run_batch(transcribe_session * session, } // Batched KV cache. - const int max_new = 512; - int max_n_kv = 1024; - while (max_n_kv < prompt_len + max_new) { - max_n_kv *= 2; - } // Decoder self-KV ceiling: dec_max_position, optionally lowered (never // raised) by the caller's n_ctx knob. Default knob (0) leaves it at // dec_max_position, so in-spec batched decode is unchanged. const int n_ctx_cap = canary_context_ceiling(cc->n_ctx, hp); + // One decode budget for the whole batch (the step loop runs every row in + // lockstep), sized from the longest surviving utterance. Same rule as run(). + const int max_new = transcribe::pick_decode_budget(T_enc_max, k_gen_reserve, prompt_len, n_ctx_cap); + int max_n_kv = 1024; + while (max_n_kv < prompt_len + max_new) { + max_n_kv *= 2; + } if (max_n_kv > n_ctx_cap) { max_n_kv = n_ctx_cap; } diff --git a/src/arch/canary_qwen/model.cpp b/src/arch/canary_qwen/model.cpp index 16f53fe4..3fdf9fa8 100644 --- a/src/arch/canary_qwen/model.cpp +++ b/src/arch/canary_qwen/model.cpp @@ -28,6 +28,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-env.h" #include "transcribe-flash-policy.h" #include "transcribe-load-common.h" @@ -116,9 +117,12 @@ constexpr float kBnEps = 1e-5f; // TRANSCRIBE_ERR_INPUT_TOO_LONG; a transcript that fills the generation budget // before end-of-stream is flagged via transcribe_was_truncated(). -// Per-run generation budget. Keep in sync with the single-utterance and -// batched step loops below. -constexpr int k_max_new = 256; +// Generation reserve, in tokens: the room the up-front input gate always +// keeps free for output, the floor under the per-run decode budget, and the +// value transcribe_capabilities::max_audio_ms subtracts. The actual per-run +// budget scales with the audio (see transcribe-decode-budget.h); this is only +// its lower bound, so a short clip decodes exactly as it always has. +constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum // (decoder.max_position_embeddings, e.g. 40960), optionally lowered — never @@ -148,7 +152,7 @@ int64_t canary_qwen_max_audio_ms(const CanaryQwenHParams & hp) { // counts; ~14 for canary_qwen). Advisory headroom, generous enough to // cover small template drift. constexpr int k_prompt_overhead = 32; - const int max_audio_tokens = hp.dec_max_position - k_prompt_overhead - k_max_new; + const int max_audio_tokens = hp.dec_max_position - k_prompt_overhead - k_gen_reserve; if (max_audio_tokens <= 0) { return 0; } @@ -554,7 +558,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->limits.has_context_cap = true; m->limits.model_max_ctx = m->hparams.dec_max_position; m->limits.prompt_overhead = 32; - m->limits.gen_reserve = k_max_new; + m->limits.gen_reserve = k_gen_reserve; // audio_tokens ≈ mel_frames / subsampling_factor ; // mel_frames = ms*sr/(hop*1000) m->limits.ms_per_audio_token = static_cast(m->hparams.enc_subsampling_factor) * @@ -910,22 +914,27 @@ transcribe_status run(transcribe_session * context, // Input-length gate: audio + prompt + generation must fit the decoder // context window. Reject an over-length clip here, before prefill/decode. const int ceiling = canary_qwen_context_ceiling(cc->n_ctx, hp); - if (T_prompt + k_max_new > ceiling) { + if (T_prompt + k_gen_reserve > ceiling) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "canary_qwen run: input too long — %d audio + %d prompt tokens " "leave no room for output within the %d-token context (need %d). " "Shorten the audio (see transcribe_capabilities.max_audio_ms) or " "split it into segments.", - T_enc, prefix_len + suffix_len, ceiling, T_prompt + k_max_new); + T_enc, prefix_len + suffix_len, ceiling, T_prompt + k_gen_reserve); return TRANSCRIBE_ERR_INPUT_TOO_LONG; } + // Per-run decode budget: scales with the audio, floored at the reserve the + // gate above just guaranteed, clamped to the context left. Replaces a flat + // 256-token cap that truncated long clips with context still free. + const int max_new = transcribe::pick_decode_budget(T_enc, k_gen_reserve, T_prompt, ceiling); + // KV cache init (grow-to-fit, clamped to the context ceiling). Size to - // hold prompt + generation budget, rounded up to a power of two (the step + // hold prompt + decode budget, rounded up to a power of two (the step // graph's flash-attn path wants pow2 attention width). A pre-allocated // smaller cache is freed and re-allocated. int want_n_ctx = 1024; - while (want_n_ctx < T_prompt + k_max_new) { + while (want_n_ctx < T_prompt + max_new) { want_n_ctx *= 2; } if (want_n_ctx > ceiling) { @@ -1073,7 +1082,6 @@ transcribe_status run(transcribe_session * context, // Step loop. const int32_t eos_id = hp.eos_token_id; - const int max_new = k_max_new; int cur_past = T_prompt; int max_n_kv = 1024; @@ -1448,13 +1456,13 @@ transcribe_status run_batch(transcribe_session * session, // Input-length gate (same as single-shot run()); reject this utterance, // the rest of the batch still runs. - if (T_prompt[b] + k_max_new > ceiling) { + if (T_prompt[b] + k_gen_reserve > ceiling) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "canary_qwen run_batch: utterance %d input too long — %d audio " "+ %d prompt tokens leave no room for output within the " "%d-token context (need %d). Shorten the audio (see " "transcribe_capabilities.max_audio_ms) or split it.", - b, T_enc[b], T_prompt[b] - T_enc[b], ceiling, T_prompt[b] + k_max_new); + b, T_enc[b], T_prompt[b] - T_enc[b], ceiling, T_prompt[b] + k_gen_reserve); fail_status[b] = TRANSCRIBE_ERR_INPUT_TOO_LONG; continue; } @@ -1477,7 +1485,9 @@ transcribe_status run_batch(transcribe_session * session, return TRANSCRIBE_OK; } T_enc_max = std::max(1, T_enc_max); - const int max_new = k_max_new; + // One decode budget for the whole batch (the step loop runs every row in + // lockstep), sized from the longest surviving utterance. Same rule as run(). + const int max_new = transcribe::pick_decode_budget(T_enc_max, k_gen_reserve, max_T_prompt, ceiling); int max_n_kv = 1024; while (max_n_kv < max_T_prompt + max_new) { max_n_kv *= 2; diff --git a/src/arch/cohere/model.cpp b/src/arch/cohere/model.cpp index a305756f..a4aafa63 100644 --- a/src/arch/cohere/model.cpp +++ b/src/arch/cohere/model.cpp @@ -15,6 +15,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-env.h" #include "transcribe-flash-policy.h" #include "transcribe-load-common.h" @@ -416,6 +417,12 @@ transcribe_status promote_conv_pw_to_f32_on_cpu(CohereModel & m) { constexpr const char k_default_variant[] = "cohere-asr"; +// Generation reserve, in tokens: the floor under the per-run decode budget. +// The budget itself scales with the audio and is clamped to the decoder +// self-KV ceiling (see transcribe-decode-budget.h), so a short clip decodes +// exactly as it always has while a long one is no longer cut at a flat 512. +constexpr int k_gen_reserve = 512; + // Forward declarations for the Arch trait below. extern transcribe_status load(Loader &, const transcribe_model_load_params *, transcribe_model **); extern transcribe_status init_context(transcribe_model *, const transcribe_session_params *, transcribe_session **); @@ -477,7 +484,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par // Fixed control-token preamble (see run()'s prompt_pieces). Audio is // in cross-KV, so there is no audio-token overhead here. m->limits.prompt_overhead = 10; - m->limits.gen_reserve = 512; // max-new-tokens cap in run() + m->limits.gen_reserve = k_gen_reserve; // ms-per-audio-token = subsampling_factor * hop_length * 1000 / sr. m->limits.ms_per_audio_token = static_cast(m->hparams.enc_subsampling_factor) * m->hparams.fe_hop_length * 1000.0 / m->hparams.fe_sample_rate; @@ -1039,7 +1046,7 @@ transcribe_status run(transcribe_session * session, // fallback is needed here. See the tokenizer.eos_id() check // in cohere::load() at the top of this file. const int eos_id = cm->hparams.eos_token_id; - const int max_tokens = std::min(512, cc->kv_cache.n_ctx - prompt_len); + const int max_tokens = transcribe::pick_decode_budget(T_enc, k_gen_reserve, prompt_len, cc->kv_cache.n_ctx); // Pick the first generated token. Fast path reads a single // int32 argmax that the GPU computed; debug path reads the @@ -1563,14 +1570,16 @@ transcribe_status run_batch(transcribe_session * session, } // ----- Allocate batched KV cache ----- - const int max_new = std::min(512, /*budget*/ 4096); - int max_n_kv = 1024; - while (max_n_kv < prompt_len + max_new) { - max_n_kv *= 2; - } // Honor the session context cap (same ceiling the single-shot path uses), // not the raw model max — so a lowered n_ctx bounds batch decoder KV too. const int n_ctx_cap = cohere_dec_ctx_ceiling(cc->n_ctx, hp); + // One decode budget for the whole batch (the step loop runs every row in + // lockstep), sized from the longest surviving utterance. Same rule as run(). + const int max_new = transcribe::pick_decode_budget(T_enc_max, k_gen_reserve, prompt_len, n_ctx_cap); + int max_n_kv = 1024; + while (max_n_kv < prompt_len + max_new) { + max_n_kv *= 2; + } if (max_n_kv > n_ctx_cap) { max_n_kv = n_ctx_cap; } diff --git a/src/arch/funasr_nano/model.cpp b/src/arch/funasr_nano/model.cpp index 436ec1ce..2ba1652a 100644 --- a/src/arch/funasr_nano/model.cpp +++ b/src/arch/funasr_nano/model.cpp @@ -13,6 +13,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-flash-policy.h" #include "transcribe-kaldi-fbank.h" #include "transcribe-load-common.h" @@ -80,8 +81,12 @@ constexpr const char k_default_variant[] = "fun-asr-nano-2512"; // transcribe_was_truncated(). // --------------------------------------------------------------------------- -// Per-run generation budget. -constexpr int k_max_new = 256; +// Generation reserve, in tokens: the room the up-front input gate always +// keeps free for output, the floor under the per-run decode budget, and the +// value transcribe_capabilities::max_audio_ms subtracts. The actual per-run +// budget scales with the audio (see transcribe-decode-budget.h); this is only +// its lower bound, so a short clip decodes exactly as it always has. +constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum, // optionally lowered — never raised — by the caller's session n_ctx knob. @@ -103,7 +108,7 @@ int64_t funasr_nano_max_audio_ms(const FunAsrNanoHParams & hp) { return 0; } constexpr int k_prompt_overhead = 48; // chat affixes; advisory - const int max_audio_tokens = hp.dec_max_position_embeddings - k_prompt_overhead - k_max_new; + const int max_audio_tokens = hp.dec_max_position_embeddings - k_prompt_overhead - k_gen_reserve; if (max_audio_tokens <= 0) { return 0; } @@ -309,7 +314,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->limits.has_context_cap = true; m->limits.model_max_ctx = m->hparams.dec_max_position_embeddings; m->limits.prompt_overhead = 48; - m->limits.gen_reserve = k_max_new; + m->limits.gen_reserve = k_gen_reserve; m->limits.ms_per_audio_token = static_cast(folds) * m->hparams.fe_lfr_n * m->hparams.fe_hop_length * 1000.0 / m->hparams.fe_sample_rate; m->limits.kv_elems_per_ctx_token = @@ -666,23 +671,28 @@ transcribe_status run(transcribe_session * session, // fixed by the input length, so reject an over-length clip here, before // KV alloc / prefill / decode, instead of walling at a fixed size. const int ceiling = funasr_nano_context_ceiling(cc->n_ctx, hp); - if (T_prompt + k_max_new > ceiling) { + if (T_prompt + k_gen_reserve > ceiling) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "funasr_nano run: input too long — %d audio + %d prompt tokens " "leave no room for output within the %d-token context (need %d). " "Shorten the audio (see transcribe_capabilities.max_audio_ms) or " "split it into segments.", - T_audio, prefix_len + suffix_len, ceiling, T_prompt + k_max_new); + T_audio, prefix_len + suffix_len, ceiling, T_prompt + k_gen_reserve); return TRANSCRIBE_ERR_INPUT_TOO_LONG; } + // Per-run decode budget: scales with the audio, floored at the reserve the + // gate above just guaranteed, clamped to the context left. Replaces a flat + // 256-token cap that truncated long clips with context still free. + const int max_new = transcribe::pick_decode_budget(T_audio, k_gen_reserve, T_prompt, ceiling); + // ---- KV cache init (grow-to-fit, clamped to the context ceiling) ---- - // Size to hold the prompt plus the generation budget, rounded up to a - // power of two (the step graph's attention width wants pow2 for the fast - // flash-attn path). The cache grows across runs as audio length demands; - // a pre-allocated smaller cache is freed and re-allocated. + // Size to hold the prompt plus the decode budget, rounded up to a power of + // two (the step graph's attention width wants pow2 for the fast flash-attn + // path). The cache grows across runs as audio length demands; a + // pre-allocated smaller cache is freed and re-allocated. int want_n_ctx = 1024; - while (want_n_ctx < T_prompt + k_max_new) { + while (want_n_ctx < T_prompt + max_new) { want_n_ctx *= 2; } if (want_n_ctx > ceiling) { @@ -817,7 +827,6 @@ transcribe_status run(transcribe_session * session, // ---- Step loop ---- const int32_t eos_id = hp.eos_token_id; - const int max_new = k_max_new; int cur_past = T_prompt; // Static step-graph shape: T_prompt prefilled + up to max_new generated. @@ -1142,25 +1151,26 @@ transcribe_status run_batch(transcribe_session * session, // Input-length gate (see docs/input-limits.md). Audio tokens + prompt + // generation must fit the decoder context window; reject an over-length // utterance here instead of walling at a fixed KV size. Mirrors the - // single-shot run() gate (T_prompt + k_max_new > ceiling). - if (T_prompt[b] + k_max_new > ceiling) { + // single-shot run() gate (T_prompt + k_gen_reserve > ceiling). + if (T_prompt[b] + k_gen_reserve > ceiling) { const int suffix = T_prompt[b] - fbank_beg - T_audio[b]; transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "funasr_nano run_batch: utterance %d input too long — %d audio + " "%d prompt tokens leave no room for output within the %d-token " "context (need %d). Shorten the audio (see " "transcribe_capabilities.max_audio_ms) or split it.", - b, T_audio[b], fbank_beg + suffix, ceiling, T_prompt[b] + k_max_new); + b, T_audio[b], fbank_beg + suffix, ceiling, T_prompt[b] + k_gen_reserve); fail_status[b] = TRANSCRIBE_ERR_INPUT_TOO_LONG; continue; } valid[b] = 1; } - int max_T_prompt = 0; + int max_T_prompt = 0, max_T_audio = 0; for (int b = 0; b < n; ++b) { if (valid[b]) { max_T_prompt = std::max(max_T_prompt, T_prompt[b]); + max_T_audio = std::max(max_T_audio, T_audio[b]); } } if (max_T_prompt == 0) { @@ -1171,7 +1181,9 @@ transcribe_status run_batch(transcribe_session * session, } return TRANSCRIBE_OK; } - const int max_new = 256; + // One decode budget for the whole batch (the step loop runs every row in + // lockstep), sized from the longest surviving utterance. Same rule as run(). + const int max_new = transcribe::pick_decode_budget(max_T_audio, k_gen_reserve, max_T_prompt, ceiling); int max_n_kv = 1024; while (max_n_kv < max_T_prompt + max_new) { max_n_kv *= 2; diff --git a/src/arch/granite/model.cpp b/src/arch/granite/model.cpp index 95d182e6..2149155a 100644 --- a/src/arch/granite/model.cpp +++ b/src/arch/granite/model.cpp @@ -11,6 +11,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-env.h" #include "transcribe-flash-policy.h" #include "transcribe-load-common.h" @@ -80,9 +81,12 @@ constexpr float kBnEps = 1e-5f; // Over-length input is rejected up front with TRANSCRIBE_ERR_INPUT_TOO_LONG // rather than silently aliasing RoPE past the trained range. -// Generation budget reserved per run. Also the KV grow-to-fit step budget, -// so an accepted clip always has room for up to this many output tokens. -constexpr int k_gen_budget = 256; +// Generation reserve, in tokens: the room the up-front input gate always +// keeps free for output, the floor under the per-run decode budget, and the +// value transcribe_capabilities::max_audio_ms subtracts. The actual per-run +// budget scales with the audio (see transcribe-decode-budget.h); this is only +// its lower bound, so a short clip decodes exactly as it always has. +constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum, // optionally lowered — never raised — by the caller's session n_ctx knob. @@ -107,7 +111,7 @@ int granite_num_queries(const GraniteHParams & hp) { // audio tokens, a representative prompt, and the generation reserve still fit // the context ceiling. This is the input bound the gate enforces; transcripts // of long-but-fitting audio may still truncate (transcribe_was_truncated) -// because the per-run output is bounded by k_gen_budget. Returns 0 ("unknown +// because the per-run output is bounded by the decode budget. Returns 0 ("unknown // / unbounded") if the rate constants are missing, so a misconfigured model // is never advertised with a wrong finite number. int64_t granite_max_audio_ms(const GraniteHParams & hp) { @@ -118,7 +122,7 @@ int64_t granite_max_audio_ms(const GraniteHParams & hp) { } // Representative non-audio prompt overhead (chat affixes); advisory. constexpr int k_prompt_overhead = 64; - const int max_audio_tokens = hp.dec_max_position_embeddings - k_prompt_overhead - k_gen_budget; + const int max_audio_tokens = hp.dec_max_position_embeddings - k_prompt_overhead - k_gen_reserve; if (max_audio_tokens <= 0) { return 0; } @@ -305,7 +309,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->limits.has_context_cap = true; m->limits.model_max_ctx = m->hparams.dec_max_position_embeddings; m->limits.prompt_overhead = 64; // match granite_max_audio_ms's k_prompt_overhead - m->limits.gen_reserve = k_gen_budget; + m->limits.gen_reserve = k_gen_reserve; // ms per audio token: granite emits num_queries tokens per // window_size encoder frames; t_enc = mel_frames/2; // mel_frames = ms*sr/(hop*1000). Inverting granite_max_audio_ms's @@ -1039,16 +1043,21 @@ transcribe_status run(transcribe_session * ctx_base, // aliasing RoPE past the trained range. Reserving the full generation // budget means an accepted clip always has room for a real transcript. const int ceiling = granite_context_ceiling(cc->n_ctx, cm->hparams); - if (T_prompt + k_gen_budget > ceiling) { + if (T_prompt + k_gen_reserve > ceiling) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "granite run: input too long — %d audio + %d prompt tokens leave " "no room for output within the %d-token context (need %d). " "Shorten the audio (see transcribe_capabilities.max_audio_ms) or " "split it into segments.", - n_audio_tokens, prefix_len + suffix_len, ceiling, T_prompt + k_gen_budget); + n_audio_tokens, prefix_len + suffix_len, ceiling, T_prompt + k_gen_reserve); return TRANSCRIBE_ERR_INPUT_TOO_LONG; } + // Per-run decode budget: scales with the audio, floored at the reserve the + // gate above just guaranteed, clamped to the context left. Replaces a flat + // 256-token cap that truncated long clips with context still free. + const int gen_budget = transcribe::pick_decode_budget(n_audio_tokens, k_gen_reserve, T_prompt, ceiling); + // Size the KV cache dynamically: T_prompt + room for the longest // generation we'll emit, clamped to the context ceiling. Matches the // HF reference's DynamicCache semantics (grows as needed) without @@ -1057,7 +1066,7 @@ transcribe_status run(transcribe_session * ctx_base, // so back-to-back runs of similar audio lengths don't keep // re-allocating. constexpr int kKvBucket = 256; - const int needed_raw = std::min(T_prompt + k_gen_budget, ceiling); + const int needed_raw = std::min(T_prompt + gen_budget, ceiling); const int needed_n_ctx = ((needed_raw + kKvBucket - 1) / kKvBucket) * kKvBucket; if (cc->kv.self_k != nullptr && cc->kv.n_ctx < needed_n_ctx) { @@ -1196,10 +1205,10 @@ transcribe_status run(transcribe_session * ctx_base, // n_ctx of the KV cache bounds the max generation length we can // attend over. const int max_n_kv = cc->kv.n_ctx; - // Bound generation by the step budget, the allocated cache, AND the - // context ceiling (the gate guarantees ceiling - T_prompt >= k_gen_budget, - // so for in-spec input this stays k_gen_budget and decode is unchanged). - const int max_steps = std::min({ k_gen_budget, max_n_kv - T_prompt, ceiling - T_prompt }); + // Bound generation by the decode budget, the allocated cache, AND the + // context ceiling. gen_budget is already clamped to ceiling - T_prompt; + // the other two terms guard the cache the bucket rounding actually gave us. + const int max_steps = std::min({ gen_budget, max_n_kv - T_prompt, ceiling - T_prompt }); ggml_context * step_ctx = nullptr; { @@ -1572,13 +1581,13 @@ transcribe_status run_batch(transcribe_session * session, T_prompt[b] = static_cast(prompt_ids[b].size()); // Input-length gate, mirroring the single-shot run() gate. - if (T_prompt[b] + k_gen_budget > ceiling) { + if (T_prompt[b] + k_gen_reserve > ceiling) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "granite run_batch: utterance %d input too long — %d audio + %d " "prompt tokens leave no room for output within the %d-token " "context (need %d). Shorten the audio (see " "transcribe_capabilities.max_audio_ms) or split it.", - b, n_audio[b], T_prompt[b] - n_audio[b], ceiling, T_prompt[b] + k_gen_budget); + b, n_audio[b], T_prompt[b] - n_audio[b], ceiling, T_prompt[b] + k_gen_reserve); fail_status[b] = TRANSCRIBE_ERR_INPUT_TOO_LONG; continue; } @@ -1601,7 +1610,9 @@ transcribe_status run_batch(transcribe_session * session, return TRANSCRIBE_OK; } n_audio_max = std::max(1, n_audio_max); - const int max_new = 256; + // One decode budget for the whole batch (the step loop runs every row in + // lockstep), sized from the longest surviving utterance. Same rule as run(). + const int max_new = transcribe::pick_decode_budget(n_audio_max, k_gen_reserve, max_T_prompt, ceiling); int max_n_kv = 1024; while (max_n_kv < max_T_prompt + max_new) { max_n_kv *= 2; diff --git a/src/arch/moss/model.cpp b/src/arch/moss/model.cpp index 3cacf0ff..1789142d 100644 --- a/src/arch/moss/model.cpp +++ b/src/arch/moss/model.cpp @@ -16,6 +16,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-env.h" #include "transcribe-flash-policy.h" #include "transcribe-load-common.h" @@ -759,10 +760,12 @@ transcribe_status run(transcribe_session * session, return TRANSCRIBE_ERR_INPUT_TOO_LONG; } - // Generation budget scales with audio length: the emergent transcript - // (text + [start]/[Sxx]/[end] markers) tracks the audio-token count, which - // for long-form far exceeds the k_max_new floor. Clamp to the context. - const int gen_budget = std::min(ceiling - T_prompt, std::max(k_max_new, 2 * T_enc + 128)); + // Generation budget scales with audio length via the shared rule + // (transcribe-decode-budget.h). moss predicts higher than the plain + // audio-token count because the emergent transcript carries speaker + // markers ([start]/[Sxx]/[end]) on top of the text; for long-form that + // far exceeds the k_max_new floor. Clamped to the context left. + const int gen_budget = transcribe::pick_decode_budget(2 * T_enc + 128, k_max_new, T_prompt, ceiling); // KV cache (grow-to-fit, clamped to ceiling). Short inputs retain the old // 1K/2K/4K buckets; longer ones grow in 4K steps so crossing 32K does not diff --git a/src/arch/qwen3_asr/model.cpp b/src/arch/qwen3_asr/model.cpp index 423f3b38..3460c90f 100644 --- a/src/arch/qwen3_asr/model.cpp +++ b/src/arch/qwen3_asr/model.cpp @@ -10,6 +10,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-env.h" #include "transcribe-flash-policy.h" #include "transcribe-load-common.h" @@ -74,8 +75,12 @@ constexpr const char k_default_variant[] = "qwen3-asr"; // transcript that fills the generation budget before end-of-stream is flagged // via transcribe_was_truncated(). -// Per-run generation budget (matches the reference dumper default). -constexpr int k_max_new = 256; +// Generation reserve, in tokens: the room the up-front input gate always +// keeps free for output, the floor under the per-run decode budget, and the +// value transcribe_capabilities::max_audio_ms subtracts. The actual per-run +// budget scales with the audio (see transcribe-decode-budget.h); this is only +// its lower bound, so a short clip decodes exactly as it always has. +constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum, // optionally lowered — never raised — by the caller's session n_ctx knob. @@ -99,7 +104,7 @@ int64_t qwen3_max_audio_ms(const QwenAsrHParams & hp) { return 0; } constexpr int k_prompt_overhead = 48; // chat affixes; advisory - const int max_audio_tokens = hp.dec_max_position_embeddings - k_prompt_overhead - k_max_new; + const int max_audio_tokens = hp.dec_max_position_embeddings - k_prompt_overhead - k_gen_reserve; if (max_audio_tokens <= 0) { return 0; } @@ -160,7 +165,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->limits.has_context_cap = true; m->limits.model_max_ctx = m->hparams.dec_max_position_embeddings; m->limits.prompt_overhead = 48; - m->limits.gen_reserve = k_max_new; + m->limits.gen_reserve = k_gen_reserve; // audio_tokens ≈ mel_frames / 8 ; mel_frames = ms*sr/(hop*1000) m->limits.ms_per_audio_token = 8.0 * m->hparams.fe_hop_length * 1000.0 / m->hparams.fe_sample_rate; m->limits.kv_elems_per_ctx_token = @@ -726,22 +731,29 @@ transcribe_status run(transcribe_session * session, // Input-length gate: audio + prompt + generation must fit the decoder // context window. Reject an over-length clip here, before prefill/decode. const int ceiling = qwen3_context_ceiling(cc->n_ctx, cm->hparams); - if (T_prompt + k_max_new > ceiling) { + if (T_prompt + k_gen_reserve > ceiling) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "qwen3_asr run: input too long — %d audio + %d prompt tokens " "leave no room for output within the %d-token context (need %d). " "Shorten the audio (see transcribe_capabilities.max_audio_ms) or " "split it into segments.", - T_enc, prefix_len + suffix_len, ceiling, T_prompt + k_max_new); + T_enc, prefix_len + suffix_len, ceiling, T_prompt + k_gen_reserve); return TRANSCRIBE_ERR_INPUT_TOO_LONG; } + // Per-run decode budget. Scales with the audio (a transcript never needs + // more text tokens than the encoder produced audio tokens), floored at the + // reserve the gate above just guaranteed and clamped to the context left. + // This replaces a flat 256-token cap that truncated every clip past ~75 s + // of speech regardless of how much context was still free. + const int max_new = transcribe::pick_decode_budget(T_enc, k_gen_reserve, T_prompt, ceiling); + // KV cache init (grow-to-fit, clamped to the context ceiling). Size to - // hold prompt + generation budget, rounded up to a power of two (the step + // hold prompt + decode budget, rounded up to a power of two (the step // graph's flash-attn path wants pow2 attention width). A pre-allocated // smaller cache is freed and re-allocated. int want_n_ctx = 1024; - while (want_n_ctx < T_prompt + k_max_new) { + while (want_n_ctx < T_prompt + max_new) { want_n_ctx *= 2; } if (want_n_ctx > ceiling) { @@ -869,9 +881,8 @@ transcribe_status run(transcribe_session * session, generated_ids.push_back(next_tok); t_prefill_logits_us = ggml_time_us() - t_prefill_logits_start; - // Step loop. + // Step loop. max_new is the per-run decode budget resolved above. const int32_t eos_id = cm->hparams.eos_token_id; - const int32_t max_new = k_max_new; int cur_past = T_prompt; // Build the step graph ONCE and reuse every step, sized for the actual @@ -1544,8 +1555,8 @@ transcribe_status run_batch(transcribe_session * session, // Prompt length bound → max_n_kv and batched-cache n_ctx. Build and keep // each utterance's prompt token ids for the batched prefill. - const int max_new = 256; int max_T_prompt = 0; + int max_T_enc = 0; int prefix_len = 0; // Per-utterance terminal status for rejected rows. Defaults to INVALID_ARG; // over-length rows below are upgraded to INPUT_TOO_LONG. @@ -1561,7 +1572,7 @@ transcribe_status run_batch(transcribe_session * session, T_prompt[b] = static_cast(prompt_ids[b].size()); prefix_len = ap.empty() ? 0 : static_cast(ap.front()); // Same gate as single-shot run(); the rest of the batch still runs. - if (T_prompt[b] + max_new > ceiling) { + if (T_prompt[b] + k_gen_reserve > ceiling) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "qwen3_asr run_batch: utterance %d input too long — %d audio + " "%d prompt tokens exceed the %d-token context. See " @@ -1572,6 +1583,7 @@ transcribe_status run_batch(transcribe_session * session, continue; } max_T_prompt = std::max(max_T_prompt, T_prompt[b]); + max_T_enc = std::max(max_T_enc, T_enc[b]); } if (max_T_prompt == 0) { // No usable utterance — emit per-row errors and return. @@ -1582,7 +1594,11 @@ transcribe_status run_batch(transcribe_session * session, } return TRANSCRIBE_OK; } - int max_n_kv = 1024; + // One decode budget for the whole batch (the step loop runs every row in + // lockstep), sized from the longest surviving utterance so no row is cut + // short. Same rule as single-shot run(). + const int max_new = transcribe::pick_decode_budget(max_T_enc, k_gen_reserve, max_T_prompt, ceiling); + int max_n_kv = 1024; while (max_n_kv < max_T_prompt + max_new) { max_n_kv *= 2; } diff --git a/src/arch/voxtral/model.cpp b/src/arch/voxtral/model.cpp index e490742b..320e2fc5 100644 --- a/src/arch/voxtral/model.cpp +++ b/src/arch/voxtral/model.cpp @@ -19,6 +19,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-flash-policy.h" #include "transcribe-load-common.h" #include "transcribe-loader.h" @@ -80,17 +81,13 @@ constexpr const char k_default_variant[] = "voxtral-mini-3b-2507"; constexpr int k_decode_budget_min = 448; // Decode budget (max new text tokens) for an utterance with `n_audio` audio -// embedding tokens. Speech yields fewer text tokens than audio frames, so the -// audio token count is a safe upper bound; clamp to the context remaining under -// the trained max so prompt+decode fits. Greedy decode stops at EOS well before -// this, so a generous ceiling costs only its KV allocation. +// embedding tokens. Thin wrapper over the shared rule every autoregressive +// family now uses (transcribe-decode-budget.h): the audio-token count is a safe +// upper bound on the transcript, floored at k_decode_budget_min and clamped to +// the context remaining under the trained max. Greedy decode stops at EOS well +// before this, so a generous ceiling costs only its KV allocation. int pick_decode_budget(int n_audio, int t_prompt, int model_max) { - int budget = std::max(k_decode_budget_min, n_audio); - const int room = model_max - t_prompt; - if (budget > room) { - budget = room; - } - return budget; + return transcribe::pick_decode_budget(n_audio, k_decode_budget_min, t_prompt, model_max); } // Chunked prefill — see decoder.h. Walks the prompt in blocks against the diff --git a/src/transcribe-decode-budget.h b/src/transcribe-decode-budget.h new file mode 100644 index 00000000..bb5e4bae --- /dev/null +++ b/src/transcribe-decode-budget.h @@ -0,0 +1,58 @@ +// transcribe-decode-budget.h - shared per-run autoregressive decode budget. +// +// INTERNAL. Header-only, like the ABI helpers in transcribe-abi.h. +// +// Autoregressive families used to hardcode their generation budget as a +// constant (256 / 512 tokens) that did not depend on the audio at all, while +// the up-front input gate accepted clips orders of magnitude longer than that +// many tokens could describe. Any clip whose natural transcript outran the +// constant came back as TRANSCRIBE_ERR_OUTPUT_TRUNCATED with a partial +// transcript, even though the decoder context had ample room left. See +// docs/input-limits.md. +// +// The budget must track the input instead. Speech yields fewer text tokens +// than the encoder yields audio tokens, so the audio-token count is a safe +// upper bound on the transcript length — the estimate voxtral has shipped +// with since its introduction, generalized here so every family shares it. +// +// This is deliberately NOT a public run parameter. The only caller-facing +// knob is transcribe_session_params::n_ctx, which lowers `ceiling` and so +// lowers the budget with it. + +#pragma once + +#include + +namespace transcribe { + +// Per-run decode budget, in transcript tokens. +// +// predicted the family's upper-bound estimate of transcript length, +// in tokens. For most families this is the audio-token +// count; a family whose output carries more than the +// transcript (moss emits speaker markers) scales it up. +// floor_tokens never plan for fewer than this. Each family passes its +// historical fixed budget, so a clip that fits today keeps +// byte-identical behavior, and the generation reserve that +// transcribe_capabilities::max_audio_ms subtracts (via +// transcribe_model::LimitsBasis::gen_reserve) stays exact. +// t_prompt prompt tokens already committed to the decoder context. +// Includes the audio embeddings for families that put audio +// in-context; 0 for encoder-decoder families whose audio +// lives in a separate cross-attention cache. +// ceiling decoder context ceiling in tokens, already lowered (never +// raised) by transcribe_session_params::n_ctx. +// +// Returns the budget clamped to the context actually left, never negative. +// A zero return means the prompt already fills the ceiling; callers gate +// that case up front (INPUT_TOO_LONG) rather than entering the step loop. +inline int pick_decode_budget(int predicted, int floor_tokens, int t_prompt, int ceiling) { + int budget = std::max(floor_tokens, predicted); + const int room = ceiling - t_prompt; + if (budget > room) { + budget = room; + } + return budget > 0 ? budget : 0; +} + +} // namespace transcribe diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 85b47282..3f8267bd 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -201,6 +201,22 @@ transcribe_apply_warnings(transcribe_prefill_chunk_mask_unit) add_test(NAME transcribe_prefill_chunk_mask_unit COMMAND transcribe_prefill_chunk_mask_unit) +# ----------------------------------------------------------------------------- +# Per-run decode budget rule (pure host, no model) +# ----------------------------------------------------------------------------- + +add_executable(transcribe_decode_budget_unit + decode_budget_unit.cpp) + +target_link_libraries(transcribe_decode_budget_unit PRIVATE transcribe ggml) + +target_include_directories(transcribe_decode_budget_unit PRIVATE + ${CMAKE_SOURCE_DIR}/src) + +transcribe_apply_warnings(transcribe_decode_budget_unit) + +add_test(NAME transcribe_decode_budget_unit COMMAND transcribe_decode_budget_unit) + # ----------------------------------------------------------------------------- # MOSS diarized-transcript parser unit test (pure host, no model) # ----------------------------------------------------------------------------- diff --git a/tests/decode_budget_unit.cpp b/tests/decode_budget_unit.cpp new file mode 100644 index 00000000..b75c2b34 --- /dev/null +++ b/tests/decode_budget_unit.cpp @@ -0,0 +1,92 @@ +// Per-run decode budget rule (pure host, no model). +// +// Every autoregressive family used to hardcode its generation budget as a +// constant that ignored the audio entirely (qwen3_asr 256, canary 512, ...). +// The up-front input gate meanwhile accepted clips orders of magnitude longer +// than that many tokens could describe — qwen3_asr advertises 87 minutes of +// audio against a 256-token output cap — so any clip past roughly a minute of +// speech came back TRANSCRIBE_ERR_OUTPUT_TRUNCATED with context to spare. +// +// transcribe::pick_decode_budget replaces those constants. This test pins the +// two properties the families depend on, because getting either wrong is +// silent: too low and long clips truncate again; too high and the KV +// allocation (112 KiB per token on qwen3-asr) balloons on every run. +// +// 1. Never below the family's floor -> a clip that fits today is unchanged, +// and transcribe_capabilities::max_audio_ms (which subtracts that same +// floor via LimitsBasis::gen_reserve) stays exact. +// 2. Never past the context left -> prompt + budget always fits the +// ceiling, which is what transcribe_session_params::n_ctx lowers. + +#include "transcribe-decode-budget.h" + +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +void check_budget(const char * what, int predicted, int floor_tokens, int t_prompt, int ceiling, int expected) { + const int got = transcribe::pick_decode_budget(predicted, floor_tokens, t_prompt, ceiling); + if (got != expected) { + std::fprintf(stderr, "FAIL %s: pick_decode_budget(%d, %d, %d, %d) = %d, expected %d\n", what, predicted, + floor_tokens, t_prompt, ceiling, got, expected); + ++g_failures; + } +} + +} // namespace + +int main(void) { + // ---- Property 1: the floor holds for short audio. ---- + // A clip whose audio-token count is below the family's historical fixed + // budget must still get that budget, so its decode is byte-identical to + // what shipped before. + check_budget("short clip keeps the floor", /*predicted=*/10, /*floor=*/256, /*t_prompt=*/64, + /*ceiling=*/65536, /*expected=*/256); + check_budget("floor applies at zero audio", 0, 256, 64, 65536, 256); + check_budget("canary floor", 300, 512, 6, 1024, 512); + + // ---- Property 2: the budget scales past the floor. ---- + // This is the fix. qwen3-asr at 80 ms per audio token: a 5-minute clip is + // 3824 audio tokens, which used to decode under a flat 256-token cap. + check_budget("5 min qwen3-asr scales", /*predicted=*/3824, /*floor=*/256, /*t_prompt=*/3872, + /*ceiling=*/65536, /*expected=*/3824); + check_budget("20 min qwen3-asr scales", 15000, 256, 15048, 65536, 15000); + + // ---- Property 3: the context ceiling always wins. ---- + // canary is the tight case: a 400 s clip is ~5000 encoder frames but the + // decoder self-KV is only 1024, so the budget clamps to what is left. + check_budget("canary clamps to dec ctx", /*predicted=*/5000, /*floor=*/512, /*t_prompt=*/6, + /*ceiling=*/1024, /*expected=*/1018); + check_budget("clamp beats the floor too", 10, 512, 900, 1024, 124); + + // ---- Property 4: n_ctx is the knob. ---- + // Lowering transcribe_session_params::n_ctx lowers `ceiling`, and the + // budget must follow it down. Same inputs as the 5-minute case above. + check_budget("full n_ctx leaves the audio-sized budget intact", 3824, 256, 3872, 8192, 3824); + check_budget("lowered n_ctx lowers the budget", 3824, 256, 3872, 6000, 2128); + check_budget("n_ctx below the floor still clamps", 3824, 256, 3872, 4000, 128); + + // ---- Property 5: never negative. ---- + // A prompt that already fills the ceiling yields 0, not a negative step + // count. Families gate this case up front with INPUT_TOO_LONG; the helper + // must not hand a negative loop bound to a step loop regardless. + check_budget("prompt exactly fills ceiling", 3824, 256, 1024, 1024, 0); + check_budget("prompt overruns ceiling", 3824, 256, 2048, 1024, 0); + + if (g_failures > 0) { + std::fprintf(stderr, "decode_budget_unit: %d failures\n", g_failures); + return 1; + } + std::fprintf(stdout, "decode_budget_unit: ok\n"); + return 0; +} diff --git a/tests/qwen3_asr_batch_truncation.cpp b/tests/qwen3_asr_batch_truncation.cpp index ae3fb2e0..2f91eadd 100644 --- a/tests/qwen3_asr_batch_truncation.cpp +++ b/tests/qwen3_asr_batch_truncation.cpp @@ -2,14 +2,20 @@ // and batch decode paths both report mid-decode OUTPUT_TRUNCATED for a // causal_lm (LLM-decoder) family. // -// qwen3_asr caps generation at max_new = 256 tokens. A long speech clip passes -// the up-front input-length gate (its audio tokens fit the 65536-token decoder -// context with room to spare) but its natural transcript exceeds 256 tokens, so -// greedy decode hits the generation budget before EOS — the transcript is -// truncated. Per docs/input-limits.md that must surface as the hard -// TRANSCRIBE_ERR_OUTPUT_TRUNCATED status (partial transcript retained, -// transcribe_was_truncated() set) in BOTH paths, while a short clip that -// finishes under the budget stays OK and the whole-batch call still returns OK. +// qwen3_asr's decode budget scales with the audio and is clamped to the +// decoder context left after the prompt (transcribe-decode-budget.h). It used +// to be a flat 256 tokens, which truncated any clip past ~75 s of speech even +// with 65000 tokens of context free; that was the bug, and the first block +// below is its regression guard — a 197 s clip must now decode to EOS. +// +// Truncation is still reachable, and still has to be reported: lowering +// transcribe_session_params::n_ctx lowers the ceiling, which lowers the budget +// with it. That is the only knob a caller has over the output length, so it is +// also how this test forces the truncation path. Per docs/input-limits.md a +// truncated decode must surface as the hard TRANSCRIBE_ERR_OUTPUT_TRUNCATED +// status (partial transcript retained, transcribe_was_truncated() set) in BOTH +// the single-shot and batch paths, while a short clip that finishes under the +// budget stays OK and the whole-batch call still returns OK. // // This is the causal_lm counterpart to moonshine_streaming_batch_truncation // (which exercises the encoder-decoder batch loop in transcribe-batch-util.cpp). @@ -20,7 +26,7 @@ // truncated batch row silently report TRANSCRIBE_OK with an incomplete // transcript — the exact failure this test catches. // -// Batch makeup: +// Batch makeup (under the lowered n_ctx): // row 0 = jfk.wav (~11 s) -> completes under the budget -> OK // row 1 = love-loss.wav (~197 s) -> exceeds the budget -> OUTPUT_TRUNCATED // @@ -105,8 +111,35 @@ int main() { return 1; } + // ---- Regression guard for the flat-256 budget bug ---- + // At the default (full) context the 197 s clip must decode all the way to + // EOS. Before the budget scaled with the audio this returned + // OUTPUT_TRUNCATED at 256 tokens with ~63000 tokens of context unused. + { + transcribe_session_params full_sp; + transcribe_session_params_init(&full_sp); + struct transcribe_session * full_s = nullptr; + if (transcribe_session_init(model, &full_sp, &full_s) != TRANSCRIBE_OK) { + std::fprintf(stderr, "session init failed\n"); + transcribe_model_free(model); + return 1; + } + const transcribe_status rl = transcribe_run(full_s, pcm_long.data(), (int) pcm_long.size(), nullptr); + CHECK(rl == TRANSCRIBE_OK); + CHECK(transcribe_was_truncated(full_s) == false); + transcribe_session_free(full_s); + } + + // ---- Lowered n_ctx: the only caller-facing control over output length ---- + // love-loss.wav is ~197 s. qwen3_asr emits one audio token per 80 ms, so + // the prompt is ~2465 audio tokens plus ~15 chat-affix tokens. A 2816-token + // ceiling therefore clears the input gate (which reserves k_gen_reserve = + // 256 on top of the prompt) while leaving only ~340 tokens of decode + // budget — well under the ~700 this clip's transcript needs, so the decode + // runs into the budget and must report it. transcribe_session_params sp; transcribe_session_params_init(&sp); + sp.n_ctx = 2816; struct transcribe_session * s = nullptr; if (transcribe_session_init(model, &sp, &s) != TRANSCRIBE_OK) { std::fprintf(stderr, "session init failed\n"); @@ -114,9 +147,7 @@ int main() { return 1; } - // ---- Single-shot baseline: the long clip truncates, the short one does not. - // Both pass the input-length gate at the default (full) context; the long - // clip simply runs the decoder into the 256-token generation budget. + // ---- Single-shot: the long clip truncates, the short one does not. ---- { const transcribe_status rl = transcribe_run(s, pcm_long.data(), (int) pcm_long.size(), nullptr); CHECK(rl == TRANSCRIBE_ERR_OUTPUT_TRUNCATED); From 7062e243904f30d8953240ca26277b5effcc2143 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 20 Sep 2026 19:40:43 +0800 Subject: [PATCH 2/6] tweaks --- docs/input-limits.md | 27 ++++++++++++++----- src/arch/canary/model.cpp | 27 +++++++++++++------ src/arch/canary_qwen/model.cpp | 11 +++++--- src/arch/cohere/model.cpp | 12 ++++++--- src/arch/funasr_nano/model.cpp | 10 ++++--- src/arch/granite/model.cpp | 12 ++++++--- src/arch/qwen3_asr/model.cpp | 9 ++++--- src/transcribe-decode-budget.h | 39 ++++++++++++++++++++++++++++ tests/decode_budget_unit.cpp | 26 +++++++++++++++++++ tests/qwen3_asr_batch_truncation.cpp | 18 ++++++++----- 10 files changed, 152 insertions(+), 39 deletions(-) diff --git a/docs/input-limits.md b/docs/input-limits.md index 5d7e81eb..7723a4db 100644 --- a/docs/input-limits.md +++ b/docs/input-limits.md @@ -101,12 +101,19 @@ budget = clamp(max(generation_reserve, predicted_transcript_tokens), 0, ceiling - prompt_tokens) ``` -`predicted_transcript_tokens` is the encoder's audio-token count (speech never -yields more text tokens than the encoder yields audio tokens, so it is a safe -upper bound; moss scales it up because its output also carries speaker -markers). `generation_reserve` is the per-family floor — the same constant the -up-front gate reserves and `max_audio_ms` subtracts — so a short clip decodes -exactly as it always has. `ceiling` is the decoder context, which +`predicted_transcript_tokens` comes from the clip's **duration**, not its +audio-token count: `seconds x 12 tokens/sec`, where `seconds` is recovered from +the family's published encoder rate (`ms_per_audio_token`). The duration form +matters because encoder rates differ about 6x — most emit one audio token per +80 ms, but `funasr_nano`'s LFR frontend emits one per ~480 ms, which is *below* +the text-token rate, so its audio-token count under-predicts. 12 tokens/sec is +deliberately generous against a measured ~3.4 for English BPE, leaving room for +denser scripts. (`moss` and `voxtral` pass their own estimates instead: moss +scales up for the speaker markers its output carries.) + +`generation_reserve` is the per-family floor — the same constant the up-front +gate reserves and `max_audio_ms` subtracts — so a short clip decodes exactly as +it always has. `ceiling` is the decoder context, which `transcribe_session_params::n_ctx` lowers. **`n_ctx` is the only caller-facing control over output length.** There is @@ -120,6 +127,14 @@ that ignored audio length entirely, so a clip well inside `max_audio_ms` could still truncate with most of the context unused. That is fixed; the reserve constants remain only as the floor. +One consequence worth knowing: on long audio some families decode into +degenerate repetition (the same phrase emitted until the budget runs out). +That is upstream model behavior under greedy decoding, not a porting defect — +the reference implementations do the same, and the old flat cap was only +hiding it by stopping the decode early. A repetition guard is tracked +separately; until it lands, a looping transcript on a long clip is expected +and matches the reference. + ### 3. Soft window — warn and proceed | Families | Window | Behavior | diff --git a/src/arch/canary/model.cpp b/src/arch/canary/model.cpp index 403dfc85..6901939b 100644 --- a/src/arch/canary/model.cpp +++ b/src/arch/canary/model.cpp @@ -395,10 +395,17 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par // effective_max_audio_ms to the encoder bound regardless of n_ctx; the // decoder self-KV (which n_ctx does lower) only bounds transcript length. if (m->hparams.dec_max_position > 0) { - m->limits.has_context_cap = true; - m->limits.audio_from_caps = true; - m->limits.model_max_ctx = m->hparams.dec_max_position; - m->limits.gen_reserve = k_gen_reserve; + m->limits.has_context_cap = true; + m->limits.audio_from_caps = true; + m->limits.model_max_ctx = m->hparams.dec_max_position; + m->limits.gen_reserve = k_gen_reserve; + // Encoder rate, for the duration-derived decode budget. Not used for + // effective_max_audio_ms here (audio_from_caps pins that to the encoder + // bound), so publishing it changes no advertised limit. + if (m->hparams.enc_subsampling_factor > 0 && m->hparams.fe_hop_length > 0 && m->hparams.fe_sample_rate > 0) { + m->limits.ms_per_audio_token = static_cast(m->hparams.enc_subsampling_factor) * + m->hparams.fe_hop_length * 1000.0 / m->hparams.fe_sample_rate; + } // Whisper-style decoder self-KV: dec_d_model per layer, K and V, no GQA. m->limits.kv_elems_per_ctx_token = (int64_t) m->hparams.dec_d_model * m->hparams.dec_n_layers * 2; } @@ -1096,8 +1103,10 @@ transcribe_status run(transcribe_session * session, cc->clear_result(); - const int eos_id = cm->hparams.eos_token_id; - const int max_tokens = transcribe::pick_decode_budget(T_enc, k_gen_reserve, prompt_len, cc->kv_cache.n_ctx); + const int eos_id = cm->hparams.eos_token_id; + const int max_tokens = + transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc, cm->limits.ms_per_audio_token), + k_gen_reserve, prompt_len, cc->kv_cache.n_ctx); int next_token = 0; if (prompt_skip_softmax && db.argmax_out != nullptr) { @@ -1621,8 +1630,10 @@ transcribe_status run_batch(transcribe_session * session, const int n_ctx_cap = canary_context_ceiling(cc->n_ctx, hp); // One decode budget for the whole batch (the step loop runs every row in // lockstep), sized from the longest surviving utterance. Same rule as run(). - const int max_new = transcribe::pick_decode_budget(T_enc_max, k_gen_reserve, prompt_len, n_ctx_cap); - int max_n_kv = 1024; + const int max_new = + transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc_max, cm->limits.ms_per_audio_token), + k_gen_reserve, prompt_len, n_ctx_cap); + int max_n_kv = 1024; while (max_n_kv < prompt_len + max_new) { max_n_kv *= 2; } diff --git a/src/arch/canary_qwen/model.cpp b/src/arch/canary_qwen/model.cpp index 3fdf9fa8..0475f6b5 100644 --- a/src/arch/canary_qwen/model.cpp +++ b/src/arch/canary_qwen/model.cpp @@ -927,7 +927,8 @@ transcribe_status run(transcribe_session * context, // Per-run decode budget: scales with the audio, floored at the reserve the // gate above just guaranteed, clamped to the context left. Replaces a flat // 256-token cap that truncated long clips with context still free. - const int max_new = transcribe::pick_decode_budget(T_enc, k_gen_reserve, T_prompt, ceiling); + const int max_new = transcribe::pick_decode_budget( + transcribe::predict_transcript_tokens(T_enc, cm->limits.ms_per_audio_token), k_gen_reserve, T_prompt, ceiling); // KV cache init (grow-to-fit, clamped to the context ceiling). Size to // hold prompt + decode budget, rounded up to a power of two (the step @@ -1484,11 +1485,13 @@ transcribe_status run_batch(transcribe_session * session, } return TRANSCRIBE_OK; } - T_enc_max = std::max(1, T_enc_max); + T_enc_max = std::max(1, T_enc_max); // One decode budget for the whole batch (the step loop runs every row in // lockstep), sized from the longest surviving utterance. Same rule as run(). - const int max_new = transcribe::pick_decode_budget(T_enc_max, k_gen_reserve, max_T_prompt, ceiling); - int max_n_kv = 1024; + const int max_new = + transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc_max, cm->limits.ms_per_audio_token), + k_gen_reserve, max_T_prompt, ceiling); + int max_n_kv = 1024; while (max_n_kv < max_T_prompt + max_new) { max_n_kv *= 2; } diff --git a/src/arch/cohere/model.cpp b/src/arch/cohere/model.cpp index a4aafa63..58aee94c 100644 --- a/src/arch/cohere/model.cpp +++ b/src/arch/cohere/model.cpp @@ -1045,8 +1045,10 @@ transcribe_status run(transcribe_session * session, // Load-time validation guarantees eos_token_id >= 0; no // fallback is needed here. See the tokenizer.eos_id() check // in cohere::load() at the top of this file. - const int eos_id = cm->hparams.eos_token_id; - const int max_tokens = transcribe::pick_decode_budget(T_enc, k_gen_reserve, prompt_len, cc->kv_cache.n_ctx); + const int eos_id = cm->hparams.eos_token_id; + const int max_tokens = + transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc, cm->limits.ms_per_audio_token), + k_gen_reserve, prompt_len, cc->kv_cache.n_ctx); // Pick the first generated token. Fast path reads a single // int32 argmax that the GPU computed; debug path reads the @@ -1575,8 +1577,10 @@ transcribe_status run_batch(transcribe_session * session, const int n_ctx_cap = cohere_dec_ctx_ceiling(cc->n_ctx, hp); // One decode budget for the whole batch (the step loop runs every row in // lockstep), sized from the longest surviving utterance. Same rule as run(). - const int max_new = transcribe::pick_decode_budget(T_enc_max, k_gen_reserve, prompt_len, n_ctx_cap); - int max_n_kv = 1024; + const int max_new = + transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc_max, cm->limits.ms_per_audio_token), + k_gen_reserve, prompt_len, n_ctx_cap); + int max_n_kv = 1024; while (max_n_kv < prompt_len + max_new) { max_n_kv *= 2; } diff --git a/src/arch/funasr_nano/model.cpp b/src/arch/funasr_nano/model.cpp index 2ba1652a..bc89e0fd 100644 --- a/src/arch/funasr_nano/model.cpp +++ b/src/arch/funasr_nano/model.cpp @@ -684,7 +684,9 @@ transcribe_status run(transcribe_session * session, // Per-run decode budget: scales with the audio, floored at the reserve the // gate above just guaranteed, clamped to the context left. Replaces a flat // 256-token cap that truncated long clips with context still free. - const int max_new = transcribe::pick_decode_budget(T_audio, k_gen_reserve, T_prompt, ceiling); + const int max_new = + transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_audio, cm->limits.ms_per_audio_token), + k_gen_reserve, T_prompt, ceiling); // ---- KV cache init (grow-to-fit, clamped to the context ceiling) ---- // Size to hold the prompt plus the decode budget, rounded up to a power of @@ -1183,8 +1185,10 @@ transcribe_status run_batch(transcribe_session * session, } // One decode budget for the whole batch (the step loop runs every row in // lockstep), sized from the longest surviving utterance. Same rule as run(). - const int max_new = transcribe::pick_decode_budget(max_T_audio, k_gen_reserve, max_T_prompt, ceiling); - int max_n_kv = 1024; + const int max_new = transcribe::pick_decode_budget( + transcribe::predict_transcript_tokens(max_T_audio, cm->limits.ms_per_audio_token), k_gen_reserve, max_T_prompt, + ceiling); + int max_n_kv = 1024; while (max_n_kv < max_T_prompt + max_new) { max_n_kv *= 2; } diff --git a/src/arch/granite/model.cpp b/src/arch/granite/model.cpp index 2149155a..2e6185b5 100644 --- a/src/arch/granite/model.cpp +++ b/src/arch/granite/model.cpp @@ -1056,7 +1056,9 @@ transcribe_status run(transcribe_session * ctx_base, // Per-run decode budget: scales with the audio, floored at the reserve the // gate above just guaranteed, clamped to the context left. Replaces a flat // 256-token cap that truncated long clips with context still free. - const int gen_budget = transcribe::pick_decode_budget(n_audio_tokens, k_gen_reserve, T_prompt, ceiling); + const int gen_budget = transcribe::pick_decode_budget( + transcribe::predict_transcript_tokens(n_audio_tokens, cm->limits.ms_per_audio_token), k_gen_reserve, T_prompt, + ceiling); // Size the KV cache dynamically: T_prompt + room for the longest // generation we'll emit, clamped to the context ceiling. Matches the @@ -1609,11 +1611,13 @@ transcribe_status run_batch(transcribe_session * session, } return TRANSCRIBE_OK; } - n_audio_max = std::max(1, n_audio_max); + n_audio_max = std::max(1, n_audio_max); // One decode budget for the whole batch (the step loop runs every row in // lockstep), sized from the longest surviving utterance. Same rule as run(). - const int max_new = transcribe::pick_decode_budget(n_audio_max, k_gen_reserve, max_T_prompt, ceiling); - int max_n_kv = 1024; + const int max_new = transcribe::pick_decode_budget( + transcribe::predict_transcript_tokens(n_audio_max, cm->limits.ms_per_audio_token), k_gen_reserve, max_T_prompt, + ceiling); + int max_n_kv = 1024; while (max_n_kv < max_T_prompt + max_new) { max_n_kv *= 2; } diff --git a/src/arch/qwen3_asr/model.cpp b/src/arch/qwen3_asr/model.cpp index 3460c90f..6944e231 100644 --- a/src/arch/qwen3_asr/model.cpp +++ b/src/arch/qwen3_asr/model.cpp @@ -746,7 +746,8 @@ transcribe_status run(transcribe_session * session, // reserve the gate above just guaranteed and clamped to the context left. // This replaces a flat 256-token cap that truncated every clip past ~75 s // of speech regardless of how much context was still free. - const int max_new = transcribe::pick_decode_budget(T_enc, k_gen_reserve, T_prompt, ceiling); + const int max_new = transcribe::pick_decode_budget( + transcribe::predict_transcript_tokens(T_enc, cm->limits.ms_per_audio_token), k_gen_reserve, T_prompt, ceiling); // KV cache init (grow-to-fit, clamped to the context ceiling). Size to // hold prompt + decode budget, rounded up to a power of two (the step @@ -1597,8 +1598,10 @@ transcribe_status run_batch(transcribe_session * session, // One decode budget for the whole batch (the step loop runs every row in // lockstep), sized from the longest surviving utterance so no row is cut // short. Same rule as single-shot run(). - const int max_new = transcribe::pick_decode_budget(max_T_enc, k_gen_reserve, max_T_prompt, ceiling); - int max_n_kv = 1024; + const int max_new = + transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(max_T_enc, cm->limits.ms_per_audio_token), + k_gen_reserve, max_T_prompt, ceiling); + int max_n_kv = 1024; while (max_n_kv < max_T_prompt + max_new) { max_n_kv *= 2; } diff --git a/src/transcribe-decode-budget.h b/src/transcribe-decode-budget.h index bb5e4bae..0d2dbc88 100644 --- a/src/transcribe-decode-budget.h +++ b/src/transcribe-decode-budget.h @@ -25,6 +25,45 @@ namespace transcribe { +// Speech-rate bound on transcript length, in text tokens per second of audio. +// +// Deliberately generous: measured English BPE runs ~3.4 tokens/sec, and CJK is +// denser, so this keeps a wide margin. It also matches what the 80 ms-per-token +// encoders were already getting from their raw audio-token count (12.5/sec), so +// adopting the duration form below leaves those families where they were. +constexpr int k_transcript_tokens_per_sec = 12; + +// Predicted transcript length, in tokens, for an utterance the encoder turned +// into `audio_tokens` outputs at `ms_per_audio_token` each. +// +// The raw audio-token count is NOT a portable proxy for transcript length: +// encoders differ ~6x in rate. Most emit one token per 80 ms (12.5/sec, safely +// above any speech rate), but funasr_nano's LFR frontend stacks frames and +// emits one per ~480 ms (2.08/sec) — below the text-token rate, so using its +// audio-token count under-predicts and the budget truncates a transcript the +// context had room for. Converting to seconds first removes the encoder rate +// from the estimate entirely. +// +// A non-positive `ms_per_audio_token` means the family did not publish its +// rate; fall back to the audio-token count, which is what every family used +// before this became rate-aware. +inline int predict_transcript_tokens(int audio_tokens, double ms_per_audio_token) { + if (audio_tokens <= 0) { + return 0; + } + if (!(ms_per_audio_token > 0.0)) { + return audio_tokens; + } + const double seconds = static_cast(audio_tokens) * ms_per_audio_token / 1000.0; + const double predicted = seconds * k_transcript_tokens_per_sec; + // Clamp into int range; callers clamp again to the context actually left. + if (predicted <= 0.0) { + return 0; + } + constexpr double k_int_max = 2147483647.0; + return predicted >= k_int_max ? 2147483647 : static_cast(predicted); +} + // Per-run decode budget, in transcript tokens. // // predicted the family's upper-bound estimate of transcript length, diff --git a/tests/decode_budget_unit.cpp b/tests/decode_budget_unit.cpp index b75c2b34..36b3524e 100644 --- a/tests/decode_budget_unit.cpp +++ b/tests/decode_budget_unit.cpp @@ -34,6 +34,15 @@ int g_failures = 0; } \ } while (0) +void check_predict(const char * what, int audio_tokens, double ms_per_audio_token, int expected) { + const int got = transcribe::predict_transcript_tokens(audio_tokens, ms_per_audio_token); + if (got != expected) { + std::fprintf(stderr, "FAIL %s: predict_transcript_tokens(%d, %.3f) = %d, expected %d\n", what, audio_tokens, + ms_per_audio_token, got, expected); + ++g_failures; + } +} + void check_budget(const char * what, int predicted, int floor_tokens, int t_prompt, int ceiling, int expected) { const int got = transcribe::pick_decode_budget(predicted, floor_tokens, t_prompt, ceiling); if (got != expected) { @@ -46,6 +55,23 @@ void check_budget(const char * what, int predicted, int floor_tokens, int t_prom } // namespace int main(void) { + // ---- Prediction is duration-based, not audio-token based. ---- + // The whole point: encoder rates differ ~6x, so two families that heard the + // same 197 seconds of speech must predict the same transcript length even + // though one emitted 6x more audio tokens than the other. + check_predict("80 ms encoder, 197 s", /*audio_tokens=*/2463, /*ms_per_audio_token=*/80.0, + /*expected=*/2364); + check_predict("480 ms LFR encoder, same 197 s", /*audio_tokens=*/410, /*ms_per_audio_token=*/480.0, + /*expected=*/2361); + // funasr_nano's real regression: its raw audio-token count (410) predicts + // far below the ~700 tokens this clip's transcript actually needs. + CHECK(transcribe::predict_transcript_tokens(410, 480.0) > 700); + + // An unpublished rate falls back to the audio-token count (prior behavior). + check_predict("unknown rate falls back", 2463, 0.0, 2463); + check_predict("negative rate falls back", 2463, -1.0, 2463); + check_predict("no audio", 0, 80.0, 0); + // ---- Property 1: the floor holds for short audio. ---- // A clip whose audio-token count is below the family's historical fixed // budget must still get that budget, so its decode is byte-identical to diff --git a/tests/qwen3_asr_batch_truncation.cpp b/tests/qwen3_asr_batch_truncation.cpp index 2f91eadd..aebadd44 100644 --- a/tests/qwen3_asr_batch_truncation.cpp +++ b/tests/qwen3_asr_batch_truncation.cpp @@ -131,15 +131,19 @@ int main() { } // ---- Lowered n_ctx: the only caller-facing control over output length ---- - // love-loss.wav is ~197 s. qwen3_asr emits one audio token per 80 ms, so - // the prompt is ~2465 audio tokens plus ~15 chat-affix tokens. A 2816-token - // ceiling therefore clears the input gate (which reserves k_gen_reserve = - // 256 on top of the prompt) while leaving only ~340 tokens of decode - // budget — well under the ~700 this clip's transcript needs, so the decode - // runs into the budget and must report it. + // Measured on love-loss.wav (~197 s): T_enc = 2563 audio tokens (the clip + // encodes at 76.9 ms/token), T_prompt = 2578 with the chat affixes, and the + // full transcript is 701-750 tokens. The input gate reserves k_gen_reserve + // (256) on top of the prompt, so the ceiling must be >= 2834 for the clip + // to be accepted at all. + // + // 3072 clears that gate by 238 tokens and leaves 494 tokens of decode + // budget, about 210 short of the full transcript. Both margins absorb small + // prompt-template drift; if this ever returns INPUT_TOO_LONG the prompt + // grew, and if it returns OK the transcript shrank. transcribe_session_params sp; transcribe_session_params_init(&sp); - sp.n_ctx = 2816; + sp.n_ctx = 3072; struct transcribe_session * s = nullptr; if (transcribe_session_init(model, &sp, &s) != TRANSCRIBE_OK) { std::fprintf(stderr, "session init failed\n"); From bbb957358900a98f54b2d5db0cf64b6274cca66b Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 21 Sep 2026 14:50:14 +0800 Subject: [PATCH 3/6] slim comments --- src/arch/canary/model.cpp | 9 ++-- src/arch/canary_qwen/model.cpp | 16 +++---- src/arch/cohere/model.cpp | 9 ++-- src/arch/funasr_nano/model.cpp | 16 +++---- src/arch/granite/model.cpp | 16 +++---- src/arch/moss/model.cpp | 7 +-- src/arch/qwen3_asr/model.cpp | 20 +++------ src/arch/voxtral/model.cpp | 7 +-- src/transcribe-decode-budget.h | 81 ++++++++++++++-------------------- 9 files changed, 69 insertions(+), 112 deletions(-) diff --git a/src/arch/canary/model.cpp b/src/arch/canary/model.cpp index 6901939b..8c2f58a8 100644 --- a/src/arch/canary/model.cpp +++ b/src/arch/canary/model.cpp @@ -219,10 +219,8 @@ constexpr float kBnEps = 1e-5f; // overrun is kept as a partial and flagged via // transcribe_was_truncated(), not rejected. -// Generation reserve, in tokens: the floor under the per-run decode budget. -// The budget itself scales with the audio and is clamped to the decoder -// self-KV ceiling (see transcribe-decode-budget.h), so a short clip decodes -// exactly as it always has while a long one is no longer cut at a flat 512. +// Generation reserve: floor under the per-run budget, which scales with the +// audio and clamps to the decoder self-KV. See transcribe-decode-budget.h. constexpr int k_gen_reserve = 512; // Predicted encoder frame count T_enc for a given mel frame count. The @@ -1628,8 +1626,7 @@ transcribe_status run_batch(transcribe_session * session, // raised) by the caller's n_ctx knob. Default knob (0) leaves it at // dec_max_position, so in-spec batched decode is unchanged. const int n_ctx_cap = canary_context_ceiling(cc->n_ctx, hp); - // One decode budget for the whole batch (the step loop runs every row in - // lockstep), sized from the longest surviving utterance. Same rule as run(). + // One budget for the whole batch, sized from the longest surviving row. const int max_new = transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc_max, cm->limits.ms_per_audio_token), k_gen_reserve, prompt_len, n_ctx_cap); diff --git a/src/arch/canary_qwen/model.cpp b/src/arch/canary_qwen/model.cpp index 0475f6b5..9b677386 100644 --- a/src/arch/canary_qwen/model.cpp +++ b/src/arch/canary_qwen/model.cpp @@ -117,11 +117,9 @@ constexpr float kBnEps = 1e-5f; // TRANSCRIBE_ERR_INPUT_TOO_LONG; a transcript that fills the generation budget // before end-of-stream is flagged via transcribe_was_truncated(). -// Generation reserve, in tokens: the room the up-front input gate always -// keeps free for output, the floor under the per-run decode budget, and the -// value transcribe_capabilities::max_audio_ms subtracts. The actual per-run -// budget scales with the audio (see transcribe-decode-budget.h); this is only -// its lower bound, so a short clip decodes exactly as it always has. +// Generation reserve: what the input gate keeps free, what max_audio_ms +// subtracts, and the floor under the per-run budget. See +// transcribe-decode-budget.h. constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum @@ -924,9 +922,8 @@ transcribe_status run(transcribe_session * context, return TRANSCRIBE_ERR_INPUT_TOO_LONG; } - // Per-run decode budget: scales with the audio, floored at the reserve the - // gate above just guaranteed, clamped to the context left. Replaces a flat - // 256-token cap that truncated long clips with context still free. + // Per-run budget: duration-derived, floored at the reserve the gate above + // just guaranteed, clamped to the context left. const int max_new = transcribe::pick_decode_budget( transcribe::predict_transcript_tokens(T_enc, cm->limits.ms_per_audio_token), k_gen_reserve, T_prompt, ceiling); @@ -1486,8 +1483,7 @@ transcribe_status run_batch(transcribe_session * session, return TRANSCRIBE_OK; } T_enc_max = std::max(1, T_enc_max); - // One decode budget for the whole batch (the step loop runs every row in - // lockstep), sized from the longest surviving utterance. Same rule as run(). + // One budget for the whole batch, sized from the longest surviving row. const int max_new = transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc_max, cm->limits.ms_per_audio_token), k_gen_reserve, max_T_prompt, ceiling); diff --git a/src/arch/cohere/model.cpp b/src/arch/cohere/model.cpp index 58aee94c..2accacf9 100644 --- a/src/arch/cohere/model.cpp +++ b/src/arch/cohere/model.cpp @@ -417,10 +417,8 @@ transcribe_status promote_conv_pw_to_f32_on_cpu(CohereModel & m) { constexpr const char k_default_variant[] = "cohere-asr"; -// Generation reserve, in tokens: the floor under the per-run decode budget. -// The budget itself scales with the audio and is clamped to the decoder -// self-KV ceiling (see transcribe-decode-budget.h), so a short clip decodes -// exactly as it always has while a long one is no longer cut at a flat 512. +// Generation reserve: floor under the per-run budget, which scales with the +// audio and clamps to the decoder self-KV. See transcribe-decode-budget.h. constexpr int k_gen_reserve = 512; // Forward declarations for the Arch trait below. @@ -1575,8 +1573,7 @@ transcribe_status run_batch(transcribe_session * session, // Honor the session context cap (same ceiling the single-shot path uses), // not the raw model max — so a lowered n_ctx bounds batch decoder KV too. const int n_ctx_cap = cohere_dec_ctx_ceiling(cc->n_ctx, hp); - // One decode budget for the whole batch (the step loop runs every row in - // lockstep), sized from the longest surviving utterance. Same rule as run(). + // One budget for the whole batch, sized from the longest surviving row. const int max_new = transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc_max, cm->limits.ms_per_audio_token), k_gen_reserve, prompt_len, n_ctx_cap); diff --git a/src/arch/funasr_nano/model.cpp b/src/arch/funasr_nano/model.cpp index bc89e0fd..1a2ab40e 100644 --- a/src/arch/funasr_nano/model.cpp +++ b/src/arch/funasr_nano/model.cpp @@ -81,11 +81,9 @@ constexpr const char k_default_variant[] = "fun-asr-nano-2512"; // transcribe_was_truncated(). // --------------------------------------------------------------------------- -// Generation reserve, in tokens: the room the up-front input gate always -// keeps free for output, the floor under the per-run decode budget, and the -// value transcribe_capabilities::max_audio_ms subtracts. The actual per-run -// budget scales with the audio (see transcribe-decode-budget.h); this is only -// its lower bound, so a short clip decodes exactly as it always has. +// Generation reserve: what the input gate keeps free, what max_audio_ms +// subtracts, and the floor under the per-run budget. See +// transcribe-decode-budget.h. constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum, @@ -681,9 +679,8 @@ transcribe_status run(transcribe_session * session, return TRANSCRIBE_ERR_INPUT_TOO_LONG; } - // Per-run decode budget: scales with the audio, floored at the reserve the - // gate above just guaranteed, clamped to the context left. Replaces a flat - // 256-token cap that truncated long clips with context still free. + // Per-run budget: duration-derived, floored at the reserve the gate above + // just guaranteed, clamped to the context left. const int max_new = transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_audio, cm->limits.ms_per_audio_token), k_gen_reserve, T_prompt, ceiling); @@ -1183,8 +1180,7 @@ transcribe_status run_batch(transcribe_session * session, } return TRANSCRIBE_OK; } - // One decode budget for the whole batch (the step loop runs every row in - // lockstep), sized from the longest surviving utterance. Same rule as run(). + // One budget for the whole batch, sized from the longest surviving row. const int max_new = transcribe::pick_decode_budget( transcribe::predict_transcript_tokens(max_T_audio, cm->limits.ms_per_audio_token), k_gen_reserve, max_T_prompt, ceiling); diff --git a/src/arch/granite/model.cpp b/src/arch/granite/model.cpp index 2e6185b5..533925ec 100644 --- a/src/arch/granite/model.cpp +++ b/src/arch/granite/model.cpp @@ -81,11 +81,9 @@ constexpr float kBnEps = 1e-5f; // Over-length input is rejected up front with TRANSCRIBE_ERR_INPUT_TOO_LONG // rather than silently aliasing RoPE past the trained range. -// Generation reserve, in tokens: the room the up-front input gate always -// keeps free for output, the floor under the per-run decode budget, and the -// value transcribe_capabilities::max_audio_ms subtracts. The actual per-run -// budget scales with the audio (see transcribe-decode-budget.h); this is only -// its lower bound, so a short clip decodes exactly as it always has. +// Generation reserve: what the input gate keeps free, what max_audio_ms +// subtracts, and the floor under the per-run budget. See +// transcribe-decode-budget.h. constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum, @@ -1053,9 +1051,8 @@ transcribe_status run(transcribe_session * ctx_base, return TRANSCRIBE_ERR_INPUT_TOO_LONG; } - // Per-run decode budget: scales with the audio, floored at the reserve the - // gate above just guaranteed, clamped to the context left. Replaces a flat - // 256-token cap that truncated long clips with context still free. + // Per-run budget: duration-derived, floored at the reserve the gate above + // just guaranteed, clamped to the context left. const int gen_budget = transcribe::pick_decode_budget( transcribe::predict_transcript_tokens(n_audio_tokens, cm->limits.ms_per_audio_token), k_gen_reserve, T_prompt, ceiling); @@ -1612,8 +1609,7 @@ transcribe_status run_batch(transcribe_session * session, return TRANSCRIBE_OK; } n_audio_max = std::max(1, n_audio_max); - // One decode budget for the whole batch (the step loop runs every row in - // lockstep), sized from the longest surviving utterance. Same rule as run(). + // One budget for the whole batch, sized from the longest surviving row. const int max_new = transcribe::pick_decode_budget( transcribe::predict_transcript_tokens(n_audio_max, cm->limits.ms_per_audio_token), k_gen_reserve, max_T_prompt, ceiling); diff --git a/src/arch/moss/model.cpp b/src/arch/moss/model.cpp index 1789142d..440e37b2 100644 --- a/src/arch/moss/model.cpp +++ b/src/arch/moss/model.cpp @@ -760,11 +760,8 @@ transcribe_status run(transcribe_session * session, return TRANSCRIBE_ERR_INPUT_TOO_LONG; } - // Generation budget scales with audio length via the shared rule - // (transcribe-decode-budget.h). moss predicts higher than the plain - // audio-token count because the emergent transcript carries speaker - // markers ([start]/[Sxx]/[end]) on top of the text; for long-form that - // far exceeds the k_max_new floor. Clamped to the context left. + // Predicts above the plain audio-token count: the transcript carries + // speaker markers ([start]/[Sxx]/[end]) on top of the text. const int gen_budget = transcribe::pick_decode_budget(2 * T_enc + 128, k_max_new, T_prompt, ceiling); // KV cache (grow-to-fit, clamped to ceiling). Short inputs retain the old diff --git a/src/arch/qwen3_asr/model.cpp b/src/arch/qwen3_asr/model.cpp index 6944e231..f2912e74 100644 --- a/src/arch/qwen3_asr/model.cpp +++ b/src/arch/qwen3_asr/model.cpp @@ -75,11 +75,9 @@ constexpr const char k_default_variant[] = "qwen3-asr"; // transcript that fills the generation budget before end-of-stream is flagged // via transcribe_was_truncated(). -// Generation reserve, in tokens: the room the up-front input gate always -// keeps free for output, the floor under the per-run decode budget, and the -// value transcribe_capabilities::max_audio_ms subtracts. The actual per-run -// budget scales with the audio (see transcribe-decode-budget.h); this is only -// its lower bound, so a short clip decodes exactly as it always has. +// Generation reserve: what the input gate keeps free, what max_audio_ms +// subtracts, and the floor under the per-run budget. See +// transcribe-decode-budget.h. constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum, @@ -741,11 +739,9 @@ transcribe_status run(transcribe_session * session, return TRANSCRIBE_ERR_INPUT_TOO_LONG; } - // Per-run decode budget. Scales with the audio (a transcript never needs - // more text tokens than the encoder produced audio tokens), floored at the - // reserve the gate above just guaranteed and clamped to the context left. - // This replaces a flat 256-token cap that truncated every clip past ~75 s - // of speech regardless of how much context was still free. + // Per-run budget: duration-derived, floored at the reserve the gate above + // just guaranteed, clamped to the context left. The old flat 256 truncated + // every clip past ~75 s of speech with the context still mostly free. const int max_new = transcribe::pick_decode_budget( transcribe::predict_transcript_tokens(T_enc, cm->limits.ms_per_audio_token), k_gen_reserve, T_prompt, ceiling); @@ -1595,9 +1591,7 @@ transcribe_status run_batch(transcribe_session * session, } return TRANSCRIBE_OK; } - // One decode budget for the whole batch (the step loop runs every row in - // lockstep), sized from the longest surviving utterance so no row is cut - // short. Same rule as single-shot run(). + // One budget for the whole batch, sized from the longest surviving row. const int max_new = transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(max_T_enc, cm->limits.ms_per_audio_token), k_gen_reserve, max_T_prompt, ceiling); diff --git a/src/arch/voxtral/model.cpp b/src/arch/voxtral/model.cpp index 320e2fc5..6f275972 100644 --- a/src/arch/voxtral/model.cpp +++ b/src/arch/voxtral/model.cpp @@ -81,11 +81,8 @@ constexpr const char k_default_variant[] = "voxtral-mini-3b-2507"; constexpr int k_decode_budget_min = 448; // Decode budget (max new text tokens) for an utterance with `n_audio` audio -// embedding tokens. Thin wrapper over the shared rule every autoregressive -// family now uses (transcribe-decode-budget.h): the audio-token count is a safe -// upper bound on the transcript, floored at k_decode_budget_min and clamped to -// the context remaining under the trained max. Greedy decode stops at EOS well -// before this, so a generous ceiling costs only its KV allocation. +// embedding tokens. Thin wrapper over transcribe-decode-budget.h. Greedy decode +// stops at EOS well before this, so a generous ceiling costs only its KV. int pick_decode_budget(int n_audio, int t_prompt, int model_max) { return transcribe::pick_decode_budget(n_audio, k_decode_budget_min, t_prompt, model_max); } diff --git a/src/transcribe-decode-budget.h b/src/transcribe-decode-budget.h index 0d2dbc88..29a152a5 100644 --- a/src/transcribe-decode-budget.h +++ b/src/transcribe-decode-budget.h @@ -2,22 +2,15 @@ // // INTERNAL. Header-only, like the ABI helpers in transcribe-abi.h. // -// Autoregressive families used to hardcode their generation budget as a -// constant (256 / 512 tokens) that did not depend on the audio at all, while -// the up-front input gate accepted clips orders of magnitude longer than that -// many tokens could describe. Any clip whose natural transcript outran the -// constant came back as TRANSCRIBE_ERR_OUTPUT_TRUNCATED with a partial -// transcript, even though the decoder context had ample room left. See -// docs/input-limits.md. +// Autoregressive families used to cap generation at a constant (256 / 512) +// that ignored the audio, while the input gate accepted clips far longer than +// that many tokens could describe — so a long clip came back +// TRANSCRIBE_ERR_OUTPUT_TRUNCATED with the context still mostly free. The +// budget has to track the input instead. See docs/input-limits.md. // -// The budget must track the input instead. Speech yields fewer text tokens -// than the encoder yields audio tokens, so the audio-token count is a safe -// upper bound on the transcript length — the estimate voxtral has shipped -// with since its introduction, generalized here so every family shares it. -// -// This is deliberately NOT a public run parameter. The only caller-facing -// knob is transcribe_session_params::n_ctx, which lowers `ceiling` and so -// lowers the budget with it. +// This is deliberately NOT a public run parameter. The only caller-facing knob +// is transcribe_session_params::n_ctx, which lowers `ceiling` and the budget +// with it. #pragma once @@ -26,27 +19,23 @@ namespace transcribe { // Speech-rate bound on transcript length, in text tokens per second of audio. -// -// Deliberately generous: measured English BPE runs ~3.4 tokens/sec, and CJK is -// denser, so this keeps a wide margin. It also matches what the 80 ms-per-token -// encoders were already getting from their raw audio-token count (12.5/sec), so -// adopting the duration form below leaves those families where they were. +// Generous on purpose: English BPE measures ~3.4/sec and CJK is denser. 12 also +// matches what the 80 ms-per-token encoders already got from their raw +// audio-token count (12.5/sec), so the duration form leaves them where they were. constexpr int k_transcript_tokens_per_sec = 12; // Predicted transcript length, in tokens, for an utterance the encoder turned // into `audio_tokens` outputs at `ms_per_audio_token` each. // -// The raw audio-token count is NOT a portable proxy for transcript length: -// encoders differ ~6x in rate. Most emit one token per 80 ms (12.5/sec, safely -// above any speech rate), but funasr_nano's LFR frontend stacks frames and -// emits one per ~480 ms (2.08/sec) — below the text-token rate, so using its -// audio-token count under-predicts and the budget truncates a transcript the -// context had room for. Converting to seconds first removes the encoder rate -// from the estimate entirely. +// The raw audio-token count is NOT a portable proxy: encoder rates differ ~6x. +// Most emit one token per 80 ms (12.5/sec, safely above any speech rate), but +// funasr_nano's LFR frontend emits one per ~480 ms (2.08/sec) — below the +// text-token rate, so its count under-predicts and the budget truncates a +// transcript the context had room for. Going via seconds removes the encoder +// rate from the estimate. // -// A non-positive `ms_per_audio_token` means the family did not publish its -// rate; fall back to the audio-token count, which is what every family used -// before this became rate-aware. +// A non-positive `ms_per_audio_token` means the family published no rate; fall +// back to the audio-token count, the pre-rate-aware behavior. inline int predict_transcript_tokens(int audio_tokens, double ms_per_audio_token) { if (audio_tokens <= 0) { return 0; @@ -66,25 +55,23 @@ inline int predict_transcript_tokens(int audio_tokens, double ms_per_audio_token // Per-run decode budget, in transcript tokens. // -// predicted the family's upper-bound estimate of transcript length, -// in tokens. For most families this is the audio-token -// count; a family whose output carries more than the -// transcript (moss emits speaker markers) scales it up. -// floor_tokens never plan for fewer than this. Each family passes its -// historical fixed budget, so a clip that fits today keeps -// byte-identical behavior, and the generation reserve that -// transcribe_capabilities::max_audio_ms subtracts (via +// predicted upper-bound estimate of transcript length, normally from +// predict_transcript_tokens() above. A family whose output +// carries more than the transcript (moss emits speaker +// markers) scales it up. +// floor_tokens never plan for fewer. Each family passes its historical +// fixed budget, so a clip that fits today stays +// byte-identical and the reserve max_audio_ms subtracts (via // transcribe_model::LimitsBasis::gen_reserve) stays exact. -// t_prompt prompt tokens already committed to the decoder context. -// Includes the audio embeddings for families that put audio -// in-context; 0 for encoder-decoder families whose audio -// lives in a separate cross-attention cache. -// ceiling decoder context ceiling in tokens, already lowered (never -// raised) by transcribe_session_params::n_ctx. +// t_prompt prompt tokens already in the decoder context. Includes the +// audio embeddings for in-context families; 0 for +// encoder-decoder families whose audio lives in a separate +// cross-attention cache. +// ceiling decoder context ceiling, already lowered (never raised) by +// transcribe_session_params::n_ctx. // -// Returns the budget clamped to the context actually left, never negative. -// A zero return means the prompt already fills the ceiling; callers gate -// that case up front (INPUT_TOO_LONG) rather than entering the step loop. +// Clamped to the context actually left, never negative. Zero means the prompt +// already fills the ceiling; callers gate that up front (INPUT_TOO_LONG). inline int pick_decode_budget(int predicted, int floor_tokens, int t_prompt, int ceiling) { int budget = std::max(floor_tokens, predicted); const int room = ceiling - t_prompt; From 2952df314cacb07749133a076f5f938366019134 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 21 Sep 2026 15:11:47 +0800 Subject: [PATCH 4/6] moss batch budget + encoder-bound docs --- docs/input-limits.md | 50 +++++++++++++++++++++++++++++++++ docs/porting/families/canary.md | 6 ++++ docs/porting/families/cohere.md | 12 ++++++++ src/arch/moss/model.cpp | 7 +++-- 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/docs/input-limits.md b/docs/input-limits.md index 7723a4db..f900391e 100644 --- a/docs/input-limits.md +++ b/docs/input-limits.md @@ -135,6 +135,43 @@ hiding it by stopping the decode early. A repetition guard is tracked separately; until it lands, a looping transcript on a long clip is expected and matches the reference. +### Encoder-bound families: `cohere` and `canary` + +For most hard-cap families the decoder context bounds the input *and* the +output, so the gate that accepts a clip also guarantees room for its +transcript. `cohere` and `canary` are the exception, and it is worth stating +plainly. + +Their audio bound is the **encoder** relative-position table +(`enc_pos_emb_max_len = 5000` on every shipped variant, ~400 s), while their +transcript bound is a **separate** 1024-token decoder self-KV +(`dec_max_position` / `dec_max_seq`). Audio lives in the cross-attention cache, +so it never consumes decoder context — which means the up-front gate has no way +to predict whether the transcript will fit. It cannot: transcript length is not +a function of the input the way audio-token count is. + +The consequence: a clip is accepted at up to ~400 s and can still return +`TRANSCRIBE_ERR_OUTPUT_TRUNCATED` once its transcript passes ~1018 tokens. +Measured on a 197 s English clip, both families truncate — canary at 1015 +tokens, cohere at 1014. This is the one place where an accepted clip is not +guaranteed a complete transcript, and it is a property of the checkpoints, not +of the decode budget: the budget already hands these families 1018 of their +1024 positions, which is everything there is. + +`max_audio_ms` is deliberately the **architectural** bound — the longest clip +the encoder table can index without aliasing — not a quality recommendation. +Upstream's recommended working window is much shorter: + +| Family | Upstream recommended clip | Architectural gate | Transcript bound | +| --- | --- | --- | --- | +| `canary` | 40 s (>40 s is chunked upstream with 1 s overlap) | ~400 s | 1024 tokens | +| `cohere` | 35 s (`max_audio_clip_s`) | ~400 s | 1024 tokens | + +Those recommended windows are advisory and are **not** currently reported +through the ABI. They belong in a future "recommended window" capability field, +landing alongside chunked long-form support — not in `max_audio_ms`, which must +keep meaning "the longest clip this model can physically accept". + ### 3. Soft window — warn and proceed | Families | Window | Behavior | @@ -241,6 +278,19 @@ detect truncation should check `transcribe_was_truncated()` after finalize. the per-run budget then scales up with the audio (see "The decode budget"). Changing a family's reserve moves its published `max_audio_ms`, so it is not a free knob — the budget rule is the thing to tune. +- For `cohere` / `canary`, `max_audio_ms` is the encoder bound and the decoder + self-KV separately bounds the transcript, so an accepted clip is *not* + guaranteed a complete transcript (see "Encoder-bound families"). Do not + "fix" this by lowering `max_audio_ms` to the upstream recommended window: + that field means the architectural maximum, and `audio_from_caps` exists + precisely to keep the encoder bound from shrinking when `n_ctx` drops. The + fix is chunked long-form plus a separate recommended-window field. +- The decoder positional encoding on both is sinusoidal, not learned (canary's + GGUF publishes `learn_positional_encodings = false`; cohere's upstream config + records the same), so the 1024-entry table is a conversion-time artifact + rather than a trained weight. Regenerating it longer is technically possible + and is still the wrong move — it would run an AED an order of magnitude past + its supported window. Upstream's own answer to long audio here is chunking. - The upfront gate and `max_audio_ms` share a shape for decoder-context-bound families but differ in precision: `max_audio_ms ≈ (ceiling − representative_prompt − generation_reserve) / tokens_per_ms`, diff --git a/docs/porting/families/canary.md b/docs/porting/families/canary.md index 5351a93c..67ced9cf 100644 --- a/docs/porting/families/canary.md +++ b/docs/porting/families/canary.md @@ -141,6 +141,12 @@ uv run scripts/bench/run.py \ - Output head: LM head over the concatenated SP vocabulary. Decoding is beam search by default for the original canary-1b (beam=5, length_penalty=1.0) and greedy by default for the flash variants (beam=1). - Tokenizer: concatenated SentencePiece — one SP model per language concatenated into a single vocabulary. canary-1b-v2 is 16,384 pieces; flash/180m-flash/1b vocab sizes are not stated on model cards (Stage 2 fills from .nemo). - Audio length contract: native ≤40 s direct inference. <1 s is symmetrically zero-padded to 1 s. >40 s is handled by an external chunked inference script with 1 s overlap (canary-1b-v2 chunk len defaults to 40 s; canary-1b-flash 10 s; canary-180m-flash 10 s). **Long-form / streaming is out of scope for the v1 port.** +- Port limits (see `docs/input-limits.md`): the input gate is the encoder + relative-position table (`enc_pos_emb_max_len = 5000`, ~400 s), not the 40 s + upstream window, while the decoder self-KV (`dec_max_position = 1024`) + separately bounds the transcript. A clip inside the gate is therefore not + guaranteed a complete transcript — measured, a 197 s English clip truncates + at 1015 tokens with `TRANSCRIBE_ERR_OUTPUT_TRUNCATED`. ## Capabilities (from intake) diff --git a/docs/porting/families/cohere.md b/docs/porting/families/cohere.md index c3abae15..3a50e305 100644 --- a/docs/porting/families/cohere.md +++ b/docs/porting/families/cohere.md @@ -15,6 +15,18 @@ native Transformers. C++ CPU validation passes locally. `[en, ar]`; config omits top-level `vocab_size` — the converter falls back to `head.num_classes`; upstream repo is gated) +## Audio length contract + +- Upstream recommended clip length: **35 s** (`max_audio_clip_s` in the + upstream config). Longer audio is expected to be segmented by the caller. +- Architectural bounds, both read from the GGUF: the encoder relative-position + table `enc_pos_emb_max_len = 5000` (~400 s) is the input gate, and the + decoder self-KV `dec_max_seq = 1024` separately bounds the transcript. +- Audio lives in the cross-attention cache and never consumes decoder context, + so a clip well inside the ~400 s gate can still exhaust the 1024-token + transcript budget and return `TRANSCRIBE_ERR_OUTPUT_TRUNCATED` (measured: a + 197 s English clip truncates at 1014 tokens). See `docs/input-limits.md`. + ## References - Canonical reference: native Hugging Face Transformers diff --git a/src/arch/moss/model.cpp b/src/arch/moss/model.cpp index 440e37b2..72e7ea7b 100644 --- a/src/arch/moss/model.cpp +++ b/src/arch/moss/model.cpp @@ -1170,9 +1170,10 @@ transcribe_status run_batch(transcribe_session * session, return TRANSCRIBE_OK; } - // Batch-wide generation budget: covers the longest utterance's transcript - // (scales with its audio tokens), clamped to the context. - const int batch_budget = std::min(ceiling - max_T_prompt, std::max(k_max_new, 2 * max_T_enc + 128)); + // One budget for the whole batch, sized from the longest surviving row. + // Same prediction as run(): above the plain audio-token count, because the + // transcript carries speaker markers on top of the text. + const int batch_budget = transcribe::pick_decode_budget(2 * max_T_enc + 128, k_max_new, max_T_prompt, ceiling); int max_n_kv = 1024; while (max_n_kv < max_T_prompt + batch_budget) { From cfc17c3ca1ac363835733f52f2d928681a1216aa Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 21 Sep 2026 15:36:12 +0800 Subject: [PATCH 5/6] remove a bunch of comments and prose --- docs/input-limits.md | 115 +++++++++------------------ docs/porting/families/canary.md | 7 +- docs/porting/families/cohere.md | 14 ++-- src/arch/canary/model.cpp | 9 +-- src/arch/canary_qwen/model.cpp | 7 +- src/arch/cohere/model.cpp | 4 +- src/arch/funasr_nano/model.cpp | 7 +- src/arch/granite/model.cpp | 12 +-- src/arch/moss/model.cpp | 6 +- src/arch/qwen3_asr/model.cpp | 10 +-- src/arch/voxtral/model.cpp | 13 +-- src/transcribe-decode-budget.h | 59 ++++---------- tests/CMakeLists.txt | 3 - tests/decode_budget_unit.cpp | 62 +++------------ tests/qwen3_asr_batch_truncation.cpp | 38 +++------ 15 files changed, 91 insertions(+), 275 deletions(-) diff --git a/docs/input-limits.md b/docs/input-limits.md index f900391e..3bdef719 100644 --- a/docs/input-limits.md +++ b/docs/input-limits.md @@ -93,7 +93,7 @@ still returns `TRANSCRIBE_OK`). ### The decode budget -How much output an accepted clip may produce is **derived from the clip**, not +How much output an accepted clip may produce is derived from the clip, not fixed. Each autoregressive family resolves a per-run decode budget as: ```text @@ -101,76 +101,40 @@ budget = clamp(max(generation_reserve, predicted_transcript_tokens), 0, ceiling - prompt_tokens) ``` -`predicted_transcript_tokens` comes from the clip's **duration**, not its -audio-token count: `seconds x 12 tokens/sec`, where `seconds` is recovered from -the family's published encoder rate (`ms_per_audio_token`). The duration form -matters because encoder rates differ about 6x — most emit one audio token per -80 ms, but `funasr_nano`'s LFR frontend emits one per ~480 ms, which is *below* -the text-token rate, so its audio-token count under-predicts. 12 tokens/sec is -deliberately generous against a measured ~3.4 for English BPE, leaving room for -denser scripts. (`moss` and `voxtral` pass their own estimates instead: moss -scales up for the speaker markers its output carries.) - -`generation_reserve` is the per-family floor — the same constant the up-front -gate reserves and `max_audio_ms` subtracts — so a short clip decodes exactly as -it always has. `ceiling` is the decoder context, which -`transcribe_session_params::n_ctx` lowers. - -**`n_ctx` is the only caller-facing control over output length.** There is -deliberately no per-run "max tokens" parameter: an ASR transcript's length is a -property of the audio, so the library derives it rather than asking. Lowering -`n_ctx` to bound memory also lowers the budget, and can turn a run that would -have completed into `OUTPUT_TRUNCATED`. - -Historically these budgets were flat per-family constants (256 or 512 tokens) -that ignored audio length entirely, so a clip well inside `max_audio_ms` could -still truncate with most of the context unused. That is fixed; the reserve -constants remain only as the floor. - -One consequence worth knowing: on long audio some families decode into -degenerate repetition (the same phrase emitted until the budget runs out). -That is upstream model behavior under greedy decoding, not a porting defect — -the reference implementations do the same, and the old flat cap was only -hiding it by stopping the decode early. A repetition guard is tracked -separately; until it lands, a looping transcript on a long clip is expected -and matches the reference. +`predicted_transcript_tokens` comes from the clip's duration (`seconds x 12`), +recovered from the family's published `ms_per_audio_token` — not from the raw +audio-token count, whose rate differs ~6x across encoders. `generation_reserve` +is the per-family floor, the same constant the up-front gate reserves and +`max_audio_ms` subtracts. `ceiling` is the decoder context, which +`transcribe_session_params::n_ctx` lowers. `moss` and `voxtral` pass their own +estimates instead. + +`n_ctx` is the only caller-facing control over output length; there is no +per-run "max tokens" parameter. Lowering it to bound memory also lowers the +budget, and can turn a run that would have completed into `OUTPUT_TRUNCATED`. ### Encoder-bound families: `cohere` and `canary` For most hard-cap families the decoder context bounds the input *and* the -output, so the gate that accepts a clip also guarantees room for its -transcript. `cohere` and `canary` are the exception, and it is worth stating -plainly. - -Their audio bound is the **encoder** relative-position table -(`enc_pos_emb_max_len = 5000` on every shipped variant, ~400 s), while their -transcript bound is a **separate** 1024-token decoder self-KV -(`dec_max_position` / `dec_max_seq`). Audio lives in the cross-attention cache, -so it never consumes decoder context — which means the up-front gate has no way -to predict whether the transcript will fit. It cannot: transcript length is not -a function of the input the way audio-token count is. - -The consequence: a clip is accepted at up to ~400 s and can still return -`TRANSCRIBE_ERR_OUTPUT_TRUNCATED` once its transcript passes ~1018 tokens. -Measured on a 197 s English clip, both families truncate — canary at 1015 -tokens, cohere at 1014. This is the one place where an accepted clip is not -guaranteed a complete transcript, and it is a property of the checkpoints, not -of the decode budget: the budget already hands these families 1018 of their -1024 positions, which is everything there is. - -`max_audio_ms` is deliberately the **architectural** bound — the longest clip -the encoder table can index without aliasing — not a quality recommendation. -Upstream's recommended working window is much shorter: +output, so the gate that accepts a clip also guarantees room for its transcript. +`cohere` and `canary` are the exception: their audio bound is the encoder +relative-position table (`enc_pos_emb_max_len = 5000`, ~400 s) while their +transcript bound is a separate 1024-token decoder self-KV (`dec_max_position` / +`dec_max_seq`). Audio lives in the cross-attention cache and never consumes +decoder context, so the up-front gate cannot predict whether the transcript +fits. A clip accepted at ~400 s can still return +`TRANSCRIBE_ERR_OUTPUT_TRUNCATED` past ~1018 tokens (measured on a 197 s English +clip: canary 1015, cohere 1014). + +`max_audio_ms` here is the architectural bound — the longest clip the encoder +table can index without aliasing — not a quality recommendation: | Family | Upstream recommended clip | Architectural gate | Transcript bound | | --- | --- | --- | --- | -| `canary` | 40 s (>40 s is chunked upstream with 1 s overlap) | ~400 s | 1024 tokens | +| `canary` | 40 s (chunked upstream with 1 s overlap) | ~400 s | 1024 tokens | | `cohere` | 35 s (`max_audio_clip_s`) | ~400 s | 1024 tokens | -Those recommended windows are advisory and are **not** currently reported -through the ABI. They belong in a future "recommended window" capability field, -landing alongside chunked long-form support — not in `max_audio_ms`, which must -keep meaning "the longest clip this model can physically accept". +The recommended windows are advisory and are not reported through the ABI. ### 3. Soft window — warn and proceed @@ -273,24 +237,17 @@ detect truncation should check `transcribe_was_truncated()` after finalize. ## Design notes (for maintainers) -- `generation_reserve` is a floor, not a cap. The gate and `max_audio_ms` - reserve it so an accepted clip is guaranteed at least that much output room; - the per-run budget then scales up with the audio (see "The decode budget"). - Changing a family's reserve moves its published `max_audio_ms`, so it is not - a free knob — the budget rule is the thing to tune. -- For `cohere` / `canary`, `max_audio_ms` is the encoder bound and the decoder - self-KV separately bounds the transcript, so an accepted clip is *not* - guaranteed a complete transcript (see "Encoder-bound families"). Do not - "fix" this by lowering `max_audio_ms` to the upstream recommended window: - that field means the architectural maximum, and `audio_from_caps` exists - precisely to keep the encoder bound from shrinking when `n_ctx` drops. The - fix is chunked long-form plus a separate recommended-window field. +- `generation_reserve` is a floor, not a cap: changing a family's reserve moves + its published `max_audio_ms`, so tune the budget rule instead. +- For `cohere` / `canary`, do not "fix" the transcript bound by lowering + `max_audio_ms` to the upstream recommended window — that field means the + architectural maximum, and `audio_from_caps` exists to keep the encoder bound + from shrinking when `n_ctx` drops. The fix is chunked long-form plus a + separate recommended-window field. - The decoder positional encoding on both is sinusoidal, not learned (canary's - GGUF publishes `learn_positional_encodings = false`; cohere's upstream config - records the same), so the 1024-entry table is a conversion-time artifact - rather than a trained weight. Regenerating it longer is technically possible - and is still the wrong move — it would run an AED an order of magnitude past - its supported window. Upstream's own answer to long audio here is chunking. + GGUF publishes `learn_positional_encodings = false`), so the 1024-entry table + is a conversion-time artifact, not a trained weight. Regenerating it longer + would still run an AED an order of magnitude past its supported window. - The upfront gate and `max_audio_ms` share a shape for decoder-context-bound families but differ in precision: `max_audio_ms ≈ (ceiling − representative_prompt − generation_reserve) / tokens_per_ms`, diff --git a/docs/porting/families/canary.md b/docs/porting/families/canary.md index 67ced9cf..82fceb5e 100644 --- a/docs/porting/families/canary.md +++ b/docs/porting/families/canary.md @@ -141,12 +141,7 @@ uv run scripts/bench/run.py \ - Output head: LM head over the concatenated SP vocabulary. Decoding is beam search by default for the original canary-1b (beam=5, length_penalty=1.0) and greedy by default for the flash variants (beam=1). - Tokenizer: concatenated SentencePiece — one SP model per language concatenated into a single vocabulary. canary-1b-v2 is 16,384 pieces; flash/180m-flash/1b vocab sizes are not stated on model cards (Stage 2 fills from .nemo). - Audio length contract: native ≤40 s direct inference. <1 s is symmetrically zero-padded to 1 s. >40 s is handled by an external chunked inference script with 1 s overlap (canary-1b-v2 chunk len defaults to 40 s; canary-1b-flash 10 s; canary-180m-flash 10 s). **Long-form / streaming is out of scope for the v1 port.** -- Port limits (see `docs/input-limits.md`): the input gate is the encoder - relative-position table (`enc_pos_emb_max_len = 5000`, ~400 s), not the 40 s - upstream window, while the decoder self-KV (`dec_max_position = 1024`) - separately bounds the transcript. A clip inside the gate is therefore not - guaranteed a complete transcript — measured, a 197 s English clip truncates - at 1015 tokens with `TRANSCRIBE_ERR_OUTPUT_TRUNCATED`. +- Port limits: the input gate is the encoder rel-pos table (`enc_pos_emb_max_len = 5000`, ~400 s), not the 40 s upstream window, while the decoder self-KV (`dec_max_position = 1024`) separately bounds the transcript — so an accepted clip is not guaranteed a complete transcript. See `docs/input-limits.md`. ## Capabilities (from intake) diff --git a/docs/porting/families/cohere.md b/docs/porting/families/cohere.md index 3a50e305..0314d445 100644 --- a/docs/porting/families/cohere.md +++ b/docs/porting/families/cohere.md @@ -17,15 +17,11 @@ native Transformers. C++ CPU validation passes locally. ## Audio length contract -- Upstream recommended clip length: **35 s** (`max_audio_clip_s` in the - upstream config). Longer audio is expected to be segmented by the caller. -- Architectural bounds, both read from the GGUF: the encoder relative-position - table `enc_pos_emb_max_len = 5000` (~400 s) is the input gate, and the - decoder self-KV `dec_max_seq = 1024` separately bounds the transcript. -- Audio lives in the cross-attention cache and never consumes decoder context, - so a clip well inside the ~400 s gate can still exhaust the 1024-token - transcript budget and return `TRANSCRIBE_ERR_OUTPUT_TRUNCATED` (measured: a - 197 s English clip truncates at 1014 tokens). See `docs/input-limits.md`. +Upstream recommends **35 s** clips (`max_audio_clip_s`); longer audio is +expected to be segmented by the caller. The port gates on the encoder +(`enc_pos_emb_max_len = 5000`, ~400 s) and bounds the transcript separately on +the decoder self-KV (`dec_max_seq = 1024`), so an accepted clip is not +guaranteed a complete transcript. See `docs/input-limits.md`. ## References diff --git a/src/arch/canary/model.cpp b/src/arch/canary/model.cpp index 8c2f58a8..775c1995 100644 --- a/src/arch/canary/model.cpp +++ b/src/arch/canary/model.cpp @@ -219,8 +219,7 @@ constexpr float kBnEps = 1e-5f; // overrun is kept as a partial and flagged via // transcribe_was_truncated(), not rejected. -// Generation reserve: floor under the per-run budget, which scales with the -// audio and clamps to the decoder self-KV. See transcribe-decode-budget.h. +// Generation reserve: floor under the per-run decode budget. constexpr int k_gen_reserve = 512; // Predicted encoder frame count T_enc for a given mel frame count. The @@ -397,9 +396,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->limits.audio_from_caps = true; m->limits.model_max_ctx = m->hparams.dec_max_position; m->limits.gen_reserve = k_gen_reserve; - // Encoder rate, for the duration-derived decode budget. Not used for - // effective_max_audio_ms here (audio_from_caps pins that to the encoder - // bound), so publishing it changes no advertised limit. + // Encoder rate, for the decode budget only: audio_from_caps pins + // effective_max_audio_ms to the encoder bound, so this moves no limit. if (m->hparams.enc_subsampling_factor > 0 && m->hparams.fe_hop_length > 0 && m->hparams.fe_sample_rate > 0) { m->limits.ms_per_audio_token = static_cast(m->hparams.enc_subsampling_factor) * m->hparams.fe_hop_length * 1000.0 / m->hparams.fe_sample_rate; @@ -1626,7 +1624,6 @@ transcribe_status run_batch(transcribe_session * session, // raised) by the caller's n_ctx knob. Default knob (0) leaves it at // dec_max_position, so in-spec batched decode is unchanged. const int n_ctx_cap = canary_context_ceiling(cc->n_ctx, hp); - // One budget for the whole batch, sized from the longest surviving row. const int max_new = transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc_max, cm->limits.ms_per_audio_token), k_gen_reserve, prompt_len, n_ctx_cap); diff --git a/src/arch/canary_qwen/model.cpp b/src/arch/canary_qwen/model.cpp index 9b677386..386fd9da 100644 --- a/src/arch/canary_qwen/model.cpp +++ b/src/arch/canary_qwen/model.cpp @@ -117,9 +117,7 @@ constexpr float kBnEps = 1e-5f; // TRANSCRIBE_ERR_INPUT_TOO_LONG; a transcript that fills the generation budget // before end-of-stream is flagged via transcribe_was_truncated(). -// Generation reserve: what the input gate keeps free, what max_audio_ms -// subtracts, and the floor under the per-run budget. See -// transcribe-decode-budget.h. +// Generation reserve: what the input gate keeps free, and the decode-budget floor. constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum @@ -922,8 +920,6 @@ transcribe_status run(transcribe_session * context, return TRANSCRIBE_ERR_INPUT_TOO_LONG; } - // Per-run budget: duration-derived, floored at the reserve the gate above - // just guaranteed, clamped to the context left. const int max_new = transcribe::pick_decode_budget( transcribe::predict_transcript_tokens(T_enc, cm->limits.ms_per_audio_token), k_gen_reserve, T_prompt, ceiling); @@ -1483,7 +1479,6 @@ transcribe_status run_batch(transcribe_session * session, return TRANSCRIBE_OK; } T_enc_max = std::max(1, T_enc_max); - // One budget for the whole batch, sized from the longest surviving row. const int max_new = transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc_max, cm->limits.ms_per_audio_token), k_gen_reserve, max_T_prompt, ceiling); diff --git a/src/arch/cohere/model.cpp b/src/arch/cohere/model.cpp index 2accacf9..bdf8050e 100644 --- a/src/arch/cohere/model.cpp +++ b/src/arch/cohere/model.cpp @@ -417,8 +417,7 @@ transcribe_status promote_conv_pw_to_f32_on_cpu(CohereModel & m) { constexpr const char k_default_variant[] = "cohere-asr"; -// Generation reserve: floor under the per-run budget, which scales with the -// audio and clamps to the decoder self-KV. See transcribe-decode-budget.h. +// Generation reserve: floor under the per-run decode budget. constexpr int k_gen_reserve = 512; // Forward declarations for the Arch trait below. @@ -1573,7 +1572,6 @@ transcribe_status run_batch(transcribe_session * session, // Honor the session context cap (same ceiling the single-shot path uses), // not the raw model max — so a lowered n_ctx bounds batch decoder KV too. const int n_ctx_cap = cohere_dec_ctx_ceiling(cc->n_ctx, hp); - // One budget for the whole batch, sized from the longest surviving row. const int max_new = transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc_max, cm->limits.ms_per_audio_token), k_gen_reserve, prompt_len, n_ctx_cap); diff --git a/src/arch/funasr_nano/model.cpp b/src/arch/funasr_nano/model.cpp index 1a2ab40e..751d6ede 100644 --- a/src/arch/funasr_nano/model.cpp +++ b/src/arch/funasr_nano/model.cpp @@ -81,9 +81,7 @@ constexpr const char k_default_variant[] = "fun-asr-nano-2512"; // transcribe_was_truncated(). // --------------------------------------------------------------------------- -// Generation reserve: what the input gate keeps free, what max_audio_ms -// subtracts, and the floor under the per-run budget. See -// transcribe-decode-budget.h. +// Generation reserve: what the input gate keeps free, and the decode-budget floor. constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum, @@ -679,8 +677,6 @@ transcribe_status run(transcribe_session * session, return TRANSCRIBE_ERR_INPUT_TOO_LONG; } - // Per-run budget: duration-derived, floored at the reserve the gate above - // just guaranteed, clamped to the context left. const int max_new = transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_audio, cm->limits.ms_per_audio_token), k_gen_reserve, T_prompt, ceiling); @@ -1180,7 +1176,6 @@ transcribe_status run_batch(transcribe_session * session, } return TRANSCRIBE_OK; } - // One budget for the whole batch, sized from the longest surviving row. const int max_new = transcribe::pick_decode_budget( transcribe::predict_transcript_tokens(max_T_audio, cm->limits.ms_per_audio_token), k_gen_reserve, max_T_prompt, ceiling); diff --git a/src/arch/granite/model.cpp b/src/arch/granite/model.cpp index 533925ec..959c314a 100644 --- a/src/arch/granite/model.cpp +++ b/src/arch/granite/model.cpp @@ -81,9 +81,7 @@ constexpr float kBnEps = 1e-5f; // Over-length input is rejected up front with TRANSCRIBE_ERR_INPUT_TOO_LONG // rather than silently aliasing RoPE past the trained range. -// Generation reserve: what the input gate keeps free, what max_audio_ms -// subtracts, and the floor under the per-run budget. See -// transcribe-decode-budget.h. +// Generation reserve: what the input gate keeps free, and the decode-budget floor. constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum, @@ -1051,8 +1049,6 @@ transcribe_status run(transcribe_session * ctx_base, return TRANSCRIBE_ERR_INPUT_TOO_LONG; } - // Per-run budget: duration-derived, floored at the reserve the gate above - // just guaranteed, clamped to the context left. const int gen_budget = transcribe::pick_decode_budget( transcribe::predict_transcript_tokens(n_audio_tokens, cm->limits.ms_per_audio_token), k_gen_reserve, T_prompt, ceiling); @@ -1204,9 +1200,8 @@ transcribe_status run(transcribe_session * ctx_base, // n_ctx of the KV cache bounds the max generation length we can // attend over. const int max_n_kv = cc->kv.n_ctx; - // Bound generation by the decode budget, the allocated cache, AND the - // context ceiling. gen_budget is already clamped to ceiling - T_prompt; - // the other two terms guard the cache the bucket rounding actually gave us. + // gen_budget is already clamped to ceiling - T_prompt; the other two terms + // guard the cache the bucket rounding actually gave us. const int max_steps = std::min({ gen_budget, max_n_kv - T_prompt, ceiling - T_prompt }); ggml_context * step_ctx = nullptr; @@ -1609,7 +1604,6 @@ transcribe_status run_batch(transcribe_session * session, return TRANSCRIBE_OK; } n_audio_max = std::max(1, n_audio_max); - // One budget for the whole batch, sized from the longest surviving row. const int max_new = transcribe::pick_decode_budget( transcribe::predict_transcript_tokens(n_audio_max, cm->limits.ms_per_audio_token), k_gen_reserve, max_T_prompt, ceiling); diff --git a/src/arch/moss/model.cpp b/src/arch/moss/model.cpp index 72e7ea7b..5133a957 100644 --- a/src/arch/moss/model.cpp +++ b/src/arch/moss/model.cpp @@ -760,8 +760,7 @@ transcribe_status run(transcribe_session * session, return TRANSCRIBE_ERR_INPUT_TOO_LONG; } - // Predicts above the plain audio-token count: the transcript carries - // speaker markers ([start]/[Sxx]/[end]) on top of the text. + // Above the plain audio-token count: the transcript carries [start]/[Sxx]/[end] markers. const int gen_budget = transcribe::pick_decode_budget(2 * T_enc + 128, k_max_new, T_prompt, ceiling); // KV cache (grow-to-fit, clamped to ceiling). Short inputs retain the old @@ -1170,9 +1169,6 @@ transcribe_status run_batch(transcribe_session * session, return TRANSCRIBE_OK; } - // One budget for the whole batch, sized from the longest surviving row. - // Same prediction as run(): above the plain audio-token count, because the - // transcript carries speaker markers on top of the text. const int batch_budget = transcribe::pick_decode_budget(2 * max_T_enc + 128, k_max_new, max_T_prompt, ceiling); int max_n_kv = 1024; diff --git a/src/arch/qwen3_asr/model.cpp b/src/arch/qwen3_asr/model.cpp index f2912e74..250bef3e 100644 --- a/src/arch/qwen3_asr/model.cpp +++ b/src/arch/qwen3_asr/model.cpp @@ -75,9 +75,7 @@ constexpr const char k_default_variant[] = "qwen3-asr"; // transcript that fills the generation budget before end-of-stream is flagged // via transcribe_was_truncated(). -// Generation reserve: what the input gate keeps free, what max_audio_ms -// subtracts, and the floor under the per-run budget. See -// transcribe-decode-budget.h. +// Generation reserve: what the input gate keeps free, and the decode-budget floor. constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum, @@ -739,9 +737,6 @@ transcribe_status run(transcribe_session * session, return TRANSCRIBE_ERR_INPUT_TOO_LONG; } - // Per-run budget: duration-derived, floored at the reserve the gate above - // just guaranteed, clamped to the context left. The old flat 256 truncated - // every clip past ~75 s of speech with the context still mostly free. const int max_new = transcribe::pick_decode_budget( transcribe::predict_transcript_tokens(T_enc, cm->limits.ms_per_audio_token), k_gen_reserve, T_prompt, ceiling); @@ -878,7 +873,7 @@ transcribe_status run(transcribe_session * session, generated_ids.push_back(next_tok); t_prefill_logits_us = ggml_time_us() - t_prefill_logits_start; - // Step loop. max_new is the per-run decode budget resolved above. + // Step loop. const int32_t eos_id = cm->hparams.eos_token_id; int cur_past = T_prompt; @@ -1591,7 +1586,6 @@ transcribe_status run_batch(transcribe_session * session, } return TRANSCRIBE_OK; } - // One budget for the whole batch, sized from the longest surviving row. const int max_new = transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(max_T_enc, cm->limits.ms_per_audio_token), k_gen_reserve, max_T_prompt, ceiling); diff --git a/src/arch/voxtral/model.cpp b/src/arch/voxtral/model.cpp index 6f275972..aba6fa73 100644 --- a/src/arch/voxtral/model.cpp +++ b/src/arch/voxtral/model.cpp @@ -77,16 +77,9 @@ namespace { constexpr const char k_default_variant[] = "voxtral-mini-3b-2507"; // Floor on the decoder text budget for short clips (also Whisper's per-chunk -// cap). Long audio scales the budget up with the audio length — see run(). +// cap); longer audio scales the budget up. See transcribe-decode-budget.h. constexpr int k_decode_budget_min = 448; -// Decode budget (max new text tokens) for an utterance with `n_audio` audio -// embedding tokens. Thin wrapper over transcribe-decode-budget.h. Greedy decode -// stops at EOS well before this, so a generous ceiling costs only its KV. -int pick_decode_budget(int n_audio, int t_prompt, int model_max) { - return transcribe::pick_decode_budget(n_audio, k_decode_budget_min, t_prompt, model_max); -} - // Chunked prefill — see decoder.h. Walks the prompt in blocks against the // growing KV cache and returns the final position's logits. The prompt is // laid out [prefix | audio | suffix], so each chunk holds at most one run of @@ -768,7 +761,7 @@ transcribe_status run(transcribe_session * session, n_audio_total, T_prompt - n_audio_total, model_max, T_prompt + k_gen_reserve); return TRANSCRIBE_ERR_INPUT_TOO_LONG; } - const int max_new = pick_decode_budget(n_audio_total, T_prompt, model_max); + const int max_new = transcribe::pick_decode_budget(n_audio_total, k_decode_budget_min, T_prompt, model_max); const int want_ctx = causal_lm::pick_kv_cache_context(T_prompt + max_new, model_max); if (cc->kv_cache.n_ctx < want_ctx) { const ggml_type kv_type = (cc->kv_type == TRANSCRIBE_KV_TYPE_F32) ? GGML_TYPE_F32 : GGML_TYPE_F16; @@ -1333,7 +1326,7 @@ transcribe_status run_batch(transcribe_session * session, // Size the batched KV cache to the longest prompt plus the decode budget, // clamped to the context ceiling. kv_init_batched grows the cache on demand. const int model_max = ctx_ceiling; - const int max_new = pick_decode_budget(T_audio_max, max_T_prompt, model_max); + const int max_new = transcribe::pick_decode_budget(T_audio_max, k_decode_budget_min, max_T_prompt, model_max); int max_n_kv = 1024; while (max_n_kv < max_T_prompt + max_new) { max_n_kv *= 2; diff --git a/src/transcribe-decode-budget.h b/src/transcribe-decode-budget.h index 29a152a5..4230d38c 100644 --- a/src/transcribe-decode-budget.h +++ b/src/transcribe-decode-budget.h @@ -1,16 +1,9 @@ // transcribe-decode-budget.h - shared per-run autoregressive decode budget. // -// INTERNAL. Header-only, like the ABI helpers in transcribe-abi.h. -// -// Autoregressive families used to cap generation at a constant (256 / 512) -// that ignored the audio, while the input gate accepted clips far longer than -// that many tokens could describe — so a long clip came back -// TRANSCRIBE_ERR_OUTPUT_TRUNCATED with the context still mostly free. The -// budget has to track the input instead. See docs/input-limits.md. -// -// This is deliberately NOT a public run parameter. The only caller-facing knob -// is transcribe_session_params::n_ctx, which lowers `ceiling` and the budget -// with it. +// The budget has to track the input: a flat per-family constant returns +// TRANSCRIBE_ERR_OUTPUT_TRUNCATED on a long clip with the decoder context still +// mostly free. Deliberately not a public run parameter — the only caller-facing +// knob is transcribe_session_params::n_ctx. See docs/input-limits.md. #pragma once @@ -19,23 +12,15 @@ namespace transcribe { // Speech-rate bound on transcript length, in text tokens per second of audio. -// Generous on purpose: English BPE measures ~3.4/sec and CJK is denser. 12 also -// matches what the 80 ms-per-token encoders already got from their raw -// audio-token count (12.5/sec), so the duration form leaves them where they were. +// Generous on purpose: English BPE measures ~3.4/sec and CJK is denser. constexpr int k_transcript_tokens_per_sec = 12; -// Predicted transcript length, in tokens, for an utterance the encoder turned -// into `audio_tokens` outputs at `ms_per_audio_token` each. -// -// The raw audio-token count is NOT a portable proxy: encoder rates differ ~6x. -// Most emit one token per 80 ms (12.5/sec, safely above any speech rate), but -// funasr_nano's LFR frontend emits one per ~480 ms (2.08/sec) — below the -// text-token rate, so its count under-predicts and the budget truncates a -// transcript the context had room for. Going via seconds removes the encoder -// rate from the estimate. +// Predicted transcript length for `audio_tokens` encoder outputs at +// `ms_per_audio_token` each. Going via seconds is required, not cosmetic: +// encoder rates differ ~6x, and funasr_nano's LFR frontend emits one token per +// ~480 ms — below the text-token rate, so its raw count under-predicts. // -// A non-positive `ms_per_audio_token` means the family published no rate; fall -// back to the audio-token count, the pre-rate-aware behavior. +// A non-positive rate means the family published none; fall back to the count. inline int predict_transcript_tokens(int audio_tokens, double ms_per_audio_token) { if (audio_tokens <= 0) { return 0; @@ -45,7 +30,6 @@ inline int predict_transcript_tokens(int audio_tokens, double ms_per_audio_token } const double seconds = static_cast(audio_tokens) * ms_per_audio_token / 1000.0; const double predicted = seconds * k_transcript_tokens_per_sec; - // Clamp into int range; callers clamp again to the context actually left. if (predicted <= 0.0) { return 0; } @@ -53,25 +37,10 @@ inline int predict_transcript_tokens(int audio_tokens, double ms_per_audio_token return predicted >= k_int_max ? 2147483647 : static_cast(predicted); } -// Per-run decode budget, in transcript tokens. -// -// predicted upper-bound estimate of transcript length, normally from -// predict_transcript_tokens() above. A family whose output -// carries more than the transcript (moss emits speaker -// markers) scales it up. -// floor_tokens never plan for fewer. Each family passes its historical -// fixed budget, so a clip that fits today stays -// byte-identical and the reserve max_audio_ms subtracts (via -// transcribe_model::LimitsBasis::gen_reserve) stays exact. -// t_prompt prompt tokens already in the decoder context. Includes the -// audio embeddings for in-context families; 0 for -// encoder-decoder families whose audio lives in a separate -// cross-attention cache. -// ceiling decoder context ceiling, already lowered (never raised) by -// transcribe_session_params::n_ctx. -// -// Clamped to the context actually left, never negative. Zero means the prompt -// already fills the ceiling; callers gate that up front (INPUT_TOO_LONG). +// Per-run decode budget: `predicted` raised to `floor_tokens` and clamped to the +// context left under `ceiling`. `floor_tokens` is the family's historical fixed +// budget, so short clips decode byte-identically and the reserve max_audio_ms +// subtracts (transcribe_model::LimitsBasis::gen_reserve) stays exact. inline int pick_decode_budget(int predicted, int floor_tokens, int t_prompt, int ceiling) { int budget = std::max(floor_tokens, predicted); const int room = ceiling - t_prompt; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3f8267bd..1e2318a0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -203,13 +203,10 @@ add_test(NAME transcribe_prefill_chunk_mask_unit COMMAND transcribe_prefill_chun # ----------------------------------------------------------------------------- # Per-run decode budget rule (pure host, no model) -# ----------------------------------------------------------------------------- add_executable(transcribe_decode_budget_unit decode_budget_unit.cpp) -target_link_libraries(transcribe_decode_budget_unit PRIVATE transcribe ggml) - target_include_directories(transcribe_decode_budget_unit PRIVATE ${CMAKE_SOURCE_DIR}/src) diff --git a/tests/decode_budget_unit.cpp b/tests/decode_budget_unit.cpp index 36b3524e..45e69272 100644 --- a/tests/decode_budget_unit.cpp +++ b/tests/decode_budget_unit.cpp @@ -1,22 +1,6 @@ -// Per-run decode budget rule (pure host, no model). -// -// Every autoregressive family used to hardcode its generation budget as a -// constant that ignored the audio entirely (qwen3_asr 256, canary 512, ...). -// The up-front input gate meanwhile accepted clips orders of magnitude longer -// than that many tokens could describe — qwen3_asr advertises 87 minutes of -// audio against a 256-token output cap — so any clip past roughly a minute of -// speech came back TRANSCRIBE_ERR_OUTPUT_TRUNCATED with context to spare. -// -// transcribe::pick_decode_budget replaces those constants. This test pins the -// two properties the families depend on, because getting either wrong is -// silent: too low and long clips truncate again; too high and the KV -// allocation (112 KiB per token on qwen3-asr) balloons on every run. -// -// 1. Never below the family's floor -> a clip that fits today is unchanged, -// and transcribe_capabilities::max_audio_ms (which subtracts that same -// floor via LimitsBasis::gen_reserve) stays exact. -// 2. Never past the context left -> prompt + budget always fits the -// ceiling, which is what transcribe_session_params::n_ctx lowers. +// Per-run decode budget rule (pure host, no model). Getting it wrong is silent +// both ways: too low and long clips truncate with context to spare, too high +// and the KV allocation balloons every run. See docs/input-limits.md. #include "transcribe-decode-budget.h" @@ -26,14 +10,6 @@ namespace { int g_failures = 0; -#define CHECK(cond) \ - do { \ - if (!(cond)) { \ - std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ - ++g_failures; \ - } \ - } while (0) - void check_predict(const char * what, int audio_tokens, double ms_per_audio_token, int expected) { const int got = transcribe::predict_transcript_tokens(audio_tokens, ms_per_audio_token); if (got != expected) { @@ -55,57 +31,39 @@ void check_budget(const char * what, int predicted, int floor_tokens, int t_prom } // namespace int main(void) { - // ---- Prediction is duration-based, not audio-token based. ---- - // The whole point: encoder rates differ ~6x, so two families that heard the - // same 197 seconds of speech must predict the same transcript length even - // though one emitted 6x more audio tokens than the other. + // Duration-based: two encoders that heard the same 197 s must agree. check_predict("80 ms encoder, 197 s", /*audio_tokens=*/2463, /*ms_per_audio_token=*/80.0, /*expected=*/2364); check_predict("480 ms LFR encoder, same 197 s", /*audio_tokens=*/410, /*ms_per_audio_token=*/480.0, /*expected=*/2361); - // funasr_nano's real regression: its raw audio-token count (410) predicts - // far below the ~700 tokens this clip's transcript actually needs. - CHECK(transcribe::predict_transcript_tokens(410, 480.0) > 700); - // An unpublished rate falls back to the audio-token count (prior behavior). + // An unpublished rate falls back to the audio-token count. check_predict("unknown rate falls back", 2463, 0.0, 2463); check_predict("negative rate falls back", 2463, -1.0, 2463); check_predict("no audio", 0, 80.0, 0); - // ---- Property 1: the floor holds for short audio. ---- - // A clip whose audio-token count is below the family's historical fixed - // budget must still get that budget, so its decode is byte-identical to - // what shipped before. + // The floor holds for short audio, so those decodes are unchanged. check_budget("short clip keeps the floor", /*predicted=*/10, /*floor=*/256, /*t_prompt=*/64, /*ceiling=*/65536, /*expected=*/256); check_budget("floor applies at zero audio", 0, 256, 64, 65536, 256); check_budget("canary floor", 300, 512, 6, 1024, 512); - // ---- Property 2: the budget scales past the floor. ---- - // This is the fix. qwen3-asr at 80 ms per audio token: a 5-minute clip is - // 3824 audio tokens, which used to decode under a flat 256-token cap. + // Past the floor the budget scales with the audio. This is the fix. check_budget("5 min qwen3-asr scales", /*predicted=*/3824, /*floor=*/256, /*t_prompt=*/3872, /*ceiling=*/65536, /*expected=*/3824); check_budget("20 min qwen3-asr scales", 15000, 256, 15048, 65536, 15000); - // ---- Property 3: the context ceiling always wins. ---- - // canary is the tight case: a 400 s clip is ~5000 encoder frames but the - // decoder self-KV is only 1024, so the budget clamps to what is left. + // The ceiling always wins: canary sees ~5000 encoder frames into a 1024 self-KV. check_budget("canary clamps to dec ctx", /*predicted=*/5000, /*floor=*/512, /*t_prompt=*/6, /*ceiling=*/1024, /*expected=*/1018); check_budget("clamp beats the floor too", 10, 512, 900, 1024, 124); - // ---- Property 4: n_ctx is the knob. ---- - // Lowering transcribe_session_params::n_ctx lowers `ceiling`, and the - // budget must follow it down. Same inputs as the 5-minute case above. + // n_ctx is the knob: lowering `ceiling` lowers the budget with it. check_budget("full n_ctx leaves the audio-sized budget intact", 3824, 256, 3872, 8192, 3824); check_budget("lowered n_ctx lowers the budget", 3824, 256, 3872, 6000, 2128); check_budget("n_ctx below the floor still clamps", 3824, 256, 3872, 4000, 128); - // ---- Property 5: never negative. ---- - // A prompt that already fills the ceiling yields 0, not a negative step - // count. Families gate this case up front with INPUT_TOO_LONG; the helper - // must not hand a negative loop bound to a step loop regardless. + // Never a negative step count. check_budget("prompt exactly fills ceiling", 3824, 256, 1024, 1024, 0); check_budget("prompt overruns ceiling", 3824, 256, 2048, 1024, 0); diff --git a/tests/qwen3_asr_batch_truncation.cpp b/tests/qwen3_asr_batch_truncation.cpp index aebadd44..9c4ad86b 100644 --- a/tests/qwen3_asr_batch_truncation.cpp +++ b/tests/qwen3_asr_batch_truncation.cpp @@ -2,20 +2,11 @@ // and batch decode paths both report mid-decode OUTPUT_TRUNCATED for a // causal_lm (LLM-decoder) family. // -// qwen3_asr's decode budget scales with the audio and is clamped to the -// decoder context left after the prompt (transcribe-decode-budget.h). It used -// to be a flat 256 tokens, which truncated any clip past ~75 s of speech even -// with 65000 tokens of context free; that was the bug, and the first block -// below is its regression guard — a 197 s clip must now decode to EOS. -// -// Truncation is still reachable, and still has to be reported: lowering -// transcribe_session_params::n_ctx lowers the ceiling, which lowers the budget -// with it. That is the only knob a caller has over the output length, so it is -// also how this test forces the truncation path. Per docs/input-limits.md a -// truncated decode must surface as the hard TRANSCRIBE_ERR_OUTPUT_TRUNCATED -// status (partial transcript retained, transcribe_was_truncated() set) in BOTH -// the single-shot and batch paths, while a short clip that finishes under the -// budget stays OK and the whole-batch call still returns OK. +// The budget scales with the audio (transcribe-decode-budget.h), so the 197 s +// clip decodes to EOS at the default context and truncation has to be forced +// with a lowered n_ctx. Per docs/input-limits.md it must then surface as the +// hard TRANSCRIBE_ERR_OUTPUT_TRUNCATED status in BOTH paths, while a short clip +// stays OK and the whole-batch call still returns OK. // // This is the causal_lm counterpart to moonshine_streaming_batch_truncation // (which exercises the encoder-decoder batch loop in transcribe-batch-util.cpp). @@ -111,10 +102,8 @@ int main() { return 1; } - // ---- Regression guard for the flat-256 budget bug ---- - // At the default (full) context the 197 s clip must decode all the way to - // EOS. Before the budget scaled with the audio this returned - // OUTPUT_TRUNCATED at 256 tokens with ~63000 tokens of context unused. + // ---- Regression guard: at full context the 197 s clip decodes to EOS ---- + // The flat 256-token budget returned OUTPUT_TRUNCATED here. { transcribe_session_params full_sp; transcribe_session_params_init(&full_sp); @@ -131,16 +120,9 @@ int main() { } // ---- Lowered n_ctx: the only caller-facing control over output length ---- - // Measured on love-loss.wav (~197 s): T_enc = 2563 audio tokens (the clip - // encodes at 76.9 ms/token), T_prompt = 2578 with the chat affixes, and the - // full transcript is 701-750 tokens. The input gate reserves k_gen_reserve - // (256) on top of the prompt, so the ceiling must be >= 2834 for the clip - // to be accepted at all. - // - // 3072 clears that gate by 238 tokens and leaves 494 tokens of decode - // budget, about 210 short of the full transcript. Both margins absorb small - // prompt-template drift; if this ever returns INPUT_TOO_LONG the prompt - // grew, and if it returns OK the transcript shrank. + // love-loss.wav (~197 s): T_prompt = 2578, transcript 701-750 tokens, gate + // needs ceiling >= 2834. So 3072 is accepted and leaves a 494-token budget, + // ~210 short. INPUT_TOO_LONG here means the prompt grew; OK means it shrank. transcribe_session_params sp; transcribe_session_params_init(&sp); sp.n_ctx = 3072; From b2b6bf150842c4b5561c09fbcd5c2b267bb2a4b4 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 21 Sep 2026 16:03:56 +0800 Subject: [PATCH 6/6] more slimming --- docs/input-limits.md | 73 ++++------------------------ docs/models/qwen3-asr.md | 8 +-- docs/porting/families/canary.md | 1 - docs/porting/families/cohere.md | 9 +--- src/transcribe-decode-budget.h | 22 ++------- tests/qwen3_asr_batch_truncation.cpp | 45 +++-------------- 6 files changed, 25 insertions(+), 133 deletions(-) diff --git a/docs/input-limits.md b/docs/input-limits.md index 3bdef719..e53123f8 100644 --- a/docs/input-limits.md +++ b/docs/input-limits.md @@ -73,13 +73,10 @@ need and do not have a length gate. | --- | --- | --- | | qwen3_asr, canary_qwen, funasr_nano, granite, granite_nar, voxtral, cohere, canary | decoder context window (`dec_max_position_embeddings` / `dec_max_seq`), or the encoder positional table (`enc_pos_emb_max_len`, for cohere/canary) — all from GGUF | KV cache grows to fit, clamped to the model's true max. Over-length input is **rejected before the decode** (or before the encoder, where the encoder table is the binding limit) with `TRANSCRIBE_ERR_INPUT_TOO_LONG`. | -These families wrap an LLM-style decoder whose context window -(`audio_tokens + prompt + generation`) is the binding constraint. The number of -tokens a clip consumes is a deterministic function of its sample count -(`n_samples → mel frames → fixed subsampling → audio tokens`), so the library -computes the prefill size *before* running the encoder and rejects an -over-length clip immediately — the caller never pays for a compute pass that -cannot fit. The rejection goes through the log callback, not raw stderr. +A clip's sample count deterministically fixes its decoder prefill size or +encoder frame count, so the library checks the relevant bound before running +and rejects over-length input immediately. The rejection goes through the log +callback, not raw stderr. The one case that cannot be predicted up front is the transcript itself running long enough to exhaust the remaining budget mid-decode. There, the run returns @@ -91,50 +88,11 @@ mistake it for complete — and the partial output is never discarded. In `transcribe_run_batch` this is a per-utterance status (the whole-batch call still returns `TRANSCRIBE_OK`). -### The decode budget - -How much output an accepted clip may produce is derived from the clip, not -fixed. Each autoregressive family resolves a per-run decode budget as: - -```text -budget = clamp(max(generation_reserve, predicted_transcript_tokens), - 0, ceiling - prompt_tokens) -``` - -`predicted_transcript_tokens` comes from the clip's duration (`seconds x 12`), -recovered from the family's published `ms_per_audio_token` — not from the raw -audio-token count, whose rate differs ~6x across encoders. `generation_reserve` -is the per-family floor, the same constant the up-front gate reserves and -`max_audio_ms` subtracts. `ceiling` is the decoder context, which -`transcribe_session_params::n_ctx` lowers. `moss` and `voxtral` pass their own -estimates instead. - -`n_ctx` is the only caller-facing control over output length; there is no -per-run "max tokens" parameter. Lowering it to bound memory also lowers the -budget, and can turn a run that would have completed into `OUTPUT_TRUNCATED`. - -### Encoder-bound families: `cohere` and `canary` - -For most hard-cap families the decoder context bounds the input *and* the -output, so the gate that accepts a clip also guarantees room for its transcript. -`cohere` and `canary` are the exception: their audio bound is the encoder -relative-position table (`enc_pos_emb_max_len = 5000`, ~400 s) while their -transcript bound is a separate 1024-token decoder self-KV (`dec_max_position` / -`dec_max_seq`). Audio lives in the cross-attention cache and never consumes -decoder context, so the up-front gate cannot predict whether the transcript -fits. A clip accepted at ~400 s can still return -`TRANSCRIBE_ERR_OUTPUT_TRUNCATED` past ~1018 tokens (measured on a 197 s English -clip: canary 1015, cohere 1014). - -`max_audio_ms` here is the architectural bound — the longest clip the encoder -table can index without aliasing — not a quality recommendation: - -| Family | Upstream recommended clip | Architectural gate | Transcript bound | -| --- | --- | --- | --- | -| `canary` | 40 s (chunked upstream with 1 s overlap) | ~400 s | 1024 tokens | -| `cohere` | 35 s (`max_audio_clip_s`) | ~400 s | 1024 tokens | - -The recommended windows are advisory and are not reported through the ABI. +Autoregressive families scale the decode budget with audio length, capped by +the remaining decoder context. Lowering `n_ctx` can therefore make truncation +more likely. For `canary` and `cohere`, input and output have separate encoder +and decoder limits; `max_audio_ms` reports the encoder limit, not a recommended +chunk size. ### 3. Soft window — warn and proceed @@ -204,7 +162,7 @@ with `TRANSCRIBE_ERR_INPUT_TOO_LONG` (one-shot and batch) or surfaced via | Situation | Status | Log | Result | | --- | --- | --- | --- | -| Input within limit | `TRANSCRIBE_OK` | — | full transcript | +| Input within limit and decode completes | `TRANSCRIBE_OK` | — | full transcript | | Over-length, hard-cap family | `TRANSCRIBE_ERR_INPUT_TOO_LONG` | `ERROR` via callback | no transcript (rejected before the decode) | | Generation ran long mid-decode | `TRANSCRIBE_ERR_OUTPUT_TRUNCATED` | `WARN` via callback | partial transcript readable; `transcribe_was_truncated() == true` | | Over-window, soft-window family | `TRANSCRIBE_OK` | `WARN` via callback | full transcript (accuracy may be degraded) | @@ -237,17 +195,6 @@ detect truncation should check `transcribe_was_truncated()` after finalize. ## Design notes (for maintainers) -- `generation_reserve` is a floor, not a cap: changing a family's reserve moves - its published `max_audio_ms`, so tune the budget rule instead. -- For `cohere` / `canary`, do not "fix" the transcript bound by lowering - `max_audio_ms` to the upstream recommended window — that field means the - architectural maximum, and `audio_from_caps` exists to keep the encoder bound - from shrinking when `n_ctx` drops. The fix is chunked long-form plus a - separate recommended-window field. -- The decoder positional encoding on both is sinusoidal, not learned (canary's - GGUF publishes `learn_positional_encodings = false`), so the 1024-entry table - is a conversion-time artifact, not a trained weight. Regenerating it longer - would still run an AED an order of magnitude past its supported window. - The upfront gate and `max_audio_ms` share a shape for decoder-context-bound families but differ in precision: `max_audio_ms ≈ (ceiling − representative_prompt − generation_reserve) / tokens_per_ms`, diff --git a/docs/models/qwen3-asr.md b/docs/models/qwen3-asr.md index 963ddbed..ed816492 100644 --- a/docs/models/qwen3-asr.md +++ b/docs/models/qwen3-asr.md @@ -47,13 +47,7 @@ family. That ceiling is there to bound memory and sits far beyond any normal clip; audio past it is rejected up front with `TRANSCRIBE_ERR_INPUT_TOO_LONG` rather than silently truncated. Lowering `--n-ctx` lowers the limit (and the KV-cache footprint), and `transcribe_session_get_limits()` reports the exact -per-session value. - -Output length is not separately capped: the decode budget scales with the audio -and is bounded only by the context left after the prompt, so a clip inside the -input limit transcribes in full. `--n-ctx` bounds both — lowering it far enough -will truncate a long transcript (`TRANSCRIBE_ERR_OUTPUT_TRUNCATED`, partial text -retained). See the [input-length contract](../input-limits.md). +per-session value. See the [input-length contract](../input-limits.md). ## Quick start diff --git a/docs/porting/families/canary.md b/docs/porting/families/canary.md index 82fceb5e..5351a93c 100644 --- a/docs/porting/families/canary.md +++ b/docs/porting/families/canary.md @@ -141,7 +141,6 @@ uv run scripts/bench/run.py \ - Output head: LM head over the concatenated SP vocabulary. Decoding is beam search by default for the original canary-1b (beam=5, length_penalty=1.0) and greedy by default for the flash variants (beam=1). - Tokenizer: concatenated SentencePiece — one SP model per language concatenated into a single vocabulary. canary-1b-v2 is 16,384 pieces; flash/180m-flash/1b vocab sizes are not stated on model cards (Stage 2 fills from .nemo). - Audio length contract: native ≤40 s direct inference. <1 s is symmetrically zero-padded to 1 s. >40 s is handled by an external chunked inference script with 1 s overlap (canary-1b-v2 chunk len defaults to 40 s; canary-1b-flash 10 s; canary-180m-flash 10 s). **Long-form / streaming is out of scope for the v1 port.** -- Port limits: the input gate is the encoder rel-pos table (`enc_pos_emb_max_len = 5000`, ~400 s), not the 40 s upstream window, while the decoder self-KV (`dec_max_position = 1024`) separately bounds the transcript — so an accepted clip is not guaranteed a complete transcript. See `docs/input-limits.md`. ## Capabilities (from intake) diff --git a/docs/porting/families/cohere.md b/docs/porting/families/cohere.md index 0314d445..b68b53ff 100644 --- a/docs/porting/families/cohere.md +++ b/docs/porting/families/cohere.md @@ -15,13 +15,8 @@ native Transformers. C++ CPU validation passes locally. `[en, ar]`; config omits top-level `vocab_size` — the converter falls back to `head.num_classes`; upstream repo is gated) -## Audio length contract - -Upstream recommends **35 s** clips (`max_audio_clip_s`); longer audio is -expected to be segmented by the caller. The port gates on the encoder -(`enc_pos_emb_max_len = 5000`, ~400 s) and bounds the transcript separately on -the decoder self-KV (`dec_max_seq = 1024`), so an accepted clip is not -guaranteed a complete transcript. See `docs/input-limits.md`. +Upstream recommends segmenting audio into 35 s clips; the port enforces the +encoder's larger architectural limit. See `docs/input-limits.md`. ## References diff --git a/src/transcribe-decode-budget.h b/src/transcribe-decode-budget.h index 4230d38c..87ee28cf 100644 --- a/src/transcribe-decode-budget.h +++ b/src/transcribe-decode-budget.h @@ -1,9 +1,4 @@ -// transcribe-decode-budget.h - shared per-run autoregressive decode budget. -// -// The budget has to track the input: a flat per-family constant returns -// TRANSCRIBE_ERR_OUTPUT_TRUNCATED on a long clip with the decoder context still -// mostly free. Deliberately not a public run parameter — the only caller-facing -// knob is transcribe_session_params::n_ctx. See docs/input-limits.md. +// Shared helpers for sizing autoregressive decode budgets. #pragma once @@ -11,16 +6,10 @@ namespace transcribe { -// Speech-rate bound on transcript length, in text tokens per second of audio. -// Generous on purpose: English BPE measures ~3.4/sec and CJK is denser. +// Conservative multilingual transcript estimate, in tokens per second. constexpr int k_transcript_tokens_per_sec = 12; -// Predicted transcript length for `audio_tokens` encoder outputs at -// `ms_per_audio_token` each. Going via seconds is required, not cosmetic: -// encoder rates differ ~6x, and funasr_nano's LFR frontend emits one token per -// ~480 ms — below the text-token rate, so its raw count under-predicts. -// -// A non-positive rate means the family published none; fall back to the count. +// Fall back to the encoder-token count when its duration is unknown. inline int predict_transcript_tokens(int audio_tokens, double ms_per_audio_token) { if (audio_tokens <= 0) { return 0; @@ -37,10 +26,7 @@ inline int predict_transcript_tokens(int audio_tokens, double ms_per_audio_token return predicted >= k_int_max ? 2147483647 : static_cast(predicted); } -// Per-run decode budget: `predicted` raised to `floor_tokens` and clamped to the -// context left under `ceiling`. `floor_tokens` is the family's historical fixed -// budget, so short clips decode byte-identically and the reserve max_audio_ms -// subtracts (transcribe_model::LimitsBasis::gen_reserve) stays exact. +// Apply the family floor without exceeding the available context. inline int pick_decode_budget(int predicted, int floor_tokens, int t_prompt, int ceiling) { int budget = std::max(floor_tokens, predicted); const int room = ceiling - t_prompt; diff --git a/tests/qwen3_asr_batch_truncation.cpp b/tests/qwen3_asr_batch_truncation.cpp index 9c4ad86b..f4661ed0 100644 --- a/tests/qwen3_asr_batch_truncation.cpp +++ b/tests/qwen3_asr_batch_truncation.cpp @@ -1,30 +1,7 @@ -// qwen3_asr_batch_truncation.cpp - real-model gated test that the single-shot -// and batch decode paths both report mid-decode OUTPUT_TRUNCATED for a -// causal_lm (LLM-decoder) family. -// -// The budget scales with the audio (transcribe-decode-budget.h), so the 197 s -// clip decodes to EOS at the default context and truncation has to be forced -// with a lowered n_ctx. Per docs/input-limits.md it must then surface as the -// hard TRANSCRIBE_ERR_OUTPUT_TRUNCATED status in BOTH paths, while a short clip -// stays OK and the whole-batch call still returns OK. -// -// This is the causal_lm counterpart to moonshine_streaming_batch_truncation -// (which exercises the encoder-decoder batch loop in transcribe-batch-util.cpp). -// It specifically guards the shared src/causal_lm batched step loop's per-row -// truncation detection: that loop marks every stopped row `finished` -// regardless of WHY it stopped, so truncation must be inferred from the last -// sampled token (!= eos), not from `!finished`. A regression there makes a -// truncated batch row silently report TRANSCRIBE_OK with an incomplete -// transcript — the exact failure this test catches. -// -// Batch makeup (under the lowered n_ctx): -// row 0 = jfk.wav (~11 s) -> completes under the budget -> OK -// row 1 = love-loss.wav (~197 s) -> exceeds the budget -> OUTPUT_TRUNCATED -// -// Gating: -// - TRANSCRIBE_BUILD_REAL_MODEL_TESTS (CMake, default OFF) builds it. -// - At runtime, TRANSCRIBE_QWEN3_ASR_0_6B_GGUF points at the GGUF. If -// unset/missing (or a sample is missing), exits 77 ("skipped"). +// Real-model regression test for decode-budget scaling and single/batch +// OUTPUT_TRUNCATED parity. The long clip completes at the default context and +// truncates under a lowered n_ctx; the short clip completes in both cases. +// Requires TRANSCRIBE_QWEN3_ASR_0_6B_GGUF; missing inputs return 77. #include "transcribe.h" #include "wav.h" @@ -102,8 +79,7 @@ int main() { return 1; } - // ---- Regression guard: at full context the 197 s clip decodes to EOS ---- - // The flat 256-token budget returned OUTPUT_TRUNCATED here. + // The long clip must complete with the default context. { transcribe_session_params full_sp; transcribe_session_params_init(&full_sp); @@ -119,10 +95,7 @@ int main() { transcribe_session_free(full_s); } - // ---- Lowered n_ctx: the only caller-facing control over output length ---- - // love-loss.wav (~197 s): T_prompt = 2578, transcript 701-750 tokens, gate - // needs ceiling >= 2834. So 3072 is accepted and leaves a 494-token budget, - // ~210 short. INPUT_TOO_LONG here means the prompt grew; OK means it shrank. + // Lower n_ctx enough to force truncation without rejecting the input. transcribe_session_params sp; transcribe_session_params_init(&sp); sp.n_ctx = 3072; @@ -133,7 +106,7 @@ int main() { return 1; } - // ---- Single-shot: the long clip truncates, the short one does not. ---- + // Single-shot truncation and reset behavior. { const transcribe_status rl = transcribe_run(s, pcm_long.data(), (int) pcm_long.size(), nullptr); CHECK(rl == TRANSCRIBE_ERR_OUTPUT_TRUNCATED); @@ -147,9 +120,7 @@ int main() { CHECK(transcribe_was_truncated(s) == false); // reset + completed } - // ---- Batch parity: the shared causal_lm batched step loop must report the - // SAME per-utterance verdict. row 0 (short) finishes -> OK; row 1 (long) - // hits the budget -> OUTPUT_TRUNCATED; whole-batch call still returns OK. + // Batch results must match the single-shot results. { const float * pcms[2] = { pcm_short.data(), pcm_long.data() }; const int lens[2] = { (int) pcm_short.size(), (int) pcm_long.size() };