diff --git a/docs/input-limits.md b/docs/input-limits.md index fb7a6ae0..e53123f8 100644 --- a/docs/input-limits.md +++ b/docs/input-limits.md @@ -73,23 +73,26 @@ need and do not have a length gate. | --- | --- | --- | | qwen3_asr, canary_qwen, funasr_nano, granite, granite_nar, voxtral, cohere, canary | decoder context window (`dec_max_position_embeddings` / `dec_max_seq`), or the encoder positional table (`enc_pos_emb_max_len`, for cohere/canary) — all from GGUF | KV cache grows to fit, clamped to the model's true max. Over-length input is **rejected before the decode** (or before the encoder, where the encoder table is the binding limit) with `TRANSCRIBE_ERR_INPUT_TOO_LONG`. | -These families wrap an LLM-style decoder whose context window -(`audio_tokens + prompt + generation`) is the binding constraint. The number of -tokens a clip consumes is a deterministic function of its sample count -(`n_samples → mel frames → fixed subsampling → audio tokens`), so the library -computes the prefill size *before* running the encoder and rejects an -over-length clip immediately — the caller never pays for a compute pass that -cannot fit. The rejection goes through the log callback, not raw stderr. +A clip's sample count deterministically fixes its decoder prefill size or +encoder frame count, so the library checks the relevant bound before running +and rejects over-length input immediately. The rejection goes through the log +callback, not raw stderr. The one case that cannot be predicted up front is the transcript itself running -long enough to exhaust the remaining budget mid-decode (rare — the output would -have to be very large for the audio length). There, the run returns the hard -status `TRANSCRIBE_ERR_OUTPUT_TRUNCATED` while keeping the partial transcript -readable (exactly like an aborted run); `transcribe_was_truncated(session)` is -also set, and a `WARN` is logged. A truncated transcript is never returned as -`TRANSCRIBE_OK` — a caller cannot mistake it for complete — and the partial -output is never discarded. In `transcribe_run_batch` this is a per-utterance -status (the whole-batch call still returns `TRANSCRIBE_OK`). +long enough to exhaust the remaining budget mid-decode. There, the run returns +the hard status `TRANSCRIBE_ERR_OUTPUT_TRUNCATED` while keeping the partial +transcript readable (exactly like an aborted run); +`transcribe_was_truncated(session)` is also set, and a `WARN` is logged. A +truncated transcript is never returned as `TRANSCRIBE_OK` — a caller cannot +mistake it for complete — and the partial output is never discarded. In +`transcribe_run_batch` this is a per-utterance status (the whole-batch call +still returns `TRANSCRIBE_OK`). + +Autoregressive families scale the decode budget with audio length, capped by +the remaining decoder context. Lowering `n_ctx` can therefore make truncation +more likely. For `canary` and `cohere`, input and output have separate encoder +and decoder limits; `max_audio_ms` reports the encoder limit, not a recommended +chunk size. ### 3. Soft window — warn and proceed @@ -159,7 +162,7 @@ with `TRANSCRIBE_ERR_INPUT_TOO_LONG` (one-shot and batch) or surfaced via | Situation | Status | Log | Result | | --- | --- | --- | --- | -| Input within limit | `TRANSCRIBE_OK` | — | full transcript | +| Input within limit and decode completes | `TRANSCRIBE_OK` | — | full transcript | | Over-length, hard-cap family | `TRANSCRIBE_ERR_INPUT_TOO_LONG` | `ERROR` via callback | no transcript (rejected before the decode) | | Generation ran long mid-decode | `TRANSCRIBE_ERR_OUTPUT_TRUNCATED` | `WARN` via callback | partial transcript readable; `transcribe_was_truncated() == true` | | Over-window, soft-window family | `TRANSCRIBE_OK` | `WARN` via callback | full transcript (accuracy may be degraded) | diff --git a/docs/porting/families/cohere.md b/docs/porting/families/cohere.md index c3abae15..b68b53ff 100644 --- a/docs/porting/families/cohere.md +++ b/docs/porting/families/cohere.md @@ -15,6 +15,9 @@ native Transformers. C++ CPU validation passes locally. `[en, ar]`; config omits top-level `vocab_size` — the converter falls back to `head.num_classes`; upstream repo is gated) +Upstream recommends segmenting audio into 35 s clips; the port enforces the +encoder's larger architectural limit. See `docs/input-limits.md`. + ## References - Canonical reference: native Hugging Face Transformers diff --git a/src/arch/canary/model.cpp b/src/arch/canary/model.cpp index 6b556357..775c1995 100644 --- a/src/arch/canary/model.cpp +++ b/src/arch/canary/model.cpp @@ -14,6 +14,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-env.h" #include "transcribe-flash-policy.h" #include "transcribe-load-common.h" @@ -214,10 +215,13 @@ constexpr float kBnEps = 1e-5f; // (a) INPUT — the encoder rel-pos table (enc_pos_emb_max_len, ~400 s). // T_enc must stay within it or the runtime table aliases past the // trained range; gated up front. Drives max_audio_ms. -// (b) DECODER self-KV (dec_max_position) + 512 max-new cap bound the -// OUTPUT length; an overrun is kept as a partial and flagged via +// (b) DECODER self-KV (dec_max_position) bounds the OUTPUT length; an +// overrun is kept as a partial and flagged via // transcribe_was_truncated(), not rejected. +// Generation reserve: floor under the per-run decode budget. +constexpr int k_gen_reserve = 512; + // Predicted encoder frame count T_enc for a given mel frame count. The // FastConformer pre-encode downsamples time via stride-2, kernel-3, pad-1 // convs; each stage maps T_in -> floor((T_in-1)/2)+1. We fold that exact @@ -388,10 +392,16 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par // effective_max_audio_ms to the encoder bound regardless of n_ctx; the // decoder self-KV (which n_ctx does lower) only bounds transcript length. if (m->hparams.dec_max_position > 0) { - m->limits.has_context_cap = true; - m->limits.audio_from_caps = true; - m->limits.model_max_ctx = m->hparams.dec_max_position; - m->limits.gen_reserve = 512; // run()'s max-new-tokens cap + m->limits.has_context_cap = true; + m->limits.audio_from_caps = true; + m->limits.model_max_ctx = m->hparams.dec_max_position; + m->limits.gen_reserve = k_gen_reserve; + // Encoder rate, for the decode budget only: audio_from_caps pins + // effective_max_audio_ms to the encoder bound, so this moves no limit. + if (m->hparams.enc_subsampling_factor > 0 && m->hparams.fe_hop_length > 0 && m->hparams.fe_sample_rate > 0) { + m->limits.ms_per_audio_token = static_cast(m->hparams.enc_subsampling_factor) * + m->hparams.fe_hop_length * 1000.0 / m->hparams.fe_sample_rate; + } // Whisper-style decoder self-KV: dec_d_model per layer, K and V, no GQA. m->limits.kv_elems_per_ctx_token = (int64_t) m->hparams.dec_d_model * m->hparams.dec_n_layers * 2; } @@ -1089,8 +1099,10 @@ transcribe_status run(transcribe_session * session, cc->clear_result(); - const int eos_id = cm->hparams.eos_token_id; - const int max_tokens = std::min(512, cc->kv_cache.n_ctx - prompt_len); + const int eos_id = cm->hparams.eos_token_id; + const int max_tokens = + transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc, cm->limits.ms_per_audio_token), + k_gen_reserve, prompt_len, cc->kv_cache.n_ctx); int next_token = 0; if (prompt_skip_softmax && db.argmax_out != nullptr) { @@ -1608,15 +1620,17 @@ transcribe_status run_batch(transcribe_session * session, } // Batched KV cache. - const int max_new = 512; - int max_n_kv = 1024; - while (max_n_kv < prompt_len + max_new) { - max_n_kv *= 2; - } // Decoder self-KV ceiling: dec_max_position, optionally lowered (never // raised) by the caller's n_ctx knob. Default knob (0) leaves it at // dec_max_position, so in-spec batched decode is unchanged. const int n_ctx_cap = canary_context_ceiling(cc->n_ctx, hp); + const int max_new = + transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc_max, cm->limits.ms_per_audio_token), + k_gen_reserve, prompt_len, n_ctx_cap); + int max_n_kv = 1024; + while (max_n_kv < prompt_len + max_new) { + max_n_kv *= 2; + } if (max_n_kv > n_ctx_cap) { max_n_kv = n_ctx_cap; } diff --git a/src/arch/canary_qwen/model.cpp b/src/arch/canary_qwen/model.cpp index 16f53fe4..386fd9da 100644 --- a/src/arch/canary_qwen/model.cpp +++ b/src/arch/canary_qwen/model.cpp @@ -28,6 +28,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-env.h" #include "transcribe-flash-policy.h" #include "transcribe-load-common.h" @@ -116,9 +117,8 @@ constexpr float kBnEps = 1e-5f; // TRANSCRIBE_ERR_INPUT_TOO_LONG; a transcript that fills the generation budget // before end-of-stream is flagged via transcribe_was_truncated(). -// Per-run generation budget. Keep in sync with the single-utterance and -// batched step loops below. -constexpr int k_max_new = 256; +// Generation reserve: what the input gate keeps free, and the decode-budget floor. +constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum // (decoder.max_position_embeddings, e.g. 40960), optionally lowered — never @@ -148,7 +148,7 @@ int64_t canary_qwen_max_audio_ms(const CanaryQwenHParams & hp) { // counts; ~14 for canary_qwen). Advisory headroom, generous enough to // cover small template drift. constexpr int k_prompt_overhead = 32; - const int max_audio_tokens = hp.dec_max_position - k_prompt_overhead - k_max_new; + const int max_audio_tokens = hp.dec_max_position - k_prompt_overhead - k_gen_reserve; if (max_audio_tokens <= 0) { return 0; } @@ -554,7 +554,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->limits.has_context_cap = true; m->limits.model_max_ctx = m->hparams.dec_max_position; m->limits.prompt_overhead = 32; - m->limits.gen_reserve = k_max_new; + m->limits.gen_reserve = k_gen_reserve; // audio_tokens ≈ mel_frames / subsampling_factor ; // mel_frames = ms*sr/(hop*1000) m->limits.ms_per_audio_token = static_cast(m->hparams.enc_subsampling_factor) * @@ -910,22 +910,25 @@ transcribe_status run(transcribe_session * context, // Input-length gate: audio + prompt + generation must fit the decoder // context window. Reject an over-length clip here, before prefill/decode. const int ceiling = canary_qwen_context_ceiling(cc->n_ctx, hp); - if (T_prompt + k_max_new > ceiling) { + if (T_prompt + k_gen_reserve > ceiling) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "canary_qwen run: input too long — %d audio + %d prompt tokens " "leave no room for output within the %d-token context (need %d). " "Shorten the audio (see transcribe_capabilities.max_audio_ms) or " "split it into segments.", - T_enc, prefix_len + suffix_len, ceiling, T_prompt + k_max_new); + T_enc, prefix_len + suffix_len, ceiling, T_prompt + k_gen_reserve); return TRANSCRIBE_ERR_INPUT_TOO_LONG; } + const int max_new = transcribe::pick_decode_budget( + transcribe::predict_transcript_tokens(T_enc, cm->limits.ms_per_audio_token), k_gen_reserve, T_prompt, ceiling); + // KV cache init (grow-to-fit, clamped to the context ceiling). Size to - // hold prompt + generation budget, rounded up to a power of two (the step + // hold prompt + decode budget, rounded up to a power of two (the step // graph's flash-attn path wants pow2 attention width). A pre-allocated // smaller cache is freed and re-allocated. int want_n_ctx = 1024; - while (want_n_ctx < T_prompt + k_max_new) { + while (want_n_ctx < T_prompt + max_new) { want_n_ctx *= 2; } if (want_n_ctx > ceiling) { @@ -1073,7 +1076,6 @@ transcribe_status run(transcribe_session * context, // Step loop. const int32_t eos_id = hp.eos_token_id; - const int max_new = k_max_new; int cur_past = T_prompt; int max_n_kv = 1024; @@ -1448,13 +1450,13 @@ transcribe_status run_batch(transcribe_session * session, // Input-length gate (same as single-shot run()); reject this utterance, // the rest of the batch still runs. - if (T_prompt[b] + k_max_new > ceiling) { + if (T_prompt[b] + k_gen_reserve > ceiling) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "canary_qwen run_batch: utterance %d input too long — %d audio " "+ %d prompt tokens leave no room for output within the " "%d-token context (need %d). Shorten the audio (see " "transcribe_capabilities.max_audio_ms) or split it.", - b, T_enc[b], T_prompt[b] - T_enc[b], ceiling, T_prompt[b] + k_max_new); + b, T_enc[b], T_prompt[b] - T_enc[b], ceiling, T_prompt[b] + k_gen_reserve); fail_status[b] = TRANSCRIBE_ERR_INPUT_TOO_LONG; continue; } @@ -1476,9 +1478,11 @@ transcribe_status run_batch(transcribe_session * session, } return TRANSCRIBE_OK; } - T_enc_max = std::max(1, T_enc_max); - const int max_new = k_max_new; - int max_n_kv = 1024; + T_enc_max = std::max(1, T_enc_max); + const int max_new = + transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc_max, cm->limits.ms_per_audio_token), + k_gen_reserve, max_T_prompt, ceiling); + int max_n_kv = 1024; while (max_n_kv < max_T_prompt + max_new) { max_n_kv *= 2; } diff --git a/src/arch/cohere/model.cpp b/src/arch/cohere/model.cpp index a305756f..bdf8050e 100644 --- a/src/arch/cohere/model.cpp +++ b/src/arch/cohere/model.cpp @@ -15,6 +15,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-env.h" #include "transcribe-flash-policy.h" #include "transcribe-load-common.h" @@ -416,6 +417,9 @@ transcribe_status promote_conv_pw_to_f32_on_cpu(CohereModel & m) { constexpr const char k_default_variant[] = "cohere-asr"; +// Generation reserve: floor under the per-run decode budget. +constexpr int k_gen_reserve = 512; + // Forward declarations for the Arch trait below. extern transcribe_status load(Loader &, const transcribe_model_load_params *, transcribe_model **); extern transcribe_status init_context(transcribe_model *, const transcribe_session_params *, transcribe_session **); @@ -477,7 +481,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par // Fixed control-token preamble (see run()'s prompt_pieces). Audio is // in cross-KV, so there is no audio-token overhead here. m->limits.prompt_overhead = 10; - m->limits.gen_reserve = 512; // max-new-tokens cap in run() + m->limits.gen_reserve = k_gen_reserve; // ms-per-audio-token = subsampling_factor * hop_length * 1000 / sr. m->limits.ms_per_audio_token = static_cast(m->hparams.enc_subsampling_factor) * m->hparams.fe_hop_length * 1000.0 / m->hparams.fe_sample_rate; @@ -1038,8 +1042,10 @@ transcribe_status run(transcribe_session * session, // Load-time validation guarantees eos_token_id >= 0; no // fallback is needed here. See the tokenizer.eos_id() check // in cohere::load() at the top of this file. - const int eos_id = cm->hparams.eos_token_id; - const int max_tokens = std::min(512, cc->kv_cache.n_ctx - prompt_len); + const int eos_id = cm->hparams.eos_token_id; + const int max_tokens = + transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc, cm->limits.ms_per_audio_token), + k_gen_reserve, prompt_len, cc->kv_cache.n_ctx); // Pick the first generated token. Fast path reads a single // int32 argmax that the GPU computed; debug path reads the @@ -1563,14 +1569,16 @@ transcribe_status run_batch(transcribe_session * session, } // ----- Allocate batched KV cache ----- - const int max_new = std::min(512, /*budget*/ 4096); - int max_n_kv = 1024; - while (max_n_kv < prompt_len + max_new) { - max_n_kv *= 2; - } // Honor the session context cap (same ceiling the single-shot path uses), // not the raw model max — so a lowered n_ctx bounds batch decoder KV too. const int n_ctx_cap = cohere_dec_ctx_ceiling(cc->n_ctx, hp); + const int max_new = + transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_enc_max, cm->limits.ms_per_audio_token), + k_gen_reserve, prompt_len, n_ctx_cap); + int max_n_kv = 1024; + while (max_n_kv < prompt_len + max_new) { + max_n_kv *= 2; + } if (max_n_kv > n_ctx_cap) { max_n_kv = n_ctx_cap; } diff --git a/src/arch/funasr_nano/model.cpp b/src/arch/funasr_nano/model.cpp index 436ec1ce..751d6ede 100644 --- a/src/arch/funasr_nano/model.cpp +++ b/src/arch/funasr_nano/model.cpp @@ -13,6 +13,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-flash-policy.h" #include "transcribe-kaldi-fbank.h" #include "transcribe-load-common.h" @@ -80,8 +81,8 @@ constexpr const char k_default_variant[] = "fun-asr-nano-2512"; // transcribe_was_truncated(). // --------------------------------------------------------------------------- -// Per-run generation budget. -constexpr int k_max_new = 256; +// Generation reserve: what the input gate keeps free, and the decode-budget floor. +constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum, // optionally lowered — never raised — by the caller's session n_ctx knob. @@ -103,7 +104,7 @@ int64_t funasr_nano_max_audio_ms(const FunAsrNanoHParams & hp) { return 0; } constexpr int k_prompt_overhead = 48; // chat affixes; advisory - const int max_audio_tokens = hp.dec_max_position_embeddings - k_prompt_overhead - k_max_new; + const int max_audio_tokens = hp.dec_max_position_embeddings - k_prompt_overhead - k_gen_reserve; if (max_audio_tokens <= 0) { return 0; } @@ -309,7 +310,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->limits.has_context_cap = true; m->limits.model_max_ctx = m->hparams.dec_max_position_embeddings; m->limits.prompt_overhead = 48; - m->limits.gen_reserve = k_max_new; + m->limits.gen_reserve = k_gen_reserve; m->limits.ms_per_audio_token = static_cast(folds) * m->hparams.fe_lfr_n * m->hparams.fe_hop_length * 1000.0 / m->hparams.fe_sample_rate; m->limits.kv_elems_per_ctx_token = @@ -666,23 +667,27 @@ transcribe_status run(transcribe_session * session, // fixed by the input length, so reject an over-length clip here, before // KV alloc / prefill / decode, instead of walling at a fixed size. const int ceiling = funasr_nano_context_ceiling(cc->n_ctx, hp); - if (T_prompt + k_max_new > ceiling) { + if (T_prompt + k_gen_reserve > ceiling) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "funasr_nano run: input too long — %d audio + %d prompt tokens " "leave no room for output within the %d-token context (need %d). " "Shorten the audio (see transcribe_capabilities.max_audio_ms) or " "split it into segments.", - T_audio, prefix_len + suffix_len, ceiling, T_prompt + k_max_new); + T_audio, prefix_len + suffix_len, ceiling, T_prompt + k_gen_reserve); return TRANSCRIBE_ERR_INPUT_TOO_LONG; } + const int max_new = + transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(T_audio, cm->limits.ms_per_audio_token), + k_gen_reserve, T_prompt, ceiling); + // ---- KV cache init (grow-to-fit, clamped to the context ceiling) ---- - // Size to hold the prompt plus the generation budget, rounded up to a - // power of two (the step graph's attention width wants pow2 for the fast - // flash-attn path). The cache grows across runs as audio length demands; - // a pre-allocated smaller cache is freed and re-allocated. + // Size to hold the prompt plus the decode budget, rounded up to a power of + // two (the step graph's attention width wants pow2 for the fast flash-attn + // path). The cache grows across runs as audio length demands; a + // pre-allocated smaller cache is freed and re-allocated. int want_n_ctx = 1024; - while (want_n_ctx < T_prompt + k_max_new) { + while (want_n_ctx < T_prompt + max_new) { want_n_ctx *= 2; } if (want_n_ctx > ceiling) { @@ -817,7 +822,6 @@ transcribe_status run(transcribe_session * session, // ---- Step loop ---- const int32_t eos_id = hp.eos_token_id; - const int max_new = k_max_new; int cur_past = T_prompt; // Static step-graph shape: T_prompt prefilled + up to max_new generated. @@ -1142,25 +1146,26 @@ transcribe_status run_batch(transcribe_session * session, // Input-length gate (see docs/input-limits.md). Audio tokens + prompt + // generation must fit the decoder context window; reject an over-length // utterance here instead of walling at a fixed KV size. Mirrors the - // single-shot run() gate (T_prompt + k_max_new > ceiling). - if (T_prompt[b] + k_max_new > ceiling) { + // single-shot run() gate (T_prompt + k_gen_reserve > ceiling). + if (T_prompt[b] + k_gen_reserve > ceiling) { const int suffix = T_prompt[b] - fbank_beg - T_audio[b]; transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "funasr_nano run_batch: utterance %d input too long — %d audio + " "%d prompt tokens leave no room for output within the %d-token " "context (need %d). Shorten the audio (see " "transcribe_capabilities.max_audio_ms) or split it.", - b, T_audio[b], fbank_beg + suffix, ceiling, T_prompt[b] + k_max_new); + b, T_audio[b], fbank_beg + suffix, ceiling, T_prompt[b] + k_gen_reserve); fail_status[b] = TRANSCRIBE_ERR_INPUT_TOO_LONG; continue; } valid[b] = 1; } - int max_T_prompt = 0; + int max_T_prompt = 0, max_T_audio = 0; for (int b = 0; b < n; ++b) { if (valid[b]) { max_T_prompt = std::max(max_T_prompt, T_prompt[b]); + max_T_audio = std::max(max_T_audio, T_audio[b]); } } if (max_T_prompt == 0) { @@ -1171,8 +1176,10 @@ transcribe_status run_batch(transcribe_session * session, } return TRANSCRIBE_OK; } - const int max_new = 256; - int max_n_kv = 1024; + const int max_new = transcribe::pick_decode_budget( + transcribe::predict_transcript_tokens(max_T_audio, cm->limits.ms_per_audio_token), k_gen_reserve, max_T_prompt, + ceiling); + int max_n_kv = 1024; while (max_n_kv < max_T_prompt + max_new) { max_n_kv *= 2; } diff --git a/src/arch/granite/model.cpp b/src/arch/granite/model.cpp index 95d182e6..959c314a 100644 --- a/src/arch/granite/model.cpp +++ b/src/arch/granite/model.cpp @@ -11,6 +11,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-env.h" #include "transcribe-flash-policy.h" #include "transcribe-load-common.h" @@ -80,9 +81,8 @@ constexpr float kBnEps = 1e-5f; // Over-length input is rejected up front with TRANSCRIBE_ERR_INPUT_TOO_LONG // rather than silently aliasing RoPE past the trained range. -// Generation budget reserved per run. Also the KV grow-to-fit step budget, -// so an accepted clip always has room for up to this many output tokens. -constexpr int k_gen_budget = 256; +// Generation reserve: what the input gate keeps free, and the decode-budget floor. +constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum, // optionally lowered — never raised — by the caller's session n_ctx knob. @@ -107,7 +107,7 @@ int granite_num_queries(const GraniteHParams & hp) { // audio tokens, a representative prompt, and the generation reserve still fit // the context ceiling. This is the input bound the gate enforces; transcripts // of long-but-fitting audio may still truncate (transcribe_was_truncated) -// because the per-run output is bounded by k_gen_budget. Returns 0 ("unknown +// because the per-run output is bounded by the decode budget. Returns 0 ("unknown // / unbounded") if the rate constants are missing, so a misconfigured model // is never advertised with a wrong finite number. int64_t granite_max_audio_ms(const GraniteHParams & hp) { @@ -118,7 +118,7 @@ int64_t granite_max_audio_ms(const GraniteHParams & hp) { } // Representative non-audio prompt overhead (chat affixes); advisory. constexpr int k_prompt_overhead = 64; - const int max_audio_tokens = hp.dec_max_position_embeddings - k_prompt_overhead - k_gen_budget; + const int max_audio_tokens = hp.dec_max_position_embeddings - k_prompt_overhead - k_gen_reserve; if (max_audio_tokens <= 0) { return 0; } @@ -305,7 +305,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->limits.has_context_cap = true; m->limits.model_max_ctx = m->hparams.dec_max_position_embeddings; m->limits.prompt_overhead = 64; // match granite_max_audio_ms's k_prompt_overhead - m->limits.gen_reserve = k_gen_budget; + m->limits.gen_reserve = k_gen_reserve; // ms per audio token: granite emits num_queries tokens per // window_size encoder frames; t_enc = mel_frames/2; // mel_frames = ms*sr/(hop*1000). Inverting granite_max_audio_ms's @@ -1039,16 +1039,20 @@ transcribe_status run(transcribe_session * ctx_base, // aliasing RoPE past the trained range. Reserving the full generation // budget means an accepted clip always has room for a real transcript. const int ceiling = granite_context_ceiling(cc->n_ctx, cm->hparams); - if (T_prompt + k_gen_budget > ceiling) { + if (T_prompt + k_gen_reserve > ceiling) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "granite run: input too long — %d audio + %d prompt tokens leave " "no room for output within the %d-token context (need %d). " "Shorten the audio (see transcribe_capabilities.max_audio_ms) or " "split it into segments.", - n_audio_tokens, prefix_len + suffix_len, ceiling, T_prompt + k_gen_budget); + n_audio_tokens, prefix_len + suffix_len, ceiling, T_prompt + k_gen_reserve); return TRANSCRIBE_ERR_INPUT_TOO_LONG; } + const int gen_budget = transcribe::pick_decode_budget( + transcribe::predict_transcript_tokens(n_audio_tokens, cm->limits.ms_per_audio_token), k_gen_reserve, T_prompt, + ceiling); + // Size the KV cache dynamically: T_prompt + room for the longest // generation we'll emit, clamped to the context ceiling. Matches the // HF reference's DynamicCache semantics (grows as needed) without @@ -1057,7 +1061,7 @@ transcribe_status run(transcribe_session * ctx_base, // so back-to-back runs of similar audio lengths don't keep // re-allocating. constexpr int kKvBucket = 256; - const int needed_raw = std::min(T_prompt + k_gen_budget, ceiling); + const int needed_raw = std::min(T_prompt + gen_budget, ceiling); const int needed_n_ctx = ((needed_raw + kKvBucket - 1) / kKvBucket) * kKvBucket; if (cc->kv.self_k != nullptr && cc->kv.n_ctx < needed_n_ctx) { @@ -1196,10 +1200,9 @@ transcribe_status run(transcribe_session * ctx_base, // n_ctx of the KV cache bounds the max generation length we can // attend over. const int max_n_kv = cc->kv.n_ctx; - // Bound generation by the step budget, the allocated cache, AND the - // context ceiling (the gate guarantees ceiling - T_prompt >= k_gen_budget, - // so for in-spec input this stays k_gen_budget and decode is unchanged). - const int max_steps = std::min({ k_gen_budget, max_n_kv - T_prompt, ceiling - T_prompt }); + // gen_budget is already clamped to ceiling - T_prompt; the other two terms + // guard the cache the bucket rounding actually gave us. + const int max_steps = std::min({ gen_budget, max_n_kv - T_prompt, ceiling - T_prompt }); ggml_context * step_ctx = nullptr; { @@ -1572,13 +1575,13 @@ transcribe_status run_batch(transcribe_session * session, T_prompt[b] = static_cast(prompt_ids[b].size()); // Input-length gate, mirroring the single-shot run() gate. - if (T_prompt[b] + k_gen_budget > ceiling) { + if (T_prompt[b] + k_gen_reserve > ceiling) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "granite run_batch: utterance %d input too long — %d audio + %d " "prompt tokens leave no room for output within the %d-token " "context (need %d). Shorten the audio (see " "transcribe_capabilities.max_audio_ms) or split it.", - b, n_audio[b], T_prompt[b] - n_audio[b], ceiling, T_prompt[b] + k_gen_budget); + b, n_audio[b], T_prompt[b] - n_audio[b], ceiling, T_prompt[b] + k_gen_reserve); fail_status[b] = TRANSCRIBE_ERR_INPUT_TOO_LONG; continue; } @@ -1600,9 +1603,11 @@ transcribe_status run_batch(transcribe_session * session, } return TRANSCRIBE_OK; } - n_audio_max = std::max(1, n_audio_max); - const int max_new = 256; - int max_n_kv = 1024; + n_audio_max = std::max(1, n_audio_max); + const int max_new = transcribe::pick_decode_budget( + transcribe::predict_transcript_tokens(n_audio_max, cm->limits.ms_per_audio_token), k_gen_reserve, max_T_prompt, + ceiling); + int max_n_kv = 1024; while (max_n_kv < max_T_prompt + max_new) { max_n_kv *= 2; } diff --git a/src/arch/moss/model.cpp b/src/arch/moss/model.cpp index 3cacf0ff..5133a957 100644 --- a/src/arch/moss/model.cpp +++ b/src/arch/moss/model.cpp @@ -16,6 +16,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-env.h" #include "transcribe-flash-policy.h" #include "transcribe-load-common.h" @@ -759,10 +760,8 @@ transcribe_status run(transcribe_session * session, return TRANSCRIBE_ERR_INPUT_TOO_LONG; } - // Generation budget scales with audio length: the emergent transcript - // (text + [start]/[Sxx]/[end] markers) tracks the audio-token count, which - // for long-form far exceeds the k_max_new floor. Clamp to the context. - const int gen_budget = std::min(ceiling - T_prompt, std::max(k_max_new, 2 * T_enc + 128)); + // Above the plain audio-token count: the transcript carries [start]/[Sxx]/[end] markers. + const int gen_budget = transcribe::pick_decode_budget(2 * T_enc + 128, k_max_new, T_prompt, ceiling); // KV cache (grow-to-fit, clamped to ceiling). Short inputs retain the old // 1K/2K/4K buckets; longer ones grow in 4K steps so crossing 32K does not @@ -1170,9 +1169,7 @@ transcribe_status run_batch(transcribe_session * session, return TRANSCRIBE_OK; } - // Batch-wide generation budget: covers the longest utterance's transcript - // (scales with its audio tokens), clamped to the context. - const int batch_budget = std::min(ceiling - max_T_prompt, std::max(k_max_new, 2 * max_T_enc + 128)); + const int batch_budget = transcribe::pick_decode_budget(2 * max_T_enc + 128, k_max_new, max_T_prompt, ceiling); int max_n_kv = 1024; while (max_n_kv < max_T_prompt + batch_budget) { diff --git a/src/arch/qwen3_asr/model.cpp b/src/arch/qwen3_asr/model.cpp index 423f3b38..250bef3e 100644 --- a/src/arch/qwen3_asr/model.cpp +++ b/src/arch/qwen3_asr/model.cpp @@ -10,6 +10,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-env.h" #include "transcribe-flash-policy.h" #include "transcribe-load-common.h" @@ -74,8 +75,8 @@ constexpr const char k_default_variant[] = "qwen3-asr"; // transcript that fills the generation budget before end-of-stream is flagged // via transcribe_was_truncated(). -// Per-run generation budget (matches the reference dumper default). -constexpr int k_max_new = 256; +// Generation reserve: what the input gate keeps free, and the decode-budget floor. +constexpr int k_gen_reserve = 256; // Effective decoder context ceiling, in tokens: the model's trained maximum, // optionally lowered — never raised — by the caller's session n_ctx knob. @@ -99,7 +100,7 @@ int64_t qwen3_max_audio_ms(const QwenAsrHParams & hp) { return 0; } constexpr int k_prompt_overhead = 48; // chat affixes; advisory - const int max_audio_tokens = hp.dec_max_position_embeddings - k_prompt_overhead - k_max_new; + const int max_audio_tokens = hp.dec_max_position_embeddings - k_prompt_overhead - k_gen_reserve; if (max_audio_tokens <= 0) { return 0; } @@ -160,7 +161,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->limits.has_context_cap = true; m->limits.model_max_ctx = m->hparams.dec_max_position_embeddings; m->limits.prompt_overhead = 48; - m->limits.gen_reserve = k_max_new; + m->limits.gen_reserve = k_gen_reserve; // audio_tokens ≈ mel_frames / 8 ; mel_frames = ms*sr/(hop*1000) m->limits.ms_per_audio_token = 8.0 * m->hparams.fe_hop_length * 1000.0 / m->hparams.fe_sample_rate; m->limits.kv_elems_per_ctx_token = @@ -726,22 +727,25 @@ transcribe_status run(transcribe_session * session, // Input-length gate: audio + prompt + generation must fit the decoder // context window. Reject an over-length clip here, before prefill/decode. const int ceiling = qwen3_context_ceiling(cc->n_ctx, cm->hparams); - if (T_prompt + k_max_new > ceiling) { + if (T_prompt + k_gen_reserve > ceiling) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "qwen3_asr run: input too long — %d audio + %d prompt tokens " "leave no room for output within the %d-token context (need %d). " "Shorten the audio (see transcribe_capabilities.max_audio_ms) or " "split it into segments.", - T_enc, prefix_len + suffix_len, ceiling, T_prompt + k_max_new); + T_enc, prefix_len + suffix_len, ceiling, T_prompt + k_gen_reserve); return TRANSCRIBE_ERR_INPUT_TOO_LONG; } + const int max_new = transcribe::pick_decode_budget( + transcribe::predict_transcript_tokens(T_enc, cm->limits.ms_per_audio_token), k_gen_reserve, T_prompt, ceiling); + // KV cache init (grow-to-fit, clamped to the context ceiling). Size to - // hold prompt + generation budget, rounded up to a power of two (the step + // hold prompt + decode budget, rounded up to a power of two (the step // graph's flash-attn path wants pow2 attention width). A pre-allocated // smaller cache is freed and re-allocated. int want_n_ctx = 1024; - while (want_n_ctx < T_prompt + k_max_new) { + while (want_n_ctx < T_prompt + max_new) { want_n_ctx *= 2; } if (want_n_ctx > ceiling) { @@ -871,7 +875,6 @@ transcribe_status run(transcribe_session * session, // Step loop. const int32_t eos_id = cm->hparams.eos_token_id; - const int32_t max_new = k_max_new; int cur_past = T_prompt; // Build the step graph ONCE and reuse every step, sized for the actual @@ -1544,8 +1547,8 @@ transcribe_status run_batch(transcribe_session * session, // Prompt length bound → max_n_kv and batched-cache n_ctx. Build and keep // each utterance's prompt token ids for the batched prefill. - const int max_new = 256; int max_T_prompt = 0; + int max_T_enc = 0; int prefix_len = 0; // Per-utterance terminal status for rejected rows. Defaults to INVALID_ARG; // over-length rows below are upgraded to INPUT_TOO_LONG. @@ -1561,7 +1564,7 @@ transcribe_status run_batch(transcribe_session * session, T_prompt[b] = static_cast(prompt_ids[b].size()); prefix_len = ap.empty() ? 0 : static_cast(ap.front()); // Same gate as single-shot run(); the rest of the batch still runs. - if (T_prompt[b] + max_new > ceiling) { + if (T_prompt[b] + k_gen_reserve > ceiling) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "qwen3_asr run_batch: utterance %d input too long — %d audio + " "%d prompt tokens exceed the %d-token context. See " @@ -1572,6 +1575,7 @@ transcribe_status run_batch(transcribe_session * session, continue; } max_T_prompt = std::max(max_T_prompt, T_prompt[b]); + max_T_enc = std::max(max_T_enc, T_enc[b]); } if (max_T_prompt == 0) { // No usable utterance — emit per-row errors and return. @@ -1582,6 +1586,9 @@ transcribe_status run_batch(transcribe_session * session, } return TRANSCRIBE_OK; } + const int max_new = + transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(max_T_enc, cm->limits.ms_per_audio_token), + k_gen_reserve, max_T_prompt, ceiling); int max_n_kv = 1024; while (max_n_kv < max_T_prompt + max_new) { max_n_kv *= 2; diff --git a/src/arch/voxtral/model.cpp b/src/arch/voxtral/model.cpp index e490742b..aba6fa73 100644 --- a/src/arch/voxtral/model.cpp +++ b/src/arch/voxtral/model.cpp @@ -19,6 +19,7 @@ #include "transcribe-arch.h" #include "transcribe-batch-util.h" #include "transcribe-debug.h" +#include "transcribe-decode-budget.h" #include "transcribe-flash-policy.h" #include "transcribe-load-common.h" #include "transcribe-loader.h" @@ -76,23 +77,9 @@ namespace { constexpr const char k_default_variant[] = "voxtral-mini-3b-2507"; // Floor on the decoder text budget for short clips (also Whisper's per-chunk -// cap). Long audio scales the budget up with the audio length — see run(). +// cap); longer audio scales the budget up. See transcribe-decode-budget.h. constexpr int k_decode_budget_min = 448; -// Decode budget (max new text tokens) for an utterance with `n_audio` audio -// embedding tokens. Speech yields fewer text tokens than audio frames, so the -// audio token count is a safe upper bound; clamp to the context remaining under -// the trained max so prompt+decode fits. Greedy decode stops at EOS well before -// this, so a generous ceiling costs only its KV allocation. -int pick_decode_budget(int n_audio, int t_prompt, int model_max) { - int budget = std::max(k_decode_budget_min, n_audio); - const int room = model_max - t_prompt; - if (budget > room) { - budget = room; - } - return budget; -} - // Chunked prefill — see decoder.h. Walks the prompt in blocks against the // growing KV cache and returns the final position's logits. The prompt is // laid out [prefix | audio | suffix], so each chunk holds at most one run of @@ -774,7 +761,7 @@ transcribe_status run(transcribe_session * session, n_audio_total, T_prompt - n_audio_total, model_max, T_prompt + k_gen_reserve); return TRANSCRIBE_ERR_INPUT_TOO_LONG; } - const int max_new = pick_decode_budget(n_audio_total, T_prompt, model_max); + const int max_new = transcribe::pick_decode_budget(n_audio_total, k_decode_budget_min, T_prompt, model_max); const int want_ctx = causal_lm::pick_kv_cache_context(T_prompt + max_new, model_max); if (cc->kv_cache.n_ctx < want_ctx) { const ggml_type kv_type = (cc->kv_type == TRANSCRIBE_KV_TYPE_F32) ? GGML_TYPE_F32 : GGML_TYPE_F16; @@ -1339,7 +1326,7 @@ transcribe_status run_batch(transcribe_session * session, // Size the batched KV cache to the longest prompt plus the decode budget, // clamped to the context ceiling. kv_init_batched grows the cache on demand. const int model_max = ctx_ceiling; - const int max_new = pick_decode_budget(T_audio_max, max_T_prompt, model_max); + const int max_new = transcribe::pick_decode_budget(T_audio_max, k_decode_budget_min, max_T_prompt, model_max); int max_n_kv = 1024; while (max_n_kv < max_T_prompt + max_new) { max_n_kv *= 2; diff --git a/src/transcribe-decode-budget.h b/src/transcribe-decode-budget.h new file mode 100644 index 00000000..87ee28cf --- /dev/null +++ b/src/transcribe-decode-budget.h @@ -0,0 +1,39 @@ +// Shared helpers for sizing autoregressive decode budgets. + +#pragma once + +#include + +namespace transcribe { + +// Conservative multilingual transcript estimate, in tokens per second. +constexpr int k_transcript_tokens_per_sec = 12; + +// Fall back to the encoder-token count when its duration is unknown. +inline int predict_transcript_tokens(int audio_tokens, double ms_per_audio_token) { + if (audio_tokens <= 0) { + return 0; + } + if (!(ms_per_audio_token > 0.0)) { + return audio_tokens; + } + const double seconds = static_cast(audio_tokens) * ms_per_audio_token / 1000.0; + const double predicted = seconds * k_transcript_tokens_per_sec; + if (predicted <= 0.0) { + return 0; + } + constexpr double k_int_max = 2147483647.0; + return predicted >= k_int_max ? 2147483647 : static_cast(predicted); +} + +// Apply the family floor without exceeding the available context. +inline int pick_decode_budget(int predicted, int floor_tokens, int t_prompt, int ceiling) { + int budget = std::max(floor_tokens, predicted); + const int room = ceiling - t_prompt; + if (budget > room) { + budget = room; + } + return budget > 0 ? budget : 0; +} + +} // namespace transcribe diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 85b47282..1e2318a0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -201,6 +201,19 @@ transcribe_apply_warnings(transcribe_prefill_chunk_mask_unit) add_test(NAME transcribe_prefill_chunk_mask_unit COMMAND transcribe_prefill_chunk_mask_unit) +# ----------------------------------------------------------------------------- +# Per-run decode budget rule (pure host, no model) + +add_executable(transcribe_decode_budget_unit + decode_budget_unit.cpp) + +target_include_directories(transcribe_decode_budget_unit PRIVATE + ${CMAKE_SOURCE_DIR}/src) + +transcribe_apply_warnings(transcribe_decode_budget_unit) + +add_test(NAME transcribe_decode_budget_unit COMMAND transcribe_decode_budget_unit) + # ----------------------------------------------------------------------------- # MOSS diarized-transcript parser unit test (pure host, no model) # ----------------------------------------------------------------------------- diff --git a/tests/decode_budget_unit.cpp b/tests/decode_budget_unit.cpp new file mode 100644 index 00000000..45e69272 --- /dev/null +++ b/tests/decode_budget_unit.cpp @@ -0,0 +1,76 @@ +// Per-run decode budget rule (pure host, no model). Getting it wrong is silent +// both ways: too low and long clips truncate with context to spare, too high +// and the KV allocation balloons every run. See docs/input-limits.md. + +#include "transcribe-decode-budget.h" + +#include + +namespace { + +int g_failures = 0; + +void check_predict(const char * what, int audio_tokens, double ms_per_audio_token, int expected) { + const int got = transcribe::predict_transcript_tokens(audio_tokens, ms_per_audio_token); + if (got != expected) { + std::fprintf(stderr, "FAIL %s: predict_transcript_tokens(%d, %.3f) = %d, expected %d\n", what, audio_tokens, + ms_per_audio_token, got, expected); + ++g_failures; + } +} + +void check_budget(const char * what, int predicted, int floor_tokens, int t_prompt, int ceiling, int expected) { + const int got = transcribe::pick_decode_budget(predicted, floor_tokens, t_prompt, ceiling); + if (got != expected) { + std::fprintf(stderr, "FAIL %s: pick_decode_budget(%d, %d, %d, %d) = %d, expected %d\n", what, predicted, + floor_tokens, t_prompt, ceiling, got, expected); + ++g_failures; + } +} + +} // namespace + +int main(void) { + // Duration-based: two encoders that heard the same 197 s must agree. + check_predict("80 ms encoder, 197 s", /*audio_tokens=*/2463, /*ms_per_audio_token=*/80.0, + /*expected=*/2364); + check_predict("480 ms LFR encoder, same 197 s", /*audio_tokens=*/410, /*ms_per_audio_token=*/480.0, + /*expected=*/2361); + + // An unpublished rate falls back to the audio-token count. + check_predict("unknown rate falls back", 2463, 0.0, 2463); + check_predict("negative rate falls back", 2463, -1.0, 2463); + check_predict("no audio", 0, 80.0, 0); + + // The floor holds for short audio, so those decodes are unchanged. + check_budget("short clip keeps the floor", /*predicted=*/10, /*floor=*/256, /*t_prompt=*/64, + /*ceiling=*/65536, /*expected=*/256); + check_budget("floor applies at zero audio", 0, 256, 64, 65536, 256); + check_budget("canary floor", 300, 512, 6, 1024, 512); + + // Past the floor the budget scales with the audio. This is the fix. + check_budget("5 min qwen3-asr scales", /*predicted=*/3824, /*floor=*/256, /*t_prompt=*/3872, + /*ceiling=*/65536, /*expected=*/3824); + check_budget("20 min qwen3-asr scales", 15000, 256, 15048, 65536, 15000); + + // The ceiling always wins: canary sees ~5000 encoder frames into a 1024 self-KV. + check_budget("canary clamps to dec ctx", /*predicted=*/5000, /*floor=*/512, /*t_prompt=*/6, + /*ceiling=*/1024, /*expected=*/1018); + check_budget("clamp beats the floor too", 10, 512, 900, 1024, 124); + + // n_ctx is the knob: lowering `ceiling` lowers the budget with it. + check_budget("full n_ctx leaves the audio-sized budget intact", 3824, 256, 3872, 8192, 3824); + check_budget("lowered n_ctx lowers the budget", 3824, 256, 3872, 6000, 2128); + check_budget("n_ctx below the floor still clamps", 3824, 256, 3872, 4000, 128); + + // Never a negative step count. + check_budget("prompt exactly fills ceiling", 3824, 256, 1024, 1024, 0); + check_budget("prompt overruns ceiling", 3824, 256, 2048, 1024, 0); + + if (g_failures > 0) { + std::fprintf(stderr, "decode_budget_unit: %d failures\n", g_failures); + return 1; + } + std::fprintf(stdout, "decode_budget_unit: ok\n"); + return 0; +} diff --git a/tests/qwen3_asr_batch_truncation.cpp b/tests/qwen3_asr_batch_truncation.cpp index ae3fb2e0..f4661ed0 100644 --- a/tests/qwen3_asr_batch_truncation.cpp +++ b/tests/qwen3_asr_batch_truncation.cpp @@ -1,33 +1,7 @@ -// qwen3_asr_batch_truncation.cpp - real-model gated test that the single-shot -// and batch decode paths both report mid-decode OUTPUT_TRUNCATED for a -// causal_lm (LLM-decoder) family. -// -// qwen3_asr caps generation at max_new = 256 tokens. A long speech clip passes -// the up-front input-length gate (its audio tokens fit the 65536-token decoder -// context with room to spare) but its natural transcript exceeds 256 tokens, so -// greedy decode hits the generation budget before EOS — the transcript is -// truncated. Per docs/input-limits.md that must surface as the hard -// TRANSCRIBE_ERR_OUTPUT_TRUNCATED status (partial transcript retained, -// transcribe_was_truncated() set) in BOTH paths, while a short clip that -// finishes under the budget stays OK and the whole-batch call still returns OK. -// -// This is the causal_lm counterpart to moonshine_streaming_batch_truncation -// (which exercises the encoder-decoder batch loop in transcribe-batch-util.cpp). -// It specifically guards the shared src/causal_lm batched step loop's per-row -// truncation detection: that loop marks every stopped row `finished` -// regardless of WHY it stopped, so truncation must be inferred from the last -// sampled token (!= eos), not from `!finished`. A regression there makes a -// truncated batch row silently report TRANSCRIBE_OK with an incomplete -// transcript — the exact failure this test catches. -// -// Batch makeup: -// row 0 = jfk.wav (~11 s) -> completes under the budget -> OK -// row 1 = love-loss.wav (~197 s) -> exceeds the budget -> OUTPUT_TRUNCATED -// -// Gating: -// - TRANSCRIBE_BUILD_REAL_MODEL_TESTS (CMake, default OFF) builds it. -// - At runtime, TRANSCRIBE_QWEN3_ASR_0_6B_GGUF points at the GGUF. If -// unset/missing (or a sample is missing), exits 77 ("skipped"). +// Real-model regression test for decode-budget scaling and single/batch +// OUTPUT_TRUNCATED parity. The long clip completes at the default context and +// truncates under a lowered n_ctx; the short clip completes in both cases. +// Requires TRANSCRIBE_QWEN3_ASR_0_6B_GGUF; missing inputs return 77. #include "transcribe.h" #include "wav.h" @@ -105,8 +79,26 @@ int main() { return 1; } + // The long clip must complete with the default context. + { + transcribe_session_params full_sp; + transcribe_session_params_init(&full_sp); + struct transcribe_session * full_s = nullptr; + if (transcribe_session_init(model, &full_sp, &full_s) != TRANSCRIBE_OK) { + std::fprintf(stderr, "session init failed\n"); + transcribe_model_free(model); + return 1; + } + const transcribe_status rl = transcribe_run(full_s, pcm_long.data(), (int) pcm_long.size(), nullptr); + CHECK(rl == TRANSCRIBE_OK); + CHECK(transcribe_was_truncated(full_s) == false); + transcribe_session_free(full_s); + } + + // Lower n_ctx enough to force truncation without rejecting the input. transcribe_session_params sp; transcribe_session_params_init(&sp); + sp.n_ctx = 3072; struct transcribe_session * s = nullptr; if (transcribe_session_init(model, &sp, &s) != TRANSCRIBE_OK) { std::fprintf(stderr, "session init failed\n"); @@ -114,9 +106,7 @@ int main() { return 1; } - // ---- Single-shot baseline: the long clip truncates, the short one does not. - // Both pass the input-length gate at the default (full) context; the long - // clip simply runs the decoder into the 256-token generation budget. + // Single-shot truncation and reset behavior. { const transcribe_status rl = transcribe_run(s, pcm_long.data(), (int) pcm_long.size(), nullptr); CHECK(rl == TRANSCRIBE_ERR_OUTPUT_TRUNCATED); @@ -130,9 +120,7 @@ int main() { CHECK(transcribe_was_truncated(s) == false); // reset + completed } - // ---- Batch parity: the shared causal_lm batched step loop must report the - // SAME per-utterance verdict. row 0 (short) finishes -> OK; row 1 (long) - // hits the budget -> OUTPUT_TRUNCATED; whole-batch call still returns OK. + // Batch results must match the single-shot results. { const float * pcms[2] = { pcm_short.data(), pcm_long.data() }; const int lens[2] = { (int) pcm_short.size(), (int) pcm_long.size() };