From 49ee0a6b46c5115163b2a16e4e4cf6f604fc6961 Mon Sep 17 00:00:00 2001 From: Carlos Augusto Date: Thu, 12 Nov 2026 02:09:57 -0700 Subject: [PATCH] Stop tokenizing user content as special - track trust in the jinja template VM Rendered chat templates interleave template-emitted text with user content. Special tokens arriving in user content were tokenized with the same tokenizer as template output, so a prompt quoting a marker such as <|vision_start|> could inject a real special-token id into the model stream. The renderer records, per output span, whether the template VM produced it or user content flowed through it: - The jinja VM emits trust runs: trusted spans for template literals and untrusted spans for injected values. Provenance travels inside values, so namespace and dict construction, attribute assignment, list and map literals, macro captures, iteration, and slicing preserve it, and an indexing or lookup key only selects a value; it never confers trust on it. - Text-transform filters that shrink text (trim and the strip family) slice both the checkpoint hints and the trust runs to the surviving byte range instead of discarding them, so a template marker emitted next to user content keeps its special-token id across a trim. - The whole rendered prompt encodes in one pass with the template tokenizer, preserving the pre-patch token ids for ordinary prompts, including BPE merges that cross trust boundaries. Suppression then works at the granularity of a maximal untrusted run, not an individual marker: if any token of a run has an id from the added vocabulary, the run's whole id span is replaced by a single encode of the run's full byte range with an ordinary tokenizer that has no added vocabulary. BPE merges inside a run therefore survive, so a request that quotes " <|im_end|>" keeps the merge between the leading space and the marker instead of re-encoding the marker as an isolated island. Ids and byte offsets are rebuilt run by run from left to right, and a trust boundary is the only splice point, so user-quoted markers cannot produce protocol ids while trusted spans keep their single-pass ids. - expand_prompt counts and splices image placeholder ids from prompt_token_ids instead of re-splitting the text, which keeps placeholder accounting correct across trust boundaries and lets ImagePrompt drop its write-only text field (the field had no runtime readers; its single reader sat in a test driver this patch fixes in the same motion). The fixture suite passes; ordinary prompts render byte-identical token ids to full-text encoding while prompts carrying real image placeholders keep their expansion through trim and mixed-trust paths. Signed-off-by: Carlos Augusto --- helios/src/frontend/deepseek.rs | 5 +- .../src/frontend/images/layouts/bracketed.rs | 4 +- helios/src/frontend/images/layouts/qwen.rs | 2 +- helios/src/frontend/images/mod.rs | 11 +- helios/src/frontend/images/patch_merge.rs | 32 +- helios/src/frontend/images/qwen.rs | 40 +- helios/src/frontend/jinja/mod.rs | 13 +- helios/src/frontend/jinja/tests.rs | 317 ++++++- helios/src/frontend/jinja/value.rs | 152 ++- helios/src/frontend/jinja/vm.rs | 423 +++++++-- helios/src/frontend/mod.rs | 874 ++++++++++++++++-- helios/src/frontend/tml.rs | 6 +- helios/tests/deepseek_frontend.rs | 121 +++ helios/tests/image_admission.rs | 4 +- helios/tests/tools/test_images.rs | 3 +- tests/frontend/prepare_templates.py | 6 + 16 files changed, 1809 insertions(+), 204 deletions(-) diff --git a/helios/src/frontend/deepseek.rs b/helios/src/frontend/deepseek.rs index 37831f0ea..8ac376de8 100644 --- a/helios/src/frontend/deepseek.rs +++ b/helios/src/frontend/deepseek.rs @@ -437,10 +437,7 @@ pub(super) fn render( .collect::>>()?; let mut token_ids = Vec::new(); let mut offsets = Vec::new(); - let plain = frontend - .ordinary_tokenizer - .as_ref() - .expect("DeepSeek plain tokenizer is initialized"); + let plain = frontend.ordinary(); let mut start = 0; let mut image_count = 0; for range in &rendered.special_ranges { diff --git a/helios/src/frontend/images/layouts/bracketed.rs b/helios/src/frontend/images/layouts/bracketed.rs index 4c7c61174..2c208037e 100644 --- a/helios/src/frontend/images/layouts/bracketed.rs +++ b/helios/src/frontend/images/layouts/bracketed.rs @@ -31,7 +31,7 @@ pub(crate) fn prepare_input( interpolation_weights: vec![], cu_seqlens: vec![0], }; - processor.expand_prompt(frontend, &rendered.text, &empty)?; + processor.expand_prompt(frontend, &rendered.token_ids, &empty)?; } return Ok(PreparedInput { tokens: rendered.token_ids.clone(), @@ -46,7 +46,7 @@ pub(crate) fn prepare_input( } let processor = processor.ok_or("image processing is unavailable")?; let processed = processor.describe_inputs(images)?; - let expanded = processor.expand_prompt(frontend, &rendered.text, &processed)?; + let expanded = processor.expand_prompt(frontend, &rendered.token_ids, &processed)?; let mut conditioning = Vec::new(); let mut image = 0; let mut feature = 0; diff --git a/helios/src/frontend/images/layouts/qwen.rs b/helios/src/frontend/images/layouts/qwen.rs index 1621958b2..eaf91e8ea 100644 --- a/helios/src/frontend/images/layouts/qwen.rs +++ b/helios/src/frontend/images/layouts/qwen.rs @@ -46,7 +46,7 @@ pub(crate) fn prepare_input( } else { let processor = processor.ok_or("image processing is unavailable")?; let processed = processor.describe_inputs(images)?; - let expanded = processor.expand_prompt(frontend, &rendered.text, &processed)?; + let expanded = processor.expand_prompt(frontend, &rendered.token_ids, &processed)?; let positions = (0..expanded.token_ids.len()) .map(|position| { [ diff --git a/helios/src/frontend/images/mod.rs b/helios/src/frontend/images/mod.rs index d25fb8c26..600c5998a 100644 --- a/helios/src/frontend/images/mod.rs +++ b/helios/src/frontend/images/mod.rs @@ -213,7 +213,6 @@ impl ProcessedImages { } pub struct ImagePrompt { - pub text: String, pub token_ids: Vec, pub mm_token_type_ids: Vec, /// Text, temporal, height, and width position rows, each of length token_ids.len(). @@ -718,12 +717,16 @@ impl ImageProcessor { pub fn expand_prompt( &self, frontend: &Frontend, - text: &str, + prompt_token_ids: &[u32], images: &ProcessedImages, ) -> Result { match &self.profile { - Profile::Qwen(profile) => profile.expand_prompt(frontend, text, images), - Profile::PatchMerge(profile) => profile.expand_prompt(frontend, text, images), + Profile::Qwen(profile) => { + profile.expand_prompt(frontend, prompt_token_ids, images) + } + Profile::PatchMerge(profile) => { + profile.expand_prompt(frontend, prompt_token_ids, images) + } Profile::DeepSeek4 | Profile::DeepseekRowMajor(_) => Err("DeepSeek image spans require its model contract".into()), Profile::MiMo(_) => Err("MiMo image expansion requires its model contract".into()), Profile::Moonvit => Err("MoonViT image expansion requires its model contract".into()), diff --git a/helios/src/frontend/images/patch_merge.rs b/helios/src/frontend/images/patch_merge.rs index acfe757c9..9ef10b0b0 100644 --- a/helios/src/frontend/images/patch_merge.rs +++ b/helios/src/frontend/images/patch_merge.rs @@ -59,7 +59,7 @@ impl Processor { pub(super) fn expand_prompt( &self, frontend: &Frontend, - text: &str, + prompt_token_ids: &[u32], images: &ProcessedImages, ) -> Result { let image_ids = frontend.encode(&self.image_token, false)?; @@ -71,26 +71,34 @@ impl Processor { return Err("image envelope marker must encode one token".into()); } } - if text.matches(&self.image_token).count() != images.geometry.len() { + let placeholder_count = prompt_token_ids + .iter() + .filter(|id| **id == *image_id) + .count(); + if placeholder_count != images.geometry.len() { return Err("image placeholder count does not match input images".into()); } - let mut parts = text.split(&self.image_token); - let mut expanded = parts.next().unwrap_or_default().to_owned(); - for (part, geometry) in parts.zip(&images.geometry) { - expanded.push_str(&self.start_token); - for _ in 0..geometry.tokens { - expanded.push_str(&self.image_token); + let start_id = frontend.encode(&self.start_token, false)?[0]; + let end_id = frontend.encode(&self.end_token, false)?[0]; + let mut expanded_ids = Vec::new(); + let mut image_index = 0usize; + for &id in prompt_token_ids { + if id == *image_id { + let tokens = images.geometry[image_index].tokens; + expanded_ids.push(start_id); + expanded_ids.extend(std::iter::repeat(*image_id).take(tokens)); + expanded_ids.push(end_id); + image_index += 1; + } else { + expanded_ids.push(id); } - expanded.push_str(&self.end_token); - expanded.push_str(part); } - let token_ids = frontend.encode(&expanded, false)?; + let token_ids = expanded_ids; let mm_token_type_ids = token_ids .iter() .map(|id| i32::from(*id == *image_id)) .collect(); Ok(ImagePrompt { - text: expanded, position_ids: std::array::from_fn(|_| (0..token_ids.len() as i64).collect()), token_ids, mm_token_type_ids, diff --git a/helios/src/frontend/images/qwen.rs b/helios/src/frontend/images/qwen.rs index a563edf9f..17c0f6289 100644 --- a/helios/src/frontend/images/qwen.rs +++ b/helios/src/frontend/images/qwen.rs @@ -78,7 +78,7 @@ impl QwenProcessor { pub fn expand_prompt( &self, frontend: &Frontend, - text: &str, + prompt_token_ids: &[u32], images: &ProcessedImages, ) -> Result { if frontend.encode(IMAGE_TOKEN, false)? != [self.image_token_id] { @@ -86,26 +86,39 @@ impl QwenProcessor { "image token ID does not match tokenizer", )))); } - if text.matches(IMAGE_TOKEN).count() != images.geometry.len() { + let placeholder_count = prompt_token_ids + .iter() + .filter(|id| **id == self.image_token_id) + .count(); + if placeholder_count != images.geometry.len() { return Err(Error::frontend(FrontendError::Input(Failure::new( "image placeholder count does not match prepared images", )))); } - let mut expanded = String::with_capacity(text.len()); - let mut parts = text.split(IMAGE_TOKEN); - expanded.push_str(parts.next().unwrap_or_default()); - for (part, geometry) in parts.zip(&images.geometry) { - for _ in 0..geometry.tokens { - expanded.push_str(IMAGE_TOKEN); - } - expanded.push_str(part); - } - let token_ids = frontend.encode(&expanded, false)?; - if token_ids.contains(&self.video_token_id) { + if prompt_token_ids.contains(&self.video_token_id) { return Err(Error::frontend(FrontendError::Input(Failure::new( "video inputs are not supported by the image processor", )))); } + let mut expanded_ids = Vec::with_capacity( + prompt_token_ids.len() + + images + .geometry + .iter() + .map(|geometry| geometry.tokens.saturating_sub(1)) + .sum::(), + ); + let mut image_index = 0usize; + for &id in prompt_token_ids { + if id == self.image_token_id { + let tokens = images.geometry[image_index].tokens; + expanded_ids.extend(std::iter::repeat(self.image_token_id).take(tokens)); + image_index += 1; + } else { + expanded_ids.push(id); + } + } + let token_ids = expanded_ids; let mm_token_type_ids: Vec = token_ids .iter() .map(|id| i32::from(*id == self.image_token_id)) @@ -159,7 +172,6 @@ impl QwenProcessor { )))); } Ok(ImagePrompt { - text: expanded, rope_delta: current - token_ids.len() as i64, token_ids, mm_token_type_ids, diff --git a/helios/src/frontend/jinja/mod.rs b/helios/src/frontend/jinja/mod.rs index 6f53e8b2f..22744ab86 100644 --- a/helios/src/frontend/jinja/mod.rs +++ b/helios/src/frontend/jinja/mod.rs @@ -45,6 +45,7 @@ pub enum Mark { pub struct RenderedTemplate { pub text: String, pub marks: Vec, + pub runs: Vec<(usize, bool)>, } pub struct CompiledTemplate { @@ -69,7 +70,17 @@ impl CompiledTemplate { context: &Map, hints: &[(usize, usize)], ) -> Result { - vm::render(self, context, hints) + use std::collections::HashSet; + self.render_with_trust(context, hints, &HashSet::new()) + } + + pub fn render_with_trust( + &self, + context: &Map, + hints: &[(usize, usize)], + trusted: &std::collections::HashSet, + ) -> Result { + vm::render(self, context, hints, trusted) } pub fn annotation_spans(&self) -> &[std::ops::Range] { diff --git a/helios/src/frontend/jinja/tests.rs b/helios/src/frontend/jinja/tests.rs index 7e2405820..0f2676efb 100644 --- a/helios/src/frontend/jinja/tests.rs +++ b/helios/src/frontend/jinja/tests.rs @@ -55,6 +55,40 @@ fn compare(source: &str, context: Json) { } } +#[test] +fn request_values_record_untrusted_runs_and_defaults_stay_trusted() { + let template = CompiledTemplate::compile("A{{ x }}B{{ 'C' }}{{ y }}").unwrap(); + let context = json!({"x":"<|pad|>", "y":"plain"}); + let mut trusted = std::collections::HashSet::new(); + trusted.insert("y".to_owned()); + let rendered = template + .render_with_trust(context.as_object().unwrap(), &[], &trusted) + .unwrap(); + assert_eq!(rendered.text, "A<|pad|>BCplain"); + assert_eq!( + rendered.runs, + vec![(1, true), (8, false), (15, true)] + ); + let rendered = template.render(context.as_object().unwrap(), &[]).unwrap(); + assert_eq!( + rendered.runs, + vec![(1, true), (8, false), (10, true), (15, false)] + ); +} + +#[test] +fn captures_preserve_trust_runs_and_concatenation_widens() { + let template = + CompiledTemplate::compile("{% set body %}<|s|>{{ m }}{% endset %}{{ body }}{{ a ~ m }}") + .unwrap(); + let context = json!({"m":"x", "a":"<|a|>"}); + let rendered = template + .render_with_trust(context.as_object().unwrap(), &[], &std::collections::HashSet::new()) + .unwrap(); + assert_eq!(rendered.text, "<|s|>x<|a|>x"); + assert_eq!(rendered.runs, vec![(5, true), (12, false)]); +} + #[test] fn expressions_and_python_methods_match_minijinja() { let context = json!({"s":" café 👋 ","a":[1,2,3,4],"m":{"z":true,"a":null},"d":{"b":2,"B":3,"a":1},"f":[1e-6,-0.0,1e16,2.5,3.0],"none":null}); @@ -171,7 +205,8 @@ fn marks_are_zero_output_and_cannot_escape_captures() { Mark::Byte(3), Mark::Byte(7), Mark::Position(3) - ] + ], + runs: vec![(7, true)], } ); assert_eq!(template.annotation_spans().len(), 3); @@ -476,3 +511,283 @@ fn json_object_members_preserves_literals_order_and_duplicate_keys() { } assert!(CompiledTemplate::compile("{{ json_object_members([]) }}").unwrap().render(&Map::new(), &[]).is_err()); } + + +#[test] +fn diag_micro_paths() { + for (source, expected_untrusted) in [ + ("{% macro r(c) %}{{ c }}{% endmacro %}{{ r(m) }}", true), + ("{% for message in messages %}{{ message.content }}{% endfor %}", true), + ("{% set body %}{{ m }}{% endset %}{{ body|trim }}", true), + ] { + let template = CompiledTemplate::compile(source).unwrap(); + let context = json!({"m":"<|x|>", "messages":[{"content":"<|y|>"}]}); + let rendered = template + .render_with_trust(context.as_object().unwrap(), &[], &std::collections::HashSet::new()) + .unwrap(); + let untrusted = rendered.runs.iter().any(|(_, t)| !t); + assert_eq!(untrusted, expected_untrusted, "source: {source}"); + } +} + +const ENVELOPE_TEMPLATE: &str = r#" +{%- macro render_content(content) -%} +{%- if content is string -%} +{{- content -}} +{%- elif content is iterable and content is not mapping -%} +{%- for item in content -%} +{%- if item.type == 'image' -%} +{{- '<|vision_start|><|image_pad|><|vision_end|>' -}} +{%- else -%} +{{- item.text -}} +{%- endif -%} +{%- endfor -%} +{%- endif -%} +{%- endmacro -%} +{%- for message in messages -%} +{{- '<|im_start|>' -}}{{- message.role -}}{{- '\n' -}}{{ render_content(message.content) }}{{- '<|im_end|>' -}}{{- '\n' -}} +{%- endfor -%} +{%- if add_generation_prompt -%} +{{- '<|im_start|>assistant\n' -}} +{%- endif -%} +"#; + +fn covering_trust(rendered: &RenderedTemplate, start: usize, end: usize) -> Option { + let mut covering = None; + let mut run_start = 0; + for (run_end, trusted) in &rendered.runs { + if run_start <= start && *run_end >= end { + covering = Some(*trusted); + } + run_start = *run_end; + } + covering +} + +#[test] +fn pasted_placeholders_render_as_untrusted_spans() { + let template = CompiledTemplate::compile(ENVELOPE_TEMPLATE).unwrap(); + let pasted = "\"<|vision_start|><|image_pad|><|vision_end|>\""; + let context = json!({ + "messages": [{ + "role": "user", + "content": [ + {"type": "image", "image": "placeholder"}, + {"type": "text", "text": format!("Between the quotes: {pasted} ok?")} + ] + }], + "add_generation_prompt": true + }); + let rendered = template + .render_with_trust(context.as_object().unwrap(), &[], &std::collections::HashSet::new()) + .unwrap(); + let envelope = "<|vision_start|><|image_pad|><|vision_end|>"; + let offsets: Vec = rendered + .text + .match_indices(envelope) + .map(|(offset, _)| offset) + .collect(); + assert_eq!(offsets.len(), 2, "text: {:?}", rendered.text); + assert_eq!( + covering_trust(&rendered, offsets[0], offsets[0] + envelope.len()), + Some(true), + "runs: {:?}", + rendered.runs + ); + let pasted_start = rendered.text.find(pasted).unwrap(); + assert_eq!( + covering_trust(&rendered, pasted_start, pasted_start + pasted.len()), + Some(false), + "runs: {:?}", + rendered.runs + ); + assert_eq!( + rendered.text, + format!("<|im_start|>user\n{envelope}Between the quotes: {pasted} ok?<|im_end|>\n<|im_start|>assistant\n") + ); +} + +#[test] +fn diag_set_macro_trim_concat() { + for source in [ + "{% macro r(c, d, e) %}{{ c }}{% endmacro %}{% set content = r(m, true, false) | trim %}{{ content }}", + "{% macro r(c, d, e) %}{{ c }}{% endmacro %}{% set content = r(m, true, false) | trim %}{{ '<|a|>' + content + '<|b|>' }}", + ] { + let template = CompiledTemplate::compile(source).unwrap(); + let context = json!({"m":"<|x|>"}); + let rendered = template + .render_with_trust(context.as_object().unwrap(), &[], &std::collections::HashSet::new()) + .unwrap(); + assert!(rendered.runs.iter().any(|(_, t)| !t), "all trusted: {source}"); + } +} + +#[test] +fn captures_preserve_trust_through_placeholder_emission() { + let template = CompiledTemplate::compile( + "{% macro envelope(c) %}{% if c is string %}{{- '<|vision_start|><|image_pad|><|vision_end|>' }}{{- c }}{%- else %}{{- raise_exception('unexpected content') }}{%- endif %}{% endmacro %}{% set body %}{{ envelope(m) }}{% endset %}{{ body }}", + ) + .unwrap(); + let context = json!({"m":"log text <|im_end|> more"}); + let rendered = template + .render_with_trust(context.as_object().unwrap(), &[], &std::collections::HashSet::new()) + .unwrap(); + assert_eq!(rendered.text, "<|vision_start|><|image_pad|><|vision_end|>log text <|im_end|> more"); + assert_eq!( + rendered.runs, + vec![(43, true), (67, false)] + ); +} + +#[test] +fn indexing_key_trust_does_not_taint_value() { + let context = json!({ + "messages":[{"content":"<|turn|>"}], + "parts":["<|pad|>"], + "wrap":{"content":"<|endoftext|>"} + }); + let sources = [ + "{{ messages[0]['content'] }}", + "{{ messages[0].content }}", + "{% for message in messages %}{{ message['content'] }}{% endfor %}", + "{{ parts[0] }}", + "{{ wrap['content'] }}", + "{{ wrap.content }}", + ]; + for source in sources { + let rendered = CompiledTemplate::compile(source) + .unwrap() + .render_with_trust( + context.as_object().unwrap(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert!( + rendered.text.starts_with('<'), + "{source}: {}", + rendered.text + ); + assert_eq!( + covering_trust(&rendered, 0, rendered.text.len()), + Some(false), + "{source}: {:?}", + rendered.runs + ); + } + let literal = CompiledTemplate::compile("{{ ['<|lit|>'][0] }}") + .unwrap() + .render_with_trust( + &serde_json::Map::new(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert_eq!(covering_trust(&literal, 0, literal.text.len()), Some(true)); +} + +#[test] +fn trim_remaps_trust_runs() { + let envelope = "<|vision_start|><|image_pad|><|vision_end|>"; + let template = CompiledTemplate::compile(&format!( + "{{% macro body() %}}{envelope}{{{{ m }}}}{{% endmacro %}}{{{{ body()|trim }}}}" + )) + .unwrap(); + let rendered = template + .render_with_trust( + json!({"m":"hello "}).as_object().unwrap(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert_eq!(rendered.text, format!("{envelope}hello")); + assert_eq!( + rendered.runs, + vec![(envelope.len(), true), (rendered.text.len(), false)] + ); + let trailing = CompiledTemplate::compile(&format!( + "{{% set body %}}{{{{ m }}}}{envelope} {{% endset %}}{{{{ body|trim }}}}" + )) + .unwrap(); + let rendered = trailing + .render_with_trust( + json!({"m":"hi "}).as_object().unwrap(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert_eq!(rendered.text, format!("hi {envelope}")); + assert_eq!(rendered.runs, vec![(3, false), (rendered.text.len(), true)]); + let method = + CompiledTemplate::compile("{% set body %}{{ m }}<|a|> {% endset %}{{ body.rstrip() }}") + .unwrap(); + let rendered = method + .render_with_trust( + json!({"m":"x"}).as_object().unwrap(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert_eq!(rendered.text, "x<|a|>"); + assert_eq!(rendered.runs, vec![(1, false), (rendered.text.len(), true)]); +} + +#[test] +fn namespace_provenance_recursive() { + let cases = [ + "{% set ns = namespace(body=m) %}{{ ns.body }}", + "{% set ns = namespace(body=m) %}{{ ns['body'] }}", + "{% set ns = namespace() %}{% set ns.body = m %}{{ ns.body }}", + "{% set d = dict(body=m) %}{{ d.body }}", + "{% set ns = namespace(body=m) %}{{ ns.body[0:] }}", + "{% set ns = namespace(body=m) %}{{ ns.body ~ 'tail' }}", + "{% set ns = namespace(body=m) %}{% set ns.body = ns.body ~ 'tail' %}{{ ns.body }}", + ]; + for source in cases { + let rendered = CompiledTemplate::compile(source) + .unwrap() + .render_with_trust( + json!({"m":"<|image_pad|>"}).as_object().unwrap(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert_eq!( + covering_trust( + &rendered, + rendered.text.find("<|image_pad|>").unwrap(), + rendered.text.find("<|image_pad|>").unwrap() + "<|image_pad|>".len() + ), + Some(false), + "{source}: {:?}", + rendered.runs + ); + } + let upper = + CompiledTemplate::compile("{% set ns = namespace(body=m) %}{{ ns.body|upper }}").unwrap(); + let rendered = upper + .render_with_trust( + json!({"m":"<|image_pad|>"}).as_object().unwrap(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert_eq!(rendered.text, "<|IMAGE_PAD|>"); + assert_eq!(rendered.runs, vec![(rendered.text.len(), false)]); + + let iterate = CompiledTemplate::compile( + "{% set ns = namespace(body=messages) %}{% for message in ns.body %}{{ message.content }}{% endfor %}", + ) + .unwrap(); + let rendered = iterate + .render_with_trust( + json!({"messages":[{"content":"<|x|>"},{"content":"<|y|>"}]}) + .as_object() + .unwrap(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert_eq!(rendered.text, "<|x|><|y|>"); + assert_eq!(rendered.runs, vec![(rendered.text.len(), false)]); +} diff --git a/helios/src/frontend/jinja/value.rs b/helios/src/frontend/jinja/value.rs index 600f9f148..acc7d1f53 100644 --- a/helios/src/frontend/jinja/value.rs +++ b/helios/src/frontend/jinja/value.rs @@ -22,7 +22,7 @@ pub(super) enum Value<'a> { UInt(u64), Float(f64), Str(&'a str), - String(Rc<(String, Vec)>), + String(Rc<(String, Vec, Vec<(usize, bool)>)>), Array(&'a [Json]), Object(&'a serde_json::Map), List(Rc>>), @@ -37,6 +37,7 @@ pub(super) enum Value<'a> { pub(super) struct LoopValue<'a> { pub items: Vec>, pub index: Cell, + pub trust: bool, } impl<'a> Value<'a> { @@ -76,7 +77,70 @@ impl<'a> Value<'a> { Self::marked(value, Vec::new()) } pub fn marked(value: String, hints: Vec) -> Self { - Self::String(Rc::new((value, hints))) + Self::String(Rc::new((value, hints, Vec::new()))) + } + pub fn marked_runs(value: String, hints: Vec, runs: Vec<(usize, bool)>) -> Self { + Self::String(Rc::new((value, hints, runs))) + } + pub fn carries_untrusted(&self) -> bool { + match self { + Self::String(s) => s.2.iter().any(|(_, trusted)| !trusted), + Self::List(items) => items.iter().any(Self::carries_untrusted), + Self::Map(pairs) => pairs.iter().any(|(_, value)| value.carries_untrusted()), + Self::Namespace(pairs) => pairs + .borrow() + .iter() + .any(|(_, value)| value.carries_untrusted()), + _ => false, + } + } + pub fn baked_untrusted(value: Self) -> Self { + match value { + Self::Str(s) => { + let runs = vec![(s.len(), false)]; + Self::marked_runs(s.to_owned(), Vec::new(), runs) + } + Self::String(s) => { + if s.2.is_empty() { + Self::String(Rc::new(( + s.0.clone(), + s.1.clone(), + vec![(s.0.len(), false)], + ))) + } else { + Self::String(s) + } + } + Self::Array(items) => Self::list( + items + .iter() + .map(|item| Self::baked_untrusted(Self::json(item))) + .collect(), + ), + Self::Object(members) => Self::Map(Rc::new( + members + .iter() + .map(|(key, item)| (key.clone(), Self::baked_untrusted(Self::json(item)))) + .collect(), + )), + Self::List(items) => { + Self::list(items.iter().cloned().map(Self::baked_untrusted).collect()) + } + Self::Map(pairs) => Self::Map(Rc::new( + pairs + .iter() + .map(|(key, item)| (key.clone(), Self::baked_untrusted(item.clone()))) + .collect(), + )), + Self::Namespace(pairs) => Self::Namespace(Rc::new(RefCell::new( + pairs + .borrow() + .iter() + .map(|(key, item)| (key.clone(), Self::baked_untrusted(item.clone()))) + .collect(), + ))), + other => other, + } } pub fn hints(&self) -> &[usize] { match self { @@ -89,6 +153,79 @@ impl<'a> Value<'a> { hints.extend(self.hints().iter().map(|end| start + end)); self.write(output); } + pub fn append_annotated( + &self, + output: &mut String, + hints: &mut Vec, + runs: &mut Vec<(usize, bool)>, + trust: bool, + ) { + let start = output.len(); + hints.extend(self.hints().iter().map(|end| start + end)); + if let Self::String(s) = self { + if !s.2.is_empty() { + output.push_str(&s.0); + for (end, trusted) in &s.2 { + let end = start + end; + match runs.last_mut() { + Some(last) if last.1 == *trusted => last.0 = end, + _ => runs.push((end, *trusted)), + } + } + return; + } + } + self.write(output); + let end = output.len(); + match runs.last_mut() { + Some(last) if last.1 == trust => last.0 = end, + _ => runs.push((end, trust)), + } + } + pub fn sliced_annotations(&self, start: usize, end: usize) -> (Vec, Vec<(usize, bool)>) { + let mut hints = Vec::new(); + let mut runs: Vec<(usize, bool)> = Vec::new(); + if let Self::String(s) = self { + for hint in &s.1 { + if *hint > start { + hints.push((*hint - start).min(end - start)); + } + } + let mut cursor = 0; + for (run_end, trusted) in &s.2 { + let run_start = cursor; + cursor = *run_end; + let low = run_start.max(start); + let high = (*run_end).min(end); + if high > low { + match runs.last_mut() { + Some(last) if last.1 == *trusted => last.0 = high - start, + _ => runs.push((high - start, *trusted)), + } + } + } + } + (hints, runs) + } + pub(super) fn trust_at_byte(&self, byte: usize) -> Option { + let Self::String(s) = self else { + return None; + }; + s.2.iter() + .find(|(end, _)| byte < *end) + .map(|(_, trusted)| *trusted) + } + fn annotated_char(&self, text: &str, byte: usize) -> Self { + let character = text[byte..].chars().next().unwrap(); + match self.trust_at_byte(byte) { + Some(trusted) => Self::marked_runs( + character.to_string(), + Vec::new(), + vec![(character.len_utf8(), trusted)], + ), + None => Self::string(character.to_string()), + } + } pub fn list(value: Vec) -> Self { Self::List(Rc::new(value)) } @@ -202,8 +339,11 @@ impl<'a> Value<'a> { Self::List(v) => v.get(index).cloned().unwrap_or_default(), _ => self .as_str() - .and_then(|s| s.chars().nth(index)) - .map(|c| Self::string(c.to_string())) + .and_then(|s| { + s.char_indices() + .nth(index) + .map(|(byte, _)| self.annotated_char(s, byte)) + }) .unwrap_or_default(), } } @@ -218,8 +358,8 @@ impl<'a> Value<'a> { _ if self.as_str().is_some() => Ok(self .as_str() .unwrap() - .chars() - .map(|c| Self::string(c.to_string())) + .char_indices() + .map(|(byte, _)| self.annotated_char(self.as_str().unwrap(), byte)) .collect()), _ => Err(Error::new("value is not iterable")), } diff --git a/helios/src/frontend/jinja/vm.rs b/helios/src/frontend/jinja/vm.rs index 3529dec70..6567a2a73 100644 --- a/helios/src/frontend/jinja/vm.rs +++ b/helios/src/frontend/jinja/vm.rs @@ -21,7 +21,7 @@ struct Scope { struct Closure<'a> { definition: u32, scope: usize, - captured: Option>>, + captured: Option<(Vec>, Vec)>, } struct Vm<'a> { template: &'a CompiledTemplate, @@ -31,11 +31,14 @@ struct Vm<'a> { serial: usize, scopes: Vec, closures: Vec>, - undo: Vec<(u32, Value<'a>, usize)>, + undo: Vec<(u32, Value<'a>, usize, bool)>, stack: Vec>, + stack_trust: Vec, + slots_trust: Vec, loops: Vec>>, output: String, - captures: Vec<(String, Vec)>, + output_runs: Vec<(usize, bool)>, + captures: Vec<(String, Vec, Vec<(usize, bool)>)>, hint_positions: Vec, marks: Vec, depth: usize, @@ -46,6 +49,7 @@ pub(super) fn render( template: &CompiledTemplate, context: &Map, hints: &[(usize, usize)], + trusted: &std::collections::HashSet, ) -> Result { let mut hint_texts = Vec::new(); for &(message, count) in hints { @@ -85,8 +89,11 @@ pub(super) fn render( closures: Vec::new(), undo: Vec::new(), stack: Vec::with_capacity(16), + stack_trust: Vec::with_capacity(16), + slots_trust: template.names.iter().map(|s| trusted.contains(s)).collect(), loops: Vec::new(), output: String::new(), + output_runs: Vec::new(), captures: Vec::new(), hint_positions: Vec::new(), marks: Vec::new(), @@ -98,6 +105,7 @@ pub(super) fn render( Ok(RenderedTemplate { text: vm.output, marks: vm.marks, + runs: vm.output_runs, }) } @@ -107,6 +115,22 @@ impl<'a> Vm<'a> { .pop() .expect("compiler balances the operand stack") } + fn pop_trust(&mut self) -> bool { + self.stack_trust + .pop() + .expect("compiler balances the operand stack") + } + fn pop_trusted(&mut self) -> (Value<'a>, bool) { + let trust = self.pop_trust(); + (self.pop(), trust) + } + fn force_trust(value: Value<'a>, trusted: bool) -> Value<'a> { + if trusted { + value + } else { + Value::baked_untrusted(value) + } + } fn enter(&mut self) { self.scopes.push(Scope { undo: self.undo.len(), @@ -123,15 +147,20 @@ impl<'a> Vm<'a> { let values = self.template.macros[closure.definition as usize] .captures .iter() - .map(|&s| self.slots[s as usize].clone()) - .collect(); - self.closures[i].captured = Some(values); + .map(|&s| (self.lexical(s, self.scope), self.lexical_trust(s, self.scope))) + .collect::>(); + let (values, trusts): (Vec<_>, Vec<_>) = values + .into_iter() + .map(|(value, trust)| (Value::baked_untrusted(value), trust)) + .unzip(); + self.closures[i].captured = Some((values, trusts)); } } let scope = self.scopes.pop().unwrap(); - for (slot, value, stamp) in self.undo.drain(scope.undo..).rev() { + for (slot, value, stamp, trust) in self.undo.drain(scope.undo..).rev() { self.slots[slot as usize] = value; self.stamps[slot as usize] = stamp; + self.slots_trust[slot as usize] = trust; } self.scope = scope.id; } @@ -139,7 +168,7 @@ impl<'a> Vm<'a> { let mut value = &self.slots[slot as usize]; let mut stamp = self.stamps[slot as usize]; if stamp > scope { - for (s, previous, previous_stamp) in self.undo.iter().rev() { + for (s, previous, previous_stamp, _) in self.undo.iter().rev() { if *s == slot { value = previous; stamp = *previous_stamp; @@ -151,7 +180,23 @@ impl<'a> Vm<'a> { } value.clone() } - fn store(&mut self, slot: u32, value: Value<'a>) { + fn lexical_trust(&self, slot: u32, scope: usize) -> bool { + let mut trust = self.slots_trust[slot as usize]; + let mut stamp = self.stamps[slot as usize]; + if stamp > scope { + for (s, _, previous_stamp, previous_trust) in self.undo.iter().rev() { + if *s == slot { + trust = *previous_trust; + stamp = *previous_stamp; + if stamp <= scope { + break; + } + } + } + } + trust + } + fn store(&mut self, slot: u32, value: Value<'a>, trust: bool) { let value = self.hinted(value); let index = slot as usize; if self.scope != 0 && self.stamps[index] != self.scope { @@ -159,28 +204,31 @@ impl<'a> Vm<'a> { slot, std::mem::take(&mut self.slots[index]), self.stamps[index], + self.slots_trust[index], )); self.stamps[index] = self.scope; } self.slots[index] = value; + self.slots_trust[index] = trust; } fn iteration(&mut self, id: u32) -> Result<()> { let value = self.loops.last().unwrap().clone(); let item = value.items[value.index.get()].clone(); - self.store(self.template.loop_symbol, Value::Loop(value)); - self.loop_variables(id, item) + let trust = value.trust; + self.store(self.template.loop_symbol, Value::Loop(value), trust); + self.loop_variables(id, item, trust) } - fn loop_variables(&mut self, id: u32, item: Value<'a>) -> Result<()> { + fn loop_variables(&mut self, id: u32, item: Value<'a>, trust: bool) -> Result<()> { let vars = &self.template.loops[id as usize].variables; if vars.len() == 1 { - self.store(vars[0], item); + self.store(vars[0], item, trust); } else { let items = item.iter()?; if items.len() != vars.len() { return Err(Error::new("cannot unpack sequence of different length")); } for (&slot, value) in vars.iter().zip(items) { - self.store(slot, value); + self.store(slot, value, trust); } } Ok(()) @@ -196,21 +244,27 @@ impl<'a> Vm<'a> { } value } - fn emit(&mut self, value: Value<'a>) { - self.hinted(value) - .append(&mut self.output, &mut self.hint_positions); + fn emit(&mut self, value: Value<'a>, trust: bool) { + self.hinted(value).append_annotated( + &mut self.output, + &mut self.hint_positions, + &mut self.output_runs, + trust, + ); } fn capture(&mut self) { self.captures.push(( std::mem::take(&mut self.output), std::mem::take(&mut self.hint_positions), + std::mem::take(&mut self.output_runs), )); } fn end_capture(&mut self) -> Value<'a> { - let (output, hints) = self.captures.pop().unwrap(); + let (output, hints, runs) = self.captures.pop().unwrap(); let text = std::mem::replace(&mut self.output, output); let positions = std::mem::replace(&mut self.hint_positions, hints); - Value::marked(text, positions) + let runs = std::mem::replace(&mut self.output_runs, runs); + Value::marked_runs(text, positions, runs) } fn run(&mut self, start: u32) -> Result<()> { self.depth += 1; @@ -223,19 +277,33 @@ impl<'a> Vm<'a> { let op = self.template.code[pc]; pc += 1; match op { - I::Text(id) => self - .output - .push_str(self.template.constants[id as usize].as_str().unwrap()), - I::Constant(id) => self - .stack - .push(Value::json(&self.template.constants[id as usize])), - I::Load(id) => self.stack.push(self.slots[id as usize].clone()), + I::Text(id) => { + self.output + .push_str(self.template.constants[id as usize].as_str().unwrap()); + self.output_runs.push((self.output.len(), true)); + } + I::Constant(id) => { + self.stack + .push(Value::json(&self.template.constants[id as usize])); + self.stack_trust.push(true); + } + I::Load(id) => { + self.stack.push(self.slots[id as usize].clone()); + self.stack_trust.push(self.slots_trust[id as usize]); + } I::Store(id) => { + let trust = self.pop_trust(); let v = self.pop(); - self.store(id, v); + self.store(id, v, trust); } I::StoreAttr(slot, name) | I::AppendAttr(slot, name) => { + let trust = self.pop_trust(); let value = self.pop(); + let value = if trust { + value + } else { + Value::baked_untrusted(value) + }; let Value::Namespace(ns) = &self.slots[slot as usize] else { return Err(Error::new( "cannot assign attribute of non-namespace object", @@ -251,6 +319,20 @@ impl<'a> Vm<'a> { { Rc::make_mut(items).extend(value.iter()?) } + (old, value) + if old.as_str().is_some() && value.as_str().is_some() => + { + let mut text = String::new(); + let mut hints = Vec::new(); + let mut runs = Vec::new(); + old.append_annotated( + &mut text, &mut hints, &mut runs, true, + ); + value.append_annotated( + &mut text, &mut hints, &mut runs, true, + ); + *old = Value::marked_runs(text, hints, runs); + } (old, value) => { *old = binary(Binary::Add, std::mem::take(old), value)? } @@ -265,114 +347,216 @@ impl<'a> Vm<'a> { } } I::Attr(name) => { + let trust = self.pop_trust(); let v = self.pop(); if matches!(v, Value::Undefined) { return Err(Error::new("undefined value")); } let value = self.hinted(v.attr(&self.template.names[name as usize])); self.stack.push(value); + self.stack_trust.push(trust); } I::Index => { + let _ = self.pop_trust(); let key = self.pop(); + let trust = self.pop_trust(); let v = self.pop(); if matches!(v, Value::Undefined) { return Err(Error::new("undefined value")); } let value = self.hinted(v.index(&key)); self.stack.push(value); + self.stack_trust.push(trust); } I::Slice => { + let _ = self.pop_trust(); let step = self.pop(); + let _ = self.pop_trust(); let end = self.pop(); + let _ = self.pop_trust(); let begin = self.pop(); + let trust = self.pop_trust(); let v = self.pop(); - self.stack.push(slice(v, begin, end, step)?); + let sliced = slice(v, begin, end, step)?; + self.stack.push(Self::force_trust(sliced, trust)); + self.stack_trust.push(trust); } I::List(n) => { + let trusts = self.stack_trust.split_off(self.stack_trust.len() - n as usize); let items = self.stack.split_off(self.stack.len() - n as usize); + let trusted = trusts.iter().all(|trusted| *trusted); + let items = items + .into_iter() + .zip(trusts) + .map(|(value, trust)| if trust { value } else { Value::baked_untrusted(value) }) + .collect(); self.stack.push(Value::list(items)); + self.stack_trust.push(trusted); } I::Map(n) => { + let trusts = self.stack_trust.split_off(self.stack_trust.len() - 2 * n as usize); let values = self.stack.split_off(self.stack.len() - 2 * n as usize); let mut pairs = Vec::with_capacity(n as usize); - let mut values = values.into_iter(); - while let Some(key) = values.next() { - pairs.push((key.text(), values.next().unwrap())); + let mut trusted = true; + let mut entries = values.into_iter().zip(trusts); + while let Some((key, key_trust)) = entries.next() { + let (value, value_trust) = entries.next().unwrap(); + trusted &= key_trust & value_trust; + let value = if value_trust { + value + } else { + Value::baked_untrusted(value) + }; + pairs.push((key.text(), value)); } self.stack.push(Value::Map(Rc::new(pairs))); + self.stack_trust.push(trusted); } I::Unary(op) => { + let trust = self.pop_trust(); let v = self.pop(); - self.stack.push(match op { - Unary::Not => Value::Bool(!v.truthy()), - Unary::Pos => v, + let (value, trusted) = match op { + Unary::Not => (Value::Bool(!v.truthy()), true), + Unary::Pos => (v, trust), Unary::Neg => match v { - Value::Int(v) => Value::Int( + Value::Int(v) => (Value::Int( v.checked_neg() .ok_or_else(|| Error::new("integer overflow"))?, - ), - _ => Value::Float( + ), true), + _ => (Value::Float( -v.number() .ok_or_else(|| Error::new("cannot negate non-number"))?, - ), + ), true), }, - }); + }; + self.stack.push(value); + self.stack_trust.push(trusted); } I::Binary(op) => { + let b_trust = self.pop_trust(); let b = self.pop(); + let a_trust = self.pop_trust(); let a = self.pop(); - self.stack.push(binary(op, a, b)?); + let mut annotated = false; + let result = if matches!(op, Binary::Concat | Binary::Add) + && a.as_str().is_some() + && b.as_str().is_some() + { + annotated = true; + let mut text = String::new(); + let mut hints = Vec::new(); + let mut runs = Vec::new(); + a.append_annotated(&mut text, &mut hints, &mut runs, a_trust); + b.append_annotated(&mut text, &mut hints, &mut runs, b_trust); + Value::marked_runs(text, hints, runs) + } else { + binary(op, a, b)? + }; + let trusted = match op { + Binary::Eq + | Binary::Ne + | Binary::In + | Binary::NotIn + | Binary::Lt + | Binary::Le + | Binary::Gt + | Binary::Ge => true, + Binary::Concat | Binary::Add if annotated => { + !result.carries_untrusted() + } + _ => a_trust && b_trust, + }; + let result = Self::force_trust(result, trusted); + self.stack.push(result); + self.stack_trust.push(trusted); } I::Test(id) | I::Filter(id) | I::Method(id) | I::Call(id) => { let call = &self.template.calls[id as usize]; let count = call.positional + call.keywords.len(); + let arg_trusts = self.stack_trust.split_off(self.stack_trust.len() - count); let arguments = self.stack.split_off(self.stack.len() - count); let name = self.template.names[call.name as usize].as_str(); let (args, kw) = arguments.split_at(call.positional); + let (arg_trusts, kw_trusts) = arg_trusts.split_at(call.positional); let kwargs: Vec<_> = call .keywords .iter() .zip(kw) - .map(|(key, v)| { - (self.template.names[*key as usize].as_str(), v.clone()) + .zip(kw_trusts) + .map(|((key, v), trust)| { + (self.template.names[*key as usize].as_str(), v.clone(), *trust) }) .collect(); + let kw_named: Vec<(&str, Value<'a>)> = kwargs + .iter() + .map(|(name, value, _)| (*name, value.clone())) + .collect(); + let arguments_trusted = arg_trusts.iter().chain(kw_trusts.iter()).all(|trusted| *trusted); + let mut input_trust = true; + let mut input_untrusted = false; + let mut macro_result = false; let result = match op { I::Call(_) => { if let Value::Macro(id) = self.slots[call.name as usize] { let direct = matches!(self.template.code.get(pc), Some(I::Emit)); - let value = self.macro_call(id, args, &kwargs, direct)?; + let value = self.macro_call( + id, + args, + &kwargs, + arg_trusts, + direct, + )?; if direct { pc += 1; continue; } + macro_result = true; value } else { - function(name, args, &kwargs)? + function(name, args, &kw_named)? } } I::Test(_) => { - let v = self.pop(); + let (v, _) = self.pop_trusted(); Value::Bool(test(name, &v, args)?) } I::Filter(_) => { - let v = self.pop(); - filter(name, v, args, &kwargs, !self.hint_texts.is_empty())? + let (v, trust) = self.pop_trusted(); + input_trust = trust; + input_untrusted = v.carries_untrusted(); + filter(name, v, args, &kw_named, !self.hint_texts.is_empty())? + } + _ => { + let (v, trust) = self.pop_trusted(); + input_trust = trust; + input_untrusted = v.carries_untrusted(); + method(name, v, args, &kw_named)? } + }; + let trusted = match op { + I::Test(_) => true, + I::Call(_) if macro_result => !result.carries_untrusted(), + I::Call(_) => arguments_trusted && !result.carries_untrusted(), _ => { - let v = self.pop(); - method(name, v, args, &kwargs)? + arguments_trusted + && input_trust + && !input_untrusted + && !result.carries_untrusted() } }; + let result = Self::force_trust(result, trusted); self.stack.push(result); + self.stack_trust.push(trusted); } I::Emit => { + let trust = self.pop_trust(); let v = self.pop(); - self.emit(v); + self.emit(v, trust); } I::Join(count, strict) | I::EmitJoined(count, strict) => { let start = self.stack.len() - count as usize; + let trusts = self.stack_trust.split_off(start); let values = &self.stack[start..]; if strict && values.iter().any(|v| v.as_str().is_none()) { return Err(Error::new("string addition requires strings")); @@ -380,24 +564,30 @@ impl<'a> Vm<'a> { let size = values.iter().map(|v| v.as_str().map_or(0, str::len)).sum(); if matches!(op, I::EmitJoined(..)) { self.output.reserve(size); - for index in start..self.stack.len() { - self.emit(self.stack[index].clone()); + for (index, trust) in (start..self.stack.len()).zip(trusts) { + let value = self.stack[index].clone(); + self.emit(value, trust); } self.stack.truncate(start); } else { let mut text = String::with_capacity(size); let mut hints = Vec::new(); - for v in self.stack.drain(start..) { - v.append(&mut text, &mut hints); + let mut runs = Vec::new(); + for (v, trust) in self.stack.drain(start..).zip(trusts) { + v.append_annotated(&mut text, &mut hints, &mut runs, trust); } - self.stack.push(Value::marked(text, hints)); + let trusted = runs.iter().all(|(_, trusted)| *trusted); + self.stack.push(Value::marked_runs(text, hints, runs)); + self.stack_trust.push(trusted); } } I::Drop => { + let _ = self.pop_trust(); self.pop(); } I::Jump(target) => pc = target as usize, I::JumpFalse(target) => { + let _ = self.pop_trust(); if !self.pop().truthy() { pc = target as usize; } @@ -406,6 +596,7 @@ impl<'a> Vm<'a> { if !self.stack.last().unwrap().truthy() { pc = target as usize; } else { + let _ = self.pop_trust(); self.pop(); } } @@ -413,17 +604,20 @@ impl<'a> Vm<'a> { if self.stack.last().unwrap().truthy() { pc = target as usize; } else { + let _ = self.pop_trust(); self.pop(); } } I::For(id) => { + let trust = self.pop_trust(); let mut items = self.pop().iter()?; if let Some(filter) = self.template.loops[id as usize].filter { self.enter(); let mut selected = Vec::new(); for item in items { - self.loop_variables(id, item.clone())?; + self.loop_variables(id, item.clone(), trust)?; self.run(filter)?; + let _ = self.pop_trust(); if self.pop().truthy() { selected.push(item); } @@ -435,6 +629,7 @@ impl<'a> Vm<'a> { self.loops.push(Rc::new(LoopValue { items, index: Cell::new(0), + trust, })); self.enter(); if empty { @@ -469,16 +664,19 @@ impl<'a> Vm<'a> { scope: self.scope, captured: None, }); - self.store(slot, Value::Macro(closure)); + self.store(slot, Value::Macro(closure), true); } I::Return => break, I::Capture => self.capture(), I::EndCapture => { let value = self.end_capture(); + let trusted = !value.carries_untrusted(); self.stack.push(value); + self.stack_trust.push(trusted); } I::Mark => self.marks.push(Mark::Byte(self.output.len())), I::MarkPosition => { + let _ = self.pop_trust(); let Value::Int(v) = self.pop() else { return Err(Error::new("mark position must be a nonnegative integer")); }; @@ -503,7 +701,8 @@ impl<'a> Vm<'a> { &mut self, id: u32, args: &[Value<'a>], - kwargs: &[(&str, Value<'a>)], + kwargs: &[(&str, Value<'a>, bool)], + arg_trusts: &[bool], direct: bool, ) -> Result> { let closure = &self.closures[id as usize]; @@ -514,28 +713,31 @@ impl<'a> Vm<'a> { } self.enter(); for (i, &slot) in mac.captures.iter().enumerate() { - let value = if let Some(captured) = &self.closures[id as usize].captured { - captured[i].clone() + let (value, trust) = if let Some((captured, trusts)) = &self.closures[id as usize].captured + { + (captured[i].clone(), trusts[i]) } else { - self.lexical(slot, scope) + let trust = self.lexical_trust(slot, scope); + (self.lexical(slot, scope), trust) }; - self.store(slot, value); + self.store(slot, value, trust); } for (i, &slot) in mac.parameters.iter().enumerate() { let name = self.template.names[slot as usize].as_str(); let value = if let Some(value) = args.get(i) { - value.clone() - } else if let Some((_, value)) = kwargs.iter().find(|(k, _)| *k == name) { - value.clone() + let trust = arg_trusts.get(i).copied().unwrap_or(true); + (value.clone(), trust) + } else if let Some((_, value, trust)) = kwargs.iter().find(|(k, _, _)| *k == name) { + (value.clone(), *trust) } else if let Some((_, pc)) = mac.defaults.iter().find(|(s, _)| *s == slot) { self.run(*pc)?; - self.pop() + self.pop_trusted() } else { - Value::Undefined + (Value::Undefined, true) }; - self.store(slot, value); + self.store(slot, value.0, value.1); } - for (name, _) in kwargs { + for (name, _, _) in kwargs { if !mac .parameters .iter() @@ -687,21 +889,50 @@ fn slice<'a>( }; let mut i = bound(&begin, if step < 0 { n - 1 } else { 0 })?; let end = bound(&end, if step < 0 { -1 } else { n })?; - let mut items = Vec::new(); + let mut indices = Vec::new(); while if step < 0 { i > end } else { i < end } { - items.push(value.index(&Value::Int(i))); + indices.push(i); i = i .checked_add(step) .ok_or_else(|| Error::new("slice index overflow"))?; } - if value.as_str().is_some() { - let mut s = String::new(); - for v in items { - v.write(&mut s); + if let Some(text) = value.as_str() { + let mut output = String::new(); + let mut hints = Vec::new(); + let mut runs: Vec<(usize, bool)> = Vec::new(); + for index in indices { + let Ok(index) = usize::try_from(index) else { + continue; + }; + let Some((byte, character)) = text.char_indices().nth(index) else { + continue; + }; + output.push(character); + let output_end = output.len(); + if let Value::String(source) = &value { + if source + .1 + .iter() + .any(|hint| *hint == byte + character.len_utf8()) + { + hints.push(output_end); + } + if let Some(trusted) = value.trust_at_byte(byte) { + match runs.last_mut() { + Some(last) if last.1 == trusted => last.0 = output_end, + _ => runs.push((output_end, trusted)), + } + } + } } - Ok(Value::string(s)) + Ok(Value::marked_runs(output, hints, runs)) } else { - Ok(Value::list(items)) + Ok(Value::list( + indices + .into_iter() + .map(|index| value.index(&Value::Int(index))) + .collect(), + )) } } @@ -879,17 +1110,15 @@ fn filter<'a>( }; let trimmed = text.trim_matches(pred); if trimmed.len() == text.len() { - Ok(v) - } else { - let start = trimmed.as_ptr() as usize - text.as_ptr() as usize; - let hints = v - .hints() - .iter() - .filter(|end| **end > start && !trimmed.is_empty()) - .map(|end| (end - start).min(trimmed.len())) - .collect(); - Ok(Value::marked(trimmed.into(), hints)) + return Ok(v); + } + let start = trimmed.as_ptr() as usize - text.as_ptr() as usize; + let end = start + trimmed.len(); + let (mut hints, runs) = v.sliced_annotations(start, end); + if trimmed.is_empty() { + hints.clear(); } + Ok(Value::marked_runs(trimmed.into(), hints, runs)) } "capitalize" => { let text = v.text().to_lowercase(); @@ -1096,14 +1325,18 @@ fn method<'a>( "strip" | "lstrip" | "rstrip" => { let chars = args.first().and_then(Value::as_str); let pred = |c: char| chars.map_or_else(|| c.is_whitespace(), |s| s.contains(c)); - Ok(Value::string( - match name { - "lstrip" => text.trim_start_matches(pred), - "rstrip" => text.trim_end_matches(pred), - _ => text.trim_matches(pred), - } - .into(), - )) + let trimmed = match name { + "lstrip" => text.trim_start_matches(pred), + "rstrip" => text.trim_end_matches(pred), + _ => text.trim_matches(pred), + }; + let start = trimmed.as_ptr() as usize - text.as_ptr() as usize; + let end = start + trimmed.len(); + let (mut hints, runs) = v.sliced_annotations(start, end); + if trimmed.is_empty() { + hints.clear(); + } + Ok(Value::marked_runs(trimmed.into(), hints, runs)) } "split" | "rsplit" => { let limit = args diff --git a/helios/src/frontend/mod.rs b/helios/src/frontend/mod.rs index e7879b206..c725fd224 100644 --- a/helios/src/frontend/mod.rs +++ b/helios/src/frontend/mod.rs @@ -297,7 +297,8 @@ pub struct Frontend { prompt_encoder: PromptEncoder, image_placeholder_token_id: Option, tokenizer: Tokenizer, - ordinary_tokenizer: Option, + ordinary_tokenizer: std::sync::OnceLock, + segmented_template: bool, template: Option, template_source: String, bos_token_id: Option, @@ -313,14 +314,93 @@ impl Frontend { } pub fn with_template_encoding(mut self, encoding: TemplateEncoding) -> Self { if self.prompt_encoder == PromptEncoder::Jinja { - self.ordinary_tokenizer = matches!(encoding, TemplateEncoding::TokenSegments).then(|| { - let mut tokenizer = self.tokenizer.clone(); - tokenizer.with_added_vocabulary(tokenizers::AddedVocabulary::new()); - tokenizer - }); + self.segmented_template = matches!(encoding, TemplateEncoding::TokenSegments); } self } + + fn ordinary_from(tokenizer: &Tokenizer) -> Tokenizer { + let mut tokenizer = tokenizer.clone(); + tokenizer.with_added_vocabulary(tokenizers::AddedVocabulary::new()); + tokenizer + } + + fn ordinary(&self) -> &Tokenizer { + self.ordinary_tokenizer.get_or_init(|| Self::ordinary_from(&self.tokenizer)) + } + + fn encode_splicing_untrusted_markers( + &self, + text: &str, + untrusted: &[(usize, usize)], + ) -> Result<(Vec, Vec<(usize, usize)>)> { + let encoding = self.tokenizer.encode(text, false).map_err(|error| { + Error::frontend(FrontendError::Tokenizer(Failure::boxed( + error.to_string(), + error, + ))) + })?; + let token_ids = encoding.get_ids().to_vec(); + let offsets = encoding.get_offsets().to_vec(); + if untrusted.is_empty() { + return Ok((token_ids, offsets)); + } + let added = self.tokenizer.get_added_tokens_decoder(); + let mut spans: Vec<(usize, usize)> = Vec::new(); + let mut token = 0usize; + for &(run_start, run_end) in untrusted { + while token < offsets.len() && offsets[token].1 <= run_start { + token += 1; + } + if token == offsets.len() || offsets[token].0 >= run_end { + continue; + } + let mut span_end = token; + while span_end < offsets.len() && offsets[span_end].0 < run_end { + span_end += 1; + } + if !token_ids[token..span_end] + .iter() + .any(|id| added.contains_key(id)) + { + continue; + } + match spans.last_mut() { + Some(previous) if token <= previous.1 => previous.1 = previous.1.max(span_end), + _ => spans.push((token, span_end)), + } + } + let mut spliced_ids = Vec::with_capacity(token_ids.len()); + let mut spliced_offsets = Vec::with_capacity(offsets.len()); + let mut cursor = 0usize; + for (span_start, span_end) in spans { + let byte_start = offsets[span_start].0; + let byte_end = offsets[span_end - 1].1; + if byte_end <= byte_start { + continue; + } + spliced_ids.extend_from_slice(&token_ids[cursor..span_start]); + spliced_offsets.extend_from_slice(&offsets[cursor..span_start]); + cursor = span_end; + let slice = text.get(byte_start..byte_end).unwrap_or(""); + let replacement = self.ordinary().encode(slice, false).map_err(|error| { + Error::frontend(FrontendError::Tokenizer(Failure::boxed( + error.to_string(), + error, + ))) + })?; + spliced_ids.extend_from_slice(replacement.get_ids()); + spliced_offsets.extend( + replacement + .get_offsets() + .iter() + .map(|&(start, end)| (byte_start + start, byte_start + end)), + ); + } + spliced_ids.extend_from_slice(&token_ids[cursor..]); + spliced_offsets.extend_from_slice(&offsets[cursor..]); + Ok((spliced_ids, spliced_offsets)) + } pub fn bos_token_id(&self) -> Option { self.bos_token_id } @@ -551,12 +631,6 @@ impl Frontend { defaults.insert("tools".into(), Json::Null); defaults.insert("documents".into(), Json::Null); defaults.extend(template_defaults); - // Programmatic encoders tokenize message text without the added vocabulary. - let ordinary_tokenizer = (prompt_encoder != PromptEncoder::Jinja).then(|| { - let mut ordinary = tokenizer.clone(); - ordinary.with_added_vocabulary(tokenizers::AddedVocabulary::new()); - ordinary - }); let frontend = Self { prompt_encoder, image_placeholder_token_id, @@ -565,7 +639,8 @@ impl Frontend { effort_levels: Default::default(), inline_system_messages: Default::default(), tokenizer, - ordinary_tokenizer, + ordinary_tokenizer: std::sync::OnceLock::new(), + segmented_template: false, template, template_source, defaults, @@ -709,11 +784,17 @@ impl Frontend { .map(|field| self.mark_continuation(&mut messages, &field)) .transpose()?; context.insert("messages".into(), Json::Array(messages)); + let trusted: std::collections::HashSet = self + .defaults + .keys() + .filter(|name| !options.contains_key(*name) && name.as_str() != "messages") + .cloned() + .collect(); let rendered = self .template .as_ref() .unwrap() - .render(&context, &checkpoint_hints) + .render_with_trust(&context, &checkpoint_hints, &trusted) .map_err(|error| { Error::frontend(FrontendError::Template(Failure::caused_by( error.message.clone(), @@ -721,6 +802,7 @@ impl Frontend { ))) })?; let marks = rendered.marks; + let runs = rendered.runs; let rendered = rendered.text; let mut hint_bytes = marks .iter() @@ -729,7 +811,7 @@ impl Frontend { _ => None, }) .collect(); - let mut segments: Vec = if self.ordinary_tokenizer.is_some() { + let mut segments: Vec = if self.segmented_template { token_segments(&rendered, &mut hint_bytes).map_err(|error| { Error::frontend(FrontendError::Template(Failure::caused_by( "chat template must render an array of text and allow_special segments", @@ -737,11 +819,7 @@ impl Frontend { ))) })? } else { - vec![TokenSegment { - text: rendered, - allow_special: true, - image_index: None, - }] + Vec::new() }; order_segment_images(&segments, &mut image_sources, &mut image_checkpoint_hints)?; let mut segment_ends = Vec::with_capacity(segments.len() + 1); @@ -753,13 +831,10 @@ impl Frontend { for mark in marks { match mark { jinja::Mark::Hint(_) => {} - jinja::Mark::Byte(byte) - if self.ordinary_tokenizer.is_none() - && byte <= *segment_ends.last().unwrap() => - { + jinja::Mark::Byte(byte) if !self.segmented_template && byte <= rendered.len() => { byte_boundaries.push(byte); } - jinja::Mark::Position(count) if self.ordinary_tokenizer.is_some() => { + jinja::Mark::Position(count) if self.segmented_template => { byte_boundaries.push(*segment_ends.get(count).ok_or_else(|| { deepseek::invalid("mark position exceeds the rendered segment count") })?); @@ -771,10 +846,14 @@ impl Frontend { } } } - let mut text: String = segments - .iter() - .map(|segment| segment.text.as_str()) - .collect(); + let mut text: String = if self.segmented_template { + segments + .iter() + .map(|segment| segment.text.as_str()) + .collect() + } else { + rendered + }; if let Some(final_text) = final_text { let marker = "CONTINUE_FINAL_MESSAGE_TAG"; let index = text.rfind(marker).filter(|_| text.contains(final_text.trim())).ok_or_else(|| Error::frontend(FrontendError::Input(Failure::new("continue_final_message is set but the final message does not appear in the chat after applying the chat template!"))))?; @@ -784,32 +863,56 @@ impl Frontend { text.truncate(text.trim_end().len()); } } - let mut remaining = text.len(); let mut token_ids = Vec::new(); let mut offsets = Vec::new(); - let mut byte_start = 0; - for segment in &mut segments { - segment.text.truncate(remaining.min(segment.text.len())); - remaining -= segment.text.len(); - let tokenizer = if segment.allow_special { - &self.tokenizer - } else { - self.ordinary_tokenizer.as_ref().unwrap_or(&self.tokenizer) - }; - let encoding = tokenizer - .encode(segment.text.as_str(), false) - .map_err(|error| { - Error::frontend(FrontendError::Tokenizer(Failure::boxed( - error.to_string(), - error, - ))) - })?; - token_ids.extend_from_slice(encoding.get_ids()); - offsets.extend(encoding.get_offsets().iter().map(|&(start,end)| (byte_start+start,byte_start+end))); - byte_start += segment.text.len(); - if remaining == 0 { - break; + if self.segmented_template { + let mut remaining = text.len(); + let mut byte_start = 0; + for segment in &mut segments { + segment.text.truncate(remaining.min(segment.text.len())); + remaining -= segment.text.len(); + let tokenizer = if segment.allow_special { + &self.tokenizer + } else { + self.ordinary() + }; + let encoding = tokenizer + .encode(segment.text.as_str(), false) + .map_err(|error| { + Error::frontend(FrontendError::Tokenizer(Failure::boxed( + error.to_string(), + error, + ))) + })?; + token_ids.extend_from_slice(encoding.get_ids()); + offsets.extend( + encoding + .get_offsets() + .iter() + .map(|&(start, end)| (byte_start + start, byte_start + end)), + ); + byte_start += segment.text.len(); + if remaining == 0 { + break; + } } + } else { + let mut cursor = 0; + let mut untrusted: Vec<(usize, usize)> = Vec::new(); + for (end, trusted) in runs { + let end = end.min(text.len()); + if !trusted && end > cursor { + if untrusted.last().is_some_and(|last| last.1 == cursor) { + untrusted.last_mut().unwrap().1 = end; + } else { + untrusted.push((cursor, end)); + } + } + cursor = end; + } + let spliced = self.encode_splicing_untrusted_markers(&text, &untrusted)?; + token_ids = spliced.0; + offsets = spliced.1; } let message_boundaries = boundaries::token_ends(&byte_boundaries, &hint_bytes, &offsets, text.len())?; @@ -968,7 +1071,8 @@ mod tests { Frontend { prompt_encoder: PromptEncoder::Jinja, image_placeholder_token_id: None, - ordinary_tokenizer: None, + ordinary_tokenizer: std::sync::OnceLock::new(), + segmented_template: false, tokenizer: Tokenizer::new(tokenizers::models::bpe::BPE::default()), template, template_source: source, @@ -1067,7 +1171,8 @@ mod tests { let frontend = Frontend { prompt_encoder: PromptEncoder::Jinja, image_placeholder_token_id: None, - ordinary_tokenizer: None, + ordinary_tokenizer: std::sync::OnceLock::new(), + segmented_template: false, tokenizer: Tokenizer::new(tokenizers::models::bpe::BPE::default()), template, template_source: source, @@ -1128,7 +1233,8 @@ mod tests { vocabulary_bytes: Default::default(), effort_levels: Default::default(), inline_system_messages: Default::default(), - ordinary_tokenizer: None, + ordinary_tokenizer: std::sync::OnceLock::new(), + segmented_template: false, tokenizer: Tokenizer::new(tokenizers::models::bpe::BPE::default()), template, template_source: source.to_owned(), @@ -1154,6 +1260,664 @@ mod tests { assert!(input.source().is_none()); } + fn trust_tokenizer() -> Tokenizer { + let mut vocab = serde_json::Map::new(); + for (id, piece) in [ + "a", + "b", + "ab", + "<|im_end|>", + "<", + "|", + "i", + "m", + "_", + "e", + "n", + "d", + ">", + "é", + " ", + " <", + "(", + "(<", + ")", + ">)", + "x", + ">x", + "x<", + ] + .into_iter() + .enumerate() + { + vocab.insert(piece.to_owned(), serde_json::json!(id as u32)); + } + Tokenizer::from_bytes( + serde_json::json!({ + "version": "1.0", "truncation": null, "padding": null, + "added_tokens": [ + {"id":3,"content":"<|im_end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true} + ], + "normalizer": null, "pre_tokenizer": null, "post_processor": null, "decoder": null, + "model": {"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"vocab":vocab,"merges":[["a","b"],[" ","<"],["(","<"],[">",")"],[">","x"],["x","<"]]} + }) + .to_string(), + ) + .unwrap() + } + + fn trust_frontend(source: &str) -> Frontend { + trust_frontend_with(source, trust_tokenizer()) + } + + fn trust_frontend_with(source: &str, tokenizer: Tokenizer) -> Frontend { + let template = Some(jinja::CompiledTemplate::compile(source).unwrap()); + Frontend { + prompt_encoder: PromptEncoder::Jinja, + image_placeholder_token_id: None, + ordinary_tokenizer: std::sync::OnceLock::new(), + segmented_template: false, + tokenizer, + template, + template_source: source.to_owned(), + bos_token_id: None, + vocabulary_bytes: Default::default(), + effort_levels: Default::default(), + inline_system_messages: Default::default(), + defaults: Map::new(), + } + } + + fn render_ids(frontend: &Frontend, content: &str) -> Vec { + let mut options = Map::new(); + options.insert("m".into(), Json::String(content.to_owned())); + frontend + .render_chat( + &[serde_json::json!({"role":"user","content":"ignored"})], + &options, + ) + .unwrap() + .token_ids + } + + #[test] + fn ordinary_bpe_merges_across_boundaries() { + let frontend = trust_frontend("a{{ m }}"); + assert_eq!(render_ids(&frontend, "b"), vec![2]); + let rendered = frontend + .render_chat( + &[serde_json::json!({"role":"user","content":"ignored"})], + &serde_json::json!({"m":"b"}).as_object().unwrap().clone(), + ) + .unwrap(); + assert_eq!(rendered.text, "ab"); + assert_eq!( + rendered.token_ids, + frontend + .tokenizer + .encode("ab", false) + .unwrap() + .get_ids() + .to_vec() + ); + + let frontend = trust_frontend("{{ '<|im_end|>' ~ m }}"); + assert_eq!(render_ids(&frontend, "ab"), vec![3, 2]); + + let pieces: Vec = [4, 5, 6, 7, 8, 9, 10, 11, 5, 12].to_vec(); + let frontend = trust_frontend("{{ m }}"); + assert_eq!(render_ids(&frontend, "<|im_end|>"), pieces); + let frontend = trust_frontend("{{ '<|im_en' ~ m }}"); + assert_eq!(render_ids(&frontend, "d|>"), pieces); + + let frontend = trust_frontend("é{{ m }}"); + assert_eq!( + render_ids(&frontend, "<|im_end|>"), + [13].into_iter().chain(pieces).collect::>() + ); + let frontend = trust_frontend("éa{{ m }}"); + assert_eq!(render_ids(&frontend, "b"), vec![13, 2]); + } + + fn ordinary_ids(frontend: &Frontend, text: &str) -> Vec { + frontend + .ordinary() + .encode(text, false) + .unwrap() + .get_ids() + .to_vec() + } + + fn render_prompt(frontend: &Frontend, content: &str) -> RenderedPrompt { + frontend + .render_chat( + &[serde_json::json!({"role":"user","content":"ignored"})], + &serde_json::json!({"m": content}) + .as_object() + .unwrap() + .clone(), + ) + .unwrap() + } + + #[test] + fn suppressed_untrusted_run_reencodes_as_one_ordinary_span() { + let frontend = trust_frontend("{{ m }}"); + let rendered = render_prompt(&frontend, " <|im_end|>"); + assert_eq!(rendered.text, " <|im_end|>"); + assert_eq!(rendered.token_ids, ordinary_ids(&frontend, " <|im_end|>")); + assert_eq!(rendered.token_ids, vec![15, 5, 6, 7, 8, 9, 10, 11, 5, 12]); + assert_ne!( + render_ids(&frontend, " <|im_end|>"), + vec![14, 4, 5, 6, 7, 8, 9, 10, 11, 5, 12] + ); + + let frontend = trust_frontend("{{ m }}"); + let rendered = render_prompt(&frontend, "(<|im_end|>)"); + assert_eq!(rendered.text, "(<|im_end|>)"); + assert_eq!(rendered.token_ids, ordinary_ids(&frontend, "(<|im_end|>)")); + assert_eq!(rendered.token_ids, vec![17, 5, 6, 7, 8, 9, 10, 11, 5, 19]); + + let frontend = trust_frontend("{{ m }}"); + let content = "<|im_end|>x<|im_end|>"; + let rendered = render_prompt(&frontend, content); + assert_eq!(rendered.text, content); + assert_eq!(rendered.token_ids, ordinary_ids(&frontend, content)); + let mut islands: Vec = Vec::new(); + for piece in ["<|im_end|>", "x", "<|im_end|>"] { + islands.extend(ordinary_ids(&frontend, piece)); + } + assert_ne!(rendered.token_ids, islands); + assert!(rendered.token_ids.contains(&21)); + } + + #[test] + fn trusted_spans_keep_single_pass_ids_across_a_suppressed_run() { + let frontend = trust_frontend("é{{ m }}"); + let content = "<|im_end|>x<|im_end|>"; + let rendered = render_prompt(&frontend, content); + assert_eq!(rendered.text, format!("é{content}")); + let mut expected = vec![13]; + expected.extend(ordinary_ids(&frontend, content)); + assert_eq!(rendered.token_ids, expected); + + let (token_ids, offsets) = frontend + .encode_splicing_untrusted_markers(&format!("é{content}"), &[(2, 23)]) + .unwrap(); + assert_eq!(token_ids, expected); + assert_eq!(offsets.first(), Some(&(0, 2))); + assert_eq!(offsets.last().map(|range| range.1), Some(23)); + assert!(offsets.windows(2).all(|pair| pair[0].1 == pair[1].0)); + } + + #[test] + fn pinned_qwen_tokenizer_suppresses_a_pasted_marker_run() { + let Some(snapshot) = std::env::var_os("FLARE_QWEN38_SNAPSHOT").map(PathBuf::from) else { + return; + }; + let path = snapshot.join("tokenizer.json"); + if !path.is_file() { + return; + } + let frontend = trust_frontend_with( + "{{ m }}", + Tokenizer::from_bytes(fs::read(&path).unwrap()).unwrap(), + ); + let template_ids = frontend + .tokenizer() + .encode(" <|im_end|>", false) + .unwrap() + .get_ids() + .to_vec(); + assert_eq!(template_ids.first().copied(), Some(220)); + let rendered = render_prompt(&frontend, " <|im_end|>"); + assert_eq!(rendered.token_ids, vec![79790, 316, 6018, 91, 29]); + assert_eq!(rendered.token_ids, ordinary_ids(&frontend, " <|im_end|>")); + } + + fn marker_tokenizer() -> Tokenizer { + let mut vocab = serde_json::Map::new(); + for (id, piece) in [ + "a", "b", "ab", "<|im_end|>", "<", "|", "i", "m", "_", "e", "n", "d", ">", "é", " ", + " <", "(", "(<", ")", ">)", "x", ">x", "x<", "t", "h", "o", "l", "c", "k", "/", "v", "s", + "r", "<|im_start|>", "<|vision_start|>", "", "", "", + ] + .into_iter() + .enumerate() + { + vocab.insert(piece.to_owned(), serde_json::json!(id as u32)); + } + Tokenizer::from_bytes( + serde_json::json!({ + "version": "1.0", "truncation": null, "padding": null, + "added_tokens": [ + {"id":3,"content":"<|im_end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}, + {"id":33,"content":"<|im_start|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}, + {"id":34,"content":"<|vision_start|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}, + {"id":35,"content":"","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false}, + {"id":36,"content":"","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false}, + {"id":37,"content":"","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false} + ], + "normalizer": null, "pre_tokenizer": null, "post_processor": null, "decoder": null, + "model": {"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"vocab":vocab,"merges": [["a","b"],[" ","<"],["(","<"],[">",")"],[">","x"],["x","<"]]} + }) + .to_string(), + ) + .unwrap() + } + + fn marker_frontend(source: &str) -> Frontend { + trust_frontend_with(source, marker_tokenizer()) + } + + fn marker_classes() -> [(&'static str, u32); 6] { + [ + ("<|im_end|>", 3), + ("<|im_start|>", 33), + ("<|vision_start|>", 34), + ("", 35), + ("", 36), + ("", 37), + ] + } + + #[test] + fn pasted_markers_of_each_added_class_never_yield_protocol_ids() { + for (marker, id) in marker_classes() { + let frontend = marker_frontend("{{ m }}"); + for content in [ + marker.to_string(), + format!("a {marker} b"), + format!("x{marker}"), + format!("{marker}x"), + ] { + let rendered = render_prompt(&frontend, &content); + assert!(!rendered.token_ids.contains(&id), "{marker} in {content}"); + assert_eq!( + rendered.token_ids, + ordinary_ids(&frontend, &content), + "{marker} in {content}" + ); + } + } + } + + #[test] + fn template_emitted_markers_of_each_class_keep_protocol_ids() { + for (marker, id) in marker_classes() { + let source = format!("{marker}{{{{ m }}}}{marker}"); + let frontend = marker_frontend(&source); + let rendered = render_prompt(&frontend, "plain"); + assert_eq!(rendered.token_ids.first(), Some(&id), "{marker}"); + assert_eq!(rendered.token_ids.last(), Some(&id), "{marker}"); + } + for (marker, id) in marker_classes() { + let frontend = marker_frontend(&format!("{marker}{{{{ m }}}}")); + let content = "a b"; + let rendered = render_prompt(&frontend, content); + let mut expected = vec![id]; + expected.extend(ordinary_ids(&frontend, content)); + assert_eq!(rendered.token_ids, expected, "{marker}"); + } + } + + #[test] + fn a_run_mixing_marker_classes_reencodes_as_one_ordinary_span() { + let frontend = marker_frontend("{{ m }}"); + let content = "a <|im_start|> b x d <|im_end|>"; + let rendered = render_prompt(&frontend, content); + assert_eq!(rendered.token_ids, ordinary_ids(&frontend, content)); + for (_, id) in marker_classes() { + assert!(!rendered.token_ids.contains(&id)); + } + } + + #[test] + fn markers_straddling_a_trust_boundary_are_pulled_into_the_run() { + for (marker, id) in marker_classes() { + let (prefix, suffix) = marker.split_at(marker.len() - 2); + let frontend = marker_frontend(&format!("{prefix}{{{{ m }}}}")); + let rendered = render_prompt(&frontend, suffix); + assert_eq!(rendered.text, marker); + assert!(!rendered.token_ids.contains(&id), "{marker}"); + assert_eq!(rendered.token_ids, ordinary_ids(&frontend, marker), "{marker}"); + + let frontend = marker_frontend(&format!("{{{{ m }}}}{suffix}")); + let rendered = render_prompt(&frontend, prefix); + assert_eq!(rendered.text, marker); + assert!(!rendered.token_ids.contains(&id), "{marker}"); + assert_eq!(rendered.token_ids, ordinary_ids(&frontend, marker), "{marker}"); + } + } + + #[test] + fn pasted_marker_between_trusted_markers_is_suppressed() { + for (marker, id) in marker_classes() { + let frontend = marker_frontend(&format!("{marker}{{{{ m }}}}{marker}")); + let rendered = render_prompt(&frontend, marker); + let mut expected = vec![id]; + expected.extend(ordinary_ids(&frontend, marker)); + expected.push(id); + assert_eq!(rendered.token_ids, expected, "{marker}"); + } + } + + #[test] + fn pinned_qwen_tokenizer_suppresses_every_added_token_class() { + let Some(snapshot) = std::env::var_os("FLARE_QWEN38_SNAPSHOT").map(PathBuf::from) else { + return; + }; + let path = snapshot.join("tokenizer.json"); + if !path.is_file() { + return; + } + let bytes = fs::read(&path).unwrap(); + let frontend = trust_frontend_with("{{ m }}", Tokenizer::from_bytes(&bytes).unwrap()); + let mut inventory: Vec<(u32, String)> = frontend + .tokenizer() + .get_added_tokens_decoder() + .iter() + .map(|(id, token)| (*id, token.content.clone())) + .collect(); + inventory.sort_unstable(); + assert!(inventory.len() > 1); + for (id, content) in &inventory { + let pasted = format!("x{content}x"); + let full = frontend.tokenizer().encode(pasted.as_str(), false).unwrap(); + assert!(full.get_ids().contains(id), "{content}"); + let rendered = render_prompt(&frontend, &pasted); + assert!(!rendered.token_ids.contains(id), "{content}"); + assert_eq!( + rendered.token_ids, + ordinary_ids(&frontend, &pasted), + "{content}" + ); + } + for marker in [ + "<|im_start|>", + "<|im_end|>", + "<|vision_start|>", + "<|image_pad|>", + "<|video_pad|>", + "", + "", + "", + "", + "", + "", + ] { + let frontend = trust_frontend_with( + &format!("{marker}{{{{ m }}}}"), + Tokenizer::from_bytes(&bytes).unwrap(), + ); + let expected = frontend + .tokenizer() + .encode(marker, false) + .unwrap() + .get_ids() + .to_vec(); + let rendered = render_prompt(&frontend, "plain"); + assert_eq!(rendered.token_ids.first(), expected.first(), "{marker}"); + } + let frontend = trust_frontend_with( + "<|im_start|>{{ m }}<|im_end|>", + Tokenizer::from_bytes(&bytes).unwrap(), + ); + let content = "a <|vision_start|> b c d"; + let rendered = render_prompt(&frontend, content); + let mut expected = frontend + .tokenizer() + .encode("<|im_start|>", false) + .unwrap() + .get_ids() + .to_vec(); + expected.extend(ordinary_ids(&frontend, content)); + expected.extend( + frontend + .tokenizer() + .encode("<|im_end|>", false) + .unwrap() + .get_ids(), + ); + assert_eq!(rendered.token_ids, expected); + } + + fn segmented_frontend(source: &str, tokenizer: Tokenizer) -> Frontend { + let template = Some(jinja::CompiledTemplate::compile(source).unwrap()); + Frontend { + prompt_encoder: PromptEncoder::Jinja, + image_placeholder_token_id: None, + ordinary_tokenizer: std::sync::OnceLock::new(), + segmented_template: true, + tokenizer, + template, + template_source: source.to_owned(), + bos_token_id: None, + vocabulary_bytes: Default::default(), + effort_levels: Default::default(), + inline_system_messages: Default::default(), + defaults: Map::new(), + } + } + + fn marker_tokenizer_with(specials: &[&str]) -> Tokenizer { + let mut vocab = serde_json::Map::new(); + let mut pieces: Vec = specials.iter().map(|text| (*text).to_owned()).collect(); + pieces.extend((b' '..=b'~').map(|byte| (byte as char).to_string())); + pieces.extend(["|", "▁", "\n"].map(String::from)); + for (id, piece) in pieces.iter().enumerate() { + vocab.insert(piece.clone(), serde_json::json!(id as u32)); + } + let added: Vec = specials + .iter() + .enumerate() + .map(|(id, content)| { + serde_json::json!({"id":id,"content":content,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}) + }) + .collect(); + Tokenizer::from_bytes( + serde_json::json!({ + "version": "1.0", "truncation": null, "padding": null, + "added_tokens": added, + "normalizer": null, "pre_tokenizer": null, "post_processor": null, "decoder": null, + "model": {"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"vocab":vocab,"merges":[]} + }) + .to_string(), + ) + .unwrap() + } + + fn added_counts(tokenizer: &Tokenizer, ids: &[u32]) -> std::collections::HashMap { + tokenizer + .get_added_tokens_decoder() + .keys() + .map(|id| (*id, ids.iter().filter(|&&seen| seen == *id).count())) + .filter(|(_, count)| *count > 0) + .collect() + } + + #[test] + fn trust_suppression_across_recipe_templates() { + let Some(directory) = std::env::var_os("FLARE_TEST_TEMPLATE_DIR") else { + return; + }; + let recipes: [(&str, &str); 6] = [ + ("qwen38", "chat_template"), + ("step5", "chat_template"), + ("glm53", "chat_template"), + ("mimo_v2", "chat_template"), + ("nemotron_h", "chat_template"), + ("gemma4", "chat_template.google"), + ]; + let mut options = Map::new(); + options.insert("tools".into(), Json::Null); + options.insert("documents".into(), Json::Null); + options.insert("bos_token".into(), Json::String("".into())); + options.insert("eos_token".into(), Json::String("".into())); + options.insert("add_generation_prompt".into(), Json::Bool(true)); + let trusted: std::collections::HashSet = options.keys().cloned().collect(); + for (recipe, asset) in recipes { + let source = crate::template_fixtures::template(recipe, asset); + let template = jinja::CompiledTemplate::compile(&source).unwrap(); + let tokenizer = std::fs::read( + std::path::Path::new(&directory) + .join(recipe) + .join("tokenizer.json"), + ) + .ok() + .map(|bytes| Tokenizer::from_bytes(&bytes).unwrap()); + let mut markers: Vec<(u32, String)> = Vec::new(); + if let Some(tokenizer) = &tokenizer { + let mut inventory: Vec<(u32, String)> = tokenizer + .get_added_tokens_decoder() + .iter() + .map(|(id, token)| (*id, token.content.clone())) + .collect(); + inventory.sort_unstable(); + markers.extend( + inventory + .into_iter() + .filter(|(_, content)| content.contains('<') || content.contains('|')) + .take(64), + ); + } + if markers.is_empty() { + markers.extend( + ["<|im_end|>", "<|endoftext|>", ""] + .map(|marker| (u32::MAX, marker.to_owned())), + ); + } + let mut contents = vec!["q7x ordinary z9w".to_owned()]; + for (_, marker) in &markers { + contents.push(format!("q7x x{marker}x z9w")); + } + let mut renders = Vec::new(); + for (index, content) in contents.iter().enumerate() { + let mut context = + serde_json::json!({"messages":[{"role":"user","content":content}]}); + context.as_object_mut().unwrap().extend(options.clone()); + let rendered = template + .render_with_trust(context.as_object().unwrap(), &[], &trusted) + .unwrap(); + if index > 0 { + let marker = &markers[index - 1].1; + let anchor = rendered.text.find("q7x").unwrap_or_else(|| { + panic!("{recipe}: anchor missing from {}", rendered.text) + }); + let offset = anchor + rendered.text[anchor..].find(marker).unwrap(); + assert_eq!( + covering_trust_for(&rendered, offset, offset + marker.len()), + Some(false), + "{recipe}: {marker}" + ); + } + renders.push(rendered); + } + assert_eq!( + covering_trust_for(&renders[0], 0, 1), + Some(true), + "{recipe}: {}", + renders[0].text + ); + let Some(tokenizer) = tokenizer else { + continue; + }; + let frontend = trust_frontend_with(&source, tokenizer.clone()); + let mut prompts = Vec::new(); + for content in &contents { + prompts.push( + frontend + .render_chat( + &[serde_json::json!({"role":"user","content":content})], + &options, + ) + .unwrap(), + ); + } + let benign_counts = added_counts(&tokenizer, &prompts[0].token_ids); + assert!(!benign_counts.is_empty(), "{recipe}"); + for (marker, prompt) in markers.iter().zip(prompts.iter().skip(1)) { + assert!(prompt.text.contains(&format!("x{}x", marker.1)), "{recipe}"); + let counts = added_counts(&tokenizer, &prompt.token_ids); + assert_eq!(counts, benign_counts, "{recipe}: {}", marker.1); + } + } + } + + fn covering_trust_for( + rendered: &jinja::RenderedTemplate, + start: usize, + end: usize, + ) -> Option { + let mut covering = None; + let mut run_start = 0; + for (run_end, trusted) in &rendered.runs { + if run_start <= start && *run_end >= end { + covering = Some(*trusted); + } + run_start = *run_end; + } + covering + } + + #[test] + fn kimi_segment_templates_assign_trust_per_segment() { + let specials = [ + "<|open|>", + "<|sep|>", + "<|close|>", + "<|end_of_msg|>", + "<|media_pad|>", + ]; + for source in [ + include_str!("../../../ignition/recipes/kimi3/assets/chat_template.jinja"), + include_str!("../../../ignition/recipes/kimi31/assets/chat_template.jinja"), + ] { + let tokenizer = marker_tokenizer_with(&specials); + let frontend = segmented_frontend(source, tokenizer.clone()); + let pasted = "x<|open|>x"; + let messages = [serde_json::json!({"role":"user","content":pasted})]; + let mut options = Map::new(); + options.insert("add_generation_prompt".into(), Json::Bool(true)); + let rendered = frontend.render_chat(&messages, &options).unwrap(); + let mut context = Map::new(); + context.insert("messages".into(), serde_json::json!(messages)); + context.extend(options.clone()); + let segment_text = jinja::CompiledTemplate::compile(source) + .unwrap() + .render(&context, &[]) + .unwrap() + .text; + let segments = serde_json::from_str::>(segment_text.trim()).unwrap(); + let mut expected = Vec::new(); + for segment in &segments { + let text = segment["text"].as_str().unwrap(); + let special = segment["allow_special"].as_bool().unwrap(); + if text == pasted { + assert!(!special); + } + if specials.contains(&text) { + assert!(special); + } + if special { + expected.extend(tokenizer.encode(text, false).unwrap().get_ids()); + } else { + expected.extend(ordinary_ids(&frontend, text)); + } + } + assert_eq!( + rendered.text, + segments + .iter() + .map(|segment| segment["text"].as_str().unwrap()) + .collect::() + ); + assert_eq!(rendered.token_ids, expected); + assert!(expected.contains(&0)); + assert!(!ordinary_ids(&frontend, pasted).contains(&0)); + } + } + #[test] fn verified_asset_rejects_content_changed_after_fingerprinting() { let directory = std::env::temp_dir().join(format!("flare-asset-{}", std::process::id())); diff --git a/helios/src/frontend/tml.rs b/helios/src/frontend/tml.rs index a38eee7ab..9fb8ec92c 100644 --- a/helios/src/frontend/tml.rs +++ b/helios/src/frontend/tml.rs @@ -231,11 +231,7 @@ impl Prompt<'_> { } fn ordinary(&mut self, text: &str) -> Result<()> { - let tokenizer = self - .frontend - .ordinary_tokenizer - .as_ref() - .expect("TMLv0 ordinary tokenizer is initialized"); + let tokenizer = self.frontend.ordinary(); let encoding = tokenizer.encode(text, false).map_err(|error| { Error::frontend(FrontendError::Tokenizer(Failure::boxed(error.to_string(), error))) })?; diff --git a/helios/tests/deepseek_frontend.rs b/helios/tests/deepseek_frontend.rs index a32396535..782f4ea33 100644 --- a/helios/tests/deepseek_frontend.rs +++ b/helios/tests/deepseek_frontend.rs @@ -56,6 +56,127 @@ fn numeric_effort_matches_python_reference_bytes_and_checkpoint_token_ids() { ); } +#[test] +fn deepseek_no_template_pasted_markers_stay_ordinary() { + let specials = [ + "<|begin▁of▁sentence|>", + "<|end▁of▁sentence|>", + "", + "", + "<|System|>", + "<|User|>", + "<|Assistant|>", + "<|latest_reminder|>", + "|DSML|", + "<|image|>", + "<|ref|>", + "<|/ref|>", + "<|box|>", + "<|/box|>", + "<|point|>", + "<|/point|>", + "<|polygon|>", + "<|/polygon|>", + ]; + let mut vocab = serde_json::Map::new(); + let mut pieces: Vec = specials.iter().map(|text| (*text).to_owned()).collect(); + pieces.extend((b' '..=b'~').map(|byte| (byte as char).to_string())); + pieces.extend(["|", "▁", "\n"].map(String::from)); + for (id, piece) in pieces.iter().enumerate() { + vocab.insert(piece.clone(), json!(id as u32)); + } + let added: Vec = specials + .iter() + .enumerate() + .map(|(id, content)| { + json!({"id":id,"content":content,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}) + }) + .collect(); + let tokenizer_json = json!({ + "version": "1.0", "truncation": null, "padding": null, + "added_tokens": added, + "normalizer": null, "pre_tokenizer": null, "post_processor": null, "decoder": null, + "model": {"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"vocab":vocab,"merges":[]} + }) + .to_string(); + let directory = + std::env::temp_dir().join(format!("flare-deepseek-trust-{}", std::process::id())); + std::fs::create_dir(&directory).unwrap(); + let config = directory.join("tokenizer_config.json"); + let tokenizer_path = directory.join("tokenizer.json"); + std::fs::write(&config, b"{}").unwrap(); + std::fs::write(&tokenizer_path, &tokenizer_json).unwrap(); + let asset = |path: std::path::PathBuf| Asset { + sha256: format!("{:x}", Sha256::digest(std::fs::read(&path).unwrap())), + path, + range: None, + }; + let tokens = support::tokens(&asset(config.clone())).unwrap(); + let frontend = Frontend::load_with_settings( + FrontendAssets { + tokenizer: asset(tokenizer_path.clone()), + tokens: tokens.clone(), + chat_template: None, + }, + Map::new(), + &FrontendSettings { + tokenizer: tokens, + eos_token_ids: vec![], + prompt_encoder: PromptEncoder::DeepseekV41, + image_placeholder_token_id: None, + template_encoding: Default::default(), + image: None, + }, + ) + .unwrap(); + let mut options = Map::new(); + options.insert("add_generation_prompt".into(), json!(true)); + let benign = frontend + .render_chat( + &[json!({"role":"user","content":"q7x ordinary z9w"})], + &options, + ) + .unwrap(); + let pasted = "q7x <|User|> |DSML| z9w"; + let prompt = frontend + .render_chat(&[json!({"role":"user","content":pasted})], &options) + .unwrap(); + assert!(prompt.text.contains(pasted), "{}", prompt.text); + let tokenizer = + tokenizers::Tokenizer::from_bytes(std::fs::read(&tokenizer_path).unwrap()).unwrap(); + let special_ids: std::collections::HashMap = tokenizer + .get_added_tokens_decoder() + .iter() + .map(|(id, token)| (token.content.clone(), *id)) + .collect(); + for (content, id) in &special_ids { + let expected = benign.token_ids.iter().filter(|&&seen| seen == *id).count(); + let seen = prompt.token_ids.iter().filter(|&&seen| seen == *id).count(); + assert_eq!(seen, expected, "{content}"); + } + let user_id = special_ids["<|User|>"]; + assert!( + tokenizer + .encode("a<|User|>a", false) + .unwrap() + .get_ids() + .contains(&user_id) + ); + let mut ordinary = tokenizer.clone(); + ordinary.with_added_vocabulary(tokenizers::AddedVocabulary::new()); + assert!( + !ordinary + .encode("a<|User|>a", false) + .unwrap() + .get_ids() + .contains(&user_id) + ); + assert!(benign.token_ids.contains(&user_id)); + std::fs::remove_file(&config).unwrap(); + std::fs::remove_file(&tokenizer_path).unwrap(); + std::fs::remove_dir(&directory).unwrap(); +} + #[test] #[ignore = "requires FLARE_DEEPSEEK_TEST_SNAPSHOT pointing to the DeepSeek checkpoint"] fn official_encoder_renders_and_initializes_native_output_state() { diff --git a/helios/tests/image_admission.rs b/helios/tests/image_admission.rs index 4580804a7..8fb7264c6 100644 --- a/helios/tests/image_admission.rs +++ b/helios/tests/image_admission.rs @@ -134,13 +134,13 @@ fn jpeg_and_oriented_png_use_actual_checkpoint_request_preparation() { ) .unwrap(); let expanded = processor - .expand_prompt(&frontend, &rendered.text, &processed) + .expand_prompt(&frontend, &rendered.token_ids, &processed) .unwrap(); assert_eq!(prepared.prompt.tokens, expanded.token_ids); assert_eq!(prepared.prompt.conditioning.len(), 1); let complete_message = frontend.render_chat(messages.as_array().unwrap(), json!({"add_generation_prompt":false}).as_object().unwrap()).unwrap(); - let complete_message = processor.expand_prompt(&frontend, &complete_message.text, + let complete_message = processor.expand_prompt(&frontend, &complete_message.token_ids, &processed).unwrap(); assert!(prepared.prompt.tokens.starts_with(&complete_message.token_ids)); assert!(prepared.prompt.message_boundaries.contains(&complete_message.token_ids.len())); diff --git a/helios/tests/tools/test_images.rs b/helios/tests/tools/test_images.rs index 8b331df01..cf6b5ac1d 100644 --- a/helios/tests/tools/test_images.rs +++ b/helios/tests/tools/test_images.rs @@ -141,9 +141,8 @@ fn run() -> Result<(), Box> { case["messages"].as_array().unwrap(), json!({"add_generation_prompt": true}).as_object().unwrap(), )?; - let prompt = processor.expand_prompt(&frontend, &rendered.text, &processed)?; + let prompt = processor.expand_prompt(&frontend, &rendered.token_ids, &processed)?; if rendered.text != case["rendered"] - || prompt.text != case["expanded"] || json!(prompt.token_ids) != case["token_ids"] || json!(prompt.mm_token_type_ids) != case["mm_token_type_ids"] || json!(prompt.rope_delta) != case["rope_delta"] diff --git a/tests/frontend/prepare_templates.py b/tests/frontend/prepare_templates.py index 954f01823..b5286f992 100644 --- a/tests/frontend/prepare_templates.py +++ b/tests/frontend/prepare_templates.py @@ -43,6 +43,12 @@ def main(): (output / f"{name}.source.jinja").write_bytes(original) (output / f"{name}.jinja").write_bytes(asset["text"].encode("utf-8")) print(output / f"{name}.jinja") + try: + path = hf_hub_download(source["repository"], "tokenizer.json", + revision=source["revision"], cache_dir=args.hub_cache) + (output / "tokenizer.json").write_bytes(Path(path).read_bytes()) + except Exception as error: + print(f"tokenizer.json unavailable for {recipe}: {error}") if __name__ == "__main__":