diff --git a/helios/src/frontend/deepseek.rs b/helios/src/frontend/deepseek.rs index 37831f0ea..8ac376de8 100644 --- a/helios/src/frontend/deepseek.rs +++ b/helios/src/frontend/deepseek.rs @@ -437,10 +437,7 @@ pub(super) fn render( .collect::>>()?; let mut token_ids = Vec::new(); let mut offsets = Vec::new(); - let plain = frontend - .ordinary_tokenizer - .as_ref() - .expect("DeepSeek plain tokenizer is initialized"); + let plain = frontend.ordinary(); let mut start = 0; let mut image_count = 0; for range in &rendered.special_ranges { diff --git a/helios/src/frontend/images/layouts/bracketed.rs b/helios/src/frontend/images/layouts/bracketed.rs index 4c7c61174..2c208037e 100644 --- a/helios/src/frontend/images/layouts/bracketed.rs +++ b/helios/src/frontend/images/layouts/bracketed.rs @@ -31,7 +31,7 @@ pub(crate) fn prepare_input( interpolation_weights: vec![], cu_seqlens: vec![0], }; - processor.expand_prompt(frontend, &rendered.text, &empty)?; + processor.expand_prompt(frontend, &rendered.token_ids, &empty)?; } return Ok(PreparedInput { tokens: rendered.token_ids.clone(), @@ -46,7 +46,7 @@ pub(crate) fn prepare_input( } let processor = processor.ok_or("image processing is unavailable")?; let processed = processor.describe_inputs(images)?; - let expanded = processor.expand_prompt(frontend, &rendered.text, &processed)?; + let expanded = processor.expand_prompt(frontend, &rendered.token_ids, &processed)?; let mut conditioning = Vec::new(); let mut image = 0; let mut feature = 0; diff --git a/helios/src/frontend/images/layouts/qwen.rs b/helios/src/frontend/images/layouts/qwen.rs index 1621958b2..eaf91e8ea 100644 --- a/helios/src/frontend/images/layouts/qwen.rs +++ b/helios/src/frontend/images/layouts/qwen.rs @@ -46,7 +46,7 @@ pub(crate) fn prepare_input( } else { let processor = processor.ok_or("image processing is unavailable")?; let processed = processor.describe_inputs(images)?; - let expanded = processor.expand_prompt(frontend, &rendered.text, &processed)?; + let expanded = processor.expand_prompt(frontend, &rendered.token_ids, &processed)?; let positions = (0..expanded.token_ids.len()) .map(|position| { [ diff --git a/helios/src/frontend/images/mod.rs b/helios/src/frontend/images/mod.rs index d25fb8c26..600c5998a 100644 --- a/helios/src/frontend/images/mod.rs +++ b/helios/src/frontend/images/mod.rs @@ -213,7 +213,6 @@ impl ProcessedImages { } pub struct ImagePrompt { - pub text: String, pub token_ids: Vec, pub mm_token_type_ids: Vec, /// Text, temporal, height, and width position rows, each of length token_ids.len(). @@ -718,12 +717,16 @@ impl ImageProcessor { pub fn expand_prompt( &self, frontend: &Frontend, - text: &str, + prompt_token_ids: &[u32], images: &ProcessedImages, ) -> Result { match &self.profile { - Profile::Qwen(profile) => profile.expand_prompt(frontend, text, images), - Profile::PatchMerge(profile) => profile.expand_prompt(frontend, text, images), + Profile::Qwen(profile) => { + profile.expand_prompt(frontend, prompt_token_ids, images) + } + Profile::PatchMerge(profile) => { + profile.expand_prompt(frontend, prompt_token_ids, images) + } Profile::DeepSeek4 | Profile::DeepseekRowMajor(_) => Err("DeepSeek image spans require its model contract".into()), Profile::MiMo(_) => Err("MiMo image expansion requires its model contract".into()), Profile::Moonvit => Err("MoonViT image expansion requires its model contract".into()), diff --git a/helios/src/frontend/images/patch_merge.rs b/helios/src/frontend/images/patch_merge.rs index acfe757c9..9ef10b0b0 100644 --- a/helios/src/frontend/images/patch_merge.rs +++ b/helios/src/frontend/images/patch_merge.rs @@ -59,7 +59,7 @@ impl Processor { pub(super) fn expand_prompt( &self, frontend: &Frontend, - text: &str, + prompt_token_ids: &[u32], images: &ProcessedImages, ) -> Result { let image_ids = frontend.encode(&self.image_token, false)?; @@ -71,26 +71,34 @@ impl Processor { return Err("image envelope marker must encode one token".into()); } } - if text.matches(&self.image_token).count() != images.geometry.len() { + let placeholder_count = prompt_token_ids + .iter() + .filter(|id| **id == *image_id) + .count(); + if placeholder_count != images.geometry.len() { return Err("image placeholder count does not match input images".into()); } - let mut parts = text.split(&self.image_token); - let mut expanded = parts.next().unwrap_or_default().to_owned(); - for (part, geometry) in parts.zip(&images.geometry) { - expanded.push_str(&self.start_token); - for _ in 0..geometry.tokens { - expanded.push_str(&self.image_token); + let start_id = frontend.encode(&self.start_token, false)?[0]; + let end_id = frontend.encode(&self.end_token, false)?[0]; + let mut expanded_ids = Vec::new(); + let mut image_index = 0usize; + for &id in prompt_token_ids { + if id == *image_id { + let tokens = images.geometry[image_index].tokens; + expanded_ids.push(start_id); + expanded_ids.extend(std::iter::repeat(*image_id).take(tokens)); + expanded_ids.push(end_id); + image_index += 1; + } else { + expanded_ids.push(id); } - expanded.push_str(&self.end_token); - expanded.push_str(part); } - let token_ids = frontend.encode(&expanded, false)?; + let token_ids = expanded_ids; let mm_token_type_ids = token_ids .iter() .map(|id| i32::from(*id == *image_id)) .collect(); Ok(ImagePrompt { - text: expanded, position_ids: std::array::from_fn(|_| (0..token_ids.len() as i64).collect()), token_ids, mm_token_type_ids, diff --git a/helios/src/frontend/images/qwen.rs b/helios/src/frontend/images/qwen.rs index a563edf9f..17c0f6289 100644 --- a/helios/src/frontend/images/qwen.rs +++ b/helios/src/frontend/images/qwen.rs @@ -78,7 +78,7 @@ impl QwenProcessor { pub fn expand_prompt( &self, frontend: &Frontend, - text: &str, + prompt_token_ids: &[u32], images: &ProcessedImages, ) -> Result { if frontend.encode(IMAGE_TOKEN, false)? != [self.image_token_id] { @@ -86,26 +86,39 @@ impl QwenProcessor { "image token ID does not match tokenizer", )))); } - if text.matches(IMAGE_TOKEN).count() != images.geometry.len() { + let placeholder_count = prompt_token_ids + .iter() + .filter(|id| **id == self.image_token_id) + .count(); + if placeholder_count != images.geometry.len() { return Err(Error::frontend(FrontendError::Input(Failure::new( "image placeholder count does not match prepared images", )))); } - let mut expanded = String::with_capacity(text.len()); - let mut parts = text.split(IMAGE_TOKEN); - expanded.push_str(parts.next().unwrap_or_default()); - for (part, geometry) in parts.zip(&images.geometry) { - for _ in 0..geometry.tokens { - expanded.push_str(IMAGE_TOKEN); - } - expanded.push_str(part); - } - let token_ids = frontend.encode(&expanded, false)?; - if token_ids.contains(&self.video_token_id) { + if prompt_token_ids.contains(&self.video_token_id) { return Err(Error::frontend(FrontendError::Input(Failure::new( "video inputs are not supported by the image processor", )))); } + let mut expanded_ids = Vec::with_capacity( + prompt_token_ids.len() + + images + .geometry + .iter() + .map(|geometry| geometry.tokens.saturating_sub(1)) + .sum::(), + ); + let mut image_index = 0usize; + for &id in prompt_token_ids { + if id == self.image_token_id { + let tokens = images.geometry[image_index].tokens; + expanded_ids.extend(std::iter::repeat(self.image_token_id).take(tokens)); + image_index += 1; + } else { + expanded_ids.push(id); + } + } + let token_ids = expanded_ids; let mm_token_type_ids: Vec = token_ids .iter() .map(|id| i32::from(*id == self.image_token_id)) @@ -159,7 +172,6 @@ impl QwenProcessor { )))); } Ok(ImagePrompt { - text: expanded, rope_delta: current - token_ids.len() as i64, token_ids, mm_token_type_ids, diff --git a/helios/src/frontend/jinja/mod.rs b/helios/src/frontend/jinja/mod.rs index 6f53e8b2f..22744ab86 100644 --- a/helios/src/frontend/jinja/mod.rs +++ b/helios/src/frontend/jinja/mod.rs @@ -45,6 +45,7 @@ pub enum Mark { pub struct RenderedTemplate { pub text: String, pub marks: Vec, + pub runs: Vec<(usize, bool)>, } pub struct CompiledTemplate { @@ -69,7 +70,17 @@ impl CompiledTemplate { context: &Map, hints: &[(usize, usize)], ) -> Result { - vm::render(self, context, hints) + use std::collections::HashSet; + self.render_with_trust(context, hints, &HashSet::new()) + } + + pub fn render_with_trust( + &self, + context: &Map, + hints: &[(usize, usize)], + trusted: &std::collections::HashSet, + ) -> Result { + vm::render(self, context, hints, trusted) } pub fn annotation_spans(&self) -> &[std::ops::Range] { diff --git a/helios/src/frontend/jinja/tests.rs b/helios/src/frontend/jinja/tests.rs index 7e2405820..0f2676efb 100644 --- a/helios/src/frontend/jinja/tests.rs +++ b/helios/src/frontend/jinja/tests.rs @@ -55,6 +55,40 @@ fn compare(source: &str, context: Json) { } } +#[test] +fn request_values_record_untrusted_runs_and_defaults_stay_trusted() { + let template = CompiledTemplate::compile("A{{ x }}B{{ 'C' }}{{ y }}").unwrap(); + let context = json!({"x":"<|pad|>", "y":"plain"}); + let mut trusted = std::collections::HashSet::new(); + trusted.insert("y".to_owned()); + let rendered = template + .render_with_trust(context.as_object().unwrap(), &[], &trusted) + .unwrap(); + assert_eq!(rendered.text, "A<|pad|>BCplain"); + assert_eq!( + rendered.runs, + vec![(1, true), (8, false), (15, true)] + ); + let rendered = template.render(context.as_object().unwrap(), &[]).unwrap(); + assert_eq!( + rendered.runs, + vec![(1, true), (8, false), (10, true), (15, false)] + ); +} + +#[test] +fn captures_preserve_trust_runs_and_concatenation_widens() { + let template = + CompiledTemplate::compile("{% set body %}<|s|>{{ m }}{% endset %}{{ body }}{{ a ~ m }}") + .unwrap(); + let context = json!({"m":"x", "a":"<|a|>"}); + let rendered = template + .render_with_trust(context.as_object().unwrap(), &[], &std::collections::HashSet::new()) + .unwrap(); + assert_eq!(rendered.text, "<|s|>x<|a|>x"); + assert_eq!(rendered.runs, vec![(5, true), (12, false)]); +} + #[test] fn expressions_and_python_methods_match_minijinja() { let context = json!({"s":" café 👋 ","a":[1,2,3,4],"m":{"z":true,"a":null},"d":{"b":2,"B":3,"a":1},"f":[1e-6,-0.0,1e16,2.5,3.0],"none":null}); @@ -171,7 +205,8 @@ fn marks_are_zero_output_and_cannot_escape_captures() { Mark::Byte(3), Mark::Byte(7), Mark::Position(3) - ] + ], + runs: vec![(7, true)], } ); assert_eq!(template.annotation_spans().len(), 3); @@ -476,3 +511,283 @@ fn json_object_members_preserves_literals_order_and_duplicate_keys() { } assert!(CompiledTemplate::compile("{{ json_object_members([]) }}").unwrap().render(&Map::new(), &[]).is_err()); } + + +#[test] +fn diag_micro_paths() { + for (source, expected_untrusted) in [ + ("{% macro r(c) %}{{ c }}{% endmacro %}{{ r(m) }}", true), + ("{% for message in messages %}{{ message.content }}{% endfor %}", true), + ("{% set body %}{{ m }}{% endset %}{{ body|trim }}", true), + ] { + let template = CompiledTemplate::compile(source).unwrap(); + let context = json!({"m":"<|x|>", "messages":[{"content":"<|y|>"}]}); + let rendered = template + .render_with_trust(context.as_object().unwrap(), &[], &std::collections::HashSet::new()) + .unwrap(); + let untrusted = rendered.runs.iter().any(|(_, t)| !t); + assert_eq!(untrusted, expected_untrusted, "source: {source}"); + } +} + +const ENVELOPE_TEMPLATE: &str = r#" +{%- macro render_content(content) -%} +{%- if content is string -%} +{{- content -}} +{%- elif content is iterable and content is not mapping -%} +{%- for item in content -%} +{%- if item.type == 'image' -%} +{{- '<|vision_start|><|image_pad|><|vision_end|>' -}} +{%- else -%} +{{- item.text -}} +{%- endif -%} +{%- endfor -%} +{%- endif -%} +{%- endmacro -%} +{%- for message in messages -%} +{{- '<|im_start|>' -}}{{- message.role -}}{{- '\n' -}}{{ render_content(message.content) }}{{- '<|im_end|>' -}}{{- '\n' -}} +{%- endfor -%} +{%- if add_generation_prompt -%} +{{- '<|im_start|>assistant\n' -}} +{%- endif -%} +"#; + +fn covering_trust(rendered: &RenderedTemplate, start: usize, end: usize) -> Option { + let mut covering = None; + let mut run_start = 0; + for (run_end, trusted) in &rendered.runs { + if run_start <= start && *run_end >= end { + covering = Some(*trusted); + } + run_start = *run_end; + } + covering +} + +#[test] +fn pasted_placeholders_render_as_untrusted_spans() { + let template = CompiledTemplate::compile(ENVELOPE_TEMPLATE).unwrap(); + let pasted = "\"<|vision_start|><|image_pad|><|vision_end|>\""; + let context = json!({ + "messages": [{ + "role": "user", + "content": [ + {"type": "image", "image": "placeholder"}, + {"type": "text", "text": format!("Between the quotes: {pasted} ok?")} + ] + }], + "add_generation_prompt": true + }); + let rendered = template + .render_with_trust(context.as_object().unwrap(), &[], &std::collections::HashSet::new()) + .unwrap(); + let envelope = "<|vision_start|><|image_pad|><|vision_end|>"; + let offsets: Vec = rendered + .text + .match_indices(envelope) + .map(|(offset, _)| offset) + .collect(); + assert_eq!(offsets.len(), 2, "text: {:?}", rendered.text); + assert_eq!( + covering_trust(&rendered, offsets[0], offsets[0] + envelope.len()), + Some(true), + "runs: {:?}", + rendered.runs + ); + let pasted_start = rendered.text.find(pasted).unwrap(); + assert_eq!( + covering_trust(&rendered, pasted_start, pasted_start + pasted.len()), + Some(false), + "runs: {:?}", + rendered.runs + ); + assert_eq!( + rendered.text, + format!("<|im_start|>user\n{envelope}Between the quotes: {pasted} ok?<|im_end|>\n<|im_start|>assistant\n") + ); +} + +#[test] +fn diag_set_macro_trim_concat() { + for source in [ + "{% macro r(c, d, e) %}{{ c }}{% endmacro %}{% set content = r(m, true, false) | trim %}{{ content }}", + "{% macro r(c, d, e) %}{{ c }}{% endmacro %}{% set content = r(m, true, false) | trim %}{{ '<|a|>' + content + '<|b|>' }}", + ] { + let template = CompiledTemplate::compile(source).unwrap(); + let context = json!({"m":"<|x|>"}); + let rendered = template + .render_with_trust(context.as_object().unwrap(), &[], &std::collections::HashSet::new()) + .unwrap(); + assert!(rendered.runs.iter().any(|(_, t)| !t), "all trusted: {source}"); + } +} + +#[test] +fn captures_preserve_trust_through_placeholder_emission() { + let template = CompiledTemplate::compile( + "{% macro envelope(c) %}{% if c is string %}{{- '<|vision_start|><|image_pad|><|vision_end|>' }}{{- c }}{%- else %}{{- raise_exception('unexpected content') }}{%- endif %}{% endmacro %}{% set body %}{{ envelope(m) }}{% endset %}{{ body }}", + ) + .unwrap(); + let context = json!({"m":"log text <|im_end|> more"}); + let rendered = template + .render_with_trust(context.as_object().unwrap(), &[], &std::collections::HashSet::new()) + .unwrap(); + assert_eq!(rendered.text, "<|vision_start|><|image_pad|><|vision_end|>log text <|im_end|> more"); + assert_eq!( + rendered.runs, + vec![(43, true), (67, false)] + ); +} + +#[test] +fn indexing_key_trust_does_not_taint_value() { + let context = json!({ + "messages":[{"content":"<|turn|>"}], + "parts":["<|pad|>"], + "wrap":{"content":"<|endoftext|>"} + }); + let sources = [ + "{{ messages[0]['content'] }}", + "{{ messages[0].content }}", + "{% for message in messages %}{{ message['content'] }}{% endfor %}", + "{{ parts[0] }}", + "{{ wrap['content'] }}", + "{{ wrap.content }}", + ]; + for source in sources { + let rendered = CompiledTemplate::compile(source) + .unwrap() + .render_with_trust( + context.as_object().unwrap(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert!( + rendered.text.starts_with('<'), + "{source}: {}", + rendered.text + ); + assert_eq!( + covering_trust(&rendered, 0, rendered.text.len()), + Some(false), + "{source}: {:?}", + rendered.runs + ); + } + let literal = CompiledTemplate::compile("{{ ['<|lit|>'][0] }}") + .unwrap() + .render_with_trust( + &serde_json::Map::new(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert_eq!(covering_trust(&literal, 0, literal.text.len()), Some(true)); +} + +#[test] +fn trim_remaps_trust_runs() { + let envelope = "<|vision_start|><|image_pad|><|vision_end|>"; + let template = CompiledTemplate::compile(&format!( + "{{% macro body() %}}{envelope}{{{{ m }}}}{{% endmacro %}}{{{{ body()|trim }}}}" + )) + .unwrap(); + let rendered = template + .render_with_trust( + json!({"m":"hello "}).as_object().unwrap(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert_eq!(rendered.text, format!("{envelope}hello")); + assert_eq!( + rendered.runs, + vec![(envelope.len(), true), (rendered.text.len(), false)] + ); + let trailing = CompiledTemplate::compile(&format!( + "{{% set body %}}{{{{ m }}}}{envelope} {{% endset %}}{{{{ body|trim }}}}" + )) + .unwrap(); + let rendered = trailing + .render_with_trust( + json!({"m":"hi "}).as_object().unwrap(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert_eq!(rendered.text, format!("hi {envelope}")); + assert_eq!(rendered.runs, vec![(3, false), (rendered.text.len(), true)]); + let method = + CompiledTemplate::compile("{% set body %}{{ m }}<|a|> {% endset %}{{ body.rstrip() }}") + .unwrap(); + let rendered = method + .render_with_trust( + json!({"m":"x"}).as_object().unwrap(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert_eq!(rendered.text, "x<|a|>"); + assert_eq!(rendered.runs, vec![(1, false), (rendered.text.len(), true)]); +} + +#[test] +fn namespace_provenance_recursive() { + let cases = [ + "{% set ns = namespace(body=m) %}{{ ns.body }}", + "{% set ns = namespace(body=m) %}{{ ns['body'] }}", + "{% set ns = namespace() %}{% set ns.body = m %}{{ ns.body }}", + "{% set d = dict(body=m) %}{{ d.body }}", + "{% set ns = namespace(body=m) %}{{ ns.body[0:] }}", + "{% set ns = namespace(body=m) %}{{ ns.body ~ 'tail' }}", + "{% set ns = namespace(body=m) %}{% set ns.body = ns.body ~ 'tail' %}{{ ns.body }}", + ]; + for source in cases { + let rendered = CompiledTemplate::compile(source) + .unwrap() + .render_with_trust( + json!({"m":"<|image_pad|>"}).as_object().unwrap(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert_eq!( + covering_trust( + &rendered, + rendered.text.find("<|image_pad|>").unwrap(), + rendered.text.find("<|image_pad|>").unwrap() + "<|image_pad|>".len() + ), + Some(false), + "{source}: {:?}", + rendered.runs + ); + } + let upper = + CompiledTemplate::compile("{% set ns = namespace(body=m) %}{{ ns.body|upper }}").unwrap(); + let rendered = upper + .render_with_trust( + json!({"m":"<|image_pad|>"}).as_object().unwrap(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert_eq!(rendered.text, "<|IMAGE_PAD|>"); + assert_eq!(rendered.runs, vec![(rendered.text.len(), false)]); + + let iterate = CompiledTemplate::compile( + "{% set ns = namespace(body=messages) %}{% for message in ns.body %}{{ message.content }}{% endfor %}", + ) + .unwrap(); + let rendered = iterate + .render_with_trust( + json!({"messages":[{"content":"<|x|>"},{"content":"<|y|>"}]}) + .as_object() + .unwrap(), + &[], + &std::collections::HashSet::new(), + ) + .unwrap(); + assert_eq!(rendered.text, "<|x|><|y|>"); + assert_eq!(rendered.runs, vec![(rendered.text.len(), false)]); +} diff --git a/helios/src/frontend/jinja/value.rs b/helios/src/frontend/jinja/value.rs index 600f9f148..acc7d1f53 100644 --- a/helios/src/frontend/jinja/value.rs +++ b/helios/src/frontend/jinja/value.rs @@ -22,7 +22,7 @@ pub(super) enum Value<'a> { UInt(u64), Float(f64), Str(&'a str), - String(Rc<(String, Vec)>), + String(Rc<(String, Vec, Vec<(usize, bool)>)>), Array(&'a [Json]), Object(&'a serde_json::Map), List(Rc>>), @@ -37,6 +37,7 @@ pub(super) enum Value<'a> { pub(super) struct LoopValue<'a> { pub items: Vec>, pub index: Cell, + pub trust: bool, } impl<'a> Value<'a> { @@ -76,7 +77,70 @@ impl<'a> Value<'a> { Self::marked(value, Vec::new()) } pub fn marked(value: String, hints: Vec) -> Self { - Self::String(Rc::new((value, hints))) + Self::String(Rc::new((value, hints, Vec::new()))) + } + pub fn marked_runs(value: String, hints: Vec, runs: Vec<(usize, bool)>) -> Self { + Self::String(Rc::new((value, hints, runs))) + } + pub fn carries_untrusted(&self) -> bool { + match self { + Self::String(s) => s.2.iter().any(|(_, trusted)| !trusted), + Self::List(items) => items.iter().any(Self::carries_untrusted), + Self::Map(pairs) => pairs.iter().any(|(_, value)| value.carries_untrusted()), + Self::Namespace(pairs) => pairs + .borrow() + .iter() + .any(|(_, value)| value.carries_untrusted()), + _ => false, + } + } + pub fn baked_untrusted(value: Self) -> Self { + match value { + Self::Str(s) => { + let runs = vec![(s.len(), false)]; + Self::marked_runs(s.to_owned(), Vec::new(), runs) + } + Self::String(s) => { + if s.2.is_empty() { + Self::String(Rc::new(( + s.0.clone(), + s.1.clone(), + vec![(s.0.len(), false)], + ))) + } else { + Self::String(s) + } + } + Self::Array(items) => Self::list( + items + .iter() + .map(|item| Self::baked_untrusted(Self::json(item))) + .collect(), + ), + Self::Object(members) => Self::Map(Rc::new( + members + .iter() + .map(|(key, item)| (key.clone(), Self::baked_untrusted(Self::json(item)))) + .collect(), + )), + Self::List(items) => { + Self::list(items.iter().cloned().map(Self::baked_untrusted).collect()) + } + Self::Map(pairs) => Self::Map(Rc::new( + pairs + .iter() + .map(|(key, item)| (key.clone(), Self::baked_untrusted(item.clone()))) + .collect(), + )), + Self::Namespace(pairs) => Self::Namespace(Rc::new(RefCell::new( + pairs + .borrow() + .iter() + .map(|(key, item)| (key.clone(), Self::baked_untrusted(item.clone()))) + .collect(), + ))), + other => other, + } } pub fn hints(&self) -> &[usize] { match self { @@ -89,6 +153,79 @@ impl<'a> Value<'a> { hints.extend(self.hints().iter().map(|end| start + end)); self.write(output); } + pub fn append_annotated( + &self, + output: &mut String, + hints: &mut Vec, + runs: &mut Vec<(usize, bool)>, + trust: bool, + ) { + let start = output.len(); + hints.extend(self.hints().iter().map(|end| start + end)); + if let Self::String(s) = self { + if !s.2.is_empty() { + output.push_str(&s.0); + for (end, trusted) in &s.2 { + let end = start + end; + match runs.last_mut() { + Some(last) if last.1 == *trusted => last.0 = end, + _ => runs.push((end, *trusted)), + } + } + return; + } + } + self.write(output); + let end = output.len(); + match runs.last_mut() { + Some(last) if last.1 == trust => last.0 = end, + _ => runs.push((end, trust)), + } + } + pub fn sliced_annotations(&self, start: usize, end: usize) -> (Vec, Vec<(usize, bool)>) { + let mut hints = Vec::new(); + let mut runs: Vec<(usize, bool)> = Vec::new(); + if let Self::String(s) = self { + for hint in &s.1 { + if *hint > start { + hints.push((*hint - start).min(end - start)); + } + } + let mut cursor = 0; + for (run_end, trusted) in &s.2 { + let run_start = cursor; + cursor = *run_end; + let low = run_start.max(start); + let high = (*run_end).min(end); + if high > low { + match runs.last_mut() { + Some(last) if last.1 == *trusted => last.0 = high - start, + _ => runs.push((high - start, *trusted)), + } + } + } + } + (hints, runs) + } + pub(super) fn trust_at_byte(&self, byte: usize) -> Option { + let Self::String(s) = self else { + return None; + }; + s.2.iter() + .find(|(end, _)| byte < *end) + .map(|(_, trusted)| *trusted) + } + fn annotated_char(&self, text: &str, byte: usize) -> Self { + let character = text[byte..].chars().next().unwrap(); + match self.trust_at_byte(byte) { + Some(trusted) => Self::marked_runs( + character.to_string(), + Vec::new(), + vec![(character.len_utf8(), trusted)], + ), + None => Self::string(character.to_string()), + } + } pub fn list(value: Vec) -> Self { Self::List(Rc::new(value)) } @@ -202,8 +339,11 @@ impl<'a> Value<'a> { Self::List(v) => v.get(index).cloned().unwrap_or_default(), _ => self .as_str() - .and_then(|s| s.chars().nth(index)) - .map(|c| Self::string(c.to_string())) + .and_then(|s| { + s.char_indices() + .nth(index) + .map(|(byte, _)| self.annotated_char(s, byte)) + }) .unwrap_or_default(), } } @@ -218,8 +358,8 @@ impl<'a> Value<'a> { _ if self.as_str().is_some() => Ok(self .as_str() .unwrap() - .chars() - .map(|c| Self::string(c.to_string())) + .char_indices() + .map(|(byte, _)| self.annotated_char(self.as_str().unwrap(), byte)) .collect()), _ => Err(Error::new("value is not iterable")), } diff --git a/helios/src/frontend/jinja/vm.rs b/helios/src/frontend/jinja/vm.rs index 3529dec70..6567a2a73 100644 --- a/helios/src/frontend/jinja/vm.rs +++ b/helios/src/frontend/jinja/vm.rs @@ -21,7 +21,7 @@ struct Scope { struct Closure<'a> { definition: u32, scope: usize, - captured: Option>>, + captured: Option<(Vec>, Vec)>, } struct Vm<'a> { template: &'a CompiledTemplate, @@ -31,11 +31,14 @@ struct Vm<'a> { serial: usize, scopes: Vec, closures: Vec>, - undo: Vec<(u32, Value<'a>, usize)>, + undo: Vec<(u32, Value<'a>, usize, bool)>, stack: Vec>, + stack_trust: Vec, + slots_trust: Vec, loops: Vec>>, output: String, - captures: Vec<(String, Vec)>, + output_runs: Vec<(usize, bool)>, + captures: Vec<(String, Vec, Vec<(usize, bool)>)>, hint_positions: Vec, marks: Vec, depth: usize, @@ -46,6 +49,7 @@ pub(super) fn render( template: &CompiledTemplate, context: &Map, hints: &[(usize, usize)], + trusted: &std::collections::HashSet, ) -> Result { let mut hint_texts = Vec::new(); for &(message, count) in hints { @@ -85,8 +89,11 @@ pub(super) fn render( closures: Vec::new(), undo: Vec::new(), stack: Vec::with_capacity(16), + stack_trust: Vec::with_capacity(16), + slots_trust: template.names.iter().map(|s| trusted.contains(s)).collect(), loops: Vec::new(), output: String::new(), + output_runs: Vec::new(), captures: Vec::new(), hint_positions: Vec::new(), marks: Vec::new(), @@ -98,6 +105,7 @@ pub(super) fn render( Ok(RenderedTemplate { text: vm.output, marks: vm.marks, + runs: vm.output_runs, }) } @@ -107,6 +115,22 @@ impl<'a> Vm<'a> { .pop() .expect("compiler balances the operand stack") } + fn pop_trust(&mut self) -> bool { + self.stack_trust + .pop() + .expect("compiler balances the operand stack") + } + fn pop_trusted(&mut self) -> (Value<'a>, bool) { + let trust = self.pop_trust(); + (self.pop(), trust) + } + fn force_trust(value: Value<'a>, trusted: bool) -> Value<'a> { + if trusted { + value + } else { + Value::baked_untrusted(value) + } + } fn enter(&mut self) { self.scopes.push(Scope { undo: self.undo.len(), @@ -123,15 +147,20 @@ impl<'a> Vm<'a> { let values = self.template.macros[closure.definition as usize] .captures .iter() - .map(|&s| self.slots[s as usize].clone()) - .collect(); - self.closures[i].captured = Some(values); + .map(|&s| (self.lexical(s, self.scope), self.lexical_trust(s, self.scope))) + .collect::>(); + let (values, trusts): (Vec<_>, Vec<_>) = values + .into_iter() + .map(|(value, trust)| (Value::baked_untrusted(value), trust)) + .unzip(); + self.closures[i].captured = Some((values, trusts)); } } let scope = self.scopes.pop().unwrap(); - for (slot, value, stamp) in self.undo.drain(scope.undo..).rev() { + for (slot, value, stamp, trust) in self.undo.drain(scope.undo..).rev() { self.slots[slot as usize] = value; self.stamps[slot as usize] = stamp; + self.slots_trust[slot as usize] = trust; } self.scope = scope.id; } @@ -139,7 +168,7 @@ impl<'a> Vm<'a> { let mut value = &self.slots[slot as usize]; let mut stamp = self.stamps[slot as usize]; if stamp > scope { - for (s, previous, previous_stamp) in self.undo.iter().rev() { + for (s, previous, previous_stamp, _) in self.undo.iter().rev() { if *s == slot { value = previous; stamp = *previous_stamp; @@ -151,7 +180,23 @@ impl<'a> Vm<'a> { } value.clone() } - fn store(&mut self, slot: u32, value: Value<'a>) { + fn lexical_trust(&self, slot: u32, scope: usize) -> bool { + let mut trust = self.slots_trust[slot as usize]; + let mut stamp = self.stamps[slot as usize]; + if stamp > scope { + for (s, _, previous_stamp, previous_trust) in self.undo.iter().rev() { + if *s == slot { + trust = *previous_trust; + stamp = *previous_stamp; + if stamp <= scope { + break; + } + } + } + } + trust + } + fn store(&mut self, slot: u32, value: Value<'a>, trust: bool) { let value = self.hinted(value); let index = slot as usize; if self.scope != 0 && self.stamps[index] != self.scope { @@ -159,28 +204,31 @@ impl<'a> Vm<'a> { slot, std::mem::take(&mut self.slots[index]), self.stamps[index], + self.slots_trust[index], )); self.stamps[index] = self.scope; } self.slots[index] = value; + self.slots_trust[index] = trust; } fn iteration(&mut self, id: u32) -> Result<()> { let value = self.loops.last().unwrap().clone(); let item = value.items[value.index.get()].clone(); - self.store(self.template.loop_symbol, Value::Loop(value)); - self.loop_variables(id, item) + let trust = value.trust; + self.store(self.template.loop_symbol, Value::Loop(value), trust); + self.loop_variables(id, item, trust) } - fn loop_variables(&mut self, id: u32, item: Value<'a>) -> Result<()> { + fn loop_variables(&mut self, id: u32, item: Value<'a>, trust: bool) -> Result<()> { let vars = &self.template.loops[id as usize].variables; if vars.len() == 1 { - self.store(vars[0], item); + self.store(vars[0], item, trust); } else { let items = item.iter()?; if items.len() != vars.len() { return Err(Error::new("cannot unpack sequence of different length")); } for (&slot, value) in vars.iter().zip(items) { - self.store(slot, value); + self.store(slot, value, trust); } } Ok(()) @@ -196,21 +244,27 @@ impl<'a> Vm<'a> { } value } - fn emit(&mut self, value: Value<'a>) { - self.hinted(value) - .append(&mut self.output, &mut self.hint_positions); + fn emit(&mut self, value: Value<'a>, trust: bool) { + self.hinted(value).append_annotated( + &mut self.output, + &mut self.hint_positions, + &mut self.output_runs, + trust, + ); } fn capture(&mut self) { self.captures.push(( std::mem::take(&mut self.output), std::mem::take(&mut self.hint_positions), + std::mem::take(&mut self.output_runs), )); } fn end_capture(&mut self) -> Value<'a> { - let (output, hints) = self.captures.pop().unwrap(); + let (output, hints, runs) = self.captures.pop().unwrap(); let text = std::mem::replace(&mut self.output, output); let positions = std::mem::replace(&mut self.hint_positions, hints); - Value::marked(text, positions) + let runs = std::mem::replace(&mut self.output_runs, runs); + Value::marked_runs(text, positions, runs) } fn run(&mut self, start: u32) -> Result<()> { self.depth += 1; @@ -223,19 +277,33 @@ impl<'a> Vm<'a> { let op = self.template.code[pc]; pc += 1; match op { - I::Text(id) => self - .output - .push_str(self.template.constants[id as usize].as_str().unwrap()), - I::Constant(id) => self - .stack - .push(Value::json(&self.template.constants[id as usize])), - I::Load(id) => self.stack.push(self.slots[id as usize].clone()), + I::Text(id) => { + self.output + .push_str(self.template.constants[id as usize].as_str().unwrap()); + self.output_runs.push((self.output.len(), true)); + } + I::Constant(id) => { + self.stack + .push(Value::json(&self.template.constants[id as usize])); + self.stack_trust.push(true); + } + I::Load(id) => { + self.stack.push(self.slots[id as usize].clone()); + self.stack_trust.push(self.slots_trust[id as usize]); + } I::Store(id) => { + let trust = self.pop_trust(); let v = self.pop(); - self.store(id, v); + self.store(id, v, trust); } I::StoreAttr(slot, name) | I::AppendAttr(slot, name) => { + let trust = self.pop_trust(); let value = self.pop(); + let value = if trust { + value + } else { + Value::baked_untrusted(value) + }; let Value::Namespace(ns) = &self.slots[slot as usize] else { return Err(Error::new( "cannot assign attribute of non-namespace object", @@ -251,6 +319,20 @@ impl<'a> Vm<'a> { { Rc::make_mut(items).extend(value.iter()?) } + (old, value) + if old.as_str().is_some() && value.as_str().is_some() => + { + let mut text = String::new(); + let mut hints = Vec::new(); + let mut runs = Vec::new(); + old.append_annotated( + &mut text, &mut hints, &mut runs, true, + ); + value.append_annotated( + &mut text, &mut hints, &mut runs, true, + ); + *old = Value::marked_runs(text, hints, runs); + } (old, value) => { *old = binary(Binary::Add, std::mem::take(old), value)? } @@ -265,114 +347,216 @@ impl<'a> Vm<'a> { } } I::Attr(name) => { + let trust = self.pop_trust(); let v = self.pop(); if matches!(v, Value::Undefined) { return Err(Error::new("undefined value")); } let value = self.hinted(v.attr(&self.template.names[name as usize])); self.stack.push(value); + self.stack_trust.push(trust); } I::Index => { + let _ = self.pop_trust(); let key = self.pop(); + let trust = self.pop_trust(); let v = self.pop(); if matches!(v, Value::Undefined) { return Err(Error::new("undefined value")); } let value = self.hinted(v.index(&key)); self.stack.push(value); + self.stack_trust.push(trust); } I::Slice => { + let _ = self.pop_trust(); let step = self.pop(); + let _ = self.pop_trust(); let end = self.pop(); + let _ = self.pop_trust(); let begin = self.pop(); + let trust = self.pop_trust(); let v = self.pop(); - self.stack.push(slice(v, begin, end, step)?); + let sliced = slice(v, begin, end, step)?; + self.stack.push(Self::force_trust(sliced, trust)); + self.stack_trust.push(trust); } I::List(n) => { + let trusts = self.stack_trust.split_off(self.stack_trust.len() - n as usize); let items = self.stack.split_off(self.stack.len() - n as usize); + let trusted = trusts.iter().all(|trusted| *trusted); + let items = items + .into_iter() + .zip(trusts) + .map(|(value, trust)| if trust { value } else { Value::baked_untrusted(value) }) + .collect(); self.stack.push(Value::list(items)); + self.stack_trust.push(trusted); } I::Map(n) => { + let trusts = self.stack_trust.split_off(self.stack_trust.len() - 2 * n as usize); let values = self.stack.split_off(self.stack.len() - 2 * n as usize); let mut pairs = Vec::with_capacity(n as usize); - let mut values = values.into_iter(); - while let Some(key) = values.next() { - pairs.push((key.text(), values.next().unwrap())); + let mut trusted = true; + let mut entries = values.into_iter().zip(trusts); + while let Some((key, key_trust)) = entries.next() { + let (value, value_trust) = entries.next().unwrap(); + trusted &= key_trust & value_trust; + let value = if value_trust { + value + } else { + Value::baked_untrusted(value) + }; + pairs.push((key.text(), value)); } self.stack.push(Value::Map(Rc::new(pairs))); + self.stack_trust.push(trusted); } I::Unary(op) => { + let trust = self.pop_trust(); let v = self.pop(); - self.stack.push(match op { - Unary::Not => Value::Bool(!v.truthy()), - Unary::Pos => v, + let (value, trusted) = match op { + Unary::Not => (Value::Bool(!v.truthy()), true), + Unary::Pos => (v, trust), Unary::Neg => match v { - Value::Int(v) => Value::Int( + Value::Int(v) => (Value::Int( v.checked_neg() .ok_or_else(|| Error::new("integer overflow"))?, - ), - _ => Value::Float( + ), true), + _ => (Value::Float( -v.number() .ok_or_else(|| Error::new("cannot negate non-number"))?, - ), + ), true), }, - }); + }; + self.stack.push(value); + self.stack_trust.push(trusted); } I::Binary(op) => { + let b_trust = self.pop_trust(); let b = self.pop(); + let a_trust = self.pop_trust(); let a = self.pop(); - self.stack.push(binary(op, a, b)?); + let mut annotated = false; + let result = if matches!(op, Binary::Concat | Binary::Add) + && a.as_str().is_some() + && b.as_str().is_some() + { + annotated = true; + let mut text = String::new(); + let mut hints = Vec::new(); + let mut runs = Vec::new(); + a.append_annotated(&mut text, &mut hints, &mut runs, a_trust); + b.append_annotated(&mut text, &mut hints, &mut runs, b_trust); + Value::marked_runs(text, hints, runs) + } else { + binary(op, a, b)? + }; + let trusted = match op { + Binary::Eq + | Binary::Ne + | Binary::In + | Binary::NotIn + | Binary::Lt + | Binary::Le + | Binary::Gt + | Binary::Ge => true, + Binary::Concat | Binary::Add if annotated => { + !result.carries_untrusted() + } + _ => a_trust && b_trust, + }; + let result = Self::force_trust(result, trusted); + self.stack.push(result); + self.stack_trust.push(trusted); } I::Test(id) | I::Filter(id) | I::Method(id) | I::Call(id) => { let call = &self.template.calls[id as usize]; let count = call.positional + call.keywords.len(); + let arg_trusts = self.stack_trust.split_off(self.stack_trust.len() - count); let arguments = self.stack.split_off(self.stack.len() - count); let name = self.template.names[call.name as usize].as_str(); let (args, kw) = arguments.split_at(call.positional); + let (arg_trusts, kw_trusts) = arg_trusts.split_at(call.positional); let kwargs: Vec<_> = call .keywords .iter() .zip(kw) - .map(|(key, v)| { - (self.template.names[*key as usize].as_str(), v.clone()) + .zip(kw_trusts) + .map(|((key, v), trust)| { + (self.template.names[*key as usize].as_str(), v.clone(), *trust) }) .collect(); + let kw_named: Vec<(&str, Value<'a>)> = kwargs + .iter() + .map(|(name, value, _)| (*name, value.clone())) + .collect(); + let arguments_trusted = arg_trusts.iter().chain(kw_trusts.iter()).all(|trusted| *trusted); + let mut input_trust = true; + let mut input_untrusted = false; + let mut macro_result = false; let result = match op { I::Call(_) => { if let Value::Macro(id) = self.slots[call.name as usize] { let direct = matches!(self.template.code.get(pc), Some(I::Emit)); - let value = self.macro_call(id, args, &kwargs, direct)?; + let value = self.macro_call( + id, + args, + &kwargs, + arg_trusts, + direct, + )?; if direct { pc += 1; continue; } + macro_result = true; value } else { - function(name, args, &kwargs)? + function(name, args, &kw_named)? } } I::Test(_) => { - let v = self.pop(); + let (v, _) = self.pop_trusted(); Value::Bool(test(name, &v, args)?) } I::Filter(_) => { - let v = self.pop(); - filter(name, v, args, &kwargs, !self.hint_texts.is_empty())? + let (v, trust) = self.pop_trusted(); + input_trust = trust; + input_untrusted = v.carries_untrusted(); + filter(name, v, args, &kw_named, !self.hint_texts.is_empty())? + } + _ => { + let (v, trust) = self.pop_trusted(); + input_trust = trust; + input_untrusted = v.carries_untrusted(); + method(name, v, args, &kw_named)? } + }; + let trusted = match op { + I::Test(_) => true, + I::Call(_) if macro_result => !result.carries_untrusted(), + I::Call(_) => arguments_trusted && !result.carries_untrusted(), _ => { - let v = self.pop(); - method(name, v, args, &kwargs)? + arguments_trusted + && input_trust + && !input_untrusted + && !result.carries_untrusted() } }; + let result = Self::force_trust(result, trusted); self.stack.push(result); + self.stack_trust.push(trusted); } I::Emit => { + let trust = self.pop_trust(); let v = self.pop(); - self.emit(v); + self.emit(v, trust); } I::Join(count, strict) | I::EmitJoined(count, strict) => { let start = self.stack.len() - count as usize; + let trusts = self.stack_trust.split_off(start); let values = &self.stack[start..]; if strict && values.iter().any(|v| v.as_str().is_none()) { return Err(Error::new("string addition requires strings")); @@ -380,24 +564,30 @@ impl<'a> Vm<'a> { let size = values.iter().map(|v| v.as_str().map_or(0, str::len)).sum(); if matches!(op, I::EmitJoined(..)) { self.output.reserve(size); - for index in start..self.stack.len() { - self.emit(self.stack[index].clone()); + for (index, trust) in (start..self.stack.len()).zip(trusts) { + let value = self.stack[index].clone(); + self.emit(value, trust); } self.stack.truncate(start); } else { let mut text = String::with_capacity(size); let mut hints = Vec::new(); - for v in self.stack.drain(start..) { - v.append(&mut text, &mut hints); + let mut runs = Vec::new(); + for (v, trust) in self.stack.drain(start..).zip(trusts) { + v.append_annotated(&mut text, &mut hints, &mut runs, trust); } - self.stack.push(Value::marked(text, hints)); + let trusted = runs.iter().all(|(_, trusted)| *trusted); + self.stack.push(Value::marked_runs(text, hints, runs)); + self.stack_trust.push(trusted); } } I::Drop => { + let _ = self.pop_trust(); self.pop(); } I::Jump(target) => pc = target as usize, I::JumpFalse(target) => { + let _ = self.pop_trust(); if !self.pop().truthy() { pc = target as usize; } @@ -406,6 +596,7 @@ impl<'a> Vm<'a> { if !self.stack.last().unwrap().truthy() { pc = target as usize; } else { + let _ = self.pop_trust(); self.pop(); } } @@ -413,17 +604,20 @@ impl<'a> Vm<'a> { if self.stack.last().unwrap().truthy() { pc = target as usize; } else { + let _ = self.pop_trust(); self.pop(); } } I::For(id) => { + let trust = self.pop_trust(); let mut items = self.pop().iter()?; if let Some(filter) = self.template.loops[id as usize].filter { self.enter(); let mut selected = Vec::new(); for item in items { - self.loop_variables(id, item.clone())?; + self.loop_variables(id, item.clone(), trust)?; self.run(filter)?; + let _ = self.pop_trust(); if self.pop().truthy() { selected.push(item); } @@ -435,6 +629,7 @@ impl<'a> Vm<'a> { self.loops.push(Rc::new(LoopValue { items, index: Cell::new(0), + trust, })); self.enter(); if empty { @@ -469,16 +664,19 @@ impl<'a> Vm<'a> { scope: self.scope, captured: None, }); - self.store(slot, Value::Macro(closure)); + self.store(slot, Value::Macro(closure), true); } I::Return => break, I::Capture => self.capture(), I::EndCapture => { let value = self.end_capture(); + let trusted = !value.carries_untrusted(); self.stack.push(value); + self.stack_trust.push(trusted); } I::Mark => self.marks.push(Mark::Byte(self.output.len())), I::MarkPosition => { + let _ = self.pop_trust(); let Value::Int(v) = self.pop() else { return Err(Error::new("mark position must be a nonnegative integer")); }; @@ -503,7 +701,8 @@ impl<'a> Vm<'a> { &mut self, id: u32, args: &[Value<'a>], - kwargs: &[(&str, Value<'a>)], + kwargs: &[(&str, Value<'a>, bool)], + arg_trusts: &[bool], direct: bool, ) -> Result> { let closure = &self.closures[id as usize]; @@ -514,28 +713,31 @@ impl<'a> Vm<'a> { } self.enter(); for (i, &slot) in mac.captures.iter().enumerate() { - let value = if let Some(captured) = &self.closures[id as usize].captured { - captured[i].clone() + let (value, trust) = if let Some((captured, trusts)) = &self.closures[id as usize].captured + { + (captured[i].clone(), trusts[i]) } else { - self.lexical(slot, scope) + let trust = self.lexical_trust(slot, scope); + (self.lexical(slot, scope), trust) }; - self.store(slot, value); + self.store(slot, value, trust); } for (i, &slot) in mac.parameters.iter().enumerate() { let name = self.template.names[slot as usize].as_str(); let value = if let Some(value) = args.get(i) { - value.clone() - } else if let Some((_, value)) = kwargs.iter().find(|(k, _)| *k == name) { - value.clone() + let trust = arg_trusts.get(i).copied().unwrap_or(true); + (value.clone(), trust) + } else if let Some((_, value, trust)) = kwargs.iter().find(|(k, _, _)| *k == name) { + (value.clone(), *trust) } else if let Some((_, pc)) = mac.defaults.iter().find(|(s, _)| *s == slot) { self.run(*pc)?; - self.pop() + self.pop_trusted() } else { - Value::Undefined + (Value::Undefined, true) }; - self.store(slot, value); + self.store(slot, value.0, value.1); } - for (name, _) in kwargs { + for (name, _, _) in kwargs { if !mac .parameters .iter() @@ -687,21 +889,50 @@ fn slice<'a>( }; let mut i = bound(&begin, if step < 0 { n - 1 } else { 0 })?; let end = bound(&end, if step < 0 { -1 } else { n })?; - let mut items = Vec::new(); + let mut indices = Vec::new(); while if step < 0 { i > end } else { i < end } { - items.push(value.index(&Value::Int(i))); + indices.push(i); i = i .checked_add(step) .ok_or_else(|| Error::new("slice index overflow"))?; } - if value.as_str().is_some() { - let mut s = String::new(); - for v in items { - v.write(&mut s); + if let Some(text) = value.as_str() { + let mut output = String::new(); + let mut hints = Vec::new(); + let mut runs: Vec<(usize, bool)> = Vec::new(); + for index in indices { + let Ok(index) = usize::try_from(index) else { + continue; + }; + let Some((byte, character)) = text.char_indices().nth(index) else { + continue; + }; + output.push(character); + let output_end = output.len(); + if let Value::String(source) = &value { + if source + .1 + .iter() + .any(|hint| *hint == byte + character.len_utf8()) + { + hints.push(output_end); + } + if let Some(trusted) = value.trust_at_byte(byte) { + match runs.last_mut() { + Some(last) if last.1 == trusted => last.0 = output_end, + _ => runs.push((output_end, trusted)), + } + } + } } - Ok(Value::string(s)) + Ok(Value::marked_runs(output, hints, runs)) } else { - Ok(Value::list(items)) + Ok(Value::list( + indices + .into_iter() + .map(|index| value.index(&Value::Int(index))) + .collect(), + )) } } @@ -879,17 +1110,15 @@ fn filter<'a>( }; let trimmed = text.trim_matches(pred); if trimmed.len() == text.len() { - Ok(v) - } else { - let start = trimmed.as_ptr() as usize - text.as_ptr() as usize; - let hints = v - .hints() - .iter() - .filter(|end| **end > start && !trimmed.is_empty()) - .map(|end| (end - start).min(trimmed.len())) - .collect(); - Ok(Value::marked(trimmed.into(), hints)) + return Ok(v); + } + let start = trimmed.as_ptr() as usize - text.as_ptr() as usize; + let end = start + trimmed.len(); + let (mut hints, runs) = v.sliced_annotations(start, end); + if trimmed.is_empty() { + hints.clear(); } + Ok(Value::marked_runs(trimmed.into(), hints, runs)) } "capitalize" => { let text = v.text().to_lowercase(); @@ -1096,14 +1325,18 @@ fn method<'a>( "strip" | "lstrip" | "rstrip" => { let chars = args.first().and_then(Value::as_str); let pred = |c: char| chars.map_or_else(|| c.is_whitespace(), |s| s.contains(c)); - Ok(Value::string( - match name { - "lstrip" => text.trim_start_matches(pred), - "rstrip" => text.trim_end_matches(pred), - _ => text.trim_matches(pred), - } - .into(), - )) + let trimmed = match name { + "lstrip" => text.trim_start_matches(pred), + "rstrip" => text.trim_end_matches(pred), + _ => text.trim_matches(pred), + }; + let start = trimmed.as_ptr() as usize - text.as_ptr() as usize; + let end = start + trimmed.len(); + let (mut hints, runs) = v.sliced_annotations(start, end); + if trimmed.is_empty() { + hints.clear(); + } + Ok(Value::marked_runs(trimmed.into(), hints, runs)) } "split" | "rsplit" => { let limit = args diff --git a/helios/src/frontend/mod.rs b/helios/src/frontend/mod.rs index e7879b206..c725fd224 100644 --- a/helios/src/frontend/mod.rs +++ b/helios/src/frontend/mod.rs @@ -297,7 +297,8 @@ pub struct Frontend { prompt_encoder: PromptEncoder, image_placeholder_token_id: Option, tokenizer: Tokenizer, - ordinary_tokenizer: Option, + ordinary_tokenizer: std::sync::OnceLock, + segmented_template: bool, template: Option, template_source: String, bos_token_id: Option, @@ -313,14 +314,93 @@ impl Frontend { } pub fn with_template_encoding(mut self, encoding: TemplateEncoding) -> Self { if self.prompt_encoder == PromptEncoder::Jinja { - self.ordinary_tokenizer = matches!(encoding, TemplateEncoding::TokenSegments).then(|| { - let mut tokenizer = self.tokenizer.clone(); - tokenizer.with_added_vocabulary(tokenizers::AddedVocabulary::new()); - tokenizer - }); + self.segmented_template = matches!(encoding, TemplateEncoding::TokenSegments); } self } + + fn ordinary_from(tokenizer: &Tokenizer) -> Tokenizer { + let mut tokenizer = tokenizer.clone(); + tokenizer.with_added_vocabulary(tokenizers::AddedVocabulary::new()); + tokenizer + } + + fn ordinary(&self) -> &Tokenizer { + self.ordinary_tokenizer.get_or_init(|| Self::ordinary_from(&self.tokenizer)) + } + + fn encode_splicing_untrusted_markers( + &self, + text: &str, + untrusted: &[(usize, usize)], + ) -> Result<(Vec, Vec<(usize, usize)>)> { + let encoding = self.tokenizer.encode(text, false).map_err(|error| { + Error::frontend(FrontendError::Tokenizer(Failure::boxed( + error.to_string(), + error, + ))) + })?; + let token_ids = encoding.get_ids().to_vec(); + let offsets = encoding.get_offsets().to_vec(); + if untrusted.is_empty() { + return Ok((token_ids, offsets)); + } + let added = self.tokenizer.get_added_tokens_decoder(); + let mut spans: Vec<(usize, usize)> = Vec::new(); + let mut token = 0usize; + for &(run_start, run_end) in untrusted { + while token < offsets.len() && offsets[token].1 <= run_start { + token += 1; + } + if token == offsets.len() || offsets[token].0 >= run_end { + continue; + } + let mut span_end = token; + while span_end < offsets.len() && offsets[span_end].0 < run_end { + span_end += 1; + } + if !token_ids[token..span_end] + .iter() + .any(|id| added.contains_key(id)) + { + continue; + } + match spans.last_mut() { + Some(previous) if token <= previous.1 => previous.1 = previous.1.max(span_end), + _ => spans.push((token, span_end)), + } + } + let mut spliced_ids = Vec::with_capacity(token_ids.len()); + let mut spliced_offsets = Vec::with_capacity(offsets.len()); + let mut cursor = 0usize; + for (span_start, span_end) in spans { + let byte_start = offsets[span_start].0; + let byte_end = offsets[span_end - 1].1; + if byte_end <= byte_start { + continue; + } + spliced_ids.extend_from_slice(&token_ids[cursor..span_start]); + spliced_offsets.extend_from_slice(&offsets[cursor..span_start]); + cursor = span_end; + let slice = text.get(byte_start..byte_end).unwrap_or(""); + let replacement = self.ordinary().encode(slice, false).map_err(|error| { + Error::frontend(FrontendError::Tokenizer(Failure::boxed( + error.to_string(), + error, + ))) + })?; + spliced_ids.extend_from_slice(replacement.get_ids()); + spliced_offsets.extend( + replacement + .get_offsets() + .iter() + .map(|&(start, end)| (byte_start + start, byte_start + end)), + ); + } + spliced_ids.extend_from_slice(&token_ids[cursor..]); + spliced_offsets.extend_from_slice(&offsets[cursor..]); + Ok((spliced_ids, spliced_offsets)) + } pub fn bos_token_id(&self) -> Option { self.bos_token_id } @@ -551,12 +631,6 @@ impl Frontend { defaults.insert("tools".into(), Json::Null); defaults.insert("documents".into(), Json::Null); defaults.extend(template_defaults); - // Programmatic encoders tokenize message text without the added vocabulary. - let ordinary_tokenizer = (prompt_encoder != PromptEncoder::Jinja).then(|| { - let mut ordinary = tokenizer.clone(); - ordinary.with_added_vocabulary(tokenizers::AddedVocabulary::new()); - ordinary - }); let frontend = Self { prompt_encoder, image_placeholder_token_id, @@ -565,7 +639,8 @@ impl Frontend { effort_levels: Default::default(), inline_system_messages: Default::default(), tokenizer, - ordinary_tokenizer, + ordinary_tokenizer: std::sync::OnceLock::new(), + segmented_template: false, template, template_source, defaults, @@ -709,11 +784,17 @@ impl Frontend { .map(|field| self.mark_continuation(&mut messages, &field)) .transpose()?; context.insert("messages".into(), Json::Array(messages)); + let trusted: std::collections::HashSet = self + .defaults + .keys() + .filter(|name| !options.contains_key(*name) && name.as_str() != "messages") + .cloned() + .collect(); let rendered = self .template .as_ref() .unwrap() - .render(&context, &checkpoint_hints) + .render_with_trust(&context, &checkpoint_hints, &trusted) .map_err(|error| { Error::frontend(FrontendError::Template(Failure::caused_by( error.message.clone(), @@ -721,6 +802,7 @@ impl Frontend { ))) })?; let marks = rendered.marks; + let runs = rendered.runs; let rendered = rendered.text; let mut hint_bytes = marks .iter() @@ -729,7 +811,7 @@ impl Frontend { _ => None, }) .collect(); - let mut segments: Vec = if self.ordinary_tokenizer.is_some() { + let mut segments: Vec = if self.segmented_template { token_segments(&rendered, &mut hint_bytes).map_err(|error| { Error::frontend(FrontendError::Template(Failure::caused_by( "chat template must render an array of text and allow_special segments", @@ -737,11 +819,7 @@ impl Frontend { ))) })? } else { - vec![TokenSegment { - text: rendered, - allow_special: true, - image_index: None, - }] + Vec::new() }; order_segment_images(&segments, &mut image_sources, &mut image_checkpoint_hints)?; let mut segment_ends = Vec::with_capacity(segments.len() + 1); @@ -753,13 +831,10 @@ impl Frontend { for mark in marks { match mark { jinja::Mark::Hint(_) => {} - jinja::Mark::Byte(byte) - if self.ordinary_tokenizer.is_none() - && byte <= *segment_ends.last().unwrap() => - { + jinja::Mark::Byte(byte) if !self.segmented_template && byte <= rendered.len() => { byte_boundaries.push(byte); } - jinja::Mark::Position(count) if self.ordinary_tokenizer.is_some() => { + jinja::Mark::Position(count) if self.segmented_template => { byte_boundaries.push(*segment_ends.get(count).ok_or_else(|| { deepseek::invalid("mark position exceeds the rendered segment count") })?); @@ -771,10 +846,14 @@ impl Frontend { } } } - let mut text: String = segments - .iter() - .map(|segment| segment.text.as_str()) - .collect(); + let mut text: String = if self.segmented_template { + segments + .iter() + .map(|segment| segment.text.as_str()) + .collect() + } else { + rendered + }; if let Some(final_text) = final_text { let marker = "CONTINUE_FINAL_MESSAGE_TAG"; let index = text.rfind(marker).filter(|_| text.contains(final_text.trim())).ok_or_else(|| Error::frontend(FrontendError::Input(Failure::new("continue_final_message is set but the final message does not appear in the chat after applying the chat template!"))))?; @@ -784,32 +863,56 @@ impl Frontend { text.truncate(text.trim_end().len()); } } - let mut remaining = text.len(); let mut token_ids = Vec::new(); let mut offsets = Vec::new(); - let mut byte_start = 0; - for segment in &mut segments { - segment.text.truncate(remaining.min(segment.text.len())); - remaining -= segment.text.len(); - let tokenizer = if segment.allow_special { - &self.tokenizer - } else { - self.ordinary_tokenizer.as_ref().unwrap_or(&self.tokenizer) - }; - let encoding = tokenizer - .encode(segment.text.as_str(), false) - .map_err(|error| { - Error::frontend(FrontendError::Tokenizer(Failure::boxed( - error.to_string(), - error, - ))) - })?; - token_ids.extend_from_slice(encoding.get_ids()); - offsets.extend(encoding.get_offsets().iter().map(|&(start,end)| (byte_start+start,byte_start+end))); - byte_start += segment.text.len(); - if remaining == 0 { - break; + if self.segmented_template { + let mut remaining = text.len(); + let mut byte_start = 0; + for segment in &mut segments { + segment.text.truncate(remaining.min(segment.text.len())); + remaining -= segment.text.len(); + let tokenizer = if segment.allow_special { + &self.tokenizer + } else { + self.ordinary() + }; + let encoding = tokenizer + .encode(segment.text.as_str(), false) + .map_err(|error| { + Error::frontend(FrontendError::Tokenizer(Failure::boxed( + error.to_string(), + error, + ))) + })?; + token_ids.extend_from_slice(encoding.get_ids()); + offsets.extend( + encoding + .get_offsets() + .iter() + .map(|&(start, end)| (byte_start + start, byte_start + end)), + ); + byte_start += segment.text.len(); + if remaining == 0 { + break; + } } + } else { + let mut cursor = 0; + let mut untrusted: Vec<(usize, usize)> = Vec::new(); + for (end, trusted) in runs { + let end = end.min(text.len()); + if !trusted && end > cursor { + if untrusted.last().is_some_and(|last| last.1 == cursor) { + untrusted.last_mut().unwrap().1 = end; + } else { + untrusted.push((cursor, end)); + } + } + cursor = end; + } + let spliced = self.encode_splicing_untrusted_markers(&text, &untrusted)?; + token_ids = spliced.0; + offsets = spliced.1; } let message_boundaries = boundaries::token_ends(&byte_boundaries, &hint_bytes, &offsets, text.len())?; @@ -968,7 +1071,8 @@ mod tests { Frontend { prompt_encoder: PromptEncoder::Jinja, image_placeholder_token_id: None, - ordinary_tokenizer: None, + ordinary_tokenizer: std::sync::OnceLock::new(), + segmented_template: false, tokenizer: Tokenizer::new(tokenizers::models::bpe::BPE::default()), template, template_source: source, @@ -1067,7 +1171,8 @@ mod tests { let frontend = Frontend { prompt_encoder: PromptEncoder::Jinja, image_placeholder_token_id: None, - ordinary_tokenizer: None, + ordinary_tokenizer: std::sync::OnceLock::new(), + segmented_template: false, tokenizer: Tokenizer::new(tokenizers::models::bpe::BPE::default()), template, template_source: source, @@ -1128,7 +1233,8 @@ mod tests { vocabulary_bytes: Default::default(), effort_levels: Default::default(), inline_system_messages: Default::default(), - ordinary_tokenizer: None, + ordinary_tokenizer: std::sync::OnceLock::new(), + segmented_template: false, tokenizer: Tokenizer::new(tokenizers::models::bpe::BPE::default()), template, template_source: source.to_owned(), @@ -1154,6 +1260,664 @@ mod tests { assert!(input.source().is_none()); } + fn trust_tokenizer() -> Tokenizer { + let mut vocab = serde_json::Map::new(); + for (id, piece) in [ + "a", + "b", + "ab", + "<|im_end|>", + "<", + "|", + "i", + "m", + "_", + "e", + "n", + "d", + ">", + "é", + " ", + " <", + "(", + "(<", + ")", + ">)", + "x", + ">x", + "x<", + ] + .into_iter() + .enumerate() + { + vocab.insert(piece.to_owned(), serde_json::json!(id as u32)); + } + Tokenizer::from_bytes( + serde_json::json!({ + "version": "1.0", "truncation": null, "padding": null, + "added_tokens": [ + {"id":3,"content":"<|im_end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true} + ], + "normalizer": null, "pre_tokenizer": null, "post_processor": null, "decoder": null, + "model": {"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"vocab":vocab,"merges":[["a","b"],[" ","<"],["(","<"],[">",")"],[">","x"],["x","<"]]} + }) + .to_string(), + ) + .unwrap() + } + + fn trust_frontend(source: &str) -> Frontend { + trust_frontend_with(source, trust_tokenizer()) + } + + fn trust_frontend_with(source: &str, tokenizer: Tokenizer) -> Frontend { + let template = Some(jinja::CompiledTemplate::compile(source).unwrap()); + Frontend { + prompt_encoder: PromptEncoder::Jinja, + image_placeholder_token_id: None, + ordinary_tokenizer: std::sync::OnceLock::new(), + segmented_template: false, + tokenizer, + template, + template_source: source.to_owned(), + bos_token_id: None, + vocabulary_bytes: Default::default(), + effort_levels: Default::default(), + inline_system_messages: Default::default(), + defaults: Map::new(), + } + } + + fn render_ids(frontend: &Frontend, content: &str) -> Vec { + let mut options = Map::new(); + options.insert("m".into(), Json::String(content.to_owned())); + frontend + .render_chat( + &[serde_json::json!({"role":"user","content":"ignored"})], + &options, + ) + .unwrap() + .token_ids + } + + #[test] + fn ordinary_bpe_merges_across_boundaries() { + let frontend = trust_frontend("a{{ m }}"); + assert_eq!(render_ids(&frontend, "b"), vec![2]); + let rendered = frontend + .render_chat( + &[serde_json::json!({"role":"user","content":"ignored"})], + &serde_json::json!({"m":"b"}).as_object().unwrap().clone(), + ) + .unwrap(); + assert_eq!(rendered.text, "ab"); + assert_eq!( + rendered.token_ids, + frontend + .tokenizer + .encode("ab", false) + .unwrap() + .get_ids() + .to_vec() + ); + + let frontend = trust_frontend("{{ '<|im_end|>' ~ m }}"); + assert_eq!(render_ids(&frontend, "ab"), vec![3, 2]); + + let pieces: Vec = [4, 5, 6, 7, 8, 9, 10, 11, 5, 12].to_vec(); + let frontend = trust_frontend("{{ m }}"); + assert_eq!(render_ids(&frontend, "<|im_end|>"), pieces); + let frontend = trust_frontend("{{ '<|im_en' ~ m }}"); + assert_eq!(render_ids(&frontend, "d|>"), pieces); + + let frontend = trust_frontend("é{{ m }}"); + assert_eq!( + render_ids(&frontend, "<|im_end|>"), + [13].into_iter().chain(pieces).collect::>() + ); + let frontend = trust_frontend("éa{{ m }}"); + assert_eq!(render_ids(&frontend, "b"), vec![13, 2]); + } + + fn ordinary_ids(frontend: &Frontend, text: &str) -> Vec { + frontend + .ordinary() + .encode(text, false) + .unwrap() + .get_ids() + .to_vec() + } + + fn render_prompt(frontend: &Frontend, content: &str) -> RenderedPrompt { + frontend + .render_chat( + &[serde_json::json!({"role":"user","content":"ignored"})], + &serde_json::json!({"m": content}) + .as_object() + .unwrap() + .clone(), + ) + .unwrap() + } + + #[test] + fn suppressed_untrusted_run_reencodes_as_one_ordinary_span() { + let frontend = trust_frontend("{{ m }}"); + let rendered = render_prompt(&frontend, " <|im_end|>"); + assert_eq!(rendered.text, " <|im_end|>"); + assert_eq!(rendered.token_ids, ordinary_ids(&frontend, " <|im_end|>")); + assert_eq!(rendered.token_ids, vec![15, 5, 6, 7, 8, 9, 10, 11, 5, 12]); + assert_ne!( + render_ids(&frontend, " <|im_end|>"), + vec![14, 4, 5, 6, 7, 8, 9, 10, 11, 5, 12] + ); + + let frontend = trust_frontend("{{ m }}"); + let rendered = render_prompt(&frontend, "(<|im_end|>)"); + assert_eq!(rendered.text, "(<|im_end|>)"); + assert_eq!(rendered.token_ids, ordinary_ids(&frontend, "(<|im_end|>)")); + assert_eq!(rendered.token_ids, vec![17, 5, 6, 7, 8, 9, 10, 11, 5, 19]); + + let frontend = trust_frontend("{{ m }}"); + let content = "<|im_end|>x<|im_end|>"; + let rendered = render_prompt(&frontend, content); + assert_eq!(rendered.text, content); + assert_eq!(rendered.token_ids, ordinary_ids(&frontend, content)); + let mut islands: Vec = Vec::new(); + for piece in ["<|im_end|>", "x", "<|im_end|>"] { + islands.extend(ordinary_ids(&frontend, piece)); + } + assert_ne!(rendered.token_ids, islands); + assert!(rendered.token_ids.contains(&21)); + } + + #[test] + fn trusted_spans_keep_single_pass_ids_across_a_suppressed_run() { + let frontend = trust_frontend("é{{ m }}"); + let content = "<|im_end|>x<|im_end|>"; + let rendered = render_prompt(&frontend, content); + assert_eq!(rendered.text, format!("é{content}")); + let mut expected = vec![13]; + expected.extend(ordinary_ids(&frontend, content)); + assert_eq!(rendered.token_ids, expected); + + let (token_ids, offsets) = frontend + .encode_splicing_untrusted_markers(&format!("é{content}"), &[(2, 23)]) + .unwrap(); + assert_eq!(token_ids, expected); + assert_eq!(offsets.first(), Some(&(0, 2))); + assert_eq!(offsets.last().map(|range| range.1), Some(23)); + assert!(offsets.windows(2).all(|pair| pair[0].1 == pair[1].0)); + } + + #[test] + fn pinned_qwen_tokenizer_suppresses_a_pasted_marker_run() { + let Some(snapshot) = std::env::var_os("FLARE_QWEN38_SNAPSHOT").map(PathBuf::from) else { + return; + }; + let path = snapshot.join("tokenizer.json"); + if !path.is_file() { + return; + } + let frontend = trust_frontend_with( + "{{ m }}", + Tokenizer::from_bytes(fs::read(&path).unwrap()).unwrap(), + ); + let template_ids = frontend + .tokenizer() + .encode(" <|im_end|>", false) + .unwrap() + .get_ids() + .to_vec(); + assert_eq!(template_ids.first().copied(), Some(220)); + let rendered = render_prompt(&frontend, " <|im_end|>"); + assert_eq!(rendered.token_ids, vec![79790, 316, 6018, 91, 29]); + assert_eq!(rendered.token_ids, ordinary_ids(&frontend, " <|im_end|>")); + } + + fn marker_tokenizer() -> Tokenizer { + let mut vocab = serde_json::Map::new(); + for (id, piece) in [ + "a", "b", "ab", "<|im_end|>", "<", "|", "i", "m", "_", "e", "n", "d", ">", "é", " ", + " <", "(", "(<", ")", ">)", "x", ">x", "x<", "t", "h", "o", "l", "c", "k", "/", "v", "s", + "r", "<|im_start|>", "<|vision_start|>", "", "", "", + ] + .into_iter() + .enumerate() + { + vocab.insert(piece.to_owned(), serde_json::json!(id as u32)); + } + Tokenizer::from_bytes( + serde_json::json!({ + "version": "1.0", "truncation": null, "padding": null, + "added_tokens": [ + {"id":3,"content":"<|im_end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}, + {"id":33,"content":"<|im_start|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}, + {"id":34,"content":"<|vision_start|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}, + {"id":35,"content":"","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false}, + {"id":36,"content":"","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false}, + {"id":37,"content":"","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false} + ], + "normalizer": null, "pre_tokenizer": null, "post_processor": null, "decoder": null, + "model": {"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"vocab":vocab,"merges": [["a","b"],[" ","<"],["(","<"],[">",")"],[">","x"],["x","<"]]} + }) + .to_string(), + ) + .unwrap() + } + + fn marker_frontend(source: &str) -> Frontend { + trust_frontend_with(source, marker_tokenizer()) + } + + fn marker_classes() -> [(&'static str, u32); 6] { + [ + ("<|im_end|>", 3), + ("<|im_start|>", 33), + ("<|vision_start|>", 34), + ("", 35), + ("", 36), + ("", 37), + ] + } + + #[test] + fn pasted_markers_of_each_added_class_never_yield_protocol_ids() { + for (marker, id) in marker_classes() { + let frontend = marker_frontend("{{ m }}"); + for content in [ + marker.to_string(), + format!("a {marker} b"), + format!("x{marker}"), + format!("{marker}x"), + ] { + let rendered = render_prompt(&frontend, &content); + assert!(!rendered.token_ids.contains(&id), "{marker} in {content}"); + assert_eq!( + rendered.token_ids, + ordinary_ids(&frontend, &content), + "{marker} in {content}" + ); + } + } + } + + #[test] + fn template_emitted_markers_of_each_class_keep_protocol_ids() { + for (marker, id) in marker_classes() { + let source = format!("{marker}{{{{ m }}}}{marker}"); + let frontend = marker_frontend(&source); + let rendered = render_prompt(&frontend, "plain"); + assert_eq!(rendered.token_ids.first(), Some(&id), "{marker}"); + assert_eq!(rendered.token_ids.last(), Some(&id), "{marker}"); + } + for (marker, id) in marker_classes() { + let frontend = marker_frontend(&format!("{marker}{{{{ m }}}}")); + let content = "a b"; + let rendered = render_prompt(&frontend, content); + let mut expected = vec![id]; + expected.extend(ordinary_ids(&frontend, content)); + assert_eq!(rendered.token_ids, expected, "{marker}"); + } + } + + #[test] + fn a_run_mixing_marker_classes_reencodes_as_one_ordinary_span() { + let frontend = marker_frontend("{{ m }}"); + let content = "a <|im_start|> b x d <|im_end|>"; + let rendered = render_prompt(&frontend, content); + assert_eq!(rendered.token_ids, ordinary_ids(&frontend, content)); + for (_, id) in marker_classes() { + assert!(!rendered.token_ids.contains(&id)); + } + } + + #[test] + fn markers_straddling_a_trust_boundary_are_pulled_into_the_run() { + for (marker, id) in marker_classes() { + let (prefix, suffix) = marker.split_at(marker.len() - 2); + let frontend = marker_frontend(&format!("{prefix}{{{{ m }}}}")); + let rendered = render_prompt(&frontend, suffix); + assert_eq!(rendered.text, marker); + assert!(!rendered.token_ids.contains(&id), "{marker}"); + assert_eq!(rendered.token_ids, ordinary_ids(&frontend, marker), "{marker}"); + + let frontend = marker_frontend(&format!("{{{{ m }}}}{suffix}")); + let rendered = render_prompt(&frontend, prefix); + assert_eq!(rendered.text, marker); + assert!(!rendered.token_ids.contains(&id), "{marker}"); + assert_eq!(rendered.token_ids, ordinary_ids(&frontend, marker), "{marker}"); + } + } + + #[test] + fn pasted_marker_between_trusted_markers_is_suppressed() { + for (marker, id) in marker_classes() { + let frontend = marker_frontend(&format!("{marker}{{{{ m }}}}{marker}")); + let rendered = render_prompt(&frontend, marker); + let mut expected = vec![id]; + expected.extend(ordinary_ids(&frontend, marker)); + expected.push(id); + assert_eq!(rendered.token_ids, expected, "{marker}"); + } + } + + #[test] + fn pinned_qwen_tokenizer_suppresses_every_added_token_class() { + let Some(snapshot) = std::env::var_os("FLARE_QWEN38_SNAPSHOT").map(PathBuf::from) else { + return; + }; + let path = snapshot.join("tokenizer.json"); + if !path.is_file() { + return; + } + let bytes = fs::read(&path).unwrap(); + let frontend = trust_frontend_with("{{ m }}", Tokenizer::from_bytes(&bytes).unwrap()); + let mut inventory: Vec<(u32, String)> = frontend + .tokenizer() + .get_added_tokens_decoder() + .iter() + .map(|(id, token)| (*id, token.content.clone())) + .collect(); + inventory.sort_unstable(); + assert!(inventory.len() > 1); + for (id, content) in &inventory { + let pasted = format!("x{content}x"); + let full = frontend.tokenizer().encode(pasted.as_str(), false).unwrap(); + assert!(full.get_ids().contains(id), "{content}"); + let rendered = render_prompt(&frontend, &pasted); + assert!(!rendered.token_ids.contains(id), "{content}"); + assert_eq!( + rendered.token_ids, + ordinary_ids(&frontend, &pasted), + "{content}" + ); + } + for marker in [ + "<|im_start|>", + "<|im_end|>", + "<|vision_start|>", + "<|image_pad|>", + "<|video_pad|>", + "", + "", + "", + "", + "", + "", + ] { + let frontend = trust_frontend_with( + &format!("{marker}{{{{ m }}}}"), + Tokenizer::from_bytes(&bytes).unwrap(), + ); + let expected = frontend + .tokenizer() + .encode(marker, false) + .unwrap() + .get_ids() + .to_vec(); + let rendered = render_prompt(&frontend, "plain"); + assert_eq!(rendered.token_ids.first(), expected.first(), "{marker}"); + } + let frontend = trust_frontend_with( + "<|im_start|>{{ m }}<|im_end|>", + Tokenizer::from_bytes(&bytes).unwrap(), + ); + let content = "a <|vision_start|> b c d"; + let rendered = render_prompt(&frontend, content); + let mut expected = frontend + .tokenizer() + .encode("<|im_start|>", false) + .unwrap() + .get_ids() + .to_vec(); + expected.extend(ordinary_ids(&frontend, content)); + expected.extend( + frontend + .tokenizer() + .encode("<|im_end|>", false) + .unwrap() + .get_ids(), + ); + assert_eq!(rendered.token_ids, expected); + } + + fn segmented_frontend(source: &str, tokenizer: Tokenizer) -> Frontend { + let template = Some(jinja::CompiledTemplate::compile(source).unwrap()); + Frontend { + prompt_encoder: PromptEncoder::Jinja, + image_placeholder_token_id: None, + ordinary_tokenizer: std::sync::OnceLock::new(), + segmented_template: true, + tokenizer, + template, + template_source: source.to_owned(), + bos_token_id: None, + vocabulary_bytes: Default::default(), + effort_levels: Default::default(), + inline_system_messages: Default::default(), + defaults: Map::new(), + } + } + + fn marker_tokenizer_with(specials: &[&str]) -> Tokenizer { + let mut vocab = serde_json::Map::new(); + let mut pieces: Vec = specials.iter().map(|text| (*text).to_owned()).collect(); + pieces.extend((b' '..=b'~').map(|byte| (byte as char).to_string())); + pieces.extend(["|", "▁", "\n"].map(String::from)); + for (id, piece) in pieces.iter().enumerate() { + vocab.insert(piece.clone(), serde_json::json!(id as u32)); + } + let added: Vec = specials + .iter() + .enumerate() + .map(|(id, content)| { + serde_json::json!({"id":id,"content":content,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}) + }) + .collect(); + Tokenizer::from_bytes( + serde_json::json!({ + "version": "1.0", "truncation": null, "padding": null, + "added_tokens": added, + "normalizer": null, "pre_tokenizer": null, "post_processor": null, "decoder": null, + "model": {"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"vocab":vocab,"merges":[]} + }) + .to_string(), + ) + .unwrap() + } + + fn added_counts(tokenizer: &Tokenizer, ids: &[u32]) -> std::collections::HashMap { + tokenizer + .get_added_tokens_decoder() + .keys() + .map(|id| (*id, ids.iter().filter(|&&seen| seen == *id).count())) + .filter(|(_, count)| *count > 0) + .collect() + } + + #[test] + fn trust_suppression_across_recipe_templates() { + let Some(directory) = std::env::var_os("FLARE_TEST_TEMPLATE_DIR") else { + return; + }; + let recipes: [(&str, &str); 6] = [ + ("qwen38", "chat_template"), + ("step5", "chat_template"), + ("glm53", "chat_template"), + ("mimo_v2", "chat_template"), + ("nemotron_h", "chat_template"), + ("gemma4", "chat_template.google"), + ]; + let mut options = Map::new(); + options.insert("tools".into(), Json::Null); + options.insert("documents".into(), Json::Null); + options.insert("bos_token".into(), Json::String("".into())); + options.insert("eos_token".into(), Json::String("".into())); + options.insert("add_generation_prompt".into(), Json::Bool(true)); + let trusted: std::collections::HashSet = options.keys().cloned().collect(); + for (recipe, asset) in recipes { + let source = crate::template_fixtures::template(recipe, asset); + let template = jinja::CompiledTemplate::compile(&source).unwrap(); + let tokenizer = std::fs::read( + std::path::Path::new(&directory) + .join(recipe) + .join("tokenizer.json"), + ) + .ok() + .map(|bytes| Tokenizer::from_bytes(&bytes).unwrap()); + let mut markers: Vec<(u32, String)> = Vec::new(); + if let Some(tokenizer) = &tokenizer { + let mut inventory: Vec<(u32, String)> = tokenizer + .get_added_tokens_decoder() + .iter() + .map(|(id, token)| (*id, token.content.clone())) + .collect(); + inventory.sort_unstable(); + markers.extend( + inventory + .into_iter() + .filter(|(_, content)| content.contains('<') || content.contains('|')) + .take(64), + ); + } + if markers.is_empty() { + markers.extend( + ["<|im_end|>", "<|endoftext|>", ""] + .map(|marker| (u32::MAX, marker.to_owned())), + ); + } + let mut contents = vec!["q7x ordinary z9w".to_owned()]; + for (_, marker) in &markers { + contents.push(format!("q7x x{marker}x z9w")); + } + let mut renders = Vec::new(); + for (index, content) in contents.iter().enumerate() { + let mut context = + serde_json::json!({"messages":[{"role":"user","content":content}]}); + context.as_object_mut().unwrap().extend(options.clone()); + let rendered = template + .render_with_trust(context.as_object().unwrap(), &[], &trusted) + .unwrap(); + if index > 0 { + let marker = &markers[index - 1].1; + let anchor = rendered.text.find("q7x").unwrap_or_else(|| { + panic!("{recipe}: anchor missing from {}", rendered.text) + }); + let offset = anchor + rendered.text[anchor..].find(marker).unwrap(); + assert_eq!( + covering_trust_for(&rendered, offset, offset + marker.len()), + Some(false), + "{recipe}: {marker}" + ); + } + renders.push(rendered); + } + assert_eq!( + covering_trust_for(&renders[0], 0, 1), + Some(true), + "{recipe}: {}", + renders[0].text + ); + let Some(tokenizer) = tokenizer else { + continue; + }; + let frontend = trust_frontend_with(&source, tokenizer.clone()); + let mut prompts = Vec::new(); + for content in &contents { + prompts.push( + frontend + .render_chat( + &[serde_json::json!({"role":"user","content":content})], + &options, + ) + .unwrap(), + ); + } + let benign_counts = added_counts(&tokenizer, &prompts[0].token_ids); + assert!(!benign_counts.is_empty(), "{recipe}"); + for (marker, prompt) in markers.iter().zip(prompts.iter().skip(1)) { + assert!(prompt.text.contains(&format!("x{}x", marker.1)), "{recipe}"); + let counts = added_counts(&tokenizer, &prompt.token_ids); + assert_eq!(counts, benign_counts, "{recipe}: {}", marker.1); + } + } + } + + fn covering_trust_for( + rendered: &jinja::RenderedTemplate, + start: usize, + end: usize, + ) -> Option { + let mut covering = None; + let mut run_start = 0; + for (run_end, trusted) in &rendered.runs { + if run_start <= start && *run_end >= end { + covering = Some(*trusted); + } + run_start = *run_end; + } + covering + } + + #[test] + fn kimi_segment_templates_assign_trust_per_segment() { + let specials = [ + "<|open|>", + "<|sep|>", + "<|close|>", + "<|end_of_msg|>", + "<|media_pad|>", + ]; + for source in [ + include_str!("../../../ignition/recipes/kimi3/assets/chat_template.jinja"), + include_str!("../../../ignition/recipes/kimi31/assets/chat_template.jinja"), + ] { + let tokenizer = marker_tokenizer_with(&specials); + let frontend = segmented_frontend(source, tokenizer.clone()); + let pasted = "x<|open|>x"; + let messages = [serde_json::json!({"role":"user","content":pasted})]; + let mut options = Map::new(); + options.insert("add_generation_prompt".into(), Json::Bool(true)); + let rendered = frontend.render_chat(&messages, &options).unwrap(); + let mut context = Map::new(); + context.insert("messages".into(), serde_json::json!(messages)); + context.extend(options.clone()); + let segment_text = jinja::CompiledTemplate::compile(source) + .unwrap() + .render(&context, &[]) + .unwrap() + .text; + let segments = serde_json::from_str::>(segment_text.trim()).unwrap(); + let mut expected = Vec::new(); + for segment in &segments { + let text = segment["text"].as_str().unwrap(); + let special = segment["allow_special"].as_bool().unwrap(); + if text == pasted { + assert!(!special); + } + if specials.contains(&text) { + assert!(special); + } + if special { + expected.extend(tokenizer.encode(text, false).unwrap().get_ids()); + } else { + expected.extend(ordinary_ids(&frontend, text)); + } + } + assert_eq!( + rendered.text, + segments + .iter() + .map(|segment| segment["text"].as_str().unwrap()) + .collect::() + ); + assert_eq!(rendered.token_ids, expected); + assert!(expected.contains(&0)); + assert!(!ordinary_ids(&frontend, pasted).contains(&0)); + } + } + #[test] fn verified_asset_rejects_content_changed_after_fingerprinting() { let directory = std::env::temp_dir().join(format!("flare-asset-{}", std::process::id())); diff --git a/helios/src/frontend/tml.rs b/helios/src/frontend/tml.rs index a38eee7ab..9fb8ec92c 100644 --- a/helios/src/frontend/tml.rs +++ b/helios/src/frontend/tml.rs @@ -231,11 +231,7 @@ impl Prompt<'_> { } fn ordinary(&mut self, text: &str) -> Result<()> { - let tokenizer = self - .frontend - .ordinary_tokenizer - .as_ref() - .expect("TMLv0 ordinary tokenizer is initialized"); + let tokenizer = self.frontend.ordinary(); let encoding = tokenizer.encode(text, false).map_err(|error| { Error::frontend(FrontendError::Tokenizer(Failure::boxed(error.to_string(), error))) })?; diff --git a/helios/tests/deepseek_frontend.rs b/helios/tests/deepseek_frontend.rs index a32396535..782f4ea33 100644 --- a/helios/tests/deepseek_frontend.rs +++ b/helios/tests/deepseek_frontend.rs @@ -56,6 +56,127 @@ fn numeric_effort_matches_python_reference_bytes_and_checkpoint_token_ids() { ); } +#[test] +fn deepseek_no_template_pasted_markers_stay_ordinary() { + let specials = [ + "<|begin▁of▁sentence|>", + "<|end▁of▁sentence|>", + "", + "", + "<|System|>", + "<|User|>", + "<|Assistant|>", + "<|latest_reminder|>", + "|DSML|", + "<|image|>", + "<|ref|>", + "<|/ref|>", + "<|box|>", + "<|/box|>", + "<|point|>", + "<|/point|>", + "<|polygon|>", + "<|/polygon|>", + ]; + let mut vocab = serde_json::Map::new(); + let mut pieces: Vec = specials.iter().map(|text| (*text).to_owned()).collect(); + pieces.extend((b' '..=b'~').map(|byte| (byte as char).to_string())); + pieces.extend(["|", "▁", "\n"].map(String::from)); + for (id, piece) in pieces.iter().enumerate() { + vocab.insert(piece.clone(), json!(id as u32)); + } + let added: Vec = specials + .iter() + .enumerate() + .map(|(id, content)| { + json!({"id":id,"content":content,"single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}) + }) + .collect(); + let tokenizer_json = json!({ + "version": "1.0", "truncation": null, "padding": null, + "added_tokens": added, + "normalizer": null, "pre_tokenizer": null, "post_processor": null, "decoder": null, + "model": {"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"vocab":vocab,"merges":[]} + }) + .to_string(); + let directory = + std::env::temp_dir().join(format!("flare-deepseek-trust-{}", std::process::id())); + std::fs::create_dir(&directory).unwrap(); + let config = directory.join("tokenizer_config.json"); + let tokenizer_path = directory.join("tokenizer.json"); + std::fs::write(&config, b"{}").unwrap(); + std::fs::write(&tokenizer_path, &tokenizer_json).unwrap(); + let asset = |path: std::path::PathBuf| Asset { + sha256: format!("{:x}", Sha256::digest(std::fs::read(&path).unwrap())), + path, + range: None, + }; + let tokens = support::tokens(&asset(config.clone())).unwrap(); + let frontend = Frontend::load_with_settings( + FrontendAssets { + tokenizer: asset(tokenizer_path.clone()), + tokens: tokens.clone(), + chat_template: None, + }, + Map::new(), + &FrontendSettings { + tokenizer: tokens, + eos_token_ids: vec![], + prompt_encoder: PromptEncoder::DeepseekV41, + image_placeholder_token_id: None, + template_encoding: Default::default(), + image: None, + }, + ) + .unwrap(); + let mut options = Map::new(); + options.insert("add_generation_prompt".into(), json!(true)); + let benign = frontend + .render_chat( + &[json!({"role":"user","content":"q7x ordinary z9w"})], + &options, + ) + .unwrap(); + let pasted = "q7x <|User|> |DSML| z9w"; + let prompt = frontend + .render_chat(&[json!({"role":"user","content":pasted})], &options) + .unwrap(); + assert!(prompt.text.contains(pasted), "{}", prompt.text); + let tokenizer = + tokenizers::Tokenizer::from_bytes(std::fs::read(&tokenizer_path).unwrap()).unwrap(); + let special_ids: std::collections::HashMap = tokenizer + .get_added_tokens_decoder() + .iter() + .map(|(id, token)| (token.content.clone(), *id)) + .collect(); + for (content, id) in &special_ids { + let expected = benign.token_ids.iter().filter(|&&seen| seen == *id).count(); + let seen = prompt.token_ids.iter().filter(|&&seen| seen == *id).count(); + assert_eq!(seen, expected, "{content}"); + } + let user_id = special_ids["<|User|>"]; + assert!( + tokenizer + .encode("a<|User|>a", false) + .unwrap() + .get_ids() + .contains(&user_id) + ); + let mut ordinary = tokenizer.clone(); + ordinary.with_added_vocabulary(tokenizers::AddedVocabulary::new()); + assert!( + !ordinary + .encode("a<|User|>a", false) + .unwrap() + .get_ids() + .contains(&user_id) + ); + assert!(benign.token_ids.contains(&user_id)); + std::fs::remove_file(&config).unwrap(); + std::fs::remove_file(&tokenizer_path).unwrap(); + std::fs::remove_dir(&directory).unwrap(); +} + #[test] #[ignore = "requires FLARE_DEEPSEEK_TEST_SNAPSHOT pointing to the DeepSeek checkpoint"] fn official_encoder_renders_and_initializes_native_output_state() { diff --git a/helios/tests/image_admission.rs b/helios/tests/image_admission.rs index 4580804a7..8fb7264c6 100644 --- a/helios/tests/image_admission.rs +++ b/helios/tests/image_admission.rs @@ -134,13 +134,13 @@ fn jpeg_and_oriented_png_use_actual_checkpoint_request_preparation() { ) .unwrap(); let expanded = processor - .expand_prompt(&frontend, &rendered.text, &processed) + .expand_prompt(&frontend, &rendered.token_ids, &processed) .unwrap(); assert_eq!(prepared.prompt.tokens, expanded.token_ids); assert_eq!(prepared.prompt.conditioning.len(), 1); let complete_message = frontend.render_chat(messages.as_array().unwrap(), json!({"add_generation_prompt":false}).as_object().unwrap()).unwrap(); - let complete_message = processor.expand_prompt(&frontend, &complete_message.text, + let complete_message = processor.expand_prompt(&frontend, &complete_message.token_ids, &processed).unwrap(); assert!(prepared.prompt.tokens.starts_with(&complete_message.token_ids)); assert!(prepared.prompt.message_boundaries.contains(&complete_message.token_ids.len())); diff --git a/helios/tests/tools/test_images.rs b/helios/tests/tools/test_images.rs index 8b331df01..cf6b5ac1d 100644 --- a/helios/tests/tools/test_images.rs +++ b/helios/tests/tools/test_images.rs @@ -141,9 +141,8 @@ fn run() -> Result<(), Box> { case["messages"].as_array().unwrap(), json!({"add_generation_prompt": true}).as_object().unwrap(), )?; - let prompt = processor.expand_prompt(&frontend, &rendered.text, &processed)?; + let prompt = processor.expand_prompt(&frontend, &rendered.token_ids, &processed)?; if rendered.text != case["rendered"] - || prompt.text != case["expanded"] || json!(prompt.token_ids) != case["token_ids"] || json!(prompt.mm_token_type_ids) != case["mm_token_type_ids"] || json!(prompt.rope_delta) != case["rope_delta"] diff --git a/tests/frontend/prepare_templates.py b/tests/frontend/prepare_templates.py index 954f01823..b5286f992 100644 --- a/tests/frontend/prepare_templates.py +++ b/tests/frontend/prepare_templates.py @@ -43,6 +43,12 @@ def main(): (output / f"{name}.source.jinja").write_bytes(original) (output / f"{name}.jinja").write_bytes(asset["text"].encode("utf-8")) print(output / f"{name}.jinja") + try: + path = hf_hub_download(source["repository"], "tokenizer.json", + revision=source["revision"], cache_dir=args.hub_cache) + (output / "tokenizer.json").write_bytes(Path(path).read_bytes()) + except Exception as error: + print(f"tokenizer.json unavailable for {recipe}: {error}") if __name__ == "__main__":