Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions helios/src/frontend/deepseek.rs
Original file line number Diff line number Diff line change
Expand Up @@ -437,10 +437,7 @@ pub(super) fn render(
.collect::<Result<Vec<_>>>()?;
let mut token_ids = Vec::new();
let mut offsets = Vec::new();
let plain = frontend
.ordinary_tokenizer
.as_ref()
.expect("DeepSeek plain tokenizer is initialized");
let plain = frontend.ordinary();
let mut start = 0;
let mut image_count = 0;
for range in &rendered.special_ranges {
Expand Down
4 changes: 2 additions & 2 deletions helios/src/frontend/images/layouts/bracketed.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ pub(crate) fn prepare_input(
interpolation_weights: vec![],
cu_seqlens: vec![0],
};
processor.expand_prompt(frontend, &rendered.text, &empty)?;
processor.expand_prompt(frontend, &rendered.token_ids, &empty)?;
}
return Ok(PreparedInput {
tokens: rendered.token_ids.clone(),
Expand All @@ -46,7 +46,7 @@ pub(crate) fn prepare_input(
}
let processor = processor.ok_or("image processing is unavailable")?;
let processed = processor.describe_inputs(images)?;
let expanded = processor.expand_prompt(frontend, &rendered.text, &processed)?;
let expanded = processor.expand_prompt(frontend, &rendered.token_ids, &processed)?;
let mut conditioning = Vec::new();
let mut image = 0;
let mut feature = 0;
Expand Down
2 changes: 1 addition & 1 deletion helios/src/frontend/images/layouts/qwen.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ pub(crate) fn prepare_input(
} else {
let processor = processor.ok_or("image processing is unavailable")?;
let processed = processor.describe_inputs(images)?;
let expanded = processor.expand_prompt(frontend, &rendered.text, &processed)?;
let expanded = processor.expand_prompt(frontend, &rendered.token_ids, &processed)?;
let positions = (0..expanded.token_ids.len())
.map(|position| {
[
Expand Down
11 changes: 7 additions & 4 deletions helios/src/frontend/images/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,6 @@ impl ProcessedImages {
}

pub struct ImagePrompt {
pub text: String,
pub token_ids: Vec<u32>,
pub mm_token_type_ids: Vec<i32>,
/// Text, temporal, height, and width position rows, each of length token_ids.len().
Expand Down Expand Up @@ -718,12 +717,16 @@ impl ImageProcessor {
pub fn expand_prompt(
&self,
frontend: &Frontend,
text: &str,
prompt_token_ids: &[u32],
images: &ProcessedImages,
) -> Result<ImagePrompt> {
match &self.profile {
Profile::Qwen(profile) => profile.expand_prompt(frontend, text, images),
Profile::PatchMerge(profile) => profile.expand_prompt(frontend, text, images),
Profile::Qwen(profile) => {
profile.expand_prompt(frontend, prompt_token_ids, images)
}
Profile::PatchMerge(profile) => {
profile.expand_prompt(frontend, prompt_token_ids, images)
}
Profile::DeepSeek4 | Profile::DeepseekRowMajor(_) => Err("DeepSeek image spans require its model contract".into()),
Profile::MiMo(_) => Err("MiMo image expansion requires its model contract".into()),
Profile::Moonvit => Err("MoonViT image expansion requires its model contract".into()),
Expand Down
32 changes: 20 additions & 12 deletions helios/src/frontend/images/patch_merge.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ impl Processor {
pub(super) fn expand_prompt(
&self,
frontend: &Frontend,
text: &str,
prompt_token_ids: &[u32],
images: &ProcessedImages,
) -> Result<ImagePrompt> {
let image_ids = frontend.encode(&self.image_token, false)?;
Expand All @@ -71,26 +71,34 @@ impl Processor {
return Err("image envelope marker must encode one token".into());
}
}
if text.matches(&self.image_token).count() != images.geometry.len() {
let placeholder_count = prompt_token_ids
.iter()
.filter(|id| **id == *image_id)
.count();
if placeholder_count != images.geometry.len() {
return Err("image placeholder count does not match input images".into());
}
let mut parts = text.split(&self.image_token);
let mut expanded = parts.next().unwrap_or_default().to_owned();
for (part, geometry) in parts.zip(&images.geometry) {
expanded.push_str(&self.start_token);
for _ in 0..geometry.tokens {
expanded.push_str(&self.image_token);
let start_id = frontend.encode(&self.start_token, false)?[0];
let end_id = frontend.encode(&self.end_token, false)?[0];
let mut expanded_ids = Vec::new();
let mut image_index = 0usize;
for &id in prompt_token_ids {
if id == *image_id {
let tokens = images.geometry[image_index].tokens;
expanded_ids.push(start_id);
expanded_ids.extend(std::iter::repeat(*image_id).take(tokens));
expanded_ids.push(end_id);
image_index += 1;
} else {
expanded_ids.push(id);
}
expanded.push_str(&self.end_token);
expanded.push_str(part);
}
let token_ids = frontend.encode(&expanded, false)?;
let token_ids = expanded_ids;
let mm_token_type_ids = token_ids
.iter()
.map(|id| i32::from(*id == *image_id))
.collect();
Ok(ImagePrompt {
text: expanded,
position_ids: std::array::from_fn(|_| (0..token_ids.len() as i64).collect()),
token_ids,
mm_token_type_ids,
Expand Down
40 changes: 26 additions & 14 deletions helios/src/frontend/images/qwen.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,34 +78,47 @@ impl QwenProcessor {
pub fn expand_prompt(
&self,
frontend: &Frontend,
text: &str,
prompt_token_ids: &[u32],
images: &ProcessedImages,
) -> Result<ImagePrompt> {
if frontend.encode(IMAGE_TOKEN, false)? != [self.image_token_id] {
return Err(Error::frontend(FrontendError::Input(Failure::new(
"image token ID does not match tokenizer",
))));
}
if text.matches(IMAGE_TOKEN).count() != images.geometry.len() {
let placeholder_count = prompt_token_ids
.iter()
.filter(|id| **id == self.image_token_id)
.count();
if placeholder_count != images.geometry.len() {
return Err(Error::frontend(FrontendError::Input(Failure::new(
"image placeholder count does not match prepared images",
))));
}
let mut expanded = String::with_capacity(text.len());
let mut parts = text.split(IMAGE_TOKEN);
expanded.push_str(parts.next().unwrap_or_default());
for (part, geometry) in parts.zip(&images.geometry) {
for _ in 0..geometry.tokens {
expanded.push_str(IMAGE_TOKEN);
}
expanded.push_str(part);
}
let token_ids = frontend.encode(&expanded, false)?;
if token_ids.contains(&self.video_token_id) {
if prompt_token_ids.contains(&self.video_token_id) {
return Err(Error::frontend(FrontendError::Input(Failure::new(
"video inputs are not supported by the image processor",
))));
}
let mut expanded_ids = Vec::with_capacity(
prompt_token_ids.len()
+ images
.geometry
.iter()
.map(|geometry| geometry.tokens.saturating_sub(1))
.sum::<usize>(),
);
let mut image_index = 0usize;
for &id in prompt_token_ids {
if id == self.image_token_id {
let tokens = images.geometry[image_index].tokens;
expanded_ids.extend(std::iter::repeat(self.image_token_id).take(tokens));
image_index += 1;
} else {
expanded_ids.push(id);
}
}
let token_ids = expanded_ids;
let mm_token_type_ids: Vec<i32> = token_ids
.iter()
.map(|id| i32::from(*id == self.image_token_id))
Expand Down Expand Up @@ -159,7 +172,6 @@ impl QwenProcessor {
))));
}
Ok(ImagePrompt {
text: expanded,
rope_delta: current - token_ids.len() as i64,
token_ids,
mm_token_type_ids,
Expand Down
13 changes: 12 additions & 1 deletion helios/src/frontend/jinja/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ pub enum Mark {
pub struct RenderedTemplate {
pub text: String,
pub marks: Vec<Mark>,
pub runs: Vec<(usize, bool)>,
}

pub struct CompiledTemplate {
Expand All @@ -69,7 +70,17 @@ impl CompiledTemplate {
context: &Map<String, Json>,
hints: &[(usize, usize)],
) -> Result<RenderedTemplate> {
vm::render(self, context, hints)
use std::collections::HashSet;
self.render_with_trust(context, hints, &HashSet::new())
}

pub fn render_with_trust(
&self,
context: &Map<String, Json>,
hints: &[(usize, usize)],
trusted: &std::collections::HashSet<String>,
) -> Result<RenderedTemplate> {
vm::render(self, context, hints, trusted)
}

pub fn annotation_spans(&self) -> &[std::ops::Range<usize>] {
Expand Down
Loading