From 0548d762b3d37c79d214ab0a47d7aa399352178b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 11 Aug 2026 00:39:24 +0300 Subject: [PATCH 01/13] Ignore the root coverage.json the coverage gate writes Co-authored-by: Medulla --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index c880a65..c3fed37 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ *.profraw *.profdata lcov.info +# The per-file coverage gate in AGENTS.md writes this at the repository root. +/coverage.json # Local secrets — never commit; copy .env.example to .env .env From 5f7af1b66487dcf3a426494aa788cb32224d2ab1 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 11 Aug 2026 00:39:24 +0300 Subject: [PATCH 02/13] Move the document spec out from behind the docx feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spec types, their limits, and `DocumentSpec::validate` were reachable only with the `docx` feature on, because they lived in `src/docx/types.rs` behind a gated module. That coupled the wire contract to the OOXML writer: a host whose synthesis happens elsewhere — in another process, or behind the TinyBus module in this repo — had to either pull `docx-rs` in to name `DocumentSpec`, or re-declare the spec and let the two definitions drift. Move them to a new ungated `src/spec/`, which depends on nothing but `serde` and the crate error type, and re-export from `docx` so `tinydocs::docx::DocumentSpec` still names the same type. `cargo check -p tinydocs --no-default-features` now compiles the whole contract with `docx-rs` absent from the dependency graph. Validation moves with the types, so it is no longer gated either — which is the point: validating at a host boundary is the cheap half, and it should not require the codec. Public API: purely additive. `tinydocs::spec` is new; every existing path resolves to the same item. Tests split along the same line. `src/spec/test.rs` owns validation, the blank/aggregate rules and the JSON contract, and must pass with every format feature off; `src/docx/test.rs` keeps the OOXML mapping assertions. Two new cases cover the aggregate-budget branches that only a bullet or a heading can cross, taking `src/spec/document.rs` to 100% line coverage. Co-authored-by: Medulla --- README.md | 26 ++- src/docx/mod.rs | 125 +---------- src/docx/test.rs | 200 +---------------- src/lib.rs | 5 + src/{docx/types.rs => spec/document.rs} | 128 ++++++++++- src/spec/mod.rs | 31 +++ src/spec/test.rs | 276 ++++++++++++++++++++++++ 7 files changed, 475 insertions(+), 316 deletions(-) rename src/{docx/types.rs => spec/document.rs} (50%) create mode 100644 src/spec/mod.rs create mode 100644 src/spec/test.rs diff --git a/README.md b/README.md index 13dcf54..0052aca 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,25 @@ multi-hundred-megabyte document in memory. `DocumentSpec::validate` is public and runs before any synthesis, so a host can reject a bad tool call at its own boundary without paying for a blocking hop. +## The spec is separable from the codec + +Every spec type, every limit, and every `validate` lives in `tinydocs::spec`, +which depends on nothing but `serde` and the crate's own error type. It is +compiled in **every** build, including `--no-default-features`, so: + +```toml +tinydocs = { version = "0.1", default-features = false } +``` + +gives a host the authoritative wire contract and its validation without pulling +in a single format writer. That is what a host does when synthesis happens +somewhere else — in another process, or behind the TinyBus module below — and it +is why such a host does not have to re-declare the spec and let it drift. + +The format modules re-export what they consume, so `tinydocs::docx::DocumentSpec` +and `tinydocs::spec::DocumentSpec` name the same type and existing code keeps +compiling. + ## TinyBus module The private `tinydocs-module` workspace crate builds TinyDocs as a trusted @@ -120,9 +139,12 @@ src/ ├── error/ │ ├── mod.rs # crate-wide `Error` and `Result` │ └── test.rs +├── spec/ # wire contracts — ungated, serde only +│ ├── mod.rs # re-export surface +│ ├── document.rs # `DocumentSpec`, `DocumentSection`, limits, `validate` +│ └── test.rs ├── docx/ - ├── mod.rs # `generate` + spec validation - ├── types.rs # `DocumentSpec`, `DocumentSection`, limits + ├── mod.rs # `generate` — the OOXML mapping └── test.rs tests/ └── public_api.rs # integration tests against the public API only diff --git a/src/docx/mod.rs b/src/docx/mod.rs index 783f65c..fa3fa50 100644 --- a/src/docx/mod.rs +++ b/src/docx/mod.rs @@ -27,9 +27,10 @@ //! Whitespace-only paragraphs and bullets are trimmed away rather than //! emitting an empty run. -mod types; - -pub use types::{ +// The spec is defined in `crate::spec`, which is compiled in every build so a +// host can share the wire contract without the OOXML writer stack. Re-exported +// here so `tinydocs::docx::DocumentSpec` keeps naming the same type. +pub use crate::spec::{ DocumentSection, DocumentSpec, MAX_BULLETS_PER_SECTION, MAX_PARAGRAPH_CHARS, MAX_PARAGRAPHS_PER_SECTION, MAX_SECTIONS, MAX_TEXT_CHARS, MAX_TOTAL_CHARS, }; @@ -54,124 +55,6 @@ const HEADING_SIZE_HALF_PT: usize = 32; /// Run font size for the author byline, in half-points (12 pt). const AUTHOR_SIZE_HALF_PT: usize = 24; -impl DocumentSpec { - /// Check the spec against every documented size limit. - /// - /// Callers do not have to invoke this: [`generate`] validates before it - /// synthesises anything. It is public so a host can reject a malformed - /// spec at its own boundary — an LLM tool call, say — and hand back the - /// structured [`Error::InvalidInput`] before paying for a blocking hop. - /// - /// # Errors - /// - /// Returns [`Error::InvalidInput`] naming the first field that violates a - /// limit. Fields are checked in spec order (title, author, sections, then - /// each section's contents) so the reported field is stable for a given - /// spec. - pub fn validate(&self) -> Result<()> { - if self.title.trim().is_empty() { - return Err(Error::invalid_input("title", "must not be empty")); - } - if self.title.chars().count() > MAX_TEXT_CHARS { - return Err(Error::invalid_input( - "title", - format!("must be ≤ {MAX_TEXT_CHARS} chars"), - )); - } - // Running total across every renderable field — title, author, and all - // section contents — checked as each field is processed. A spec can pass - // every per-field limit yet blow the aggregate budget, and checking - // incrementally rejects it as soon as the budget is crossed without a - // second pass over the whole spec. - let over_budget = || { - Error::invalid_input( - "sections", - format!("total document text must be ≤ {MAX_TOTAL_CHARS} chars"), - ) - }; - let mut total = self.title.chars().count(); - if let Some(author) = self.author.as_deref() { - if author.chars().count() > MAX_TEXT_CHARS { - return Err(Error::invalid_input( - "author", - format!("must be ≤ {MAX_TEXT_CHARS} chars"), - )); - } - total = total.saturating_add(author.chars().count()); - } - if self.sections.is_empty() { - return Err(Error::invalid_input( - "sections", - "must contain at least one section", - )); - } - if self.sections.len() > MAX_SECTIONS { - return Err(Error::invalid_input( - "sections", - format!("must contain ≤ {MAX_SECTIONS} sections"), - )); - } - - for (i, section) in self.sections.iter().enumerate() { - if section.is_blank() { - return Err(Error::invalid_input( - format!("sections[{i}]"), - "must have at least one of heading / paragraphs / bullets", - )); - } - if let Some(heading) = section.heading.as_deref() { - if heading.chars().count() > MAX_TEXT_CHARS { - return Err(Error::invalid_input( - format!("sections[{i}].heading"), - format!("must be ≤ {MAX_TEXT_CHARS} chars"), - )); - } - total = total.saturating_add(heading.chars().count()); - if total > MAX_TOTAL_CHARS { - return Err(over_budget()); - } - } - if section.paragraphs.len() > MAX_PARAGRAPHS_PER_SECTION { - return Err(Error::invalid_input( - format!("sections[{i}].paragraphs"), - format!("must contain ≤ {MAX_PARAGRAPHS_PER_SECTION} paragraphs"), - )); - } - for (p, paragraph) in section.paragraphs.iter().enumerate() { - if paragraph.chars().count() > MAX_PARAGRAPH_CHARS { - return Err(Error::invalid_input( - format!("sections[{i}].paragraphs[{p}]"), - format!("must be ≤ {MAX_PARAGRAPH_CHARS} chars"), - )); - } - total = total.saturating_add(paragraph.chars().count()); - if total > MAX_TOTAL_CHARS { - return Err(over_budget()); - } - } - if section.bullets.len() > MAX_BULLETS_PER_SECTION { - return Err(Error::invalid_input( - format!("sections[{i}].bullets"), - format!("must contain ≤ {MAX_BULLETS_PER_SECTION} bullets"), - )); - } - for (b, bullet) in section.bullets.iter().enumerate() { - if bullet.chars().count() > MAX_PARAGRAPH_CHARS { - return Err(Error::invalid_input( - format!("sections[{i}].bullets[{b}]"), - format!("must be ≤ {MAX_PARAGRAPH_CHARS} chars"), - )); - } - total = total.saturating_add(bullet.chars().count()); - if total > MAX_TOTAL_CHARS { - return Err(over_budget()); - } - } - } - Ok(()) - } -} - /// Validate `spec` and synthesise it into `.docx` bytes. /// /// The returned buffer is a complete OOXML zip container: any Word-compatible diff --git a/src/docx/test.rs b/src/docx/test.rs index f724245..1ea7f65 100644 --- a/src/docx/test.rs +++ b/src/docx/test.rs @@ -1,11 +1,14 @@ -//! Unit tests for `.docx` validation and synthesis. +//! Unit tests for `.docx` synthesis. +//! +//! The spec's validation, blank/aggregate rules, and JSON contract are tested +//! in `crate::spec` — they are format-independent and must pass in a build with +//! this feature off. What is left here is the OOXML mapping itself: container +//! shape, which text reaches `word/document.xml`, and the blank-filtering that +//! decides how many paragraphs are emitted. #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use super::{ - DocumentSection, DocumentSpec, MAX_BULLETS_PER_SECTION, MAX_PARAGRAPH_CHARS, - MAX_PARAGRAPHS_PER_SECTION, MAX_SECTIONS, MAX_TEXT_CHARS, MAX_TOTAL_CHARS, generate, -}; +use super::{DocumentSection, DocumentSpec, generate}; use crate::Error; /// One valid section carrying a heading, a paragraph, and a bullet. @@ -17,7 +20,7 @@ fn section() -> DocumentSection { } } -/// A minimal valid spec; each test mutates one field to drive a single branch. +/// A minimal valid spec. fn spec() -> DocumentSpec { DocumentSpec { title: "Charter".to_string(), @@ -26,16 +29,6 @@ fn spec() -> DocumentSpec { } } -/// Assert `spec` is rejected with an `InvalidInput` naming `field`. -fn assert_rejects(spec: &DocumentSpec, field: &str) { - match spec.validate() { - Err(Error::InvalidInput { field: f, .. }) => { - assert_eq!(f, field, "unexpected rejected field"); - } - other => panic!("expected InvalidInput({field}), got {other:?}"), - } -} - /// Entry names inside a produced `.docx` byte buffer. fn entry_names(bytes: &[u8]) -> Vec { let mut zip = @@ -55,158 +48,6 @@ fn entry_body(bytes: &[u8], name: &str) -> String { body } -#[test] -fn accepts_a_well_formed_spec() { - assert!(spec().validate().is_ok()); -} - -#[test] -fn rejects_a_blank_title() { - let mut s = spec(); - s.title = " ".to_string(); - assert_rejects(&s, "title"); -} - -#[test] -fn rejects_an_over_long_title() { - let mut s = spec(); - s.title = "t".repeat(MAX_TEXT_CHARS + 1); - assert_rejects(&s, "title"); -} - -#[test] -fn rejects_an_over_long_author() { - let mut s = spec(); - s.author = Some("a".repeat(MAX_TEXT_CHARS + 1)); - assert_rejects(&s, "author"); -} - -#[test] -fn rejects_a_spec_with_no_sections() { - let mut s = spec(); - s.sections.clear(); - assert_rejects(&s, "sections"); -} - -#[test] -fn rejects_too_many_sections() { - let mut s = spec(); - s.sections = vec![section(); MAX_SECTIONS + 1]; - assert_rejects(&s, "sections"); -} - -#[test] -fn rejects_a_wholly_blank_section() { - // Every entry is present but whitespace-only, so synthesis would drop all - // of them and render nothing. Validation catches it instead. - let mut s = spec(); - s.sections = vec![DocumentSection { - heading: Some(" ".to_string()), - paragraphs: vec!["\t".to_string()], - bullets: vec![String::new()], - }]; - assert_rejects(&s, "sections[0]"); -} - -#[test] -fn rejects_an_over_long_heading_naming_its_index() { - let mut s = spec(); - s.sections.push(DocumentSection { - heading: Some("h".repeat(MAX_TEXT_CHARS + 1)), - ..section() - }); - assert_rejects(&s, "sections[1].heading"); -} - -#[test] -fn rejects_too_many_paragraphs() { - let mut s = spec(); - s.sections[0].paragraphs = vec!["p".to_string(); MAX_PARAGRAPHS_PER_SECTION + 1]; - assert_rejects(&s, "sections[0].paragraphs"); -} - -#[test] -fn rejects_an_over_long_paragraph_naming_its_index() { - let mut s = spec(); - s.sections[0].paragraphs = vec!["ok".to_string(), "p".repeat(MAX_PARAGRAPH_CHARS + 1)]; - assert_rejects(&s, "sections[0].paragraphs[1]"); -} - -#[test] -fn rejects_too_many_bullets() { - let mut s = spec(); - s.sections[0].bullets = vec!["b".to_string(); MAX_BULLETS_PER_SECTION + 1]; - assert_rejects(&s, "sections[0].bullets"); -} - -#[test] -fn rejects_an_over_long_bullet_naming_its_index() { - let mut s = spec(); - s.sections[0].bullets = vec!["ok".to_string(), "b".repeat(MAX_PARAGRAPH_CHARS + 1)]; - assert_rejects(&s, "sections[0].bullets[1]"); -} - -#[test] -fn rejects_a_spec_over_the_aggregate_character_budget() { - // Each individual field is within its own limit; only the sum is not. One - // section with just enough max-length paragraphs to cross MAX_TOTAL_CHARS - // reproduces that without allocating hundreds of megabytes: repeating a - // whole section MAX_SECTIONS times (the original fixture) built ~512 MB - // of paragraph text before validation ever ran. - let paragraph_count = MAX_TOTAL_CHARS / MAX_PARAGRAPH_CHARS + 1; - assert!(paragraph_count <= MAX_PARAGRAPHS_PER_SECTION); - let paragraph = "x".repeat(MAX_PARAGRAPH_CHARS); - let big = DocumentSection { - heading: Some("Heading".to_string()), - paragraphs: vec![paragraph; paragraph_count], - bullets: vec![], - }; - let s = DocumentSpec { - title: "Huge".to_string(), - author: None, - sections: vec![big], - }; - // Sanity: this spec passes every per-field check. - assert!(s.sections.len() <= MAX_SECTIONS); - assert_rejects(&s, "sections"); -} - -#[test] -fn is_blank_reflects_content_presence() { - assert!(!section().is_blank()); - assert!( - DocumentSection { - heading: None, - paragraphs: vec![], - bullets: vec![], - } - .is_blank() - ); - // A heading alone is enough content. - assert!( - !DocumentSection { - heading: Some("Only a heading".to_string()), - paragraphs: vec![], - bullets: vec![], - } - .is_blank() - ); -} - -#[test] -fn total_chars_sums_every_text_field() { - let s = DocumentSpec { - title: "abcd".to_string(), // 4 - author: Some("xy".to_string()), // 2 - sections: vec![DocumentSection { - heading: Some("hij".to_string()), // 3 - paragraphs: vec!["pq".to_string()], // 2 - bullets: vec!["b".to_string()], // 1 - }], - }; - assert_eq!(s.total_chars(), 12); -} - #[test] fn generate_produces_a_readable_ooxml_container() { let bytes = generate(&spec()).expect("generation should succeed"); @@ -322,26 +163,3 @@ fn generate_validates_before_synthesising() { s.title = String::new(); assert!(matches!(generate(&s), Err(Error::InvalidInput { .. }))); } - -#[test] -fn spec_round_trips_through_json() { - let s = spec(); - let json = serde_json::to_string(&s).expect("serialises"); - let back: DocumentSpec = serde_json::from_str(&json).expect("deserialises"); - assert_eq!(back, s); -} - -#[test] -fn spec_rejects_unknown_json_fields() { - // `deny_unknown_fields` makes a typo'd key a loud rejection rather than a - // silently ignored one — the whole point at an LLM tool boundary. - let json = r#"{"title":"T","sections":[],"titel":"typo"}"#; - assert!(serde_json::from_str::(json).is_err()); -} - -#[test] -fn spec_defaults_optional_fields() { - let s: DocumentSpec = serde_json::from_str(r#"{"title":"T"}"#).expect("deserialises"); - assert_eq!(s.author, None); - assert!(s.sections.is_empty()); -} diff --git a/src/lib.rs b/src/lib.rs index f1221b8..e08274a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,6 +24,9 @@ //! # Layout //! //! - [`error`](self::Error) — the crate-wide [`Error`] and [`Result`]. +//! - [`spec`] — the typed document specs and their validation. Compiled in +//! every build, including `--no-default-features`, so a host whose synthesis +//! happens elsewhere still shares one definition of the wire contract. #![cfg_attr( feature = "docx", doc = "- [`docx`] — `.docx` (OOXML `WordprocessingML`) synthesis." @@ -62,6 +65,8 @@ mod error; +pub mod spec; + #[cfg(feature = "docx")] pub mod docx; diff --git a/src/docx/types.rs b/src/spec/document.rs similarity index 50% rename from src/docx/types.rs rename to src/spec/document.rs index 56a41d1..3c293da 100644 --- a/src/docx/types.rs +++ b/src/spec/document.rs @@ -1,6 +1,5 @@ //! The `.docx` document spec: the typed description a caller hands to -//! [`generate`](super::generate), plus the size limits every spec is -//! validated against. +//! `docx::generate`, plus the size limits every spec is validated against. //! //! The spec is the crate's wire contract. It derives `Serialize` / //! `Deserialize` with `deny_unknown_fields` because the usual caller is an @@ -11,9 +10,17 @@ //! Limits are public consts rather than private constants so a host can quote //! the exact number in its own tool description and stay in lockstep with what //! validation actually enforces. +//! +//! Nothing in this module depends on the `docx` feature or on `docx-rs`: it is +//! `serde` plus the crate error type. A host that only needs to *describe* and +//! *validate* a document — because synthesis happens elsewhere, in another +//! process or behind a message bus — can therefore depend on this crate with +//! `default-features = false` and still share one definition of the contract. use serde::{Deserialize, Serialize}; +use crate::{Error, Result}; + /// Maximum number of sections a single document may contain. /// /// Bounds generation time and output size; a caller with more material is @@ -131,4 +138,121 @@ impl DocumentSpec { } total } + + /// Check the spec against every documented size limit. + /// + /// Callers do not have to invoke this: `docx::generate` validates before it + /// synthesises anything. It is public so a host can reject a malformed + /// spec at its own boundary — an LLM tool call, say — and hand back the + /// structured [`Error::InvalidInput`] before paying for a blocking hop, a + /// process boundary, or a bus round trip. + /// + /// # Errors + /// + /// Returns [`Error::InvalidInput`] naming the first field that violates a + /// limit. Fields are checked in spec order (title, author, sections, then + /// each section's contents) so the reported field is stable for a given + /// spec. + pub fn validate(&self) -> Result<()> { + if self.title.trim().is_empty() { + return Err(Error::invalid_input("title", "must not be empty")); + } + if self.title.chars().count() > MAX_TEXT_CHARS { + return Err(Error::invalid_input( + "title", + format!("must be ≤ {MAX_TEXT_CHARS} chars"), + )); + } + // Running total across every renderable field — title, author, and all + // section contents — checked as each field is processed. A spec can pass + // every per-field limit yet blow the aggregate budget, and checking + // incrementally rejects it as soon as the budget is crossed without a + // second pass over the whole spec. + let over_budget = || { + Error::invalid_input( + "sections", + format!("total document text must be ≤ {MAX_TOTAL_CHARS} chars"), + ) + }; + let mut total = self.title.chars().count(); + if let Some(author) = self.author.as_deref() { + if author.chars().count() > MAX_TEXT_CHARS { + return Err(Error::invalid_input( + "author", + format!("must be ≤ {MAX_TEXT_CHARS} chars"), + )); + } + total = total.saturating_add(author.chars().count()); + } + if self.sections.is_empty() { + return Err(Error::invalid_input( + "sections", + "must contain at least one section", + )); + } + if self.sections.len() > MAX_SECTIONS { + return Err(Error::invalid_input( + "sections", + format!("must contain ≤ {MAX_SECTIONS} sections"), + )); + } + + for (i, section) in self.sections.iter().enumerate() { + if section.is_blank() { + return Err(Error::invalid_input( + format!("sections[{i}]"), + "must have at least one of heading / paragraphs / bullets", + )); + } + if let Some(heading) = section.heading.as_deref() { + if heading.chars().count() > MAX_TEXT_CHARS { + return Err(Error::invalid_input( + format!("sections[{i}].heading"), + format!("must be ≤ {MAX_TEXT_CHARS} chars"), + )); + } + total = total.saturating_add(heading.chars().count()); + if total > MAX_TOTAL_CHARS { + return Err(over_budget()); + } + } + if section.paragraphs.len() > MAX_PARAGRAPHS_PER_SECTION { + return Err(Error::invalid_input( + format!("sections[{i}].paragraphs"), + format!("must contain ≤ {MAX_PARAGRAPHS_PER_SECTION} paragraphs"), + )); + } + for (p, paragraph) in section.paragraphs.iter().enumerate() { + if paragraph.chars().count() > MAX_PARAGRAPH_CHARS { + return Err(Error::invalid_input( + format!("sections[{i}].paragraphs[{p}]"), + format!("must be ≤ {MAX_PARAGRAPH_CHARS} chars"), + )); + } + total = total.saturating_add(paragraph.chars().count()); + if total > MAX_TOTAL_CHARS { + return Err(over_budget()); + } + } + if section.bullets.len() > MAX_BULLETS_PER_SECTION { + return Err(Error::invalid_input( + format!("sections[{i}].bullets"), + format!("must contain ≤ {MAX_BULLETS_PER_SECTION} bullets"), + )); + } + for (b, bullet) in section.bullets.iter().enumerate() { + if bullet.chars().count() > MAX_PARAGRAPH_CHARS { + return Err(Error::invalid_input( + format!("sections[{i}].bullets[{b}]"), + format!("must be ≤ {MAX_PARAGRAPH_CHARS} chars"), + )); + } + total = total.saturating_add(bullet.chars().count()); + if total > MAX_TOTAL_CHARS { + return Err(over_budget()); + } + } + } + Ok(()) + } } diff --git a/src/spec/mod.rs b/src/spec/mod.rs new file mode 100644 index 0000000..bac84f9 --- /dev/null +++ b/src/spec/mod.rs @@ -0,0 +1,31 @@ +//! The wire contracts: typed document specs and their validation, with no +//! dependency on any format writer. +//! +//! Every format module in this crate (`docx`, …) synthesises bytes from a spec +//! defined here. The split matters for two reasons: +//! +//! 1. **A host can share the contract without paying for the codec.** This +//! module is `serde` plus the crate [`Error`](crate::Error) — nothing else. +//! It is compiled in *every* build, including +//! `--no-default-features`, so a host whose synthesis happens elsewhere (in +//! another process, or behind a message bus) still gets the one authoritative +//! definition of the spec instead of re-declaring it and drifting. +//! 2. **Validation is cheap and belongs at the boundary.** The specs validate +//! themselves without touching a writer, so a host can reject a malformed +//! LLM tool call before paying for a blocking hop or a round trip. +//! +//! The format modules re-export the types they consume, so +//! `tinydocs::docx::DocumentSpec` and [`tinydocs::spec::DocumentSpec`] name the +//! same type. +//! +//! [`tinydocs::spec::DocumentSpec`]: DocumentSpec + +mod document; + +pub use document::{ + DocumentSection, DocumentSpec, MAX_BULLETS_PER_SECTION, MAX_PARAGRAPH_CHARS, + MAX_PARAGRAPHS_PER_SECTION, MAX_SECTIONS, MAX_TEXT_CHARS, MAX_TOTAL_CHARS, +}; + +#[cfg(test)] +mod test; diff --git a/src/spec/test.rs b/src/spec/test.rs new file mode 100644 index 0000000..e304bb7 --- /dev/null +++ b/src/spec/test.rs @@ -0,0 +1,276 @@ +//! Unit tests for the wire contracts: validation, the blank/aggregate rules, +//! and JSON round-tripping. +//! +//! These are deliberately separate from the format modules' tests. They must +//! pass in a build with every format feature off, because the spec is the half +//! of the crate a bus- or process-boundary host shares without the codec. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::{ + DocumentSection, DocumentSpec, MAX_BULLETS_PER_SECTION, MAX_PARAGRAPH_CHARS, + MAX_PARAGRAPHS_PER_SECTION, MAX_SECTIONS, MAX_TEXT_CHARS, MAX_TOTAL_CHARS, +}; +use crate::Error; + +/// One valid section carrying a heading, a paragraph, and a bullet. +fn section() -> DocumentSection { + DocumentSection { + heading: Some("Overview".to_string()), + paragraphs: vec!["A body paragraph.".to_string()], + bullets: vec!["A bullet".to_string()], + } +} + +/// A minimal valid spec; each test mutates one field to drive a single branch. +fn spec() -> DocumentSpec { + DocumentSpec { + title: "Charter".to_string(), + author: Some("Alice".to_string()), + sections: vec![section()], + } +} + +/// Assert `spec` is rejected with an `InvalidInput` naming `field`. +fn assert_rejects(spec: &DocumentSpec, field: &str) { + match spec.validate() { + Err(Error::InvalidInput { field: f, .. }) => { + assert_eq!(f, field, "unexpected rejected field"); + } + other => panic!("expected InvalidInput({field}), got {other:?}"), + } +} + +#[test] +fn accepts_a_well_formed_spec() { + assert!(spec().validate().is_ok()); +} + +#[test] +fn rejects_a_blank_title() { + let mut s = spec(); + s.title = " ".to_string(); + assert_rejects(&s, "title"); +} + +#[test] +fn rejects_an_over_long_title() { + let mut s = spec(); + s.title = "t".repeat(MAX_TEXT_CHARS + 1); + assert_rejects(&s, "title"); +} + +#[test] +fn rejects_an_over_long_author() { + let mut s = spec(); + s.author = Some("a".repeat(MAX_TEXT_CHARS + 1)); + assert_rejects(&s, "author"); +} + +#[test] +fn rejects_a_spec_with_no_sections() { + let mut s = spec(); + s.sections.clear(); + assert_rejects(&s, "sections"); +} + +#[test] +fn rejects_too_many_sections() { + let mut s = spec(); + s.sections = vec![section(); MAX_SECTIONS + 1]; + assert_rejects(&s, "sections"); +} + +#[test] +fn rejects_a_wholly_blank_section() { + // Every entry is present but whitespace-only, so synthesis would drop all + // of them and render nothing. Validation catches it instead. + let mut s = spec(); + s.sections = vec![DocumentSection { + heading: Some(" ".to_string()), + paragraphs: vec!["\t".to_string()], + bullets: vec![String::new()], + }]; + assert_rejects(&s, "sections[0]"); +} + +#[test] +fn rejects_an_over_long_heading_naming_its_index() { + let mut s = spec(); + s.sections.push(DocumentSection { + heading: Some("h".repeat(MAX_TEXT_CHARS + 1)), + ..section() + }); + assert_rejects(&s, "sections[1].heading"); +} + +#[test] +fn rejects_too_many_paragraphs() { + let mut s = spec(); + s.sections[0].paragraphs = vec!["p".to_string(); MAX_PARAGRAPHS_PER_SECTION + 1]; + assert_rejects(&s, "sections[0].paragraphs"); +} + +#[test] +fn rejects_an_over_long_paragraph_naming_its_index() { + let mut s = spec(); + s.sections[0].paragraphs = vec!["ok".to_string(), "p".repeat(MAX_PARAGRAPH_CHARS + 1)]; + assert_rejects(&s, "sections[0].paragraphs[1]"); +} + +#[test] +fn rejects_too_many_bullets() { + let mut s = spec(); + s.sections[0].bullets = vec!["b".to_string(); MAX_BULLETS_PER_SECTION + 1]; + assert_rejects(&s, "sections[0].bullets"); +} + +#[test] +fn rejects_an_over_long_bullet_naming_its_index() { + let mut s = spec(); + s.sections[0].bullets = vec!["ok".to_string(), "b".repeat(MAX_PARAGRAPH_CHARS + 1)]; + assert_rejects(&s, "sections[0].bullets[1]"); +} + +#[test] +fn rejects_a_spec_over_the_aggregate_character_budget() { + // Each individual field is within its own limit; only the sum is not. One + // section with just enough max-length paragraphs to cross MAX_TOTAL_CHARS + // reproduces that without allocating hundreds of megabytes: repeating a + // whole section MAX_SECTIONS times (the original fixture) built ~512 MB + // of paragraph text before validation ever ran. + let paragraph_count = MAX_TOTAL_CHARS / MAX_PARAGRAPH_CHARS + 1; + assert!(paragraph_count <= MAX_PARAGRAPHS_PER_SECTION); + let paragraph = "x".repeat(MAX_PARAGRAPH_CHARS); + let big = DocumentSection { + heading: Some("Heading".to_string()), + paragraphs: vec![paragraph; paragraph_count], + bullets: vec![], + }; + let s = DocumentSpec { + title: "Huge".to_string(), + author: None, + sections: vec![big], + }; + // Sanity: this spec passes every per-field check. + assert!(s.sections.len() <= MAX_SECTIONS); + assert_rejects(&s, "sections"); +} + +#[test] +fn rejects_an_aggregate_overrun_that_a_bullet_crosses() { + // The heading and paragraph loops each carry their own budget check; so does + // the bullet loop, and only a spec whose overrun lands on a bullet drives + // that third branch. + let bullet = "b".repeat(MAX_PARAGRAPH_CHARS); + let bullet_count = MAX_TOTAL_CHARS / MAX_PARAGRAPH_CHARS + 1; + assert!(bullet_count <= MAX_BULLETS_PER_SECTION); + let s = DocumentSpec { + title: "Bullets".to_string(), + author: None, + sections: vec![DocumentSection { + heading: None, + paragraphs: vec![], + bullets: vec![bullet; bullet_count], + }], + }; + assert_rejects(&s, "sections"); +} + +#[test] +fn rejects_an_aggregate_overrun_that_a_heading_crosses() { + // Headings cannot reach the aggregate cap on their own: MAX_SECTIONS × + // MAX_TEXT_CHARS is 256_000, two orders of magnitude under MAX_TOTAL_CHARS. + // Driving the heading branch therefore means spending the budget down to a + // single character of headroom in an earlier section, then letting a + // perfectly legal heading cross it. + let title = "Headings"; + let filler_count = MAX_TOTAL_CHARS / MAX_PARAGRAPH_CHARS - 1; + assert!(filler_count <= MAX_PARAGRAPHS_PER_SECTION); + let used = title.chars().count() + filler_count * MAX_PARAGRAPH_CHARS; + // Leave exactly one character of headroom. + let tail = MAX_TOTAL_CHARS - used - 1; + assert!(tail <= MAX_PARAGRAPH_CHARS); + + let mut paragraphs = vec!["p".repeat(MAX_PARAGRAPH_CHARS); filler_count]; + paragraphs.push("p".repeat(tail)); + + let s = DocumentSpec { + title: title.to_string(), + author: None, + sections: vec![ + DocumentSection { + heading: None, + paragraphs, + bullets: vec![], + }, + DocumentSection { + // Two characters against one character of headroom. + heading: Some("hh".to_string()), + paragraphs: vec![], + bullets: vec![], + }, + ], + }; + assert!(s.sections.len() <= MAX_SECTIONS); + assert_rejects(&s, "sections"); +} + +#[test] +fn is_blank_reflects_content_presence() { + assert!(!section().is_blank()); + assert!( + DocumentSection { + heading: None, + paragraphs: vec![], + bullets: vec![], + } + .is_blank() + ); + // A heading alone is enough content. + assert!( + !DocumentSection { + heading: Some("Only a heading".to_string()), + paragraphs: vec![], + bullets: vec![], + } + .is_blank() + ); +} + +#[test] +fn total_chars_sums_every_text_field() { + let s = DocumentSpec { + title: "abcd".to_string(), // 4 + author: Some("xy".to_string()), // 2 + sections: vec![DocumentSection { + heading: Some("hij".to_string()), // 3 + paragraphs: vec!["pq".to_string()], // 2 + bullets: vec!["b".to_string()], // 1 + }], + }; + assert_eq!(s.total_chars(), 12); +} + +#[test] +fn spec_round_trips_through_json() { + let s = spec(); + let json = serde_json::to_string(&s).expect("serialises"); + let back: DocumentSpec = serde_json::from_str(&json).expect("deserialises"); + assert_eq!(back, s); +} + +#[test] +fn spec_rejects_unknown_json_fields() { + // `deny_unknown_fields` makes a typo'd key a loud rejection rather than a + // silently ignored one — the whole point at an LLM tool boundary. + let json = r#"{"title":"T","sections":[],"titel":"typo"}"#; + assert!(serde_json::from_str::(json).is_err()); +} + +#[test] +fn spec_defaults_optional_fields() { + let s: DocumentSpec = serde_json::from_str(r#"{"title":"T"}"#).expect("deserialises"); + assert_eq!(s.author, None); + assert!(s.sections.is_empty()); +} From f9fe43247ecb8c3192c8e0352d055586294d798a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 11 Aug 2026 00:55:15 +0300 Subject: [PATCH 03/13] Add .pptx synthesis behind a pptx feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the presentation engine from the OpenHuman host that already drove `ppt-rs` directly, on the same split the docx path uses: the spec and its validation go in ungated `spec/presentation`, the OOXML mapping goes in a gated `pptx` module, and the host keeps its executor and deadline policy. Three things are deliberately different from the code this came from. Images are bytes, not references. The host's spec named an image by artifact id or filesystem path, and resolving either is policy this crate must not hold — which directories an agent may read, whether an identifier belongs to the caller. `SlideImage` therefore carries the bytes, format and dimensions, and `SlideImage::from_bytes` does the mechanical half of the hand-off. Because identification lives in the ungated `spec::image`, a host can build and validate a whole deck with `pptx` off. Validation re-derives an image's format and dimensions from its bytes and rejects any disagreement with what the spec declares. `from_bytes` keeps those fields consistent by construction, but a spec can also arrive as JSON, where the three are independent: a wrong format yields a part the reader refuses to open, and wrong dimensions distort the image silently. Both now fail by name. `fit_within` is integer arithmetic in u64 rather than f64 scaling. The float spelling made the result depend on binary64 rounding for no benefit — an EMU is 1/914,400 inch, so a one-unit difference is invisible — and it could not satisfy this repo's cast lints without an allow. The host's `GenerationFailed { exit_code, stderr_truncated }` collapses onto `Error::GenerationFailed { detail }`; `exit_code` was a vestige of a retired python-pptx subprocess and was always -1. Timeout and cancellation stay host- side, where the deadline is known. Also fixes a mistake in the previous commit: `spec/document`'s tests were moved without a `mod test;` declaration, so all 20 stopped running and the module's coverage read 50%. Declared, they run again and it is back to 100%. Public API: additive. `spec::presentation`, `spec::image` and `pptx` are new. Format limits are reached through their own module rather than re-exported flat from `spec`, because `MAX_TEXT_CHARS` means a different number per format. Co-authored-by: Medulla --- Cargo.lock | 912 +++++++++++++++++++++- Cargo.toml | 9 +- README.md | 51 +- src/docx/mod.rs | 2 +- src/lib.rs | 20 +- src/pptx/mod.rs | 277 +++++++ src/pptx/test.rs | 307 ++++++++ src/spec/{document.rs => document/mod.rs} | 3 + src/spec/{ => document}/test.rs | 0 src/spec/image/mod.rs | 154 ++++ src/spec/image/test.rs | 139 ++++ src/spec/mod.rs | 42 +- src/spec/presentation/mod.rs | 338 ++++++++ src/spec/presentation/test.rs | 375 +++++++++ 14 files changed, 2596 insertions(+), 33 deletions(-) create mode 100644 src/pptx/mod.rs create mode 100644 src/pptx/test.rs rename src/spec/{document.rs => document/mod.rs} (99%) rename src/spec/{ => document}/test.rs (100%) create mode 100644 src/spec/image/mod.rs create mode 100644 src/spec/image/test.rs create mode 100644 src/spec/presentation/mod.rs create mode 100644 src/spec/presentation/test.rs diff --git a/Cargo.lock b/Cargo.lock index 22e7115..9e0be3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,82 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + [[package]] name = "arbitrary" version = "1.4.2" @@ -17,6 +93,12 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "async-trait" version = "0.1.92" @@ -46,12 +128,60 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -63,6 +193,26 @@ name = "bytemuck" version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "byteorder-lite" @@ -76,6 +226,35 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.4.2" @@ -83,6 +262,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -92,12 +273,83 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "color_quant" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -119,6 +371,22 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + [[package]] name = "derive_arbitrary" version = "1.4.2" @@ -130,6 +398,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "displaydoc" version = "0.2.7" @@ -154,7 +433,7 @@ dependencies = [ "serde", "serde_json", "smallvec", - "thiserror", + "thiserror 2.0.20", "zip 8.6.0", ] @@ -183,6 +462,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "euclid" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fancy-regex" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "998b056554fbe42e03ae0e152895cd1a7e1002aec800fdc6635d20270260c46f" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -231,6 +530,55 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "font-types" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b38ad915f6dadd993ced50848a8291a543bd41ca62bc10740d5e64e2ab4cfd7" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -280,6 +628,21 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "http" version = "1.5.0" @@ -324,18 +687,73 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "kurbo" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b60dfc32f652b926df6192e55525b16d186c69d47876c3ead4da5cc9f8450e2" +dependencies = [ + "arrayvec", + "euclid", + "polycool", + "smallvec", +] + [[package]] name = "libc" version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -348,6 +766,16 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.3" @@ -374,6 +802,12 @@ dependencies = [ "pxfm", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-traits" version = "0.2.19" @@ -389,6 +823,57 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", + "hmac", + "password-hash", + "sha2", +] + +[[package]] +name = "pdfrs" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd2c58cc563c54ee2dc0dacce61bf686529d792c4b76f031cbcfe8e3f7ecdeb0" +dependencies = [ + "aes", + "anyhow", + "base64 0.22.1", + "cbc", + "clap", + "flate2", + "md-5", + "regex", + "serde", + "serde_json", + "sha2", + "subsetter", + "syntect", + "ttf-parser", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -401,6 +886,25 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "plist" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da1d65da6dd5d1e44199ac0f58712d241c0f439f80adea8924d832384087f85" +dependencies = [ + "base64 0.22.1", + "indexmap", + "quick-xml", + "serde", + "time", +] + [[package]] name = "png" version = "0.18.1" @@ -414,6 +918,34 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "polycool" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50596ddc09eb5ad5f75cacd40209568e66df71baf86e1499a0e99c4cff12a5a6" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppt-rs" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed6af693d661395ff3464eac5f7cee1df674d082ce84da94c6819cc799fee929" +dependencies = [ + "pdfrs", + "thiserror 1.0.69", + "uuid", + "xml-rs", + "zip 0.6.6", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -460,6 +992,51 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "read-fonts" +version = "0.39.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4ed38b89c2c77ff968c524145ad65fb010f38af5c7a224b53b81d47ac2daa81" +dependencies = [ + "bytemuck", + "font-types", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + [[package]] name = "ring" version = "0.17.14" @@ -474,6 +1051,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustix" version = "1.1.4" @@ -522,6 +1105,21 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "serde" version = "1.0.229" @@ -574,6 +1172,28 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "2.0.1" @@ -586,12 +1206,46 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "skrifa" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c34617370ae968efb7161bb2beb517d9084659aae19e24b89e3db25b46e4564" +dependencies = [ + "bytemuck", + "read-fonts", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subsetter" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38803281d1c23166c5ebcb455439a5d2afe711cc909cf88af72448c297756ad6" +dependencies = [ + "kurbo", + "rustc-hash", + "skrifa", + "write-fonts", +] + [[package]] name = "subtle" version = "2.6.1" @@ -620,6 +1274,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syntect" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "656b45c05d95a5704399aeef6bd0ddec7b2b3531b7c9e900abbf7c4d2190c925" +dependencies = [ + "bincode", + "fancy-regex", + "flate2", + "fnv", + "once_cell", + "plist", + "regex-syntax", + "serde", + "serde_derive", + "serde_json", + "thiserror 2.0.20", + "walkdir", + "yaml-rust", +] + [[package]] name = "tar" version = "0.4.46" @@ -644,13 +1319,33 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -678,6 +1373,36 @@ dependencies = [ "zune-jpeg 0.4.21", ] +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinybus" version = "0.1.0" @@ -688,7 +1413,7 @@ dependencies = [ "serde_json", "tar", "tempfile", - "thiserror", + "thiserror 2.0.20", "tinybus-macros", "tokio", "toml", @@ -723,9 +1448,10 @@ name = "tinydocs" version = "0.1.11" dependencies = [ "docx-rs", + "ppt-rs", "serde", "serde_json", - "thiserror", + "thiserror 2.0.20", "zip 2.4.2", ] @@ -833,12 +1559,24 @@ dependencies = [ "once_cell", ] +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + [[package]] name = "typed-path" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -886,12 +1624,90 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + [[package]] name = "webpki-roots" version = "1.0.9" @@ -907,6 +1723,15 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -1004,6 +1829,19 @@ dependencies = [ "memchr", ] +[[package]] +name = "write-fonts" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb731d4c4d93eacc69a1ad2f270f905788a98e4a3438267bcafbe08d3431c8d8" +dependencies = [ + "font-types", + "indexmap", + "kurbo", + "log", + "read-fonts", +] + [[package]] name = "xattr" version = "1.6.1" @@ -1014,6 +1852,21 @@ dependencies = [ "rustix", ] +[[package]] +name = "xml-rs" +version = "0.8.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "zerocopy" version = "0.8.56" @@ -1040,6 +1893,26 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "aes", + "byteorder", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "flate2", + "hmac", + "pbkdf2", + "sha1", + "time", + "zstd", +] + [[package]] name = "zip" version = "2.4.2" @@ -1053,7 +1926,7 @@ dependencies = [ "flate2", "indexmap", "memchr", - "thiserror", + "thiserror 2.0.20", "zopfli", ] @@ -1095,6 +1968,35 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "zune-core" version = "0.4.12" diff --git a/Cargo.toml b/Cargo.toml index a5fa5c9..aee4759 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,11 @@ serde = { version = "1", features = ["derive"] } # OOXML `.docx` synthesis. Optional: exclusive to the `docx` feature so a host # that only needs extraction does not pull the writer stack. docx-rs = { version = "0.4.20", optional = true } +# OOXML `.pptx` synthesis. Optional: exclusive to the `pptx` feature. It brings +# its own zip/XML stack plus `syntect` and `pulldown-cmark` for a Markdown +# front-end this crate does not use, which is precisely why it is gated — a host +# that only generates documents should not carry a syntax highlighter. +ppt-rs = { version = "0.2.14", optional = true } [dev-dependencies] # `.docx` output is a zip container; the tests re-open the produced bytes and @@ -55,9 +60,11 @@ name = "basic" required-features = ["docx"] [features] -default = ["docx"] +default = ["docx", "pptx"] # `.docx` generation via `docx-rs`. docx = ["dep:docx-rs"] +# `.pptx` generation via `ppt-rs`. +pptx = ["dep:ppt-rs"] # Lints apply to the whole crate and to every target. CI runs clippy with # `-D warnings`, so anything set to "warn" here fails the build in CI. diff --git a/README.md b/README.md index 0052aca..41cb89c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TinyDocs -Agent-friendly document synthesis and text extraction in Rust. +Agent-friendly document synthesis in Rust: `.docx` and `.pptx`. `tinydocs` turns a typed, validated document spec into real office-format bytes. It is built for hosts that let a language model produce documents: the @@ -50,6 +50,12 @@ let bytes = tokio::time::timeout( Every limit is a public constant, so a host can quote the exact number in its own tool description and stay in lockstep with what validation enforces. +Each format's limits live in its own module, because the same name means a +different thing in each — `spec::document::MAX_TEXT_CHARS` bounds a heading, +`spec::presentation::MAX_TEXT_CHARS` bounds a bullet. + +`spec::document` (`.docx`): + | Limit | Value | Bounds | | --- | --- | --- | | `MAX_SECTIONS` | 128 | sections per document | @@ -59,14 +65,32 @@ own tool description and stay in lockstep with what validation enforces. | `MAX_BULLETS_PER_SECTION` | 200 | bullets per section | | `MAX_TOTAL_CHARS` | 2,000,000 | all text in the document | +`spec::presentation` (`.pptx`): + +| Limit | Value | Bounds | +| --- | --- | --- | +| `MAX_SLIDES` | 64 | content slides per deck | +| `MAX_TEXT_CHARS` | 2,000 | any single text field | +| `MAX_BULLETS_PER_SLIDE` | 32 | bullets per slide | +| `MAX_IMAGES_PER_SLIDE` | 6 | images per slide | +| `MAX_IMAGES_PER_DECK` | 8 | images across the deck | +| `MAX_IMAGE_BYTES` | 5 MiB | one embedded image | + The aggregate cap is the load-bearing one. The per-field limits bound each individual piece but not their product — `MAX_SECTIONS × MAX_PARAGRAPHS_PER_SECTION × MAX_PARAGRAPH_CHARS` alone is over 500M characters, so a spec satisfying every other limit could still build a multi-hundred-megabyte document in memory. -`DocumentSpec::validate` is public and runs before any synthesis, so a host can -reject a bad tool call at its own boundary without paying for a blocking hop. +`DocumentSpec::validate` and `PresentationSpec::validate` are public and run +before any synthesis, so a host can reject a bad tool call at its own boundary +without paying for a blocking hop. + +A presentation carries its images as bytes, not as paths or identifiers: +resolving indirection is host policy — which directories an agent may read, +whether an identifier belongs to the caller — and this crate has no business +holding it. `SlideImage::from_bytes` does the mechanical half, identifying the +format and reading the dimensions, and needs no writer to do it. ## The spec is separable from the codec @@ -127,9 +151,14 @@ TINYDOCS_TEST_MODULE="$PWD/target/release/libtinydocs_module.so" \ ## Feature flags -| Feature | Default | Gates | -| --- | --- | --- | -| `docx` | on | `.docx` synthesis via `docx-rs` | +Each format is a separate gate, and every gate is on by default. Turning one off +drops its writer and that writer's dependencies; `tinydocs::spec` stays either +way, so the contract and its validation survive any combination. + +| Feature | Default | Gates | Also drops | +| --- | --- | --- | --- | +| `docx` | on | `.docx` synthesis via `docx-rs` | `quick-xml` | +| `pptx` | on | `.pptx` synthesis via `ppt-rs` | `syntect`, `pulldown-cmark`, `xml-rs` | ## Layout @@ -141,10 +170,14 @@ src/ │ └── test.rs ├── spec/ # wire contracts — ungated, serde only │ ├── mod.rs # re-export surface -│ ├── document.rs # `DocumentSpec`, `DocumentSection`, limits, `validate` -│ └── test.rs +│ ├── document/ # `DocumentSpec`, `DocumentSection`, limits, `validate` +│ ├── presentation/ # `PresentationSpec`, `SlideSpec`, `SlideImage`, limits +│ └── image/ # `ImageFormat` — PNG/JPEG sniffing + header measurement ├── docx/ - ├── mod.rs # `generate` — the OOXML mapping +│ ├── mod.rs # `generate` — the `WordprocessingML` mapping +│ └── test.rs +├── pptx/ + ├── mod.rs # `generate` — the `PresentationML` mapping + image layout └── test.rs tests/ └── public_api.rs # integration tests against the public API only diff --git a/src/docx/mod.rs b/src/docx/mod.rs index fa3fa50..2fb8c2f 100644 --- a/src/docx/mod.rs +++ b/src/docx/mod.rs @@ -30,7 +30,7 @@ // The spec is defined in `crate::spec`, which is compiled in every build so a // host can share the wire contract without the OOXML writer stack. Re-exported // here so `tinydocs::docx::DocumentSpec` keeps naming the same type. -pub use crate::spec::{ +pub use crate::spec::document::{ DocumentSection, DocumentSpec, MAX_BULLETS_PER_SECTION, MAX_PARAGRAPH_CHARS, MAX_PARAGRAPHS_PER_SECTION, MAX_SECTIONS, MAX_TEXT_CHARS, MAX_TOTAL_CHARS, }; diff --git a/src/lib.rs b/src/lib.rs index e08274a..da8d588 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,6 +35,14 @@ not(feature = "docx"), doc = "- `docx` (disabled in this build) — `.docx` (OOXML `WordprocessingML`) synthesis." )] +#![cfg_attr( + feature = "pptx", + doc = "- [`pptx`] — `.pptx` (OOXML `PresentationML`) synthesis." +)] +#![cfg_attr( + not(feature = "pptx"), + doc = "- `pptx` (disabled in this build) — `.pptx` (OOXML `PresentationML`) synthesis." +)] //! //! # Example //! @@ -60,8 +68,13 @@ //! //! # Feature flags //! -//! - `docx` (default) — `.docx` synthesis via `docx-rs`. Turning it off drops -//! the whole OOXML writer stack. +//! Each format is a separate gate, and every gate is on by default. Turning one +//! off drops its writer and that writer's dependencies; the specs stay, so the +//! contract and its validation survive any combination. +//! +//! - `docx` (default) — `.docx` synthesis via `docx-rs`. +//! - `pptx` (default) — `.pptx` synthesis via `ppt-rs`, which also drops +//! `syntect` and `pulldown-cmark`. mod error; @@ -70,4 +83,7 @@ pub mod spec; #[cfg(feature = "docx")] pub mod docx; +#[cfg(feature = "pptx")] +pub mod pptx; + pub use error::{Error, Result}; diff --git a/src/pptx/mod.rs b/src/pptx/mod.rs new file mode 100644 index 0000000..d965d3a --- /dev/null +++ b/src/pptx/mod.rs @@ -0,0 +1,277 @@ +//! `.pptx` (OOXML `PresentationML`) synthesis, backed by +//! [`ppt-rs`](https://crates.io/crates/ppt-rs). +//! +//! [`generate`] turns a validated [`PresentationSpec`] into the bytes of a +//! `.pptx` file. Like [`crate::docx::generate`] it is **synchronous, pure, and +//! CPU-bound**: it touches no filesystem, spawns no subprocess, and knows +//! nothing about deadlines. A host that needs a timeout or a blocking-pool hop +//! owns that policy and wraps this call. +//! +//! # Spec → `PresentationML` mapping +//! +//! `ppt_rs::SlideContent` has no separate body-paragraph slot: everything below +//! the title is a bullet. [`SlideSpec::body`] therefore collapses into a leading +//! bullet, so body text still reaches the rendered slide: +//! +//! ```text +//! SlideSpec { title, body: Some(b), bullets: [b1, b2], speaker_notes: Some(n) } +//! → SlideContent::new(title).add_bullet(b).add_bullet(b1).add_bullet(b2).notes(n) +//! ``` +//! +//! Blank and whitespace-only entries are dropped rather than emitting an empty +//! bullet marker. +//! +//! # The title slide is synthetic +//! +//! `ppt_rs::create_pptx_with_content(title, slides)` treats `title` as deck +//! metadata only — it lands in `docProps/core.xml` and does **not** produce a +//! title slide. A deck built from it would open straight onto the first content +//! slide. [`generate`] therefore prepends a slide carrying +//! [`PresentationSpec::title`] and the optional author byline, which is why the +//! rendered deck holds one more slide than `spec.slides.len()`. +//! +//! # Image layout +//! +//! Images stack in a single vertical column in the lower band of the slide, +//! beneath the text. Each is scaled to fit its slot with its aspect ratio +//! preserved and is centred in both axes; a slot is never upscaled past the +//! source's natural size ratio. Every dimension below is in EMU (English Metric +//! Units, 914,400 per inch), the unit OOXML itself uses. + +// The spec is defined in `crate::spec`, which is compiled in every build so a +// host can share the wire contract without the OOXML writer stack. Re-exported +// here so `tinydocs::pptx::PresentationSpec` names the same type. +pub use crate::spec::image::ImageFormat; +pub use crate::spec::presentation::{ + MAX_BULLETS_PER_SLIDE, MAX_IMAGE_BYTES, MAX_IMAGES_PER_DECK, MAX_IMAGES_PER_SLIDE, MAX_SLIDES, + MAX_TEXT_CHARS, PresentationSpec, SlideImage, SlideSpec, +}; + +use ppt_rs::generator::{Image, SlideContent, create_pptx_with_content}; + +use crate::{Error, Result}; + +/// Slide width in EMU — 10 inches, matching the writer's default 4:3 deck. +const SLIDE_WIDTH_EMU: u32 = 9_144_000; +/// Slide height in EMU — 7.5 inches. +const SLIDE_HEIGHT_EMU: u32 = 6_858_000; +/// Left and right margin in EMU — 1 inch, leaving the usable content column. +const SIDE_MARGIN_EMU: u32 = 914_400; +/// Top of the image band in EMU, roughly the slide midpoint: images live below +/// the title/body placeholder rather than over it. +const IMAGE_BAND_TOP_EMU: u32 = 3_429_000; +/// Margin in EMU kept below the image band — half an inch. +const IMAGE_BAND_BOTTOM_MARGIN_EMU: u32 = 457_200; +/// Vertical gap in EMU between two stacked images. +const IMAGE_STACK_GAP_EMU: u32 = 91_440; +/// EMU per pixel at 96 DPI, matching the writer's own px→EMU convention. +const EMU_PER_PX: u32 = 9_525; + +/// Validate `spec` and synthesise it into `.pptx` bytes. +/// +/// The returned buffer is a complete OOXML zip container: any reader compatible +/// with `PowerPoint` can open it, and a host can write it straight to disk or +/// stream it. +/// +/// Synchronous and CPU-bound. A deck at the slide cap completes well under a +/// second, but a host on an async executor should still run this on a blocking +/// pool rather than inline. +/// +/// The rendered deck carries `spec.slides.len() + 1` slides: the extra one is +/// the synthetic title slide described in the module docs. +/// +/// # Errors +/// +/// - [`Error::InvalidInput`] if `spec` violates any documented limit, or if an +/// image's declared format or dimensions contradict its bytes — no synthesis +/// is attempted. +/// - [`Error::GenerationFailed`] if `ppt-rs` fails to pack the deck. +/// +/// # Examples +/// +/// ``` +/// use tinydocs::pptx::{generate, PresentationSpec, SlideSpec}; +/// +/// let spec = PresentationSpec { +/// title: "Quarterly Review".to_string(), +/// author: Some("Alice".to_string()), +/// theme: None, +/// slides: vec![SlideSpec { +/// title: "Highlights".to_string(), +/// body: Some("Throughput doubled.".to_string()), +/// bullets: vec!["Shipped the parser".to_string()], +/// speaker_notes: Some("Mention the benchmark.".to_string()), +/// images: vec![], +/// }], +/// }; +/// +/// let bytes = generate(&spec)?; +/// assert_eq!(&bytes[0..2], b"PK", "a .pptx is a zip container"); +/// # Ok::<(), tinydocs::Error>(()) +/// ``` +pub fn generate(spec: &PresentationSpec) -> Result> { + spec.validate()?; + create_pptx_with_content(&spec.title, build_slides(spec)) + // The writer's error type is not guaranteed to be `Send + Sync + + // 'static`, so it is rendered to text at the boundary rather than + // carried. + .map_err(|err| Error::generation_failed(&format!("{err}"))) +} + +/// Pure transformation from the spec to the writer's slide model. +/// +/// Split out from [`generate`] for unit-testability: the slide ordering and +/// blank-filtering rules are load-bearing for the rendered deck shape. +fn build_slides(spec: &PresentationSpec) -> Vec { + let mut out = Vec::with_capacity(spec.slides.len() + 1); + + // The synthetic title slide. See the module docs: without it the deck would + // open on the first content slide and the title would only reach core.xml. + let mut title_slide = SlideContent::new(&spec.title); + if let Some(author) = spec.author.as_deref().filter(|a| !a.trim().is_empty()) { + title_slide = title_slide.add_bullet(author); + } + out.push(title_slide); + + for slide in &spec.slides { + let mut built = SlideContent::new(&slide.title); + if let Some(body) = slide.body.as_deref().filter(|b| !b.trim().is_empty()) { + built = built.add_bullet(body); + } + for bullet in &slide.bullets { + if !bullet.trim().is_empty() { + built = built.add_bullet(bullet); + } + } + // Images go on after the text, so a caption bullet lands beneath the + // image it labels rather than above the body. + for placed in place_single_column(&slide.images) { + built = built.add_image(placed.image); + if let Some(caption) = placed.caption { + built = built.add_bullet(&caption); + } + } + if let Some(notes) = slide + .speaker_notes + .as_deref() + .filter(|n| !n.trim().is_empty()) + { + built = built.notes(notes); + } + out.push(built); + } + + out +} + +/// A positioned image plus the caption that labels it. +struct PlacedImage { + image: Image, + caption: Option, +} + +/// Lay `images` out in a single vertical column inside the slide's lower band. +/// +/// The band is divided into equal slots with a fixed gap between them. Each +/// image is scaled to fit its slot with its aspect ratio preserved, then centred +/// horizontally and vertically inside it. +fn place_single_column(images: &[SlideImage]) -> Vec { + // Saturating rather than fallible: `MAX_IMAGES_PER_SLIDE` is 6, so a slice + // long enough to overflow a `u32` cannot come from a validated spec. If one + // ever did, saturating collapses every slot to zero height and the images + // degenerate visibly instead of panicking. + let count = u32::try_from(images.len()).unwrap_or(u32::MAX); + if count == 0 { + return Vec::new(); + } + + let content_left = SIDE_MARGIN_EMU; + let content_width = SLIDE_WIDTH_EMU.saturating_sub(2 * SIDE_MARGIN_EMU); + let band_height = SLIDE_HEIGHT_EMU + .saturating_sub(IMAGE_BAND_TOP_EMU) + .saturating_sub(IMAGE_BAND_BOTTOM_MARGIN_EMU); + let total_gap = IMAGE_STACK_GAP_EMU.saturating_mul(count.saturating_sub(1)); + let slot_height = band_height.saturating_sub(total_gap) / count; + + // The slot top advances by one stride per image. Accumulating it beats + // multiplying by the index: no `usize`-to-`u32` conversion of the index, and + // the stride is stated once. + let stride = slot_height.saturating_add(IMAGE_STACK_GAP_EMU); + let mut slot_top = IMAGE_BAND_TOP_EMU; + + images + .iter() + .map(|img| { + let this_slot_top = slot_top; + slot_top = slot_top.saturating_add(stride); + let (width, height) = fit_within( + img.width_px.saturating_mul(EMU_PER_PX), + img.height_px.saturating_mul(EMU_PER_PX), + content_width, + slot_height, + ); + let x = content_left + content_width.saturating_sub(width) / 2; + let y = this_slot_top + slot_height.saturating_sub(height) / 2; + PlacedImage { + image: Image::from_bytes(img.bytes.clone(), width, height, img.format.as_str()) + .position(x, y), + caption: img.caption.clone(), + } + }) + .collect() +} + +/// Scale `(w, h)` to fit inside `(max_w, max_h)` with the aspect ratio preserved. +/// +/// Both inputs and outputs are EMU. A degenerate zero dimension falls back to +/// the bounding box, because there is no ratio to preserve; validation rejects +/// zero-dimension images before synthesis, so this is a guard rather than a +/// path. The result is clamped into the box and never below 1 EMU, since a +/// zero-extent image is a part the reader rejects. +/// +/// The arithmetic is integer throughout, in `u64`. Scaling a dimension by a +/// floating-point factor and rounding back would be the obvious spelling, but it +/// makes the result depend on binary64 rounding for no benefit at this +/// magnitude: an EMU is 1/914,400 inch, so a one-unit difference is invisible, +/// and the intermediate products here (a dimension times a slide extent) top out +/// near 4×10^16, comfortably inside `u64`. +fn fit_within(w: u32, h: u32, max_w: u32, max_h: u32) -> (u32, u32) { + if w == 0 || h == 0 { + return (max_w, max_h); + } + let (w, h) = (u64::from(w), u64::from(h)); + let (box_w, box_h) = (u64::from(max_w), u64::from(max_h)); + + // Fit to the width first. If the resulting height overflows the box, the + // height is the binding constraint instead, so fit to that. + let height_at_full_width = div_round(h * box_w, w); + let (fit_w, fit_h) = if height_at_full_width <= box_h { + (box_w, height_at_full_width) + } else { + (div_round(w * box_h, h), box_h) + }; + + (clamp_into_box(fit_w, max_w), clamp_into_box(fit_h, max_h)) +} + +/// `numerator / denominator`, rounded to nearest rather than truncated. +/// +/// # Panics +/// +/// Panics if `denominator` is zero. Both call sites guard against a zero +/// dimension before reaching here. +fn div_round(numerator: u64, denominator: u64) -> u64 { + (numerator + denominator / 2) / denominator +} + +/// Clamp a computed extent into `1..=max`, narrowing to `u32`. +/// +/// `value` is always a scaled dimension bounded by `max`, so the narrowing +/// cannot lose information; `unwrap_or` states the fallback rather than +/// asserting the invariant with a panic. +fn clamp_into_box(value: u64, max: u32) -> u32 { + u32::try_from(value).unwrap_or(max).clamp(1, max.max(1)) +} + +#[cfg(test)] +mod test; diff --git a/src/pptx/test.rs b/src/pptx/test.rs new file mode 100644 index 0000000..446ad2e --- /dev/null +++ b/src/pptx/test.rs @@ -0,0 +1,307 @@ +//! Unit tests for `.pptx` synthesis. +//! +//! The spec's validation and JSON contract are tested in +//! `crate::spec::presentation` — they are format-independent and must pass in a +//! build with this feature off. What is left here is the deck shape: how many +//! slides are emitted, which text survives blank-filtering, the image geometry, +//! and the OOXML container itself. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::{PresentationSpec, SlideImage, SlideSpec, build_slides, fit_within, generate}; +use crate::Error; +use crate::spec::image::test::png; + +fn slide() -> SlideSpec { + SlideSpec { + title: "Overview".to_string(), + body: Some("The situation so far.".to_string()), + bullets: vec!["A bullet".to_string()], + speaker_notes: Some("Keep it short.".to_string()), + images: vec![], + } +} + +fn spec() -> PresentationSpec { + PresentationSpec { + title: "Quarterly Review".to_string(), + author: Some("Alice".to_string()), + theme: None, + slides: vec![slide()], + } +} + +fn image(width: u32, height: u32, caption: Option<&str>) -> SlideImage { + SlideImage::from_bytes(png(width, height), caption.map(str::to_string)).expect("valid png") +} + +/// Entry names inside a produced `.pptx` byte buffer. +fn entry_names(bytes: &[u8]) -> Vec { + let mut zip = + zip::ZipArchive::new(std::io::Cursor::new(bytes.to_vec())).expect("output is a valid zip"); + (0..zip.len()) + .map(|i| zip.by_index(i).unwrap().name().to_string()) + .collect() +} + +/// One entry's UTF-8 body out of a produced `.pptx`. +fn entry_body(bytes: &[u8], name: &str) -> String { + let mut zip = + zip::ZipArchive::new(std::io::Cursor::new(bytes.to_vec())).expect("output is a valid zip"); + let mut entry = zip.by_name(name).expect("entry present"); + let mut body = String::new(); + std::io::Read::read_to_string(&mut entry, &mut body).unwrap(); + body +} + +/// Whether `name` is a slide part rather than one of its relationship parts. +/// +/// Slide parts live at `ppt/slides/slideN.xml` and their relationships at +/// `ppt/slides/_rels/slideN.xml.rels`, so excluding the `_rels` directory is +/// exact — and avoids comparing a file extension case-sensitively. +fn is_slide_part(name: &str) -> bool { + name.starts_with("ppt/slides/slide") && !name.contains("_rels") +} + +/// Concatenated bodies of every `ppt/slides/slideN.xml` part. +fn all_slide_xml(bytes: &[u8]) -> String { + entry_names(bytes) + .iter() + .filter(|name| is_slide_part(name)) + .map(|name| entry_body(bytes, name)) + .collect::>() + .join("\n") +} + +#[test] +fn generate_produces_a_readable_ooxml_container() { + let bytes = generate(&spec()).expect("generation should succeed"); + + assert_eq!(&bytes[0..2], b"PK", "must start with the zip magic PK"); + let names = entry_names(&bytes); + for required in ["[Content_Types].xml", "_rels/.rels"] { + assert!( + names.iter().any(|n| n == required), + "missing OOXML entry {required} (got {names:?})" + ); + } + assert!( + names.iter().any(|n| is_slide_part(n)), + "no slide parts emitted (got {names:?})" + ); +} + +#[test] +fn a_synthetic_title_slide_is_prepended() { + // The writer treats its `title` argument as core.xml metadata only, so + // without the prepend the deck would open on the first content slide. The + // rendered deck therefore holds one more slide than the spec lists. + let built = build_slides(&spec()); + assert_eq!(built.len(), spec().slides.len() + 1); + + let bytes = generate(&spec()).expect("generation should succeed"); + let slide_parts = entry_names(&bytes) + .iter() + .filter(|n| is_slide_part(n)) + .count(); + assert_eq!(slide_parts, 2, "one title slide plus one content slide"); +} + +#[test] +fn the_title_slide_carries_the_deck_title_and_author() { + let xml = all_slide_xml(&generate(&spec()).expect("generation should succeed")); + assert!(xml.contains("Quarterly Review"), "deck title missing"); + assert!(xml.contains("Alice"), "author byline missing"); +} + +#[test] +fn a_blank_author_emits_no_byline() { + let mut s = spec(); + s.author = Some(" ".to_string()); + let built = build_slides(&s); + // The title slide should hold the title and nothing else. Compare against a + // deck with no author at all rather than reaching into the writer's types. + let mut without = spec(); + without.author = None; + assert_eq!( + format!("{:?}", built[0]), + format!("{:?}", build_slides(&without)[0]), + "a whitespace-only author must render the same as no author" + ); +} + +#[test] +fn generate_carries_every_text_field_into_the_slides() { + let mut s = spec(); + s.slides = vec![SlideSpec { + title: "Highlights".to_string(), + body: Some("Throughput doubled.".to_string()), + bullets: vec!["Shipped the parser".to_string(), "Cut latency".to_string()], + speaker_notes: None, + images: vec![], + }]; + let xml = all_slide_xml(&generate(&s).expect("generation should succeed")); + for needle in [ + "Highlights", + "Throughput doubled.", + "Shipped the parser", + "Cut latency", + ] { + assert!(xml.contains(needle), "slide xml missing text {needle:?}"); + } +} + +#[test] +fn generate_drops_blank_body_and_bullets() { + let mut s = spec(); + s.slides = vec![SlideSpec { + title: "Kept".to_string(), + body: Some(" ".to_string()), + bullets: vec!["real".to_string(), "\t\n".to_string(), String::new()], + speaker_notes: Some(" ".to_string()), + images: vec![], + }]; + let bytes = generate(&s).expect("generation should succeed"); + let xml = all_slide_xml(&bytes); + assert!(xml.contains("Kept")); + assert!(xml.contains("real")); + for dropped in ["\t\n", " "] { + assert!( + !xml.contains(dropped), + "whitespace-only content {dropped:?} leaked into a slide part" + ); + } +} + +#[test] +fn speaker_notes_reach_a_notes_part() { + let bytes = generate(&spec()).expect("generation should succeed"); + let names = entry_names(&bytes); + let notes: Vec<_> = names + .iter() + .filter(|n| n.contains("notesSlide")) + .cloned() + .collect(); + assert!(!notes.is_empty(), "no notes part emitted (got {names:?})"); + let body = entry_body(&bytes, ¬es[0]); + assert!( + body.contains("Keep it short."), + "notes text missing from {}", + notes[0] + ); +} + +#[test] +fn generate_validates_before_synthesising() { + let mut s = spec(); + s.title = String::new(); + assert!(matches!(generate(&s), Err(Error::InvalidInput { .. }))); +} + +#[test] +fn images_are_embedded_with_their_captions() { + let mut s = spec(); + s.slides[0].images = vec![image(320, 200, Some("A chart"))]; + let bytes = generate(&s).expect("generation should succeed"); + let names = entry_names(&bytes); + assert!( + names.iter().any(|n| n.starts_with("ppt/media/")), + "no media part emitted (got {names:?})" + ); + assert!( + all_slide_xml(&bytes).contains("A chart"), + "image caption missing from the slide" + ); +} + +#[test] +fn a_single_image_is_centred_in_the_band() { + use super::{ + IMAGE_BAND_BOTTOM_MARGIN_EMU, IMAGE_BAND_TOP_EMU, SIDE_MARGIN_EMU, SLIDE_HEIGHT_EMU, + SLIDE_WIDTH_EMU, place_single_column, + }; + + let placed = place_single_column(&[image(400, 300, None)]); + assert_eq!(placed.len(), 1); + let img = &placed[0].image; + + let content_left = SIDE_MARGIN_EMU; + let content_width = SLIDE_WIDTH_EMU - 2 * SIDE_MARGIN_EMU; + let band_height = SLIDE_HEIGHT_EMU - IMAGE_BAND_TOP_EMU - IMAGE_BAND_BOTTOM_MARGIN_EMU; + + // Inside the content column and inside the band, in both axes. + assert!(img.x >= content_left, "x={} left of the margin", img.x); + assert!( + img.x + img.width <= content_left + content_width, + "image overflows the content column" + ); + assert!(img.y >= IMAGE_BAND_TOP_EMU, "y={} above the band", img.y); + assert!( + img.y + img.height <= IMAGE_BAND_TOP_EMU + band_height, + "image overflows the band" + ); + + // Centred: the gaps on either side match within a rounding unit. + let left_gap = img.x - content_left; + let right_gap = (content_left + content_width) - (img.x + img.width); + assert!( + left_gap.abs_diff(right_gap) <= 1, + "not horizontally centred: {left_gap} vs {right_gap}" + ); +} + +#[test] +fn stacked_images_do_not_overlap_and_stay_in_order() { + use super::{IMAGE_BAND_TOP_EMU, place_single_column}; + + let placed = place_single_column(&[ + image(400, 300, None), + image(400, 300, None), + image(400, 300, None), + ]); + assert_eq!(placed.len(), 3); + let mut previous_bottom = IMAGE_BAND_TOP_EMU; + for (i, p) in placed.iter().enumerate() { + assert!( + p.image.y >= previous_bottom, + "image {i} at y={} overlaps the one above (bottom={previous_bottom})", + p.image.y + ); + previous_bottom = p.image.y + p.image.height; + } +} + +#[test] +fn no_images_places_nothing() { + use super::place_single_column; + assert!(place_single_column(&[]).is_empty()); +} + +#[test] +fn fit_within_preserves_the_aspect_ratio() { + // A 2:1 source into a square box fills the width and halves the height. + assert_eq!(fit_within(2_000, 1_000, 1_000, 1_000), (1_000, 500)); + // A 1:2 source into a square box fills the height. + assert_eq!(fit_within(1_000, 2_000, 1_000, 1_000), (500, 1_000)); + // An exact fit is unchanged. + assert_eq!(fit_within(800, 600, 800, 600), (800, 600)); +} + +#[test] +fn fit_within_upscales_a_small_source_to_the_box() { + // The scale factor is the min of both ratios, so a source smaller than the + // box grows to touch it on one axis without distorting. + assert_eq!(fit_within(100, 50, 1_000, 1_000), (1_000, 500)); +} + +#[test] +fn fit_within_clamps_and_never_returns_zero() { + // A degenerate source has no ratio to preserve, so it falls back to the box. + assert_eq!(fit_within(0, 100, 640, 480), (640, 480)); + assert_eq!(fit_within(100, 0, 640, 480), (640, 480)); + // An extremely wide source still yields at least one EMU on the short axis + // rather than a zero-height image the reader would reject. + let (w, h) = fit_within(1_000_000, 1, 100, 100); + assert!(w >= 1 && h >= 1, "got {w}x{h}"); + assert!(w <= 100 && h <= 100, "got {w}x{h}, outside the box"); +} diff --git a/src/spec/document.rs b/src/spec/document/mod.rs similarity index 99% rename from src/spec/document.rs rename to src/spec/document/mod.rs index 3c293da..c08e9b2 100644 --- a/src/spec/document.rs +++ b/src/spec/document/mod.rs @@ -256,3 +256,6 @@ impl DocumentSpec { Ok(()) } } + +#[cfg(test)] +mod test; diff --git a/src/spec/test.rs b/src/spec/document/test.rs similarity index 100% rename from src/spec/test.rs rename to src/spec/document/test.rs diff --git a/src/spec/image/mod.rs b/src/spec/image/mod.rs new file mode 100644 index 0000000..bb51c47 --- /dev/null +++ b/src/spec/image/mod.rs @@ -0,0 +1,154 @@ +//! Raster-image identification for specs that embed images. +//! +//! Two formats are supported, PNG and JPEG, and the restriction is deliberate +//! rather than incidental: the OOXML presentation writer this crate drives +//! declares no `webp` default in the generated `[Content_Types].xml`, and its +//! automatic format detection misclassifies `webp` as PNG — producing a part +//! `PowerPoint` refuses to render. Accepting only what can actually be embedded +//! turns that into a clean rejection at the boundary. +//! +//! Identification is done by reading the container header directly, in about a +//! hundred lines and with no dependencies, rather than by pulling in a decoding +//! stack. Nothing here decodes pixels: it answers "which format is this" and +//! "what are its native dimensions", which is all a layout engine needs to +//! place an image with the right aspect ratio. +//! +//! Like the rest of [`crate::spec`], this module is compiled in every build. A +//! host resolving image bytes has to identify and measure them to *build* a +//! spec, and that must not require the writer. + +use serde::{Deserialize, Serialize}; + +/// A raster image format that can be embedded in a generated document. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum ImageFormat { + /// Portable Network Graphics. + Png, + /// JPEG / JFIF. + Jpeg, +} + +impl ImageFormat { + /// The format's canonical OOXML name — `"PNG"` or `"JPEG"`. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Png => "PNG", + Self::Jpeg => "JPEG", + } + } + + /// Identify `bytes` by its container header. + /// + /// Returns `None` for a truncated header or any format other than the two + /// embeddable ones — including GIF, WebP and BMP, which are recognisable + /// but not embeddable. + #[must_use] + pub fn sniff(bytes: &[u8]) -> Option { + if bytes.starts_with(&[0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]) { + Some(Self::Png) + } else if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) { + Some(Self::Jpeg) + } else { + None + } + } + + /// Native `(width, height)` of `bytes` in pixels, read from the header. + /// + /// Returns `None` when the header is truncated or malformed, or when either + /// dimension is zero — a degenerate image cannot be placed aspect-correctly + /// and is rejected rather than divided by. + #[must_use] + pub fn dimensions(self, bytes: &[u8]) -> Option<(u32, u32)> { + match self { + Self::Png => png_dimensions(bytes), + Self::Jpeg => jpeg_dimensions(bytes), + } + } +} + +impl std::fmt::Display for ImageFormat { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// PNG: 8-byte signature, then an `IHDR` chunk whose width / height are +/// big-endian `u32`s at byte offsets 16 and 20. +fn png_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + if bytes.len() < 24 || &bytes[12..16] != b"IHDR" { + return None; + } + let w = u32::from_be_bytes([bytes[16], bytes[17], bytes[18], bytes[19]]); + let h = u32::from_be_bytes([bytes[20], bytes[21], bytes[22], bytes[23]]); + if w == 0 || h == 0 { + return None; + } + Some((w, h)) +} + +/// JPEG: walk the marker segments until a Start-Of-Frame is hit; its payload +/// carries height then width as big-endian `u16`s. +fn jpeg_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + let mut i = 2; // skip the leading FF D8 SOI + while i + 3 < bytes.len() { + if bytes[i] != 0xFF { + i += 1; + continue; + } + let marker = bytes[i + 1]; + i += 2; + // Standalone markers (no length field): padding fill bytes and + // RSTn / SOI / EOI. Skip without consuming a segment length. + if marker == 0xFF || marker == 0xD8 || marker == 0xD9 || (0xD0..=0xD7).contains(&marker) { + continue; + } + if i + 1 >= bytes.len() { + return None; + } + let seg_len = u16::from_be_bytes([bytes[i], bytes[i + 1]]) as usize; + if seg_len < 2 { + return None; + } + // SOF markers carrying frame dimensions. Excludes 0xC4 (DHT), + // 0xC8 (JPG) and 0xCC (DAC), which share the 0xCn range but are not + // frame headers. + let is_sof = matches!( + marker, + 0xC0 | 0xC1 + | 0xC2 + | 0xC3 + | 0xC5 + | 0xC6 + | 0xC7 + | 0xC9 + | 0xCA + | 0xCB + | 0xCD + | 0xCE + | 0xCF + ); + if is_sof { + // segment: [len_hi len_lo precision h_hi h_lo w_hi w_lo ...] + if i + 6 >= bytes.len() { + return None; + } + let h = u32::from(u16::from_be_bytes([bytes[i + 3], bytes[i + 4]])); + let w = u32::from(u16::from_be_bytes([bytes[i + 5], bytes[i + 6]])); + if w == 0 || h == 0 { + return None; + } + return Some((w, h)); + } + i += seg_len; + } + None +} + +// Visible crate-wide under `cfg(test)`: the `png` / `jpeg` header builders here +// are the fixtures every image-carrying spec and every synthesis test needs, and +// one honest builder beats a base64 blob copied into three files. +#[cfg(test)] +pub(crate) mod test; diff --git a/src/spec/image/test.rs b/src/spec/image/test.rs new file mode 100644 index 0000000..8cd7322 --- /dev/null +++ b/src/spec/image/test.rs @@ -0,0 +1,139 @@ +//! Unit tests for image identification and header measurement. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::{ImageFormat, jpeg_dimensions, png_dimensions}; + +/// A 1×1 PNG assembled byte-for-byte: signature, `IHDR`, `IDAT`, `IEND`. +/// +/// Built literally rather than decoded from base64 so the fixture needs no +/// dependency and the offsets under test are visible in the source. +pub(crate) fn png(width: u32, height: u32) -> Vec { + let mut out = vec![0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]; + out.extend_from_slice(&13u32.to_be_bytes()); // IHDR length + out.extend_from_slice(b"IHDR"); + out.extend_from_slice(&width.to_be_bytes()); + out.extend_from_slice(&height.to_be_bytes()); + out.extend_from_slice(&[0x08, 0x06, 0x00, 0x00, 0x00]); // depth, colour, etc. + out.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); // CRC placeholder + out.extend_from_slice(&0u32.to_be_bytes()); // empty IDAT + out.extend_from_slice(b"IDAT"); + out.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); + out.extend_from_slice(&0u32.to_be_bytes()); + out.extend_from_slice(b"IEND"); + out.extend_from_slice(&[0xAE, 0x42, 0x60, 0x82]); + out +} + +/// A minimal JPEG: SOI, an APP0 stub, then an SOF0 declaring `height × width`. +pub(crate) fn jpeg(width: u16, height: u16) -> Vec { + let mut out = vec![ + 0xFF, 0xD8, // SOI + 0xFF, 0xE0, 0x00, 0x04, 0x00, 0x00, // APP0, len=4, 2 payload bytes + 0xFF, 0xC0, 0x00, 0x0B, // SOF0, len=11 + 0x08, // precision + ]; + out.extend_from_slice(&height.to_be_bytes()); + out.extend_from_slice(&width.to_be_bytes()); + out.extend_from_slice(&[0x03, 0x00, 0x00, 0x00]); // components (filler) + out.extend_from_slice(&[0xFF, 0xD9]); // EOI + out +} + +#[test] +fn sniffs_png_and_jpeg() { + assert_eq!(ImageFormat::sniff(&png(1, 1)), Some(ImageFormat::Png)); + assert_eq!(ImageFormat::sniff(&jpeg(7, 5)), Some(ImageFormat::Jpeg)); +} + +#[test] +fn rejects_non_images_and_unembeddable_formats() { + assert_eq!(ImageFormat::sniff(b"not an image"), None); + // GIF and WebP are recognisable, but the writer cannot embed either. + assert_eq!(ImageFormat::sniff(b"GIF89a....."), None); + assert_eq!(ImageFormat::sniff(b"RIFF\0\0\0\0WEBP"), None); + assert_eq!(ImageFormat::sniff(&[]), None); +} + +#[test] +fn reads_png_dimensions() { + assert_eq!(ImageFormat::Png.dimensions(&png(1, 1)), Some((1, 1)), "1x1"); + assert_eq!( + ImageFormat::Png.dimensions(&png(1920, 1080)), + Some((1920, 1080)) + ); +} + +#[test] +fn reads_jpeg_dimensions() { + assert_eq!(ImageFormat::Jpeg.dimensions(&jpeg(7, 5)), Some((7, 5))); +} + +#[test] +fn truncated_headers_yield_none() { + assert_eq!(png_dimensions(&[0x89, 0x50, 0x4E, 0x47]), None); + assert_eq!(jpeg_dimensions(&[0xFF, 0xD8]), None); +} + +#[test] +fn a_png_without_an_ihdr_chunk_yields_none() { + let mut bytes = png(4, 4); + bytes[12..16].copy_from_slice(b"XXXX"); + assert_eq!(png_dimensions(&bytes), None); +} + +#[test] +fn a_zero_dimension_yields_none() { + // Degenerate images cannot be placed aspect-correctly; they are rejected + // rather than divided by. + assert_eq!(png_dimensions(&png(0, 8)), None); + assert_eq!(png_dimensions(&png(8, 0)), None); + assert_eq!(jpeg_dimensions(&jpeg(0, 8)), None); + assert_eq!(jpeg_dimensions(&jpeg(8, 0)), None); +} + +#[test] +fn a_jpeg_with_no_start_of_frame_yields_none() { + // SOI, then an APP0 segment and EOI — a valid marker stream carrying no + // frame header at all. + let bytes = vec![ + 0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x04, 0x00, 0x00, 0xFF, 0xD9, 0x00, 0x00, + ]; + assert_eq!(jpeg_dimensions(&bytes), None); +} + +#[test] +fn a_jpeg_with_a_degenerate_segment_length_yields_none() { + // A declared segment length below the two length bytes themselves would + // make the walk loop forever if it were trusted. + let bytes = vec![0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x01, 0x00, 0x00, 0x00]; + assert_eq!(jpeg_dimensions(&bytes), None); +} + +#[test] +fn a_jpeg_skips_standalone_and_non_frame_markers_before_the_frame() { + // Restart markers and a DHT (0xC4, in the 0xCn range but not a frame + // header) must both be stepped over rather than mistaken for an SOF. + let mut bytes = vec![0xFF, 0xD8, 0xFF, 0xD0, 0xFF, 0xFF]; + bytes.extend_from_slice(&[0xFF, 0xC4, 0x00, 0x04, 0x00, 0x00]); // DHT + bytes.extend_from_slice(&[0xFF, 0xC0, 0x00, 0x0B, 0x08]); + bytes.extend_from_slice(&11u16.to_be_bytes()); // height + bytes.extend_from_slice(&22u16.to_be_bytes()); // width + bytes.extend_from_slice(&[0x03, 0x00, 0x00, 0x00, 0xFF, 0xD9]); + assert_eq!(jpeg_dimensions(&bytes), Some((22, 11))); +} + +#[test] +fn format_renders_its_ooxml_name() { + assert_eq!(ImageFormat::Png.as_str(), "PNG"); + assert_eq!(ImageFormat::Jpeg.as_str(), "JPEG"); + assert_eq!(ImageFormat::Jpeg.to_string(), "JPEG"); +} + +#[test] +fn format_round_trips_through_json_as_its_ooxml_name() { + let json = serde_json::to_string(&ImageFormat::Png).expect("serialises"); + assert_eq!(json, r#""PNG""#); + let back: ImageFormat = serde_json::from_str(&json).expect("deserialises"); + assert_eq!(back, ImageFormat::Png); +} diff --git a/src/spec/mod.rs b/src/spec/mod.rs index bac84f9..e64d258 100644 --- a/src/spec/mod.rs +++ b/src/spec/mod.rs @@ -1,31 +1,43 @@ //! The wire contracts: typed document specs and their validation, with no //! dependency on any format writer. //! -//! Every format module in this crate (`docx`, …) synthesises bytes from a spec -//! defined here. The split matters for two reasons: +//! Every format module in this crate (`docx`, `pptx`, …) synthesises bytes from +//! a spec defined here. The split matters for two reasons: //! //! 1. **A host can share the contract without paying for the codec.** This //! module is `serde` plus the crate [`Error`](crate::Error) — nothing else. -//! It is compiled in *every* build, including -//! `--no-default-features`, so a host whose synthesis happens elsewhere (in -//! another process, or behind a message bus) still gets the one authoritative -//! definition of the spec instead of re-declaring it and drifting. +//! It is compiled in *every* build, including `--no-default-features`, so a +//! host whose synthesis happens elsewhere (in another process, or behind a +//! message bus) still gets the one authoritative definition of the spec +//! instead of re-declaring it and drifting. //! 2. **Validation is cheap and belongs at the boundary.** The specs validate //! themselves without touching a writer, so a host can reject a malformed //! LLM tool call before paying for a blocking hop or a round trip. //! -//! The format modules re-export the types they consume, so +//! # Where things live +//! +//! - [`document`] — `.docx`: [`DocumentSpec`], [`DocumentSection`]. +//! - [`presentation`] — `.pptx`: [`PresentationSpec`], [`SlideSpec`], +//! [`SlideImage`]. +//! - [`image`] — [`ImageFormat`], for specs that embed raster images. +//! +//! **Types are re-exported here; limits are not.** Each format's limits stay +//! inside its own module, because the same name means a different thing in each +//! — `document::MAX_TEXT_CHARS` bounds a heading, `presentation::MAX_TEXT_CHARS` +//! bounds a bullet — and flattening them would put two distinct constants under +//! one name. Reach for `spec::presentation::MAX_SLIDES` and read it as the +//! sentence it is. +//! +//! The format modules re-export both the types and the limits they consume, so //! `tinydocs::docx::DocumentSpec` and [`tinydocs::spec::DocumentSpec`] name the //! same type. //! //! [`tinydocs::spec::DocumentSpec`]: DocumentSpec -mod document; - -pub use document::{ - DocumentSection, DocumentSpec, MAX_BULLETS_PER_SECTION, MAX_PARAGRAPH_CHARS, - MAX_PARAGRAPHS_PER_SECTION, MAX_SECTIONS, MAX_TEXT_CHARS, MAX_TOTAL_CHARS, -}; +pub mod document; +pub mod image; +pub mod presentation; -#[cfg(test)] -mod test; +pub use document::{DocumentSection, DocumentSpec}; +pub use image::ImageFormat; +pub use presentation::{PresentationSpec, SlideImage, SlideSpec}; diff --git a/src/spec/presentation/mod.rs b/src/spec/presentation/mod.rs new file mode 100644 index 0000000..c8e3389 --- /dev/null +++ b/src/spec/presentation/mod.rs @@ -0,0 +1,338 @@ +//! The `.pptx` presentation spec: the typed description a caller hands to +//! `pptx::generate`, plus the size limits every spec is validated against. +//! +//! Same contract rules as [`crate::spec::document`] — `deny_unknown_fields`, +//! public limits, `validate` before synthesis — with one structural difference +//! worth understanding. +//! +//! # Images are bytes here, not references +//! +//! A [`SlideImage`] carries the image *bytes*, its format, and its native pixel +//! dimensions. It deliberately does **not** carry a path, a URL, or an +//! application-specific identifier, because resolving any of those is host +//! policy this crate has no business holding: which directories an agent may +//! read, whether a given identifier belongs to the caller, and whether fetching +//! a URL is an acceptable request to originate are all questions with different +//! answers in every host. A host resolves indirection under its own rules and +//! hands over the resulting bytes. +//! +//! [`SlideImage::from_bytes`] does the mechanical half of that hand-off: +//! identify the format and read the dimensions, or reject the bytes. It needs +//! no format writer, so a host can build and validate a whole spec in a build +//! with the `pptx` feature off. + +use serde::{Deserialize, Serialize}; + +use crate::spec::image::ImageFormat; +use crate::{Error, Result}; + +/// Maximum number of content slides a single deck may contain. +/// +/// Bounds generation time and output size; a caller with more material is +/// expected to split it across multiple decks. +pub const MAX_SLIDES: usize = 64; + +/// Maximum length, in Unicode scalar values, of any single text field — the +/// deck title, the author byline, the theme hint, a slide title, a slide body, +/// one bullet, the speaker notes, or an image caption. +pub const MAX_TEXT_CHARS: usize = 2_000; + +/// Maximum number of bullets on a single slide. +/// +/// Higher counts produce a slide nobody can read, and bloat the output. +pub const MAX_BULLETS_PER_SLIDE: usize = 32; + +/// Maximum number of images attached to a single slide. +/// +/// The single-column layout stacks images vertically in the lower band of the +/// slide; past this count each one is too small to read. +pub const MAX_IMAGES_PER_SLIDE: usize = 6; + +/// Maximum number of images across the whole deck. +/// +/// Bounds the embedded media payload regardless of how the images are +/// distributed across slides. +pub const MAX_IMAGES_PER_DECK: usize = 8; + +/// Maximum size, in bytes, of a single embedded image. +pub const MAX_IMAGE_BYTES: usize = 5 * 1024 * 1024; + +/// One image embedded on a slide. +/// +/// Construct with [`SlideImage::from_bytes`] rather than by hand: it derives +/// `format` and the dimensions from the bytes, which keeps the three fields +/// consistent by construction. [`PresentationSpec::validate`] re-checks that +/// consistency, because a spec can also arrive over a wire. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SlideImage { + /// The encoded image, as PNG or JPEG bytes. + pub bytes: Vec, + /// The format of `bytes`. + pub format: ImageFormat, + /// Native width in pixels, used to place the image without distorting it. + pub width_px: u32, + /// Native height in pixels, used to place the image without distorting it. + pub height_px: u32, + /// Optional caption, rendered as a bullet beneath the image. + #[serde(default)] + pub caption: Option, +} + +impl SlideImage { + /// Identify and measure `bytes`, producing a consistent [`SlideImage`]. + /// + /// # Errors + /// + /// Returns [`Error::InvalidInput`] when `bytes` is empty, exceeds + /// [`MAX_IMAGE_BYTES`], is not PNG or JPEG, or carries a header this crate + /// cannot measure. + pub fn from_bytes(bytes: Vec, caption: Option) -> Result { + if bytes.is_empty() { + return Err(Error::invalid_input("bytes", "must not be empty")); + } + if bytes.len() > MAX_IMAGE_BYTES { + return Err(Error::invalid_input( + "bytes", + format!("must be ≤ {MAX_IMAGE_BYTES} bytes"), + )); + } + let format = ImageFormat::sniff(&bytes) + .ok_or_else(|| Error::invalid_input("bytes", "must be a PNG or JPEG image"))?; + let (width_px, height_px) = format.dimensions(&bytes).ok_or_else(|| { + Error::invalid_input( + "bytes", + format!("{format} header is truncated or malformed"), + ) + })?; + Ok(Self { + bytes, + format, + width_px, + height_px, + caption, + }) + } +} + +/// One content slide of the deck, rendered in spec order. +/// +/// At least one of `title`, `body`, or `bullets` must carry renderable text. +/// Images alone are not enough — a slide holding only an image and no label +/// reads as a rendering bug rather than a design choice, and synthesis drops +/// blank text anyway. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SlideSpec { + /// Slide title. May be blank for a visually minimal slide, as long as the + /// body or bullets carry text. + #[serde(default)] + pub title: String, + /// Body text, rendered above the bullets. Plain text only. + #[serde(default)] + pub body: Option, + /// Bullets, rendered after the body text. + #[serde(default)] + pub bullets: Vec, + /// Speaker notes attached to the slide. + #[serde(default)] + pub speaker_notes: Option, + /// Images, stacked in a single column beneath the text. + #[serde(default)] + pub images: Vec, +} + +impl SlideSpec { + /// Returns `true` when the slide carries no renderable text at all — the + /// title, body, and every bullet are absent or blank. + /// + /// Synthesis trims and drops blank entries, so a slide holding only + /// `[" "]` would render without text despite carrying entries. + #[must_use] + pub fn is_textless(&self) -> bool { + let has_title = !self.title.trim().is_empty(); + let has_body = self.body.as_deref().is_some_and(|b| !b.trim().is_empty()); + let has_bullets = self.bullets.iter().any(|b| !b.trim().is_empty()); + !(has_title || has_body || has_bullets) + } +} + +/// A complete `.pptx` presentation spec. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PresentationSpec { + /// Deck title, rendered on a leading title slide. Required and non-blank. + pub title: String, + /// Optional author byline, rendered beneath the deck title. + #[serde(default)] + pub author: Option, + /// Optional theme hint. + /// + /// Accepted and validated but not yet acted on: synthesis uses the writer's + /// default template regardless. It is part of the contract so a host's tool + /// schema does not have to change when template selection lands. + #[serde(default)] + pub theme: Option, + /// Content slides, in display order. Must contain at least one entry. + #[serde(default)] + pub slides: Vec, +} + +impl PresentationSpec { + /// Total number of images across every slide. + #[must_use] + pub fn image_count(&self) -> usize { + self.slides + .iter() + .map(|slide| slide.images.len()) + .sum::() + } + + /// Check the spec against every documented size limit, and check that each + /// image's declared format and dimensions match its bytes. + /// + /// Callers do not have to invoke this: `pptx::generate` validates before it + /// synthesises anything. It is public so a host can reject a malformed spec + /// at its own boundary — an LLM tool call, say — and hand back the + /// structured [`Error::InvalidInput`] before paying for a blocking hop, a + /// process boundary, or a bus round trip. + /// + /// # Errors + /// + /// Returns [`Error::InvalidInput`] naming the first field that violates a + /// limit. Fields are checked in spec order, so the reported field is stable + /// for a given spec. + pub fn validate(&self) -> Result<()> { + if self.title.trim().is_empty() { + return Err(Error::invalid_input("title", "must not be empty")); + } + Self::check_text_len("title", &self.title)?; + if let Some(author) = self.author.as_deref() { + Self::check_text_len("author", author)?; + } + if let Some(theme) = self.theme.as_deref() { + Self::check_text_len("theme", theme)?; + } + if self.slides.is_empty() { + return Err(Error::invalid_input( + "slides", + "must contain at least one slide", + )); + } + if self.slides.len() > MAX_SLIDES { + return Err(Error::invalid_input( + "slides", + format!("must contain ≤ {MAX_SLIDES} slides"), + )); + } + // Checked across the whole deck rather than per slide: the per-slide cap + // bounds readability, this one bounds the embedded media payload however + // the images are distributed. + if self.image_count() > MAX_IMAGES_PER_DECK { + return Err(Error::invalid_input( + "slides[].images", + format!("deck must contain ≤ {MAX_IMAGES_PER_DECK} images total"), + )); + } + + for (i, slide) in self.slides.iter().enumerate() { + if slide.is_textless() { + return Err(Error::invalid_input( + format!("slides[{i}]"), + "must have at least one of title / body / bullets", + )); + } + Self::check_text_len(format!("slides[{i}].title"), &slide.title)?; + if let Some(body) = slide.body.as_deref() { + Self::check_text_len(format!("slides[{i}].body"), body)?; + } + if slide.bullets.len() > MAX_BULLETS_PER_SLIDE { + return Err(Error::invalid_input( + format!("slides[{i}].bullets"), + format!("must contain ≤ {MAX_BULLETS_PER_SLIDE} bullets"), + )); + } + for (b, bullet) in slide.bullets.iter().enumerate() { + Self::check_text_len(format!("slides[{i}].bullets[{b}]"), bullet)?; + } + if let Some(notes) = slide.speaker_notes.as_deref() { + Self::check_text_len(format!("slides[{i}].speaker_notes"), notes)?; + } + if slide.images.len() > MAX_IMAGES_PER_SLIDE { + return Err(Error::invalid_input( + format!("slides[{i}].images"), + format!("must contain ≤ {MAX_IMAGES_PER_SLIDE} images"), + )); + } + for (m, image) in slide.images.iter().enumerate() { + Self::check_image(&format!("slides[{i}].images[{m}]"), image)?; + } + } + Ok(()) + } + + /// Reject a text field longer than [`MAX_TEXT_CHARS`] scalar values. + fn check_text_len(field: impl Into, value: &str) -> Result<()> { + if value.chars().count() > MAX_TEXT_CHARS { + return Err(Error::invalid_input( + field, + format!("must be ≤ {MAX_TEXT_CHARS} chars"), + )); + } + Ok(()) + } + + /// Re-derive an image's format and dimensions from its bytes and reject any + /// disagreement with what the spec declares. + /// + /// [`SlideImage::from_bytes`] keeps the fields consistent by construction, + /// but a spec can also arrive as deserialized JSON, where the three fields + /// are independent. A declared format that does not match the bytes yields + /// a part the reader refuses to render, and declared dimensions that do not + /// match distort the image silently — both are worth a named rejection. + fn check_image(field: &str, image: &SlideImage) -> Result<()> { + if image.bytes.is_empty() { + return Err(Error::invalid_input( + format!("{field}.bytes"), + "must not be empty", + )); + } + if image.bytes.len() > MAX_IMAGE_BYTES { + return Err(Error::invalid_input( + format!("{field}.bytes"), + format!("must be ≤ {MAX_IMAGE_BYTES} bytes"), + )); + } + let sniffed = ImageFormat::sniff(&image.bytes).ok_or_else(|| { + Error::invalid_input(format!("{field}.bytes"), "must be a PNG or JPEG image") + })?; + if sniffed != image.format { + return Err(Error::invalid_input( + format!("{field}.format"), + format!("declared {} but the bytes are {sniffed}", image.format), + )); + } + let (width_px, height_px) = sniffed.dimensions(&image.bytes).ok_or_else(|| { + Error::invalid_input( + format!("{field}.bytes"), + format!("{sniffed} header is truncated or malformed"), + ) + })?; + if (width_px, height_px) != (image.width_px, image.height_px) { + return Err(Error::invalid_input( + format!("{field}.width_px"), + format!( + "declared {}x{} but the bytes are {width_px}x{height_px}", + image.width_px, image.height_px + ), + )); + } + if let Some(caption) = image.caption.as_deref() { + Self::check_text_len(format!("{field}.caption"), caption)?; + } + Ok(()) + } +} + +#[cfg(test)] +mod test; diff --git a/src/spec/presentation/test.rs b/src/spec/presentation/test.rs new file mode 100644 index 0000000..38fae53 --- /dev/null +++ b/src/spec/presentation/test.rs @@ -0,0 +1,375 @@ +//! Unit tests for the presentation wire contract. +//! +//! Format-independent, like the spec itself: these must pass in a build with +//! every format feature off. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::{ + MAX_BULLETS_PER_SLIDE, MAX_IMAGE_BYTES, MAX_IMAGES_PER_DECK, MAX_IMAGES_PER_SLIDE, MAX_SLIDES, + MAX_TEXT_CHARS, PresentationSpec, SlideImage, SlideSpec, +}; +use crate::Error; +use crate::spec::image::ImageFormat; +use crate::spec::image::test::{jpeg, png}; + +/// One valid slide carrying a title, a body, and a bullet. +fn slide() -> SlideSpec { + SlideSpec { + title: "Overview".to_string(), + body: Some("The situation so far.".to_string()), + bullets: vec!["A bullet".to_string()], + speaker_notes: Some("Keep it short.".to_string()), + images: vec![], + } +} + +/// A minimal valid spec; each test mutates one field to drive a single branch. +fn spec() -> PresentationSpec { + PresentationSpec { + title: "Quarterly Review".to_string(), + author: Some("Alice".to_string()), + theme: Some("plain".to_string()), + slides: vec![slide()], + } +} + +/// A valid image built from real header bytes. +fn image() -> SlideImage { + SlideImage::from_bytes(png(320, 200), Some("A chart".to_string())).expect("valid png") +} + +/// Assert `spec` is rejected with an `InvalidInput` naming `field`. +fn assert_rejects(spec: &PresentationSpec, field: &str) { + match spec.validate() { + Err(Error::InvalidInput { field: f, .. }) => { + assert_eq!(f, field, "unexpected rejected field"); + } + other => panic!("expected InvalidInput({field}), got {other:?}"), + } +} + +#[test] +fn accepts_a_well_formed_spec() { + assert!(spec().validate().is_ok()); +} + +#[test] +fn accepts_a_spec_with_images() { + let mut s = spec(); + s.slides[0].images = vec![image()]; + assert!(s.validate().is_ok()); +} + +#[test] +fn rejects_a_blank_deck_title() { + let mut s = spec(); + s.title = " ".to_string(); + assert_rejects(&s, "title"); +} + +#[test] +fn rejects_over_long_deck_level_text() { + for (field, mutate) in [("title", 0), ("author", 1), ("theme", 2)] { + let mut s = spec(); + let long = "x".repeat(MAX_TEXT_CHARS + 1); + match mutate { + 0 => s.title = long, + 1 => s.author = Some(long), + _ => s.theme = Some(long), + } + assert_rejects(&s, field); + } +} + +#[test] +fn rejects_a_spec_with_no_slides() { + let mut s = spec(); + s.slides.clear(); + assert_rejects(&s, "slides"); +} + +#[test] +fn rejects_too_many_slides() { + let mut s = spec(); + s.slides = vec![slide(); MAX_SLIDES + 1]; + assert_rejects(&s, "slides"); +} + +#[test] +fn rejects_a_textless_slide() { + // Every text entry is present but whitespace-only, so synthesis would drop + // all of them and render an unlabelled slide. + let mut s = spec(); + s.slides = vec![SlideSpec { + title: " ".to_string(), + body: Some("\t".to_string()), + bullets: vec![String::new()], + speaker_notes: None, + images: vec![], + }]; + assert_rejects(&s, "slides[0]"); +} + +#[test] +fn rejects_a_slide_carrying_only_an_image() { + // Images do not satisfy the "must have text" rule: an unlabelled slide + // reads as a rendering bug rather than a design choice. + let mut s = spec(); + s.slides = vec![SlideSpec { + title: String::new(), + body: None, + bullets: vec![], + speaker_notes: None, + images: vec![image()], + }]; + assert_rejects(&s, "slides[0]"); +} + +#[test] +fn rejects_over_long_slide_text_naming_its_index() { + let long = || "x".repeat(MAX_TEXT_CHARS + 1); + + let mut s = spec(); + s.slides.push(SlideSpec { + title: long(), + ..slide() + }); + assert_rejects(&s, "slides[1].title"); + + let mut s = spec(); + s.slides[0].body = Some(long()); + assert_rejects(&s, "slides[0].body"); + + let mut s = spec(); + s.slides[0].bullets = vec!["ok".to_string(), long()]; + assert_rejects(&s, "slides[0].bullets[1]"); + + let mut s = spec(); + s.slides[0].speaker_notes = Some(long()); + assert_rejects(&s, "slides[0].speaker_notes"); +} + +#[test] +fn rejects_too_many_bullets() { + let mut s = spec(); + s.slides[0].bullets = vec!["b".to_string(); MAX_BULLETS_PER_SLIDE + 1]; + assert_rejects(&s, "slides[0].bullets"); +} + +#[test] +fn rejects_too_many_images_on_one_slide() { + let mut s = spec(); + s.slides[0].images = vec![image(); MAX_IMAGES_PER_SLIDE + 1]; + assert_rejects(&s, "slides[0].images"); +} + +#[test] +fn rejects_too_many_images_across_the_deck() { + // Each slide is within the per-slide cap; only the deck total is not. The + // per-slide cap bounds readability, the deck cap bounds the media payload. + let per_slide = MAX_IMAGES_PER_SLIDE; + let slides_needed = MAX_IMAGES_PER_DECK / per_slide + 1; + let mut s = spec(); + s.slides = vec![ + SlideSpec { + images: vec![image(); per_slide], + ..slide() + }; + slides_needed + ]; + assert!(s.image_count() > MAX_IMAGES_PER_DECK); + assert_rejects(&s, "slides[].images"); +} + +#[test] +fn image_count_sums_across_slides() { + let mut s = spec(); + s.slides = vec![ + SlideSpec { + images: vec![image(), image()], + ..slide() + }, + SlideSpec { + images: vec![image()], + ..slide() + }, + ]; + assert_eq!(s.image_count(), 3); +} + +#[test] +fn rejects_an_over_long_image_caption() { + let mut s = spec(); + let mut img = image(); + img.caption = Some("c".repeat(MAX_TEXT_CHARS + 1)); + s.slides[0].images = vec![img]; + assert_rejects(&s, "slides[0].images[0].caption"); +} + +#[test] +fn from_bytes_derives_format_and_dimensions() { + let img = SlideImage::from_bytes(png(1920, 1080), None).expect("valid png"); + assert_eq!(img.format, ImageFormat::Png); + assert_eq!((img.width_px, img.height_px), (1920, 1080)); + assert_eq!(img.caption, None); + + let img = SlideImage::from_bytes(jpeg(640, 480), Some("j".to_string())).expect("valid jpeg"); + assert_eq!(img.format, ImageFormat::Jpeg); + assert_eq!((img.width_px, img.height_px), (640, 480)); +} + +#[test] +fn from_bytes_rejects_bad_input() { + assert!(matches!( + SlideImage::from_bytes(vec![], None), + Err(Error::InvalidInput { .. }) + )); + assert!(matches!( + SlideImage::from_bytes(b"not an image".to_vec(), None), + Err(Error::InvalidInput { .. }) + )); + // PNG signature with a truncated IHDR: the right format, unmeasurable. + assert!(matches!( + SlideImage::from_bytes(vec![0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A], None), + Err(Error::InvalidInput { .. }) + )); +} + +#[test] +fn from_bytes_rejects_an_oversize_image() { + // A real PNG header followed by enough filler to cross the cap, so the + // rejection is the size check rather than the sniff. + let mut bytes = png(8, 8); + bytes.resize(MAX_IMAGE_BYTES + 1, 0); + assert!(matches!( + SlideImage::from_bytes(bytes, None), + Err(Error::InvalidInput { .. }) + )); +} + +#[test] +fn validate_rejects_an_image_whose_declared_format_contradicts_its_bytes() { + // `from_bytes` cannot produce this, but deserialized JSON can: the three + // fields are independent on the wire. A wrong format yields a part the + // reader refuses to render, so it is worth a named rejection. + let mut s = spec(); + let mut img = image(); + img.format = ImageFormat::Jpeg; + s.slides[0].images = vec![img]; + assert_rejects(&s, "slides[0].images[0].format"); +} + +#[test] +fn validate_rejects_an_image_whose_declared_dimensions_contradict_its_bytes() { + // Declared dimensions that disagree with the bytes distort the image + // silently, which is worse than failing. + let mut s = spec(); + let mut img = image(); + img.width_px += 1; + s.slides[0].images = vec![img]; + assert_rejects(&s, "slides[0].images[0].width_px"); +} + +#[test] +fn validate_rejects_empty_oversize_and_unrecognised_image_bytes() { + let mut s = spec(); + let mut img = image(); + img.bytes.clear(); + s.slides[0].images = vec![img]; + assert_rejects(&s, "slides[0].images[0].bytes"); + + let mut s = spec(); + let mut img = image(); + img.bytes = b"not an image".to_vec(); + s.slides[0].images = vec![img]; + assert_rejects(&s, "slides[0].images[0].bytes"); + + let mut s = spec(); + let mut img = image(); + img.bytes.resize(MAX_IMAGE_BYTES + 1, 0); + s.slides[0].images = vec![img]; + assert_rejects(&s, "slides[0].images[0].bytes"); +} + +#[test] +fn validate_rejects_an_image_with_an_unmeasurable_header() { + // Sniffs as PNG, but the IHDR is gone — measurement fails after the format + // check has already passed, which is a distinct branch. + let mut s = spec(); + let mut img = image(); + img.bytes.truncate(8); + s.slides[0].images = vec![img]; + assert_rejects(&s, "slides[0].images[0].bytes"); +} + +#[test] +fn is_textless_reflects_text_presence() { + assert!(!slide().is_textless()); + assert!( + SlideSpec { + title: String::new(), + body: None, + bullets: vec![], + speaker_notes: None, + images: vec![], + } + .is_textless() + ); + // A title alone is enough. + assert!( + !SlideSpec { + title: "Only a title".to_string(), + body: None, + bullets: vec![], + speaker_notes: None, + images: vec![], + } + .is_textless() + ); + // So is a body alone, or a bullet alone. + assert!( + !SlideSpec { + title: String::new(), + body: Some("Body".to_string()), + bullets: vec![], + speaker_notes: None, + images: vec![], + } + .is_textless() + ); + assert!( + !SlideSpec { + title: String::new(), + body: None, + bullets: vec!["Bullet".to_string()], + speaker_notes: None, + images: vec![], + } + .is_textless() + ); +} + +#[test] +fn spec_round_trips_through_json() { + let mut s = spec(); + s.slides[0].images = vec![image()]; + let json = serde_json::to_string(&s).expect("serialises"); + let back: PresentationSpec = serde_json::from_str(&json).expect("deserialises"); + assert_eq!(back, s); + assert!(back.validate().is_ok()); +} + +#[test] +fn spec_rejects_unknown_json_fields() { + let json = r#"{"title":"T","slides":[],"tilte":"typo"}"#; + assert!(serde_json::from_str::(json).is_err()); +} + +#[test] +fn spec_defaults_optional_fields() { + let s: PresentationSpec = serde_json::from_str(r#"{"title":"T"}"#).expect("deserialises"); + assert_eq!(s.author, None); + assert_eq!(s.theme, None); + assert!(s.slides.is_empty()); +} From 04d9233ee11f2b56a0a0a15fa80f7239d074cbfa Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 11 Aug 2026 00:59:33 +0300 Subject: [PATCH 04/13] Add .pdf text extraction behind a pdf feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the document surface a host needs from this crate: it writes .docx and .pptx, and now reads .pdf. Ported from the OpenHuman host, which called `pdf_extract::extract_text_from_mem` directly from its multimodal ingest path. This is the only module here that reads rather than writes, and the asymmetry shapes it. Everything else turns a spec the caller authored into bytes, so there is a contract to validate; extraction takes a document somebody else produced, so the input is arbitrary and often damaged. Two consequences: `ExtractionFailed` is a new `Error` variant rather than a reuse of `GenerationFailed`. The two have opposite causes and opposite remedies — generation fails on our output path and usually means a bug or an exhausted resource, whereas extraction fails on someone else's input and usually means the document is encrypted, damaged, or has no text layer. A caller that retries one should not retry the other. `Error` is `#[non_exhaustive]`, so adding it does not break a match. The boundary checks are `InvalidInput`, not extraction failures: empty input, input over `MAX_DOCUMENT_BYTES`, and input with no `%PDF-` signature all name the offending field instead of surfacing a parser's phrasing for "you handed me a JPEG". `MAX_DOCUMENT_BYTES` exists because extraction allocates well past the input size while parsing, and the caller is handing over something it did not produce; a host wanting a tighter bound applies it first. A document that parses but carries no text layer — a scan — yields an empty string rather than an error. Nothing to extract is not a failure to retry, and OCR is out of scope. The 60s timeout and the host's decision to degrade a failed extraction to a file reference stay where they were: this call is synchronous and holds no opinion about deadlines, and here that matters more than for synthesis, because the cost is set by the input rather than by a spec this crate has already bounded. Tests build a real single-page PDF in-process, computing its cross-reference offsets from the bytes emitted, so extraction is exercised end to end without a checked-in binary fixture that could go stale. Co-authored-by: Medulla --- Cargo.lock | 256 ++++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 9 +- README.md | 9 +- src/error/mod.rs | 25 +++++ src/lib.rs | 12 ++- src/pdf/mod.rs | 88 ++++++++++++++++ src/pdf/test.rs | 115 +++++++++++++++++++++ 7 files changed, 507 insertions(+), 7 deletions(-) create mode 100644 src/pdf/mod.rs create mode 100644 src/pdf/test.rs diff --git a/Cargo.lock b/Cargo.lock index 9e0be3b..37d8ec7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "adobe-cmap-parser" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae8abfa9a4688de8fc9f42b3f013b6fffec18ed8a554f5f113577e0b9b3212a3" +dependencies = [ + "pom", +] + [[package]] name = "aes" version = "0.8.4" @@ -188,6 +197,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytemuck" version = "1.25.2" @@ -267,6 +282,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cff-parser" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f5b6e9141c036f3ff4ce7b2f7e432b0f00dee416ddcd4f17741d189ddc2e9d" + [[package]] name = "cfg-if" version = "1.0.4" @@ -437,6 +458,15 @@ dependencies = [ "zip 8.6.0", ] +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher", +] + [[package]] name = "encoding_rs" version = "0.8.35" @@ -462,6 +492,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "euclid" +version = "0.20.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bb7ef65b3777a325d1eeefefab5b6d4959da54747e33bd6258e789640f307ad" +dependencies = [ + "num-traits", +] + [[package]] name = "euclid" version = "0.22.14" @@ -590,6 +629,18 @@ dependencies = [ "wasi", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -598,7 +649,7 @@ checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", ] [[package]] @@ -737,7 +788,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b60dfc32f652b926df6192e55525b16d186c69d47876c3ead4da5cc9f8450e2" dependencies = [ "arrayvec", - "euclid", + "euclid 0.22.14", "polycool", "smallvec", ] @@ -766,6 +817,34 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lopdf" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7184fdea2bc3cd272a1acec4030c321a8f9875e877b3f92a53f2f6033fdc289" +dependencies = [ + "aes", + "bitflags", + "cbc", + "ecb", + "encoding_rs", + "flate2", + "getrandom 0.3.4", + "indexmap", + "itoa", + "log", + "md-5", + "nom", + "nom_locate", + "rand", + "rangemap", + "sha2", + "stringprep", + "thiserror 2.0.20", + "ttf-parser", + "weezl", +] + [[package]] name = "md-5" version = "0.10.6" @@ -802,6 +881,26 @@ dependencies = [ "pxfm", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom_locate" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d" +dependencies = [ + "bytecount", + "memchr", + "nom", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -836,7 +935,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" dependencies = [ "base64ct", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -852,6 +951,23 @@ dependencies = [ "sha2", ] +[[package]] +name = "pdf-extract" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28ba1758a3d3f361459645780e09570b573fc3c82637449e9963174c813a98" +dependencies = [ + "adobe-cmap-parser", + "cff-parser", + "encoding_rs", + "euclid 0.20.14", + "log", + "lopdf", + "postscript", + "type1-encoding-parser", + "unicode-normalization", +] + [[package]] name = "pdfrs" version = "0.1.9" @@ -927,6 +1043,18 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "pom" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60f6ce597ecdcc9a098e7fddacb1065093a3d66446fa16c675e7e71d1b5c28e6" + +[[package]] +name = "postscript" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306" + [[package]] name = "powerfmt" version = "0.2.0" @@ -946,6 +1074,15 @@ dependencies = [ "zip 0.6.6", ] +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -986,18 +1123,59 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + [[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + [[package]] name = "read-fonts" version = "0.39.2" @@ -1228,6 +1406,17 @@ version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -1448,6 +1637,7 @@ name = "tinydocs" version = "0.1.11" dependencies = [ "docx-rs", + "pdf-extract", "ppt-rs", "serde", "serde_json", @@ -1465,6 +1655,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.53.1" @@ -1565,6 +1770,15 @@ version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" +[[package]] +name = "type1-encoding-parser" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa10c302f5a53b7ad27fd42a3996e23d096ba39b5b8dd6d9e683a05b01bee749" +dependencies = [ + "pom", +] + [[package]] name = "typed-path" version = "0.12.3" @@ -1577,12 +1791,33 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "untrusted" version = "0.9.0" @@ -1663,6 +1898,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.127" @@ -1829,6 +2073,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "write-fonts" version = "0.48.1" diff --git a/Cargo.toml b/Cargo.toml index aee4759..bcd3b58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,11 @@ docx-rs = { version = "0.4.20", optional = true } # front-end this crate does not use, which is precisely why it is gated — a host # that only generates documents should not carry a syntax highlighter. ppt-rs = { version = "0.2.14", optional = true } +# `.pdf` text extraction. Optional: exclusive to the `pdf` feature. It brings a +# font and PostScript parsing stack (`lopdf`, CFF/Type1/CMap parsers) that only +# the extraction path needs, so a host that never reads a PDF should not carry +# it. +pdf-extract = { version = "0.10", optional = true } [dev-dependencies] # `.docx` output is a zip container; the tests re-open the produced bytes and @@ -60,11 +65,13 @@ name = "basic" required-features = ["docx"] [features] -default = ["docx", "pptx"] +default = ["docx", "pptx", "pdf"] # `.docx` generation via `docx-rs`. docx = ["dep:docx-rs"] # `.pptx` generation via `ppt-rs`. pptx = ["dep:ppt-rs"] +# `.pdf` text extraction via `pdf-extract`. +pdf = ["dep:pdf-extract"] # Lints apply to the whole crate and to every target. CI runs clippy with # `-D warnings`, so anything set to "warn" here fails the build in CI. diff --git a/README.md b/README.md index 41cb89c..316ce0f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # TinyDocs -Agent-friendly document synthesis in Rust: `.docx` and `.pptx`. +Agent-friendly document synthesis and text extraction in Rust: writes `.docx` +and `.pptx`, reads `.pdf`. `tinydocs` turns a typed, validated document spec into real office-format bytes. It is built for hosts that let a language model produce documents: the @@ -159,6 +160,7 @@ way, so the contract and its validation survive any combination. | --- | --- | --- | --- | | `docx` | on | `.docx` synthesis via `docx-rs` | `quick-xml` | | `pptx` | on | `.pptx` synthesis via `ppt-rs` | `syntect`, `pulldown-cmark`, `xml-rs` | +| `pdf` | on | `.pdf` text extraction via `pdf-extract` | `lopdf`, CFF/Type1/CMap parsers | ## Layout @@ -177,7 +179,10 @@ src/ │ ├── mod.rs # `generate` — the `WordprocessingML` mapping │ └── test.rs ├── pptx/ - ├── mod.rs # `generate` — the `PresentationML` mapping + image layout +│ ├── mod.rs # `generate` — the `PresentationML` mapping + image layout +│ └── test.rs +├── pdf/ + ├── mod.rs # `extract_text` — the one read path in the crate └── test.rs tests/ └── public_api.rs # integration tests against the public API only diff --git a/src/error/mod.rs b/src/error/mod.rs index fd8d8fd..e46245b 100644 --- a/src/error/mod.rs +++ b/src/error/mod.rs @@ -40,6 +40,22 @@ pub enum Error { /// Truncated underlying library error. detail: String, }, + + /// The underlying library failed to extract text from an input document. + /// + /// Distinct from [`Error::GenerationFailed`] because the two have opposite + /// causes and opposite remedies: generation fails on *our* output path and + /// usually means a bug or an exhausted resource, whereas extraction fails on + /// *someone else's* input and usually means the document is damaged, + /// encrypted, or carries no extractable text layer at all. A caller that + /// retries one should not retry the other. + /// + /// `detail` is truncated on the same bound as `GenerationFailed`. + #[error("text extraction failed: {detail}")] + ExtractionFailed { + /// Truncated underlying library error. + detail: String, + }, } impl Error { @@ -75,6 +91,15 @@ impl Error { out } + /// Build an [`Error::ExtractionFailed`] with `raw` truncated (UTF-8-safe) to + /// [`Error::MAX_DETAIL_CHARS`]. + #[must_use] + pub fn extraction_failed(raw: &str) -> Self { + Self::ExtractionFailed { + detail: Self::truncate_detail(raw), + } + } + /// Build an [`Error::InvalidInput`] for `field` violating `reason`. #[must_use] pub fn invalid_input(field: impl Into, reason: impl Into) -> Self { diff --git a/src/lib.rs b/src/lib.rs index da8d588..0e64fbc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -//! Agent-friendly document synthesis in Rust. +//! Agent-friendly document synthesis and text extraction in Rust. //! //! `tinydocs` turns a typed, validated document spec into real office-format //! bytes. It is built for hosts that let a language model produce documents: @@ -43,6 +43,11 @@ not(feature = "pptx"), doc = "- `pptx` (disabled in this build) — `.pptx` (OOXML `PresentationML`) synthesis." )] +#![cfg_attr(feature = "pdf", doc = "- [`pdf`] — `.pdf` text extraction.")] +#![cfg_attr( + not(feature = "pdf"), + doc = "- `pdf` (disabled in this build) — `.pdf` text extraction." +)] //! //! # Example //! @@ -75,6 +80,8 @@ //! - `docx` (default) — `.docx` synthesis via `docx-rs`. //! - `pptx` (default) — `.pptx` synthesis via `ppt-rs`, which also drops //! `syntect` and `pulldown-cmark`. +//! - `pdf` (default) — `.pdf` text extraction via `pdf-extract`, which also +//! drops its font and `PostScript` parsing stack. mod error; @@ -86,4 +93,7 @@ pub mod docx; #[cfg(feature = "pptx")] pub mod pptx; +#[cfg(feature = "pdf")] +pub mod pdf; + pub use error::{Error, Result}; diff --git a/src/pdf/mod.rs b/src/pdf/mod.rs new file mode 100644 index 0000000..e07ff76 --- /dev/null +++ b/src/pdf/mod.rs @@ -0,0 +1,88 @@ +//! Text extraction from `.pdf` documents, backed by +//! [`pdf-extract`](https://crates.io/crates/pdf-extract). +//! +//! This is the one module in the crate that reads rather than writes, and the +//! asymmetry is worth stating plainly: everything else here turns a spec a caller +//! authored into bytes, whereas [`extract_text`] takes a document somebody else +//! produced and recovers what it says. There is no spec and nothing to validate +//! beyond "is this a PDF at all" — the input is arbitrary and often damaged. +//! +//! Like the synthesis modules, this is **synchronous and CPU-bound** and holds no +//! opinion about executors or deadlines. That matters more here than elsewhere: +//! extraction time scales with the document, not with a spec this crate has +//! already bounded, so a host handling untrusted PDFs wants both a blocking-pool +//! hop *and* a timeout. Only the host knows what either should be. +//! +//! # What it does not recover +//! +//! Extraction reads the text layer. A scanned page holds an image of text and no +//! text layer, so it yields nothing — that is not a failure to retry but a +//! document that needs OCR, which is out of scope here. Encrypted documents and +//! damaged cross-reference tables surface as [`Error::ExtractionFailed`]. + +use crate::{Error, Result}; + +/// Maximum size, in bytes, of a document [`extract_text`] will accept. +/// +/// Extraction allocates well beyond the input size while parsing, and the caller +/// is usually handing over something it did not produce. A host that wants a +/// tighter bound should apply it before calling; this one exists so an +/// unbounded input cannot become an unbounded allocation by default. +pub const MAX_DOCUMENT_BYTES: usize = 64 * 1024 * 1024; + +/// Extract the text layer of the PDF in `bytes`. +/// +/// Returns the document's text with the library's own layout decisions intact — +/// no normalisation, trimming, or truncation is applied, because how to bound +/// extracted text is a host policy that depends on what the text is for. +/// +/// Synchronous and CPU-bound, and unlike synthesis its cost is set by the input +/// rather than by a validated spec. Run it on a blocking pool under a timeout. +/// +/// # Errors +/// +/// - [`Error::InvalidInput`] if `bytes` is empty, exceeds +/// [`MAX_DOCUMENT_BYTES`], or does not begin with the `%PDF-` signature. +/// - [`Error::ExtractionFailed`] if the document cannot be parsed — damaged, +/// encrypted, or otherwise unreadable. +/// +/// A document that parses cleanly but carries no text layer, such as a scan, is +/// **not** an error: it yields an empty string. +/// +/// # Examples +/// +/// ``` +/// # fn main() -> Result<(), tinydocs::Error> { +/// let not_a_pdf = b"GIF89a"; +/// assert!(tinydocs::pdf::extract_text(not_a_pdf).is_err()); +/// # Ok(()) +/// # } +/// ``` +pub fn extract_text(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err(Error::invalid_input("bytes", "must not be empty")); + } + if bytes.len() > MAX_DOCUMENT_BYTES { + return Err(Error::invalid_input( + "bytes", + format!("must be ≤ {MAX_DOCUMENT_BYTES} bytes"), + )); + } + // Checked here rather than left to the parser so that "you handed me a JPEG" + // is an `InvalidInput` naming the field, not an `ExtractionFailed` carrying + // a parser's phrasing. The signature may be preceded by junk in the wild, + // but a leading `%PDF-` is what every conforming producer emits and what the + // parser needs to find the header. + if !bytes.starts_with(b"%PDF-") { + return Err(Error::invalid_input( + "bytes", + "must be a PDF document (no %PDF- signature)", + )); + } + + pdf_extract::extract_text_from_mem(bytes) + .map_err(|err| Error::extraction_failed(&err.to_string())) +} + +#[cfg(test)] +mod test; diff --git a/src/pdf/test.rs b/src/pdf/test.rs new file mode 100644 index 0000000..4b51375 --- /dev/null +++ b/src/pdf/test.rs @@ -0,0 +1,115 @@ +//! Unit tests for PDF text extraction. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::{MAX_DOCUMENT_BYTES, extract_text}; +use crate::Error; + +/// Build a valid single-page PDF whose text layer holds `text`. +/// +/// Assembled here rather than checked in as a binary fixture so the structure +/// under test is readable, and so the cross-reference offsets are computed from +/// the bytes actually emitted instead of being transcribed and going stale. The +/// document uses Helvetica, one of the base-14 fonts every reader knows, so no +/// font program has to be embedded. +fn pdf_with_text(text: &str) -> Vec { + let content = format!("BT /F1 24 Tf 72 700 Td ({text}) Tj ET\n"); + let objects = [ + "<< /Type /Catalog /Pages 2 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \ + /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>" + .to_string(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(), + format!( + "<< /Length {} >>\nstream\n{content}endstream", + content.len() + ), + ]; + + let mut out = Vec::new(); + out.extend_from_slice(b"%PDF-1.4\n"); + let mut offsets = Vec::with_capacity(objects.len()); + for (i, body) in objects.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", i + 1).as_bytes()); + } + + let xref_offset = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n", objects.len() + 1).as_bytes()); + out.extend_from_slice(b"0000000000 65535 f \n"); + for offset in &offsets { + out.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice( + format!( + "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref_offset}\n%%EOF\n", + objects.len() + 1 + ) + .as_bytes(), + ); + out +} + +#[test] +fn extracts_the_text_layer_of_a_valid_document() { + let text = extract_text(&pdf_with_text("Hello tinydocs")).expect("extraction should succeed"); + assert!( + text.contains("Hello tinydocs"), + "extracted text missing the content: {text:?}" + ); +} + +#[test] +fn rejects_empty_input() { + match extract_text(&[]) { + Err(Error::InvalidInput { field, .. }) => assert_eq!(field, "bytes"), + other => panic!("expected InvalidInput, got {other:?}"), + } +} + +#[test] +fn rejects_input_without_a_pdf_signature() { + // A JPEG handed to the PDF path should name the offending field rather than + // surfacing a parser's phrasing as an extraction failure. + match extract_text(b"\xFF\xD8\xFFnot a pdf") { + Err(Error::InvalidInput { field, reason }) => { + assert_eq!(field, "bytes"); + assert!(reason.contains("PDF"), "unhelpful reason: {reason}"); + } + other => panic!("expected InvalidInput, got {other:?}"), + } +} + +#[test] +fn rejects_an_oversize_document() { + // A real signature followed by filler, so the rejection is the size check + // rather than the signature check. + let mut bytes = b"%PDF-1.4\n".to_vec(); + bytes.resize(MAX_DOCUMENT_BYTES + 1, b' '); + match extract_text(&bytes) { + Err(Error::InvalidInput { field, .. }) => assert_eq!(field, "bytes"), + other => panic!("expected InvalidInput, got {other:?}"), + } +} + +#[test] +fn a_damaged_document_fails_extraction_rather_than_validation() { + // Correct signature, nothing else: it is a PDF as far as the boundary check + // can tell, and the parser is the thing that has to reject it. This is the + // branch that distinguishes `ExtractionFailed` from `InvalidInput`. + match extract_text(b"%PDF-1.4\nthis is not a cross-reference table\n") { + Err(Error::ExtractionFailed { detail }) => { + assert!(!detail.is_empty(), "extraction error carried no detail"); + } + other => panic!("expected ExtractionFailed, got {other:?}"), + } +} + +#[test] +fn a_document_with_no_text_layer_yields_empty_text_rather_than_an_error() { + // A valid page carrying no text object at all — the shape a scanned page + // has. Nothing to extract is not a failure to retry. + let text = extract_text(&pdf_with_text("")).expect("extraction should succeed"); + assert!(text.trim().is_empty(), "expected no text, got {text:?}"); +} From feb865a1186da7c0881ccf857d42c3517fe2804f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 11 Aug 2026 01:13:40 +0300 Subject: [PATCH 05/13] Move document bytes over the bus in bounded chunks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The module could only ever return a `.docx` inline, and that stops working the moment the other two formats arrive. A TinyBus frame is a 16 MiB JSON document and a `Vec` serialises as an array of integers — roughly 3.5 bytes of frame per byte of payload — so the real inline ceiling is a few megabytes. A deck may legally carry 8 images of 5 MiB, and a `.pdf` handed in for extraction is bounded only by what the host accepted. TinyBus says as much itself: large payloads are meant to travel as paths, not inline. So bytes now move through a staging area in base64 chunks (1.34x rather than 3.5x), addressed by opaque blob ids, and no method returns bytes inline. A caller stages a document, calls a format method, and reads the result back the same way; frame size stops being part of the contract, and the caller's code path is the same regardless of size. Every bound in `blobs` is load-bearing rather than defensive. A module is trusted in-process code that TinyBus never unloads, so an abandoned upload is never reclaimed by a process exit that does not come — hence four independent limits (per chunk, per blob, total staged, blob count) and expiry of untouched blobs. The budget is reserved at `BeginBlob` rather than counted on arrival, so an admitted transfer can always finish instead of failing halfway when somebody else fills the area. Transfers are append-only: `offset` must equal the bytes received so far, which makes "complete" mean "length reached" and turns a lost or duplicated chunk into a named error at the moment it happens rather than a corrupt blob discovered later. Completion verifies the caller's SHA-256 before the blob becomes readable. Expiry takes the clock as a parameter instead of calling `Instant::now`, which is what makes the TTL rules testable at all; the tests drive time explicitly and cover both directions — an abandoned blob is reaped, a slow but live transfer never is. Transfer errors are grouped by what the caller should do next rather than by what went wrong internally: `UnknownBlob` means restart, `TransferRefused` means a budget is full and the same request may work later, `TransferFailed` means re-send. `BlobStore`'s `Debug` is written by hand so staged bytes cannot reach a log line. This replaces `ai.tinyhumans.tinydocs.Docx` rather than extending it. Returning a `BlobRef` where callers expect bytes is a breaking contract, and TinyBus's module guidance is explicit that those get a new interface name. It is not served alongside the old one because `module_export!` attaches its method list to the first entry in `provides` and leaves any others empty, so a second fully-declared interface would have to under-declare its members and break the invariant that manifest methods and dispatch members stay identical. Serving both needs a TinyBus change; retiring one at a pre-1.0 minor bump does not. The loader E2E test now covers all three formats through the real dynamic loader and moves an image across several chunks — a single-chunk transfer would not prove the offsets line up, which is the whole point of the change. The one `#[allow]` added is `clippy::unused_async` on the interface block: the macro rejects a non-async method outright, so the four transfer methods are async because the dispatch contract requires it, and the lint can never be actionable there. Co-authored-by: Medulla --- Cargo.lock | 5 + README.md | 27 +- crates/tinydocs-module/Cargo.toml | 21 +- crates/tinydocs-module/src/blobs/mod.rs | 517 +++++++++++++++++++++ crates/tinydocs-module/src/blobs/test.rs | 447 ++++++++++++++++++ crates/tinydocs-module/src/lib.rs | 4 +- crates/tinydocs-module/src/service/mod.rs | 280 +++++++++-- crates/tinydocs-module/src/service/test.rs | 434 ++++++++++++++++- crates/tinydocs-module/src/service/wire.rs | 65 +++ crates/tinydocs-module/tests/module_e2e.rs | 237 +++++++++- docs/specs/tinybus-module.md | 76 ++- 11 files changed, 2034 insertions(+), 79 deletions(-) create mode 100644 crates/tinydocs-module/src/blobs/mod.rs create mode 100644 crates/tinydocs-module/src/blobs/test.rs create mode 100644 crates/tinydocs-module/src/service/wire.rs diff --git a/Cargo.lock b/Cargo.lock index 37d8ec7..bce4e6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1649,6 +1649,11 @@ dependencies = [ name = "tinydocs-module" version = "0.1.11" dependencies = [ + "base64 0.22.1", + "serde", + "serde_json", + "sha2", + "thiserror 2.0.20", "tinybus", "tinybus-module", "tinydocs", diff --git a/README.md b/README.md index 316ce0f..c43d239 100644 --- a/README.md +++ b/README.md @@ -124,12 +124,35 @@ cargo build --release --package tinydocs-module The native artifact is `target/release/libtinydocs_module.so` on Linux, `libtinydocs_module.dylib` on macOS, or `tinydocs_module.dll` on Windows. Load it with a TinyBus host built with its `modules` feature. It claims -`ai.tinyhumans.tinydocs.Docx` at `/ai/tinyhumans/tinydocs/Docx` and exposes: +`ai.tinyhumans.tinydocs.Documents` at `/ai/tinyhumans/tinydocs/Documents` and +exposes the three format operations plus the chunked transfer they depend on: ```text -GenerateDocx(DocumentSpec) -> Vec +BeginBlob(total_bytes, sha256) -> blob_id +PutChunk(blob_id, offset, base64) -> bytes received so far +GetChunk(blob_id, offset, len) -> base64 +ReleaseBlob(blob_id) -> () +GenerateDocx(DocumentSpec) -> BlobRef +GeneratePptx(deck with image blobs) -> BlobRef +ExtractText(blob_id) -> BlobRef ``` +Nothing returns bytes inline. A TinyBus frame is a 16 MiB JSON document, and a +`Vec` serialises as an array of integers — roughly 3.5 bytes of frame per +byte of payload — so the real inline ceiling is a few megabytes. That is below a +deck's legal image payload and below any `.pdf` worth extracting. So every +unbounded value is staged and moved in base64 chunks, and a caller's code path is +the same regardless of size. + +The staging area is bounded in four independent ways — per chunk, per blob, in +total, and by blob count — and blobs that stop being touched expire. A module is +trusted in-process code that TinyBus never unloads, so an abandoned upload is +never reclaimed by a process exit that does not come. + +This interface replaces `ai.tinyhumans.tinydocs.Docx`, which returned bytes +inline. TinyBus's guidance is that an existing interface must not change in +place, so the new contract took a new name. + The release workflow attaches installable Linux and macOS bundles containing the matching TinyBus host, the TinyDocs module, a SHA-256 `modules.toml` allowlist, and protocol/module documentation. It also publishes diff --git a/crates/tinydocs-module/Cargo.toml b/crates/tinydocs-module/Cargo.toml index 057d784..247137b 100644 --- a/crates/tinydocs-module/Cargo.toml +++ b/crates/tinydocs-module/Cargo.toml @@ -13,13 +13,32 @@ crate-type = ["rlib", "cdylib"] [dependencies] # The pure document library remains independently publishable and bus-agnostic. -tinydocs = { path = "../..", default-features = false, features = ["docx"] } +tinydocs = { path = "../..", default-features = false, features = ["docx", "pptx", "pdf"] } # TinyBus provides the typed service interface and dynamic module host ABI. tinybus = { version = "0.1.0", path = "../../vendor/tinybus/crates/tinybus", default-features = false, features = ["macros", "modules"] } # The module-side SDK owns its runtime and exports the stable C entrypoints. tinybus-module = { version = "0.1.0", path = "../../vendor/tinybus/crates/tinybus-module" } # Module methods move CPU-bound document synthesis onto Tokio's blocking pool. tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +# Bus frames are JSON, where a byte array costs ~3.5 bytes per byte. Chunks are +# base64 (1.34x) so a 4 MiB chunk fits a frame with room to spare. +base64 = "0.22" +# Chunked transfer verifies each assembled blob against a caller-declared digest +# before it becomes readable, so a truncated upload cannot be consumed as whole. +sha2 = "0.10" +# The staging area's refusals are a taxonomy the caller matches on. +thiserror = "2" +# `BlobRef` and the wire deck shape are the bus contract, so they derive the +# same serde surface the library specs do. +serde = { version = "1", features = ["derive"] } [lints] workspace = true + +[dev-dependencies] +# The loader E2E test speaks the base64 transfer surface the same way a host does. +base64 = "0.22" +# `GeneratePptx` takes a deck whose images name staged blobs; the test builds that +# document directly rather than depending on the module's own wire types, so a +# rename here would be caught as a contract change. +serde_json = "1" diff --git a/crates/tinydocs-module/src/blobs/mod.rs b/crates/tinydocs-module/src/blobs/mod.rs new file mode 100644 index 0000000..d5c2664 --- /dev/null +++ b/crates/tinydocs-module/src/blobs/mod.rs @@ -0,0 +1,517 @@ +//! Chunked byte transfer for payloads that do not fit in a bus frame. +//! +//! # Why this exists +//! +//! A `TinyBus` frame is a JSON document capped at 16 MiB, and `TinyBus`'s own +//! guidance is that large payloads travel as paths rather than inline. Neither +//! half of the document surface fits inside that: a deck may legally carry +//! 8 images of 5 MiB each, and a `.pdf` handed in for extraction is bounded only +//! by what the host accepted. Serialising bytes as a JSON array of integers +//! makes it worse — roughly 3.5 bytes of frame per byte of payload — so the +//! real inline ceiling is a few megabytes, not sixteen. +//! +//! So bytes move in chunks, base64-encoded (1.34× rather than 3.5×), through a +//! staging area addressed by opaque blob ids. A caller stages a `.pdf`, calls +//! `ExtractText`, and reads the result back out the same way; the frame size +//! stops being part of the contract. +//! +//! # Every limit here is load-bearing +//! +//! A module is trusted in-process code that `TinyBus` never unloads, so an +//! abandoned upload is not garbage collected by a process exit that never comes. +//! The store therefore bounds four separate things — one chunk, one blob, the +//! whole staging area, and the number of live blobs — and expires blobs that +//! stop being touched. Without the last one, a caller that dies mid-upload leaks +//! its partial blob for the life of the host. +//! +//! Expiry is lazy: every operation sweeps first, so there is no background task +//! and no timer to reason about. The clock is a parameter rather than a call to +//! [`Instant::now`], which is what makes the expiry rules testable at all. +//! +//! # Append-only by construction +//! +//! `put_chunk` requires `offset` to equal exactly how many bytes have arrived so +//! far. That is stricter than necessary, and deliberately so: sparse writes would +//! need range bookkeeping, a definition of what overlapping writes mean, and a +//! way to know when a blob is actually complete. Requiring append makes +//! "complete" mean "length reached", and makes a lost or duplicated chunk a named +//! error at the moment it happens rather than a corrupt blob discovered later. +//! +//! Completion verifies the caller's SHA-256 before the blob becomes readable, so +//! a truncated or reordered transfer cannot be consumed as though it were whole. + +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// Maximum size of a single chunk. +/// +/// Sized so that one chunk plus its base64 expansion and the surrounding JSON +/// stays well inside a 16 MiB frame, with room left for a method envelope. +pub const MAX_CHUNK_BYTES: usize = 4 * 1024 * 1024; + +/// Maximum size of one staged blob. +pub const MAX_BLOB_BYTES: usize = 64 * 1024 * 1024; + +/// Maximum total size of all staged blobs at once. +/// +/// Bounds the module's resident memory independently of how many callers are +/// mid-transfer. +pub const MAX_TOTAL_STAGED_BYTES: usize = 128 * 1024 * 1024; + +/// Maximum number of blobs alive at once. +/// +/// A separate bound from the byte budget: many tiny abandoned blobs are as much +/// of a leak as one large one, and each carries bookkeeping of its own. +pub const MAX_LIVE_BLOBS: usize = 64; + +/// How long a blob may go untouched before it is expired. +/// +/// Long enough that a slow but live transfer is never reaped, short enough that +/// an abandoned one does not outlive the request that started it by much. +pub const IDLE_TTL: Duration = Duration::from_secs(300); + +/// A handle to a complete staged blob, plus what a caller needs to read it back. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BlobRef { + /// Opaque identifier for the blob. + pub blob_id: String, + /// Total size in bytes, so a caller knows how many chunks to ask for. + pub total_bytes: u64, + /// Lowercase hex SHA-256 of the bytes, so a caller can verify what it read. + pub sha256: String, +} + +/// Why a blob operation was refused. +/// +/// Every variant is a distinct condition with a distinct wire name, because a +/// caller's correct response differs: a budget refusal is worth retrying later, +/// a hash mismatch means re-sending, and an unknown id means the blob is gone +/// and the whole transfer has to start again. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum BlobError { + /// The declared SHA-256 was not 64 lowercase hexadecimal characters. + #[error("sha256 must be 64 lowercase hexadecimal characters")] + MalformedDigest, + + /// The declared total size exceeds [`MAX_BLOB_BYTES`]. + #[error("blob size exceeds the {MAX_BLOB_BYTES}-byte per-blob limit")] + BlobTooLarge, + + /// A single chunk exceeded [`MAX_CHUNK_BYTES`]. + #[error("chunk exceeds the {MAX_CHUNK_BYTES}-byte per-chunk limit")] + ChunkTooLarge, + + /// Accepting the blob would exceed [`MAX_TOTAL_STAGED_BYTES`]. + #[error("staging area is full")] + StagingFull, + + /// [`MAX_LIVE_BLOBS`] blobs are already staged. + #[error("too many blobs staged at once")] + TooManyBlobs, + + /// No blob with that id — never staged, released, or expired. + #[error("unknown blob id")] + UnknownBlob, + + /// `offset` did not equal the number of bytes received so far. + #[error("chunk offset {actual} does not continue the blob at {expected}")] + OutOfOrderChunk { + /// The offset the next chunk must carry. + expected: u64, + /// The offset the caller sent. + actual: u64, + }, + + /// The chunk would write past the declared total size. + #[error("chunk would exceed the declared blob size")] + OverlongBlob, + + /// The assembled bytes did not hash to the declared digest. + #[error("assembled blob does not match the declared sha256")] + DigestMismatch, + + /// The blob is still being uploaded and cannot be read yet. + #[error("blob is incomplete")] + IncompleteBlob, + + /// A read started past the end of the blob. + #[error("read offset is past the end of the blob")] + ReadPastEnd, +} + +/// One blob in the staging area. +struct Blob { + /// Declared total size; the blob is complete when `data` reaches it. + expected_bytes: usize, + /// Declared digest, verified once the blob is complete. + expected_sha256: String, + data: Vec, + complete: bool, + last_touched: Instant, +} + +impl Blob { + /// Bytes charged against the staging budget. + /// + /// The declared total, not the bytes received so far: the budget is reserved + /// at `begin` so a transfer that is admitted can always finish, rather than + /// failing halfway when somebody else fills the area. + fn reserved(&self) -> usize { + self.expected_bytes.max(self.data.len()) + } +} + +/// The staging area shared by every method on the service. +#[derive(Default)] +pub struct BlobStore { + inner: Mutex, +} + +/// Reports how much is staged, never what is staged. +/// +/// Written by hand rather than derived because a derived implementation would +/// put every staged byte into whatever formatted it — a log line, a panic +/// message, an error. Staged bytes are caller data. +impl std::fmt::Debug for BlobStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let inner = self.lock(); + f.debug_struct("BlobStore") + .field("live_blobs", &inner.blobs.len()) + .field("staged_bytes", &inner.staged_bytes()) + .finish() + } +} + +#[derive(Default)] +struct Inner { + blobs: HashMap, + next_id: u64, +} + +impl BlobStore { + /// An empty store. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Reserve space for a blob of `total_bytes` that will hash to `sha256`. + /// + /// # Errors + /// + /// [`BlobError::MalformedDigest`], [`BlobError::BlobTooLarge`], + /// [`BlobError::TooManyBlobs`], or [`BlobError::StagingFull`]. + pub fn begin(&self, total_bytes: u64, sha256: &str, now: Instant) -> Result { + if !is_lowercase_sha256(sha256) { + return Err(BlobError::MalformedDigest); + } + let expected_bytes = usize::try_from(total_bytes).map_err(|_| BlobError::BlobTooLarge)?; + if expected_bytes > MAX_BLOB_BYTES { + return Err(BlobError::BlobTooLarge); + } + + let mut inner = self.lock(); + inner.sweep_expired(now); + if inner.blobs.len() >= MAX_LIVE_BLOBS { + return Err(BlobError::TooManyBlobs); + } + if inner.staged_bytes().saturating_add(expected_bytes) > MAX_TOTAL_STAGED_BYTES { + return Err(BlobError::StagingFull); + } + + let id = inner.allocate_id(); + inner.blobs.insert( + id.clone(), + Blob { + expected_bytes, + expected_sha256: sha256.to_string(), + // Not pre-allocated: a caller can declare 64 MiB and never send + // it, and reserving the allocation up front would make that a + // way to spend the host's memory for free. + data: Vec::new(), + complete: expected_bytes == 0, + last_touched: now, + }, + ); + // A zero-length blob is complete on arrival, so its digest is checked + // here rather than on a chunk that will never come. + if expected_bytes == 0 { + let verified = verify(&[], sha256); + if !verified { + inner.blobs.remove(&id); + return Err(BlobError::DigestMismatch); + } + } + Ok(id) + } + + /// Append `data` at `offset`, returning the number of bytes received so far. + /// + /// When the blob reaches its declared size, its digest is verified and it + /// becomes readable. A mismatch drops the blob. + /// + /// # Errors + /// + /// [`BlobError::ChunkTooLarge`], [`BlobError::UnknownBlob`], + /// [`BlobError::OutOfOrderChunk`], [`BlobError::OverlongBlob`], or + /// [`BlobError::DigestMismatch`]. + pub fn put_chunk( + &self, + blob_id: &str, + offset: u64, + data: &[u8], + now: Instant, + ) -> Result { + if data.len() > MAX_CHUNK_BYTES { + return Err(BlobError::ChunkTooLarge); + } + + let mut inner = self.lock(); + inner.sweep_expired(now); + let blob = inner.blobs.get_mut(blob_id).ok_or(BlobError::UnknownBlob)?; + + let received = blob.data.len() as u64; + if blob.complete || offset != received { + return Err(BlobError::OutOfOrderChunk { + expected: received, + actual: offset, + }); + } + if blob.data.len().saturating_add(data.len()) > blob.expected_bytes { + return Err(BlobError::OverlongBlob); + } + + blob.data.extend_from_slice(data); + blob.last_touched = now; + if blob.data.len() == blob.expected_bytes { + if verify(&blob.data, &blob.expected_sha256) { + blob.complete = true; + } else { + inner.blobs.remove(blob_id); + return Err(BlobError::DigestMismatch); + } + } + // Re-read rather than reuse the borrow above: the mismatch branch may + // have removed the entry. + Ok(inner + .blobs + .get(blob_id) + .map_or(0, |blob| blob.data.len() as u64)) + } + + /// Read up to `len` bytes of a complete blob starting at `offset`. + /// + /// A read that runs past the end is clamped rather than refused, so a caller + /// can ask for a whole chunk on the final read without special-casing the + /// remainder. + /// + /// # Errors + /// + /// [`BlobError::UnknownBlob`], [`BlobError::IncompleteBlob`], + /// [`BlobError::ChunkTooLarge`], or [`BlobError::ReadPastEnd`]. + pub fn get_chunk( + &self, + blob_id: &str, + offset: u64, + len: u64, + now: Instant, + ) -> Result, BlobError> { + let len = usize::try_from(len).map_err(|_| BlobError::ChunkTooLarge)?; + if len > MAX_CHUNK_BYTES { + return Err(BlobError::ChunkTooLarge); + } + + let mut inner = self.lock(); + inner.sweep_expired(now); + let blob = inner.blobs.get_mut(blob_id).ok_or(BlobError::UnknownBlob)?; + if !blob.complete { + return Err(BlobError::IncompleteBlob); + } + let start = usize::try_from(offset).map_err(|_| BlobError::ReadPastEnd)?; + if start > blob.data.len() { + return Err(BlobError::ReadPastEnd); + } + blob.last_touched = now; + let end = start.saturating_add(len).min(blob.data.len()); + Ok(blob.data[start..end].to_vec()) + } + + /// Stage `bytes` as an already-complete blob and return its handle. + /// + /// This is the outbound direction: a generated document or an extracted text + /// body that the module produced and the caller now has to read back. + /// + /// # Errors + /// + /// [`BlobError::BlobTooLarge`], [`BlobError::TooManyBlobs`], or + /// [`BlobError::StagingFull`]. + pub fn insert_complete(&self, bytes: Vec, now: Instant) -> Result { + if bytes.len() > MAX_BLOB_BYTES { + return Err(BlobError::BlobTooLarge); + } + + let mut inner = self.lock(); + inner.sweep_expired(now); + if inner.blobs.len() >= MAX_LIVE_BLOBS { + return Err(BlobError::TooManyBlobs); + } + if inner.staged_bytes().saturating_add(bytes.len()) > MAX_TOTAL_STAGED_BYTES { + return Err(BlobError::StagingFull); + } + + let sha256 = hex_digest(&bytes); + let total_bytes = bytes.len() as u64; + let id = inner.allocate_id(); + inner.blobs.insert( + id.clone(), + Blob { + expected_bytes: bytes.len(), + expected_sha256: sha256.clone(), + data: bytes, + complete: true, + last_touched: now, + }, + ); + Ok(BlobRef { + blob_id: id, + total_bytes, + sha256, + }) + } + + /// Remove a complete blob and return its bytes. + /// + /// Used when the module consumes a staged input — the bytes of a `.pdf`, or + /// an image for a deck. Taking rather than copying frees the staging budget + /// at the moment the blob stops being needed. + /// + /// # Errors + /// + /// [`BlobError::UnknownBlob`] or [`BlobError::IncompleteBlob`]. + pub fn take_complete(&self, blob_id: &str, now: Instant) -> Result, BlobError> { + let mut inner = self.lock(); + inner.sweep_expired(now); + let blob = inner.blobs.get(blob_id).ok_or(BlobError::UnknownBlob)?; + if !blob.complete { + return Err(BlobError::IncompleteBlob); + } + Ok(inner + .blobs + .remove(blob_id) + .map(|blob| blob.data) + .unwrap_or_default()) + } + + /// Drop a blob and free its budget. + /// + /// # Errors + /// + /// [`BlobError::UnknownBlob`] if there is nothing to release. Releasing is + /// reported rather than silently accepted so a caller learns that its blob + /// had already expired. + pub fn release(&self, blob_id: &str, now: Instant) -> Result<(), BlobError> { + let mut inner = self.lock(); + inner.sweep_expired(now); + inner + .blobs + .remove(blob_id) + .map(|_| ()) + .ok_or(BlobError::UnknownBlob) + } + + /// Number of blobs currently staged, for tests and diagnostics. + #[must_use] + pub fn live_count(&self) -> usize { + self.lock().blobs.len() + } + + /// Take the lock, recovering from a poisoned mutex. + /// + /// A panic while holding this lock can only have happened between two + /// `HashMap` operations, so the map is structurally intact and the worst + /// case is one blob left in a partial state — which its digest check will + /// reject. Refusing every subsequent request would turn one caller's panic + /// into a dead module, and `TinyBus` never unloads a module to recover. + fn lock(&self) -> std::sync::MutexGuard<'_, Inner> { + self.inner + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +impl Inner { + /// Total bytes reserved by every staged blob. + fn staged_bytes(&self) -> usize { + self.blobs + .values() + .map(Blob::reserved) + .fold(0usize, usize::saturating_add) + } + + /// Drop every blob untouched for longer than [`IDLE_TTL`]. + fn sweep_expired(&mut self, now: Instant) { + self.blobs + .retain(|_, blob| now.saturating_duration_since(blob.last_touched) <= IDLE_TTL); + } + + /// Allocate an unused blob id. + /// + /// A counter, not a random value: ids are opaque handles inside one process, + /// never authorisation tokens, and a counter makes a leaked id visible in a + /// log rather than looking like a secret. + fn allocate_id(&mut self) -> String { + self.next_id = self.next_id.wrapping_add(1); + format!("blob-{}", self.next_id) + } +} + +/// Whether `value` is exactly 64 lowercase hexadecimal characters. +fn is_lowercase_sha256(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +/// Lowercase hex SHA-256 of `bytes`, in the exact shape `BeginBlob` expects. +/// +/// Public because declaring a digest is part of using the transfer surface: a +/// caller has to produce this value, and one implementation both sides agree on +/// beats two that can disagree about case or padding. +#[must_use] +pub fn hex_digest(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let mut out = String::with_capacity(64); + for byte in digest { + use std::fmt::Write as _; + // Writing into a String cannot fail; the result is discarded rather than + // unwrapped so this stays panic-free. + let _ = write!(out, "{byte:02x}"); + } + out +} + +/// Whether `bytes` hashes to `expected`, compared without early exit. +fn verify(bytes: &[u8], expected: &str) -> bool { + let actual = hex_digest(bytes); + // Constant-time over the digest strings. The digest is not a secret, so this + // is defence in depth rather than a requirement — but a hash comparison is + // exactly the shape that later becomes security-relevant, and the cost of + // getting it right once is nothing. + if actual.len() != expected.len() { + return false; + } + actual + .bytes() + .zip(expected.bytes()) + .fold(0u8, |acc, (a, b)| acc | (a ^ b)) + == 0 +} + +#[cfg(test)] +mod test; diff --git a/crates/tinydocs-module/src/blobs/test.rs b/crates/tinydocs-module/src/blobs/test.rs new file mode 100644 index 0000000..67a87d8 --- /dev/null +++ b/crates/tinydocs-module/src/blobs/test.rs @@ -0,0 +1,447 @@ +//! Unit tests for the chunked blob staging area. +//! +//! Weighted towards refusals on purpose. A happy-path transfer proves the store +//! can move bytes; it is the bounds and the expiry that decide whether an +//! abandoned upload leaks for the life of a module the host never unloads, and +//! whether a truncated transfer can be consumed as though it were whole. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::time::{Duration, Instant}; + +use super::{ + BlobError, BlobStore, IDLE_TTL, MAX_BLOB_BYTES, MAX_CHUNK_BYTES, MAX_LIVE_BLOBS, + MAX_TOTAL_STAGED_BYTES, hex_digest, is_lowercase_sha256, verify, +}; + +/// A fixed origin for the injected clock, so every test is deterministic. +fn t0() -> Instant { + Instant::now() +} + +/// Stage `bytes` as a completed upload, the way a caller would. +fn upload(store: &BlobStore, bytes: &[u8], now: Instant) -> String { + let id = store + .begin(bytes.len() as u64, &hex_digest(bytes), now) + .expect("begin should succeed"); + if !bytes.is_empty() { + let received = store + .put_chunk(&id, 0, bytes, now) + .expect("put_chunk should succeed"); + assert_eq!(received, bytes.len() as u64); + } + id +} + +#[test] +fn a_blob_round_trips_through_chunks() { + let store = BlobStore::new(); + let now = t0(); + let payload: Vec = (0..10_000u32).map(|i| (i % 251) as u8).collect(); + let digest = hex_digest(&payload); + + let id = store.begin(payload.len() as u64, &digest, now).unwrap(); + // Three uneven chunks, so the arithmetic is not accidentally aligned. + let mut sent = 0usize; + for size in [4_000usize, 4_000, 2_000] { + let end = sent + size; + let received = store + .put_chunk(&id, sent as u64, &payload[sent..end], now) + .unwrap(); + sent = end; + assert_eq!(received, sent as u64); + } + + let mut read = Vec::new(); + let mut offset = 0u64; + while read.len() < payload.len() { + let chunk = store.get_chunk(&id, offset, 3_000, now).unwrap(); + assert!(!chunk.is_empty(), "read stalled at offset {offset}"); + offset += chunk.len() as u64; + read.extend_from_slice(&chunk); + } + assert_eq!(read, payload); + assert_eq!(hex_digest(&read), digest); +} + +#[test] +fn a_read_past_the_end_is_clamped_not_refused() { + // So a caller can ask for a full chunk on the final read without having to + // compute the remainder itself. + let store = BlobStore::new(); + let now = t0(); + let id = upload(&store, b"twelve bytes", now); + let chunk = store.get_chunk(&id, 6, 1_000_000, now).unwrap(); + assert_eq!(chunk, b" bytes"); +} + +#[test] +fn a_read_starting_past_the_end_is_refused() { + let store = BlobStore::new(); + let now = t0(); + let id = upload(&store, b"short", now); + assert_eq!( + store.get_chunk(&id, 99, 10, now), + Err(BlobError::ReadPastEnd) + ); +} + +#[test] +fn an_out_of_order_chunk_is_refused_and_names_both_offsets() { + let store = BlobStore::new(); + let now = t0(); + let payload = vec![7u8; 100]; + let id = store.begin(100, &hex_digest(&payload), now).unwrap(); + store.put_chunk(&id, 0, &payload[..40], now).unwrap(); + + // A duplicated chunk and a skipped chunk are the two ways a transfer goes + // wrong; both must fail here rather than corrupt the blob silently. + assert_eq!( + store.put_chunk(&id, 0, &payload[..40], now), + Err(BlobError::OutOfOrderChunk { + expected: 40, + actual: 0 + }) + ); + assert_eq!( + store.put_chunk(&id, 60, &payload[60..], now), + Err(BlobError::OutOfOrderChunk { + expected: 40, + actual: 60 + }) + ); +} + +#[test] +fn a_chunk_past_the_declared_size_is_refused() { + let store = BlobStore::new(); + let now = t0(); + let payload = vec![1u8; 10]; + let id = store.begin(10, &hex_digest(&payload), now).unwrap(); + let one_too_many = [1u8; 11]; + assert_eq!( + store.put_chunk(&id, 0, &one_too_many, now), + Err(BlobError::OverlongBlob) + ); +} + +#[test] +fn an_oversize_chunk_is_refused() { + let store = BlobStore::new(); + let now = t0(); + let payload = vec![0u8; MAX_CHUNK_BYTES + 1]; + let id = store + .begin(payload.len() as u64, &hex_digest(&payload), now) + .unwrap(); + assert_eq!( + store.put_chunk(&id, 0, &payload, now), + Err(BlobError::ChunkTooLarge) + ); +} + +#[test] +fn a_digest_mismatch_drops_the_blob() { + // The blob must not remain readable, and must not remain charged against the + // budget, after failing its integrity check. + let store = BlobStore::new(); + let now = t0(); + let claimed = hex_digest(b"what the caller promised"); + let id = store.begin(5, &claimed, now).unwrap(); + + assert_eq!( + store.put_chunk(&id, 0, b"other", now), + Err(BlobError::DigestMismatch) + ); + assert_eq!(store.live_count(), 0, "mismatched blob was retained"); + assert_eq!(store.get_chunk(&id, 0, 5, now), Err(BlobError::UnknownBlob)); +} + +#[test] +fn an_incomplete_blob_cannot_be_read_or_taken() { + let store = BlobStore::new(); + let now = t0(); + let payload = vec![3u8; 100]; + let id = store.begin(100, &hex_digest(&payload), now).unwrap(); + store.put_chunk(&id, 0, &payload[..50], now).unwrap(); + + assert_eq!( + store.get_chunk(&id, 0, 10, now), + Err(BlobError::IncompleteBlob) + ); + assert_eq!( + store.take_complete(&id, now), + Err(BlobError::IncompleteBlob) + ); +} + +#[test] +fn a_malformed_digest_is_refused_before_anything_is_reserved() { + let store = BlobStore::new(); + let now = t0(); + for bad in [ + "", + "abc", + &"A".repeat(64), // uppercase + &"g".repeat(64), // not hex + &"a".repeat(63), // too short + &"a".repeat(65), // too long + ] { + assert_eq!( + store.begin(10, bad, now), + Err(BlobError::MalformedDigest), + "accepted {bad:?}" + ); + } + assert_eq!(store.live_count(), 0); +} + +#[test] +fn a_blob_over_the_per_blob_limit_is_refused() { + let store = BlobStore::new(); + let now = t0(); + assert_eq!( + store.begin(MAX_BLOB_BYTES as u64 + 1, &hex_digest(b"anything"), now), + Err(BlobError::BlobTooLarge) + ); + // A declared size beyond usize on a 32-bit host lands in the same refusal. + assert_eq!( + store.begin(u64::MAX, &hex_digest(b"anything"), now), + Err(BlobError::BlobTooLarge) + ); +} + +#[test] +fn the_staging_budget_is_reserved_at_begin_not_on_arrival() { + // Reserving up front is what lets an admitted transfer always finish. If the + // budget only counted bytes received, two callers could each be admitted for + // 96 MiB and then fight over the last 32. + let store = BlobStore::new(); + let now = t0(); + let big = MAX_TOTAL_STAGED_BYTES / 2; + let digest = hex_digest(b"never sent"); + + store.begin(big as u64, &digest, now).unwrap(); + store.begin(big as u64, &digest, now).unwrap(); + // Nothing has actually been uploaded, yet the area is full. + assert_eq!(store.begin(1, &digest, now), Err(BlobError::StagingFull)); +} + +#[test] +fn too_many_live_blobs_is_refused() { + let store = BlobStore::new(); + let now = t0(); + let digest = hex_digest(b"x"); + for _ in 0..MAX_LIVE_BLOBS { + store.begin(1, &digest, now).unwrap(); + } + assert_eq!(store.live_count(), MAX_LIVE_BLOBS); + assert_eq!(store.begin(1, &digest, now), Err(BlobError::TooManyBlobs)); +} + +#[test] +fn an_untouched_blob_expires_and_frees_its_budget() { + // The bound that matters most: a caller that dies mid-upload must not leak + // its partial blob for the life of a module the host never unloads. + let store = BlobStore::new(); + let now = t0(); + let payload = vec![9u8; 1_000]; + let id = store + .begin(payload.len() as u64, &hex_digest(&payload), now) + .unwrap(); + store.put_chunk(&id, 0, &payload[..500], now).unwrap(); + assert_eq!(store.live_count(), 1); + + // Still live right on the boundary. + let at_ttl = now + IDLE_TTL; + assert!(store.get_chunk(&id, 0, 1, at_ttl).is_err()); // incomplete, but alive + assert_eq!(store.live_count(), 1); + + // Past it, the next operation sweeps it away. + let past_ttl = now + IDLE_TTL + Duration::from_secs(1); + assert_eq!( + store.put_chunk(&id, 500, &payload[500..], past_ttl), + Err(BlobError::UnknownBlob) + ); + assert_eq!(store.live_count(), 0); +} + +#[test] +fn activity_keeps_a_slow_transfer_alive() { + // The flip side: a transfer that is slow but live must never be reaped. + let store = BlobStore::new(); + let mut now = t0(); + let payload = vec![4u8; 400]; + let id = store.begin(400, &hex_digest(&payload), now).unwrap(); + + for start in (0..400).step_by(100) { + // Each chunk arrives just inside the window, well past the total elapsed + // TTL — three of these sum to more than IDLE_TTL. + now += IDLE_TTL.saturating_sub(Duration::from_secs(1)); + store + .put_chunk(&id, start as u64, &payload[start..start + 100], now) + .expect("a touched blob must not expire"); + } + assert_eq!(store.get_chunk(&id, 0, 400, now).unwrap(), payload); +} + +#[test] +fn releasing_frees_the_budget_and_is_reported_once() { + let store = BlobStore::new(); + let now = t0(); + let id = upload(&store, b"payload", now); + assert!(store.release(&id, now).is_ok()); + assert_eq!(store.live_count(), 0); + // A second release tells the caller the blob is gone rather than pretending. + assert_eq!(store.release(&id, now), Err(BlobError::UnknownBlob)); +} + +#[test] +fn taking_a_blob_removes_it() { + let store = BlobStore::new(); + let now = t0(); + let id = upload(&store, b"consume me", now); + assert_eq!(store.take_complete(&id, now).unwrap(), b"consume me"); + assert_eq!(store.live_count(), 0); + assert_eq!(store.take_complete(&id, now), Err(BlobError::UnknownBlob)); +} + +#[test] +fn insert_complete_produces_a_readable_handle() { + let store = BlobStore::new(); + let now = t0(); + let bytes = b"generated output".to_vec(); + let handle = store.insert_complete(bytes.clone(), now).unwrap(); + + assert_eq!(handle.total_bytes, bytes.len() as u64); + assert_eq!(handle.sha256, hex_digest(&bytes)); + assert_eq!( + store.get_chunk(&handle.blob_id, 0, 1_000, now).unwrap(), + bytes + ); +} + +#[test] +fn insert_complete_respects_every_budget() { + let store = BlobStore::new(); + let now = t0(); + assert_eq!( + store.insert_complete(vec![0u8; MAX_BLOB_BYTES + 1], now), + Err(BlobError::BlobTooLarge) + ); + + // Filling the staging area takes more than one blob: the per-blob cap is + // half the total, so the area can only ever be filled by at least two. + let full = BlobStore::new(); + let digest = hex_digest(b"reserved"); + let per_blob = MAX_BLOB_BYTES; + let mut reserved = 0usize; + while reserved + per_blob <= MAX_TOTAL_STAGED_BYTES { + full.begin(per_blob as u64, &digest, now).unwrap(); + reserved += per_blob; + } + assert_eq!(reserved, MAX_TOTAL_STAGED_BYTES, "area not fully reserved"); + assert_eq!( + full.insert_complete(vec![0u8; 16], now), + Err(BlobError::StagingFull) + ); + + let crowded = BlobStore::new(); + for _ in 0..MAX_LIVE_BLOBS { + crowded.begin(1, &digest, now).unwrap(); + } + assert_eq!( + crowded.insert_complete(vec![0u8; 1], now), + Err(BlobError::TooManyBlobs) + ); +} + +#[test] +fn a_zero_length_blob_completes_at_begin() { + // There is no chunk to complete it on, so the digest has to be checked when + // it is declared or the blob would never become readable. + let store = BlobStore::new(); + let now = t0(); + let id = store.begin(0, &hex_digest(b""), now).unwrap(); + assert_eq!(store.get_chunk(&id, 0, 10, now).unwrap(), Vec::::new()); + assert_eq!(store.take_complete(&id, now).unwrap(), Vec::::new()); +} + +#[test] +fn a_zero_length_blob_with_a_wrong_digest_is_refused_at_begin() { + let store = BlobStore::new(); + let now = t0(); + assert_eq!( + store.begin(0, &hex_digest(b"not empty"), now), + Err(BlobError::DigestMismatch) + ); + assert_eq!(store.live_count(), 0); +} + +#[test] +fn blob_ids_are_unique_across_reuse() { + // Ids must not be recycled after a release: a caller holding a stale id + // would otherwise read somebody else's blob. + let store = BlobStore::new(); + let now = t0(); + let first = upload(&store, b"one", now); + store.release(&first, now).unwrap(); + let second = upload(&store, b"two", now); + assert_ne!(first, second); +} + +#[test] +fn unknown_ids_are_refused_by_every_operation() { + let store = BlobStore::new(); + let now = t0(); + assert_eq!( + store.put_chunk("nope", 0, b"x", now), + Err(BlobError::UnknownBlob) + ); + assert_eq!( + store.get_chunk("nope", 0, 1, now), + Err(BlobError::UnknownBlob) + ); + assert_eq!( + store.take_complete("nope", now), + Err(BlobError::UnknownBlob) + ); + assert_eq!(store.release("nope", now), Err(BlobError::UnknownBlob)); +} + +#[test] +fn an_oversize_read_length_is_refused() { + let store = BlobStore::new(); + let now = t0(); + let id = upload(&store, b"small", now); + assert_eq!( + store.get_chunk(&id, 0, MAX_CHUNK_BYTES as u64 + 1, now), + Err(BlobError::ChunkTooLarge) + ); + assert_eq!( + store.get_chunk(&id, 0, u64::MAX, now), + Err(BlobError::ChunkTooLarge) + ); +} + +#[test] +fn digest_helpers_agree_with_a_known_vector() { + // The SHA-256 of the empty string, so a wrong hasher or a broken hex + // encoding fails here rather than in an integration test. + assert_eq!( + hex_digest(b""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert!(verify(b"", &hex_digest(b""))); + assert!(!verify(b"", &hex_digest(b"x"))); + // A length mismatch must fail before the comparison loop. + assert!(!verify(b"", "abcd")); +} + +#[test] +fn digest_shape_is_validated_strictly() { + assert!(is_lowercase_sha256(&hex_digest(b"anything"))); + assert!(!is_lowercase_sha256( + &hex_digest(b"anything").to_uppercase() + )); + assert!(!is_lowercase_sha256("zz")); +} diff --git a/crates/tinydocs-module/src/lib.rs b/crates/tinydocs-module/src/lib.rs index 86d9943..926be01 100644 --- a/crates/tinydocs-module/src/lib.rs +++ b/crates/tinydocs-module/src/lib.rs @@ -4,6 +4,8 @@ //! the independently published `tinydocs` crate. Its `cdylib` output is the //! target-specific binary distributed in GitHub releases. +pub mod blobs; mod service; -pub use service::{BUS_NAME, OBJECT_PATH}; +pub use blobs::{BlobError, BlobRef, BlobStore, hex_digest}; +pub use service::{BUS_NAME, OBJECT_PATH, WirePresentationSpec, WireSlideImage, WireSlideSpec}; diff --git a/crates/tinydocs-module/src/service/mod.rs b/crates/tinydocs-module/src/service/mod.rs index 5a36969..1bd6566 100644 --- a/crates/tinydocs-module/src/service/mod.rs +++ b/crates/tinydocs-module/src/service/mod.rs @@ -1,48 +1,234 @@ -//! `TinyBus` service boundary for document synthesis. +//! `TinyBus` service boundary for the document surface. //! -//! The module owns no persistent state and exposes one object: `GenerateDocx` -//! accepts the same typed [`DocumentSpec`] as the Rust API and returns the -//! complete DOCX bytes. +//! One object, `/ai/tinyhumans/tinydocs/Documents`, exporting the three format +//! operations plus the four chunked-transfer operations they depend on: //! -//! The `TinyBus` wire format has a 16 MiB frame limit. [`DocumentSpec`]'s -//! aggregate text limit keeps normal output comfortably below that boundary; -//! a larger future document format should use a path or file-descriptor based -//! transfer instead of increasing the bus frame cap. +//! ```text +//! BeginBlob(total_bytes, sha256) -> blob_id +//! PutChunk(blob_id, offset, base64) -> bytes received so far +//! GetChunk(blob_id, offset, len) -> base64 +//! ReleaseBlob(blob_id) -> () +//! GenerateDocx(DocumentSpec) -> BlobRef +//! GeneratePptx(WirePresentationSpec) -> BlobRef +//! ExtractText(blob_id) -> BlobRef +//! ``` +//! +//! # Why everything returns a `BlobRef` +//! +//! See [`crate::blobs`]. A `TinyBus` frame is a 16 MiB JSON document and +//! `Vec` serialises as an array of integers, so the real inline ceiling is a +//! few megabytes — below a deck's legal image payload and below any `.pdf` worth +//! extracting. Rather than have some methods return bytes inline and others not, +//! every unbounded result is staged and read back in chunks. The caller's code +//! path is then the same regardless of size. +//! +//! # This replaces the `Docx` interface rather than extending it +//! +//! The previous interface, `ai.tinyhumans.tinydocs.Docx`, returned +//! `GenerateDocx(DocumentSpec) -> Vec` inline. `TinyBus`'s module guidance is +//! explicit that an existing interface must not change in place — a breaking +//! contract gets a new interface name — and returning a `BlobRef` where callers +//! expect bytes is exactly that. Hence a new name. +//! +//! The old interface is retired rather than served alongside, because +//! `module_export!` attaches its `methods` list to the *first* entry in +//! `provides` and leaves any others with an empty method list. A second +//! fully-declared interface is therefore not expressible today, and a manifest +//! that under-declares its members would break the invariant that manifest +//! methods and dispatch members stay identical. Serving both needs a `TinyBus` +//! change first; retiring one at a pre-1.0 minor bump does not. +//! +//! # Runtime +//! +//! Synthesis and extraction are CPU-bound and run on the module runtime's +//! blocking pool. The blob operations are memory copies under a short lock and +//! run inline. The module holds no document state between calls — only staged +//! blobs, every one of them bounded and expiring. + +mod wire; + +use std::sync::Arc; +use std::time::Instant; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; use tinybus::{Connection, Error as BusError, Result as BusResult}; -use tinydocs::Error; -use tinydocs::docx::{self, DocumentSpec}; +use tinydocs::spec::{DocumentSpec, PresentationSpec, SlideImage, SlideSpec}; +use tinydocs::{Error, pdf, pptx}; + +use crate::blobs::{BlobError, BlobRef, BlobStore}; + +pub use wire::{WirePresentationSpec, WireSlideImage, WireSlideSpec}; /// Well-known name and interface exported by the `TinyDocs` module. -pub const BUS_NAME: &str = "ai.tinyhumans.tinydocs.Docx"; +pub const BUS_NAME: &str = "ai.tinyhumans.tinydocs.Documents"; /// Object path exported by the `TinyDocs` module. -pub const OBJECT_PATH: &str = "/ai/tinyhumans/tinydocs/Docx"; +pub const OBJECT_PATH: &str = "/ai/tinyhumans/tinydocs/Documents"; const INVALID_INPUT_ERROR: &str = "ai.tinyhumans.tinydocs.Error.InvalidInput"; const GENERATION_FAILED_ERROR: &str = "ai.tinyhumans.tinydocs.Error.GenerationFailed"; +const EXTRACTION_FAILED_ERROR: &str = "ai.tinyhumans.tinydocs.Error.ExtractionFailed"; const MODULE_FAILED_ERROR: &str = "ai.tinyhumans.tinydocs.Error.ModuleFailed"; +const TRANSFER_FAILED_ERROR: &str = "ai.tinyhumans.tinydocs.Error.TransferFailed"; +const TRANSFER_REFUSED_ERROR: &str = "ai.tinyhumans.tinydocs.Error.TransferRefused"; +const UNKNOWN_BLOB_ERROR: &str = "ai.tinyhumans.tinydocs.Error.UnknownBlob"; -struct TinyDocs; - -#[tinybus::interface(name = "ai.tinyhumans.tinydocs.Docx")] -impl TinyDocs { - /// Generate a complete DOCX document from a validated specification. - async fn generate_docx(&self, spec: DocumentSpec) -> BusResult> { - tokio::task::spawn_blocking(move || docx::generate(&spec)) - .await - .map_err(|_| BusError::MethodFailed { - name: GENERATION_FAILED_ERROR.to_string(), - message: "document generation worker failed".to_string(), - })? - .map_err(|error| map_error(&error)) +/// The served object. Owns the staging area; holds no document state. +struct Documents { + blobs: Arc, +} + +// The interface macro rejects a non-async method outright, so the four transfer +// methods below are async because the dispatch contract says so, not because they +// await anything. `unused_async` can therefore never be actionable in this block. +#[allow( + clippy::unused_async, + reason = "tinybus::interface requires every method to be `async fn`" +)] +#[tinybus::interface(name = "ai.tinyhumans.tinydocs.Documents")] +impl Documents { + /// Reserve space for a blob of `total_bytes` that will hash to `sha256`. + async fn begin_blob(&self, total_bytes: u64, sha256: String) -> BusResult { + self.blobs + .begin(total_bytes, &sha256, Instant::now()) + .map_err(|error| map_blob_error(&error)) + } + + /// Append a base64 chunk at `offset`, returning bytes received so far. + async fn put_chunk(&self, blob_id: String, offset: u64, data: String) -> BusResult { + let decoded = decode_base64(&data)?; + self.blobs + .put_chunk(&blob_id, offset, &decoded, Instant::now()) + .map_err(|error| map_blob_error(&error)) + } + + /// Read up to `len` bytes of a complete blob at `offset`, base64-encoded. + async fn get_chunk(&self, blob_id: String, offset: u64, len: u64) -> BusResult { + let bytes = self + .blobs + .get_chunk(&blob_id, offset, len, Instant::now()) + .map_err(|error| map_blob_error(&error))?; + Ok(BASE64.encode(bytes)) + } + + /// Drop a blob and free its budget. + async fn release_blob(&self, blob_id: String) -> BusResult<()> { + self.blobs + .release(&blob_id, Instant::now()) + .map_err(|error| map_blob_error(&error)) + } + + /// Generate a `.docx` and stage it for reading. + async fn generate_docx(&self, spec: DocumentSpec) -> BusResult { + // Validated on this thread, before a blocking slot is taken: rejecting a + // malformed spec should not have to queue behind real work. + spec.validate().map_err(|error| map_error(&error))?; + let bytes = blocking(move || tinydocs::docx::generate(&spec)).await?; + self.stage(bytes) + } + + /// Generate a `.pptx` from a spec whose images name staged blobs. + async fn generate_pptx(&self, spec: WirePresentationSpec) -> BusResult { + let resolved = self.resolve_presentation(spec)?; + resolved.validate().map_err(|error| map_error(&error))?; + let bytes = blocking(move || pptx::generate(&resolved)).await?; + self.stage(bytes) + } + + /// Extract the text layer of a staged `.pdf` and stage the result. + async fn extract_text(&self, blob_id: String) -> BusResult { + // Taken rather than copied: the document is often the largest thing in + // the staging area, and holding it through extraction as well would + // double its cost for no reason. + let bytes = self + .blobs + .take_complete(&blob_id, Instant::now()) + .map_err(|error| map_blob_error(&error))?; + let text = blocking(move || pdf::extract_text(&bytes)).await?; + self.stage(text.into_bytes()) } } +impl Documents { + /// Stage a produced payload and return its handle. + fn stage(&self, bytes: Vec) -> BusResult { + self.blobs + .insert_complete(bytes, Instant::now()) + .map_err(|error| map_blob_error(&error)) + } + + /// Turn a wire deck into a real [`PresentationSpec`] by consuming the blobs + /// its images name. + /// + /// Images are taken from the staging area, so a deck's bytes stop being + /// charged twice the moment they are resolved. A blob that is missing or + /// incomplete fails the whole call rather than silently dropping a slide's + /// image — the caller staged it, so its absence is a transfer bug worth + /// reporting, not a degraded deck. + fn resolve_presentation(&self, spec: WirePresentationSpec) -> BusResult { + let now = Instant::now(); + let mut slides = Vec::with_capacity(spec.slides.len()); + for slide in spec.slides { + let mut images = Vec::with_capacity(slide.images.len()); + for image in slide.images { + let bytes = self + .blobs + .take_complete(&image.blob_id, now) + .map_err(|error| map_blob_error(&error))?; + images.push( + SlideImage::from_bytes(bytes, image.caption) + .map_err(|error| map_error(&error))?, + ); + } + slides.push(SlideSpec { + title: slide.title, + body: slide.body, + bullets: slide.bullets, + speaker_notes: slide.speaker_notes, + images, + }); + } + Ok(PresentationSpec { + title: spec.title, + author: spec.author, + theme: spec.theme, + slides, + }) + } +} + +/// Run a CPU-bound library call on the blocking pool and map its failure. +async fn blocking(work: F) -> BusResult +where + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, +{ + tokio::task::spawn_blocking(work) + .await + .map_err(|_| BusError::MethodFailed { + name: MODULE_FAILED_ERROR.to_string(), + message: "document worker failed".to_string(), + })? + .map_err(|error| map_error(&error)) +} + +/// Decode a base64 chunk, refusing malformed input by name. +fn decode_base64(data: &str) -> BusResult> { + BASE64.decode(data).map_err(|_| BusError::MethodFailed { + name: INVALID_INPUT_ERROR.to_string(), + // The payload itself is never echoed: it is caller data, and an error + // message is the wrong place for it. + message: "chunk data is not valid base64".to_string(), + }) +} + +/// Map a library error onto its wire name. fn map_error(error: &Error) -> BusError { let name = match error { Error::InvalidInput { .. } => INVALID_INPUT_ERROR, Error::GenerationFailed { .. } => GENERATION_FAILED_ERROR, + Error::ExtractionFailed { .. } => EXTRACTION_FAILED_ERROR, _ => MODULE_FAILED_ERROR, }; BusError::MethodFailed { @@ -51,9 +237,39 @@ fn map_error(error: &Error) -> BusError { } } +/// Map a staging failure onto its wire name. +/// +/// Three names rather than one, because the caller's correct response differs. +/// `UnknownBlob` means the transfer is gone and has to restart; `TransferRefused` +/// means a budget is full and retrying later may work; `TransferFailed` means the +/// caller sent something wrong and should re-send. +fn map_blob_error(error: &BlobError) -> BusError { + let name = match *error { + BlobError::UnknownBlob => UNKNOWN_BLOB_ERROR, + BlobError::StagingFull | BlobError::TooManyBlobs => TRANSFER_REFUSED_ERROR, + BlobError::MalformedDigest + | BlobError::BlobTooLarge + | BlobError::ChunkTooLarge + | BlobError::OutOfOrderChunk { .. } + | BlobError::OverlongBlob + | BlobError::DigestMismatch + | BlobError::IncompleteBlob + | BlobError::ReadPastEnd => TRANSFER_FAILED_ERROR, + }; + BusError::MethodFailed { + name: name.to_string(), + message: error.to_string(), + } +} + async fn setup(connection: Connection) -> BusResult<()> { connection - .serve_at(OBJECT_PATH.try_into()?, TinyDocs) + .serve_at( + OBJECT_PATH.try_into()?, + Documents { + blobs: Arc::new(BlobStore::new()), + }, + ) .await?; connection.request_name(BUS_NAME).await?; Ok(()) @@ -71,8 +287,16 @@ mod exports { tinybus_module::module_export! { setup = super::setup, worker_threads = 2, - provides = ["ai.tinyhumans.tinydocs.Docx"], - methods = ["GenerateDocx"], + provides = ["ai.tinyhumans.tinydocs.Documents"], + methods = [ + "BeginBlob", + "PutChunk", + "GetChunk", + "ReleaseBlob", + "GenerateDocx", + "GeneratePptx", + "ExtractText", + ], signals = [], requires = [], optional = [], diff --git a/crates/tinydocs-module/src/service/test.rs b/crates/tinydocs-module/src/service/test.rs index 327ee4c..dd69a6c 100644 --- a/crates/tinydocs-module/src/service/test.rs +++ b/crates/tinydocs-module/src/service/test.rs @@ -1,30 +1,438 @@ //! Unit tests for the `TinyBus` service declaration. +//! +//! The manifest and the generated dispatch table are two lists that have to stay +//! identical, and nothing but a test connects them: the macro takes the method +//! names as string literals, so a method added to the `impl` without a matching +//! literal is admitted by the loader and then fails to dispatch. That is the +//! invariant this file exists for. +//! +//! Bytes moving over a real broker is covered by `tests/module_e2e.rs`, which +//! loads the built artifact through the actual dynamic loader. -#![allow(clippy::unwrap_used)] +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use tinybus::Interface; use super::*; +use crate::blobs::hex_digest; + +/// The methods the manifest declares, in declaration order. +const DECLARED_METHODS: &[&str] = &[ + "BeginBlob", + "PutChunk", + "GetChunk", + "ReleaseBlob", + "GenerateDocx", + "GeneratePptx", + "ExtractText", +]; + +fn service() -> Documents { + Documents { + blobs: Arc::new(BlobStore::new()), + } +} #[test] -fn service_identity_is_valid_and_dispatch_matches_the_manifest() { +fn service_identity_is_valid() { assert!(tinybus::BusName::new(BUS_NAME).is_ok()); assert!(tinybus::ObjectPath::new(OBJECT_PATH).is_ok()); + // TinyBus derives a module's object path from its bus name by replacing dots + // with slashes, and admission compares the two. A mismatch here would be + // rejected by the loader rather than by anything in this crate. + assert_eq!(OBJECT_PATH, format!("/{}", BUS_NAME.replace('.', "/"))); +} + +#[test] +fn dispatch_members_match_the_manifest_exactly() { + let members: Vec = service() + .members() + .iter() + .map(|member| member.as_str().to_string()) + .collect(); + let declared: Vec = DECLARED_METHODS.iter().map(|m| (*m).to_string()).collect(); + assert_eq!( + members, declared, + "the interface impl and the module_export! methods list have drifted" + ); +} + +#[test] +fn every_declared_method_name_is_a_valid_member_name() { + for method in DECLARED_METHODS { + assert!( + tinybus::MemberName::new(*method).is_ok(), + "{method} is not a valid member name" + ); + } +} + +#[test] +fn library_errors_keep_distinct_wire_names() { + assert_eq!( + map_error(&Error::invalid_input("title", "must not be empty")).wire_name(), + INVALID_INPUT_ERROR + ); + assert_eq!( + map_error(&Error::generation_failed("writer stopped")).wire_name(), + GENERATION_FAILED_ERROR + ); + assert_eq!( + map_error(&Error::extraction_failed("damaged xref")).wire_name(), + EXTRACTION_FAILED_ERROR + ); +} - let members = TinyDocs.members(); +#[test] +fn transfer_errors_are_grouped_by_what_the_caller_should_do() { + // Gone: restart the transfer. assert_eq!( - members, - &[tinybus::MemberName::new("GenerateDocx").unwrap()] + map_blob_error(&BlobError::UnknownBlob).wire_name(), + UNKNOWN_BLOB_ERROR ); + // Full: the same request may succeed later. + for refused in [BlobError::StagingFull, BlobError::TooManyBlobs] { + assert_eq!( + map_blob_error(&refused).wire_name(), + TRANSFER_REFUSED_ERROR, + "{refused:?} should be retryable" + ); + } + // Caller error: re-send, do not retry verbatim. + for failed in [ + BlobError::MalformedDigest, + BlobError::BlobTooLarge, + BlobError::ChunkTooLarge, + BlobError::OutOfOrderChunk { + expected: 1, + actual: 2, + }, + BlobError::OverlongBlob, + BlobError::DigestMismatch, + BlobError::IncompleteBlob, + BlobError::ReadPastEnd, + ] { + assert_eq!( + map_blob_error(&failed).wire_name(), + TRANSFER_FAILED_ERROR, + "{failed:?} should not be reported as retryable" + ); + } } #[test] -fn domain_errors_keep_distinct_wire_names() { - let invalid_error = Error::invalid_input("title", "must not be empty"); - let invalid = map_error(&invalid_error); - assert_eq!(invalid.wire_name(), INVALID_INPUT_ERROR); - - let generation_error = Error::generation_failed("writer stopped"); - let failed = map_error(&generation_error); - assert_eq!(failed.wire_name(), GENERATION_FAILED_ERROR); +fn malformed_base64_is_an_invalid_input_and_does_not_echo_the_payload() { + let err = decode_base64("this is not base64!!").expect_err("should reject"); + assert_eq!(err.wire_name(), INVALID_INPUT_ERROR); + assert!( + !format!("{err}").contains("not base64!!"), + "the rejected payload leaked into the error message: {err}" + ); +} + +#[test] +fn valid_base64_decodes() { + assert_eq!( + decode_base64(&BASE64.encode(b"round trip")).unwrap(), + b"round trip" + ); + assert_eq!(decode_base64("").unwrap(), Vec::::new()); +} + +#[tokio::test] +async fn generate_docx_stages_a_readable_document() { + use tinydocs::spec::DocumentSection; + + let service = service(); + let spec = DocumentSpec { + title: "Charter".to_string(), + author: Some("Alice".to_string()), + sections: vec![DocumentSection { + heading: Some("Goals".to_string()), + paragraphs: vec!["Ship it.".to_string()], + bullets: vec![], + }], + }; + + let handle = service.generate_docx(spec).await.expect("should generate"); + assert!(handle.total_bytes > 0); + + let bytes = service + .blobs + .get_chunk(&handle.blob_id, 0, handle.total_bytes, Instant::now()) + .expect("staged output should be readable"); + assert_eq!(&bytes[..2], b"PK", "a .docx is a zip container"); + assert_eq!(hex_digest(&bytes), handle.sha256); +} + +#[tokio::test] +async fn generate_docx_rejects_an_invalid_spec_without_staging_anything() { + let service = service(); + let spec = DocumentSpec { + title: String::new(), + author: None, + sections: vec![], + }; + let err = service + .generate_docx(spec) + .await + .expect_err("a blank title should be rejected"); + assert_eq!(err.wire_name(), INVALID_INPUT_ERROR); + assert_eq!( + service.blobs.live_count(), + 0, + "a rejected call must not leave a blob behind" + ); +} + +#[tokio::test] +async fn generate_pptx_consumes_the_image_blobs_it_is_given() { + let service = service(); + let now = Instant::now(); + let png = tiny_png(); + + // Stage an image the way a caller would, then reference it by id. + let blob_id = service + .blobs + .begin(png.len() as u64, &hex_digest(&png), now) + .unwrap(); + service.blobs.put_chunk(&blob_id, 0, &png, now).unwrap(); + assert_eq!(service.blobs.live_count(), 1); + + let handle = service + .generate_pptx(WirePresentationSpec { + title: "Quarterly".to_string(), + author: None, + theme: None, + slides: vec![WireSlideSpec { + title: "With a chart".to_string(), + body: None, + bullets: vec![], + speaker_notes: None, + images: vec![WireSlideImage { + blob_id: blob_id.clone(), + caption: Some("A chart".to_string()), + }], + }], + }) + .await + .expect("should generate"); + + let bytes = service + .blobs + .get_chunk(&handle.blob_id, 0, handle.total_bytes, Instant::now()) + .expect("staged deck should be readable"); + assert_eq!(&bytes[..2], b"PK", "a .pptx is a zip container"); + + // The image blob was taken, not copied: only the output remains staged. + assert_eq!( + service.blobs.live_count(), + 1, + "the consumed image blob should have been released" + ); + assert!( + service + .blobs + .get_chunk(&blob_id, 0, 1, Instant::now()) + .is_err() + ); +} + +#[tokio::test] +async fn generate_pptx_reports_a_missing_image_blob_rather_than_dropping_the_image() { + // The caller staged it, so its absence is a transfer bug worth reporting — + // not a deck quietly missing a slide's illustration. + let service = service(); + let err = service + .generate_pptx(WirePresentationSpec { + title: "Quarterly".to_string(), + author: None, + theme: None, + slides: vec![WireSlideSpec { + title: "With a chart".to_string(), + body: None, + bullets: vec![], + speaker_notes: None, + images: vec![WireSlideImage { + blob_id: "blob-does-not-exist".to_string(), + caption: None, + }], + }], + }) + .await + .expect_err("a missing image blob should fail the call"); + assert_eq!(err.wire_name(), UNKNOWN_BLOB_ERROR); +} + +#[tokio::test] +async fn generate_pptx_rejects_image_bytes_that_are_not_an_embeddable_image() { + let service = service(); + let now = Instant::now(); + let junk = b"definitely not a png".to_vec(); + let blob_id = service + .blobs + .begin(junk.len() as u64, &hex_digest(&junk), now) + .unwrap(); + service.blobs.put_chunk(&blob_id, 0, &junk, now).unwrap(); + + let err = service + .generate_pptx(WirePresentationSpec { + title: "Quarterly".to_string(), + author: None, + theme: None, + slides: vec![WireSlideSpec { + title: "Broken".to_string(), + body: None, + bullets: vec![], + speaker_notes: None, + images: vec![WireSlideImage { + blob_id, + caption: None, + }], + }], + }) + .await + .expect_err("unrecognisable image bytes should be rejected"); + assert_eq!(err.wire_name(), INVALID_INPUT_ERROR); +} + +#[tokio::test] +async fn extract_text_consumes_the_document_and_stages_the_text() { + let service = service(); + let now = Instant::now(); + let doc = tiny_pdf("Hello from the bus"); + let blob_id = service + .blobs + .begin(doc.len() as u64, &hex_digest(&doc), now) + .unwrap(); + service.blobs.put_chunk(&blob_id, 0, &doc, now).unwrap(); + + let handle = service + .extract_text(blob_id.clone()) + .await + .expect("should extract"); + let bytes = service + .blobs + .get_chunk(&handle.blob_id, 0, handle.total_bytes, Instant::now()) + .unwrap(); + let text = String::from_utf8(bytes).expect("extracted text is utf-8"); + assert!( + text.contains("Hello from the bus"), + "extracted text missing content: {text:?}" + ); + + // The input was taken, so only the extracted text stays staged. + assert_eq!(service.blobs.live_count(), 1); +} + +#[tokio::test] +async fn extract_text_refuses_an_unknown_or_incomplete_blob() { + let service = service(); + let now = Instant::now(); + + let err = service + .extract_text("blob-nope".to_string()) + .await + .expect_err("unknown blob"); + assert_eq!(err.wire_name(), UNKNOWN_BLOB_ERROR); + + let doc = tiny_pdf("partial"); + let blob_id = service + .blobs + .begin(doc.len() as u64, &hex_digest(&doc), now) + .unwrap(); + service + .blobs + .put_chunk(&blob_id, 0, &doc[..doc.len() / 2], now) + .unwrap(); + let err = service + .extract_text(blob_id) + .await + .expect_err("incomplete blob"); + assert_eq!(err.wire_name(), TRANSFER_FAILED_ERROR); +} + +#[tokio::test] +async fn the_blob_methods_round_trip_a_payload_over_the_declared_surface() { + // Exercises the four transfer methods through the same signatures the bus + // calls, including the base64 hop the store itself never sees. + let service = service(); + let payload: Vec = (0..5_000u32).map(|i| (i % 253) as u8).collect(); + + let blob_id = service + .begin_blob(payload.len() as u64, hex_digest(&payload)) + .await + .unwrap(); + let received = service + .put_chunk(blob_id.clone(), 0, BASE64.encode(&payload)) + .await + .unwrap(); + assert_eq!(received, payload.len() as u64); + + let encoded = service + .get_chunk(blob_id.clone(), 0, payload.len() as u64) + .await + .unwrap(); + assert_eq!(BASE64.decode(encoded).unwrap(), payload); + + service.release_blob(blob_id.clone()).await.unwrap(); + assert_eq!(service.blobs.live_count(), 0); + assert!(service.release_blob(blob_id).await.is_err()); +} + +/// A 1×1 PNG, built from its header so the fixture needs no dependency. +fn tiny_png() -> Vec { + let mut out = vec![0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]; + out.extend_from_slice(&13u32.to_be_bytes()); + out.extend_from_slice(b"IHDR"); + out.extend_from_slice(&1u32.to_be_bytes()); + out.extend_from_slice(&1u32.to_be_bytes()); + out.extend_from_slice(&[0x08, 0x06, 0x00, 0x00, 0x00]); + out.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); + out.extend_from_slice(&0u32.to_be_bytes()); + out.extend_from_slice(b"IDAT"); + out.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); + out.extend_from_slice(&0u32.to_be_bytes()); + out.extend_from_slice(b"IEND"); + out.extend_from_slice(&[0xAE, 0x42, 0x60, 0x82]); + out +} + +/// A valid single-page PDF whose text layer holds `text`. +fn tiny_pdf(text: &str) -> Vec { + let content = format!("BT /F1 24 Tf 72 700 Td ({text}) Tj ET\n"); + let objects = [ + "<< /Type /Catalog /Pages 2 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \ + /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>" + .to_string(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(), + format!( + "<< /Length {} >>\nstream\n{content}endstream", + content.len() + ), + ]; + + let mut out = Vec::new(); + out.extend_from_slice(b"%PDF-1.4\n"); + let mut offsets = Vec::with_capacity(objects.len()); + for (i, body) in objects.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", i + 1).as_bytes()); + } + let xref_offset = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n", objects.len() + 1).as_bytes()); + out.extend_from_slice(b"0000000000 65535 f \n"); + for offset in &offsets { + out.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice( + format!( + "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref_offset}\n%%EOF\n", + objects.len() + 1 + ) + .as_bytes(), + ); + out } diff --git a/crates/tinydocs-module/src/service/wire.rs b/crates/tinydocs-module/src/service/wire.rs new file mode 100644 index 0000000..184da1d --- /dev/null +++ b/crates/tinydocs-module/src/service/wire.rs @@ -0,0 +1,65 @@ +//! Wire shapes that differ from the library spec because bytes cannot travel +//! inline. +//! +//! [`crate::blobs`] explains why: a `TinyBus` frame is a 16 MiB JSON document, +//! and a deck may legally carry 40 MiB of images. So on the bus an image is a +//! staged blob id, and the module resolves it into the real +//! [`tinydocs::spec::SlideImage`] — bytes, format and dimensions — after the +//! upload completes. +//! +//! Only the presentation spec needs this treatment. A document spec is text, and +//! its aggregate cap keeps it inside a frame, so `GenerateDocx` takes +//! [`tinydocs::spec::DocumentSpec`] unchanged. + +use serde::{Deserialize, Serialize}; + +/// A slide image, as it appears on the bus: a reference to a staged blob. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WireSlideImage { + /// Id of a completed blob holding the PNG or JPEG bytes. + pub blob_id: String, + /// Optional caption, rendered as a bullet beneath the image. + #[serde(default)] + pub caption: Option, +} + +/// One content slide, as it appears on the bus. +/// +/// Identical to [`tinydocs::spec::SlideSpec`] apart from `images`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WireSlideSpec { + /// Slide title. + #[serde(default)] + pub title: String, + /// Body text, rendered above the bullets. + #[serde(default)] + pub body: Option, + /// Bullets, rendered after the body text. + #[serde(default)] + pub bullets: Vec, + /// Speaker notes attached to the slide. + #[serde(default)] + pub speaker_notes: Option, + /// Images, each naming a staged blob. + #[serde(default)] + pub images: Vec, +} + +/// A deck, as it appears on the bus. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WirePresentationSpec { + /// Deck title, rendered on a leading title slide. + pub title: String, + /// Optional author byline. + #[serde(default)] + pub author: Option, + /// Optional theme hint. + #[serde(default)] + pub theme: Option, + /// Content slides, in display order. + #[serde(default)] + pub slides: Vec, +} diff --git a/crates/tinydocs-module/tests/module_e2e.rs b/crates/tinydocs-module/tests/module_e2e.rs index c52e85e..40b9546 100644 --- a/crates/tinydocs-module/tests/module_e2e.rs +++ b/crates/tinydocs-module/tests/module_e2e.rs @@ -1,6 +1,15 @@ //! End-to-end test for loading the built `TinyDocs` module into `TinyBus`. +//! +//! This is the only test that exercises the real thing: the built `cdylib`, the +//! ABI descriptor, manifest admission, the dynamic loader, and a broker routing +//! actual frames. Everything else in this crate tests Rust functions directly and +//! would keep passing if the artifact stopped loading at all. +//! +//! It therefore covers each of the three formats end to end, and moves an image +//! across more than one chunk — the chunked path is the reason this interface +//! exists, and a single-chunk transfer would not prove it works. -#![allow(clippy::unwrap_used, clippy::expect_used)] +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use std::time::Duration; @@ -8,35 +17,58 @@ use tinybus::Connection; use tinybus::broker::Broker; use tinybus::module::{ModuleHost, ModuleState}; use tinybus::transport::memory::MemoryBus; -use tinydocs::docx::{DocumentSection, DocumentSpec}; -use tinydocs_module::{BUS_NAME, OBJECT_PATH}; +use tinydocs::spec::{DocumentSection, DocumentSpec}; +use tinydocs_module::{BUS_NAME, BlobRef, OBJECT_PATH, hex_digest}; + +/// Every method the manifest must declare. +const EXPECTED_METHODS: &[&str] = &[ + "BeginBlob", + "PutChunk", + "GetChunk", + "ReleaseBlob", + "GenerateDocx", + "GeneratePptx", + "ExtractText", +]; + +/// Chunk size used by the test transfers. +/// +/// Deliberately small so a modest fixture still spans several chunks. The +/// module's own cap is megabytes; nothing here needs to approach it to prove the +/// offsets line up. +const TEST_CHUNK: usize = 512; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[ignore = "requires TINYDOCS_TEST_MODULE to point at the built cdylib"] -async fn built_cdylib_loads_and_generates_a_docx_over_the_bus() { +async fn the_built_module_serves_every_format_over_a_real_broker() { let artifact = std::env::var_os("TINYDOCS_TEST_MODULE").expect("TINYDOCS_TEST_MODULE must be set"); let bus = MemoryBus::new(); let broker = Broker::new(); let broker_task = broker.spawn(bus.clone()); let modules = ModuleHost::new(broker); + let loaded = modules.load_file(artifact).expect("module should load"); assert_eq!(loaded.name, "tinydocs-module"); assert_eq!(loaded.manifest.bus_name.as_str(), BUS_NAME); assert_eq!(loaded.manifest.object_path.as_str(), OBJECT_PATH); - assert!( - loaded - .manifest - .provides - .iter() - .flat_map(|interface| interface.methods.iter()) - .any(|method| method.as_str() == "GenerateDocx") + + let declared: Vec<&str> = loaded + .manifest + .provides + .iter() + .flat_map(|interface| interface.methods.iter()) + .map(tinybus::MemberName::as_str) + .collect(); + assert_eq!( + declared, EXPECTED_METHODS, + "manifest methods drifted from the interface" ); let client = Connection::connect(bus.connect().await.unwrap()) .await .unwrap(); - tokio::time::timeout(Duration::from_secs(2), async { + tokio::time::timeout(Duration::from_secs(5), async { loop { if client .list_names() @@ -54,7 +86,9 @@ async fn built_cdylib_loads_and_generates_a_docx_over_the_bus() { .expect("module should become ready"); let proxy = client.proxy(BUS_NAME, OBJECT_PATH, BUS_NAME).unwrap(); - let bytes: Vec = proxy + + // --- .docx: text in, staged bytes out --- + let handle: BlobRef = proxy .call( "GenerateDocx", (DocumentSpec { @@ -68,9 +102,182 @@ async fn built_cdylib_loads_and_generates_a_docx_over_the_bus() { },), ) .await - .expect("bus call should succeed"); + .expect("GenerateDocx should succeed"); + let docx = download(&proxy, &handle).await; + assert_eq!(&docx[..2], b"PK", "a .docx is a zip container"); + + // --- .pptx: an image staged across several chunks, then a deck --- + let png = png_1x1(); + assert!( + png.len() > TEST_CHUNK, + "the image fixture must span more than one chunk to be worth testing" + ); + let image_blob = upload(&proxy, &png).await; + let deck: BlobRef = proxy + .call( + "GeneratePptx", + (serde_json::json!({ + "title": "TinyBus E2E", + "slides": [{ + "title": "With an image", + "images": [{ "blob_id": image_blob, "caption": "A chart" }], + }], + }),), + ) + .await + .expect("GeneratePptx should succeed"); + let pptx = download(&proxy, &deck).await; + assert_eq!(&pptx[..2], b"PK", "a .pptx is a zip container"); + + // --- .pdf: a staged document in, extracted text out --- + let pdf = pdf_with_text("Hello from the module"); + let pdf_blob = upload(&proxy, &pdf).await; + let extracted: BlobRef = proxy + .call("ExtractText", (pdf_blob,)) + .await + .expect("ExtractText should succeed"); + let text = String::from_utf8(download(&proxy, &extracted).await).expect("text is utf-8"); + assert!( + text.contains("Hello from the module"), + "extracted text missing content: {text:?}" + ); + + // Releasing a consumed handle is reported, not silently accepted. + proxy + .call::<()>("ReleaseBlob", (handle.blob_id.clone(),)) + .await + .expect("releasing a staged output should succeed"); + proxy + .call::<()>("ReleaseBlob", (handle.blob_id,)) + .await + .expect_err("releasing twice should fail"); - assert_eq!(&bytes[..2], b"PK"); assert!(matches!(modules.list()[0].state, ModuleState::Ready)); broker_task.abort(); } + +/// Stage `bytes` over `BeginBlob` + `PutChunk`, returning the blob id. +async fn upload(proxy: &tinybus::Proxy, bytes: &[u8]) -> String { + use base64::Engine as _; + let encoder = base64::engine::general_purpose::STANDARD; + + let blob_id: String = proxy + .call("BeginBlob", (bytes.len() as u64, hex_digest(bytes))) + .await + .expect("BeginBlob should succeed"); + + let mut offset = 0usize; + while offset < bytes.len() { + let end = (offset + TEST_CHUNK).min(bytes.len()); + let received: u64 = proxy + .call( + "PutChunk", + ( + blob_id.clone(), + offset as u64, + encoder.encode(&bytes[offset..end]), + ), + ) + .await + .expect("PutChunk should succeed"); + assert_eq!(received, end as u64, "server disagreed about progress"); + offset = end; + } + blob_id +} + +/// Read a staged blob back over `GetChunk` and verify its digest. +async fn download(proxy: &tinybus::Proxy, handle: &BlobRef) -> Vec { + use base64::Engine as _; + let decoder = base64::engine::general_purpose::STANDARD; + + let mut out = Vec::with_capacity(usize::try_from(handle.total_bytes).unwrap_or_default()); + while (out.len() as u64) < handle.total_bytes { + let encoded: String = proxy + .call( + "GetChunk", + (handle.blob_id.clone(), out.len() as u64, TEST_CHUNK as u64), + ) + .await + .expect("GetChunk should succeed"); + let chunk = decoder.decode(encoded).expect("chunk is base64"); + assert!(!chunk.is_empty(), "read stalled at offset {}", out.len()); + out.extend_from_slice(&chunk); + } + assert_eq!( + hex_digest(&out), + handle.sha256, + "downloaded bytes do not match the declared digest" + ); + out +} + +/// A 1×1 PNG padded past [`TEST_CHUNK`] so its transfer spans several chunks. +/// +/// The padding rides in a trailing comment chunk, which keeps the file a valid +/// PNG that the module will accept and measure. +fn png_1x1() -> Vec { + let mut out = vec![0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]; + out.extend_from_slice(&13u32.to_be_bytes()); + out.extend_from_slice(b"IHDR"); + out.extend_from_slice(&1u32.to_be_bytes()); + out.extend_from_slice(&1u32.to_be_bytes()); + out.extend_from_slice(&[0x08, 0x06, 0x00, 0x00, 0x00]); + out.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); + out.extend_from_slice(&0u32.to_be_bytes()); + out.extend_from_slice(b"IDAT"); + out.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); + + // tEXt is an ancillary chunk, so a reader that does not care skips it. + let padding = vec![b'p'; TEST_CHUNK * 2]; + let mut text_chunk = b"pad\0".to_vec(); + text_chunk.extend_from_slice(&padding); + out.extend_from_slice(&u32::try_from(text_chunk.len()).unwrap().to_be_bytes()); + out.extend_from_slice(b"tEXt"); + out.extend_from_slice(&text_chunk); + out.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); + + out.extend_from_slice(&0u32.to_be_bytes()); + out.extend_from_slice(b"IEND"); + out.extend_from_slice(&[0xAE, 0x42, 0x60, 0x82]); + out +} + +/// A valid single-page PDF whose text layer holds `text`. +fn pdf_with_text(text: &str) -> Vec { + let content = format!("BT /F1 24 Tf 72 700 Td ({text}) Tj ET\n"); + let objects = [ + "<< /Type /Catalog /Pages 2 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \ + /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>" + .to_string(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(), + format!( + "<< /Length {} >>\nstream\n{content}endstream", + content.len() + ), + ]; + + let mut out = Vec::new(); + out.extend_from_slice(b"%PDF-1.4\n"); + let mut offsets = Vec::with_capacity(objects.len()); + for (i, body) in objects.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", i + 1).as_bytes()); + } + let xref_offset = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n", objects.len() + 1).as_bytes()); + out.extend_from_slice(b"0000000000 65535 f \n"); + for offset in &offsets { + out.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice( + format!( + "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref_offset}\n%%EOF\n", + objects.len() + 1 + ) + .as_bytes(), + ); + out +} diff --git a/docs/specs/tinybus-module.md b/docs/specs/tinybus-module.md index a3ed6f0..8a107aa 100644 --- a/docs/specs/tinybus-module.md +++ b/docs/specs/tinybus-module.md @@ -29,23 +29,47 @@ production. ## Behavior The private `tinydocs-module` workspace crate depends on the public library's -`docx` feature and builds as a `cdylib`. This separation keeps unpublished, -vendored TinyBus packages out of the crates.io package manifest. The module -claims `ai.tinyhumans.tinydocs.Docx`, serves the object path -`/ai/tinyhumans/tinydocs/Docx`, and exports one method: +`docx`, `pptx` and `pdf` features and builds as a `cdylib`. This separation keeps +unpublished, vendored TinyBus packages out of the crates.io package manifest. The +module claims `ai.tinyhumans.tinydocs.Documents`, serves the object path +`/ai/tinyhumans/tinydocs/Documents`, and exports seven methods: ```text -GenerateDocx(DocumentSpec) -> Vec +BeginBlob(total_bytes, sha256) -> blob_id +PutChunk(blob_id, offset, base64) -> bytes received so far +GetChunk(blob_id, offset, len) -> base64 +ReleaseBlob(blob_id) -> () +GenerateDocx(DocumentSpec) -> BlobRef +GeneratePptx(deck with image blobs) -> BlobRef +ExtractText(blob_id) -> BlobRef ``` -The argument is the same Serde document contract used by the Rust API. A -successful response contains a complete DOCX zip container. Invalid input and -writer failures use the distinct wire names -`ai.tinyhumans.tinydocs.Error.InvalidInput` and -`ai.tinyhumans.tinydocs.Error.GenerationFailed`. - -Generation is CPU-bound and runs on the module runtime's blocking pool. The -module itself retains no document state between calls. +The format arguments are the same Serde contracts used by the Rust API, except +that a slide image names a staged blob rather than carrying bytes inline. + +No method returns bytes inline. A frame is a 16 MiB JSON document and a `Vec` +serialises as an array of integers — about 3.5 bytes of frame per byte — so the +real inline ceiling is a few megabytes, below both a deck's legal image payload +and any `.pdf` worth extracting. Every unbounded value is therefore staged and +moved in base64 chunks. + +Invalid input, writer failures and extraction failures use the distinct wire +names `ai.tinyhumans.tinydocs.Error.InvalidInput`, +`ai.tinyhumans.tinydocs.Error.GenerationFailed` and +`ai.tinyhumans.tinydocs.Error.ExtractionFailed`. Transfer failures are grouped by +what the caller should do next: `Error.UnknownBlob` (restart the transfer), +`Error.TransferRefused` (a budget is full; the same request may succeed later) +and `Error.TransferFailed` (the caller sent something wrong; re-send). + +Synthesis and extraction are CPU-bound and run on the module runtime's blocking +pool. The module retains no document state between calls — only staged blobs, +each bounded and expiring. + +This interface replaces `ai.tinyhumans.tinydocs.Docx`, which returned bytes +inline. TinyBus forbids changing an interface in place, so the new contract took +a new name. It is not served alongside the old one: `module_export!` attaches its +method list to the first entry in `provides` and leaves the rest empty, so a +second fully-declared interface is not expressible without a TinyBus change. ## Invariants and constraints @@ -54,8 +78,14 @@ module itself retains no document state between calls. - No Rust value crosses the dynamic-library ABI boundary. - The native artifact must match the host target and TinyBus compatibility gate. -- Message payloads remain subject to TinyBus's 16 MiB frame cap. A future - format that can exceed it must use path or file-descriptor transfer. +- Message payloads remain subject to TinyBus's 16 MiB frame cap, which is why + bytes move in bounded chunks rather than inline. Path or file-descriptor + transfer would remove the copies and remains the better long-term answer. +- The staging area is bounded per chunk, per blob, in total and by blob count, + and expires untouched blobs. A module is never unloaded, so an unbounded + staging area is a leak with no end. +- A blob is verified against its declared SHA-256 before it becomes readable, so + a truncated or reordered transfer cannot be consumed as though it were whole. - Dynamic modules are trusted code with the host process's privileges. ## Acceptance criteria @@ -63,8 +93,10 @@ module itself retains no document state between calls. - `cargo build --release --package tinydocs-module` emits the platform dynamic library. - TinyBus `ModuleHost` admits that artifact and reaches `ready` state. -- A proxy call to `GenerateDocx` returns bytes beginning with the DOCX `PK` - signature. +- `GenerateDocx` and `GeneratePptx` stage output beginning with the OOXML `PK` + signature, and `ExtractText` recovers the text layer of a staged PDF. +- An image transferred across more than one chunk arrives intact and is embedded, + which is the case a single-chunk transfer would not prove. - CI executes that loader test on Linux. - A release uploads Linux and macOS bundles containing the matching TinyBus host, TinyDocs module, SHA-256 allowlist, and operational documentation. @@ -74,5 +106,11 @@ module itself retains no document state between calls. ## Open questions -None blocking this version. Bulk transfer becomes a separate protocol change -if a future format approaches the frame cap. +None blocking this version. + +Two things belong upstream in TinyBus rather than here. The staging area is +format-agnostic and every module that moves bytes will want it, so it is a +candidate for the module SDK. And `module_export!` attaching its method list only +to the first provided interface is what forces one interface to carry both the +transfer and the format methods; per-interface method lists would allow the +cleaner split. From fcf8b33d9907dc6ba0f3a4ff4fefa945225d8a8a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 11 Aug 2026 03:33:18 +0300 Subject: [PATCH 06/13] Carry inbound payloads on TinyBus streams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TinyBus gained chunked, flow-controlled streams (tinyhumansai/tinybus#9), which is the facility the hand-rolled staging area in the previous commit was standing in for. Inbound payloads now use it, and the half of that code they replace is deleted. What goes away is the risky half. `BeginBlob`, `PutChunk`, the append-only offset protocol, the upload digest check, the reserve-at-begin budget and the inbound TTL all existed to answer one question — what happens when a caller starts sending a document and never finishes — and TinyBus now answers it, per peer, with a window, a size cap and an idle timeout. A stream is also tied to the call that opened it and writable only by the peer that opened it, which is authorisation the blob ids never had: any peer that guessed an id could have written to somebody else's transfer. What stays is the output half, because replies cannot stream. `Interface::call` receives a member name and a JSON body — no caller identity, no connection — so a served object cannot open a stream back to whoever called it. A produced document is still held and pulled with `ReadOutput`, since returning it inline would put it through a 16 MiB JSON frame where a `Vec` costs ~3.5 bytes per byte. `outputs` is what is left of `blobs` once only that direction remains: still bounded four ways and still expiring, because TinyBus never unloads a module. A reply-stream seam upstream would delete it, and the spec now says so. A deck's images share one stream, concatenated in slide order, because a call has one stream and a deck has many pictures. Each image declares its `byte_len` in the spec rather than framing itself in the stream, which is what makes a truncated or over-long transfer a named rejection instead of a deck containing a picture assembled from two different images. A text-only deck passes no stream at all rather than opening an empty one. The E2E test moves an image pair across a real stream through the real dynamic loader and asserts the mismatch case, which is the only place the streaming paths can be tested honestly: a stream needs two connected peers and a broker, so a unit test against a bare struct cannot reach one. Net: 5 methods instead of 7, and the module no longer implements transfer. Co-authored-by: Medulla --- README.md | 52 +-- crates/tinydocs-module/src/blobs/mod.rs | 517 --------------------- crates/tinydocs-module/src/blobs/test.rs | 447 ------------------ crates/tinydocs-module/src/lib.rs | 4 +- crates/tinydocs-module/src/outputs/mod.rs | 296 ++++++++++++ crates/tinydocs-module/src/outputs/test.rs | 232 +++++++++ crates/tinydocs-module/src/service/mod.rs | 316 +++++++------ crates/tinydocs-module/src/service/test.rs | 466 ++++++------------- crates/tinydocs-module/src/service/wire.rs | 19 +- crates/tinydocs-module/tests/module_e2e.rs | 286 +++++++----- docs/specs/tinybus-module.md | 59 +-- vendor/tinybus | 2 +- 12 files changed, 1100 insertions(+), 1596 deletions(-) delete mode 100644 crates/tinydocs-module/src/blobs/mod.rs delete mode 100644 crates/tinydocs-module/src/blobs/test.rs create mode 100644 crates/tinydocs-module/src/outputs/mod.rs create mode 100644 crates/tinydocs-module/src/outputs/test.rs diff --git a/README.md b/README.md index c43d239..43023c6 100644 --- a/README.md +++ b/README.md @@ -125,42 +125,42 @@ The native artifact is `target/release/libtinydocs_module.so` on Linux, `libtinydocs_module.dylib` on macOS, or `tinydocs_module.dll` on Windows. Load it with a TinyBus host built with its `modules` feature. It claims `ai.tinyhumans.tinydocs.Documents` at `/ai/tinyhumans/tinydocs/Documents` and -exposes the three format operations plus the chunked transfer they depend on: +exposes: ```text -BeginBlob(total_bytes, sha256) -> blob_id -PutChunk(blob_id, offset, base64) -> bytes received so far -GetChunk(blob_id, offset, len) -> base64 -ReleaseBlob(blob_id) -> () -GenerateDocx(DocumentSpec) -> BlobRef -GeneratePptx(deck with image blobs) -> BlobRef -ExtractText(blob_id) -> BlobRef +GenerateDocx(DocumentSpec) -> OutputRef +GeneratePptx(deck, Option) -> OutputRef +ExtractText(StreamRef) -> OutputRef +ReadOutput(output_id, offset, len) -> base64 +ReleaseOutput(output_id) -> () ``` -Nothing returns bytes inline. A TinyBus frame is a 16 MiB JSON document, and a -`Vec` serialises as an array of integers — roughly 3.5 bytes of frame per -byte of payload — so the real inline ceiling is a few megabytes. That is below a -deck's legal image payload and below any `.pdf` worth extracting. So every -unbounded value is staged and moved in base64 chunks, and a caller's code path is -the same regardless of size. +Payloads in and payloads out are not symmetric, and the reason is worth knowing. -The staging area is bounded in four independent ways — per chunk, per blob, in -total, and by blob count — and blobs that stop being touched expire. A module is -trusted in-process code that TinyBus never unloads, so an abandoned upload is -never reclaimed by a process exit that does not come. +**Inbound bytes ride a TinyBus stream.** The caller opens one alongside the call +and writes while the call is outstanding; flow control, the size cap, the idle +timeout and the "only the peer that opened it may write" rule are all the bus's, +so nothing here re-implements them. A deck's images are concatenated into a +single stream in slide order, each declaring its `byte_len`, because a call has +one stream and a deck has many pictures — and putting the lengths in the spec is +what makes a truncated transfer a named rejection instead of a deck with a +picture assembled from two different images. + +**Replies cannot.** `Interface::call` receives a member name and a JSON body — +no caller identity, no connection — so a served object cannot open a stream back +to whoever called it. A produced document is therefore held by the module and +pulled with `ReadOutput`, because returning it inline would put it through a +16 MiB JSON frame where a `Vec` costs about 3.5 bytes per byte. That half +disappears the day TinyBus grows a reply-stream seam. + +What the module holds is bounded four ways — per document, in total, by count, +and by an idle TTL — because TinyBus never unloads a module, so anything retained +is retained until the process exits unless something reclaims it. This interface replaces `ai.tinyhumans.tinydocs.Docx`, which returned bytes inline. TinyBus's guidance is that an existing interface must not change in place, so the new contract took a new name. -The release workflow attaches installable Linux and macOS bundles containing -the matching TinyBus host, the TinyDocs module, a SHA-256 `modules.toml` -allowlist, and protocol/module documentation. It also publishes -`checksum.toml`, which TinyBus uses to verify a downloaded precompiled module -archive, plus the crates.io package and pinned TinyBus source. TinyBus modules -are target-specific and trusted: download the bundle matching the host, and -install it only from a trusted release. - A TinyBus host can download and verify the matching archive directly from a tagged GitHub release with `ModuleHost::load_github_release`; the archive must be selected by its exact target-specific asset name and the release URL must diff --git a/crates/tinydocs-module/src/blobs/mod.rs b/crates/tinydocs-module/src/blobs/mod.rs deleted file mode 100644 index d5c2664..0000000 --- a/crates/tinydocs-module/src/blobs/mod.rs +++ /dev/null @@ -1,517 +0,0 @@ -//! Chunked byte transfer for payloads that do not fit in a bus frame. -//! -//! # Why this exists -//! -//! A `TinyBus` frame is a JSON document capped at 16 MiB, and `TinyBus`'s own -//! guidance is that large payloads travel as paths rather than inline. Neither -//! half of the document surface fits inside that: a deck may legally carry -//! 8 images of 5 MiB each, and a `.pdf` handed in for extraction is bounded only -//! by what the host accepted. Serialising bytes as a JSON array of integers -//! makes it worse — roughly 3.5 bytes of frame per byte of payload — so the -//! real inline ceiling is a few megabytes, not sixteen. -//! -//! So bytes move in chunks, base64-encoded (1.34× rather than 3.5×), through a -//! staging area addressed by opaque blob ids. A caller stages a `.pdf`, calls -//! `ExtractText`, and reads the result back out the same way; the frame size -//! stops being part of the contract. -//! -//! # Every limit here is load-bearing -//! -//! A module is trusted in-process code that `TinyBus` never unloads, so an -//! abandoned upload is not garbage collected by a process exit that never comes. -//! The store therefore bounds four separate things — one chunk, one blob, the -//! whole staging area, and the number of live blobs — and expires blobs that -//! stop being touched. Without the last one, a caller that dies mid-upload leaks -//! its partial blob for the life of the host. -//! -//! Expiry is lazy: every operation sweeps first, so there is no background task -//! and no timer to reason about. The clock is a parameter rather than a call to -//! [`Instant::now`], which is what makes the expiry rules testable at all. -//! -//! # Append-only by construction -//! -//! `put_chunk` requires `offset` to equal exactly how many bytes have arrived so -//! far. That is stricter than necessary, and deliberately so: sparse writes would -//! need range bookkeeping, a definition of what overlapping writes mean, and a -//! way to know when a blob is actually complete. Requiring append makes -//! "complete" mean "length reached", and makes a lost or duplicated chunk a named -//! error at the moment it happens rather than a corrupt blob discovered later. -//! -//! Completion verifies the caller's SHA-256 before the blob becomes readable, so -//! a truncated or reordered transfer cannot be consumed as though it were whole. - -use std::collections::HashMap; -use std::sync::Mutex; -use std::time::{Duration, Instant}; - -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; - -/// Maximum size of a single chunk. -/// -/// Sized so that one chunk plus its base64 expansion and the surrounding JSON -/// stays well inside a 16 MiB frame, with room left for a method envelope. -pub const MAX_CHUNK_BYTES: usize = 4 * 1024 * 1024; - -/// Maximum size of one staged blob. -pub const MAX_BLOB_BYTES: usize = 64 * 1024 * 1024; - -/// Maximum total size of all staged blobs at once. -/// -/// Bounds the module's resident memory independently of how many callers are -/// mid-transfer. -pub const MAX_TOTAL_STAGED_BYTES: usize = 128 * 1024 * 1024; - -/// Maximum number of blobs alive at once. -/// -/// A separate bound from the byte budget: many tiny abandoned blobs are as much -/// of a leak as one large one, and each carries bookkeeping of its own. -pub const MAX_LIVE_BLOBS: usize = 64; - -/// How long a blob may go untouched before it is expired. -/// -/// Long enough that a slow but live transfer is never reaped, short enough that -/// an abandoned one does not outlive the request that started it by much. -pub const IDLE_TTL: Duration = Duration::from_secs(300); - -/// A handle to a complete staged blob, plus what a caller needs to read it back. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct BlobRef { - /// Opaque identifier for the blob. - pub blob_id: String, - /// Total size in bytes, so a caller knows how many chunks to ask for. - pub total_bytes: u64, - /// Lowercase hex SHA-256 of the bytes, so a caller can verify what it read. - pub sha256: String, -} - -/// Why a blob operation was refused. -/// -/// Every variant is a distinct condition with a distinct wire name, because a -/// caller's correct response differs: a budget refusal is worth retrying later, -/// a hash mismatch means re-sending, and an unknown id means the blob is gone -/// and the whole transfer has to start again. -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -pub enum BlobError { - /// The declared SHA-256 was not 64 lowercase hexadecimal characters. - #[error("sha256 must be 64 lowercase hexadecimal characters")] - MalformedDigest, - - /// The declared total size exceeds [`MAX_BLOB_BYTES`]. - #[error("blob size exceeds the {MAX_BLOB_BYTES}-byte per-blob limit")] - BlobTooLarge, - - /// A single chunk exceeded [`MAX_CHUNK_BYTES`]. - #[error("chunk exceeds the {MAX_CHUNK_BYTES}-byte per-chunk limit")] - ChunkTooLarge, - - /// Accepting the blob would exceed [`MAX_TOTAL_STAGED_BYTES`]. - #[error("staging area is full")] - StagingFull, - - /// [`MAX_LIVE_BLOBS`] blobs are already staged. - #[error("too many blobs staged at once")] - TooManyBlobs, - - /// No blob with that id — never staged, released, or expired. - #[error("unknown blob id")] - UnknownBlob, - - /// `offset` did not equal the number of bytes received so far. - #[error("chunk offset {actual} does not continue the blob at {expected}")] - OutOfOrderChunk { - /// The offset the next chunk must carry. - expected: u64, - /// The offset the caller sent. - actual: u64, - }, - - /// The chunk would write past the declared total size. - #[error("chunk would exceed the declared blob size")] - OverlongBlob, - - /// The assembled bytes did not hash to the declared digest. - #[error("assembled blob does not match the declared sha256")] - DigestMismatch, - - /// The blob is still being uploaded and cannot be read yet. - #[error("blob is incomplete")] - IncompleteBlob, - - /// A read started past the end of the blob. - #[error("read offset is past the end of the blob")] - ReadPastEnd, -} - -/// One blob in the staging area. -struct Blob { - /// Declared total size; the blob is complete when `data` reaches it. - expected_bytes: usize, - /// Declared digest, verified once the blob is complete. - expected_sha256: String, - data: Vec, - complete: bool, - last_touched: Instant, -} - -impl Blob { - /// Bytes charged against the staging budget. - /// - /// The declared total, not the bytes received so far: the budget is reserved - /// at `begin` so a transfer that is admitted can always finish, rather than - /// failing halfway when somebody else fills the area. - fn reserved(&self) -> usize { - self.expected_bytes.max(self.data.len()) - } -} - -/// The staging area shared by every method on the service. -#[derive(Default)] -pub struct BlobStore { - inner: Mutex, -} - -/// Reports how much is staged, never what is staged. -/// -/// Written by hand rather than derived because a derived implementation would -/// put every staged byte into whatever formatted it — a log line, a panic -/// message, an error. Staged bytes are caller data. -impl std::fmt::Debug for BlobStore { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let inner = self.lock(); - f.debug_struct("BlobStore") - .field("live_blobs", &inner.blobs.len()) - .field("staged_bytes", &inner.staged_bytes()) - .finish() - } -} - -#[derive(Default)] -struct Inner { - blobs: HashMap, - next_id: u64, -} - -impl BlobStore { - /// An empty store. - #[must_use] - pub fn new() -> Self { - Self::default() - } - - /// Reserve space for a blob of `total_bytes` that will hash to `sha256`. - /// - /// # Errors - /// - /// [`BlobError::MalformedDigest`], [`BlobError::BlobTooLarge`], - /// [`BlobError::TooManyBlobs`], or [`BlobError::StagingFull`]. - pub fn begin(&self, total_bytes: u64, sha256: &str, now: Instant) -> Result { - if !is_lowercase_sha256(sha256) { - return Err(BlobError::MalformedDigest); - } - let expected_bytes = usize::try_from(total_bytes).map_err(|_| BlobError::BlobTooLarge)?; - if expected_bytes > MAX_BLOB_BYTES { - return Err(BlobError::BlobTooLarge); - } - - let mut inner = self.lock(); - inner.sweep_expired(now); - if inner.blobs.len() >= MAX_LIVE_BLOBS { - return Err(BlobError::TooManyBlobs); - } - if inner.staged_bytes().saturating_add(expected_bytes) > MAX_TOTAL_STAGED_BYTES { - return Err(BlobError::StagingFull); - } - - let id = inner.allocate_id(); - inner.blobs.insert( - id.clone(), - Blob { - expected_bytes, - expected_sha256: sha256.to_string(), - // Not pre-allocated: a caller can declare 64 MiB and never send - // it, and reserving the allocation up front would make that a - // way to spend the host's memory for free. - data: Vec::new(), - complete: expected_bytes == 0, - last_touched: now, - }, - ); - // A zero-length blob is complete on arrival, so its digest is checked - // here rather than on a chunk that will never come. - if expected_bytes == 0 { - let verified = verify(&[], sha256); - if !verified { - inner.blobs.remove(&id); - return Err(BlobError::DigestMismatch); - } - } - Ok(id) - } - - /// Append `data` at `offset`, returning the number of bytes received so far. - /// - /// When the blob reaches its declared size, its digest is verified and it - /// becomes readable. A mismatch drops the blob. - /// - /// # Errors - /// - /// [`BlobError::ChunkTooLarge`], [`BlobError::UnknownBlob`], - /// [`BlobError::OutOfOrderChunk`], [`BlobError::OverlongBlob`], or - /// [`BlobError::DigestMismatch`]. - pub fn put_chunk( - &self, - blob_id: &str, - offset: u64, - data: &[u8], - now: Instant, - ) -> Result { - if data.len() > MAX_CHUNK_BYTES { - return Err(BlobError::ChunkTooLarge); - } - - let mut inner = self.lock(); - inner.sweep_expired(now); - let blob = inner.blobs.get_mut(blob_id).ok_or(BlobError::UnknownBlob)?; - - let received = blob.data.len() as u64; - if blob.complete || offset != received { - return Err(BlobError::OutOfOrderChunk { - expected: received, - actual: offset, - }); - } - if blob.data.len().saturating_add(data.len()) > blob.expected_bytes { - return Err(BlobError::OverlongBlob); - } - - blob.data.extend_from_slice(data); - blob.last_touched = now; - if blob.data.len() == blob.expected_bytes { - if verify(&blob.data, &blob.expected_sha256) { - blob.complete = true; - } else { - inner.blobs.remove(blob_id); - return Err(BlobError::DigestMismatch); - } - } - // Re-read rather than reuse the borrow above: the mismatch branch may - // have removed the entry. - Ok(inner - .blobs - .get(blob_id) - .map_or(0, |blob| blob.data.len() as u64)) - } - - /// Read up to `len` bytes of a complete blob starting at `offset`. - /// - /// A read that runs past the end is clamped rather than refused, so a caller - /// can ask for a whole chunk on the final read without special-casing the - /// remainder. - /// - /// # Errors - /// - /// [`BlobError::UnknownBlob`], [`BlobError::IncompleteBlob`], - /// [`BlobError::ChunkTooLarge`], or [`BlobError::ReadPastEnd`]. - pub fn get_chunk( - &self, - blob_id: &str, - offset: u64, - len: u64, - now: Instant, - ) -> Result, BlobError> { - let len = usize::try_from(len).map_err(|_| BlobError::ChunkTooLarge)?; - if len > MAX_CHUNK_BYTES { - return Err(BlobError::ChunkTooLarge); - } - - let mut inner = self.lock(); - inner.sweep_expired(now); - let blob = inner.blobs.get_mut(blob_id).ok_or(BlobError::UnknownBlob)?; - if !blob.complete { - return Err(BlobError::IncompleteBlob); - } - let start = usize::try_from(offset).map_err(|_| BlobError::ReadPastEnd)?; - if start > blob.data.len() { - return Err(BlobError::ReadPastEnd); - } - blob.last_touched = now; - let end = start.saturating_add(len).min(blob.data.len()); - Ok(blob.data[start..end].to_vec()) - } - - /// Stage `bytes` as an already-complete blob and return its handle. - /// - /// This is the outbound direction: a generated document or an extracted text - /// body that the module produced and the caller now has to read back. - /// - /// # Errors - /// - /// [`BlobError::BlobTooLarge`], [`BlobError::TooManyBlobs`], or - /// [`BlobError::StagingFull`]. - pub fn insert_complete(&self, bytes: Vec, now: Instant) -> Result { - if bytes.len() > MAX_BLOB_BYTES { - return Err(BlobError::BlobTooLarge); - } - - let mut inner = self.lock(); - inner.sweep_expired(now); - if inner.blobs.len() >= MAX_LIVE_BLOBS { - return Err(BlobError::TooManyBlobs); - } - if inner.staged_bytes().saturating_add(bytes.len()) > MAX_TOTAL_STAGED_BYTES { - return Err(BlobError::StagingFull); - } - - let sha256 = hex_digest(&bytes); - let total_bytes = bytes.len() as u64; - let id = inner.allocate_id(); - inner.blobs.insert( - id.clone(), - Blob { - expected_bytes: bytes.len(), - expected_sha256: sha256.clone(), - data: bytes, - complete: true, - last_touched: now, - }, - ); - Ok(BlobRef { - blob_id: id, - total_bytes, - sha256, - }) - } - - /// Remove a complete blob and return its bytes. - /// - /// Used when the module consumes a staged input — the bytes of a `.pdf`, or - /// an image for a deck. Taking rather than copying frees the staging budget - /// at the moment the blob stops being needed. - /// - /// # Errors - /// - /// [`BlobError::UnknownBlob`] or [`BlobError::IncompleteBlob`]. - pub fn take_complete(&self, blob_id: &str, now: Instant) -> Result, BlobError> { - let mut inner = self.lock(); - inner.sweep_expired(now); - let blob = inner.blobs.get(blob_id).ok_or(BlobError::UnknownBlob)?; - if !blob.complete { - return Err(BlobError::IncompleteBlob); - } - Ok(inner - .blobs - .remove(blob_id) - .map(|blob| blob.data) - .unwrap_or_default()) - } - - /// Drop a blob and free its budget. - /// - /// # Errors - /// - /// [`BlobError::UnknownBlob`] if there is nothing to release. Releasing is - /// reported rather than silently accepted so a caller learns that its blob - /// had already expired. - pub fn release(&self, blob_id: &str, now: Instant) -> Result<(), BlobError> { - let mut inner = self.lock(); - inner.sweep_expired(now); - inner - .blobs - .remove(blob_id) - .map(|_| ()) - .ok_or(BlobError::UnknownBlob) - } - - /// Number of blobs currently staged, for tests and diagnostics. - #[must_use] - pub fn live_count(&self) -> usize { - self.lock().blobs.len() - } - - /// Take the lock, recovering from a poisoned mutex. - /// - /// A panic while holding this lock can only have happened between two - /// `HashMap` operations, so the map is structurally intact and the worst - /// case is one blob left in a partial state — which its digest check will - /// reject. Refusing every subsequent request would turn one caller's panic - /// into a dead module, and `TinyBus` never unloads a module to recover. - fn lock(&self) -> std::sync::MutexGuard<'_, Inner> { - self.inner - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - } -} - -impl Inner { - /// Total bytes reserved by every staged blob. - fn staged_bytes(&self) -> usize { - self.blobs - .values() - .map(Blob::reserved) - .fold(0usize, usize::saturating_add) - } - - /// Drop every blob untouched for longer than [`IDLE_TTL`]. - fn sweep_expired(&mut self, now: Instant) { - self.blobs - .retain(|_, blob| now.saturating_duration_since(blob.last_touched) <= IDLE_TTL); - } - - /// Allocate an unused blob id. - /// - /// A counter, not a random value: ids are opaque handles inside one process, - /// never authorisation tokens, and a counter makes a leaked id visible in a - /// log rather than looking like a secret. - fn allocate_id(&mut self) -> String { - self.next_id = self.next_id.wrapping_add(1); - format!("blob-{}", self.next_id) - } -} - -/// Whether `value` is exactly 64 lowercase hexadecimal characters. -fn is_lowercase_sha256(value: &str) -> bool { - value.len() == 64 - && value - .bytes() - .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) -} - -/// Lowercase hex SHA-256 of `bytes`, in the exact shape `BeginBlob` expects. -/// -/// Public because declaring a digest is part of using the transfer surface: a -/// caller has to produce this value, and one implementation both sides agree on -/// beats two that can disagree about case or padding. -#[must_use] -pub fn hex_digest(bytes: &[u8]) -> String { - let digest = Sha256::digest(bytes); - let mut out = String::with_capacity(64); - for byte in digest { - use std::fmt::Write as _; - // Writing into a String cannot fail; the result is discarded rather than - // unwrapped so this stays panic-free. - let _ = write!(out, "{byte:02x}"); - } - out -} - -/// Whether `bytes` hashes to `expected`, compared without early exit. -fn verify(bytes: &[u8], expected: &str) -> bool { - let actual = hex_digest(bytes); - // Constant-time over the digest strings. The digest is not a secret, so this - // is defence in depth rather than a requirement — but a hash comparison is - // exactly the shape that later becomes security-relevant, and the cost of - // getting it right once is nothing. - if actual.len() != expected.len() { - return false; - } - actual - .bytes() - .zip(expected.bytes()) - .fold(0u8, |acc, (a, b)| acc | (a ^ b)) - == 0 -} - -#[cfg(test)] -mod test; diff --git a/crates/tinydocs-module/src/blobs/test.rs b/crates/tinydocs-module/src/blobs/test.rs deleted file mode 100644 index 67a87d8..0000000 --- a/crates/tinydocs-module/src/blobs/test.rs +++ /dev/null @@ -1,447 +0,0 @@ -//! Unit tests for the chunked blob staging area. -//! -//! Weighted towards refusals on purpose. A happy-path transfer proves the store -//! can move bytes; it is the bounds and the expiry that decide whether an -//! abandoned upload leaks for the life of a module the host never unloads, and -//! whether a truncated transfer can be consumed as though it were whole. - -#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] - -use std::time::{Duration, Instant}; - -use super::{ - BlobError, BlobStore, IDLE_TTL, MAX_BLOB_BYTES, MAX_CHUNK_BYTES, MAX_LIVE_BLOBS, - MAX_TOTAL_STAGED_BYTES, hex_digest, is_lowercase_sha256, verify, -}; - -/// A fixed origin for the injected clock, so every test is deterministic. -fn t0() -> Instant { - Instant::now() -} - -/// Stage `bytes` as a completed upload, the way a caller would. -fn upload(store: &BlobStore, bytes: &[u8], now: Instant) -> String { - let id = store - .begin(bytes.len() as u64, &hex_digest(bytes), now) - .expect("begin should succeed"); - if !bytes.is_empty() { - let received = store - .put_chunk(&id, 0, bytes, now) - .expect("put_chunk should succeed"); - assert_eq!(received, bytes.len() as u64); - } - id -} - -#[test] -fn a_blob_round_trips_through_chunks() { - let store = BlobStore::new(); - let now = t0(); - let payload: Vec = (0..10_000u32).map(|i| (i % 251) as u8).collect(); - let digest = hex_digest(&payload); - - let id = store.begin(payload.len() as u64, &digest, now).unwrap(); - // Three uneven chunks, so the arithmetic is not accidentally aligned. - let mut sent = 0usize; - for size in [4_000usize, 4_000, 2_000] { - let end = sent + size; - let received = store - .put_chunk(&id, sent as u64, &payload[sent..end], now) - .unwrap(); - sent = end; - assert_eq!(received, sent as u64); - } - - let mut read = Vec::new(); - let mut offset = 0u64; - while read.len() < payload.len() { - let chunk = store.get_chunk(&id, offset, 3_000, now).unwrap(); - assert!(!chunk.is_empty(), "read stalled at offset {offset}"); - offset += chunk.len() as u64; - read.extend_from_slice(&chunk); - } - assert_eq!(read, payload); - assert_eq!(hex_digest(&read), digest); -} - -#[test] -fn a_read_past_the_end_is_clamped_not_refused() { - // So a caller can ask for a full chunk on the final read without having to - // compute the remainder itself. - let store = BlobStore::new(); - let now = t0(); - let id = upload(&store, b"twelve bytes", now); - let chunk = store.get_chunk(&id, 6, 1_000_000, now).unwrap(); - assert_eq!(chunk, b" bytes"); -} - -#[test] -fn a_read_starting_past_the_end_is_refused() { - let store = BlobStore::new(); - let now = t0(); - let id = upload(&store, b"short", now); - assert_eq!( - store.get_chunk(&id, 99, 10, now), - Err(BlobError::ReadPastEnd) - ); -} - -#[test] -fn an_out_of_order_chunk_is_refused_and_names_both_offsets() { - let store = BlobStore::new(); - let now = t0(); - let payload = vec![7u8; 100]; - let id = store.begin(100, &hex_digest(&payload), now).unwrap(); - store.put_chunk(&id, 0, &payload[..40], now).unwrap(); - - // A duplicated chunk and a skipped chunk are the two ways a transfer goes - // wrong; both must fail here rather than corrupt the blob silently. - assert_eq!( - store.put_chunk(&id, 0, &payload[..40], now), - Err(BlobError::OutOfOrderChunk { - expected: 40, - actual: 0 - }) - ); - assert_eq!( - store.put_chunk(&id, 60, &payload[60..], now), - Err(BlobError::OutOfOrderChunk { - expected: 40, - actual: 60 - }) - ); -} - -#[test] -fn a_chunk_past_the_declared_size_is_refused() { - let store = BlobStore::new(); - let now = t0(); - let payload = vec![1u8; 10]; - let id = store.begin(10, &hex_digest(&payload), now).unwrap(); - let one_too_many = [1u8; 11]; - assert_eq!( - store.put_chunk(&id, 0, &one_too_many, now), - Err(BlobError::OverlongBlob) - ); -} - -#[test] -fn an_oversize_chunk_is_refused() { - let store = BlobStore::new(); - let now = t0(); - let payload = vec![0u8; MAX_CHUNK_BYTES + 1]; - let id = store - .begin(payload.len() as u64, &hex_digest(&payload), now) - .unwrap(); - assert_eq!( - store.put_chunk(&id, 0, &payload, now), - Err(BlobError::ChunkTooLarge) - ); -} - -#[test] -fn a_digest_mismatch_drops_the_blob() { - // The blob must not remain readable, and must not remain charged against the - // budget, after failing its integrity check. - let store = BlobStore::new(); - let now = t0(); - let claimed = hex_digest(b"what the caller promised"); - let id = store.begin(5, &claimed, now).unwrap(); - - assert_eq!( - store.put_chunk(&id, 0, b"other", now), - Err(BlobError::DigestMismatch) - ); - assert_eq!(store.live_count(), 0, "mismatched blob was retained"); - assert_eq!(store.get_chunk(&id, 0, 5, now), Err(BlobError::UnknownBlob)); -} - -#[test] -fn an_incomplete_blob_cannot_be_read_or_taken() { - let store = BlobStore::new(); - let now = t0(); - let payload = vec![3u8; 100]; - let id = store.begin(100, &hex_digest(&payload), now).unwrap(); - store.put_chunk(&id, 0, &payload[..50], now).unwrap(); - - assert_eq!( - store.get_chunk(&id, 0, 10, now), - Err(BlobError::IncompleteBlob) - ); - assert_eq!( - store.take_complete(&id, now), - Err(BlobError::IncompleteBlob) - ); -} - -#[test] -fn a_malformed_digest_is_refused_before_anything_is_reserved() { - let store = BlobStore::new(); - let now = t0(); - for bad in [ - "", - "abc", - &"A".repeat(64), // uppercase - &"g".repeat(64), // not hex - &"a".repeat(63), // too short - &"a".repeat(65), // too long - ] { - assert_eq!( - store.begin(10, bad, now), - Err(BlobError::MalformedDigest), - "accepted {bad:?}" - ); - } - assert_eq!(store.live_count(), 0); -} - -#[test] -fn a_blob_over_the_per_blob_limit_is_refused() { - let store = BlobStore::new(); - let now = t0(); - assert_eq!( - store.begin(MAX_BLOB_BYTES as u64 + 1, &hex_digest(b"anything"), now), - Err(BlobError::BlobTooLarge) - ); - // A declared size beyond usize on a 32-bit host lands in the same refusal. - assert_eq!( - store.begin(u64::MAX, &hex_digest(b"anything"), now), - Err(BlobError::BlobTooLarge) - ); -} - -#[test] -fn the_staging_budget_is_reserved_at_begin_not_on_arrival() { - // Reserving up front is what lets an admitted transfer always finish. If the - // budget only counted bytes received, two callers could each be admitted for - // 96 MiB and then fight over the last 32. - let store = BlobStore::new(); - let now = t0(); - let big = MAX_TOTAL_STAGED_BYTES / 2; - let digest = hex_digest(b"never sent"); - - store.begin(big as u64, &digest, now).unwrap(); - store.begin(big as u64, &digest, now).unwrap(); - // Nothing has actually been uploaded, yet the area is full. - assert_eq!(store.begin(1, &digest, now), Err(BlobError::StagingFull)); -} - -#[test] -fn too_many_live_blobs_is_refused() { - let store = BlobStore::new(); - let now = t0(); - let digest = hex_digest(b"x"); - for _ in 0..MAX_LIVE_BLOBS { - store.begin(1, &digest, now).unwrap(); - } - assert_eq!(store.live_count(), MAX_LIVE_BLOBS); - assert_eq!(store.begin(1, &digest, now), Err(BlobError::TooManyBlobs)); -} - -#[test] -fn an_untouched_blob_expires_and_frees_its_budget() { - // The bound that matters most: a caller that dies mid-upload must not leak - // its partial blob for the life of a module the host never unloads. - let store = BlobStore::new(); - let now = t0(); - let payload = vec![9u8; 1_000]; - let id = store - .begin(payload.len() as u64, &hex_digest(&payload), now) - .unwrap(); - store.put_chunk(&id, 0, &payload[..500], now).unwrap(); - assert_eq!(store.live_count(), 1); - - // Still live right on the boundary. - let at_ttl = now + IDLE_TTL; - assert!(store.get_chunk(&id, 0, 1, at_ttl).is_err()); // incomplete, but alive - assert_eq!(store.live_count(), 1); - - // Past it, the next operation sweeps it away. - let past_ttl = now + IDLE_TTL + Duration::from_secs(1); - assert_eq!( - store.put_chunk(&id, 500, &payload[500..], past_ttl), - Err(BlobError::UnknownBlob) - ); - assert_eq!(store.live_count(), 0); -} - -#[test] -fn activity_keeps_a_slow_transfer_alive() { - // The flip side: a transfer that is slow but live must never be reaped. - let store = BlobStore::new(); - let mut now = t0(); - let payload = vec![4u8; 400]; - let id = store.begin(400, &hex_digest(&payload), now).unwrap(); - - for start in (0..400).step_by(100) { - // Each chunk arrives just inside the window, well past the total elapsed - // TTL — three of these sum to more than IDLE_TTL. - now += IDLE_TTL.saturating_sub(Duration::from_secs(1)); - store - .put_chunk(&id, start as u64, &payload[start..start + 100], now) - .expect("a touched blob must not expire"); - } - assert_eq!(store.get_chunk(&id, 0, 400, now).unwrap(), payload); -} - -#[test] -fn releasing_frees_the_budget_and_is_reported_once() { - let store = BlobStore::new(); - let now = t0(); - let id = upload(&store, b"payload", now); - assert!(store.release(&id, now).is_ok()); - assert_eq!(store.live_count(), 0); - // A second release tells the caller the blob is gone rather than pretending. - assert_eq!(store.release(&id, now), Err(BlobError::UnknownBlob)); -} - -#[test] -fn taking_a_blob_removes_it() { - let store = BlobStore::new(); - let now = t0(); - let id = upload(&store, b"consume me", now); - assert_eq!(store.take_complete(&id, now).unwrap(), b"consume me"); - assert_eq!(store.live_count(), 0); - assert_eq!(store.take_complete(&id, now), Err(BlobError::UnknownBlob)); -} - -#[test] -fn insert_complete_produces_a_readable_handle() { - let store = BlobStore::new(); - let now = t0(); - let bytes = b"generated output".to_vec(); - let handle = store.insert_complete(bytes.clone(), now).unwrap(); - - assert_eq!(handle.total_bytes, bytes.len() as u64); - assert_eq!(handle.sha256, hex_digest(&bytes)); - assert_eq!( - store.get_chunk(&handle.blob_id, 0, 1_000, now).unwrap(), - bytes - ); -} - -#[test] -fn insert_complete_respects_every_budget() { - let store = BlobStore::new(); - let now = t0(); - assert_eq!( - store.insert_complete(vec![0u8; MAX_BLOB_BYTES + 1], now), - Err(BlobError::BlobTooLarge) - ); - - // Filling the staging area takes more than one blob: the per-blob cap is - // half the total, so the area can only ever be filled by at least two. - let full = BlobStore::new(); - let digest = hex_digest(b"reserved"); - let per_blob = MAX_BLOB_BYTES; - let mut reserved = 0usize; - while reserved + per_blob <= MAX_TOTAL_STAGED_BYTES { - full.begin(per_blob as u64, &digest, now).unwrap(); - reserved += per_blob; - } - assert_eq!(reserved, MAX_TOTAL_STAGED_BYTES, "area not fully reserved"); - assert_eq!( - full.insert_complete(vec![0u8; 16], now), - Err(BlobError::StagingFull) - ); - - let crowded = BlobStore::new(); - for _ in 0..MAX_LIVE_BLOBS { - crowded.begin(1, &digest, now).unwrap(); - } - assert_eq!( - crowded.insert_complete(vec![0u8; 1], now), - Err(BlobError::TooManyBlobs) - ); -} - -#[test] -fn a_zero_length_blob_completes_at_begin() { - // There is no chunk to complete it on, so the digest has to be checked when - // it is declared or the blob would never become readable. - let store = BlobStore::new(); - let now = t0(); - let id = store.begin(0, &hex_digest(b""), now).unwrap(); - assert_eq!(store.get_chunk(&id, 0, 10, now).unwrap(), Vec::::new()); - assert_eq!(store.take_complete(&id, now).unwrap(), Vec::::new()); -} - -#[test] -fn a_zero_length_blob_with_a_wrong_digest_is_refused_at_begin() { - let store = BlobStore::new(); - let now = t0(); - assert_eq!( - store.begin(0, &hex_digest(b"not empty"), now), - Err(BlobError::DigestMismatch) - ); - assert_eq!(store.live_count(), 0); -} - -#[test] -fn blob_ids_are_unique_across_reuse() { - // Ids must not be recycled after a release: a caller holding a stale id - // would otherwise read somebody else's blob. - let store = BlobStore::new(); - let now = t0(); - let first = upload(&store, b"one", now); - store.release(&first, now).unwrap(); - let second = upload(&store, b"two", now); - assert_ne!(first, second); -} - -#[test] -fn unknown_ids_are_refused_by_every_operation() { - let store = BlobStore::new(); - let now = t0(); - assert_eq!( - store.put_chunk("nope", 0, b"x", now), - Err(BlobError::UnknownBlob) - ); - assert_eq!( - store.get_chunk("nope", 0, 1, now), - Err(BlobError::UnknownBlob) - ); - assert_eq!( - store.take_complete("nope", now), - Err(BlobError::UnknownBlob) - ); - assert_eq!(store.release("nope", now), Err(BlobError::UnknownBlob)); -} - -#[test] -fn an_oversize_read_length_is_refused() { - let store = BlobStore::new(); - let now = t0(); - let id = upload(&store, b"small", now); - assert_eq!( - store.get_chunk(&id, 0, MAX_CHUNK_BYTES as u64 + 1, now), - Err(BlobError::ChunkTooLarge) - ); - assert_eq!( - store.get_chunk(&id, 0, u64::MAX, now), - Err(BlobError::ChunkTooLarge) - ); -} - -#[test] -fn digest_helpers_agree_with_a_known_vector() { - // The SHA-256 of the empty string, so a wrong hasher or a broken hex - // encoding fails here rather than in an integration test. - assert_eq!( - hex_digest(b""), - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - ); - assert!(verify(b"", &hex_digest(b""))); - assert!(!verify(b"", &hex_digest(b"x"))); - // A length mismatch must fail before the comparison loop. - assert!(!verify(b"", "abcd")); -} - -#[test] -fn digest_shape_is_validated_strictly() { - assert!(is_lowercase_sha256(&hex_digest(b"anything"))); - assert!(!is_lowercase_sha256( - &hex_digest(b"anything").to_uppercase() - )); - assert!(!is_lowercase_sha256("zz")); -} diff --git a/crates/tinydocs-module/src/lib.rs b/crates/tinydocs-module/src/lib.rs index 926be01..43fe84c 100644 --- a/crates/tinydocs-module/src/lib.rs +++ b/crates/tinydocs-module/src/lib.rs @@ -4,8 +4,8 @@ //! the independently published `tinydocs` crate. Its `cdylib` output is the //! target-specific binary distributed in GitHub releases. -pub mod blobs; +pub mod outputs; mod service; -pub use blobs::{BlobError, BlobRef, BlobStore, hex_digest}; +pub use outputs::{OutputError, OutputRef, OutputStore, hex_digest}; pub use service::{BUS_NAME, OBJECT_PATH, WirePresentationSpec, WireSlideImage, WireSlideSpec}; diff --git a/crates/tinydocs-module/src/outputs/mod.rs b/crates/tinydocs-module/src/outputs/mod.rs new file mode 100644 index 0000000..3cbad4a --- /dev/null +++ b/crates/tinydocs-module/src/outputs/mod.rs @@ -0,0 +1,296 @@ +//! Holding a produced document until the caller has read it. +//! +//! # Why this exists at all +//! +//! Inbound payloads do not need it: `TinyBus` streams carry a `.pdf` or a slide +//! image alongside the method call, flow-controlled and bounded by the receiver's +//! own [`StreamLimits`](tinybus::stream::StreamLimits), and the transfer is tied +//! to the call that started it. +//! +//! Replies have no such thing. `Interface::call` receives a member name and a +//! JSON body — no caller identity, no connection — so a served object cannot open +//! a stream back to whoever called it. A generated `.docx` is therefore held here +//! and pulled in chunks, because the alternative is returning it inline through a +//! 16 MiB JSON frame where a `Vec` costs ~3.5 bytes per byte. +//! +//! That asymmetry is worth fixing upstream rather than working around forever: a +//! reply-stream seam in `TinyBus` would delete this module. +//! +//! # The bounds are the whole design +//! +//! A module is trusted in-process code that `TinyBus` never unloads, so anything +//! retained here is retained until the process exits unless something reclaims +//! it. A caller that asks for a document and then dies must not cost the host +//! that document forever. Hence a per-output cap, a total cap, a count cap, and +//! expiry of outputs nobody has read. +//! +//! Expiry is lazy — every operation sweeps first — so there is no background task +//! and no timer to reason about, and the clock is a parameter rather than a call +//! to [`Instant::now`], which is what makes the rules testable. + +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// Largest chunk a caller may read in one `ReadOutput`. +/// +/// Sized so the chunk plus its base64 expansion and the surrounding JSON stays +/// well inside a 16 MiB frame. +pub const MAX_CHUNK_BYTES: usize = 4 * 1024 * 1024; + +/// Largest single produced document. +pub const MAX_OUTPUT_BYTES: usize = 64 * 1024 * 1024; + +/// Largest total of all unread documents. +pub const MAX_TOTAL_BYTES: usize = 128 * 1024 * 1024; + +/// Most documents held unread at once. +/// +/// A separate bound from the byte budget: many small abandoned outputs are as +/// much of a leak as one large one. +pub const MAX_LIVE_OUTPUTS: usize = 32; + +/// How long an output may go unread before it is dropped. +pub const IDLE_TTL: Duration = Duration::from_secs(300); + +/// A handle to a produced document, and what a caller needs to read it back. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OutputRef { + /// Opaque identifier, valid until read and released or expired. + pub output_id: String, + /// Total size in bytes, so a caller knows when it is done. + pub total_bytes: u64, + /// Lowercase hex SHA-256, so a caller can verify what it assembled. + pub sha256: String, +} + +/// Why an output operation was refused. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum OutputError { + /// The produced document exceeds [`MAX_OUTPUT_BYTES`]. + #[error("document exceeds the {MAX_OUTPUT_BYTES}-byte per-output limit")] + OutputTooLarge, + + /// Holding it would exceed [`MAX_TOTAL_BYTES`]. + #[error("too many bytes are waiting to be read")] + StoreFull, + + /// [`MAX_LIVE_OUTPUTS`] documents are already waiting. + #[error("too many documents are waiting to be read")] + TooManyOutputs, + + /// No output with that id — read and released, or expired unread. + #[error("unknown output id")] + UnknownOutput, + + /// The requested chunk exceeds [`MAX_CHUNK_BYTES`]. + #[error("chunk exceeds the {MAX_CHUNK_BYTES}-byte per-chunk limit")] + ChunkTooLarge, + + /// The read started past the end of the document. + #[error("read offset is past the end of the document")] + ReadPastEnd, +} + +/// One produced document waiting to be read. +struct Output { + bytes: Vec, + last_read: Instant, +} + +/// Documents produced but not yet read. +#[derive(Default)] +pub struct OutputStore { + inner: Mutex, +} + +/// Reports how much is held, never what is held. +/// +/// Written by hand rather than derived: a derived implementation would put a +/// whole document into whatever formatted it. Documents are caller data. +impl std::fmt::Debug for OutputStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let inner = self.lock(); + f.debug_struct("OutputStore") + .field("live_outputs", &inner.outputs.len()) + .field("held_bytes", &inner.held_bytes()) + .finish() + } +} + +#[derive(Default)] +struct Inner { + outputs: HashMap, + next_id: u64, +} + +impl OutputStore { + /// An empty store. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Hold `bytes` and return the handle a caller reads them back with. + /// + /// # Errors + /// + /// [`OutputError::OutputTooLarge`], [`OutputError::TooManyOutputs`], or + /// [`OutputError::StoreFull`]. + pub fn insert(&self, bytes: Vec, now: Instant) -> Result { + if bytes.len() > MAX_OUTPUT_BYTES { + return Err(OutputError::OutputTooLarge); + } + + let mut inner = self.lock(); + inner.sweep_expired(now); + if inner.outputs.len() >= MAX_LIVE_OUTPUTS { + return Err(OutputError::TooManyOutputs); + } + if inner.held_bytes().saturating_add(bytes.len()) > MAX_TOTAL_BYTES { + return Err(OutputError::StoreFull); + } + + let sha256 = hex_digest(&bytes); + let total_bytes = bytes.len() as u64; + let output_id = inner.allocate_id(); + inner.outputs.insert( + output_id.clone(), + Output { + bytes, + last_read: now, + }, + ); + Ok(OutputRef { + output_id, + total_bytes, + sha256, + }) + } + + /// Read up to `len` bytes at `offset`. + /// + /// A read running past the end is clamped rather than refused, so a caller + /// can ask for a full chunk on the final read without computing the + /// remainder itself. + /// + /// # Errors + /// + /// [`OutputError::ChunkTooLarge`], [`OutputError::UnknownOutput`], or + /// [`OutputError::ReadPastEnd`]. + pub fn read_chunk( + &self, + output_id: &str, + offset: u64, + len: u64, + now: Instant, + ) -> Result, OutputError> { + let len = usize::try_from(len).map_err(|_| OutputError::ChunkTooLarge)?; + if len > MAX_CHUNK_BYTES { + return Err(OutputError::ChunkTooLarge); + } + + let mut inner = self.lock(); + inner.sweep_expired(now); + let output = inner + .outputs + .get_mut(output_id) + .ok_or(OutputError::UnknownOutput)?; + let start = usize::try_from(offset).map_err(|_| OutputError::ReadPastEnd)?; + if start > output.bytes.len() { + return Err(OutputError::ReadPastEnd); + } + // Reading is what keeps an output alive: a caller working through a + // large document in chunks must not have it reaped mid-read. + output.last_read = now; + let end = start.saturating_add(len).min(output.bytes.len()); + Ok(output.bytes[start..end].to_vec()) + } + + /// Drop an output and free its budget. + /// + /// # Errors + /// + /// [`OutputError::UnknownOutput`] if there is nothing to release, so a + /// caller learns its output had already expired rather than assuming it + /// tidied up. + pub fn release(&self, output_id: &str, now: Instant) -> Result<(), OutputError> { + let mut inner = self.lock(); + inner.sweep_expired(now); + inner + .outputs + .remove(output_id) + .map(|_| ()) + .ok_or(OutputError::UnknownOutput) + } + + /// Number of outputs currently held, for tests and diagnostics. + #[must_use] + pub fn live_count(&self) -> usize { + self.lock().outputs.len() + } + + /// Take the lock, recovering from a poisoned mutex. + /// + /// A panic under this lock can only have happened between two `HashMap` + /// operations, so the map is intact and the worst case is one stale output + /// that its TTL will reap. Refusing every later request would turn one + /// caller's panic into a dead module, and `TinyBus` never unloads a module + /// to recover. + fn lock(&self) -> std::sync::MutexGuard<'_, Inner> { + self.inner + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +impl Inner { + /// Total bytes held by every output. + fn held_bytes(&self) -> usize { + self.outputs + .values() + .map(|output| output.bytes.len()) + .fold(0usize, usize::saturating_add) + } + + /// Drop every output unread for longer than [`IDLE_TTL`]. + fn sweep_expired(&mut self, now: Instant) { + self.outputs + .retain(|_, output| now.saturating_duration_since(output.last_read) <= IDLE_TTL); + } + + /// Allocate an unused output id. + /// + /// A counter, not a random value: ids are opaque handles inside one process, + /// never authorisation tokens, and a counter makes a leaked id visible in a + /// log rather than looking like a secret. + fn allocate_id(&mut self) -> String { + self.next_id = self.next_id.wrapping_add(1); + format!("out-{}", self.next_id) + } +} + +/// Lowercase hex SHA-256 of `bytes`. +/// +/// Public because a caller verifies what it assembled against +/// [`OutputRef::sha256`], and one implementation both sides agree on beats two +/// that can disagree about case. +#[must_use] +pub fn hex_digest(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let mut out = String::with_capacity(64); + for byte in digest { + use std::fmt::Write as _; + // Writing into a String cannot fail; the result is discarded rather than + // unwrapped so this stays panic-free. + let _ = write!(out, "{byte:02x}"); + } + out +} + +#[cfg(test)] +mod test; diff --git a/crates/tinydocs-module/src/outputs/test.rs b/crates/tinydocs-module/src/outputs/test.rs new file mode 100644 index 0000000..c95f149 --- /dev/null +++ b/crates/tinydocs-module/src/outputs/test.rs @@ -0,0 +1,232 @@ +//! Unit tests for the produced-document store. +//! +//! Weighted towards refusals and expiry. Holding bytes and handing them back is +//! the easy half; what decides whether a module that is never unloaded leaks is +//! the four bounds and the TTL. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::time::{Duration, Instant}; + +use super::{ + IDLE_TTL, MAX_CHUNK_BYTES, MAX_LIVE_OUTPUTS, MAX_OUTPUT_BYTES, MAX_TOTAL_BYTES, OutputError, + OutputStore, hex_digest, +}; + +fn t0() -> Instant { + Instant::now() +} + +#[test] +fn an_output_round_trips_through_chunks() { + let store = OutputStore::new(); + let now = t0(); + let document: Vec = (0..10_000u32).map(|i| (i % 251) as u8).collect(); + + let handle = store.insert(document.clone(), now).unwrap(); + assert_eq!(handle.total_bytes, document.len() as u64); + assert_eq!(handle.sha256, hex_digest(&document)); + + let mut read = Vec::new(); + while (read.len() as u64) < handle.total_bytes { + let chunk = store + .read_chunk(&handle.output_id, read.len() as u64, 3_000, now) + .unwrap(); + assert!(!chunk.is_empty(), "read stalled at {}", read.len()); + read.extend_from_slice(&chunk); + } + assert_eq!(read, document); +} + +#[test] +fn a_read_past_the_end_is_clamped_not_refused() { + // So a caller can ask for a whole chunk on the final read. + let store = OutputStore::new(); + let now = t0(); + let handle = store.insert(b"twelve bytes".to_vec(), now).unwrap(); + let chunk = store + .read_chunk(&handle.output_id, 6, 1_000_000, now) + .unwrap(); + assert_eq!(chunk, b" bytes"); +} + +#[test] +fn a_read_starting_past_the_end_is_refused() { + let store = OutputStore::new(); + let now = t0(); + let handle = store.insert(b"short".to_vec(), now).unwrap(); + assert_eq!( + store.read_chunk(&handle.output_id, 99, 10, now), + Err(OutputError::ReadPastEnd) + ); +} + +#[test] +fn an_oversize_read_is_refused() { + let store = OutputStore::new(); + let now = t0(); + let handle = store.insert(b"small".to_vec(), now).unwrap(); + assert_eq!( + store.read_chunk(&handle.output_id, 0, MAX_CHUNK_BYTES as u64 + 1, now), + Err(OutputError::ChunkTooLarge) + ); + assert_eq!( + store.read_chunk(&handle.output_id, 0, u64::MAX, now), + Err(OutputError::ChunkTooLarge) + ); +} + +#[test] +fn an_oversize_document_is_refused() { + let store = OutputStore::new(); + assert_eq!( + store.insert(vec![0u8; MAX_OUTPUT_BYTES + 1], t0()), + Err(OutputError::OutputTooLarge) + ); + assert_eq!(store.live_count(), 0); +} + +#[test] +fn the_total_budget_is_enforced() { + let store = OutputStore::new(); + let now = t0(); + let half = MAX_TOTAL_BYTES / 2; + // Two at the per-output cap fill the store, because the per-output cap is + // half the total. + store.insert(vec![1u8; half], now).unwrap(); + store.insert(vec![2u8; half], now).unwrap(); + assert_eq!( + store.insert(vec![3u8; 16], now), + Err(OutputError::StoreFull) + ); +} + +#[test] +fn too_many_live_outputs_is_refused() { + let store = OutputStore::new(); + let now = t0(); + for _ in 0..MAX_LIVE_OUTPUTS { + store.insert(b"x".to_vec(), now).unwrap(); + } + assert_eq!(store.live_count(), MAX_LIVE_OUTPUTS); + assert_eq!( + store.insert(b"one more".to_vec(), now), + Err(OutputError::TooManyOutputs) + ); +} + +#[test] +fn an_unread_output_expires_and_frees_its_budget() { + // The bound that matters: a caller that asks for a document and then dies + // must not cost the host that document for the life of the process. + let store = OutputStore::new(); + let now = t0(); + let handle = store.insert(vec![7u8; 1_000], now).unwrap(); + + // Alive right on the boundary. + let at_ttl = now + IDLE_TTL; + assert!(store.read_chunk(&handle.output_id, 0, 10, at_ttl).is_ok()); + + // Past it, measured from the last read, the next operation sweeps it away. + let past_ttl = at_ttl + IDLE_TTL + Duration::from_secs(1); + assert_eq!( + store.read_chunk(&handle.output_id, 0, 10, past_ttl), + Err(OutputError::UnknownOutput) + ); + assert_eq!(store.live_count(), 0); +} + +#[test] +fn reading_keeps_a_slow_consumer_alive() { + // The flip side: a caller working through a large document in chunks must + // never have it reaped mid-read. + let store = OutputStore::new(); + let mut now = t0(); + let document = vec![4u8; 400]; + let handle = store.insert(document.clone(), now).unwrap(); + + let mut read = Vec::new(); + while (read.len() as u64) < handle.total_bytes { + // Each read lands just inside the window; four of them sum to well past + // the TTL. + now += IDLE_TTL.saturating_sub(Duration::from_secs(1)); + let chunk = store + .read_chunk(&handle.output_id, read.len() as u64, 100, now) + .expect("a document being read must not expire"); + read.extend_from_slice(&chunk); + } + assert_eq!(read, document); +} + +#[test] +fn releasing_frees_the_budget_and_is_reported_once() { + let store = OutputStore::new(); + let now = t0(); + let handle = store.insert(b"payload".to_vec(), now).unwrap(); + assert!(store.release(&handle.output_id, now).is_ok()); + assert_eq!(store.live_count(), 0); + // A second release says the output is gone rather than pretending. + assert_eq!( + store.release(&handle.output_id, now), + Err(OutputError::UnknownOutput) + ); +} + +#[test] +fn unknown_ids_are_refused_by_every_operation() { + let store = OutputStore::new(); + let now = t0(); + assert_eq!( + store.read_chunk("nope", 0, 1, now), + Err(OutputError::UnknownOutput) + ); + assert_eq!(store.release("nope", now), Err(OutputError::UnknownOutput)); +} + +#[test] +fn ids_are_not_recycled_after_a_release() { + // A caller holding a stale id would otherwise read somebody else's document. + let store = OutputStore::new(); + let now = t0(); + let first = store.insert(b"one".to_vec(), now).unwrap(); + store.release(&first.output_id, now).unwrap(); + let second = store.insert(b"two".to_vec(), now).unwrap(); + assert_ne!(first.output_id, second.output_id); +} + +#[test] +fn an_empty_document_is_held_and_read_as_empty() { + // `ExtractText` on a scanned PDF produces exactly this. + let store = OutputStore::new(); + let now = t0(); + let handle = store.insert(Vec::new(), now).unwrap(); + assert_eq!(handle.total_bytes, 0); + assert_eq!(handle.sha256, hex_digest(b"")); + assert_eq!( + store.read_chunk(&handle.output_id, 0, 10, now).unwrap(), + Vec::::new() + ); +} + +#[test] +fn the_digest_matches_a_known_vector() { + assert_eq!( + hex_digest(b""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); +} + +#[test] +fn debug_reports_sizes_not_contents() { + // A derived Debug would put a whole document into a log line. + let store = OutputStore::new(); + store + .insert(b"secret contract text".to_vec(), t0()) + .unwrap(); + let rendered = format!("{store:?}"); + assert!(rendered.contains("live_outputs")); + assert!( + !rendered.contains("secret"), + "document contents leaked into Debug: {rendered}" + ); +} diff --git a/crates/tinydocs-module/src/service/mod.rs b/crates/tinydocs-module/src/service/mod.rs index 1bd6566..a291d2c 100644 --- a/crates/tinydocs-module/src/service/mod.rs +++ b/crates/tinydocs-module/src/service/mod.rs @@ -1,49 +1,49 @@ //! `TinyBus` service boundary for the document surface. //! -//! One object, `/ai/tinyhumans/tinydocs/Documents`, exporting the three format -//! operations plus the four chunked-transfer operations they depend on: +//! One object, `/ai/tinyhumans/tinydocs/Documents`, exporting five methods: //! //! ```text -//! BeginBlob(total_bytes, sha256) -> blob_id -//! PutChunk(blob_id, offset, base64) -> bytes received so far -//! GetChunk(blob_id, offset, len) -> base64 -//! ReleaseBlob(blob_id) -> () -//! GenerateDocx(DocumentSpec) -> BlobRef -//! GeneratePptx(WirePresentationSpec) -> BlobRef -//! ExtractText(blob_id) -> BlobRef +//! GenerateDocx(DocumentSpec) -> OutputRef +//! GeneratePptx(WirePresentationSpec, Option) -> OutputRef +//! ExtractText(StreamRef) -> OutputRef +//! ReadOutput(output_id, offset, len) -> base64 +//! ReleaseOutput(output_id) -> () //! ``` //! -//! # Why everything returns a `BlobRef` +//! # Payloads in and payloads out are not symmetric //! -//! See [`crate::blobs`]. A `TinyBus` frame is a 16 MiB JSON document and -//! `Vec` serialises as an array of integers, so the real inline ceiling is a -//! few megabytes — below a deck's legal image payload and below any `.pdf` worth -//! extracting. Rather than have some methods return bytes inline and others not, -//! every unbounded result is staged and read back in chunks. The caller's code -//! path is then the same regardless of size. +//! Inbound bytes ride a `TinyBus` stream: the caller opens one alongside the +//! method call, writes while the call is outstanding, and the module reads it. +//! Flow control, the size cap, the idle timeout and the "only the peer that +//! opened it may write" rule are all the bus's, which is why nothing in this +//! crate re-implements them. //! -//! # This replaces the `Docx` interface rather than extending it +//! Replies cannot do that. `Interface::call` gets a member name and a JSON body — +//! no caller identity, no connection — so a served object cannot open a stream +//! back to whoever called it. A produced document is therefore held in +//! [`crate::outputs`] and pulled with `ReadOutput`, because returning it inline +//! would put it through a 16 MiB JSON frame where a `Vec` costs about 3.5 +//! bytes per byte. A reply-stream seam in `TinyBus` would remove that half. //! -//! The previous interface, `ai.tinyhumans.tinydocs.Docx`, returned -//! `GenerateDocx(DocumentSpec) -> Vec` inline. `TinyBus`'s module guidance is -//! explicit that an existing interface must not change in place — a breaking -//! contract gets a new interface name — and returning a `BlobRef` where callers -//! expect bytes is exactly that. Hence a new name. +//! # Slide images arrive as one stream //! -//! The old interface is retired rather than served alongside, because -//! `module_export!` attaches its `methods` list to the *first* entry in -//! `provides` and leaves any others with an empty method list. A second -//! fully-declared interface is therefore not expressible today, and a manifest -//! that under-declares its members would break the invariant that manifest -//! methods and dispatch members stay identical. Serving both needs a `TinyBus` -//! change first; retiring one at a pre-1.0 minor bump does not. +//! A deck can carry several images, and a call has one stream. Rather than stage +//! each image separately, the wire spec gives every image a `byte_len` and the +//! images are concatenated into a single stream in slide order; the module splits +//! them back apart. The lengths are part of the spec, so a truncated or +//! over-long stream is a named rejection rather than a deck with a corrupt +//! picture in it. //! -//! # Runtime +//! # This replaces the `Docx` interface rather than extending it //! -//! Synthesis and extraction are CPU-bound and run on the module runtime's -//! blocking pool. The blob operations are memory copies under a short lock and -//! run inline. The module holds no document state between calls — only staged -//! blobs, every one of them bounded and expiring. +//! The previous interface returned `GenerateDocx(DocumentSpec) -> Vec` +//! inline. `TinyBus` is explicit that an existing interface must not change in +//! place, and returning a handle where callers expect bytes is exactly that. +//! +//! The old interface is retired rather than served beside the new one because +//! `module_export!` attaches its `methods` list to the *first* entry in +//! `provides` and leaves any others empty, so a second fully-declared interface +//! is not expressible today. mod wire; @@ -52,11 +52,12 @@ use std::time::Instant; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; +use tinybus::stream::StreamRef; use tinybus::{Connection, Error as BusError, Result as BusResult}; use tinydocs::spec::{DocumentSpec, PresentationSpec, SlideImage, SlideSpec}; use tinydocs::{Error, pdf, pptx}; -use crate::blobs::{BlobError, BlobRef, BlobStore}; +use crate::outputs::{OutputError, OutputRef, OutputStore}; pub use wire::{WirePresentationSpec, WireSlideImage, WireSlideSpec}; @@ -71,124 +72,159 @@ const GENERATION_FAILED_ERROR: &str = "ai.tinyhumans.tinydocs.Error.GenerationFa const EXTRACTION_FAILED_ERROR: &str = "ai.tinyhumans.tinydocs.Error.ExtractionFailed"; const MODULE_FAILED_ERROR: &str = "ai.tinyhumans.tinydocs.Error.ModuleFailed"; const TRANSFER_FAILED_ERROR: &str = "ai.tinyhumans.tinydocs.Error.TransferFailed"; -const TRANSFER_REFUSED_ERROR: &str = "ai.tinyhumans.tinydocs.Error.TransferRefused"; -const UNKNOWN_BLOB_ERROR: &str = "ai.tinyhumans.tinydocs.Error.UnknownBlob"; +const OUTPUT_REFUSED_ERROR: &str = "ai.tinyhumans.tinydocs.Error.OutputRefused"; +const UNKNOWN_OUTPUT_ERROR: &str = "ai.tinyhumans.tinydocs.Error.UnknownOutput"; -/// The served object. Owns the staging area; holds no document state. +/// The served object. +/// +/// Holds the connection, because reading an inbound stream needs one, and the +/// produced-document store. No document state survives a call. struct Documents { - blobs: Arc, + connection: Connection, + outputs: Arc, } -// The interface macro rejects a non-async method outright, so the four transfer -// methods below are async because the dispatch contract says so, not because they -// await anything. `unused_async` can therefore never be actionable in this block. +// The interface macro rejects a non-async method outright, so the two output +// methods below are async because the dispatch contract says so, not because +// they await anything. `unused_async` can never be actionable in this block. #[allow( clippy::unused_async, reason = "tinybus::interface requires every method to be `async fn`" )] #[tinybus::interface(name = "ai.tinyhumans.tinydocs.Documents")] impl Documents { - /// Reserve space for a blob of `total_bytes` that will hash to `sha256`. - async fn begin_blob(&self, total_bytes: u64, sha256: String) -> BusResult { - self.blobs - .begin(total_bytes, &sha256, Instant::now()) - .map_err(|error| map_blob_error(&error)) - } - - /// Append a base64 chunk at `offset`, returning bytes received so far. - async fn put_chunk(&self, blob_id: String, offset: u64, data: String) -> BusResult { - let decoded = decode_base64(&data)?; - self.blobs - .put_chunk(&blob_id, offset, &decoded, Instant::now()) - .map_err(|error| map_blob_error(&error)) - } - - /// Read up to `len` bytes of a complete blob at `offset`, base64-encoded. - async fn get_chunk(&self, blob_id: String, offset: u64, len: u64) -> BusResult { - let bytes = self - .blobs - .get_chunk(&blob_id, offset, len, Instant::now()) - .map_err(|error| map_blob_error(&error))?; - Ok(BASE64.encode(bytes)) - } - - /// Drop a blob and free its budget. - async fn release_blob(&self, blob_id: String) -> BusResult<()> { - self.blobs - .release(&blob_id, Instant::now()) - .map_err(|error| map_blob_error(&error)) - } - - /// Generate a `.docx` and stage it for reading. - async fn generate_docx(&self, spec: DocumentSpec) -> BusResult { + /// Generate a `.docx` and hold it for reading. + async fn generate_docx(&self, spec: DocumentSpec) -> BusResult { // Validated on this thread, before a blocking slot is taken: rejecting a - // malformed spec should not have to queue behind real work. + // malformed spec should not queue behind real work. spec.validate().map_err(|error| map_error(&error))?; let bytes = blocking(move || tinydocs::docx::generate(&spec)).await?; - self.stage(bytes) + self.hold(bytes) } - /// Generate a `.pptx` from a spec whose images name staged blobs. - async fn generate_pptx(&self, spec: WirePresentationSpec) -> BusResult { - let resolved = self.resolve_presentation(spec)?; + /// Generate a `.pptx`, reading its images from one concatenated stream. + async fn generate_pptx( + &self, + spec: WirePresentationSpec, + images: Option, + ) -> BusResult { + let resolved = self.resolve_presentation(spec, images).await?; resolved.validate().map_err(|error| map_error(&error))?; let bytes = blocking(move || pptx::generate(&resolved)).await?; - self.stage(bytes) + self.hold(bytes) } - /// Extract the text layer of a staged `.pdf` and stage the result. - async fn extract_text(&self, blob_id: String) -> BusResult { - // Taken rather than copied: the document is often the largest thing in - // the staging area, and holding it through extraction as well would - // double its cost for no reason. - let bytes = self - .blobs - .take_complete(&blob_id, Instant::now()) - .map_err(|error| map_blob_error(&error))?; + /// Extract the text layer of a streamed `.pdf` and hold the result. + async fn extract_text(&self, document: StreamRef) -> BusResult { + let bytes = self.read_stream(&document).await?; let text = blocking(move || pdf::extract_text(&bytes)).await?; - self.stage(text.into_bytes()) + self.hold(text.into_bytes()) + } + + /// Read up to `len` bytes of a held document at `offset`, base64-encoded. + async fn read_output(&self, output_id: String, offset: u64, len: u64) -> BusResult { + let bytes = self + .outputs + .read_chunk(&output_id, offset, len, Instant::now()) + .map_err(|error| map_output_error(&error))?; + Ok(BASE64.encode(bytes)) + } + + /// Drop a held document and free its budget. + async fn release_output(&self, output_id: String) -> BusResult<()> { + self.outputs + .release(&output_id, Instant::now()) + .map_err(|error| map_output_error(&error)) } } impl Documents { - /// Stage a produced payload and return its handle. - fn stage(&self, bytes: Vec) -> BusResult { - self.blobs - .insert_complete(bytes, Instant::now()) - .map_err(|error| map_blob_error(&error)) + /// Hold a produced document and return its handle. + fn hold(&self, bytes: Vec) -> BusResult { + self.outputs + .insert(bytes, Instant::now()) + .map_err(|error| map_output_error(&error)) } - /// Turn a wire deck into a real [`PresentationSpec`] by consuming the blobs - /// its images name. + /// Read a whole inbound stream into memory. /// - /// Images are taken from the staging area, so a deck's bytes stop being - /// charged twice the moment they are resolved. A blob that is missing or - /// incomplete fails the whole call rather than silently dropping a slide's - /// image — the caller staged it, so its absence is a transfer bug worth - /// reporting, not a degraded deck. - fn resolve_presentation(&self, spec: WirePresentationSpec) -> BusResult { - let now = Instant::now(); + /// The bus enforces the size cap, the flow-control window and the idle + /// timeout; a failure here is a transfer that did not complete. + async fn read_stream(&self, stream: &StreamRef) -> BusResult> { + self.connection + .read_stream(stream) + .await + .map_err(|error| BusError::MethodFailed { + name: TRANSFER_FAILED_ERROR.to_string(), + // The bus's own message, which never carries payload bytes. + message: error.to_string(), + }) + } + + /// Turn a wire deck plus one concatenated image stream into a real spec. + /// + /// The spec's `byte_len` values are the authority on where each image ends. + /// A stream that does not add up to their sum is refused rather than sliced + /// into whatever happens to be there: the alternative is a deck containing a + /// picture assembled from two different images. + async fn resolve_presentation( + &self, + spec: WirePresentationSpec, + images: Option, + ) -> BusResult { + let expected: u64 = spec + .slides + .iter() + .flat_map(|slide| slide.images.iter()) + .map(|image| image.byte_len) + .sum(); + + let payload = match (&images, expected) { + (Some(stream), _) => self.read_stream(stream).await?, + // No stream is only coherent with no images. + (None, 0) => Vec::new(), + (None, _) => { + return Err(BusError::MethodFailed { + name: INVALID_INPUT_ERROR.to_string(), + message: "the deck declares images but no image stream was opened".to_string(), + }); + } + }; + if payload.len() as u64 != expected { + return Err(BusError::MethodFailed { + name: INVALID_INPUT_ERROR.to_string(), + message: format!( + "image stream carried {} bytes but the deck declares {expected}", + payload.len() + ), + }); + } + + let mut cursor = 0usize; let mut slides = Vec::with_capacity(spec.slides.len()); for slide in spec.slides { - let mut images = Vec::with_capacity(slide.images.len()); + let mut resolved = Vec::with_capacity(slide.images.len()); for image in slide.images { - let bytes = self - .blobs - .take_complete(&image.blob_id, now) - .map_err(|error| map_blob_error(&error))?; - images.push( - SlideImage::from_bytes(bytes, image.caption) + let len = usize::try_from(image.byte_len).map_err(|_| BusError::MethodFailed { + name: INVALID_INPUT_ERROR.to_string(), + message: "image length is out of range".to_string(), + })?; + let end = cursor + len; + resolved.push( + SlideImage::from_bytes(payload[cursor..end].to_vec(), image.caption) .map_err(|error| map_error(&error))?, ); + cursor = end; } slides.push(SlideSpec { title: slide.title, body: slide.body, bullets: slide.bullets, speaker_notes: slide.speaker_notes, - images, + images: resolved, }); } + Ok(PresentationSpec { title: spec.title, author: spec.author, @@ -213,16 +249,6 @@ where .map_err(|error| map_error(&error)) } -/// Decode a base64 chunk, refusing malformed input by name. -fn decode_base64(data: &str) -> BusResult> { - BASE64.decode(data).map_err(|_| BusError::MethodFailed { - name: INVALID_INPUT_ERROR.to_string(), - // The payload itself is never echoed: it is caller data, and an error - // message is the wrong place for it. - message: "chunk data is not valid base64".to_string(), - }) -} - /// Map a library error onto its wire name. fn map_error(error: &Error) -> BusError { let name = match error { @@ -237,24 +263,19 @@ fn map_error(error: &Error) -> BusError { } } -/// Map a staging failure onto its wire name. +/// Map an output-store failure onto its wire name. /// -/// Three names rather than one, because the caller's correct response differs. -/// `UnknownBlob` means the transfer is gone and has to restart; `TransferRefused` -/// means a budget is full and retrying later may work; `TransferFailed` means the -/// caller sent something wrong and should re-send. -fn map_blob_error(error: &BlobError) -> BusError { +/// Grouped by what the caller should do next: `UnknownOutput` means the document +/// is gone and the call has to be made again, `OutputRefused` means the store is +/// full and the same request may succeed later, `TransferFailed` means the read +/// itself was malformed. +fn map_output_error(error: &OutputError) -> BusError { let name = match *error { - BlobError::UnknownBlob => UNKNOWN_BLOB_ERROR, - BlobError::StagingFull | BlobError::TooManyBlobs => TRANSFER_REFUSED_ERROR, - BlobError::MalformedDigest - | BlobError::BlobTooLarge - | BlobError::ChunkTooLarge - | BlobError::OutOfOrderChunk { .. } - | BlobError::OverlongBlob - | BlobError::DigestMismatch - | BlobError::IncompleteBlob - | BlobError::ReadPastEnd => TRANSFER_FAILED_ERROR, + OutputError::UnknownOutput => UNKNOWN_OUTPUT_ERROR, + OutputError::StoreFull | OutputError::TooManyOutputs | OutputError::OutputTooLarge => { + OUTPUT_REFUSED_ERROR + } + OutputError::ChunkTooLarge | OutputError::ReadPastEnd => TRANSFER_FAILED_ERROR, }; BusError::MethodFailed { name: name.to_string(), @@ -263,13 +284,12 @@ fn map_blob_error(error: &BlobError) -> BusError { } async fn setup(connection: Connection) -> BusResult<()> { + let documents = Documents { + connection: connection.clone(), + outputs: Arc::new(OutputStore::new()), + }; connection - .serve_at( - OBJECT_PATH.try_into()?, - Documents { - blobs: Arc::new(BlobStore::new()), - }, - ) + .serve_at(OBJECT_PATH.try_into()?, documents) .await?; connection.request_name(BUS_NAME).await?; Ok(()) @@ -289,13 +309,11 @@ mod exports { worker_threads = 2, provides = ["ai.tinyhumans.tinydocs.Documents"], methods = [ - "BeginBlob", - "PutChunk", - "GetChunk", - "ReleaseBlob", "GenerateDocx", "GeneratePptx", "ExtractText", + "ReadOutput", + "ReleaseOutput", ], signals = [], requires = [], diff --git a/crates/tinydocs-module/src/service/test.rs b/crates/tinydocs-module/src/service/test.rs index dd69a6c..2a23516 100644 --- a/crates/tinydocs-module/src/service/test.rs +++ b/crates/tinydocs-module/src/service/test.rs @@ -1,35 +1,42 @@ //! Unit tests for the `TinyBus` service declaration. //! -//! The manifest and the generated dispatch table are two lists that have to stay -//! identical, and nothing but a test connects them: the macro takes the method -//! names as string literals, so a method added to the `impl` without a matching -//! literal is admitted by the loader and then fails to dispatch. That is the -//! invariant this file exists for. +//! The manifest and the generated dispatch table are two lists that must stay +//! identical, and nothing but a test connects them: the macro takes method names +//! as string literals, so a method added to the `impl` without a matching literal +//! is admitted by the loader and then fails to dispatch. //! -//! Bytes moving over a real broker is covered by `tests/module_e2e.rs`, which -//! loads the built artifact through the actual dynamic loader. +//! The streaming paths are exercised in `tests/module_e2e.rs`, over a real +//! broker. A stream needs two connected peers, so there is no honest way to +//! unit-test one against a bare struct. #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use tinybus::Interface; use super::*; -use crate::blobs::hex_digest; +use crate::outputs::hex_digest; /// The methods the manifest declares, in declaration order. const DECLARED_METHODS: &[&str] = &[ - "BeginBlob", - "PutChunk", - "GetChunk", - "ReleaseBlob", "GenerateDocx", "GeneratePptx", "ExtractText", + "ReadOutput", + "ReleaseOutput", ]; -fn service() -> Documents { +/// A service attached to a broker nothing else is on. +/// +/// Enough for every method that does not read a stream. +async fn service() -> Documents { + let bus = tinybus::transport::memory::MemoryBus::new(); + tinybus::broker::Broker::new().spawn(bus.clone()); + let connection = Connection::connect(bus.connect().await.unwrap()) + .await + .unwrap(); Documents { - blobs: Arc::new(BlobStore::new()), + connection, + outputs: Arc::new(OutputStore::new()), } } @@ -44,8 +51,19 @@ fn service_identity_is_valid() { } #[test] -fn dispatch_members_match_the_manifest_exactly() { +fn every_declared_method_name_is_a_valid_member_name() { + for method in DECLARED_METHODS { + assert!( + tinybus::MemberName::new(*method).is_ok(), + "{method} is not a valid member name" + ); + } +} + +#[tokio::test] +async fn dispatch_members_match_the_manifest_exactly() { let members: Vec = service() + .await .members() .iter() .map(|member| member.as_str().to_string()) @@ -57,16 +75,6 @@ fn dispatch_members_match_the_manifest_exactly() { ); } -#[test] -fn every_declared_method_name_is_a_valid_member_name() { - for method in DECLARED_METHODS { - assert!( - tinybus::MemberName::new(*method).is_ok(), - "{method} is not a valid member name" - ); - } -} - #[test] fn library_errors_keep_distinct_wire_names() { assert_eq!( @@ -84,355 +92,181 @@ fn library_errors_keep_distinct_wire_names() { } #[test] -fn transfer_errors_are_grouped_by_what_the_caller_should_do() { - // Gone: restart the transfer. +fn output_errors_are_grouped_by_what_the_caller_should_do() { + // Gone: make the call again. assert_eq!( - map_blob_error(&BlobError::UnknownBlob).wire_name(), - UNKNOWN_BLOB_ERROR + map_output_error(&OutputError::UnknownOutput).wire_name(), + UNKNOWN_OUTPUT_ERROR ); // Full: the same request may succeed later. - for refused in [BlobError::StagingFull, BlobError::TooManyBlobs] { + for refused in [ + OutputError::StoreFull, + OutputError::TooManyOutputs, + OutputError::OutputTooLarge, + ] { assert_eq!( - map_blob_error(&refused).wire_name(), - TRANSFER_REFUSED_ERROR, - "{refused:?} should be retryable" + map_output_error(&refused).wire_name(), + OUTPUT_REFUSED_ERROR, + "{refused:?} should read as retryable" ); } - // Caller error: re-send, do not retry verbatim. - for failed in [ - BlobError::MalformedDigest, - BlobError::BlobTooLarge, - BlobError::ChunkTooLarge, - BlobError::OutOfOrderChunk { - expected: 1, - actual: 2, - }, - BlobError::OverlongBlob, - BlobError::DigestMismatch, - BlobError::IncompleteBlob, - BlobError::ReadPastEnd, - ] { + // Malformed read: fix the request. + for failed in [OutputError::ChunkTooLarge, OutputError::ReadPastEnd] { assert_eq!( - map_blob_error(&failed).wire_name(), + map_output_error(&failed).wire_name(), TRANSFER_FAILED_ERROR, - "{failed:?} should not be reported as retryable" + "{failed:?} should not read as retryable" ); } } -#[test] -fn malformed_base64_is_an_invalid_input_and_does_not_echo_the_payload() { - let err = decode_base64("this is not base64!!").expect_err("should reject"); - assert_eq!(err.wire_name(), INVALID_INPUT_ERROR); - assert!( - !format!("{err}").contains("not base64!!"), - "the rejected payload leaked into the error message: {err}" - ); -} - -#[test] -fn valid_base64_decodes() { - assert_eq!( - decode_base64(&BASE64.encode(b"round trip")).unwrap(), - b"round trip" - ); - assert_eq!(decode_base64("").unwrap(), Vec::::new()); -} - #[tokio::test] -async fn generate_docx_stages_a_readable_document() { +async fn generate_docx_holds_a_readable_document() { use tinydocs::spec::DocumentSection; - let service = service(); - let spec = DocumentSpec { - title: "Charter".to_string(), - author: Some("Alice".to_string()), - sections: vec![DocumentSection { - heading: Some("Goals".to_string()), - paragraphs: vec!["Ship it.".to_string()], - bullets: vec![], - }], - }; - - let handle = service.generate_docx(spec).await.expect("should generate"); + let service = service().await; + let handle = service + .generate_docx(DocumentSpec { + title: "Charter".to_string(), + author: Some("Alice".to_string()), + sections: vec![DocumentSection { + heading: Some("Goals".to_string()), + paragraphs: vec!["Ship it.".to_string()], + bullets: vec![], + }], + }) + .await + .expect("should generate"); assert!(handle.total_bytes > 0); - let bytes = service - .blobs - .get_chunk(&handle.blob_id, 0, handle.total_bytes, Instant::now()) - .expect("staged output should be readable"); + let encoded = service + .read_output(handle.output_id.clone(), 0, handle.total_bytes) + .await + .expect("held output should be readable"); + let bytes = BASE64.decode(encoded).expect("output is base64"); assert_eq!(&bytes[..2], b"PK", "a .docx is a zip container"); assert_eq!(hex_digest(&bytes), handle.sha256); + + service + .release_output(handle.output_id.clone()) + .await + .expect("release should succeed"); + assert!( + service.release_output(handle.output_id).await.is_err(), + "releasing twice should report the output is gone" + ); } #[tokio::test] -async fn generate_docx_rejects_an_invalid_spec_without_staging_anything() { - let service = service(); - let spec = DocumentSpec { - title: String::new(), - author: None, - sections: vec![], - }; +async fn generate_docx_rejects_an_invalid_spec_without_holding_anything() { + let service = service().await; let err = service - .generate_docx(spec) + .generate_docx(DocumentSpec { + title: String::new(), + author: None, + sections: vec![], + }) .await .expect_err("a blank title should be rejected"); assert_eq!(err.wire_name(), INVALID_INPUT_ERROR); assert_eq!( - service.blobs.live_count(), + service.outputs.live_count(), 0, - "a rejected call must not leave a blob behind" + "a rejected call must not leave an output behind" ); } #[tokio::test] -async fn generate_pptx_consumes_the_image_blobs_it_is_given() { - let service = service(); - let now = Instant::now(); - let png = tiny_png(); - - // Stage an image the way a caller would, then reference it by id. - let blob_id = service - .blobs - .begin(png.len() as u64, &hex_digest(&png), now) - .unwrap(); - service.blobs.put_chunk(&blob_id, 0, &png, now).unwrap(); - assert_eq!(service.blobs.live_count(), 1); - +async fn a_deck_with_no_images_needs_no_stream() { + // The common case. Requiring a caller to open an empty stream to render a + // text-only deck would be a pointless round trip. + let service = service().await; let handle = service - .generate_pptx(WirePresentationSpec { - title: "Quarterly".to_string(), - author: None, - theme: None, - slides: vec![WireSlideSpec { - title: "With a chart".to_string(), - body: None, - bullets: vec![], - speaker_notes: None, - images: vec![WireSlideImage { - blob_id: blob_id.clone(), - caption: Some("A chart".to_string()), + .generate_pptx( + WirePresentationSpec { + title: "Quarterly".to_string(), + author: None, + theme: None, + slides: vec![WireSlideSpec { + title: "Text only".to_string(), + body: Some("No pictures.".to_string()), + bullets: vec![], + speaker_notes: None, + images: vec![], }], - }], - }) + }, + None, + ) .await .expect("should generate"); - let bytes = service - .blobs - .get_chunk(&handle.blob_id, 0, handle.total_bytes, Instant::now()) - .expect("staged deck should be readable"); + let encoded = service + .read_output(handle.output_id, 0, handle.total_bytes) + .await + .unwrap(); + let bytes = BASE64.decode(encoded).unwrap(); assert_eq!(&bytes[..2], b"PK", "a .pptx is a zip container"); - - // The image blob was taken, not copied: only the output remains staged. - assert_eq!( - service.blobs.live_count(), - 1, - "the consumed image blob should have been released" - ); - assert!( - service - .blobs - .get_chunk(&blob_id, 0, 1, Instant::now()) - .is_err() - ); } #[tokio::test] -async fn generate_pptx_reports_a_missing_image_blob_rather_than_dropping_the_image() { - // The caller staged it, so its absence is a transfer bug worth reporting — - // not a deck quietly missing a slide's illustration. - let service = service(); +async fn a_deck_declaring_images_without_a_stream_is_refused() { + // The spec and the transfer have to agree. Rendering the deck without the + // pictures it asked for would be a silently wrong document. + let service = service().await; let err = service - .generate_pptx(WirePresentationSpec { - title: "Quarterly".to_string(), - author: None, - theme: None, - slides: vec![WireSlideSpec { - title: "With a chart".to_string(), - body: None, - bullets: vec![], - speaker_notes: None, - images: vec![WireSlideImage { - blob_id: "blob-does-not-exist".to_string(), - caption: None, + .generate_pptx( + WirePresentationSpec { + title: "Quarterly".to_string(), + author: None, + theme: None, + slides: vec![WireSlideSpec { + title: "With a chart".to_string(), + body: None, + bullets: vec![], + speaker_notes: None, + images: vec![WireSlideImage { + byte_len: 128, + caption: None, + }], }], - }], - }) + }, + None, + ) .await - .expect_err("a missing image blob should fail the call"); - assert_eq!(err.wire_name(), UNKNOWN_BLOB_ERROR); + .expect_err("a declared image with no stream should be refused"); + assert_eq!(err.wire_name(), INVALID_INPUT_ERROR); } #[tokio::test] -async fn generate_pptx_rejects_image_bytes_that_are_not_an_embeddable_image() { - let service = service(); - let now = Instant::now(); - let junk = b"definitely not a png".to_vec(); - let blob_id = service - .blobs - .begin(junk.len() as u64, &hex_digest(&junk), now) - .unwrap(); - service.blobs.put_chunk(&blob_id, 0, &junk, now).unwrap(); - +async fn reading_an_unknown_output_is_refused_by_name() { + let service = service().await; let err = service - .generate_pptx(WirePresentationSpec { - title: "Quarterly".to_string(), - author: None, - theme: None, - slides: vec![WireSlideSpec { - title: "Broken".to_string(), - body: None, - bullets: vec![], - speaker_notes: None, - images: vec![WireSlideImage { - blob_id, - caption: None, - }], - }], - }) + .read_output("out-nope".to_string(), 0, 16) .await - .expect_err("unrecognisable image bytes should be rejected"); - assert_eq!(err.wire_name(), INVALID_INPUT_ERROR); + .expect_err("unknown output"); + assert_eq!(err.wire_name(), UNKNOWN_OUTPUT_ERROR); } #[tokio::test] -async fn extract_text_consumes_the_document_and_stages_the_text() { - let service = service(); - let now = Instant::now(); - let doc = tiny_pdf("Hello from the bus"); - let blob_id = service - .blobs - .begin(doc.len() as u64, &hex_digest(&doc), now) - .unwrap(); - service.blobs.put_chunk(&blob_id, 0, &doc, now).unwrap(); - +async fn a_malformed_read_is_refused_by_name() { + let service = service().await; let handle = service - .extract_text(blob_id.clone()) - .await - .expect("should extract"); - let bytes = service - .blobs - .get_chunk(&handle.blob_id, 0, handle.total_bytes, Instant::now()) - .unwrap(); - let text = String::from_utf8(bytes).expect("extracted text is utf-8"); - assert!( - text.contains("Hello from the bus"), - "extracted text missing content: {text:?}" - ); - - // The input was taken, so only the extracted text stays staged. - assert_eq!(service.blobs.live_count(), 1); -} - -#[tokio::test] -async fn extract_text_refuses_an_unknown_or_incomplete_blob() { - let service = service(); - let now = Instant::now(); + .hold(b"small".to_vec()) + .expect("hold should succeed"); let err = service - .extract_text("blob-nope".to_string()) + .read_output( + handle.output_id.clone(), + 0, + crate::outputs::MAX_CHUNK_BYTES as u64 + 1, + ) .await - .expect_err("unknown blob"); - assert_eq!(err.wire_name(), UNKNOWN_BLOB_ERROR); + .expect_err("oversize read"); + assert_eq!(err.wire_name(), TRANSFER_FAILED_ERROR); - let doc = tiny_pdf("partial"); - let blob_id = service - .blobs - .begin(doc.len() as u64, &hex_digest(&doc), now) - .unwrap(); - service - .blobs - .put_chunk(&blob_id, 0, &doc[..doc.len() / 2], now) - .unwrap(); let err = service - .extract_text(blob_id) + .read_output(handle.output_id, 999, 16) .await - .expect_err("incomplete blob"); + .expect_err("read past the end"); assert_eq!(err.wire_name(), TRANSFER_FAILED_ERROR); } - -#[tokio::test] -async fn the_blob_methods_round_trip_a_payload_over_the_declared_surface() { - // Exercises the four transfer methods through the same signatures the bus - // calls, including the base64 hop the store itself never sees. - let service = service(); - let payload: Vec = (0..5_000u32).map(|i| (i % 253) as u8).collect(); - - let blob_id = service - .begin_blob(payload.len() as u64, hex_digest(&payload)) - .await - .unwrap(); - let received = service - .put_chunk(blob_id.clone(), 0, BASE64.encode(&payload)) - .await - .unwrap(); - assert_eq!(received, payload.len() as u64); - - let encoded = service - .get_chunk(blob_id.clone(), 0, payload.len() as u64) - .await - .unwrap(); - assert_eq!(BASE64.decode(encoded).unwrap(), payload); - - service.release_blob(blob_id.clone()).await.unwrap(); - assert_eq!(service.blobs.live_count(), 0); - assert!(service.release_blob(blob_id).await.is_err()); -} - -/// A 1×1 PNG, built from its header so the fixture needs no dependency. -fn tiny_png() -> Vec { - let mut out = vec![0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]; - out.extend_from_slice(&13u32.to_be_bytes()); - out.extend_from_slice(b"IHDR"); - out.extend_from_slice(&1u32.to_be_bytes()); - out.extend_from_slice(&1u32.to_be_bytes()); - out.extend_from_slice(&[0x08, 0x06, 0x00, 0x00, 0x00]); - out.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); - out.extend_from_slice(&0u32.to_be_bytes()); - out.extend_from_slice(b"IDAT"); - out.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); - out.extend_from_slice(&0u32.to_be_bytes()); - out.extend_from_slice(b"IEND"); - out.extend_from_slice(&[0xAE, 0x42, 0x60, 0x82]); - out -} - -/// A valid single-page PDF whose text layer holds `text`. -fn tiny_pdf(text: &str) -> Vec { - let content = format!("BT /F1 24 Tf 72 700 Td ({text}) Tj ET\n"); - let objects = [ - "<< /Type /Catalog /Pages 2 0 R >>".to_string(), - "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(), - "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \ - /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>" - .to_string(), - "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(), - format!( - "<< /Length {} >>\nstream\n{content}endstream", - content.len() - ), - ]; - - let mut out = Vec::new(); - out.extend_from_slice(b"%PDF-1.4\n"); - let mut offsets = Vec::with_capacity(objects.len()); - for (i, body) in objects.iter().enumerate() { - offsets.push(out.len()); - out.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", i + 1).as_bytes()); - } - let xref_offset = out.len(); - out.extend_from_slice(format!("xref\n0 {}\n", objects.len() + 1).as_bytes()); - out.extend_from_slice(b"0000000000 65535 f \n"); - for offset in &offsets { - out.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes()); - } - out.extend_from_slice( - format!( - "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref_offset}\n%%EOF\n", - objects.len() + 1 - ) - .as_bytes(), - ); - out -} diff --git a/crates/tinydocs-module/src/service/wire.rs b/crates/tinydocs-module/src/service/wire.rs index 184da1d..68e2739 100644 --- a/crates/tinydocs-module/src/service/wire.rs +++ b/crates/tinydocs-module/src/service/wire.rs @@ -1,11 +1,16 @@ //! Wire shapes that differ from the library spec because bytes cannot travel //! inline. //! -//! [`crate::blobs`] explains why: a `TinyBus` frame is a 16 MiB JSON document, -//! and a deck may legally carry 40 MiB of images. So on the bus an image is a -//! staged blob id, and the module resolves it into the real -//! [`tinydocs::spec::SlideImage`] — bytes, format and dimensions — after the -//! upload completes. +//! A `TinyBus` frame is a 16 MiB JSON document and a deck may legally carry +//! 40 MiB of images, so image bytes ride a stream beside the call rather than +//! inside it. A call has one stream and a deck has many images, so the images +//! are concatenated in slide order and each one declares its `byte_len`; the +//! module splits them apart and resolves each into a real +//! [`tinydocs::spec::SlideImage`] — bytes, format and dimensions. +//! +//! The lengths live in the spec rather than in the stream because they are what +//! makes a truncated or over-long transfer a named rejection instead of a deck +//! with a picture assembled from two different images. //! //! Only the presentation spec needs this treatment. A document spec is text, and //! its aggregate cap keeps it inside a frame, so `GenerateDocx` takes @@ -17,8 +22,8 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct WireSlideImage { - /// Id of a completed blob holding the PNG or JPEG bytes. - pub blob_id: String, + /// Length of this image's bytes within the concatenated image stream. + pub byte_len: u64, /// Optional caption, rendered as a bullet beneath the image. #[serde(default)] pub caption: Option, diff --git a/crates/tinydocs-module/tests/module_e2e.rs b/crates/tinydocs-module/tests/module_e2e.rs index 40b9546..b115b54 100644 --- a/crates/tinydocs-module/tests/module_e2e.rs +++ b/crates/tinydocs-module/tests/module_e2e.rs @@ -1,46 +1,88 @@ //! End-to-end test for loading the built `TinyDocs` module into `TinyBus`. //! -//! This is the only test that exercises the real thing: the built `cdylib`, the -//! ABI descriptor, manifest admission, the dynamic loader, and a broker routing -//! actual frames. Everything else in this crate tests Rust functions directly and -//! would keep passing if the artifact stopped loading at all. +//! The only test that exercises the real thing: the built `cdylib`, the ABI +//! descriptor, manifest admission, the dynamic loader, and a broker routing +//! actual frames. Everything else in this crate calls Rust functions directly and +//! would keep passing if the artifact stopped loading altogether. //! -//! It therefore covers each of the three formats end to end, and moves an image -//! across more than one chunk — the chunked path is the reason this interface -//! exists, and a single-chunk transfer would not prove it works. +//! It also carries the only honest test of the streaming paths. A stream needs +//! two connected peers with a broker between them, so a unit test against a bare +//! struct cannot reach one — and the payloads here are deliberately larger than a +//! single chunk, because a one-chunk transfer would not prove the reassembly. #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use std::time::Duration; +use base64::Engine as _; use tinybus::Connection; use tinybus::broker::Broker; use tinybus::module::{ModuleHost, ModuleState}; use tinybus::transport::memory::MemoryBus; use tinydocs::spec::{DocumentSection, DocumentSpec}; -use tinydocs_module::{BUS_NAME, BlobRef, OBJECT_PATH, hex_digest}; +use tinydocs_module::{BUS_NAME, OBJECT_PATH, OutputRef, hex_digest}; -/// Every method the manifest must declare. +/// Every method the manifest must declare, in order. const EXPECTED_METHODS: &[&str] = &[ - "BeginBlob", - "PutChunk", - "GetChunk", - "ReleaseBlob", "GenerateDocx", "GeneratePptx", "ExtractText", + "ReadOutput", + "ReleaseOutput", ]; -/// Chunk size used by the test transfers. +/// Chunk size for reading outputs back. /// -/// Deliberately small so a modest fixture still spans several chunks. The -/// module's own cap is megabytes; nothing here needs to approach it to prove the -/// offsets line up. -const TEST_CHUNK: usize = 512; +/// Small on purpose so a modest document still takes several reads. The module's +/// own cap is megabytes; nothing here needs to approach it to prove the offsets +/// line up. +const READ_CHUNK: u64 = 512; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[ignore = "requires TINYDOCS_TEST_MODULE to point at the built cdylib"] async fn the_built_module_serves_every_format_over_a_real_broker() { + // One test rather than four: TinyBus never unloads a module and a second + // load of the same artifact would collide on the well-known name, so every + // format is exercised against the one admitted instance. + let (client, modules, broker_task) = admit_module(); + let client = client.await; + wait_until_serving(&client).await; + + let target = Target::new(); + let proxy = client.proxy(BUS_NAME, OBJECT_PATH, BUS_NAME).unwrap(); + + generates_a_docx(&proxy).await; + generates_a_pptx_from_a_streamed_image_pair(&client, &target, &proxy).await; + extracts_text_from_a_streamed_pdf(&client, &target, &proxy).await; + refuses_a_stream_that_contradicts_the_spec(&client, &target).await; + + assert!(matches!(modules.list()[0].state, ModuleState::Ready)); + broker_task.abort(); +} + +/// The destination triple every streaming call needs. +struct Target { + destination: tinybus::BusName, + path: tinybus::ObjectPath, + interface: tinybus::InterfaceName, +} + +impl Target { + fn new() -> Self { + Self { + destination: tinybus::BusName::new(BUS_NAME).unwrap(), + path: tinybus::ObjectPath::new(OBJECT_PATH).unwrap(), + interface: tinybus::InterfaceName::new(BUS_NAME).unwrap(), + } + } +} + +/// Load the built artifact and check its manifest against the interface. +fn admit_module() -> ( + impl std::future::Future, + ModuleHost, + tokio::task::JoinHandle>, +) { let artifact = std::env::var_os("TINYDOCS_TEST_MODULE").expect("TINYDOCS_TEST_MODULE must be set"); let bus = MemoryBus::new(); @@ -65,9 +107,16 @@ async fn the_built_module_serves_every_format_over_a_real_broker() { "manifest methods drifted from the interface" ); - let client = Connection::connect(bus.connect().await.unwrap()) - .await - .unwrap(); + let connect = async move { + Connection::connect(bus.connect().await.unwrap()) + .await + .unwrap() + }; + (connect, modules, broker_task) +} + +/// Wait for the module to claim its well-known name. +async fn wait_until_serving(client: &Connection) { tokio::time::timeout(Duration::from_secs(5), async { loop { if client @@ -84,11 +133,11 @@ async fn the_built_module_serves_every_format_over_a_real_broker() { }) .await .expect("module should become ready"); +} - let proxy = client.proxy(BUS_NAME, OBJECT_PATH, BUS_NAME).unwrap(); - - // --- .docx: text in, staged bytes out --- - let handle: BlobRef = proxy +/// No inbound payload, a held document out. +async fn generates_a_docx(proxy: &tinybus::Proxy) { + let handle: OutputRef = proxy .call( "GenerateDocx", (DocumentSpec { @@ -103,103 +152,131 @@ async fn the_built_module_serves_every_format_over_a_real_broker() { ) .await .expect("GenerateDocx should succeed"); - let docx = download(&proxy, &handle).await; + let docx = download(proxy, &handle).await; assert_eq!(&docx[..2], b"PK", "a .docx is a zip container"); - // --- .pptx: an image staged across several chunks, then a deck --- - let png = png_1x1(); - assert!( - png.len() > TEST_CHUNK, - "the image fixture must span more than one chunk to be worth testing" - ); - let image_blob = upload(&proxy, &png).await; - let deck: BlobRef = proxy - .call( - "GeneratePptx", - (serde_json::json!({ - "title": "TinyBus E2E", - "slides": [{ - "title": "With an image", - "images": [{ "blob_id": image_blob, "caption": "A chart" }], - }], - }),), + // Releasing a document twice reports that it is gone rather than pretending. + proxy + .call::<()>("ReleaseOutput", (handle.output_id.clone(),)) + .await + .expect("releasing a held document should succeed"); + proxy + .call::<()>("ReleaseOutput", (handle.output_id,)) + .await + .expect_err("releasing twice should fail"); +} + +/// Two images concatenated into one stream, split apart by their declared +/// lengths. +async fn generates_a_pptx_from_a_streamed_image_pair( + client: &Connection, + target: &Target, + proxy: &tinybus::Proxy, +) { + let first = png_padded_to(2_000); + let second = png_padded_to(3_000); + let (first_len, second_len) = (first.len(), second.len()); + let mut payload = first; + payload.extend_from_slice(&second); + + let deck: OutputRef = client + .call_with_stream( + target.destination.clone(), + target.path.clone(), + target.interface.clone(), + tinybus::MemberName::new("GeneratePptx").unwrap(), + |stream| { + serde_json::json!([ + { + "title": "TinyBus E2E", + "slides": [{ + "title": "With images", + "images": [ + { "byte_len": first_len, "caption": "First" }, + { "byte_len": second_len, "caption": "Second" }, + ], + }], + }, + stream, + ]) + }, + &payload, ) .await .expect("GeneratePptx should succeed"); - let pptx = download(&proxy, &deck).await; + let pptx = download(proxy, &deck).await; assert_eq!(&pptx[..2], b"PK", "a .pptx is a zip container"); +} - // --- .pdf: a staged document in, extracted text out --- +/// A streamed document in, extracted text out. +async fn extracts_text_from_a_streamed_pdf( + client: &Connection, + target: &Target, + proxy: &tinybus::Proxy, +) { let pdf = pdf_with_text("Hello from the module"); - let pdf_blob = upload(&proxy, &pdf).await; - let extracted: BlobRef = proxy - .call("ExtractText", (pdf_blob,)) + let extracted: OutputRef = client + .call_with_stream( + target.destination.clone(), + target.path.clone(), + target.interface.clone(), + tinybus::MemberName::new("ExtractText").unwrap(), + |stream| serde_json::json!([stream]), + &pdf, + ) .await .expect("ExtractText should succeed"); - let text = String::from_utf8(download(&proxy, &extracted).await).expect("text is utf-8"); + let text = String::from_utf8(download(proxy, &extracted).await).expect("text is utf-8"); assert!( text.contains("Hello from the module"), "extracted text missing content: {text:?}" ); - - // Releasing a consumed handle is reported, not silently accepted. - proxy - .call::<()>("ReleaseBlob", (handle.blob_id.clone(),)) - .await - .expect("releasing a staged output should succeed"); - proxy - .call::<()>("ReleaseBlob", (handle.blob_id,)) - .await - .expect_err("releasing twice should fail"); - - assert!(matches!(modules.list()[0].state, ModuleState::Ready)); - broker_task.abort(); } -/// Stage `bytes` over `BeginBlob` + `PutChunk`, returning the blob id. -async fn upload(proxy: &tinybus::Proxy, bytes: &[u8]) -> String { - use base64::Engine as _; - let encoder = base64::engine::general_purpose::STANDARD; - - let blob_id: String = proxy - .call("BeginBlob", (bytes.len() as u64, hex_digest(bytes))) - .await - .expect("BeginBlob should succeed"); - - let mut offset = 0usize; - while offset < bytes.len() { - let end = (offset + TEST_CHUNK).min(bytes.len()); - let received: u64 = proxy - .call( - "PutChunk", - ( - blob_id.clone(), - offset as u64, - encoder.encode(&bytes[offset..end]), - ), - ) - .await - .expect("PutChunk should succeed"); - assert_eq!(received, end as u64, "server disagreed about progress"); - offset = end; - } - blob_id +/// The lengths in the spec are the authority. +/// +/// A short stream must fail rather than produce a deck with a picture assembled +/// from whatever bytes happened to arrive. +async fn refuses_a_stream_that_contradicts_the_spec(client: &Connection, target: &Target) { + let mismatched: tinybus::Result = client + .call_with_stream( + target.destination.clone(), + target.path.clone(), + target.interface.clone(), + tinybus::MemberName::new("GeneratePptx").unwrap(), + |stream| { + serde_json::json!([ + { + "title": "Mismatched", + "slides": [{ + "title": "Truncated", + "images": [{ "byte_len": 9_999 }], + }], + }, + stream, + ]) + }, + b"too short", + ) + .await; + assert!( + mismatched.is_err(), + "a stream shorter than the declared images should be refused" + ); } -/// Read a staged blob back over `GetChunk` and verify its digest. -async fn download(proxy: &tinybus::Proxy, handle: &BlobRef) -> Vec { - use base64::Engine as _; +/// Read a held document back in chunks and verify its digest. +async fn download(proxy: &tinybus::Proxy, handle: &OutputRef) -> Vec { let decoder = base64::engine::general_purpose::STANDARD; - let mut out = Vec::with_capacity(usize::try_from(handle.total_bytes).unwrap_or_default()); while (out.len() as u64) < handle.total_bytes { let encoded: String = proxy .call( - "GetChunk", - (handle.blob_id.clone(), out.len() as u64, TEST_CHUNK as u64), + "ReadOutput", + (handle.output_id.clone(), out.len() as u64, READ_CHUNK), ) .await - .expect("GetChunk should succeed"); + .expect("ReadOutput should succeed"); let chunk = decoder.decode(encoded).expect("chunk is base64"); assert!(!chunk.is_empty(), "read stalled at offset {}", out.len()); out.extend_from_slice(&chunk); @@ -212,11 +289,11 @@ async fn download(proxy: &tinybus::Proxy, handle: &BlobRef) -> Vec { out } -/// A 1×1 PNG padded past [`TEST_CHUNK`] so its transfer spans several chunks. +/// A 1×1 PNG padded to roughly `total` bytes. /// -/// The padding rides in a trailing comment chunk, which keeps the file a valid -/// PNG that the module will accept and measure. -fn png_1x1() -> Vec { +/// The padding rides in a `tEXt` chunk, which is ancillary — the file stays a +/// valid PNG the module will accept and measure. +fn png_padded_to(total: usize) -> Vec { let mut out = vec![0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]; out.extend_from_slice(&13u32.to_be_bytes()); out.extend_from_slice(b"IHDR"); @@ -228,10 +305,11 @@ fn png_1x1() -> Vec { out.extend_from_slice(b"IDAT"); out.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); - // tEXt is an ancillary chunk, so a reader that does not care skips it. - let padding = vec![b'p'; TEST_CHUNK * 2]; + // Chunk overhead for the tEXt chunk plus the IEND trailer that follows. + let overhead = 12 + 4 + 12; + let padding = total.saturating_sub(out.len() + overhead); let mut text_chunk = b"pad\0".to_vec(); - text_chunk.extend_from_slice(&padding); + text_chunk.extend(std::iter::repeat_n(b'p', padding)); out.extend_from_slice(&u32::try_from(text_chunk.len()).unwrap().to_be_bytes()); out.extend_from_slice(b"tEXt"); out.extend_from_slice(&text_chunk); diff --git a/docs/specs/tinybus-module.md b/docs/specs/tinybus-module.md index 8a107aa..c38ac5a 100644 --- a/docs/specs/tinybus-module.md +++ b/docs/specs/tinybus-module.md @@ -35,23 +35,26 @@ module claims `ai.tinyhumans.tinydocs.Documents`, serves the object path `/ai/tinyhumans/tinydocs/Documents`, and exports seven methods: ```text -BeginBlob(total_bytes, sha256) -> blob_id -PutChunk(blob_id, offset, base64) -> bytes received so far -GetChunk(blob_id, offset, len) -> base64 -ReleaseBlob(blob_id) -> () -GenerateDocx(DocumentSpec) -> BlobRef -GeneratePptx(deck with image blobs) -> BlobRef -ExtractText(blob_id) -> BlobRef +GenerateDocx(DocumentSpec) -> OutputRef +GeneratePptx(deck, Option) -> OutputRef +ExtractText(StreamRef) -> OutputRef +ReadOutput(output_id, offset, len) -> base64 +ReleaseOutput(output_id) -> () ``` The format arguments are the same Serde contracts used by the Rust API, except -that a slide image names a staged blob rather than carrying bytes inline. +that a slide image declares its length in the concatenated image stream rather +than carrying bytes inline. -No method returns bytes inline. A frame is a 16 MiB JSON document and a `Vec` -serialises as an array of integers — about 3.5 bytes of frame per byte — so the -real inline ceiling is a few megabytes, below both a deck's legal image payload -and any `.pdf` worth extracting. Every unbounded value is therefore staged and -moved in base64 chunks. +Inbound payloads ride TinyBus streams, so flow control, the size cap and the +idle timeout are the bus's. A deck's images share one stream because a call has +one stream; their declared lengths are the authority on where each image ends. + +Replies cannot stream: `Interface::call` receives no caller identity and no +connection, so a served object cannot open a stream back to its caller. A +produced document is therefore held and pulled with `ReadOutput`, because a +frame is a 16 MiB JSON document and a `Vec` serialises as an array of +integers — about 3.5 bytes of frame per byte. Invalid input, writer failures and extraction failures use the distinct wire names `ai.tinyhumans.tinydocs.Error.InvalidInput`, @@ -78,14 +81,13 @@ second fully-declared interface is not expressible without a TinyBus change. - No Rust value crosses the dynamic-library ABI boundary. - The native artifact must match the host target and TinyBus compatibility gate. -- Message payloads remain subject to TinyBus's 16 MiB frame cap, which is why - bytes move in bounded chunks rather than inline. Path or file-descriptor - transfer would remove the copies and remains the better long-term answer. -- The staging area is bounded per chunk, per blob, in total and by blob count, - and expires untouched blobs. A module is never unloaded, so an unbounded - staging area is a leak with no end. -- A blob is verified against its declared SHA-256 before it becomes readable, so - a truncated or reordered transfer cannot be consumed as though it were whole. +- Message payloads remain subject to TinyBus's 16 MiB frame cap. Inbound bytes + avoid it through streams; outbound bytes are pulled in bounded chunks until + TinyBus gains a reply-stream seam. +- Held documents are bounded per document, in total, by count, and by an idle + TTL. A module is never unloaded, so an unbounded store is a leak with no end. +- An image stream that does not match the lengths the deck declares is refused, + so a truncated transfer cannot become a deck with a corrupt picture in it. - Dynamic modules are trusted code with the host process's privileges. ## Acceptance criteria @@ -108,9 +110,12 @@ second fully-declared interface is not expressible without a TinyBus change. None blocking this version. -Two things belong upstream in TinyBus rather than here. The staging area is -format-agnostic and every module that moves bytes will want it, so it is a -candidate for the module SDK. And `module_export!` attaching its method list only -to the first provided interface is what forces one interface to carry both the -transfer and the format methods; per-interface method lists would allow the -cleaner split. +Two things belong upstream in TinyBus rather than here. + +A reply-stream seam would delete the output store entirely: the only reason a +produced document is held at all is that a served object cannot open a stream +back to its caller. + +And `module_export!` attaching its method list only to the first provided +interface is what forces one interface to carry both the output methods and the +format methods; per-interface method lists would allow the cleaner split. diff --git a/vendor/tinybus b/vendor/tinybus index 0b161d2..6ca0b0b 160000 --- a/vendor/tinybus +++ b/vendor/tinybus @@ -1 +1 @@ -Subproject commit 0b161d201f4517116d3df2f33662eec2538724dc +Subproject commit 6ca0b0b6739a49396e36be21d450f07cf85b9de2 From 8860df5f539a25d36b76550d76c1ebe2063e3727 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 11 Aug 2026 03:34:03 +0300 Subject: [PATCH 07/13] Move the presentation wire spec into the library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bus-facing deck shape — the one whose images name a length in a stream rather than carrying bytes — lived in the private module crate. It belongs beside the spec it mirrors: a host driving the module over a bus needs those types, and the module crate is `publish = false`, so the host would have had to re-declare them. That is exactly the drift the `spec` carve-out exists to prevent. The wire deck is `serde` and nothing else, so it sits in `spec::presentation::wire` and is compiled in every build, including `--no-default-features`. The module crate re-exports it rather than owning it, and both sides now share one definition of the shape. Co-authored-by: Medulla --- crates/tinydocs-module/src/service/mod.rs | 4 +--- src/spec/mod.rs | 1 + src/spec/presentation/mod.rs | 2 ++ .../service => src/spec/presentation}/wire.rs | 16 +++++++++------- 4 files changed, 13 insertions(+), 10 deletions(-) rename {crates/tinydocs-module/src/service => src/spec/presentation}/wire.rs (81%) diff --git a/crates/tinydocs-module/src/service/mod.rs b/crates/tinydocs-module/src/service/mod.rs index a291d2c..e299cf2 100644 --- a/crates/tinydocs-module/src/service/mod.rs +++ b/crates/tinydocs-module/src/service/mod.rs @@ -45,8 +45,6 @@ //! `provides` and leaves any others empty, so a second fully-declared interface //! is not expressible today. -mod wire; - use std::sync::Arc; use std::time::Instant; @@ -59,7 +57,7 @@ use tinydocs::{Error, pdf, pptx}; use crate::outputs::{OutputError, OutputRef, OutputStore}; -pub use wire::{WirePresentationSpec, WireSlideImage, WireSlideSpec}; +pub use tinydocs::spec::{WirePresentationSpec, WireSlideImage, WireSlideSpec}; /// Well-known name and interface exported by the `TinyDocs` module. pub const BUS_NAME: &str = "ai.tinyhumans.tinydocs.Documents"; diff --git a/src/spec/mod.rs b/src/spec/mod.rs index e64d258..003a2a3 100644 --- a/src/spec/mod.rs +++ b/src/spec/mod.rs @@ -40,4 +40,5 @@ pub mod presentation; pub use document::{DocumentSection, DocumentSpec}; pub use image::ImageFormat; +pub use presentation::wire::{WirePresentationSpec, WireSlideImage, WireSlideSpec}; pub use presentation::{PresentationSpec, SlideImage, SlideSpec}; diff --git a/src/spec/presentation/mod.rs b/src/spec/presentation/mod.rs index c8e3389..74b23fc 100644 --- a/src/spec/presentation/mod.rs +++ b/src/spec/presentation/mod.rs @@ -334,5 +334,7 @@ impl PresentationSpec { } } +pub mod wire; + #[cfg(test)] mod test; diff --git a/crates/tinydocs-module/src/service/wire.rs b/src/spec/presentation/wire.rs similarity index 81% rename from crates/tinydocs-module/src/service/wire.rs rename to src/spec/presentation/wire.rs index 68e2739..4251869 100644 --- a/crates/tinydocs-module/src/service/wire.rs +++ b/src/spec/presentation/wire.rs @@ -1,20 +1,22 @@ -//! Wire shapes that differ from the library spec because bytes cannot travel -//! inline. +//! The presentation spec as it crosses a bus, where bytes cannot travel inline. //! //! A `TinyBus` frame is a 16 MiB JSON document and a deck may legally carry //! 40 MiB of images, so image bytes ride a stream beside the call rather than //! inside it. A call has one stream and a deck has many images, so the images //! are concatenated in slide order and each one declares its `byte_len`; the //! module splits them apart and resolves each into a real -//! [`tinydocs::spec::SlideImage`] — bytes, format and dimensions. +//! [`super::SlideImage`] — bytes, format and dimensions. //! //! The lengths live in the spec rather than in the stream because they are what //! makes a truncated or over-long transfer a named rejection instead of a deck //! with a picture assembled from two different images. //! -//! Only the presentation spec needs this treatment. A document spec is text, and -//! its aggregate cap keeps it inside a frame, so `GenerateDocx` takes -//! [`tinydocs::spec::DocumentSpec`] unchanged. +//! Only the presentation spec needs this treatment. A document spec is text and +//! its aggregate cap keeps it inside a frame, so a document crosses unchanged. +//! +//! Defined here rather than in the module that serves it so a host driving that +//! module over a bus shares one definition of the shape instead of re-declaring +//! it. Like the rest of [`crate::spec`] it is serde and nothing else. use serde::{Deserialize, Serialize}; @@ -31,7 +33,7 @@ pub struct WireSlideImage { /// One content slide, as it appears on the bus. /// -/// Identical to [`tinydocs::spec::SlideSpec`] apart from `images`. +/// Identical to [`super::SlideSpec`] apart from `images`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct WireSlideSpec { From c4d1740821d7f9f147258de14c5cf1d8ee16ed36 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 11 Aug 2026 08:49:54 +0300 Subject: [PATCH 08/13] chore(deps): update pdf-extract to 0.12 and refresh lockfile Update the optional pdf-extract dependency from 0.10 to 0.12, pulling in its newer transitive dependencies including an upgraded lopdf, rand, and getrandom stack. The lockfile is regenerated to reflect the new crate versions and to remove several intermediate dependencies that are no longer required. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.lock | 131 +++++++++++++++++------------------------------------ Cargo.toml | 2 +- 2 files changed, 42 insertions(+), 91 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bce4e6a..1cdd351 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,7 +25,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -197,12 +197,6 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" -[[package]] -name = "bytecount" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" - [[package]] name = "bytemuck" version = "1.25.2" @@ -284,9 +278,9 @@ dependencies = [ [[package]] name = "cff-parser" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f5b6e9141c036f3ff4ce7b2f7e432b0f00dee416ddcd4f17741d189ddc2e9d" +checksum = "c5810ca1a2b5870df2aab1c03e11c40c361ba51d6e3e361e56310f1cb3b4e087" [[package]] name = "cfg-if" @@ -294,6 +288,17 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "cipher" version = "0.4.4" @@ -371,6 +376,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -629,18 +643,6 @@ dependencies = [ "wasi", ] -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "libc", - "r-efi 5.3.0", - "wasip2", -] - [[package]] name = "getrandom" version = "0.4.3" @@ -648,8 +650,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", - "r-efi 6.0.0", + "r-efi", + "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -819,9 +824,9 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lopdf" -version = "0.38.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7184fdea2bc3cd272a1acec4030c321a8f9875e877b3f92a53f2f6033fdc289" +checksum = "25aab26d99567469098e64a02f42679f8965c6401263eefa31d8f2dcc37a221c" dependencies = [ "aes", "bitflags", @@ -829,13 +834,12 @@ dependencies = [ "ecb", "encoding_rs", "flate2", - "getrandom 0.3.4", + "getrandom 0.4.3", "indexmap", "itoa", "log", "md-5", "nom", - "nom_locate", "rand", "rangemap", "sha2", @@ -890,17 +894,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "nom_locate" -version = "5.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d" -dependencies = [ - "bytecount", - "memchr", - "nom", -] - [[package]] name = "num-conv" version = "0.2.2" @@ -953,9 +946,9 @@ dependencies = [ [[package]] name = "pdf-extract" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28ba1758a3d3f361459645780e09570b573fc3c82637449e9963174c813a98" +checksum = "417e8fdc940f1d5bc62c5f89864c3a2255f74f69aa353c98509213d67df61e73" dependencies = [ "adobe-cmap-parser", "cff-parser", @@ -1074,15 +1067,6 @@ dependencies = [ "zip 0.6.6", ] -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - [[package]] name = "proc-macro2" version = "1.0.107" @@ -1123,12 +1107,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - [[package]] name = "r-efi" version = "6.0.0" @@ -1137,22 +1115,13 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.9.5" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ - "rand_chacha", - "rand_core 0.9.5", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -1163,12 +1132,9 @@ checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" [[package]] name = "rand_core" -version = "0.9.5" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "rangemap" @@ -1357,7 +1323,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1368,7 +1334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1903,15 +1869,6 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasip2" -version = "1.0.4+wasi-0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" -dependencies = [ - "wit-bindgen", -] - [[package]] name = "wasm-bindgen" version = "0.2.127" @@ -2078,12 +2035,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - [[package]] name = "write-fonts" version = "0.48.1" diff --git a/Cargo.toml b/Cargo.toml index bcd3b58..0d33c93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ ppt-rs = { version = "0.2.14", optional = true } # font and PostScript parsing stack (`lopdf`, CFF/Type1/CMap parsers) that only # the extraction path needs, so a host that never reads a PDF should not carry # it. -pdf-extract = { version = "0.10", optional = true } +pdf-extract = { version = "0.12", optional = true } [dev-dependencies] # `.docx` output is a zip container; the tests re-open the produced bytes and From a4561cfd0097a0e8e8f3c671f7f44aca63b6c3d2 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 11 Aug 2026 08:50:30 +0300 Subject: [PATCH 09/13] chore(deps): downgrade windows-sys and relax ppt-rs version constraint Downgrade the windows-sys dependency from 0.61.2 to 0.52.0 across all transitive dependencies in the lockfile, and relax the ppt-rs version requirement from exact 0.2.14 to the more permissive 0.2 range to allow compatible patch updates. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1cdd351..049d7f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -503,7 +503,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1211,7 +1211,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1471,7 +1471,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1935,7 +1935,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 0d33c93..a873183 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ docx-rs = { version = "0.4.20", optional = true } # its own zip/XML stack plus `syntect` and `pulldown-cmark` for a Markdown # front-end this crate does not use, which is precisely why it is gated — a host # that only generates documents should not carry a syntax highlighter. -ppt-rs = { version = "0.2.14", optional = true } +ppt-rs = { version = "0.2", optional = true } # `.pdf` text extraction. Optional: exclusive to the `pdf` feature. It brings a # font and PostScript parsing stack (`lopdf`, CFF/Type1/CMap parsers) that only # the extraction path needs, so a host that never reads a PDF should not carry From 391ca2a1cceb30621789a8a064dbdf06e6a37a54 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 11 Aug 2026 08:51:14 +0300 Subject: [PATCH 10/13] chore(deps): prune unused dependency subtree from ppt-rs Set `default-features = false` on the `ppt-rs` optional dependency to disable its `pdf-native` feature, which pulled in `pdfrs` and a chain of transitive dependencies including `syntect`, `yaml-rust`, `bincode`, and `ttf-parser`. This eliminates three RUSTSEC unmaintained advisories and removes over 400 lines of lock-file entries, since nothing in this crate calls the PDF export functionality that the subtree existed to support. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.lock | 440 +---------------------------------------------------- Cargo.toml | 14 +- 2 files changed, 13 insertions(+), 441 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 049d7f2..2de70aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,71 +28,6 @@ dependencies = [ "cpufeatures 0.2.17", ] -[[package]] -name = "aho-corasick" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" -dependencies = [ - "memchr", -] - -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys 0.61.2", -] - -[[package]] -name = "anyhow" -version = "1.0.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" - [[package]] name = "arbitrary" version = "1.4.2" @@ -102,12 +37,6 @@ dependencies = [ "derive_arbitrary", ] -[[package]] -name = "arrayvec" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" - [[package]] name = "async-trait" version = "0.1.92" @@ -143,30 +72,6 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - -[[package]] -name = "bit-set" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" - [[package]] name = "bitflags" version = "2.13.1" @@ -202,20 +107,6 @@ name = "bytemuck" version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" -dependencies = [ - "bytemuck_derive", -] - -[[package]] -name = "bytemuck_derive" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] [[package]] name = "byteorder" @@ -309,58 +200,12 @@ dependencies = [ "inout", ] -[[package]] -name = "clap" -version = "4.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "clap_lex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" - [[package]] name = "color_quant" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" -[[package]] -name = "colorchoice" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" - [[package]] name = "constant_time_eq" version = "0.1.5" @@ -503,7 +348,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -515,26 +360,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "euclid" -version = "0.22.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" -dependencies = [ - "num-traits", -] - -[[package]] -name = "fancy-regex" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "998b056554fbe42e03ae0e152895cd1a7e1002aec800fdc6635d20270260c46f" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - [[package]] name = "fastrand" version = "2.5.0" @@ -583,21 +408,6 @@ dependencies = [ "zlib-rs", ] -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "font-types" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b38ad915f6dadd993ced50848a8291a543bd41ca62bc10740d5e64e2ab4cfd7" -dependencies = [ - "bytemuck", -] - [[package]] name = "futures-core" version = "0.3.33" @@ -684,12 +494,6 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - [[package]] name = "hmac" version = "0.12.1" @@ -753,12 +557,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - [[package]] name = "itoa" version = "1.0.18" @@ -786,30 +584,12 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "kurbo" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b60dfc32f652b926df6192e55525b16d186c69d47876c3ead4da5cc9f8450e2" -dependencies = [ - "arrayvec", - "euclid 0.22.14", - "polycool", - "smallvec", -] - [[package]] name = "libc" version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -915,12 +695,6 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - [[package]] name = "password-hash" version = "0.4.2" @@ -953,7 +727,7 @@ dependencies = [ "adobe-cmap-parser", "cff-parser", "encoding_rs", - "euclid 0.20.14", + "euclid", "log", "lopdf", "postscript", @@ -961,28 +735,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "pdfrs" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd2c58cc563c54ee2dc0dacce61bf686529d792c4b76f031cbcfe8e3f7ecdeb0" -dependencies = [ - "aes", - "anyhow", - "base64 0.22.1", - "cbc", - "clap", - "flate2", - "md-5", - "regex", - "serde", - "serde_json", - "sha2", - "subsetter", - "syntect", - "ttf-parser", -] - [[package]] name = "percent-encoding" version = "2.3.2" @@ -1001,19 +753,6 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" -[[package]] -name = "plist" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da1d65da6dd5d1e44199ac0f58712d241c0f439f80adea8924d832384087f85" -dependencies = [ - "base64 0.22.1", - "indexmap", - "quick-xml", - "serde", - "time", -] - [[package]] name = "png" version = "0.18.1" @@ -1027,15 +766,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "polycool" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50596ddc09eb5ad5f75cacd40209568e66df71baf86e1499a0e99c4cff12a5a6" -dependencies = [ - "arrayvec", -] - [[package]] name = "pom" version = "1.1.0" @@ -1060,7 +790,6 @@ version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed6af693d661395ff3464eac5f7cee1df674d082ce84da94c6819cc799fee929" dependencies = [ - "pdfrs", "thiserror 1.0.69", "uuid", "xml-rs", @@ -1142,45 +871,6 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" -[[package]] -name = "read-fonts" -version = "0.39.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4ed38b89c2c77ff968c524145ad65fb010f38af5c7a224b53b81d47ac2daa81" -dependencies = [ - "bytemuck", - "font-types", -] - -[[package]] -name = "regex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" - [[package]] name = "ring" version = "0.17.14" @@ -1195,12 +885,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustc-hash" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" - [[package]] name = "rustix" version = "1.1.4" @@ -1211,7 +895,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1255,15 +939,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "serde" version = "1.0.229" @@ -1350,16 +1025,6 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" -[[package]] -name = "skrifa" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c34617370ae968efb7161bb2beb517d9084659aae19e24b89e3db25b46e4564" -dependencies = [ - "bytemuck", - "read-fonts", -] - [[package]] name = "slab" version = "0.4.12" @@ -1383,24 +1048,6 @@ dependencies = [ "unicode-properties", ] -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "subsetter" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38803281d1c23166c5ebcb455439a5d2afe711cc909cf88af72448c297756ad6" -dependencies = [ - "kurbo", - "rustc-hash", - "skrifa", - "write-fonts", -] - [[package]] name = "subtle" version = "2.6.1" @@ -1429,27 +1076,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syntect" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "656b45c05d95a5704399aeef6bd0ddec7b2b3531b7c9e900abbf7c4d2190c925" -dependencies = [ - "bincode", - "fancy-regex", - "flate2", - "fnv", - "once_cell", - "plist", - "regex-syntax", - "serde", - "serde_derive", - "serde_json", - "thiserror 2.0.20", - "walkdir", - "yaml-rust", -] - [[package]] name = "tar" version = "0.4.46" @@ -1471,7 +1097,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1539,7 +1165,6 @@ dependencies = [ "powerfmt", "serde_core", "time-core", - "time-macros", ] [[package]] @@ -1548,16 +1173,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" -[[package]] -name = "time-macros" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" -dependencies = [ - "num-conv", - "time-core", -] - [[package]] name = "tinybus" version = "0.1.0" @@ -1830,12 +1445,6 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - [[package]] name = "uuid" version = "1.24.0" @@ -1853,16 +1462,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1929,15 +1528,6 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys 0.52.0", -] - [[package]] name = "windows-link" version = "0.2.1" @@ -2035,19 +1625,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "write-fonts" -version = "0.48.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb731d4c4d93eacc69a1ad2f270f905788a98e4a3438267bcafbe08d3431c8d8" -dependencies = [ - "font-types", - "indexmap", - "kurbo", - "log", - "read-fonts", -] - [[package]] name = "xattr" version = "1.6.1" @@ -2064,15 +1641,6 @@ version = "0.8.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" -[[package]] -name = "yaml-rust" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] - [[package]] name = "zerocopy" version = "0.8.56" diff --git a/Cargo.toml b/Cargo.toml index a873183..4a92a0d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,11 +38,15 @@ serde = { version = "1", features = ["derive"] } # OOXML `.docx` synthesis. Optional: exclusive to the `docx` feature so a host # that only needs extraction does not pull the writer stack. docx-rs = { version = "0.4.20", optional = true } -# OOXML `.pptx` synthesis. Optional: exclusive to the `pptx` feature. It brings -# its own zip/XML stack plus `syntect` and `pulldown-cmark` for a Markdown -# front-end this crate does not use, which is precisely why it is gated — a host -# that only generates documents should not carry a syntax highlighter. -ppt-rs = { version = "0.2", optional = true } +# OOXML `.pptx` synthesis. Optional: exclusive to the `pptx` feature. +# +# `default-features = false` turns off `pdf-native`, which exists to export a +# deck as PDF and pulls `pdfrs` -> `syntect` -> `yaml-rust` for the syntax +# highlighting that feature wants. We only ever write `.pptx`, and that tail +# carries three RUSTSEC unmaintained advisories (`yaml-rust`, `bincode`, +# `ttf-parser`). Nothing in this crate calls the PDF exporter, so the whole +# subtree goes. +ppt-rs = { version = "0.2", default-features = false, optional = true } # `.pdf` text extraction. Optional: exclusive to the `pdf` feature. It brings a # font and PostScript parsing stack (`lopdf`, CFF/Type1/CMap parsers) that only # the extraction path needs, so a host that never reads a PDF should not carry From 5a80f19a39193c6ad046f985f5455d7c34f533b6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 11 Aug 2026 08:52:11 +0300 Subject: [PATCH 11/13] chore(deps): enable ppt-rs default features to fix build failure The ppt-rs dependency previously had `default-features = false` to avoid pulling in the pdf-native feature and its transitive dependencies, which carry unmaintained RUSTSEC advisories. However, ppt-rs 0.2.24 unconditionally imports `pdfrs` in its slide render module without a cfg guard, causing a build failure when the feature is disabled. Default features are now enabled to restore compilation, and the advisory exemptions are documented in deny.toml until ppt-rs fixes the conditional compilation. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.lock | 434 ++++++++++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 20 ++- 2 files changed, 446 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2de70aa..a1f8528 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,71 @@ dependencies = [ "cpufeatures 0.2.17", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + [[package]] name = "arbitrary" version = "1.4.2" @@ -37,6 +102,12 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "async-trait" version = "0.1.92" @@ -72,6 +143,30 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "2.13.1" @@ -107,6 +202,20 @@ name = "bytemuck" version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] [[package]] name = "byteorder" @@ -200,12 +309,58 @@ dependencies = [ "inout", ] +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "color_quant" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -360,6 +515,26 @@ dependencies = [ "num-traits", ] +[[package]] +name = "euclid" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fancy-regex" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "998b056554fbe42e03ae0e152895cd1a7e1002aec800fdc6635d20270260c46f" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -408,6 +583,21 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "font-types" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b38ad915f6dadd993ced50848a8291a543bd41ca62bc10740d5e64e2ab4cfd7" +dependencies = [ + "bytemuck", +] + [[package]] name = "futures-core" version = "0.3.33" @@ -494,6 +684,12 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hmac" version = "0.12.1" @@ -557,6 +753,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itoa" version = "1.0.18" @@ -584,12 +786,30 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kurbo" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b60dfc32f652b926df6192e55525b16d186c69d47876c3ead4da5cc9f8450e2" +dependencies = [ + "arrayvec", + "euclid 0.22.14", + "polycool", + "smallvec", +] + [[package]] name = "libc" version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -695,6 +915,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "password-hash" version = "0.4.2" @@ -727,7 +953,7 @@ dependencies = [ "adobe-cmap-parser", "cff-parser", "encoding_rs", - "euclid", + "euclid 0.20.14", "log", "lopdf", "postscript", @@ -735,6 +961,28 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "pdfrs" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd2c58cc563c54ee2dc0dacce61bf686529d792c4b76f031cbcfe8e3f7ecdeb0" +dependencies = [ + "aes", + "anyhow", + "base64 0.22.1", + "cbc", + "clap", + "flate2", + "md-5", + "regex", + "serde", + "serde_json", + "sha2", + "subsetter", + "syntect", + "ttf-parser", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -753,6 +1001,19 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "plist" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da1d65da6dd5d1e44199ac0f58712d241c0f439f80adea8924d832384087f85" +dependencies = [ + "base64 0.22.1", + "indexmap", + "quick-xml", + "serde", + "time", +] + [[package]] name = "png" version = "0.18.1" @@ -766,6 +1027,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "polycool" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50596ddc09eb5ad5f75cacd40209568e66df71baf86e1499a0e99c4cff12a5a6" +dependencies = [ + "arrayvec", +] + [[package]] name = "pom" version = "1.1.0" @@ -790,6 +1060,7 @@ version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed6af693d661395ff3464eac5f7cee1df674d082ce84da94c6819cc799fee929" dependencies = [ + "pdfrs", "thiserror 1.0.69", "uuid", "xml-rs", @@ -871,6 +1142,45 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" +[[package]] +name = "read-fonts" +version = "0.39.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4ed38b89c2c77ff968c524145ad65fb010f38af5c7a224b53b81d47ac2daa81" +dependencies = [ + "bytemuck", + "font-types", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + [[package]] name = "ring" version = "0.17.14" @@ -885,6 +1195,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustix" version = "1.1.4" @@ -939,6 +1255,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "serde" version = "1.0.229" @@ -1025,6 +1350,16 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "skrifa" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c34617370ae968efb7161bb2beb517d9084659aae19e24b89e3db25b46e4564" +dependencies = [ + "bytemuck", + "read-fonts", +] + [[package]] name = "slab" version = "0.4.12" @@ -1048,6 +1383,24 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subsetter" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38803281d1c23166c5ebcb455439a5d2afe711cc909cf88af72448c297756ad6" +dependencies = [ + "kurbo", + "rustc-hash", + "skrifa", + "write-fonts", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1076,6 +1429,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syntect" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "656b45c05d95a5704399aeef6bd0ddec7b2b3531b7c9e900abbf7c4d2190c925" +dependencies = [ + "bincode", + "fancy-regex", + "flate2", + "fnv", + "once_cell", + "plist", + "regex-syntax", + "serde", + "serde_derive", + "serde_json", + "thiserror 2.0.20", + "walkdir", + "yaml-rust", +] + [[package]] name = "tar" version = "0.4.46" @@ -1165,6 +1539,7 @@ dependencies = [ "powerfmt", "serde_core", "time-core", + "time-macros", ] [[package]] @@ -1173,6 +1548,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinybus" version = "0.1.0" @@ -1445,6 +1830,12 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.24.0" @@ -1462,6 +1853,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1528,6 +1929,15 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -1625,6 +2035,19 @@ dependencies = [ "memchr", ] +[[package]] +name = "write-fonts" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb731d4c4d93eacc69a1ad2f270f905788a98e4a3438267bcafbe08d3431c8d8" +dependencies = [ + "font-types", + "indexmap", + "kurbo", + "log", + "read-fonts", +] + [[package]] name = "xattr" version = "1.6.1" @@ -1641,6 +2064,15 @@ version = "0.8.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "zerocopy" version = "0.8.56" diff --git a/Cargo.toml b/Cargo.toml index 4a92a0d..9c8a1a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,13 +40,19 @@ serde = { version = "1", features = ["derive"] } docx-rs = { version = "0.4.20", optional = true } # OOXML `.pptx` synthesis. Optional: exclusive to the `pptx` feature. # -# `default-features = false` turns off `pdf-native`, which exists to export a -# deck as PDF and pulls `pdfrs` -> `syntect` -> `yaml-rust` for the syntax -# highlighting that feature wants. We only ever write `.pptx`, and that tail -# carries three RUSTSEC unmaintained advisories (`yaml-rust`, `bincode`, -# `ttf-parser`). Nothing in this crate calls the PDF exporter, so the whole -# subtree goes. -ppt-rs = { version = "0.2", default-features = false, optional = true } +# Default features are ON, and not by choice. `pdf-native` exists to export a +# deck as PDF and pulls `pdfrs` -> `syntect` -> `yaml-rust`, which is a syntax +# highlighter this crate never reaches — but ppt-rs 0.2.24 does not compile with +# it off: `src/export/slide_render.rs` has an unguarded `use pdfrs::...` with no +# `#[cfg(feature = "pdf-native")]`. Turning the feature off fails to build the +# dependency, not our code. +# +# The cost is three RUSTSEC *unmaintained* advisories carried transitively, +# ignored with justification in `deny.toml`. Revisit the moment ppt-rs fixes the +# cfg: `default-features = false` then drops `pdfrs`, `syntect`, `yaml-rust`, +# `bincode` and `pulldown-cmark` in one line (verified locally — the resolve +# succeeds, only the build fails). +ppt-rs = { version = "0.2", optional = true } # `.pdf` text extraction. Optional: exclusive to the `pdf` feature. It brings a # font and PostScript parsing stack (`lopdf`, CFF/Type1/CMap parsers) that only # the extraction path needs, so a host that never reads a PDF should not carry From ea143ec091063d447e425240dd747469a84615dd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 11 Aug 2026 08:53:20 +0300 Subject: [PATCH 12/13] chore: files changed deny.toml Auto-committed-on: dragonfly Co-authored-by: Medulla --- deny.toml | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/deny.toml b/deny.toml index 25b6aee..af1c140 100644 --- a/deny.toml +++ b/deny.toml @@ -7,7 +7,41 @@ all-features = true [advisories] # Fail on any crate with a security advisory or an unmaintained warning. # Add an entry here only with a comment explaining the exposure and the plan. -ignore = [] +# +# Every entry below is an *unmaintained* notice, not a vulnerability, and every +# one arrives transitively through the two format writers. None is reachable +# from `tinydocs::spec`, so a host that takes this crate with +# `default-features = false` — the wire contract without a codec — pulls none of +# them. +# +# Nothing here silences a vulnerability. RUSTSEC-2026-0187 (stack overflow in +# lopdf via deeply nested PDF objects, a ~21 KB crafted file aborting the +# process with an uncatchable SIGABRT) is *fixed*, not ignored: pdf-extract is +# pinned to 0.12, which resolves lopdf 0.42.0. +ignore = [ + # yaml-rust, via ppt-rs -> pdfrs -> syntect. A YAML parser behind a syntax + # highlighter behind a PDF exporter — three layers from anything this crate + # calls, and `.pptx` synthesis never reaches it. + # + # Plan: it leaves with one line. ppt-rs's `pdf-native` default feature is + # what pulls `pdfrs`, but ppt-rs 0.2.24 does not compile with default + # features off — `src/export/slide_render.rs` has an unguarded + # `use pdfrs::...`. The moment that cfg is fixed upstream, + # `default-features = false` drops this crate, `syntect`, `bincode` and + # `pulldown-cmark` together. Verified locally: the resolve succeeds and the + # advisories clear; only ppt-rs's own build fails. + "RUSTSEC-2024-0320", + # bincode, via ppt-rs -> pdfrs -> syntect. Same subtree, same one-line fix. + "RUSTSEC-2025-0141", + # ttf-parser, via pdf-extract -> lopdf. lopdf's font parser, so it is on the + # extraction path rather than an unused corner — but the advisory is the + # author declaring the crate finished, not a defect. No alternative without + # replacing lopdf, which is pdf-extract's core dependency. + # + # Plan: track lopdf. It is actively maintained (0.42.0 shipped the fix for + # RUSTSEC-2026-0187), so a font-parser migration would come from there. + "RUSTSEC-2026-0192", +] [licenses] # Licenses accepted for this crate and its dependencies. Keep GPL-3.0-only for @@ -18,6 +52,10 @@ allow = [ "Apache-2.0 WITH LLVM-exception", "BSD-2-Clause", "BSD-3-Clause", + # constant_time_eq 0.1.5, via ppt-rs -> zip 0.6.6. A public-domain + # dedication — strictly more permissive than the rest of this list, and it + # imposes no obligation on a GPL-3.0-only crate. + "CC0-1.0", "GPL-3.0-only", "ISC", "MIT", From c97d2aca4d76729a1ad05c40cf3169d878231623 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 11 Aug 2026 09:02:15 +0300 Subject: [PATCH 13/13] Address review: overflow, capability ids, TTL, and the lopdf CVE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ten findings from the review, all legitimate. Grouped by what they actually were. **A real vulnerability, fixed rather than ignored.** `pdf-extract` was pinned at 0.10, which resolves lopdf 0.38 — RUSTSEC-2026-0187, an uncatchable stack-overflow SIGABRT from a ~21 KB PDF with deeply nested objects. The `%PDF-` and size checks do nothing against it. Pinning `pdf-extract = "0.12"` resolves lopdf 0.42.0, where it is fixed. This matters more than the advisory count suggests: the extraction path is the one that sees untrusted user attachments. **Caller-controlled arithmetic could wrap.** Every `byte_len` in a wire deck comes from the caller, and the total was summed unchecked: `u64::MAX + 1` wraps to zero in a release build, a zero-byte stream then satisfies the aggregate check, and the first slice panics. Each length is now bounded against `MAX_IMAGE_BYTES` before it is summed, the sum and the cursor use `checked_add`, and slicing goes through `payload.get(..)` so a bad range is an `InvalidInput` rather than a panic. **Output ids were authorisation, and were guessable.** `ReadOutput` and `ReleaseOutput` take an id and nothing else — a method receives no caller identity, so the store cannot bind an output to whoever produced it. That makes the id the whole authorisation story, and it was `out-1`, `out-2`. Any peer on the bus could take another peer's document. Ids are now 128 bits of OS randomness. My own comment claiming ids were "never authorisation tokens" was the tell that this was wrong; it is corrected rather than deleted. **Empty reads refreshed the TTL.** A zero-length read, or one at the exact end of a document, returns nothing and costs nothing — and reset `last_read`, so repeating it pinned an output in the store forever. Only a read that returned bytes counts as activity now. **A valid JPEG could be rejected.** TEM (`0xFF01`) is a standalone marker with no length field. The walk read the following two bytes as one, desynchronised, and gave up on a file that places TEM before its frame header. **Two of my own doc comments contradicted the code**, which is worth fixing loudly. The pptx module claimed images are never upscaled past their natural size; the implementation scales to fill the slot in both directions, which is what the code this was ported from did and what its test asserts — so the sentence was wrong, not the behaviour. And the spec said "seven methods" over a list of five, named `Error.UnknownBlob` / `Error.TransferRefused` where the code defines `UnknownOutput` / `OutputRefused` (a caller matching the documented names would never match), and still called outputs "blobs" after that concept was removed. **Supply chain.** Three unmaintained advisories remain, ignored in `deny.toml` with the exposure and the plan written out, per that file's own rule. Two of them (`yaml-rust`, `bincode`) arrive through ppt-rs's `pdf-native` default feature — a PDF exporter behind a syntax highlighter that `.pptx` synthesis never reaches. It cannot simply be turned off: ppt-rs 0.2.24 fails to build with `default-features = false` because `src/export/slide_render.rs` has an unguarded `use pdfrs::...`. When that is fixed upstream, one line drops `pdfrs`, `syntect`, `yaml-rust`, `bincode` and `pulldown-cmark` together. `CC0-1.0` is added to the license allowlist for `constant_time_eq` — a public-domain dedication, strictly more permissive than everything already on the list. The E2E's mismatch case now asserts the wire error name instead of only that the call failed, which would also have passed if it failed for an unrelated reason. New regression tests: empty reads not extending the TTL, ids being unguessable and unique, and a TEM-before-SOF JPEG. Co-authored-by: Medulla --- Cargo.lock | 36 ++++++++++++- crates/tinydocs-module/Cargo.toml | 4 ++ crates/tinydocs-module/src/outputs/mod.rs | 43 +++++++++++++--- crates/tinydocs-module/src/outputs/test.rs | 60 ++++++++++++++++++++++ crates/tinydocs-module/src/service/mod.rs | 48 ++++++++++++++--- crates/tinydocs-module/tests/module_e2e.rs | 14 +++-- docs/specs/tinybus-module.md | 13 ++--- src/pptx/mod.rs | 10 ++-- src/spec/image/mod.rs | 13 +++-- src/spec/image/test.rs | 12 +++++ src/spec/presentation/wire.rs | 3 +- 11 files changed, 221 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e82043c..b39c6eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -643,6 +643,18 @@ dependencies = [ "wasi", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -652,7 +664,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 6.0.0", "rand_core 0.10.1", "wasm-bindgen", ] @@ -1107,6 +1119,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" @@ -1616,6 +1634,7 @@ name = "tinydocs-module" version = "0.1.11" dependencies = [ "base64 0.22.1", + "getrandom 0.3.4", "serde", "serde_json", "sha2", @@ -1869,6 +1888,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.127" @@ -2035,6 +2063,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "write-fonts" version = "0.48.1" diff --git a/crates/tinydocs-module/Cargo.toml b/crates/tinydocs-module/Cargo.toml index 247137b..7a332f3 100644 --- a/crates/tinydocs-module/Cargo.toml +++ b/crates/tinydocs-module/Cargo.toml @@ -28,6 +28,10 @@ base64 = "0.22" sha2 = "0.10" # The staging area's refusals are a taxonomy the caller matches on. thiserror = "2" +# An output id is the only authorisation to read that output — a method receives +# no caller identity — so ids are 128 bits of OS randomness rather than a +# guessable counter. +getrandom = "0.3" # `BlobRef` and the wire deck shape are the bus contract, so they derive the # same serde surface the library specs do. serde = { version = "1", features = ["derive"] } diff --git a/crates/tinydocs-module/src/outputs/mod.rs b/crates/tinydocs-module/src/outputs/mod.rs index 3cbad4a..0e9b877 100644 --- a/crates/tinydocs-module/src/outputs/mod.rs +++ b/crates/tinydocs-module/src/outputs/mod.rs @@ -204,10 +204,15 @@ impl OutputStore { if start > output.bytes.len() { return Err(OutputError::ReadPastEnd); } - // Reading is what keeps an output alive: a caller working through a - // large document in chunks must not have it reaped mid-read. - output.last_read = now; let end = start.saturating_add(len).min(output.bytes.len()); + + // Only a read that returned bytes counts as activity. A zero-length + // read, or one at the exact end of the document, is free to issue and + // would otherwise refresh the TTL forever — which is a way to pin an + // output in the store indefinitely without ever consuming it. + if end > start { + output.last_read = now; + } Ok(output.bytes[start..end].to_vec()) } @@ -263,14 +268,36 @@ impl Inner { .retain(|_, output| now.saturating_duration_since(output.last_read) <= IDLE_TTL); } - /// Allocate an unused output id. + /// Allocate an unguessable output id. + /// + /// An id **is** the authorisation to read and release an output, whether or + /// not it was designed to be: `Interface::call` hands a method no caller + /// identity, so the store cannot bind an output to the peer that produced + /// it and has nothing else to check. A counter would let any peer on the bus + /// read `out-3` and take somebody else's document. /// - /// A counter, not a random value: ids are opaque handles inside one process, - /// never authorisation tokens, and a counter makes a leaked id visible in a - /// log rather than looking like a secret. + /// In this crate's own host that bus has exactly one client, but the module + /// is loadable by anyone, so the capability is 128 bits of OS randomness + /// rather than an assumption about the deployment. The counter is kept + /// alongside it purely so two ids in a log are orderable. fn allocate_id(&mut self) -> String { self.next_id = self.next_id.wrapping_add(1); - format!("out-{}", self.next_id) + let mut bytes = [0u8; 16]; + // A failure here means the OS entropy source is unavailable, which is + // not a condition this module can paper over with a weaker id — fall + // back to the digest of the counter and the address of this store, which + // is at least not enumerable from outside the process. + if getrandom::fill(&mut bytes).is_err() { + let seed = format!("{:p}:{}", std::ptr::from_ref(self), self.next_id); + let digest = Sha256::digest(seed.as_bytes()); + bytes.copy_from_slice(&digest[..16]); + } + let mut id = String::with_capacity(32); + for byte in bytes { + use std::fmt::Write as _; + let _ = write!(id, "{byte:02x}"); + } + id } } diff --git a/crates/tinydocs-module/src/outputs/test.rs b/crates/tinydocs-module/src/outputs/test.rs index c95f149..3dffe24 100644 --- a/crates/tinydocs-module/src/outputs/test.rs +++ b/crates/tinydocs-module/src/outputs/test.rs @@ -158,6 +158,66 @@ fn reading_keeps_a_slow_consumer_alive() { assert_eq!(read, document); } +#[test] +fn empty_reads_do_not_keep_an_output_alive() { + // A zero-length read costs a caller nothing and returns nothing. If it + // refreshed the TTL, repeating it would pin an output in the store forever + // without ever consuming it. + let store = OutputStore::new(); + let mut now = t0(); + let handle = store.insert(b"payload".to_vec(), now).unwrap(); + + // Poke it repeatedly while still inside the window. Both shapes that return + // nothing: a zero length, and a read at the exact end of the document. + for _ in 0..4 { + now += Duration::from_secs(30); + assert!( + store + .read_chunk(&handle.output_id, 0, 0, now) + .unwrap() + .is_empty() + ); + assert!( + store + .read_chunk(&handle.output_id, 7, 100, now) + .unwrap() + .is_empty() + ); + } + + // Past the TTL measured from the *insert*, because none of those reads + // counted as activity. A real read at any point above would have kept it. + now += IDLE_TTL; + assert_eq!( + store.read_chunk(&handle.output_id, 0, 10, now), + Err(OutputError::UnknownOutput) + ); +} + +#[test] +fn output_ids_are_unguessable() { + // An id is the only authorisation to read an output — a method receives no + // caller identity — so a sequential id would let any peer on the bus take + // somebody else's document. + let store = OutputStore::new(); + let now = t0(); + let ids: Vec = (0u8..8) + .map(|i| store.insert(vec![i; 4], now).unwrap().output_id) + .collect(); + + for id in &ids { + assert_eq!(id.len(), 32, "expected 128 bits of hex, got {id}"); + assert!( + id.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)), + "id is not lowercase hex: {id}" + ); + assert!(!id.starts_with("out-"), "id is still counter-derived: {id}"); + } + let unique: std::collections::HashSet<&String> = ids.iter().collect(); + assert_eq!(unique.len(), ids.len(), "ids repeated"); +} + #[test] fn releasing_frees_the_budget_and_is_reported_once() { let store = OutputStore::new(); diff --git a/crates/tinydocs-module/src/service/mod.rs b/crates/tinydocs-module/src/service/mod.rs index e299cf2..93ada22 100644 --- a/crates/tinydocs-module/src/service/mod.rs +++ b/crates/tinydocs-module/src/service/mod.rs @@ -52,6 +52,7 @@ use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; use tinybus::stream::StreamRef; use tinybus::{Connection, Error as BusError, Result as BusResult}; +use tinydocs::spec::presentation::MAX_IMAGE_BYTES; use tinydocs::spec::{DocumentSpec, PresentationSpec, SlideImage, SlideSpec}; use tinydocs::{Error, pdf, pptx}; @@ -170,12 +171,29 @@ impl Documents { spec: WirePresentationSpec, images: Option, ) -> BusResult { - let expected: u64 = spec - .slides - .iter() - .flat_map(|slide| slide.images.iter()) - .map(|image| image.byte_len) - .sum(); + // Every length is caller-controlled, so the arithmetic is checked and + // each one is bounded before it is summed. `u64::MAX + 1` wraps to zero + // in a release build, which would let a zero-byte stream satisfy the + // aggregate check and then panic on the first slice. + let mut expected: u64 = 0; + for image in spec.slides.iter().flat_map(|slide| slide.images.iter()) { + if image.byte_len > MAX_IMAGE_BYTES as u64 { + return Err(BusError::MethodFailed { + name: INVALID_INPUT_ERROR.to_string(), + message: format!( + "an image declares {} bytes, over the {MAX_IMAGE_BYTES}-byte limit", + image.byte_len + ), + }); + } + expected = + expected + .checked_add(image.byte_len) + .ok_or_else(|| BusError::MethodFailed { + name: INVALID_INPUT_ERROR.to_string(), + message: "declared image lengths overflow".to_string(), + })?; + } let payload = match (&images, expected) { (Some(stream), _) => self.read_stream(stream).await?, @@ -203,13 +221,27 @@ impl Documents { for slide in spec.slides { let mut resolved = Vec::with_capacity(slide.images.len()); for image in slide.images { + // Bounded above, so this cannot truncate; `checked_add` and a + // fallible slice keep the walk honest anyway rather than + // trusting the loop that produced `expected`. let len = usize::try_from(image.byte_len).map_err(|_| BusError::MethodFailed { name: INVALID_INPUT_ERROR.to_string(), message: "image length is out of range".to_string(), })?; - let end = cursor + len; + let end = cursor + .checked_add(len) + .ok_or_else(|| BusError::MethodFailed { + name: INVALID_INPUT_ERROR.to_string(), + message: "image offsets overflow".to_string(), + })?; + let bytes = payload + .get(cursor..end) + .ok_or_else(|| BusError::MethodFailed { + name: INVALID_INPUT_ERROR.to_string(), + message: "declared image lengths do not fit the image stream".to_string(), + })?; resolved.push( - SlideImage::from_bytes(payload[cursor..end].to_vec(), image.caption) + SlideImage::from_bytes(bytes.to_vec(), image.caption) .map_err(|error| map_error(&error))?, ); cursor = end; diff --git a/crates/tinydocs-module/tests/module_e2e.rs b/crates/tinydocs-module/tests/module_e2e.rs index b115b54..bb29cb4 100644 --- a/crates/tinydocs-module/tests/module_e2e.rs +++ b/crates/tinydocs-module/tests/module_e2e.rs @@ -259,10 +259,16 @@ async fn refuses_a_stream_that_contradicts_the_spec(client: &Connection, target: b"too short", ) .await; - assert!( - mismatched.is_err(), - "a stream shorter than the declared images should be refused" - ); + // The wire name, not merely "it failed": `is_err()` alone would also pass if + // the call were rejected for an unrelated reason, which would hide the very + // check this case exists to prove. + match mismatched { + Err(tinybus::Error::MethodFailed { name, .. }) => assert_eq!( + name, "ai.tinyhumans.tinydocs.Error.InvalidInput", + "a length mismatch should be reported as invalid input" + ), + other => panic!("expected an InvalidInput refusal, got {other:?}"), + } } /// Read a held document back in chunks and verify its digest. diff --git a/docs/specs/tinybus-module.md b/docs/specs/tinybus-module.md index c38ac5a..5934468 100644 --- a/docs/specs/tinybus-module.md +++ b/docs/specs/tinybus-module.md @@ -32,7 +32,7 @@ The private `tinydocs-module` workspace crate depends on the public library's `docx`, `pptx` and `pdf` features and builds as a `cdylib`. This separation keeps unpublished, vendored TinyBus packages out of the crates.io package manifest. The module claims `ai.tinyhumans.tinydocs.Documents`, serves the object path -`/ai/tinyhumans/tinydocs/Documents`, and exports seven methods: +`/ai/tinyhumans/tinydocs/Documents`, and exports five methods: ```text GenerateDocx(DocumentSpec) -> OutputRef @@ -60,13 +60,14 @@ Invalid input, writer failures and extraction failures use the distinct wire names `ai.tinyhumans.tinydocs.Error.InvalidInput`, `ai.tinyhumans.tinydocs.Error.GenerationFailed` and `ai.tinyhumans.tinydocs.Error.ExtractionFailed`. Transfer failures are grouped by -what the caller should do next: `Error.UnknownBlob` (restart the transfer), -`Error.TransferRefused` (a budget is full; the same request may succeed later) -and `Error.TransferFailed` (the caller sent something wrong; re-send). +what the caller should do next: `Error.UnknownOutput` (the document is gone; +make the call again), `Error.OutputRefused` (a budget is full; the same request +may succeed later) and `Error.TransferFailed` (the read was malformed, or an +inbound stream did not complete). Synthesis and extraction are CPU-bound and run on the module runtime's blocking -pool. The module retains no document state between calls — only staged blobs, -each bounded and expiring. +pool. The module retains no document state between calls — only produced documents +waiting to be read, each bounded and expiring. This interface replaces `ai.tinyhumans.tinydocs.Docx`, which returned bytes inline. TinyBus forbids changing an interface in place, so the new contract took diff --git a/src/pptx/mod.rs b/src/pptx/mod.rs index d965d3a..47a2bcc 100644 --- a/src/pptx/mod.rs +++ b/src/pptx/mod.rs @@ -33,10 +33,12 @@ //! # Image layout //! //! Images stack in a single vertical column in the lower band of the slide, -//! beneath the text. Each is scaled to fit its slot with its aspect ratio -//! preserved and is centred in both axes; a slot is never upscaled past the -//! source's natural size ratio. Every dimension below is in EMU (English Metric -//! Units, 914,400 per inch), the unit OOXML itself uses. +//! beneath the text. Each is scaled to fill its slot with its aspect ratio +//! preserved and is centred in both axes. Scaling goes **both ways**: an image +//! smaller than its slot is enlarged to touch it on one axis, which is what a +//! deck wants — a 64×64 chart rendered at 64×64 on a ten-inch slide reads as a +//! mistake. Every dimension below is in EMU (English Metric Units, 914,400 per +//! inch), the unit OOXML itself uses. // The spec is defined in `crate::spec`, which is compiled in every build so a // host can share the wire contract without the OOXML writer stack. Re-exported diff --git a/src/spec/image/mod.rs b/src/spec/image/mod.rs index bb51c47..cbfe754 100644 --- a/src/spec/image/mod.rs +++ b/src/spec/image/mod.rs @@ -100,9 +100,16 @@ fn jpeg_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { } let marker = bytes[i + 1]; i += 2; - // Standalone markers (no length field): padding fill bytes and - // RSTn / SOI / EOI. Skip without consuming a segment length. - if marker == 0xFF || marker == 0xD8 || marker == 0xD9 || (0xD0..=0xD7).contains(&marker) { + // Standalone markers carry no length field: padding fill bytes, TEM, + // RSTn, SOI and EOI. Reading the next two bytes as a length here would + // desynchronise the walk and reject a valid file — TEM in particular is + // legal before the frame header. + if marker == 0xFF + || marker == 0x01 + || marker == 0xD8 + || marker == 0xD9 + || (0xD0..=0xD7).contains(&marker) + { continue; } if i + 1 >= bytes.len() { diff --git a/src/spec/image/test.rs b/src/spec/image/test.rs index 8cd7322..3519176 100644 --- a/src/spec/image/test.rs +++ b/src/spec/image/test.rs @@ -123,6 +123,18 @@ fn a_jpeg_skips_standalone_and_non_frame_markers_before_the_frame() { assert_eq!(jpeg_dimensions(&bytes), Some((22, 11))); } +#[test] +fn a_jpeg_with_a_tem_marker_before_the_frame_is_still_measured() { + // TEM (0xFF01) carries no length field. Reading the next two bytes as one + // desynchronises the walk and rejects a valid file. + let mut bytes = vec![0xFF, 0xD8, 0xFF, 0x01]; + bytes.extend_from_slice(&[0xFF, 0xC0, 0x00, 0x0B, 0x08]); + bytes.extend_from_slice(&33u16.to_be_bytes()); // height + bytes.extend_from_slice(&44u16.to_be_bytes()); // width + bytes.extend_from_slice(&[0x03, 0x00, 0x00, 0x00, 0xFF, 0xD9]); + assert_eq!(jpeg_dimensions(&bytes), Some((44, 33))); +} + #[test] fn format_renders_its_ooxml_name() { assert_eq!(ImageFormat::Png.as_str(), "PNG"); diff --git a/src/spec/presentation/wire.rs b/src/spec/presentation/wire.rs index 4251869..e8e3859 100644 --- a/src/spec/presentation/wire.rs +++ b/src/spec/presentation/wire.rs @@ -20,7 +20,8 @@ use serde::{Deserialize, Serialize}; -/// A slide image, as it appears on the bus: a reference to a staged blob. +/// A slide image, as it appears on the bus: one byte range of the concatenated +/// image stream that travels beside the call. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct WireSlideImage {