From 02f0b5f93d929cc35d950efeae884cc9ca706b14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 23:31:35 +0200 Subject: [PATCH 1/8] Add portable RustiQ scientific artifact containers --- Cargo.lock | 22 +- crates/rustiq-core/Cargo.toml | 3 +- .../src/persistence/calculation.rs | 466 +++++++++++++ crates/rustiq-core/src/persistence/data.rs | 2 + .../src/persistence/data/portable.rs | 217 ++++++ .../src/persistence/data/portable/tests.rs | 631 ++++++++++++++++++ .../src/persistence/data/rustiq_data.rs | 60 +- crates/rustiq-core/src/persistence/error.rs | 47 ++ .../rustiq-core/src/persistence/identity.rs | 56 +- .../rustiq-core/src/persistence/manifest.rs | 15 + crates/rustiq-core/src/persistence/mod.rs | 4 +- crates/rustiq-core/src/persistence/npy.rs | 51 ++ crates/rustiq-core/src/persistence/storage.rs | 89 ++- .../src/persistence/storage/zip.rs | 336 ++++++++++ .../data/persistence/calculation-h2-v1.json | 101 +++ .../portable-python-big-v1.rustiq.hex | 1 + .../portable-python-little-v1.rustiq.hex | 1 + crates/rustiq-core/tests/portable_artifact.rs | 56 ++ docs/persistence-format-v1.md | 193 +++++- schemas/calculation-snapshot-v1.schema.json | 379 +++++++++++ tools/reference/generate_portable_fixtures.py | 124 ++++ tools/reference/test_persistence_interop.py | 78 ++- 22 files changed, 2878 insertions(+), 54 deletions(-) create mode 100644 crates/rustiq-core/src/persistence/calculation.rs create mode 100644 crates/rustiq-core/src/persistence/data/portable.rs create mode 100644 crates/rustiq-core/src/persistence/data/portable/tests.rs create mode 100644 crates/rustiq-core/src/persistence/storage/zip.rs create mode 100644 crates/rustiq-core/tests/data/persistence/calculation-h2-v1.json create mode 100644 crates/rustiq-core/tests/data/persistence/portable-python-big-v1.rustiq.hex create mode 100644 crates/rustiq-core/tests/data/persistence/portable-python-little-v1.rustiq.hex create mode 100644 crates/rustiq-core/tests/portable_artifact.rs create mode 100644 schemas/calculation-snapshot-v1.schema.json create mode 100644 tools/reference/generate_portable_fixtures.py diff --git a/Cargo.lock b/Cargo.lock index 1df32a16..e483fcff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -902,7 +902,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aad4cf7e378685c1af30f41b4832c29700325524c648d47752cabd5b97e7b09b" dependencies = [ - "zip", + "zip 0.6.6", ] [[package]] @@ -2651,6 +2651,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "zip 8.6.0", ] [[package]] @@ -3523,6 +3524,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -4201,6 +4208,19 @@ dependencies = [ "flate2", ] +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", +] + [[package]] name = "zlib-rs" version = "0.6.7" diff --git a/crates/rustiq-core/Cargo.toml b/crates/rustiq-core/Cargo.toml index 3d156b35..d0cfb1c8 100644 --- a/crates/rustiq-core/Cargo.toml +++ b/crates/rustiq-core/Cargo.toml @@ -29,7 +29,7 @@ periodic_table.workspace = true rayon.workspace = true relative-path.workspace = true tokio = { workspace = true, features = ["fs", "io-util"], optional = true } -serde_json.workspace = true +serde_json = { workspace = true, features = ["float_roundtrip"] } serde.workspace = true sha2 = "0.11.0" delegate.workspace = true @@ -47,6 +47,7 @@ smallvec = "1" tempfile.workspace = true bytesize.workspace = true sysinfo.workspace = true +zip = { version = "8", default-features = false, features = ["deflate-flate2-zlib-rs"] } [dev-dependencies] approx.workspace = true diff --git a/crates/rustiq-core/src/persistence/calculation.rs b/crates/rustiq-core/src/persistence/calculation.rs new file mode 100644 index 00000000..c3bc4051 --- /dev/null +++ b/crates/rustiq-core/src/persistence/calculation.rs @@ -0,0 +1,466 @@ +//! Explicit V1 wire records; domain structs are never serialized as the format. +use serde::{Deserialize, Serialize}; + +use super::{ + identity::CanonicalBytes, PortableError, ScientificIdentity, AO_ERI_COMPUTATION_VERSION, +}; +use crate::{ + calculation::PreparedCalculation, + config::{ + random_config::{DistributionConfig, RandomConfig}, + DensityGuessConfig, ResolvedHfMethod, + }, +}; + +pub(crate) const CALCULATION_PATH: &str = "calculation.json"; +pub(crate) const MAX_CALCULATION_BYTES: u64 = 64 * 1024 * 1024; + +/// Read-only, resolved scientific context. Coordinates and centers are in Bohr. +#[derive(Clone, Debug)] +pub struct CalculationContext(pub(crate) Snapshot); + +impl CalculationContext { + pub fn atoms(&self) -> impl ExactSizeIterator + '_ { + self.0 + .atoms + .iter() + .map(|atom| (atom.atomic_number, atom.position)) + } + pub fn charge(&self) -> i32 { + self.0.charge + } + pub fn multiplicity(&self) -> u8 { + self.0.multiplicity + } + pub fn basis(&self) -> impl ExactSizeIterator> { + self.0.basis.iter().map(ResolvedAo) + } + pub fn hf_method(&self) -> ResolvedHfMethod { + match self.0.hf.method { + Method::Rhf => ResolvedHfMethod::Rhf, + Method::Uhf => ResolvedHfMethod::Uhf, + } + } + pub fn max_iterations(&self) -> usize { + self.0.hf.max_iterations + } + pub fn convergence_threshold(&self) -> f64 { + self.0.hf.convergence_threshold + } + pub fn linear_dependency_threshold(&self) -> f64 { + self.0.hf.linear_dependency_threshold + } + pub fn eri_schwarz_threshold(&self) -> Option { + self.0.hf.eri_schwarz_threshold + } + pub fn diis_size(&self) -> Option { + self.0.hf.diis_size + } + /// `None` means HF only; `Some(n)` requests MP2 with n frozen orbitals. + pub fn mp2_frozen_orbitals(&self) -> Option { + self.0.mp2.as_ref().map(|mp2| mp2.frozen_orbitals) + } + /// The initial density strategy; random settings do not promise an exact restart. + pub fn density_guess(&self) -> crate::config::DensityGuessConfig { + self.0.hf.guess.to_config() + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct Snapshot { + format: String, + version: u32, + units: String, + atoms: Vec, + charge: i32, + multiplicity: u8, + basis: Vec, + hf: Hf, + mp2: Option, + ao_eri_computation_version: u32, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Atom { + atomic_number: u32, + position: [f64; 3], +} + +/// One effective AO, in the same order as the integral engine. +#[derive(Clone, Copy, Debug)] +pub struct ResolvedAo<'a>(&'a Ao); +impl<'a> ResolvedAo<'a> { + pub fn center(&self) -> [f64; 3] { + self.0.center + } + pub fn components(&self) -> impl ExactSizeIterator> { + self.0.components.iter().map(ResolvedComponent) + } +} +/// A Cartesian component with effective normalized primitive coefficients. +#[derive(Clone, Copy, Debug)] +pub struct ResolvedComponent<'a>(&'a Component); +impl ResolvedComponent<'_> { + pub fn angular_momentum(&self) -> [u8; 3] { + self.0.angular_momentum + } + /// Returns (exponent, effective normalized coefficient) pairs. + pub fn primitives(&self) -> impl ExactSizeIterator + '_ { + self.0 + .primitives + .iter() + .map(|p| (p.exponent, p.coefficient)) + } +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Ao { + center: [f64; 3], + components: Vec, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Component { + angular_momentum: [u8; 3], + primitives: Vec, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Primitive { + exponent: f64, + coefficient: f64, +} +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +enum Method { + Rhf, + Uhf, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Hf { + method: Method, + max_iterations: usize, + convergence_threshold: f64, + linear_dependency_threshold: f64, + eri_schwarz_threshold: Option, + diis_size: Option, + guess: Guess, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Mp2 { + frozen_orbitals: usize, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum Guess { + CoreHamiltonian { perturbation: Option }, + OneElectron { perturbation: Option }, + Random { random: Random }, + Zero, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Random { + seed: Option, + distribution: Distribution, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum Distribution { + Uniform { min: f64, max: f64 }, + Normal { mean: f64, std_dev: f64 }, +} + +impl Snapshot { + pub(crate) fn from_calculation( + calculation: &PreparedCalculation, + ) -> Result { + let molecule = calculation.get_molecule(); + let basis = calculation.get_basis(); + let hf = calculation.hf_config(); + let method = calculation.hf_method(); + let mp2 = calculation.mp2_config(); + let snapshot = Self { + format: "rustiq-calculation".into(), + version: 1, + units: "bohr".into(), + atoms: molecule + .atoms + .iter() + .map(|atom| Atom { + atomic_number: atom.element.atomic_number, + position: atom.position.into(), + }) + .collect(), + charge: molecule.charge(), + multiplicity: molecule.multiplicity().get(), + basis: basis + .shell_ids + .iter() + .zip(&basis.normalized_components) + .map(|(shell, components)| Ao { + center: basis.shells[*shell].origin.into(), + components: components + .iter() + .map(|component| Component { + angular_momentum: component.angular_momentum.into(), + primitives: component + .primitives + .iter() + .map(|p| Primitive { + exponent: p.exponent, + coefficient: p.coefficient, + }) + .collect(), + }) + .collect(), + }) + .collect(), + hf: Hf { + method: match method { + ResolvedHfMethod::Rhf => Method::Rhf, + ResolvedHfMethod::Uhf => Method::Uhf, + }, + max_iterations: hf.max_iterations.get(), + convergence_threshold: hf.convergence_threshold.into_inner(), + linear_dependency_threshold: hf.linear_dependency_threshold.value.into_inner(), + eri_schwarz_threshold: hf.eri_schwarz_threshold.map(|value| value.into_inner()), + diis_size: hf.diis.then(|| hf.diis_size.into_inner()), + guess: match hf.guess.value { + DensityGuessConfig::CoreHamiltonian { perturbation } => { + Guess::CoreHamiltonian { + perturbation: perturbation.map(|p| p.random.into()), + } + } + DensityGuessConfig::OneElectron { perturbation } => Guess::OneElectron { + perturbation: perturbation.map(|p| p.random.into()), + }, + DensityGuessConfig::Random { config } => Guess::Random { + random: config.random.into(), + }, + DensityGuessConfig::Zero => Guess::Zero, + }, + }, + mp2: mp2.map(|mp2| Mp2 { + frozen_orbitals: mp2.frozen_orbitals.value, + }), + ao_eri_computation_version: AO_ERI_COMPUTATION_VERSION, + }; + snapshot.validate()?; + Ok(snapshot) + } + + pub(crate) fn validate(&self) -> Result<(), PortableError> { + if self.format != "rustiq-calculation" || self.version != 1 || self.units != "bohr" { + return Err(PortableError::UnsupportedVersion); + } + let invalid = |message: &str| PortableError::InvalidCalculation(message.into()); + if self.atoms.is_empty() + || self.basis.is_empty() + || self.multiplicity == 0 + || self.ao_eri_computation_version == 0 + { + return Err(invalid( + "empty molecule/basis or invalid version/multiplicity", + )); + } + let mut nuclear_charge = 0_i64; + for atom in &self.atoms { + if !(1..=118).contains(&atom.atomic_number) + || !atom.position.iter().all(|v| v.is_finite()) + { + return Err(invalid("invalid atom")); + } + nuclear_charge += i64::from(atom.atomic_number); + } + let electrons = nuclear_charge - i64::from(self.charge); + let unpaired = i64::from(self.multiplicity) - 1; + if electrons <= 0 + || electrons < unpaired + || (electrons - unpaired) % 2 != 0 + || (matches!(self.hf.method, Method::Rhf) && (unpaired != 0 || electrons % 2 != 0)) + { + return Err(invalid( + "inconsistent electron count, multiplicity or HF method", + )); + } + for ao in &self.basis { + if !ao.center.iter().all(|v| v.is_finite()) || ao.components.is_empty() { + return Err(invalid("invalid AO center or empty components")); + } + for component in &ao.components { + if component.primitives.is_empty() + || component + .primitives + .iter() + .any(|p| !positive(p.exponent) || !p.coefficient.is_finite()) + { + return Err(invalid("invalid AO primitives")); + } + } + } + if crate::eri::CompactEri::checked_storage_len(self.basis.len()) + .and_then(|n| n.checked_mul(8)) + .is_none() + { + return Err(invalid("basis dimensions overflow")); + } + if self.hf.max_iterations == 0 + || !positive(self.hf.convergence_threshold) + || !self.hf.linear_dependency_threshold.is_finite() + || self.hf.linear_dependency_threshold < 0.0 + || self.hf.eri_schwarz_threshold.is_some_and(|v| !positive(v)) + || self.hf.diis_size.is_some_and(|v| v < 2) + { + return Err(invalid("invalid HF settings")); + } + let random = match &self.hf.guess { + Guess::CoreHamiltonian { perturbation } | Guess::OneElectron { perturbation } => { + perturbation.as_ref() + } + Guess::Random { random } => Some(random), + Guess::Zero => None, + }; + if let Some(random) = random { + let valid = match random.distribution { + Distribution::Uniform { min, max } => { + min.is_finite() && max.is_finite() && min < max && (max - min).is_finite() + } + Distribution::Normal { mean, std_dev } => mean.is_finite() && positive(std_dev), + }; + if !valid { + return Err(invalid("invalid random distribution")); + } + } + Ok(()) + } + + pub(crate) fn basis_functions(&self) -> usize { + self.basis.len() + } + pub(crate) fn computation_version(&self) -> u32 { + self.ao_eri_computation_version + } + pub(crate) fn identity(&self) -> ScientificIdentity { + let mut bytes = CanonicalBytes::ao_eri(self.ao_eri_computation_version); + bytes.len(self.atoms.len()); + for atom in &self.atoms { + bytes.u32(atom.atomic_number); + for coordinate in atom.position { + bytes.f64(coordinate); + } + } + bytes.len(self.basis.len()); + for ao in &self.basis { + for coordinate in ao.center { + bytes.f64(coordinate); + } + bytes.len(ao.components.len()); + for component in &ao.components { + for axis in component.angular_momentum { + bytes.u8(axis); + } + bytes.len(component.primitives.len()); + for primitive in &component.primitives { + bytes.f64(primitive.exponent); + bytes.f64(primitive.coefficient); + } + } + } + bytes.finish_ao_eri(self.hf.eri_schwarz_threshold) + } +} +fn positive(value: f64) -> bool { + value.is_finite() && value > 0.0 +} +impl From for Random { + fn from(config: RandomConfig) -> Self { + Self { + seed: config.seed, + distribution: match config.distribution { + DistributionConfig::Uniform { config } => Distribution::Uniform { + min: config.min, + max: config.max, + }, + DistributionConfig::Normal { config } => Distribution::Normal { + mean: config.mean, + std_dev: config.std_dev.into_inner(), + }, + }, + } + } +} +impl Random { + fn to_config(&self) -> RandomConfig { + use crate::config::random_config::distribution_config::{ + NormalDistributionConfig, UniformDistributionConfig, + }; + RandomConfig { + seed: self.seed, + distribution: match self.distribution { + Distribution::Uniform { min, max } => DistributionConfig::Uniform { + config: UniformDistributionConfig { min, max }, + }, + Distribution::Normal { mean, std_dev } => DistributionConfig::Normal { + config: NormalDistributionConfig { + mean, + std_dev: crate::config::validated::PositiveFiniteF64::try_new(std_dev) + .expect("validated snapshot"), + }, + }, + }, + } + } +} +impl Guess { + fn to_config(&self) -> DensityGuessConfig { + use crate::config::{GuessPerturbationConfig, RandomGuessConfig}; + match self { + Self::CoreHamiltonian { perturbation } => DensityGuessConfig::CoreHamiltonian { + perturbation: perturbation.as_ref().map(|p| GuessPerturbationConfig { + random: p.to_config(), + }), + }, + Self::OneElectron { perturbation } => DensityGuessConfig::OneElectron { + perturbation: perturbation.as_ref().map(|p| GuessPerturbationConfig { + random: p.to_config(), + }), + }, + Self::Random { random } => DensityGuessConfig::Random { + config: RandomGuessConfig { + random: random.to_config(), + }, + }, + Self::Zero => DensityGuessConfig::Zero, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + + proptest! { + #[test] + fn json_round_trip_preserves_binary64_scientific_identity( + coordinate in any::().prop_filter("finite coordinate", |v| v.is_finite()), + exponent in any::().prop_filter("positive finite exponent", |v| v.is_finite() && *v > 0.0), + coefficient in any::().prop_filter("finite coefficient", |v| v.is_finite()), + ) { + let mut snapshot: Snapshot = serde_json::from_str(include_str!("../../tests/data/persistence/calculation-h2-v1.json")).unwrap(); + snapshot.atoms[0].position[0] = coordinate; + snapshot.basis[0].center[0] = coordinate; + snapshot.basis[0].components[0].primitives[0].exponent = exponent; + snapshot.basis[0].components[0].primitives[0].coefficient = coefficient; + snapshot.validate().unwrap(); + let bytes = serde_json::to_vec(&snapshot).unwrap(); + let restored: Snapshot = serde_json::from_slice(&bytes).unwrap(); + restored.validate().unwrap(); + prop_assert_eq!(snapshot.identity(), restored.identity()); + } + } +} diff --git a/crates/rustiq-core/src/persistence/data.rs b/crates/rustiq-core/src/persistence/data.rs index 86dccc87..e0f1dd36 100644 --- a/crates/rustiq-core/src/persistence/data.rs +++ b/crates/rustiq-core/src/persistence/data.rs @@ -6,3 +6,5 @@ pub use ao_eri_artifact::AoEriArtifact; pub use artifact::Artifact; pub use rustiq_data::RustiQData; pub(crate) use rustiq_data::AO_ERI_ARTIFACT; + +mod portable; diff --git a/crates/rustiq-core/src/persistence/data/portable.rs b/crates/rustiq-core/src/persistence/data/portable.rs new file mode 100644 index 00000000..16b7c13d --- /dev/null +++ b/crates/rustiq-core/src/persistence/data/portable.rs @@ -0,0 +1,217 @@ +use super::super::{ + calculation::{Snapshot, CALCULATION_PATH, MAX_CALCULATION_BYTES}, + ArtifactAttributes, ArtifactError, CalculationContext, ManifestKind, PortableError, Storage, + AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, +}; +use super::{RustiQData, AO_ERI_ARTIFACT}; +use crate::calculation::PreparedCalculation; +use relative_path::RelativePath; +#[cfg(any(unix, test))] +use std::fs::File; +use std::path::Path; + +impl RustiQData { + /// Creates portable scientific data from effective inputs, without running HF. + /// Add known artifacts with the typed setters before calling [`Self::write`]. + pub fn from_calculation(calculation: &PreparedCalculation) -> Result { + let snapshot = Snapshot::from_calculation(calculation)?; + let mut data = Self::new_with_identity( + snapshot.identity(), + snapshot.basis_functions(), + ManifestKind::Portable, + ); + data.context = Some(CalculationContext(snapshot)); + Ok(data) + } + + /// Opens a portable archive and validates its context and artifact index. + /// Numerical payloads are verified and decoded lazily on first access. + pub fn open(path: impl AsRef) -> Result { + let mut data = Self::read_from(Storage::open_zip(path.as_ref())?)?; + if data.manifest.kind != ManifestKind::Portable { + return Err(PortableError::UnsupportedVersion); + } + let reference = + data.manifest.calculation.as_ref().ok_or_else(|| { + PortableError::InvalidArchive("missing calculation reference".into()) + })?; + if reference.version != 1 { + return Err(PortableError::UnsupportedVersion); + } + if reference.path != CALCULATION_PATH || reference.size > MAX_CALCULATION_BYTES { + return Err(PortableError::InvalidArchive( + "invalid calculation reference".into(), + )); + } + let source = data.source.as_mut().expect("opened storage"); + let path = RelativePath::new(CALCULATION_PATH); + if source.artifact_size(path)? != reference.size { + return Err(ArtifactError::IntegrityMismatch(CALCULATION_PATH.into()).into()); + } + let metadata = source.artifact_metadata(path)?; + if metadata.digest != reference.digest { + return Err(ArtifactError::IntegrityMismatch(CALCULATION_PATH.into()).into()); + } + let snapshot: Snapshot = source + .read_json(path, MAX_CALCULATION_BYTES) + .map_err(|error| PortableError::InvalidCalculation(error.to_string()))?; + snapshot.validate()?; + let identity = snapshot.identity(); + if identity.version != data.manifest.scientific_identity.version + || identity.digest != data.manifest.scientific_identity.digest + { + return Err(PortableError::InvalidCalculation( + "scientific identity does not match the resolved inputs".into(), + )); + } + for (name, artifact) in &data.manifest.artifacts { + if artifact + .path + .as_str() + .split('/') + .next() + .is_some_and(|part| { + part.eq_ignore_ascii_case(CALCULATION_PATH) + || part.eq_ignore_ascii_case("manifest.json") + }) + { + return Err(ArtifactError::InvalidPath(artifact.path.to_string()).into()); + } + if source.artifact_size(&artifact.path)? != artifact.size { + return Err(ArtifactError::IntegrityMismatch(artifact.path.to_string()).into()); + } + if let ArtifactAttributes::AoEri(attributes) = &artifact.attributes { + if name != AO_ERI_ARTIFACT + || artifact.path.as_str() != AO_ERI_PATH + || attributes.basis_functions != snapshot.basis_functions() + || attributes.computation_version != snapshot.computation_version() + { + return Err(ArtifactError::InvalidMetadata( + "AO ERI metadata disagrees with calculation context".into(), + ) + .into()); + } + } + } + data.basis_functions = Some(snapshot.basis_functions()); + data.context = Some(CalculationContext(snapshot)); + Ok(data) + } + + /// Returns the portable context, absent only for internal directory-cache data. + pub fn calculation(&self) -> Option<&CalculationContext> { + self.context.as_ref() + } + + /// Producer name and version recorded as provenance, not compatibility policy. + pub fn producer(&self) -> (&str, &str) { + ( + &self.manifest.producer.name, + &self.manifest.producer.version, + ) + } + + /// Lists scientific artifact names and representation identifiers without loading arrays. + /// Unknown representations remain visible; listing does not verify their payloads. + pub fn artifact_representations(&self) -> impl Iterator { + self.ao_eri + .as_ref() + .map(|_| (AO_ERI_ARTIFACT, COMPACT_ERI_REPRESENTATION)) + .into_iter() + .chain( + self.manifest + .artifacts + .iter() + .filter_map(|(name, artifact)| { + (name != AO_ERI_ARTIFACT || self.ao_eri.is_none()) + .then_some((name.as_str(), artifact.representation.as_str())) + }), + ) + } + + /// Tests AO ERI compatibility with another resolved calculation, without reading arrays. + /// Requested HF/MP2 methods do not participate in deterministic integral identity. + pub fn eri_is_compatible(&self, calculation: &PreparedCalculation) -> bool { + if self.ao_eri.is_some() { + return self.matches_eri_identity(calculation); + } + let Some(artifact) = self.manifest.artifacts.get(AO_ERI_ARTIFACT) else { + return false; + }; + matches!(&artifact.attributes, ArtifactAttributes::AoEri(attributes) + if attributes.computation_version == super::super::AO_ERI_COMPUTATION_VERSION) + && artifact.representation == COMPACT_ERI_REPRESENTATION + && self.matches_eri_identity(calculation) + } + + fn matches_eri_identity(&self, calculation: &PreparedCalculation) -> bool { + let hf = calculation.hf_config(); + let identity = super::super::ao_eri_identity( + calculation.get_molecule(), + calculation.get_basis(), + hf.eri_schwarz_threshold, + ); + self.manifest.scientific_identity.version == identity.version + && self.manifest.scientific_identity.digest == identity.digest + } + + /// Atomically publishes a portable archive. Never overwrites an existing path. + /// The source archive is not modified; unloaded artifacts are copied and verified. + pub fn write(&mut self, path: impl AsRef) -> Result<(), PortableError> { + if self.context.is_none() || self.manifest.kind != ManifestKind::Portable { + return Err(PortableError::InvalidCalculation( + "portable output requires resolved calculation context".into(), + )); + } + // Validate any known, unloaded artifact before preserving it as scientific data. + if self.ao_eri.is_none() && self.manifest.artifacts.get(AO_ERI_ARTIFACT) + .is_some_and(|artifact| artifact.representation == COMPACT_ERI_REPRESENTATION + && matches!(&artifact.attributes, ArtifactAttributes::AoEri(attributes) if attributes.computation_version == super::super::AO_ERI_COMPUTATION_VERSION)) { + self.validate_unloaded_eri()?; + } + let path = path.as_ref(); + match std::fs::symlink_metadata(path) { + Ok(_) => return Err(PortableError::AlreadyExists), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let temporary = tempfile::NamedTempFile::new_in(parent)?; + self.write_to(Storage::create_zip(temporary.reopen()?))?; + // Reopening validates the finalized container, context, index and publication limits. + drop(Self::open(temporary.path())?); + temporary.persist_noclobber(path).map_err(|error| { + if error.error.kind() == std::io::ErrorKind::AlreadyExists { + PortableError::AlreadyExists + } else { + PortableError::Io(error.error) + } + })?; + #[cfg(unix)] + File::open(parent)?.sync_all()?; + Ok(()) + } + + fn validate_unloaded_eri(&mut self) -> Result<(), PortableError> { + let artifact = &self.manifest.artifacts[AO_ERI_ARTIFACT]; + let ArtifactAttributes::AoEri(attributes) = &artifact.attributes else { + unreachable!() + }; + let source = self.source.as_mut().ok_or(ArtifactError::Missing)?; + source.with_artifact::<_, ArtifactError, _>(&artifact.path, |reader| { + super::super::npy::checked_eri_prefix( + reader, + attributes.basis_functions, + artifact.size, + )?; + Ok(()) + })?; + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustiq-core/src/persistence/data/portable/tests.rs b/crates/rustiq-core/src/persistence/data/portable/tests.rs new file mode 100644 index 00000000..e8e81fcb --- /dev/null +++ b/crates/rustiq-core/src/persistence/data/portable/tests.rs @@ -0,0 +1,631 @@ +use super::super::super::{sha256, AoEriArtifact}; +use super::*; +use crate::{ + calculation::CalculationBuilder, + config::{HfConfig, HfMethod, MoleculeConfig, Mp2Config}, + eri::CompactEri, + molecules::geometry::Geometry, + test_utils::load_minimal_basis_file, +}; +use std::{ + fs, + io::{Read, Write}, + num::NonZeroU8, +}; +use zip::{write::SimpleFileOptions, CompressionMethod, ZipArchive, ZipWriter}; + +fn calculation() -> PreparedCalculation { + let geometry = Geometry::from_source("private.xyz", "2\nH2\nH 0 0 0\nH 1.4 0 0\n").unwrap(); + CalculationBuilder::new(&geometry, &load_minimal_basis_file()) + .prepare() + .unwrap() +} +fn data() -> RustiQData { + let mut data = RustiQData::from_calculation(&calculation()).unwrap(); + data.set_eri(CompactEri::from_ordered_values(2, vec![0.5, 1.5, 2.5, 3.5, 4.5, 5.5]).unwrap()) + .unwrap(); + data +} +fn members(path: &Path) -> Vec<(String, Vec)> { + let mut archive = ZipArchive::new(File::open(path).unwrap()).unwrap(); + (0..archive.len()) + .map(|i| { + let mut member = archive.by_index(i).unwrap(); + let name = member.name().to_owned(); + let mut bytes = Vec::new(); + member.read_to_end(&mut bytes).unwrap(); + (name, bytes) + }) + .collect() +} +fn write_members(path: &Path, members: &[(String, Vec)]) { + let mut writer = ZipWriter::new(File::create(path).unwrap()); + for (name, bytes) in members { + writer + .start_file( + name, + SimpleFileOptions::default() + .compression_method(CompressionMethod::Stored) + .large_file(true), + ) + .unwrap(); + writer.write_all(bytes).unwrap(); + } + writer.finish().unwrap(); +} +fn edit_json( + members: &mut [(String, Vec)], + name: &str, + edit: impl FnOnce(&mut serde_json::Value), +) { + let bytes = &mut members.iter_mut().find(|(n, _)| n == name).unwrap().1; + let mut json = serde_json::from_slice(bytes).unwrap(); + edit(&mut json); + *bytes = serde_json::to_vec(&json).unwrap(); +} +fn refresh_snapshot_digest(members: &mut [(String, Vec)]) { + let snapshot = &members + .iter() + .find(|(n, _)| n == CALCULATION_PATH) + .unwrap() + .1; + let (size, digest) = (snapshot.len(), sha256(snapshot).to_string()); + edit_json(members, "manifest.json", |m| { + m["calculation"]["size"] = size.into(); + m["calculation"]["digest"] = digest.into(); + }); +} +fn refresh_eri_digest(members: &mut [(String, Vec)]) { + let payload = &members.iter().find(|(n, _)| n == AO_ERI_PATH).unwrap().1; + let (size, digest) = (payload.len(), sha256(payload).to_string()); + edit_json(members, "manifest.json", |m| { + m["artifacts"]["ao_eri"]["size"] = size.into(); + m["artifacts"]["ao_eri"]["digest"] = digest.into(); + }); +} + +#[test] +fn rust_round_trip_is_lazy_self_describing_and_reproducible() { + let dir = tempfile::tempdir().unwrap(); + let first = dir.path().join("first.rustiq"); + let second = dir.path().join("second.rustiq"); + data().write(&first).unwrap(); + data().write(&second).unwrap(); + assert_eq!(fs::read(&first).unwrap(), fs::read(&second).unwrap()); + let mut restored = RustiQData::open(&first).unwrap(); + assert!(restored.ao_eri.is_none()); + let context = restored.calculation().unwrap(); + assert_eq!( + context.atoms().collect::>(), + vec![(1, [0.0, 0.0, 0.0]), (1, [1.4, 0.0, 0.0])] + ); + assert_eq!(context.hf_method(), crate::config::ResolvedHfMethod::Rhf); + assert_eq!(context.basis().len(), 2); + assert!(restored.eri_is_compatible(&calculation())); + assert_eq!( + restored.read_eri().unwrap().ordered_values(), + &[0.5, 1.5, 2.5, 3.5, 4.5, 5.5] + ); + let pointer = restored.read_eri().unwrap() as *const CompactEri; + assert_eq!(restored.read_eri().unwrap() as *const CompactEri, pointer); + let zip = ZipArchive::new(File::open(&first).unwrap()).unwrap(); + assert_eq!(zip.len(), 3); + let snapshot = members(&first) + .into_iter() + .find(|(n, _)| n == CALCULATION_PATH) + .unwrap() + .1; + assert_eq!( + serde_json::from_slice::(&snapshot).unwrap(), + serde_json::from_str::(include_str!( + "../../../../tests/data/persistence/calculation-h2-v1.json" + )) + .unwrap() + ); +} + +#[test] +fn empty_artifacts_uhf_and_mp2_context_round_trip() { + let geometry = Geometry::from_source("H.xyz", "1\nH\nH 0 0 0\n").unwrap(); + let basis = load_minimal_basis_file(); + let prepared = CalculationBuilder::new(&geometry, &basis) + .with_molecule_config(MoleculeConfig { + multiplicity: NonZeroU8::new(2).unwrap().into(), + ..Default::default() + }) + .with_hf(HfConfig { + method: HfMethod::Uhf.into(), + diis: true, + ..Default::default() + }) + .with_mp2(Mp2Config::default()) + .prepare() + .unwrap(); + let mut data = RustiQData::from_calculation(&prepared).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("h.rustiq"); + data.write(&path).unwrap(); + let mut restored = RustiQData::open(&path).unwrap(); + assert!(restored.get::().unwrap().is_none()); + let context = restored.calculation().unwrap(); + assert_eq!(context.multiplicity(), 2); + assert_eq!(context.hf_method(), crate::config::ResolvedHfMethod::Uhf); + assert_eq!(context.diis_size(), Some(6)); + assert_eq!(context.mp2_frozen_orbitals(), Some(0)); +} + +#[test] +fn compatibility_depends_on_effective_integral_inputs_only() { + let prepared = calculation(); + let mut data = data(); + let mp2 = CalculationBuilder::new(prepared.get_molecule(), &load_minimal_basis_file()) + .with_mp2(Mp2Config::default()) + .prepare() + .unwrap(); + assert!(data.eri_is_compatible(&mp2)); + let changed = CalculationBuilder::new(prepared.get_molecule(), &load_minimal_basis_file()) + .with_hf(HfConfig { + eri_schwarz_threshold: None, + ..Default::default() + }) + .prepare() + .unwrap(); + assert!(!data.eri_is_compatible(&changed)); + data.ao_eri = None; + assert!(!data.eri_is_compatible(&prepared)); +} + +#[test] +fn publication_never_overwrites_and_concurrent_writer_has_one_winner() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("target.rustiq"); + fs::write(&path, b"user data").unwrap(); + assert!(matches!( + data().write(&path), + Err(PortableError::AlreadyExists) + )); + assert_eq!(fs::read(&path).unwrap(), b"user data"); + fs::remove_file(&path).unwrap(); + let barrier = std::sync::Barrier::new(2); + let results = std::thread::scope(|scope| { + let first = scope.spawn(|| { + let mut data = data(); + barrier.wait(); + data.write(&path) + }); + let second = scope.spawn(|| { + let mut data = data(); + barrier.wait(); + data.write(&path) + }); + [first.join().unwrap(), second.join().unwrap()] + }); + assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1); + assert_eq!( + results + .iter() + .filter(|r| matches!(r, Err(PortableError::AlreadyExists))) + .count(), + 1 + ); + assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 1); + RustiQData::open(&path).unwrap().read_eri().unwrap(); +} + +#[test] +fn corrupt_payload_is_lazy_but_cannot_be_read_or_republished() { + let dir = tempfile::tempdir().unwrap(); + let original = dir.path().join("original.rustiq"); + data().write(&original).unwrap(); + let mut contents = members(&original); + *contents + .iter_mut() + .find(|(n, _)| n == AO_ERI_PATH) + .unwrap() + .1 + .last_mut() + .unwrap() ^= 1; + let corrupt = dir.path().join("corrupt.rustiq"); + write_members(&corrupt, &contents); + let mut restored = RustiQData::open(&corrupt).unwrap(); + assert!(restored.read_eri().is_err()); + let output = dir.path().join("output.rustiq"); + assert!(restored.write(&output).is_err()); + assert!(!output.exists()); + assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 2); +} + +#[test] +fn unknown_artifacts_survive_verified_streaming_copy() { + let dir = tempfile::tempdir().unwrap(); + let first = dir.path().join("first.rustiq"); + data().write(&first).unwrap(); + let mut contents = members(&first); + let opaque = b"future payload".to_vec(); + contents.push(("arrays/future.bin".into(), opaque.clone())); + edit_json(&mut contents, "manifest.json", |m| { + m["artifacts"]["future"] = serde_json::json!({"path":"arrays/future.bin","size":opaque.len(),"digest":sha256(&opaque),"representation":"future-v99","attributes":{"opaque":17}}); + }); + let source = dir.path().join("source.rustiq"); + write_members(&source, &contents); + let mut restored = RustiQData::open(&source).unwrap(); + let output = dir.path().join("copy.rustiq"); + restored.write(&output).unwrap(); + assert!(restored.ao_eri.is_none()); + assert!(members(&output).contains(&("arrays/future.bin".into(), opaque))); + assert_eq!( + RustiQData::open(&output).unwrap().manifest.artifacts["future"], + restored.manifest.artifacts["future"] + ); + assert_eq!(members(&source), contents); +} + +#[test] +fn rejects_invalid_context_manifest_and_artifact_metadata() { + let dir = tempfile::tempdir().unwrap(); + let original = dir.path().join("original.rustiq"); + data().write(&original).unwrap(); + let original = members(&original); + for (target, field, value) in [ + ("manifest.json", "/format_version", serde_json::json!(999)), + ( + "manifest.json", + "/scientific_identity/version", + serde_json::json!(99), + ), + ( + "manifest.json", + "/artifacts/ao_eri/attributes/basis_functions", + serde_json::json!(3), + ), + ( + "manifest.json", + "/artifacts/ao_eri/attributes/computation_version", + serde_json::json!(99), + ), + ( + "manifest.json", + "/artifacts/ao_eri/path", + serde_json::json!("CALCULATION.JSON"), + ), + ( + "manifest.json", + "/artifacts/ao_eri/size", + serde_json::json!(1), + ), + (CALCULATION_PATH, "/version", serde_json::json!(99)), + (CALCULATION_PATH, "/units", serde_json::json!("angstrom")), + (CALCULATION_PATH, "/charge", serde_json::json!(100)), + (CALCULATION_PATH, "/multiplicity", serde_json::json!(0)), + ( + CALCULATION_PATH, + "/atoms/0/position/0", + serde_json::json!(5.0), + ), + ( + CALCULATION_PATH, + "/basis/0/components/0/primitives/0/exponent", + serde_json::json!(-1.0), + ), + (CALCULATION_PATH, "/hf/diis_size", serde_json::json!(1)), + ( + CALCULATION_PATH, + "/hf/convergence_threshold", + serde_json::json!(0), + ), + ] { + let mut contents = original.clone(); + edit_json(&mut contents, target, |j| { + *j.pointer_mut(field).unwrap() = value + }); + if target == CALCULATION_PATH { + refresh_snapshot_digest(&mut contents); + } + let path = dir.path().join("invalid.rustiq"); + write_members(&path, &contents); + assert!(RustiQData::open(&path).is_err(), "{target} {field}"); + } + let mut contents = original.clone(); + contents.retain(|(n, _)| n != CALCULATION_PATH); + let path = dir.path().join("missing.rustiq"); + write_members(&path, &contents); + assert!(RustiQData::open(path).is_err()); +} + +#[test] +fn rejects_unsafe_member_paths_and_local_header_disagreement() { + let dir = tempfile::tempdir().unwrap(); + for name in [ + "../evil", + "/evil", + "C:/evil", + "a\\evil", + "a/CON", + "a/", + "a//b", + "a/./b", + "a/../b", + "a/b.", + "a/b ", + "a/b?", + "manifest.json/child", + ] { + let path = dir.path().join("bad.rustiq"); + let mut contents = vec![("manifest.json".into(), b"{}".to_vec())]; + contents.push((name.into(), vec![])); + write_members(&path, &contents); + assert!(RustiQData::open(&path).is_err(), "{name}"); + } + let path = dir.path().join("good.rustiq"); + data().write(&path).unwrap(); + let original = fs::read(&path).unwrap(); + for offset in [30, 6, 8] { + let mut bytes = original.clone(); + bytes[offset] ^= 1; + let invalid = dir.path().join("invalid.rustiq"); + fs::write(&invalid, bytes).unwrap(); + assert!(RustiQData::open(&invalid).is_err()); + } + for len in [0, 10, original.len() / 2, original.len() - 1] { + let invalid = dir.path().join("truncated.rustiq"); + fs::write(&invalid, &original[..len]).unwrap(); + assert!(RustiQData::open(invalid).is_err()); + } +} + +#[test] +fn rejects_duplicate_members_before_zip_library_deduplication() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("duplicate.rustiq"); + write_members( + &path, + &[("one.json".into(), vec![]), ("two.json".into(), vec![])], + ); + let mut bytes = fs::read(&path).unwrap(); + for index in 0..bytes.len() - 8 { + if &bytes[index..index + 8] == b"two.json" { + bytes[index..index + 8].copy_from_slice(b"one.json"); + } + } + fs::write(&path, bytes).unwrap(); + assert!(RustiQData::open(&path) + .unwrap_err() + .to_string() + .contains("duplicate")); + write_members( + &path, + &[ + ("manifest.json".into(), vec![]), + ("MANIFEST.JSON".into(), vec![]), + ], + ); + assert!(RustiQData::open(path) + .unwrap_err() + .to_string() + .contains("duplicate")); +} + +#[test] +fn malformed_npy_is_rejected_even_with_correct_digest() { + let dir = tempfile::tempdir().unwrap(); + let good = dir.path().join("good.rustiq"); + data().write(&good).unwrap(); + for payload in [ + b"not npy".to_vec(), + b"\x93NUMPY\x02\x00\xff\xff\xff\x7f".to_vec(), + ] { + let mut contents = members(&good); + contents + .iter_mut() + .find(|(n, _)| n == AO_ERI_PATH) + .unwrap() + .1 = payload; + refresh_eri_digest(&mut contents); + let bad = dir.path().join("bad.rustiq"); + write_members(&bad, &contents); + let mut restored = RustiQData::open(&bad).unwrap(); + assert!(restored.read_eri().is_err()); + assert!(restored.write(dir.path().join("copy.rustiq")).is_err()); + } +} + +#[test] +#[ignore = "writes a portable artifact for Python interoperability"] +fn writes_archive_for_python_interoperability() { + let path = std::env::var_os("RUSTIQ_ARCHIVE_TEST_OUTPUT").expect("output path"); + data().write(path).unwrap(); +} + +#[test] +fn reads_python_zip64_fixtures_with_both_npy_byte_orders() { + for fixture in [ + include_str!("../../../../tests/data/persistence/portable-python-little-v1.rustiq.hex"), + include_str!("../../../../tests/data/persistence/portable-python-big-v1.rustiq.hex"), + ] { + let bytes: Vec = fixture + .trim() + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect(); + assert!(bytes.windows(4).any(|w| w == b"PK\x06\x06")); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("python.rustiq"); + fs::write(&path, bytes).unwrap(); + let mut data = RustiQData::open(&path).unwrap(); + assert!(data.eri_is_compatible(&calculation())); + assert_eq!( + data.read_eri().unwrap().ordered_values(), + &[0.5, 1.5, 2.5, 3.5, 4.5, 5.5] + ); + } +} + +#[test] +#[cfg(unix)] +fn zip64_handles_offsets_beyond_four_gib_without_allocating_large_payloads() { + use std::io::{Seek, SeekFrom}; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sparse.rustiq"); + let mut file = File::create(&path).unwrap(); + file.seek(SeekFrom::Start(u64::from(u32::MAX) + 1)).unwrap(); + data().write_to(Storage::create_zip(file)).unwrap(); + assert!(fs::metadata(&path).unwrap().len() > u64::from(u32::MAX)); + let mut restored = RustiQData::open(&path).unwrap(); + assert_eq!( + restored.read_eri().unwrap().ordered_values(), + &[0.5, 1.5, 2.5, 3.5, 4.5, 5.5] + ); +} + +#[test] +fn rejects_links_special_entries_encryption_and_oversized_metadata() { + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("source.rustiq"); + data().write(&source).unwrap(); + let original = fs::read(&source).unwrap(); + let central = original + .windows(4) + .position(|w| w == b"PK\x01\x02") + .unwrap(); + for mode in [0o120777_u32, 0o020666, 0o040755] { + let mut bytes = original.clone(); + bytes[central + 38..central + 42].copy_from_slice(&(mode << 16).to_le_bytes()); + let path = dir.path().join("special.rustiq"); + fs::write(&path, bytes).unwrap(); + assert!(RustiQData::open(path).is_err()); + } + for flags in [1_u16, 0x40, 0x2000] { + let mut bytes = original.clone(); + bytes[central + 8..central + 10].copy_from_slice(&flags.to_le_bytes()); + let path = dir.path().join("encrypted.rustiq"); + fs::write(&path, bytes).unwrap(); + assert!(RustiQData::open(path).is_err()); + } + let mut contents = members(&source); + let manifest = &mut contents + .iter_mut() + .find(|(n, _)| n == "manifest.json") + .unwrap() + .1; + manifest.resize(1024 * 1024 + 1, b' '); + let path = dir.path().join("huge.rustiq"); + write_members(&path, &contents); + assert!(RustiQData::open(path).is_err()); + let mut bytes = original.clone(); + let end = bytes.len() - 22; + bytes[end + 8..end + 10].copy_from_slice(&4097_u16.to_le_bytes()); + bytes[end + 10..end + 12].copy_from_slice(&4097_u16.to_le_bytes()); + let path = dir.path().join("many.rustiq"); + fs::write(&path, bytes).unwrap(); + assert!(RustiQData::open(path).is_err()); + let mut contents = members(&source); + edit_json(&mut contents, "manifest.json", |m| { + m["calculation"]["size"] = (MAX_CALCULATION_BYTES + 1).into() + }); + let path = dir.path().join("snapshot-limit.rustiq"); + write_members(&path, &contents); + assert!(RustiQData::open(path).is_err()); +} + +#[test] +fn crc_and_actual_decompressed_size_are_checked() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("valid.rustiq"); + data().write(&path).unwrap(); + // Metadata is deflated. Tamper with its declared length in the central directory. + let mut bytes = fs::read(&path).unwrap(); + let central = bytes.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + bytes[central + 24..central + 28].copy_from_slice(&1_u32.to_le_bytes()); + let bad = dir.path().join("size.rustiq"); + fs::write(&bad, bytes).unwrap(); + assert!(RustiQData::open(bad).is_err()); + let mut bytes = fs::read(&path).unwrap(); + bytes[central + 16] ^= 1; + let bad = dir.path().join("crc.rustiq"); + fs::write(&bad, bytes).unwrap(); + assert!(RustiQData::open(bad).is_err()); +} + +#[test] +fn settings_are_resolved_and_machine_policy_is_excluded() { + use crate::config::{ + DensityGuessConfig, GuessPerturbationConfig, MemoryLimit, RandomGuessConfig, + }; + use crate::molecules::units::Units; + let basis = load_minimal_basis_file(); + let geometry = Geometry::from_source( + "source-path-must-not-leak.xyz", + "2\nH2\nH 0 0 0\nH 0.74 0 0\n", + ) + .unwrap(); + for guess in [ + DensityGuessConfig::Zero, + DensityGuessConfig::CoreHamiltonian { + perturbation: Some(GuessPerturbationConfig::default()), + }, + DensityGuessConfig::OneElectron { + perturbation: Some(GuessPerturbationConfig::default()), + }, + DensityGuessConfig::Random { + config: RandomGuessConfig::default(), + }, + ] { + let prepared = CalculationBuilder::new(&geometry, &basis) + .with_molecule_config(MoleculeConfig { + units: Units::Angstrom, + ..Default::default() + }) + .with_hf(HfConfig { + guess: guess.into(), + ..Default::default() + }) + .with_mp2(Mp2Config { + memory_limit: MemoryLimit::Fixed(bytesize::ByteSize::mib(42)).into(), + ..Default::default() + }) + .prepare() + .unwrap(); + let data = RustiQData::from_calculation(&prepared).unwrap(); + let snapshot = &data.calculation().unwrap().0; + let json = serde_json::to_string(snapshot).unwrap(); + for forbidden in [ + "source-path", + "memory_limit", + "cache", + "span", + "auto", + "angstrom", + ] { + assert!(!json.contains(forbidden), "{forbidden}"); + } + let restored: Snapshot = serde_json::from_str(&json).unwrap(); + restored.validate().unwrap(); + assert_eq!(snapshot.identity(), restored.identity()); + assert_eq!( + format!("{:?}", CalculationContext(restored).density_guess()), + format!("{guess:?}") + ); + assert!(data.calculation().unwrap().atoms().nth(1).unwrap().1[0] > 1.0); + } +} + +#[test] +fn unsupported_representation_is_inspectable_but_not_scientifically_usable() { + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("source.rustiq"); + data().write(&source).unwrap(); + let mut contents = members(&source); + edit_json(&mut contents, "manifest.json", |m| { + m["artifacts"]["ao_eri"]["representation"] = "future-eri-v2".into() + }); + let unknown = dir.path().join("unknown.rustiq"); + write_members(&unknown, &contents); + let mut restored = RustiQData::open(&unknown).unwrap(); + assert!(restored.calculation().is_some()); + assert!(!restored.eri_is_compatible(&calculation())); + assert!(restored.read_eri().is_err()); + restored.write(dir.path().join("copy.rustiq")).unwrap(); +} diff --git a/crates/rustiq-core/src/persistence/data/rustiq_data.rs b/crates/rustiq-core/src/persistence/data/rustiq_data.rs index 6ab7648c..995afde0 100644 --- a/crates/rustiq-core/src/persistence/data/rustiq_data.rs +++ b/crates/rustiq-core/src/persistence/data/rustiq_data.rs @@ -5,10 +5,10 @@ use relative_path::RelativePath; use crate::eri::CompactEri; use super::super::{ - read_compact_eri, validate_storage_path, write_compact_eri, AoEriAttributes, - ArtifactAttributes, ArtifactError, ArtifactManifest, Manifest, ManifestError, ManifestKind, - PersistenceReadError, PersistenceWriteError, Producer, ScientificIdentityManifest, Storage, - StorageError, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, + validate_storage_path, write_compact_eri, AoEriAttributes, ArtifactAttributes, ArtifactError, + ArtifactManifest, Manifest, ManifestError, ManifestKind, PersistenceReadError, + PersistenceWriteError, Producer, ScientificIdentityManifest, Storage, StorageError, + AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, FORMAT_VERSION, MANIFEST_PATH, }; use super::artifact::Artifact; @@ -20,9 +20,10 @@ const MAX_MANIFEST_BYTES: u64 = 1024 * 1024; #[derive(Debug)] pub struct RustiQData { pub(super) manifest: Manifest, - source: Option, + pub(super) source: Option, pub(super) ao_eri: Option, - basis_functions: Option, + pub(super) basis_functions: Option, + pub(super) context: Option, } impl RustiQData { @@ -55,8 +56,10 @@ impl RustiQData { digest: identity.digest, }, artifacts: BTreeMap::new(), + calculation: None, }, source: None, + context: None, ao_eri: None, basis_functions: Some(basis_functions), } @@ -92,6 +95,7 @@ impl RustiQData { Ok(Self { manifest, source: Some(source), + context: None, ao_eri: None, basis_functions, }) @@ -103,6 +107,15 @@ impl RustiQData { /// Replaces the AO ERI artifact after checking its compact length. pub fn set_eri(&mut self, eri: CompactEri) -> Result<(), ArtifactError> { + if self + .context + .as_ref() + .is_some_and(|context| context.0.computation_version() != AO_ERI_COMPUTATION_VERSION) + { + return Err(ArtifactError::UnsupportedRepresentation( + "AO ERI computation version".into(), + )); + } let basis_functions = self.basis_functions.ok_or(ArtifactError::Missing)?; validate_eri_len(&eri, basis_functions)?; self.ao_eri = Some(eri); @@ -132,16 +145,25 @@ impl RustiQData { } let source = self.source.as_mut().ok_or(ArtifactError::Missing)?; + if source.artifact_size(&artifact.path)? != artifact.size { + return Err(ArtifactError::IntegrityMismatch(artifact.path.to_string())); + } let metadata = source.artifact_metadata(&artifact.path)?; if metadata.size != artifact.size || metadata.digest != artifact.digest { return Err(ArtifactError::IntegrityMismatch(artifact.path.to_string())); } - self.ao_eri = Some( - source.with_artifact::<_, ArtifactError, _>(&artifact.path, |reader| { - read_compact_eri(reader, attributes.basis_functions).map_err(Into::into) - })?, - ); + self.ao_eri = Some(source.with_artifact::<_, ArtifactError, _>( + &artifact.path, + |reader| { + super::super::npy::read_checked_compact_eri( + reader, + attributes.basis_functions, + artifact.size, + ) + .map_err(Into::into) + }, + )?); } Ok(self @@ -176,6 +198,19 @@ impl RustiQData { ) -> Result<(), PersistenceWriteError> { let mut manifest = self.manifest.clone(); + if let Some(context) = &self.context { + let metadata = destination.write_json( + RelativePath::new(super::super::calculation::CALCULATION_PATH), + &context.0, + )?; + manifest.calculation = Some(super::super::manifest::CalculationManifest { + path: super::super::calculation::CALCULATION_PATH.into(), + version: 1, + size: metadata.size, + digest: metadata.digest, + }); + } + for (name, artifact) in &self.manifest.artifacts { if name == AO_ERI_ARTIFACT && eri.is_some() { continue; @@ -186,6 +221,9 @@ impl RustiQData { })?; validate_artifact_path(&artifact.path)?; + if source.artifact_size(&artifact.path)? != artifact.size { + return Err(ArtifactError::IntegrityMismatch(artifact.path.to_string()).into()); + } let metadata = source.with_artifact::<_, PersistenceWriteError, _>(&artifact.path, |input| { destination.write_artifact::( diff --git a/crates/rustiq-core/src/persistence/error.rs b/crates/rustiq-core/src/persistence/error.rs index 02e115c0..f14e3fda 100644 --- a/crates/rustiq-core/src/persistence/error.rs +++ b/crates/rustiq-core/src/persistence/error.rs @@ -4,6 +4,8 @@ use thiserror::Error; #[derive(Debug, Error)] pub(crate) enum StorageError { + #[error("invalid portable archive: {0}")] + Archive(String), #[error("storage I/O failed: {0}")] Io(#[from] io::Error), #[error("unsafe storage path: {0}")] @@ -12,6 +14,51 @@ pub(crate) enum StorageError { UnexpectedEntryType(String), } +/// Portable persistence failures, independent of the physical container library. +#[derive(Debug, Error)] +pub enum PortableError { + #[error("portable artifact I/O failed: {0}")] + Io(#[from] io::Error), + #[error("the destination already exists")] + AlreadyExists, + #[error("invalid portable artifact: {0}")] + InvalidArchive(String), + #[error("unsupported portable calculation format or version")] + UnsupportedVersion, + #[error("invalid resolved calculation: {0}")] + InvalidCalculation(String), + #[error(transparent)] + Artifact(#[from] ArtifactError), +} + +impl From for PortableError { + fn from(error: StorageError) -> Self { + match error { + StorageError::Io(error) => Self::Io(error), + other => Self::InvalidArchive(other.to_string()), + } + } +} + +impl From for PortableError { + fn from(error: PersistenceReadError) -> Self { + match error { + PersistenceReadError::Artifact(error) => Self::Artifact(error), + other => Self::InvalidArchive(other.to_string()), + } + } +} + +impl From for PortableError { + fn from(error: PersistenceWriteError) -> Self { + match error { + PersistenceWriteError::Artifact(error) => Self::Artifact(error), + PersistenceWriteError::Storage(error) => error.into(), + other => Self::InvalidArchive(other.to_string()), + } + } +} + #[derive(Debug, Error)] pub enum NpyError { #[error("could not read NPY data: {0}")] diff --git a/crates/rustiq-core/src/persistence/identity.rs b/crates/rustiq-core/src/persistence/identity.rs index 94c5e439..c84b5569 100644 --- a/crates/rustiq-core/src/persistence/identity.rs +++ b/crates/rustiq-core/src/persistence/identity.rs @@ -35,11 +35,7 @@ fn ao_eri_identity_with_computation_version( schwarz_threshold: Option, computation_version: u32, ) -> ScientificIdentity { - let mut bytes = CanonicalBytes::default(); - bytes.text(b"scientific-identity-v1"); - bytes.text(b"ao-eri-computation-version"); - bytes.u32(computation_version); - bytes.text(COMPACT_ERI_REPRESENTATION.as_bytes()); + let mut bytes = CanonicalBytes::ao_eri(computation_version); bytes.len(geometry.atoms.len()); for atom in &geometry.atoms { @@ -70,41 +66,49 @@ fn ao_eri_identity_with_computation_version( } } - match schwarz_threshold { - None => bytes.u8(0), - Some(threshold) => { - bytes.u8(1); - bytes.f64(threshold.into_inner()); - } - } - - ScientificIdentity { - version: SCIENTIFIC_IDENTITY_VERSION, - digest: sha256(&bytes.0), - } + bytes.finish_ao_eri(schwarz_threshold.map(|value| value.into_inner())) } #[derive(Default)] -struct CanonicalBytes(Vec); +pub(crate) struct CanonicalBytes(Vec); impl CanonicalBytes { - fn u8(&mut self, value: u8) { - self.0.push(value); + pub(crate) fn ao_eri(computation_version: u32) -> Self { + let mut bytes = Self::default(); + bytes.text(b"scientific-identity-v1"); + bytes.text(b"ao-eri-computation-version"); + bytes.u32(computation_version); + bytes.text(COMPACT_ERI_REPRESENTATION.as_bytes()); + bytes + } + + pub(crate) fn finish_ao_eri(mut self, threshold: Option) -> ScientificIdentity { + match threshold { + None => self.u8(0), + Some(value) => { + self.u8(1); + self.f64(value); + } + } + ScientificIdentity { + version: SCIENTIFIC_IDENTITY_VERSION, + digest: sha256(&self.0), + } } - fn u32(&mut self, value: u32) { + pub(crate) fn u8(&mut self, value: u8) { + self.0.push(value); + } + pub(crate) fn u32(&mut self, value: u32) { self.0.extend_from_slice(&value.to_be_bytes()); } - - fn len(&mut self, value: usize) { + pub(crate) fn len(&mut self, value: usize) { self.0.extend_from_slice(&(value as u64).to_be_bytes()); } - - fn f64(&mut self, value: f64) { + pub(crate) fn f64(&mut self, value: f64) { let canonical = if value == 0.0 { 0.0 } else { value }; self.0.extend_from_slice(&canonical.to_bits().to_be_bytes()); } - fn text(&mut self, value: &[u8]) { self.len(value.len()); self.0.extend_from_slice(value); diff --git a/crates/rustiq-core/src/persistence/manifest.rs b/crates/rustiq-core/src/persistence/manifest.rs index 4a0ca36b..4715ced6 100644 --- a/crates/rustiq-core/src/persistence/manifest.rs +++ b/crates/rustiq-core/src/persistence/manifest.rs @@ -9,6 +9,7 @@ use super::{Sha256Digest, COMPACT_ERI_REPRESENTATION}; #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum ManifestKind { IntegralCache, + Portable, Unknown(String), } @@ -16,6 +17,7 @@ impl ManifestKind { pub(crate) fn as_str(&self) -> &str { match self { Self::IntegralCache => "integral-cache", + Self::Portable => "portable", Self::Unknown(value) => value, } } @@ -38,6 +40,7 @@ impl<'de> Deserialize<'de> for ManifestKind { let value = String::deserialize(deserializer)?; Ok(match value.as_str() { "integral-cache" => Self::IntegralCache, + "portable" => Self::Portable, _ => Self::Unknown(value), }) } @@ -51,6 +54,17 @@ pub(crate) struct Manifest { pub producer: Producer, pub scientific_identity: ScientificIdentityManifest, pub artifacts: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub calculation: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct CalculationManifest { + pub path: String, + pub version: u32, + pub size: u64, + pub digest: Sha256Digest, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -161,6 +175,7 @@ mod tests { ); let manifest = Manifest { + calculation: None, format: FORMAT_NAME.to_string(), format_version: FORMAT_VERSION, kind: ManifestKind::IntegralCache, diff --git a/crates/rustiq-core/src/persistence/mod.rs b/crates/rustiq-core/src/persistence/mod.rs index 1a5bad46..2ce6db51 100644 --- a/crates/rustiq-core/src/persistence/mod.rs +++ b/crates/rustiq-core/src/persistence/mod.rs @@ -5,6 +5,8 @@ //! location policy remains outside `rustiq-core`. mod cache_names; +mod calculation; +pub use calculation::{CalculationContext, ResolvedAo, ResolvedComponent}; mod checksum; mod data; mod eri_cache; @@ -18,7 +20,7 @@ pub use crate::eri::CompactEri; pub use checksum::{sha256, sha256_reader, verify_sha256, Sha256Digest, Sha256DigestParseError}; pub use data::{AoEriArtifact, Artifact, RustiQData}; pub use eri_cache::{EriCache, EriCacheEntry}; -pub use error::{ArtifactError, NpyError}; +pub use error::{ArtifactError, NpyError, PortableError}; pub(crate) use error::{ManifestError, PersistenceReadError, PersistenceWriteError, StorageError}; #[allow(unused_imports)] pub(crate) use identity::{ao_eri_identity, ScientificIdentity, AO_ERI_COMPUTATION_VERSION}; diff --git a/crates/rustiq-core/src/persistence/npy.rs b/crates/rustiq-core/src/persistence/npy.rs index 70014189..9d45647a 100644 --- a/crates/rustiq-core/src/persistence/npy.rs +++ b/crates/rustiq-core/src/persistence/npy.rs @@ -115,6 +115,57 @@ pub(crate) fn read_compact_eri( CompactEri::try_read_with_shape(reader, basis_functions) } +/// Validates the small NPY header and exact payload length before allocating values. +pub(crate) fn checked_eri_prefix( + reader: &mut dyn Read, + basis_functions: usize, + file_size: u64, +) -> Result, NpyError> { + let mut prefix = [0_u8; 8]; + reader.read_exact(&mut prefix).map_err(NpyError::Read)?; + if &prefix[..6] != b"\x93NUMPY" || !matches!((prefix[6], prefix[7]), (1..=3, 0)) { + return Err(NpyError::Read(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "invalid NPY prefix", + ))); + } + let length_bytes = if prefix[6] == 1 { 2 } else { 4 }; + let mut length = [0_u8; 4]; + reader + .read_exact(&mut length[..length_bytes]) + .map_err(NpyError::Read)?; + let header_len = u32::from_le_bytes(length) as usize; + if header_len > 65536 { + return Err(NpyError::Read(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "NPY header exceeds 64 KiB", + ))); + } + let mut header = Vec::with_capacity(8 + length_bytes + header_len); + header.extend_from_slice(&prefix); + header.extend_from_slice(&length[..length_bytes]); + header.resize(8 + length_bytes + header_len, 0); + reader + .read_exact(&mut header[8 + length_bytes..]) + .map_err(NpyError::Read)?; + if !validate_compact_eri_header(std::io::Cursor::new(&header), basis_functions, file_size) { + return Err(NpyError::Read(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "NPY dtype, shape or payload length is invalid", + ))); + } + Ok(header) +} + +pub(crate) fn read_checked_compact_eri( + reader: &mut dyn Read, + basis_functions: usize, + file_size: u64, +) -> Result { + let header = checked_eri_prefix(reader, basis_functions, file_size)?; + read_compact_eri(std::io::Cursor::new(header).chain(reader), basis_functions) +} + fn decode_compact_eri( npy: NpyFile, basis_functions: usize, diff --git a/crates/rustiq-core/src/persistence/storage.rs b/crates/rustiq-core/src/persistence/storage.rs index 7633beb9..726a7351 100644 --- a/crates/rustiq-core/src/persistence/storage.rs +++ b/crates/rustiq-core/src/persistence/storage.rs @@ -10,9 +10,13 @@ use sha2::{Digest, Sha256}; use super::{sha256_reader, ManifestError, Sha256Digest, StorageError}; +mod zip; + #[derive(Debug)] pub(crate) enum Storage { Folder(PathBuf), + ZipReader(Box<::zip::ZipArchive>), + ZipWriter(Box<::zip::ZipWriter>>), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -66,6 +70,26 @@ impl Write for DigestWriter { } impl Storage { + pub(crate) fn open_zip(path: &std::path::Path) -> Result { + Ok(Self::ZipReader(Box::new(zip::open(path)?))) + } + + pub(crate) fn create_zip(file: File) -> Self { + Self::ZipWriter(Box::new(zip::writer(file))) + } + + pub(crate) fn artifact_size(&mut self, path: &RelativePath) -> Result { + validate_path(path)?; + match self { + Self::Folder(root) => FolderStorage { root: root.clone() }.artifact_size(path), + Self::ZipReader(archive) => Ok(archive + .by_name(path.as_str()) + .map_err(zip::zip_error)? + .size()), + Self::ZipWriter(_) => Err(zip::invalid("cannot read an archive writer")), + } + } + pub(crate) fn folder(root: impl Into) -> Self { Self::Folder(root.into()) } @@ -76,6 +100,17 @@ impl Storage { ) -> Result { match self { Self::Folder(root) => FolderStorage { root: root.clone() }.artifact_metadata(path), + Self::ZipReader(archive) => { + let size = archive + .by_name(path.as_str()) + .map_err(zip::zip_error)? + .size(); + let digest = zip::with_member::<_, StorageError, _>(archive, path, |reader| { + Ok(sha256_reader(reader)?) + })?; + Ok(ArtifactMetadata { size, digest }) + } + Self::ZipWriter(_) => Err(zip::invalid("cannot read an archive writer")), } } @@ -86,6 +121,8 @@ impl Storage { { match self { Self::Folder(root) => FolderStorage { root: root.clone() }.with_artifact(path, read), + Self::ZipReader(archive) => zip::with_member(archive, path, read), + Self::ZipWriter(_) => Err(E::from(zip::invalid("cannot read an archive writer"))), } } @@ -100,6 +137,18 @@ impl Storage { { match self { Self::Folder(root) => FolderStorage { root: root.clone() }.write_artifact(path, write), + Self::ZipWriter(archive) => { + validate_path(path).map_err(E::from)?; + archive + .start_file(path.as_str(), zip::options(path)) + .map_err(zip::zip_error) + .map_err(E::from)?; + let mut writer = DigestWriter::new(archive.as_mut()); + write(&mut writer)?; + let (_, metadata) = writer.finish(); + Ok(metadata) + } + Self::ZipReader(_) => Err(E::from(zip::invalid("cannot write an archive reader"))), } } @@ -108,15 +157,21 @@ impl Storage { path: &RelativePath, max_size: u64, ) -> Result { - let size = match self { - Self::Folder(root) => FolderStorage { root: root.clone() }.artifact_size(path)?, - }; + let size = self.artifact_size(path)?; if size > max_size { return Err(ManifestError::TooLarge); } self.with_artifact(path, |reader| { - serde_json::from_reader(reader).map_err(ManifestError::from) + let mut bytes = Vec::new(); + reader + .take(max_size + 1) + .read_to_end(&mut bytes) + .map_err(StorageError::from)?; + if bytes.len() as u64 > max_size { + return Err(ManifestError::TooLarge); + } + serde_json::from_slice(&bytes).map_err(ManifestError::from) }) } @@ -124,18 +179,32 @@ impl Storage { &mut self, path: &RelativePath, value: &T, - ) -> Result<(), ManifestError> { + ) -> Result { + let mut bytes = serde_json::to_vec_pretty(value)?; + bytes.push(b'\n'); + let limit = if path.as_str() == super::calculation::CALCULATION_PATH { + super::calculation::MAX_CALCULATION_BYTES + } else { + 1024 * 1024 + }; + if bytes.len() as u64 > limit { + return Err(ManifestError::TooLarge); + } self.write_artifact::(path, |writer| { - serde_json::to_writer_pretty(&mut *writer, value)?; - writer.write_all(b"\n").map_err(StorageError::from)?; + writer.write_all(&bytes).map_err(StorageError::from)?; Ok(()) - })?; - Ok(()) + }) } pub(crate) fn finish(self) -> Result<(), StorageError> { match self { - Self::Folder(_) => Ok(()), + Self::Folder(_) | Self::ZipReader(_) => Ok(()), + Self::ZipWriter(archive) => { + let mut file = archive.finish().map_err(zip::zip_error)?; + file.flush()?; + file.get_ref().sync_all()?; + Ok(()) + } } } } diff --git a/crates/rustiq-core/src/persistence/storage/zip.rs b/crates/rustiq-core/src/persistence/storage/zip.rs new file mode 100644 index 00000000..0821f717 --- /dev/null +++ b/crates/rustiq-core/src/persistence/storage/zip.rs @@ -0,0 +1,336 @@ +//! Bounded ZIP indexing before handing the archive to the container library. +//! `zip` indexes by name and can hide duplicate central-directory records, so +//! validate the original records before constructing its index. +use super::{validate_path, StorageError}; +use relative_path::RelativePath; +use std::{ + collections::BTreeSet, + fs::File, + io::{self, BufReader, BufWriter, Read, Seek, SeekFrom}, + path::Path, +}; +use zip::{write::SimpleFileOptions, CompressionMethod, ZipArchive, ZipWriter}; + +pub(super) const MAX_MEMBERS: usize = 4096; +const MAX_DIRECTORY_BYTES: u64 = 16 * 1024 * 1024; +const MAX_END_RECORD_BYTES: u64 = 65536; + +pub(super) fn invalid(message: impl Into) -> StorageError { + StorageError::Archive(message.into()) +} +pub(super) fn zip_error(error: zip::result::ZipError) -> StorageError { + invalid(error.to_string()) +} + +pub(super) fn open(path: &Path) -> Result, StorageError> { + let mut file = File::open(path)?; + let (count, directory_start) = preflight(&mut file)?; + file.rewind()?; + let mut archive = ZipArchive::new(file).map_err(zip_error)?; + if archive.len() != count + || archive.central_directory_start() != directory_start + || archive.offset() != 0 + { + return Err(invalid("inconsistent archive directory")); + } + if archive.has_overlapping_files().map_err(zip_error)? { + return Err(invalid("overlapping ZIP members")); + } + for index in 0..archive.len() { + let member = archive.by_index_raw(index).map_err(zip_error)?; + if member + .data_start() + .and_then(|start| start.checked_add(member.compressed_size())) + .is_none_or(|end| end > directory_start) + { + return Err(invalid("member extends into the central directory")); + } + } + Ok(archive) +} + +pub(super) fn options(path: &RelativePath) -> SimpleFileOptions { + let metadata = matches!(path.as_str(), "manifest.json" | "calculation.json"); + SimpleFileOptions::default() + .compression_method(if metadata { + CompressionMethod::Deflated + } else { + CompressionMethod::Stored + }) + .last_modified_time(zip::DateTime::default()) + .unix_permissions(0o644) + // Streaming artifacts have no known size when their local header is emitted. + // Reserve ZIP64 fields even for small arrays, without buffering the payload. + .large_file(!metadata) +} + +pub(super) fn writer(file: File) -> ZipWriter> { + ZipWriter::new(BufWriter::new(file)) +} + +fn u16_at(bytes: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(bytes[offset..offset + 2].try_into().unwrap()) +} +fn u32_at(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) +} +fn u64_at(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap()) +} + +fn preflight(file: &mut File) -> Result<(usize, u64), StorageError> { + let length = file.metadata()?.len(); + let tail_len = length.min(22 + u64::from(u16::MAX)); + file.seek(SeekFrom::Start(length - tail_len))?; + let mut tail = vec![0; tail_len as usize]; + file.read_exact(&mut tail)?; + let end = (0..tail.len().saturating_sub(21)) + .rev() + .find(|&i| { + tail[i..i + 4] == *b"PK\x05\x06" + && i + 22 + usize::from(u16_at(&tail, i + 20)) == tail.len() + }) + .ok_or_else(|| invalid("missing or truncated ZIP end record"))?; + let eocd = &tail[end..]; + let end_offset = length - tail_len + end as u64; + if u16_at(eocd, 4) != 0 || u16_at(eocd, 6) != 0 || u16_at(eocd, 8) != u16_at(eocd, 10) { + return Err(invalid("multi-disk archives are unsupported")); + } + let mut count = u64::from(u16_at(eocd, 10)); + let mut size = u64::from(u32_at(eocd, 12)); + let mut offset = u64::from(u32_at(eocd, 16)); + let mut directory_end = end_offset; + // A locator can exist even when classic EOCD fields have not overflowed. + let mut locator = [0; 20]; + let has_locator = if end_offset >= 20 { + file.seek(SeekFrom::Start(end_offset - 20))?; + file.read_exact(&mut locator)?; + locator[..4] == *b"PK\x06\x07" + } else { + false + }; + if has_locator { + if u32_at(&locator, 4) != 0 || u32_at(&locator, 16) != 1 { + return Err(invalid("multi-disk ZIP64 archive")); + } + let record_offset = u64_at(&locator, 8); + if record_offset + .checked_add(56) + .is_none_or(|end| end > end_offset - 20) + { + return Err(invalid("invalid ZIP64 end offset")); + } + file.seek(SeekFrom::Start(record_offset))?; + let mut record = [0; 56]; + file.read_exact(&mut record)?; + let record_size = u64_at(&record, 4); + if record[..4] != *b"PK\x06\x06" + || !(44..=MAX_END_RECORD_BYTES).contains(&record_size) + || record_offset + .checked_add(12) + .and_then(|n| n.checked_add(record_size)) + != Some(end_offset - 20) + || u32_at(&record, 16) != 0 + || u32_at(&record, 20) != 0 + || u64_at(&record, 24) != u64_at(&record, 32) + { + return Err(invalid("invalid ZIP64 end record")); + } + let (count64, size64, offset64) = ( + u64_at(&record, 32), + u64_at(&record, 40), + u64_at(&record, 48), + ); + if (count != u64::from(u16::MAX) && count != count64) + || (size != u64::from(u32::MAX) && size != size64) + || (offset != u64::from(u32::MAX) && offset != offset64) + { + return Err(invalid("inconsistent ZIP64 directory fields")); + } + (count, size, offset) = (count64, size64, offset64); + directory_end = record_offset; + } else if count == u64::from(u16::MAX) + || size == u64::from(u32::MAX) + || offset == u64::from(u32::MAX) + { + return Err(invalid("missing ZIP64 end record")); + } + if count > MAX_MEMBERS as u64 + || size > MAX_DIRECTORY_BYTES + || offset.checked_add(size) != Some(directory_end) + { + return Err(invalid("oversized or inconsistent ZIP directory")); + } + file.seek(SeekFrom::Start(offset))?; + let mut directory = vec![0; size as usize]; + file.read_exact(&mut directory)?; + let mut cursor = 0_usize; + let mut paths = BTreeSet::new(); + for _ in 0..count { + let header = directory + .get(cursor..cursor + 46) + .ok_or_else(|| invalid("truncated central header"))?; + if header[..4] != *b"PK\x01\x02" { + return Err(invalid("invalid central header")); + } + let flags = u16_at(header, 8); + let method = u16_at(header, 10); + if flags & !0x080e != 0 || !matches!(method, 0 | 8) { + return Err(invalid("unsupported encryption, flags or compression")); + } + let attributes = u32_at(header, 38); + let file_type = (attributes >> 16) & 0o170000; + if attributes & 0x10 != 0 || !matches!(file_type, 0 | 0o100000) { + return Err(invalid("only regular file members are supported")); + } + if u16_at(header, 34) != 0 { + return Err(invalid("multi-disk member")); + } + let name_len = usize::from(u16_at(header, 28)); + let extra_len = usize::from(u16_at(header, 30)); + let comment_len = usize::from(u16_at(header, 32)); + let name_start = cursor + 46; + let next = name_start + name_len + extra_len + comment_len; + if next > directory.len() { + return Err(invalid("truncated directory member")); + } + let name_bytes = &directory[name_start..name_start + name_len]; + if !name_bytes.is_ascii() && flags & 0x0800 == 0 { + return Err(invalid("non-ASCII member names must use the UTF-8 flag")); + } + validate_extras(&directory[name_start + name_len..name_start + name_len + extra_len])?; + let name = std::str::from_utf8(name_bytes).map_err(|_| invalid("non-UTF-8 member path"))?; + validate_path(RelativePath::new(name))?; + let key = name.to_lowercase(); + if !paths.insert(key.clone()) { + return Err(invalid("duplicate member path")); + } + for (index, _) in key.match_indices('/') { + if paths.contains(&key[..index]) { + return Err(invalid("file/directory path conflict")); + } + } + if paths + .range(format!("{key}/")..) + .next() + .is_some_and(|p| p.starts_with(&format!("{key}/"))) + { + return Err(invalid("file/directory path conflict")); + } + // Resolve only the local offset from ZIP64; the library validates full extras. + let mut local_offset = u64::from(u32_at(header, 42)); + if local_offset == u64::from(u32::MAX) { + let extra = &directory[name_start + name_len..name_start + name_len + extra_len]; + let skip = 8 + * (usize::from(u32_at(header, 24) == u32::MAX) + + usize::from(u32_at(header, 20) == u32::MAX)); + local_offset = zip64_offset(extra, skip)?; + } + if local_offset.checked_add(30).is_none_or(|end| end > offset) { + return Err(invalid("invalid local header offset")); + } + file.seek(SeekFrom::Start(local_offset))?; + let mut local = [0; 30]; + file.read_exact(&mut local)?; + let local_name_len = usize::from(u16_at(&local, 26)); + let local_extra_len = u64::from(u16_at(&local, 28)); + if local[..4] != *b"PK\x03\x04" + || u16_at(&local, 6) != flags + || u16_at(&local, 8) != method + || local_name_len != name_len + || local_offset + .checked_add(30 + name_len as u64 + local_extra_len) + .is_none_or(|end| end > offset) + { + return Err(invalid("inconsistent local header")); + } + let mut local_name = vec![0; name_len]; + file.read_exact(&mut local_name)?; + if local_name != name_bytes { + return Err(invalid("local/central member names differ")); + } + let mut local_extra = vec![0; local_extra_len as usize]; + file.read_exact(&mut local_extra)?; + validate_extras(&local_extra)?; + cursor = next; + } + if cursor != directory.len() { + return Err(invalid("unindexed central-directory records")); + } + Ok((count as usize, offset)) +} +fn validate_extras(mut extra: &[u8]) -> Result<(), StorageError> { + let mut kinds = BTreeSet::new(); + while !extra.is_empty() { + if extra.len() < 4 { + return Err(invalid("truncated extra field")); + } + let kind = u16_at(extra, 0); + let len = usize::from(u16_at(extra, 2)); + if kind == 0x7075 || !kinds.insert(kind) { + return Err(invalid( + "alternate Unicode paths or duplicate extra fields are unsupported", + )); + } + extra = extra + .get(4 + len..) + .ok_or_else(|| invalid("truncated extra field"))?; + } + Ok(()) +} + +fn zip64_offset(mut extra: &[u8], skip: usize) -> Result { + while extra.len() >= 4 { + let kind = u16_at(extra, 0); + let len = usize::from(u16_at(extra, 2)); + let field = extra + .get(4..4 + len) + .ok_or_else(|| invalid("truncated extra field"))?; + if kind == 1 { + let value = field + .get(skip..skip + 8) + .ok_or_else(|| invalid("missing ZIP64 offset"))?; + return Ok(u64_at(value, 0)); + } + extra = &extra[4 + len..]; + } + Err(invalid("missing ZIP64 extra field")) +} + +/// Stops decompression at the declared size and checks actual EOF/CRC even when +/// a consumer (such as an NPY decoder) reads exactly the declared payload. +pub(super) fn with_member( + archive: &mut ZipArchive, + path: &RelativePath, + read: F, +) -> Result +where + E: From, + F: FnOnce(&mut dyn Read) -> Result, +{ + let member = archive + .by_name(path.as_str()) + .map_err(zip_error) + .map_err(E::from)?; + let size = member.size(); + let mut bounded = BufReader::new(member.take(size)); + let value = read(&mut bounded)?; + io::copy(&mut bounded, &mut io::sink()) + .map_err(StorageError::from) + .map_err(E::from)?; + let bounded = bounded.into_inner(); + if bounded.limit() != 0 { + return Err(E::from(invalid("truncated ZIP member"))); + } + let mut member = bounded.into_inner(); + let mut trailing = [0; 1]; + if member + .read(&mut trailing) + .map_err(StorageError::from) + .map_err(E::from)? + != 0 + { + return Err(E::from(invalid("member exceeds declared size"))); + } + Ok(value) +} diff --git a/crates/rustiq-core/tests/data/persistence/calculation-h2-v1.json b/crates/rustiq-core/tests/data/persistence/calculation-h2-v1.json new file mode 100644 index 00000000..c8bd2f5a --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/calculation-h2-v1.json @@ -0,0 +1,101 @@ +{ + "format": "rustiq-calculation", + "version": 1, + "units": "bohr", + "atoms": [ + { + "atomic_number": 1, + "position": [ + 0.0, + 0.0, + 0.0 + ] + }, + { + "atomic_number": 1, + "position": [ + 1.4, + 0.0, + 0.0 + ] + } + ], + "charge": 0, + "multiplicity": 1, + "basis": [ + { + "center": [ + 0.0, + 0.0, + 0.0 + ], + "components": [ + { + "angular_momentum": [ + 0, + 0, + 0 + ], + "primitives": [ + { + "exponent": 3.425250914, + "coefficient": 0.2769343550790519 + }, + { + "exponent": 0.6239137298, + "coefficient": 0.26783885160947885 + }, + { + "exponent": 0.168855404, + "coefficient": 0.08347367112984119 + } + ] + } + ] + }, + { + "center": [ + 1.4, + 0.0, + 0.0 + ], + "components": [ + { + "angular_momentum": [ + 0, + 0, + 0 + ], + "primitives": [ + { + "exponent": 3.425250914, + "coefficient": 0.2769343550790519 + }, + { + "exponent": 0.6239137298, + "coefficient": 0.26783885160947885 + }, + { + "exponent": 0.168855404, + "coefficient": 0.08347367112984119 + } + ] + } + ] + } + ], + "hf": { + "method": "rhf", + "max_iterations": 100, + "convergence_threshold": 1e-8, + "linear_dependency_threshold": 1e-8, + "eri_schwarz_threshold": 1e-12, + "diis_size": null, + "guess": { + "kind": "core_hamiltonian", + "perturbation": null + } + }, + "mp2": null, + "ao_eri_computation_version": 1 +} diff --git a/crates/rustiq-core/tests/data/persistence/portable-python-big-v1.rustiq.hex b/crates/rustiq-core/tests/data/persistence/portable-python-big-v1.rustiq.hex new file mode 100644 index 00000000..61e6bb94 --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/portable-python-big-v1.rustiq.hex @@ -0,0 +1 @@ +504b03042d0000000800000021003e7a0d31ffffffffffffffff0d0014006d616e69666573742e6a736f6e010010003c03000000000000ad0100000000000075525b6edb3010fcf7290c7dc7091f161f39456f602cb9cb98ad2d292415d40d7cf7927e444683fe08e2ec706766b99fabf5ba0b633a42e95ed75d9a7389ef9b89528eb9d0e0a97b5a18bb8f868f4365f20bfc2b0ed8ae4d632ae00e37f294469c3da55af9ace78a0c70a4c6fb712afb7158c7a1501aab08b87888e5b40ef17799d3f57aa52f321def2a74beb4cd3ed25062887e17b1fd95d3a2f08fb38a607ca37c0995f7207af5ca020a0dc082e5561bc5552f84776acba4e14ea093ca714472ceea2003f6120c934a68a6b9f6cafbc58987839f0f50ae82370713947d537b283effcc95f12dd4dd618e7fda54b8b1e27f9e7b2e2492e70a04b7bde352588b3a6c9d6d611c33d6c9a0ac0cbe76719e056724859a8a91b2a1ef17cf90eae8c097bc38867147297e9d1f32404a70ca2fed9dde121cf20b8c9b4a7d1ea6d32dce837dadbea06ffe111d0a2990340f5bc134a22150ae97e48dc6adc1207b8e4e188b5e736218d896505b634c1fd085452dd19428d767bf8ffdbeab7e3c4e3557f3b7f9e0cb052825453717ca0f092beea0aef62ecc836f8d5a513c2dd5d66dbe6a3c6efb8d705eddbfe7d579f517504b03042d000000080000002100021adb15ffffffffffffffff1000140063616c63756c6174696f6e2e6a736f6e010010006407000000000000e201000000000000ed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b03042d000000000000002100feb6142cffffffffffffffff1b0014006172726179732f696e74656772616c732f616f2d6572692e6e707901001000b000000000000000b000000000000000934e554d5059010076007b276465736372273a20273e6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a3fe00000000000003ff80000000000004004000000000000400c00000000000040120000000000004016000000000000504b01022d032d0000000800000021003e7a0d31ffffffffffffffff0d0014000000000000000000a481000000006d616e69666573742e6a736f6e010010003c03000000000000ad01000000000000504b01022d032d000000080000002100021adb15ffffffffffffffff10001c000000000000000000a481ffffffff63616c63756c6174696f6e2e6a736f6e010018006407000000000000e201000000000000ec01000000000000504b01022d032d000000000000002100feb6142cffffffffffffffff1b001c000000000000000000a481ffffffff6172726179732f696e74656772616c732f616f2d6572692e6e707901001800b000000000000000b0000000000000001004000000000000504b06062c000000000000002d002d000000000000000000030000000000000003000000000000000e010000000000000d05000000000000504b0607000000001b0600000000000001000000504b050600000000030003000e0100000d0500000000 diff --git a/crates/rustiq-core/tests/data/persistence/portable-python-little-v1.rustiq.hex b/crates/rustiq-core/tests/data/persistence/portable-python-little-v1.rustiq.hex new file mode 100644 index 00000000..03cd60a0 --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/portable-python-little-v1.rustiq.hex @@ -0,0 +1 @@ +504b03042d00000008000000210048a229b3ffffffffffffffff0d0014006d616e69666573742e6a736f6e010010003c03000000000000ac0100000000000075525b72dc2010fcdf53a8f4edb5792c087c8adc606b80c14ba2956440ae28aebd7b601f962aaefc50d0d34c770f7cee9aa6f5633c436e5f9b36ce2987f7fd8431859471b0d83ead8ce347c5c7a130e915fe150657af4d63cc60fa3b798aa39b2dc652f92ce7820c70c6cafbb1e4d3383461c818c7220226f4212f8d0fbff31c6fd70b7d9569695ba0cbb56db201871c7cb0c7e0ea2e2fabc23fce0ae2c21ba66ba8740226e42bf18e7500c46baa3b25a9148c59230f842b6a98335c1aea1c1aa33bcfbd131c14e19275a4a39d95d6ae4e2cf476ee21df04ef0e26c8a7aab6293eff4c85f12dd4c3610a7fea54a8d2ec7f9e0565dca1a51218d5c250ceb4769d3f185dc318a2b4e15e6aee6de9622cf14671f4251541a9bd10ab6788657460735a1dc378c418bece9b0c10232ce9a5bed35b843ebdc0b82fd4e7615aee7136f63bf9057df3ef1457440b4f34a30e9d605628ec34f59e8223de0aa19ca5e6600e1c0065291b3c08a4daea8e97fdaa16718a98cab33fc6fef8ab763c4f2557f5b7ffa0eb05c8390633674c9b84053750bef6d1cf83ad8d6a913dadd5da6dbe696c7ffb9d70d93dd6cbeeb2fb0b504b03042d000000080000002100021adb15ffffffffffffffff1000140063616c63756c6174696f6e2e6a736f6e010010006407000000000000e201000000000000ed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b03042d000000000000002100db95d32dffffffffffffffff1b0014006172726179732f696e74656772616c732f616f2d6572692e6e707901001000b000000000000000b000000000000000934e554d5059010076007b276465736372273a20273c6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a000000000000e03f000000000000f83f00000000000004400000000000000c4000000000000012400000000000001640504b01022d032d00000008000000210048a229b3ffffffffffffffff0d0014000000000000000000a481000000006d616e69666573742e6a736f6e010010003c03000000000000ac01000000000000504b01022d032d000000080000002100021adb15ffffffffffffffff10001c000000000000000000a481ffffffff63616c63756c6174696f6e2e6a736f6e010018006407000000000000e201000000000000eb01000000000000504b01022d032d000000000000002100db95d32dffffffffffffffff1b001c000000000000000000a481ffffffff6172726179732f696e74656772616c732f616f2d6572692e6e707901001800b000000000000000b0000000000000000f04000000000000504b06062c000000000000002d002d000000000000000000030000000000000003000000000000000e010000000000000c05000000000000504b0607000000001a0600000000000001000000504b050600000000030003000e0100000c0500000000 diff --git a/crates/rustiq-core/tests/portable_artifact.rs b/crates/rustiq-core/tests/portable_artifact.rs new file mode 100644 index 00000000..99893039 --- /dev/null +++ b/crates/rustiq-core/tests/portable_artifact.rs @@ -0,0 +1,56 @@ +use rustiq_core::{ + basis::BasisFile, + calculation::CalculationBuilder, + molecules::geometry::Geometry, + persistence::{AoEriArtifact, CompactEri, PortableError, RustiQData}, +}; + +#[test] +fn public_api_creates_inspects_and_recovers_a_portable_artifact() { + let geometry = Geometry::from_source("h2.xyz", "2\nH2\nH 0 0 0\nH 1.4 0 0\n").unwrap(); + let basis = BasisFile::from_reader(&include_bytes!("data/sto-3g.json")[..]).unwrap(); + let prepared = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + let mut data = RustiQData::from_calculation(&prepared).unwrap(); + let values = vec![0.5, 1.5, 2.5, 3.5, 4.5, 5.5]; + let indices = [ + (0, 0, 0, 0), + (1, 0, 0, 0), + (1, 0, 1, 0), + (1, 1, 0, 0), + (1, 1, 1, 0), + (1, 1, 1, 1), + ]; + let mut eri = CompactEri::Zeroed(2); + for (index, value) in indices.into_iter().zip(&values) { + eri[index] = *value; + } + data.set::(eri).unwrap(); + assert_eq!( + data.artifact_representations().collect::>(), + vec![("ao_eri", "rustiq-compact-eri-v1")] + ); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("h2.rustiq"); + data.write(&path).unwrap(); + assert!(matches!( + data.write(&path), + Err(PortableError::AlreadyExists) + )); + let mut restored = RustiQData::open(&path).unwrap(); + let context = restored.calculation().unwrap(); + assert_eq!(context.charge(), 0); + assert_eq!(context.multiplicity(), 1); + let ao = context.basis().next().unwrap(); + assert_eq!(ao.center(), [0.0; 3]); + let component = ao.components().next().unwrap(); + assert_eq!(component.angular_momentum(), [0; 3]); + assert_eq!(component.primitives().len(), 3); + assert_eq!(restored.producer().0, "RustiQ"); + assert!(restored.eri_is_compatible(&prepared)); + let eri = restored.get::().unwrap().unwrap(); + for (index, value) in indices.into_iter().zip(values) { + assert_eq!(eri[index], value); + } +} diff --git a/docs/persistence-format-v1.md b/docs/persistence-format-v1.md index f7e2f655..998241f7 100644 --- a/docs/persistence-format-v1.md +++ b/docs/persistence-format-v1.md @@ -1,7 +1,7 @@ # RustiQ logical persistence format V1 This document specifies the storage-independent logical format used by RustiQ -scientific persistence. It does not specify an archive container. +scientific persistence, including the portable `.rustiq` ZIP/ZIP64 container. ## Active AO ERI cache @@ -89,8 +89,8 @@ untouched. Names and fingerprints are validated rather than interpreted as paths The core Rust API exposes `persistence::RustiQData` as a typed scientific facade. Physical storage selection is an internal persistence concern: the -directory-backed ERI cache resolves folder storage internally, and the future -portable `.rustiq` API will resolve ZIP/ZIP64 internally rather than exposing a +directory-backed ERI cache resolves folder storage internally, and the +portable `.rustiq` API resolves ZIP/ZIP64 internally rather than exposing a generic storage backend. Internal readers load the bounded manifest first and open scientific artifacts on demand. `read_eri` validates and decodes the AO ERI NPY on first access and keeps the resulting `CompactEri` for subsequent accesses. @@ -232,3 +232,190 @@ as the AO ERI artifact attribute `attributes.computation_version`. Bumping that version changes the fingerprint and causes older ERI artifacts to be reported as `invalid`, even when their inputs and storage representation are otherwise unchanged. + +## Portable `.rustiq` V1 + +A portable artifact is a ZIP archive, with ZIP64 support for large members and +large offsets. It contains `manifest.json`, `calculation.json`, and zero or more +indexed scientific artifacts. It is not an NPZ file. Readers access members by +name without extracting files. The active directory cache remains a separate, +disposable machine-local product with its existing format and behavior. + +The portable manifest uses `kind: "portable"`. Its format/version, producer, +scientific identity and artifact envelope are the same as the logical format +above. It additionally requires a calculation reference: + +```json +"calculation": { + "path": "calculation.json", + "version": 1, + "size": 1234, + "digest": "sha256:..." +} +``` + +`size` and `digest` describe the exact uncompressed JSON bytes, including any +whitespace. Cache manifests do not require this field. Producer version remains +provenance, not an invalidation rule. Archives may contain no numerical artifacts; +the scientific context remains useful by itself. + +### Resolved scientific snapshot + +The normative structural schema is +[`calculation-snapshot-v1.schema.json`](../schemas/calculation-snapshot-v1.schema.json). +The [H2 golden snapshot](../crates/rustiq-core/tests/data/persistence/calculation-h2-v1.json) +provides a complete example. Wire records are explicitly defined, independently +of domain Rust struct layouts and package versions. + +- `format: "rustiq-calculation"`, `version: 1`, `units: "bohr"` identify the + snapshot contract and coordinate unit. +- `atoms` contains ordered atomic numbers and positions; `charge` and + `multiplicity` define the molecular electronic state. +- `basis` contains effective AOs in integral-engine order. Each has a center, + ordered Cartesian components with three angular-momentum integers, and ordered + primitives with positive exponents and effective normalized coefficients. + These include any spherical-component weights already applied by the engine. + Consumers must not normalize them again. A basis name or source file is not + needed to interpret the stored AO representation. +- `hf` contains the resolved `rhf`/`uhf` method, iteration limit, convergence and + linear-dependency thresholds, Schwarz threshold, DIIS history size, and density + guess. `null`/omitted Schwarz threshold means screening disabled; + `null`/omitted DIIS size means DIIS disabled. +- Density guesses are tagged as `core_hamiltonian`, `one_electron`, `random`, or + `zero`. Perturbations and random guesses record distribution parameters and an + optional seed. An absent seed records an unseeded request, not the random + generator state or an exact continuation promise. +- `mp2` is absent/null for HF-only requests, or contains `frozen_orbitals`. + This records the request, not an MP2 result or proof of convergence. +- `ao_eri_computation_version` states the scientific algorithm identity used for + AO ERIs; the snapshot independently reproduces `scientific-identity-v1`. + +Coordinates and all floating parameters must be finite. Exponents and positive +thresholds must be greater than zero. Electron counts, multiplicity and RHF method +must be mutually consistent. Random uniform bounds must be finite, ordered, and +have a finite width; normal standard deviation must be positive. Arrays must be +nonempty where the schema requires them. Cross-field scientific checks supplement +the structural JSON schema. Unsupported snapshot versions and unknown fields in +known snapshot records are errors. + +The snapshot excludes cache locations, source paths/spans, output presentation, +and memory budgets. Floating-point JSON parsing preserves binary64 round trips, +so reparsing a snapshot cannot change its canonical scientific identity. +The snapshot does not yet provide direct archive execution or reconstruct an SCF +restart state. + +### Rust API + +`RustiQData::from_calculation(&PreparedCalculation)` captures resolved inputs +without executing HF. `set_eri` / `set::` supply an owned +`CompactEri` whose dimension matches those inputs. The caller is responsible for +supplying the ERIs computed for that calculation, rather than unrelated values +of the same shape. + +```rust,ignore +use rustiq_core::persistence::{CompactEri, RustiQData}; + +// `prepared` is produced by CalculationBuilder; `eri` belongs to these inputs. +let mut data = RustiQData::from_calculation(&prepared)?; +data.set_eri(eri)?; +data.write("water.rustiq")?; + +let mut restored = RustiQData::open("water.rustiq")?; +let context = restored.calculation().expect("portable context"); +println!("{:?}: {} AOs", context.hf_method(), context.basis().len()); +for (name, representation) in restored.artifact_representations() { + println!("{name}: {representation}"); +} +if restored.eri_is_compatible(&prepared) { + let eri: &CompactEri = restored.read_eri()?; +} +``` + +`calculation()` exposes read-only typed context, including atoms, effective basis, +HF settings and MP2 request. `producer()` exposes provenance, and +`artifact_representations()` lists known and unknown artifacts without loading +arrays. These inspections do not certify payload integrity. + +`open` validates the container index, manifest, snapshot digest/schema, scientific +identity, and declared artifact sizes. ERI dimensions and computation-version +metadata must agree with the snapshot. It does not decode numerical arrays. +The first typed read verifies SHA-256, bounded NPY header, f64 dtype, shape and +exact payload length before decoding. Further reads return the same object. +Missing ERIs are represented by `get::() == Ok(None)`; an +unsupported representation is never silently treated as usable scientific data. + +`eri_is_compatible` compares effective integral inputs with another prepared +calculation without loading arrays. Changing an HF-only request to MP2 does not +invalidate AO ERIs. Compatibility does not imply that the payload has already +been verified. There is no CLI reuse orchestration in this API release. + +Writing a reopened archive preserves unloaded artifacts by verified streaming +copy, including unknown representations and their attributes. Known ERI headers +are checked before preservation; overwritten ERIs use the current representation. +Errors use `PortableError` (including `AlreadyExists`, unsupported version, +invalid context/archive, I/O and artifact errors), without public ZIP types. + +### Publication and container policy + +Writing creates a sibling temporary file, writes snapshot and artifacts with +SHA-256 metadata, writes the manifest, finalizes the ZIP, syncs the file, and +reopens it to validate the final index/context. Publication uses an atomic +no-clobber operation. **An existing destination is never replaced**, including +when two producers race to create it. Unix additionally syncs the parent +directory after publication. Failures before publication leave the destination +untouched and clean up the temporary file. A directory-sync error after +publication can leave a complete output file and is still reported as an error. + +Writers use `Stored` for numerical/opaque artifacts and `Deflated` for the two +JSON members. Streaming payloads reserve ZIP64 local-header fields even for +small arrays, allowing growth past 4 GiB without buffering. ZIP64 central/end +records are emitted when required. Timestamps are fixed to 1980-01-01 00:00:00, +permissions to regular files with mode 0644, and member order is deterministic. +Repeated writes of the same data by the same implementation produce identical +bytes; compression-library changes are not promised byte-identical output and +never change scientific identity. + +### Defensive reading limits + +Portable V1 accepts only single-disk ZIP archives with regular file members; +directory entries are unnecessary and rejected. Supported compression is Stored +or Deflated. Encryption, links, devices and other special entry types are errors. +The reader validates original central-directory records before the ZIP library +indexes them, so duplicate names cannot be silently collapsed. + +Limits applied before parsing or allocating payloads: + +| Structure | Limit | +| --- | ---: | +| Uncompressed manifest | 1 MiB | +| Uncompressed calculation snapshot | 64 MiB | +| Members | 4,096 | +| Central directory | 16 MiB | +| ZIP64 end-record body | 64 KiB | +| NPY header body | 64 KiB | + +All member paths obey the logical path rules above. File/ancestor conflicts and +case-insensitive collisions are rejected; metadata paths are reserved. Names must +be valid UTF-8; non-ASCII names require the ZIP UTF-8 flag. Alternative Unicode +path extra fields and duplicate extra fields are rejected to avoid competing +names. Local/central names, flags and compression must agree. Invalid offsets, +overlapping payloads, truncated directory records, and inconsistent ZIP64 fields +are rejected. ZIP metadata names are not extracted onto the filesystem. + +Actual decompressed length and ZIP CRC are checked, in addition to independent +SHA-256 digests. Streamed members cannot exceed their declared uncompressed size. +NPY object/pickle arrays, unsupported dtype/rank, dimension overflow, extra bytes, +and incomplete arrays are rejected. Large legitimate ERIs still require memory +for the decoded `CompactEri`; there is no artificial 4 GiB payload limit. + +### Interoperability fixtures + +`tools/reference/generate_portable_fixtures.py` regenerates Python-produced +ZIP64 fixtures using only standard-library modules and the existing NumPy byte +fixtures. Both little- and big-endian f64 payloads are covered. The Python tests +independently reproduce the scientific identity and verify ZIP/JSON/NumPy output +written by Rust. Rust tests also exercise a sparse archive with offsets beyond +4 GiB without allocating a multi-gigabyte payload. + +Run `uv run --locked pytest tools/reference` for interoperability and scientific +reference comparisons, separately from the Cargo suite. diff --git a/schemas/calculation-snapshot-v1.schema.json b/schemas/calculation-snapshot-v1.schema.json new file mode 100644 index 00000000..5bacf66e --- /dev/null +++ b/schemas/calculation-snapshot-v1.schema.json @@ -0,0 +1,379 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/mveril/RustiQ/blob/main/schemas/calculation-snapshot-v1.schema.json", + "title": "RustiQ resolved scientific calculation V1", + "type": "object", + "additionalProperties": false, + "required": [ + "format", + "version", + "units", + "atoms", + "charge", + "multiplicity", + "basis", + "hf", + "ao_eri_computation_version" + ], + "properties": { + "format": { + "const": "rustiq-calculation" + }, + "version": { + "const": 1 + }, + "units": { + "const": "bohr" + }, + "atoms": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/atom" + } + }, + "charge": { + "type": "integer", + "minimum": -2147483648, + "maximum": 2147483647 + }, + "multiplicity": { + "type": "integer", + "minimum": 1, + "maximum": 255 + }, + "basis": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/ao" + } + }, + "hf": { + "$ref": "#/$defs/hf" + }, + "mp2": { + "anyOf": [ + { + "$ref": "#/$defs/mp2" + }, + { + "type": "null" + } + ] + }, + "ao_eri_computation_version": { + "type": "integer", + "minimum": 1, + "maximum": 4294967295 + } + }, + "$defs": { + "atom": { + "type": "object", + "additionalProperties": false, + "required": [ + "atomic_number", + "position" + ], + "properties": { + "atomic_number": { + "type": "integer", + "minimum": 1, + "maximum": 118 + }, + "position": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "items": { + "type": "number" + } + } + } + }, + "primitive": { + "type": "object", + "additionalProperties": false, + "required": [ + "exponent", + "coefficient" + ], + "properties": { + "exponent": { + "type": "number", + "exclusiveMinimum": 0 + }, + "coefficient": { + "type": "number" + } + } + }, + "component": { + "type": "object", + "additionalProperties": false, + "required": [ + "angular_momentum", + "primitives" + ], + "properties": { + "angular_momentum": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "items": { + "type": "integer", + "minimum": 0, + "maximum": 255 + } + }, + "primitives": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/primitive" + } + } + } + }, + "ao": { + "type": "object", + "additionalProperties": false, + "required": [ + "center", + "components" + ], + "properties": { + "center": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "items": { + "type": "number" + } + }, + "components": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/component" + } + } + } + }, + "distribution": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "min", + "max" + ], + "properties": { + "kind": { + "const": "uniform" + }, + "min": { + "type": "number" + }, + "max": { + "type": "number" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "mean", + "std_dev" + ], + "properties": { + "kind": { + "const": "normal" + }, + "mean": { + "type": "number" + }, + "std_dev": { + "type": "number", + "exclusiveMinimum": 0 + } + } + } + ] + }, + "random": { + "type": "object", + "additionalProperties": false, + "required": [ + "distribution" + ], + "properties": { + "seed": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615 + }, + { + "type": "null" + } + ] + }, + "distribution": { + "$ref": "#/$defs/distribution" + } + } + }, + "guess": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "core_hamiltonian" + }, + "perturbation": { + "anyOf": [ + { + "$ref": "#/$defs/random" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "one_electron" + }, + "perturbation": { + "anyOf": [ + { + "$ref": "#/$defs/random" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "random" + ], + "properties": { + "kind": { + "const": "random" + }, + "random": { + "$ref": "#/$defs/random" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "zero" + } + } + } + ] + }, + "hf": { + "type": "object", + "additionalProperties": false, + "required": [ + "method", + "max_iterations", + "convergence_threshold", + "linear_dependency_threshold", + "guess" + ], + "properties": { + "method": { + "enum": [ + "rhf", + "uhf" + ] + }, + "max_iterations": { + "type": "integer", + "minimum": 1 + }, + "convergence_threshold": { + "type": "number", + "exclusiveMinimum": 0 + }, + "linear_dependency_threshold": { + "type": "number", + "minimum": 0 + }, + "eri_schwarz_threshold": { + "anyOf": [ + { + "type": "number", + "exclusiveMinimum": 0 + }, + { + "type": "null" + } + ] + }, + "diis_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 2 + }, + { + "type": "null" + } + ] + }, + "guess": { + "$ref": "#/$defs/guess" + } + } + }, + "mp2": { + "type": "object", + "additionalProperties": false, + "required": [ + "frozen_orbitals" + ], + "properties": { + "frozen_orbitals": { + "type": "integer", + "minimum": 0 + } + } + } + } +} diff --git a/tools/reference/generate_portable_fixtures.py b/tools/reference/generate_portable_fixtures.py new file mode 100644 index 00000000..28817c57 --- /dev/null +++ b/tools/reference/generate_portable_fixtures.py @@ -0,0 +1,124 @@ +"""Regenerate small Python ZIP64 fixtures using the documented V1 wire format. + +Uses only the standard library and the checked-in NumPy payload fixtures. +""" + +import hashlib +import io +import json +import struct +import zipfile +from pathlib import Path + +FIXTURES = ( + Path(__file__).resolve().parents[2] / "crates/rustiq-core/tests/data/persistence" +) + + +def scientific_identity(snapshot: dict) -> str: + canonical = bytearray() + + def length(value: int) -> None: + canonical.extend(struct.pack(">Q", value)) + + def text(value: str) -> None: + encoded = value.encode("utf-8") + length(len(encoded)) + canonical.extend(encoded) + + def number(value: float) -> None: + canonical.extend(struct.pack(">d", 0.0 if value == 0 else value)) + + text("scientific-identity-v1") + text("ao-eri-computation-version") + canonical.extend(struct.pack(">I", snapshot["ao_eri_computation_version"])) + text("rustiq-compact-eri-v1") + length(len(snapshot["atoms"])) + for atom in snapshot["atoms"]: + canonical.extend(struct.pack(">I", atom["atomic_number"])) + for coordinate in atom["position"]: + number(coordinate) + length(len(snapshot["basis"])) + for ao in snapshot["basis"]: + for coordinate in ao["center"]: + number(coordinate) + length(len(ao["components"])) + for component in ao["components"]: + canonical.extend(bytes(component["angular_momentum"])) + length(len(component["primitives"])) + for primitive in component["primitives"]: + number(primitive["exponent"]) + number(primitive["coefficient"]) + threshold = snapshot["hf"]["eri_schwarz_threshold"] + canonical.append(int(threshold is not None)) + if threshold is not None: + number(threshold) + return digest(canonical) + + +def digest(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def generate() -> None: + snapshot = (FIXTURES / "calculation-h2-v1.json").read_bytes() + for suffix, npy_fixture in [ + ("little", "ao-eri-python-v1.npy.hex"), + ("big", "ao-eri-python-big-endian-v1.npy.hex"), + ]: + payload = bytes.fromhex((FIXTURES / npy_fixture).read_text()) + manifest = { + "format": "rustiq-persistence", + "format_version": 1, + "kind": "portable", + "producer": {"name": "Python interoperability fixture", "version": "1"}, + "scientific_identity": { + "version": 1, + "digest": scientific_identity(json.loads(snapshot)), + }, + "calculation": { + "path": "calculation.json", + "version": 1, + "size": len(snapshot), + "digest": digest(snapshot), + }, + "artifacts": { + "ao_eri": { + "path": "arrays/integrals/ao-eri.npy", + "size": len(payload), + "digest": digest(payload), + "representation": "rustiq-compact-eri-v1", + "attributes": {"basis_functions": 2, "computation_version": 1}, + } + }, + } + output = io.BytesIO() + # Force full ZIP64 EOCD/locator/central extras on small fixtures. + previous_limit = zipfile.ZIP64_LIMIT + try: + zipfile.ZIP64_LIMIT = 0 + with zipfile.ZipFile(output, "w", allowZip64=True) as archive: + for name, content in [ + ("manifest.json", json.dumps(manifest, indent=2).encode() + b"\n"), + ("calculation.json", snapshot), + ("arrays/integrals/ao-eri.npy", payload), + ]: + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = 0o100644 << 16 + info.compress_type = ( + zipfile.ZIP_STORED + if name.endswith(".npy") + else zipfile.ZIP_DEFLATED + ) + with archive.open(info, "w", force_zip64=True) as member: + member.write(content) + finally: + zipfile.ZIP64_LIMIT = previous_limit + (FIXTURES / f"portable-python-{suffix}-v1.rustiq.hex").write_text( + output.getvalue().hex() + "\n", encoding="ascii" + ) + + +if __name__ == "__main__": + generate() diff --git a/tools/reference/test_persistence_interop.py b/tools/reference/test_persistence_interop.py index 2753e5e8..82fd495e 100644 --- a/tools/reference/test_persistence_interop.py +++ b/tools/reference/test_persistence_interop.py @@ -1,7 +1,7 @@ import io import os -from pathlib import Path import subprocess +from pathlib import Path import numpy as np @@ -56,3 +56,79 @@ def test_python_numpy_big_endian_ao_eri_fixture() -> None: assert values.dtype == np.dtype(">f8") assert values.shape == (6,) np.testing.assert_array_equal(values, [0.5, 1.5, 2.5, 3.5, 4.5, 5.5]) + + +def test_numpy_reads_rust_portable_archive(tmp_path: Path) -> None: + import json + import zipfile + + from generate_portable_fixtures import digest, scientific_identity + + output = tmp_path / "h2.rustiq" + environment = os.environ.copy() + environment["RUSTIQ_ARCHIVE_TEST_OUTPUT"] = str(output) + subprocess.run( + [ + "cargo", + "test", + "-p", + "rustiq-core", + "--no-default-features", + "persistence::data::portable::tests::writes_archive_for_python_interoperability", + "--", + "--ignored", + "--exact", + ], + check=True, + cwd=ROOT, + env=environment, + ) + with zipfile.ZipFile(output) as archive: + assert archive.testzip() is None + assert set(archive.namelist()) == { + "manifest.json", + "calculation.json", + "arrays/integrals/ao-eri.npy", + } + manifest = json.loads(archive.read("manifest.json")) + snapshot_bytes = archive.read("calculation.json") + snapshot = json.loads(snapshot_bytes) + assert manifest["scientific_identity"]["digest"] == scientific_identity( + snapshot + ) + assert manifest["calculation"]["digest"] == digest(snapshot_bytes) + assert snapshot["hf"]["method"] == "rhf" + assert snapshot["units"] == "bohr" + payload = archive.read("arrays/integrals/ao-eri.npy") + assert manifest["artifacts"]["ao_eri"]["digest"] == digest(payload) + _assert_ao_eri_values(np.load(io.BytesIO(payload), allow_pickle=False)) + assert ( + archive.getinfo("arrays/integrals/ao-eri.npy").compress_type + == zipfile.ZIP_STORED + ) + assert all( + info.date_time == (1980, 1, 1, 0, 0, 0) for info in archive.infolist() + ) + + +def test_python_portable_golden_fixtures() -> None: + import json + import zipfile + + from generate_portable_fixtures import digest, scientific_identity + + for byte_order in ("little", "big"): + content = bytes.fromhex( + (FIXTURE_DIR / f"portable-python-{byte_order}-v1.rustiq.hex").read_text() + ) + assert b"PK\x06\x06" in content + with zipfile.ZipFile(io.BytesIO(content)) as archive: + manifest = json.loads(archive.read("manifest.json")) + snapshot = json.loads(archive.read("calculation.json")) + assert ( + scientific_identity(snapshot) + == manifest["scientific_identity"]["digest"] + ) + payload = archive.read("arrays/integrals/ao-eri.npy") + assert digest(payload) == manifest["artifacts"]["ao_eri"]["digest"] + _assert_ao_eri_values(np.load(io.BytesIO(payload), allow_pickle=False)) From f2520827903a08d4effcb69ff1c117ceff992f81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Fri, 2 Oct 2026 20:46:34 +0200 Subject: [PATCH 2/8] Persist normalized requests and opaque source provenance --- .../src/persistence/calculation.rs | 148 ++++++-- .../src/persistence/data/portable.rs | 162 +++++++-- .../src/persistence/data/portable/tests.rs | 231 +++++++++++- .../src/persistence/data/rustiq_data.rs | 44 ++- crates/rustiq-core/src/persistence/error.rs | 2 + .../rustiq-core/src/persistence/manifest.rs | 20 +- crates/rustiq-core/src/persistence/mod.rs | 3 + .../rustiq-core/src/persistence/provenance.rs | 21 ++ crates/rustiq-core/src/persistence/request.rs | 217 +++++++++++ crates/rustiq-core/src/persistence/storage.rs | 8 +- .../src/persistence/storage/zip.rs | 5 +- .../portable-python-big-v1.rustiq.hex | 2 +- .../portable-python-little-v1.rustiq.hex | 2 +- .../tests/data/persistence/request-h2-v1.json | 40 +++ .../persistence/source-original-v1.toml.hex | 1 + crates/rustiq-core/tests/portable_artifact.rs | 14 + docs/persistence-format-v1.md | 77 +++- schemas/request-snapshot-v1.schema.json | 336 ++++++++++++++++++ tools/reference/generate_portable_fixtures.py | 19 + tools/reference/test_persistence_interop.py | 22 ++ 20 files changed, 1297 insertions(+), 77 deletions(-) create mode 100644 crates/rustiq-core/src/persistence/provenance.rs create mode 100644 crates/rustiq-core/src/persistence/request.rs create mode 100644 crates/rustiq-core/tests/data/persistence/request-h2-v1.json create mode 100644 crates/rustiq-core/tests/data/persistence/source-original-v1.toml.hex create mode 100644 schemas/request-snapshot-v1.schema.json diff --git a/crates/rustiq-core/src/persistence/calculation.rs b/crates/rustiq-core/src/persistence/calculation.rs index c3bc4051..649f153d 100644 --- a/crates/rustiq-core/src/persistence/calculation.rs +++ b/crates/rustiq-core/src/persistence/calculation.rs @@ -155,7 +155,7 @@ struct Mp2 { } #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -enum Guess { +pub(super) enum Guess { CoreHamiltonian { perturbation: Option }, OneElectron { perturbation: Option }, Random { random: Random }, @@ -163,11 +163,11 @@ enum Guess { } #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -struct Random { +pub(super) struct Random { seed: Option, distribution: Distribution, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] enum Distribution { Uniform { min: f64, max: f64 }, @@ -229,20 +229,7 @@ impl Snapshot { linear_dependency_threshold: hf.linear_dependency_threshold.value.into_inner(), eri_schwarz_threshold: hf.eri_schwarz_threshold.map(|value| value.into_inner()), diis_size: hf.diis.then(|| hf.diis_size.into_inner()), - guess: match hf.guess.value { - DensityGuessConfig::CoreHamiltonian { perturbation } => { - Guess::CoreHamiltonian { - perturbation: perturbation.map(|p| p.random.into()), - } - } - DensityGuessConfig::OneElectron { perturbation } => Guess::OneElectron { - perturbation: perturbation.map(|p| p.random.into()), - }, - DensityGuessConfig::Random { config } => Guess::Random { - random: config.random.into(), - }, - DensityGuessConfig::Zero => Guess::Zero, - }, + guess: hf.guess.value.into(), }, mp2: mp2.map(|mp2| Mp2 { frozen_orbitals: mp2.frozen_orbitals.value, @@ -317,23 +304,48 @@ impl Snapshot { { return Err(invalid("invalid HF settings")); } - let random = match &self.hf.guess { - Guess::CoreHamiltonian { perturbation } | Guess::OneElectron { perturbation } => { - perturbation.as_ref() - } - Guess::Random { random } => Some(random), - Guess::Zero => None, - }; - if let Some(random) = random { - let valid = match random.distribution { - Distribution::Uniform { min, max } => { - min.is_finite() && max.is_finite() && min < max && (max - min).is_finite() - } - Distribution::Normal { mean, std_dev } => mean.is_finite() && positive(std_dev), - }; - if !valid { - return Err(invalid("invalid random distribution")); - } + self.hf.guess.validate()?; + Ok(()) + } + + pub(crate) fn validate_request( + &self, + request: &crate::calculation::CalculationRequest, + ) -> Result<(), PortableError> { + let invalid = + || PortableError::InvalidRequest("request disagrees with resolved calculation".into()); + let mut molecule = request + .molecule() + .build(request.geometry().clone()) + .map_err(|_| invalid())?; + molecule.convert_to(crate::molecules::units::Units::Bohr); + let method = request + .hf() + .resolve_method(&molecule) + .map_err(|_| invalid())?; + let hf = request.hf(); + if molecule.charge() != self.charge + || molecule.multiplicity().get() != self.multiplicity + || molecule.atoms.len() != self.atoms.len() + || molecule.atoms.iter().zip(&self.atoms).any(|(a, b)| { + a.element.atomic_number != b.atomic_number + || <[f64; 3]>::from(a.position) != b.position + }) + || !matches!( + (method, self.hf.method), + (ResolvedHfMethod::Rhf, Method::Rhf) | (ResolvedHfMethod::Uhf, Method::Uhf) + ) + || hf.max_iterations.get() != self.hf.max_iterations + || hf.convergence_threshold.into_inner() != self.hf.convergence_threshold + || hf.linear_dependency_threshold.value.into_inner() + != self.hf.linear_dependency_threshold + || hf.eri_schwarz_threshold.map(|v| v.into_inner()) != self.hf.eri_schwarz_threshold + || hf.diis.then(|| hf.diis_size.into_inner()) != self.hf.diis_size + || request.mp2().map(|v| v.frozen_orbitals.value) + != self.mp2.as_ref().map(|v| v.frozen_orbitals) + || !self.hf.guess.is_resolution_of(&hf.guess.value.into()) + { + return Err(invalid()); } Ok(()) } @@ -416,7 +428,56 @@ impl Random { } } impl Guess { - fn to_config(&self) -> DensityGuessConfig { + fn is_resolution_of(&self, requested: &Self) -> bool { + let random_matches = |resolved: &Random, requested: &Random| { + resolved.distribution == requested.distribution + && requested + .seed + .is_none_or(|seed| resolved.seed == Some(seed)) + }; + match (self, requested) { + ( + Self::CoreHamiltonian { perturbation: a }, + Self::CoreHamiltonian { perturbation: b }, + ) + | (Self::OneElectron { perturbation: a }, Self::OneElectron { perturbation: b }) => { + match (a, b) { + (None, None) => true, + (Some(a), Some(b)) => random_matches(a, b), + _ => false, + } + } + (Self::Random { random: a }, Self::Random { random: b }) => random_matches(a, b), + (Self::Zero, Self::Zero) => true, + _ => false, + } + } + + pub(super) fn validate(&self) -> Result<(), PortableError> { + let random = match self { + Guess::CoreHamiltonian { perturbation } | Guess::OneElectron { perturbation } => { + perturbation.as_ref() + } + Guess::Random { random } => Some(random), + Guess::Zero => None, + }; + if let Some(random) = random { + let valid = match random.distribution { + Distribution::Uniform { min, max } => { + min.is_finite() && max.is_finite() && min < max && (max - min).is_finite() + } + Distribution::Normal { mean, std_dev } => mean.is_finite() && positive(std_dev), + }; + if !valid { + return Err(PortableError::InvalidCalculation( + "invalid random distribution".into(), + )); + } + } + Ok(()) + } + + pub(super) fn to_config(&self) -> DensityGuessConfig { use crate::config::{GuessPerturbationConfig, RandomGuessConfig}; match self { Self::CoreHamiltonian { perturbation } => DensityGuessConfig::CoreHamiltonian { @@ -439,6 +500,23 @@ impl Guess { } } +impl From for Guess { + fn from(config: DensityGuessConfig) -> Self { + match config { + DensityGuessConfig::CoreHamiltonian { perturbation } => Guess::CoreHamiltonian { + perturbation: perturbation.map(|p| p.random.into()), + }, + DensityGuessConfig::OneElectron { perturbation } => Guess::OneElectron { + perturbation: perturbation.map(|p| p.random.into()), + }, + DensityGuessConfig::Random { config } => Guess::Random { + random: config.random.into(), + }, + DensityGuessConfig::Zero => Guess::Zero, + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/rustiq-core/src/persistence/data/portable.rs b/crates/rustiq-core/src/persistence/data/portable.rs index 16b7c13d..f8069c49 100644 --- a/crates/rustiq-core/src/persistence/data/portable.rs +++ b/crates/rustiq-core/src/persistence/data/portable.rs @@ -1,13 +1,16 @@ use super::super::{ calculation::{Snapshot, CALCULATION_PATH, MAX_CALCULATION_BYTES}, - ArtifactAttributes, ArtifactError, CalculationContext, ManifestKind, PortableError, Storage, - AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, + provenance::{MAX_SOURCES, MAX_SOURCE_BYTES, MAX_SOURCE_NAME_BYTES}, + request::{RequestSnapshot, MAX_REQUEST_BYTES, REQUEST_PATH}, + ArtifactAttributes, ArtifactError, CalculationContext, ManifestKind, PortableError, + SourceProvenance, Storage, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, }; use super::{RustiQData, AO_ERI_ARTIFACT}; use crate::calculation::PreparedCalculation; use relative_path::RelativePath; #[cfg(any(unix, test))] use std::fs::File; +use std::io::Read; use std::path::Path; impl RustiQData { @@ -20,6 +23,7 @@ impl RustiQData { snapshot.basis_functions(), ManifestKind::Portable, ); + data.request = Some(calculation.request().clone()); data.context = Some(CalculationContext(snapshot)); Ok(data) } @@ -31,31 +35,75 @@ impl RustiQData { if data.manifest.kind != ManifestKind::Portable { return Err(PortableError::UnsupportedVersion); } + let source = data.source.as_mut().expect("opened storage"); let reference = data.manifest.calculation.as_ref().ok_or_else(|| { PortableError::InvalidArchive("missing calculation reference".into()) })?; - if reference.version != 1 { - return Err(PortableError::UnsupportedVersion); - } - if reference.path != CALCULATION_PATH || reference.size > MAX_CALCULATION_BYTES { + verify_reference(source, reference, CALCULATION_PATH, MAX_CALCULATION_BYTES)?; + let snapshot: Snapshot = source + .read_json(RelativePath::new(CALCULATION_PATH), MAX_CALCULATION_BYTES) + .map_err(|error| PortableError::InvalidCalculation(error.to_string()))?; + snapshot.validate()?; + let reference = data + .manifest + .request + .as_ref() + .ok_or_else(|| PortableError::InvalidArchive("missing request reference".into()))?; + verify_reference(source, reference, REQUEST_PATH, MAX_REQUEST_BYTES)?; + let request_snapshot: RequestSnapshot = source + .read_json(RelativePath::new(REQUEST_PATH), MAX_REQUEST_BYTES) + .map_err(|error| PortableError::InvalidRequest(error.to_string()))?; + let request = request_snapshot.to_request()?; + snapshot.validate_request(&request)?; + data.request = Some(request); + if data.manifest.sources.len() > MAX_SOURCES { return Err(PortableError::InvalidArchive( - "invalid calculation reference".into(), + "too many source payloads".into(), )); } - let source = data.source.as_mut().expect("opened storage"); - let path = RelativePath::new(CALCULATION_PATH); - if source.artifact_size(path)? != reference.size { - return Err(ArtifactError::IntegrityMismatch(CALCULATION_PATH.into()).into()); - } - let metadata = source.artifact_metadata(path)?; - if metadata.digest != reference.digest { - return Err(ArtifactError::IntegrityMismatch(CALCULATION_PATH.into()).into()); + let mut total = 0_u64; + let mut paths = std::collections::HashSet::new(); + for reference in &data.manifest.sources { + let path = RelativePath::new(&reference.path); + super::super::validate_storage_path(path)?; + total = total + .checked_add(reference.size) + .filter(|total| *total <= MAX_SOURCE_BYTES) + .ok_or_else(|| { + PortableError::InvalidArchive("source payloads exceed size limit".into()) + })?; + if reference.version != 1 { + return Err(PortableError::UnsupportedVersion); + } + if !reference.path.starts_with("sources/") + || reference.original_name.len() > MAX_SOURCE_NAME_BYTES + || !paths.insert(reference.path.to_lowercase()) + { + return Err(PortableError::InvalidArchive( + "invalid source reference".into(), + )); + } + let expected = super::super::manifest::SnapshotManifest { + path: reference.path.clone(), + version: reference.version, + size: reference.size, + digest: reference.digest, + }; + verify_reference(source, &expected, &reference.path, MAX_SOURCE_BYTES)?; + let bytes = source.with_artifact::<_, PortableError, _>(path, |reader| { + let mut bytes = Vec::new(); + reader.take(reference.size + 1).read_to_end(&mut bytes)?; + if bytes.len() as u64 != reference.size { + return Err(ArtifactError::IntegrityMismatch(reference.path.clone()).into()); + } + Ok(bytes) + })?; + data.sources.push(SourceProvenance { + original_name: reference.original_name.clone(), + bytes, + }); } - let snapshot: Snapshot = source - .read_json(path, MAX_CALCULATION_BYTES) - .map_err(|error| PortableError::InvalidCalculation(error.to_string()))?; - snapshot.validate()?; let identity = snapshot.identity(); if identity.version != data.manifest.scientific_identity.version || identity.digest != data.manifest.scientific_identity.digest @@ -73,6 +121,8 @@ impl RustiQData { .is_some_and(|part| { part.eq_ignore_ascii_case(CALCULATION_PATH) || part.eq_ignore_ascii_case("manifest.json") + || part.eq_ignore_ascii_case(REQUEST_PATH) + || part.eq_ignore_ascii_case("sources") }) { return Err(ArtifactError::InvalidPath(artifact.path.to_string()).into()); @@ -98,6 +148,50 @@ impl RustiQData { Ok(data) } + /// Returns the normalized pre-resolution request, absent for directory-cache data. + pub fn request(&self) -> Option<&crate::calculation::CalculationRequest> { + self.request.as_ref() + } + + /// Captured sources are opaque provenance and never affect scientific compatibility. + pub fn sources(&self) -> impl ExactSizeIterator { + self.sources.iter() + } + + /// Captures exact source bytes; the original name is informational, never an archive path. + pub fn add_source( + &mut self, + original_name: impl Into, + bytes: impl Into>, + ) -> Result<(), PortableError> { + if self.context.is_none() { + return Err(PortableError::InvalidCalculation( + "source provenance requires portable context".into(), + )); + } + let original_name = original_name.into(); + let bytes = bytes.into(); + let total = self + .sources + .iter() + .map(|s| s.bytes().len() as u64) + .sum::() + + bytes.len() as u64; + if self.sources.len() >= MAX_SOURCES + || total > MAX_SOURCE_BYTES + || original_name.len() > MAX_SOURCE_NAME_BYTES + { + return Err(PortableError::InvalidArchive( + "source provenance exceeds supported limits".into(), + )); + } + self.sources.push(SourceProvenance { + original_name, + bytes, + }); + Ok(()) + } + /// Returns the portable context, absent only for internal directory-cache data. pub fn calculation(&self) -> Option<&CalculationContext> { self.context.as_ref() @@ -158,7 +252,10 @@ impl RustiQData { /// Atomically publishes a portable archive. Never overwrites an existing path. /// The source archive is not modified; unloaded artifacts are copied and verified. pub fn write(&mut self, path: impl AsRef) -> Result<(), PortableError> { - if self.context.is_none() || self.manifest.kind != ManifestKind::Portable { + if self.context.is_none() + || self.request.is_none() + || self.manifest.kind != ManifestKind::Portable + { return Err(PortableError::InvalidCalculation( "portable output requires resolved calculation context".into(), )); @@ -213,5 +310,30 @@ impl RustiQData { } } +fn verify_reference( + source: &mut Storage, + reference: &super::super::manifest::SnapshotManifest, + expected_path: &str, + limit: u64, +) -> Result<(), PortableError> { + if reference.version != 1 { + return Err(PortableError::UnsupportedVersion); + } + if reference.path != expected_path || reference.size > limit { + return Err(PortableError::InvalidArchive( + "invalid snapshot/source reference".into(), + )); + } + let path = RelativePath::new(expected_path); + if source.artifact_size(path)? != reference.size { + return Err(ArtifactError::IntegrityMismatch(expected_path.into()).into()); + } + let metadata = source.artifact_metadata(path)?; + if metadata.size != reference.size || metadata.digest != reference.digest { + return Err(ArtifactError::IntegrityMismatch(expected_path.into()).into()); + } + Ok(()) +} + #[cfg(test)] mod tests; diff --git a/crates/rustiq-core/src/persistence/data/portable/tests.rs b/crates/rustiq-core/src/persistence/data/portable/tests.rs index e8e81fcb..d7cec6bd 100644 --- a/crates/rustiq-core/src/persistence/data/portable/tests.rs +++ b/crates/rustiq-core/src/persistence/data/portable/tests.rs @@ -26,6 +26,16 @@ fn data() -> RustiQData { .unwrap(); data } +fn source_fixture() -> Vec { + include_str!("../../../../tests/data/persistence/source-original-v1.toml.hex") + .trim() + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect() +} fn members(path: &Path) -> Vec<(String, Vec)> { let mut archive = ZipArchive::new(File::open(path).unwrap()).unwrap(); (0..archive.len()) @@ -109,7 +119,20 @@ fn rust_round_trip_is_lazy_self_describing_and_reproducible() { let pointer = restored.read_eri().unwrap() as *const CompactEri; assert_eq!(restored.read_eri().unwrap() as *const CompactEri, pointer); let zip = ZipArchive::new(File::open(&first).unwrap()).unwrap(); - assert_eq!(zip.len(), 3); + assert_eq!(zip.len(), 4); + let request = members(&first) + .into_iter() + .find(|(n, _)| n == REQUEST_PATH) + .unwrap() + .1; + assert_eq!( + serde_json::from_slice::(&request).unwrap(), + serde_json::from_str::(include_str!( + "../../../../tests/data/persistence/request-h2-v1.json" + )) + .unwrap() + ); + assert_eq!(restored.sources().len(), 0); let snapshot = members(&first) .into_iter() .find(|(n, _)| n == CALCULATION_PATH) @@ -124,6 +147,205 @@ fn rust_round_trip_is_lazy_self_describing_and_reproducible() { ); } +#[test] +fn normalized_request_round_trip_keeps_auto_angstrom_defaults_and_requested_options() { + use crate::config::{DensityGuessConfig, MemoryLimit, RandomGuessConfig}; + use crate::molecules::units::Units; + let geometry = + Geometry::from_source("input.xyz", "2\nignored comment\nH 0 0 0\nH 0.74 0 0\n").unwrap(); + let basis = load_minimal_basis_file(); + let defaulted = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + let explicit = CalculationBuilder::new(&geometry, &basis) + .with_hf(HfConfig::default()) + .with_molecule_config(MoleculeConfig::default()) + .prepare() + .unwrap(); + assert_eq!( + serde_json::to_value(RequestSnapshot::from_request(defaulted.request())).unwrap(), + serde_json::to_value(RequestSnapshot::from_request(explicit.request())).unwrap() + ); + let prepared = CalculationBuilder::new(&geometry, &basis) + .with_basis_label("portable-basis-label") + .with_molecule_config(MoleculeConfig { + units: Units::Angstrom, + ..Default::default() + }) + .with_hf(HfConfig { + guess: DensityGuessConfig::Random { + config: RandomGuessConfig::default(), + } + .into(), + ..Default::default() + }) + .with_mp2(Mp2Config { + frozen_orbitals: 1.into(), + memory_limit: MemoryLimit::Fixed(bytesize::ByteSize::mib(42)).into(), + }) + .prepare() + .unwrap(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("round-trip.rustiq"); + RustiQData::from_calculation(&prepared) + .unwrap() + .write(&path) + .unwrap(); + let restored = RustiQData::open(&path).unwrap(); + let request = restored.request().unwrap(); + assert_eq!(request.hf().method.value, HfMethod::Auto); + assert_eq!(request.molecule().units, Units::Angstrom); + assert_eq!(request.geometry().atoms[1].position[0], 0.74); + assert_eq!(request.basis_name(), "portable-basis-label"); + assert_eq!( + request.mp2().unwrap().memory_limit.value, + prepared.request().mp2().unwrap().memory_limit.value + ); + assert!(request.geometry().comment.is_empty()); + let DensityGuessConfig::Random { config } = request.hf().guess.value else { + panic!() + }; + assert_eq!(config.random.seed, None); + let context = restored.calculation().unwrap(); + assert_eq!(context.hf_method(), crate::config::ResolvedHfMethod::Rhf); + assert!(context.atoms().nth(1).unwrap().1[0] > 1.0); + let DensityGuessConfig::Random { config } = context.density_guess() else { + panic!() + }; + assert!(config.random.seed.is_some()); + assert_eq!( + serde_json::to_value(RequestSnapshot::from_request(request)).unwrap(), + serde_json::to_value(RequestSnapshot::from_request(prepared.request())).unwrap() + ); +} + +#[test] +fn opaque_sources_preserve_exact_bytes_without_affecting_scientific_state() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rustiq"); + let output = dir.path().join("copied.rustiq"); + let mut original = data(); + let sources: [(&str, &[u8]); 2] = [ + ( + "../../private/calculation.toml", + b"# preserve comments\r\n[hf]\r\nmethod = 'Auto'\r\n", + ), + ( + "C:\\private\\molecule.xyz", + b"2\r\noriginal comment\r\nH 0.000 0 0\r\nH 1.4000 0 0\r\n\xff", + ), + ]; + for (name, bytes) in sources { + original.add_source(name, bytes).unwrap(); + } + original.write(&path).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o444)).unwrap(); + } + let mut restored = RustiQData::open(&path).unwrap(); + assert!(restored.ao_eri.is_none()); + assert!(restored.eri_is_compatible(&calculation())); + for (source, (name, bytes)) in restored.sources().zip(sources) { + assert_eq!(source.original_name(), name); + assert_eq!(source.bytes(), bytes); + } + let archive_members = members(&path); + assert!(archive_members.iter().any(|(name, _)| name == "sources/0")); + assert!(!archive_members + .iter() + .any(|(name, _)| name.contains("private"))); + restored.write(&output).unwrap(); + assert_eq!(members(&output), archive_members); + assert_eq!(members(&path), archive_members); +} + +#[test] +fn request_and_source_integrity_versions_and_limits_are_checked_on_open() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("good.rustiq"); + let mut original = data(); + original + .add_source("input.toml", b"original bytes".as_slice()) + .unwrap(); + original.write(&path).unwrap(); + let original = members(&path); + let broken = dir.path().join("broken.rustiq"); + for (target, field, value) in [ + ("manifest.json", "/request", serde_json::Value::Null), + ( + "manifest.json", + "/request/path", + serde_json::json!(CALCULATION_PATH), + ), + ("manifest.json", "/request/version", serde_json::json!(99)), + ( + "manifest.json", + "/request/size", + serde_json::json!(MAX_REQUEST_BYTES + 1), + ), + ( + "manifest.json", + "/sources/0/path", + serde_json::json!("../input.toml"), + ), + ("manifest.json", "/sources/0/version", serde_json::json!(99)), + ( + "manifest.json", + "/sources/0/size", + serde_json::json!(MAX_SOURCE_BYTES + 1), + ), + (REQUEST_PATH, "/version", serde_json::json!(99)), + (REQUEST_PATH, "/units", serde_json::json!("invalid")), + (REQUEST_PATH, "/hf/method", serde_json::json!("invalid")), + (REQUEST_PATH, "/hf/max_iterations", serde_json::json!(0)), + (REQUEST_PATH, "/hf/diis_size", serde_json::json!(1)), + ( + REQUEST_PATH, + "/hf/convergence_threshold", + serde_json::json!(-1), + ), + ( + REQUEST_PATH, + "/atoms/0/atomic_number", + serde_json::json!(119), + ), + (REQUEST_PATH, "/atoms/0/position/0", serde_json::json!(12)), + ] { + let mut contents = original.clone(); + edit_json(&mut contents, target, |json| { + *json.pointer_mut(field).unwrap() = value + }); + if target == REQUEST_PATH { + let bytes = &contents + .iter() + .find(|(name, _)| name == REQUEST_PATH) + .unwrap() + .1; + let (size, digest) = (bytes.len(), sha256(bytes).to_string()); + edit_json(&mut contents, "manifest.json", |m| { + m["request"]["size"] = size.into(); + m["request"]["digest"] = digest.into(); + }); + } + write_members(&broken, &contents); + assert!(RustiQData::open(&broken).is_err(), "{target} {field}"); + } + for name in [REQUEST_PATH, "sources/0"] { + let mut contents = original.clone(); + contents.iter_mut().find(|(n, _)| n == name).unwrap().1[0] ^= 1; + write_members(&broken, &contents); + assert!(matches!( + RustiQData::open(&broken), + Err(PortableError::Artifact(ArtifactError::IntegrityMismatch(_))) + )); + contents.retain(|(n, _)| n != name); + write_members(&broken, &contents); + assert!(RustiQData::open(&broken).is_err()); + } +} + #[test] fn empty_artifacts_uhf_and_mp2_context_round_trip() { let geometry = Geometry::from_source("H.xyz", "1\nH\nH 0 0 0\n").unwrap(); @@ -433,7 +655,10 @@ fn malformed_npy_is_rejected_even_with_correct_digest() { #[ignore = "writes a portable artifact for Python interoperability"] fn writes_archive_for_python_interoperability() { let path = std::env::var_os("RUSTIQ_ARCHIVE_TEST_OUTPUT").expect("output path"); - data().write(path).unwrap(); + let mut data = data(); + data.add_source("../../calculation.toml", source_fixture()) + .unwrap(); + data.write(path).unwrap(); } #[test] @@ -606,7 +831,7 @@ fn settings_are_resolved_and_machine_policy_is_excluded() { assert_eq!(snapshot.identity(), restored.identity()); assert_eq!( format!("{:?}", CalculationContext(restored).density_guess()), - format!("{guess:?}") + format!("{:?}", prepared.hf_config().guess.value) ); assert!(data.calculation().unwrap().atoms().nth(1).unwrap().1[0] > 1.0); } diff --git a/crates/rustiq-core/src/persistence/data/rustiq_data.rs b/crates/rustiq-core/src/persistence/data/rustiq_data.rs index 995afde0..400a7c69 100644 --- a/crates/rustiq-core/src/persistence/data/rustiq_data.rs +++ b/crates/rustiq-core/src/persistence/data/rustiq_data.rs @@ -23,6 +23,8 @@ pub struct RustiQData { pub(super) source: Option, pub(super) ao_eri: Option, pub(super) basis_functions: Option, + pub(super) request: Option, + pub(super) sources: Vec, pub(super) context: Option, } @@ -57,9 +59,13 @@ impl RustiQData { }, artifacts: BTreeMap::new(), calculation: None, + request: None, + sources: Vec::new(), }, source: None, context: None, + request: None, + sources: Vec::new(), ao_eri: None, basis_functions: Some(basis_functions), } @@ -96,6 +102,8 @@ impl RustiQData { manifest, source: Some(source), context: None, + request: None, + sources: Vec::new(), ao_eri: None, basis_functions, }) @@ -203,7 +211,7 @@ impl RustiQData { RelativePath::new(super::super::calculation::CALCULATION_PATH), &context.0, )?; - manifest.calculation = Some(super::super::manifest::CalculationManifest { + manifest.calculation = Some(super::super::manifest::SnapshotManifest { path: super::super::calculation::CALCULATION_PATH.into(), version: 1, size: metadata.size, @@ -211,6 +219,40 @@ impl RustiQData { }); } + if let Some(request) = &self.request { + let metadata = destination.write_json( + RelativePath::new(super::super::request::REQUEST_PATH), + &super::super::request::RequestSnapshot::from_request(request), + )?; + manifest.request = Some(super::super::manifest::SnapshotManifest { + path: super::super::request::REQUEST_PATH.into(), + version: 1, + size: metadata.size, + digest: metadata.digest, + }); + } + manifest.sources.clear(); + for (index, source) in self.sources.iter().enumerate() { + let path = format!("sources/{index}"); + let metadata = destination.write_artifact::( + RelativePath::new(&path), + |writer| { + std::io::Write::write_all(writer, source.bytes()) + .map_err(StorageError::from)?; + Ok(()) + }, + )?; + manifest + .sources + .push(super::super::manifest::SourceManifest { + original_name: source.original_name().into(), + path, + version: 1, + size: metadata.size, + digest: metadata.digest, + }); + } + for (name, artifact) in &self.manifest.artifacts { if name == AO_ERI_ARTIFACT && eri.is_some() { continue; diff --git a/crates/rustiq-core/src/persistence/error.rs b/crates/rustiq-core/src/persistence/error.rs index f14e3fda..da9de026 100644 --- a/crates/rustiq-core/src/persistence/error.rs +++ b/crates/rustiq-core/src/persistence/error.rs @@ -27,6 +27,8 @@ pub enum PortableError { UnsupportedVersion, #[error("invalid resolved calculation: {0}")] InvalidCalculation(String), + #[error("invalid normalized request: {0}")] + InvalidRequest(String), #[error(transparent)] Artifact(#[from] ArtifactError), } diff --git a/crates/rustiq-core/src/persistence/manifest.rs b/crates/rustiq-core/src/persistence/manifest.rs index 4715ced6..9af57a31 100644 --- a/crates/rustiq-core/src/persistence/manifest.rs +++ b/crates/rustiq-core/src/persistence/manifest.rs @@ -55,12 +55,26 @@ pub(crate) struct Manifest { pub scientific_identity: ScientificIdentityManifest, pub artifacts: BTreeMap, #[serde(default, skip_serializing_if = "Option::is_none")] - pub calculation: Option, + pub calculation: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub sources: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SourceManifest { + pub original_name: String, + pub path: String, + pub version: u32, + pub size: u64, + pub digest: Sha256Digest, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub(crate) struct CalculationManifest { +pub(crate) struct SnapshotManifest { pub path: String, pub version: u32, pub size: u64, @@ -176,6 +190,8 @@ mod tests { let manifest = Manifest { calculation: None, + request: None, + sources: Vec::new(), format: FORMAT_NAME.to_string(), format_version: FORMAT_VERSION, kind: ManifestKind::IntegralCache, diff --git a/crates/rustiq-core/src/persistence/mod.rs b/crates/rustiq-core/src/persistence/mod.rs index 2ce6db51..0c2980de 100644 --- a/crates/rustiq-core/src/persistence/mod.rs +++ b/crates/rustiq-core/src/persistence/mod.rs @@ -6,7 +6,10 @@ mod cache_names; mod calculation; +mod provenance; +mod request; pub use calculation::{CalculationContext, ResolvedAo, ResolvedComponent}; +pub use provenance::SourceProvenance; mod checksum; mod data; mod eri_cache; diff --git a/crates/rustiq-core/src/persistence/provenance.rs b/crates/rustiq-core/src/persistence/provenance.rs new file mode 100644 index 00000000..31464eff --- /dev/null +++ b/crates/rustiq-core/src/persistence/provenance.rs @@ -0,0 +1,21 @@ +//! Optional opaque provenance, kept outside the scientific request and identity. +pub(crate) const MAX_SOURCE_BYTES: u64 = 64 * 1024 * 1024; +pub(crate) const MAX_SOURCES: usize = 256; +pub(crate) const MAX_SOURCE_NAME_BYTES: usize = 4096; + +/// Exact original source bytes with an informational original name. +/// The name is never an archive member name or an extraction instruction. +#[derive(Debug, Clone)] +pub struct SourceProvenance { + pub(crate) original_name: String, + pub(crate) bytes: Vec, +} + +impl SourceProvenance { + pub fn original_name(&self) -> &str { + &self.original_name + } + pub fn bytes(&self) -> &[u8] { + &self.bytes + } +} diff --git a/crates/rustiq-core/src/persistence/request.rs b/crates/rustiq-core/src/persistence/request.rs new file mode 100644 index 00000000..cfcd4b66 --- /dev/null +++ b/crates/rustiq-core/src/persistence/request.rs @@ -0,0 +1,217 @@ +//! Explicit normalized-request V1 wire records, independent of frontend syntax. +use std::num::{NonZeroU8, NonZeroUsize}; + +use serde::{Deserialize, Serialize}; + +use super::{calculation::Guess, PortableError}; +use crate::{ + calculation::CalculationRequest, + config::{ + validated::{DiisSize, NonNegativeFiniteF64, PositiveFiniteF64}, + HfConfig, HfMethod, MemoryLimit, MoleculeConfig, Mp2Config, + }, + molecules::{atom::Atom, geometry::Geometry, units::Units}, +}; + +pub(crate) const REQUEST_PATH: &str = "request.json"; +pub(crate) const MAX_REQUEST_BYTES: u64 = 64 * 1024 * 1024; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RequestSnapshot { + format: String, + version: u32, + units: CoordinateUnits, + atoms: Vec, + charge: i32, + multiplicity: u8, + basis: String, + hf: RequestedHf, + mp2: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +enum CoordinateUnits { + Bohr, + Angstrom, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RequestAtom { + atomic_number: u32, + position: [f64; 3], +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +enum RequestedMethod { + Auto, + Rhf, + Uhf, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RequestedHf { + method: RequestedMethod, + max_iterations: usize, + convergence_threshold: f64, + linear_dependency_threshold: f64, + eri_schwarz_threshold: Option, + diis: bool, + diis_size: usize, + guess: Guess, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RequestedMp2 { + frozen_orbitals: usize, + memory_limit: RequestedMemory, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum RequestedMemory { + Auto, + Fixed { bytes: u64 }, +} + +impl RequestSnapshot { + pub(crate) fn from_request(request: &CalculationRequest) -> Self { + let hf = request.hf(); + Self { + format: "rustiq-request".into(), + version: 1, + units: match request.molecule().units { + Units::Bohr => CoordinateUnits::Bohr, + Units::Angstrom => CoordinateUnits::Angstrom, + }, + atoms: request + .geometry() + .atoms + .iter() + .map(|atom| RequestAtom { + atomic_number: atom.element.atomic_number, + position: atom.position.into(), + }) + .collect(), + charge: request.molecule().charge.value, + multiplicity: request.molecule().multiplicity.value.get(), + basis: request.basis_name().into(), + hf: RequestedHf { + method: match hf.method.value { + HfMethod::Auto => RequestedMethod::Auto, + HfMethod::Rhf => RequestedMethod::Rhf, + HfMethod::Uhf => RequestedMethod::Uhf, + }, + max_iterations: hf.max_iterations.get(), + convergence_threshold: hf.convergence_threshold.into_inner(), + linear_dependency_threshold: hf.linear_dependency_threshold.value.into_inner(), + eri_schwarz_threshold: hf.eri_schwarz_threshold.map(|v| v.into_inner()), + diis: hf.diis, + diis_size: hf.diis_size.into_inner(), + guess: hf.guess.value.into(), + }, + mp2: request.mp2().map(|mp2| RequestedMp2 { + frozen_orbitals: mp2.frozen_orbitals.value, + memory_limit: match mp2.memory_limit.value { + MemoryLimit::Auto => RequestedMemory::Auto, + MemoryLimit::Fixed(bytes) => RequestedMemory::Fixed { + bytes: bytes.as_u64(), + }, + }, + }), + } + } + + pub(crate) fn to_request(&self) -> Result { + if self.format != "rustiq-request" || self.version != 1 { + return Err(PortableError::UnsupportedVersion); + } + let invalid = || PortableError::InvalidRequest("invalid normalized request".into()); + if self.atoms.is_empty() { + return Err(invalid()); + } + let atoms = self + .atoms + .iter() + .map(|atom| { + let element = periodic_table::periodic_table() + .iter() + .find(|e| e.atomic_number == atom.atomic_number) + .copied() + .ok_or_else(invalid)?; + if !atom.position.iter().all(|v| v.is_finite()) { + return Err(invalid()); + } + Ok(Atom::new(element, atom.position.into())) + }) + .collect::, PortableError>>()?; + self.hf + .guess + .validate() + .map_err(|error| PortableError::InvalidRequest(error.to_string()))?; + let hf = HfConfig { + method: match self.hf.method { + RequestedMethod::Auto => HfMethod::Auto, + RequestedMethod::Rhf => HfMethod::Rhf, + RequestedMethod::Uhf => HfMethod::Uhf, + } + .into(), + max_iterations: NonZeroUsize::new(self.hf.max_iterations).ok_or_else(invalid)?, + convergence_threshold: PositiveFiniteF64::try_new(self.hf.convergence_threshold) + .map_err(|_| invalid())?, + linear_dependency_threshold: NonNegativeFiniteF64::try_new( + self.hf.linear_dependency_threshold, + ) + .map_err(|_| invalid())? + .into(), + eri_schwarz_threshold: self + .hf + .eri_schwarz_threshold + .map(PositiveFiniteF64::try_new) + .transpose() + .map_err(|_| invalid())?, + guess: self.hf.guess.to_config().into(), + diis: self.hf.diis, + diis_size: DiisSize::try_new(self.hf.diis_size).map_err(|_| invalid())?, + }; + let request = CalculationRequest { + geometry: Geometry::new(String::new(), atoms), + molecule: MoleculeConfig { + units: match self.units { + CoordinateUnits::Bohr => Units::Bohr, + CoordinateUnits::Angstrom => Units::Angstrom, + }, + charge: self.charge.into(), + multiplicity: NonZeroU8::new(self.multiplicity) + .ok_or_else(invalid)? + .into(), + }, + basis_name: self.basis.clone(), + hf, + mp2: self.mp2.as_ref().map(|mp2| Mp2Config { + frozen_orbitals: mp2.frozen_orbitals.into(), + memory_limit: match mp2.memory_limit { + RequestedMemory::Auto => MemoryLimit::Auto, + RequestedMemory::Fixed { bytes } => { + MemoryLimit::Fixed(bytesize::ByteSize(bytes)) + } + } + .into(), + }), + }; + let molecule = request + .molecule() + .build(request.geometry().clone()) + .map_err(|error| PortableError::InvalidRequest(error.to_string()))?; + request + .hf() + .resolve_method(&molecule) + .map_err(|error| PortableError::InvalidRequest(error.to_string()))?; + Ok(request) + } +} diff --git a/crates/rustiq-core/src/persistence/storage.rs b/crates/rustiq-core/src/persistence/storage.rs index 726a7351..e8de4f11 100644 --- a/crates/rustiq-core/src/persistence/storage.rs +++ b/crates/rustiq-core/src/persistence/storage.rs @@ -182,10 +182,10 @@ impl Storage { ) -> Result { let mut bytes = serde_json::to_vec_pretty(value)?; bytes.push(b'\n'); - let limit = if path.as_str() == super::calculation::CALCULATION_PATH { - super::calculation::MAX_CALCULATION_BYTES - } else { - 1024 * 1024 + let limit = match path.as_str() { + super::calculation::CALCULATION_PATH => super::calculation::MAX_CALCULATION_BYTES, + super::request::REQUEST_PATH => super::request::MAX_REQUEST_BYTES, + _ => 1024 * 1024, }; if bytes.len() as u64 > limit { return Err(ManifestError::TooLarge); diff --git a/crates/rustiq-core/src/persistence/storage/zip.rs b/crates/rustiq-core/src/persistence/storage/zip.rs index 0821f717..25742b06 100644 --- a/crates/rustiq-core/src/persistence/storage/zip.rs +++ b/crates/rustiq-core/src/persistence/storage/zip.rs @@ -50,7 +50,10 @@ pub(super) fn open(path: &Path) -> Result, StorageError> { } pub(super) fn options(path: &RelativePath) -> SimpleFileOptions { - let metadata = matches!(path.as_str(), "manifest.json" | "calculation.json"); + let metadata = matches!( + path.as_str(), + "manifest.json" | "calculation.json" | "request.json" + ); SimpleFileOptions::default() .compression_method(if metadata { CompressionMethod::Deflated diff --git a/crates/rustiq-core/tests/data/persistence/portable-python-big-v1.rustiq.hex b/crates/rustiq-core/tests/data/persistence/portable-python-big-v1.rustiq.hex index 61e6bb94..a712eac2 100644 --- a/crates/rustiq-core/tests/data/persistence/portable-python-big-v1.rustiq.hex +++ b/crates/rustiq-core/tests/data/persistence/portable-python-big-v1.rustiq.hex @@ -1 +1 @@ -504b03042d0000000800000021003e7a0d31ffffffffffffffff0d0014006d616e69666573742e6a736f6e010010003c03000000000000ad0100000000000075525b6edb3010fcf7290c7dc7091f161f39456f602cb9cb98ad2d292415d40d7cf7927e444683fe08e2ec706766b99fabf5ba0b633a42e95ed75d9a7389ef9b89528eb9d0e0a97b5a18bb8f868f4365f20bfc2b0ed8ae4d632ae00e37f294469c3da55af9ace78a0c70a4c6fb712afb7158c7a1501aab08b87888e5b40ef17799d3f57aa52f321def2a74beb4cd3ed25062887e17b1fd95d3a2f08fb38a607ca37c0995f7207af5ca020a0dc082e5561bc5552f84776acba4e14ea093ca714472ceea2003f6120c934a68a6b9f6cafbc58987839f0f50ae82370713947d537b283effcc95f12dd4dd618e7fda54b8b1e27f9e7b2e2492e70a04b7bde352588b3a6c9d6d611c33d6c9a0ac0cbe76719e056724859a8a91b2a1ef17cf90eae8c097bc38867147297e9d1f32404a70ca2fed9dde121cf20b8c9b4a7d1ea6d32dce837dadbea06ffe111d0a2990340f5bc134a22150ae97e48dc6adc1207b8e4e188b5e736218d896505b634c1fd085452dd19428d767bf8ffdbeab7e3c4e3557f3b7f9e0cb052825453717ca0f092beea0aef62ecc836f8d5a513c2dd5d66dbe6a3c6efb8d705eddbfe7d579f517504b03042d000000080000002100021adb15ffffffffffffffff1000140063616c63756c6174696f6e2e6a736f6e010010006407000000000000e201000000000000ed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b03042d000000000000002100feb6142cffffffffffffffff1b0014006172726179732f696e74656772616c732f616f2d6572692e6e707901001000b000000000000000b000000000000000934e554d5059010076007b276465736372273a20273e6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a3fe00000000000003ff80000000000004004000000000000400c00000000000040120000000000004016000000000000504b01022d032d0000000800000021003e7a0d31ffffffffffffffff0d0014000000000000000000a481000000006d616e69666573742e6a736f6e010010003c03000000000000ad01000000000000504b01022d032d000000080000002100021adb15ffffffffffffffff10001c000000000000000000a481ffffffff63616c63756c6174696f6e2e6a736f6e010018006407000000000000e201000000000000ec01000000000000504b01022d032d000000000000002100feb6142cffffffffffffffff1b001c000000000000000000a481ffffffff6172726179732f696e74656772616c732f616f2d6572692e6e707901001800b000000000000000b0000000000000001004000000000000504b06062c000000000000002d002d000000000000000000030000000000000003000000000000000e010000000000000d05000000000000504b0607000000001b0600000000000001000000504b050600000000030003000e0100000d0500000000 +504b03042d00000008000000210076c69d7fffffffffffffffff0d0014006d616e69666573742e6a736f6e01001000d3040000000000004c020000000000008d54db6edb300c7def5704796e13dd2ffd8abd0f43208954ab2eb153592e9615f9f7d1b9d4418b02030cc3268fc873a823bddf2d16cbdcd75d68cbc7c5b28e432baf0f7bac43191a760997f73362f336c5fb8e90fc14fe5d3a9896edfbda42dc5ec0fbdac398b052e69dfe29d2851d4eb81f87f6dc778bd235ac3d3509b16c4b3b2c72f9d3c67a5e4ef0b9cd922f29743c951d52c1ae955cd2a6c0f4d50e73874fcc2802e5098793a8e139086d1e5906614360d9736f9de1460b91a2514c3a1e0544692207c018bdcd328396c13169846596db64529a99547c1dcfb52fddf7a13d9fc6774eac5e06a2f245cc95d950fe4ed330c67c4735b26cc0e9e44d663e092958d4d6aac485d23a0606a8b8972958cd6322008512701098912991cc4c35856d1ab7a19d197ca27b93fc3fcadc79f11d67cd85044cdc04c1bd8e5c0aefc16615fd34f7c89c8f321b2f73a22a31b11c9dc44c1bc0d0f8acf5cd46f7634d3850ed9fa75e67d694e86b792a5dd86eae7e5aadd6f4dcca68fd6e7b917123f45271cde6d4178db34aa73e225f4d84ca2913658220d17a411e8200ca062b3293641fcd93d64e29ce910967b940149e7ca6b853999d768674d2fbd7496ba8e4e890da30ef4ee83758cbc7ff8d8c506b380cebe9f83cd5b01dd6a17f20e8aadb1f969f45706bbe570110815c0568795682590087c1442d31390bca41969a4314ce43b22404325308d63be7748698e76e15f715073a8d578b5daf90d4eff6a46be2f7f0c6e705a1b55ae2d870b85148f118e8c6d9e4b14b53a12929eee7ec546d3cf7b8bd842e80e3c7508f77c7bb7f504b03042d000000080000002100021adb15ffffffffffffffff1000140063616c63756c6174696f6e2e6a736f6e010010006407000000000000e201000000000000ed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b03042d0000000800000021004100ae2bffffffffffffffff0c001400726571756573742e6a736f6e010010009a020000000000003001000000000000a592c14ec3300c86ef7b8a28e76e6a07428817e0b803dca6a94a536fb14893ce49806ddabb93a4d940489390b8a4f5efcfb1fdb7a719637c6b69109e3f314ec179dccf09f6019ce755cabe0339b426a69b1c0783de25b8b38a2644783b24691d03c64ef92c32cad684a103bad4e7cc681dfae9d275d118ab1775753328ef9bfc3c57ff6ad42ceeffda289e9bbca254827610afc9341f82f6386a94e80f57673ae1303bf3f2ba9adf3d4fdea86d54a649f9005ed93e112278cbaba28acf163d904883a6faa62e2371694db47f074642eb15815356a7fa06e6f56361341a10d4f63082e92379b8490261eba4fa1074fccd34cbc2f488ae75784cab3e146d177f0677dd220a6f68f216d212b44a0ca8bd35280cfff61dc807ea44f1de04ad7f7eb9dc26ea5ba11dcc8acc87717961cfb32f504b03042d000000080000002100564c8e8fffffffffffffffff09001400736f75726365732f300100100054000000000000005400000000000000535670ad484c2e5128ce2f2d4a4e552828ca2f4bcd4bcc4b4eb55248cecfcd4dcd2b295648cc4b51700ef27153482c02a9482d4e2d2a4b4dd1e3e58ace488be5e5ca4d2dc9c84f51b05550722c2dc957e2e50200504b03042d000000000000002100feb6142cffffffffffffffff1b0014006172726179732f696e74656772616c732f616f2d6572692e6e707901001000b000000000000000b000000000000000934e554d5059010076007b276465736372273a20273e6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a3fe00000000000003ff80000000000004004000000000000400c00000000000040120000000000004016000000000000504b01022d032d00000008000000210076c69d7fffffffffffffffff0d0014000000000000000000a481000000006d616e69666573742e6a736f6e01001000d3040000000000004c02000000000000504b01022d032d000000080000002100021adb15ffffffffffffffff10001c000000000000000000a481ffffffff63616c63756c6174696f6e2e6a736f6e010018006407000000000000e2010000000000008b02000000000000504b01022d032d0000000800000021004100ae2bffffffffffffffff0c001c000000000000000000a481ffffffff726571756573742e6a736f6e010018009a020000000000003001000000000000af04000000000000504b01022d032d000000080000002100564c8e8fffffffffffffffff09001c000000000000000000a481ffffffff736f75726365732f3001001800540000000000000054000000000000001d06000000000000504b01022d032d000000000000002100feb6142cffffffffffffffff1b001c000000000000000000a481ffffffff6172726179732f696e74656772616c732f616f2d6572692e6e707901001800b000000000000000b000000000000000ac06000000000000504b06062c000000000000002d002d00000000000000000005000000000000000500000000000000b701000000000000a907000000000000504b060700000000600900000000000001000000504b05060000000005000500b7010000a90700000000 diff --git a/crates/rustiq-core/tests/data/persistence/portable-python-little-v1.rustiq.hex b/crates/rustiq-core/tests/data/persistence/portable-python-little-v1.rustiq.hex index 03cd60a0..d3b26d4a 100644 --- a/crates/rustiq-core/tests/data/persistence/portable-python-little-v1.rustiq.hex +++ b/crates/rustiq-core/tests/data/persistence/portable-python-little-v1.rustiq.hex @@ -1 +1 @@ -504b03042d00000008000000210048a229b3ffffffffffffffff0d0014006d616e69666573742e6a736f6e010010003c03000000000000ac0100000000000075525b72dc2010fcdf53a8f4edb5792c087c8adc606b80c14ba2956440ae28aebd7b601f962aaefc50d0d34c770f7cee9aa6f5633c436e5f9b36ce2987f7fd8431859471b0d83ead8ce347c5c7a130e915fe150657af4d63cc60fa3b798aa39b2dc652f92ce7820c70c6cafbb1e4d3383461c818c7220226f4212f8d0fbff31c6fd70b7d9569695ba0cbb56db201871c7cb0c7e0ea2e2fabc23fce0ae2c21ba66ba8740226e42bf18e7500c46baa3b25a9148c59230f842b6a98335c1aea1c1aa33bcfbd131c14e19275a4a39d95d6ae4e2cf476ee21df04ef0e26c8a7aab6293eff4c85f12dd4c3610a7fea54a8d2ec7f9e0565dca1a51218d5c250ceb4769d3f185dc318a2b4e15e6aee6de9622cf14671f4251541a9bd10ab6788657460735a1dc378c418bece9b0c10232ce9a5bed35b843ebdc0b82fd4e7615aee7136f63bf9057df3ef1457440b4f34a30e9d605628ec34f59e8223de0aa19ca5e6600e1c0065291b3c08a4daea8e97fdaa16718a98cab33fc6fef8ab763c4f2557f5b7ffa0eb05c8390633674c9b84053750bef6d1cf83ad8d6a913dadd5da6dbe696c7ffb9d70d93dd6cbeeb2fb0b504b03042d000000080000002100021adb15ffffffffffffffff1000140063616c63756c6174696f6e2e6a736f6e010010006407000000000000e201000000000000ed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b03042d000000000000002100db95d32dffffffffffffffff1b0014006172726179732f696e74656772616c732f616f2d6572692e6e707901001000b000000000000000b000000000000000934e554d5059010076007b276465736372273a20273c6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a000000000000e03f000000000000f83f00000000000004400000000000000c4000000000000012400000000000001640504b01022d032d00000008000000210048a229b3ffffffffffffffff0d0014000000000000000000a481000000006d616e69666573742e6a736f6e010010003c03000000000000ac01000000000000504b01022d032d000000080000002100021adb15ffffffffffffffff10001c000000000000000000a481ffffffff63616c63756c6174696f6e2e6a736f6e010018006407000000000000e201000000000000eb01000000000000504b01022d032d000000000000002100db95d32dffffffffffffffff1b001c000000000000000000a481ffffffff6172726179732f696e74656772616c732f616f2d6572692e6e707901001800b000000000000000b0000000000000000f04000000000000504b06062c000000000000002d002d000000000000000000030000000000000003000000000000000e010000000000000c05000000000000504b0607000000001a0600000000000001000000504b050600000000030003000e0100000c0500000000 +504b03042d000000080000002100001eb9fdffffffffffffffff0d0014006d616e69666573742e6a736f6e01001000d3040000000000004b020000000000008d54db6edb300c7def5704796e13dd2ffd8abd0f43404954ab2eb153592e9615f9f7d1b9d4418b02030c43228fc873484aef778bc532f775076df9b858d67168e5f5618f752843c32ee2f27e466cde267bdf11929fccbf4b97a663fbbe3608db0b785ffb3446ace479a73d593ad8e184fb7168cf7db7285dc3da531208655bda6191cb9f36d6f37182cf69967c49a6e329ec100b76ade41237254dab7698337c624696549e7038891a9e4168f3c87212168065cfbd75861b2d440c4631e97810294813784a1882b759e6a42538268db0cc721b4d8c33938aafe339f625fb1edaf3a97c67c7ea65202a5fc45c990de5ef540d63cc775403cb26391dbdc9cc4721050bda5a15b9505a07600915f73282d53c44029029269e0466644a4433538db08de316da99c127ba37ceffa3cc9d17df71d65cc884911b10dcebc0a5f03ed9ac829fea1e98f34166e3658e14254496839398a9010c8dcf5adf34ba1f6bc48162ff3ce53ab326475fcb53e960bbb9ced36ab5a6ef5646eb77db8b8c1ba197886b36bbbe689c553af561f93a44a89c3241c60412ad1734430992b260456692c647f3a8b5538a7364c2592e1085a73953dca9cc4e9d219df4ff75d20a95261a621be6ee40bfc15a3ef63732a056380cebe9fa3c55d80e6be81f08baeaf687e56711dc9aef5524271df39a864bf084498ba81da9e13973482c4f0a52e441052501d0903ba0d2c87df456d27ace56715f71a0db781db1eb1312fbdd9e744dfc1edef87c005aab258c0d871b85640f402fce268f5d9c024d4e713f7ba768e339c7ed2374011c3f8a7abc3bdefd03504b03042d000000080000002100021adb15ffffffffffffffff1000140063616c63756c6174696f6e2e6a736f6e010010006407000000000000e201000000000000ed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b03042d0000000800000021004100ae2bffffffffffffffff0c001400726571756573742e6a736f6e010010009a020000000000003001000000000000a592c14ec3300c86ef7b8a28e76e6a07428817e0b803dca6a94a536fb14893ce49806ddabb93a4d940489390b8a4f5efcfb1fdb7a719637c6b69109e3f314ec179dccf09f6019ce755cabe0339b426a69b1c0783de25b8b38a2644783b24691d03c64ef92c32cad684a103bad4e7cc681dfae9d275d118ab1775753328ef9bfc3c57ff6ad42ceeffda289e9bbca254827610afc9341f82f6386a94e80f57673ae1303bf3f2ba9adf3d4fdea86d54a649f9005ed93e112278cbaba28acf163d904883a6faa62e2371694db47f074642eb15815356a7fa06e6f56361341a10d4f63082e92379b8490261eba4fa1074fccd34cbc2f488ae75784cab3e146d177f0677dd220a6f68f216d212b44a0ca8bd35280cfff61dc807ea44f1de04ad7f7eb9dc26ea5ba11dcc8acc87717961cfb32f504b03042d000000080000002100564c8e8fffffffffffffffff09001400736f75726365732f300100100054000000000000005400000000000000535670ad484c2e5128ce2f2d4a4e552828ca2f4bcd4bcc4b4eb55248cecfcd4dcd2b295648cc4b51700ef27153482c02a9482d4e2d2a4b4dd1e3e58ace488be5e5ca4d2dc9c84f51b05550722c2dc957e2e50200504b03042d000000000000002100db95d32dffffffffffffffff1b0014006172726179732f696e74656772616c732f616f2d6572692e6e707901001000b000000000000000b000000000000000934e554d5059010076007b276465736372273a20273c6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a000000000000e03f000000000000f83f00000000000004400000000000000c4000000000000012400000000000001640504b01022d032d000000080000002100001eb9fdffffffffffffffff0d0014000000000000000000a481000000006d616e69666573742e6a736f6e01001000d3040000000000004b02000000000000504b01022d032d000000080000002100021adb15ffffffffffffffff10001c000000000000000000a481ffffffff63616c63756c6174696f6e2e6a736f6e010018006407000000000000e2010000000000008a02000000000000504b01022d032d0000000800000021004100ae2bffffffffffffffff0c001c000000000000000000a481ffffffff726571756573742e6a736f6e010018009a020000000000003001000000000000ae04000000000000504b01022d032d000000080000002100564c8e8fffffffffffffffff09001c000000000000000000a481ffffffff736f75726365732f3001001800540000000000000054000000000000001c06000000000000504b01022d032d000000000000002100db95d32dffffffffffffffff1b001c000000000000000000a481ffffffff6172726179732f696e74656772616c732f616f2d6572692e6e707901001800b000000000000000b000000000000000ab06000000000000504b06062c000000000000002d002d00000000000000000005000000000000000500000000000000b701000000000000a807000000000000504b0607000000005f0900000000000001000000504b05060000000005000500b7010000a80700000000 diff --git a/crates/rustiq-core/tests/data/persistence/request-h2-v1.json b/crates/rustiq-core/tests/data/persistence/request-h2-v1.json new file mode 100644 index 00000000..6ed0d621 --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/request-h2-v1.json @@ -0,0 +1,40 @@ +{ + "format": "rustiq-request", + "version": 1, + "units": "bohr", + "atoms": [ + { + "atomic_number": 1, + "position": [ + 0.0, + 0.0, + 0.0 + ] + }, + { + "atomic_number": 1, + "position": [ + 1.4, + 0.0, + 0.0 + ] + } + ], + "charge": 0, + "multiplicity": 1, + "basis": "STO-3G", + "hf": { + "method": "auto", + "max_iterations": 100, + "convergence_threshold": 1e-08, + "linear_dependency_threshold": 1e-08, + "eri_schwarz_threshold": 1e-12, + "diis_size": 6, + "guess": { + "kind": "core_hamiltonian", + "perturbation": null + }, + "diis": false + }, + "mp2": null +} diff --git a/crates/rustiq-core/tests/data/persistence/source-original-v1.toml.hex b/crates/rustiq-core/tests/data/persistence/source-original-v1.toml.hex new file mode 100644 index 00000000..6926b914 --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/source-original-v1.toml.hex @@ -0,0 +1 @@ +2320457861637420736f757263652070726f76656e616e63653a20636f6d6d656e747320616e642043524c4620617265207072657365727665642e0d0a5b68665d0d0a6d6574686f64203d20224175746f220d0a diff --git a/crates/rustiq-core/tests/portable_artifact.rs b/crates/rustiq-core/tests/portable_artifact.rs index 99893039..99aed97f 100644 --- a/crates/rustiq-core/tests/portable_artifact.rs +++ b/crates/rustiq-core/tests/portable_artifact.rs @@ -13,6 +13,9 @@ fn public_api_creates_inspects_and_recovers_a_portable_artifact() { .prepare() .unwrap(); let mut data = RustiQData::from_calculation(&prepared).unwrap(); + let original = b"# source comments are provenance\r\n[hf]\r\nmethod = 'Auto'\r\n"; + data.add_source("../calculation.toml", original.as_slice()) + .unwrap(); let values = vec![0.5, 1.5, 2.5, 3.5, 4.5, 5.5]; let indices = [ (0, 0, 0, 0), @@ -39,6 +42,17 @@ fn public_api_creates_inspects_and_recovers_a_portable_artifact() { Err(PortableError::AlreadyExists) )); let mut restored = RustiQData::open(&path).unwrap(); + assert_eq!( + restored.request().unwrap().hf().method.value, + rustiq_core::config::HfMethod::Auto + ); + assert_eq!( + restored.request().unwrap().molecule().units, + rustiq_core::molecules::units::Units::Bohr + ); + let source = restored.sources().next().unwrap(); + assert_eq!(source.original_name(), "../calculation.toml"); + assert_eq!(source.bytes(), original); let context = restored.calculation().unwrap(); assert_eq!(context.charge(), 0); assert_eq!(context.multiplicity(), 1); diff --git a/docs/persistence-format-v1.md b/docs/persistence-format-v1.md index 998241f7..0a46ca33 100644 --- a/docs/persistence-format-v1.md +++ b/docs/persistence-format-v1.md @@ -236,16 +236,23 @@ unchanged. ## Portable `.rustiq` V1 A portable artifact is a ZIP archive, with ZIP64 support for large members and -large offsets. It contains `manifest.json`, `calculation.json`, and zero or more +large offsets. It contains `manifest.json`, independently versioned `request.json` +and `calculation.json`, optional opaque `sources/` payloads, and zero or more indexed scientific artifacts. It is not an NPZ file. Readers access members by name without extracting files. The active directory cache remains a separate, disposable machine-local product with its existing format and behavior. The portable manifest uses `kind: "portable"`. Its format/version, producer, scientific identity and artifact envelope are the same as the logical format -above. It additionally requires a calculation reference: +above. It additionally requires separate `request` and `calculation` references: ```json +"request": { + "path": "request.json", + "version": 1, + "size": 1234, + "digest": "sha256:..." +}, "calculation": { "path": "calculation.json", "version": 1, @@ -255,10 +262,55 @@ above. It additionally requires a calculation reference: ``` `size` and `digest` describe the exact uncompressed JSON bytes, including any -whitespace. Cache manifests do not require this field. Producer version remains +whitespace. Each reference version selects its own snapshot schema; it is not +tied to the package version or the other snapshot version. Cache manifests omit +both references and source provenance. Producer version remains provenance, not an invalidation rule. Archives may contain no numerical artifacts; the scientific context remains useful by itself. +### Normalized request and optional source provenance + +The normative request schema is +[`request-snapshot-v1.schema.json`](../schemas/request-snapshot-v1.schema.json), +with an [H2 golden request](../crates/rustiq-core/tests/data/persistence/request-h2-v1.json). +`request.json` is constructed from `PreparedCalculation::request()`, not source +TOML/XYZ or the resolved snapshot. Its `format: "rustiq-request"` and `version: 1` +identify the independent wire contract. Reopening reconstructs a typed +`CalculationRequest`, without depending on original input files. + +The request preserves ordered atoms in the requested `bohr`/`angstrom` units, +charge, multiplicity, a portable basis label, requested `auto`/`rhf`/`uhf` method, +defaulted HF settings, and optional MP2 settings. DIIS enablement and history size +remain distinct even when disabled. The requested MP2 memory limit is tagged +`{"kind":"auto"}` or `{"kind":"fixed","bytes":...}`; no machine-resolved +memory budget is stored. Random seeds omitted in the request remain absent even +when preparation chooses a seed for the resolved calculation. Comments, source +spans, source paths, cache configuration, and output/UI settings are excluded. +The basis label is informational; resolved AO contents define scientific identity. + +Sources are optional opaque bytes, including non-UTF-8 content. They are never +parsed to reconstruct either semantic snapshot and never enter compatibility. +The optional manifest `sources` array contains records such as: + +```json +{"original_name":"../input.toml","path":"sources/0","version":1, + "size":123,"digest":"sha256:..."} +``` + +`original_name` is informational provenance and may contain an original path. +It is never used as an extraction or publication path. Writers assign safe member +names (`sources/0`, `sources/1`, ...); readers accept safe relative members under +`sources/` and reject duplicate references. Sources have size/SHA-256 checks and +are read eagerly with a combined 64 MiB limit, at most 256 sources, and at most +4096 UTF-8 bytes per original name. Missing sources are valid for direct Rust API +use. `request.json` and `calculation.json` are required independently of sources. + +The reader validates request semantics and their consistency with the resolved +molecule, method, scientific settings, and MP2 selection. Requested Angstrom +coordinates resolve to Bohr and `auto` resolves to RHF/UHF; a missing requested +random seed may resolve to a generated seed. Neither snapshot substitutes for +the other. + ### Resolved scientific snapshot The normative structural schema is @@ -283,8 +335,9 @@ of domain Rust struct layouts and package versions. `null`/omitted DIIS size means DIIS disabled. - Density guesses are tagged as `core_hamiltonian`, `one_electron`, `random`, or `zero`. Perturbations and random guesses record distribution parameters and an - optional seed. An absent seed records an unseeded request, not the random - generator state or an exact continuation promise. + resolved seed. Seeds chosen during preparation are captured here, while an + initially absent seed stays absent in `request.json`. This does not capture + generator state or promise exact SCF continuation. - `mp2` is absent/null for HF-only requests, or contains `frozen_orbitals`. This records the request, not an MP2 result or proof of convergence. - `ao_eri_computation_version` states the scientific algorithm identity used for @@ -306,7 +359,7 @@ restart state. ### Rust API -`RustiQData::from_calculation(&PreparedCalculation)` captures resolved inputs +`RustiQData::from_calculation(&PreparedCalculation)` captures requested and resolved inputs without executing HF. `set_eri` / `set::` supply an owned `CompactEri` whose dimension matches those inputs. The caller is responsible for supplying the ERIs computed for that calculation, rather than unrelated values @@ -317,6 +370,8 @@ use rustiq_core::persistence::{CompactEri, RustiQData}; // `prepared` is produced by CalculationBuilder; `eri` belongs to these inputs. let mut data = RustiQData::from_calculation(&prepared)?; +// Optional: preserve exact source bytes separately from scientific state. +data.add_source("calculation.toml", original_toml_bytes)?; data.set_eri(eri)?; data.write("water.rustiq")?; @@ -331,12 +386,15 @@ if restored.eri_is_compatible(&prepared) { } ``` -`calculation()` exposes read-only typed context, including atoms, effective basis, +`request()` exposes the reconstructed typed normalized request. `sources()` +exposes exact captured bytes and informational original names. `calculation()` +exposes read-only typed resolved context, including atoms, effective basis, HF settings and MP2 request. `producer()` exposes provenance, and `artifact_representations()` lists known and unknown artifacts without loading arrays. These inspections do not certify payload integrity. -`open` validates the container index, manifest, snapshot digest/schema, scientific +`open` requires only read access to the source. It validates the container index, +manifest, both snapshot versions/digests/schemas, source sizes/digests, scientific identity, and declared artifact sizes. ERI dimensions and computation-version metadata must agree with the snapshot. It does not decode numerical arrays. The first typed read verifies SHA-256, bounded NPY header, f64 dtype, shape and @@ -349,7 +407,8 @@ calculation without loading arrays. Changing an HF-only request to MP2 does not invalidate AO ERIs. Compatibility does not imply that the payload has already been verified. There is no CLI reuse orchestration in this API release. -Writing a reopened archive preserves unloaded artifacts by verified streaming +Writing a reopened archive preserves both semantic snapshots and exact captured +source bytes. It preserves unloaded artifacts by verified streaming copy, including unknown representations and their attributes. Known ERI headers are checked before preservation; overwritten ERIs use the current representation. Errors use `PortableError` (including `AlreadyExists`, unsupported version, diff --git a/schemas/request-snapshot-v1.schema.json b/schemas/request-snapshot-v1.schema.json new file mode 100644 index 00000000..63f81766 --- /dev/null +++ b/schemas/request-snapshot-v1.schema.json @@ -0,0 +1,336 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/mveril/RustiQ/blob/main/schemas/request-snapshot-v1.schema.json", + "title": "RustiQ normalized requested calculation V1", + "type": "object", + "additionalProperties": false, + "required": [ + "format", + "version", + "units", + "atoms", + "charge", + "multiplicity", + "basis", + "hf" + ], + "properties": { + "format": { + "const": "rustiq-request" + }, + "version": { + "const": 1 + }, + "units": { + "enum": [ + "bohr", + "angstrom" + ] + }, + "atoms": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/atom" + } + }, + "charge": { + "type": "integer", + "minimum": -2147483648, + "maximum": 2147483647 + }, + "multiplicity": { + "type": "integer", + "minimum": 1, + "maximum": 255 + }, + "basis": { + "type": "string" + }, + "hf": { + "$ref": "#/$defs/hf" + }, + "mp2": { + "anyOf": [ + { + "$ref": "#/$defs/mp2" + }, + { + "type": "null" + } + ] + } + }, + "$defs": { + "atom": { + "type": "object", + "additionalProperties": false, + "required": [ + "atomic_number", + "position" + ], + "properties": { + "atomic_number": { + "type": "integer", + "minimum": 1, + "maximum": 118 + }, + "position": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "items": { + "type": "number" + } + } + } + }, + "distribution": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "min", + "max" + ], + "properties": { + "kind": { + "const": "uniform" + }, + "min": { + "type": "number" + }, + "max": { + "type": "number" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "mean", + "std_dev" + ], + "properties": { + "kind": { + "const": "normal" + }, + "mean": { + "type": "number" + }, + "std_dev": { + "type": "number", + "exclusiveMinimum": 0 + } + } + } + ] + }, + "random": { + "type": "object", + "additionalProperties": false, + "required": [ + "distribution" + ], + "properties": { + "seed": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615 + }, + { + "type": "null" + } + ] + }, + "distribution": { + "$ref": "#/$defs/distribution" + } + } + }, + "guess": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "core_hamiltonian" + }, + "perturbation": { + "anyOf": [ + { + "$ref": "#/$defs/random" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "one_electron" + }, + "perturbation": { + "anyOf": [ + { + "$ref": "#/$defs/random" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "random" + ], + "properties": { + "kind": { + "const": "random" + }, + "random": { + "$ref": "#/$defs/random" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "zero" + } + } + } + ] + }, + "hf": { + "type": "object", + "additionalProperties": false, + "required": [ + "method", + "max_iterations", + "convergence_threshold", + "linear_dependency_threshold", + "guess", + "diis" + ], + "properties": { + "method": { + "enum": [ + "auto", + "rhf", + "uhf" + ] + }, + "max_iterations": { + "type": "integer", + "minimum": 1 + }, + "convergence_threshold": { + "type": "number", + "exclusiveMinimum": 0 + }, + "linear_dependency_threshold": { + "type": "number", + "minimum": 0 + }, + "eri_schwarz_threshold": { + "anyOf": [ + { + "type": "number", + "exclusiveMinimum": 0 + }, + { + "type": "null" + } + ] + }, + "diis_size": { + "type": "integer", + "minimum": 2 + }, + "guess": { + "$ref": "#/$defs/guess" + }, + "diis": { + "type": "boolean" + } + } + }, + "mp2": { + "type": "object", + "additionalProperties": false, + "required": [ + "frozen_orbitals", + "memory_limit" + ], + "properties": { + "frozen_orbitals": { + "type": "integer", + "minimum": 0 + }, + "memory_limit": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "auto" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "bytes" + ], + "properties": { + "kind": { + "const": "fixed" + }, + "bytes": { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615 + } + } + } + ] + } + } + } + } +} diff --git a/tools/reference/generate_portable_fixtures.py b/tools/reference/generate_portable_fixtures.py index 28817c57..5a56f3a4 100644 --- a/tools/reference/generate_portable_fixtures.py +++ b/tools/reference/generate_portable_fixtures.py @@ -62,6 +62,8 @@ def digest(payload: bytes) -> str: def generate() -> None: snapshot = (FIXTURES / "calculation-h2-v1.json").read_bytes() + request = (FIXTURES / "request-h2-v1.json").read_bytes() + source = bytes.fromhex((FIXTURES / "source-original-v1.toml.hex").read_text()) for suffix, npy_fixture in [ ("little", "ao-eri-python-v1.npy.hex"), ("big", "ao-eri-python-big-endian-v1.npy.hex"), @@ -76,12 +78,27 @@ def generate() -> None: "version": 1, "digest": scientific_identity(json.loads(snapshot)), }, + "request": { + "path": "request.json", + "version": 1, + "size": len(request), + "digest": digest(request), + }, "calculation": { "path": "calculation.json", "version": 1, "size": len(snapshot), "digest": digest(snapshot), }, + "sources": [ + { + "original_name": "../../calculation.toml", + "path": "sources/0", + "version": 1, + "size": len(source), + "digest": digest(source), + } + ], "artifacts": { "ao_eri": { "path": "arrays/integrals/ao-eri.npy", @@ -101,6 +118,8 @@ def generate() -> None: for name, content in [ ("manifest.json", json.dumps(manifest, indent=2).encode() + b"\n"), ("calculation.json", snapshot), + ("request.json", request), + ("sources/0", source), ("arrays/integrals/ao-eri.npy", payload), ]: info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) diff --git a/tools/reference/test_persistence_interop.py b/tools/reference/test_persistence_interop.py index 82fd495e..76ef919a 100644 --- a/tools/reference/test_persistence_interop.py +++ b/tools/reference/test_persistence_interop.py @@ -88,9 +88,23 @@ def test_numpy_reads_rust_portable_archive(tmp_path: Path) -> None: assert set(archive.namelist()) == { "manifest.json", "calculation.json", + "request.json", + "sources/0", "arrays/integrals/ao-eri.npy", } manifest = json.loads(archive.read("manifest.json")) + request_bytes = archive.read("request.json") + request = json.loads(request_bytes) + assert manifest["request"]["digest"] == digest(request_bytes) + assert manifest["request"]["size"] == len(request_bytes) + assert request["hf"]["method"] == "auto" + assert request["units"] == "bohr" + source_bytes = archive.read("sources/0") + assert source_bytes == bytes.fromhex( + (FIXTURE_DIR / "source-original-v1.toml.hex").read_text() + ) + assert manifest["sources"][0]["digest"] == digest(source_bytes) + assert manifest["sources"][0]["original_name"] == "../../calculation.toml" snapshot_bytes = archive.read("calculation.json") snapshot = json.loads(snapshot_bytes) assert manifest["scientific_identity"]["digest"] == scientific_identity( @@ -124,6 +138,14 @@ def test_python_portable_golden_fixtures() -> None: assert b"PK\x06\x06" in content with zipfile.ZipFile(io.BytesIO(content)) as archive: manifest = json.loads(archive.read("manifest.json")) + source_bytes = archive.read("sources/0") + assert source_bytes == bytes.fromhex( + (FIXTURE_DIR / "source-original-v1.toml.hex").read_text() + ) + assert digest(source_bytes) == manifest["sources"][0]["digest"] + request_bytes = archive.read("request.json") + assert digest(request_bytes) == manifest["request"]["digest"] + assert json.loads(request_bytes)["hf"]["method"] == "auto" snapshot = json.loads(archive.read("calculation.json")) assert ( scientific_identity(snapshot) From 3d5b50ad6daff7305bf316d2e990586743a99385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sat, 3 Oct 2026 12:28:28 +0200 Subject: [PATCH 3/8] Fix portable persistence V1 contracts --- .../src/persistence/calculation.rs | 208 ++++++++++++++++-- .../src/persistence/data/portable.rs | 22 +- .../src/persistence/data/portable/tests.rs | 184 +++++++++++++++- .../src/persistence/data/rustiq_data.rs | 9 +- .../rustiq-core/src/persistence/manifest.rs | 55 ++++- crates/rustiq-core/src/persistence/npy.rs | 7 +- crates/rustiq-core/src/persistence/request.rs | 161 +++++++++++--- crates/rustiq-core/src/persistence/storage.rs | 4 +- .../src/persistence/storage/zip.rs | 24 +- .../portable-python-angstrom-v1.rustiq.hex | 1 + docs/persistence-format-v1.md | 32 ++- schemas/calculation-snapshot-v1.schema.json | 9 +- schemas/request-snapshot-v1.schema.json | 12 +- tools/reference/generate_portable_fixtures.py | 56 +++++ tools/reference/test_persistence_interop.py | 36 +++ 15 files changed, 733 insertions(+), 87 deletions(-) create mode 100644 crates/rustiq-core/tests/data/persistence/portable-python-angstrom-v1.rustiq.hex diff --git a/crates/rustiq-core/src/persistence/calculation.rs b/crates/rustiq-core/src/persistence/calculation.rs index 649f153d..d6cc646b 100644 --- a/crates/rustiq-core/src/persistence/calculation.rs +++ b/crates/rustiq-core/src/persistence/calculation.rs @@ -17,9 +17,42 @@ pub(crate) const MAX_CALCULATION_BYTES: u64 = 64 * 1024 * 1024; /// Read-only, resolved scientific context. Coordinates and centers are in Bohr. #[derive(Clone, Debug)] -pub struct CalculationContext(pub(crate) Snapshot); +pub struct CalculationContext(pub(crate) Snapshot, DomainCounts); + +// Checked once at the wire boundary; public accessors retain their domain types. +#[derive(Clone, Debug)] +struct DomainCounts { + max_iterations: usize, + diis_size: Option, + frozen_orbitals: Option, +} +impl DomainCounts { + fn from_snapshot(snapshot: &Snapshot) -> Result { + let invalid = + |_| PortableError::InvalidCalculation("integer exceeds host usize range".into()); + Ok(Self { + max_iterations: usize::try_from(snapshot.hf.max_iterations).map_err(invalid)?, + diis_size: snapshot + .hf + .diis_size + .map(usize::try_from) + .transpose() + .map_err(invalid)?, + frozen_orbitals: snapshot + .mp2 + .as_ref() + .map(|v| usize::try_from(v.frozen_orbitals)) + .transpose() + .map_err(invalid)?, + }) + } +} impl CalculationContext { + pub(crate) fn from_snapshot(snapshot: Snapshot) -> Result { + let counts = DomainCounts::from_snapshot(&snapshot)?; + Ok(Self(snapshot, counts)) + } pub fn atoms(&self) -> impl ExactSizeIterator + '_ { self.0 .atoms @@ -42,7 +75,7 @@ impl CalculationContext { } } pub fn max_iterations(&self) -> usize { - self.0.hf.max_iterations + self.1.max_iterations } pub fn convergence_threshold(&self) -> f64 { self.0.hf.convergence_threshold @@ -54,11 +87,11 @@ impl CalculationContext { self.0.hf.eri_schwarz_threshold } pub fn diis_size(&self) -> Option { - self.0.hf.diis_size + self.1.diis_size } /// `None` means HF only; `Some(n)` requests MP2 with n frozen orbitals. pub fn mp2_frozen_orbitals(&self) -> Option { - self.0.mp2.as_ref().map(|mp2| mp2.frozen_orbitals) + self.1.frozen_orbitals } /// The initial density strategy; random settings do not promise an exact restart. pub fn density_guess(&self) -> crate::config::DensityGuessConfig { @@ -141,17 +174,17 @@ enum Method { #[serde(deny_unknown_fields)] struct Hf { method: Method, - max_iterations: usize, + max_iterations: u64, convergence_threshold: f64, linear_dependency_threshold: f64, eri_schwarz_threshold: Option, - diis_size: Option, + diis_size: Option, guess: Guess, } #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct Mp2 { - frozen_orbitals: usize, + frozen_orbitals: u64, } #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] @@ -178,6 +211,7 @@ impl Snapshot { pub(crate) fn from_calculation( calculation: &PreparedCalculation, ) -> Result { + let invalid = |_| PortableError::InvalidCalculation("integer exceeds V1 u64 range".into()); let molecule = calculation.get_molecule(); let basis = calculation.get_basis(); let hf = calculation.hf_config(); @@ -224,16 +258,25 @@ impl Snapshot { ResolvedHfMethod::Rhf => Method::Rhf, ResolvedHfMethod::Uhf => Method::Uhf, }, - max_iterations: hf.max_iterations.get(), + max_iterations: u64::try_from(hf.max_iterations.get()).map_err(invalid)?, convergence_threshold: hf.convergence_threshold.into_inner(), linear_dependency_threshold: hf.linear_dependency_threshold.value.into_inner(), eri_schwarz_threshold: hf.eri_schwarz_threshold.map(|value| value.into_inner()), - diis_size: hf.diis.then(|| hf.diis_size.into_inner()), + diis_size: hf + .diis + .then(|| u64::try_from(hf.diis_size.into_inner())) + .transpose() + .map_err(invalid)?, guess: hf.guess.value.into(), }, - mp2: mp2.map(|mp2| Mp2 { - frozen_orbitals: mp2.frozen_orbitals.value, - }), + mp2: mp2 + .map(|mp2| { + Ok::<_, PortableError>(Mp2 { + frozen_orbitals: u64::try_from(mp2.frozen_orbitals.value) + .map_err(invalid)?, + }) + }) + .transpose()?, ao_eri_computation_version: AO_ERI_COMPUTATION_VERSION, }; snapshot.validate()?; @@ -304,6 +347,7 @@ impl Snapshot { { return Err(invalid("invalid HF settings")); } + DomainCounts::from_snapshot(self)?; self.hf.guess.validate()?; Ok(()) } @@ -314,11 +358,10 @@ impl Snapshot { ) -> Result<(), PortableError> { let invalid = || PortableError::InvalidRequest("request disagrees with resolved calculation".into()); - let mut molecule = request + let molecule = request .molecule() .build(request.geometry().clone()) .map_err(|_| invalid())?; - molecule.convert_to(crate::molecules::units::Units::Bohr); let method = request .hf() .resolve_method(&molecule) @@ -329,19 +372,34 @@ impl Snapshot { || molecule.atoms.len() != self.atoms.len() || molecule.atoms.iter().zip(&self.atoms).any(|(a, b)| { a.element.atomic_number != b.atomic_number - || <[f64; 3]>::from(a.position) != b.position + || !<[f64; 3]>::from(a.position) + .into_iter() + .zip(b.position) + .all(|(requested, resolved)| { + coordinate_matches_v1(requested, resolved, request.molecule().units) + }) }) || !matches!( (method, self.hf.method), (ResolvedHfMethod::Rhf, Method::Rhf) | (ResolvedHfMethod::Uhf, Method::Uhf) ) - || hf.max_iterations.get() != self.hf.max_iterations + || u64::try_from(hf.max_iterations.get()).map_err(|_| invalid())? + != self.hf.max_iterations || hf.convergence_threshold.into_inner() != self.hf.convergence_threshold || hf.linear_dependency_threshold.value.into_inner() != self.hf.linear_dependency_threshold || hf.eri_schwarz_threshold.map(|v| v.into_inner()) != self.hf.eri_schwarz_threshold - || hf.diis.then(|| hf.diis_size.into_inner()) != self.hf.diis_size - || request.mp2().map(|v| v.frozen_orbitals.value) + || hf + .diis + .then(|| u64::try_from(hf.diis_size.into_inner())) + .transpose() + .map_err(|_| invalid())? + != self.hf.diis_size + || request + .mp2() + .map(|v| u64::try_from(v.frozen_orbitals.value)) + .transpose() + .map_err(|_| invalid())? != self.mp2.as_ref().map(|v| v.frozen_orbitals) || !self.hf.guess.is_resolution_of(&hf.guess.value.into()) { @@ -385,6 +443,40 @@ impl Snapshot { bytes.finish_ao_eri(self.hf.eri_schwarz_threshold) } } +// V1 pins this decimal value (CODATA 2018 Bohr radius in Angstrom), rather +// than inheriting a future physical_constants release. Parse as binary64 and divide. +const V1_BOHR_IN_ANGSTROM: f64 = 0.529_177_210_903; + +fn coordinate_matches_v1( + requested: f64, + resolved: f64, + units: crate::molecules::units::Units, +) -> bool { + use crate::molecules::units::Units; + let expected = match units { + Units::Bohr => requested, + Units::Angstrom => requested / V1_BOHR_IN_ANGSTROM, + }; + if !expected.is_finite() || !resolved.is_finite() { + return false; + } + if units == Units::Bohr { + return expected == resolved; + } + // Four adjacent binary64 values cover division versus reciprocal multiplication + // and final rounding of independently produced conversions. There is no absolute + // chemistry-level floor: even near zero only four subnormal steps are allowed. + fn ordered(value: f64) -> u64 { + let bits = if value == 0.0 { 0 } else { value.to_bits() }; + if bits >> 63 == 0 { + bits | (1 << 63) + } else { + (!bits) + 1 + } + } + ordered(expected).abs_diff(ordered(resolved)) <= 4 +} + fn positive(value: f64) -> bool { value.is_finite() && value > 0.0 } @@ -542,3 +634,83 @@ mod tests { } } } + +#[cfg(test)] +mod v1_contract_tests { + use super::*; + use crate::molecules::units::Units; + use serde_json::{json, Value}; + + #[test] + fn conversion_comparison_is_bounded_in_binary64_steps() { + for requested in [0.74, -0.74, 1e-200, -1e-200, 1e200] { + let expected = requested / V1_BOHR_IN_ANGSTROM; + let four = f64::from_bits(expected.to_bits() + 4); + let five = f64::from_bits(expected.to_bits() + 5); + assert!(coordinate_matches_v1(requested, four, Units::Angstrom)); + assert!(!coordinate_matches_v1(requested, five, Units::Angstrom)); + assert!(!coordinate_matches_v1(requested, four, Units::Bohr)); + } + assert!(coordinate_matches_v1(0.0, -0.0, Units::Angstrom)); + assert!(coordinate_matches_v1( + 0.0, + f64::from_bits(4), + Units::Angstrom + )); + assert!(!coordinate_matches_v1( + 0.0, + f64::from_bits(5), + Units::Angstrom + )); + assert!(!coordinate_matches_v1(0.0, 1e-15, Units::Angstrom)); + assert!(!coordinate_matches_v1(f64::MAX, f64::MAX, Units::Angstrom)); + assert!(!coordinate_matches_v1(0.74, f64::NAN, Units::Angstrom)); + assert!(!coordinate_matches_v1(0.74, f64::INFINITY, Units::Angstrom)); + } + + #[test] + fn resolved_counts_use_u64_wire_and_checked_host_values() { + let schema: Value = serde_json::from_str(include_str!( + "../../../../schemas/calculation-snapshot-v1.schema.json" + )) + .unwrap(); + for (definition, field) in [ + ("hf", "max_iterations"), + ("hf", "diis_size"), + ("mp2", "frozen_orbitals"), + ] { + let property = &schema["$defs"][definition]["properties"][field]; + let integer = if field == "diis_size" { + &property["anyOf"][0] + } else { + property + }; + assert_eq!(integer["maximum"], json!(u64::MAX)); + let mut value: Value = serde_json::from_str(include_str!( + "../../tests/data/persistence/calculation-h2-v1.json" + )) + .unwrap(); + value["mp2"] = json!({"frozen_orbitals":0}); + value[definition][field] = json!(u64::MAX); + let snapshot: Snapshot = serde_json::from_value(value.clone()).unwrap(); + assert_eq!( + snapshot.validate().is_ok(), + usize::try_from(u64::MAX).is_ok() + ); + assert_eq!( + CalculationContext::from_snapshot(snapshot.clone()).is_ok(), + usize::try_from(u64::MAX).is_ok() + ); + assert_eq!( + serde_json::to_value(snapshot).unwrap()[definition][field], + json!(u64::MAX) + ); + for invalid in ["-1", "18446744073709551616"] { + let encoded = serde_json::to_string(&value) + .unwrap() + .replace("18446744073709551615", invalid); + assert!(serde_json::from_str::(&encoded).is_err()); + } + } + } +} diff --git a/crates/rustiq-core/src/persistence/data/portable.rs b/crates/rustiq-core/src/persistence/data/portable.rs index f8069c49..4dcba95d 100644 --- a/crates/rustiq-core/src/persistence/data/portable.rs +++ b/crates/rustiq-core/src/persistence/data/portable.rs @@ -18,13 +18,15 @@ impl RustiQData { /// Add known artifacts with the typed setters before calling [`Self::write`]. pub fn from_calculation(calculation: &PreparedCalculation) -> Result { let snapshot = Snapshot::from_calculation(calculation)?; + RequestSnapshot::from_request(calculation.request())?; + snapshot.validate_request(calculation.request())?; let mut data = Self::new_with_identity( snapshot.identity(), snapshot.basis_functions(), ManifestKind::Portable, ); data.request = Some(calculation.request().clone()); - data.context = Some(CalculationContext(snapshot)); + data.context = Some(CalculationContext::from_snapshot(snapshot)?); Ok(data) } @@ -94,7 +96,10 @@ impl RustiQData { let bytes = source.with_artifact::<_, PortableError, _>(path, |reader| { let mut bytes = Vec::new(); reader.take(reference.size + 1).read_to_end(&mut bytes)?; - if bytes.len() as u64 != reference.size { + if u64::try_from(bytes.len()).map_err(|_| { + PortableError::InvalidArchive("source size exceeds V1 range".into()) + })? != reference.size + { return Err(ArtifactError::IntegrityMismatch(reference.path.clone()).into()); } Ok(bytes) @@ -144,7 +149,7 @@ impl RustiQData { } } data.basis_functions = Some(snapshot.basis_functions()); - data.context = Some(CalculationContext(snapshot)); + data.context = Some(CalculationContext::from_snapshot(snapshot)?); Ok(data) } @@ -174,9 +179,14 @@ impl RustiQData { let total = self .sources .iter() - .map(|s| s.bytes().len() as u64) - .sum::() - + bytes.len() as u64; + .map(|source| source.bytes().len()) + .chain(std::iter::once(bytes.len())) + .try_fold(0_u64, |total, length| { + u64::try_from(length) + .ok() + .and_then(|size| total.checked_add(size)) + }) + .ok_or_else(|| PortableError::InvalidArchive("source size exceeds V1 range".into()))?; if self.sources.len() >= MAX_SOURCES || total > MAX_SOURCE_BYTES || original_name.len() > MAX_SOURCE_NAME_BYTES diff --git a/crates/rustiq-core/src/persistence/data/portable/tests.rs b/crates/rustiq-core/src/persistence/data/portable/tests.rs index d7cec6bd..70eadf89 100644 --- a/crates/rustiq-core/src/persistence/data/portable/tests.rs +++ b/crates/rustiq-core/src/persistence/data/portable/tests.rs @@ -163,8 +163,8 @@ fn normalized_request_round_trip_keeps_auto_angstrom_defaults_and_requested_opti .prepare() .unwrap(); assert_eq!( - serde_json::to_value(RequestSnapshot::from_request(defaulted.request())).unwrap(), - serde_json::to_value(RequestSnapshot::from_request(explicit.request())).unwrap() + serde_json::to_value(RequestSnapshot::from_request(defaulted.request()).unwrap()).unwrap(), + serde_json::to_value(RequestSnapshot::from_request(explicit.request()).unwrap()).unwrap() ); let prepared = CalculationBuilder::new(&geometry, &basis) .with_basis_label("portable-basis-label") @@ -214,8 +214,8 @@ fn normalized_request_round_trip_keeps_auto_angstrom_defaults_and_requested_opti }; assert!(config.random.seed.is_some()); assert_eq!( - serde_json::to_value(RequestSnapshot::from_request(request)).unwrap(), - serde_json::to_value(RequestSnapshot::from_request(prepared.request())).unwrap() + serde_json::to_value(RequestSnapshot::from_request(request).unwrap()).unwrap(), + serde_json::to_value(RequestSnapshot::from_request(prepared.request()).unwrap()).unwrap() ); } @@ -830,7 +830,12 @@ fn settings_are_resolved_and_machine_policy_is_excluded() { restored.validate().unwrap(); assert_eq!(snapshot.identity(), restored.identity()); assert_eq!( - format!("{:?}", CalculationContext(restored).density_guess()), + format!( + "{:?}", + CalculationContext::from_snapshot(restored) + .unwrap() + .density_guess() + ), format!("{:?}", prepared.hf_config().guess.value) ); assert!(data.calculation().unwrap().atoms().nth(1).unwrap().1[0] > 1.0); @@ -854,3 +859,172 @@ fn unsupported_representation_is_inspectable_but_not_scientifically_usable() { assert!(restored.read_eri().is_err()); restored.write(dir.path().join("copy.rustiq")).unwrap(); } + +#[test] +fn rewriting_foreign_producer_updates_provenance_only() { + let dir = tempfile::tempdir().unwrap(); + let first = dir.path().join("first.rustiq"); + data().write(&first).unwrap(); + let mut contents = members(&first); + edit_json(&mut contents, "manifest.json", |m| { + m["producer"] = serde_json::json!({"name":"Python","version":"old"}); + }); + let foreign = dir.path().join("foreign.rustiq"); + write_members(&foreign, &contents); + let mut restored = RustiQData::open(&foreign).unwrap(); + assert_eq!(restored.producer(), ("Python", "old")); + let output = dir.path().join("rewritten.rustiq"); + restored.write(&output).unwrap(); + assert!(restored.ao_eri.is_none()); + let mut rewritten = RustiQData::open(&output).unwrap(); + assert_eq!(rewritten.producer(), ("RustiQ", env!("CARGO_PKG_VERSION"))); + assert_eq!( + rewritten.manifest.scientific_identity, + restored.manifest.scientific_identity + ); + assert_eq!(rewritten.manifest.artifacts, restored.manifest.artifacts); + assert!(rewritten.eri_is_compatible(&calculation())); + assert_eq!( + rewritten.read_eri().unwrap().ordered_values(), + &[0.5, 1.5, 2.5, 3.5, 4.5, 5.5] + ); + assert_eq!(members(&foreign), contents); + // Producer changes do not disturb reproducibility for the current writer. + assert_eq!(fs::read(&first).unwrap(), fs::read(&output).unwrap()); +} + +#[test] +fn independently_rounded_python_angstrom_conversion_is_accepted() { + let fixture = + include_str!("../../../../tests/data/persistence/portable-python-angstrom-v1.rustiq.hex"); + let bytes: Vec = fixture + .trim() + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("python.rustiq"); + fs::write(&path, bytes).unwrap(); + let mut restored = RustiQData::open(&path).unwrap(); + let requested = restored.request().unwrap().geometry().atoms[1].position[0]; + let resolved = restored.calculation().unwrap().atoms().nth(1).unwrap().1[0]; + // Python Decimal divided the exact binary64 request by the decimal V1 + // constant, rounding only the final result. It differs by one ULP from + // dividing by the rounded binary64 constant, so exact equality would fail. + assert_eq!(requested, 0.74); + assert_ne!(requested / 0.529_177_210_903, resolved); + assert_eq!( + (requested / 0.529_177_210_903) + .to_bits() + .abs_diff(resolved.to_bits()), + 1 + ); + let identity = restored.manifest.scientific_identity.clone(); + let rewritten = dir.path().join("rewritten.rustiq"); + restored.write(&rewritten).unwrap(); + assert_eq!( + RustiQData::open(&rewritten) + .unwrap() + .manifest + .scientific_identity, + identity + ); + let mut contents = members(&path); + edit_json(&mut contents, REQUEST_PATH, |r| { + r["atoms"][1]["position"][0] = (requested + 1e-10).into() + }); + let request = &contents.iter().find(|(n, _)| n == REQUEST_PATH).unwrap().1; + let (size, digest) = (request.len(), sha256(request).to_string()); + edit_json(&mut contents, "manifest.json", |m| { + m["request"]["size"] = size.into(); + m["request"]["digest"] = digest.into(); + }); + let invalid = dir.path().join("changed.rustiq"); + write_members(&invalid, &contents); + assert!(matches!( + RustiQData::open(&invalid), + Err(PortableError::InvalidRequest(_)) + )); +} + +#[test] +fn published_required_fields_match_v1_decoders() { + use serde_json::{json, Value}; + let cases = [ + ( + include_str!("../../../../../../schemas/request-snapshot-v1.schema.json"), + include_str!("../../../../tests/data/persistence/request-h2-v1.json"), + true, + vec![ + ("", ""), + ("/hf", "hf"), + ("/atoms/0", "atom"), + ("/mp2", "mp2"), + ], + ), + ( + include_str!("../../../../../../schemas/calculation-snapshot-v1.schema.json"), + include_str!("../../../../tests/data/persistence/calculation-h2-v1.json"), + false, + vec![ + ("", ""), + ("/hf", "hf"), + ("/atoms/0", "atom"), + ("/mp2", "mp2"), + ("/basis/0", "ao"), + ("/basis/0/components/0", "component"), + ("/basis/0/components/0/primitives/0", "primitive"), + ], + ), + ]; + for (schema, fixture, is_request, records) in cases { + let schema: Value = serde_json::from_str(schema).unwrap(); + let mut value: Value = serde_json::from_str(fixture).unwrap(); + value["mp2"] = if is_request { + json!({"frozen_orbitals":0,"memory_limit":{"kind":"auto"}}) + } else { + json!({"frozen_orbitals":0}) + }; + for (path, definition) in records { + let record_schema = if definition.is_empty() { + &schema + } else { + &schema["$defs"][definition] + }; + let required = record_schema["required"].as_array().unwrap(); + let properties = record_schema["properties"].as_object().unwrap(); + let decode = |candidate: Value| { + if is_request { + serde_json::from_value::(candidate).is_ok() + } else { + serde_json::from_value::(candidate).is_ok() + } + }; + for field in properties.keys() { + let mut candidate = value.clone(); + candidate + .pointer_mut(path) + .unwrap() + .as_object_mut() + .unwrap() + .remove(field); + assert_eq!( + decode(candidate), + !required.contains(&json!(field)), + "{is_request} {definition} {field}" + ); + } + let mut candidate = value.clone(); + candidate + .pointer_mut(path) + .unwrap() + .as_object_mut() + .unwrap() + .insert("unknown_v1_field".into(), json!(0)); + assert!(!decode(candidate), "unknown field in {definition}"); + } + } +} diff --git a/crates/rustiq-core/src/persistence/data/rustiq_data.rs b/crates/rustiq-core/src/persistence/data/rustiq_data.rs index 400a7c69..6bc98988 100644 --- a/crates/rustiq-core/src/persistence/data/rustiq_data.rs +++ b/crates/rustiq-core/src/persistence/data/rustiq_data.rs @@ -205,6 +205,12 @@ impl RustiQData { eri: Option<&CompactEri>, ) -> Result<(), PersistenceWriteError> { let mut manifest = self.manifest.clone(); + if manifest.kind == ManifestKind::Portable { + manifest.producer = Producer { + name: "RustiQ".into(), + version: env!("CARGO_PKG_VERSION").into(), + }; + } if let Some(context) = &self.context { let metadata = destination.write_json( @@ -222,7 +228,8 @@ impl RustiQData { if let Some(request) = &self.request { let metadata = destination.write_json( RelativePath::new(super::super::request::REQUEST_PATH), - &super::super::request::RequestSnapshot::from_request(request), + &super::super::request::RequestSnapshot::from_request(request) + .map_err(|error| ArtifactError::InvalidMetadata(error.to_string()))?, )?; manifest.request = Some(super::super::manifest::SnapshotManifest { path: super::super::request::REQUEST_PATH.into(), diff --git a/crates/rustiq-core/src/persistence/manifest.rs b/crates/rustiq-core/src/persistence/manifest.rs index 9af57a31..2eb7deb9 100644 --- a/crates/rustiq-core/src/persistence/manifest.rs +++ b/crates/rustiq-core/src/persistence/manifest.rs @@ -104,13 +104,40 @@ pub(crate) enum ArtifactAttributes { Unknown(BTreeMap), } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] +#[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct AoEriAttributes { pub basis_functions: usize, pub computation_version: u32, } +// Keep the folder-cache/domain API while defining a pointer-width-independent wire record. +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct AoEriAttributesV1 { + basis_functions: u64, + computation_version: u32, +} +impl Serialize for AoEriAttributes { + fn serialize(&self, serializer: S) -> Result { + AoEriAttributesV1 { + basis_functions: u64::try_from(self.basis_functions) + .map_err(serde::ser::Error::custom)?, + computation_version: self.computation_version, + } + .serialize(serializer) + } +} +impl<'de> Deserialize<'de> for AoEriAttributes { + fn deserialize>(deserializer: D) -> Result { + let wire = AoEriAttributesV1::deserialize(deserializer)?; + Ok(Self { + basis_functions: usize::try_from(wire.basis_functions) + .map_err(serde::de::Error::custom)?, + computation_version: wire.computation_version, + }) + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub(crate) struct ArtifactManifest { pub path: RelativePathBuf, @@ -342,3 +369,27 @@ mod tests { assert!(serde_json::from_str::(json).is_err()); } } + +#[cfg(test)] +mod wire_tests { + use super::*; + + #[test] + fn ao_dimension_wire_is_u64_with_checked_host_conversion() { + let encoded = format!( + r#"{{"basis_functions":{},"computation_version":1}}"#, + u64::MAX + ); + let result = serde_json::from_str::(&encoded); + assert_eq!(result.is_ok(), usize::try_from(u64::MAX).is_ok()); + if let Ok(attributes) = result { + assert_eq!(serde_json::to_string(&attributes).unwrap(), encoded); + } + for invalid in ["-1", "18446744073709551616"] { + assert!(serde_json::from_str::( + &encoded.replace("18446744073709551615", invalid) + ) + .is_err()); + } + } +} diff --git a/crates/rustiq-core/src/persistence/npy.rs b/crates/rustiq-core/src/persistence/npy.rs index 9d45647a..9d1fbd51 100644 --- a/crates/rustiq-core/src/persistence/npy.rs +++ b/crates/rustiq-core/src/persistence/npy.rs @@ -57,7 +57,7 @@ impl NpyConvert for CompactEri { } fn write_npy(&self, writer: impl Write) -> Result<(), NpyError> { - let shape = [self.len() as u64]; + let shape = [u64::try_from(self.len()).map_err(|_| NpyError::DimensionOverflow)?]; let mut writer = npyz::WriteOptions::new() .default_dtype() .shape(&shape) @@ -89,7 +89,7 @@ impl NpyConvert for DMatrix { } fn write_npy(&self, writer: impl Write) -> Result<(), NpyError> { - let shape = [self.nrows() as u64, self.ncols() as u64]; + let shape = Self::npy_shape((self.nrows(), self.ncols()))?; let mut writer = npyz::WriteOptions::new() .default_dtype() .order(npyz::Order::Fortran) @@ -134,7 +134,8 @@ pub(crate) fn checked_eri_prefix( reader .read_exact(&mut length[..length_bytes]) .map_err(NpyError::Read)?; - let header_len = u32::from_le_bytes(length) as usize; + let header_len = + usize::try_from(u32::from_le_bytes(length)).map_err(|_| NpyError::DimensionOverflow)?; if header_len > 65536 { return Err(NpyError::Read(std::io::Error::new( std::io::ErrorKind::InvalidData, diff --git a/crates/rustiq-core/src/persistence/request.rs b/crates/rustiq-core/src/persistence/request.rs index cfcd4b66..6c478a52 100644 --- a/crates/rustiq-core/src/persistence/request.rs +++ b/crates/rustiq-core/src/persistence/request.rs @@ -56,19 +56,19 @@ enum RequestedMethod { #[serde(deny_unknown_fields)] struct RequestedHf { method: RequestedMethod, - max_iterations: usize, + max_iterations: u64, convergence_threshold: f64, linear_dependency_threshold: f64, eri_schwarz_threshold: Option, diis: bool, - diis_size: usize, + diis_size: u64, guess: Guess, } #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct RequestedMp2 { - frozen_orbitals: usize, + frozen_orbitals: u64, memory_limit: RequestedMemory, } @@ -80,9 +80,10 @@ enum RequestedMemory { } impl RequestSnapshot { - pub(crate) fn from_request(request: &CalculationRequest) -> Self { + pub(crate) fn from_request(request: &CalculationRequest) -> Result { + let invalid = || PortableError::InvalidRequest("integer exceeds V1 u64 range".into()); let hf = request.hf(); - Self { + Ok(Self { format: "rustiq-request".into(), version: 1, units: match request.molecule().units { @@ -107,24 +108,30 @@ impl RequestSnapshot { HfMethod::Rhf => RequestedMethod::Rhf, HfMethod::Uhf => RequestedMethod::Uhf, }, - max_iterations: hf.max_iterations.get(), + max_iterations: u64::try_from(hf.max_iterations.get()).map_err(|_| invalid())?, convergence_threshold: hf.convergence_threshold.into_inner(), linear_dependency_threshold: hf.linear_dependency_threshold.value.into_inner(), eri_schwarz_threshold: hf.eri_schwarz_threshold.map(|v| v.into_inner()), diis: hf.diis, - diis_size: hf.diis_size.into_inner(), + diis_size: u64::try_from(hf.diis_size.into_inner()).map_err(|_| invalid())?, guess: hf.guess.value.into(), }, - mp2: request.mp2().map(|mp2| RequestedMp2 { - frozen_orbitals: mp2.frozen_orbitals.value, - memory_limit: match mp2.memory_limit.value { - MemoryLimit::Auto => RequestedMemory::Auto, - MemoryLimit::Fixed(bytes) => RequestedMemory::Fixed { - bytes: bytes.as_u64(), - }, - }, - }), - } + mp2: request + .mp2() + .map(|mp2| { + Ok::<_, PortableError>(RequestedMp2 { + frozen_orbitals: u64::try_from(mp2.frozen_orbitals.value) + .map_err(|_| invalid())?, + memory_limit: match mp2.memory_limit.value { + MemoryLimit::Auto => RequestedMemory::Auto, + MemoryLimit::Fixed(bytes) => RequestedMemory::Fixed { + bytes: bytes.as_u64(), + }, + }, + }) + }) + .transpose()?, + }) } pub(crate) fn to_request(&self) -> Result { @@ -161,7 +168,10 @@ impl RequestSnapshot { RequestedMethod::Uhf => HfMethod::Uhf, } .into(), - max_iterations: NonZeroUsize::new(self.hf.max_iterations).ok_or_else(invalid)?, + max_iterations: NonZeroUsize::new( + usize::try_from(self.hf.max_iterations).map_err(|_| invalid())?, + ) + .ok_or_else(invalid)?, convergence_threshold: PositiveFiniteF64::try_new(self.hf.convergence_threshold) .map_err(|_| invalid())?, linear_dependency_threshold: NonNegativeFiniteF64::try_new( @@ -177,7 +187,10 @@ impl RequestSnapshot { .map_err(|_| invalid())?, guess: self.hf.guess.to_config().into(), diis: self.hf.diis, - diis_size: DiisSize::try_new(self.hf.diis_size).map_err(|_| invalid())?, + diis_size: DiisSize::try_new( + usize::try_from(self.hf.diis_size).map_err(|_| invalid())?, + ) + .map_err(|_| invalid())?, }; let request = CalculationRequest { geometry: Geometry::new(String::new(), atoms), @@ -193,16 +206,24 @@ impl RequestSnapshot { }, basis_name: self.basis.clone(), hf, - mp2: self.mp2.as_ref().map(|mp2| Mp2Config { - frozen_orbitals: mp2.frozen_orbitals.into(), - memory_limit: match mp2.memory_limit { - RequestedMemory::Auto => MemoryLimit::Auto, - RequestedMemory::Fixed { bytes } => { - MemoryLimit::Fixed(bytesize::ByteSize(bytes)) - } - } - .into(), - }), + mp2: self + .mp2 + .as_ref() + .map(|mp2| { + Ok::<_, PortableError>(Mp2Config { + frozen_orbitals: usize::try_from(mp2.frozen_orbitals) + .map_err(|_| invalid())? + .into(), + memory_limit: match mp2.memory_limit { + RequestedMemory::Auto => MemoryLimit::Auto, + RequestedMemory::Fixed { bytes } => { + MemoryLimit::Fixed(bytesize::ByteSize(bytes)) + } + } + .into(), + }) + }) + .transpose()?, }; let molecule = request .molecule() @@ -215,3 +236,85 @@ impl RequestSnapshot { Ok(request) } } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::{json, Value}; + + fn golden() -> Value { + serde_json::from_str(include_str!( + "../../tests/data/persistence/request-h2-v1.json" + )) + .unwrap() + } + + #[test] + fn schema_and_decoder_require_diis_size_even_when_disabled() { + let schema: Value = serde_json::from_str(include_str!( + "../../../../schemas/request-snapshot-v1.schema.json" + )) + .unwrap(); + assert!(schema["$defs"]["hf"]["required"] + .as_array() + .unwrap() + .contains(&json!("diis_size"))); + for enabled in [false, true] { + let mut value = golden(); + value["hf"]["diis"] = json!(enabled); + value["hf"]["diis_size"] = json!(17); + let snapshot: RequestSnapshot = serde_json::from_value(value.clone()).unwrap(); + let request = snapshot.to_request().unwrap(); + assert_eq!(request.hf().diis, enabled); + assert_eq!(request.hf().diis_size.into_inner(), 17); + assert_eq!( + serde_json::to_value(RequestSnapshot::from_request(&request).unwrap()).unwrap() + ["hf"]["diis_size"], + json!(17) + ); + value["hf"].as_object_mut().unwrap().remove("diis_size"); + assert!(serde_json::from_value::(value).is_err()); + } + } + + #[test] + fn unsigned_wire_limits_and_checked_domain_conversions() { + let schema: Value = serde_json::from_str(include_str!( + "../../../../schemas/request-snapshot-v1.schema.json" + )) + .unwrap(); + for (definition, field) in [ + ("hf", "max_iterations"), + ("hf", "diis_size"), + ("mp2", "frozen_orbitals"), + ] { + assert_eq!( + schema["$defs"][definition]["properties"][field]["maximum"], + json!(u64::MAX) + ); + let mut value = golden(); + value["mp2"] = json!({"frozen_orbitals":0,"memory_limit":{"kind":"auto"}}); + value[definition][field] = json!(u64::MAX); + let snapshot: RequestSnapshot = serde_json::from_value(value.clone()).unwrap(); + let decoded = snapshot.to_request(); + assert_eq!(decoded.is_ok(), usize::try_from(u64::MAX).is_ok()); + if let Ok(request) = decoded { + assert_eq!( + serde_json::to_value(RequestSnapshot::from_request(&request).unwrap()).unwrap() + [definition][field], + json!(u64::MAX) + ); + } + for invalid in ["-1", "18446744073709551616"] { + let encoded = serde_json::to_string(&value) + .unwrap() + .replace("18446744073709551615", invalid); + assert!(serde_json::from_str::(&encoded).is_err()); + } + // A 32-bit reader must reject this valid V1 integer, not truncate it. + value[definition][field] = json!(u64::from(u32::MAX) + 1); + let snapshot: RequestSnapshot = serde_json::from_value(value).unwrap(); + assert_eq!(snapshot.to_request().is_ok(), usize::BITS > 32); + } + } +} diff --git a/crates/rustiq-core/src/persistence/storage.rs b/crates/rustiq-core/src/persistence/storage.rs index e8de4f11..ffabaa27 100644 --- a/crates/rustiq-core/src/persistence/storage.rs +++ b/crates/rustiq-core/src/persistence/storage.rs @@ -168,7 +168,7 @@ impl Storage { .take(max_size + 1) .read_to_end(&mut bytes) .map_err(StorageError::from)?; - if bytes.len() as u64 > max_size { + if u64::try_from(bytes.len()).map_err(|_| ManifestError::TooLarge)? > max_size { return Err(ManifestError::TooLarge); } serde_json::from_slice(&bytes).map_err(ManifestError::from) @@ -187,7 +187,7 @@ impl Storage { super::request::REQUEST_PATH => super::request::MAX_REQUEST_BYTES, _ => 1024 * 1024, }; - if bytes.len() as u64 > limit { + if u64::try_from(bytes.len()).map_err(|_| ManifestError::TooLarge)? > limit { return Err(ManifestError::TooLarge); } self.write_artifact::(path, |writer| { diff --git a/crates/rustiq-core/src/persistence/storage/zip.rs b/crates/rustiq-core/src/persistence/storage/zip.rs index 25742b06..9e1cd95b 100644 --- a/crates/rustiq-core/src/persistence/storage/zip.rs +++ b/crates/rustiq-core/src/persistence/storage/zip.rs @@ -85,7 +85,8 @@ fn preflight(file: &mut File) -> Result<(usize, u64), StorageError> { let length = file.metadata()?.len(); let tail_len = length.min(22 + u64::from(u16::MAX)); file.seek(SeekFrom::Start(length - tail_len))?; - let mut tail = vec![0; tail_len as usize]; + let mut tail = + vec![0; usize::try_from(tail_len).map_err(|_| invalid("tail size exceeds host range"))?]; file.read_exact(&mut tail)?; let end = (0..tail.len().saturating_sub(21)) .rev() @@ -95,7 +96,8 @@ fn preflight(file: &mut File) -> Result<(usize, u64), StorageError> { }) .ok_or_else(|| invalid("missing or truncated ZIP end record"))?; let eocd = &tail[end..]; - let end_offset = length - tail_len + end as u64; + let end_offset = length - tail_len + + u64::try_from(end).map_err(|_| invalid("end offset exceeds V1 range"))?; if u16_at(eocd, 4) != 0 || u16_at(eocd, 6) != 0 || u16_at(eocd, 8) != u16_at(eocd, 10) { return Err(invalid("multi-disk archives are unsupported")); } @@ -158,14 +160,15 @@ fn preflight(file: &mut File) -> Result<(usize, u64), StorageError> { { return Err(invalid("missing ZIP64 end record")); } - if count > MAX_MEMBERS as u64 + if count > u64::try_from(MAX_MEMBERS).map_err(|_| invalid("member limit exceeds V1 range"))? || size > MAX_DIRECTORY_BYTES || offset.checked_add(size) != Some(directory_end) { return Err(invalid("oversized or inconsistent ZIP directory")); } file.seek(SeekFrom::Start(offset))?; - let mut directory = vec![0; size as usize]; + let mut directory = + vec![0; usize::try_from(size).map_err(|_| invalid("directory size exceeds host range"))?]; file.read_exact(&mut directory)?; let mut cursor = 0_usize; let mut paths = BTreeSet::new(); @@ -242,7 +245,7 @@ fn preflight(file: &mut File) -> Result<(usize, u64), StorageError> { || u16_at(&local, 8) != method || local_name_len != name_len || local_offset - .checked_add(30 + name_len as u64 + local_extra_len) + .checked_add(30 + u64::from(u16_at(header, 28)) + local_extra_len) .is_none_or(|end| end > offset) { return Err(invalid("inconsistent local header")); @@ -252,7 +255,11 @@ fn preflight(file: &mut File) -> Result<(usize, u64), StorageError> { if local_name != name_bytes { return Err(invalid("local/central member names differ")); } - let mut local_extra = vec![0; local_extra_len as usize]; + let mut local_extra = vec![ + 0; + usize::try_from(local_extra_len) + .map_err(|_| invalid("extra field exceeds host range"))? + ]; file.read_exact(&mut local_extra)?; validate_extras(&local_extra)?; cursor = next; @@ -260,7 +267,10 @@ fn preflight(file: &mut File) -> Result<(usize, u64), StorageError> { if cursor != directory.len() { return Err(invalid("unindexed central-directory records")); } - Ok((count as usize, offset)) + Ok(( + usize::try_from(count).map_err(|_| invalid("member count exceeds host range"))?, + offset, + )) } fn validate_extras(mut extra: &[u8]) -> Result<(), StorageError> { let mut kinds = BTreeSet::new(); diff --git a/crates/rustiq-core/tests/data/persistence/portable-python-angstrom-v1.rustiq.hex b/crates/rustiq-core/tests/data/persistence/portable-python-angstrom-v1.rustiq.hex new file mode 100644 index 00000000..3c1e9b84 --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/portable-python-angstrom-v1.rustiq.hex @@ -0,0 +1 @@ +504b030414000000080000002100039cbc2966010000ad0200000d0000006d616e69666573742e6a736f6e8d914d6edc300c85f7730ac3ebb69068fd669d03f4060145511db51e7b22cb45a7c1dcbdf24c0a07290214da488f8fe447eae5d0757d9acb096bffd0f5655d6a7efe7ce6b2e4a5f244dc7fda1d4f3f377d9e9a53dee41f798a5bda792e15c3f86a3e9739aec4a5455edabb29139e78f37dbdd4e33c758f4cf9846397a7ca656ecd30e431d74b97f2afba967b9996b6b7eb65dfa4ebadfc4299a79a53a6a71cb75bbdec9dde113625e66fbcdc865b8e08da3ca841876483550e4414c1d0c06823468d1a8414c86248682c810e3088280715da61e224140c817792c2cfebbdf66bf733d6e36d8df7c097ef4b43f96798bf644bfebd6dc5d8e12354f03a45024b3638214142a0a41ca38f6d00ef935682d020a8864dd106159db50a74730aa7fd1b54c291d611eb9de01dee9be0ff214bafcd47cc3ea2f3e878a0e8a1d1920150469b4418bc2308883e38655cd4522101c7046d8220b5060f8276662ced9791eab2115f0fd7c31f504b030414000000080000002100db07d4a038010000a10200000c000000726571756573742e6a736f6ea5524d4fc3300cbdef575439b753bb2140fc018e1ce0364d55d67aad453e3ac701b669ff9d34cd3684342e5c92f8f9c5f67bc9719665626b494b164f9920ef187705c1ce8363918fd90f2087d6847415636f90dd4896a6734c564f34c9568ff02a0459768c6b82b1a98dd71ba0738d9819ac439e0aaf129665e5bccc6f06e9bc8efb29ff67a387bb6bf1a29c57cb3f7a85751d5536bda40e42a53899d05e310e0a1be4fdc5a08d74180d7a7d7b2996cf933dfd3620d3b04203f7b68d167ab6224fa8fcaa9181e438eb78bf2a937cd158135ea103d340cd3d81ebad1aef5750948f89a3d080a4ba85014c1b98fb9b4c20ac5dd37f4a3afce6548bc469115dedf0304abd4f5817fe84bba808c03b9aa8a2b104752f352ab606a51157eb81d8d34626fb8d57eae7e3c53601df4ae5609660a187c5997b9a7d03504b0304140000000800000021002bb22568fd010000a40700001000000063616c63756c6174696f6e2e6a736f6eed55c972a33010bdfb2b28ced8a58535bf927251b22c1bd5208991442689cbff3e2dc138d81367d6633880ba79bdbd6ee8d32a49d283b18af9f42149ede8bcfcbae6ace763cfbc343acd02e24958178487044779d4d2bb60b0339d9d20cc1b15548f2024c929de67b5e4ad1ed54ed81ff6f1cd609cf493d3c75997246883b2bbc27cdec6e739fba74078439b9a3615a584108acaa67e0bb5461b92176599d3a62408d302e10ff280fb3632c03b668f02a2c4ac5335f65e0ebde4d2bf5c88db3127df61890bed63d67fc8c4a5466ed460347871574e4e975360481fa1a9b6554601705457c8e838fb505c48dbe5ab74b05201c34fc2fde4f2742501563c4f7902926e725290023538cf6e61dc88c301a89b90d08faa6c684e8b02550d2a7073853f67bf1d136d4a421b4c2bb26cf8bda06555d3ba2e70899abc82c3df47c525981739fa75a5a8a67945cb0a634831c7b7a52e9bb0bad5beff65bc335cff63fa3f672f5c9fb377e76fd81dc0e59458aa84efcc3e2e185067b3923db7120633ae99d03d8ce6ae435e1a36ce51682e5adf59e13ad307732cd668a62eeda51630507b3108bd07e4cb5da4b0b275bcfbc6eceb2d069319b397d2b54ebe86dfb71efb7e561f47e1dca50e507c913ad6c18d156dc794ecbdd192e9f46ddb08eb47bb63f3c609ce2ecc9ca7c5309045949499366418bea2d147b376b16e57e7d577504b0102140314000000080000002100039cbc2966010000ad0200000d0000000000000000000000a481000000006d616e69666573742e6a736f6e504b0102140314000000080000002100db07d4a038010000a10200000c0000000000000000000000a48191010000726571756573742e6a736f6e504b01021403140000000800000021002bb22568fd010000a4070000100000000000000000000000a481f302000063616c63756c6174696f6e2e6a736f6e504b05060000000003000300b30000001e0500000000 diff --git a/docs/persistence-format-v1.md b/docs/persistence-format-v1.md index 0a46ca33..435d0359 100644 --- a/docs/persistence-format-v1.md +++ b/docs/persistence-format-v1.md @@ -280,8 +280,9 @@ identify the independent wire contract. Reopening reconstructs a typed The request preserves ordered atoms in the requested `bohr`/`angstrom` units, charge, multiplicity, a portable basis label, requested `auto`/`rhf`/`uhf` method, -defaulted HF settings, and optional MP2 settings. DIIS enablement and history size -remain distinct even when disabled. The requested MP2 memory limit is tagged +defaulted HF settings, and optional MP2 settings. The requested history size +`diis_size` is required and remains present even when DIIS is disabled. +`diis` and `diis_size` are independent requested values. The requested MP2 memory limit is tagged `{"kind":"auto"}` or `{"kind":"fixed","bytes":...}`; no machine-resolved memory budget is stored. Random seeds omitted in the request remain absent even when preparation chooses a seed for the resolved calculation. Comments, source @@ -307,7 +308,16 @@ use. `request.json` and `calculation.json` are required independently of sources The reader validates request semantics and their consistency with the resolved molecule, method, scientific settings, and MP2 selection. Requested Angstrom -coordinates resolve to Bohr and `auto` resolves to RHF/UHF; a missing requested +coordinates are checked using the V1 constant `0.529177210903` Angstrom per +Bohr, rounded to binary64, and binary64 division. A resolved coordinate may differ +from that result by at most four adjacent binary64 values (ULPs), allowing final +rounding differences between independent implementations. This includes at most +four subnormal steps near zero; there is no absolute chemistry-level tolerance. +Both values and the converted result must be finite. Bohr requests compare +numerically exactly, treating signed zeros as equal. This V1 rule is independent +of domain conversion constants or dependency updates. `calculation.json` remains +authoritative for resolved coordinates and scientific identity; validation never +replaces or rounds them. `auto` resolves to RHF/UHF; a missing requested random seed may resolve to a generated seed. Neither snapshot substitutes for the other. @@ -317,7 +327,10 @@ The normative structural schema is [`calculation-snapshot-v1.schema.json`](../schemas/calculation-snapshot-v1.schema.json). The [H2 golden snapshot](../crates/rustiq-core/tests/data/persistence/calculation-h2-v1.json) provides a complete example. Wire records are explicitly defined, independently -of domain Rust struct layouts and package versions. +of domain Rust struct layouts and package versions. Iteration limits, DIIS sizes, +and frozen-orbital counts use unsigned 64-bit JSON integers (maximum +18446744073709551615). Domain conversions are checked in both directions; +a reader rejects values exceeding its host integer range. - `format: "rustiq-calculation"`, `version: 1`, `units: "bohr"` identify the snapshot contract and coordinate unit. @@ -407,6 +420,10 @@ calculation without loading arrays. Changing an HF-only request to MP2 does not invalidate AO ERIs. Compatibility does not imply that the payload has already been verified. There is no CLI reuse orchestration in this API release. +Every new archive written by RustiQ records producer name `RustiQ` and the +current package version, including when rewriting an older or foreign archive. +Producer provenance does not change scientific identity. + Writing a reopened archive preserves both semantic snapshots and exact captured source bytes. It preserves unloaded artifacts by verified streaming copy, including unknown representations and their attributes. Known ERI headers @@ -425,9 +442,10 @@ directory after publication. Failures before publication leave the destination untouched and clean up the temporary file. A directory-sync error after publication can leave a complete output file and is still reported as an error. -Writers use `Stored` for numerical/opaque artifacts and `Deflated` for the two -JSON members. Streaming payloads reserve ZIP64 local-header fields even for -small arrays, allowing growth past 4 GiB without buffering. ZIP64 central/end +Writers use `Stored` for numerical/opaque artifacts and `Deflated` for all three +JSON members (`manifest.json`, `request.json`, and `calculation.json`). Streaming +payloads reserve ZIP64 local-header fields even for small arrays, allowing growth +past 4 GiB without buffering. ZIP64 central/end records are emitted when required. Timestamps are fixed to 1980-01-01 00:00:00, permissions to regular files with mode 0644, and member order is deterministic. Repeated writes of the same data by the same implementation produce identical diff --git a/schemas/calculation-snapshot-v1.schema.json b/schemas/calculation-snapshot-v1.schema.json index 5bacf66e..5b15925f 100644 --- a/schemas/calculation-snapshot-v1.schema.json +++ b/schemas/calculation-snapshot-v1.schema.json @@ -325,7 +325,8 @@ }, "max_iterations": { "type": "integer", - "minimum": 1 + "minimum": 1, + "maximum": 18446744073709551615 }, "convergence_threshold": { "type": "number", @@ -350,7 +351,8 @@ "anyOf": [ { "type": "integer", - "minimum": 2 + "minimum": 2, + "maximum": 18446744073709551615 }, { "type": "null" @@ -371,7 +373,8 @@ "properties": { "frozen_orbitals": { "type": "integer", - "minimum": 0 + "minimum": 0, + "maximum": 18446744073709551615 } } } diff --git a/schemas/request-snapshot-v1.schema.json b/schemas/request-snapshot-v1.schema.json index 63f81766..6a6c291a 100644 --- a/schemas/request-snapshot-v1.schema.json +++ b/schemas/request-snapshot-v1.schema.json @@ -239,7 +239,8 @@ "convergence_threshold", "linear_dependency_threshold", "guess", - "diis" + "diis", + "diis_size" ], "properties": { "method": { @@ -251,7 +252,8 @@ }, "max_iterations": { "type": "integer", - "minimum": 1 + "minimum": 1, + "maximum": 18446744073709551615 }, "convergence_threshold": { "type": "number", @@ -274,7 +276,8 @@ }, "diis_size": { "type": "integer", - "minimum": 2 + "minimum": 2, + "maximum": 18446744073709551615 }, "guess": { "$ref": "#/$defs/guess" @@ -294,7 +297,8 @@ "properties": { "frozen_orbitals": { "type": "integer", - "minimum": 0 + "minimum": 0, + "maximum": 18446744073709551615 }, "memory_limit": { "oneOf": [ diff --git a/tools/reference/generate_portable_fixtures.py b/tools/reference/generate_portable_fixtures.py index 5a56f3a4..b98e7759 100644 --- a/tools/reference/generate_portable_fixtures.py +++ b/tools/reference/generate_portable_fixtures.py @@ -139,5 +139,61 @@ def generate() -> None: ) +def generate_angstrom() -> None: + """Produce coordinates independently, rounding only after Decimal division.""" + from decimal import Decimal, localcontext + + request = json.loads((FIXTURES / "request-h2-v1.json").read_bytes()) + snapshot = json.loads((FIXTURES / "calculation-h2-v1.json").read_bytes()) + request["units"] = "angstrom" + request["atoms"][1]["position"] = [0.74, -0.13, 0.0] + with localcontext() as context: + context.prec = 100 + coordinates = [ + float(Decimal.from_float(x) / Decimal("0.529177210903")) + for x in request["atoms"][1]["position"] + ] + snapshot["atoms"][1]["position"] = coordinates + snapshot["basis"][1]["center"] = coordinates + request_bytes = json.dumps(request, indent=2).encode() + b"\n" + snapshot_bytes = json.dumps(snapshot, indent=2).encode() + b"\n" + manifest = { + "format": "rustiq-persistence", + "format_version": 1, + "kind": "portable", + "producer": {"name": "Python Decimal interoperability fixture", "version": "1"}, + "scientific_identity": {"version": 1, "digest": scientific_identity(snapshot)}, + "request": { + "path": "request.json", + "version": 1, + "size": len(request_bytes), + "digest": digest(request_bytes), + }, + "calculation": { + "path": "calculation.json", + "version": 1, + "size": len(snapshot_bytes), + "digest": digest(snapshot_bytes), + }, + "artifacts": {}, + } + output = io.BytesIO() + with zipfile.ZipFile(output, "w", allowZip64=True) as archive: + for name, content in [ + ("manifest.json", json.dumps(manifest, indent=2).encode() + b"\n"), + ("request.json", request_bytes), + ("calculation.json", snapshot_bytes), + ]: + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = 0o100644 << 16 + info.compress_type = zipfile.ZIP_DEFLATED + archive.writestr(info, content) + (FIXTURES / "portable-python-angstrom-v1.rustiq.hex").write_text( + output.getvalue().hex() + "\n", encoding="ascii" + ) + + if __name__ == "__main__": generate() + generate_angstrom() diff --git a/tools/reference/test_persistence_interop.py b/tools/reference/test_persistence_interop.py index 76ef919a..16e9fb2c 100644 --- a/tools/reference/test_persistence_interop.py +++ b/tools/reference/test_persistence_interop.py @@ -154,3 +154,39 @@ def test_python_portable_golden_fixtures() -> None: payload = archive.read("arrays/integrals/ao-eri.npy") assert digest(payload) == manifest["artifacts"]["ao_eri"]["digest"] _assert_ao_eri_values(np.load(io.BytesIO(payload), allow_pickle=False)) + + +def test_python_angstrom_conversion_fixture() -> None: + import json + import zipfile + from decimal import Decimal, localcontext + + from generate_portable_fixtures import digest, scientific_identity + + content = bytes.fromhex( + (FIXTURE_DIR / "portable-python-angstrom-v1.rustiq.hex").read_text() + ) + with zipfile.ZipFile(io.BytesIO(content)) as archive: + manifest = json.loads(archive.read("manifest.json")) + request_bytes = archive.read("request.json") + snapshot_bytes = archive.read("calculation.json") + request = json.loads(request_bytes) + snapshot = json.loads(snapshot_bytes) + assert request["units"] == "angstrom" + assert snapshot["units"] == "bohr" + assert digest(request_bytes) == manifest["request"]["digest"] + assert digest(snapshot_bytes) == manifest["calculation"]["digest"] + assert ( + scientific_identity(snapshot) == manifest["scientific_identity"]["digest"] + ) + with localcontext() as context: + context.prec = 100 + for requested, resolved in zip( + request["atoms"][1]["position"], + snapshot["atoms"][1]["position"], + strict=True, + ): + assert resolved == float( + Decimal.from_float(requested) / Decimal("0.529177210903") + ) + assert snapshot["atoms"][1]["position"][0] != 0.74 / 0.529177210903 From 844eff37b9c2f7531b9ab2885df523407a6fef24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sat, 3 Oct 2026 13:40:14 +0200 Subject: [PATCH 4/8] Freeze V1 HF method resolution --- .../src/persistence/calculation.rs | 82 +++++++++++++++++-- crates/rustiq-core/src/persistence/request.rs | 11 +-- docs/persistence-format-v1.md | 9 +- 3 files changed, 88 insertions(+), 14 deletions(-) diff --git a/crates/rustiq-core/src/persistence/calculation.rs b/crates/rustiq-core/src/persistence/calculation.rs index d6cc646b..307ee922 100644 --- a/crates/rustiq-core/src/persistence/calculation.rs +++ b/crates/rustiq-core/src/persistence/calculation.rs @@ -8,8 +8,9 @@ use crate::{ calculation::PreparedCalculation, config::{ random_config::{DistributionConfig, RandomConfig}, - DensityGuessConfig, ResolvedHfMethod, + DensityGuessConfig, HfMethod, ResolvedHfMethod, }, + molecules::molecule::Molecule, }; pub(crate) const CALCULATION_PATH: &str = "calculation.json"; @@ -362,10 +363,8 @@ impl Snapshot { .molecule() .build(request.geometry().clone()) .map_err(|_| invalid())?; - let method = request - .hf() - .resolve_method(&molecule) - .map_err(|_| invalid())?; + let method = + resolve_hf_method_v1(request.hf().method.value, &molecule).ok_or_else(invalid)?; let hf = request.hf(); if molecule.charge() != self.charge || molecule.multiplicity().get() != self.multiplicity @@ -443,6 +442,25 @@ impl Snapshot { bytes.finish_ao_eri(self.hf.eri_schwarz_threshold) } } +/// Resolves the HF request according to the frozen V1 persistence contract. +/// +/// V1 predates any future method that may become eligible for `Auto`, so archive +/// validation must not inherit changes to the current domain-level resolver. +pub(super) fn resolve_hf_method_v1( + method: HfMethod, + molecule: &Molecule, +) -> Option { + let closed_shell_singlet = + molecule.multiplicity().get() == 1 && molecule.total_electrons().is_multiple_of(2); + match method { + HfMethod::Rhf if closed_shell_singlet => Some(ResolvedHfMethod::Rhf), + HfMethod::Rhf => None, + HfMethod::Uhf => Some(ResolvedHfMethod::Uhf), + HfMethod::Auto if closed_shell_singlet => Some(ResolvedHfMethod::Rhf), + HfMethod::Auto => Some(ResolvedHfMethod::Uhf), + } +} + // V1 pins this decimal value (CODATA 2018 Bohr radius in Angstrom), rather // than inheriting a future physical_constants release. Parse as binary64 and divide. const V1_BOHR_IN_ANGSTROM: f64 = 0.529_177_210_903; @@ -638,8 +656,60 @@ mod tests { #[cfg(test)] mod v1_contract_tests { use super::*; - use crate::molecules::units::Units; + use crate::molecules::{atom::Atom, geometry::Geometry, units::Units}; + use nalgebra::point; use serde_json::{json, Value}; + use std::num::NonZeroU8; + + fn molecule(symbols: &[&str], multiplicity: u8) -> Molecule { + let elements = periodic_table::periodic_table(); + let atoms = symbols + .iter() + .enumerate() + .map(|(index, symbol)| { + let element = elements + .iter() + .find(|element| element.symbol == *symbol) + .unwrap(); + Atom::new(element, point![0.0, 0.0, index as f64]) + }) + .collect(); + Molecule::try_new( + Geometry::new("v1-method-resolution".into(), atoms), + Units::Bohr, + 0, + NonZeroU8::new(multiplicity).unwrap(), + ) + .unwrap() + } + + #[test] + fn hf_method_resolution_is_frozen_for_v1() { + let closed_shell = molecule(&["H", "H"], 1); + let open_shell = molecule(&["H"], 2); + + assert_eq!( + resolve_hf_method_v1(HfMethod::Auto, &closed_shell), + Some(ResolvedHfMethod::Rhf) + ); + assert_eq!( + resolve_hf_method_v1(HfMethod::Auto, &open_shell), + Some(ResolvedHfMethod::Uhf) + ); + assert_eq!( + resolve_hf_method_v1(HfMethod::Rhf, &closed_shell), + Some(ResolvedHfMethod::Rhf) + ); + assert_eq!(resolve_hf_method_v1(HfMethod::Rhf, &open_shell), None); + assert_eq!( + resolve_hf_method_v1(HfMethod::Uhf, &closed_shell), + Some(ResolvedHfMethod::Uhf) + ); + assert_eq!( + resolve_hf_method_v1(HfMethod::Uhf, &open_shell), + Some(ResolvedHfMethod::Uhf) + ); + } #[test] fn conversion_comparison_is_bounded_in_binary64_steps() { diff --git a/crates/rustiq-core/src/persistence/request.rs b/crates/rustiq-core/src/persistence/request.rs index 6c478a52..051d5852 100644 --- a/crates/rustiq-core/src/persistence/request.rs +++ b/crates/rustiq-core/src/persistence/request.rs @@ -3,7 +3,10 @@ use std::num::{NonZeroU8, NonZeroUsize}; use serde::{Deserialize, Serialize}; -use super::{calculation::Guess, PortableError}; +use super::{ + calculation::{resolve_hf_method_v1, Guess}, + PortableError, +}; use crate::{ calculation::CalculationRequest, config::{ @@ -229,10 +232,8 @@ impl RequestSnapshot { .molecule() .build(request.geometry().clone()) .map_err(|error| PortableError::InvalidRequest(error.to_string()))?; - request - .hf() - .resolve_method(&molecule) - .map_err(|error| PortableError::InvalidRequest(error.to_string()))?; + resolve_hf_method_v1(request.hf().method.value, &molecule) + .ok_or_else(|| PortableError::InvalidRequest("invalid V1 HF method resolution".into()))?; Ok(request) } } diff --git a/docs/persistence-format-v1.md b/docs/persistence-format-v1.md index 435d0359..5cf9a265 100644 --- a/docs/persistence-format-v1.md +++ b/docs/persistence-format-v1.md @@ -317,9 +317,12 @@ Both values and the converted result must be finite. Bohr requests compare numerically exactly, treating signed zeros as equal. This V1 rule is independent of domain conversion constants or dependency updates. `calculation.json` remains authoritative for resolved coordinates and scientific identity; validation never -replaces or rounds them. `auto` resolves to RHF/UHF; a missing requested -random seed may resolve to a generated seed. Neither snapshot substitutes for -the other. +replaces or rounds them. HF method resolution is also frozen by V1 rather than +delegated to the current RustiQ domain policy: `auto` resolves to RHF for a +closed-shell singlet (multiplicity 1 with an even electron count) and to UHF +otherwise; explicit RHF is valid only for a closed-shell singlet, while explicit +UHF remains UHF. A missing requested random seed may resolve to a generated seed. +Neither snapshot substitutes for the other. ### Resolved scientific snapshot From 4ab485819a561b48978b9cc27faa16eb66cbf489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sat, 3 Oct 2026 14:32:16 +0200 Subject: [PATCH 5/8] Require resolved random seeds in V1 --- .../src/persistence/calculation.rs | 180 ++++++++++++++++-- crates/rustiq-core/src/persistence/request.rs | 5 +- docs/persistence-format-v1.md | 13 +- schemas/calculation-snapshot-v1.schema.json | 14 +- 4 files changed, 178 insertions(+), 34 deletions(-) diff --git a/crates/rustiq-core/src/persistence/calculation.rs b/crates/rustiq-core/src/persistence/calculation.rs index 307ee922..593d7633 100644 --- a/crates/rustiq-core/src/persistence/calculation.rs +++ b/crates/rustiq-core/src/persistence/calculation.rs @@ -180,7 +180,7 @@ struct Hf { linear_dependency_threshold: f64, eri_schwarz_threshold: Option, diis_size: Option, - guess: Guess, + guess: ResolvedGuess, } #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -201,6 +201,24 @@ pub(super) struct Random { seed: Option, distribution: Distribution, } +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum ResolvedGuess { + CoreHamiltonian { + perturbation: Option, + }, + OneElectron { + perturbation: Option, + }, + Random { random: ResolvedRandom }, + Zero, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct ResolvedRandom { + seed: u64, + distribution: Distribution, +} #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] enum Distribution { @@ -268,7 +286,7 @@ impl Snapshot { .then(|| u64::try_from(hf.diis_size.into_inner())) .transpose() .map_err(invalid)?, - guess: hf.guess.value.into(), + guess: ResolvedGuess::from_config(hf.guess.value)?, }, mp2: mp2 .map(|mp2| { @@ -537,32 +555,121 @@ impl Random { } } } -impl Guess { - fn is_resolution_of(&self, requested: &Self) -> bool { - let random_matches = |resolved: &Random, requested: &Random| { +impl ResolvedRandom { + fn from_config(config: RandomConfig) -> Result { + let requested = Random::from(config); + let seed = requested.seed.ok_or_else(|| { + PortableError::InvalidCalculation( + "resolved random configuration is missing its seed".into(), + ) + })?; + Ok(Self { + seed, + distribution: requested.distribution, + }) + } + + fn to_config(&self) -> RandomConfig { + Random { + seed: Some(self.seed), + distribution: self.distribution.clone(), + } + .to_config() + } +} + +impl ResolvedGuess { + fn from_config(config: DensityGuessConfig) -> Result { + Ok(match config { + DensityGuessConfig::CoreHamiltonian { perturbation } => Self::CoreHamiltonian { + perturbation: perturbation + .map(|value| ResolvedRandom::from_config(value.random)) + .transpose()?, + }, + DensityGuessConfig::OneElectron { perturbation } => Self::OneElectron { + perturbation: perturbation + .map(|value| ResolvedRandom::from_config(value.random)) + .transpose()?, + }, + DensityGuessConfig::Random { config } => Self::Random { + random: ResolvedRandom::from_config(config.random)?, + }, + DensityGuessConfig::Zero => Self::Zero, + }) + } + + fn is_resolution_of(&self, requested: &Guess) -> bool { + let random_matches = |resolved: &ResolvedRandom, requested: &Random| { resolved.distribution == requested.distribution - && requested - .seed - .is_none_or(|seed| resolved.seed == Some(seed)) + && requested.seed.is_none_or(|seed| resolved.seed == seed) }; match (self, requested) { ( Self::CoreHamiltonian { perturbation: a }, - Self::CoreHamiltonian { perturbation: b }, + Guess::CoreHamiltonian { perturbation: b }, ) - | (Self::OneElectron { perturbation: a }, Self::OneElectron { perturbation: b }) => { - match (a, b) { - (None, None) => true, - (Some(a), Some(b)) => random_matches(a, b), - _ => false, + | ( + Self::OneElectron { perturbation: a }, + Guess::OneElectron { perturbation: b }, + ) => match (a, b) { + (None, None) => true, + (Some(a), Some(b)) => random_matches(a, b), + _ => false, + }, + (Self::Random { random: a }, Guess::Random { random: b }) => random_matches(a, b), + (Self::Zero, Guess::Zero) => true, + _ => false, + } + } + + fn validate(&self) -> Result<(), PortableError> { + let random = match self { + Self::CoreHamiltonian { perturbation } | Self::OneElectron { perturbation } => { + perturbation.as_ref() + } + Self::Random { random } => Some(random), + Self::Zero => None, + }; + if let Some(random) = random { + let valid = match random.distribution { + Distribution::Uniform { min, max } => { + min.is_finite() && max.is_finite() && min < max && (max - min).is_finite() } + Distribution::Normal { mean, std_dev } => mean.is_finite() && positive(std_dev), + }; + if !valid { + return Err(PortableError::InvalidCalculation( + "invalid random distribution".into(), + )); } - (Self::Random { random: a }, Self::Random { random: b }) => random_matches(a, b), - (Self::Zero, Self::Zero) => true, - _ => false, } + Ok(()) } + fn to_config(&self) -> DensityGuessConfig { + use crate::config::{GuessPerturbationConfig, RandomGuessConfig}; + match self { + Self::CoreHamiltonian { perturbation } => DensityGuessConfig::CoreHamiltonian { + perturbation: perturbation.as_ref().map(|value| GuessPerturbationConfig { + random: value.to_config(), + }), + }, + Self::OneElectron { perturbation } => DensityGuessConfig::OneElectron { + perturbation: perturbation.as_ref().map(|value| GuessPerturbationConfig { + random: value.to_config(), + }), + }, + Self::Random { random } => DensityGuessConfig::Random { + config: RandomGuessConfig { + random: random.to_config(), + }, + }, + Self::Zero => DensityGuessConfig::Zero, + } + } +} + +impl Guess { pub(super) fn validate(&self) -> Result<(), PortableError> { let random = match self { Guess::CoreHamiltonian { perturbation } | Guess::OneElectron { perturbation } => { @@ -711,6 +818,45 @@ mod v1_contract_tests { ); } + #[test] + fn resolved_random_seed_is_required_by_v1() { + let schema: Value = serde_json::from_str(include_str!( + "../../../../schemas/calculation-snapshot-v1.schema.json" + )) + .unwrap(); + assert!(schema["$defs"]["random"]["required"] + .as_array() + .unwrap() + .contains(&json!("seed"))); + assert_eq!( + schema["$defs"]["random"]["properties"]["seed"]["type"], + json!("integer") + ); + + let mut value: Value = serde_json::from_str(include_str!( + "../../tests/data/persistence/calculation-h2-v1.json" + )) + .unwrap(); + value["hf"]["guess"] = json!({ + "kind": "random", + "random": { + "seed": 42, + "distribution": {"kind": "uniform", "min": -1.0, "max": 1.0} + } + }); + let snapshot: Snapshot = serde_json::from_value(value.clone()).unwrap(); + snapshot.validate().unwrap(); + + value["hf"]["guess"]["random"] + .as_object_mut() + .unwrap() + .remove("seed"); + assert!(serde_json::from_value::(value.clone()).is_err()); + + value["hf"]["guess"]["random"]["seed"] = Value::Null; + assert!(serde_json::from_value::(value).is_err()); + } + #[test] fn conversion_comparison_is_bounded_in_binary64_steps() { for requested in [0.74, -0.74, 1e-200, -1e-200, 1e200] { diff --git a/crates/rustiq-core/src/persistence/request.rs b/crates/rustiq-core/src/persistence/request.rs index 051d5852..45585449 100644 --- a/crates/rustiq-core/src/persistence/request.rs +++ b/crates/rustiq-core/src/persistence/request.rs @@ -232,8 +232,9 @@ impl RequestSnapshot { .molecule() .build(request.geometry().clone()) .map_err(|error| PortableError::InvalidRequest(error.to_string()))?; - resolve_hf_method_v1(request.hf().method.value, &molecule) - .ok_or_else(|| PortableError::InvalidRequest("invalid V1 HF method resolution".into()))?; + resolve_hf_method_v1(request.hf().method.value, &molecule).ok_or_else(|| { + PortableError::InvalidRequest("invalid V1 HF method resolution".into()) + })?; Ok(request) } } diff --git a/docs/persistence-format-v1.md b/docs/persistence-format-v1.md index 5cf9a265..50f4dfe1 100644 --- a/docs/persistence-format-v1.md +++ b/docs/persistence-format-v1.md @@ -284,8 +284,10 @@ defaulted HF settings, and optional MP2 settings. The requested history size `diis_size` is required and remains present even when DIIS is disabled. `diis` and `diis_size` are independent requested values. The requested MP2 memory limit is tagged `{"kind":"auto"}` or `{"kind":"fixed","bytes":...}`; no machine-resolved -memory budget is stored. Random seeds omitted in the request remain absent even -when preparation chooses a seed for the resolved calculation. Comments, source +memory budget is stored. A random seed may remain unspecified in the request; +this deliberately asks preparation to choose a fresh seed. The unresolved request +keeps that state while `calculation.json` records the concrete seed actually used. +Comments, source spans, source paths, cache configuration, and output/UI settings are excluded. The basis label is informational; resolved AO contents define scientific identity. @@ -350,9 +352,10 @@ a reader rejects values exceeding its host integer range. guess. `null`/omitted Schwarz threshold means screening disabled; `null`/omitted DIIS size means DIIS disabled. - Density guesses are tagged as `core_hamiltonian`, `one_electron`, `random`, or - `zero`. Perturbations and random guesses record distribution parameters and an - resolved seed. Seeds chosen during preparation are captured here, while an - initially absent seed stays absent in `request.json`. This does not capture + `zero`. Perturbations and random guesses record distribution parameters and a + required concrete resolved seed. A missing or null seed is invalid in + `calculation.json`. Seeds chosen during preparation are captured here, while an + initially unspecified seed stays unspecified in `request.json`. This does not capture generator state or promise exact SCF continuation. - `mp2` is absent/null for HF-only requests, or contains `frozen_orbitals`. This records the request, not an MP2 result or proof of convergence. diff --git a/schemas/calculation-snapshot-v1.schema.json b/schemas/calculation-snapshot-v1.schema.json index 5b15925f..60bf843d 100644 --- a/schemas/calculation-snapshot-v1.schema.json +++ b/schemas/calculation-snapshot-v1.schema.json @@ -210,20 +210,14 @@ "type": "object", "additionalProperties": false, "required": [ + "seed", "distribution" ], "properties": { "seed": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 18446744073709551615 - }, - { - "type": "null" - } - ] + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615 }, "distribution": { "$ref": "#/$defs/distribution" From 42fc72c571e54623491debff4bc14e2490ffa0d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Mon, 5 Oct 2026 14:39:28 +0200 Subject: [PATCH 6/8] Finalize multi-calculation portable persistence V1 --- .../src/persistence/calculation.rs | 50 ++- crates/rustiq-core/src/persistence/data.rs | 3 + .../src/persistence/data/bundle.rs | 306 +++++++++++++++++ .../src/persistence/data/portable.rs | 165 +++++---- .../src/persistence/data/portable/tests.rs | 324 ++++++++++++++++-- .../src/persistence/data/rustiq_data.rs | 34 +- .../rustiq-core/src/persistence/manifest.rs | 23 ++ crates/rustiq-core/src/persistence/mod.rs | 2 +- .../rustiq-core/src/persistence/provenance.rs | 2 +- crates/rustiq-core/src/persistence/request.rs | 64 ++-- crates/rustiq-core/src/persistence/storage.rs | 38 +- .../src/persistence/storage/zip.rs | 2 +- .../persistence/portable-manifest-h2-v1.json | 51 +++ .../portable-python-angstrom-v1.rustiq.hex | 2 +- .../portable-python-big-v1.rustiq.hex | 2 +- .../portable-python-little-v1.rustiq.hex | 2 +- .../portable-python-multi-v1.rustiq.hex | 1 + crates/rustiq-core/tests/portable_artifact.rs | 78 +++++ docs/persistence-format-v1.md | 103 ++++-- schemas/portable-manifest-v1.schema.json | 229 +++++++++++++ tools/reference/generate_portable_fixtures.py | 78 ++++- tools/reference/test_persistence_interop.py | 129 +++++-- 22 files changed, 1472 insertions(+), 216 deletions(-) create mode 100644 crates/rustiq-core/src/persistence/data/bundle.rs create mode 100644 crates/rustiq-core/tests/data/persistence/portable-manifest-h2-v1.json create mode 100644 crates/rustiq-core/tests/data/persistence/portable-python-multi-v1.rustiq.hex create mode 100644 schemas/portable-manifest-v1.schema.json diff --git a/crates/rustiq-core/src/persistence/calculation.rs b/crates/rustiq-core/src/persistence/calculation.rs index 593d7633..2645982f 100644 --- a/crates/rustiq-core/src/persistence/calculation.rs +++ b/crates/rustiq-core/src/persistence/calculation.rs @@ -210,7 +210,9 @@ enum ResolvedGuess { OneElectron { perturbation: Option, }, - Random { random: ResolvedRandom }, + Random { + random: ResolvedRandom, + }, Zero, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -279,11 +281,20 @@ impl Snapshot { }, max_iterations: u64::try_from(hf.max_iterations.get()).map_err(invalid)?, convergence_threshold: hf.convergence_threshold.into_inner(), - linear_dependency_threshold: hf.linear_dependency_threshold.value.into_inner(), - eri_schwarz_threshold: hf.eri_schwarz_threshold.map(|value| value.into_inner()), + linear_dependency_threshold: hf + .orthogonalization + .linear_dependency_threshold + .value + .into_inner(), + eri_schwarz_threshold: calculation + .integral_config() + .schwarz_threshold + .value + .map(|value| value.into_inner()), diis_size: hf .diis - .then(|| u64::try_from(hf.diis_size.into_inner())) + .enabled + .then(|| u64::try_from(hf.diis.max_history.value.into_inner())) .transpose() .map_err(invalid)?, guess: ResolvedGuess::from_config(hf.guess.value)?, @@ -403,12 +414,22 @@ impl Snapshot { || u64::try_from(hf.max_iterations.get()).map_err(|_| invalid())? != self.hf.max_iterations || hf.convergence_threshold.into_inner() != self.hf.convergence_threshold - || hf.linear_dependency_threshold.value.into_inner() + || hf + .orthogonalization + .linear_dependency_threshold + .value + .into_inner() != self.hf.linear_dependency_threshold - || hf.eri_schwarz_threshold.map(|v| v.into_inner()) != self.hf.eri_schwarz_threshold + || request + .integrals() + .schwarz_threshold + .value + .map(|v| v.into_inner()) + != self.hf.eri_schwarz_threshold || hf .diis - .then(|| u64::try_from(hf.diis_size.into_inner())) + .enabled + .then(|| u64::try_from(hf.diis.max_history.value.into_inner())) .transpose() .map_err(|_| invalid())? != self.hf.diis_size @@ -608,14 +629,13 @@ impl ResolvedGuess { Self::CoreHamiltonian { perturbation: a }, Guess::CoreHamiltonian { perturbation: b }, ) - | ( - Self::OneElectron { perturbation: a }, - Guess::OneElectron { perturbation: b }, - ) => match (a, b) { - (None, None) => true, - (Some(a), Some(b)) => random_matches(a, b), - _ => false, - }, + | (Self::OneElectron { perturbation: a }, Guess::OneElectron { perturbation: b }) => { + match (a, b) { + (None, None) => true, + (Some(a), Some(b)) => random_matches(a, b), + _ => false, + } + } (Self::Random { random: a }, Guess::Random { random: b }) => random_matches(a, b), (Self::Zero, Guess::Zero) => true, _ => false, diff --git a/crates/rustiq-core/src/persistence/data.rs b/crates/rustiq-core/src/persistence/data.rs index e0f1dd36..1283e495 100644 --- a/crates/rustiq-core/src/persistence/data.rs +++ b/crates/rustiq-core/src/persistence/data.rs @@ -8,3 +8,6 @@ pub use rustiq_data::RustiQData; pub(crate) use rustiq_data::AO_ERI_ARTIFACT; mod portable; + +mod bundle; +pub use bundle::RustiQBundle; diff --git a/crates/rustiq-core/src/persistence/data/bundle.rs b/crates/rustiq-core/src/persistence/data/bundle.rs new file mode 100644 index 00000000..37ad8c62 --- /dev/null +++ b/crates/rustiq-core/src/persistence/data/bundle.rs @@ -0,0 +1,306 @@ +//! A non-empty portable collection; scientific APIs still operate on one calculation. +use std::{ + collections::HashSet, + path::Path, + sync::{Arc, Mutex}, +}; + +use relative_path::{RelativePath, RelativePathBuf}; + +use super::{portable, rustiq_data::MAX_MANIFEST_BYTES, RustiQData}; +use crate::persistence::{ + manifest::{PortableCalculation, PortableManifest, SourceManifest}, + ArtifactAttributes, Manifest, ManifestKind, PortableError, Producer, SourceProvenance, Storage, + COMPACT_ERI_REPRESENTATION, FORMAT_NAME, FORMAT_VERSION, MANIFEST_PATH, +}; + +/// A portable snapshot containing one or more independently reusable calculations. +/// Sources are shared opaque provenance. ZIP details remain private. +#[derive(Debug)] +pub struct RustiQBundle { + calculations: Vec, + sources: Vec, +} + +impl RustiQBundle { + /// Collects portable calculation data, rejecting empty or cache-only collections. + /// Exact duplicate sources are stored once at the bundle level. + pub fn new(mut calculations: Vec) -> Result { + if calculations.is_empty() { + return Err(invalid("portable bundles require at least one calculation")); + } + let mut sources = Vec::new(); + for data in &mut calculations { + if data.request.is_none() || data.context.is_none() { + return Err(invalid( + "portable entries require request and resolved snapshots", + )); + } + for source in &data.sources { + if !sources.contains(source) { + sources.push(source.clone()); + } + } + } + validate_sources(&sources)?; + for data in &mut calculations { + data.sources.clear(); + } + Ok(Self { + calculations, + sources, + }) + } + + /// Opens read-only, validating all snapshots and references without loading arrays. + pub fn open(path: impl AsRef) -> Result { + let mut storage = Storage::open_zip(path.as_ref())?; + let manifest: PortableManifest = storage + .read_json(RelativePath::new(MANIFEST_PATH), MAX_MANIFEST_BYTES) + .map_err(crate::persistence::PersistenceReadError::from)?; + if manifest.format != FORMAT_NAME + || manifest.format_version != FORMAT_VERSION + || manifest.kind != ManifestKind::Portable + { + return Err(PortableError::UnsupportedVersion); + } + if manifest.calculations.is_empty() { + return Err(invalid("portable bundles require at least one calculation")); + } + let sources = portable::read_sources(&mut storage, &manifest.sources)?; + let storage = Arc::new(Mutex::new(storage)); + let mut ids = HashSet::new(); + let mut calculations = Vec::new(); + for entry in manifest.calculations { + let digits = entry.id.strip_prefix("calculation-").unwrap_or_default(); + if digits.is_empty() + || !digits.bytes().all(|b| b.is_ascii_digit()) + || !ids.insert(entry.id.clone()) + { + return Err(invalid("invalid or duplicate calculation identifier")); + } + let prefix = format!("calculations/{}", entry.id); + let mut request = entry.request; + let mut calculation = entry.calculation; + request.path = local_path(&prefix, &request.path)?; + calculation.path = local_path(&prefix, &calculation.path)?; + let mut artifacts = entry.artifacts; + for artifact in artifacts.values_mut() { + artifact.path = RelativePathBuf::from(local_path(&prefix, artifact.path.as_str())?); + } + let internal = Manifest { + format: manifest.format.clone(), + format_version: manifest.format_version, + kind: ManifestKind::Portable, + producer: manifest.producer.clone(), + scientific_identity: entry.scientific_identity, + artifacts, + request: Some(request), + calculation: Some(calculation), + sources: Vec::new(), + }; + let data = + RustiQData::read_manifest(Storage::scoped(storage.clone(), prefix), internal)?; + calculations.push(RustiQData::validate_portable(data)?); + } + Ok(Self { + calculations, + sources, + }) + } + + /// Entries retain manifest order; each exposes its own typed artifact access. + pub fn calculations(&self) -> &[RustiQData] { + &self.calculations + } + + /// Mutates scientific artifacts within entries, without allowing an empty collection. + pub fn calculations_mut(&mut self) -> &mut [RustiQData] { + &mut self.calculations + } + + pub fn sources(&self) -> impl ExactSizeIterator { + self.sources.iter() + } + + /// Adds shared exact source bytes; the original name never becomes a member path. + pub fn add_source( + &mut self, + original_name: impl Into, + bytes: impl Into>, + ) -> Result<(), PortableError> { + let source = SourceProvenance { + original_name: original_name.into(), + bytes: bytes.into(), + }; + let mut sources = self.sources.clone(); + sources.push(source); + validate_sources(&sources)?; + self.sources = sources; + Ok(()) + } + + /// Publishes a complete validated snapshot atomically, without overwriting a destination. + pub fn write(&mut self, path: impl AsRef) -> Result<(), PortableError> { + // Sources attached through individual entry APIs are lifted to shared provenance. + for data in &mut self.calculations { + for source in &data.sources { + if !self.sources.contains(source) { + self.sources.push(source.clone()); + } + } + } + validate_sources(&self.sources)?; + write_bundle(&mut self.calculations, &self.sources, path.as_ref()) + } + + pub(super) fn into_parts(self) -> (Vec, Vec) { + (self.calculations, self.sources) + } +} + +fn invalid(message: &str) -> PortableError { + PortableError::InvalidArchive(message.into()) +} + +fn local_path(prefix: &str, path: &str) -> Result { + crate::persistence::validate_storage_path(RelativePath::new(path))?; + path.strip_prefix(&format!("{prefix}/")) + .map(str::to_owned) + .ok_or_else(|| invalid("reference crosses calculation namespace")) +} + +fn validate_sources(sources: &[SourceProvenance]) -> Result<(), PortableError> { + use crate::persistence::provenance::{MAX_SOURCES, MAX_SOURCE_BYTES, MAX_SOURCE_NAME_BYTES}; + let total = sources.iter().try_fold(0_u64, |total, source| { + total.checked_add(u64::try_from(source.bytes.len()).ok()?) + }); + if sources.len() > MAX_SOURCES + || total.is_none_or(|total| total > MAX_SOURCE_BYTES) + || sources + .iter() + .any(|source| source.original_name.len() > MAX_SOURCE_NAME_BYTES) + { + return Err(invalid("source provenance exceeds supported limits")); + } + Ok(()) +} + +pub(super) fn write_bundle( + calculations: &mut [RustiQData], + sources: &[SourceProvenance], + path: &Path, +) -> Result<(), PortableError> { + if calculations.is_empty() { + return Err(invalid("portable bundles require at least one calculation")); + } + validate_sources(sources)?; + match std::fs::symlink_metadata(path) { + Ok(_) => return Err(PortableError::AlreadyExists), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let temporary = tempfile::NamedTempFile::new_in(parent)?; + write_snapshot( + calculations, + sources, + Storage::create_zip(temporary.reopen()?), + )?; + drop(RustiQBundle::open(temporary.path())?); + temporary.persist_noclobber(path).map_err(|error| { + if error.error.kind() == std::io::ErrorKind::AlreadyExists { + PortableError::AlreadyExists + } else { + PortableError::Io(error.error) + } + })?; + #[cfg(unix)] + std::fs::File::open(parent)?.sync_all()?; + Ok(()) +} + +pub(super) fn write_snapshot( + calculations: &mut [RustiQData], + sources: &[SourceProvenance], + destination: Storage, +) -> Result<(), PortableError> { + let storage = Arc::new(Mutex::new(destination)); + let mut manifest = PortableManifest { + format: FORMAT_NAME.into(), + format_version: FORMAT_VERSION, + kind: ManifestKind::Portable, + producer: Producer { + name: "RustiQ".into(), + version: env!("CARGO_PKG_VERSION").into(), + }, + sources: Vec::new(), + calculations: Vec::new(), + }; + for (index, source) in sources.iter().enumerate() { + let path = format!("sources/{index}"); + let metadata = storage + .lock() + .expect("storage mutex poisoned") + .write_artifact::(RelativePath::new(&path), |writer| { + writer.write_all(source.bytes())?; + Ok(()) + })?; + manifest.sources.push(SourceManifest { + original_name: source.original_name.clone(), + path, + version: 1, + size: metadata.size, + digest: metadata.digest, + }); + } + for (index, data) in calculations.iter_mut().enumerate() { + if data.context.is_none() || data.request.is_none() { + return Err(invalid("missing calculation snapshots")); + } + if data.ao_eri.is_none() && data.manifest.artifacts.get(super::AO_ERI_ARTIFACT).is_some_and(|artifact| + artifact.representation == COMPACT_ERI_REPRESENTATION && matches!(&artifact.attributes, + ArtifactAttributes::AoEri(attributes) if attributes.computation_version == crate::persistence::AO_ERI_COMPUTATION_VERSION)) { + data.validate_unloaded_eri()?; + } + let id = format!("calculation-{index}"); + let prefix = format!("calculations/{id}"); + let saved_sources = std::mem::take(&mut data.sources); + let result = data.write_entry(Storage::scoped(storage.clone(), prefix.clone())); + data.sources = saved_sources; + let entry = result?; + let mut request = entry + .request + .ok_or_else(|| invalid("missing request snapshot"))?; + let mut calculation = entry + .calculation + .ok_or_else(|| invalid("missing resolved snapshot"))?; + request.path = format!("{prefix}/{}", request.path); + calculation.path = format!("{prefix}/{}", calculation.path); + let mut artifacts = entry.artifacts; + for artifact in artifacts.values_mut() { + artifact.path = RelativePathBuf::from(format!("{prefix}/{}", artifact.path)); + } + manifest.calculations.push(PortableCalculation { + id, + scientific_identity: entry.scientific_identity, + request, + calculation, + artifacts, + }); + } + storage + .lock() + .expect("storage mutex poisoned") + .write_json(RelativePath::new(MANIFEST_PATH), &manifest) + .map_err(crate::persistence::PersistenceWriteError::from)?; + Arc::try_unwrap(storage) + .expect("all entry writers released") + .into_inner() + .expect("storage mutex poisoned") + .finish()?; + Ok(()) +} diff --git a/crates/rustiq-core/src/persistence/data/portable.rs b/crates/rustiq-core/src/persistence/data/portable.rs index 4dcba95d..bb8bf618 100644 --- a/crates/rustiq-core/src/persistence/data/portable.rs +++ b/crates/rustiq-core/src/persistence/data/portable.rs @@ -8,8 +8,6 @@ use super::super::{ use super::{RustiQData, AO_ERI_ARTIFACT}; use crate::calculation::PreparedCalculation; use relative_path::RelativePath; -#[cfg(any(unix, test))] -use std::fs::File; use std::io::Read; use std::path::Path; @@ -32,8 +30,21 @@ impl RustiQData { /// Opens a portable archive and validates its context and artifact index. /// Numerical payloads are verified and decoded lazily on first access. + /// Multi-calculation archives require [`super::RustiQBundle::open`]. pub fn open(path: impl AsRef) -> Result { - let mut data = Self::read_from(Storage::open_zip(path.as_ref())?)?; + let bundle = super::RustiQBundle::open(path)?; + let (mut calculations, sources) = bundle.into_parts(); + if calculations.len() != 1 { + return Err(PortableError::InvalidArchive( + "single-calculation API requires exactly one calculation".into(), + )); + } + let mut data = calculations.remove(0); + data.sources = sources; + Ok(data) + } + + pub(crate) fn validate_portable(mut data: Self) -> Result { if data.manifest.kind != ManifestKind::Portable { return Err(PortableError::UnsupportedVersion); } @@ -59,56 +70,6 @@ impl RustiQData { let request = request_snapshot.to_request()?; snapshot.validate_request(&request)?; data.request = Some(request); - if data.manifest.sources.len() > MAX_SOURCES { - return Err(PortableError::InvalidArchive( - "too many source payloads".into(), - )); - } - let mut total = 0_u64; - let mut paths = std::collections::HashSet::new(); - for reference in &data.manifest.sources { - let path = RelativePath::new(&reference.path); - super::super::validate_storage_path(path)?; - total = total - .checked_add(reference.size) - .filter(|total| *total <= MAX_SOURCE_BYTES) - .ok_or_else(|| { - PortableError::InvalidArchive("source payloads exceed size limit".into()) - })?; - if reference.version != 1 { - return Err(PortableError::UnsupportedVersion); - } - if !reference.path.starts_with("sources/") - || reference.original_name.len() > MAX_SOURCE_NAME_BYTES - || !paths.insert(reference.path.to_lowercase()) - { - return Err(PortableError::InvalidArchive( - "invalid source reference".into(), - )); - } - let expected = super::super::manifest::SnapshotManifest { - path: reference.path.clone(), - version: reference.version, - size: reference.size, - digest: reference.digest, - }; - verify_reference(source, &expected, &reference.path, MAX_SOURCE_BYTES)?; - let bytes = source.with_artifact::<_, PortableError, _>(path, |reader| { - let mut bytes = Vec::new(); - reader.take(reference.size + 1).read_to_end(&mut bytes)?; - if u64::try_from(bytes.len()).map_err(|_| { - PortableError::InvalidArchive("source size exceeds V1 range".into()) - })? != reference.size - { - return Err(ArtifactError::IntegrityMismatch(reference.path.clone()).into()); - } - Ok(bytes) - })?; - data.sources.push(SourceProvenance { - original_name: reference.original_name.clone(), - bytes, - }); - } let identity = snapshot.identity(); if identity.version != data.manifest.scientific_identity.version || identity.digest != data.manifest.scientific_identity.digest @@ -249,11 +210,10 @@ impl RustiQData { } fn matches_eri_identity(&self, calculation: &PreparedCalculation) -> bool { - let hf = calculation.hf_config(); let identity = super::super::ao_eri_identity( calculation.get_molecule(), calculation.get_basis(), - hf.eri_schwarz_threshold, + calculation.integral_config().schwarz_threshold.value, ); self.manifest.scientific_identity.version == identity.version && self.manifest.scientific_identity.digest == identity.digest @@ -270,39 +230,14 @@ impl RustiQData { "portable output requires resolved calculation context".into(), )); } - // Validate any known, unloaded artifact before preserving it as scientific data. - if self.ao_eri.is_none() && self.manifest.artifacts.get(AO_ERI_ARTIFACT) - .is_some_and(|artifact| artifact.representation == COMPACT_ERI_REPRESENTATION - && matches!(&artifact.attributes, ArtifactAttributes::AoEri(attributes) if attributes.computation_version == super::super::AO_ERI_COMPUTATION_VERSION)) { - self.validate_unloaded_eri()?; - } - let path = path.as_ref(); - match std::fs::symlink_metadata(path) { - Ok(_) => return Err(PortableError::AlreadyExists), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(error.into()), - } - let parent = path - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .unwrap_or(Path::new(".")); - let temporary = tempfile::NamedTempFile::new_in(parent)?; - self.write_to(Storage::create_zip(temporary.reopen()?))?; - // Reopening validates the finalized container, context, index and publication limits. - drop(Self::open(temporary.path())?); - temporary.persist_noclobber(path).map_err(|error| { - if error.error.kind() == std::io::ErrorKind::AlreadyExists { - PortableError::AlreadyExists - } else { - PortableError::Io(error.error) - } - })?; - #[cfg(unix)] - File::open(parent)?.sync_all()?; - Ok(()) + let sources = std::mem::take(&mut self.sources); + let result = + super::bundle::write_bundle(std::slice::from_mut(self), &sources, path.as_ref()); + self.sources = sources; + result } - fn validate_unloaded_eri(&mut self) -> Result<(), PortableError> { + pub(crate) fn validate_unloaded_eri(&mut self) -> Result<(), PortableError> { let artifact = &self.manifest.artifacts[AO_ERI_ARTIFACT]; let ArtifactAttributes::AoEri(attributes) = &artifact.attributes else { unreachable!() @@ -320,6 +255,64 @@ impl RustiQData { } } +pub(super) fn read_sources( + source: &mut Storage, + references: &[super::super::manifest::SourceManifest], +) -> Result, PortableError> { + let mut sources = Vec::new(); + if references.len() > MAX_SOURCES { + return Err(PortableError::InvalidArchive( + "too many source payloads".into(), + )); + } + let mut total = 0_u64; + let mut paths = std::collections::HashSet::new(); + for reference in references { + let path = RelativePath::new(&reference.path); + super::super::validate_storage_path(path)?; + total = total + .checked_add(reference.size) + .filter(|total| *total <= MAX_SOURCE_BYTES) + .ok_or_else(|| { + PortableError::InvalidArchive("source payloads exceed size limit".into()) + })?; + if reference.version != 1 { + return Err(PortableError::UnsupportedVersion); + } + if !reference.path.starts_with("sources/") + || reference.original_name.len() > MAX_SOURCE_NAME_BYTES + || !paths.insert(reference.path.to_lowercase()) + { + return Err(PortableError::InvalidArchive( + "invalid source reference".into(), + )); + } + let expected = super::super::manifest::SnapshotManifest { + path: reference.path.clone(), + version: reference.version, + size: reference.size, + digest: reference.digest, + }; + verify_reference(source, &expected, &reference.path, MAX_SOURCE_BYTES)?; + let bytes = source.with_artifact::<_, PortableError, _>(path, |reader| { + let mut bytes = Vec::new(); + reader.take(reference.size + 1).read_to_end(&mut bytes)?; + if u64::try_from(bytes.len()) + .map_err(|_| PortableError::InvalidArchive("source size exceeds V1 range".into()))? + != reference.size + { + return Err(ArtifactError::IntegrityMismatch(reference.path.clone()).into()); + } + Ok(bytes) + })?; + sources.push(SourceProvenance { + original_name: reference.original_name.clone(), + bytes, + }); + } + Ok(sources) +} + fn verify_reference( source: &mut Storage, reference: &super::super::manifest::SnapshotManifest, diff --git a/crates/rustiq-core/src/persistence/data/portable/tests.rs b/crates/rustiq-core/src/persistence/data/portable/tests.rs index 70eadf89..a0184ece 100644 --- a/crates/rustiq-core/src/persistence/data/portable/tests.rs +++ b/crates/rustiq-core/src/persistence/data/portable/tests.rs @@ -8,12 +8,16 @@ use crate::{ test_utils::load_minimal_basis_file, }; use std::{ - fs, + fs::{self, File}, io::{Read, Write}, num::NonZeroU8, }; use zip::{write::SimpleFileOptions, CompressionMethod, ZipArchive, ZipWriter}; +const CALCULATION_PATH: &str = "calculations/calculation-0/calculation.json"; +const REQUEST_PATH: &str = "calculations/calculation-0/request.json"; +const AO_ERI_PATH: &str = "calculations/calculation-0/arrays/integrals/ao-eri.npy"; + fn calculation() -> PreparedCalculation { let geometry = Geometry::from_source("private.xyz", "2\nH2\nH 0 0 0\nH 1.4 0 0\n").unwrap(); CalculationBuilder::new(&geometry, &load_minimal_basis_file()) @@ -81,16 +85,16 @@ fn refresh_snapshot_digest(members: &mut [(String, Vec)]) { .1; let (size, digest) = (snapshot.len(), sha256(snapshot).to_string()); edit_json(members, "manifest.json", |m| { - m["calculation"]["size"] = size.into(); - m["calculation"]["digest"] = digest.into(); + m["calculations"][0]["calculation"]["size"] = size.into(); + m["calculations"][0]["calculation"]["digest"] = digest.into(); }); } fn refresh_eri_digest(members: &mut [(String, Vec)]) { let payload = &members.iter().find(|(n, _)| n == AO_ERI_PATH).unwrap().1; let (size, digest) = (payload.len(), sha256(payload).to_string()); edit_json(members, "manifest.json", |m| { - m["artifacts"]["ao_eri"]["size"] = size.into(); - m["artifacts"]["ao_eri"]["digest"] = digest.into(); + m["calculations"][0]["artifacts"]["ao_eri"]["size"] = size.into(); + m["calculations"][0]["artifacts"]["ao_eri"]["digest"] = digest.into(); }); } @@ -273,16 +277,24 @@ fn request_and_source_integrity_versions_and_limits_are_checked_on_open() { let original = members(&path); let broken = dir.path().join("broken.rustiq"); for (target, field, value) in [ - ("manifest.json", "/request", serde_json::Value::Null), ( "manifest.json", - "/request/path", + "/calculations/0/request", + serde_json::Value::Null, + ), + ( + "manifest.json", + "/calculations/0/request/path", serde_json::json!(CALCULATION_PATH), ), - ("manifest.json", "/request/version", serde_json::json!(99)), ( "manifest.json", - "/request/size", + "/calculations/0/request/version", + serde_json::json!(99), + ), + ( + "manifest.json", + "/calculations/0/request/size", serde_json::json!(MAX_REQUEST_BYTES + 1), ), ( @@ -325,8 +337,8 @@ fn request_and_source_integrity_versions_and_limits_are_checked_on_open() { .1; let (size, digest) = (bytes.len(), sha256(bytes).to_string()); edit_json(&mut contents, "manifest.json", |m| { - m["request"]["size"] = size.into(); - m["request"]["digest"] = digest.into(); + m["calculations"][0]["request"]["size"] = size.into(); + m["calculations"][0]["request"]["digest"] = digest.into(); }); } write_members(&broken, &contents); @@ -357,7 +369,10 @@ fn empty_artifacts_uhf_and_mp2_context_round_trip() { }) .with_hf(HfConfig { method: HfMethod::Uhf.into(), - diis: true, + diis: crate::config::DiisConfig { + enabled: true, + ..Default::default() + }, ..Default::default() }) .with_mp2(Mp2Config::default()) @@ -386,9 +401,8 @@ fn compatibility_depends_on_effective_integral_inputs_only() { .unwrap(); assert!(data.eri_is_compatible(&mp2)); let changed = CalculationBuilder::new(prepared.get_molecule(), &load_minimal_basis_file()) - .with_hf(HfConfig { - eri_schwarz_threshold: None, - ..Default::default() + .with_integrals(crate::config::IntegralConfig { + schwarz_threshold: None.into(), }) .prepare() .unwrap(); @@ -464,9 +478,12 @@ fn unknown_artifacts_survive_verified_streaming_copy() { data().write(&first).unwrap(); let mut contents = members(&first); let opaque = b"future payload".to_vec(); - contents.push(("arrays/future.bin".into(), opaque.clone())); + contents.push(( + "calculations/calculation-0/arrays/future.bin".into(), + opaque.clone(), + )); edit_json(&mut contents, "manifest.json", |m| { - m["artifacts"]["future"] = serde_json::json!({"path":"arrays/future.bin","size":opaque.len(),"digest":sha256(&opaque),"representation":"future-v99","attributes":{"opaque":17}}); + m["calculations"][0]["artifacts"]["future"] = serde_json::json!({"path":"calculations/calculation-0/arrays/future.bin","size":opaque.len(),"digest":sha256(&opaque),"representation":"future-v99","attributes":{"opaque":17}}); }); let source = dir.path().join("source.rustiq"); write_members(&source, &contents); @@ -474,7 +491,10 @@ fn unknown_artifacts_survive_verified_streaming_copy() { let output = dir.path().join("copy.rustiq"); restored.write(&output).unwrap(); assert!(restored.ao_eri.is_none()); - assert!(members(&output).contains(&("arrays/future.bin".into(), opaque))); + assert!(members(&output).contains(&( + "calculations/calculation-0/arrays/future.bin".into(), + opaque + ))); assert_eq!( RustiQData::open(&output).unwrap().manifest.artifacts["future"], restored.manifest.artifacts["future"] @@ -492,27 +512,27 @@ fn rejects_invalid_context_manifest_and_artifact_metadata() { ("manifest.json", "/format_version", serde_json::json!(999)), ( "manifest.json", - "/scientific_identity/version", + "/calculations/0/scientific_identity/version", serde_json::json!(99), ), ( "manifest.json", - "/artifacts/ao_eri/attributes/basis_functions", + "/calculations/0/artifacts/ao_eri/attributes/basis_functions", serde_json::json!(3), ), ( "manifest.json", - "/artifacts/ao_eri/attributes/computation_version", + "/calculations/0/artifacts/ao_eri/attributes/computation_version", serde_json::json!(99), ), ( "manifest.json", - "/artifacts/ao_eri/path", + "/calculations/0/artifacts/ao_eri/path", serde_json::json!("CALCULATION.JSON"), ), ( "manifest.json", - "/artifacts/ao_eri/size", + "/calculations/0/artifacts/ao_eri/size", serde_json::json!(1), ), (CALCULATION_PATH, "/version", serde_json::json!(99)), @@ -696,7 +716,12 @@ fn zip64_handles_offsets_beyond_four_gib_without_allocating_large_payloads() { let path = dir.path().join("sparse.rustiq"); let mut file = File::create(&path).unwrap(); file.seek(SeekFrom::Start(u64::from(u32::MAX) + 1)).unwrap(); - data().write_to(Storage::create_zip(file)).unwrap(); + super::super::bundle::write_snapshot( + std::slice::from_mut(&mut data()), + &[], + Storage::create_zip(file), + ) + .unwrap(); assert!(fs::metadata(&path).unwrap().len() > u64::from(u32::MAX)); let mut restored = RustiQData::open(&path).unwrap(); assert_eq!( @@ -748,7 +773,7 @@ fn rejects_links_special_entries_encryption_and_oversized_metadata() { assert!(RustiQData::open(path).is_err()); let mut contents = members(&source); edit_json(&mut contents, "manifest.json", |m| { - m["calculation"]["size"] = (MAX_CALCULATION_BYTES + 1).into() + m["calculations"][0]["calculation"]["size"] = (MAX_CALCULATION_BYTES + 1).into() }); let path = dir.path().join("snapshot-limit.rustiq"); write_members(&path, &contents); @@ -849,7 +874,7 @@ fn unsupported_representation_is_inspectable_but_not_scientifically_usable() { data().write(&source).unwrap(); let mut contents = members(&source); edit_json(&mut contents, "manifest.json", |m| { - m["artifacts"]["ao_eri"]["representation"] = "future-eri-v2".into() + m["calculations"][0]["artifacts"]["ao_eri"]["representation"] = "future-eri-v2".into() }); let unknown = dir.path().join("unknown.rustiq"); write_members(&unknown, &contents); @@ -939,8 +964,8 @@ fn independently_rounded_python_angstrom_conversion_is_accepted() { let request = &contents.iter().find(|(n, _)| n == REQUEST_PATH).unwrap().1; let (size, digest) = (request.len(), sha256(request).to_string()); edit_json(&mut contents, "manifest.json", |m| { - m["request"]["size"] = size.into(); - m["request"]["digest"] = digest.into(); + m["calculations"][0]["request"]["size"] = size.into(); + m["calculations"][0]["request"]["digest"] = digest.into(); }); let invalid = dir.path().join("changed.rustiq"); write_members(&invalid, &contents); @@ -1028,3 +1053,246 @@ fn published_required_fields_match_v1_decoders() { } } } + +#[test] +fn bundle_entries_have_independent_lazy_artifacts_and_shared_sources() { + use crate::persistence::RustiQBundle; + let basis = load_minimal_basis_file(); + let prepared = calculation(); + let changed = CalculationBuilder::new(prepared.get_molecule(), &basis) + .with_integrals(crate::config::IntegralConfig { + schwarz_threshold: None.into(), + }) + .prepare() + .unwrap(); + let mut first = data(); + first + .add_source("batch.ncl", b"shared source".as_slice()) + .unwrap(); + let mut second = RustiQData::from_calculation(&changed).unwrap(); + second + .add_source("batch.ncl", b"shared source".as_slice()) + .unwrap(); + second.set_eri(CompactEri::Zeroed(2)).unwrap(); + let mut bundle = RustiQBundle::new(vec![first, second]).unwrap(); + assert_eq!(bundle.sources().len(), 1); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("batch.rustiq"); + bundle.write(&path).unwrap(); + let contents = members(&path); + let manifest: serde_json::Value = serde_json::from_slice( + &contents + .iter() + .find(|(n, _)| n == "manifest.json") + .unwrap() + .1, + ) + .unwrap(); + assert!(manifest.get("scientific_identity").is_none()); + assert_eq!(manifest["calculations"].as_array().unwrap().len(), 2); + assert_eq!( + contents + .iter() + .filter(|(n, _)| n.starts_with("sources/")) + .count(), + 1 + ); + assert!(RustiQData::open(&path) + .unwrap_err() + .to_string() + .contains("exactly one")); + let mut restored = RustiQBundle::open(&path).unwrap(); + assert!(restored + .calculations() + .iter() + .all(|data| data.ao_eri.is_none())); + assert!(restored.calculations()[0].eri_is_compatible(&prepared)); + assert!(!restored.calculations()[1].eri_is_compatible(&prepared)); + assert!(restored.calculations()[1].eri_is_compatible(&changed)); + assert_eq!( + restored.calculations_mut()[0] + .read_eri() + .unwrap() + .ordered_values()[0], + 0.5 + ); + assert!(restored.calculations()[1].ao_eri.is_none()); + assert_eq!( + restored.calculations_mut()[1] + .read_eri() + .unwrap() + .ordered_values()[0], + 0.0 + ); + let copy = dir.path().join("copy.rustiq"); + restored.write(©).unwrap(); + assert_eq!(fs::read(&path).unwrap(), fs::read(©).unwrap()); +} + +#[test] +fn bundle_rejects_empty_duplicate_ids_cross_entry_and_conflicting_references() { + use crate::persistence::RustiQBundle; + assert!(RustiQBundle::new(vec![]).is_err()); + let dir = tempfile::tempdir().unwrap(); + let good = dir.path().join("good.rustiq"); + RustiQBundle::new(vec![data(), data()]) + .unwrap() + .write(&good) + .unwrap(); + let original = members(&good); + for (field, value) in [ + ("/calculations", serde_json::json!([])), + ("/calculations/1/id", serde_json::json!("calculation-0")), + ("/calculations/1/id", serde_json::json!("../invalid")), + ( + "/calculations/1/request/path", + serde_json::json!(REQUEST_PATH), + ), + ( + "/calculations/0/artifacts/ao_eri/path", + serde_json::json!(CALCULATION_PATH), + ), + ] { + let mut contents = original.clone(); + edit_json(&mut contents, "manifest.json", |manifest| { + *manifest.pointer_mut(field).unwrap() = value; + }); + let bad = dir.path().join("bad.rustiq"); + write_members(&bad, &contents); + assert!(RustiQBundle::open(&bad).is_err(), "{field}"); + } + let mut contents = original; + edit_json(&mut contents, "manifest.json", |manifest| { + manifest["scientific_identity"] = + manifest["calculations"][0]["scientific_identity"].clone(); + }); + let bad = dir.path().join("aggregate.rustiq"); + write_members(&bad, &contents); + assert!(RustiQBundle::open(&bad).is_err()); +} + +#[test] +fn corrupt_bundle_entry_does_not_prevent_independent_artifact_access_or_publish_partial_output() { + use crate::persistence::RustiQBundle; + let dir = tempfile::tempdir().unwrap(); + let good = dir.path().join("good.rustiq"); + RustiQBundle::new(vec![data(), data()]) + .unwrap() + .write(&good) + .unwrap(); + let mut contents = members(&good); + *contents + .iter_mut() + .find(|(n, _)| n == AO_ERI_PATH) + .unwrap() + .1 + .last_mut() + .unwrap() ^= 1; + let bad = dir.path().join("bad.rustiq"); + write_members(&bad, &contents); + let original = fs::read(&bad).unwrap(); + let mut bundle = RustiQBundle::open(&bad).unwrap(); + assert!(bundle.calculations_mut()[0].read_eri().is_err()); + assert_eq!( + bundle.calculations_mut()[1] + .read_eri() + .unwrap() + .ordered_values()[0], + 0.5 + ); + let output = dir.path().join("output.rustiq"); + assert!(bundle.write(&output).is_err()); + assert!(!output.exists()); + assert_eq!(fs::read(&bad).unwrap(), original); + assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 2); +} + +#[test] +fn reads_python_multi_calculation_fixture() { + let fixture = + include_str!("../../../../tests/data/persistence/portable-python-multi-v1.rustiq.hex"); + let bytes: Vec = fixture + .trim() + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("multi.rustiq"); + fs::write(&path, bytes).unwrap(); + let mut bundle = crate::persistence::RustiQBundle::open(&path).unwrap(); + assert_eq!(bundle.sources().len(), 1); + assert_eq!(bundle.calculations().len(), 2); + for data in bundle.calculations_mut() { + assert!(data.ao_eri.is_none()); + assert_eq!( + data.read_eri().unwrap().ordered_values(), + &[0.5, 1.5, 2.5, 3.5, 4.5, 5.5] + ); + } +} + +#[test] +#[ignore = "writes a multi-calculation artifact for Python interoperability"] +fn writes_bundle_for_python_interoperability() { + let path = std::env::var_os("RUSTIQ_ARCHIVE_TEST_OUTPUT").expect("output path"); + let mut bundle = crate::persistence::RustiQBundle::new(vec![data(), data()]).unwrap(); + bundle + .add_source("batch.ncl", b"shared source".as_slice()) + .unwrap(); + bundle.write(path).unwrap(); +} + +#[test] +fn portable_manifest_golden_and_schema_required_fields_match_decoder() { + use crate::persistence::manifest::PortableManifest; + use serde_json::{json, Value}; + let value: Value = serde_json::from_str(include_str!( + "../../../../tests/data/persistence/portable-manifest-h2-v1.json" + )) + .unwrap(); + let decoded: PortableManifest = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(serde_json::to_value(decoded).unwrap(), value); + let schema: Value = serde_json::from_str(include_str!( + "../../../../../../schemas/portable-manifest-v1.schema.json" + )) + .unwrap(); + assert_eq!(schema["properties"]["calculations"]["minItems"], 1); + for (path, definition) in [ + ("", ""), + ("/calculations/0", "calculation"), + ("/calculations/0/request", "snapshot"), + ("/sources/0", "source"), + ] { + let record = if definition.is_empty() { + &schema + } else { + &schema["$defs"][definition] + }; + let required = record["required"].as_array().unwrap(); + for field in record["properties"].as_object().unwrap().keys() { + let mut candidate = value.clone(); + candidate + .pointer_mut(path) + .unwrap() + .as_object_mut() + .unwrap() + .remove(field); + assert_eq!( + serde_json::from_value::(candidate).is_ok(), + !required.contains(&json!(field)), + "{definition} {field}" + ); + } + let mut candidate = value.clone(); + candidate + .pointer_mut(path) + .unwrap() + .as_object_mut() + .unwrap() + .insert("unknown_v1_field".into(), json!(0)); + assert!(serde_json::from_value::(candidate).is_err()); + } +} diff --git a/crates/rustiq-core/src/persistence/data/rustiq_data.rs b/crates/rustiq-core/src/persistence/data/rustiq_data.rs index 6bc98988..16ac8e21 100644 --- a/crates/rustiq-core/src/persistence/data/rustiq_data.rs +++ b/crates/rustiq-core/src/persistence/data/rustiq_data.rs @@ -14,7 +14,7 @@ use super::super::{ use super::artifact::Artifact; pub(crate) const AO_ERI_ARTIFACT: &str = "ao_eri"; -const MAX_MANIFEST_BYTES: u64 = 1024 * 1024; +pub(super) const MAX_MANIFEST_BYTES: u64 = 1024 * 1024; /// Known scientific artifacts and their manifest, with values loaded on demand. #[derive(Debug)] @@ -74,6 +74,13 @@ impl RustiQData { pub(crate) fn read_from(mut source: Storage) -> Result { let manifest: Manifest = source.read_json(RelativePath::new(MANIFEST_PATH), MAX_MANIFEST_BYTES)?; + Self::read_manifest(source, manifest) + } + + pub(crate) fn read_manifest( + source: Storage, + manifest: Manifest, + ) -> Result { if manifest.format != FORMAT_NAME || manifest.format_version != FORMAT_VERSION { return Err(ManifestError::UnsupportedFormat.into()); } @@ -186,7 +193,9 @@ impl RustiQData { pub(crate) fn write_to(&mut self, destination: Storage) -> Result<(), PersistenceWriteError> { let eri = self.ao_eri.take(); - let result = self.write_inner(destination, eri.as_ref()); + let result = self + .write_inner(destination, eri.as_ref(), true) + .map(|_| ()); self.ao_eri = eri; result } @@ -196,14 +205,25 @@ impl RustiQData { destination: Storage, eri: &CompactEri, ) -> Result<(), PersistenceWriteError> { - self.write_inner(destination, Some(eri)) + self.write_inner(destination, Some(eri), true).map(|_| ()) + } + + pub(crate) fn write_entry( + &mut self, + destination: Storage, + ) -> Result { + let eri = self.ao_eri.take(); + let result = self.write_inner(destination, eri.as_ref(), false); + self.ao_eri = eri; + result } fn write_inner( &mut self, mut destination: Storage, eri: Option<&CompactEri>, - ) -> Result<(), PersistenceWriteError> { + publish_manifest: bool, + ) -> Result { let mut manifest = self.manifest.clone(); if manifest.kind == ManifestKind::Portable { manifest.producer = Producer { @@ -315,9 +335,11 @@ impl RustiQData { ); } - destination.write_json(RelativePath::new(MANIFEST_PATH), &manifest)?; + if publish_manifest { + destination.write_json(RelativePath::new(MANIFEST_PATH), &manifest)?; + } destination.finish()?; - Ok(()) + Ok(manifest) } } diff --git a/crates/rustiq-core/src/persistence/manifest.rs b/crates/rustiq-core/src/persistence/manifest.rs index 2eb7deb9..514f1890 100644 --- a/crates/rustiq-core/src/persistence/manifest.rs +++ b/crates/rustiq-core/src/persistence/manifest.rs @@ -393,3 +393,26 @@ mod wire_tests { } } } + +/// Portable wire format is independent of the single-artifact directory cache. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct PortableManifest { + pub format: String, + pub format_version: u32, + pub kind: ManifestKind, + pub producer: Producer, + #[serde(default)] + pub sources: Vec, + pub calculations: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct PortableCalculation { + pub id: String, + pub scientific_identity: ScientificIdentityManifest, + pub request: SnapshotManifest, + pub calculation: SnapshotManifest, + pub artifacts: BTreeMap, +} diff --git a/crates/rustiq-core/src/persistence/mod.rs b/crates/rustiq-core/src/persistence/mod.rs index 0c2980de..203edb2c 100644 --- a/crates/rustiq-core/src/persistence/mod.rs +++ b/crates/rustiq-core/src/persistence/mod.rs @@ -21,7 +21,7 @@ mod storage; pub use crate::eri::CompactEri; pub use checksum::{sha256, sha256_reader, verify_sha256, Sha256Digest, Sha256DigestParseError}; -pub use data::{AoEriArtifact, Artifact, RustiQData}; +pub use data::{AoEriArtifact, Artifact, RustiQBundle, RustiQData}; pub use eri_cache::{EriCache, EriCacheEntry}; pub use error::{ArtifactError, NpyError, PortableError}; pub(crate) use error::{ManifestError, PersistenceReadError, PersistenceWriteError, StorageError}; diff --git a/crates/rustiq-core/src/persistence/provenance.rs b/crates/rustiq-core/src/persistence/provenance.rs index 31464eff..67e78c8b 100644 --- a/crates/rustiq-core/src/persistence/provenance.rs +++ b/crates/rustiq-core/src/persistence/provenance.rs @@ -5,7 +5,7 @@ pub(crate) const MAX_SOURCE_NAME_BYTES: usize = 4096; /// Exact original source bytes with an informational original name. /// The name is never an archive member name or an extraction instruction. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct SourceProvenance { pub(crate) original_name: String, pub(crate) bytes: Vec, diff --git a/crates/rustiq-core/src/persistence/request.rs b/crates/rustiq-core/src/persistence/request.rs index 45585449..bc4ab6a1 100644 --- a/crates/rustiq-core/src/persistence/request.rs +++ b/crates/rustiq-core/src/persistence/request.rs @@ -11,7 +11,8 @@ use crate::{ calculation::CalculationRequest, config::{ validated::{DiisSize, NonNegativeFiniteF64, PositiveFiniteF64}, - HfConfig, HfMethod, MemoryLimit, MoleculeConfig, Mp2Config, + DiisConfig, HfConfig, HfMethod, IntegralConfig, MemoryLimit, MoleculeConfig, Mp2Config, + OrthogonalizationConfig, }, molecules::{atom::Atom, geometry::Geometry, units::Units}, }; @@ -113,10 +114,19 @@ impl RequestSnapshot { }, max_iterations: u64::try_from(hf.max_iterations.get()).map_err(|_| invalid())?, convergence_threshold: hf.convergence_threshold.into_inner(), - linear_dependency_threshold: hf.linear_dependency_threshold.value.into_inner(), - eri_schwarz_threshold: hf.eri_schwarz_threshold.map(|v| v.into_inner()), - diis: hf.diis, - diis_size: u64::try_from(hf.diis_size.into_inner()).map_err(|_| invalid())?, + linear_dependency_threshold: hf + .orthogonalization + .linear_dependency_threshold + .value + .into_inner(), + eri_schwarz_threshold: request + .integrals() + .schwarz_threshold + .value + .map(|v| v.into_inner()), + diis: hf.diis.enabled, + diis_size: u64::try_from(hf.diis.max_history.value.into_inner()) + .map_err(|_| invalid())?, guess: hf.guess.value.into(), }, mp2: request @@ -177,23 +187,22 @@ impl RequestSnapshot { .ok_or_else(invalid)?, convergence_threshold: PositiveFiniteF64::try_new(self.hf.convergence_threshold) .map_err(|_| invalid())?, - linear_dependency_threshold: NonNegativeFiniteF64::try_new( - self.hf.linear_dependency_threshold, - ) - .map_err(|_| invalid())? - .into(), - eri_schwarz_threshold: self - .hf - .eri_schwarz_threshold - .map(PositiveFiniteF64::try_new) - .transpose() - .map_err(|_| invalid())?, + orthogonalization: OrthogonalizationConfig { + linear_dependency_threshold: NonNegativeFiniteF64::try_new( + self.hf.linear_dependency_threshold, + ) + .map_err(|_| invalid())? + .into(), + }, guess: self.hf.guess.to_config().into(), - diis: self.hf.diis, - diis_size: DiisSize::try_new( - usize::try_from(self.hf.diis_size).map_err(|_| invalid())?, - ) - .map_err(|_| invalid())?, + diis: DiisConfig { + enabled: self.hf.diis, + max_history: DiisSize::try_new( + usize::try_from(self.hf.diis_size).map_err(|_| invalid())?, + ) + .map_err(|_| invalid())? + .into(), + }, }; let request = CalculationRequest { geometry: Geometry::new(String::new(), atoms), @@ -209,6 +218,15 @@ impl RequestSnapshot { }, basis_name: self.basis.clone(), hf, + integrals: IntegralConfig { + schwarz_threshold: self + .hf + .eri_schwarz_threshold + .map(PositiveFiniteF64::try_new) + .transpose() + .map_err(|_| invalid())? + .into(), + }, mp2: self .mp2 .as_ref() @@ -267,8 +285,8 @@ mod tests { value["hf"]["diis_size"] = json!(17); let snapshot: RequestSnapshot = serde_json::from_value(value.clone()).unwrap(); let request = snapshot.to_request().unwrap(); - assert_eq!(request.hf().diis, enabled); - assert_eq!(request.hf().diis_size.into_inner(), 17); + assert_eq!(request.hf().diis.enabled, enabled); + assert_eq!(request.hf().diis.max_history.value.into_inner(), 17); assert_eq!( serde_json::to_value(RequestSnapshot::from_request(&request).unwrap()).unwrap() ["hf"]["diis_size"], diff --git a/crates/rustiq-core/src/persistence/storage.rs b/crates/rustiq-core/src/persistence/storage.rs index ffabaa27..c5438423 100644 --- a/crates/rustiq-core/src/persistence/storage.rs +++ b/crates/rustiq-core/src/persistence/storage.rs @@ -15,6 +15,10 @@ mod zip; #[derive(Debug)] pub(crate) enum Storage { Folder(PathBuf), + Scoped { + storage: std::sync::Arc>, + prefix: String, + }, ZipReader(Box<::zip::ZipArchive>), ZipWriter(Box<::zip::ZipWriter>>), } @@ -70,6 +74,10 @@ impl Write for DigestWriter { } impl Storage { + pub(crate) fn scoped(storage: std::sync::Arc>, prefix: String) -> Self { + Self::Scoped { storage, prefix } + } + pub(crate) fn open_zip(path: &std::path::Path) -> Result { Ok(Self::ZipReader(Box::new(zip::open(path)?))) } @@ -81,6 +89,10 @@ impl Storage { pub(crate) fn artifact_size(&mut self, path: &RelativePath) -> Result { validate_path(path)?; match self { + Self::Scoped { storage, prefix } => storage + .lock() + .expect("storage mutex poisoned") + .artifact_size(&prefixed_path(prefix, path)?), Self::Folder(root) => FolderStorage { root: root.clone() }.artifact_size(path), Self::ZipReader(archive) => Ok(archive .by_name(path.as_str()) @@ -99,6 +111,10 @@ impl Storage { path: &RelativePath, ) -> Result { match self { + Self::Scoped { storage, prefix } => storage + .lock() + .expect("storage mutex poisoned") + .artifact_metadata(&prefixed_path(prefix, path)?), Self::Folder(root) => FolderStorage { root: root.clone() }.artifact_metadata(path), Self::ZipReader(archive) => { let size = archive @@ -120,6 +136,10 @@ impl Storage { F: FnOnce(&mut dyn Read) -> Result, { match self { + Self::Scoped { storage, prefix } => storage + .lock() + .expect("storage mutex poisoned") + .with_artifact(&prefixed_path(prefix, path).map_err(E::from)?, read), Self::Folder(root) => FolderStorage { root: root.clone() }.with_artifact(path, read), Self::ZipReader(archive) => zip::with_member(archive, path, read), Self::ZipWriter(_) => Err(E::from(zip::invalid("cannot read an archive writer"))), @@ -136,6 +156,10 @@ impl Storage { F: FnOnce(&mut dyn Write) -> Result<(), E>, { match self { + Self::Scoped { storage, prefix } => storage + .lock() + .expect("storage mutex poisoned") + .write_artifact(&prefixed_path(prefix, path).map_err(E::from)?, write), Self::Folder(root) => FolderStorage { root: root.clone() }.write_artifact(path, write), Self::ZipWriter(archive) => { validate_path(path).map_err(E::from)?; @@ -182,7 +206,7 @@ impl Storage { ) -> Result { let mut bytes = serde_json::to_vec_pretty(value)?; bytes.push(b'\n'); - let limit = match path.as_str() { + let limit = match path.file_name().unwrap_or_default() { super::calculation::CALCULATION_PATH => super::calculation::MAX_CALCULATION_BYTES, super::request::REQUEST_PATH => super::request::MAX_REQUEST_BYTES, _ => 1024 * 1024, @@ -198,7 +222,7 @@ impl Storage { pub(crate) fn finish(self) -> Result<(), StorageError> { match self { - Self::Folder(_) | Self::ZipReader(_) => Ok(()), + Self::Folder(_) | Self::ZipReader(_) | Self::Scoped { .. } => Ok(()), Self::ZipWriter(archive) => { let mut file = archive.finish().map_err(zip::zip_error)?; file.flush()?; @@ -209,6 +233,16 @@ impl Storage { } } +fn prefixed_path( + prefix: &str, + path: &RelativePath, +) -> Result { + validate_path(path)?; + let full = relative_path::RelativePathBuf::from(format!("{prefix}/{}", path.as_str())); + validate_path(&full)?; + Ok(full) +} + #[derive(Debug)] struct FolderStorage { root: PathBuf, diff --git a/crates/rustiq-core/src/persistence/storage/zip.rs b/crates/rustiq-core/src/persistence/storage/zip.rs index 9e1cd95b..a5c3e8b5 100644 --- a/crates/rustiq-core/src/persistence/storage/zip.rs +++ b/crates/rustiq-core/src/persistence/storage/zip.rs @@ -51,7 +51,7 @@ pub(super) fn open(path: &Path) -> Result, StorageError> { pub(super) fn options(path: &RelativePath) -> SimpleFileOptions { let metadata = matches!( - path.as_str(), + path.file_name().unwrap_or_default(), "manifest.json" | "calculation.json" | "request.json" ); SimpleFileOptions::default() diff --git a/crates/rustiq-core/tests/data/persistence/portable-manifest-h2-v1.json b/crates/rustiq-core/tests/data/persistence/portable-manifest-h2-v1.json new file mode 100644 index 00000000..e064de0e --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/portable-manifest-h2-v1.json @@ -0,0 +1,51 @@ +{ + "format": "rustiq-persistence", + "format_version": 1, + "kind": "portable", + "producer": { + "name": "Python interoperability fixture", + "version": "1" + }, + "sources": [ + { + "original_name": "../../calculation.toml", + "path": "sources/0", + "version": 1, + "size": 84, + "digest": "sha256:0e4846b3cda3e792522dad47a72f03dde51c5584411e028712ee292cb4184f06" + } + ], + "calculations": [ + { + "id": "calculation-0", + "scientific_identity": { + "version": 1, + "digest": "sha256:0fd27aa0f91978616522cb640381b2db36b1ddebb97f3fd53a8036270717c6cc" + }, + "request": { + "path": "calculations/calculation-0/request.json", + "version": 1, + "size": 666, + "digest": "sha256:b0f6d85c96f09c2320b5774c12455ba0de4193ca751bc9c2ba0cd1d2efe042c6" + }, + "calculation": { + "path": "calculations/calculation-0/calculation.json", + "version": 1, + "size": 1892, + "digest": "sha256:5123dec16a2195b13299d7f4b9fd27b089b3f693fc189bc0fb83efcb60e69f55" + }, + "artifacts": { + "ao_eri": { + "path": "calculations/calculation-0/arrays/integrals/ao-eri.npy", + "size": 176, + "digest": "sha256:d838095f0921ded52c58e791ff1ad0fc558dc1b4b43aae62c5be45e19c9735be", + "representation": "rustiq-compact-eri-v1", + "attributes": { + "basis_functions": 2, + "computation_version": 1 + } + } + } + } + ] +} diff --git a/crates/rustiq-core/tests/data/persistence/portable-python-angstrom-v1.rustiq.hex b/crates/rustiq-core/tests/data/persistence/portable-python-angstrom-v1.rustiq.hex index 3c1e9b84..e9a510d3 100644 --- a/crates/rustiq-core/tests/data/persistence/portable-python-angstrom-v1.rustiq.hex +++ b/crates/rustiq-core/tests/data/persistence/portable-python-angstrom-v1.rustiq.hex @@ -1 +1 @@ -504b030414000000080000002100039cbc2966010000ad0200000d0000006d616e69666573742e6a736f6e8d914d6edc300c85f7730ac3ebb69068fd669d03f4060145511db51e7b22cb45a7c1dcbdf24c0a07290214da488f8fe447eae5d0757d9acb096bffd0f5655d6a7efe7ce6b2e4a5f244dc7fda1d4f3f377d9e9a53dee41f798a5bda792e15c3f86a3e9739aec4a5455edabb29139e78f37dbdd4e33c758f4cf9846397a7ca656ecd30e431d74b97f2afba967b9996b6b7eb65dfa4ebadfc4299a79a53a6a71cb75bbdec9dde113625e66fbcdc865b8e08da3ca841876483550e4414c1d0c06823468d1a8414c86248682c810e3088280715da61e224140c817792c2cfebbdf66bf733d6e36d8df7c097ef4b43f96798bf644bfebd6dc5d8e12354f03a45024b3638214142a0a41ca38f6d00ef935682d020a8864dd106159db50a74730aa7fd1b54c291d611eb9de01dee9be0ff214bafcd47cc3ea2f3e878a0e8a1d1920150469b4418bc2308883e38655cd4522101c7046d8220b5060f8276662ced9791eab2115f0fd7c31f504b030414000000080000002100db07d4a038010000a10200000c000000726571756573742e6a736f6ea5524d4fc3300cbdef575439b753bb2140fc018e1ce0364d55d67aad453e3ac701b669ff9d34cd3684342e5c92f8f9c5f67bc9719665626b494b164f9920ef187705c1ce8363918fd90f2087d6847415636f90dd4896a6734c564f34c9568ff02a0459768c6b82b1a98dd71ba0738d9819ac439e0aaf129665e5bccc6f06e9bc8efb29ff67a387bb6bf1a29c57cb3f7a85751d5536bda40e42a53899d05e310e0a1be4fdc5a08d74180d7a7d7b2996cf933dfd3620d3b04203f7b68d167ab6224fa8fcaa9181e438eb78bf2a937cd158135ea103d340cd3d81ebad1aef5750948f89a3d080a4ba85014c1b98fb9b4c20ac5dd37f4a3afce6548bc469115dedf0304abd4f5817fe84bba808c03b9aa8a2b104752f352ab606a51157eb81d8d34626fb8d57eae7e3c53601df4ae5609660a187c5997b9a7d03504b0304140000000800000021002bb22568fd010000a40700001000000063616c63756c6174696f6e2e6a736f6eed55c972a33010bdfb2b28ced8a58535bf927251b22c1bd5208991442689cbff3e2dc138d81367d6633880ba79bdbd6ee8d32a49d283b18af9f42149ede8bcfcbae6ace763cfbc343acd02e24958178487044779d4d2bb60b0339d9d20cc1b15548f2024c929de67b5e4ad1ed54ed81ff6f1cd609cf493d3c75997246883b2bbc27cdec6e739fba74078439b9a3615a584108acaa67e0bb5461b92176599d3a62408d302e10ff280fb3632c03b668f02a2c4ac5335f65e0ebde4d2bf5c88db3127df61890bed63d67fc8c4a5466ed460347871574e4e975360481fa1a9b6554601705457c8e838fb505c48dbe5ab74b05201c34fc2fde4f2742501563c4f7902926e725290023538cf6e61dc88c301a89b90d08faa6c684e8b02550d2a7073853f67bf1d136d4a421b4c2bb26cf8bda06555d3ba2e70899abc82c3df47c525981739fa75a5a8a67945cb0a634831c7b7a52e9bb0bad5beff65bc335cff63fa3f672f5c9fb377e76fd81dc0e59458aa84efcc3e2e185067b3923db7120633ae99d03d8ce6ae435e1a36ce51682e5adf59e13ad307732cd668a62eeda51630507b3108bd07e4cb5da4b0b275bcfbc6eceb2d069319b397d2b54ebe86dfb71efb7e561f47e1dca50e507c913ad6c18d156dc794ecbdd192e9f46ddb08eb47bb63f3c609ce2ecc9ca7c5309045949499366418bea2d147b376b16e57e7d577504b0102140314000000080000002100039cbc2966010000ad0200000d0000000000000000000000a481000000006d616e69666573742e6a736f6e504b0102140314000000080000002100db07d4a038010000a10200000c0000000000000000000000a48191010000726571756573742e6a736f6e504b01021403140000000800000021002bb22568fd010000a4070000100000000000000000000000a481f302000063616c63756c6174696f6e2e6a736f6e504b05060000000003000300b30000001e0500000000 +504b0304140000000800000021002758fe798f010000680300000d0000006d616e69666573742e6a736f6e95924d6edb301085f73e85a0b5d35023fe66dd03641f14c17048d66c65c921e9a2aee1bb97928dd84d8b148100427cf338f33e89c755d3b4614a5b2ced43d3a67d2ef1e56ee7538eb9f8917cbbbe3a9e7fccfa345667b7c8dfe3e8e663bb2915b4c3c5bc4b93db934fb572acfbaa8cb8f5b3eff15036d3d87cf614b73834712c3e4d7518da38c4726842fc59f6e9dca61ebb8e6bbbb64aa7a53de140fb014bade45a7a5abce741b51a9740379e3b7669578b99a21f4b0c919ea39bdfcae135e49b89ddfaaabaf8d5e7e5fbe40d82900fbc173628abb806e69895d47b540e9d4001ac63e8591f502a0261a167aeebb9ad8f271f1887defaf6d2fbf49a2cf997fd79c64d9a1d96cd1b9a7cff07dafde5dca76fb9a65eff0f24c75ff37f90aa7f8f0e8c088e4091b29a75d081a5c0b547e32ab3314170462811782525a72c775a290ea23a9916e61f7437993f4c78b3fb186567847c0fd338d406b5efc919a8802401b81432105aa3092ca2b19a4bed44c791c0bb0015da76428001467f6362aa770ba9ccd7f2785ad479fdb23aad7e03504b030414000000080000002100db07d4a038010000a10200002700000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6ea5524d4fc3300cbdef575439b753bb2140fc018e1ce0364d55d67aad453e3ac701b669ff9d34cd3684342e5c92f8f9c5f67bc9719665626b494b164f9920ef187705c1ce8363918fd90f2087d6847415636f90dd4896a6734c564f34c9568ff02a0459768c6b82b1a98dd71ba0738d9819ac439e0aaf129665e5bccc6f06e9bc8efb29ff67a387bb6bf1a29c57cb3f7a85751d5536bda40e42a53899d05e310e0a1be4fdc5a08d74180d7a7d7b2996cf933dfd3620d3b04203f7b68d167ab6224fa8fcaa9181e438eb78bf2a937cd158135ea103d340cd3d81ebad1aef5750948f89a3d080a4ba85014c1b98fb9b4c20ac5dd37f4a3afce6548bc469115dedf0304abd4f5817fe84bba808c03b9aa8a2b104752f352ab606a51157eb81d8d34626fb8d57eae7e3c53601df4ae5609660a187c5997b9a7d03504b0304140000000800000021002bb22568fd010000a40700002b00000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6eed55c972a33010bdfb2b28ced8a58535bf927251b22c1bd5208991442689cbff3e2dc138d81367d6633880ba79bdbd6ee8d32a49d283b18af9f42149ede8bcfcbae6ace763cfbc343acd02e24958178487044779d4d2bb60b0339d9d20cc1b15548f2024c929de67b5e4ad1ed54ed81ff6f1cd609cf493d3c75997246883b2bbc27cdec6e739fba74078439b9a3615a584108acaa67e0bb5461b92176599d3a62408d302e10ff280fb3632c03b668f02a2c4ac5335f65e0ebde4d2bf5c88db3127df61890bed63d67fc8c4a5466ed460347871574e4e975360481fa1a9b6554601705457c8e838fb505c48dbe5ab74b05201c34fc2fde4f2742501563c4f7902926e725290023538cf6e61dc88c301a89b90d08faa6c684e8b02550d2a7073853f67bf1d136d4a421b4c2bb26cf8bda06555d3ba2e70899abc82c3df47c525981739fa75a5a8a67945cb0a634831c7b7a52e9bb0bad5beff65bc335cff63fa3f672f5c9fb377e76fd81dc0e59458aa84efcc3e2e185067b3923db7120633ae99d03d8ce6ae435e1a36ce51682e5adf59e13ad307732cd668a62eeda51630507b3108bd07e4cb5da4b0b275bcfbc6eceb2d069319b397d2b54ebe86dfb71efb7e561f47e1dca50e507c913ad6c18d156dc794ecbdd192e9f46ddb08eb47bb63f3c609ce2ecc9ca7c5309045949499366418bea2d147b376b16e57e7d577504b01021403140000000800000021002758fe798f010000680300000d0000000000000000000000a481000000006d616e69666573742e6a736f6e504b0102140314000000080000002100db07d4a038010000a1020000270000000000000000000000a481ba01000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6e504b01021403140000000800000021002bb22568fd010000a40700002b0000000000000000000000a4813703000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6e504b05060000000003000300e90000007d0500000000 diff --git a/crates/rustiq-core/tests/data/persistence/portable-python-big-v1.rustiq.hex b/crates/rustiq-core/tests/data/persistence/portable-python-big-v1.rustiq.hex index a712eac2..906e0cfa 100644 --- a/crates/rustiq-core/tests/data/persistence/portable-python-big-v1.rustiq.hex +++ b/crates/rustiq-core/tests/data/persistence/portable-python-big-v1.rustiq.hex @@ -1 +1 @@ -504b03042d00000008000000210076c69d7fffffffffffffffff0d0014006d616e69666573742e6a736f6e01001000d3040000000000004c020000000000008d54db6edb300c7def5704796e13dd2ffd8abd0f43208954ab2eb153592e9615f9f7d1b9d4418b02030cc3268fc873a823bddf2d16cbdcd75d68cbc7c5b28e432baf0f7bac43191a760997f73362f336c5fb8e90fc14fe5d3a9896edfbda42dc5ec0fbdac398b052e69dfe29d2851d4eb81f87f6dc778bd235ac3d3509b16c4b3b2c72f9d3c67a5e4ef0b9cd922f29743c951d52c1ae955cd2a6c0f4d50e73874fcc2802e5098793a8e139086d1e5906614360d9736f9de1460b91a2514c3a1e0544692207c018bdcd328396c13169846596db64529a99547c1dcfb52fddf7a13d9fc6774eac5e06a2f245cc95d950fe4ed330c67c4735b26cc0e9e44d663e092958d4d6aac485d23a0606a8b8972958cd6322008512701098912991cc4c35856d1ab7a19d197ca27b93fc3fcadc79f11d67cd85044cdc04c1bd8e5c0aefc16615fd34f7c89c8f321b2f73a22a31b11c9dc44c1bc0d0f8acf5cd46f7634d3850ed9fa75e67d694e86b792a5dd86eae7e5aadd6f4dcca68fd6e7b917123f45271cde6d4178db34aa73e225f4d84ca2913658220d17a411e8200ca062b3293641fcd93d64e29ce910967b940149e7ca6b853999d768674d2fbd7496ba8e4e890da30ef4ee83758cbc7ff8d8c506b380cebe9f83cd5b01dd6a17f20e8aadb1f969f45706bbe570110815c0568795682590087c1442d31390bca41969a4314ce43b22404325308d63be7748698e76e15f715073a8d578b5daf90d4eff6a46be2f7f0c6e705a1b55ae2d870b85148f118e8c6d9e4b14b53a12929eee7ec546d3cf7b8bd842e80e3c7508f77c7bb7f504b03042d000000080000002100021adb15ffffffffffffffff1000140063616c63756c6174696f6e2e6a736f6e010010006407000000000000e201000000000000ed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b03042d0000000800000021004100ae2bffffffffffffffff0c001400726571756573742e6a736f6e010010009a020000000000003001000000000000a592c14ec3300c86ef7b8a28e76e6a07428817e0b803dca6a94a536fb14893ce49806ddabb93a4d940489390b8a4f5efcfb1fdb7a719637c6b69109e3f314ec179dccf09f6019ce755cabe0339b426a69b1c0783de25b8b38a2644783b24691d03c64ef92c32cad684a103bad4e7cc681dfae9d275d118ab1775753328ef9bfc3c57ff6ad42ceeffda289e9bbca254827610afc9341f82f6386a94e80f57673ae1303bf3f2ba9adf3d4fdea86d54a649f9005ed93e112278cbaba28acf163d904883a6faa62e2371694db47f074642eb15815356a7fa06e6f56361341a10d4f63082e92379b8490261eba4fa1074fccd34cbc2f488ae75784cab3e146d177f0677dd220a6f68f216d212b44a0ca8bd35280cfff61dc807ea44f1de04ad7f7eb9dc26ea5ba11dcc8acc87717961cfb32f504b03042d000000080000002100564c8e8fffffffffffffffff09001400736f75726365732f300100100054000000000000005400000000000000535670ad484c2e5128ce2f2d4a4e552828ca2f4bcd4bcc4b4eb55248cecfcd4dcd2b295648cc4b51700ef27153482c02a9482d4e2d2a4b4dd1e3e58ace488be5e5ca4d2dc9c84f51b05550722c2dc957e2e50200504b03042d000000000000002100feb6142cffffffffffffffff1b0014006172726179732f696e74656772616c732f616f2d6572692e6e707901001000b000000000000000b000000000000000934e554d5059010076007b276465736372273a20273e6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a3fe00000000000003ff80000000000004004000000000000400c00000000000040120000000000004016000000000000504b01022d032d00000008000000210076c69d7fffffffffffffffff0d0014000000000000000000a481000000006d616e69666573742e6a736f6e01001000d3040000000000004c02000000000000504b01022d032d000000080000002100021adb15ffffffffffffffff10001c000000000000000000a481ffffffff63616c63756c6174696f6e2e6a736f6e010018006407000000000000e2010000000000008b02000000000000504b01022d032d0000000800000021004100ae2bffffffffffffffff0c001c000000000000000000a481ffffffff726571756573742e6a736f6e010018009a020000000000003001000000000000af04000000000000504b01022d032d000000080000002100564c8e8fffffffffffffffff09001c000000000000000000a481ffffffff736f75726365732f3001001800540000000000000054000000000000001d06000000000000504b01022d032d000000000000002100feb6142cffffffffffffffff1b001c000000000000000000a481ffffffff6172726179732f696e74656772616c732f616f2d6572692e6e707901001800b000000000000000b000000000000000ac06000000000000504b06062c000000000000002d002d00000000000000000005000000000000000500000000000000b701000000000000a907000000000000504b060700000000600900000000000001000000504b05060000000005000500b7010000a90700000000 +504b03042d000000080000002100bd8db22dffffffffffffffff0d0014006d616e69666573742e6a736f6e01001000d5050000000000006d020000000000009594db72da301086ef790ac6d701743ee4297adfe93092564ad41a9b4872a634c3bb5736104cd2a6cd0cc3d8ab5fffeeb76be965b15c36a14f3b539afb6593865ce2d36aef538eb9f8cef9e6eeaad83e8ff1beab4a3c857fc40ec66dfb3e1563dbb3789f7a189c4f75e5a5bed74867767ed47d3994c7be5bc6aef8d4d724c6c63696c332c49f6548a7ed557e4dd3e0a6868e936dee87e47caed1af93ece45d17fa141f6267daed25cd7abda93f675a37b4a654a775e977edd97d2cd094c7517776dca0ebd21bc22996e3afd156b1d708c4079fa786e54743b8b8479e29262c7560a8979a7042c000934692802880e7d871ae18c3d823a22426de134d9c6558b1804433391febffb7897556fb1f80e3d4f499663523c82efaaec410dd36c2f8540eaf83f81be21f910210690c0a1a6ba9041695c959c11055d812b054585cc1acd532d0009c1a85a82012492c9d70ae397b1f5f2b4bfe6938e598557399c59c787383b639ef5b7fcfb5eabb7f819ca72584f888cea2204071a74540da114a90e552328709e3dc1a049e614d9d911c5b570535e40003f1c123469c784f37abf9d384f34ff553945869f21126c784827758188235b79812ad410666f5385d8b94b634084d83ab4ed6a16015f5a18e1979a103e7ef314daadf967125df429a7eeb53bc89fd1fba49c91cf266bc121e9269f3c6f4abeab4eef687591766c452dc84df210358a8f3042f7160044900e58db09c7aa724300581720c96280d4ed6d30801310f522ba578001b6eb326bf4f3ed7637419ece58a74fd6e5fdb30d6ba7ac6b79b4c2929daa1f8fca62175cd9a7ab36ec3d0b9cbe12677b78ad17938e59b5fb833d171f1f6e978bd3f16c7c56f504b03042d000000080000002100021adb15ffffffffffffffff2b00140063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6e010010006407000000000000e201000000000000ed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b03042d0000000800000021004100ae2bffffffffffffffff2700140063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6e010010009a020000000000003001000000000000a592c14ec3300c86ef7b8a28e76e6a07428817e0b803dca6a94a536fb14893ce49806ddabb93a4d940489390b8a4f5efcfb1fdb7a719637c6b69109e3f314ec179dccf09f6019ce755cabe0339b426a69b1c0783de25b8b38a2644783b24691d03c64ef92c32cad684a103bad4e7cc681dfae9d275d118ab1775753328ef9bfc3c57ff6ad42ceeffda289e9bbca254827610afc9341f82f6386a94e80f57673ae1303bf3f2ba9adf3d4fdea86d54a649f9005ed93e112278cbaba28acf163d904883a6faa62e2371694db47f074642eb15815356a7fa06e6f56361341a10d4f63082e92379b8490261eba4fa1074fccd34cbc2f488ae75784cab3e146d177f0677dd220a6f68f216d212b44a0ca8bd35280cfff61dc807ea44f1de04ad7f7eb9dc26ea5ba11dcc8acc87717961cfb32f504b03042d000000080000002100564c8e8fffffffffffffffff09001400736f75726365732f300100100054000000000000005400000000000000535670ad484c2e5128ce2f2d4a4e552828ca2f4bcd4bcc4b4eb55248cecfcd4dcd2b295648cc4b51700ef27153482c02a9482d4e2d2a4b4dd1e3e58ace488be5e5ca4d2dc9c84f51b05550722c2dc957e2e50200504b03042d000000000000002100feb6142cffffffffffffffff3600140063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f6172726179732f696e74656772616c732f616f2d6572692e6e707901001000b000000000000000b000000000000000934e554d5059010076007b276465736372273a20273e6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a3fe00000000000003ff80000000000004004000000000000400c00000000000040120000000000004016000000000000504b01022d032d000000080000002100bd8db22dffffffffffffffff0d0014000000000000000000a481000000006d616e69666573742e6a736f6e01001000d5050000000000006d02000000000000504b01022d032d000000080000002100021adb15ffffffffffffffff2b001c000000000000000000a481ffffffff63616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6e010018006407000000000000e201000000000000ac02000000000000504b01022d032d0000000800000021004100ae2bffffffffffffffff27001c000000000000000000a481ffffffff63616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6e010018009a020000000000003001000000000000eb04000000000000504b01022d032d000000080000002100564c8e8fffffffffffffffff09001c000000000000000000a481ffffffff736f75726365732f3001001800540000000000000054000000000000007406000000000000504b01022d032d000000000000002100feb6142cffffffffffffffff36001c000000000000000000a481ffffffff63616c63756c6174696f6e732f63616c63756c6174696f6e2d302f6172726179732f696e74656772616c732f616f2d6572692e6e707901001800b000000000000000b0000000000000000307000000000000504b06062c000000000000002d002d0000000000000000000500000000000000050000000000000008020000000000001b08000000000000504b060700000000230a00000000000001000000504b05060000000005000500080200001b0800000000 diff --git a/crates/rustiq-core/tests/data/persistence/portable-python-little-v1.rustiq.hex b/crates/rustiq-core/tests/data/persistence/portable-python-little-v1.rustiq.hex index d3b26d4a..33045ae7 100644 --- a/crates/rustiq-core/tests/data/persistence/portable-python-little-v1.rustiq.hex +++ b/crates/rustiq-core/tests/data/persistence/portable-python-little-v1.rustiq.hex @@ -1 +1 @@ -504b03042d000000080000002100001eb9fdffffffffffffffff0d0014006d616e69666573742e6a736f6e01001000d3040000000000004b020000000000008d54db6edb300c7def5704796e13dd2ffd8abd0f43404954ab2eb153592e9615f9f7d1b9d4418b02030c43228fc873484aef778bc532f775076df9b858d67168e5f5618f752843c32ee2f27e466cde267bdf11929fccbf4b97a663fbbe3608db0b785ffb3446ace479a73d593ad8e184fb7168cf7db7285dc3da531208655bda6191cb9f36d6f37182cf69967c49a6e329ec100b76ade41237254dab7698337c624696549e7038891a9e4168f3c87212168065cfbd75861b2d440c4631e97810294813784a1882b759e6a42538268db0cc721b4d8c33938aafe339f625fb1edaf3a97c67c7ea65202a5fc45c990de5ef540d63cc775403cb26391dbdc9cc4721050bda5a15b9505a07600915f73282d53c44029029269e0466644a4433538db08de316da99c127ba37ceffa3cc9d17df71d65cc884911b10dcebc0a5f03ed9ac829fea1e98f34166e3658e14254496839398a9010c8dcf5adf34ba1f6bc48162ff3ce53ab326475fcb53e960bbb9ced36ab5a6ef5646eb77db8b8c1ba197886b36bbbe689c553af561f93a44a89c3241c60412ad1734430992b260456692c647f3a8b5538a7364c2592e1085a73953dca9cc4e9d219df4ff75d20a95261a621be6ee40bfc15a3ef63732a056380cebe9fa3c55d80e6be81f08baeaf687e56711dc9aef5524271df39a864bf084498ba81da9e13973482c4f0a52e441052501d0903ba0d2c87df456d27ace56715f71a0db781db1eb1312fbdd9e744dfc1edef87c005aab258c0d871b85640f402fce268f5d9c024d4e713f7ba768e339c7ed2374011c3f8a7abc3bdefd03504b03042d000000080000002100021adb15ffffffffffffffff1000140063616c63756c6174696f6e2e6a736f6e010010006407000000000000e201000000000000ed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b03042d0000000800000021004100ae2bffffffffffffffff0c001400726571756573742e6a736f6e010010009a020000000000003001000000000000a592c14ec3300c86ef7b8a28e76e6a07428817e0b803dca6a94a536fb14893ce49806ddabb93a4d940489390b8a4f5efcfb1fdb7a719637c6b69109e3f314ec179dccf09f6019ce755cabe0339b426a69b1c0783de25b8b38a2644783b24691d03c64ef92c32cad684a103bad4e7cc681dfae9d275d118ab1775753328ef9bfc3c57ff6ad42ceeffda289e9bbca254827610afc9341f82f6386a94e80f57673ae1303bf3f2ba9adf3d4fdea86d54a649f9005ed93e112278cbaba28acf163d904883a6faa62e2371694db47f074642eb15815356a7fa06e6f56361341a10d4f63082e92379b8490261eba4fa1074fccd34cbc2f488ae75784cab3e146d177f0677dd220a6f68f216d212b44a0ca8bd35280cfff61dc807ea44f1de04ad7f7eb9dc26ea5ba11dcc8acc87717961cfb32f504b03042d000000080000002100564c8e8fffffffffffffffff09001400736f75726365732f300100100054000000000000005400000000000000535670ad484c2e5128ce2f2d4a4e552828ca2f4bcd4bcc4b4eb55248cecfcd4dcd2b295648cc4b51700ef27153482c02a9482d4e2d2a4b4dd1e3e58ace488be5e5ca4d2dc9c84f51b05550722c2dc957e2e50200504b03042d000000000000002100db95d32dffffffffffffffff1b0014006172726179732f696e74656772616c732f616f2d6572692e6e707901001000b000000000000000b000000000000000934e554d5059010076007b276465736372273a20273c6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a000000000000e03f000000000000f83f00000000000004400000000000000c4000000000000012400000000000001640504b01022d032d000000080000002100001eb9fdffffffffffffffff0d0014000000000000000000a481000000006d616e69666573742e6a736f6e01001000d3040000000000004b02000000000000504b01022d032d000000080000002100021adb15ffffffffffffffff10001c000000000000000000a481ffffffff63616c63756c6174696f6e2e6a736f6e010018006407000000000000e2010000000000008a02000000000000504b01022d032d0000000800000021004100ae2bffffffffffffffff0c001c000000000000000000a481ffffffff726571756573742e6a736f6e010018009a020000000000003001000000000000ae04000000000000504b01022d032d000000080000002100564c8e8fffffffffffffffff09001c000000000000000000a481ffffffff736f75726365732f3001001800540000000000000054000000000000001c06000000000000504b01022d032d000000000000002100db95d32dffffffffffffffff1b001c000000000000000000a481ffffffff6172726179732f696e74656772616c732f616f2d6572692e6e707901001800b000000000000000b000000000000000ab06000000000000504b06062c000000000000002d002d00000000000000000005000000000000000500000000000000b701000000000000a807000000000000504b0607000000005f0900000000000001000000504b05060000000005000500b7010000a80700000000 +504b03042d000000080000002100caae2935ffffffffffffffff0d0014006d616e69666573742e6a736f6e01001000d5050000000000006d020000000000009594d96edb301045dffd15869e639bfb92afe87b51185c86095b5972482aa81bf8df4bc9762c276dda0082400d2fefcce1887c592c974de8d3ce94e67ed9a42197f8b4da43ca3117e81c347757c5f6798cf75d55e229fc23767e5cb6ef5331b63d8bf7a9f7838354675eea778d746607a3eecba13cf6dd327605525f93181bdb580ecb107f96219d9657f9354d839b1a3a4eb6b91f92835ca35f27d9c9bb4ef4293ec4ceb4db4b9af57a531f675a37b4a654a775e977edd97d2cd094c7517776dca0ebd41bc22996e3afd156b1d7888f0f90a70dcb8f8670718f8029262c75de50909a7042bcf14c1a4902a2de03c78e73c518c68088929800104d9c6558b1804433391febfbdbc43aabfd0fc071daf499663523c82e425762886e1bfd382a87d746fc0df18f48c113690c0a1a6ba9041695c959c11055d8126fa9b0b88259ab65a0c1736a14a2824824b174c2b9e6ec7d7cad2cc1d370ca31abe6d28b39f1e6066d735eb7fe9e6bd577ff0239774b08f1119d454178c59d160169472841964bc91c268c736b900786357546726c5d15d490f3d81308801871e23dddace64f13ce7fd54f5162a5c947981c13eac1616108d6dc624ab4f63230abc7ee5aa4b4a541681a5c75b20e05ab2884da66044207cedf639a54ff2de34abe8534fd1652bc89fd1fba49c91cf266bc121e9269f3c6f4abeab4eef687d92ecc88a5b809bf43f68a2aa4796d2bc11e3c278eab7a247108d87814c663e81db6cc326a0c883a6d8171c0da6949ebf8366b827d825c8fd1a5b1972bd2f5bb7ddd86b1d6d533be5d644a49d10e05f29b0da973d6d49b751b86ce5d0e37b9bb558ccec329dffcc29d898e8bb7a3e3f5fe581c17bf01504b03042d000000080000002100021adb15ffffffffffffffff2b00140063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6e010010006407000000000000e201000000000000ed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b03042d0000000800000021004100ae2bffffffffffffffff2700140063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6e010010009a020000000000003001000000000000a592c14ec3300c86ef7b8a28e76e6a07428817e0b803dca6a94a536fb14893ce49806ddabb93a4d940489390b8a4f5efcfb1fdb7a719637c6b69109e3f314ec179dccf09f6019ce755cabe0339b426a69b1c0783de25b8b38a2644783b24691d03c64ef92c32cad684a103bad4e7cc681dfae9d275d118ab1775753328ef9bfc3c57ff6ad42ceeffda289e9bbca254827610afc9341f82f6386a94e80f57673ae1303bf3f2ba9adf3d4fdea86d54a649f9005ed93e112278cbaba28acf163d904883a6faa62e2371694db47f074642eb15815356a7fa06e6f56361341a10d4f63082e92379b8490261eba4fa1074fccd34cbc2f488ae75784cab3e146d177f0677dd220a6f68f216d212b44a0ca8bd35280cfff61dc807ea44f1de04ad7f7eb9dc26ea5ba11dcc8acc87717961cfb32f504b03042d000000080000002100564c8e8fffffffffffffffff09001400736f75726365732f300100100054000000000000005400000000000000535670ad484c2e5128ce2f2d4a4e552828ca2f4bcd4bcc4b4eb55248cecfcd4dcd2b295648cc4b51700ef27153482c02a9482d4e2d2a4b4dd1e3e58ace488be5e5ca4d2dc9c84f51b05550722c2dc957e2e50200504b03042d000000000000002100db95d32dffffffffffffffff3600140063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f6172726179732f696e74656772616c732f616f2d6572692e6e707901001000b000000000000000b000000000000000934e554d5059010076007b276465736372273a20273c6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a000000000000e03f000000000000f83f00000000000004400000000000000c4000000000000012400000000000001640504b01022d032d000000080000002100caae2935ffffffffffffffff0d0014000000000000000000a481000000006d616e69666573742e6a736f6e01001000d5050000000000006d02000000000000504b01022d032d000000080000002100021adb15ffffffffffffffff2b001c000000000000000000a481ffffffff63616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6e010018006407000000000000e201000000000000ac02000000000000504b01022d032d0000000800000021004100ae2bffffffffffffffff27001c000000000000000000a481ffffffff63616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6e010018009a020000000000003001000000000000eb04000000000000504b01022d032d000000080000002100564c8e8fffffffffffffffff09001c000000000000000000a481ffffffff736f75726365732f3001001800540000000000000054000000000000007406000000000000504b01022d032d000000000000002100db95d32dffffffffffffffff36001c000000000000000000a481ffffffff63616c63756c6174696f6e732f63616c63756c6174696f6e2d302f6172726179732f696e74656772616c732f616f2d6572692e6e707901001800b000000000000000b0000000000000000307000000000000504b06062c000000000000002d002d0000000000000000000500000000000000050000000000000008020000000000001b08000000000000504b060700000000230a00000000000001000000504b05060000000005000500080200001b0800000000 diff --git a/crates/rustiq-core/tests/data/persistence/portable-python-multi-v1.rustiq.hex b/crates/rustiq-core/tests/data/persistence/portable-python-multi-v1.rustiq.hex new file mode 100644 index 00000000..1a4e2268 --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/portable-python-multi-v1.rustiq.hex @@ -0,0 +1 @@ +504b030414000000080000002100564c8e8f540000005400000009000000736f75726365732f30535670ad484c2e5128ce2f2d4a4e552828ca2f4bcd4bcc4b4eb55248cecfcd4dcd2b295648cc4b51700ef27153482c02a9482d4e2d2a4b4dd1e3e58ace488be5e5ca4d2dc9c84f51b05550722c2dc957e2e50200504b030414000000080000002100021adb15e2010000640700002b00000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6eed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b0304140000000800000021004100ae2b300100009a0200002700000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6ea592c14ec3300c86ef7b8a28e76e6a07428817e0b803dca6a94a536fb14893ce49806ddabb93a4d940489390b8a4f5efcfb1fdb7a719637c6b69109e3f314ec179dccf09f6019ce755cabe0339b426a69b1c0783de25b8b38a2644783b24691d03c64ef92c32cad684a103bad4e7cc681dfae9d275d118ab1775753328ef9bfc3c57ff6ad42ceeffda289e9bbca254827610afc9341f82f6386a94e80f57673ae1303bf3f2ba9adf3d4fdea86d54a649f9005ed93e112278cbaba28acf163d904883a6faa62e2371694db47f074642eb15815356a7fa06e6f56361341a10d4f63082e92379b8490261eba4fa1074fccd34cbc2f488ae75784cab3e146d177f0677dd220a6f68f216d212b44a0ca8bd35280cfff61dc807ea44f1de04ad7f7eb9dc26ea5ba11dcc8acc87717961cfb32f504b030414000000000000002100db95d32db0000000b00000003600000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f6172726179732f696e74656772616c732f616f2d6572692e6e7079934e554d5059010076007b276465736372273a20273c6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a000000000000e03f000000000000f83f00000000000004400000000000000c4000000000000012400000000000001640504b030414000000080000002100021adb15e2010000640700002b00000063616c63756c6174696f6e732f63616c63756c6174696f6e2d312f63616c63756c6174696f6e2e6a736f6eed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b0304140000000800000021004100ae2b300100009a0200002700000063616c63756c6174696f6e732f63616c63756c6174696f6e2d312f726571756573742e6a736f6ea592c14ec3300c86ef7b8a28e76e6a07428817e0b803dca6a94a536fb14893ce49806ddabb93a4d940489390b8a4f5efcfb1fdb7a719637c6b69109e3f314ec179dccf09f6019ce755cabe0339b426a69b1c0783de25b8b38a2644783b24691d03c64ef92c32cad684a103bad4e7cc681dfae9d275d118ab1775753328ef9bfc3c57ff6ad42ceeffda289e9bbca254827610afc9341f82f6386a94e80f57673ae1303bf3f2ba9adf3d4fdea86d54a649f9005ed93e112278cbaba28acf163d904883a6faa62e2371694db47f074642eb15815356a7fa06e6f56361341a10d4f63082e92379b8490261eba4fa1074fccd34cbc2f488ae75784cab3e146d177f0677dd220a6f68f216d212b44a0ca8bd35280cfff61dc807ea44f1de04ad7f7eb9dc26ea5ba11dcc8acc87717961cfb32f504b030414000000000000002100feb6142cb0000000b00000003600000063616c63756c6174696f6e732f63616c63756c6174696f6e2d312f6172726179732f696e74656772616c732f616f2d6572692e6e7079934e554d5059010076007b276465736372273a20273e6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a3fe00000000000003ff80000000000004004000000000000400c00000000000040120000000000004016000000000000504b030414000000080000002100e1699ac2b4020000fd0900000d0000006d616e69666573742e6a736f6ee556cb6e1b3910bcfb2b049d6d89cd37f315b9070b8164376d6ee41985c309561be8df97a3873db2b34e72c82109204854b158ddd53d6ce9cbcd62b14c7d79f475f96eb12ce350f3a7bb1d95210f95ba48cbdb67c6e6f384f75d63c211fe983b9c8eedfa527dd89ec9bbd2e318a9b49d2fed7b433aff4813effdbe3ef4dd2277954adf82f890b7b9ee1729ff53c7723adee8cf6196b06cd0e1283bf463893434f4c39176d26e1b7dc9f7b9f3dbcd25cc6ab56eafe8b771dcfada9456b57fdc9ed5a7047d7d987867c5357bde7ae1f0880df9df49d6ca2704f33d0dc7820d0f9e2bfd8e91b4520711d10b328e2bced1a334def0c4042229884a59290188716b801371c763906065627a79543eb4f7bf8e5e67b97fc5703e167dc6b99b391862a6aee694e326e3b4aafba746fc9fc5af5a4ac88df72c3970c66ad0cd530c5a326121700c420768c642702689844a78cb84e686193051c7b83c6b1f9e322bf4693cc5986573e9c5dcf1facadafa7c6ef5f7d0b2befd969173b7b4d66fb90b2c69b42a3a9d988b5c701694313202974a05cf90243811bd5110622334282220a7444cf2a85fbb9be5fcc30ee78fea0fb904ebf85b3615708114417b0e4e0510dc3934490637753730eb8248da89149b52882c052b28b53633d22e29f5daa62fedd9f2b10ed7267dbfa192afb0efb3ee4bf1fb613d8d84fbe2b7c3daf7774d69d5edf6b32acc1c1b7d05bfb28c5658e6546b2b0724543c2adbae24a4041e599aae2146083248e13de9b61d482a02179d116d7d1db5d0aed0d0aed1a5b1971119fbc75d2bc394ebdd67b83ee46b2d398c95861705697bc1b7c9ba4963172f979bdf5e3326e5f1146f3e7067a4c3cdcbd5e9f3dca537e604fcbe73027efb39017fee9c809f30273060eb2792812439338896bc0e4a50b406a5c5241460e0d66134ed571b139384c6596b55c2907ee53931fdcfb839dcfc07504b0102140314000000080000002100564c8e8f5400000054000000090000000000000000000000a48100000000736f75726365732f30504b0102140314000000080000002100021adb15e2010000640700002b0000000000000000000000a4817b00000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6e504b01021403140000000800000021004100ae2b300100009a020000270000000000000000000000a481a602000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6e504b0102140314000000000000002100db95d32db0000000b0000000360000000000000000000000a4811b04000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f6172726179732f696e74656772616c732f616f2d6572692e6e7079504b0102140314000000080000002100021adb15e2010000640700002b0000000000000000000000a4811f05000063616c63756c6174696f6e732f63616c63756c6174696f6e2d312f63616c63756c6174696f6e2e6a736f6e504b01021403140000000800000021004100ae2b300100009a020000270000000000000000000000a4814a07000063616c63756c6174696f6e732f63616c63756c6174696f6e2d312f726571756573742e6a736f6e504b0102140314000000000000002100feb6142cb0000000b0000000360000000000000000000000a481bf08000063616c63756c6174696f6e732f63616c63756c6174696f6e2d312f6172726179732f696e74656772616c732f616f2d6572692e6e7079504b0102140314000000080000002100e1699ac2b4020000fd0900000d0000000000000000000000a481c30900006d616e69666573742e6a736f6e504b0506000000000800080096020000a20c00000000 diff --git a/crates/rustiq-core/tests/portable_artifact.rs b/crates/rustiq-core/tests/portable_artifact.rs index 99aed97f..a12d19ed 100644 --- a/crates/rustiq-core/tests/portable_artifact.rs +++ b/crates/rustiq-core/tests/portable_artifact.rs @@ -68,3 +68,81 @@ fn public_api_creates_inspects_and_recovers_a_portable_artifact() { assert_eq!(eri[index], value); } } + +#[test] +fn public_bundle_api_preserves_restructured_options_for_multiple_calculations() { + use rustiq_core::{ + config::{ + validated::{DiisSize, NonNegativeFiniteF64}, + DiisConfig, HfConfig, IntegralConfig, OrthogonalizationConfig, + }, + persistence::RustiQBundle, + }; + let geometry = Geometry::from_source("h2.xyz", "2\nH2\nH 0 0 0\nH 1.4 0 0\n").unwrap(); + let basis = BasisFile::from_reader(&include_bytes!("data/sto-3g.json")[..]).unwrap(); + let prepared = CalculationBuilder::new(&geometry, &basis) + .with_hf(HfConfig { + diis: DiisConfig { + enabled: true, + max_history: DiisSize::try_new(9).unwrap().into(), + }, + orthogonalization: OrthogonalizationConfig { + linear_dependency_threshold: NonNegativeFiniteF64::try_new(1e-7).unwrap().into(), + }, + ..Default::default() + }) + .with_integrals(IntegralConfig { + schwarz_threshold: None.into(), + }) + .prepare() + .unwrap(); + let mut bundle = RustiQBundle::new(vec![ + RustiQData::from_calculation(&prepared).unwrap(), + RustiQData::from_calculation(&prepared).unwrap(), + ]) + .unwrap(); + bundle + .add_source("input.ncl", b"opaque input".as_slice()) + .unwrap(); + bundle.calculations_mut()[1] + .set::(CompactEri::Zeroed(2)) + .unwrap(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("bundle.rustiq"); + bundle.write(&path).unwrap(); + let mut reopened = RustiQBundle::open(&path).unwrap(); + assert_eq!(reopened.sources().next().unwrap().bytes(), b"opaque input"); + for entry in reopened.calculations() { + let request = entry.request().unwrap(); + assert!(request.hf().diis.enabled); + assert_eq!(request.hf().diis.max_history.value.into_inner(), 9); + assert_eq!( + request + .hf() + .orthogonalization + .linear_dependency_threshold + .value + .into_inner(), + 1e-7 + ); + assert!(request.integrals().schwarz_threshold.value.is_none()); + assert_eq!(entry.calculation().unwrap().diis_size(), Some(9)); + assert_eq!( + entry.calculation().unwrap().linear_dependency_threshold(), + 1e-7 + ); + assert!(entry + .calculation() + .unwrap() + .eri_schwarz_threshold() + .is_none()); + } + assert!(reopened.calculations_mut()[0] + .get::() + .unwrap() + .is_none()); + assert!(reopened.calculations_mut()[1] + .get::() + .unwrap() + .is_some()); +} diff --git a/docs/persistence-format-v1.md b/docs/persistence-format-v1.md index 50f4dfe1..15b70bad 100644 --- a/docs/persistence-format-v1.md +++ b/docs/persistence-format-v1.md @@ -110,6 +110,8 @@ basis-function count from their typed manifest attributes; the NPY length is nev used to infer that scientific context. Matrix readers accept C and Fortran order and matrix writers emit Fortran order. +For the directory cache: + - `manifest.json` is UTF-8 JSON and describes the format, producer, scientific identity and artifacts. - `arrays/integrals/ao-eri.npy` is the AO electron-repulsion integral artifact. @@ -155,7 +157,7 @@ independent of its container. ## Generic artifact manifest -The root manifest is an index of versioned scientific artifacts rather than a +The cache root manifest, or each portable calculation entry, indexes scientific artifacts rather than a union of every scientific state RustiQ may ever persist. A representative entry has this shape: @@ -235,38 +237,60 @@ unchanged. ## Portable `.rustiq` V1 -A portable artifact is a ZIP archive, with ZIP64 support for large members and -large offsets. It contains `manifest.json`, independently versioned `request.json` -and `calculation.json`, optional opaque `sources/` payloads, and zero or more -indexed scientific artifacts. It is not an NPZ file. Readers access members by -name without extracting files. The active directory cache remains a separate, -disposable machine-local product with its existing format and behavior. +A portable artifact is a ZIP archive with ZIP64 support, containing a non-empty +collection of independently reusable calculations. Single-calculation archives +use the same V1 format with exactly one entry. The normative root schema is +[`portable-manifest-v1.schema.json`](../schemas/portable-manifest-v1.schema.json). +A [golden H2 manifest](../crates/rustiq-core/tests/data/persistence/portable-manifest-h2-v1.json) +records the ordinary single-calculation case. +The directory cache retains its existing single-artifact manifest and layout. + +```text +manifest.json +sources/0 optional shared opaque provenance +calculations/calculation-0/request.json normalized requested state +calculations/calculation-0/calculation.json resolved scientific state +calculations/calculation-0/arrays/integrals/ao-eri.npy +calculations/calculation-1/request.json +calculations/calculation-1/calculation.json +... +``` + +The root contains `format: "rustiq-persistence"`, `format_version: 1`, +`kind: "portable"`, `producer`, optional `sources`, and `calculations`. +Each calculation entry contains `id`, `scientific_identity`, `request`, +`calculation`, and `artifacts`. There is no root scientific identity or artifact +map. Compatibility remains specific to each calculation and artifact. -The portable manifest uses `kind: "portable"`. Its format/version, producer, -scientific identity and artifact envelope are the same as the logical format -above. It additionally requires separate `request` and `calculation` references: +Writers assign deterministic identifiers `calculation-0`, `calculation-1`, ... +in collection order. Readers accept unique `calculation-` identifiers. +Identifiers are references within a snapshot, not scientific identities; neither +identifiers, source order nor user labels enter scientific compatibility. +Request/resolved references and artifacts must stay inside their entry's +`calculations//` namespace. References are full archive member paths: ```json "request": { - "path": "request.json", + "path": "calculations/calculation-0/request.json", "version": 1, "size": 1234, "digest": "sha256:..." }, "calculation": { - "path": "calculation.json", + "path": "calculations/calculation-0/calculation.json", "version": 1, "size": 1234, "digest": "sha256:..." } ``` -`size` and `digest` describe the exact uncompressed JSON bytes, including any -whitespace. Each reference version selects its own snapshot schema; it is not -tied to the package version or the other snapshot version. Cache manifests omit -both references and source provenance. Producer version remains -provenance, not an invalidation rule. Archives may contain no numerical artifacts; -the scientific context remains useful by itself. +`size` and `digest` describe exact uncompressed bytes, including whitespace. +Each snapshot has an independently versioned schema. Producer version is +provenance, not an invalidation rule. An entry may contain no numerical artifacts. +A bundle represents the supplied calculation collection, not an append-only history. +Shared sources are stored once; numerical payload deduplication is not implemented. +The pre-release single-calculation portable layout is replaced by this V1 contract; +there is no migration reader for that unpublished layout. ### Normalized request and optional source provenance @@ -289,6 +313,11 @@ this deliberately asks preparation to choose a fresh seed. The unresolved reques keeps that state while `calculation.json` records the concrete seed actually used. Comments, source spans, source paths, cache configuration, and output/UI settings are excluded. +The V1 wire fields remain independent of Rust struct organization: integral screening +is encoded as `hf.eri_schwarz_threshold`, orthogonalization as +`hf.linear_dependency_threshold`, and DIIS as `hf.diis` / `hf.diis_size`. +The decoder maps these to `IntegralConfig`, `OrthogonalizationConfig`, and +`DiisConfig`, respectively. The basis label is informational; resolved AO contents define scientific identity. Sources are optional opaque bytes, including non-UTF-8 content. They are never @@ -378,6 +407,35 @@ restart state. ### Rust API +`RustiQBundle::new(Vec)` validates a non-empty portable collection. +`calculations()` and `calculations_mut()` expose entries in manifest order, with +independent typed artifact access and compatibility. The mutable slice cannot +change cardinality. `add_source` / `sources` manage shared provenance; construction +also lifts and deduplicates identical source name/byte pairs attached to entries. +`RustiQBundle::open` reads all request/resolved snapshots but leaves arrays lazy. +`write` publishes all supplied entries as one complete snapshot. + +```rust,ignore +use rustiq_core::persistence::{RustiQBundle, RustiQData}; + +let entries = prepared_calculations.iter() + .map(RustiQData::from_calculation) + .collect::, _>>()?; +let mut bundle = RustiQBundle::new(entries)?; +bundle.add_source("batch.ncl", original_source_bytes)?; +bundle.write("batch.rustiq")?; +let mut reopened = RustiQBundle::open("batch.rustiq")?; +for entry in reopened.calculations_mut() { + println!("{:?}", entry.calculation().expect("portable context").hf_method()); +} +``` + +The existing single-calculation `RustiQData::open/write` methods delegate to the +same bundle implementation. `open` rejects multi-calculation archives explicitly, +so callers cannot silently drop entries. Sources returned by `RustiQData::open` +are the single bundle's shared provenance. Bundle entries use `bundle.sources()`. + + `RustiQData::from_calculation(&PreparedCalculation)` captures requested and resolved inputs without executing HF. `set_eri` / `set::` supply an owned `CompactEri` whose dimension matches those inputs. The caller is responsible for @@ -448,7 +506,7 @@ directory after publication. Failures before publication leave the destination untouched and clean up the temporary file. A directory-sync error after publication can leave a complete output file and is still reported as an error. -Writers use `Stored` for numerical/opaque artifacts and `Deflated` for all three +Writers use `Stored` for numerical/opaque artifacts and `Deflated` for JSON members (`manifest.json`, `request.json`, and `calculation.json`). Streaming payloads reserve ZIP64 local-header fields even for small arrays, allowing growth past 4 GiB without buffering. ZIP64 central/end @@ -471,7 +529,9 @@ Limits applied before parsing or allocating payloads: | Structure | Limit | | --- | ---: | | Uncompressed manifest | 1 MiB | -| Uncompressed calculation snapshot | 64 MiB | +| Uncompressed request snapshot, per calculation | 64 MiB | +| Uncompressed calculation snapshot, per calculation | 64 MiB | +| Combined shared sources | 64 MiB | | Members | 4,096 | | Central directory | 16 MiB | | ZIP64 end-record body | 64 KiB | @@ -495,7 +555,8 @@ for the decoded `CompactEri`; there is no artificial 4 GiB payload limit. `tools/reference/generate_portable_fixtures.py` regenerates Python-produced ZIP64 fixtures using only standard-library modules and the existing NumPy byte -fixtures. Both little- and big-endian f64 payloads are covered. The Python tests +fixtures. Single- and multi-calculation bundles, shared sources, and both little- and +big-endian f64 payloads are covered. The Python tests independently reproduce the scientific identity and verify ZIP/JSON/NumPy output written by Rust. Rust tests also exercise a sparse archive with offsets beyond 4 GiB without allocating a multi-gigabyte payload. diff --git a/schemas/portable-manifest-v1.schema.json b/schemas/portable-manifest-v1.schema.json new file mode 100644 index 00000000..048c4334 --- /dev/null +++ b/schemas/portable-manifest-v1.schema.json @@ -0,0 +1,229 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "RustiQ portable manifest V1", + "type": "object", + "additionalProperties": false, + "required": [ + "format", + "format_version", + "kind", + "producer", + "calculations" + ], + "properties": { + "format": { + "const": "rustiq-persistence" + }, + "format_version": { + "const": 1 + }, + "kind": { + "const": "portable" + }, + "producer": { + "type": "object", + "required": [ + "name", + "version" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "sources": { + "type": "array", + "maxItems": 256, + "items": { + "$ref": "#/$defs/source" + } + }, + "calculations": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/calculation" + } + } + }, + "$defs": { + "digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "snapshot": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "version", + "size", + "digest" + ], + "properties": { + "path": { + "type": "string" + }, + "version": { + "const": 1 + }, + "size": { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615 + }, + "digest": { + "$ref": "#/$defs/digest" + } + } + }, + "identity": { + "type": "object", + "required": [ + "version", + "digest" + ], + "properties": { + "version": { + "const": 1 + }, + "digest": { + "$ref": "#/$defs/digest" + } + } + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": [ + "original_name", + "path", + "version", + "size", + "digest" + ], + "properties": { + "original_name": { + "type": "string" + }, + "path": { + "type": "string", + "pattern": "^sources/[^\\\\]+$" + }, + "version": { + "const": 1 + }, + "size": { + "type": "integer", + "minimum": 0, + "maximum": 67108864 + }, + "digest": { + "$ref": "#/$defs/digest" + } + } + }, + "artifact": { + "type": "object", + "required": [ + "path", + "size", + "representation", + "digest" + ], + "properties": { + "path": { + "type": "string" + }, + "size": { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615 + }, + "representation": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/digest" + }, + "attributes": { + "type": "object" + } + }, + "allOf": [ + { + "if": { + "properties": { + "representation": { + "const": "rustiq-compact-eri-v1" + } + } + }, + "then": { + "required": [ + "attributes" + ], + "properties": { + "attributes": { + "type": "object", + "additionalProperties": false, + "required": [ + "basis_functions", + "computation_version" + ], + "properties": { + "basis_functions": { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615 + }, + "computation_version": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295 + } + } + } + } + } + } + ] + }, + "calculation": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "scientific_identity", + "request", + "calculation", + "artifacts" + ], + "properties": { + "id": { + "type": "string", + "pattern": "^calculation-[0-9]+$" + }, + "scientific_identity": { + "$ref": "#/$defs/identity" + }, + "request": { + "$ref": "#/$defs/snapshot" + }, + "calculation": { + "$ref": "#/$defs/snapshot" + }, + "artifacts": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/artifact" + } + } + } + } + } +} diff --git a/tools/reference/generate_portable_fixtures.py b/tools/reference/generate_portable_fixtures.py index b98e7759..cc35c292 100644 --- a/tools/reference/generate_portable_fixtures.py +++ b/tools/reference/generate_portable_fixtures.py @@ -60,6 +60,18 @@ def digest(payload: bytes) -> str: return "sha256:" + hashlib.sha256(payload).hexdigest() +def portable_manifest(manifest: dict) -> dict: + entry = {"id": "calculation-0"} + for field in ("scientific_identity", "request", "calculation", "artifacts"): + entry[field] = manifest.pop(field) + for field in ("request", "calculation"): + entry[field]["path"] = "calculations/calculation-0/" + entry[field]["path"] + for artifact in entry["artifacts"].values(): + artifact["path"] = "calculations/calculation-0/" + artifact["path"] + manifest["calculations"] = [entry] + return manifest + + def generate() -> None: snapshot = (FIXTURES / "calculation-h2-v1.json").read_bytes() request = (FIXTURES / "request-h2-v1.json").read_bytes() @@ -109,6 +121,7 @@ def generate() -> None: } }, } + manifest = portable_manifest(manifest) output = io.BytesIO() # Force full ZIP64 EOCD/locator/central extras on small fixtures. previous_limit = zipfile.ZIP64_LIMIT @@ -117,10 +130,10 @@ def generate() -> None: with zipfile.ZipFile(output, "w", allowZip64=True) as archive: for name, content in [ ("manifest.json", json.dumps(manifest, indent=2).encode() + b"\n"), - ("calculation.json", snapshot), - ("request.json", request), + ("calculations/calculation-0/calculation.json", snapshot), + ("calculations/calculation-0/request.json", request), ("sources/0", source), - ("arrays/integrals/ao-eri.npy", payload), + ("calculations/calculation-0/arrays/integrals/ao-eri.npy", payload), ]: info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) info.create_system = 3 @@ -177,12 +190,13 @@ def generate_angstrom() -> None: }, "artifacts": {}, } + manifest = portable_manifest(manifest) output = io.BytesIO() with zipfile.ZipFile(output, "w", allowZip64=True) as archive: for name, content in [ ("manifest.json", json.dumps(manifest, indent=2).encode() + b"\n"), - ("request.json", request_bytes), - ("calculation.json", snapshot_bytes), + ("calculations/calculation-0/request.json", request_bytes), + ("calculations/calculation-0/calculation.json", snapshot_bytes), ]: info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) info.create_system = 3 @@ -194,6 +208,60 @@ def generate_angstrom() -> None: ) +def generate_multi() -> None: + """Two independent entries share source provenance and use opposite NPY byte orders.""" + + def write_member(target: zipfile.ZipFile, name: str, content: bytes) -> None: + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = 0o100644 << 16 + info.compress_type = ( + zipfile.ZIP_STORED if name.endswith(".npy") else zipfile.ZIP_DEFLATED + ) + target.writestr(info, content) + + output = io.BytesIO() + combined = None + with zipfile.ZipFile(output, "w", allowZip64=True) as target: + for index, order in enumerate(("little", "big")): + content = bytes.fromhex( + (FIXTURES / f"portable-python-{order}-v1.rustiq.hex").read_text() + ) + with zipfile.ZipFile(io.BytesIO(content)) as source: + manifest = json.loads(source.read("manifest.json")) + entry = manifest["calculations"][0] + old_prefix = "calculations/calculation-0/" + new_prefix = f"calculations/calculation-{index}/" + entry["id"] = f"calculation-{index}" + for reference in ( + entry["request"], + entry["calculation"], + *entry["artifacts"].values(), + ): + reference["path"] = reference["path"].replace( + old_prefix, new_prefix + ) + if combined is None: + combined = manifest + write_member(target, "sources/0", source.read("sources/0")) + else: + combined["calculations"].append(entry) + for name in source.namelist(): + if name.startswith(old_prefix): + write_member( + target, + name.replace(old_prefix, new_prefix), + source.read(name), + ) + write_member( + target, "manifest.json", json.dumps(combined, indent=2).encode() + b"\n" + ) + (FIXTURES / "portable-python-multi-v1.rustiq.hex").write_text( + output.getvalue().hex() + "\n", encoding="ascii" + ) + + if __name__ == "__main__": generate() generate_angstrom() + generate_multi() diff --git a/tools/reference/test_persistence_interop.py b/tools/reference/test_persistence_interop.py index 16e9fb2c..0eb906cc 100644 --- a/tools/reference/test_persistence_interop.py +++ b/tools/reference/test_persistence_interop.py @@ -87,16 +87,16 @@ def test_numpy_reads_rust_portable_archive(tmp_path: Path) -> None: assert archive.testzip() is None assert set(archive.namelist()) == { "manifest.json", - "calculation.json", - "request.json", + "calculations/calculation-0/calculation.json", + "calculations/calculation-0/request.json", "sources/0", - "arrays/integrals/ao-eri.npy", + "calculations/calculation-0/arrays/integrals/ao-eri.npy", } manifest = json.loads(archive.read("manifest.json")) - request_bytes = archive.read("request.json") + request_bytes = archive.read("calculations/calculation-0/request.json") request = json.loads(request_bytes) - assert manifest["request"]["digest"] == digest(request_bytes) - assert manifest["request"]["size"] == len(request_bytes) + assert manifest["calculations"][0]["request"]["digest"] == digest(request_bytes) + assert manifest["calculations"][0]["request"]["size"] == len(request_bytes) assert request["hf"]["method"] == "auto" assert request["units"] == "bohr" source_bytes = archive.read("sources/0") @@ -105,19 +105,25 @@ def test_numpy_reads_rust_portable_archive(tmp_path: Path) -> None: ) assert manifest["sources"][0]["digest"] == digest(source_bytes) assert manifest["sources"][0]["original_name"] == "../../calculation.toml" - snapshot_bytes = archive.read("calculation.json") + snapshot_bytes = archive.read("calculations/calculation-0/calculation.json") snapshot = json.loads(snapshot_bytes) - assert manifest["scientific_identity"]["digest"] == scientific_identity( - snapshot + assert manifest["calculations"][0]["scientific_identity"][ + "digest" + ] == scientific_identity(snapshot) + assert manifest["calculations"][0]["calculation"]["digest"] == digest( + snapshot_bytes ) - assert manifest["calculation"]["digest"] == digest(snapshot_bytes) assert snapshot["hf"]["method"] == "rhf" assert snapshot["units"] == "bohr" - payload = archive.read("arrays/integrals/ao-eri.npy") - assert manifest["artifacts"]["ao_eri"]["digest"] == digest(payload) + payload = archive.read("calculations/calculation-0/arrays/integrals/ao-eri.npy") + assert manifest["calculations"][0]["artifacts"]["ao_eri"]["digest"] == digest( + payload + ) _assert_ao_eri_values(np.load(io.BytesIO(payload), allow_pickle=False)) assert ( - archive.getinfo("arrays/integrals/ao-eri.npy").compress_type + archive.getinfo( + "calculations/calculation-0/arrays/integrals/ao-eri.npy" + ).compress_type == zipfile.ZIP_STORED ) assert all( @@ -143,16 +149,26 @@ def test_python_portable_golden_fixtures() -> None: (FIXTURE_DIR / "source-original-v1.toml.hex").read_text() ) assert digest(source_bytes) == manifest["sources"][0]["digest"] - request_bytes = archive.read("request.json") - assert digest(request_bytes) == manifest["request"]["digest"] + request_bytes = archive.read("calculations/calculation-0/request.json") + assert ( + digest(request_bytes) + == manifest["calculations"][0]["request"]["digest"] + ) assert json.loads(request_bytes)["hf"]["method"] == "auto" - snapshot = json.loads(archive.read("calculation.json")) + snapshot = json.loads( + archive.read("calculations/calculation-0/calculation.json") + ) assert ( scientific_identity(snapshot) - == manifest["scientific_identity"]["digest"] + == manifest["calculations"][0]["scientific_identity"]["digest"] + ) + payload = archive.read( + "calculations/calculation-0/arrays/integrals/ao-eri.npy" + ) + assert ( + digest(payload) + == manifest["calculations"][0]["artifacts"]["ao_eri"]["digest"] ) - payload = archive.read("arrays/integrals/ao-eri.npy") - assert digest(payload) == manifest["artifacts"]["ao_eri"]["digest"] _assert_ao_eri_values(np.load(io.BytesIO(payload), allow_pickle=False)) @@ -168,16 +184,20 @@ def test_python_angstrom_conversion_fixture() -> None: ) with zipfile.ZipFile(io.BytesIO(content)) as archive: manifest = json.loads(archive.read("manifest.json")) - request_bytes = archive.read("request.json") - snapshot_bytes = archive.read("calculation.json") + request_bytes = archive.read("calculations/calculation-0/request.json") + snapshot_bytes = archive.read("calculations/calculation-0/calculation.json") request = json.loads(request_bytes) snapshot = json.loads(snapshot_bytes) assert request["units"] == "angstrom" assert snapshot["units"] == "bohr" - assert digest(request_bytes) == manifest["request"]["digest"] - assert digest(snapshot_bytes) == manifest["calculation"]["digest"] + assert digest(request_bytes) == manifest["calculations"][0]["request"]["digest"] + assert ( + digest(snapshot_bytes) + == manifest["calculations"][0]["calculation"]["digest"] + ) assert ( - scientific_identity(snapshot) == manifest["scientific_identity"]["digest"] + scientific_identity(snapshot) + == manifest["calculations"][0]["scientific_identity"]["digest"] ) with localcontext() as context: context.prec = 100 @@ -190,3 +210,64 @@ def test_python_angstrom_conversion_fixture() -> None: Decimal.from_float(requested) / Decimal("0.529177210903") ) assert snapshot["atoms"][1]["position"][0] != 0.74 / 0.529177210903 + + +def _assert_multi_archive(content: bytes) -> None: + import json + import zipfile + + from generate_portable_fixtures import digest, scientific_identity + + with zipfile.ZipFile(io.BytesIO(content)) as archive: + assert archive.testzip() is None + manifest = json.loads(archive.read("manifest.json")) + assert "scientific_identity" not in manifest + assert len(manifest["calculations"]) == 2 + assert len(manifest["sources"]) == 1 + source = manifest["sources"][0] + assert digest(archive.read(source["path"])) == source["digest"] + for index, entry in enumerate(manifest["calculations"]): + assert entry["id"] == f"calculation-{index}" + for field in ("request", "calculation"): + reference = entry[field] + payload = archive.read(reference["path"]) + assert len(payload) == reference["size"] + assert digest(payload) == reference["digest"] + snapshot = json.loads(archive.read(entry["calculation"]["path"])) + assert ( + scientific_identity(snapshot) == entry["scientific_identity"]["digest"] + ) + reference = entry["artifacts"]["ao_eri"] + payload = archive.read(reference["path"]) + assert digest(payload) == reference["digest"] + _assert_ao_eri_values(np.load(io.BytesIO(payload), allow_pickle=False)) + + +def test_python_multi_calculation_fixture() -> None: + content = bytes.fromhex( + (FIXTURE_DIR / "portable-python-multi-v1.rustiq.hex").read_text() + ) + _assert_multi_archive(content) + + +def test_numpy_reads_rust_multi_calculation_archive(tmp_path: Path) -> None: + output = tmp_path / "batch.rustiq" + environment = os.environ.copy() + environment["RUSTIQ_ARCHIVE_TEST_OUTPUT"] = str(output) + subprocess.run( + [ + "cargo", + "test", + "-p", + "rustiq-core", + "--no-default-features", + "persistence::data::portable::tests::writes_bundle_for_python_interoperability", + "--", + "--ignored", + "--exact", + ], + check=True, + cwd=ROOT, + env=environment, + ) + _assert_multi_archive(output.read_bytes()) From 9e56654e96bf993dc847ddc9ca423f50103699bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Mon, 5 Oct 2026 20:56:53 +0200 Subject: [PATCH 7/8] Integrate portable artifacts into calculation execution --- .gitignore | 1 + .../examples/calculate_from_rustiq.rs | 24 ++ crates/rustiq-core/examples/common/mod.rs | 26 ++ crates/rustiq-core/examples/copy_rustiq.rs | 24 ++ crates/rustiq-core/examples/hf_calculation.rs | 23 ++ crates/rustiq-core/examples/read_rustiq.rs | 15 + crates/rustiq-core/examples/write_rustiq.rs | 15 + .../rustiq-core/src/basis/gaussian/basis.rs | 82 ++++++ crates/rustiq-core/src/calculation.rs | 11 +- .../src/calculation/artifact_reuse.rs | 86 ++++++ crates/rustiq-core/src/calculation/builder.rs | 1 + .../rustiq-core/src/calculation/execution.rs | 16 ++ .../src/calculation/prepared_calculation.rs | 172 +++++++++++- .../rustiq-core/src/calculation/solution.rs | 13 + crates/rustiq-core/src/eri.rs | 6 + crates/rustiq-core/src/eri/compact.rs | 38 ++- crates/rustiq-core/src/hf/scf_setup.rs | 23 +- .../src/persistence/data/ao_eri_artifact.rs | 13 + .../src/persistence/data/artifact.rs | 11 + .../src/persistence/data/bundle.rs | 29 +- .../src/persistence/data/portable.rs | 66 ++++- .../src/persistence/data/portable/tests.rs | 13 + .../src/persistence/data/rustiq_data.rs | 23 +- .../rustiq-core/src/persistence/eri_cache.rs | 2 +- crates/rustiq-core/src/persistence/error.rs | 2 + .../tests/automatic_artifact_reuse.rs | 263 ++++++++++++++++++ crates/rustiq-core/tests/portable_artifact.rs | 148 ++++++++++ docs/persistence-format-v1.md | 81 ++++++ 28 files changed, 1195 insertions(+), 32 deletions(-) create mode 100644 crates/rustiq-core/examples/calculate_from_rustiq.rs create mode 100644 crates/rustiq-core/examples/common/mod.rs create mode 100644 crates/rustiq-core/examples/copy_rustiq.rs create mode 100644 crates/rustiq-core/examples/hf_calculation.rs create mode 100644 crates/rustiq-core/examples/read_rustiq.rs create mode 100644 crates/rustiq-core/examples/write_rustiq.rs create mode 100644 crates/rustiq-core/src/calculation/artifact_reuse.rs create mode 100644 crates/rustiq-core/tests/automatic_artifact_reuse.rs diff --git a/.gitignore b/.gitignore index 4b4f0154..07a69d34 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ __pycache__/ .fake /result /result-* +/crates/rustiq-core/examples/data/ diff --git a/crates/rustiq-core/examples/calculate_from_rustiq.rs b/crates/rustiq-core/examples/calculate_from_rustiq.rs new file mode 100644 index 00000000..16621bee --- /dev/null +++ b/crates/rustiq-core/examples/calculate_from_rustiq.rs @@ -0,0 +1,24 @@ +//! Restore a calculation and its AO ERI from a single `.rustiq` archive. +//! Create the default archive with `cargo run -p rustiq-core --example write_rustiq`, +//! then run this example with `cargo run -p rustiq-core --example calculate_from_rustiq`. +#[allow(dead_code)] +mod common; + +use rustiq_core::{calculation::CalculationExecution, persistence::RustiQData}; + +fn main() -> Result<(), Box> { + let input = common::data_path("h2.rustiq")?; + let data = RustiQData::open(&input)?; + let prepared = data.prepare_calculation()?; + let result = prepared.execute()?; + println!("Retained AO ERI values: {}", result.hf.ao_eri().len()); + + println!( + "HF energy: {:.12} Eh", + result.hf.summary().scf.electronic_energy + ); + if let Some(mp2) = result.mp2 { + println!("MP2 correlation energy: {:.12} Eh", mp2.correlation_energy); + } + Ok(()) +} diff --git a/crates/rustiq-core/examples/common/mod.rs b/crates/rustiq-core/examples/common/mod.rs new file mode 100644 index 00000000..da450c98 --- /dev/null +++ b/crates/rustiq-core/examples/common/mod.rs @@ -0,0 +1,26 @@ +use std::fs::File; +use std::path::PathBuf; + +use rustiq_core::{basis::BasisFile, molecules::geometry::Geometry}; + +pub fn h2_inputs() -> Result<(Geometry, BasisFile), Box> { + let geometry_path = workspace_path("samples/h2/molecule.xyz")?.canonicalize()?; + let basis_path = workspace_path("tests/data/sto-3g.json")?.canonicalize()?; + let geometry = Geometry::from_file(File::open(geometry_path)?)?; + let basis = BasisFile::from_reader(File::open(basis_path)?)?; + Ok((geometry, basis)) +} + +pub fn workspace_path(path: &str) -> Result> { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize()?; + Ok(root.join(path)) +} + +#[allow(dead_code)] +pub fn data_path(filename: &str) -> Result> { + let directory = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("examples/data"); + std::fs::create_dir_all(&directory)?; + Ok(directory.canonicalize()?.join(filename)) +} diff --git a/crates/rustiq-core/examples/copy_rustiq.rs b/crates/rustiq-core/examples/copy_rustiq.rs new file mode 100644 index 00000000..ed505a74 --- /dev/null +++ b/crates/rustiq-core/examples/copy_rustiq.rs @@ -0,0 +1,24 @@ +//! Read h2.rustiq, run HF and MP2, then write the retained AO ERI to h2-copy.rustiq. +mod common; + +use rustiq_core::{calculation::CalculationBuilder, config::Mp2Config, persistence::RustiQData}; + +fn main() -> Result<(), Box> { + let (geometry, basis) = common::h2_inputs()?; + let prepared = CalculationBuilder::new(&geometry, &basis) + .with_mp2(Mp2Config::default()) + .prepare()?; + let input = common::data_path("h2.rustiq")?; + let mut data = RustiQData::open(input)?; + let eri = data.take_compatible_eri(&prepared)?; + let result = prepared.execute_with_eri(eri)?; + let mut output = RustiQData::from_calculation(&prepared)?; + let output_path = common::data_path("h2-copy.rustiq")?; + output.write_with_eri(&output_path, result.hf.ao_eri())?; + println!( + "HF energy: {:.12} Eh", + result.hf.summary().scf.electronic_energy + ); + println!("Wrote {}", output_path.display()); + Ok(()) +} diff --git a/crates/rustiq-core/examples/hf_calculation.rs b/crates/rustiq-core/examples/hf_calculation.rs new file mode 100644 index 00000000..5fcb7df3 --- /dev/null +++ b/crates/rustiq-core/examples/hf_calculation.rs @@ -0,0 +1,23 @@ +//! Run HF and MP2 directly from the H2 sample inputs. +mod common; + +use rustiq_core::{ + calculation::{CalculationBuilder, CalculationExecution}, + config::Mp2Config, +}; + +fn main() -> Result<(), Box> { + let (geometry, basis) = common::h2_inputs()?; + let prepared = CalculationBuilder::new(&geometry, &basis) + .with_mp2(Mp2Config::default()) + .prepare()?; + let result = prepared.execute()?; + println!( + "HF energy: {:.12} Eh", + result.hf.summary().scf.electronic_energy + ); + if let Some(mp2) = result.mp2 { + println!("MP2 correlation energy: {:.12} Eh", mp2.correlation_energy); + } + Ok(()) +} diff --git a/crates/rustiq-core/examples/read_rustiq.rs b/crates/rustiq-core/examples/read_rustiq.rs new file mode 100644 index 00000000..9f9bc4d0 --- /dev/null +++ b/crates/rustiq-core/examples/read_rustiq.rs @@ -0,0 +1,15 @@ +//! Run HF using the AO ERI stored in h2.rustiq. +mod common; + +use rustiq_core::{calculation::CalculationBuilder, persistence::RustiQData}; + +fn main() -> Result<(), Box> { + let (geometry, basis) = common::h2_inputs()?; + let prepared = CalculationBuilder::new(&geometry, &basis).prepare()?; + let input = common::data_path("h2.rustiq")?; + let mut data = RustiQData::open(input)?; + let eri = data.take_compatible_eri(&prepared)?; + let hf = prepared.run_hf_with_eri(eri)?; + println!("HF energy: {:.12} Eh", hf.summary().scf.electronic_energy); + Ok(()) +} diff --git a/crates/rustiq-core/examples/write_rustiq.rs b/crates/rustiq-core/examples/write_rustiq.rs new file mode 100644 index 00000000..6db30fa3 --- /dev/null +++ b/crates/rustiq-core/examples/write_rustiq.rs @@ -0,0 +1,15 @@ +//! Run HF and write its retained AO ERI to h2.rustiq. +mod common; + +use rustiq_core::calculation::CalculationBuilder; + +fn main() -> Result<(), Box> { + let (geometry, basis) = common::h2_inputs()?; + let prepared = CalculationBuilder::new(&geometry, &basis).prepare()?; + let hf = prepared.run_hf()?; + let mut data = rustiq_core::persistence::RustiQData::from_calculation(&prepared)?; + let output = common::data_path("h2.rustiq")?; + data.write_with_eri(&output, hf.ao_eri())?; + println!("Wrote {}", output.display()); + Ok(()) +} diff --git a/crates/rustiq-core/src/basis/gaussian/basis.rs b/crates/rustiq-core/src/basis/gaussian/basis.rs index c7da3bd8..1792921d 100644 --- a/crates/rustiq-core/src/basis/gaussian/basis.rs +++ b/crates/rustiq-core/src/basis/gaussian/basis.rs @@ -137,6 +137,88 @@ pub struct NormalizedPrimitive { } impl Basis { + /// Rebuilds the resolved AO basis stored in portable calculation context. + /// Its normalized primitive coefficients are copied verbatim. Resolved AO + /// snapshots retain effective components, not the original shell grouping; + /// integral evaluation only needs the centers alongside those components. + pub(crate) fn from_calculation_context( + context: &crate::persistence::CalculationContext, + ) -> Self { + let mut shells = Vec::new(); + let mut shell_ids = Vec::new(); + let mut angular_momenta = Vec::new(); + let mut angular_components = Vec::new(); + let mut normalized_components = Vec::new(); + + for ao in context.basis() { + let components: Vec<_> = ao.components().collect(); + let first = components + .first() + .expect("validated portable AO has at least one component"); + let shell_id = shells.len(); + let center = ao.center(); + shells.push(Shell { + alpha: nalgebra::DVector::zeros(0), + contr: Vec::new(), + origin: Point3::new(center[0], center[1], center[2]), + max_ln_coeff: nalgebra::DVector::zeros(0), + }); + shell_ids.push(shell_id); + let first_angular_momentum = first.angular_momentum(); + angular_momenta.push(Vector3::new( + first_angular_momentum[0], + first_angular_momentum[1], + first_angular_momentum[2], + )); + angular_components.push( + components + .iter() + .map(|component| { + let angular_momentum = component.angular_momentum(); + ( + Vector3::new( + angular_momentum[0], + angular_momentum[1], + angular_momentum[2], + ), + 1.0, + ) + }) + .collect(), + ); + normalized_components.push( + components + .iter() + .map(|component| NormalizedComponent { + angular_momentum: { + let angular_momentum = component.angular_momentum(); + Vector3::new( + angular_momentum[0], + angular_momentum[1], + angular_momentum[2], + ) + }, + primitives: component + .primitives() + .map(|(exponent, coefficient)| NormalizedPrimitive { + exponent, + coefficient, + }) + .collect(), + }) + .collect(), + ); + } + + Self { + shells, + shell_ids, + angular_momenta, + angular_components, + normalized_components, + } + } + pub(crate) fn new(shells: Vec) -> Self { let mut shell_ids = Vec::new(); let mut angular_momenta = Vec::new(); diff --git a/crates/rustiq-core/src/calculation.rs b/crates/rustiq-core/src/calculation.rs index 4404e958..8cfca874 100644 --- a/crates/rustiq-core/src/calculation.rs +++ b/crates/rustiq-core/src/calculation.rs @@ -78,6 +78,7 @@ pub use setup_error::HfSetupError; mod solution; use crate::hf::scf_result::ScfOutcome; pub use solution::{CalculationExecutionError, Converged, HfOutcome, HfSolution, Unconverged}; +mod artifact_reuse; mod builder; mod execution; mod prepared_calculation; @@ -101,8 +102,8 @@ pub use crate::{ }; pub use builder::CalculationBuilder; pub use execution::{ - CalculationEvent, CalculationExecution, CalculationResult, EriCacheAction, EriCacheEvent, - HfCalculationResult, Mp2MemoryPlan, + ArtifactReuseDecision, ArtifactReuseEvent, CalculationEvent, CalculationExecution, + CalculationResult, EriCacheAction, EriCacheEvent, HfCalculationResult, Mp2MemoryPlan, }; pub use prepared_calculation::PreparedCalculation; pub use request::CalculationRequest; @@ -155,6 +156,8 @@ pub enum CalculationError { Numerical(#[from] NumericalError), #[error(transparent)] Diis(#[from] DiisError), + #[error(transparent)] + Artifact(#[from] crate::persistence::ArtifactError), #[error("{error}")] Mp2 { #[source] @@ -222,17 +225,20 @@ impl<'a> HfCalculation<'a> { config, integrals, eri_cache, + None, progress, |_| {}, ) } + #[allow(clippy::too_many_arguments)] pub(crate) fn new_with_progress_and_cache( molecule: &'a Molecule, basis: &'a Basis, config: &HfConfig, integrals: &IntegralConfig, eri_cache: Option<&'a EriCache>, + supplied_eri: Option, mut progress: impl FnMut(ScfSetupStep), mut cache_event: impl FnMut(EriCacheEvent), ) -> Result { @@ -251,6 +257,7 @@ impl<'a> HfCalculation<'a> { &config.orthogonalization, integrals, eri_cache, + supplied_eri, &mut progress, &mut cache_event, ) diff --git a/crates/rustiq-core/src/calculation/artifact_reuse.rs b/crates/rustiq-core/src/calculation/artifact_reuse.rs new file mode 100644 index 00000000..b64ca8c3 --- /dev/null +++ b/crates/rustiq-core/src/calculation/artifact_reuse.rs @@ -0,0 +1,86 @@ +use std::sync::Mutex; + +use crate::persistence::{Artifact, ArtifactError, PortableError, RustiQData}; + +use super::{ArtifactReuseDecision, ArtifactReuseEvent, PreparedCalculation}; + +/// The origin of reusable state does not change the scientific execution API. +pub(super) enum CalculationSource { + Configuration, + Portable(Box), +} + +/// Read-only persistence input with lazily decoded typed domain values. +/// Numerical algorithms receive domain objects, never archive paths or keys. +pub(super) struct ArtifactReuse { + source: Mutex, + resolved: Mutex, +} + +impl ArtifactReuse { + pub(super) fn new( + data: RustiQData, + calculation: &PreparedCalculation, + ) -> Result { + Ok(Self { + source: Mutex::new(data), + resolved: Mutex::new(RustiQData::from_calculation(calculation)?), + }) + } + + pub(super) fn resolve( + &self, + calculation: &PreparedCalculation, + ) -> Result<(Option, ArtifactReuseEvent), ArtifactError> + where + A::Value: Clone, + { + let mut resolved = self.resolved.lock().map_err(|_| { + ArtifactError::AccessFailed("resolved artifact mutex is poisoned".into()) + })?; + if let Some(value) = resolved.get::()?.cloned() { + return Ok(( + Some(value), + ArtifactReuseEvent { + artifact: A::NAME, + decision: ArtifactReuseDecision::Reused, + }, + )); + } + let mut data = self + .source + .lock() + .map_err(|_| ArtifactError::AccessFailed("artifact source mutex is poisoned".into()))?; + let (value, decision) = if !A::is_present(&data) { + (None, ArtifactReuseDecision::Missing) + } else if !A::is_compatible(&data, calculation) { + (None, ArtifactReuseDecision::Incompatible) + } else { + let value = data.get::()?.cloned(); + let decision = if value.is_some() { + ArtifactReuseDecision::Reused + } else { + ArtifactReuseDecision::Missing + }; + (value, decision) + }; + if let Some(value) = &value { + resolved.set::(value.clone())?; + } + Ok(( + value, + ArtifactReuseEvent { + artifact: A::NAME, + decision, + }, + )) + } + + /// Retains newly computed domain values in memory; never writes the source archive. + pub(super) fn remember(&self, value: A::Value) -> Result<(), ArtifactError> { + self.resolved + .lock() + .map_err(|_| ArtifactError::AccessFailed("resolved artifact mutex is poisoned".into()))? + .set::(value) + } +} diff --git a/crates/rustiq-core/src/calculation/builder.rs b/crates/rustiq-core/src/calculation/builder.rs index 3707f50a..80845b4b 100644 --- a/crates/rustiq-core/src/calculation/builder.rs +++ b/crates/rustiq-core/src/calculation/builder.rs @@ -203,6 +203,7 @@ impl<'a> CalculationBuilder<'a> { integrals: self.integrals, mp2: self.mp2, eri_cache: self.eri_cache.clone(), + source: super::artifact_reuse::CalculationSource::Configuration, }) } diff --git a/crates/rustiq-core/src/calculation/execution.rs b/crates/rustiq-core/src/calculation/execution.rs index f1da5dda..101dd3c8 100644 --- a/crates/rustiq-core/src/calculation/execution.rs +++ b/crates/rustiq-core/src/calculation/execution.rs @@ -28,6 +28,21 @@ pub struct EriCacheEvent { pub fingerprint: String, } +/// Per-artifact reuse decision. Ignoring an artifact never means deleting it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ArtifactReuseDecision { + Reused, + Missing, + Incompatible, + Ignored, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ArtifactReuseEvent { + pub artifact: &'static str, + pub decision: ArtifactReuseDecision, +} + /// Results of the requested scientific stages, without presentation choices. #[derive(Debug, Clone)] pub struct CalculationResult { @@ -55,6 +70,7 @@ pub enum CalculationEvent<'a> { config: &'a HfConfig, }, EriCache(EriCacheEvent), + ArtifactReuse(ArtifactReuseEvent), ScfSetup(ScfSetupStep), ScfIteration(&'a ScfIteration), HfCompleted(&'a HfOutcome), diff --git a/crates/rustiq-core/src/calculation/prepared_calculation.rs b/crates/rustiq-core/src/calculation/prepared_calculation.rs index bd51a1b7..c1c4a2af 100644 --- a/crates/rustiq-core/src/calculation/prepared_calculation.rs +++ b/crates/rustiq-core/src/calculation/prepared_calculation.rs @@ -2,20 +2,31 @@ use super::{ CalculationError, CalculationEvent, CalculationExecution, CalculationExecutionError, CalculationResult, HfCalculation, HfCalculationResult, HfOutcome, HfSolution, }; +use crate::eri::CompactEri; use crate::hf::scf_result::ScfOutcome; use crate::{ basis::Basis, - config::{HfConfig, IntegralConfig, Mp2Config, ResolvedHfMethod}, - molecules::molecule::Molecule, - persistence::EriCache, + config::{ + validated::{DiisSize, NonNegativeFiniteF64, PositiveFiniteF64}, + DiisConfig, HfConfig, HfMethod, IntegralConfig, Mp2Config, OrthogonalizationConfig, + ResolvedHfMethod, + }, + molecules::{atom::Atom, geometry::Geometry, molecule::Molecule, units::Units}, + persistence::{AoEriArtifact, EriCache, PortableError, RustiQData}, +}; +use std::{ + cell::RefCell, + num::{NonZeroU8, NonZeroUsize}, }; -use std::cell::RefCell; +use super::artifact_reuse::{ArtifactReuse, CalculationSource}; use super::CalculationRequest; -/// A validated molecule in Bohr and its basis, prepared together by the builder. +/// A validated molecule in Bohr and its basis, prepared from configuration or +/// restored from portable scientific data through the same execution interface. /// /// Each execution starts fresh HF state and uses the same immutable inputs. +/// Available compatible typed artifacts are reused automatically when a source is attached. /// This avoids self-referential SCF storage and allows reuse of the basis. pub struct PreparedCalculation { pub(super) request: CalculationRequest, @@ -26,9 +37,97 @@ pub struct PreparedCalculation { pub(super) integrals: IntegralConfig, pub(super) mp2: Option, pub(super) eri_cache: Option, + pub(super) source: CalculationSource, } impl PreparedCalculation { + pub(crate) fn from_portable_context( + request: CalculationRequest, + context: &crate::persistence::CalculationContext, + ) -> Self { + let atoms = context + .atoms() + .map(|(atomic_number, position)| { + let element = periodic_table::periodic_table() + .iter() + .find(|element| element.atomic_number == atomic_number) + .expect("validated portable context uses a known element"); + Atom::new( + element, + nalgebra::Point3::new(position[0], position[1], position[2]), + ) + }) + .collect(); + let geometry = Geometry::new("Restored from RustiQ data".into(), atoms); + let molecule = Molecule::try_new( + geometry, + Units::Bohr, + context.charge(), + NonZeroU8::new(context.multiplicity()).expect("validated multiplicity is nonzero"), + ) + .expect("validated portable context has a valid molecular state"); + let basis = Basis::from_calculation_context(context); + + let diis_size = context + .diis_size() + .map(|size| DiisSize::try_new(size).expect("validated DIIS size is valid")); + let hf = HfConfig { + method: match context.hf_method() { + ResolvedHfMethod::Rhf => HfMethod::Rhf, + ResolvedHfMethod::Uhf => HfMethod::Uhf, + } + .into(), + max_iterations: NonZeroUsize::new(context.max_iterations()) + .expect("validated iteration count is nonzero"), + convergence_threshold: PositiveFiniteF64::try_new(context.convergence_threshold()) + .expect("validated convergence threshold is positive and finite"), + guess: context.density_guess().into(), + diis: DiisConfig { + enabled: diis_size.is_some(), + max_history: diis_size + .unwrap_or_else(|| DiisSize::try_new(6).expect("default DIIS size is valid")) + .into(), + }, + orthogonalization: OrthogonalizationConfig { + linear_dependency_threshold: NonNegativeFiniteF64::try_new( + context.linear_dependency_threshold(), + ) + .expect("validated linear dependency threshold is finite and nonnegative") + .into(), + }, + }; + let integrals = IntegralConfig { + schwarz_threshold: context + .eri_schwarz_threshold() + .map(|threshold| { + PositiveFiniteF64::try_new(threshold) + .expect("validated ERI threshold is positive and finite") + }) + .into(), + }; + + Self { + basis_name: request.basis_name().to_owned(), + mp2: request.mp2().copied(), + request, + molecule, + basis, + hf: (hf, context.hf_method()), + integrals, + eri_cache: None, + source: CalculationSource::Configuration, + } + } + + /// Supplies a read-only source of reusable typed scientific artifacts. + /// Normal execution checks compatibility per artifact, then uses the enabled + /// local cache or computes missing/incompatible values. Corrupt required + /// payloads are errors. Newly computed values are retained in memory. + pub fn with_reuse_data(mut self, data: RustiQData) -> Result { + self.source = CalculationSource::Portable(Box::new(ArtifactReuse::new(data, &self)?)); + Ok(self) + } + /// Returns the normalized inputs as requested before scientific resolution. pub fn request(&self) -> &CalculationRequest { &self.request @@ -84,18 +183,48 @@ impl PreparedCalculation { &self, events: impl FnMut(CalculationEvent<'_>), ) -> Result { + self.run_hf_from_eri(None, events) + } + + /// Runs HF with an owned AO ERI, bypassing integral computation and the ERI cache. + pub fn run_hf_with_eri(&self, eri: CompactEri) -> Result { + self.run_hf_from_eri(Some(eri), |_| {}) + } + + fn run_hf_from_eri( + &self, + eri: Option, + events: impl FnMut(CalculationEvent<'_>), + ) -> Result { + let retain_artifact = eri.is_none(); let events = RefCell::new(events); let (config, method) = &self.hf; events.borrow_mut()(CalculationEvent::HfStarted { method: *method, config, }); + let eri = match (eri, &self.source) { + (Some(eri), _) => Some(eri), + (None, CalculationSource::Portable(reuse)) => { + let (eri, decision) = reuse + .resolve::(self) + .map_err(CalculationError::from)?; + events.borrow_mut()(CalculationEvent::ArtifactReuse(decision)); + eri + } + (None, CalculationSource::Configuration) => None, + }; let mut calculation = HfCalculation::new_with_progress_and_cache( &self.molecule, &self.basis, config, &self.integrals, - self.eri_cache.as_ref(), + if eri.is_some() { + None + } else { + self.eri_cache.as_ref() + }, + eri, |step| events.borrow_mut()(CalculationEvent::ScfSetup(step)), |event| events.borrow_mut()(CalculationEvent::EriCache(event)), )?; @@ -118,17 +247,33 @@ impl PreparedCalculation { calculation.state, )), }; + if retain_artifact { + if let CalculationSource::Portable(reuse) = &self.source { + reuse + .remember::(hf.ao_eri().clone()) + .map_err(|error| hf.clone().execution_error(CalculationError::from(error)))?; + } + } events.borrow_mut()(CalculationEvent::HfCompleted(&hf)); Ok(hf) } } -impl CalculationExecution for PreparedCalculation { - fn execute_with_events( +impl PreparedCalculation { + /// Executes HF and optional MP2 using the supplied AO ERI without copying it. + pub fn execute_with_eri( + &self, + eri: CompactEri, + ) -> Result { + self.execute_from_eri(Some(eri), |_| {}) + } + + fn execute_from_eri( &self, + eri: Option, mut events: impl FnMut(CalculationEvent<'_>), ) -> Result { - let hf = self.run_hf_with_events(&mut events)?; + let hf = self.run_hf_from_eri(eri, &mut events)?; let mp2 = self .mp2 .as_ref() @@ -156,3 +301,12 @@ impl CalculationExecution for PreparedCalculation { Ok(CalculationResult { hf, mp2 }) } } + +impl CalculationExecution for PreparedCalculation { + fn execute_with_events( + &self, + events: impl FnMut(CalculationEvent<'_>), + ) -> Result { + self.execute_from_eri(None, events) + } +} diff --git a/crates/rustiq-core/src/calculation/solution.rs b/crates/rustiq-core/src/calculation/solution.rs index e738ff85..8f6ed428 100644 --- a/crates/rustiq-core/src/calculation/solution.rs +++ b/crates/rustiq-core/src/calculation/solution.rs @@ -48,6 +48,14 @@ pub enum HfOutcome { } impl HfOutcome { + /// Borrows the AO ERI tensor, including when HF did not converge. + pub fn ao_eri(&self) -> &CompactEri { + match self { + Self::Converged(hf) => hf.ao_eri(), + Self::Unconverged(hf) => hf.ao_eri(), + } + } + /// Returns the resolved Hartree-Fock method, even when HF did not converge. pub fn method(&self) -> ResolvedHfMethod { self.summary().method @@ -73,6 +81,11 @@ impl HfOutcome { } impl HfSolution { + /// Borrows the retained AO ERI tensor without copying it. + pub fn ao_eri(&self) -> &CompactEri { + &self.0.integrals + } + pub(super) fn from_state(summary: HfCalculationResult, state: HfState<'_>) -> Self { let (orbitals, integrals) = match state { HfState::Rhf(scf) => ( diff --git a/crates/rustiq-core/src/eri.rs b/crates/rustiq-core/src/eri.rs index d9a51d4d..26378e3d 100644 --- a/crates/rustiq-core/src/eri.rs +++ b/crates/rustiq-core/src/eri.rs @@ -149,6 +149,12 @@ pub fn compute_eri_primitive( /// A 4D tensor containing all ERI integrals. #[derive(Debug, Error, PartialEq)] pub enum EriError { + #[error("supplied AO ERI has {actual} values, expected {expected} for {basis_functions} basis functions")] + InvalidValueCount { + basis_functions: usize, + expected: usize, + actual: usize, + }, #[error("non-finite Coulomb self-integral for AO pair {pair_index}: {value}")] NonFiniteSelfIntegral { pair_index: usize, value: f64 }, #[error( diff --git a/crates/rustiq-core/src/eri/compact.rs b/crates/rustiq-core/src/eri/compact.rs index fe50ca2d..9f611ca2 100644 --- a/crates/rustiq-core/src/eri/compact.rs +++ b/crates/rustiq-core/src/eri/compact.rs @@ -1,4 +1,7 @@ -use std::ops::{Index, IndexMut}; +use std::{ + ops::{Index, IndexMut}, + sync::Arc, +}; use rayon::prelude::*; use thiserror::Error; @@ -11,9 +14,10 @@ pub(crate) enum CompactEriBuildError { InvalidLength { expected: usize, actual: usize }, } -#[derive(Debug)] +/// Compact AO ERI values. Clones share storage until a value is modified. +#[derive(Debug, Clone)] pub struct CompactEri { - storage: Box<[f64]>, + storage: Arc<[f64]>, } impl CompactEri { @@ -64,7 +68,7 @@ impl CompactEri { return Err(CompactEriBuildError::InvalidLength { expected, actual }); } Ok(Self { - storage: values.into_boxed_slice(), + storage: values.into(), }) } @@ -92,9 +96,11 @@ impl CompactEri { }); } - let storage = par_iter.collect(); + let storage: Vec = par_iter.collect(); - Ok(Self { storage }) + Ok(Self { + storage: storage.into(), + }) } } @@ -108,7 +114,7 @@ impl Index for CompactEri { impl IndexMut for CompactEri { fn index_mut(&mut self, index: EriIndex) -> &mut Self::Output { - &mut self.storage[index.0] + &mut Arc::make_mut(&mut self.storage)[index.0] } } @@ -124,7 +130,7 @@ impl Index<(usize, usize, usize, usize)> for CompactEri { impl IndexMut<(usize, usize, usize, usize)> for CompactEri { fn index_mut(&mut self, index: (usize, usize, usize, usize)) -> &mut Self::Output { let (mu, nu, lambda, sigma) = index; - &mut self.storage[EriIndex::new(mu, nu, lambda, sigma).0] + &mut self[EriIndex::new(mu, nu, lambda, sigma)] } } @@ -135,6 +141,22 @@ mod tests { use ndarray::Array4; use proptest::prelude::*; + #[test] + fn clones_share_values_and_detach_on_mutation() { + let mut original = CompactEri::Zeroed(2); + original[(0, 0, 0, 0)] = 1.0; + let mut cloned = original.clone(); + assert_eq!( + original.ordered_values().as_ptr(), + cloned.ordered_values().as_ptr() + ); + cloned[EriIndex::new(0, 0, 0, 0)] = 2.0; + assert_eq!(original[(0, 0, 0, 0)], 1.0); + assert_eq!(cloned[(0, 0, 0, 0)], 2.0); + original[(0, 0, 0, 0)] = 3.0; + assert_eq!(cloned[(0, 0, 0, 0)], 2.0); + } + #[test] fn test_compact_eri_allocates_unique_quartets() { let basis_functions = 4; diff --git a/crates/rustiq-core/src/hf/scf_setup.rs b/crates/rustiq-core/src/hf/scf_setup.rs index c356d6ef..5a72c860 100644 --- a/crates/rustiq-core/src/hf/scf_setup.rs +++ b/crates/rustiq-core/src/hf/scf_setup.rs @@ -10,7 +10,7 @@ use crate::{ validated::PositiveFiniteF64, IntegralConfig, OrthogonalizationConfig, DEFAULT_ERI_SCHWARZ_THRESHOLD, }, - eri::EriError, + eri::{CompactEri, EriError}, molecules::molecule::Molecule, persistence::EriCache, }; @@ -70,6 +70,7 @@ pub(crate) fn prepare_scf_setup( .expect("default ERI Schwarz threshold is valid"), ), None, + None, progress, |_| {}, ) @@ -83,6 +84,7 @@ pub(crate) fn prepare_configured_scf_setup( orthogonalization: &OrthogonalizationConfig, integrals: &IntegralConfig, eri_cache: Option<&EriCache>, + supplied_eri: Option, progress: impl FnMut(ScfSetupStep), cache_event: impl FnMut(EriCacheEvent), ) -> Result { @@ -96,6 +98,7 @@ pub(crate) fn prepare_configured_scf_setup( .into_inner(), integrals.schwarz_threshold.value, eri_cache, + supplied_eri, progress, cache_event, ) @@ -109,6 +112,7 @@ fn prepare_scf_setup_with_thresholds( linear_dependency_threshold: f64, eri_schwarz_threshold: Option, eri_cache: Option<&EriCache>, + supplied_eri: Option, mut progress: impl FnMut(ScfSetupStep), mut cache_event: impl FnMut(EriCacheEvent), ) -> Result { @@ -139,7 +143,22 @@ fn prepare_scf_setup_with_thresholds( progress(ScfSetupStep::ElectronRepulsionIntegrals); let step_start = Instant::now(); - let (electron_repulsion, eri_cache_event) = builder.electron_repulsion()?; + let (electron_repulsion, eri_cache_event) = match supplied_eri { + Some(eri) => { + let basis_functions = basis.nbasis(); + let expected = CompactEri::storage_len(basis_functions); + if eri.len() != expected { + return Err(EriError::InvalidValueCount { + basis_functions, + expected, + actual: eri.len(), + } + .into()); + } + (eri, None) + } + None => builder.electron_repulsion()?, + }; if let Some(event) = eri_cache_event { cache_event(event); } diff --git a/crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs b/crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs index 3e93b560..7b418618 100644 --- a/crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs +++ b/crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs @@ -13,6 +13,19 @@ impl private::Sealed for AoEriArtifact {} impl Artifact for AoEriArtifact { type Value = CompactEri; + const NAME: &'static str = AO_ERI_ARTIFACT; + + fn is_present(data: &RustiQData) -> bool { + data.ao_eri.is_some() || data.manifest.artifacts.contains_key(AO_ERI_ARTIFACT) + } + + fn is_compatible( + data: &RustiQData, + calculation: &crate::calculation::PreparedCalculation, + ) -> bool { + data.matches_eri_identity(calculation) + } + fn get(data: &mut RustiQData) -> Result, ArtifactError> { if data.ao_eri.is_some() { return Ok(data.ao_eri.as_ref()); diff --git a/crates/rustiq-core/src/persistence/data/artifact.rs b/crates/rustiq-core/src/persistence/data/artifact.rs index dd1df18b..88dccaf4 100644 --- a/crates/rustiq-core/src/persistence/data/artifact.rs +++ b/crates/rustiq-core/src/persistence/data/artifact.rs @@ -9,6 +9,17 @@ pub(crate) mod private { pub trait Artifact: private::Sealed { type Value; + /// Stable logical artifact name, independent of its storage path. + const NAME: &'static str; + + #[doc(hidden)] + fn is_present(data: &RustiQData) -> bool; + #[doc(hidden)] + fn is_compatible( + data: &RustiQData, + calculation: &crate::calculation::PreparedCalculation, + ) -> bool; + #[doc(hidden)] fn get(data: &mut RustiQData) -> Result, ArtifactError>; #[doc(hidden)] diff --git a/crates/rustiq-core/src/persistence/data/bundle.rs b/crates/rustiq-core/src/persistence/data/bundle.rs index 37ad8c62..ca9527c4 100644 --- a/crates/rustiq-core/src/persistence/data/bundle.rs +++ b/crates/rustiq-core/src/persistence/data/bundle.rs @@ -190,6 +190,15 @@ pub(super) fn write_bundle( calculations: &mut [RustiQData], sources: &[SourceProvenance], path: &Path, +) -> Result<(), PortableError> { + write_bundle_with_eri(calculations, sources, path, None) +} + +pub(super) fn write_bundle_with_eri( + calculations: &mut [RustiQData], + sources: &[SourceProvenance], + path: &Path, + eri: Option<&crate::eri::CompactEri>, ) -> Result<(), PortableError> { if calculations.is_empty() { return Err(invalid("portable bundles require at least one calculation")); @@ -205,10 +214,11 @@ pub(super) fn write_bundle( .filter(|parent| !parent.as_os_str().is_empty()) .unwrap_or(Path::new(".")); let temporary = tempfile::NamedTempFile::new_in(parent)?; - write_snapshot( + write_snapshot_inner( calculations, sources, Storage::create_zip(temporary.reopen()?), + eri, )?; drop(RustiQBundle::open(temporary.path())?); temporary.persist_noclobber(path).map_err(|error| { @@ -227,6 +237,15 @@ pub(super) fn write_snapshot( calculations: &mut [RustiQData], sources: &[SourceProvenance], destination: Storage, +) -> Result<(), PortableError> { + write_snapshot_inner(calculations, sources, destination, None) +} + +fn write_snapshot_inner( + calculations: &mut [RustiQData], + sources: &[SourceProvenance], + destination: Storage, + eri: Option<&crate::eri::CompactEri>, ) -> Result<(), PortableError> { let storage = Arc::new(Mutex::new(destination)); let mut manifest = PortableManifest { @@ -261,7 +280,7 @@ pub(super) fn write_snapshot( if data.context.is_none() || data.request.is_none() { return Err(invalid("missing calculation snapshots")); } - if data.ao_eri.is_none() && data.manifest.artifacts.get(super::AO_ERI_ARTIFACT).is_some_and(|artifact| + if eri.is_none() && data.ao_eri.is_none() && data.manifest.artifacts.get(super::AO_ERI_ARTIFACT).is_some_and(|artifact| artifact.representation == COMPACT_ERI_REPRESENTATION && matches!(&artifact.attributes, ArtifactAttributes::AoEri(attributes) if attributes.computation_version == crate::persistence::AO_ERI_COMPUTATION_VERSION)) { data.validate_unloaded_eri()?; @@ -269,7 +288,11 @@ pub(super) fn write_snapshot( let id = format!("calculation-{index}"); let prefix = format!("calculations/{id}"); let saved_sources = std::mem::take(&mut data.sources); - let result = data.write_entry(Storage::scoped(storage.clone(), prefix.clone())); + let destination = Storage::scoped(storage.clone(), prefix.clone()); + let result = match eri { + Some(eri) => data.write_entry_with_eri(destination, eri), + None => data.write_entry(destination), + }; data.sources = saved_sources; let entry = result?; let mut request = entry diff --git a/crates/rustiq-core/src/persistence/data/portable.rs b/crates/rustiq-core/src/persistence/data/portable.rs index bb8bf618..e0e43caf 100644 --- a/crates/rustiq-core/src/persistence/data/portable.rs +++ b/crates/rustiq-core/src/persistence/data/portable.rs @@ -119,6 +119,26 @@ impl RustiQData { self.request.as_ref() } + /// Rebuilds an executable calculation from the resolved context in this portable archive. + /// The archive becomes a read-only source of typed artifacts for normal execution. + /// Payloads are loaded and verified only when their scientific stage needs them. + /// Missing or incompatible artifacts fall back to computation; corrupt compatible + /// artifacts produce an execution error. The source archive is never modified. + pub fn prepare_calculation(self) -> Result { + let context = self.context.as_ref().ok_or_else(|| { + PortableError::InvalidCalculation( + "calculation preparation requires portable calculation context".into(), + ) + })?; + let request = self.request.as_ref().ok_or_else(|| { + PortableError::InvalidCalculation( + "calculation preparation requires a portable request".into(), + ) + })?; + let prepared = PreparedCalculation::from_portable_context(request.clone(), context); + prepared.with_reuse_data(self) + } + /// Captured sources are opaque provenance and never affect scientific compatibility. pub fn sources(&self) -> impl ExactSizeIterator { self.sources.iter() @@ -209,7 +229,7 @@ impl RustiQData { && self.matches_eri_identity(calculation) } - fn matches_eri_identity(&self, calculation: &PreparedCalculation) -> bool { + pub(super) fn matches_eri_identity(&self, calculation: &PreparedCalculation) -> bool { let identity = super::super::ao_eri_identity( calculation.get_molecule(), calculation.get_basis(), @@ -237,6 +257,50 @@ impl RustiQData { result } + /// Writes a single-calculation V1 bundle with a borrowed AO ERI tensor. + pub fn write_with_eri( + &mut self, + path: impl AsRef, + eri: &crate::eri::CompactEri, + ) -> Result<(), PortableError> { + self.validate_eri(eri)?; + let sources = std::mem::take(&mut self.sources); + let result = super::bundle::write_bundle_with_eri( + std::slice::from_mut(self), + &sources, + path.as_ref(), + Some(eri), + ); + self.sources = sources; + result + } + + /// Validates compatibility and lazily decodes the ERI, then transfers ownership. + /// Missing, unsupported, incompatible, or corrupt artifacts are errors. + pub fn take_compatible_eri( + &mut self, + calculation: &PreparedCalculation, + ) -> Result { + if self.ao_eri.is_none() { + let artifact = self + .manifest + .artifacts + .get(AO_ERI_ARTIFACT) + .ok_or(ArtifactError::Missing)?; + if artifact.representation != COMPACT_ERI_REPRESENTATION { + return Err(ArtifactError::UnsupportedRepresentation( + artifact.representation.clone(), + ) + .into()); + } + } + if !self.eri_is_compatible(calculation) { + return Err(PortableError::IncompatibleEri); + } + self.read_eri()?; + Ok(self.take_eri().expect("validated ERI is loaded")) + } + pub(crate) fn validate_unloaded_eri(&mut self) -> Result<(), PortableError> { let artifact = &self.manifest.artifacts[AO_ERI_ARTIFACT]; let ArtifactAttributes::AoEri(attributes) = &artifact.attributes else { diff --git a/crates/rustiq-core/src/persistence/data/portable/tests.rs b/crates/rustiq-core/src/persistence/data/portable/tests.rs index a0184ece..f83d9a76 100644 --- a/crates/rustiq-core/src/persistence/data/portable/tests.rs +++ b/crates/rustiq-core/src/persistence/data/portable/tests.rs @@ -463,7 +463,18 @@ fn corrupt_payload_is_lazy_but_cannot_be_read_or_republished() { .unwrap() ^= 1; let corrupt = dir.path().join("corrupt.rustiq"); write_members(&corrupt, &contents); + let portable_calculation = RustiQData::open(&corrupt) + .unwrap() + .prepare_calculation() + .unwrap(); + use crate::calculation::{CalculationError, CalculationExecution}; + let error = portable_calculation.execute().unwrap_err(); + assert!(matches!( + error.cause(), + CalculationError::Artifact(ArtifactError::IntegrityMismatch(_)) + )); let mut restored = RustiQData::open(&corrupt).unwrap(); + assert!(restored.take_compatible_eri(&calculation()).is_err()); assert!(restored.read_eri().is_err()); let output = dir.path().join("output.rustiq"); assert!(restored.write(&output).is_err()); @@ -666,6 +677,7 @@ fn malformed_npy_is_rejected_even_with_correct_digest() { let bad = dir.path().join("bad.rustiq"); write_members(&bad, &contents); let mut restored = RustiQData::open(&bad).unwrap(); + assert!(restored.take_compatible_eri(&calculation()).is_err()); assert!(restored.read_eri().is_err()); assert!(restored.write(dir.path().join("copy.rustiq")).is_err()); } @@ -881,6 +893,7 @@ fn unsupported_representation_is_inspectable_but_not_scientifically_usable() { let mut restored = RustiQData::open(&unknown).unwrap(); assert!(restored.calculation().is_some()); assert!(!restored.eri_is_compatible(&calculation())); + assert!(restored.take_compatible_eri(&calculation()).is_err()); assert!(restored.read_eri().is_err()); restored.write(dir.path().join("copy.rustiq")).unwrap(); } diff --git a/crates/rustiq-core/src/persistence/data/rustiq_data.rs b/crates/rustiq-core/src/persistence/data/rustiq_data.rs index 16ac8e21..e24ef039 100644 --- a/crates/rustiq-core/src/persistence/data/rustiq_data.rs +++ b/crates/rustiq-core/src/persistence/data/rustiq_data.rs @@ -122,6 +122,12 @@ impl RustiQData { /// Replaces the AO ERI artifact after checking its compact length. pub fn set_eri(&mut self, eri: CompactEri) -> Result<(), ArtifactError> { + self.validate_eri(&eri)?; + self.ao_eri = Some(eri); + Ok(()) + } + + pub(super) fn validate_eri(&self, eri: &CompactEri) -> Result<(), ArtifactError> { if self .context .as_ref() @@ -131,10 +137,7 @@ impl RustiQData { "AO ERI computation version".into(), )); } - let basis_functions = self.basis_functions.ok_or(ArtifactError::Missing)?; - validate_eri_len(&eri, basis_functions)?; - self.ao_eri = Some(eri); - Ok(()) + validate_eri_len(eri, self.basis_functions.ok_or(ArtifactError::Missing)?) } /// Validates and decodes the AO ERI on first access, then reuses the object. @@ -200,7 +203,7 @@ impl RustiQData { result } - pub(crate) fn write_with_eri( + pub(crate) fn write_storage_with_eri( &mut self, destination: Storage, eri: &CompactEri, @@ -208,6 +211,14 @@ impl RustiQData { self.write_inner(destination, Some(eri), true).map(|_| ()) } + pub(crate) fn write_entry_with_eri( + &mut self, + destination: Storage, + eri: &CompactEri, + ) -> Result { + self.write_inner(destination, Some(eri), false) + } + pub(crate) fn write_entry( &mut self, destination: Storage, @@ -313,7 +324,7 @@ impl RustiQData { if let Some(eri) = eri { let basis_functions = self.basis_functions.ok_or(ArtifactError::Missing)?; - validate_eri_len(eri, basis_functions)?; + self.validate_eri(eri)?; let path = RelativePath::new(AO_ERI_PATH); let metadata = destination .write_artifact::(path, |writer| { diff --git a/crates/rustiq-core/src/persistence/eri_cache.rs b/crates/rustiq-core/src/persistence/eri_cache.rs index 771018c8..debbb4a3 100644 --- a/crates/rustiq-core/src/persistence/eri_cache.rs +++ b/crates/rustiq-core/src/persistence/eri_cache.rs @@ -341,7 +341,7 @@ impl EriCache { let mut entry_data = RustiQData::new_with_identity(identity, basis_functions, ManifestKind::IntegralCache); entry_data - .write_with_eri(Storage::folder(temporary.path()), eri) + .write_storage_with_eri(Storage::folder(temporary.path()), eri) .map_err(io::Error::other)?; let temporary_path = temporary.keep(); if fs::symlink_metadata(&final_entry).is_ok() { diff --git a/crates/rustiq-core/src/persistence/error.rs b/crates/rustiq-core/src/persistence/error.rs index da9de026..f48e40c6 100644 --- a/crates/rustiq-core/src/persistence/error.rs +++ b/crates/rustiq-core/src/persistence/error.rs @@ -17,6 +17,8 @@ pub(crate) enum StorageError { /// Portable persistence failures, independent of the physical container library. #[derive(Debug, Error)] pub enum PortableError { + #[error("AO ERI is incompatible with the prepared calculation")] + IncompatibleEri, #[error("portable artifact I/O failed: {0}")] Io(#[from] io::Error), #[error("the destination already exists")] diff --git a/crates/rustiq-core/tests/automatic_artifact_reuse.rs b/crates/rustiq-core/tests/automatic_artifact_reuse.rs new file mode 100644 index 00000000..8ecc8199 --- /dev/null +++ b/crates/rustiq-core/tests/automatic_artifact_reuse.rs @@ -0,0 +1,263 @@ +use rustiq_core::{ + basis::BasisFile, + calculation::{ + ArtifactReuseDecision, CalculationBuilder, CalculationEvent, CalculationExecution, + }, + config::Mp2Config, + molecules::geometry::Geometry, + persistence::{CompactEri, EriCache, RustiQData}, +}; + +fn inputs(distance: f64) -> (Geometry, BasisFile) { + let geometry = Geometry::from_source( + "h2.xyz", + &format!("2\nH2 in Bohr\nH 0 0 0\nH {distance} 0 0\n"), + ) + .unwrap(); + let basis = BasisFile::from_reader(&include_bytes!("data/sto-3g.json")[..]).unwrap(); + (geometry, basis) +} + +#[test] +fn execute_automatically_reuses_portable_eri_and_preserves_mp2_results() { + let (geometry, basis) = inputs(1.4); + let original = CalculationBuilder::new(&geometry, &basis) + .with_mp2(Mp2Config::default()) + .prepare() + .unwrap(); + let expected = original.execute().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("h2.rustiq"); + RustiQData::from_calculation(&original) + .unwrap() + .write_with_eri(&path, expected.hf.ao_eri()) + .unwrap(); + let archive_bytes = std::fs::read(&path).unwrap(); + let restored = RustiQData::open(&path) + .unwrap() + .prepare_calculation() + .unwrap(); + let mut decisions = Vec::new(); + let result = restored + .execute_with_events(|event| { + if let CalculationEvent::ArtifactReuse(event) = event { + assert_eq!(event.artifact, "ao_eri"); + decisions.push(event.decision); + } + }) + .unwrap(); + assert_eq!(decisions, [ArtifactReuseDecision::Reused]); + assert_eq!( + result.hf.summary().scf.total_energy, + expected.hf.summary().scf.total_energy + ); + assert_eq!( + result.mp2.unwrap().correlation_energy, + expected.mp2.unwrap().correlation_energy + ); + + let overridden = restored.run_hf_with_eri(CompactEri::Zeroed(2)).unwrap(); + assert_eq!(overridden.ao_eri()[(0, 0, 0, 0)], 0.0); + let repeated = restored.execute().unwrap(); + let hf_only = restored.run_hf().unwrap(); + let pointer = &result.hf.ao_eri()[(0, 0, 0, 0)] as *const f64; + assert_eq!(pointer, &repeated.hf.ao_eri()[(0, 0, 0, 0)] as *const f64); + assert_eq!(pointer, &hf_only.ao_eri()[(0, 0, 0, 0)] as *const f64); + assert_eq!(std::fs::read(path).unwrap(), archive_bytes); +} + +#[test] +fn missing_eri_are_computed_once_and_retained_for_normal_execution() { + let (geometry, basis) = inputs(1.4); + let original = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + let expected = original.execute().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("inputs-only.rustiq"); + RustiQData::from_calculation(&original) + .unwrap() + .write(&path) + .unwrap(); + let archive_bytes = std::fs::read(&path).unwrap(); + let restored = RustiQData::open(&path) + .unwrap() + .prepare_calculation() + .unwrap(); + let mut decisions = Vec::new(); + let mut execute = || { + restored + .execute_with_events(|event| { + if let CalculationEvent::ArtifactReuse(event) = event { + decisions.push(event.decision); + } + }) + .unwrap() + }; + let first = execute(); + let second = execute(); + assert_eq!( + decisions, + [ + ArtifactReuseDecision::Missing, + ArtifactReuseDecision::Reused + ] + ); + assert_eq!( + first.hf.summary().scf.total_energy, + expected.hf.summary().scf.total_energy + ); + assert_eq!( + &first.hf.ao_eri()[(0, 0, 0, 0)] as *const f64, + &second.hf.ao_eri()[(0, 0, 0, 0)] as *const f64, + ); + assert_eq!(std::fs::read(path).unwrap(), archive_bytes); +} + +#[test] +fn incompatible_artifacts_fall_back_to_the_current_calculation_and_local_cache() { + let (source_geometry, basis) = inputs(1.4); + let source_calculation = CalculationBuilder::new(&source_geometry, &basis) + .prepare() + .unwrap(); + let mut source = RustiQData::from_calculation(&source_calculation).unwrap(); + source.set_eri(CompactEri::Zeroed(2)).unwrap(); + let (geometry, _) = inputs(1.6); + let expected = CalculationBuilder::new(&geometry, &basis) + .execute() + .unwrap(); + let directory = tempfile::tempdir().unwrap(); + let cache_root = directory.path().join("cache"); + let calculation = CalculationBuilder::new(&geometry, &basis) + .with_eri_cache(EriCache::new(&cache_root)) + .prepare() + .unwrap() + .with_reuse_data(source) + .unwrap(); + let mut decisions = Vec::new(); + let mut cache_events = Vec::new(); + let result = calculation + .execute_with_events(|event| match event { + CalculationEvent::ArtifactReuse(event) => decisions.push(event.decision), + CalculationEvent::EriCache(event) => cache_events.push(event.action), + _ => {} + }) + .unwrap(); + assert_eq!(decisions, [ArtifactReuseDecision::Incompatible]); + assert_eq!( + cache_events, + [rustiq_core::calculation::EriCacheAction::Stored] + ); + assert_eq!( + result.hf.summary().scf.total_energy, + expected.hf.summary().scf.total_energy + ); + assert!(result.hf.ao_eri()[(0, 0, 0, 0)] > 0.0); +} + +#[test] +fn explicit_portable_eri_take_precedence_over_local_cache_and_computation() { + let (geometry, basis) = inputs(1.4); + let original = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + let mut source = RustiQData::from_calculation(&original).unwrap(); + source.set_eri(CompactEri::Zeroed(2)).unwrap(); + let directory = tempfile::tempdir().unwrap(); + let cache_root = directory.path().join("unused-cache"); + let calculation = CalculationBuilder::new(&geometry, &basis) + .with_eri_cache(EriCache::new(&cache_root)) + .prepare() + .unwrap() + .with_reuse_data(source) + .unwrap(); + let result = calculation.execute().unwrap(); + assert_eq!(result.hf.ao_eri()[(0, 0, 0, 0)], 0.0); + assert!(!cache_root.exists()); +} + +#[test] +fn restored_open_shell_calculation_preserves_requested_units_and_resolved_method() { + use rustiq_core::{ + config::{DiisConfig, HfConfig, MoleculeConfig, ResolvedHfMethod}, + molecules::units::Units, + }; + use std::num::NonZeroU8; + let geometry = Geometry::from_source("oh.xyz", "2\nOH\nO 0 0 0\nH 0 0 0.97\n").unwrap(); + let (_, basis) = inputs(1.4); + let original = CalculationBuilder::new(&geometry, &basis) + .with_molecule_config(MoleculeConfig { + units: Units::Angstrom, + multiplicity: NonZeroU8::new(2).unwrap().into(), + ..Default::default() + }) + .with_hf(HfConfig { + diis: DiisConfig { + enabled: true, + ..Default::default() + }, + ..Default::default() + }) + .prepare() + .unwrap(); + let expected = original.execute().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("oh.rustiq"); + RustiQData::from_calculation(&original) + .unwrap() + .write_with_eri(&path, expected.hf.ao_eri()) + .unwrap(); + let restored = RustiQData::open(path) + .unwrap() + .prepare_calculation() + .unwrap(); + assert_eq!(restored.hf_method(), ResolvedHfMethod::Uhf); + assert_eq!(restored.request().molecule().units, Units::Angstrom); + assert!(restored.hf_config().diis.enabled); + let result = restored.execute().unwrap(); + assert!(result.hf.is_converged()); + assert_eq!( + result.hf.summary().scf.total_energy, + expected.hf.summary().scf.total_energy + ); +} + +#[test] +fn resolved_spherical_ao_components_are_restored_without_renormalization() { + let (geometry, _) = inputs(1.4); + let mut basis_json: serde_json::Value = + serde_json::from_slice(include_bytes!("data/sto-3g.json")).unwrap(); + basis_json["elements"]["1"]["electron_shells"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "function_type": "gto_spherical", + "angular_momentum": [2], + "exponents": ["0.8"], + "coefficients": [["1.0"]] + })); + let basis = + BasisFile::from_reader(serde_json::to_vec(&basis_json).unwrap().as_slice()).unwrap(); + let original = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + let restored = RustiQData::from_calculation(&original) + .unwrap() + .prepare_calculation() + .unwrap(); + assert_eq!(original.get_basis().nbasis(), 12); + assert_eq!( + restored.get_basis().overlap_ints(), + original.get_basis().overlap_ints() + ); + assert_eq!( + restored.get_basis().kinetic_ints(), + original.get_basis().kinetic_ints() + ); + let expected = original.execute().unwrap(); + let result = restored.execute().unwrap(); + assert_eq!( + result.hf.summary().scf.total_energy, + expected.hf.summary().scf.total_energy + ); +} diff --git a/crates/rustiq-core/tests/portable_artifact.rs b/crates/rustiq-core/tests/portable_artifact.rs index a12d19ed..ee09c887 100644 --- a/crates/rustiq-core/tests/portable_artifact.rs +++ b/crates/rustiq-core/tests/portable_artifact.rs @@ -146,3 +146,151 @@ fn public_bundle_api_preserves_restructured_options_for_multiple_calculations() .unwrap() .is_some()); } + +#[test] +fn borrowed_output_and_owned_input_preserve_eri_and_mp2() { + use rustiq_core::{ + calculation::CalculationExecution, config::Mp2Config, persistence::EriCache, + }; + let geometry = Geometry::from_source("h2.xyz", "2\nH2\nH 0 0 0\nH 1.4 0 0\n").unwrap(); + let basis = BasisFile::from_reader(&include_bytes!("data/sto-3g.json")[..]).unwrap(); + let prepared = CalculationBuilder::new(&geometry, &basis) + .with_mp2(Mp2Config::default()) + .prepare() + .unwrap(); + let ordinary = prepared.execute().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("h2.rustiq"); + let mut data = RustiQData::from_calculation(&prepared).unwrap(); + data.add_source("input.xyz", b"provenance".as_slice()) + .unwrap(); + let tensor = ordinary.hf.ao_eri(); + let pointer = &tensor[(0, 0, 0, 0)] as *const f64; + data.write_with_eri(&path, tensor).unwrap(); + assert_eq!(pointer, &ordinary.hf.ao_eri()[(0, 0, 0, 0)] as *const f64); + assert!(data.get::().unwrap().is_none()); + let mut restored = RustiQData::open(&path).unwrap(); + assert_eq!(restored.sources().len(), 1); + assert_eq!(restored.calculation().unwrap().charge(), 0); + let cache_root = directory.path().join("unused-cache"); + let cached = CalculationBuilder::new(&geometry, &basis) + .with_mp2(Mp2Config::default()) + .with_eri_cache(EriCache::new(&cache_root)) + .prepare() + .unwrap(); + let eri = restored.take_compatible_eri(&cached).unwrap(); + let pointer = &eri[(0, 0, 0, 0)] as *const f64; + let supplied = cached.execute_with_eri(eri).unwrap(); + assert_eq!(pointer, &supplied.hf.ao_eri()[(0, 0, 0, 0)] as *const f64); + assert!(!cache_root.exists()); + assert_eq!( + ordinary.hf.summary().scf.electronic_energy, + supplied.hf.summary().scf.electronic_energy + ); + assert_eq!( + ordinary.mp2.unwrap().correlation_energy, + supplied.mp2.unwrap().correlation_energy + ); + assert!(cached.execute_with_eri(CompactEri::Zeroed(3)).is_err()); + let zero = cached.run_hf_with_eri(CompactEri::Zeroed(2)).unwrap(); + assert_eq!(zero.ao_eri()[(0, 0, 0, 0)], 0.0); + assert!(!cache_root.exists()); +} + +#[test] +fn explicit_input_rejects_missing_incompatible_and_multiple_calculations() { + use rustiq_core::{ + config::IntegralConfig, + persistence::{ArtifactError, RustiQBundle}, + }; + let geometry = Geometry::from_source("h2.xyz", "2\nH2\nH 0 0 0\nH 1.4 0 0\n").unwrap(); + let basis = BasisFile::from_reader(&include_bytes!("data/sto-3g.json")[..]).unwrap(); + let prepared = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + let mut data = RustiQData::from_calculation(&prepared).unwrap(); + assert!(matches!( + data.take_compatible_eri(&prepared), + Err(PortableError::Artifact(ArtifactError::Missing)) + )); + data.set_eri(CompactEri::Zeroed(2)).unwrap(); + let changed = CalculationBuilder::new(&geometry, &basis) + .with_integrals(IntegralConfig { + schwarz_threshold: None.into(), + }) + .prepare() + .unwrap(); + assert!(matches!( + data.take_compatible_eri(&changed), + Err(PortableError::IncompatibleEri) + )); + assert!(data.read_eri().is_ok()); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("multi.rustiq"); + RustiQBundle::new(vec![data, RustiQData::from_calculation(&prepared).unwrap()]) + .unwrap() + .write(&path) + .unwrap(); + assert!(RustiQData::open(path).is_err()); +} + +#[test] +fn explicit_input_reports_corrupt_and_unsupported_eri() { + use rustiq_core::persistence::ArtifactError; + use std::{ + fs::File, + io::{Read, Write}, + }; + use zip::{write::SimpleFileOptions, ZipArchive, ZipWriter}; + let geometry = Geometry::from_source("h2.xyz", "2\nH2\nH 0 0 0\nH 1.4 0 0\n").unwrap(); + let basis = BasisFile::from_reader(&include_bytes!("data/sto-3g.json")[..]).unwrap(); + let prepared = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + let hf = prepared.run_hf().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let original = directory.path().join("original.rustiq"); + RustiQData::from_calculation(&prepared) + .unwrap() + .write_with_eri(&original, hf.ao_eri()) + .unwrap(); + for unsupported in [false, true] { + let path = directory + .path() + .join(format!("changed-{unsupported}.rustiq")); + let mut archive = ZipArchive::new(File::open(&original).unwrap()).unwrap(); + let mut writer = ZipWriter::new(File::create(&path).unwrap()); + for index in 0..archive.len() { + let mut member = archive.by_index(index).unwrap(); + let name = member.name().to_owned(); + let mut bytes = Vec::new(); + member.read_to_end(&mut bytes).unwrap(); + if unsupported && name == "manifest.json" { + let mut manifest: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + manifest["calculations"][0]["artifacts"]["ao_eri"]["representation"] = + "future-eri-v2".into(); + bytes = serde_json::to_vec(&manifest).unwrap(); + } else if !unsupported && name.ends_with("ao-eri.npy") { + *bytes.last_mut().unwrap() ^= 1; + } + writer + .start_file(name, SimpleFileOptions::default()) + .unwrap(); + writer.write_all(&bytes).unwrap(); + } + writer.finish().unwrap(); + let mut data = RustiQData::open(path).unwrap(); + let error = data.take_compatible_eri(&prepared).unwrap_err(); + if unsupported { + assert!(matches!( + error, + PortableError::Artifact(ArtifactError::UnsupportedRepresentation(_)) + )); + } else { + assert!(matches!( + error, + PortableError::Artifact(ArtifactError::IntegrityMismatch(_)) + )); + } + } +} diff --git a/docs/persistence-format-v1.md b/docs/persistence-format-v1.md index 15b70bad..fea1eadd 100644 --- a/docs/persistence-format-v1.md +++ b/docs/persistence-format-v1.md @@ -563,3 +563,84 @@ written by Rust. Rust tests also exercise a sparse archive with offsets beyond Run `uv run --locked pytest tools/reference` for interoperability and scientific reference comparisons, separately from the Cargo suite. + +## Single-calculation execution APIs + +Portable V1 can contain multiple calculations. The current convenience execution +API operates on one calculation, so `RustiQData` represents exactly one entry and +`RustiQData::open` rejects multi-calculation archives. Newly written single-entry +archives use `calculation-0` within the normal V1 bundle layout. Callers should +neither depend on nor manipulate the numeric index. + +```rust +// Calculation -> .rustiq +let prepared = CalculationBuilder::new(&geometry, &basis_file).prepare()?; +let hf = prepared.run_hf()?; +let mut data = RustiQData::from_calculation(&prepared)?; +data.write_with_eri("water.rustiq", hf.ao_eri())?; +``` + +The writer borrows the retained AO ERI without cloning or recomputing it. +`set_eri` remains available when the caller owns the tensor. + +```rust +// .rustiq -> Calculation +let data = RustiQData::open("water.rustiq")?; +let prepared = data.prepare_calculation()?; +let result = prepared.execute()?; +let eri = result.hf.ao_eri(); +``` + +`prepare_calculation` consumes the data facade and restores the resolved molecule, +AO basis, and scientific options. It retains the archive as a read-only artifact +source. Payloads are decoded lazily when execution needs them. Compatible AO ERIs +are verified and reused automatically by `execute`, `execute_with_events`, and +`run_hf`; missing or incompatible artifacts fall back to the enabled local cache +and then computation. Corrupt or unsupported compatible payloads fail explicitly. +The source archive is never modified. + +A new calculation can also reuse a previous archive with different requested options: + +```rust +let prepared = CalculationBuilder::new(&geometry, &basis_file) + .with_mp2(Mp2Config::default()) + .prepare()? + .with_reuse_data(RustiQData::open("water-hf.rustiq")?)?; +let result = prepared.execute()?; +``` + +The execution interface is independent of the input source. An internal enum +distinguishes configuration-only execution from portable artifact reuse. The +resolver uses the existing sealed `Artifact` markers for presence, compatibility, +and loading, rather than archive paths or a public storage/backend trait. Future +HF solution and restart markers can participate at their respective stages through +the same resolver; those representations are not implemented yet. + +`CalculationEvent::ArtifactReuse` reports per-artifact reused, missing, or +incompatible decisions. Its decision type also distinguishes intentionally ignored +state for future selection policies. Computed and reused values are retained in a +separate in-memory typed data facade for subsequent executions. Compact ERI clones +share immutable storage; mutation detaches that storage. Neither ordinary reuse nor +this memory retention rewrites the source snapshot or stores runtime scratch data. + +For callers managing tensors explicitly, `take_compatible_eri`, +`execute_with_eri`, and `run_hf_with_eri` remain available. This explicit path +rejects missing, incompatible, corrupt, or unsupported artifacts, and validates +tensor dimensions. ERIs remain accessible in the HF solution and are reused by MP2. + +A complete H2/STO-3G example set is in +[`crates/rustiq-core/examples`](../crates/rustiq-core/examples). Each file shows +one workflow. `calculate_from_rustiq` uses only the archive; the other examples +load geometry and basis from repository sample and test data: + +```sh +cargo run -p rustiq-core --example hf_calculation +cargo run -p rustiq-core --example write_rustiq +cargo run -p rustiq-core --example read_rustiq +cargo run -p rustiq-core --example calculate_from_rustiq +cargo run -p rustiq-core --example copy_rustiq +``` + +The examples read and write archives in the ignored +`crates/rustiq-core/examples/data/` directory. The read examples expect +`h2.rustiq` to have been created by `write_rustiq`. From 34491812b6660fa891d84180ff1a763241d01e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Mon, 5 Oct 2026 21:21:32 +0200 Subject: [PATCH 8/8] Polish portable artifact reuse API --- .../src/calculation/artifact_reuse.rs | 2 + .../rustiq-core/src/calculation/execution.rs | 4 +- .../src/persistence/data/artifact.rs | 5 +++ .../src/persistence/data/bundle.rs | 5 +++ .../src/persistence/data/portable/tests.rs | 21 +++++++++ crates/rustiq-core/src/prelude.rs | 39 +++++++++++++++-- crates/rustiq-core/tests/prelude_portable.rs | 43 +++++++++++++++++++ 7 files changed, 114 insertions(+), 5 deletions(-) create mode 100644 crates/rustiq-core/tests/prelude_portable.rs diff --git a/crates/rustiq-core/src/calculation/artifact_reuse.rs b/crates/rustiq-core/src/calculation/artifact_reuse.rs index b64ca8c3..4c84cfe2 100644 --- a/crates/rustiq-core/src/calculation/artifact_reuse.rs +++ b/crates/rustiq-core/src/calculation/artifact_reuse.rs @@ -28,6 +28,8 @@ impl ArtifactReuse { }) } + /// Resolve cached values through cheap `Clone`; artifact implementations document this + /// invariant because large payloads must not be duplicated during reuse. pub(super) fn resolve( &self, calculation: &PreparedCalculation, diff --git a/crates/rustiq-core/src/calculation/execution.rs b/crates/rustiq-core/src/calculation/execution.rs index 101dd3c8..c58630e0 100644 --- a/crates/rustiq-core/src/calculation/execution.rs +++ b/crates/rustiq-core/src/calculation/execution.rs @@ -28,13 +28,13 @@ pub struct EriCacheEvent { pub fingerprint: String, } -/// Per-artifact reuse decision. Ignoring an artifact never means deleting it. +/// Per-artifact reuse decision. +#[non_exhaustive] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ArtifactReuseDecision { Reused, Missing, Incompatible, - Ignored, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/rustiq-core/src/persistence/data/artifact.rs b/crates/rustiq-core/src/persistence/data/artifact.rs index 88dccaf4..e24c1425 100644 --- a/crates/rustiq-core/src/persistence/data/artifact.rs +++ b/crates/rustiq-core/src/persistence/data/artifact.rs @@ -6,6 +6,11 @@ pub(crate) mod private { } /// A known scientific artifact. Only RustiQ's declared artifact markers implement this trait. +/// +/// Values for reusable heavy artifacts must be cheap to clone: use shared immutable backing +/// storage or another representation that avoids copying the payload. The resolver may clone +/// values while moving them between the source, resolved state, and execution. Implementing +/// `Clone` as a deep copy for a large artifact is incorrect for this architecture. pub trait Artifact: private::Sealed { type Value; diff --git a/crates/rustiq-core/src/persistence/data/bundle.rs b/crates/rustiq-core/src/persistence/data/bundle.rs index ca9527c4..b1e2e6ac 100644 --- a/crates/rustiq-core/src/persistence/data/bundle.rs +++ b/crates/rustiq-core/src/persistence/data/bundle.rs @@ -203,6 +203,11 @@ pub(super) fn write_bundle_with_eri( if calculations.is_empty() { return Err(invalid("portable bundles require at least one calculation")); } + if eri.is_some() && calculations.len() != 1 { + return Err(invalid( + "a borrowed AO ERI can only be written with a single calculation", + )); + } validate_sources(sources)?; match std::fs::symlink_metadata(path) { Ok(_) => return Err(PortableError::AlreadyExists), diff --git a/crates/rustiq-core/src/persistence/data/portable/tests.rs b/crates/rustiq-core/src/persistence/data/portable/tests.rs index f83d9a76..3f974b18 100644 --- a/crates/rustiq-core/src/persistence/data/portable/tests.rs +++ b/crates/rustiq-core/src/persistence/data/portable/tests.rs @@ -30,6 +30,27 @@ fn data() -> RustiQData { .unwrap(); data } + +#[test] +fn borrowed_eri_writer_rejects_multi_calculation_input() { + let first = RustiQData::from_calculation(&calculation()).unwrap(); + let second = RustiQData::from_calculation(&calculation()).unwrap(); + let mut calculations = [first, second]; + let directory = tempfile::tempdir().unwrap(); + let destination = directory.path().join("invalid.rustiq"); + + let error = super::super::bundle::write_bundle_with_eri( + &mut calculations, + &[], + &destination, + Some(&CompactEri::Zeroed(2)), + ) + .unwrap_err(); + + assert!(matches!(error, PortableError::InvalidArchive(_))); + assert!(!destination.exists()); +} + fn source_fixture() -> Vec { include_str!("../../../../tests/data/persistence/source-original-v1.toml.hex") .trim() diff --git a/crates/rustiq-core/src/prelude.rs b/crates/rustiq-core/src/prelude.rs index 78b0f99b..b1f4c2bd 100644 --- a/crates/rustiq-core/src/prelude.rs +++ b/crates/rustiq-core/src/prelude.rs @@ -1,4 +1,4 @@ -//! Common types and the execution trait for direct Rust calculations. +//! Common types for direct calculations and portable `.rustiq` workflows. //! //! ```no_run //! use rustiq_core::prelude::*; @@ -22,13 +22,46 @@ //! # Ok(()) //! # } //! ``` +//! +//! A portable archive can also supply reusable artifacts to normal execution: +//! +//! ```no_run +//! use rustiq_core::prelude::*; +//! +//! # fn main() -> Result<(), Box> { +//! let geometry = Geometry::from_reader(std::io::Cursor::new( +//! "2\nH2\nH 0 0 -0.37\nH 0 0 0.37\n", +//! ))?; +//! let basis = BasisFile::from_reader(std::fs::File::open("sto-3g.json")?)?; +//! let prepared = CalculationBuilder::new(&geometry, &basis).prepare()?; +//! let hf = prepared.run_hf()?; +//! let eri: &CompactEri = hf.ao_eri(); +//! +//! let directory = tempfile::tempdir()?; +//! let path = directory.path().join("water.rustiq"); +//! let mut data = RustiQData::from_calculation(&prepared)?; +//! data.write_with_eri(&path, eri)?; +//! +//! let restored = RustiQData::open(&path)?.prepare_calculation()?; +//! let result = restored.execute()?; +//! +//! // Strict ownership-transfer path: missing or incompatible ERIs are errors. +//! let mut data = RustiQData::open(&path)?; +//! let eri = data.take_compatible_eri(&prepared)?; +//! let result = prepared.execute_with_eri(eri)?; +//! let _ = (result, Option::::None); +//! # Ok(()) +//! # } +//! ``` pub use crate::{ basis::BasisFile, calculation::{ - CalculationBuilder, CalculationExecution, CalculationExecutionError, CalculationResult, - Converged, HfOutcome, HfSolution, PreparedCalculation, Unconverged, + ArtifactReuseDecision, ArtifactReuseEvent, CalculationBuilder, CalculationEvent, + CalculationExecution, CalculationExecutionError, CalculationResult, Converged, HfOutcome, + HfSolution, PreparedCalculation, Unconverged, }, config::{HfConfig, HfMethod, MoleculeConfig, Mp2Config}, molecules::{atom::Atom, geometry::Geometry, units::Units}, + persistence::{CompactEri, RustiQBundle, RustiQData}, }; diff --git a/crates/rustiq-core/tests/prelude_portable.rs b/crates/rustiq-core/tests/prelude_portable.rs new file mode 100644 index 00000000..b63ff469 --- /dev/null +++ b/crates/rustiq-core/tests/prelude_portable.rs @@ -0,0 +1,43 @@ +use rustiq_core::prelude::*; + +#[test] +fn portable_workflow_uses_only_the_public_prelude() { + let geometry = + Geometry::from_reader(std::io::Cursor::new("2\nH2 in Bohr\nH 0 0 0\nH 1.4 0 0\n")).unwrap(); + let basis = BasisFile::from_reader(&include_bytes!("data/sto-3g.json")[..]).unwrap(); + let prepared = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + + let hf = prepared.run_hf().unwrap(); + let eri: &CompactEri = hf.ao_eri(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("h2.rustiq"); + let mut data = RustiQData::from_calculation(&prepared).unwrap(); + data.write_with_eri(&path, eri).unwrap(); + + let restored = RustiQData::open(&path) + .unwrap() + .prepare_calculation() + .unwrap(); + let mut reused = false; + let result = restored + .execute_with_events(|event| { + if let CalculationEvent::ArtifactReuse(event) = event { + reused = event.decision == ArtifactReuseDecision::Reused; + assert_eq!(event.artifact, "ao_eri"); + } + }) + .unwrap(); + assert!(reused); + let _: &HfOutcome = &result.hf; + + let mut strict_data = RustiQData::open(&path).unwrap(); + let transferred: CompactEri = strict_data.take_compatible_eri(&prepared).unwrap(); + let strict_result = prepared.execute_with_eri(transferred).unwrap(); + let _: &CompactEri = strict_result.hf.ao_eri(); + + let _: Option = None; + let _: Option = None; + let _: Option = None; +}