diff --git a/.gitignore b/.gitignore index 4b4f0154..07a69d34 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ __pycache__/ .fake /result /result-* +/crates/rustiq-core/examples/data/ diff --git a/Cargo.lock b/Cargo.lock index 1df32a16..e483fcff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -902,7 +902,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aad4cf7e378685c1af30f41b4832c29700325524c648d47752cabd5b97e7b09b" dependencies = [ - "zip", + "zip 0.6.6", ] [[package]] @@ -2651,6 +2651,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "zip 8.6.0", ] [[package]] @@ -3523,6 +3524,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -4201,6 +4208,19 @@ dependencies = [ "flate2", ] +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", +] + [[package]] name = "zlib-rs" version = "0.6.7" diff --git a/crates/rustiq-core/Cargo.toml b/crates/rustiq-core/Cargo.toml index 3d156b35..d0cfb1c8 100644 --- a/crates/rustiq-core/Cargo.toml +++ b/crates/rustiq-core/Cargo.toml @@ -29,7 +29,7 @@ periodic_table.workspace = true rayon.workspace = true relative-path.workspace = true tokio = { workspace = true, features = ["fs", "io-util"], optional = true } -serde_json.workspace = true +serde_json = { workspace = true, features = ["float_roundtrip"] } serde.workspace = true sha2 = "0.11.0" delegate.workspace = true @@ -47,6 +47,7 @@ smallvec = "1" tempfile.workspace = true bytesize.workspace = true sysinfo.workspace = true +zip = { version = "8", default-features = false, features = ["deflate-flate2-zlib-rs"] } [dev-dependencies] approx.workspace = true diff --git a/crates/rustiq-core/examples/calculate_from_rustiq.rs b/crates/rustiq-core/examples/calculate_from_rustiq.rs new file mode 100644 index 00000000..16621bee --- /dev/null +++ b/crates/rustiq-core/examples/calculate_from_rustiq.rs @@ -0,0 +1,24 @@ +//! Restore a calculation and its AO ERI from a single `.rustiq` archive. +//! Create the default archive with `cargo run -p rustiq-core --example write_rustiq`, +//! then run this example with `cargo run -p rustiq-core --example calculate_from_rustiq`. +#[allow(dead_code)] +mod common; + +use rustiq_core::{calculation::CalculationExecution, persistence::RustiQData}; + +fn main() -> Result<(), Box> { + let input = common::data_path("h2.rustiq")?; + let data = RustiQData::open(&input)?; + let prepared = data.prepare_calculation()?; + let result = prepared.execute()?; + println!("Retained AO ERI values: {}", result.hf.ao_eri().len()); + + println!( + "HF energy: {:.12} Eh", + result.hf.summary().scf.electronic_energy + ); + if let Some(mp2) = result.mp2 { + println!("MP2 correlation energy: {:.12} Eh", mp2.correlation_energy); + } + Ok(()) +} diff --git a/crates/rustiq-core/examples/common/mod.rs b/crates/rustiq-core/examples/common/mod.rs new file mode 100644 index 00000000..da450c98 --- /dev/null +++ b/crates/rustiq-core/examples/common/mod.rs @@ -0,0 +1,26 @@ +use std::fs::File; +use std::path::PathBuf; + +use rustiq_core::{basis::BasisFile, molecules::geometry::Geometry}; + +pub fn h2_inputs() -> Result<(Geometry, BasisFile), Box> { + let geometry_path = workspace_path("samples/h2/molecule.xyz")?.canonicalize()?; + let basis_path = workspace_path("tests/data/sto-3g.json")?.canonicalize()?; + let geometry = Geometry::from_file(File::open(geometry_path)?)?; + let basis = BasisFile::from_reader(File::open(basis_path)?)?; + Ok((geometry, basis)) +} + +pub fn workspace_path(path: &str) -> Result> { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize()?; + Ok(root.join(path)) +} + +#[allow(dead_code)] +pub fn data_path(filename: &str) -> Result> { + let directory = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("examples/data"); + std::fs::create_dir_all(&directory)?; + Ok(directory.canonicalize()?.join(filename)) +} diff --git a/crates/rustiq-core/examples/copy_rustiq.rs b/crates/rustiq-core/examples/copy_rustiq.rs new file mode 100644 index 00000000..ed505a74 --- /dev/null +++ b/crates/rustiq-core/examples/copy_rustiq.rs @@ -0,0 +1,24 @@ +//! Read h2.rustiq, run HF and MP2, then write the retained AO ERI to h2-copy.rustiq. +mod common; + +use rustiq_core::{calculation::CalculationBuilder, config::Mp2Config, persistence::RustiQData}; + +fn main() -> Result<(), Box> { + let (geometry, basis) = common::h2_inputs()?; + let prepared = CalculationBuilder::new(&geometry, &basis) + .with_mp2(Mp2Config::default()) + .prepare()?; + let input = common::data_path("h2.rustiq")?; + let mut data = RustiQData::open(input)?; + let eri = data.take_compatible_eri(&prepared)?; + let result = prepared.execute_with_eri(eri)?; + let mut output = RustiQData::from_calculation(&prepared)?; + let output_path = common::data_path("h2-copy.rustiq")?; + output.write_with_eri(&output_path, result.hf.ao_eri())?; + println!( + "HF energy: {:.12} Eh", + result.hf.summary().scf.electronic_energy + ); + println!("Wrote {}", output_path.display()); + Ok(()) +} diff --git a/crates/rustiq-core/examples/hf_calculation.rs b/crates/rustiq-core/examples/hf_calculation.rs new file mode 100644 index 00000000..5fcb7df3 --- /dev/null +++ b/crates/rustiq-core/examples/hf_calculation.rs @@ -0,0 +1,23 @@ +//! Run HF and MP2 directly from the H2 sample inputs. +mod common; + +use rustiq_core::{ + calculation::{CalculationBuilder, CalculationExecution}, + config::Mp2Config, +}; + +fn main() -> Result<(), Box> { + let (geometry, basis) = common::h2_inputs()?; + let prepared = CalculationBuilder::new(&geometry, &basis) + .with_mp2(Mp2Config::default()) + .prepare()?; + let result = prepared.execute()?; + println!( + "HF energy: {:.12} Eh", + result.hf.summary().scf.electronic_energy + ); + if let Some(mp2) = result.mp2 { + println!("MP2 correlation energy: {:.12} Eh", mp2.correlation_energy); + } + Ok(()) +} diff --git a/crates/rustiq-core/examples/read_rustiq.rs b/crates/rustiq-core/examples/read_rustiq.rs new file mode 100644 index 00000000..9f9bc4d0 --- /dev/null +++ b/crates/rustiq-core/examples/read_rustiq.rs @@ -0,0 +1,15 @@ +//! Run HF using the AO ERI stored in h2.rustiq. +mod common; + +use rustiq_core::{calculation::CalculationBuilder, persistence::RustiQData}; + +fn main() -> Result<(), Box> { + let (geometry, basis) = common::h2_inputs()?; + let prepared = CalculationBuilder::new(&geometry, &basis).prepare()?; + let input = common::data_path("h2.rustiq")?; + let mut data = RustiQData::open(input)?; + let eri = data.take_compatible_eri(&prepared)?; + let hf = prepared.run_hf_with_eri(eri)?; + println!("HF energy: {:.12} Eh", hf.summary().scf.electronic_energy); + Ok(()) +} diff --git a/crates/rustiq-core/examples/write_rustiq.rs b/crates/rustiq-core/examples/write_rustiq.rs new file mode 100644 index 00000000..6db30fa3 --- /dev/null +++ b/crates/rustiq-core/examples/write_rustiq.rs @@ -0,0 +1,15 @@ +//! Run HF and write its retained AO ERI to h2.rustiq. +mod common; + +use rustiq_core::calculation::CalculationBuilder; + +fn main() -> Result<(), Box> { + let (geometry, basis) = common::h2_inputs()?; + let prepared = CalculationBuilder::new(&geometry, &basis).prepare()?; + let hf = prepared.run_hf()?; + let mut data = rustiq_core::persistence::RustiQData::from_calculation(&prepared)?; + let output = common::data_path("h2.rustiq")?; + data.write_with_eri(&output, hf.ao_eri())?; + println!("Wrote {}", output.display()); + Ok(()) +} diff --git a/crates/rustiq-core/src/basis/gaussian/basis.rs b/crates/rustiq-core/src/basis/gaussian/basis.rs index c7da3bd8..1792921d 100644 --- a/crates/rustiq-core/src/basis/gaussian/basis.rs +++ b/crates/rustiq-core/src/basis/gaussian/basis.rs @@ -137,6 +137,88 @@ pub struct NormalizedPrimitive { } impl Basis { + /// Rebuilds the resolved AO basis stored in portable calculation context. + /// Its normalized primitive coefficients are copied verbatim. Resolved AO + /// snapshots retain effective components, not the original shell grouping; + /// integral evaluation only needs the centers alongside those components. + pub(crate) fn from_calculation_context( + context: &crate::persistence::CalculationContext, + ) -> Self { + let mut shells = Vec::new(); + let mut shell_ids = Vec::new(); + let mut angular_momenta = Vec::new(); + let mut angular_components = Vec::new(); + let mut normalized_components = Vec::new(); + + for ao in context.basis() { + let components: Vec<_> = ao.components().collect(); + let first = components + .first() + .expect("validated portable AO has at least one component"); + let shell_id = shells.len(); + let center = ao.center(); + shells.push(Shell { + alpha: nalgebra::DVector::zeros(0), + contr: Vec::new(), + origin: Point3::new(center[0], center[1], center[2]), + max_ln_coeff: nalgebra::DVector::zeros(0), + }); + shell_ids.push(shell_id); + let first_angular_momentum = first.angular_momentum(); + angular_momenta.push(Vector3::new( + first_angular_momentum[0], + first_angular_momentum[1], + first_angular_momentum[2], + )); + angular_components.push( + components + .iter() + .map(|component| { + let angular_momentum = component.angular_momentum(); + ( + Vector3::new( + angular_momentum[0], + angular_momentum[1], + angular_momentum[2], + ), + 1.0, + ) + }) + .collect(), + ); + normalized_components.push( + components + .iter() + .map(|component| NormalizedComponent { + angular_momentum: { + let angular_momentum = component.angular_momentum(); + Vector3::new( + angular_momentum[0], + angular_momentum[1], + angular_momentum[2], + ) + }, + primitives: component + .primitives() + .map(|(exponent, coefficient)| NormalizedPrimitive { + exponent, + coefficient, + }) + .collect(), + }) + .collect(), + ); + } + + Self { + shells, + shell_ids, + angular_momenta, + angular_components, + normalized_components, + } + } + pub(crate) fn new(shells: Vec) -> Self { let mut shell_ids = Vec::new(); let mut angular_momenta = Vec::new(); diff --git a/crates/rustiq-core/src/calculation.rs b/crates/rustiq-core/src/calculation.rs index 4404e958..8cfca874 100644 --- a/crates/rustiq-core/src/calculation.rs +++ b/crates/rustiq-core/src/calculation.rs @@ -78,6 +78,7 @@ pub use setup_error::HfSetupError; mod solution; use crate::hf::scf_result::ScfOutcome; pub use solution::{CalculationExecutionError, Converged, HfOutcome, HfSolution, Unconverged}; +mod artifact_reuse; mod builder; mod execution; mod prepared_calculation; @@ -101,8 +102,8 @@ pub use crate::{ }; pub use builder::CalculationBuilder; pub use execution::{ - CalculationEvent, CalculationExecution, CalculationResult, EriCacheAction, EriCacheEvent, - HfCalculationResult, Mp2MemoryPlan, + ArtifactReuseDecision, ArtifactReuseEvent, CalculationEvent, CalculationExecution, + CalculationResult, EriCacheAction, EriCacheEvent, HfCalculationResult, Mp2MemoryPlan, }; pub use prepared_calculation::PreparedCalculation; pub use request::CalculationRequest; @@ -155,6 +156,8 @@ pub enum CalculationError { Numerical(#[from] NumericalError), #[error(transparent)] Diis(#[from] DiisError), + #[error(transparent)] + Artifact(#[from] crate::persistence::ArtifactError), #[error("{error}")] Mp2 { #[source] @@ -222,17 +225,20 @@ impl<'a> HfCalculation<'a> { config, integrals, eri_cache, + None, progress, |_| {}, ) } + #[allow(clippy::too_many_arguments)] pub(crate) fn new_with_progress_and_cache( molecule: &'a Molecule, basis: &'a Basis, config: &HfConfig, integrals: &IntegralConfig, eri_cache: Option<&'a EriCache>, + supplied_eri: Option, mut progress: impl FnMut(ScfSetupStep), mut cache_event: impl FnMut(EriCacheEvent), ) -> Result { @@ -251,6 +257,7 @@ impl<'a> HfCalculation<'a> { &config.orthogonalization, integrals, eri_cache, + supplied_eri, &mut progress, &mut cache_event, ) diff --git a/crates/rustiq-core/src/calculation/artifact_reuse.rs b/crates/rustiq-core/src/calculation/artifact_reuse.rs new file mode 100644 index 00000000..4c84cfe2 --- /dev/null +++ b/crates/rustiq-core/src/calculation/artifact_reuse.rs @@ -0,0 +1,88 @@ +use std::sync::Mutex; + +use crate::persistence::{Artifact, ArtifactError, PortableError, RustiQData}; + +use super::{ArtifactReuseDecision, ArtifactReuseEvent, PreparedCalculation}; + +/// The origin of reusable state does not change the scientific execution API. +pub(super) enum CalculationSource { + Configuration, + Portable(Box), +} + +/// Read-only persistence input with lazily decoded typed domain values. +/// Numerical algorithms receive domain objects, never archive paths or keys. +pub(super) struct ArtifactReuse { + source: Mutex, + resolved: Mutex, +} + +impl ArtifactReuse { + pub(super) fn new( + data: RustiQData, + calculation: &PreparedCalculation, + ) -> Result { + Ok(Self { + source: Mutex::new(data), + resolved: Mutex::new(RustiQData::from_calculation(calculation)?), + }) + } + + /// Resolve cached values through cheap `Clone`; artifact implementations document this + /// invariant because large payloads must not be duplicated during reuse. + pub(super) fn resolve( + &self, + calculation: &PreparedCalculation, + ) -> Result<(Option, ArtifactReuseEvent), ArtifactError> + where + A::Value: Clone, + { + let mut resolved = self.resolved.lock().map_err(|_| { + ArtifactError::AccessFailed("resolved artifact mutex is poisoned".into()) + })?; + if let Some(value) = resolved.get::()?.cloned() { + return Ok(( + Some(value), + ArtifactReuseEvent { + artifact: A::NAME, + decision: ArtifactReuseDecision::Reused, + }, + )); + } + let mut data = self + .source + .lock() + .map_err(|_| ArtifactError::AccessFailed("artifact source mutex is poisoned".into()))?; + let (value, decision) = if !A::is_present(&data) { + (None, ArtifactReuseDecision::Missing) + } else if !A::is_compatible(&data, calculation) { + (None, ArtifactReuseDecision::Incompatible) + } else { + let value = data.get::()?.cloned(); + let decision = if value.is_some() { + ArtifactReuseDecision::Reused + } else { + ArtifactReuseDecision::Missing + }; + (value, decision) + }; + if let Some(value) = &value { + resolved.set::(value.clone())?; + } + Ok(( + value, + ArtifactReuseEvent { + artifact: A::NAME, + decision, + }, + )) + } + + /// Retains newly computed domain values in memory; never writes the source archive. + pub(super) fn remember(&self, value: A::Value) -> Result<(), ArtifactError> { + self.resolved + .lock() + .map_err(|_| ArtifactError::AccessFailed("resolved artifact mutex is poisoned".into()))? + .set::(value) + } +} diff --git a/crates/rustiq-core/src/calculation/builder.rs b/crates/rustiq-core/src/calculation/builder.rs index 3707f50a..80845b4b 100644 --- a/crates/rustiq-core/src/calculation/builder.rs +++ b/crates/rustiq-core/src/calculation/builder.rs @@ -203,6 +203,7 @@ impl<'a> CalculationBuilder<'a> { integrals: self.integrals, mp2: self.mp2, eri_cache: self.eri_cache.clone(), + source: super::artifact_reuse::CalculationSource::Configuration, }) } diff --git a/crates/rustiq-core/src/calculation/execution.rs b/crates/rustiq-core/src/calculation/execution.rs index f1da5dda..c58630e0 100644 --- a/crates/rustiq-core/src/calculation/execution.rs +++ b/crates/rustiq-core/src/calculation/execution.rs @@ -28,6 +28,21 @@ pub struct EriCacheEvent { pub fingerprint: String, } +/// Per-artifact reuse decision. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ArtifactReuseDecision { + Reused, + Missing, + Incompatible, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ArtifactReuseEvent { + pub artifact: &'static str, + pub decision: ArtifactReuseDecision, +} + /// Results of the requested scientific stages, without presentation choices. #[derive(Debug, Clone)] pub struct CalculationResult { @@ -55,6 +70,7 @@ pub enum CalculationEvent<'a> { config: &'a HfConfig, }, EriCache(EriCacheEvent), + ArtifactReuse(ArtifactReuseEvent), ScfSetup(ScfSetupStep), ScfIteration(&'a ScfIteration), HfCompleted(&'a HfOutcome), diff --git a/crates/rustiq-core/src/calculation/prepared_calculation.rs b/crates/rustiq-core/src/calculation/prepared_calculation.rs index bd51a1b7..c1c4a2af 100644 --- a/crates/rustiq-core/src/calculation/prepared_calculation.rs +++ b/crates/rustiq-core/src/calculation/prepared_calculation.rs @@ -2,20 +2,31 @@ use super::{ CalculationError, CalculationEvent, CalculationExecution, CalculationExecutionError, CalculationResult, HfCalculation, HfCalculationResult, HfOutcome, HfSolution, }; +use crate::eri::CompactEri; use crate::hf::scf_result::ScfOutcome; use crate::{ basis::Basis, - config::{HfConfig, IntegralConfig, Mp2Config, ResolvedHfMethod}, - molecules::molecule::Molecule, - persistence::EriCache, + config::{ + validated::{DiisSize, NonNegativeFiniteF64, PositiveFiniteF64}, + DiisConfig, HfConfig, HfMethod, IntegralConfig, Mp2Config, OrthogonalizationConfig, + ResolvedHfMethod, + }, + molecules::{atom::Atom, geometry::Geometry, molecule::Molecule, units::Units}, + persistence::{AoEriArtifact, EriCache, PortableError, RustiQData}, +}; +use std::{ + cell::RefCell, + num::{NonZeroU8, NonZeroUsize}, }; -use std::cell::RefCell; +use super::artifact_reuse::{ArtifactReuse, CalculationSource}; use super::CalculationRequest; -/// A validated molecule in Bohr and its basis, prepared together by the builder. +/// A validated molecule in Bohr and its basis, prepared from configuration or +/// restored from portable scientific data through the same execution interface. /// /// Each execution starts fresh HF state and uses the same immutable inputs. +/// Available compatible typed artifacts are reused automatically when a source is attached. /// This avoids self-referential SCF storage and allows reuse of the basis. pub struct PreparedCalculation { pub(super) request: CalculationRequest, @@ -26,9 +37,97 @@ pub struct PreparedCalculation { pub(super) integrals: IntegralConfig, pub(super) mp2: Option, pub(super) eri_cache: Option, + pub(super) source: CalculationSource, } impl PreparedCalculation { + pub(crate) fn from_portable_context( + request: CalculationRequest, + context: &crate::persistence::CalculationContext, + ) -> Self { + let atoms = context + .atoms() + .map(|(atomic_number, position)| { + let element = periodic_table::periodic_table() + .iter() + .find(|element| element.atomic_number == atomic_number) + .expect("validated portable context uses a known element"); + Atom::new( + element, + nalgebra::Point3::new(position[0], position[1], position[2]), + ) + }) + .collect(); + let geometry = Geometry::new("Restored from RustiQ data".into(), atoms); + let molecule = Molecule::try_new( + geometry, + Units::Bohr, + context.charge(), + NonZeroU8::new(context.multiplicity()).expect("validated multiplicity is nonzero"), + ) + .expect("validated portable context has a valid molecular state"); + let basis = Basis::from_calculation_context(context); + + let diis_size = context + .diis_size() + .map(|size| DiisSize::try_new(size).expect("validated DIIS size is valid")); + let hf = HfConfig { + method: match context.hf_method() { + ResolvedHfMethod::Rhf => HfMethod::Rhf, + ResolvedHfMethod::Uhf => HfMethod::Uhf, + } + .into(), + max_iterations: NonZeroUsize::new(context.max_iterations()) + .expect("validated iteration count is nonzero"), + convergence_threshold: PositiveFiniteF64::try_new(context.convergence_threshold()) + .expect("validated convergence threshold is positive and finite"), + guess: context.density_guess().into(), + diis: DiisConfig { + enabled: diis_size.is_some(), + max_history: diis_size + .unwrap_or_else(|| DiisSize::try_new(6).expect("default DIIS size is valid")) + .into(), + }, + orthogonalization: OrthogonalizationConfig { + linear_dependency_threshold: NonNegativeFiniteF64::try_new( + context.linear_dependency_threshold(), + ) + .expect("validated linear dependency threshold is finite and nonnegative") + .into(), + }, + }; + let integrals = IntegralConfig { + schwarz_threshold: context + .eri_schwarz_threshold() + .map(|threshold| { + PositiveFiniteF64::try_new(threshold) + .expect("validated ERI threshold is positive and finite") + }) + .into(), + }; + + Self { + basis_name: request.basis_name().to_owned(), + mp2: request.mp2().copied(), + request, + molecule, + basis, + hf: (hf, context.hf_method()), + integrals, + eri_cache: None, + source: CalculationSource::Configuration, + } + } + + /// Supplies a read-only source of reusable typed scientific artifacts. + /// Normal execution checks compatibility per artifact, then uses the enabled + /// local cache or computes missing/incompatible values. Corrupt required + /// payloads are errors. Newly computed values are retained in memory. + pub fn with_reuse_data(mut self, data: RustiQData) -> Result { + self.source = CalculationSource::Portable(Box::new(ArtifactReuse::new(data, &self)?)); + Ok(self) + } + /// Returns the normalized inputs as requested before scientific resolution. pub fn request(&self) -> &CalculationRequest { &self.request @@ -84,18 +183,48 @@ impl PreparedCalculation { &self, events: impl FnMut(CalculationEvent<'_>), ) -> Result { + self.run_hf_from_eri(None, events) + } + + /// Runs HF with an owned AO ERI, bypassing integral computation and the ERI cache. + pub fn run_hf_with_eri(&self, eri: CompactEri) -> Result { + self.run_hf_from_eri(Some(eri), |_| {}) + } + + fn run_hf_from_eri( + &self, + eri: Option, + events: impl FnMut(CalculationEvent<'_>), + ) -> Result { + let retain_artifact = eri.is_none(); let events = RefCell::new(events); let (config, method) = &self.hf; events.borrow_mut()(CalculationEvent::HfStarted { method: *method, config, }); + let eri = match (eri, &self.source) { + (Some(eri), _) => Some(eri), + (None, CalculationSource::Portable(reuse)) => { + let (eri, decision) = reuse + .resolve::(self) + .map_err(CalculationError::from)?; + events.borrow_mut()(CalculationEvent::ArtifactReuse(decision)); + eri + } + (None, CalculationSource::Configuration) => None, + }; let mut calculation = HfCalculation::new_with_progress_and_cache( &self.molecule, &self.basis, config, &self.integrals, - self.eri_cache.as_ref(), + if eri.is_some() { + None + } else { + self.eri_cache.as_ref() + }, + eri, |step| events.borrow_mut()(CalculationEvent::ScfSetup(step)), |event| events.borrow_mut()(CalculationEvent::EriCache(event)), )?; @@ -118,17 +247,33 @@ impl PreparedCalculation { calculation.state, )), }; + if retain_artifact { + if let CalculationSource::Portable(reuse) = &self.source { + reuse + .remember::(hf.ao_eri().clone()) + .map_err(|error| hf.clone().execution_error(CalculationError::from(error)))?; + } + } events.borrow_mut()(CalculationEvent::HfCompleted(&hf)); Ok(hf) } } -impl CalculationExecution for PreparedCalculation { - fn execute_with_events( +impl PreparedCalculation { + /// Executes HF and optional MP2 using the supplied AO ERI without copying it. + pub fn execute_with_eri( + &self, + eri: CompactEri, + ) -> Result { + self.execute_from_eri(Some(eri), |_| {}) + } + + fn execute_from_eri( &self, + eri: Option, mut events: impl FnMut(CalculationEvent<'_>), ) -> Result { - let hf = self.run_hf_with_events(&mut events)?; + let hf = self.run_hf_from_eri(eri, &mut events)?; let mp2 = self .mp2 .as_ref() @@ -156,3 +301,12 @@ impl CalculationExecution for PreparedCalculation { Ok(CalculationResult { hf, mp2 }) } } + +impl CalculationExecution for PreparedCalculation { + fn execute_with_events( + &self, + events: impl FnMut(CalculationEvent<'_>), + ) -> Result { + self.execute_from_eri(None, events) + } +} diff --git a/crates/rustiq-core/src/calculation/solution.rs b/crates/rustiq-core/src/calculation/solution.rs index e738ff85..8f6ed428 100644 --- a/crates/rustiq-core/src/calculation/solution.rs +++ b/crates/rustiq-core/src/calculation/solution.rs @@ -48,6 +48,14 @@ pub enum HfOutcome { } impl HfOutcome { + /// Borrows the AO ERI tensor, including when HF did not converge. + pub fn ao_eri(&self) -> &CompactEri { + match self { + Self::Converged(hf) => hf.ao_eri(), + Self::Unconverged(hf) => hf.ao_eri(), + } + } + /// Returns the resolved Hartree-Fock method, even when HF did not converge. pub fn method(&self) -> ResolvedHfMethod { self.summary().method @@ -73,6 +81,11 @@ impl HfOutcome { } impl HfSolution { + /// Borrows the retained AO ERI tensor without copying it. + pub fn ao_eri(&self) -> &CompactEri { + &self.0.integrals + } + pub(super) fn from_state(summary: HfCalculationResult, state: HfState<'_>) -> Self { let (orbitals, integrals) = match state { HfState::Rhf(scf) => ( diff --git a/crates/rustiq-core/src/eri.rs b/crates/rustiq-core/src/eri.rs index d9a51d4d..26378e3d 100644 --- a/crates/rustiq-core/src/eri.rs +++ b/crates/rustiq-core/src/eri.rs @@ -149,6 +149,12 @@ pub fn compute_eri_primitive( /// A 4D tensor containing all ERI integrals. #[derive(Debug, Error, PartialEq)] pub enum EriError { + #[error("supplied AO ERI has {actual} values, expected {expected} for {basis_functions} basis functions")] + InvalidValueCount { + basis_functions: usize, + expected: usize, + actual: usize, + }, #[error("non-finite Coulomb self-integral for AO pair {pair_index}: {value}")] NonFiniteSelfIntegral { pair_index: usize, value: f64 }, #[error( diff --git a/crates/rustiq-core/src/eri/compact.rs b/crates/rustiq-core/src/eri/compact.rs index fe50ca2d..9f611ca2 100644 --- a/crates/rustiq-core/src/eri/compact.rs +++ b/crates/rustiq-core/src/eri/compact.rs @@ -1,4 +1,7 @@ -use std::ops::{Index, IndexMut}; +use std::{ + ops::{Index, IndexMut}, + sync::Arc, +}; use rayon::prelude::*; use thiserror::Error; @@ -11,9 +14,10 @@ pub(crate) enum CompactEriBuildError { InvalidLength { expected: usize, actual: usize }, } -#[derive(Debug)] +/// Compact AO ERI values. Clones share storage until a value is modified. +#[derive(Debug, Clone)] pub struct CompactEri { - storage: Box<[f64]>, + storage: Arc<[f64]>, } impl CompactEri { @@ -64,7 +68,7 @@ impl CompactEri { return Err(CompactEriBuildError::InvalidLength { expected, actual }); } Ok(Self { - storage: values.into_boxed_slice(), + storage: values.into(), }) } @@ -92,9 +96,11 @@ impl CompactEri { }); } - let storage = par_iter.collect(); + let storage: Vec = par_iter.collect(); - Ok(Self { storage }) + Ok(Self { + storage: storage.into(), + }) } } @@ -108,7 +114,7 @@ impl Index for CompactEri { impl IndexMut for CompactEri { fn index_mut(&mut self, index: EriIndex) -> &mut Self::Output { - &mut self.storage[index.0] + &mut Arc::make_mut(&mut self.storage)[index.0] } } @@ -124,7 +130,7 @@ impl Index<(usize, usize, usize, usize)> for CompactEri { impl IndexMut<(usize, usize, usize, usize)> for CompactEri { fn index_mut(&mut self, index: (usize, usize, usize, usize)) -> &mut Self::Output { let (mu, nu, lambda, sigma) = index; - &mut self.storage[EriIndex::new(mu, nu, lambda, sigma).0] + &mut self[EriIndex::new(mu, nu, lambda, sigma)] } } @@ -135,6 +141,22 @@ mod tests { use ndarray::Array4; use proptest::prelude::*; + #[test] + fn clones_share_values_and_detach_on_mutation() { + let mut original = CompactEri::Zeroed(2); + original[(0, 0, 0, 0)] = 1.0; + let mut cloned = original.clone(); + assert_eq!( + original.ordered_values().as_ptr(), + cloned.ordered_values().as_ptr() + ); + cloned[EriIndex::new(0, 0, 0, 0)] = 2.0; + assert_eq!(original[(0, 0, 0, 0)], 1.0); + assert_eq!(cloned[(0, 0, 0, 0)], 2.0); + original[(0, 0, 0, 0)] = 3.0; + assert_eq!(cloned[(0, 0, 0, 0)], 2.0); + } + #[test] fn test_compact_eri_allocates_unique_quartets() { let basis_functions = 4; diff --git a/crates/rustiq-core/src/hf/scf_setup.rs b/crates/rustiq-core/src/hf/scf_setup.rs index c356d6ef..5a72c860 100644 --- a/crates/rustiq-core/src/hf/scf_setup.rs +++ b/crates/rustiq-core/src/hf/scf_setup.rs @@ -10,7 +10,7 @@ use crate::{ validated::PositiveFiniteF64, IntegralConfig, OrthogonalizationConfig, DEFAULT_ERI_SCHWARZ_THRESHOLD, }, - eri::EriError, + eri::{CompactEri, EriError}, molecules::molecule::Molecule, persistence::EriCache, }; @@ -70,6 +70,7 @@ pub(crate) fn prepare_scf_setup( .expect("default ERI Schwarz threshold is valid"), ), None, + None, progress, |_| {}, ) @@ -83,6 +84,7 @@ pub(crate) fn prepare_configured_scf_setup( orthogonalization: &OrthogonalizationConfig, integrals: &IntegralConfig, eri_cache: Option<&EriCache>, + supplied_eri: Option, progress: impl FnMut(ScfSetupStep), cache_event: impl FnMut(EriCacheEvent), ) -> Result { @@ -96,6 +98,7 @@ pub(crate) fn prepare_configured_scf_setup( .into_inner(), integrals.schwarz_threshold.value, eri_cache, + supplied_eri, progress, cache_event, ) @@ -109,6 +112,7 @@ fn prepare_scf_setup_with_thresholds( linear_dependency_threshold: f64, eri_schwarz_threshold: Option, eri_cache: Option<&EriCache>, + supplied_eri: Option, mut progress: impl FnMut(ScfSetupStep), mut cache_event: impl FnMut(EriCacheEvent), ) -> Result { @@ -139,7 +143,22 @@ fn prepare_scf_setup_with_thresholds( progress(ScfSetupStep::ElectronRepulsionIntegrals); let step_start = Instant::now(); - let (electron_repulsion, eri_cache_event) = builder.electron_repulsion()?; + let (electron_repulsion, eri_cache_event) = match supplied_eri { + Some(eri) => { + let basis_functions = basis.nbasis(); + let expected = CompactEri::storage_len(basis_functions); + if eri.len() != expected { + return Err(EriError::InvalidValueCount { + basis_functions, + expected, + actual: eri.len(), + } + .into()); + } + (eri, None) + } + None => builder.electron_repulsion()?, + }; if let Some(event) = eri_cache_event { cache_event(event); } diff --git a/crates/rustiq-core/src/persistence/calculation.rs b/crates/rustiq-core/src/persistence/calculation.rs new file mode 100644 index 00000000..2645982f --- /dev/null +++ b/crates/rustiq-core/src/persistence/calculation.rs @@ -0,0 +1,952 @@ +//! Explicit V1 wire records; domain structs are never serialized as the format. +use serde::{Deserialize, Serialize}; + +use super::{ + identity::CanonicalBytes, PortableError, ScientificIdentity, AO_ERI_COMPUTATION_VERSION, +}; +use crate::{ + calculation::PreparedCalculation, + config::{ + random_config::{DistributionConfig, RandomConfig}, + DensityGuessConfig, HfMethod, ResolvedHfMethod, + }, + molecules::molecule::Molecule, +}; + +pub(crate) const CALCULATION_PATH: &str = "calculation.json"; +pub(crate) const MAX_CALCULATION_BYTES: u64 = 64 * 1024 * 1024; + +/// Read-only, resolved scientific context. Coordinates and centers are in Bohr. +#[derive(Clone, Debug)] +pub struct CalculationContext(pub(crate) Snapshot, DomainCounts); + +// Checked once at the wire boundary; public accessors retain their domain types. +#[derive(Clone, Debug)] +struct DomainCounts { + max_iterations: usize, + diis_size: Option, + frozen_orbitals: Option, +} +impl DomainCounts { + fn from_snapshot(snapshot: &Snapshot) -> Result { + let invalid = + |_| PortableError::InvalidCalculation("integer exceeds host usize range".into()); + Ok(Self { + max_iterations: usize::try_from(snapshot.hf.max_iterations).map_err(invalid)?, + diis_size: snapshot + .hf + .diis_size + .map(usize::try_from) + .transpose() + .map_err(invalid)?, + frozen_orbitals: snapshot + .mp2 + .as_ref() + .map(|v| usize::try_from(v.frozen_orbitals)) + .transpose() + .map_err(invalid)?, + }) + } +} + +impl CalculationContext { + pub(crate) fn from_snapshot(snapshot: Snapshot) -> Result { + let counts = DomainCounts::from_snapshot(&snapshot)?; + Ok(Self(snapshot, counts)) + } + pub fn atoms(&self) -> impl ExactSizeIterator + '_ { + self.0 + .atoms + .iter() + .map(|atom| (atom.atomic_number, atom.position)) + } + pub fn charge(&self) -> i32 { + self.0.charge + } + pub fn multiplicity(&self) -> u8 { + self.0.multiplicity + } + pub fn basis(&self) -> impl ExactSizeIterator> { + self.0.basis.iter().map(ResolvedAo) + } + pub fn hf_method(&self) -> ResolvedHfMethod { + match self.0.hf.method { + Method::Rhf => ResolvedHfMethod::Rhf, + Method::Uhf => ResolvedHfMethod::Uhf, + } + } + pub fn max_iterations(&self) -> usize { + self.1.max_iterations + } + pub fn convergence_threshold(&self) -> f64 { + self.0.hf.convergence_threshold + } + pub fn linear_dependency_threshold(&self) -> f64 { + self.0.hf.linear_dependency_threshold + } + pub fn eri_schwarz_threshold(&self) -> Option { + self.0.hf.eri_schwarz_threshold + } + pub fn diis_size(&self) -> Option { + self.1.diis_size + } + /// `None` means HF only; `Some(n)` requests MP2 with n frozen orbitals. + pub fn mp2_frozen_orbitals(&self) -> Option { + self.1.frozen_orbitals + } + /// The initial density strategy; random settings do not promise an exact restart. + pub fn density_guess(&self) -> crate::config::DensityGuessConfig { + self.0.hf.guess.to_config() + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct Snapshot { + format: String, + version: u32, + units: String, + atoms: Vec, + charge: i32, + multiplicity: u8, + basis: Vec, + hf: Hf, + mp2: Option, + ao_eri_computation_version: u32, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Atom { + atomic_number: u32, + position: [f64; 3], +} + +/// One effective AO, in the same order as the integral engine. +#[derive(Clone, Copy, Debug)] +pub struct ResolvedAo<'a>(&'a Ao); +impl<'a> ResolvedAo<'a> { + pub fn center(&self) -> [f64; 3] { + self.0.center + } + pub fn components(&self) -> impl ExactSizeIterator> { + self.0.components.iter().map(ResolvedComponent) + } +} +/// A Cartesian component with effective normalized primitive coefficients. +#[derive(Clone, Copy, Debug)] +pub struct ResolvedComponent<'a>(&'a Component); +impl ResolvedComponent<'_> { + pub fn angular_momentum(&self) -> [u8; 3] { + self.0.angular_momentum + } + /// Returns (exponent, effective normalized coefficient) pairs. + pub fn primitives(&self) -> impl ExactSizeIterator + '_ { + self.0 + .primitives + .iter() + .map(|p| (p.exponent, p.coefficient)) + } +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Ao { + center: [f64; 3], + components: Vec, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Component { + angular_momentum: [u8; 3], + primitives: Vec, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Primitive { + exponent: f64, + coefficient: f64, +} +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +enum Method { + Rhf, + Uhf, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Hf { + method: Method, + max_iterations: u64, + convergence_threshold: f64, + linear_dependency_threshold: f64, + eri_schwarz_threshold: Option, + diis_size: Option, + guess: ResolvedGuess, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Mp2 { + frozen_orbitals: u64, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub(super) enum Guess { + CoreHamiltonian { perturbation: Option }, + OneElectron { perturbation: Option }, + Random { random: Random }, + Zero, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Random { + seed: Option, + distribution: Distribution, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum ResolvedGuess { + CoreHamiltonian { + perturbation: Option, + }, + OneElectron { + perturbation: Option, + }, + Random { + random: ResolvedRandom, + }, + Zero, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct ResolvedRandom { + seed: u64, + distribution: Distribution, +} +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum Distribution { + Uniform { min: f64, max: f64 }, + Normal { mean: f64, std_dev: f64 }, +} + +impl Snapshot { + pub(crate) fn from_calculation( + calculation: &PreparedCalculation, + ) -> Result { + let invalid = |_| PortableError::InvalidCalculation("integer exceeds V1 u64 range".into()); + let molecule = calculation.get_molecule(); + let basis = calculation.get_basis(); + let hf = calculation.hf_config(); + let method = calculation.hf_method(); + let mp2 = calculation.mp2_config(); + let snapshot = Self { + format: "rustiq-calculation".into(), + version: 1, + units: "bohr".into(), + atoms: molecule + .atoms + .iter() + .map(|atom| Atom { + atomic_number: atom.element.atomic_number, + position: atom.position.into(), + }) + .collect(), + charge: molecule.charge(), + multiplicity: molecule.multiplicity().get(), + basis: basis + .shell_ids + .iter() + .zip(&basis.normalized_components) + .map(|(shell, components)| Ao { + center: basis.shells[*shell].origin.into(), + components: components + .iter() + .map(|component| Component { + angular_momentum: component.angular_momentum.into(), + primitives: component + .primitives + .iter() + .map(|p| Primitive { + exponent: p.exponent, + coefficient: p.coefficient, + }) + .collect(), + }) + .collect(), + }) + .collect(), + hf: Hf { + method: match method { + ResolvedHfMethod::Rhf => Method::Rhf, + ResolvedHfMethod::Uhf => Method::Uhf, + }, + max_iterations: u64::try_from(hf.max_iterations.get()).map_err(invalid)?, + convergence_threshold: hf.convergence_threshold.into_inner(), + linear_dependency_threshold: hf + .orthogonalization + .linear_dependency_threshold + .value + .into_inner(), + eri_schwarz_threshold: calculation + .integral_config() + .schwarz_threshold + .value + .map(|value| value.into_inner()), + diis_size: hf + .diis + .enabled + .then(|| u64::try_from(hf.diis.max_history.value.into_inner())) + .transpose() + .map_err(invalid)?, + guess: ResolvedGuess::from_config(hf.guess.value)?, + }, + mp2: mp2 + .map(|mp2| { + Ok::<_, PortableError>(Mp2 { + frozen_orbitals: u64::try_from(mp2.frozen_orbitals.value) + .map_err(invalid)?, + }) + }) + .transpose()?, + ao_eri_computation_version: AO_ERI_COMPUTATION_VERSION, + }; + snapshot.validate()?; + Ok(snapshot) + } + + pub(crate) fn validate(&self) -> Result<(), PortableError> { + if self.format != "rustiq-calculation" || self.version != 1 || self.units != "bohr" { + return Err(PortableError::UnsupportedVersion); + } + let invalid = |message: &str| PortableError::InvalidCalculation(message.into()); + if self.atoms.is_empty() + || self.basis.is_empty() + || self.multiplicity == 0 + || self.ao_eri_computation_version == 0 + { + return Err(invalid( + "empty molecule/basis or invalid version/multiplicity", + )); + } + let mut nuclear_charge = 0_i64; + for atom in &self.atoms { + if !(1..=118).contains(&atom.atomic_number) + || !atom.position.iter().all(|v| v.is_finite()) + { + return Err(invalid("invalid atom")); + } + nuclear_charge += i64::from(atom.atomic_number); + } + let electrons = nuclear_charge - i64::from(self.charge); + let unpaired = i64::from(self.multiplicity) - 1; + if electrons <= 0 + || electrons < unpaired + || (electrons - unpaired) % 2 != 0 + || (matches!(self.hf.method, Method::Rhf) && (unpaired != 0 || electrons % 2 != 0)) + { + return Err(invalid( + "inconsistent electron count, multiplicity or HF method", + )); + } + for ao in &self.basis { + if !ao.center.iter().all(|v| v.is_finite()) || ao.components.is_empty() { + return Err(invalid("invalid AO center or empty components")); + } + for component in &ao.components { + if component.primitives.is_empty() + || component + .primitives + .iter() + .any(|p| !positive(p.exponent) || !p.coefficient.is_finite()) + { + return Err(invalid("invalid AO primitives")); + } + } + } + if crate::eri::CompactEri::checked_storage_len(self.basis.len()) + .and_then(|n| n.checked_mul(8)) + .is_none() + { + return Err(invalid("basis dimensions overflow")); + } + if self.hf.max_iterations == 0 + || !positive(self.hf.convergence_threshold) + || !self.hf.linear_dependency_threshold.is_finite() + || self.hf.linear_dependency_threshold < 0.0 + || self.hf.eri_schwarz_threshold.is_some_and(|v| !positive(v)) + || self.hf.diis_size.is_some_and(|v| v < 2) + { + return Err(invalid("invalid HF settings")); + } + DomainCounts::from_snapshot(self)?; + self.hf.guess.validate()?; + Ok(()) + } + + pub(crate) fn validate_request( + &self, + request: &crate::calculation::CalculationRequest, + ) -> Result<(), PortableError> { + let invalid = + || PortableError::InvalidRequest("request disagrees with resolved calculation".into()); + let molecule = request + .molecule() + .build(request.geometry().clone()) + .map_err(|_| invalid())?; + let method = + resolve_hf_method_v1(request.hf().method.value, &molecule).ok_or_else(invalid)?; + let hf = request.hf(); + if molecule.charge() != self.charge + || molecule.multiplicity().get() != self.multiplicity + || molecule.atoms.len() != self.atoms.len() + || molecule.atoms.iter().zip(&self.atoms).any(|(a, b)| { + a.element.atomic_number != b.atomic_number + || !<[f64; 3]>::from(a.position) + .into_iter() + .zip(b.position) + .all(|(requested, resolved)| { + coordinate_matches_v1(requested, resolved, request.molecule().units) + }) + }) + || !matches!( + (method, self.hf.method), + (ResolvedHfMethod::Rhf, Method::Rhf) | (ResolvedHfMethod::Uhf, Method::Uhf) + ) + || u64::try_from(hf.max_iterations.get()).map_err(|_| invalid())? + != self.hf.max_iterations + || hf.convergence_threshold.into_inner() != self.hf.convergence_threshold + || hf + .orthogonalization + .linear_dependency_threshold + .value + .into_inner() + != self.hf.linear_dependency_threshold + || request + .integrals() + .schwarz_threshold + .value + .map(|v| v.into_inner()) + != self.hf.eri_schwarz_threshold + || hf + .diis + .enabled + .then(|| u64::try_from(hf.diis.max_history.value.into_inner())) + .transpose() + .map_err(|_| invalid())? + != self.hf.diis_size + || request + .mp2() + .map(|v| u64::try_from(v.frozen_orbitals.value)) + .transpose() + .map_err(|_| invalid())? + != self.mp2.as_ref().map(|v| v.frozen_orbitals) + || !self.hf.guess.is_resolution_of(&hf.guess.value.into()) + { + return Err(invalid()); + } + Ok(()) + } + + pub(crate) fn basis_functions(&self) -> usize { + self.basis.len() + } + pub(crate) fn computation_version(&self) -> u32 { + self.ao_eri_computation_version + } + pub(crate) fn identity(&self) -> ScientificIdentity { + let mut bytes = CanonicalBytes::ao_eri(self.ao_eri_computation_version); + bytes.len(self.atoms.len()); + for atom in &self.atoms { + bytes.u32(atom.atomic_number); + for coordinate in atom.position { + bytes.f64(coordinate); + } + } + bytes.len(self.basis.len()); + for ao in &self.basis { + for coordinate in ao.center { + bytes.f64(coordinate); + } + bytes.len(ao.components.len()); + for component in &ao.components { + for axis in component.angular_momentum { + bytes.u8(axis); + } + bytes.len(component.primitives.len()); + for primitive in &component.primitives { + bytes.f64(primitive.exponent); + bytes.f64(primitive.coefficient); + } + } + } + bytes.finish_ao_eri(self.hf.eri_schwarz_threshold) + } +} +/// Resolves the HF request according to the frozen V1 persistence contract. +/// +/// V1 predates any future method that may become eligible for `Auto`, so archive +/// validation must not inherit changes to the current domain-level resolver. +pub(super) fn resolve_hf_method_v1( + method: HfMethod, + molecule: &Molecule, +) -> Option { + let closed_shell_singlet = + molecule.multiplicity().get() == 1 && molecule.total_electrons().is_multiple_of(2); + match method { + HfMethod::Rhf if closed_shell_singlet => Some(ResolvedHfMethod::Rhf), + HfMethod::Rhf => None, + HfMethod::Uhf => Some(ResolvedHfMethod::Uhf), + HfMethod::Auto if closed_shell_singlet => Some(ResolvedHfMethod::Rhf), + HfMethod::Auto => Some(ResolvedHfMethod::Uhf), + } +} + +// V1 pins this decimal value (CODATA 2018 Bohr radius in Angstrom), rather +// than inheriting a future physical_constants release. Parse as binary64 and divide. +const V1_BOHR_IN_ANGSTROM: f64 = 0.529_177_210_903; + +fn coordinate_matches_v1( + requested: f64, + resolved: f64, + units: crate::molecules::units::Units, +) -> bool { + use crate::molecules::units::Units; + let expected = match units { + Units::Bohr => requested, + Units::Angstrom => requested / V1_BOHR_IN_ANGSTROM, + }; + if !expected.is_finite() || !resolved.is_finite() { + return false; + } + if units == Units::Bohr { + return expected == resolved; + } + // Four adjacent binary64 values cover division versus reciprocal multiplication + // and final rounding of independently produced conversions. There is no absolute + // chemistry-level floor: even near zero only four subnormal steps are allowed. + fn ordered(value: f64) -> u64 { + let bits = if value == 0.0 { 0 } else { value.to_bits() }; + if bits >> 63 == 0 { + bits | (1 << 63) + } else { + (!bits) + 1 + } + } + ordered(expected).abs_diff(ordered(resolved)) <= 4 +} + +fn positive(value: f64) -> bool { + value.is_finite() && value > 0.0 +} +impl From for Random { + fn from(config: RandomConfig) -> Self { + Self { + seed: config.seed, + distribution: match config.distribution { + DistributionConfig::Uniform { config } => Distribution::Uniform { + min: config.min, + max: config.max, + }, + DistributionConfig::Normal { config } => Distribution::Normal { + mean: config.mean, + std_dev: config.std_dev.into_inner(), + }, + }, + } + } +} +impl Random { + fn to_config(&self) -> RandomConfig { + use crate::config::random_config::distribution_config::{ + NormalDistributionConfig, UniformDistributionConfig, + }; + RandomConfig { + seed: self.seed, + distribution: match self.distribution { + Distribution::Uniform { min, max } => DistributionConfig::Uniform { + config: UniformDistributionConfig { min, max }, + }, + Distribution::Normal { mean, std_dev } => DistributionConfig::Normal { + config: NormalDistributionConfig { + mean, + std_dev: crate::config::validated::PositiveFiniteF64::try_new(std_dev) + .expect("validated snapshot"), + }, + }, + }, + } + } +} +impl ResolvedRandom { + fn from_config(config: RandomConfig) -> Result { + let requested = Random::from(config); + let seed = requested.seed.ok_or_else(|| { + PortableError::InvalidCalculation( + "resolved random configuration is missing its seed".into(), + ) + })?; + Ok(Self { + seed, + distribution: requested.distribution, + }) + } + + fn to_config(&self) -> RandomConfig { + Random { + seed: Some(self.seed), + distribution: self.distribution.clone(), + } + .to_config() + } +} + +impl ResolvedGuess { + fn from_config(config: DensityGuessConfig) -> Result { + Ok(match config { + DensityGuessConfig::CoreHamiltonian { perturbation } => Self::CoreHamiltonian { + perturbation: perturbation + .map(|value| ResolvedRandom::from_config(value.random)) + .transpose()?, + }, + DensityGuessConfig::OneElectron { perturbation } => Self::OneElectron { + perturbation: perturbation + .map(|value| ResolvedRandom::from_config(value.random)) + .transpose()?, + }, + DensityGuessConfig::Random { config } => Self::Random { + random: ResolvedRandom::from_config(config.random)?, + }, + DensityGuessConfig::Zero => Self::Zero, + }) + } + + fn is_resolution_of(&self, requested: &Guess) -> bool { + let random_matches = |resolved: &ResolvedRandom, requested: &Random| { + resolved.distribution == requested.distribution + && requested.seed.is_none_or(|seed| resolved.seed == seed) + }; + match (self, requested) { + ( + Self::CoreHamiltonian { perturbation: a }, + Guess::CoreHamiltonian { perturbation: b }, + ) + | (Self::OneElectron { perturbation: a }, Guess::OneElectron { perturbation: b }) => { + match (a, b) { + (None, None) => true, + (Some(a), Some(b)) => random_matches(a, b), + _ => false, + } + } + (Self::Random { random: a }, Guess::Random { random: b }) => random_matches(a, b), + (Self::Zero, Guess::Zero) => true, + _ => false, + } + } + + fn validate(&self) -> Result<(), PortableError> { + let random = match self { + Self::CoreHamiltonian { perturbation } | Self::OneElectron { perturbation } => { + perturbation.as_ref() + } + Self::Random { random } => Some(random), + Self::Zero => None, + }; + if let Some(random) = random { + let valid = match random.distribution { + Distribution::Uniform { min, max } => { + min.is_finite() && max.is_finite() && min < max && (max - min).is_finite() + } + Distribution::Normal { mean, std_dev } => mean.is_finite() && positive(std_dev), + }; + if !valid { + return Err(PortableError::InvalidCalculation( + "invalid random distribution".into(), + )); + } + } + Ok(()) + } + + fn to_config(&self) -> DensityGuessConfig { + use crate::config::{GuessPerturbationConfig, RandomGuessConfig}; + match self { + Self::CoreHamiltonian { perturbation } => DensityGuessConfig::CoreHamiltonian { + perturbation: perturbation.as_ref().map(|value| GuessPerturbationConfig { + random: value.to_config(), + }), + }, + Self::OneElectron { perturbation } => DensityGuessConfig::OneElectron { + perturbation: perturbation.as_ref().map(|value| GuessPerturbationConfig { + random: value.to_config(), + }), + }, + Self::Random { random } => DensityGuessConfig::Random { + config: RandomGuessConfig { + random: random.to_config(), + }, + }, + Self::Zero => DensityGuessConfig::Zero, + } + } +} + +impl Guess { + pub(super) fn validate(&self) -> Result<(), PortableError> { + let random = match self { + Guess::CoreHamiltonian { perturbation } | Guess::OneElectron { perturbation } => { + perturbation.as_ref() + } + Guess::Random { random } => Some(random), + Guess::Zero => None, + }; + if let Some(random) = random { + let valid = match random.distribution { + Distribution::Uniform { min, max } => { + min.is_finite() && max.is_finite() && min < max && (max - min).is_finite() + } + Distribution::Normal { mean, std_dev } => mean.is_finite() && positive(std_dev), + }; + if !valid { + return Err(PortableError::InvalidCalculation( + "invalid random distribution".into(), + )); + } + } + Ok(()) + } + + pub(super) fn to_config(&self) -> DensityGuessConfig { + use crate::config::{GuessPerturbationConfig, RandomGuessConfig}; + match self { + Self::CoreHamiltonian { perturbation } => DensityGuessConfig::CoreHamiltonian { + perturbation: perturbation.as_ref().map(|p| GuessPerturbationConfig { + random: p.to_config(), + }), + }, + Self::OneElectron { perturbation } => DensityGuessConfig::OneElectron { + perturbation: perturbation.as_ref().map(|p| GuessPerturbationConfig { + random: p.to_config(), + }), + }, + Self::Random { random } => DensityGuessConfig::Random { + config: RandomGuessConfig { + random: random.to_config(), + }, + }, + Self::Zero => DensityGuessConfig::Zero, + } + } +} + +impl From for Guess { + fn from(config: DensityGuessConfig) -> Self { + match config { + DensityGuessConfig::CoreHamiltonian { perturbation } => Guess::CoreHamiltonian { + perturbation: perturbation.map(|p| p.random.into()), + }, + DensityGuessConfig::OneElectron { perturbation } => Guess::OneElectron { + perturbation: perturbation.map(|p| p.random.into()), + }, + DensityGuessConfig::Random { config } => Guess::Random { + random: config.random.into(), + }, + DensityGuessConfig::Zero => Guess::Zero, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + + proptest! { + #[test] + fn json_round_trip_preserves_binary64_scientific_identity( + coordinate in any::().prop_filter("finite coordinate", |v| v.is_finite()), + exponent in any::().prop_filter("positive finite exponent", |v| v.is_finite() && *v > 0.0), + coefficient in any::().prop_filter("finite coefficient", |v| v.is_finite()), + ) { + let mut snapshot: Snapshot = serde_json::from_str(include_str!("../../tests/data/persistence/calculation-h2-v1.json")).unwrap(); + snapshot.atoms[0].position[0] = coordinate; + snapshot.basis[0].center[0] = coordinate; + snapshot.basis[0].components[0].primitives[0].exponent = exponent; + snapshot.basis[0].components[0].primitives[0].coefficient = coefficient; + snapshot.validate().unwrap(); + let bytes = serde_json::to_vec(&snapshot).unwrap(); + let restored: Snapshot = serde_json::from_slice(&bytes).unwrap(); + restored.validate().unwrap(); + prop_assert_eq!(snapshot.identity(), restored.identity()); + } + } +} + +#[cfg(test)] +mod v1_contract_tests { + use super::*; + use crate::molecules::{atom::Atom, geometry::Geometry, units::Units}; + use nalgebra::point; + use serde_json::{json, Value}; + use std::num::NonZeroU8; + + fn molecule(symbols: &[&str], multiplicity: u8) -> Molecule { + let elements = periodic_table::periodic_table(); + let atoms = symbols + .iter() + .enumerate() + .map(|(index, symbol)| { + let element = elements + .iter() + .find(|element| element.symbol == *symbol) + .unwrap(); + Atom::new(element, point![0.0, 0.0, index as f64]) + }) + .collect(); + Molecule::try_new( + Geometry::new("v1-method-resolution".into(), atoms), + Units::Bohr, + 0, + NonZeroU8::new(multiplicity).unwrap(), + ) + .unwrap() + } + + #[test] + fn hf_method_resolution_is_frozen_for_v1() { + let closed_shell = molecule(&["H", "H"], 1); + let open_shell = molecule(&["H"], 2); + + assert_eq!( + resolve_hf_method_v1(HfMethod::Auto, &closed_shell), + Some(ResolvedHfMethod::Rhf) + ); + assert_eq!( + resolve_hf_method_v1(HfMethod::Auto, &open_shell), + Some(ResolvedHfMethod::Uhf) + ); + assert_eq!( + resolve_hf_method_v1(HfMethod::Rhf, &closed_shell), + Some(ResolvedHfMethod::Rhf) + ); + assert_eq!(resolve_hf_method_v1(HfMethod::Rhf, &open_shell), None); + assert_eq!( + resolve_hf_method_v1(HfMethod::Uhf, &closed_shell), + Some(ResolvedHfMethod::Uhf) + ); + assert_eq!( + resolve_hf_method_v1(HfMethod::Uhf, &open_shell), + Some(ResolvedHfMethod::Uhf) + ); + } + + #[test] + fn resolved_random_seed_is_required_by_v1() { + let schema: Value = serde_json::from_str(include_str!( + "../../../../schemas/calculation-snapshot-v1.schema.json" + )) + .unwrap(); + assert!(schema["$defs"]["random"]["required"] + .as_array() + .unwrap() + .contains(&json!("seed"))); + assert_eq!( + schema["$defs"]["random"]["properties"]["seed"]["type"], + json!("integer") + ); + + let mut value: Value = serde_json::from_str(include_str!( + "../../tests/data/persistence/calculation-h2-v1.json" + )) + .unwrap(); + value["hf"]["guess"] = json!({ + "kind": "random", + "random": { + "seed": 42, + "distribution": {"kind": "uniform", "min": -1.0, "max": 1.0} + } + }); + let snapshot: Snapshot = serde_json::from_value(value.clone()).unwrap(); + snapshot.validate().unwrap(); + + value["hf"]["guess"]["random"] + .as_object_mut() + .unwrap() + .remove("seed"); + assert!(serde_json::from_value::(value.clone()).is_err()); + + value["hf"]["guess"]["random"]["seed"] = Value::Null; + assert!(serde_json::from_value::(value).is_err()); + } + + #[test] + fn conversion_comparison_is_bounded_in_binary64_steps() { + for requested in [0.74, -0.74, 1e-200, -1e-200, 1e200] { + let expected = requested / V1_BOHR_IN_ANGSTROM; + let four = f64::from_bits(expected.to_bits() + 4); + let five = f64::from_bits(expected.to_bits() + 5); + assert!(coordinate_matches_v1(requested, four, Units::Angstrom)); + assert!(!coordinate_matches_v1(requested, five, Units::Angstrom)); + assert!(!coordinate_matches_v1(requested, four, Units::Bohr)); + } + assert!(coordinate_matches_v1(0.0, -0.0, Units::Angstrom)); + assert!(coordinate_matches_v1( + 0.0, + f64::from_bits(4), + Units::Angstrom + )); + assert!(!coordinate_matches_v1( + 0.0, + f64::from_bits(5), + Units::Angstrom + )); + assert!(!coordinate_matches_v1(0.0, 1e-15, Units::Angstrom)); + assert!(!coordinate_matches_v1(f64::MAX, f64::MAX, Units::Angstrom)); + assert!(!coordinate_matches_v1(0.74, f64::NAN, Units::Angstrom)); + assert!(!coordinate_matches_v1(0.74, f64::INFINITY, Units::Angstrom)); + } + + #[test] + fn resolved_counts_use_u64_wire_and_checked_host_values() { + let schema: Value = serde_json::from_str(include_str!( + "../../../../schemas/calculation-snapshot-v1.schema.json" + )) + .unwrap(); + for (definition, field) in [ + ("hf", "max_iterations"), + ("hf", "diis_size"), + ("mp2", "frozen_orbitals"), + ] { + let property = &schema["$defs"][definition]["properties"][field]; + let integer = if field == "diis_size" { + &property["anyOf"][0] + } else { + property + }; + assert_eq!(integer["maximum"], json!(u64::MAX)); + let mut value: Value = serde_json::from_str(include_str!( + "../../tests/data/persistence/calculation-h2-v1.json" + )) + .unwrap(); + value["mp2"] = json!({"frozen_orbitals":0}); + value[definition][field] = json!(u64::MAX); + let snapshot: Snapshot = serde_json::from_value(value.clone()).unwrap(); + assert_eq!( + snapshot.validate().is_ok(), + usize::try_from(u64::MAX).is_ok() + ); + assert_eq!( + CalculationContext::from_snapshot(snapshot.clone()).is_ok(), + usize::try_from(u64::MAX).is_ok() + ); + assert_eq!( + serde_json::to_value(snapshot).unwrap()[definition][field], + json!(u64::MAX) + ); + for invalid in ["-1", "18446744073709551616"] { + let encoded = serde_json::to_string(&value) + .unwrap() + .replace("18446744073709551615", invalid); + assert!(serde_json::from_str::(&encoded).is_err()); + } + } + } +} diff --git a/crates/rustiq-core/src/persistence/data.rs b/crates/rustiq-core/src/persistence/data.rs index 86dccc87..1283e495 100644 --- a/crates/rustiq-core/src/persistence/data.rs +++ b/crates/rustiq-core/src/persistence/data.rs @@ -6,3 +6,8 @@ pub use ao_eri_artifact::AoEriArtifact; pub use artifact::Artifact; pub use rustiq_data::RustiQData; pub(crate) use rustiq_data::AO_ERI_ARTIFACT; + +mod portable; + +mod bundle; +pub use bundle::RustiQBundle; diff --git a/crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs b/crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs index 3e93b560..7b418618 100644 --- a/crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs +++ b/crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs @@ -13,6 +13,19 @@ impl private::Sealed for AoEriArtifact {} impl Artifact for AoEriArtifact { type Value = CompactEri; + const NAME: &'static str = AO_ERI_ARTIFACT; + + fn is_present(data: &RustiQData) -> bool { + data.ao_eri.is_some() || data.manifest.artifacts.contains_key(AO_ERI_ARTIFACT) + } + + fn is_compatible( + data: &RustiQData, + calculation: &crate::calculation::PreparedCalculation, + ) -> bool { + data.matches_eri_identity(calculation) + } + fn get(data: &mut RustiQData) -> Result, ArtifactError> { if data.ao_eri.is_some() { return Ok(data.ao_eri.as_ref()); diff --git a/crates/rustiq-core/src/persistence/data/artifact.rs b/crates/rustiq-core/src/persistence/data/artifact.rs index dd1df18b..e24c1425 100644 --- a/crates/rustiq-core/src/persistence/data/artifact.rs +++ b/crates/rustiq-core/src/persistence/data/artifact.rs @@ -6,9 +6,25 @@ pub(crate) mod private { } /// A known scientific artifact. Only RustiQ's declared artifact markers implement this trait. +/// +/// Values for reusable heavy artifacts must be cheap to clone: use shared immutable backing +/// storage or another representation that avoids copying the payload. The resolver may clone +/// values while moving them between the source, resolved state, and execution. Implementing +/// `Clone` as a deep copy for a large artifact is incorrect for this architecture. pub trait Artifact: private::Sealed { type Value; + /// Stable logical artifact name, independent of its storage path. + const NAME: &'static str; + + #[doc(hidden)] + fn is_present(data: &RustiQData) -> bool; + #[doc(hidden)] + fn is_compatible( + data: &RustiQData, + calculation: &crate::calculation::PreparedCalculation, + ) -> bool; + #[doc(hidden)] fn get(data: &mut RustiQData) -> Result, ArtifactError>; #[doc(hidden)] diff --git a/crates/rustiq-core/src/persistence/data/bundle.rs b/crates/rustiq-core/src/persistence/data/bundle.rs new file mode 100644 index 00000000..b1e2e6ac --- /dev/null +++ b/crates/rustiq-core/src/persistence/data/bundle.rs @@ -0,0 +1,334 @@ +//! A non-empty portable collection; scientific APIs still operate on one calculation. +use std::{ + collections::HashSet, + path::Path, + sync::{Arc, Mutex}, +}; + +use relative_path::{RelativePath, RelativePathBuf}; + +use super::{portable, rustiq_data::MAX_MANIFEST_BYTES, RustiQData}; +use crate::persistence::{ + manifest::{PortableCalculation, PortableManifest, SourceManifest}, + ArtifactAttributes, Manifest, ManifestKind, PortableError, Producer, SourceProvenance, Storage, + COMPACT_ERI_REPRESENTATION, FORMAT_NAME, FORMAT_VERSION, MANIFEST_PATH, +}; + +/// A portable snapshot containing one or more independently reusable calculations. +/// Sources are shared opaque provenance. ZIP details remain private. +#[derive(Debug)] +pub struct RustiQBundle { + calculations: Vec, + sources: Vec, +} + +impl RustiQBundle { + /// Collects portable calculation data, rejecting empty or cache-only collections. + /// Exact duplicate sources are stored once at the bundle level. + pub fn new(mut calculations: Vec) -> Result { + if calculations.is_empty() { + return Err(invalid("portable bundles require at least one calculation")); + } + let mut sources = Vec::new(); + for data in &mut calculations { + if data.request.is_none() || data.context.is_none() { + return Err(invalid( + "portable entries require request and resolved snapshots", + )); + } + for source in &data.sources { + if !sources.contains(source) { + sources.push(source.clone()); + } + } + } + validate_sources(&sources)?; + for data in &mut calculations { + data.sources.clear(); + } + Ok(Self { + calculations, + sources, + }) + } + + /// Opens read-only, validating all snapshots and references without loading arrays. + pub fn open(path: impl AsRef) -> Result { + let mut storage = Storage::open_zip(path.as_ref())?; + let manifest: PortableManifest = storage + .read_json(RelativePath::new(MANIFEST_PATH), MAX_MANIFEST_BYTES) + .map_err(crate::persistence::PersistenceReadError::from)?; + if manifest.format != FORMAT_NAME + || manifest.format_version != FORMAT_VERSION + || manifest.kind != ManifestKind::Portable + { + return Err(PortableError::UnsupportedVersion); + } + if manifest.calculations.is_empty() { + return Err(invalid("portable bundles require at least one calculation")); + } + let sources = portable::read_sources(&mut storage, &manifest.sources)?; + let storage = Arc::new(Mutex::new(storage)); + let mut ids = HashSet::new(); + let mut calculations = Vec::new(); + for entry in manifest.calculations { + let digits = entry.id.strip_prefix("calculation-").unwrap_or_default(); + if digits.is_empty() + || !digits.bytes().all(|b| b.is_ascii_digit()) + || !ids.insert(entry.id.clone()) + { + return Err(invalid("invalid or duplicate calculation identifier")); + } + let prefix = format!("calculations/{}", entry.id); + let mut request = entry.request; + let mut calculation = entry.calculation; + request.path = local_path(&prefix, &request.path)?; + calculation.path = local_path(&prefix, &calculation.path)?; + let mut artifacts = entry.artifacts; + for artifact in artifacts.values_mut() { + artifact.path = RelativePathBuf::from(local_path(&prefix, artifact.path.as_str())?); + } + let internal = Manifest { + format: manifest.format.clone(), + format_version: manifest.format_version, + kind: ManifestKind::Portable, + producer: manifest.producer.clone(), + scientific_identity: entry.scientific_identity, + artifacts, + request: Some(request), + calculation: Some(calculation), + sources: Vec::new(), + }; + let data = + RustiQData::read_manifest(Storage::scoped(storage.clone(), prefix), internal)?; + calculations.push(RustiQData::validate_portable(data)?); + } + Ok(Self { + calculations, + sources, + }) + } + + /// Entries retain manifest order; each exposes its own typed artifact access. + pub fn calculations(&self) -> &[RustiQData] { + &self.calculations + } + + /// Mutates scientific artifacts within entries, without allowing an empty collection. + pub fn calculations_mut(&mut self) -> &mut [RustiQData] { + &mut self.calculations + } + + pub fn sources(&self) -> impl ExactSizeIterator { + self.sources.iter() + } + + /// Adds shared exact source bytes; the original name never becomes a member path. + pub fn add_source( + &mut self, + original_name: impl Into, + bytes: impl Into>, + ) -> Result<(), PortableError> { + let source = SourceProvenance { + original_name: original_name.into(), + bytes: bytes.into(), + }; + let mut sources = self.sources.clone(); + sources.push(source); + validate_sources(&sources)?; + self.sources = sources; + Ok(()) + } + + /// Publishes a complete validated snapshot atomically, without overwriting a destination. + pub fn write(&mut self, path: impl AsRef) -> Result<(), PortableError> { + // Sources attached through individual entry APIs are lifted to shared provenance. + for data in &mut self.calculations { + for source in &data.sources { + if !self.sources.contains(source) { + self.sources.push(source.clone()); + } + } + } + validate_sources(&self.sources)?; + write_bundle(&mut self.calculations, &self.sources, path.as_ref()) + } + + pub(super) fn into_parts(self) -> (Vec, Vec) { + (self.calculations, self.sources) + } +} + +fn invalid(message: &str) -> PortableError { + PortableError::InvalidArchive(message.into()) +} + +fn local_path(prefix: &str, path: &str) -> Result { + crate::persistence::validate_storage_path(RelativePath::new(path))?; + path.strip_prefix(&format!("{prefix}/")) + .map(str::to_owned) + .ok_or_else(|| invalid("reference crosses calculation namespace")) +} + +fn validate_sources(sources: &[SourceProvenance]) -> Result<(), PortableError> { + use crate::persistence::provenance::{MAX_SOURCES, MAX_SOURCE_BYTES, MAX_SOURCE_NAME_BYTES}; + let total = sources.iter().try_fold(0_u64, |total, source| { + total.checked_add(u64::try_from(source.bytes.len()).ok()?) + }); + if sources.len() > MAX_SOURCES + || total.is_none_or(|total| total > MAX_SOURCE_BYTES) + || sources + .iter() + .any(|source| source.original_name.len() > MAX_SOURCE_NAME_BYTES) + { + return Err(invalid("source provenance exceeds supported limits")); + } + Ok(()) +} + +pub(super) fn write_bundle( + calculations: &mut [RustiQData], + sources: &[SourceProvenance], + path: &Path, +) -> Result<(), PortableError> { + write_bundle_with_eri(calculations, sources, path, None) +} + +pub(super) fn write_bundle_with_eri( + calculations: &mut [RustiQData], + sources: &[SourceProvenance], + path: &Path, + eri: Option<&crate::eri::CompactEri>, +) -> Result<(), PortableError> { + if calculations.is_empty() { + return Err(invalid("portable bundles require at least one calculation")); + } + if eri.is_some() && calculations.len() != 1 { + return Err(invalid( + "a borrowed AO ERI can only be written with a single calculation", + )); + } + validate_sources(sources)?; + match std::fs::symlink_metadata(path) { + Ok(_) => return Err(PortableError::AlreadyExists), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let temporary = tempfile::NamedTempFile::new_in(parent)?; + write_snapshot_inner( + calculations, + sources, + Storage::create_zip(temporary.reopen()?), + eri, + )?; + drop(RustiQBundle::open(temporary.path())?); + temporary.persist_noclobber(path).map_err(|error| { + if error.error.kind() == std::io::ErrorKind::AlreadyExists { + PortableError::AlreadyExists + } else { + PortableError::Io(error.error) + } + })?; + #[cfg(unix)] + std::fs::File::open(parent)?.sync_all()?; + Ok(()) +} + +pub(super) fn write_snapshot( + calculations: &mut [RustiQData], + sources: &[SourceProvenance], + destination: Storage, +) -> Result<(), PortableError> { + write_snapshot_inner(calculations, sources, destination, None) +} + +fn write_snapshot_inner( + calculations: &mut [RustiQData], + sources: &[SourceProvenance], + destination: Storage, + eri: Option<&crate::eri::CompactEri>, +) -> Result<(), PortableError> { + let storage = Arc::new(Mutex::new(destination)); + let mut manifest = PortableManifest { + format: FORMAT_NAME.into(), + format_version: FORMAT_VERSION, + kind: ManifestKind::Portable, + producer: Producer { + name: "RustiQ".into(), + version: env!("CARGO_PKG_VERSION").into(), + }, + sources: Vec::new(), + calculations: Vec::new(), + }; + for (index, source) in sources.iter().enumerate() { + let path = format!("sources/{index}"); + let metadata = storage + .lock() + .expect("storage mutex poisoned") + .write_artifact::(RelativePath::new(&path), |writer| { + writer.write_all(source.bytes())?; + Ok(()) + })?; + manifest.sources.push(SourceManifest { + original_name: source.original_name.clone(), + path, + version: 1, + size: metadata.size, + digest: metadata.digest, + }); + } + for (index, data) in calculations.iter_mut().enumerate() { + if data.context.is_none() || data.request.is_none() { + return Err(invalid("missing calculation snapshots")); + } + if eri.is_none() && data.ao_eri.is_none() && data.manifest.artifacts.get(super::AO_ERI_ARTIFACT).is_some_and(|artifact| + artifact.representation == COMPACT_ERI_REPRESENTATION && matches!(&artifact.attributes, + ArtifactAttributes::AoEri(attributes) if attributes.computation_version == crate::persistence::AO_ERI_COMPUTATION_VERSION)) { + data.validate_unloaded_eri()?; + } + let id = format!("calculation-{index}"); + let prefix = format!("calculations/{id}"); + let saved_sources = std::mem::take(&mut data.sources); + let destination = Storage::scoped(storage.clone(), prefix.clone()); + let result = match eri { + Some(eri) => data.write_entry_with_eri(destination, eri), + None => data.write_entry(destination), + }; + data.sources = saved_sources; + let entry = result?; + let mut request = entry + .request + .ok_or_else(|| invalid("missing request snapshot"))?; + let mut calculation = entry + .calculation + .ok_or_else(|| invalid("missing resolved snapshot"))?; + request.path = format!("{prefix}/{}", request.path); + calculation.path = format!("{prefix}/{}", calculation.path); + let mut artifacts = entry.artifacts; + for artifact in artifacts.values_mut() { + artifact.path = RelativePathBuf::from(format!("{prefix}/{}", artifact.path)); + } + manifest.calculations.push(PortableCalculation { + id, + scientific_identity: entry.scientific_identity, + request, + calculation, + artifacts, + }); + } + storage + .lock() + .expect("storage mutex poisoned") + .write_json(RelativePath::new(MANIFEST_PATH), &manifest) + .map_err(crate::persistence::PersistenceWriteError::from)?; + Arc::try_unwrap(storage) + .expect("all entry writers released") + .into_inner() + .expect("storage mutex poisoned") + .finish()?; + Ok(()) +} diff --git a/crates/rustiq-core/src/persistence/data/portable.rs b/crates/rustiq-core/src/persistence/data/portable.rs new file mode 100644 index 00000000..e0e43caf --- /dev/null +++ b/crates/rustiq-core/src/persistence/data/portable.rs @@ -0,0 +1,406 @@ +use super::super::{ + calculation::{Snapshot, CALCULATION_PATH, MAX_CALCULATION_BYTES}, + provenance::{MAX_SOURCES, MAX_SOURCE_BYTES, MAX_SOURCE_NAME_BYTES}, + request::{RequestSnapshot, MAX_REQUEST_BYTES, REQUEST_PATH}, + ArtifactAttributes, ArtifactError, CalculationContext, ManifestKind, PortableError, + SourceProvenance, Storage, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, +}; +use super::{RustiQData, AO_ERI_ARTIFACT}; +use crate::calculation::PreparedCalculation; +use relative_path::RelativePath; +use std::io::Read; +use std::path::Path; + +impl RustiQData { + /// Creates portable scientific data from effective inputs, without running HF. + /// Add known artifacts with the typed setters before calling [`Self::write`]. + pub fn from_calculation(calculation: &PreparedCalculation) -> Result { + let snapshot = Snapshot::from_calculation(calculation)?; + RequestSnapshot::from_request(calculation.request())?; + snapshot.validate_request(calculation.request())?; + let mut data = Self::new_with_identity( + snapshot.identity(), + snapshot.basis_functions(), + ManifestKind::Portable, + ); + data.request = Some(calculation.request().clone()); + data.context = Some(CalculationContext::from_snapshot(snapshot)?); + Ok(data) + } + + /// Opens a portable archive and validates its context and artifact index. + /// Numerical payloads are verified and decoded lazily on first access. + /// Multi-calculation archives require [`super::RustiQBundle::open`]. + pub fn open(path: impl AsRef) -> Result { + let bundle = super::RustiQBundle::open(path)?; + let (mut calculations, sources) = bundle.into_parts(); + if calculations.len() != 1 { + return Err(PortableError::InvalidArchive( + "single-calculation API requires exactly one calculation".into(), + )); + } + let mut data = calculations.remove(0); + data.sources = sources; + Ok(data) + } + + pub(crate) fn validate_portable(mut data: Self) -> Result { + if data.manifest.kind != ManifestKind::Portable { + return Err(PortableError::UnsupportedVersion); + } + let source = data.source.as_mut().expect("opened storage"); + let reference = + data.manifest.calculation.as_ref().ok_or_else(|| { + PortableError::InvalidArchive("missing calculation reference".into()) + })?; + verify_reference(source, reference, CALCULATION_PATH, MAX_CALCULATION_BYTES)?; + let snapshot: Snapshot = source + .read_json(RelativePath::new(CALCULATION_PATH), MAX_CALCULATION_BYTES) + .map_err(|error| PortableError::InvalidCalculation(error.to_string()))?; + snapshot.validate()?; + let reference = data + .manifest + .request + .as_ref() + .ok_or_else(|| PortableError::InvalidArchive("missing request reference".into()))?; + verify_reference(source, reference, REQUEST_PATH, MAX_REQUEST_BYTES)?; + let request_snapshot: RequestSnapshot = source + .read_json(RelativePath::new(REQUEST_PATH), MAX_REQUEST_BYTES) + .map_err(|error| PortableError::InvalidRequest(error.to_string()))?; + let request = request_snapshot.to_request()?; + snapshot.validate_request(&request)?; + data.request = Some(request); + let identity = snapshot.identity(); + if identity.version != data.manifest.scientific_identity.version + || identity.digest != data.manifest.scientific_identity.digest + { + return Err(PortableError::InvalidCalculation( + "scientific identity does not match the resolved inputs".into(), + )); + } + for (name, artifact) in &data.manifest.artifacts { + if artifact + .path + .as_str() + .split('/') + .next() + .is_some_and(|part| { + part.eq_ignore_ascii_case(CALCULATION_PATH) + || part.eq_ignore_ascii_case("manifest.json") + || part.eq_ignore_ascii_case(REQUEST_PATH) + || part.eq_ignore_ascii_case("sources") + }) + { + return Err(ArtifactError::InvalidPath(artifact.path.to_string()).into()); + } + if source.artifact_size(&artifact.path)? != artifact.size { + return Err(ArtifactError::IntegrityMismatch(artifact.path.to_string()).into()); + } + if let ArtifactAttributes::AoEri(attributes) = &artifact.attributes { + if name != AO_ERI_ARTIFACT + || artifact.path.as_str() != AO_ERI_PATH + || attributes.basis_functions != snapshot.basis_functions() + || attributes.computation_version != snapshot.computation_version() + { + return Err(ArtifactError::InvalidMetadata( + "AO ERI metadata disagrees with calculation context".into(), + ) + .into()); + } + } + } + data.basis_functions = Some(snapshot.basis_functions()); + data.context = Some(CalculationContext::from_snapshot(snapshot)?); + Ok(data) + } + + /// Returns the normalized pre-resolution request, absent for directory-cache data. + pub fn request(&self) -> Option<&crate::calculation::CalculationRequest> { + self.request.as_ref() + } + + /// Rebuilds an executable calculation from the resolved context in this portable archive. + /// The archive becomes a read-only source of typed artifacts for normal execution. + /// Payloads are loaded and verified only when their scientific stage needs them. + /// Missing or incompatible artifacts fall back to computation; corrupt compatible + /// artifacts produce an execution error. The source archive is never modified. + pub fn prepare_calculation(self) -> Result { + let context = self.context.as_ref().ok_or_else(|| { + PortableError::InvalidCalculation( + "calculation preparation requires portable calculation context".into(), + ) + })?; + let request = self.request.as_ref().ok_or_else(|| { + PortableError::InvalidCalculation( + "calculation preparation requires a portable request".into(), + ) + })?; + let prepared = PreparedCalculation::from_portable_context(request.clone(), context); + prepared.with_reuse_data(self) + } + + /// Captured sources are opaque provenance and never affect scientific compatibility. + pub fn sources(&self) -> impl ExactSizeIterator { + self.sources.iter() + } + + /// Captures exact source bytes; the original name is informational, never an archive path. + pub fn add_source( + &mut self, + original_name: impl Into, + bytes: impl Into>, + ) -> Result<(), PortableError> { + if self.context.is_none() { + return Err(PortableError::InvalidCalculation( + "source provenance requires portable context".into(), + )); + } + let original_name = original_name.into(); + let bytes = bytes.into(); + let total = self + .sources + .iter() + .map(|source| source.bytes().len()) + .chain(std::iter::once(bytes.len())) + .try_fold(0_u64, |total, length| { + u64::try_from(length) + .ok() + .and_then(|size| total.checked_add(size)) + }) + .ok_or_else(|| PortableError::InvalidArchive("source size exceeds V1 range".into()))?; + if self.sources.len() >= MAX_SOURCES + || total > MAX_SOURCE_BYTES + || original_name.len() > MAX_SOURCE_NAME_BYTES + { + return Err(PortableError::InvalidArchive( + "source provenance exceeds supported limits".into(), + )); + } + self.sources.push(SourceProvenance { + original_name, + bytes, + }); + Ok(()) + } + + /// Returns the portable context, absent only for internal directory-cache data. + pub fn calculation(&self) -> Option<&CalculationContext> { + self.context.as_ref() + } + + /// Producer name and version recorded as provenance, not compatibility policy. + pub fn producer(&self) -> (&str, &str) { + ( + &self.manifest.producer.name, + &self.manifest.producer.version, + ) + } + + /// Lists scientific artifact names and representation identifiers without loading arrays. + /// Unknown representations remain visible; listing does not verify their payloads. + pub fn artifact_representations(&self) -> impl Iterator { + self.ao_eri + .as_ref() + .map(|_| (AO_ERI_ARTIFACT, COMPACT_ERI_REPRESENTATION)) + .into_iter() + .chain( + self.manifest + .artifacts + .iter() + .filter_map(|(name, artifact)| { + (name != AO_ERI_ARTIFACT || self.ao_eri.is_none()) + .then_some((name.as_str(), artifact.representation.as_str())) + }), + ) + } + + /// Tests AO ERI compatibility with another resolved calculation, without reading arrays. + /// Requested HF/MP2 methods do not participate in deterministic integral identity. + pub fn eri_is_compatible(&self, calculation: &PreparedCalculation) -> bool { + if self.ao_eri.is_some() { + return self.matches_eri_identity(calculation); + } + let Some(artifact) = self.manifest.artifacts.get(AO_ERI_ARTIFACT) else { + return false; + }; + matches!(&artifact.attributes, ArtifactAttributes::AoEri(attributes) + if attributes.computation_version == super::super::AO_ERI_COMPUTATION_VERSION) + && artifact.representation == COMPACT_ERI_REPRESENTATION + && self.matches_eri_identity(calculation) + } + + pub(super) fn matches_eri_identity(&self, calculation: &PreparedCalculation) -> bool { + let identity = super::super::ao_eri_identity( + calculation.get_molecule(), + calculation.get_basis(), + calculation.integral_config().schwarz_threshold.value, + ); + self.manifest.scientific_identity.version == identity.version + && self.manifest.scientific_identity.digest == identity.digest + } + + /// Atomically publishes a portable archive. Never overwrites an existing path. + /// The source archive is not modified; unloaded artifacts are copied and verified. + pub fn write(&mut self, path: impl AsRef) -> Result<(), PortableError> { + if self.context.is_none() + || self.request.is_none() + || self.manifest.kind != ManifestKind::Portable + { + return Err(PortableError::InvalidCalculation( + "portable output requires resolved calculation context".into(), + )); + } + let sources = std::mem::take(&mut self.sources); + let result = + super::bundle::write_bundle(std::slice::from_mut(self), &sources, path.as_ref()); + self.sources = sources; + result + } + + /// Writes a single-calculation V1 bundle with a borrowed AO ERI tensor. + pub fn write_with_eri( + &mut self, + path: impl AsRef, + eri: &crate::eri::CompactEri, + ) -> Result<(), PortableError> { + self.validate_eri(eri)?; + let sources = std::mem::take(&mut self.sources); + let result = super::bundle::write_bundle_with_eri( + std::slice::from_mut(self), + &sources, + path.as_ref(), + Some(eri), + ); + self.sources = sources; + result + } + + /// Validates compatibility and lazily decodes the ERI, then transfers ownership. + /// Missing, unsupported, incompatible, or corrupt artifacts are errors. + pub fn take_compatible_eri( + &mut self, + calculation: &PreparedCalculation, + ) -> Result { + if self.ao_eri.is_none() { + let artifact = self + .manifest + .artifacts + .get(AO_ERI_ARTIFACT) + .ok_or(ArtifactError::Missing)?; + if artifact.representation != COMPACT_ERI_REPRESENTATION { + return Err(ArtifactError::UnsupportedRepresentation( + artifact.representation.clone(), + ) + .into()); + } + } + if !self.eri_is_compatible(calculation) { + return Err(PortableError::IncompatibleEri); + } + self.read_eri()?; + Ok(self.take_eri().expect("validated ERI is loaded")) + } + + pub(crate) fn validate_unloaded_eri(&mut self) -> Result<(), PortableError> { + let artifact = &self.manifest.artifacts[AO_ERI_ARTIFACT]; + let ArtifactAttributes::AoEri(attributes) = &artifact.attributes else { + unreachable!() + }; + let source = self.source.as_mut().ok_or(ArtifactError::Missing)?; + source.with_artifact::<_, ArtifactError, _>(&artifact.path, |reader| { + super::super::npy::checked_eri_prefix( + reader, + attributes.basis_functions, + artifact.size, + )?; + Ok(()) + })?; + Ok(()) + } +} + +pub(super) fn read_sources( + source: &mut Storage, + references: &[super::super::manifest::SourceManifest], +) -> Result, PortableError> { + let mut sources = Vec::new(); + if references.len() > MAX_SOURCES { + return Err(PortableError::InvalidArchive( + "too many source payloads".into(), + )); + } + let mut total = 0_u64; + let mut paths = std::collections::HashSet::new(); + for reference in references { + let path = RelativePath::new(&reference.path); + super::super::validate_storage_path(path)?; + total = total + .checked_add(reference.size) + .filter(|total| *total <= MAX_SOURCE_BYTES) + .ok_or_else(|| { + PortableError::InvalidArchive("source payloads exceed size limit".into()) + })?; + if reference.version != 1 { + return Err(PortableError::UnsupportedVersion); + } + if !reference.path.starts_with("sources/") + || reference.original_name.len() > MAX_SOURCE_NAME_BYTES + || !paths.insert(reference.path.to_lowercase()) + { + return Err(PortableError::InvalidArchive( + "invalid source reference".into(), + )); + } + let expected = super::super::manifest::SnapshotManifest { + path: reference.path.clone(), + version: reference.version, + size: reference.size, + digest: reference.digest, + }; + verify_reference(source, &expected, &reference.path, MAX_SOURCE_BYTES)?; + let bytes = source.with_artifact::<_, PortableError, _>(path, |reader| { + let mut bytes = Vec::new(); + reader.take(reference.size + 1).read_to_end(&mut bytes)?; + if u64::try_from(bytes.len()) + .map_err(|_| PortableError::InvalidArchive("source size exceeds V1 range".into()))? + != reference.size + { + return Err(ArtifactError::IntegrityMismatch(reference.path.clone()).into()); + } + Ok(bytes) + })?; + sources.push(SourceProvenance { + original_name: reference.original_name.clone(), + bytes, + }); + } + Ok(sources) +} + +fn verify_reference( + source: &mut Storage, + reference: &super::super::manifest::SnapshotManifest, + expected_path: &str, + limit: u64, +) -> Result<(), PortableError> { + if reference.version != 1 { + return Err(PortableError::UnsupportedVersion); + } + if reference.path != expected_path || reference.size > limit { + return Err(PortableError::InvalidArchive( + "invalid snapshot/source reference".into(), + )); + } + let path = RelativePath::new(expected_path); + if source.artifact_size(path)? != reference.size { + return Err(ArtifactError::IntegrityMismatch(expected_path.into()).into()); + } + let metadata = source.artifact_metadata(path)?; + if metadata.size != reference.size || metadata.digest != reference.digest { + return Err(ArtifactError::IntegrityMismatch(expected_path.into()).into()); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/rustiq-core/src/persistence/data/portable/tests.rs b/crates/rustiq-core/src/persistence/data/portable/tests.rs new file mode 100644 index 00000000..3f974b18 --- /dev/null +++ b/crates/rustiq-core/src/persistence/data/portable/tests.rs @@ -0,0 +1,1332 @@ +use super::super::super::{sha256, AoEriArtifact}; +use super::*; +use crate::{ + calculation::CalculationBuilder, + config::{HfConfig, HfMethod, MoleculeConfig, Mp2Config}, + eri::CompactEri, + molecules::geometry::Geometry, + test_utils::load_minimal_basis_file, +}; +use std::{ + fs::{self, File}, + io::{Read, Write}, + num::NonZeroU8, +}; +use zip::{write::SimpleFileOptions, CompressionMethod, ZipArchive, ZipWriter}; + +const CALCULATION_PATH: &str = "calculations/calculation-0/calculation.json"; +const REQUEST_PATH: &str = "calculations/calculation-0/request.json"; +const AO_ERI_PATH: &str = "calculations/calculation-0/arrays/integrals/ao-eri.npy"; + +fn calculation() -> PreparedCalculation { + let geometry = Geometry::from_source("private.xyz", "2\nH2\nH 0 0 0\nH 1.4 0 0\n").unwrap(); + CalculationBuilder::new(&geometry, &load_minimal_basis_file()) + .prepare() + .unwrap() +} +fn data() -> RustiQData { + let mut data = RustiQData::from_calculation(&calculation()).unwrap(); + data.set_eri(CompactEri::from_ordered_values(2, vec![0.5, 1.5, 2.5, 3.5, 4.5, 5.5]).unwrap()) + .unwrap(); + data +} + +#[test] +fn borrowed_eri_writer_rejects_multi_calculation_input() { + let first = RustiQData::from_calculation(&calculation()).unwrap(); + let second = RustiQData::from_calculation(&calculation()).unwrap(); + let mut calculations = [first, second]; + let directory = tempfile::tempdir().unwrap(); + let destination = directory.path().join("invalid.rustiq"); + + let error = super::super::bundle::write_bundle_with_eri( + &mut calculations, + &[], + &destination, + Some(&CompactEri::Zeroed(2)), + ) + .unwrap_err(); + + assert!(matches!(error, PortableError::InvalidArchive(_))); + assert!(!destination.exists()); +} + +fn source_fixture() -> Vec { + include_str!("../../../../tests/data/persistence/source-original-v1.toml.hex") + .trim() + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect() +} +fn members(path: &Path) -> Vec<(String, Vec)> { + let mut archive = ZipArchive::new(File::open(path).unwrap()).unwrap(); + (0..archive.len()) + .map(|i| { + let mut member = archive.by_index(i).unwrap(); + let name = member.name().to_owned(); + let mut bytes = Vec::new(); + member.read_to_end(&mut bytes).unwrap(); + (name, bytes) + }) + .collect() +} +fn write_members(path: &Path, members: &[(String, Vec)]) { + let mut writer = ZipWriter::new(File::create(path).unwrap()); + for (name, bytes) in members { + writer + .start_file( + name, + SimpleFileOptions::default() + .compression_method(CompressionMethod::Stored) + .large_file(true), + ) + .unwrap(); + writer.write_all(bytes).unwrap(); + } + writer.finish().unwrap(); +} +fn edit_json( + members: &mut [(String, Vec)], + name: &str, + edit: impl FnOnce(&mut serde_json::Value), +) { + let bytes = &mut members.iter_mut().find(|(n, _)| n == name).unwrap().1; + let mut json = serde_json::from_slice(bytes).unwrap(); + edit(&mut json); + *bytes = serde_json::to_vec(&json).unwrap(); +} +fn refresh_snapshot_digest(members: &mut [(String, Vec)]) { + let snapshot = &members + .iter() + .find(|(n, _)| n == CALCULATION_PATH) + .unwrap() + .1; + let (size, digest) = (snapshot.len(), sha256(snapshot).to_string()); + edit_json(members, "manifest.json", |m| { + m["calculations"][0]["calculation"]["size"] = size.into(); + m["calculations"][0]["calculation"]["digest"] = digest.into(); + }); +} +fn refresh_eri_digest(members: &mut [(String, Vec)]) { + let payload = &members.iter().find(|(n, _)| n == AO_ERI_PATH).unwrap().1; + let (size, digest) = (payload.len(), sha256(payload).to_string()); + edit_json(members, "manifest.json", |m| { + m["calculations"][0]["artifacts"]["ao_eri"]["size"] = size.into(); + m["calculations"][0]["artifacts"]["ao_eri"]["digest"] = digest.into(); + }); +} + +#[test] +fn rust_round_trip_is_lazy_self_describing_and_reproducible() { + let dir = tempfile::tempdir().unwrap(); + let first = dir.path().join("first.rustiq"); + let second = dir.path().join("second.rustiq"); + data().write(&first).unwrap(); + data().write(&second).unwrap(); + assert_eq!(fs::read(&first).unwrap(), fs::read(&second).unwrap()); + let mut restored = RustiQData::open(&first).unwrap(); + assert!(restored.ao_eri.is_none()); + let context = restored.calculation().unwrap(); + assert_eq!( + context.atoms().collect::>(), + vec![(1, [0.0, 0.0, 0.0]), (1, [1.4, 0.0, 0.0])] + ); + assert_eq!(context.hf_method(), crate::config::ResolvedHfMethod::Rhf); + assert_eq!(context.basis().len(), 2); + assert!(restored.eri_is_compatible(&calculation())); + assert_eq!( + restored.read_eri().unwrap().ordered_values(), + &[0.5, 1.5, 2.5, 3.5, 4.5, 5.5] + ); + let pointer = restored.read_eri().unwrap() as *const CompactEri; + assert_eq!(restored.read_eri().unwrap() as *const CompactEri, pointer); + let zip = ZipArchive::new(File::open(&first).unwrap()).unwrap(); + assert_eq!(zip.len(), 4); + let request = members(&first) + .into_iter() + .find(|(n, _)| n == REQUEST_PATH) + .unwrap() + .1; + assert_eq!( + serde_json::from_slice::(&request).unwrap(), + serde_json::from_str::(include_str!( + "../../../../tests/data/persistence/request-h2-v1.json" + )) + .unwrap() + ); + assert_eq!(restored.sources().len(), 0); + let snapshot = members(&first) + .into_iter() + .find(|(n, _)| n == CALCULATION_PATH) + .unwrap() + .1; + assert_eq!( + serde_json::from_slice::(&snapshot).unwrap(), + serde_json::from_str::(include_str!( + "../../../../tests/data/persistence/calculation-h2-v1.json" + )) + .unwrap() + ); +} + +#[test] +fn normalized_request_round_trip_keeps_auto_angstrom_defaults_and_requested_options() { + use crate::config::{DensityGuessConfig, MemoryLimit, RandomGuessConfig}; + use crate::molecules::units::Units; + let geometry = + Geometry::from_source("input.xyz", "2\nignored comment\nH 0 0 0\nH 0.74 0 0\n").unwrap(); + let basis = load_minimal_basis_file(); + let defaulted = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + let explicit = CalculationBuilder::new(&geometry, &basis) + .with_hf(HfConfig::default()) + .with_molecule_config(MoleculeConfig::default()) + .prepare() + .unwrap(); + assert_eq!( + serde_json::to_value(RequestSnapshot::from_request(defaulted.request()).unwrap()).unwrap(), + serde_json::to_value(RequestSnapshot::from_request(explicit.request()).unwrap()).unwrap() + ); + let prepared = CalculationBuilder::new(&geometry, &basis) + .with_basis_label("portable-basis-label") + .with_molecule_config(MoleculeConfig { + units: Units::Angstrom, + ..Default::default() + }) + .with_hf(HfConfig { + guess: DensityGuessConfig::Random { + config: RandomGuessConfig::default(), + } + .into(), + ..Default::default() + }) + .with_mp2(Mp2Config { + frozen_orbitals: 1.into(), + memory_limit: MemoryLimit::Fixed(bytesize::ByteSize::mib(42)).into(), + }) + .prepare() + .unwrap(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("round-trip.rustiq"); + RustiQData::from_calculation(&prepared) + .unwrap() + .write(&path) + .unwrap(); + let restored = RustiQData::open(&path).unwrap(); + let request = restored.request().unwrap(); + assert_eq!(request.hf().method.value, HfMethod::Auto); + assert_eq!(request.molecule().units, Units::Angstrom); + assert_eq!(request.geometry().atoms[1].position[0], 0.74); + assert_eq!(request.basis_name(), "portable-basis-label"); + assert_eq!( + request.mp2().unwrap().memory_limit.value, + prepared.request().mp2().unwrap().memory_limit.value + ); + assert!(request.geometry().comment.is_empty()); + let DensityGuessConfig::Random { config } = request.hf().guess.value else { + panic!() + }; + assert_eq!(config.random.seed, None); + let context = restored.calculation().unwrap(); + assert_eq!(context.hf_method(), crate::config::ResolvedHfMethod::Rhf); + assert!(context.atoms().nth(1).unwrap().1[0] > 1.0); + let DensityGuessConfig::Random { config } = context.density_guess() else { + panic!() + }; + assert!(config.random.seed.is_some()); + assert_eq!( + serde_json::to_value(RequestSnapshot::from_request(request).unwrap()).unwrap(), + serde_json::to_value(RequestSnapshot::from_request(prepared.request()).unwrap()).unwrap() + ); +} + +#[test] +fn opaque_sources_preserve_exact_bytes_without_affecting_scientific_state() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rustiq"); + let output = dir.path().join("copied.rustiq"); + let mut original = data(); + let sources: [(&str, &[u8]); 2] = [ + ( + "../../private/calculation.toml", + b"# preserve comments\r\n[hf]\r\nmethod = 'Auto'\r\n", + ), + ( + "C:\\private\\molecule.xyz", + b"2\r\noriginal comment\r\nH 0.000 0 0\r\nH 1.4000 0 0\r\n\xff", + ), + ]; + for (name, bytes) in sources { + original.add_source(name, bytes).unwrap(); + } + original.write(&path).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o444)).unwrap(); + } + let mut restored = RustiQData::open(&path).unwrap(); + assert!(restored.ao_eri.is_none()); + assert!(restored.eri_is_compatible(&calculation())); + for (source, (name, bytes)) in restored.sources().zip(sources) { + assert_eq!(source.original_name(), name); + assert_eq!(source.bytes(), bytes); + } + let archive_members = members(&path); + assert!(archive_members.iter().any(|(name, _)| name == "sources/0")); + assert!(!archive_members + .iter() + .any(|(name, _)| name.contains("private"))); + restored.write(&output).unwrap(); + assert_eq!(members(&output), archive_members); + assert_eq!(members(&path), archive_members); +} + +#[test] +fn request_and_source_integrity_versions_and_limits_are_checked_on_open() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("good.rustiq"); + let mut original = data(); + original + .add_source("input.toml", b"original bytes".as_slice()) + .unwrap(); + original.write(&path).unwrap(); + let original = members(&path); + let broken = dir.path().join("broken.rustiq"); + for (target, field, value) in [ + ( + "manifest.json", + "/calculations/0/request", + serde_json::Value::Null, + ), + ( + "manifest.json", + "/calculations/0/request/path", + serde_json::json!(CALCULATION_PATH), + ), + ( + "manifest.json", + "/calculations/0/request/version", + serde_json::json!(99), + ), + ( + "manifest.json", + "/calculations/0/request/size", + serde_json::json!(MAX_REQUEST_BYTES + 1), + ), + ( + "manifest.json", + "/sources/0/path", + serde_json::json!("../input.toml"), + ), + ("manifest.json", "/sources/0/version", serde_json::json!(99)), + ( + "manifest.json", + "/sources/0/size", + serde_json::json!(MAX_SOURCE_BYTES + 1), + ), + (REQUEST_PATH, "/version", serde_json::json!(99)), + (REQUEST_PATH, "/units", serde_json::json!("invalid")), + (REQUEST_PATH, "/hf/method", serde_json::json!("invalid")), + (REQUEST_PATH, "/hf/max_iterations", serde_json::json!(0)), + (REQUEST_PATH, "/hf/diis_size", serde_json::json!(1)), + ( + REQUEST_PATH, + "/hf/convergence_threshold", + serde_json::json!(-1), + ), + ( + REQUEST_PATH, + "/atoms/0/atomic_number", + serde_json::json!(119), + ), + (REQUEST_PATH, "/atoms/0/position/0", serde_json::json!(12)), + ] { + let mut contents = original.clone(); + edit_json(&mut contents, target, |json| { + *json.pointer_mut(field).unwrap() = value + }); + if target == REQUEST_PATH { + let bytes = &contents + .iter() + .find(|(name, _)| name == REQUEST_PATH) + .unwrap() + .1; + let (size, digest) = (bytes.len(), sha256(bytes).to_string()); + edit_json(&mut contents, "manifest.json", |m| { + m["calculations"][0]["request"]["size"] = size.into(); + m["calculations"][0]["request"]["digest"] = digest.into(); + }); + } + write_members(&broken, &contents); + assert!(RustiQData::open(&broken).is_err(), "{target} {field}"); + } + for name in [REQUEST_PATH, "sources/0"] { + let mut contents = original.clone(); + contents.iter_mut().find(|(n, _)| n == name).unwrap().1[0] ^= 1; + write_members(&broken, &contents); + assert!(matches!( + RustiQData::open(&broken), + Err(PortableError::Artifact(ArtifactError::IntegrityMismatch(_))) + )); + contents.retain(|(n, _)| n != name); + write_members(&broken, &contents); + assert!(RustiQData::open(&broken).is_err()); + } +} + +#[test] +fn empty_artifacts_uhf_and_mp2_context_round_trip() { + let geometry = Geometry::from_source("H.xyz", "1\nH\nH 0 0 0\n").unwrap(); + let basis = load_minimal_basis_file(); + let prepared = CalculationBuilder::new(&geometry, &basis) + .with_molecule_config(MoleculeConfig { + multiplicity: NonZeroU8::new(2).unwrap().into(), + ..Default::default() + }) + .with_hf(HfConfig { + method: HfMethod::Uhf.into(), + diis: crate::config::DiisConfig { + enabled: true, + ..Default::default() + }, + ..Default::default() + }) + .with_mp2(Mp2Config::default()) + .prepare() + .unwrap(); + let mut data = RustiQData::from_calculation(&prepared).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("h.rustiq"); + data.write(&path).unwrap(); + let mut restored = RustiQData::open(&path).unwrap(); + assert!(restored.get::().unwrap().is_none()); + let context = restored.calculation().unwrap(); + assert_eq!(context.multiplicity(), 2); + assert_eq!(context.hf_method(), crate::config::ResolvedHfMethod::Uhf); + assert_eq!(context.diis_size(), Some(6)); + assert_eq!(context.mp2_frozen_orbitals(), Some(0)); +} + +#[test] +fn compatibility_depends_on_effective_integral_inputs_only() { + let prepared = calculation(); + let mut data = data(); + let mp2 = CalculationBuilder::new(prepared.get_molecule(), &load_minimal_basis_file()) + .with_mp2(Mp2Config::default()) + .prepare() + .unwrap(); + assert!(data.eri_is_compatible(&mp2)); + let changed = CalculationBuilder::new(prepared.get_molecule(), &load_minimal_basis_file()) + .with_integrals(crate::config::IntegralConfig { + schwarz_threshold: None.into(), + }) + .prepare() + .unwrap(); + assert!(!data.eri_is_compatible(&changed)); + data.ao_eri = None; + assert!(!data.eri_is_compatible(&prepared)); +} + +#[test] +fn publication_never_overwrites_and_concurrent_writer_has_one_winner() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("target.rustiq"); + fs::write(&path, b"user data").unwrap(); + assert!(matches!( + data().write(&path), + Err(PortableError::AlreadyExists) + )); + assert_eq!(fs::read(&path).unwrap(), b"user data"); + fs::remove_file(&path).unwrap(); + let barrier = std::sync::Barrier::new(2); + let results = std::thread::scope(|scope| { + let first = scope.spawn(|| { + let mut data = data(); + barrier.wait(); + data.write(&path) + }); + let second = scope.spawn(|| { + let mut data = data(); + barrier.wait(); + data.write(&path) + }); + [first.join().unwrap(), second.join().unwrap()] + }); + assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1); + assert_eq!( + results + .iter() + .filter(|r| matches!(r, Err(PortableError::AlreadyExists))) + .count(), + 1 + ); + assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 1); + RustiQData::open(&path).unwrap().read_eri().unwrap(); +} + +#[test] +fn corrupt_payload_is_lazy_but_cannot_be_read_or_republished() { + let dir = tempfile::tempdir().unwrap(); + let original = dir.path().join("original.rustiq"); + data().write(&original).unwrap(); + let mut contents = members(&original); + *contents + .iter_mut() + .find(|(n, _)| n == AO_ERI_PATH) + .unwrap() + .1 + .last_mut() + .unwrap() ^= 1; + let corrupt = dir.path().join("corrupt.rustiq"); + write_members(&corrupt, &contents); + let portable_calculation = RustiQData::open(&corrupt) + .unwrap() + .prepare_calculation() + .unwrap(); + use crate::calculation::{CalculationError, CalculationExecution}; + let error = portable_calculation.execute().unwrap_err(); + assert!(matches!( + error.cause(), + CalculationError::Artifact(ArtifactError::IntegrityMismatch(_)) + )); + let mut restored = RustiQData::open(&corrupt).unwrap(); + assert!(restored.take_compatible_eri(&calculation()).is_err()); + assert!(restored.read_eri().is_err()); + let output = dir.path().join("output.rustiq"); + assert!(restored.write(&output).is_err()); + assert!(!output.exists()); + assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 2); +} + +#[test] +fn unknown_artifacts_survive_verified_streaming_copy() { + let dir = tempfile::tempdir().unwrap(); + let first = dir.path().join("first.rustiq"); + data().write(&first).unwrap(); + let mut contents = members(&first); + let opaque = b"future payload".to_vec(); + contents.push(( + "calculations/calculation-0/arrays/future.bin".into(), + opaque.clone(), + )); + edit_json(&mut contents, "manifest.json", |m| { + m["calculations"][0]["artifacts"]["future"] = serde_json::json!({"path":"calculations/calculation-0/arrays/future.bin","size":opaque.len(),"digest":sha256(&opaque),"representation":"future-v99","attributes":{"opaque":17}}); + }); + let source = dir.path().join("source.rustiq"); + write_members(&source, &contents); + let mut restored = RustiQData::open(&source).unwrap(); + let output = dir.path().join("copy.rustiq"); + restored.write(&output).unwrap(); + assert!(restored.ao_eri.is_none()); + assert!(members(&output).contains(&( + "calculations/calculation-0/arrays/future.bin".into(), + opaque + ))); + assert_eq!( + RustiQData::open(&output).unwrap().manifest.artifacts["future"], + restored.manifest.artifacts["future"] + ); + assert_eq!(members(&source), contents); +} + +#[test] +fn rejects_invalid_context_manifest_and_artifact_metadata() { + let dir = tempfile::tempdir().unwrap(); + let original = dir.path().join("original.rustiq"); + data().write(&original).unwrap(); + let original = members(&original); + for (target, field, value) in [ + ("manifest.json", "/format_version", serde_json::json!(999)), + ( + "manifest.json", + "/calculations/0/scientific_identity/version", + serde_json::json!(99), + ), + ( + "manifest.json", + "/calculations/0/artifacts/ao_eri/attributes/basis_functions", + serde_json::json!(3), + ), + ( + "manifest.json", + "/calculations/0/artifacts/ao_eri/attributes/computation_version", + serde_json::json!(99), + ), + ( + "manifest.json", + "/calculations/0/artifacts/ao_eri/path", + serde_json::json!("CALCULATION.JSON"), + ), + ( + "manifest.json", + "/calculations/0/artifacts/ao_eri/size", + serde_json::json!(1), + ), + (CALCULATION_PATH, "/version", serde_json::json!(99)), + (CALCULATION_PATH, "/units", serde_json::json!("angstrom")), + (CALCULATION_PATH, "/charge", serde_json::json!(100)), + (CALCULATION_PATH, "/multiplicity", serde_json::json!(0)), + ( + CALCULATION_PATH, + "/atoms/0/position/0", + serde_json::json!(5.0), + ), + ( + CALCULATION_PATH, + "/basis/0/components/0/primitives/0/exponent", + serde_json::json!(-1.0), + ), + (CALCULATION_PATH, "/hf/diis_size", serde_json::json!(1)), + ( + CALCULATION_PATH, + "/hf/convergence_threshold", + serde_json::json!(0), + ), + ] { + let mut contents = original.clone(); + edit_json(&mut contents, target, |j| { + *j.pointer_mut(field).unwrap() = value + }); + if target == CALCULATION_PATH { + refresh_snapshot_digest(&mut contents); + } + let path = dir.path().join("invalid.rustiq"); + write_members(&path, &contents); + assert!(RustiQData::open(&path).is_err(), "{target} {field}"); + } + let mut contents = original.clone(); + contents.retain(|(n, _)| n != CALCULATION_PATH); + let path = dir.path().join("missing.rustiq"); + write_members(&path, &contents); + assert!(RustiQData::open(path).is_err()); +} + +#[test] +fn rejects_unsafe_member_paths_and_local_header_disagreement() { + let dir = tempfile::tempdir().unwrap(); + for name in [ + "../evil", + "/evil", + "C:/evil", + "a\\evil", + "a/CON", + "a/", + "a//b", + "a/./b", + "a/../b", + "a/b.", + "a/b ", + "a/b?", + "manifest.json/child", + ] { + let path = dir.path().join("bad.rustiq"); + let mut contents = vec![("manifest.json".into(), b"{}".to_vec())]; + contents.push((name.into(), vec![])); + write_members(&path, &contents); + assert!(RustiQData::open(&path).is_err(), "{name}"); + } + let path = dir.path().join("good.rustiq"); + data().write(&path).unwrap(); + let original = fs::read(&path).unwrap(); + for offset in [30, 6, 8] { + let mut bytes = original.clone(); + bytes[offset] ^= 1; + let invalid = dir.path().join("invalid.rustiq"); + fs::write(&invalid, bytes).unwrap(); + assert!(RustiQData::open(&invalid).is_err()); + } + for len in [0, 10, original.len() / 2, original.len() - 1] { + let invalid = dir.path().join("truncated.rustiq"); + fs::write(&invalid, &original[..len]).unwrap(); + assert!(RustiQData::open(invalid).is_err()); + } +} + +#[test] +fn rejects_duplicate_members_before_zip_library_deduplication() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("duplicate.rustiq"); + write_members( + &path, + &[("one.json".into(), vec![]), ("two.json".into(), vec![])], + ); + let mut bytes = fs::read(&path).unwrap(); + for index in 0..bytes.len() - 8 { + if &bytes[index..index + 8] == b"two.json" { + bytes[index..index + 8].copy_from_slice(b"one.json"); + } + } + fs::write(&path, bytes).unwrap(); + assert!(RustiQData::open(&path) + .unwrap_err() + .to_string() + .contains("duplicate")); + write_members( + &path, + &[ + ("manifest.json".into(), vec![]), + ("MANIFEST.JSON".into(), vec![]), + ], + ); + assert!(RustiQData::open(path) + .unwrap_err() + .to_string() + .contains("duplicate")); +} + +#[test] +fn malformed_npy_is_rejected_even_with_correct_digest() { + let dir = tempfile::tempdir().unwrap(); + let good = dir.path().join("good.rustiq"); + data().write(&good).unwrap(); + for payload in [ + b"not npy".to_vec(), + b"\x93NUMPY\x02\x00\xff\xff\xff\x7f".to_vec(), + ] { + let mut contents = members(&good); + contents + .iter_mut() + .find(|(n, _)| n == AO_ERI_PATH) + .unwrap() + .1 = payload; + refresh_eri_digest(&mut contents); + let bad = dir.path().join("bad.rustiq"); + write_members(&bad, &contents); + let mut restored = RustiQData::open(&bad).unwrap(); + assert!(restored.take_compatible_eri(&calculation()).is_err()); + assert!(restored.read_eri().is_err()); + assert!(restored.write(dir.path().join("copy.rustiq")).is_err()); + } +} + +#[test] +#[ignore = "writes a portable artifact for Python interoperability"] +fn writes_archive_for_python_interoperability() { + let path = std::env::var_os("RUSTIQ_ARCHIVE_TEST_OUTPUT").expect("output path"); + let mut data = data(); + data.add_source("../../calculation.toml", source_fixture()) + .unwrap(); + data.write(path).unwrap(); +} + +#[test] +fn reads_python_zip64_fixtures_with_both_npy_byte_orders() { + for fixture in [ + include_str!("../../../../tests/data/persistence/portable-python-little-v1.rustiq.hex"), + include_str!("../../../../tests/data/persistence/portable-python-big-v1.rustiq.hex"), + ] { + let bytes: Vec = fixture + .trim() + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect(); + assert!(bytes.windows(4).any(|w| w == b"PK\x06\x06")); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("python.rustiq"); + fs::write(&path, bytes).unwrap(); + let mut data = RustiQData::open(&path).unwrap(); + assert!(data.eri_is_compatible(&calculation())); + assert_eq!( + data.read_eri().unwrap().ordered_values(), + &[0.5, 1.5, 2.5, 3.5, 4.5, 5.5] + ); + } +} + +#[test] +#[cfg(unix)] +fn zip64_handles_offsets_beyond_four_gib_without_allocating_large_payloads() { + use std::io::{Seek, SeekFrom}; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sparse.rustiq"); + let mut file = File::create(&path).unwrap(); + file.seek(SeekFrom::Start(u64::from(u32::MAX) + 1)).unwrap(); + super::super::bundle::write_snapshot( + std::slice::from_mut(&mut data()), + &[], + Storage::create_zip(file), + ) + .unwrap(); + assert!(fs::metadata(&path).unwrap().len() > u64::from(u32::MAX)); + let mut restored = RustiQData::open(&path).unwrap(); + assert_eq!( + restored.read_eri().unwrap().ordered_values(), + &[0.5, 1.5, 2.5, 3.5, 4.5, 5.5] + ); +} + +#[test] +fn rejects_links_special_entries_encryption_and_oversized_metadata() { + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("source.rustiq"); + data().write(&source).unwrap(); + let original = fs::read(&source).unwrap(); + let central = original + .windows(4) + .position(|w| w == b"PK\x01\x02") + .unwrap(); + for mode in [0o120777_u32, 0o020666, 0o040755] { + let mut bytes = original.clone(); + bytes[central + 38..central + 42].copy_from_slice(&(mode << 16).to_le_bytes()); + let path = dir.path().join("special.rustiq"); + fs::write(&path, bytes).unwrap(); + assert!(RustiQData::open(path).is_err()); + } + for flags in [1_u16, 0x40, 0x2000] { + let mut bytes = original.clone(); + bytes[central + 8..central + 10].copy_from_slice(&flags.to_le_bytes()); + let path = dir.path().join("encrypted.rustiq"); + fs::write(&path, bytes).unwrap(); + assert!(RustiQData::open(path).is_err()); + } + let mut contents = members(&source); + let manifest = &mut contents + .iter_mut() + .find(|(n, _)| n == "manifest.json") + .unwrap() + .1; + manifest.resize(1024 * 1024 + 1, b' '); + let path = dir.path().join("huge.rustiq"); + write_members(&path, &contents); + assert!(RustiQData::open(path).is_err()); + let mut bytes = original.clone(); + let end = bytes.len() - 22; + bytes[end + 8..end + 10].copy_from_slice(&4097_u16.to_le_bytes()); + bytes[end + 10..end + 12].copy_from_slice(&4097_u16.to_le_bytes()); + let path = dir.path().join("many.rustiq"); + fs::write(&path, bytes).unwrap(); + assert!(RustiQData::open(path).is_err()); + let mut contents = members(&source); + edit_json(&mut contents, "manifest.json", |m| { + m["calculations"][0]["calculation"]["size"] = (MAX_CALCULATION_BYTES + 1).into() + }); + let path = dir.path().join("snapshot-limit.rustiq"); + write_members(&path, &contents); + assert!(RustiQData::open(path).is_err()); +} + +#[test] +fn crc_and_actual_decompressed_size_are_checked() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("valid.rustiq"); + data().write(&path).unwrap(); + // Metadata is deflated. Tamper with its declared length in the central directory. + let mut bytes = fs::read(&path).unwrap(); + let central = bytes.windows(4).position(|w| w == b"PK\x01\x02").unwrap(); + bytes[central + 24..central + 28].copy_from_slice(&1_u32.to_le_bytes()); + let bad = dir.path().join("size.rustiq"); + fs::write(&bad, bytes).unwrap(); + assert!(RustiQData::open(bad).is_err()); + let mut bytes = fs::read(&path).unwrap(); + bytes[central + 16] ^= 1; + let bad = dir.path().join("crc.rustiq"); + fs::write(&bad, bytes).unwrap(); + assert!(RustiQData::open(bad).is_err()); +} + +#[test] +fn settings_are_resolved_and_machine_policy_is_excluded() { + use crate::config::{ + DensityGuessConfig, GuessPerturbationConfig, MemoryLimit, RandomGuessConfig, + }; + use crate::molecules::units::Units; + let basis = load_minimal_basis_file(); + let geometry = Geometry::from_source( + "source-path-must-not-leak.xyz", + "2\nH2\nH 0 0 0\nH 0.74 0 0\n", + ) + .unwrap(); + for guess in [ + DensityGuessConfig::Zero, + DensityGuessConfig::CoreHamiltonian { + perturbation: Some(GuessPerturbationConfig::default()), + }, + DensityGuessConfig::OneElectron { + perturbation: Some(GuessPerturbationConfig::default()), + }, + DensityGuessConfig::Random { + config: RandomGuessConfig::default(), + }, + ] { + let prepared = CalculationBuilder::new(&geometry, &basis) + .with_molecule_config(MoleculeConfig { + units: Units::Angstrom, + ..Default::default() + }) + .with_hf(HfConfig { + guess: guess.into(), + ..Default::default() + }) + .with_mp2(Mp2Config { + memory_limit: MemoryLimit::Fixed(bytesize::ByteSize::mib(42)).into(), + ..Default::default() + }) + .prepare() + .unwrap(); + let data = RustiQData::from_calculation(&prepared).unwrap(); + let snapshot = &data.calculation().unwrap().0; + let json = serde_json::to_string(snapshot).unwrap(); + for forbidden in [ + "source-path", + "memory_limit", + "cache", + "span", + "auto", + "angstrom", + ] { + assert!(!json.contains(forbidden), "{forbidden}"); + } + let restored: Snapshot = serde_json::from_str(&json).unwrap(); + restored.validate().unwrap(); + assert_eq!(snapshot.identity(), restored.identity()); + assert_eq!( + format!( + "{:?}", + CalculationContext::from_snapshot(restored) + .unwrap() + .density_guess() + ), + format!("{:?}", prepared.hf_config().guess.value) + ); + assert!(data.calculation().unwrap().atoms().nth(1).unwrap().1[0] > 1.0); + } +} + +#[test] +fn unsupported_representation_is_inspectable_but_not_scientifically_usable() { + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("source.rustiq"); + data().write(&source).unwrap(); + let mut contents = members(&source); + edit_json(&mut contents, "manifest.json", |m| { + m["calculations"][0]["artifacts"]["ao_eri"]["representation"] = "future-eri-v2".into() + }); + let unknown = dir.path().join("unknown.rustiq"); + write_members(&unknown, &contents); + let mut restored = RustiQData::open(&unknown).unwrap(); + assert!(restored.calculation().is_some()); + assert!(!restored.eri_is_compatible(&calculation())); + assert!(restored.take_compatible_eri(&calculation()).is_err()); + assert!(restored.read_eri().is_err()); + restored.write(dir.path().join("copy.rustiq")).unwrap(); +} + +#[test] +fn rewriting_foreign_producer_updates_provenance_only() { + let dir = tempfile::tempdir().unwrap(); + let first = dir.path().join("first.rustiq"); + data().write(&first).unwrap(); + let mut contents = members(&first); + edit_json(&mut contents, "manifest.json", |m| { + m["producer"] = serde_json::json!({"name":"Python","version":"old"}); + }); + let foreign = dir.path().join("foreign.rustiq"); + write_members(&foreign, &contents); + let mut restored = RustiQData::open(&foreign).unwrap(); + assert_eq!(restored.producer(), ("Python", "old")); + let output = dir.path().join("rewritten.rustiq"); + restored.write(&output).unwrap(); + assert!(restored.ao_eri.is_none()); + let mut rewritten = RustiQData::open(&output).unwrap(); + assert_eq!(rewritten.producer(), ("RustiQ", env!("CARGO_PKG_VERSION"))); + assert_eq!( + rewritten.manifest.scientific_identity, + restored.manifest.scientific_identity + ); + assert_eq!(rewritten.manifest.artifacts, restored.manifest.artifacts); + assert!(rewritten.eri_is_compatible(&calculation())); + assert_eq!( + rewritten.read_eri().unwrap().ordered_values(), + &[0.5, 1.5, 2.5, 3.5, 4.5, 5.5] + ); + assert_eq!(members(&foreign), contents); + // Producer changes do not disturb reproducibility for the current writer. + assert_eq!(fs::read(&first).unwrap(), fs::read(&output).unwrap()); +} + +#[test] +fn independently_rounded_python_angstrom_conversion_is_accepted() { + let fixture = + include_str!("../../../../tests/data/persistence/portable-python-angstrom-v1.rustiq.hex"); + let bytes: Vec = fixture + .trim() + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("python.rustiq"); + fs::write(&path, bytes).unwrap(); + let mut restored = RustiQData::open(&path).unwrap(); + let requested = restored.request().unwrap().geometry().atoms[1].position[0]; + let resolved = restored.calculation().unwrap().atoms().nth(1).unwrap().1[0]; + // Python Decimal divided the exact binary64 request by the decimal V1 + // constant, rounding only the final result. It differs by one ULP from + // dividing by the rounded binary64 constant, so exact equality would fail. + assert_eq!(requested, 0.74); + assert_ne!(requested / 0.529_177_210_903, resolved); + assert_eq!( + (requested / 0.529_177_210_903) + .to_bits() + .abs_diff(resolved.to_bits()), + 1 + ); + let identity = restored.manifest.scientific_identity.clone(); + let rewritten = dir.path().join("rewritten.rustiq"); + restored.write(&rewritten).unwrap(); + assert_eq!( + RustiQData::open(&rewritten) + .unwrap() + .manifest + .scientific_identity, + identity + ); + let mut contents = members(&path); + edit_json(&mut contents, REQUEST_PATH, |r| { + r["atoms"][1]["position"][0] = (requested + 1e-10).into() + }); + let request = &contents.iter().find(|(n, _)| n == REQUEST_PATH).unwrap().1; + let (size, digest) = (request.len(), sha256(request).to_string()); + edit_json(&mut contents, "manifest.json", |m| { + m["calculations"][0]["request"]["size"] = size.into(); + m["calculations"][0]["request"]["digest"] = digest.into(); + }); + let invalid = dir.path().join("changed.rustiq"); + write_members(&invalid, &contents); + assert!(matches!( + RustiQData::open(&invalid), + Err(PortableError::InvalidRequest(_)) + )); +} + +#[test] +fn published_required_fields_match_v1_decoders() { + use serde_json::{json, Value}; + let cases = [ + ( + include_str!("../../../../../../schemas/request-snapshot-v1.schema.json"), + include_str!("../../../../tests/data/persistence/request-h2-v1.json"), + true, + vec![ + ("", ""), + ("/hf", "hf"), + ("/atoms/0", "atom"), + ("/mp2", "mp2"), + ], + ), + ( + include_str!("../../../../../../schemas/calculation-snapshot-v1.schema.json"), + include_str!("../../../../tests/data/persistence/calculation-h2-v1.json"), + false, + vec![ + ("", ""), + ("/hf", "hf"), + ("/atoms/0", "atom"), + ("/mp2", "mp2"), + ("/basis/0", "ao"), + ("/basis/0/components/0", "component"), + ("/basis/0/components/0/primitives/0", "primitive"), + ], + ), + ]; + for (schema, fixture, is_request, records) in cases { + let schema: Value = serde_json::from_str(schema).unwrap(); + let mut value: Value = serde_json::from_str(fixture).unwrap(); + value["mp2"] = if is_request { + json!({"frozen_orbitals":0,"memory_limit":{"kind":"auto"}}) + } else { + json!({"frozen_orbitals":0}) + }; + for (path, definition) in records { + let record_schema = if definition.is_empty() { + &schema + } else { + &schema["$defs"][definition] + }; + let required = record_schema["required"].as_array().unwrap(); + let properties = record_schema["properties"].as_object().unwrap(); + let decode = |candidate: Value| { + if is_request { + serde_json::from_value::(candidate).is_ok() + } else { + serde_json::from_value::(candidate).is_ok() + } + }; + for field in properties.keys() { + let mut candidate = value.clone(); + candidate + .pointer_mut(path) + .unwrap() + .as_object_mut() + .unwrap() + .remove(field); + assert_eq!( + decode(candidate), + !required.contains(&json!(field)), + "{is_request} {definition} {field}" + ); + } + let mut candidate = value.clone(); + candidate + .pointer_mut(path) + .unwrap() + .as_object_mut() + .unwrap() + .insert("unknown_v1_field".into(), json!(0)); + assert!(!decode(candidate), "unknown field in {definition}"); + } + } +} + +#[test] +fn bundle_entries_have_independent_lazy_artifacts_and_shared_sources() { + use crate::persistence::RustiQBundle; + let basis = load_minimal_basis_file(); + let prepared = calculation(); + let changed = CalculationBuilder::new(prepared.get_molecule(), &basis) + .with_integrals(crate::config::IntegralConfig { + schwarz_threshold: None.into(), + }) + .prepare() + .unwrap(); + let mut first = data(); + first + .add_source("batch.ncl", b"shared source".as_slice()) + .unwrap(); + let mut second = RustiQData::from_calculation(&changed).unwrap(); + second + .add_source("batch.ncl", b"shared source".as_slice()) + .unwrap(); + second.set_eri(CompactEri::Zeroed(2)).unwrap(); + let mut bundle = RustiQBundle::new(vec![first, second]).unwrap(); + assert_eq!(bundle.sources().len(), 1); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("batch.rustiq"); + bundle.write(&path).unwrap(); + let contents = members(&path); + let manifest: serde_json::Value = serde_json::from_slice( + &contents + .iter() + .find(|(n, _)| n == "manifest.json") + .unwrap() + .1, + ) + .unwrap(); + assert!(manifest.get("scientific_identity").is_none()); + assert_eq!(manifest["calculations"].as_array().unwrap().len(), 2); + assert_eq!( + contents + .iter() + .filter(|(n, _)| n.starts_with("sources/")) + .count(), + 1 + ); + assert!(RustiQData::open(&path) + .unwrap_err() + .to_string() + .contains("exactly one")); + let mut restored = RustiQBundle::open(&path).unwrap(); + assert!(restored + .calculations() + .iter() + .all(|data| data.ao_eri.is_none())); + assert!(restored.calculations()[0].eri_is_compatible(&prepared)); + assert!(!restored.calculations()[1].eri_is_compatible(&prepared)); + assert!(restored.calculations()[1].eri_is_compatible(&changed)); + assert_eq!( + restored.calculations_mut()[0] + .read_eri() + .unwrap() + .ordered_values()[0], + 0.5 + ); + assert!(restored.calculations()[1].ao_eri.is_none()); + assert_eq!( + restored.calculations_mut()[1] + .read_eri() + .unwrap() + .ordered_values()[0], + 0.0 + ); + let copy = dir.path().join("copy.rustiq"); + restored.write(©).unwrap(); + assert_eq!(fs::read(&path).unwrap(), fs::read(©).unwrap()); +} + +#[test] +fn bundle_rejects_empty_duplicate_ids_cross_entry_and_conflicting_references() { + use crate::persistence::RustiQBundle; + assert!(RustiQBundle::new(vec![]).is_err()); + let dir = tempfile::tempdir().unwrap(); + let good = dir.path().join("good.rustiq"); + RustiQBundle::new(vec![data(), data()]) + .unwrap() + .write(&good) + .unwrap(); + let original = members(&good); + for (field, value) in [ + ("/calculations", serde_json::json!([])), + ("/calculations/1/id", serde_json::json!("calculation-0")), + ("/calculations/1/id", serde_json::json!("../invalid")), + ( + "/calculations/1/request/path", + serde_json::json!(REQUEST_PATH), + ), + ( + "/calculations/0/artifacts/ao_eri/path", + serde_json::json!(CALCULATION_PATH), + ), + ] { + let mut contents = original.clone(); + edit_json(&mut contents, "manifest.json", |manifest| { + *manifest.pointer_mut(field).unwrap() = value; + }); + let bad = dir.path().join("bad.rustiq"); + write_members(&bad, &contents); + assert!(RustiQBundle::open(&bad).is_err(), "{field}"); + } + let mut contents = original; + edit_json(&mut contents, "manifest.json", |manifest| { + manifest["scientific_identity"] = + manifest["calculations"][0]["scientific_identity"].clone(); + }); + let bad = dir.path().join("aggregate.rustiq"); + write_members(&bad, &contents); + assert!(RustiQBundle::open(&bad).is_err()); +} + +#[test] +fn corrupt_bundle_entry_does_not_prevent_independent_artifact_access_or_publish_partial_output() { + use crate::persistence::RustiQBundle; + let dir = tempfile::tempdir().unwrap(); + let good = dir.path().join("good.rustiq"); + RustiQBundle::new(vec![data(), data()]) + .unwrap() + .write(&good) + .unwrap(); + let mut contents = members(&good); + *contents + .iter_mut() + .find(|(n, _)| n == AO_ERI_PATH) + .unwrap() + .1 + .last_mut() + .unwrap() ^= 1; + let bad = dir.path().join("bad.rustiq"); + write_members(&bad, &contents); + let original = fs::read(&bad).unwrap(); + let mut bundle = RustiQBundle::open(&bad).unwrap(); + assert!(bundle.calculations_mut()[0].read_eri().is_err()); + assert_eq!( + bundle.calculations_mut()[1] + .read_eri() + .unwrap() + .ordered_values()[0], + 0.5 + ); + let output = dir.path().join("output.rustiq"); + assert!(bundle.write(&output).is_err()); + assert!(!output.exists()); + assert_eq!(fs::read(&bad).unwrap(), original); + assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 2); +} + +#[test] +fn reads_python_multi_calculation_fixture() { + let fixture = + include_str!("../../../../tests/data/persistence/portable-python-multi-v1.rustiq.hex"); + let bytes: Vec = fixture + .trim() + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("multi.rustiq"); + fs::write(&path, bytes).unwrap(); + let mut bundle = crate::persistence::RustiQBundle::open(&path).unwrap(); + assert_eq!(bundle.sources().len(), 1); + assert_eq!(bundle.calculations().len(), 2); + for data in bundle.calculations_mut() { + assert!(data.ao_eri.is_none()); + assert_eq!( + data.read_eri().unwrap().ordered_values(), + &[0.5, 1.5, 2.5, 3.5, 4.5, 5.5] + ); + } +} + +#[test] +#[ignore = "writes a multi-calculation artifact for Python interoperability"] +fn writes_bundle_for_python_interoperability() { + let path = std::env::var_os("RUSTIQ_ARCHIVE_TEST_OUTPUT").expect("output path"); + let mut bundle = crate::persistence::RustiQBundle::new(vec![data(), data()]).unwrap(); + bundle + .add_source("batch.ncl", b"shared source".as_slice()) + .unwrap(); + bundle.write(path).unwrap(); +} + +#[test] +fn portable_manifest_golden_and_schema_required_fields_match_decoder() { + use crate::persistence::manifest::PortableManifest; + use serde_json::{json, Value}; + let value: Value = serde_json::from_str(include_str!( + "../../../../tests/data/persistence/portable-manifest-h2-v1.json" + )) + .unwrap(); + let decoded: PortableManifest = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(serde_json::to_value(decoded).unwrap(), value); + let schema: Value = serde_json::from_str(include_str!( + "../../../../../../schemas/portable-manifest-v1.schema.json" + )) + .unwrap(); + assert_eq!(schema["properties"]["calculations"]["minItems"], 1); + for (path, definition) in [ + ("", ""), + ("/calculations/0", "calculation"), + ("/calculations/0/request", "snapshot"), + ("/sources/0", "source"), + ] { + let record = if definition.is_empty() { + &schema + } else { + &schema["$defs"][definition] + }; + let required = record["required"].as_array().unwrap(); + for field in record["properties"].as_object().unwrap().keys() { + let mut candidate = value.clone(); + candidate + .pointer_mut(path) + .unwrap() + .as_object_mut() + .unwrap() + .remove(field); + assert_eq!( + serde_json::from_value::(candidate).is_ok(), + !required.contains(&json!(field)), + "{definition} {field}" + ); + } + let mut candidate = value.clone(); + candidate + .pointer_mut(path) + .unwrap() + .as_object_mut() + .unwrap() + .insert("unknown_v1_field".into(), json!(0)); + assert!(serde_json::from_value::(candidate).is_err()); + } +} diff --git a/crates/rustiq-core/src/persistence/data/rustiq_data.rs b/crates/rustiq-core/src/persistence/data/rustiq_data.rs index 6ab7648c..e24ef039 100644 --- a/crates/rustiq-core/src/persistence/data/rustiq_data.rs +++ b/crates/rustiq-core/src/persistence/data/rustiq_data.rs @@ -5,24 +5,27 @@ use relative_path::RelativePath; use crate::eri::CompactEri; use super::super::{ - read_compact_eri, validate_storage_path, write_compact_eri, AoEriAttributes, - ArtifactAttributes, ArtifactError, ArtifactManifest, Manifest, ManifestError, ManifestKind, - PersistenceReadError, PersistenceWriteError, Producer, ScientificIdentityManifest, Storage, - StorageError, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, + validate_storage_path, write_compact_eri, AoEriAttributes, ArtifactAttributes, ArtifactError, + ArtifactManifest, Manifest, ManifestError, ManifestKind, PersistenceReadError, + PersistenceWriteError, Producer, ScientificIdentityManifest, Storage, StorageError, + AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, FORMAT_VERSION, MANIFEST_PATH, }; use super::artifact::Artifact; pub(crate) const AO_ERI_ARTIFACT: &str = "ao_eri"; -const MAX_MANIFEST_BYTES: u64 = 1024 * 1024; +pub(super) const MAX_MANIFEST_BYTES: u64 = 1024 * 1024; /// Known scientific artifacts and their manifest, with values loaded on demand. #[derive(Debug)] pub struct RustiQData { pub(super) manifest: Manifest, - source: Option, + pub(super) source: Option, pub(super) ao_eri: Option, - basis_functions: Option, + pub(super) basis_functions: Option, + pub(super) request: Option, + pub(super) sources: Vec, + pub(super) context: Option, } impl RustiQData { @@ -55,8 +58,14 @@ impl RustiQData { digest: identity.digest, }, artifacts: BTreeMap::new(), + calculation: None, + request: None, + sources: Vec::new(), }, source: None, + context: None, + request: None, + sources: Vec::new(), ao_eri: None, basis_functions: Some(basis_functions), } @@ -65,6 +74,13 @@ impl RustiQData { pub(crate) fn read_from(mut source: Storage) -> Result { let manifest: Manifest = source.read_json(RelativePath::new(MANIFEST_PATH), MAX_MANIFEST_BYTES)?; + Self::read_manifest(source, manifest) + } + + pub(crate) fn read_manifest( + source: Storage, + manifest: Manifest, + ) -> Result { if manifest.format != FORMAT_NAME || manifest.format_version != FORMAT_VERSION { return Err(ManifestError::UnsupportedFormat.into()); } @@ -92,6 +108,9 @@ impl RustiQData { Ok(Self { manifest, source: Some(source), + context: None, + request: None, + sources: Vec::new(), ao_eri: None, basis_functions, }) @@ -103,12 +122,24 @@ impl RustiQData { /// Replaces the AO ERI artifact after checking its compact length. pub fn set_eri(&mut self, eri: CompactEri) -> Result<(), ArtifactError> { - let basis_functions = self.basis_functions.ok_or(ArtifactError::Missing)?; - validate_eri_len(&eri, basis_functions)?; + self.validate_eri(&eri)?; self.ao_eri = Some(eri); Ok(()) } + pub(super) fn validate_eri(&self, eri: &CompactEri) -> Result<(), ArtifactError> { + if self + .context + .as_ref() + .is_some_and(|context| context.0.computation_version() != AO_ERI_COMPUTATION_VERSION) + { + return Err(ArtifactError::UnsupportedRepresentation( + "AO ERI computation version".into(), + )); + } + validate_eri_len(eri, self.basis_functions.ok_or(ArtifactError::Missing)?) + } + /// Validates and decodes the AO ERI on first access, then reuses the object. pub fn read_eri(&mut self) -> Result<&CompactEri, ArtifactError> { if self.ao_eri.is_none() { @@ -132,16 +163,25 @@ impl RustiQData { } let source = self.source.as_mut().ok_or(ArtifactError::Missing)?; + if source.artifact_size(&artifact.path)? != artifact.size { + return Err(ArtifactError::IntegrityMismatch(artifact.path.to_string())); + } let metadata = source.artifact_metadata(&artifact.path)?; if metadata.size != artifact.size || metadata.digest != artifact.digest { return Err(ArtifactError::IntegrityMismatch(artifact.path.to_string())); } - self.ao_eri = Some( - source.with_artifact::<_, ArtifactError, _>(&artifact.path, |reader| { - read_compact_eri(reader, attributes.basis_functions).map_err(Into::into) - })?, - ); + self.ao_eri = Some(source.with_artifact::<_, ArtifactError, _>( + &artifact.path, + |reader| { + super::super::npy::read_checked_compact_eri( + reader, + attributes.basis_functions, + artifact.size, + ) + .map_err(Into::into) + }, + )?); } Ok(self @@ -156,25 +196,100 @@ impl RustiQData { pub(crate) fn write_to(&mut self, destination: Storage) -> Result<(), PersistenceWriteError> { let eri = self.ao_eri.take(); - let result = self.write_inner(destination, eri.as_ref()); + let result = self + .write_inner(destination, eri.as_ref(), true) + .map(|_| ()); self.ao_eri = eri; result } - pub(crate) fn write_with_eri( + pub(crate) fn write_storage_with_eri( &mut self, destination: Storage, eri: &CompactEri, ) -> Result<(), PersistenceWriteError> { - self.write_inner(destination, Some(eri)) + self.write_inner(destination, Some(eri), true).map(|_| ()) + } + + pub(crate) fn write_entry_with_eri( + &mut self, + destination: Storage, + eri: &CompactEri, + ) -> Result { + self.write_inner(destination, Some(eri), false) + } + + pub(crate) fn write_entry( + &mut self, + destination: Storage, + ) -> Result { + let eri = self.ao_eri.take(); + let result = self.write_inner(destination, eri.as_ref(), false); + self.ao_eri = eri; + result } fn write_inner( &mut self, mut destination: Storage, eri: Option<&CompactEri>, - ) -> Result<(), PersistenceWriteError> { + publish_manifest: bool, + ) -> Result { let mut manifest = self.manifest.clone(); + if manifest.kind == ManifestKind::Portable { + manifest.producer = Producer { + name: "RustiQ".into(), + version: env!("CARGO_PKG_VERSION").into(), + }; + } + + if let Some(context) = &self.context { + let metadata = destination.write_json( + RelativePath::new(super::super::calculation::CALCULATION_PATH), + &context.0, + )?; + manifest.calculation = Some(super::super::manifest::SnapshotManifest { + path: super::super::calculation::CALCULATION_PATH.into(), + version: 1, + size: metadata.size, + digest: metadata.digest, + }); + } + + if let Some(request) = &self.request { + let metadata = destination.write_json( + RelativePath::new(super::super::request::REQUEST_PATH), + &super::super::request::RequestSnapshot::from_request(request) + .map_err(|error| ArtifactError::InvalidMetadata(error.to_string()))?, + )?; + manifest.request = Some(super::super::manifest::SnapshotManifest { + path: super::super::request::REQUEST_PATH.into(), + version: 1, + size: metadata.size, + digest: metadata.digest, + }); + } + manifest.sources.clear(); + for (index, source) in self.sources.iter().enumerate() { + let path = format!("sources/{index}"); + let metadata = destination.write_artifact::( + RelativePath::new(&path), + |writer| { + std::io::Write::write_all(writer, source.bytes()) + .map_err(StorageError::from)?; + Ok(()) + }, + )?; + manifest + .sources + .push(super::super::manifest::SourceManifest { + original_name: source.original_name().into(), + path, + version: 1, + size: metadata.size, + digest: metadata.digest, + }); + } for (name, artifact) in &self.manifest.artifacts { if name == AO_ERI_ARTIFACT && eri.is_some() { @@ -186,6 +301,9 @@ impl RustiQData { })?; validate_artifact_path(&artifact.path)?; + if source.artifact_size(&artifact.path)? != artifact.size { + return Err(ArtifactError::IntegrityMismatch(artifact.path.to_string()).into()); + } let metadata = source.with_artifact::<_, PersistenceWriteError, _>(&artifact.path, |input| { destination.write_artifact::( @@ -206,7 +324,7 @@ impl RustiQData { if let Some(eri) = eri { let basis_functions = self.basis_functions.ok_or(ArtifactError::Missing)?; - validate_eri_len(eri, basis_functions)?; + self.validate_eri(eri)?; let path = RelativePath::new(AO_ERI_PATH); let metadata = destination .write_artifact::(path, |writer| { @@ -228,9 +346,11 @@ impl RustiQData { ); } - destination.write_json(RelativePath::new(MANIFEST_PATH), &manifest)?; + if publish_manifest { + destination.write_json(RelativePath::new(MANIFEST_PATH), &manifest)?; + } destination.finish()?; - Ok(()) + Ok(manifest) } } diff --git a/crates/rustiq-core/src/persistence/eri_cache.rs b/crates/rustiq-core/src/persistence/eri_cache.rs index 771018c8..debbb4a3 100644 --- a/crates/rustiq-core/src/persistence/eri_cache.rs +++ b/crates/rustiq-core/src/persistence/eri_cache.rs @@ -341,7 +341,7 @@ impl EriCache { let mut entry_data = RustiQData::new_with_identity(identity, basis_functions, ManifestKind::IntegralCache); entry_data - .write_with_eri(Storage::folder(temporary.path()), eri) + .write_storage_with_eri(Storage::folder(temporary.path()), eri) .map_err(io::Error::other)?; let temporary_path = temporary.keep(); if fs::symlink_metadata(&final_entry).is_ok() { diff --git a/crates/rustiq-core/src/persistence/error.rs b/crates/rustiq-core/src/persistence/error.rs index 02e115c0..f48e40c6 100644 --- a/crates/rustiq-core/src/persistence/error.rs +++ b/crates/rustiq-core/src/persistence/error.rs @@ -4,6 +4,8 @@ use thiserror::Error; #[derive(Debug, Error)] pub(crate) enum StorageError { + #[error("invalid portable archive: {0}")] + Archive(String), #[error("storage I/O failed: {0}")] Io(#[from] io::Error), #[error("unsafe storage path: {0}")] @@ -12,6 +14,55 @@ pub(crate) enum StorageError { UnexpectedEntryType(String), } +/// Portable persistence failures, independent of the physical container library. +#[derive(Debug, Error)] +pub enum PortableError { + #[error("AO ERI is incompatible with the prepared calculation")] + IncompatibleEri, + #[error("portable artifact I/O failed: {0}")] + Io(#[from] io::Error), + #[error("the destination already exists")] + AlreadyExists, + #[error("invalid portable artifact: {0}")] + InvalidArchive(String), + #[error("unsupported portable calculation format or version")] + UnsupportedVersion, + #[error("invalid resolved calculation: {0}")] + InvalidCalculation(String), + #[error("invalid normalized request: {0}")] + InvalidRequest(String), + #[error(transparent)] + Artifact(#[from] ArtifactError), +} + +impl From for PortableError { + fn from(error: StorageError) -> Self { + match error { + StorageError::Io(error) => Self::Io(error), + other => Self::InvalidArchive(other.to_string()), + } + } +} + +impl From for PortableError { + fn from(error: PersistenceReadError) -> Self { + match error { + PersistenceReadError::Artifact(error) => Self::Artifact(error), + other => Self::InvalidArchive(other.to_string()), + } + } +} + +impl From for PortableError { + fn from(error: PersistenceWriteError) -> Self { + match error { + PersistenceWriteError::Artifact(error) => Self::Artifact(error), + PersistenceWriteError::Storage(error) => error.into(), + other => Self::InvalidArchive(other.to_string()), + } + } +} + #[derive(Debug, Error)] pub enum NpyError { #[error("could not read NPY data: {0}")] diff --git a/crates/rustiq-core/src/persistence/identity.rs b/crates/rustiq-core/src/persistence/identity.rs index 94c5e439..c84b5569 100644 --- a/crates/rustiq-core/src/persistence/identity.rs +++ b/crates/rustiq-core/src/persistence/identity.rs @@ -35,11 +35,7 @@ fn ao_eri_identity_with_computation_version( schwarz_threshold: Option, computation_version: u32, ) -> ScientificIdentity { - let mut bytes = CanonicalBytes::default(); - bytes.text(b"scientific-identity-v1"); - bytes.text(b"ao-eri-computation-version"); - bytes.u32(computation_version); - bytes.text(COMPACT_ERI_REPRESENTATION.as_bytes()); + let mut bytes = CanonicalBytes::ao_eri(computation_version); bytes.len(geometry.atoms.len()); for atom in &geometry.atoms { @@ -70,41 +66,49 @@ fn ao_eri_identity_with_computation_version( } } - match schwarz_threshold { - None => bytes.u8(0), - Some(threshold) => { - bytes.u8(1); - bytes.f64(threshold.into_inner()); - } - } - - ScientificIdentity { - version: SCIENTIFIC_IDENTITY_VERSION, - digest: sha256(&bytes.0), - } + bytes.finish_ao_eri(schwarz_threshold.map(|value| value.into_inner())) } #[derive(Default)] -struct CanonicalBytes(Vec); +pub(crate) struct CanonicalBytes(Vec); impl CanonicalBytes { - fn u8(&mut self, value: u8) { - self.0.push(value); + pub(crate) fn ao_eri(computation_version: u32) -> Self { + let mut bytes = Self::default(); + bytes.text(b"scientific-identity-v1"); + bytes.text(b"ao-eri-computation-version"); + bytes.u32(computation_version); + bytes.text(COMPACT_ERI_REPRESENTATION.as_bytes()); + bytes + } + + pub(crate) fn finish_ao_eri(mut self, threshold: Option) -> ScientificIdentity { + match threshold { + None => self.u8(0), + Some(value) => { + self.u8(1); + self.f64(value); + } + } + ScientificIdentity { + version: SCIENTIFIC_IDENTITY_VERSION, + digest: sha256(&self.0), + } } - fn u32(&mut self, value: u32) { + pub(crate) fn u8(&mut self, value: u8) { + self.0.push(value); + } + pub(crate) fn u32(&mut self, value: u32) { self.0.extend_from_slice(&value.to_be_bytes()); } - - fn len(&mut self, value: usize) { + pub(crate) fn len(&mut self, value: usize) { self.0.extend_from_slice(&(value as u64).to_be_bytes()); } - - fn f64(&mut self, value: f64) { + pub(crate) fn f64(&mut self, value: f64) { let canonical = if value == 0.0 { 0.0 } else { value }; self.0.extend_from_slice(&canonical.to_bits().to_be_bytes()); } - fn text(&mut self, value: &[u8]) { self.len(value.len()); self.0.extend_from_slice(value); diff --git a/crates/rustiq-core/src/persistence/manifest.rs b/crates/rustiq-core/src/persistence/manifest.rs index 4a0ca36b..514f1890 100644 --- a/crates/rustiq-core/src/persistence/manifest.rs +++ b/crates/rustiq-core/src/persistence/manifest.rs @@ -9,6 +9,7 @@ use super::{Sha256Digest, COMPACT_ERI_REPRESENTATION}; #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum ManifestKind { IntegralCache, + Portable, Unknown(String), } @@ -16,6 +17,7 @@ impl ManifestKind { pub(crate) fn as_str(&self) -> &str { match self { Self::IntegralCache => "integral-cache", + Self::Portable => "portable", Self::Unknown(value) => value, } } @@ -38,6 +40,7 @@ impl<'de> Deserialize<'de> for ManifestKind { let value = String::deserialize(deserializer)?; Ok(match value.as_str() { "integral-cache" => Self::IntegralCache, + "portable" => Self::Portable, _ => Self::Unknown(value), }) } @@ -51,6 +54,31 @@ pub(crate) struct Manifest { pub producer: Producer, pub scientific_identity: ScientificIdentityManifest, pub artifacts: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub calculation: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub sources: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SourceManifest { + pub original_name: String, + pub path: String, + pub version: u32, + pub size: u64, + pub digest: Sha256Digest, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SnapshotManifest { + pub path: String, + pub version: u32, + pub size: u64, + pub digest: Sha256Digest, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -76,13 +104,40 @@ pub(crate) enum ArtifactAttributes { Unknown(BTreeMap), } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] +#[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct AoEriAttributes { pub basis_functions: usize, pub computation_version: u32, } +// Keep the folder-cache/domain API while defining a pointer-width-independent wire record. +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct AoEriAttributesV1 { + basis_functions: u64, + computation_version: u32, +} +impl Serialize for AoEriAttributes { + fn serialize(&self, serializer: S) -> Result { + AoEriAttributesV1 { + basis_functions: u64::try_from(self.basis_functions) + .map_err(serde::ser::Error::custom)?, + computation_version: self.computation_version, + } + .serialize(serializer) + } +} +impl<'de> Deserialize<'de> for AoEriAttributes { + fn deserialize>(deserializer: D) -> Result { + let wire = AoEriAttributesV1::deserialize(deserializer)?; + Ok(Self { + basis_functions: usize::try_from(wire.basis_functions) + .map_err(serde::de::Error::custom)?, + computation_version: wire.computation_version, + }) + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub(crate) struct ArtifactManifest { pub path: RelativePathBuf, @@ -161,6 +216,9 @@ mod tests { ); let manifest = Manifest { + calculation: None, + request: None, + sources: Vec::new(), format: FORMAT_NAME.to_string(), format_version: FORMAT_VERSION, kind: ManifestKind::IntegralCache, @@ -311,3 +369,50 @@ mod tests { assert!(serde_json::from_str::(json).is_err()); } } + +#[cfg(test)] +mod wire_tests { + use super::*; + + #[test] + fn ao_dimension_wire_is_u64_with_checked_host_conversion() { + let encoded = format!( + r#"{{"basis_functions":{},"computation_version":1}}"#, + u64::MAX + ); + let result = serde_json::from_str::(&encoded); + assert_eq!(result.is_ok(), usize::try_from(u64::MAX).is_ok()); + if let Ok(attributes) = result { + assert_eq!(serde_json::to_string(&attributes).unwrap(), encoded); + } + for invalid in ["-1", "18446744073709551616"] { + assert!(serde_json::from_str::( + &encoded.replace("18446744073709551615", invalid) + ) + .is_err()); + } + } +} + +/// Portable wire format is independent of the single-artifact directory cache. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct PortableManifest { + pub format: String, + pub format_version: u32, + pub kind: ManifestKind, + pub producer: Producer, + #[serde(default)] + pub sources: Vec, + pub calculations: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct PortableCalculation { + pub id: String, + pub scientific_identity: ScientificIdentityManifest, + pub request: SnapshotManifest, + pub calculation: SnapshotManifest, + pub artifacts: BTreeMap, +} diff --git a/crates/rustiq-core/src/persistence/mod.rs b/crates/rustiq-core/src/persistence/mod.rs index 1a5bad46..203edb2c 100644 --- a/crates/rustiq-core/src/persistence/mod.rs +++ b/crates/rustiq-core/src/persistence/mod.rs @@ -5,6 +5,11 @@ //! location policy remains outside `rustiq-core`. mod cache_names; +mod calculation; +mod provenance; +mod request; +pub use calculation::{CalculationContext, ResolvedAo, ResolvedComponent}; +pub use provenance::SourceProvenance; mod checksum; mod data; mod eri_cache; @@ -16,9 +21,9 @@ mod storage; pub use crate::eri::CompactEri; pub use checksum::{sha256, sha256_reader, verify_sha256, Sha256Digest, Sha256DigestParseError}; -pub use data::{AoEriArtifact, Artifact, RustiQData}; +pub use data::{AoEriArtifact, Artifact, RustiQBundle, RustiQData}; pub use eri_cache::{EriCache, EriCacheEntry}; -pub use error::{ArtifactError, NpyError}; +pub use error::{ArtifactError, NpyError, PortableError}; pub(crate) use error::{ManifestError, PersistenceReadError, PersistenceWriteError, StorageError}; #[allow(unused_imports)] pub(crate) use identity::{ao_eri_identity, ScientificIdentity, AO_ERI_COMPUTATION_VERSION}; diff --git a/crates/rustiq-core/src/persistence/npy.rs b/crates/rustiq-core/src/persistence/npy.rs index 70014189..9d1fbd51 100644 --- a/crates/rustiq-core/src/persistence/npy.rs +++ b/crates/rustiq-core/src/persistence/npy.rs @@ -57,7 +57,7 @@ impl NpyConvert for CompactEri { } fn write_npy(&self, writer: impl Write) -> Result<(), NpyError> { - let shape = [self.len() as u64]; + let shape = [u64::try_from(self.len()).map_err(|_| NpyError::DimensionOverflow)?]; let mut writer = npyz::WriteOptions::new() .default_dtype() .shape(&shape) @@ -89,7 +89,7 @@ impl NpyConvert for DMatrix { } fn write_npy(&self, writer: impl Write) -> Result<(), NpyError> { - let shape = [self.nrows() as u64, self.ncols() as u64]; + let shape = Self::npy_shape((self.nrows(), self.ncols()))?; let mut writer = npyz::WriteOptions::new() .default_dtype() .order(npyz::Order::Fortran) @@ -115,6 +115,58 @@ pub(crate) fn read_compact_eri( CompactEri::try_read_with_shape(reader, basis_functions) } +/// Validates the small NPY header and exact payload length before allocating values. +pub(crate) fn checked_eri_prefix( + reader: &mut dyn Read, + basis_functions: usize, + file_size: u64, +) -> Result, NpyError> { + let mut prefix = [0_u8; 8]; + reader.read_exact(&mut prefix).map_err(NpyError::Read)?; + if &prefix[..6] != b"\x93NUMPY" || !matches!((prefix[6], prefix[7]), (1..=3, 0)) { + return Err(NpyError::Read(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "invalid NPY prefix", + ))); + } + let length_bytes = if prefix[6] == 1 { 2 } else { 4 }; + let mut length = [0_u8; 4]; + reader + .read_exact(&mut length[..length_bytes]) + .map_err(NpyError::Read)?; + let header_len = + usize::try_from(u32::from_le_bytes(length)).map_err(|_| NpyError::DimensionOverflow)?; + if header_len > 65536 { + return Err(NpyError::Read(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "NPY header exceeds 64 KiB", + ))); + } + let mut header = Vec::with_capacity(8 + length_bytes + header_len); + header.extend_from_slice(&prefix); + header.extend_from_slice(&length[..length_bytes]); + header.resize(8 + length_bytes + header_len, 0); + reader + .read_exact(&mut header[8 + length_bytes..]) + .map_err(NpyError::Read)?; + if !validate_compact_eri_header(std::io::Cursor::new(&header), basis_functions, file_size) { + return Err(NpyError::Read(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "NPY dtype, shape or payload length is invalid", + ))); + } + Ok(header) +} + +pub(crate) fn read_checked_compact_eri( + reader: &mut dyn Read, + basis_functions: usize, + file_size: u64, +) -> Result { + let header = checked_eri_prefix(reader, basis_functions, file_size)?; + read_compact_eri(std::io::Cursor::new(header).chain(reader), basis_functions) +} + fn decode_compact_eri( npy: NpyFile, basis_functions: usize, diff --git a/crates/rustiq-core/src/persistence/provenance.rs b/crates/rustiq-core/src/persistence/provenance.rs new file mode 100644 index 00000000..67e78c8b --- /dev/null +++ b/crates/rustiq-core/src/persistence/provenance.rs @@ -0,0 +1,21 @@ +//! Optional opaque provenance, kept outside the scientific request and identity. +pub(crate) const MAX_SOURCE_BYTES: u64 = 64 * 1024 * 1024; +pub(crate) const MAX_SOURCES: usize = 256; +pub(crate) const MAX_SOURCE_NAME_BYTES: usize = 4096; + +/// Exact original source bytes with an informational original name. +/// The name is never an archive member name or an extraction instruction. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceProvenance { + pub(crate) original_name: String, + pub(crate) bytes: Vec, +} + +impl SourceProvenance { + pub fn original_name(&self) -> &str { + &self.original_name + } + pub fn bytes(&self) -> &[u8] { + &self.bytes + } +} diff --git a/crates/rustiq-core/src/persistence/request.rs b/crates/rustiq-core/src/persistence/request.rs new file mode 100644 index 00000000..bc4ab6a1 --- /dev/null +++ b/crates/rustiq-core/src/persistence/request.rs @@ -0,0 +1,340 @@ +//! Explicit normalized-request V1 wire records, independent of frontend syntax. +use std::num::{NonZeroU8, NonZeroUsize}; + +use serde::{Deserialize, Serialize}; + +use super::{ + calculation::{resolve_hf_method_v1, Guess}, + PortableError, +}; +use crate::{ + calculation::CalculationRequest, + config::{ + validated::{DiisSize, NonNegativeFiniteF64, PositiveFiniteF64}, + DiisConfig, HfConfig, HfMethod, IntegralConfig, MemoryLimit, MoleculeConfig, Mp2Config, + OrthogonalizationConfig, + }, + molecules::{atom::Atom, geometry::Geometry, units::Units}, +}; + +pub(crate) const REQUEST_PATH: &str = "request.json"; +pub(crate) const MAX_REQUEST_BYTES: u64 = 64 * 1024 * 1024; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RequestSnapshot { + format: String, + version: u32, + units: CoordinateUnits, + atoms: Vec, + charge: i32, + multiplicity: u8, + basis: String, + hf: RequestedHf, + mp2: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +enum CoordinateUnits { + Bohr, + Angstrom, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RequestAtom { + atomic_number: u32, + position: [f64; 3], +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +enum RequestedMethod { + Auto, + Rhf, + Uhf, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RequestedHf { + method: RequestedMethod, + max_iterations: u64, + convergence_threshold: f64, + linear_dependency_threshold: f64, + eri_schwarz_threshold: Option, + diis: bool, + diis_size: u64, + guess: Guess, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RequestedMp2 { + frozen_orbitals: u64, + memory_limit: RequestedMemory, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum RequestedMemory { + Auto, + Fixed { bytes: u64 }, +} + +impl RequestSnapshot { + pub(crate) fn from_request(request: &CalculationRequest) -> Result { + let invalid = || PortableError::InvalidRequest("integer exceeds V1 u64 range".into()); + let hf = request.hf(); + Ok(Self { + format: "rustiq-request".into(), + version: 1, + units: match request.molecule().units { + Units::Bohr => CoordinateUnits::Bohr, + Units::Angstrom => CoordinateUnits::Angstrom, + }, + atoms: request + .geometry() + .atoms + .iter() + .map(|atom| RequestAtom { + atomic_number: atom.element.atomic_number, + position: atom.position.into(), + }) + .collect(), + charge: request.molecule().charge.value, + multiplicity: request.molecule().multiplicity.value.get(), + basis: request.basis_name().into(), + hf: RequestedHf { + method: match hf.method.value { + HfMethod::Auto => RequestedMethod::Auto, + HfMethod::Rhf => RequestedMethod::Rhf, + HfMethod::Uhf => RequestedMethod::Uhf, + }, + max_iterations: u64::try_from(hf.max_iterations.get()).map_err(|_| invalid())?, + convergence_threshold: hf.convergence_threshold.into_inner(), + linear_dependency_threshold: hf + .orthogonalization + .linear_dependency_threshold + .value + .into_inner(), + eri_schwarz_threshold: request + .integrals() + .schwarz_threshold + .value + .map(|v| v.into_inner()), + diis: hf.diis.enabled, + diis_size: u64::try_from(hf.diis.max_history.value.into_inner()) + .map_err(|_| invalid())?, + guess: hf.guess.value.into(), + }, + mp2: request + .mp2() + .map(|mp2| { + Ok::<_, PortableError>(RequestedMp2 { + frozen_orbitals: u64::try_from(mp2.frozen_orbitals.value) + .map_err(|_| invalid())?, + memory_limit: match mp2.memory_limit.value { + MemoryLimit::Auto => RequestedMemory::Auto, + MemoryLimit::Fixed(bytes) => RequestedMemory::Fixed { + bytes: bytes.as_u64(), + }, + }, + }) + }) + .transpose()?, + }) + } + + pub(crate) fn to_request(&self) -> Result { + if self.format != "rustiq-request" || self.version != 1 { + return Err(PortableError::UnsupportedVersion); + } + let invalid = || PortableError::InvalidRequest("invalid normalized request".into()); + if self.atoms.is_empty() { + return Err(invalid()); + } + let atoms = self + .atoms + .iter() + .map(|atom| { + let element = periodic_table::periodic_table() + .iter() + .find(|e| e.atomic_number == atom.atomic_number) + .copied() + .ok_or_else(invalid)?; + if !atom.position.iter().all(|v| v.is_finite()) { + return Err(invalid()); + } + Ok(Atom::new(element, atom.position.into())) + }) + .collect::, PortableError>>()?; + self.hf + .guess + .validate() + .map_err(|error| PortableError::InvalidRequest(error.to_string()))?; + let hf = HfConfig { + method: match self.hf.method { + RequestedMethod::Auto => HfMethod::Auto, + RequestedMethod::Rhf => HfMethod::Rhf, + RequestedMethod::Uhf => HfMethod::Uhf, + } + .into(), + max_iterations: NonZeroUsize::new( + usize::try_from(self.hf.max_iterations).map_err(|_| invalid())?, + ) + .ok_or_else(invalid)?, + convergence_threshold: PositiveFiniteF64::try_new(self.hf.convergence_threshold) + .map_err(|_| invalid())?, + orthogonalization: OrthogonalizationConfig { + linear_dependency_threshold: NonNegativeFiniteF64::try_new( + self.hf.linear_dependency_threshold, + ) + .map_err(|_| invalid())? + .into(), + }, + guess: self.hf.guess.to_config().into(), + diis: DiisConfig { + enabled: self.hf.diis, + max_history: DiisSize::try_new( + usize::try_from(self.hf.diis_size).map_err(|_| invalid())?, + ) + .map_err(|_| invalid())? + .into(), + }, + }; + let request = CalculationRequest { + geometry: Geometry::new(String::new(), atoms), + molecule: MoleculeConfig { + units: match self.units { + CoordinateUnits::Bohr => Units::Bohr, + CoordinateUnits::Angstrom => Units::Angstrom, + }, + charge: self.charge.into(), + multiplicity: NonZeroU8::new(self.multiplicity) + .ok_or_else(invalid)? + .into(), + }, + basis_name: self.basis.clone(), + hf, + integrals: IntegralConfig { + schwarz_threshold: self + .hf + .eri_schwarz_threshold + .map(PositiveFiniteF64::try_new) + .transpose() + .map_err(|_| invalid())? + .into(), + }, + mp2: self + .mp2 + .as_ref() + .map(|mp2| { + Ok::<_, PortableError>(Mp2Config { + frozen_orbitals: usize::try_from(mp2.frozen_orbitals) + .map_err(|_| invalid())? + .into(), + memory_limit: match mp2.memory_limit { + RequestedMemory::Auto => MemoryLimit::Auto, + RequestedMemory::Fixed { bytes } => { + MemoryLimit::Fixed(bytesize::ByteSize(bytes)) + } + } + .into(), + }) + }) + .transpose()?, + }; + let molecule = request + .molecule() + .build(request.geometry().clone()) + .map_err(|error| PortableError::InvalidRequest(error.to_string()))?; + resolve_hf_method_v1(request.hf().method.value, &molecule).ok_or_else(|| { + PortableError::InvalidRequest("invalid V1 HF method resolution".into()) + })?; + Ok(request) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::{json, Value}; + + fn golden() -> Value { + serde_json::from_str(include_str!( + "../../tests/data/persistence/request-h2-v1.json" + )) + .unwrap() + } + + #[test] + fn schema_and_decoder_require_diis_size_even_when_disabled() { + let schema: Value = serde_json::from_str(include_str!( + "../../../../schemas/request-snapshot-v1.schema.json" + )) + .unwrap(); + assert!(schema["$defs"]["hf"]["required"] + .as_array() + .unwrap() + .contains(&json!("diis_size"))); + for enabled in [false, true] { + let mut value = golden(); + value["hf"]["diis"] = json!(enabled); + value["hf"]["diis_size"] = json!(17); + let snapshot: RequestSnapshot = serde_json::from_value(value.clone()).unwrap(); + let request = snapshot.to_request().unwrap(); + assert_eq!(request.hf().diis.enabled, enabled); + assert_eq!(request.hf().diis.max_history.value.into_inner(), 17); + assert_eq!( + serde_json::to_value(RequestSnapshot::from_request(&request).unwrap()).unwrap() + ["hf"]["diis_size"], + json!(17) + ); + value["hf"].as_object_mut().unwrap().remove("diis_size"); + assert!(serde_json::from_value::(value).is_err()); + } + } + + #[test] + fn unsigned_wire_limits_and_checked_domain_conversions() { + let schema: Value = serde_json::from_str(include_str!( + "../../../../schemas/request-snapshot-v1.schema.json" + )) + .unwrap(); + for (definition, field) in [ + ("hf", "max_iterations"), + ("hf", "diis_size"), + ("mp2", "frozen_orbitals"), + ] { + assert_eq!( + schema["$defs"][definition]["properties"][field]["maximum"], + json!(u64::MAX) + ); + let mut value = golden(); + value["mp2"] = json!({"frozen_orbitals":0,"memory_limit":{"kind":"auto"}}); + value[definition][field] = json!(u64::MAX); + let snapshot: RequestSnapshot = serde_json::from_value(value.clone()).unwrap(); + let decoded = snapshot.to_request(); + assert_eq!(decoded.is_ok(), usize::try_from(u64::MAX).is_ok()); + if let Ok(request) = decoded { + assert_eq!( + serde_json::to_value(RequestSnapshot::from_request(&request).unwrap()).unwrap() + [definition][field], + json!(u64::MAX) + ); + } + for invalid in ["-1", "18446744073709551616"] { + let encoded = serde_json::to_string(&value) + .unwrap() + .replace("18446744073709551615", invalid); + assert!(serde_json::from_str::(&encoded).is_err()); + } + // A 32-bit reader must reject this valid V1 integer, not truncate it. + value[definition][field] = json!(u64::from(u32::MAX) + 1); + let snapshot: RequestSnapshot = serde_json::from_value(value).unwrap(); + assert_eq!(snapshot.to_request().is_ok(), usize::BITS > 32); + } + } +} diff --git a/crates/rustiq-core/src/persistence/storage.rs b/crates/rustiq-core/src/persistence/storage.rs index 7633beb9..c5438423 100644 --- a/crates/rustiq-core/src/persistence/storage.rs +++ b/crates/rustiq-core/src/persistence/storage.rs @@ -10,9 +10,17 @@ use sha2::{Digest, Sha256}; use super::{sha256_reader, ManifestError, Sha256Digest, StorageError}; +mod zip; + #[derive(Debug)] pub(crate) enum Storage { Folder(PathBuf), + Scoped { + storage: std::sync::Arc>, + prefix: String, + }, + ZipReader(Box<::zip::ZipArchive>), + ZipWriter(Box<::zip::ZipWriter>>), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -66,6 +74,34 @@ impl Write for DigestWriter { } impl Storage { + pub(crate) fn scoped(storage: std::sync::Arc>, prefix: String) -> Self { + Self::Scoped { storage, prefix } + } + + pub(crate) fn open_zip(path: &std::path::Path) -> Result { + Ok(Self::ZipReader(Box::new(zip::open(path)?))) + } + + pub(crate) fn create_zip(file: File) -> Self { + Self::ZipWriter(Box::new(zip::writer(file))) + } + + pub(crate) fn artifact_size(&mut self, path: &RelativePath) -> Result { + validate_path(path)?; + match self { + Self::Scoped { storage, prefix } => storage + .lock() + .expect("storage mutex poisoned") + .artifact_size(&prefixed_path(prefix, path)?), + Self::Folder(root) => FolderStorage { root: root.clone() }.artifact_size(path), + Self::ZipReader(archive) => Ok(archive + .by_name(path.as_str()) + .map_err(zip::zip_error)? + .size()), + Self::ZipWriter(_) => Err(zip::invalid("cannot read an archive writer")), + } + } + pub(crate) fn folder(root: impl Into) -> Self { Self::Folder(root.into()) } @@ -75,7 +111,22 @@ impl Storage { path: &RelativePath, ) -> Result { match self { + Self::Scoped { storage, prefix } => storage + .lock() + .expect("storage mutex poisoned") + .artifact_metadata(&prefixed_path(prefix, path)?), Self::Folder(root) => FolderStorage { root: root.clone() }.artifact_metadata(path), + Self::ZipReader(archive) => { + let size = archive + .by_name(path.as_str()) + .map_err(zip::zip_error)? + .size(); + let digest = zip::with_member::<_, StorageError, _>(archive, path, |reader| { + Ok(sha256_reader(reader)?) + })?; + Ok(ArtifactMetadata { size, digest }) + } + Self::ZipWriter(_) => Err(zip::invalid("cannot read an archive writer")), } } @@ -85,7 +136,13 @@ impl Storage { F: FnOnce(&mut dyn Read) -> Result, { match self { + Self::Scoped { storage, prefix } => storage + .lock() + .expect("storage mutex poisoned") + .with_artifact(&prefixed_path(prefix, path).map_err(E::from)?, read), Self::Folder(root) => FolderStorage { root: root.clone() }.with_artifact(path, read), + Self::ZipReader(archive) => zip::with_member(archive, path, read), + Self::ZipWriter(_) => Err(E::from(zip::invalid("cannot read an archive writer"))), } } @@ -99,7 +156,23 @@ impl Storage { F: FnOnce(&mut dyn Write) -> Result<(), E>, { match self { + Self::Scoped { storage, prefix } => storage + .lock() + .expect("storage mutex poisoned") + .write_artifact(&prefixed_path(prefix, path).map_err(E::from)?, write), Self::Folder(root) => FolderStorage { root: root.clone() }.write_artifact(path, write), + Self::ZipWriter(archive) => { + validate_path(path).map_err(E::from)?; + archive + .start_file(path.as_str(), zip::options(path)) + .map_err(zip::zip_error) + .map_err(E::from)?; + let mut writer = DigestWriter::new(archive.as_mut()); + write(&mut writer)?; + let (_, metadata) = writer.finish(); + Ok(metadata) + } + Self::ZipReader(_) => Err(E::from(zip::invalid("cannot write an archive reader"))), } } @@ -108,15 +181,21 @@ impl Storage { path: &RelativePath, max_size: u64, ) -> Result { - let size = match self { - Self::Folder(root) => FolderStorage { root: root.clone() }.artifact_size(path)?, - }; + let size = self.artifact_size(path)?; if size > max_size { return Err(ManifestError::TooLarge); } self.with_artifact(path, |reader| { - serde_json::from_reader(reader).map_err(ManifestError::from) + let mut bytes = Vec::new(); + reader + .take(max_size + 1) + .read_to_end(&mut bytes) + .map_err(StorageError::from)?; + if u64::try_from(bytes.len()).map_err(|_| ManifestError::TooLarge)? > max_size { + return Err(ManifestError::TooLarge); + } + serde_json::from_slice(&bytes).map_err(ManifestError::from) }) } @@ -124,22 +203,46 @@ impl Storage { &mut self, path: &RelativePath, value: &T, - ) -> Result<(), ManifestError> { + ) -> Result { + let mut bytes = serde_json::to_vec_pretty(value)?; + bytes.push(b'\n'); + let limit = match path.file_name().unwrap_or_default() { + super::calculation::CALCULATION_PATH => super::calculation::MAX_CALCULATION_BYTES, + super::request::REQUEST_PATH => super::request::MAX_REQUEST_BYTES, + _ => 1024 * 1024, + }; + if u64::try_from(bytes.len()).map_err(|_| ManifestError::TooLarge)? > limit { + return Err(ManifestError::TooLarge); + } self.write_artifact::(path, |writer| { - serde_json::to_writer_pretty(&mut *writer, value)?; - writer.write_all(b"\n").map_err(StorageError::from)?; + writer.write_all(&bytes).map_err(StorageError::from)?; Ok(()) - })?; - Ok(()) + }) } pub(crate) fn finish(self) -> Result<(), StorageError> { match self { - Self::Folder(_) => Ok(()), + Self::Folder(_) | Self::ZipReader(_) | Self::Scoped { .. } => Ok(()), + Self::ZipWriter(archive) => { + let mut file = archive.finish().map_err(zip::zip_error)?; + file.flush()?; + file.get_ref().sync_all()?; + Ok(()) + } } } } +fn prefixed_path( + prefix: &str, + path: &RelativePath, +) -> Result { + validate_path(path)?; + let full = relative_path::RelativePathBuf::from(format!("{prefix}/{}", path.as_str())); + validate_path(&full)?; + Ok(full) +} + #[derive(Debug)] struct FolderStorage { root: PathBuf, diff --git a/crates/rustiq-core/src/persistence/storage/zip.rs b/crates/rustiq-core/src/persistence/storage/zip.rs new file mode 100644 index 00000000..a5c3e8b5 --- /dev/null +++ b/crates/rustiq-core/src/persistence/storage/zip.rs @@ -0,0 +1,349 @@ +//! Bounded ZIP indexing before handing the archive to the container library. +//! `zip` indexes by name and can hide duplicate central-directory records, so +//! validate the original records before constructing its index. +use super::{validate_path, StorageError}; +use relative_path::RelativePath; +use std::{ + collections::BTreeSet, + fs::File, + io::{self, BufReader, BufWriter, Read, Seek, SeekFrom}, + path::Path, +}; +use zip::{write::SimpleFileOptions, CompressionMethod, ZipArchive, ZipWriter}; + +pub(super) const MAX_MEMBERS: usize = 4096; +const MAX_DIRECTORY_BYTES: u64 = 16 * 1024 * 1024; +const MAX_END_RECORD_BYTES: u64 = 65536; + +pub(super) fn invalid(message: impl Into) -> StorageError { + StorageError::Archive(message.into()) +} +pub(super) fn zip_error(error: zip::result::ZipError) -> StorageError { + invalid(error.to_string()) +} + +pub(super) fn open(path: &Path) -> Result, StorageError> { + let mut file = File::open(path)?; + let (count, directory_start) = preflight(&mut file)?; + file.rewind()?; + let mut archive = ZipArchive::new(file).map_err(zip_error)?; + if archive.len() != count + || archive.central_directory_start() != directory_start + || archive.offset() != 0 + { + return Err(invalid("inconsistent archive directory")); + } + if archive.has_overlapping_files().map_err(zip_error)? { + return Err(invalid("overlapping ZIP members")); + } + for index in 0..archive.len() { + let member = archive.by_index_raw(index).map_err(zip_error)?; + if member + .data_start() + .and_then(|start| start.checked_add(member.compressed_size())) + .is_none_or(|end| end > directory_start) + { + return Err(invalid("member extends into the central directory")); + } + } + Ok(archive) +} + +pub(super) fn options(path: &RelativePath) -> SimpleFileOptions { + let metadata = matches!( + path.file_name().unwrap_or_default(), + "manifest.json" | "calculation.json" | "request.json" + ); + SimpleFileOptions::default() + .compression_method(if metadata { + CompressionMethod::Deflated + } else { + CompressionMethod::Stored + }) + .last_modified_time(zip::DateTime::default()) + .unix_permissions(0o644) + // Streaming artifacts have no known size when their local header is emitted. + // Reserve ZIP64 fields even for small arrays, without buffering the payload. + .large_file(!metadata) +} + +pub(super) fn writer(file: File) -> ZipWriter> { + ZipWriter::new(BufWriter::new(file)) +} + +fn u16_at(bytes: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(bytes[offset..offset + 2].try_into().unwrap()) +} +fn u32_at(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) +} +fn u64_at(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap()) +} + +fn preflight(file: &mut File) -> Result<(usize, u64), StorageError> { + let length = file.metadata()?.len(); + let tail_len = length.min(22 + u64::from(u16::MAX)); + file.seek(SeekFrom::Start(length - tail_len))?; + let mut tail = + vec![0; usize::try_from(tail_len).map_err(|_| invalid("tail size exceeds host range"))?]; + file.read_exact(&mut tail)?; + let end = (0..tail.len().saturating_sub(21)) + .rev() + .find(|&i| { + tail[i..i + 4] == *b"PK\x05\x06" + && i + 22 + usize::from(u16_at(&tail, i + 20)) == tail.len() + }) + .ok_or_else(|| invalid("missing or truncated ZIP end record"))?; + let eocd = &tail[end..]; + let end_offset = length - tail_len + + u64::try_from(end).map_err(|_| invalid("end offset exceeds V1 range"))?; + if u16_at(eocd, 4) != 0 || u16_at(eocd, 6) != 0 || u16_at(eocd, 8) != u16_at(eocd, 10) { + return Err(invalid("multi-disk archives are unsupported")); + } + let mut count = u64::from(u16_at(eocd, 10)); + let mut size = u64::from(u32_at(eocd, 12)); + let mut offset = u64::from(u32_at(eocd, 16)); + let mut directory_end = end_offset; + // A locator can exist even when classic EOCD fields have not overflowed. + let mut locator = [0; 20]; + let has_locator = if end_offset >= 20 { + file.seek(SeekFrom::Start(end_offset - 20))?; + file.read_exact(&mut locator)?; + locator[..4] == *b"PK\x06\x07" + } else { + false + }; + if has_locator { + if u32_at(&locator, 4) != 0 || u32_at(&locator, 16) != 1 { + return Err(invalid("multi-disk ZIP64 archive")); + } + let record_offset = u64_at(&locator, 8); + if record_offset + .checked_add(56) + .is_none_or(|end| end > end_offset - 20) + { + return Err(invalid("invalid ZIP64 end offset")); + } + file.seek(SeekFrom::Start(record_offset))?; + let mut record = [0; 56]; + file.read_exact(&mut record)?; + let record_size = u64_at(&record, 4); + if record[..4] != *b"PK\x06\x06" + || !(44..=MAX_END_RECORD_BYTES).contains(&record_size) + || record_offset + .checked_add(12) + .and_then(|n| n.checked_add(record_size)) + != Some(end_offset - 20) + || u32_at(&record, 16) != 0 + || u32_at(&record, 20) != 0 + || u64_at(&record, 24) != u64_at(&record, 32) + { + return Err(invalid("invalid ZIP64 end record")); + } + let (count64, size64, offset64) = ( + u64_at(&record, 32), + u64_at(&record, 40), + u64_at(&record, 48), + ); + if (count != u64::from(u16::MAX) && count != count64) + || (size != u64::from(u32::MAX) && size != size64) + || (offset != u64::from(u32::MAX) && offset != offset64) + { + return Err(invalid("inconsistent ZIP64 directory fields")); + } + (count, size, offset) = (count64, size64, offset64); + directory_end = record_offset; + } else if count == u64::from(u16::MAX) + || size == u64::from(u32::MAX) + || offset == u64::from(u32::MAX) + { + return Err(invalid("missing ZIP64 end record")); + } + if count > u64::try_from(MAX_MEMBERS).map_err(|_| invalid("member limit exceeds V1 range"))? + || size > MAX_DIRECTORY_BYTES + || offset.checked_add(size) != Some(directory_end) + { + return Err(invalid("oversized or inconsistent ZIP directory")); + } + file.seek(SeekFrom::Start(offset))?; + let mut directory = + vec![0; usize::try_from(size).map_err(|_| invalid("directory size exceeds host range"))?]; + file.read_exact(&mut directory)?; + let mut cursor = 0_usize; + let mut paths = BTreeSet::new(); + for _ in 0..count { + let header = directory + .get(cursor..cursor + 46) + .ok_or_else(|| invalid("truncated central header"))?; + if header[..4] != *b"PK\x01\x02" { + return Err(invalid("invalid central header")); + } + let flags = u16_at(header, 8); + let method = u16_at(header, 10); + if flags & !0x080e != 0 || !matches!(method, 0 | 8) { + return Err(invalid("unsupported encryption, flags or compression")); + } + let attributes = u32_at(header, 38); + let file_type = (attributes >> 16) & 0o170000; + if attributes & 0x10 != 0 || !matches!(file_type, 0 | 0o100000) { + return Err(invalid("only regular file members are supported")); + } + if u16_at(header, 34) != 0 { + return Err(invalid("multi-disk member")); + } + let name_len = usize::from(u16_at(header, 28)); + let extra_len = usize::from(u16_at(header, 30)); + let comment_len = usize::from(u16_at(header, 32)); + let name_start = cursor + 46; + let next = name_start + name_len + extra_len + comment_len; + if next > directory.len() { + return Err(invalid("truncated directory member")); + } + let name_bytes = &directory[name_start..name_start + name_len]; + if !name_bytes.is_ascii() && flags & 0x0800 == 0 { + return Err(invalid("non-ASCII member names must use the UTF-8 flag")); + } + validate_extras(&directory[name_start + name_len..name_start + name_len + extra_len])?; + let name = std::str::from_utf8(name_bytes).map_err(|_| invalid("non-UTF-8 member path"))?; + validate_path(RelativePath::new(name))?; + let key = name.to_lowercase(); + if !paths.insert(key.clone()) { + return Err(invalid("duplicate member path")); + } + for (index, _) in key.match_indices('/') { + if paths.contains(&key[..index]) { + return Err(invalid("file/directory path conflict")); + } + } + if paths + .range(format!("{key}/")..) + .next() + .is_some_and(|p| p.starts_with(&format!("{key}/"))) + { + return Err(invalid("file/directory path conflict")); + } + // Resolve only the local offset from ZIP64; the library validates full extras. + let mut local_offset = u64::from(u32_at(header, 42)); + if local_offset == u64::from(u32::MAX) { + let extra = &directory[name_start + name_len..name_start + name_len + extra_len]; + let skip = 8 + * (usize::from(u32_at(header, 24) == u32::MAX) + + usize::from(u32_at(header, 20) == u32::MAX)); + local_offset = zip64_offset(extra, skip)?; + } + if local_offset.checked_add(30).is_none_or(|end| end > offset) { + return Err(invalid("invalid local header offset")); + } + file.seek(SeekFrom::Start(local_offset))?; + let mut local = [0; 30]; + file.read_exact(&mut local)?; + let local_name_len = usize::from(u16_at(&local, 26)); + let local_extra_len = u64::from(u16_at(&local, 28)); + if local[..4] != *b"PK\x03\x04" + || u16_at(&local, 6) != flags + || u16_at(&local, 8) != method + || local_name_len != name_len + || local_offset + .checked_add(30 + u64::from(u16_at(header, 28)) + local_extra_len) + .is_none_or(|end| end > offset) + { + return Err(invalid("inconsistent local header")); + } + let mut local_name = vec![0; name_len]; + file.read_exact(&mut local_name)?; + if local_name != name_bytes { + return Err(invalid("local/central member names differ")); + } + let mut local_extra = vec![ + 0; + usize::try_from(local_extra_len) + .map_err(|_| invalid("extra field exceeds host range"))? + ]; + file.read_exact(&mut local_extra)?; + validate_extras(&local_extra)?; + cursor = next; + } + if cursor != directory.len() { + return Err(invalid("unindexed central-directory records")); + } + Ok(( + usize::try_from(count).map_err(|_| invalid("member count exceeds host range"))?, + offset, + )) +} +fn validate_extras(mut extra: &[u8]) -> Result<(), StorageError> { + let mut kinds = BTreeSet::new(); + while !extra.is_empty() { + if extra.len() < 4 { + return Err(invalid("truncated extra field")); + } + let kind = u16_at(extra, 0); + let len = usize::from(u16_at(extra, 2)); + if kind == 0x7075 || !kinds.insert(kind) { + return Err(invalid( + "alternate Unicode paths or duplicate extra fields are unsupported", + )); + } + extra = extra + .get(4 + len..) + .ok_or_else(|| invalid("truncated extra field"))?; + } + Ok(()) +} + +fn zip64_offset(mut extra: &[u8], skip: usize) -> Result { + while extra.len() >= 4 { + let kind = u16_at(extra, 0); + let len = usize::from(u16_at(extra, 2)); + let field = extra + .get(4..4 + len) + .ok_or_else(|| invalid("truncated extra field"))?; + if kind == 1 { + let value = field + .get(skip..skip + 8) + .ok_or_else(|| invalid("missing ZIP64 offset"))?; + return Ok(u64_at(value, 0)); + } + extra = &extra[4 + len..]; + } + Err(invalid("missing ZIP64 extra field")) +} + +/// Stops decompression at the declared size and checks actual EOF/CRC even when +/// a consumer (such as an NPY decoder) reads exactly the declared payload. +pub(super) fn with_member( + archive: &mut ZipArchive, + path: &RelativePath, + read: F, +) -> Result +where + E: From, + F: FnOnce(&mut dyn Read) -> Result, +{ + let member = archive + .by_name(path.as_str()) + .map_err(zip_error) + .map_err(E::from)?; + let size = member.size(); + let mut bounded = BufReader::new(member.take(size)); + let value = read(&mut bounded)?; + io::copy(&mut bounded, &mut io::sink()) + .map_err(StorageError::from) + .map_err(E::from)?; + let bounded = bounded.into_inner(); + if bounded.limit() != 0 { + return Err(E::from(invalid("truncated ZIP member"))); + } + let mut member = bounded.into_inner(); + let mut trailing = [0; 1]; + if member + .read(&mut trailing) + .map_err(StorageError::from) + .map_err(E::from)? + != 0 + { + return Err(E::from(invalid("member exceeds declared size"))); + } + Ok(value) +} diff --git a/crates/rustiq-core/src/prelude.rs b/crates/rustiq-core/src/prelude.rs index 78b0f99b..b1f4c2bd 100644 --- a/crates/rustiq-core/src/prelude.rs +++ b/crates/rustiq-core/src/prelude.rs @@ -1,4 +1,4 @@ -//! Common types and the execution trait for direct Rust calculations. +//! Common types for direct calculations and portable `.rustiq` workflows. //! //! ```no_run //! use rustiq_core::prelude::*; @@ -22,13 +22,46 @@ //! # Ok(()) //! # } //! ``` +//! +//! A portable archive can also supply reusable artifacts to normal execution: +//! +//! ```no_run +//! use rustiq_core::prelude::*; +//! +//! # fn main() -> Result<(), Box> { +//! let geometry = Geometry::from_reader(std::io::Cursor::new( +//! "2\nH2\nH 0 0 -0.37\nH 0 0 0.37\n", +//! ))?; +//! let basis = BasisFile::from_reader(std::fs::File::open("sto-3g.json")?)?; +//! let prepared = CalculationBuilder::new(&geometry, &basis).prepare()?; +//! let hf = prepared.run_hf()?; +//! let eri: &CompactEri = hf.ao_eri(); +//! +//! let directory = tempfile::tempdir()?; +//! let path = directory.path().join("water.rustiq"); +//! let mut data = RustiQData::from_calculation(&prepared)?; +//! data.write_with_eri(&path, eri)?; +//! +//! let restored = RustiQData::open(&path)?.prepare_calculation()?; +//! let result = restored.execute()?; +//! +//! // Strict ownership-transfer path: missing or incompatible ERIs are errors. +//! let mut data = RustiQData::open(&path)?; +//! let eri = data.take_compatible_eri(&prepared)?; +//! let result = prepared.execute_with_eri(eri)?; +//! let _ = (result, Option::::None); +//! # Ok(()) +//! # } +//! ``` pub use crate::{ basis::BasisFile, calculation::{ - CalculationBuilder, CalculationExecution, CalculationExecutionError, CalculationResult, - Converged, HfOutcome, HfSolution, PreparedCalculation, Unconverged, + ArtifactReuseDecision, ArtifactReuseEvent, CalculationBuilder, CalculationEvent, + CalculationExecution, CalculationExecutionError, CalculationResult, Converged, HfOutcome, + HfSolution, PreparedCalculation, Unconverged, }, config::{HfConfig, HfMethod, MoleculeConfig, Mp2Config}, molecules::{atom::Atom, geometry::Geometry, units::Units}, + persistence::{CompactEri, RustiQBundle, RustiQData}, }; diff --git a/crates/rustiq-core/tests/automatic_artifact_reuse.rs b/crates/rustiq-core/tests/automatic_artifact_reuse.rs new file mode 100644 index 00000000..8ecc8199 --- /dev/null +++ b/crates/rustiq-core/tests/automatic_artifact_reuse.rs @@ -0,0 +1,263 @@ +use rustiq_core::{ + basis::BasisFile, + calculation::{ + ArtifactReuseDecision, CalculationBuilder, CalculationEvent, CalculationExecution, + }, + config::Mp2Config, + molecules::geometry::Geometry, + persistence::{CompactEri, EriCache, RustiQData}, +}; + +fn inputs(distance: f64) -> (Geometry, BasisFile) { + let geometry = Geometry::from_source( + "h2.xyz", + &format!("2\nH2 in Bohr\nH 0 0 0\nH {distance} 0 0\n"), + ) + .unwrap(); + let basis = BasisFile::from_reader(&include_bytes!("data/sto-3g.json")[..]).unwrap(); + (geometry, basis) +} + +#[test] +fn execute_automatically_reuses_portable_eri_and_preserves_mp2_results() { + let (geometry, basis) = inputs(1.4); + let original = CalculationBuilder::new(&geometry, &basis) + .with_mp2(Mp2Config::default()) + .prepare() + .unwrap(); + let expected = original.execute().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("h2.rustiq"); + RustiQData::from_calculation(&original) + .unwrap() + .write_with_eri(&path, expected.hf.ao_eri()) + .unwrap(); + let archive_bytes = std::fs::read(&path).unwrap(); + let restored = RustiQData::open(&path) + .unwrap() + .prepare_calculation() + .unwrap(); + let mut decisions = Vec::new(); + let result = restored + .execute_with_events(|event| { + if let CalculationEvent::ArtifactReuse(event) = event { + assert_eq!(event.artifact, "ao_eri"); + decisions.push(event.decision); + } + }) + .unwrap(); + assert_eq!(decisions, [ArtifactReuseDecision::Reused]); + assert_eq!( + result.hf.summary().scf.total_energy, + expected.hf.summary().scf.total_energy + ); + assert_eq!( + result.mp2.unwrap().correlation_energy, + expected.mp2.unwrap().correlation_energy + ); + + let overridden = restored.run_hf_with_eri(CompactEri::Zeroed(2)).unwrap(); + assert_eq!(overridden.ao_eri()[(0, 0, 0, 0)], 0.0); + let repeated = restored.execute().unwrap(); + let hf_only = restored.run_hf().unwrap(); + let pointer = &result.hf.ao_eri()[(0, 0, 0, 0)] as *const f64; + assert_eq!(pointer, &repeated.hf.ao_eri()[(0, 0, 0, 0)] as *const f64); + assert_eq!(pointer, &hf_only.ao_eri()[(0, 0, 0, 0)] as *const f64); + assert_eq!(std::fs::read(path).unwrap(), archive_bytes); +} + +#[test] +fn missing_eri_are_computed_once_and_retained_for_normal_execution() { + let (geometry, basis) = inputs(1.4); + let original = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + let expected = original.execute().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("inputs-only.rustiq"); + RustiQData::from_calculation(&original) + .unwrap() + .write(&path) + .unwrap(); + let archive_bytes = std::fs::read(&path).unwrap(); + let restored = RustiQData::open(&path) + .unwrap() + .prepare_calculation() + .unwrap(); + let mut decisions = Vec::new(); + let mut execute = || { + restored + .execute_with_events(|event| { + if let CalculationEvent::ArtifactReuse(event) = event { + decisions.push(event.decision); + } + }) + .unwrap() + }; + let first = execute(); + let second = execute(); + assert_eq!( + decisions, + [ + ArtifactReuseDecision::Missing, + ArtifactReuseDecision::Reused + ] + ); + assert_eq!( + first.hf.summary().scf.total_energy, + expected.hf.summary().scf.total_energy + ); + assert_eq!( + &first.hf.ao_eri()[(0, 0, 0, 0)] as *const f64, + &second.hf.ao_eri()[(0, 0, 0, 0)] as *const f64, + ); + assert_eq!(std::fs::read(path).unwrap(), archive_bytes); +} + +#[test] +fn incompatible_artifacts_fall_back_to_the_current_calculation_and_local_cache() { + let (source_geometry, basis) = inputs(1.4); + let source_calculation = CalculationBuilder::new(&source_geometry, &basis) + .prepare() + .unwrap(); + let mut source = RustiQData::from_calculation(&source_calculation).unwrap(); + source.set_eri(CompactEri::Zeroed(2)).unwrap(); + let (geometry, _) = inputs(1.6); + let expected = CalculationBuilder::new(&geometry, &basis) + .execute() + .unwrap(); + let directory = tempfile::tempdir().unwrap(); + let cache_root = directory.path().join("cache"); + let calculation = CalculationBuilder::new(&geometry, &basis) + .with_eri_cache(EriCache::new(&cache_root)) + .prepare() + .unwrap() + .with_reuse_data(source) + .unwrap(); + let mut decisions = Vec::new(); + let mut cache_events = Vec::new(); + let result = calculation + .execute_with_events(|event| match event { + CalculationEvent::ArtifactReuse(event) => decisions.push(event.decision), + CalculationEvent::EriCache(event) => cache_events.push(event.action), + _ => {} + }) + .unwrap(); + assert_eq!(decisions, [ArtifactReuseDecision::Incompatible]); + assert_eq!( + cache_events, + [rustiq_core::calculation::EriCacheAction::Stored] + ); + assert_eq!( + result.hf.summary().scf.total_energy, + expected.hf.summary().scf.total_energy + ); + assert!(result.hf.ao_eri()[(0, 0, 0, 0)] > 0.0); +} + +#[test] +fn explicit_portable_eri_take_precedence_over_local_cache_and_computation() { + let (geometry, basis) = inputs(1.4); + let original = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + let mut source = RustiQData::from_calculation(&original).unwrap(); + source.set_eri(CompactEri::Zeroed(2)).unwrap(); + let directory = tempfile::tempdir().unwrap(); + let cache_root = directory.path().join("unused-cache"); + let calculation = CalculationBuilder::new(&geometry, &basis) + .with_eri_cache(EriCache::new(&cache_root)) + .prepare() + .unwrap() + .with_reuse_data(source) + .unwrap(); + let result = calculation.execute().unwrap(); + assert_eq!(result.hf.ao_eri()[(0, 0, 0, 0)], 0.0); + assert!(!cache_root.exists()); +} + +#[test] +fn restored_open_shell_calculation_preserves_requested_units_and_resolved_method() { + use rustiq_core::{ + config::{DiisConfig, HfConfig, MoleculeConfig, ResolvedHfMethod}, + molecules::units::Units, + }; + use std::num::NonZeroU8; + let geometry = Geometry::from_source("oh.xyz", "2\nOH\nO 0 0 0\nH 0 0 0.97\n").unwrap(); + let (_, basis) = inputs(1.4); + let original = CalculationBuilder::new(&geometry, &basis) + .with_molecule_config(MoleculeConfig { + units: Units::Angstrom, + multiplicity: NonZeroU8::new(2).unwrap().into(), + ..Default::default() + }) + .with_hf(HfConfig { + diis: DiisConfig { + enabled: true, + ..Default::default() + }, + ..Default::default() + }) + .prepare() + .unwrap(); + let expected = original.execute().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("oh.rustiq"); + RustiQData::from_calculation(&original) + .unwrap() + .write_with_eri(&path, expected.hf.ao_eri()) + .unwrap(); + let restored = RustiQData::open(path) + .unwrap() + .prepare_calculation() + .unwrap(); + assert_eq!(restored.hf_method(), ResolvedHfMethod::Uhf); + assert_eq!(restored.request().molecule().units, Units::Angstrom); + assert!(restored.hf_config().diis.enabled); + let result = restored.execute().unwrap(); + assert!(result.hf.is_converged()); + assert_eq!( + result.hf.summary().scf.total_energy, + expected.hf.summary().scf.total_energy + ); +} + +#[test] +fn resolved_spherical_ao_components_are_restored_without_renormalization() { + let (geometry, _) = inputs(1.4); + let mut basis_json: serde_json::Value = + serde_json::from_slice(include_bytes!("data/sto-3g.json")).unwrap(); + basis_json["elements"]["1"]["electron_shells"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "function_type": "gto_spherical", + "angular_momentum": [2], + "exponents": ["0.8"], + "coefficients": [["1.0"]] + })); + let basis = + BasisFile::from_reader(serde_json::to_vec(&basis_json).unwrap().as_slice()).unwrap(); + let original = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + let restored = RustiQData::from_calculation(&original) + .unwrap() + .prepare_calculation() + .unwrap(); + assert_eq!(original.get_basis().nbasis(), 12); + assert_eq!( + restored.get_basis().overlap_ints(), + original.get_basis().overlap_ints() + ); + assert_eq!( + restored.get_basis().kinetic_ints(), + original.get_basis().kinetic_ints() + ); + let expected = original.execute().unwrap(); + let result = restored.execute().unwrap(); + assert_eq!( + result.hf.summary().scf.total_energy, + expected.hf.summary().scf.total_energy + ); +} diff --git a/crates/rustiq-core/tests/data/persistence/calculation-h2-v1.json b/crates/rustiq-core/tests/data/persistence/calculation-h2-v1.json new file mode 100644 index 00000000..c8bd2f5a --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/calculation-h2-v1.json @@ -0,0 +1,101 @@ +{ + "format": "rustiq-calculation", + "version": 1, + "units": "bohr", + "atoms": [ + { + "atomic_number": 1, + "position": [ + 0.0, + 0.0, + 0.0 + ] + }, + { + "atomic_number": 1, + "position": [ + 1.4, + 0.0, + 0.0 + ] + } + ], + "charge": 0, + "multiplicity": 1, + "basis": [ + { + "center": [ + 0.0, + 0.0, + 0.0 + ], + "components": [ + { + "angular_momentum": [ + 0, + 0, + 0 + ], + "primitives": [ + { + "exponent": 3.425250914, + "coefficient": 0.2769343550790519 + }, + { + "exponent": 0.6239137298, + "coefficient": 0.26783885160947885 + }, + { + "exponent": 0.168855404, + "coefficient": 0.08347367112984119 + } + ] + } + ] + }, + { + "center": [ + 1.4, + 0.0, + 0.0 + ], + "components": [ + { + "angular_momentum": [ + 0, + 0, + 0 + ], + "primitives": [ + { + "exponent": 3.425250914, + "coefficient": 0.2769343550790519 + }, + { + "exponent": 0.6239137298, + "coefficient": 0.26783885160947885 + }, + { + "exponent": 0.168855404, + "coefficient": 0.08347367112984119 + } + ] + } + ] + } + ], + "hf": { + "method": "rhf", + "max_iterations": 100, + "convergence_threshold": 1e-8, + "linear_dependency_threshold": 1e-8, + "eri_schwarz_threshold": 1e-12, + "diis_size": null, + "guess": { + "kind": "core_hamiltonian", + "perturbation": null + } + }, + "mp2": null, + "ao_eri_computation_version": 1 +} diff --git a/crates/rustiq-core/tests/data/persistence/portable-manifest-h2-v1.json b/crates/rustiq-core/tests/data/persistence/portable-manifest-h2-v1.json new file mode 100644 index 00000000..e064de0e --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/portable-manifest-h2-v1.json @@ -0,0 +1,51 @@ +{ + "format": "rustiq-persistence", + "format_version": 1, + "kind": "portable", + "producer": { + "name": "Python interoperability fixture", + "version": "1" + }, + "sources": [ + { + "original_name": "../../calculation.toml", + "path": "sources/0", + "version": 1, + "size": 84, + "digest": "sha256:0e4846b3cda3e792522dad47a72f03dde51c5584411e028712ee292cb4184f06" + } + ], + "calculations": [ + { + "id": "calculation-0", + "scientific_identity": { + "version": 1, + "digest": "sha256:0fd27aa0f91978616522cb640381b2db36b1ddebb97f3fd53a8036270717c6cc" + }, + "request": { + "path": "calculations/calculation-0/request.json", + "version": 1, + "size": 666, + "digest": "sha256:b0f6d85c96f09c2320b5774c12455ba0de4193ca751bc9c2ba0cd1d2efe042c6" + }, + "calculation": { + "path": "calculations/calculation-0/calculation.json", + "version": 1, + "size": 1892, + "digest": "sha256:5123dec16a2195b13299d7f4b9fd27b089b3f693fc189bc0fb83efcb60e69f55" + }, + "artifacts": { + "ao_eri": { + "path": "calculations/calculation-0/arrays/integrals/ao-eri.npy", + "size": 176, + "digest": "sha256:d838095f0921ded52c58e791ff1ad0fc558dc1b4b43aae62c5be45e19c9735be", + "representation": "rustiq-compact-eri-v1", + "attributes": { + "basis_functions": 2, + "computation_version": 1 + } + } + } + } + ] +} diff --git a/crates/rustiq-core/tests/data/persistence/portable-python-angstrom-v1.rustiq.hex b/crates/rustiq-core/tests/data/persistence/portable-python-angstrom-v1.rustiq.hex new file mode 100644 index 00000000..e9a510d3 --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/portable-python-angstrom-v1.rustiq.hex @@ -0,0 +1 @@ +504b0304140000000800000021002758fe798f010000680300000d0000006d616e69666573742e6a736f6e95924d6edb301085f73e85a0b5d35023fe66dd03641f14c17048d66c65c921e9a2aee1bb97928dd84d8b148100427cf338f33e89c755d3b4614a5b2ced43d3a67d2ef1e56ee7538eb9f8917cbbbe3a9e7fccfa345667b7c8dfe3e8e663bb2915b4c3c5bc4b93db934fb572acfbaa8cb8f5b3eff15036d3d87cf614b73834712c3e4d7518da38c4726842fc59f6e9dca61ebb8e6bbbb64aa7a53de140fb014bade45a7a5abce741b51a9740379e3b7669578b99a21f4b0c919ea39bdfcae135e49b89ddfaaabaf8d5e7e5fbe40d82900fbc173628abb806e69895d47b540e9d4001ac63e8591f502a0261a167aeebb9ad8f271f1887defaf6d2fbf49a2cf997fd79c64d9a1d96cd1b9a7cff07dafde5dca76fb9a65eff0f24c75ff37f90aa7f8f0e8c088e4091b29a75d081a5c0b547e32ab3314170462811782525a72c775a290ea23a9916e61f7437993f4c78b3fb186567847c0fd338d406b5efc919a8802401b81432105aa3092ca2b19a4bed44c791c0bb0015da76428001467f6362aa770ba9ccd7f2785ad479fdb23aad7e03504b030414000000080000002100db07d4a038010000a10200002700000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6ea5524d4fc3300cbdef575439b753bb2140fc018e1ce0364d55d67aad453e3ac701b669ff9d34cd3684342e5c92f8f9c5f67bc9719665626b494b164f9920ef187705c1ce8363918fd90f2087d6847415636f90dd4896a6734c564f34c9568ff02a0459768c6b82b1a98dd71ba0738d9819ac439e0aaf129665e5bccc6f06e9bc8efb29ff67a387bb6bf1a29c57cb3f7a85751d5536bda40e42a53899d05e310e0a1be4fdc5a08d74180d7a7d7b2996cf933dfd3620d3b04203f7b68d167ab6224fa8fcaa9181e438eb78bf2a937cd158135ea103d340cd3d81ebad1aef5750948f89a3d080a4ba85014c1b98fb9b4c20ac5dd37f4a3afce6548bc469115dedf0304abd4f5817fe84bba808c03b9aa8a2b104752f352ab606a51157eb81d8d34626fb8d57eae7e3c53601df4ae5609660a187c5997b9a7d03504b0304140000000800000021002bb22568fd010000a40700002b00000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6eed55c972a33010bdfb2b28ced8a58535bf927251b22c1bd5208991442689cbff3e2dc138d81367d6633880ba79bdbd6ee8d32a49d283b18af9f42149ede8bcfcbae6ace763cfbc343acd02e24958178487044779d4d2bb60b0339d9d20cc1b15548f2024c929de67b5e4ad1ed54ed81ff6f1cd609cf493d3c75997246883b2bbc27cdec6e739fba74078439b9a3615a584108acaa67e0bb5461b92176599d3a62408d302e10ff280fb3632c03b668f02a2c4ac5335f65e0ebde4d2bf5c88db3127df61890bed63d67fc8c4a5466ed460347871574e4e975360481fa1a9b6554601705457c8e838fb505c48dbe5ab74b05201c34fc2fde4f2742501563c4f7902926e725290023538cf6e61dc88c301a89b90d08faa6c684e8b02550d2a7073853f67bf1d136d4a421b4c2bb26cf8bda06555d3ba2e70899abc82c3df47c525981739fa75a5a8a67945cb0a634831c7b7a52e9bb0bad5beff65bc335cff63fa3f672f5c9fb377e76fd81dc0e59458aa84efcc3e2e185067b3923db7120633ae99d03d8ce6ae435e1a36ce51682e5adf59e13ad307732cd668a62eeda51630507b3108bd07e4cb5da4b0b275bcfbc6eceb2d069319b397d2b54ebe86dfb71efb7e561f47e1dca50e507c913ad6c18d156dc794ecbdd192e9f46ddb08eb47bb63f3c609ce2ecc9ca7c5309045949499366418bea2d147b376b16e57e7d577504b01021403140000000800000021002758fe798f010000680300000d0000000000000000000000a481000000006d616e69666573742e6a736f6e504b0102140314000000080000002100db07d4a038010000a1020000270000000000000000000000a481ba01000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6e504b01021403140000000800000021002bb22568fd010000a40700002b0000000000000000000000a4813703000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6e504b05060000000003000300e90000007d0500000000 diff --git a/crates/rustiq-core/tests/data/persistence/portable-python-big-v1.rustiq.hex b/crates/rustiq-core/tests/data/persistence/portable-python-big-v1.rustiq.hex new file mode 100644 index 00000000..906e0cfa --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/portable-python-big-v1.rustiq.hex @@ -0,0 +1 @@ +504b03042d000000080000002100bd8db22dffffffffffffffff0d0014006d616e69666573742e6a736f6e01001000d5050000000000006d020000000000009594db72da301086ef790ac6d701743ee4297adfe93092564ad41a9b4872a634c3bb5736104cd2a6cd0cc3d8ab5fffeeb76be965b15c36a14f3b539afb6593865ce2d36aef538eb9f8cef9e6eeaad83e8ff1beab4a3c857fc40ec66dfb3e1563dbb3789f7a189c4f75e5a5bed74867767ed47d3994c7be5bc6aef8d4d724c6c63696c332c49f6548a7ed557e4dd3e0a6868e936dee87e47caed1af93ece45d17fa141f6267daed25cd7abda93f675a37b4a654a775e977edd97d2cd094c7517776dca0ebd21bc22996e3afd156b1d708c4079fa786e54743b8b8479e29262c7560a8979a7042c000934692802880e7d871ae18c3d823a22426de134d9c6558b1804433391febffb7897556fb1f80e3d4f499663523c82efaaec410dd36c2f8540eaf83f81be21f910210690c0a1a6ba9041695c959c11055d812b054585cc1acd532d0009c1a85a82012492c9d70ae397b1f5f2b4bfe6938e598557399c59c787383b639ef5b7fcfb5eabb7f819ca72584f888cea2204071a74540da114a90e552328709e3dc1a049e614d9d911c5b570535e40003f1c123469c784f37abf9d384f34ff553945869f21126c784827758188235b79812ad410666f5385d8b94b634084d83ab4ed6a16015f5a18e1979a103e7ef314daadf967125df429a7eeb53bc89fd1fba49c91cf266bc121e9269f3c6f4abeab4eef687591766c452dc84df210358a8f3042f7160044900e58db09c7aa724300581720c96280d4ed6d30801310f522ba578001b6eb326bf4f3ed7637419ece58a74fd6e5fdb30d6ba7ac6b79b4c2929daa1f8fca62175cd9a7ab36ec3d0b9cbe12677b78ad17938e59b5fb833d171f1f6e978bd3f16c7c56f504b03042d000000080000002100021adb15ffffffffffffffff2b00140063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6e010010006407000000000000e201000000000000ed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b03042d0000000800000021004100ae2bffffffffffffffff2700140063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6e010010009a020000000000003001000000000000a592c14ec3300c86ef7b8a28e76e6a07428817e0b803dca6a94a536fb14893ce49806ddabb93a4d940489390b8a4f5efcfb1fdb7a719637c6b69109e3f314ec179dccf09f6019ce755cabe0339b426a69b1c0783de25b8b38a2644783b24691d03c64ef92c32cad684a103bad4e7cc681dfae9d275d118ab1775753328ef9bfc3c57ff6ad42ceeffda289e9bbca254827610afc9341f82f6386a94e80f57673ae1303bf3f2ba9adf3d4fdea86d54a649f9005ed93e112278cbaba28acf163d904883a6faa62e2371694db47f074642eb15815356a7fa06e6f56361341a10d4f63082e92379b8490261eba4fa1074fccd34cbc2f488ae75784cab3e146d177f0677dd220a6f68f216d212b44a0ca8bd35280cfff61dc807ea44f1de04ad7f7eb9dc26ea5ba11dcc8acc87717961cfb32f504b03042d000000080000002100564c8e8fffffffffffffffff09001400736f75726365732f300100100054000000000000005400000000000000535670ad484c2e5128ce2f2d4a4e552828ca2f4bcd4bcc4b4eb55248cecfcd4dcd2b295648cc4b51700ef27153482c02a9482d4e2d2a4b4dd1e3e58ace488be5e5ca4d2dc9c84f51b05550722c2dc957e2e50200504b03042d000000000000002100feb6142cffffffffffffffff3600140063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f6172726179732f696e74656772616c732f616f2d6572692e6e707901001000b000000000000000b000000000000000934e554d5059010076007b276465736372273a20273e6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a3fe00000000000003ff80000000000004004000000000000400c00000000000040120000000000004016000000000000504b01022d032d000000080000002100bd8db22dffffffffffffffff0d0014000000000000000000a481000000006d616e69666573742e6a736f6e01001000d5050000000000006d02000000000000504b01022d032d000000080000002100021adb15ffffffffffffffff2b001c000000000000000000a481ffffffff63616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6e010018006407000000000000e201000000000000ac02000000000000504b01022d032d0000000800000021004100ae2bffffffffffffffff27001c000000000000000000a481ffffffff63616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6e010018009a020000000000003001000000000000eb04000000000000504b01022d032d000000080000002100564c8e8fffffffffffffffff09001c000000000000000000a481ffffffff736f75726365732f3001001800540000000000000054000000000000007406000000000000504b01022d032d000000000000002100feb6142cffffffffffffffff36001c000000000000000000a481ffffffff63616c63756c6174696f6e732f63616c63756c6174696f6e2d302f6172726179732f696e74656772616c732f616f2d6572692e6e707901001800b000000000000000b0000000000000000307000000000000504b06062c000000000000002d002d0000000000000000000500000000000000050000000000000008020000000000001b08000000000000504b060700000000230a00000000000001000000504b05060000000005000500080200001b0800000000 diff --git a/crates/rustiq-core/tests/data/persistence/portable-python-little-v1.rustiq.hex b/crates/rustiq-core/tests/data/persistence/portable-python-little-v1.rustiq.hex new file mode 100644 index 00000000..33045ae7 --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/portable-python-little-v1.rustiq.hex @@ -0,0 +1 @@ +504b03042d000000080000002100caae2935ffffffffffffffff0d0014006d616e69666573742e6a736f6e01001000d5050000000000006d020000000000009594d96edb301045dffd15869e639bfb92afe87b51185c86095b5972482aa81bf8df4bc9762c276dda0082400d2fefcce1887c592c974de8d3ce94e67ed9a42197f8b4da43ca3117e81c347757c5f6798cf75d55e229fc23767e5cb6ef5331b63d8bf7a9f7838354675eea778d746607a3eecba13cf6dd327605525f93181bdb580ecb107f96219d9657f9354d839b1a3a4eb6b91f92835ca35f27d9c9bb4ef4293ec4ceb4db4b9af57a531f675a37b4a654a775e977edd97d2cd094c7517776dca0ebd41bc22996e3afd156b1d7888f0f90a70dcb8f8670718f8029262c75de50909a7042bcf14c1a4902a2de03c78e73c518c68088929800104d9c6558b1804433391febfbdbc43aabfd0fc071daf499663523c82e425762886e1bfd382a87d746fc0df18f48c113690c0a1a6ba9041695c959c11055d8126fa9b0b88259ab65a0c1736a14a2824824b174c2b9e6ec7d7cad2cc1d370ca31abe6d28b39f1e6066d735eb7fe9e6bd577ff0239774b08f1119d454178c59d160169472841964bc91c268c736b900786357546726c5d15d490f3d81308801871e23dddace64f13ce7fd54f5162a5c947981c13eac1616108d6dc624ab4f63230abc7ee5aa4b4a541681a5c75b20e05ab2884da66044207cedf639a54ff2de34abe8534fd1652bc89fd1fba49c91cf266bc121e9269f3c6f4abeab4eef687d92ecc88a5b809bf43f68a2aa4796d2bc11e3c278eab7a247108d87814c663e81db6cc326a0c883a6d8171c0da6949ebf8366b827d825c8fd1a5b1972bd2f5bb7ddd86b1d6d533be5d644a49d10e05f29b0da973d6d49b751b86ce5d0e37b9bb558ccec329dffcc29d898e8bb7a3e3f5fe581c17bf01504b03042d000000080000002100021adb15ffffffffffffffff2b00140063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6e010010006407000000000000e201000000000000ed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b03042d0000000800000021004100ae2bffffffffffffffff2700140063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6e010010009a020000000000003001000000000000a592c14ec3300c86ef7b8a28e76e6a07428817e0b803dca6a94a536fb14893ce49806ddabb93a4d940489390b8a4f5efcfb1fdb7a719637c6b69109e3f314ec179dccf09f6019ce755cabe0339b426a69b1c0783de25b8b38a2644783b24691d03c64ef92c32cad684a103bad4e7cc681dfae9d275d118ab1775753328ef9bfc3c57ff6ad42ceeffda289e9bbca254827610afc9341f82f6386a94e80f57673ae1303bf3f2ba9adf3d4fdea86d54a649f9005ed93e112278cbaba28acf163d904883a6faa62e2371694db47f074642eb15815356a7fa06e6f56361341a10d4f63082e92379b8490261eba4fa1074fccd34cbc2f488ae75784cab3e146d177f0677dd220a6f68f216d212b44a0ca8bd35280cfff61dc807ea44f1de04ad7f7eb9dc26ea5ba11dcc8acc87717961cfb32f504b03042d000000080000002100564c8e8fffffffffffffffff09001400736f75726365732f300100100054000000000000005400000000000000535670ad484c2e5128ce2f2d4a4e552828ca2f4bcd4bcc4b4eb55248cecfcd4dcd2b295648cc4b51700ef27153482c02a9482d4e2d2a4b4dd1e3e58ace488be5e5ca4d2dc9c84f51b05550722c2dc957e2e50200504b03042d000000000000002100db95d32dffffffffffffffff3600140063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f6172726179732f696e74656772616c732f616f2d6572692e6e707901001000b000000000000000b000000000000000934e554d5059010076007b276465736372273a20273c6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a000000000000e03f000000000000f83f00000000000004400000000000000c4000000000000012400000000000001640504b01022d032d000000080000002100caae2935ffffffffffffffff0d0014000000000000000000a481000000006d616e69666573742e6a736f6e01001000d5050000000000006d02000000000000504b01022d032d000000080000002100021adb15ffffffffffffffff2b001c000000000000000000a481ffffffff63616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6e010018006407000000000000e201000000000000ac02000000000000504b01022d032d0000000800000021004100ae2bffffffffffffffff27001c000000000000000000a481ffffffff63616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6e010018009a020000000000003001000000000000eb04000000000000504b01022d032d000000080000002100564c8e8fffffffffffffffff09001c000000000000000000a481ffffffff736f75726365732f3001001800540000000000000054000000000000007406000000000000504b01022d032d000000000000002100db95d32dffffffffffffffff36001c000000000000000000a481ffffffff63616c63756c6174696f6e732f63616c63756c6174696f6e2d302f6172726179732f696e74656772616c732f616f2d6572692e6e707901001800b000000000000000b0000000000000000307000000000000504b06062c000000000000002d002d0000000000000000000500000000000000050000000000000008020000000000001b08000000000000504b060700000000230a00000000000001000000504b05060000000005000500080200001b0800000000 diff --git a/crates/rustiq-core/tests/data/persistence/portable-python-multi-v1.rustiq.hex b/crates/rustiq-core/tests/data/persistence/portable-python-multi-v1.rustiq.hex new file mode 100644 index 00000000..1a4e2268 --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/portable-python-multi-v1.rustiq.hex @@ -0,0 +1 @@ +504b030414000000080000002100564c8e8f540000005400000009000000736f75726365732f30535670ad484c2e5128ce2f2d4a4e552828ca2f4bcd4bcc4b4eb55248cecfcd4dcd2b295648cc4b51700ef27153482c02a9482d4e2d2a4b4dd1e3e58ace488be5e5ca4d2dc9c84f51b05550722c2dc957e2e50200504b030414000000080000002100021adb15e2010000640700002b00000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6eed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b0304140000000800000021004100ae2b300100009a0200002700000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6ea592c14ec3300c86ef7b8a28e76e6a07428817e0b803dca6a94a536fb14893ce49806ddabb93a4d940489390b8a4f5efcfb1fdb7a719637c6b69109e3f314ec179dccf09f6019ce755cabe0339b426a69b1c0783de25b8b38a2644783b24691d03c64ef92c32cad684a103bad4e7cc681dfae9d275d118ab1775753328ef9bfc3c57ff6ad42ceeffda289e9bbca254827610afc9341f82f6386a94e80f57673ae1303bf3f2ba9adf3d4fdea86d54a649f9005ed93e112278cbaba28acf163d904883a6faa62e2371694db47f074642eb15815356a7fa06e6f56361341a10d4f63082e92379b8490261eba4fa1074fccd34cbc2f488ae75784cab3e146d177f0677dd220a6f68f216d212b44a0ca8bd35280cfff61dc807ea44f1de04ad7f7eb9dc26ea5ba11dcc8acc87717961cfb32f504b030414000000000000002100db95d32db0000000b00000003600000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f6172726179732f696e74656772616c732f616f2d6572692e6e7079934e554d5059010076007b276465736372273a20273c6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a000000000000e03f000000000000f83f00000000000004400000000000000c4000000000000012400000000000001640504b030414000000080000002100021adb15e2010000640700002b00000063616c63756c6174696f6e732f63616c63756c6174696f6e2d312f63616c63756c6174696f6e2e6a736f6eed55db729b30107df757303c1346e24e7e25e3616459369a2289ea9226f1f8dfb31284daaedd38ed6b7810dae5ecd98b16ed611545f14e69416cfc18c5da19cb7f3e5032503710cb958c138f7866da78e131c24176925be30d36aad713845825bcea0984283a84755673da4927364c7fd8872fa332dc4ea44fb32e8a508a929bc2bc5f87f731f92f47382dee7504eb3aa4487ba2f70c68023a166eb07c1c38e5f675a9cc86187ea50c94491bc2fa62aa4b1254895149603167248765e74b20f7706aba134a00d0893364204efe2a9e48ebd34ff1a8b980123e33f307e5e14c022c7b99e204649e16599995a8c5457209a38aed7650ba0989d2acaedabcc8cb12d52d2a717b863f2677fb446995e52dceebac6d3e775ad54dde3425ae505bd4b0f977afb802f3b2409f678a9abca8f3aac618422cf065aaa787b0bad45e6ffd2bcd75577b7f37977fbe9bebc675d7ef80720a2c16ccf66a1b4604a89359495e3a0e9d1706853f3d8ce65387b824cc8c3d939475b6d7ccf46af0e6983dcc958b072e19f4d3968d4c6e01f87a0bc834ef0ced7f11fd7609c1d98cd9726e3ac3dffced2cdd30cceabd63c62c5980e20797210baa34eb7a22f86095e444c6bfa705d3d6e90d992786275bea729ceefd313bf11213d5f908fd3fe46c30eb4ec6e5eab87a07504b0304140000000800000021004100ae2b300100009a0200002700000063616c63756c6174696f6e732f63616c63756c6174696f6e2d312f726571756573742e6a736f6ea592c14ec3300c86ef7b8a28e76e6a07428817e0b803dca6a94a536fb14893ce49806ddabb93a4d940489390b8a4f5efcfb1fdb7a719637c6b69109e3f314ec179dccf09f6019ce755cabe0339b426a69b1c0783de25b8b38a2644783b24691d03c64ef92c32cad684a103bad4e7cc681dfae9d275d118ab1775753328ef9bfc3c57ff6ad42ceeffda289e9bbca254827610afc9341f82f6386a94e80f57673ae1303bf3f2ba9adf3d4fdea86d54a649f9005ed93e112278cbaba28acf163d904883a6faa62e2371694db47f074642eb15815356a7fa06e6f56361341a10d4f63082e92379b8490261eba4fa1074fccd34cbc2f488ae75784cab3e146d177f0677dd220a6f68f216d212b44a0ca8bd35280cfff61dc807ea44f1de04ad7f7eb9dc26ea5ba11dcc8acc87717961cfb32f504b030414000000000000002100feb6142cb0000000b00000003600000063616c63756c6174696f6e732f63616c63756c6174696f6e2d312f6172726179732f696e74656772616c732f616f2d6572692e6e7079934e554d5059010076007b276465736372273a20273e6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028362c292c207d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a3fe00000000000003ff80000000000004004000000000000400c00000000000040120000000000004016000000000000504b030414000000080000002100e1699ac2b4020000fd0900000d0000006d616e69666573742e6a736f6ee556cb6e1b3910bcfb2b049d6d89cd37f315b9070b8164376d6ee41985c309561be8df97a3873db2b34e72c82109204854b158ddd53d6ce9cbcd62b14c7d79f475f96eb12ce350f3a7bb1d95210f95ba48cbdb67c6e6f384f75d63c211fe983b9c8eedfa527dd89ec9bbd2e318a9b49d2fed7b433aff4813effdbe3ef4dd2277954adf82f890b7b9ee1729ff53c7723adee8cf6196b06cd0e1283bf463893434f4c39176d26e1b7dc9f7b9f3dbcd25cc6ab56eafe8b771dcfada9456b57fdc9ed5a7047d7d987867c5357bde7ae1f0880df9df49d6ca2704f33d0dc7820d0f9e2bfd8e91b4520711d10b328e2bced1a334def0c4042229884a59290188716b801371c763906065627a79543eb4f7bf8e5e67b97fc5703e167dc6b99b391862a6aee694e326e3b4aafba746fc9fc5af5a4ac88df72c3970c66ad0cd530c5a326121700c420768c642702689844a78cb84e686193051c7b83c6b1f9e322bf4693cc5986573e9c5dcf1facadafa7c6ef5f7d0b2befd969173b7b4d66fb90b2c69b42a3a9d988b5c701694313202974a05cf90243811bd5110622334282220a7444cf2a85fbb9be5fcc30ee78fea0fb904ebf85b3615708114417b0e4e0510dc3934490637753730eb8248da89149b52882c052b28b53633d22e29f5daa62fedd9f2b10ed7267dbfa192afb0efb3ee4bf1fb613d8d84fbe2b7c3daf7774d69d5edf6b32acc1c1b7d05bfb28c5658e6546b2b0724543c2adbae24a4041e599aae2146083248e13de9b61d482a02179d116d7d1db5d0aed0d0aed1a5b1971119fbc75d2bc394ebdd67b83ee46b2d398c95861705697bc1b7c9ba4963172f979bdf5e3326e5f1146f3e7067a4c3cdcbd5e9f3dca537e604fcbe73027efb39017fee9c809f30273060eb2792812439338896bc0e4a50b406a5c5241460e0d66134ed571b139384c6596b55c2907ee53931fdcfb839dcfc07504b0102140314000000080000002100564c8e8f5400000054000000090000000000000000000000a48100000000736f75726365732f30504b0102140314000000080000002100021adb15e2010000640700002b0000000000000000000000a4817b00000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f63616c63756c6174696f6e2e6a736f6e504b01021403140000000800000021004100ae2b300100009a020000270000000000000000000000a481a602000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f726571756573742e6a736f6e504b0102140314000000000000002100db95d32db0000000b0000000360000000000000000000000a4811b04000063616c63756c6174696f6e732f63616c63756c6174696f6e2d302f6172726179732f696e74656772616c732f616f2d6572692e6e7079504b0102140314000000080000002100021adb15e2010000640700002b0000000000000000000000a4811f05000063616c63756c6174696f6e732f63616c63756c6174696f6e2d312f63616c63756c6174696f6e2e6a736f6e504b01021403140000000800000021004100ae2b300100009a020000270000000000000000000000a4814a07000063616c63756c6174696f6e732f63616c63756c6174696f6e2d312f726571756573742e6a736f6e504b0102140314000000000000002100feb6142cb0000000b0000000360000000000000000000000a481bf08000063616c63756c6174696f6e732f63616c63756c6174696f6e2d312f6172726179732f696e74656772616c732f616f2d6572692e6e7079504b0102140314000000080000002100e1699ac2b4020000fd0900000d0000000000000000000000a481c30900006d616e69666573742e6a736f6e504b0506000000000800080096020000a20c00000000 diff --git a/crates/rustiq-core/tests/data/persistence/request-h2-v1.json b/crates/rustiq-core/tests/data/persistence/request-h2-v1.json new file mode 100644 index 00000000..6ed0d621 --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/request-h2-v1.json @@ -0,0 +1,40 @@ +{ + "format": "rustiq-request", + "version": 1, + "units": "bohr", + "atoms": [ + { + "atomic_number": 1, + "position": [ + 0.0, + 0.0, + 0.0 + ] + }, + { + "atomic_number": 1, + "position": [ + 1.4, + 0.0, + 0.0 + ] + } + ], + "charge": 0, + "multiplicity": 1, + "basis": "STO-3G", + "hf": { + "method": "auto", + "max_iterations": 100, + "convergence_threshold": 1e-08, + "linear_dependency_threshold": 1e-08, + "eri_schwarz_threshold": 1e-12, + "diis_size": 6, + "guess": { + "kind": "core_hamiltonian", + "perturbation": null + }, + "diis": false + }, + "mp2": null +} diff --git a/crates/rustiq-core/tests/data/persistence/source-original-v1.toml.hex b/crates/rustiq-core/tests/data/persistence/source-original-v1.toml.hex new file mode 100644 index 00000000..6926b914 --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/source-original-v1.toml.hex @@ -0,0 +1 @@ +2320457861637420736f757263652070726f76656e616e63653a20636f6d6d656e747320616e642043524c4620617265207072657365727665642e0d0a5b68665d0d0a6d6574686f64203d20224175746f220d0a diff --git a/crates/rustiq-core/tests/portable_artifact.rs b/crates/rustiq-core/tests/portable_artifact.rs new file mode 100644 index 00000000..ee09c887 --- /dev/null +++ b/crates/rustiq-core/tests/portable_artifact.rs @@ -0,0 +1,296 @@ +use rustiq_core::{ + basis::BasisFile, + calculation::CalculationBuilder, + molecules::geometry::Geometry, + persistence::{AoEriArtifact, CompactEri, PortableError, RustiQData}, +}; + +#[test] +fn public_api_creates_inspects_and_recovers_a_portable_artifact() { + let geometry = Geometry::from_source("h2.xyz", "2\nH2\nH 0 0 0\nH 1.4 0 0\n").unwrap(); + let basis = BasisFile::from_reader(&include_bytes!("data/sto-3g.json")[..]).unwrap(); + let prepared = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + let mut data = RustiQData::from_calculation(&prepared).unwrap(); + let original = b"# source comments are provenance\r\n[hf]\r\nmethod = 'Auto'\r\n"; + data.add_source("../calculation.toml", original.as_slice()) + .unwrap(); + let values = vec![0.5, 1.5, 2.5, 3.5, 4.5, 5.5]; + let indices = [ + (0, 0, 0, 0), + (1, 0, 0, 0), + (1, 0, 1, 0), + (1, 1, 0, 0), + (1, 1, 1, 0), + (1, 1, 1, 1), + ]; + let mut eri = CompactEri::Zeroed(2); + for (index, value) in indices.into_iter().zip(&values) { + eri[index] = *value; + } + data.set::(eri).unwrap(); + assert_eq!( + data.artifact_representations().collect::>(), + vec![("ao_eri", "rustiq-compact-eri-v1")] + ); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("h2.rustiq"); + data.write(&path).unwrap(); + assert!(matches!( + data.write(&path), + Err(PortableError::AlreadyExists) + )); + let mut restored = RustiQData::open(&path).unwrap(); + assert_eq!( + restored.request().unwrap().hf().method.value, + rustiq_core::config::HfMethod::Auto + ); + assert_eq!( + restored.request().unwrap().molecule().units, + rustiq_core::molecules::units::Units::Bohr + ); + let source = restored.sources().next().unwrap(); + assert_eq!(source.original_name(), "../calculation.toml"); + assert_eq!(source.bytes(), original); + let context = restored.calculation().unwrap(); + assert_eq!(context.charge(), 0); + assert_eq!(context.multiplicity(), 1); + let ao = context.basis().next().unwrap(); + assert_eq!(ao.center(), [0.0; 3]); + let component = ao.components().next().unwrap(); + assert_eq!(component.angular_momentum(), [0; 3]); + assert_eq!(component.primitives().len(), 3); + assert_eq!(restored.producer().0, "RustiQ"); + assert!(restored.eri_is_compatible(&prepared)); + let eri = restored.get::().unwrap().unwrap(); + for (index, value) in indices.into_iter().zip(values) { + assert_eq!(eri[index], value); + } +} + +#[test] +fn public_bundle_api_preserves_restructured_options_for_multiple_calculations() { + use rustiq_core::{ + config::{ + validated::{DiisSize, NonNegativeFiniteF64}, + DiisConfig, HfConfig, IntegralConfig, OrthogonalizationConfig, + }, + persistence::RustiQBundle, + }; + let geometry = Geometry::from_source("h2.xyz", "2\nH2\nH 0 0 0\nH 1.4 0 0\n").unwrap(); + let basis = BasisFile::from_reader(&include_bytes!("data/sto-3g.json")[..]).unwrap(); + let prepared = CalculationBuilder::new(&geometry, &basis) + .with_hf(HfConfig { + diis: DiisConfig { + enabled: true, + max_history: DiisSize::try_new(9).unwrap().into(), + }, + orthogonalization: OrthogonalizationConfig { + linear_dependency_threshold: NonNegativeFiniteF64::try_new(1e-7).unwrap().into(), + }, + ..Default::default() + }) + .with_integrals(IntegralConfig { + schwarz_threshold: None.into(), + }) + .prepare() + .unwrap(); + let mut bundle = RustiQBundle::new(vec![ + RustiQData::from_calculation(&prepared).unwrap(), + RustiQData::from_calculation(&prepared).unwrap(), + ]) + .unwrap(); + bundle + .add_source("input.ncl", b"opaque input".as_slice()) + .unwrap(); + bundle.calculations_mut()[1] + .set::(CompactEri::Zeroed(2)) + .unwrap(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("bundle.rustiq"); + bundle.write(&path).unwrap(); + let mut reopened = RustiQBundle::open(&path).unwrap(); + assert_eq!(reopened.sources().next().unwrap().bytes(), b"opaque input"); + for entry in reopened.calculations() { + let request = entry.request().unwrap(); + assert!(request.hf().diis.enabled); + assert_eq!(request.hf().diis.max_history.value.into_inner(), 9); + assert_eq!( + request + .hf() + .orthogonalization + .linear_dependency_threshold + .value + .into_inner(), + 1e-7 + ); + assert!(request.integrals().schwarz_threshold.value.is_none()); + assert_eq!(entry.calculation().unwrap().diis_size(), Some(9)); + assert_eq!( + entry.calculation().unwrap().linear_dependency_threshold(), + 1e-7 + ); + assert!(entry + .calculation() + .unwrap() + .eri_schwarz_threshold() + .is_none()); + } + assert!(reopened.calculations_mut()[0] + .get::() + .unwrap() + .is_none()); + assert!(reopened.calculations_mut()[1] + .get::() + .unwrap() + .is_some()); +} + +#[test] +fn borrowed_output_and_owned_input_preserve_eri_and_mp2() { + use rustiq_core::{ + calculation::CalculationExecution, config::Mp2Config, persistence::EriCache, + }; + let geometry = Geometry::from_source("h2.xyz", "2\nH2\nH 0 0 0\nH 1.4 0 0\n").unwrap(); + let basis = BasisFile::from_reader(&include_bytes!("data/sto-3g.json")[..]).unwrap(); + let prepared = CalculationBuilder::new(&geometry, &basis) + .with_mp2(Mp2Config::default()) + .prepare() + .unwrap(); + let ordinary = prepared.execute().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("h2.rustiq"); + let mut data = RustiQData::from_calculation(&prepared).unwrap(); + data.add_source("input.xyz", b"provenance".as_slice()) + .unwrap(); + let tensor = ordinary.hf.ao_eri(); + let pointer = &tensor[(0, 0, 0, 0)] as *const f64; + data.write_with_eri(&path, tensor).unwrap(); + assert_eq!(pointer, &ordinary.hf.ao_eri()[(0, 0, 0, 0)] as *const f64); + assert!(data.get::().unwrap().is_none()); + let mut restored = RustiQData::open(&path).unwrap(); + assert_eq!(restored.sources().len(), 1); + assert_eq!(restored.calculation().unwrap().charge(), 0); + let cache_root = directory.path().join("unused-cache"); + let cached = CalculationBuilder::new(&geometry, &basis) + .with_mp2(Mp2Config::default()) + .with_eri_cache(EriCache::new(&cache_root)) + .prepare() + .unwrap(); + let eri = restored.take_compatible_eri(&cached).unwrap(); + let pointer = &eri[(0, 0, 0, 0)] as *const f64; + let supplied = cached.execute_with_eri(eri).unwrap(); + assert_eq!(pointer, &supplied.hf.ao_eri()[(0, 0, 0, 0)] as *const f64); + assert!(!cache_root.exists()); + assert_eq!( + ordinary.hf.summary().scf.electronic_energy, + supplied.hf.summary().scf.electronic_energy + ); + assert_eq!( + ordinary.mp2.unwrap().correlation_energy, + supplied.mp2.unwrap().correlation_energy + ); + assert!(cached.execute_with_eri(CompactEri::Zeroed(3)).is_err()); + let zero = cached.run_hf_with_eri(CompactEri::Zeroed(2)).unwrap(); + assert_eq!(zero.ao_eri()[(0, 0, 0, 0)], 0.0); + assert!(!cache_root.exists()); +} + +#[test] +fn explicit_input_rejects_missing_incompatible_and_multiple_calculations() { + use rustiq_core::{ + config::IntegralConfig, + persistence::{ArtifactError, RustiQBundle}, + }; + let geometry = Geometry::from_source("h2.xyz", "2\nH2\nH 0 0 0\nH 1.4 0 0\n").unwrap(); + let basis = BasisFile::from_reader(&include_bytes!("data/sto-3g.json")[..]).unwrap(); + let prepared = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + let mut data = RustiQData::from_calculation(&prepared).unwrap(); + assert!(matches!( + data.take_compatible_eri(&prepared), + Err(PortableError::Artifact(ArtifactError::Missing)) + )); + data.set_eri(CompactEri::Zeroed(2)).unwrap(); + let changed = CalculationBuilder::new(&geometry, &basis) + .with_integrals(IntegralConfig { + schwarz_threshold: None.into(), + }) + .prepare() + .unwrap(); + assert!(matches!( + data.take_compatible_eri(&changed), + Err(PortableError::IncompatibleEri) + )); + assert!(data.read_eri().is_ok()); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("multi.rustiq"); + RustiQBundle::new(vec![data, RustiQData::from_calculation(&prepared).unwrap()]) + .unwrap() + .write(&path) + .unwrap(); + assert!(RustiQData::open(path).is_err()); +} + +#[test] +fn explicit_input_reports_corrupt_and_unsupported_eri() { + use rustiq_core::persistence::ArtifactError; + use std::{ + fs::File, + io::{Read, Write}, + }; + use zip::{write::SimpleFileOptions, ZipArchive, ZipWriter}; + let geometry = Geometry::from_source("h2.xyz", "2\nH2\nH 0 0 0\nH 1.4 0 0\n").unwrap(); + let basis = BasisFile::from_reader(&include_bytes!("data/sto-3g.json")[..]).unwrap(); + let prepared = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + let hf = prepared.run_hf().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let original = directory.path().join("original.rustiq"); + RustiQData::from_calculation(&prepared) + .unwrap() + .write_with_eri(&original, hf.ao_eri()) + .unwrap(); + for unsupported in [false, true] { + let path = directory + .path() + .join(format!("changed-{unsupported}.rustiq")); + let mut archive = ZipArchive::new(File::open(&original).unwrap()).unwrap(); + let mut writer = ZipWriter::new(File::create(&path).unwrap()); + for index in 0..archive.len() { + let mut member = archive.by_index(index).unwrap(); + let name = member.name().to_owned(); + let mut bytes = Vec::new(); + member.read_to_end(&mut bytes).unwrap(); + if unsupported && name == "manifest.json" { + let mut manifest: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + manifest["calculations"][0]["artifacts"]["ao_eri"]["representation"] = + "future-eri-v2".into(); + bytes = serde_json::to_vec(&manifest).unwrap(); + } else if !unsupported && name.ends_with("ao-eri.npy") { + *bytes.last_mut().unwrap() ^= 1; + } + writer + .start_file(name, SimpleFileOptions::default()) + .unwrap(); + writer.write_all(&bytes).unwrap(); + } + writer.finish().unwrap(); + let mut data = RustiQData::open(path).unwrap(); + let error = data.take_compatible_eri(&prepared).unwrap_err(); + if unsupported { + assert!(matches!( + error, + PortableError::Artifact(ArtifactError::UnsupportedRepresentation(_)) + )); + } else { + assert!(matches!( + error, + PortableError::Artifact(ArtifactError::IntegrityMismatch(_)) + )); + } + } +} diff --git a/crates/rustiq-core/tests/prelude_portable.rs b/crates/rustiq-core/tests/prelude_portable.rs new file mode 100644 index 00000000..b63ff469 --- /dev/null +++ b/crates/rustiq-core/tests/prelude_portable.rs @@ -0,0 +1,43 @@ +use rustiq_core::prelude::*; + +#[test] +fn portable_workflow_uses_only_the_public_prelude() { + let geometry = + Geometry::from_reader(std::io::Cursor::new("2\nH2 in Bohr\nH 0 0 0\nH 1.4 0 0\n")).unwrap(); + let basis = BasisFile::from_reader(&include_bytes!("data/sto-3g.json")[..]).unwrap(); + let prepared = CalculationBuilder::new(&geometry, &basis) + .prepare() + .unwrap(); + + let hf = prepared.run_hf().unwrap(); + let eri: &CompactEri = hf.ao_eri(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("h2.rustiq"); + let mut data = RustiQData::from_calculation(&prepared).unwrap(); + data.write_with_eri(&path, eri).unwrap(); + + let restored = RustiQData::open(&path) + .unwrap() + .prepare_calculation() + .unwrap(); + let mut reused = false; + let result = restored + .execute_with_events(|event| { + if let CalculationEvent::ArtifactReuse(event) = event { + reused = event.decision == ArtifactReuseDecision::Reused; + assert_eq!(event.artifact, "ao_eri"); + } + }) + .unwrap(); + assert!(reused); + let _: &HfOutcome = &result.hf; + + let mut strict_data = RustiQData::open(&path).unwrap(); + let transferred: CompactEri = strict_data.take_compatible_eri(&prepared).unwrap(); + let strict_result = prepared.execute_with_eri(transferred).unwrap(); + let _: &CompactEri = strict_result.hf.ao_eri(); + + let _: Option = None; + let _: Option = None; + let _: Option = None; +} diff --git a/docs/persistence-format-v1.md b/docs/persistence-format-v1.md index f7e2f655..fea1eadd 100644 --- a/docs/persistence-format-v1.md +++ b/docs/persistence-format-v1.md @@ -1,7 +1,7 @@ # RustiQ logical persistence format V1 This document specifies the storage-independent logical format used by RustiQ -scientific persistence. It does not specify an archive container. +scientific persistence, including the portable `.rustiq` ZIP/ZIP64 container. ## Active AO ERI cache @@ -89,8 +89,8 @@ untouched. Names and fingerprints are validated rather than interpreted as paths The core Rust API exposes `persistence::RustiQData` as a typed scientific facade. Physical storage selection is an internal persistence concern: the -directory-backed ERI cache resolves folder storage internally, and the future -portable `.rustiq` API will resolve ZIP/ZIP64 internally rather than exposing a +directory-backed ERI cache resolves folder storage internally, and the +portable `.rustiq` API resolves ZIP/ZIP64 internally rather than exposing a generic storage backend. Internal readers load the bounded manifest first and open scientific artifacts on demand. `read_eri` validates and decodes the AO ERI NPY on first access and keeps the resulting `CompactEri` for subsequent accesses. @@ -110,6 +110,8 @@ basis-function count from their typed manifest attributes; the NPY length is nev used to infer that scientific context. Matrix readers accept C and Fortran order and matrix writers emit Fortran order. +For the directory cache: + - `manifest.json` is UTF-8 JSON and describes the format, producer, scientific identity and artifacts. - `arrays/integrals/ao-eri.npy` is the AO electron-repulsion integral artifact. @@ -155,7 +157,7 @@ independent of its container. ## Generic artifact manifest -The root manifest is an index of versioned scientific artifacts rather than a +The cache root manifest, or each portable calculation entry, indexes scientific artifacts rather than a union of every scientific state RustiQ may ever persist. A representative entry has this shape: @@ -232,3 +234,413 @@ as the AO ERI artifact attribute `attributes.computation_version`. Bumping that version changes the fingerprint and causes older ERI artifacts to be reported as `invalid`, even when their inputs and storage representation are otherwise unchanged. + +## Portable `.rustiq` V1 + +A portable artifact is a ZIP archive with ZIP64 support, containing a non-empty +collection of independently reusable calculations. Single-calculation archives +use the same V1 format with exactly one entry. The normative root schema is +[`portable-manifest-v1.schema.json`](../schemas/portable-manifest-v1.schema.json). +A [golden H2 manifest](../crates/rustiq-core/tests/data/persistence/portable-manifest-h2-v1.json) +records the ordinary single-calculation case. +The directory cache retains its existing single-artifact manifest and layout. + +```text +manifest.json +sources/0 optional shared opaque provenance +calculations/calculation-0/request.json normalized requested state +calculations/calculation-0/calculation.json resolved scientific state +calculations/calculation-0/arrays/integrals/ao-eri.npy +calculations/calculation-1/request.json +calculations/calculation-1/calculation.json +... +``` + +The root contains `format: "rustiq-persistence"`, `format_version: 1`, +`kind: "portable"`, `producer`, optional `sources`, and `calculations`. +Each calculation entry contains `id`, `scientific_identity`, `request`, +`calculation`, and `artifacts`. There is no root scientific identity or artifact +map. Compatibility remains specific to each calculation and artifact. + +Writers assign deterministic identifiers `calculation-0`, `calculation-1`, ... +in collection order. Readers accept unique `calculation-` identifiers. +Identifiers are references within a snapshot, not scientific identities; neither +identifiers, source order nor user labels enter scientific compatibility. +Request/resolved references and artifacts must stay inside their entry's +`calculations//` namespace. References are full archive member paths: + +```json +"request": { + "path": "calculations/calculation-0/request.json", + "version": 1, + "size": 1234, + "digest": "sha256:..." +}, +"calculation": { + "path": "calculations/calculation-0/calculation.json", + "version": 1, + "size": 1234, + "digest": "sha256:..." +} +``` + +`size` and `digest` describe exact uncompressed bytes, including whitespace. +Each snapshot has an independently versioned schema. Producer version is +provenance, not an invalidation rule. An entry may contain no numerical artifacts. +A bundle represents the supplied calculation collection, not an append-only history. +Shared sources are stored once; numerical payload deduplication is not implemented. +The pre-release single-calculation portable layout is replaced by this V1 contract; +there is no migration reader for that unpublished layout. + +### Normalized request and optional source provenance + +The normative request schema is +[`request-snapshot-v1.schema.json`](../schemas/request-snapshot-v1.schema.json), +with an [H2 golden request](../crates/rustiq-core/tests/data/persistence/request-h2-v1.json). +`request.json` is constructed from `PreparedCalculation::request()`, not source +TOML/XYZ or the resolved snapshot. Its `format: "rustiq-request"` and `version: 1` +identify the independent wire contract. Reopening reconstructs a typed +`CalculationRequest`, without depending on original input files. + +The request preserves ordered atoms in the requested `bohr`/`angstrom` units, +charge, multiplicity, a portable basis label, requested `auto`/`rhf`/`uhf` method, +defaulted HF settings, and optional MP2 settings. The requested history size +`diis_size` is required and remains present even when DIIS is disabled. +`diis` and `diis_size` are independent requested values. The requested MP2 memory limit is tagged +`{"kind":"auto"}` or `{"kind":"fixed","bytes":...}`; no machine-resolved +memory budget is stored. A random seed may remain unspecified in the request; +this deliberately asks preparation to choose a fresh seed. The unresolved request +keeps that state while `calculation.json` records the concrete seed actually used. +Comments, source +spans, source paths, cache configuration, and output/UI settings are excluded. +The V1 wire fields remain independent of Rust struct organization: integral screening +is encoded as `hf.eri_schwarz_threshold`, orthogonalization as +`hf.linear_dependency_threshold`, and DIIS as `hf.diis` / `hf.diis_size`. +The decoder maps these to `IntegralConfig`, `OrthogonalizationConfig`, and +`DiisConfig`, respectively. +The basis label is informational; resolved AO contents define scientific identity. + +Sources are optional opaque bytes, including non-UTF-8 content. They are never +parsed to reconstruct either semantic snapshot and never enter compatibility. +The optional manifest `sources` array contains records such as: + +```json +{"original_name":"../input.toml","path":"sources/0","version":1, + "size":123,"digest":"sha256:..."} +``` + +`original_name` is informational provenance and may contain an original path. +It is never used as an extraction or publication path. Writers assign safe member +names (`sources/0`, `sources/1`, ...); readers accept safe relative members under +`sources/` and reject duplicate references. Sources have size/SHA-256 checks and +are read eagerly with a combined 64 MiB limit, at most 256 sources, and at most +4096 UTF-8 bytes per original name. Missing sources are valid for direct Rust API +use. `request.json` and `calculation.json` are required independently of sources. + +The reader validates request semantics and their consistency with the resolved +molecule, method, scientific settings, and MP2 selection. Requested Angstrom +coordinates are checked using the V1 constant `0.529177210903` Angstrom per +Bohr, rounded to binary64, and binary64 division. A resolved coordinate may differ +from that result by at most four adjacent binary64 values (ULPs), allowing final +rounding differences between independent implementations. This includes at most +four subnormal steps near zero; there is no absolute chemistry-level tolerance. +Both values and the converted result must be finite. Bohr requests compare +numerically exactly, treating signed zeros as equal. This V1 rule is independent +of domain conversion constants or dependency updates. `calculation.json` remains +authoritative for resolved coordinates and scientific identity; validation never +replaces or rounds them. HF method resolution is also frozen by V1 rather than +delegated to the current RustiQ domain policy: `auto` resolves to RHF for a +closed-shell singlet (multiplicity 1 with an even electron count) and to UHF +otherwise; explicit RHF is valid only for a closed-shell singlet, while explicit +UHF remains UHF. A missing requested random seed may resolve to a generated seed. +Neither snapshot substitutes for the other. + +### Resolved scientific snapshot + +The normative structural schema is +[`calculation-snapshot-v1.schema.json`](../schemas/calculation-snapshot-v1.schema.json). +The [H2 golden snapshot](../crates/rustiq-core/tests/data/persistence/calculation-h2-v1.json) +provides a complete example. Wire records are explicitly defined, independently +of domain Rust struct layouts and package versions. Iteration limits, DIIS sizes, +and frozen-orbital counts use unsigned 64-bit JSON integers (maximum +18446744073709551615). Domain conversions are checked in both directions; +a reader rejects values exceeding its host integer range. + +- `format: "rustiq-calculation"`, `version: 1`, `units: "bohr"` identify the + snapshot contract and coordinate unit. +- `atoms` contains ordered atomic numbers and positions; `charge` and + `multiplicity` define the molecular electronic state. +- `basis` contains effective AOs in integral-engine order. Each has a center, + ordered Cartesian components with three angular-momentum integers, and ordered + primitives with positive exponents and effective normalized coefficients. + These include any spherical-component weights already applied by the engine. + Consumers must not normalize them again. A basis name or source file is not + needed to interpret the stored AO representation. +- `hf` contains the resolved `rhf`/`uhf` method, iteration limit, convergence and + linear-dependency thresholds, Schwarz threshold, DIIS history size, and density + guess. `null`/omitted Schwarz threshold means screening disabled; + `null`/omitted DIIS size means DIIS disabled. +- Density guesses are tagged as `core_hamiltonian`, `one_electron`, `random`, or + `zero`. Perturbations and random guesses record distribution parameters and a + required concrete resolved seed. A missing or null seed is invalid in + `calculation.json`. Seeds chosen during preparation are captured here, while an + initially unspecified seed stays unspecified in `request.json`. This does not capture + generator state or promise exact SCF continuation. +- `mp2` is absent/null for HF-only requests, or contains `frozen_orbitals`. + This records the request, not an MP2 result or proof of convergence. +- `ao_eri_computation_version` states the scientific algorithm identity used for + AO ERIs; the snapshot independently reproduces `scientific-identity-v1`. + +Coordinates and all floating parameters must be finite. Exponents and positive +thresholds must be greater than zero. Electron counts, multiplicity and RHF method +must be mutually consistent. Random uniform bounds must be finite, ordered, and +have a finite width; normal standard deviation must be positive. Arrays must be +nonempty where the schema requires them. Cross-field scientific checks supplement +the structural JSON schema. Unsupported snapshot versions and unknown fields in +known snapshot records are errors. + +The snapshot excludes cache locations, source paths/spans, output presentation, +and memory budgets. Floating-point JSON parsing preserves binary64 round trips, +so reparsing a snapshot cannot change its canonical scientific identity. +The snapshot does not yet provide direct archive execution or reconstruct an SCF +restart state. + +### Rust API + +`RustiQBundle::new(Vec)` validates a non-empty portable collection. +`calculations()` and `calculations_mut()` expose entries in manifest order, with +independent typed artifact access and compatibility. The mutable slice cannot +change cardinality. `add_source` / `sources` manage shared provenance; construction +also lifts and deduplicates identical source name/byte pairs attached to entries. +`RustiQBundle::open` reads all request/resolved snapshots but leaves arrays lazy. +`write` publishes all supplied entries as one complete snapshot. + +```rust,ignore +use rustiq_core::persistence::{RustiQBundle, RustiQData}; + +let entries = prepared_calculations.iter() + .map(RustiQData::from_calculation) + .collect::, _>>()?; +let mut bundle = RustiQBundle::new(entries)?; +bundle.add_source("batch.ncl", original_source_bytes)?; +bundle.write("batch.rustiq")?; +let mut reopened = RustiQBundle::open("batch.rustiq")?; +for entry in reopened.calculations_mut() { + println!("{:?}", entry.calculation().expect("portable context").hf_method()); +} +``` + +The existing single-calculation `RustiQData::open/write` methods delegate to the +same bundle implementation. `open` rejects multi-calculation archives explicitly, +so callers cannot silently drop entries. Sources returned by `RustiQData::open` +are the single bundle's shared provenance. Bundle entries use `bundle.sources()`. + + +`RustiQData::from_calculation(&PreparedCalculation)` captures requested and resolved inputs +without executing HF. `set_eri` / `set::` supply an owned +`CompactEri` whose dimension matches those inputs. The caller is responsible for +supplying the ERIs computed for that calculation, rather than unrelated values +of the same shape. + +```rust,ignore +use rustiq_core::persistence::{CompactEri, RustiQData}; + +// `prepared` is produced by CalculationBuilder; `eri` belongs to these inputs. +let mut data = RustiQData::from_calculation(&prepared)?; +// Optional: preserve exact source bytes separately from scientific state. +data.add_source("calculation.toml", original_toml_bytes)?; +data.set_eri(eri)?; +data.write("water.rustiq")?; + +let mut restored = RustiQData::open("water.rustiq")?; +let context = restored.calculation().expect("portable context"); +println!("{:?}: {} AOs", context.hf_method(), context.basis().len()); +for (name, representation) in restored.artifact_representations() { + println!("{name}: {representation}"); +} +if restored.eri_is_compatible(&prepared) { + let eri: &CompactEri = restored.read_eri()?; +} +``` + +`request()` exposes the reconstructed typed normalized request. `sources()` +exposes exact captured bytes and informational original names. `calculation()` +exposes read-only typed resolved context, including atoms, effective basis, +HF settings and MP2 request. `producer()` exposes provenance, and +`artifact_representations()` lists known and unknown artifacts without loading +arrays. These inspections do not certify payload integrity. + +`open` requires only read access to the source. It validates the container index, +manifest, both snapshot versions/digests/schemas, source sizes/digests, scientific +identity, and declared artifact sizes. ERI dimensions and computation-version +metadata must agree with the snapshot. It does not decode numerical arrays. +The first typed read verifies SHA-256, bounded NPY header, f64 dtype, shape and +exact payload length before decoding. Further reads return the same object. +Missing ERIs are represented by `get::() == Ok(None)`; an +unsupported representation is never silently treated as usable scientific data. + +`eri_is_compatible` compares effective integral inputs with another prepared +calculation without loading arrays. Changing an HF-only request to MP2 does not +invalidate AO ERIs. Compatibility does not imply that the payload has already +been verified. There is no CLI reuse orchestration in this API release. + +Every new archive written by RustiQ records producer name `RustiQ` and the +current package version, including when rewriting an older or foreign archive. +Producer provenance does not change scientific identity. + +Writing a reopened archive preserves both semantic snapshots and exact captured +source bytes. It preserves unloaded artifacts by verified streaming +copy, including unknown representations and their attributes. Known ERI headers +are checked before preservation; overwritten ERIs use the current representation. +Errors use `PortableError` (including `AlreadyExists`, unsupported version, +invalid context/archive, I/O and artifact errors), without public ZIP types. + +### Publication and container policy + +Writing creates a sibling temporary file, writes snapshot and artifacts with +SHA-256 metadata, writes the manifest, finalizes the ZIP, syncs the file, and +reopens it to validate the final index/context. Publication uses an atomic +no-clobber operation. **An existing destination is never replaced**, including +when two producers race to create it. Unix additionally syncs the parent +directory after publication. Failures before publication leave the destination +untouched and clean up the temporary file. A directory-sync error after +publication can leave a complete output file and is still reported as an error. + +Writers use `Stored` for numerical/opaque artifacts and `Deflated` for +JSON members (`manifest.json`, `request.json`, and `calculation.json`). Streaming +payloads reserve ZIP64 local-header fields even for small arrays, allowing growth +past 4 GiB without buffering. ZIP64 central/end +records are emitted when required. Timestamps are fixed to 1980-01-01 00:00:00, +permissions to regular files with mode 0644, and member order is deterministic. +Repeated writes of the same data by the same implementation produce identical +bytes; compression-library changes are not promised byte-identical output and +never change scientific identity. + +### Defensive reading limits + +Portable V1 accepts only single-disk ZIP archives with regular file members; +directory entries are unnecessary and rejected. Supported compression is Stored +or Deflated. Encryption, links, devices and other special entry types are errors. +The reader validates original central-directory records before the ZIP library +indexes them, so duplicate names cannot be silently collapsed. + +Limits applied before parsing or allocating payloads: + +| Structure | Limit | +| --- | ---: | +| Uncompressed manifest | 1 MiB | +| Uncompressed request snapshot, per calculation | 64 MiB | +| Uncompressed calculation snapshot, per calculation | 64 MiB | +| Combined shared sources | 64 MiB | +| Members | 4,096 | +| Central directory | 16 MiB | +| ZIP64 end-record body | 64 KiB | +| NPY header body | 64 KiB | + +All member paths obey the logical path rules above. File/ancestor conflicts and +case-insensitive collisions are rejected; metadata paths are reserved. Names must +be valid UTF-8; non-ASCII names require the ZIP UTF-8 flag. Alternative Unicode +path extra fields and duplicate extra fields are rejected to avoid competing +names. Local/central names, flags and compression must agree. Invalid offsets, +overlapping payloads, truncated directory records, and inconsistent ZIP64 fields +are rejected. ZIP metadata names are not extracted onto the filesystem. + +Actual decompressed length and ZIP CRC are checked, in addition to independent +SHA-256 digests. Streamed members cannot exceed their declared uncompressed size. +NPY object/pickle arrays, unsupported dtype/rank, dimension overflow, extra bytes, +and incomplete arrays are rejected. Large legitimate ERIs still require memory +for the decoded `CompactEri`; there is no artificial 4 GiB payload limit. + +### Interoperability fixtures + +`tools/reference/generate_portable_fixtures.py` regenerates Python-produced +ZIP64 fixtures using only standard-library modules and the existing NumPy byte +fixtures. Single- and multi-calculation bundles, shared sources, and both little- and +big-endian f64 payloads are covered. The Python tests +independently reproduce the scientific identity and verify ZIP/JSON/NumPy output +written by Rust. Rust tests also exercise a sparse archive with offsets beyond +4 GiB without allocating a multi-gigabyte payload. + +Run `uv run --locked pytest tools/reference` for interoperability and scientific +reference comparisons, separately from the Cargo suite. + +## Single-calculation execution APIs + +Portable V1 can contain multiple calculations. The current convenience execution +API operates on one calculation, so `RustiQData` represents exactly one entry and +`RustiQData::open` rejects multi-calculation archives. Newly written single-entry +archives use `calculation-0` within the normal V1 bundle layout. Callers should +neither depend on nor manipulate the numeric index. + +```rust +// Calculation -> .rustiq +let prepared = CalculationBuilder::new(&geometry, &basis_file).prepare()?; +let hf = prepared.run_hf()?; +let mut data = RustiQData::from_calculation(&prepared)?; +data.write_with_eri("water.rustiq", hf.ao_eri())?; +``` + +The writer borrows the retained AO ERI without cloning or recomputing it. +`set_eri` remains available when the caller owns the tensor. + +```rust +// .rustiq -> Calculation +let data = RustiQData::open("water.rustiq")?; +let prepared = data.prepare_calculation()?; +let result = prepared.execute()?; +let eri = result.hf.ao_eri(); +``` + +`prepare_calculation` consumes the data facade and restores the resolved molecule, +AO basis, and scientific options. It retains the archive as a read-only artifact +source. Payloads are decoded lazily when execution needs them. Compatible AO ERIs +are verified and reused automatically by `execute`, `execute_with_events`, and +`run_hf`; missing or incompatible artifacts fall back to the enabled local cache +and then computation. Corrupt or unsupported compatible payloads fail explicitly. +The source archive is never modified. + +A new calculation can also reuse a previous archive with different requested options: + +```rust +let prepared = CalculationBuilder::new(&geometry, &basis_file) + .with_mp2(Mp2Config::default()) + .prepare()? + .with_reuse_data(RustiQData::open("water-hf.rustiq")?)?; +let result = prepared.execute()?; +``` + +The execution interface is independent of the input source. An internal enum +distinguishes configuration-only execution from portable artifact reuse. The +resolver uses the existing sealed `Artifact` markers for presence, compatibility, +and loading, rather than archive paths or a public storage/backend trait. Future +HF solution and restart markers can participate at their respective stages through +the same resolver; those representations are not implemented yet. + +`CalculationEvent::ArtifactReuse` reports per-artifact reused, missing, or +incompatible decisions. Its decision type also distinguishes intentionally ignored +state for future selection policies. Computed and reused values are retained in a +separate in-memory typed data facade for subsequent executions. Compact ERI clones +share immutable storage; mutation detaches that storage. Neither ordinary reuse nor +this memory retention rewrites the source snapshot or stores runtime scratch data. + +For callers managing tensors explicitly, `take_compatible_eri`, +`execute_with_eri`, and `run_hf_with_eri` remain available. This explicit path +rejects missing, incompatible, corrupt, or unsupported artifacts, and validates +tensor dimensions. ERIs remain accessible in the HF solution and are reused by MP2. + +A complete H2/STO-3G example set is in +[`crates/rustiq-core/examples`](../crates/rustiq-core/examples). Each file shows +one workflow. `calculate_from_rustiq` uses only the archive; the other examples +load geometry and basis from repository sample and test data: + +```sh +cargo run -p rustiq-core --example hf_calculation +cargo run -p rustiq-core --example write_rustiq +cargo run -p rustiq-core --example read_rustiq +cargo run -p rustiq-core --example calculate_from_rustiq +cargo run -p rustiq-core --example copy_rustiq +``` + +The examples read and write archives in the ignored +`crates/rustiq-core/examples/data/` directory. The read examples expect +`h2.rustiq` to have been created by `write_rustiq`. diff --git a/schemas/calculation-snapshot-v1.schema.json b/schemas/calculation-snapshot-v1.schema.json new file mode 100644 index 00000000..60bf843d --- /dev/null +++ b/schemas/calculation-snapshot-v1.schema.json @@ -0,0 +1,376 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/mveril/RustiQ/blob/main/schemas/calculation-snapshot-v1.schema.json", + "title": "RustiQ resolved scientific calculation V1", + "type": "object", + "additionalProperties": false, + "required": [ + "format", + "version", + "units", + "atoms", + "charge", + "multiplicity", + "basis", + "hf", + "ao_eri_computation_version" + ], + "properties": { + "format": { + "const": "rustiq-calculation" + }, + "version": { + "const": 1 + }, + "units": { + "const": "bohr" + }, + "atoms": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/atom" + } + }, + "charge": { + "type": "integer", + "minimum": -2147483648, + "maximum": 2147483647 + }, + "multiplicity": { + "type": "integer", + "minimum": 1, + "maximum": 255 + }, + "basis": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/ao" + } + }, + "hf": { + "$ref": "#/$defs/hf" + }, + "mp2": { + "anyOf": [ + { + "$ref": "#/$defs/mp2" + }, + { + "type": "null" + } + ] + }, + "ao_eri_computation_version": { + "type": "integer", + "minimum": 1, + "maximum": 4294967295 + } + }, + "$defs": { + "atom": { + "type": "object", + "additionalProperties": false, + "required": [ + "atomic_number", + "position" + ], + "properties": { + "atomic_number": { + "type": "integer", + "minimum": 1, + "maximum": 118 + }, + "position": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "items": { + "type": "number" + } + } + } + }, + "primitive": { + "type": "object", + "additionalProperties": false, + "required": [ + "exponent", + "coefficient" + ], + "properties": { + "exponent": { + "type": "number", + "exclusiveMinimum": 0 + }, + "coefficient": { + "type": "number" + } + } + }, + "component": { + "type": "object", + "additionalProperties": false, + "required": [ + "angular_momentum", + "primitives" + ], + "properties": { + "angular_momentum": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "items": { + "type": "integer", + "minimum": 0, + "maximum": 255 + } + }, + "primitives": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/primitive" + } + } + } + }, + "ao": { + "type": "object", + "additionalProperties": false, + "required": [ + "center", + "components" + ], + "properties": { + "center": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "items": { + "type": "number" + } + }, + "components": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/component" + } + } + } + }, + "distribution": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "min", + "max" + ], + "properties": { + "kind": { + "const": "uniform" + }, + "min": { + "type": "number" + }, + "max": { + "type": "number" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "mean", + "std_dev" + ], + "properties": { + "kind": { + "const": "normal" + }, + "mean": { + "type": "number" + }, + "std_dev": { + "type": "number", + "exclusiveMinimum": 0 + } + } + } + ] + }, + "random": { + "type": "object", + "additionalProperties": false, + "required": [ + "seed", + "distribution" + ], + "properties": { + "seed": { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615 + }, + "distribution": { + "$ref": "#/$defs/distribution" + } + } + }, + "guess": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "core_hamiltonian" + }, + "perturbation": { + "anyOf": [ + { + "$ref": "#/$defs/random" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "one_electron" + }, + "perturbation": { + "anyOf": [ + { + "$ref": "#/$defs/random" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "random" + ], + "properties": { + "kind": { + "const": "random" + }, + "random": { + "$ref": "#/$defs/random" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "zero" + } + } + } + ] + }, + "hf": { + "type": "object", + "additionalProperties": false, + "required": [ + "method", + "max_iterations", + "convergence_threshold", + "linear_dependency_threshold", + "guess" + ], + "properties": { + "method": { + "enum": [ + "rhf", + "uhf" + ] + }, + "max_iterations": { + "type": "integer", + "minimum": 1, + "maximum": 18446744073709551615 + }, + "convergence_threshold": { + "type": "number", + "exclusiveMinimum": 0 + }, + "linear_dependency_threshold": { + "type": "number", + "minimum": 0 + }, + "eri_schwarz_threshold": { + "anyOf": [ + { + "type": "number", + "exclusiveMinimum": 0 + }, + { + "type": "null" + } + ] + }, + "diis_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 2, + "maximum": 18446744073709551615 + }, + { + "type": "null" + } + ] + }, + "guess": { + "$ref": "#/$defs/guess" + } + } + }, + "mp2": { + "type": "object", + "additionalProperties": false, + "required": [ + "frozen_orbitals" + ], + "properties": { + "frozen_orbitals": { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615 + } + } + } + } +} diff --git a/schemas/portable-manifest-v1.schema.json b/schemas/portable-manifest-v1.schema.json new file mode 100644 index 00000000..048c4334 --- /dev/null +++ b/schemas/portable-manifest-v1.schema.json @@ -0,0 +1,229 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "RustiQ portable manifest V1", + "type": "object", + "additionalProperties": false, + "required": [ + "format", + "format_version", + "kind", + "producer", + "calculations" + ], + "properties": { + "format": { + "const": "rustiq-persistence" + }, + "format_version": { + "const": 1 + }, + "kind": { + "const": "portable" + }, + "producer": { + "type": "object", + "required": [ + "name", + "version" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "sources": { + "type": "array", + "maxItems": 256, + "items": { + "$ref": "#/$defs/source" + } + }, + "calculations": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/calculation" + } + } + }, + "$defs": { + "digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "snapshot": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "version", + "size", + "digest" + ], + "properties": { + "path": { + "type": "string" + }, + "version": { + "const": 1 + }, + "size": { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615 + }, + "digest": { + "$ref": "#/$defs/digest" + } + } + }, + "identity": { + "type": "object", + "required": [ + "version", + "digest" + ], + "properties": { + "version": { + "const": 1 + }, + "digest": { + "$ref": "#/$defs/digest" + } + } + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": [ + "original_name", + "path", + "version", + "size", + "digest" + ], + "properties": { + "original_name": { + "type": "string" + }, + "path": { + "type": "string", + "pattern": "^sources/[^\\\\]+$" + }, + "version": { + "const": 1 + }, + "size": { + "type": "integer", + "minimum": 0, + "maximum": 67108864 + }, + "digest": { + "$ref": "#/$defs/digest" + } + } + }, + "artifact": { + "type": "object", + "required": [ + "path", + "size", + "representation", + "digest" + ], + "properties": { + "path": { + "type": "string" + }, + "size": { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615 + }, + "representation": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/digest" + }, + "attributes": { + "type": "object" + } + }, + "allOf": [ + { + "if": { + "properties": { + "representation": { + "const": "rustiq-compact-eri-v1" + } + } + }, + "then": { + "required": [ + "attributes" + ], + "properties": { + "attributes": { + "type": "object", + "additionalProperties": false, + "required": [ + "basis_functions", + "computation_version" + ], + "properties": { + "basis_functions": { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615 + }, + "computation_version": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295 + } + } + } + } + } + } + ] + }, + "calculation": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "scientific_identity", + "request", + "calculation", + "artifacts" + ], + "properties": { + "id": { + "type": "string", + "pattern": "^calculation-[0-9]+$" + }, + "scientific_identity": { + "$ref": "#/$defs/identity" + }, + "request": { + "$ref": "#/$defs/snapshot" + }, + "calculation": { + "$ref": "#/$defs/snapshot" + }, + "artifacts": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/artifact" + } + } + } + } + } +} diff --git a/schemas/request-snapshot-v1.schema.json b/schemas/request-snapshot-v1.schema.json new file mode 100644 index 00000000..6a6c291a --- /dev/null +++ b/schemas/request-snapshot-v1.schema.json @@ -0,0 +1,340 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/mveril/RustiQ/blob/main/schemas/request-snapshot-v1.schema.json", + "title": "RustiQ normalized requested calculation V1", + "type": "object", + "additionalProperties": false, + "required": [ + "format", + "version", + "units", + "atoms", + "charge", + "multiplicity", + "basis", + "hf" + ], + "properties": { + "format": { + "const": "rustiq-request" + }, + "version": { + "const": 1 + }, + "units": { + "enum": [ + "bohr", + "angstrom" + ] + }, + "atoms": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/atom" + } + }, + "charge": { + "type": "integer", + "minimum": -2147483648, + "maximum": 2147483647 + }, + "multiplicity": { + "type": "integer", + "minimum": 1, + "maximum": 255 + }, + "basis": { + "type": "string" + }, + "hf": { + "$ref": "#/$defs/hf" + }, + "mp2": { + "anyOf": [ + { + "$ref": "#/$defs/mp2" + }, + { + "type": "null" + } + ] + } + }, + "$defs": { + "atom": { + "type": "object", + "additionalProperties": false, + "required": [ + "atomic_number", + "position" + ], + "properties": { + "atomic_number": { + "type": "integer", + "minimum": 1, + "maximum": 118 + }, + "position": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "items": { + "type": "number" + } + } + } + }, + "distribution": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "min", + "max" + ], + "properties": { + "kind": { + "const": "uniform" + }, + "min": { + "type": "number" + }, + "max": { + "type": "number" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "mean", + "std_dev" + ], + "properties": { + "kind": { + "const": "normal" + }, + "mean": { + "type": "number" + }, + "std_dev": { + "type": "number", + "exclusiveMinimum": 0 + } + } + } + ] + }, + "random": { + "type": "object", + "additionalProperties": false, + "required": [ + "distribution" + ], + "properties": { + "seed": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615 + }, + { + "type": "null" + } + ] + }, + "distribution": { + "$ref": "#/$defs/distribution" + } + } + }, + "guess": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "core_hamiltonian" + }, + "perturbation": { + "anyOf": [ + { + "$ref": "#/$defs/random" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "one_electron" + }, + "perturbation": { + "anyOf": [ + { + "$ref": "#/$defs/random" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "random" + ], + "properties": { + "kind": { + "const": "random" + }, + "random": { + "$ref": "#/$defs/random" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "zero" + } + } + } + ] + }, + "hf": { + "type": "object", + "additionalProperties": false, + "required": [ + "method", + "max_iterations", + "convergence_threshold", + "linear_dependency_threshold", + "guess", + "diis", + "diis_size" + ], + "properties": { + "method": { + "enum": [ + "auto", + "rhf", + "uhf" + ] + }, + "max_iterations": { + "type": "integer", + "minimum": 1, + "maximum": 18446744073709551615 + }, + "convergence_threshold": { + "type": "number", + "exclusiveMinimum": 0 + }, + "linear_dependency_threshold": { + "type": "number", + "minimum": 0 + }, + "eri_schwarz_threshold": { + "anyOf": [ + { + "type": "number", + "exclusiveMinimum": 0 + }, + { + "type": "null" + } + ] + }, + "diis_size": { + "type": "integer", + "minimum": 2, + "maximum": 18446744073709551615 + }, + "guess": { + "$ref": "#/$defs/guess" + }, + "diis": { + "type": "boolean" + } + } + }, + "mp2": { + "type": "object", + "additionalProperties": false, + "required": [ + "frozen_orbitals", + "memory_limit" + ], + "properties": { + "frozen_orbitals": { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615 + }, + "memory_limit": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "auto" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "bytes" + ], + "properties": { + "kind": { + "const": "fixed" + }, + "bytes": { + "type": "integer", + "minimum": 0, + "maximum": 18446744073709551615 + } + } + } + ] + } + } + } + } +} diff --git a/tools/reference/generate_portable_fixtures.py b/tools/reference/generate_portable_fixtures.py new file mode 100644 index 00000000..cc35c292 --- /dev/null +++ b/tools/reference/generate_portable_fixtures.py @@ -0,0 +1,267 @@ +"""Regenerate small Python ZIP64 fixtures using the documented V1 wire format. + +Uses only the standard library and the checked-in NumPy payload fixtures. +""" + +import hashlib +import io +import json +import struct +import zipfile +from pathlib import Path + +FIXTURES = ( + Path(__file__).resolve().parents[2] / "crates/rustiq-core/tests/data/persistence" +) + + +def scientific_identity(snapshot: dict) -> str: + canonical = bytearray() + + def length(value: int) -> None: + canonical.extend(struct.pack(">Q", value)) + + def text(value: str) -> None: + encoded = value.encode("utf-8") + length(len(encoded)) + canonical.extend(encoded) + + def number(value: float) -> None: + canonical.extend(struct.pack(">d", 0.0 if value == 0 else value)) + + text("scientific-identity-v1") + text("ao-eri-computation-version") + canonical.extend(struct.pack(">I", snapshot["ao_eri_computation_version"])) + text("rustiq-compact-eri-v1") + length(len(snapshot["atoms"])) + for atom in snapshot["atoms"]: + canonical.extend(struct.pack(">I", atom["atomic_number"])) + for coordinate in atom["position"]: + number(coordinate) + length(len(snapshot["basis"])) + for ao in snapshot["basis"]: + for coordinate in ao["center"]: + number(coordinate) + length(len(ao["components"])) + for component in ao["components"]: + canonical.extend(bytes(component["angular_momentum"])) + length(len(component["primitives"])) + for primitive in component["primitives"]: + number(primitive["exponent"]) + number(primitive["coefficient"]) + threshold = snapshot["hf"]["eri_schwarz_threshold"] + canonical.append(int(threshold is not None)) + if threshold is not None: + number(threshold) + return digest(canonical) + + +def digest(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def portable_manifest(manifest: dict) -> dict: + entry = {"id": "calculation-0"} + for field in ("scientific_identity", "request", "calculation", "artifacts"): + entry[field] = manifest.pop(field) + for field in ("request", "calculation"): + entry[field]["path"] = "calculations/calculation-0/" + entry[field]["path"] + for artifact in entry["artifacts"].values(): + artifact["path"] = "calculations/calculation-0/" + artifact["path"] + manifest["calculations"] = [entry] + return manifest + + +def generate() -> None: + snapshot = (FIXTURES / "calculation-h2-v1.json").read_bytes() + request = (FIXTURES / "request-h2-v1.json").read_bytes() + source = bytes.fromhex((FIXTURES / "source-original-v1.toml.hex").read_text()) + for suffix, npy_fixture in [ + ("little", "ao-eri-python-v1.npy.hex"), + ("big", "ao-eri-python-big-endian-v1.npy.hex"), + ]: + payload = bytes.fromhex((FIXTURES / npy_fixture).read_text()) + manifest = { + "format": "rustiq-persistence", + "format_version": 1, + "kind": "portable", + "producer": {"name": "Python interoperability fixture", "version": "1"}, + "scientific_identity": { + "version": 1, + "digest": scientific_identity(json.loads(snapshot)), + }, + "request": { + "path": "request.json", + "version": 1, + "size": len(request), + "digest": digest(request), + }, + "calculation": { + "path": "calculation.json", + "version": 1, + "size": len(snapshot), + "digest": digest(snapshot), + }, + "sources": [ + { + "original_name": "../../calculation.toml", + "path": "sources/0", + "version": 1, + "size": len(source), + "digest": digest(source), + } + ], + "artifacts": { + "ao_eri": { + "path": "arrays/integrals/ao-eri.npy", + "size": len(payload), + "digest": digest(payload), + "representation": "rustiq-compact-eri-v1", + "attributes": {"basis_functions": 2, "computation_version": 1}, + } + }, + } + manifest = portable_manifest(manifest) + output = io.BytesIO() + # Force full ZIP64 EOCD/locator/central extras on small fixtures. + previous_limit = zipfile.ZIP64_LIMIT + try: + zipfile.ZIP64_LIMIT = 0 + with zipfile.ZipFile(output, "w", allowZip64=True) as archive: + for name, content in [ + ("manifest.json", json.dumps(manifest, indent=2).encode() + b"\n"), + ("calculations/calculation-0/calculation.json", snapshot), + ("calculations/calculation-0/request.json", request), + ("sources/0", source), + ("calculations/calculation-0/arrays/integrals/ao-eri.npy", payload), + ]: + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = 0o100644 << 16 + info.compress_type = ( + zipfile.ZIP_STORED + if name.endswith(".npy") + else zipfile.ZIP_DEFLATED + ) + with archive.open(info, "w", force_zip64=True) as member: + member.write(content) + finally: + zipfile.ZIP64_LIMIT = previous_limit + (FIXTURES / f"portable-python-{suffix}-v1.rustiq.hex").write_text( + output.getvalue().hex() + "\n", encoding="ascii" + ) + + +def generate_angstrom() -> None: + """Produce coordinates independently, rounding only after Decimal division.""" + from decimal import Decimal, localcontext + + request = json.loads((FIXTURES / "request-h2-v1.json").read_bytes()) + snapshot = json.loads((FIXTURES / "calculation-h2-v1.json").read_bytes()) + request["units"] = "angstrom" + request["atoms"][1]["position"] = [0.74, -0.13, 0.0] + with localcontext() as context: + context.prec = 100 + coordinates = [ + float(Decimal.from_float(x) / Decimal("0.529177210903")) + for x in request["atoms"][1]["position"] + ] + snapshot["atoms"][1]["position"] = coordinates + snapshot["basis"][1]["center"] = coordinates + request_bytes = json.dumps(request, indent=2).encode() + b"\n" + snapshot_bytes = json.dumps(snapshot, indent=2).encode() + b"\n" + manifest = { + "format": "rustiq-persistence", + "format_version": 1, + "kind": "portable", + "producer": {"name": "Python Decimal interoperability fixture", "version": "1"}, + "scientific_identity": {"version": 1, "digest": scientific_identity(snapshot)}, + "request": { + "path": "request.json", + "version": 1, + "size": len(request_bytes), + "digest": digest(request_bytes), + }, + "calculation": { + "path": "calculation.json", + "version": 1, + "size": len(snapshot_bytes), + "digest": digest(snapshot_bytes), + }, + "artifacts": {}, + } + manifest = portable_manifest(manifest) + output = io.BytesIO() + with zipfile.ZipFile(output, "w", allowZip64=True) as archive: + for name, content in [ + ("manifest.json", json.dumps(manifest, indent=2).encode() + b"\n"), + ("calculations/calculation-0/request.json", request_bytes), + ("calculations/calculation-0/calculation.json", snapshot_bytes), + ]: + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = 0o100644 << 16 + info.compress_type = zipfile.ZIP_DEFLATED + archive.writestr(info, content) + (FIXTURES / "portable-python-angstrom-v1.rustiq.hex").write_text( + output.getvalue().hex() + "\n", encoding="ascii" + ) + + +def generate_multi() -> None: + """Two independent entries share source provenance and use opposite NPY byte orders.""" + + def write_member(target: zipfile.ZipFile, name: str, content: bytes) -> None: + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = 0o100644 << 16 + info.compress_type = ( + zipfile.ZIP_STORED if name.endswith(".npy") else zipfile.ZIP_DEFLATED + ) + target.writestr(info, content) + + output = io.BytesIO() + combined = None + with zipfile.ZipFile(output, "w", allowZip64=True) as target: + for index, order in enumerate(("little", "big")): + content = bytes.fromhex( + (FIXTURES / f"portable-python-{order}-v1.rustiq.hex").read_text() + ) + with zipfile.ZipFile(io.BytesIO(content)) as source: + manifest = json.loads(source.read("manifest.json")) + entry = manifest["calculations"][0] + old_prefix = "calculations/calculation-0/" + new_prefix = f"calculations/calculation-{index}/" + entry["id"] = f"calculation-{index}" + for reference in ( + entry["request"], + entry["calculation"], + *entry["artifacts"].values(), + ): + reference["path"] = reference["path"].replace( + old_prefix, new_prefix + ) + if combined is None: + combined = manifest + write_member(target, "sources/0", source.read("sources/0")) + else: + combined["calculations"].append(entry) + for name in source.namelist(): + if name.startswith(old_prefix): + write_member( + target, + name.replace(old_prefix, new_prefix), + source.read(name), + ) + write_member( + target, "manifest.json", json.dumps(combined, indent=2).encode() + b"\n" + ) + (FIXTURES / "portable-python-multi-v1.rustiq.hex").write_text( + output.getvalue().hex() + "\n", encoding="ascii" + ) + + +if __name__ == "__main__": + generate() + generate_angstrom() + generate_multi() diff --git a/tools/reference/test_persistence_interop.py b/tools/reference/test_persistence_interop.py index 2753e5e8..0eb906cc 100644 --- a/tools/reference/test_persistence_interop.py +++ b/tools/reference/test_persistence_interop.py @@ -1,7 +1,7 @@ import io import os -from pathlib import Path import subprocess +from pathlib import Path import numpy as np @@ -56,3 +56,218 @@ def test_python_numpy_big_endian_ao_eri_fixture() -> None: assert values.dtype == np.dtype(">f8") assert values.shape == (6,) np.testing.assert_array_equal(values, [0.5, 1.5, 2.5, 3.5, 4.5, 5.5]) + + +def test_numpy_reads_rust_portable_archive(tmp_path: Path) -> None: + import json + import zipfile + + from generate_portable_fixtures import digest, scientific_identity + + output = tmp_path / "h2.rustiq" + environment = os.environ.copy() + environment["RUSTIQ_ARCHIVE_TEST_OUTPUT"] = str(output) + subprocess.run( + [ + "cargo", + "test", + "-p", + "rustiq-core", + "--no-default-features", + "persistence::data::portable::tests::writes_archive_for_python_interoperability", + "--", + "--ignored", + "--exact", + ], + check=True, + cwd=ROOT, + env=environment, + ) + with zipfile.ZipFile(output) as archive: + assert archive.testzip() is None + assert set(archive.namelist()) == { + "manifest.json", + "calculations/calculation-0/calculation.json", + "calculations/calculation-0/request.json", + "sources/0", + "calculations/calculation-0/arrays/integrals/ao-eri.npy", + } + manifest = json.loads(archive.read("manifest.json")) + request_bytes = archive.read("calculations/calculation-0/request.json") + request = json.loads(request_bytes) + assert manifest["calculations"][0]["request"]["digest"] == digest(request_bytes) + assert manifest["calculations"][0]["request"]["size"] == len(request_bytes) + assert request["hf"]["method"] == "auto" + assert request["units"] == "bohr" + source_bytes = archive.read("sources/0") + assert source_bytes == bytes.fromhex( + (FIXTURE_DIR / "source-original-v1.toml.hex").read_text() + ) + assert manifest["sources"][0]["digest"] == digest(source_bytes) + assert manifest["sources"][0]["original_name"] == "../../calculation.toml" + snapshot_bytes = archive.read("calculations/calculation-0/calculation.json") + snapshot = json.loads(snapshot_bytes) + assert manifest["calculations"][0]["scientific_identity"][ + "digest" + ] == scientific_identity(snapshot) + assert manifest["calculations"][0]["calculation"]["digest"] == digest( + snapshot_bytes + ) + assert snapshot["hf"]["method"] == "rhf" + assert snapshot["units"] == "bohr" + payload = archive.read("calculations/calculation-0/arrays/integrals/ao-eri.npy") + assert manifest["calculations"][0]["artifacts"]["ao_eri"]["digest"] == digest( + payload + ) + _assert_ao_eri_values(np.load(io.BytesIO(payload), allow_pickle=False)) + assert ( + archive.getinfo( + "calculations/calculation-0/arrays/integrals/ao-eri.npy" + ).compress_type + == zipfile.ZIP_STORED + ) + assert all( + info.date_time == (1980, 1, 1, 0, 0, 0) for info in archive.infolist() + ) + + +def test_python_portable_golden_fixtures() -> None: + import json + import zipfile + + from generate_portable_fixtures import digest, scientific_identity + + for byte_order in ("little", "big"): + content = bytes.fromhex( + (FIXTURE_DIR / f"portable-python-{byte_order}-v1.rustiq.hex").read_text() + ) + assert b"PK\x06\x06" in content + with zipfile.ZipFile(io.BytesIO(content)) as archive: + manifest = json.loads(archive.read("manifest.json")) + source_bytes = archive.read("sources/0") + assert source_bytes == bytes.fromhex( + (FIXTURE_DIR / "source-original-v1.toml.hex").read_text() + ) + assert digest(source_bytes) == manifest["sources"][0]["digest"] + request_bytes = archive.read("calculations/calculation-0/request.json") + assert ( + digest(request_bytes) + == manifest["calculations"][0]["request"]["digest"] + ) + assert json.loads(request_bytes)["hf"]["method"] == "auto" + snapshot = json.loads( + archive.read("calculations/calculation-0/calculation.json") + ) + assert ( + scientific_identity(snapshot) + == manifest["calculations"][0]["scientific_identity"]["digest"] + ) + payload = archive.read( + "calculations/calculation-0/arrays/integrals/ao-eri.npy" + ) + assert ( + digest(payload) + == manifest["calculations"][0]["artifacts"]["ao_eri"]["digest"] + ) + _assert_ao_eri_values(np.load(io.BytesIO(payload), allow_pickle=False)) + + +def test_python_angstrom_conversion_fixture() -> None: + import json + import zipfile + from decimal import Decimal, localcontext + + from generate_portable_fixtures import digest, scientific_identity + + content = bytes.fromhex( + (FIXTURE_DIR / "portable-python-angstrom-v1.rustiq.hex").read_text() + ) + with zipfile.ZipFile(io.BytesIO(content)) as archive: + manifest = json.loads(archive.read("manifest.json")) + request_bytes = archive.read("calculations/calculation-0/request.json") + snapshot_bytes = archive.read("calculations/calculation-0/calculation.json") + request = json.loads(request_bytes) + snapshot = json.loads(snapshot_bytes) + assert request["units"] == "angstrom" + assert snapshot["units"] == "bohr" + assert digest(request_bytes) == manifest["calculations"][0]["request"]["digest"] + assert ( + digest(snapshot_bytes) + == manifest["calculations"][0]["calculation"]["digest"] + ) + assert ( + scientific_identity(snapshot) + == manifest["calculations"][0]["scientific_identity"]["digest"] + ) + with localcontext() as context: + context.prec = 100 + for requested, resolved in zip( + request["atoms"][1]["position"], + snapshot["atoms"][1]["position"], + strict=True, + ): + assert resolved == float( + Decimal.from_float(requested) / Decimal("0.529177210903") + ) + assert snapshot["atoms"][1]["position"][0] != 0.74 / 0.529177210903 + + +def _assert_multi_archive(content: bytes) -> None: + import json + import zipfile + + from generate_portable_fixtures import digest, scientific_identity + + with zipfile.ZipFile(io.BytesIO(content)) as archive: + assert archive.testzip() is None + manifest = json.loads(archive.read("manifest.json")) + assert "scientific_identity" not in manifest + assert len(manifest["calculations"]) == 2 + assert len(manifest["sources"]) == 1 + source = manifest["sources"][0] + assert digest(archive.read(source["path"])) == source["digest"] + for index, entry in enumerate(manifest["calculations"]): + assert entry["id"] == f"calculation-{index}" + for field in ("request", "calculation"): + reference = entry[field] + payload = archive.read(reference["path"]) + assert len(payload) == reference["size"] + assert digest(payload) == reference["digest"] + snapshot = json.loads(archive.read(entry["calculation"]["path"])) + assert ( + scientific_identity(snapshot) == entry["scientific_identity"]["digest"] + ) + reference = entry["artifacts"]["ao_eri"] + payload = archive.read(reference["path"]) + assert digest(payload) == reference["digest"] + _assert_ao_eri_values(np.load(io.BytesIO(payload), allow_pickle=False)) + + +def test_python_multi_calculation_fixture() -> None: + content = bytes.fromhex( + (FIXTURE_DIR / "portable-python-multi-v1.rustiq.hex").read_text() + ) + _assert_multi_archive(content) + + +def test_numpy_reads_rust_multi_calculation_archive(tmp_path: Path) -> None: + output = tmp_path / "batch.rustiq" + environment = os.environ.copy() + environment["RUSTIQ_ARCHIVE_TEST_OUTPUT"] = str(output) + subprocess.run( + [ + "cargo", + "test", + "-p", + "rustiq-core", + "--no-default-features", + "persistence::data::portable::tests::writes_bundle_for_python_interoperability", + "--", + "--ignored", + "--exact", + ], + check=True, + cwd=ROOT, + env=environment, + ) + _assert_multi_archive(output.read_bytes())