From acbea8fced8fa56d3435bd125fc49b4eaa03683b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sat, 26 Sep 2026 19:36:42 +0200 Subject: [PATCH 01/62] Add typed scientific persistence artifacts --- crates/rustiq-core/src/persistence/data.rs | 499 ++++++++++++++++++ .../rustiq-core/src/persistence/eri_cache.rs | 116 +--- crates/rustiq-core/src/persistence/mod.rs | 19 + crates/rustiq-core/src/persistence/npy.rs | 236 ++++++++- docs/persistence-format-v1.md | 19 + 5 files changed, 781 insertions(+), 108 deletions(-) create mode 100644 crates/rustiq-core/src/persistence/data.rs diff --git a/crates/rustiq-core/src/persistence/data.rs b/crates/rustiq-core/src/persistence/data.rs new file mode 100644 index 0000000..0246ca5 --- /dev/null +++ b/crates/rustiq-core/src/persistence/data.rs @@ -0,0 +1,499 @@ +use std::{ + collections::{BTreeMap, HashSet}, + fs::{self, File}, + io::{BufReader, BufWriter, Seek, Write}, + path::{Component, Path, PathBuf}, +}; + +use crate::{ + basis::Basis, config::validated::PositiveFiniteF64, eri::CompactEri, + molecules::geometry::Geometry, +}; + +use super::{ + ao_eri_identity, sha256_reader, AoEriAttributes, ArtifactAttributes, ArtifactManifest, + Manifest, NpyConvert, PersistenceError, Producer, ScientificIdentityManifest, + AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, + FORMAT_VERSION, MANIFEST_PATH, +}; + +pub(crate) const AO_ERI_ARTIFACT: &str = "ao_eri"; +pub(crate) const CACHE_KIND: &str = "integral-cache"; +const MAX_MANIFEST_BYTES: u64 = 1024 * 1024; + +mod private { + pub trait Sealed {} +} + +/// A known scientific artifact. Only RustiQ's declared artifact markers implement this trait. +pub trait Artifact: private::Sealed { + type Value; + + #[doc(hidden)] + fn get(data: &mut RustiQData) -> Result, PersistenceError>; + #[doc(hidden)] + fn set(data: &mut RustiQData, value: Self::Value) -> Result<(), PersistenceError>; +} + +/// AO electron-repulsion integrals in compact storage order. +pub struct AoEriArtifact; + +impl private::Sealed for AoEriArtifact {} + +impl Artifact for AoEriArtifact { + type Value = CompactEri; + + fn get(data: &mut RustiQData) -> Result, PersistenceError> { + if data.ao_eri.is_some() { + return Ok(data.ao_eri.as_ref()); + } + if !data.manifest.artifacts.contains_key(AO_ERI_ARTIFACT) { + return Ok(None); + } + data.read_eri().map(Some) + } + + fn set(data: &mut RustiQData, value: CompactEri) -> Result<(), PersistenceError> { + data.set_eri(value) + } +} + +/// Known scientific artifacts and their manifest, with NPY values loaded on demand. +/// +/// Add a typed field and accessors here when RustiQ gains a new scientific artifact. +/// Unknown manifest entries remain opaque and are copied when writing a new entry. +#[derive(Debug)] +pub struct RustiQData { + manifest: Manifest, + source: Option, + ao_eri: Option, + basis_functions: Option, +} + +impl RustiQData { + /// Gets a known artifact, loading and caching its value on first access. + pub fn get(&mut self) -> Result, PersistenceError> { + A::get(self) + } + + /// Sets a known artifact using its statically selected value type. + pub fn set(&mut self, value: A::Value) -> Result<(), PersistenceError> { + A::set(self, value) + } + + /// Starts a new AO ERI entry with the same scientific identity as the cache. + pub fn new(geometry: &Geometry, basis: &Basis, threshold: Option) -> Self { + let identity = ao_eri_identity(geometry, basis, threshold); + Self::new_with_identity(identity, basis.nbasis()) + } + + pub(crate) fn new_with_identity( + identity: super::ScientificIdentity, + basis_functions: usize, + ) -> Self { + Self { + manifest: Manifest { + format: FORMAT_NAME.to_owned(), + format_version: FORMAT_VERSION, + kind: CACHE_KIND.to_owned(), + producer: Producer { + name: "RustiQ".to_owned(), + version: env!("CARGO_PKG_VERSION").to_owned(), + }, + scientific_identity: ScientificIdentityManifest { + version: identity.version, + digest: identity.digest, + }, + artifacts: BTreeMap::new(), + }, + source: None, + ao_eri: None, + basis_functions: Some(basis_functions), + } + } + + /// Reads only the bounded manifest. NPY values are opened on demand. + pub fn read(directory: impl AsRef) -> Result { + let directory = directory.as_ref(); + let metadata = fs::symlink_metadata(directory)?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(PersistenceError::InvalidManifest( + "entry is not a regular directory".into(), + )); + } + let manifest_path = directory.join(MANIFEST_PATH); + let metadata = fs::symlink_metadata(&manifest_path)?; + if !metadata.is_file() + || metadata.file_type().is_symlink() + || metadata.len() > MAX_MANIFEST_BYTES + { + return Err(PersistenceError::InvalidManifest( + "manifest is not a bounded regular file".into(), + )); + } + let manifest: Manifest = + serde_json::from_reader(BufReader::new(File::open(manifest_path)?))?; + if manifest.format != FORMAT_NAME || manifest.format_version != FORMAT_VERSION { + return Err(PersistenceError::InvalidManifest( + "unsupported format or version".into(), + )); + } + let mut paths: HashSet<&str> = HashSet::new(); + for artifact in manifest.artifacts.values() { + relative_artifact_path(&artifact.path)?; + if paths + .iter() + .any(|other| paths_conflict(other, &artifact.path)) + { + return Err(PersistenceError::InvalidManifest(format!( + "artifact path conflicts with another artifact: {}", + artifact.path + ))); + } + paths.insert(artifact.path.as_str()); + } + let basis_functions = manifest + .artifacts + .get(AO_ERI_ARTIFACT) + .and_then(|artifact| { + if let ArtifactAttributes::AoEri(attributes) = &artifact.attributes { + Some(attributes.basis_functions) + } else { + None + } + }); + Ok(Self { + manifest, + source: Some(fs::canonicalize(directory)?), + ao_eri: None, + basis_functions, + }) + } + + pub(crate) fn manifest(&self) -> &Manifest { + &self.manifest + } + + /// Replaces the AO ERI artifact after checking its compact length. + pub fn set_eri(&mut self, eri: CompactEri) -> Result<(), PersistenceError> { + let basis_functions = self.basis_functions.ok_or(PersistenceError::MissingEri)?; + validate_eri_len(&eri, basis_functions)?; + self.ao_eri = Some(eri); + Ok(()) + } + + /// Validates and decodes the AO ERI NPY on first access, then reuses the object. + pub fn read_eri(&mut self) -> Result<&CompactEri, PersistenceError> { + if self.ao_eri.is_none() { + let artifact = self + .manifest + .artifacts + .get(AO_ERI_ARTIFACT) + .ok_or(PersistenceError::MissingEri)?; + let ArtifactAttributes::AoEri(attributes) = &artifact.attributes else { + return Err(PersistenceError::InvalidArtifact( + "AO ERI attributes are missing".into(), + )); + }; + if artifact.path != AO_ERI_PATH + || artifact.representation != COMPACT_ERI_REPRESENTATION + || attributes.computation_version != AO_ERI_COMPUTATION_VERSION + { + return Err(PersistenceError::InvalidArtifact( + "unsupported AO ERI representation".into(), + )); + } + let source = self.source.as_ref().ok_or(PersistenceError::MissingEri)?; + let mut file = open_artifact(source, artifact)?; + if sha256_reader(&mut file)? != artifact.digest { + return Err(PersistenceError::InvalidArtifact( + "AO ERI digest mismatch".into(), + )); + } + file.rewind()?; + self.ao_eri = Some(CompactEri::try_read_with_shape( + BufReader::new(file), + attributes.basis_functions, + )?); + } + Ok(self + .ao_eri + .as_ref() + .expect("ERI was loaded or already present")) + } + + pub(crate) fn take_eri(&mut self) -> Option { + self.ao_eri.take() + } + + /// Writes to a new directory, copying unloaded artifacts without decoding them. + pub fn write(&self, directory: impl AsRef) -> Result<(), PersistenceError> { + self.write_inner(directory.as_ref(), self.ao_eri.as_ref()) + } + + pub(crate) fn write_with_eri( + &self, + directory: impl AsRef, + eri: Option<&CompactEri>, + ) -> Result<(), PersistenceError> { + self.write_inner(directory.as_ref(), eri) + } + + fn write_inner( + &self, + directory: &Path, + eri: Option<&CompactEri>, + ) -> Result<(), PersistenceError> { + if directory.exists() { + return Err(PersistenceError::Io(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "persistence destination already exists", + ))); + } + let parent = directory + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent)?; + let staging = tempfile::Builder::new() + .prefix(".rustiq-data-") + .tempdir_in(parent)?; + let mut manifest = self.manifest.clone(); + + for (name, artifact) in &self.manifest.artifacts { + if name == AO_ERI_ARTIFACT && eri.is_some() { + continue; + } + let source = self.source.as_ref().ok_or_else(|| { + PersistenceError::InvalidArtifact(format!("artifact {name} has no source file")) + })?; + let mut input = open_artifact(source, artifact)?; + let output_path = staging.path().join(relative_artifact_path(&artifact.path)?); + fs::create_dir_all(output_path.parent().expect("artifact has a parent"))?; + let mut output = BufWriter::new(File::create(&output_path)?); + std::io::copy(&mut input, &mut output)?; + output.flush()?; + output + .into_inner() + .map_err(|error| error.into_error())? + .sync_all()?; + verify_artifact(&output_path, artifact)?; + } + + if let Some(eri) = eri { + let basis_functions = self.basis_functions.ok_or(PersistenceError::MissingEri)?; + validate_eri_len(eri, basis_functions)?; + let path = staging.path().join(AO_ERI_PATH); + fs::create_dir_all(path.parent().expect("ERI artifact has a parent"))?; + let mut output = BufWriter::new(File::create(&path)?); + eri.write_npy(&mut output)?; + output.flush()?; + output + .into_inner() + .map_err(|error| error.into_error())? + .sync_all()?; + manifest.artifacts.insert( + AO_ERI_ARTIFACT.to_owned(), + ArtifactManifest { + path: AO_ERI_PATH.to_owned(), + size: fs::metadata(&path)?.len(), + representation: COMPACT_ERI_REPRESENTATION.to_owned(), + digest: sha256_reader(BufReader::new(File::open(&path)?))?, + attributes: ArtifactAttributes::AoEri(AoEriAttributes { + basis_functions, + computation_version: AO_ERI_COMPUTATION_VERSION, + }), + }, + ); + } + if manifest.kind == CACHE_KIND && !manifest.artifacts.contains_key(AO_ERI_ARTIFACT) { + return Err(PersistenceError::MissingEri); + } + let mut output = BufWriter::new(File::create(staging.path().join(MANIFEST_PATH))?); + serde_json::to_writer_pretty(&mut output, &manifest)?; + output.write_all(b"\n")?; + output.flush()?; + output + .into_inner() + .map_err(|error| error.into_error())? + .sync_all()?; + fs::rename(staging.path(), directory)?; + Ok(()) + } +} + +fn validate_eri_len(eri: &CompactEri, basis_functions: usize) -> Result<(), PersistenceError> { + let expected = CompactEri::checked_storage_len(basis_functions).ok_or_else(|| { + PersistenceError::InvalidArtifact( + "basis-function count overflows compact ERI storage".into(), + ) + })?; + if eri.len() != expected { + return Err(PersistenceError::InvalidValueCount { + basis_functions, + expected, + actual: eri.len(), + }); + } + Ok(()) +} + +fn relative_artifact_path(path: &str) -> Result<&Path, PersistenceError> { + let path = Path::new(path); + if path.as_os_str().is_empty() + || !path + .components() + .all(|component| matches!(component, Component::Normal(_))) + || path.starts_with(MANIFEST_PATH) + { + return Err(PersistenceError::InvalidArtifact(format!( + "unsafe artifact path: {}", + path.display() + ))); + } + Ok(path) +} + +fn paths_conflict(left: &str, right: &str) -> bool { + Path::new(left).starts_with(right) || Path::new(right).starts_with(left) +} + +fn open_artifact(root: &Path, artifact: &ArtifactManifest) -> Result { + let relative = relative_artifact_path(&artifact.path)?; + let mut path = root.to_path_buf(); + for component in relative.components() { + path.push(component); + let metadata = fs::symlink_metadata(&path)?; + if metadata.file_type().is_symlink() { + return Err(PersistenceError::InvalidArtifact(format!( + "artifact contains a symbolic link: {}", + artifact.path + ))); + } + } + let metadata = fs::symlink_metadata(&path)?; + if !metadata.is_file() || metadata.len() != artifact.size { + return Err(PersistenceError::InvalidArtifact(format!( + "artifact size or file type mismatch: {}", + artifact.path + ))); + } + Ok(File::open(path)?) +} + +fn verify_artifact(path: &Path, artifact: &ArtifactManifest) -> Result<(), PersistenceError> { + let file = File::open(path)?; + if file.metadata()?.len() != artifact.size + || sha256_reader(BufReader::new(file))? != artifact.digest + { + return Err(PersistenceError::InvalidArtifact(format!( + "artifact digest or size mismatch: {}", + artifact.path + ))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::persistence::{sha256, Sha256Digest, SCIENTIFIC_IDENTITY_VERSION}; + + fn new_data() -> RustiQData { + RustiQData::new_with_identity( + super::super::ScientificIdentity { + version: SCIENTIFIC_IDENTITY_VERSION, + digest: Sha256Digest::from([7; 32]), + }, + 2, + ) + } + + #[test] + fn eri_is_loaded_only_on_request_and_then_cached_as_an_object() { + let root = tempfile::tempdir().unwrap(); + let entry = root.path().join("entry"); + let mut data = new_data(); + assert!(data.get::().unwrap().is_none()); + data.set::(CompactEri::Zeroed(2)).unwrap(); + data.write(&entry).unwrap(); + let mut restored = RustiQData::read(&entry).unwrap(); + assert!(restored.ao_eri.is_none()); + let first = restored.get::().unwrap().unwrap() as *const CompactEri; + fs::remove_file(entry.join(AO_ERI_PATH)).unwrap(); + let second = restored.get::().unwrap().unwrap() as *const CompactEri; + assert_eq!(first, second); + assert_eq!(restored.read_eri().unwrap().len(), 6); + } + + #[test] + fn unloaded_unknown_artifact_is_copied_without_decoding() { + let root = tempfile::tempdir().unwrap(); + let first = root.path().join("first"); + let second = root.path().join("second"); + let mut data = new_data(); + data.set_eri(CompactEri::Zeroed(2)).unwrap(); + data.write(&first).unwrap(); + let unknown = b"opaque future data"; + let unknown_path = first.join("arrays/post-hf/future.npy"); + fs::create_dir_all(unknown_path.parent().unwrap()).unwrap(); + fs::write(&unknown_path, unknown).unwrap(); + let manifest_path = first.join(MANIFEST_PATH); + let mut manifest: Manifest = + serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap(); + manifest.artifacts.insert( + "future".into(), + ArtifactManifest { + path: "arrays/post-hf/future.npy".into(), + size: unknown.len() as u64, + representation: "rustiq-future-v1".into(), + digest: sha256(unknown), + attributes: ArtifactAttributes::Unknown(BTreeMap::new()), + }, + ); + fs::write(&manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap(); + let restored = RustiQData::read(&first).unwrap(); + assert!(restored.ao_eri.is_none()); + restored.write(&second).unwrap(); + assert_eq!( + fs::read(second.join("arrays/post-hf/future.npy")).unwrap(), + unknown + ); + assert_eq!(RustiQData::read(&second).unwrap().manifest, manifest); + } + + #[test] + fn rejects_corrupt_payload_and_conflicting_paths() { + let root = tempfile::tempdir().unwrap(); + let first = root.path().join("first"); + let second = root.path().join("second"); + let mut data = new_data(); + data.set_eri(CompactEri::Zeroed(2)).unwrap(); + data.write(&first).unwrap(); + fs::write(first.join(AO_ERI_PATH), b"bad").unwrap(); + assert!(matches!( + RustiQData::read(&first).unwrap().write(&second), + Err(PersistenceError::InvalidArtifact(_)) + )); + assert!(!second.exists()); + + let manifest_path = first.join(MANIFEST_PATH); + let mut manifest: Manifest = + serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap(); + manifest.artifacts.insert( + "future".into(), + ArtifactManifest { + path: "arrays/integrals".into(), + size: 0, + representation: "future-v1".into(), + digest: sha256(b""), + attributes: ArtifactAttributes::Unknown(BTreeMap::new()), + }, + ); + fs::write(&manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap(); + assert!(matches!( + RustiQData::read(&first), + Err(PersistenceError::InvalidManifest(_)) + )); + } +} diff --git a/crates/rustiq-core/src/persistence/eri_cache.rs b/crates/rustiq-core/src/persistence/eri_cache.rs index c60f70a..d364e43 100644 --- a/crates/rustiq-core/src/persistence/eri_cache.rs +++ b/crates/rustiq-core/src/persistence/eri_cache.rs @@ -1,6 +1,6 @@ use std::{ fs::{self, File}, - io::{self, BufReader, BufWriter, Seek, Write}, + io::{self, Seek}, path::{Path, PathBuf}, }; @@ -12,16 +12,13 @@ use crate::{ }; use super::{ - ao_eri_identity, read_compact_eri, sha256_reader, validate_compact_eri_header, AoEriAttributes, - ArtifactAttributes, ArtifactManifest, Manifest, Producer, ScientificIdentity, - ScientificIdentityManifest, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, - COMPACT_ERI_REPRESENTATION, FORMAT_NAME, FORMAT_VERSION, MANIFEST_PATH, - SCIENTIFIC_IDENTITY_VERSION, + ao_eri_identity, sha256_reader, validate_compact_eri_header, AoEriAttributes, + ArtifactAttributes, ArtifactManifest, Manifest, RustiQData, ScientificIdentity, + AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, + FORMAT_VERSION, SCIENTIFIC_IDENTITY_VERSION, }; -const CACHE_KIND: &str = "integral-cache"; -const AO_ERI_ARTIFACT: &str = "ao_eri"; -const MAX_MANIFEST_BYTES: u64 = 1024 * 1024; +use super::data::{AO_ERI_ARTIFACT, CACHE_KIND}; /// A directory-backed AO ERI cache entry available for management. #[derive(Clone, Debug, PartialEq, Eq)] @@ -312,14 +309,15 @@ impl EriCache { if !metadata.is_dir() || metadata.file_type().is_symlink() { return None; } - let manifest = read_manifest(&entry)?; + let mut data = RustiQData::read(&entry).ok()?; + let manifest = data.manifest(); let artifact = manifest.artifacts.get(AO_ERI_ARTIFACT)?; let attributes = ao_eri_attributes(artifact)?; - if !manifest_is_valid(&manifest, identity) || attributes.basis_functions != basis_functions - { + if !manifest_is_valid(manifest, identity) || attributes.basis_functions != basis_functions { return None; } - read_validated_payload(&entry, artifact, basis_functions) + data.read_eri().ok()?; + data.take_eri() } fn store_identity( @@ -340,53 +338,13 @@ impl EriCache { } fs::create_dir_all(parent)?; let temporary = Builder::new().prefix(".rustiq-eri-").tempdir_in(parent)?; - let payload_path = temporary.path().join(AO_ERI_PATH); - fs::create_dir_all(payload_path.parent().expect("AO ERI path has a parent"))?; - { - let mut writer = BufWriter::new(File::create(&payload_path)?); - super::write_compact_eri(&mut writer, eri) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; - writer.flush()?; - writer.into_inner()?.sync_all()?; - } - let payload_metadata = fs::metadata(&payload_path)?; - let payload_digest = sha256_reader(BufReader::new(File::open(&payload_path)?))?; - let manifest = Manifest { - format: FORMAT_NAME.to_owned(), - format_version: FORMAT_VERSION, - kind: CACHE_KIND.to_owned(), - producer: Producer { - name: "RustiQ".to_owned(), - version: env!("CARGO_PKG_VERSION").to_owned(), - }, - scientific_identity: ScientificIdentityManifest { - version: identity.version, - digest: identity.digest, - }, - artifacts: [( - AO_ERI_ARTIFACT.to_owned(), - ArtifactManifest { - path: AO_ERI_PATH.to_owned(), - size: payload_metadata.len(), - representation: COMPACT_ERI_REPRESENTATION.to_owned(), - digest: payload_digest, - attributes: ArtifactAttributes::AoEri(AoEriAttributes { - basis_functions, - computation_version: AO_ERI_COMPUTATION_VERSION, - }), - }, - )] - .into_iter() - .collect(), - }; - { - let mut writer = BufWriter::new(File::create(temporary.path().join(MANIFEST_PATH))?); - serde_json::to_writer_pretty(&mut writer, &manifest).map_err(io::Error::other)?; - writer.write_all(b"\n")?; - writer.flush()?; - writer.into_inner()?.sync_all()?; - } + let entry_data = RustiQData::new_with_identity(identity, basis_functions); + let staged_entry = temporary.path().join("entry"); + entry_data + .write_with_eri(&staged_entry, Some(eri)) + .map_err(io::Error::other)?; let temporary_path = temporary.keep(); + let staged_entry = temporary_path.join("entry"); if fs::symlink_metadata(&final_entry).is_ok() { if self.load_identity(identity, basis_functions).is_some() { let _ = fs::remove_dir_all(temporary_path); @@ -397,8 +355,11 @@ impl EriCache { return Err(error); } } - match fs::rename(&temporary_path, &final_entry) { - Ok(()) => Ok(()), + match fs::rename(&staged_entry, &final_entry) { + Ok(()) => { + let _ = fs::remove_dir_all(temporary_path); + Ok(()) + } Err(error) => { let _ = fs::remove_dir_all(temporary_path); if self.load_identity(identity, basis_functions).is_some() { @@ -412,15 +373,9 @@ impl EriCache { } fn read_manifest(entry: &Path) -> Option { - let manifest_path = entry.join(MANIFEST_PATH); - fs::symlink_metadata(&manifest_path) + RustiQData::read(entry) .ok() - .filter(|metadata| { - metadata.is_file() - && !metadata.file_type().is_symlink() - && metadata.len() <= MAX_MANIFEST_BYTES - })?; - serde_json::from_reader(BufReader::new(File::open(manifest_path).ok()?)).ok() + .map(|data| data.manifest().clone()) } fn manifest_is_valid(manifest: &Manifest, identity: ScientificIdentity) -> bool { @@ -448,28 +403,6 @@ fn ao_eri_attributes(artifact: &ArtifactManifest) -> Option<&AoEriAttributes> { } } -fn read_validated_payload( - entry: &Path, - artifact: &ArtifactManifest, - basis_functions: usize, -) -> Option { - CompactEri::checked_storage_len(basis_functions)?; - let payload_path = entry.join(AO_ERI_PATH); - let metadata = fs::symlink_metadata(&payload_path).ok()?; - if !metadata.is_file() || metadata.file_type().is_symlink() { - return None; - } - let mut file = File::open(payload_path).ok()?; - if file.metadata().ok()?.len() != artifact.size { - return None; - } - if sha256_reader(&mut file).ok()? != artifact.digest { - return None; - } - file.rewind().ok()?; - read_compact_eri(BufReader::new(file), basis_functions).ok() -} - fn validate_payload(entry: &Path, artifact: &ArtifactManifest, basis_functions: usize) -> bool { if CompactEri::checked_storage_len(basis_functions).is_none() { return false; @@ -525,6 +458,7 @@ pub(super) fn is_fingerprint(value: &str) -> bool { #[cfg(test)] mod tests { use super::*; + use crate::persistence::MANIFEST_PATH; use crate::{config::DEFAULT_ERI_SCHWARZ_THRESHOLD, test_utils::load_sto3g_basis}; fn input() -> (Molecule, Basis) { diff --git a/crates/rustiq-core/src/persistence/mod.rs b/crates/rustiq-core/src/persistence/mod.rs index a65860c..f21f960 100644 --- a/crates/rustiq-core/src/persistence/mod.rs +++ b/crates/rustiq-core/src/persistence/mod.rs @@ -6,12 +6,15 @@ mod cache_names; mod checksum; +mod data; mod eri_cache; mod identity; mod manifest; mod npy; +pub use crate::eri::CompactEri; pub use checksum::{sha256, sha256_reader, verify_sha256, Sha256Digest, Sha256DigestParseError}; +pub use data::{AoEriArtifact, Artifact, RustiQData}; pub use eri_cache::{EriCache, EriCacheEntry}; #[allow(unused_imports)] pub(crate) use identity::{ao_eri_identity, ScientificIdentity, AO_ERI_COMPUTATION_VERSION}; @@ -19,6 +22,7 @@ pub use manifest::{ AoEriAttributes, ArtifactAttributes, ArtifactManifest, Manifest, Producer, ScientificIdentityManifest, }; +pub use npy::NpyConvert; #[allow(unused_imports)] pub(crate) use npy::{ @@ -36,6 +40,16 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum PersistenceError { + #[error("persistence I/O failed: {0}")] + Io(#[from] std::io::Error), + #[error("could not read or write persistence manifest: {0}")] + Manifest(#[from] serde_json::Error), + #[error("invalid persistence manifest: {0}")] + InvalidManifest(String), + #[error("invalid persistence artifact: {0}")] + InvalidArtifact(String), + #[error("AO ERI artifact is missing")] + MissingEri, #[error("could not read NPY data: {0}")] NpyRead(#[source] std::io::Error), #[error("could not write NPY data: {0}")] @@ -44,6 +58,11 @@ pub enum PersistenceError { InvalidEriShape(Vec), #[error("matrix NPY must be two-dimensional, found shape {0:?}")] InvalidMatrixShape(Vec), + #[error("NPY has shape {actual:?}, expected {expected:?}")] + InvalidNpyShape { + expected: Vec, + actual: Vec, + }, #[error("AO ERI payload has {actual} values, expected {expected} for {basis_functions} basis functions")] InvalidValueCount { basis_functions: usize, diff --git a/crates/rustiq-core/src/persistence/npy.rs b/crates/rustiq-core/src/persistence/npy.rs index cf6cb5d..23fcc21 100644 --- a/crates/rustiq-core/src/persistence/npy.rs +++ b/crates/rustiq-core/src/persistence/npy.rs @@ -7,6 +7,103 @@ use crate::eri::CompactEri; use super::PersistenceError; +/// Conversion between scientific values and an NPY byte stream. +pub trait NpyConvert: Sized { + type Shape; + + /// Converts a parsed NPY array into the scientific value. + fn from_npy(npy: NpyFile) -> Result; + + /// Reads from any byte stream, including an in-memory buffer. + fn read_npy(reader: impl Read) -> Result { + Self::from_npy(NpyFile::new(reader).map_err(PersistenceError::NpyRead)?) + } + + fn npy_shape(shape: Self::Shape) -> Result, PersistenceError>; + + /// Checks a parsed header before decoding any array values. + fn try_from_npy_with_shape( + npy: NpyFile, + shape: Self::Shape, + ) -> Result { + let expected = Self::npy_shape(shape)?; + let actual = npy.shape().to_vec(); + if actual != expected { + return Err(PersistenceError::InvalidNpyShape { expected, actual }); + } + Self::from_npy(npy) + } + + /// Parses a stream once, checks its shape, then decodes its values. + fn try_read_with_shape( + reader: impl Read, + shape: Self::Shape, + ) -> Result { + Self::try_from_npy_with_shape( + NpyFile::new(reader).map_err(PersistenceError::NpyRead)?, + shape, + ) + } + + fn write_npy(&self, writer: impl Write) -> Result<(), PersistenceError>; +} + +impl NpyConvert for CompactEri { + type Shape = usize; + + fn write_npy(&self, writer: impl Write) -> Result<(), PersistenceError> { + write_compact_eri(writer, self) + } + + fn from_npy(npy: NpyFile) -> Result { + decode_compact_eri(npy, None) + } + + fn npy_shape(basis_functions: usize) -> Result, PersistenceError> { + let length = CompactEri::checked_storage_len(basis_functions).ok_or_else(|| { + PersistenceError::InvalidArtifact( + "basis-function count overflows compact ERI storage".into(), + ) + })?; + Ok(vec![u64::try_from(length).map_err(|_| { + PersistenceError::InvalidArtifact("compact ERI length exceeds NPY limits".into()) + })?]) + } +} + +impl NpyConvert for DMatrix { + type Shape = (usize, usize); + + fn from_npy(npy: NpyFile) -> Result { + decode_dmatrix(npy) + } + + fn npy_shape((rows, columns): (usize, usize)) -> Result, PersistenceError> { + let rows = u64::try_from(rows).map_err(|_| { + PersistenceError::InvalidArtifact("matrix rows exceed NPY limits".into()) + })?; + let columns = u64::try_from(columns).map_err(|_| { + PersistenceError::InvalidArtifact("matrix columns exceed NPY limits".into()) + })?; + Ok(vec![rows, columns]) + } + + fn write_npy(&self, writer: impl Write) -> Result<(), PersistenceError> { + let shape = [self.nrows() as u64, self.ncols() as u64]; + let mut writer = npyz::WriteOptions::new() + .default_dtype() + .order(npyz::Order::Fortran) + .shape(&shape) + .writer(writer) + .begin_nd() + .map_err(PersistenceError::NpyWrite)?; + writer + .extend(self.as_slice().iter().copied()) + .map_err(PersistenceError::NpyWrite)?; + writer.finish().map_err(PersistenceError::NpyWrite) + } +} + pub(crate) fn write_compact_eri( writer: impl Write, eri: &CompactEri, @@ -28,25 +125,41 @@ pub(crate) fn read_compact_eri( reader: impl Read, basis_functions: usize, ) -> Result { - let expected = CompactEri::checked_storage_len(basis_functions).ok_or( - PersistenceError::InvalidValueCount { - basis_functions, - expected: 0, - actual: 0, - }, - )?; - let npy = NpyFile::new(reader).map_err(PersistenceError::NpyRead)?; + decode_compact_eri( + NpyFile::new(reader).map_err(PersistenceError::NpyRead)?, + Some(basis_functions), + ) +} + +fn decode_compact_eri( + npy: NpyFile, + basis_functions: Option, +) -> Result { if npy.shape().len() != 1 { return Err(PersistenceError::InvalidEriShape(npy.shape().to_vec())); } - let actual = usize::try_from(npy.shape()[0]).unwrap_or(usize::MAX); - if actual != expected { - return Err(PersistenceError::InvalidValueCount { - basis_functions, - expected, - actual, - }); - } + let actual = usize::try_from(npy.shape()[0]) + .map_err(|_| PersistenceError::InvalidEriShape(npy.shape().to_vec()))?; + let basis_functions = if let Some(basis_functions) = basis_functions { + let expected = CompactEri::checked_storage_len(basis_functions).ok_or( + PersistenceError::InvalidValueCount { + basis_functions, + expected: 0, + actual, + }, + )?; + if actual != expected { + return Err(PersistenceError::InvalidValueCount { + basis_functions, + expected, + actual, + }); + } + basis_functions + } else { + basis_functions_for_len(actual) + .ok_or_else(|| PersistenceError::InvalidEriShape(npy.shape().to_vec()))? + }; let values = npy.into_vec::().map_err(|error| { if error.kind() == std::io::ErrorKind::InvalidData { PersistenceError::InvalidDtype(error.to_string()) @@ -65,6 +178,19 @@ pub(crate) fn read_compact_eri( }) } +fn basis_functions_for_len(length: usize) -> Option { + let (mut low, mut high) = (0, usize::MAX); + while low <= high { + let middle = low + (high - low) / 2; + match CompactEri::checked_storage_len(middle) { + Some(actual) if actual == length => return Some(middle), + Some(actual) if actual < length => low = middle + 1, + _ => high = middle.checked_sub(1)?, + } + } + None +} + pub(crate) fn validate_compact_eri_header( reader: impl Read + Seek, basis_functions: usize, @@ -105,7 +231,10 @@ pub(crate) fn validate_compact_eri_header( } pub(crate) fn read_dmatrix(reader: impl Read) -> Result, PersistenceError> { - let npy = NpyFile::new(reader).map_err(PersistenceError::NpyRead)?; + decode_dmatrix(NpyFile::new(reader).map_err(PersistenceError::NpyRead)?) +} + +fn decode_dmatrix(npy: NpyFile) -> Result, PersistenceError> { if npy.shape().len() != 2 { return Err(PersistenceError::InvalidMatrixShape(npy.shape().to_vec())); } @@ -155,6 +284,24 @@ mod tests { write_compact_eri(&mut bytes, &source).unwrap(); let restored = read_compact_eri(bytes.as_slice(), basis_functions).unwrap(); assert_eq!(restored.ordered_values(), source.ordered_values()); + let inferred = CompactEri::read_npy(bytes.as_slice()).unwrap(); + assert_eq!(inferred.ordered_values(), source.ordered_values()); + assert_eq!( + CompactEri::from_npy(NpyFile::new(bytes.as_slice()).unwrap()) + .unwrap() + .ordered_values(), + source.ordered_values() + ); + assert_eq!( + CompactEri::try_read_with_shape(bytes.as_slice(), basis_functions) + .unwrap() + .ordered_values(), + source.ordered_values() + ); + assert!(matches!( + CompactEri::try_read_with_shape(bytes.as_slice(), basis_functions - 1), + Err(PersistenceError::InvalidNpyShape { .. }) + )); } #[test] @@ -183,6 +330,61 @@ mod tests { assert_eq!(matrix[(1, 2)], 6.0); } + #[test] + fn dmatrix_trait_writes_fortran_order_and_reads_both_orders() { + let matrix = DMatrix::from_row_slice(2, 3, &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); + let mut bytes = Vec::new(); + matrix.write_npy(&mut bytes).unwrap(); + let npy = NpyFile::new(bytes.as_slice()).unwrap(); + assert_eq!(npy.order(), npyz::Order::Fortran); + assert_eq!(npy.shape(), &[2, 3]); + assert_eq!(npy.into_vec::().unwrap(), matrix.as_slice()); + assert_eq!(DMatrix::::read_npy(bytes.as_slice()).unwrap(), matrix); + assert_eq!( + DMatrix::::from_npy(NpyFile::new(bytes.as_slice()).unwrap()).unwrap(), + matrix + ); + assert_eq!( + DMatrix::::try_from_npy_with_shape( + NpyFile::new(bytes.as_slice()).unwrap(), + (2, 3) + ) + .unwrap(), + matrix + ); + assert_eq!( + DMatrix::::try_read_with_shape(bytes.as_slice(), (2, 3)).unwrap(), + matrix + ); + assert!(matches!( + DMatrix::::try_read_with_shape(bytes.as_slice(), (3, 2)), + Err(PersistenceError::InvalidNpyShape { .. }) + )); + let truncated = &bytes[..bytes.len() - 8]; + assert!(matches!( + DMatrix::::try_read_with_shape(truncated, (3, 2)), + Err(PersistenceError::InvalidNpyShape { .. }) + )); + assert!(matches!( + DMatrix::::try_read_with_shape(truncated, (2, 3)), + Err(PersistenceError::NpyRead(_)) + )); + + let mut c_bytes = Vec::new(); + let mut writer = npyz::WriteOptions::new() + .default_dtype() + .shape(&[2, 3]) + .writer(&mut c_bytes) + .begin_nd() + .unwrap(); + writer.extend([1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap(); + writer.finish().unwrap(); + assert_eq!( + DMatrix::::read_npy(c_bytes.as_slice()).unwrap(), + matrix + ); + } + #[test] fn rejects_wrong_value_count() { let mut bytes = Vec::new(); diff --git a/docs/persistence-format-v1.md b/docs/persistence-format-v1.md index 444093d..a332328 100644 --- a/docs/persistence-format-v1.md +++ b/docs/persistence-format-v1.md @@ -87,6 +87,25 @@ untouched. Names and fingerprints are validated rather than interpreted as paths ## Logical entries +The core Rust API exposes `persistence::RustiQData` for reading and writing a +directory in this format. `RustiQData::read` reads the manifest only. Its +`read_eri` method validates and decodes the AO ERI NPY on first access and keeps +the resulting `CompactEri` for subsequent accesses. Writing to a new directory +copies artifacts that have not been decoded, including unknown representations, +as verified byte streams. Its public scientific API is typed: `set_eri` and +`read_eri` operate on `CompactEri`. The generic `get::()` returns +`Result, PersistenceError>`, while +`set::(value)` accepts only `CompactEri`; only declared artifact +marker types are accepted. A future known artifact gets its own marker, typed field, +and accessors in `RustiQData`. The `NpyConvert` trait handles NPY byte streams +for `CompactEri` and `DMatrix`. Its associated `Shape` type is `usize` for +the ERI basis-function count and `(usize, usize)` for matrix dimensions; +`try_read_with_shape` checks the declared shape before constructing the value. +`from_npy` and `try_from_npy_with_shape` also accept an already parsed +`npyz::NpyFile`, so callers can parse a stream once without using a filesystem +path. Matrix readers accept C and Fortran order and matrix writers emit Fortran +order. + - `manifest.json` is UTF-8 JSON and describes the format, producer, scientific identity and artifacts. - `arrays/integrals/ao-eri.npy` is the AO electron-repulsion integral artifact. From 510716b747d07c7f36e7a6414a107758803429b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 10:18:03 +0200 Subject: [PATCH 02/62] Harden typed persistence boundaries --- Cargo.lock | 10 ++ Cargo.toml | 1 + crates/rustiq-core/Cargo.toml | 1 + crates/rustiq-core/src/persistence/data.rs | 89 ++++++++--- .../rustiq-core/src/persistence/eri_cache.rs | 2 +- .../rustiq-core/src/persistence/manifest.rs | 7 +- crates/rustiq-core/src/persistence/mod.rs | 1 - crates/rustiq-core/src/persistence/npy.rs | 141 +++++++----------- docs/persistence-format-v1.md | 18 ++- 9 files changed, 147 insertions(+), 123 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1de1627..4cdf9ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2494,6 +2494,15 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" +[[package]] +name = "relative-path" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bca40a312222d8ba74837cb474edef44b37f561da5f773981007a10bbaa992b0" +dependencies = [ + "serde", +] + [[package]] name = "reqwest" version = "0.13.4" @@ -2629,6 +2638,7 @@ dependencies = [ "rand_chacha 0.10.0", "rand_distr", "rayon", + "relative-path 2.0.1", "reqwest", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 8e8da1e..927d4e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ periodic_table = "0.5" proptest = "1" rand = "0.10" rayon = "1" +relative-path = { version = "2", features = ["serde"] } serde = { version = "1", features = ["derive"] } serde_json = "1.0.*" sysinfo = { version = "0.39.6", default-features = false, features = ["system"] } diff --git a/crates/rustiq-core/Cargo.toml b/crates/rustiq-core/Cargo.toml index 703bbac..3d156b3 100644 --- a/crates/rustiq-core/Cargo.toml +++ b/crates/rustiq-core/Cargo.toml @@ -27,6 +27,7 @@ npyz = "0.9.1" physical_constants = "0.5" periodic_table.workspace = true rayon.workspace = true +relative-path.workspace = true tokio = { workspace = true, features = ["fs", "io-util"], optional = true } serde_json.workspace = true serde.workspace = true diff --git a/crates/rustiq-core/src/persistence/data.rs b/crates/rustiq-core/src/persistence/data.rs index 0246ca5..f849b5f 100644 --- a/crates/rustiq-core/src/persistence/data.rs +++ b/crates/rustiq-core/src/persistence/data.rs @@ -2,17 +2,20 @@ use std::{ collections::{BTreeMap, HashSet}, fs::{self, File}, io::{BufReader, BufWriter, Seek, Write}, - path::{Component, Path, PathBuf}, + path::{Path, PathBuf}, }; +use relative_path::RelativePath; + use crate::{ basis::Basis, config::validated::PositiveFiniteF64, eri::CompactEri, molecules::geometry::Geometry, }; use super::{ - ao_eri_identity, sha256_reader, AoEriAttributes, ArtifactAttributes, ArtifactManifest, - Manifest, NpyConvert, PersistenceError, Producer, ScientificIdentityManifest, + ao_eri_identity, read_compact_eri, sha256_reader, write_compact_eri, AoEriAttributes, + ArtifactAttributes, ArtifactManifest, Manifest, PersistenceError, Producer, + ScientificIdentityManifest, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, FORMAT_VERSION, MANIFEST_PATH, }; @@ -140,10 +143,10 @@ impl RustiQData { } let mut paths: HashSet<&str> = HashSet::new(); for artifact in manifest.artifacts.values() { - relative_artifact_path(&artifact.path)?; + validate_artifact_path(&artifact.path)?; if paths .iter() - .any(|other| paths_conflict(other, &artifact.path)) + .any(|other| paths_conflict(other, artifact.path.as_str())) { return Err(PersistenceError::InvalidManifest(format!( "artifact path conflicts with another artifact: {}", @@ -195,7 +198,7 @@ impl RustiQData { "AO ERI attributes are missing".into(), )); }; - if artifact.path != AO_ERI_PATH + if artifact.path.as_str() != AO_ERI_PATH || artifact.representation != COMPACT_ERI_REPRESENTATION || attributes.computation_version != AO_ERI_COMPUTATION_VERSION { @@ -211,7 +214,7 @@ impl RustiQData { )); } file.rewind()?; - self.ao_eri = Some(CompactEri::try_read_with_shape( + self.ao_eri = Some(read_compact_eri( BufReader::new(file), attributes.basis_functions, )?); @@ -268,7 +271,8 @@ impl RustiQData { PersistenceError::InvalidArtifact(format!("artifact {name} has no source file")) })?; let mut input = open_artifact(source, artifact)?; - let output_path = staging.path().join(relative_artifact_path(&artifact.path)?); + validate_artifact_path(&artifact.path)?; + let output_path = artifact.path.to_path(staging.path()); fs::create_dir_all(output_path.parent().expect("artifact has a parent"))?; let mut output = BufWriter::new(File::create(&output_path)?); std::io::copy(&mut input, &mut output)?; @@ -283,10 +287,10 @@ impl RustiQData { if let Some(eri) = eri { let basis_functions = self.basis_functions.ok_or(PersistenceError::MissingEri)?; validate_eri_len(eri, basis_functions)?; - let path = staging.path().join(AO_ERI_PATH); + let path = RelativePath::new(AO_ERI_PATH).to_path(staging.path()); fs::create_dir_all(path.parent().expect("ERI artifact has a parent"))?; let mut output = BufWriter::new(File::create(&path)?); - eri.write_npy(&mut output)?; + write_compact_eri(&mut output, eri)?; output.flush()?; output .into_inner() @@ -295,7 +299,7 @@ impl RustiQData { manifest.artifacts.insert( AO_ERI_ARTIFACT.to_owned(), ArtifactManifest { - path: AO_ERI_PATH.to_owned(), + path: AO_ERI_PATH.into(), size: fs::metadata(&path)?.len(), representation: COMPACT_ERI_REPRESENTATION.to_owned(), digest: sha256_reader(BufReader::new(File::open(&path)?))?, @@ -338,30 +342,41 @@ fn validate_eri_len(eri: &CompactEri, basis_functions: usize) -> Result<(), Pers Ok(()) } -fn relative_artifact_path(path: &str) -> Result<&Path, PersistenceError> { - let path = Path::new(path); - if path.as_os_str().is_empty() - || !path - .components() - .all(|component| matches!(component, Component::Normal(_))) - || path.starts_with(MANIFEST_PATH) +fn validate_artifact_path(path: &RelativePath) -> Result<(), PersistenceError> { + let value = path.as_str(); + let invalid_component = value + .split('/') + .any(|component| component.is_empty() || matches!(component, "." | "..")); + let conflicts_with_manifest = value.split('/').next() == Some(MANIFEST_PATH); + + if value.is_empty() + || value.starts_with('/') + || value.contains('\\') + || value.contains(':') + || invalid_component + || conflicts_with_manifest { return Err(PersistenceError::InvalidArtifact(format!( - "unsafe artifact path: {}", - path.display() + "unsafe artifact path: {path}" ))); } - Ok(path) + Ok(()) } fn paths_conflict(left: &str, right: &str) -> bool { - Path::new(left).starts_with(right) || Path::new(right).starts_with(left) + left == right + || left + .strip_prefix(right) + .is_some_and(|suffix| suffix.starts_with('/')) + || right + .strip_prefix(left) + .is_some_and(|suffix| suffix.starts_with('/')) } fn open_artifact(root: &Path, artifact: &ArtifactManifest) -> Result { - let relative = relative_artifact_path(&artifact.path)?; + validate_artifact_path(&artifact.path)?; let mut path = root.to_path_buf(); - for component in relative.components() { + for component in artifact.path.as_str().split('/') { path.push(component); let metadata = fs::symlink_metadata(&path)?; if metadata.file_type().is_symlink() { @@ -462,6 +477,32 @@ mod tests { assert_eq!(RustiQData::read(&second).unwrap().manifest, manifest); } + #[test] + fn artifact_paths_are_portable_and_relative() { + assert!(validate_artifact_path(RelativePath::new( + "arrays/integrals/ao-eri.npy" + )) + .is_ok()); + + for path in [ + "", + "/arrays/integrals/ao-eri.npy", + "arrays\\integrals\\ao-eri.npy", + "C:/arrays/ao-eri.npy", + "arrays/../ao-eri.npy", + "arrays/./ao-eri.npy", + "arrays//ao-eri.npy", + "arrays/ao-eri.npy/", + "manifest.json", + "manifest.json/child", + ] { + assert!( + validate_artifact_path(RelativePath::new(path)).is_err(), + "{path} must be rejected" + ); + } + } + #[test] fn rejects_corrupt_payload_and_conflicting_paths() { let root = tempfile::tempdir().unwrap(); diff --git a/crates/rustiq-core/src/persistence/eri_cache.rs b/crates/rustiq-core/src/persistence/eri_cache.rs index d364e43..fddc4c7 100644 --- a/crates/rustiq-core/src/persistence/eri_cache.rs +++ b/crates/rustiq-core/src/persistence/eri_cache.rs @@ -388,7 +388,7 @@ fn manifest_is_valid(manifest: &Manifest, identity: ScientificIdentity) -> bool .artifacts .get(AO_ERI_ARTIFACT) .is_some_and(|artifact| { - artifact.path == AO_ERI_PATH + artifact.path.as_str() == AO_ERI_PATH && artifact.representation == COMPACT_ERI_REPRESENTATION && ao_eri_attributes(artifact).is_some_and(|attributes| { attributes.computation_version == AO_ERI_COMPUTATION_VERSION diff --git a/crates/rustiq-core/src/persistence/manifest.rs b/crates/rustiq-core/src/persistence/manifest.rs index 61a89d9..cc7f201 100644 --- a/crates/rustiq-core/src/persistence/manifest.rs +++ b/crates/rustiq-core/src/persistence/manifest.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; +use relative_path::RelativePathBuf; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value; @@ -47,7 +48,7 @@ pub struct AoEriAttributes { #[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub struct ArtifactManifest { - pub path: String, + pub path: RelativePathBuf, pub size: u64, pub representation: String, pub digest: Sha256Digest, @@ -56,7 +57,7 @@ pub struct ArtifactManifest { #[derive(Deserialize)] struct RawArtifactManifest { - path: String, + path: RelativePathBuf, size: u64, representation: String, digest: Sha256Digest, @@ -109,7 +110,7 @@ mod tests { artifacts.insert( "ao_eri".to_string(), ArtifactManifest { - path: AO_ERI_PATH.to_string(), + path: RelativePathBuf::from(AO_ERI_PATH), size: 176, representation: COMPACT_ERI_REPRESENTATION.to_string(), digest: Sha256Digest::from([0x22; 32]), diff --git a/crates/rustiq-core/src/persistence/mod.rs b/crates/rustiq-core/src/persistence/mod.rs index f21f960..102543c 100644 --- a/crates/rustiq-core/src/persistence/mod.rs +++ b/crates/rustiq-core/src/persistence/mod.rs @@ -22,7 +22,6 @@ pub use manifest::{ AoEriAttributes, ArtifactAttributes, ArtifactManifest, Manifest, Producer, ScientificIdentityManifest, }; -pub use npy::NpyConvert; #[allow(unused_imports)] pub(crate) use npy::{ diff --git a/crates/rustiq-core/src/persistence/npy.rs b/crates/rustiq-core/src/persistence/npy.rs index 23fcc21..562e3b3 100644 --- a/crates/rustiq-core/src/persistence/npy.rs +++ b/crates/rustiq-core/src/persistence/npy.rs @@ -8,19 +8,16 @@ use crate::eri::CompactEri; use super::PersistenceError; /// Conversion between scientific values and an NPY byte stream. -pub trait NpyConvert: Sized { - type Shape; - - /// Converts a parsed NPY array into the scientific value. - fn from_npy(npy: NpyFile) -> Result; - - /// Reads from any byte stream, including an in-memory buffer. - fn read_npy(reader: impl Read) -> Result { - Self::from_npy(NpyFile::new(reader).map_err(PersistenceError::NpyRead)?) - } +trait NpyConvert: Sized { + type Shape: Copy; fn npy_shape(shape: Self::Shape) -> Result, PersistenceError>; + fn decode_with_shape( + npy: NpyFile, + shape: Self::Shape, + ) -> Result; + /// Checks a parsed header before decoding any array values. fn try_from_npy_with_shape( npy: NpyFile, @@ -31,7 +28,7 @@ pub trait NpyConvert: Sized { if actual != expected { return Err(PersistenceError::InvalidNpyShape { expected, actual }); } - Self::from_npy(npy) + Self::decode_with_shape(npy, shape) } /// Parses a stream once, checks its shape, then decodes its values. @@ -51,14 +48,6 @@ pub trait NpyConvert: Sized { impl NpyConvert for CompactEri { type Shape = usize; - fn write_npy(&self, writer: impl Write) -> Result<(), PersistenceError> { - write_compact_eri(writer, self) - } - - fn from_npy(npy: NpyFile) -> Result { - decode_compact_eri(npy, None) - } - fn npy_shape(basis_functions: usize) -> Result, PersistenceError> { let length = CompactEri::checked_storage_len(basis_functions).ok_or_else(|| { PersistenceError::InvalidArtifact( @@ -69,15 +58,32 @@ impl NpyConvert for CompactEri { PersistenceError::InvalidArtifact("compact ERI length exceeds NPY limits".into()) })?]) } + + fn decode_with_shape( + npy: NpyFile, + basis_functions: usize, + ) -> Result { + decode_compact_eri(npy, basis_functions) + } + + fn write_npy(&self, writer: impl Write) -> Result<(), PersistenceError> { + let shape = [self.len() as u64]; + let mut writer = npyz::WriteOptions::new() + .default_dtype() + .shape(&shape) + .writer(writer) + .begin_nd() + .map_err(PersistenceError::NpyWrite)?; + writer + .extend(self.ordered_values().iter().copied()) + .map_err(PersistenceError::NpyWrite)?; + writer.finish().map_err(PersistenceError::NpyWrite) + } } impl NpyConvert for DMatrix { type Shape = (usize, usize); - fn from_npy(npy: NpyFile) -> Result { - decode_dmatrix(npy) - } - fn npy_shape((rows, columns): (usize, usize)) -> Result, PersistenceError> { let rows = u64::try_from(rows).map_err(|_| { PersistenceError::InvalidArtifact("matrix rows exceed NPY limits".into()) @@ -88,6 +94,13 @@ impl NpyConvert for DMatrix { Ok(vec![rows, columns]) } + fn decode_with_shape( + npy: NpyFile, + _shape: (usize, usize), + ) -> Result { + decode_dmatrix(npy) + } + fn write_npy(&self, writer: impl Write) -> Result<(), PersistenceError> { let shape = [self.nrows() as u64, self.ncols() as u64]; let mut writer = npyz::WriteOptions::new() @@ -108,58 +121,39 @@ pub(crate) fn write_compact_eri( writer: impl Write, eri: &CompactEri, ) -> Result<(), PersistenceError> { - let shape = [eri.len() as u64]; - let mut writer = npyz::WriteOptions::new() - .default_dtype() - .shape(&shape) - .writer(writer) - .begin_nd() - .map_err(PersistenceError::NpyWrite)?; - writer - .extend(eri.ordered_values().iter().copied()) - .map_err(PersistenceError::NpyWrite)?; - writer.finish().map_err(PersistenceError::NpyWrite) + eri.write_npy(writer) } pub(crate) fn read_compact_eri( reader: impl Read, basis_functions: usize, ) -> Result { - decode_compact_eri( - NpyFile::new(reader).map_err(PersistenceError::NpyRead)?, - Some(basis_functions), - ) + CompactEri::try_read_with_shape(reader, basis_functions) } fn decode_compact_eri( npy: NpyFile, - basis_functions: Option, + basis_functions: usize, ) -> Result { if npy.shape().len() != 1 { return Err(PersistenceError::InvalidEriShape(npy.shape().to_vec())); } let actual = usize::try_from(npy.shape()[0]) .map_err(|_| PersistenceError::InvalidEriShape(npy.shape().to_vec()))?; - let basis_functions = if let Some(basis_functions) = basis_functions { - let expected = CompactEri::checked_storage_len(basis_functions).ok_or( - PersistenceError::InvalidValueCount { - basis_functions, - expected: 0, - actual, - }, - )?; - if actual != expected { - return Err(PersistenceError::InvalidValueCount { - basis_functions, - expected, - actual, - }); - } - basis_functions - } else { - basis_functions_for_len(actual) - .ok_or_else(|| PersistenceError::InvalidEriShape(npy.shape().to_vec()))? - }; + let expected = CompactEri::checked_storage_len(basis_functions).ok_or( + PersistenceError::InvalidValueCount { + basis_functions, + expected: 0, + actual, + }, + )?; + if actual != expected { + return Err(PersistenceError::InvalidValueCount { + basis_functions, + expected, + actual, + }); + } let values = npy.into_vec::().map_err(|error| { if error.kind() == std::io::ErrorKind::InvalidData { PersistenceError::InvalidDtype(error.to_string()) @@ -178,19 +172,6 @@ fn decode_compact_eri( }) } -fn basis_functions_for_len(length: usize) -> Option { - let (mut low, mut high) = (0, usize::MAX); - while low <= high { - let middle = low + (high - low) / 2; - match CompactEri::checked_storage_len(middle) { - Some(actual) if actual == length => return Some(middle), - Some(actual) if actual < length => low = middle + 1, - _ => high = middle.checked_sub(1)?, - } - } - None -} - pub(crate) fn validate_compact_eri_header( reader: impl Read + Seek, basis_functions: usize, @@ -284,14 +265,6 @@ mod tests { write_compact_eri(&mut bytes, &source).unwrap(); let restored = read_compact_eri(bytes.as_slice(), basis_functions).unwrap(); assert_eq!(restored.ordered_values(), source.ordered_values()); - let inferred = CompactEri::read_npy(bytes.as_slice()).unwrap(); - assert_eq!(inferred.ordered_values(), source.ordered_values()); - assert_eq!( - CompactEri::from_npy(NpyFile::new(bytes.as_slice()).unwrap()) - .unwrap() - .ordered_values(), - source.ordered_values() - ); assert_eq!( CompactEri::try_read_with_shape(bytes.as_slice(), basis_functions) .unwrap() @@ -339,11 +312,7 @@ mod tests { assert_eq!(npy.order(), npyz::Order::Fortran); assert_eq!(npy.shape(), &[2, 3]); assert_eq!(npy.into_vec::().unwrap(), matrix.as_slice()); - assert_eq!(DMatrix::::read_npy(bytes.as_slice()).unwrap(), matrix); - assert_eq!( - DMatrix::::from_npy(NpyFile::new(bytes.as_slice()).unwrap()).unwrap(), - matrix - ); + assert_eq!(read_dmatrix(bytes.as_slice()).unwrap(), matrix); assert_eq!( DMatrix::::try_from_npy_with_shape( NpyFile::new(bytes.as_slice()).unwrap(), @@ -399,7 +368,7 @@ mod tests { writer.finish().unwrap(); assert!(matches!( read_compact_eri(bytes.as_slice(), 2), - Err(PersistenceError::InvalidValueCount { .. }) + Err(PersistenceError::InvalidNpyShape { .. }) )); } diff --git a/docs/persistence-format-v1.md b/docs/persistence-format-v1.md index a332328..e9fd650 100644 --- a/docs/persistence-format-v1.md +++ b/docs/persistence-format-v1.md @@ -97,19 +97,21 @@ as verified byte streams. Its public scientific API is typed: `set_eri` and `Result, PersistenceError>`, while `set::(value)` accepts only `CompactEri`; only declared artifact marker types are accepted. A future known artifact gets its own marker, typed field, -and accessors in `RustiQData`. The `NpyConvert` trait handles NPY byte streams -for `CompactEri` and `DMatrix`. Its associated `Shape` type is `usize` for -the ERI basis-function count and `(usize, usize)` for matrix dimensions; -`try_read_with_shape` checks the declared shape before constructing the value. -`from_npy` and `try_from_npy_with_shape` also accept an already parsed -`npyz::NpyFile`, so callers can parse a stream once without using a filesystem -path. Matrix readers accept C and Fortran order and matrix writers emit Fortran -order. +and accessors in `RustiQData`. NPY parsing and conversion remain internal +persistence details. Compact ERIs are decoded only with the basis-function count +from their typed manifest attributes; the NPY length is never used to infer that +scientific context. Matrix readers accept C and Fortran order and matrix writers +emit Fortran order. - `manifest.json` is UTF-8 JSON and describes the format, producer, scientific identity and artifacts. - `arrays/integrals/ao-eri.npy` is the AO electron-repulsion integral artifact. +Artifact paths are portable relative UTF-8 paths with `/` as the only separator, +independent of the host operating system. Empty components, `.`, `..`, +backslashes, drive-like prefixes containing `:`, absolute paths, and paths +conflicting with `manifest.json` are rejected before filesystem access. + Every artifact records common envelope metadata: - logical path; From 68ef17892054e3382bec9105bb78eb676c2ef6fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 10:18:47 +0200 Subject: [PATCH 03/62] Fix private NPY matrix test --- crates/rustiq-core/src/persistence/npy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rustiq-core/src/persistence/npy.rs b/crates/rustiq-core/src/persistence/npy.rs index 562e3b3..3eddecc 100644 --- a/crates/rustiq-core/src/persistence/npy.rs +++ b/crates/rustiq-core/src/persistence/npy.rs @@ -349,7 +349,7 @@ mod tests { writer.extend([1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap(); writer.finish().unwrap(); assert_eq!( - DMatrix::::read_npy(c_bytes.as_slice()).unwrap(), + read_dmatrix(c_bytes.as_slice()).unwrap(), matrix ); } From 4ebc31288970d1a377d949cf1aea08386fc997ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 10:47:31 +0200 Subject: [PATCH 04/62] Format persistence paths --- crates/rustiq-core/src/persistence/data.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/crates/rustiq-core/src/persistence/data.rs b/crates/rustiq-core/src/persistence/data.rs index f849b5f..85a4da9 100644 --- a/crates/rustiq-core/src/persistence/data.rs +++ b/crates/rustiq-core/src/persistence/data.rs @@ -15,9 +15,8 @@ use crate::{ use super::{ ao_eri_identity, read_compact_eri, sha256_reader, write_compact_eri, AoEriAttributes, ArtifactAttributes, ArtifactManifest, Manifest, PersistenceError, Producer, - ScientificIdentityManifest, - AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, - FORMAT_VERSION, MANIFEST_PATH, + ScientificIdentityManifest, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, + COMPACT_ERI_REPRESENTATION, FORMAT_NAME, FORMAT_VERSION, MANIFEST_PATH, }; pub(crate) const AO_ERI_ARTIFACT: &str = "ao_eri"; @@ -479,10 +478,7 @@ mod tests { #[test] fn artifact_paths_are_portable_and_relative() { - assert!(validate_artifact_path(RelativePath::new( - "arrays/integrals/ao-eri.npy" - )) - .is_ok()); + assert!(validate_artifact_path(RelativePath::new("arrays/integrals/ao-eri.npy")).is_ok()); for path in [ "", From 2d0350eeb2db1ba3b08a0d6b0cc754e1a2cb5b17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 10:47:34 +0200 Subject: [PATCH 05/62] Format NPY persistence tests --- crates/rustiq-core/src/persistence/npy.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/rustiq-core/src/persistence/npy.rs b/crates/rustiq-core/src/persistence/npy.rs index 3eddecc..ba02d42 100644 --- a/crates/rustiq-core/src/persistence/npy.rs +++ b/crates/rustiq-core/src/persistence/npy.rs @@ -348,10 +348,7 @@ mod tests { .unwrap(); writer.extend([1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap(); writer.finish().unwrap(); - assert_eq!( - read_dmatrix(c_bytes.as_slice()).unwrap(), - matrix - ); + assert_eq!(read_dmatrix(c_bytes.as_slice()).unwrap(), matrix); } #[test] From 72c218cfb889a6cc66b06ee7c0358f92934f7b6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 12:29:17 +0200 Subject: [PATCH 06/62] Hide persistence manifest internals --- crates/rustiq-core/src/persistence/mod.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/rustiq-core/src/persistence/mod.rs b/crates/rustiq-core/src/persistence/mod.rs index 102543c..81ab577 100644 --- a/crates/rustiq-core/src/persistence/mod.rs +++ b/crates/rustiq-core/src/persistence/mod.rs @@ -18,7 +18,7 @@ pub use data::{AoEriArtifact, Artifact, RustiQData}; pub use eri_cache::{EriCache, EriCacheEntry}; #[allow(unused_imports)] pub(crate) use identity::{ao_eri_identity, ScientificIdentity, AO_ERI_COMPUTATION_VERSION}; -pub use manifest::{ +pub(crate) use manifest::{ AoEriAttributes, ArtifactAttributes, ArtifactManifest, Manifest, Producer, ScientificIdentityManifest, }; @@ -28,12 +28,12 @@ pub(crate) use npy::{ read_compact_eri, read_dmatrix, validate_compact_eri_header, write_compact_eri, }; -pub const FORMAT_NAME: &str = "rustiq-persistence"; -pub const FORMAT_VERSION: u32 = 1; -pub const MANIFEST_PATH: &str = "manifest.json"; -pub const AO_ERI_PATH: &str = "arrays/integrals/ao-eri.npy"; -pub const COMPACT_ERI_REPRESENTATION: &str = "rustiq-compact-eri-v1"; -pub const SCIENTIFIC_IDENTITY_VERSION: u32 = 1; +pub(crate) const FORMAT_NAME: &str = "rustiq-persistence"; +pub(crate) const FORMAT_VERSION: u32 = 1; +pub(crate) const MANIFEST_PATH: &str = "manifest.json"; +pub(crate) const AO_ERI_PATH: &str = "arrays/integrals/ao-eri.npy"; +pub(crate) const COMPACT_ERI_REPRESENTATION: &str = "rustiq-compact-eri-v1"; +pub(crate) const SCIENTIFIC_IDENTITY_VERSION: u32 = 1; use thiserror::Error; From 806f51ff538d4335af21937d6acec7ed51d279bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 12:29:19 +0200 Subject: [PATCH 07/62] Make manifest types crate-private --- crates/rustiq-core/src/persistence/manifest.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/rustiq-core/src/persistence/manifest.rs b/crates/rustiq-core/src/persistence/manifest.rs index cc7f201..e7c7431 100644 --- a/crates/rustiq-core/src/persistence/manifest.rs +++ b/crates/rustiq-core/src/persistence/manifest.rs @@ -7,7 +7,7 @@ use serde_json::Value; use super::{Sha256Digest, COMPACT_ERI_REPRESENTATION}; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct Manifest { +pub(crate) struct Manifest { pub format: String, pub format_version: u32, pub kind: String, @@ -17,13 +17,13 @@ pub struct Manifest { } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct Producer { +pub(crate) struct Producer { pub name: String, pub version: String, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ScientificIdentityManifest { +pub(crate) struct ScientificIdentityManifest { pub version: u32, pub digest: Sha256Digest, } @@ -34,20 +34,20 @@ pub struct ScientificIdentityManifest { /// retain their attributes so newer manifests remain inspectable by older readers. #[derive(Clone, Debug, PartialEq, Eq, Serialize)] #[serde(untagged)] -pub enum ArtifactAttributes { +pub(crate) enum ArtifactAttributes { AoEri(AoEriAttributes), Unknown(BTreeMap), } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct AoEriAttributes { +pub(crate) struct AoEriAttributes { pub basis_functions: usize, pub computation_version: u32, } #[derive(Clone, Debug, PartialEq, Eq, Serialize)] -pub struct ArtifactManifest { +pub(crate) struct ArtifactManifest { pub path: RelativePathBuf, pub size: u64, pub representation: String, From f252b12ffe0629a23c62359384f6c937a29c368a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 13:04:18 +0200 Subject: [PATCH 08/62] Add internal persistence storage --- crates/rustiq-core/src/persistence/storage.rs | 270 ++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 crates/rustiq-core/src/persistence/storage.rs diff --git a/crates/rustiq-core/src/persistence/storage.rs b/crates/rustiq-core/src/persistence/storage.rs new file mode 100644 index 0000000..502d80a --- /dev/null +++ b/crates/rustiq-core/src/persistence/storage.rs @@ -0,0 +1,270 @@ +use std::{ + fs::{self, File, OpenOptions}, + io::{BufReader, BufWriter, Read, Write}, + path::PathBuf, +}; + +use relative_path::RelativePath; +use serde::{de::DeserializeOwned, Serialize}; + +use super::{sha256_reader, PersistenceError, Sha256Digest}; + +#[derive(Debug)] +pub(crate) enum Storage { + Folder(FolderStorage), + // Later: Zip(ZipStorage). +} + +#[derive(Debug)] +pub(crate) struct FolderStorage { + root: PathBuf, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ArtifactMetadata { + pub(crate) size: u64, + pub(crate) digest: Sha256Digest, +} + +impl Storage { + pub(crate) fn folder(root: impl Into) -> Self { + Self::Folder(FolderStorage { root: root.into() }) + } + + pub(crate) fn artifact_metadata( + &self, + path: &RelativePath, + ) -> Result { + match self { + Self::Folder(folder) => folder.artifact_metadata(path), + } + } + + pub(crate) fn with_artifact( + &self, + path: &RelativePath, + read: F, + ) -> Result + where + F: FnOnce(&mut dyn Read) -> Result, + { + match self { + Self::Folder(folder) => folder.with_artifact(path, read), + } + } + + pub(crate) fn write_artifact( + &mut self, + path: &RelativePath, + write: F, + ) -> Result + where + F: FnOnce(&mut dyn Write) -> Result<(), PersistenceError>, + { + match self { + Self::Folder(folder) => folder.write_artifact(path, write), + } + } + + pub(crate) fn read_json( + &self, + path: &RelativePath, + max_size: u64, + ) -> Result { + let size = match self { + Self::Folder(folder) => folder.artifact_size(path)?, + }; + if size > max_size { + return Err(PersistenceError::InvalidManifest( + "manifest is larger than the supported limit".into(), + )); + } + + self.with_artifact(path, |reader| { + serde_json::from_reader(reader).map_err(PersistenceError::from) + }) + } + + pub(crate) fn write_json( + &mut self, + path: &RelativePath, + value: &T, + ) -> Result<(), PersistenceError> { + self.write_artifact(path, |writer| { + serde_json::to_writer_pretty(&mut *writer, value)?; + writer.write_all(b"\n")?; + Ok(()) + })?; + Ok(()) + } +} + +impl FolderStorage { + fn artifact_size(&self, path: &RelativePath) -> Result { + Ok(self.open_artifact(path)?.metadata()?.len()) + } + + fn artifact_metadata( + &self, + path: &RelativePath, + ) -> Result { + let file = self.open_artifact(path)?; + let size = file.metadata()?.len(); + let digest = sha256_reader(BufReader::new(file))?; + Ok(ArtifactMetadata { size, digest }) + } + + fn with_artifact( + &self, + path: &RelativePath, + read: F, + ) -> Result + where + F: FnOnce(&mut dyn Read) -> Result, + { + let mut reader = BufReader::new(self.open_artifact(path)?); + read(&mut reader) + } + + fn write_artifact( + &mut self, + path: &RelativePath, + write: F, + ) -> Result + where + F: FnOnce(&mut dyn Write) -> Result<(), PersistenceError>, + { + let file = self.create_artifact(path)?; + let mut writer = BufWriter::new(file); + write(&mut writer)?; + writer.flush()?; + let file = writer + .into_inner() + .map_err(|error| PersistenceError::Io(error.into_error()))?; + file.sync_all()?; + drop(file); + self.artifact_metadata(path) + } + + fn open_artifact(&self, path: &RelativePath) -> Result { + validate_path(path)?; + let components: Vec<_> = path.as_str().split('/').collect(); + let mut native = self.root.clone(); + + for (index, component) in components.iter().enumerate() { + native.push(component); + let metadata = fs::symlink_metadata(&native)?; + if metadata.file_type().is_symlink() { + return Err(PersistenceError::InvalidArtifact(format!( + "artifact contains a symbolic link: {path}" + ))); + } + + let is_last = index + 1 == components.len(); + if (!is_last && !metadata.is_dir()) || (is_last && !metadata.is_file()) { + return Err(PersistenceError::InvalidArtifact(format!( + "artifact path has an unexpected file type: {path}" + ))); + } + } + + Ok(File::open(native)?) + } + + fn create_artifact(&self, path: &RelativePath) -> Result { + validate_path(path)?; + let root_metadata = fs::symlink_metadata(&self.root)?; + if !root_metadata.is_dir() || root_metadata.file_type().is_symlink() { + return Err(PersistenceError::InvalidArtifact( + "storage root is not a regular directory".into(), + )); + } + + let mut components = path.as_str().split('/').peekable(); + let mut parent = self.root.clone(); + while let Some(component) = components.next() { + if components.peek().is_none() { + parent.push(component); + return Ok(OpenOptions::new() + .write(true) + .create_new(true) + .open(parent)?); + } + + parent.push(component); + match fs::symlink_metadata(&parent) { + Ok(metadata) => { + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(PersistenceError::InvalidArtifact(format!( + "artifact parent is not a regular directory: {path}" + ))); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + fs::create_dir(&parent)?; + } + Err(error) => return Err(error.into()), + } + } + + Err(PersistenceError::InvalidArtifact( + "artifact path is empty".into(), + )) + } +} + +pub(crate) fn validate_path(path: &RelativePath) -> Result<(), PersistenceError> { + let value = path.as_str(); + let invalid_component = value + .split('/') + .any(|component| component.is_empty() || matches!(component, "." | "..")); + + if value.is_empty() + || value.starts_with('/') + || value.contains('\\') + || value.contains(':') + || invalid_component + { + return Err(PersistenceError::InvalidArtifact(format!( + "unsafe artifact path: {path}" + ))); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn folder_storage_streams_artifacts_and_json() { + let root = tempfile::tempdir().unwrap(); + let mut storage = Storage::folder(root.path()); + let artifact_path = RelativePath::new("arrays/test.bin"); + + let metadata = storage + .write_artifact(artifact_path, |writer| { + writer.write_all(b"payload")?; + Ok(()) + }) + .unwrap(); + assert_eq!(metadata.size, 7); + + let payload = storage + .with_artifact(artifact_path, |reader| { + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes)?; + Ok(bytes) + }) + .unwrap(); + assert_eq!(payload, b"payload"); + + let manifest_path = RelativePath::new("manifest.json"); + storage + .write_json(manifest_path, &serde_json::json!({"version": 1})) + .unwrap(); + let manifest: serde_json::Value = storage.read_json(manifest_path, 1024).unwrap(); + assert_eq!(manifest["version"], 1); + } +} From 37cabda4993b8a28bac4dadb1a0c669d6d5b1747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 13:04:27 +0200 Subject: [PATCH 09/62] Wire persistence storage --- crates/rustiq-core/src/persistence/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/rustiq-core/src/persistence/mod.rs b/crates/rustiq-core/src/persistence/mod.rs index 81ab577..c7c2651 100644 --- a/crates/rustiq-core/src/persistence/mod.rs +++ b/crates/rustiq-core/src/persistence/mod.rs @@ -11,6 +11,7 @@ mod eri_cache; mod identity; mod manifest; mod npy; +mod storage; pub use crate::eri::CompactEri; pub use checksum::{sha256, sha256_reader, verify_sha256, Sha256Digest, Sha256DigestParseError}; @@ -27,6 +28,7 @@ pub(crate) use manifest::{ pub(crate) use npy::{ read_compact_eri, read_dmatrix, validate_compact_eri_header, write_compact_eri, }; +pub(crate) use storage::{validate_path as validate_storage_path, Storage}; pub(crate) const FORMAT_NAME: &str = "rustiq-persistence"; pub(crate) const FORMAT_VERSION: u32 = 1; From 025801bf971cbc1f5517ec723802c302d7ec6895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 13:04:52 +0200 Subject: [PATCH 10/62] Route RustiQData through storage --- crates/rustiq-core/src/persistence/data.rs | 142 ++++++--------------- 1 file changed, 36 insertions(+), 106 deletions(-) diff --git a/crates/rustiq-core/src/persistence/data.rs b/crates/rustiq-core/src/persistence/data.rs index 85a4da9..e45e456 100644 --- a/crates/rustiq-core/src/persistence/data.rs +++ b/crates/rustiq-core/src/persistence/data.rs @@ -1,8 +1,7 @@ use std::{ collections::{BTreeMap, HashSet}, - fs::{self, File}, - io::{BufReader, BufWriter, Seek, Write}, - path::{Path, PathBuf}, + fs, + path::Path, }; use relative_path::RelativePath; @@ -13,9 +12,9 @@ use crate::{ }; use super::{ - ao_eri_identity, read_compact_eri, sha256_reader, write_compact_eri, AoEriAttributes, - ArtifactAttributes, ArtifactManifest, Manifest, PersistenceError, Producer, - ScientificIdentityManifest, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, + ao_eri_identity, read_compact_eri, validate_storage_path, write_compact_eri, + AoEriAttributes, ArtifactAttributes, ArtifactManifest, Manifest, PersistenceError, Producer, + ScientificIdentityManifest, Storage, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, FORMAT_VERSION, MANIFEST_PATH, }; @@ -67,7 +66,7 @@ impl Artifact for AoEriArtifact { #[derive(Debug)] pub struct RustiQData { manifest: Manifest, - source: Option, + source: Option, ao_eri: Option, basis_functions: Option, } @@ -123,18 +122,9 @@ impl RustiQData { "entry is not a regular directory".into(), )); } - let manifest_path = directory.join(MANIFEST_PATH); - let metadata = fs::symlink_metadata(&manifest_path)?; - if !metadata.is_file() - || metadata.file_type().is_symlink() - || metadata.len() > MAX_MANIFEST_BYTES - { - return Err(PersistenceError::InvalidManifest( - "manifest is not a bounded regular file".into(), - )); - } + let source = Storage::folder(fs::canonicalize(directory)?); let manifest: Manifest = - serde_json::from_reader(BufReader::new(File::open(manifest_path)?))?; + source.read_json(RelativePath::new(MANIFEST_PATH), MAX_MANIFEST_BYTES)?; if manifest.format != FORMAT_NAME || manifest.format_version != FORMAT_VERSION { return Err(PersistenceError::InvalidManifest( "unsupported format or version".into(), @@ -166,7 +156,7 @@ impl RustiQData { }); Ok(Self { manifest, - source: Some(fs::canonicalize(directory)?), + source: Some(source), ao_eri: None, basis_functions, }) @@ -206,17 +196,15 @@ impl RustiQData { )); } let source = self.source.as_ref().ok_or(PersistenceError::MissingEri)?; - let mut file = open_artifact(source, artifact)?; - if sha256_reader(&mut file)? != artifact.digest { + let metadata = source.artifact_metadata(&artifact.path)?; + if metadata.size != artifact.size || metadata.digest != artifact.digest { return Err(PersistenceError::InvalidArtifact( - "AO ERI digest mismatch".into(), + "AO ERI digest or size mismatch".into(), )); } - file.rewind()?; - self.ao_eri = Some(read_compact_eri( - BufReader::new(file), - attributes.basis_functions, - )?); + self.ao_eri = Some(source.with_artifact(&artifact.path, |reader| { + read_compact_eri(reader, attributes.basis_functions) + })?); } Ok(self .ao_eri @@ -261,6 +249,7 @@ impl RustiQData { .prefix(".rustiq-data-") .tempdir_in(parent)?; let mut manifest = self.manifest.clone(); + let mut destination = Storage::folder(staging.path()); for (name, artifact) in &self.manifest.artifacts { if name == AO_ERI_ARTIFACT && eri.is_some() { @@ -269,39 +258,34 @@ impl RustiQData { let source = self.source.as_ref().ok_or_else(|| { PersistenceError::InvalidArtifact(format!("artifact {name} has no source file")) })?; - let mut input = open_artifact(source, artifact)?; validate_artifact_path(&artifact.path)?; - let output_path = artifact.path.to_path(staging.path()); - fs::create_dir_all(output_path.parent().expect("artifact has a parent"))?; - let mut output = BufWriter::new(File::create(&output_path)?); - std::io::copy(&mut input, &mut output)?; - output.flush()?; - output - .into_inner() - .map_err(|error| error.into_error())? - .sync_all()?; - verify_artifact(&output_path, artifact)?; + let metadata = source.with_artifact(&artifact.path, |input| { + destination.write_artifact(&artifact.path, |output| { + std::io::copy(input, output)?; + Ok(()) + }) + })?; + if metadata.size != artifact.size || metadata.digest != artifact.digest { + return Err(PersistenceError::InvalidArtifact(format!( + "artifact digest or size mismatch: {}", + artifact.path + ))); + } } if let Some(eri) = eri { let basis_functions = self.basis_functions.ok_or(PersistenceError::MissingEri)?; validate_eri_len(eri, basis_functions)?; - let path = RelativePath::new(AO_ERI_PATH).to_path(staging.path()); - fs::create_dir_all(path.parent().expect("ERI artifact has a parent"))?; - let mut output = BufWriter::new(File::create(&path)?); - write_compact_eri(&mut output, eri)?; - output.flush()?; - output - .into_inner() - .map_err(|error| error.into_error())? - .sync_all()?; + let path = RelativePath::new(AO_ERI_PATH); + let metadata = + destination.write_artifact(path, |writer| write_compact_eri(writer, eri))?; manifest.artifacts.insert( AO_ERI_ARTIFACT.to_owned(), ArtifactManifest { path: AO_ERI_PATH.into(), - size: fs::metadata(&path)?.len(), + size: metadata.size, representation: COMPACT_ERI_REPRESENTATION.to_owned(), - digest: sha256_reader(BufReader::new(File::open(&path)?))?, + digest: metadata.digest, attributes: ArtifactAttributes::AoEri(AoEriAttributes { basis_functions, computation_version: AO_ERI_COMPUTATION_VERSION, @@ -312,14 +296,7 @@ impl RustiQData { if manifest.kind == CACHE_KIND && !manifest.artifacts.contains_key(AO_ERI_ARTIFACT) { return Err(PersistenceError::MissingEri); } - let mut output = BufWriter::new(File::create(staging.path().join(MANIFEST_PATH))?); - serde_json::to_writer_pretty(&mut output, &manifest)?; - output.write_all(b"\n")?; - output.flush()?; - output - .into_inner() - .map_err(|error| error.into_error())? - .sync_all()?; + destination.write_json(RelativePath::new(MANIFEST_PATH), &manifest)?; fs::rename(staging.path(), directory)?; Ok(()) } @@ -342,19 +319,8 @@ fn validate_eri_len(eri: &CompactEri, basis_functions: usize) -> Result<(), Pers } fn validate_artifact_path(path: &RelativePath) -> Result<(), PersistenceError> { - let value = path.as_str(); - let invalid_component = value - .split('/') - .any(|component| component.is_empty() || matches!(component, "." | "..")); - let conflicts_with_manifest = value.split('/').next() == Some(MANIFEST_PATH); - - if value.is_empty() - || value.starts_with('/') - || value.contains('\\') - || value.contains(':') - || invalid_component - || conflicts_with_manifest - { + validate_storage_path(path)?; + if path.as_str().split('/').next() == Some(MANIFEST_PATH) { return Err(PersistenceError::InvalidArtifact(format!( "unsafe artifact path: {path}" ))); @@ -372,42 +338,6 @@ fn paths_conflict(left: &str, right: &str) -> bool { .is_some_and(|suffix| suffix.starts_with('/')) } -fn open_artifact(root: &Path, artifact: &ArtifactManifest) -> Result { - validate_artifact_path(&artifact.path)?; - let mut path = root.to_path_buf(); - for component in artifact.path.as_str().split('/') { - path.push(component); - let metadata = fs::symlink_metadata(&path)?; - if metadata.file_type().is_symlink() { - return Err(PersistenceError::InvalidArtifact(format!( - "artifact contains a symbolic link: {}", - artifact.path - ))); - } - } - let metadata = fs::symlink_metadata(&path)?; - if !metadata.is_file() || metadata.len() != artifact.size { - return Err(PersistenceError::InvalidArtifact(format!( - "artifact size or file type mismatch: {}", - artifact.path - ))); - } - Ok(File::open(path)?) -} - -fn verify_artifact(path: &Path, artifact: &ArtifactManifest) -> Result<(), PersistenceError> { - let file = File::open(path)?; - if file.metadata()?.len() != artifact.size - || sha256_reader(BufReader::new(file))? != artifact.digest - { - return Err(PersistenceError::InvalidArtifact(format!( - "artifact digest or size mismatch: {}", - artifact.path - ))); - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; From 87047afe56010db00a4685b0d0281504431d7ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 13:46:34 +0200 Subject: [PATCH 11/62] Harden persistence storage --- crates/rustiq-core/src/persistence/storage.rs | 144 +++++++++++++++--- 1 file changed, 119 insertions(+), 25 deletions(-) diff --git a/crates/rustiq-core/src/persistence/storage.rs b/crates/rustiq-core/src/persistence/storage.rs index 502d80a..7334329 100644 --- a/crates/rustiq-core/src/persistence/storage.rs +++ b/crates/rustiq-core/src/persistence/storage.rs @@ -1,11 +1,12 @@ use std::{ fs::{self, File, OpenOptions}, - io::{BufReader, BufWriter, Read, Write}, + io::{self, BufReader, BufWriter, Read, Write}, path::PathBuf, }; use relative_path::RelativePath; use serde::{de::DeserializeOwned, Serialize}; +use sha2::{Digest, Sha256}; use super::{sha256_reader, PersistenceError, Sha256Digest}; @@ -26,13 +27,57 @@ pub(crate) struct ArtifactMetadata { pub(crate) digest: Sha256Digest, } +struct DigestWriter { + inner: W, + hasher: Sha256, + size: u64, +} + +impl DigestWriter { + fn new(inner: W) -> Self { + Self { + inner, + hasher: Sha256::new(), + size: 0, + } + } + + fn finish(self) -> (W, ArtifactMetadata) { + ( + self.inner, + ArtifactMetadata { + size: self.size, + digest: self.hasher.finalize().into(), + }, + ) + } +} + +impl Write for DigestWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + let written = self.inner.write(buffer)?; + self.hasher.update(&buffer[..written]); + let written = u64::try_from(written) + .map_err(|_| io::Error::other("artifact size exceeds u64"))?; + self.size = self + .size + .checked_add(written) + .ok_or_else(|| io::Error::other("artifact size exceeds u64"))?; + Ok(usize::try_from(written).expect("written byte count originated as usize")) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + impl Storage { pub(crate) fn folder(root: impl Into) -> Self { Self::Folder(FolderStorage { root: root.into() }) } pub(crate) fn artifact_metadata( - &self, + &mut self, path: &RelativePath, ) -> Result { match self { @@ -41,7 +86,7 @@ impl Storage { } pub(crate) fn with_artifact( - &self, + &mut self, path: &RelativePath, read: F, ) -> Result @@ -67,7 +112,7 @@ impl Storage { } pub(crate) fn read_json( - &self, + &mut self, path: &RelativePath, max_size: u64, ) -> Result { @@ -97,6 +142,12 @@ impl Storage { })?; Ok(()) } + + pub(crate) fn finish(self) -> Result<(), PersistenceError> { + match self { + Self::Folder(_) => Ok(()), + } + } } impl FolderStorage { @@ -104,10 +155,7 @@ impl FolderStorage { Ok(self.open_artifact(path)?.metadata()?.len()) } - fn artifact_metadata( - &self, - path: &RelativePath, - ) -> Result { + fn artifact_metadata(&self, path: &RelativePath) -> Result { let file = self.open_artifact(path)?; let size = file.metadata()?.len(); let digest = sha256_reader(BufReader::new(file))?; @@ -135,15 +183,16 @@ impl FolderStorage { F: FnOnce(&mut dyn Write) -> Result<(), PersistenceError>, { let file = self.create_artifact(path)?; - let mut writer = BufWriter::new(file); + let writer = BufWriter::new(file); + let mut writer = DigestWriter::new(writer); write(&mut writer)?; writer.flush()?; + let (writer, metadata) = writer.finish(); let file = writer .into_inner() .map_err(|error| PersistenceError::Io(error.into_error()))?; file.sync_all()?; - drop(file); - self.artifact_metadata(path) + Ok(metadata) } fn open_artifact(&self, path: &RelativePath) -> Result { @@ -200,7 +249,7 @@ impl FolderStorage { ))); } } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + Err(error) if error.kind() == io::ErrorKind::NotFound => { fs::create_dir(&parent)?; } Err(error) => return Err(error.into()), @@ -213,26 +262,52 @@ impl FolderStorage { } } +pub(crate) fn portable_path_key(path: &RelativePath) -> Result { + validate_path(path)?; + Ok(path.as_str().to_lowercase()) +} + pub(crate) fn validate_path(path: &RelativePath) -> Result<(), PersistenceError> { let value = path.as_str(); - let invalid_component = value - .split('/') - .any(|component| component.is_empty() || matches!(component, "." | "..")); - - if value.is_empty() - || value.starts_with('/') - || value.contains('\\') - || value.contains(':') - || invalid_component - { - return Err(PersistenceError::InvalidArtifact(format!( - "unsafe artifact path: {path}" - ))); + if value.is_empty() || value.starts_with('/') || value.contains('\\') { + return Err(unsafe_path(path)); + } + + for component in value.split('/') { + if component.is_empty() + || matches!(component, "." | "..") + || component.ends_with(['.', ' ']) + || component.chars().any(|character| { + character < '\u{20}' || matches!(character, '<' | '>' | ':' | '"' | '|' | '?' | '*') + }) + || is_windows_reserved_name(component) + { + return Err(unsafe_path(path)); + } } Ok(()) } +fn is_windows_reserved_name(component: &str) -> bool { + let stem = component + .split('.') + .next() + .unwrap_or(component) + .to_ascii_uppercase(); + + matches!( + stem.as_str(), + "CON" | "PRN" | "AUX" | "NUL" | "CONIN$" | "CONOUT$" + ) || (stem.len() == 4 + && matches!(&stem[..3], "COM" | "LPT") + && matches!(stem.as_bytes()[3], b'1'..=b'9')) +} + +fn unsafe_path(path: &RelativePath) -> PersistenceError { + PersistenceError::InvalidArtifact(format!("unsafe artifact path: {path}")) +} + #[cfg(test)] mod tests { use super::*; @@ -250,6 +325,7 @@ mod tests { }) .unwrap(); assert_eq!(metadata.size, 7); + assert_eq!(metadata.digest, super::super::sha256(b"payload")); let payload = storage .with_artifact(artifact_path, |reader| { @@ -266,5 +342,23 @@ mod tests { .unwrap(); let manifest: serde_json::Value = storage.read_json(manifest_path, 1024).unwrap(); assert_eq!(manifest["version"], 1); + storage.finish().unwrap(); + } + + #[test] + fn rejects_non_portable_windows_paths() { + for path in [ + "CON", + "con.npy", + "arrays/NUL.bin", + "arrays/COM1.npy", + "arrays/LPT9.npy", + "arrays/trailing.", + "arrays/trailing ", + "arrays/bad?.npy", + "arrays/bad|name.npy", + ] { + assert!(validate_path(RelativePath::new(path)).is_err(), "{path}"); + } } } From e71034112c6315d63d9ef926238ce7e5394d2bcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 13:46:55 +0200 Subject: [PATCH 12/62] Prepare RustiQData for multiple storages --- crates/rustiq-core/src/persistence/data.rs | 23 +++++++++++----------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/crates/rustiq-core/src/persistence/data.rs b/crates/rustiq-core/src/persistence/data.rs index e45e456..0d9f0ab 100644 --- a/crates/rustiq-core/src/persistence/data.rs +++ b/crates/rustiq-core/src/persistence/data.rs @@ -122,7 +122,7 @@ impl RustiQData { "entry is not a regular directory".into(), )); } - let source = Storage::folder(fs::canonicalize(directory)?); + let mut source = Storage::folder(fs::canonicalize(directory)?); let manifest: Manifest = source.read_json(RelativePath::new(MANIFEST_PATH), MAX_MANIFEST_BYTES)?; if manifest.format != FORMAT_NAME || manifest.format_version != FORMAT_VERSION { @@ -130,19 +130,17 @@ impl RustiQData { "unsupported format or version".into(), )); } - let mut paths: HashSet<&str> = HashSet::new(); + let mut paths: HashSet = HashSet::new(); for artifact in manifest.artifacts.values() { validate_artifact_path(&artifact.path)?; - if paths - .iter() - .any(|other| paths_conflict(other, artifact.path.as_str())) - { + let key = artifact.path.as_str().to_lowercase(); + if paths.iter().any(|other| paths_conflict(other, &key)) { return Err(PersistenceError::InvalidManifest(format!( "artifact path conflicts with another artifact: {}", artifact.path ))); } - paths.insert(artifact.path.as_str()); + paths.insert(key); } let basis_functions = manifest .artifacts @@ -195,7 +193,7 @@ impl RustiQData { "unsupported AO ERI representation".into(), )); } - let source = self.source.as_ref().ok_or(PersistenceError::MissingEri)?; + let source = self.source.as_mut().ok_or(PersistenceError::MissingEri)?; let metadata = source.artifact_metadata(&artifact.path)?; if metadata.size != artifact.size || metadata.digest != artifact.digest { return Err(PersistenceError::InvalidArtifact( @@ -217,12 +215,12 @@ impl RustiQData { } /// Writes to a new directory, copying unloaded artifacts without decoding them. - pub fn write(&self, directory: impl AsRef) -> Result<(), PersistenceError> { + pub fn write(&mut self, directory: impl AsRef) -> Result<(), PersistenceError> { self.write_inner(directory.as_ref(), self.ao_eri.as_ref()) } pub(crate) fn write_with_eri( - &self, + &mut self, directory: impl AsRef, eri: Option<&CompactEri>, ) -> Result<(), PersistenceError> { @@ -230,7 +228,7 @@ impl RustiQData { } fn write_inner( - &self, + &mut self, directory: &Path, eri: Option<&CompactEri>, ) -> Result<(), PersistenceError> { @@ -255,7 +253,7 @@ impl RustiQData { if name == AO_ERI_ARTIFACT && eri.is_some() { continue; } - let source = self.source.as_ref().ok_or_else(|| { + let source = self.source.as_mut().ok_or_else(|| { PersistenceError::InvalidArtifact(format!("artifact {name} has no source file")) })?; validate_artifact_path(&artifact.path)?; @@ -297,6 +295,7 @@ impl RustiQData { return Err(PersistenceError::MissingEri); } destination.write_json(RelativePath::new(MANIFEST_PATH), &manifest)?; + destination.finish()?; fs::rename(staging.path(), directory)?; Ok(()) } From de30a642c60ed02457064607dd94f7de2fd15c50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 13:49:38 +0200 Subject: [PATCH 13/62] Expose selectable persistence storage --- crates/rustiq-core/src/persistence/storage.rs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/crates/rustiq-core/src/persistence/storage.rs b/crates/rustiq-core/src/persistence/storage.rs index 7334329..4e037b5 100644 --- a/crates/rustiq-core/src/persistence/storage.rs +++ b/crates/rustiq-core/src/persistence/storage.rs @@ -11,16 +11,11 @@ use sha2::{Digest, Sha256}; use super::{sha256_reader, PersistenceError, Sha256Digest}; #[derive(Debug)] -pub(crate) enum Storage { - Folder(FolderStorage), +pub enum Storage { + Folder(PathBuf), // Later: Zip(ZipStorage). } -#[derive(Debug)] -pub(crate) struct FolderStorage { - root: PathBuf, -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct ArtifactMetadata { pub(crate) size: u64, @@ -72,8 +67,8 @@ impl Write for DigestWriter { } impl Storage { - pub(crate) fn folder(root: impl Into) -> Self { - Self::Folder(FolderStorage { root: root.into() }) + pub fn folder(root: impl Into) -> Self { + Self::Folder(root.into()) } pub(crate) fn artifact_metadata( @@ -81,7 +76,7 @@ impl Storage { path: &RelativePath, ) -> Result { match self { - Self::Folder(folder) => folder.artifact_metadata(path), + Self::Folder(root) => FolderStorage { root: root.clone() }.artifact_metadata(path), } } @@ -94,7 +89,7 @@ impl Storage { F: FnOnce(&mut dyn Read) -> Result, { match self { - Self::Folder(folder) => folder.with_artifact(path, read), + Self::Folder(root) => FolderStorage { root: root.clone() }.with_artifact(path, read), } } @@ -107,7 +102,7 @@ impl Storage { F: FnOnce(&mut dyn Write) -> Result<(), PersistenceError>, { match self { - Self::Folder(folder) => folder.write_artifact(path, write), + Self::Folder(root) => FolderStorage { root: root.clone() }.write_artifact(path, write), } } @@ -117,7 +112,7 @@ impl Storage { max_size: u64, ) -> Result { let size = match self { - Self::Folder(folder) => folder.artifact_size(path)?, + Self::Folder(root) => FolderStorage { root: root.clone() }.artifact_size(path)?, }; if size > max_size { return Err(PersistenceError::InvalidManifest( @@ -150,6 +145,11 @@ impl Storage { } } +#[derive(Debug)] +struct FolderStorage { + root: PathBuf, +} + impl FolderStorage { fn artifact_size(&self, path: &RelativePath) -> Result { Ok(self.open_artifact(path)?.metadata()?.len()) From b4a17204bfb68e9ea07548fd8c393299ae010e7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 13:50:27 +0200 Subject: [PATCH 14/62] Expose persistence storage selection --- crates/rustiq-core/src/persistence/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/rustiq-core/src/persistence/mod.rs b/crates/rustiq-core/src/persistence/mod.rs index c7c2651..d7c40a3 100644 --- a/crates/rustiq-core/src/persistence/mod.rs +++ b/crates/rustiq-core/src/persistence/mod.rs @@ -17,6 +17,7 @@ pub use crate::eri::CompactEri; pub use checksum::{sha256, sha256_reader, verify_sha256, Sha256Digest, Sha256DigestParseError}; pub use data::{AoEriArtifact, Artifact, RustiQData}; pub use eri_cache::{EriCache, EriCacheEntry}; +pub use storage::Storage; #[allow(unused_imports)] pub(crate) use identity::{ao_eri_identity, ScientificIdentity, AO_ERI_COMPUTATION_VERSION}; pub(crate) use manifest::{ @@ -28,7 +29,7 @@ pub(crate) use manifest::{ pub(crate) use npy::{ read_compact_eri, read_dmatrix, validate_compact_eri_header, write_compact_eri, }; -pub(crate) use storage::{validate_path as validate_storage_path, Storage}; +pub(crate) use storage::validate_path as validate_storage_path; pub(crate) const FORMAT_NAME: &str = "rustiq-persistence"; pub(crate) const FORMAT_VERSION: u32 = 1; From 4a74c44af583688551abb57e3264e679c848a353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 13:50:30 +0200 Subject: [PATCH 15/62] Make RustiQData storage-selectable --- crates/rustiq-core/src/persistence/data.rs | 70 +++++++--------------- 1 file changed, 22 insertions(+), 48 deletions(-) diff --git a/crates/rustiq-core/src/persistence/data.rs b/crates/rustiq-core/src/persistence/data.rs index 0d9f0ab..bf38271 100644 --- a/crates/rustiq-core/src/persistence/data.rs +++ b/crates/rustiq-core/src/persistence/data.rs @@ -1,8 +1,4 @@ -use std::{ - collections::{BTreeMap, HashSet}, - fs, - path::Path, -}; +use std::collections::{BTreeMap, HashSet}; use relative_path::RelativePath; @@ -113,16 +109,8 @@ impl RustiQData { } } - /// Reads only the bounded manifest. NPY values are opened on demand. - pub fn read(directory: impl AsRef) -> Result { - let directory = directory.as_ref(); - let metadata = fs::symlink_metadata(directory)?; - if !metadata.is_dir() || metadata.file_type().is_symlink() { - return Err(PersistenceError::InvalidManifest( - "entry is not a regular directory".into(), - )); - } - let mut source = Storage::folder(fs::canonicalize(directory)?); + /// Reads only the bounded manifest. Scientific artifacts are opened on demand. + pub fn read(mut source: Storage) -> Result { let manifest: Manifest = source.read_json(RelativePath::new(MANIFEST_PATH), MAX_MANIFEST_BYTES)?; if manifest.format != FORMAT_NAME || manifest.format_version != FORMAT_VERSION { @@ -214,40 +202,28 @@ impl RustiQData { self.ao_eri.take() } - /// Writes to a new directory, copying unloaded artifacts without decoding them. - pub fn write(&mut self, directory: impl AsRef) -> Result<(), PersistenceError> { - self.write_inner(directory.as_ref(), self.ao_eri.as_ref()) + /// Writes to the selected storage, copying unloaded artifacts without decoding them. + pub fn write(&mut self, destination: Storage) -> Result<(), PersistenceError> { + let eri = self.ao_eri.take(); + let result = self.write_inner(destination, eri.as_ref()); + self.ao_eri = eri; + result } pub(crate) fn write_with_eri( &mut self, - directory: impl AsRef, + destination: Storage, eri: Option<&CompactEri>, ) -> Result<(), PersistenceError> { - self.write_inner(directory.as_ref(), eri) + self.write_inner(destination, eri) } fn write_inner( &mut self, - directory: &Path, + mut destination: Storage, eri: Option<&CompactEri>, ) -> Result<(), PersistenceError> { - if directory.exists() { - return Err(PersistenceError::Io(std::io::Error::new( - std::io::ErrorKind::AlreadyExists, - "persistence destination already exists", - ))); - } - let parent = directory - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .unwrap_or_else(|| Path::new(".")); - fs::create_dir_all(parent)?; - let staging = tempfile::Builder::new() - .prefix(".rustiq-data-") - .tempdir_in(parent)?; let mut manifest = self.manifest.clone(); - let mut destination = Storage::folder(staging.path()); for (name, artifact) in &self.manifest.artifacts { if name == AO_ERI_ARTIFACT && eri.is_some() { @@ -295,9 +271,7 @@ impl RustiQData { return Err(PersistenceError::MissingEri); } destination.write_json(RelativePath::new(MANIFEST_PATH), &manifest)?; - destination.finish()?; - fs::rename(staging.path(), directory)?; - Ok(()) + destination.finish() } } @@ -359,8 +333,8 @@ mod tests { let mut data = new_data(); assert!(data.get::().unwrap().is_none()); data.set::(CompactEri::Zeroed(2)).unwrap(); - data.write(&entry).unwrap(); - let mut restored = RustiQData::read(&entry).unwrap(); + data.write(Storage::folder(&entry)).unwrap(); + let mut restored = RustiQData::read(Storage::folder(&entry)).unwrap(); assert!(restored.ao_eri.is_none()); let first = restored.get::().unwrap().unwrap() as *const CompactEri; fs::remove_file(entry.join(AO_ERI_PATH)).unwrap(); @@ -376,7 +350,7 @@ mod tests { let second = root.path().join("second"); let mut data = new_data(); data.set_eri(CompactEri::Zeroed(2)).unwrap(); - data.write(&first).unwrap(); + data.write(Storage::folder(&first)).unwrap(); let unknown = b"opaque future data"; let unknown_path = first.join("arrays/post-hf/future.npy"); fs::create_dir_all(unknown_path.parent().unwrap()).unwrap(); @@ -395,14 +369,14 @@ mod tests { }, ); fs::write(&manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap(); - let restored = RustiQData::read(&first).unwrap(); + let restored = RustiQData::read(Storage::folder(&first)).unwrap(); assert!(restored.ao_eri.is_none()); - restored.write(&second).unwrap(); + restored.write(Storage::folder(&second)).unwrap(); assert_eq!( fs::read(second.join("arrays/post-hf/future.npy")).unwrap(), unknown ); - assert_eq!(RustiQData::read(&second).unwrap().manifest, manifest); + assert_eq!(RustiQData::read(Storage::folder(&second)).unwrap().manifest, manifest); } #[test] @@ -435,10 +409,10 @@ mod tests { let second = root.path().join("second"); let mut data = new_data(); data.set_eri(CompactEri::Zeroed(2)).unwrap(); - data.write(&first).unwrap(); + data.write(Storage::folder(&first)).unwrap(); fs::write(first.join(AO_ERI_PATH), b"bad").unwrap(); assert!(matches!( - RustiQData::read(&first).unwrap().write(&second), + RustiQData::read(Storage::folder(&first)).unwrap().write(&second), Err(PersistenceError::InvalidArtifact(_)) )); assert!(!second.exists()); @@ -458,7 +432,7 @@ mod tests { ); fs::write(&manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap(); assert!(matches!( - RustiQData::read(&first), + RustiQData::read(Storage::folder(&first)), Err(PersistenceError::InvalidManifest(_)) )); } From 9f1caa96d144c4eabb38f4286651f224989fe97f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 13:50:48 +0200 Subject: [PATCH 16/62] Use selectable storage in ERI cache --- .../rustiq-core/src/persistence/eri_cache.rs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/crates/rustiq-core/src/persistence/eri_cache.rs b/crates/rustiq-core/src/persistence/eri_cache.rs index fddc4c7..653ba53 100644 --- a/crates/rustiq-core/src/persistence/eri_cache.rs +++ b/crates/rustiq-core/src/persistence/eri_cache.rs @@ -13,7 +13,7 @@ use crate::{ use super::{ ao_eri_identity, sha256_reader, validate_compact_eri_header, AoEriAttributes, - ArtifactAttributes, ArtifactManifest, Manifest, RustiQData, ScientificIdentity, + ArtifactAttributes, ArtifactManifest, Manifest, RustiQData, ScientificIdentity, Storage, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, FORMAT_VERSION, SCIENTIFIC_IDENTITY_VERSION, }; @@ -309,7 +309,7 @@ impl EriCache { if !metadata.is_dir() || metadata.file_type().is_symlink() { return None; } - let mut data = RustiQData::read(&entry).ok()?; + let mut data = RustiQData::read(Storage::folder(&entry)).ok()?; let manifest = data.manifest(); let artifact = manifest.artifacts.get(AO_ERI_ARTIFACT)?; let attributes = ao_eri_attributes(artifact)?; @@ -338,13 +338,11 @@ impl EriCache { } fs::create_dir_all(parent)?; let temporary = Builder::new().prefix(".rustiq-eri-").tempdir_in(parent)?; - let entry_data = RustiQData::new_with_identity(identity, basis_functions); - let staged_entry = temporary.path().join("entry"); + let mut entry_data = RustiQData::new_with_identity(identity, basis_functions); entry_data - .write_with_eri(&staged_entry, Some(eri)) + .write_with_eri(Storage::folder(temporary.path()), Some(eri)) .map_err(io::Error::other)?; let temporary_path = temporary.keep(); - let staged_entry = temporary_path.join("entry"); if fs::symlink_metadata(&final_entry).is_ok() { if self.load_identity(identity, basis_functions).is_some() { let _ = fs::remove_dir_all(temporary_path); @@ -355,11 +353,8 @@ impl EriCache { return Err(error); } } - match fs::rename(&staged_entry, &final_entry) { - Ok(()) => { - let _ = fs::remove_dir_all(temporary_path); - Ok(()) - } + match fs::rename(&temporary_path, &final_entry) { + Ok(()) => Ok(()), Err(error) => { let _ = fs::remove_dir_all(temporary_path); if self.load_identity(identity, basis_functions).is_some() { @@ -373,7 +368,7 @@ impl EriCache { } fn read_manifest(entry: &Path) -> Option { - RustiQData::read(entry) + RustiQData::read(Storage::folder(entry)) .ok() .map(|data| data.manifest().clone()) } From 40b37505266a37bd4ea1594500b52d3723c020a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 13:51:08 +0200 Subject: [PATCH 17/62] Add NumPy matrix interoperability fixture --- .../tests/data/persistence/dmatrix-python-c-v1.npy.hex | 1 + 1 file changed, 1 insertion(+) create mode 100644 crates/rustiq-core/tests/data/persistence/dmatrix-python-c-v1.npy.hex diff --git a/crates/rustiq-core/tests/data/persistence/dmatrix-python-c-v1.npy.hex b/crates/rustiq-core/tests/data/persistence/dmatrix-python-c-v1.npy.hex new file mode 100644 index 0000000..becde52 --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/dmatrix-python-c-v1.npy.hex @@ -0,0 +1 @@ +934e554d5059010076007b276465736372273a20273c6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028322c2033292c207d20202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a000000000000f03f00000000000000400000000000000840000000000000104000000000000014400000000000001840 From d5312ed73965fef4d2a6bcff72d1cca981409b8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 13:51:10 +0200 Subject: [PATCH 18/62] Add NumPy matrix interoperability fixture --- .../tests/data/persistence/dmatrix-python-fortran-v1.npy.hex | 1 + 1 file changed, 1 insertion(+) create mode 100644 crates/rustiq-core/tests/data/persistence/dmatrix-python-fortran-v1.npy.hex diff --git a/crates/rustiq-core/tests/data/persistence/dmatrix-python-fortran-v1.npy.hex b/crates/rustiq-core/tests/data/persistence/dmatrix-python-fortran-v1.npy.hex new file mode 100644 index 0000000..34e329d --- /dev/null +++ b/crates/rustiq-core/tests/data/persistence/dmatrix-python-fortran-v1.npy.hex @@ -0,0 +1 @@ +934e554d5059010076007b276465736372273a20273c6638272c2027666f727472616e5f6f72646572273a20547275652c20277368617065273a2028322c2033292c207d20202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a000000000000f03f00000000000010400000000000000040000000000000144000000000000008400000000000001840 From 9f9d0c11a18cb23d15411dfca37434e82c283e91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 13:51:19 +0200 Subject: [PATCH 19/62] Test NumPy matrix interoperability --- crates/rustiq-core/src/persistence/npy.rs | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/crates/rustiq-core/src/persistence/npy.rs b/crates/rustiq-core/src/persistence/npy.rs index ba02d42..512c19c 100644 --- a/crates/rustiq-core/src/persistence/npy.rs +++ b/crates/rustiq-core/src/persistence/npy.rs @@ -351,6 +351,32 @@ mod tests { assert_eq!(read_dmatrix(c_bytes.as_slice()).unwrap(), matrix); } + #[test] + fn reads_python_numpy_matrix_fixtures_in_c_and_fortran_order() { + for hex in [ + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/data/persistence/dmatrix-python-c-v1.npy.hex" + )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/data/persistence/dmatrix-python-fortran-v1.npy.hex" + )), + ] { + let (pairs, remainder) = hex.trim().as_bytes().as_chunks::<2>(); + assert!(remainder.is_empty()); + let bytes: Vec = pairs + .iter() + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect(); + + assert_eq!( + read_dmatrix(bytes.as_slice()).unwrap(), + DMatrix::from_row_slice(2, 3, &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) + ); + } + } + #[test] fn rejects_wrong_value_count() { let mut bytes = Vec::new(); From b3710ba3002cdf9f88bfb16272dc1ca757a80bd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 13:51:37 +0200 Subject: [PATCH 20/62] Document selectable persistence storage --- docs/persistence-format-v1.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/persistence-format-v1.md b/docs/persistence-format-v1.md index e9fd650..e74ee04 100644 --- a/docs/persistence-format-v1.md +++ b/docs/persistence-format-v1.md @@ -87,12 +87,13 @@ untouched. Names and fingerprints are validated rather than interpreted as paths ## Logical entries -The core Rust API exposes `persistence::RustiQData` for reading and writing a -directory in this format. `RustiQData::read` reads the manifest only. Its +The core Rust API exposes `persistence::RustiQData` together with a selectable +`persistence::Storage`. V1 currently provides `Storage::Folder(PathBuf)`; a ZIP-backed +variant can be added without changing the `RustiQData` type. `RustiQData::read(storage)` +reads the manifest only. Its `read_eri` method validates and decodes the AO ERI NPY on first access and keeps -the resulting `CompactEri` for subsequent accesses. Writing to a new directory -copies artifacts that have not been decoded, including unknown representations, -as verified byte streams. Its public scientific API is typed: `set_eri` and +the resulting `CompactEri` for subsequent accesses. Writing to a selected storage copies artifacts that have not been decoded, +including unknown representations, as verified byte streams. Its public scientific API is typed: `set_eri` and `read_eri` operate on `CompactEri`. The generic `get::()` returns `Result, PersistenceError>`, while `set::(value)` accepts only `CompactEri`; only declared artifact @@ -109,8 +110,10 @@ emit Fortran order. Artifact paths are portable relative UTF-8 paths with `/` as the only separator, independent of the host operating system. Empty components, `.`, `..`, -backslashes, drive-like prefixes containing `:`, absolute paths, and paths -conflicting with `manifest.json` are rejected before filesystem access. +backslashes, drive-like prefixes containing `:`, absolute paths, Windows-reserved +device names, Windows-invalid filename characters, trailing dots/spaces, and paths +conflicting with `manifest.json` are rejected before storage access. Logical paths +are also checked case-insensitively to avoid cross-platform collisions. Every artifact records common envelope metadata: From 503aafdcc75399903b20781d6a7a9001391a58a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 13:52:13 +0200 Subject: [PATCH 21/62] Make storage enum extensible --- crates/rustiq-core/src/persistence/storage.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/rustiq-core/src/persistence/storage.rs b/crates/rustiq-core/src/persistence/storage.rs index 4e037b5..771d1ab 100644 --- a/crates/rustiq-core/src/persistence/storage.rs +++ b/crates/rustiq-core/src/persistence/storage.rs @@ -10,10 +10,14 @@ use sha2::{Digest, Sha256}; use super::{sha256_reader, PersistenceError, Sha256Digest}; +/// Physical storage selected for RustiQ persistence. +/// +/// The folder variant is available in V1. Additional variants, such as the +/// portable ZIP/ZIP64 container, can be added without changing `RustiQData`. +#[non_exhaustive] #[derive(Debug)] pub enum Storage { Folder(PathBuf), - // Later: Zip(ZipStorage). } #[derive(Clone, Copy, Debug, PartialEq, Eq)] From 8fb142d743b16a7b8f65d7b3a71c3e5a619ada0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 13:52:15 +0200 Subject: [PATCH 22/62] Fix storage-aware persistence tests --- crates/rustiq-core/src/persistence/data.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/rustiq-core/src/persistence/data.rs b/crates/rustiq-core/src/persistence/data.rs index bf38271..e6d4cab 100644 --- a/crates/rustiq-core/src/persistence/data.rs +++ b/crates/rustiq-core/src/persistence/data.rs @@ -369,7 +369,7 @@ mod tests { }, ); fs::write(&manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap(); - let restored = RustiQData::read(Storage::folder(&first)).unwrap(); + let mut restored = RustiQData::read(Storage::folder(&first)).unwrap(); assert!(restored.ao_eri.is_none()); restored.write(Storage::folder(&second)).unwrap(); assert_eq!( @@ -411,8 +411,9 @@ mod tests { data.set_eri(CompactEri::Zeroed(2)).unwrap(); data.write(Storage::folder(&first)).unwrap(); fs::write(first.join(AO_ERI_PATH), b"bad").unwrap(); + let mut restored = RustiQData::read(Storage::folder(&first)).unwrap(); assert!(matches!( - RustiQData::read(Storage::folder(&first)).unwrap().write(&second), + restored.write(Storage::folder(&second)), Err(PersistenceError::InvalidArtifact(_)) )); assert!(!second.exists()); From ee045520ca848749b5631f6cbaef9719992a8be3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 12:27:21 +0000 Subject: [PATCH 23/62] Split ile and avoid vector shape --- Cargo.lock | 2 +- crates/rustiq-core/src/persistence/data.rs | 448 +----------------- .../src/persistence/data/ao_eri_artifact.rs | 29 ++ .../src/persistence/data/artifact.rs | 16 + .../src/persistence/data/rustiq_data.rs | 407 ++++++++++++++++ crates/rustiq-core/src/persistence/mod.rs | 6 +- crates/rustiq-core/src/persistence/npy.rs | 21 +- 7 files changed, 477 insertions(+), 452 deletions(-) create mode 100644 crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs create mode 100644 crates/rustiq-core/src/persistence/data/artifact.rs create mode 100644 crates/rustiq-core/src/persistence/data/rustiq_data.rs diff --git a/Cargo.lock b/Cargo.lock index 4cdf9ce..2ad6567 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2588,7 +2588,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "relative-path", + "relative-path 1.9.3", "rustc_version", "syn 2.0.119", "unicode-ident", diff --git a/crates/rustiq-core/src/persistence/data.rs b/crates/rustiq-core/src/persistence/data.rs index e6d4cab..4f47954 100644 --- a/crates/rustiq-core/src/persistence/data.rs +++ b/crates/rustiq-core/src/persistence/data.rs @@ -1,440 +1,8 @@ -use std::collections::{BTreeMap, HashSet}; - -use relative_path::RelativePath; - -use crate::{ - basis::Basis, config::validated::PositiveFiniteF64, eri::CompactEri, - molecules::geometry::Geometry, -}; - -use super::{ - ao_eri_identity, read_compact_eri, validate_storage_path, write_compact_eri, - AoEriAttributes, ArtifactAttributes, ArtifactManifest, Manifest, PersistenceError, Producer, - ScientificIdentityManifest, Storage, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, - COMPACT_ERI_REPRESENTATION, FORMAT_NAME, FORMAT_VERSION, MANIFEST_PATH, -}; - -pub(crate) const AO_ERI_ARTIFACT: &str = "ao_eri"; -pub(crate) const CACHE_KIND: &str = "integral-cache"; -const MAX_MANIFEST_BYTES: u64 = 1024 * 1024; - -mod private { - pub trait Sealed {} -} - -/// A known scientific artifact. Only RustiQ's declared artifact markers implement this trait. -pub trait Artifact: private::Sealed { - type Value; - - #[doc(hidden)] - fn get(data: &mut RustiQData) -> Result, PersistenceError>; - #[doc(hidden)] - fn set(data: &mut RustiQData, value: Self::Value) -> Result<(), PersistenceError>; -} - -/// AO electron-repulsion integrals in compact storage order. -pub struct AoEriArtifact; - -impl private::Sealed for AoEriArtifact {} - -impl Artifact for AoEriArtifact { - type Value = CompactEri; - - fn get(data: &mut RustiQData) -> Result, PersistenceError> { - if data.ao_eri.is_some() { - return Ok(data.ao_eri.as_ref()); - } - if !data.manifest.artifacts.contains_key(AO_ERI_ARTIFACT) { - return Ok(None); - } - data.read_eri().map(Some) - } - - fn set(data: &mut RustiQData, value: CompactEri) -> Result<(), PersistenceError> { - data.set_eri(value) - } -} - -/// Known scientific artifacts and their manifest, with NPY values loaded on demand. -/// -/// Add a typed field and accessors here when RustiQ gains a new scientific artifact. -/// Unknown manifest entries remain opaque and are copied when writing a new entry. -#[derive(Debug)] -pub struct RustiQData { - manifest: Manifest, - source: Option, - ao_eri: Option, - basis_functions: Option, -} - -impl RustiQData { - /// Gets a known artifact, loading and caching its value on first access. - pub fn get(&mut self) -> Result, PersistenceError> { - A::get(self) - } - - /// Sets a known artifact using its statically selected value type. - pub fn set(&mut self, value: A::Value) -> Result<(), PersistenceError> { - A::set(self, value) - } - - /// Starts a new AO ERI entry with the same scientific identity as the cache. - pub fn new(geometry: &Geometry, basis: &Basis, threshold: Option) -> Self { - let identity = ao_eri_identity(geometry, basis, threshold); - Self::new_with_identity(identity, basis.nbasis()) - } - - pub(crate) fn new_with_identity( - identity: super::ScientificIdentity, - basis_functions: usize, - ) -> Self { - Self { - manifest: Manifest { - format: FORMAT_NAME.to_owned(), - format_version: FORMAT_VERSION, - kind: CACHE_KIND.to_owned(), - producer: Producer { - name: "RustiQ".to_owned(), - version: env!("CARGO_PKG_VERSION").to_owned(), - }, - scientific_identity: ScientificIdentityManifest { - version: identity.version, - digest: identity.digest, - }, - artifacts: BTreeMap::new(), - }, - source: None, - ao_eri: None, - basis_functions: Some(basis_functions), - } - } - - /// Reads only the bounded manifest. Scientific artifacts are opened on demand. - pub fn read(mut source: Storage) -> Result { - let manifest: Manifest = - source.read_json(RelativePath::new(MANIFEST_PATH), MAX_MANIFEST_BYTES)?; - if manifest.format != FORMAT_NAME || manifest.format_version != FORMAT_VERSION { - return Err(PersistenceError::InvalidManifest( - "unsupported format or version".into(), - )); - } - let mut paths: HashSet = HashSet::new(); - for artifact in manifest.artifacts.values() { - validate_artifact_path(&artifact.path)?; - let key = artifact.path.as_str().to_lowercase(); - if paths.iter().any(|other| paths_conflict(other, &key)) { - return Err(PersistenceError::InvalidManifest(format!( - "artifact path conflicts with another artifact: {}", - artifact.path - ))); - } - paths.insert(key); - } - let basis_functions = manifest - .artifacts - .get(AO_ERI_ARTIFACT) - .and_then(|artifact| { - if let ArtifactAttributes::AoEri(attributes) = &artifact.attributes { - Some(attributes.basis_functions) - } else { - None - } - }); - Ok(Self { - manifest, - source: Some(source), - ao_eri: None, - basis_functions, - }) - } - - pub(crate) fn manifest(&self) -> &Manifest { - &self.manifest - } - - /// Replaces the AO ERI artifact after checking its compact length. - pub fn set_eri(&mut self, eri: CompactEri) -> Result<(), PersistenceError> { - let basis_functions = self.basis_functions.ok_or(PersistenceError::MissingEri)?; - validate_eri_len(&eri, basis_functions)?; - self.ao_eri = Some(eri); - Ok(()) - } - - /// Validates and decodes the AO ERI NPY on first access, then reuses the object. - pub fn read_eri(&mut self) -> Result<&CompactEri, PersistenceError> { - if self.ao_eri.is_none() { - let artifact = self - .manifest - .artifacts - .get(AO_ERI_ARTIFACT) - .ok_or(PersistenceError::MissingEri)?; - let ArtifactAttributes::AoEri(attributes) = &artifact.attributes else { - return Err(PersistenceError::InvalidArtifact( - "AO ERI attributes are missing".into(), - )); - }; - if artifact.path.as_str() != AO_ERI_PATH - || artifact.representation != COMPACT_ERI_REPRESENTATION - || attributes.computation_version != AO_ERI_COMPUTATION_VERSION - { - return Err(PersistenceError::InvalidArtifact( - "unsupported AO ERI representation".into(), - )); - } - let source = self.source.as_mut().ok_or(PersistenceError::MissingEri)?; - let metadata = source.artifact_metadata(&artifact.path)?; - if metadata.size != artifact.size || metadata.digest != artifact.digest { - return Err(PersistenceError::InvalidArtifact( - "AO ERI digest or size mismatch".into(), - )); - } - self.ao_eri = Some(source.with_artifact(&artifact.path, |reader| { - read_compact_eri(reader, attributes.basis_functions) - })?); - } - Ok(self - .ao_eri - .as_ref() - .expect("ERI was loaded or already present")) - } - - pub(crate) fn take_eri(&mut self) -> Option { - self.ao_eri.take() - } - - /// Writes to the selected storage, copying unloaded artifacts without decoding them. - pub fn write(&mut self, destination: Storage) -> Result<(), PersistenceError> { - let eri = self.ao_eri.take(); - let result = self.write_inner(destination, eri.as_ref()); - self.ao_eri = eri; - result - } - - pub(crate) fn write_with_eri( - &mut self, - destination: Storage, - eri: Option<&CompactEri>, - ) -> Result<(), PersistenceError> { - self.write_inner(destination, eri) - } - - fn write_inner( - &mut self, - mut destination: Storage, - eri: Option<&CompactEri>, - ) -> Result<(), PersistenceError> { - let mut manifest = self.manifest.clone(); - - for (name, artifact) in &self.manifest.artifacts { - if name == AO_ERI_ARTIFACT && eri.is_some() { - continue; - } - let source = self.source.as_mut().ok_or_else(|| { - PersistenceError::InvalidArtifact(format!("artifact {name} has no source file")) - })?; - validate_artifact_path(&artifact.path)?; - let metadata = source.with_artifact(&artifact.path, |input| { - destination.write_artifact(&artifact.path, |output| { - std::io::copy(input, output)?; - Ok(()) - }) - })?; - if metadata.size != artifact.size || metadata.digest != artifact.digest { - return Err(PersistenceError::InvalidArtifact(format!( - "artifact digest or size mismatch: {}", - artifact.path - ))); - } - } - - if let Some(eri) = eri { - let basis_functions = self.basis_functions.ok_or(PersistenceError::MissingEri)?; - validate_eri_len(eri, basis_functions)?; - let path = RelativePath::new(AO_ERI_PATH); - let metadata = - destination.write_artifact(path, |writer| write_compact_eri(writer, eri))?; - manifest.artifacts.insert( - AO_ERI_ARTIFACT.to_owned(), - ArtifactManifest { - path: AO_ERI_PATH.into(), - size: metadata.size, - representation: COMPACT_ERI_REPRESENTATION.to_owned(), - digest: metadata.digest, - attributes: ArtifactAttributes::AoEri(AoEriAttributes { - basis_functions, - computation_version: AO_ERI_COMPUTATION_VERSION, - }), - }, - ); - } - if manifest.kind == CACHE_KIND && !manifest.artifacts.contains_key(AO_ERI_ARTIFACT) { - return Err(PersistenceError::MissingEri); - } - destination.write_json(RelativePath::new(MANIFEST_PATH), &manifest)?; - destination.finish() - } -} - -fn validate_eri_len(eri: &CompactEri, basis_functions: usize) -> Result<(), PersistenceError> { - let expected = CompactEri::checked_storage_len(basis_functions).ok_or_else(|| { - PersistenceError::InvalidArtifact( - "basis-function count overflows compact ERI storage".into(), - ) - })?; - if eri.len() != expected { - return Err(PersistenceError::InvalidValueCount { - basis_functions, - expected, - actual: eri.len(), - }); - } - Ok(()) -} - -fn validate_artifact_path(path: &RelativePath) -> Result<(), PersistenceError> { - validate_storage_path(path)?; - if path.as_str().split('/').next() == Some(MANIFEST_PATH) { - return Err(PersistenceError::InvalidArtifact(format!( - "unsafe artifact path: {path}" - ))); - } - Ok(()) -} - -fn paths_conflict(left: &str, right: &str) -> bool { - left == right - || left - .strip_prefix(right) - .is_some_and(|suffix| suffix.starts_with('/')) - || right - .strip_prefix(left) - .is_some_and(|suffix| suffix.starts_with('/')) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::persistence::{sha256, Sha256Digest, SCIENTIFIC_IDENTITY_VERSION}; - - fn new_data() -> RustiQData { - RustiQData::new_with_identity( - super::super::ScientificIdentity { - version: SCIENTIFIC_IDENTITY_VERSION, - digest: Sha256Digest::from([7; 32]), - }, - 2, - ) - } - - #[test] - fn eri_is_loaded_only_on_request_and_then_cached_as_an_object() { - let root = tempfile::tempdir().unwrap(); - let entry = root.path().join("entry"); - let mut data = new_data(); - assert!(data.get::().unwrap().is_none()); - data.set::(CompactEri::Zeroed(2)).unwrap(); - data.write(Storage::folder(&entry)).unwrap(); - let mut restored = RustiQData::read(Storage::folder(&entry)).unwrap(); - assert!(restored.ao_eri.is_none()); - let first = restored.get::().unwrap().unwrap() as *const CompactEri; - fs::remove_file(entry.join(AO_ERI_PATH)).unwrap(); - let second = restored.get::().unwrap().unwrap() as *const CompactEri; - assert_eq!(first, second); - assert_eq!(restored.read_eri().unwrap().len(), 6); - } - - #[test] - fn unloaded_unknown_artifact_is_copied_without_decoding() { - let root = tempfile::tempdir().unwrap(); - let first = root.path().join("first"); - let second = root.path().join("second"); - let mut data = new_data(); - data.set_eri(CompactEri::Zeroed(2)).unwrap(); - data.write(Storage::folder(&first)).unwrap(); - let unknown = b"opaque future data"; - let unknown_path = first.join("arrays/post-hf/future.npy"); - fs::create_dir_all(unknown_path.parent().unwrap()).unwrap(); - fs::write(&unknown_path, unknown).unwrap(); - let manifest_path = first.join(MANIFEST_PATH); - let mut manifest: Manifest = - serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap(); - manifest.artifacts.insert( - "future".into(), - ArtifactManifest { - path: "arrays/post-hf/future.npy".into(), - size: unknown.len() as u64, - representation: "rustiq-future-v1".into(), - digest: sha256(unknown), - attributes: ArtifactAttributes::Unknown(BTreeMap::new()), - }, - ); - fs::write(&manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap(); - let mut restored = RustiQData::read(Storage::folder(&first)).unwrap(); - assert!(restored.ao_eri.is_none()); - restored.write(Storage::folder(&second)).unwrap(); - assert_eq!( - fs::read(second.join("arrays/post-hf/future.npy")).unwrap(), - unknown - ); - assert_eq!(RustiQData::read(Storage::folder(&second)).unwrap().manifest, manifest); - } - - #[test] - fn artifact_paths_are_portable_and_relative() { - assert!(validate_artifact_path(RelativePath::new("arrays/integrals/ao-eri.npy")).is_ok()); - - for path in [ - "", - "/arrays/integrals/ao-eri.npy", - "arrays\\integrals\\ao-eri.npy", - "C:/arrays/ao-eri.npy", - "arrays/../ao-eri.npy", - "arrays/./ao-eri.npy", - "arrays//ao-eri.npy", - "arrays/ao-eri.npy/", - "manifest.json", - "manifest.json/child", - ] { - assert!( - validate_artifact_path(RelativePath::new(path)).is_err(), - "{path} must be rejected" - ); - } - } - - #[test] - fn rejects_corrupt_payload_and_conflicting_paths() { - let root = tempfile::tempdir().unwrap(); - let first = root.path().join("first"); - let second = root.path().join("second"); - let mut data = new_data(); - data.set_eri(CompactEri::Zeroed(2)).unwrap(); - data.write(Storage::folder(&first)).unwrap(); - fs::write(first.join(AO_ERI_PATH), b"bad").unwrap(); - let mut restored = RustiQData::read(Storage::folder(&first)).unwrap(); - assert!(matches!( - restored.write(Storage::folder(&second)), - Err(PersistenceError::InvalidArtifact(_)) - )); - assert!(!second.exists()); - - let manifest_path = first.join(MANIFEST_PATH); - let mut manifest: Manifest = - serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap(); - manifest.artifacts.insert( - "future".into(), - ArtifactManifest { - path: "arrays/integrals".into(), - size: 0, - representation: "future-v1".into(), - digest: sha256(b""), - attributes: ArtifactAttributes::Unknown(BTreeMap::new()), - }, - ); - fs::write(&manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap(); - assert!(matches!( - RustiQData::read(Storage::folder(&first)), - Err(PersistenceError::InvalidManifest(_)) - )); - } -} +mod ao_eri_artifact; +mod artifact; +mod rustiq_data; + +pub use ao_eri_artifact::AoEriArtifact; +pub use artifact::Artifact; +pub use rustiq_data::RustiQData; +pub(crate) use rustiq_data::{AO_ERI_ARTIFACT, CACHE_KIND}; diff --git a/crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs b/crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs new file mode 100644 index 0000000..7309102 --- /dev/null +++ b/crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs @@ -0,0 +1,29 @@ +use crate::{eri::CompactEri, persistence::PersistenceError}; + +use super::{ + artifact::{private, Artifact}, + rustiq_data::{RustiQData, AO_ERI_ARTIFACT}, +}; + +/// AO electron-repulsion integrals in compact storage order. +pub struct AoEriArtifact; + +impl private::Sealed for AoEriArtifact {} + +impl Artifact for AoEriArtifact { + type Value = CompactEri; + + fn get(data: &mut RustiQData) -> Result, PersistenceError> { + if data.ao_eri.is_some() { + return Ok(data.ao_eri.as_ref()); + } + if !data.manifest.artifacts.contains_key(AO_ERI_ARTIFACT) { + return Ok(None); + } + data.read_eri().map(Some) + } + + fn set(data: &mut RustiQData, value: CompactEri) -> Result<(), PersistenceError> { + data.set_eri(value) + } +} diff --git a/crates/rustiq-core/src/persistence/data/artifact.rs b/crates/rustiq-core/src/persistence/data/artifact.rs new file mode 100644 index 0000000..91968fd --- /dev/null +++ b/crates/rustiq-core/src/persistence/data/artifact.rs @@ -0,0 +1,16 @@ +use super::rustiq_data::RustiQData; +use crate::persistence::PersistenceError; + +pub(crate) mod private { + pub trait Sealed {} +} + +/// A known scientific artifact. Only RustiQ's declared artifact markers implement this trait. +pub trait Artifact: private::Sealed { + type Value; + + #[doc(hidden)] + fn get(data: &mut RustiQData) -> Result, PersistenceError>; + #[doc(hidden)] + fn set(data: &mut RustiQData, value: Self::Value) -> Result<(), PersistenceError>; +} diff --git a/crates/rustiq-core/src/persistence/data/rustiq_data.rs b/crates/rustiq-core/src/persistence/data/rustiq_data.rs new file mode 100644 index 0000000..c68797d --- /dev/null +++ b/crates/rustiq-core/src/persistence/data/rustiq_data.rs @@ -0,0 +1,407 @@ +use std::collections::{BTreeMap, HashSet}; + +use relative_path::RelativePath; + +use crate::{ + basis::Basis, config::validated::PositiveFiniteF64, eri::CompactEri, + molecules::geometry::Geometry, +}; + +use super::super::{ + ao_eri_identity, read_compact_eri, validate_storage_path, write_compact_eri, AoEriAttributes, + ArtifactAttributes, ArtifactManifest, Manifest, PersistenceError, Producer, + ScientificIdentityManifest, Storage, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, + COMPACT_ERI_REPRESENTATION, FORMAT_NAME, FORMAT_VERSION, MANIFEST_PATH, +}; +use super::artifact::Artifact; + +pub(crate) const AO_ERI_ARTIFACT: &str = "ao_eri"; +pub(crate) const CACHE_KIND: &str = "integral-cache"; +const MAX_MANIFEST_BYTES: u64 = 1024 * 1024; + +/// Known scientific artifacts and their manifest, with NPY values loaded on demand. +/// +/// Add a typed field and accessors here when RustiQ gains a new scientific artifact. +/// Unknown manifest entries remain opaque and are copied when writing a new entry. +#[derive(Debug)] +pub struct RustiQData { + pub(super) manifest: Manifest, + source: Option, + pub(super) ao_eri: Option, + basis_functions: Option, +} + +impl RustiQData { + /// Gets a known artifact, loading and caching its value on first access. + pub fn get(&mut self) -> Result, PersistenceError> { + A::get(self) + } + + /// Sets a known artifact using its statically selected value type. + pub fn set(&mut self, value: A::Value) -> Result<(), PersistenceError> { + A::set(self, value) + } + + /// Starts a new AO ERI entry with the same scientific identity as the cache. + pub fn new(geometry: &Geometry, basis: &Basis, threshold: Option) -> Self { + let identity = ao_eri_identity(geometry, basis, threshold); + Self::new_with_identity(identity, basis.nbasis()) + } + + pub(crate) fn new_with_identity( + identity: super::super::ScientificIdentity, + basis_functions: usize, + ) -> Self { + Self { + manifest: Manifest { + format: FORMAT_NAME.to_owned(), + format_version: FORMAT_VERSION, + kind: CACHE_KIND.to_owned(), + producer: Producer { + name: "RustiQ".to_owned(), + version: env!("CARGO_PKG_VERSION").to_owned(), + }, + scientific_identity: ScientificIdentityManifest { + version: identity.version, + digest: identity.digest, + }, + artifacts: BTreeMap::new(), + }, + source: None, + ao_eri: None, + basis_functions: Some(basis_functions), + } + } + + /// Reads only the bounded manifest. Scientific artifacts are opened on demand. + pub fn read(mut source: Storage) -> Result { + let manifest: Manifest = + source.read_json(RelativePath::new(MANIFEST_PATH), MAX_MANIFEST_BYTES)?; + if manifest.format != FORMAT_NAME || manifest.format_version != FORMAT_VERSION { + return Err(PersistenceError::InvalidManifest( + "unsupported format or version".into(), + )); + } + let mut paths: HashSet = HashSet::new(); + for artifact in manifest.artifacts.values() { + validate_artifact_path(&artifact.path)?; + let key = artifact.path.as_str().to_lowercase(); + if paths.iter().any(|other| paths_conflict(other, &key)) { + return Err(PersistenceError::InvalidManifest(format!( + "artifact path conflicts with another artifact: {}", + artifact.path + ))); + } + paths.insert(key); + } + let basis_functions = manifest + .artifacts + .get(AO_ERI_ARTIFACT) + .and_then(|artifact| { + if let ArtifactAttributes::AoEri(attributes) = &artifact.attributes { + Some(attributes.basis_functions) + } else { + None + } + }); + Ok(Self { + manifest, + source: Some(source), + ao_eri: None, + basis_functions, + }) + } + + pub(crate) fn manifest(&self) -> &Manifest { + &self.manifest + } + + /// Replaces the AO ERI artifact after checking its compact length. + pub fn set_eri(&mut self, eri: CompactEri) -> Result<(), PersistenceError> { + let basis_functions = self.basis_functions.ok_or(PersistenceError::MissingEri)?; + validate_eri_len(&eri, basis_functions)?; + self.ao_eri = Some(eri); + Ok(()) + } + + /// Validates and decodes the AO ERI NPY on first access, then reuses the object. + pub fn read_eri(&mut self) -> Result<&CompactEri, PersistenceError> { + if self.ao_eri.is_none() { + let artifact = self + .manifest + .artifacts + .get(AO_ERI_ARTIFACT) + .ok_or(PersistenceError::MissingEri)?; + let ArtifactAttributes::AoEri(attributes) = &artifact.attributes else { + return Err(PersistenceError::InvalidArtifact( + "AO ERI attributes are missing".into(), + )); + }; + if artifact.path.as_str() != AO_ERI_PATH + || artifact.representation != COMPACT_ERI_REPRESENTATION + || attributes.computation_version != AO_ERI_COMPUTATION_VERSION + { + return Err(PersistenceError::InvalidArtifact( + "unsupported AO ERI representation".into(), + )); + } + let source = self.source.as_mut().ok_or(PersistenceError::MissingEri)?; + let metadata = source.artifact_metadata(&artifact.path)?; + if metadata.size != artifact.size || metadata.digest != artifact.digest { + return Err(PersistenceError::InvalidArtifact( + "AO ERI digest or size mismatch".into(), + )); + } + self.ao_eri = Some(source.with_artifact(&artifact.path, |reader| { + read_compact_eri(reader, attributes.basis_functions) + })?); + } + Ok(self + .ao_eri + .as_ref() + .expect("ERI was loaded or already present")) + } + + pub(crate) fn take_eri(&mut self) -> Option { + self.ao_eri.take() + } + + /// Writes to the selected storage, copying unloaded artifacts without decoding them. + pub fn write(&mut self, destination: Storage) -> Result<(), PersistenceError> { + let eri = self.ao_eri.take(); + let result = self.write_inner(destination, eri.as_ref()); + self.ao_eri = eri; + result + } + + pub(crate) fn write_with_eri( + &mut self, + destination: Storage, + eri: Option<&CompactEri>, + ) -> Result<(), PersistenceError> { + self.write_inner(destination, eri) + } + + fn write_inner( + &mut self, + mut destination: Storage, + eri: Option<&CompactEri>, + ) -> Result<(), PersistenceError> { + let mut manifest = self.manifest.clone(); + + for (name, artifact) in &self.manifest.artifacts { + if name == AO_ERI_ARTIFACT && eri.is_some() { + continue; + } + let source = self.source.as_mut().ok_or_else(|| { + PersistenceError::InvalidArtifact(format!("artifact {name} has no source file")) + })?; + validate_artifact_path(&artifact.path)?; + let metadata = source.with_artifact(&artifact.path, |input| { + destination.write_artifact(&artifact.path, |output| { + std::io::copy(input, output)?; + Ok(()) + }) + })?; + if metadata.size != artifact.size || metadata.digest != artifact.digest { + return Err(PersistenceError::InvalidArtifact(format!( + "artifact digest or size mismatch: {}", + artifact.path + ))); + } + } + + if let Some(eri) = eri { + let basis_functions = self.basis_functions.ok_or(PersistenceError::MissingEri)?; + validate_eri_len(eri, basis_functions)?; + let path = RelativePath::new(AO_ERI_PATH); + let metadata = + destination.write_artifact(path, |writer| write_compact_eri(writer, eri))?; + manifest.artifacts.insert( + AO_ERI_ARTIFACT.to_owned(), + ArtifactManifest { + path: AO_ERI_PATH.into(), + size: metadata.size, + representation: COMPACT_ERI_REPRESENTATION.to_owned(), + digest: metadata.digest, + attributes: ArtifactAttributes::AoEri(AoEriAttributes { + basis_functions, + computation_version: AO_ERI_COMPUTATION_VERSION, + }), + }, + ); + } + if manifest.kind == CACHE_KIND && !manifest.artifacts.contains_key(AO_ERI_ARTIFACT) { + return Err(PersistenceError::MissingEri); + } + destination.write_json(RelativePath::new(MANIFEST_PATH), &manifest)?; + destination.finish() + } +} + +fn validate_eri_len(eri: &CompactEri, basis_functions: usize) -> Result<(), PersistenceError> { + let expected = CompactEri::checked_storage_len(basis_functions).ok_or_else(|| { + PersistenceError::InvalidArtifact( + "basis-function count overflows compact ERI storage".into(), + ) + })?; + if eri.len() != expected { + return Err(PersistenceError::InvalidValueCount { + basis_functions, + expected, + actual: eri.len(), + }); + } + Ok(()) +} + +fn validate_artifact_path(path: &RelativePath) -> Result<(), PersistenceError> { + validate_storage_path(path)?; + if path.as_str().split('/').next() == Some(MANIFEST_PATH) { + return Err(PersistenceError::InvalidArtifact(format!( + "unsafe artifact path: {path}" + ))); + } + Ok(()) +} + +fn paths_conflict(left: &str, right: &str) -> bool { + left == right + || left + .strip_prefix(right) + .is_some_and(|suffix| suffix.starts_with('/')) + || right + .strip_prefix(left) + .is_some_and(|suffix| suffix.starts_with('/')) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::persistence::{sha256, Sha256Digest, SCIENTIFIC_IDENTITY_VERSION}; + + fn new_data() -> RustiQData { + RustiQData::new_with_identity( + crate::persistence::ScientificIdentity { + version: SCIENTIFIC_IDENTITY_VERSION, + digest: Sha256Digest::from([7; 32]), + }, + 2, + ) + } + + #[test] + fn eri_is_loaded_only_on_request_and_then_cached_as_an_object() { + let root = tempfile::tempdir().unwrap(); + let entry = root.path().join("entry"); + let mut data = new_data(); + assert!(data.get::().unwrap().is_none()); + data.set::(CompactEri::Zeroed(2)).unwrap(); + data.write(Storage::folder(&entry)).unwrap(); + let mut restored = RustiQData::read(Storage::folder(&entry)).unwrap(); + assert!(restored.ao_eri.is_none()); + let first = restored.get::().unwrap().unwrap() as *const CompactEri; + fs::remove_file(entry.join(AO_ERI_PATH)).unwrap(); + let second = restored.get::().unwrap().unwrap() as *const CompactEri; + assert_eq!(first, second); + assert_eq!(restored.read_eri().unwrap().len(), 6); + } + + #[test] + fn unloaded_unknown_artifact_is_copied_without_decoding() { + let root = tempfile::tempdir().unwrap(); + let first = root.path().join("first"); + let second = root.path().join("second"); + let mut data = new_data(); + data.set_eri(CompactEri::Zeroed(2)).unwrap(); + data.write(Storage::folder(&first)).unwrap(); + let unknown = b"opaque future data"; + let unknown_path = first.join("arrays/post-hf/future.npy"); + fs::create_dir_all(unknown_path.parent().unwrap()).unwrap(); + fs::write(&unknown_path, unknown).unwrap(); + let manifest_path = first.join(MANIFEST_PATH); + let mut manifest: Manifest = + serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap(); + manifest.artifacts.insert( + "future".into(), + ArtifactManifest { + path: "arrays/post-hf/future.npy".into(), + size: unknown.len() as u64, + representation: "rustiq-future-v1".into(), + digest: sha256(unknown), + attributes: ArtifactAttributes::Unknown(BTreeMap::new()), + }, + ); + fs::write(&manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap(); + let mut restored = RustiQData::read(Storage::folder(&first)).unwrap(); + assert!(restored.ao_eri.is_none()); + restored.write(Storage::folder(&second)).unwrap(); + assert_eq!( + fs::read(second.join("arrays/post-hf/future.npy")).unwrap(), + unknown + ); + assert_eq!( + RustiQData::read(Storage::folder(&second)).unwrap().manifest, + manifest + ); + } + + #[test] + fn artifact_paths_are_portable_and_relative() { + assert!(validate_artifact_path(RelativePath::new("arrays/integrals/ao-eri.npy")).is_ok()); + + for path in [ + "", + "/arrays/integrals/ao-eri.npy", + "arrays\\integrals\\ao-eri.npy", + "C:/arrays/ao-eri.npy", + "arrays/../ao-eri.npy", + "arrays/./ao-eri.npy", + "arrays//ao-eri.npy", + "arrays/ao-eri.npy/", + "manifest.json", + "manifest.json/child", + ] { + assert!( + validate_artifact_path(RelativePath::new(path)).is_err(), + "{path} must be rejected" + ); + } + } + + #[test] + fn rejects_corrupt_payload_and_conflicting_paths() { + let root = tempfile::tempdir().unwrap(); + let first = root.path().join("first"); + let second = root.path().join("second"); + let mut data = new_data(); + data.set_eri(CompactEri::Zeroed(2)).unwrap(); + data.write(Storage::folder(&first)).unwrap(); + fs::write(first.join(AO_ERI_PATH), b"bad").unwrap(); + let mut restored = RustiQData::read(Storage::folder(&first)).unwrap(); + assert!(matches!( + restored.write(Storage::folder(&second)), + Err(PersistenceError::InvalidArtifact(_)) + )); + assert!(!second.exists()); + + let manifest_path = first.join(MANIFEST_PATH); + let mut manifest: Manifest = + serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap(); + manifest.artifacts.insert( + "future".into(), + ArtifactManifest { + path: "arrays/integrals".into(), + size: 0, + representation: "future-v1".into(), + digest: sha256(b""), + attributes: ArtifactAttributes::Unknown(BTreeMap::new()), + }, + ); + fs::write(&manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap(); + assert!(matches!( + RustiQData::read(Storage::folder(&first)), + Err(PersistenceError::InvalidManifest(_)) + )); + } +} diff --git a/crates/rustiq-core/src/persistence/mod.rs b/crates/rustiq-core/src/persistence/mod.rs index d7c40a3..7758793 100644 --- a/crates/rustiq-core/src/persistence/mod.rs +++ b/crates/rustiq-core/src/persistence/mod.rs @@ -17,13 +17,13 @@ pub use crate::eri::CompactEri; pub use checksum::{sha256, sha256_reader, verify_sha256, Sha256Digest, Sha256DigestParseError}; pub use data::{AoEriArtifact, Artifact, RustiQData}; pub use eri_cache::{EriCache, EriCacheEntry}; -pub use storage::Storage; #[allow(unused_imports)] pub(crate) use identity::{ao_eri_identity, ScientificIdentity, AO_ERI_COMPUTATION_VERSION}; pub(crate) use manifest::{ AoEriAttributes, ArtifactAttributes, ArtifactManifest, Manifest, Producer, ScientificIdentityManifest, }; +pub use storage::Storage; #[allow(unused_imports)] pub(crate) use npy::{ @@ -62,8 +62,8 @@ pub enum PersistenceError { InvalidMatrixShape(Vec), #[error("NPY has shape {actual:?}, expected {expected:?}")] InvalidNpyShape { - expected: Vec, - actual: Vec, + expected: Box<[u64]>, + actual: Box<[u64]>, }, #[error("AO ERI payload has {actual} values, expected {expected} for {basis_functions} basis functions")] InvalidValueCount { diff --git a/crates/rustiq-core/src/persistence/npy.rs b/crates/rustiq-core/src/persistence/npy.rs index 512c19c..6aaca70 100644 --- a/crates/rustiq-core/src/persistence/npy.rs +++ b/crates/rustiq-core/src/persistence/npy.rs @@ -10,8 +10,9 @@ use super::PersistenceError; /// Conversion between scientific values and an NPY byte stream. trait NpyConvert: Sized { type Shape: Copy; + type NpyShape: AsRef<[u64]>; - fn npy_shape(shape: Self::Shape) -> Result, PersistenceError>; + fn npy_shape(shape: Self::Shape) -> Result; fn decode_with_shape( npy: NpyFile, @@ -24,9 +25,11 @@ trait NpyConvert: Sized { shape: Self::Shape, ) -> Result { let expected = Self::npy_shape(shape)?; - let actual = npy.shape().to_vec(); - if actual != expected { - return Err(PersistenceError::InvalidNpyShape { expected, actual }); + if npy.shape() != expected.as_ref() { + return Err(PersistenceError::InvalidNpyShape { + expected: expected.as_ref().to_vec().into_boxed_slice(), + actual: npy.shape().to_vec().into_boxed_slice(), + }); } Self::decode_with_shape(npy, shape) } @@ -47,14 +50,15 @@ trait NpyConvert: Sized { impl NpyConvert for CompactEri { type Shape = usize; + type NpyShape = [u64; 1]; - fn npy_shape(basis_functions: usize) -> Result, PersistenceError> { + fn npy_shape(basis_functions: usize) -> Result { let length = CompactEri::checked_storage_len(basis_functions).ok_or_else(|| { PersistenceError::InvalidArtifact( "basis-function count overflows compact ERI storage".into(), ) })?; - Ok(vec![u64::try_from(length).map_err(|_| { + Ok([u64::try_from(length).map_err(|_| { PersistenceError::InvalidArtifact("compact ERI length exceeds NPY limits".into()) })?]) } @@ -83,15 +87,16 @@ impl NpyConvert for CompactEri { impl NpyConvert for DMatrix { type Shape = (usize, usize); + type NpyShape = [u64; 2]; - fn npy_shape((rows, columns): (usize, usize)) -> Result, PersistenceError> { + fn npy_shape((rows, columns): (usize, usize)) -> Result { let rows = u64::try_from(rows).map_err(|_| { PersistenceError::InvalidArtifact("matrix rows exceed NPY limits".into()) })?; let columns = u64::try_from(columns).map_err(|_| { PersistenceError::InvalidArtifact("matrix columns exceed NPY limits".into()) })?; - Ok(vec![rows, columns]) + Ok([rows, columns]) } fn decode_with_shape( From 237d20400288738770e0cba4481ca2d5416a010c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:41:07 +0200 Subject: [PATCH 24/62] Split persistence error domains --- crates/rustiq-core/src/persistence/error.rs | 94 +++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 crates/rustiq-core/src/persistence/error.rs diff --git a/crates/rustiq-core/src/persistence/error.rs b/crates/rustiq-core/src/persistence/error.rs new file mode 100644 index 0000000..632d602 --- /dev/null +++ b/crates/rustiq-core/src/persistence/error.rs @@ -0,0 +1,94 @@ +use std::io; + +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum StorageError { + #[error("storage I/O failed: {0}")] + Io(#[from] io::Error), + #[error("unsafe storage path: {0}")] + InvalidPath(String), + #[error("storage entry has an unexpected type: {0}")] + UnexpectedEntryType(String), + #[error("artifact size exceeds the supported limit")] + SizeOverflow, +} + +#[derive(Debug, Error)] +pub enum NpyError { + #[error("could not read NPY data: {0}")] + Read(#[source] io::Error), + #[error("could not write NPY data: {0}")] + Write(#[source] io::Error), + #[error("AO ERI NPY must be one-dimensional, found shape {0:?}")] + InvalidEriShape(Box<[u64]>), + #[error("matrix NPY must be two-dimensional, found shape {0:?}")] + InvalidMatrixShape(Box<[u64]>), + #[error("NPY has shape {actual:?}, expected {expected:?}")] + InvalidShape { + expected: Box<[u64]>, + actual: Box<[u64]>, + }, + #[error("NPY dtype is not a supported f64 representation: {0}")] + InvalidDtype(String), + #[error("NPY dimensions exceed supported limits")] + DimensionOverflow, +} + +#[derive(Debug, Error)] +pub enum ManifestError { + #[error("manifest storage failed: {0}")] + Storage(#[from] StorageError), + #[error("could not decode or encode persistence manifest: {0}")] + Json(#[from] serde_json::Error), + #[error("manifest is larger than the supported limit")] + TooLarge, + #[error("unsupported persistence format or version")] + UnsupportedFormat, + #[error("artifact path conflicts with another artifact: {0}")] + ConflictingArtifactPath(String), +} + +#[derive(Debug, Error)] +pub enum ArtifactError { + #[error("AO ERI artifact is missing")] + Missing, + #[error("unsupported artifact representation: {0}")] + UnsupportedRepresentation(String), + #[error("artifact integrity check failed: {0}")] + IntegrityMismatch(String), + #[error("invalid artifact path: {0}")] + InvalidPath(String), + #[error( + "AO ERI payload has {actual} values, expected {expected} for {basis_functions} basis functions" + )] + InvalidValueCount { + basis_functions: usize, + expected: usize, + actual: usize, + }, + #[error("artifact storage failed: {0}")] + Storage(#[from] StorageError), + #[error("artifact NPY data is invalid: {0}")] + Npy(#[from] NpyError), +} + +#[derive(Debug, Error)] +pub(crate) enum PersistenceReadError { + #[error(transparent)] + Manifest(#[from] ManifestError), + #[error(transparent)] + Artifact(#[from] ArtifactError), +} + +#[derive(Debug, Error)] +pub(crate) enum PersistenceWriteError { + #[error(transparent)] + Storage(#[from] StorageError), + #[error(transparent)] + Manifest(#[from] ManifestError), + #[error(transparent)] + Artifact(#[from] ArtifactError), + #[error(transparent)] + Npy(#[from] NpyError), +} From e2cb9ec220f9393fd2878f6ec19d8bd0489e060c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:41:21 +0200 Subject: [PATCH 25/62] Expose focused persistence errors --- crates/rustiq-core/src/persistence/mod.rs | 40 ++--------------------- 1 file changed, 3 insertions(+), 37 deletions(-) diff --git a/crates/rustiq-core/src/persistence/mod.rs b/crates/rustiq-core/src/persistence/mod.rs index 7758793..b299198 100644 --- a/crates/rustiq-core/src/persistence/mod.rs +++ b/crates/rustiq-core/src/persistence/mod.rs @@ -8,6 +8,7 @@ mod cache_names; mod checksum; mod data; mod eri_cache; +mod error; mod identity; mod manifest; mod npy; @@ -17,13 +18,14 @@ pub use crate::eri::CompactEri; pub use checksum::{sha256, sha256_reader, verify_sha256, Sha256Digest, Sha256DigestParseError}; pub use data::{AoEriArtifact, Artifact, RustiQData}; pub use eri_cache::{EriCache, EriCacheEntry}; +pub use error::{ArtifactError, ManifestError, NpyError, StorageError}; #[allow(unused_imports)] pub(crate) use identity::{ao_eri_identity, ScientificIdentity, AO_ERI_COMPUTATION_VERSION}; pub(crate) use manifest::{ AoEriAttributes, ArtifactAttributes, ArtifactManifest, Manifest, Producer, ScientificIdentityManifest, }; -pub use storage::Storage; +pub(crate) use storage::Storage; #[allow(unused_imports)] pub(crate) use npy::{ @@ -38,39 +40,3 @@ pub(crate) const AO_ERI_PATH: &str = "arrays/integrals/ao-eri.npy"; pub(crate) const COMPACT_ERI_REPRESENTATION: &str = "rustiq-compact-eri-v1"; pub(crate) const SCIENTIFIC_IDENTITY_VERSION: u32 = 1; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum PersistenceError { - #[error("persistence I/O failed: {0}")] - Io(#[from] std::io::Error), - #[error("could not read or write persistence manifest: {0}")] - Manifest(#[from] serde_json::Error), - #[error("invalid persistence manifest: {0}")] - InvalidManifest(String), - #[error("invalid persistence artifact: {0}")] - InvalidArtifact(String), - #[error("AO ERI artifact is missing")] - MissingEri, - #[error("could not read NPY data: {0}")] - NpyRead(#[source] std::io::Error), - #[error("could not write NPY data: {0}")] - NpyWrite(#[source] std::io::Error), - #[error("AO ERI NPY must be one-dimensional, found shape {0:?}")] - InvalidEriShape(Vec), - #[error("matrix NPY must be two-dimensional, found shape {0:?}")] - InvalidMatrixShape(Vec), - #[error("NPY has shape {actual:?}, expected {expected:?}")] - InvalidNpyShape { - expected: Box<[u64]>, - actual: Box<[u64]>, - }, - #[error("AO ERI payload has {actual} values, expected {expected} for {basis_functions} basis functions")] - InvalidValueCount { - basis_functions: usize, - expected: usize, - actual: usize, - }, - #[error("NPY dtype is not a supported f64 representation: {0}")] - InvalidDtype(String), -} From a51a79d7fdcfaf5e825a6a1f5eadf38511f179a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:41:23 +0200 Subject: [PATCH 26/62] Use artifact-specific errors --- crates/rustiq-core/src/persistence/data/artifact.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/rustiq-core/src/persistence/data/artifact.rs b/crates/rustiq-core/src/persistence/data/artifact.rs index 91968fd..dd1df18 100644 --- a/crates/rustiq-core/src/persistence/data/artifact.rs +++ b/crates/rustiq-core/src/persistence/data/artifact.rs @@ -1,5 +1,5 @@ use super::rustiq_data::RustiQData; -use crate::persistence::PersistenceError; +use crate::persistence::ArtifactError; pub(crate) mod private { pub trait Sealed {} @@ -10,7 +10,7 @@ pub trait Artifact: private::Sealed { type Value; #[doc(hidden)] - fn get(data: &mut RustiQData) -> Result, PersistenceError>; + fn get(data: &mut RustiQData) -> Result, ArtifactError>; #[doc(hidden)] - fn set(data: &mut RustiQData, value: Self::Value) -> Result<(), PersistenceError>; + fn set(data: &mut RustiQData, value: Self::Value) -> Result<(), ArtifactError>; } From 963879a62ffbce86c6ef7263b5772195c8f26dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:41:25 +0200 Subject: [PATCH 27/62] Use artifact errors for AO ERI access --- crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs b/crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs index 7309102..3e93b56 100644 --- a/crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs +++ b/crates/rustiq-core/src/persistence/data/ao_eri_artifact.rs @@ -1,4 +1,4 @@ -use crate::{eri::CompactEri, persistence::PersistenceError}; +use crate::{eri::CompactEri, persistence::ArtifactError}; use super::{ artifact::{private, Artifact}, @@ -13,7 +13,7 @@ impl private::Sealed for AoEriArtifact {} impl Artifact for AoEriArtifact { type Value = CompactEri; - fn get(data: &mut RustiQData) -> Result, PersistenceError> { + fn get(data: &mut RustiQData) -> Result, ArtifactError> { if data.ao_eri.is_some() { return Ok(data.ao_eri.as_ref()); } @@ -23,7 +23,7 @@ impl Artifact for AoEriArtifact { data.read_eri().map(Some) } - fn set(data: &mut RustiQData, value: CompactEri) -> Result<(), PersistenceError> { + fn set(data: &mut RustiQData, value: CompactEri) -> Result<(), ArtifactError> { data.set_eri(value) } } From e4ba9d6a3bd5c0371120cd95709f3ce7ff09a7f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:41:49 +0200 Subject: [PATCH 28/62] Keep infrastructure errors internal --- crates/rustiq-core/src/persistence/error.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/rustiq-core/src/persistence/error.rs b/crates/rustiq-core/src/persistence/error.rs index 632d602..670a961 100644 --- a/crates/rustiq-core/src/persistence/error.rs +++ b/crates/rustiq-core/src/persistence/error.rs @@ -3,7 +3,7 @@ use std::io; use thiserror::Error; #[derive(Debug, Error)] -pub enum StorageError { +pub(crate) enum StorageError { #[error("storage I/O failed: {0}")] Io(#[from] io::Error), #[error("unsafe storage path: {0}")] @@ -15,7 +15,7 @@ pub enum StorageError { } #[derive(Debug, Error)] -pub enum NpyError { +pub(crate) enum NpyError { #[error("could not read NPY data: {0}")] Read(#[source] io::Error), #[error("could not write NPY data: {0}")] @@ -36,7 +36,7 @@ pub enum NpyError { } #[derive(Debug, Error)] -pub enum ManifestError { +pub(crate) enum ManifestError { #[error("manifest storage failed: {0}")] Storage(#[from] StorageError), #[error("could not decode or encode persistence manifest: {0}")] From 5028a0803b476e7c62d5bbce86232ccecf009363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:41:51 +0200 Subject: [PATCH 29/62] Keep persistence infrastructure private --- crates/rustiq-core/src/persistence/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/rustiq-core/src/persistence/mod.rs b/crates/rustiq-core/src/persistence/mod.rs index b299198..b9af756 100644 --- a/crates/rustiq-core/src/persistence/mod.rs +++ b/crates/rustiq-core/src/persistence/mod.rs @@ -18,7 +18,8 @@ pub use crate::eri::CompactEri; pub use checksum::{sha256, sha256_reader, verify_sha256, Sha256Digest, Sha256DigestParseError}; pub use data::{AoEriArtifact, Artifact, RustiQData}; pub use eri_cache::{EriCache, EriCacheEntry}; -pub use error::{ArtifactError, ManifestError, NpyError, StorageError}; +pub use error::ArtifactError; +pub(crate) use error::{ManifestError, NpyError, PersistenceReadError, PersistenceWriteError, StorageError}; #[allow(unused_imports)] pub(crate) use identity::{ao_eri_identity, ScientificIdentity, AO_ERI_COMPUTATION_VERSION}; pub(crate) use manifest::{ From b977d7b99bef6632aa91bb87c249f02b544423a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:42:24 +0200 Subject: [PATCH 30/62] Isolate storage errors --- crates/rustiq-core/src/persistence/storage.rs | 134 ++++++++---------- 1 file changed, 59 insertions(+), 75 deletions(-) diff --git a/crates/rustiq-core/src/persistence/storage.rs b/crates/rustiq-core/src/persistence/storage.rs index 771d1ab..2b152c1 100644 --- a/crates/rustiq-core/src/persistence/storage.rs +++ b/crates/rustiq-core/src/persistence/storage.rs @@ -8,15 +8,10 @@ use relative_path::RelativePath; use serde::{de::DeserializeOwned, Serialize}; use sha2::{Digest, Sha256}; -use super::{sha256_reader, PersistenceError, Sha256Digest}; +use super::{sha256_reader, ManifestError, Sha256Digest, StorageError}; -/// Physical storage selected for RustiQ persistence. -/// -/// The folder variant is available in V1. Additional variants, such as the -/// portable ZIP/ZIP64 container, can be added without changing `RustiQData`. -#[non_exhaustive] #[derive(Debug)] -pub enum Storage { +pub(crate) enum Storage { Folder(PathBuf), } @@ -56,13 +51,13 @@ impl Write for DigestWriter { fn write(&mut self, buffer: &[u8]) -> io::Result { let written = self.inner.write(buffer)?; self.hasher.update(&buffer[..written]); - let written = u64::try_from(written) - .map_err(|_| io::Error::other("artifact size exceeds u64"))?; + let written_u64 = + u64::try_from(written).map_err(|_| io::Error::other("artifact size exceeds u64"))?; self.size = self .size - .checked_add(written) + .checked_add(written_u64) .ok_or_else(|| io::Error::other("artifact size exceeds u64"))?; - Ok(usize::try_from(written).expect("written byte count originated as usize")) + Ok(written) } fn flush(&mut self) -> io::Result<()> { @@ -71,42 +66,47 @@ impl Write for DigestWriter { } impl Storage { - pub fn folder(root: impl Into) -> Self { + pub(crate) fn folder(root: impl Into) -> Self { Self::Folder(root.into()) } pub(crate) fn artifact_metadata( &mut self, path: &RelativePath, - ) -> Result { + ) -> Result { match self { Self::Folder(root) => FolderStorage { root: root.clone() }.artifact_metadata(path), } } - pub(crate) fn with_artifact( + pub(crate) fn with_artifact( &mut self, path: &RelativePath, read: F, - ) -> Result + ) -> Result where - F: FnOnce(&mut dyn Read) -> Result, + E: From, + F: FnOnce(&mut dyn Read) -> Result, { match self { - Self::Folder(root) => FolderStorage { root: root.clone() }.with_artifact(path, read), + Self::Folder(root) => FolderStorage { root: root.clone() } + .with_artifact(path, read) + .map_err(Into::into), } } - pub(crate) fn write_artifact( + pub(crate) fn write_artifact( &mut self, path: &RelativePath, write: F, - ) -> Result + ) -> Result where - F: FnOnce(&mut dyn Write) -> Result<(), PersistenceError>, + E: From, + F: FnOnce(&mut dyn Write) -> Result<(), E>, { match self { - Self::Folder(root) => FolderStorage { root: root.clone() }.write_artifact(path, write), + Self::Folder(root) => FolderStorage { root: root.clone() } + .write_artifact(path, write), } } @@ -114,18 +114,16 @@ impl Storage { &mut self, path: &RelativePath, max_size: u64, - ) -> Result { + ) -> Result { let size = match self { Self::Folder(root) => FolderStorage { root: root.clone() }.artifact_size(path)?, }; if size > max_size { - return Err(PersistenceError::InvalidManifest( - "manifest is larger than the supported limit".into(), - )); + return Err(ManifestError::TooLarge); } self.with_artifact(path, |reader| { - serde_json::from_reader(reader).map_err(PersistenceError::from) + serde_json::from_reader(reader).map_err(ManifestError::from) }) } @@ -133,16 +131,16 @@ impl Storage { &mut self, path: &RelativePath, value: &T, - ) -> Result<(), PersistenceError> { + ) -> Result<(), ManifestError> { self.write_artifact(path, |writer| { serde_json::to_writer_pretty(&mut *writer, value)?; - writer.write_all(b"\n")?; + writer.write_all(b"\n").map_err(StorageError::from)?; Ok(()) })?; Ok(()) } - pub(crate) fn finish(self) -> Result<(), PersistenceError> { + pub(crate) fn finish(self) -> Result<(), StorageError> { match self { Self::Folder(_) => Ok(()), } @@ -155,51 +153,54 @@ struct FolderStorage { } impl FolderStorage { - fn artifact_size(&self, path: &RelativePath) -> Result { + fn artifact_size(&self, path: &RelativePath) -> Result { Ok(self.open_artifact(path)?.metadata()?.len()) } - fn artifact_metadata(&self, path: &RelativePath) -> Result { + fn artifact_metadata(&self, path: &RelativePath) -> Result { let file = self.open_artifact(path)?; let size = file.metadata()?.len(); let digest = sha256_reader(BufReader::new(file))?; Ok(ArtifactMetadata { size, digest }) } - fn with_artifact( - &self, - path: &RelativePath, - read: F, - ) -> Result + fn with_artifact(&self, path: &RelativePath, read: F) -> Result where - F: FnOnce(&mut dyn Read) -> Result, + E: From, + F: FnOnce(&mut dyn Read) -> Result, { let mut reader = BufReader::new(self.open_artifact(path)?); - read(&mut reader) + read(&mut reader).map_err(|_| { + StorageError::Io(io::Error::other( + "artifact reader callback failed outside the storage layer", + )) + }) } - fn write_artifact( + fn write_artifact( &mut self, path: &RelativePath, write: F, - ) -> Result + ) -> Result where - F: FnOnce(&mut dyn Write) -> Result<(), PersistenceError>, + E: From, + F: FnOnce(&mut dyn Write) -> Result<(), E>, { - let file = self.create_artifact(path)?; + let file = self.create_artifact(path).map_err(E::from)?; let writer = BufWriter::new(file); let mut writer = DigestWriter::new(writer); write(&mut writer)?; - writer.flush()?; + writer.flush().map_err(StorageError::from).map_err(E::from)?; let (writer, metadata) = writer.finish(); let file = writer .into_inner() - .map_err(|error| PersistenceError::Io(error.into_error()))?; - file.sync_all()?; + .map_err(|error| StorageError::Io(error.into_error())) + .map_err(E::from)?; + file.sync_all().map_err(StorageError::from).map_err(E::from)?; Ok(metadata) } - fn open_artifact(&self, path: &RelativePath) -> Result { + fn open_artifact(&self, path: &RelativePath) -> Result { validate_path(path)?; let components: Vec<_> = path.as_str().split('/').collect(); let mut native = self.root.clone(); @@ -208,28 +209,24 @@ impl FolderStorage { native.push(component); let metadata = fs::symlink_metadata(&native)?; if metadata.file_type().is_symlink() { - return Err(PersistenceError::InvalidArtifact(format!( - "artifact contains a symbolic link: {path}" - ))); + return Err(StorageError::UnexpectedEntryType(path.to_string())); } let is_last = index + 1 == components.len(); if (!is_last && !metadata.is_dir()) || (is_last && !metadata.is_file()) { - return Err(PersistenceError::InvalidArtifact(format!( - "artifact path has an unexpected file type: {path}" - ))); + return Err(StorageError::UnexpectedEntryType(path.to_string())); } } Ok(File::open(native)?) } - fn create_artifact(&self, path: &RelativePath) -> Result { + fn create_artifact(&self, path: &RelativePath) -> Result { validate_path(path)?; let root_metadata = fs::symlink_metadata(&self.root)?; if !root_metadata.is_dir() || root_metadata.file_type().is_symlink() { - return Err(PersistenceError::InvalidArtifact( - "storage root is not a regular directory".into(), + return Err(StorageError::UnexpectedEntryType( + self.root.display().to_string(), )); } @@ -248,9 +245,7 @@ impl FolderStorage { match fs::symlink_metadata(&parent) { Ok(metadata) => { if !metadata.is_dir() || metadata.file_type().is_symlink() { - return Err(PersistenceError::InvalidArtifact(format!( - "artifact parent is not a regular directory: {path}" - ))); + return Err(StorageError::UnexpectedEntryType(path.to_string())); } } Err(error) if error.kind() == io::ErrorKind::NotFound => { @@ -260,21 +255,14 @@ impl FolderStorage { } } - Err(PersistenceError::InvalidArtifact( - "artifact path is empty".into(), - )) + Err(StorageError::InvalidPath(path.to_string())) } } -pub(crate) fn portable_path_key(path: &RelativePath) -> Result { - validate_path(path)?; - Ok(path.as_str().to_lowercase()) -} - -pub(crate) fn validate_path(path: &RelativePath) -> Result<(), PersistenceError> { +pub(crate) fn validate_path(path: &RelativePath) -> Result<(), StorageError> { let value = path.as_str(); if value.is_empty() || value.starts_with('/') || value.contains('\\') { - return Err(unsafe_path(path)); + return Err(StorageError::InvalidPath(path.to_string())); } for component in value.split('/') { @@ -286,7 +274,7 @@ pub(crate) fn validate_path(path: &RelativePath) -> Result<(), PersistenceError> }) || is_windows_reserved_name(component) { - return Err(unsafe_path(path)); + return Err(StorageError::InvalidPath(path.to_string())); } } @@ -308,10 +296,6 @@ fn is_windows_reserved_name(component: &str) -> bool { && matches!(stem.as_bytes()[3], b'1'..=b'9')) } -fn unsafe_path(path: &RelativePath) -> PersistenceError { - PersistenceError::InvalidArtifact(format!("unsafe artifact path: {path}")) -} - #[cfg(test)] mod tests { use super::*; @@ -323,7 +307,7 @@ mod tests { let artifact_path = RelativePath::new("arrays/test.bin"); let metadata = storage - .write_artifact(artifact_path, |writer| { + .write_artifact::(artifact_path, |writer| { writer.write_all(b"payload")?; Ok(()) }) @@ -332,7 +316,7 @@ mod tests { assert_eq!(metadata.digest, super::super::sha256(b"payload")); let payload = storage - .with_artifact(artifact_path, |reader| { + .with_artifact::<_, StorageError, _>(artifact_path, |reader| { let mut bytes = Vec::new(); reader.read_to_end(&mut bytes)?; Ok(bytes) From 0682875491f36e65f26462bd8e48745ed996560d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:42:35 +0200 Subject: [PATCH 31/62] Preserve storage callback errors --- crates/rustiq-core/src/persistence/storage.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/crates/rustiq-core/src/persistence/storage.rs b/crates/rustiq-core/src/persistence/storage.rs index 2b152c1..026bb4d 100644 --- a/crates/rustiq-core/src/persistence/storage.rs +++ b/crates/rustiq-core/src/persistence/storage.rs @@ -164,17 +164,13 @@ impl FolderStorage { Ok(ArtifactMetadata { size, digest }) } - fn with_artifact(&self, path: &RelativePath, read: F) -> Result + fn with_artifact(&self, path: &RelativePath, read: F) -> Result where E: From, F: FnOnce(&mut dyn Read) -> Result, { - let mut reader = BufReader::new(self.open_artifact(path)?); - read(&mut reader).map_err(|_| { - StorageError::Io(io::Error::other( - "artifact reader callback failed outside the storage layer", - )) - }) + let mut reader = BufReader::new(self.open_artifact(path).map_err(E::from)?); + read(&mut reader) } fn write_artifact( From 3d8e3a6475cf609b96093a5876492278eca55dcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:43:02 +0200 Subject: [PATCH 32/62] Refine NPY validation errors --- crates/rustiq-core/src/persistence/error.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/rustiq-core/src/persistence/error.rs b/crates/rustiq-core/src/persistence/error.rs index 670a961..5787ca9 100644 --- a/crates/rustiq-core/src/persistence/error.rs +++ b/crates/rustiq-core/src/persistence/error.rs @@ -31,6 +31,14 @@ pub(crate) enum NpyError { }, #[error("NPY dtype is not a supported f64 representation: {0}")] InvalidDtype(String), + #[error( + "AO ERI NPY has {actual} values, expected {expected} for {basis_functions} basis functions" + )] + InvalidValueCount { + basis_functions: usize, + expected: usize, + actual: usize, + }, #[error("NPY dimensions exceed supported limits")] DimensionOverflow, } From 6824c3216337eb1c2744a8f8632bdafdc794bfb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:43:05 +0200 Subject: [PATCH 33/62] Use NPY-specific errors --- crates/rustiq-core/src/persistence/npy.rs | 102 +++++++++++----------- 1 file changed, 50 insertions(+), 52 deletions(-) diff --git a/crates/rustiq-core/src/persistence/npy.rs b/crates/rustiq-core/src/persistence/npy.rs index 6aaca70..f7940fa 100644 --- a/crates/rustiq-core/src/persistence/npy.rs +++ b/crates/rustiq-core/src/persistence/npy.rs @@ -5,28 +5,28 @@ use npyz::{DType, NpyFile, TypeChar, WriterBuilder}; use crate::eri::CompactEri; -use super::PersistenceError; +use super::NpyError; /// Conversion between scientific values and an NPY byte stream. trait NpyConvert: Sized { type Shape: Copy; type NpyShape: AsRef<[u64]>; - fn npy_shape(shape: Self::Shape) -> Result; + fn npy_shape(shape: Self::Shape) -> Result; fn decode_with_shape( npy: NpyFile, shape: Self::Shape, - ) -> Result; + ) -> Result; /// Checks a parsed header before decoding any array values. fn try_from_npy_with_shape( npy: NpyFile, shape: Self::Shape, - ) -> Result { + ) -> Result { let expected = Self::npy_shape(shape)?; if npy.shape() != expected.as_ref() { - return Err(PersistenceError::InvalidNpyShape { + return Err(NpyError::InvalidShape { expected: expected.as_ref().to_vec().into_boxed_slice(), actual: npy.shape().to_vec().into_boxed_slice(), }); @@ -38,50 +38,48 @@ trait NpyConvert: Sized { fn try_read_with_shape( reader: impl Read, shape: Self::Shape, - ) -> Result { + ) -> Result { Self::try_from_npy_with_shape( - NpyFile::new(reader).map_err(PersistenceError::NpyRead)?, + NpyFile::new(reader).map_err(NpyError::Read)?, shape, ) } - fn write_npy(&self, writer: impl Write) -> Result<(), PersistenceError>; + fn write_npy(&self, writer: impl Write) -> Result<(), NpyError>; } impl NpyConvert for CompactEri { type Shape = usize; type NpyShape = [u64; 1]; - fn npy_shape(basis_functions: usize) -> Result { + fn npy_shape(basis_functions: usize) -> Result { let length = CompactEri::checked_storage_len(basis_functions).ok_or_else(|| { - PersistenceError::InvalidArtifact( - "basis-function count overflows compact ERI storage".into(), - ) + NpyError::DimensionOverflow })?; Ok([u64::try_from(length).map_err(|_| { - PersistenceError::InvalidArtifact("compact ERI length exceeds NPY limits".into()) + NpyError::DimensionOverflow })?]) } fn decode_with_shape( npy: NpyFile, basis_functions: usize, - ) -> Result { + ) -> Result { decode_compact_eri(npy, basis_functions) } - fn write_npy(&self, writer: impl Write) -> Result<(), PersistenceError> { + fn write_npy(&self, writer: impl Write) -> Result<(), NpyError> { let shape = [self.len() as u64]; let mut writer = npyz::WriteOptions::new() .default_dtype() .shape(&shape) .writer(writer) .begin_nd() - .map_err(PersistenceError::NpyWrite)?; + .map_err(NpyError::Write)?; writer .extend(self.ordered_values().iter().copied()) - .map_err(PersistenceError::NpyWrite)?; - writer.finish().map_err(PersistenceError::NpyWrite) + .map_err(NpyError::Write)?; + writer.finish().map_err(NpyError::Write) } } @@ -89,12 +87,12 @@ impl NpyConvert for DMatrix { type Shape = (usize, usize); type NpyShape = [u64; 2]; - fn npy_shape((rows, columns): (usize, usize)) -> Result { + fn npy_shape((rows, columns): (usize, usize)) -> Result { let rows = u64::try_from(rows).map_err(|_| { - PersistenceError::InvalidArtifact("matrix rows exceed NPY limits".into()) + NpyError::DimensionOverflow })?; let columns = u64::try_from(columns).map_err(|_| { - PersistenceError::InvalidArtifact("matrix columns exceed NPY limits".into()) + NpyError::DimensionOverflow })?; Ok([rows, columns]) } @@ -102,11 +100,11 @@ impl NpyConvert for DMatrix { fn decode_with_shape( npy: NpyFile, _shape: (usize, usize), - ) -> Result { + ) -> Result { decode_dmatrix(npy) } - fn write_npy(&self, writer: impl Write) -> Result<(), PersistenceError> { + fn write_npy(&self, writer: impl Write) -> Result<(), NpyError> { let shape = [self.nrows() as u64, self.ncols() as u64]; let mut writer = npyz::WriteOptions::new() .default_dtype() @@ -114,46 +112,46 @@ impl NpyConvert for DMatrix { .shape(&shape) .writer(writer) .begin_nd() - .map_err(PersistenceError::NpyWrite)?; + .map_err(NpyError::Write)?; writer .extend(self.as_slice().iter().copied()) - .map_err(PersistenceError::NpyWrite)?; - writer.finish().map_err(PersistenceError::NpyWrite) + .map_err(NpyError::Write)?; + writer.finish().map_err(NpyError::Write) } } pub(crate) fn write_compact_eri( writer: impl Write, eri: &CompactEri, -) -> Result<(), PersistenceError> { +) -> Result<(), NpyError> { eri.write_npy(writer) } pub(crate) fn read_compact_eri( reader: impl Read, basis_functions: usize, -) -> Result { +) -> Result { CompactEri::try_read_with_shape(reader, basis_functions) } fn decode_compact_eri( npy: NpyFile, basis_functions: usize, -) -> Result { +) -> Result { if npy.shape().len() != 1 { - return Err(PersistenceError::InvalidEriShape(npy.shape().to_vec())); + return Err(NpyError::InvalidEriShape(npy.shape().to_vec().into_boxed_slice())); } let actual = usize::try_from(npy.shape()[0]) - .map_err(|_| PersistenceError::InvalidEriShape(npy.shape().to_vec()))?; + .map_err(|_| NpyError::InvalidEriShape(npy.shape().to_vec().into_boxed_slice()))?; let expected = CompactEri::checked_storage_len(basis_functions).ok_or( - PersistenceError::InvalidValueCount { + NpyError::InvalidValueCount { basis_functions, expected: 0, actual, }, )?; if actual != expected { - return Err(PersistenceError::InvalidValueCount { + return Err(NpyError::InvalidValueCount { basis_functions, expected, actual, @@ -161,14 +159,14 @@ fn decode_compact_eri( } let values = npy.into_vec::().map_err(|error| { if error.kind() == std::io::ErrorKind::InvalidData { - PersistenceError::InvalidDtype(error.to_string()) + NpyError::InvalidDtype(error.to_string()) } else { - PersistenceError::NpyRead(error) + NpyError::Read(error) } })?; CompactEri::from_ordered_values(basis_functions, values).map_err(|error| match error { crate::eri::CompactEriBuildError::InvalidLength { expected, actual } => { - PersistenceError::InvalidValueCount { + NpyError::InvalidValueCount { basis_functions, expected, actual, @@ -216,26 +214,26 @@ pub(crate) fn validate_compact_eri_header( .is_some_and(|end| end == file_size) } -pub(crate) fn read_dmatrix(reader: impl Read) -> Result, PersistenceError> { - decode_dmatrix(NpyFile::new(reader).map_err(PersistenceError::NpyRead)?) +pub(crate) fn read_dmatrix(reader: impl Read) -> Result, NpyError> { + decode_dmatrix(NpyFile::new(reader).map_err(NpyError::Read)?) } -fn decode_dmatrix(npy: NpyFile) -> Result, PersistenceError> { +fn decode_dmatrix(npy: NpyFile) -> Result, NpyError> { if npy.shape().len() != 2 { - return Err(PersistenceError::InvalidMatrixShape(npy.shape().to_vec())); + return Err(NpyError::InvalidMatrixShape(npy.shape().to_vec().into_boxed_slice())); } let rows = usize::try_from(npy.shape()[0]) - .map_err(|_| PersistenceError::InvalidMatrixShape(npy.shape().to_vec()))?; + .map_err(|_| NpyError::InvalidMatrixShape(npy.shape().to_vec().into_boxed_slice()))?; let columns = usize::try_from(npy.shape()[1]) - .map_err(|_| PersistenceError::InvalidMatrixShape(npy.shape().to_vec()))?; + .map_err(|_| NpyError::InvalidMatrixShape(npy.shape().to_vec().into_boxed_slice()))?; rows.checked_mul(columns) - .ok_or_else(|| PersistenceError::InvalidMatrixShape(npy.shape().to_vec()))?; + .ok_or_else(|| NpyError::InvalidMatrixShape(npy.shape().to_vec().into_boxed_slice()))?; let order = npy.order(); let values = npy.into_vec::().map_err(|error| { if error.kind() == std::io::ErrorKind::InvalidData { - PersistenceError::InvalidDtype(error.to_string()) + NpyError::InvalidDtype(error.to_string()) } else { - PersistenceError::NpyRead(error) + NpyError::Read(error) } })?; Ok(matrix_from_npy_values(rows, columns, order, values)) @@ -278,7 +276,7 @@ mod tests { ); assert!(matches!( CompactEri::try_read_with_shape(bytes.as_slice(), basis_functions - 1), - Err(PersistenceError::InvalidNpyShape { .. }) + Err(NpyError::InvalidShape { .. }) )); } @@ -332,16 +330,16 @@ mod tests { ); assert!(matches!( DMatrix::::try_read_with_shape(bytes.as_slice(), (3, 2)), - Err(PersistenceError::InvalidNpyShape { .. }) + Err(NpyError::InvalidShape { .. }) )); let truncated = &bytes[..bytes.len() - 8]; assert!(matches!( DMatrix::::try_read_with_shape(truncated, (3, 2)), - Err(PersistenceError::InvalidNpyShape { .. }) + Err(NpyError::InvalidShape { .. }) )); assert!(matches!( DMatrix::::try_read_with_shape(truncated, (2, 3)), - Err(PersistenceError::NpyRead(_)) + Err(NpyError::Read(_)) )); let mut c_bytes = Vec::new(); @@ -396,7 +394,7 @@ mod tests { writer.finish().unwrap(); assert!(matches!( read_compact_eri(bytes.as_slice(), 2), - Err(PersistenceError::InvalidNpyShape { .. }) + Err(NpyError::InvalidShape { .. }) )); } @@ -414,7 +412,7 @@ mod tests { writer.finish().unwrap(); assert!(matches!( read_compact_eri(bytes.as_slice(), 1), - Err(PersistenceError::InvalidDtype(_)) + Err(NpyError::InvalidDtype(_)) )); } @@ -426,7 +424,7 @@ mod tests { bytes.truncate(bytes.len() - 1); assert!(matches!( read_compact_eri(bytes.as_slice(), 3), - Err(PersistenceError::NpyRead(_)) + Err(NpyError::Read(_)) )); } From 263a5b6fd1453de7038df36ceb39da8e25485a2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:44:09 +0200 Subject: [PATCH 34/62] Refine artifact errors --- crates/rustiq-core/src/persistence/error.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/rustiq-core/src/persistence/error.rs b/crates/rustiq-core/src/persistence/error.rs index 5787ca9..efa665a 100644 --- a/crates/rustiq-core/src/persistence/error.rs +++ b/crates/rustiq-core/src/persistence/error.rs @@ -63,6 +63,8 @@ pub enum ArtifactError { Missing, #[error("unsupported artifact representation: {0}")] UnsupportedRepresentation(String), + #[error("invalid artifact metadata: {0}")] + InvalidMetadata(String), #[error("artifact integrity check failed: {0}")] IntegrityMismatch(String), #[error("invalid artifact path: {0}")] From 7d73b3dfb4a536a000ff17add52f3d35561cbd4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:44:11 +0200 Subject: [PATCH 35/62] Hide storage behind RustiQData --- .../src/persistence/data/rustiq_data.rs | 204 ++++++++++-------- 1 file changed, 119 insertions(+), 85 deletions(-) diff --git a/crates/rustiq-core/src/persistence/data/rustiq_data.rs b/crates/rustiq-core/src/persistence/data/rustiq_data.rs index c68797d..69c88eb 100644 --- a/crates/rustiq-core/src/persistence/data/rustiq_data.rs +++ b/crates/rustiq-core/src/persistence/data/rustiq_data.rs @@ -9,9 +9,10 @@ use crate::{ use super::super::{ ao_eri_identity, read_compact_eri, validate_storage_path, write_compact_eri, AoEriAttributes, - ArtifactAttributes, ArtifactManifest, Manifest, PersistenceError, Producer, - ScientificIdentityManifest, Storage, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, - COMPACT_ERI_REPRESENTATION, FORMAT_NAME, FORMAT_VERSION, MANIFEST_PATH, + ArtifactAttributes, ArtifactError, ArtifactManifest, Manifest, ManifestError, + PersistenceReadError, PersistenceWriteError, Producer, ScientificIdentityManifest, Storage, + StorageError, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, + FORMAT_VERSION, MANIFEST_PATH, }; use super::artifact::Artifact; @@ -19,10 +20,7 @@ pub(crate) const AO_ERI_ARTIFACT: &str = "ao_eri"; pub(crate) const CACHE_KIND: &str = "integral-cache"; const MAX_MANIFEST_BYTES: u64 = 1024 * 1024; -/// Known scientific artifacts and their manifest, with NPY values loaded on demand. -/// -/// Add a typed field and accessors here when RustiQ gains a new scientific artifact. -/// Unknown manifest entries remain opaque and are copied when writing a new entry. +/// Known scientific artifacts and their manifest, with values loaded on demand. #[derive(Debug)] pub struct RustiQData { pub(super) manifest: Manifest, @@ -32,17 +30,17 @@ pub struct RustiQData { } impl RustiQData { - /// Gets a known artifact, loading and caching its value on first access. - pub fn get(&mut self) -> Result, PersistenceError> { + /// Gets a known scientific artifact, loading and caching it on first access. + pub fn get(&mut self) -> Result, ArtifactError> { A::get(self) } - /// Sets a known artifact using its statically selected value type. - pub fn set(&mut self, value: A::Value) -> Result<(), PersistenceError> { + /// Sets a known scientific artifact using its statically selected value type. + pub fn set(&mut self, value: A::Value) -> Result<(), ArtifactError> { A::set(self, value) } - /// Starts a new AO ERI entry with the same scientific identity as the cache. + /// Starts a new AO ERI data set with the current scientific identity. pub fn new(geometry: &Geometry, basis: &Basis, threshold: Option) -> Self { let identity = ao_eri_identity(geometry, basis, threshold); Self::new_with_identity(identity, basis.nbasis()) @@ -73,37 +71,33 @@ impl RustiQData { } } - /// Reads only the bounded manifest. Scientific artifacts are opened on demand. - pub fn read(mut source: Storage) -> Result { + pub(crate) fn read_from(mut source: Storage) -> Result { let manifest: Manifest = source.read_json(RelativePath::new(MANIFEST_PATH), MAX_MANIFEST_BYTES)?; if manifest.format != FORMAT_NAME || manifest.format_version != FORMAT_VERSION { - return Err(PersistenceError::InvalidManifest( - "unsupported format or version".into(), - )); + return Err(ManifestError::UnsupportedFormat.into()); } - let mut paths: HashSet = HashSet::new(); + + let mut paths = HashSet::new(); for artifact in manifest.artifacts.values() { validate_artifact_path(&artifact.path)?; let key = artifact.path.as_str().to_lowercase(); if paths.iter().any(|other| paths_conflict(other, &key)) { - return Err(PersistenceError::InvalidManifest(format!( - "artifact path conflicts with another artifact: {}", - artifact.path - ))); + return Err( + ManifestError::ConflictingArtifactPath(artifact.path.to_string()).into(), + ); } paths.insert(key); } + let basis_functions = manifest .artifacts .get(AO_ERI_ARTIFACT) - .and_then(|artifact| { - if let ArtifactAttributes::AoEri(attributes) = &artifact.attributes { - Some(attributes.basis_functions) - } else { - None - } + .and_then(|artifact| match &artifact.attributes { + ArtifactAttributes::AoEri(attributes) => Some(attributes.basis_functions), + ArtifactAttributes::Unknown(_) => None, }); + Ok(Self { manifest, source: Some(source), @@ -117,23 +111,23 @@ impl RustiQData { } /// Replaces the AO ERI artifact after checking its compact length. - pub fn set_eri(&mut self, eri: CompactEri) -> Result<(), PersistenceError> { - let basis_functions = self.basis_functions.ok_or(PersistenceError::MissingEri)?; + pub fn set_eri(&mut self, eri: CompactEri) -> Result<(), ArtifactError> { + let basis_functions = self.basis_functions.ok_or(ArtifactError::Missing)?; validate_eri_len(&eri, basis_functions)?; self.ao_eri = Some(eri); Ok(()) } - /// Validates and decodes the AO ERI NPY on first access, then reuses the object. - pub fn read_eri(&mut self) -> Result<&CompactEri, PersistenceError> { + /// Validates and decodes the AO ERI on first access, then reuses the object. + pub fn read_eri(&mut self) -> Result<&CompactEri, ArtifactError> { if self.ao_eri.is_none() { let artifact = self .manifest .artifacts .get(AO_ERI_ARTIFACT) - .ok_or(PersistenceError::MissingEri)?; + .ok_or(ArtifactError::Missing)?; let ArtifactAttributes::AoEri(attributes) = &artifact.attributes else { - return Err(PersistenceError::InvalidArtifact( + return Err(ArtifactError::InvalidMetadata( "AO ERI attributes are missing".into(), )); }; @@ -141,21 +135,25 @@ impl RustiQData { || artifact.representation != COMPACT_ERI_REPRESENTATION || attributes.computation_version != AO_ERI_COMPUTATION_VERSION { - return Err(PersistenceError::InvalidArtifact( - "unsupported AO ERI representation".into(), + return Err(ArtifactError::UnsupportedRepresentation( + artifact.representation.clone(), )); } - let source = self.source.as_mut().ok_or(PersistenceError::MissingEri)?; + + let source = self.source.as_mut().ok_or(ArtifactError::Missing)?; let metadata = source.artifact_metadata(&artifact.path)?; if metadata.size != artifact.size || metadata.digest != artifact.digest { - return Err(PersistenceError::InvalidArtifact( - "AO ERI digest or size mismatch".into(), + return Err(ArtifactError::IntegrityMismatch( + artifact.path.to_string(), )); } - self.ao_eri = Some(source.with_artifact(&artifact.path, |reader| { - read_compact_eri(reader, attributes.basis_functions) - })?); + + self.ao_eri = Some(source.with_artifact::<_, ArtifactError, _>( + &artifact.path, + |reader| read_compact_eri(reader, attributes.basis_functions).map_err(Into::into), + )?); } + Ok(self .ao_eri .as_ref() @@ -166,8 +164,10 @@ impl RustiQData { self.ao_eri.take() } - /// Writes to the selected storage, copying unloaded artifacts without decoding them. - pub fn write(&mut self, destination: Storage) -> Result<(), PersistenceError> { + pub(crate) fn write_to( + &mut self, + destination: Storage, + ) -> Result<(), PersistenceWriteError> { let eri = self.ao_eri.take(); let result = self.write_inner(destination, eri.as_ref()); self.ao_eri = eri; @@ -178,7 +178,7 @@ impl RustiQData { &mut self, destination: Storage, eri: Option<&CompactEri>, - ) -> Result<(), PersistenceError> { + ) -> Result<(), PersistenceWriteError> { self.write_inner(destination, eri) } @@ -186,37 +186,50 @@ impl RustiQData { &mut self, mut destination: Storage, eri: Option<&CompactEri>, - ) -> Result<(), PersistenceError> { + ) -> Result<(), PersistenceWriteError> { let mut manifest = self.manifest.clone(); for (name, artifact) in &self.manifest.artifacts { if name == AO_ERI_ARTIFACT && eri.is_some() { continue; } + let source = self.source.as_mut().ok_or_else(|| { - PersistenceError::InvalidArtifact(format!("artifact {name} has no source file")) + ArtifactError::InvalidMetadata(format!("artifact {name} has no source")) })?; validate_artifact_path(&artifact.path)?; - let metadata = source.with_artifact(&artifact.path, |input| { - destination.write_artifact(&artifact.path, |output| { - std::io::copy(input, output)?; - Ok(()) - }) - })?; + + let metadata = source.with_artifact::<_, PersistenceWriteError, _>( + &artifact.path, + |input| { + destination.write_artifact::( + &artifact.path, + |output| { + std::io::copy(input, output) + .map_err(StorageError::from) + .map_err(PersistenceWriteError::from)?; + Ok(()) + }, + ) + }, + )?; + if metadata.size != artifact.size || metadata.digest != artifact.digest { - return Err(PersistenceError::InvalidArtifact(format!( - "artifact digest or size mismatch: {}", - artifact.path - ))); + return Err( + ArtifactError::IntegrityMismatch(artifact.path.to_string()).into(), + ); } } if let Some(eri) = eri { - let basis_functions = self.basis_functions.ok_or(PersistenceError::MissingEri)?; + let basis_functions = self.basis_functions.ok_or(ArtifactError::Missing)?; validate_eri_len(eri, basis_functions)?; let path = RelativePath::new(AO_ERI_PATH); - let metadata = - destination.write_artifact(path, |writer| write_compact_eri(writer, eri))?; + let metadata = destination.write_artifact::( + path, + |writer| write_compact_eri(writer, eri).map_err(Into::into), + )?; + manifest.artifacts.insert( AO_ERI_ARTIFACT.to_owned(), ArtifactManifest { @@ -231,22 +244,25 @@ impl RustiQData { }, ); } + if manifest.kind == CACHE_KIND && !manifest.artifacts.contains_key(AO_ERI_ARTIFACT) { - return Err(PersistenceError::MissingEri); + return Err(ArtifactError::Missing.into()); } + destination.write_json(RelativePath::new(MANIFEST_PATH), &manifest)?; - destination.finish() + destination.finish()?; + Ok(()) } } -fn validate_eri_len(eri: &CompactEri, basis_functions: usize) -> Result<(), PersistenceError> { +fn validate_eri_len(eri: &CompactEri, basis_functions: usize) -> Result<(), ArtifactError> { let expected = CompactEri::checked_storage_len(basis_functions).ok_or_else(|| { - PersistenceError::InvalidArtifact( + ArtifactError::InvalidMetadata( "basis-function count overflows compact ERI storage".into(), ) })?; if eri.len() != expected { - return Err(PersistenceError::InvalidValueCount { + return Err(ArtifactError::InvalidValueCount { basis_functions, expected, actual: eri.len(), @@ -255,12 +271,10 @@ fn validate_eri_len(eri: &CompactEri, basis_functions: usize) -> Result<(), Pers Ok(()) } -fn validate_artifact_path(path: &RelativePath) -> Result<(), PersistenceError> { - validate_storage_path(path)?; +fn validate_artifact_path(path: &RelativePath) -> Result<(), ArtifactError> { + validate_storage_path(path).map_err(|_| ArtifactError::InvalidPath(path.to_string()))?; if path.as_str().split('/').next() == Some(MANIFEST_PATH) { - return Err(PersistenceError::InvalidArtifact(format!( - "unsafe artifact path: {path}" - ))); + return Err(ArtifactError::InvalidPath(path.to_string())); } Ok(()) } @@ -277,8 +291,12 @@ fn paths_conflict(left: &str, right: &str) -> bool { #[cfg(test)] mod tests { + use std::fs; + use super::*; - use crate::persistence::{sha256, Sha256Digest, SCIENTIFIC_IDENTITY_VERSION}; + use crate::persistence::{ + data::AoEriArtifact, sha256, Sha256Digest, SCIENTIFIC_IDENTITY_VERSION, + }; fn new_data() -> RustiQData { RustiQData::new_with_identity( @@ -294,11 +312,13 @@ mod tests { fn eri_is_loaded_only_on_request_and_then_cached_as_an_object() { let root = tempfile::tempdir().unwrap(); let entry = root.path().join("entry"); + fs::create_dir(&entry).unwrap(); let mut data = new_data(); assert!(data.get::().unwrap().is_none()); data.set::(CompactEri::Zeroed(2)).unwrap(); - data.write(Storage::folder(&entry)).unwrap(); - let mut restored = RustiQData::read(Storage::folder(&entry)).unwrap(); + data.write_to(Storage::folder(&entry)).unwrap(); + + let mut restored = RustiQData::read_from(Storage::folder(&entry)).unwrap(); assert!(restored.ao_eri.is_none()); let first = restored.get::().unwrap().unwrap() as *const CompactEri; fs::remove_file(entry.join(AO_ERI_PATH)).unwrap(); @@ -312,13 +332,18 @@ mod tests { let root = tempfile::tempdir().unwrap(); let first = root.path().join("first"); let second = root.path().join("second"); + fs::create_dir(&first).unwrap(); + fs::create_dir(&second).unwrap(); + let mut data = new_data(); data.set_eri(CompactEri::Zeroed(2)).unwrap(); - data.write(Storage::folder(&first)).unwrap(); + data.write_to(Storage::folder(&first)).unwrap(); + let unknown = b"opaque future data"; let unknown_path = first.join("arrays/post-hf/future.npy"); fs::create_dir_all(unknown_path.parent().unwrap()).unwrap(); fs::write(&unknown_path, unknown).unwrap(); + let manifest_path = first.join(MANIFEST_PATH); let mut manifest: Manifest = serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap(); @@ -333,15 +358,17 @@ mod tests { }, ); fs::write(&manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap(); - let mut restored = RustiQData::read(Storage::folder(&first)).unwrap(); - assert!(restored.ao_eri.is_none()); - restored.write(Storage::folder(&second)).unwrap(); + + let mut restored = RustiQData::read_from(Storage::folder(&first)).unwrap(); + restored.write_to(Storage::folder(&second)).unwrap(); assert_eq!( fs::read(second.join("arrays/post-hf/future.npy")).unwrap(), unknown ); assert_eq!( - RustiQData::read(Storage::folder(&second)).unwrap().manifest, + RustiQData::read_from(Storage::folder(&second)) + .unwrap() + .manifest, manifest ); } @@ -374,16 +401,21 @@ mod tests { let root = tempfile::tempdir().unwrap(); let first = root.path().join("first"); let second = root.path().join("second"); + fs::create_dir(&first).unwrap(); + fs::create_dir(&second).unwrap(); + let mut data = new_data(); data.set_eri(CompactEri::Zeroed(2)).unwrap(); - data.write(Storage::folder(&first)).unwrap(); + data.write_to(Storage::folder(&first)).unwrap(); fs::write(first.join(AO_ERI_PATH), b"bad").unwrap(); - let mut restored = RustiQData::read(Storage::folder(&first)).unwrap(); + + let mut restored = RustiQData::read_from(Storage::folder(&first)).unwrap(); assert!(matches!( - restored.write(Storage::folder(&second)), - Err(PersistenceError::InvalidArtifact(_)) + restored.write_to(Storage::folder(&second)), + Err(PersistenceWriteError::Artifact( + ArtifactError::IntegrityMismatch(_) + )) )); - assert!(!second.exists()); let manifest_path = first.join(MANIFEST_PATH); let mut manifest: Manifest = @@ -400,8 +432,10 @@ mod tests { ); fs::write(&manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap(); assert!(matches!( - RustiQData::read(Storage::folder(&first)), - Err(PersistenceError::InvalidManifest(_)) + RustiQData::read_from(Storage::folder(&first)), + Err(PersistenceReadError::Manifest( + ManifestError::ConflictingArtifactPath(_) + )) )); } } From b8f1664827543e7a5154e2df7fd206bff219dfbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:44:21 +0200 Subject: [PATCH 36/62] Keep cache storage internal --- crates/rustiq-core/src/persistence/eri_cache.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rustiq-core/src/persistence/eri_cache.rs b/crates/rustiq-core/src/persistence/eri_cache.rs index 653ba53..296078b 100644 --- a/crates/rustiq-core/src/persistence/eri_cache.rs +++ b/crates/rustiq-core/src/persistence/eri_cache.rs @@ -309,7 +309,7 @@ impl EriCache { if !metadata.is_dir() || metadata.file_type().is_symlink() { return None; } - let mut data = RustiQData::read(Storage::folder(&entry)).ok()?; + let mut data = RustiQData::read_from(Storage::folder(&entry)).ok()?; let manifest = data.manifest(); let artifact = manifest.artifacts.get(AO_ERI_ARTIFACT)?; let attributes = ao_eri_attributes(artifact)?; @@ -368,7 +368,7 @@ impl EriCache { } fn read_manifest(entry: &Path) -> Option { - RustiQData::read(Storage::folder(entry)) + RustiQData::read_from(Storage::folder(entry)) .ok() .map(|data| data.manifest().clone()) } From a92f67c803884034241816aa37db133e26337588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:44:57 +0200 Subject: [PATCH 37/62] Hide storage errors from artifact API --- crates/rustiq-core/src/persistence/error.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/rustiq-core/src/persistence/error.rs b/crates/rustiq-core/src/persistence/error.rs index efa665a..37d085a 100644 --- a/crates/rustiq-core/src/persistence/error.rs +++ b/crates/rustiq-core/src/persistence/error.rs @@ -15,7 +15,7 @@ pub(crate) enum StorageError { } #[derive(Debug, Error)] -pub(crate) enum NpyError { +pub enum NpyError { #[error("could not read NPY data: {0}")] Read(#[source] io::Error), #[error("could not write NPY data: {0}")] @@ -77,8 +77,8 @@ pub enum ArtifactError { expected: usize, actual: usize, }, - #[error("artifact storage failed: {0}")] - Storage(#[from] StorageError), + #[error("artifact access failed: {0}")] + AccessFailed(String), #[error("artifact NPY data is invalid: {0}")] Npy(#[from] NpyError), } @@ -102,3 +102,12 @@ pub(crate) enum PersistenceWriteError { #[error(transparent)] Npy(#[from] NpyError), } + +impl From for ArtifactError { + fn from(error: StorageError) -> Self { + match error { + StorageError::InvalidPath(path) => Self::InvalidPath(path), + other => Self::AccessFailed(other.to_string()), + } + } +} From e84ecd3641cade1bde39429b0b1a03fdd3de50aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:45:11 +0200 Subject: [PATCH 38/62] Expose NPY error semantics --- crates/rustiq-core/src/persistence/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rustiq-core/src/persistence/mod.rs b/crates/rustiq-core/src/persistence/mod.rs index b9af756..95da712 100644 --- a/crates/rustiq-core/src/persistence/mod.rs +++ b/crates/rustiq-core/src/persistence/mod.rs @@ -18,8 +18,8 @@ pub use crate::eri::CompactEri; pub use checksum::{sha256, sha256_reader, verify_sha256, Sha256Digest, Sha256DigestParseError}; pub use data::{AoEriArtifact, Artifact, RustiQData}; pub use eri_cache::{EriCache, EriCacheEntry}; -pub use error::ArtifactError; -pub(crate) use error::{ManifestError, NpyError, PersistenceReadError, PersistenceWriteError, StorageError}; +pub use error::{ArtifactError, NpyError}; +pub(crate) use error::{ManifestError, PersistenceReadError, PersistenceWriteError, StorageError}; #[allow(unused_imports)] pub(crate) use identity::{ao_eri_identity, ScientificIdentity, AO_ERI_COMPUTATION_VERSION}; pub(crate) use manifest::{ From 74dd07f1eef8b4f27e92f9e6ab6dbdc94bb55003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:45:14 +0200 Subject: [PATCH 39/62] Simplify storage callback propagation --- crates/rustiq-core/src/persistence/storage.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/rustiq-core/src/persistence/storage.rs b/crates/rustiq-core/src/persistence/storage.rs index 026bb4d..11fe6e0 100644 --- a/crates/rustiq-core/src/persistence/storage.rs +++ b/crates/rustiq-core/src/persistence/storage.rs @@ -90,8 +90,7 @@ impl Storage { { match self { Self::Folder(root) => FolderStorage { root: root.clone() } - .with_artifact(path, read) - .map_err(Into::into), + .with_artifact(path, read), } } From bccc6184d25f958f6ca21c9191fd6c9e93696678 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:45:16 +0200 Subject: [PATCH 40/62] Remove unused storage error variant --- crates/rustiq-core/src/persistence/error.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/rustiq-core/src/persistence/error.rs b/crates/rustiq-core/src/persistence/error.rs index 37d085a..02e115c 100644 --- a/crates/rustiq-core/src/persistence/error.rs +++ b/crates/rustiq-core/src/persistence/error.rs @@ -10,8 +10,6 @@ pub(crate) enum StorageError { InvalidPath(String), #[error("storage entry has an unexpected type: {0}")] UnexpectedEntryType(String), - #[error("artifact size exceeds the supported limit")] - SizeOverflow, } #[derive(Debug, Error)] From 9b8281eaaa876a6c0fce66bb0218f12f388f23e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:45:33 +0200 Subject: [PATCH 41/62] Document internal storage boundary --- docs/persistence-format-v1.md | 36 ++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/docs/persistence-format-v1.md b/docs/persistence-format-v1.md index e74ee04..f7e2f65 100644 --- a/docs/persistence-format-v1.md +++ b/docs/persistence-format-v1.md @@ -87,22 +87,28 @@ untouched. Names and fingerprints are validated rather than interpreted as paths ## Logical entries -The core Rust API exposes `persistence::RustiQData` together with a selectable -`persistence::Storage`. V1 currently provides `Storage::Folder(PathBuf)`; a ZIP-backed -variant can be added without changing the `RustiQData` type. `RustiQData::read(storage)` -reads the manifest only. Its -`read_eri` method validates and decodes the AO ERI NPY on first access and keeps -the resulting `CompactEri` for subsequent accesses. Writing to a selected storage copies artifacts that have not been decoded, -including unknown representations, as verified byte streams. Its public scientific API is typed: `set_eri` and -`read_eri` operate on `CompactEri`. The generic `get::()` returns -`Result, PersistenceError>`, while +The core Rust API exposes `persistence::RustiQData` as a typed scientific +facade. Physical storage selection is an internal persistence concern: the +directory-backed ERI cache resolves folder storage internally, and the future +portable `.rustiq` API will resolve ZIP/ZIP64 internally rather than exposing a +generic storage backend. Internal readers load the bounded manifest first and +open scientific artifacts on demand. `read_eri` validates and decodes the AO ERI +NPY on first access and keeps the resulting `CompactEri` for subsequent accesses. +Unknown representations can be copied internally as verified byte streams without +being decoded. + +The public scientific API is typed: `set_eri` and `read_eri` operate on +`CompactEri`. The generic `get::()` returns +`Result, ArtifactError>`, while `set::(value)` accepts only `CompactEri`; only declared artifact -marker types are accepted. A future known artifact gets its own marker, typed field, -and accessors in `RustiQData`. NPY parsing and conversion remain internal -persistence details. Compact ERIs are decoded only with the basis-function count -from their typed manifest attributes; the NPY length is never used to infer that -scientific context. Matrix readers accept C and Fortran order and matrix writers -emit Fortran order. +marker types are accepted. Artifact access reports `ArtifactError`, while NPY +format failures are represented by `NpyError`; storage, manifest, and persistence +orchestration errors remain internal. A future known artifact gets its own marker, +typed field, and accessors in `RustiQData`. NPY parsing and conversion remain +internal persistence details. Compact ERIs are decoded only with the +basis-function count from their typed manifest attributes; the NPY length is never +used to infer that scientific context. Matrix readers accept C and Fortran order +and matrix writers emit Fortran order. - `manifest.json` is UTF-8 JSON and describes the format, producer, scientific identity and artifacts. From ff2b718035a864e391b78f643177aa3e3a8c0a6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:47:47 +0200 Subject: [PATCH 42/62] Refine persistence API boundaries From 45cc330d91ce9dfb6893b2f4a4425c2e6bd2d140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:51:25 +0200 Subject: [PATCH 43/62] Format persistence refactor --- .../src/persistence/data/rustiq_data.rs | 42 ++++++--------- crates/rustiq-core/src/persistence/mod.rs | 1 - crates/rustiq-core/src/persistence/npy.rs | 52 +++++++------------ crates/rustiq-core/src/persistence/storage.rs | 27 ++++------ 4 files changed, 45 insertions(+), 77 deletions(-) diff --git a/crates/rustiq-core/src/persistence/data/rustiq_data.rs b/crates/rustiq-core/src/persistence/data/rustiq_data.rs index 69c88eb..184296c 100644 --- a/crates/rustiq-core/src/persistence/data/rustiq_data.rs +++ b/crates/rustiq-core/src/persistence/data/rustiq_data.rs @@ -143,15 +143,14 @@ impl RustiQData { let source = self.source.as_mut().ok_or(ArtifactError::Missing)?; let metadata = source.artifact_metadata(&artifact.path)?; if metadata.size != artifact.size || metadata.digest != artifact.digest { - return Err(ArtifactError::IntegrityMismatch( - artifact.path.to_string(), - )); + return Err(ArtifactError::IntegrityMismatch(artifact.path.to_string())); } - self.ao_eri = Some(source.with_artifact::<_, ArtifactError, _>( - &artifact.path, - |reader| read_compact_eri(reader, attributes.basis_functions).map_err(Into::into), - )?); + self.ao_eri = Some( + source.with_artifact::<_, ArtifactError, _>(&artifact.path, |reader| { + read_compact_eri(reader, attributes.basis_functions).map_err(Into::into) + })?, + ); } Ok(self @@ -164,10 +163,7 @@ impl RustiQData { self.ao_eri.take() } - pub(crate) fn write_to( - &mut self, - destination: Storage, - ) -> Result<(), PersistenceWriteError> { + pub(crate) fn write_to(&mut self, destination: Storage) -> Result<(), PersistenceWriteError> { let eri = self.ao_eri.take(); let result = self.write_inner(destination, eri.as_ref()); self.ao_eri = eri; @@ -199,9 +195,8 @@ impl RustiQData { })?; validate_artifact_path(&artifact.path)?; - let metadata = source.with_artifact::<_, PersistenceWriteError, _>( - &artifact.path, - |input| { + let metadata = + source.with_artifact::<_, PersistenceWriteError, _>(&artifact.path, |input| { destination.write_artifact::( &artifact.path, |output| { @@ -211,13 +206,10 @@ impl RustiQData { Ok(()) }, ) - }, - )?; + })?; if metadata.size != artifact.size || metadata.digest != artifact.digest { - return Err( - ArtifactError::IntegrityMismatch(artifact.path.to_string()).into(), - ); + return Err(ArtifactError::IntegrityMismatch(artifact.path.to_string()).into()); } } @@ -225,10 +217,10 @@ impl RustiQData { let basis_functions = self.basis_functions.ok_or(ArtifactError::Missing)?; validate_eri_len(eri, basis_functions)?; let path = RelativePath::new(AO_ERI_PATH); - let metadata = destination.write_artifact::( - path, - |writer| write_compact_eri(writer, eri).map_err(Into::into), - )?; + let metadata = destination + .write_artifact::(path, |writer| { + write_compact_eri(writer, eri).map_err(Into::into) + })?; manifest.artifacts.insert( AO_ERI_ARTIFACT.to_owned(), @@ -257,9 +249,7 @@ impl RustiQData { fn validate_eri_len(eri: &CompactEri, basis_functions: usize) -> Result<(), ArtifactError> { let expected = CompactEri::checked_storage_len(basis_functions).ok_or_else(|| { - ArtifactError::InvalidMetadata( - "basis-function count overflows compact ERI storage".into(), - ) + ArtifactError::InvalidMetadata("basis-function count overflows compact ERI storage".into()) })?; if eri.len() != expected { return Err(ArtifactError::InvalidValueCount { diff --git a/crates/rustiq-core/src/persistence/mod.rs b/crates/rustiq-core/src/persistence/mod.rs index 95da712..771d5d8 100644 --- a/crates/rustiq-core/src/persistence/mod.rs +++ b/crates/rustiq-core/src/persistence/mod.rs @@ -40,4 +40,3 @@ pub(crate) const MANIFEST_PATH: &str = "manifest.json"; pub(crate) const AO_ERI_PATH: &str = "arrays/integrals/ao-eri.npy"; pub(crate) const COMPACT_ERI_REPRESENTATION: &str = "rustiq-compact-eri-v1"; pub(crate) const SCIENTIFIC_IDENTITY_VERSION: u32 = 1; - diff --git a/crates/rustiq-core/src/persistence/npy.rs b/crates/rustiq-core/src/persistence/npy.rs index f7940fa..64a2ffb 100644 --- a/crates/rustiq-core/src/persistence/npy.rs +++ b/crates/rustiq-core/src/persistence/npy.rs @@ -14,10 +14,7 @@ trait NpyConvert: Sized { fn npy_shape(shape: Self::Shape) -> Result; - fn decode_with_shape( - npy: NpyFile, - shape: Self::Shape, - ) -> Result; + fn decode_with_shape(npy: NpyFile, shape: Self::Shape) -> Result; /// Checks a parsed header before decoding any array values. fn try_from_npy_with_shape( @@ -35,14 +32,8 @@ trait NpyConvert: Sized { } /// Parses a stream once, checks its shape, then decodes its values. - fn try_read_with_shape( - reader: impl Read, - shape: Self::Shape, - ) -> Result { - Self::try_from_npy_with_shape( - NpyFile::new(reader).map_err(NpyError::Read)?, - shape, - ) + fn try_read_with_shape(reader: impl Read, shape: Self::Shape) -> Result { + Self::try_from_npy_with_shape(NpyFile::new(reader).map_err(NpyError::Read)?, shape) } fn write_npy(&self, writer: impl Write) -> Result<(), NpyError>; @@ -53,12 +44,9 @@ impl NpyConvert for CompactEri { type NpyShape = [u64; 1]; fn npy_shape(basis_functions: usize) -> Result { - let length = CompactEri::checked_storage_len(basis_functions).ok_or_else(|| { - NpyError::DimensionOverflow - })?; - Ok([u64::try_from(length).map_err(|_| { - NpyError::DimensionOverflow - })?]) + let length = CompactEri::checked_storage_len(basis_functions) + .ok_or_else(|| NpyError::DimensionOverflow)?; + Ok([u64::try_from(length).map_err(|_| NpyError::DimensionOverflow)?]) } fn decode_with_shape( @@ -88,12 +76,8 @@ impl NpyConvert for DMatrix { type NpyShape = [u64; 2]; fn npy_shape((rows, columns): (usize, usize)) -> Result { - let rows = u64::try_from(rows).map_err(|_| { - NpyError::DimensionOverflow - })?; - let columns = u64::try_from(columns).map_err(|_| { - NpyError::DimensionOverflow - })?; + let rows = u64::try_from(rows).map_err(|_| NpyError::DimensionOverflow)?; + let columns = u64::try_from(columns).map_err(|_| NpyError::DimensionOverflow)?; Ok([rows, columns]) } @@ -120,10 +104,7 @@ impl NpyConvert for DMatrix { } } -pub(crate) fn write_compact_eri( - writer: impl Write, - eri: &CompactEri, -) -> Result<(), NpyError> { +pub(crate) fn write_compact_eri(writer: impl Write, eri: &CompactEri) -> Result<(), NpyError> { eri.write_npy(writer) } @@ -139,17 +120,18 @@ fn decode_compact_eri( basis_functions: usize, ) -> Result { if npy.shape().len() != 1 { - return Err(NpyError::InvalidEriShape(npy.shape().to_vec().into_boxed_slice())); + return Err(NpyError::InvalidEriShape( + npy.shape().to_vec().into_boxed_slice(), + )); } let actual = usize::try_from(npy.shape()[0]) .map_err(|_| NpyError::InvalidEriShape(npy.shape().to_vec().into_boxed_slice()))?; - let expected = CompactEri::checked_storage_len(basis_functions).ok_or( - NpyError::InvalidValueCount { + let expected = + CompactEri::checked_storage_len(basis_functions).ok_or(NpyError::InvalidValueCount { basis_functions, expected: 0, actual, - }, - )?; + })?; if actual != expected { return Err(NpyError::InvalidValueCount { basis_functions, @@ -220,7 +202,9 @@ pub(crate) fn read_dmatrix(reader: impl Read) -> Result, NpyError> fn decode_dmatrix(npy: NpyFile) -> Result, NpyError> { if npy.shape().len() != 2 { - return Err(NpyError::InvalidMatrixShape(npy.shape().to_vec().into_boxed_slice())); + return Err(NpyError::InvalidMatrixShape( + npy.shape().to_vec().into_boxed_slice(), + )); } let rows = usize::try_from(npy.shape()[0]) .map_err(|_| NpyError::InvalidMatrixShape(npy.shape().to_vec().into_boxed_slice()))?; diff --git a/crates/rustiq-core/src/persistence/storage.rs b/crates/rustiq-core/src/persistence/storage.rs index 11fe6e0..a15b898 100644 --- a/crates/rustiq-core/src/persistence/storage.rs +++ b/crates/rustiq-core/src/persistence/storage.rs @@ -79,18 +79,13 @@ impl Storage { } } - pub(crate) fn with_artifact( - &mut self, - path: &RelativePath, - read: F, - ) -> Result + pub(crate) fn with_artifact(&mut self, path: &RelativePath, read: F) -> Result where E: From, F: FnOnce(&mut dyn Read) -> Result, { match self { - Self::Folder(root) => FolderStorage { root: root.clone() } - .with_artifact(path, read), + Self::Folder(root) => FolderStorage { root: root.clone() }.with_artifact(path, read), } } @@ -104,8 +99,7 @@ impl Storage { F: FnOnce(&mut dyn Write) -> Result<(), E>, { match self { - Self::Folder(root) => FolderStorage { root: root.clone() } - .write_artifact(path, write), + Self::Folder(root) => FolderStorage { root: root.clone() }.write_artifact(path, write), } } @@ -172,11 +166,7 @@ impl FolderStorage { read(&mut reader) } - fn write_artifact( - &mut self, - path: &RelativePath, - write: F, - ) -> Result + fn write_artifact(&mut self, path: &RelativePath, write: F) -> Result where E: From, F: FnOnce(&mut dyn Write) -> Result<(), E>, @@ -185,13 +175,18 @@ impl FolderStorage { let writer = BufWriter::new(file); let mut writer = DigestWriter::new(writer); write(&mut writer)?; - writer.flush().map_err(StorageError::from).map_err(E::from)?; + writer + .flush() + .map_err(StorageError::from) + .map_err(E::from)?; let (writer, metadata) = writer.finish(); let file = writer .into_inner() .map_err(|error| StorageError::Io(error.into_error())) .map_err(E::from)?; - file.sync_all().map_err(StorageError::from).map_err(E::from)?; + file.sync_all() + .map_err(StorageError::from) + .map_err(E::from)?; Ok(metadata) } From 05e2eb63f2ca03c08a21cc17431099200b6ed181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:54:23 +0200 Subject: [PATCH 44/62] Fix persistence type inference --- crates/rustiq-core/src/persistence/data/rustiq_data.rs | 2 +- crates/rustiq-core/src/persistence/storage.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rustiq-core/src/persistence/data/rustiq_data.rs b/crates/rustiq-core/src/persistence/data/rustiq_data.rs index 184296c..e51a9ac 100644 --- a/crates/rustiq-core/src/persistence/data/rustiq_data.rs +++ b/crates/rustiq-core/src/persistence/data/rustiq_data.rs @@ -78,7 +78,7 @@ impl RustiQData { return Err(ManifestError::UnsupportedFormat.into()); } - let mut paths = HashSet::new(); + let mut paths: HashSet = HashSet::new(); for artifact in manifest.artifacts.values() { validate_artifact_path(&artifact.path)?; let key = artifact.path.as_str().to_lowercase(); diff --git a/crates/rustiq-core/src/persistence/storage.rs b/crates/rustiq-core/src/persistence/storage.rs index a15b898..7633beb 100644 --- a/crates/rustiq-core/src/persistence/storage.rs +++ b/crates/rustiq-core/src/persistence/storage.rs @@ -125,7 +125,7 @@ impl Storage { path: &RelativePath, value: &T, ) -> Result<(), ManifestError> { - self.write_artifact(path, |writer| { + self.write_artifact::(path, |writer| { serde_json::to_writer_pretty(&mut *writer, value)?; writer.write_all(b"\n").map_err(StorageError::from)?; Ok(()) From 2d027b9ef4266d230462dae7ebf3bb572dcb058c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:56:45 +0200 Subject: [PATCH 45/62] Fix persistence clippy lint --- crates/rustiq-core/src/persistence/npy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rustiq-core/src/persistence/npy.rs b/crates/rustiq-core/src/persistence/npy.rs index 64a2ffb..43ed42b 100644 --- a/crates/rustiq-core/src/persistence/npy.rs +++ b/crates/rustiq-core/src/persistence/npy.rs @@ -45,7 +45,7 @@ impl NpyConvert for CompactEri { fn npy_shape(basis_functions: usize) -> Result { let length = CompactEri::checked_storage_len(basis_functions) - .ok_or_else(|| NpyError::DimensionOverflow)?; + .ok_or(NpyError::DimensionOverflow)?; Ok([u64::try_from(length).map_err(|_| NpyError::DimensionOverflow)?]) } From 3b1083473e46081c1747d9c4a4de9d2b75252cce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 16:58:11 +0200 Subject: [PATCH 46/62] Format NPY clippy fix --- crates/rustiq-core/src/persistence/npy.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rustiq-core/src/persistence/npy.rs b/crates/rustiq-core/src/persistence/npy.rs index 43ed42b..7001418 100644 --- a/crates/rustiq-core/src/persistence/npy.rs +++ b/crates/rustiq-core/src/persistence/npy.rs @@ -44,8 +44,8 @@ impl NpyConvert for CompactEri { type NpyShape = [u64; 1]; fn npy_shape(basis_functions: usize) -> Result { - let length = CompactEri::checked_storage_len(basis_functions) - .ok_or(NpyError::DimensionOverflow)?; + let length = + CompactEri::checked_storage_len(basis_functions).ok_or(NpyError::DimensionOverflow)?; Ok([u64::try_from(length).map_err(|_| NpyError::DimensionOverflow)?]) } From a57855de867698e327a48a34b54e362abead2372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 17:01:30 +0200 Subject: [PATCH 47/62] Fix NumPy matrix interoperability fixture --- .../tests/data/persistence/dmatrix-python-c-v1.npy.hex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rustiq-core/tests/data/persistence/dmatrix-python-c-v1.npy.hex b/crates/rustiq-core/tests/data/persistence/dmatrix-python-c-v1.npy.hex index becde52..3491ea1 100644 --- a/crates/rustiq-core/tests/data/persistence/dmatrix-python-c-v1.npy.hex +++ b/crates/rustiq-core/tests/data/persistence/dmatrix-python-c-v1.npy.hex @@ -1 +1 @@ -934e554d5059010076007b276465736372273a20273c6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028322c2033292c207d20202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a000000000000f03f00000000000000400000000000000840000000000000104000000000000014400000000000001840 +934e554d5059010076007b276465736372273a20273c6638272c2027666f727472616e5f6f72646572273a2046616c73652c20277368617065273a2028322c2033292c207d202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200a000000000000f03f00000000000000400000000000000840000000000000104000000000000014400000000000001840 From 435af65fab0e842c18acfc314ea24b29250c4f9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 17:01:33 +0200 Subject: [PATCH 48/62] Fix NumPy matrix interoperability fixture From 91921c3ec36596e5b10cc080627454b7cb02f602 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 17:29:03 +0200 Subject: [PATCH 49/62] Decouple RustiQData from cache policy --- .../src/persistence/data/rustiq_data.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/crates/rustiq-core/src/persistence/data/rustiq_data.rs b/crates/rustiq-core/src/persistence/data/rustiq_data.rs index e51a9ac..fdf594d 100644 --- a/crates/rustiq-core/src/persistence/data/rustiq_data.rs +++ b/crates/rustiq-core/src/persistence/data/rustiq_data.rs @@ -17,7 +17,7 @@ use super::super::{ use super::artifact::Artifact; pub(crate) const AO_ERI_ARTIFACT: &str = "ao_eri"; -pub(crate) const CACHE_KIND: &str = "integral-cache"; +const CHECKPOINT_KIND: &str = "checkpoint"; const MAX_MANIFEST_BYTES: u64 = 1024 * 1024; /// Known scientific artifacts and their manifest, with values loaded on demand. @@ -40,21 +40,22 @@ impl RustiQData { A::set(self, value) } - /// Starts a new AO ERI data set with the current scientific identity. + /// Starts a new checkpoint data set with the current scientific identity. pub fn new(geometry: &Geometry, basis: &Basis, threshold: Option) -> Self { let identity = ao_eri_identity(geometry, basis, threshold); - Self::new_with_identity(identity, basis.nbasis()) + Self::new_with_identity(identity, basis.nbasis(), CHECKPOINT_KIND) } pub(crate) fn new_with_identity( identity: super::super::ScientificIdentity, basis_functions: usize, + kind: impl Into, ) -> Self { Self { manifest: Manifest { format: FORMAT_NAME.to_owned(), format_version: FORMAT_VERSION, - kind: CACHE_KIND.to_owned(), + kind: kind.into(), producer: Producer { name: "RustiQ".to_owned(), version: env!("CARGO_PKG_VERSION").to_owned(), @@ -173,9 +174,9 @@ impl RustiQData { pub(crate) fn write_with_eri( &mut self, destination: Storage, - eri: Option<&CompactEri>, + eri: &CompactEri, ) -> Result<(), PersistenceWriteError> { - self.write_inner(destination, eri) + self.write_inner(destination, Some(eri)) } fn write_inner( @@ -237,10 +238,6 @@ impl RustiQData { ); } - if manifest.kind == CACHE_KIND && !manifest.artifacts.contains_key(AO_ERI_ARTIFACT) { - return Err(ArtifactError::Missing.into()); - } - destination.write_json(RelativePath::new(MANIFEST_PATH), &manifest)?; destination.finish()?; Ok(()) @@ -295,6 +292,7 @@ mod tests { digest: Sha256Digest::from([7; 32]), }, 2, + CHECKPOINT_KIND, ) } From 49de5b594a46592bfacdef7cf605a36ec8a1783a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 17:29:06 +0200 Subject: [PATCH 50/62] Own cache persistence policy in EriCache --- crates/rustiq-core/src/persistence/eri_cache.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/rustiq-core/src/persistence/eri_cache.rs b/crates/rustiq-core/src/persistence/eri_cache.rs index 296078b..354ce0e 100644 --- a/crates/rustiq-core/src/persistence/eri_cache.rs +++ b/crates/rustiq-core/src/persistence/eri_cache.rs @@ -18,7 +18,9 @@ use super::{ FORMAT_VERSION, SCIENTIFIC_IDENTITY_VERSION, }; -use super::data::{AO_ERI_ARTIFACT, CACHE_KIND}; +use super::data::AO_ERI_ARTIFACT; + +const CACHE_KIND: &str = "integral-cache"; /// A directory-backed AO ERI cache entry available for management. #[derive(Clone, Debug, PartialEq, Eq)] @@ -338,9 +340,10 @@ impl EriCache { } fs::create_dir_all(parent)?; let temporary = Builder::new().prefix(".rustiq-eri-").tempdir_in(parent)?; - let mut entry_data = RustiQData::new_with_identity(identity, basis_functions); + let mut entry_data = + RustiQData::new_with_identity(identity, basis_functions, CACHE_KIND); entry_data - .write_with_eri(Storage::folder(temporary.path()), Some(eri)) + .write_with_eri(Storage::folder(temporary.path()), eri) .map_err(io::Error::other)?; let temporary_path = temporary.keep(); if fs::symlink_metadata(&final_entry).is_ok() { From 214b2da10e9b2b735d86b326f0368209c353c983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 17:55:21 +0200 Subject: [PATCH 51/62] Remove stale cache kind re-export --- crates/rustiq-core/src/persistence/data.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rustiq-core/src/persistence/data.rs b/crates/rustiq-core/src/persistence/data.rs index 4f47954..86dccc8 100644 --- a/crates/rustiq-core/src/persistence/data.rs +++ b/crates/rustiq-core/src/persistence/data.rs @@ -5,4 +5,4 @@ mod rustiq_data; pub use ao_eri_artifact::AoEriArtifact; pub use artifact::Artifact; pub use rustiq_data::RustiQData; -pub(crate) use rustiq_data::{AO_ERI_ARTIFACT, CACHE_KIND}; +pub(crate) use rustiq_data::AO_ERI_ARTIFACT; From 19a8369f189c76bfd9d48f2cbf0d572a306f0526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 17:55:24 +0200 Subject: [PATCH 52/62] Format cache persistence update --- crates/rustiq-core/src/persistence/eri_cache.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/rustiq-core/src/persistence/eri_cache.rs b/crates/rustiq-core/src/persistence/eri_cache.rs index 354ce0e..dc10e47 100644 --- a/crates/rustiq-core/src/persistence/eri_cache.rs +++ b/crates/rustiq-core/src/persistence/eri_cache.rs @@ -340,8 +340,7 @@ impl EriCache { } fs::create_dir_all(parent)?; let temporary = Builder::new().prefix(".rustiq-eri-").tempdir_in(parent)?; - let mut entry_data = - RustiQData::new_with_identity(identity, basis_functions, CACHE_KIND); + let mut entry_data = RustiQData::new_with_identity(identity, basis_functions, CACHE_KIND); entry_data .write_with_eri(Storage::folder(temporary.path()), eri) .map_err(io::Error::other)?; From b242f10c1dddd2efa0afc50d1ff056acaad0b59d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 18:02:37 +0200 Subject: [PATCH 53/62] Type known manifest kinds --- crates/rustiq-core/src/persistence/manifest.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/rustiq-core/src/persistence/manifest.rs b/crates/rustiq-core/src/persistence/manifest.rs index e7c7431..c8657b6 100644 --- a/crates/rustiq-core/src/persistence/manifest.rs +++ b/crates/rustiq-core/src/persistence/manifest.rs @@ -6,6 +6,21 @@ use serde_json::Value; use super::{Sha256Digest, COMPACT_ERI_REPRESENTATION}; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ManifestKind { + Checkpoint, + IntegralCache, +} + +impl ManifestKind { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Checkpoint => "checkpoint", + Self::IntegralCache => "integral-cache", + } + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub(crate) struct Manifest { pub format: String, From 66fb6def8ac98ca51ae36cbadbf64f94020e7585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 18:02:41 +0200 Subject: [PATCH 54/62] Expose known manifest kinds internally --- crates/rustiq-core/src/persistence/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rustiq-core/src/persistence/mod.rs b/crates/rustiq-core/src/persistence/mod.rs index 771d5d8..1a5bad4 100644 --- a/crates/rustiq-core/src/persistence/mod.rs +++ b/crates/rustiq-core/src/persistence/mod.rs @@ -23,7 +23,7 @@ pub(crate) use error::{ManifestError, PersistenceReadError, PersistenceWriteErro #[allow(unused_imports)] pub(crate) use identity::{ao_eri_identity, ScientificIdentity, AO_ERI_COMPUTATION_VERSION}; pub(crate) use manifest::{ - AoEriAttributes, ArtifactAttributes, ArtifactManifest, Manifest, Producer, + AoEriAttributes, ArtifactAttributes, ArtifactManifest, Manifest, ManifestKind, Producer, ScientificIdentityManifest, }; pub(crate) use storage::Storage; From 8a3550cfbca21afea112bd5866fdcc7b373c5e54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 18:03:02 +0200 Subject: [PATCH 55/62] Test checkpoint persistence boundary --- .../src/persistence/data/rustiq_data.rs | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/crates/rustiq-core/src/persistence/data/rustiq_data.rs b/crates/rustiq-core/src/persistence/data/rustiq_data.rs index fdf594d..c4f808e 100644 --- a/crates/rustiq-core/src/persistence/data/rustiq_data.rs +++ b/crates/rustiq-core/src/persistence/data/rustiq_data.rs @@ -9,7 +9,7 @@ use crate::{ use super::super::{ ao_eri_identity, read_compact_eri, validate_storage_path, write_compact_eri, AoEriAttributes, - ArtifactAttributes, ArtifactError, ArtifactManifest, Manifest, ManifestError, + ArtifactAttributes, ArtifactError, ArtifactManifest, Manifest, ManifestError, ManifestKind, PersistenceReadError, PersistenceWriteError, Producer, ScientificIdentityManifest, Storage, StorageError, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, FORMAT_VERSION, MANIFEST_PATH, @@ -17,7 +17,6 @@ use super::super::{ use super::artifact::Artifact; pub(crate) const AO_ERI_ARTIFACT: &str = "ao_eri"; -const CHECKPOINT_KIND: &str = "checkpoint"; const MAX_MANIFEST_BYTES: u64 = 1024 * 1024; /// Known scientific artifacts and their manifest, with values loaded on demand. @@ -43,19 +42,19 @@ impl RustiQData { /// Starts a new checkpoint data set with the current scientific identity. pub fn new(geometry: &Geometry, basis: &Basis, threshold: Option) -> Self { let identity = ao_eri_identity(geometry, basis, threshold); - Self::new_with_identity(identity, basis.nbasis(), CHECKPOINT_KIND) + Self::new_with_identity(identity, basis.nbasis(), ManifestKind::Checkpoint) } pub(crate) fn new_with_identity( identity: super::super::ScientificIdentity, basis_functions: usize, - kind: impl Into, + kind: ManifestKind, ) -> Self { Self { manifest: Manifest { format: FORMAT_NAME.to_owned(), format_version: FORMAT_VERSION, - kind: kind.into(), + kind: kind.as_str().to_owned(), producer: Producer { name: "RustiQ".to_owned(), version: env!("CARGO_PKG_VERSION").to_owned(), @@ -292,10 +291,24 @@ mod tests { digest: Sha256Digest::from([7; 32]), }, 2, - CHECKPOINT_KIND, + ManifestKind::Checkpoint, ) } + #[test] + fn checkpoint_can_be_written_without_ao_eri() { + let root = tempfile::tempdir().unwrap(); + let entry = root.path().join("checkpoint"); + fs::create_dir(&entry).unwrap(); + + let mut data = new_data(); + data.write_to(Storage::folder(&entry)).unwrap(); + + let restored = RustiQData::read_from(Storage::folder(&entry)).unwrap(); + assert_eq!(restored.manifest().kind, ManifestKind::Checkpoint.as_str()); + assert!(!restored.manifest().artifacts.contains_key(AO_ERI_ARTIFACT)); + } + #[test] fn eri_is_loaded_only_on_request_and_then_cached_as_an_object() { let root = tempfile::tempdir().unwrap(); From 2ddf0633c9c66c1ec4e256c2066edc6043c8c42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 18:03:04 +0200 Subject: [PATCH 56/62] Test cache persistence boundary --- .../rustiq-core/src/persistence/eri_cache.rs | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/crates/rustiq-core/src/persistence/eri_cache.rs b/crates/rustiq-core/src/persistence/eri_cache.rs index dc10e47..8383ded 100644 --- a/crates/rustiq-core/src/persistence/eri_cache.rs +++ b/crates/rustiq-core/src/persistence/eri_cache.rs @@ -13,15 +13,14 @@ use crate::{ use super::{ ao_eri_identity, sha256_reader, validate_compact_eri_header, AoEriAttributes, - ArtifactAttributes, ArtifactManifest, Manifest, RustiQData, ScientificIdentity, Storage, + ArtifactAttributes, ArtifactManifest, Manifest, ManifestKind, RustiQData, ScientificIdentity, + Storage, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, FORMAT_VERSION, SCIENTIFIC_IDENTITY_VERSION, }; use super::data::AO_ERI_ARTIFACT; -const CACHE_KIND: &str = "integral-cache"; - /// A directory-backed AO ERI cache entry available for management. #[derive(Clone, Debug, PartialEq, Eq)] pub struct EriCacheEntry { @@ -340,7 +339,8 @@ impl EriCache { } fs::create_dir_all(parent)?; let temporary = Builder::new().prefix(".rustiq-eri-").tempdir_in(parent)?; - let mut entry_data = RustiQData::new_with_identity(identity, basis_functions, CACHE_KIND); + let mut entry_data = + RustiQData::new_with_identity(identity, basis_functions, ManifestKind::IntegralCache); entry_data .write_with_eri(Storage::folder(temporary.path()), eri) .map_err(io::Error::other)?; @@ -378,7 +378,7 @@ fn read_manifest(entry: &Path) -> Option { fn manifest_is_valid(manifest: &Manifest, identity: ScientificIdentity) -> bool { manifest.format == FORMAT_NAME && manifest.format_version == FORMAT_VERSION - && manifest.kind == CACHE_KIND + && manifest.kind == ManifestKind::IntegralCache.as_str() && manifest.scientific_identity.version == identity.version && manifest.scientific_identity.digest == identity.digest && manifest @@ -493,6 +493,42 @@ mod tests { eri.ordered_values() ); } + #[test] + fn stored_entry_is_explicitly_an_integral_cache_with_ao_eri() { + let temporary = tempfile::tempdir().unwrap(); + let cache = EriCache::new(temporary.path()); + let (molecule, basis) = input(); + let eri = CompactEri::Zeroed(basis.nbasis()); + + cache.store(&molecule, &basis, threshold(), &eri).unwrap(); + + let identity = ao_eri_identity(molecule.geometry(), &basis, threshold()); + let manifest = read_manifest(&cache.entry_path(identity)).unwrap(); + assert_eq!(manifest.kind, ManifestKind::IntegralCache.as_str()); + assert!(manifest.artifacts.contains_key(AO_ERI_ARTIFACT)); + } + + #[test] + fn checkpoint_is_never_accepted_as_a_cache_hit() { + let temporary = tempfile::tempdir().unwrap(); + let cache = EriCache::new(temporary.path()); + let (molecule, basis) = input(); + let eri = CompactEri::Zeroed(basis.nbasis()); + cache.store(&molecule, &basis, threshold(), &eri).unwrap(); + + let identity = ao_eri_identity(molecule.geometry(), &basis, threshold()); + let manifest_path = cache.entry_path(identity).join(MANIFEST_PATH); + let mut manifest: serde_json::Value = + serde_json::from_reader(File::open(&manifest_path).unwrap()).unwrap(); + manifest["kind"] = serde_json::json!(ManifestKind::Checkpoint.as_str()); + fs::write(&manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap(); + + assert!(cache.load(&molecule, &basis, threshold()).is_none()); + let entries = cache.entries().unwrap(); + assert_eq!(entries.len(), 1); + assert!(!entries[0].verified); + } + #[test] fn corruption_is_a_cache_miss() { let temporary = tempfile::tempdir().unwrap(); From e6a0413567e1de966322b0a06aa246d70f87a808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 18:18:31 +0200 Subject: [PATCH 57/62] Preserve unknown manifest kinds --- .../rustiq-core/src/persistence/manifest.rs | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/crates/rustiq-core/src/persistence/manifest.rs b/crates/rustiq-core/src/persistence/manifest.rs index c8657b6..159f290 100644 --- a/crates/rustiq-core/src/persistence/manifest.rs +++ b/crates/rustiq-core/src/persistence/manifest.rs @@ -6,26 +6,51 @@ use serde_json::Value; use super::{Sha256Digest, COMPACT_ERI_REPRESENTATION}; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum ManifestKind { Checkpoint, IntegralCache, + Unknown(String), } impl ManifestKind { - pub(crate) const fn as_str(self) -> &'static str { + pub(crate) fn as_str(&self) -> &str { match self { Self::Checkpoint => "checkpoint", Self::IntegralCache => "integral-cache", + Self::Unknown(value) => value, } } } +impl Serialize for ManifestKind { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for ManifestKind { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Ok(match value.as_str() { + "checkpoint" => Self::Checkpoint, + "integral-cache" => Self::IntegralCache, + _ => Self::Unknown(value), + }) + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub(crate) struct Manifest { pub format: String, pub format_version: u32, - pub kind: String, + pub kind: ManifestKind, pub producer: Producer, pub scientific_identity: ScientificIdentityManifest, pub artifacts: BTreeMap, @@ -113,6 +138,8 @@ fn decode_attributes( #[cfg(test)] mod tests { + use proptest::prelude::*; + use super::*; use crate::persistence::{ AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, FORMAT_NAME, FORMAT_VERSION, @@ -139,7 +166,7 @@ mod tests { let manifest = Manifest { format: FORMAT_NAME.to_string(), format_version: FORMAT_VERSION, - kind: "integral-cache".to_string(), + kind: ManifestKind::IntegralCache, producer: Producer { name: "RustiQ".to_string(), version: "0.1.0".to_string(), @@ -185,6 +212,19 @@ mod tests { assert_eq!(serde_json::from_str::(&json).unwrap(), manifest); } + proptest! { + #[test] + fn unknown_manifest_kind_round_trips(value in "[A-Za-z0-9_-]{1,64}") { + prop_assume!(value != "checkpoint" && value != "integral-cache"); + + let kind = ManifestKind::Unknown(value.clone()); + let json = serde_json::to_string(&kind).unwrap(); + let restored: ManifestKind = serde_json::from_str(&json).unwrap(); + + prop_assert_eq!(restored, ManifestKind::Unknown(value)); + } + } + #[test] fn unknown_artifact_attributes_are_preserved() { let json = r#"{ From c9c3e28a26a6042cee387768423c68ea93739ee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 18:18:34 +0200 Subject: [PATCH 58/62] Store typed manifest kinds --- crates/rustiq-core/src/persistence/data/rustiq_data.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rustiq-core/src/persistence/data/rustiq_data.rs b/crates/rustiq-core/src/persistence/data/rustiq_data.rs index c4f808e..2e4212b 100644 --- a/crates/rustiq-core/src/persistence/data/rustiq_data.rs +++ b/crates/rustiq-core/src/persistence/data/rustiq_data.rs @@ -54,7 +54,7 @@ impl RustiQData { manifest: Manifest { format: FORMAT_NAME.to_owned(), format_version: FORMAT_VERSION, - kind: kind.as_str().to_owned(), + kind, producer: Producer { name: "RustiQ".to_owned(), version: env!("CARGO_PKG_VERSION").to_owned(), @@ -305,7 +305,7 @@ mod tests { data.write_to(Storage::folder(&entry)).unwrap(); let restored = RustiQData::read_from(Storage::folder(&entry)).unwrap(); - assert_eq!(restored.manifest().kind, ManifestKind::Checkpoint.as_str()); + assert_eq!(restored.manifest().kind, ManifestKind::Checkpoint); assert!(!restored.manifest().artifacts.contains_key(AO_ERI_ARTIFACT)); } From cdf0fcefe82ac3d56097202b8e0140a6ae6c8be3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 18:18:37 +0200 Subject: [PATCH 59/62] Use typed manifest kind matching --- crates/rustiq-core/src/persistence/eri_cache.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rustiq-core/src/persistence/eri_cache.rs b/crates/rustiq-core/src/persistence/eri_cache.rs index 8383ded..fdae3cf 100644 --- a/crates/rustiq-core/src/persistence/eri_cache.rs +++ b/crates/rustiq-core/src/persistence/eri_cache.rs @@ -378,7 +378,7 @@ fn read_manifest(entry: &Path) -> Option { fn manifest_is_valid(manifest: &Manifest, identity: ScientificIdentity) -> bool { manifest.format == FORMAT_NAME && manifest.format_version == FORMAT_VERSION - && manifest.kind == ManifestKind::IntegralCache.as_str() + && manifest.kind == ManifestKind::IntegralCache && manifest.scientific_identity.version == identity.version && manifest.scientific_identity.digest == identity.digest && manifest @@ -504,7 +504,7 @@ mod tests { let identity = ao_eri_identity(molecule.geometry(), &basis, threshold()); let manifest = read_manifest(&cache.entry_path(identity)).unwrap(); - assert_eq!(manifest.kind, ManifestKind::IntegralCache.as_str()); + assert_eq!(manifest.kind, ManifestKind::IntegralCache); assert!(manifest.artifacts.contains_key(AO_ERI_ARTIFACT)); } From f7cf24053a586a5e0ed6051c31cdc73081ad5e74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 18:37:43 +0200 Subject: [PATCH 60/62] Remove unsupported checkpoint kind --- crates/rustiq-core/src/persistence/manifest.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/rustiq-core/src/persistence/manifest.rs b/crates/rustiq-core/src/persistence/manifest.rs index 159f290..4a0ca36 100644 --- a/crates/rustiq-core/src/persistence/manifest.rs +++ b/crates/rustiq-core/src/persistence/manifest.rs @@ -8,7 +8,6 @@ use super::{Sha256Digest, COMPACT_ERI_REPRESENTATION}; #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum ManifestKind { - Checkpoint, IntegralCache, Unknown(String), } @@ -16,7 +15,6 @@ pub(crate) enum ManifestKind { impl ManifestKind { pub(crate) fn as_str(&self) -> &str { match self { - Self::Checkpoint => "checkpoint", Self::IntegralCache => "integral-cache", Self::Unknown(value) => value, } @@ -39,7 +37,6 @@ impl<'de> Deserialize<'de> for ManifestKind { { let value = String::deserialize(deserializer)?; Ok(match value.as_str() { - "checkpoint" => Self::Checkpoint, "integral-cache" => Self::IntegralCache, _ => Self::Unknown(value), }) @@ -215,7 +212,7 @@ mod tests { proptest! { #[test] fn unknown_manifest_kind_round_trips(value in "[A-Za-z0-9_-]{1,64}") { - prop_assume!(value != "checkpoint" && value != "integral-cache"); + prop_assume!(value != "integral-cache"); let kind = ManifestKind::Unknown(value.clone()); let json = serde_json::to_string(&kind).unwrap(); @@ -230,7 +227,7 @@ mod tests { let json = r#"{ "format": "rustiq-persistence", "format_version": 1, - "kind": "checkpoint", + "kind": "future-state", "producer": {"name": "RustiQ", "version": "0.2.0"}, "scientific_identity": { "version": 1, From 0ae3e548e2fcdee9cadb9b5e67c17ed1f2ccdb0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 18:37:46 +0200 Subject: [PATCH 61/62] Remove unsupported checkpoint constructor --- .../src/persistence/data/rustiq_data.rs | 29 ++----------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/crates/rustiq-core/src/persistence/data/rustiq_data.rs b/crates/rustiq-core/src/persistence/data/rustiq_data.rs index 2e4212b..6ab7648 100644 --- a/crates/rustiq-core/src/persistence/data/rustiq_data.rs +++ b/crates/rustiq-core/src/persistence/data/rustiq_data.rs @@ -2,13 +2,10 @@ use std::collections::{BTreeMap, HashSet}; use relative_path::RelativePath; -use crate::{ - basis::Basis, config::validated::PositiveFiniteF64, eri::CompactEri, - molecules::geometry::Geometry, -}; +use crate::eri::CompactEri; use super::super::{ - ao_eri_identity, read_compact_eri, validate_storage_path, write_compact_eri, AoEriAttributes, + read_compact_eri, validate_storage_path, write_compact_eri, AoEriAttributes, ArtifactAttributes, ArtifactError, ArtifactManifest, Manifest, ManifestError, ManifestKind, PersistenceReadError, PersistenceWriteError, Producer, ScientificIdentityManifest, Storage, StorageError, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, @@ -39,12 +36,6 @@ impl RustiQData { A::set(self, value) } - /// Starts a new checkpoint data set with the current scientific identity. - pub fn new(geometry: &Geometry, basis: &Basis, threshold: Option) -> Self { - let identity = ao_eri_identity(geometry, basis, threshold); - Self::new_with_identity(identity, basis.nbasis(), ManifestKind::Checkpoint) - } - pub(crate) fn new_with_identity( identity: super::super::ScientificIdentity, basis_functions: usize, @@ -291,24 +282,10 @@ mod tests { digest: Sha256Digest::from([7; 32]), }, 2, - ManifestKind::Checkpoint, + ManifestKind::Unknown("test-data".to_owned()), ) } - #[test] - fn checkpoint_can_be_written_without_ao_eri() { - let root = tempfile::tempdir().unwrap(); - let entry = root.path().join("checkpoint"); - fs::create_dir(&entry).unwrap(); - - let mut data = new_data(); - data.write_to(Storage::folder(&entry)).unwrap(); - - let restored = RustiQData::read_from(Storage::folder(&entry)).unwrap(); - assert_eq!(restored.manifest().kind, ManifestKind::Checkpoint); - assert!(!restored.manifest().artifacts.contains_key(AO_ERI_ARTIFACT)); - } - #[test] fn eri_is_loaded_only_on_request_and_then_cached_as_an_object() { let root = tempfile::tempdir().unwrap(); From 7fd41477c2b9d32e4f27e5f25191daf23782e9c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20V=C3=A9ril?= Date: Sun, 27 Sep 2026 18:37:49 +0200 Subject: [PATCH 62/62] Test unknown kind cache rejection --- crates/rustiq-core/src/persistence/eri_cache.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/rustiq-core/src/persistence/eri_cache.rs b/crates/rustiq-core/src/persistence/eri_cache.rs index fdae3cf..771018c 100644 --- a/crates/rustiq-core/src/persistence/eri_cache.rs +++ b/crates/rustiq-core/src/persistence/eri_cache.rs @@ -14,8 +14,7 @@ use crate::{ use super::{ ao_eri_identity, sha256_reader, validate_compact_eri_header, AoEriAttributes, ArtifactAttributes, ArtifactManifest, Manifest, ManifestKind, RustiQData, ScientificIdentity, - Storage, - AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, + Storage, AO_ERI_COMPUTATION_VERSION, AO_ERI_PATH, COMPACT_ERI_REPRESENTATION, FORMAT_NAME, FORMAT_VERSION, SCIENTIFIC_IDENTITY_VERSION, }; @@ -509,7 +508,7 @@ mod tests { } #[test] - fn checkpoint_is_never_accepted_as_a_cache_hit() { + fn unknown_kind_is_never_accepted_as_a_cache_hit() { let temporary = tempfile::tempdir().unwrap(); let cache = EriCache::new(temporary.path()); let (molecule, basis) = input(); @@ -520,7 +519,8 @@ mod tests { let manifest_path = cache.entry_path(identity).join(MANIFEST_PATH); let mut manifest: serde_json::Value = serde_json::from_reader(File::open(&manifest_path).unwrap()).unwrap(); - manifest["kind"] = serde_json::json!(ManifestKind::Checkpoint.as_str()); + manifest["kind"] = + serde_json::to_value(ManifestKind::Unknown("future-state".to_owned())).unwrap(); fs::write(&manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap(); assert!(cache.load(&molecule, &basis, threshold()).is_none());