From 65a6c894e4fac057eba16084406a91ba9b2166a8 Mon Sep 17 00:00:00 2001 From: Connor Stone Date: Sun, 8 Nov 2026 15:14:22 -0500 Subject: [PATCH] append_state(batch=k): append a batch of states in one write per parameter After set_values with a batched array of k parameter vectors, every parameter value carries a leading axis of k states; append_state(path, batch=k) now appends them all at once (one resize and one write per parameter instead of one file open per state), repeating a parameter whose value has no state axis (static, or a pointer to static ones) for every state. A value that fits neither shape raises. The single-state path reads the value once (it was evaluated twice per parameter). Co-Authored-By: Claude Fable 5.1 --- src/caskade/base.py | 18 +++++++++++++----- src/caskade/param.py | 23 +++++++++++++++++++---- tests/test_save.py | 38 +++++++++++++++++++++++++++++++++++++- 3 files changed, 69 insertions(+), 10 deletions(-) diff --git a/src/caskade/base.py b/src/caskade/base.py index f398049..59d4c9b 100644 --- a/src/caskade/base.py +++ b/src/caskade/base.py @@ -522,12 +522,12 @@ def _append_state_cleanup(self): """Cleanup the state of the node and its children after appending to HDF5.""" pass - def _append_state_hdf5(self, h5group): + def _append_state_hdf5(self, h5group, batch: Optional[int] = None): """Append the state of the node and its children to an existing HDF5 file.""" for key, child in self.children.items(): - child._append_state_hdf5(h5group[key]) + child._append_state_hdf5(h5group[key], batch=batch) - def append_state(self, saveto: Union[str, "File"]): + def append_state(self, saveto: Union[str, "File"], batch: Optional[int] = None): """ Append the current state to an existing HDF5 file. @@ -540,6 +540,14 @@ def append_state(self, saveto: Union[str, "File"]): saveto : str or File Path to an HDF5 file (``'.h5'`` or ``'.hdf5'``) or an open HDF5 ``File`` object. + batch : int or None, optional + Append ``batch`` states at once: the parameter values carry a + leading axis of that length (e.g. after ``set_values`` with a + batched array of ``batch`` parameter vectors). A parameter whose + value has no such axis (a static parameter, or a pointer to + static ones) is repeated for every state. One file open and one + write per parameter instead of one per state. Default ``None`` + appends the single current state. Raises ------ @@ -552,7 +560,7 @@ def append_state(self, saveto: Union[str, "File"]): if saveto.endswith(".h5") or saveto.endswith(".hdf5"): with h5py.File(saveto, "a") as h5file: self._check_append_state_hdf5(h5file[self.name]) - self._append_state_hdf5(h5file[self.name]) + self._append_state_hdf5(h5file[self.name], batch=batch) for node in self.topological_ordering(): node._append_state_cleanup() @@ -562,7 +570,7 @@ def append_state(self, saveto: Union[str, "File"]): ) else: # assume saveto is an HDF5 File object self._check_append_state_hdf5(saveto[self.name]) - self._append_state_hdf5(saveto[self.name]) + self._append_state_hdf5(saveto[self.name], batch=batch) for node in self.topological_ordering(): node._append_state_cleanup() diff --git a/src/caskade/param.py b/src/caskade/param.py index 5ff9bca..37a0f43 100644 --- a/src/caskade/param.py +++ b/src/caskade/param.py @@ -675,8 +675,8 @@ def _append_state_cleanup(self): super()._append_state_cleanup() del self.appended - def _append_state_hdf5(self, h5group): - super()._append_state_hdf5(h5group) + def _append_state_hdf5(self, h5group, batch: Optional[int] = None): + super()._append_state_hdf5(h5group, batch=batch) if not hasattr(self, "appended"): self.appended = True try: @@ -684,8 +684,23 @@ def _append_state_hdf5(self, h5group): except: value = None if value is not None: - h5group["value"].resize((h5group["value"].shape[0] + 1,) + h5group["value"].shape[1:]) - h5group["value"][-1] = self.value + value = backend.to_numpy(value) + dataset = h5group["value"] + state_shape = dataset.shape[1:] + if batch is None: + rows = value.reshape((1,) + state_shape) + elif value.shape == (batch,) + state_shape: + rows = value + elif value.shape == state_shape: # no state axis: one value for every state + rows = np.broadcast_to(value, (batch,) + state_shape) + else: + raise ValueError( + f"Param {self.name}: value of shape {value.shape} is neither one state of shape " + f"{state_shape} nor a batch of {batch} states of that shape" + ) + n = dataset.shape[0] + dataset.resize((n + rows.shape[0],) + state_shape) + dataset[n:] = rows def _load_state_hdf5(self, h5group, index: int = -1, _done_load: set = None): super()._load_state_hdf5(h5group, index=index, _done_load=_done_load) diff --git a/tests/test_save.py b/tests/test_save.py index 9321357..3b7e6fc 100644 --- a/tests/test_save.py +++ b/tests/test_save.py @@ -1,4 +1,5 @@ -from caskade import Module, Param, GraphError, SaveStateWarning +from caskade import Module, Param, GraphError, SaveStateWarning, backend +import os import numpy as np import gc import h5py @@ -177,6 +178,41 @@ def test_save_append_load_batched_param(): np.testing.assert_array_equal(loaded_module.batched.value, np.arange(12, 24).reshape(2, 2, 3)) +def test_append_state_batch(): + """append_state(batch=k) writes k states at once from values with a + leading state axis (set_values with a batched array); a static parameter + is repeated for every state; the result loads like k single appends.""" + module = Module("main") + module.dyn = Param("dyn", np.zeros((2, 3)), shape=(2, 3), dynamic=True) + module.scalar = Param("scalar", 1.0, shape=(), dynamic=True) + module.fixed = Param("fixed", np.arange(4.0)) + path = "test_save_append_batch.h5" + module.save_state(path, appendable=True) + + states = np.random.default_rng(0).normal(size=(5, 7)) # 5 parameter vectors of dyn (6) + scalar (1) + module.set_values(backend.as_array(states)) + module.append_state(path, batch=5) + with h5py.File(path, "r") as h5file: + assert h5file["main/dyn/value"].shape == (6, 2, 3) + np.testing.assert_allclose(h5file["main/dyn/value"][1:], states[:, :6].reshape(5, 2, 3), rtol=1e-6) + np.testing.assert_allclose(h5file["main/scalar/value"][1:], states[:, 6], rtol=1e-6) + np.testing.assert_array_equal(h5file["main/fixed/value"][()], np.tile(np.arange(4.0), (6, 1))) + + loaded = Module("main") + loaded.dyn = Param("dyn", np.zeros((2, 3)), shape=(2, 3), dynamic=True) + loaded.scalar = Param("scalar", 0.0, shape=(), dynamic=True) + loaded.fixed = Param("fixed", np.zeros(4)) + loaded.load_state(path, index=3) + np.testing.assert_allclose(backend.to_numpy(loaded.dyn.value), states[2, :6].reshape(2, 3)) + np.testing.assert_allclose(backend.to_numpy(loaded.scalar.value), states[2, 6]) + + # a value that is neither one state nor a batch of the stated size is an error + module.set_values(backend.as_array(states[:3])) + with pytest.raises(ValueError): + module.append_state(path, batch=5) + os.remove(path) + + def test_missing_h5py(): with mock.patch.dict(sys.modules, {"h5py": None}): import importlib