Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions src/caskade/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -522,12 +522,12 @@ def _append_state_cleanup(self):
"""Cleanup the state of the node and its children after appending to HDF5."""
pass

def _append_state_hdf5(self, h5group):
def _append_state_hdf5(self, h5group, batch: Optional[int] = None):
"""Append the state of the node and its children to an existing HDF5 file."""
for key, child in self.children.items():
child._append_state_hdf5(h5group[key])
child._append_state_hdf5(h5group[key], batch=batch)

def append_state(self, saveto: Union[str, "File"]):
def append_state(self, saveto: Union[str, "File"], batch: Optional[int] = None):
"""
Append the current state to an existing HDF5 file.

Expand All @@ -540,6 +540,14 @@ def append_state(self, saveto: Union[str, "File"]):
saveto : str or File
Path to an HDF5 file (``'.h5'`` or ``'.hdf5'``) or an open
HDF5 ``File`` object.
batch : int or None, optional
Append ``batch`` states at once: the parameter values carry a
leading axis of that length (e.g. after ``set_values`` with a
batched array of ``batch`` parameter vectors). A parameter whose
value has no such axis (a static parameter, or a pointer to
static ones) is repeated for every state. One file open and one
write per parameter instead of one per state. Default ``None``
appends the single current state.

Raises
------
Expand All @@ -552,7 +560,7 @@ def append_state(self, saveto: Union[str, "File"]):
if saveto.endswith(".h5") or saveto.endswith(".hdf5"):
with h5py.File(saveto, "a") as h5file:
self._check_append_state_hdf5(h5file[self.name])
self._append_state_hdf5(h5file[self.name])
self._append_state_hdf5(h5file[self.name], batch=batch)

for node in self.topological_ordering():
node._append_state_cleanup()
Expand All @@ -562,7 +570,7 @@ def append_state(self, saveto: Union[str, "File"]):
)
else: # assume saveto is an HDF5 File object
self._check_append_state_hdf5(saveto[self.name])
self._append_state_hdf5(saveto[self.name])
self._append_state_hdf5(saveto[self.name], batch=batch)

for node in self.topological_ordering():
node._append_state_cleanup()
Expand Down
23 changes: 19 additions & 4 deletions src/caskade/param.py
Original file line number Diff line number Diff line change
Expand Up @@ -675,17 +675,32 @@ def _append_state_cleanup(self):
super()._append_state_cleanup()
del self.appended

def _append_state_hdf5(self, h5group):
super()._append_state_hdf5(h5group)
def _append_state_hdf5(self, h5group, batch: Optional[int] = None):
super()._append_state_hdf5(h5group, batch=batch)
if not hasattr(self, "appended"):
self.appended = True
try:
value = self.value
except:
value = None
if value is not None:
h5group["value"].resize((h5group["value"].shape[0] + 1,) + h5group["value"].shape[1:])
h5group["value"][-1] = self.value
value = backend.to_numpy(value)
dataset = h5group["value"]
state_shape = dataset.shape[1:]
if batch is None:
rows = value.reshape((1,) + state_shape)
elif value.shape == (batch,) + state_shape:
rows = value
elif value.shape == state_shape: # no state axis: one value for every state
rows = np.broadcast_to(value, (batch,) + state_shape)
else:
raise ValueError(
f"Param {self.name}: value of shape {value.shape} is neither one state of shape "
f"{state_shape} nor a batch of {batch} states of that shape"
)
Comment on lines +697 to +700
n = dataset.shape[0]
dataset.resize((n + rows.shape[0],) + state_shape)
dataset[n:] = rows
Comment on lines +702 to +703

def _load_state_hdf5(self, h5group, index: int = -1, _done_load: set = None):
super()._load_state_hdf5(h5group, index=index, _done_load=_done_load)
Expand Down
38 changes: 37 additions & 1 deletion tests/test_save.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from caskade import Module, Param, GraphError, SaveStateWarning
from caskade import Module, Param, GraphError, SaveStateWarning, backend
import os
import numpy as np
import gc
import h5py
Expand Down Expand Up @@ -177,6 +178,41 @@ def test_save_append_load_batched_param():
np.testing.assert_array_equal(loaded_module.batched.value, np.arange(12, 24).reshape(2, 2, 3))


def test_append_state_batch():
"""append_state(batch=k) writes k states at once from values with a
leading state axis (set_values with a batched array); a static parameter
is repeated for every state; the result loads like k single appends."""
module = Module("main")
module.dyn = Param("dyn", np.zeros((2, 3)), shape=(2, 3), dynamic=True)
module.scalar = Param("scalar", 1.0, shape=(), dynamic=True)
module.fixed = Param("fixed", np.arange(4.0))
path = "test_save_append_batch.h5"
module.save_state(path, appendable=True)

states = np.random.default_rng(0).normal(size=(5, 7)) # 5 parameter vectors of dyn (6) + scalar (1)
module.set_values(backend.as_array(states))
module.append_state(path, batch=5)
with h5py.File(path, "r") as h5file:
assert h5file["main/dyn/value"].shape == (6, 2, 3)
np.testing.assert_allclose(h5file["main/dyn/value"][1:], states[:, :6].reshape(5, 2, 3), rtol=1e-6)
np.testing.assert_allclose(h5file["main/scalar/value"][1:], states[:, 6], rtol=1e-6)
np.testing.assert_array_equal(h5file["main/fixed/value"][()], np.tile(np.arange(4.0), (6, 1)))

loaded = Module("main")
loaded.dyn = Param("dyn", np.zeros((2, 3)), shape=(2, 3), dynamic=True)
loaded.scalar = Param("scalar", 0.0, shape=(), dynamic=True)
loaded.fixed = Param("fixed", np.zeros(4))
loaded.load_state(path, index=3)
np.testing.assert_allclose(backend.to_numpy(loaded.dyn.value), states[2, :6].reshape(2, 3))
np.testing.assert_allclose(backend.to_numpy(loaded.scalar.value), states[2, 6])

# a value that is neither one state nor a batch of the stated size is an error
module.set_values(backend.as_array(states[:3]))
with pytest.raises(ValueError):
module.append_state(path, batch=5)
os.remove(path)


def test_missing_h5py():
with mock.patch.dict(sys.modules, {"h5py": None}):
import importlib
Expand Down
Loading