Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 19 additions & 5 deletions src/caskade/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,9 @@ def to(self, device=None, dtype=None):

return self

def _save_state_hdf5(self, h5group, appendable: bool = False, _done_save: set = None):
def _save_state_hdf5(
self, h5group, appendable: bool = False, _done_save: set = None, compression="gzip"
):
"""Save the state of the node and its children to HDF5."""
if id(self) not in _done_save:
for attr in self.saveattrs:
Expand All @@ -441,9 +443,11 @@ def _save_state_hdf5(self, h5group, appendable: bool = False, _done_save: set =
child._h5group = h5group.create_group(key)
elif key not in h5group:
h5group[key] = child._h5group
child._save_state_hdf5(h5group[key], appendable=appendable, _done_save=_done_save)
child._save_state_hdf5(
h5group[key], appendable=appendable, _done_save=_done_save, compression=compression
)

def save_state(self, saveto: Union[str, "File"], appendable: bool = False):
def save_state(self, saveto: Union[str, "File"], appendable: bool = False, compression="gzip"):
"""
Save the state of the node and its children, currently only works for
HDF5 file types (.h5 and .hdf5).
Expand Down Expand Up @@ -485,14 +489,22 @@ def save_state(self, saveto: Union[str, "File"], appendable: bool = False):
Whether to save the state in an appendable format. If True, the
values will have an extra dimension for the number of samples.
Defaults to False.
compression: (str or None, optional)
The h5py compression filter for the appendable value datasets
(``"gzip"`` by default). ``None`` writes them uncompressed, which
makes each append cheaper when the file is written often (e.g. a
sampler saving every few steps). Ignored when appendable is False.

"""
if isinstance(saveto, str):
if saveto.endswith(".h5") or saveto.endswith(".hdf5"):
with h5py.File(saveto, "w") as h5file:
self._h5group = h5file.create_group(self.name)
self._save_state_hdf5(
h5file[self.name], appendable=appendable, _done_save=set()
h5file[self.name],
appendable=appendable,
_done_save=set(),
compression=compression,
)

for node in self.topological_ordering():
Expand All @@ -503,7 +515,9 @@ def save_state(self, saveto: Union[str, "File"], appendable: bool = False):
)
else: # assume saveto is an HDF5 File object
self._h5group = saveto.create_group(self.name)
self._save_state_hdf5(saveto[self.name], appendable=appendable, _done_save=set())
self._save_state_hdf5(
saveto[self.name], appendable=appendable, _done_save=set(), compression=compression
)

for node in self.topological_ordering():
del node._h5group
Expand Down
10 changes: 7 additions & 3 deletions src/caskade/param.py
Original file line number Diff line number Diff line change
Expand Up @@ -628,8 +628,12 @@ def cyclic(self, cyclic: bool):
self._cyclic = cyclic
self.valid = self.valid

def _save_state_hdf5(self, h5group, appendable: bool = False, _done_save: set = None):
super()._save_state_hdf5(h5group, appendable=appendable, _done_save=_done_save)
def _save_state_hdf5(
self, h5group, appendable: bool = False, _done_save: set = None, compression="gzip"
):
super()._save_state_hdf5(
h5group, appendable=appendable, _done_save=_done_save, compression=compression
)
if "value" not in self._h5group:
try:
value = self.value
Expand All @@ -648,7 +652,7 @@ def _save_state_hdf5(self, h5group, appendable: bool = False, _done_save: set =
data=value,
chunks=False if isinstance(value, str) else True,
maxshape=None if isinstance(value, str) else (None,) + tuple(value.shape[1:]),
compression=None if isinstance(value, str) else "gzip",
compression=None if isinstance(value, str) else compression,
)
else:
self._h5group.create_dataset(
Expand Down
17 changes: 17 additions & 0 deletions tests/test_save.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,22 @@ def test_save_append_load_batched_param():
np.testing.assert_array_equal(loaded_module.batched.value, np.arange(12, 24).reshape(2, 2, 3))


def test_save_append_uncompressed():
main = _build_test_module()
main.save_state("test_save_uncompressed.h5", appendable=True, compression=None)
main.m1.p1.value = 2.0
main.append_state("test_save_uncompressed.h5")
with h5py.File("test_save_uncompressed.h5", "r") as f:
assert f["main/m1/p1/value"].compression is None
assert f["main/m1/p1/value"].shape == (2,)
assert np.allclose(f["main/m1/p1/value"][()], [1.0, 2.0])
main.save_state("test_save_compressed.h5", appendable=True)
with h5py.File("test_save_compressed.h5", "r") as f:
assert f["main/m1/p1/value"].compression == "gzip"
main.load_state("test_save_uncompressed.h5", index=0)
assert np.allclose(main.m1.p1.value, 1.0)


def test_missing_h5py():
with mock.patch.dict(sys.modules, {"h5py": None}):
import importlib
Expand All @@ -186,3 +202,4 @@ def test_missing_h5py():
importlib.reload(module)

assert module.h5py is None

Loading