From e636c59774826cac98109e1d8a7903c77f839497 Mon Sep 17 00:00:00 2001 From: Connor Stone Date: Mon, 9 Nov 2026 15:52:15 -0500 Subject: [PATCH] save_state takes a compression option for appendable files Appendable value datasets were always gzip compressed. A sampler that appends a state every few steps pays the filter on every append, and float samples compress poorly, so save_state(..., compression=None) now writes them uncompressed. The default stays "gzip". Co-Authored-By: Claude Fable 5.1 --- src/caskade/base.py | 24 +++++++++++++++++++----- src/caskade/param.py | 10 +++++++--- tests/test_save.py | 17 +++++++++++++++++ 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/caskade/base.py b/src/caskade/base.py index f398049..52b917c 100644 --- a/src/caskade/base.py +++ b/src/caskade/base.py @@ -418,7 +418,9 @@ def to(self, device=None, dtype=None): return self - def _save_state_hdf5(self, h5group, appendable: bool = False, _done_save: set = None): + def _save_state_hdf5( + self, h5group, appendable: bool = False, _done_save: set = None, compression="gzip" + ): """Save the state of the node and its children to HDF5.""" if id(self) not in _done_save: for attr in self.saveattrs: @@ -441,9 +443,11 @@ def _save_state_hdf5(self, h5group, appendable: bool = False, _done_save: set = child._h5group = h5group.create_group(key) elif key not in h5group: h5group[key] = child._h5group - child._save_state_hdf5(h5group[key], appendable=appendable, _done_save=_done_save) + child._save_state_hdf5( + h5group[key], appendable=appendable, _done_save=_done_save, compression=compression + ) - def save_state(self, saveto: Union[str, "File"], appendable: bool = False): + def save_state(self, saveto: Union[str, "File"], appendable: bool = False, compression="gzip"): """ Save the state of the node and its children, currently only works for HDF5 file types (.h5 and .hdf5). @@ -485,6 +489,11 @@ def save_state(self, saveto: Union[str, "File"], appendable: bool = False): Whether to save the state in an appendable format. If True, the values will have an extra dimension for the number of samples. Defaults to False. + compression: (str or None, optional) + The h5py compression filter for the appendable value datasets + (``"gzip"`` by default). ``None`` writes them uncompressed, which + makes each append cheaper when the file is written often (e.g. a + sampler saving every few steps). Ignored when appendable is False. """ if isinstance(saveto, str): @@ -492,7 +501,10 @@ def save_state(self, saveto: Union[str, "File"], appendable: bool = False): with h5py.File(saveto, "w") as h5file: self._h5group = h5file.create_group(self.name) self._save_state_hdf5( - h5file[self.name], appendable=appendable, _done_save=set() + h5file[self.name], + appendable=appendable, + _done_save=set(), + compression=compression, ) for node in self.topological_ordering(): @@ -503,7 +515,9 @@ def save_state(self, saveto: Union[str, "File"], appendable: bool = False): ) else: # assume saveto is an HDF5 File object self._h5group = saveto.create_group(self.name) - self._save_state_hdf5(saveto[self.name], appendable=appendable, _done_save=set()) + self._save_state_hdf5( + saveto[self.name], appendable=appendable, _done_save=set(), compression=compression + ) for node in self.topological_ordering(): del node._h5group diff --git a/src/caskade/param.py b/src/caskade/param.py index 5ff9bca..f011401 100644 --- a/src/caskade/param.py +++ b/src/caskade/param.py @@ -628,8 +628,12 @@ def cyclic(self, cyclic: bool): self._cyclic = cyclic self.valid = self.valid - def _save_state_hdf5(self, h5group, appendable: bool = False, _done_save: set = None): - super()._save_state_hdf5(h5group, appendable=appendable, _done_save=_done_save) + def _save_state_hdf5( + self, h5group, appendable: bool = False, _done_save: set = None, compression="gzip" + ): + super()._save_state_hdf5( + h5group, appendable=appendable, _done_save=_done_save, compression=compression + ) if "value" not in self._h5group: try: value = self.value @@ -648,7 +652,7 @@ def _save_state_hdf5(self, h5group, appendable: bool = False, _done_save: set = data=value, chunks=False if isinstance(value, str) else True, maxshape=None if isinstance(value, str) else (None,) + tuple(value.shape[1:]), - compression=None if isinstance(value, str) else "gzip", + compression=None if isinstance(value, str) else compression, ) else: self._h5group.create_dataset( diff --git a/tests/test_save.py b/tests/test_save.py index 9321357..1e3c11f 100644 --- a/tests/test_save.py +++ b/tests/test_save.py @@ -177,6 +177,22 @@ def test_save_append_load_batched_param(): np.testing.assert_array_equal(loaded_module.batched.value, np.arange(12, 24).reshape(2, 2, 3)) +def test_save_append_uncompressed(): + main = _build_test_module() + main.save_state("test_save_uncompressed.h5", appendable=True, compression=None) + main.m1.p1.value = 2.0 + main.append_state("test_save_uncompressed.h5") + with h5py.File("test_save_uncompressed.h5", "r") as f: + assert f["main/m1/p1/value"].compression is None + assert f["main/m1/p1/value"].shape == (2,) + assert np.allclose(f["main/m1/p1/value"][()], [1.0, 2.0]) + main.save_state("test_save_compressed.h5", appendable=True) + with h5py.File("test_save_compressed.h5", "r") as f: + assert f["main/m1/p1/value"].compression == "gzip" + main.load_state("test_save_uncompressed.h5", index=0) + assert np.allclose(main.m1.p1.value, 1.0) + + def test_missing_h5py(): with mock.patch.dict(sys.modules, {"h5py": None}): import importlib @@ -186,3 +202,4 @@ def test_missing_h5py(): importlib.reload(module) assert module.h5py is None +