From 6ac58488403e5d80bc50b400a7bd13c57bafb154 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 8 Oct 2026 09:12:22 -0400 Subject: [PATCH 1/2] Encode atomic JSON once and preserve failure cleanup Closes #30. Default to compact parsed-equivalent metadata while retaining explicit indentation, locks, fsync and atomic replacement. --- docs/source/api/atomic.rst | 4 ++- src/polystore/atomic.py | 44 +++++++++++++++++---------- tests/test_atomic_lock.py | 62 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 17 deletions(-) diff --git a/docs/source/api/atomic.rst b/docs/source/api/atomic.rst index 647db5f..72696d4 100644 --- a/docs/source/api/atomic.rst +++ b/docs/source/api/atomic.rst @@ -7,7 +7,9 @@ PolyStore exports cross-platform lock and JSON update primitives: Exclusive advisory lock context manager. ``atomic_write_json(path, data, ...)`` - Write JSON through a temporary file and atomic replacement. + Write compact JSON through a temporary file and atomic replacement. Pass + ``indent=2`` (or another explicit indentation) for human-readable output. + Whitespace is a presentation choice; readers consume the parsed JSON document. ``atomic_update_json(path, update_func, ...)`` Lock, read, transform, and replace a JSON document. diff --git a/src/polystore/atomic.py b/src/polystore/atomic.py index 0771f12..a0c06c8 100644 --- a/src/polystore/atomic.py +++ b/src/polystore/atomic.py @@ -31,7 +31,7 @@ class LockConfig: DEFAULT_POLL_INTERVAL: float = 0.1 LOCK_SUFFIX: str = ".lock" TEMP_PREFIX: str = ".tmp" - JSON_INDENT: int = 2 + JSON_INDENT: int | None = None def lock_path(self, file_path: str | Path) -> Path: """Derive the lock owned by one atomic file transaction.""" @@ -118,15 +118,16 @@ def _cleanup_lock(lock_fd: int, lock_path: Path) -> None: def atomic_write_json( file_path: str | Path, data: dict[str, Any], - indent: int = LOCK_CONFIG.JSON_INDENT, + indent: int | None = LOCK_CONFIG.JSON_INDENT, ensure_directory: bool = True, ) -> None: - """Atomically write JSON data to file using temporary file + rename.""" + """Atomically write compact JSON, or explicit indentation, through one encoding.""" file_path = Path(file_path) if ensure_directory: file_path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = None try: tmp_path = _write_to_temp_file(file_path, data, indent) # Use os.replace() instead of os.rename() for atomic replacement on all platforms @@ -135,21 +136,32 @@ def atomic_write_json( logger.debug(f"Atomically wrote JSON to {file_path}") except Exception as e: raise FileLockError(f"Atomic JSON write failed for {file_path}: {e}") from e + finally: + if tmp_path is not None: + Path(tmp_path).unlink(missing_ok=True) -def _write_to_temp_file(file_path: Path, data: dict[str, Any], indent: int) -> str: - """Write data to temporary file and return path.""" - with tempfile.NamedTemporaryFile( - mode="w", - dir=file_path.parent, - prefix=f"{LOCK_CONFIG.TEMP_PREFIX}{file_path.name}", - suffix=".json", - delete=False, - ) as tmp_file: - json.dump(data, tmp_file, indent=indent) - tmp_file.flush() - os.fsync(tmp_file.fileno()) - return tmp_file.name +def _write_to_temp_file(file_path: Path, data: dict[str, Any], indent: int | None) -> str: + """Encode once, then durably write the owned temporary JSON file.""" + encoded = json.dumps(data, indent=indent) + tmp_path = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + dir=file_path.parent, + prefix=f"{LOCK_CONFIG.TEMP_PREFIX}{file_path.name}", + suffix=".json", + delete=False, + ) as tmp_file: + tmp_path = tmp_file.name + tmp_file.write(encoded) + tmp_file.flush() + os.fsync(tmp_file.fileno()) + return tmp_path + except BaseException: + if tmp_path is not None: + Path(tmp_path).unlink(missing_ok=True) + raise def atomic_update_json( diff --git a/tests/test_atomic_lock.py b/tests/test_atomic_lock.py index 82b3120..9f35143 100644 --- a/tests/test_atomic_lock.py +++ b/tests/test_atomic_lock.py @@ -46,3 +46,65 @@ def reject_lock(_lock_path): pytest.fail("unavailable lock must not be entered") assert attempts == 1 + + +@pytest.mark.parametrize("indent", (None, 2)) +def test_atomic_json_preserves_parsed_values_and_explicit_presentation(tmp_path, indent): + import json + + from polystore.atomic import atomic_write_json + + path = tmp_path / "metadata.json" + document = {"nested": [None, True, 2**60, 1.25, {"label": 'λ\nquoted "value"'}]} + atomic_write_json(path, document, indent=indent) + + assert json.loads(path.read_text()) == document + assert path.read_text() == json.dumps(document, indent=indent) + assert not tuple(tmp_path.glob(".tmp*")) + + +def test_atomic_update_preserves_transform_and_releases_lock(tmp_path): + import json + + from polystore.atomic import atomic_update_json, atomic_write_json + + path = tmp_path / "metadata.json" + atomic_write_json(path, {"revision": 1, "nested": {"unchanged": True}}) + + def update(document): + document["revision"] += 1 + return document + + atomic_update_json(path, update) + assert json.loads(path.read_text()) == {"revision": 2, "nested": {"unchanged": True}} + with file_lock(LockConfig().lock_path(path), timeout=0): + pass + + +@pytest.mark.parametrize("failure", ("encoding", "fsync", "replace", "update")) +def test_atomic_json_failure_preserves_original_and_removes_temporary( + tmp_path, monkeypatch, failure +): + from polystore import atomic + + path = tmp_path / "metadata.json" + original = '{"revision": 1}\n' + path.write_text(original) + + def fail(*_args): + raise OSError("injected failure") + + if failure in {"fsync", "replace"}: + monkeypatch.setattr(atomic.os, failure, fail) + if failure == "update": + with pytest.raises(atomic.FileLockError, match="Update function failed"): + atomic.atomic_update_json(path, fail) + else: + data = {"unsupported": object()} if failure == "encoding" else {"revision": 2} + with pytest.raises(atomic.FileLockError, match="Atomic JSON write failed"): + atomic.atomic_write_json(path, data) + + assert path.read_text() == original + assert not tuple(tmp_path.glob(".tmp*")) + with file_lock(LockConfig().lock_path(path), timeout=0): + pass From 597daa2ec96d1a8ea248b95a6caff6404abd3e7a Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Thu, 8 Oct 2026 09:13:41 -0400 Subject: [PATCH 2/2] Release atomic JSON encoding fix as 0.3.4 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b1219c6..d06a3c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "polystore" -version = "0.3.3" +version = "0.3.4" description = "Framework-agnostic multi-backend storage abstraction for ML and scientific computing" readme = "README.md" requires-python = ">=3.11"