Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/source/api/atomic.rst
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ PolyStore exports cross-platform lock and JSON update primitives:
Exclusive advisory lock context manager.

``atomic_write_json(path, data, ...)``
Write JSON through a temporary file and atomic replacement.
Write compact JSON through a temporary file and atomic replacement. Pass
``indent=2`` (or another explicit indentation) for human-readable output.
Whitespace is a presentation choice; readers consume the parsed JSON document.

``atomic_update_json(path, update_func, ...)``
Lock, read, transform, and replace a JSON document.
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "polystore"
version = "0.3.3"
version = "0.3.4"
description = "Framework-agnostic multi-backend storage abstraction for ML and scientific computing"
readme = "README.md"
requires-python = ">=3.11"
Expand Down
44 changes: 28 additions & 16 deletions src/polystore/atomic.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ class LockConfig:
DEFAULT_POLL_INTERVAL: float = 0.1
LOCK_SUFFIX: str = ".lock"
TEMP_PREFIX: str = ".tmp"
JSON_INDENT: int = 2
JSON_INDENT: int | None = None

def lock_path(self, file_path: str | Path) -> Path:
"""Derive the lock owned by one atomic file transaction."""
Expand Down Expand Up @@ -118,15 +118,16 @@ def _cleanup_lock(lock_fd: int, lock_path: Path) -> None:
def atomic_write_json(
file_path: str | Path,
data: dict[str, Any],
indent: int = LOCK_CONFIG.JSON_INDENT,
indent: int | None = LOCK_CONFIG.JSON_INDENT,
ensure_directory: bool = True,
) -> None:
"""Atomically write JSON data to file using temporary file + rename."""
"""Atomically write compact JSON, or explicit indentation, through one encoding."""
file_path = Path(file_path)

if ensure_directory:
file_path.parent.mkdir(parents=True, exist_ok=True)

tmp_path = None
try:
tmp_path = _write_to_temp_file(file_path, data, indent)
# Use os.replace() instead of os.rename() for atomic replacement on all platforms
Expand All @@ -135,21 +136,32 @@ def atomic_write_json(
logger.debug(f"Atomically wrote JSON to {file_path}")
except Exception as e:
raise FileLockError(f"Atomic JSON write failed for {file_path}: {e}") from e
finally:
if tmp_path is not None:
Path(tmp_path).unlink(missing_ok=True)


def _write_to_temp_file(file_path: Path, data: dict[str, Any], indent: int) -> str:
"""Write data to temporary file and return path."""
with tempfile.NamedTemporaryFile(
mode="w",
dir=file_path.parent,
prefix=f"{LOCK_CONFIG.TEMP_PREFIX}{file_path.name}",
suffix=".json",
delete=False,
) as tmp_file:
json.dump(data, tmp_file, indent=indent)
tmp_file.flush()
os.fsync(tmp_file.fileno())
return tmp_file.name
def _write_to_temp_file(file_path: Path, data: dict[str, Any], indent: int | None) -> str:
"""Encode once, then durably write the owned temporary JSON file."""
encoded = json.dumps(data, indent=indent)
tmp_path = None
try:
with tempfile.NamedTemporaryFile(
mode="w",
dir=file_path.parent,
prefix=f"{LOCK_CONFIG.TEMP_PREFIX}{file_path.name}",
suffix=".json",
delete=False,
) as tmp_file:
tmp_path = tmp_file.name
tmp_file.write(encoded)
tmp_file.flush()
os.fsync(tmp_file.fileno())
return tmp_path
except BaseException:
if tmp_path is not None:
Path(tmp_path).unlink(missing_ok=True)
raise


def atomic_update_json(
Expand Down
62 changes: 62 additions & 0 deletions tests/test_atomic_lock.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,65 @@ def reject_lock(_lock_path):
pytest.fail("unavailable lock must not be entered")

assert attempts == 1


@pytest.mark.parametrize("indent", (None, 2))
def test_atomic_json_preserves_parsed_values_and_explicit_presentation(tmp_path, indent):
import json

from polystore.atomic import atomic_write_json

path = tmp_path / "metadata.json"
document = {"nested": [None, True, 2**60, 1.25, {"label": 'λ\nquoted "value"'}]}
atomic_write_json(path, document, indent=indent)

assert json.loads(path.read_text()) == document
assert path.read_text() == json.dumps(document, indent=indent)
assert not tuple(tmp_path.glob(".tmp*"))


def test_atomic_update_preserves_transform_and_releases_lock(tmp_path):
import json

from polystore.atomic import atomic_update_json, atomic_write_json

path = tmp_path / "metadata.json"
atomic_write_json(path, {"revision": 1, "nested": {"unchanged": True}})

def update(document):
document["revision"] += 1
return document

atomic_update_json(path, update)
assert json.loads(path.read_text()) == {"revision": 2, "nested": {"unchanged": True}}
with file_lock(LockConfig().lock_path(path), timeout=0):
pass


@pytest.mark.parametrize("failure", ("encoding", "fsync", "replace", "update"))
def test_atomic_json_failure_preserves_original_and_removes_temporary(
tmp_path, monkeypatch, failure
):
from polystore import atomic

path = tmp_path / "metadata.json"
original = '{"revision": 1}\n'
path.write_text(original)

def fail(*_args):
raise OSError("injected failure")

if failure in {"fsync", "replace"}:
monkeypatch.setattr(atomic.os, failure, fail)
if failure == "update":
with pytest.raises(atomic.FileLockError, match="Update function failed"):
atomic.atomic_update_json(path, fail)
else:
data = {"unsupported": object()} if failure == "encoding" else {"revision": 2}
with pytest.raises(atomic.FileLockError, match="Atomic JSON write failed"):
atomic.atomic_write_json(path, data)

assert path.read_text() == original
assert not tuple(tmp_path.glob(".tmp*"))
with file_lock(LockConfig().lock_path(path), timeout=0):
pass
Loading