Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion src/datasets/arrow_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -584,7 +584,16 @@ def __enter__(self):
return self

def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
# Leaving the block must produce a complete, readable file. `close()` only closes the
# stream; it is `finalize()` that flushes the examples still staged in the buffer, so
# `with ArrowWriter(...) as writer:` used to discard everything below `writer_batch_size`
# and leave a 0-byte file. Stay quiet, as before, when the block raised, when nothing was
# written (there is no schema to infer), or when the caller already finalized.
has_staged_data = bool(self.current_examples or self.current_rows) or self.pa_writer is not None
if exc_type is None and not getattr(self.stream, "closed", False) and has_staged_data:
self.finalize()
else:
self.close()

def close(self):
# Try closing if opened; if closed: pyarrow.lib.ArrowInvalid: Invalid operation on closed file
Expand Down
15 changes: 15 additions & 0 deletions tests/test_arrow_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,21 @@ def test_arrow_writer_closes_stream(raise_exception, tmp_path):
assert writer.stream.closed


def test_arrow_writer_context_manager_writes_staged_examples(tmp_path):
# Leaving a `with ArrowWriter(...)` block must produce a complete file, including the examples
# still staged in the buffer below `writer_batch_size`. `__exit__` only called `close()`, which
# closes the stream without flushing the buffer, so the file stayed at 0 bytes and unreadable.
path = str(tmp_path / "dataset-train.arrow")
with ArrowWriter(path=path) as writer:
for i in range(3):
writer.write({"a": i})
assert writer.stream.closed
assert os.path.getsize(path) > 0
table = pa.ipc.open_stream(path).read_all()
assert table.num_rows == 3
assert table.column("a").to_pylist() == [0, 1, 2]


def test_arrow_writer_with_filesystem(mockfs):
path = "mock://dataset-train.arrow"
with ArrowWriter(path=path, storage_options=mockfs.storage_options) as writer:
Expand Down