diff --git a/src/datasets/arrow_writer.py b/src/datasets/arrow_writer.py index 64024090994..fd23b996dd3 100644 --- a/src/datasets/arrow_writer.py +++ b/src/datasets/arrow_writer.py @@ -584,7 +584,16 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): - self.close() + # Leaving the block must produce a complete, readable file. `close()` only closes the + # stream; it is `finalize()` that flushes the examples still staged in the buffer, so + # `with ArrowWriter(...) as writer:` used to discard everything below `writer_batch_size` + # and leave a 0-byte file. Stay quiet, as before, when the block raised, when nothing was + # written (there is no schema to infer), or when the caller already finalized. + has_staged_data = bool(self.current_examples or self.current_rows) or self.pa_writer is not None + if exc_type is None and not getattr(self.stream, "closed", False) and has_staged_data: + self.finalize() + else: + self.close() def close(self): # Try closing if opened; if closed: pyarrow.lib.ArrowInvalid: Invalid operation on closed file diff --git a/tests/test_arrow_writer.py b/tests/test_arrow_writer.py index 6e2fd14ef7f..f5243c5cc7a 100644 --- a/tests/test_arrow_writer.py +++ b/tests/test_arrow_writer.py @@ -308,6 +308,21 @@ def test_arrow_writer_closes_stream(raise_exception, tmp_path): assert writer.stream.closed +def test_arrow_writer_context_manager_writes_staged_examples(tmp_path): + # Leaving a `with ArrowWriter(...)` block must produce a complete file, including the examples + # still staged in the buffer below `writer_batch_size`. `__exit__` only called `close()`, which + # closes the stream without flushing the buffer, so the file stayed at 0 bytes and unreadable. + path = str(tmp_path / "dataset-train.arrow") + with ArrowWriter(path=path) as writer: + for i in range(3): + writer.write({"a": i}) + assert writer.stream.closed + assert os.path.getsize(path) > 0 + table = pa.ipc.open_stream(path).read_all() + assert table.num_rows == 3 + assert table.column("a").to_pylist() == [0, 1, 2] + + def test_arrow_writer_with_filesystem(mockfs): path = "mock://dataset-train.arrow" with ArrowWriter(path=path, storage_options=mockfs.storage_options) as writer: