Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -121,8 +121,8 @@ jobs:
run: pip install --upgrade uv
- name: Install dependencies
run: uv pip install --system "datasets[tests] @ ."
- name: Install tsfile (py3.14 only)
run: uv pip install --system "tsfile>=2.3.0"
- name: Install tsfile extra (py3.14 only)
run: uv pip install --system "datasets[tsfile] @ ."
- name: Print dependencies
run: uv pip list
- name: Test with pytest
Expand Down
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,9 @@ pip install datasets[vision]
# For PDFs/NIfTI (pdfplumber, nibabel)
pip install datasets[pdfs,nibabel]

# For Apache TsFile (time-series)
pip install datasets[tsfile]

# For PyTorch/TensorFlow/JAX integration
pip install datasets[torch,tensorflow,jax]

Expand Down
2 changes: 1 addition & 1 deletion docs/source/about_dataset_load.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ Under the hood, 🤗 Datasets will use an appropriate [`DatasetBuilder`] based o
* [`datasets.packaged_modules.parquet.Parquet`] for Parquet
* [`datasets.packaged_modules.arrow.Arrow`] for Arrow (streaming file format)
* [`datasets.packaged_modules.sql.Sql`] for SQL databases
* [`datasets.packaged_modules.tsfile.TsFile`] for TsFile (time-series data)
* [`datasets.packaged_modules.tsfile.TsFile`] for TsFile (time-series data; requires `pip install "datasets[tsfile]"`)
* [`datasets.packaged_modules.imagefolder.ImageFolder`] for image folders
* [`datasets.packaged_modules.audiofolder.AudioFolder`] for audio folders

Expand Down
6 changes: 6 additions & 0 deletions docs/source/loading.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,12 @@ To load a TsFile:
>>> dataset = load_dataset("tsfile", data_files="my_data.tsfile")
```

The Apache TsFile SDK is an optional extra:

```bash
pip install "datasets[tsfile]"
```

Filter by time range — bounds are pushed down to TsFile's internal time index and accept `int` epochs, `datetime`, `date`, ISO-8601 strings, or `pyarrow` timestamp scalars:

```py
Expand Down
4 changes: 2 additions & 2 deletions docs/source/tsfile_load.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ This loader is provided as a separate guide because it does not follow the usual

## Installation

The loader depends on the [`tsfile`](https://pypi.org/project/tsfile/) Python package:
The loader depends on the [`tsfile`](https://pypi.org/project/tsfile/) Python package, installed via the optional extra:

```bash
pip install "tsfile>=2.3.0"
pip install "datasets[tsfile]"
```

## Data model and output layout
Expand Down
7 changes: 7 additions & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@
"nibabel>=5.3.1",
"trimesh>=4.10.0",
"teich==0.1.5",
"tsfile>=2.3.0; python_version >= '3.14'",
]

NUMPY2_INCOMPATIBLE_LIBRARIES = [
Expand All @@ -220,6 +221,11 @@

ICEBERG_REQUIRE = ["pyiceberg>=0.7.0"]

# Optional: Apache TsFile SDK. On Python < 3.14 the published wheels pin
# pyarrow<20, which conflicts with datasets' pyarrow>=21, so CI only installs
# this extra on 3.14 (see .github/workflows/ci.yml).
TSFILE_REQUIRE = ["tsfile>=2.3.0"]

EXTRAS_REQUIRE = {
"audio": AUDIO_REQUIRE,
"vision": VISION_REQUIRE,
Expand All @@ -240,6 +246,7 @@
"pdfs": PDFS_REQUIRE,
"nibabel": NIBABEL_REQUIRE,
"iceberg": ICEBERG_REQUIRE,
"tsfile": TSFILE_REQUIRE,
}

setup(
Expand Down
1 change: 1 addition & 0 deletions src/datasets/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@
NIBABEL_AVAILABLE = importlib.util.find_spec("nibabel") is not None
TRIMESH_AVAILABLE = importlib.util.find_spec("trimesh") is not None
TEICH_AVAILABLE = importlib.util.find_spec("teich") is not None
TSFILE_AVAILABLE = importlib.util.find_spec("tsfile") is not None

# Optional compression tools
RARFILE_AVAILABLE = importlib.util.find_spec("rarfile") is not None
Expand Down
24 changes: 24 additions & 0 deletions src/datasets/packaged_modules/tsfile/tsfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,24 @@

logger = datasets.utils.logging.get_logger(__name__)

_TSFILE_INSTALL_ERROR = (
"Loading TsFile datasets requires the Apache TsFile Python SDK. "
"Please install it with `pip install 'datasets[tsfile]'`."
)


def _require_tsfile() -> None:
"""Import the optional ``tsfile`` SDK or raise an actionable error.

The SDK is an extra (``datasets[tsfile]``), not a hard dependency. Bare
``ModuleNotFoundError: No module named 'tsfile'`` is what dataset-viewer
workers currently surface (huggingface/datasets#8256, #8499).
"""
try:
import tsfile # noqa: F401
except ImportError as err:
raise ImportError(_TSFILE_INSTALL_ERROR) from err


# ---------------------------------------------------------------------------
# Type helpers
Expand All @@ -56,6 +74,7 @@

def _arrow_type(ts_dtype, *, unit: str, tz: Optional[str]) -> pa.DataType:
"""Map a tsfile ``TSDataType`` to its Arrow representation."""
_require_tsfile()
from tsfile.constants import TSDataType

return {
Expand Down Expand Up @@ -86,6 +105,7 @@ def _promote_tsdatatype(a, b):
if a == b:
return a

_require_tsfile()
from tsfile.constants import TSDataType

table = {
Expand Down Expand Up @@ -301,6 +321,7 @@ def _generate_tables(self, files):

def _scan_metadata(self, files) -> Optional[dict]:
"""Walk every file and unify table name, TAG columns, FIELD types."""
_require_tsfile()
from tsfile.constants import TIME_COLUMN, ColumnCategory

wanted_table = self._table
Expand Down Expand Up @@ -431,6 +452,7 @@ def _build_device_index(self, readers: dict):
- ``file_meta``: maps each readable file to its per-file context
(``tag_cols``, ``field_cols``, ``time_col``).
"""
_require_tsfile()
from tsfile.constants import ColumnCategory

device_to_files: dict[tuple, list[str]] = {}
Expand Down Expand Up @@ -541,6 +563,7 @@ def _read_device_from_file(self, reader, meta: dict, device_key: tuple) -> tuple
field columns that this file owns *and* that the builder requested;
callers fill missing fields with all-null contributions.
"""
_require_tsfile()
from tsfile import tag_eq

file_tag_cols: list[str] = meta["tag_cols"]
Expand Down Expand Up @@ -730,6 +753,7 @@ def _open_reader(file: str):
segfaults. The 6-byte ``TsFile`` magic header is checked first to
bail out cleanly.
"""
_require_tsfile()
from tsfile import TsFileReader

try:
Expand Down
3 changes: 3 additions & 0 deletions tests/packaged_modules/test_tsfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
# `tsfile` requires pyarrow<20 for python<3.14, which conflicts with datasets'
# pyarrow>=21.0.0. It is therefore only installed in the py3.14 CI. Skip this
# whole module (at collection time) when tsfile is not importable.
# `@require_tsfile` is the per-test equivalent (see tests/utils.py); importorskip
# is still required here because the module imports TsFileWriter at import time.
pytest.importorskip("tsfile")

from tsfile import ColumnCategory, ColumnSchema, TableSchema, Tablet, TsFileWriter # noqa: E402
Expand All @@ -22,6 +24,7 @@
from datasets.builder import InvalidConfigName # noqa: E402
from datasets.data_files import DataFilesList # noqa: E402
from datasets.packaged_modules.tsfile.tsfile import TsFileConfig, _to_epoch # noqa: E402
from ..utils import require_tsfile # noqa: F401 (marker lives in tests.utils; collection skip is importorskip)


# ---------------------------------------------------------------------------
Expand Down
29 changes: 29 additions & 0 deletions tests/packaged_modules/test_tsfile_optional.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
"""Tests for the TsFile builder that must run without the optional SDK."""

from __future__ import annotations

from unittest.mock import patch

import pytest

from datasets.packaged_modules.tsfile.tsfile import _TSFILE_INSTALL_ERROR, _require_tsfile


def test_require_tsfile_missing_sdk_raises_actionable_import_error():
real_import = __import__

def blocked(name, *args, **kwargs):
if name == "tsfile" or (isinstance(name, str) and name.startswith("tsfile.")):
raise ImportError("No module named 'tsfile'")
return real_import(name, *args, **kwargs)

with patch("builtins.__import__", side_effect=blocked):
with pytest.raises(ImportError) as ei:
_require_tsfile()

err = ei.value
assert type(err) is ImportError
assert "datasets[tsfile]" in str(err)
assert _TSFILE_INSTALL_ERROR in str(err)
assert err.__cause__ is not None
assert "tsfile" in str(err.__cause__)
12 changes: 12 additions & 0 deletions tests/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,18 @@ def require_torchcodec(test_case):
return test_case


def require_tsfile(test_case):
"""
Decorator marking a test that requires the Apache TsFile Python SDK.

These tests are skipped when tsfile isn't installed.

"""
if not config.TSFILE_AVAILABLE:
test_case = unittest.skip("test requires tsfile")(test_case)
return test_case


def require_pdfplumber(test_case):
"""
Decorator marking a test that requires pdfplumber.
Expand Down