diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3615482590..5b6149ecd1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,8 +121,8 @@ jobs: run: pip install --upgrade uv - name: Install dependencies run: uv pip install --system "datasets[tests] @ ." - - name: Install tsfile (py3.14 only) - run: uv pip install --system "tsfile>=2.3.0" + - name: Install tsfile extra (py3.14 only) + run: uv pip install --system "datasets[tsfile] @ ." - name: Print dependencies run: uv pip list - name: Test with pytest diff --git a/README.md b/README.md index d96cbda38ab..f75440b6153 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,9 @@ pip install datasets[vision] # For PDFs/NIfTI (pdfplumber, nibabel) pip install datasets[pdfs,nibabel] +# For Apache TsFile (time-series) +pip install datasets[tsfile] + # For PyTorch/TensorFlow/JAX integration pip install datasets[torch,tensorflow,jax] diff --git a/docs/source/about_dataset_load.mdx b/docs/source/about_dataset_load.mdx index dac34dacc11..38b63e86f23 100644 --- a/docs/source/about_dataset_load.mdx +++ b/docs/source/about_dataset_load.mdx @@ -23,7 +23,7 @@ Under the hood, 🤗 Datasets will use an appropriate [`DatasetBuilder`] based o * [`datasets.packaged_modules.parquet.Parquet`] for Parquet * [`datasets.packaged_modules.arrow.Arrow`] for Arrow (streaming file format) * [`datasets.packaged_modules.sql.Sql`] for SQL databases -* [`datasets.packaged_modules.tsfile.TsFile`] for TsFile (time-series data) +* [`datasets.packaged_modules.tsfile.TsFile`] for TsFile (time-series data; requires `pip install "datasets[tsfile]"`) * [`datasets.packaged_modules.imagefolder.ImageFolder`] for image folders * [`datasets.packaged_modules.audiofolder.AudioFolder`] for audio folders diff --git a/docs/source/loading.mdx b/docs/source/loading.mdx index c39b0e3e8e2..f993cc1a1f7 100644 --- a/docs/source/loading.mdx +++ b/docs/source/loading.mdx @@ -213,6 +213,12 @@ To load a TsFile: >>> dataset = load_dataset("tsfile", data_files="my_data.tsfile") ``` +The Apache TsFile SDK is an optional extra: + +```bash +pip install "datasets[tsfile]" +``` + Filter by time range — bounds are pushed down to TsFile's internal time index and accept `int` epochs, `datetime`, `date`, ISO-8601 strings, or `pyarrow` timestamp scalars: ```py diff --git a/docs/source/tsfile_load.mdx b/docs/source/tsfile_load.mdx index 6c37eae29fd..dd375d2244a 100644 --- a/docs/source/tsfile_load.mdx +++ b/docs/source/tsfile_load.mdx @@ -6,10 +6,10 @@ This loader is provided as a separate guide because it does not follow the usual ## Installation -The loader depends on the [`tsfile`](https://pypi.org/project/tsfile/) Python package: +The loader depends on the [`tsfile`](https://pypi.org/project/tsfile/) Python package, installed via the optional extra: ```bash -pip install "tsfile>=2.3.0" +pip install "datasets[tsfile]" ``` ## Data model and output layout diff --git a/setup.py b/setup.py index c536574a15e..511b06c5085 100644 --- a/setup.py +++ b/setup.py @@ -195,6 +195,7 @@ "nibabel>=5.3.1", "trimesh>=4.10.0", "teich==0.1.5", + "tsfile>=2.3.0; python_version >= '3.14'", ] NUMPY2_INCOMPATIBLE_LIBRARIES = [ @@ -220,6 +221,11 @@ ICEBERG_REQUIRE = ["pyiceberg>=0.7.0"] +# Optional: Apache TsFile SDK. On Python < 3.14 the published wheels pin +# pyarrow<20, which conflicts with datasets' pyarrow>=21, so CI only installs +# this extra on 3.14 (see .github/workflows/ci.yml). +TSFILE_REQUIRE = ["tsfile>=2.3.0"] + EXTRAS_REQUIRE = { "audio": AUDIO_REQUIRE, "vision": VISION_REQUIRE, @@ -240,6 +246,7 @@ "pdfs": PDFS_REQUIRE, "nibabel": NIBABEL_REQUIRE, "iceberg": ICEBERG_REQUIRE, + "tsfile": TSFILE_REQUIRE, } setup( diff --git a/src/datasets/config.py b/src/datasets/config.py index 5bbd646caea..0eb1bbe6616 100644 --- a/src/datasets/config.py +++ b/src/datasets/config.py @@ -142,6 +142,7 @@ NIBABEL_AVAILABLE = importlib.util.find_spec("nibabel") is not None TRIMESH_AVAILABLE = importlib.util.find_spec("trimesh") is not None TEICH_AVAILABLE = importlib.util.find_spec("teich") is not None +TSFILE_AVAILABLE = importlib.util.find_spec("tsfile") is not None # Optional compression tools RARFILE_AVAILABLE = importlib.util.find_spec("rarfile") is not None diff --git a/src/datasets/packaged_modules/tsfile/tsfile.py b/src/datasets/packaged_modules/tsfile/tsfile.py index 96712b4516e..253b8169c03 100644 --- a/src/datasets/packaged_modules/tsfile/tsfile.py +++ b/src/datasets/packaged_modules/tsfile/tsfile.py @@ -48,6 +48,24 @@ logger = datasets.utils.logging.get_logger(__name__) +_TSFILE_INSTALL_ERROR = ( + "Loading TsFile datasets requires the Apache TsFile Python SDK. " + "Please install it with `pip install 'datasets[tsfile]'`." +) + + +def _require_tsfile() -> None: + """Import the optional ``tsfile`` SDK or raise an actionable error. + + The SDK is an extra (``datasets[tsfile]``), not a hard dependency. Bare + ``ModuleNotFoundError: No module named 'tsfile'`` is what dataset-viewer + workers currently surface (huggingface/datasets#8256, #8499). + """ + try: + import tsfile # noqa: F401 + except ImportError as err: + raise ImportError(_TSFILE_INSTALL_ERROR) from err + # --------------------------------------------------------------------------- # Type helpers @@ -56,6 +74,7 @@ def _arrow_type(ts_dtype, *, unit: str, tz: Optional[str]) -> pa.DataType: """Map a tsfile ``TSDataType`` to its Arrow representation.""" + _require_tsfile() from tsfile.constants import TSDataType return { @@ -86,6 +105,7 @@ def _promote_tsdatatype(a, b): if a == b: return a + _require_tsfile() from tsfile.constants import TSDataType table = { @@ -301,6 +321,7 @@ def _generate_tables(self, files): def _scan_metadata(self, files) -> Optional[dict]: """Walk every file and unify table name, TAG columns, FIELD types.""" + _require_tsfile() from tsfile.constants import TIME_COLUMN, ColumnCategory wanted_table = self._table @@ -431,6 +452,7 @@ def _build_device_index(self, readers: dict): - ``file_meta``: maps each readable file to its per-file context (``tag_cols``, ``field_cols``, ``time_col``). """ + _require_tsfile() from tsfile.constants import ColumnCategory device_to_files: dict[tuple, list[str]] = {} @@ -541,6 +563,7 @@ def _read_device_from_file(self, reader, meta: dict, device_key: tuple) -> tuple field columns that this file owns *and* that the builder requested; callers fill missing fields with all-null contributions. """ + _require_tsfile() from tsfile import tag_eq file_tag_cols: list[str] = meta["tag_cols"] @@ -730,6 +753,7 @@ def _open_reader(file: str): segfaults. The 6-byte ``TsFile`` magic header is checked first to bail out cleanly. """ + _require_tsfile() from tsfile import TsFileReader try: diff --git a/tests/packaged_modules/test_tsfile.py b/tests/packaged_modules/test_tsfile.py index b640c8e579a..283c2e05a21 100644 --- a/tests/packaged_modules/test_tsfile.py +++ b/tests/packaged_modules/test_tsfile.py @@ -13,6 +13,8 @@ # `tsfile` requires pyarrow<20 for python<3.14, which conflicts with datasets' # pyarrow>=21.0.0. It is therefore only installed in the py3.14 CI. Skip this # whole module (at collection time) when tsfile is not importable. +# `@require_tsfile` is the per-test equivalent (see tests/utils.py); importorskip +# is still required here because the module imports TsFileWriter at import time. pytest.importorskip("tsfile") from tsfile import ColumnCategory, ColumnSchema, TableSchema, Tablet, TsFileWriter # noqa: E402 @@ -22,6 +24,7 @@ from datasets.builder import InvalidConfigName # noqa: E402 from datasets.data_files import DataFilesList # noqa: E402 from datasets.packaged_modules.tsfile.tsfile import TsFileConfig, _to_epoch # noqa: E402 +from ..utils import require_tsfile # noqa: F401 (marker lives in tests.utils; collection skip is importorskip) # --------------------------------------------------------------------------- diff --git a/tests/packaged_modules/test_tsfile_optional.py b/tests/packaged_modules/test_tsfile_optional.py new file mode 100644 index 00000000000..8f9e5719baf --- /dev/null +++ b/tests/packaged_modules/test_tsfile_optional.py @@ -0,0 +1,29 @@ +"""Tests for the TsFile builder that must run without the optional SDK.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from datasets.packaged_modules.tsfile.tsfile import _TSFILE_INSTALL_ERROR, _require_tsfile + + +def test_require_tsfile_missing_sdk_raises_actionable_import_error(): + real_import = __import__ + + def blocked(name, *args, **kwargs): + if name == "tsfile" or (isinstance(name, str) and name.startswith("tsfile.")): + raise ImportError("No module named 'tsfile'") + return real_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=blocked): + with pytest.raises(ImportError) as ei: + _require_tsfile() + + err = ei.value + assert type(err) is ImportError + assert "datasets[tsfile]" in str(err) + assert _TSFILE_INSTALL_ERROR in str(err) + assert err.__cause__ is not None + assert "tsfile" in str(err.__cause__) diff --git a/tests/utils.py b/tests/utils.py index d1d6a4a15a6..8466f4af8eb 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -239,6 +239,18 @@ def require_torchcodec(test_case): return test_case +def require_tsfile(test_case): + """ + Decorator marking a test that requires the Apache TsFile Python SDK. + + These tests are skipped when tsfile isn't installed. + + """ + if not config.TSFILE_AVAILABLE: + test_case = unittest.skip("test requires tsfile")(test_case) + return test_case + + def require_pdfplumber(test_case): """ Decorator marking a test that requires pdfplumber.