From 937fbbd6fefb03f60a2d021ad4aa78086a02bf44 Mon Sep 17 00:00:00 2001 From: AxelNoun Date: Wed, 26 Aug 2026 06:49:36 +0200 Subject: [PATCH] fix(tsfile): guard the optional Apache TsFile SDK import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TsFile builder imported the third-party `tsfile` SDK inside `_scan_metadata` without a guard, surfacing a bare `ModuleNotFoundError` to users and to the dataset-viewer workers. Scope note: this is a usability and packaging fix. It does not fix the Format filter reported in #8499 — `tsfile` is not registered in the Hub's format taxonomy, which accounts for that symptom on its own and is not actionable in this repository. It also does not fix Hub-inspect / streaming file-open failures (separate PR). On Python 3.10-3.13 the SDK cannot co-install with `datasets` (pyarrow<20 vs pyarrow>=21); the ImportError must say so, because an environment-marked `datasets[tsfile]` extra is silently empty there. - raise a version-aware ImportError (point at `datasets[tsfile]` only on 3.14+) - declare `tsfile>=2.3.0; python_version >= '3.14'` and mirror it in test extras - document the `.tsfile` extension and the 3.14+ co-install constraint - add missing-SDK and message tests; keep round-trips behind `require_tsfile` Refs #8499, #8160 Co-authored-by: Cursor --- .github/workflows/ci.yml | 4 +- docs/source/about_dataset_load.mdx | 2 +- docs/source/loading.mdx | 7 +++ docs/source/repository_structure.mdx | 4 +- docs/source/tsfile_load.mdx | 6 ++- setup.py | 8 +++ src/datasets/config.py | 1 + .../packaged_modules/tsfile/tsfile.py | 51 +++++++++++++++++++ tests/packaged_modules/test_tsfile_guard.py | 41 +++++++++++++++ tests/utils.py | 12 +++++ 10 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 tests/packaged_modules/test_tsfile_guard.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3615482590..5b6149ecd1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,8 +121,8 @@ jobs: run: pip install --upgrade uv - name: Install dependencies run: uv pip install --system "datasets[tests] @ ." - - name: Install tsfile (py3.14 only) - run: uv pip install --system "tsfile>=2.3.0" + - name: Install tsfile extra (py3.14 only) + run: uv pip install --system "datasets[tsfile] @ ." - name: Print dependencies run: uv pip list - name: Test with pytest diff --git a/docs/source/about_dataset_load.mdx b/docs/source/about_dataset_load.mdx index dac34dacc11..dab4f76c21a 100644 --- a/docs/source/about_dataset_load.mdx +++ b/docs/source/about_dataset_load.mdx @@ -23,7 +23,7 @@ Under the hood, 🤗 Datasets will use an appropriate [`DatasetBuilder`] based o * [`datasets.packaged_modules.parquet.Parquet`] for Parquet * [`datasets.packaged_modules.arrow.Arrow`] for Arrow (streaming file format) * [`datasets.packaged_modules.sql.Sql`] for SQL databases -* [`datasets.packaged_modules.tsfile.TsFile`] for TsFile (time-series data) +* [`datasets.packaged_modules.tsfile.TsFile`] for TsFile (time-series data; requires `pip install "datasets[tsfile]"`, Python 3.14+) * [`datasets.packaged_modules.imagefolder.ImageFolder`] for image folders * [`datasets.packaged_modules.audiofolder.AudioFolder`] for audio folders diff --git a/docs/source/loading.mdx b/docs/source/loading.mdx index c39b0e3e8e2..c9e5ddae43b 100644 --- a/docs/source/loading.mdx +++ b/docs/source/loading.mdx @@ -213,6 +213,13 @@ To load a TsFile: >>> dataset = load_dataset("tsfile", data_files="my_data.tsfile") ``` +Install the optional Apache TsFile SDK first (`datasets[tsfile]`). The SDK's +pyarrow requirement currently only resolves against `datasets` on Python 3.14+: + +```bash +pip install "datasets[tsfile]" +``` + Filter by time range — bounds are pushed down to TsFile's internal time index and accept `int` epochs, `datetime`, `date`, ISO-8601 strings, or `pyarrow` timestamp scalars: ```py diff --git a/docs/source/repository_structure.mdx b/docs/source/repository_structure.mdx index acb2bb1ed3d..2792b086d8a 100644 --- a/docs/source/repository_structure.mdx +++ b/docs/source/repository_structure.mdx @@ -3,7 +3,9 @@ To host and share your dataset, create a dataset repository on the Hugging Face Hub and upload your data files. This guide will show you how to structure your dataset repository when you upload it. -A dataset with a supported structure and file format (`.txt`, `.csv`, `.parquet`, `.jsonl`, `.mp3`, `.jpg`, `.zip` etc.) are loaded automatically with [`~datasets.load_dataset`], and it'll have a dataset viewer on its dataset page on the Hub. +A dataset with a supported structure and file format (`.txt`, `.csv`, `.parquet`, `.jsonl`, `.mp3`, `.jpg`, `.zip`, `.tsfile` etc.) are loaded automatically with [`~datasets.load_dataset`], and it'll have a dataset viewer on its dataset page on the Hub. + +`.tsfile` — requires the optional Apache TsFile SDK (`datasets[tsfile]`). The SDK's pyarrow requirement currently only resolves against `datasets` on Python 3.14+. ## Main use-case diff --git a/docs/source/tsfile_load.mdx b/docs/source/tsfile_load.mdx index 6c37eae29fd..40be9d84443 100644 --- a/docs/source/tsfile_load.mdx +++ b/docs/source/tsfile_load.mdx @@ -6,12 +6,14 @@ This loader is provided as a separate guide because it does not follow the usual ## Installation -The loader depends on the [`tsfile`](https://pypi.org/project/tsfile/) Python package: +The loader depends on the [`tsfile`](https://pypi.org/project/tsfile/) Python package, installed via the optional extra: ```bash -pip install "tsfile>=2.3.0" +pip install "datasets[tsfile]" ``` +The SDK's pyarrow requirement currently only resolves against `datasets` on Python 3.14+. + ## Data model and output layout The loader follows the TsFile *table model*. Each table column is one of: diff --git a/setup.py b/setup.py index c536574a15e..863724f00bb 100644 --- a/setup.py +++ b/setup.py @@ -195,6 +195,9 @@ "nibabel>=5.3.1", "trimesh>=4.10.0", "teich==0.1.5", + # Apache TsFile SDK — only co-installable with datasets on Python 3.14+ + # (pyarrow pins: SDK <20 on 3.10-3.13 vs datasets >=21). See #8499. + "tsfile>=2.3.0; python_version >= '3.14'", ] NUMPY2_INCOMPATIBLE_LIBRARIES = [ @@ -220,6 +223,10 @@ ICEBERG_REQUIRE = ["pyiceberg>=0.7.0"] +# Optional: Apache TsFile SDK. Environment marker keeps the extra empty on +# Python < 3.14 where pyarrow pins conflict with datasets (see #8499). +TSFILE_REQUIRE = ["tsfile>=2.3.0; python_version >= '3.14'"] + EXTRAS_REQUIRE = { "audio": AUDIO_REQUIRE, "vision": VISION_REQUIRE, @@ -240,6 +247,7 @@ "pdfs": PDFS_REQUIRE, "nibabel": NIBABEL_REQUIRE, "iceberg": ICEBERG_REQUIRE, + "tsfile": TSFILE_REQUIRE, } setup( diff --git a/src/datasets/config.py b/src/datasets/config.py index 5bbd646caea..0eb1bbe6616 100644 --- a/src/datasets/config.py +++ b/src/datasets/config.py @@ -142,6 +142,7 @@ NIBABEL_AVAILABLE = importlib.util.find_spec("nibabel") is not None TRIMESH_AVAILABLE = importlib.util.find_spec("trimesh") is not None TEICH_AVAILABLE = importlib.util.find_spec("teich") is not None +TSFILE_AVAILABLE = importlib.util.find_spec("tsfile") is not None # Optional compression tools RARFILE_AVAILABLE = importlib.util.find_spec("rarfile") is not None diff --git a/src/datasets/packaged_modules/tsfile/tsfile.py b/src/datasets/packaged_modules/tsfile/tsfile.py index 96712b4516e..c6d088cca9c 100644 --- a/src/datasets/packaged_modules/tsfile/tsfile.py +++ b/src/datasets/packaged_modules/tsfile/tsfile.py @@ -34,6 +34,7 @@ from __future__ import annotations import datetime as _dt +import sys from dataclasses import dataclass from typing import Any, Literal, Optional @@ -48,6 +49,50 @@ logger = datasets.utils.logging.get_logger(__name__) +# The Apache TsFile SDK pins pyarrow per Python version: +# 3.9 -> pyarrow>=16,<18 +# 3.10-3.13 -> pyarrow>=18,<20 +# 3.14+ -> pyarrow>=22,<24 +# `datasets` requires pyarrow>=21 (since 4.8.5), so the SDK is only co-installable +# on Python 3.14+. Verified against tsfile 2.3.0/2.3.1/2.4.0 on 2026-08-25. +# Lower bound is 2.3.0 on purpose: 2.2.1 declares no pyarrow constraint at all. +# Revisit if the SDK relaxes the <20 bound -- see #8499. +_TSFILE_MIN_PYTHON = (3, 14) + + +def _tsfile_install_message() -> str: + base = ( + "Loading TsFile datasets requires the Apache TsFile Python SDK, " + "which is not installed." + ) + # [:2] so tests can monkeypatch sys.version_info with a plain tuple. + py = sys.version_info[:2] + if py >= _TSFILE_MIN_PYTHON: + return f"{base} Install it with `pip install 'datasets[tsfile]'`." + return ( + f"{base} On Python {py[0]}.{py[1]} it " + "cannot be installed alongside `datasets`: the SDK requires pyarrow<20 on " + "Python 3.10-3.13, while `datasets` requires pyarrow>=21, and no set of " + "versions satisfies both. Note that `pip install 'datasets[tsfile]'` will " + "report success without installing anything on this interpreter. " + f"Python {_TSFILE_MIN_PYTHON[0]}.{_TSFILE_MIN_PYTHON[1]}+ is currently the " + "only combination that resolves. " + "See https://github.com/huggingface/datasets/issues/8499" + ) + + +def _require_tsfile() -> None: + """Import the optional ``tsfile`` SDK or raise an actionable error. + + The SDK is an extra (``datasets[tsfile]``), not a hard dependency. Bare + ``ModuleNotFoundError: No module named 'tsfile'`` is what dataset-viewer + workers currently surface (huggingface/datasets#8256, #8499). + """ + try: + import tsfile # noqa: F401 + except ImportError as err: + raise ImportError(_tsfile_install_message()) from err + # --------------------------------------------------------------------------- # Type helpers @@ -56,6 +101,7 @@ def _arrow_type(ts_dtype, *, unit: str, tz: Optional[str]) -> pa.DataType: """Map a tsfile ``TSDataType`` to its Arrow representation.""" + _require_tsfile() from tsfile.constants import TSDataType return { @@ -86,6 +132,7 @@ def _promote_tsdatatype(a, b): if a == b: return a + _require_tsfile() from tsfile.constants import TSDataType table = { @@ -301,6 +348,7 @@ def _generate_tables(self, files): def _scan_metadata(self, files) -> Optional[dict]: """Walk every file and unify table name, TAG columns, FIELD types.""" + _require_tsfile() from tsfile.constants import TIME_COLUMN, ColumnCategory wanted_table = self._table @@ -431,6 +479,7 @@ def _build_device_index(self, readers: dict): - ``file_meta``: maps each readable file to its per-file context (``tag_cols``, ``field_cols``, ``time_col``). """ + _require_tsfile() from tsfile.constants import ColumnCategory device_to_files: dict[tuple, list[str]] = {} @@ -541,6 +590,7 @@ def _read_device_from_file(self, reader, meta: dict, device_key: tuple) -> tuple field columns that this file owns *and* that the builder requested; callers fill missing fields with all-null contributions. """ + _require_tsfile() from tsfile import tag_eq file_tag_cols: list[str] = meta["tag_cols"] @@ -730,6 +780,7 @@ def _open_reader(file: str): segfaults. The 6-byte ``TsFile`` magic header is checked first to bail out cleanly. """ + _require_tsfile() from tsfile import TsFileReader try: diff --git a/tests/packaged_modules/test_tsfile_guard.py b/tests/packaged_modules/test_tsfile_guard.py new file mode 100644 index 00000000000..d6ed47a1808 --- /dev/null +++ b/tests/packaged_modules/test_tsfile_guard.py @@ -0,0 +1,41 @@ +"""Tests for the TsFile optional-SDK guard (must run without the SDK).""" + +from __future__ import annotations + +import sys +from unittest.mock import patch + +import pytest + +from datasets.packaged_modules.tsfile.tsfile import _require_tsfile, _tsfile_install_message + + +def test_install_message_is_version_aware(monkeypatch): + monkeypatch.setattr(sys, "version_info", (3, 12, 0, "final", 0)) + msg = _tsfile_install_message() + assert "cannot be installed alongside" in msg + assert "without installing anything" in msg + + monkeypatch.setattr(sys, "version_info", (3, 14, 0, "final", 0)) + msg = _tsfile_install_message() + assert "datasets[tsfile]" in msg + assert "cannot be installed" not in msg + + +def test_require_tsfile_missing_sdk_raises_actionable_import_error(): + real_import = __import__ + + def blocked(name, *args, **kwargs): + if name == "tsfile" or (isinstance(name, str) and name.startswith("tsfile.")): + raise ImportError("No module named 'tsfile'") + return real_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=blocked): + with pytest.raises(ImportError) as ei: + _require_tsfile() + + err = ei.value + assert type(err) is ImportError + assert "Apache TsFile" in str(err) + assert err.__cause__ is not None + assert "tsfile" in str(err.__cause__) diff --git a/tests/utils.py b/tests/utils.py index d1d6a4a15a6..8466f4af8eb 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -239,6 +239,18 @@ def require_torchcodec(test_case): return test_case +def require_tsfile(test_case): + """ + Decorator marking a test that requires the Apache TsFile Python SDK. + + These tests are skipped when tsfile isn't installed. + + """ + if not config.TSFILE_AVAILABLE: + test_case = unittest.skip("test requires tsfile")(test_case) + return test_case + + def require_pdfplumber(test_case): """ Decorator marking a test that requires pdfplumber.