Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -121,8 +121,8 @@ jobs:
run: pip install --upgrade uv
- name: Install dependencies
run: uv pip install --system "datasets[tests] @ ."
- name: Install tsfile (py3.14 only)
run: uv pip install --system "tsfile>=2.3.0"
- name: Install tsfile extra (py3.14 only)
run: uv pip install --system "datasets[tsfile] @ ."
- name: Print dependencies
run: uv pip list
- name: Test with pytest
Expand Down
2 changes: 1 addition & 1 deletion docs/source/about_dataset_load.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ Under the hood, 🤗 Datasets will use an appropriate [`DatasetBuilder`] based o
* [`datasets.packaged_modules.parquet.Parquet`] for Parquet
* [`datasets.packaged_modules.arrow.Arrow`] for Arrow (streaming file format)
* [`datasets.packaged_modules.sql.Sql`] for SQL databases
* [`datasets.packaged_modules.tsfile.TsFile`] for TsFile (time-series data)
* [`datasets.packaged_modules.tsfile.TsFile`] for TsFile (time-series data; requires `pip install "datasets[tsfile]"`, Python 3.14+)
* [`datasets.packaged_modules.imagefolder.ImageFolder`] for image folders
* [`datasets.packaged_modules.audiofolder.AudioFolder`] for audio folders

Expand Down
7 changes: 7 additions & 0 deletions docs/source/loading.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,13 @@ To load a TsFile:
>>> dataset = load_dataset("tsfile", data_files="my_data.tsfile")
```

Install the optional Apache TsFile SDK first (`datasets[tsfile]`). The SDK's
pyarrow requirement currently only resolves against `datasets` on Python 3.14+:

```bash
pip install "datasets[tsfile]"
```

Filter by time range — bounds are pushed down to TsFile's internal time index and accept `int` epochs, `datetime`, `date`, ISO-8601 strings, or `pyarrow` timestamp scalars:

```py
Expand Down
4 changes: 3 additions & 1 deletion docs/source/repository_structure.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
To host and share your dataset, create a dataset repository on the Hugging Face Hub and upload your data files.

This guide will show you how to structure your dataset repository when you upload it.
A dataset with a supported structure and file format (`.txt`, `.csv`, `.parquet`, `.jsonl`, `.mp3`, `.jpg`, `.zip` etc.) are loaded automatically with [`~datasets.load_dataset`], and it'll have a dataset viewer on its dataset page on the Hub.
A dataset with a supported structure and file format (`.txt`, `.csv`, `.parquet`, `.jsonl`, `.mp3`, `.jpg`, `.zip`, `.tsfile` etc.) are loaded automatically with [`~datasets.load_dataset`], and it'll have a dataset viewer on its dataset page on the Hub.

`.tsfile` — requires the optional Apache TsFile SDK (`datasets[tsfile]`). The SDK's pyarrow requirement currently only resolves against `datasets` on Python 3.14+.

## Main use-case

Expand Down
6 changes: 4 additions & 2 deletions docs/source/tsfile_load.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,14 @@ This loader is provided as a separate guide because it does not follow the usual

## Installation

The loader depends on the [`tsfile`](https://pypi.org/project/tsfile/) Python package:
The loader depends on the [`tsfile`](https://pypi.org/project/tsfile/) Python package, installed via the optional extra:

```bash
pip install "tsfile>=2.3.0"
pip install "datasets[tsfile]"
```

The SDK's pyarrow requirement currently only resolves against `datasets` on Python 3.14+.

## Data model and output layout

The loader follows the TsFile *table model*. Each table column is one of:
Expand Down
8 changes: 8 additions & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,9 @@
"nibabel>=5.3.1",
"trimesh>=4.10.0",
"teich==0.1.5",
# Apache TsFile SDK — only co-installable with datasets on Python 3.14+
# (pyarrow pins: SDK <20 on 3.10-3.13 vs datasets >=21). See #8499.
"tsfile>=2.3.0; python_version >= '3.14'",
]

NUMPY2_INCOMPATIBLE_LIBRARIES = [
Expand All @@ -220,6 +223,10 @@

ICEBERG_REQUIRE = ["pyiceberg>=0.7.0"]

# Optional: Apache TsFile SDK. Environment marker keeps the extra empty on
# Python < 3.14 where pyarrow pins conflict with datasets (see #8499).
TSFILE_REQUIRE = ["tsfile>=2.3.0; python_version >= '3.14'"]

EXTRAS_REQUIRE = {
"audio": AUDIO_REQUIRE,
"vision": VISION_REQUIRE,
Expand All @@ -240,6 +247,7 @@
"pdfs": PDFS_REQUIRE,
"nibabel": NIBABEL_REQUIRE,
"iceberg": ICEBERG_REQUIRE,
"tsfile": TSFILE_REQUIRE,
}

setup(
Expand Down
1 change: 1 addition & 0 deletions src/datasets/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@
NIBABEL_AVAILABLE = importlib.util.find_spec("nibabel") is not None
TRIMESH_AVAILABLE = importlib.util.find_spec("trimesh") is not None
TEICH_AVAILABLE = importlib.util.find_spec("teich") is not None
TSFILE_AVAILABLE = importlib.util.find_spec("tsfile") is not None

# Optional compression tools
RARFILE_AVAILABLE = importlib.util.find_spec("rarfile") is not None
Expand Down
51 changes: 51 additions & 0 deletions src/datasets/packaged_modules/tsfile/tsfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
from __future__ import annotations

import datetime as _dt
import sys
from dataclasses import dataclass
from typing import Any, Literal, Optional

Expand All @@ -48,6 +49,50 @@

logger = datasets.utils.logging.get_logger(__name__)

# The Apache TsFile SDK pins pyarrow per Python version:
# 3.9 -> pyarrow>=16,<18
# 3.10-3.13 -> pyarrow>=18,<20
# 3.14+ -> pyarrow>=22,<24
# `datasets` requires pyarrow>=21 (since 4.8.5), so the SDK is only co-installable
# on Python 3.14+. Verified against tsfile 2.3.0/2.3.1/2.4.0 on 2026-08-25.
# Lower bound is 2.3.0 on purpose: 2.2.1 declares no pyarrow constraint at all.
# Revisit if the SDK relaxes the <20 bound -- see #8499.
_TSFILE_MIN_PYTHON = (3, 14)


def _tsfile_install_message() -> str:
base = (
"Loading TsFile datasets requires the Apache TsFile Python SDK, "
"which is not installed."
)
# [:2] so tests can monkeypatch sys.version_info with a plain tuple.
py = sys.version_info[:2]
if py >= _TSFILE_MIN_PYTHON:
return f"{base} Install it with `pip install 'datasets[tsfile]'`."
return (
f"{base} On Python {py[0]}.{py[1]} it "
"cannot be installed alongside `datasets`: the SDK requires pyarrow<20 on "
"Python 3.10-3.13, while `datasets` requires pyarrow>=21, and no set of "
"versions satisfies both. Note that `pip install 'datasets[tsfile]'` will "
"report success without installing anything on this interpreter. "
f"Python {_TSFILE_MIN_PYTHON[0]}.{_TSFILE_MIN_PYTHON[1]}+ is currently the "
"only combination that resolves. "
"See https://github.com/huggingface/datasets/issues/8499"
)


def _require_tsfile() -> None:
"""Import the optional ``tsfile`` SDK or raise an actionable error.

The SDK is an extra (``datasets[tsfile]``), not a hard dependency. Bare
``ModuleNotFoundError: No module named 'tsfile'`` is what dataset-viewer
workers currently surface (huggingface/datasets#8256, #8499).
"""
try:
import tsfile # noqa: F401
except ImportError as err:
raise ImportError(_tsfile_install_message()) from err


# ---------------------------------------------------------------------------
# Type helpers
Expand All @@ -56,6 +101,7 @@

def _arrow_type(ts_dtype, *, unit: str, tz: Optional[str]) -> pa.DataType:
"""Map a tsfile ``TSDataType`` to its Arrow representation."""
_require_tsfile()
from tsfile.constants import TSDataType

return {
Expand Down Expand Up @@ -86,6 +132,7 @@ def _promote_tsdatatype(a, b):
if a == b:
return a

_require_tsfile()
from tsfile.constants import TSDataType

table = {
Expand Down Expand Up @@ -301,6 +348,7 @@ def _generate_tables(self, files):

def _scan_metadata(self, files) -> Optional[dict]:
"""Walk every file and unify table name, TAG columns, FIELD types."""
_require_tsfile()
from tsfile.constants import TIME_COLUMN, ColumnCategory

wanted_table = self._table
Expand Down Expand Up @@ -431,6 +479,7 @@ def _build_device_index(self, readers: dict):
- ``file_meta``: maps each readable file to its per-file context
(``tag_cols``, ``field_cols``, ``time_col``).
"""
_require_tsfile()
from tsfile.constants import ColumnCategory

device_to_files: dict[tuple, list[str]] = {}
Expand Down Expand Up @@ -541,6 +590,7 @@ def _read_device_from_file(self, reader, meta: dict, device_key: tuple) -> tuple
field columns that this file owns *and* that the builder requested;
callers fill missing fields with all-null contributions.
"""
_require_tsfile()
from tsfile import tag_eq

file_tag_cols: list[str] = meta["tag_cols"]
Expand Down Expand Up @@ -730,6 +780,7 @@ def _open_reader(file: str):
segfaults. The 6-byte ``TsFile`` magic header is checked first to
bail out cleanly.
"""
_require_tsfile()
from tsfile import TsFileReader

try:
Expand Down
41 changes: 41 additions & 0 deletions tests/packaged_modules/test_tsfile_guard.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
"""Tests for the TsFile optional-SDK guard (must run without the SDK)."""

from __future__ import annotations

import sys
from unittest.mock import patch

import pytest

from datasets.packaged_modules.tsfile.tsfile import _require_tsfile, _tsfile_install_message


def test_install_message_is_version_aware(monkeypatch):
monkeypatch.setattr(sys, "version_info", (3, 12, 0, "final", 0))
msg = _tsfile_install_message()
assert "cannot be installed alongside" in msg
assert "without installing anything" in msg

monkeypatch.setattr(sys, "version_info", (3, 14, 0, "final", 0))
msg = _tsfile_install_message()
assert "datasets[tsfile]" in msg
assert "cannot be installed" not in msg


def test_require_tsfile_missing_sdk_raises_actionable_import_error():
real_import = __import__

def blocked(name, *args, **kwargs):
if name == "tsfile" or (isinstance(name, str) and name.startswith("tsfile.")):
raise ImportError("No module named 'tsfile'")
return real_import(name, *args, **kwargs)

with patch("builtins.__import__", side_effect=blocked):
with pytest.raises(ImportError) as ei:
_require_tsfile()

err = ei.value
assert type(err) is ImportError
assert "Apache TsFile" in str(err)
assert err.__cause__ is not None
assert "tsfile" in str(err.__cause__)
12 changes: 12 additions & 0 deletions tests/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,18 @@ def require_torchcodec(test_case):
return test_case


def require_tsfile(test_case):
"""
Decorator marking a test that requires the Apache TsFile Python SDK.

These tests are skipped when tsfile isn't installed.

"""
if not config.TSFILE_AVAILABLE:
test_case = unittest.skip("test requires tsfile")(test_case)
return test_case


def require_pdfplumber(test_case):
"""
Decorator marking a test that requires pdfplumber.
Expand Down