Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions .github/workflows/wheels.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ jobs:
- name: Install build tools
run: |
python -m pip install -U pip wheel
python -m pip install cibuildwheel==2.21.3
python -m pip install cibuildwheel==3.4.1

- name: Pre-download virtualenv for cibuildwheel
run: |
Expand Down Expand Up @@ -108,8 +108,8 @@ jobs:
CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*"
CIBW_SKIP: "pp* *-musllinux*"

CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014"
CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux2014"
CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"

MACOSX_DEPLOYMENT_TARGET: "12.0"

Expand Down Expand Up @@ -222,7 +222,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
steps:
- name: Checkout
uses: actions/checkout@v4
Expand Down Expand Up @@ -268,8 +268,13 @@ jobs:
# Build wheel via Maven (no clean — keep C++ artifacts from previous job)
chmod +x mvnw || true
cd python
BLACK_VER_PROP=""
case "${{ matrix.python-version }}" in
3.9) BLACK_VER_PROP="-Dblack.version=25.11.0" ;;
esac
../mvnw package -DskipTests \
-Dspotless.check.skip=true -Dspotless.apply.skip=true
-Dspotless.check.skip=true -Dspotless.apply.skip=true \
${BLACK_VER_PROP}
ls -la dist/

- name: Verify wheel
Expand Down
6 changes: 4 additions & 2 deletions python/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,11 @@ maintainers = [
]
dependencies = [
"numpy>=2.0.0,<3",
"pandas>=2.0",
"pandas>=2.0,<2.3; python_full_version < '3.14.0'",
"pandas>=2.3.3; python_full_version >= '3.14.0'",
"pyarrow>=16.0,<18; python_version<'3.10'",
"pyarrow>=18.0,<20; python_version>='3.10'"
"pyarrow>=18.0,<20; python_version>='3.10' and python_version<'3.14'",
"pyarrow>=22.0,<24; python_version>='3.14'"
]

[project.urls]
Expand Down
6 changes: 4 additions & 2 deletions python/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@
#

cython==3.0.10
black==26.3.1
black==25.11.0; python_version < "3.10"
black==26.3.1; python_version >= "3.10"
numpy>=2.0.0,<3
pandas==2.2.2
pandas==2.2.2; python_full_version < "3.14.0"
pandas>=2.3.3; python_full_version >= "3.14.0"
setuptools==78.1.1
wheel==0.46.2
pyarrow>=8.0.0
13 changes: 13 additions & 0 deletions python/tests/test_dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,13 @@ def convert_to_nullable_types(df):
df[col] = df[col].astype("Float32")
elif dtype == "bool":
df[col] = df[col].astype("boolean")
elif pd.api.types.is_object_dtype(df[col]):
non_null = df[col].dropna()
if len(non_null) and non_null.map(lambda x: isinstance(x, str)).all():
df[col] = df[col].astype("string")
elif pd.api.types.is_string_dtype(df[col]):
# NumPy/pandas str dtype (e.g. Py3.14) vs object: unify for Series.equals
df[col] = df[col].astype("string")
return df


Expand Down Expand Up @@ -92,6 +99,7 @@ def test_write_dataframe_basic():
df_sorted = convert_to_nullable_types(
df.sort_values("time").reset_index(drop=True)
)
df_read = convert_to_nullable_types(df_read)
assert df_read.shape == (100, 4)
assert df_read[TIME_COLUMN].equals(df_sorted["time"])
assert df_read["device"].equals(df_sorted["device"])
Expand Down Expand Up @@ -128,6 +136,7 @@ def test_write_dataframe_with_index():
df_read = df_read.sort_values(TIME_COLUMN).reset_index(drop=True)
df_sorted = df.sort_index()
df_sorted = convert_to_nullable_types(df_sorted.reset_index(drop=True))
df_read = convert_to_nullable_types(df_read)
time_series = pd.Series(df.sort_index().index.values, dtype="Int64")
assert df_read.shape == (50, 3)
assert df_read[TIME_COLUMN].equals(time_series)
Expand Down Expand Up @@ -166,6 +175,7 @@ def test_write_dataframe_case_insensitive():
df_sorted = convert_to_nullable_types(
df.sort_values("Time").reset_index(drop=True)
)
df_read = convert_to_nullable_types(df_read)
assert df_read.shape == (30, 3)
assert df_read[TIME_COLUMN].equals(df_sorted["Time"])
assert df_read["device"].equals(df_sorted["Device"])
Expand Down Expand Up @@ -270,6 +280,7 @@ def test_write_dataframe_all_datatypes():
df_sorted = convert_to_nullable_types(
df.sort_values("time").reset_index(drop=True)
)
df_read = convert_to_nullable_types(df_read)
assert df_read.shape == (50, 11)
assert df_read["bool_col"].equals(df_sorted["bool_col"])
assert df_read["int32_col"].equals(df_sorted["int32_col"])
Expand Down Expand Up @@ -317,6 +328,7 @@ def test_write_dataframe_schema_time_column():
df_sorted = convert_to_nullable_types(
df.sort_values("time").reset_index(drop=True)
)
df_read = convert_to_nullable_types(df_read)
assert df_read.shape == (50, 3)
assert df_read[TIME_COLUMN].equals(df_sorted[TIME_COLUMN])
assert df_read["device"].equals(df_sorted["device"])
Expand Down Expand Up @@ -354,6 +366,7 @@ def test_write_dataframe_schema_time_and_dataframe_time():
df_sorted = convert_to_nullable_types(
df.sort_values("Time").rename(columns=str.lower).reset_index(drop=True)
)
df_read = convert_to_nullable_types(df_read)
assert df_read.shape == (30, 3)
assert df_read["time"].equals(df_sorted["time"])
assert df_read["device"].equals(df_sorted["device"])
Expand Down
13 changes: 13 additions & 0 deletions python/tests/test_to_tsfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ def convert_to_nullable_types(df):
df[col] = df[col].astype("Float32")
elif dtype == "bool":
df[col] = df[col].astype("boolean")
elif pd.api.types.is_object_dtype(df[col]):
non_null = df[col].dropna()
if len(non_null) and non_null.map(lambda x: isinstance(x, str)).all():
df[col] = df[col].astype("string")
elif pd.api.types.is_string_dtype(df[col]):
df[col] = df[col].astype("string")
return df


Expand All @@ -65,6 +71,7 @@ def test_dataframe_to_tsfile_basic():
df_sorted = convert_to_nullable_types(
df.sort_values("time").reset_index(drop=True)
)
df_read = convert_to_nullable_types(df_read)

assert df_read.shape == (100, 4)
assert df_read["time"].equals(df_sorted["time"])
Expand Down Expand Up @@ -109,6 +116,7 @@ def test_dataframe_to_tsfile_with_index():

df_read = to_dataframe(tsfile_path, table_name="test_table")
df_read = df_read.sort_values("time").reset_index(drop=True)
df_read = convert_to_nullable_types(df_read)
time_expected = pd.Series(df.index.values, dtype="Int64")
assert df_read.shape == (30, 3)
assert df_read["time"].equals(time_expected)
Expand Down Expand Up @@ -146,6 +154,7 @@ def test_dataframe_to_tsfile_custom_time_column():
df_sorted = convert_to_nullable_types(
df.sort_values("timestamp").reset_index(drop=True)
)
df_read = convert_to_nullable_types(df_read)

assert df_read.shape == (30, 3)
assert df_read["timestamp"].equals(df_sorted["timestamp"])
Expand All @@ -169,6 +178,7 @@ def test_dataframe_to_tsfile_case_insensitive_time():
dataframe_to_tsfile(df, tsfile_path, table_name="test_table")

df_read = to_dataframe(tsfile_path, table_name="test_table")
df_read = convert_to_nullable_types(df_read)
assert df_read.shape == (20, 2)
assert df_read["time"].equals(pd.Series([i for i in range(20)], dtype="Int64"))
finally:
Expand Down Expand Up @@ -200,6 +210,7 @@ def test_dataframe_to_tsfile_with_tag_columns():
df_sorted = convert_to_nullable_types(
df.sort_values("time").reset_index(drop=True)
)
df_read = convert_to_nullable_types(df_read)

assert df_read.shape == (20, 4)
assert df_read["device"].equals(df_sorted["device"])
Expand Down Expand Up @@ -242,6 +253,7 @@ def test_dataframe_to_tsfile_tag_time_unsorted():
df_read = to_dataframe(tsfile_path, table_name="test_table")
df_expected = df.sort_values(by=["device", "time"]).reset_index(drop=True)
df_expected = convert_to_nullable_types(df_expected)
df_read = convert_to_nullable_types(df_read)

assert df_read.shape == (10, 3)
assert df_read["device"].equals(df_expected["device"])
Expand Down Expand Up @@ -281,6 +293,7 @@ def test_dataframe_to_tsfile_all_datatypes():
df_sorted = convert_to_nullable_types(
df.sort_values("time").reset_index(drop=True)
)
df_read = convert_to_nullable_types(df_read)

assert df_read.shape == (50, 11)
assert df_read["bool_col"].equals(df_sorted["bool_col"])
Expand Down
14 changes: 12 additions & 2 deletions python/tsfile/tsfile_table_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,19 @@ def validate_dataframe_for_tsfile(df: pd.DataFrame) -> None:
seen = set()
duplicates = []
for c in columns:
if c is None or (isinstance(c, str) and len(c) == 0):
if isinstance(c, str):
if len(c) == 0:
raise ValueError("Column name cannot be None or empty")
lower = c.lower()
elif c is None:
raise ValueError("Column name cannot be None or empty")
lower = c.lower()
else:
try:
if pd.isna(c):
raise ValueError("Column name cannot be None or empty")
except TypeError:
pass
lower = str(c).lower()
if lower in seen:
duplicates.append(c)
seen.add(lower)
Expand Down
Loading