diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index a506e3ae0..84af04f42 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -63,7 +63,7 @@ jobs: - name: Install build tools run: | python -m pip install -U pip wheel - python -m pip install cibuildwheel==2.21.3 + python -m pip install cibuildwheel==3.4.1 - name: Pre-download virtualenv for cibuildwheel run: | @@ -108,8 +108,8 @@ jobs: CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" CIBW_SKIP: "pp* *-musllinux*" - CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014" - CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux2014" + CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28" + CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28" MACOSX_DEPLOYMENT_TARGET: "12.0" @@ -222,7 +222,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - name: Checkout uses: actions/checkout@v4 @@ -268,8 +268,13 @@ jobs: # Build wheel via Maven (no clean — keep C++ artifacts from previous job) chmod +x mvnw || true cd python + BLACK_VER_PROP="" + case "${{ matrix.python-version }}" in + 3.9) BLACK_VER_PROP="-Dblack.version=25.11.0" ;; + esac ../mvnw package -DskipTests \ - -Dspotless.check.skip=true -Dspotless.apply.skip=true + -Dspotless.check.skip=true -Dspotless.apply.skip=true \ + ${BLACK_VER_PROP} ls -la dist/ - name: Verify wheel diff --git a/python/pyproject.toml b/python/pyproject.toml index 476f91231..403b1fa49 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -37,9 +37,11 @@ maintainers = [ ] dependencies = [ "numpy>=2.0.0,<3", - "pandas>=2.0", + "pandas>=2.0,<2.3; python_full_version < '3.14.0'", + "pandas>=2.3.3; python_full_version >= '3.14.0'", "pyarrow>=16.0,<18; python_version<'3.10'", - "pyarrow>=18.0,<20; python_version>='3.10'" + "pyarrow>=18.0,<20; python_version>='3.10' and python_version<'3.14'", + "pyarrow>=22.0,<24; python_version>='3.14'" ] [project.urls] diff --git a/python/requirements.txt b/python/requirements.txt index 54e32aa9d..8733d9887 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -18,9 +18,11 @@ # cython==3.0.10 -black==26.3.1 +black==25.11.0; python_version < "3.10" +black==26.3.1; python_version >= "3.10" numpy>=2.0.0,<3 -pandas==2.2.2 +pandas==2.2.2; python_full_version < "3.14.0" +pandas>=2.3.3; python_full_version >= "3.14.0" setuptools==78.1.1 wheel==0.46.2 pyarrow>=8.0.0 diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index e376add7a..aecaa877c 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -51,6 +51,13 @@ def convert_to_nullable_types(df): df[col] = df[col].astype("Float32") elif dtype == "bool": df[col] = df[col].astype("boolean") + elif pd.api.types.is_object_dtype(df[col]): + non_null = df[col].dropna() + if len(non_null) and non_null.map(lambda x: isinstance(x, str)).all(): + df[col] = df[col].astype("string") + elif pd.api.types.is_string_dtype(df[col]): + # NumPy/pandas str dtype (e.g. Py3.14) vs object: unify for Series.equals + df[col] = df[col].astype("string") return df @@ -92,6 +99,7 @@ def test_write_dataframe_basic(): df_sorted = convert_to_nullable_types( df.sort_values("time").reset_index(drop=True) ) + df_read = convert_to_nullable_types(df_read) assert df_read.shape == (100, 4) assert df_read[TIME_COLUMN].equals(df_sorted["time"]) assert df_read["device"].equals(df_sorted["device"]) @@ -128,6 +136,7 @@ def test_write_dataframe_with_index(): df_read = df_read.sort_values(TIME_COLUMN).reset_index(drop=True) df_sorted = df.sort_index() df_sorted = convert_to_nullable_types(df_sorted.reset_index(drop=True)) + df_read = convert_to_nullable_types(df_read) time_series = pd.Series(df.sort_index().index.values, dtype="Int64") assert df_read.shape == (50, 3) assert df_read[TIME_COLUMN].equals(time_series) @@ -166,6 +175,7 @@ def test_write_dataframe_case_insensitive(): df_sorted = convert_to_nullable_types( df.sort_values("Time").reset_index(drop=True) ) + df_read = convert_to_nullable_types(df_read) assert df_read.shape == (30, 3) assert df_read[TIME_COLUMN].equals(df_sorted["Time"]) assert df_read["device"].equals(df_sorted["Device"]) @@ -270,6 +280,7 @@ def test_write_dataframe_all_datatypes(): df_sorted = convert_to_nullable_types( df.sort_values("time").reset_index(drop=True) ) + df_read = convert_to_nullable_types(df_read) assert df_read.shape == (50, 11) assert df_read["bool_col"].equals(df_sorted["bool_col"]) assert df_read["int32_col"].equals(df_sorted["int32_col"]) @@ -317,6 +328,7 @@ def test_write_dataframe_schema_time_column(): df_sorted = convert_to_nullable_types( df.sort_values("time").reset_index(drop=True) ) + df_read = convert_to_nullable_types(df_read) assert df_read.shape == (50, 3) assert df_read[TIME_COLUMN].equals(df_sorted[TIME_COLUMN]) assert df_read["device"].equals(df_sorted["device"]) @@ -354,6 +366,7 @@ def test_write_dataframe_schema_time_and_dataframe_time(): df_sorted = convert_to_nullable_types( df.sort_values("Time").rename(columns=str.lower).reset_index(drop=True) ) + df_read = convert_to_nullable_types(df_read) assert df_read.shape == (30, 3) assert df_read["time"].equals(df_sorted["time"]) assert df_read["device"].equals(df_sorted["device"]) diff --git a/python/tests/test_to_tsfile.py b/python/tests/test_to_tsfile.py index 446403867..147b0bea8 100644 --- a/python/tests/test_to_tsfile.py +++ b/python/tests/test_to_tsfile.py @@ -40,6 +40,12 @@ def convert_to_nullable_types(df): df[col] = df[col].astype("Float32") elif dtype == "bool": df[col] = df[col].astype("boolean") + elif pd.api.types.is_object_dtype(df[col]): + non_null = df[col].dropna() + if len(non_null) and non_null.map(lambda x: isinstance(x, str)).all(): + df[col] = df[col].astype("string") + elif pd.api.types.is_string_dtype(df[col]): + df[col] = df[col].astype("string") return df @@ -65,6 +71,7 @@ def test_dataframe_to_tsfile_basic(): df_sorted = convert_to_nullable_types( df.sort_values("time").reset_index(drop=True) ) + df_read = convert_to_nullable_types(df_read) assert df_read.shape == (100, 4) assert df_read["time"].equals(df_sorted["time"]) @@ -109,6 +116,7 @@ def test_dataframe_to_tsfile_with_index(): df_read = to_dataframe(tsfile_path, table_name="test_table") df_read = df_read.sort_values("time").reset_index(drop=True) + df_read = convert_to_nullable_types(df_read) time_expected = pd.Series(df.index.values, dtype="Int64") assert df_read.shape == (30, 3) assert df_read["time"].equals(time_expected) @@ -146,6 +154,7 @@ def test_dataframe_to_tsfile_custom_time_column(): df_sorted = convert_to_nullable_types( df.sort_values("timestamp").reset_index(drop=True) ) + df_read = convert_to_nullable_types(df_read) assert df_read.shape == (30, 3) assert df_read["timestamp"].equals(df_sorted["timestamp"]) @@ -169,6 +178,7 @@ def test_dataframe_to_tsfile_case_insensitive_time(): dataframe_to_tsfile(df, tsfile_path, table_name="test_table") df_read = to_dataframe(tsfile_path, table_name="test_table") + df_read = convert_to_nullable_types(df_read) assert df_read.shape == (20, 2) assert df_read["time"].equals(pd.Series([i for i in range(20)], dtype="Int64")) finally: @@ -200,6 +210,7 @@ def test_dataframe_to_tsfile_with_tag_columns(): df_sorted = convert_to_nullable_types( df.sort_values("time").reset_index(drop=True) ) + df_read = convert_to_nullable_types(df_read) assert df_read.shape == (20, 4) assert df_read["device"].equals(df_sorted["device"]) @@ -242,6 +253,7 @@ def test_dataframe_to_tsfile_tag_time_unsorted(): df_read = to_dataframe(tsfile_path, table_name="test_table") df_expected = df.sort_values(by=["device", "time"]).reset_index(drop=True) df_expected = convert_to_nullable_types(df_expected) + df_read = convert_to_nullable_types(df_read) assert df_read.shape == (10, 3) assert df_read["device"].equals(df_expected["device"]) @@ -281,6 +293,7 @@ def test_dataframe_to_tsfile_all_datatypes(): df_sorted = convert_to_nullable_types( df.sort_values("time").reset_index(drop=True) ) + df_read = convert_to_nullable_types(df_read) assert df_read.shape == (50, 11) assert df_read["bool_col"].equals(df_sorted["bool_col"]) diff --git a/python/tsfile/tsfile_table_writer.py b/python/tsfile/tsfile_table_writer.py index acc5fefb1..9f3a257e6 100644 --- a/python/tsfile/tsfile_table_writer.py +++ b/python/tsfile/tsfile_table_writer.py @@ -34,9 +34,19 @@ def validate_dataframe_for_tsfile(df: pd.DataFrame) -> None: seen = set() duplicates = [] for c in columns: - if c is None or (isinstance(c, str) and len(c) == 0): + if isinstance(c, str): + if len(c) == 0: + raise ValueError("Column name cannot be None or empty") + lower = c.lower() + elif c is None: raise ValueError("Column name cannot be None or empty") - lower = c.lower() + else: + try: + if pd.isna(c): + raise ValueError("Column name cannot be None or empty") + except TypeError: + pass + lower = str(c).lower() if lower in seen: duplicates.append(c) seen.add(lower)