From 7474040f577882d0930f3890e07858127db9119e Mon Sep 17 00:00:00 2001 From: Hongzhi Gao Date: Tue, 9 Jun 2026 15:13:14 +0800 Subject: [PATCH 1/3] Fix Windows CI failures for the Python wheel build. (#831) * fix(cpp): remove FORCE_INLINE from AndFilter/OrFilter for MinGW build GCC on Windows CI fails when always_inline virtual methods exceed the max-inline-insns-single limit during expression.cc compilation. * fix(python): preload MinGW runtime DLLs on Windows wheel import Restore MinGW runtime preloading in __init__.py removed during MSVC support. Improve setup.py to locate MinGW DLLs via MSYSTEM_PREFIX and harden wheels.yml Windows build/verify steps. * fix(python): fix MinGW .pyd linking and skip tests during wheel build Link gcc_s/stdc++ before libtsfile on MinGW so _Unwind_Resume is not imported from libtsfile.dll (WinError 127). Honor -DskipTests in exec-maven-plugin for pytest, and add -Dmaven.test.skip=true in wheels CI. * Fix Windows wheel MinGW runtime packaging * Remove unrelated setup.py version bump * Fix cpp formatting and sync Python version * Fix MinGW filter inlining for rc build * Avoid forced inline for nested time filters * Avoid forced inline for nested statistic filters --- .github/workflows/wheels.yml | 25 +++++++++++++++++---- cpp/CMakeLists.txt | 2 +- cpp/src/reader/filter/and_filter.h | 10 ++++----- cpp/src/reader/filter/or_filter.h | 10 ++++----- python/pom.xml | 2 ++ python/pyproject.toml | 2 +- python/setup.py | 35 +++++++++++++++++++++++++----- python/tsfile/__init__.py | 23 +++++++++++++++++--- 8 files changed, 83 insertions(+), 26 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 148266ce9..104e36433 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -206,6 +206,9 @@ jobs: -Dspotless.check.skip=true -Dspotless.apply.skip=true test -d cpp/target/build/lib test -d cpp/target/build/include + cp /mingw64/bin/libstdc++-6.dll cpp/target/build/lib/ + cp /mingw64/bin/libgcc_s_seh-1.dll cpp/target/build/lib/ + cp /mingw64/bin/libwinpthread-1.dll cpp/target/build/lib/ - name: Upload C++ build output uses: actions/upload-artifact@v7 @@ -257,13 +260,25 @@ jobs: name: tsfile-cpp-windows path: cpp/target/build/ + - name: Verify C++ artifacts exist + shell: msys2 {0} + run: | + set -euxo pipefail + ls -laR cpp/target/build/lib/ + ls -laR cpp/target/build/include/ + test -f cpp/target/build/lib/libtsfile.dll + test -f cpp/target/build/lib/libtsfile.dll.a + test -f cpp/target/build/lib/libstdc++-6.dll + test -f cpp/target/build/lib/libgcc_s_seh-1.dll + test -f cpp/target/build/lib/libwinpthread-1.dll + - name: Build wheel shell: msys2 {0} run: | set -euxo pipefail export JAVA_HOME="$(cygpath "$JAVA_HOME")" export PYTHON_HOME="$(cygpath "$pythonLocation")" - export PATH="$PYTHON_HOME:$PYTHON_HOME/Scripts:$JAVA_HOME/bin:$PATH" + export PATH="/mingw64/bin:$PYTHON_HOME:$PYTHON_HOME/Scripts:$JAVA_HOME/bin:$PATH" # Build wheel via Maven (no clean — keep C++ artifacts from previous job) chmod +x mvnw || true @@ -272,14 +287,17 @@ jobs: case "${{ matrix.python-version }}" in 3.9) BLACK_VER_PROP="-Dblack.version=25.11.0" ;; esac - ../mvnw package -DskipTests \ + ../mvnw package -DskipTests -Dmaven.test.skip=true \ -Dspotless.check.skip=true -Dspotless.apply.skip=true \ ${BLACK_VER_PROP} ls -la dist/ - name: Verify wheel - shell: bash + shell: msys2 {0} run: | + set -euxo pipefail + export PYTHON_HOME="$(cygpath "$pythonLocation")" + export PATH="$PYTHON_HOME:$PYTHON_HOME/Scripts:$PATH" python -m pip install python/dist/*.whl python -c "import tsfile, tsfile.tsfile_reader as r; print('import-ok')" @@ -288,4 +306,3 @@ jobs: with: name: tsfile-wheels-windows-py${{ matrix.python-version }} path: python/dist/*.whl - diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 98d93fcfe..a3b1d6214 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -29,7 +29,7 @@ endif () if (POLICY CMP0079) cmake_policy(SET CMP0079 NEW) endif () -set(TsFile_CPP_VERSION 2.2.1.dev) +set(TsFile_CPP_VERSION 2.3.1) if (MSVC) # MSVC does not provide a /std:c++11 flag; C++11 is its implicit baseline. diff --git a/cpp/src/reader/filter/and_filter.h b/cpp/src/reader/filter/and_filter.h index 0d01000f8..b324a3f81 100644 --- a/cpp/src/reader/filter/and_filter.h +++ b/cpp/src/reader/filter/and_filter.h @@ -30,22 +30,20 @@ class AndFilter : public BinaryFilter { ~AndFilter() {} AndFilter(Filter* left, Filter* right) : BinaryFilter(left, right) {} - FORCE_INLINE bool satisfy(Statistic* statistic) { + bool satisfy(Statistic* statistic) { return left_->satisfy(statistic) && right_->satisfy(statistic); } - FORCE_INLINE bool satisfy(int64_t time, int64_t value) { + bool satisfy(int64_t time, int64_t value) { return left_->satisfy(time, value) && right_->satisfy(time, value); } - FORCE_INLINE bool satisfy_start_end_time(int64_t start_time, - int64_t end_time) { + bool satisfy_start_end_time(int64_t start_time, int64_t end_time) { return left_->satisfy_start_end_time(start_time, end_time) && right_->satisfy_start_end_time(start_time, end_time); } - FORCE_INLINE bool contain_start_end_time(int64_t start_time, - int64_t end_time) { + bool contain_start_end_time(int64_t start_time, int64_t end_time) { return left_->contain_start_end_time(start_time, end_time) && right_->contain_start_end_time(start_time, end_time); } diff --git a/cpp/src/reader/filter/or_filter.h b/cpp/src/reader/filter/or_filter.h index 1d4aa6aa7..fc8d4a2cf 100644 --- a/cpp/src/reader/filter/or_filter.h +++ b/cpp/src/reader/filter/or_filter.h @@ -30,22 +30,20 @@ class OrFilter : public BinaryFilter { OrFilter(Filter* left, Filter* right) : BinaryFilter(left, right) {} ~OrFilter() {} - FORCE_INLINE bool satisfy(Statistic* statistic) { + bool satisfy(Statistic* statistic) { return left_->satisfy(statistic) || right_->satisfy(statistic); } - FORCE_INLINE bool satisfy(int64_t time, int64_t value) { + bool satisfy(int64_t time, int64_t value) { return left_->satisfy(time, value) || right_->satisfy(time, value); } - FORCE_INLINE bool satisfy_start_end_time(int64_t start_time, - int64_t end_time) { + bool satisfy_start_end_time(int64_t start_time, int64_t end_time) { return left_->satisfy_start_end_time(start_time, end_time) || right_->satisfy_start_end_time(start_time, end_time); } - FORCE_INLINE bool contain_start_end_time(int64_t start_time, - int64_t end_time) { + bool contain_start_end_time(int64_t start_time, int64_t end_time) { return left_->contain_start_end_time(start_time, end_time) || right_->contain_start_end_time(start_time, end_time); } diff --git a/python/pom.xml b/python/pom.xml index fb773711a..933976c20 100644 --- a/python/pom.xml +++ b/python/pom.xml @@ -142,6 +142,7 @@ exec + ${skipTests:-false} ${python.venv.bin}${python.exe.bin} -m @@ -158,6 +159,7 @@ exec + ${skipTests:-false} ${python.venv.bin}pytest ${project.basedir}/tests diff --git a/python/pyproject.toml b/python/pyproject.toml index 3a2789853..920f0a6ef 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -28,7 +28,7 @@ build-backend = "setuptools.build_meta" [project] name = "tsfile" -version = "2.2.1.dev" +version = "2.3.1" requires-python = ">=3.9" description = "TsFile Python" readme = {file = "README.md", content-type = "text/markdown"} diff --git a/python/setup.py b/python/setup.py index 19ada7921..08ca44044 100644 --- a/python/setup.py +++ b/python/setup.py @@ -30,7 +30,7 @@ ROOT = Path(__file__).parent.resolve() PKG = ROOT / "tsfile" -version = "2.2.1.dev" +version = "2.3.1" def _find_cpp_build(): @@ -151,19 +151,40 @@ def _find_lib(root, patterns): # Copy MinGW runtime DLLs next to tsfile.dll so Python can find them. # Python 3.8+ does not search PATH for DLLs; they must sit in the # same directory as the .pyd extensions (os.add_dll_directory). + # Prefer runtime DLLs captured from the same MSYS2 environment that + # built libtsfile.dll. Mixing MinGW runtime DLLs from a different + # install can load successfully but fail later with WinError 127 when + # libtsfile.dll imports a newer runtime symbol. + _mingw_search_dirs = [CPP_LIB] + for _dir in os.environ.get("PATH", "").split(os.pathsep): + if _dir: + _mingw_search_dirs.append(Path(_dir)) + _msys_prefix = os.environ.get("MSYSTEM_PREFIX") + if _msys_prefix: + _mingw_search_dirs.append(Path(_msys_prefix) / "bin") + for _extra in ( + Path("C:/msys64/mingw64/bin"), + Path("C:/ProgramData/mingw64/mingw64/bin"), + ): + if _extra.is_dir(): + _mingw_search_dirs.append(_extra) + for _mingw_dll in ( "libstdc++-6.dll", "libgcc_s_seh-1.dll", "libwinpthread-1.dll", ): - for _dir in os.environ.get("PATH", "").split(os.pathsep): - _src = Path(_dir) / _mingw_dll + for _dir in _mingw_search_dirs: + _src = _dir / _mingw_dll if _src.is_file(): shutil.copy2(_src, PKG / _mingw_dll) print(f"setup.py: copied {_mingw_dll} from {_src}") break else: - print(f"setup.py: WARNING - {_mingw_dll} not found on PATH") + raise FileNotFoundError( + f"setup.py: MinGW runtime DLL {_mingw_dll} not found; " + "ensure mingw-w64-x86_64-gcc is on PATH or MSYSTEM_PREFIX is set" + ) else: raise RuntimeError(f"Unsupported platform: {sys.platform}") @@ -197,8 +218,11 @@ def finalize_options(self): extra_compile_args += ["-O3", "-std=c++11", "-fvisibility=hidden", "-fPIC"] extra_link_args += ["-Wl,-rpath,@loader_path", "-stdlib=libc++"] elif sys.platform == "win32": - libraries = ["tsfile"] if win_toolchain == "mingw": + # Resolve EH/runtime symbols from MinGW runtimes before libtsfile.dll.a. + # Otherwise _Unwind_Resume is recorded against libtsfile.dll and loading + # the .pyd fails with WinError 127 on Windows. + libraries = ["gcc_s", "stdc++", "tsfile"] extra_compile_args += [ "-O2", "-std=c++11", @@ -208,6 +232,7 @@ def finalize_options(self): "-D_WIN64", ] else: # msvc + libraries = ["tsfile"] # cl.exe rejects the GCC-style flags above. Mirror the options the # C++ module itself is built with (see cpp/CMakeLists.txt). C++17 is # required because Cython 3 emits inline variables (error C7525); the diff --git a/python/tsfile/__init__.py b/python/tsfile/__init__.py index 1026ec673..ac8b6b853 100644 --- a/python/tsfile/__init__.py +++ b/python/tsfile/__init__.py @@ -22,15 +22,32 @@ _pkg_dir = os.path.dirname(os.path.abspath(__file__)) + +def _preload_dll(path): + if not os.path.isfile(path): + return + try: + ctypes.CDLL(path) + except OSError: + pass + + if sys.platform == "win32": # Keep the handle alive for the lifetime of this module. CPython's reference # counting frees the object immediately if not stored, which calls # RemoveDllDirectory and undoes the registration before any .pyd is loaded. _dll_dir = os.add_dll_directory(_pkg_dir) - # Preload the tsfile DLL so Windows finds it by base-name when loading the - # Cython extensions. Store the handle to prevent the DLL from being - # unloaded prematurely. + # Preload MinGW runtime DLLs before libtsfile.dll. Python 3.8+ does not + # search PATH for DLL dependencies; they must live next to the extensions. + for _mingw_dll in ( + "libwinpthread-1.dll", + "libgcc_s_seh-1.dll", + "libstdc++-6.dll", + ): + _preload_dll(os.path.join(_pkg_dir, _mingw_dll)) + # Preload the tsfile DLL so Windows finds it when loading Cython extensions. # MSVC builds produce "tsfile.dll"; MinGW builds produce "libtsfile.dll". + _tsfile_cdll = None for _dll_name in ("tsfile.dll", "libtsfile.dll"): _tsfile_dll = os.path.join(_pkg_dir, _dll_name) if os.path.isfile(_tsfile_dll): From c02fe1483037bbb3c2a84fbbd6e4b73fc89d0fa3 Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Tue, 9 Jun 2026 18:23:11 +0800 Subject: [PATCH 2/3] Sync C++/Python version to 2.3.2.dev with Maven 2.3.2-SNAPSHOT --- cpp/CMakeLists.txt | 2 +- python/pyproject.toml | 2 +- python/setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a3b1d6214..55fa0f38c 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -29,7 +29,7 @@ endif () if (POLICY CMP0079) cmake_policy(SET CMP0079 NEW) endif () -set(TsFile_CPP_VERSION 2.3.1) +set(TsFile_CPP_VERSION 2.3.2.dev) if (MSVC) # MSVC does not provide a /std:c++11 flag; C++11 is its implicit baseline. diff --git a/python/pyproject.toml b/python/pyproject.toml index 920f0a6ef..6a60de0a2 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -28,7 +28,7 @@ build-backend = "setuptools.build_meta" [project] name = "tsfile" -version = "2.3.1" +version = "2.3.2.dev" requires-python = ">=3.9" description = "TsFile Python" readme = {file = "README.md", content-type = "text/markdown"} diff --git a/python/setup.py b/python/setup.py index 08ca44044..d8e9a502f 100644 --- a/python/setup.py +++ b/python/setup.py @@ -30,7 +30,7 @@ ROOT = Path(__file__).parent.resolve() PKG = ROOT / "tsfile" -version = "2.3.1" +version = "2.3.2.dev" def _find_cpp_build(): From 9f1b4c9ddf243b1cdec0172b25a3980a70a5d07f Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Tue, 9 Jun 2026 16:37:04 +0800 Subject: [PATCH 3/3] Fix C example string duplication for Linux wheels --- cpp/examples/c_examples/demo_write.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/cpp/examples/c_examples/demo_write.c b/cpp/examples/c_examples/demo_write.c index 925e3af8c..cf5ffea28 100644 --- a/cpp/examples/c_examples/demo_write.c +++ b/cpp/examples/c_examples/demo_write.c @@ -24,6 +24,15 @@ #include "c_examples.h" +static char* duplicate_string(const char* src) { + size_t len = strlen(src) + 1; + char* dst = (char*)malloc(len); + if (dst != NULL) { + memcpy(dst, src, len); + } + return dst; +} + // This example shows you how to write tsfile. ERRNO write_tsfile() { ERRNO code = 0; @@ -39,20 +48,20 @@ ERRNO write_tsfile() { // Create table schema to describe a table in a tsfile. TableSchema table_schema; - table_schema.table_name = strdup(table_name); + table_schema.table_name = duplicate_string(table_name); table_schema.column_num = 3; table_schema.column_schemas = (ColumnSchema*)malloc(sizeof(ColumnSchema) * 3); table_schema.column_schemas[0] = - (ColumnSchema){.column_name = strdup("id1"), + (ColumnSchema){.column_name = duplicate_string("id1"), .data_type = TS_DATATYPE_STRING, .column_category = TAG}; table_schema.column_schemas[1] = - (ColumnSchema){.column_name = strdup("id2"), + (ColumnSchema){.column_name = duplicate_string("id2"), .data_type = TS_DATATYPE_STRING, .column_category = TAG}; table_schema.column_schemas[2] = - (ColumnSchema){.column_name = strdup("s1"), + (ColumnSchema){.column_name = duplicate_string("s1"), .data_type = TS_DATATYPE_INT32, .column_category = FIELD}; @@ -98,4 +107,4 @@ ERRNO write_tsfile() { free_write_file(&file); return 0; -} \ No newline at end of file +}