From 82dd04bf681b4ccc6b5adca1cbdc0d3aa760dcae Mon Sep 17 00:00:00 2001 From: "Murilo M. Marinho" Date: Thu, 1 Oct 2026 08:04:39 +0100 Subject: [PATCH] Build dqrobotics from a pinned submodule, statically, into _core Add dqrobotics/cpp as submodules/dqrobotics_cpp, pinned to 674eae4, the commit the required dqrobotics 26.4.0a7 Python release was built from (its cpp submodule at dqrobotics/python e8f43c1 = v26.04.0 + 7 commits). _core links it statically with hidden visibility, as working-needlemanipulation does; DQ and DQ_SerialManipulator objects are still the dqrobotics package's, through pybind11's shared type registry, so the two copies must keep the same class layouts. sas_cpp is bumped to the branch that uses a parent's dqrobotics::dqrobotics target (MarinhoLab/sas_cpp#12). Its headers no longer bring Eigen's names into scope, so sas_robot_driver_py.cpp uses Eigen::VectorXd explicitly. The workflow no longer installs the DQ Robotics PPA or builds dqrobotics on macOS and Windows, and no longer repairs the macOS and Windows wheels with delocate and delvewheel (auditwheel still sets the manylinux tag). The Docker pipeline drops the PPA and checks dqrobotics interop and that _core links no libdqrobotics. The Eigen include shim is gone: no header of the pinned dqrobotics includes . Co-Authored-By: Claude Opus 5.5 --- .github/workflows/publish.yml | 94 +++++++---------------------------- .gitmodules | 3 ++ CMakeLists.txt | 52 ++++++++++--------- README.md | 33 ++++++------ docker/Dockerfile | 23 +++------ docker/build.sh | 27 ++++++++++ src/sas_robot_driver_py.cpp | 3 ++ submodules/dqrobotics_cpp | 1 + submodules/sas_cpp | 2 +- 9 files changed, 104 insertions(+), 134 deletions(-) create mode 160000 submodules/dqrobotics_cpp diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 917a895..f2e4b52 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,8 +19,7 @@ jobs: os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - # macOS: the oldest macOS the wheel supports (the dqrobotics library that - # delocate bundles into it is built with the same target). + # macOS: the oldest macOS the wheel supports. env: MACOSX_DEPLOYMENT_TARGET: "14.0" @@ -39,47 +38,31 @@ jobs: cache: 'pip' # ── Ubuntu dependencies ───────────────────────────────────── - # libdqrobotics comes from the DQ Robotics PPA; patchelf is a native + # dqrobotics is built from the submodules/dqrobotics_cpp submodule as + # part of the wheel, so only Eigen is needed. patchelf is a native # prerequisite of `auditwheel repair` below. - name: Install system dependencies (Ubuntu) if: runner.os == 'Linux' run: | sudo apt-get update - sudo apt-get install -y build-essential patchelf - sudo apt-get install -y libeigen3-dev - sudo add-apt-repository -y ppa:dqrobotics-dev/development - sudo apt-get update - sudo apt-get install -y libdqrobotics - # dqrobotics headers use without the eigen3/ prefix: - sudo ln -s /usr/include/eigen3/Eigen /usr/include/Eigen + sudo apt-get install -y build-essential patchelf libeigen3-dev # ── macOS dependencies ────────────────────────────────────── - # Eigen from Homebrew, and dqrobotics built from source (there is no - # Homebrew formula) and installed into the runner's temporary directory, - # which the build step passes to CMake. dqrobotics and this package use - # the same Eigen, since Eigen types cross their interface. - name: Install dependencies (macOS) if: runner.os == 'macOS' run: | brew update brew install eigen - git clone --depth 1 https://github.com/dqrobotics/cpp.git dqrobotics/cpp - cmake -S dqrobotics/cpp -B dqrobotics/cpp/build \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH="$(brew --prefix)" \ - -DCMAKE_INSTALL_PREFIX="$RUNNER_TEMP/dqrobotics" - cmake --build dqrobotics/cpp/build -j - cmake --install dqrobotics/cpp/build - echo "CMAKE_ARGS=-DCMAKE_PREFIX_PATH=$(brew --prefix);$RUNNER_TEMP/dqrobotics" >> "$GITHUB_ENV" + echo "CMAKE_ARGS=-DCMAKE_PREFIX_PATH=$(brew --prefix)" >> "$GITHUB_ENV" # The python.org Pythons of setup-python are universal2 builds, so - # setuptools would tag the wheel universal2; the extension and the - # dqrobotics it links are arm64 only. ARCHFLAGS makes setup.py pass - # the architecture to CMake, and _PYTHON_HOST_PLATFORM tags the wheel. + # setuptools would tag the wheel universal2; the extension is built + # for arm64 only. ARCHFLAGS makes setup.py pass the architecture to + # CMake, and _PYTHON_HOST_PLATFORM tags the wheel. echo "ARCHFLAGS=-arch arm64" >> "$GITHUB_ENV" echo "_PYTHON_HOST_PLATFORM=macosx-${MACOSX_DEPLOYMENT_TARGET}-arm64" >> "$GITHUB_ENV" # ── Windows dependencies ──────────────────────────────────── - # dqrobotics/cpp expects the vcpkg toolchain at C:/vcpkg. + # Eigen from vcpkg; CMakeLists.txt uses the vcpkg toolchain at C:/vcpkg. - name: Install vcpkg at C:/vcpkg if: runner.os == 'Windows' run: | @@ -87,38 +70,18 @@ jobs: C:/vcpkg/./bootstrap-vcpkg.bat C:/vcpkg/./vcpkg install eigen3:x64-windows - - name: Build & install dqrobotics (Windows) - if: runner.os == 'Windows' - # '^' line-continuation is cmd.exe syntax (the default runner shell - # is pwsh, where it would be passed literally to cmake). - shell: cmd - run: | - :: Use the DEFAULT (Visual Studio / MSVC) generator so dqrobotics is - :: built with cl.exe, the same compiler the wheel uses. Do NOT use - :: -G Ninja here: on this runner Ninja picks the pre-installed MinGW - :: g++ and produces a libdqrobotics.dll.a that MSVC cannot link. - git clone --depth 1 https://github.com/dqrobotics/cpp.git dqrobotics\cpp - cmake -S dqrobotics\cpp -B dqrobotics\cpp\build ^ - -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake ^ - -DVCPKG_TARGET_TRIPLET=x64-windows ^ - -DCMAKE_INSTALL_PREFIX=C:\dqrobotics - cmake --build dqrobotics\cpp\build --config Release -j - cmake --install dqrobotics\cpp\build --config Release - - # ── Build ─────────────────────────────────────────────────── - # On Windows, point sas_cpp's dqrobotics lookup at C:/dqrobotics. + # ── Build ────────────────────────────────────────────────── # setup.py forwards CMAKE_ARGS to the CMake configure step. - name: Build wheel run: | python -m pip install --upgrade pip pip install build python -m build --wheel - env: - CMAKE_ARGS: ${{ runner.os == 'Windows' && '-DCMAKE_PREFIX_PATH=C:/dqrobotics' || env.CMAKE_ARGS }} # PyPI rejects wheels tagged `linux_x86_64`/`linux_aarch64` (it requires - # a PEP 600 `manylinux` tag). auditwheel relinks/bundles as needed and - # retags the wheel. + # a PEP 600 `manylinux` tag). auditwheel checks the wheel's external + # dependencies (only system libraries: dqrobotics is linked statically) + # and retags it. macOS and Windows wheels need no repair. # - `--plat auto` picks the tightest compatible manylinux tag from the # wheel's actual symbols (the ubuntu-latest glibc varies, so do not # hardcode a `manylinux_2_X` tag). @@ -132,31 +95,6 @@ jobs: auditwheel repair --plat auto -w dist/ dist/*linux_${arch}.whl rm -f dist/*linux_${arch}.whl - # macOS: bundle libdqrobotics.dylib into the wheel (as auditwheel does - # on Linux) and retag it for MACOSX_DEPLOYMENT_TARGET. - - name: Repair wheel (macOS, delocate) - if: runner.os == 'macOS' - run: | - pip install delocate - delocate-wheel -v -w dist/repaired dist/*.whl - rm -f dist/*.whl - mv dist/repaired/*.whl dist/ - rmdir dist/repaired - - # Windows: bundle dqrobotics.dll into the wheel (as auditwheel and - # delocate do on Linux and macOS). Without it, importing _core fails - # with "DLL load failed while importing _core" wherever dqrobotics.dll - # is not on the DLL search path, i.e. on every user's machine. - - name: Repair wheel (Windows, delvewheel) - if: runner.os == 'Windows' - shell: bash - run: | - pip install delvewheel - delvewheel repair --add-path "C:/dqrobotics/bin" -w dist/repaired dist/*.whl - rm -f dist/*.whl - mv dist/repaired/*.whl dist/ - rmdir dist/repaired - # Install the wheel in a clean environment and use it. --no-deps: only # papers.icra2019 needs marinholab-solvers-qpoases, whose wheels do not # cover every platform and Python version; dqrobotics comes from its @@ -178,7 +116,11 @@ jobs: actuation_types=[SM.ActuationType.RX, SM.ActuationType.RY, SM.ActuationType.RZ]) q = np.array([0.1, -0.2, 0.3]) assert np.shape(arm.raw_pose_jacobian(q, 2)) == (8, 3) - print(arm.raw_fkm(q, 2)) + # Inherited DQ_SerialManipulator methods run in the dqrobotics + # package's own copy of the library, on an object built by _core. + assert arm.fkm(q) == arm.raw_fkm(q, 2) + assert np.allclose(arm.pose_jacobian(q), arm.raw_pose_jacobian(q, 2)) + print(arm.fkm(q)) " - name: Upload wheel artifact diff --git a/.gitmodules b/.gitmodules index b7b3924..2aa88b1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,3 +6,6 @@ path = submodules/pybind11 url = https://github.com/pybind/pybind11.git branch = v3.0 +[submodule "submodules/dqrobotics_cpp"] + path = submodules/dqrobotics_cpp + url = https://github.com/dqrobotics/cpp.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 3efc53e..1d46665 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,34 +10,38 @@ project(marinholab_sas_py LANGUAGES CXX) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -# The pybind11 module is self-contained: build marinholab_sas_core as a static -# library and link it into _core, so the resulting extension module has no -# dependency on a separately installed marinholab_sas_core shared library. -# (Matches the working-needlemanipulation approach, where dqrobotics is built -# in-tree as static.) +# The pybind11 module is self-contained: dqrobotics (submodules/dqrobotics_cpp) +# and marinholab_sas_core (submodules/sas_cpp) are built from source as static +# libraries and linked into _core, so the extension module does not depend on +# any separately installed shared library. (This is the working-needlemanipulation +# approach.) The DQ and DQ_Kinematics objects that cross into Python are still +# the dqrobotics Python package's types: pybind11's type registry is shared +# between extension modules, so _core receives and returns the objects of the +# `dqrobotics` package. The two copies of dqrobotics must therefore have the +# same class layouts, which is why the submodule is pinned to the dqrobotics/cpp +# commit that the required `dqrobotics` Python release was built from. set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) set(MARINHO_LAB_SAS_CORE_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) +set(MARINHO_LAB_SAS_CORE_INSTALL OFF CACHE BOOL "" FORCE) +# The static libraries end up in a shared object (_core), so they need PIC. +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +# Keep their symbols private to _core (pybind11 already builds _core itself +# with hidden visibility), so that they never interpose with the copy of +# dqrobotics inside the `dqrobotics` Python package's extension module. +set(CMAKE_CXX_VISIBILITY_PRESET hidden) +set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) -# The dqrobotics headers include both and . -# The Eigen3 package provides /include/eigen3, which resolves the first; -# the second needs /include, which is on the default search path on -# Linux (/usr/include) and Windows (the vcpkg include root) but not on macOS -# (Homebrew's /opt/homebrew/include). Rather than adding all of that prefix -# (which could shadow, e.g., the pybind11 submodule with a Homebrew pybind11), -# expose only Eigen: a directory in the build tree holding an "eigen3" link to -# Eigen's include directory. include_directories applies to sas_cpp as well, -# since it is added below. find_package(Eigen3 REQUIRED) -if(NOT WIN32) - get_target_property(_eigen3_include_dirs Eigen3::Eigen INTERFACE_INCLUDE_DIRECTORIES) - list(GET _eigen3_include_dirs 0 _eigen3_include_dir) - set(_eigen3_shim "${CMAKE_BINARY_DIR}/eigen3_include_shim") - file(MAKE_DIRECTORY "${_eigen3_shim}") - if(NOT EXISTS "${_eigen3_shim}/eigen3") - file(CREATE_LINK "${_eigen3_include_dir}" "${_eigen3_shim}/eigen3" SYMBOLIC) - endif() - include_directories(SYSTEM "${_eigen3_shim}") -endif() + +# dqrobotics, as a static library. EXCLUDE_FROM_ALL: only what _core links is +# built. Its warning flags are PUBLIC compile options, which would propagate to +# everything that links it; they are for dqrobotics' own development, so drop +# them from its interface. +add_subdirectory(submodules/dqrobotics_cpp EXCLUDE_FROM_ALL) +set_property(TARGET dqrobotics PROPERTY INTERFACE_COMPILE_OPTIONS "") +# sas_cpp links dqrobotics::dqrobotics, and uses this target instead of looking +# for an installed dqrobotics when it is defined. +add_library(dqrobotics::dqrobotics ALIAS dqrobotics) add_subdirectory(submodules/sas_cpp) diff --git a/README.md b/README.md index 759eca9..a53bb41 100644 --- a/README.md +++ b/README.md @@ -23,13 +23,18 @@ - `example_*.py` — example scripts (also installed as commands). - `src/` — the C++ binding sources (ported from `SmartArmStack/sas_core`). - `submodules/sas_cpp` — the C++ core (git submodule, consumed via CMake). +- `submodules/dqrobotics_cpp` — the dqrobotics C++ library (git submodule, + pinned to the commit the required `dqrobotics` Python release is built from). - `submodules/pybind11` — pybind11 (git submodule, pinned to v3.0.4). - `docker/` — `ubuntu:noble` build environment and full test pipeline. -The C++ core depends on **Eigen3** and **dqrobotics** (the latter is located -by CMake at build time; on Debian/Ubuntu it comes from the DQ Robotics PPA). -The Python package additionally depends on the `dqrobotics` Python package, -which registers the dual-quaternion types the bindings expose. +The C++ core depends on **Eigen3** and **dqrobotics**. dqrobotics is compiled +from the `submodules/dqrobotics_cpp` submodule and linked statically into the +extension module, together with the C++ core, so no dqrobotics library needs +to be installed. The Python package additionally depends on the `dqrobotics` +Python package, which registers the dual-quaternion and robot-model types the +bindings expose (pybind11 shares them between the two extension modules, so +the submodule is kept at the same dqrobotics/cpp commit as that release). ## Installation @@ -51,29 +56,23 @@ cd sas_py pip install . --no-build-isolation ``` -Building requires `cmake` (>= 3.16), `ninja`, a C++17 compiler, Eigen3, and -`dqrobotics`. On Debian/Ubuntu: +Building requires `cmake` (>= 3.16), `ninja`, a C++17 compiler and Eigen3. +On Debian/Ubuntu: ```bash sudo apt install build-essential g++ cmake ninja-build python3-dev libeigen3-dev -# dqrobotics from the DQ Robotics PPA: -sudo add-apt-repository -y ppa:dqrobotics-dev/development -sudo apt install libdqrobotics ``` -On macOS, Eigen comes from Homebrew and `dqrobotics` is built from source -(there is no Homebrew formula), with the same Eigen: +On macOS, with Eigen from Homebrew: ```bash brew install eigen cmake ninja -git clone --depth 1 https://github.com/dqrobotics/cpp.git dqrobotics-cpp -cmake -S dqrobotics-cpp -B dqrobotics-cpp/build -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH="$(brew --prefix)" -DCMAKE_INSTALL_PREFIX="$HOME/.local/dqrobotics" -cmake --build dqrobotics-cpp/build -j && cmake --install dqrobotics-cpp/build -CMAKE_ARGS="-DCMAKE_PREFIX_PATH=$(brew --prefix);$HOME/.local/dqrobotics" \ - pip install . --no-build-isolation +CMAKE_ARGS="-DCMAKE_PREFIX_PATH=$(brew --prefix)" pip install . --no-build-isolation ``` +On Windows, Eigen comes from vcpkg, expected at `C:/vcpkg` +(`vcpkg install eigen3:x64-windows`). + ## Usage ```python diff --git a/docker/Dockerfile b/docker/Dockerfile index 6fed6f7..fefd95b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,34 +5,25 @@ SHELL ["/bin/bash", "-c"] ENV DEBIAN_FRONTEND=noninteractive -# Build tooling, C++ dependencies and the Python toolchain. -# libdqrobotics (headers + shared library) is not in the Ubuntu repositories; -# it is provided by the DQ Robotics PPA (noble, amd64/arm64). +# Build tooling, C++ dependencies and the Python toolchain. dqrobotics is +# built from the submodules/dqrobotics_cpp submodule as part of the package, +# so only Eigen is needed from the system. RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential g++ cmake make pkg-config ninja-build \ python3-dev python3-pip \ libeigen3-dev \ - software-properties-common ca-certificates \ + ca-certificates \ && rm -rf /var/lib/apt/lists/* -RUN add-apt-repository -y ppa:dqrobotics-dev/development \ - && apt-get update \ - && apt-get install -y --no-install-recommends libdqrobotics \ - && rm -rf /var/lib/apt/lists/* - -# The dqrobotics headers use (without the eigen3/ prefix); -# provide the conventional include path for them. -RUN ln -s /usr/include/eigen3/Eigen /usr/include/Eigen - # Build-time Python packages (used with `pip install --no-build-isolation`). RUN python3 -m pip install --no-cache-dir --break-system-packages \ setuptools wheel ninja setuptools-git-versioning WORKDIR /opt/sas_py # `COPY .` includes the populated git-submodule trees (submodules/sas_cpp, -# submodules/pybind11) from the build context. No `git submodule update` is -# needed inside the image because CMake consumes them as plain source trees -# (via add_subdirectory), not via git. +# submodules/dqrobotics_cpp, submodules/pybind11) from the build context. No +# `git submodule update` is needed inside the image because CMake consumes them +# as plain source trees (via add_subdirectory), not via git. COPY . /opt/sas_py CMD ["bash", "/opt/sas_py/docker/build.sh"] diff --git a/docker/build.sh b/docker/build.sh index d07a742..165280b 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -62,6 +62,25 @@ assert max_err < 1e-6, f"Jacobian/FDM mismatch: {max_err}" assert np.allclose(m.get_lower_q_limit(), -10.0) assert np.allclose(m.get_upper_q_limit(), 10.0) +# dqrobotics interop: _core links its own static copy of dqrobotics, while +# DQ and DQ_SerialManipulator are the `dqrobotics` package's Python types. +# Methods inherited from DQ_SerialManipulator run in the dqrobotics package's +# copy, on an object built by _core, and dqrobotics' functions accept it. +from dqrobotics import i_, k_, E_ +from dqrobotics.robot_modeling import DQ_Kinematics +x = m.fkm(q) +assert isinstance(x, DQ) and x == m.raw_fkm(q, 2), x +Jx = m.pose_jacobian(q) +assert np.allclose(Jx, J), "fkm/pose_jacobian via dqrobotics differ from raw_*" +assert m.get_dim_configuration_space() == 3 +base = 1 + 0.5 * E_ * k_ +m.set_base_frame(base) +m.set_reference_frame(base) +assert m.get_base_frame() == base +assert np.allclose(m.fkm(q).vec8(), (base * m.raw_fkm(q, 2)).vec8()) +Jt = DQ_Kinematics.translation_jacobian(m.pose_jacobian(q), m.fkm(q)) +assert Jt.shape == (4, 3), Jt.shape + print("import + API OK (incl. modeling, FD Jacobian err=%.2e)" % max_err) EOF @@ -70,6 +89,14 @@ sas_core_clock_example sas_core_clock_sched_fifo_example sas_core_robot_driver_subclass_example +echo "=== _core links no dqrobotics shared library ===" +core_so=$(python3 -c "import marinholab.sas.core._core as c; print(c.__file__)") +ldd "$core_so" +if ldd "$core_so" | grep -qi dqrobotics; then + echo "_core depends on a dqrobotics shared library" >&2 + exit 1 +fi + echo "=== Verify installed files ===" python3 - <<'EOF' import marinholab.sas.core, os diff --git a/src/sas_robot_driver_py.cpp b/src/sas_robot_driver_py.cpp index 74611a4..76e489b 100644 --- a/src/sas_robot_driver_py.cpp +++ b/src/sas_robot_driver_py.cpp @@ -35,6 +35,9 @@ namespace marinholab::sas::core { +// The sas_cpp headers no longer bring Eigen's names into scope. +using Eigen::VectorXd; + //https://pybind11.readthedocs.io/en/stable/advanced/classes.html //Trampoline class class RobotDriverPy: public RobotDriver, public py::trampoline_self_life_support diff --git a/submodules/dqrobotics_cpp b/submodules/dqrobotics_cpp new file mode 160000 index 0000000..674eae4 --- /dev/null +++ b/submodules/dqrobotics_cpp @@ -0,0 +1 @@ +Subproject commit 674eae4d8804baca6dc22bbbe3afc3b323c05cdf diff --git a/submodules/sas_cpp b/submodules/sas_cpp index 9926db9..b25253b 160000 --- a/submodules/sas_cpp +++ b/submodules/sas_cpp @@ -1 +1 @@ -Subproject commit 9926db96528c5d7e4c338c1f490c590aca745e33 +Subproject commit b25253b60d130914aa1328d0e28d4ebc814caa16