From 7100e66a23eb114315b038db65349fbaf2c8708f Mon Sep 17 00:00:00 2001 From: ouyangshun <1246787391@qq.com> Date: Tue, 17 Nov 2026 16:58:11 +0800 Subject: [PATCH 01/11] fix(scheduler): release retired commands outside the scheduler lock Retained callbacks can destroy a backend and acquire the runtime lock. Release completed and discarded requests after unlocking to remove the inverse order with runtime send observers. Keep completion allocation-free. --- src/hand/unified/command_scheduler.cpp | 71 ++++++++++++++----- src/hand/unified/detail/command_scheduler.hpp | 5 +- 2 files changed, 57 insertions(+), 19 deletions(-) diff --git a/src/hand/unified/command_scheduler.cpp b/src/hand/unified/command_scheduler.cpp index 34e0fa2..f7b8607 100644 --- a/src/hand/unified/command_scheduler.cpp +++ b/src/hand/unified/command_scheduler.cpp @@ -142,10 +142,14 @@ namespace linkerbot::hand::unified_detail { CommandHandle CommandScheduler::accept(CommandRequest request, AckSupport support, ErrorContext context, Execute execute) { + // Requests can retain a backend whose destruction enters DeviceRuntime. + // Declare retired work before the lock so every exit releases it unlocked. + std::vector retired; + std::optional incoming; const auto family = request.operation ? CommandFamily::kDiscreteOperation : command_family(request.semantic); std::unique_lock lock(mutex_); const auto now = now_(); - expire(now); + expire(now, retired); if (stop_) { throw DeviceNotReadyError("command scheduler is stopping", context); } @@ -156,7 +160,8 @@ namespace linkerbot::hand::unified_detail { throw InternalError("command sequence exhausted", context); } CommandHandle handle(next_id_, family, support, request.ack_expectation, now); - Job job{std::move(request), handle, std::move(context), now, {}, std::move(execute)}; + incoming.emplace(Job{std::move(request), handle, std::move(context), now, {}, std::move(execute)}); + auto &job = *incoming; job.deadline = python_behavior_ ? Clock::time_point::max() : now + job.request.timeout; job.request.deadline = job.deadline; job.request.completion = handle; @@ -195,6 +200,7 @@ namespace linkerbot::hand::unified_detail { command_detail::CommandHandleAccess::mark_superseded(old->second.handle, now); removed(old->second, true); ++metrics.value.superseded; + retired.push_back(std::move(old->second)); old->second = std::move(job); } else { latest_.emplace(family, std::move(job)); @@ -238,7 +244,7 @@ namespace linkerbot::hand::unified_detail { } } - void CommandScheduler::expire(Clock::time_point now) { + void CommandScheduler::expire(Clock::time_point now, std::vector &retired) { const auto expired = [&](Job &job) { if (now < job.deadline) { return false; @@ -249,6 +255,7 @@ namespace linkerbot::hand::unified_detail { }; for (auto it = fifo_.begin(); it != fifo_.end();) { if (expired(*it)) { + retired.push_back(std::move(*it)); it = fifo_.erase(it); } else { ++it; @@ -256,6 +263,7 @@ namespace linkerbot::hand::unified_detail { } for (auto it = latest_.begin(); it != latest_.end();) { if (expired(it->second)) { + retired.push_back(std::move(it->second)); it = latest_.erase(it); } else { ++it; @@ -275,6 +283,7 @@ namespace linkerbot::hand::unified_detail { } void CommandScheduler::invalidate(std::exception_ptr error) { + std::vector retired; const std::lock_guard lock(mutex_); ++invalidation_; if (preparing_) { @@ -283,10 +292,12 @@ namespace linkerbot::hand::unified_detail { for (auto &job : fifo_) { fail(job, error); removed(job, true); + retired.push_back(std::move(job)); } for (auto &entry : latest_) { fail(entry.second, error); removed(entry.second, true); + retired.push_back(std::move(entry.second)); } fifo_.clear(); latest_.clear(); @@ -303,9 +314,16 @@ namespace linkerbot::hand::unified_detail { } void CommandScheduler::watch_deadlines() { - std::unique_lock lock(mutex_); - while (!stop_) { - expire(now_()); + for (;;) { + std::vector retired; + std::unique_lock lock(mutex_); + if (stop_) { + return; + } + expire(now_(), retired); + if (!retired.empty()) { + continue; + } const auto deadline = next_deadline(); if (manual_clock_ || deadline == Clock::time_point::max()) { changed_.wait(lock); @@ -316,10 +334,20 @@ namespace linkerbot::hand::unified_detail { } void CommandScheduler::run() { - std::unique_lock lock(mutex_); - while (!stop_) { + for (;;) { + std::vector retired; + std::optional completed; + std::shared_ptr execution_lease; + decltype(CommandRequest::prepare_execution) prepare; + std::unique_lock lock(mutex_); + if (stop_) { + return; + } auto now = now_(); - expire(now); + expire(now, retired); + if (!retired.empty()) { + continue; + } if (active_ || !pending_size()) { changed_.wait(lock); continue; @@ -334,10 +362,9 @@ namespace linkerbot::hand::unified_detail { } // Claim a target only after the device transaction lane is available. // Until then latest replacement and FIFO capacity still apply to it. - std::shared_ptr execution_lease; if (python_behavior_) { const auto &candidate = fifo_.empty() ? latest_.begin()->second : fifo_.front(); - const auto prepare = candidate.request.prepare_execution; + prepare = candidate.request.prepare_execution; if (prepare) { const auto epoch = invalidation_; preparing_.emplace(); @@ -358,11 +385,13 @@ namespace linkerbot::hand::unified_detail { for (auto &queued : fifo_) { fail(queued, error); removed(queued, true); + retired.push_back(std::move(queued)); } for (auto &[family, queued] : latest_) { (void)family; fail(queued, error); removed(queued, true); + retired.push_back(std::move(queued)); } fifo_.clear(); latest_.clear(); @@ -391,11 +420,12 @@ namespace linkerbot::hand::unified_detail { auto &m = metrics_.at(job.handle.family()); sample(m.queue_times, now - job.accepted); next_start_ = now + interval_; - execute_active(lock, now); + execute_active(lock, now, completed); } } - std::exception_ptr CommandScheduler::execute_active(std::unique_lock &lock, Clock::time_point started) { + std::exception_ptr CommandScheduler::execute_active(std::unique_lock &lock, Clock::time_point started, + std::optional &completed) { auto &job = ::linkerbot::hand::detail::checked_value(active_, "missing active command"); auto &m = metrics_.at(job.handle.family()); lock.unlock(); @@ -439,14 +469,18 @@ namespace linkerbot::hand::unified_detail { } } } + // Transfer ownership without allocating on normal command completion. + // The caller releases the completed request after unlocking mutex_. + completed.emplace(std::move(job)); active_.reset(); changed_.notify_all(); return error; } std::optional CommandScheduler::pending_angle_at() { + std::vector retired; const std::lock_guard lock(mutex_); - expire(now_()); + expire(now_(), retired); if (!python_behavior_ || active_ || latest_.count(CommandFamily::kAngleTarget) == 0) { return {}; } @@ -454,9 +488,11 @@ namespace linkerbot::hand::unified_detail { } bool CommandScheduler::run_angle(const std::shared_ptr &lane) { + std::vector retired; + std::optional completed; std::unique_lock lock(mutex_); const auto now = now_(); - expire(now); + expire(now, retired); auto found = latest_.find(CommandFamily::kAngleTarget); if (!python_behavior_ || stop_ || active_ || now < next_start_ || found == latest_.end()) { return false; @@ -468,7 +504,7 @@ namespace linkerbot::hand::unified_detail { removed(job, false); sample(metrics_.at(job.handle.family()).queue_times, now - job.accepted); next_start_ = now + interval_; - const auto error = execute_active(lock, now); + const auto error = execute_active(lock, now, completed); lock.unlock(); ::linkerbot::hand::detail::rethrow_nested_connection_failure(error); return true; @@ -487,8 +523,9 @@ namespace linkerbot::hand::unified_detail { } void CommandScheduler::notify_clock_advanced() { + std::vector retired; const std::lock_guard lock(mutex_); - expire(now_()); + expire(now_(), retired); changed_.notify_all(); } } diff --git a/src/hand/unified/detail/command_scheduler.hpp b/src/hand/unified/detail/command_scheduler.hpp index 271ac67..e826174 100644 --- a/src/hand/unified/detail/command_scheduler.hpp +++ b/src/hand/unified/detail/command_scheduler.hpp @@ -65,12 +65,13 @@ namespace linkerbot::hand::unified_detail { }; static bool latest(CommandFamily family); void fail(Job &job, std::exception_ptr error); - void expire(Clock::time_point now); + void expire(Clock::time_point now, std::vector &retired); Clock::time_point next_deadline() const; size_t pending_size() const; void removed(Job &job, bool dropped); void run(); - std::exception_ptr execute_active(std::unique_lock &lock, Clock::time_point started); + std::exception_ptr execute_active(std::unique_lock &lock, Clock::time_point started, + std::optional &completed); void watch_deadlines(); const Clock::duration interval_; From ce19c40ca21855d958b438c89679d47e9ec62bf7 Mon Sep 17 00:00:00 2001 From: ouyangshun <1246787391@qq.com> Date: Tue, 17 Nov 2026 16:58:11 +0800 Subject: [PATCH 02/11] test(scheduler): cover command resource release outside the lock Use synchronization barriers to check completion, latest replacement, invalidation and timeout cleanup. Scheduler inspection must remain available while retained resources are being destroyed. --- tests/hand/device/test_command_scheduler.cpp | 57 ++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/hand/device/test_command_scheduler.cpp b/tests/hand/device/test_command_scheduler.cpp index 0d4769b..d875b5f 100644 --- a/tests/hand/device/test_command_scheduler.cpp +++ b/tests/hand/device/test_command_scheduler.cpp @@ -152,6 +152,63 @@ TEST_CASE("M2 FIFO capacity excludes latest and executing requests", "[comm][m2] } } +TEST_CASE("scheduler releases retired command resources without holding its mutex", "[comm][scheduler][regression]") { + auto scheduler = std::make_unique(1000, 16); + Barrier execution; + Barrier destruction; + auto first = + scheduler->accept(request(CommandSemantic::kAccelerations, 0), AckSupport::kUnsupported, {}, [&](const auto &) { + execution.entered.set_value(); + execution.gate.wait(); + return delivered(); + }); + REQUIRE(execution.entered.get_future().wait_for(std::chrono::seconds{2}) == std::future_status::ready); + + auto retained = request(CommandSemantic::kAngles, 1); + retained.generation_valid = [lifetime = std::shared_ptr(nullptr, [&](void *) { + destruction.entered.set_value(); + destruction.gate.wait(); + })] { return true; }; + std::function retire; + SECTION("completed command") { + retire = [&](CommandHandle handle) { + execution.open(); + (void)handle.wait_for(std::chrono::seconds{2}); + }; + } + SECTION("superseded latest command") { + retire = [&](CommandHandle) { + (void)scheduler->accept(request(CommandSemantic::kAngles, 2), AckSupport::kUnsupported, {}, + [](const auto &) { return delivered(); }); + }; + } + SECTION("invalidated pending commands") { + retire = [&](CommandHandle) { scheduler->invalidate(std::make_exception_ptr(ClosedError("test close"))); }; + } + SECTION("expired pending command") { + retained.timeout = std::chrono::milliseconds{20}; + retire = [](CommandHandle handle) { (void)handle.wait_for(std::chrono::seconds{2}); }; + } + const auto pending = + scheduler->accept(std::move(retained), AckSupport::kUnsupported, {}, [](const auto &) { return delivered(); }); + auto retiring = std::async(std::launch::async, [&] { retire(pending); }); + const auto entered = destruction.entered.get_future().wait_for(std::chrono::seconds{2}); + auto inspection = std::async(std::launch::async, [&] { return scheduler->metrics(); }); + const auto inspected = inspection.wait_for(std::chrono::seconds{1}); + // Always release the destructor before asserting, so a broken implementation + // fails the test without leaving a worker or async future permanently stuck. + destruction.open(); + retiring.get(); + execution.open(); + (void)inspection.get(); + // A terminal handle may precede the worker's resource release. Join it + // before destroying the barriers that its retained request still uses. + scheduler.reset(); + REQUIRE(entered == std::future_status::ready); + REQUIRE(inspected == std::future_status::ready); + REQUIRE(first.wait_for(std::chrono::seconds{2})); +} + TEST_CASE("M2 pending deadlines expire independently of executor and throttle", "[comm][m2][scheduler]") { CommandScheduler scheduler(0.5, 1); // Throttle must not delay expiry. Barrier barrier; From 9af38975d438eaa0c3b8e76ebf6ae408effd0dd3 Mon Sep 17 00:00:00 2001 From: ouyangshun <1246787391@qq.com> Date: Tue, 17 Nov 2026 16:58:11 +0800 Subject: [PATCH 03/11] test(cmake): add optional process isolation for offline test names Discover communication and discovery cases through Catch2 when requested. Separate test names to avoid stale lock graphs across reused mutex addresses. Preserve default grouping and keep deadlock detection enabled. --- tests/CMakeLists.txt | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7da1afa..5fa6dc5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -96,11 +96,27 @@ foreach(batch RANGE 1 5) string(APPEND python_policy_filter "[comm][python-phase${batch}]") string(APPEND comm_filter "~[python-phase${batch}]") endforeach() -add_test(NAME comm COMMAND tests "${comm_filter}" --durations yes) -add_test(NAME python-policy-regressions COMMAND tests "${python_policy_filter}" --durations yes) # Discovery contract and adapter tests are offline: injected enumerators and # FakeTransport cover platform sources without opening host CAN hardware. -add_test(NAME discovery COMMAND tests "[discovery]" --durations yes) +option(LINKERBOT_ISOLATE_OFFLINE_TESTS "Run offline Catch2 cases in separate processes" OFF) +if(LINKERBOT_ISOLATE_OFFLINE_TESTS) + # libstdc++ mutexes have trivial destruction on Linux. TSan's deadlock + # detector can therefore retain lock-order edges when later cases reuse + # their stack addresses. Keep every case and its deadlock checks, but give + # each discovered test name a fresh process rather than sharing that graph. + list(APPEND CMAKE_MODULE_PATH "${Catch2_DIR}" "${catch2_SOURCE_DIR}/extras") + include(Catch) + catch_discover_tests(tests + TEST_SPEC "[comm],[discovery]" + TEST_PREFIX "offline::" + EXTRA_ARGS --durations yes + PROPERTIES TIMEOUT 300 + DISCOVERY_MODE PRE_TEST) +else() + add_test(NAME comm COMMAND tests "${comm_filter}" --durations yes) + add_test(NAME python-policy-regressions COMMAND tests "${python_policy_filter}" --durations yes) + add_test(NAME discovery COMMAND tests "[discovery]" --durations yes) +endif() if(TARGET l6_read_version) add_dependencies(tests l6_read_version) From 093f148316297bcf520abf2a085189759b19556f Mon Sep 17 00:00:00 2001 From: ouyangshun <1246787391@qq.com> Date: Tue, 17 Nov 2026 16:58:11 +0800 Subject: [PATCH 04/11] ci(sanitizers): enable leak detection in the ASan job Enable LeakSanitizer and use exit code 23 for leak findings alongside existing address and undefined-behavior checks. --- .github/workflows/sanitizers.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 26e920c..55f53ce 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -62,13 +62,14 @@ jobs: --compiler g++-13 sanitizers: - name: ASan and UBSan + name: ASan, UBSan and LSan runs-on: ubuntu-24.04 timeout-minutes: 30 env: CC: gcc-13 CXX: g++-13 - ASAN_OPTIONS: detect_leaks=0:halt_on_error=1:strict_string_checks=1 + ASAN_OPTIONS: detect_leaks=1:halt_on_error=1:strict_string_checks=1 + LSAN_OPTIONS: exitcode=23 UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 steps: From 707fd3ad8b88da83667e8fbc8eb53feee18af342 Mon Sep 17 00:00:00 2001 From: ouyangshun <1246787391@qq.com> Date: Tue, 17 Nov 2026 16:58:48 +0800 Subject: [PATCH 05/11] ci(sanitizers): retain CTest diagnostics after sanitizer failures Fail when no tests are registered and produce JUnit results. Upload ASan, UBSan and LSan diagnostics even when an earlier step fails. --- .github/workflows/sanitizers.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 55f53ce..a14d423 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -117,4 +117,16 @@ jobs: - name: Run sanitizer tests working-directory: hand-cpp - run: ctest --test-dir build-sanitized --output-on-failure --timeout 180 + run: | + ctest --test-dir build-sanitized --output-on-failure --no-tests=error \ + --timeout 180 --output-junit "$PWD/build-sanitized/sanitizer-tests.xml" + + - name: Upload sanitizer diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: asan-ubsan-lsan-diagnostics + if-no-files-found: warn + path: | + hand-cpp/build-sanitized/sanitizer-tests.xml + hand-cpp/build-sanitized/Testing/Temporary/ From 9ddae3811075c45080e3f4f907c86d2e3dc36ebe Mon Sep 17 00:00:00 2001 From: ouyangshun <1246787391@qq.com> Date: Tue, 17 Nov 2026 16:58:49 +0800 Subject: [PATCH 06/11] ci(sanitizers): add an isolated Clang 18 TSan job Instrument the SDK and paired dependencies with Clang 18 ThreadSanitizer. Use an independent build, compatible runner address-space settings and offline test isolation while retaining race and deadlock checks. Upload CTest results and failure diagnostics. --- .github/workflows/sanitizers.yml | 74 ++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index a14d423..41269ab 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -130,3 +130,77 @@ jobs: path: | hand-cpp/build-sanitized/sanitizer-tests.xml hand-cpp/build-sanitized/Testing/Temporary/ + + thread-sanitizer: + name: TSan + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + CC: clang-18 + CXX: clang++-18 + TSAN_OPTIONS: halt_on_error=1:exitcode=66:second_deadlock_stack=1:external_symbolizer_path=/usr/bin/llvm-symbolizer-18 + + steps: + - name: Set offline configuration directory + run: echo "XDG_CONFIG_HOME=${RUNNER_TEMP}/xdg-config" >> "$GITHUB_ENV" + + - name: Check out hand SDK + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: hand-cpp + + - name: Check out communication library + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ env.COMM_REPOSITORY }} + ref: ${{ env.COMM_REF }} + path: linkerbot-transport-cpp + token: ${{ secrets.LINKERBOT_COMM_TOKEN || github.token }} + + - name: Install build tools + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + cmake ninja-build clang-18 libclang-rt-18-dev llvm-18 + + - name: Configure TSan address space + # TSan needs a compatible shadow-memory layout with Ubuntu 24.04's ASLR. + # This setting affects only this ephemeral runner. + run: sudo sysctl -w vm.mmap_rnd_bits=28 + + - name: Configure TSan build + working-directory: hand-cpp + run: | + cmake -S . -B build-tsan -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_FLAGS="-O1 -fsanitize=thread -fno-omit-frame-pointer" \ + -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=thread" \ + -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=thread" \ + -DLINKERBOT_COMM_SOURCE_DIR="${GITHUB_WORKSPACE}/linkerbot-transport-cpp" \ + -DLINKERBOT_BUILD_EXAMPLES=OFF \ + -DLINKERBOT_BUILD_TESTS=ON \ + -DLINKERBOT_ISOLATE_OFFLINE_TESTS=ON \ + -DLINKERBOT_BUILD_TOOLS=ON \ + -DLINKERBOT_COMM_BUILD_TESTS=OFF + + - name: Build TSan test targets + working-directory: hand-cpp + run: | + cmake --build build-tsan --parallel 4 --target \ + tests protocol-tests l6_read_version l6_frequency_benchmark + + - name: Run TSan tests + working-directory: hand-cpp + run: | + ctest --test-dir build-tsan --output-on-failure --no-tests=error \ + --timeout 300 --output-junit "$PWD/build-tsan/tsan-tests.xml" + + - name: Upload TSan diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: tsan-diagnostics + if-no-files-found: warn + path: | + hand-cpp/build-tsan/tsan-tests.xml + hand-cpp/build-tsan/Testing/Temporary/ From 15cc6fd344fc2c83c94c987772503963cfb14156 Mon Sep 17 00:00:00 2001 From: ouyangshun <1246787391@qq.com> Date: Tue, 17 Nov 2026 16:58:49 +0800 Subject: [PATCH 07/11] docs(quality): document leak and thread sanitizer coverage Explain both sanitizer builds, test isolation, runtime requirements and diagnostic artifacts in English and Chinese. State the offline scope and distinguish initialization errors from clean sanitizer results. --- docs/en/guides/quality.md | 24 ++++++++++++++++++++++++ docs/zh/guides/quality.md | 21 +++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/docs/en/guides/quality.md b/docs/en/guides/quality.md index 05b7bb3..ea0192c 100644 --- a/docs/en/guides/quality.md +++ b/docs/en/guides/quality.md @@ -2,6 +2,30 @@ [English](quality.md) | [简体中文](../../zh/guides/quality.md) +## Runtime sanitizers + +The `Sanitizers` workflow runs two independent Linux Debug builds against the pinned +transport source. `ASan, UBSan and LSan` checks memory access, undefined behavior and +leaks (`detect_leaks=1`); `TSan` checks data races in a separate Clang 18 build using +`-fsanitize=thread`. Both run the registered offline CTest suite, including communication, +Python-policy, discovery and protocol regressions. They do not open real CAN devices. +SDK and paired transport sources are instrumented together. Findings fail the job; +CTest logs and JUnit results are uploaded even after failures. + +The TSan build enables `LINKERBOT_ISOLATE_OFFLINE_TESTS`: Catch2 discovers every +`[comm]` and `[discovery]` case and CTest runs each test name in its own process. +Cases sharing a name remain selected together by Catch2. This prevents +libstdc++'s trivial mutex destruction and stack-address reuse between cases from +creating stale lock-order cycles. Data-race and deadlock detection remain enabled +within each case. The normal ASan/UBSan/LSan build keeps the grouped test layout. + +Clang 18 TSan recognizes the `pthread_mutex_clocklock` calls used by libstdc++ timed +mutexes; the GCC runtime on Ubuntu 24.04 reports false unlock errors for these calls. +TSan also needs a compatible shadow-memory layout. Its job sets `vm.mmap_rnd_bits=28` +on the ephemeral Ubuntu runner before testing. Local environments need a compatible +layout too; an initialization failure is not a clean race check. Runtime sanitizers +cover only executed paths and do not replace static analysis or hardware acceptance. + ## Report Card The `C++ Report Card` workflow runs on PRs, main/master pushes and manual dispatch. diff --git a/docs/zh/guides/quality.md b/docs/zh/guides/quality.md index 5f08fab..61449bf 100644 --- a/docs/zh/guides/quality.md +++ b/docs/zh/guides/quality.md @@ -2,6 +2,27 @@ [English](../../en/guides/quality.md) | [简体中文](quality.md) +## 运行时检测 + +`Sanitizers` 工作流基于固定的 transport 源码,运行两套独立的 Linux Debug 构建。 +`ASan, UBSan and LSan` 检查内存访问、未定义行为和泄漏(`detect_leaks=1`); +`TSan` 使用独立的 Clang 18 `-fsanitize=thread` 构建检查数据竞争。两者都运行已注册的离线 +CTest 测试,包括通信、Python 策略、发现和协议回归,不打开真实 CAN 设备。 +SDK 与配套 transport 源码一起插桩。检测到问题会使任务失败;失败后仍上传 +CTest 日志和 JUnit 测试结果。 + +TSan 构建开启 `LINKERBOT_ISOLATE_OFFLINE_TESTS`:Catch2 发现全部 `[comm]` 和 +`[discovery]` 用例,CTest 按用例名称启动独立进程,同名用例仍由 Catch2 一起匹配。 +这样可避免 libstdc++ 互斥锁的平凡析构和 +跨用例栈地址复用形成过期的锁顺序环。每个用例内的数据竞争和死锁检测仍保持开启; +常规 ASan/UBSan/LSan 构建继续按原有分组运行测试。 + +Clang 18 TSan 能识别 libstdc++ 定时互斥锁使用的 `pthread_mutex_clocklock`; +Ubuntu 24.04 的 GCC 检测运行库会对此误报解锁错误。 +TSan 还需要兼容的影子内存布局,因此该任务在临时 Ubuntu runner 上先设置 +`vm.mmap_rnd_bits=28`。本地环境也需要兼容的布局;初始化失败不代表没有数据竞争。 +运行时检测只覆盖实际执行的路径,不能替代静态分析或实机验收。 + ## Report Card `C++ Report Card` 工作流在 PR、main/master 推送和手动触发时运行。 From 94269cc195c6d3857b4f645c2cb15abb02e71a31 Mon Sep 17 00:00:00 2001 From: ouyangshun <1246787391@qq.com> Date: Tue, 17 Nov 2026 17:16:12 +0800 Subject: [PATCH 08/11] fix(ci): pin the TSan standard library toolchain to GCC 13 Clang 18 selects GCC 14 headers on the hosted runner and fails compiling std::timed_mutex calls before tests start. Install GCC 13 explicitly and select its installation for every TSan C++ compile and link invocation. Validated toolchain selection, DeviceRuntime compilation and a TSan timed-mutex probe with Clang 18 and GCC 13 on Linux. --- .github/workflows/sanitizers.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 41269ab..1c7281e 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -161,7 +161,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends \ - cmake ninja-build clang-18 libclang-rt-18-dev llvm-18 + cmake ninja-build gcc-13 g++-13 clang-18 libclang-rt-18-dev llvm-18 - name: Configure TSan address space # TSan needs a compatible shadow-memory layout with Ubuntu 24.04's ASLR. @@ -171,9 +171,12 @@ jobs: - name: Configure TSan build working-directory: hand-cpp run: | + # Clang otherwise selects the runner's newest GCC installation, which + # can have incompatible timed-mutex headers. Match the tested GCC 13 baseline. + gcc_install_dir="$(dirname "$(gcc-13 -print-libgcc-file-name)")" cmake -S . -B build-tsan -G Ninja \ -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_FLAGS="-O1 -fsanitize=thread -fno-omit-frame-pointer" \ + -DCMAKE_CXX_FLAGS="--gcc-install-dir=${gcc_install_dir} -O1 -fsanitize=thread -fno-omit-frame-pointer" \ -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=thread" \ -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=thread" \ -DLINKERBOT_COMM_SOURCE_DIR="${GITHUB_WORKSPACE}/linkerbot-transport-cpp" \ From a47a464cbc0e307e9899f99677bc7ba8997770db Mon Sep 17 00:00:00 2001 From: ouyangshun <1246787391@qq.com> Date: Tue, 17 Nov 2026 17:16:12 +0800 Subject: [PATCH 09/11] docs(quality): explain the pinned TSan standard library baseline Document the Clang 18 and GCC 14 timed-mutex compilation failure and the explicit GCC 13 selection in both language guides. --- docs/en/guides/quality.md | 5 +++++ docs/zh/guides/quality.md | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/docs/en/guides/quality.md b/docs/en/guides/quality.md index ea0192c..5890cb3 100644 --- a/docs/en/guides/quality.md +++ b/docs/en/guides/quality.md @@ -19,6 +19,11 @@ libstdc++'s trivial mutex destruction and stack-address reuse between cases from creating stale lock-order cycles. Data-race and deadlock detection remain enabled within each case. The normal ASan/UBSan/LSan build keeps the grouped test layout. +The TSan job explicitly selects the GCC 13 libstdc++ headers and libraries with +Clang's `--gcc-install-dir`; runner image updates must not silently select a newer +GCC installation. Clang 18 with the runner's GCC 14 headers fails to compile the +SDK's timed-mutex calls, before sanitizer tests can run. + Clang 18 TSan recognizes the `pthread_mutex_clocklock` calls used by libstdc++ timed mutexes; the GCC runtime on Ubuntu 24.04 reports false unlock errors for these calls. TSan also needs a compatible shadow-memory layout. Its job sets `vm.mmap_rnd_bits=28` diff --git a/docs/zh/guides/quality.md b/docs/zh/guides/quality.md index 61449bf..7a77c1b 100644 --- a/docs/zh/guides/quality.md +++ b/docs/zh/guides/quality.md @@ -17,6 +17,10 @@ TSan 构建开启 `LINKERBOT_ISOLATE_OFFLINE_TESTS`:Catch2 发现全部 `[comm 跨用例栈地址复用形成过期的锁顺序环。每个用例内的数据竞争和死锁检测仍保持开启; 常规 ASan/UBSan/LSan 构建继续按原有分组运行测试。 +TSan 任务通过 Clang 的 `--gcc-install-dir` 显式选择 GCC 13 的 libstdc++ 头文件与库, +避免 runner 镜像更新后自动选择更高版本的 GCC。Clang 18 搭配 runner 的 GCC 14 +头文件时,SDK 的定时互斥锁调用会在编译阶段失败,检测测试尚未开始。 + Clang 18 TSan 能识别 libstdc++ 定时互斥锁使用的 `pthread_mutex_clocklock`; Ubuntu 24.04 的 GCC 检测运行库会对此误报解锁错误。 TSan 还需要兼容的影子内存布局,因此该任务在临时 Ubuntu runner 上先设置 From dff2c88f5fd4f53f7f306b10b1d9c414eb853a16 Mon Sep 17 00:00:00 2001 From: ouyangshun <1246787391@qq.com> Date: Tue, 17 Nov 2026 17:34:45 +0800 Subject: [PATCH 10/11] test(runtime): assert waiter failures on the main test thread The close regression ran REQUIRE_THROWS in an async worker concurrently with the main thread assertion. TSan detects a race in Catch2 output redirection because assertion reporting is not thread-safe. Keep request.wait in the worker and assert its exception through future.get on the main thread. Preserve the close and wakeup checks. The corrected case passed 100 consecutive Clang 18 TSan runs on Linux. --- tests/runtime/test_device_runtime_comm.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/runtime/test_device_runtime_comm.cpp b/tests/runtime/test_device_runtime_comm.cpp index 595400e..d51ef8d 100644 --- a/tests/runtime/test_device_runtime_comm.cpp +++ b/tests/runtime/test_device_runtime_comm.cpp @@ -726,10 +726,10 @@ TEST_CASE("comm: command queue counts logical jobs independently of frames", "[c TEST_CASE("comm: runtime close wakes waiter and clears pending cache", "[comm][runtime]") { Rig rig; auto request = rig.runtime->submit(plan(), Clock::now() + 30s); - auto waiting = std::async(std::launch::async, [&] { REQUIRE_THROWS(request.wait()); }); + auto waiting = std::async(std::launch::async, [&] { return request.wait(); }); rig.runtime->close(); REQUIRE(waiting.wait_for(500ms) == std::future_status::ready); - waiting.get(); + REQUIRE_THROWS(waiting.get()); REQUIRE(rig.runtime->runtime_health().in_flight == 0); REQUIRE(rig.runtime->health().lifecycle_state == linkerbot::hand::LifecycleState::kClosed); REQUIRE_FALSE(rig.runtime->health().connected()); From f79edeb38b0bb8fa862c3ab6980cf52e1c19a65a Mon Sep 17 00:00:00 2001 From: ouyangshun <1246787391@qq.com> Date: Wed, 18 Nov 2026 14:20:29 +0800 Subject: [PATCH 11/11] ci(report-card): increase analyzer timeout to 400 seconds --- .cppreportcard.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cppreportcard.json b/.cppreportcard.json index 1d1969c..b2beba8 100644 --- a/.cppreportcard.json +++ b/.cppreportcard.json @@ -8,7 +8,7 @@ ], "tidy_checks": "-*,clang-analyzer-*,bugprone-*", "format_style": "LLVM", - "timeout_seconds": 300, + "timeout_seconds": 400, "jobs": 4, "tools": { "clang-format": "clang-format-18",