diff --git a/.cppreportcard.json b/.cppreportcard.json index 1d1969c..b2beba8 100644 --- a/.cppreportcard.json +++ b/.cppreportcard.json @@ -8,7 +8,7 @@ ], "tidy_checks": "-*,clang-analyzer-*,bugprone-*", "format_style": "LLVM", - "timeout_seconds": 300, + "timeout_seconds": 400, "jobs": 4, "tools": { "clang-format": "clang-format-18", diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 26e920c..1c7281e 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -62,13 +62,14 @@ jobs: --compiler g++-13 sanitizers: - name: ASan and UBSan + name: ASan, UBSan and LSan runs-on: ubuntu-24.04 timeout-minutes: 30 env: CC: gcc-13 CXX: g++-13 - ASAN_OPTIONS: detect_leaks=0:halt_on_error=1:strict_string_checks=1 + ASAN_OPTIONS: detect_leaks=1:halt_on_error=1:strict_string_checks=1 + LSAN_OPTIONS: exitcode=23 UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 steps: @@ -116,4 +117,93 @@ jobs: - name: Run sanitizer tests working-directory: hand-cpp - run: ctest --test-dir build-sanitized --output-on-failure --timeout 180 + run: | + ctest --test-dir build-sanitized --output-on-failure --no-tests=error \ + --timeout 180 --output-junit "$PWD/build-sanitized/sanitizer-tests.xml" + + - name: Upload sanitizer diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: asan-ubsan-lsan-diagnostics + if-no-files-found: warn + path: | + hand-cpp/build-sanitized/sanitizer-tests.xml + hand-cpp/build-sanitized/Testing/Temporary/ + + thread-sanitizer: + name: TSan + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + CC: clang-18 + CXX: clang++-18 + TSAN_OPTIONS: halt_on_error=1:exitcode=66:second_deadlock_stack=1:external_symbolizer_path=/usr/bin/llvm-symbolizer-18 + + steps: + - name: Set offline configuration directory + run: echo "XDG_CONFIG_HOME=${RUNNER_TEMP}/xdg-config" >> "$GITHUB_ENV" + + - name: Check out hand SDK + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: hand-cpp + + - name: Check out communication library + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ env.COMM_REPOSITORY }} + ref: ${{ env.COMM_REF }} + path: linkerbot-transport-cpp + token: ${{ secrets.LINKERBOT_COMM_TOKEN || github.token }} + + - name: Install build tools + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + cmake ninja-build gcc-13 g++-13 clang-18 libclang-rt-18-dev llvm-18 + + - name: Configure TSan address space + # TSan needs a compatible shadow-memory layout with Ubuntu 24.04's ASLR. + # This setting affects only this ephemeral runner. + run: sudo sysctl -w vm.mmap_rnd_bits=28 + + - name: Configure TSan build + working-directory: hand-cpp + run: | + # Clang otherwise selects the runner's newest GCC installation, which + # can have incompatible timed-mutex headers. Match the tested GCC 13 baseline. + gcc_install_dir="$(dirname "$(gcc-13 -print-libgcc-file-name)")" + cmake -S . -B build-tsan -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_FLAGS="--gcc-install-dir=${gcc_install_dir} -O1 -fsanitize=thread -fno-omit-frame-pointer" \ + -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=thread" \ + -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=thread" \ + -DLINKERBOT_COMM_SOURCE_DIR="${GITHUB_WORKSPACE}/linkerbot-transport-cpp" \ + -DLINKERBOT_BUILD_EXAMPLES=OFF \ + -DLINKERBOT_BUILD_TESTS=ON \ + -DLINKERBOT_ISOLATE_OFFLINE_TESTS=ON \ + -DLINKERBOT_BUILD_TOOLS=ON \ + -DLINKERBOT_COMM_BUILD_TESTS=OFF + + - name: Build TSan test targets + working-directory: hand-cpp + run: | + cmake --build build-tsan --parallel 4 --target \ + tests protocol-tests l6_read_version l6_frequency_benchmark + + - name: Run TSan tests + working-directory: hand-cpp + run: | + ctest --test-dir build-tsan --output-on-failure --no-tests=error \ + --timeout 300 --output-junit "$PWD/build-tsan/tsan-tests.xml" + + - name: Upload TSan diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: tsan-diagnostics + if-no-files-found: warn + path: | + hand-cpp/build-tsan/tsan-tests.xml + hand-cpp/build-tsan/Testing/Temporary/ diff --git a/docs/en/guides/quality.md b/docs/en/guides/quality.md index 05b7bb3..5890cb3 100644 --- a/docs/en/guides/quality.md +++ b/docs/en/guides/quality.md @@ -2,6 +2,35 @@ [English](quality.md) | [简体中文](../../zh/guides/quality.md) +## Runtime sanitizers + +The `Sanitizers` workflow runs two independent Linux Debug builds against the pinned +transport source. `ASan, UBSan and LSan` checks memory access, undefined behavior and +leaks (`detect_leaks=1`); `TSan` checks data races in a separate Clang 18 build using +`-fsanitize=thread`. Both run the registered offline CTest suite, including communication, +Python-policy, discovery and protocol regressions. They do not open real CAN devices. +SDK and paired transport sources are instrumented together. Findings fail the job; +CTest logs and JUnit results are uploaded even after failures. + +The TSan build enables `LINKERBOT_ISOLATE_OFFLINE_TESTS`: Catch2 discovers every +`[comm]` and `[discovery]` case and CTest runs each test name in its own process. +Cases sharing a name remain selected together by Catch2. This prevents +libstdc++'s trivial mutex destruction and stack-address reuse between cases from +creating stale lock-order cycles. Data-race and deadlock detection remain enabled +within each case. The normal ASan/UBSan/LSan build keeps the grouped test layout. + +The TSan job explicitly selects the GCC 13 libstdc++ headers and libraries with +Clang's `--gcc-install-dir`; runner image updates must not silently select a newer +GCC installation. Clang 18 with the runner's GCC 14 headers fails to compile the +SDK's timed-mutex calls, before sanitizer tests can run. + +Clang 18 TSan recognizes the `pthread_mutex_clocklock` calls used by libstdc++ timed +mutexes; the GCC runtime on Ubuntu 24.04 reports false unlock errors for these calls. +TSan also needs a compatible shadow-memory layout. Its job sets `vm.mmap_rnd_bits=28` +on the ephemeral Ubuntu runner before testing. Local environments need a compatible +layout too; an initialization failure is not a clean race check. Runtime sanitizers +cover only executed paths and do not replace static analysis or hardware acceptance. + ## Report Card The `C++ Report Card` workflow runs on PRs, main/master pushes and manual dispatch. diff --git a/docs/zh/guides/quality.md b/docs/zh/guides/quality.md index 5f08fab..7a77c1b 100644 --- a/docs/zh/guides/quality.md +++ b/docs/zh/guides/quality.md @@ -2,6 +2,31 @@ [English](../../en/guides/quality.md) | [简体中文](quality.md) +## 运行时检测 + +`Sanitizers` 工作流基于固定的 transport 源码,运行两套独立的 Linux Debug 构建。 +`ASan, UBSan and LSan` 检查内存访问、未定义行为和泄漏(`detect_leaks=1`); +`TSan` 使用独立的 Clang 18 `-fsanitize=thread` 构建检查数据竞争。两者都运行已注册的离线 +CTest 测试,包括通信、Python 策略、发现和协议回归,不打开真实 CAN 设备。 +SDK 与配套 transport 源码一起插桩。检测到问题会使任务失败;失败后仍上传 +CTest 日志和 JUnit 测试结果。 + +TSan 构建开启 `LINKERBOT_ISOLATE_OFFLINE_TESTS`:Catch2 发现全部 `[comm]` 和 +`[discovery]` 用例,CTest 按用例名称启动独立进程,同名用例仍由 Catch2 一起匹配。 +这样可避免 libstdc++ 互斥锁的平凡析构和 +跨用例栈地址复用形成过期的锁顺序环。每个用例内的数据竞争和死锁检测仍保持开启; +常规 ASan/UBSan/LSan 构建继续按原有分组运行测试。 + +TSan 任务通过 Clang 的 `--gcc-install-dir` 显式选择 GCC 13 的 libstdc++ 头文件与库, +避免 runner 镜像更新后自动选择更高版本的 GCC。Clang 18 搭配 runner 的 GCC 14 +头文件时,SDK 的定时互斥锁调用会在编译阶段失败,检测测试尚未开始。 + +Clang 18 TSan 能识别 libstdc++ 定时互斥锁使用的 `pthread_mutex_clocklock`; +Ubuntu 24.04 的 GCC 检测运行库会对此误报解锁错误。 +TSan 还需要兼容的影子内存布局,因此该任务在临时 Ubuntu runner 上先设置 +`vm.mmap_rnd_bits=28`。本地环境也需要兼容的布局;初始化失败不代表没有数据竞争。 +运行时检测只覆盖实际执行的路径,不能替代静态分析或实机验收。 + ## Report Card `C++ Report Card` 工作流在 PR、main/master 推送和手动触发时运行。 diff --git a/src/hand/unified/command_scheduler.cpp b/src/hand/unified/command_scheduler.cpp index 34e0fa2..f7b8607 100644 --- a/src/hand/unified/command_scheduler.cpp +++ b/src/hand/unified/command_scheduler.cpp @@ -142,10 +142,14 @@ namespace linkerbot::hand::unified_detail { CommandHandle CommandScheduler::accept(CommandRequest request, AckSupport support, ErrorContext context, Execute execute) { + // Requests can retain a backend whose destruction enters DeviceRuntime. + // Declare retired work before the lock so every exit releases it unlocked. + std::vector retired; + std::optional incoming; const auto family = request.operation ? CommandFamily::kDiscreteOperation : command_family(request.semantic); std::unique_lock lock(mutex_); const auto now = now_(); - expire(now); + expire(now, retired); if (stop_) { throw DeviceNotReadyError("command scheduler is stopping", context); } @@ -156,7 +160,8 @@ namespace linkerbot::hand::unified_detail { throw InternalError("command sequence exhausted", context); } CommandHandle handle(next_id_, family, support, request.ack_expectation, now); - Job job{std::move(request), handle, std::move(context), now, {}, std::move(execute)}; + incoming.emplace(Job{std::move(request), handle, std::move(context), now, {}, std::move(execute)}); + auto &job = *incoming; job.deadline = python_behavior_ ? Clock::time_point::max() : now + job.request.timeout; job.request.deadline = job.deadline; job.request.completion = handle; @@ -195,6 +200,7 @@ namespace linkerbot::hand::unified_detail { command_detail::CommandHandleAccess::mark_superseded(old->second.handle, now); removed(old->second, true); ++metrics.value.superseded; + retired.push_back(std::move(old->second)); old->second = std::move(job); } else { latest_.emplace(family, std::move(job)); @@ -238,7 +244,7 @@ namespace linkerbot::hand::unified_detail { } } - void CommandScheduler::expire(Clock::time_point now) { + void CommandScheduler::expire(Clock::time_point now, std::vector &retired) { const auto expired = [&](Job &job) { if (now < job.deadline) { return false; @@ -249,6 +255,7 @@ namespace linkerbot::hand::unified_detail { }; for (auto it = fifo_.begin(); it != fifo_.end();) { if (expired(*it)) { + retired.push_back(std::move(*it)); it = fifo_.erase(it); } else { ++it; @@ -256,6 +263,7 @@ namespace linkerbot::hand::unified_detail { } for (auto it = latest_.begin(); it != latest_.end();) { if (expired(it->second)) { + retired.push_back(std::move(it->second)); it = latest_.erase(it); } else { ++it; @@ -275,6 +283,7 @@ namespace linkerbot::hand::unified_detail { } void CommandScheduler::invalidate(std::exception_ptr error) { + std::vector retired; const std::lock_guard lock(mutex_); ++invalidation_; if (preparing_) { @@ -283,10 +292,12 @@ namespace linkerbot::hand::unified_detail { for (auto &job : fifo_) { fail(job, error); removed(job, true); + retired.push_back(std::move(job)); } for (auto &entry : latest_) { fail(entry.second, error); removed(entry.second, true); + retired.push_back(std::move(entry.second)); } fifo_.clear(); latest_.clear(); @@ -303,9 +314,16 @@ namespace linkerbot::hand::unified_detail { } void CommandScheduler::watch_deadlines() { - std::unique_lock lock(mutex_); - while (!stop_) { - expire(now_()); + for (;;) { + std::vector retired; + std::unique_lock lock(mutex_); + if (stop_) { + return; + } + expire(now_(), retired); + if (!retired.empty()) { + continue; + } const auto deadline = next_deadline(); if (manual_clock_ || deadline == Clock::time_point::max()) { changed_.wait(lock); @@ -316,10 +334,20 @@ namespace linkerbot::hand::unified_detail { } void CommandScheduler::run() { - std::unique_lock lock(mutex_); - while (!stop_) { + for (;;) { + std::vector retired; + std::optional completed; + std::shared_ptr execution_lease; + decltype(CommandRequest::prepare_execution) prepare; + std::unique_lock lock(mutex_); + if (stop_) { + return; + } auto now = now_(); - expire(now); + expire(now, retired); + if (!retired.empty()) { + continue; + } if (active_ || !pending_size()) { changed_.wait(lock); continue; @@ -334,10 +362,9 @@ namespace linkerbot::hand::unified_detail { } // Claim a target only after the device transaction lane is available. // Until then latest replacement and FIFO capacity still apply to it. - std::shared_ptr execution_lease; if (python_behavior_) { const auto &candidate = fifo_.empty() ? latest_.begin()->second : fifo_.front(); - const auto prepare = candidate.request.prepare_execution; + prepare = candidate.request.prepare_execution; if (prepare) { const auto epoch = invalidation_; preparing_.emplace(); @@ -358,11 +385,13 @@ namespace linkerbot::hand::unified_detail { for (auto &queued : fifo_) { fail(queued, error); removed(queued, true); + retired.push_back(std::move(queued)); } for (auto &[family, queued] : latest_) { (void)family; fail(queued, error); removed(queued, true); + retired.push_back(std::move(queued)); } fifo_.clear(); latest_.clear(); @@ -391,11 +420,12 @@ namespace linkerbot::hand::unified_detail { auto &m = metrics_.at(job.handle.family()); sample(m.queue_times, now - job.accepted); next_start_ = now + interval_; - execute_active(lock, now); + execute_active(lock, now, completed); } } - std::exception_ptr CommandScheduler::execute_active(std::unique_lock &lock, Clock::time_point started) { + std::exception_ptr CommandScheduler::execute_active(std::unique_lock &lock, Clock::time_point started, + std::optional &completed) { auto &job = ::linkerbot::hand::detail::checked_value(active_, "missing active command"); auto &m = metrics_.at(job.handle.family()); lock.unlock(); @@ -439,14 +469,18 @@ namespace linkerbot::hand::unified_detail { } } } + // Transfer ownership without allocating on normal command completion. + // The caller releases the completed request after unlocking mutex_. + completed.emplace(std::move(job)); active_.reset(); changed_.notify_all(); return error; } std::optional CommandScheduler::pending_angle_at() { + std::vector retired; const std::lock_guard lock(mutex_); - expire(now_()); + expire(now_(), retired); if (!python_behavior_ || active_ || latest_.count(CommandFamily::kAngleTarget) == 0) { return {}; } @@ -454,9 +488,11 @@ namespace linkerbot::hand::unified_detail { } bool CommandScheduler::run_angle(const std::shared_ptr &lane) { + std::vector retired; + std::optional completed; std::unique_lock lock(mutex_); const auto now = now_(); - expire(now); + expire(now, retired); auto found = latest_.find(CommandFamily::kAngleTarget); if (!python_behavior_ || stop_ || active_ || now < next_start_ || found == latest_.end()) { return false; @@ -468,7 +504,7 @@ namespace linkerbot::hand::unified_detail { removed(job, false); sample(metrics_.at(job.handle.family()).queue_times, now - job.accepted); next_start_ = now + interval_; - const auto error = execute_active(lock, now); + const auto error = execute_active(lock, now, completed); lock.unlock(); ::linkerbot::hand::detail::rethrow_nested_connection_failure(error); return true; @@ -487,8 +523,9 @@ namespace linkerbot::hand::unified_detail { } void CommandScheduler::notify_clock_advanced() { + std::vector retired; const std::lock_guard lock(mutex_); - expire(now_()); + expire(now_(), retired); changed_.notify_all(); } } diff --git a/src/hand/unified/detail/command_scheduler.hpp b/src/hand/unified/detail/command_scheduler.hpp index 271ac67..e826174 100644 --- a/src/hand/unified/detail/command_scheduler.hpp +++ b/src/hand/unified/detail/command_scheduler.hpp @@ -65,12 +65,13 @@ namespace linkerbot::hand::unified_detail { }; static bool latest(CommandFamily family); void fail(Job &job, std::exception_ptr error); - void expire(Clock::time_point now); + void expire(Clock::time_point now, std::vector &retired); Clock::time_point next_deadline() const; size_t pending_size() const; void removed(Job &job, bool dropped); void run(); - std::exception_ptr execute_active(std::unique_lock &lock, Clock::time_point started); + std::exception_ptr execute_active(std::unique_lock &lock, Clock::time_point started, + std::optional &completed); void watch_deadlines(); const Clock::duration interval_; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7da1afa..5fa6dc5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -96,11 +96,27 @@ foreach(batch RANGE 1 5) string(APPEND python_policy_filter "[comm][python-phase${batch}]") string(APPEND comm_filter "~[python-phase${batch}]") endforeach() -add_test(NAME comm COMMAND tests "${comm_filter}" --durations yes) -add_test(NAME python-policy-regressions COMMAND tests "${python_policy_filter}" --durations yes) # Discovery contract and adapter tests are offline: injected enumerators and # FakeTransport cover platform sources without opening host CAN hardware. -add_test(NAME discovery COMMAND tests "[discovery]" --durations yes) +option(LINKERBOT_ISOLATE_OFFLINE_TESTS "Run offline Catch2 cases in separate processes" OFF) +if(LINKERBOT_ISOLATE_OFFLINE_TESTS) + # libstdc++ mutexes have trivial destruction on Linux. TSan's deadlock + # detector can therefore retain lock-order edges when later cases reuse + # their stack addresses. Keep every case and its deadlock checks, but give + # each discovered test name a fresh process rather than sharing that graph. + list(APPEND CMAKE_MODULE_PATH "${Catch2_DIR}" "${catch2_SOURCE_DIR}/extras") + include(Catch) + catch_discover_tests(tests + TEST_SPEC "[comm],[discovery]" + TEST_PREFIX "offline::" + EXTRA_ARGS --durations yes + PROPERTIES TIMEOUT 300 + DISCOVERY_MODE PRE_TEST) +else() + add_test(NAME comm COMMAND tests "${comm_filter}" --durations yes) + add_test(NAME python-policy-regressions COMMAND tests "${python_policy_filter}" --durations yes) + add_test(NAME discovery COMMAND tests "[discovery]" --durations yes) +endif() if(TARGET l6_read_version) add_dependencies(tests l6_read_version) diff --git a/tests/hand/device/test_command_scheduler.cpp b/tests/hand/device/test_command_scheduler.cpp index 0d4769b..d875b5f 100644 --- a/tests/hand/device/test_command_scheduler.cpp +++ b/tests/hand/device/test_command_scheduler.cpp @@ -152,6 +152,63 @@ TEST_CASE("M2 FIFO capacity excludes latest and executing requests", "[comm][m2] } } +TEST_CASE("scheduler releases retired command resources without holding its mutex", "[comm][scheduler][regression]") { + auto scheduler = std::make_unique(1000, 16); + Barrier execution; + Barrier destruction; + auto first = + scheduler->accept(request(CommandSemantic::kAccelerations, 0), AckSupport::kUnsupported, {}, [&](const auto &) { + execution.entered.set_value(); + execution.gate.wait(); + return delivered(); + }); + REQUIRE(execution.entered.get_future().wait_for(std::chrono::seconds{2}) == std::future_status::ready); + + auto retained = request(CommandSemantic::kAngles, 1); + retained.generation_valid = [lifetime = std::shared_ptr(nullptr, [&](void *) { + destruction.entered.set_value(); + destruction.gate.wait(); + })] { return true; }; + std::function retire; + SECTION("completed command") { + retire = [&](CommandHandle handle) { + execution.open(); + (void)handle.wait_for(std::chrono::seconds{2}); + }; + } + SECTION("superseded latest command") { + retire = [&](CommandHandle) { + (void)scheduler->accept(request(CommandSemantic::kAngles, 2), AckSupport::kUnsupported, {}, + [](const auto &) { return delivered(); }); + }; + } + SECTION("invalidated pending commands") { + retire = [&](CommandHandle) { scheduler->invalidate(std::make_exception_ptr(ClosedError("test close"))); }; + } + SECTION("expired pending command") { + retained.timeout = std::chrono::milliseconds{20}; + retire = [](CommandHandle handle) { (void)handle.wait_for(std::chrono::seconds{2}); }; + } + const auto pending = + scheduler->accept(std::move(retained), AckSupport::kUnsupported, {}, [](const auto &) { return delivered(); }); + auto retiring = std::async(std::launch::async, [&] { retire(pending); }); + const auto entered = destruction.entered.get_future().wait_for(std::chrono::seconds{2}); + auto inspection = std::async(std::launch::async, [&] { return scheduler->metrics(); }); + const auto inspected = inspection.wait_for(std::chrono::seconds{1}); + // Always release the destructor before asserting, so a broken implementation + // fails the test without leaving a worker or async future permanently stuck. + destruction.open(); + retiring.get(); + execution.open(); + (void)inspection.get(); + // A terminal handle may precede the worker's resource release. Join it + // before destroying the barriers that its retained request still uses. + scheduler.reset(); + REQUIRE(entered == std::future_status::ready); + REQUIRE(inspected == std::future_status::ready); + REQUIRE(first.wait_for(std::chrono::seconds{2})); +} + TEST_CASE("M2 pending deadlines expire independently of executor and throttle", "[comm][m2][scheduler]") { CommandScheduler scheduler(0.5, 1); // Throttle must not delay expiry. Barrier barrier; diff --git a/tests/runtime/test_device_runtime_comm.cpp b/tests/runtime/test_device_runtime_comm.cpp index 595400e..d51ef8d 100644 --- a/tests/runtime/test_device_runtime_comm.cpp +++ b/tests/runtime/test_device_runtime_comm.cpp @@ -726,10 +726,10 @@ TEST_CASE("comm: command queue counts logical jobs independently of frames", "[c TEST_CASE("comm: runtime close wakes waiter and clears pending cache", "[comm][runtime]") { Rig rig; auto request = rig.runtime->submit(plan(), Clock::now() + 30s); - auto waiting = std::async(std::launch::async, [&] { REQUIRE_THROWS(request.wait()); }); + auto waiting = std::async(std::launch::async, [&] { return request.wait(); }); rig.runtime->close(); REQUIRE(waiting.wait_for(500ms) == std::future_status::ready); - waiting.get(); + REQUIRE_THROWS(waiting.get()); REQUIRE(rig.runtime->runtime_health().in_flight == 0); REQUIRE(rig.runtime->health().lifecycle_state == linkerbot::hand::LifecycleState::kClosed); REQUIRE_FALSE(rig.runtime->health().connected());