From 62e3a36d430b9dfb1e75d204257bf470c643c54a Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sat, 8 Aug 2026 17:09:02 -0500 Subject: [PATCH 01/18] refactor: stopped calling os clock. using rdtsc() based timing --- src/OrderBook.cpp | 85 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 3 deletions(-) diff --git a/src/OrderBook.cpp b/src/OrderBook.cpp index ffb2224..ebff683 100644 --- a/src/OrderBook.cpp +++ b/src/OrderBook.cpp @@ -2,14 +2,95 @@ #include #include +#include #include +#if defined(_MSC_VER) +#include +#endif + #include "Order.h" #include "Side.h" #include "Types.h" namespace orderbook { +namespace +{ + +#if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__) + +std::uint64_t readTsc() noexcept +{ +#if defined(_MSC_VER) + return __rdtsc(); +#else + return __builtin_ia32_rdtsc(); +#endif +} + +struct TscCalibration +{ + std::uint64_t baseCycles; + double nsPerCycle; + std::chrono::nanoseconds baseEpoch; +}; + +const TscCalibration& tscCalibration() +{ + static const TscCalibration calibration = [] + { + const auto wallStart = + std::chrono::time_point_cast(std::chrono::system_clock::now()); + const std::uint64_t cyclesStart = readTsc(); + + auto wallNow = wallStart; + std::uint64_t cyclesNow = cyclesStart; + do + { + wallNow = std::chrono::time_point_cast( + std::chrono::system_clock::now()); + cyclesNow = readTsc(); + } while (wallNow - wallStart < std::chrono::milliseconds{ 1 }); + + const double elapsedNs = + static_cast((wallNow - wallStart).count()); + const double nsPerCycle = elapsedNs / static_cast(cyclesNow - cyclesStart); + + return TscCalibration{ cyclesStart, nsPerCycle, wallStart.time_since_epoch() }; + }(); + + return calibration; +} + +// Calibrated once per process (a ~1ms one-time cost, amortized to nothing); +// every call after that is a TSC read plus a multiply, not a clock_gettime +// syscall/vDSO round trip. Requires an invariant TSC (constant_tsc, +// nonstop_tsc) to be meaningful across cores, which every x86-64 CPU built +// in the last decade-plus has. +Timestamp tscNow() noexcept +{ + const TscCalibration& calibration = tscCalibration(); + const std::uint64_t elapsedCycles = readTsc() - calibration.baseCycles; + const auto elapsedNs = + static_cast(static_cast(elapsedCycles) * calibration.nsPerCycle); + + return Timestamp{ calibration.baseEpoch + std::chrono::nanoseconds{ elapsedNs } }; +} + +#else + +Timestamp tscNow() noexcept +{ + const auto now = + std::chrono::time_point_cast(std::chrono::system_clock::now()); + return Timestamp{ now.time_since_epoch() }; +} + +#endif + +} // namespace + void OrderBook::pushBack(PriceLevel& level, Order* order) { order->prev = level.tail; @@ -198,9 +279,7 @@ void OrderBook::cleanupAfterTrade(Order* incoming, void OrderBook::matchOrder(Order* incoming) { - const auto now = - std::chrono::time_point_cast(std::chrono::system_clock::now()); - const Timestamp arrivalTime{ now.time_since_epoch() }; + const Timestamp arrivalTime = tscNow(); while (!incoming->isFilled() && canMatch(incoming)) { From 7b159d0edfda0c9f9c7b53c03de6b92577110fe4 Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sat, 8 Aug 2026 17:30:57 -0500 Subject: [PATCH 02/18] added: lock-free MPMC ConcurrentQueue for order ingest --- src/lockfree/ConcurrentQueue.h | 144 +++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 src/lockfree/ConcurrentQueue.h diff --git a/src/lockfree/ConcurrentQueue.h b/src/lockfree/ConcurrentQueue.h new file mode 100644 index 0000000..7014c91 --- /dev/null +++ b/src/lockfree/ConcurrentQueue.h @@ -0,0 +1,144 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace orderbook { + +// Bounded MPMC ring buffer (Dmitry Vyukov's per-cell sequence number design). +// Producers/consumers only ever contend on enqueuePos_/dequeuePos_ via CAS; +// each cell's own sequence number is what makes concurrent access to that +// cell's storage safe (see the acquire/release comments below). +template +class ConcurrentQueue +{ + // Power of two so `pos & Mask` is a cheap wraparound. Must be >= 2: + // at Capacity == 1, the "just filled" sequence value (pos + 1) and the + // "ready for next lap" value (pos + Capacity) are the same number, so a + // push immediately following another push (no pop between) would be + // wrongly accepted as if the slot were free. + static_assert(Capacity >= 2 && (Capacity & (Capacity - 1)) == 0, + "ConcurrentQueue capacity must be a power of two >= 2"); + +public: + ConcurrentQueue() + { + for (std::size_t i = 0; i < Capacity; ++i) + { + cells_[i].sequence.store(i, std::memory_order_relaxed); + } + } + + ~ConcurrentQueue() + { + // Drain any still-live elements directly rather than through tryPop: + // tryPop moves into an existing T&, and T (e.g. Order) need not be + // default-constructible, so callers may have nothing to move into. + // Assumes no concurrent push/pop is in flight during destruction, + // same assumption any standard container makes. + std::size_t head = dequeuePos_.load(std::memory_order_relaxed); + const std::size_t tail = enqueuePos_.load(std::memory_order_relaxed); + for (; head != tail; ++head) + { + cells_[head & Mask].storage.value.~T(); + } + } + + ConcurrentQueue(const ConcurrentQueue&) = delete; + ConcurrentQueue& operator=(const ConcurrentQueue&) = delete; + ConcurrentQueue(ConcurrentQueue&&) = delete; + ConcurrentQueue& operator=(ConcurrentQueue&&) = delete; + + // Takes T&& rather than T-by-value so a failed push (queue full) leaves + // the caller's object untouched -- by-value would move-construct the + // parameter unconditionally and then silently discard it on failure. + [[nodiscard]] bool tryPush(T&& value) + { + std::size_t pos = enqueuePos_.load(std::memory_order_relaxed); + for (;;) + { + Cell& cell = cells_[pos & Mask]; + const std::size_t seq = cell.sequence.load(std::memory_order_acquire); + const auto diff = static_cast(seq) - static_cast(pos); + + if (diff == 0) + { + if (enqueuePos_.compare_exchange_weak(pos, pos + 1, std::memory_order_relaxed)) + { + new (&cell.storage.value) T(std::move(value)); + cell.sequence.store(pos + 1, std::memory_order_release); + return true; + } + } + else if (diff < 0) + { + return false; + } + else + { + pos = enqueuePos_.load(std::memory_order_relaxed); + } + } + } + + [[nodiscard]] bool tryPop(T& out) + { + std::size_t pos = dequeuePos_.load(std::memory_order_relaxed); + for (;;) + { + Cell& cell = cells_[pos & Mask]; + const std::size_t seq = cell.sequence.load(std::memory_order_acquire); + const auto diff = + static_cast(seq) - static_cast(pos + 1); + + if (diff == 0) + { + if (dequeuePos_.compare_exchange_weak(pos, pos + 1, std::memory_order_relaxed)) + { + out = std::move(cell.storage.value); + cell.storage.value.~T(); + cell.sequence.store(pos + Capacity, std::memory_order_release); + return true; + } + } + else if (diff < 0) + { + return false; + } + else + { + pos = dequeuePos_.load(std::memory_order_relaxed); + } + } + } + +private: + static constexpr std::size_t Mask = Capacity - 1; + + struct Cell + { + std::atomic sequence; + + union Storage + { + T value; + + Storage() {} + Storage(const Storage&) = delete; + Storage& operator=(const Storage&) = delete; + Storage(Storage&&) = delete; + Storage& operator=(Storage&&) = delete; + ~Storage() {} + } storage; + }; + + alignas(64) std::atomic enqueuePos_{ 0 }; + alignas(64) std::atomic dequeuePos_{ 0 }; + std::array cells_{}; +}; + +} From d0fe35df32de6e77e424968d95c3914d10dbad91 Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sat, 8 Aug 2026 19:42:34 -0500 Subject: [PATCH 03/18] added: ConcurrentQueue why comments and CMake wiring for its tests --- CMakeLists.txt | 5 ++- src/lockfree/ConcurrentQueue.h | 65 ++++++++++++++++++++-------------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 00e7623..2f72ce0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,6 +95,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) include(FetchContent) enable_testing() + find_package(Threads REQUIRED) + FetchContent_Declare( googletest GIT_REPOSITORY https://github.com/google/googletest.git @@ -122,10 +124,11 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) tests/order_pool_tests.cpp tests/book_printer_tests.cpp tests/order_index_tests.cpp + tests/concurrent_queue_tests.cpp ) add_executable(orderbook_tests ${TEST_SOURCES}) - target_link_libraries(orderbook_tests PRIVATE orderbook GTest::gtest_main) + target_link_libraries(orderbook_tests PRIVATE orderbook GTest::gtest_main Threads::Threads) orderbook_apply_hardening(orderbook_tests) if(ipo_ok) set_target_properties(orderbook_tests PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ON) diff --git a/src/lockfree/ConcurrentQueue.h b/src/lockfree/ConcurrentQueue.h index 7014c91..362f014 100644 --- a/src/lockfree/ConcurrentQueue.h +++ b/src/lockfree/ConcurrentQueue.h @@ -7,24 +7,20 @@ #include #include -namespace orderbook { +namespace orderbook +{ -// Bounded MPMC ring buffer (Dmitry Vyukov's per-cell sequence number design). -// Producers/consumers only ever contend on enqueuePos_/dequeuePos_ via CAS; -// each cell's own sequence number is what makes concurrent access to that -// cell's storage safe (see the acquire/release comments below). -template +// Bounded MPMC ring buffer, Vyukov's per-cell sequence number design. +// Each cell's own sequence number is what makes it safe, not a lock. +template class ConcurrentQueue { - // Power of two so `pos & Mask` is a cheap wraparound. Must be >= 2: - // at Capacity == 1, the "just filled" sequence value (pos + 1) and the - // "ready for next lap" value (pos + Capacity) are the same number, so a - // push immediately following another push (no pop between) would be - // wrongly accepted as if the slot were free. + // Power of two so `pos & Mask` is a cheap wraparound. Capacity of 1 + // breaks the lap math, since "just filled" and "next lap" collide. static_assert(Capacity >= 2 && (Capacity & (Capacity - 1)) == 0, "ConcurrentQueue capacity must be a power of two >= 2"); -public: + public: ConcurrentQueue() { for (std::size_t i = 0; i < Capacity; ++i) @@ -35,11 +31,8 @@ class ConcurrentQueue ~ConcurrentQueue() { - // Drain any still-live elements directly rather than through tryPop: - // tryPop moves into an existing T&, and T (e.g. Order) need not be - // default-constructible, so callers may have nothing to move into. - // Assumes no concurrent push/pop is in flight during destruction, - // same assumption any standard container makes. + // tryPop needs a T& to move into, and T may have no default + // constructor, so we destroy what's left directly instead. std::size_t head = dequeuePos_.load(std::memory_order_relaxed); const std::size_t tail = enqueuePos_.load(std::memory_order_relaxed); for (; head != tail; ++head) @@ -53,20 +46,25 @@ class ConcurrentQueue ConcurrentQueue(ConcurrentQueue&&) = delete; ConcurrentQueue& operator=(ConcurrentQueue&&) = delete; - // Takes T&& rather than T-by-value so a failed push (queue full) leaves - // the caller's object untouched -- by-value would move-construct the - // parameter unconditionally and then silently discard it on failure. + // T&& instead of T by value: a failed push should leave the caller's + // object alone, and by value would move-construct it regardless. [[nodiscard]] bool tryPush(T&& value) { std::size_t pos = enqueuePos_.load(std::memory_order_relaxed); for (;;) { Cell& cell = cells_[pos & Mask]; + + // Acquire pairs with the release stores below and in tryPop. + // It guarantees we see whatever the last thread wrote here. const std::size_t seq = cell.sequence.load(std::memory_order_acquire); const auto diff = static_cast(seq) - static_cast(pos); if (diff == 0) { + // enqueuePos_ only tracks whose turn is next, not data, + // so relaxed is fine here. The acquire/release pair on + // sequence is what actually protects the storage. if (enqueuePos_.compare_exchange_weak(pos, pos + 1, std::memory_order_relaxed)) { new (&cell.storage.value) T(std::move(value)); @@ -76,10 +74,13 @@ class ConcurrentQueue } else if (diff < 0) { + // Nobody has freed this slot yet, so the queue is full. return false; } else { + // Someone else already took this turn. Our pos is stale, + // so reload it and try again. pos = enqueuePos_.load(std::memory_order_relaxed); } } @@ -92,8 +93,10 @@ class ConcurrentQueue { Cell& cell = cells_[pos & Mask]; const std::size_t seq = cell.sequence.load(std::memory_order_acquire); - const auto diff = - static_cast(seq) - static_cast(pos + 1); + + // A slot is only poppable once a producer fills it, one lap + // ahead of empty. That's the + 1 tryPush doesn't have. + const auto diff = static_cast(seq) - static_cast(pos + 1); if (diff == 0) { @@ -101,12 +104,16 @@ class ConcurrentQueue { out = std::move(cell.storage.value); cell.storage.value.~T(); + + // Republish as pos + Capacity, not pos + 1. That's the + // number this same slot needs one lap from now. cell.sequence.store(pos + Capacity, std::memory_order_release); return true; } } else if (diff < 0) { + // Nothing has filled this slot yet, so the queue is empty. return false; } else @@ -116,13 +123,16 @@ class ConcurrentQueue } } -private: + private: static constexpr std::size_t Mask = Capacity - 1; struct Cell { std::atomic sequence; + // We use a union instead of a plain T member because Order has no + // default constructor and can't be copied. OrderSlab's Slot union + // solves the same problem the same way. union Storage { T value; @@ -136,9 +146,12 @@ class ConcurrentQueue } storage; }; - alignas(64) std::atomic enqueuePos_{ 0 }; - alignas(64) std::atomic dequeuePos_{ 0 }; + // alignas(64) prevents enqueuePos_ and dequeuePos_ from being on the + // same cache line. If they were, producer and consumer threads would + // thrash it every push and pop, even though the counters are unrelated. + alignas(64) std::atomic enqueuePos_{0}; + alignas(64) std::atomic dequeuePos_{0}; std::array cells_{}; }; -} +} // namespace orderbook From b69ffe54986b9b7fe258ca3c72d18bd14c300e7d Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sat, 8 Aug 2026 19:43:25 -0500 Subject: [PATCH 04/18] added: unit and multithreaded stress tests for ConcurrentQueue --- tests/concurrent_queue_tests.cpp | 271 +++++++++++++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 tests/concurrent_queue_tests.cpp diff --git a/tests/concurrent_queue_tests.cpp b/tests/concurrent_queue_tests.cpp new file mode 100644 index 0000000..3c75eb9 --- /dev/null +++ b/tests/concurrent_queue_tests.cpp @@ -0,0 +1,271 @@ +#include +#include +#include +#include +#include + +#include + +#include "Order.h" +#include "lockfree/ConcurrentQueue.h" + +using namespace orderbook; + +namespace +{ + +// Mirrors Order: no default constructor, move-only. Counts its own +// destructor calls so the drain tests can check nothing leaked. +struct Tracked +{ + int value; + int* dtorCount; + + Tracked(int v, int* counter) : value(v), dtorCount(counter) {} + Tracked(const Tracked&) = delete; + Tracked& operator=(const Tracked&) = delete; + + // A defaulted move would copy dtorCount without clearing the source, + // so a moved-from husk would double count when it's later destroyed. + // We null it out here so only the current owner ever counts. + Tracked(Tracked&& other) noexcept : value(other.value), dtorCount(other.dtorCount) + { + other.dtorCount = nullptr; + } + Tracked& operator=(Tracked&& other) noexcept + { + value = other.value; + dtorCount = other.dtorCount; + other.dtorCount = nullptr; + return *this; + } + + ~Tracked() + { + if (dtorCount != nullptr) + { + ++(*dtorCount); + } + } +}; + +} // namespace + +TEST(ConcurrentQueue, PushThenPopReturnsSameValue) +{ + ConcurrentQueue queue; + int out = 0; + + EXPECT_TRUE(queue.tryPush(42)); + EXPECT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out, 42); +} + +TEST(ConcurrentQueue, PopOnEmptyReturnsFalse) +{ + ConcurrentQueue queue; + int out = 0; + + EXPECT_FALSE(queue.tryPop(out)); +} + +TEST(ConcurrentQueue, PushToCapacityThenOneMoreFails) +{ + ConcurrentQueue queue; + + EXPECT_TRUE(queue.tryPush(1)); + EXPECT_TRUE(queue.tryPush(2)); + EXPECT_TRUE(queue.tryPush(3)); + EXPECT_TRUE(queue.tryPush(4)); + + // No pop has happened, so every slot is still on its first lap. + EXPECT_FALSE(queue.tryPush(5)); +} + +TEST(ConcurrentQueue, PreservesFifoOrder) +{ + ConcurrentQueue queue; + + EXPECT_TRUE(queue.tryPush(10)); + EXPECT_TRUE(queue.tryPush(20)); + EXPECT_TRUE(queue.tryPush(30)); + + int out = 0; + EXPECT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out, 10); + EXPECT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out, 20); + EXPECT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out, 30); +} + +TEST(ConcurrentQueue, FailedPushLeavesTheCallersValueUntouched) +{ + // unique_ptr proves tryPush only moves from its argument on success. + // A wrong implementation would null ptr out even though this fails. + ConcurrentQueue, 2> queue; + + EXPECT_TRUE(queue.tryPush(std::make_unique(1))); + EXPECT_TRUE(queue.tryPush(std::make_unique(2))); + + std::unique_ptr ptr = std::make_unique(99); + EXPECT_FALSE(queue.tryPush(std::move(ptr))); + ASSERT_NE(ptr, nullptr); + EXPECT_EQ(*ptr, 99); +} + +TEST(ConcurrentQueue, WorksWithOrderItsActualMotivatingType) +{ + // Order has no default constructor and can't be copied. This is the + // exact constraint the queue was designed around. + ConcurrentQueue queue; + + Order order = OrderBuilder{} + .id(OrderId{ 7 }) + .buy() + .goodTillCancel() + .price(Price{ 100 }) + .quantity(Quantity{ 5 }) + .build(); + + EXPECT_TRUE(queue.tryPush(std::move(order))); + + Order out = OrderBuilder{} + .id(OrderId{ 0 }) + .buy() + .goodTillCancel() + .price(Price{ 1 }) + .quantity(Quantity{ 1 }) + .build(); + EXPECT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out.getOrderId(), OrderId{ 7 }); + EXPECT_EQ(out.getPrice(), Price{ 100 }); + EXPECT_EQ(out.getRemainingQuantity(), Quantity{ 5 }); +} + +TEST(ConcurrentQueue, DestructorDrainsWhateverIsStillInTheQueue) +{ + int dtorCount = 0; + + { + ConcurrentQueue queue; + EXPECT_TRUE(queue.tryPush(Tracked{ 1, &dtorCount })); + EXPECT_TRUE(queue.tryPush(Tracked{ 2, &dtorCount })); + EXPECT_TRUE(queue.tryPush(Tracked{ 3, &dtorCount })); + + // Only pop one. The other two are still live when queue is + // destroyed below, so the destructor's drain has to clean them up. + Tracked out{ 0, nullptr }; + EXPECT_TRUE(queue.tryPop(out)); + } + + EXPECT_EQ(dtorCount, 3); +} + +TEST(ConcurrentQueue, WrapsAroundCorrectlyAcrossManyLaps) +{ + ConcurrentQueue queue; + + // An off-by-one-lap bug only shows up once a slot gets reused, not on + // the first pass. 50 laps forces every slot through many reuses. + for (int lap = 0; lap < 50; ++lap) + { + for (int i = 0; i < 4; ++i) + { + ASSERT_TRUE(queue.tryPush(lap * 100 + i)) << "lap " << lap << " push " << i; + } + for (int i = 0; i < 4; ++i) + { + int out = 0; + ASSERT_TRUE(queue.tryPop(out)) << "lap " << lap << " pop " << i; + ASSERT_EQ(out, lap * 100 + i) << "lap " << lap << " pop " << i; + } + } +} + +TEST(ConcurrentQueue, HandlesFillDrainPatternsNotAlignedToCapacity) +{ + ConcurrentQueue queue; + + for (int i = 0; i < 4; ++i) + { + ASSERT_TRUE(queue.tryPush(int{ i })); + } + + int out = 0; + ASSERT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out, 0); + ASSERT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out, 1); + + // These two land on an offset that isn't a multiple of capacity. + ASSERT_TRUE(queue.tryPush(4)); + ASSERT_TRUE(queue.tryPush(5)); + + for (int expected : { 2, 3, 4, 5 }) + { + ASSERT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out, expected); + } + EXPECT_FALSE(queue.tryPop(out)); +} + +TEST(ConcurrentQueue, ManyProducersOneConsumerSeeEveryValueExactlyOnce) +{ + constexpr int ProducerCount = 4; + constexpr int PerProducer = 50000; + constexpr int Total = ProducerCount * PerProducer; + + ConcurrentQueue queue; + std::vector> seenCount(Total); + for (auto& counter : seenCount) + { + counter.store(0, std::memory_order_relaxed); + } + + std::vector producers; + for (int p = 0; p < ProducerCount; ++p) + { + producers.emplace_back( + [p, &queue] + { + for (int i = 0; i < PerProducer; ++i) + { + // Each producer owns a disjoint range, so one unique + // value proves it was neither dropped nor duplicated. + const int value = p * PerProducer + i; + while (!queue.tryPush(int{ value })) + { + // The queue is bounded, so a full ring under real + // contention is expected. Just retry. + } + } + }); + } + + std::thread consumer( + [&] + { + int received = 0; + while (received < Total) + { + int value = 0; + if (queue.tryPop(value)) + { + seenCount[static_cast(value)].fetch_add(1, std::memory_order_relaxed); + ++received; + } + } + }); + + for (auto& producer : producers) + { + producer.join(); + } + consumer.join(); + + for (int i = 0; i < Total; ++i) + { + ASSERT_EQ(seenCount[static_cast(i)].load(), 1) << "value " << i; + } +} From 4bfe6231e2c8100c82671e34fefec286170cc47d Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sat, 8 Aug 2026 19:43:42 -0500 Subject: [PATCH 05/18] added: mutex vs lock-free contention benchmark for ConcurrentQueue --- CMakeLists.txt | 6 ++- bench/concurrent_queue_bench.cpp | 83 ++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 bench/concurrent_queue_bench.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f72ce0..b3b78b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -141,12 +141,13 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) bench/cancel_order_bench.cpp bench/depth_bench.cpp bench/level_info_bench.cpp + bench/concurrent_queue_bench.cpp ) foreach(BENCH_SRC ${BENCH_SOURCES}) get_filename_component(BENCH_NAME ${BENCH_SRC} NAME_WE) add_executable(${BENCH_NAME} ${BENCH_SRC}) - target_link_libraries(${BENCH_NAME} PRIVATE orderbook benchmark::benchmark) + target_link_libraries(${BENCH_NAME} PRIVATE orderbook benchmark::benchmark Threads::Threads) target_include_directories(${BENCH_NAME} PRIVATE bench) orderbook_apply_hardening(${BENCH_NAME}) if(ipo_ok) @@ -164,8 +165,9 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) bench/cancel_order_bench.cpp bench/depth_bench.cpp bench/level_info_bench.cpp + bench/concurrent_queue_bench.cpp ) - target_link_libraries(orderbook_bench_all PRIVATE orderbook benchmark::benchmark_main) + target_link_libraries(orderbook_bench_all PRIVATE orderbook benchmark::benchmark_main Threads::Threads) target_include_directories(orderbook_bench_all PRIVATE bench) target_compile_definitions(orderbook_bench_all PRIVATE BENCHMARK_COMBINED) orderbook_apply_hardening(orderbook_bench_all) diff --git a/bench/concurrent_queue_bench.cpp b/bench/concurrent_queue_bench.cpp new file mode 100644 index 0000000..415cd12 --- /dev/null +++ b/bench/concurrent_queue_bench.cpp @@ -0,0 +1,83 @@ +#include + +#include +#include +#include + +#include "lockfree/ConcurrentQueue.h" + +using orderbook::ConcurrentQueue; + +namespace +{ +constexpr std::size_t kCapacity = 4096; +} + +// Baseline: a single mutex-guarded std::queue, contended by N threads. +static void BM_MutexQueue_PushPop(benchmark::State& state) +{ + static std::mutex mutex; + static std::queue queue; + + std::uint64_t value = 0; + for (auto _ : state) + { + { + std::lock_guard lock(mutex); + queue.push(value); + } + + // Retry until a pop actually succeeds, so both benchmarks do the + // same amount of work: one push, one pop, every iteration. + std::uint64_t popped = 0; + bool ok = false; + while (!ok) + { + std::lock_guard lock(mutex); + if (!queue.empty()) + { + popped = queue.front(); + queue.pop(); + ok = true; + } + else + { + __builtin_ia32_pause(); + } + } + + benchmark::DoNotOptimize(popped); + ++value; + } +} +BENCHMARK(BM_MutexQueue_PushPop)->Threads(1)->Threads(2)->Threads(4)->Threads(8); + +// Same workload, lock-free queue instead of a mutex. +static void BM_ConcurrentQueue_PushPop(benchmark::State& state) +{ + static ConcurrentQueue queue; + + std::uint64_t value = 0; + for (auto _ : state) + { + std::uint64_t toPush = value; + while (!queue.tryPush(std::move(toPush))) + { + __builtin_ia32_pause(); + } + + std::uint64_t popped = 0; + while (!queue.tryPop(popped)) + { + __builtin_ia32_pause(); + } + + benchmark::DoNotOptimize(popped); + ++value; + } +} +BENCHMARK(BM_ConcurrentQueue_PushPop)->Threads(1)->Threads(2)->Threads(4)->Threads(8); + +#ifndef BENCHMARK_COMBINED +BENCHMARK_MAIN(); +#endif From 22ae3dbd7e677ea5fea04174c3a7c07281149d62 Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sun, 9 Aug 2026 17:33:31 -0500 Subject: [PATCH 06/18] added: N producer to 1 consumer ingest pipeline benchmark --- CMakeLists.txt | 2 + bench/ingest_pipeline_bench.cpp | 147 ++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 bench/ingest_pipeline_bench.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b3b78b0..bb65b75 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,6 +142,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) bench/depth_bench.cpp bench/level_info_bench.cpp bench/concurrent_queue_bench.cpp + bench/ingest_pipeline_bench.cpp ) foreach(BENCH_SRC ${BENCH_SOURCES}) @@ -166,6 +167,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) bench/depth_bench.cpp bench/level_info_bench.cpp bench/concurrent_queue_bench.cpp + bench/ingest_pipeline_bench.cpp ) target_link_libraries(orderbook_bench_all PRIVATE orderbook benchmark::benchmark_main Threads::Threads) target_include_directories(orderbook_bench_all PRIVATE bench) diff --git a/bench/ingest_pipeline_bench.cpp b/bench/ingest_pipeline_bench.cpp new file mode 100644 index 0000000..9130ea5 --- /dev/null +++ b/bench/ingest_pipeline_bench.cpp @@ -0,0 +1,147 @@ +#include + +#include +#include +#include + +#include "Order.h" +#include "OrderBook.h" +#include "lockfree/ConcurrentQueue.h" + +using orderbook::ConcurrentQueue; +using orderbook::Order; +using orderbook::OrderBook; +using orderbook::OrderBuilder; +using orderbook::OrderId; +using orderbook::Price; +using orderbook::Quantity; + +namespace +{ +constexpr std::size_t kQueueCapacity = 4096; +constexpr Price kMinPrice{ 1 }; +constexpr Price kMaxPrice{ 12'000 }; +constexpr std::size_t kBookCapacity = 2'000'000; + +Order makeOrder(std::uint64_t id) +{ + return OrderBuilder{}.id(OrderId{ id }).buy().goodTillCancel().price(Price{ 100 }).quantity(Quantity{ 10 }).build(); +} +} // namespace + +// N producer threads, thread 0 is the sole consumer draining into OrderBook. +// This is the actual topology v3 is built for: every atomic in the queue is +// only ever contended from one side, unlike a benchmark where every thread +// does both a push and a pop. +static void BM_IngestPipeline_Mutex(benchmark::State& state) +{ + static std::mutex* mutex = nullptr; + static std::queue* queue = nullptr; + static OrderBook* book = nullptr; + + const int producerCount = state.threads() - 1; + + if (state.thread_index() == 0) + { + mutex = new std::mutex(); + queue = new std::queue(); + book = new OrderBook(kMinPrice, kMaxPrice, kBookCapacity); + } + + if (state.thread_index() == 0) + { + for (auto _ : state) + { + for (int p = 0; p < producerCount; ++p) + { + for (;;) + { + { + std::lock_guard lock(*mutex); + if (!queue->empty()) + { + Order order = std::move(queue->front()); + queue->pop(); + auto trades = book->addOrder(std::move(order)); + benchmark::DoNotOptimize(trades); + break; + } + } + __builtin_ia32_pause(); + } + } + } + } + else + { + std::uint64_t id = static_cast(state.thread_index()) * 100'000'000ULL; + for (auto _ : state) + { + std::lock_guard lock(*mutex); + queue->push(makeOrder(++id)); + } + } + + if (state.thread_index() == 0) + { + delete book; + delete queue; + delete mutex; + } +} +BENCHMARK(BM_IngestPipeline_Mutex)->Threads(2)->Threads(4)->Threads(8)->Iterations(20000); + +// Same topology, lock-free queue instead of a mutex. +static void BM_IngestPipeline_LockFree(benchmark::State& state) +{ + static ConcurrentQueue* queue = nullptr; + static OrderBook* book = nullptr; + + const int producerCount = state.threads() - 1; + + if (state.thread_index() == 0) + { + queue = new ConcurrentQueue(); + book = new OrderBook(kMinPrice, kMaxPrice, kBookCapacity); + } + + if (state.thread_index() == 0) + { + Order slot = makeOrder(0); + for (auto _ : state) + { + for (int p = 0; p < producerCount; ++p) + { + while (!queue->tryPop(slot)) + { + __builtin_ia32_pause(); + } + auto trades = book->addOrder(std::move(slot)); + benchmark::DoNotOptimize(trades); + } + } + } + else + { + std::uint64_t id = static_cast(state.thread_index()) * 100'000'000ULL; + for (auto _ : state) + { + Order order = makeOrder(++id); + while (!queue->tryPush(std::move(order))) + { + __builtin_ia32_pause(); + } + } + } + + if (state.thread_index() == 0) + { + delete book; + delete queue; + } +} +BENCHMARK(BM_IngestPipeline_LockFree)->Threads(2)->Threads(4)->Threads(8)->Iterations(20000); + +#ifndef BENCHMARK_COMBINED +BENCHMARK_MAIN(); +#endif From 35d73a795304b9ce2e034869db465488e333b707 Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sun, 9 Aug 2026 17:33:49 -0500 Subject: [PATCH 07/18] removed: symmetric push-pop ConcurrentQueue bench, contention pattern didn't match real usage --- CMakeLists.txt | 2 - bench/concurrent_queue_bench.cpp | 83 -------------------------------- 2 files changed, 85 deletions(-) delete mode 100644 bench/concurrent_queue_bench.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index bb65b75..99c7495 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -141,7 +141,6 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) bench/cancel_order_bench.cpp bench/depth_bench.cpp bench/level_info_bench.cpp - bench/concurrent_queue_bench.cpp bench/ingest_pipeline_bench.cpp ) @@ -166,7 +165,6 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) bench/cancel_order_bench.cpp bench/depth_bench.cpp bench/level_info_bench.cpp - bench/concurrent_queue_bench.cpp bench/ingest_pipeline_bench.cpp ) target_link_libraries(orderbook_bench_all PRIVATE orderbook benchmark::benchmark_main Threads::Threads) diff --git a/bench/concurrent_queue_bench.cpp b/bench/concurrent_queue_bench.cpp deleted file mode 100644 index 415cd12..0000000 --- a/bench/concurrent_queue_bench.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include - -#include -#include -#include - -#include "lockfree/ConcurrentQueue.h" - -using orderbook::ConcurrentQueue; - -namespace -{ -constexpr std::size_t kCapacity = 4096; -} - -// Baseline: a single mutex-guarded std::queue, contended by N threads. -static void BM_MutexQueue_PushPop(benchmark::State& state) -{ - static std::mutex mutex; - static std::queue queue; - - std::uint64_t value = 0; - for (auto _ : state) - { - { - std::lock_guard lock(mutex); - queue.push(value); - } - - // Retry until a pop actually succeeds, so both benchmarks do the - // same amount of work: one push, one pop, every iteration. - std::uint64_t popped = 0; - bool ok = false; - while (!ok) - { - std::lock_guard lock(mutex); - if (!queue.empty()) - { - popped = queue.front(); - queue.pop(); - ok = true; - } - else - { - __builtin_ia32_pause(); - } - } - - benchmark::DoNotOptimize(popped); - ++value; - } -} -BENCHMARK(BM_MutexQueue_PushPop)->Threads(1)->Threads(2)->Threads(4)->Threads(8); - -// Same workload, lock-free queue instead of a mutex. -static void BM_ConcurrentQueue_PushPop(benchmark::State& state) -{ - static ConcurrentQueue queue; - - std::uint64_t value = 0; - for (auto _ : state) - { - std::uint64_t toPush = value; - while (!queue.tryPush(std::move(toPush))) - { - __builtin_ia32_pause(); - } - - std::uint64_t popped = 0; - while (!queue.tryPop(popped)) - { - __builtin_ia32_pause(); - } - - benchmark::DoNotOptimize(popped); - ++value; - } -} -BENCHMARK(BM_ConcurrentQueue_PushPop)->Threads(1)->Threads(2)->Threads(4)->Threads(8); - -#ifndef BENCHMARK_COMBINED -BENCHMARK_MAIN(); -#endif From 6f2a9a3db0ca64ad24f610a345211ea823a235b3 Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sun, 9 Aug 2026 17:33:55 -0500 Subject: [PATCH 08/18] fixed: depth benchmarks taking 80+ seconds from auto-calibrated iteration counts multiplying O(depth) refill cost --- bench/depth_bench.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bench/depth_bench.cpp b/bench/depth_bench.cpp index 8a10923..045c697 100644 --- a/bench/depth_bench.cpp +++ b/bench/depth_bench.cpp @@ -29,7 +29,9 @@ static void BM_Depth_AddAtBest(benchmark::State& state) { benchmark::DoNotOptimize(trades); } } -BENCHMARK(BM_Depth_AddAtBest)->Arg(0)->Arg(100)->Arg(1000)->Arg(10000); +// Iterations capped. Refill above is O(depth), and auto-calibrated +// iteration counts in the millions turn that into minutes of wall time. +BENCHMARK(BM_Depth_AddAtBest)->Arg(0)->Arg(100)->Arg(1000)->Arg(10000)->Iterations(20000); // Add orders at random prices across the price range. // Forces tree traversal on every add. Measures worst-case map descent. @@ -56,7 +58,7 @@ static void BM_Depth_AddScatter(benchmark::State& state) { benchmark::DoNotOptimize(trades); } } -BENCHMARK(BM_Depth_AddScatter)->Arg(100)->Arg(1000)->Arg(10000); +BENCHMARK(BM_Depth_AddScatter)->Arg(100)->Arg(1000)->Arg(10000)->Iterations(20000); // Matching that sweeps through multiple price levels. // Aggressive buy crosses spread and walks up the ask ladder. @@ -106,7 +108,7 @@ static void BM_Depth_SweepLevels(benchmark::State& state) { benchmark::DoNotOptimize(trades); } } -BENCHMARK(BM_Depth_SweepLevels)->Arg(2)->Arg(5)->Arg(10)->Arg(20); +BENCHMARK(BM_Depth_SweepLevels)->Arg(2)->Arg(5)->Arg(10)->Arg(20)->Iterations(20000); #ifndef BENCHMARK_COMBINED BENCHMARK_MAIN(); From b2e997bbb13db91bb483853f62d2590a927c3185 Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sun, 9 Aug 2026 17:39:26 -0500 Subject: [PATCH 09/18] ci: added linux-tsan job for the ConcurrentQueue race tests --- .github/workflows/ci.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ec551a..e87649d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,34 @@ jobs: UBSAN_OPTIONS: print_stacktrace=1,halt_on_error=1 run: ctest --test-dir build --output-on-failure + linux-tsan: + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v4 + + - name: Install toolchain + run: | + sudo apt-get update -y + sudo apt-get install -y clang + + - name: Configure CMake + env: + CC: clang + CXX: clang++ + run: > + cmake -B build -DCMAKE_BUILD_TYPE=Debug + -DORDERBOOK_WARNINGS_AS_ERRORS=ON + -DORDERBOOK_SANITIZERS=thread + + - name: Build + run: cmake --build build -j + + - name: Run tests under TSan + env: + TSAN_OPTIONS: halt_on_error=1 + run: ctest --test-dir build --output-on-failure + clang-tidy: runs-on: ubuntu-24.04 From 2e506c052079ffc329940a7389ee8700f79d8fe6 Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sun, 9 Aug 2026 17:46:15 -0500 Subject: [PATCH 10/18] perf: packed Order into one cache line (72 to 64 bytes), cache-aligned OrderSlab slots --- src/Order.h | 13 ++++++++----- src/OrderSlab.h | 4 +++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/Order.h b/src/Order.h index a7fcd5e..495419b 100644 --- a/src/Order.h +++ b/src/Order.h @@ -17,13 +17,13 @@ class Order Order* next{ nullptr }; Order(OrderType type, OrderId id, Side side, Price price, Quantity quantity, Timestamp ts) - : orderType_{ type } - , orderId_{ id } - , side_{ side } + : orderId_{ id } , price_{ price } , initialQuantity_{ quantity } , remainingQuantity_{ quantity } , timestamp_{ ts } + , orderType_{ type } + , side_{ side } { } @@ -52,13 +52,16 @@ class Order static Timestamp now(); private: - OrderType orderType_; + // Order is one cache line (64 bytes) with this field order. The two + // 1-byte enums are kept together so neither pads out to an 8-byte + // boundary on its own. OrderId orderId_; - Side side_; Price price_; Quantity initialQuantity_; Quantity remainingQuantity_; Timestamp timestamp_; + OrderType orderType_; + Side side_; }; class OrderBuilder diff --git a/src/OrderSlab.h b/src/OrderSlab.h index 57da193..f669200 100644 --- a/src/OrderSlab.h +++ b/src/OrderSlab.h @@ -49,7 +49,9 @@ class OrderSlab ~OrderSlab() = default; private: - union Slot + // alignas(64) so each slot starts on its own cache line. Order is sized + // to exactly fill one, so this stops adjacent orders from splitting one. + union alignas(64) Slot { Order order; Slot* nextFree; From ac16d8a72d6409faa13ff01090fa5803405e5716 Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sun, 9 Aug 2026 17:46:18 -0500 Subject: [PATCH 11/18] perf: moved Trade into tradeBuffer_ instead of copying it --- src/OrderBook.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/OrderBook.cpp b/src/OrderBook.cpp index ebff683..9c918fb 100644 --- a/src/OrderBook.cpp +++ b/src/OrderBook.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #if defined(_MSC_VER) @@ -300,7 +301,7 @@ void OrderBook::matchOrder(Order* incoming) incoming->fill(tradeQuantity); resting->fill(tradeQuantity); - tradeBuffer_.push_back(trade); + tradeBuffer_.push_back(std::move(trade)); cleanupAfterTrade(incoming, resting, *level, tradeQuantity); } From d6bc3f86baff8352b84c90db6146a8a5aaab6708 Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sun, 9 Aug 2026 18:19:53 -0500 Subject: [PATCH 12/18] perf: fused OrderIndex duplicate check and insert into one hash and probe --- src/OrderBook.cpp | 9 +++++---- src/OrderBook.h | 6 +++++- src/OrderIndex.h | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/src/OrderBook.cpp b/src/OrderBook.cpp index 9c918fb..522177b 100644 --- a/src/OrderBook.cpp +++ b/src/OrderBook.cpp @@ -136,7 +136,7 @@ void OrderBook::unlink(PriceLevel& level, Order* order) level.orderCount--; } -void OrderBook::addToBook(Order* order) +void OrderBook::addToBook(Order* order, std::size_t indexHint) { const Price price = order->getPrice(); const Side side = order->getSide(); @@ -145,7 +145,7 @@ void OrderBook::addToBook(Order* order) pushBack(level, order); - orders_.insert(order->getOrderId(), order); + orders_.insertAt(indexHint, order->getOrderId(), order); } bool OrderBook::acceptsPrice(const Order& order) const noexcept @@ -311,7 +311,8 @@ std::span OrderBook::addOrder(Order order) { tradeBuffer_.clear(); - if (orders_.contains(order.getOrderId())) + const OrderIndex::SlotLookup slot = orders_.findSlot(order.getOrderId()); + if (slot.occupied) { return {}; } @@ -340,7 +341,7 @@ std::span OrderBook::addOrder(Order order) if (restsInBook) { - addToBook(incoming); + addToBook(incoming, slot.index); } else { diff --git a/src/OrderBook.h b/src/OrderBook.h index 0d85da7..26d9167 100644 --- a/src/OrderBook.h +++ b/src/OrderBook.h @@ -51,7 +51,11 @@ class OrderBook Trade createTrade(Order* incoming, Order* resting, Quantity quantity, Timestamp timestamp); void cleanupAfterTrade(Order* incoming, Order* resting, PriceLevel& level, Quantity tradeQty); void matchOrder(Order* incomingOrder); - void addToBook(Order* order); + + // indexHint must come from the OrderIndex::SlotLookup that addOrder got + // from findSlot for this same order. It lets insertAt skip re-hashing + // and re-probing an id we already located once. + void addToBook(Order* order, std::size_t indexHint); bool removeOrder(OrderId orderId); static void pushBack(PriceLevel& level, Order* order); diff --git a/src/OrderIndex.h b/src/OrderIndex.h index 63a64e8..2b52745 100644 --- a/src/OrderIndex.h +++ b/src/OrderIndex.h @@ -46,6 +46,46 @@ class OrderIndex [[nodiscard]] bool contains(OrderId id) const noexcept { return find(id) != nullptr; } + struct SlotLookup + { + std::size_t index; + bool occupied; + }; + + // OrderBook calls this once to both check for a duplicate id and find + // where a new one would go, then reuses the index with insertAt below + // instead of hashing and probing the same id a second time. + [[nodiscard]] SlotLookup findSlot(OrderId id) noexcept + { + if ((size_ + 1) * MaxLoadDenominator > slots_.size() * MaxLoadNumerator) + { + grow(); + } + + const std::size_t mask = slots_.size() - 1; + std::size_t index = hash(id) & mask; + + while (slots_[index].order != nullptr) + { + if (slots_[index].id == id) + { + return { index, true }; + } + index = (index + 1) & mask; + } + + return { index, false }; + } + + // Only valid for an index just returned by findSlot, before any other + // insert on this table. grow() would move everything and invalidate it, + // but findSlot already grows up front so that can't happen in between. + void insertAt(std::size_t index, OrderId id, Order* order) noexcept + { + slots_[index] = Slot{ .id = id, .order = order }; + ++size_; + } + void insert(OrderId id, Order* order) { if ((size_ + 1) * MaxLoadDenominator > slots_.size() * MaxLoadNumerator) From 7ee1626a2300972e0d7888d71712c2fd93afad9d Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sun, 9 Aug 2026 18:21:03 -0500 Subject: [PATCH 13/18] perf: order construction now uses the shared TSC clock instead of system_clock --- src/Order.cpp | 7 ++-- src/OrderBook.cpp | 83 +--------------------------------------- src/TscClock.h | 96 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 86 deletions(-) create mode 100644 src/TscClock.h diff --git a/src/Order.cpp b/src/Order.cpp index f325128..c26b7e8 100644 --- a/src/Order.cpp +++ b/src/Order.cpp @@ -1,9 +1,10 @@ #include "Order.h" -#include #include #include +#include "TscClock.h" + namespace orderbook { void Order::fill(Quantity qty) @@ -20,9 +21,7 @@ void Order::fill(Quantity qty) Timestamp Order::now() { - auto n = - std::chrono::time_point_cast(std::chrono::system_clock::now()); - return Timestamp{ n.time_since_epoch() }; + return tscNow(); } OrderBuilder& OrderBuilder::id(OrderId id) diff --git a/src/OrderBook.cpp b/src/OrderBook.cpp index 522177b..3c89107 100644 --- a/src/OrderBook.cpp +++ b/src/OrderBook.cpp @@ -1,97 +1,16 @@ #include "OrderBook.h" #include -#include -#include #include #include -#if defined(_MSC_VER) -#include -#endif - #include "Order.h" #include "Side.h" +#include "TscClock.h" #include "Types.h" namespace orderbook { -namespace -{ - -#if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__) - -std::uint64_t readTsc() noexcept -{ -#if defined(_MSC_VER) - return __rdtsc(); -#else - return __builtin_ia32_rdtsc(); -#endif -} - -struct TscCalibration -{ - std::uint64_t baseCycles; - double nsPerCycle; - std::chrono::nanoseconds baseEpoch; -}; - -const TscCalibration& tscCalibration() -{ - static const TscCalibration calibration = [] - { - const auto wallStart = - std::chrono::time_point_cast(std::chrono::system_clock::now()); - const std::uint64_t cyclesStart = readTsc(); - - auto wallNow = wallStart; - std::uint64_t cyclesNow = cyclesStart; - do - { - wallNow = std::chrono::time_point_cast( - std::chrono::system_clock::now()); - cyclesNow = readTsc(); - } while (wallNow - wallStart < std::chrono::milliseconds{ 1 }); - - const double elapsedNs = - static_cast((wallNow - wallStart).count()); - const double nsPerCycle = elapsedNs / static_cast(cyclesNow - cyclesStart); - - return TscCalibration{ cyclesStart, nsPerCycle, wallStart.time_since_epoch() }; - }(); - - return calibration; -} - -// Calibrated once per process (a ~1ms one-time cost, amortized to nothing); -// every call after that is a TSC read plus a multiply, not a clock_gettime -// syscall/vDSO round trip. Requires an invariant TSC (constant_tsc, -// nonstop_tsc) to be meaningful across cores, which every x86-64 CPU built -// in the last decade-plus has. -Timestamp tscNow() noexcept -{ - const TscCalibration& calibration = tscCalibration(); - const std::uint64_t elapsedCycles = readTsc() - calibration.baseCycles; - const auto elapsedNs = - static_cast(static_cast(elapsedCycles) * calibration.nsPerCycle); - - return Timestamp{ calibration.baseEpoch + std::chrono::nanoseconds{ elapsedNs } }; -} - -#else - -Timestamp tscNow() noexcept -{ - const auto now = - std::chrono::time_point_cast(std::chrono::system_clock::now()); - return Timestamp{ now.time_since_epoch() }; -} - -#endif - -} // namespace - void OrderBook::pushBack(PriceLevel& level, Order* order) { order->prev = level.tail; diff --git a/src/TscClock.h b/src/TscClock.h new file mode 100644 index 0000000..935b0c3 --- /dev/null +++ b/src/TscClock.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include + +#include "Types.h" + +#if defined(_MSC_VER) +#include +#endif + +namespace orderbook { + +namespace detail { + +#if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__) + +inline std::uint64_t readTsc() noexcept +{ +#if defined(_MSC_VER) + return __rdtsc(); +#else + return __builtin_ia32_rdtsc(); +#endif +} + +struct TscCalibration +{ + std::uint64_t baseCycles; + double nsPerCycle; + std::chrono::nanoseconds baseEpoch; +}; + +// inline so every translation unit that includes this header shares the +// same function-local static, and so the same one-time calibration, instead +// of each .cpp file calibrating its own clock against a different baseline. +inline const TscCalibration& tscCalibration() +{ + static const TscCalibration calibration = [] + { + const auto wallStart = + std::chrono::time_point_cast(std::chrono::system_clock::now()); + const std::uint64_t cyclesStart = readTsc(); + + auto wallNow = wallStart; + std::uint64_t cyclesNow = cyclesStart; + do + { + wallNow = std::chrono::time_point_cast( + std::chrono::system_clock::now()); + cyclesNow = readTsc(); + } while (wallNow - wallStart < std::chrono::milliseconds{ 1 }); + + const double elapsedNs = + static_cast((wallNow - wallStart).count()); + const double nsPerCycle = elapsedNs / static_cast(cyclesNow - cyclesStart); + + return TscCalibration{ cyclesStart, nsPerCycle, wallStart.time_since_epoch() }; + }(); + + return calibration; +} + +#endif + +} // namespace detail + +#if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__) + +// Calibrated once per process (a ~1ms one-time cost, amortized to nothing); +// every call after that is a TSC read plus a multiply, not a clock_gettime +// syscall/vDSO round trip. Requires an invariant TSC (constant_tsc, +// nonstop_tsc) to be meaningful across cores, which every x86-64 CPU built +// in the last decade-plus has. +inline Timestamp tscNow() noexcept +{ + const detail::TscCalibration& calibration = detail::tscCalibration(); + const std::uint64_t elapsedCycles = detail::readTsc() - calibration.baseCycles; + const auto elapsedNs = + static_cast(static_cast(elapsedCycles) * calibration.nsPerCycle); + + return Timestamp{ calibration.baseEpoch + std::chrono::nanoseconds{ elapsedNs } }; +} + +#else + +inline Timestamp tscNow() noexcept +{ + const auto now = + std::chrono::time_point_cast(std::chrono::system_clock::now()); + return Timestamp{ now.time_since_epoch() }; +} + +#endif + +} // namespace orderbook From 6850854b14e6a69aaffe952d67d38490c0eb66f0 Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sun, 9 Aug 2026 18:39:16 -0500 Subject: [PATCH 14/18] perf: fused OrderIndex find and erase in the cancel path into one hash and probe --- src/OrderBook.cpp | 5 +++-- src/OrderIndex.h | 52 +++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/OrderBook.cpp b/src/OrderBook.cpp index 3c89107..6e020c6 100644 --- a/src/OrderBook.cpp +++ b/src/OrderBook.cpp @@ -272,7 +272,8 @@ std::span OrderBook::addOrder(Order order) bool OrderBook::removeOrder(OrderId orderId) { - Order* order = orders_.find(orderId); + const OrderIndex::IndexLookup lookup = orders_.findIndex(orderId); + Order* order = lookup.order; if (order == nullptr) { return false; @@ -304,7 +305,7 @@ bool OrderBook::removeOrder(OrderId orderId) } } - orders_.erase(orderId); + orders_.eraseAt(lookup.index); orderSlab_.release(order); return true; diff --git a/src/OrderIndex.h b/src/OrderIndex.h index 2b52745..4469ccd 100644 --- a/src/OrderIndex.h +++ b/src/OrderIndex.h @@ -110,20 +110,39 @@ class OrderIndex ++size_; } - bool erase(OrderId id) noexcept + struct IndexLookup + { + std::size_t index; + Order* order; + }; + + // Same idea as findSlot: OrderBook's cancel path needs to find an order + // before it can remove it anyway, so hand back the index and let + // eraseAt reuse it instead of erase() re-deriving it from the id. + [[nodiscard]] IndexLookup findIndex(OrderId id) const noexcept { const std::size_t mask = slots_.size() - 1; - std::size_t hole = hash(id) & mask; + std::size_t index = hash(id) & mask; - while (slots_[hole].order != nullptr && slots_[hole].id != id) + while (slots_[index].order != nullptr) { - hole = (hole + 1) & mask; + if (slots_[index].id == id) + { + return { index, slots_[index].order }; + } + index = (index + 1) & mask; } - if (slots_[hole].order == nullptr) - { - return false; - } + return { index, nullptr }; + } + + // Only valid for an index just returned by findIndex, before any other + // insert or erase on this table. Unlike insertAt, there's no grow() to + // worry about here: erasing never resizes the table. + void eraseAt(std::size_t index) noexcept + { + const std::size_t mask = slots_.size() - 1; + std::size_t hole = index; slots_[hole].order = nullptr; --size_; @@ -149,7 +168,24 @@ class OrderIndex slots_[probe].order = nullptr; hole = probe; } + } + + bool erase(OrderId id) noexcept + { + const std::size_t mask = slots_.size() - 1; + std::size_t hole = hash(id) & mask; + + while (slots_[hole].order != nullptr && slots_[hole].id != id) + { + hole = (hole + 1) & mask; + } + + if (slots_[hole].order == nullptr) + { + return false; + } + eraseAt(hole); return true; } From a9e63d5434190b4d83d98dd150664f7db60cee72 Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sun, 9 Aug 2026 18:47:45 -0500 Subject: [PATCH 15/18] perf: fused canMatch and getBestOppositeLevel so matchOrder's loop looks up best() once per iteration --- src/OrderBook.cpp | 25 +++++++++++++++++++++---- src/OrderBook.h | 2 +- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/OrderBook.cpp b/src/OrderBook.cpp index 6e020c6..0bd39f1 100644 --- a/src/OrderBook.cpp +++ b/src/OrderBook.cpp @@ -132,9 +132,26 @@ bool OrderBook::canFullyFill(Side side, Price price, Quantity qty) const return sufficient; } -PriceLevel* OrderBook::getBestOppositeLevel(const Order* order) +// Does what canMatch does and what the old getBestOppositeLevel did, in one +// call. matchOrder's loop used to call both every iteration, which meant +// asks_.best()/bids_.best() got looked up twice for the same answer. +PriceLevel* OrderBook::matchableLevel(const Order* order) { - return order->getSide() == Side::Buy ? asks_.best() : bids_.best(); + const Price orderPrice = order->getPrice(); + + if (order->getOrderType() == OrderType::Market) + { + return order->getSide() == Side::Buy ? asks_.best() : bids_.best(); + } + + if (order->getSide() == Side::Buy) + { + PriceLevel* bestAsk = asks_.best(); + return (bestAsk != nullptr && orderPrice >= bestAsk->price) ? bestAsk : nullptr; + } + + PriceLevel* bestBid = bids_.best(); + return (bestBid != nullptr && orderPrice <= bestBid->price) ? bestBid : nullptr; } Trade OrderBook::createTrade(Order* incomingOrder, @@ -201,9 +218,9 @@ void OrderBook::matchOrder(Order* incoming) { const Timestamp arrivalTime = tscNow(); - while (!incoming->isFilled() && canMatch(incoming)) + while (!incoming->isFilled()) { - PriceLevel* level = getBestOppositeLevel(incoming); + PriceLevel* level = matchableLevel(incoming); if (level == nullptr || level->head == nullptr) { diff --git a/src/OrderBook.h b/src/OrderBook.h index 26d9167..55ded6f 100644 --- a/src/OrderBook.h +++ b/src/OrderBook.h @@ -47,7 +47,7 @@ class OrderBook [[nodiscard]] bool acceptsPrice(const Order& order) const noexcept; [[nodiscard]] bool canMatch(const Order* order) const; [[nodiscard]] bool canFullyFill(Side side, Price price, Quantity qty) const; - [[nodiscard]] PriceLevel* getBestOppositeLevel(const Order* order); + [[nodiscard]] PriceLevel* matchableLevel(const Order* order); Trade createTrade(Order* incoming, Order* resting, Quantity quantity, Timestamp timestamp); void cleanupAfterTrade(Order* incoming, Order* resting, PriceLevel& level, Quantity tradeQty); void matchOrder(Order* incomingOrder); From ac5621d399ba17c8b02129ebf409329c65847af6 Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sun, 9 Aug 2026 19:07:47 -0500 Subject: [PATCH 16/18] refactor: OrderIndex insert delegates to findSlot and insertAt instead of duplicating the probe loop --- src/OrderIndex.h | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/OrderIndex.h b/src/OrderIndex.h index 4469ccd..d7fee15 100644 --- a/src/OrderIndex.h +++ b/src/OrderIndex.h @@ -86,28 +86,20 @@ class OrderIndex ++size_; } + // Upsert, for OrderIndex's own general use (grow's rehashing loop below, + // and callers who don't already have a findSlot result to reuse). Built + // on findSlot/insertAt rather than its own probe loop, so there's one + // place that implements probing, not two copies to keep in sync. void insert(OrderId id, Order* order) { - if ((size_ + 1) * MaxLoadDenominator > slots_.size() * MaxLoadNumerator) + const SlotLookup slot = findSlot(id); + if (slot.occupied) { - grow(); + slots_[slot.index].order = order; + return; } - const std::size_t mask = slots_.size() - 1; - std::size_t index = hash(id) & mask; - - while (slots_[index].order != nullptr) - { - if (slots_[index].id == id) - { - slots_[index].order = order; - return; - } - index = (index + 1) & mask; - } - - slots_[index] = Slot{ .id = id, .order = order }; - ++size_; + insertAt(slot.index, id, order); } struct IndexLookup From 2664f4c5b7966ed913c79aad03dc53474d80385d Mon Sep 17 00:00:00 2001 From: Kieran Finger Date: Sun, 9 Aug 2026 19:07:51 -0500 Subject: [PATCH 17/18] refactor: moved list ops to PriceLevel, added side-dispatch helpers, dropped removeOrder indirection --- src/OrderBook.cpp | 92 +++++++++++------------------------------------ src/OrderBook.h | 14 +++++--- src/PriceLevel.h | 44 +++++++++++++++++++++++ 3 files changed, 73 insertions(+), 77 deletions(-) diff --git a/src/OrderBook.cpp b/src/OrderBook.cpp index 0bd39f1..1b06685 100644 --- a/src/OrderBook.cpp +++ b/src/OrderBook.cpp @@ -11,58 +11,33 @@ namespace orderbook { -void OrderBook::pushBack(PriceLevel& level, Order* order) +PriceLevel& OrderBook::getOrCreateLevel(Side side, Price price) { - order->prev = level.tail; - order->next = nullptr; - - if (level.tail != nullptr) - { - level.tail->next = order; - } - else - { - level.head = order; - } - - level.tail = order; - level.orderCount++; - level.totalQuantity += order->getRemainingQuantity(); + return side == Side::Buy ? bids_.getOrCreate(price) : asks_.getOrCreate(price); } -void OrderBook::unlink(PriceLevel& level, Order* order) +PriceLevel* OrderBook::findLevel(Side side, Price price) { - if (order->prev != nullptr) - { - order->prev->next = order->next; - } - else - { - level.head = order->next; - } + return side == Side::Buy ? bids_.find(price) : asks_.find(price); +} - if (order->next != nullptr) +void OrderBook::eraseLevel(Side side, Price price) +{ + if (side == Side::Buy) { - order->next->prev = order->prev; + bids_.erase(price); } else { - level.tail = order->prev; + asks_.erase(price); } - - order->prev = nullptr; - order->next = nullptr; - level.orderCount--; } void OrderBook::addToBook(Order* order, std::size_t indexHint) { - const Price price = order->getPrice(); - const Side side = order->getSide(); + PriceLevel& level = getOrCreateLevel(order->getSide(), order->getPrice()); - PriceLevel& level = (side == Side::Buy) ? bids_.getOrCreate(price) : asks_.getOrCreate(price); - - pushBack(level, order); + level.pushBack(order); orders_.insertAt(indexHint, order->getOrderId(), order); } @@ -185,32 +160,20 @@ Trade OrderBook::createTrade(Order* incomingOrder, return trade; } -void OrderBook::cleanupAfterTrade(Order* incoming, - Order* resting, - PriceLevel& level, - Quantity tradeQty) +void OrderBook::cleanupAfterTrade(Order* resting, PriceLevel& level, Quantity tradeQty) { - const Price restingPrice = resting->getPrice(); - level.totalQuantity -= tradeQty; if (resting->isFilled()) { orders_.erase(resting->getOrderId()); - unlink(level, resting); + level.unlink(resting); orderSlab_.release(resting); } if (level.orderCount == 0) { - if (incoming->getSide() == Side::Buy) - { - asks_.erase(restingPrice); - } - else - { - bids_.erase(restingPrice); - } + eraseLevel(resting->getSide(), resting->getPrice()); } } @@ -239,7 +202,7 @@ void OrderBook::matchOrder(Order* incoming) tradeBuffer_.push_back(std::move(trade)); - cleanupAfterTrade(incoming, resting, *level, tradeQuantity); + cleanupAfterTrade(resting, *level, tradeQuantity); } } @@ -287,7 +250,7 @@ std::span OrderBook::addOrder(Order order) return tradeBuffer_; } -bool OrderBook::removeOrder(OrderId orderId) +bool OrderBook::cancelOrder(OrderId orderId) { const OrderIndex::IndexLookup lookup = orders_.findIndex(orderId); Order* order = lookup.order; @@ -296,10 +259,7 @@ bool OrderBook::removeOrder(OrderId orderId) return false; } - const Side side = order->getSide(); - const Price price = order->getPrice(); - - PriceLevel* level = (side == Side::Buy) ? bids_.find(price) : asks_.find(price); + PriceLevel* level = findLevel(order->getSide(), order->getPrice()); if (level == nullptr) { @@ -308,18 +268,11 @@ bool OrderBook::removeOrder(OrderId orderId) level->totalQuantity -= order->getRemainingQuantity(); - unlink(*level, order); + level->unlink(order); if (level->orderCount == 0) { - if (side == Side::Buy) - { - bids_.erase(price); - } - else - { - asks_.erase(price); - } + eraseLevel(order->getSide(), order->getPrice()); } orders_.eraseAt(lookup.index); @@ -328,11 +281,6 @@ bool OrderBook::removeOrder(OrderId orderId) return true; } -bool OrderBook::cancelOrder(OrderId orderId) -{ - return removeOrder(orderId); -} - OrderBookLevelInfos OrderBook::getLevelInfos() const { OrderBookLevelInfos result; diff --git a/src/OrderBook.h b/src/OrderBook.h index 55ded6f..422b1d7 100644 --- a/src/OrderBook.h +++ b/src/OrderBook.h @@ -48,18 +48,22 @@ class OrderBook [[nodiscard]] bool canMatch(const Order* order) const; [[nodiscard]] bool canFullyFill(Side side, Price price, Quantity qty) const; [[nodiscard]] PriceLevel* matchableLevel(const Order* order); + + // side selects which of bids_/asks_ to hit. They're different template + // instantiations, not a common base, so this ternary can't be avoided + // by returning a shared reference; these just keep it in one place. + [[nodiscard]] PriceLevel& getOrCreateLevel(Side side, Price price); + [[nodiscard]] PriceLevel* findLevel(Side side, Price price); + void eraseLevel(Side side, Price price); + Trade createTrade(Order* incoming, Order* resting, Quantity quantity, Timestamp timestamp); - void cleanupAfterTrade(Order* incoming, Order* resting, PriceLevel& level, Quantity tradeQty); + void cleanupAfterTrade(Order* resting, PriceLevel& level, Quantity tradeQty); void matchOrder(Order* incomingOrder); // indexHint must come from the OrderIndex::SlotLookup that addOrder got // from findSlot for this same order. It lets insertAt skip re-hashing // and re-probing an id we already located once. void addToBook(Order* order, std::size_t indexHint); - bool removeOrder(OrderId orderId); - - static void pushBack(PriceLevel& level, Order* order); - static void unlink(PriceLevel& level, Order* order); }; } diff --git a/src/PriceLevel.h b/src/PriceLevel.h index 4b740e1..34cfe4f 100644 --- a/src/PriceLevel.h +++ b/src/PriceLevel.h @@ -14,6 +14,50 @@ struct PriceLevel Order* head{ nullptr }; Order* tail{ nullptr }; std::size_t orderCount{ 0 }; + + void pushBack(Order* order) + { + order->prev = tail; + order->next = nullptr; + + if (tail != nullptr) + { + tail->next = order; + } + else + { + head = order; + } + + tail = order; + orderCount++; + totalQuantity += order->getRemainingQuantity(); + } + + void unlink(Order* order) + { + if (order->prev != nullptr) + { + order->prev->next = order->next; + } + else + { + head = order->next; + } + + if (order->next != nullptr) + { + order->next->prev = order->prev; + } + else + { + tail = order->prev; + } + + order->prev = nullptr; + order->next = nullptr; + orderCount--; + } }; } From a48f21bc542030dbf7fd12a4e37b5fa74dd267a0 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 9 Aug 2026 20:42:14 -0500 Subject: [PATCH 18/18] v3: Performance fixes, Concurrent Queue tests, update docs --- .gitignore | 3 + CMakeLists.txt | 2 +- README.md | 93 +++++++-------- bench/depth_bench.cpp | 5 +- bench/ingest_pipeline_bench.cpp | 7 +- docs/BENCHMARKS.md | 197 ++++++++++++++++++------------- src/Order.h | 3 - src/OrderBook.cpp | 7 +- src/OrderBook.h | 9 +- src/OrderIndex.h | 35 +++--- src/OrderSlab.h | 4 +- src/TscClock.h | 15 ++- src/Version.h | 6 +- src/lockfree/ConcurrentQueue.h | 43 ++++--- tests/concurrent_queue_tests.cpp | 32 +++-- 15 files changed, 242 insertions(+), 219 deletions(-) diff --git a/.gitignore b/.gitignore index 19ba091..3c96c83 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,9 @@ compile_commands.json # VSCode PLAN.md +V3.md +v3-info.md +architecture-deep-dive.md .vscode/ # Visual Studio diff --git a/CMakeLists.txt b/CMakeLists.txt index 99c7495..579a0e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.15) project(LimitOrderBook - VERSION 2.3.0 + VERSION 3.0.0 LANGUAGES CXX ) diff --git a/README.md b/README.md index 9f31c21..75a9b9f 100644 --- a/README.md +++ b/README.md @@ -1,60 +1,61 @@ # liborderbook -A high-performance C++20 limit order book matching engine with sub-100ns add-order latency. - [![Build](https://github.com/K-Finger/liborderbook/actions/workflows/ci.yml/badge.svg)](https://github.com/K-Finger/liborderbook/actions) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) -## Features +A C++20 limit order book matching engine with sub-50ns operations on Linux. + +## Properties + +- Supports GTC, market, IOC, and FOK orders +- Ingests orders through a lock free queue when multiple threads submit at once +- Matches by price time priority +- Never allocates on the hot path +- Indexes price levels directly by price instead of using a tree -- **Sub-100ns latency** on the critical path -- **Order types**: GTC, Market, IOC, FOK -- **Price-time priority** matching (FIFO) -- **Strong types** for compile-time safety -- **Slab allocator** for cache locality -- **Zero dependencies** beyond STL +## What's New in v3.0.0 -## Quick Start +Adds a lock free queue for concurrent order ingest. Also cuts add and cancel latency further. The public API has not changed. Details and numbers are in [docs/BENCHMARKS.md](docs/BENCHMARKS.md). -### Build +## Build ```sh cmake -S . -B build cmake --build build --config Release ``` -### Integrate +## Integrate ```cmake add_subdirectory(liborderbook) target_link_libraries(your_app PRIVATE orderbook::orderbook) ``` -### Use +## Use ```cpp +#include + #include "OrderBook.h" +#include "BookPrinter.h" using namespace orderbook; OrderBook book; -// Place limit orders book.addOrder(OrderBuilder{}.id(OrderId{1}).buy().goodTillCancel() .price(Price{100}).quantity(Quantity{10}).build()); book.addOrder(OrderBuilder{}.id(OrderId{2}).sell().goodTillCancel() .price(Price{101}).quantity(Quantity{20}).build()); -// Market order crosses the spread +// Market order crosses the spread and returns the trades it produced. auto trades = book.addOrder(OrderBuilder{}.id(OrderId{3}).buy().market() .quantity(Quantity{5}).build()); book.cancelOrder(OrderId{1}); -book.printBook(); +printBook(std::cout, book); ``` -Output: - ``` === ASKS === 101 | 15 (1) @@ -64,8 +65,6 @@ Output: ## API -### OrderBuilder - ```cpp OrderBuilder{} .id(OrderId{42}) @@ -74,60 +73,54 @@ OrderBuilder{} .price(Price{100}) // omit for market orders .quantity(Quantity{10}) .build(); -``` - -### OrderBook -```cpp -std::vector addOrder(Order order); // Match + rest remainder -bool cancelOrder(OrderId id); // Remove resting order -std::size_t size() const; // Live order count -OrderBookLevelInfos getLevelInfos() const; // Aggregated depth -const std::vector& getTrades() const; // Trade history -void printBook(); // Debug output +std::span addOrder(Order order); // Match against the book, rest the remainder +bool cancelOrder(OrderId id); // Remove a resting order by id +std::size_t size() const; // Live resting order count +OrderBookLevelInfos getLevelInfos() const; // Aggregated depth, both sides ``` +`Order`, `OrderId`, `Price`, and `Quantity` are strong types. A price cannot be passed where a quantity is expected. + ## Benchmarks -Single thread, MSVC Release with IPO and AVX2: +Single core, Linux, GCC Release `-O3`: | Operation | Latency | |-----------|---------| -| Add, no match | 67 ns | -| Add, single match | 143 ns | -| Add into 10k levels | 58 ns | -| Cancel | 75 ns | +| Add, no match | 37 ns | +| Add, single match | 37.5 ns | +| Add into 10k price levels | 34 ns | +| Cancel | 20 ns | ```sh -./build/Release/orderbook_bench_all +./build/orderbook_bench_all ``` -See [BENCHMARKS.md](docs/BENCHMARKS.md) for history. +Full history in [docs/BENCHMARKS.md](docs/BENCHMARKS.md). ## Tests ```sh cmake --build build --target orderbook_tests -./build/Release/orderbook_tests +./build/orderbook_tests ``` +CI runs the suite on GCC and Clang, in Release and Debug, and under AddressSanitizer, UndefinedBehaviorSanitizer, and ThreadSanitizer. + ## Documentation -| Document | Description | -|----------|-------------| -| [Introduction](docs/introduction.md) | Architecture and design | -| [Installation](docs/installation.md) | Build and integration | -| [Guide](docs/guide.md) | Core concepts and operations | -| [Examples](docs/examples.md) | Usage patterns | -| [Support](docs/support.md) | FAQ, contributing, license | +- [User Guide](docs/user_guide.md) +- [Benchmarks](docs/BENCHMARKS.md) ## References -- [How to Build a Fast Limit Order Book](https://web.archive.org/web/20110219163448/http://howtohft.wordpress.com/2011/02/15/how-to-build-a-fast-limit-order-book/) -- [Limit Order Book in C++](https://alexabosi.wordpress.com/2014/08/28/limit-order-book-implementation-for-low-latency-trading-in-c/) — Abosi -- [brprojects/Limit-Order-Book](https://github.com/brprojects/Limit-Order-Book) -- [Strong types with CRTP](https://youtu.be/fWcnp7Bulc8) — Fluent C++ -- [TomaszRewak/cpp-allocator](https://github.com/TomaszRewak/cpp-allocator) — slab allocator +- [How to Build a Fast Limit Order Book](https://web.archive.org/web/20110219163448/http://howtohft.wordpress.com/2011/02/15/how-to-build-a-fast-limit-order-book/) - the array indexed price level technique this book uses +- [Bounded MPMC queue](https://web.archive.org/web/20160310160656/http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue) - Dmitry Vyukov, the design `ConcurrentQueue` is built on +- [Limit Order Book in C++](https://alexabosi.wordpress.com/2014/08/28/limit-order-book-implementation-for-low-latency-trading-in-c/) - Abosi +- [Bit Twiddling Hacks](https://graphics.stanford.edu/~seander/bithacks.html) - Sean Eron Anderson, Stanford. The bit scan technique behind the occupancy bitset +- [TomaszRewak/cpp-allocator](https://github.com/TomaszRewak/cpp-allocator) - slab allocator +- [Strong types with CRTP](https://youtu.be/fWcnp7Bulc8) - Fluent C++ ## License diff --git a/bench/depth_bench.cpp b/bench/depth_bench.cpp index 045c697..a0cf719 100644 --- a/bench/depth_bench.cpp +++ b/bench/depth_bench.cpp @@ -29,8 +29,9 @@ static void BM_Depth_AddAtBest(benchmark::State& state) { benchmark::DoNotOptimize(trades); } } -// Iterations capped. Refill above is O(depth), and auto-calibrated -// iteration counts in the millions turn that into minutes of wall time. +// Iterations are capped. The refill above is O(depth). Without this cap, +// auto calibration picks iteration counts in the millions, which turns +// that refill cost into minutes of wall time. BENCHMARK(BM_Depth_AddAtBest)->Arg(0)->Arg(100)->Arg(1000)->Arg(10000)->Iterations(20000); // Add orders at random prices across the price range. diff --git a/bench/ingest_pipeline_bench.cpp b/bench/ingest_pipeline_bench.cpp index 9130ea5..44dbd86 100644 --- a/bench/ingest_pipeline_bench.cpp +++ b/bench/ingest_pipeline_bench.cpp @@ -29,10 +29,6 @@ Order makeOrder(std::uint64_t id) } } // namespace -// N producer threads, thread 0 is the sole consumer draining into OrderBook. -// This is the actual topology v3 is built for: every atomic in the queue is -// only ever contended from one side, unlike a benchmark where every thread -// does both a push and a pop. static void BM_IngestPipeline_Mutex(benchmark::State& state) { static std::mutex* mutex = nullptr; @@ -91,7 +87,8 @@ static void BM_IngestPipeline_Mutex(benchmark::State& state) } BENCHMARK(BM_IngestPipeline_Mutex)->Threads(2)->Threads(4)->Threads(8)->Iterations(20000); -// Same topology, lock-free queue instead of a mutex. +// Same topology as above. This one uses a lock free queue instead of a +// mutex. static void BM_IngestPipeline_LockFree(benchmark::State& state) { static ConcurrentQueue* queue = nullptr; diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 008eb6e..753e8b4 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -9,71 +9,77 @@ Latency of core operations measured with [Google Benchmark](https://github.com/g - Linux, GCC Release -O3 - Single threaded -Each operation is timed in a steady-state book. A batch of pre-built orders is replayed; timing pauses to rebuild the book between batches so allocation and teardown are excluded. +Each benchmark runs on a book that already has orders in it. -## Current Results (v2.3) +## Current Results (v3.0.0) ``` -BM_AddOrder_NoMatch 43.3 ns -BM_AddOrder_SingleMatch 42.7 ns -BM_AddOrder_MultiMatch/2 68.0 ns -BM_AddOrder_MultiMatch/5 163 ns -BM_AddOrder_MultiMatch/10 326 ns +BM_AddOrder_NoMatch 37.0 ns +BM_AddOrder_SingleMatch 37.5 ns +BM_AddOrder_MultiMatch/2 55.3 ns +BM_AddOrder_MultiMatch/5 129 ns +BM_AddOrder_MultiMatch/10 241 ns + +BM_CancelOrder 20.2 ns +BM_CancelOrder_MidQueue/10 18.3 ns +BM_CancelOrder_MidQueue/100 6.67 ns +BM_CancelOrder_MidQueue/1000 13.5 ns + +BM_Depth_AddAtBest/0 20.3 ns +BM_Depth_AddAtBest/100 21.8 ns +BM_Depth_AddAtBest/1000 39.9 ns +BM_Depth_AddAtBest/10000 34.0 ns +BM_Depth_AddScatter/100 25.6 ns +BM_Depth_AddScatter/1000 46.0 ns +BM_Depth_AddScatter/10000 38.2 ns +BM_Depth_SweepLevels/2 31.1 ns +BM_Depth_SweepLevels/5 79.4 ns +BM_Depth_SweepLevels/10 209 ns +BM_Depth_SweepLevels/20 417 ns + +BM_GetLevelInfos/10 32.8 ns +BM_GetLevelInfos/100 352 ns +BM_GetLevelInfos/1000 3101 ns +BM_GetLevelInfos/10000 31416 ns +BM_GetLevelInfos_BothSides/10 57.7 ns +BM_GetLevelInfos_BothSides/100 657 ns +BM_GetLevelInfos_BothSides/1000 6116 ns +``` -BM_CancelOrder 21.0 ns -BM_CancelOrder_MidQueue/10 22.8 ns -BM_CancelOrder_MidQueue/100 6.57 ns -BM_CancelOrder_MidQueue/1000 14.4 ns +Every number dropped from v2.3. See "v3.0.0" under Optimization History for why. -BM_Depth_AddAtBest/0 26.1 ns -BM_Depth_AddAtBest/100 27.1 ns -BM_Depth_AddAtBest/1000 44.1 ns -BM_Depth_AddAtBest/10000 34.8 ns -BM_Depth_AddScatter/100 28.2 ns -BM_Depth_AddScatter/1000 46.9 ns -BM_Depth_AddScatter/10000 37.8 ns -BM_Depth_SweepLevels/2 43.5 ns -BM_Depth_SweepLevels/5 93.6 ns -BM_Depth_SweepLevels/10 205 ns -BM_Depth_SweepLevels/20 449 ns +### Lock free queue + +v3.0.0 adds `ConcurrentQueue`. It is a bounded lock free MPMC ring buffer for order ingest. Many producer threads push orders into it. One consumer thread drains it into the matching engine. The matching engine stays single threaded. This keeps it correct and fast. -BM_GetLevelInfos/10 29.8 ns -BM_GetLevelInfos/100 395 ns -BM_GetLevelInfos/1000 2977 ns -BM_GetLevelInfos/10000 34664 ns -BM_GetLevelInfos_BothSides/10 57.0 ns -BM_GetLevelInfos_BothSides/100 693 ns -BM_GetLevelInfos_BothSides/1000 5876 ns +``` + threads:2 threads:4 threads:8 +Mutex + queue 158 ns 386 ns 943 ns +ConcurrentQueue 42.3 ns 260 ns 945 ns ``` -The v2.3 array-indexed price ladder replaced the old `std::map`-backed price index, so the -benchmark suite was rewritten alongside it (`bench/add_order_bench.cpp`, -`cancel_order_bench.cpp`, `depth_bench.cpp`, `level_info_bench.cpp`) to isolate the new -access patterns — see "What Each Benchmark Measures" below. The old `AddOrder_PreBuilt` / -`AddOrder_AtDepth` names from the v1/v2/v2.1 history further down no longer exist; the -closest current equivalents are `AddOrder_NoMatch` and `Depth_AddAtBest`, but the numbers -aren't directly comparable since the harness and scenarios changed. +The single consumer becomes the bottleneck at high thread counts. ## What Each Benchmark Measures -- **AddOrder_NoMatch** - Resting insert, no match, single price level (best case for cache locality) -- **AddOrder_SingleMatch** - Add + immediate single match, full crossing-spread latency -- **AddOrder_MultiMatch/N** - Aggressive order sweeps N resting orders at the same price level -- **CancelOrder** - Locate by id and unlink, book has one order per price level -- **CancelOrder_MidQueue/N** - Cancel from the middle of an N-deep price-level queue (intrusive list unlink, not head/tail) -- **Depth_AddAtBest/N** - Insert against a book holding N price levels, all adds at the best price (isolates level-lookup cost as the ladder fills) -- **Depth_AddScatter/N** - Insert at random prices across an N-level book (worst-case index lookup, no cache locality between adds) -- **Depth_SweepLevels/N** - Aggressive order matches across N price levels (multi-level book walk) -- **GetLevelInfos/N** - Snapshot one side of a book holding N price levels -- **GetLevelInfos_BothSides/N** - Snapshot both sides of a book holding N price levels each +- **AddOrder_NoMatch**: Adds one order that rests with no match. This is the best case for cache locality. +- **AddOrder_SingleMatch**: Adds one order that matches and fills in one trade. This is the full cost of crossing the spread. +- **AddOrder_MultiMatch/N**: An aggressive order sweeps N resting orders at the same price level. +- **CancelOrder**: Finds an order by id and unlinks it. The book has one order per price level. +- **CancelOrder_MidQueue/N**: Cancels an order from the middle of a price level queue that is N orders deep. This tests unlinking from the middle of the list, not the head or tail. +- **Depth_AddAtBest/N**: Adds orders to a book that already holds N price levels. All adds go to the best price. This isolates the cost of finding a level as the ladder fills up. +- **Depth_AddScatter/N**: Adds orders at random prices across a book holding N price levels. This is the worst case for locating a level, since there is no cache locality between adds. +- **Depth_SweepLevels/N**: An aggressive order matches across N price levels. This walks the book across multiple levels. +- **GetLevelInfos/N**: Takes a snapshot of one side of a book holding N price levels. +- **GetLevelInfos_BothSides/N**: Takes a snapshot of both sides of a book, each holding N price levels. ## Optimization History -### v1 - Baseline +### v1: Baseline `std::map` per side, `std::list` per level, `std::shared_ptr` ownership. -Each add allocates a list node. Each level walk costs two cache hops. Ownership via shared_ptr adds a control block and atomic refcount per order. +Each add allocates a list node. Each level walk costs two cache hops. Ownership through shared_ptr adds a control block and an atomic refcount to every order. ``` AddOrder_PreBuilt 170 ns @@ -83,11 +89,11 @@ AddOrder_AtDepth/10000 121 ns CancelOrder 115 ns ``` -### v2 - Intrusive Linked List +### v2: Intrusive Linked List -Order carries its own prev/next. PriceLevel holds head/tail. No per-add list-node allocation. shared_ptr swapped for unique_ptr. +Order carries its own prev and next pointers. PriceLevel holds a head and a tail. Adding an order no longer allocates a list node. shared_ptr was replaced with unique_ptr. -~65 ns drop on resting path. +Resting an order got about 65 ns faster. ``` AddOrder_PreBuilt 91.7 ns @@ -97,11 +103,11 @@ AddOrder_AtDepth/10000 84.1 ns CancelOrder 94.0 ns ``` -### v2.1 - Slab Allocator +### v2.1: Slab Allocator -Orders live in a preallocated slab with LIFO free list. Allocation is a free-list pop or high-water bump. Just-freed slots are hot in cache on reuse. +Orders live in a slab that is allocated once, with a free list for reuse. Allocating an order pops from the free list or bumps the high water mark forward. A slot that was just freed is still hot in cache when it gets reused. -~27% improvement on resting path. +Resting an order got about 27% faster. ``` AddOrder_PreBuilt 67.0 ns @@ -111,46 +117,69 @@ AddOrder_AtDepth/10000 58.3 ns CancelOrder 75.3 ns ``` -### v2.2 - Reusable Trade Buffer - -`addOrder` returned a freshly allocated `std::vector` on every call, and the engine -kept an unbounded internal trade log growing for the lifetime of the book. Both removed: -trades are now written into a `std::span` over a single reusable buffer cleared at the start -of each `addOrder` call, and the unbounded log is gone. +### v2.2: Reusable Trade Buffer -### v2.3 - Array-Indexed Price Ladder +`addOrder` used to allocate a new `std::vector` on every call. The engine also kept an internal trade log that grew for the whole life of the book. Both are gone now. Trades are written into a `std::span` over one reusable buffer, and that buffer is cleared at the start of each `addOrder` call. -The `std::map` price index was the remaining bottleneck. Every add, cancel and level-drain -paid a red-black tree descent: `try_emplace` on insert, `at` on cancel, `erase` when a level -emptied. Each descent was a chain of dependent pointer loads into scattered heap nodes, so the -cost was cache misses rather than comparisons, and it grew with book depth. +### v2.3: Array Indexed Price Ladder +The `std::map` price index was the remaining bottleneck. Every add, cancel, and level drain walked down a red black tree: a `try_emplace` on insert, an `at` on cancel, an `erase` when a level emptied. Each walk was a chain of pointer loads into scattered heap nodes. The replacement is a dense ladder indexed directly by price. -**Layout.** One `PriceLevel` array per side, sized to a fixed tick band chosen at construction, -where `index = price - minPrice`. Locating a level becomes an add and a load. There is no tree, -no per-level node allocation, and no rebalancing. +**Layout.** Each side gets one `PriceLevel` array. Its size is a fixed tick band chosen at construction. A level's index is `price minus minPrice`. Finding a level is just an add and a load. There is no tree, no node allocation for each level, and no rebalancing. + +**Occupancy bitset.** A dense array is mostly empty, so a linear scan cannot answer "which level is best now". Occupancy is mirrored into a bitset, with 64 levels per word. The bitset is scanned with `std::countr_zero` and `std::countl_zero`. When the best level drains, finding the next one usually takes a single word test, not a walk. + +**Tracked extent.** The ladder remembers its best and worst occupied index. Traversals like `getLevelInfos` and the FOK liquidity check only scan the populated region, not the whole band. A book with 10 levels never scans an array of 110,000 levels. + +**Side as a template parameter.** `PriceLadder` treats the highest occupied price as best. `PriceLadder` treats the lowest occupied price as best. This is resolved at compile time, so it removes a branch from every check for the best level, and it lets the matching code treat both sides the same way. + +Alongside the ladder, the order index moved from `std::unordered_map` to an open addressing table. Resting orders are indexed by pointer alone, so the hot path no longer needs an id lookup. + +``` +BM_AddOrder_NoMatch 43.3 ns +BM_AddOrder_SingleMatch 42.7 ns +BM_AddOrder_MultiMatch/2 68.0 ns +BM_AddOrder_MultiMatch/5 163 ns +BM_AddOrder_MultiMatch/10 326 ns + +BM_CancelOrder 21.0 ns +BM_CancelOrder_MidQueue/10 22.8 ns +BM_CancelOrder_MidQueue/100 6.57 ns +BM_CancelOrder_MidQueue/1000 14.4 ns + +BM_Depth_AddAtBest/0 26.1 ns +BM_Depth_AddAtBest/100 27.1 ns +BM_Depth_AddAtBest/1000 44.1 ns +BM_Depth_AddAtBest/10000 34.8 ns +BM_Depth_AddScatter/100 28.2 ns +BM_Depth_AddScatter/1000 46.9 ns +BM_Depth_AddScatter/10000 37.8 ns +BM_Depth_SweepLevels/2 43.5 ns +BM_Depth_SweepLevels/5 93.6 ns +BM_Depth_SweepLevels/10 205 ns +BM_Depth_SweepLevels/20 449 ns + +BM_GetLevelInfos/10 29.8 ns +BM_GetLevelInfos/100 395 ns +BM_GetLevelInfos/1000 2977 ns +BM_GetLevelInfos/10000 34664 ns +BM_GetLevelInfos_BothSides/10 57.0 ns +BM_GetLevelInfos_BothSides/100 693 ns +BM_GetLevelInfos_BothSides/1000 5876 ns +``` + +The benchmark suite was rewritten alongside this change (`bench/add_order_bench.cpp`, `cancel_order_bench.cpp`, `depth_bench.cpp`, `level_info_bench.cpp`) to isolate the new access patterns. See "What Each Benchmark Measures" above. The old `AddOrder_PreBuilt` and `AddOrder_AtDepth` names from v1, v2, and v2.1 no longer exist. The closest equivalents are `AddOrder_NoMatch` and `Depth_AddAtBest`, but the numbers are not directly comparable, since the harness and scenarios changed. -**Occupancy bitset.** A dense array is mostly empty, so "which level is best now?" cannot be a -linear scan. Occupancy is mirrored into a bitset, 64 levels per word, scanned with -`std::countr_zero` / `std::countl_zero`. When the best level drains, finding the next populated -one is typically a single word test rather than a walk. +### v3.0.0 performance changes: -**Tracked extent.** The ladder remembers its best and worst occupied index. Traversals -(`getLevelInfos`, the FOK liquidity check) are bounded to the populated region instead of the -whole band, so a 10-level book never scans a 110k-level array. +`matchOrder` timestamped every trade with `system_clock::now()`, which is a syscall on most platforms. It now reads the CPU's own cycle counter and converts that into a timestamp with one multiply. This clock is calibrated once per process at startup. -**Side as a template parameter.** `PriceLadder` treats the highest occupied price as -best, `PriceLadder` the lowest. Resolving this at compile time removes a branch from -every best-level access and lets the matching code treat both sides symmetrically. +`Order` was 72 bytes. It spilled across two cache lines because two 1 byte fields sat on either side of an 8 byte field. Reordering the fields brought it down to exactly 64 bytes, which is one cache line. As a side effect, the order slab shrank by 8MB at the default capacity of 1 million orders. -**Trade-off.** The band is bounded, so memory is `O(band)` rather than `O(levels)` and prices -outside the band are rejected — the same price-collar constraint real venues enforce. The -default band is deliberately wide enough to cover ordinary use and still fits in a few MB. +The order index hashed and probed an order's id twice on the common paths. `addOrder` checked that the id was not a duplicate. Then `addToBook` hashed and probed the same id again to insert it. The cancel path had the same problem between finding an order and erasing it. Both paths now do the lookup once and reuse the result. -Alongside the ladder, the order index moved from `std::unordered_map` to an -open-addressing table, and resting orders are indexed by pointer alone rather than id lookup -on the hot path — see "Current Results (v2.3)" above for the combined effect. +The code copied a `Trade` into the trade buffer where a move works. ## Running Benchmarks diff --git a/src/Order.h b/src/Order.h index 495419b..0319f75 100644 --- a/src/Order.h +++ b/src/Order.h @@ -52,9 +52,6 @@ class Order static Timestamp now(); private: - // Order is one cache line (64 bytes) with this field order. The two - // 1-byte enums are kept together so neither pads out to an 8-byte - // boundary on its own. OrderId orderId_; Price price_; Quantity initialQuantity_; diff --git a/src/OrderBook.cpp b/src/OrderBook.cpp index 1b06685..84253a3 100644 --- a/src/OrderBook.cpp +++ b/src/OrderBook.cpp @@ -107,9 +107,10 @@ bool OrderBook::canFullyFill(Side side, Price price, Quantity qty) const return sufficient; } -// Does what canMatch does and what the old getBestOppositeLevel did, in one -// call. matchOrder's loop used to call both every iteration, which meant -// asks_.best()/bids_.best() got looked up twice for the same answer. +// This does what canMatch does and what the old getBestOppositeLevel did, +// in one call. matchOrder's loop used to call both on every iteration. +// That meant asks_.best() or bids_.best() got looked up twice for the same +// answer. PriceLevel* OrderBook::matchableLevel(const Order* order) { const Price orderPrice = order->getPrice(); diff --git a/src/OrderBook.h b/src/OrderBook.h index 422b1d7..e18ff1e 100644 --- a/src/OrderBook.h +++ b/src/OrderBook.h @@ -49,9 +49,6 @@ class OrderBook [[nodiscard]] bool canFullyFill(Side side, Price price, Quantity qty) const; [[nodiscard]] PriceLevel* matchableLevel(const Order* order); - // side selects which of bids_/asks_ to hit. They're different template - // instantiations, not a common base, so this ternary can't be avoided - // by returning a shared reference; these just keep it in one place. [[nodiscard]] PriceLevel& getOrCreateLevel(Side side, Price price); [[nodiscard]] PriceLevel* findLevel(Side side, Price price); void eraseLevel(Side side, Price price); @@ -60,9 +57,9 @@ class OrderBook void cleanupAfterTrade(Order* resting, PriceLevel& level, Quantity tradeQty); void matchOrder(Order* incomingOrder); - // indexHint must come from the OrderIndex::SlotLookup that addOrder got - // from findSlot for this same order. It lets insertAt skip re-hashing - // and re-probing an id we already located once. + // indexHint must come from the OrderIndex::SlotLookup that addOrder + // already got from findSlot for this same order. It lets insertAt skip + // hashing and probing an id that was already found once. void addToBook(Order* order, std::size_t indexHint); }; diff --git a/src/OrderIndex.h b/src/OrderIndex.h index d7fee15..8a8d768 100644 --- a/src/OrderIndex.h +++ b/src/OrderIndex.h @@ -52,9 +52,9 @@ class OrderIndex bool occupied; }; - // OrderBook calls this once to both check for a duplicate id and find - // where a new one would go, then reuses the index with insertAt below - // instead of hashing and probing the same id a second time. + // OrderBook calls this once to check for a duplicate id and find where + // a new one would go. It then reuses the index with insertAt below. + // This avoids hashing and probing the same id twice. [[nodiscard]] SlotLookup findSlot(OrderId id) noexcept { if ((size_ + 1) * MaxLoadDenominator > slots_.size() * MaxLoadNumerator) @@ -77,19 +77,21 @@ class OrderIndex return { index, false }; } - // Only valid for an index just returned by findSlot, before any other - // insert on this table. grow() would move everything and invalidate it, - // but findSlot already grows up front so that can't happen in between. + // Only use an index that findSlot just returned, before any other + // insert on this table. grow() would move everything and make the + // index invalid, but findSlot already grows the table first, so that + // cannot happen in between. void insertAt(std::size_t index, OrderId id, Order* order) noexcept { slots_[index] = Slot{ .id = id, .order = order }; ++size_; } - // Upsert, for OrderIndex's own general use (grow's rehashing loop below, - // and callers who don't already have a findSlot result to reuse). Built - // on findSlot/insertAt rather than its own probe loop, so there's one - // place that implements probing, not two copies to keep in sync. + // This is an upsert for OrderIndex's own general use. grow uses it to + // rehash below, and any caller without a findSlot result to reuse can + // call it too. It is built on findSlot and insertAt instead of its own + // probe loop, so there is one place that does probing, not two copies + // to keep in sync. void insert(OrderId id, Order* order) { const SlotLookup slot = findSlot(id); @@ -108,9 +110,10 @@ class OrderIndex Order* order; }; - // Same idea as findSlot: OrderBook's cancel path needs to find an order - // before it can remove it anyway, so hand back the index and let - // eraseAt reuse it instead of erase() re-deriving it from the id. + // Same idea as findSlot. OrderBook's cancel path already needs to find + // an order before it can remove it. So this hands back the index too, + // and eraseAt reuses it instead of erase() looking it up again from + // the id. [[nodiscard]] IndexLookup findIndex(OrderId id) const noexcept { const std::size_t mask = slots_.size() - 1; @@ -128,9 +131,9 @@ class OrderIndex return { index, nullptr }; } - // Only valid for an index just returned by findIndex, before any other - // insert or erase on this table. Unlike insertAt, there's no grow() to - // worry about here: erasing never resizes the table. + // Only use an index that findIndex just returned, before any other + // insert or erase on this table. Unlike insertAt, there is no grow() + // to worry about here. Erasing never resizes the table. void eraseAt(std::size_t index) noexcept { const std::size_t mask = slots_.size() - 1; diff --git a/src/OrderSlab.h b/src/OrderSlab.h index f669200..342762e 100644 --- a/src/OrderSlab.h +++ b/src/OrderSlab.h @@ -49,8 +49,8 @@ class OrderSlab ~OrderSlab() = default; private: - // alignas(64) so each slot starts on its own cache line. Order is sized - // to exactly fill one, so this stops adjacent orders from splitting one. + // alignas(64) puts each slot on its own cache line. Order is exactly + // one cache line in size, so two orders never share a line. union alignas(64) Slot { Order order; diff --git a/src/TscClock.h b/src/TscClock.h index 935b0c3..3a9728a 100644 --- a/src/TscClock.h +++ b/src/TscClock.h @@ -31,9 +31,9 @@ struct TscCalibration std::chrono::nanoseconds baseEpoch; }; -// inline so every translation unit that includes this header shares the -// same function-local static, and so the same one-time calibration, instead -// of each .cpp file calibrating its own clock against a different baseline. +// This is inline so every file that includes this header shares the same +// static variable. That means one calibration for the whole program, not a +// different one in every file. inline const TscCalibration& tscCalibration() { static const TscCalibration calibration = [] @@ -67,11 +67,10 @@ inline const TscCalibration& tscCalibration() #if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__) -// Calibrated once per process (a ~1ms one-time cost, amortized to nothing); -// every call after that is a TSC read plus a multiply, not a clock_gettime -// syscall/vDSO round trip. Requires an invariant TSC (constant_tsc, -// nonstop_tsc) to be meaningful across cores, which every x86-64 CPU built -// in the last decade-plus has. +// Calibration runs once per process and costs about 1ms. After that, every +// call is just a TSC read and a multiply, not a clock_gettime syscall. This +// needs an invariant TSC (constant_tsc, nonstop_tsc) to stay correct across +// cores. Every x86-64 CPU built in the last ten years or so has one. inline Timestamp tscNow() noexcept { const detail::TscCalibration& calibration = detail::tscCalibration(); diff --git a/src/Version.h b/src/Version.h index ef851ce..1645b32 100644 --- a/src/Version.h +++ b/src/Version.h @@ -1,10 +1,10 @@ #pragma once -#define ORDERBOOK_VERSION_MAJOR 2 -#define ORDERBOOK_VERSION_MINOR 3 +#define ORDERBOOK_VERSION_MAJOR 3 +#define ORDERBOOK_VERSION_MINOR 0 #define ORDERBOOK_VERSION_PATCH 0 -#define ORDERBOOK_VERSION_STRING "2.3.0" +#define ORDERBOOK_VERSION_STRING "3.0.0" #define ORDERBOOK_VERSION \ (ORDERBOOK_VERSION_MAJOR * 10000 + ORDERBOOK_VERSION_MINOR * 100 + ORDERBOOK_VERSION_PATCH) diff --git a/src/lockfree/ConcurrentQueue.h b/src/lockfree/ConcurrentQueue.h index 362f014..fdc0751 100644 --- a/src/lockfree/ConcurrentQueue.h +++ b/src/lockfree/ConcurrentQueue.h @@ -10,13 +10,14 @@ namespace orderbook { -// Bounded MPMC ring buffer, Vyukov's per-cell sequence number design. -// Each cell's own sequence number is what makes it safe, not a lock. +// A bounded MPMC ring buffer. It uses Vyukov's design of a sequence number +// per cell. Each cell's own sequence number is what makes it safe, not a lock. template class ConcurrentQueue { - // Power of two so `pos & Mask` is a cheap wraparound. Capacity of 1 - // breaks the lap math, since "just filled" and "next lap" collide. + // Capacity must be a power of two so `pos & Mask` is a cheap wraparound. + // A capacity of 1 breaks the lap math, because "just filled" and "next + // lap" end up as the same number. static_assert(Capacity >= 2 && (Capacity & (Capacity - 1)) == 0, "ConcurrentQueue capacity must be a power of two >= 2"); @@ -32,7 +33,7 @@ class ConcurrentQueue ~ConcurrentQueue() { // tryPop needs a T& to move into, and T may have no default - // constructor, so we destroy what's left directly instead. + // constructor. So we destroy what is left directly instead. std::size_t head = dequeuePos_.load(std::memory_order_relaxed); const std::size_t tail = enqueuePos_.load(std::memory_order_relaxed); for (; head != tail; ++head) @@ -46,8 +47,9 @@ class ConcurrentQueue ConcurrentQueue(ConcurrentQueue&&) = delete; ConcurrentQueue& operator=(ConcurrentQueue&&) = delete; - // T&& instead of T by value: a failed push should leave the caller's - // object alone, and by value would move-construct it regardless. + // This takes T&& instead of T by value. A failed push should leave the + // caller's object alone, but passing by value would move construct it + // regardless. [[nodiscard]] bool tryPush(T&& value) { std::size_t pos = enqueuePos_.load(std::memory_order_relaxed); @@ -62,9 +64,10 @@ class ConcurrentQueue if (diff == 0) { - // enqueuePos_ only tracks whose turn is next, not data, - // so relaxed is fine here. The acquire/release pair on - // sequence is what actually protects the storage. + // enqueuePos_ only tracks whose turn is next. It does not + // track data, so relaxed is fine here. The acquire and + // release pair on sequence is what actually protects the + // storage. if (enqueuePos_.compare_exchange_weak(pos, pos + 1, std::memory_order_relaxed)) { new (&cell.storage.value) T(std::move(value)); @@ -94,8 +97,8 @@ class ConcurrentQueue Cell& cell = cells_[pos & Mask]; const std::size_t seq = cell.sequence.load(std::memory_order_acquire); - // A slot is only poppable once a producer fills it, one lap - // ahead of empty. That's the + 1 tryPush doesn't have. + // A slot can only be popped after a producer fills it, one lap + // ahead of empty. That is the + 1 that tryPush does not have. const auto diff = static_cast(seq) - static_cast(pos + 1); if (diff == 0) @@ -105,8 +108,9 @@ class ConcurrentQueue out = std::move(cell.storage.value); cell.storage.value.~T(); - // Republish as pos + Capacity, not pos + 1. That's the - // number this same slot needs one lap from now. + // This republishes the slot as pos + Capacity, not + // pos + 1. That is the number this same slot will need + // one lap from now. cell.sequence.store(pos + Capacity, std::memory_order_release); return true; } @@ -130,8 +134,8 @@ class ConcurrentQueue { std::atomic sequence; - // We use a union instead of a plain T member because Order has no - // default constructor and can't be copied. OrderSlab's Slot union + // This is a union instead of a plain T member because Order has no + // default constructor and cannot be copied. OrderSlab's Slot union // solves the same problem the same way. union Storage { @@ -146,9 +150,10 @@ class ConcurrentQueue } storage; }; - // alignas(64) prevents enqueuePos_ and dequeuePos_ from being on the - // same cache line. If they were, producer and consumer threads would - // thrash it every push and pop, even though the counters are unrelated. + // alignas(64) keeps enqueuePos_ and dequeuePos_ off the same cache line. + // If they shared one, producer and consumer threads would fight over + // that line on every push and pop, even though the two counters have + // nothing to do with each other. alignas(64) std::atomic enqueuePos_{0}; alignas(64) std::atomic dequeuePos_{0}; std::array cells_{}; diff --git a/tests/concurrent_queue_tests.cpp b/tests/concurrent_queue_tests.cpp index 3c75eb9..b061605 100644 --- a/tests/concurrent_queue_tests.cpp +++ b/tests/concurrent_queue_tests.cpp @@ -14,8 +14,6 @@ using namespace orderbook; namespace { -// Mirrors Order: no default constructor, move-only. Counts its own -// destructor calls so the drain tests can check nothing leaked. struct Tracked { int value; @@ -25,9 +23,10 @@ struct Tracked Tracked(const Tracked&) = delete; Tracked& operator=(const Tracked&) = delete; - // A defaulted move would copy dtorCount without clearing the source, - // so a moved-from husk would double count when it's later destroyed. - // We null it out here so only the current owner ever counts. + // A defaulted move would copy dtorCount without clearing the source. + // The object it moved from would then double count when it gets + // destroyed later. This nulls dtorCount out instead, so only the + // current owner ever counts. Tracked(Tracked&& other) noexcept : value(other.value), dtorCount(other.dtorCount) { other.dtorCount = nullptr; @@ -101,8 +100,9 @@ TEST(ConcurrentQueue, PreservesFifoOrder) TEST(ConcurrentQueue, FailedPushLeavesTheCallersValueUntouched) { - // unique_ptr proves tryPush only moves from its argument on success. - // A wrong implementation would null ptr out even though this fails. + // unique_ptr proves that tryPush only moves from its argument on + // success. A wrong implementation would null out ptr even though + // this push fails. ConcurrentQueue, 2> queue; EXPECT_TRUE(queue.tryPush(std::make_unique(1))); @@ -116,8 +116,6 @@ TEST(ConcurrentQueue, FailedPushLeavesTheCallersValueUntouched) TEST(ConcurrentQueue, WorksWithOrderItsActualMotivatingType) { - // Order has no default constructor and can't be copied. This is the - // exact constraint the queue was designed around. ConcurrentQueue queue; Order order = OrderBuilder{} @@ -153,8 +151,9 @@ TEST(ConcurrentQueue, DestructorDrainsWhateverIsStillInTheQueue) EXPECT_TRUE(queue.tryPush(Tracked{ 2, &dtorCount })); EXPECT_TRUE(queue.tryPush(Tracked{ 3, &dtorCount })); - // Only pop one. The other two are still live when queue is - // destroyed below, so the destructor's drain has to clean them up. + // This only pops one. The other two are still live when the queue + // is destroyed below, so the destructor's drain has to clean them + // up. Tracked out{ 0, nullptr }; EXPECT_TRUE(queue.tryPop(out)); } @@ -166,7 +165,7 @@ TEST(ConcurrentQueue, WrapsAroundCorrectlyAcrossManyLaps) { ConcurrentQueue queue; - // An off-by-one-lap bug only shows up once a slot gets reused, not on + // A bug in the lap count only shows up once a slot gets reused, not on // the first pass. 50 laps forces every slot through many reuses. for (int lap = 0; lap < 50; ++lap) { @@ -198,7 +197,7 @@ TEST(ConcurrentQueue, HandlesFillDrainPatternsNotAlignedToCapacity) ASSERT_TRUE(queue.tryPop(out)); EXPECT_EQ(out, 1); - // These two land on an offset that isn't a multiple of capacity. + // These two land on an offset that is not a multiple of capacity. ASSERT_TRUE(queue.tryPush(4)); ASSERT_TRUE(queue.tryPush(5)); @@ -231,13 +230,12 @@ TEST(ConcurrentQueue, ManyProducersOneConsumerSeeEveryValueExactlyOnce) { for (int i = 0; i < PerProducer; ++i) { - // Each producer owns a disjoint range, so one unique - // value proves it was neither dropped nor duplicated. + // Each producer owns a disjoint range of values. So + // seeing one unique value proves it was neither dropped + // nor duplicated. const int value = p * PerProducer + i; while (!queue.tryPush(int{ value })) { - // The queue is bounded, so a full ring under real - // contention is expected. Just retry. } } });