diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ec551a..e87649d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,34 @@ jobs: UBSAN_OPTIONS: print_stacktrace=1,halt_on_error=1 run: ctest --test-dir build --output-on-failure + linux-tsan: + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v4 + + - name: Install toolchain + run: | + sudo apt-get update -y + sudo apt-get install -y clang + + - name: Configure CMake + env: + CC: clang + CXX: clang++ + run: > + cmake -B build -DCMAKE_BUILD_TYPE=Debug + -DORDERBOOK_WARNINGS_AS_ERRORS=ON + -DORDERBOOK_SANITIZERS=thread + + - name: Build + run: cmake --build build -j + + - name: Run tests under TSan + env: + TSAN_OPTIONS: halt_on_error=1 + run: ctest --test-dir build --output-on-failure + clang-tidy: runs-on: ubuntu-24.04 diff --git a/.gitignore b/.gitignore index 19ba091..3c96c83 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,9 @@ compile_commands.json # VSCode PLAN.md +V3.md +v3-info.md +architecture-deep-dive.md .vscode/ # Visual Studio diff --git a/CMakeLists.txt b/CMakeLists.txt index 00e7623..579a0e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.15) project(LimitOrderBook - VERSION 2.3.0 + VERSION 3.0.0 LANGUAGES CXX ) @@ -95,6 +95,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) include(FetchContent) enable_testing() + find_package(Threads REQUIRED) + FetchContent_Declare( googletest GIT_REPOSITORY https://github.com/google/googletest.git @@ -122,10 +124,11 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) tests/order_pool_tests.cpp tests/book_printer_tests.cpp tests/order_index_tests.cpp + tests/concurrent_queue_tests.cpp ) add_executable(orderbook_tests ${TEST_SOURCES}) - target_link_libraries(orderbook_tests PRIVATE orderbook GTest::gtest_main) + target_link_libraries(orderbook_tests PRIVATE orderbook GTest::gtest_main Threads::Threads) orderbook_apply_hardening(orderbook_tests) if(ipo_ok) set_target_properties(orderbook_tests PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ON) @@ -138,12 +141,13 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) bench/cancel_order_bench.cpp bench/depth_bench.cpp bench/level_info_bench.cpp + bench/ingest_pipeline_bench.cpp ) foreach(BENCH_SRC ${BENCH_SOURCES}) get_filename_component(BENCH_NAME ${BENCH_SRC} NAME_WE) add_executable(${BENCH_NAME} ${BENCH_SRC}) - target_link_libraries(${BENCH_NAME} PRIVATE orderbook benchmark::benchmark) + target_link_libraries(${BENCH_NAME} PRIVATE orderbook benchmark::benchmark Threads::Threads) target_include_directories(${BENCH_NAME} PRIVATE bench) orderbook_apply_hardening(${BENCH_NAME}) if(ipo_ok) @@ -161,8 +165,9 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) bench/cancel_order_bench.cpp bench/depth_bench.cpp bench/level_info_bench.cpp + bench/ingest_pipeline_bench.cpp ) - target_link_libraries(orderbook_bench_all PRIVATE orderbook benchmark::benchmark_main) + target_link_libraries(orderbook_bench_all PRIVATE orderbook benchmark::benchmark_main Threads::Threads) target_include_directories(orderbook_bench_all PRIVATE bench) target_compile_definitions(orderbook_bench_all PRIVATE BENCHMARK_COMBINED) orderbook_apply_hardening(orderbook_bench_all) diff --git a/README.md b/README.md index 9f31c21..75a9b9f 100644 --- a/README.md +++ b/README.md @@ -1,60 +1,61 @@ # liborderbook -A high-performance C++20 limit order book matching engine with sub-100ns add-order latency. - [![Build](https://github.com/K-Finger/liborderbook/actions/workflows/ci.yml/badge.svg)](https://github.com/K-Finger/liborderbook/actions) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) -## Features +A C++20 limit order book matching engine with sub-50ns operations on Linux. + +## Properties + +- Supports GTC, market, IOC, and FOK orders +- Ingests orders through a lock free queue when multiple threads submit at once +- Matches by price time priority +- Never allocates on the hot path +- Indexes price levels directly by price instead of using a tree -- **Sub-100ns latency** on the critical path -- **Order types**: GTC, Market, IOC, FOK -- **Price-time priority** matching (FIFO) -- **Strong types** for compile-time safety -- **Slab allocator** for cache locality -- **Zero dependencies** beyond STL +## What's New in v3.0.0 -## Quick Start +Adds a lock free queue for concurrent order ingest. Also cuts add and cancel latency further. The public API has not changed. Details and numbers are in [docs/BENCHMARKS.md](docs/BENCHMARKS.md). -### Build +## Build ```sh cmake -S . -B build cmake --build build --config Release ``` -### Integrate +## Integrate ```cmake add_subdirectory(liborderbook) target_link_libraries(your_app PRIVATE orderbook::orderbook) ``` -### Use +## Use ```cpp +#include + #include "OrderBook.h" +#include "BookPrinter.h" using namespace orderbook; OrderBook book; -// Place limit orders book.addOrder(OrderBuilder{}.id(OrderId{1}).buy().goodTillCancel() .price(Price{100}).quantity(Quantity{10}).build()); book.addOrder(OrderBuilder{}.id(OrderId{2}).sell().goodTillCancel() .price(Price{101}).quantity(Quantity{20}).build()); -// Market order crosses the spread +// Market order crosses the spread and returns the trades it produced. auto trades = book.addOrder(OrderBuilder{}.id(OrderId{3}).buy().market() .quantity(Quantity{5}).build()); book.cancelOrder(OrderId{1}); -book.printBook(); +printBook(std::cout, book); ``` -Output: - ``` === ASKS === 101 | 15 (1) @@ -64,8 +65,6 @@ Output: ## API -### OrderBuilder - ```cpp OrderBuilder{} .id(OrderId{42}) @@ -74,60 +73,54 @@ OrderBuilder{} .price(Price{100}) // omit for market orders .quantity(Quantity{10}) .build(); -``` - -### OrderBook -```cpp -std::vector addOrder(Order order); // Match + rest remainder -bool cancelOrder(OrderId id); // Remove resting order -std::size_t size() const; // Live order count -OrderBookLevelInfos getLevelInfos() const; // Aggregated depth -const std::vector& getTrades() const; // Trade history -void printBook(); // Debug output +std::span addOrder(Order order); // Match against the book, rest the remainder +bool cancelOrder(OrderId id); // Remove a resting order by id +std::size_t size() const; // Live resting order count +OrderBookLevelInfos getLevelInfos() const; // Aggregated depth, both sides ``` +`Order`, `OrderId`, `Price`, and `Quantity` are strong types. A price cannot be passed where a quantity is expected. + ## Benchmarks -Single thread, MSVC Release with IPO and AVX2: +Single core, Linux, GCC Release `-O3`: | Operation | Latency | |-----------|---------| -| Add, no match | 67 ns | -| Add, single match | 143 ns | -| Add into 10k levels | 58 ns | -| Cancel | 75 ns | +| Add, no match | 37 ns | +| Add, single match | 37.5 ns | +| Add into 10k price levels | 34 ns | +| Cancel | 20 ns | ```sh -./build/Release/orderbook_bench_all +./build/orderbook_bench_all ``` -See [BENCHMARKS.md](docs/BENCHMARKS.md) for history. +Full history in [docs/BENCHMARKS.md](docs/BENCHMARKS.md). ## Tests ```sh cmake --build build --target orderbook_tests -./build/Release/orderbook_tests +./build/orderbook_tests ``` +CI runs the suite on GCC and Clang, in Release and Debug, and under AddressSanitizer, UndefinedBehaviorSanitizer, and ThreadSanitizer. + ## Documentation -| Document | Description | -|----------|-------------| -| [Introduction](docs/introduction.md) | Architecture and design | -| [Installation](docs/installation.md) | Build and integration | -| [Guide](docs/guide.md) | Core concepts and operations | -| [Examples](docs/examples.md) | Usage patterns | -| [Support](docs/support.md) | FAQ, contributing, license | +- [User Guide](docs/user_guide.md) +- [Benchmarks](docs/BENCHMARKS.md) ## References -- [How to Build a Fast Limit Order Book](https://web.archive.org/web/20110219163448/http://howtohft.wordpress.com/2011/02/15/how-to-build-a-fast-limit-order-book/) -- [Limit Order Book in C++](https://alexabosi.wordpress.com/2014/08/28/limit-order-book-implementation-for-low-latency-trading-in-c/) — Abosi -- [brprojects/Limit-Order-Book](https://github.com/brprojects/Limit-Order-Book) -- [Strong types with CRTP](https://youtu.be/fWcnp7Bulc8) — Fluent C++ -- [TomaszRewak/cpp-allocator](https://github.com/TomaszRewak/cpp-allocator) — slab allocator +- [How to Build a Fast Limit Order Book](https://web.archive.org/web/20110219163448/http://howtohft.wordpress.com/2011/02/15/how-to-build-a-fast-limit-order-book/) - the array indexed price level technique this book uses +- [Bounded MPMC queue](https://web.archive.org/web/20160310160656/http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue) - Dmitry Vyukov, the design `ConcurrentQueue` is built on +- [Limit Order Book in C++](https://alexabosi.wordpress.com/2014/08/28/limit-order-book-implementation-for-low-latency-trading-in-c/) - Abosi +- [Bit Twiddling Hacks](https://graphics.stanford.edu/~seander/bithacks.html) - Sean Eron Anderson, Stanford. The bit scan technique behind the occupancy bitset +- [TomaszRewak/cpp-allocator](https://github.com/TomaszRewak/cpp-allocator) - slab allocator +- [Strong types with CRTP](https://youtu.be/fWcnp7Bulc8) - Fluent C++ ## License diff --git a/bench/depth_bench.cpp b/bench/depth_bench.cpp index 8a10923..a0cf719 100644 --- a/bench/depth_bench.cpp +++ b/bench/depth_bench.cpp @@ -29,7 +29,10 @@ static void BM_Depth_AddAtBest(benchmark::State& state) { benchmark::DoNotOptimize(trades); } } -BENCHMARK(BM_Depth_AddAtBest)->Arg(0)->Arg(100)->Arg(1000)->Arg(10000); +// Iterations are capped. The refill above is O(depth). Without this cap, +// auto calibration picks iteration counts in the millions, which turns +// that refill cost into minutes of wall time. +BENCHMARK(BM_Depth_AddAtBest)->Arg(0)->Arg(100)->Arg(1000)->Arg(10000)->Iterations(20000); // Add orders at random prices across the price range. // Forces tree traversal on every add. Measures worst-case map descent. @@ -56,7 +59,7 @@ static void BM_Depth_AddScatter(benchmark::State& state) { benchmark::DoNotOptimize(trades); } } -BENCHMARK(BM_Depth_AddScatter)->Arg(100)->Arg(1000)->Arg(10000); +BENCHMARK(BM_Depth_AddScatter)->Arg(100)->Arg(1000)->Arg(10000)->Iterations(20000); // Matching that sweeps through multiple price levels. // Aggressive buy crosses spread and walks up the ask ladder. @@ -106,7 +109,7 @@ static void BM_Depth_SweepLevels(benchmark::State& state) { benchmark::DoNotOptimize(trades); } } -BENCHMARK(BM_Depth_SweepLevels)->Arg(2)->Arg(5)->Arg(10)->Arg(20); +BENCHMARK(BM_Depth_SweepLevels)->Arg(2)->Arg(5)->Arg(10)->Arg(20)->Iterations(20000); #ifndef BENCHMARK_COMBINED BENCHMARK_MAIN(); diff --git a/bench/ingest_pipeline_bench.cpp b/bench/ingest_pipeline_bench.cpp new file mode 100644 index 0000000..44dbd86 --- /dev/null +++ b/bench/ingest_pipeline_bench.cpp @@ -0,0 +1,144 @@ +#include + +#include +#include +#include + +#include "Order.h" +#include "OrderBook.h" +#include "lockfree/ConcurrentQueue.h" + +using orderbook::ConcurrentQueue; +using orderbook::Order; +using orderbook::OrderBook; +using orderbook::OrderBuilder; +using orderbook::OrderId; +using orderbook::Price; +using orderbook::Quantity; + +namespace +{ +constexpr std::size_t kQueueCapacity = 4096; +constexpr Price kMinPrice{ 1 }; +constexpr Price kMaxPrice{ 12'000 }; +constexpr std::size_t kBookCapacity = 2'000'000; + +Order makeOrder(std::uint64_t id) +{ + return OrderBuilder{}.id(OrderId{ id }).buy().goodTillCancel().price(Price{ 100 }).quantity(Quantity{ 10 }).build(); +} +} // namespace + +static void BM_IngestPipeline_Mutex(benchmark::State& state) +{ + static std::mutex* mutex = nullptr; + static std::queue* queue = nullptr; + static OrderBook* book = nullptr; + + const int producerCount = state.threads() - 1; + + if (state.thread_index() == 0) + { + mutex = new std::mutex(); + queue = new std::queue(); + book = new OrderBook(kMinPrice, kMaxPrice, kBookCapacity); + } + + if (state.thread_index() == 0) + { + for (auto _ : state) + { + for (int p = 0; p < producerCount; ++p) + { + for (;;) + { + { + std::lock_guard lock(*mutex); + if (!queue->empty()) + { + Order order = std::move(queue->front()); + queue->pop(); + auto trades = book->addOrder(std::move(order)); + benchmark::DoNotOptimize(trades); + break; + } + } + __builtin_ia32_pause(); + } + } + } + } + else + { + std::uint64_t id = static_cast(state.thread_index()) * 100'000'000ULL; + for (auto _ : state) + { + std::lock_guard lock(*mutex); + queue->push(makeOrder(++id)); + } + } + + if (state.thread_index() == 0) + { + delete book; + delete queue; + delete mutex; + } +} +BENCHMARK(BM_IngestPipeline_Mutex)->Threads(2)->Threads(4)->Threads(8)->Iterations(20000); + +// Same topology as above. This one uses a lock free queue instead of a +// mutex. +static void BM_IngestPipeline_LockFree(benchmark::State& state) +{ + static ConcurrentQueue* queue = nullptr; + static OrderBook* book = nullptr; + + const int producerCount = state.threads() - 1; + + if (state.thread_index() == 0) + { + queue = new ConcurrentQueue(); + book = new OrderBook(kMinPrice, kMaxPrice, kBookCapacity); + } + + if (state.thread_index() == 0) + { + Order slot = makeOrder(0); + for (auto _ : state) + { + for (int p = 0; p < producerCount; ++p) + { + while (!queue->tryPop(slot)) + { + __builtin_ia32_pause(); + } + auto trades = book->addOrder(std::move(slot)); + benchmark::DoNotOptimize(trades); + } + } + } + else + { + std::uint64_t id = static_cast(state.thread_index()) * 100'000'000ULL; + for (auto _ : state) + { + Order order = makeOrder(++id); + while (!queue->tryPush(std::move(order))) + { + __builtin_ia32_pause(); + } + } + } + + if (state.thread_index() == 0) + { + delete book; + delete queue; + } +} +BENCHMARK(BM_IngestPipeline_LockFree)->Threads(2)->Threads(4)->Threads(8)->Iterations(20000); + +#ifndef BENCHMARK_COMBINED +BENCHMARK_MAIN(); +#endif diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 008eb6e..753e8b4 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -9,71 +9,77 @@ Latency of core operations measured with [Google Benchmark](https://github.com/g - Linux, GCC Release -O3 - Single threaded -Each operation is timed in a steady-state book. A batch of pre-built orders is replayed; timing pauses to rebuild the book between batches so allocation and teardown are excluded. +Each benchmark runs on a book that already has orders in it. -## Current Results (v2.3) +## Current Results (v3.0.0) ``` -BM_AddOrder_NoMatch 43.3 ns -BM_AddOrder_SingleMatch 42.7 ns -BM_AddOrder_MultiMatch/2 68.0 ns -BM_AddOrder_MultiMatch/5 163 ns -BM_AddOrder_MultiMatch/10 326 ns +BM_AddOrder_NoMatch 37.0 ns +BM_AddOrder_SingleMatch 37.5 ns +BM_AddOrder_MultiMatch/2 55.3 ns +BM_AddOrder_MultiMatch/5 129 ns +BM_AddOrder_MultiMatch/10 241 ns + +BM_CancelOrder 20.2 ns +BM_CancelOrder_MidQueue/10 18.3 ns +BM_CancelOrder_MidQueue/100 6.67 ns +BM_CancelOrder_MidQueue/1000 13.5 ns + +BM_Depth_AddAtBest/0 20.3 ns +BM_Depth_AddAtBest/100 21.8 ns +BM_Depth_AddAtBest/1000 39.9 ns +BM_Depth_AddAtBest/10000 34.0 ns +BM_Depth_AddScatter/100 25.6 ns +BM_Depth_AddScatter/1000 46.0 ns +BM_Depth_AddScatter/10000 38.2 ns +BM_Depth_SweepLevels/2 31.1 ns +BM_Depth_SweepLevels/5 79.4 ns +BM_Depth_SweepLevels/10 209 ns +BM_Depth_SweepLevels/20 417 ns + +BM_GetLevelInfos/10 32.8 ns +BM_GetLevelInfos/100 352 ns +BM_GetLevelInfos/1000 3101 ns +BM_GetLevelInfos/10000 31416 ns +BM_GetLevelInfos_BothSides/10 57.7 ns +BM_GetLevelInfos_BothSides/100 657 ns +BM_GetLevelInfos_BothSides/1000 6116 ns +``` -BM_CancelOrder 21.0 ns -BM_CancelOrder_MidQueue/10 22.8 ns -BM_CancelOrder_MidQueue/100 6.57 ns -BM_CancelOrder_MidQueue/1000 14.4 ns +Every number dropped from v2.3. See "v3.0.0" under Optimization History for why. -BM_Depth_AddAtBest/0 26.1 ns -BM_Depth_AddAtBest/100 27.1 ns -BM_Depth_AddAtBest/1000 44.1 ns -BM_Depth_AddAtBest/10000 34.8 ns -BM_Depth_AddScatter/100 28.2 ns -BM_Depth_AddScatter/1000 46.9 ns -BM_Depth_AddScatter/10000 37.8 ns -BM_Depth_SweepLevels/2 43.5 ns -BM_Depth_SweepLevels/5 93.6 ns -BM_Depth_SweepLevels/10 205 ns -BM_Depth_SweepLevels/20 449 ns +### Lock free queue + +v3.0.0 adds `ConcurrentQueue`. It is a bounded lock free MPMC ring buffer for order ingest. Many producer threads push orders into it. One consumer thread drains it into the matching engine. The matching engine stays single threaded. This keeps it correct and fast. -BM_GetLevelInfos/10 29.8 ns -BM_GetLevelInfos/100 395 ns -BM_GetLevelInfos/1000 2977 ns -BM_GetLevelInfos/10000 34664 ns -BM_GetLevelInfos_BothSides/10 57.0 ns -BM_GetLevelInfos_BothSides/100 693 ns -BM_GetLevelInfos_BothSides/1000 5876 ns +``` + threads:2 threads:4 threads:8 +Mutex + queue 158 ns 386 ns 943 ns +ConcurrentQueue 42.3 ns 260 ns 945 ns ``` -The v2.3 array-indexed price ladder replaced the old `std::map`-backed price index, so the -benchmark suite was rewritten alongside it (`bench/add_order_bench.cpp`, -`cancel_order_bench.cpp`, `depth_bench.cpp`, `level_info_bench.cpp`) to isolate the new -access patterns — see "What Each Benchmark Measures" below. The old `AddOrder_PreBuilt` / -`AddOrder_AtDepth` names from the v1/v2/v2.1 history further down no longer exist; the -closest current equivalents are `AddOrder_NoMatch` and `Depth_AddAtBest`, but the numbers -aren't directly comparable since the harness and scenarios changed. +The single consumer becomes the bottleneck at high thread counts. ## What Each Benchmark Measures -- **AddOrder_NoMatch** - Resting insert, no match, single price level (best case for cache locality) -- **AddOrder_SingleMatch** - Add + immediate single match, full crossing-spread latency -- **AddOrder_MultiMatch/N** - Aggressive order sweeps N resting orders at the same price level -- **CancelOrder** - Locate by id and unlink, book has one order per price level -- **CancelOrder_MidQueue/N** - Cancel from the middle of an N-deep price-level queue (intrusive list unlink, not head/tail) -- **Depth_AddAtBest/N** - Insert against a book holding N price levels, all adds at the best price (isolates level-lookup cost as the ladder fills) -- **Depth_AddScatter/N** - Insert at random prices across an N-level book (worst-case index lookup, no cache locality between adds) -- **Depth_SweepLevels/N** - Aggressive order matches across N price levels (multi-level book walk) -- **GetLevelInfos/N** - Snapshot one side of a book holding N price levels -- **GetLevelInfos_BothSides/N** - Snapshot both sides of a book holding N price levels each +- **AddOrder_NoMatch**: Adds one order that rests with no match. This is the best case for cache locality. +- **AddOrder_SingleMatch**: Adds one order that matches and fills in one trade. This is the full cost of crossing the spread. +- **AddOrder_MultiMatch/N**: An aggressive order sweeps N resting orders at the same price level. +- **CancelOrder**: Finds an order by id and unlinks it. The book has one order per price level. +- **CancelOrder_MidQueue/N**: Cancels an order from the middle of a price level queue that is N orders deep. This tests unlinking from the middle of the list, not the head or tail. +- **Depth_AddAtBest/N**: Adds orders to a book that already holds N price levels. All adds go to the best price. This isolates the cost of finding a level as the ladder fills up. +- **Depth_AddScatter/N**: Adds orders at random prices across a book holding N price levels. This is the worst case for locating a level, since there is no cache locality between adds. +- **Depth_SweepLevels/N**: An aggressive order matches across N price levels. This walks the book across multiple levels. +- **GetLevelInfos/N**: Takes a snapshot of one side of a book holding N price levels. +- **GetLevelInfos_BothSides/N**: Takes a snapshot of both sides of a book, each holding N price levels. ## Optimization History -### v1 - Baseline +### v1: Baseline `std::map` per side, `std::list` per level, `std::shared_ptr` ownership. -Each add allocates a list node. Each level walk costs two cache hops. Ownership via shared_ptr adds a control block and atomic refcount per order. +Each add allocates a list node. Each level walk costs two cache hops. Ownership through shared_ptr adds a control block and an atomic refcount to every order. ``` AddOrder_PreBuilt 170 ns @@ -83,11 +89,11 @@ AddOrder_AtDepth/10000 121 ns CancelOrder 115 ns ``` -### v2 - Intrusive Linked List +### v2: Intrusive Linked List -Order carries its own prev/next. PriceLevel holds head/tail. No per-add list-node allocation. shared_ptr swapped for unique_ptr. +Order carries its own prev and next pointers. PriceLevel holds a head and a tail. Adding an order no longer allocates a list node. shared_ptr was replaced with unique_ptr. -~65 ns drop on resting path. +Resting an order got about 65 ns faster. ``` AddOrder_PreBuilt 91.7 ns @@ -97,11 +103,11 @@ AddOrder_AtDepth/10000 84.1 ns CancelOrder 94.0 ns ``` -### v2.1 - Slab Allocator +### v2.1: Slab Allocator -Orders live in a preallocated slab with LIFO free list. Allocation is a free-list pop or high-water bump. Just-freed slots are hot in cache on reuse. +Orders live in a slab that is allocated once, with a free list for reuse. Allocating an order pops from the free list or bumps the high water mark forward. A slot that was just freed is still hot in cache when it gets reused. -~27% improvement on resting path. +Resting an order got about 27% faster. ``` AddOrder_PreBuilt 67.0 ns @@ -111,46 +117,69 @@ AddOrder_AtDepth/10000 58.3 ns CancelOrder 75.3 ns ``` -### v2.2 - Reusable Trade Buffer - -`addOrder` returned a freshly allocated `std::vector` on every call, and the engine -kept an unbounded internal trade log growing for the lifetime of the book. Both removed: -trades are now written into a `std::span` over a single reusable buffer cleared at the start -of each `addOrder` call, and the unbounded log is gone. +### v2.2: Reusable Trade Buffer -### v2.3 - Array-Indexed Price Ladder +`addOrder` used to allocate a new `std::vector` on every call. The engine also kept an internal trade log that grew for the whole life of the book. Both are gone now. Trades are written into a `std::span` over one reusable buffer, and that buffer is cleared at the start of each `addOrder` call. -The `std::map` price index was the remaining bottleneck. Every add, cancel and level-drain -paid a red-black tree descent: `try_emplace` on insert, `at` on cancel, `erase` when a level -emptied. Each descent was a chain of dependent pointer loads into scattered heap nodes, so the -cost was cache misses rather than comparisons, and it grew with book depth. +### v2.3: Array Indexed Price Ladder +The `std::map` price index was the remaining bottleneck. Every add, cancel, and level drain walked down a red black tree: a `try_emplace` on insert, an `at` on cancel, an `erase` when a level emptied. Each walk was a chain of pointer loads into scattered heap nodes. The replacement is a dense ladder indexed directly by price. -**Layout.** One `PriceLevel` array per side, sized to a fixed tick band chosen at construction, -where `index = price - minPrice`. Locating a level becomes an add and a load. There is no tree, -no per-level node allocation, and no rebalancing. +**Layout.** Each side gets one `PriceLevel` array. Its size is a fixed tick band chosen at construction. A level's index is `price minus minPrice`. Finding a level is just an add and a load. There is no tree, no node allocation for each level, and no rebalancing. + +**Occupancy bitset.** A dense array is mostly empty, so a linear scan cannot answer "which level is best now". Occupancy is mirrored into a bitset, with 64 levels per word. The bitset is scanned with `std::countr_zero` and `std::countl_zero`. When the best level drains, finding the next one usually takes a single word test, not a walk. + +**Tracked extent.** The ladder remembers its best and worst occupied index. Traversals like `getLevelInfos` and the FOK liquidity check only scan the populated region, not the whole band. A book with 10 levels never scans an array of 110,000 levels. + +**Side as a template parameter.** `PriceLadder` treats the highest occupied price as best. `PriceLadder` treats the lowest occupied price as best. This is resolved at compile time, so it removes a branch from every check for the best level, and it lets the matching code treat both sides the same way. + +Alongside the ladder, the order index moved from `std::unordered_map` to an open addressing table. Resting orders are indexed by pointer alone, so the hot path no longer needs an id lookup. + +``` +BM_AddOrder_NoMatch 43.3 ns +BM_AddOrder_SingleMatch 42.7 ns +BM_AddOrder_MultiMatch/2 68.0 ns +BM_AddOrder_MultiMatch/5 163 ns +BM_AddOrder_MultiMatch/10 326 ns + +BM_CancelOrder 21.0 ns +BM_CancelOrder_MidQueue/10 22.8 ns +BM_CancelOrder_MidQueue/100 6.57 ns +BM_CancelOrder_MidQueue/1000 14.4 ns + +BM_Depth_AddAtBest/0 26.1 ns +BM_Depth_AddAtBest/100 27.1 ns +BM_Depth_AddAtBest/1000 44.1 ns +BM_Depth_AddAtBest/10000 34.8 ns +BM_Depth_AddScatter/100 28.2 ns +BM_Depth_AddScatter/1000 46.9 ns +BM_Depth_AddScatter/10000 37.8 ns +BM_Depth_SweepLevels/2 43.5 ns +BM_Depth_SweepLevels/5 93.6 ns +BM_Depth_SweepLevels/10 205 ns +BM_Depth_SweepLevels/20 449 ns + +BM_GetLevelInfos/10 29.8 ns +BM_GetLevelInfos/100 395 ns +BM_GetLevelInfos/1000 2977 ns +BM_GetLevelInfos/10000 34664 ns +BM_GetLevelInfos_BothSides/10 57.0 ns +BM_GetLevelInfos_BothSides/100 693 ns +BM_GetLevelInfos_BothSides/1000 5876 ns +``` + +The benchmark suite was rewritten alongside this change (`bench/add_order_bench.cpp`, `cancel_order_bench.cpp`, `depth_bench.cpp`, `level_info_bench.cpp`) to isolate the new access patterns. See "What Each Benchmark Measures" above. The old `AddOrder_PreBuilt` and `AddOrder_AtDepth` names from v1, v2, and v2.1 no longer exist. The closest equivalents are `AddOrder_NoMatch` and `Depth_AddAtBest`, but the numbers are not directly comparable, since the harness and scenarios changed. -**Occupancy bitset.** A dense array is mostly empty, so "which level is best now?" cannot be a -linear scan. Occupancy is mirrored into a bitset, 64 levels per word, scanned with -`std::countr_zero` / `std::countl_zero`. When the best level drains, finding the next populated -one is typically a single word test rather than a walk. +### v3.0.0 performance changes: -**Tracked extent.** The ladder remembers its best and worst occupied index. Traversals -(`getLevelInfos`, the FOK liquidity check) are bounded to the populated region instead of the -whole band, so a 10-level book never scans a 110k-level array. +`matchOrder` timestamped every trade with `system_clock::now()`, which is a syscall on most platforms. It now reads the CPU's own cycle counter and converts that into a timestamp with one multiply. This clock is calibrated once per process at startup. -**Side as a template parameter.** `PriceLadder` treats the highest occupied price as -best, `PriceLadder` the lowest. Resolving this at compile time removes a branch from -every best-level access and lets the matching code treat both sides symmetrically. +`Order` was 72 bytes. It spilled across two cache lines because two 1 byte fields sat on either side of an 8 byte field. Reordering the fields brought it down to exactly 64 bytes, which is one cache line. As a side effect, the order slab shrank by 8MB at the default capacity of 1 million orders. -**Trade-off.** The band is bounded, so memory is `O(band)` rather than `O(levels)` and prices -outside the band are rejected — the same price-collar constraint real venues enforce. The -default band is deliberately wide enough to cover ordinary use and still fits in a few MB. +The order index hashed and probed an order's id twice on the common paths. `addOrder` checked that the id was not a duplicate. Then `addToBook` hashed and probed the same id again to insert it. The cancel path had the same problem between finding an order and erasing it. Both paths now do the lookup once and reuse the result. -Alongside the ladder, the order index moved from `std::unordered_map` to an -open-addressing table, and resting orders are indexed by pointer alone rather than id lookup -on the hot path — see "Current Results (v2.3)" above for the combined effect. +The code copied a `Trade` into the trade buffer where a move works. ## Running Benchmarks diff --git a/src/Order.cpp b/src/Order.cpp index f325128..c26b7e8 100644 --- a/src/Order.cpp +++ b/src/Order.cpp @@ -1,9 +1,10 @@ #include "Order.h" -#include #include #include +#include "TscClock.h" + namespace orderbook { void Order::fill(Quantity qty) @@ -20,9 +21,7 @@ void Order::fill(Quantity qty) Timestamp Order::now() { - auto n = - std::chrono::time_point_cast(std::chrono::system_clock::now()); - return Timestamp{ n.time_since_epoch() }; + return tscNow(); } OrderBuilder& OrderBuilder::id(OrderId id) diff --git a/src/Order.h b/src/Order.h index a7fcd5e..0319f75 100644 --- a/src/Order.h +++ b/src/Order.h @@ -17,13 +17,13 @@ class Order Order* next{ nullptr }; Order(OrderType type, OrderId id, Side side, Price price, Quantity quantity, Timestamp ts) - : orderType_{ type } - , orderId_{ id } - , side_{ side } + : orderId_{ id } , price_{ price } , initialQuantity_{ quantity } , remainingQuantity_{ quantity } , timestamp_{ ts } + , orderType_{ type } + , side_{ side } { } @@ -52,13 +52,13 @@ class Order static Timestamp now(); private: - OrderType orderType_; OrderId orderId_; - Side side_; Price price_; Quantity initialQuantity_; Quantity remainingQuantity_; Timestamp timestamp_; + OrderType orderType_; + Side side_; }; class OrderBuilder diff --git a/src/OrderBook.cpp b/src/OrderBook.cpp index ffb2224..84253a3 100644 --- a/src/OrderBook.cpp +++ b/src/OrderBook.cpp @@ -1,69 +1,45 @@ #include "OrderBook.h" #include -#include +#include #include #include "Order.h" #include "Side.h" +#include "TscClock.h" #include "Types.h" namespace orderbook { -void OrderBook::pushBack(PriceLevel& level, Order* order) +PriceLevel& OrderBook::getOrCreateLevel(Side side, Price price) { - order->prev = level.tail; - order->next = nullptr; - - if (level.tail != nullptr) - { - level.tail->next = order; - } - else - { - level.head = order; - } - - level.tail = order; - level.orderCount++; - level.totalQuantity += order->getRemainingQuantity(); + return side == Side::Buy ? bids_.getOrCreate(price) : asks_.getOrCreate(price); } -void OrderBook::unlink(PriceLevel& level, Order* order) +PriceLevel* OrderBook::findLevel(Side side, Price price) { - if (order->prev != nullptr) - { - order->prev->next = order->next; - } - else - { - level.head = order->next; - } + return side == Side::Buy ? bids_.find(price) : asks_.find(price); +} - if (order->next != nullptr) +void OrderBook::eraseLevel(Side side, Price price) +{ + if (side == Side::Buy) { - order->next->prev = order->prev; + bids_.erase(price); } else { - level.tail = order->prev; + asks_.erase(price); } - - order->prev = nullptr; - order->next = nullptr; - level.orderCount--; } -void OrderBook::addToBook(Order* order) +void OrderBook::addToBook(Order* order, std::size_t indexHint) { - const Price price = order->getPrice(); - const Side side = order->getSide(); + PriceLevel& level = getOrCreateLevel(order->getSide(), order->getPrice()); - PriceLevel& level = (side == Side::Buy) ? bids_.getOrCreate(price) : asks_.getOrCreate(price); + level.pushBack(order); - pushBack(level, order); - - orders_.insert(order->getOrderId(), order); + orders_.insertAt(indexHint, order->getOrderId(), order); } bool OrderBook::acceptsPrice(const Order& order) const noexcept @@ -131,9 +107,27 @@ bool OrderBook::canFullyFill(Side side, Price price, Quantity qty) const return sufficient; } -PriceLevel* OrderBook::getBestOppositeLevel(const Order* order) +// This does what canMatch does and what the old getBestOppositeLevel did, +// in one call. matchOrder's loop used to call both on every iteration. +// That meant asks_.best() or bids_.best() got looked up twice for the same +// answer. +PriceLevel* OrderBook::matchableLevel(const Order* order) { - return order->getSide() == Side::Buy ? asks_.best() : bids_.best(); + const Price orderPrice = order->getPrice(); + + if (order->getOrderType() == OrderType::Market) + { + return order->getSide() == Side::Buy ? asks_.best() : bids_.best(); + } + + if (order->getSide() == Side::Buy) + { + PriceLevel* bestAsk = asks_.best(); + return (bestAsk != nullptr && orderPrice >= bestAsk->price) ? bestAsk : nullptr; + } + + PriceLevel* bestBid = bids_.best(); + return (bestBid != nullptr && orderPrice <= bestBid->price) ? bestBid : nullptr; } Trade OrderBook::createTrade(Order* incomingOrder, @@ -167,44 +161,30 @@ Trade OrderBook::createTrade(Order* incomingOrder, return trade; } -void OrderBook::cleanupAfterTrade(Order* incoming, - Order* resting, - PriceLevel& level, - Quantity tradeQty) +void OrderBook::cleanupAfterTrade(Order* resting, PriceLevel& level, Quantity tradeQty) { - const Price restingPrice = resting->getPrice(); - level.totalQuantity -= tradeQty; if (resting->isFilled()) { orders_.erase(resting->getOrderId()); - unlink(level, resting); + level.unlink(resting); orderSlab_.release(resting); } if (level.orderCount == 0) { - if (incoming->getSide() == Side::Buy) - { - asks_.erase(restingPrice); - } - else - { - bids_.erase(restingPrice); - } + eraseLevel(resting->getSide(), resting->getPrice()); } } void OrderBook::matchOrder(Order* incoming) { - const auto now = - std::chrono::time_point_cast(std::chrono::system_clock::now()); - const Timestamp arrivalTime{ now.time_since_epoch() }; + const Timestamp arrivalTime = tscNow(); - while (!incoming->isFilled() && canMatch(incoming)) + while (!incoming->isFilled()) { - PriceLevel* level = getBestOppositeLevel(incoming); + PriceLevel* level = matchableLevel(incoming); if (level == nullptr || level->head == nullptr) { @@ -221,9 +201,9 @@ void OrderBook::matchOrder(Order* incoming) incoming->fill(tradeQuantity); resting->fill(tradeQuantity); - tradeBuffer_.push_back(trade); + tradeBuffer_.push_back(std::move(trade)); - cleanupAfterTrade(incoming, resting, *level, tradeQuantity); + cleanupAfterTrade(resting, *level, tradeQuantity); } } @@ -231,7 +211,8 @@ std::span OrderBook::addOrder(Order order) { tradeBuffer_.clear(); - if (orders_.contains(order.getOrderId())) + const OrderIndex::SlotLookup slot = orders_.findSlot(order.getOrderId()); + if (slot.occupied) { return {}; } @@ -260,7 +241,7 @@ std::span OrderBook::addOrder(Order order) if (restsInBook) { - addToBook(incoming); + addToBook(incoming, slot.index); } else { @@ -270,18 +251,16 @@ std::span OrderBook::addOrder(Order order) return tradeBuffer_; } -bool OrderBook::removeOrder(OrderId orderId) +bool OrderBook::cancelOrder(OrderId orderId) { - Order* order = orders_.find(orderId); + const OrderIndex::IndexLookup lookup = orders_.findIndex(orderId); + Order* order = lookup.order; if (order == nullptr) { return false; } - const Side side = order->getSide(); - const Price price = order->getPrice(); - - PriceLevel* level = (side == Side::Buy) ? bids_.find(price) : asks_.find(price); + PriceLevel* level = findLevel(order->getSide(), order->getPrice()); if (level == nullptr) { @@ -290,31 +269,19 @@ bool OrderBook::removeOrder(OrderId orderId) level->totalQuantity -= order->getRemainingQuantity(); - unlink(*level, order); + level->unlink(order); if (level->orderCount == 0) { - if (side == Side::Buy) - { - bids_.erase(price); - } - else - { - asks_.erase(price); - } + eraseLevel(order->getSide(), order->getPrice()); } - orders_.erase(orderId); + orders_.eraseAt(lookup.index); orderSlab_.release(order); return true; } -bool OrderBook::cancelOrder(OrderId orderId) -{ - return removeOrder(orderId); -} - OrderBookLevelInfos OrderBook::getLevelInfos() const { OrderBookLevelInfos result; diff --git a/src/OrderBook.h b/src/OrderBook.h index 0d85da7..e18ff1e 100644 --- a/src/OrderBook.h +++ b/src/OrderBook.h @@ -47,15 +47,20 @@ class OrderBook [[nodiscard]] bool acceptsPrice(const Order& order) const noexcept; [[nodiscard]] bool canMatch(const Order* order) const; [[nodiscard]] bool canFullyFill(Side side, Price price, Quantity qty) const; - [[nodiscard]] PriceLevel* getBestOppositeLevel(const Order* order); + [[nodiscard]] PriceLevel* matchableLevel(const Order* order); + + [[nodiscard]] PriceLevel& getOrCreateLevel(Side side, Price price); + [[nodiscard]] PriceLevel* findLevel(Side side, Price price); + void eraseLevel(Side side, Price price); + Trade createTrade(Order* incoming, Order* resting, Quantity quantity, Timestamp timestamp); - void cleanupAfterTrade(Order* incoming, Order* resting, PriceLevel& level, Quantity tradeQty); + void cleanupAfterTrade(Order* resting, PriceLevel& level, Quantity tradeQty); void matchOrder(Order* incomingOrder); - void addToBook(Order* order); - bool removeOrder(OrderId orderId); - static void pushBack(PriceLevel& level, Order* order); - static void unlink(PriceLevel& level, Order* order); + // indexHint must come from the OrderIndex::SlotLookup that addOrder + // already got from findSlot for this same order. It lets insertAt skip + // hashing and probing an id that was already found once. + void addToBook(Order* order, std::size_t indexHint); }; } diff --git a/src/OrderIndex.h b/src/OrderIndex.h index 63a64e8..8a8d768 100644 --- a/src/OrderIndex.h +++ b/src/OrderIndex.h @@ -46,7 +46,16 @@ class OrderIndex [[nodiscard]] bool contains(OrderId id) const noexcept { return find(id) != nullptr; } - void insert(OrderId id, Order* order) + struct SlotLookup + { + std::size_t index; + bool occupied; + }; + + // OrderBook calls this once to check for a duplicate id and find where + // a new one would go. It then reuses the index with insertAt below. + // This avoids hashing and probing the same id twice. + [[nodiscard]] SlotLookup findSlot(OrderId id) noexcept { if ((size_ + 1) * MaxLoadDenominator > slots_.size() * MaxLoadNumerator) { @@ -60,31 +69,76 @@ class OrderIndex { if (slots_[index].id == id) { - slots_[index].order = order; - return; + return { index, true }; } index = (index + 1) & mask; } + return { index, false }; + } + + // Only use an index that findSlot just returned, before any other + // insert on this table. grow() would move everything and make the + // index invalid, but findSlot already grows the table first, so that + // cannot happen in between. + void insertAt(std::size_t index, OrderId id, Order* order) noexcept + { slots_[index] = Slot{ .id = id, .order = order }; ++size_; } - bool erase(OrderId id) noexcept + // This is an upsert for OrderIndex's own general use. grow uses it to + // rehash below, and any caller without a findSlot result to reuse can + // call it too. It is built on findSlot and insertAt instead of its own + // probe loop, so there is one place that does probing, not two copies + // to keep in sync. + void insert(OrderId id, Order* order) { - const std::size_t mask = slots_.size() - 1; - std::size_t hole = hash(id) & mask; - - while (slots_[hole].order != nullptr && slots_[hole].id != id) + const SlotLookup slot = findSlot(id); + if (slot.occupied) { - hole = (hole + 1) & mask; + slots_[slot.index].order = order; + return; } - if (slots_[hole].order == nullptr) + insertAt(slot.index, id, order); + } + + struct IndexLookup + { + std::size_t index; + Order* order; + }; + + // Same idea as findSlot. OrderBook's cancel path already needs to find + // an order before it can remove it. So this hands back the index too, + // and eraseAt reuses it instead of erase() looking it up again from + // the id. + [[nodiscard]] IndexLookup findIndex(OrderId id) const noexcept + { + const std::size_t mask = slots_.size() - 1; + std::size_t index = hash(id) & mask; + + while (slots_[index].order != nullptr) { - return false; + if (slots_[index].id == id) + { + return { index, slots_[index].order }; + } + index = (index + 1) & mask; } + return { index, nullptr }; + } + + // Only use an index that findIndex just returned, before any other + // insert or erase on this table. Unlike insertAt, there is no grow() + // to worry about here. Erasing never resizes the table. + void eraseAt(std::size_t index) noexcept + { + const std::size_t mask = slots_.size() - 1; + std::size_t hole = index; + slots_[hole].order = nullptr; --size_; @@ -109,7 +163,24 @@ class OrderIndex slots_[probe].order = nullptr; hole = probe; } + } + + bool erase(OrderId id) noexcept + { + const std::size_t mask = slots_.size() - 1; + std::size_t hole = hash(id) & mask; + + while (slots_[hole].order != nullptr && slots_[hole].id != id) + { + hole = (hole + 1) & mask; + } + + if (slots_[hole].order == nullptr) + { + return false; + } + eraseAt(hole); return true; } diff --git a/src/OrderSlab.h b/src/OrderSlab.h index 57da193..342762e 100644 --- a/src/OrderSlab.h +++ b/src/OrderSlab.h @@ -49,7 +49,9 @@ class OrderSlab ~OrderSlab() = default; private: - union Slot + // alignas(64) puts each slot on its own cache line. Order is exactly + // one cache line in size, so two orders never share a line. + union alignas(64) Slot { Order order; Slot* nextFree; diff --git a/src/PriceLevel.h b/src/PriceLevel.h index 4b740e1..34cfe4f 100644 --- a/src/PriceLevel.h +++ b/src/PriceLevel.h @@ -14,6 +14,50 @@ struct PriceLevel Order* head{ nullptr }; Order* tail{ nullptr }; std::size_t orderCount{ 0 }; + + void pushBack(Order* order) + { + order->prev = tail; + order->next = nullptr; + + if (tail != nullptr) + { + tail->next = order; + } + else + { + head = order; + } + + tail = order; + orderCount++; + totalQuantity += order->getRemainingQuantity(); + } + + void unlink(Order* order) + { + if (order->prev != nullptr) + { + order->prev->next = order->next; + } + else + { + head = order->next; + } + + if (order->next != nullptr) + { + order->next->prev = order->prev; + } + else + { + tail = order->prev; + } + + order->prev = nullptr; + order->next = nullptr; + orderCount--; + } }; } diff --git a/src/TscClock.h b/src/TscClock.h new file mode 100644 index 0000000..3a9728a --- /dev/null +++ b/src/TscClock.h @@ -0,0 +1,95 @@ +#pragma once + +#include +#include + +#include "Types.h" + +#if defined(_MSC_VER) +#include +#endif + +namespace orderbook { + +namespace detail { + +#if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__) + +inline std::uint64_t readTsc() noexcept +{ +#if defined(_MSC_VER) + return __rdtsc(); +#else + return __builtin_ia32_rdtsc(); +#endif +} + +struct TscCalibration +{ + std::uint64_t baseCycles; + double nsPerCycle; + std::chrono::nanoseconds baseEpoch; +}; + +// This is inline so every file that includes this header shares the same +// static variable. That means one calibration for the whole program, not a +// different one in every file. +inline const TscCalibration& tscCalibration() +{ + static const TscCalibration calibration = [] + { + const auto wallStart = + std::chrono::time_point_cast(std::chrono::system_clock::now()); + const std::uint64_t cyclesStart = readTsc(); + + auto wallNow = wallStart; + std::uint64_t cyclesNow = cyclesStart; + do + { + wallNow = std::chrono::time_point_cast( + std::chrono::system_clock::now()); + cyclesNow = readTsc(); + } while (wallNow - wallStart < std::chrono::milliseconds{ 1 }); + + const double elapsedNs = + static_cast((wallNow - wallStart).count()); + const double nsPerCycle = elapsedNs / static_cast(cyclesNow - cyclesStart); + + return TscCalibration{ cyclesStart, nsPerCycle, wallStart.time_since_epoch() }; + }(); + + return calibration; +} + +#endif + +} // namespace detail + +#if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__) + +// Calibration runs once per process and costs about 1ms. After that, every +// call is just a TSC read and a multiply, not a clock_gettime syscall. This +// needs an invariant TSC (constant_tsc, nonstop_tsc) to stay correct across +// cores. Every x86-64 CPU built in the last ten years or so has one. +inline Timestamp tscNow() noexcept +{ + const detail::TscCalibration& calibration = detail::tscCalibration(); + const std::uint64_t elapsedCycles = detail::readTsc() - calibration.baseCycles; + const auto elapsedNs = + static_cast(static_cast(elapsedCycles) * calibration.nsPerCycle); + + return Timestamp{ calibration.baseEpoch + std::chrono::nanoseconds{ elapsedNs } }; +} + +#else + +inline Timestamp tscNow() noexcept +{ + const auto now = + std::chrono::time_point_cast(std::chrono::system_clock::now()); + return Timestamp{ now.time_since_epoch() }; +} + +#endif + +} // namespace orderbook diff --git a/src/Version.h b/src/Version.h index ef851ce..1645b32 100644 --- a/src/Version.h +++ b/src/Version.h @@ -1,10 +1,10 @@ #pragma once -#define ORDERBOOK_VERSION_MAJOR 2 -#define ORDERBOOK_VERSION_MINOR 3 +#define ORDERBOOK_VERSION_MAJOR 3 +#define ORDERBOOK_VERSION_MINOR 0 #define ORDERBOOK_VERSION_PATCH 0 -#define ORDERBOOK_VERSION_STRING "2.3.0" +#define ORDERBOOK_VERSION_STRING "3.0.0" #define ORDERBOOK_VERSION \ (ORDERBOOK_VERSION_MAJOR * 10000 + ORDERBOOK_VERSION_MINOR * 100 + ORDERBOOK_VERSION_PATCH) diff --git a/src/lockfree/ConcurrentQueue.h b/src/lockfree/ConcurrentQueue.h new file mode 100644 index 0000000..fdc0751 --- /dev/null +++ b/src/lockfree/ConcurrentQueue.h @@ -0,0 +1,162 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace orderbook +{ + +// A bounded MPMC ring buffer. It uses Vyukov's design of a sequence number +// per cell. Each cell's own sequence number is what makes it safe, not a lock. +template +class ConcurrentQueue +{ + // Capacity must be a power of two so `pos & Mask` is a cheap wraparound. + // A capacity of 1 breaks the lap math, because "just filled" and "next + // lap" end up as the same number. + static_assert(Capacity >= 2 && (Capacity & (Capacity - 1)) == 0, + "ConcurrentQueue capacity must be a power of two >= 2"); + + public: + ConcurrentQueue() + { + for (std::size_t i = 0; i < Capacity; ++i) + { + cells_[i].sequence.store(i, std::memory_order_relaxed); + } + } + + ~ConcurrentQueue() + { + // tryPop needs a T& to move into, and T may have no default + // constructor. So we destroy what is left directly instead. + std::size_t head = dequeuePos_.load(std::memory_order_relaxed); + const std::size_t tail = enqueuePos_.load(std::memory_order_relaxed); + for (; head != tail; ++head) + { + cells_[head & Mask].storage.value.~T(); + } + } + + ConcurrentQueue(const ConcurrentQueue&) = delete; + ConcurrentQueue& operator=(const ConcurrentQueue&) = delete; + ConcurrentQueue(ConcurrentQueue&&) = delete; + ConcurrentQueue& operator=(ConcurrentQueue&&) = delete; + + // This takes T&& instead of T by value. A failed push should leave the + // caller's object alone, but passing by value would move construct it + // regardless. + [[nodiscard]] bool tryPush(T&& value) + { + std::size_t pos = enqueuePos_.load(std::memory_order_relaxed); + for (;;) + { + Cell& cell = cells_[pos & Mask]; + + // Acquire pairs with the release stores below and in tryPop. + // It guarantees we see whatever the last thread wrote here. + const std::size_t seq = cell.sequence.load(std::memory_order_acquire); + const auto diff = static_cast(seq) - static_cast(pos); + + if (diff == 0) + { + // enqueuePos_ only tracks whose turn is next. It does not + // track data, so relaxed is fine here. The acquire and + // release pair on sequence is what actually protects the + // storage. + if (enqueuePos_.compare_exchange_weak(pos, pos + 1, std::memory_order_relaxed)) + { + new (&cell.storage.value) T(std::move(value)); + cell.sequence.store(pos + 1, std::memory_order_release); + return true; + } + } + else if (diff < 0) + { + // Nobody has freed this slot yet, so the queue is full. + return false; + } + else + { + // Someone else already took this turn. Our pos is stale, + // so reload it and try again. + pos = enqueuePos_.load(std::memory_order_relaxed); + } + } + } + + [[nodiscard]] bool tryPop(T& out) + { + std::size_t pos = dequeuePos_.load(std::memory_order_relaxed); + for (;;) + { + Cell& cell = cells_[pos & Mask]; + const std::size_t seq = cell.sequence.load(std::memory_order_acquire); + + // A slot can only be popped after a producer fills it, one lap + // ahead of empty. That is the + 1 that tryPush does not have. + const auto diff = static_cast(seq) - static_cast(pos + 1); + + if (diff == 0) + { + if (dequeuePos_.compare_exchange_weak(pos, pos + 1, std::memory_order_relaxed)) + { + out = std::move(cell.storage.value); + cell.storage.value.~T(); + + // This republishes the slot as pos + Capacity, not + // pos + 1. That is the number this same slot will need + // one lap from now. + cell.sequence.store(pos + Capacity, std::memory_order_release); + return true; + } + } + else if (diff < 0) + { + // Nothing has filled this slot yet, so the queue is empty. + return false; + } + else + { + pos = dequeuePos_.load(std::memory_order_relaxed); + } + } + } + + private: + static constexpr std::size_t Mask = Capacity - 1; + + struct Cell + { + std::atomic sequence; + + // This is a union instead of a plain T member because Order has no + // default constructor and cannot be copied. OrderSlab's Slot union + // solves the same problem the same way. + union Storage + { + T value; + + Storage() {} + Storage(const Storage&) = delete; + Storage& operator=(const Storage&) = delete; + Storage(Storage&&) = delete; + Storage& operator=(Storage&&) = delete; + ~Storage() {} + } storage; + }; + + // alignas(64) keeps enqueuePos_ and dequeuePos_ off the same cache line. + // If they shared one, producer and consumer threads would fight over + // that line on every push and pop, even though the two counters have + // nothing to do with each other. + alignas(64) std::atomic enqueuePos_{0}; + alignas(64) std::atomic dequeuePos_{0}; + std::array cells_{}; +}; + +} // namespace orderbook diff --git a/tests/concurrent_queue_tests.cpp b/tests/concurrent_queue_tests.cpp new file mode 100644 index 0000000..b061605 --- /dev/null +++ b/tests/concurrent_queue_tests.cpp @@ -0,0 +1,269 @@ +#include +#include +#include +#include +#include + +#include + +#include "Order.h" +#include "lockfree/ConcurrentQueue.h" + +using namespace orderbook; + +namespace +{ + +struct Tracked +{ + int value; + int* dtorCount; + + Tracked(int v, int* counter) : value(v), dtorCount(counter) {} + Tracked(const Tracked&) = delete; + Tracked& operator=(const Tracked&) = delete; + + // A defaulted move would copy dtorCount without clearing the source. + // The object it moved from would then double count when it gets + // destroyed later. This nulls dtorCount out instead, so only the + // current owner ever counts. + Tracked(Tracked&& other) noexcept : value(other.value), dtorCount(other.dtorCount) + { + other.dtorCount = nullptr; + } + Tracked& operator=(Tracked&& other) noexcept + { + value = other.value; + dtorCount = other.dtorCount; + other.dtorCount = nullptr; + return *this; + } + + ~Tracked() + { + if (dtorCount != nullptr) + { + ++(*dtorCount); + } + } +}; + +} // namespace + +TEST(ConcurrentQueue, PushThenPopReturnsSameValue) +{ + ConcurrentQueue queue; + int out = 0; + + EXPECT_TRUE(queue.tryPush(42)); + EXPECT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out, 42); +} + +TEST(ConcurrentQueue, PopOnEmptyReturnsFalse) +{ + ConcurrentQueue queue; + int out = 0; + + EXPECT_FALSE(queue.tryPop(out)); +} + +TEST(ConcurrentQueue, PushToCapacityThenOneMoreFails) +{ + ConcurrentQueue queue; + + EXPECT_TRUE(queue.tryPush(1)); + EXPECT_TRUE(queue.tryPush(2)); + EXPECT_TRUE(queue.tryPush(3)); + EXPECT_TRUE(queue.tryPush(4)); + + // No pop has happened, so every slot is still on its first lap. + EXPECT_FALSE(queue.tryPush(5)); +} + +TEST(ConcurrentQueue, PreservesFifoOrder) +{ + ConcurrentQueue queue; + + EXPECT_TRUE(queue.tryPush(10)); + EXPECT_TRUE(queue.tryPush(20)); + EXPECT_TRUE(queue.tryPush(30)); + + int out = 0; + EXPECT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out, 10); + EXPECT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out, 20); + EXPECT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out, 30); +} + +TEST(ConcurrentQueue, FailedPushLeavesTheCallersValueUntouched) +{ + // unique_ptr proves that tryPush only moves from its argument on + // success. A wrong implementation would null out ptr even though + // this push fails. + ConcurrentQueue, 2> queue; + + EXPECT_TRUE(queue.tryPush(std::make_unique(1))); + EXPECT_TRUE(queue.tryPush(std::make_unique(2))); + + std::unique_ptr ptr = std::make_unique(99); + EXPECT_FALSE(queue.tryPush(std::move(ptr))); + ASSERT_NE(ptr, nullptr); + EXPECT_EQ(*ptr, 99); +} + +TEST(ConcurrentQueue, WorksWithOrderItsActualMotivatingType) +{ + ConcurrentQueue queue; + + Order order = OrderBuilder{} + .id(OrderId{ 7 }) + .buy() + .goodTillCancel() + .price(Price{ 100 }) + .quantity(Quantity{ 5 }) + .build(); + + EXPECT_TRUE(queue.tryPush(std::move(order))); + + Order out = OrderBuilder{} + .id(OrderId{ 0 }) + .buy() + .goodTillCancel() + .price(Price{ 1 }) + .quantity(Quantity{ 1 }) + .build(); + EXPECT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out.getOrderId(), OrderId{ 7 }); + EXPECT_EQ(out.getPrice(), Price{ 100 }); + EXPECT_EQ(out.getRemainingQuantity(), Quantity{ 5 }); +} + +TEST(ConcurrentQueue, DestructorDrainsWhateverIsStillInTheQueue) +{ + int dtorCount = 0; + + { + ConcurrentQueue queue; + EXPECT_TRUE(queue.tryPush(Tracked{ 1, &dtorCount })); + EXPECT_TRUE(queue.tryPush(Tracked{ 2, &dtorCount })); + EXPECT_TRUE(queue.tryPush(Tracked{ 3, &dtorCount })); + + // This only pops one. The other two are still live when the queue + // is destroyed below, so the destructor's drain has to clean them + // up. + Tracked out{ 0, nullptr }; + EXPECT_TRUE(queue.tryPop(out)); + } + + EXPECT_EQ(dtorCount, 3); +} + +TEST(ConcurrentQueue, WrapsAroundCorrectlyAcrossManyLaps) +{ + ConcurrentQueue queue; + + // A bug in the lap count only shows up once a slot gets reused, not on + // the first pass. 50 laps forces every slot through many reuses. + for (int lap = 0; lap < 50; ++lap) + { + for (int i = 0; i < 4; ++i) + { + ASSERT_TRUE(queue.tryPush(lap * 100 + i)) << "lap " << lap << " push " << i; + } + for (int i = 0; i < 4; ++i) + { + int out = 0; + ASSERT_TRUE(queue.tryPop(out)) << "lap " << lap << " pop " << i; + ASSERT_EQ(out, lap * 100 + i) << "lap " << lap << " pop " << i; + } + } +} + +TEST(ConcurrentQueue, HandlesFillDrainPatternsNotAlignedToCapacity) +{ + ConcurrentQueue queue; + + for (int i = 0; i < 4; ++i) + { + ASSERT_TRUE(queue.tryPush(int{ i })); + } + + int out = 0; + ASSERT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out, 0); + ASSERT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out, 1); + + // These two land on an offset that is not a multiple of capacity. + ASSERT_TRUE(queue.tryPush(4)); + ASSERT_TRUE(queue.tryPush(5)); + + for (int expected : { 2, 3, 4, 5 }) + { + ASSERT_TRUE(queue.tryPop(out)); + EXPECT_EQ(out, expected); + } + EXPECT_FALSE(queue.tryPop(out)); +} + +TEST(ConcurrentQueue, ManyProducersOneConsumerSeeEveryValueExactlyOnce) +{ + constexpr int ProducerCount = 4; + constexpr int PerProducer = 50000; + constexpr int Total = ProducerCount * PerProducer; + + ConcurrentQueue queue; + std::vector> seenCount(Total); + for (auto& counter : seenCount) + { + counter.store(0, std::memory_order_relaxed); + } + + std::vector producers; + for (int p = 0; p < ProducerCount; ++p) + { + producers.emplace_back( + [p, &queue] + { + for (int i = 0; i < PerProducer; ++i) + { + // Each producer owns a disjoint range of values. So + // seeing one unique value proves it was neither dropped + // nor duplicated. + const int value = p * PerProducer + i; + while (!queue.tryPush(int{ value })) + { + } + } + }); + } + + std::thread consumer( + [&] + { + int received = 0; + while (received < Total) + { + int value = 0; + if (queue.tryPop(value)) + { + seenCount[static_cast(value)].fetch_add(1, std::memory_order_relaxed); + ++received; + } + } + }); + + for (auto& producer : producers) + { + producer.join(); + } + consumer.join(); + + for (int i = 0; i < Total; ++i) + { + ASSERT_EQ(seenCount[static_cast(i)].load(), 1) << "value " << i; + } +}