From 72ca69163f2fb21d40f7bc681876134650a4aa67 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:44:27 +0900 Subject: [PATCH 01/16] Implement N-gram indexer --- include/iris/ngram.hpp | 163 +++++++++++++++++++++++++++++++++++++++++ iris.natvis | 15 ++++ test/CMakeLists.txt | 1 + test/ngram.cpp | 124 +++++++++++++++++++++++++++++++ 4 files changed, 303 insertions(+) create mode 100644 include/iris/ngram.hpp create mode 100644 test/ngram.cpp diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp new file mode 100644 index 0000000..f5ecadb --- /dev/null +++ b/include/iris/ngram.hpp @@ -0,0 +1,163 @@ +#ifndef IRIS_NGRAM_HPP +#define IRIS_NGRAM_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace iris { + +enum struct ngram_document_id : unsigned {}; + +} // iris + +template +struct std::formatter + : std::formatter, CharT> +{ + using base_type = std::formatter, CharT>; + + template + Ctx::iterator format(iris::ngram_document_id doc_id, Ctx& ctx) const + { + return base_type::format(std::to_underlying(doc_id), ctx); + } +}; + +namespace iris { + +struct ngram_occurrence +{ + ngram_document_id doc_id; + int pos; + + [[nodiscard]] constexpr bool operator==(ngram_occurrence const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram_occurrence const&) const noexcept = default; +}; + +template +struct ngram +{ + // TODO: optimize for N=1 + std::array chars; + + [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; +}; + +inline namespace ngram_literals { + +[[nodiscard]] constexpr ngram_document_id operator ""_doc_id(unsigned long long id) noexcept +{ + return ngram_document_id{static_cast>(id)}; +} + +[[nodiscard]] constexpr auto operator ""_2gram(char32_t const* str, std::size_t len) noexcept +{ + assert(len == 2); + return ngram<2, char32_t>{str[0], str[1]}; +} + +} // ngram_literals + + +namespace detail { + +template +struct ngram_index +{ + void append(ngram ng, ngram_document_id doc_id, int pos) + { + occs[ng].emplace_back(doc_id, pos); + } + + std::flat_map, std::vector> occs; +}; + +template +struct ngram_index_storage +{ + ngram_index<1, CharT> uni_idx; + ngram_index<2, CharT> bi_idx; +}; + +} // detail + +template +class ngram_database +{ +public: + [[nodiscard]] + ngram_document_id add_document(std::basic_string_view const doc_text) + { + ngram_document_id const doc_id{max_doc_id_}; + max_doc_id_ = ngram_document_id{std::to_underlying(max_doc_id_) + 1u}; + + for (std::size_t i = 0; i < doc_text.size(); ++i) { + store_.uni_idx.append(ngram<1, CharT>{doc_text[i]}, doc_id, int(i)); + } + + auto const do_ngram = [&](detail::ngram_index& idx) { + if (doc_text.size() < N) return; + + ngram ng; + std::size_t i = 0; + for (; i < N; ++i) { // TODO: loop unroll + ng.chars[i] = doc_text[i]; + } + idx.append(ng, doc_id, 0); + + for (; i < doc_text.size(); ++i) { + std::ranges::shift_left(ng.chars, 1); + ng.chars[N - 1] = doc_text[i]; + idx.append(ng, doc_id, int(i - N + 1)); + } + }; + do_ngram(get_index<2>()); + + return doc_id; + } + + template + [[nodiscard]] + std::vector const* get_occurrences(ngram ng) const noexcept + { + auto const& idx = get_index(); + auto const it = idx.occs.find(ng); + if (it == idx.occs.end()) return nullptr; + return &it->second; + } + +private: + template + [[nodiscard]] auto& get_index(this auto& self) noexcept + { + if constexpr (N == 1) { + return self.store_.uni_idx; + } else if constexpr (N == 2) { + return self.store_.bi_idx; + } else { + static_assert(false, "unhandled N"); + } + } + + ngram_document_id max_doc_id_{0_doc_id}; + detail::ngram_index_storage store_; +}; + + +} // iris + +#endif diff --git a/iris.natvis b/iris.natvis index b928d79..76f86a6 100644 --- a/iris.natvis +++ b/iris.natvis @@ -252,4 +252,19 @@ (int)index_ + + + + + {chars._Elems,na1} + + + {chars._Elems,na2} + + + {chars._Elems,na3} + + + {chars._Elems,na4} + diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6a46bc3..8d5d2b4 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -189,6 +189,7 @@ if(PROJECT_IS_TOP_LEVEL) interval_algo interval_set snippet + ngram ) foreach(test_name IN LISTS IRIS_TEST_IRIS_TESTS) diff --git a/test/ngram.cpp b/test/ngram.cpp new file mode 100644 index 0000000..e722d9d --- /dev/null +++ b/test/ngram.cpp @@ -0,0 +1,124 @@ +#include "iris_test.hpp" + +#include + +#include +#include +#include + +#ifdef _MSC_VER +# include +#endif + +namespace iris { + +inline std::ostream& operator<<(std::ostream& os, iris::ngram_occurrence const& occ) +{ + return os << std::format("{}:{}", occ.doc_id, occ.pos); +} + +} // iris + +// -------------------------------------------------- + +using namespace iris::ngram_literals; +using iris::ngram_occurrence; + +[[nodiscard]] +constexpr auto make_occurrences(std::initializer_list occs) +{ + return std::vector{occs}; +} + +#define IRIS_CHECK_OCCURRENCE(ng_str, ...) do { \ + std::vector const* occs = nullptr; \ + CHECK((occs = ngram_db.get_occurrences(U ## ng_str ## _2gram))); \ + if (occs) { \ + CHECK(*occs == make_occurrences({__VA_ARGS__})); \ + } \ + } while (false); + +TEST_CASE("ngram") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + // https://gihyo.jp/dev/serial/01/make-findspot/0005 + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日は良い天気です。"); + + IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("日は", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("は良", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("良い", {0_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("い天", {0_doc_id, 4}); + IRIS_CHECK_OCCURRENCE("天気", {0_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("気で", {0_doc_id, 6}); + IRIS_CHECK_OCCURRENCE("です", {0_doc_id, 7}); + IRIS_CHECK_OCCURRENCE("す。", {0_doc_id, 8}); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日は大雨です。"); + + IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("日は", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("は大", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("大雨", {0_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("雨で", {0_doc_id, 4}); + IRIS_CHECK_OCCURRENCE("です", {0_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("す。", {0_doc_id, 6}); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日の東海地方は大雨でしょう。"); + + IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("日の", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("の東", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("東海", {0_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("海地", {0_doc_id, 4}); + IRIS_CHECK_OCCURRENCE("地方", {0_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("方は", {0_doc_id, 6}); + IRIS_CHECK_OCCURRENCE("は大", {0_doc_id, 7}); + IRIS_CHECK_OCCURRENCE("大雨", {0_doc_id, 8}); + IRIS_CHECK_OCCURRENCE("雨で", {0_doc_id, 9}); + IRIS_CHECK_OCCURRENCE("でし", {0_doc_id, 10}); + IRIS_CHECK_OCCURRENCE("しょ", {0_doc_id, 11}); + IRIS_CHECK_OCCURRENCE("ょう", {0_doc_id, 12}); + IRIS_CHECK_OCCURRENCE("う。", {0_doc_id, 13}); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日は良い天気です。"); + (void)ngram_db.add_document(U"今日は大雨です。"); + (void)ngram_db.add_document(U"今日の東海地方は大雨でしょう。"); + + IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}, {1_doc_id, 0}, {2_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("日は", {0_doc_id, 1}, {1_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("は良", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("良い", {0_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("い天", {0_doc_id, 4}); + IRIS_CHECK_OCCURRENCE("天気", {0_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("気で", {0_doc_id, 6}); + IRIS_CHECK_OCCURRENCE("です", {0_doc_id, 7}, {1_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("す。", {0_doc_id, 8}, {1_doc_id, 6}); + IRIS_CHECK_OCCURRENCE("は大", {1_doc_id, 2}, {2_doc_id, 7}); + IRIS_CHECK_OCCURRENCE("大雨", {1_doc_id, 3}, {2_doc_id, 8}); + IRIS_CHECK_OCCURRENCE("雨で", {1_doc_id, 4}, {2_doc_id, 9}); + IRIS_CHECK_OCCURRENCE("日の", {2_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("の東", {2_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("東海", {2_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("海地", {2_doc_id, 4}); + IRIS_CHECK_OCCURRENCE("地方", {2_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("方は", {2_doc_id, 6}); + IRIS_CHECK_OCCURRENCE("でし", {2_doc_id, 10}); + IRIS_CHECK_OCCURRENCE("しょ", {2_doc_id, 11}); + IRIS_CHECK_OCCURRENCE("ょう", {2_doc_id, 12}); + IRIS_CHECK_OCCURRENCE("う。", {2_doc_id, 13}); + } +} From d84b60f494f8de8af048aa032b0cd0eaaed8af55 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:55:28 +0900 Subject: [PATCH 02/16] std::ranges::shift_left IS NOT IMPLEMENTED --- include/iris/ngram.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index f5ecadb..c6b9d0d 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -120,7 +120,7 @@ class ngram_database idx.append(ng, doc_id, 0); for (; i < doc_text.size(); ++i) { - std::ranges::shift_left(ng.chars, 1); + std::shift_left(ng.chars.begin(), ng.chars.end(), 1); ng.chars[N - 1] = doc_text[i]; idx.append(ng, doc_id, int(i - N + 1)); } From a1d5d6bd915313cdc45cf31ffac390bff9f19d0f Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:41:31 +0900 Subject: [PATCH 03/16] Add more basic tests --- include/iris/ngram.hpp | 44 +++++++++++++++++++++++++++++++++++++++--- test/ngram.cpp | 44 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index c6b9d0d..88c7ab3 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -29,7 +29,7 @@ struct std::formatter { using base_type = std::formatter, CharT>; - template + template Ctx::iterator format(iris::ngram_document_id doc_id, Ctx& ctx) const { return base_type::format(std::to_underlying(doc_id), ctx); @@ -38,6 +38,12 @@ struct std::formatter namespace iris { +namespace detail { + +inline constexpr std::size_t N_GRAM_MAX_OPTIMIZED_N = 2; + +} // detail + struct ngram_occurrence { ngram_document_id doc_id; @@ -50,9 +56,28 @@ struct ngram_occurrence template struct ngram { + static_assert(N >= 1); + // TODO: optimize for N=1 std::array chars; + template + [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept + { + assert(chars[Len - 1] == static_cast(0)); + + if constexpr (N == 1) { + return ngram{chars[0]}; + } else if constexpr (N == 2) { + return ngram{chars[0], chars[1]}; + } else { + static_assert(detail::N_GRAM_MAX_OPTIMIZED_N == 2); + ngram ng; + std::ranges::copy_n(chars, Len - 1, ng.chars.begin()); + return ng; + } + } + [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; }; @@ -70,8 +95,20 @@ inline namespace ngram_literals { return ngram<2, char32_t>{str[0], str[1]}; } +[[nodiscard]] constexpr auto operator ""_1gram(char32_t const* str, std::size_t len) noexcept +{ + assert(len == 1); + return ngram<1, char32_t>{str[0]}; +} + } // ngram_literals +template +[[nodiscard]] ngram to_ngram(CharT const (&chars)[N]) noexcept +{ + return ngram::from_c_array(chars); +} + namespace detail { @@ -114,7 +151,7 @@ class ngram_database ngram ng; std::size_t i = 0; - for (; i < N; ++i) { // TODO: loop unroll + for (; i < N; ++i) { ng.chars[i] = doc_text[i]; } idx.append(ng, doc_id, 0); @@ -126,6 +163,7 @@ class ngram_database } }; do_ngram(get_index<2>()); + static_assert(detail::N_GRAM_MAX_OPTIMIZED_N == 2); return doc_id; } @@ -149,7 +187,7 @@ class ngram_database } else if constexpr (N == 2) { return self.store_.bi_idx; } else { - static_assert(false, "unhandled N"); + static_assert(detail::N_GRAM_MAX_OPTIMIZED_N == 2); } } diff --git a/test/ngram.cpp b/test/ngram.cpp index e722d9d..0c51e41 100644 --- a/test/ngram.cpp +++ b/test/ngram.cpp @@ -30,15 +30,55 @@ constexpr auto make_occurrences(std::initializer_list occs) return std::vector{occs}; } +#define IRIS_CHECK_NO_OCCURRENCE(ng_str) do { \ + CHECK(!ngram_db.get_occurrences(iris::to_ngram(U ## ng_str))); \ + } while (false); + #define IRIS_CHECK_OCCURRENCE(ng_str, ...) do { \ std::vector const* occs = nullptr; \ - CHECK((occs = ngram_db.get_occurrences(U ## ng_str ## _2gram))); \ + CHECK((occs = ngram_db.get_occurrences(iris::to_ngram(U ## ng_str)))); \ if (occs) { \ CHECK(*occs == make_occurrences({__VA_ARGS__})); \ } \ } while (false); -TEST_CASE("ngram") +TEST_CASE("ngram (minimal input)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U""); + IRIS_CHECK_NO_OCCURRENCE("今"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今"); + IRIS_CHECK_OCCURRENCE("今", {0_doc_id, 0}); + IRIS_CHECK_NO_OCCURRENCE("無"); + IRIS_CHECK_NO_OCCURRENCE("今日"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今"); + IRIS_CHECK_OCCURRENCE("今", {0_doc_id, 0}); + IRIS_CHECK_NO_OCCURRENCE("無"); + IRIS_CHECK_NO_OCCURRENCE("今日"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日"); + IRIS_CHECK_OCCURRENCE("今", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("日", {0_doc_id, 1}); + IRIS_CHECK_NO_OCCURRENCE("無"); + IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); + IRIS_CHECK_NO_OCCURRENCE("今無"); + } +} + +TEST_CASE("ngram (realistic input)") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); From c43b05dfd49009b6b9dedee71fa85befb5dc8682 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:47:38 +0900 Subject: [PATCH 04/16] Organize formatters --- include/iris/ngram.hpp | 34 +++++++++++++++++++++++----------- test/ngram.cpp | 6 +++--- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index 88c7ab3..61fee8f 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -1,7 +1,8 @@ #ifndef IRIS_NGRAM_HPP #define IRIS_NGRAM_HPP -#include +#include + #include #include #include @@ -21,6 +22,15 @@ namespace iris { enum struct ngram_document_id : unsigned {}; +struct ngram_occurrence +{ + ngram_document_id doc_id; + int pos; + + [[nodiscard]] constexpr bool operator==(ngram_occurrence const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram_occurrence const&) const noexcept = default; +}; + } // iris template @@ -36,6 +46,17 @@ struct std::formatter } }; +template +struct std::formatter + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram_occurrence const& occ, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{}:{}", occ.doc_id, occ.pos); + } +}; + namespace iris { namespace detail { @@ -44,15 +65,6 @@ inline constexpr std::size_t N_GRAM_MAX_OPTIMIZED_N = 2; } // detail -struct ngram_occurrence -{ - ngram_document_id doc_id; - int pos; - - [[nodiscard]] constexpr bool operator==(ngram_occurrence const&) const noexcept = default; - [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram_occurrence const&) const noexcept = default; -}; - template struct ngram { @@ -170,7 +182,7 @@ class ngram_database template [[nodiscard]] - std::vector const* get_occurrences(ngram ng) const noexcept + std::vector const* find_occurrences(ngram ng) const noexcept { auto const& idx = get_index(); auto const it = idx.occs.find(ng); diff --git a/test/ngram.cpp b/test/ngram.cpp index 0c51e41..88a237a 100644 --- a/test/ngram.cpp +++ b/test/ngram.cpp @@ -14,7 +14,7 @@ namespace iris { inline std::ostream& operator<<(std::ostream& os, iris::ngram_occurrence const& occ) { - return os << std::format("{}:{}", occ.doc_id, occ.pos); + return os << std::format("{}", occ); } } // iris @@ -31,12 +31,12 @@ constexpr auto make_occurrences(std::initializer_list occs) } #define IRIS_CHECK_NO_OCCURRENCE(ng_str) do { \ - CHECK(!ngram_db.get_occurrences(iris::to_ngram(U ## ng_str))); \ + CHECK(!ngram_db.find_occurrences(iris::to_ngram(U ## ng_str))); \ } while (false); #define IRIS_CHECK_OCCURRENCE(ng_str, ...) do { \ std::vector const* occs = nullptr; \ - CHECK((occs = ngram_db.get_occurrences(iris::to_ngram(U ## ng_str)))); \ + CHECK((occs = ngram_db.find_occurrences(iris::to_ngram(U ## ng_str)))); \ if (occs) { \ CHECK(*occs == make_occurrences({__VA_ARGS__})); \ } \ From 14f61eb6fb0ed41ed9eb139a78b0a0b88333b7e9 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Wed, 12 Aug 2026 04:35:46 +0900 Subject: [PATCH 05/16] Implement search functionality --- include/iris/format.hpp | 2 + include/iris/ngram.hpp | 475 +++++++++++++++++++++++++++++++++++----- test/CMakeLists.txt | 1 + test/ngram.cpp | 46 +++- 4 files changed, 468 insertions(+), 56 deletions(-) diff --git a/include/iris/format.hpp b/include/iris/format.hpp index 5539380..4c63876 100644 --- a/include/iris/format.hpp +++ b/include/iris/format.hpp @@ -3,6 +3,8 @@ // SPDX-License-Identifier: MIT +// SPDX-License-Identifier: MIT + #include #include diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index 61fee8f..513652b 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -1,8 +1,15 @@ #ifndef IRIS_NGRAM_HPP #define IRIS_NGRAM_HPP +// SPDX-License-Identifier: MIT + +#include +#include +#include +#include #include +#include #include #include #include @@ -12,9 +19,9 @@ #include #include #include -#include #include #include +#include #include @@ -73,23 +80,34 @@ struct ngram // TODO: optimize for N=1 std::array chars; - template - [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept + template + [[nodiscard]] static constexpr ngram from_copy_n(It it) + noexcept(noexcept(*it++)) { - assert(chars[Len - 1] == static_cast(0)); - if constexpr (N == 1) { - return ngram{chars[0]}; + ngram ng; + ng.chars[0] = *it; + return ng; } else if constexpr (N == 2) { - return ngram{chars[0], chars[1]}; + ngram ng; + ng.chars[0] = *it++; + ng.chars[1] = *it; + return ng; } else { static_assert(detail::N_GRAM_MAX_OPTIMIZED_N == 2); ngram ng; - std::ranges::copy_n(chars, Len - 1, ng.chars.begin()); + std::ranges::copy_n(it, N, ng.chars.begin()); return ng; } } + template + [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept + { + assert(chars[Len - 1] == static_cast(0)); + return ngram::from_copy_n(std::ranges::begin(chars)); + } + [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; }; @@ -104,12 +122,14 @@ inline namespace ngram_literals { [[nodiscard]] constexpr auto operator ""_2gram(char32_t const* str, std::size_t len) noexcept { assert(len == 2); + (void)len; return ngram<2, char32_t>{str[0], str[1]}; } [[nodiscard]] constexpr auto operator ""_1gram(char32_t const* str, std::size_t len) noexcept { assert(len == 1); + (void)len; return ngram<1, char32_t>{str[0]}; } @@ -124,82 +144,439 @@ template namespace detail { -template +enum struct [[nodiscard]] search_continuation : bool +{ + abort = false, + proceed = true, +}; + +struct ngram_posting +{ + ngram_document_id doc_id; + unsigned pos_offset = 0; + unsigned pos_count = 0; +}; + +struct ngram_posting_list +{ + std::vector postings; + std::vector positions; + + void append(ngram_document_id const doc_id, int pos) + { + if (postings.empty() || postings.back().doc_id != doc_id) { + if (!postings.empty() && postings.back().doc_id > doc_id) { + throw std::invalid_argument{"documents must be indexed in non-decreasing order of document ID"}; + } + postings.emplace_back( + doc_id, + static_cast(positions.size()), + 0 + ); + } + ++postings.back().pos_count; + positions.emplace_back(pos); + } + + void to_occurrence_list(std::vector& occs) const + { + occs.clear(); + for (auto const& posting : postings) { + for (std::size_t i = posting.pos_offset; i < posting.pos_offset + posting.pos_count; ++i) { + occs.emplace_back(posting.doc_id, positions[i]); + } + } + } + + template + void for_each_documents(F&& f) const + { + static_assert(std::invocable>); + + constexpr bool f_returns_continuation = std::same_as< + std::invoke_result_t>, + search_continuation + >; + + for (auto const& posting : postings) { + std::span const posting_span{ + positions.begin() + posting.pos_offset, + static_cast(posting.pos_count) + }; + + if constexpr (f_returns_continuation) { + search_continuation const cont = f(posting.doc_id, posting_span); + if (cont == search_continuation::abort) break; + } else { + f(posting.doc_id, posting_span); + } + } + } +}; + +template struct ngram_index { - void append(ngram ng, ngram_document_id doc_id, int pos) + void append(ngram const ng, ngram_document_id const doc_id, int const pos) + { + gram_entries[ng].append(doc_id, pos); + } + + [[nodiscard]] + bool empty() const noexcept + { + return gram_entries.empty(); + } + + void find_occurrences(ngram const ng, std::vector& occs) const { - occs[ng].emplace_back(doc_id, pos); + occs.clear(); + auto const it = gram_entries.find(ng); + if (it == gram_entries.end()) return; + + it->second.to_occurrence_list(occs); } - std::flat_map, std::vector> occs; + template + void search(ngram const ng, F&& f) const + { + auto const it = gram_entries.find(ng); + if (it == gram_entries.end()) return; + it->second.for_each_documents(f); + } + + std::flat_map, PostingListT> gram_entries; }; -template + +template struct ngram_index_storage { - ngram_index<1, CharT> uni_idx; - ngram_index<2, CharT> bi_idx; + ngram_index<1, CharT, PostingListT> uni_idx; + ngram_index<2, CharT, PostingListT> bi_idx; + + template + [[nodiscard]] auto& get_index(this auto& self) noexcept + { + if constexpr (N == 1) { + return self.uni_idx; + } else if constexpr (N == 2) { + return self.bi_idx; + } else { + static_assert(N_GRAM_MAX_OPTIMIZED_N == 2); + } + } + + void append_index(ngram_document_id const doc_id, std::basic_string_view const input) + { + this->template append_index<1>(doc_id, this->template get_index<1>(), input); + this->template append_index<2>(doc_id, this->template get_index<2>(), input); + } + + [[nodiscard]] + bool empty() const noexcept + { + return uni_idx.empty() && bi_idx.empty(); + } + + template + void search(ngram const ng, F&& f) const + { + this->template get_index().search(ng, f); + } + +private: + template + void append_index( + ngram_document_id const doc_id, + ngram_index& idx, + std::basic_string_view const input + ) + { + if (input.size() < N) return; + + if constexpr (N == 1) { + for (std::size_t i = 0; i < input.size(); ++i) { + idx.append(ngram<1, CharT>{input[i]}, doc_id, int(i)); + } + + } else { + auto ng = ngram::from_copy_n(input.begin()); + idx.append(ng, doc_id, 0); + + for (std::size_t i = N; i < input.size(); ++i) { + std::shift_left(ng.chars.begin(), ng.chars.end(), 1); + ng.chars[N - 1] = input[i]; + idx.append(ng, doc_id, int(i - N + 1)); + } + } + } }; } // detail + template -class ngram_database +struct ngram_search_query { + explicit ngram_search_query(std::basic_string_view input_sv) + { + std::basic_string input{input_sv}; + iris::compact_spaces(input); + if (input.empty()) return; + + words_ = input + | std::views::split(detail::string_algo_traits::space) + | std::views::transform([](auto const& r) { + return std::basic_string{std::from_range, r}; + }) + | std::ranges::to(); + + std::ranges::sort(words_); + { + auto const [first, last] = std::ranges::unique(words_); + words_.erase(first, last); + } + } + + // ------------------------------------------ + + [[nodiscard]] + auto const& words() const noexcept + { + return words_; + } + + [[nodiscard]] bool empty() const noexcept + { + return words_.empty(); + } + + [[nodiscard]] bool operator==(ngram_search_query const& other) const noexcept + { + return words_ == other.words_; + } + +private: + std::vector> words_; +}; + +} // iris + +template +struct std::formatter, CharT> + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram_search_query const& query, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{}", query.words() | std::views::transform([](std::u32string_view ustr) { + return iris::unicode::transcode(ustr); + })); + } +}; + + +namespace iris { + +class [[nodiscard]] ngram_search_result +{ + struct word_matches_t + { + int word_id = 0; + std::vector> matches; + }; + + using doc_matches_map = std::flat_map>; + + struct word_matches_handle + { + doc_matches_map::iterator map_it; + std::vector>* word_matches = nullptr; + + [[nodiscard]] + std::vector>* operator->() const noexcept + { + return word_matches; + } + + [[nodiscard]] explicit operator bool() const noexcept + { + return word_matches; + } + }; + public: [[nodiscard]] - ngram_document_id add_document(std::basic_string_view const doc_text) + bool has_document(ngram_document_id const doc_id) const noexcept { - ngram_document_id const doc_id{max_doc_id_}; - max_doc_id_ = ngram_document_id{std::to_underlying(max_doc_id_) + 1u}; + return doc_matches_.contains(doc_id); + } + + [[nodiscard]] + auto const& doc_matches() const noexcept { return doc_matches_; } - for (std::size_t i = 0; i < doc_text.size(); ++i) { - store_.uni_idx.append(ngram<1, CharT>{doc_text[i]}, doc_id, int(i)); + // Returns whether search must continue + template + [[nodiscard]] + bool init_word_matches(ngram_document_id const doc_id, int const word_id, std::span const positions) + { + auto doc_matches_it = doc_matches_.find(doc_id); + if (doc_matches_it == doc_matches_.end()) { + if (word_id != 0) return false; + doc_matches_it = doc_matches_.try_emplace(doc_id).first; } - auto const do_ngram = [&](detail::ngram_index& idx) { - if (doc_text.size() < N) return; + auto& word_matches = doc_matches_it->second.emplace_back(word_id); + word_matches.matches.assign_range(positions | std::views::transform([](int const pos) -> interval { + return {pos, pos + static_cast(N)}; + })); + return true; + } - ngram ng; - std::size_t i = 0; - for (; i < N; ++i) { - ng.chars[i] = doc_text[i]; - } - idx.append(ng, doc_id, 0); + [[nodiscard]] + word_matches_handle get_word_matches(ngram_document_id const doc_id, int const word_id) + { + auto const doc_matches_it = doc_matches_.find(doc_id); + if (doc_matches_it == doc_matches_.end()) return {}; - for (; i < doc_text.size(); ++i) { - std::shift_left(ng.chars.begin(), ng.chars.end(), 1); - ng.chars[N - 1] = doc_text[i]; - idx.append(ng, doc_id, int(i - N + 1)); - } - }; - do_ngram(get_index<2>()); - static_assert(detail::N_GRAM_MAX_OPTIMIZED_N == 2); + auto const it = std::ranges::find(doc_matches_it->second, word_id, &word_matches_t::word_id); + if (it == doc_matches_it->second.end()) return {}; + return {doc_matches_it, &it->matches}; + } + void erase_word_matches(word_matches_handle const& handle) + { + doc_matches_.erase(handle.map_it); + } + + void clear() noexcept + { + doc_matches_.clear(); + } + + [[nodiscard]] + bool empty() const noexcept + { + return doc_matches_.empty(); + } + + [[nodiscard]] + explicit operator bool() const noexcept + { + return !this->empty(); + } + +private: + doc_matches_map doc_matches_; +}; + +template +class ngram_database +{ +public: + [[nodiscard]] + ngram_document_id add_document(std::basic_string_view const doc_text) + { + ngram_document_id const doc_id{max_doc_id_}; + max_doc_id_ = ngram_document_id{std::to_underlying(max_doc_id_) + 1u}; + + store_.append_index(doc_id, doc_text); return doc_id; } template + void find_occurrences(ngram ng, std::vector& occs) const noexcept + { + occs.clear(); + auto const& idx = store_.template get_index(); + idx.find_occurrences(ng, occs); + } + [[nodiscard]] - std::vector const* find_occurrences(ngram ng) const noexcept + ngram_search_result search(ngram_search_query const& query) const { - auto const& idx = get_index(); - auto const it = idx.occs.find(ng); - if (it == idx.occs.end()) return nullptr; - return &it->second; + if (query.empty()) return {}; + if (store_.empty()) return {}; + + ngram_search_result search_res; + + int word_id = 0; + auto it = query.words().begin(); + assert(!it->empty()); + this->search_word(search_res, word_id++, *it++); + if (search_res.empty()) return search_res; + + for (; it != query.words().end(); ++it) { + assert(!it->empty()); + this->search_word(search_res, word_id++, *it); + if (search_res.empty()) break; + } + return search_res; } private: - template - [[nodiscard]] auto& get_index(this auto& self) noexcept + template + void search_word(ngram_search_result& search_res, int const word_id, std::basic_string_view const word) const { - if constexpr (N == 1) { - return self.store_.uni_idx; - } else if constexpr (N == 2) { - return self.store_.bi_idx; + if (word.empty()) return; + + if (word.size() == 1) { + this->search_word_impl(search_res, word_id, word); } else { - static_assert(detail::N_GRAM_MAX_OPTIMIZED_N == 2); + this->search_word_impl(search_res, word_id, word); + } + } + + template + void search_word_impl(ngram_search_result& search_res, int const word_id, std::basic_string_view const word) const + { + auto ng = ngram::from_copy_n(word.begin()); + + std::size_t available_doc_count = 0; + store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { + if (search_res.init_word_matches(doc_id, word_id, positions)) { + ++available_doc_count; + } + }); + + if constexpr (!IsFirstWord) { + if (available_doc_count == 0) { + search_res.clear(); + return; + } + } + + for (std::size_t i = N; i < word.size(); i += N) { + std::ranges::copy_n(word.begin() + i, N, ng.chars.begin()); + + store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { + auto word_matches = search_res.get_word_matches(doc_id, word_id); + if (!word_matches) return detail::search_continuation::proceed; + assert(!word_matches->empty()); + + for (auto it = word_matches->begin(); it != word_matches->end();) { + auto& prev_pos = *it; + + // TODO: make this binary search + if (std::ranges::any_of(positions, [prev_pos](int const pos) { + return pos == prev_pos.right; + })) { + // Matched; the current word's current n-gram is contiguous to the previous n-gram + prev_pos.right += N; + ++it; + continue; + } + + it = word_matches->erase(it); + } + + if (word_matches->empty()) { + search_res.erase_word_matches(word_matches); + if (search_res.empty()) return detail::search_continuation::abort; + } + return detail::search_continuation::proceed; + }); } } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8d5d2b4..590803b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -184,6 +184,7 @@ if(PROJECT_IS_TOP_LEVEL) indirect colorize_format preprocess + interval string_algo interval interval_algo diff --git a/test/ngram.cpp b/test/ngram.cpp index 88a237a..e96313e 100644 --- a/test/ngram.cpp +++ b/test/ngram.cpp @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: MIT + #include "iris_test.hpp" #include @@ -12,11 +14,17 @@ namespace iris { -inline std::ostream& operator<<(std::ostream& os, iris::ngram_occurrence const& occ) +inline std::ostream& operator<<(std::ostream& os, ngram_occurrence const& occ) { return os << std::format("{}", occ); } +template +inline std::ostream& operator<<(std::ostream& os, interval const& iv) +{ + return os << std::format("{}", iv); +} + } // iris // -------------------------------------------------- @@ -31,15 +39,15 @@ constexpr auto make_occurrences(std::initializer_list occs) } #define IRIS_CHECK_NO_OCCURRENCE(ng_str) do { \ - CHECK(!ngram_db.find_occurrences(iris::to_ngram(U ## ng_str))); \ + std::vector occs; \ + ngram_db.find_occurrences(iris::to_ngram(U ## ng_str), occs); \ + CHECK(occs.empty()); \ } while (false); #define IRIS_CHECK_OCCURRENCE(ng_str, ...) do { \ - std::vector const* occs = nullptr; \ - CHECK((occs = ngram_db.find_occurrences(iris::to_ngram(U ## ng_str)))); \ - if (occs) { \ - CHECK(*occs == make_occurrences({__VA_ARGS__})); \ - } \ + std::vector occs; \ + ngram_db.find_occurrences(iris::to_ngram(U ## ng_str), occs); \ + CHECK(occs == make_occurrences({__VA_ARGS__})); \ } while (false); TEST_CASE("ngram (minimal input)") @@ -162,3 +170,27 @@ TEST_CASE("ngram (realistic input)") IRIS_CHECK_OCCURRENCE("う。", {2_doc_id, 13}); } } + +TEST_CASE("ngram search") +{ + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日は良い天気です。"); + //(void)ngram_db.add_document(U"今日は大雨です。"); + //(void)ngram_db.add_document(U"今日の東海地方は大雨でしょう。"); + + iris::ngram_search_query<> query{U"良い天気"}; + + auto const search_res = ngram_db.search(query); + + auto const& doc_matches = search_res.doc_matches(); + + REQUIRE(doc_matches.contains(0_doc_id)); + auto const& word_map = doc_matches.at(0_doc_id); + + REQUIRE(word_map.size() == 1); + CHECK(word_map[0].word_id == 0); + REQUIRE(word_map[0].matches.size() == 1); + CHECK(word_map[0].matches[0] == iris::interval{3, 7}); + } +} From bbf227ae437608c92148377f8c93679aba16a68e Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Thu, 13 Aug 2026 04:55:09 +0900 Subject: [PATCH 06/16] Complete implementation for basic search functionality --- include/iris/ngram.hpp | 435 ++++++++++++++++++++-------- test/ngram.cpp | 624 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 894 insertions(+), 165 deletions(-) diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index 513652b..ec6c646 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -4,6 +4,7 @@ // SPDX-License-Identifier: MIT #include +#include #include #include #include @@ -38,34 +39,6 @@ struct ngram_occurrence [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram_occurrence const&) const noexcept = default; }; -} // iris - -template -struct std::formatter - : std::formatter, CharT> -{ - using base_type = std::formatter, CharT>; - - template - Ctx::iterator format(iris::ngram_document_id doc_id, Ctx& ctx) const - { - return base_type::format(std::to_underlying(doc_id), ctx); - } -}; - -template -struct std::formatter - : iris::no_spec_formatter -{ - template - Ctx::iterator format(iris::ngram_occurrence const& occ, Ctx& ctx) const - { - return std::format_to(ctx.out(), "{}:{}", occ.doc_id, occ.pos); - } -}; - -namespace iris { - namespace detail { inline constexpr std::size_t N_GRAM_MAX_OPTIMIZED_N = 2; @@ -248,68 +221,137 @@ struct ngram_index std::flat_map, PostingListT> gram_entries; }; +template +struct ngram_pos_t +{ + ngram ng; + int pos; + + [[nodiscard]] constexpr bool operator==(ngram_pos_t const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram_pos_t const&) const noexcept = default; +}; template struct ngram_index_storage { - ngram_index<1, CharT, PostingListT> uni_idx; - ngram_index<2, CharT, PostingListT> bi_idx; - - template - [[nodiscard]] auto& get_index(this auto& self) noexcept - { - if constexpr (N == 1) { - return self.uni_idx; - } else if constexpr (N == 2) { - return self.bi_idx; - } else { - static_assert(N_GRAM_MAX_OPTIMIZED_N == 2); - } - } - void append_index(ngram_document_id const doc_id, std::basic_string_view const input) { - this->template append_index<1>(doc_id, this->template get_index<1>(), input); - this->template append_index<2>(doc_id, this->template get_index<2>(), input); + this->template append_index<1>(doc_id, input); + this->template append_index<2>(doc_id, input); } [[nodiscard]] bool empty() const noexcept { - return uni_idx.empty() && bi_idx.empty(); + return + this->template get_data<1>().idx.empty() && + this->template get_data<2>().idx.empty(); } template void search(ngram const ng, F&& f) const { - this->template get_index().search(ng, f); + this->template get_data().idx.search(ng, f); + } + + template + [[nodiscard]] auto& get_index(this auto& self) noexcept IRIS_LIFETIMEBOUND + { + return self.template get_data().idx; } private: + template + struct ngram_index_storage_data + { + ngram_index idx; + + // Caches + std::vector, default_init_allocator>> + batch_grams; + + std::vector, PostingListT>> + batch_pending; + }; + + ngram_index_storage_data<1> uni_data_; + ngram_index_storage_data<2> bi_data_; + + template + [[nodiscard]] auto& get_data(this auto& self) noexcept IRIS_LIFETIMEBOUND + { + if constexpr (N == 1) { + return self.uni_data_; + } else if constexpr (N == 2) { + return self.bi_data_; + } else { + static_assert(N_GRAM_MAX_OPTIMIZED_N == 2); + } + } + template void append_index( ngram_document_id const doc_id, - ngram_index& idx, std::basic_string_view const input ) { if (input.size() < N) return; + ngram_index_storage_data& data = this->template get_data(); + + // Naive per-gram insertion into flat_map is expensive: each *new* key + // shifts the underlying vectors, so building an index of vocabulary + // size V costs O(V^2) overall. Instead, per document: + // + // 1. Collect grams+positions --- O(G) G = grams in this doc + // 2. Sort them ----------------- O(G log G) + // 3. Existing keys ------------- O(D log V) D = distinct grams (D <= G) + // 4. New keys ------------------ O(V + P) P = brand-new keys (P <= D) + // + // Once the vocabulary saturates (P ~ 0, typical after a few documents), + // step 4 is a no-op and each document costs only O(G log G + D log V). + // + // Note: initially implemented by @saki7, then the complexity math is + // double-checked by Claude. + + data.batch_grams.clear(); + data.batch_grams.resize(input.size() - N + 1); if constexpr (N == 1) { for (std::size_t i = 0; i < input.size(); ++i) { - idx.append(ngram<1, CharT>{input[i]}, doc_id, int(i)); + data.batch_grams[i].ng.chars[0] = input[i]; + data.batch_grams[i].pos = static_cast(i); } } else { - auto ng = ngram::from_copy_n(input.begin()); - idx.append(ng, doc_id, 0); + for (std::size_t i = 0; i + N <= input.size(); ++i) { + std::ranges::copy_n(input.begin() + i, N, data.batch_grams[i].ng.chars.begin()); + data.batch_grams[i].pos = static_cast(i); + } + } + std::ranges::sort(data.batch_grams); + + data.batch_pending.clear(); - for (std::size_t i = N; i < input.size(); ++i) { - std::shift_left(ng.chars.begin(), ng.chars.end(), 1); - ng.chars[N - 1] = input[i]; - idx.append(ng, doc_id, int(i - N + 1)); + for (auto const& chunk : data.batch_grams | std::views::chunk_by( + [](auto const& a, auto const& b) { return a.ng == b.ng; } + )) { + auto const& key = chunk.front().ng; + if (auto const it = data.idx.gram_entries.find(key); it != data.idx.gram_entries.end()) { + for (auto const& gp : chunk) { + it->second.append(doc_id, gp.pos); + } + } else { + auto& pl = data.batch_pending.emplace_back(key, PostingListT{}).second; + for (auto const& gp : chunk) { + pl.append(doc_id, gp.pos); + } } } + data.idx.gram_entries.insert( + std::sorted_unique, + std::make_move_iterator(data.batch_pending.begin()), + std::make_move_iterator(data.batch_pending.end()) + ); } }; @@ -361,48 +403,52 @@ struct ngram_search_query std::vector> words_; }; -} // iris +template +ngram_search_query(CharT const(&)[N]) -> ngram_search_query; -template -struct std::formatter, CharT> - : iris::no_spec_formatter + +struct [[nodiscard]] ngram_search_word_match { - template - Ctx::iterator format(iris::ngram_search_query const& query, Ctx& ctx) const + ngram_search_word_match() = default; + + explicit ngram_search_word_match(int word_id) + : word_id(word_id) + {} + + ngram_search_word_match(int word_id, std::initializer_list> spans) + : word_id(word_id) + , spans(spans) + {} + + int word_id = 0; + unsigned successful_ngrams = 1; // due to the class layout, this must be placed here + std::vector> spans; + + [[nodiscard]] + bool operator==(ngram_search_word_match const& other) const noexcept { - return std::format_to(ctx.out(), "{}", query.words() | std::views::transform([](std::u32string_view ustr) { - return iris::unicode::transcode(ustr); - })); + return word_id == other.word_id && spans == other.spans; } }; - -namespace iris { - class [[nodiscard]] ngram_search_result { - struct word_matches_t - { - int word_id = 0; - std::vector> matches; - }; - - using doc_matches_map = std::flat_map>; + using doc_matches_map = std::flat_map>; struct word_matches_handle { - doc_matches_map::iterator map_it; - std::vector>* word_matches = nullptr; + doc_matches_map::iterator doc_it; + ngram_search_word_match* word_match = nullptr; [[nodiscard]] - std::vector>* operator->() const noexcept + ngram_search_word_match* operator->() const noexcept { - return word_matches; + return word_match; } [[nodiscard]] explicit operator bool() const noexcept { - return word_matches; + return word_match; } }; @@ -417,18 +463,25 @@ class [[nodiscard]] ngram_search_result auto const& doc_matches() const noexcept { return doc_matches_; } // Returns whether search must continue - template + template [[nodiscard]] bool init_word_matches(ngram_document_id const doc_id, int const word_id, std::span const positions) { - auto doc_matches_it = doc_matches_.find(doc_id); - if (doc_matches_it == doc_matches_.end()) { - if (word_id != 0) return false; - doc_matches_it = doc_matches_.try_emplace(doc_id).first; + assert(!positions.empty()); + + doc_matches_map::iterator doc_matches_it; + if constexpr (IsFirstWord) { + assert(word_id == 0); + assert(doc_matches_.empty() || doc_matches_.rbegin()->first < doc_id); + doc_matches_it = doc_matches_.try_emplace(doc_matches_.end(), doc_id); // hint: append + } else { + doc_matches_it = doc_matches_.find(doc_id); + if (doc_matches_it == doc_matches_.end()) return false; // no new docs after word 0 } - auto& word_matches = doc_matches_it->second.emplace_back(word_id); - word_matches.matches.assign_range(positions | std::views::transform([](int const pos) -> interval { + assert(!std::ranges::contains(doc_matches_it->second, word_id, &ngram_search_word_match::word_id)); + auto& word_match = doc_matches_it->second.emplace_back(word_id); + word_match.spans.assign_range(positions | std::views::transform([](int const pos) -> interval { return {pos, pos + static_cast(N)}; })); return true; @@ -440,14 +493,52 @@ class [[nodiscard]] ngram_search_result auto const doc_matches_it = doc_matches_.find(doc_id); if (doc_matches_it == doc_matches_.end()) return {}; - auto const it = std::ranges::find(doc_matches_it->second, word_id, &word_matches_t::word_id); - if (it == doc_matches_it->second.end()) return {}; - return {doc_matches_it, &it->matches}; + // We don't need to do *full* `std::find` here; the word match is + // always inserted sequentially so if it exists, it is always placed + // at the *back* of the vector. + if ( + doc_matches_it->second.empty() || + doc_matches_it->second.back().word_id != word_id + ) { + assert( + doc_matches_it->second.empty() || + // Make sure the matching element does not exist at the position except for *back* + !std::ranges::contains(doc_matches_it->second, word_id, &ngram_search_word_match::word_id) + ); + return {}; + } + assert(!doc_matches_it->second.back().spans.empty()); + return {doc_matches_it, &doc_matches_it->second.back()}; + } + + void erase_document(word_matches_handle const& handle) + { + doc_matches_.erase(handle.doc_it); } - void erase_word_matches(word_matches_handle const& handle) + void remove_stale_document_matches(int const word_id, unsigned const expected_ngrams) { - doc_matches_.erase(handle.map_it); + auto [keys, values] = std::move(doc_matches_).extract(); + + std::size_t out = 0; + for (std::size_t in = 0; in < keys.size(); ++in) { + auto& word_matches = values[in]; + bool has_word = false; + std::erase_if(word_matches, [&](ngram_search_word_match const& wm) { + if (wm.word_id != word_id) return false; + has_word = true; + return wm.successful_ngrams != expected_ngrams; + }); + if (!has_word || word_matches.empty()) continue; + if (out != in) { + keys[out] = keys[in]; + values[out] = std::move(values[in]); + } + ++out; + } + keys.resize(out); + values.resize(out); + doc_matches_.replace(std::move(keys), std::move(values)); } void clear() noexcept @@ -519,7 +610,7 @@ class ngram_database template void search_word(ngram_search_result& search_res, int const word_id, std::basic_string_view const word) const { - if (word.empty()) return; + assert(!word.empty()); if (word.size() == 1) { this->search_word_impl(search_res, word_id, word); @@ -531,60 +622,168 @@ class ngram_database template void search_word_impl(ngram_search_result& search_res, int const word_id, std::basic_string_view const word) const { + assert(word.size() >= N); auto ng = ngram::from_copy_n(word.begin()); - std::size_t available_doc_count = 0; - store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { - if (search_res.init_word_matches(doc_id, word_id, positions)) { - ++available_doc_count; - } - }); + if constexpr (IsFirstWord) { + store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { + (void)search_res.init_word_matches(doc_id, word_id, positions); + }); + if (search_res.empty()) return; - if constexpr (!IsFirstWord) { + } else { + std::size_t available_doc_count = 0; + store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { + if (search_res.init_word_matches(doc_id, word_id, positions)) { + ++available_doc_count; + } + }); if (available_doc_count == 0) { search_res.clear(); return; } } - for (std::size_t i = N; i < word.size(); i += N) { - std::ranges::copy_n(word.begin() + i, N, ng.chars.begin()); - - store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { - auto word_matches = search_res.get_word_matches(doc_id, word_id); - if (!word_matches) return detail::search_continuation::proceed; - assert(!word_matches->empty()); + unsigned current_ngram = 1; + auto const do_search = [&](int remaining_chars) { + return [&, remaining_chars, overlapping_chars = int(N) - remaining_chars](ngram_document_id const doc_id, std::span const positions) { + // Find the existing match set from the previous iteration. + // If none exists, any subsequent characters of the document will not match. + // + // For example, when the document is "今日は晴れです" and current `ng` is "は晴", + // - When previous `ng` was "昨日", `search_res` contians no matches => omit further sequence + // - When previous `ng` was "今日", `search_res` contains matches => proceed with "は晴" + auto word_match = search_res.get_word_matches(doc_id, word_id); + if (!word_match) return detail::search_continuation::proceed; + + // Prevent *resurrecting* the false-positive match on "match -> unmatch -> match" pattern. + // For example, when the document is "abef" and the query is "abXXef", + // - ngram{"ab"} -> match (successful_ngrams = 1) + // - ngram{"XX"} -> no match (successful_ngrams is untouched) + // - ngram{"ef"} -> successful_ngrams does not match current_ngram! + if (word_match->successful_ngrams != current_ngram) { + search_res.erase_document(word_match); + if (search_res.empty()) return detail::search_continuation::abort; + return detail::search_continuation::proceed; + } - for (auto it = word_matches->begin(); it != word_matches->end();) { + // Find contiguous match; document has [previous ng, current ng] + for (auto it = word_match->spans.begin(); it != word_match->spans.end();) { auto& prev_pos = *it; - // TODO: make this binary search - if (std::ranges::any_of(positions, [prev_pos](int const pos) { - return pos == prev_pos.right; - })) { + if (std::ranges::binary_search(positions, prev_pos.right - overlapping_chars)) { // Matched; the current word's current n-gram is contiguous to the previous n-gram - prev_pos.right += N; + prev_pos.right += remaining_chars; ++it; continue; } - - it = word_matches->erase(it); + // Erase exiting match that indicates the below structure + // [previous ng, ...some unrelated chars..., current ng] + it = word_match->spans.erase(it); } - if (word_matches->empty()) { - search_res.erase_word_matches(word_matches); + // Even if *all* existing matches fit + // [previous ng, ...some unrelated chars..., current ng], + // we can always remove the entire document from the candidate pool. + if (word_match->spans.empty()) { + search_res.erase_document(word_match); if (search_res.empty()) return detail::search_continuation::abort; + return detail::search_continuation::proceed; } + + ++word_match->successful_ngrams; return detail::search_continuation::proceed; - }); + }; + }; + + std::size_t i = N; + for (; i + N <= word.size(); i += N) { + std::ranges::copy_n(word.begin() + i, N, ng.chars.begin()); + store_.search(ng, do_search(N)); + if (search_res.empty()) return; + ++current_ngram; + } + + // When the remaining character count is remainder of `word.size() % N`, + // search by the *slided* remaining characters. + // + // For example, when the document is "今日は晴れです": + // + // When doing 3-gram search with "今日は雨": + // 1. Search by "今日は" in the normal loop + // + // 2. Then, + // i == 3 + // remaining_chars == word.size() - i == 1 + // overlapping_chars == N - remaining_chars == 2 + // next_search_pos = i - overlapping_chars == 1 + // + // 3. Try to match "日は雨" in the last loop + if (int const remaining_chars = static_cast(word.size() - i); remaining_chars > 0) { + assert(remaining_chars < N); + std::shift_left(ng.chars.begin(), ng.chars.end(), remaining_chars); + std::ranges::copy_n(word.begin() + i, remaining_chars, ng.chars.begin() + (N - remaining_chars)); + store_.search(ng, do_search(remaining_chars)); + if (search_res.empty()) return; + ++current_ngram; } + + search_res.remove_stale_document_matches(word_id, current_ngram); } ngram_document_id max_doc_id_{0_doc_id}; detail::ngram_index_storage store_; }; - } // iris + +template +struct std::formatter + : std::formatter, CharT> +{ + using base_type = std::formatter, CharT>; + + template + Ctx::iterator format(iris::ngram_document_id doc_id, Ctx& ctx) const + { + return base_type::format(std::to_underlying(doc_id), ctx); + } +}; + +template +struct std::formatter + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram_occurrence const& occ, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{}:{}", occ.doc_id, occ.pos); + } +}; + +template +struct std::formatter, CharT> + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram_search_query const& query, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{}", query.words() | std::views::transform([](std::u32string_view ustr) { + return iris::unicode::transcode(ustr); + })); + } +}; + +template +struct std::formatter + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram_search_word_match const& word_match, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{{word: #{}, spans: {}}}", word_match.word_id, word_match.spans); + } +}; + #endif diff --git a/test/ngram.cpp b/test/ngram.cpp index e96313e..7e81475 100644 --- a/test/ngram.cpp +++ b/test/ngram.cpp @@ -12,25 +12,9 @@ # include #endif -namespace iris { - -inline std::ostream& operator<<(std::ostream& os, ngram_occurrence const& occ) -{ - return os << std::format("{}", occ); -} - -template -inline std::ostream& operator<<(std::ostream& os, interval const& iv) -{ - return os << std::format("{}", iv); -} - -} // iris - -// -------------------------------------------------- - using namespace iris::ngram_literals; using iris::ngram_occurrence; +using iris::interval; [[nodiscard]] constexpr auto make_occurrences(std::initializer_list occs) @@ -42,13 +26,13 @@ constexpr auto make_occurrences(std::initializer_list occs) std::vector occs; \ ngram_db.find_occurrences(iris::to_ngram(U ## ng_str), occs); \ CHECK(occs.empty()); \ - } while (false); + } while (false) #define IRIS_CHECK_OCCURRENCE(ng_str, ...) do { \ std::vector occs; \ ngram_db.find_occurrences(iris::to_ngram(U ## ng_str), occs); \ CHECK(occs == make_occurrences({__VA_ARGS__})); \ - } while (false); + } while (false) TEST_CASE("ngram (minimal input)") { @@ -59,30 +43,62 @@ TEST_CASE("ngram (minimal input)") { iris::ngram_database<> ngram_db; (void)ngram_db.add_document(U""); - IRIS_CHECK_NO_OCCURRENCE("今"); + IRIS_CHECK_NO_OCCURRENCE("a"); } { iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"今"); - IRIS_CHECK_OCCURRENCE("今", {0_doc_id, 0}); - IRIS_CHECK_NO_OCCURRENCE("無"); - IRIS_CHECK_NO_OCCURRENCE("今日"); + (void)ngram_db.add_document(U"a"); + IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); + IRIS_CHECK_NO_OCCURRENCE("X"); + IRIS_CHECK_NO_OCCURRENCE("XX"); } { iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"今"); - IRIS_CHECK_OCCURRENCE("今", {0_doc_id, 0}); - IRIS_CHECK_NO_OCCURRENCE("無"); - IRIS_CHECK_NO_OCCURRENCE("今日"); + (void)ngram_db.add_document(U"ab"); + IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); + IRIS_CHECK_NO_OCCURRENCE("X"); + IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); + IRIS_CHECK_NO_OCCURRENCE("XX"); } { iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"今日"); - IRIS_CHECK_OCCURRENCE("今", {0_doc_id, 0}); - IRIS_CHECK_OCCURRENCE("日", {0_doc_id, 1}); - IRIS_CHECK_NO_OCCURRENCE("無"); - IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); - IRIS_CHECK_NO_OCCURRENCE("今無"); + (void)ngram_db.add_document(U"abc"); + IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("c", {0_doc_id, 2}); + IRIS_CHECK_NO_OCCURRENCE("X"); + IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("bc", {0_doc_id, 1}); + IRIS_CHECK_NO_OCCURRENCE("XX"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("c", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("d", {0_doc_id, 3}); + IRIS_CHECK_NO_OCCURRENCE("X"); + IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("bc", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("cd", {0_doc_id, 2}); + IRIS_CHECK_NO_OCCURRENCE("XX"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abcde"); + IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("c", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("d", {0_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("e", {0_doc_id, 4}); + IRIS_CHECK_NO_OCCURRENCE("X"); + IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("bc", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("cd", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("de", {0_doc_id, 3}); + IRIS_CHECK_NO_OCCURRENCE("XX"); } } @@ -171,26 +187,540 @@ TEST_CASE("ngram (realistic input)") } } -TEST_CASE("ngram search") +struct DocumentMatch +{ + iris::ngram_document_id doc_id; + std::vector word_matches; + + DocumentMatch(iris::ngram_document_id doc_id, std::initializer_list word_matches) + : doc_id(doc_id) + , word_matches(word_matches) + {} + + DocumentMatch(iris::ngram_document_id doc_id, std::vector word_matches) + : doc_id(doc_id) + , word_matches(std::move(word_matches)) + {} + + [[nodiscard]] + bool operator==(DocumentMatch const&) const noexcept = default; +}; + +template +struct std::formatter + : iris::no_spec_formatter { + template + Ctx::iterator format(DocumentMatch const& doc_match, Ctx& ctx) const + { + return std::format_to(ctx.out(), "(doc: #{}, word_matches: {})", doc_match.doc_id, doc_match.word_matches); + } +}; + +#define IRIS_CHECK_SEARCH(query_input, ...) do { \ + iris::ngram_search_query const query{U ## query_input}; \ + auto const search_res = ngram_db.search(query); \ + auto const& doc_matches = search_res.doc_matches(); \ + \ + std::vector const expected_doc_matches{ \ + std::initializer_list{__VA_ARGS__} \ + }; \ + \ + auto const actual_doc_matches = doc_matches | std::views::transform([](auto const& kv) { \ + return DocumentMatch{kv.first, kv.second}; \ + }) | std::ranges::to(); \ + CHECK(actual_doc_matches == expected_doc_matches); \ + } while (false) + +TEST_CASE("ngram search (document chars = 0)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + { iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"今日は良い天気です。"); - //(void)ngram_db.add_document(U"今日は大雨です。"); - //(void)ngram_db.add_document(U"今日の東海地方は大雨でしょう。"); + IRIS_CHECK_SEARCH(""); + IRIS_CHECK_SEARCH("X"); + IRIS_CHECK_SEARCH("XX"); + } +} - iris::ngram_search_query<> query{U"良い天気"}; +// 1-gram document +TEST_CASE("ngram search (document chars = 1)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif - auto const search_res = ngram_db.search(query); + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"a"); + IRIS_CHECK_SEARCH(""); + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + IRIS_CHECK_SEARCH("XX"); + } +} - auto const& doc_matches = search_res.doc_matches(); +// 2-gram document +TEST_CASE("ngram search (document chars = 2)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aa"); + IRIS_CHECK_SEARCH(""); - REQUIRE(doc_matches.contains(0_doc_id)); - auto const& word_map = doc_matches.at(0_doc_id); + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("XXX"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + } +} + +// 2-gram + 1-gram document +TEST_CASE("ngram search (document chars = 3, aaa/baa)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif - REQUIRE(word_map.size() == 1); - CHECK(word_map[0].word_id == 0); - REQUIRE(word_map[0].matches.size() == 1); - CHECK(word_map[0].matches[0] == iris::interval{3, 7}); + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aaa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}, interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aaa", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("aaaX"); + IRIS_CHECK_SEARCH("Xaaa"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXXX"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"baa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{1, 2}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "baa", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("baX"); + IRIS_CHECK_SEARCH("Xba"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("baaX"); + IRIS_CHECK_SEARCH("Xbaa"); + IRIS_CHECK_SEARCH("baXX"); + IRIS_CHECK_SEARCH("XXba"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2-gram + 1-gram document +TEST_CASE("ngram search (document chars = 3, aba/aab)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aba"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aba", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("baX"); + IRIS_CHECK_SEARCH("Xba"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("abaX"); + IRIS_CHECK_SEARCH("Xaba"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXba"); + IRIS_CHECK_SEARCH("baXX"); + IRIS_CHECK_SEARCH("XXXX"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aab", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("aabX"); + IRIS_CHECK_SEARCH("Xaab"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2-gram + 1-gram document +TEST_CASE("ngram search (document chars = 3, abc)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abc"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "c", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("bcX"); + IRIS_CHECK_SEARCH("Xbc"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("abcX"); + IRIS_CHECK_SEARCH("Xabc"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXbc"); + IRIS_CHECK_SEARCH("bcXX"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +TEST_CASE("ngram search (document chars = 4)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + // TODO + + // 2x 2-gram document + //{ + // iris::ngram_database<> ngram_db; + // (void)ngram_db.add_document(U"aaaa"); + // IRIS_CHECK_SEARCH(""); + + // IRIS_CHECK_SEARCH( + // "a", + // {0_doc_id, { + // {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}, interval{3, 4}}}, + // }}, + // ); + // IRIS_CHECK_SEARCH("X"); + + // IRIS_CHECK_SEARCH( + // "aa", + // {0_doc_id, { + // {0, {interval{0, 2}, interval{2, 4}}}, + // }}, + // ); + // IRIS_CHECK_SEARCH("XX"); + //} + +} + +TEST_CASE("ngram search (minimal input, dependency on previous match)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXXef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXXefef"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abefef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abXXef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abXXefef"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXefef"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abxcd"); // trap document + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH( + "abcd", + {1_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab"); + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH( + "ab cd", + {1_doc_id, { + {0, {interval{0, 2}}}, + {1, {interval{2, 4}}}, + }}, + ); } } From c1ee2f648100b833318204262ab57ac87aad9f45 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:28:12 +0900 Subject: [PATCH 07/16] Optimize implementation and add tests --- include/iris/ngram.hpp | 364 ++++++++++++++++------- test/CMakeLists.txt | 3 + test/ngram.cpp | 563 +----------------------------------- test/ngram_search_2.cpp | 104 +++++++ test/ngram_search_3.cpp | 287 ++++++++++++++++++ test/ngram_search_4_dep.cpp | 407 ++++++++++++++++++++++++++ test/ngram_test.hpp | 67 +++++ 7 files changed, 1144 insertions(+), 651 deletions(-) create mode 100644 test/ngram_search_2.cpp create mode 100644 test/ngram_search_3.cpp create mode 100644 test/ngram_search_4_dep.cpp create mode 100644 test/ngram_test.hpp diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index ec6c646..1cf1e5e 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -25,6 +26,7 @@ #include #include +#include namespace iris { @@ -41,42 +43,131 @@ struct ngram_occurrence namespace detail { -inline constexpr std::size_t N_GRAM_MAX_OPTIMIZED_N = 2; +template struct ngram_value; +template<> struct ngram_value<1> { using type = std::uint8_t; }; +template<> struct ngram_value<2> { using type = std::uint16_t; }; +template<> struct ngram_value<4> { using type = std::uint32_t; }; +template<> struct ngram_value<8> { using type = std::uint64_t; }; + +template +using ngram_value_t = ngram_value::type; } // detail template struct ngram { - static_assert(N >= 1); + static_assert(N >= 3); - // TODO: optimize for N=1 - std::array chars; + std::array data; + template + constexpr void copy_n(It it) + noexcept(noexcept(*it++)) + { + std::ranges::copy_n(it, N, data.begin()); + } template [[nodiscard]] static constexpr ngram from_copy_n(It it) + noexcept(noexcept(std::declval().copy_n(std::move(it)))) + { + ngram ng; + ng.copy_n(std::move(it)); + return ng; + } + + template + constexpr void shift_copy(It it, int const remaining_chars) + noexcept( + noexcept(std::shift_left(data.begin(), data.end(), remaining_chars)) && + noexcept(std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars))) + ) + { + assert(remaining_chars < N); + std::shift_left(data.begin(), data.end(), remaining_chars); + std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars)); + } + + template + [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept + { + static_assert(Len == N + 1); + assert(chars[Len - 1] == static_cast(0)); + return ngram::from_copy_n(std::ranges::begin(chars)); + } + + [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; +}; + +template +struct ngram<1, CharT> +{ + CharT data; + + template + constexpr void copy_n(It it) + noexcept(noexcept(*it)) + { + data = *it; + } + template + [[nodiscard]] static constexpr ngram from_copy_n(It it) + noexcept(noexcept(std::declval().copy_n(std::move(it)))) + { + ngram ng; + ng.copy_n(std::move(it)); + return ng; + } + + template + [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept + { + static_assert(Len == 1 + 1); + assert(chars[Len - 1] == static_cast(0)); + return ngram::from_copy_n(std::ranges::begin(chars)); + } + + [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; +}; + +template +struct ngram<2, CharT> +{ + using value_type = detail::ngram_value_t; + value_type data; + + template + constexpr void copy_n(It it) noexcept(noexcept(*it++)) { - if constexpr (N == 1) { - ngram ng; - ng.chars[0] = *it; - return ng; - } else if constexpr (N == 2) { - ngram ng; - ng.chars[0] = *it++; - ng.chars[1] = *it; - return ng; - } else { - static_assert(detail::N_GRAM_MAX_OPTIMIZED_N == 2); - ngram ng; - std::ranges::copy_n(it, N, ng.chars.begin()); - return ng; - } + using uchar = std::make_unsigned_t; + data = value_type(static_cast(*it++)) << (sizeof(CharT) * 8); + data |= value_type(static_cast(*it)); + } + template + [[nodiscard]] static constexpr ngram from_copy_n(It it) + noexcept(noexcept(std::declval().copy_n(std::move(it)))) + { + ngram ng; + ng.copy_n(std::move(it)); + return ng; + } + + template + constexpr void shift_copy(It it, int const remaining_chars) + noexcept(noexcept(*it)) + { + assert(remaining_chars == 1); + (void)remaining_chars; + data = (data << (sizeof(CharT) * 8)) | value_type(static_cast>(*it)); } template [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept { + static_assert(Len == 2 + 1); assert(chars[Len - 1] == static_cast(0)); return ngram::from_copy_n(std::ranges::begin(chars)); } @@ -92,20 +183,6 @@ inline namespace ngram_literals { return ngram_document_id{static_cast>(id)}; } -[[nodiscard]] constexpr auto operator ""_2gram(char32_t const* str, std::size_t len) noexcept -{ - assert(len == 2); - (void)len; - return ngram<2, char32_t>{str[0], str[1]}; -} - -[[nodiscard]] constexpr auto operator ""_1gram(char32_t const* str, std::size_t len) noexcept -{ - assert(len == 1); - (void)len; - return ngram<1, char32_t>{str[0]}; -} - } // ngram_literals template @@ -188,37 +265,101 @@ struct ngram_posting_list }; template -struct ngram_index +class ngram_index { - void append(ngram const ng, ngram_document_id const doc_id, int const pos) - { - gram_entries[ng].append(doc_id, pos); - } + using entry_map = std::flat_map, std::unique_ptr>; + static constexpr std::size_t side_merge_threshold = 2048; +public: [[nodiscard]] - bool empty() const noexcept + auto find_list(this auto&& self, ngram const ng) { - return gram_entries.empty(); + if (auto const it = self.gram_entries_.find(ng); it != self.gram_entries_.end()) { + return it->second.get(); + } + if (auto const it = self.side_entries_.find(ng); it != self.side_entries_.end()) { + return it->second.get(); + } + return static_cast(nullptr); } void find_occurrences(ngram const ng, std::vector& occs) const { occs.clear(); - auto const it = gram_entries.find(ng); - if (it == gram_entries.end()) return; - - it->second.to_occurrence_list(occs); + auto const* list = this->find_list(ng); + if (!list) return; + list->to_occurrence_list(occs); } template void search(ngram const ng, F&& f) const { - auto const it = gram_entries.find(ng); - if (it == gram_entries.end()) return; - it->second.for_each_documents(f); + auto const* list = this->find_list(ng); + if (!list) return; + list->for_each_documents(f); + } + + [[nodiscard]] + bool empty() const noexcept + { + return gram_entries_.empty() && side_entries_.empty(); + } + + void merge_new_entries(std::vector, std::unique_ptr>>& pending) + { + if (pending.empty()) return; // vocabulary saturated + + for (auto& [key, pl] : pending) { + [[maybe_unused]] + auto const it = side_entries_.try_emplace( + side_entries_.end(), // hint + key, std::move(pl) + ); + assert(it->second != nullptr && pl == nullptr); + } + if (side_entries_.size() >= side_merge_threshold) { + this->flush_side(); + } + } + +private: + void flush_side() + { + if (side_entries_.empty()) return; + + auto [skeys, svalues] = std::move(side_entries_).extract(); + auto [keys, values] = std::move(gram_entries_).extract(); + + std::size_t const old_size = keys.size(); + std::size_t const add = skeys.size(); + keys.resize(old_size + add); + values.resize(old_size + add); + + // Backward merge + std::size_t out = old_size + add; + std::size_t i = old_size; + std::size_t j = add; + while (j > 0) { + if (i > 0 && skeys[j - 1] < keys[i - 1]) { + --out; + --i; + keys[out] = keys[i]; + values[out] = std::move(values[i]); + } else { + assert(i == 0 || keys[i - 1] < skeys[j - 1]); + --out; + --j; + keys[out] = skeys[j]; + values[out] = std::move(svalues[j]); + } + } + assert(out == i); + + gram_entries_.replace(std::move(keys), std::move(values)); } - std::flat_map, PostingListT> gram_entries; + // Double-buffered to reduce insertion cost + entry_map gram_entries_, side_entries_; }; template @@ -255,7 +396,7 @@ struct ngram_index_storage } template - [[nodiscard]] auto& get_index(this auto& self) noexcept IRIS_LIFETIMEBOUND + [[nodiscard]] auto& get_index(this auto& self IRIS_LIFETIMEBOUND) noexcept { return self.template get_data().idx; } @@ -270,7 +411,7 @@ struct ngram_index_storage std::vector, default_init_allocator>> batch_grams; - std::vector, PostingListT>> + std::vector, std::unique_ptr>> batch_pending; }; @@ -278,14 +419,14 @@ struct ngram_index_storage ngram_index_storage_data<2> bi_data_; template - [[nodiscard]] auto& get_data(this auto& self) noexcept IRIS_LIFETIMEBOUND + [[nodiscard]] auto& get_data(this auto& self IRIS_LIFETIMEBOUND) noexcept { if constexpr (N == 1) { return self.uni_data_; } else if constexpr (N == 2) { return self.bi_data_; } else { - static_assert(N_GRAM_MAX_OPTIMIZED_N == 2); + static_assert(false); } } @@ -318,13 +459,13 @@ struct ngram_index_storage if constexpr (N == 1) { for (std::size_t i = 0; i < input.size(); ++i) { - data.batch_grams[i].ng.chars[0] = input[i]; + data.batch_grams[i].ng.data = input[i]; data.batch_grams[i].pos = static_cast(i); } } else { for (std::size_t i = 0; i + N <= input.size(); ++i) { - std::ranges::copy_n(input.begin() + i, N, data.batch_grams[i].ng.chars.begin()); + data.batch_grams[i].ng.copy_n(input.begin() + i); data.batch_grams[i].pos = static_cast(i); } } @@ -336,22 +477,20 @@ struct ngram_index_storage [](auto const& a, auto const& b) { return a.ng == b.ng; } )) { auto const& key = chunk.front().ng; - if (auto const it = data.idx.gram_entries.find(key); it != data.idx.gram_entries.end()) { + if (PostingListT* const pl = data.idx.find_list(key)) { for (auto const& gp : chunk) { - it->second.append(doc_id, gp.pos); + pl->append(doc_id, gp.pos); } + } else { - auto& pl = data.batch_pending.emplace_back(key, PostingListT{}).second; + auto& new_pl = data.batch_pending.emplace_back(key, std::make_unique()).second; for (auto const& gp : chunk) { - pl.append(doc_id, gp.pos); + new_pl->append(doc_id, gp.pos); } } } - data.idx.gram_entries.insert( - std::sorted_unique, - std::make_move_iterator(data.batch_pending.begin()), - std::make_move_iterator(data.batch_pending.end()) - ); + + data.idx.merge_new_entries(data.batch_pending); } }; @@ -456,7 +595,10 @@ class [[nodiscard]] ngram_search_result [[nodiscard]] bool has_document(ngram_document_id const doc_id) const noexcept { - return doc_matches_.contains(doc_id); + auto const it = doc_matches_.find(doc_id); + // An entry with no word matches is a tombstone (soft-erased document + // awaiting the next sweep), not a match. + return it != doc_matches_.end() && !it->second.empty(); } [[nodiscard]] @@ -474,9 +616,12 @@ class [[nodiscard]] ngram_search_result assert(word_id == 0); assert(doc_matches_.empty() || doc_matches_.rbegin()->first < doc_id); doc_matches_it = doc_matches_.try_emplace(doc_matches_.end(), doc_id); // hint: append + ++live_doc_count_; + } else { doc_matches_it = doc_matches_.find(doc_id); if (doc_matches_it == doc_matches_.end()) return false; // no new docs after word 0 + if (doc_matches_it->second.empty()) return false; // tombstoned (soft-erased) document; skip } assert(!std::ranges::contains(doc_matches_it->second, word_id, &ngram_search_word_match::word_id)); @@ -513,7 +658,14 @@ class [[nodiscard]] ngram_search_result void erase_document(word_matches_handle const& handle) { - doc_matches_.erase(handle.doc_it); + // This is slow because + // k erases x O(n) shift each =~ O(n^2) per word + //doc_matches_.erase(handle.doc_it); + + assert(!handle.doc_it->second.empty()); // never double-tombstone + handle.doc_it->second.clear(); // make this tombstone + assert(live_doc_count_ >= 1); + --live_doc_count_; } void remove_stale_document_matches(int const word_id, unsigned const expected_ngrams) @@ -523,13 +675,15 @@ class [[nodiscard]] ngram_search_result std::size_t out = 0; for (std::size_t in = 0; in < keys.size(); ++in) { auto& word_matches = values[in]; - bool has_word = false; + bool is_word_survived = false; std::erase_if(word_matches, [&](ngram_search_word_match const& wm) { if (wm.word_id != word_id) return false; - has_word = true; - return wm.successful_ngrams != expected_ngrams; + if (wm.successful_ngrams != expected_ngrams) return true; + is_word_survived = true; + return false; }); - if (!has_word || word_matches.empty()) continue; + if (!is_word_survived || word_matches.empty()) continue; + if (out != in) { keys[out] = keys[in]; values[out] = std::move(values[in]); @@ -539,17 +693,19 @@ class [[nodiscard]] ngram_search_result keys.resize(out); values.resize(out); doc_matches_.replace(std::move(keys), std::move(values)); + live_doc_count_ = out; } - void clear() noexcept + void reset() noexcept { doc_matches_.clear(); + live_doc_count_ = 0; } [[nodiscard]] bool empty() const noexcept { - return doc_matches_.empty(); + return live_doc_count_ == 0; } [[nodiscard]] @@ -560,6 +716,7 @@ class [[nodiscard]] ngram_search_result private: doc_matches_map doc_matches_; + std::size_t live_doc_count_ = 0; }; template @@ -577,7 +734,7 @@ class ngram_database } template - void find_occurrences(ngram ng, std::vector& occs) const noexcept + void find_occurrences(ngram ng, std::vector& occs) const { occs.clear(); auto const& idx = store_.template get_index(); @@ -596,12 +753,18 @@ class ngram_database auto it = query.words().begin(); assert(!it->empty()); this->search_word(search_res, word_id++, *it++); - if (search_res.empty()) return search_res; + if (search_res.empty()) { + search_res.reset(); // remove tombstones + return search_res; + } for (; it != query.words().end(); ++it) { assert(!it->empty()); this->search_word(search_res, word_id++, *it); - if (search_res.empty()) break; + if (search_res.empty()) { + search_res.reset(); // remove tombstones + break; + } } return search_res; } @@ -639,7 +802,7 @@ class ngram_database } }); if (available_doc_count == 0) { - search_res.clear(); + search_res.reset(); return; } } @@ -698,36 +861,41 @@ class ngram_database std::size_t i = N; for (; i + N <= word.size(); i += N) { - std::ranges::copy_n(word.begin() + i, N, ng.chars.begin()); + ng.copy_n(word.begin() + i); store_.search(ng, do_search(N)); if (search_res.empty()) return; ++current_ngram; } - // When the remaining character count is remainder of `word.size() % N`, - // search by the *slided* remaining characters. - // - // For example, when the document is "今日は晴れです": - // - // When doing 3-gram search with "今日は雨": - // 1. Search by "今日は" in the normal loop - // - // 2. Then, - // i == 3 - // remaining_chars == word.size() - i == 1 - // overlapping_chars == N - remaining_chars == 2 - // next_search_pos = i - overlapping_chars == 1 - // - // 3. Try to match "日は雨" in the last loop - if (int const remaining_chars = static_cast(word.size() - i); remaining_chars > 0) { - assert(remaining_chars < N); - std::shift_left(ng.chars.begin(), ng.chars.end(), remaining_chars); - std::ranges::copy_n(word.begin() + i, remaining_chars, ng.chars.begin() + (N - remaining_chars)); - store_.search(ng, do_search(remaining_chars)); - if (search_res.empty()) return; - ++current_ngram; + if constexpr (N >= 2) { + // When the remaining character count is remainder of `word.size() % N`, + // search by the *slided* remaining characters. + // + // For example, when the document is "今日は晴れです": + // + // When doing 3-gram search with "今日は雨": + // 1. Search by "今日は" in the normal loop + // + // 2. Then, + // i == 3 + // remaining_chars == word.size() - i == 1 + // overlapping_chars == N - remaining_chars == 2 + // next_search_pos = i - overlapping_chars == 1 + // + // 3. Try to match "日は雨" in the last loop + if (int const remaining_chars = static_cast(word.size() - i); remaining_chars > 0) { + assert(remaining_chars < N); + ng.shift_copy(word.begin() + i, remaining_chars); + store_.search(ng, do_search(remaining_chars)); + if (search_res.empty()) return; + ++current_ngram; + } } + if constexpr (IsFirstWord) { + // A first word of exactly one n-gram runs no continuation searches + if (current_ngram == 1) return; + } search_res.remove_stale_document_matches(word_id, current_ngram); } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 590803b..e151928 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -191,6 +191,9 @@ if(PROJECT_IS_TOP_LEVEL) interval_set snippet ngram + ngram_search_2 + ngram_search_3 + ngram_search_4_dep ) foreach(test_name IN LISTS IRIS_TEST_IRIS_TESTS) diff --git a/test/ngram.cpp b/test/ngram.cpp index 7e81475..7a1d39c 100644 --- a/test/ngram.cpp +++ b/test/ngram.cpp @@ -1,20 +1,6 @@ // SPDX-License-Identifier: MIT -#include "iris_test.hpp" - -#include - -#include -#include -#include - -#ifdef _MSC_VER -# include -#endif - -using namespace iris::ngram_literals; -using iris::ngram_occurrence; -using iris::interval; +#include "ngram_test.hpp" [[nodiscard]] constexpr auto make_occurrences(std::initializer_list occs) @@ -34,6 +20,15 @@ constexpr auto make_occurrences(std::initializer_list occs) CHECK(occs == make_occurrences({__VA_ARGS__})); \ } while (false) +TEST_CASE("ngram (type traits)") +{ + STATIC_CHECK(std::same_as::data), char>); + STATIC_CHECK(std::same_as::data), std::uint16_t>); + + STATIC_CHECK(std::same_as::data), char32_t>); + STATIC_CHECK(std::same_as::data), std::uint64_t>); +} + TEST_CASE("ngram (minimal input)") { #ifdef _MSC_VER @@ -186,541 +181,3 @@ TEST_CASE("ngram (realistic input)") IRIS_CHECK_OCCURRENCE("う。", {2_doc_id, 13}); } } - -struct DocumentMatch -{ - iris::ngram_document_id doc_id; - std::vector word_matches; - - DocumentMatch(iris::ngram_document_id doc_id, std::initializer_list word_matches) - : doc_id(doc_id) - , word_matches(word_matches) - {} - - DocumentMatch(iris::ngram_document_id doc_id, std::vector word_matches) - : doc_id(doc_id) - , word_matches(std::move(word_matches)) - {} - - [[nodiscard]] - bool operator==(DocumentMatch const&) const noexcept = default; -}; - -template -struct std::formatter - : iris::no_spec_formatter -{ - template - Ctx::iterator format(DocumentMatch const& doc_match, Ctx& ctx) const - { - return std::format_to(ctx.out(), "(doc: #{}, word_matches: {})", doc_match.doc_id, doc_match.word_matches); - } -}; - -#define IRIS_CHECK_SEARCH(query_input, ...) do { \ - iris::ngram_search_query const query{U ## query_input}; \ - auto const search_res = ngram_db.search(query); \ - auto const& doc_matches = search_res.doc_matches(); \ - \ - std::vector const expected_doc_matches{ \ - std::initializer_list{__VA_ARGS__} \ - }; \ - \ - auto const actual_doc_matches = doc_matches | std::views::transform([](auto const& kv) { \ - return DocumentMatch{kv.first, kv.second}; \ - }) | std::ranges::to(); \ - CHECK(actual_doc_matches == expected_doc_matches); \ - } while (false) - -TEST_CASE("ngram search (document chars = 0)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - { - iris::ngram_database<> ngram_db; - IRIS_CHECK_SEARCH(""); - IRIS_CHECK_SEARCH("X"); - IRIS_CHECK_SEARCH("XX"); - } -} - -// 1-gram document -TEST_CASE("ngram search (document chars = 1)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"a"); - IRIS_CHECK_SEARCH(""); - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{0, 1}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - IRIS_CHECK_SEARCH("XX"); - } -} - -// 2-gram document -TEST_CASE("ngram search (document chars = 2)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"aa"); - IRIS_CHECK_SEARCH(""); - - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{0, 1}, interval{1, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - - IRIS_CHECK_SEARCH( - "aa", - {0_doc_id, { - {0, {interval{0, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH("XX"); - - IRIS_CHECK_SEARCH("aaX"); - IRIS_CHECK_SEARCH("Xaa"); - IRIS_CHECK_SEARCH("XXX"); - } - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"ab"); - IRIS_CHECK_SEARCH(""); - - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{0, 1}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "b", - {0_doc_id, { - {0, {interval{1, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - - IRIS_CHECK_SEARCH( - "ab", - {0_doc_id, { - {0, {interval{0, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH("XX"); - - IRIS_CHECK_SEARCH("abX"); - IRIS_CHECK_SEARCH("Xab"); - IRIS_CHECK_SEARCH("XXX"); - } -} - -// 2-gram + 1-gram document -TEST_CASE("ngram search (document chars = 3, aaa/baa)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"aaa"); - IRIS_CHECK_SEARCH(""); - - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - - IRIS_CHECK_SEARCH( - "aa", - {0_doc_id, { - {0, {interval{0, 2}, interval{1, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("XX"); - - IRIS_CHECK_SEARCH( - "aaa", - {0_doc_id, { - {0, {interval{0, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("aaX"); - IRIS_CHECK_SEARCH("Xaa"); - IRIS_CHECK_SEARCH("XXX"); - - IRIS_CHECK_SEARCH("aaaX"); - IRIS_CHECK_SEARCH("Xaaa"); - IRIS_CHECK_SEARCH("XXaa"); - IRIS_CHECK_SEARCH("aaXX"); - IRIS_CHECK_SEARCH("XXXX"); - } - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"baa"); - IRIS_CHECK_SEARCH(""); - - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{1, 2}, interval{2, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "b", - {0_doc_id, { - {0, {interval{0, 1}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - - IRIS_CHECK_SEARCH( - "aa", - {0_doc_id, { - {0, {interval{1, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "ba", - {0_doc_id, { - {0, {interval{0, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH("XX"); - - IRIS_CHECK_SEARCH( - "baa", - {0_doc_id, { - {0, {interval{0, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("aaX"); - IRIS_CHECK_SEARCH("Xaa"); - IRIS_CHECK_SEARCH("baX"); - IRIS_CHECK_SEARCH("Xba"); - IRIS_CHECK_SEARCH("XXX"); - - IRIS_CHECK_SEARCH("baaX"); - IRIS_CHECK_SEARCH("Xbaa"); - IRIS_CHECK_SEARCH("baXX"); - IRIS_CHECK_SEARCH("XXba"); - IRIS_CHECK_SEARCH("aaXX"); - IRIS_CHECK_SEARCH("XXaa"); - IRIS_CHECK_SEARCH("XXXX"); - } -} - -// 2-gram + 1-gram document -TEST_CASE("ngram search (document chars = 3, aba/aab)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"aba"); - IRIS_CHECK_SEARCH(""); - - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{0, 1}, interval{2, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "b", - {0_doc_id, { - {0, {interval{1, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - - IRIS_CHECK_SEARCH( - "ab", - {0_doc_id, { - {0, {interval{0, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "ba", - {0_doc_id, { - {0, {interval{1, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("XX"); - - IRIS_CHECK_SEARCH( - "aba", - {0_doc_id, { - {0, {interval{0, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("abX"); - IRIS_CHECK_SEARCH("Xab"); - IRIS_CHECK_SEARCH("baX"); - IRIS_CHECK_SEARCH("Xba"); - IRIS_CHECK_SEARCH("XXX"); - - IRIS_CHECK_SEARCH("abaX"); - IRIS_CHECK_SEARCH("Xaba"); - IRIS_CHECK_SEARCH("XXab"); - IRIS_CHECK_SEARCH("abXX"); - IRIS_CHECK_SEARCH("XXba"); - IRIS_CHECK_SEARCH("baXX"); - IRIS_CHECK_SEARCH("XXXX"); - } - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"aab"); - IRIS_CHECK_SEARCH(""); - - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{0, 1}, interval{1, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "b", - {0_doc_id, { - {0, {interval{2, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - - IRIS_CHECK_SEARCH( - "aa", - {0_doc_id, { - {0, {interval{0, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "ab", - {0_doc_id, { - {0, {interval{1, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("XX"); - - IRIS_CHECK_SEARCH( - "aab", - {0_doc_id, { - {0, {interval{0, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("aaX"); - IRIS_CHECK_SEARCH("Xaa"); - IRIS_CHECK_SEARCH("abX"); - IRIS_CHECK_SEARCH("Xab"); - IRIS_CHECK_SEARCH("XXX"); - - IRIS_CHECK_SEARCH("aabX"); - IRIS_CHECK_SEARCH("Xaab"); - IRIS_CHECK_SEARCH("XXaa"); - IRIS_CHECK_SEARCH("aaXX"); - IRIS_CHECK_SEARCH("XXab"); - IRIS_CHECK_SEARCH("abXX"); - IRIS_CHECK_SEARCH("XXXX"); - } -} - -// 2-gram + 1-gram document -TEST_CASE("ngram search (document chars = 3, abc)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"abc"); - IRIS_CHECK_SEARCH(""); - - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{0, 1}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "b", - {0_doc_id, { - {0, {interval{1, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "c", - {0_doc_id, { - {0, {interval{2, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - - IRIS_CHECK_SEARCH( - "ab", - {0_doc_id, { - {0, {interval{0, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "bc", - {0_doc_id, { - {0, {interval{1, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("XX"); - - IRIS_CHECK_SEARCH( - "abc", - {0_doc_id, { - {0, {interval{0, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("abX"); - IRIS_CHECK_SEARCH("Xab"); - IRIS_CHECK_SEARCH("bcX"); - IRIS_CHECK_SEARCH("Xbc"); - IRIS_CHECK_SEARCH("XXX"); - - IRIS_CHECK_SEARCH("abcX"); - IRIS_CHECK_SEARCH("Xabc"); - IRIS_CHECK_SEARCH("XXab"); - IRIS_CHECK_SEARCH("abXX"); - IRIS_CHECK_SEARCH("XXbc"); - IRIS_CHECK_SEARCH("bcXX"); - IRIS_CHECK_SEARCH("XXXX"); - } -} - -TEST_CASE("ngram search (document chars = 4)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - // TODO - - // 2x 2-gram document - //{ - // iris::ngram_database<> ngram_db; - // (void)ngram_db.add_document(U"aaaa"); - // IRIS_CHECK_SEARCH(""); - - // IRIS_CHECK_SEARCH( - // "a", - // {0_doc_id, { - // {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}, interval{3, 4}}}, - // }}, - // ); - // IRIS_CHECK_SEARCH("X"); - - // IRIS_CHECK_SEARCH( - // "aa", - // {0_doc_id, { - // {0, {interval{0, 2}, interval{2, 4}}}, - // }}, - // ); - // IRIS_CHECK_SEARCH("XX"); - //} - -} - -TEST_CASE("ngram search (minimal input, dependency on previous match)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"abef"); - IRIS_CHECK_SEARCH("abXXef"); - } - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"abef"); - IRIS_CHECK_SEARCH("abXXefef"); - } - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"ab..ef"); - IRIS_CHECK_SEARCH("abef"); - } - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"ab..ef"); - IRIS_CHECK_SEARCH("abefef"); - } - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"ab..ef"); - IRIS_CHECK_SEARCH("abXXef"); - } - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"ab..ef"); - IRIS_CHECK_SEARCH("abXXefef"); - } - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"abef"); - IRIS_CHECK_SEARCH("abXef"); - } - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"abef"); - IRIS_CHECK_SEARCH("abXefef"); - } - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"abxcd"); // trap document - (void)ngram_db.add_document(U"abcd"); - IRIS_CHECK_SEARCH( - "abcd", - {1_doc_id, { - {0, {interval{0, 4}}}, - }}, - ); - } - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"ab"); - (void)ngram_db.add_document(U"abcd"); - IRIS_CHECK_SEARCH( - "ab cd", - {1_doc_id, { - {0, {interval{0, 2}}}, - {1, {interval{2, 4}}}, - }}, - ); - } -} diff --git a/test/ngram_search_2.cpp b/test/ngram_search_2.cpp new file mode 100644 index 0000000..2f85570 --- /dev/null +++ b/test/ngram_search_2.cpp @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: MIT + +#include "ngram_test.hpp" + +TEST_CASE("ngram search (document chars = 0)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + IRIS_CHECK_SEARCH(""); + IRIS_CHECK_SEARCH("X"); + IRIS_CHECK_SEARCH("XX"); + } +} + +// 1-gram document +TEST_CASE("ngram search (document chars = 1)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"a"); + IRIS_CHECK_SEARCH(""); + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + IRIS_CHECK_SEARCH("XX"); + } +} + +// 2-gram document +TEST_CASE("ngram search (document chars = 2)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("XXX"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + } +} diff --git a/test/ngram_search_3.cpp b/test/ngram_search_3.cpp new file mode 100644 index 0000000..dc9040d --- /dev/null +++ b/test/ngram_search_3.cpp @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: MIT + +#include "ngram_test.hpp" + +// 2-gram + 1-gram document +TEST_CASE("ngram search (document chars = 3, aaa/baa)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aaa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}, interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aaa", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("aaaX"); + IRIS_CHECK_SEARCH("Xaaa"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXXX"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"baa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{1, 2}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "baa", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("baX"); + IRIS_CHECK_SEARCH("Xba"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("baaX"); + IRIS_CHECK_SEARCH("Xbaa"); + IRIS_CHECK_SEARCH("baXX"); + IRIS_CHECK_SEARCH("XXba"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2-gram + 1-gram document +TEST_CASE("ngram search (document chars = 3, aba/aab)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aba"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aba", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("baX"); + IRIS_CHECK_SEARCH("Xba"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("abaX"); + IRIS_CHECK_SEARCH("Xaba"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXba"); + IRIS_CHECK_SEARCH("baXX"); + IRIS_CHECK_SEARCH("XXXX"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aab", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("aabX"); + IRIS_CHECK_SEARCH("Xaab"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2-gram + 1-gram document +TEST_CASE("ngram search (document chars = 3, abc)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abc"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "c", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("bcX"); + IRIS_CHECK_SEARCH("Xbc"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("abcX"); + IRIS_CHECK_SEARCH("Xabc"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXbc"); + IRIS_CHECK_SEARCH("bcXX"); + IRIS_CHECK_SEARCH("XXXX"); + } +} diff --git a/test/ngram_search_4_dep.cpp b/test/ngram_search_4_dep.cpp new file mode 100644 index 0000000..8045632 --- /dev/null +++ b/test/ngram_search_4_dep.cpp @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: MIT + +#include "ngram_test.hpp" + +// 2x 2-gram document +TEST_CASE("ngram search (document chars = 4, aaaa)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aaaa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}, interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}, interval{1, 3}, interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aaa", + {0_doc_id, { + {0, {interval{0, 3}, interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "aaaa", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaaX"); + IRIS_CHECK_SEARCH("Xaaa"); + IRIS_CHECK_SEARCH("XXXX"); + + IRIS_CHECK_SEARCH("aaaaX"); + IRIS_CHECK_SEARCH("Xaaaa"); + IRIS_CHECK_SEARCH("XXaaa"); + IRIS_CHECK_SEARCH("aaaXX"); + IRIS_CHECK_SEARCH("XXXXX"); + } +} + +// 2x 2-gram document +TEST_CASE("ngram search (document chars = 4, abab)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}, interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}, interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aba", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bab", + {0_doc_id, { + {0, {interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "abab", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("ababX"); + IRIS_CHECK_SEARCH("Xabab"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2x 2-gram document +TEST_CASE("ngram search (document chars = 4, abca)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abca"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "c", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ca", + {0_doc_id, { + {0, {interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bca", + {0_doc_id, { + {0, {interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xbc"); + IRIS_CHECK_SEARCH("caX"); + IRIS_CHECK_SEARCH("Xca"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "abca", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abcaX"); + IRIS_CHECK_SEARCH("Xabca"); + IRIS_CHECK_SEARCH("caab"); // "ca" and "ab" both exist but not contiguous as "caab" + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2x 2-gram document +TEST_CASE("ngram search (document chars = 4, abcd)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "d", + {0_doc_id, { + {0, {interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "cd", + {0_doc_id, { + {0, {interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bcd", + {0_doc_id, { + {0, {interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "abcd", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abcX"); + IRIS_CHECK_SEARCH("Xbcd"); + IRIS_CHECK_SEARCH("abcdX"); + IRIS_CHECK_SEARCH("Xabcd"); + IRIS_CHECK_SEARCH("abXcd"); // both halves exist; broken by X in the middle... but see note below! + IRIS_CHECK_SEARCH("acbd"); // all chars exist; order scrambled + IRIS_CHECK_SEARCH("XXXX"); + } +} + +TEST_CASE("ngram search (dependency on previous match)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH("abXX"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXXef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXXefef"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abefef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abXXef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abXXefef"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXefef"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abxcd"); // trap document + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH( + "abcd", + {1_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abcd"); + (void)ngram_db.add_document(U"abxcd"); // trap document + IRIS_CHECK_SEARCH( + "abcd", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab"); + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH( + "ab cd", + {1_doc_id, { + {0, {interval{0, 2}}}, + {1, {interval{2, 4}}}, + }}, + ); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abXcd"); + (void)ngram_db.add_document(U"abcdX"); + IRIS_CHECK_SEARCH("abcdc"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abXXabcd"); + IRIS_CHECK_SEARCH( + "abcd", + {0_doc_id, { + {0, {interval{4, 8}}}, + }}, + ); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab cdXf"); + (void)ngram_db.add_document(U"ab cdef"); + IRIS_CHECK_SEARCH( + "ab cdef", + {1_doc_id, { + {0, {interval{0, 2}}}, + {1, {interval{3, 7}}}, + }}, + ); + } +} diff --git a/test/ngram_test.hpp b/test/ngram_test.hpp new file mode 100644 index 0000000..ff49a87 --- /dev/null +++ b/test/ngram_test.hpp @@ -0,0 +1,67 @@ +#ifndef IRIS_ZZ_TEST_NGRAM_TEST_HPP +#define IRIS_ZZ_TEST_NGRAM_TEST_HPP + +// SPDX-License-Identifier: MIT + +#include "iris_test.hpp" + +#include + +#include +#include +#include + +#ifdef _MSC_VER +# include +#endif + +using namespace iris::ngram_literals; +using iris::ngram_occurrence; +using iris::interval; + +struct DocumentMatch +{ + iris::ngram_document_id doc_id; + std::vector word_matches; + + DocumentMatch(iris::ngram_document_id doc_id, std::initializer_list word_matches) + : doc_id(doc_id) + , word_matches(word_matches) + {} + + DocumentMatch(iris::ngram_document_id doc_id, std::vector word_matches) + : doc_id(doc_id) + , word_matches(std::move(word_matches)) + {} + + [[nodiscard]] + bool operator==(DocumentMatch const&) const noexcept = default; +}; + +template +struct std::formatter + : iris::no_spec_formatter +{ + template + Ctx::iterator format(DocumentMatch const& doc_match, Ctx& ctx) const + { + return std::format_to(ctx.out(), "(doc: #{}, word_matches: {})", doc_match.doc_id, doc_match.word_matches); + } +}; + +#define IRIS_CHECK_SEARCH(query_input, ...) do { \ + iris::ngram_search_query const query{U ## query_input}; \ + auto const search_res = ngram_db.search(query); \ + auto const& doc_matches = search_res.doc_matches(); \ + \ + std::vector const expected_doc_matches{ \ + std::initializer_list{__VA_ARGS__} \ + }; \ + \ + auto const actual_doc_matches = doc_matches | std::views::transform([](auto const& kv) { \ + return DocumentMatch{kv.first, kv.second}; \ + }) | std::ranges::to(); \ + CHECK(actual_doc_matches == expected_doc_matches); \ + } while (false) + +#endif From 5de5b49f6b9497893fc26f3ef20c8db4912660f6 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:36:45 +0900 Subject: [PATCH 08/16] Fix sign comparison --- include/iris/ngram.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index 1cf1e5e..b3319e2 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -83,7 +83,7 @@ struct ngram noexcept(std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars))) ) { - assert(remaining_chars < N); + assert(remaining_chars < int(N)); std::shift_left(data.begin(), data.end(), remaining_chars); std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars)); } @@ -884,7 +884,7 @@ class ngram_database // // 3. Try to match "日は雨" in the last loop if (int const remaining_chars = static_cast(word.size() - i); remaining_chars > 0) { - assert(remaining_chars < N); + assert(remaining_chars < int(N)); ng.shift_copy(word.begin() + i, remaining_chars); store_.search(ng, do_search(remaining_chars)); if (search_res.empty()) return; From eb6e2a714b00bf74df3b9820440523ca1df9242f Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Fri, 14 Aug 2026 03:50:56 +0900 Subject: [PATCH 09/16] Current --- include/iris/format.hpp | 2 -- include/iris/ngram.hpp | 8 ++++---- test/CMakeLists.txt | 1 - 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/include/iris/format.hpp b/include/iris/format.hpp index 4c63876..5539380 100644 --- a/include/iris/format.hpp +++ b/include/iris/format.hpp @@ -3,8 +3,6 @@ // SPDX-License-Identifier: MIT -// SPDX-License-Identifier: MIT - #include #include diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index b3319e2..60ded71 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -1,5 +1,5 @@ -#ifndef IRIS_NGRAM_HPP -#define IRIS_NGRAM_HPP +#ifndef IRIS_ZZ_NGRAM_HPP +#define IRIS_ZZ_NGRAM_HPP // SPDX-License-Identifier: MIT @@ -834,9 +834,9 @@ class ngram_database for (auto it = word_match->spans.begin(); it != word_match->spans.end();) { auto& prev_pos = *it; - if (std::ranges::binary_search(positions, prev_pos.right - overlapping_chars)) { + if (std::ranges::binary_search(positions, prev_pos.upper - overlapping_chars)) { // Matched; the current word's current n-gram is contiguous to the previous n-gram - prev_pos.right += remaining_chars; + prev_pos.upper += remaining_chars; ++it; continue; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e151928..0f97bbb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -184,7 +184,6 @@ if(PROJECT_IS_TOP_LEVEL) indirect colorize_format preprocess - interval string_algo interval interval_algo From f3c754d30b9dfe55e8eba23a344eeaf710721bbc Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Sat, 15 Aug 2026 20:34:39 +0900 Subject: [PATCH 10/16] Add clear() function --- include/iris/ngram.hpp | 46 ++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index 60ded71..02d6083 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -271,8 +271,7 @@ class ngram_index static constexpr std::size_t side_merge_threshold = 2048; public: - [[nodiscard]] - auto find_list(this auto&& self, ngram const ng) + [[nodiscard]] auto find_list(this auto&& self, ngram const ng) { if (auto const it = self.gram_entries_.find(ng); it != self.gram_entries_.end()) { return it->second.get(); @@ -299,12 +298,17 @@ class ngram_index list->for_each_documents(f); } - [[nodiscard]] - bool empty() const noexcept + [[nodiscard]] bool empty() const noexcept { return gram_entries_.empty() && side_entries_.empty(); } + void clear() noexcept + { + gram_entries_.clear(); + side_entries_.clear(); + } + void merge_new_entries(std::vector, std::unique_ptr>>& pending) { if (pending.empty()) return; // vocabulary saturated @@ -381,8 +385,13 @@ struct ngram_index_storage this->template append_index<2>(doc_id, input); } - [[nodiscard]] - bool empty() const noexcept + void clear() noexcept + { + uni_data_.clear(); + bi_data_.clear(); + } + + [[nodiscard]] bool empty() const noexcept { return this->template get_data<1>().idx.empty() && @@ -413,6 +422,13 @@ struct ngram_index_storage std::vector, std::unique_ptr>> batch_pending; + + void clear() noexcept + { + idx.clear(); + batch_grams.clear(); + batch_pending.clear(); + } }; ngram_index_storage_data<1> uni_data_; @@ -723,16 +739,21 @@ template class ngram_database { public: - [[nodiscard]] - ngram_document_id add_document(std::basic_string_view const doc_text) + [[nodiscard]] ngram_document_id add_document(std::basic_string_view const doc_text) { - ngram_document_id const doc_id{max_doc_id_}; - max_doc_id_ = ngram_document_id{std::to_underlying(max_doc_id_) + 1u}; + ngram_document_id const doc_id{next_doc_id_}; + next_doc_id_ = ngram_document_id{std::to_underlying(next_doc_id_) + 1u}; store_.append_index(doc_id, doc_text); return doc_id; } + void clear() noexcept + { + next_doc_id_ = 0_doc_id; + store_.clear(); + } + template void find_occurrences(ngram ng, std::vector& occs) const { @@ -741,8 +762,7 @@ class ngram_database idx.find_occurrences(ng, occs); } - [[nodiscard]] - ngram_search_result search(ngram_search_query const& query) const + [[nodiscard]] ngram_search_result search(ngram_search_query const& query) const { if (query.empty()) return {}; if (store_.empty()) return {}; @@ -899,7 +919,7 @@ class ngram_database search_res.remove_stale_document_matches(word_id, current_ngram); } - ngram_document_id max_doc_id_{0_doc_id}; + ngram_document_id next_doc_id_{0_doc_id}; detail::ngram_index_storage store_; }; From db20850475bd026dc0b9010e8f655533b2549e31 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Sat, 15 Aug 2026 20:41:44 +0900 Subject: [PATCH 11/16] Split ngram_search_query.hpp and ngram_database.hpp --- .../iris/{ngram.hpp => ngram_database.hpp} | 55 +-------------- include/iris/ngram_search_query.hpp | 67 +++++++++++++++++++ test/ngram_test.hpp | 3 +- 3 files changed, 72 insertions(+), 53 deletions(-) rename include/iris/{ngram.hpp => ngram_database.hpp} (95%) create mode 100644 include/iris/ngram_search_query.hpp diff --git a/include/iris/ngram.hpp b/include/iris/ngram_database.hpp similarity index 95% rename from include/iris/ngram.hpp rename to include/iris/ngram_database.hpp index 02d6083..b1f41b8 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram_database.hpp @@ -1,9 +1,10 @@ -#ifndef IRIS_ZZ_NGRAM_HPP -#define IRIS_ZZ_NGRAM_HPP +#ifndef IRIS_ZZ_NGRAM_DATABASE_HPP +#define IRIS_ZZ_NGRAM_DATABASE_HPP // SPDX-License-Identifier: MIT #include +#include #include #include #include @@ -512,56 +513,6 @@ struct ngram_index_storage } // detail - -template -struct ngram_search_query -{ - explicit ngram_search_query(std::basic_string_view input_sv) - { - std::basic_string input{input_sv}; - iris::compact_spaces(input); - if (input.empty()) return; - - words_ = input - | std::views::split(detail::string_algo_traits::space) - | std::views::transform([](auto const& r) { - return std::basic_string{std::from_range, r}; - }) - | std::ranges::to(); - - std::ranges::sort(words_); - { - auto const [first, last] = std::ranges::unique(words_); - words_.erase(first, last); - } - } - - // ------------------------------------------ - - [[nodiscard]] - auto const& words() const noexcept - { - return words_; - } - - [[nodiscard]] bool empty() const noexcept - { - return words_.empty(); - } - - [[nodiscard]] bool operator==(ngram_search_query const& other) const noexcept - { - return words_ == other.words_; - } - -private: - std::vector> words_; -}; - -template -ngram_search_query(CharT const(&)[N]) -> ngram_search_query; - - struct [[nodiscard]] ngram_search_word_match { ngram_search_word_match() = default; diff --git a/include/iris/ngram_search_query.hpp b/include/iris/ngram_search_query.hpp new file mode 100644 index 0000000..133e2eb --- /dev/null +++ b/include/iris/ngram_search_query.hpp @@ -0,0 +1,67 @@ +#ifndef IRIS_ZZ_NGRAM_SEARCH_QUERY_HPP +#define IRIS_ZZ_NGRAM_SEARCH_QUERY_HPP + +// SPDX-License-Identifier: MIT + +#include +#include + +#include +#include +#include +#include +#include + +namespace iris { + +template +struct ngram_search_query +{ + explicit ngram_search_query(std::basic_string_view input_sv) + { + std::basic_string input{input_sv}; + iris::compact_spaces(input); + if (input.empty()) return; + + words_ = input + | std::views::split(detail::string_algo_traits::space) + | std::views::transform([](auto const& r) { + return std::basic_string{std::from_range, r}; + }) + | std::ranges::to(); + + std::ranges::sort(words_); + { + auto const [first, last] = std::ranges::unique(words_); + words_.erase(first, last); + } + } + + // ------------------------------------------ + + [[nodiscard]] + auto const& words() const noexcept + { + return words_; + } + + [[nodiscard]] bool empty() const noexcept + { + return words_.empty(); + } + + [[nodiscard]] bool operator==(ngram_search_query const& other) const noexcept + { + return words_ == other.words_; + } + +private: + std::vector> words_; +}; + +template +ngram_search_query(CharT const(&)[N]) -> ngram_search_query; + +} // iris + +#endif diff --git a/test/ngram_test.hpp b/test/ngram_test.hpp index ff49a87..f7367f5 100644 --- a/test/ngram_test.hpp +++ b/test/ngram_test.hpp @@ -5,7 +5,8 @@ #include "iris_test.hpp" -#include +#include +#include #include #include From 5ffa08e40ddabfb1059f44ef09b41e9af32856dd Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Sun, 16 Aug 2026 03:30:42 +0900 Subject: [PATCH 12/16] Improve interface --- include/iris/ngram_database.hpp | 48 +++++++++++++++++------------ include/iris/ngram_search_query.hpp | 19 ++++++++++++ test/ngram_test.hpp | 3 +- 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/include/iris/ngram_database.hpp b/include/iris/ngram_database.hpp index b1f41b8..f4507b0 100644 --- a/include/iris/ngram_database.hpp +++ b/include/iris/ngram_database.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include +#include namespace iris { @@ -225,6 +227,7 @@ struct ngram_posting_list 0 ); } + ++postings.back().pos_count; positions.emplace_back(pos); } @@ -699,10 +702,25 @@ class ngram_database return doc_id; } + [[nodiscard]] bool is_visible(ngram_document_id const doc_id) const noexcept + { + return !invisible_documents_.contains(doc_id); + } + + void set_visible(ngram_document_id const doc_id, bool const flag) + { + if (flag) { + invisible_documents_.erase(doc_id); + } else { + invisible_documents_.emplace(doc_id); + } + } + void clear() noexcept { next_doc_id_ = 0_doc_id; store_.clear(); + invisible_documents_.clear(); } template @@ -713,12 +731,10 @@ class ngram_database idx.find_occurrences(ng, occs); } - [[nodiscard]] ngram_search_result search(ngram_search_query const& query) const + bool search(ngram_search_query const& query, ngram_search_result& search_res) const { - if (query.empty()) return {}; - if (store_.empty()) return {}; - - ngram_search_result search_res; + if (query.empty()) return false; + if (store_.empty()) return false; int word_id = 0; auto it = query.words().begin(); @@ -726,7 +742,7 @@ class ngram_database this->search_word(search_res, word_id++, *it++); if (search_res.empty()) { search_res.reset(); // remove tombstones - return search_res; + return false; } for (; it != query.words().end(); ++it) { @@ -737,7 +753,7 @@ class ngram_database break; } } - return search_res; + return !search_res.empty(); } private: @@ -761,6 +777,7 @@ class ngram_database if constexpr (IsFirstWord) { store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { + if (!is_visible(doc_id)) return; (void)search_res.init_word_matches(doc_id, word_id, positions); }); if (search_res.empty()) return; @@ -768,6 +785,7 @@ class ngram_database } else { std::size_t available_doc_count = 0; store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { + if (!is_visible(doc_id)) return; if (search_res.init_word_matches(doc_id, word_id, positions)) { ++available_doc_count; } @@ -781,6 +799,8 @@ class ngram_database unsigned current_ngram = 1; auto const do_search = [&](int remaining_chars) { return [&, remaining_chars, overlapping_chars = int(N) - remaining_chars](ngram_document_id const doc_id, std::span const positions) { + if (!is_visible(doc_id)) return detail::search_continuation::proceed; + // Find the existing match set from the previous iteration. // If none exists, any subsequent characters of the document will not match. // @@ -872,6 +892,7 @@ class ngram_database ngram_document_id next_doc_id_{0_doc_id}; detail::ngram_index_storage store_; + std::unordered_set invisible_documents_; }; } // iris @@ -901,19 +922,6 @@ struct std::formatter } }; -template -struct std::formatter, CharT> - : iris::no_spec_formatter -{ - template - Ctx::iterator format(iris::ngram_search_query const& query, Ctx& ctx) const - { - return std::format_to(ctx.out(), "{}", query.words() | std::views::transform([](std::u32string_view ustr) { - return iris::unicode::transcode(ustr); - })); - } -}; - template struct std::formatter : iris::no_spec_formatter diff --git a/include/iris/ngram_search_query.hpp b/include/iris/ngram_search_query.hpp index 133e2eb..e04d8e8 100644 --- a/include/iris/ngram_search_query.hpp +++ b/include/iris/ngram_search_query.hpp @@ -5,11 +5,15 @@ #include #include +#include + +#include #include #include #include #include +#include #include namespace iris { @@ -17,6 +21,8 @@ namespace iris { template struct ngram_search_query { + ngram_search_query() = default; + explicit ngram_search_query(std::basic_string_view input_sv) { std::basic_string input{input_sv}; @@ -64,4 +70,17 @@ ngram_search_query(CharT const(&)[N]) -> ngram_search_query; } // iris +template +struct std::formatter, CharT> + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram_search_query const& query, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{}", query.words() | std::views::transform([](std::u32string_view ustr) { + return iris::unicode::transcode(ustr); + })); + } +}; + #endif diff --git a/test/ngram_test.hpp b/test/ngram_test.hpp index f7367f5..5dedc30 100644 --- a/test/ngram_test.hpp +++ b/test/ngram_test.hpp @@ -52,7 +52,8 @@ struct std::formatter #define IRIS_CHECK_SEARCH(query_input, ...) do { \ iris::ngram_search_query const query{U ## query_input}; \ - auto const search_res = ngram_db.search(query); \ + iris::ngram_search_result search_res; \ + ngram_db.search(query, search_res); \ auto const& doc_matches = search_res.doc_matches(); \ \ std::vector const expected_doc_matches{ \ From 22ea7b8e1685458b0519a89bd24d12cf3ddf5524 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Mon, 17 Aug 2026 03:13:51 +0900 Subject: [PATCH 13/16] Reorganize library structure --- .../database.hpp} | 361 +++++------------- include/iris/ngram/gram.hpp | 168 ++++++++ include/iris/ngram/id.hpp | 31 ++ .../search_query.hpp} | 20 +- test/CMakeLists.txt | 62 +-- test/{ => ngram}/ngram.cpp | 42 +- test/{ => ngram}/ngram_test.hpp | 18 +- .../search_2.cpp} | 14 +- .../search_3.cpp} | 16 +- .../search_4_dep.cpp} | 48 +-- test/rvariant/CMakeLists.txt | 4 +- test/unicode/string/CMakeLists.txt | 4 +- 12 files changed, 421 insertions(+), 367 deletions(-) rename include/iris/{ngram_database.hpp => ngram/database.hpp} (64%) create mode 100644 include/iris/ngram/gram.hpp create mode 100644 include/iris/ngram/id.hpp rename include/iris/{ngram_search_query.hpp => ngram/search_query.hpp} (74%) rename test/{ => ngram}/ngram.cpp (85%) rename test/{ => ngram}/ngram_test.hpp (73%) rename test/{ngram_search_2.cpp => ngram/search_2.cpp} (86%) rename test/{ngram_search_3.cpp => ngram/search_3.cpp} (94%) rename test/{ngram_search_4_dep.cpp => ngram/search_4_dep.cpp} (89%) diff --git a/include/iris/ngram_database.hpp b/include/iris/ngram/database.hpp similarity index 64% rename from include/iris/ngram_database.hpp rename to include/iris/ngram/database.hpp index f4507b0..8173ecd 100644 --- a/include/iris/ngram_database.hpp +++ b/include/iris/ngram/database.hpp @@ -4,11 +4,13 @@ // SPDX-License-Identifier: MIT #include -#include + +#include +#include +#include + #include #include -#include -#include #include #include @@ -16,10 +18,8 @@ #include #include #include -#include #include #include -#include #include #include #include @@ -28,173 +28,18 @@ #include #include -#include -#include - -namespace iris { -enum struct ngram_document_id : unsigned {}; +namespace iris::ngram { -struct ngram_occurrence +struct gram_occurrence { - ngram_document_id doc_id; + document_id doc_id; int pos; - [[nodiscard]] constexpr bool operator==(ngram_occurrence const&) const noexcept = default; - [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram_occurrence const&) const noexcept = default; -}; - -namespace detail { - -template struct ngram_value; -template<> struct ngram_value<1> { using type = std::uint8_t; }; -template<> struct ngram_value<2> { using type = std::uint16_t; }; -template<> struct ngram_value<4> { using type = std::uint32_t; }; -template<> struct ngram_value<8> { using type = std::uint64_t; }; - -template -using ngram_value_t = ngram_value::type; - -} // detail - -template -struct ngram -{ - static_assert(N >= 3); - - std::array data; - - template - constexpr void copy_n(It it) - noexcept(noexcept(*it++)) - { - std::ranges::copy_n(it, N, data.begin()); - } - template - [[nodiscard]] static constexpr ngram from_copy_n(It it) - noexcept(noexcept(std::declval().copy_n(std::move(it)))) - { - ngram ng; - ng.copy_n(std::move(it)); - return ng; - } - - template - constexpr void shift_copy(It it, int const remaining_chars) - noexcept( - noexcept(std::shift_left(data.begin(), data.end(), remaining_chars)) && - noexcept(std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars))) - ) - { - assert(remaining_chars < int(N)); - std::shift_left(data.begin(), data.end(), remaining_chars); - std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars)); - } - - template - [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept - { - static_assert(Len == N + 1); - assert(chars[Len - 1] == static_cast(0)); - return ngram::from_copy_n(std::ranges::begin(chars)); - } - - [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; - [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; -}; - -template -struct ngram<1, CharT> -{ - CharT data; - - template - constexpr void copy_n(It it) - noexcept(noexcept(*it)) - { - data = *it; - } - template - [[nodiscard]] static constexpr ngram from_copy_n(It it) - noexcept(noexcept(std::declval().copy_n(std::move(it)))) - { - ngram ng; - ng.copy_n(std::move(it)); - return ng; - } - - template - [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept - { - static_assert(Len == 1 + 1); - assert(chars[Len - 1] == static_cast(0)); - return ngram::from_copy_n(std::ranges::begin(chars)); - } - - [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; - [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; -}; - -template -struct ngram<2, CharT> -{ - using value_type = detail::ngram_value_t; - value_type data; - - template - constexpr void copy_n(It it) - noexcept(noexcept(*it++)) - { - using uchar = std::make_unsigned_t; - data = value_type(static_cast(*it++)) << (sizeof(CharT) * 8); - data |= value_type(static_cast(*it)); - } - template - [[nodiscard]] static constexpr ngram from_copy_n(It it) - noexcept(noexcept(std::declval().copy_n(std::move(it)))) - { - ngram ng; - ng.copy_n(std::move(it)); - return ng; - } - - template - constexpr void shift_copy(It it, int const remaining_chars) - noexcept(noexcept(*it)) - { - assert(remaining_chars == 1); - (void)remaining_chars; - data = (data << (sizeof(CharT) * 8)) | value_type(static_cast>(*it)); - } - - template - [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept - { - static_assert(Len == 2 + 1); - assert(chars[Len - 1] == static_cast(0)); - return ngram::from_copy_n(std::ranges::begin(chars)); - } - - [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; - [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; + [[nodiscard]] constexpr bool operator==(gram_occurrence const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(gram_occurrence const&) const noexcept = default; }; -inline namespace ngram_literals { - -[[nodiscard]] constexpr ngram_document_id operator ""_doc_id(unsigned long long id) noexcept -{ - return ngram_document_id{static_cast>(id)}; -} - -} // ngram_literals - -template -[[nodiscard]] ngram to_ngram(CharT const (&chars)[N]) noexcept -{ - return ngram::from_c_array(chars); -} - - namespace detail { enum struct [[nodiscard]] search_continuation : bool @@ -203,41 +48,32 @@ enum struct [[nodiscard]] search_continuation : bool proceed = true, }; -struct ngram_posting -{ - ngram_document_id doc_id; - unsigned pos_offset = 0; - unsigned pos_count = 0; -}; - -struct ngram_posting_list +class posting_list { - std::vector postings; - std::vector positions; - - void append(ngram_document_id const doc_id, int pos) +public: + void append(document_id const doc_id, int pos) { - if (postings.empty() || postings.back().doc_id != doc_id) { - if (!postings.empty() && postings.back().doc_id > doc_id) { + if (postings_.empty() || postings_.back().doc_id != doc_id) { + if (!postings_.empty() && postings_.back().doc_id > doc_id) { throw std::invalid_argument{"documents must be indexed in non-decreasing order of document ID"}; } - postings.emplace_back( + postings_.emplace_back( doc_id, - static_cast(positions.size()), + static_cast(positions_.size()), 0 ); } - ++postings.back().pos_count; - positions.emplace_back(pos); + ++postings_.back().pos_count; + positions_.emplace_back(pos); } - void to_occurrence_list(std::vector& occs) const + void to_occurrence_list(std::vector& occs) const { occs.clear(); - for (auto const& posting : postings) { + for (auto const& posting : postings_) { for (std::size_t i = posting.pos_offset; i < posting.pos_offset + posting.pos_count; ++i) { - occs.emplace_back(posting.doc_id, positions[i]); + occs.emplace_back(posting.doc_id, positions_[i]); } } } @@ -245,16 +81,16 @@ struct ngram_posting_list template void for_each_documents(F&& f) const { - static_assert(std::invocable>); + static_assert(std::invocable>); constexpr bool f_returns_continuation = std::same_as< - std::invoke_result_t>, + std::invoke_result_t>, search_continuation >; - for (auto const& posting : postings) { + for (auto const& posting : postings_) { std::span const posting_span{ - positions.begin() + posting.pos_offset, + positions_.begin() + posting.pos_offset, static_cast(posting.pos_count) }; @@ -266,16 +102,27 @@ struct ngram_posting_list } } } + +private: + struct posting_t + { + document_id doc_id; + unsigned pos_offset = 0; + unsigned pos_count = 0; + }; + + std::vector postings_; + std::vector positions_; }; -template -class ngram_index +template +class gram_index { - using entry_map = std::flat_map, std::unique_ptr>; + using entry_map = std::flat_map, std::unique_ptr>; static constexpr std::size_t side_merge_threshold = 2048; public: - [[nodiscard]] auto find_list(this auto&& self, ngram const ng) + [[nodiscard]] auto find_list(this auto&& self, gram const ng) { if (auto const it = self.gram_entries_.find(ng); it != self.gram_entries_.end()) { return it->second.get(); @@ -286,7 +133,7 @@ class ngram_index return static_cast(nullptr); } - void find_occurrences(ngram const ng, std::vector& occs) const + void find_occurrences(gram const ng, std::vector& occs) const { occs.clear(); auto const* list = this->find_list(ng); @@ -295,7 +142,7 @@ class ngram_index } template - void search(ngram const ng, F&& f) const + void search(gram const ng, F&& f) const { auto const* list = this->find_list(ng); if (!list) return; @@ -313,7 +160,7 @@ class ngram_index side_entries_.clear(); } - void merge_new_entries(std::vector, std::unique_ptr>>& pending) + void merge_new_entries(std::vector, std::unique_ptr>>& pending) { if (pending.empty()) return; // vocabulary saturated @@ -371,19 +218,19 @@ class ngram_index }; template -struct ngram_pos_t +struct gram_pos_t { - ngram ng; + gram ng; int pos; - [[nodiscard]] constexpr bool operator==(ngram_pos_t const&) const noexcept = default; - [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram_pos_t const&) const noexcept = default; + [[nodiscard]] constexpr bool operator==(gram_pos_t const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(gram_pos_t const&) const noexcept = default; }; -template -struct ngram_index_storage +template +struct index_storage { - void append_index(ngram_document_id const doc_id, std::basic_string_view const input) + void append_index(document_id const doc_id, std::basic_string_view const input) { this->template append_index<1>(doc_id, input); this->template append_index<2>(doc_id, input); @@ -403,7 +250,7 @@ struct ngram_index_storage } template - void search(ngram const ng, F&& f) const + void search(gram const ng, F&& f) const { this->template get_data().idx.search(ng, f); } @@ -416,15 +263,15 @@ struct ngram_index_storage private: template - struct ngram_index_storage_data + struct gram_index_storage { - ngram_index idx; + gram_index idx; // Caches - std::vector, default_init_allocator>> + std::vector, default_init_allocator>> batch_grams; - std::vector, std::unique_ptr>> + std::vector, std::unique_ptr>> batch_pending; void clear() noexcept @@ -435,8 +282,8 @@ struct ngram_index_storage } }; - ngram_index_storage_data<1> uni_data_; - ngram_index_storage_data<2> bi_data_; + gram_index_storage<1> uni_data_; + gram_index_storage<2> bi_data_; template [[nodiscard]] auto& get_data(this auto& self IRIS_LIFETIMEBOUND) noexcept @@ -452,12 +299,12 @@ struct ngram_index_storage template void append_index( - ngram_document_id const doc_id, + document_id const doc_id, std::basic_string_view const input ) { if (input.size() < N) return; - ngram_index_storage_data& data = this->template get_data(); + gram_index_storage& data = this->template get_data(); // Naive per-gram insertion into flat_map is expensive: each *new* key // shifts the underlying vectors, so building an index of vocabulary @@ -516,15 +363,15 @@ struct ngram_index_storage } // detail -struct [[nodiscard]] ngram_search_word_match +struct [[nodiscard]] search_word_match { - ngram_search_word_match() = default; + search_word_match() = default; - explicit ngram_search_word_match(int word_id) + explicit search_word_match(int word_id) : word_id(word_id) {} - ngram_search_word_match(int word_id, std::initializer_list> spans) + search_word_match(int word_id, std::initializer_list> spans) : word_id(word_id) , spans(spans) {} @@ -534,23 +381,23 @@ struct [[nodiscard]] ngram_search_word_match std::vector> spans; [[nodiscard]] - bool operator==(ngram_search_word_match const& other) const noexcept + bool operator==(search_word_match const& other) const noexcept { return word_id == other.word_id && spans == other.spans; } }; -class [[nodiscard]] ngram_search_result +class [[nodiscard]] search_result { - using doc_matches_map = std::flat_map>; + using doc_matches_map = std::flat_map>; struct word_matches_handle { doc_matches_map::iterator doc_it; - ngram_search_word_match* word_match = nullptr; + search_word_match* word_match = nullptr; [[nodiscard]] - ngram_search_word_match* operator->() const noexcept + search_word_match* operator->() const noexcept { return word_match; } @@ -563,7 +410,7 @@ class [[nodiscard]] ngram_search_result public: [[nodiscard]] - bool has_document(ngram_document_id const doc_id) const noexcept + bool has_document(document_id const doc_id) const noexcept { auto const it = doc_matches_.find(doc_id); // An entry with no word matches is a tombstone (soft-erased document @@ -577,7 +424,7 @@ class [[nodiscard]] ngram_search_result // Returns whether search must continue template [[nodiscard]] - bool init_word_matches(ngram_document_id const doc_id, int const word_id, std::span const positions) + bool init_word_matches(document_id const doc_id, int const word_id, std::span const positions) { assert(!positions.empty()); @@ -594,7 +441,7 @@ class [[nodiscard]] ngram_search_result if (doc_matches_it->second.empty()) return false; // tombstoned (soft-erased) document; skip } - assert(!std::ranges::contains(doc_matches_it->second, word_id, &ngram_search_word_match::word_id)); + assert(!std::ranges::contains(doc_matches_it->second, word_id, &search_word_match::word_id)); auto& word_match = doc_matches_it->second.emplace_back(word_id); word_match.spans.assign_range(positions | std::views::transform([](int const pos) -> interval { return {pos, pos + static_cast(N)}; @@ -603,7 +450,7 @@ class [[nodiscard]] ngram_search_result } [[nodiscard]] - word_matches_handle get_word_matches(ngram_document_id const doc_id, int const word_id) + word_matches_handle get_word_matches(document_id const doc_id, int const word_id) { auto const doc_matches_it = doc_matches_.find(doc_id); if (doc_matches_it == doc_matches_.end()) return {}; @@ -618,7 +465,7 @@ class [[nodiscard]] ngram_search_result assert( doc_matches_it->second.empty() || // Make sure the matching element does not exist at the position except for *back* - !std::ranges::contains(doc_matches_it->second, word_id, &ngram_search_word_match::word_id) + !std::ranges::contains(doc_matches_it->second, word_id, &search_word_match::word_id) ); return {}; } @@ -646,7 +493,7 @@ class [[nodiscard]] ngram_search_result for (std::size_t in = 0; in < keys.size(); ++in) { auto& word_matches = values[in]; bool is_word_survived = false; - std::erase_if(word_matches, [&](ngram_search_word_match const& wm) { + std::erase_if(word_matches, [&](search_word_match const& wm) { if (wm.word_id != word_id) return false; if (wm.successful_ngrams != expected_ngrams) return true; is_word_survived = true; @@ -690,24 +537,24 @@ class [[nodiscard]] ngram_search_result }; template -class ngram_database +class database { public: - [[nodiscard]] ngram_document_id add_document(std::basic_string_view const doc_text) + [[nodiscard]] document_id add_document(std::basic_string_view const doc_text) { - ngram_document_id const doc_id{next_doc_id_}; - next_doc_id_ = ngram_document_id{std::to_underlying(next_doc_id_) + 1u}; + document_id const doc_id{next_doc_id_}; + next_doc_id_ = document_id{std::to_underlying(next_doc_id_) + 1u}; store_.append_index(doc_id, doc_text); return doc_id; } - [[nodiscard]] bool is_visible(ngram_document_id const doc_id) const noexcept + [[nodiscard]] bool is_visible(document_id const doc_id) const noexcept { return !invisible_documents_.contains(doc_id); } - void set_visible(ngram_document_id const doc_id, bool const flag) + void set_visible(document_id const doc_id, bool const flag) { if (flag) { invisible_documents_.erase(doc_id); @@ -724,14 +571,14 @@ class ngram_database } template - void find_occurrences(ngram ng, std::vector& occs) const + void find_occurrences(gram ng, std::vector& occs) const { occs.clear(); auto const& idx = store_.template get_index(); idx.find_occurrences(ng, occs); } - bool search(ngram_search_query const& query, ngram_search_result& search_res) const + bool search(search_query const& query, search_result& search_res) const { if (query.empty()) return false; if (store_.empty()) return false; @@ -758,7 +605,7 @@ class ngram_database private: template - void search_word(ngram_search_result& search_res, int const word_id, std::basic_string_view const word) const + void search_word(search_result& search_res, int const word_id, std::basic_string_view const word) const { assert(!word.empty()); @@ -770,13 +617,13 @@ class ngram_database } template - void search_word_impl(ngram_search_result& search_res, int const word_id, std::basic_string_view const word) const + void search_word_impl(search_result& search_res, int const word_id, std::basic_string_view const word) const { assert(word.size() >= N); - auto ng = ngram::from_copy_n(word.begin()); + auto ng = gram::from_copy_n(word.begin()); if constexpr (IsFirstWord) { - store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { + store_.search(ng, [&](document_id const doc_id, std::span const positions) { if (!is_visible(doc_id)) return; (void)search_res.init_word_matches(doc_id, word_id, positions); }); @@ -784,7 +631,7 @@ class ngram_database } else { std::size_t available_doc_count = 0; - store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { + store_.search(ng, [&](document_id const doc_id, std::span const positions) { if (!is_visible(doc_id)) return; if (search_res.init_word_matches(doc_id, word_id, positions)) { ++available_doc_count; @@ -798,7 +645,7 @@ class ngram_database unsigned current_ngram = 1; auto const do_search = [&](int remaining_chars) { - return [&, remaining_chars, overlapping_chars = int(N) - remaining_chars](ngram_document_id const doc_id, std::span const positions) { + return [&, remaining_chars, overlapping_chars = int(N) - remaining_chars](document_id const doc_id, std::span const positions) { if (!is_visible(doc_id)) return detail::search_continuation::proceed; // Find the existing match set from the previous iteration. @@ -812,9 +659,9 @@ class ngram_database // Prevent *resurrecting* the false-positive match on "match -> unmatch -> match" pattern. // For example, when the document is "abef" and the query is "abXXef", - // - ngram{"ab"} -> match (successful_ngrams = 1) - // - ngram{"XX"} -> no match (successful_ngrams is untouched) - // - ngram{"ef"} -> successful_ngrams does not match current_ngram! + // - gram{"ab"} -> match (successful_ngrams = 1) + // - gram{"XX"} -> no match (successful_ngrams is untouched) + // - gram{"ef"} -> successful_ngrams does not match current_ngram! if (word_match->successful_ngrams != current_ngram) { search_res.erase_document(word_match); if (search_res.empty()) return detail::search_continuation::abort; @@ -890,44 +737,44 @@ class ngram_database search_res.remove_stale_document_matches(word_id, current_ngram); } - ngram_document_id next_doc_id_{0_doc_id}; - detail::ngram_index_storage store_; - std::unordered_set invisible_documents_; + document_id next_doc_id_{0_doc_id}; + detail::index_storage store_; + std::unordered_set invisible_documents_; }; -} // iris +} // iris::gram template -struct std::formatter - : std::formatter, CharT> +struct std::formatter + : std::formatter, CharT> { - using base_type = std::formatter, CharT>; + using base_type = std::formatter, CharT>; template - Ctx::iterator format(iris::ngram_document_id doc_id, Ctx& ctx) const + Ctx::iterator format(iris::ngram::document_id doc_id, Ctx& ctx) const { return base_type::format(std::to_underlying(doc_id), ctx); } }; template -struct std::formatter +struct std::formatter : iris::no_spec_formatter { template - Ctx::iterator format(iris::ngram_occurrence const& occ, Ctx& ctx) const + Ctx::iterator format(iris::ngram::gram_occurrence const& occ, Ctx& ctx) const { return std::format_to(ctx.out(), "{}:{}", occ.doc_id, occ.pos); } }; template -struct std::formatter +struct std::formatter : iris::no_spec_formatter { template - Ctx::iterator format(iris::ngram_search_word_match const& word_match, Ctx& ctx) const + Ctx::iterator format(iris::ngram::search_word_match const& word_match, Ctx& ctx) const { return std::format_to(ctx.out(), "{{word: #{}, spans: {}}}", word_match.word_id, word_match.spans); } diff --git a/include/iris/ngram/gram.hpp b/include/iris/ngram/gram.hpp new file mode 100644 index 0000000..f13aeea --- /dev/null +++ b/include/iris/ngram/gram.hpp @@ -0,0 +1,168 @@ +#ifndef IRIS_ZZ_NGRAM_GRAM_HPP +#define IRIS_ZZ_NGRAM_GRAM_HPP + +// SPDX-License-Identifier: MIT + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace iris::ngram { + +namespace detail { + +template struct gram_value; +template<> struct gram_value<1> { using type = std::uint8_t; }; +template<> struct gram_value<2> { using type = std::uint16_t; }; +template<> struct gram_value<4> { using type = std::uint32_t; }; +template<> struct gram_value<8> { using type = std::uint64_t; }; + +template +using gram_value_t = gram_value::type; + +} // detail + +template +struct gram +{ + static_assert(N >= 3); + + std::array data; + + template + constexpr void copy_n(It it) + noexcept(noexcept(*it++)) + { + std::ranges::copy_n(it, N, data.begin()); + } + template + [[nodiscard]] static constexpr gram from_copy_n(It it) + noexcept(noexcept(std::declval().copy_n(std::move(it)))) + { + gram ng; + ng.copy_n(std::move(it)); + return ng; + } + + template + constexpr void shift_copy(It it, int const remaining_chars) + noexcept( + noexcept(std::shift_left(data.begin(), data.end(), remaining_chars)) && + noexcept(std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars))) + ) + { + assert(remaining_chars < int(N)); + std::shift_left(data.begin(), data.end(), remaining_chars); + std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars)); + } + + template + [[nodiscard]] static constexpr gram from_c_array(CharT const (&chars)[Len]) noexcept + { + static_assert(Len == N + 1); + assert(chars[Len - 1] == static_cast(0)); + return gram::from_copy_n(std::ranges::begin(chars)); + } + + [[nodiscard]] constexpr bool operator==(gram const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(gram const&) const noexcept = default; +}; + +template +struct gram<1, CharT> +{ + CharT data; + + template + constexpr void copy_n(It it) + noexcept(noexcept(*it)) + { + data = *it; + } + template + [[nodiscard]] static constexpr gram from_copy_n(It it) + noexcept(noexcept(std::declval().copy_n(std::move(it)))) + { + gram ng; + ng.copy_n(std::move(it)); + return ng; + } + + template + [[nodiscard]] static constexpr gram from_c_array(CharT const (&chars)[Len]) noexcept + { + static_assert(Len == 1 + 1); + assert(chars[Len - 1] == static_cast(0)); + return gram::from_copy_n(std::ranges::begin(chars)); + } + + [[nodiscard]] constexpr bool operator==(gram const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(gram const&) const noexcept = default; +}; + +template +struct gram<2, CharT> +{ + using value_type = detail::gram_value_t; + value_type data; + + template + constexpr void copy_n(It it) + noexcept(noexcept(*it++)) + { + using uchar = std::make_unsigned_t; + data = value_type(static_cast(*it++)) << (sizeof(CharT) * 8); + data |= value_type(static_cast(*it)); + } + template + [[nodiscard]] static constexpr gram from_copy_n(It it) + noexcept(noexcept(std::declval().copy_n(std::move(it)))) + { + gram ng; + ng.copy_n(std::move(it)); + return ng; + } + + template + constexpr void shift_copy(It it, int const remaining_chars) + noexcept(noexcept(*it)) + { + assert(remaining_chars == 1); + (void)remaining_chars; + data = (data << (sizeof(CharT) * 8)) | value_type(static_cast>(*it)); + } + + template + [[nodiscard]] static constexpr gram from_c_array(CharT const (&chars)[Len]) noexcept + { + static_assert(Len == 2 + 1); + assert(chars[Len - 1] == static_cast(0)); + return gram::from_copy_n(std::ranges::begin(chars)); + } + + [[nodiscard]] constexpr bool operator==(gram const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(gram const&) const noexcept = default; +}; + +} // iris::ngram + +namespace iris { + +template +[[nodiscard]] ngram::gram to_ngram(CharT const (&chars)[N]) noexcept +{ + return ngram::gram::from_c_array(chars); +} + +} // iris + +#endif diff --git a/include/iris/ngram/id.hpp b/include/iris/ngram/id.hpp new file mode 100644 index 0000000..30bc03f --- /dev/null +++ b/include/iris/ngram/id.hpp @@ -0,0 +1,31 @@ +#ifndef IRIS_ZZ_NGRAM_ID_HPP +#define IRIS_ZZ_NGRAM_ID_HPP + +// SPDX-License-Identifier: MIT + +#include + +#include + +#include + +namespace iris::ngram { + +enum struct document_id : std::uint32_t {}; + +} // iris::ngram + +namespace iris { + +inline namespace ngram_literals { + +[[nodiscard]] constexpr ngram::document_id operator ""_doc_id(unsigned long long id) noexcept +{ + return ngram::document_id{static_cast>(id)}; +} + +} // ngram_literals + +} // iris + +#endif diff --git a/include/iris/ngram_search_query.hpp b/include/iris/ngram/search_query.hpp similarity index 74% rename from include/iris/ngram_search_query.hpp rename to include/iris/ngram/search_query.hpp index e04d8e8..203d793 100644 --- a/include/iris/ngram_search_query.hpp +++ b/include/iris/ngram/search_query.hpp @@ -16,21 +16,21 @@ #include #include -namespace iris { +namespace iris::ngram { template -struct ngram_search_query +struct search_query { - ngram_search_query() = default; + search_query() = default; - explicit ngram_search_query(std::basic_string_view input_sv) + explicit search_query(std::basic_string_view input_sv) { std::basic_string input{input_sv}; iris::compact_spaces(input); if (input.empty()) return; words_ = input - | std::views::split(detail::string_algo_traits::space) + | std::views::split(iris::detail::string_algo_traits::space) | std::views::transform([](auto const& r) { return std::basic_string{std::from_range, r}; }) @@ -56,7 +56,7 @@ struct ngram_search_query return words_.empty(); } - [[nodiscard]] bool operator==(ngram_search_query const& other) const noexcept + [[nodiscard]] bool operator==(search_query const& other) const noexcept { return words_ == other.words_; } @@ -66,16 +66,16 @@ struct ngram_search_query }; template -ngram_search_query(CharT const(&)[N]) -> ngram_search_query; +search_query(CharT const(&)[N]) -> search_query; -} // iris +} // iris::gram template -struct std::formatter, CharT> +struct std::formatter, CharT> : iris::no_spec_formatter { template - Ctx::iterator format(iris::ngram_search_query const& query, Ctx& ctx) const + Ctx::iterator format(iris::ngram::search_query const& query, Ctx& ctx) const { return std::format_to(ctx.out(), "{}", query.words() | std::views::transform([](std::u32string_view ustr) { return iris::unicode::transcode(ustr); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0f97bbb..7ea03d5 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -69,23 +69,6 @@ target_link_libraries(Catch2WithMain PRIVATE iris_cxx_test_external) target_link_libraries(iris_cxx_test INTERFACE Catch2::Catch2) - -# ----------------------------------------------------------------- -# Iris internal test targets - -add_library(_iris_internal_test INTERFACE) -target_include_directories(_iris_internal_test INTERFACE ${CMAKE_CURRENT_LIST_DIR}) - -if(MSVC) - target_sources(_iris_internal_test INTERFACE "${CMAKE_CURRENT_LIST_DIR}/cpp.hint") -endif() - -function(iris_define_internal_test test_name) - iris_define_test(${test_name} ${ARGN}) - target_link_libraries(${test_name}_test PRIVATE _iris_internal_test) -endfunction() - - # ----------------------------------------------------------------- # Common CMake utilities for testing @@ -167,14 +150,33 @@ function(iris_define_library_test library_type test_name srcs) _iris_define_test_impl(${test_name} Catch2::Catch2) endfunction() +# ----------------------------------------------------------------- +# Iris internal test targets + +add_library(_iris_internal_test_base INTERFACE) +target_include_directories(_iris_internal_test_base INTERFACE ${CMAKE_CURRENT_LIST_DIR}) +if(MSVC) + target_sources(_iris_internal_test_base INTERFACE "${CMAKE_CURRENT_LIST_DIR}/cpp.hint") +endif() + +function(iris_define_internal_test test_name) + iris_define_test(iris_${test_name} ${ARGN}) + target_link_libraries(iris_${test_name}_test PRIVATE _iris_internal_test_base) + target_sources(iris_${test_name}_test PRIVATE FILE_SET HEADERS BASE_DIRS ${CMAKE_CURRENT_FUNCTION_LIST_DIR} FILES ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/iris_test.hpp) + set_target_properties(iris_${test_name}_test PROPERTIES FOLDER "test/iris") +endfunction() + +function(iris_define_internal_subdir_test subdir test_name) + list(TRANSFORM ARGN PREPEND "${subdir}/") + iris_define_internal_test(${subdir}_${test_name} ${ARGN}) + set_target_properties(iris_${subdir}_${test_name}_test PROPERTIES FOLDER "test/iris/${subdir}") +endfunction() # ----------------------------------------------------------------- # Iris tests if(PROJECT_IS_TOP_LEVEL) if(NOT DEFINED IRIS_CI_COMPONENT OR IRIS_CI_COMPONENT STREQUAL iris) - add_subdirectory(rvariant) - set( IRIS_TEST_IRIS_TESTS core @@ -189,18 +191,24 @@ if(PROJECT_IS_TOP_LEVEL) interval_algo interval_set snippet - ngram - ngram_search_2 - ngram_search_3 - ngram_search_4_dep ) - foreach(test_name IN LISTS IRIS_TEST_IRIS_TESTS) - iris_define_internal_test(iris_${test_name} ${test_name}.cpp) - iris_define_test_headers(iris_${test_name} iris_test.hpp) - set_target_properties(iris_${test_name}_test PROPERTIES FOLDER "test/iris") + iris_define_internal_test(${test_name} ${test_name}.cpp) + endforeach() + + set( + IRIS_TEST_NGRAM_TESTS + ngram + search_2 + search_3 + search_4_dep + ) + foreach(test_name IN LISTS IRIS_TEST_NGRAM_TESTS) + iris_define_internal_subdir_test(ngram ${test_name} ${test_name}.cpp) + iris_define_test_headers(iris_ngram_${test_name} ngram/ngram_test.hpp) endforeach() + add_subdirectory(rvariant) add_subdirectory(unicode) endif() endif() diff --git a/test/ngram.cpp b/test/ngram/ngram.cpp similarity index 85% rename from test/ngram.cpp rename to test/ngram/ngram.cpp index 7a1d39c..6207615 100644 --- a/test/ngram.cpp +++ b/test/ngram/ngram.cpp @@ -3,52 +3,52 @@ #include "ngram_test.hpp" [[nodiscard]] -constexpr auto make_occurrences(std::initializer_list occs) +constexpr auto make_occurrences(std::initializer_list occs) { - return std::vector{occs}; + return std::vector{occs}; } #define IRIS_CHECK_NO_OCCURRENCE(ng_str) do { \ - std::vector occs; \ + std::vector occs; \ ngram_db.find_occurrences(iris::to_ngram(U ## ng_str), occs); \ CHECK(occs.empty()); \ } while (false) #define IRIS_CHECK_OCCURRENCE(ng_str, ...) do { \ - std::vector occs; \ + std::vector occs; \ ngram_db.find_occurrences(iris::to_ngram(U ## ng_str), occs); \ CHECK(occs == make_occurrences({__VA_ARGS__})); \ } while (false) -TEST_CASE("ngram (type traits)") +TEST_CASE("gram (type traits)") { - STATIC_CHECK(std::same_as::data), char>); - STATIC_CHECK(std::same_as::data), std::uint16_t>); + STATIC_CHECK(std::same_as::data), char>); + STATIC_CHECK(std::same_as::data), std::uint16_t>); - STATIC_CHECK(std::same_as::data), char32_t>); - STATIC_CHECK(std::same_as::data), std::uint64_t>); + STATIC_CHECK(std::same_as::data), char32_t>); + STATIC_CHECK(std::same_as::data), std::uint64_t>); } -TEST_CASE("ngram (minimal input)") +TEST_CASE("gram (minimal input)") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); #endif { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U""); IRIS_CHECK_NO_OCCURRENCE("a"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"a"); IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); IRIS_CHECK_NO_OCCURRENCE("X"); IRIS_CHECK_NO_OCCURRENCE("XX"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"ab"); IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); @@ -57,7 +57,7 @@ TEST_CASE("ngram (minimal input)") IRIS_CHECK_NO_OCCURRENCE("XX"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abc"); IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); @@ -68,7 +68,7 @@ TEST_CASE("ngram (minimal input)") IRIS_CHECK_NO_OCCURRENCE("XX"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abcd"); IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); @@ -81,7 +81,7 @@ TEST_CASE("ngram (minimal input)") IRIS_CHECK_NO_OCCURRENCE("XX"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abcde"); IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); @@ -97,7 +97,7 @@ TEST_CASE("ngram (minimal input)") } } -TEST_CASE("ngram (realistic input)") +TEST_CASE("gram (realistic input)") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); @@ -106,7 +106,7 @@ TEST_CASE("ngram (realistic input)") // https://gihyo.jp/dev/serial/01/make-findspot/0005 { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"今日は良い天気です。"); IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); @@ -120,7 +120,7 @@ TEST_CASE("ngram (realistic input)") IRIS_CHECK_OCCURRENCE("す。", {0_doc_id, 8}); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"今日は大雨です。"); IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); @@ -132,7 +132,7 @@ TEST_CASE("ngram (realistic input)") IRIS_CHECK_OCCURRENCE("す。", {0_doc_id, 6}); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"今日の東海地方は大雨でしょう。"); IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); @@ -152,7 +152,7 @@ TEST_CASE("ngram (realistic input)") } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"今日は良い天気です。"); (void)ngram_db.add_document(U"今日は大雨です。"); (void)ngram_db.add_document(U"今日の東海地方は大雨でしょう。"); diff --git a/test/ngram_test.hpp b/test/ngram/ngram_test.hpp similarity index 73% rename from test/ngram_test.hpp rename to test/ngram/ngram_test.hpp index 5dedc30..899a39c 100644 --- a/test/ngram_test.hpp +++ b/test/ngram/ngram_test.hpp @@ -5,8 +5,8 @@ #include "iris_test.hpp" -#include -#include +#include +#include #include #include @@ -17,20 +17,20 @@ #endif using namespace iris::ngram_literals; -using iris::ngram_occurrence; +using iris::ngram::gram_occurrence; using iris::interval; struct DocumentMatch { - iris::ngram_document_id doc_id; - std::vector word_matches; + iris::ngram::document_id doc_id; + std::vector word_matches; - DocumentMatch(iris::ngram_document_id doc_id, std::initializer_list word_matches) + DocumentMatch(iris::ngram::document_id doc_id, std::initializer_list word_matches) : doc_id(doc_id) , word_matches(word_matches) {} - DocumentMatch(iris::ngram_document_id doc_id, std::vector word_matches) + DocumentMatch(iris::ngram::document_id doc_id, std::vector word_matches) : doc_id(doc_id) , word_matches(std::move(word_matches)) {} @@ -51,8 +51,8 @@ struct std::formatter }; #define IRIS_CHECK_SEARCH(query_input, ...) do { \ - iris::ngram_search_query const query{U ## query_input}; \ - iris::ngram_search_result search_res; \ + iris::ngram::search_query const query{U ## query_input}; \ + iris::ngram::search_result search_res; \ ngram_db.search(query, search_res); \ auto const& doc_matches = search_res.doc_matches(); \ \ diff --git a/test/ngram_search_2.cpp b/test/ngram/search_2.cpp similarity index 86% rename from test/ngram_search_2.cpp rename to test/ngram/search_2.cpp index 2f85570..916af2e 100644 --- a/test/ngram_search_2.cpp +++ b/test/ngram/search_2.cpp @@ -2,14 +2,14 @@ #include "ngram_test.hpp" -TEST_CASE("ngram search (document chars = 0)") +TEST_CASE("gram search (document chars = 0)") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); #endif { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; IRIS_CHECK_SEARCH(""); IRIS_CHECK_SEARCH("X"); IRIS_CHECK_SEARCH("XX"); @@ -17,14 +17,14 @@ TEST_CASE("ngram search (document chars = 0)") } // 1-gram document -TEST_CASE("ngram search (document chars = 1)") +TEST_CASE("gram search (document chars = 1)") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); #endif { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"a"); IRIS_CHECK_SEARCH(""); IRIS_CHECK_SEARCH( @@ -39,14 +39,14 @@ TEST_CASE("ngram search (document chars = 1)") } // 2-gram document -TEST_CASE("ngram search (document chars = 2)") +TEST_CASE("gram search (document chars = 2)") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); #endif { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"aa"); IRIS_CHECK_SEARCH(""); @@ -71,7 +71,7 @@ TEST_CASE("ngram search (document chars = 2)") IRIS_CHECK_SEARCH("XXX"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"ab"); IRIS_CHECK_SEARCH(""); diff --git a/test/ngram_search_3.cpp b/test/ngram/search_3.cpp similarity index 94% rename from test/ngram_search_3.cpp rename to test/ngram/search_3.cpp index dc9040d..c99e050 100644 --- a/test/ngram_search_3.cpp +++ b/test/ngram/search_3.cpp @@ -3,14 +3,14 @@ #include "ngram_test.hpp" // 2-gram + 1-gram document -TEST_CASE("ngram search (document chars = 3, aaa/baa)") +TEST_CASE("gram search (document chars = 3, aaa/baa)") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); #endif { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"aaa"); IRIS_CHECK_SEARCH(""); @@ -48,7 +48,7 @@ TEST_CASE("ngram search (document chars = 3, aaa/baa)") } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"baa"); IRIS_CHECK_SEARCH(""); @@ -103,14 +103,14 @@ TEST_CASE("ngram search (document chars = 3, aaa/baa)") } // 2-gram + 1-gram document -TEST_CASE("ngram search (document chars = 3, aba/aab)") +TEST_CASE("gram search (document chars = 3, aba/aab)") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); #endif { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"aba"); IRIS_CHECK_SEARCH(""); @@ -164,7 +164,7 @@ TEST_CASE("ngram search (document chars = 3, aba/aab)") } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"aab"); IRIS_CHECK_SEARCH(""); @@ -219,14 +219,14 @@ TEST_CASE("ngram search (document chars = 3, aba/aab)") } // 2-gram + 1-gram document -TEST_CASE("ngram search (document chars = 3, abc)") +TEST_CASE("gram search (document chars = 3, abc)") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); #endif { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abc"); IRIS_CHECK_SEARCH(""); diff --git a/test/ngram_search_4_dep.cpp b/test/ngram/search_4_dep.cpp similarity index 89% rename from test/ngram_search_4_dep.cpp rename to test/ngram/search_4_dep.cpp index 8045632..714bb22 100644 --- a/test/ngram_search_4_dep.cpp +++ b/test/ngram/search_4_dep.cpp @@ -3,14 +3,14 @@ #include "ngram_test.hpp" // 2x 2-gram document -TEST_CASE("ngram search (document chars = 4, aaaa)") +TEST_CASE("gram search (document chars = 4, aaaa)") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); #endif { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"aaaa"); IRIS_CHECK_SEARCH(""); @@ -57,14 +57,14 @@ TEST_CASE("ngram search (document chars = 4, aaaa)") } // 2x 2-gram document -TEST_CASE("ngram search (document chars = 4, abab)") +TEST_CASE("gram search (document chars = 4, abab)") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); #endif { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abab"); IRIS_CHECK_SEARCH(""); @@ -125,14 +125,14 @@ TEST_CASE("ngram search (document chars = 4, abab)") } // 2x 2-gram document -TEST_CASE("ngram search (document chars = 4, abca)") +TEST_CASE("gram search (document chars = 4, abca)") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); #endif { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abca"); IRIS_CHECK_SEARCH(""); @@ -208,14 +208,14 @@ TEST_CASE("ngram search (document chars = 4, abca)") } // 2x 2-gram document -TEST_CASE("ngram search (document chars = 4, abcd)") +TEST_CASE("gram search (document chars = 4, abcd)") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); #endif { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abcd"); IRIS_CHECK_SEARCH(""); @@ -283,63 +283,63 @@ TEST_CASE("ngram search (document chars = 4, abcd)") } } -TEST_CASE("ngram search (dependency on previous match)") +TEST_CASE("gram search (dependency on previous match)") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); #endif { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abcd"); IRIS_CHECK_SEARCH("abXX"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abef"); IRIS_CHECK_SEARCH("abXXef"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abef"); IRIS_CHECK_SEARCH("abXXefef"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"ab..ef"); IRIS_CHECK_SEARCH("abef"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"ab..ef"); IRIS_CHECK_SEARCH("abefef"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"ab..ef"); IRIS_CHECK_SEARCH("abXXef"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"ab..ef"); IRIS_CHECK_SEARCH("abXXefef"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abef"); IRIS_CHECK_SEARCH("abXef"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abef"); IRIS_CHECK_SEARCH("abXefef"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abxcd"); // trap document (void)ngram_db.add_document(U"abcd"); IRIS_CHECK_SEARCH( @@ -350,7 +350,7 @@ TEST_CASE("ngram search (dependency on previous match)") ); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abcd"); (void)ngram_db.add_document(U"abxcd"); // trap document IRIS_CHECK_SEARCH( @@ -362,7 +362,7 @@ TEST_CASE("ngram search (dependency on previous match)") } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"ab"); (void)ngram_db.add_document(U"abcd"); IRIS_CHECK_SEARCH( @@ -375,14 +375,14 @@ TEST_CASE("ngram search (dependency on previous match)") } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abXcd"); (void)ngram_db.add_document(U"abcdX"); IRIS_CHECK_SEARCH("abcdc"); } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"abXXabcd"); IRIS_CHECK_SEARCH( "abcd", @@ -393,7 +393,7 @@ TEST_CASE("ngram search (dependency on previous match)") } { - iris::ngram_database<> ngram_db; + iris::ngram::database<> ngram_db; (void)ngram_db.add_document(U"ab cdXf"); (void)ngram_db.add_document(U"ab cdef"); IRIS_CHECK_SEARCH( diff --git a/test/rvariant/CMakeLists.txt b/test/rvariant/CMakeLists.txt index 0404d61..cb81abe 100644 --- a/test/rvariant/CMakeLists.txt +++ b/test/rvariant/CMakeLists.txt @@ -16,6 +16,6 @@ endif() foreach(test_name IN LISTS IRIS_TEST_RVARIANT_TESTS) iris_define_internal_test(rvariant_${test_name} ${test_name}.cpp) - iris_define_test_headers(rvariant_${test_name} iris_rvariant_test.hpp) - set_target_properties(rvariant_${test_name}_test PROPERTIES FOLDER "test/rvariant") + iris_define_test_headers(iris_rvariant_${test_name} iris_rvariant_test.hpp) + set_target_properties(iris_rvariant_${test_name}_test PROPERTIES FOLDER "test/rvariant") endforeach() diff --git a/test/unicode/string/CMakeLists.txt b/test/unicode/string/CMakeLists.txt index a7d1e48..68c9eff 100644 --- a/test/unicode/string/CMakeLists.txt +++ b/test/unicode/string/CMakeLists.txt @@ -8,7 +8,7 @@ set( foreach(test_name IN LISTS IRIS_TEST_UNICODE_STRING_TESTS) iris_define_internal_test(unicode_string_${test_name} ${test_name}.cpp) - set_target_properties(unicode_string_${test_name}_test PROPERTIES FOLDER "test/unicode/string") + set_target_properties(iris_unicode_string_${test_name}_test PROPERTIES FOLDER "test/unicode/string") endforeach() -target_sources(unicode_string_utf8_invalid_test PRIVATE test_data/utf8_invalid.txt) +target_sources(iris_unicode_string_utf8_invalid_test PRIVATE test_data/utf8_invalid.txt) From f6778138c6e1c4749608a0e1d1ce16454ddc7446 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:15:25 +0900 Subject: [PATCH 14/16] Split `document_id` and `document_slot`, redesign interface --- include/iris/config.hpp | 2 + include/iris/hash/string_like_hash.hpp | 36 + include/iris/ngram/database.hpp | 710 +++--------------- include/iris/ngram/detail/id_store.hpp | 194 +++++ include/iris/ngram/detail/index.hpp | 339 +++++++++ .../iris/ngram/detail/search_result_cache.hpp | 154 ++++ include/iris/ngram/detail/slot.hpp | 29 + include/iris/ngram/gram.hpp | 2 +- include/iris/ngram/id.hpp | 34 +- include/iris/ngram/keyed_database.hpp | 109 +++ include/iris/ngram/search_query.hpp | 8 +- include/iris/ngram/search_result.hpp | 120 +++ test/ngram/ngram.cpp | 324 ++++---- test/ngram/ngram_test.hpp | 40 +- 14 files changed, 1317 insertions(+), 784 deletions(-) create mode 100644 include/iris/hash/string_like_hash.hpp create mode 100644 include/iris/ngram/detail/id_store.hpp create mode 100644 include/iris/ngram/detail/index.hpp create mode 100644 include/iris/ngram/detail/search_result_cache.hpp create mode 100644 include/iris/ngram/detail/slot.hpp create mode 100644 include/iris/ngram/keyed_database.hpp create mode 100644 include/iris/ngram/search_result.hpp diff --git a/include/iris/config.hpp b/include/iris/config.hpp index 8137d53..d258061 100644 --- a/include/iris/config.hpp +++ b/include/iris/config.hpp @@ -3,6 +3,8 @@ // SPDX-License-Identifier: MIT +// IWYU pragma: always_keep + #include #if _MSC_VER diff --git a/include/iris/hash/string_like_hash.hpp b/include/iris/hash/string_like_hash.hpp new file mode 100644 index 0000000..6827adb --- /dev/null +++ b/include/iris/hash/string_like_hash.hpp @@ -0,0 +1,36 @@ +#ifndef IRIS_ZZ_HASH_STRING_LIKE_HASH_HPP +#define IRIS_ZZ_HASH_STRING_LIKE_HASH_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include + +#include +#include +#include + +namespace iris { + +template> +struct basic_string_like_hash +{ + using is_transparent = int; + + [[nodiscard]] static std::size_t operator()(std::basic_string_view sv) + noexcept(is_nothrow_hashable_v>) + { + return std::hash>{}(sv); + } +}; + +using string_like_hash = basic_string_like_hash; +using wstring_like_hash = basic_string_like_hash; +using u8string_like_hash = basic_string_like_hash; +using u16string_like_hash = basic_string_like_hash; +using u32string_like_hash = basic_string_like_hash; + +} // iris + +#endif diff --git a/include/iris/ngram/database.hpp b/include/iris/ngram/database.hpp index 8173ecd..6ada138 100644 --- a/include/iris/ngram/database.hpp +++ b/include/iris/ngram/database.hpp @@ -3,650 +3,182 @@ // SPDX-License-Identifier: MIT -#include +#include // IWYU pragma: keep +#include +#include +#include +#include #include -#include #include +#include +#include -#include -#include -#include - -#include -#include #include #include -#include #include #include #include #include #include #include -#include -#include #include namespace iris::ngram { -struct gram_occurrence -{ - document_id doc_id; - int pos; - - [[nodiscard]] constexpr bool operator==(gram_occurrence const&) const noexcept = default; - [[nodiscard]] constexpr std::strong_ordering operator<=>(gram_occurrence const&) const noexcept = default; -}; - -namespace detail { - -enum struct [[nodiscard]] search_continuation : bool -{ - abort = false, - proceed = true, -}; - -class posting_list -{ -public: - void append(document_id const doc_id, int pos) - { - if (postings_.empty() || postings_.back().doc_id != doc_id) { - if (!postings_.empty() && postings_.back().doc_id > doc_id) { - throw std::invalid_argument{"documents must be indexed in non-decreasing order of document ID"}; - } - postings_.emplace_back( - doc_id, - static_cast(positions_.size()), - 0 - ); - } - - ++postings_.back().pos_count; - positions_.emplace_back(pos); - } - - void to_occurrence_list(std::vector& occs) const - { - occs.clear(); - for (auto const& posting : postings_) { - for (std::size_t i = posting.pos_offset; i < posting.pos_offset + posting.pos_count; ++i) { - occs.emplace_back(posting.doc_id, positions_[i]); - } - } - } - - template - void for_each_documents(F&& f) const - { - static_assert(std::invocable>); - - constexpr bool f_returns_continuation = std::same_as< - std::invoke_result_t>, - search_continuation - >; - - for (auto const& posting : postings_) { - std::span const posting_span{ - positions_.begin() + posting.pos_offset, - static_cast(posting.pos_count) - }; - - if constexpr (f_returns_continuation) { - search_continuation const cont = f(posting.doc_id, posting_span); - if (cont == search_continuation::abort) break; - } else { - f(posting.doc_id, posting_span); - } - } - } - -private: - struct posting_t - { - document_id doc_id; - unsigned pos_offset = 0; - unsigned pos_count = 0; - }; - - std::vector postings_; - std::vector positions_; -}; - -template -class gram_index -{ - using entry_map = std::flat_map, std::unique_ptr>; - static constexpr std::size_t side_merge_threshold = 2048; - -public: - [[nodiscard]] auto find_list(this auto&& self, gram const ng) - { - if (auto const it = self.gram_entries_.find(ng); it != self.gram_entries_.end()) { - return it->second.get(); - } - if (auto const it = self.side_entries_.find(ng); it != self.side_entries_.end()) { - return it->second.get(); - } - return static_cast(nullptr); - } - - void find_occurrences(gram const ng, std::vector& occs) const - { - occs.clear(); - auto const* list = this->find_list(ng); - if (!list) return; - list->to_occurrence_list(occs); - } - - template - void search(gram const ng, F&& f) const - { - auto const* list = this->find_list(ng); - if (!list) return; - list->for_each_documents(f); - } - - [[nodiscard]] bool empty() const noexcept - { - return gram_entries_.empty() && side_entries_.empty(); - } - - void clear() noexcept - { - gram_entries_.clear(); - side_entries_.clear(); - } - - void merge_new_entries(std::vector, std::unique_ptr>>& pending) - { - if (pending.empty()) return; // vocabulary saturated - - for (auto& [key, pl] : pending) { - [[maybe_unused]] - auto const it = side_entries_.try_emplace( - side_entries_.end(), // hint - key, std::move(pl) - ); - assert(it->second != nullptr && pl == nullptr); - } - if (side_entries_.size() >= side_merge_threshold) { - this->flush_side(); - } - } - -private: - void flush_side() - { - if (side_entries_.empty()) return; - - auto [skeys, svalues] = std::move(side_entries_).extract(); - auto [keys, values] = std::move(gram_entries_).extract(); - - std::size_t const old_size = keys.size(); - std::size_t const add = skeys.size(); - keys.resize(old_size + add); - values.resize(old_size + add); - - // Backward merge - std::size_t out = old_size + add; - std::size_t i = old_size; - std::size_t j = add; - while (j > 0) { - if (i > 0 && skeys[j - 1] < keys[i - 1]) { - --out; - --i; - keys[out] = keys[i]; - values[out] = std::move(values[i]); - } else { - assert(i == 0 || keys[i - 1] < skeys[j - 1]); - --out; - --j; - keys[out] = skeys[j]; - values[out] = std::move(svalues[j]); - } - } - assert(out == i); - - gram_entries_.replace(std::move(keys), std::move(values)); - } - - // Double-buffered to reduce insertion cost - entry_map gram_entries_, side_entries_; -}; - -template -struct gram_pos_t -{ - gram ng; - int pos; - - [[nodiscard]] constexpr bool operator==(gram_pos_t const&) const noexcept = default; - [[nodiscard]] constexpr std::strong_ordering operator<=>(gram_pos_t const&) const noexcept = default; -}; - -template -struct index_storage -{ - void append_index(document_id const doc_id, std::basic_string_view const input) - { - this->template append_index<1>(doc_id, input); - this->template append_index<2>(doc_id, input); - } - - void clear() noexcept - { - uni_data_.clear(); - bi_data_.clear(); - } - - [[nodiscard]] bool empty() const noexcept - { - return - this->template get_data<1>().idx.empty() && - this->template get_data<2>().idx.empty(); - } - - template - void search(gram const ng, F&& f) const - { - this->template get_data().idx.search(ng, f); - } - - template - [[nodiscard]] auto& get_index(this auto& self IRIS_LIFETIMEBOUND) noexcept - { - return self.template get_data().idx; - } - -private: - template - struct gram_index_storage - { - gram_index idx; - - // Caches - std::vector, default_init_allocator>> - batch_grams; - - std::vector, std::unique_ptr>> - batch_pending; - - void clear() noexcept - { - idx.clear(); - batch_grams.clear(); - batch_pending.clear(); - } - }; - - gram_index_storage<1> uni_data_; - gram_index_storage<2> bi_data_; - - template - [[nodiscard]] auto& get_data(this auto& self IRIS_LIFETIMEBOUND) noexcept - { - if constexpr (N == 1) { - return self.uni_data_; - } else if constexpr (N == 2) { - return self.bi_data_; - } else { - static_assert(false); - } - } - - template - void append_index( - document_id const doc_id, - std::basic_string_view const input - ) - { - if (input.size() < N) return; - gram_index_storage& data = this->template get_data(); - - // Naive per-gram insertion into flat_map is expensive: each *new* key - // shifts the underlying vectors, so building an index of vocabulary - // size V costs O(V^2) overall. Instead, per document: - // - // 1. Collect grams+positions --- O(G) G = grams in this doc - // 2. Sort them ----------------- O(G log G) - // 3. Existing keys ------------- O(D log V) D = distinct grams (D <= G) - // 4. New keys ------------------ O(V + P) P = brand-new keys (P <= D) - // - // Once the vocabulary saturates (P ~ 0, typical after a few documents), - // step 4 is a no-op and each document costs only O(G log G + D log V). - // - // Note: initially implemented by @saki7, then the complexity math is - // double-checked by Claude. - - data.batch_grams.clear(); - data.batch_grams.resize(input.size() - N + 1); - - if constexpr (N == 1) { - for (std::size_t i = 0; i < input.size(); ++i) { - data.batch_grams[i].ng.data = input[i]; - data.batch_grams[i].pos = static_cast(i); - } - - } else { - for (std::size_t i = 0; i + N <= input.size(); ++i) { - data.batch_grams[i].ng.copy_n(input.begin() + i); - data.batch_grams[i].pos = static_cast(i); - } - } - std::ranges::sort(data.batch_grams); - - data.batch_pending.clear(); - - for (auto const& chunk : data.batch_grams | std::views::chunk_by( - [](auto const& a, auto const& b) { return a.ng == b.ng; } - )) { - auto const& key = chunk.front().ng; - if (PostingListT* const pl = data.idx.find_list(key)) { - for (auto const& gp : chunk) { - pl->append(doc_id, gp.pos); - } - - } else { - auto& new_pl = data.batch_pending.emplace_back(key, std::make_unique()).second; - for (auto const& gp : chunk) { - new_pl->append(doc_id, gp.pos); - } - } - } - - data.idx.merge_new_entries(data.batch_pending); - } -}; - -} // detail - -struct [[nodiscard]] search_word_match -{ - search_word_match() = default; - - explicit search_word_match(int word_id) - : word_id(word_id) - {} - - search_word_match(int word_id, std::initializer_list> spans) - : word_id(word_id) - , spans(spans) - {} - - int word_id = 0; - unsigned successful_ngrams = 1; // due to the class layout, this must be placed here - std::vector> spans; - - [[nodiscard]] - bool operator==(search_word_match const& other) const noexcept - { - return word_id == other.word_id && spans == other.spans; - } -}; - -class [[nodiscard]] search_result +template +class database { - using doc_matches_map = std::flat_map>; - - struct word_matches_handle - { - doc_matches_map::iterator doc_it; - search_word_match* word_match = nullptr; - - [[nodiscard]] - search_word_match* operator->() const noexcept - { - return word_match; - } - - [[nodiscard]] explicit operator bool() const noexcept - { - return word_match; - } - }; - public: - [[nodiscard]] - bool has_document(document_id const doc_id) const noexcept - { - auto const it = doc_matches_.find(doc_id); - // An entry with no word matches is a tombstone (soft-erased document - // awaiting the next sweep), not a match. - return it != doc_matches_.end() && !it->second.empty(); - } - - [[nodiscard]] - auto const& doc_matches() const noexcept { return doc_matches_; } - - // Returns whether search must continue - template - [[nodiscard]] - bool init_word_matches(document_id const doc_id, int const word_id, std::span const positions) - { - assert(!positions.empty()); - - doc_matches_map::iterator doc_matches_it; - if constexpr (IsFirstWord) { - assert(word_id == 0); - assert(doc_matches_.empty() || doc_matches_.rbegin()->first < doc_id); - doc_matches_it = doc_matches_.try_emplace(doc_matches_.end(), doc_id); // hint: append - ++live_doc_count_; - - } else { - doc_matches_it = doc_matches_.find(doc_id); - if (doc_matches_it == doc_matches_.end()) return false; // no new docs after word 0 - if (doc_matches_it->second.empty()) return false; // tombstoned (soft-erased) document; skip - } - - assert(!std::ranges::contains(doc_matches_it->second, word_id, &search_word_match::word_id)); - auto& word_match = doc_matches_it->second.emplace_back(word_id); - word_match.spans.assign_range(positions | std::views::transform([](int const pos) -> interval { - return {pos, pos + static_cast(N)}; - })); - return true; - } - - [[nodiscard]] - word_matches_handle get_word_matches(document_id const doc_id, int const word_id) - { - auto const doc_matches_it = doc_matches_.find(doc_id); - if (doc_matches_it == doc_matches_.end()) return {}; - - // We don't need to do *full* `std::find` here; the word match is - // always inserted sequentially so if it exists, it is always placed - // at the *back* of the vector. - if ( - doc_matches_it->second.empty() || - doc_matches_it->second.back().word_id != word_id - ) { - assert( - doc_matches_it->second.empty() || - // Make sure the matching element does not exist at the position except for *back* - !std::ranges::contains(doc_matches_it->second, word_id, &search_word_match::word_id) - ); - return {}; - } - assert(!doc_matches_it->second.back().spans.empty()); - return {doc_matches_it, &doc_matches_it->second.back()}; - } - - void erase_document(word_matches_handle const& handle) - { - // This is slow because - // k erases x O(n) shift each =~ O(n^2) per word - //doc_matches_.erase(handle.doc_it); - - assert(!handle.doc_it->second.empty()); // never double-tombstone - handle.doc_it->second.clear(); // make this tombstone - assert(live_doc_count_ >= 1); - --live_doc_count_; - } + using document_id_type = document_id; - void remove_stale_document_matches(int const word_id, unsigned const expected_ngrams) + [[nodiscard]] document_id add_document(std::basic_string_view const doc_text) { - auto [keys, values] = std::move(doc_matches_).extract(); - - std::size_t out = 0; - for (std::size_t in = 0; in < keys.size(); ++in) { - auto& word_matches = values[in]; - bool is_word_survived = false; - std::erase_if(word_matches, [&](search_word_match const& wm) { - if (wm.word_id != word_id) return false; - if (wm.successful_ngrams != expected_ngrams) return true; - is_word_survived = true; - return false; - }); - if (!is_word_survived || word_matches.empty()) continue; - - if (out != in) { - keys[out] = keys[in]; - values[out] = std::move(values[in]); - } - ++out; - } - keys.resize(out); - values.resize(out); - doc_matches_.replace(std::move(keys), std::move(values)); - live_doc_count_ = out; + auto transaction = id_store_.add_document(); + store_.append_index(transaction.new_slot(), doc_text); + transaction.commit(); + return transaction.new_id(); } - void reset() noexcept + void update_document(document_id const doc_id, std::basic_string_view const doc_text) { - doc_matches_.clear(); - live_doc_count_ = 0; + auto transaction = id_store_.update_document(doc_id); + store_.append_index(transaction.new_slot(), doc_text); + transaction.commit(); } - [[nodiscard]] - bool empty() const noexcept + void remove_document(document_id const doc_id) { - return live_doc_count_ == 0; + id_store_.remove_document(doc_id); } - [[nodiscard]] - explicit operator bool() const noexcept + [[nodiscard]] bool has_document(document_id const doc_id) const { - return !this->empty(); + return id_store_.has_document(doc_id); } -private: - doc_matches_map doc_matches_; - std::size_t live_doc_count_ = 0; -}; - -template -class database -{ -public: - [[nodiscard]] document_id add_document(std::basic_string_view const doc_text) + [[nodiscard]] bool is_visible(document_id const doc_id) const { - document_id const doc_id{next_doc_id_}; - next_doc_id_ = document_id{std::to_underlying(next_doc_id_) + 1u}; - - store_.append_index(doc_id, doc_text); - return doc_id; - } - - [[nodiscard]] bool is_visible(document_id const doc_id) const noexcept - { - return !invisible_documents_.contains(doc_id); + return id_store_.is_visible(doc_id); } void set_visible(document_id const doc_id, bool const flag) { - if (flag) { - invisible_documents_.erase(doc_id); - } else { - invisible_documents_.emplace(doc_id); - } + id_store_.set_visible(doc_id, flag); } void clear() noexcept { - next_doc_id_ = 0_doc_id; + id_store_.clear(); store_.clear(); - invisible_documents_.clear(); } - template - void find_occurrences(gram ng, std::vector& occs) const - { - occs.clear(); - auto const& idx = store_.template get_index(); - idx.find_occurrences(ng, occs); - } - - bool search(search_query const& query, search_result& search_res) const + template + bool search(this auto const& db, search_query const& query, search_result& search_res) { + search_res.clear(); if (query.empty()) return false; - if (store_.empty()) return false; + if (db.store_.empty()) return false; + + db.search_res_cache_.reset(); int word_id = 0; auto it = query.words().begin(); assert(!it->empty()); - this->search_word(search_res, word_id++, *it++); - if (search_res.empty()) { - search_res.reset(); // remove tombstones + db.template search_word(word_id++, *it++); + if (db.search_res_cache_.empty()) { + db.search_res_cache_.reset(); // remove tombstones return false; } for (; it != query.words().end(); ++it) { assert(!it->empty()); - this->search_word(search_res, word_id++, *it); - if (search_res.empty()) { - search_res.reset(); // remove tombstones - break; + db.template search_word(word_id++, *it); + if (db.search_res_cache_.empty()) { + db.search_res_cache_.reset(); // remove tombstones + return false; } } - return !search_res.empty(); + + assert(!db.search_res_cache_.empty()); + assert(search_res.empty()); + + using document_id_maybe_ref_t = std::conditional_t< + std::is_reference_v()))>, + DocumentID const&, + DocumentID + >; + // Convert cache into real result + search_res.assign( + db.search_res_cache_.doc_matches() | std::views::transform([&db](auto&& kv) + -> std::pair&&> { + return std::pair&&>{ + db.make_document_id(db.id_store_.get_info(std::get<0>(kv)).doc_id), + std::move(std::get<1>(kv)) + }; + }) + ); + // Clear the cache (only contains the moved-from buffer though) + db.search_res_cache_.reset(); + + return true; } private: + [[nodiscard]] static document_id make_document_id(document_id doc_id) noexcept + { + return doc_id; + } + template - void search_word(search_result& search_res, int const word_id, std::basic_string_view const word) const + void search_word(int const word_id, std::basic_string_view const word) const { assert(!word.empty()); if (word.size() == 1) { - this->search_word_impl(search_res, word_id, word); + this->search_word_impl(word_id, word); } else { - this->search_word_impl(search_res, word_id, word); + this->search_word_impl(word_id, word); } } template - void search_word_impl(search_result& search_res, int const word_id, std::basic_string_view const word) const + void search_word_impl(int const word_id, std::basic_string_view const word) const { assert(word.size() >= N); auto ng = gram::from_copy_n(word.begin()); if constexpr (IsFirstWord) { - store_.search(ng, [&](document_id const doc_id, std::span const positions) { - if (!is_visible(doc_id)) return; - (void)search_res.init_word_matches(doc_id, word_id, positions); + store_.search(ng, [&](detail::document_slot const doc_slot, std::span const positions) { + auto const& slot_info = id_store_.get_info(doc_slot); + if (!slot_info.is_used_for_search()) return; + (void)search_res_cache_.init_word_matches(doc_slot, word_id, positions); }); - if (search_res.empty()) return; + if (search_res_cache_.empty()) return; } else { std::size_t available_doc_count = 0; - store_.search(ng, [&](document_id const doc_id, std::span const positions) { - if (!is_visible(doc_id)) return; - if (search_res.init_word_matches(doc_id, word_id, positions)) { + store_.search(ng, [&](detail::document_slot const doc_slot, std::span const positions) { + auto const& slot_info = id_store_.get_info(doc_slot); + if (!slot_info.is_used_for_search()) return; + if (search_res_cache_.init_word_matches(doc_slot, word_id, positions)) { ++available_doc_count; } }); if (available_doc_count == 0) { - search_res.reset(); + search_res_cache_.reset(); return; } } unsigned current_ngram = 1; auto const do_search = [&](int remaining_chars) { - return [&, remaining_chars, overlapping_chars = int(N) - remaining_chars](document_id const doc_id, std::span const positions) { - if (!is_visible(doc_id)) return detail::search_continuation::proceed; + return [&, remaining_chars, overlapping_chars = int(N) - remaining_chars](detail::document_slot const doc_slot, std::span const positions) { + auto const& slot_info = id_store_.get_info(doc_slot); + if (!slot_info.is_used_for_search()) { + return detail::search_continuation::proceed; + } // Find the existing match set from the previous iteration. // If none exists, any subsequent characters of the document will not match. @@ -654,7 +186,7 @@ class database // For example, when the document is "今日は晴れです" and current `ng` is "は晴", // - When previous `ng` was "昨日", `search_res` contians no matches => omit further sequence // - When previous `ng` was "今日", `search_res` contains matches => proceed with "は晴" - auto word_match = search_res.get_word_matches(doc_id, word_id); + auto word_match = search_res_cache_.get_word_matches(doc_slot, word_id); if (!word_match) return detail::search_continuation::proceed; // Prevent *resurrecting* the false-positive match on "match -> unmatch -> match" pattern. @@ -662,14 +194,14 @@ class database // - gram{"ab"} -> match (successful_ngrams = 1) // - gram{"XX"} -> no match (successful_ngrams is untouched) // - gram{"ef"} -> successful_ngrams does not match current_ngram! - if (word_match->successful_ngrams != current_ngram) { - search_res.erase_document(word_match); - if (search_res.empty()) return detail::search_continuation::abort; + if (word_match->successful_ngrams_ != current_ngram) { + search_res_cache_.erase_document(word_match); + if (search_res_cache_.empty()) return detail::search_continuation::abort; return detail::search_continuation::proceed; } // Find contiguous match; document has [previous ng, current ng] - for (auto it = word_match->spans.begin(); it != word_match->spans.end();) { + for (auto it = word_match->spans_.begin(); it != word_match->spans_.end();) { auto& prev_pos = *it; if (std::ranges::binary_search(positions, prev_pos.upper - overlapping_chars)) { @@ -680,19 +212,19 @@ class database } // Erase exiting match that indicates the below structure // [previous ng, ...some unrelated chars..., current ng] - it = word_match->spans.erase(it); + it = word_match->spans_.erase(it); } // Even if *all* existing matches fit // [previous ng, ...some unrelated chars..., current ng], // we can always remove the entire document from the candidate pool. - if (word_match->spans.empty()) { - search_res.erase_document(word_match); - if (search_res.empty()) return detail::search_continuation::abort; + if (word_match->spans_.empty()) { + search_res_cache_.erase_document(word_match); + if (search_res_cache_.empty()) return detail::search_continuation::abort; return detail::search_continuation::proceed; } - ++word_match->successful_ngrams; + ++word_match->successful_ngrams_; return detail::search_continuation::proceed; }; }; @@ -701,7 +233,7 @@ class database for (; i + N <= word.size(); i += N) { ng.copy_n(word.begin() + i); store_.search(ng, do_search(N)); - if (search_res.empty()) return; + if (search_res_cache_.empty()) return; ++current_ngram; } @@ -725,7 +257,7 @@ class database assert(remaining_chars < int(N)); ng.shift_copy(word.begin() + i, remaining_chars); store_.search(ng, do_search(remaining_chars)); - if (search_res.empty()) return; + if (search_res_cache_.empty()) return; ++current_ngram; } } @@ -734,50 +266,14 @@ class database // A first word of exactly one n-gram runs no continuation searches if (current_ngram == 1) return; } - search_res.remove_stale_document_matches(word_id, current_ngram); + search_res_cache_.remove_stale_document_matches(word_id, current_ngram); } - document_id next_doc_id_{0_doc_id}; + detail::id_store id_store_; detail::index_storage store_; - std::unordered_set invisible_documents_; + mutable detail::search_result_cache search_res_cache_; }; -} // iris::gram - - -template -struct std::formatter - : std::formatter, CharT> -{ - using base_type = std::formatter, CharT>; - - template - Ctx::iterator format(iris::ngram::document_id doc_id, Ctx& ctx) const - { - return base_type::format(std::to_underlying(doc_id), ctx); - } -}; - -template -struct std::formatter - : iris::no_spec_formatter -{ - template - Ctx::iterator format(iris::ngram::gram_occurrence const& occ, Ctx& ctx) const - { - return std::format_to(ctx.out(), "{}:{}", occ.doc_id, occ.pos); - } -}; - -template -struct std::formatter - : iris::no_spec_formatter -{ - template - Ctx::iterator format(iris::ngram::search_word_match const& word_match, Ctx& ctx) const - { - return std::format_to(ctx.out(), "{{word: #{}, spans: {}}}", word_match.word_id, word_match.spans); - } -}; +} // iris::ngram #endif diff --git a/include/iris/ngram/detail/id_store.hpp b/include/iris/ngram/detail/id_store.hpp new file mode 100644 index 0000000..d996394 --- /dev/null +++ b/include/iris/ngram/detail/id_store.hpp @@ -0,0 +1,194 @@ +#ifndef IRIS_ZZ_NGRAM_DETAIL_ID_STORE_HPP +#define IRIS_ZZ_NGRAM_DETAIL_ID_STORE_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include +#include + +#include + +#include +#include + +#include + +namespace iris::ngram::detail { + +class id_store +{ +public: + struct slot_info_t + { + document_id doc_id; + bool is_visible = true; + + [[nodiscard]] bool is_stale() const noexcept { return doc_id == document_id::tombstone; } + [[nodiscard]] bool is_used_for_search() const noexcept { return doc_id != document_id::tombstone && is_visible; } + }; + +private: + struct [[nodiscard]] add_document_transaction; + friend add_document_transaction; + struct add_document_transaction + { + add_document_transaction(id_store* store, document_id new_id, document_slot new_slot) noexcept + : store_(store) + , new_id_(new_id) + , new_slot_(new_slot) + {} + + [[nodiscard]] document_id new_id() const noexcept { return new_id_; } + [[nodiscard]] document_slot new_slot() const noexcept { return new_slot_; } + + void commit() + { + store_->id_to_slot_.emplace_back(new_slot_); + store_->slot_infos_.emplace_back(new_id_); + } + + private: + id_store* store_; + document_id new_id_; + document_slot new_slot_; + }; + +public: + add_document_transaction add_document() + { + return { + this, + static_cast(id_to_slot_.size()), + static_cast(slot_infos_.size()) + }; + } + +private: + struct [[nodiscard]] update_document_transaction; + friend update_document_transaction; + struct update_document_transaction + { + update_document_transaction(id_store* store, slot_info_t& old_slot_info, document_slot new_slot) noexcept + : store_(store) + , old_slot_info_(old_slot_info) + , new_slot_(new_slot) + {} + + [[nodiscard]] document_slot new_slot() const noexcept { return new_slot_; } + + void commit() + { + auto const doc_id = old_slot_info_.doc_id; + old_slot_info_.doc_id = document_id::tombstone; + store_->slot_infos_.emplace_back(doc_id, old_slot_info_.is_visible); + store_->id_to_slot_[to_index(doc_id)] = new_slot_; + } + + private: + id_store* store_; + slot_info_t& old_slot_info_; + document_slot new_slot_; + }; + +public: + update_document_transaction update_document(document_id const doc_id) + { + if (to_index(doc_id) >= id_to_slot_.size()) { + throwf("document id #{} is beyond the range of ids issued by this database", doc_id); + } + + document_slot const& doc_slot = id_to_slot_[to_index(doc_id)]; + if (doc_slot == document_slot::tombstone) { + throwf("cannot update a removed document #{}", doc_id); + } + + assert(to_index(doc_slot) < slot_infos_.size()); + auto& old_slot_info = slot_infos_[to_index(doc_slot)]; + assert(old_slot_info.doc_id == doc_id); + + return { + this, old_slot_info, static_cast(slot_infos_.size()) + }; + } + + void remove_document(document_id const doc_id) + { + if (to_index(doc_id) >= id_to_slot_.size()) { + throwf("document id #{} is beyond the range of ids issued by this database", doc_id); + } + + document_slot& doc_slot = id_to_slot_[to_index(doc_id)]; + if (doc_slot == document_slot::tombstone) { + // Removing an already-removed document is no-op; same semantics as STL containers + return; + } + + assert(to_index(doc_slot) < slot_infos_.size()); + auto& slot_info = slot_infos_[to_index(doc_slot)]; + + assert(!slot_info.is_stale()); + slot_info.doc_id = document_id::tombstone; + + doc_slot = document_slot::tombstone; + } + + [[nodiscard]] bool has_document(document_id const doc_id) const noexcept + { + return detail::to_index(doc_id) < id_to_slot_.size(); + } + + // This method is intentionally `noexcept` and invokes only assertion because + // it is heavily accessed via actual lookup + [[nodiscard]] slot_info_t const& get_info(document_slot const doc_slot) const noexcept + { + assert(to_index(doc_slot) < slot_infos_.size()); + return slot_infos_[to_index(doc_slot)]; + } + [[nodiscard]] slot_info_t& get_info(document_slot const doc_slot) noexcept + { + assert(to_index(doc_slot) < slot_infos_.size()); + return slot_infos_[to_index(doc_slot)]; + } + + [[nodiscard]] bool is_visible(document_id const doc_id) const + { + if (to_index(doc_id) >= id_to_slot_.size()) { + throwf("document id #{} is beyond the namespace acquired by the database", doc_id); + } + + document_slot const doc_slot = id_to_slot_[to_index(doc_id)]; + if (doc_slot == document_slot::tombstone) { + throwf("cannot fetch the visibility of already-removed document"); + } + return get_info(doc_slot).is_visible; + } + + void set_visible(document_id const doc_id, bool flag) + { + if (to_index(doc_id) >= id_to_slot_.size()) { + throwf("document id #{} is beyond the namespace acquired by the database", doc_id); + } + + document_slot const doc_slot = id_to_slot_[to_index(doc_id)]; + if (doc_slot == document_slot::tombstone) { + throwf("cannot change the visibility of already-removed document"); + } + get_info(doc_slot).is_visible = flag; + } + + void clear() noexcept + { + id_to_slot_.clear(); + slot_infos_.clear(); + } + +private: + std::vector id_to_slot_; + std::vector slot_infos_; +}; + +} // iris::ngram::detail + +#endif diff --git a/include/iris/ngram/detail/index.hpp b/include/iris/ngram/detail/index.hpp new file mode 100644 index 0000000..825b0aa --- /dev/null +++ b/include/iris/ngram/detail/index.hpp @@ -0,0 +1,339 @@ +#ifndef IRIS_ZZ_NGRAM_DETAIL_INDEX_HPP +#define IRIS_ZZ_NGRAM_DETAIL_INDEX_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +// Make sure we don't include `ngram/id.hpp` so that we can assure the +// internal logic never uses the external id type +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include // IWYU pragma: keep + +namespace iris::ngram::detail { + +enum struct [[nodiscard]] search_continuation : bool +{ + abort = false, + proceed = true, +}; + +class posting_list +{ +public: + void append(document_slot const doc_slot, int pos) + { + assert(postings_.empty() || postings_.back().doc_slot == document_slot::sentinel); + assert(postings_.empty() || postings_.back().pos_offset == positions_.size()); + + if (postings_.empty()) { + postings_.emplace_back(doc_slot, 0u); + postings_.emplace_back(document_slot::sentinel, 0u); + + } else if (auto last_posting = postings_.end() - 2; last_posting->doc_slot != doc_slot) { + if (last_posting->doc_slot > doc_slot) { + throw std::logic_error{"documents must be indexed in non-decreasing order of document_slot"}; + } + assert(postings_.back().doc_slot == document_slot::sentinel); + + // Promote the sentinel into a real posting + postings_.back().doc_slot = doc_slot; + postings_.emplace_back(document_slot::sentinel, static_cast(positions_.size())); + } + + positions_.emplace_back(pos); + ++postings_.back().pos_offset; + } + + template + void for_each_documents(F&& f) const + { + static_assert(std::invocable>); + + constexpr bool f_returns_continuation = std::same_as< + std::invoke_result_t>, + search_continuation + >; + + for (auto const& [posting, next] : postings_ | std::views::pairwise) { + std::span const posting_span{ + std::next(positions_.begin(), posting.pos_offset), + static_cast(next.pos_offset - posting.pos_offset) + }; + + if constexpr (f_returns_continuation) { + search_continuation const cont = f(posting.doc_slot, posting_span); + if (cont == search_continuation::abort) break; + } else { + f(posting.doc_slot, posting_span); + } + } + } + +private: + struct posting_t + { + document_slot doc_slot; + unsigned pos_offset = 0; + }; + std::vector postings_; + std::vector positions_; +}; + +template +class gram_index +{ + using gram_posting_map = std::flat_map, std::unique_ptr>; + static constexpr std::size_t side_merge_threshold = 2048; + +public: + [[nodiscard]] auto find_list(this auto&& self, gram const ng) + { + if (auto const it = self.gram_entries_.find(ng); it != self.gram_entries_.end()) { + return it->second.get(); + } + if (auto const it = self.side_entries_.find(ng); it != self.side_entries_.end()) { + return it->second.get(); + } + return static_cast(nullptr); + } + + template + void search(gram const ng, F&& f) const + { + auto const* list = this->find_list(ng); + if (!list) return; + list->for_each_documents(f); + } + + [[nodiscard]] bool empty() const noexcept + { + return gram_entries_.empty() && side_entries_.empty(); + } + + void clear() noexcept + { + gram_entries_.clear(); + side_entries_.clear(); + } + + void merge_new_entries(std::vector, std::unique_ptr>>& pending) + { + if (pending.empty()) return; // vocabulary saturated + + for (auto& [key, pl] : pending) { + [[maybe_unused]] auto const it = side_entries_.try_emplace( + side_entries_.end(), // hint + key, std::move(pl) + ); + assert(it->second != nullptr && pl == nullptr); + } + if (side_entries_.size() >= side_merge_threshold) { + this->flush_side(); + } + } + +private: + void flush_side() + { + if (side_entries_.empty()) return; + + auto [skeys, svalues] = std::move(side_entries_).extract(); + auto [keys, values] = std::move(gram_entries_).extract(); + + std::size_t const old_size = keys.size(); + std::size_t const add = skeys.size(); + keys.resize(old_size + add); + values.resize(old_size + add); + + // Backward merge + std::size_t out = old_size + add; + std::size_t i = old_size; + std::size_t j = add; + while (j > 0) { + if (i > 0 && skeys[j - 1] < keys[i - 1]) { + --out; + --i; + keys[out] = keys[i]; + values[out] = std::move(values[i]); + } else { + assert(i == 0 || keys[i - 1] < skeys[j - 1]); + --out; + --j; + keys[out] = skeys[j]; + values[out] = std::move(svalues[j]); + } + } + assert(out == i); + + gram_entries_.replace(std::move(keys), std::move(values)); + } + + // Double-buffered to reduce insertion cost + gram_posting_map gram_entries_, side_entries_; +}; + +template +struct gram_pos_t +{ + gram ng; + int pos; + + [[nodiscard]] constexpr bool operator==(gram_pos_t const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(gram_pos_t const&) const noexcept = default; +}; + +template +struct index_storage +{ + void append_index(document_slot const doc_slot, std::basic_string_view const input) + { + this->template append_index<1>(doc_slot, input); + this->template append_index<2>(doc_slot, input); + } + + void clear() noexcept + { + uni_data_.clear(); + bi_data_.clear(); + } + + [[nodiscard]] bool empty() const noexcept + { + return + this->template get_data<1>().idx.empty() && + this->template get_data<2>().idx.empty(); + } + + template + void search(gram const ng, F&& f) const + { + this->template get_data().idx.search(ng, f); + } + + template + [[nodiscard]] auto& get_index(this auto& self IRIS_LIFETIMEBOUND) noexcept + { + return self.template get_data().idx; + } + +private: + template + struct gram_index_storage + { + gram_index idx; + + // Caches + std::vector, default_init_allocator>> + batch_grams; + + std::vector, std::unique_ptr>> + batch_pending; + + void clear() noexcept + { + idx.clear(); + batch_grams.clear(); + batch_pending.clear(); + } + }; + + gram_index_storage<1> uni_data_; + gram_index_storage<2> bi_data_; + + template + [[nodiscard]] auto& get_data(this auto& self IRIS_LIFETIMEBOUND) noexcept + { + if constexpr (N == 1) { + return self.uni_data_; + } else if constexpr (N == 2) { + return self.bi_data_; + } else { + static_assert(false); + } + } + + template + void append_index( + document_slot const doc_slot, + std::basic_string_view const input + ) + { + if (input.size() < N) return; + gram_index_storage& data = this->template get_data(); + + // Naive per-gram insertion into flat_map is expensive: each *new* key + // shifts the underlying vectors, so building an index of vocabulary + // size V costs O(V^2) overall. Instead, per document: + // + // 1. Collect grams+positions --- O(G) G = grams in this doc + // 2. Sort them ----------------- O(G log G) + // 3. Existing keys ------------- O(D log V) D = distinct grams (D <= G) + // 4. New keys ------------------ O(V + P) P = brand-new keys (P <= D) + // + // Once the vocabulary saturates (P ~ 0, typical after a few documents), + // step 4 is a no-op and each document costs only O(G log G + D log V). + // + // Note: initially implemented by @saki7, then the complexity math is + // double-checked by Claude. + + data.batch_grams.clear(); + data.batch_grams.resize(input.size() - N + 1); + + if constexpr (N == 1) { + for (std::size_t i = 0; i < input.size(); ++i) { + data.batch_grams[i].ng.data = input[i]; + data.batch_grams[i].pos = static_cast(i); + } + + } else { + for (std::size_t i = 0; i + N <= input.size(); ++i) { + data.batch_grams[i].ng.copy_n(input.begin() + i); + data.batch_grams[i].pos = static_cast(i); + } + } + std::ranges::sort(data.batch_grams); + + data.batch_pending.clear(); + + for (auto const& chunk : data.batch_grams | std::views::chunk_by( + [](auto const& a, auto const& b) { return a.ng == b.ng; } + )) { + auto const& key = chunk.front().ng; + if (posting_list* const pl = data.idx.find_list(key)) { + for (auto const& gp : chunk) { + pl->append(doc_slot, gp.pos); + } + + } else { + auto& new_pl = data.batch_pending.emplace_back(key, std::make_unique()).second; + for (auto const& gp : chunk) { + new_pl->append(doc_slot, gp.pos); + } + } + } + + data.idx.merge_new_entries(data.batch_pending); + } +}; + +} // iris::ngram::detail + +#endif diff --git a/include/iris/ngram/detail/search_result_cache.hpp b/include/iris/ngram/detail/search_result_cache.hpp new file mode 100644 index 0000000..00c421d --- /dev/null +++ b/include/iris/ngram/detail/search_result_cache.hpp @@ -0,0 +1,154 @@ +#ifndef IRIS_ZZ_NGRAM_DETAIL_SEARCH_RESULT_CACHE_HPP +#define IRIS_ZZ_NGRAM_DETAIL_SEARCH_RESULT_CACHE_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace iris::ngram::detail { + +class [[nodiscard]] search_result_cache +{ + using doc_matches_map = std::flat_map>; + + struct word_matches_handle + { + doc_matches_map::iterator doc_it; + search_word_match* word_match = nullptr; + + [[nodiscard]] + search_word_match* operator->() const noexcept + { + return word_match; + } + + [[nodiscard]] explicit operator bool() const noexcept + { + return word_match; + } + }; + +public: + [[nodiscard]] auto& doc_matches() noexcept { return doc_matches_; } + + // Returns whether search must continue + template + [[nodiscard]] bool init_word_matches(document_slot const doc_slot, int const word_id, std::span const positions) + { + assert(!positions.empty()); + + doc_matches_map::iterator doc_matches_it; + if constexpr (IsFirstWord) { + assert(word_id == 0); + assert(doc_matches_.empty() || doc_matches_.rbegin()->first < doc_slot); + doc_matches_it = doc_matches_.try_emplace(doc_matches_.end(), doc_slot); // hint: append + ++live_doc_count_; + + } else { + doc_matches_it = doc_matches_.find(doc_slot); + if (doc_matches_it == doc_matches_.end()) return false; // no new docs after word 0 + if (doc_matches_it->second.empty()) return false; // tombstoned (soft-erased) document; skip + } + + assert(!std::ranges::contains(doc_matches_it->second, word_id, &search_word_match::word_id_)); + auto& word_match = doc_matches_it->second.emplace_back(word_id); + word_match.spans_.assign_range(positions | std::views::transform([](int const pos) -> interval { + return {pos, pos + static_cast(N)}; + })); + return true; + } + + [[nodiscard]] word_matches_handle get_word_matches(document_slot const doc_slot, int const word_id) + { + auto const doc_matches_it = doc_matches_.find(doc_slot); + if (doc_matches_it == doc_matches_.end()) return {}; + + // We don't need to do *full* `std::find` here; the word match is + // always inserted sequentially so if it exists, it is always placed + // at the *back* of the vector. + if ( + doc_matches_it->second.empty() || + doc_matches_it->second.back().word_id_ != word_id + ) { + assert( + doc_matches_it->second.empty() || + // Make sure the matching element does not exist at the position except for *back* + !std::ranges::contains(doc_matches_it->second, word_id, &search_word_match::word_id_) + ); + return {}; + } + assert(!doc_matches_it->second.back().spans_.empty()); + return {doc_matches_it, &doc_matches_it->second.back()}; + } + + void erase_document(word_matches_handle const& handle) + { + assert(!handle.doc_it->second.empty()); // never double-tombstone + handle.doc_it->second.clear(); // make this tombstone + assert(live_doc_count_ >= 1); + --live_doc_count_; + } + + void remove_stale_document_matches(int const word_id, unsigned const expected_ngrams) + { + auto [keys, values] = std::move(doc_matches_).extract(); + + std::size_t out = 0; + for (std::size_t in = 0; in < keys.size(); ++in) { + auto& word_matches = values[in]; + bool is_word_survived = false; + std::erase_if(word_matches, [&](search_word_match const& wm) { + if (wm.word_id_ != word_id) return false; + if (wm.successful_ngrams_ != expected_ngrams) return true; + is_word_survived = true; + return false; + }); + if (!is_word_survived || word_matches.empty()) continue; + + if (out != in) { + keys[out] = keys[in]; + values[out] = std::move(values[in]); + } + ++out; + } + keys.resize(out); + values.resize(out); + doc_matches_.replace(std::move(keys), std::move(values)); + live_doc_count_ = out; + } + + // Clears the search result and tombstones + void reset() noexcept + { + doc_matches_.clear(); + live_doc_count_ = 0; + } + + [[nodiscard]] bool empty() const noexcept + { + return live_doc_count_ == 0; + } + +private: + doc_matches_map doc_matches_; + std::size_t live_doc_count_ = 0; +}; + +} // iris::ngram::detail + +#endif diff --git a/include/iris/ngram/detail/slot.hpp b/include/iris/ngram/detail/slot.hpp new file mode 100644 index 0000000..287acae --- /dev/null +++ b/include/iris/ngram/detail/slot.hpp @@ -0,0 +1,29 @@ +#ifndef IRIS_ZZ_NGRAM_DETAIL_SLOT_HPP +#define IRIS_ZZ_NGRAM_DETAIL_SLOT_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include +#include // IWYU pragma: keep + +namespace iris::ngram::detail { + +// A monotonically increasing internal index used for bookkeeping. +// Obsolete documents and their posting data may still refer to this index (harmlessly) +// until `compact()` is requested on the database. +enum struct document_slot : std::uint32_t +{ + sentinel = static_cast(-1), + tombstone = static_cast(-2), +}; + +[[nodiscard]] constexpr std::size_t to_index(document_slot doc_slot) noexcept +{ + return static_cast(doc_slot); +} + +} // iris::ngram::detail + +#endif diff --git a/include/iris/ngram/gram.hpp b/include/iris/ngram/gram.hpp index f13aeea..650daed 100644 --- a/include/iris/ngram/gram.hpp +++ b/include/iris/ngram/gram.hpp @@ -3,7 +3,7 @@ // SPDX-License-Identifier: MIT -#include +#include // IWYU pragma: keep #include #include diff --git a/include/iris/ngram/id.hpp b/include/iris/ngram/id.hpp index 30bc03f..1c8a620 100644 --- a/include/iris/ngram/id.hpp +++ b/include/iris/ngram/id.hpp @@ -3,18 +3,34 @@ // SPDX-License-Identifier: MIT -#include +#include // IWYU pragma: keep +#include #include #include +#include // IWYU pragma: keep namespace iris::ngram { -enum struct document_id : std::uint32_t {}; +// An external id that is always *stable* across document updates or removal. +enum struct document_id : std::uint32_t +{ + tombstone = static_cast(-2), +}; + +namespace detail { + +[[nodiscard]] constexpr std::size_t to_index(document_id doc_id) noexcept +{ + return static_cast(doc_id); +} + +} // detail } // iris::ngram + namespace iris { inline namespace ngram_literals { @@ -28,4 +44,18 @@ inline namespace ngram_literals { } // iris + +template +struct std::formatter + : std::formatter, CharT> +{ + using base_type = std::formatter, CharT>; + + template + Ctx::iterator format(iris::ngram::document_id doc_id, Ctx& ctx) const + { + return base_type::format(std::to_underlying(doc_id), ctx); + } +}; + #endif diff --git a/include/iris/ngram/keyed_database.hpp b/include/iris/ngram/keyed_database.hpp new file mode 100644 index 0000000..48bb89f --- /dev/null +++ b/include/iris/ngram/keyed_database.hpp @@ -0,0 +1,109 @@ +#ifndef IRIS_ZZ_NGRAM_KEYED_DATABASE_HPP +#define IRIS_ZZ_NGRAM_KEYED_DATABASE_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include +#include + +#include + +#include +#include +#include + +namespace iris::ngram { + +template, class EqualT = std::equal_to<>, class CharT = char32_t> +class keyed_database : private database +{ + using base_type = database; + friend base_type; + +public: + using document_id_type = KeyT; + + template + requires std::is_constructible_v + void add_document(KeyLikeT&& key_like, std::basic_string_view const doc_text) + { + auto const doc_id = base_type::add_document(doc_text); + auto const [it, inserted] = key_to_doc_id_.emplace(std::forward(key_like), doc_id); + if (!inserted) { + if constexpr (std::is_pointer_v) { + throwf("key `{}` already exists in keyed_database", static_cast(it->first)); + } else { + throwf("key `{}` already exists in keyed_database", it->first); + } + } + assert(doc_id_to_key_.size() == detail::to_index(doc_id)); + doc_id_to_key_.emplace_back(std::addressof(it->first)); + } + + template + void update_document(KeyLikeT const& key_like, std::basic_string_view const doc_text) + { + base_type::update_document(this->get_document_id(key_like), doc_text); + } + + template + void remove_document(KeyLikeT const& key_like) + { + base_type::remove_document(this->get_document_id(key_like)); + } + + template + [[nodiscard]] bool has_document(KeyLikeT const& key_like) const + { + return key_to_doc_id_.find(key_like) != key_to_doc_id_.end(); + } + + template + [[nodiscard]] bool is_visible(KeyLikeT const& key_like) const + { + return base_type::is_visible(this->get_document_id(key_like)); + } + + template + void set_visible(KeyLikeT const& key_like, bool const flag) + { + base_type::set_visible(this->get_document_id(key_like), flag); + } + + void clear() noexcept + { + base_type::clear(); + key_to_doc_id_.clear(); + doc_id_to_key_.clear(); + } + + using base_type::search; + +private: + [[nodiscard]] KeyT const& make_document_id(document_id doc_id) const noexcept + { + assert(detail::to_index(doc_id) < doc_id_to_key_.size()); + return *doc_id_to_key_[detail::to_index(doc_id)]; + } + + template + [[nodiscard]] document_id get_document_id(KeyLikeT const& key_like) const + { + // TODO: __cpp_lib_associative_heterogeneous_insertion + auto const it = key_to_doc_id_.find(key_like); + if (it == key_to_doc_id_.end()) throw std::out_of_range{"key not found"}; + return it->second; + } + + std::unordered_map + key_to_doc_id_; + + std::vector + doc_id_to_key_; +}; + +} // iris::ngram + +#endif diff --git a/include/iris/ngram/search_query.hpp b/include/iris/ngram/search_query.hpp index 203d793..949c276 100644 --- a/include/iris/ngram/search_query.hpp +++ b/include/iris/ngram/search_query.hpp @@ -3,7 +3,8 @@ // SPDX-License-Identifier: MIT -#include +#include // IWYU pragma: keep + #include #include @@ -45,8 +46,7 @@ struct search_query // ------------------------------------------ - [[nodiscard]] - auto const& words() const noexcept + [[nodiscard]] auto const& words() const noexcept { return words_; } @@ -68,7 +68,7 @@ struct search_query template search_query(CharT const(&)[N]) -> search_query; -} // iris::gram +} // iris::ngram template struct std::formatter, CharT> diff --git a/include/iris/ngram/search_result.hpp b/include/iris/ngram/search_result.hpp new file mode 100644 index 0000000..7e39c5f --- /dev/null +++ b/include/iris/ngram/search_result.hpp @@ -0,0 +1,120 @@ +#ifndef IRIS_ZZ_NGRAM_SEARCH_RESULT_HPP +#define IRIS_ZZ_NGRAM_SEARCH_RESULT_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace iris::ngram { + +namespace detail { +class search_result_cache; +} // detail + +template +class database; + +struct [[nodiscard]] search_word_match +{ + search_word_match() = default; + + explicit search_word_match(int word_id) + : word_id_(word_id) + {} + + search_word_match(int word_id, std::initializer_list> spans) + : word_id_(word_id) + , spans_(spans) + {} + + [[nodiscard]] int word_id() const noexcept { return word_id_; } + [[nodiscard]] auto const& spans() const noexcept { return spans_; } + + [[nodiscard]] bool operator==(search_word_match const& other) const noexcept + { + return word_id_ == other.word_id_ && spans_ == other.spans_; + } + +private: + friend class detail::search_result_cache; + + template + friend class database; + + int word_id_ = 0; + unsigned successful_ngrams_ = 1; // due to the class layout, this must be placed here + std::vector> spans_; +}; + +template +struct [[nodiscard]] search_result +{ + using document_id_type = DocumentID; + using map_type = std::unordered_map>; + + search_result() = default; + + template + requires requires(map_type& doc_matches) { + doc_matches.insert_range(std::declval()); + } + void assign(DocumentMatchMap&& doc_match_map) + { + doc_matches_.clear(); + if constexpr (std::ranges::sized_range) { + doc_matches_.reserve(std::ranges::size(doc_match_map)); + } + doc_matches_.insert_range(std::forward(doc_match_map)); + } + + [[nodiscard]] map_type const& doc_matches() const noexcept + { + return doc_matches_; + } + + void clear() noexcept + { + doc_matches_.clear(); + } + + [[nodiscard]] bool empty() const noexcept + { + return doc_matches_.empty(); + } + + [[nodiscard]] explicit operator bool() const noexcept + { + return !this->empty(); + } + +private: + map_type doc_matches_; +}; + +} // iris::ngram + +template +struct std::formatter + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram::search_word_match const& word_match, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{{word: #{}, spans: {}}}", word_match.word_id(), word_match.spans()); + } +}; + +#endif diff --git a/test/ngram/ngram.cpp b/test/ngram/ngram.cpp index 6207615..3d20788 100644 --- a/test/ngram/ngram.cpp +++ b/test/ngram/ngram.cpp @@ -2,34 +2,31 @@ #include "ngram_test.hpp" -[[nodiscard]] -constexpr auto make_occurrences(std::initializer_list occs) -{ - return std::vector{occs}; -} +#include -#define IRIS_CHECK_NO_OCCURRENCE(ng_str) do { \ - std::vector occs; \ - ngram_db.find_occurrences(iris::to_ngram(U ## ng_str), occs); \ - CHECK(occs.empty()); \ - } while (false) +#include -#define IRIS_CHECK_OCCURRENCE(ng_str, ...) do { \ - std::vector occs; \ - ngram_db.find_occurrences(iris::to_ngram(U ## ng_str), occs); \ - CHECK(occs == make_occurrences({__VA_ARGS__})); \ - } while (false) +#include +#include -TEST_CASE("gram (type traits)") +TEST_CASE("ngram: type traits") { - STATIC_CHECK(std::same_as::data), char>); - STATIC_CHECK(std::same_as::data), std::uint16_t>); - - STATIC_CHECK(std::same_as::data), char32_t>); - STATIC_CHECK(std::same_as::data), std::uint64_t>); + STATIC_CHECK(sizeof(iris::ngram::gram<1, char>) == 1); + STATIC_CHECK(sizeof(iris::ngram::gram<2, char>) == 2); + STATIC_CHECK(sizeof(iris::ngram::gram<3, char>) == 3); + STATIC_CHECK(std::is_trivially_copyable_v>); + STATIC_CHECK(std::is_trivially_copyable_v>); + STATIC_CHECK(std::is_trivially_copyable_v>); + + STATIC_CHECK(sizeof(iris::ngram::gram<1, char32_t>) == 4); + STATIC_CHECK(sizeof(iris::ngram::gram<2, char32_t>) == 8); + STATIC_CHECK(sizeof(iris::ngram::gram<3, char32_t>) == 12); // TODO: optimize + STATIC_CHECK(std::is_trivially_copyable_v>); + STATIC_CHECK(std::is_trivially_copyable_v>); + STATIC_CHECK(std::is_trivially_copyable_v>); } -TEST_CASE("gram (minimal input)") +TEST_CASE("ngram: update document") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); @@ -37,147 +34,170 @@ TEST_CASE("gram (minimal input)") { iris::ngram::database<> ngram_db; - (void)ngram_db.add_document(U""); - IRIS_CHECK_NO_OCCURRENCE("a"); - } - { - iris::ngram::database<> ngram_db; - (void)ngram_db.add_document(U"a"); - IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); - IRIS_CHECK_NO_OCCURRENCE("X"); - IRIS_CHECK_NO_OCCURRENCE("XX"); - } - { - iris::ngram::database<> ngram_db; - (void)ngram_db.add_document(U"ab"); - IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); - IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); - IRIS_CHECK_NO_OCCURRENCE("X"); - IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); - IRIS_CHECK_NO_OCCURRENCE("XX"); - } - { - iris::ngram::database<> ngram_db; - (void)ngram_db.add_document(U"abc"); - IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); - IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); - IRIS_CHECK_OCCURRENCE("c", {0_doc_id, 2}); - IRIS_CHECK_NO_OCCURRENCE("X"); - IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); - IRIS_CHECK_OCCURRENCE("bc", {0_doc_id, 1}); - IRIS_CHECK_NO_OCCURRENCE("XX"); - } - { - iris::ngram::database<> ngram_db; - (void)ngram_db.add_document(U"abcd"); - IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); - IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); - IRIS_CHECK_OCCURRENCE("c", {0_doc_id, 2}); - IRIS_CHECK_OCCURRENCE("d", {0_doc_id, 3}); - IRIS_CHECK_NO_OCCURRENCE("X"); - IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); - IRIS_CHECK_OCCURRENCE("bc", {0_doc_id, 1}); - IRIS_CHECK_OCCURRENCE("cd", {0_doc_id, 2}); - IRIS_CHECK_NO_OCCURRENCE("XX"); + + auto const doc_id = ngram_db.add_document(U"abc"); + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + + ngram_db.set_visible(doc_id, false); + IRIS_CHECK_SEARCH("abc"); + IRIS_CHECK_SEARCH("ab"); + IRIS_CHECK_SEARCH("bc"); + + ngram_db.set_visible(doc_id, true); + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + + ngram_db.remove_document(doc_id); + IRIS_CHECK_SEARCH("abc"); + IRIS_CHECK_SEARCH("ab"); + IRIS_CHECK_SEARCH("bc"); } + { iris::ngram::database<> ngram_db; - (void)ngram_db.add_document(U"abcde"); - IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); - IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); - IRIS_CHECK_OCCURRENCE("c", {0_doc_id, 2}); - IRIS_CHECK_OCCURRENCE("d", {0_doc_id, 3}); - IRIS_CHECK_OCCURRENCE("e", {0_doc_id, 4}); - IRIS_CHECK_NO_OCCURRENCE("X"); - IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); - IRIS_CHECK_OCCURRENCE("bc", {0_doc_id, 1}); - IRIS_CHECK_OCCURRENCE("cd", {0_doc_id, 2}); - IRIS_CHECK_OCCURRENCE("de", {0_doc_id, 3}); - IRIS_CHECK_NO_OCCURRENCE("XX"); + + auto const doc_id = ngram_db.add_document(U"abc"); + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + + ngram_db.update_document(doc_id, U"abd"); + IRIS_CHECK_SEARCH("abc"); + IRIS_CHECK_SEARCH( + "abd", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("bc"); + IRIS_CHECK_SEARCH( + "bd", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + + ngram_db.remove_document(doc_id); + IRIS_CHECK_SEARCH("abc"); + IRIS_CHECK_SEARCH("ab"); + IRIS_CHECK_SEARCH("bc"); + IRIS_CHECK_SEARCH("abd"); + IRIS_CHECK_SEARCH("bd"); } } -TEST_CASE("gram (realistic input)") +TEST_CASE("ngram: keyed_database") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); #endif - // https://gihyo.jp/dev/serial/01/make-findspot/0005 - { - iris::ngram::database<> ngram_db; - (void)ngram_db.add_document(U"今日は良い天気です。"); - - IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); - IRIS_CHECK_OCCURRENCE("日は", {0_doc_id, 1}); - IRIS_CHECK_OCCURRENCE("は良", {0_doc_id, 2}); - IRIS_CHECK_OCCURRENCE("良い", {0_doc_id, 3}); - IRIS_CHECK_OCCURRENCE("い天", {0_doc_id, 4}); - IRIS_CHECK_OCCURRENCE("天気", {0_doc_id, 5}); - IRIS_CHECK_OCCURRENCE("気で", {0_doc_id, 6}); - IRIS_CHECK_OCCURRENCE("です", {0_doc_id, 7}); - IRIS_CHECK_OCCURRENCE("す。", {0_doc_id, 8}); - } - { - iris::ngram::database<> ngram_db; - (void)ngram_db.add_document(U"今日は大雨です。"); - - IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); - IRIS_CHECK_OCCURRENCE("日は", {0_doc_id, 1}); - IRIS_CHECK_OCCURRENCE("は大", {0_doc_id, 2}); - IRIS_CHECK_OCCURRENCE("大雨", {0_doc_id, 3}); - IRIS_CHECK_OCCURRENCE("雨で", {0_doc_id, 4}); - IRIS_CHECK_OCCURRENCE("です", {0_doc_id, 5}); - IRIS_CHECK_OCCURRENCE("す。", {0_doc_id, 6}); - } - { - iris::ngram::database<> ngram_db; - (void)ngram_db.add_document(U"今日の東海地方は大雨でしょう。"); - - IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); - IRIS_CHECK_OCCURRENCE("日の", {0_doc_id, 1}); - IRIS_CHECK_OCCURRENCE("の東", {0_doc_id, 2}); - IRIS_CHECK_OCCURRENCE("東海", {0_doc_id, 3}); - IRIS_CHECK_OCCURRENCE("海地", {0_doc_id, 4}); - IRIS_CHECK_OCCURRENCE("地方", {0_doc_id, 5}); - IRIS_CHECK_OCCURRENCE("方は", {0_doc_id, 6}); - IRIS_CHECK_OCCURRENCE("は大", {0_doc_id, 7}); - IRIS_CHECK_OCCURRENCE("大雨", {0_doc_id, 8}); - IRIS_CHECK_OCCURRENCE("雨で", {0_doc_id, 9}); - IRIS_CHECK_OCCURRENCE("でし", {0_doc_id, 10}); - IRIS_CHECK_OCCURRENCE("しょ", {0_doc_id, 11}); - IRIS_CHECK_OCCURRENCE("ょう", {0_doc_id, 12}); - IRIS_CHECK_OCCURRENCE("う。", {0_doc_id, 13}); - } - - { - iris::ngram::database<> ngram_db; - (void)ngram_db.add_document(U"今日は良い天気です。"); - (void)ngram_db.add_document(U"今日は大雨です。"); - (void)ngram_db.add_document(U"今日の東海地方は大雨でしょう。"); - - IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}, {1_doc_id, 0}, {2_doc_id, 0}); - IRIS_CHECK_OCCURRENCE("日は", {0_doc_id, 1}, {1_doc_id, 1}); - IRIS_CHECK_OCCURRENCE("は良", {0_doc_id, 2}); - IRIS_CHECK_OCCURRENCE("良い", {0_doc_id, 3}); - IRIS_CHECK_OCCURRENCE("い天", {0_doc_id, 4}); - IRIS_CHECK_OCCURRENCE("天気", {0_doc_id, 5}); - IRIS_CHECK_OCCURRENCE("気で", {0_doc_id, 6}); - IRIS_CHECK_OCCURRENCE("です", {0_doc_id, 7}, {1_doc_id, 5}); - IRIS_CHECK_OCCURRENCE("す。", {0_doc_id, 8}, {1_doc_id, 6}); - IRIS_CHECK_OCCURRENCE("は大", {1_doc_id, 2}, {2_doc_id, 7}); - IRIS_CHECK_OCCURRENCE("大雨", {1_doc_id, 3}, {2_doc_id, 8}); - IRIS_CHECK_OCCURRENCE("雨で", {1_doc_id, 4}, {2_doc_id, 9}); - IRIS_CHECK_OCCURRENCE("日の", {2_doc_id, 1}); - IRIS_CHECK_OCCURRENCE("の東", {2_doc_id, 2}); - IRIS_CHECK_OCCURRENCE("東海", {2_doc_id, 3}); - IRIS_CHECK_OCCURRENCE("海地", {2_doc_id, 4}); - IRIS_CHECK_OCCURRENCE("地方", {2_doc_id, 5}); - IRIS_CHECK_OCCURRENCE("方は", {2_doc_id, 6}); - IRIS_CHECK_OCCURRENCE("でし", {2_doc_id, 10}); - IRIS_CHECK_OCCURRENCE("しょ", {2_doc_id, 11}); - IRIS_CHECK_OCCURRENCE("ょう", {2_doc_id, 12}); - IRIS_CHECK_OCCURRENCE("う。", {2_doc_id, 13}); + iris::ngram::keyed_database + ngram_db; + + ngram_db.add_document("doc0", U"abc"); + IRIS_CHECK_SEARCH( + "abc", + {"doc0", { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {"doc0", { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {"doc0", { + {0, {interval{1, 3}}}, + }}, + ); + + ngram_db.set_visible("doc0", false); + IRIS_CHECK_SEARCH("abc"); + IRIS_CHECK_SEARCH("ab"); + IRIS_CHECK_SEARCH("bc"); + + ngram_db.set_visible("doc0", true); + IRIS_CHECK_SEARCH( + "abc", + {"doc0", { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {"doc0", { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {"doc0", { + {0, {interval{1, 3}}}, + }}, + ); + + ngram_db.remove_document("doc0"); + IRIS_CHECK_SEARCH("abc"); + IRIS_CHECK_SEARCH("ab"); + IRIS_CHECK_SEARCH("bc"); } } diff --git a/test/ngram/ngram_test.hpp b/test/ngram/ngram_test.hpp index 899a39c..18fe66c 100644 --- a/test/ngram/ngram_test.hpp +++ b/test/ngram/ngram_test.hpp @@ -8,30 +8,34 @@ #include #include -#include -#include -#include +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export #ifdef _MSC_VER -# include +# include // IWYU pragma: export #endif using namespace iris::ngram_literals; -using iris::ngram::gram_occurrence; using iris::interval; +template struct DocumentMatch { - iris::ngram::document_id doc_id; + DocumentID doc_id; std::vector word_matches; - DocumentMatch(iris::ngram::document_id doc_id, std::initializer_list word_matches) - : doc_id(doc_id) + template + DocumentMatch(ID&& doc_id, std::initializer_list word_matches) + : doc_id(std::forward(doc_id)) , word_matches(word_matches) {} - DocumentMatch(iris::ngram::document_id doc_id, std::vector word_matches) - : doc_id(doc_id) + template + DocumentMatch(ID&& doc_id, std::vector word_matches) + : doc_id(std::forward(doc_id)) , word_matches(std::move(word_matches)) {} @@ -39,29 +43,29 @@ struct DocumentMatch bool operator==(DocumentMatch const&) const noexcept = default; }; -template -struct std::formatter +template +struct std::formatter, CharT> : iris::no_spec_formatter { template - Ctx::iterator format(DocumentMatch const& doc_match, Ctx& ctx) const + Ctx::iterator format(DocumentMatch const& doc_match, Ctx& ctx) const { - return std::format_to(ctx.out(), "(doc: #{}, word_matches: {})", doc_match.doc_id, doc_match.word_matches); + return std::format_to(ctx.out(), "(doc: `{}`, word_matches: {})", doc_match.doc_id, doc_match.word_matches); } }; #define IRIS_CHECK_SEARCH(query_input, ...) do { \ iris::ngram::search_query const query{U ## query_input}; \ - iris::ngram::search_result search_res; \ + iris::ngram::search_result::document_id_type> search_res; \ ngram_db.search(query, search_res); \ auto const& doc_matches = search_res.doc_matches(); \ \ - std::vector const expected_doc_matches{ \ - std::initializer_list{__VA_ARGS__} \ + std::vector::document_id_type>> const expected_doc_matches{ \ + std::initializer_list::document_id_type>>{__VA_ARGS__} \ }; \ \ auto const actual_doc_matches = doc_matches | std::views::transform([](auto const& kv) { \ - return DocumentMatch{kv.first, kv.second}; \ + return DocumentMatch::document_id_type>{kv.first, kv.second}; \ }) | std::ranges::to(); \ CHECK(actual_doc_matches == expected_doc_matches); \ } while (false) From 45438dcad655dd8a82a3c8d57f80c2f3c00861c5 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:35:46 +0900 Subject: [PATCH 15/16] Fix natvis --- iris.natvis | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iris.natvis b/iris.natvis index 76f86a6..77c0f8e 100644 --- a/iris.natvis +++ b/iris.natvis @@ -255,16 +255,16 @@ - + {chars._Elems,na1} - + {chars._Elems,na2} - + {chars._Elems,na3} - + {chars._Elems,na4} From ce3503f12779d6fa3210fbd964586576e09dce0f Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Wed, 19 Aug 2026 00:33:41 +0900 Subject: [PATCH 16/16] Add `add_or_update_document` --- include/iris/ngram/keyed_database.hpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/include/iris/ngram/keyed_database.hpp b/include/iris/ngram/keyed_database.hpp index 48bb89f..8d0334b 100644 --- a/include/iris/ngram/keyed_database.hpp +++ b/include/iris/ngram/keyed_database.hpp @@ -48,6 +48,20 @@ class keyed_database : private database base_type::update_document(this->get_document_id(key_like), doc_text); } + template + requires std::is_constructible_v + void add_or_update_document(KeyLikeT&& key_like, std::basic_string_view const doc_text) + { + KeyT key{std::forward(key_like)}; + auto const it = key_to_doc_id_.find(key); + + if (it == key_to_doc_id_.end()) { + this->add_document(std::move(key), doc_text); + } else { + base_type::update_document(it->second, doc_text); + } + } + template void remove_document(KeyLikeT const& key_like) {