diff --git a/include/iris/config.hpp b/include/iris/config.hpp index 8137d53..d258061 100644 --- a/include/iris/config.hpp +++ b/include/iris/config.hpp @@ -3,6 +3,8 @@ // SPDX-License-Identifier: MIT +// IWYU pragma: always_keep + #include #if _MSC_VER diff --git a/include/iris/hash/string_like_hash.hpp b/include/iris/hash/string_like_hash.hpp new file mode 100644 index 0000000..6827adb --- /dev/null +++ b/include/iris/hash/string_like_hash.hpp @@ -0,0 +1,36 @@ +#ifndef IRIS_ZZ_HASH_STRING_LIKE_HASH_HPP +#define IRIS_ZZ_HASH_STRING_LIKE_HASH_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include + +#include +#include +#include + +namespace iris { + +template> +struct basic_string_like_hash +{ + using is_transparent = int; + + [[nodiscard]] static std::size_t operator()(std::basic_string_view sv) + noexcept(is_nothrow_hashable_v>) + { + return std::hash>{}(sv); + } +}; + +using string_like_hash = basic_string_like_hash; +using wstring_like_hash = basic_string_like_hash; +using u8string_like_hash = basic_string_like_hash; +using u16string_like_hash = basic_string_like_hash; +using u32string_like_hash = basic_string_like_hash; + +} // iris + +#endif diff --git a/include/iris/ngram/database.hpp b/include/iris/ngram/database.hpp new file mode 100644 index 0000000..6ada138 --- /dev/null +++ b/include/iris/ngram/database.hpp @@ -0,0 +1,279 @@ +#ifndef IRIS_ZZ_NGRAM_DATABASE_HPP +#define IRIS_ZZ_NGRAM_DATABASE_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace iris::ngram { + +template +class database +{ +public: + using document_id_type = document_id; + + [[nodiscard]] document_id add_document(std::basic_string_view const doc_text) + { + auto transaction = id_store_.add_document(); + store_.append_index(transaction.new_slot(), doc_text); + transaction.commit(); + return transaction.new_id(); + } + + void update_document(document_id const doc_id, std::basic_string_view const doc_text) + { + auto transaction = id_store_.update_document(doc_id); + store_.append_index(transaction.new_slot(), doc_text); + transaction.commit(); + } + + void remove_document(document_id const doc_id) + { + id_store_.remove_document(doc_id); + } + + [[nodiscard]] bool has_document(document_id const doc_id) const + { + return id_store_.has_document(doc_id); + } + + [[nodiscard]] bool is_visible(document_id const doc_id) const + { + return id_store_.is_visible(doc_id); + } + + void set_visible(document_id const doc_id, bool const flag) + { + id_store_.set_visible(doc_id, flag); + } + + void clear() noexcept + { + id_store_.clear(); + store_.clear(); + } + + template + bool search(this auto const& db, search_query const& query, search_result& search_res) + { + search_res.clear(); + if (query.empty()) return false; + if (db.store_.empty()) return false; + + db.search_res_cache_.reset(); + + int word_id = 0; + auto it = query.words().begin(); + assert(!it->empty()); + db.template search_word(word_id++, *it++); + if (db.search_res_cache_.empty()) { + db.search_res_cache_.reset(); // remove tombstones + return false; + } + + for (; it != query.words().end(); ++it) { + assert(!it->empty()); + db.template search_word(word_id++, *it); + if (db.search_res_cache_.empty()) { + db.search_res_cache_.reset(); // remove tombstones + return false; + } + } + + assert(!db.search_res_cache_.empty()); + assert(search_res.empty()); + + using document_id_maybe_ref_t = std::conditional_t< + std::is_reference_v()))>, + DocumentID const&, + DocumentID + >; + // Convert cache into real result + search_res.assign( + db.search_res_cache_.doc_matches() | std::views::transform([&db](auto&& kv) + -> std::pair&&> { + return std::pair&&>{ + db.make_document_id(db.id_store_.get_info(std::get<0>(kv)).doc_id), + std::move(std::get<1>(kv)) + }; + }) + ); + // Clear the cache (only contains the moved-from buffer though) + db.search_res_cache_.reset(); + + return true; + } + +private: + [[nodiscard]] static document_id make_document_id(document_id doc_id) noexcept + { + return doc_id; + } + + template + void search_word(int const word_id, std::basic_string_view const word) const + { + assert(!word.empty()); + + if (word.size() == 1) { + this->search_word_impl(word_id, word); + } else { + this->search_word_impl(word_id, word); + } + } + + template + void search_word_impl(int const word_id, std::basic_string_view const word) const + { + assert(word.size() >= N); + auto ng = gram::from_copy_n(word.begin()); + + if constexpr (IsFirstWord) { + store_.search(ng, [&](detail::document_slot const doc_slot, std::span const positions) { + auto const& slot_info = id_store_.get_info(doc_slot); + if (!slot_info.is_used_for_search()) return; + (void)search_res_cache_.init_word_matches(doc_slot, word_id, positions); + }); + if (search_res_cache_.empty()) return; + + } else { + std::size_t available_doc_count = 0; + store_.search(ng, [&](detail::document_slot const doc_slot, std::span const positions) { + auto const& slot_info = id_store_.get_info(doc_slot); + if (!slot_info.is_used_for_search()) return; + if (search_res_cache_.init_word_matches(doc_slot, word_id, positions)) { + ++available_doc_count; + } + }); + if (available_doc_count == 0) { + search_res_cache_.reset(); + return; + } + } + + unsigned current_ngram = 1; + auto const do_search = [&](int remaining_chars) { + return [&, remaining_chars, overlapping_chars = int(N) - remaining_chars](detail::document_slot const doc_slot, std::span const positions) { + auto const& slot_info = id_store_.get_info(doc_slot); + if (!slot_info.is_used_for_search()) { + return detail::search_continuation::proceed; + } + + // Find the existing match set from the previous iteration. + // If none exists, any subsequent characters of the document will not match. + // + // For example, when the document is "今日は晴れです" and current `ng` is "は晴", + // - When previous `ng` was "昨日", `search_res` contians no matches => omit further sequence + // - When previous `ng` was "今日", `search_res` contains matches => proceed with "は晴" + auto word_match = search_res_cache_.get_word_matches(doc_slot, word_id); + if (!word_match) return detail::search_continuation::proceed; + + // Prevent *resurrecting* the false-positive match on "match -> unmatch -> match" pattern. + // For example, when the document is "abef" and the query is "abXXef", + // - gram{"ab"} -> match (successful_ngrams = 1) + // - gram{"XX"} -> no match (successful_ngrams is untouched) + // - gram{"ef"} -> successful_ngrams does not match current_ngram! + if (word_match->successful_ngrams_ != current_ngram) { + search_res_cache_.erase_document(word_match); + if (search_res_cache_.empty()) return detail::search_continuation::abort; + return detail::search_continuation::proceed; + } + + // Find contiguous match; document has [previous ng, current ng] + for (auto it = word_match->spans_.begin(); it != word_match->spans_.end();) { + auto& prev_pos = *it; + + if (std::ranges::binary_search(positions, prev_pos.upper - overlapping_chars)) { + // Matched; the current word's current n-gram is contiguous to the previous n-gram + prev_pos.upper += remaining_chars; + ++it; + continue; + } + // Erase exiting match that indicates the below structure + // [previous ng, ...some unrelated chars..., current ng] + it = word_match->spans_.erase(it); + } + + // Even if *all* existing matches fit + // [previous ng, ...some unrelated chars..., current ng], + // we can always remove the entire document from the candidate pool. + if (word_match->spans_.empty()) { + search_res_cache_.erase_document(word_match); + if (search_res_cache_.empty()) return detail::search_continuation::abort; + return detail::search_continuation::proceed; + } + + ++word_match->successful_ngrams_; + return detail::search_continuation::proceed; + }; + }; + + std::size_t i = N; + for (; i + N <= word.size(); i += N) { + ng.copy_n(word.begin() + i); + store_.search(ng, do_search(N)); + if (search_res_cache_.empty()) return; + ++current_ngram; + } + + if constexpr (N >= 2) { + // When the remaining character count is remainder of `word.size() % N`, + // search by the *slided* remaining characters. + // + // For example, when the document is "今日は晴れです": + // + // When doing 3-gram search with "今日は雨": + // 1. Search by "今日は" in the normal loop + // + // 2. Then, + // i == 3 + // remaining_chars == word.size() - i == 1 + // overlapping_chars == N - remaining_chars == 2 + // next_search_pos = i - overlapping_chars == 1 + // + // 3. Try to match "日は雨" in the last loop + if (int const remaining_chars = static_cast(word.size() - i); remaining_chars > 0) { + assert(remaining_chars < int(N)); + ng.shift_copy(word.begin() + i, remaining_chars); + store_.search(ng, do_search(remaining_chars)); + if (search_res_cache_.empty()) return; + ++current_ngram; + } + } + + if constexpr (IsFirstWord) { + // A first word of exactly one n-gram runs no continuation searches + if (current_ngram == 1) return; + } + search_res_cache_.remove_stale_document_matches(word_id, current_ngram); + } + + detail::id_store id_store_; + detail::index_storage store_; + mutable detail::search_result_cache search_res_cache_; +}; + +} // iris::ngram + +#endif diff --git a/include/iris/ngram/detail/id_store.hpp b/include/iris/ngram/detail/id_store.hpp new file mode 100644 index 0000000..d996394 --- /dev/null +++ b/include/iris/ngram/detail/id_store.hpp @@ -0,0 +1,194 @@ +#ifndef IRIS_ZZ_NGRAM_DETAIL_ID_STORE_HPP +#define IRIS_ZZ_NGRAM_DETAIL_ID_STORE_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include +#include + +#include + +#include +#include + +#include + +namespace iris::ngram::detail { + +class id_store +{ +public: + struct slot_info_t + { + document_id doc_id; + bool is_visible = true; + + [[nodiscard]] bool is_stale() const noexcept { return doc_id == document_id::tombstone; } + [[nodiscard]] bool is_used_for_search() const noexcept { return doc_id != document_id::tombstone && is_visible; } + }; + +private: + struct [[nodiscard]] add_document_transaction; + friend add_document_transaction; + struct add_document_transaction + { + add_document_transaction(id_store* store, document_id new_id, document_slot new_slot) noexcept + : store_(store) + , new_id_(new_id) + , new_slot_(new_slot) + {} + + [[nodiscard]] document_id new_id() const noexcept { return new_id_; } + [[nodiscard]] document_slot new_slot() const noexcept { return new_slot_; } + + void commit() + { + store_->id_to_slot_.emplace_back(new_slot_); + store_->slot_infos_.emplace_back(new_id_); + } + + private: + id_store* store_; + document_id new_id_; + document_slot new_slot_; + }; + +public: + add_document_transaction add_document() + { + return { + this, + static_cast(id_to_slot_.size()), + static_cast(slot_infos_.size()) + }; + } + +private: + struct [[nodiscard]] update_document_transaction; + friend update_document_transaction; + struct update_document_transaction + { + update_document_transaction(id_store* store, slot_info_t& old_slot_info, document_slot new_slot) noexcept + : store_(store) + , old_slot_info_(old_slot_info) + , new_slot_(new_slot) + {} + + [[nodiscard]] document_slot new_slot() const noexcept { return new_slot_; } + + void commit() + { + auto const doc_id = old_slot_info_.doc_id; + old_slot_info_.doc_id = document_id::tombstone; + store_->slot_infos_.emplace_back(doc_id, old_slot_info_.is_visible); + store_->id_to_slot_[to_index(doc_id)] = new_slot_; + } + + private: + id_store* store_; + slot_info_t& old_slot_info_; + document_slot new_slot_; + }; + +public: + update_document_transaction update_document(document_id const doc_id) + { + if (to_index(doc_id) >= id_to_slot_.size()) { + throwf("document id #{} is beyond the range of ids issued by this database", doc_id); + } + + document_slot const& doc_slot = id_to_slot_[to_index(doc_id)]; + if (doc_slot == document_slot::tombstone) { + throwf("cannot update a removed document #{}", doc_id); + } + + assert(to_index(doc_slot) < slot_infos_.size()); + auto& old_slot_info = slot_infos_[to_index(doc_slot)]; + assert(old_slot_info.doc_id == doc_id); + + return { + this, old_slot_info, static_cast(slot_infos_.size()) + }; + } + + void remove_document(document_id const doc_id) + { + if (to_index(doc_id) >= id_to_slot_.size()) { + throwf("document id #{} is beyond the range of ids issued by this database", doc_id); + } + + document_slot& doc_slot = id_to_slot_[to_index(doc_id)]; + if (doc_slot == document_slot::tombstone) { + // Removing an already-removed document is no-op; same semantics as STL containers + return; + } + + assert(to_index(doc_slot) < slot_infos_.size()); + auto& slot_info = slot_infos_[to_index(doc_slot)]; + + assert(!slot_info.is_stale()); + slot_info.doc_id = document_id::tombstone; + + doc_slot = document_slot::tombstone; + } + + [[nodiscard]] bool has_document(document_id const doc_id) const noexcept + { + return detail::to_index(doc_id) < id_to_slot_.size(); + } + + // This method is intentionally `noexcept` and invokes only assertion because + // it is heavily accessed via actual lookup + [[nodiscard]] slot_info_t const& get_info(document_slot const doc_slot) const noexcept + { + assert(to_index(doc_slot) < slot_infos_.size()); + return slot_infos_[to_index(doc_slot)]; + } + [[nodiscard]] slot_info_t& get_info(document_slot const doc_slot) noexcept + { + assert(to_index(doc_slot) < slot_infos_.size()); + return slot_infos_[to_index(doc_slot)]; + } + + [[nodiscard]] bool is_visible(document_id const doc_id) const + { + if (to_index(doc_id) >= id_to_slot_.size()) { + throwf("document id #{} is beyond the namespace acquired by the database", doc_id); + } + + document_slot const doc_slot = id_to_slot_[to_index(doc_id)]; + if (doc_slot == document_slot::tombstone) { + throwf("cannot fetch the visibility of already-removed document"); + } + return get_info(doc_slot).is_visible; + } + + void set_visible(document_id const doc_id, bool flag) + { + if (to_index(doc_id) >= id_to_slot_.size()) { + throwf("document id #{} is beyond the namespace acquired by the database", doc_id); + } + + document_slot const doc_slot = id_to_slot_[to_index(doc_id)]; + if (doc_slot == document_slot::tombstone) { + throwf("cannot change the visibility of already-removed document"); + } + get_info(doc_slot).is_visible = flag; + } + + void clear() noexcept + { + id_to_slot_.clear(); + slot_infos_.clear(); + } + +private: + std::vector id_to_slot_; + std::vector slot_infos_; +}; + +} // iris::ngram::detail + +#endif diff --git a/include/iris/ngram/detail/index.hpp b/include/iris/ngram/detail/index.hpp new file mode 100644 index 0000000..825b0aa --- /dev/null +++ b/include/iris/ngram/detail/index.hpp @@ -0,0 +1,339 @@ +#ifndef IRIS_ZZ_NGRAM_DETAIL_INDEX_HPP +#define IRIS_ZZ_NGRAM_DETAIL_INDEX_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +// Make sure we don't include `ngram/id.hpp` so that we can assure the +// internal logic never uses the external id type +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include // IWYU pragma: keep + +namespace iris::ngram::detail { + +enum struct [[nodiscard]] search_continuation : bool +{ + abort = false, + proceed = true, +}; + +class posting_list +{ +public: + void append(document_slot const doc_slot, int pos) + { + assert(postings_.empty() || postings_.back().doc_slot == document_slot::sentinel); + assert(postings_.empty() || postings_.back().pos_offset == positions_.size()); + + if (postings_.empty()) { + postings_.emplace_back(doc_slot, 0u); + postings_.emplace_back(document_slot::sentinel, 0u); + + } else if (auto last_posting = postings_.end() - 2; last_posting->doc_slot != doc_slot) { + if (last_posting->doc_slot > doc_slot) { + throw std::logic_error{"documents must be indexed in non-decreasing order of document_slot"}; + } + assert(postings_.back().doc_slot == document_slot::sentinel); + + // Promote the sentinel into a real posting + postings_.back().doc_slot = doc_slot; + postings_.emplace_back(document_slot::sentinel, static_cast(positions_.size())); + } + + positions_.emplace_back(pos); + ++postings_.back().pos_offset; + } + + template + void for_each_documents(F&& f) const + { + static_assert(std::invocable>); + + constexpr bool f_returns_continuation = std::same_as< + std::invoke_result_t>, + search_continuation + >; + + for (auto const& [posting, next] : postings_ | std::views::pairwise) { + std::span const posting_span{ + std::next(positions_.begin(), posting.pos_offset), + static_cast(next.pos_offset - posting.pos_offset) + }; + + if constexpr (f_returns_continuation) { + search_continuation const cont = f(posting.doc_slot, posting_span); + if (cont == search_continuation::abort) break; + } else { + f(posting.doc_slot, posting_span); + } + } + } + +private: + struct posting_t + { + document_slot doc_slot; + unsigned pos_offset = 0; + }; + std::vector postings_; + std::vector positions_; +}; + +template +class gram_index +{ + using gram_posting_map = std::flat_map, std::unique_ptr>; + static constexpr std::size_t side_merge_threshold = 2048; + +public: + [[nodiscard]] auto find_list(this auto&& self, gram const ng) + { + if (auto const it = self.gram_entries_.find(ng); it != self.gram_entries_.end()) { + return it->second.get(); + } + if (auto const it = self.side_entries_.find(ng); it != self.side_entries_.end()) { + return it->second.get(); + } + return static_cast(nullptr); + } + + template + void search(gram const ng, F&& f) const + { + auto const* list = this->find_list(ng); + if (!list) return; + list->for_each_documents(f); + } + + [[nodiscard]] bool empty() const noexcept + { + return gram_entries_.empty() && side_entries_.empty(); + } + + void clear() noexcept + { + gram_entries_.clear(); + side_entries_.clear(); + } + + void merge_new_entries(std::vector, std::unique_ptr>>& pending) + { + if (pending.empty()) return; // vocabulary saturated + + for (auto& [key, pl] : pending) { + [[maybe_unused]] auto const it = side_entries_.try_emplace( + side_entries_.end(), // hint + key, std::move(pl) + ); + assert(it->second != nullptr && pl == nullptr); + } + if (side_entries_.size() >= side_merge_threshold) { + this->flush_side(); + } + } + +private: + void flush_side() + { + if (side_entries_.empty()) return; + + auto [skeys, svalues] = std::move(side_entries_).extract(); + auto [keys, values] = std::move(gram_entries_).extract(); + + std::size_t const old_size = keys.size(); + std::size_t const add = skeys.size(); + keys.resize(old_size + add); + values.resize(old_size + add); + + // Backward merge + std::size_t out = old_size + add; + std::size_t i = old_size; + std::size_t j = add; + while (j > 0) { + if (i > 0 && skeys[j - 1] < keys[i - 1]) { + --out; + --i; + keys[out] = keys[i]; + values[out] = std::move(values[i]); + } else { + assert(i == 0 || keys[i - 1] < skeys[j - 1]); + --out; + --j; + keys[out] = skeys[j]; + values[out] = std::move(svalues[j]); + } + } + assert(out == i); + + gram_entries_.replace(std::move(keys), std::move(values)); + } + + // Double-buffered to reduce insertion cost + gram_posting_map gram_entries_, side_entries_; +}; + +template +struct gram_pos_t +{ + gram ng; + int pos; + + [[nodiscard]] constexpr bool operator==(gram_pos_t const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(gram_pos_t const&) const noexcept = default; +}; + +template +struct index_storage +{ + void append_index(document_slot const doc_slot, std::basic_string_view const input) + { + this->template append_index<1>(doc_slot, input); + this->template append_index<2>(doc_slot, input); + } + + void clear() noexcept + { + uni_data_.clear(); + bi_data_.clear(); + } + + [[nodiscard]] bool empty() const noexcept + { + return + this->template get_data<1>().idx.empty() && + this->template get_data<2>().idx.empty(); + } + + template + void search(gram const ng, F&& f) const + { + this->template get_data().idx.search(ng, f); + } + + template + [[nodiscard]] auto& get_index(this auto& self IRIS_LIFETIMEBOUND) noexcept + { + return self.template get_data().idx; + } + +private: + template + struct gram_index_storage + { + gram_index idx; + + // Caches + std::vector, default_init_allocator>> + batch_grams; + + std::vector, std::unique_ptr>> + batch_pending; + + void clear() noexcept + { + idx.clear(); + batch_grams.clear(); + batch_pending.clear(); + } + }; + + gram_index_storage<1> uni_data_; + gram_index_storage<2> bi_data_; + + template + [[nodiscard]] auto& get_data(this auto& self IRIS_LIFETIMEBOUND) noexcept + { + if constexpr (N == 1) { + return self.uni_data_; + } else if constexpr (N == 2) { + return self.bi_data_; + } else { + static_assert(false); + } + } + + template + void append_index( + document_slot const doc_slot, + std::basic_string_view const input + ) + { + if (input.size() < N) return; + gram_index_storage& data = this->template get_data(); + + // Naive per-gram insertion into flat_map is expensive: each *new* key + // shifts the underlying vectors, so building an index of vocabulary + // size V costs O(V^2) overall. Instead, per document: + // + // 1. Collect grams+positions --- O(G) G = grams in this doc + // 2. Sort them ----------------- O(G log G) + // 3. Existing keys ------------- O(D log V) D = distinct grams (D <= G) + // 4. New keys ------------------ O(V + P) P = brand-new keys (P <= D) + // + // Once the vocabulary saturates (P ~ 0, typical after a few documents), + // step 4 is a no-op and each document costs only O(G log G + D log V). + // + // Note: initially implemented by @saki7, then the complexity math is + // double-checked by Claude. + + data.batch_grams.clear(); + data.batch_grams.resize(input.size() - N + 1); + + if constexpr (N == 1) { + for (std::size_t i = 0; i < input.size(); ++i) { + data.batch_grams[i].ng.data = input[i]; + data.batch_grams[i].pos = static_cast(i); + } + + } else { + for (std::size_t i = 0; i + N <= input.size(); ++i) { + data.batch_grams[i].ng.copy_n(input.begin() + i); + data.batch_grams[i].pos = static_cast(i); + } + } + std::ranges::sort(data.batch_grams); + + data.batch_pending.clear(); + + for (auto const& chunk : data.batch_grams | std::views::chunk_by( + [](auto const& a, auto const& b) { return a.ng == b.ng; } + )) { + auto const& key = chunk.front().ng; + if (posting_list* const pl = data.idx.find_list(key)) { + for (auto const& gp : chunk) { + pl->append(doc_slot, gp.pos); + } + + } else { + auto& new_pl = data.batch_pending.emplace_back(key, std::make_unique()).second; + for (auto const& gp : chunk) { + new_pl->append(doc_slot, gp.pos); + } + } + } + + data.idx.merge_new_entries(data.batch_pending); + } +}; + +} // iris::ngram::detail + +#endif diff --git a/include/iris/ngram/detail/search_result_cache.hpp b/include/iris/ngram/detail/search_result_cache.hpp new file mode 100644 index 0000000..00c421d --- /dev/null +++ b/include/iris/ngram/detail/search_result_cache.hpp @@ -0,0 +1,154 @@ +#ifndef IRIS_ZZ_NGRAM_DETAIL_SEARCH_RESULT_CACHE_HPP +#define IRIS_ZZ_NGRAM_DETAIL_SEARCH_RESULT_CACHE_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace iris::ngram::detail { + +class [[nodiscard]] search_result_cache +{ + using doc_matches_map = std::flat_map>; + + struct word_matches_handle + { + doc_matches_map::iterator doc_it; + search_word_match* word_match = nullptr; + + [[nodiscard]] + search_word_match* operator->() const noexcept + { + return word_match; + } + + [[nodiscard]] explicit operator bool() const noexcept + { + return word_match; + } + }; + +public: + [[nodiscard]] auto& doc_matches() noexcept { return doc_matches_; } + + // Returns whether search must continue + template + [[nodiscard]] bool init_word_matches(document_slot const doc_slot, int const word_id, std::span const positions) + { + assert(!positions.empty()); + + doc_matches_map::iterator doc_matches_it; + if constexpr (IsFirstWord) { + assert(word_id == 0); + assert(doc_matches_.empty() || doc_matches_.rbegin()->first < doc_slot); + doc_matches_it = doc_matches_.try_emplace(doc_matches_.end(), doc_slot); // hint: append + ++live_doc_count_; + + } else { + doc_matches_it = doc_matches_.find(doc_slot); + if (doc_matches_it == doc_matches_.end()) return false; // no new docs after word 0 + if (doc_matches_it->second.empty()) return false; // tombstoned (soft-erased) document; skip + } + + assert(!std::ranges::contains(doc_matches_it->second, word_id, &search_word_match::word_id_)); + auto& word_match = doc_matches_it->second.emplace_back(word_id); + word_match.spans_.assign_range(positions | std::views::transform([](int const pos) -> interval { + return {pos, pos + static_cast(N)}; + })); + return true; + } + + [[nodiscard]] word_matches_handle get_word_matches(document_slot const doc_slot, int const word_id) + { + auto const doc_matches_it = doc_matches_.find(doc_slot); + if (doc_matches_it == doc_matches_.end()) return {}; + + // We don't need to do *full* `std::find` here; the word match is + // always inserted sequentially so if it exists, it is always placed + // at the *back* of the vector. + if ( + doc_matches_it->second.empty() || + doc_matches_it->second.back().word_id_ != word_id + ) { + assert( + doc_matches_it->second.empty() || + // Make sure the matching element does not exist at the position except for *back* + !std::ranges::contains(doc_matches_it->second, word_id, &search_word_match::word_id_) + ); + return {}; + } + assert(!doc_matches_it->second.back().spans_.empty()); + return {doc_matches_it, &doc_matches_it->second.back()}; + } + + void erase_document(word_matches_handle const& handle) + { + assert(!handle.doc_it->second.empty()); // never double-tombstone + handle.doc_it->second.clear(); // make this tombstone + assert(live_doc_count_ >= 1); + --live_doc_count_; + } + + void remove_stale_document_matches(int const word_id, unsigned const expected_ngrams) + { + auto [keys, values] = std::move(doc_matches_).extract(); + + std::size_t out = 0; + for (std::size_t in = 0; in < keys.size(); ++in) { + auto& word_matches = values[in]; + bool is_word_survived = false; + std::erase_if(word_matches, [&](search_word_match const& wm) { + if (wm.word_id_ != word_id) return false; + if (wm.successful_ngrams_ != expected_ngrams) return true; + is_word_survived = true; + return false; + }); + if (!is_word_survived || word_matches.empty()) continue; + + if (out != in) { + keys[out] = keys[in]; + values[out] = std::move(values[in]); + } + ++out; + } + keys.resize(out); + values.resize(out); + doc_matches_.replace(std::move(keys), std::move(values)); + live_doc_count_ = out; + } + + // Clears the search result and tombstones + void reset() noexcept + { + doc_matches_.clear(); + live_doc_count_ = 0; + } + + [[nodiscard]] bool empty() const noexcept + { + return live_doc_count_ == 0; + } + +private: + doc_matches_map doc_matches_; + std::size_t live_doc_count_ = 0; +}; + +} // iris::ngram::detail + +#endif diff --git a/include/iris/ngram/detail/slot.hpp b/include/iris/ngram/detail/slot.hpp new file mode 100644 index 0000000..287acae --- /dev/null +++ b/include/iris/ngram/detail/slot.hpp @@ -0,0 +1,29 @@ +#ifndef IRIS_ZZ_NGRAM_DETAIL_SLOT_HPP +#define IRIS_ZZ_NGRAM_DETAIL_SLOT_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include +#include // IWYU pragma: keep + +namespace iris::ngram::detail { + +// A monotonically increasing internal index used for bookkeeping. +// Obsolete documents and their posting data may still refer to this index (harmlessly) +// until `compact()` is requested on the database. +enum struct document_slot : std::uint32_t +{ + sentinel = static_cast(-1), + tombstone = static_cast(-2), +}; + +[[nodiscard]] constexpr std::size_t to_index(document_slot doc_slot) noexcept +{ + return static_cast(doc_slot); +} + +} // iris::ngram::detail + +#endif diff --git a/include/iris/ngram/gram.hpp b/include/iris/ngram/gram.hpp new file mode 100644 index 0000000..650daed --- /dev/null +++ b/include/iris/ngram/gram.hpp @@ -0,0 +1,168 @@ +#ifndef IRIS_ZZ_NGRAM_GRAM_HPP +#define IRIS_ZZ_NGRAM_GRAM_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace iris::ngram { + +namespace detail { + +template struct gram_value; +template<> struct gram_value<1> { using type = std::uint8_t; }; +template<> struct gram_value<2> { using type = std::uint16_t; }; +template<> struct gram_value<4> { using type = std::uint32_t; }; +template<> struct gram_value<8> { using type = std::uint64_t; }; + +template +using gram_value_t = gram_value::type; + +} // detail + +template +struct gram +{ + static_assert(N >= 3); + + std::array data; + + template + constexpr void copy_n(It it) + noexcept(noexcept(*it++)) + { + std::ranges::copy_n(it, N, data.begin()); + } + template + [[nodiscard]] static constexpr gram from_copy_n(It it) + noexcept(noexcept(std::declval().copy_n(std::move(it)))) + { + gram ng; + ng.copy_n(std::move(it)); + return ng; + } + + template + constexpr void shift_copy(It it, int const remaining_chars) + noexcept( + noexcept(std::shift_left(data.begin(), data.end(), remaining_chars)) && + noexcept(std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars))) + ) + { + assert(remaining_chars < int(N)); + std::shift_left(data.begin(), data.end(), remaining_chars); + std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars)); + } + + template + [[nodiscard]] static constexpr gram from_c_array(CharT const (&chars)[Len]) noexcept + { + static_assert(Len == N + 1); + assert(chars[Len - 1] == static_cast(0)); + return gram::from_copy_n(std::ranges::begin(chars)); + } + + [[nodiscard]] constexpr bool operator==(gram const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(gram const&) const noexcept = default; +}; + +template +struct gram<1, CharT> +{ + CharT data; + + template + constexpr void copy_n(It it) + noexcept(noexcept(*it)) + { + data = *it; + } + template + [[nodiscard]] static constexpr gram from_copy_n(It it) + noexcept(noexcept(std::declval().copy_n(std::move(it)))) + { + gram ng; + ng.copy_n(std::move(it)); + return ng; + } + + template + [[nodiscard]] static constexpr gram from_c_array(CharT const (&chars)[Len]) noexcept + { + static_assert(Len == 1 + 1); + assert(chars[Len - 1] == static_cast(0)); + return gram::from_copy_n(std::ranges::begin(chars)); + } + + [[nodiscard]] constexpr bool operator==(gram const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(gram const&) const noexcept = default; +}; + +template +struct gram<2, CharT> +{ + using value_type = detail::gram_value_t; + value_type data; + + template + constexpr void copy_n(It it) + noexcept(noexcept(*it++)) + { + using uchar = std::make_unsigned_t; + data = value_type(static_cast(*it++)) << (sizeof(CharT) * 8); + data |= value_type(static_cast(*it)); + } + template + [[nodiscard]] static constexpr gram from_copy_n(It it) + noexcept(noexcept(std::declval().copy_n(std::move(it)))) + { + gram ng; + ng.copy_n(std::move(it)); + return ng; + } + + template + constexpr void shift_copy(It it, int const remaining_chars) + noexcept(noexcept(*it)) + { + assert(remaining_chars == 1); + (void)remaining_chars; + data = (data << (sizeof(CharT) * 8)) | value_type(static_cast>(*it)); + } + + template + [[nodiscard]] static constexpr gram from_c_array(CharT const (&chars)[Len]) noexcept + { + static_assert(Len == 2 + 1); + assert(chars[Len - 1] == static_cast(0)); + return gram::from_copy_n(std::ranges::begin(chars)); + } + + [[nodiscard]] constexpr bool operator==(gram const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(gram const&) const noexcept = default; +}; + +} // iris::ngram + +namespace iris { + +template +[[nodiscard]] ngram::gram to_ngram(CharT const (&chars)[N]) noexcept +{ + return ngram::gram::from_c_array(chars); +} + +} // iris + +#endif diff --git a/include/iris/ngram/id.hpp b/include/iris/ngram/id.hpp new file mode 100644 index 0000000..1c8a620 --- /dev/null +++ b/include/iris/ngram/id.hpp @@ -0,0 +1,61 @@ +#ifndef IRIS_ZZ_NGRAM_ID_HPP +#define IRIS_ZZ_NGRAM_ID_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include +#include + +#include +#include // IWYU pragma: keep + +namespace iris::ngram { + +// An external id that is always *stable* across document updates or removal. +enum struct document_id : std::uint32_t +{ + tombstone = static_cast(-2), +}; + +namespace detail { + +[[nodiscard]] constexpr std::size_t to_index(document_id doc_id) noexcept +{ + return static_cast(doc_id); +} + +} // detail + +} // iris::ngram + + +namespace iris { + +inline namespace ngram_literals { + +[[nodiscard]] constexpr ngram::document_id operator ""_doc_id(unsigned long long id) noexcept +{ + return ngram::document_id{static_cast>(id)}; +} + +} // ngram_literals + +} // iris + + +template +struct std::formatter + : std::formatter, CharT> +{ + using base_type = std::formatter, CharT>; + + template + Ctx::iterator format(iris::ngram::document_id doc_id, Ctx& ctx) const + { + return base_type::format(std::to_underlying(doc_id), ctx); + } +}; + +#endif diff --git a/include/iris/ngram/keyed_database.hpp b/include/iris/ngram/keyed_database.hpp new file mode 100644 index 0000000..8d0334b --- /dev/null +++ b/include/iris/ngram/keyed_database.hpp @@ -0,0 +1,123 @@ +#ifndef IRIS_ZZ_NGRAM_KEYED_DATABASE_HPP +#define IRIS_ZZ_NGRAM_KEYED_DATABASE_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include +#include + +#include + +#include +#include +#include + +namespace iris::ngram { + +template, class EqualT = std::equal_to<>, class CharT = char32_t> +class keyed_database : private database +{ + using base_type = database; + friend base_type; + +public: + using document_id_type = KeyT; + + template + requires std::is_constructible_v + void add_document(KeyLikeT&& key_like, std::basic_string_view const doc_text) + { + auto const doc_id = base_type::add_document(doc_text); + auto const [it, inserted] = key_to_doc_id_.emplace(std::forward(key_like), doc_id); + if (!inserted) { + if constexpr (std::is_pointer_v) { + throwf("key `{}` already exists in keyed_database", static_cast(it->first)); + } else { + throwf("key `{}` already exists in keyed_database", it->first); + } + } + assert(doc_id_to_key_.size() == detail::to_index(doc_id)); + doc_id_to_key_.emplace_back(std::addressof(it->first)); + } + + template + void update_document(KeyLikeT const& key_like, std::basic_string_view const doc_text) + { + base_type::update_document(this->get_document_id(key_like), doc_text); + } + + template + requires std::is_constructible_v + void add_or_update_document(KeyLikeT&& key_like, std::basic_string_view const doc_text) + { + KeyT key{std::forward(key_like)}; + auto const it = key_to_doc_id_.find(key); + + if (it == key_to_doc_id_.end()) { + this->add_document(std::move(key), doc_text); + } else { + base_type::update_document(it->second, doc_text); + } + } + + template + void remove_document(KeyLikeT const& key_like) + { + base_type::remove_document(this->get_document_id(key_like)); + } + + template + [[nodiscard]] bool has_document(KeyLikeT const& key_like) const + { + return key_to_doc_id_.find(key_like) != key_to_doc_id_.end(); + } + + template + [[nodiscard]] bool is_visible(KeyLikeT const& key_like) const + { + return base_type::is_visible(this->get_document_id(key_like)); + } + + template + void set_visible(KeyLikeT const& key_like, bool const flag) + { + base_type::set_visible(this->get_document_id(key_like), flag); + } + + void clear() noexcept + { + base_type::clear(); + key_to_doc_id_.clear(); + doc_id_to_key_.clear(); + } + + using base_type::search; + +private: + [[nodiscard]] KeyT const& make_document_id(document_id doc_id) const noexcept + { + assert(detail::to_index(doc_id) < doc_id_to_key_.size()); + return *doc_id_to_key_[detail::to_index(doc_id)]; + } + + template + [[nodiscard]] document_id get_document_id(KeyLikeT const& key_like) const + { + // TODO: __cpp_lib_associative_heterogeneous_insertion + auto const it = key_to_doc_id_.find(key_like); + if (it == key_to_doc_id_.end()) throw std::out_of_range{"key not found"}; + return it->second; + } + + std::unordered_map + key_to_doc_id_; + + std::vector + doc_id_to_key_; +}; + +} // iris::ngram + +#endif diff --git a/include/iris/ngram/search_query.hpp b/include/iris/ngram/search_query.hpp new file mode 100644 index 0000000..949c276 --- /dev/null +++ b/include/iris/ngram/search_query.hpp @@ -0,0 +1,86 @@ +#ifndef IRIS_ZZ_NGRAM_SEARCH_QUERY_HPP +#define IRIS_ZZ_NGRAM_SEARCH_QUERY_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace iris::ngram { + +template +struct search_query +{ + search_query() = default; + + explicit search_query(std::basic_string_view input_sv) + { + std::basic_string input{input_sv}; + iris::compact_spaces(input); + if (input.empty()) return; + + words_ = input + | std::views::split(iris::detail::string_algo_traits::space) + | std::views::transform([](auto const& r) { + return std::basic_string{std::from_range, r}; + }) + | std::ranges::to(); + + std::ranges::sort(words_); + { + auto const [first, last] = std::ranges::unique(words_); + words_.erase(first, last); + } + } + + // ------------------------------------------ + + [[nodiscard]] auto const& words() const noexcept + { + return words_; + } + + [[nodiscard]] bool empty() const noexcept + { + return words_.empty(); + } + + [[nodiscard]] bool operator==(search_query const& other) const noexcept + { + return words_ == other.words_; + } + +private: + std::vector> words_; +}; + +template +search_query(CharT const(&)[N]) -> search_query; + +} // iris::ngram + +template +struct std::formatter, CharT> + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram::search_query const& query, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{}", query.words() | std::views::transform([](std::u32string_view ustr) { + return iris::unicode::transcode(ustr); + })); + } +}; + +#endif diff --git a/include/iris/ngram/search_result.hpp b/include/iris/ngram/search_result.hpp new file mode 100644 index 0000000..7e39c5f --- /dev/null +++ b/include/iris/ngram/search_result.hpp @@ -0,0 +1,120 @@ +#ifndef IRIS_ZZ_NGRAM_SEARCH_RESULT_HPP +#define IRIS_ZZ_NGRAM_SEARCH_RESULT_HPP + +// SPDX-License-Identifier: MIT + +#include // IWYU pragma: keep + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace iris::ngram { + +namespace detail { +class search_result_cache; +} // detail + +template +class database; + +struct [[nodiscard]] search_word_match +{ + search_word_match() = default; + + explicit search_word_match(int word_id) + : word_id_(word_id) + {} + + search_word_match(int word_id, std::initializer_list> spans) + : word_id_(word_id) + , spans_(spans) + {} + + [[nodiscard]] int word_id() const noexcept { return word_id_; } + [[nodiscard]] auto const& spans() const noexcept { return spans_; } + + [[nodiscard]] bool operator==(search_word_match const& other) const noexcept + { + return word_id_ == other.word_id_ && spans_ == other.spans_; + } + +private: + friend class detail::search_result_cache; + + template + friend class database; + + int word_id_ = 0; + unsigned successful_ngrams_ = 1; // due to the class layout, this must be placed here + std::vector> spans_; +}; + +template +struct [[nodiscard]] search_result +{ + using document_id_type = DocumentID; + using map_type = std::unordered_map>; + + search_result() = default; + + template + requires requires(map_type& doc_matches) { + doc_matches.insert_range(std::declval()); + } + void assign(DocumentMatchMap&& doc_match_map) + { + doc_matches_.clear(); + if constexpr (std::ranges::sized_range) { + doc_matches_.reserve(std::ranges::size(doc_match_map)); + } + doc_matches_.insert_range(std::forward(doc_match_map)); + } + + [[nodiscard]] map_type const& doc_matches() const noexcept + { + return doc_matches_; + } + + void clear() noexcept + { + doc_matches_.clear(); + } + + [[nodiscard]] bool empty() const noexcept + { + return doc_matches_.empty(); + } + + [[nodiscard]] explicit operator bool() const noexcept + { + return !this->empty(); + } + +private: + map_type doc_matches_; +}; + +} // iris::ngram + +template +struct std::formatter + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram::search_word_match const& word_match, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{{word: #{}, spans: {}}}", word_match.word_id(), word_match.spans()); + } +}; + +#endif diff --git a/iris.natvis b/iris.natvis index b928d79..77c0f8e 100644 --- a/iris.natvis +++ b/iris.natvis @@ -252,4 +252,19 @@ (int)index_ + + + + + {chars._Elems,na1} + + + {chars._Elems,na2} + + + {chars._Elems,na3} + + + {chars._Elems,na4} + diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6a46bc3..7ea03d5 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -69,23 +69,6 @@ target_link_libraries(Catch2WithMain PRIVATE iris_cxx_test_external) target_link_libraries(iris_cxx_test INTERFACE Catch2::Catch2) - -# ----------------------------------------------------------------- -# Iris internal test targets - -add_library(_iris_internal_test INTERFACE) -target_include_directories(_iris_internal_test INTERFACE ${CMAKE_CURRENT_LIST_DIR}) - -if(MSVC) - target_sources(_iris_internal_test INTERFACE "${CMAKE_CURRENT_LIST_DIR}/cpp.hint") -endif() - -function(iris_define_internal_test test_name) - iris_define_test(${test_name} ${ARGN}) - target_link_libraries(${test_name}_test PRIVATE _iris_internal_test) -endfunction() - - # ----------------------------------------------------------------- # Common CMake utilities for testing @@ -167,14 +150,33 @@ function(iris_define_library_test library_type test_name srcs) _iris_define_test_impl(${test_name} Catch2::Catch2) endfunction() +# ----------------------------------------------------------------- +# Iris internal test targets + +add_library(_iris_internal_test_base INTERFACE) +target_include_directories(_iris_internal_test_base INTERFACE ${CMAKE_CURRENT_LIST_DIR}) +if(MSVC) + target_sources(_iris_internal_test_base INTERFACE "${CMAKE_CURRENT_LIST_DIR}/cpp.hint") +endif() + +function(iris_define_internal_test test_name) + iris_define_test(iris_${test_name} ${ARGN}) + target_link_libraries(iris_${test_name}_test PRIVATE _iris_internal_test_base) + target_sources(iris_${test_name}_test PRIVATE FILE_SET HEADERS BASE_DIRS ${CMAKE_CURRENT_FUNCTION_LIST_DIR} FILES ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/iris_test.hpp) + set_target_properties(iris_${test_name}_test PROPERTIES FOLDER "test/iris") +endfunction() + +function(iris_define_internal_subdir_test subdir test_name) + list(TRANSFORM ARGN PREPEND "${subdir}/") + iris_define_internal_test(${subdir}_${test_name} ${ARGN}) + set_target_properties(iris_${subdir}_${test_name}_test PROPERTIES FOLDER "test/iris/${subdir}") +endfunction() # ----------------------------------------------------------------- # Iris tests if(PROJECT_IS_TOP_LEVEL) if(NOT DEFINED IRIS_CI_COMPONENT OR IRIS_CI_COMPONENT STREQUAL iris) - add_subdirectory(rvariant) - set( IRIS_TEST_IRIS_TESTS core @@ -190,13 +192,23 @@ if(PROJECT_IS_TOP_LEVEL) interval_set snippet ) - foreach(test_name IN LISTS IRIS_TEST_IRIS_TESTS) - iris_define_internal_test(iris_${test_name} ${test_name}.cpp) - iris_define_test_headers(iris_${test_name} iris_test.hpp) - set_target_properties(iris_${test_name}_test PROPERTIES FOLDER "test/iris") + iris_define_internal_test(${test_name} ${test_name}.cpp) endforeach() + set( + IRIS_TEST_NGRAM_TESTS + ngram + search_2 + search_3 + search_4_dep + ) + foreach(test_name IN LISTS IRIS_TEST_NGRAM_TESTS) + iris_define_internal_subdir_test(ngram ${test_name} ${test_name}.cpp) + iris_define_test_headers(iris_ngram_${test_name} ngram/ngram_test.hpp) + endforeach() + + add_subdirectory(rvariant) add_subdirectory(unicode) endif() endif() diff --git a/test/ngram/ngram.cpp b/test/ngram/ngram.cpp new file mode 100644 index 0000000..3d20788 --- /dev/null +++ b/test/ngram/ngram.cpp @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: MIT + +#include "ngram_test.hpp" + +#include + +#include + +#include +#include + +TEST_CASE("ngram: type traits") +{ + STATIC_CHECK(sizeof(iris::ngram::gram<1, char>) == 1); + STATIC_CHECK(sizeof(iris::ngram::gram<2, char>) == 2); + STATIC_CHECK(sizeof(iris::ngram::gram<3, char>) == 3); + STATIC_CHECK(std::is_trivially_copyable_v>); + STATIC_CHECK(std::is_trivially_copyable_v>); + STATIC_CHECK(std::is_trivially_copyable_v>); + + STATIC_CHECK(sizeof(iris::ngram::gram<1, char32_t>) == 4); + STATIC_CHECK(sizeof(iris::ngram::gram<2, char32_t>) == 8); + STATIC_CHECK(sizeof(iris::ngram::gram<3, char32_t>) == 12); // TODO: optimize + STATIC_CHECK(std::is_trivially_copyable_v>); + STATIC_CHECK(std::is_trivially_copyable_v>); + STATIC_CHECK(std::is_trivially_copyable_v>); +} + +TEST_CASE("ngram: update document") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram::database<> ngram_db; + + auto const doc_id = ngram_db.add_document(U"abc"); + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + + ngram_db.set_visible(doc_id, false); + IRIS_CHECK_SEARCH("abc"); + IRIS_CHECK_SEARCH("ab"); + IRIS_CHECK_SEARCH("bc"); + + ngram_db.set_visible(doc_id, true); + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + + ngram_db.remove_document(doc_id); + IRIS_CHECK_SEARCH("abc"); + IRIS_CHECK_SEARCH("ab"); + IRIS_CHECK_SEARCH("bc"); + } + + { + iris::ngram::database<> ngram_db; + + auto const doc_id = ngram_db.add_document(U"abc"); + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + + ngram_db.update_document(doc_id, U"abd"); + IRIS_CHECK_SEARCH("abc"); + IRIS_CHECK_SEARCH( + "abd", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("bc"); + IRIS_CHECK_SEARCH( + "bd", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + + ngram_db.remove_document(doc_id); + IRIS_CHECK_SEARCH("abc"); + IRIS_CHECK_SEARCH("ab"); + IRIS_CHECK_SEARCH("bc"); + IRIS_CHECK_SEARCH("abd"); + IRIS_CHECK_SEARCH("bd"); + } +} + +TEST_CASE("ngram: keyed_database") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram::keyed_database + ngram_db; + + ngram_db.add_document("doc0", U"abc"); + IRIS_CHECK_SEARCH( + "abc", + {"doc0", { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {"doc0", { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {"doc0", { + {0, {interval{1, 3}}}, + }}, + ); + + ngram_db.set_visible("doc0", false); + IRIS_CHECK_SEARCH("abc"); + IRIS_CHECK_SEARCH("ab"); + IRIS_CHECK_SEARCH("bc"); + + ngram_db.set_visible("doc0", true); + IRIS_CHECK_SEARCH( + "abc", + {"doc0", { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {"doc0", { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {"doc0", { + {0, {interval{1, 3}}}, + }}, + ); + + ngram_db.remove_document("doc0"); + IRIS_CHECK_SEARCH("abc"); + IRIS_CHECK_SEARCH("ab"); + IRIS_CHECK_SEARCH("bc"); + } +} diff --git a/test/ngram/ngram_test.hpp b/test/ngram/ngram_test.hpp new file mode 100644 index 0000000..18fe66c --- /dev/null +++ b/test/ngram/ngram_test.hpp @@ -0,0 +1,73 @@ +#ifndef IRIS_ZZ_TEST_NGRAM_TEST_HPP +#define IRIS_ZZ_TEST_NGRAM_TEST_HPP + +// SPDX-License-Identifier: MIT + +#include "iris_test.hpp" + +#include +#include + +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export + +#ifdef _MSC_VER +# include // IWYU pragma: export +#endif + +using namespace iris::ngram_literals; +using iris::interval; + +template +struct DocumentMatch +{ + DocumentID doc_id; + std::vector word_matches; + + template + DocumentMatch(ID&& doc_id, std::initializer_list word_matches) + : doc_id(std::forward(doc_id)) + , word_matches(word_matches) + {} + + template + DocumentMatch(ID&& doc_id, std::vector word_matches) + : doc_id(std::forward(doc_id)) + , word_matches(std::move(word_matches)) + {} + + [[nodiscard]] + bool operator==(DocumentMatch const&) const noexcept = default; +}; + +template +struct std::formatter, CharT> + : iris::no_spec_formatter +{ + template + Ctx::iterator format(DocumentMatch const& doc_match, Ctx& ctx) const + { + return std::format_to(ctx.out(), "(doc: `{}`, word_matches: {})", doc_match.doc_id, doc_match.word_matches); + } +}; + +#define IRIS_CHECK_SEARCH(query_input, ...) do { \ + iris::ngram::search_query const query{U ## query_input}; \ + iris::ngram::search_result::document_id_type> search_res; \ + ngram_db.search(query, search_res); \ + auto const& doc_matches = search_res.doc_matches(); \ + \ + std::vector::document_id_type>> const expected_doc_matches{ \ + std::initializer_list::document_id_type>>{__VA_ARGS__} \ + }; \ + \ + auto const actual_doc_matches = doc_matches | std::views::transform([](auto const& kv) { \ + return DocumentMatch::document_id_type>{kv.first, kv.second}; \ + }) | std::ranges::to(); \ + CHECK(actual_doc_matches == expected_doc_matches); \ + } while (false) + +#endif diff --git a/test/ngram/search_2.cpp b/test/ngram/search_2.cpp new file mode 100644 index 0000000..916af2e --- /dev/null +++ b/test/ngram/search_2.cpp @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: MIT + +#include "ngram_test.hpp" + +TEST_CASE("gram search (document chars = 0)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram::database<> ngram_db; + IRIS_CHECK_SEARCH(""); + IRIS_CHECK_SEARCH("X"); + IRIS_CHECK_SEARCH("XX"); + } +} + +// 1-gram document +TEST_CASE("gram search (document chars = 1)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"a"); + IRIS_CHECK_SEARCH(""); + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + IRIS_CHECK_SEARCH("XX"); + } +} + +// 2-gram document +TEST_CASE("gram search (document chars = 2)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"aa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("XXX"); + } + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"ab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + } +} diff --git a/test/ngram/search_3.cpp b/test/ngram/search_3.cpp new file mode 100644 index 0000000..c99e050 --- /dev/null +++ b/test/ngram/search_3.cpp @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: MIT + +#include "ngram_test.hpp" + +// 2-gram + 1-gram document +TEST_CASE("gram search (document chars = 3, aaa/baa)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"aaa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}, interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aaa", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("aaaX"); + IRIS_CHECK_SEARCH("Xaaa"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXXX"); + } + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"baa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{1, 2}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "baa", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("baX"); + IRIS_CHECK_SEARCH("Xba"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("baaX"); + IRIS_CHECK_SEARCH("Xbaa"); + IRIS_CHECK_SEARCH("baXX"); + IRIS_CHECK_SEARCH("XXba"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2-gram + 1-gram document +TEST_CASE("gram search (document chars = 3, aba/aab)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"aba"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aba", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("baX"); + IRIS_CHECK_SEARCH("Xba"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("abaX"); + IRIS_CHECK_SEARCH("Xaba"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXba"); + IRIS_CHECK_SEARCH("baXX"); + IRIS_CHECK_SEARCH("XXXX"); + } + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"aab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aab", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("aabX"); + IRIS_CHECK_SEARCH("Xaab"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2-gram + 1-gram document +TEST_CASE("gram search (document chars = 3, abc)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"abc"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "c", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("bcX"); + IRIS_CHECK_SEARCH("Xbc"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("abcX"); + IRIS_CHECK_SEARCH("Xabc"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXbc"); + IRIS_CHECK_SEARCH("bcXX"); + IRIS_CHECK_SEARCH("XXXX"); + } +} diff --git a/test/ngram/search_4_dep.cpp b/test/ngram/search_4_dep.cpp new file mode 100644 index 0000000..714bb22 --- /dev/null +++ b/test/ngram/search_4_dep.cpp @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: MIT + +#include "ngram_test.hpp" + +// 2x 2-gram document +TEST_CASE("gram search (document chars = 4, aaaa)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"aaaa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}, interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}, interval{1, 3}, interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aaa", + {0_doc_id, { + {0, {interval{0, 3}, interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "aaaa", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaaX"); + IRIS_CHECK_SEARCH("Xaaa"); + IRIS_CHECK_SEARCH("XXXX"); + + IRIS_CHECK_SEARCH("aaaaX"); + IRIS_CHECK_SEARCH("Xaaaa"); + IRIS_CHECK_SEARCH("XXaaa"); + IRIS_CHECK_SEARCH("aaaXX"); + IRIS_CHECK_SEARCH("XXXXX"); + } +} + +// 2x 2-gram document +TEST_CASE("gram search (document chars = 4, abab)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"abab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}, interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}, interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aba", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bab", + {0_doc_id, { + {0, {interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "abab", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("ababX"); + IRIS_CHECK_SEARCH("Xabab"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2x 2-gram document +TEST_CASE("gram search (document chars = 4, abca)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"abca"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "c", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ca", + {0_doc_id, { + {0, {interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bca", + {0_doc_id, { + {0, {interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xbc"); + IRIS_CHECK_SEARCH("caX"); + IRIS_CHECK_SEARCH("Xca"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "abca", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abcaX"); + IRIS_CHECK_SEARCH("Xabca"); + IRIS_CHECK_SEARCH("caab"); // "ca" and "ab" both exist but not contiguous as "caab" + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2x 2-gram document +TEST_CASE("gram search (document chars = 4, abcd)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "d", + {0_doc_id, { + {0, {interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "cd", + {0_doc_id, { + {0, {interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bcd", + {0_doc_id, { + {0, {interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "abcd", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abcX"); + IRIS_CHECK_SEARCH("Xbcd"); + IRIS_CHECK_SEARCH("abcdX"); + IRIS_CHECK_SEARCH("Xabcd"); + IRIS_CHECK_SEARCH("abXcd"); // both halves exist; broken by X in the middle... but see note below! + IRIS_CHECK_SEARCH("acbd"); // all chars exist; order scrambled + IRIS_CHECK_SEARCH("XXXX"); + } +} + +TEST_CASE("gram search (dependency on previous match)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH("abXX"); + } + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXXef"); + } + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXXefef"); + } + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abef"); + } + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abefef"); + } + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abXXef"); + } + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abXXefef"); + } + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXef"); + } + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXefef"); + } + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"abxcd"); // trap document + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH( + "abcd", + {1_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + } + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"abcd"); + (void)ngram_db.add_document(U"abxcd"); // trap document + IRIS_CHECK_SEARCH( + "abcd", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + } + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"ab"); + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH( + "ab cd", + {1_doc_id, { + {0, {interval{0, 2}}}, + {1, {interval{2, 4}}}, + }}, + ); + } + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"abXcd"); + (void)ngram_db.add_document(U"abcdX"); + IRIS_CHECK_SEARCH("abcdc"); + } + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"abXXabcd"); + IRIS_CHECK_SEARCH( + "abcd", + {0_doc_id, { + {0, {interval{4, 8}}}, + }}, + ); + } + + { + iris::ngram::database<> ngram_db; + (void)ngram_db.add_document(U"ab cdXf"); + (void)ngram_db.add_document(U"ab cdef"); + IRIS_CHECK_SEARCH( + "ab cdef", + {1_doc_id, { + {0, {interval{0, 2}}}, + {1, {interval{3, 7}}}, + }}, + ); + } +} diff --git a/test/rvariant/CMakeLists.txt b/test/rvariant/CMakeLists.txt index 0404d61..cb81abe 100644 --- a/test/rvariant/CMakeLists.txt +++ b/test/rvariant/CMakeLists.txt @@ -16,6 +16,6 @@ endif() foreach(test_name IN LISTS IRIS_TEST_RVARIANT_TESTS) iris_define_internal_test(rvariant_${test_name} ${test_name}.cpp) - iris_define_test_headers(rvariant_${test_name} iris_rvariant_test.hpp) - set_target_properties(rvariant_${test_name}_test PROPERTIES FOLDER "test/rvariant") + iris_define_test_headers(iris_rvariant_${test_name} iris_rvariant_test.hpp) + set_target_properties(iris_rvariant_${test_name}_test PROPERTIES FOLDER "test/rvariant") endforeach() diff --git a/test/unicode/string/CMakeLists.txt b/test/unicode/string/CMakeLists.txt index a7d1e48..68c9eff 100644 --- a/test/unicode/string/CMakeLists.txt +++ b/test/unicode/string/CMakeLists.txt @@ -8,7 +8,7 @@ set( foreach(test_name IN LISTS IRIS_TEST_UNICODE_STRING_TESTS) iris_define_internal_test(unicode_string_${test_name} ${test_name}.cpp) - set_target_properties(unicode_string_${test_name}_test PROPERTIES FOLDER "test/unicode/string") + set_target_properties(iris_unicode_string_${test_name}_test PROPERTIES FOLDER "test/unicode/string") endforeach() -target_sources(unicode_string_utf8_invalid_test PRIVATE test_data/utf8_invalid.txt) +target_sources(iris_unicode_string_utf8_invalid_test PRIVATE test_data/utf8_invalid.txt)