From c51690c385c8a2dc703cfe7042ebe1278cd10ee3 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 21 Sep 2026 08:42:53 +0200 Subject: [PATCH 1/6] Add zstd dictionary compression strategy Add ZstdDictCompressionStrategy that uses pre-trained zstd dictionaries for improved compression of small values. The dictionary is passed at construction time and used for both compression and decompression via ZSTD_CDict/ZSTD_DDict. Co-Authored-By: Claude Code --- .../keyvi/compression/compression_algorithm.h | 1 + .../keyvi/compression/compression_selector.h | 1 + .../zstd_dict_compression_strategy.h | 104 ++++++++++++++ .../keyvi/dictionary/fsa/internal/constants.h | 2 +- .../zstd_dict_compression_strategy_test.cpp | 132 ++++++++++++++++++ 5 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h create mode 100644 keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp diff --git a/keyvi/include/keyvi/compression/compression_algorithm.h b/keyvi/include/keyvi/compression/compression_algorithm.h index 2a3186df9..29c898bb0 100644 --- a/keyvi/include/keyvi/compression/compression_algorithm.h +++ b/keyvi/include/keyvi/compression/compression_algorithm.h @@ -26,6 +26,7 @@ enum CompressionAlgorithm { ZLIB_COMPRESSION = 1, SNAPPY_COMPRESSION = 2, ZSTD_COMPRESSION = 3, + ZSTD_DICT_COMPRESSION = 4, }; } /* namespace compression */ diff --git a/keyvi/include/keyvi/compression/compression_selector.h b/keyvi/include/keyvi/compression/compression_selector.h index 5daf4167f..1255f65d6 100644 --- a/keyvi/include/keyvi/compression/compression_selector.h +++ b/keyvi/include/keyvi/compression/compression_selector.h @@ -37,6 +37,7 @@ #include "keyvi/compression/snappy_compression_strategy.h" #include "keyvi/compression/zlib_compression_strategy.h" #include "keyvi/compression/zstd_compression_strategy.h" +#include "keyvi/compression/zstd_dict_compression_strategy.h" // #define ENABLE_TRACING #include "keyvi/dictionary/util/trace.h" diff --git a/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h b/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h new file mode 100644 index 000000000..3fed6f682 --- /dev/null +++ b/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h @@ -0,0 +1,104 @@ +/* * keyvi - A key value store. + * + * Copyright 2025 Hendrik Muhs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef KEYVI_COMPRESSION_ZSTD_DICT_COMPRESSION_STRATEGY_H_ +#define KEYVI_COMPRESSION_ZSTD_DICT_COMPRESSION_STRATEGY_H_ + +#include + +#include +#include + +#include "keyvi/compression/compression_strategy.h" + +#ifndef ZSTD_DEFAULT_CLEVEL +#define ZSTD_DEFAULT_CLEVEL 3 +#endif + +namespace keyvi { +namespace compression { + +struct ZstdDictCompressionStrategy final : public CompressionStrategy { + ZstdDictCompressionStrategy(const char* dict_data, size_t dict_size, + int compression_level = ZSTD_DEFAULT_CLEVEL) + : cctx_(ZSTD_createCCtx()), + dctx_(ZSTD_createDCtx()), + cdict_(ZSTD_createCDict(dict_data, dict_size, compression_level)), + ddict_(ZSTD_createDDict(dict_data, dict_size)) { + if (!cctx_ || !dctx_ || !cdict_ || !ddict_) { + Cleanup(); + throw std::runtime_error("failed to initialize zstd dictionary compression"); + } + } + + ~ZstdDictCompressionStrategy() { Cleanup(); } + + ZstdDictCompressionStrategy(const ZstdDictCompressionStrategy&) = delete; + ZstdDictCompressionStrategy& operator=(const ZstdDictCompressionStrategy&) = delete; + + using CompressionStrategy::Compress; + + void Compress(buffer_t* buffer, const char* raw, size_t raw_size) override { + size_t output_length = ZSTD_compressBound(raw_size); + buffer->resize(output_length + 1); + buffer->data()[0] = static_cast(ZSTD_DICT_COMPRESSION); + + output_length = ZSTD_compress_usingCDict(cctx_, buffer->data() + 1, output_length, raw, raw_size, cdict_); + if (ZSTD_isError(output_length)) { + throw std::runtime_error(std::string("zstd dict compression failed: ") + ZSTD_getErrorName(output_length)); + } + buffer->resize(output_length + 1); + } + + std::string Decompress(const char* data, const size_t size) override { + const size_t dest_size = ZSTD_getFrameContentSize(data + 1, size - 1); + if (dest_size == ZSTD_CONTENTSIZE_UNKNOWN || dest_size == ZSTD_CONTENTSIZE_ERROR) { + throw std::runtime_error("zstd dict decompression failed: unable to determine content size"); + } + + std::string uncompressed; + uncompressed.resize(dest_size); + size_t result = ZSTD_decompress_usingDDict(dctx_, uncompressed.data(), dest_size, data + 1, size - 1, ddict_); + if (ZSTD_isError(result)) { + throw std::runtime_error(std::string("zstd dict decompression failed: ") + ZSTD_getErrorName(result)); + } + + return uncompressed; + } + + std::string name() const override { return "zstd_dict"; } + + uint64_t GetFileVersionMin() const override { return 4; } + + private: + void Cleanup() { + if (cctx_) ZSTD_freeCCtx(cctx_); + if (dctx_) ZSTD_freeDCtx(dctx_); + if (cdict_) ZSTD_freeCDict(cdict_); + if (ddict_) ZSTD_freeDDict(ddict_); + } + + ZSTD_CCtx* cctx_; + ZSTD_DCtx* dctx_; + ZSTD_CDict* cdict_; + ZSTD_DDict* ddict_; +}; + +} /* namespace compression */ +} /* namespace keyvi */ + +#endif // KEYVI_COMPRESSION_ZSTD_DICT_COMPRESSION_STRATEGY_H_ diff --git a/keyvi/include/keyvi/dictionary/fsa/internal/constants.h b/keyvi/include/keyvi/dictionary/fsa/internal/constants.h index cfbd81a12..558afcb96 100644 --- a/keyvi/include/keyvi/dictionary/fsa/internal/constants.h +++ b/keyvi/include/keyvi/dictionary/fsa/internal/constants.h @@ -37,7 +37,7 @@ static const size_t KEYVI_FILE_MAGIC_LEN = 8; // min version of the file format static const uint64_t KEYVI_FILE_VERSION_MIN = 2; // max version of the file format supported -static const uint64_t KEYVI_FILE_VERSION_MAX = 3; +static const uint64_t KEYVI_FILE_VERSION_MAX = 4; // min version of the persistence part static const int KEYVI_FILE_PERSISTENCE_VERSION_MIN = 2; diff --git a/keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp b/keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp new file mode 100644 index 000000000..32672c487 --- /dev/null +++ b/keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp @@ -0,0 +1,132 @@ +/* keyvi - A key value store. + * + * Copyright 2025 Hendrik Muhs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include + +#include + +#include "keyvi/compression/zstd_dict_compression_strategy.h" + +namespace keyvi { +namespace compression { + +BOOST_AUTO_TEST_SUITE(ZstdDictCompressionStrategyTests) + +namespace { + +std::vector TrainDictionary(const std::vector& samples, size_t dict_capacity = 4096) { + std::vector combined; + std::vector sample_sizes; + for (const auto& s : samples) { + combined.insert(combined.end(), s.begin(), s.end()); + sample_sizes.push_back(s.size()); + } + + std::vector dict_buffer(dict_capacity); + size_t dict_size = ZDICT_trainFromBuffer(dict_buffer.data(), dict_buffer.size(), combined.data(), sample_sizes.data(), + static_cast(samples.size())); + if (ZSTD_isError(dict_size)) { + dict_buffer.clear(); + return dict_buffer; + } + dict_buffer.resize(dict_size); + return dict_buffer; +} + +} // namespace + +BOOST_AUTO_TEST_CASE(CompressAndDecompress) { + std::vector samples; + for (int i = 0; i < 200; ++i) { + samples.push_back("the quick brown fox jumps over the lazy dog " + std::to_string(i)); + } + + auto dict = TrainDictionary(samples); + BOOST_REQUIRE(!dict.empty()); + + ZstdDictCompressionStrategy strategy(dict.data(), dict.size()); + + const std::string input = "the quick brown fox jumps over the lazy dog 42"; + auto compressed = strategy.Compress(input); + + BOOST_CHECK_EQUAL(static_cast(compressed[0]), ZSTD_DICT_COMPRESSION); + + auto decompressed = strategy.Decompress(compressed.data(), compressed.size()); + BOOST_CHECK_EQUAL(input, decompressed); +} + +BOOST_AUTO_TEST_CASE(CompressedSmallerThanPlainZstd) { + std::vector samples; + for (int i = 0; i < 200; ++i) { + samples.push_back("the quick brown fox jumps over the lazy dog " + std::to_string(i)); + } + + auto dict = TrainDictionary(samples); + BOOST_REQUIRE(!dict.empty()); + + ZstdDictCompressionStrategy dict_strategy(dict.data(), dict.size()); + + const std::string input = "the quick brown fox jumps over the lazy dog 99"; + + buffer_t dict_buf; + dict_strategy.Compress(&dict_buf, input.data(), input.size()); + + buffer_t plain_buf; + plain_buf.resize(ZSTD_compressBound(input.size()) + 1); + size_t plain_size = ZSTD_compress(plain_buf.data(), plain_buf.size(), input.data(), input.size(), ZSTD_DEFAULT_CLEVEL); + + BOOST_CHECK(dict_buf.size() <= plain_size + 1); +} + +BOOST_AUTO_TEST_CASE(EmptyInput) { + std::vector samples; + for (int i = 0; i < 200; ++i) { + samples.push_back("sample data " + std::to_string(i)); + } + + auto dict = TrainDictionary(samples); + BOOST_REQUIRE(!dict.empty()); + + ZstdDictCompressionStrategy strategy(dict.data(), dict.size()); + + const std::string input; + auto compressed = strategy.Compress(input); + auto decompressed = strategy.Decompress(compressed.data(), compressed.size()); + BOOST_CHECK_EQUAL(input, decompressed); +} + +BOOST_AUTO_TEST_CASE(Name) { + std::vector samples; + for (int i = 0; i < 200; ++i) { + samples.push_back("sample " + std::to_string(i)); + } + + auto dict = TrainDictionary(samples); + BOOST_REQUIRE(!dict.empty()); + + ZstdDictCompressionStrategy strategy(dict.data(), dict.size()); + BOOST_CHECK_EQUAL("zstd_dict", strategy.name()); +} + +BOOST_AUTO_TEST_SUITE_END() + +} // namespace compression +} // namespace keyvi From 937a389b304470b79ddc68cad5c85d05dd2f0983 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 21 Sep 2026 08:50:04 +0200 Subject: [PATCH 2/6] Fix clang-format issues in zstd dict compression Co-Authored-By: Claude Code --- .../keyvi/compression/zstd_dict_compression_strategy.h | 3 +-- .../compression/zstd_dict_compression_strategy_test.cpp | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h b/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h index 3fed6f682..49e2544d1 100644 --- a/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h +++ b/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h @@ -33,8 +33,7 @@ namespace keyvi { namespace compression { struct ZstdDictCompressionStrategy final : public CompressionStrategy { - ZstdDictCompressionStrategy(const char* dict_data, size_t dict_size, - int compression_level = ZSTD_DEFAULT_CLEVEL) + ZstdDictCompressionStrategy(const char* dict_data, size_t dict_size, int compression_level = ZSTD_DEFAULT_CLEVEL) : cctx_(ZSTD_createCCtx()), dctx_(ZSTD_createDCtx()), cdict_(ZSTD_createCDict(dict_data, dict_size, compression_level)), diff --git a/keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp b/keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp index 32672c487..c157ac764 100644 --- a/keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp +++ b/keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp @@ -42,7 +42,7 @@ std::vector TrainDictionary(const std::vector& samples, size_ std::vector dict_buffer(dict_capacity); size_t dict_size = ZDICT_trainFromBuffer(dict_buffer.data(), dict_buffer.size(), combined.data(), sample_sizes.data(), - static_cast(samples.size())); + static_cast(samples.size())); if (ZSTD_isError(dict_size)) { dict_buffer.clear(); return dict_buffer; @@ -91,7 +91,8 @@ BOOST_AUTO_TEST_CASE(CompressedSmallerThanPlainZstd) { buffer_t plain_buf; plain_buf.resize(ZSTD_compressBound(input.size()) + 1); - size_t plain_size = ZSTD_compress(plain_buf.data(), plain_buf.size(), input.data(), input.size(), ZSTD_DEFAULT_CLEVEL); + size_t plain_size = + ZSTD_compress(plain_buf.data(), plain_buf.size(), input.data(), input.size(), ZSTD_DEFAULT_CLEVEL); BOOST_CHECK(dict_buf.size() <= plain_size + 1); } From 41b0120208e53ca7208800b2d812fb4e3012a1f3 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 21 Sep 2026 09:05:12 +0200 Subject: [PATCH 3/6] Fix clang-tidy warnings in zstd dict compression strategy - Add override to destructor - Add deleted move constructor/assignment operator - Use explicit nullptr comparisons instead of implicit bool conversion - Use explicit != 0u for ZSTD_isError checks - Add braces around single-line if bodies - Use operator[] instead of data()[0] - Make result variable const - Add NOLINTNEXTLINE for unavoidable pointer arithmetic - Add missing direct includes for cstddef, cstdint, compression_algorithm.h Co-Authored-By: Claude Code --- .../zstd_dict_compression_strategy.h | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h b/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h index 49e2544d1..e0ec2a7ec 100644 --- a/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h +++ b/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h @@ -20,9 +20,12 @@ #include +#include +#include #include #include +#include "keyvi/compression/compression_algorithm.h" #include "keyvi/compression/compression_strategy.h" #ifndef ZSTD_DEFAULT_CLEVEL @@ -38,32 +41,36 @@ struct ZstdDictCompressionStrategy final : public CompressionStrategy { dctx_(ZSTD_createDCtx()), cdict_(ZSTD_createCDict(dict_data, dict_size, compression_level)), ddict_(ZSTD_createDDict(dict_data, dict_size)) { - if (!cctx_ || !dctx_ || !cdict_ || !ddict_) { + if (cctx_ == nullptr || dctx_ == nullptr || cdict_ == nullptr || ddict_ == nullptr) { Cleanup(); throw std::runtime_error("failed to initialize zstd dictionary compression"); } } - ~ZstdDictCompressionStrategy() { Cleanup(); } + ~ZstdDictCompressionStrategy() override { Cleanup(); } ZstdDictCompressionStrategy(const ZstdDictCompressionStrategy&) = delete; ZstdDictCompressionStrategy& operator=(const ZstdDictCompressionStrategy&) = delete; + ZstdDictCompressionStrategy(ZstdDictCompressionStrategy&&) = delete; + ZstdDictCompressionStrategy& operator=(ZstdDictCompressionStrategy&&) = delete; using CompressionStrategy::Compress; void Compress(buffer_t* buffer, const char* raw, size_t raw_size) override { size_t output_length = ZSTD_compressBound(raw_size); buffer->resize(output_length + 1); - buffer->data()[0] = static_cast(ZSTD_DICT_COMPRESSION); + (*buffer)[0] = static_cast(ZSTD_DICT_COMPRESSION); + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) output_length = ZSTD_compress_usingCDict(cctx_, buffer->data() + 1, output_length, raw, raw_size, cdict_); - if (ZSTD_isError(output_length)) { + if (ZSTD_isError(output_length) != 0u) { throw std::runtime_error(std::string("zstd dict compression failed: ") + ZSTD_getErrorName(output_length)); } buffer->resize(output_length + 1); } std::string Decompress(const char* data, const size_t size) override { + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) const size_t dest_size = ZSTD_getFrameContentSize(data + 1, size - 1); if (dest_size == ZSTD_CONTENTSIZE_UNKNOWN || dest_size == ZSTD_CONTENTSIZE_ERROR) { throw std::runtime_error("zstd dict decompression failed: unable to determine content size"); @@ -71,8 +78,9 @@ struct ZstdDictCompressionStrategy final : public CompressionStrategy { std::string uncompressed; uncompressed.resize(dest_size); - size_t result = ZSTD_decompress_usingDDict(dctx_, uncompressed.data(), dest_size, data + 1, size - 1, ddict_); - if (ZSTD_isError(result)) { + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + const size_t result = ZSTD_decompress_usingDDict(dctx_, uncompressed.data(), dest_size, data + 1, size - 1, ddict_); + if (ZSTD_isError(result) != 0u) { throw std::runtime_error(std::string("zstd dict decompression failed: ") + ZSTD_getErrorName(result)); } @@ -85,10 +93,18 @@ struct ZstdDictCompressionStrategy final : public CompressionStrategy { private: void Cleanup() { - if (cctx_) ZSTD_freeCCtx(cctx_); - if (dctx_) ZSTD_freeDCtx(dctx_); - if (cdict_) ZSTD_freeCDict(cdict_); - if (ddict_) ZSTD_freeDDict(ddict_); + if (cctx_ != nullptr) { + ZSTD_freeCCtx(cctx_); + } + if (dctx_ != nullptr) { + ZSTD_freeDCtx(dctx_); + } + if (cdict_ != nullptr) { + ZSTD_freeCDict(cdict_); + } + if (ddict_ != nullptr) { + ZSTD_freeDDict(ddict_); + } } ZSTD_CCtx* cctx_; From 0b47a1932907d739f14be66a6982e5d187781cbd Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 21 Sep 2026 09:17:06 +0200 Subject: [PATCH 4/6] Fix remaining clang-tidy warnings - Use uppercase literal suffix 0U instead of 0u - Add [[nodiscard]] to name() and GetFileVersionMin() - Remove unused include from compression_selector.h - Add samples.reserve(200) before loops in tests - Make dict_size, plain_size, strategy const where possible Co-Authored-By: Claude Code --- .../keyvi/compression/compression_selector.h | 1 - .../compression/zstd_dict_compression_strategy.h | 8 ++++---- .../zstd_dict_compression_strategy_test.cpp | 14 +++++++++----- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/keyvi/include/keyvi/compression/compression_selector.h b/keyvi/include/keyvi/compression/compression_selector.h index 1255f65d6..5daf4167f 100644 --- a/keyvi/include/keyvi/compression/compression_selector.h +++ b/keyvi/include/keyvi/compression/compression_selector.h @@ -37,7 +37,6 @@ #include "keyvi/compression/snappy_compression_strategy.h" #include "keyvi/compression/zlib_compression_strategy.h" #include "keyvi/compression/zstd_compression_strategy.h" -#include "keyvi/compression/zstd_dict_compression_strategy.h" // #define ENABLE_TRACING #include "keyvi/dictionary/util/trace.h" diff --git a/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h b/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h index e0ec2a7ec..abdefd2e2 100644 --- a/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h +++ b/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h @@ -63,7 +63,7 @@ struct ZstdDictCompressionStrategy final : public CompressionStrategy { // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) output_length = ZSTD_compress_usingCDict(cctx_, buffer->data() + 1, output_length, raw, raw_size, cdict_); - if (ZSTD_isError(output_length) != 0u) { + if (ZSTD_isError(output_length) != 0U) { throw std::runtime_error(std::string("zstd dict compression failed: ") + ZSTD_getErrorName(output_length)); } buffer->resize(output_length + 1); @@ -80,16 +80,16 @@ struct ZstdDictCompressionStrategy final : public CompressionStrategy { uncompressed.resize(dest_size); // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) const size_t result = ZSTD_decompress_usingDDict(dctx_, uncompressed.data(), dest_size, data + 1, size - 1, ddict_); - if (ZSTD_isError(result) != 0u) { + if (ZSTD_isError(result) != 0U) { throw std::runtime_error(std::string("zstd dict decompression failed: ") + ZSTD_getErrorName(result)); } return uncompressed; } - std::string name() const override { return "zstd_dict"; } + [[nodiscard]] std::string name() const override { return "zstd_dict"; } - uint64_t GetFileVersionMin() const override { return 4; } + [[nodiscard]] uint64_t GetFileVersionMin() const override { return 4; } private: void Cleanup() { diff --git a/keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp b/keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp index c157ac764..887890913 100644 --- a/keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp +++ b/keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp @@ -41,9 +41,9 @@ std::vector TrainDictionary(const std::vector& samples, size_ } std::vector dict_buffer(dict_capacity); - size_t dict_size = ZDICT_trainFromBuffer(dict_buffer.data(), dict_buffer.size(), combined.data(), sample_sizes.data(), - static_cast(samples.size())); - if (ZSTD_isError(dict_size)) { + const size_t dict_size = ZDICT_trainFromBuffer(dict_buffer.data(), dict_buffer.size(), combined.data(), + sample_sizes.data(), static_cast(samples.size())); + if (ZSTD_isError(dict_size) != 0U) { dict_buffer.clear(); return dict_buffer; } @@ -55,6 +55,7 @@ std::vector TrainDictionary(const std::vector& samples, size_ BOOST_AUTO_TEST_CASE(CompressAndDecompress) { std::vector samples; + samples.reserve(200); for (int i = 0; i < 200; ++i) { samples.push_back("the quick brown fox jumps over the lazy dog " + std::to_string(i)); } @@ -75,6 +76,7 @@ BOOST_AUTO_TEST_CASE(CompressAndDecompress) { BOOST_AUTO_TEST_CASE(CompressedSmallerThanPlainZstd) { std::vector samples; + samples.reserve(200); for (int i = 0; i < 200; ++i) { samples.push_back("the quick brown fox jumps over the lazy dog " + std::to_string(i)); } @@ -91,7 +93,7 @@ BOOST_AUTO_TEST_CASE(CompressedSmallerThanPlainZstd) { buffer_t plain_buf; plain_buf.resize(ZSTD_compressBound(input.size()) + 1); - size_t plain_size = + const size_t plain_size = ZSTD_compress(plain_buf.data(), plain_buf.size(), input.data(), input.size(), ZSTD_DEFAULT_CLEVEL); BOOST_CHECK(dict_buf.size() <= plain_size + 1); @@ -99,6 +101,7 @@ BOOST_AUTO_TEST_CASE(CompressedSmallerThanPlainZstd) { BOOST_AUTO_TEST_CASE(EmptyInput) { std::vector samples; + samples.reserve(200); for (int i = 0; i < 200; ++i) { samples.push_back("sample data " + std::to_string(i)); } @@ -116,6 +119,7 @@ BOOST_AUTO_TEST_CASE(EmptyInput) { BOOST_AUTO_TEST_CASE(Name) { std::vector samples; + samples.reserve(200); for (int i = 0; i < 200; ++i) { samples.push_back("sample " + std::to_string(i)); } @@ -123,7 +127,7 @@ BOOST_AUTO_TEST_CASE(Name) { auto dict = TrainDictionary(samples); BOOST_REQUIRE(!dict.empty()); - ZstdDictCompressionStrategy strategy(dict.data(), dict.size()); + const ZstdDictCompressionStrategy strategy(dict.data(), dict.size()); BOOST_CHECK_EQUAL("zstd_dict", strategy.name()); } From de76d2e649afeec803bb0c5cd5aabeab42f42687 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 21 Sep 2026 09:30:19 +0200 Subject: [PATCH 5/6] Replace macro with constexpr and use concatenated namespace - Replace ZSTD_DEFAULT_CLEVEL macro with constexpr kZstdDefaultCompressionLevel - Use C++17 concatenated namespace keyvi::compression Co-Authored-By: Claude Code --- .../compression/zstd_dict_compression_strategy.h | 12 ++++++------ .../zstd_dict_compression_strategy_test.cpp | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h b/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h index abdefd2e2..ccc14dffe 100644 --- a/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h +++ b/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h @@ -29,14 +29,15 @@ #include "keyvi/compression/compression_strategy.h" #ifndef ZSTD_DEFAULT_CLEVEL -#define ZSTD_DEFAULT_CLEVEL 3 +constexpr int kZstdDefaultCompressionLevel = 3; +#else +constexpr int kZstdDefaultCompressionLevel = ZSTD_DEFAULT_CLEVEL; #endif -namespace keyvi { -namespace compression { +namespace keyvi::compression { struct ZstdDictCompressionStrategy final : public CompressionStrategy { - ZstdDictCompressionStrategy(const char* dict_data, size_t dict_size, int compression_level = ZSTD_DEFAULT_CLEVEL) + ZstdDictCompressionStrategy(const char* dict_data, size_t dict_size, int compression_level = kZstdDefaultCompressionLevel) : cctx_(ZSTD_createCCtx()), dctx_(ZSTD_createDCtx()), cdict_(ZSTD_createCDict(dict_data, dict_size, compression_level)), @@ -113,7 +114,6 @@ struct ZstdDictCompressionStrategy final : public CompressionStrategy { ZSTD_DDict* ddict_; }; -} /* namespace compression */ -} /* namespace keyvi */ +} // namespace keyvi::compression #endif // KEYVI_COMPRESSION_ZSTD_DICT_COMPRESSION_STRATEGY_H_ diff --git a/keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp b/keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp index 887890913..763147a0c 100644 --- a/keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp +++ b/keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp @@ -94,7 +94,7 @@ BOOST_AUTO_TEST_CASE(CompressedSmallerThanPlainZstd) { buffer_t plain_buf; plain_buf.resize(ZSTD_compressBound(input.size()) + 1); const size_t plain_size = - ZSTD_compress(plain_buf.data(), plain_buf.size(), input.data(), input.size(), ZSTD_DEFAULT_CLEVEL); + ZSTD_compress(plain_buf.data(), plain_buf.size(), input.data(), input.size(), kZstdDefaultCompressionLevel); BOOST_CHECK(dict_buf.size() <= plain_size + 1); } From 5b150783220ba6864bdaaebb8e52fa9b0ead1611 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 21 Sep 2026 10:29:49 +0200 Subject: [PATCH 6/6] Fix clang-format line wrapping Co-Authored-By: Claude Code --- .../keyvi/compression/zstd_dict_compression_strategy.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h b/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h index ccc14dffe..44fbe41d7 100644 --- a/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h +++ b/keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h @@ -37,7 +37,8 @@ constexpr int kZstdDefaultCompressionLevel = ZSTD_DEFAULT_CLEVEL; namespace keyvi::compression { struct ZstdDictCompressionStrategy final : public CompressionStrategy { - ZstdDictCompressionStrategy(const char* dict_data, size_t dict_size, int compression_level = kZstdDefaultCompressionLevel) + ZstdDictCompressionStrategy(const char* dict_data, size_t dict_size, + int compression_level = kZstdDefaultCompressionLevel) : cctx_(ZSTD_createCCtx()), dctx_(ZSTD_createDCtx()), cdict_(ZSTD_createCDict(dict_data, dict_size, compression_level)), @@ -114,6 +115,6 @@ struct ZstdDictCompressionStrategy final : public CompressionStrategy { ZSTD_DDict* ddict_; }; -} // namespace keyvi::compression +} // namespace keyvi::compression #endif // KEYVI_COMPRESSION_ZSTD_DICT_COMPRESSION_STRATEGY_H_