Add zstd dictionary compression strategy

keyvi/include/keyvi/compression/compression_algorithm.h

@@ -26,6 +26,7 @@ enum CompressionAlgorithm {
   ZLIB_COMPRESSION = 1,
   SNAPPY_COMPRESSION = 2,
   ZSTD_COMPRESSION = 3,
+  ZSTD_DICT_COMPRESSION = 4,
 };
 
 } /* namespace compression */

keyvi/include/keyvi/compression/zstd_dict_compression_strategy.h

@@ -0,0 +1,120 @@
+/* * keyvi - A key value store.
+ *
+ * Copyright 2025 Hendrik Muhs<hendrik.muhs@gmail.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef KEYVI_COMPRESSION_ZSTD_DICT_COMPRESSION_STRATEGY_H_
+#define KEYVI_COMPRESSION_ZSTD_DICT_COMPRESSION_STRATEGY_H_
+
+#include <zstd.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <stdexcept>
+#include <string>
+
+#include "keyvi/compression/compression_algorithm.h"
+#include "keyvi/compression/compression_strategy.h"
+
+#ifndef ZSTD_DEFAULT_CLEVEL
+constexpr int kZstdDefaultCompressionLevel = 3;
+#else
+constexpr int kZstdDefaultCompressionLevel = ZSTD_DEFAULT_CLEVEL;
+#endif
+
+namespace keyvi::compression {
+
+struct ZstdDictCompressionStrategy final : public CompressionStrategy {
+  ZstdDictCompressionStrategy(const char* dict_data, size_t dict_size,
+                              int compression_level = kZstdDefaultCompressionLevel)
+      : cctx_(ZSTD_createCCtx()),
+        dctx_(ZSTD_createDCtx()),
+        cdict_(ZSTD_createCDict(dict_data, dict_size, compression_level)),
+        ddict_(ZSTD_createDDict(dict_data, dict_size)) {
+    if (cctx_ == nullptr || dctx_ == nullptr || cdict_ == nullptr || ddict_ == nullptr) {
+      Cleanup();
+      throw std::runtime_error("failed to initialize zstd dictionary compression");
+    }
+  }
+
+  ~ZstdDictCompressionStrategy() override { Cleanup(); }
+
+  ZstdDictCompressionStrategy(const ZstdDictCompressionStrategy&) = delete;
+  ZstdDictCompressionStrategy& operator=(const ZstdDictCompressionStrategy&) = delete;
+  ZstdDictCompressionStrategy(ZstdDictCompressionStrategy&&) = delete;
+  ZstdDictCompressionStrategy& operator=(ZstdDictCompressionStrategy&&) = delete;
+
+  using CompressionStrategy::Compress;
+
+  void Compress(buffer_t* buffer, const char* raw, size_t raw_size) override {
+    size_t output_length = ZSTD_compressBound(raw_size);
+    buffer->resize(output_length + 1);
+    (*buffer)[0] = static_cast<char>(ZSTD_DICT_COMPRESSION);
+
+    // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
+    output_length = ZSTD_compress_usingCDict(cctx_, buffer->data() + 1, output_length, raw, raw_size, cdict_);
+    if (ZSTD_isError(output_length) != 0U) {
+      throw std::runtime_error(std::string("zstd dict compression failed: ") + ZSTD_getErrorName(output_length));
+    }
+    buffer->resize(output_length + 1);
+  }
+
+  std::string Decompress(const char* data, const size_t size) override {
+    // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
+    const size_t dest_size = ZSTD_getFrameContentSize(data + 1, size - 1);
+    if (dest_size == ZSTD_CONTENTSIZE_UNKNOWN || dest_size == ZSTD_CONTENTSIZE_ERROR) {
+      throw std::runtime_error("zstd dict decompression failed: unable to determine content size");
+    }
+
+    std::string uncompressed;
+    uncompressed.resize(dest_size);
+    // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
+    const size_t result = ZSTD_decompress_usingDDict(dctx_, uncompressed.data(), dest_size, data + 1, size - 1, ddict_);
+    if (ZSTD_isError(result) != 0U) {
+      throw std::runtime_error(std::string("zstd dict decompression failed: ") + ZSTD_getErrorName(result));
+    }
+
+    return uncompressed;
+  }
+
+  [[nodiscard]] std::string name() const override { return "zstd_dict"; }
+
+  [[nodiscard]] uint64_t GetFileVersionMin() const override { return 4; }
+
+ private:
+  void Cleanup() {
+    if (cctx_ != nullptr) {
+      ZSTD_freeCCtx(cctx_);
+    }
+    if (dctx_ != nullptr) {
+      ZSTD_freeDCtx(dctx_);
+    }
+    if (cdict_ != nullptr) {
+      ZSTD_freeCDict(cdict_);
+    }
+    if (ddict_ != nullptr) {
+      ZSTD_freeDDict(ddict_);
+    }
+  }
+
+  ZSTD_CCtx* cctx_;
+  ZSTD_DCtx* dctx_;
+  ZSTD_CDict* cdict_;
+  ZSTD_DDict* ddict_;
+};
+
+}  // namespace keyvi::compression
+
+#endif  // KEYVI_COMPRESSION_ZSTD_DICT_COMPRESSION_STRATEGY_H_

keyvi/include/keyvi/dictionary/fsa/internal/constants.h

@@ -37,7 +37,7 @@ static const size_t KEYVI_FILE_MAGIC_LEN = 8;
 // min version of the file format
 static const uint64_t KEYVI_FILE_VERSION_MIN = 2;
 // max version of the file format supported
-static const uint64_t KEYVI_FILE_VERSION_MAX = 3;
+static const uint64_t KEYVI_FILE_VERSION_MAX = 4;
 
 // min version of the persistence part
 static const int KEYVI_FILE_PERSISTENCE_VERSION_MIN = 2;

keyvi/tests/keyvi/compression/zstd_dict_compression_strategy_test.cpp

@@ -0,0 +1,137 @@
+/* keyvi - A key value store.
+ *
+ * Copyright 2025 Hendrik Muhs<hendrik.muhs@gmail.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <zdict.h>
+#include <zstd.h>
+
+#include <string>
+#include <vector>
+
+#include <boost/test/unit_test.hpp>
+
+#include "keyvi/compression/zstd_dict_compression_strategy.h"
+
+namespace keyvi {
+namespace compression {
+
+BOOST_AUTO_TEST_SUITE(ZstdDictCompressionStrategyTests)
+
+namespace {
+
+std::vector<char> TrainDictionary(const std::vector<std::string>& samples, size_t dict_capacity = 4096) {
+  std::vector<char> combined;
+  std::vector<size_t> sample_sizes;
+  for (const auto& s : samples) {
+    combined.insert(combined.end(), s.begin(), s.end());
+    sample_sizes.push_back(s.size());
+  }
+
+  std::vector<char> dict_buffer(dict_capacity);
+  const size_t dict_size = ZDICT_trainFromBuffer(dict_buffer.data(), dict_buffer.size(), combined.data(),
+                                                 sample_sizes.data(), static_cast<unsigned>(samples.size()));
+  if (ZSTD_isError(dict_size) != 0U) {
+    dict_buffer.clear();
+    return dict_buffer;
+  }
+  dict_buffer.resize(dict_size);
+  return dict_buffer;
+}
+
+}  // namespace
+
+BOOST_AUTO_TEST_CASE(CompressAndDecompress) {
+  std::vector<std::string> samples;
+  samples.reserve(200);
+  for (int i = 0; i < 200; ++i) {
+    samples.push_back("the quick brown fox jumps over the lazy dog " + std::to_string(i));
+  }
+
+  auto dict = TrainDictionary(samples);
+  BOOST_REQUIRE(!dict.empty());
+
+  ZstdDictCompressionStrategy strategy(dict.data(), dict.size());
+
+  const std::string input = "the quick brown fox jumps over the lazy dog 42";
+  auto compressed = strategy.Compress(input);
+
+  BOOST_CHECK_EQUAL(static_cast<unsigned char>(compressed[0]), ZSTD_DICT_COMPRESSION);
+
+  auto decompressed = strategy.Decompress(compressed.data(), compressed.size());
+  BOOST_CHECK_EQUAL(input, decompressed);
+}
+
+BOOST_AUTO_TEST_CASE(CompressedSmallerThanPlainZstd) {
+  std::vector<std::string> samples;
+  samples.reserve(200);
+  for (int i = 0; i < 200; ++i) {
+    samples.push_back("the quick brown fox jumps over the lazy dog " + std::to_string(i));
+  }
+
+  auto dict = TrainDictionary(samples);
+  BOOST_REQUIRE(!dict.empty());
+
+  ZstdDictCompressionStrategy dict_strategy(dict.data(), dict.size());
+
+  const std::string input = "the quick brown fox jumps over the lazy dog 99";
+
+  buffer_t dict_buf;
+  dict_strategy.Compress(&dict_buf, input.data(), input.size());
+
+  buffer_t plain_buf;
+  plain_buf.resize(ZSTD_compressBound(input.size()) + 1);
+  const size_t plain_size =
+      ZSTD_compress(plain_buf.data(), plain_buf.size(), input.data(), input.size(), kZstdDefaultCompressionLevel);
+
+  BOOST_CHECK(dict_buf.size() <= plain_size + 1);
+}
+
+BOOST_AUTO_TEST_CASE(EmptyInput) {
+  std::vector<std::string> samples;
+  samples.reserve(200);
+  for (int i = 0; i < 200; ++i) {
+    samples.push_back("sample data " + std::to_string(i));
+  }
+
+  auto dict = TrainDictionary(samples);
+  BOOST_REQUIRE(!dict.empty());
+
+  ZstdDictCompressionStrategy strategy(dict.data(), dict.size());
+
+  const std::string input;
+  auto compressed = strategy.Compress(input);
+  auto decompressed = strategy.Decompress(compressed.data(), compressed.size());
+  BOOST_CHECK_EQUAL(input, decompressed);
+}
+
+BOOST_AUTO_TEST_CASE(Name) {
+  std::vector<std::string> samples;
+  samples.reserve(200);
+  for (int i = 0; i < 200; ++i) {
+    samples.push_back("sample " + std::to_string(i));
+  }
+
+  auto dict = TrainDictionary(samples);
+  BOOST_REQUIRE(!dict.empty());
+
+  const ZstdDictCompressionStrategy strategy(dict.data(), dict.size());
+  BOOST_CHECK_EQUAL("zstd_dict", strategy.name());
+}
+
+BOOST_AUTO_TEST_SUITE_END()
+
+}  // namespace compression
+}  // namespace keyvi