Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ bazel_dep(name = "riegeli", version = "0.0.0-20250822-9f2744d")
bazel_dep(name = "platforms", version = "1.0.0")
bazel_dep(name = "rules_rust", version = "0.68.1")
bazel_dep(name = "glib", version = "2.82.2.bcr.7")
bazel_dep(name = "brotli", version = "1.2.0")
bazel_dep(name = "brotli", version = "1.2.1")
bazel_dep(name = "bazel_skylib", version = "1.9.0")

# Frequency Data
Expand Down
4 changes: 4 additions & 0 deletions brotli/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,16 @@ cc_library(
cc_library(
name = "shared_brotli_encoder",
hdrs = [
"brotli_arena.h",
"shared_brotli_encoder.h",
],
visibility = [
"//visibility:public",
],
deps = [
"@abseil-cpp//absl/log",
"@abseil-cpp//absl/strings",
"@abseil-cpp//absl/types:span",
"@brotli//:brotli_inc",
"@brotli//:brotlienc",
],
Expand Down
98 changes: 98 additions & 0 deletions brotli/brotli_arena.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
#ifndef BROTLI_BROTLI_ARENA_H_
#define BROTLI_BROTLI_ARENA_H_

#include <cstddef>
#include <cstdint>
#include <iterator>
#include <memory>
#include <vector>

#include "absl/log/log.h"

namespace brotli {

/* Reusable bump-pointer arena for Brotli encoder and dictionary allocations. */
class BrotliArena {
public:
static constexpr size_t kAlignment = alignof(std::max_align_t);
static_assert(kAlignment >= sizeof(size_t));

explicit BrotliArena(size_t capacity = 256u * 1024u * 1024u)
: capacity_(capacity), offset_(0), buf_(nullptr) {}

BrotliArena(const BrotliArena&) = delete;
BrotliArena& operator=(const BrotliArena&) = delete;
BrotliArena(BrotliArena&&) = default;
BrotliArena& operator=(BrotliArena&&) = default;

static void* Alloc(void* opaque, size_t size) {
if (!opaque) {
return nullptr;
}
auto* self = static_cast<BrotliArena*>(opaque);
if (!self->buf_) {
self->buf_.reset(new uint8_t[self->capacity_]);
}
// Round size up to multiple of kAlignment bytes for proper alignment.
size_t payload_size = (size + kAlignment - 1) & ~(kAlignment - 1);
size_t total_size = payload_size + kAlignment;
if (self->offset_ < self->capacity_ &&
total_size <= self->capacity_ - self->offset_) {
*reinterpret_cast<size_t*>(self->buf_.get() + self->offset_) =
payload_size;
void* ptr = self->buf_.get() + self->offset_ + kAlignment;
self->offset_ += total_size;
return ptr;
}
// Fall back to heap allocation if arena capacity is exceeded.
LOG(WARNING) << "BrotliArena capacity exceeded (requested " << size
<< " bytes at offset " << self->offset_ << " of "
<< self->capacity_ << "), falling back to new.";
self->fallback_allocs_.emplace_back(new uint8_t[size]);
return self->fallback_allocs_.back().get();
}

static void Free(void* opaque, void* address) {
if (!address || !opaque) {
return;
}
auto* self = static_cast<BrotliArena*>(opaque);
auto addr_val = reinterpret_cast<std::uintptr_t>(address);
auto buf_start = reinterpret_cast<std::uintptr_t>(self->buf_.get());
if (self->buf_ && addr_val >= buf_start + kAlignment &&
addr_val <= buf_start + self->offset_) {
size_t alloc_offset = addr_val - buf_start;
size_t payload_size = *reinterpret_cast<const size_t*>(
self->buf_.get() + (alloc_offset - kAlignment));
if (alloc_offset + payload_size == self->offset_) {
self->offset_ = alloc_offset - kAlignment;
}
return;
}
for (auto it = self->fallback_allocs_.rbegin();
it != self->fallback_allocs_.rend(); ++it) {
if (it->get() == address) {
self->fallback_allocs_.erase(std::next(it).base());
return;
}
}
}

void Reset() {
offset_ = 0;
fallback_allocs_.clear();
}

size_t offset() const { return offset_; }
size_t capacity() const { return capacity_; }

private:
size_t capacity_;
size_t offset_;
std::unique_ptr<uint8_t[]> buf_;
std::vector<std::unique_ptr<uint8_t[]>> fallback_allocs_;
};

} // namespace brotli

#endif // BROTLI_BROTLI_ARENA_H_
16 changes: 12 additions & 4 deletions brotli/shared_brotli_encoder.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@
#define BROTLI_SHARED_BROTLI_ENCODER_H_

#include <memory>
#include <vector>

#include "absl/log/log.h"
#include "absl/strings/string_view.h"
#include "absl/types/span.h"
#include "brotli/brotli_arena.h"
#include "brotli/encode.h"

namespace brotli {
Expand All @@ -20,19 +22,25 @@ typedef std::unique_ptr<BrotliEncoderPreparedDictionary,
/* A collection of utilities that ease using the existing brotli encoder API. */
class SharedBrotliEncoder {
public:
static DictionaryPointer CreateDictionary(absl::Span<const uint8_t> data) {
static DictionaryPointer CreateDictionary(absl::Span<const uint8_t> data,
BrotliArena* arena = nullptr) {
brotli_alloc_func alloc_func = arena ? &BrotliArena::Alloc : nullptr;
brotli_free_func free_func = arena ? &BrotliArena::Free : nullptr;
return DictionaryPointer(
BrotliEncoderPrepareDictionary(
BROTLI_SHARED_DICTIONARY_RAW, data.size(), data.data(),
BROTLI_MAX_QUALITY, nullptr, nullptr, nullptr),
BROTLI_MAX_QUALITY, alloc_func, free_func, arena),
&BrotliEncoderDestroyPreparedDictionary);
}

static EncoderStatePointer CreateEncoder(
unsigned quality, size_t font_size, unsigned stream_offset,
const BrotliEncoderPreparedDictionary* dictionary) {
const BrotliEncoderPreparedDictionary* dictionary,
BrotliArena* arena = nullptr) {
brotli_alloc_func alloc_func = arena ? &BrotliArena::Alloc : nullptr;
brotli_free_func free_func = arena ? &BrotliArena::Free : nullptr;
EncoderStatePointer state = EncoderStatePointer(
BrotliEncoderCreateInstance(nullptr, nullptr, nullptr),
BrotliEncoderCreateInstance(alloc_func, free_func, arena),
&BrotliEncoderDestroyInstance);

if (!BrotliEncoderSetParameter(state.get(), BROTLI_PARAM_QUALITY,
Expand Down
2 changes: 2 additions & 0 deletions ift/common/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -174,10 +174,12 @@ cc_test(
":common",
":mocks",
":test_font_loader",
"//brotli:shared_brotli_encoder",
"@abseil-cpp//absl/container:btree",
"@abseil-cpp//absl/container:flat_hash_map",
"@abseil-cpp//absl/container:flat_hash_set",
"@abseil-cpp//absl/hash:hash_testing",
"@abseil-cpp//absl/log",
"@abseil-cpp//absl/status",
"@abseil-cpp//absl/strings",
"@abseil-cpp//absl/types:span",
Expand Down
30 changes: 28 additions & 2 deletions ift/common/brotli_binary_diff.cc
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "ift/common/brotli_binary_diff.h"

#include <chrono>
#include <vector>

#include "absl/log/log.h"
Expand Down Expand Up @@ -39,11 +40,25 @@ Status BrotliBinaryDiff::Diff(const FontData& font_base,
Status BrotliBinaryDiff::Diff(const FontData& font_base, string_view data,
unsigned stream_offset, bool is_last,
std::vector<uint8_t>& sink) const {

total_calls_++;
total_input_bytes_ += data.size();
total_dict_bytes_ += font_base.size();
auto start = std::chrono::high_resolution_clock::now();

// Reset the arena on scope exit, ensuring any allocated state and dictionary
// are cleaned up first before offset is reset.
struct ArenaResetGuard {
brotli::BrotliArena& arena;
~ArenaResetGuard() { arena.Reset(); }
} reset_guard{arena_};

// There's a decent amount of overhead in creating a dictionary, even if it's
// completely empty. So don't set a dictionary unless it's non-empty.
DictionaryPointer dictionary(nullptr, nullptr);
if (font_base.size() > 0) {
dictionary = SharedBrotliEncoder::CreateDictionary(font_base.span());
dictionary =
SharedBrotliEncoder::CreateDictionary(font_base.span(), &arena_);
if (!dictionary) {
return absl::InternalError("Failed to create the shared dictionary.");
}
Expand All @@ -52,7 +67,7 @@ Status BrotliBinaryDiff::Diff(const FontData& font_base, string_view data,
// Don't give the encoder an estimated size if this is not all the data.
unsigned data_size = !stream_offset && is_last ? data.size() : 0;
EncoderStatePointer state = SharedBrotliEncoder::CreateEncoder(
quality_, data_size, stream_offset, dictionary.get());
quality_, data_size, stream_offset, dictionary.get(), &arena_);
if (!state) {
return absl::InternalError("Failed to create the encoder.");
}
Expand All @@ -63,6 +78,17 @@ Status BrotliBinaryDiff::Diff(const FontData& font_base, string_view data,
return absl::InternalError("Failed to encode brotli binary patch.");
}

auto end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double, std::milli> elapsed = end - start;
total_time_ += elapsed.count();

if (total_calls_ % 1000 == 0) {
VLOG(1) << "BrotliBinaryDiff(q" << quality_ << "): " << total_calls_ << " ops, "
<< (total_input_bytes_ / total_calls_) << " bytes/call, "
<< (total_dict_bytes_ / total_calls_) << " dict bytes/call, "
<< (total_time_ / (double) total_calls_) << " ms/call.";
}

return absl::OkStatus();
}

Expand Down
10 changes: 9 additions & 1 deletion ift/common/brotli_binary_diff.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include <vector>

#include "absl/status/status.h"
#include "brotli/brotli_arena.h"
#include "ift/common/binary_diff.h"
#include "ift/common/font_data.h"

Expand All @@ -24,8 +25,15 @@ class BrotliBinaryDiff : public BinaryDiff {
unsigned stream_offset, bool is_last,
std::vector<uint8_t>& sink) const;

const brotli::BrotliArena& arena() const { return arena_; }

private:
unsigned quality_;
const unsigned quality_;
mutable brotli::BrotliArena arena_;
mutable uint64_t total_input_bytes_ = 0;
mutable uint64_t total_dict_bytes_ = 0;
mutable uint64_t total_calls_ = 0;
mutable double total_time_ = 0;
};

} // namespace ift::common
Expand Down
87 changes: 87 additions & 0 deletions ift/common/brotli_patching_test.cc
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "absl/status/status.h"
#include "absl/types/span.h"
#include "brotli/brotli_arena.h"
#include "gtest/gtest.h"
#include "ift/common/brotli_binary_diff.h"
#include "ift/common/brotli_binary_patch.h"
Expand Down Expand Up @@ -124,4 +125,90 @@ TEST_F(BrotliPatchingTest, DiffAndPatchWithSmallerDerived) {
EXPECT_EQ(Span<const char>(patched), Span<const char>(subset_a_));
}

TEST(BrotliArenaTest, BumpAllocationAndAlignment) {
brotli::BrotliArena arena(1024);
EXPECT_EQ(arena.offset(), 0);
EXPECT_EQ(arena.capacity(), 1024);

constexpr size_t kAlign = brotli::BrotliArena::kAlignment;

void* p1 = brotli::BrotliArena::Alloc(&arena, 10);
ASSERT_NE(p1, nullptr);
EXPECT_EQ(reinterpret_cast<std::uintptr_t>(p1) % kAlign, 0);
// Header + aligned payload
size_t expected_p1_offset = kAlign + ((10 + kAlign - 1) & ~(kAlign - 1));
EXPECT_EQ(arena.offset(), expected_p1_offset);

void* p2 = brotli::BrotliArena::Alloc(&arena, 24);
ASSERT_NE(p2, nullptr);
EXPECT_EQ(reinterpret_cast<std::uintptr_t>(p2) % kAlign, 0);
size_t expected_p2_offset =
expected_p1_offset + kAlign + ((24 + kAlign - 1) & ~(kAlign - 1));
EXPECT_EQ(arena.offset(), expected_p2_offset);

// LIFO free p2 then p1
brotli::BrotliArena::Free(&arena, p2);
EXPECT_EQ(arena.offset(), expected_p1_offset);

brotli::BrotliArena::Free(&arena, p1);
EXPECT_EQ(arena.offset(), 0);
}

TEST(BrotliArenaTest, ResetAndReallocation) {
brotli::BrotliArena arena(1024);
void* p1 = brotli::BrotliArena::Alloc(&arena, 64);
EXPECT_GT(arena.offset(), 0);

arena.Reset();
EXPECT_EQ(arena.offset(), 0);

void* p2 = brotli::BrotliArena::Alloc(&arena, 64);
EXPECT_EQ(p1, p2); // Reuses the exact same backing memory address
EXPECT_GT(arena.offset(), 0);

arena.Reset();
EXPECT_EQ(arena.offset(), 0);
}

TEST(BrotliArenaTest, OversizedAllocationFallsBackToMalloc) {
brotli::BrotliArena arena(256);
// Allocate more than capacity
void* p = brotli::BrotliArena::Alloc(&arena, 1024);
ASSERT_NE(p, nullptr);
EXPECT_EQ(arena.offset(), 0); // Arena was bypassed

// Freeing fallback allocation does not corrupt arena
brotli::BrotliArena::Free(&arena, p);
EXPECT_EQ(arena.offset(), 0);
}

TEST_F(BrotliPatchingTest, DiffReusesArenaAcrossMultipleCalls) {
EXPECT_EQ(diff_->arena().offset(), 0);

FontData patch1;
EXPECT_EQ(diff_->Diff(subset_a_, subset_b_, &patch1), absl::OkStatus());
EXPECT_EQ(diff_->arena().offset(), 0); // Cleaned up after call

FontData patch2;
EXPECT_EQ(diff_->Diff(subset_a_, subset_b_, &patch2), absl::OkStatus());
EXPECT_EQ(diff_->arena().offset(), 0); // Cleaned up after call

// Output must be bit-for-bit identical across calls
EXPECT_EQ(Span<const char>(patch1), Span<const char>(patch2));

// Also test with quality 11 (uses H10 binary tree hasher)
BrotliBinaryDiff diff11(11);
EXPECT_EQ(diff11.arena().offset(), 0);

FontData patch11_1;
EXPECT_EQ(diff11.Diff(subset_a_, subset_b_, &patch11_1), absl::OkStatus());
EXPECT_EQ(diff11.arena().offset(), 0);

FontData patch11_2;
EXPECT_EQ(diff11.Diff(subset_a_, subset_b_, &patch11_2), absl::OkStatus());
EXPECT_EQ(diff11.arena().offset(), 0);

EXPECT_EQ(Span<const char>(patch11_1), Span<const char>(patch11_2));
}

} // namespace ift::common
1 change: 0 additions & 1 deletion ift/encoder/candidate_merge.cc
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ using absl::flat_hash_map;
using absl::flat_hash_set;
using absl::Status;
using absl::StatusOr;
using ift::GlyphKeyedDiff;
using ift::common::CodepointSet;
using ift::common::CompatId;
using ift::common::FontData;
Expand Down
1 change: 0 additions & 1 deletion ift/encoder/closure_glyph_segmenter.cc
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ using absl::Span;
using absl::Status;
using absl::StatusOr;
using absl::StrCat;
using ift::GlyphKeyedDiff;
using ift::common::CodepointSet;
using ift::common::CompatId;
using ift::common::FontData;
Expand Down
Loading
Loading