Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
262 changes: 262 additions & 0 deletions ift/encoder/candidate_merge_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1366,4 +1366,266 @@ TEST_F(CandidateMergeTest, AssessMerge_CostDeltas_UnicodeRangeIgnoresNonExclusiv
EXPECT_NEAR((*r_f_i)->CostDelta(), expected_delta_f_i, 1e-9);
}

namespace {

class HighDefaultPatchSizeCache : public PatchSizeCache {
public:
explicit HighDefaultPatchSizeCache(uint32_t default_size)
: default_size_(default_size) {}

absl::StatusOr<uint32_t> GetPatchSize(
const ift::common::GlyphSet& gids) override {
auto it = patch_sizes_.find(gids);
if (it != patch_sizes_.end()) {
return it->second;
}
return default_size_;
}

void LogBrotliCallCount() const override {}

void SetPatchSize(const ift::common::GlyphSet& gids, uint32_t size) {
patch_sizes_[gids] = size;
}

private:
uint32_t default_size_;
absl::flat_hash_map<ift::common::GlyphSet, uint32_t> patch_sizes_;
};

} // namespace

TEST_F(CandidateMergeTest,
TryNextPatchMerge_ExcludesGlyphsThatBecameExclusiveDuringSegmentMerging) {

// s0 (aacute) and s1 (eacute) share acute accent glyphs via (s0 OR s1).
// s2 ('f'), s3 ('i'), s4 ('l') form composite ligature patches (s2 AND s3)
// and (s2 AND s4).
std::vector<Segment> segments = {
{{0x00E1}}, // aacute
{{0x00E9}}, // eacute
{{'f'}},
{{'i'}},
{{'l'}},
};

freq::UnicodeFrequencies frequencies{
{{' ', ' '}, 100},
{{0x00E1, 0x00E1}, 90},
{{0x00E9, 0x00E9}, 90},
{{'f', 'f'}, 40},
{{'i', 'i'}, 40},
{{'l', 'l'}, 40},
};

ClosureGlyphSegmenter segmenter(8, 8, PATCH, CLOSURE_ONLY, resolver);
auto context = SegmentationContext::InitializeSegmentationContext(
roboto.get(), {}, segments, segmenter.unmapped_glyph_handling(),
segmenter.condition_analysis_mode(), segmenter.brotli_quality(),
segmenter.init_font_merging_brotli_quality(), resolver);
ASSERT_TRUE(context.ok()) << context.status();

GlyphSet g0 = context->glyph_groupings.ExclusiveGlyphs(0);
GlyphSet g1 = context->glyph_groupings.ExclusiveGlyphs(1);
ASSERT_FALSE(g0.empty());
ASSERT_FALSE(g1.empty());
ASSERT_TRUE(context->glyph_groupings.UnmappedGlyphs().empty());

ASSERT_TRUE(context->glyph_groupings.ConditionsAndGlyphs().contains(
ActivationCondition::or_segments({0, 1}, 0)));
GlyphSet g_acute = context->glyph_groupings.ConditionsAndGlyphs().at(
ActivationCondition::or_segments({0, 1}, 0));
ASSERT_FALSE(g_acute.empty());

ASSERT_TRUE(context->glyph_groupings.ConditionsAndGlyphs().contains(
ActivationCondition::and_segments({2, 3}, 0)));
GlyphSet g_fi = context->glyph_groupings.ConditionsAndGlyphs().at(
ActivationCondition::and_segments({2, 3}, 0));
ASSERT_FALSE(g_fi.empty());

ASSERT_TRUE(context->glyph_groupings.ConditionsAndGlyphs().contains(
ActivationCondition::and_segments({2, 4}, 0)));
GlyphSet g_fl = context->glyph_groupings.ConditionsAndGlyphs().at(
ActivationCondition::and_segments({2, 4}, 0));
ASSERT_FALSE(g_fl.empty());

auto* size_cache = new HighDefaultPatchSizeCache(/*default_size=*/5000);
size_cache->SetPatchSize({}, 10);
for (const auto& [cond, glyphs] :
context->glyph_groupings.ConditionsAndGlyphs()) {
size_cache->SetPatchSize(glyphs, 30);
}

GlyphSet g01 = g0;
g01.union_set(g1);
g01.union_set(g_acute);
size_cache->SetPatchSize(g01, 40);

GlyphSet g_fi_fl = g_fi;
g_fi_fl.union_set(g_fl);
size_cache->SetPatchSize(g_fi_fl, 30);

// Make merging s0's post-merge exclusive patch with the composite patches
// strongly cost-reducing if s0 were ever leaked into TryNextPatchMerge().
GlyphSet g01_fi = g01;
g01_fi.union_set(g_fi);
size_cache->SetPatchSize(g01_fi, 40);
GlyphSet g01_fl = g01;
g01_fl.union_set(g_fl);
size_cache->SetPatchSize(g01_fl, 40);
GlyphSet g01_fi_fl = g01;
g01_fi_fl.union_set(g_fi_fl);
size_cache->SetPatchSize(g01_fi_fl, 40);

context->patch_size_cache.reset(size_cache);

MergeStrategy strategy = MergeStrategy::CostBased(
*ProbabilityProfile::Unigram(std::move(frequencies)),
/*network_overhead_cost=*/75,
/*min_group_size=*/1);
strategy.SetUsePatchMerges(true);
strategy.SetOptimizationCutoffFraction(0.0);

Merger merger = *Merger::New(*context, strategy, all, all);

// Step 1: Merges segments {0, 1} into segment 0.
auto step1 = merger.TryNextMerge();
ASSERT_TRUE(step1.ok()) << step1.status();
ASSERT_TRUE(step1->has_value());
EXPECT_EQ((*step1)->base_segment, 0u);
EXPECT_EQ((*step1)->segments, (SegmentSet{0, 1}));
ASSERT_TRUE(context->ReprocessChanged(**step1).ok());

// g_acute has now collapsed into the exclusive patch for segment 0.
EXPECT_EQ(context->glyph_groupings.ExclusiveGlyphs(0), g01);

// Step 2 & 3: Patch merging combines composite ligature patches.
auto step2 = merger.TryNextMerge();
ASSERT_TRUE(step2.ok()) << step2.status();
ASSERT_TRUE(step2->has_value());
ASSERT_TRUE(context->ReprocessChanged(**step2).ok());

auto step3 = merger.TryNextMerge();
ASSERT_TRUE(step3.ok()) << step3.status();
ASSERT_TRUE(step3->has_value());
ASSERT_TRUE(context->ReprocessChanged(**step3).ok());

// Step 4: No further merges remain. The exclusive patch for segment 0 must
// NOT be merged into any composite patch.
auto step4 = merger.TryNextMerge();
ASSERT_TRUE(step4.ok()) << step4.status();
EXPECT_FALSE(step4->has_value());

EXPECT_EQ(context->glyph_groupings.ExclusiveGlyphs(0), g01);
EXPECT_EQ(context->glyph_groupings.ConditionsAndGlyphs().at(
ActivationCondition::exclusive_segment(0, 0)),
g01);
}

TEST_F(CandidateMergeTest,
TryNextPatchMerge_ExcludesConditionSimplifiedToExclusiveDuringPatchMerging) {
std::vector<Segment> segments = {
{{'f', 'b'}},
{{'i', 'd'}},
{{'l', 'g'}},
{{0x00F2, 'h'}}, // ograve, 'h'
{{0x00F9, 'm'}}, // ugrave, 'm'
};

freq::UnicodeFrequencies frequencies{
{{' ', ' '}, 100},
{{'f', 'f'}, 90},
{{'i', 'i'}, 80},
{{'l', 'l'}, 80},
{{0x00F2, 0x00F2}, 40},
{{0x00F9, 0x00F9}, 40},
};

ClosureGlyphSegmenter segmenter(
8, 8, PATCH, DEP_GRAPH_ONLY_WITH_SIMPLIFICATION, resolver);
auto context = SegmentationContext::InitializeSegmentationContext(
roboto.get(), {}, segments, segmenter.unmapped_glyph_handling(),
segmenter.condition_analysis_mode(), segmenter.brotli_quality(),
segmenter.init_font_merging_brotli_quality(), resolver);
ASSERT_TRUE(context.ok()) << context.status();

GlyphSet g0 = context->glyph_groupings.ExclusiveGlyphs(0);
ASSERT_FALSE(g0.empty());

ASSERT_TRUE(context->glyph_groupings.ConditionsAndGlyphs().contains(
ActivationCondition::and_segments({0, 1}, 0)));
GlyphSet g_fi = context->glyph_groupings.ConditionsAndGlyphs().at(
ActivationCondition::and_segments({0, 1}, 0));
ASSERT_FALSE(g_fi.empty());

ASSERT_TRUE(context->glyph_groupings.ConditionsAndGlyphs().contains(
ActivationCondition::and_segments({0, 2}, 0)));
GlyphSet g_fl = context->glyph_groupings.ConditionsAndGlyphs().at(
ActivationCondition::and_segments({0, 2}, 0));
ASSERT_FALSE(g_fl.empty());

ASSERT_TRUE(context->glyph_groupings.ConditionsAndGlyphs().contains(
ActivationCondition::or_segments({3, 4}, 0)));
GlyphSet g_grave = context->glyph_groupings.ConditionsAndGlyphs().at(
ActivationCondition::or_segments({3, 4}, 0));
ASSERT_FALSE(g_grave.empty());

auto* size_cache = new HighDefaultPatchSizeCache(/*default_size=*/5000);
size_cache->SetPatchSize({}, 10);
for (const auto& [cond, glyphs] :
context->glyph_groupings.ConditionsAndGlyphs()) {
size_cache->SetPatchSize(glyphs, 30);
}

GlyphSet g_fi_fl = g_fi;
g_fi_fl.union_set(g_fl);
size_cache->SetPatchSize(g_fi_fl, 40);

GlyphSet g0_with_fi_fl = g0;
g0_with_fi_fl.union_set(g_fi_fl);
size_cache->SetPatchSize(g0_with_fi_fl, 40);

// Make merging the simplified exclusive condition if (s0) with (s3 OR s4)
// strongly cost-reducing if if (s0) were not excluded on subsequent calls.
GlyphSet g0_with_fi_fl_and_grave = g0_with_fi_fl;
g0_with_fi_fl_and_grave.union_set(g_grave);
size_cache->SetPatchSize(g0_with_fi_fl_and_grave, 40);

context->patch_size_cache.reset(size_cache);

MergeStrategy strategy = MergeStrategy::CostBased(
*ProbabilityProfile::Unigram(std::move(frequencies)),
/*network_overhead_cost=*/75,
/*min_group_size=*/1);
strategy.SetUsePatchMerges(true);
strategy.SetOptimizationCutoffFraction(0.0);

Merger merger = *Merger::New(*context, strategy, all, all);

// Step 1: Merges (s0 AND s1) with (s0 AND s2). Under
// DEP_GRAPH_ONLY_WITH_SIMPLIFICATION, NonCompositeSuperset() simplifies
// s0 AND (s1 OR s2) to exclusive condition if (s0) then p0.
auto step1 = merger.TryNextMerge();
ASSERT_TRUE(step1.ok()) << step1.status();
ASSERT_TRUE(step1->has_value());
ASSERT_TRUE(context->ReprocessChanged(**step1).ok());

EXPECT_EQ(context->glyph_groupings.ConditionsAndGlyphs().at(
ActivationCondition::exclusive_segment(0, 0)),
g0_with_fi_fl);

// Step 2: The simplified condition if (s0) is exclusive and must NOT be
// merged with if ((s3 OR s4)). No further patch merges remain.
auto step2 = merger.TryNextMerge();
ASSERT_TRUE(step2.ok()) << step2.status();
EXPECT_FALSE(step2->has_value());

EXPECT_EQ(context->glyph_groupings.ConditionsAndGlyphs().at(
ActivationCondition::exclusive_segment(0, 0)),
g0_with_fi_fl);
EXPECT_EQ(context->glyph_groupings.ConditionsAndGlyphs().at(
ActivationCondition::or_segments({3, 4}, 0)),
g_grave);
}

} // namespace ift::encoder
33 changes: 26 additions & 7 deletions ift/encoder/merger.cc
Original file line number Diff line number Diff line change
Expand Up @@ -108,12 +108,29 @@ StatusOr<std::optional<InvalidationSet>> Merger::TryNextPatchMerge() {
return std::nullopt;
}

if (!candidate_patch_merge_glyphs_.has_value()) {
candidate_patch_merge_glyphs_ = ComputeCandidatePatchMergeGlyphs(
*context_, strategy_, inscope_segments_);
}
if (candidate_patch_merge_glyphs_->empty()) {
return std::nullopt;
}

// Work through the candidate composite conditions in order of descending
// probability.
std::vector<std::pair<ActivationCondition, double>> ordered_candidates;
for (const auto& [condition, glyphs] :
Context().glyph_groupings.ConditionsAndGlyphs()) {
if (!glyphs.intersects(candidate_patch_merge_glyphs_)) {
ActivationCondition last_exclusive =
ActivationCondition::exclusive_segment(UINT32_MAX, 0);
for (auto it = Context().glyph_groupings.OrderedConditions().lower_bound(
last_exclusive);
it != Context().glyph_groupings.OrderedConditions().end(); ++it) {
const auto& condition = *it;
if (condition.IsFallback() || condition.IsExclusive()) {
continue;
}
const GlyphSet& glyphs =
Context().glyph_groupings.ConditionsAndGlyphs().at(condition);
if (!glyphs.intersects(*candidate_patch_merge_glyphs_)) {
continue;
}
ordered_candidates.emplace_back(condition,
Expand All @@ -136,7 +153,7 @@ StatusOr<std::optional<InvalidationSet>> Merger::TryNextPatchMerge() {
if (!invalidation.has_value()) {
// No merges left for this base, remove it from the candidate list.
ordered_candidates.erase(ordered_candidates.begin());
candidate_patch_merge_glyphs_.subtract(base_glyphs);
candidate_patch_merge_glyphs_->subtract(base_glyphs);
} else {
return invalidation;
}
Expand Down Expand Up @@ -325,6 +342,7 @@ Status Merger::ResetSegmentProbabilities() const {
Status Merger::ReassignInitSubset() {
candidate_segments_ =
ComputeCandidateSegments(Context(), strategy_, inscope_segments_);
candidate_patch_merge_glyphs_ = std::nullopt;
TRYV(ResetSegmentProbabilities());
TRYV(InitOptimizationCutoff());
return absl::OkStatus();
Expand Down Expand Up @@ -746,9 +764,10 @@ Status Merger::CollectCompositeCandidateMerges(

GlyphSet Merger::ComputeCandidatePatchMergeGlyphs(
SegmentationContext& context, const MergeStrategy& strategy,
const SegmentSet& candidate_segments, const SegmentSet& inscope_segments,
segment_index_t optimization_cutoff_segment) {
const SegmentSet& inscope_segments) {
GlyphSet out;
SegmentSet active_inscope_segments =
ComputeCandidateSegments(context, strategy, inscope_segments);

ActivationCondition last_exclusive =
ActivationCondition::exclusive_segment(UINT32_MAX, 0);
Expand All @@ -763,7 +782,7 @@ GlyphSet Merger::ComputeCandidatePatchMergeGlyphs(

SegmentSet triggering_segments = condition.TriggeringSegments();

if (!triggering_segments.intersects(candidate_segments) ||
if (!triggering_segments.intersects(active_inscope_segments) ||
!triggering_segments.is_subset_of(inscope_segments)) {
// all triggering segments must be inscope otherwise this merge crosses
// merge group boundaries.
Expand Down
9 changes: 2 additions & 7 deletions ift/encoder/merger.h
Original file line number Diff line number Diff line change
Expand Up @@ -148,9 +148,6 @@ class Merger {
inscope_segments_(inscope_segments),
candidate_segments_(
ComputeCandidateSegments(*context_, strategy_, inscope_segments_)),
candidate_patch_merge_glyphs_(ComputeCandidatePatchMergeGlyphs(
*context_, strategy_, candidate_segments_, inscope_segments,
optimization_cutoff_segment)),
inscope_segments_for_init_move_(inscope_segments_for_init_move),
optimization_cutoff_segment_(optimization_cutoff_segment) {}

Expand All @@ -160,9 +157,7 @@ class Merger {

static ift::common::GlyphSet ComputeCandidatePatchMergeGlyphs(
SegmentationContext& context, const MergeStrategy& strategy,
const ift::common::SegmentSet& candidate_segments,
const ift::common::SegmentSet& inscope_segments,
segment_index_t optimization_cutoff_segment);
const ift::common::SegmentSet& inscope_segments);

absl::StatusOr<std::optional<InvalidationSet>> TryNextPatchMerge();

Expand Down Expand Up @@ -264,7 +259,7 @@ class Merger {
// The current set of segments under consideration for being merged.
const ift::common::SegmentSet inscope_segments_;
ift::common::SegmentSet candidate_segments_;
ift::common::GlyphSet candidate_patch_merge_glyphs_;
std::optional<ift::common::GlyphSet> candidate_patch_merge_glyphs_;

// This is the set of segments under consideration for being merged into the
// init font. Typically contains segments that were removed from
Expand Down
Loading