diff --git a/ift/encoder/candidate_merge_test.cc b/ift/encoder/candidate_merge_test.cc index 5e5baa10..40b1f8bb 100644 --- a/ift/encoder/candidate_merge_test.cc +++ b/ift/encoder/candidate_merge_test.cc @@ -1366,4 +1366,266 @@ TEST_F(CandidateMergeTest, AssessMerge_CostDeltas_UnicodeRangeIgnoresNonExclusiv EXPECT_NEAR((*r_f_i)->CostDelta(), expected_delta_f_i, 1e-9); } +namespace { + +class HighDefaultPatchSizeCache : public PatchSizeCache { + public: + explicit HighDefaultPatchSizeCache(uint32_t default_size) + : default_size_(default_size) {} + + absl::StatusOr GetPatchSize( + const ift::common::GlyphSet& gids) override { + auto it = patch_sizes_.find(gids); + if (it != patch_sizes_.end()) { + return it->second; + } + return default_size_; + } + + void LogBrotliCallCount() const override {} + + void SetPatchSize(const ift::common::GlyphSet& gids, uint32_t size) { + patch_sizes_[gids] = size; + } + + private: + uint32_t default_size_; + absl::flat_hash_map patch_sizes_; +}; + +} // namespace + +TEST_F(CandidateMergeTest, + TryNextPatchMerge_ExcludesGlyphsThatBecameExclusiveDuringSegmentMerging) { + + // s0 (aacute) and s1 (eacute) share acute accent glyphs via (s0 OR s1). + // s2 ('f'), s3 ('i'), s4 ('l') form composite ligature patches (s2 AND s3) + // and (s2 AND s4). + std::vector segments = { + {{0x00E1}}, // aacute + {{0x00E9}}, // eacute + {{'f'}}, + {{'i'}}, + {{'l'}}, + }; + + freq::UnicodeFrequencies frequencies{ + {{' ', ' '}, 100}, + {{0x00E1, 0x00E1}, 90}, + {{0x00E9, 0x00E9}, 90}, + {{'f', 'f'}, 40}, + {{'i', 'i'}, 40}, + {{'l', 'l'}, 40}, + }; + + ClosureGlyphSegmenter segmenter(8, 8, PATCH, CLOSURE_ONLY, resolver); + auto context = SegmentationContext::InitializeSegmentationContext( + roboto.get(), {}, segments, segmenter.unmapped_glyph_handling(), + segmenter.condition_analysis_mode(), segmenter.brotli_quality(), + segmenter.init_font_merging_brotli_quality(), resolver); + ASSERT_TRUE(context.ok()) << context.status(); + + GlyphSet g0 = context->glyph_groupings.ExclusiveGlyphs(0); + GlyphSet g1 = context->glyph_groupings.ExclusiveGlyphs(1); + ASSERT_FALSE(g0.empty()); + ASSERT_FALSE(g1.empty()); + ASSERT_TRUE(context->glyph_groupings.UnmappedGlyphs().empty()); + + ASSERT_TRUE(context->glyph_groupings.ConditionsAndGlyphs().contains( + ActivationCondition::or_segments({0, 1}, 0))); + GlyphSet g_acute = context->glyph_groupings.ConditionsAndGlyphs().at( + ActivationCondition::or_segments({0, 1}, 0)); + ASSERT_FALSE(g_acute.empty()); + + ASSERT_TRUE(context->glyph_groupings.ConditionsAndGlyphs().contains( + ActivationCondition::and_segments({2, 3}, 0))); + GlyphSet g_fi = context->glyph_groupings.ConditionsAndGlyphs().at( + ActivationCondition::and_segments({2, 3}, 0)); + ASSERT_FALSE(g_fi.empty()); + + ASSERT_TRUE(context->glyph_groupings.ConditionsAndGlyphs().contains( + ActivationCondition::and_segments({2, 4}, 0))); + GlyphSet g_fl = context->glyph_groupings.ConditionsAndGlyphs().at( + ActivationCondition::and_segments({2, 4}, 0)); + ASSERT_FALSE(g_fl.empty()); + + auto* size_cache = new HighDefaultPatchSizeCache(/*default_size=*/5000); + size_cache->SetPatchSize({}, 10); + for (const auto& [cond, glyphs] : + context->glyph_groupings.ConditionsAndGlyphs()) { + size_cache->SetPatchSize(glyphs, 30); + } + + GlyphSet g01 = g0; + g01.union_set(g1); + g01.union_set(g_acute); + size_cache->SetPatchSize(g01, 40); + + GlyphSet g_fi_fl = g_fi; + g_fi_fl.union_set(g_fl); + size_cache->SetPatchSize(g_fi_fl, 30); + + // Make merging s0's post-merge exclusive patch with the composite patches + // strongly cost-reducing if s0 were ever leaked into TryNextPatchMerge(). + GlyphSet g01_fi = g01; + g01_fi.union_set(g_fi); + size_cache->SetPatchSize(g01_fi, 40); + GlyphSet g01_fl = g01; + g01_fl.union_set(g_fl); + size_cache->SetPatchSize(g01_fl, 40); + GlyphSet g01_fi_fl = g01; + g01_fi_fl.union_set(g_fi_fl); + size_cache->SetPatchSize(g01_fi_fl, 40); + + context->patch_size_cache.reset(size_cache); + + MergeStrategy strategy = MergeStrategy::CostBased( + *ProbabilityProfile::Unigram(std::move(frequencies)), + /*network_overhead_cost=*/75, + /*min_group_size=*/1); + strategy.SetUsePatchMerges(true); + strategy.SetOptimizationCutoffFraction(0.0); + + Merger merger = *Merger::New(*context, strategy, all, all); + + // Step 1: Merges segments {0, 1} into segment 0. + auto step1 = merger.TryNextMerge(); + ASSERT_TRUE(step1.ok()) << step1.status(); + ASSERT_TRUE(step1->has_value()); + EXPECT_EQ((*step1)->base_segment, 0u); + EXPECT_EQ((*step1)->segments, (SegmentSet{0, 1})); + ASSERT_TRUE(context->ReprocessChanged(**step1).ok()); + + // g_acute has now collapsed into the exclusive patch for segment 0. + EXPECT_EQ(context->glyph_groupings.ExclusiveGlyphs(0), g01); + + // Step 2 & 3: Patch merging combines composite ligature patches. + auto step2 = merger.TryNextMerge(); + ASSERT_TRUE(step2.ok()) << step2.status(); + ASSERT_TRUE(step2->has_value()); + ASSERT_TRUE(context->ReprocessChanged(**step2).ok()); + + auto step3 = merger.TryNextMerge(); + ASSERT_TRUE(step3.ok()) << step3.status(); + ASSERT_TRUE(step3->has_value()); + ASSERT_TRUE(context->ReprocessChanged(**step3).ok()); + + // Step 4: No further merges remain. The exclusive patch for segment 0 must + // NOT be merged into any composite patch. + auto step4 = merger.TryNextMerge(); + ASSERT_TRUE(step4.ok()) << step4.status(); + EXPECT_FALSE(step4->has_value()); + + EXPECT_EQ(context->glyph_groupings.ExclusiveGlyphs(0), g01); + EXPECT_EQ(context->glyph_groupings.ConditionsAndGlyphs().at( + ActivationCondition::exclusive_segment(0, 0)), + g01); +} + +TEST_F(CandidateMergeTest, + TryNextPatchMerge_ExcludesConditionSimplifiedToExclusiveDuringPatchMerging) { + std::vector segments = { + {{'f', 'b'}}, + {{'i', 'd'}}, + {{'l', 'g'}}, + {{0x00F2, 'h'}}, // ograve, 'h' + {{0x00F9, 'm'}}, // ugrave, 'm' + }; + + freq::UnicodeFrequencies frequencies{ + {{' ', ' '}, 100}, + {{'f', 'f'}, 90}, + {{'i', 'i'}, 80}, + {{'l', 'l'}, 80}, + {{0x00F2, 0x00F2}, 40}, + {{0x00F9, 0x00F9}, 40}, + }; + + ClosureGlyphSegmenter segmenter( + 8, 8, PATCH, DEP_GRAPH_ONLY_WITH_SIMPLIFICATION, resolver); + auto context = SegmentationContext::InitializeSegmentationContext( + roboto.get(), {}, segments, segmenter.unmapped_glyph_handling(), + segmenter.condition_analysis_mode(), segmenter.brotli_quality(), + segmenter.init_font_merging_brotli_quality(), resolver); + ASSERT_TRUE(context.ok()) << context.status(); + + GlyphSet g0 = context->glyph_groupings.ExclusiveGlyphs(0); + ASSERT_FALSE(g0.empty()); + + ASSERT_TRUE(context->glyph_groupings.ConditionsAndGlyphs().contains( + ActivationCondition::and_segments({0, 1}, 0))); + GlyphSet g_fi = context->glyph_groupings.ConditionsAndGlyphs().at( + ActivationCondition::and_segments({0, 1}, 0)); + ASSERT_FALSE(g_fi.empty()); + + ASSERT_TRUE(context->glyph_groupings.ConditionsAndGlyphs().contains( + ActivationCondition::and_segments({0, 2}, 0))); + GlyphSet g_fl = context->glyph_groupings.ConditionsAndGlyphs().at( + ActivationCondition::and_segments({0, 2}, 0)); + ASSERT_FALSE(g_fl.empty()); + + ASSERT_TRUE(context->glyph_groupings.ConditionsAndGlyphs().contains( + ActivationCondition::or_segments({3, 4}, 0))); + GlyphSet g_grave = context->glyph_groupings.ConditionsAndGlyphs().at( + ActivationCondition::or_segments({3, 4}, 0)); + ASSERT_FALSE(g_grave.empty()); + + auto* size_cache = new HighDefaultPatchSizeCache(/*default_size=*/5000); + size_cache->SetPatchSize({}, 10); + for (const auto& [cond, glyphs] : + context->glyph_groupings.ConditionsAndGlyphs()) { + size_cache->SetPatchSize(glyphs, 30); + } + + GlyphSet g_fi_fl = g_fi; + g_fi_fl.union_set(g_fl); + size_cache->SetPatchSize(g_fi_fl, 40); + + GlyphSet g0_with_fi_fl = g0; + g0_with_fi_fl.union_set(g_fi_fl); + size_cache->SetPatchSize(g0_with_fi_fl, 40); + + // Make merging the simplified exclusive condition if (s0) with (s3 OR s4) + // strongly cost-reducing if if (s0) were not excluded on subsequent calls. + GlyphSet g0_with_fi_fl_and_grave = g0_with_fi_fl; + g0_with_fi_fl_and_grave.union_set(g_grave); + size_cache->SetPatchSize(g0_with_fi_fl_and_grave, 40); + + context->patch_size_cache.reset(size_cache); + + MergeStrategy strategy = MergeStrategy::CostBased( + *ProbabilityProfile::Unigram(std::move(frequencies)), + /*network_overhead_cost=*/75, + /*min_group_size=*/1); + strategy.SetUsePatchMerges(true); + strategy.SetOptimizationCutoffFraction(0.0); + + Merger merger = *Merger::New(*context, strategy, all, all); + + // Step 1: Merges (s0 AND s1) with (s0 AND s2). Under + // DEP_GRAPH_ONLY_WITH_SIMPLIFICATION, NonCompositeSuperset() simplifies + // s0 AND (s1 OR s2) to exclusive condition if (s0) then p0. + auto step1 = merger.TryNextMerge(); + ASSERT_TRUE(step1.ok()) << step1.status(); + ASSERT_TRUE(step1->has_value()); + ASSERT_TRUE(context->ReprocessChanged(**step1).ok()); + + EXPECT_EQ(context->glyph_groupings.ConditionsAndGlyphs().at( + ActivationCondition::exclusive_segment(0, 0)), + g0_with_fi_fl); + + // Step 2: The simplified condition if (s0) is exclusive and must NOT be + // merged with if ((s3 OR s4)). No further patch merges remain. + auto step2 = merger.TryNextMerge(); + ASSERT_TRUE(step2.ok()) << step2.status(); + EXPECT_FALSE(step2->has_value()); + + EXPECT_EQ(context->glyph_groupings.ConditionsAndGlyphs().at( + ActivationCondition::exclusive_segment(0, 0)), + g0_with_fi_fl); + EXPECT_EQ(context->glyph_groupings.ConditionsAndGlyphs().at( + ActivationCondition::or_segments({3, 4}, 0)), + g_grave); +} + } // namespace ift::encoder diff --git a/ift/encoder/merger.cc b/ift/encoder/merger.cc index 19df4c20..f4fd96c1 100644 --- a/ift/encoder/merger.cc +++ b/ift/encoder/merger.cc @@ -108,12 +108,29 @@ StatusOr> Merger::TryNextPatchMerge() { return std::nullopt; } + if (!candidate_patch_merge_glyphs_.has_value()) { + candidate_patch_merge_glyphs_ = ComputeCandidatePatchMergeGlyphs( + *context_, strategy_, inscope_segments_); + } + if (candidate_patch_merge_glyphs_->empty()) { + return std::nullopt; + } + // Work through the candidate composite conditions in order of descending // probability. std::vector> ordered_candidates; - for (const auto& [condition, glyphs] : - Context().glyph_groupings.ConditionsAndGlyphs()) { - if (!glyphs.intersects(candidate_patch_merge_glyphs_)) { + ActivationCondition last_exclusive = + ActivationCondition::exclusive_segment(UINT32_MAX, 0); + for (auto it = Context().glyph_groupings.OrderedConditions().lower_bound( + last_exclusive); + it != Context().glyph_groupings.OrderedConditions().end(); ++it) { + const auto& condition = *it; + if (condition.IsFallback() || condition.IsExclusive()) { + continue; + } + const GlyphSet& glyphs = + Context().glyph_groupings.ConditionsAndGlyphs().at(condition); + if (!glyphs.intersects(*candidate_patch_merge_glyphs_)) { continue; } ordered_candidates.emplace_back(condition, @@ -136,7 +153,7 @@ StatusOr> Merger::TryNextPatchMerge() { if (!invalidation.has_value()) { // No merges left for this base, remove it from the candidate list. ordered_candidates.erase(ordered_candidates.begin()); - candidate_patch_merge_glyphs_.subtract(base_glyphs); + candidate_patch_merge_glyphs_->subtract(base_glyphs); } else { return invalidation; } @@ -325,6 +342,7 @@ Status Merger::ResetSegmentProbabilities() const { Status Merger::ReassignInitSubset() { candidate_segments_ = ComputeCandidateSegments(Context(), strategy_, inscope_segments_); + candidate_patch_merge_glyphs_ = std::nullopt; TRYV(ResetSegmentProbabilities()); TRYV(InitOptimizationCutoff()); return absl::OkStatus(); @@ -746,9 +764,10 @@ Status Merger::CollectCompositeCandidateMerges( GlyphSet Merger::ComputeCandidatePatchMergeGlyphs( SegmentationContext& context, const MergeStrategy& strategy, - const SegmentSet& candidate_segments, const SegmentSet& inscope_segments, - segment_index_t optimization_cutoff_segment) { + const SegmentSet& inscope_segments) { GlyphSet out; + SegmentSet active_inscope_segments = + ComputeCandidateSegments(context, strategy, inscope_segments); ActivationCondition last_exclusive = ActivationCondition::exclusive_segment(UINT32_MAX, 0); @@ -763,7 +782,7 @@ GlyphSet Merger::ComputeCandidatePatchMergeGlyphs( SegmentSet triggering_segments = condition.TriggeringSegments(); - if (!triggering_segments.intersects(candidate_segments) || + if (!triggering_segments.intersects(active_inscope_segments) || !triggering_segments.is_subset_of(inscope_segments)) { // all triggering segments must be inscope otherwise this merge crosses // merge group boundaries. diff --git a/ift/encoder/merger.h b/ift/encoder/merger.h index 30bf0432..1c5b9e17 100644 --- a/ift/encoder/merger.h +++ b/ift/encoder/merger.h @@ -148,9 +148,6 @@ class Merger { inscope_segments_(inscope_segments), candidate_segments_( ComputeCandidateSegments(*context_, strategy_, inscope_segments_)), - candidate_patch_merge_glyphs_(ComputeCandidatePatchMergeGlyphs( - *context_, strategy_, candidate_segments_, inscope_segments, - optimization_cutoff_segment)), inscope_segments_for_init_move_(inscope_segments_for_init_move), optimization_cutoff_segment_(optimization_cutoff_segment) {} @@ -160,9 +157,7 @@ class Merger { static ift::common::GlyphSet ComputeCandidatePatchMergeGlyphs( SegmentationContext& context, const MergeStrategy& strategy, - const ift::common::SegmentSet& candidate_segments, - const ift::common::SegmentSet& inscope_segments, - segment_index_t optimization_cutoff_segment); + const ift::common::SegmentSet& inscope_segments); absl::StatusOr> TryNextPatchMerge(); @@ -264,7 +259,7 @@ class Merger { // The current set of segments under consideration for being merged. const ift::common::SegmentSet inscope_segments_; ift::common::SegmentSet candidate_segments_; - ift::common::GlyphSet candidate_patch_merge_glyphs_; + std::optional candidate_patch_merge_glyphs_; // This is the set of segments under consideration for being merged into the // init font. Typically contains segments that were removed from