From 19a19fef033487add032353dd97bd550c27f0c1b Mon Sep 17 00:00:00 2001 From: Garret Rieger Date: Mon, 2 Nov 2026 23:39:45 +0000 Subject: [PATCH] Add auto config flags for specifying default features and codepoints. Default features/codepoints are automatically add to the init font when generating a segmentation. While this could previously be accomplished by manually editting the segmenter config, these new flags make it easier to specify directly via command line options. --- docs/creating_ift_fonts.md | 16 ++- ift/config/auto_segmenter_config.cc | 60 +++++--- ift/config/auto_segmenter_config.h | 13 +- ift/config/auto_segmenter_config_test.cc | 135 ++++++++++++++++++ ift/config/segmenter_config_util.cc | 3 +- ift/config/segmenter_config_util_test.cc | 33 +++++ util/BUILD | 32 +++++ util/auto_config_flags.cc | 171 +++++++++++++++++++++++ util/auto_config_flags.h | 28 ++++ util/auto_config_flags_test.cc | 140 +++++++++++++++++++ util/font2ift.cc | 13 +- util/font2ur.cc | 14 +- util/gen_ift_segmentation_plan.cc | 16 +-- util/gen_ift_segmenter_config.cc | 23 ++- 14 files changed, 636 insertions(+), 61 deletions(-) create mode 100644 util/auto_config_flags_test.cc diff --git a/docs/creating_ift_fonts.md b/docs/creating_ift_fonts.md index 3cc30a14..af71ebb5 100644 --- a/docs/creating_ift_fonts.md +++ b/docs/creating_ift_fonts.md @@ -22,7 +22,7 @@ bazel run -c opt //util:font2ift -- \ This will analyze the input font, decide how to segment it, and then produce the final IFT encoded font and patches. -When utilizing auto config there are two optional flags which can be used to adjust the behaviour: +When utilizing auto config there are optional flags which can be used to adjust the behaviour: * `--auto_config_primary_script`: this tells the config generator which language/script the font is intended to be used with. It has two effects: first the codepoints of the primary script are eligible to be moved into the initial font. Second for scripts with large overlaps, such as CJK, primary script selects which @@ -35,6 +35,14 @@ When utilizing auto config there are two optional flags which can be used to adj values increase encoding times but typically result in a more efficient end IFT font (ie. less bytes transferred by clients using it). +* `--auto_config_additional_default_features`: Takes a comma-separated list of 4-character OpenType feature tags + (e.g. `smcp,dlig`) which are added to the initial font (`initial_segment.features`) in the produced segmenter + config so that those layout features are always available in the base font. + +* `--auto_config_default_codepoints`: Takes a comma-separated list of Unicode codepoints or codepoint ranges + (in hex with `0x` or `U+` prefix, or decimal, with `-` or `..` for ranges, e.g. `0x20,0x41-0x5A,U+FE00..U+FE0F`) + which are added to the initial font (`initial_segment.codepoints`) in the produced segmenter config. + Example command line with optional flags: ```bash @@ -84,8 +92,10 @@ There are two main options for generating a segmenter config: ``` This analyzes the input font and tries to pick appropriate config values automatically. As discussed in - the previous "Producing IFT Encoded Fonts" section there is a configurable quality level. If needed - the auto generated config can be hand tweaked after generation. + the previous "Producing IFT Encoded Fonts" section there is a configurable quality level. Additional flags + including `--primary_script`, `--additional_default_features`, and `--default_codepoints` can also be + provided to control the initial font segment and script prioritization. If needed the auto generated config + can be hand tweaked after generation. ### Step 2: Generating Segmentation Plan diff --git a/ift/config/auto_segmenter_config.cc b/ift/config/auto_segmenter_config.cc index 2d3ac1be..5ebd695b 100644 --- a/ift/config/auto_segmenter_config.cc +++ b/ift/config/auto_segmenter_config.cc @@ -351,7 +351,9 @@ static CodepointSet OverlapCodepoints( static Status GroupInteractingScripts( hb_face_t* face, const std::vector& scripts, const std::vector& script_codepoints, - const ift::common::DataFileResolver& resolver, GlyphPartition& partition) { + const ift::common::DataFileResolver& resolver, GlyphPartition& partition, + const CodepointSet& default_codepoints, + const btree_set& additional_default_features) { if (script_codepoints.size() <= 1) { return absl::OkStatus(); } @@ -360,6 +362,8 @@ static Status GroupInteractingScripts( TRY(GlyphClosureCache::Create(face, resolver)); SubsetDefinition init_segment; + init_segment.codepoints = default_codepoints; + init_segment.feature_tags = additional_default_features; AddInitSubsetDefaults(init_segment); std::vector segments; @@ -420,7 +424,9 @@ static StatusOr>> GroupOverlappingScripts( const btree_set& detected_scripts, const flat_hash_map& freq_list, const CodepointSet& unicodes, - const ift::common::DataFileResolver& resolver) { + const ift::common::DataFileResolver& resolver, + const CodepointSet& default_codepoints, + const btree_set& additional_default_features) { std::vector scripts(detected_scripts.begin(), detected_scripts.end()); std::vector codepoints; @@ -507,7 +513,8 @@ static StatusOr>> GroupOverlappingScripts( if (target_mode == UNICODE_RANGE) { TRYV(GroupInteractingScripts(face, scripts, codepoints, resolver, - partition)); + partition, default_codepoints, + additional_default_features)); } std::vector> groups; @@ -1061,9 +1068,12 @@ static std::vector PlaceIgnoreOverlapCodepoints( return out; } -static bool HasFeatureSegments(hb_face_t* face) { +static bool HasFeatureSegments( + hb_face_t* face, + const btree_set& additional_default_features) { for (hb_tag_t tag : FontHelper::GetFeatureTags(face)) { - if (!DefaultFeatureTags().contains(tag)) { + if (!DefaultFeatureTags().contains(tag) && + !additional_default_features.contains(tag)) { return true; } } @@ -1120,13 +1130,14 @@ uint64_t AutoSegmenterConfig::EstimateTableKeyedPatchCount( static uint32_t CountTableKeyedSegments( hb_face_t* face, const CodepointSet& unicodes, const flat_hash_map& freq_list, - const btree_set& detected_scripts) { + const btree_set& detected_scripts, + const btree_set& additional_default_features) { uint32_t num_segments = detected_scripts.size(); if (CoveredCodepoints(detected_scripts, freq_list, unicodes).size() < unicodes.size()) { num_segments++; } - if (HasFeatureSegments(face)) { + if (HasFeatureSegments(face, additional_default_features)) { num_segments++; } return num_segments; @@ -1191,8 +1202,17 @@ static void AddMergeGroup(const std::vector& scripts, StatusOr AutoSegmenterConfig::GenerateConfig( hb_face_t* face, const ift::common::DataFileResolver& resolver, std::optional primary_script, - std::optional quality_level, TargetMode target_mode) { + std::optional quality_level, TargetMode target_mode, + const ift::common::CodepointSet& default_codepoints, + const absl::btree_set& additional_default_features) { SegmenterConfig config; + for (hb_codepoint_t cp : default_codepoints) { + config.mutable_initial_segment()->mutable_codepoints()->add_values(cp); + } + for (hb_tag_t tag : additional_default_features) { + config.mutable_initial_segment()->mutable_features()->add_values( + FontHelper::ToString(tag)); + } if (target_mode == UNICODE_RANGE) { config.set_target_mode(UNICODE_RANGE); } else { @@ -1209,6 +1229,9 @@ StatusOr AutoSegmenterConfig::GenerateConfig( CodepointSet unicodes = FontHelper::ToCodepointsSet(face); uint32_t codepoint_count = unicodes.size(); + CodepointSet non_init_unicodes = unicodes; + non_init_unicodes.subtract(default_codepoints); + Quality quality = AutoPickQuality(codepoint_count); if (quality_level.has_value() && quality_level.value() >= MIN && @@ -1236,7 +1259,8 @@ StatusOr AutoSegmenterConfig::GenerateConfig( if (target_mode != UNICODE_RANGE) { ConfigureMaxDepth( - CountTableKeyedSegments(face, unicodes, freq_list, detected_scripts), + CountTableKeyedSegments(face, non_init_unicodes, freq_list, + detected_scripts, additional_default_features), *config.mutable_base_segmentation_plan()); } @@ -1245,13 +1269,17 @@ StatusOr AutoSegmenterConfig::GenerateConfig( // shared codepoints are cost merged instead of falling back to heuristic // merging. auto script_groups = TRY( - GroupOverlappingScripts(face, target_mode, detected_scripts, freq_list, unicodes, resolver)); + GroupOverlappingScripts(face, target_mode, detected_scripts, freq_list, + non_init_unicodes, resolver, default_codepoints, + additional_default_features)); std::vector codepoints_added_to_group(script_groups.size()); if (target_mode == UNICODE_RANGE) { // Any of ignore overlap codepoints may be in more than one merge group, // in unicode range mode we need to place thse into a single group. - codepoints_added_to_group = PlaceIgnoreOverlapCodepoints(target_mode, primary_script_file, script_groups, freq_list, unicodes); + codepoints_added_to_group = PlaceIgnoreOverlapCodepoints( + target_mode, primary_script_file, script_groups, freq_list, + non_init_unicodes); } for (uint32_t group_index = 0; group_index < script_groups.size(); group_index++) { @@ -1261,7 +1289,7 @@ StatusOr AutoSegmenterConfig::GenerateConfig( primary_script_file) == script_group.end()) { AddMergeGroup(script_group, primary_script_file, - GroupCodepointSetOverride(group_index, script_group, freq_list, codepoints_added_to_group, unicodes), + GroupCodepointSetOverride(group_index, script_group, freq_list, codepoints_added_to_group, non_init_unicodes), config); continue; } @@ -1272,7 +1300,7 @@ StatusOr AutoSegmenterConfig::GenerateConfig( // be treated as equally important as each of them, so instead give it a // merge group of its own. std::optional primary_script_override = - GroupCodepointSetOverride(group_index, {primary_script_file}, freq_list, codepoints_added_to_group, unicodes); + GroupCodepointSetOverride(group_index, {primary_script_file}, freq_list, codepoints_added_to_group, non_init_unicodes); AddMergeGroup({primary_script_file}, primary_script_file, primary_script_override, config); @@ -1289,13 +1317,13 @@ StatusOr AutoSegmenterConfig::GenerateConfig( } CodepointSet remaining_codepoints = - GroupCodepointSetOverride(group_index, remaining_scripts, freq_list, codepoints_added_to_group, unicodes) - .value_or(CoveredCodepoints(remaining_scripts, freq_list, unicodes)); + GroupCodepointSetOverride(group_index, remaining_scripts, freq_list, codepoints_added_to_group, non_init_unicodes) + .value_or(CoveredCodepoints(remaining_scripts, freq_list, non_init_unicodes)); remaining_codepoints.subtract( primary_script_override.value_or( CoveredCodepoints( - std::vector{primary_script_file}, freq_list, unicodes))); + std::vector{primary_script_file}, freq_list, non_init_unicodes))); if (remaining_codepoints.empty()) { // The primary script covers everything the rest of the group does, no diff --git a/ift/config/auto_segmenter_config.h b/ift/config/auto_segmenter_config.h index 2964dc68..f3f9c652 100644 --- a/ift/config/auto_segmenter_config.h +++ b/ift/config/auto_segmenter_config.h @@ -4,10 +4,12 @@ #include #include +#include "absl/container/btree_set.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "hb.h" #include "ift/common/data_file_resolver.h" +#include "ift/common/int_set.h" #include "ift/config/segmenter_config.pb.h" namespace ift::config { @@ -31,11 +33,20 @@ class AutoSegmenterConfig { // // target_mode: the target format mode for the generated segmentation config // (IFT or UNICODE_RANGE). Defaults to IFT. + // + // default_codepoints: optional set of codepoints to place into the initial + // font (initial_segment.codepoints). + // + // additional_default_features: optional set of layout feature tags to place + // into the initial font + // (initial_segment.features). static absl::StatusOr GenerateConfig( hb_face_t* face, const ift::common::DataFileResolver& resolver, std::optional primary_script = std::nullopt, std::optional quality_level = std::nullopt, - TargetMode target_mode = IFT); + TargetMode target_mode = IFT, + const ift::common::CodepointSet& default_codepoints = {}, + const absl::btree_set& additional_default_features = {}); // Returns the base script for a given language. // For example, "Language_fr" -> "Script_latin". diff --git a/ift/config/auto_segmenter_config_test.cc b/ift/config/auto_segmenter_config_test.cc index 790d8b6a..1d378ac0 100644 --- a/ift/config/auto_segmenter_config_test.cc +++ b/ift/config/auto_segmenter_config_test.cc @@ -861,5 +861,140 @@ TEST_F(AutoSegmenterConfigTest, UnicodeRange_PrimaryScriptSplitting) { EXPECT_FALSE(remaining_covered->intersects(*japanese_covered)); } +TEST_F(AutoSegmenterConfigTest, Ift_DefaultCodepointsAndFeatures) { + CodepointSet default_cps{0x41, 0x42}; + absl::btree_set default_features{HB_TAG('s', 'm', 'c', 'p')}; + + auto config_or = AutoSegmenterConfig::GenerateConfig( + face_.get(), *resolver, std::nullopt, std::nullopt, IFT, default_cps, + default_features); + ASSERT_TRUE(config_or.ok()) << config_or.status(); + + EXPECT_THAT(config_or->initial_segment().codepoints().values(), + UnorderedElementsAre(0x41, 0x42)); + EXPECT_THAT(config_or->initial_segment().features().values(), + UnorderedElementsAre("smcp")); +} + +TEST_F(AutoSegmenterConfigTest, MaxDepth_AllFeaturesInDefaultFeatures) { + ASSERT_TRUE(cjk_face_) << "NotoSansJP-Regular.ttf not found"; + + auto freq_list = BuiltInFrequenciesList(*resolver); + ASSERT_TRUE(freq_list.ok()) << freq_list.status(); + + // Collect only the codepoints covered by the 7 scripts used with Script_CJK. + CodepointSet covered_codepoints; + for (const auto& [ _, scripts ] : + {kEmojiLatinAndSymbols, kGreek, kCyrillic, kUnifiedCJK, kFallback}) { + for (const std::string& script : scripts) { + covered_codepoints.union_set(freq_list->at(script)); + } + } + covered_codepoints.intersect(FontHelper::ToCodepointsSet(cjk_face_.get())); + + // Font with no uncovered codepoints has 7 scripts + 1 feature-only = 8 table + // keyed segments (no max_depth). + // Now if we also put all non-default features in additional_default_features, + // table keyed segments becomes 7 (still no max_depth). + hb_subset_input_t* input_no_uncovered = hb_subset_input_create_or_fail(); + for (hb_codepoint_t cp : covered_codepoints) { + hb_set_add(hb_subset_input_unicode_set(input_no_uncovered), cp); + } + hb_set_invert(hb_subset_input_set(input_no_uncovered, + HB_SUBSET_SETS_LAYOUT_FEATURE_TAG)); + hb_face_unique_ptr subset_no_uncovered = + make_hb_face(hb_subset_or_fail(cjk_face_.get(), input_no_uncovered)); + hb_subset_input_destroy(input_no_uncovered); + + auto non_default_features = + FontHelper::GetNonDefaultFeatureTags(subset_no_uncovered.get()); + ASSERT_FALSE(non_default_features.empty()); + + auto config = AutoSegmenterConfig::GenerateConfig( + subset_no_uncovered.get(), *resolver, "Script_CJK", std::nullopt, IFT, {}, + non_default_features); + ASSERT_TRUE(config.ok()) << config.status(); + EXPECT_FALSE(config->base_segmentation_plan().has_max_depth()); +} + +TEST_F(AutoSegmenterConfigTest, + Roboto_UnspecifiedPrimary_UnicodeRange_DefaultCodepoints) { + hb_subset_input_t* input = hb_subset_input_create_or_fail(); + ASSERT_TRUE(input); + CodepointSet subset { + 0x20, // Shared w/ All + 0x301, // Shared w/ Latin + Cyrillic + // Latin + 'A', + 'B', + 'C', + // Greek + 0x384, + 0x385, + 0x394, + // Cyrillic + 0x401, + 0x402, + 0x403, + }; + subset.union_into(hb_subset_input_unicode_set(input)); + hb_set_clear( + hb_subset_input_set(input, HB_SUBSET_SETS_LAYOUT_FEATURE_TAG)); + hb_face_unique_ptr subset_face = + make_hb_face(hb_subset_or_fail(face_.get(), input)); + hb_subset_input_destroy(input); + + // Moving 0x20 and 0x301 to default_codepoints means they are in initial_segment + // and do not cause Latin + Cyrillic to group or 0x20 to need explicit segment_ids. + auto config_or = AutoSegmenterConfig::GenerateConfig( + subset_face.get(), *resolver, std::nullopt, std::nullopt, UNICODE_RANGE, + {0x20, 0x301}); + ASSERT_TRUE(config_or.ok()) << config_or.status(); + + EXPECT_THAT(config_or->initial_segment().codepoints().values(), + UnorderedElementsAre(0x20, 0x301)); + EXPECT_THAT(GetScripts(*config_or), + UnorderedElementsAre(kLatin, kCyrillic, kGreek)); + EXPECT_FALSE(GetSegmentIds(*config_or, kLatin.first).has_value()); + EXPECT_FALSE(GetSegmentIds(*config_or, kCyrillic.first).has_value()); + EXPECT_FALSE(GetSegmentIds(*config_or, kGreek.first).has_value()); +} + +TEST_F(AutoSegmenterConfigTest, + Roboto_InteractingScripts_UnicodeRange_DefaultCodepoints) { + hb_subset_input_t* input = hb_subset_input_create_or_fail(); + ASSERT_TRUE(input); + CodepointSet subset{ + // Latin + 'A', + 'B', + 'C', + // Greek (0x386 is a composite glyph using 'A' as a component) + 0x384, + 0x385, + 0x386, + // Cyrillic + 0x401, + 0x402, + 0x403, + }; + subset.union_into(hb_subset_input_unicode_set(input)); + hb_face_unique_ptr subset_face = + make_hb_face(hb_subset_or_fail(face_.get(), input)); + hb_subset_input_destroy(input); + + // Placing 'A' into default_codepoints means 'A' is in the initial segment + // (available in all subsets), so Greek and Latin no longer interact on 'A'. + auto config_or = AutoSegmenterConfig::GenerateConfig( + subset_face.get(), *resolver, std::nullopt, std::nullopt, UNICODE_RANGE, + {'A'}); + ASSERT_TRUE(config_or.ok()) << config_or.status(); + + EXPECT_THAT(config_or->initial_segment().codepoints().values(), + UnorderedElementsAre('A')); + EXPECT_THAT(GetScripts(*config_or), + UnorderedElementsAre(kGreek, kLatin, kCyrillic)); +} + } // namespace } // namespace ift::config diff --git a/ift/config/segmenter_config_util.cc b/ift/config/segmenter_config_util.cc index d9980947..fec939dc 100644 --- a/ift/config/segmenter_config_util.cc +++ b/ift/config/segmenter_config_util.cc @@ -80,7 +80,8 @@ std::vector SegmenterConfigUtil::ConfigToSegments( } else if (config.generate_feature_segments()) { uint32_t id = 0; for (hb_tag_t tag : font_features) { - if (DefaultFeatureTags().contains(tag)) { + if (DefaultFeatureTags().contains(tag) || + init_segment.feature_tags.contains(tag)) { continue; } SubsetDefinition def; diff --git a/ift/config/segmenter_config_util_test.cc b/ift/config/segmenter_config_util_test.cc index 6be79d24..9814a2de 100644 --- a/ift/config/segmenter_config_util_test.cc +++ b/ift/config/segmenter_config_util_test.cc @@ -311,6 +311,39 @@ TEST_F(SegmenterConfigUtilTest, ConfigToMergeGroups_FeatureSegmentsInferred) { ASSERT_EQ(*groups, (btree_map{})); } +TEST_F(SegmenterConfigUtilTest, + ConfigToMergeGroups_FeatureSegmentsInferred_InitFeaturesExcluded) { + SegmenterConfig config; + config.set_generate_feature_segments(true); + config.mutable_initial_segment()->mutable_features()->add_values("foo "); + + CodepointSet font_codepoints{1, 2, 4}; + btree_set font_features{HB_TAG('f', 'o', 'o', ' '), + HB_TAG('b', 'a', 'r', ' '), + HB_TAG('c', 'u', 'r', 's')}; + + SegmenterConfigUtil util("", resolver); + + std::vector segments_out; + auto groups = util.ConfigToMergeGroups(config, font_codepoints, font_features, + segments_out); + ASSERT_TRUE(groups.ok()) << groups.status(); + + SubsetDefinition bar; + bar.feature_tags = { + HB_TAG('b', 'a', 'r', ' '), + }; + + ASSERT_EQ(segments_out, (std::vector{ + bar, + {1}, + {2}, + {4}, + })); + + ASSERT_EQ(*groups, (btree_map{})); +} + TEST_F(SegmenterConfigUtilTest, ConfigToMergeGroups_SegmentsProvided_Heuristic) { // Minimal config with one heuristic merge group that covers everything. diff --git a/util/BUILD b/util/BUILD index 424ae9c3..b93989b7 100644 --- a/util/BUILD +++ b/util/BUILD @@ -177,7 +177,36 @@ cc_library( srcs = ["auto_config_flags.cc"], hdrs = ["auto_config_flags.h"], deps = [ + "//ift/common", + "//ift/common:data_file_resolver", + "//ift/common:try", + "//ift/config:auto_segmenter_config", + "//ift/config:segmenter_config_cc_proto", + "@abseil-cpp//absl/container:btree", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/types:span", + "@harfbuzz", + ], +) + +cc_test( + name = "auto_config_flags_test", + size = "small", + srcs = ["auto_config_flags_test.cc"], + data = ["//ift/common:testdata"], + deps = [ + ":auto_config_flags", + "//ift/common", + "//ift/common:data_file_resolver", + "//ift/config:load_codepoints", "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/status", + "@googletest//:gtest", + "@googletest//:gtest_main", + "@harfbuzz", ], ) @@ -187,18 +216,21 @@ cc_binary( "gen_ift_segmenter_config.cc", ], deps = [ + ":auto_config_flags", "//ift/common", "//ift/common:data_file_resolver", "//ift/common:try", "//ift/config:auto_segmenter_config", "//ift/config:load_codepoints", "//ift/config:segmenter_config_cc_proto", + "@abseil-cpp//absl/container:btree", "@abseil-cpp//absl/flags:flag", "@abseil-cpp//absl/flags:parse", "@abseil-cpp//absl/flags:usage", "@abseil-cpp//absl/log:globals", "@abseil-cpp//absl/log:initialize", "@abseil-cpp//absl/status", + "@abseil-cpp//absl/strings", "@harfbuzz", "@protobuf", ], diff --git a/util/auto_config_flags.cc b/util/auto_config_flags.cc index 3f786a7c..82e780c1 100644 --- a/util/auto_config_flags.cc +++ b/util/auto_config_flags.cc @@ -1,8 +1,28 @@ #include "util/auto_config_flags.h" +#include +#include #include +#include +#include "absl/container/btree_set.h" #include "absl/flags/flag.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/ascii.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "hb.h" +#include "ift/common/data_file_resolver.h" +#include "ift/common/font_helper.h" +#include "ift/common/int_set.h" +#include "ift/common/try.h" +#include "ift/config/auto_segmenter_config.h" +#include "ift/config/segmenter_config.pb.h" ABSL_FLAG( int, auto_config_quality, 0, @@ -24,3 +44,154 @@ ABSL_FLAG( bool, auto_config_unicode_range, false, "When auto_config is enabled, generate a segmenter config with target_mode " "set to UNICODE_RANGE instead of IFT."); + +ABSL_FLAG( + std::vector, auto_config_additional_default_features, {}, + "Comma-separated list of OpenType feature tags (e.g. smcp,dlig) to add to " + "the initial font in the produced segmenter config."); + +ABSL_FLAG( + std::vector, auto_config_default_codepoints, {}, + "Comma-separated list of Unicode codepoints or ranges (in hex like 0x41 or " + "U+0041, or decimal, with ranges using - or .. like 0xFE00-0xFE0F) to add " + "to the initial font in the produced segmenter config."); + +namespace ift::util { + +static absl::StatusOr ParseSingleCodepoint( + absl::string_view token) { + token = absl::StripAsciiWhitespace(token); + if (token.empty()) { + return absl::InvalidArgumentError("Empty codepoint"); + } + + uint32_t val = 0; + if (absl::StartsWithIgnoreCase(token, "0x") || + absl::StartsWithIgnoreCase(token, "u+")) { + absl::string_view hex_part = token.substr(2); + if (hex_part.empty()) { + return absl::InvalidArgumentError( + absl::StrCat("Invalid hex codepoint: ", token)); + } + for (char c : hex_part) { + if (!absl::ascii_isxdigit(c)) { + return absl::InvalidArgumentError( + absl::StrCat("Invalid character in hex codepoint: ", token)); + } + } + if (!absl::SimpleHexAtoi(hex_part, &val)) { + return absl::InvalidArgumentError( + absl::StrCat("Failed to parse hex codepoint: ", token)); + } + } else { + for (char c : token) { + if (!absl::ascii_isdigit(c)) { + return absl::InvalidArgumentError( + absl::StrCat("Invalid character in decimal codepoint: ", token)); + } + } + if (!absl::SimpleAtoi(token, &val)) { + return absl::InvalidArgumentError( + absl::StrCat("Failed to parse decimal codepoint: ", token)); + } + } + + if (val > 0x10FFFF) { + return absl::InvalidArgumentError( + absl::StrCat("Codepoint value out of range (max 0x10FFFF): ", token)); + } + return static_cast(val); +} + +absl::StatusOr ParseCodepointsFlag( + absl::Span items) { + ift::common::CodepointSet result; + for (const std::string& item : items) { + for (absl::string_view token : + absl::StrSplit(item, ',', absl::SkipEmpty())) { + token = absl::StripAsciiWhitespace(token); + if (token.empty()) { + continue; + } + + size_t sep_pos = token.find(".."); + size_t sep_len = 2; + if (sep_pos == absl::string_view::npos) { + sep_pos = token.find('-'); + sep_len = 1; + } + if (sep_pos != absl::string_view::npos) { + absl::string_view start_str = token.substr(0, sep_pos); + absl::string_view end_str = token.substr(sep_pos + sep_len); + hb_codepoint_t start = TRY(ParseSingleCodepoint(start_str)); + hb_codepoint_t end = TRY(ParseSingleCodepoint(end_str)); + if (start > end) { + return absl::InvalidArgumentError( + absl::StrCat("Range start must be <= end: ", token)); + } + result.insert_range(start, end); + continue; + } + + hb_codepoint_t cp = TRY(ParseSingleCodepoint(token)); + result.insert(cp); + } + } + return result; +} + +absl::StatusOr> ParseFeatureTagsFlag( + absl::Span items) { + absl::btree_set result; + for (const std::string& item : items) { + for (absl::string_view tag_str : + absl::StrSplit(item, ',', absl::SkipEmpty())) { + if (tag_str.empty()) { + continue; + } + if (tag_str.size() > 4) { + tag_str = absl::StripAsciiWhitespace(tag_str); + } + if (tag_str.size() != 4) { + return absl::InvalidArgumentError( + absl::StrCat("Invalid feature tag (must be 4 characters): ", + tag_str)); + } + result.insert(ift::common::FontHelper::ToTag(std::string(tag_str))); + } + } + return result; +} + +absl::StatusOr +GenerateAutoSegmenterConfigFromFlags( + hb_face_t* font, const ift::common::DataFileResolver& resolver, + std::optional target_mode_override) { + std::optional quality_level = std::nullopt; + if (absl::GetFlag(FLAGS_auto_config_quality) > 0) { + quality_level = absl::GetFlag(FLAGS_auto_config_quality); + } + + std::optional primary_script = std::nullopt; + if (!absl::GetFlag(FLAGS_auto_config_primary_script).empty()) { + primary_script = absl::GetFlag(FLAGS_auto_config_primary_script); + } + + ift::config::TargetMode target_mode = + target_mode_override.value_or( + absl::GetFlag(FLAGS_auto_config_unicode_range) + ? ift::config::UNICODE_RANGE + : ift::config::IFT); + + auto default_codepoints = TRY( + ParseCodepointsFlag(absl::GetFlag(FLAGS_auto_config_default_codepoints))); + + auto additional_default_features = TRY(ParseFeatureTagsFlag( + absl::GetFlag(FLAGS_auto_config_additional_default_features))); + + return ift::config::AutoSegmenterConfig::GenerateConfig( + font, resolver, primary_script, quality_level, target_mode, + default_codepoints, additional_default_features); +} + +} // namespace ift::util diff --git a/util/auto_config_flags.h b/util/auto_config_flags.h index 53da5bf3..bfef0b6c 100644 --- a/util/auto_config_flags.h +++ b/util/auto_config_flags.h @@ -1,12 +1,40 @@ #ifndef UTIL_AUTO_CONFIG_FLAGS_H_ #define UTIL_AUTO_CONFIG_FLAGS_H_ +#include #include +#include +#include "absl/container/btree_set.h" #include "absl/flags/declare.h" +#include "absl/status/statusor.h" +#include "absl/types/span.h" +#include "hb.h" +#include "ift/common/data_file_resolver.h" +#include "ift/common/int_set.h" +#include "ift/config/auto_segmenter_config.h" +#include "ift/config/segmenter_config.pb.h" ABSL_DECLARE_FLAG(int, auto_config_quality); ABSL_DECLARE_FLAG(std::string, auto_config_primary_script); ABSL_DECLARE_FLAG(bool, auto_config_unicode_range); +ABSL_DECLARE_FLAG(std::vector, + auto_config_additional_default_features); +ABSL_DECLARE_FLAG(std::vector, auto_config_default_codepoints); + +namespace ift::util { + +absl::StatusOr ParseCodepointsFlag( + absl::Span items); + +absl::StatusOr> ParseFeatureTagsFlag( + absl::Span items); + +absl::StatusOr +GenerateAutoSegmenterConfigFromFlags( + hb_face_t* font, const ift::common::DataFileResolver& resolver, + std::optional target_mode_override = std::nullopt); + +} // namespace ift::util #endif // UTIL_AUTO_CONFIG_FLAGS_H_ diff --git a/util/auto_config_flags_test.cc b/util/auto_config_flags_test.cc new file mode 100644 index 00000000..c0eca912 --- /dev/null +++ b/util/auto_config_flags_test.cc @@ -0,0 +1,140 @@ +#include "util/auto_config_flags.h" + +#include +#include + +#include "absl/flags/flag.h" +#include "absl/status/status.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "hb.h" +#include "ift/common/bazel_data_file_resolver.h" +#include "ift/common/font_helper.h" +#include "ift/common/int_set.h" +#include "ift/config/load_codepoints.h" + +namespace ift::util { +namespace { + +using ::testing::ElementsAre; +using ::testing::UnorderedElementsAre; + +TEST(AutoConfigFlagsTest, ParseCodepointsFlag_Empty) { + auto result = ParseCodepointsFlag({}); + ASSERT_TRUE(result.ok()) << result.status(); + EXPECT_TRUE(result->empty()); + + result = ParseCodepointsFlag({""}); + ASSERT_TRUE(result.ok()) << result.status(); + EXPECT_TRUE(result->empty()); +} + +TEST(AutoConfigFlagsTest, ParseCodepointsFlag_SingleValues) { + auto result = ParseCodepointsFlag({"0x41", "0X42", "U+0043", "u+0044", "69"}); + ASSERT_TRUE(result.ok()) << result.status(); + EXPECT_THAT(result->to_vector(), ElementsAre(0x41, 0x42, 0x43, 0x44, 69)); +} + +TEST(AutoConfigFlagsTest, ParseCodepointsFlag_Ranges) { + auto result = ParseCodepointsFlag({"0x30-0x32", "U+0100..U+0102", "65-67"}); + ASSERT_TRUE(result.ok()) << result.status(); + EXPECT_THAT(result->to_vector(), + ElementsAre(0x30, 0x31, 0x32, 65, 66, 67, 0x100, 0x101, 0x102)); +} + +TEST(AutoConfigFlagsTest, ParseCodepointsFlag_CommaSeparatedAndWhitespace) { + auto result = ParseCodepointsFlag( + {" 0x41 , 0x42 ", " 0x50 - 0x52 , U+60..U+61 "}); + ASSERT_TRUE(result.ok()) << result.status(); + EXPECT_THAT(result->to_vector(), + ElementsAre(0x41, 0x42, 0x50, 0x51, 0x52, 0x60, 0x61)); +} + +TEST(AutoConfigFlagsTest, ParseCodepointsFlag_InvalidFormats) { + // Invalid hex + EXPECT_EQ(ParseCodepointsFlag({"0x"}).status().code(), + absl::StatusCode::kInvalidArgument); + EXPECT_EQ(ParseCodepointsFlag({"0xZZ"}).status().code(), + absl::StatusCode::kInvalidArgument); + EXPECT_EQ(ParseCodepointsFlag({"U+G"}).status().code(), + absl::StatusCode::kInvalidArgument); + + // Invalid decimal + EXPECT_EQ(ParseCodepointsFlag({"12a"}).status().code(), + absl::StatusCode::kInvalidArgument); + EXPECT_EQ(ParseCodepointsFlag({"-5"}).status().code(), + absl::StatusCode::kInvalidArgument); + + // Out of range (> 0x10FFFF) + EXPECT_EQ(ParseCodepointsFlag({"0x110000"}).status().code(), + absl::StatusCode::kInvalidArgument); + EXPECT_EQ(ParseCodepointsFlag({"1114112"}).status().code(), + absl::StatusCode::kInvalidArgument); + + // Inverted range + EXPECT_EQ(ParseCodepointsFlag({"0x50-0x40"}).status().code(), + absl::StatusCode::kInvalidArgument); + EXPECT_EQ(ParseCodepointsFlag({"100..50"}).status().code(), + absl::StatusCode::kInvalidArgument); + + // Malformed range + EXPECT_EQ(ParseCodepointsFlag({"0x40-"}).status().code(), + absl::StatusCode::kInvalidArgument); + EXPECT_EQ(ParseCodepointsFlag({"-0x40"}).status().code(), + absl::StatusCode::kInvalidArgument); + EXPECT_EQ(ParseCodepointsFlag({"0x40.."}).status().code(), + absl::StatusCode::kInvalidArgument); +} + +TEST(AutoConfigFlagsTest, ParseFeatureTagsFlag_Valid) { + auto result = ParseFeatureTagsFlag({"smcp", "dlig"}); + ASSERT_TRUE(result.ok()) << result.status(); + EXPECT_THAT( + *result, + UnorderedElementsAre(HB_TAG('s', 'm', 'c', 'p'), HB_TAG('d', 'l', 'i', 'g'))); + + // Comma separated with whitespace + result = ParseFeatureTagsFlag({"smcp, dlig", "cv01"}); + ASSERT_TRUE(result.ok()) << result.status(); + EXPECT_THAT( + *result, + UnorderedElementsAre(HB_TAG('s', 'm', 'c', 'p'), HB_TAG('d', 'l', 'i', 'g'), + HB_TAG('c', 'v', '0', '1'))); +} + +TEST(AutoConfigFlagsTest, ParseFeatureTagsFlag_InvalidLength) { + EXPECT_EQ(ParseFeatureTagsFlag({"a"}).status().code(), + absl::StatusCode::kInvalidArgument); + EXPECT_EQ(ParseFeatureTagsFlag({"abc"}).status().code(), + absl::StatusCode::kInvalidArgument); + EXPECT_EQ(ParseFeatureTagsFlag({"abcde"}).status().code(), + absl::StatusCode::kInvalidArgument); +} + +TEST(AutoConfigFlagsTest, GenerateAutoSegmenterConfigFromFlags) { + auto resolver = ift::common::BazelDataFileResolver::CreateForTest(); + ASSERT_TRUE(resolver.ok()) << resolver.status(); + + auto font_data = + ift::config::LoadFile("ift/common/testdata/Roboto-Regular.ttf"); + ASSERT_TRUE(font_data.ok()) << font_data.status(); + auto face = font_data->face(); + + absl::SetFlag(&FLAGS_auto_config_quality, 5); + absl::SetFlag(&FLAGS_auto_config_primary_script, "Script_latin"); + absl::SetFlag(&FLAGS_auto_config_default_codepoints, + std::vector{"0x20", "0x41-0x43"}); + absl::SetFlag(&FLAGS_auto_config_additional_default_features, + std::vector{"smcp"}); + + auto config = GenerateAutoSegmenterConfigFromFlags(face.get(), **resolver); + ASSERT_TRUE(config.ok()) << config.status(); + + EXPECT_THAT(config->initial_segment().codepoints().values(), + UnorderedElementsAre(0x20, 0x41, 0x42, 0x43)); + EXPECT_THAT(config->initial_segment().features().values(), + UnorderedElementsAre("smcp")); +} + +} // namespace +} // namespace ift::util diff --git a/util/font2ift.cc b/util/font2ift.cc index 187d85f0..b1192915 100644 --- a/util/font2ift.cc +++ b/util/font2ift.cc @@ -158,17 +158,8 @@ StatusOr CreateSegmentationPlan( if (absl::GetFlag(FLAGS_plan).empty() || absl::GetFlag(FLAGS_plan) == "auto") { std::cerr << ">> auto generating segmentation plan:" << std::endl; - std::optional quality_level = std::nullopt; - if (absl::GetFlag(FLAGS_auto_config_quality) > 0) { - quality_level = absl::GetFlag(FLAGS_auto_config_quality); - } - - std::optional primary_script = std::nullopt; - if (!absl::GetFlag(FLAGS_auto_config_primary_script).empty()) { - primary_script = absl::GetFlag(FLAGS_auto_config_primary_script); - } - auto config = AutoSegmenterConfig::GenerateConfig( - font, *resolver, primary_script, quality_level); + auto config = ift::util::GenerateAutoSegmenterConfigFromFlags( + font, *resolver, ift::config::IFT); if (!config.ok()) { return absl::InternalError( StrCat("Failed to generate config: ", config.status().message())); diff --git a/util/font2ur.cc b/util/font2ur.cc index e1e0eb04..c6432b54 100644 --- a/util/font2ur.cc +++ b/util/font2ur.cc @@ -129,18 +129,8 @@ static StatusOr CreateSegmentationPlan( absl::GetFlag(FLAGS_plan) == "auto") { std::cerr << ">> auto generating unicode-range segmentation plan:" << std::endl; - std::optional quality_level = std::nullopt; - if (absl::GetFlag(FLAGS_auto_config_quality) > 0) { - quality_level = absl::GetFlag(FLAGS_auto_config_quality); - } - - std::optional primary_script = std::nullopt; - if (!absl::GetFlag(FLAGS_auto_config_primary_script).empty()) { - primary_script = absl::GetFlag(FLAGS_auto_config_primary_script); - } - auto config = AutoSegmenterConfig::GenerateConfig( - font, *resolver, primary_script, quality_level, - ift::config::UNICODE_RANGE); + auto config = ift::util::GenerateAutoSegmenterConfigFromFlags( + font, *resolver, ift::config::UNICODE_RANGE); if (!config.ok()) { return absl::InternalError( StrCat("Failed to generate config: ", config.status().message())); diff --git a/util/gen_ift_segmentation_plan.cc b/util/gen_ift_segmentation_plan.cc index 2f226424..5cc9c022 100644 --- a/util/gen_ift_segmentation_plan.cc +++ b/util/gen_ift_segmentation_plan.cc @@ -112,21 +112,7 @@ using ift::freq::UnigramProbabilityCalculator; static StatusOr LoadConfig(hb_face_t* font, const DataFileResolver& resolver) { if (absl::GetFlag(FLAGS_config) == "auto") { - std::optional quality_level = std::nullopt; - if (absl::GetFlag(FLAGS_auto_config_quality) > 0) { - quality_level = absl::GetFlag(FLAGS_auto_config_quality); - } - - std::optional primary_script = std::nullopt; - if (!absl::GetFlag(FLAGS_auto_config_primary_script).empty()) { - primary_script = absl::GetFlag(FLAGS_auto_config_primary_script); - } - ift::config::TargetMode target_mode = - absl::GetFlag(FLAGS_auto_config_unicode_range) - ? ift::config::UNICODE_RANGE - : ift::config::IFT; - return AutoSegmenterConfig::GenerateConfig(font, resolver, primary_script, - quality_level, target_mode); + return ift::util::GenerateAutoSegmenterConfigFromFlags(font, resolver); } FontData config_text = diff --git a/util/gen_ift_segmenter_config.cc b/util/gen_ift_segmenter_config.cc index 879d2cbf..aa2c485e 100644 --- a/util/gen_ift_segmenter_config.cc +++ b/util/gen_ift_segmenter_config.cc @@ -15,6 +15,7 @@ #include "ift/common/try.h" #include "ift/config/auto_segmenter_config.h" #include "ift/config/load_codepoints.h" +#include "util/auto_config_flags.h" using ift::config::SegmenterConfig; @@ -39,6 +40,17 @@ ABSL_FLAG(bool, unicode_range, false, "If true, generate a segmenter config with target_mode set to " "UNICODE_RANGE instead of IFT."); +ABSL_FLAG( + std::vector, additional_default_features, {}, + "Comma-separated list of OpenType feature tags (e.g. smcp,dlig) to add to " + "the initial font in the produced segmenter config."); + +ABSL_FLAG( + std::vector, default_codepoints, {}, + "Comma-separated list of Unicode codepoints or ranges (in hex like 0x41 or " + "U+0041, or decimal, with ranges using - or .. like 0xFE00-0xFE0F) to add " + "to the initial font in the produced segmenter config."); + ABSL_FLAG( int, verbosity, 0, "Log verbosity level from. 0 is least verbose, higher values are more."); @@ -67,8 +79,13 @@ static Status Main(const std::vector args) { ift::config::TargetMode target_mode = absl::GetFlag(FLAGS_unicode_range) ? ift::config::UNICODE_RANGE : ift::config::IFT; + auto default_codepoints = + TRY(ift::util::ParseCodepointsFlag(absl::GetFlag(FLAGS_default_codepoints))); + auto additional_default_features = TRY(ift::util::ParseFeatureTagsFlag( + absl::GetFlag(FLAGS_additional_default_features))); auto config = TRY(AutoSegmenterConfig::GenerateConfig( - font.get(), *resolver, primary_script, quality_level, target_mode)); + font.get(), *resolver, primary_script, quality_level, target_mode, + default_codepoints, additional_default_features)); std::string output; if (!google::protobuf::TextFormat::PrintToString(config, &output)) { @@ -85,7 +102,9 @@ int main(int argc, char** argv) { "Generates a segmenter config to use with a font.\n" "\n" "Usage: gen_ift_segmenter_config --font=\"myfont.ttf\" [--quality=n] " - "[--primary_script=Script_foo] [--unicode_range]\n"); + "[--primary_script=Script_foo] [--unicode_range] " + "[--additional_default_features=tag1,tag2] " + "[--default_codepoints=cp1,cp2-cp3]\n"); absl::SetStderrThreshold(absl::LogSeverityAtLeast::kInfo); absl::SetGlobalVLogLevel(absl::GetFlag(FLAGS_verbosity)); auto args = absl::ParseCommandLine(argc, argv);