Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions docs/creating_ift_fonts.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ bazel run -c opt //util:font2ift -- \
This will analyze the input font, decide how to segment it, and then produce the final IFT encoded font
and patches.

When utilizing auto config there are two optional flags which can be used to adjust the behaviour:
When utilizing auto config there are optional flags which can be used to adjust the behaviour:
* `--auto_config_primary_script`: this tells the config generator which language/script the font is intended
to be used with. It has two effects: first the codepoints of the primary script are eligible to be moved
into the initial font. Second for scripts with large overlaps, such as CJK, primary script selects which
Expand All @@ -35,6 +35,14 @@ When utilizing auto config there are two optional flags which can be used to adj
values increase encoding times but typically result in a more efficient end IFT font (ie. less bytes
transferred by clients using it).

* `--auto_config_additional_default_features`: Takes a comma-separated list of 4-character OpenType feature tags
(e.g. `smcp,dlig`) which are added to the initial font (`initial_segment.features`) in the produced segmenter
config so that those layout features are always available in the base font.

* `--auto_config_default_codepoints`: Takes a comma-separated list of Unicode codepoints or codepoint ranges
(in hex with `0x` or `U+` prefix, or decimal, with `-` or `..` for ranges, e.g. `0x20,0x41-0x5A,U+FE00..U+FE0F`)
which are added to the initial font (`initial_segment.codepoints`) in the produced segmenter config.

Example command line with optional flags:

```bash
Expand Down Expand Up @@ -84,8 +92,10 @@ There are two main options for generating a segmenter config:
```

This analyzes the input font and tries to pick appropriate config values automatically. As discussed in
the previous "Producing IFT Encoded Fonts" section there is a configurable quality level. If needed
the auto generated config can be hand tweaked after generation.
the previous "Producing IFT Encoded Fonts" section there is a configurable quality level. Additional flags
including `--primary_script`, `--additional_default_features`, and `--default_codepoints` can also be
provided to control the initial font segment and script prioritization. If needed the auto generated config
can be hand tweaked after generation.

### Step 2: Generating Segmentation Plan

Expand Down
60 changes: 44 additions & 16 deletions ift/config/auto_segmenter_config.cc
Original file line number Diff line number Diff line change
Expand Up @@ -351,7 +351,9 @@ static CodepointSet OverlapCodepoints(
static Status GroupInteractingScripts(
hb_face_t* face, const std::vector<std::string>& scripts,
const std::vector<CodepointSet>& script_codepoints,
const ift::common::DataFileResolver& resolver, GlyphPartition& partition) {
const ift::common::DataFileResolver& resolver, GlyphPartition& partition,
const CodepointSet& default_codepoints,
const btree_set<hb_tag_t>& additional_default_features) {
if (script_codepoints.size() <= 1) {
return absl::OkStatus();
}
Expand All @@ -360,6 +362,8 @@ static Status GroupInteractingScripts(
TRY(GlyphClosureCache::Create(face, resolver));

SubsetDefinition init_segment;
init_segment.codepoints = default_codepoints;
init_segment.feature_tags = additional_default_features;
AddInitSubsetDefaults(init_segment);

std::vector<Segment> segments;
Expand Down Expand Up @@ -420,7 +424,9 @@ static StatusOr<std::vector<std::vector<std::string>>> GroupOverlappingScripts(
const btree_set<std::string>& detected_scripts,
const flat_hash_map<std::string, CodepointSet>& freq_list,
const CodepointSet& unicodes,
const ift::common::DataFileResolver& resolver) {
const ift::common::DataFileResolver& resolver,
const CodepointSet& default_codepoints,
const btree_set<hb_tag_t>& additional_default_features) {
std::vector<std::string> scripts(detected_scripts.begin(),
detected_scripts.end());
std::vector<CodepointSet> codepoints;
Expand Down Expand Up @@ -507,7 +513,8 @@ static StatusOr<std::vector<std::vector<std::string>>> GroupOverlappingScripts(

if (target_mode == UNICODE_RANGE) {
TRYV(GroupInteractingScripts(face, scripts, codepoints, resolver,
partition));
partition, default_codepoints,
additional_default_features));
}

std::vector<std::vector<std::string>> groups;
Expand Down Expand Up @@ -1061,9 +1068,12 @@ static std::vector<CodepointSet> PlaceIgnoreOverlapCodepoints(
return out;
}

static bool HasFeatureSegments(hb_face_t* face) {
static bool HasFeatureSegments(
hb_face_t* face,
const btree_set<hb_tag_t>& additional_default_features) {
for (hb_tag_t tag : FontHelper::GetFeatureTags(face)) {
if (!DefaultFeatureTags().contains(tag)) {
if (!DefaultFeatureTags().contains(tag) &&
!additional_default_features.contains(tag)) {
return true;
}
}
Expand Down Expand Up @@ -1120,13 +1130,14 @@ uint64_t AutoSegmenterConfig::EstimateTableKeyedPatchCount(
static uint32_t CountTableKeyedSegments(
hb_face_t* face, const CodepointSet& unicodes,
const flat_hash_map<std::string, CodepointSet>& freq_list,
const btree_set<std::string>& detected_scripts) {
const btree_set<std::string>& detected_scripts,
const btree_set<hb_tag_t>& additional_default_features) {
uint32_t num_segments = detected_scripts.size();
if (CoveredCodepoints(detected_scripts, freq_list, unicodes).size() <
unicodes.size()) {
num_segments++;
}
if (HasFeatureSegments(face)) {
if (HasFeatureSegments(face, additional_default_features)) {
num_segments++;
}
return num_segments;
Expand Down Expand Up @@ -1191,8 +1202,17 @@ static void AddMergeGroup(const std::vector<std::string>& scripts,
StatusOr<SegmenterConfig> AutoSegmenterConfig::GenerateConfig(
hb_face_t* face, const ift::common::DataFileResolver& resolver,
std::optional<std::string> primary_script,
std::optional<int> quality_level, TargetMode target_mode) {
std::optional<int> quality_level, TargetMode target_mode,
const ift::common::CodepointSet& default_codepoints,
const absl::btree_set<hb_tag_t>& additional_default_features) {
SegmenterConfig config;
for (hb_codepoint_t cp : default_codepoints) {
config.mutable_initial_segment()->mutable_codepoints()->add_values(cp);
}
for (hb_tag_t tag : additional_default_features) {
config.mutable_initial_segment()->mutable_features()->add_values(
FontHelper::ToString(tag));
}
if (target_mode == UNICODE_RANGE) {
config.set_target_mode(UNICODE_RANGE);
} else {
Expand All @@ -1209,6 +1229,9 @@ StatusOr<SegmenterConfig> AutoSegmenterConfig::GenerateConfig(
CodepointSet unicodes = FontHelper::ToCodepointsSet(face);
uint32_t codepoint_count = unicodes.size();

CodepointSet non_init_unicodes = unicodes;
non_init_unicodes.subtract(default_codepoints);

Quality quality = AutoPickQuality(codepoint_count);

if (quality_level.has_value() && quality_level.value() >= MIN &&
Expand Down Expand Up @@ -1236,7 +1259,8 @@ StatusOr<SegmenterConfig> AutoSegmenterConfig::GenerateConfig(

if (target_mode != UNICODE_RANGE) {
ConfigureMaxDepth(
CountTableKeyedSegments(face, unicodes, freq_list, detected_scripts),
CountTableKeyedSegments(face, non_init_unicodes, freq_list,
detected_scripts, additional_default_features),
*config.mutable_base_segmentation_plan());
}

Expand All @@ -1245,13 +1269,17 @@ StatusOr<SegmenterConfig> AutoSegmenterConfig::GenerateConfig(
// shared codepoints are cost merged instead of falling back to heuristic
// merging.
auto script_groups = TRY(
GroupOverlappingScripts(face, target_mode, detected_scripts, freq_list, unicodes, resolver));
GroupOverlappingScripts(face, target_mode, detected_scripts, freq_list,
non_init_unicodes, resolver, default_codepoints,
additional_default_features));

std::vector<CodepointSet> codepoints_added_to_group(script_groups.size());
if (target_mode == UNICODE_RANGE) {
// Any of ignore overlap codepoints may be in more than one merge group,
// in unicode range mode we need to place thse into a single group.
codepoints_added_to_group = PlaceIgnoreOverlapCodepoints(target_mode, primary_script_file, script_groups, freq_list, unicodes);
codepoints_added_to_group = PlaceIgnoreOverlapCodepoints(
target_mode, primary_script_file, script_groups, freq_list,
non_init_unicodes);
}

for (uint32_t group_index = 0; group_index < script_groups.size(); group_index++) {
Expand All @@ -1261,7 +1289,7 @@ StatusOr<SegmenterConfig> AutoSegmenterConfig::GenerateConfig(
primary_script_file) == script_group.end()) {
AddMergeGroup(script_group,
primary_script_file,
GroupCodepointSetOverride(group_index, script_group, freq_list, codepoints_added_to_group, unicodes),
GroupCodepointSetOverride(group_index, script_group, freq_list, codepoints_added_to_group, non_init_unicodes),
config);
continue;
}
Expand All @@ -1272,7 +1300,7 @@ StatusOr<SegmenterConfig> AutoSegmenterConfig::GenerateConfig(
// be treated as equally important as each of them, so instead give it a
// merge group of its own.
std::optional<CodepointSet> primary_script_override =
GroupCodepointSetOverride(group_index, {primary_script_file}, freq_list, codepoints_added_to_group, unicodes);
GroupCodepointSetOverride(group_index, {primary_script_file}, freq_list, codepoints_added_to_group, non_init_unicodes);
AddMergeGroup({primary_script_file}, primary_script_file,
primary_script_override,
config);
Expand All @@ -1289,13 +1317,13 @@ StatusOr<SegmenterConfig> AutoSegmenterConfig::GenerateConfig(
}

CodepointSet remaining_codepoints =
GroupCodepointSetOverride(group_index, remaining_scripts, freq_list, codepoints_added_to_group, unicodes)
.value_or(CoveredCodepoints(remaining_scripts, freq_list, unicodes));
GroupCodepointSetOverride(group_index, remaining_scripts, freq_list, codepoints_added_to_group, non_init_unicodes)
.value_or(CoveredCodepoints(remaining_scripts, freq_list, non_init_unicodes));

remaining_codepoints.subtract(
primary_script_override.value_or(
CoveredCodepoints(
std::vector<std::string>{primary_script_file}, freq_list, unicodes)));
std::vector<std::string>{primary_script_file}, freq_list, non_init_unicodes)));

if (remaining_codepoints.empty()) {
// The primary script covers everything the rest of the group does, no
Expand Down
13 changes: 12 additions & 1 deletion ift/config/auto_segmenter_config.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@
#include <optional>
#include <string>

#include "absl/container/btree_set.h"
#include "absl/status/statusor.h"
#include "absl/strings/string_view.h"
#include "hb.h"
#include "ift/common/data_file_resolver.h"
#include "ift/common/int_set.h"
#include "ift/config/segmenter_config.pb.h"

namespace ift::config {
Expand All @@ -31,11 +33,20 @@ class AutoSegmenterConfig {
//
// target_mode: the target format mode for the generated segmentation config
// (IFT or UNICODE_RANGE). Defaults to IFT.
//
// default_codepoints: optional set of codepoints to place into the initial
// font (initial_segment.codepoints).
//
// additional_default_features: optional set of layout feature tags to place
// into the initial font
// (initial_segment.features).
static absl::StatusOr<SegmenterConfig> GenerateConfig(
hb_face_t* face, const ift::common::DataFileResolver& resolver,
std::optional<std::string> primary_script = std::nullopt,
std::optional<int> quality_level = std::nullopt,
TargetMode target_mode = IFT);
TargetMode target_mode = IFT,
const ift::common::CodepointSet& default_codepoints = {},
const absl::btree_set<hb_tag_t>& additional_default_features = {});

// Returns the base script for a given language.
// For example, "Language_fr" -> "Script_latin".
Expand Down
135 changes: 135 additions & 0 deletions ift/config/auto_segmenter_config_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -861,5 +861,140 @@ TEST_F(AutoSegmenterConfigTest, UnicodeRange_PrimaryScriptSplitting) {
EXPECT_FALSE(remaining_covered->intersects(*japanese_covered));
}

TEST_F(AutoSegmenterConfigTest, Ift_DefaultCodepointsAndFeatures) {
CodepointSet default_cps{0x41, 0x42};
absl::btree_set<hb_tag_t> default_features{HB_TAG('s', 'm', 'c', 'p')};

auto config_or = AutoSegmenterConfig::GenerateConfig(
face_.get(), *resolver, std::nullopt, std::nullopt, IFT, default_cps,
default_features);
ASSERT_TRUE(config_or.ok()) << config_or.status();

EXPECT_THAT(config_or->initial_segment().codepoints().values(),
UnorderedElementsAre(0x41, 0x42));
EXPECT_THAT(config_or->initial_segment().features().values(),
UnorderedElementsAre("smcp"));
}

TEST_F(AutoSegmenterConfigTest, MaxDepth_AllFeaturesInDefaultFeatures) {
ASSERT_TRUE(cjk_face_) << "NotoSansJP-Regular.ttf not found";

auto freq_list = BuiltInFrequenciesList(*resolver);
ASSERT_TRUE(freq_list.ok()) << freq_list.status();

// Collect only the codepoints covered by the 7 scripts used with Script_CJK.
CodepointSet covered_codepoints;
for (const auto& [ _, scripts ] :
{kEmojiLatinAndSymbols, kGreek, kCyrillic, kUnifiedCJK, kFallback}) {
for (const std::string& script : scripts) {
covered_codepoints.union_set(freq_list->at(script));
}
}
covered_codepoints.intersect(FontHelper::ToCodepointsSet(cjk_face_.get()));

// Font with no uncovered codepoints has 7 scripts + 1 feature-only = 8 table
// keyed segments (no max_depth).
// Now if we also put all non-default features in additional_default_features,
// table keyed segments becomes 7 (still no max_depth).
hb_subset_input_t* input_no_uncovered = hb_subset_input_create_or_fail();
for (hb_codepoint_t cp : covered_codepoints) {
hb_set_add(hb_subset_input_unicode_set(input_no_uncovered), cp);
}
hb_set_invert(hb_subset_input_set(input_no_uncovered,
HB_SUBSET_SETS_LAYOUT_FEATURE_TAG));
hb_face_unique_ptr subset_no_uncovered =
make_hb_face(hb_subset_or_fail(cjk_face_.get(), input_no_uncovered));
hb_subset_input_destroy(input_no_uncovered);

auto non_default_features =
FontHelper::GetNonDefaultFeatureTags(subset_no_uncovered.get());
ASSERT_FALSE(non_default_features.empty());

auto config = AutoSegmenterConfig::GenerateConfig(
subset_no_uncovered.get(), *resolver, "Script_CJK", std::nullopt, IFT, {},
non_default_features);
ASSERT_TRUE(config.ok()) << config.status();
EXPECT_FALSE(config->base_segmentation_plan().has_max_depth());
}

TEST_F(AutoSegmenterConfigTest,
Roboto_UnspecifiedPrimary_UnicodeRange_DefaultCodepoints) {
hb_subset_input_t* input = hb_subset_input_create_or_fail();
ASSERT_TRUE(input);
CodepointSet subset {
0x20, // Shared w/ All
0x301, // Shared w/ Latin + Cyrillic
// Latin
'A',
'B',
'C',
// Greek
0x384,
0x385,
0x394,
// Cyrillic
0x401,
0x402,
0x403,
};
subset.union_into(hb_subset_input_unicode_set(input));
hb_set_clear(
hb_subset_input_set(input, HB_SUBSET_SETS_LAYOUT_FEATURE_TAG));
hb_face_unique_ptr subset_face =
make_hb_face(hb_subset_or_fail(face_.get(), input));
hb_subset_input_destroy(input);

// Moving 0x20 and 0x301 to default_codepoints means they are in initial_segment
// and do not cause Latin + Cyrillic to group or 0x20 to need explicit segment_ids.
auto config_or = AutoSegmenterConfig::GenerateConfig(
subset_face.get(), *resolver, std::nullopt, std::nullopt, UNICODE_RANGE,
{0x20, 0x301});
ASSERT_TRUE(config_or.ok()) << config_or.status();

EXPECT_THAT(config_or->initial_segment().codepoints().values(),
UnorderedElementsAre(0x20, 0x301));
EXPECT_THAT(GetScripts(*config_or),
UnorderedElementsAre(kLatin, kCyrillic, kGreek));
EXPECT_FALSE(GetSegmentIds(*config_or, kLatin.first).has_value());
EXPECT_FALSE(GetSegmentIds(*config_or, kCyrillic.first).has_value());
EXPECT_FALSE(GetSegmentIds(*config_or, kGreek.first).has_value());
}

TEST_F(AutoSegmenterConfigTest,
Roboto_InteractingScripts_UnicodeRange_DefaultCodepoints) {
hb_subset_input_t* input = hb_subset_input_create_or_fail();
ASSERT_TRUE(input);
CodepointSet subset{
// Latin
'A',
'B',
'C',
// Greek (0x386 is a composite glyph using 'A' as a component)
0x384,
0x385,
0x386,
// Cyrillic
0x401,
0x402,
0x403,
};
subset.union_into(hb_subset_input_unicode_set(input));
hb_face_unique_ptr subset_face =
make_hb_face(hb_subset_or_fail(face_.get(), input));
hb_subset_input_destroy(input);

// Placing 'A' into default_codepoints means 'A' is in the initial segment
// (available in all subsets), so Greek and Latin no longer interact on 'A'.
auto config_or = AutoSegmenterConfig::GenerateConfig(
subset_face.get(), *resolver, std::nullopt, std::nullopt, UNICODE_RANGE,
{'A'});
ASSERT_TRUE(config_or.ok()) << config_or.status();

EXPECT_THAT(config_or->initial_segment().codepoints().values(),
UnorderedElementsAre('A'));
EXPECT_THAT(GetScripts(*config_or),
UnorderedElementsAre(kGreek, kLatin, kCyrillic));
}

} // namespace
} // namespace ift::config
3 changes: 2 additions & 1 deletion ift/config/segmenter_config_util.cc
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ std::vector<SubsetDefinition> SegmenterConfigUtil::ConfigToSegments(
} else if (config.generate_feature_segments()) {
uint32_t id = 0;
for (hb_tag_t tag : font_features) {
if (DefaultFeatureTags().contains(tag)) {
if (DefaultFeatureTags().contains(tag) ||
init_segment.feature_tags.contains(tag)) {
continue;
}
SubsetDefinition def;
Expand Down
Loading
Loading