Skip to content

Read the symbol table from lang-symbols.tsv instead of parsing lang.def #270

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xroche merged 2 commits into from
Nov 1, 2026
Merged

Read the symbol table from lang-symbols.tsv instead of parsing lang.def #270

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ea1d01c
Read the symbol table from lang-symbols.tsv instead of parsing lang.def
xroche Nov 1, 2026
3e5f004
Drop the guards the engine's own generator already makes
xroche Nov 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 64 additions & 44 deletions tools/build_strings-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,44 +1,32 @@
#!/usr/bin/env bash
#
# Exercise build_strings.sh on catalogs keyed by a lang.def symbol instead of by
# English text. No shipped catalog uses one yet, so this is the only cover.
# Exercise build_strings.sh on catalogs keyed by a symbol instead of by English
# text. No shipped catalog uses one yet, so this is the only cover.
set -euo pipefail

HERE="$(cd "$(dirname "$0")" && pwd)"
SUT="$HERE/build_strings.sh"
tmp="$(mktemp -d)"
trap 'rm -rf "$tmp"' EXIT

# lang.def pairs a symbol with its English text, symbol first. This fixture
# copies the real file, including its LANGUAGE_ header rows and two symbols
# sharing one English text, as 27 of its texts do.
# lang-symbols.tsv gives kind, symbol and text, one row each. Two string symbols
# share one English text here, as 27 of the real texts do. The columns are
# tab-separated, so write the file with printf rather than a quoted heredoc.
mkdir -p "$tmp/lang" "$tmp/res/values"
cat >"$tmp/lang.def" <<'DEF'
Francais
LANGUAGE_1
LANGUAGE_NAME
LANGUAGE_NAME
LANGUAGE_FILE
LANGUAGE_FILE
LANGUAGE_ISO
LANGUAGE_ISO
LANG_OK
OK
LANG_CANCEL
Cancel
LANG_B3
Cancel
LANG_START
Start
LISTDEF_1
One\nTwo
LANG_LITERAL
LANG_OK
LANG_B3
Cancel
LANG_INFO
Info type:
DEF
printf '# a comment row, as the real file carries five of\n' >"$tmp/lang-symbols.tsv"
printf '%s\t%s\t%s\n' \
language LANGUAGE_1 Francais \
meta LANGUAGE_NAME LANGUAGE_NAME \
meta LANGUAGE_FILE LANGUAGE_FILE \
meta LANGUAGE_ISO LANGUAGE_ISO \
string LANG_OK OK \
string LANG_CANCEL Cancel \
string LANG_B3 Cancel \
string LANG_START Start \
string LISTDEF_1 'One\nTwo' \
string LANG_LITERAL LANG_OK \
string LANG_B3 Cancel \
string LANG_INFO 'Info type:' >>"$tmp/lang-symbols.tsv"

cat >"$tmp/res/values/strings.xml" <<'XML'
<?xml version="1.0" encoding="utf-8"?>
Expand Down Expand Up @@ -122,16 +110,16 @@ has "a LISTDEF_ symbol splits on \\n, second line" two Deux
# fragment filled, because a fragment belongs to the symbol that carried it.
has "a split fragment keeps its symbol's claim" one Un
missing "a string no catalog line carries stays empty" start Start
# lang.def gives "LANG_OK" as a string's own English text, so a line keyed that
# The table gives "LANG_OK" as a string's own English text, so a line keyed that
# way is text, not the symbol of the same name.
has "a key lang.def knows as text is not read as a symbol" literal TexteLitteral
# lang.def spells this text with a trailing colon and strings.xml does not, so
# the resolved key matches only when lang.def goes through lang_filter as well.
has "lang.def text is cleaned up like a catalog key" info "Type info"
has "a key the table knows as text is not read as a symbol" literal TexteLitteral
# The table spells this text with a trailing colon and strings.xml does not, so
# the resolved key matches only when the text column goes through lang_filter too.
has "table text is cleaned up like a catalog key" info "Type info"
if grep -q 'LANG_NOSUCH' "$err"; then
pass "a symbol absent from lang.def is reported"
pass "a symbol absent from the table is reported"
else
bad "a symbol absent from lang.def is reported"
bad "a symbol absent from the table is reported"
fi

# A catalog mid-re-key holds both kinds of line for one string. The symbol must
Expand Down Expand Up @@ -196,14 +184,46 @@ else
diff "$tmp/text-keyed.xml" "$out" || true
fi

# The symbol-to-text table comes from lang.def, so a missing one must stop the
# run rather than quietly drop every symbol-keyed line.
# A missing table must stop the run rather than quietly drop every symbol-keyed
# line.
rc=0
bash "$SUT" "$tmp/lang" "$tmp/res" /nonexistent/lang.def >/dev/null 2>&1 || rc=$?
if test "$rc" -eq 1; then
pass "a missing lang.def fails the run"
bash "$SUT" "$tmp/lang" "$tmp/res" /nonexistent/lang-symbols.tsv >/dev/null 2>"$err" || rc=$?
if test "$rc" -eq 1 && grep -q 'lang-symbols.tsv not found' "$err"; then
pass "a missing table fails the run and says so"
else
bad "a missing lang.def fails the run (exit $rc)"
bad "a missing table fails the run and says so (exit $rc)"
cat "$err" || true
fi

# Every case above runs on a fixture, so none would notice the pinned engine losing
# lang-symbols.tsv or changing its shape. Resolve one of the real table's own symbols.
engine="$HERE/../app/src/main/jni/httrack"
if ! test -d "$engine/lang"; then
echo "SKIP: no engine submodule, so the pinned table is unchecked" >&2
elif ! test -f "$engine/lang-symbols.tsv"; then
bad "the pinned engine carries lang-symbols.tsv"
else
table="$engine/lang-symbols.tsv"
sym="$(awk -F'\t' '$1 == "string" { print $2; exit }' "$table")"
text="$(awk -F'\t' '$1 == "string" { print $3; exit }' "$table")"
# A glob, not find piped to head, which pipefail turns into a failure.
catalogs=("$engine"/lang/*.txt)
catalog="${catalogs[0]}"
pin="$tmp/pin"
mkdir -p "$pin/lang" "$pin/res/values"
printf '<?xml version="1.0" encoding="utf-8"?>\n<resources>\n <string name="probe">%s</string>\n</resources>\n' \
"$text" >"$pin/res/values/strings.xml"
{
printf 'LANGUAGE_NAME\nFrancais\nLANGUAGE_FILE\nFrancais\nLANGUAGE_ISO\nfr\n'
printf '%s\nTraduit\n' "$sym"
} >"$pin/lang/$(basename "$catalog")"
bash "$SUT" "$pin/lang" "$pin/res" "$table" >/dev/null 2>&1
if grep -qF '<string name="probe">Traduit</string>' "$pin/res/values-fr/string.xml"; then
pass "the pinned table resolves its own first symbol ($sym)"
else
bad "the pinned table resolves its own first symbol ($sym)"
grep -F 'name="probe"' "$pin/res/values-fr/string.xml" || true
fi
fi

test "$fail" -eq 0 && echo "build_strings-test: all cases pass"
Expand Down
33 changes: 16 additions & 17 deletions tools/build_strings.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,19 @@

# Convert httrack lang/*;txt files into android xml resources

# usage: $0 path-to/lang path-to/res [path-to/lang.def]
# lang.def defaults to the one next to the lang directory
# usage: $0 path-to/lang path-to/res [path-to/lang-symbols.tsv]
# lang-symbols.tsv defaults to the one next to the lang directory
# example:
# rm -rf res/values-?? res/values-??-r?? # language dirs only: values-* also takes the
# # hand-kept values-v34, values-night and the rest
# bash tools/build_strings.sh /temp/httrack-3.47.21/lang res/
# bash tools/build_strings.sh app/src/main/jni/httrack/lang app/src/main/res/

function norm_key {
echo "$1" | sed -e 's/[^a-zA-Z0-9]/_/g'
}

# Clean up catalog text. lang.def goes through it too, so a symbol resolves to
# the same key as its English text.
# Clean up catalog text. A table text goes through it too, so a symbol resolves
# to the same key as its English text.
function lang_filter {
tr -d '\r' \
| sed -e 's/\\r//g' \
Expand All @@ -26,7 +26,7 @@ function lang_filter {
| sed -e "s/(C)/\&#169;/g"
}

# Store one translation, keyed by the lang.def symbol in $3, or by its text when
# Store one translation, keyed by the table symbol in $3, or by its text when
# $3 is empty. A symbol line beats a text line, and is skipped only when the same
# symbol already set this key, so several symbols sharing a text keep the last.
function put_str {
Expand All @@ -39,23 +39,22 @@ function put_str {
arr["$1"]="$2"
}

test -d "$1" || ! echo "lang directory not found: $1" || exit 1
test -d "$2" || ! echo "res directory not found: $2" || exit 1
langdef="${3:-$1/../lang.def}"
test -f "$langdef" || ! echo "lang.def not found: $langdef (give its path as the 3rd argument)" || exit 1
test -d "$1" || ! echo "lang directory not found: $1" >&2 || exit 1
test -d "$2" || ! echo "res directory not found: $2" >&2 || exit 1
symtab="${3:-$1/../lang-symbols.tsv}"
test -f "$symtab" || ! echo "lang-symbols.tsv not found: $symtab (give its path as the 3rd argument)" >&2 || exit 1

# lang.def pairs each string's LANG_ symbol with its English text, symbol first.
# The string rows are the translatable ones, in file order, so a text several
# symbols share resolves to the last of them.
# sym_text resolves a symbol, english_text answers "is this key a known text?".
declare -A sym_text
declare -A english_text
while IFS= read -r dk && IFS= read -r dv; do
while IFS=$'\t' read -r dk dv; do
if test -n "$dk" -a -n "$dv"; then
english_text["$dv"]=1
case "$dk" in
LANG_*|LISTDEF_*) sym_text["$dk"]="$dv" ;;
esac
sym_text["$dk"]="$dv"
fi
done < <(lang_filter < "$langdef")
done < <(awk -F'\t' '$1 == "string" { print $2 "\t" $3 }' "$symtab" | lang_filter)

for i in $1/*.txt ; do
echo -n "processing $i .. " >&2
Expand Down Expand Up @@ -86,7 +85,7 @@ LANG_*|LISTDEF_*)
if test -z "${english_text[$rk]}"; then
if test -z "${sym_text[$rk]}"; then
# No resource is keyed by a bare symbol, so drop the line rather than store it.
echo "warning: $i: no lang.def entry for key $rk, line skipped" >&2
echo "warning: $i: no string row for symbol $rk, line skipped" >&2
continue
fi
symbol="$rk"
Expand Down
Loading