From ea1d01c27b540ba48c831d64088bf9e93e4681e4 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Sun, 1 Nov 2026 08:51:12 +0100 Subject: [PATCH 1/2] Read the symbol table from lang-symbols.tsv instead of parsing lang.def lang.def states nothing about its own shape: the first 60 lines pair a language name with its symbol, the next 10 are self-pairs, and the rest pair a symbol with its text the other way round. Three repositories each hand-parsed that, and the engine now generates lang.def from lang-symbols.tsv, whose kind column says which block a row belongs to. So read the table and drop this repository's parser. A row kind is what selects the rows now, not the symbol's prefix. That matters because 11 string symbols are LISTDEF_ rather than LANG_, so a prefix test silently drops them. Only the text column goes through lang_filter. Over a whole row it would also strip a text's leading space, where lang.def gave every text a line of its own. Proved equivalent by generating res/values-* from the engine's 30 catalogs with both versions: the output is byte-identical apart from the script path the generated header carries. The harness fixture becomes a TSV, keeps every case, and gains two: a row whose kind is not string must not resolve, and a row with a fourth column must stop the run. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Xavier Roche --- tools/build_strings-test.sh | 95 ++++++++++++++++++++----------------- tools/build_strings.sh | 27 ++++++----- 2 files changed, 68 insertions(+), 54 deletions(-) diff --git a/tools/build_strings-test.sh b/tools/build_strings-test.sh index a8ad6bd..895768b 100755 --- a/tools/build_strings-test.sh +++ b/tools/build_strings-test.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # -# Exercise build_strings.sh on catalogs keyed by a lang.def symbol instead of by -# English text. No shipped catalog uses one yet, so this is the only cover. +# Exercise build_strings.sh on catalogs keyed by a symbol instead of by English +# text. No shipped catalog uses one yet, so this is the only cover. set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" @@ -9,36 +9,25 @@ SUT="$HERE/build_strings.sh" tmp="$(mktemp -d)" trap 'rm -rf "$tmp"' EXIT -# lang.def pairs a symbol with its English text, symbol first. This fixture -# copies the real file, including its LANGUAGE_ header rows and two symbols -# sharing one English text, as 27 of its texts do. +# lang-symbols.tsv gives kind, symbol and text, one row each. This fixture +# copies the real file: its language and meta blocks come first, and two string +# symbols share one English text, as 27 of its texts do. The row kinds are +# tab-separated, so write the file with printf rather than a quoted heredoc. mkdir -p "$tmp/lang" "$tmp/res/values" -cat >"$tmp/lang.def" <<'DEF' -Francais -LANGUAGE_1 -LANGUAGE_NAME -LANGUAGE_NAME -LANGUAGE_FILE -LANGUAGE_FILE -LANGUAGE_ISO -LANGUAGE_ISO -LANG_OK -OK -LANG_CANCEL -Cancel -LANG_B3 -Cancel -LANG_START -Start -LISTDEF_1 -One\nTwo -LANG_LITERAL -LANG_OK -LANG_B3 -Cancel -LANG_INFO -Info type: -DEF +printf '%s\t%s\t%s\n' \ + language LANGUAGE_1 Francais \ + meta LANGUAGE_NAME LANGUAGE_NAME \ + meta LANGUAGE_FILE LANGUAGE_FILE \ + meta LANGUAGE_ISO LANGUAGE_ISO \ + string LANG_OK OK \ + string LANG_CANCEL Cancel \ + string LANG_B3 Cancel \ + string LANG_START Start \ + string LISTDEF_1 'One\nTwo' \ + string LANG_LITERAL LANG_OK \ + string LANG_B3 Cancel \ + string LANG_INFO 'Info type:' \ + meta LANG_UNSELECTED Jamais >"$tmp/lang-symbols.tsv" cat >"$tmp/res/values/strings.xml" <<'XML' @@ -107,6 +96,8 @@ LISTDEF_1 Un\nDeux LANG_NOSUCH Jamais +LANG_UNSELECTED +Jamais LANG_OK TexteLitteral LANG_INFO @@ -122,16 +113,23 @@ has "a LISTDEF_ symbol splits on \\n, second line" two Deux # fragment filled, because a fragment belongs to the symbol that carried it. has "a split fragment keeps its symbol's claim" one Un missing "a string no catalog line carries stays empty" start Start -# lang.def gives "LANG_OK" as a string's own English text, so a line keyed that +# The table gives "LANG_OK" as a string's own English text, so a line keyed that # way is text, not the symbol of the same name. -has "a key lang.def knows as text is not read as a symbol" literal TexteLitteral -# lang.def spells this text with a trailing colon and strings.xml does not, so -# the resolved key matches only when lang.def goes through lang_filter as well. -has "lang.def text is cleaned up like a catalog key" info "Type info" +has "a key the table knows as text is not read as a symbol" literal TexteLitteral +# The table spells this text with a trailing colon and strings.xml does not, so +# the resolved key matches only when the text column goes through lang_filter too. +has "table text is cleaned up like a catalog key" info "Type info" if grep -q 'LANG_NOSUCH' "$err"; then - pass "a symbol absent from lang.def is reported" + pass "a symbol absent from the table is reported" +else + bad "a symbol absent from the table is reported" +fi +# The kind column selects the rows, not the symbol's prefix, so a LANG_ symbol +# outside the string block must not resolve. +if grep -q 'LANG_UNSELECTED' "$err"; then + pass "a symbol in a non-string row does not resolve" else - bad "a symbol absent from lang.def is reported" + bad "a symbol in a non-string row does not resolve" fi # A catalog mid-re-key holds both kinds of line for one string. The symbol must @@ -196,14 +194,25 @@ else diff "$tmp/text-keyed.xml" "$out" || true fi -# The symbol-to-text table comes from lang.def, so a missing one must stop the -# run rather than quietly drop every symbol-keyed line. +# A missing table must stop the run rather than quietly drop every symbol-keyed +# line. +rc=0 +bash "$SUT" "$tmp/lang" "$tmp/res" /nonexistent/lang-symbols.tsv >/dev/null 2>&1 || rc=$? +if test "$rc" -eq 1; then + pass "a missing table fails the run" +else + bad "a missing table fails the run (exit $rc)" +fi + +# A real tab in a text splits it into a fourth column, which would translate +# against a truncated text, so the run must stop instead. +printf 'string\tLANG_OK\tOK\tstray\n' >"$tmp/four-columns.tsv" rc=0 -bash "$SUT" "$tmp/lang" "$tmp/res" /nonexistent/lang.def >/dev/null 2>&1 || rc=$? +bash "$SUT" "$tmp/lang" "$tmp/res" "$tmp/four-columns.tsv" >/dev/null 2>&1 || rc=$? if test "$rc" -eq 1; then - pass "a missing lang.def fails the run" + pass "a row with four columns fails the run" else - bad "a missing lang.def fails the run (exit $rc)" + bad "a row with four columns fails the run (exit $rc)" fi test "$fail" -eq 0 && echo "build_strings-test: all cases pass" diff --git a/tools/build_strings.sh b/tools/build_strings.sh index a17d4d3..9e3804f 100755 --- a/tools/build_strings.sh +++ b/tools/build_strings.sh @@ -3,8 +3,8 @@ # Convert httrack lang/*;txt files into android xml resources -# usage: $0 path-to/lang path-to/res [path-to/lang.def] -# lang.def defaults to the one next to the lang directory +# usage: $0 path-to/lang path-to/res [path-to/lang-symbols.tsv] +# lang-symbols.tsv defaults to the one next to the lang directory # example: # rm -rf res/values-?? res/values-??-r?? # language dirs only: values-* also takes the # # hand-kept values-v34, values-night and the rest @@ -41,21 +41,26 @@ function put_str { test -d "$1" || ! echo "lang directory not found: $1" || exit 1 test -d "$2" || ! echo "res directory not found: $2" || exit 1 -langdef="${3:-$1/../lang.def}" -test -f "$langdef" || ! echo "lang.def not found: $langdef (give its path as the 3rd argument)" || exit 1 +symtab="${3:-$1/../lang-symbols.tsv}" +test -f "$symtab" || ! echo "lang-symbols.tsv not found: $symtab (give its path as the 3rd argument)" || exit 1 -# lang.def pairs each string's LANG_ symbol with its English text, symbol first. +# A real tab in a text would shift it into a fourth column, so refuse the file. +awk -F'\t' '!/^#/ && NF != 3 { print FILENAME ":" FNR ": expected 3 tab-separated columns" > "/dev/stderr"; bad = 1 } END { exit bad }' "$symtab" || exit 1 + +# The string rows are the translatable ones, in file order, so a text several +# symbols share resolves to the last of them. # sym_text resolves a symbol, english_text answers "is this key a known text?". +# lang_filter sees the text column alone, because over a whole row it would also +# strip a text's leading space. declare -A sym_text declare -A english_text -while IFS= read -r dk && IFS= read -r dv; do +while IFS=$'\t' read -r dk dv; do if test -n "$dk" -a -n "$dv"; then english_text["$dv"]=1 -case "$dk" in -LANG_*|LISTDEF_*) sym_text["$dk"]="$dv" ;; -esac +sym_text["$dk"]="$dv" fi -done < <(lang_filter < "$langdef") +done < <(paste <(awk -F'\t' '$1 == "string" { print $2 }' "$symtab") \ + <(awk -F'\t' '$1 == "string" { print $3 }' "$symtab" | lang_filter)) for i in $1/*.txt ; do echo -n "processing $i .. " >&2 @@ -86,7 +91,7 @@ LANG_*|LISTDEF_*) if test -z "${english_text[$rk]}"; then if test -z "${sym_text[$rk]}"; then # No resource is keyed by a bare symbol, so drop the line rather than store it. -echo "warning: $i: no lang.def entry for key $rk, line skipped" >&2 +echo "warning: $i: no string row for symbol $rk, line skipped" >&2 continue fi symbol="$rk" From 3e5f004892a984638e10a9aef69206c8cb9b0cba Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Sun, 1 Nov 2026 09:07:09 +0100 Subject: [PATCH 2/2] Drop the guards the engine's own generator already makes The review killed both guards I had added, and it was right. The engine's tools/gen-lang-def.py exits on a row without three columns, on a text that differs from its strip(), on a duplicate symbol, and on a meta row whose text does not repeat its symbol. We consume a committed blob that has already passed those checks. The column-count check was also stricter than the producer, which allows a blank line, so it would have refused a legal future table. The paste of two awks, which kept lang_filter off the symbol column, defended a padded text the producer forbids and took on an alignment risk instead: a later lang_filter rule that dropped a line would misalign every symbol from its text. One awk emitting both columns cannot. The case for "the kind column selects the rows, not the prefix" needed a meta row carrying a LANG_ symbol, which the producer forbids too. On any legal table a prefix test and a kind test agree, because language and meta symbols are all LANGUAGE_*, so that case was testing against a world that cannot exist. What the harness checks instead is the pinned table itself: it resolves that table's own first string symbol. Nothing else here reads it, so a pin losing lang-symbols.tsv would otherwise keep CI green. Verified by mutation: a crafted engine directory without the table reports it rather than skipping. Also moves the three not-found messages to stderr, which is where an error belongs and what lets the missing-table case assert the message rather than only the exit code. Still byte-identical over the engine's 30 catalogs. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Xavier Roche --- tools/build_strings-test.sh | 63 ++++++++++++++++++++++--------------- tools/build_strings.sh | 22 +++++-------- 2 files changed, 45 insertions(+), 40 deletions(-) diff --git a/tools/build_strings-test.sh b/tools/build_strings-test.sh index 895768b..980ae3c 100755 --- a/tools/build_strings-test.sh +++ b/tools/build_strings-test.sh @@ -9,11 +9,11 @@ SUT="$HERE/build_strings.sh" tmp="$(mktemp -d)" trap 'rm -rf "$tmp"' EXIT -# lang-symbols.tsv gives kind, symbol and text, one row each. This fixture -# copies the real file: its language and meta blocks come first, and two string -# symbols share one English text, as 27 of its texts do. The row kinds are +# lang-symbols.tsv gives kind, symbol and text, one row each. Two string symbols +# share one English text here, as 27 of the real texts do. The columns are # tab-separated, so write the file with printf rather than a quoted heredoc. mkdir -p "$tmp/lang" "$tmp/res/values" +printf '# a comment row, as the real file carries five of\n' >"$tmp/lang-symbols.tsv" printf '%s\t%s\t%s\n' \ language LANGUAGE_1 Francais \ meta LANGUAGE_NAME LANGUAGE_NAME \ @@ -26,8 +26,7 @@ printf '%s\t%s\t%s\n' \ string LISTDEF_1 'One\nTwo' \ string LANG_LITERAL LANG_OK \ string LANG_B3 Cancel \ - string LANG_INFO 'Info type:' \ - meta LANG_UNSELECTED Jamais >"$tmp/lang-symbols.tsv" + string LANG_INFO 'Info type:' >>"$tmp/lang-symbols.tsv" cat >"$tmp/res/values/strings.xml" <<'XML' @@ -96,8 +95,6 @@ LISTDEF_1 Un\nDeux LANG_NOSUCH Jamais -LANG_UNSELECTED -Jamais LANG_OK TexteLitteral LANG_INFO @@ -124,13 +121,6 @@ if grep -q 'LANG_NOSUCH' "$err"; then else bad "a symbol absent from the table is reported" fi -# The kind column selects the rows, not the symbol's prefix, so a LANG_ symbol -# outside the string block must not resolve. -if grep -q 'LANG_UNSELECTED' "$err"; then - pass "a symbol in a non-string row does not resolve" -else - bad "a symbol in a non-string row does not resolve" -fi # A catalog mid-re-key holds both kinds of line for one string. The symbol must # win either way round, or the result would depend on row order. @@ -197,22 +187,43 @@ fi # A missing table must stop the run rather than quietly drop every symbol-keyed # line. rc=0 -bash "$SUT" "$tmp/lang" "$tmp/res" /nonexistent/lang-symbols.tsv >/dev/null 2>&1 || rc=$? -if test "$rc" -eq 1; then - pass "a missing table fails the run" +bash "$SUT" "$tmp/lang" "$tmp/res" /nonexistent/lang-symbols.tsv >/dev/null 2>"$err" || rc=$? +if test "$rc" -eq 1 && grep -q 'lang-symbols.tsv not found' "$err"; then + pass "a missing table fails the run and says so" else - bad "a missing table fails the run (exit $rc)" + bad "a missing table fails the run and says so (exit $rc)" + cat "$err" || true fi -# A real tab in a text splits it into a fourth column, which would translate -# against a truncated text, so the run must stop instead. -printf 'string\tLANG_OK\tOK\tstray\n' >"$tmp/four-columns.tsv" -rc=0 -bash "$SUT" "$tmp/lang" "$tmp/res" "$tmp/four-columns.tsv" >/dev/null 2>&1 || rc=$? -if test "$rc" -eq 1; then - pass "a row with four columns fails the run" +# Every case above runs on a fixture, so none would notice the pinned engine losing +# lang-symbols.tsv or changing its shape. Resolve one of the real table's own symbols. +engine="$HERE/../app/src/main/jni/httrack" +if ! test -d "$engine/lang"; then + echo "SKIP: no engine submodule, so the pinned table is unchecked" >&2 +elif ! test -f "$engine/lang-symbols.tsv"; then + bad "the pinned engine carries lang-symbols.tsv" else - bad "a row with four columns fails the run (exit $rc)" + table="$engine/lang-symbols.tsv" + sym="$(awk -F'\t' '$1 == "string" { print $2; exit }' "$table")" + text="$(awk -F'\t' '$1 == "string" { print $3; exit }' "$table")" + # A glob, not find piped to head, which pipefail turns into a failure. + catalogs=("$engine"/lang/*.txt) + catalog="${catalogs[0]}" + pin="$tmp/pin" + mkdir -p "$pin/lang" "$pin/res/values" + printf '\n\n %s\n\n' \ + "$text" >"$pin/res/values/strings.xml" + { + printf 'LANGUAGE_NAME\nFrancais\nLANGUAGE_FILE\nFrancais\nLANGUAGE_ISO\nfr\n' + printf '%s\nTraduit\n' "$sym" + } >"$pin/lang/$(basename "$catalog")" + bash "$SUT" "$pin/lang" "$pin/res" "$table" >/dev/null 2>&1 + if grep -qF 'Traduit' "$pin/res/values-fr/string.xml"; then + pass "the pinned table resolves its own first symbol ($sym)" + else + bad "the pinned table resolves its own first symbol ($sym)" + grep -F 'name="probe"' "$pin/res/values-fr/string.xml" || true + fi fi test "$fail" -eq 0 && echo "build_strings-test: all cases pass" diff --git a/tools/build_strings.sh b/tools/build_strings.sh index 9e3804f..b7e593e 100755 --- a/tools/build_strings.sh +++ b/tools/build_strings.sh @@ -8,14 +8,14 @@ # example: # rm -rf res/values-?? res/values-??-r?? # language dirs only: values-* also takes the # # hand-kept values-v34, values-night and the rest -# bash tools/build_strings.sh /temp/httrack-3.47.21/lang res/ +# bash tools/build_strings.sh app/src/main/jni/httrack/lang app/src/main/res/ function norm_key { echo "$1" | sed -e 's/[^a-zA-Z0-9]/_/g' } -# Clean up catalog text. lang.def goes through it too, so a symbol resolves to -# the same key as its English text. +# Clean up catalog text. A table text goes through it too, so a symbol resolves +# to the same key as its English text. function lang_filter { tr -d '\r' \ | sed -e 's/\\r//g' \ @@ -26,7 +26,7 @@ function lang_filter { | sed -e "s/(C)/\©/g" } -# Store one translation, keyed by the lang.def symbol in $3, or by its text when +# Store one translation, keyed by the table symbol in $3, or by its text when # $3 is empty. A symbol line beats a text line, and is skipped only when the same # symbol already set this key, so several symbols sharing a text keep the last. function put_str { @@ -39,19 +39,14 @@ function put_str { arr["$1"]="$2" } -test -d "$1" || ! echo "lang directory not found: $1" || exit 1 -test -d "$2" || ! echo "res directory not found: $2" || exit 1 +test -d "$1" || ! echo "lang directory not found: $1" >&2 || exit 1 +test -d "$2" || ! echo "res directory not found: $2" >&2 || exit 1 symtab="${3:-$1/../lang-symbols.tsv}" -test -f "$symtab" || ! echo "lang-symbols.tsv not found: $symtab (give its path as the 3rd argument)" || exit 1 - -# A real tab in a text would shift it into a fourth column, so refuse the file. -awk -F'\t' '!/^#/ && NF != 3 { print FILENAME ":" FNR ": expected 3 tab-separated columns" > "/dev/stderr"; bad = 1 } END { exit bad }' "$symtab" || exit 1 +test -f "$symtab" || ! echo "lang-symbols.tsv not found: $symtab (give its path as the 3rd argument)" >&2 || exit 1 # The string rows are the translatable ones, in file order, so a text several # symbols share resolves to the last of them. # sym_text resolves a symbol, english_text answers "is this key a known text?". -# lang_filter sees the text column alone, because over a whole row it would also -# strip a text's leading space. declare -A sym_text declare -A english_text while IFS=$'\t' read -r dk dv; do @@ -59,8 +54,7 @@ if test -n "$dk" -a -n "$dv"; then english_text["$dv"]=1 sym_text["$dk"]="$dv" fi -done < <(paste <(awk -F'\t' '$1 == "string" { print $2 }' "$symtab") \ - <(awk -F'\t' '$1 == "string" { print $3 }' "$symtab" | lang_filter)) +done < <(awk -F'\t' '$1 == "string" { print $2 "\t" $3 }' "$symtab" | lang_filter) for i in $1/*.txt ; do echo -n "processing $i .. " >&2