From a962c8fb7e4ee9384457a776787c502efbe327e5 Mon Sep 17 00:00:00 2001 From: autoresearch Date: Sat, 10 Oct 2026 04:40:36 -0400 Subject: [PATCH 1/2] CONTROL_TOKENS_PATTERN, uppercase; the misspelled name stays as an alias words-segmentation <= 0.0.6 imports CONTROl_TOKENS_PATTERN, so it remains an alias of the corrected name. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015Xm62qoeaTunFSU5GYCQ1f --- tests/test_control.py | 8 +++++--- utf8_tokenizer/control.py | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_control.py b/tests/test_control.py index 9147150..980828e 100644 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -43,11 +43,13 @@ def test_control_visualization(): def test_control_tokens_pattern_matches_non_whitespace_controls(): - """CONTROl_TOKENS_PATTERN is public API: words-segmentation builds its regexes from it.""" + """CONTROL_TOKENS_PATTERN is public API: words-segmentation builds its regexes from it (older versions from its + misspelled former name, kept as an alias).""" import re - from utf8_tokenizer.control import CONTROl_TOKENS_PATTERN + from utf8_tokenizer.control import CONTROL_TOKENS_PATTERN, CONTROl_TOKENS_PATTERN - pattern = re.compile(f"[{CONTROl_TOKENS_PATTERN}]") + assert CONTROl_TOKENS_PATTERN == CONTROL_TOKENS_PATTERN + pattern = re.compile(f"[{CONTROL_TOKENS_PATTERN}]") assert [c for c in map(chr, range(128)) if pattern.fullmatch(c)] == \ [*map(chr, range(0x01, 0x09)), *map(chr, range(0x0E, 0x20)), "\x7f"] diff --git a/utf8_tokenizer/control.py b/utf8_tokenizer/control.py index 95879af..12f30a9 100644 --- a/utf8_tokenizer/control.py +++ b/utf8_tokenizer/control.py @@ -37,7 +37,8 @@ class ControlTokens: # Public: a regex character range of the non-whitespace control tokens (used by words-segmentation) -CONTROl_TOKENS_PATTERN = "\x01-\x08\x0e-\x1f\x7f" +CONTROL_TOKENS_PATTERN = "\x01-\x08\x0e-\x1f\x7f" +CONTROl_TOKENS_PATTERN = CONTROL_TOKENS_PATTERN # Its former (misspelled) name, for words-segmentation <= 0.0.6 CONTROL_WHITESPACES = { ControlTokens.HorizontalTab, From 5ef6557fbb8c84fde509c86e3a24b712837c05fd Mon Sep 17 00:00:00 2001 From: autoresearch Date: Sat, 10 Oct 2026 04:42:07 -0400 Subject: [PATCH 2/2] No alias: words-segmentation switches to CONTROL_TOKENS_PATTERN Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015Xm62qoeaTunFSU5GYCQ1f --- tests/test_control.py | 6 ++---- utf8_tokenizer/control.py | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_control.py b/tests/test_control.py index 980828e..5a65e47 100644 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -43,13 +43,11 @@ def test_control_visualization(): def test_control_tokens_pattern_matches_non_whitespace_controls(): - """CONTROL_TOKENS_PATTERN is public API: words-segmentation builds its regexes from it (older versions from its - misspelled former name, kept as an alias).""" + """CONTROL_TOKENS_PATTERN is public API: words-segmentation builds its regexes from it.""" import re - from utf8_tokenizer.control import CONTROL_TOKENS_PATTERN, CONTROl_TOKENS_PATTERN + from utf8_tokenizer.control import CONTROL_TOKENS_PATTERN - assert CONTROl_TOKENS_PATTERN == CONTROL_TOKENS_PATTERN pattern = re.compile(f"[{CONTROL_TOKENS_PATTERN}]") assert [c for c in map(chr, range(128)) if pattern.fullmatch(c)] == \ [*map(chr, range(0x01, 0x09)), *map(chr, range(0x0E, 0x20)), "\x7f"] diff --git a/utf8_tokenizer/control.py b/utf8_tokenizer/control.py index 12f30a9..df14f8a 100644 --- a/utf8_tokenizer/control.py +++ b/utf8_tokenizer/control.py @@ -38,7 +38,6 @@ class ControlTokens: # Public: a regex character range of the non-whitespace control tokens (used by words-segmentation) CONTROL_TOKENS_PATTERN = "\x01-\x08\x0e-\x1f\x7f" -CONTROl_TOKENS_PATTERN = CONTROL_TOKENS_PATTERN # Its former (misspelled) name, for words-segmentation <= 0.0.6 CONTROL_WHITESPACES = { ControlTokens.HorizontalTab,