diff --git a/release/ait/ait.mnw.mon/HISTORY.md b/release/ait/ait.mnw.mon/HISTORY.md
index 4de705a2..a87e0dc7 100644
--- a/release/ait/ait.mnw.mon/HISTORY.md
+++ b/release/ait/ait.mnw.mon/HISTORY.md
@@ -1,6 +1,10 @@
Mon dictionary Change History
====================
+1.2 (2026-10-27)
+----------------
+* Improved wordbreaker handling of edge cases & duplicated words in context range
+
1.1 (2024-09-16)
----------------
* Rebuild with 17.0.329 compiler to address missing low-frequency words
diff --git a/release/ait/ait.mnw.mon/ait.mnw.mon.kpj b/release/ait/ait.mnw.mon/ait.mnw.mon.kpj
index 18be241b..e4a99571 100644
--- a/release/ait/ait.mnw.mon/ait.mnw.mon.kpj
+++ b/release/ait/ait.mnw.mon/ait.mnw.mon.kpj
@@ -12,19 +12,19 @@
id_a0a76ecb3384b957c34b9a90d62d20ea
ait.mnw.mon.model.ts
source\ait.mnw.mon.model.ts
- 1.0
+ 1.2
.ts
id_e06131155a3cd6fa86531f339ad6dc6c
ait.mnw.mon.model.kps
source\ait.mnw.mon.model.kps
- 1.0
+ 1.2
.kps
Mon dictionary Lexical Model
© 2021 Anonta Mon
- 1.0
+ 1.2
diff --git a/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.kps b/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.kps
index d15a57b7..f23dcc41 100644
--- a/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.kps
+++ b/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.kps
@@ -19,7 +19,7 @@
Mon dictionary Lexical Model
© 2021 Anonta Mon
Anonta Mon
- 1.1
+ 1.2
Mon predictive text lexical model.
diff --git a/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.ts b/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.ts
index 234a223c..29fcc904 100644
--- a/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.ts
+++ b/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.ts
@@ -6,19 +6,57 @@
const source: LexicalModelSource = {
format: 'trie-1.0',
- wordBreaker: 'default',
sources: ['wordlist.tsv'],
languageUsesCasing: true,
- wordBreaker: function(str: string) {
- return str.split(/\s|\u200b/).map(function(token) {
+ wordBreaker: function(str: string) {
+ let latestIndex = 0;
+
+ let emptyToken = {
+ left: str.length,
+ start: str.length,
+ right: str.length,
+ end: str.length,
+ length: 0,
+ text: ''
+ };
+
+ let tokens = str.split(/\s|\u200b/);
+ let finalizedTokens = tokens.map(function(token, index) {
+ // Empty tokens need special handling.
+ if(token == '') {
+ // If it's in the middle of the text, remove it. We probably
+ // were given two adjacent spaces.
+ if(index != tokens.length - 1) {
+ return null;
+ } else {
+ // If it's at the end of the text, we're in a "new word"
+ // position; make sure we indicate this well!
+ return emptyToken;
+ }
+ }
+
+ // Always check for the position after the end of the previous token.
+ // The sequence should always be from start to end.
+ const start = str.indexOf(token, latestIndex);
+ latestIndex = start + token.length;
return {
- left: str.indexOf(token),
- start: str.indexOf(token),
- right: str.indexOf(token) + token.length,
- end: str.indexOf(token) + token.length,
+ left: start,
+ start: start,
+ right: start + token.length,
+ end: start + token.length,
+ length: token.length,
text: token
- }
- });
+ };
+ }).filter((t) => !!t); // remove any `null` entries (for middle-of-text empty strings)
+
+ // If a new word should be starting at the current caret position,
+ // create an empty '' token to indicate this.
+ let tailToken = finalizedTokens[finalizedTokens.length - 1];
+ if(!tailToken || tailToken.end != str.length) {
+ finalizedTokens.push(emptyToken);
+ }
+
+ return finalizedTokens;
},
punctuation: {
insertAfterWord: ""