fix: improve ait.mnw.mon wordbreaking

release/ait/ait.mnw.mon/HISTORY.md

@@ -1,6 +1,10 @@
 Mon dictionary Change History
 ====================
 
+1.2 (2026-10-27)
+----------------
+* Improved wordbreaker handling of edge cases & duplicated words in context range
+
 1.1 (2024-09-16)
 ----------------
 * Rebuild with 17.0.329 compiler to address missing low-frequency words

release/ait/ait.mnw.mon/ait.mnw.mon.kpj

@@ -12,19 +12,19 @@
       <ID>id_a0a76ecb3384b957c34b9a90d62d20ea</ID>
       <Filename>ait.mnw.mon.model.ts</Filename>
       <Filepath>source\ait.mnw.mon.model.ts</Filepath>
-      <FileVersion>1.0</FileVersion>
+      <FileVersion>1.2</FileVersion>
       <FileType>.ts</FileType>
     </File>
     <File>
       <ID>id_e06131155a3cd6fa86531f339ad6dc6c</ID>
       <Filename>ait.mnw.mon.model.kps</Filename>
       <Filepath>source\ait.mnw.mon.model.kps</Filepath>
-      <FileVersion>1.0</FileVersion>
+      <FileVersion>1.2</FileVersion>
       <FileType>.kps</FileType>
       <Details>
         <Name>Mon dictionary Lexical Model</Name>
         <Copyright>© 2021 Anonta Mon</Copyright>
-        <Version>1.0</Version>
+        <Version>1.2</Version>
       </Details>
     </File>
     <File>

release/ait/ait.mnw.mon/source/ait.mnw.mon.model.kps

@@ -19,7 +19,7 @@
     <Name URL="">Mon dictionary Lexical Model</Name>
     <Copyright URL="">© 2021 Anonta Mon</Copyright>
     <Author URL="mailto:monfonts@gmail.com">Anonta Mon</Author>
-    <Version URL="">1.1</Version>
+    <Version URL="">1.2</Version>
     <Description>Mon predictive text lexical model.</Description>
   </Info>
   <Files>

release/ait/ait.mnw.mon/source/ait.mnw.mon.model.ts

@@ -6,19 +6,57 @@
 
 const source: LexicalModelSource = {
   format: 'trie-1.0',
-  wordBreaker: 'default',
   sources: ['wordlist.tsv'],
   languageUsesCasing: true,
-    wordBreaker: function(str: string) {
-    return str.split(/\s|\u200b/).map(function(token) {
+  wordBreaker: function(str: string) {
+    let latestIndex = 0;
+
+    let emptyToken = {
+      left: str.length,
+      start: str.length,
+      right: str.length,
+      end: str.length,
+      length: 0,
+      text: ''
+    };
+
+    let tokens = str.split(/\s|\u200b/);
+    let finalizedTokens = tokens.map(function(token, index) {
+      // Empty tokens need special handling.
+      if(token == '') {
+        // If it's in the middle of the text, remove it.  We probably
+        // were given two adjacent spaces.
+        if(index != tokens.length - 1) {
+          return null;
+        } else {
+          // If it's at the end of the text, we're in a "new word"
+          // position; make sure we indicate this well!
+          return emptyToken;
+        }
+      }
+
+      // Always check for the position after the end of the previous token.
+      // The sequence should always be from start to end.
+      const start = str.indexOf(token, latestIndex);
+      latestIndex = start + token.length;
       return {
-        left: str.indexOf(token),
-        start: str.indexOf(token),
-        right: str.indexOf(token) + token.length,
-        end: str.indexOf(token) + token.length,
+        left: start,
+        start: start,
+        right: start + token.length,
+        end: start + token.length,
+        length: token.length,
         text: token
-      }
-    });
+      };
+    }).filter((t) => !!t); // remove any `null` entries (for middle-of-text empty strings)
+
+    // If a new word should be starting at the current caret position,
+    // create an empty '' token to indicate this.
+    let tailToken = finalizedTokens[finalizedTokens.length - 1];
+    if(!tailToken || tailToken.end != str.length) {
+      finalizedTokens.push(emptyToken);
+    }
+
+    return finalizedTokens;
   },
   punctuation: {
     insertAfterWord: ""