diff --git a/release/ait/ait.mnw.mon/HISTORY.md b/release/ait/ait.mnw.mon/HISTORY.md index 4de705a2..a87e0dc7 100644 --- a/release/ait/ait.mnw.mon/HISTORY.md +++ b/release/ait/ait.mnw.mon/HISTORY.md @@ -1,6 +1,10 @@ Mon dictionary Change History ==================== +1.2 (2026-10-27) +---------------- +* Improved wordbreaker handling of edge cases & duplicated words in context range + 1.1 (2024-09-16) ---------------- * Rebuild with 17.0.329 compiler to address missing low-frequency words diff --git a/release/ait/ait.mnw.mon/ait.mnw.mon.kpj b/release/ait/ait.mnw.mon/ait.mnw.mon.kpj index 18be241b..e4a99571 100644 --- a/release/ait/ait.mnw.mon/ait.mnw.mon.kpj +++ b/release/ait/ait.mnw.mon/ait.mnw.mon.kpj @@ -12,19 +12,19 @@ id_a0a76ecb3384b957c34b9a90d62d20ea ait.mnw.mon.model.ts source\ait.mnw.mon.model.ts - 1.0 + 1.2 .ts id_e06131155a3cd6fa86531f339ad6dc6c ait.mnw.mon.model.kps source\ait.mnw.mon.model.kps - 1.0 + 1.2 .kps
Mon dictionary Lexical Model © 2021 Anonta Mon - 1.0 + 1.2
diff --git a/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.kps b/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.kps index d15a57b7..f23dcc41 100644 --- a/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.kps +++ b/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.kps @@ -19,7 +19,7 @@ Mon dictionary Lexical Model © 2021 Anonta Mon Anonta Mon - 1.1 + 1.2 Mon predictive text lexical model. diff --git a/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.ts b/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.ts index 234a223c..29fcc904 100644 --- a/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.ts +++ b/release/ait/ait.mnw.mon/source/ait.mnw.mon.model.ts @@ -6,19 +6,57 @@ const source: LexicalModelSource = { format: 'trie-1.0', - wordBreaker: 'default', sources: ['wordlist.tsv'], languageUsesCasing: true, - wordBreaker: function(str: string) { - return str.split(/\s|\u200b/).map(function(token) { + wordBreaker: function(str: string) { + let latestIndex = 0; + + let emptyToken = { + left: str.length, + start: str.length, + right: str.length, + end: str.length, + length: 0, + text: '' + }; + + let tokens = str.split(/\s|\u200b/); + let finalizedTokens = tokens.map(function(token, index) { + // Empty tokens need special handling. + if(token == '') { + // If it's in the middle of the text, remove it. We probably + // were given two adjacent spaces. + if(index != tokens.length - 1) { + return null; + } else { + // If it's at the end of the text, we're in a "new word" + // position; make sure we indicate this well! + return emptyToken; + } + } + + // Always check for the position after the end of the previous token. + // The sequence should always be from start to end. + const start = str.indexOf(token, latestIndex); + latestIndex = start + token.length; return { - left: str.indexOf(token), - start: str.indexOf(token), - right: str.indexOf(token) + token.length, - end: str.indexOf(token) + token.length, + left: start, + start: start, + right: start + token.length, + end: start + token.length, + length: token.length, text: token - } - }); + }; + }).filter((t) => !!t); // remove any `null` entries (for middle-of-text empty strings) + + // If a new word should be starting at the current caret position, + // create an empty '' token to indicate this. + let tailToken = finalizedTokens[finalizedTokens.length - 1]; + if(!tailToken || tailToken.end != str.length) { + finalizedTokens.push(emptyToken); + } + + return finalizedTokens; }, punctuation: { insertAfterWord: ""