diff --git a/release/sil/sil.km.ggc/HISTORY.md b/release/sil/sil.km.ggc/HISTORY.md index 75bcf35a..c565e552 100644 --- a/release/sil/sil.km.ggc/HISTORY.md +++ b/release/sil/sil.km.ggc/HISTORY.md @@ -1,5 +1,10 @@ Google Crawler Change History ==================== + +2.1 (2026-Oct-27) +---------------- +* update wordbreaking function to better handle edge cases + 2.0 (2024-Oct-24) ---------------- * update wordlist and documentation diff --git a/release/sil/sil.km.ggc/sil.km.ggc.kpj b/release/sil/sil.km.ggc/sil.km.ggc.kpj index e7f285c0..071d98c6 100644 --- a/release/sil/sil.km.ggc/sil.km.ggc.kpj +++ b/release/sil/sil.km.ggc/sil.km.ggc.kpj @@ -18,12 +18,12 @@ id_a02fd3894fae146aa99da3213c3509d4 sil.km.ggc.model.kps source\sil.km.ggc.model.kps - 2.0 + 2.1 .kps
Khmer Google Crawler © SIL International - 2.0 + 2.1
diff --git a/release/sil/sil.km.ggc/source/sil.km.ggc.model.kps b/release/sil/sil.km.ggc/source/sil.km.ggc.model.kps index ff811de8..62d8ab74 100644 --- a/release/sil/sil.km.ggc/source/sil.km.ggc.model.kps +++ b/release/sil/sil.km.ggc/source/sil.km.ggc.model.kps @@ -19,7 +19,7 @@ Khmer Google Crawler © SIL International SIL International - 2.0 + 2.1 Google Crawler generated from template diff --git a/release/sil/sil.km.ggc/source/sil.km.ggc.model.ts b/release/sil/sil.km.ggc/source/sil.km.ggc.model.ts index 122b9150..3ecc911c 100644 --- a/release/sil/sil.km.ggc/source/sil.km.ggc.model.ts +++ b/release/sil/sil.km.ggc/source/sil.km.ggc.model.ts @@ -60,8 +60,25 @@ const source: LexicalModelSource = { } } + let emptyToken = { + left: str.length, + start: str.length, + right: str.length, + end: str.length, + length: 0, + text: '' + }; + let latestIndex = 0; - return tokens.map(function(token) { + const finalizedTokens = tokens.map(function(token, index) { + // Make sure we properly place a new end-of-context token; + // indexOf doesn't do this well by default. + if(token == '' && index == tokens.length - 1) { + return emptyToken; + } + + // Find the first occurrence of the word that appears after the end of + // the last word we checked. const start = str.indexOf(token, latestIndex); latestIndex = start + token.length; return { @@ -73,6 +90,15 @@ const source: LexicalModelSource = { text: token } }); + + // If a new word should be starting at the current caret position, + // create an empty '' token to indicate this. + let tailToken = finalizedTokens[tokens.length - 1]; + if(!tailToken || tailToken.end != str.length) { + finalizedTokens.push(emptyToken); + } + + return finalizedTokens; }, punctuation: { insertAfterWord: "\u200B"