diff --git a/release/sil/sil.km.ggc/HISTORY.md b/release/sil/sil.km.ggc/HISTORY.md
index 75bcf35a..c565e552 100644
--- a/release/sil/sil.km.ggc/HISTORY.md
+++ b/release/sil/sil.km.ggc/HISTORY.md
@@ -1,5 +1,10 @@
Google Crawler Change History
====================
+
+2.1 (2026-Oct-27)
+----------------
+* update wordbreaking function to better handle edge cases
+
2.0 (2024-Oct-24)
----------------
* update wordlist and documentation
diff --git a/release/sil/sil.km.ggc/sil.km.ggc.kpj b/release/sil/sil.km.ggc/sil.km.ggc.kpj
index e7f285c0..071d98c6 100644
--- a/release/sil/sil.km.ggc/sil.km.ggc.kpj
+++ b/release/sil/sil.km.ggc/sil.km.ggc.kpj
@@ -18,12 +18,12 @@
id_a02fd3894fae146aa99da3213c3509d4
sil.km.ggc.model.kps
source\sil.km.ggc.model.kps
- 2.0
+ 2.1
.kps
Khmer Google Crawler
© SIL International
- 2.0
+ 2.1
diff --git a/release/sil/sil.km.ggc/source/sil.km.ggc.model.kps b/release/sil/sil.km.ggc/source/sil.km.ggc.model.kps
index ff811de8..62d8ab74 100644
--- a/release/sil/sil.km.ggc/source/sil.km.ggc.model.kps
+++ b/release/sil/sil.km.ggc/source/sil.km.ggc.model.kps
@@ -19,7 +19,7 @@
Khmer Google Crawler
© SIL International
SIL International
- 2.0
+ 2.1
Google Crawler generated from template
diff --git a/release/sil/sil.km.ggc/source/sil.km.ggc.model.ts b/release/sil/sil.km.ggc/source/sil.km.ggc.model.ts
index 122b9150..3ecc911c 100644
--- a/release/sil/sil.km.ggc/source/sil.km.ggc.model.ts
+++ b/release/sil/sil.km.ggc/source/sil.km.ggc.model.ts
@@ -60,8 +60,25 @@ const source: LexicalModelSource = {
}
}
+ let emptyToken = {
+ left: str.length,
+ start: str.length,
+ right: str.length,
+ end: str.length,
+ length: 0,
+ text: ''
+ };
+
let latestIndex = 0;
- return tokens.map(function(token) {
+ const finalizedTokens = tokens.map(function(token, index) {
+ // Make sure we properly place a new end-of-context token;
+ // indexOf doesn't do this well by default.
+ if(token == '' && index == tokens.length - 1) {
+ return emptyToken;
+ }
+
+ // Find the first occurrence of the word that appears after the end of
+ // the last word we checked.
const start = str.indexOf(token, latestIndex);
latestIndex = start + token.length;
return {
@@ -73,6 +90,15 @@ const source: LexicalModelSource = {
text: token
}
});
+
+ // If a new word should be starting at the current caret position,
+ // create an empty '' token to indicate this.
+ let tailToken = finalizedTokens[tokens.length - 1];
+ if(!tailToken || tailToken.end != str.length) {
+ finalizedTokens.push(emptyToken);
+ }
+
+ return finalizedTokens;
},
punctuation: {
insertAfterWord: "\u200B"