From 15ebeeb0542df5d3e45b8a64b70babb19e267ba9 Mon Sep 17 00:00:00 2001 From: Zarxrax Date: Mon, 11 May 2026 21:29:02 -0400 Subject: [PATCH 1/3] Handle line breaks and full width spaces more closely to the behavior on jreadability.net --- src/jreadability/jreadability.py | 37 +++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/src/jreadability/jreadability.py b/src/jreadability/jreadability.py index 18271dc..5a3fa7d 100644 --- a/src/jreadability/jreadability.py +++ b/src/jreadability/jreadability.py @@ -8,6 +8,7 @@ from fugashi import Tagger from typing import List, Optional from fugashi.fugashi import UnidicNode +import re def compute_readability(text: str, tagger: Optional[Tagger] = None) -> float: @@ -21,35 +22,45 @@ def compute_readability(text: str, tagger: Optional[Tagger] = None) -> float: Returns: float: A float representing the readability score of the text. """ - if tagger is None: # initialize mecab parser tagger = Tagger() doc = tagger(text) - def split_japanese_sentences(doc: List[UnidicNode]) -> List[List[UnidicNode]]: + #Remove full-width spaces, standard spaces, and empty tokens + doc = [t for t in doc if t.surface.strip() and t.surface not in ("　")] + + + def split_japanese_sentences(text, tagger): """ Helper function that breaks the parsed text into lists of sentences. """ - + text = text.replace("\r\n", "\n").replace("\r", "\n") + paragraphs = re.split(r'\n\s*\n+', text) sentences = [] - current_sentence = [] - for token in doc: - current_sentence.append(token) - if token.surface in ("。", "？", "！", "．"): - sentences.append(current_sentence) - current_sentence = [] + for paragraph in paragraphs: + + current_sentence = [] + doc = tagger(paragraph) + doc = [t for t in doc if t.surface.strip() and t.surface not in ("　")] + + for token in doc: + current_sentence.append(token) - # if there's any leftover sentence that doesn't end with sentence-ending punctuation - if current_sentence: - sentences.append(current_sentence) + if token.surface in ("。", "？", "！", "．"): + sentences.append(current_sentence) + current_sentence = [] + + # if there's any leftover sentence that doesn't end with sentence-ending punctuation + if current_sentence: + sentences.append(current_sentence) return sentences # first, compute mean sentence length (in words, not characters) - sentences = split_japanese_sentences(doc) + sentences = split_japanese_sentences(text, tagger) sentence_lengths = [] for sentence_doc in sentences: From 211f87e90e1e734aeeb5890d53c1231114cb12cb Mon Sep 17 00:00:00 2001 From: Zarxrax Date: Mon, 11 May 2026 21:35:15 -0400 Subject: [PATCH 2/3] remove unused imports --- src/jreadability/jreadability.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/jreadability/jreadability.py b/src/jreadability/jreadability.py index 5a3fa7d..57eae53 100644 --- a/src/jreadability/jreadability.py +++ b/src/jreadability/jreadability.py @@ -6,8 +6,7 @@ """ from fugashi import Tagger -from typing import List, Optional -from fugashi.fugashi import UnidicNode +from typing import Optional import re From 87a0f321dfdf7eb59a832638cf1cd827fde6f303 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 01:46:33 +0000 Subject: [PATCH 3/3] [autofix.ci] apply automated fixes --- src/jreadability/jreadability.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/jreadability/jreadability.py b/src/jreadability/jreadability.py index 57eae53..ce135c1 100644 --- a/src/jreadability/jreadability.py +++ b/src/jreadability/jreadability.py @@ -27,24 +27,22 @@ def compute_readability(text: str, tagger: Optional[Tagger] = None) -> float: doc = tagger(text) - #Remove full-width spaces, standard spaces, and empty tokens + # Remove full-width spaces, standard spaces, and empty tokens doc = [t for t in doc if t.surface.strip() and t.surface not in ("　")] - def split_japanese_sentences(text, tagger): """ Helper function that breaks the parsed text into lists of sentences. """ text = text.replace("\r\n", "\n").replace("\r", "\n") - paragraphs = re.split(r'\n\s*\n+', text) + paragraphs = re.split(r"\n\s*\n+", text) sentences = [] for paragraph in paragraphs: - current_sentence = [] doc = tagger(paragraph) doc = [t for t in doc if t.surface.strip() and t.surface not in ("　")] - + for token in doc: current_sentence.append(token)