diff --git a/src/jreadability/jreadability.py b/src/jreadability/jreadability.py index 18271dc..ce135c1 100644 --- a/src/jreadability/jreadability.py +++ b/src/jreadability/jreadability.py @@ -6,8 +6,8 @@ """ from fugashi import Tagger -from typing import List, Optional -from fugashi.fugashi import UnidicNode +from typing import Optional +import re def compute_readability(text: str, tagger: Optional[Tagger] = None) -> float: @@ -21,35 +21,43 @@ def compute_readability(text: str, tagger: Optional[Tagger] = None) -> float: Returns: float: A float representing the readability score of the text. """ - if tagger is None: # initialize mecab parser tagger = Tagger() doc = tagger(text) - def split_japanese_sentences(doc: List[UnidicNode]) -> List[List[UnidicNode]]: + # Remove full-width spaces, standard spaces, and empty tokens + doc = [t for t in doc if t.surface.strip() and t.surface not in (" ")] + + def split_japanese_sentences(text, tagger): """ Helper function that breaks the parsed text into lists of sentences. """ - + text = text.replace("\r\n", "\n").replace("\r", "\n") + paragraphs = re.split(r"\n\s*\n+", text) sentences = [] - current_sentence = [] - for token in doc: - current_sentence.append(token) - if token.surface in ("。", "?", "!", "."): - sentences.append(current_sentence) - current_sentence = [] + for paragraph in paragraphs: + current_sentence = [] + doc = tagger(paragraph) + doc = [t for t in doc if t.surface.strip() and t.surface not in (" ")] - # if there's any leftover sentence that doesn't end with sentence-ending punctuation - if current_sentence: - sentences.append(current_sentence) + for token in doc: + current_sentence.append(token) + + if token.surface in ("。", "?", "!", "."): + sentences.append(current_sentence) + current_sentence = [] + + # if there's any leftover sentence that doesn't end with sentence-ending punctuation + if current_sentence: + sentences.append(current_sentence) return sentences # first, compute mean sentence length (in words, not characters) - sentences = split_japanese_sentences(doc) + sentences = split_japanese_sentences(text, tagger) sentence_lengths = [] for sentence_doc in sentences: