From bf845ede3b50d55a2854615024b85013321b6af2 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Sun, 1 Nov 2026 14:42:02 -0500 Subject: [PATCH] fix(asr/streaming): join a window that opens mid-word onto the previous text (#912) With vocabulary boosting, finish() builds the transcript from window text joined with spaces, while the token stream glues a window's first piece onto the previous token when that piece starts inside a word. A seam that cut before punctuation or mid-word therefore read "detail . As a result", or left a stray "y" from "Andre|y". The opening fragment now attaches like the token stream: punctuation always, a continuation's letters unless rescoring replaced the previous word (its term already spells the word), and a fragment rescoring rewrote keeps its own word. --- .../SlidingWindowAsrManager.swift | 69 +++++++++++++++++-- .../TDT/AsrManager+Transcription.swift | 16 +++++ .../SlidingWindowAsrManagerTests.swift | 66 ++++++++++++++++++ 3 files changed, 145 insertions(+), 6 deletions(-) diff --git a/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/SlidingWindowAsrManager.swift b/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/SlidingWindowAsrManager.swift index 0056b7ce5..219addf4d 100644 --- a/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/SlidingWindowAsrManager.swift +++ b/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/SlidingWindowAsrManager.swift @@ -681,7 +681,17 @@ public actor SlidingWindowAsrManager { ) } - await updateTranscriptionState(with: displayResult, shouldConfirm: shouldConfirm) + // A window that opens inside a word continues the previous text: the + // token stream glues its first piece on, so the text state must too, or + // the seam reads "detail . As" or leaves a stray "y" from "Andre|y" (#912). + var transcriptText = displayResult.text + if retainedTokenCount > 0, let manager = asrManager, await manager.startsMidWord(tokens) { + let previousRawWord = await manager.trailingWordText( + Array(accumulatedTokens.prefix(retainedTokenCount))) + transcriptText = joinMidWordWindow( + transcriptText, rawWindowText: interim.text, previousRawWord: previousRawWord) + } + await updateTranscriptionState(with: displayResult, text: transcriptText, shouldConfirm: shouldConfirm) lastWindowRenderedLastWord = Self.renderedLastWord( rawText: interim.text, renderedText: displayResult.text, replacements: appliedReplacements) @@ -739,7 +749,54 @@ public actor SlidingWindowAsrManager { } } - private func updateTranscriptionState(with result: ASRResult, shouldConfirm: Bool) async { + /// Glue the opening fragment of a window that starts inside a word onto the + /// end of the transcript text, and return the rest of the window's text (#912). + private func joinMidWordWindow(_ windowText: String, rawWindowText: String, previousRawWord: String) -> String { + let extendsVolatile = !volatileTranscript.isEmpty + let previous = extendsVolatile ? volatileTranscript : confirmedTranscript + guard !previous.isEmpty else { return windowText } + let joined = Self.joiningMidWordStart( + previous: previous, windowText: windowText, rawWindowText: rawWindowText, + previousRawWord: previousRawWord) + if extendsVolatile { + volatileTranscript = joined.previous + } else { + confirmedTranscript = joined.previous + } + return joined.rest + } + + /// Join a window that opens inside a word onto the previous text the way the + /// token stream joins its pieces (#912). The window's first fragment (up to + /// its first space) continues the previous word: punctuation attaches + /// ("detail" + "." → "detail."), and so do a continuation's letters ("Andre" + /// + "y" → "Andrey") unless rescoring replaced the previous word, whose + /// vocabulary term already spells the whole word ("Andrzej" + "y" → + /// "Andrzej"). An apostrophe suffix ("'s") always attaches. A fragment that + /// rescoring rewrote (`windowText` no longer opens with `rawWindowText`'s + /// fragment) is a whole vocabulary term and keeps its own word. Returns the + /// new previous text and the rest of the window's text. Pure. + static func joiningMidWordStart( + previous: String, windowText: String, rawWindowText: String, previousRawWord: String + ) -> (previous: String, rest: String) { + func split(_ text: String) -> (fragment: Substring, rest: String) { + let trimmed = text.drop(while: { $0 == " " }) + let fragment = trimmed.prefix(while: { $0 != " " }) + return (fragment, String(trimmed.dropFirst(fragment.count).drop(while: { $0 == " " }))) + } + let (fragment, rest) = split(windowText) + guard !fragment.isEmpty, fragment == split(rawWindowText).fragment else { return (previous, windowText) } + + func core(_ word: S) -> String { + word.lowercased().trimmingCharacters(in: .punctuationCharacters.union(.symbols)) + } + let letters = fragment.prefix(while: { !$0.isPunctuation && !$0.isSymbol }) + let previousWord = previous.split(separator: " ").last ?? "" + let replaced = !letters.isEmpty && core(previousWord) != core(previousRawWord) + return (previous + (replaced ? fragment.dropFirst(letters.count) : fragment), rest) + } + + private func updateTranscriptionState(with result: ASRResult, text: String, shouldConfirm: Bool) async { let totalAudioProcessed = Double(bufferStartIndex + sampleBuffer.count) / 16000.0 if shouldConfirm { @@ -751,9 +808,9 @@ public actor SlidingWindowAsrManager { components.append(volatileTranscript) confirmedTranscript = components.joined(separator: " ") } - volatileTranscript = result.text + volatileTranscript = text logger.debug( - "CONFIRMED (\(result.confidence), \(String(format: "%.1f", totalAudioProcessed))s context): promoted to confirmed; new volatile '\(result.text)'" + "CONFIRMED (\(result.confidence), \(String(format: "%.1f", totalAudioProcessed))s context): promoted to confirmed; new volatile '\(text)'" ) } else { // Each window carries new audio, so an unconfirmed window extends the @@ -761,12 +818,12 @@ public actor SlidingWindowAsrManager { // window's text whenever two consecutive windows went unconfirmed — with // boosting on, finish() builds from this text, and a trailing empty flush // window returned an empty transcript for a 15 s clip (#851). - volatileTranscript = Self.appendingVolatile(volatileTranscript, result.text) + volatileTranscript = Self.appendingVolatile(volatileTranscript, text) let hasMinimumContext = totalAudioProcessed >= config.minContextForConfirmation let reason = !hasMinimumContext ? "insufficient context (\(String(format: "%.1f", totalAudioProcessed))s)" : "low confidence" - logger.debug("VOLATILE (\(result.confidence)): \(reason) - updated volatile '\(result.text)'") + logger.debug("VOLATILE (\(result.confidence)): \(reason) - updated volatile '\(text)'") } } diff --git a/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/TDT/AsrManager+Transcription.swift b/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/TDT/AsrManager+Transcription.swift index 97e1394a6..851e4ddd7 100644 --- a/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/TDT/AsrManager+Transcription.swift +++ b/Sources/FluidAudio/ASR/Parakeet/SlidingWindow/TDT/AsrManager+Transcription.swift @@ -476,6 +476,22 @@ extension AsrManager { Self.trailingWordStartIndex(pieces: tokens.map { vocabulary[$0] ?? "" }) ?? 0 } + /// Whether `tokens` open inside a word: their first piece carries no word + /// boundary (a continuation such as the "y" of "Andre|y", or punctuation), + /// so the token stream glues it onto the previous token (#912). + func startsMidWord(_ tokens: [Int]) -> Bool { + guard + let first = tokens.lazy.compactMap({ self.vocabulary[$0] }) + .first(where: { !$0.isEmpty && $0 != ASRConstants.sentencePieceUnknownToken }) + else { return false } + return !first.hasPrefix(ASRConstants.sentencePieceWordBoundary) && !first.hasPrefix(" ") + } + + /// The text of the trailing word of `tokens`, from its last word-start piece. + func trailingWordText(_ tokens: [Int]) -> String { + convertTokensToText(Array(tokens[stableStreamingTokenCount(tokens)...])) + } + /// Chunk transcription that preserves decoder state between calls. /// Used by SlidingWindowAsrManager for overlapping-window processing with token deduplication. func transcribeChunk( diff --git a/Tests/FluidAudioTests/ASR/Parakeet/SlidingWindow/SlidingWindowAsrManagerTests.swift b/Tests/FluidAudioTests/ASR/Parakeet/SlidingWindow/SlidingWindowAsrManagerTests.swift index 50836f94e..465eec2c7 100644 --- a/Tests/FluidAudioTests/ASR/Parakeet/SlidingWindow/SlidingWindowAsrManagerTests.swift +++ b/Tests/FluidAudioTests/ASR/Parakeet/SlidingWindow/SlidingWindowAsrManagerTests.swift @@ -320,6 +320,72 @@ final class SlidingWindowAsrManagerTests: XCTestCase { XCTAssertEqual(SlidingWindowAsrManager.appendingVolatile("", "only"), "only") } + // MARK: - Mid-word window starts (#912) + + /// The seams of earnings22 `4329526_chunk18` and `4470253_chunk31` (#912): the + /// next window opened with punctuation, and with the "y" of "Andre|y". The token + /// stream glues both on; the text state joined them with a space. + func testMidWordWindowStartJoinsLikeTheTokenStream() { + let punctuation = SlidingWindowAsrManager.joiningMidWordStart( + previous: "that later Andres will detail", windowText: ". As a result,", rawWindowText: ". As a result,", + previousRawWord: "detail") + XCTAssertEqual(punctuation.previous, "that later Andres will detail.") + XCTAssertEqual(punctuation.rest, "As a result,") + + let continuation = SlidingWindowAsrManager.joiningMidWordStart( + previous: "give the floor to Andre", windowText: "y Kinsborg, who", rawWindowText: "y Kinsborg, who", + previousRawWord: "Andre") + XCTAssertEqual(continuation.previous, "give the floor to Andrey") + XCTAssertEqual(continuation.rest, "Kinsborg, who") + } + + /// A vocabulary term that replaced the previous word already spells the + /// whole word: the continuation's letters go, its punctuation and an + /// apostrophe suffix stay. + func testMidWordWindowStartAfterAReplacementKeepsOnlyPunctuationAndPossessive() { + let letters = SlidingWindowAsrManager.joiningMidWordStart( + previous: "give the floor to Andrzej", windowText: "y Kensbok, who", rawWindowText: "y Kinsborg, who", + previousRawWord: "Andre") + XCTAssertEqual(letters.previous, "give the floor to Andrzej") + XCTAssertEqual(letters.rest, "Kensbok, who") + + let lettersAndComma = SlidingWindowAsrManager.joiningMidWordStart( + previous: "thanks Andrzej", windowText: "y, next", rawWindowText: "y, next", + previousRawWord: "Andre") + XCTAssertEqual(lettersAndComma.previous, "thanks Andrzej,") + XCTAssertEqual(lettersAndComma.rest, "next") + + let possessive = SlidingWindowAsrManager.joiningMidWordStart( + previous: "about Quilter", windowText: "'s products", rawWindowText: "'s products", + previousRawWord: "Quiltor") + XCTAssertEqual(possessive.previous, "about Quilter's") + XCTAssertEqual(possessive.rest, "products") + } + + /// Earnings22 `4329526_chunk132` and `4452058_chunk35`: rescoring rewrote the + /// window's opening fragment ("?", "sa") into a vocabulary term, a word of its own. + func testMidWordWindowStartRewrittenByRescoringKeepsItsOwnWord() { + let punctuation = SlidingWindowAsrManager.joiningMidWordStart( + previous: "what type of discussion", windowText: "Sabby", rawWindowText: "?", + previousRawWord: "discussion") + XCTAssertEqual(punctuation.previous, "what type of discussion") + XCTAssertEqual(punctuation.rest, "Sabby") + + let continuation = SlidingWindowAsrManager.joiningMidWordStart( + previous: "Hepsiburada HepsiExpress.", windowText: "HepsiExpress offers both", + rawWindowText: "sa Express offers both", previousRawWord: "Express.") + XCTAssertEqual(continuation.previous, "Hepsiburada HepsiExpress.") + XCTAssertEqual(continuation.rest, "HepsiExpress offers both") + } + + func testMidWordWindowStartOfOnlyAFragmentLeavesNoRest() { + let joined = SlidingWindowAsrManager.joiningMidWordStart( + previous: "we have done so", windowText: ".", rawWindowText: ".", + previousRawWord: "so") + XCTAssertEqual(joined.previous, "we have done so.") + XCTAssertEqual(joined.rest, "") + } + /// Seam retirement removes the previous window's last word from the text /// state; with vocabulary boosting that text may carry a replacement, so /// the rendered form is tracked (#897 review).