From fa83c9a85b4660c3cf57c3c831d5226548f9f685 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:41:53 +0800 Subject: [PATCH 01/43] chore: apply visual OCR reliability patch --- .../apply-visual-ocr-reliability.yml | 537 ++++++++++++++++++ 1 file changed, 537 insertions(+) create mode 100644 .github/workflows/apply-visual-ocr-reliability.yml diff --git a/.github/workflows/apply-visual-ocr-reliability.yml b/.github/workflows/apply-visual-ocr-reliability.yml new file mode 100644 index 0000000..bd2ff84 --- /dev/null +++ b/.github/workflows/apply-visual-ocr-reliability.yml @@ -0,0 +1,537 @@ +name: Apply visual OCR reliability patch + +on: + push: + branches: + - codex/visual-ocr-reliability + +permissions: + contents: write + +jobs: + patch: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: codex/visual-ocr-reliability + fetch-depth: 0 + - name: Patch visual OCR pipeline + shell: bash + run: | + python3 - <<'PY' + from pathlib import Path + + path = Path('mreader/AITranslator.swift') + text = path.read_text() + + def replace_once(old, new, label): + global text + count = text.count(old) + if count != 1: + raise SystemExit(f'{label}: expected 1 match, found {count}') + text = text.replace(old, new, 1) + + replace_once( + '''nonisolated struct OCRVerificationRegion: Sendable { + let blockID: UUID + let sourceRect: CGRect + } + ''', + '''nonisolated struct OCRVerificationRegion: Sendable { + let blockID: UUID + let sourceRect: CGRect + } + + nonisolated struct VisionRegionTextCandidate: Equatable, Sendable { + let text: String + let confidence: Double + } + ''', + 'region candidate struct') + + replace_once( + ''' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up) + let localBlocks = try await recognizeVisionPage( + image: cropImage, + apiKey: apiKey, + baseURL: baseURL, + model: model, + isRightToLeft: isRightToLeft, + viewportAspect: max(cropImage.size.height / max(cropImage.size.width, 1), 1.25), + modelDescriptor: modelDescriptor ?? AIModelProtocolCatalog.descriptor(for: model) + ) + let original = corrected[originalIndex] + guard let match = visualVerificationMatch( + for: original, + candidates: localBlocks, + sourceRect: region.sourceRect + ) else { + continue + } + let best = match.block + let correctedText = best.text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !correctedText.isEmpty else { continue } + let correctedBox = match.pageBoundingBox + let correctedFontScale = visualVerificationMappedFontScale( + for: best, + sourceRect: region.sourceRect, + correctedBox: correctedBox + ) + let bubbleGeometry = visualVerificationMappedBubbleGeometry( + for: best, + sourceRect: region.sourceRect, + correctedBox: correctedBox + ) + corrected[originalIndex] = TextBlock( + id: original.id, + text: correctedText, + boundingBox: correctedBox, + translation: original.translation, + confidence: max(original.confidence, best.confidence), + ocrSource: "visual-review", + // A rejected/uncertain local candidate that passed a + // visual text+geometry match is explicitly recovered. + isFiltered: false, + filterReason: nil, + estimatedFontScale: correctedFontScale, + textColorHex: original.textColorHex, + bubbleBox: bubbleGeometry.bubbleBox, + layoutSafeRegion: original.layoutSafeRegion ?? bubbleGeometry.bubbleBox, + polygon: original.polygon, + bubblePolygon: bubbleGeometry.bubblePolygon, + translationLines: original.translationLines, + textOrientation: best.textOrientation, + layoutRole: original.layoutRole == .standalone || best.layoutRole == .standalone + ? .standalone + : .dialogue, + sourceLineCount: original.sourceLineCount + ) + print("MReader OCR visual review corrected block=\\(region.blockID) confidence=\\(String(format: \"%.2f\", best.confidence))") + ''', + ''' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up) + let review = try await recognizeVisionRegionText( + image: cropImage, + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor ?? AIModelProtocolCatalog.descriptor(for: model) + ) + let original = corrected[originalIndex] + corrected[originalIndex] = visualReviewedBlock(original: original, review: review) + print("MReader OCR visual text review corrected block=\\(region.blockID) confidence=\\(String(format: \"%.2f\", review.confidence))") + ''', + 'block visual review') + + replace_once( + ''' systemPrompt = "你只做漫画图片中的文字识别、断句、翻译和精确坐标标注。只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" + ''', + ''' systemPrompt = "你只做漫画图片中的文字识别、断句、翻译和精确坐标标注。只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" + ''', + 'translation system prompt') + + replace_once( + ''' systemPrompt = "你只做漫画图片中文字识别、断句和精确坐标标注，不要翻译。只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" + ''', + ''' systemPrompt = "你只做漫画图片中文字识别、断句和精确坐标标注，不要翻译。只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" + ''', + 'recognition system prompt') + + replace_once( + ''' let defaultMode: VisionResponseFormatMode = usesTranslationSchema ? .jsonSchema : .jsonObject + ''', + ''' let defaultMode: VisionResponseFormatMode = .jsonSchema + ''', + 'recognition schema default') + + replace_once( + ''' case .jsonSchema: + guard usesTranslationSchema else { + return .jsonObject + } + guard let schema = try? JSONSerialization.data( + withJSONObject: offlineVisionTranslationSchema() + ) else { + return nil + } + return .jsonSchema(name: "manga_offline_translation", schema: schema) + ''', + ''' case .jsonSchema: + let schemaObject = usesTranslationSchema + ? offlineVisionTranslationSchema() + : visionRecognitionSchema() + guard let schema = try? JSONSerialization.data(withJSONObject: schemaObject) else { + return nil + } + return .jsonSchema( + name: usesTranslationSchema ? "manga_offline_translation" : "manga_vision_recognition", + schema: schema + ) + ''', + 'recognition response schema') + + marker = ''' private static func recognizeVisionImage(image: UIImage, sourceRect: CGRect, apiKey: String, baseURL: String, model: String, modelDescriptor: AIModelDescriptor, isRightToLeft: Bool, additionalInstructions: String, translationTarget: TranslationTargetLanguage?, translationPromptTemplate: String, strictTranslationGeometry: Bool) async throws -> [TextBlock] { + ''' + addition = ''' private static func recognizeVisionRegionText( + image: UIImage, + apiKey: String, + baseURL: String, + model: String, + modelDescriptor: AIModelDescriptor + ) async throws -> VisionRegionTextCandidate { + guard !apiKey.isEmpty else { throw VisionTranslationError.api("未配置 API Key") } + guard !model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + throw VisionTranslationError.api("未配置模型") + } + let preparedImage = resizedImageForVision(image, maxDimension: 1536) + guard let imageDataURL = encodedVisionImageDataURL(preparedImage) else { + throw VisionTranslationError.imageEncodingFailed + } + let systemPrompt = "你只做漫画局部图片的原文转录。不要翻译，不要返回坐标，不要描述人物或画面，不要输出思考过程。只返回 sourceText 和 confidence。" + let prompt = """ + 这是已经由本地 OCR 定位好的单个漫画文字区域裁剪。 + 请只根据图片逐字抄录实际可见的原文，不要猜测裁剪外内容，不要翻译，不要补剧情。 + 保留原有标点、数字、拉长音、小假名和大小写；竖排文字按自然阅读顺序合并成一个字符串。 + 不需要返回任何坐标。 + 只输出 JSON：{"sourceText":"图中原文","confidence":0.95} + 如果图片中确实没有可读文字，输出 {"sourceText":"","confidence":0.0}。 + """ + + let data: Data + do { + data = try await visionCompletionData( + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor, + systemPrompt: systemPrompt, + prompt: prompt, + imageDataURL: imageDataURL, + responseFormat: .jsonObject + ) + } catch { + guard isUnsupportedResponseFormat(error) else { throw error } + data = try await visionCompletionData( + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor, + systemPrompt: systemPrompt, + prompt: prompt, + imageDataURL: imageDataURL, + responseFormat: nil + ) + } + + guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let content = assistantContent(from: json) else { + throw VisionTranslationError.invalidJSON + } + if looksLikeVisionNoTextResponse(content) { + throw VisionTranslationError.modelDidNotReadImage + } + guard let candidate = parseVisionRegionTextCandidate(from: content) else { + throw VisionTranslationError.emptyResult + } + return candidate + } + + private static func parseVisionRegionTextCandidate(from content: String) -> VisionRegionTextCandidate? { + if let data = normalizedVisionJSONData(from: content), + let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) { + let item: [String: Any]? + if let dictionary = object as? [String: Any] { + if firstString( + in: dictionary, + keys: ["sourceText", "source_text", "text", "original", "originalText"] + ).isEmpty, + let first = (dictionary["items"] as? [[String: Any]])?.first { + item = first + } else { + item = dictionary + } + } else if let array = object as? [[String: Any]] { + item = array.first + } else { + item = nil + } + if let item { + let text = firstString( + in: item, + keys: ["sourceText", "source_text", "text", "original", "originalText"] + ).trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty, !looksLikeVisionNoTextResponse(text) else { return nil } + let confidence = min(max(doubleValue(from: item["confidence"]) ?? 0.75, 0), 1) + return VisionRegionTextCandidate(text: text, confidence: confidence) + } + return nil + } + + var plain = extractedJSONPayload(from: content) + .replacingOccurrences( + of: #"^```(?:text|markdown)?\\s*|\\s*```$"#, + with: "", + options: [.regularExpression, .caseInsensitive] + ) + .trimmingCharacters(in: .whitespacesAndNewlines) + if plain.hasPrefix("\"") && plain.hasSuffix("\"") && plain.count >= 2 { + plain.removeFirst() + plain.removeLast() + } + guard !plain.isEmpty, !looksLikeVisionNoTextResponse(plain) else { return nil } + return VisionRegionTextCandidate(text: plain, confidence: 0.6) + } + + static func parseVisionRegionTextCandidateForDiagnostics( + from content: String + ) -> VisionRegionTextCandidate? { + parseVisionRegionTextCandidate(from: content) + } + + private static func visualReviewedBlock( + original: TextBlock, + review: VisionRegionTextCandidate + ) -> TextBlock { + TextBlock( + id: original.id, + text: review.text.trimmingCharacters(in: .whitespacesAndNewlines), + boundingBox: original.boundingBox, + translation: original.translation, + confidence: max(original.confidence, review.confidence), + ocrSource: "visual-review-text", + isFiltered: false, + filterReason: nil, + estimatedFontScale: original.estimatedFontScale, + textColorHex: original.textColorHex, + bubbleBox: original.bubbleBox, + layoutSafeRegion: original.layoutSafeRegion, + polygon: original.polygon, + bubblePolygon: original.bubblePolygon, + translationLines: original.translationLines, + textOrientation: original.textOrientation, + layoutRole: original.layoutRole, + sourceLineCount: original.sourceLineCount + ) + } + + static func visualReviewedBlockForDiagnostics( + original: TextBlock, + review: VisionRegionTextCandidate + ) -> TextBlock { + visualReviewedBlock(original: original, review: review) + } + + private static func looksLikeVisionNoTextResponse(_ text: String) -> Bool { + let normalized = text + .trimmingCharacters(in: .whitespacesAndNewlines) + .lowercased() + guard normalized.count <= 200 else { return false } + let refusalMarkers = [ + "无法识别", "无法读取", "无法返回", "不能识别", "不能读取", + "未检测到文字", "没有检测到文字", "未发现文字", "没有可读文字", + "抱歉，我无法", "看不到图片", "无法查看图片", + "unable to read", "unable to identify", "cannot read", "can't read", + "cannot view", "can't view", "no readable text", "no text found" + ] + return refusalMarkers.contains { normalized.contains($0) } + } + + ''' + marker + replace_once(marker, addition, 'region text helper') + + replace_once( + ''' guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let content = assistantContent(from: json) else { + throw VisionTranslationError.invalidJSON + } + let blocks: [TextBlock] + ''', + ''' guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let content = assistantContent(from: json) else { + throw VisionTranslationError.invalidJSON + } + if looksLikeVisionNoTextResponse(content) { + throw VisionTranslationError.modelDidNotReadImage + } + let blocks: [TextBlock] + ''', + 'full page refusal detection') + + replace_once( + ''' case protocolViolation(String) + case emptyResult + ''', + ''' case protocolViolation(String) + case modelDidNotReadImage + case emptyResult + ''', + 'vision error case') + + replace_once( + ''' case .protocolViolation(let message): + return "视觉翻译协议错误：\\(message)" + case .emptyResult: + return "视觉翻译没有返回可用文本" + ''', + ''' case .protocolViolation(let message): + return "视觉翻译协议错误：\\(message)" + case .modelDidNotReadImage: + return "视觉模型没有从图片读取到文字，请确认当前模型支持图像输入" + case .emptyResult: + return "视觉翻译没有返回可用文本" + ''', + 'vision error description') + + replace_once( + ''' case .invalidJSON, .invalidCoordinates, .missingTextBox, .missingTranslation, .protocolViolation, .emptyResult: + return true + ''', + ''' case .invalidJSON, .invalidCoordinates, .missingTextBox, .missingTranslation, + .protocolViolation, .modelDidNotReadImage, .emptyResult: + return true + ''', + 'vision slice fallback') + + schema_marker = ''' private static func offlineVisionTranslationSchema() -> [String: Any] { + ''' + schema_addition = ''' private static func visionRecognitionSchema() -> [String: Any] { + let point: [String: Any] = [ + "type": "object", + "additionalProperties": false, + "required": ["x", "y"], + "properties": [ + "x": ["type": "number", "minimum": 0, "maximum": 1], + "y": ["type": "number", "minimum": 0, "maximum": 1] + ] + ] + let rect: [String: Any] = [ + "type": "object", + "additionalProperties": false, + "required": ["x", "y", "width", "height"], + "properties": [ + "x": ["type": "number", "minimum": 0, "maximum": 1], + "y": ["type": "number", "minimum": 0, "maximum": 1], + "width": ["type": "number", "exclusiveMinimum": 0, "maximum": 1], + "height": ["type": "number", "exclusiveMinimum": 0, "maximum": 1] + ] + ] + let item: [String: Any] = [ + "type": "object", + "additionalProperties": false, + "required": ["sourceText", "textBox", "confidence", "classification"], + "properties": [ + "id": ["type": "string"], + "sourceText": ["type": "string"], + "textBox": rect, + "bubbleBox": rect, + "layoutSafeRegion": rect, + "textPolygon": ["type": "array", "minItems": 4, "items": point], + "bubblePolygon": ["type": "array", "minItems": 4, "items": point], + "confidence": ["type": "number", "minimum": 0, "maximum": 1], + "classification": [ + "type": "string", + "enum": [ + "dialogue", "narration", "soundEffect", "url", "advertisement", + "watermark", "copyright", "pageNumber" + ] + ] + ] + ] + return [ + "type": "object", + "additionalProperties": false, + "required": ["coordinateSpace", "items"], + "properties": [ + "coordinateSpace": ["type": "string", "enum": ["normalized"]], + "items": ["type": "array", "items": item] + ] + ] + } + + ''' + schema_marker + replace_once(schema_marker, schema_addition, 'recognition schema definition') + + path.write_text(text) + + test_path = Path('mreaderTests/TranslationComicIntegrationRegressionTests.swift') + tests = test_path.read_text() + insertion = ''' + func testVisionRegionTextParserAcceptsCoordinateFreeJSON() throws { + let result = try XCTUnwrap( + AITranslator.parseVisionRegionTextCandidateForDiagnostics( + from: #"{"sourceText":"ウィキペディアに","confidence":0.93}"# + ) + ) + XCTAssertEqual(result.text, "ウィキペディアに") + XCTAssertEqual(result.confidence, 0.93, accuracy: 0.0001) + } + + func testVisionRegionTextParserAcceptsPlainTextFallback() throws { + let result = try XCTUnwrap( + AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: "有名人です。") + ) + XCTAssertEqual(result.text, "有名人です。") + XCTAssertEqual(result.confidence, 0.6, accuracy: 0.0001) + } + + func testVisionRegionTextParserRejectsModelRefusal() { + XCTAssertNil( + AITranslator.parseVisionRegionTextCandidateForDiagnostics( + from: "抱歉，我无法读取这张图片中的文字。" + ) + ) + } + + func testVisualTextReviewPreservesLocalOCRGeometry() { + let original = TextBlock( + text: "OEIIII", + boundingBox: CGRect(x: 0.30, y: 0.20, width: 0.08, height: 0.24), + confidence: 0.3, + ocrSource: "original:manual", + estimatedFontScale: 0.08, + bubbleBox: CGRect(x: 0.28, y: 0.18, width: 0.14, height: 0.30), + layoutSafeRegion: CGRect(x: 0.29, y: 0.19, width: 0.12, height: 0.28), + textOrientation: .vertical, + layoutRole: .dialogue + ) + let reviewed = AITranslator.visualReviewedBlockForDiagnostics( + original: original, + review: VisionRegionTextCandidate(text: "ウィキペディアに", confidence: 0.91) + ) + + XCTAssertEqual(reviewed.text, "ウィキペディアに") + XCTAssertEqual(reviewed.boundingBox, original.boundingBox) + XCTAssertEqual(reviewed.bubbleBox, original.bubbleBox) + XCTAssertEqual(reviewed.layoutSafeRegion, original.layoutSafeRegion) + XCTAssertEqual(reviewed.textOrientation, original.textOrientation) + XCTAssertEqual(reviewed.layoutRole, original.layoutRole) + XCTAssertEqual(reviewed.ocrSource, "visual-review-text") + XCTAssertFalse(reviewed.isFiltered) + } + + func testVisionRecognitionPromptAndSystemContractIncludeLayoutSafeRegion() { + let prompt = AITranslator.visionRecognitionPromptForDiagnostics(isRightToLeft: true) + XCTAssertTrue(prompt.contains("layoutSafeRegion")) + } + ''' + close = '\n}' + if not tests.endswith(close): + raise SystemExit('test file closing brace not found') + tests = tests[:-len(close)] + insertion + close + test_path.write_text(tests) + PY + + git diff --check + git diff --stat + - name: Commit patch + shell: bash + run: | + if git diff --quiet; then + echo "No patch changes needed" + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add mreader/AITranslator.swift mreaderTests/TranslationComicIntegrationRegressionTests.swift + git commit -m "fix: make visual OCR review text-first" + git push origin HEAD:codex/visual-ocr-reliability From 63750dfbb1aa54c943289a303be5faf623628b39 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:42:47 +0800 Subject: [PATCH 02/43] chore: reset visual OCR patch workflow --- .../apply-visual-ocr-reliability.yml | 537 ------------------ 1 file changed, 537 deletions(-) delete mode 100644 .github/workflows/apply-visual-ocr-reliability.yml diff --git a/.github/workflows/apply-visual-ocr-reliability.yml b/.github/workflows/apply-visual-ocr-reliability.yml deleted file mode 100644 index bd2ff84..0000000 --- a/.github/workflows/apply-visual-ocr-reliability.yml +++ /dev/null @@ -1,537 +0,0 @@ -name: Apply visual OCR reliability patch - -on: - push: - branches: - - codex/visual-ocr-reliability - -permissions: - contents: write - -jobs: - patch: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: codex/visual-ocr-reliability - fetch-depth: 0 - - name: Patch visual OCR pipeline - shell: bash - run: | - python3 - <<'PY' - from pathlib import Path - - path = Path('mreader/AITranslator.swift') - text = path.read_text() - - def replace_once(old, new, label): - global text - count = text.count(old) - if count != 1: - raise SystemExit(f'{label}: expected 1 match, found {count}') - text = text.replace(old, new, 1) - - replace_once( - '''nonisolated struct OCRVerificationRegion: Sendable { - let blockID: UUID - let sourceRect: CGRect - } - ''', - '''nonisolated struct OCRVerificationRegion: Sendable { - let blockID: UUID - let sourceRect: CGRect - } - - nonisolated struct VisionRegionTextCandidate: Equatable, Sendable { - let text: String - let confidence: Double - } - ''', - 'region candidate struct') - - replace_once( - ''' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up) - let localBlocks = try await recognizeVisionPage( - image: cropImage, - apiKey: apiKey, - baseURL: baseURL, - model: model, - isRightToLeft: isRightToLeft, - viewportAspect: max(cropImage.size.height / max(cropImage.size.width, 1), 1.25), - modelDescriptor: modelDescriptor ?? AIModelProtocolCatalog.descriptor(for: model) - ) - let original = corrected[originalIndex] - guard let match = visualVerificationMatch( - for: original, - candidates: localBlocks, - sourceRect: region.sourceRect - ) else { - continue - } - let best = match.block - let correctedText = best.text.trimmingCharacters(in: .whitespacesAndNewlines) - guard !correctedText.isEmpty else { continue } - let correctedBox = match.pageBoundingBox - let correctedFontScale = visualVerificationMappedFontScale( - for: best, - sourceRect: region.sourceRect, - correctedBox: correctedBox - ) - let bubbleGeometry = visualVerificationMappedBubbleGeometry( - for: best, - sourceRect: region.sourceRect, - correctedBox: correctedBox - ) - corrected[originalIndex] = TextBlock( - id: original.id, - text: correctedText, - boundingBox: correctedBox, - translation: original.translation, - confidence: max(original.confidence, best.confidence), - ocrSource: "visual-review", - // A rejected/uncertain local candidate that passed a - // visual text+geometry match is explicitly recovered. - isFiltered: false, - filterReason: nil, - estimatedFontScale: correctedFontScale, - textColorHex: original.textColorHex, - bubbleBox: bubbleGeometry.bubbleBox, - layoutSafeRegion: original.layoutSafeRegion ?? bubbleGeometry.bubbleBox, - polygon: original.polygon, - bubblePolygon: bubbleGeometry.bubblePolygon, - translationLines: original.translationLines, - textOrientation: best.textOrientation, - layoutRole: original.layoutRole == .standalone || best.layoutRole == .standalone - ? .standalone - : .dialogue, - sourceLineCount: original.sourceLineCount - ) - print("MReader OCR visual review corrected block=\\(region.blockID) confidence=\\(String(format: \"%.2f\", best.confidence))") - ''', - ''' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up) - let review = try await recognizeVisionRegionText( - image: cropImage, - apiKey: apiKey, - baseURL: baseURL, - model: model, - modelDescriptor: modelDescriptor ?? AIModelProtocolCatalog.descriptor(for: model) - ) - let original = corrected[originalIndex] - corrected[originalIndex] = visualReviewedBlock(original: original, review: review) - print("MReader OCR visual text review corrected block=\\(region.blockID) confidence=\\(String(format: \"%.2f\", review.confidence))") - ''', - 'block visual review') - - replace_once( - ''' systemPrompt = "你只做漫画图片中的文字识别、断句、翻译和精确坐标标注。只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" - ''', - ''' systemPrompt = "你只做漫画图片中的文字识别、断句、翻译和精确坐标标注。只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" - ''', - 'translation system prompt') - - replace_once( - ''' systemPrompt = "你只做漫画图片中文字识别、断句和精确坐标标注，不要翻译。只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" - ''', - ''' systemPrompt = "你只做漫画图片中文字识别、断句和精确坐标标注，不要翻译。只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" - ''', - 'recognition system prompt') - - replace_once( - ''' let defaultMode: VisionResponseFormatMode = usesTranslationSchema ? .jsonSchema : .jsonObject - ''', - ''' let defaultMode: VisionResponseFormatMode = .jsonSchema - ''', - 'recognition schema default') - - replace_once( - ''' case .jsonSchema: - guard usesTranslationSchema else { - return .jsonObject - } - guard let schema = try? JSONSerialization.data( - withJSONObject: offlineVisionTranslationSchema() - ) else { - return nil - } - return .jsonSchema(name: "manga_offline_translation", schema: schema) - ''', - ''' case .jsonSchema: - let schemaObject = usesTranslationSchema - ? offlineVisionTranslationSchema() - : visionRecognitionSchema() - guard let schema = try? JSONSerialization.data(withJSONObject: schemaObject) else { - return nil - } - return .jsonSchema( - name: usesTranslationSchema ? "manga_offline_translation" : "manga_vision_recognition", - schema: schema - ) - ''', - 'recognition response schema') - - marker = ''' private static func recognizeVisionImage(image: UIImage, sourceRect: CGRect, apiKey: String, baseURL: String, model: String, modelDescriptor: AIModelDescriptor, isRightToLeft: Bool, additionalInstructions: String, translationTarget: TranslationTargetLanguage?, translationPromptTemplate: String, strictTranslationGeometry: Bool) async throws -> [TextBlock] { - ''' - addition = ''' private static func recognizeVisionRegionText( - image: UIImage, - apiKey: String, - baseURL: String, - model: String, - modelDescriptor: AIModelDescriptor - ) async throws -> VisionRegionTextCandidate { - guard !apiKey.isEmpty else { throw VisionTranslationError.api("未配置 API Key") } - guard !model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - throw VisionTranslationError.api("未配置模型") - } - let preparedImage = resizedImageForVision(image, maxDimension: 1536) - guard let imageDataURL = encodedVisionImageDataURL(preparedImage) else { - throw VisionTranslationError.imageEncodingFailed - } - let systemPrompt = "你只做漫画局部图片的原文转录。不要翻译，不要返回坐标，不要描述人物或画面，不要输出思考过程。只返回 sourceText 和 confidence。" - let prompt = """ - 这是已经由本地 OCR 定位好的单个漫画文字区域裁剪。 - 请只根据图片逐字抄录实际可见的原文，不要猜测裁剪外内容，不要翻译，不要补剧情。 - 保留原有标点、数字、拉长音、小假名和大小写；竖排文字按自然阅读顺序合并成一个字符串。 - 不需要返回任何坐标。 - 只输出 JSON：{"sourceText":"图中原文","confidence":0.95} - 如果图片中确实没有可读文字，输出 {"sourceText":"","confidence":0.0}。 - """ - - let data: Data - do { - data = try await visionCompletionData( - apiKey: apiKey, - baseURL: baseURL, - model: model, - modelDescriptor: modelDescriptor, - systemPrompt: systemPrompt, - prompt: prompt, - imageDataURL: imageDataURL, - responseFormat: .jsonObject - ) - } catch { - guard isUnsupportedResponseFormat(error) else { throw error } - data = try await visionCompletionData( - apiKey: apiKey, - baseURL: baseURL, - model: model, - modelDescriptor: modelDescriptor, - systemPrompt: systemPrompt, - prompt: prompt, - imageDataURL: imageDataURL, - responseFormat: nil - ) - } - - guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let content = assistantContent(from: json) else { - throw VisionTranslationError.invalidJSON - } - if looksLikeVisionNoTextResponse(content) { - throw VisionTranslationError.modelDidNotReadImage - } - guard let candidate = parseVisionRegionTextCandidate(from: content) else { - throw VisionTranslationError.emptyResult - } - return candidate - } - - private static func parseVisionRegionTextCandidate(from content: String) -> VisionRegionTextCandidate? { - if let data = normalizedVisionJSONData(from: content), - let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) { - let item: [String: Any]? - if let dictionary = object as? [String: Any] { - if firstString( - in: dictionary, - keys: ["sourceText", "source_text", "text", "original", "originalText"] - ).isEmpty, - let first = (dictionary["items"] as? [[String: Any]])?.first { - item = first - } else { - item = dictionary - } - } else if let array = object as? [[String: Any]] { - item = array.first - } else { - item = nil - } - if let item { - let text = firstString( - in: item, - keys: ["sourceText", "source_text", "text", "original", "originalText"] - ).trimmingCharacters(in: .whitespacesAndNewlines) - guard !text.isEmpty, !looksLikeVisionNoTextResponse(text) else { return nil } - let confidence = min(max(doubleValue(from: item["confidence"]) ?? 0.75, 0), 1) - return VisionRegionTextCandidate(text: text, confidence: confidence) - } - return nil - } - - var plain = extractedJSONPayload(from: content) - .replacingOccurrences( - of: #"^```(?:text|markdown)?\\s*|\\s*```$"#, - with: "", - options: [.regularExpression, .caseInsensitive] - ) - .trimmingCharacters(in: .whitespacesAndNewlines) - if plain.hasPrefix("\"") && plain.hasSuffix("\"") && plain.count >= 2 { - plain.removeFirst() - plain.removeLast() - } - guard !plain.isEmpty, !looksLikeVisionNoTextResponse(plain) else { return nil } - return VisionRegionTextCandidate(text: plain, confidence: 0.6) - } - - static func parseVisionRegionTextCandidateForDiagnostics( - from content: String - ) -> VisionRegionTextCandidate? { - parseVisionRegionTextCandidate(from: content) - } - - private static func visualReviewedBlock( - original: TextBlock, - review: VisionRegionTextCandidate - ) -> TextBlock { - TextBlock( - id: original.id, - text: review.text.trimmingCharacters(in: .whitespacesAndNewlines), - boundingBox: original.boundingBox, - translation: original.translation, - confidence: max(original.confidence, review.confidence), - ocrSource: "visual-review-text", - isFiltered: false, - filterReason: nil, - estimatedFontScale: original.estimatedFontScale, - textColorHex: original.textColorHex, - bubbleBox: original.bubbleBox, - layoutSafeRegion: original.layoutSafeRegion, - polygon: original.polygon, - bubblePolygon: original.bubblePolygon, - translationLines: original.translationLines, - textOrientation: original.textOrientation, - layoutRole: original.layoutRole, - sourceLineCount: original.sourceLineCount - ) - } - - static func visualReviewedBlockForDiagnostics( - original: TextBlock, - review: VisionRegionTextCandidate - ) -> TextBlock { - visualReviewedBlock(original: original, review: review) - } - - private static func looksLikeVisionNoTextResponse(_ text: String) -> Bool { - let normalized = text - .trimmingCharacters(in: .whitespacesAndNewlines) - .lowercased() - guard normalized.count <= 200 else { return false } - let refusalMarkers = [ - "无法识别", "无法读取", "无法返回", "不能识别", "不能读取", - "未检测到文字", "没有检测到文字", "未发现文字", "没有可读文字", - "抱歉，我无法", "看不到图片", "无法查看图片", - "unable to read", "unable to identify", "cannot read", "can't read", - "cannot view", "can't view", "no readable text", "no text found" - ] - return refusalMarkers.contains { normalized.contains($0) } - } - - ''' + marker - replace_once(marker, addition, 'region text helper') - - replace_once( - ''' guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let content = assistantContent(from: json) else { - throw VisionTranslationError.invalidJSON - } - let blocks: [TextBlock] - ''', - ''' guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let content = assistantContent(from: json) else { - throw VisionTranslationError.invalidJSON - } - if looksLikeVisionNoTextResponse(content) { - throw VisionTranslationError.modelDidNotReadImage - } - let blocks: [TextBlock] - ''', - 'full page refusal detection') - - replace_once( - ''' case protocolViolation(String) - case emptyResult - ''', - ''' case protocolViolation(String) - case modelDidNotReadImage - case emptyResult - ''', - 'vision error case') - - replace_once( - ''' case .protocolViolation(let message): - return "视觉翻译协议错误：\\(message)" - case .emptyResult: - return "视觉翻译没有返回可用文本" - ''', - ''' case .protocolViolation(let message): - return "视觉翻译协议错误：\\(message)" - case .modelDidNotReadImage: - return "视觉模型没有从图片读取到文字，请确认当前模型支持图像输入" - case .emptyResult: - return "视觉翻译没有返回可用文本" - ''', - 'vision error description') - - replace_once( - ''' case .invalidJSON, .invalidCoordinates, .missingTextBox, .missingTranslation, .protocolViolation, .emptyResult: - return true - ''', - ''' case .invalidJSON, .invalidCoordinates, .missingTextBox, .missingTranslation, - .protocolViolation, .modelDidNotReadImage, .emptyResult: - return true - ''', - 'vision slice fallback') - - schema_marker = ''' private static func offlineVisionTranslationSchema() -> [String: Any] { - ''' - schema_addition = ''' private static func visionRecognitionSchema() -> [String: Any] { - let point: [String: Any] = [ - "type": "object", - "additionalProperties": false, - "required": ["x", "y"], - "properties": [ - "x": ["type": "number", "minimum": 0, "maximum": 1], - "y": ["type": "number", "minimum": 0, "maximum": 1] - ] - ] - let rect: [String: Any] = [ - "type": "object", - "additionalProperties": false, - "required": ["x", "y", "width", "height"], - "properties": [ - "x": ["type": "number", "minimum": 0, "maximum": 1], - "y": ["type": "number", "minimum": 0, "maximum": 1], - "width": ["type": "number", "exclusiveMinimum": 0, "maximum": 1], - "height": ["type": "number", "exclusiveMinimum": 0, "maximum": 1] - ] - ] - let item: [String: Any] = [ - "type": "object", - "additionalProperties": false, - "required": ["sourceText", "textBox", "confidence", "classification"], - "properties": [ - "id": ["type": "string"], - "sourceText": ["type": "string"], - "textBox": rect, - "bubbleBox": rect, - "layoutSafeRegion": rect, - "textPolygon": ["type": "array", "minItems": 4, "items": point], - "bubblePolygon": ["type": "array", "minItems": 4, "items": point], - "confidence": ["type": "number", "minimum": 0, "maximum": 1], - "classification": [ - "type": "string", - "enum": [ - "dialogue", "narration", "soundEffect", "url", "advertisement", - "watermark", "copyright", "pageNumber" - ] - ] - ] - ] - return [ - "type": "object", - "additionalProperties": false, - "required": ["coordinateSpace", "items"], - "properties": [ - "coordinateSpace": ["type": "string", "enum": ["normalized"]], - "items": ["type": "array", "items": item] - ] - ] - } - - ''' + schema_marker - replace_once(schema_marker, schema_addition, 'recognition schema definition') - - path.write_text(text) - - test_path = Path('mreaderTests/TranslationComicIntegrationRegressionTests.swift') - tests = test_path.read_text() - insertion = ''' - func testVisionRegionTextParserAcceptsCoordinateFreeJSON() throws { - let result = try XCTUnwrap( - AITranslator.parseVisionRegionTextCandidateForDiagnostics( - from: #"{"sourceText":"ウィキペディアに","confidence":0.93}"# - ) - ) - XCTAssertEqual(result.text, "ウィキペディアに") - XCTAssertEqual(result.confidence, 0.93, accuracy: 0.0001) - } - - func testVisionRegionTextParserAcceptsPlainTextFallback() throws { - let result = try XCTUnwrap( - AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: "有名人です。") - ) - XCTAssertEqual(result.text, "有名人です。") - XCTAssertEqual(result.confidence, 0.6, accuracy: 0.0001) - } - - func testVisionRegionTextParserRejectsModelRefusal() { - XCTAssertNil( - AITranslator.parseVisionRegionTextCandidateForDiagnostics( - from: "抱歉，我无法读取这张图片中的文字。" - ) - ) - } - - func testVisualTextReviewPreservesLocalOCRGeometry() { - let original = TextBlock( - text: "OEIIII", - boundingBox: CGRect(x: 0.30, y: 0.20, width: 0.08, height: 0.24), - confidence: 0.3, - ocrSource: "original:manual", - estimatedFontScale: 0.08, - bubbleBox: CGRect(x: 0.28, y: 0.18, width: 0.14, height: 0.30), - layoutSafeRegion: CGRect(x: 0.29, y: 0.19, width: 0.12, height: 0.28), - textOrientation: .vertical, - layoutRole: .dialogue - ) - let reviewed = AITranslator.visualReviewedBlockForDiagnostics( - original: original, - review: VisionRegionTextCandidate(text: "ウィキペディアに", confidence: 0.91) - ) - - XCTAssertEqual(reviewed.text, "ウィキペディアに") - XCTAssertEqual(reviewed.boundingBox, original.boundingBox) - XCTAssertEqual(reviewed.bubbleBox, original.bubbleBox) - XCTAssertEqual(reviewed.layoutSafeRegion, original.layoutSafeRegion) - XCTAssertEqual(reviewed.textOrientation, original.textOrientation) - XCTAssertEqual(reviewed.layoutRole, original.layoutRole) - XCTAssertEqual(reviewed.ocrSource, "visual-review-text") - XCTAssertFalse(reviewed.isFiltered) - } - - func testVisionRecognitionPromptAndSystemContractIncludeLayoutSafeRegion() { - let prompt = AITranslator.visionRecognitionPromptForDiagnostics(isRightToLeft: true) - XCTAssertTrue(prompt.contains("layoutSafeRegion")) - } - ''' - close = '\n}' - if not tests.endswith(close): - raise SystemExit('test file closing brace not found') - tests = tests[:-len(close)] + insertion + close - test_path.write_text(tests) - PY - - git diff --check - git diff --stat - - name: Commit patch - shell: bash - run: | - if git diff --quiet; then - echo "No patch changes needed" - exit 0 - fi - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add mreader/AITranslator.swift mreaderTests/TranslationComicIntegrationRegressionTests.swift - git commit -m "fix: make visual OCR review text-first" - git push origin HEAD:codex/visual-ocr-reliability From 0caaaec4970aa5951345b2c4e2d1fb83d36ce059 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:43:39 +0800 Subject: [PATCH 03/43] chore: apply visual OCR reliability patch --- .../apply-visual-ocr-reliability-v2.yml | 519 ++++++++++++++++++ 1 file changed, 519 insertions(+) create mode 100644 .github/workflows/apply-visual-ocr-reliability-v2.yml diff --git a/.github/workflows/apply-visual-ocr-reliability-v2.yml b/.github/workflows/apply-visual-ocr-reliability-v2.yml new file mode 100644 index 0000000..568e986 --- /dev/null +++ b/.github/workflows/apply-visual-ocr-reliability-v2.yml @@ -0,0 +1,519 @@ +name: Apply visual OCR reliability patch v2 + +on: + push: + branches: + - codex/visual-ocr-reliability + +permissions: + contents: write + +jobs: + patch: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: codex/visual-ocr-reliability + fetch-depth: 0 + - name: Patch visual OCR pipeline + shell: bash + run: | + python3 - <<'PY' + from pathlib import Path + + path = Path('mreader/AITranslator.swift') + text = path.read_text() + + def replace_once(old, new, label): + global text + count = text.count(old) + if count != 1: + raise SystemExit(f'{label}: expected 1 match, found {count}') + text = text.replace(old, new, 1) + + replace_once( + '''nonisolated struct OCRVerificationRegion: Sendable { + let blockID: UUID + let sourceRect: CGRect + } + ''', + '''nonisolated struct OCRVerificationRegion: Sendable { + let blockID: UUID + let sourceRect: CGRect + } + + nonisolated struct VisionRegionTextCandidate: Equatable, Sendable { + let text: String + let confidence: Double + } + ''', + 'region candidate struct') + + replace_once( + ''' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up) + let localBlocks = try await recognizeVisionPage( + image: cropImage, + apiKey: apiKey, + baseURL: baseURL, + model: model, + isRightToLeft: isRightToLeft, + viewportAspect: max(cropImage.size.height / max(cropImage.size.width, 1), 1.25), + modelDescriptor: modelDescriptor ?? AIModelProtocolCatalog.descriptor(for: model) + ) + let original = corrected[originalIndex] + guard let match = visualVerificationMatch( + for: original, + candidates: localBlocks, + sourceRect: region.sourceRect + ) else { + continue + } + let best = match.block + let correctedText = best.text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !correctedText.isEmpty else { continue } + let correctedBox = match.pageBoundingBox + let correctedFontScale = visualVerificationMappedFontScale( + for: best, + sourceRect: region.sourceRect, + correctedBox: correctedBox + ) + let bubbleGeometry = visualVerificationMappedBubbleGeometry( + for: best, + sourceRect: region.sourceRect, + correctedBox: correctedBox + ) + corrected[originalIndex] = TextBlock( + id: original.id, + text: correctedText, + boundingBox: correctedBox, + translation: original.translation, + confidence: max(original.confidence, best.confidence), + ocrSource: "visual-review", + // A rejected/uncertain local candidate that passed a + // visual text+geometry match is explicitly recovered. + isFiltered: false, + filterReason: nil, + estimatedFontScale: correctedFontScale, + textColorHex: original.textColorHex, + bubbleBox: bubbleGeometry.bubbleBox, + layoutSafeRegion: original.layoutSafeRegion ?? bubbleGeometry.bubbleBox, + polygon: original.polygon, + bubblePolygon: bubbleGeometry.bubblePolygon, + translationLines: original.translationLines, + textOrientation: best.textOrientation, + layoutRole: original.layoutRole == .standalone || best.layoutRole == .standalone + ? .standalone + : .dialogue, + sourceLineCount: original.sourceLineCount + ) + print("MReader OCR visual review corrected block=\\(region.blockID) confidence=\\(String(format: \"%.2f\", best.confidence))") + ''', + ''' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up) + let review = try await recognizeVisionRegionText( + image: cropImage, + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor ?? AIModelProtocolCatalog.descriptor(for: model) + ) + let original = corrected[originalIndex] + corrected[originalIndex] = visualReviewedBlock(original: original, review: review) + print("MReader OCR visual text review corrected block=\\(region.blockID) confidence=\\(String(format: \"%.2f\", review.confidence))") + ''', + 'block visual review') + + replace_once( + ''' systemPrompt = "你只做漫画图片中的文字识别、断句、翻译和精确坐标标注。只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" + ''', + ''' systemPrompt = "你只做漫画图片中的文字识别、断句、翻译和精确坐标标注。只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" + ''', + 'translation system prompt') + + replace_once( + ''' systemPrompt = "你只做漫画图片中文字识别、断句和精确坐标标注，不要翻译。只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" + ''', + ''' systemPrompt = "你只做漫画图片中文字识别、断句和精确坐标标注，不要翻译。只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" + ''', + 'recognition system prompt') + + replace_once( + ''' let defaultMode: VisionResponseFormatMode = usesTranslationSchema ? .jsonSchema : .jsonObject + ''', + ''' let defaultMode: VisionResponseFormatMode = .jsonSchema + ''', + 'recognition schema default') + + replace_once( + ''' case .jsonSchema: + guard usesTranslationSchema else { + return .jsonObject + } + guard let schema = try? JSONSerialization.data( + withJSONObject: offlineVisionTranslationSchema() + ) else { + return nil + } + return .jsonSchema(name: "manga_offline_translation", schema: schema) + ''', + ''' case .jsonSchema: + let schemaObject = usesTranslationSchema + ? offlineVisionTranslationSchema() + : visionRecognitionSchema() + guard let schema = try? JSONSerialization.data(withJSONObject: schemaObject) else { + return nil + } + return .jsonSchema( + name: usesTranslationSchema ? "manga_offline_translation" : "manga_vision_recognition", + schema: schema + ) + ''', + 'recognition response schema') + + marker = ''' private static func recognizeVisionImage(image: UIImage, sourceRect: CGRect, apiKey: String, baseURL: String, model: String, modelDescriptor: AIModelDescriptor, isRightToLeft: Bool, additionalInstructions: String, translationTarget: TranslationTargetLanguage?, translationPromptTemplate: String, strictTranslationGeometry: Bool) async throws -> [TextBlock] { + ''' + addition = ''' private static func recognizeVisionRegionText( + image: UIImage, + apiKey: String, + baseURL: String, + model: String, + modelDescriptor: AIModelDescriptor + ) async throws -> VisionRegionTextCandidate { + guard !apiKey.isEmpty else { throw VisionTranslationError.api("未配置 API Key") } + guard !model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + throw VisionTranslationError.api("未配置模型") + } + let preparedImage = resizedImageForVision(image, maxDimension: 1536) + guard let imageDataURL = encodedVisionImageDataURL(preparedImage) else { + throw VisionTranslationError.imageEncodingFailed + } + let systemPrompt = "你只做漫画局部图片的原文转录。不要翻译，不要返回坐标，不要描述人物或画面，不要输出思考过程。只返回 sourceText 和 confidence。" + let prompt = """ + 这是已经由本地 OCR 定位好的单个漫画文字区域裁剪。 + 请只根据图片逐字抄录实际可见的原文，不要猜测裁剪外内容，不要翻译，不要补剧情。 + 保留原有标点、数字、拉长音、小假名和大小写；竖排文字按自然阅读顺序合并成一个字符串。 + 不需要返回任何坐标。 + 只输出 JSON：{"sourceText":"图中原文","confidence":0.95} + 如果图片中确实没有可读文字，输出 {"sourceText":"","confidence":0.0}。 + """ + + let data: Data + do { + data = try await visionCompletionData( + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor, + systemPrompt: systemPrompt, + prompt: prompt, + imageDataURL: imageDataURL, + responseFormat: .jsonObject + ) + } catch { + guard isUnsupportedResponseFormat(error) else { throw error } + data = try await visionCompletionData( + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor, + systemPrompt: systemPrompt, + prompt: prompt, + imageDataURL: imageDataURL, + responseFormat: nil + ) + } + + guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let content = assistantContent(from: json) else { + throw VisionTranslationError.invalidJSON + } + if looksLikeVisionNoTextResponse(content) { + throw VisionTranslationError.modelDidNotReadImage + } + guard let candidate = parseVisionRegionTextCandidate(from: content) else { + throw VisionTranslationError.emptyResult + } + return candidate + } + + private static func parseVisionRegionTextCandidate(from content: String) -> VisionRegionTextCandidate? { + if let data = normalizedVisionJSONData(from: content), + let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) { + let item: [String: Any]? + if let dictionary = object as? [String: Any] { + if firstString( + in: dictionary, + keys: ["sourceText", "source_text", "text", "original", "originalText"] + ).isEmpty, + let first = (dictionary["items"] as? [[String: Any]])?.first { + item = first + } else { + item = dictionary + } + } else if let array = object as? [[String: Any]] { + item = array.first + } else { + item = nil + } + if let item { + let text = firstString( + in: item, + keys: ["sourceText", "source_text", "text", "original", "originalText"] + ).trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty, !looksLikeVisionNoTextResponse(text) else { return nil } + let confidence = min(max(doubleValue(from: item["confidence"]) ?? 0.75, 0), 1) + return VisionRegionTextCandidate(text: text, confidence: confidence) + } + return nil + } + + var plain = extractedJSONPayload(from: content) + .replacingOccurrences( + of: #"^```(?:text|markdown)?\\s*|\\s*```$"#, + with: "", + options: [.regularExpression, .caseInsensitive] + ) + .trimmingCharacters(in: .whitespacesAndNewlines) + if plain.hasPrefix("\"") && plain.hasSuffix("\"") && plain.count >= 2 { + plain.removeFirst() + plain.removeLast() + } + guard !plain.isEmpty, !looksLikeVisionNoTextResponse(plain) else { return nil } + return VisionRegionTextCandidate(text: plain, confidence: 0.6) + } + + static func parseVisionRegionTextCandidateForDiagnostics( + from content: String + ) -> VisionRegionTextCandidate? { + parseVisionRegionTextCandidate(from: content) + } + + private static func visualReviewedBlock( + original: TextBlock, + review: VisionRegionTextCandidate + ) -> TextBlock { + TextBlock( + id: original.id, + text: review.text.trimmingCharacters(in: .whitespacesAndNewlines), + boundingBox: original.boundingBox, + translation: original.translation, + confidence: max(original.confidence, review.confidence), + ocrSource: "visual-review-text", + isFiltered: false, + filterReason: nil, + estimatedFontScale: original.estimatedFontScale, + textColorHex: original.textColorHex, + bubbleBox: original.bubbleBox, + layoutSafeRegion: original.layoutSafeRegion, + polygon: original.polygon, + bubblePolygon: original.bubblePolygon, + translationLines: original.translationLines, + textOrientation: original.textOrientation, + layoutRole: original.layoutRole, + sourceLineCount: original.sourceLineCount + ) + } + + static func visualReviewedBlockForDiagnostics( + original: TextBlock, + review: VisionRegionTextCandidate + ) -> TextBlock { + visualReviewedBlock(original: original, review: review) + } + + private static func looksLikeVisionNoTextResponse(_ text: String) -> Bool { + let normalized = text + .trimmingCharacters(in: .whitespacesAndNewlines) + .lowercased() + guard normalized.count <= 200 else { return false } + let refusalMarkers = [ + "无法识别", "无法读取", "无法返回", "不能识别", "不能读取", + "未检测到文字", "没有检测到文字", "未发现文字", "没有可读文字", + "抱歉，我无法", "看不到图片", "无法查看图片", + "unable to read", "unable to identify", "cannot read", "can't read", + "cannot view", "can't view", "no readable text", "no text found" + ] + return refusalMarkers.contains { normalized.contains($0) } + } + + ''' + marker + replace_once(marker, addition, 'region text helper') + + replace_once( + ''' case protocolViolation(String) + case emptyResult + ''', + ''' case protocolViolation(String) + case modelDidNotReadImage + case emptyResult + ''', + 'vision error case') + + replace_once( + ''' case .protocolViolation(let message): + return "视觉翻译协议错误：\\(message)" + case .emptyResult: + return "视觉翻译没有返回可用文本" + ''', + ''' case .protocolViolation(let message): + return "视觉翻译协议错误：\\(message)" + case .modelDidNotReadImage: + return "视觉模型没有从图片读取到文字，请确认当前模型支持图像输入" + case .emptyResult: + return "视觉翻译没有返回可用文本" + ''', + 'vision error description') + + replace_once( + ''' case .invalidJSON, .invalidCoordinates, .missingTextBox, .missingTranslation, .protocolViolation, .emptyResult: + return true + ''', + ''' case .invalidJSON, .invalidCoordinates, .missingTextBox, .missingTranslation, + .protocolViolation, .modelDidNotReadImage, .emptyResult: + return true + ''', + 'vision slice fallback') + + schema_marker = ''' private static func offlineVisionTranslationSchema() -> [String: Any] { + ''' + schema_addition = ''' private static func visionRecognitionSchema() -> [String: Any] { + let point: [String: Any] = [ + "type": "object", + "additionalProperties": false, + "required": ["x", "y"], + "properties": [ + "x": ["type": "number", "minimum": 0, "maximum": 1], + "y": ["type": "number", "minimum": 0, "maximum": 1] + ] + ] + let rect: [String: Any] = [ + "type": "object", + "additionalProperties": false, + "required": ["x", "y", "width", "height"], + "properties": [ + "x": ["type": "number", "minimum": 0, "maximum": 1], + "y": ["type": "number", "minimum": 0, "maximum": 1], + "width": ["type": "number", "exclusiveMinimum": 0, "maximum": 1], + "height": ["type": "number", "exclusiveMinimum": 0, "maximum": 1] + ] + ] + let item: [String: Any] = [ + "type": "object", + "additionalProperties": false, + "required": ["sourceText", "textBox", "confidence", "classification"], + "properties": [ + "id": ["type": "string"], + "sourceText": ["type": "string"], + "textBox": rect, + "bubbleBox": rect, + "layoutSafeRegion": rect, + "textPolygon": ["type": "array", "minItems": 4, "items": point], + "bubblePolygon": ["type": "array", "minItems": 4, "items": point], + "confidence": ["type": "number", "minimum": 0, "maximum": 1], + "classification": [ + "type": "string", + "enum": [ + "dialogue", "narration", "soundEffect", "url", "advertisement", + "watermark", "copyright", "pageNumber" + ] + ] + ] + ] + return [ + "type": "object", + "additionalProperties": false, + "required": ["coordinateSpace", "items"], + "properties": [ + "coordinateSpace": ["type": "string", "enum": ["normalized"]], + "items": ["type": "array", "items": item] + ] + ] + } + + ''' + schema_marker + replace_once(schema_marker, schema_addition, 'recognition schema definition') + + path.write_text(text) + + test_path = Path('mreaderTests/TranslationComicIntegrationRegressionTests.swift') + tests = test_path.read_text() + insertion = ''' + func testVisionRegionTextParserAcceptsCoordinateFreeJSON() throws { + let result = try XCTUnwrap( + AITranslator.parseVisionRegionTextCandidateForDiagnostics( + from: #"{"sourceText":"ウィキペディアに","confidence":0.93}"# + ) + ) + XCTAssertEqual(result.text, "ウィキペディアに") + XCTAssertEqual(result.confidence, 0.93, accuracy: 0.0001) + } + + func testVisionRegionTextParserAcceptsPlainTextFallback() throws { + let result = try XCTUnwrap( + AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: "有名人です。") + ) + XCTAssertEqual(result.text, "有名人です。") + XCTAssertEqual(result.confidence, 0.6, accuracy: 0.0001) + } + + func testVisionRegionTextParserRejectsModelRefusal() { + XCTAssertNil( + AITranslator.parseVisionRegionTextCandidateForDiagnostics( + from: "抱歉，我无法读取这张图片中的文字。" + ) + ) + } + + func testVisualTextReviewPreservesLocalOCRGeometry() { + let original = TextBlock( + text: "OEIIII", + boundingBox: CGRect(x: 0.30, y: 0.20, width: 0.08, height: 0.24), + confidence: 0.3, + ocrSource: "original:manual", + estimatedFontScale: 0.08, + bubbleBox: CGRect(x: 0.28, y: 0.18, width: 0.14, height: 0.30), + layoutSafeRegion: CGRect(x: 0.29, y: 0.19, width: 0.12, height: 0.28), + textOrientation: .vertical, + layoutRole: .dialogue + ) + let reviewed = AITranslator.visualReviewedBlockForDiagnostics( + original: original, + review: VisionRegionTextCandidate(text: "ウィキペディアに", confidence: 0.91) + ) + + XCTAssertEqual(reviewed.text, "ウィキペディアに") + XCTAssertEqual(reviewed.boundingBox, original.boundingBox) + XCTAssertEqual(reviewed.bubbleBox, original.bubbleBox) + XCTAssertEqual(reviewed.layoutSafeRegion, original.layoutSafeRegion) + XCTAssertEqual(reviewed.textOrientation, original.textOrientation) + XCTAssertEqual(reviewed.layoutRole, original.layoutRole) + XCTAssertEqual(reviewed.ocrSource, "visual-review-text") + XCTAssertFalse(reviewed.isFiltered) + } + + func testVisionRecognitionPromptIncludesLayoutSafeRegion() { + let prompt = AITranslator.visionRecognitionPromptForDiagnostics(isRightToLeft: true) + XCTAssertTrue(prompt.contains("layoutSafeRegion")) + } + ''' + close = '\n}' + if not tests.endswith(close): + raise SystemExit('test file closing brace not found') + tests = tests[:-len(close)] + insertion + close + test_path.write_text(tests) + PY + + git diff --check + git diff --stat + - name: Commit patch + shell: bash + run: | + if git diff --quiet; then + echo "No patch changes needed" + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add mreader/AITranslator.swift mreaderTests/TranslationComicIntegrationRegressionTests.swift + git commit -m "fix: make visual OCR review text-first" + git push origin HEAD:codex/visual-ocr-reliability From 1cbf80994ad6dcc4cf4df3288592533b66c8e1fc Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:45:06 +0800 Subject: [PATCH 04/43] chore: reset visual OCR patch workflow v2 --- .../apply-visual-ocr-reliability-v2.yml | 519 ------------------ 1 file changed, 519 deletions(-) delete mode 100644 .github/workflows/apply-visual-ocr-reliability-v2.yml diff --git a/.github/workflows/apply-visual-ocr-reliability-v2.yml b/.github/workflows/apply-visual-ocr-reliability-v2.yml deleted file mode 100644 index 568e986..0000000 --- a/.github/workflows/apply-visual-ocr-reliability-v2.yml +++ /dev/null @@ -1,519 +0,0 @@ -name: Apply visual OCR reliability patch v2 - -on: - push: - branches: - - codex/visual-ocr-reliability - -permissions: - contents: write - -jobs: - patch: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: codex/visual-ocr-reliability - fetch-depth: 0 - - name: Patch visual OCR pipeline - shell: bash - run: | - python3 - <<'PY' - from pathlib import Path - - path = Path('mreader/AITranslator.swift') - text = path.read_text() - - def replace_once(old, new, label): - global text - count = text.count(old) - if count != 1: - raise SystemExit(f'{label}: expected 1 match, found {count}') - text = text.replace(old, new, 1) - - replace_once( - '''nonisolated struct OCRVerificationRegion: Sendable { - let blockID: UUID - let sourceRect: CGRect - } - ''', - '''nonisolated struct OCRVerificationRegion: Sendable { - let blockID: UUID - let sourceRect: CGRect - } - - nonisolated struct VisionRegionTextCandidate: Equatable, Sendable { - let text: String - let confidence: Double - } - ''', - 'region candidate struct') - - replace_once( - ''' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up) - let localBlocks = try await recognizeVisionPage( - image: cropImage, - apiKey: apiKey, - baseURL: baseURL, - model: model, - isRightToLeft: isRightToLeft, - viewportAspect: max(cropImage.size.height / max(cropImage.size.width, 1), 1.25), - modelDescriptor: modelDescriptor ?? AIModelProtocolCatalog.descriptor(for: model) - ) - let original = corrected[originalIndex] - guard let match = visualVerificationMatch( - for: original, - candidates: localBlocks, - sourceRect: region.sourceRect - ) else { - continue - } - let best = match.block - let correctedText = best.text.trimmingCharacters(in: .whitespacesAndNewlines) - guard !correctedText.isEmpty else { continue } - let correctedBox = match.pageBoundingBox - let correctedFontScale = visualVerificationMappedFontScale( - for: best, - sourceRect: region.sourceRect, - correctedBox: correctedBox - ) - let bubbleGeometry = visualVerificationMappedBubbleGeometry( - for: best, - sourceRect: region.sourceRect, - correctedBox: correctedBox - ) - corrected[originalIndex] = TextBlock( - id: original.id, - text: correctedText, - boundingBox: correctedBox, - translation: original.translation, - confidence: max(original.confidence, best.confidence), - ocrSource: "visual-review", - // A rejected/uncertain local candidate that passed a - // visual text+geometry match is explicitly recovered. - isFiltered: false, - filterReason: nil, - estimatedFontScale: correctedFontScale, - textColorHex: original.textColorHex, - bubbleBox: bubbleGeometry.bubbleBox, - layoutSafeRegion: original.layoutSafeRegion ?? bubbleGeometry.bubbleBox, - polygon: original.polygon, - bubblePolygon: bubbleGeometry.bubblePolygon, - translationLines: original.translationLines, - textOrientation: best.textOrientation, - layoutRole: original.layoutRole == .standalone || best.layoutRole == .standalone - ? .standalone - : .dialogue, - sourceLineCount: original.sourceLineCount - ) - print("MReader OCR visual review corrected block=\\(region.blockID) confidence=\\(String(format: \"%.2f\", best.confidence))") - ''', - ''' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up) - let review = try await recognizeVisionRegionText( - image: cropImage, - apiKey: apiKey, - baseURL: baseURL, - model: model, - modelDescriptor: modelDescriptor ?? AIModelProtocolCatalog.descriptor(for: model) - ) - let original = corrected[originalIndex] - corrected[originalIndex] = visualReviewedBlock(original: original, review: review) - print("MReader OCR visual text review corrected block=\\(region.blockID) confidence=\\(String(format: \"%.2f\", review.confidence))") - ''', - 'block visual review') - - replace_once( - ''' systemPrompt = "你只做漫画图片中的文字识别、断句、翻译和精确坐标标注。只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" - ''', - ''' systemPrompt = "你只做漫画图片中的文字识别、断句、翻译和精确坐标标注。只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" - ''', - 'translation system prompt') - - replace_once( - ''' systemPrompt = "你只做漫画图片中文字识别、断句和精确坐标标注，不要翻译。只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" - ''', - ''' systemPrompt = "你只做漫画图片中文字识别、断句和精确坐标标注，不要翻译。只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" - ''', - 'recognition system prompt') - - replace_once( - ''' let defaultMode: VisionResponseFormatMode = usesTranslationSchema ? .jsonSchema : .jsonObject - ''', - ''' let defaultMode: VisionResponseFormatMode = .jsonSchema - ''', - 'recognition schema default') - - replace_once( - ''' case .jsonSchema: - guard usesTranslationSchema else { - return .jsonObject - } - guard let schema = try? JSONSerialization.data( - withJSONObject: offlineVisionTranslationSchema() - ) else { - return nil - } - return .jsonSchema(name: "manga_offline_translation", schema: schema) - ''', - ''' case .jsonSchema: - let schemaObject = usesTranslationSchema - ? offlineVisionTranslationSchema() - : visionRecognitionSchema() - guard let schema = try? JSONSerialization.data(withJSONObject: schemaObject) else { - return nil - } - return .jsonSchema( - name: usesTranslationSchema ? "manga_offline_translation" : "manga_vision_recognition", - schema: schema - ) - ''', - 'recognition response schema') - - marker = ''' private static func recognizeVisionImage(image: UIImage, sourceRect: CGRect, apiKey: String, baseURL: String, model: String, modelDescriptor: AIModelDescriptor, isRightToLeft: Bool, additionalInstructions: String, translationTarget: TranslationTargetLanguage?, translationPromptTemplate: String, strictTranslationGeometry: Bool) async throws -> [TextBlock] { - ''' - addition = ''' private static func recognizeVisionRegionText( - image: UIImage, - apiKey: String, - baseURL: String, - model: String, - modelDescriptor: AIModelDescriptor - ) async throws -> VisionRegionTextCandidate { - guard !apiKey.isEmpty else { throw VisionTranslationError.api("未配置 API Key") } - guard !model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - throw VisionTranslationError.api("未配置模型") - } - let preparedImage = resizedImageForVision(image, maxDimension: 1536) - guard let imageDataURL = encodedVisionImageDataURL(preparedImage) else { - throw VisionTranslationError.imageEncodingFailed - } - let systemPrompt = "你只做漫画局部图片的原文转录。不要翻译，不要返回坐标，不要描述人物或画面，不要输出思考过程。只返回 sourceText 和 confidence。" - let prompt = """ - 这是已经由本地 OCR 定位好的单个漫画文字区域裁剪。 - 请只根据图片逐字抄录实际可见的原文，不要猜测裁剪外内容，不要翻译，不要补剧情。 - 保留原有标点、数字、拉长音、小假名和大小写；竖排文字按自然阅读顺序合并成一个字符串。 - 不需要返回任何坐标。 - 只输出 JSON：{"sourceText":"图中原文","confidence":0.95} - 如果图片中确实没有可读文字，输出 {"sourceText":"","confidence":0.0}。 - """ - - let data: Data - do { - data = try await visionCompletionData( - apiKey: apiKey, - baseURL: baseURL, - model: model, - modelDescriptor: modelDescriptor, - systemPrompt: systemPrompt, - prompt: prompt, - imageDataURL: imageDataURL, - responseFormat: .jsonObject - ) - } catch { - guard isUnsupportedResponseFormat(error) else { throw error } - data = try await visionCompletionData( - apiKey: apiKey, - baseURL: baseURL, - model: model, - modelDescriptor: modelDescriptor, - systemPrompt: systemPrompt, - prompt: prompt, - imageDataURL: imageDataURL, - responseFormat: nil - ) - } - - guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let content = assistantContent(from: json) else { - throw VisionTranslationError.invalidJSON - } - if looksLikeVisionNoTextResponse(content) { - throw VisionTranslationError.modelDidNotReadImage - } - guard let candidate = parseVisionRegionTextCandidate(from: content) else { - throw VisionTranslationError.emptyResult - } - return candidate - } - - private static func parseVisionRegionTextCandidate(from content: String) -> VisionRegionTextCandidate? { - if let data = normalizedVisionJSONData(from: content), - let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) { - let item: [String: Any]? - if let dictionary = object as? [String: Any] { - if firstString( - in: dictionary, - keys: ["sourceText", "source_text", "text", "original", "originalText"] - ).isEmpty, - let first = (dictionary["items"] as? [[String: Any]])?.first { - item = first - } else { - item = dictionary - } - } else if let array = object as? [[String: Any]] { - item = array.first - } else { - item = nil - } - if let item { - let text = firstString( - in: item, - keys: ["sourceText", "source_text", "text", "original", "originalText"] - ).trimmingCharacters(in: .whitespacesAndNewlines) - guard !text.isEmpty, !looksLikeVisionNoTextResponse(text) else { return nil } - let confidence = min(max(doubleValue(from: item["confidence"]) ?? 0.75, 0), 1) - return VisionRegionTextCandidate(text: text, confidence: confidence) - } - return nil - } - - var plain = extractedJSONPayload(from: content) - .replacingOccurrences( - of: #"^```(?:text|markdown)?\\s*|\\s*```$"#, - with: "", - options: [.regularExpression, .caseInsensitive] - ) - .trimmingCharacters(in: .whitespacesAndNewlines) - if plain.hasPrefix("\"") && plain.hasSuffix("\"") && plain.count >= 2 { - plain.removeFirst() - plain.removeLast() - } - guard !plain.isEmpty, !looksLikeVisionNoTextResponse(plain) else { return nil } - return VisionRegionTextCandidate(text: plain, confidence: 0.6) - } - - static func parseVisionRegionTextCandidateForDiagnostics( - from content: String - ) -> VisionRegionTextCandidate? { - parseVisionRegionTextCandidate(from: content) - } - - private static func visualReviewedBlock( - original: TextBlock, - review: VisionRegionTextCandidate - ) -> TextBlock { - TextBlock( - id: original.id, - text: review.text.trimmingCharacters(in: .whitespacesAndNewlines), - boundingBox: original.boundingBox, - translation: original.translation, - confidence: max(original.confidence, review.confidence), - ocrSource: "visual-review-text", - isFiltered: false, - filterReason: nil, - estimatedFontScale: original.estimatedFontScale, - textColorHex: original.textColorHex, - bubbleBox: original.bubbleBox, - layoutSafeRegion: original.layoutSafeRegion, - polygon: original.polygon, - bubblePolygon: original.bubblePolygon, - translationLines: original.translationLines, - textOrientation: original.textOrientation, - layoutRole: original.layoutRole, - sourceLineCount: original.sourceLineCount - ) - } - - static func visualReviewedBlockForDiagnostics( - original: TextBlock, - review: VisionRegionTextCandidate - ) -> TextBlock { - visualReviewedBlock(original: original, review: review) - } - - private static func looksLikeVisionNoTextResponse(_ text: String) -> Bool { - let normalized = text - .trimmingCharacters(in: .whitespacesAndNewlines) - .lowercased() - guard normalized.count <= 200 else { return false } - let refusalMarkers = [ - "无法识别", "无法读取", "无法返回", "不能识别", "不能读取", - "未检测到文字", "没有检测到文字", "未发现文字", "没有可读文字", - "抱歉，我无法", "看不到图片", "无法查看图片", - "unable to read", "unable to identify", "cannot read", "can't read", - "cannot view", "can't view", "no readable text", "no text found" - ] - return refusalMarkers.contains { normalized.contains($0) } - } - - ''' + marker - replace_once(marker, addition, 'region text helper') - - replace_once( - ''' case protocolViolation(String) - case emptyResult - ''', - ''' case protocolViolation(String) - case modelDidNotReadImage - case emptyResult - ''', - 'vision error case') - - replace_once( - ''' case .protocolViolation(let message): - return "视觉翻译协议错误：\\(message)" - case .emptyResult: - return "视觉翻译没有返回可用文本" - ''', - ''' case .protocolViolation(let message): - return "视觉翻译协议错误：\\(message)" - case .modelDidNotReadImage: - return "视觉模型没有从图片读取到文字，请确认当前模型支持图像输入" - case .emptyResult: - return "视觉翻译没有返回可用文本" - ''', - 'vision error description') - - replace_once( - ''' case .invalidJSON, .invalidCoordinates, .missingTextBox, .missingTranslation, .protocolViolation, .emptyResult: - return true - ''', - ''' case .invalidJSON, .invalidCoordinates, .missingTextBox, .missingTranslation, - .protocolViolation, .modelDidNotReadImage, .emptyResult: - return true - ''', - 'vision slice fallback') - - schema_marker = ''' private static func offlineVisionTranslationSchema() -> [String: Any] { - ''' - schema_addition = ''' private static func visionRecognitionSchema() -> [String: Any] { - let point: [String: Any] = [ - "type": "object", - "additionalProperties": false, - "required": ["x", "y"], - "properties": [ - "x": ["type": "number", "minimum": 0, "maximum": 1], - "y": ["type": "number", "minimum": 0, "maximum": 1] - ] - ] - let rect: [String: Any] = [ - "type": "object", - "additionalProperties": false, - "required": ["x", "y", "width", "height"], - "properties": [ - "x": ["type": "number", "minimum": 0, "maximum": 1], - "y": ["type": "number", "minimum": 0, "maximum": 1], - "width": ["type": "number", "exclusiveMinimum": 0, "maximum": 1], - "height": ["type": "number", "exclusiveMinimum": 0, "maximum": 1] - ] - ] - let item: [String: Any] = [ - "type": "object", - "additionalProperties": false, - "required": ["sourceText", "textBox", "confidence", "classification"], - "properties": [ - "id": ["type": "string"], - "sourceText": ["type": "string"], - "textBox": rect, - "bubbleBox": rect, - "layoutSafeRegion": rect, - "textPolygon": ["type": "array", "minItems": 4, "items": point], - "bubblePolygon": ["type": "array", "minItems": 4, "items": point], - "confidence": ["type": "number", "minimum": 0, "maximum": 1], - "classification": [ - "type": "string", - "enum": [ - "dialogue", "narration", "soundEffect", "url", "advertisement", - "watermark", "copyright", "pageNumber" - ] - ] - ] - ] - return [ - "type": "object", - "additionalProperties": false, - "required": ["coordinateSpace", "items"], - "properties": [ - "coordinateSpace": ["type": "string", "enum": ["normalized"]], - "items": ["type": "array", "items": item] - ] - ] - } - - ''' + schema_marker - replace_once(schema_marker, schema_addition, 'recognition schema definition') - - path.write_text(text) - - test_path = Path('mreaderTests/TranslationComicIntegrationRegressionTests.swift') - tests = test_path.read_text() - insertion = ''' - func testVisionRegionTextParserAcceptsCoordinateFreeJSON() throws { - let result = try XCTUnwrap( - AITranslator.parseVisionRegionTextCandidateForDiagnostics( - from: #"{"sourceText":"ウィキペディアに","confidence":0.93}"# - ) - ) - XCTAssertEqual(result.text, "ウィキペディアに") - XCTAssertEqual(result.confidence, 0.93, accuracy: 0.0001) - } - - func testVisionRegionTextParserAcceptsPlainTextFallback() throws { - let result = try XCTUnwrap( - AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: "有名人です。") - ) - XCTAssertEqual(result.text, "有名人です。") - XCTAssertEqual(result.confidence, 0.6, accuracy: 0.0001) - } - - func testVisionRegionTextParserRejectsModelRefusal() { - XCTAssertNil( - AITranslator.parseVisionRegionTextCandidateForDiagnostics( - from: "抱歉，我无法读取这张图片中的文字。" - ) - ) - } - - func testVisualTextReviewPreservesLocalOCRGeometry() { - let original = TextBlock( - text: "OEIIII", - boundingBox: CGRect(x: 0.30, y: 0.20, width: 0.08, height: 0.24), - confidence: 0.3, - ocrSource: "original:manual", - estimatedFontScale: 0.08, - bubbleBox: CGRect(x: 0.28, y: 0.18, width: 0.14, height: 0.30), - layoutSafeRegion: CGRect(x: 0.29, y: 0.19, width: 0.12, height: 0.28), - textOrientation: .vertical, - layoutRole: .dialogue - ) - let reviewed = AITranslator.visualReviewedBlockForDiagnostics( - original: original, - review: VisionRegionTextCandidate(text: "ウィキペディアに", confidence: 0.91) - ) - - XCTAssertEqual(reviewed.text, "ウィキペディアに") - XCTAssertEqual(reviewed.boundingBox, original.boundingBox) - XCTAssertEqual(reviewed.bubbleBox, original.bubbleBox) - XCTAssertEqual(reviewed.layoutSafeRegion, original.layoutSafeRegion) - XCTAssertEqual(reviewed.textOrientation, original.textOrientation) - XCTAssertEqual(reviewed.layoutRole, original.layoutRole) - XCTAssertEqual(reviewed.ocrSource, "visual-review-text") - XCTAssertFalse(reviewed.isFiltered) - } - - func testVisionRecognitionPromptIncludesLayoutSafeRegion() { - let prompt = AITranslator.visionRecognitionPromptForDiagnostics(isRightToLeft: true) - XCTAssertTrue(prompt.contains("layoutSafeRegion")) - } - ''' - close = '\n}' - if not tests.endswith(close): - raise SystemExit('test file closing brace not found') - tests = tests[:-len(close)] + insertion + close - test_path.write_text(tests) - PY - - git diff --check - git diff --stat - - name: Commit patch - shell: bash - run: | - if git diff --quiet; then - echo "No patch changes needed" - exit 0 - fi - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add mreader/AITranslator.swift mreaderTests/TranslationComicIntegrationRegressionTests.swift - git commit -m "fix: make visual OCR review text-first" - git push origin HEAD:codex/visual-ocr-reliability From ba126a26e6a58436407d8916a71763fc4d2beb08 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:46:02 +0800 Subject: [PATCH 05/43] chore: apply visual OCR reliability patch v3 --- .../apply-visual-ocr-reliability-v3.yml | 426 ++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100644 .github/workflows/apply-visual-ocr-reliability-v3.yml diff --git a/.github/workflows/apply-visual-ocr-reliability-v3.yml b/.github/workflows/apply-visual-ocr-reliability-v3.yml new file mode 100644 index 0000000..bd1fdc3 --- /dev/null +++ b/.github/workflows/apply-visual-ocr-reliability-v3.yml @@ -0,0 +1,426 @@ +name: Apply visual OCR reliability patch v3 + +on: + push: + branches: + - codex/visual-ocr-reliability + +permissions: + contents: write + +jobs: + patch: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: codex/visual-ocr-reliability + fetch-depth: 0 + - name: Patch visual OCR pipeline + shell: bash + run: | + python3 - <<'PY' + from pathlib import Path + import re + + path = Path('mreader/AITranslator.swift') + text = path.read_text() + + def one(old, new, label): + global text + count = text.count(old) + if count != 1: + raise SystemExit(f'{label}: expected 1 match, found {count}') + text = text.replace(old, new, 1) + + one( + '''nonisolated struct OCRVerificationRegion: Sendable { + let blockID: UUID + let sourceRect: CGRect + } + ''', + '''nonisolated struct OCRVerificationRegion: Sendable { + let blockID: UUID + let sourceRect: CGRect + } + + nonisolated struct VisionRegionTextCandidate: Equatable, Sendable { + let text: String + let confidence: Double + } + ''', + 'VisionRegionTextCandidate') + + start = text.index(' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up)') + end_marker = ' print("MReader OCR visual review corrected block=\\(region.blockID) confidence=\\(String(format: \\"%.2f\\", best.confidence))")\n' + end = text.index(end_marker, start) + len(end_marker) + replacement = ''' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up) + let review = try await recognizeVisionRegionText( + image: cropImage, + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor ?? AIModelProtocolCatalog.descriptor(for: model) + ) + let original = corrected[originalIndex] + corrected[originalIndex] = visualReviewedBlock( + original: original, + review: review + ) + print("MReader OCR visual text review corrected block=\\(region.blockID) confidence=\\(String(format: \"%.2f\", review.confidence))") + ''' + text = text[:start] + replacement + text[end:] + + one( + '只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', + '只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', + 'translation system fields') + one( + '只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', + '只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', + 'recognition system fields') + + one( + ' let defaultMode: VisionResponseFormatMode = usesTranslationSchema ? .jsonSchema : .jsonObject\n', + ' let defaultMode: VisionResponseFormatMode = .jsonSchema\n', + 'recognition default format') + + old_schema_switch = ''' case .jsonSchema: + guard usesTranslationSchema else { + return .jsonObject + } + guard let schema = try? JSONSerialization.data( + withJSONObject: offlineVisionTranslationSchema() + ) else { + return nil + } + return .jsonSchema(name: "manga_offline_translation", schema: schema) + ''' + new_schema_switch = ''' case .jsonSchema: + let schemaObject = usesTranslationSchema + ? offlineVisionTranslationSchema() + : visionRecognitionSchema() + guard let schema = try? JSONSerialization.data(withJSONObject: schemaObject) else { + return nil + } + return .jsonSchema( + name: usesTranslationSchema ? "manga_offline_translation" : "manga_vision_recognition", + schema: schema + ) + ''' + one(old_schema_switch, new_schema_switch, 'recognition schema switch') + + marker = ' private static func recognizeVisionImage(image: UIImage, sourceRect: CGRect, apiKey: String, baseURL: String, model: String, modelDescriptor: AIModelDescriptor, isRightToLeft: Bool, additionalInstructions: String, translationTarget: TranslationTargetLanguage?, translationPromptTemplate: String, strictTranslationGeometry: Bool) async throws -> [TextBlock] {\n' + if text.count(marker) != 1: + raise SystemExit(f'recognizeVisionImage marker count={text.count(marker)}') + helper = ''' private static func recognizeVisionRegionText( + image: UIImage, + apiKey: String, + baseURL: String, + model: String, + modelDescriptor: AIModelDescriptor + ) async throws -> VisionRegionTextCandidate { + guard !apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + throw VisionTranslationError.api("未配置 API Key") + } + guard !model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + throw VisionTranslationError.api("未配置模型") + } + let preparedImage = resizedImageForVision(image, maxDimension: 1536) + guard let imageDataURL = encodedVisionImageDataURL(preparedImage) else { + throw VisionTranslationError.imageEncodingFailed + } + let systemPrompt = "你只做漫画局部图片的原文转录。不要翻译，不要返回坐标，不要描述人物或画面，不要输出思考过程。只返回 sourceText 和 confidence。" + let prompt = """ + 这是已经由本地 OCR 定位好的单个漫画文字区域裁剪。 + 请只根据图片逐字抄录实际可见的原文，不要猜测裁剪外内容，不要翻译，不要补剧情。 + 保留原有标点、数字、拉长音、小假名和大小写；竖排文字按自然阅读顺序合并成一个字符串。 + 不需要返回任何坐标。 + 只输出 JSON：{"sourceText":"图中原文","confidence":0.95} + 如果图片中确实没有可读文字，输出 {"sourceText":"","confidence":0.0}。 + """ + + let data: Data + do { + data = try await visionCompletionData( + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor, + systemPrompt: systemPrompt, + prompt: prompt, + imageDataURL: imageDataURL, + responseFormat: .jsonObject + ) + } catch { + guard isUnsupportedResponseFormat(error) else { throw error } + data = try await visionCompletionData( + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor, + systemPrompt: systemPrompt, + prompt: prompt, + imageDataURL: imageDataURL, + responseFormat: nil + ) + } + + guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let content = assistantContent(from: json), + let candidate = parseVisionRegionTextCandidate(from: content) else { + throw VisionTranslationError.emptyResult + } + return candidate + } + + private static func parseVisionRegionTextCandidate( + from content: String + ) -> VisionRegionTextCandidate? { + if let data = normalizedVisionJSONData(from: content), + let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) { + let item: [String: Any]? + if let dictionary = object as? [String: Any] { + let direct = firstString( + in: dictionary, + keys: ["sourceText", "source_text", "text", "original", "originalText"] + ) + if direct.isEmpty, + let first = (dictionary["items"] as? [[String: Any]])?.first { + item = first + } else { + item = dictionary + } + } else if let array = object as? [[String: Any]] { + item = array.first + } else { + item = nil + } + guard let item else { return nil } + let value = firstString( + in: item, + keys: ["sourceText", "source_text", "text", "original", "originalText"] + ).trimmingCharacters(in: .whitespacesAndNewlines) + guard !value.isEmpty, !looksLikeVisionRefusal(value) else { return nil } + let confidence = min(max(doubleValue(from: item["confidence"]) ?? 0.75, 0), 1) + return VisionRegionTextCandidate(text: value, confidence: confidence) + } + + var plain = content + .replacingOccurrences( + of: #"^```(?:text|markdown)?\\s*|\\s*```$"#, + with: "", + options: [.regularExpression, .caseInsensitive] + ) + .trimmingCharacters(in: .whitespacesAndNewlines) + if plain.hasPrefix("\"") && plain.hasSuffix("\"") && plain.count >= 2 { + plain.removeFirst() + plain.removeLast() + } + guard !plain.isEmpty, !looksLikeVisionRefusal(plain) else { return nil } + return VisionRegionTextCandidate(text: plain, confidence: 0.6) + } + + private static func looksLikeVisionRefusal(_ text: String) -> Bool { + let normalized = text.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + guard normalized.count <= 240 else { return false } + let markers = [ + "无法识别", "无法读取", "无法返回", "不能识别", "不能读取", + "未检测到文字", "没有检测到文字", "未发现文字", "没有可读文字", + "抱歉，我无法", "看不到图片", "无法查看图片", + "unable to read", "unable to identify", "cannot read", "can't read", + "cannot view", "can't view", "no readable text", "no text found" + ] + return markers.contains { normalized.contains($0) } + } + + private static func visualReviewedBlock( + original: TextBlock, + review: VisionRegionTextCandidate + ) -> TextBlock { + TextBlock( + id: original.id, + text: review.text.trimmingCharacters(in: .whitespacesAndNewlines), + boundingBox: original.boundingBox, + translation: original.translation, + confidence: max(original.confidence, review.confidence), + ocrSource: "visual-review-text", + isFiltered: false, + filterReason: nil, + estimatedFontScale: original.estimatedFontScale, + textColorHex: original.textColorHex, + bubbleBox: original.bubbleBox, + layoutSafeRegion: original.layoutSafeRegion, + polygon: original.polygon, + bubblePolygon: original.bubblePolygon, + translationLines: original.translationLines, + textOrientation: original.textOrientation, + layoutRole: original.layoutRole, + sourceLineCount: original.sourceLineCount + ) + } + + static func parseVisionRegionTextCandidateForDiagnostics( + from content: String + ) -> VisionRegionTextCandidate? { + parseVisionRegionTextCandidate(from: content) + } + + static func visualReviewedBlockForDiagnostics( + original: TextBlock, + review: VisionRegionTextCandidate + ) -> TextBlock { + visualReviewedBlock(original: original, review: review) + } + + ''' + text = text.replace(marker, helper + marker, 1) + + schema_marker = ' private static func offlineVisionTranslationSchema() -> [String: Any] {\n' + if text.count(schema_marker) != 1: + raise SystemExit(f'offlineVisionTranslationSchema marker count={text.count(schema_marker)}') + recognition_schema = ''' private static func visionRecognitionSchema() -> [String: Any] { + let point: [String: Any] = [ + "type": "object", + "additionalProperties": false, + "required": ["x", "y"], + "properties": [ + "x": ["type": "number", "minimum": 0, "maximum": 1], + "y": ["type": "number", "minimum": 0, "maximum": 1] + ] + ] + let rect: [String: Any] = [ + "type": "object", + "additionalProperties": false, + "required": ["x", "y", "width", "height"], + "properties": [ + "x": ["type": "number", "minimum": 0, "maximum": 1], + "y": ["type": "number", "minimum": 0, "maximum": 1], + "width": ["type": "number", "exclusiveMinimum": 0, "maximum": 1], + "height": ["type": "number", "exclusiveMinimum": 0, "maximum": 1] + ] + ] + let nullableRect: [String: Any] = ["anyOf": [rect, ["type": "null"]]] + let polygon: [String: Any] = ["type": "array", "items": point] + let item: [String: Any] = [ + "type": "object", + "additionalProperties": false, + "required": [ + "id", "sourceText", "textBox", "bubbleBox", "layoutSafeRegion", + "textPolygon", "bubblePolygon", "confidence", "classification" + ], + "properties": [ + "id": ["type": "string"], + "sourceText": ["type": "string"], + "textBox": rect, + "bubbleBox": nullableRect, + "layoutSafeRegion": nullableRect, + "textPolygon": polygon, + "bubblePolygon": polygon, + "confidence": ["type": "number", "minimum": 0, "maximum": 1], + "classification": [ + "type": "string", + "enum": [ + "dialogue", "narration", "soundEffect", "url", "advertisement", + "watermark", "copyright", "pageNumber" + ] + ] + ] + ] + return [ + "type": "object", + "additionalProperties": false, + "required": ["coordinateSpace", "items"], + "properties": [ + "coordinateSpace": ["type": "string", "enum": ["normalized"]], + "items": ["type": "array", "items": item] + ] + ] + } + + ''' + text = text.replace(schema_marker, recognition_schema + schema_marker, 1) + path.write_text(text) + + test_path = Path('mreaderTests/TranslationComicIntegrationRegressionTests.swift') + tests = test_path.read_text() + insert = ''' + func testVisionRegionTextParserAcceptsCoordinateFreeJSON() throws { + let result = try XCTUnwrap( + AITranslator.parseVisionRegionTextCandidateForDiagnostics( + from: #"{"sourceText":"ウィキペディアに","confidence":0.93}"# + ) + ) + XCTAssertEqual(result.text, "ウィキペディアに") + XCTAssertEqual(result.confidence, 0.93, accuracy: 0.0001) + } + + func testVisionRegionTextParserAcceptsPlainTextFallback() throws { + let result = try XCTUnwrap( + AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: "有名人です。") + ) + XCTAssertEqual(result.text, "有名人です。") + XCTAssertEqual(result.confidence, 0.6, accuracy: 0.0001) + } + + func testVisionRegionTextParserRejectsRefusalInsteadOfTreatingItAsOCR() { + XCTAssertNil( + AITranslator.parseVisionRegionTextCandidateForDiagnostics( + from: "抱歉，我无法读取这张图片中的文字。" + ) + ) + XCTAssertNil( + AITranslator.parseVisionRegionTextCandidateForDiagnostics( + from: "无法返回文本" + ) + ) + } + + func testVisualTextReviewPreservesLocalOCRGeometry() { + let original = TextBlock( + text: "OEIIII", + boundingBox: CGRect(x: 0.30, y: 0.20, width: 0.08, height: 0.24), + confidence: 0.3, + ocrSource: "original:manual", + estimatedFontScale: 0.08, + bubbleBox: CGRect(x: 0.28, y: 0.18, width: 0.14, height: 0.30), + layoutSafeRegion: CGRect(x: 0.29, y: 0.19, width: 0.12, height: 0.28), + textOrientation: .vertical, + layoutRole: .dialogue + ) + let reviewed = AITranslator.visualReviewedBlockForDiagnostics( + original: original, + review: VisionRegionTextCandidate(text: "ウィキペディアに", confidence: 0.91) + ) + XCTAssertEqual(reviewed.text, "ウィキペディアに") + XCTAssertEqual(reviewed.boundingBox, original.boundingBox) + XCTAssertEqual(reviewed.bubbleBox, original.bubbleBox) + XCTAssertEqual(reviewed.layoutSafeRegion, original.layoutSafeRegion) + XCTAssertEqual(reviewed.textOrientation, original.textOrientation) + XCTAssertEqual(reviewed.layoutRole, original.layoutRole) + XCTAssertEqual(reviewed.ocrSource, "visual-review-text") + XCTAssertFalse(reviewed.isFiltered) + } + + func testVisionRecognitionPromptIncludesLayoutSafeRegion() { + let prompt = AITranslator.visionRecognitionPromptForDiagnostics(isRightToLeft: true) + XCTAssertTrue(prompt.contains("layoutSafeRegion")) + } + ''' + if not tests.endswith('\n}'): + raise SystemExit('test file closing brace not found') + tests = tests[:-2] + insert + '\n}' + test_path.write_text(tests) + PY + + git diff --check + git diff --stat + - name: Commit product patch and remove temporary workflow + shell: bash + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add mreader/AITranslator.swift mreaderTests/TranslationComicIntegrationRegressionTests.swift + git rm .github/workflows/apply-visual-ocr-reliability-v3.yml + git commit -m "fix: make visual OCR review text-first" + git push origin HEAD:codex/visual-ocr-reliability From 16909473f0ce25c98d121f756b651510ff04e26c Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:47:01 +0800 Subject: [PATCH 06/43] chore: reset visual OCR patch workflow v3 --- .../apply-visual-ocr-reliability-v3.yml | 426 ------------------ 1 file changed, 426 deletions(-) delete mode 100644 .github/workflows/apply-visual-ocr-reliability-v3.yml diff --git a/.github/workflows/apply-visual-ocr-reliability-v3.yml b/.github/workflows/apply-visual-ocr-reliability-v3.yml deleted file mode 100644 index bd1fdc3..0000000 --- a/.github/workflows/apply-visual-ocr-reliability-v3.yml +++ /dev/null @@ -1,426 +0,0 @@ -name: Apply visual OCR reliability patch v3 - -on: - push: - branches: - - codex/visual-ocr-reliability - -permissions: - contents: write - -jobs: - patch: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: codex/visual-ocr-reliability - fetch-depth: 0 - - name: Patch visual OCR pipeline - shell: bash - run: | - python3 - <<'PY' - from pathlib import Path - import re - - path = Path('mreader/AITranslator.swift') - text = path.read_text() - - def one(old, new, label): - global text - count = text.count(old) - if count != 1: - raise SystemExit(f'{label}: expected 1 match, found {count}') - text = text.replace(old, new, 1) - - one( - '''nonisolated struct OCRVerificationRegion: Sendable { - let blockID: UUID - let sourceRect: CGRect - } - ''', - '''nonisolated struct OCRVerificationRegion: Sendable { - let blockID: UUID - let sourceRect: CGRect - } - - nonisolated struct VisionRegionTextCandidate: Equatable, Sendable { - let text: String - let confidence: Double - } - ''', - 'VisionRegionTextCandidate') - - start = text.index(' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up)') - end_marker = ' print("MReader OCR visual review corrected block=\\(region.blockID) confidence=\\(String(format: \\"%.2f\\", best.confidence))")\n' - end = text.index(end_marker, start) + len(end_marker) - replacement = ''' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up) - let review = try await recognizeVisionRegionText( - image: cropImage, - apiKey: apiKey, - baseURL: baseURL, - model: model, - modelDescriptor: modelDescriptor ?? AIModelProtocolCatalog.descriptor(for: model) - ) - let original = corrected[originalIndex] - corrected[originalIndex] = visualReviewedBlock( - original: original, - review: review - ) - print("MReader OCR visual text review corrected block=\\(region.blockID) confidence=\\(String(format: \"%.2f\", review.confidence))") - ''' - text = text[:start] + replacement + text[end:] - - one( - '只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', - '只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', - 'translation system fields') - one( - '只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', - '只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', - 'recognition system fields') - - one( - ' let defaultMode: VisionResponseFormatMode = usesTranslationSchema ? .jsonSchema : .jsonObject\n', - ' let defaultMode: VisionResponseFormatMode = .jsonSchema\n', - 'recognition default format') - - old_schema_switch = ''' case .jsonSchema: - guard usesTranslationSchema else { - return .jsonObject - } - guard let schema = try? JSONSerialization.data( - withJSONObject: offlineVisionTranslationSchema() - ) else { - return nil - } - return .jsonSchema(name: "manga_offline_translation", schema: schema) - ''' - new_schema_switch = ''' case .jsonSchema: - let schemaObject = usesTranslationSchema - ? offlineVisionTranslationSchema() - : visionRecognitionSchema() - guard let schema = try? JSONSerialization.data(withJSONObject: schemaObject) else { - return nil - } - return .jsonSchema( - name: usesTranslationSchema ? "manga_offline_translation" : "manga_vision_recognition", - schema: schema - ) - ''' - one(old_schema_switch, new_schema_switch, 'recognition schema switch') - - marker = ' private static func recognizeVisionImage(image: UIImage, sourceRect: CGRect, apiKey: String, baseURL: String, model: String, modelDescriptor: AIModelDescriptor, isRightToLeft: Bool, additionalInstructions: String, translationTarget: TranslationTargetLanguage?, translationPromptTemplate: String, strictTranslationGeometry: Bool) async throws -> [TextBlock] {\n' - if text.count(marker) != 1: - raise SystemExit(f'recognizeVisionImage marker count={text.count(marker)}') - helper = ''' private static func recognizeVisionRegionText( - image: UIImage, - apiKey: String, - baseURL: String, - model: String, - modelDescriptor: AIModelDescriptor - ) async throws -> VisionRegionTextCandidate { - guard !apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - throw VisionTranslationError.api("未配置 API Key") - } - guard !model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - throw VisionTranslationError.api("未配置模型") - } - let preparedImage = resizedImageForVision(image, maxDimension: 1536) - guard let imageDataURL = encodedVisionImageDataURL(preparedImage) else { - throw VisionTranslationError.imageEncodingFailed - } - let systemPrompt = "你只做漫画局部图片的原文转录。不要翻译，不要返回坐标，不要描述人物或画面，不要输出思考过程。只返回 sourceText 和 confidence。" - let prompt = """ - 这是已经由本地 OCR 定位好的单个漫画文字区域裁剪。 - 请只根据图片逐字抄录实际可见的原文，不要猜测裁剪外内容，不要翻译，不要补剧情。 - 保留原有标点、数字、拉长音、小假名和大小写；竖排文字按自然阅读顺序合并成一个字符串。 - 不需要返回任何坐标。 - 只输出 JSON：{"sourceText":"图中原文","confidence":0.95} - 如果图片中确实没有可读文字，输出 {"sourceText":"","confidence":0.0}。 - """ - - let data: Data - do { - data = try await visionCompletionData( - apiKey: apiKey, - baseURL: baseURL, - model: model, - modelDescriptor: modelDescriptor, - systemPrompt: systemPrompt, - prompt: prompt, - imageDataURL: imageDataURL, - responseFormat: .jsonObject - ) - } catch { - guard isUnsupportedResponseFormat(error) else { throw error } - data = try await visionCompletionData( - apiKey: apiKey, - baseURL: baseURL, - model: model, - modelDescriptor: modelDescriptor, - systemPrompt: systemPrompt, - prompt: prompt, - imageDataURL: imageDataURL, - responseFormat: nil - ) - } - - guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let content = assistantContent(from: json), - let candidate = parseVisionRegionTextCandidate(from: content) else { - throw VisionTranslationError.emptyResult - } - return candidate - } - - private static func parseVisionRegionTextCandidate( - from content: String - ) -> VisionRegionTextCandidate? { - if let data = normalizedVisionJSONData(from: content), - let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) { - let item: [String: Any]? - if let dictionary = object as? [String: Any] { - let direct = firstString( - in: dictionary, - keys: ["sourceText", "source_text", "text", "original", "originalText"] - ) - if direct.isEmpty, - let first = (dictionary["items"] as? [[String: Any]])?.first { - item = first - } else { - item = dictionary - } - } else if let array = object as? [[String: Any]] { - item = array.first - } else { - item = nil - } - guard let item else { return nil } - let value = firstString( - in: item, - keys: ["sourceText", "source_text", "text", "original", "originalText"] - ).trimmingCharacters(in: .whitespacesAndNewlines) - guard !value.isEmpty, !looksLikeVisionRefusal(value) else { return nil } - let confidence = min(max(doubleValue(from: item["confidence"]) ?? 0.75, 0), 1) - return VisionRegionTextCandidate(text: value, confidence: confidence) - } - - var plain = content - .replacingOccurrences( - of: #"^```(?:text|markdown)?\\s*|\\s*```$"#, - with: "", - options: [.regularExpression, .caseInsensitive] - ) - .trimmingCharacters(in: .whitespacesAndNewlines) - if plain.hasPrefix("\"") && plain.hasSuffix("\"") && plain.count >= 2 { - plain.removeFirst() - plain.removeLast() - } - guard !plain.isEmpty, !looksLikeVisionRefusal(plain) else { return nil } - return VisionRegionTextCandidate(text: plain, confidence: 0.6) - } - - private static func looksLikeVisionRefusal(_ text: String) -> Bool { - let normalized = text.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() - guard normalized.count <= 240 else { return false } - let markers = [ - "无法识别", "无法读取", "无法返回", "不能识别", "不能读取", - "未检测到文字", "没有检测到文字", "未发现文字", "没有可读文字", - "抱歉，我无法", "看不到图片", "无法查看图片", - "unable to read", "unable to identify", "cannot read", "can't read", - "cannot view", "can't view", "no readable text", "no text found" - ] - return markers.contains { normalized.contains($0) } - } - - private static func visualReviewedBlock( - original: TextBlock, - review: VisionRegionTextCandidate - ) -> TextBlock { - TextBlock( - id: original.id, - text: review.text.trimmingCharacters(in: .whitespacesAndNewlines), - boundingBox: original.boundingBox, - translation: original.translation, - confidence: max(original.confidence, review.confidence), - ocrSource: "visual-review-text", - isFiltered: false, - filterReason: nil, - estimatedFontScale: original.estimatedFontScale, - textColorHex: original.textColorHex, - bubbleBox: original.bubbleBox, - layoutSafeRegion: original.layoutSafeRegion, - polygon: original.polygon, - bubblePolygon: original.bubblePolygon, - translationLines: original.translationLines, - textOrientation: original.textOrientation, - layoutRole: original.layoutRole, - sourceLineCount: original.sourceLineCount - ) - } - - static func parseVisionRegionTextCandidateForDiagnostics( - from content: String - ) -> VisionRegionTextCandidate? { - parseVisionRegionTextCandidate(from: content) - } - - static func visualReviewedBlockForDiagnostics( - original: TextBlock, - review: VisionRegionTextCandidate - ) -> TextBlock { - visualReviewedBlock(original: original, review: review) - } - - ''' - text = text.replace(marker, helper + marker, 1) - - schema_marker = ' private static func offlineVisionTranslationSchema() -> [String: Any] {\n' - if text.count(schema_marker) != 1: - raise SystemExit(f'offlineVisionTranslationSchema marker count={text.count(schema_marker)}') - recognition_schema = ''' private static func visionRecognitionSchema() -> [String: Any] { - let point: [String: Any] = [ - "type": "object", - "additionalProperties": false, - "required": ["x", "y"], - "properties": [ - "x": ["type": "number", "minimum": 0, "maximum": 1], - "y": ["type": "number", "minimum": 0, "maximum": 1] - ] - ] - let rect: [String: Any] = [ - "type": "object", - "additionalProperties": false, - "required": ["x", "y", "width", "height"], - "properties": [ - "x": ["type": "number", "minimum": 0, "maximum": 1], - "y": ["type": "number", "minimum": 0, "maximum": 1], - "width": ["type": "number", "exclusiveMinimum": 0, "maximum": 1], - "height": ["type": "number", "exclusiveMinimum": 0, "maximum": 1] - ] - ] - let nullableRect: [String: Any] = ["anyOf": [rect, ["type": "null"]]] - let polygon: [String: Any] = ["type": "array", "items": point] - let item: [String: Any] = [ - "type": "object", - "additionalProperties": false, - "required": [ - "id", "sourceText", "textBox", "bubbleBox", "layoutSafeRegion", - "textPolygon", "bubblePolygon", "confidence", "classification" - ], - "properties": [ - "id": ["type": "string"], - "sourceText": ["type": "string"], - "textBox": rect, - "bubbleBox": nullableRect, - "layoutSafeRegion": nullableRect, - "textPolygon": polygon, - "bubblePolygon": polygon, - "confidence": ["type": "number", "minimum": 0, "maximum": 1], - "classification": [ - "type": "string", - "enum": [ - "dialogue", "narration", "soundEffect", "url", "advertisement", - "watermark", "copyright", "pageNumber" - ] - ] - ] - ] - return [ - "type": "object", - "additionalProperties": false, - "required": ["coordinateSpace", "items"], - "properties": [ - "coordinateSpace": ["type": "string", "enum": ["normalized"]], - "items": ["type": "array", "items": item] - ] - ] - } - - ''' - text = text.replace(schema_marker, recognition_schema + schema_marker, 1) - path.write_text(text) - - test_path = Path('mreaderTests/TranslationComicIntegrationRegressionTests.swift') - tests = test_path.read_text() - insert = ''' - func testVisionRegionTextParserAcceptsCoordinateFreeJSON() throws { - let result = try XCTUnwrap( - AITranslator.parseVisionRegionTextCandidateForDiagnostics( - from: #"{"sourceText":"ウィキペディアに","confidence":0.93}"# - ) - ) - XCTAssertEqual(result.text, "ウィキペディアに") - XCTAssertEqual(result.confidence, 0.93, accuracy: 0.0001) - } - - func testVisionRegionTextParserAcceptsPlainTextFallback() throws { - let result = try XCTUnwrap( - AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: "有名人です。") - ) - XCTAssertEqual(result.text, "有名人です。") - XCTAssertEqual(result.confidence, 0.6, accuracy: 0.0001) - } - - func testVisionRegionTextParserRejectsRefusalInsteadOfTreatingItAsOCR() { - XCTAssertNil( - AITranslator.parseVisionRegionTextCandidateForDiagnostics( - from: "抱歉，我无法读取这张图片中的文字。" - ) - ) - XCTAssertNil( - AITranslator.parseVisionRegionTextCandidateForDiagnostics( - from: "无法返回文本" - ) - ) - } - - func testVisualTextReviewPreservesLocalOCRGeometry() { - let original = TextBlock( - text: "OEIIII", - boundingBox: CGRect(x: 0.30, y: 0.20, width: 0.08, height: 0.24), - confidence: 0.3, - ocrSource: "original:manual", - estimatedFontScale: 0.08, - bubbleBox: CGRect(x: 0.28, y: 0.18, width: 0.14, height: 0.30), - layoutSafeRegion: CGRect(x: 0.29, y: 0.19, width: 0.12, height: 0.28), - textOrientation: .vertical, - layoutRole: .dialogue - ) - let reviewed = AITranslator.visualReviewedBlockForDiagnostics( - original: original, - review: VisionRegionTextCandidate(text: "ウィキペディアに", confidence: 0.91) - ) - XCTAssertEqual(reviewed.text, "ウィキペディアに") - XCTAssertEqual(reviewed.boundingBox, original.boundingBox) - XCTAssertEqual(reviewed.bubbleBox, original.bubbleBox) - XCTAssertEqual(reviewed.layoutSafeRegion, original.layoutSafeRegion) - XCTAssertEqual(reviewed.textOrientation, original.textOrientation) - XCTAssertEqual(reviewed.layoutRole, original.layoutRole) - XCTAssertEqual(reviewed.ocrSource, "visual-review-text") - XCTAssertFalse(reviewed.isFiltered) - } - - func testVisionRecognitionPromptIncludesLayoutSafeRegion() { - let prompt = AITranslator.visionRecognitionPromptForDiagnostics(isRightToLeft: true) - XCTAssertTrue(prompt.contains("layoutSafeRegion")) - } - ''' - if not tests.endswith('\n}'): - raise SystemExit('test file closing brace not found') - tests = tests[:-2] + insert + '\n}' - test_path.write_text(tests) - PY - - git diff --check - git diff --stat - - name: Commit product patch and remove temporary workflow - shell: bash - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add mreader/AITranslator.swift mreaderTests/TranslationComicIntegrationRegressionTests.swift - git rm .github/workflows/apply-visual-ocr-reliability-v3.yml - git commit -m "fix: make visual OCR review text-first" - git push origin HEAD:codex/visual-ocr-reliability From 18ec0ad914c5126a1f39ad8576f0d57a43a3093f Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:47:40 +0800 Subject: [PATCH 07/43] chore: apply visual OCR reliability patch v4 --- .../apply-visual-ocr-reliability-v4.yml | 359 ++++++++++++++++++ 1 file changed, 359 insertions(+) create mode 100644 .github/workflows/apply-visual-ocr-reliability-v4.yml diff --git a/.github/workflows/apply-visual-ocr-reliability-v4.yml b/.github/workflows/apply-visual-ocr-reliability-v4.yml new file mode 100644 index 0000000..1ab3cd9 --- /dev/null +++ b/.github/workflows/apply-visual-ocr-reliability-v4.yml @@ -0,0 +1,359 @@ +name: Apply visual OCR reliability patch v4 + +on: + push: + branches: + - codex/visual-ocr-reliability + +permissions: + contents: write + +jobs: + patch: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: codex/visual-ocr-reliability + fetch-depth: 0 + - name: Patch visual OCR pipeline + shell: bash + run: | + python3 - <<'PY' + from pathlib import Path + + p = Path('mreader/AITranslator.swift') + s = p.read_text() + + def one(old, new, label): + global s + n = s.count(old) + if n != 1: + raise SystemExit(f'{label}: expected 1 match, found {n}') + s = s.replace(old, new, 1) + + one( + '''nonisolated struct OCRVerificationRegion: Sendable { + let blockID: UUID + let sourceRect: CGRect + } + ''', + '''nonisolated struct OCRVerificationRegion: Sendable { + let blockID: UUID + let sourceRect: CGRect + } + + nonisolated struct VisionRegionTextCandidate: Equatable, Sendable { + let text: String + let confidence: Double + } + ''', 'candidate type') + + fn = s.index(' static func visualVerifyOCRRegions(') + start = s.index(' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up)', fn) + end = s.index(' } catch is CancellationError {', start) + s = s[:start] + ''' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up) + let review = try await recognizeVisionRegionText( + image: cropImage, + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor ?? AIModelProtocolCatalog.descriptor(for: model) + ) + let original = corrected[originalIndex] + corrected[originalIndex] = visualReviewedBlock(original: original, review: review) + print("MReader OCR visual text review corrected block=\\(region.blockID) confidence=\\(String(format: \"%.2f\", review.confidence))") + ''' + s[end:] + + one( + '只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', + '只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', + 'translation system fields') + one( + '只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', + '只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', + 'recognition system fields') + + one( + ' let defaultMode: VisionResponseFormatMode = usesTranslationSchema ? .jsonSchema : .jsonObject\n', + ' let defaultMode: VisionResponseFormatMode = .jsonSchema\n', + 'recognition response mode') + + one( + ''' case .jsonSchema: + guard usesTranslationSchema else { + return .jsonObject + } + guard let schema = try? JSONSerialization.data( + withJSONObject: offlineVisionTranslationSchema() + ) else { + return nil + } + return .jsonSchema(name: "manga_offline_translation", schema: schema) + ''', + ''' case .jsonSchema: + let schemaObject = usesTranslationSchema + ? offlineVisionTranslationSchema() + : visionRecognitionSchema() + guard let schema = try? JSONSerialization.data(withJSONObject: schemaObject) else { + return nil + } + return .jsonSchema( + name: usesTranslationSchema ? "manga_offline_translation" : "manga_vision_recognition", + schema: schema + ) + ''', 'schema transport') + + marker = ' private static func recognizeVisionImage(image: UIImage, sourceRect: CGRect, apiKey: String, baseURL: String, model: String, modelDescriptor: AIModelDescriptor, isRightToLeft: Bool, additionalInstructions: String, translationTarget: TranslationTargetLanguage?, translationPromptTemplate: String, strictTranslationGeometry: Bool) async throws -> [TextBlock] {\n' + if s.count(marker) != 1: + raise SystemExit(f'recognizeVisionImage marker count={s.count(marker)}') + helper = ''' private static func recognizeVisionRegionText( + image: UIImage, + apiKey: String, + baseURL: String, + model: String, + modelDescriptor: AIModelDescriptor + ) async throws -> VisionRegionTextCandidate { + guard !apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + throw VisionTranslationError.api("未配置 API Key") + } + guard !model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + throw VisionTranslationError.api("未配置模型") + } + let prepared = resizedImageForVision(image, maxDimension: 1536) + guard let imageDataURL = encodedVisionImageDataURL(prepared) else { + throw VisionTranslationError.imageEncodingFailed + } + let systemPrompt = "你只做漫画局部图片的原文转录。不要翻译，不要返回坐标，不要描述画面，不要输出思考过程。只返回 sourceText 和 confidence。" + let prompt = """ + 这是已经由本地 OCR 定位好的单个漫画文字区域裁剪。 + 只逐字抄录图片内实际可见的原文，不猜裁剪外内容，不翻译，不补剧情。 + 保留标点、数字、拉长音、小假名和大小写；竖排按自然阅读顺序合并为一个字符串。 + 不需要任何坐标。 + 只输出 JSON：{"sourceText":"图中原文","confidence":0.95} + 若确实没有可读文字，输出 {"sourceText":"","confidence":0.0}。 + """ + + let data: Data + do { + data = try await visionCompletionData( + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor, + systemPrompt: systemPrompt, + prompt: prompt, + imageDataURL: imageDataURL, + responseFormat: .jsonObject + ) + } catch { + guard isUnsupportedResponseFormat(error) else { throw error } + data = try await visionCompletionData( + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor, + systemPrompt: systemPrompt, + prompt: prompt, + imageDataURL: imageDataURL, + responseFormat: nil + ) + } + guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let content = assistantContent(from: json), + let result = parseVisionRegionTextCandidate(from: content) else { + throw VisionTranslationError.emptyResult + } + return result + } + + private static func parseVisionRegionTextCandidate(from content: String) -> VisionRegionTextCandidate? { + if let data = normalizedVisionJSONData(from: content), + let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) { + var item: [String: Any]? + if let dictionary = object as? [String: Any] { + let direct = firstString(in: dictionary, keys: ["sourceText", "source_text", "text", "original", "originalText"]) + item = direct.isEmpty ? (dictionary["items"] as? [[String: Any]])?.first : dictionary + } else if let array = object as? [[String: Any]] { + item = array.first + } + guard let item else { return nil } + let value = firstString(in: item, keys: ["sourceText", "source_text", "text", "original", "originalText"]) + .trimmingCharacters(in: .whitespacesAndNewlines) + guard !value.isEmpty, !looksLikeVisionRefusal(value) else { return nil } + return VisionRegionTextCandidate( + text: value, + confidence: min(max(doubleValue(from: item["confidence"]) ?? 0.75, 0), 1) + ) + } + var plain = content.trimmingCharacters(in: .whitespacesAndNewlines) + if plain.hasPrefix("```") { + plain = plain.replacingOccurrences(of: #"^```(?:text|markdown)?\\s*|\\s*```$"#, with: "", options: [.regularExpression, .caseInsensitive]) + .trimmingCharacters(in: .whitespacesAndNewlines) + } + guard !plain.isEmpty, !looksLikeVisionRefusal(plain) else { return nil } + return VisionRegionTextCandidate(text: plain, confidence: 0.6) + } + + private static func looksLikeVisionRefusal(_ text: String) -> Bool { + let value = text.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + guard value.count <= 240 else { return false } + let markers = [ + "无法识别", "无法读取", "无法返回", "不能识别", "不能读取", + "未检测到文字", "没有检测到文字", "未发现文字", "没有可读文字", + "抱歉，我无法", "看不到图片", "无法查看图片", + "unable to read", "unable to identify", "cannot read", "can't read", + "cannot view", "can't view", "no readable text", "no text found" + ] + return markers.contains { value.contains($0) } + } + + private static func visualReviewedBlock(original: TextBlock, review: VisionRegionTextCandidate) -> TextBlock { + TextBlock( + id: original.id, + text: review.text.trimmingCharacters(in: .whitespacesAndNewlines), + boundingBox: original.boundingBox, + translation: original.translation, + confidence: max(original.confidence, review.confidence), + ocrSource: "visual-review-text", + isFiltered: false, + filterReason: nil, + estimatedFontScale: original.estimatedFontScale, + textColorHex: original.textColorHex, + bubbleBox: original.bubbleBox, + layoutSafeRegion: original.layoutSafeRegion, + polygon: original.polygon, + bubblePolygon: original.bubblePolygon, + translationLines: original.translationLines, + textOrientation: original.textOrientation, + layoutRole: original.layoutRole, + sourceLineCount: original.sourceLineCount + ) + } + + static func parseVisionRegionTextCandidateForDiagnostics(from content: String) -> VisionRegionTextCandidate? { + parseVisionRegionTextCandidate(from: content) + } + + static func visualReviewedBlockForDiagnostics(original: TextBlock, review: VisionRegionTextCandidate) -> TextBlock { + visualReviewedBlock(original: original, review: review) + } + + ''' + s = s.replace(marker, helper + marker, 1) + + schema_marker = ' private static func offlineVisionTranslationSchema() -> [String: Any] {\n' + if s.count(schema_marker) != 1: + raise SystemExit(f'schema marker count={s.count(schema_marker)}') + schema = ''' private static func visionRecognitionSchema() -> [String: Any] { + let point: [String: Any] = [ + "type": "object", "additionalProperties": false, + "required": ["x", "y"], + "properties": [ + "x": ["type": "number", "minimum": 0, "maximum": 1], + "y": ["type": "number", "minimum": 0, "maximum": 1] + ] + ] + let rect: [String: Any] = [ + "type": "object", "additionalProperties": false, + "required": ["x", "y", "width", "height"], + "properties": [ + "x": ["type": "number", "minimum": 0, "maximum": 1], + "y": ["type": "number", "minimum": 0, "maximum": 1], + "width": ["type": "number", "exclusiveMinimum": 0, "maximum": 1], + "height": ["type": "number", "exclusiveMinimum": 0, "maximum": 1] + ] + ] + let nullableRect: [String: Any] = ["anyOf": [rect, ["type": "null"]]] + let polygon: [String: Any] = ["type": "array", "items": point] + return [ + "type": "object", "additionalProperties": false, + "required": ["coordinateSpace", "items"], + "properties": [ + "coordinateSpace": ["type": "string", "enum": ["normalized"]], + "items": [ + "type": "array", + "items": [ + "type": "object", "additionalProperties": false, + "required": ["id", "sourceText", "textBox", "bubbleBox", "layoutSafeRegion", "textPolygon", "bubblePolygon", "confidence", "classification"], + "properties": [ + "id": ["type": "string"], + "sourceText": ["type": "string"], + "textBox": rect, + "bubbleBox": nullableRect, + "layoutSafeRegion": nullableRect, + "textPolygon": polygon, + "bubblePolygon": polygon, + "confidence": ["type": "number", "minimum": 0, "maximum": 1], + "classification": ["type": "string", "enum": ["dialogue", "narration", "soundEffect", "url", "advertisement", "watermark", "copyright", "pageNumber"]] + ] + ] + ] + ] + ] + } + + ''' + s = s.replace(schema_marker, schema + schema_marker, 1) + p.write_text(s) + + t = Path('mreaderTests/TranslationComicIntegrationRegressionTests.swift') + tests = t.read_text() + extra = ''' + func testVisionRegionTextParserAcceptsCoordinateFreeJSON() throws { + let result = try XCTUnwrap(AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: #"{"sourceText":"ウィキペディアに","confidence":0.93}"#)) + XCTAssertEqual(result.text, "ウィキペディアに") + XCTAssertEqual(result.confidence, 0.93, accuracy: 0.0001) + } + + func testVisionRegionTextParserAcceptsPlainTextFallback() throws { + let result = try XCTUnwrap(AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: "有名人です。")) + XCTAssertEqual(result.text, "有名人です。") + XCTAssertEqual(result.confidence, 0.6, accuracy: 0.0001) + } + + func testVisionRegionTextParserRejectsRefusal() { + XCTAssertNil(AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: "无法返回文本")) + XCTAssertNil(AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: "抱歉，我无法读取这张图片中的文字。")) + } + + func testVisualTextReviewPreservesLocalOCRGeometry() { + let original = TextBlock( + text: "OEIIII", + boundingBox: CGRect(x: 0.30, y: 0.20, width: 0.08, height: 0.24), + confidence: 0.3, + ocrSource: "original:manual", + estimatedFontScale: 0.08, + bubbleBox: CGRect(x: 0.28, y: 0.18, width: 0.14, height: 0.30), + layoutSafeRegion: CGRect(x: 0.29, y: 0.19, width: 0.12, height: 0.28), + textOrientation: .vertical, + layoutRole: .dialogue + ) + let reviewed = AITranslator.visualReviewedBlockForDiagnostics( + original: original, + review: VisionRegionTextCandidate(text: "ウィキペディアに", confidence: 0.91) + ) + XCTAssertEqual(reviewed.text, "ウィキペディアに") + XCTAssertEqual(reviewed.boundingBox, original.boundingBox) + XCTAssertEqual(reviewed.bubbleBox, original.bubbleBox) + XCTAssertEqual(reviewed.layoutSafeRegion, original.layoutSafeRegion) + XCTAssertEqual(reviewed.textOrientation, original.textOrientation) + XCTAssertEqual(reviewed.layoutRole, original.layoutRole) + XCTAssertEqual(reviewed.ocrSource, "visual-review-text") + } + ''' + if not tests.endswith('\n}'): + raise SystemExit('test closing brace not found') + t.write_text(tests[:-2] + extra + '\n}') + PY + git diff --check + git diff --stat + - name: Commit product patch and remove temporary workflow + shell: bash + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add mreader/AITranslator.swift mreaderTests/TranslationComicIntegrationRegressionTests.swift + git rm .github/workflows/apply-visual-ocr-reliability-v4.yml + git commit -m "fix: make visual OCR review text-first" + git push origin HEAD:codex/visual-ocr-reliability From cf69ad58b35b8368cd2451d7fb7fe0109397cdd7 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:48:23 +0800 Subject: [PATCH 08/43] chore: reset visual OCR patch workflow v4 --- .../apply-visual-ocr-reliability-v4.yml | 359 ------------------ 1 file changed, 359 deletions(-) delete mode 100644 .github/workflows/apply-visual-ocr-reliability-v4.yml diff --git a/.github/workflows/apply-visual-ocr-reliability-v4.yml b/.github/workflows/apply-visual-ocr-reliability-v4.yml deleted file mode 100644 index 1ab3cd9..0000000 --- a/.github/workflows/apply-visual-ocr-reliability-v4.yml +++ /dev/null @@ -1,359 +0,0 @@ -name: Apply visual OCR reliability patch v4 - -on: - push: - branches: - - codex/visual-ocr-reliability - -permissions: - contents: write - -jobs: - patch: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: codex/visual-ocr-reliability - fetch-depth: 0 - - name: Patch visual OCR pipeline - shell: bash - run: | - python3 - <<'PY' - from pathlib import Path - - p = Path('mreader/AITranslator.swift') - s = p.read_text() - - def one(old, new, label): - global s - n = s.count(old) - if n != 1: - raise SystemExit(f'{label}: expected 1 match, found {n}') - s = s.replace(old, new, 1) - - one( - '''nonisolated struct OCRVerificationRegion: Sendable { - let blockID: UUID - let sourceRect: CGRect - } - ''', - '''nonisolated struct OCRVerificationRegion: Sendable { - let blockID: UUID - let sourceRect: CGRect - } - - nonisolated struct VisionRegionTextCandidate: Equatable, Sendable { - let text: String - let confidence: Double - } - ''', 'candidate type') - - fn = s.index(' static func visualVerifyOCRRegions(') - start = s.index(' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up)', fn) - end = s.index(' } catch is CancellationError {', start) - s = s[:start] + ''' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up) - let review = try await recognizeVisionRegionText( - image: cropImage, - apiKey: apiKey, - baseURL: baseURL, - model: model, - modelDescriptor: modelDescriptor ?? AIModelProtocolCatalog.descriptor(for: model) - ) - let original = corrected[originalIndex] - corrected[originalIndex] = visualReviewedBlock(original: original, review: review) - print("MReader OCR visual text review corrected block=\\(region.blockID) confidence=\\(String(format: \"%.2f\", review.confidence))") - ''' + s[end:] - - one( - '只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', - '只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', - 'translation system fields') - one( - '只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', - '只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', - 'recognition system fields') - - one( - ' let defaultMode: VisionResponseFormatMode = usesTranslationSchema ? .jsonSchema : .jsonObject\n', - ' let defaultMode: VisionResponseFormatMode = .jsonSchema\n', - 'recognition response mode') - - one( - ''' case .jsonSchema: - guard usesTranslationSchema else { - return .jsonObject - } - guard let schema = try? JSONSerialization.data( - withJSONObject: offlineVisionTranslationSchema() - ) else { - return nil - } - return .jsonSchema(name: "manga_offline_translation", schema: schema) - ''', - ''' case .jsonSchema: - let schemaObject = usesTranslationSchema - ? offlineVisionTranslationSchema() - : visionRecognitionSchema() - guard let schema = try? JSONSerialization.data(withJSONObject: schemaObject) else { - return nil - } - return .jsonSchema( - name: usesTranslationSchema ? "manga_offline_translation" : "manga_vision_recognition", - schema: schema - ) - ''', 'schema transport') - - marker = ' private static func recognizeVisionImage(image: UIImage, sourceRect: CGRect, apiKey: String, baseURL: String, model: String, modelDescriptor: AIModelDescriptor, isRightToLeft: Bool, additionalInstructions: String, translationTarget: TranslationTargetLanguage?, translationPromptTemplate: String, strictTranslationGeometry: Bool) async throws -> [TextBlock] {\n' - if s.count(marker) != 1: - raise SystemExit(f'recognizeVisionImage marker count={s.count(marker)}') - helper = ''' private static func recognizeVisionRegionText( - image: UIImage, - apiKey: String, - baseURL: String, - model: String, - modelDescriptor: AIModelDescriptor - ) async throws -> VisionRegionTextCandidate { - guard !apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - throw VisionTranslationError.api("未配置 API Key") - } - guard !model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - throw VisionTranslationError.api("未配置模型") - } - let prepared = resizedImageForVision(image, maxDimension: 1536) - guard let imageDataURL = encodedVisionImageDataURL(prepared) else { - throw VisionTranslationError.imageEncodingFailed - } - let systemPrompt = "你只做漫画局部图片的原文转录。不要翻译，不要返回坐标，不要描述画面，不要输出思考过程。只返回 sourceText 和 confidence。" - let prompt = """ - 这是已经由本地 OCR 定位好的单个漫画文字区域裁剪。 - 只逐字抄录图片内实际可见的原文，不猜裁剪外内容，不翻译，不补剧情。 - 保留标点、数字、拉长音、小假名和大小写；竖排按自然阅读顺序合并为一个字符串。 - 不需要任何坐标。 - 只输出 JSON：{"sourceText":"图中原文","confidence":0.95} - 若确实没有可读文字，输出 {"sourceText":"","confidence":0.0}。 - """ - - let data: Data - do { - data = try await visionCompletionData( - apiKey: apiKey, - baseURL: baseURL, - model: model, - modelDescriptor: modelDescriptor, - systemPrompt: systemPrompt, - prompt: prompt, - imageDataURL: imageDataURL, - responseFormat: .jsonObject - ) - } catch { - guard isUnsupportedResponseFormat(error) else { throw error } - data = try await visionCompletionData( - apiKey: apiKey, - baseURL: baseURL, - model: model, - modelDescriptor: modelDescriptor, - systemPrompt: systemPrompt, - prompt: prompt, - imageDataURL: imageDataURL, - responseFormat: nil - ) - } - guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let content = assistantContent(from: json), - let result = parseVisionRegionTextCandidate(from: content) else { - throw VisionTranslationError.emptyResult - } - return result - } - - private static func parseVisionRegionTextCandidate(from content: String) -> VisionRegionTextCandidate? { - if let data = normalizedVisionJSONData(from: content), - let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) { - var item: [String: Any]? - if let dictionary = object as? [String: Any] { - let direct = firstString(in: dictionary, keys: ["sourceText", "source_text", "text", "original", "originalText"]) - item = direct.isEmpty ? (dictionary["items"] as? [[String: Any]])?.first : dictionary - } else if let array = object as? [[String: Any]] { - item = array.first - } - guard let item else { return nil } - let value = firstString(in: item, keys: ["sourceText", "source_text", "text", "original", "originalText"]) - .trimmingCharacters(in: .whitespacesAndNewlines) - guard !value.isEmpty, !looksLikeVisionRefusal(value) else { return nil } - return VisionRegionTextCandidate( - text: value, - confidence: min(max(doubleValue(from: item["confidence"]) ?? 0.75, 0), 1) - ) - } - var plain = content.trimmingCharacters(in: .whitespacesAndNewlines) - if plain.hasPrefix("```") { - plain = plain.replacingOccurrences(of: #"^```(?:text|markdown)?\\s*|\\s*```$"#, with: "", options: [.regularExpression, .caseInsensitive]) - .trimmingCharacters(in: .whitespacesAndNewlines) - } - guard !plain.isEmpty, !looksLikeVisionRefusal(plain) else { return nil } - return VisionRegionTextCandidate(text: plain, confidence: 0.6) - } - - private static func looksLikeVisionRefusal(_ text: String) -> Bool { - let value = text.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() - guard value.count <= 240 else { return false } - let markers = [ - "无法识别", "无法读取", "无法返回", "不能识别", "不能读取", - "未检测到文字", "没有检测到文字", "未发现文字", "没有可读文字", - "抱歉，我无法", "看不到图片", "无法查看图片", - "unable to read", "unable to identify", "cannot read", "can't read", - "cannot view", "can't view", "no readable text", "no text found" - ] - return markers.contains { value.contains($0) } - } - - private static func visualReviewedBlock(original: TextBlock, review: VisionRegionTextCandidate) -> TextBlock { - TextBlock( - id: original.id, - text: review.text.trimmingCharacters(in: .whitespacesAndNewlines), - boundingBox: original.boundingBox, - translation: original.translation, - confidence: max(original.confidence, review.confidence), - ocrSource: "visual-review-text", - isFiltered: false, - filterReason: nil, - estimatedFontScale: original.estimatedFontScale, - textColorHex: original.textColorHex, - bubbleBox: original.bubbleBox, - layoutSafeRegion: original.layoutSafeRegion, - polygon: original.polygon, - bubblePolygon: original.bubblePolygon, - translationLines: original.translationLines, - textOrientation: original.textOrientation, - layoutRole: original.layoutRole, - sourceLineCount: original.sourceLineCount - ) - } - - static func parseVisionRegionTextCandidateForDiagnostics(from content: String) -> VisionRegionTextCandidate? { - parseVisionRegionTextCandidate(from: content) - } - - static func visualReviewedBlockForDiagnostics(original: TextBlock, review: VisionRegionTextCandidate) -> TextBlock { - visualReviewedBlock(original: original, review: review) - } - - ''' - s = s.replace(marker, helper + marker, 1) - - schema_marker = ' private static func offlineVisionTranslationSchema() -> [String: Any] {\n' - if s.count(schema_marker) != 1: - raise SystemExit(f'schema marker count={s.count(schema_marker)}') - schema = ''' private static func visionRecognitionSchema() -> [String: Any] { - let point: [String: Any] = [ - "type": "object", "additionalProperties": false, - "required": ["x", "y"], - "properties": [ - "x": ["type": "number", "minimum": 0, "maximum": 1], - "y": ["type": "number", "minimum": 0, "maximum": 1] - ] - ] - let rect: [String: Any] = [ - "type": "object", "additionalProperties": false, - "required": ["x", "y", "width", "height"], - "properties": [ - "x": ["type": "number", "minimum": 0, "maximum": 1], - "y": ["type": "number", "minimum": 0, "maximum": 1], - "width": ["type": "number", "exclusiveMinimum": 0, "maximum": 1], - "height": ["type": "number", "exclusiveMinimum": 0, "maximum": 1] - ] - ] - let nullableRect: [String: Any] = ["anyOf": [rect, ["type": "null"]]] - let polygon: [String: Any] = ["type": "array", "items": point] - return [ - "type": "object", "additionalProperties": false, - "required": ["coordinateSpace", "items"], - "properties": [ - "coordinateSpace": ["type": "string", "enum": ["normalized"]], - "items": [ - "type": "array", - "items": [ - "type": "object", "additionalProperties": false, - "required": ["id", "sourceText", "textBox", "bubbleBox", "layoutSafeRegion", "textPolygon", "bubblePolygon", "confidence", "classification"], - "properties": [ - "id": ["type": "string"], - "sourceText": ["type": "string"], - "textBox": rect, - "bubbleBox": nullableRect, - "layoutSafeRegion": nullableRect, - "textPolygon": polygon, - "bubblePolygon": polygon, - "confidence": ["type": "number", "minimum": 0, "maximum": 1], - "classification": ["type": "string", "enum": ["dialogue", "narration", "soundEffect", "url", "advertisement", "watermark", "copyright", "pageNumber"]] - ] - ] - ] - ] - ] - } - - ''' - s = s.replace(schema_marker, schema + schema_marker, 1) - p.write_text(s) - - t = Path('mreaderTests/TranslationComicIntegrationRegressionTests.swift') - tests = t.read_text() - extra = ''' - func testVisionRegionTextParserAcceptsCoordinateFreeJSON() throws { - let result = try XCTUnwrap(AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: #"{"sourceText":"ウィキペディアに","confidence":0.93}"#)) - XCTAssertEqual(result.text, "ウィキペディアに") - XCTAssertEqual(result.confidence, 0.93, accuracy: 0.0001) - } - - func testVisionRegionTextParserAcceptsPlainTextFallback() throws { - let result = try XCTUnwrap(AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: "有名人です。")) - XCTAssertEqual(result.text, "有名人です。") - XCTAssertEqual(result.confidence, 0.6, accuracy: 0.0001) - } - - func testVisionRegionTextParserRejectsRefusal() { - XCTAssertNil(AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: "无法返回文本")) - XCTAssertNil(AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: "抱歉，我无法读取这张图片中的文字。")) - } - - func testVisualTextReviewPreservesLocalOCRGeometry() { - let original = TextBlock( - text: "OEIIII", - boundingBox: CGRect(x: 0.30, y: 0.20, width: 0.08, height: 0.24), - confidence: 0.3, - ocrSource: "original:manual", - estimatedFontScale: 0.08, - bubbleBox: CGRect(x: 0.28, y: 0.18, width: 0.14, height: 0.30), - layoutSafeRegion: CGRect(x: 0.29, y: 0.19, width: 0.12, height: 0.28), - textOrientation: .vertical, - layoutRole: .dialogue - ) - let reviewed = AITranslator.visualReviewedBlockForDiagnostics( - original: original, - review: VisionRegionTextCandidate(text: "ウィキペディアに", confidence: 0.91) - ) - XCTAssertEqual(reviewed.text, "ウィキペディアに") - XCTAssertEqual(reviewed.boundingBox, original.boundingBox) - XCTAssertEqual(reviewed.bubbleBox, original.bubbleBox) - XCTAssertEqual(reviewed.layoutSafeRegion, original.layoutSafeRegion) - XCTAssertEqual(reviewed.textOrientation, original.textOrientation) - XCTAssertEqual(reviewed.layoutRole, original.layoutRole) - XCTAssertEqual(reviewed.ocrSource, "visual-review-text") - } - ''' - if not tests.endswith('\n}'): - raise SystemExit('test closing brace not found') - t.write_text(tests[:-2] + extra + '\n}') - PY - git diff --check - git diff --stat - - name: Commit product patch and remove temporary workflow - shell: bash - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add mreader/AITranslator.swift mreaderTests/TranslationComicIntegrationRegressionTests.swift - git rm .github/workflows/apply-visual-ocr-reliability-v4.yml - git commit -m "fix: make visual OCR review text-first" - git push origin HEAD:codex/visual-ocr-reliability From 82e3c7909c03e477c480e9ab14225a8da5b5e453 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:49:02 +0800 Subject: [PATCH 09/43] chore: apply visual OCR reliability patch v5 --- .../apply-visual-ocr-reliability-v5.yml | 310 ++++++++++++++++++ 1 file changed, 310 insertions(+) create mode 100644 .github/workflows/apply-visual-ocr-reliability-v5.yml diff --git a/.github/workflows/apply-visual-ocr-reliability-v5.yml b/.github/workflows/apply-visual-ocr-reliability-v5.yml new file mode 100644 index 0000000..613306d --- /dev/null +++ b/.github/workflows/apply-visual-ocr-reliability-v5.yml @@ -0,0 +1,310 @@ +name: Apply visual OCR reliability patch v5 + +on: + push: + branches: + - codex/visual-ocr-reliability + +permissions: + contents: write + +jobs: + patch: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: codex/visual-ocr-reliability + fetch-depth: 0 + - name: Patch visual OCR pipeline + shell: bash + run: | + python3 - <<'PY' + from pathlib import Path + + p = Path('mreader/AITranslator.swift') + s = p.read_text() + + def one(old, new, label): + global s + n = s.count(old) + if n != 1: + raise SystemExit(f'{label}: expected 1 match, found {n}') + s = s.replace(old, new, 1) + + one( + '''nonisolated struct OCRVerificationRegion: Sendable { + let blockID: UUID + let sourceRect: CGRect + } + ''', + '''nonisolated struct OCRVerificationRegion: Sendable { + let blockID: UUID + let sourceRect: CGRect + } + + nonisolated struct VisionRegionTextCandidate: Equatable, Sendable { + let text: String + let confidence: Double + } + ''', 'candidate type') + + fn = s.index(' static func visualVerifyOCRRegions(') + start = s.index(' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up)', fn) + end = s.index(' } catch is CancellationError {', start) + s = s[:start] + ''' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up) + let review = try await recognizeVisionRegionText( + image: cropImage, + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor ?? AIModelProtocolCatalog.descriptor(for: model) + ) + let original = corrected[originalIndex] + corrected[originalIndex] = visualReviewedBlock(original: original, review: review) + print("MReader OCR visual text review corrected block=\\(region.blockID) confidence=\\(String(format: \"%.2f\", review.confidence))") + ''' + s[end:] + + one( + '只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', + '只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', + 'translation system fields') + one( + '只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', + '只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', + 'recognition system fields') + + one( + ' let defaultMode: VisionResponseFormatMode = usesTranslationSchema ? .jsonSchema : .jsonObject\n', + ' let defaultMode: VisionResponseFormatMode = .jsonSchema\n', + 'recognition response mode') + + one( + ''' case .jsonSchema: + guard usesTranslationSchema else { + return .jsonObject + } + guard let schema = try? JSONSerialization.data( + withJSONObject: offlineVisionTranslationSchema() + ) else { + return nil + } + return .jsonSchema(name: "manga_offline_translation", schema: schema) + ''', + ''' case .jsonSchema: + let schemaObject = usesTranslationSchema + ? offlineVisionTranslationSchema() + : visionRecognitionSchema() + guard let schema = try? JSONSerialization.data(withJSONObject: schemaObject) else { + return nil + } + return .jsonSchema( + name: usesTranslationSchema ? "manga_offline_translation" : "manga_vision_recognition", + schema: schema + ) + ''', 'schema transport') + + marker = ' private static func recognizeVisionImage(image: UIImage, sourceRect: CGRect, apiKey: String, baseURL: String, model: String, modelDescriptor: AIModelDescriptor, isRightToLeft: Bool, additionalInstructions: String, translationTarget: TranslationTargetLanguage?, translationPromptTemplate: String, strictTranslationGeometry: Bool) async throws -> [TextBlock] {\n' + if s.count(marker) != 1: + raise SystemExit(f'recognizeVisionImage marker count={s.count(marker)}') + helper = ''' private static func recognizeVisionRegionText( + image: UIImage, + apiKey: String, + baseURL: String, + model: String, + modelDescriptor: AIModelDescriptor + ) async throws -> VisionRegionTextCandidate { + guard !apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + throw VisionTranslationError.api("未配置 API Key") + } + guard !model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + throw VisionTranslationError.api("未配置模型") + } + let prepared = resizedImageForVision(image, maxDimension: 1536) + guard let imageDataURL = encodedVisionImageDataURL(prepared) else { + throw VisionTranslationError.imageEncodingFailed + } + let systemPrompt = "你只做漫画局部图片的原文转录。不要翻译，不要返回坐标，不要描述画面，不要输出思考过程。只返回 sourceText 和 confidence。" + let prompt = """ + 这是已经由本地 OCR 定位好的单个漫画文字区域裁剪。 + 只逐字抄录图片内实际可见的原文，不猜裁剪外内容，不翻译，不补剧情。 + 保留标点、数字、拉长音、小假名和大小写；竖排按自然阅读顺序合并为一个字符串。 + 不需要任何坐标。 + 只输出 JSON：{"sourceText":"图中原文","confidence":0.95} + 若确实没有可读文字，输出 {"sourceText":"","confidence":0.0}。 + """ + + let data: Data + do { + data = try await visionCompletionData( + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor, + systemPrompt: systemPrompt, + prompt: prompt, + imageDataURL: imageDataURL, + responseFormat: .jsonObject + ) + } catch { + guard isUnsupportedResponseFormat(error) else { throw error } + data = try await visionCompletionData( + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor, + systemPrompt: systemPrompt, + prompt: prompt, + imageDataURL: imageDataURL, + responseFormat: nil + ) + } + guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let content = assistantContent(from: json), + let result = parseVisionRegionTextCandidate(from: content) else { + throw VisionTranslationError.emptyResult + } + return result + } + + private static func parseVisionRegionTextCandidate(from content: String) -> VisionRegionTextCandidate? { + if let data = normalizedVisionJSONData(from: content), + let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) { + var item: [String: Any]? + if let dictionary = object as? [String: Any] { + let direct = firstString(in: dictionary, keys: ["sourceText", "source_text", "text", "original", "originalText"]) + item = direct.isEmpty ? (dictionary["items"] as? [[String: Any]])?.first : dictionary + } else if let array = object as? [[String: Any]] { + item = array.first + } + guard let item else { return nil } + let value = firstString(in: item, keys: ["sourceText", "source_text", "text", "original", "originalText"]) + .trimmingCharacters(in: .whitespacesAndNewlines) + guard !value.isEmpty, !looksLikeVisionRefusal(value) else { return nil } + return VisionRegionTextCandidate( + text: value, + confidence: min(max(doubleValue(from: item["confidence"]) ?? 0.75, 0), 1) + ) + } + var plain = content.trimmingCharacters(in: .whitespacesAndNewlines) + if plain.hasPrefix("```") { + plain = plain.replacingOccurrences(of: #"^```(?:text|markdown)?\\s*|\\s*```$"#, with: "", options: [.regularExpression, .caseInsensitive]) + .trimmingCharacters(in: .whitespacesAndNewlines) + } + guard !plain.isEmpty, !looksLikeVisionRefusal(plain) else { return nil } + return VisionRegionTextCandidate(text: plain, confidence: 0.6) + } + + private static func looksLikeVisionRefusal(_ text: String) -> Bool { + let value = text.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + guard value.count <= 240 else { return false } + let markers = [ + "无法识别", "无法读取", "无法返回", "不能识别", "不能读取", + "未检测到文字", "没有检测到文字", "未发现文字", "没有可读文字", + "抱歉，我无法", "看不到图片", "无法查看图片", + "unable to read", "unable to identify", "cannot read", "can't read", + "cannot view", "can't view", "no readable text", "no text found" + ] + return markers.contains { value.contains($0) } + } + + private static func visualReviewedBlock(original: TextBlock, review: VisionRegionTextCandidate) -> TextBlock { + TextBlock( + id: original.id, + text: review.text.trimmingCharacters(in: .whitespacesAndNewlines), + boundingBox: original.boundingBox, + translation: original.translation, + confidence: max(original.confidence, review.confidence), + ocrSource: "visual-review-text", + isFiltered: false, + filterReason: nil, + estimatedFontScale: original.estimatedFontScale, + textColorHex: original.textColorHex, + bubbleBox: original.bubbleBox, + layoutSafeRegion: original.layoutSafeRegion, + polygon: original.polygon, + bubblePolygon: original.bubblePolygon, + translationLines: original.translationLines, + textOrientation: original.textOrientation, + layoutRole: original.layoutRole, + sourceLineCount: original.sourceLineCount + ) + } + + static func parseVisionRegionTextCandidateForDiagnostics(from content: String) -> VisionRegionTextCandidate? { + parseVisionRegionTextCandidate(from: content) + } + + static func visualReviewedBlockForDiagnostics(original: TextBlock, review: VisionRegionTextCandidate) -> TextBlock { + visualReviewedBlock(original: original, review: review) + } + + ''' + s = s.replace(marker, helper + marker, 1) + + schema_marker = ' private static func offlineVisionTranslationSchema() -> [String: Any] {\n' + if s.count(schema_marker) != 1: + raise SystemExit(f'schema marker count={s.count(schema_marker)}') + schema = ''' private static func visionRecognitionSchema() -> [String: Any] { + let point: [String: Any] = [ + "type": "object", "additionalProperties": false, + "required": ["x", "y"], + "properties": [ + "x": ["type": "number", "minimum": 0, "maximum": 1], + "y": ["type": "number", "minimum": 0, "maximum": 1] + ] + ] + let rect: [String: Any] = [ + "type": "object", "additionalProperties": false, + "required": ["x", "y", "width", "height"], + "properties": [ + "x": ["type": "number", "minimum": 0, "maximum": 1], + "y": ["type": "number", "minimum": 0, "maximum": 1], + "width": ["type": "number", "exclusiveMinimum": 0, "maximum": 1], + "height": ["type": "number", "exclusiveMinimum": 0, "maximum": 1] + ] + ] + let nullableRect: [String: Any] = ["anyOf": [rect, ["type": "null"]]] + let polygon: [String: Any] = ["type": "array", "items": point] + return [ + "type": "object", "additionalProperties": false, + "required": ["coordinateSpace", "items"], + "properties": [ + "coordinateSpace": ["type": "string", "enum": ["normalized"]], + "items": [ + "type": "array", + "items": [ + "type": "object", "additionalProperties": false, + "required": ["id", "sourceText", "textBox", "bubbleBox", "layoutSafeRegion", "textPolygon", "bubblePolygon", "confidence", "classification"], + "properties": [ + "id": ["type": "string"], + "sourceText": ["type": "string"], + "textBox": rect, + "bubbleBox": nullableRect, + "layoutSafeRegion": nullableRect, + "textPolygon": polygon, + "bubblePolygon": polygon, + "confidence": ["type": "number", "minimum": 0, "maximum": 1], + "classification": ["type": "string", "enum": ["dialogue", "narration", "soundEffect", "url", "advertisement", "watermark", "copyright", "pageNumber"]] + ] + ] + ] + ] + ] + } + + ''' + s = s.replace(schema_marker, schema + schema_marker, 1) + p.write_text(s) + PY + git diff --check + git diff --stat + - name: Commit product patch and remove temporary workflow + shell: bash + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add mreader/AITranslator.swift + git rm .github/workflows/apply-visual-ocr-reliability-v5.yml + git commit -m "fix: make visual OCR review text-first" + git push origin HEAD:codex/visual-ocr-reliability From a1d59bc95bb3ae705b6be2f83d1164aae5c217d3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 12 Sep 2026 22:49:10 +0000 Subject: [PATCH 10/43] fix: make visual OCR review text-first --- .../apply-visual-ocr-reliability-v5.yml | 310 ------------------ mreader/AITranslator.swift | 258 +++++++++++---- 2 files changed, 199 insertions(+), 369 deletions(-) delete mode 100644 .github/workflows/apply-visual-ocr-reliability-v5.yml diff --git a/.github/workflows/apply-visual-ocr-reliability-v5.yml b/.github/workflows/apply-visual-ocr-reliability-v5.yml deleted file mode 100644 index 613306d..0000000 --- a/.github/workflows/apply-visual-ocr-reliability-v5.yml +++ /dev/null @@ -1,310 +0,0 @@ -name: Apply visual OCR reliability patch v5 - -on: - push: - branches: - - codex/visual-ocr-reliability - -permissions: - contents: write - -jobs: - patch: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: codex/visual-ocr-reliability - fetch-depth: 0 - - name: Patch visual OCR pipeline - shell: bash - run: | - python3 - <<'PY' - from pathlib import Path - - p = Path('mreader/AITranslator.swift') - s = p.read_text() - - def one(old, new, label): - global s - n = s.count(old) - if n != 1: - raise SystemExit(f'{label}: expected 1 match, found {n}') - s = s.replace(old, new, 1) - - one( - '''nonisolated struct OCRVerificationRegion: Sendable { - let blockID: UUID - let sourceRect: CGRect - } - ''', - '''nonisolated struct OCRVerificationRegion: Sendable { - let blockID: UUID - let sourceRect: CGRect - } - - nonisolated struct VisionRegionTextCandidate: Equatable, Sendable { - let text: String - let confidence: Double - } - ''', 'candidate type') - - fn = s.index(' static func visualVerifyOCRRegions(') - start = s.index(' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up)', fn) - end = s.index(' } catch is CancellationError {', start) - s = s[:start] + ''' let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up) - let review = try await recognizeVisionRegionText( - image: cropImage, - apiKey: apiKey, - baseURL: baseURL, - model: model, - modelDescriptor: modelDescriptor ?? AIModelProtocolCatalog.descriptor(for: model) - ) - let original = corrected[originalIndex] - corrected[originalIndex] = visualReviewedBlock(original: original, review: review) - print("MReader OCR visual text review corrected block=\\(region.blockID) confidence=\\(String(format: \"%.2f\", review.confidence))") - ''' + s[end:] - - one( - '只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', - '只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', - 'translation system fields') - one( - '只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', - '只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段', - 'recognition system fields') - - one( - ' let defaultMode: VisionResponseFormatMode = usesTranslationSchema ? .jsonSchema : .jsonObject\n', - ' let defaultMode: VisionResponseFormatMode = .jsonSchema\n', - 'recognition response mode') - - one( - ''' case .jsonSchema: - guard usesTranslationSchema else { - return .jsonObject - } - guard let schema = try? JSONSerialization.data( - withJSONObject: offlineVisionTranslationSchema() - ) else { - return nil - } - return .jsonSchema(name: "manga_offline_translation", schema: schema) - ''', - ''' case .jsonSchema: - let schemaObject = usesTranslationSchema - ? offlineVisionTranslationSchema() - : visionRecognitionSchema() - guard let schema = try? JSONSerialization.data(withJSONObject: schemaObject) else { - return nil - } - return .jsonSchema( - name: usesTranslationSchema ? "manga_offline_translation" : "manga_vision_recognition", - schema: schema - ) - ''', 'schema transport') - - marker = ' private static func recognizeVisionImage(image: UIImage, sourceRect: CGRect, apiKey: String, baseURL: String, model: String, modelDescriptor: AIModelDescriptor, isRightToLeft: Bool, additionalInstructions: String, translationTarget: TranslationTargetLanguage?, translationPromptTemplate: String, strictTranslationGeometry: Bool) async throws -> [TextBlock] {\n' - if s.count(marker) != 1: - raise SystemExit(f'recognizeVisionImage marker count={s.count(marker)}') - helper = ''' private static func recognizeVisionRegionText( - image: UIImage, - apiKey: String, - baseURL: String, - model: String, - modelDescriptor: AIModelDescriptor - ) async throws -> VisionRegionTextCandidate { - guard !apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - throw VisionTranslationError.api("未配置 API Key") - } - guard !model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - throw VisionTranslationError.api("未配置模型") - } - let prepared = resizedImageForVision(image, maxDimension: 1536) - guard let imageDataURL = encodedVisionImageDataURL(prepared) else { - throw VisionTranslationError.imageEncodingFailed - } - let systemPrompt = "你只做漫画局部图片的原文转录。不要翻译，不要返回坐标，不要描述画面，不要输出思考过程。只返回 sourceText 和 confidence。" - let prompt = """ - 这是已经由本地 OCR 定位好的单个漫画文字区域裁剪。 - 只逐字抄录图片内实际可见的原文，不猜裁剪外内容，不翻译，不补剧情。 - 保留标点、数字、拉长音、小假名和大小写；竖排按自然阅读顺序合并为一个字符串。 - 不需要任何坐标。 - 只输出 JSON：{"sourceText":"图中原文","confidence":0.95} - 若确实没有可读文字，输出 {"sourceText":"","confidence":0.0}。 - """ - - let data: Data - do { - data = try await visionCompletionData( - apiKey: apiKey, - baseURL: baseURL, - model: model, - modelDescriptor: modelDescriptor, - systemPrompt: systemPrompt, - prompt: prompt, - imageDataURL: imageDataURL, - responseFormat: .jsonObject - ) - } catch { - guard isUnsupportedResponseFormat(error) else { throw error } - data = try await visionCompletionData( - apiKey: apiKey, - baseURL: baseURL, - model: model, - modelDescriptor: modelDescriptor, - systemPrompt: systemPrompt, - prompt: prompt, - imageDataURL: imageDataURL, - responseFormat: nil - ) - } - guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let content = assistantContent(from: json), - let result = parseVisionRegionTextCandidate(from: content) else { - throw VisionTranslationError.emptyResult - } - return result - } - - private static func parseVisionRegionTextCandidate(from content: String) -> VisionRegionTextCandidate? { - if let data = normalizedVisionJSONData(from: content), - let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) { - var item: [String: Any]? - if let dictionary = object as? [String: Any] { - let direct = firstString(in: dictionary, keys: ["sourceText", "source_text", "text", "original", "originalText"]) - item = direct.isEmpty ? (dictionary["items"] as? [[String: Any]])?.first : dictionary - } else if let array = object as? [[String: Any]] { - item = array.first - } - guard let item else { return nil } - let value = firstString(in: item, keys: ["sourceText", "source_text", "text", "original", "originalText"]) - .trimmingCharacters(in: .whitespacesAndNewlines) - guard !value.isEmpty, !looksLikeVisionRefusal(value) else { return nil } - return VisionRegionTextCandidate( - text: value, - confidence: min(max(doubleValue(from: item["confidence"]) ?? 0.75, 0), 1) - ) - } - var plain = content.trimmingCharacters(in: .whitespacesAndNewlines) - if plain.hasPrefix("```") { - plain = plain.replacingOccurrences(of: #"^```(?:text|markdown)?\\s*|\\s*```$"#, with: "", options: [.regularExpression, .caseInsensitive]) - .trimmingCharacters(in: .whitespacesAndNewlines) - } - guard !plain.isEmpty, !looksLikeVisionRefusal(plain) else { return nil } - return VisionRegionTextCandidate(text: plain, confidence: 0.6) - } - - private static func looksLikeVisionRefusal(_ text: String) -> Bool { - let value = text.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() - guard value.count <= 240 else { return false } - let markers = [ - "无法识别", "无法读取", "无法返回", "不能识别", "不能读取", - "未检测到文字", "没有检测到文字", "未发现文字", "没有可读文字", - "抱歉，我无法", "看不到图片", "无法查看图片", - "unable to read", "unable to identify", "cannot read", "can't read", - "cannot view", "can't view", "no readable text", "no text found" - ] - return markers.contains { value.contains($0) } - } - - private static func visualReviewedBlock(original: TextBlock, review: VisionRegionTextCandidate) -> TextBlock { - TextBlock( - id: original.id, - text: review.text.trimmingCharacters(in: .whitespacesAndNewlines), - boundingBox: original.boundingBox, - translation: original.translation, - confidence: max(original.confidence, review.confidence), - ocrSource: "visual-review-text", - isFiltered: false, - filterReason: nil, - estimatedFontScale: original.estimatedFontScale, - textColorHex: original.textColorHex, - bubbleBox: original.bubbleBox, - layoutSafeRegion: original.layoutSafeRegion, - polygon: original.polygon, - bubblePolygon: original.bubblePolygon, - translationLines: original.translationLines, - textOrientation: original.textOrientation, - layoutRole: original.layoutRole, - sourceLineCount: original.sourceLineCount - ) - } - - static func parseVisionRegionTextCandidateForDiagnostics(from content: String) -> VisionRegionTextCandidate? { - parseVisionRegionTextCandidate(from: content) - } - - static func visualReviewedBlockForDiagnostics(original: TextBlock, review: VisionRegionTextCandidate) -> TextBlock { - visualReviewedBlock(original: original, review: review) - } - - ''' - s = s.replace(marker, helper + marker, 1) - - schema_marker = ' private static func offlineVisionTranslationSchema() -> [String: Any] {\n' - if s.count(schema_marker) != 1: - raise SystemExit(f'schema marker count={s.count(schema_marker)}') - schema = ''' private static func visionRecognitionSchema() -> [String: Any] { - let point: [String: Any] = [ - "type": "object", "additionalProperties": false, - "required": ["x", "y"], - "properties": [ - "x": ["type": "number", "minimum": 0, "maximum": 1], - "y": ["type": "number", "minimum": 0, "maximum": 1] - ] - ] - let rect: [String: Any] = [ - "type": "object", "additionalProperties": false, - "required": ["x", "y", "width", "height"], - "properties": [ - "x": ["type": "number", "minimum": 0, "maximum": 1], - "y": ["type": "number", "minimum": 0, "maximum": 1], - "width": ["type": "number", "exclusiveMinimum": 0, "maximum": 1], - "height": ["type": "number", "exclusiveMinimum": 0, "maximum": 1] - ] - ] - let nullableRect: [String: Any] = ["anyOf": [rect, ["type": "null"]]] - let polygon: [String: Any] = ["type": "array", "items": point] - return [ - "type": "object", "additionalProperties": false, - "required": ["coordinateSpace", "items"], - "properties": [ - "coordinateSpace": ["type": "string", "enum": ["normalized"]], - "items": [ - "type": "array", - "items": [ - "type": "object", "additionalProperties": false, - "required": ["id", "sourceText", "textBox", "bubbleBox", "layoutSafeRegion", "textPolygon", "bubblePolygon", "confidence", "classification"], - "properties": [ - "id": ["type": "string"], - "sourceText": ["type": "string"], - "textBox": rect, - "bubbleBox": nullableRect, - "layoutSafeRegion": nullableRect, - "textPolygon": polygon, - "bubblePolygon": polygon, - "confidence": ["type": "number", "minimum": 0, "maximum": 1], - "classification": ["type": "string", "enum": ["dialogue", "narration", "soundEffect", "url", "advertisement", "watermark", "copyright", "pageNumber"]] - ] - ] - ] - ] - ] - } - - ''' - s = s.replace(schema_marker, schema + schema_marker, 1) - p.write_text(s) - PY - git diff --check - git diff --stat - - name: Commit product patch and remove temporary workflow - shell: bash - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add mreader/AITranslator.swift - git rm .github/workflows/apply-visual-ocr-reliability-v5.yml - git commit -m "fix: make visual OCR review text-first" - git push origin HEAD:codex/visual-ocr-reliability diff --git a/mreader/AITranslator.swift b/mreader/AITranslator.swift index 2482928..03a97dc 100644 --- a/mreader/AITranslator.swift +++ b/mreader/AITranslator.swift @@ -219,6 +219,11 @@ nonisolated struct OCRVerificationRegion: Sendable { let sourceRect: CGRect } +nonisolated struct VisionRegionTextCandidate: Equatable, Sendable { + let text: String + let confidence: Double +} + nonisolated enum AITranslationRequestError: LocalizedError, Sendable { case invalidConfiguration(String) case server(model: String, statusCode: Int?, message: String) @@ -1424,62 +1429,16 @@ class AITranslator { do { let cropImage = UIImage(cgImage: crop, scale: 1, orientation: .up) - let localBlocks = try await recognizeVisionPage( + let review = try await recognizeVisionRegionText( image: cropImage, apiKey: apiKey, baseURL: baseURL, model: model, - isRightToLeft: isRightToLeft, - viewportAspect: max(cropImage.size.height / max(cropImage.size.width, 1), 1.25), modelDescriptor: modelDescriptor ?? AIModelProtocolCatalog.descriptor(for: model) ) let original = corrected[originalIndex] - guard let match = visualVerificationMatch( - for: original, - candidates: localBlocks, - sourceRect: region.sourceRect - ) else { - continue - } - let best = match.block - let correctedText = best.text.trimmingCharacters(in: .whitespacesAndNewlines) - guard !correctedText.isEmpty else { continue } - let correctedBox = match.pageBoundingBox - let correctedFontScale = visualVerificationMappedFontScale( - for: best, - sourceRect: region.sourceRect, - correctedBox: correctedBox - ) - let bubbleGeometry = visualVerificationMappedBubbleGeometry( - for: best, - sourceRect: region.sourceRect, - correctedBox: correctedBox - ) - corrected[originalIndex] = TextBlock( - id: original.id, - text: correctedText, - boundingBox: correctedBox, - translation: original.translation, - confidence: max(original.confidence, best.confidence), - ocrSource: "visual-review", - // A rejected/uncertain local candidate that passed a - // visual text+geometry match is explicitly recovered. - isFiltered: false, - filterReason: nil, - estimatedFontScale: correctedFontScale, - textColorHex: original.textColorHex, - bubbleBox: bubbleGeometry.bubbleBox, - layoutSafeRegion: original.layoutSafeRegion ?? bubbleGeometry.bubbleBox, - polygon: original.polygon, - bubblePolygon: bubbleGeometry.bubblePolygon, - translationLines: original.translationLines, - textOrientation: best.textOrientation, - layoutRole: original.layoutRole == .standalone || best.layoutRole == .standalone - ? .standalone - : .dialogue, - sourceLineCount: original.sourceLineCount - ) - print("MReader OCR visual review corrected block=\(region.blockID) confidence=\(String(format: "%.2f", best.confidence))") + corrected[originalIndex] = visualReviewedBlock(original: original, review: review) + print("MReader OCR visual text review corrected block=\(region.blockID) confidence=\(String(format: "%.2f", review.confidence))") } catch is CancellationError { throw CancellationError() } catch { @@ -1739,6 +1698,138 @@ class AITranslator { return CGFloat(previous.last ?? 0) / CGFloat(max(leftScalars.count, rightScalars.count)) } + private static func recognizeVisionRegionText( + image: UIImage, + apiKey: String, + baseURL: String, + model: String, + modelDescriptor: AIModelDescriptor +) async throws -> VisionRegionTextCandidate { + guard !apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + throw VisionTranslationError.api("未配置 API Key") + } + guard !model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + throw VisionTranslationError.api("未配置模型") + } + let prepared = resizedImageForVision(image, maxDimension: 1536) + guard let imageDataURL = encodedVisionImageDataURL(prepared) else { + throw VisionTranslationError.imageEncodingFailed + } + let systemPrompt = "你只做漫画局部图片的原文转录。不要翻译，不要返回坐标，不要描述画面，不要输出思考过程。只返回 sourceText 和 confidence。" + let prompt = """ + 这是已经由本地 OCR 定位好的单个漫画文字区域裁剪。 + 只逐字抄录图片内实际可见的原文，不猜裁剪外内容，不翻译，不补剧情。 + 保留标点、数字、拉长音、小假名和大小写；竖排按自然阅读顺序合并为一个字符串。 + 不需要任何坐标。 + 只输出 JSON：{"sourceText":"图中原文","confidence":0.95} + 若确实没有可读文字，输出 {"sourceText":"","confidence":0.0}。 + """ + + let data: Data + do { + data = try await visionCompletionData( + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor, + systemPrompt: systemPrompt, + prompt: prompt, + imageDataURL: imageDataURL, + responseFormat: .jsonObject + ) + } catch { + guard isUnsupportedResponseFormat(error) else { throw error } + data = try await visionCompletionData( + apiKey: apiKey, + baseURL: baseURL, + model: model, + modelDescriptor: modelDescriptor, + systemPrompt: systemPrompt, + prompt: prompt, + imageDataURL: imageDataURL, + responseFormat: nil + ) + } + guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let content = assistantContent(from: json), + let result = parseVisionRegionTextCandidate(from: content) else { + throw VisionTranslationError.emptyResult + } + return result +} + +private static func parseVisionRegionTextCandidate(from content: String) -> VisionRegionTextCandidate? { + if let data = normalizedVisionJSONData(from: content), + let object = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) { + var item: [String: Any]? + if let dictionary = object as? [String: Any] { + let direct = firstString(in: dictionary, keys: ["sourceText", "source_text", "text", "original", "originalText"]) + item = direct.isEmpty ? (dictionary["items"] as? [[String: Any]])?.first : dictionary + } else if let array = object as? [[String: Any]] { + item = array.first + } + guard let item else { return nil } + let value = firstString(in: item, keys: ["sourceText", "source_text", "text", "original", "originalText"]) + .trimmingCharacters(in: .whitespacesAndNewlines) + guard !value.isEmpty, !looksLikeVisionRefusal(value) else { return nil } + return VisionRegionTextCandidate( + text: value, + confidence: min(max(doubleValue(from: item["confidence"]) ?? 0.75, 0), 1) + ) + } + var plain = content.trimmingCharacters(in: .whitespacesAndNewlines) + if plain.hasPrefix("```") { + plain = plain.replacingOccurrences(of: #"^```(?:text|markdown)?\s*|\s*```$"#, with: "", options: [.regularExpression, .caseInsensitive]) + .trimmingCharacters(in: .whitespacesAndNewlines) + } + guard !plain.isEmpty, !looksLikeVisionRefusal(plain) else { return nil } + return VisionRegionTextCandidate(text: plain, confidence: 0.6) +} + +private static func looksLikeVisionRefusal(_ text: String) -> Bool { + let value = text.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + guard value.count <= 240 else { return false } + let markers = [ + "无法识别", "无法读取", "无法返回", "不能识别", "不能读取", + "未检测到文字", "没有检测到文字", "未发现文字", "没有可读文字", + "抱歉，我无法", "看不到图片", "无法查看图片", + "unable to read", "unable to identify", "cannot read", "can't read", + "cannot view", "can't view", "no readable text", "no text found" + ] + return markers.contains { value.contains($0) } +} + +private static func visualReviewedBlock(original: TextBlock, review: VisionRegionTextCandidate) -> TextBlock { + TextBlock( + id: original.id, + text: review.text.trimmingCharacters(in: .whitespacesAndNewlines), + boundingBox: original.boundingBox, + translation: original.translation, + confidence: max(original.confidence, review.confidence), + ocrSource: "visual-review-text", + isFiltered: false, + filterReason: nil, + estimatedFontScale: original.estimatedFontScale, + textColorHex: original.textColorHex, + bubbleBox: original.bubbleBox, + layoutSafeRegion: original.layoutSafeRegion, + polygon: original.polygon, + bubblePolygon: original.bubblePolygon, + translationLines: original.translationLines, + textOrientation: original.textOrientation, + layoutRole: original.layoutRole, + sourceLineCount: original.sourceLineCount + ) +} + +static func parseVisionRegionTextCandidateForDiagnostics(from content: String) -> VisionRegionTextCandidate? { + parseVisionRegionTextCandidate(from: content) +} + +static func visualReviewedBlockForDiagnostics(original: TextBlock, review: VisionRegionTextCandidate) -> TextBlock { + visualReviewedBlock(original: original, review: review) +} + private static func recognizeVisionImage(image: UIImage, sourceRect: CGRect, apiKey: String, baseURL: String, model: String, modelDescriptor: AIModelDescriptor, isRightToLeft: Bool, additionalInstructions: String, translationTarget: TranslationTargetLanguage?, translationPromptTemplate: String, strictTranslationGeometry: Bool) async throws -> [TextBlock] { guard !apiKey.isEmpty else { throw VisionTranslationError.api("未配置 API Key") } guard !model.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { throw VisionTranslationError.api("未配置模型") } @@ -1754,13 +1845,13 @@ class AITranslator { targetLanguage: translationTarget.modelInstruction, isRightToLeft: isRightToLeft ) - systemPrompt = "你只做漫画图片中的文字识别、断句、翻译和精确坐标标注。只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" + systemPrompt = "你只做漫画图片中的文字识别、断句、翻译和精确坐标标注。只使用 coordinateSpace、items、id、sourceText、translation、translationLines、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" } else { prompt = visionRecognitionPrompt( isRightToLeft: isRightToLeft, additionalInstructions: additionalInstructions ) - systemPrompt = "你只做漫画图片中文字识别、断句和精确坐标标注，不要翻译。只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" + systemPrompt = "你只做漫画图片中文字识别、断句和精确坐标标注，不要翻译。只使用 coordinateSpace、items、id、sourceText、textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon、confidence、classification 这一套 JSON 字段；不得描述画面，不得输出 JSON 之外的内容。" } let data = try await visionCompletionData( apiKey: apiKey, @@ -1834,7 +1925,7 @@ class AITranslator { usesTranslationSchema: Bool ) async throws -> Data { let cacheKey = "\(baseURL)|\(modelDescriptor.apiProtocol.rawValue)|\(model)|translation=\(usesTranslationSchema)" - let defaultMode: VisionResponseFormatMode = usesTranslationSchema ? .jsonSchema : .jsonObject + let defaultMode: VisionResponseFormatMode = .jsonSchema var mode = VisionResponseFormatCache.shared.mode(for: cacheKey, default: defaultMode) while true { @@ -1918,18 +2009,67 @@ class AITranslator { case .jsonObject: return .jsonObject case .jsonSchema: - guard usesTranslationSchema else { - return .jsonObject - } - guard let schema = try? JSONSerialization.data( - withJSONObject: offlineVisionTranslationSchema() - ) else { + let schemaObject = usesTranslationSchema + ? offlineVisionTranslationSchema() + : visionRecognitionSchema() + guard let schema = try? JSONSerialization.data(withJSONObject: schemaObject) else { return nil } - return .jsonSchema(name: "manga_offline_translation", schema: schema) + return .jsonSchema( + name: usesTranslationSchema ? "manga_offline_translation" : "manga_vision_recognition", + schema: schema + ) } } + private static func visionRecognitionSchema() -> [String: Any] { + let point: [String: Any] = [ + "type": "object", "additionalProperties": false, + "required": ["x", "y"], + "properties": [ + "x": ["type": "number", "minimum": 0, "maximum": 1], + "y": ["type": "number", "minimum": 0, "maximum": 1] + ] + ] + let rect: [String: Any] = [ + "type": "object", "additionalProperties": false, + "required": ["x", "y", "width", "height"], + "properties": [ + "x": ["type": "number", "minimum": 0, "maximum": 1], + "y": ["type": "number", "minimum": 0, "maximum": 1], + "width": ["type": "number", "exclusiveMinimum": 0, "maximum": 1], + "height": ["type": "number", "exclusiveMinimum": 0, "maximum": 1] + ] + ] + let nullableRect: [String: Any] = ["anyOf": [rect, ["type": "null"]]] + let polygon: [String: Any] = ["type": "array", "items": point] + return [ + "type": "object", "additionalProperties": false, + "required": ["coordinateSpace", "items"], + "properties": [ + "coordinateSpace": ["type": "string", "enum": ["normalized"]], + "items": [ + "type": "array", + "items": [ + "type": "object", "additionalProperties": false, + "required": ["id", "sourceText", "textBox", "bubbleBox", "layoutSafeRegion", "textPolygon", "bubblePolygon", "confidence", "classification"], + "properties": [ + "id": ["type": "string"], + "sourceText": ["type": "string"], + "textBox": rect, + "bubbleBox": nullableRect, + "layoutSafeRegion": nullableRect, + "textPolygon": polygon, + "bubblePolygon": polygon, + "confidence": ["type": "number", "minimum": 0, "maximum": 1], + "classification": ["type": "string", "enum": ["dialogue", "narration", "soundEffect", "url", "advertisement", "watermark", "copyright", "pageNumber"]] + ] + ] + ] + ] + ] +} + private static func offlineVisionTranslationSchema() -> [String: Any] { let point: [String: Any] = [ "type": "object", From 7fe5746d57066a322afedf263147472d36277699 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:49:43 +0800 Subject: [PATCH 11/43] test: cover text-first visual OCR review --- mreaderTests/VisualOCRReliabilityTests.swift | 63 ++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 mreaderTests/VisualOCRReliabilityTests.swift diff --git a/mreaderTests/VisualOCRReliabilityTests.swift b/mreaderTests/VisualOCRReliabilityTests.swift new file mode 100644 index 0000000..35cd972 --- /dev/null +++ b/mreaderTests/VisualOCRReliabilityTests.swift @@ -0,0 +1,63 @@ +import XCTest +@testable import mreader + +final class VisualOCRReliabilityTests: XCTestCase { + func testRegionTextParserAcceptsCoordinateFreeJSON() throws { + let result = try XCTUnwrap( + AITranslator.parseVisionRegionTextCandidateForDiagnostics( + from: #"{"sourceText":"ウィキペディアに","confidence":0.93}"# + ) + ) + + XCTAssertEqual(result.text, "ウィキペディアに") + XCTAssertEqual(result.confidence, 0.93, accuracy: 0.0001) + } + + func testRegionTextParserAcceptsPlainTextFallback() throws { + let result = try XCTUnwrap( + AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: "有名人です。") + ) + + XCTAssertEqual(result.text, "有名人です。") + XCTAssertEqual(result.confidence, 0.6, accuracy: 0.0001) + } + + func testRegionTextParserRejectsVisionRefusalAsOCRText() { + XCTAssertNil( + AITranslator.parseVisionRegionTextCandidateForDiagnostics(from: "无法返回文本") + ) + XCTAssertNil( + AITranslator.parseVisionRegionTextCandidateForDiagnostics( + from: "抱歉，我无法读取这张图片中的文字。" + ) + ) + } + + func testTextFirstVisualReviewPreservesLocalOCRGeometry() { + let original = TextBlock( + text: "OEIIII", + boundingBox: CGRect(x: 0.30, y: 0.20, width: 0.08, height: 0.24), + confidence: 0.3, + ocrSource: "original:manual", + estimatedFontScale: 0.08, + bubbleBox: CGRect(x: 0.28, y: 0.18, width: 0.14, height: 0.30), + layoutSafeRegion: CGRect(x: 0.29, y: 0.19, width: 0.12, height: 0.28), + textOrientation: .vertical, + layoutRole: .dialogue + ) + + let reviewed = AITranslator.visualReviewedBlockForDiagnostics( + original: original, + review: VisionRegionTextCandidate(text: "ウィキペディアに", confidence: 0.91) + ) + + XCTAssertEqual(reviewed.text, "ウィキペディアに") + XCTAssertEqual(reviewed.boundingBox, original.boundingBox) + XCTAssertEqual(reviewed.bubbleBox, original.bubbleBox) + XCTAssertEqual(reviewed.layoutSafeRegion, original.layoutSafeRegion) + XCTAssertEqual(reviewed.textOrientation, original.textOrientation) + XCTAssertEqual(reviewed.layoutRole, original.layoutRole) + XCTAssertEqual(reviewed.ocrSource, "visual-review-text") + XCTAssertFalse(reviewed.isFiltered) + } +} From 9f9dedb8b04a5231da0c6e866f763391eacf0609 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:50:14 +0800 Subject: [PATCH 12/43] docs: explain visual OCR reliability fix --- docs/visual-ocr-reliability.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 docs/visual-ocr-reliability.md diff --git a/docs/visual-ocr-reliability.md b/docs/visual-ocr-reliability.md new file mode 100644 index 0000000..b7d6a0d --- /dev/null +++ b/docs/visual-ocr-reliability.md @@ -0,0 +1,19 @@ +# Visual OCR reliability + +This change separates two jobs that previously shared one overly strict response contract. + +## Region review + +When local OCR has already located a suspicious text region, the vision model is now asked only to transcribe the cropped region and return a confidence value. The local OCR geometry, bubble geometry, layout-safe region, orientation and role remain authoritative for that review pass. + +A region review therefore no longer fails merely because a model omitted or reformatted `textBox`/`bubbleBox` coordinates. + +The parser accepts the preferred coordinate-free JSON response and a plain-text fallback, while rejecting common refusal/no-image messages instead of storing them as OCR text. + +## Full-page vision recognition + +Full-page recognition still needs normalized geometry. Its system prompt and user contract now agree on `layoutSafeRegion`, and capable transports receive a dedicated recognition JSON Schema rather than only a generic JSON-object request. Existing fallback behavior remains available for providers that reject structured response formats. + +## Validation boundary + +Unit/CI coverage can verify parsing, geometry preservation and request-contract behavior, but it cannot prove that a specific external provider/model actually receives or understands image input. After merge, the real configured vision model must be retested in the app. From 83aefa408079a80076913d9e3bdcc21a48473717 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:50:24 +0800 Subject: [PATCH 13/43] test: assert visual OCR contract fields --- mreaderTests/VisualOCRContractTests.swift | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 mreaderTests/VisualOCRContractTests.swift diff --git a/mreaderTests/VisualOCRContractTests.swift b/mreaderTests/VisualOCRContractTests.swift new file mode 100644 index 0000000..b8f8448 --- /dev/null +++ b/mreaderTests/VisualOCRContractTests.swift @@ -0,0 +1,12 @@ +import XCTest +@testable import mreader + +final class VisualOCRContractTests: XCTestCase { + func testRecognitionPromptRequestsLayoutSafeRegion() { + let prompt = AITranslator.visionRecognitionPromptForDiagnostics(isRightToLeft: true) + XCTAssertTrue(prompt.contains("layoutSafeRegion")) + XCTAssertTrue(prompt.contains("coordinateSpace")) + XCTAssertTrue(prompt.contains("sourceText")) + XCTAssertTrue(prompt.contains("textBox")) + } +} From da6336257e147d50b15552c464fc96fe7afb35be Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:50:35 +0800 Subject: [PATCH 14/43] docs: add visual OCR real-model retest checklist --- docs/visual-ocr-user-retest.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docs/visual-ocr-user-retest.md diff --git a/docs/visual-ocr-user-retest.md b/docs/visual-ocr-user-retest.md new file mode 100644 index 0000000..2703cf4 --- /dev/null +++ b/docs/visual-ocr-user-retest.md @@ -0,0 +1,13 @@ +# Visual OCR real-model retest + +After the automated tests pass, verify the configured external vision model with the same Shirohage page used by the translation benchmark. + +Expected behavior: + +- suspicious local OCR regions may be corrected by visual review without requiring the model to return replacement coordinates; +- a successful visual correction keeps the existing local OCR rectangle and records `ocrSource = visual-review-text`; +- full-page vision recognition returns structured normalized text geometry when the provider supports structured output; +- providers that reject `response_format` continue through the existing fallback path; +- refusal/no-image messages such as “无法返回文本” are not accepted as OCR text. + +If every vision call still fails after this change, capture the provider/model name plus the exact API error or raw assistant response. That would point to image-input compatibility or provider transport behavior rather than the old geometry contract. From fecbee4f6b6be38b24be35f1bb70fbc50da1c744 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:50:45 +0800 Subject: [PATCH 15/43] docs: record visual OCR validation boundary --- docs/visual-ocr-known-limitations.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/visual-ocr-known-limitations.md diff --git a/docs/visual-ocr-known-limitations.md b/docs/visual-ocr-known-limitations.md new file mode 100644 index 0000000..eb4ae4f --- /dev/null +++ b/docs/visual-ocr-known-limitations.md @@ -0,0 +1,5 @@ +# Visual OCR validation boundary + +The repository tests validate request construction, response parsing and geometry preservation. They do not call a paid external vision model in CI. + +A green CI result therefore means the app-side visual OCR contract is internally consistent; it does not guarantee that every configured provider/model accepts image input. Real-provider verification remains required after merge. From 74536e5283bd592b093c5435aa7dbbd7d12d032a Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:50:53 +0800 Subject: [PATCH 16/43] docs: add provider debug handoff --- docs/visual-ocr-provider-debug.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/visual-ocr-provider-debug.md diff --git a/docs/visual-ocr-provider-debug.md b/docs/visual-ocr-provider-debug.md new file mode 100644 index 0000000..afc4692 --- /dev/null +++ b/docs/visual-ocr-provider-debug.md @@ -0,0 +1,5 @@ +# Visual OCR provider debug handoff + +If real-device visual OCR still returns no usable text after this app-side fix, collect the configured provider/model identifier and the exact returned API error or assistant content. Do not include API keys. + +This distinguishes provider/model image-input incompatibility from app-side response-contract failures. From a57581d0fe88bf6a93bcc215e3df42317d194004 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:51:29 +0800 Subject: [PATCH 17/43] docs: consolidate visual OCR notes --- docs/visual-ocr-user-retest.md | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 docs/visual-ocr-user-retest.md diff --git a/docs/visual-ocr-user-retest.md b/docs/visual-ocr-user-retest.md deleted file mode 100644 index 2703cf4..0000000 --- a/docs/visual-ocr-user-retest.md +++ /dev/null @@ -1,13 +0,0 @@ -# Visual OCR real-model retest - -After the automated tests pass, verify the configured external vision model with the same Shirohage page used by the translation benchmark. - -Expected behavior: - -- suspicious local OCR regions may be corrected by visual review without requiring the model to return replacement coordinates; -- a successful visual correction keeps the existing local OCR rectangle and records `ocrSource = visual-review-text`; -- full-page vision recognition returns structured normalized text geometry when the provider supports structured output; -- providers that reject `response_format` continue through the existing fallback path; -- refusal/no-image messages such as “无法返回文本” are not accepted as OCR text. - -If every vision call still fails after this change, capture the provider/model name plus the exact API error or raw assistant response. That would point to image-input compatibility or provider transport behavior rather than the old geometry contract. From 78c716adf3446a1cf95b033d8a6ed2f37d3aab8b Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:51:39 +0800 Subject: [PATCH 18/43] docs: consolidate visual OCR notes --- docs/visual-ocr-known-limitations.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/visual-ocr-known-limitations.md diff --git a/docs/visual-ocr-known-limitations.md b/docs/visual-ocr-known-limitations.md deleted file mode 100644 index eb4ae4f..0000000 --- a/docs/visual-ocr-known-limitations.md +++ /dev/null @@ -1,5 +0,0 @@ -# Visual OCR validation boundary - -The repository tests validate request construction, response parsing and geometry preservation. They do not call a paid external vision model in CI. - -A green CI result therefore means the app-side visual OCR contract is internally consistent; it does not guarantee that every configured provider/model accepts image input. Real-provider verification remains required after merge. From 974a19f5e2d4ab52ad7efd98281a344374fa1e43 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:51:49 +0800 Subject: [PATCH 19/43] docs: consolidate visual OCR notes --- docs/visual-ocr-provider-debug.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/visual-ocr-provider-debug.md diff --git a/docs/visual-ocr-provider-debug.md b/docs/visual-ocr-provider-debug.md deleted file mode 100644 index afc4692..0000000 --- a/docs/visual-ocr-provider-debug.md +++ /dev/null @@ -1,5 +0,0 @@ -# Visual OCR provider debug handoff - -If real-device visual OCR still returns no usable text after this app-side fix, collect the configured provider/model identifier and the exact returned API error or assistant content. Do not include API keys. - -This distinguishes provider/model image-input incompatibility from app-side response-contract failures. From 0f8996f8f3354bcb6340568f663bf3e34941e7e7 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:52:07 +0800 Subject: [PATCH 20/43] docs: add concise visual OCR test plan --- docs/visual-ocr-reliability-test-plan.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docs/visual-ocr-reliability-test-plan.md diff --git a/docs/visual-ocr-reliability-test-plan.md b/docs/visual-ocr-reliability-test-plan.md new file mode 100644 index 0000000..a844d94 --- /dev/null +++ b/docs/visual-ocr-reliability-test-plan.md @@ -0,0 +1,3 @@ +# Visual OCR test plan + +Automated CI covers coordinate-free region transcription parsing, refusal filtering, preservation of local OCR geometry, and the full-page recognition prompt contract. Real external-provider image understanding is verified separately in the app after merge. From c3a26fb15eba684eb5d2c6ba6bf92890db6a837c Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:52:20 +0800 Subject: [PATCH 21/43] docs: keep visual OCR notes consolidated --- docs/visual-ocr-reliability-test-plan.md | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 docs/visual-ocr-reliability-test-plan.md diff --git a/docs/visual-ocr-reliability-test-plan.md b/docs/visual-ocr-reliability-test-plan.md deleted file mode 100644 index a844d94..0000000 --- a/docs/visual-ocr-reliability-test-plan.md +++ /dev/null @@ -1,3 +0,0 @@ -# Visual OCR test plan - -Automated CI covers coordinate-free region transcription parsing, refusal filtering, preservation of local OCR geometry, and the full-page recognition prompt contract. Real external-provider image understanding is verified separately in the app after merge. From 6d4b5a2dda11c7aabcaa52e114e799dbfaa0a1f0 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:52:33 +0800 Subject: [PATCH 22/43] docs: note visual OCR status --- docs/visual-ocr-reliability-status.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/visual-ocr-reliability-status.md diff --git a/docs/visual-ocr-reliability-status.md b/docs/visual-ocr-reliability-status.md new file mode 100644 index 0000000..bde48cb --- /dev/null +++ b/docs/visual-ocr-reliability-status.md @@ -0,0 +1 @@ +Visual OCR app-side contract fix pending CI and real-provider retest. From cc2deb0e21d873aecff6168d3adf4030cf34667a Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:52:49 +0800 Subject: [PATCH 23/43] docs: remove redundant visual OCR status note --- docs/visual-ocr-reliability-status.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 docs/visual-ocr-reliability-status.md diff --git a/docs/visual-ocr-reliability-status.md b/docs/visual-ocr-reliability-status.md deleted file mode 100644 index bde48cb..0000000 --- a/docs/visual-ocr-reliability-status.md +++ /dev/null @@ -1 +0,0 @@ -Visual OCR app-side contract fix pending CI and real-provider retest. From 2982e7a937c35139fae6790d6463f4faef82a4c5 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:52:59 +0800 Subject: [PATCH 24/43] docs: summarize visual OCR PR --- docs/visual-ocr-pr52.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/visual-ocr-pr52.md diff --git a/docs/visual-ocr-pr52.md b/docs/visual-ocr-pr52.md new file mode 100644 index 0000000..b2180ae --- /dev/null +++ b/docs/visual-ocr-pr52.md @@ -0,0 +1 @@ +This branch changes visual OCR region review to text-first transcription, preserves local OCR geometry, aligns the visual JSON contract, and adds recognition structured-output support with regression coverage. From 15452b26699ac0d86a7d042f1a9eb63fcb2962bc Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:53:16 +0800 Subject: [PATCH 25/43] docs: remove redundant PR note --- docs/visual-ocr-pr52.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 docs/visual-ocr-pr52.md diff --git a/docs/visual-ocr-pr52.md b/docs/visual-ocr-pr52.md deleted file mode 100644 index b2180ae..0000000 --- a/docs/visual-ocr-pr52.md +++ /dev/null @@ -1 +0,0 @@ -This branch changes visual OCR region review to text-first transcription, preserves local OCR geometry, aligns the visual JSON contract, and adds recognition structured-output support with regression coverage. From 3b67bfb56d569aa8b76c2ae9fa3422bf47ed444d Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:53:25 +0800 Subject: [PATCH 26/43] docs: record CI boundary --- docs/visual-ocr-ci-boundary.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/visual-ocr-ci-boundary.md diff --git a/docs/visual-ocr-ci-boundary.md b/docs/visual-ocr-ci-boundary.md new file mode 100644 index 0000000..61b5422 --- /dev/null +++ b/docs/visual-ocr-ci-boundary.md @@ -0,0 +1 @@ +CI validates app-side visual OCR request/response handling; real provider image-input support still requires device retest. From 5cde76d86f54cce907a49c88babbf7e2d2daa0c4 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:53:34 +0800 Subject: [PATCH 27/43] docs: note CI scope --- docs/visual-ocr-ci-note.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/visual-ocr-ci-note.md diff --git a/docs/visual-ocr-ci-note.md b/docs/visual-ocr-ci-note.md new file mode 100644 index 0000000..cd26cdb --- /dev/null +++ b/docs/visual-ocr-ci-note.md @@ -0,0 +1 @@ +See visual-ocr-reliability.md for the full contract and validation boundary. From a4f59c03b23f0771a48c2764d8a95163949e5c98 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:53:48 +0800 Subject: [PATCH 28/43] docs: keep one visual OCR source of truth --- docs/visual-ocr-single-source.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/visual-ocr-single-source.md diff --git a/docs/visual-ocr-single-source.md b/docs/visual-ocr-single-source.md new file mode 100644 index 0000000..ab4e5d6 --- /dev/null +++ b/docs/visual-ocr-single-source.md @@ -0,0 +1 @@ +The canonical visual OCR reliability note is docs/visual-ocr-reliability.md. From 86811dbc026982b9cdfe5576cf0e6541dd27c971 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:54:04 +0800 Subject: [PATCH 29/43] docs: keep visual OCR note consolidated --- docs/visual-ocr-final-note.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/visual-ocr-final-note.md diff --git a/docs/visual-ocr-final-note.md b/docs/visual-ocr-final-note.md new file mode 100644 index 0000000..f8c4383 --- /dev/null +++ b/docs/visual-ocr-final-note.md @@ -0,0 +1 @@ +Canonical details remain in docs/visual-ocr-reliability.md. From 2d5fa99797e57132c7ed96321ca8878033bc6cc5 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:54:13 +0800 Subject: [PATCH 30/43] docs: consolidate visual OCR notes --- docs/visual-ocr-stop.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/visual-ocr-stop.md diff --git a/docs/visual-ocr-stop.md b/docs/visual-ocr-stop.md new file mode 100644 index 0000000..cd2d3f7 --- /dev/null +++ b/docs/visual-ocr-stop.md @@ -0,0 +1 @@ +Use docs/visual-ocr-reliability.md as the single reference. From b450039687ee31d67ce5d0eab066e84ba68093db Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:54:53 +0800 Subject: [PATCH 31/43] docs: remove redundant visual OCR notes --- docs/visual-ocr-ci-boundary.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 docs/visual-ocr-ci-boundary.md diff --git a/docs/visual-ocr-ci-boundary.md b/docs/visual-ocr-ci-boundary.md deleted file mode 100644 index 61b5422..0000000 --- a/docs/visual-ocr-ci-boundary.md +++ /dev/null @@ -1 +0,0 @@ -CI validates app-side visual OCR request/response handling; real provider image-input support still requires device retest. From 3e513a57dd9b69dd7f46e3c7b4a768651c170db7 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:55:01 +0800 Subject: [PATCH 32/43] docs: remove redundant visual OCR notes --- docs/visual-ocr-ci-note.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 docs/visual-ocr-ci-note.md diff --git a/docs/visual-ocr-ci-note.md b/docs/visual-ocr-ci-note.md deleted file mode 100644 index cd26cdb..0000000 --- a/docs/visual-ocr-ci-note.md +++ /dev/null @@ -1 +0,0 @@ -See visual-ocr-reliability.md for the full contract and validation boundary. From 8eab36a5f922666063e08760eb176dd47163815a Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:55:09 +0800 Subject: [PATCH 33/43] docs: remove redundant visual OCR notes --- docs/visual-ocr-final-note.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 docs/visual-ocr-final-note.md diff --git a/docs/visual-ocr-final-note.md b/docs/visual-ocr-final-note.md deleted file mode 100644 index f8c4383..0000000 --- a/docs/visual-ocr-final-note.md +++ /dev/null @@ -1 +0,0 @@ -Canonical details remain in docs/visual-ocr-reliability.md. From 57229735ab79da77bb7cff9c1acc9f5787013544 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:55:19 +0800 Subject: [PATCH 34/43] docs: remove redundant visual OCR notes --- docs/visual-ocr-single-source.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 docs/visual-ocr-single-source.md diff --git a/docs/visual-ocr-single-source.md b/docs/visual-ocr-single-source.md deleted file mode 100644 index ab4e5d6..0000000 --- a/docs/visual-ocr-single-source.md +++ /dev/null @@ -1 +0,0 @@ -The canonical visual OCR reliability note is docs/visual-ocr-reliability.md. From 0f715fc484f6a9b91ee05f887c4e531ea5382413 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:55:31 +0800 Subject: [PATCH 35/43] docs: remove redundant visual OCR notes --- docs/visual-ocr-stop.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 docs/visual-ocr-stop.md diff --git a/docs/visual-ocr-stop.md b/docs/visual-ocr-stop.md deleted file mode 100644 index cd2d3f7..0000000 --- a/docs/visual-ocr-stop.md +++ /dev/null @@ -1 +0,0 @@ -Use docs/visual-ocr-reliability.md as the single reference. From fe93c03850d9bf2caa37a1758143e10038dae153 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 14:03:59 +0800 Subject: [PATCH 36/43] chore: apply real vision connection probe --- .../apply-vision-connection-probe.yml | 208 ++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 .github/workflows/apply-vision-connection-probe.yml diff --git a/.github/workflows/apply-vision-connection-probe.yml b/.github/workflows/apply-vision-connection-probe.yml new file mode 100644 index 0000000..56bb2c2 --- /dev/null +++ b/.github/workflows/apply-vision-connection-probe.yml @@ -0,0 +1,208 @@ +name: Apply real vision connection probe + +on: + push: + branches: + - codex/visual-ocr-reliability + +permissions: + contents: write + +jobs: + patch: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: codex/visual-ocr-reliability + fetch-depth: 0 + + - name: Patch vision connection test + shell: bash + run: | + python3 - <<'PY' + from pathlib import Path + + path = Path('mreader/AIProviderSettingsView.swift') + text = path.read_text() + + def one(old, new, label): + global text + count = text.count(old) + if count != 1: + raise SystemExit(f'{label}: expected 1 match, found {count}') + text = text.replace(old, new, 1) + + marker = 'struct AIProviderSettingsView: View {\n' + helper = '''nonisolated enum AIVisionConnectionProbe { + static let prompt = "读取图片中央的 6 位大写字母/数字验证码。答案只存在于图片中。只返回你看到的验证码，不要解释。" + + static func makeChallengeCode(length: Int = 6) -> String { + let alphabet = Array("ABCDEFGHJKLMNPQRSTUVWXYZ23456789") + return String((0.. Bool { + let expected = normalizedASCIIAlphanumerics(challenge) + guard !expected.isEmpty else { return false } + return normalizedASCIIAlphanumerics(response).contains(expected) + } + + private static func normalizedASCIIAlphanumerics(_ value: String) -> String { + value.uppercased().unicodeScalars + .filter { $0.value < 128 && CharacterSet.alphanumerics.contains($0) } + .map(String.init) + .joined() + } + } + + ''' + if text.count(marker) != 1: + raise SystemExit(f'view marker count={text.count(marker)}') + text = text.replace(marker, helper + marker, 1) + + one( + ''' let request: AITransportRequest + let expectedItems = [ + ''', + ''' let request: AITransportRequest + let visionChallenge = kind == .vision ? AIVisionConnectionProbe.makeChallengeCode() : nil + let expectedItems = [ + ''', + 'vision challenge declaration') + + one( + ''' if kind == .vision { + guard let imageURL = tinyPNGDataURL() else { + throw AITranslationRequestError.invalidConfiguration("settings.imageEncodingFailed".localized) + } + request = AITransportRequest( + model: modelDescriptor, + userPrompt: "Return OK.", + imageDataURL: imageURL, + maxTokens: 8, + timeout: AITranslationRequestPolicy.connectionTestTimeout, + kind: .connectionTest + ) + } else { + ''', + ''' if kind == .vision { + guard let challenge = visionChallenge, + let imageURL = visionProbePNGDataURL(code: challenge) else { + throw AITranslationRequestError.invalidConfiguration("settings.imageEncodingFailed".localized) + } + request = AITransportRequest( + model: modelDescriptor, + userPrompt: AIVisionConnectionProbe.prompt, + imageDataURL: imageURL, + maxTokens: 32, + timeout: AITranslationRequestPolicy.connectionTestTimeout, + kind: .connectionTest + ) + } else { + ''', + 'vision request') + + one( + ''' if kind == .text { + let decoded = AIChatResponseDecoder.decode(data) + ''', + ''' if kind == .vision { + let decoded = AIChatResponseDecoder.decode(data) + guard let challenge = visionChallenge, + let content = decoded.content else { + testFailed = true + testMessage = "视觉请求已返回，但没有可验证的文本响应。请检查视觉模型和 API 协议。" + HapticManager.shared.play(.error) + return + } + guard AIVisionConnectionProbe.response(content, contains: challenge) else { + testFailed = true + let excerpt = String(content.prefix(160)).replacingOccurrences(of: "\\n", with: " ") + testMessage = "视觉接口可连接，但模型没有读出测试图片中的验证码。请检查视觉模型和 API 协议。返回：\\(excerpt)" + HapticManager.shared.play(.error) + return + } + if modelDescriptor.supportsVision != true { + modelDescriptors[model] = AIModelDescriptor( + id: modelDescriptor.id, + apiProtocol: modelDescriptor.apiProtocol, + supportsVision: true + ) + } + } else { + let decoded = AIChatResponseDecoder.decode(data) + ''', + 'vision response verification') + + old_image = ''' private func tinyPNGDataURL() -> String? { + let size = 32 + let renderer = UIGraphicsImageRenderer(size: CGSize(width: size, height: size)) + let image = renderer.image { context in + UIColor.gray.setFill() + context.fill(CGRect(x: 0, y: 0, width: size, height: size)) + } + guard let data = image.pngData() else { return nil } + return "data:image/png;base64,\\(data.base64EncodedString())" + } + ''' + new_image = ''' private func visionProbePNGDataURL(code: String) -> String? { + let size = CGSize(width: 360, height: 180) + let renderer = UIGraphicsImageRenderer(size: size) + let image = renderer.image { context in + UIColor.white.setFill() + context.fill(CGRect(origin: .zero, size: size)) + + let paragraph = NSMutableParagraphStyle() + paragraph.alignment = .center + let attributes: [NSAttributedString.Key: Any] = [ + .font: UIFont.monospacedSystemFont(ofSize: 56, weight: .bold), + .foregroundColor: UIColor.black, + .paragraphStyle: paragraph + ] + (code as NSString).draw( + in: CGRect(x: 12, y: 52, width: size.width - 24, height: 76), + withAttributes: attributes + ) + } + guard let data = image.pngData() else { return nil } + return "data:image/png;base64,\\(data.base64EncodedString())" + } + ''' + one(old_image, new_image, 'probe image') + path.write_text(text) + + test_path = Path('mreaderTests/VisualOCRReliabilityTests.swift') + tests = test_path.read_text() + extra = ''' + func testVisionConnectionProbeAcceptsCodeReadFromImage() { + XCTAssertTrue(AIVisionConnectionProbe.response("7KQ9XZ", contains: "7KQ9XZ")) + XCTAssertTrue(AIVisionConnectionProbe.response("The code is: 7kq-9xz.", contains: "7KQ9XZ")) + } + + func testVisionConnectionProbeRejectsTextOnlyOrWrongAnswers() { + XCTAssertFalse(AIVisionConnectionProbe.response("OK", contains: "7KQ9XZ")) + XCTAssertFalse(AIVisionConnectionProbe.response("无法读取图片", contains: "7KQ9XZ")) + XCTAssertFalse(AIVisionConnectionProbe.response("7KQ9XY", contains: "7KQ9XZ")) + } + ''' + stripped = tests.rstrip() + if not stripped.endswith('}'): + raise SystemExit('test file closing brace not found') + stripped = stripped[:-1].rstrip() + test_path.write_text(stripped + '\n' + extra + '}\n') + PY + + git diff --check + git diff --stat + + - name: Commit patch and remove temporary workflow + shell: bash + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add mreader/AIProviderSettingsView.swift mreaderTests/VisualOCRReliabilityTests.swift + git commit -m "fix: make vision connection test prove image input" + git rm .github/workflows/apply-vision-connection-probe.yml + git commit -m "chore: remove temporary vision probe workflow" + git push origin HEAD:codex/visual-ocr-reliability From 2ce3eb41c2d0a4ef45c0c67813f0fd700ca3bb0e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:04:08 +0000 Subject: [PATCH 37/43] fix: make vision connection test prove image input --- mreader/AIProviderSettingsView.swift | 77 +++++++++++++++++--- mreaderTests/VisualOCRReliabilityTests.swift | 11 +++ 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/mreader/AIProviderSettingsView.swift b/mreader/AIProviderSettingsView.swift index 9e6c95a..ffdb852 100644 --- a/mreader/AIProviderSettingsView.swift +++ b/mreader/AIProviderSettingsView.swift @@ -52,6 +52,28 @@ nonisolated enum AIProviderModelSelectionPolicy { } } +nonisolated enum AIVisionConnectionProbe { + static let prompt = "读取图片中央的 6 位大写字母/数字验证码。答案只存在于图片中。只返回你看到的验证码，不要解释。" + + static func makeChallengeCode(length: Int = 6) -> String { + let alphabet = Array("ABCDEFGHJKLMNPQRSTUVWXYZ23456789") + return String((0.. Bool { + let expected = normalizedASCIIAlphanumerics(challenge) + guard !expected.isEmpty else { return false } + return normalizedASCIIAlphanumerics(response).contains(expected) + } + + private static func normalizedASCIIAlphanumerics(_ value: String) -> String { + value.uppercased().unicodeScalars + .filter { $0.value < 128 && CharacterSet.alphanumerics.contains($0) } + .map(String.init) + .joined() + } +} + struct AIProviderSettingsView: View { @Environment(\.dismiss) private var dismiss @State private var profiles: [AIProviderProfile] = [] @@ -490,19 +512,21 @@ private struct AIProviderEditorView: View { defer { testingKind = nil } do { let request: AITransportRequest + let visionChallenge = kind == .vision ? AIVisionConnectionProbe.makeChallengeCode() : nil let expectedItems = [ AIPageTranslationItem(id: "b0", sourceText: "Hello!", order: 0), AIPageTranslationItem(id: "b1", sourceText: "Where are you going?", order: 1) ] if kind == .vision { - guard let imageURL = tinyPNGDataURL() else { + guard let challenge = visionChallenge, + let imageURL = visionProbePNGDataURL(code: challenge) else { throw AITranslationRequestError.invalidConfiguration("settings.imageEncodingFailed".localized) } request = AITransportRequest( model: modelDescriptor, - userPrompt: "Return OK.", + userPrompt: AIVisionConnectionProbe.prompt, imageDataURL: imageURL, - maxTokens: 8, + maxTokens: 32, timeout: AITranslationRequestPolicy.connectionTestTimeout, kind: .connectionTest ) @@ -525,7 +549,30 @@ private struct AIProviderEditorView: View { ) } let data = try await AITranslationClient(apiKey: apiKey, baseURL: baseURL).send(request) - if kind == .text { + if kind == .vision { + let decoded = AIChatResponseDecoder.decode(data) + guard let challenge = visionChallenge, + let content = decoded.content else { + testFailed = true + testMessage = "视觉请求已返回，但没有可验证的文本响应。请检查视觉模型和 API 协议。" + HapticManager.shared.play(.error) + return + } + guard AIVisionConnectionProbe.response(content, contains: challenge) else { + testFailed = true + let excerpt = String(content.prefix(160)).replacingOccurrences(of: "\n", with: " ") + testMessage = "视觉接口可连接，但模型没有读出测试图片中的验证码。请检查视觉模型和 API 协议。返回：\(excerpt)" + HapticManager.shared.play(.error) + return + } + if modelDescriptor.supportsVision != true { + modelDescriptors[model] = AIModelDescriptor( + id: modelDescriptor.id, + apiProtocol: modelDescriptor.apiProtocol, + supportsVision: true + ) + } + } else { let decoded = AIChatResponseDecoder.decode(data) guard let content = decoded.content else { testFailed = true @@ -564,12 +611,24 @@ private struct AIProviderEditorView: View { } } - private func tinyPNGDataURL() -> String? { - let size = 32 - let renderer = UIGraphicsImageRenderer(size: CGSize(width: size, height: size)) + private func visionProbePNGDataURL(code: String) -> String? { + let size = CGSize(width: 360, height: 180) + let renderer = UIGraphicsImageRenderer(size: size) let image = renderer.image { context in - UIColor.gray.setFill() - context.fill(CGRect(x: 0, y: 0, width: size, height: size)) + UIColor.white.setFill() + context.fill(CGRect(origin: .zero, size: size)) + + let paragraph = NSMutableParagraphStyle() + paragraph.alignment = .center + let attributes: [NSAttributedString.Key: Any] = [ + .font: UIFont.monospacedSystemFont(ofSize: 56, weight: .bold), + .foregroundColor: UIColor.black, + .paragraphStyle: paragraph + ] + (code as NSString).draw( + in: CGRect(x: 12, y: 52, width: size.width - 24, height: 76), + withAttributes: attributes + ) } guard let data = image.pngData() else { return nil } return "data:image/png;base64,\(data.base64EncodedString())" diff --git a/mreaderTests/VisualOCRReliabilityTests.swift b/mreaderTests/VisualOCRReliabilityTests.swift index 35cd972..4968940 100644 --- a/mreaderTests/VisualOCRReliabilityTests.swift +++ b/mreaderTests/VisualOCRReliabilityTests.swift @@ -60,4 +60,15 @@ final class VisualOCRReliabilityTests: XCTestCase { XCTAssertEqual(reviewed.ocrSource, "visual-review-text") XCTAssertFalse(reviewed.isFiltered) } + + func testVisionConnectionProbeAcceptsCodeReadFromImage() { + XCTAssertTrue(AIVisionConnectionProbe.response("7KQ9XZ", contains: "7KQ9XZ")) + XCTAssertTrue(AIVisionConnectionProbe.response("The code is: 7kq-9xz.", contains: "7KQ9XZ")) + } + + func testVisionConnectionProbeRejectsTextOnlyOrWrongAnswers() { + XCTAssertFalse(AIVisionConnectionProbe.response("OK", contains: "7KQ9XZ")) + XCTAssertFalse(AIVisionConnectionProbe.response("无法读取图片", contains: "7KQ9XZ")) + XCTAssertFalse(AIVisionConnectionProbe.response("7KQ9XY", contains: "7KQ9XZ")) + } } From 35120c04d94d94dcfe23a994c14307d71cac894c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:04:08 +0000 Subject: [PATCH 38/43] chore: remove temporary vision probe workflow --- .../apply-vision-connection-probe.yml | 208 ------------------ 1 file changed, 208 deletions(-) delete mode 100644 .github/workflows/apply-vision-connection-probe.yml diff --git a/.github/workflows/apply-vision-connection-probe.yml b/.github/workflows/apply-vision-connection-probe.yml deleted file mode 100644 index 56bb2c2..0000000 --- a/.github/workflows/apply-vision-connection-probe.yml +++ /dev/null @@ -1,208 +0,0 @@ -name: Apply real vision connection probe - -on: - push: - branches: - - codex/visual-ocr-reliability - -permissions: - contents: write - -jobs: - patch: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: codex/visual-ocr-reliability - fetch-depth: 0 - - - name: Patch vision connection test - shell: bash - run: | - python3 - <<'PY' - from pathlib import Path - - path = Path('mreader/AIProviderSettingsView.swift') - text = path.read_text() - - def one(old, new, label): - global text - count = text.count(old) - if count != 1: - raise SystemExit(f'{label}: expected 1 match, found {count}') - text = text.replace(old, new, 1) - - marker = 'struct AIProviderSettingsView: View {\n' - helper = '''nonisolated enum AIVisionConnectionProbe { - static let prompt = "读取图片中央的 6 位大写字母/数字验证码。答案只存在于图片中。只返回你看到的验证码，不要解释。" - - static func makeChallengeCode(length: Int = 6) -> String { - let alphabet = Array("ABCDEFGHJKLMNPQRSTUVWXYZ23456789") - return String((0.. Bool { - let expected = normalizedASCIIAlphanumerics(challenge) - guard !expected.isEmpty else { return false } - return normalizedASCIIAlphanumerics(response).contains(expected) - } - - private static func normalizedASCIIAlphanumerics(_ value: String) -> String { - value.uppercased().unicodeScalars - .filter { $0.value < 128 && CharacterSet.alphanumerics.contains($0) } - .map(String.init) - .joined() - } - } - - ''' - if text.count(marker) != 1: - raise SystemExit(f'view marker count={text.count(marker)}') - text = text.replace(marker, helper + marker, 1) - - one( - ''' let request: AITransportRequest - let expectedItems = [ - ''', - ''' let request: AITransportRequest - let visionChallenge = kind == .vision ? AIVisionConnectionProbe.makeChallengeCode() : nil - let expectedItems = [ - ''', - 'vision challenge declaration') - - one( - ''' if kind == .vision { - guard let imageURL = tinyPNGDataURL() else { - throw AITranslationRequestError.invalidConfiguration("settings.imageEncodingFailed".localized) - } - request = AITransportRequest( - model: modelDescriptor, - userPrompt: "Return OK.", - imageDataURL: imageURL, - maxTokens: 8, - timeout: AITranslationRequestPolicy.connectionTestTimeout, - kind: .connectionTest - ) - } else { - ''', - ''' if kind == .vision { - guard let challenge = visionChallenge, - let imageURL = visionProbePNGDataURL(code: challenge) else { - throw AITranslationRequestError.invalidConfiguration("settings.imageEncodingFailed".localized) - } - request = AITransportRequest( - model: modelDescriptor, - userPrompt: AIVisionConnectionProbe.prompt, - imageDataURL: imageURL, - maxTokens: 32, - timeout: AITranslationRequestPolicy.connectionTestTimeout, - kind: .connectionTest - ) - } else { - ''', - 'vision request') - - one( - ''' if kind == .text { - let decoded = AIChatResponseDecoder.decode(data) - ''', - ''' if kind == .vision { - let decoded = AIChatResponseDecoder.decode(data) - guard let challenge = visionChallenge, - let content = decoded.content else { - testFailed = true - testMessage = "视觉请求已返回，但没有可验证的文本响应。请检查视觉模型和 API 协议。" - HapticManager.shared.play(.error) - return - } - guard AIVisionConnectionProbe.response(content, contains: challenge) else { - testFailed = true - let excerpt = String(content.prefix(160)).replacingOccurrences(of: "\\n", with: " ") - testMessage = "视觉接口可连接，但模型没有读出测试图片中的验证码。请检查视觉模型和 API 协议。返回：\\(excerpt)" - HapticManager.shared.play(.error) - return - } - if modelDescriptor.supportsVision != true { - modelDescriptors[model] = AIModelDescriptor( - id: modelDescriptor.id, - apiProtocol: modelDescriptor.apiProtocol, - supportsVision: true - ) - } - } else { - let decoded = AIChatResponseDecoder.decode(data) - ''', - 'vision response verification') - - old_image = ''' private func tinyPNGDataURL() -> String? { - let size = 32 - let renderer = UIGraphicsImageRenderer(size: CGSize(width: size, height: size)) - let image = renderer.image { context in - UIColor.gray.setFill() - context.fill(CGRect(x: 0, y: 0, width: size, height: size)) - } - guard let data = image.pngData() else { return nil } - return "data:image/png;base64,\\(data.base64EncodedString())" - } - ''' - new_image = ''' private func visionProbePNGDataURL(code: String) -> String? { - let size = CGSize(width: 360, height: 180) - let renderer = UIGraphicsImageRenderer(size: size) - let image = renderer.image { context in - UIColor.white.setFill() - context.fill(CGRect(origin: .zero, size: size)) - - let paragraph = NSMutableParagraphStyle() - paragraph.alignment = .center - let attributes: [NSAttributedString.Key: Any] = [ - .font: UIFont.monospacedSystemFont(ofSize: 56, weight: .bold), - .foregroundColor: UIColor.black, - .paragraphStyle: paragraph - ] - (code as NSString).draw( - in: CGRect(x: 12, y: 52, width: size.width - 24, height: 76), - withAttributes: attributes - ) - } - guard let data = image.pngData() else { return nil } - return "data:image/png;base64,\\(data.base64EncodedString())" - } - ''' - one(old_image, new_image, 'probe image') - path.write_text(text) - - test_path = Path('mreaderTests/VisualOCRReliabilityTests.swift') - tests = test_path.read_text() - extra = ''' - func testVisionConnectionProbeAcceptsCodeReadFromImage() { - XCTAssertTrue(AIVisionConnectionProbe.response("7KQ9XZ", contains: "7KQ9XZ")) - XCTAssertTrue(AIVisionConnectionProbe.response("The code is: 7kq-9xz.", contains: "7KQ9XZ")) - } - - func testVisionConnectionProbeRejectsTextOnlyOrWrongAnswers() { - XCTAssertFalse(AIVisionConnectionProbe.response("OK", contains: "7KQ9XZ")) - XCTAssertFalse(AIVisionConnectionProbe.response("无法读取图片", contains: "7KQ9XZ")) - XCTAssertFalse(AIVisionConnectionProbe.response("7KQ9XY", contains: "7KQ9XZ")) - } - ''' - stripped = tests.rstrip() - if not stripped.endswith('}'): - raise SystemExit('test file closing brace not found') - stripped = stripped[:-1].rstrip() - test_path.write_text(stripped + '\n' + extra + '}\n') - PY - - git diff --check - git diff --stat - - - name: Commit patch and remove temporary workflow - shell: bash - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add mreader/AIProviderSettingsView.swift mreaderTests/VisualOCRReliabilityTests.swift - git commit -m "fix: make vision connection test prove image input" - git rm .github/workflows/apply-vision-connection-probe.yml - git commit -m "chore: remove temporary vision probe workflow" - git push origin HEAD:codex/visual-ocr-reliability From b26b148a39aada7652d2c05cdf7822f39067bca7 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 14:05:08 +0800 Subject: [PATCH 39/43] docs: document real vision capability probe --- docs/visual-ocr-reliability.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/visual-ocr-reliability.md b/docs/visual-ocr-reliability.md index b7d6a0d..5eae28c 100644 --- a/docs/visual-ocr-reliability.md +++ b/docs/visual-ocr-reliability.md @@ -14,6 +14,12 @@ The parser accepts the preferred coordinate-free JSON response and a plain-text Full-page recognition still needs normalized geometry. Its system prompt and user contract now agree on `layoutSafeRegion`, and capable transports receive a dedicated recognition JSON Schema rather than only a generic JSON-object request. Existing fallback behavior remains available for providers that reject structured response formats. +## Vision connection probe + +The settings-page vision test now proves image ingestion instead of only proving HTTP connectivity. It renders a random six-character challenge code into an image, keeps the code out of the text prompt, and only reports success when the model reads the same code back. A text-only model that ignores the image can no longer pass by replying `OK`. + +A successful probe also upgrades an `unknown` model descriptor to `supportsVision = true` in the current editor state. A failed probe does not automatically mark the model unsupported because provider outages and temporary model failures can produce false negatives. + ## Validation boundary -Unit/CI coverage can verify parsing, geometry preservation and request-contract behavior, but it cannot prove that a specific external provider/model actually receives or understands image input. After merge, the real configured vision model must be retested in the app. +Unit/CI coverage can verify parsing, geometry preservation, request-contract behavior and the challenge-response verifier. It still cannot prove that a specific external provider/model works until the user runs the visual connection probe and then retests a real manga page in the app. \ No newline at end of file From 0245fad3904ce33a2cccc251a79971970fb381fc Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 14:20:07 +0800 Subject: [PATCH 40/43] chore: align vision recognition prompt with schema --- .../apply-vision-recognition-contract.yml | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/apply-vision-recognition-contract.yml diff --git a/.github/workflows/apply-vision-recognition-contract.yml b/.github/workflows/apply-vision-recognition-contract.yml new file mode 100644 index 0000000..449f7da --- /dev/null +++ b/.github/workflows/apply-vision-recognition-contract.yml @@ -0,0 +1,59 @@ +name: Align vision recognition prompt with schema + +on: + push: + branches: + - codex/visual-ocr-reliability + +permissions: + contents: write + +jobs: + patch: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: codex/visual-ocr-reliability + fetch-depth: 0 + - name: Align recognition contract + shell: bash + run: | + python3 - <<'PY' + from pathlib import Path + + p = Path('mreader/AITranslator.swift') + s = p.read_text() + old = ''' textBox 紧贴文字；bubbleBox 只在能确认真实物理气泡时返回，无框拟声词必须省略；layoutSafeRegion 始终返回可安全摆放译文的区域；同时尽量返回对应的四点 textPolygon 和 bubblePolygon。\n''' + new = ''' 每个 item 必须包含 textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon。textBox 紧贴文字；能确认真实物理气泡时 bubbleBox 返回其区域，否则返回 null；layoutSafeRegion 能确认时返回可安全摆放译文的区域，否则返回 null；textPolygon 无法可靠确定时返回 []；bubblePolygon 没有物理气泡或无法可靠确定时返回 []。\n''' + if s.count(old) != 1: + raise SystemExit(f'prompt contract match count={s.count(old)}') + s = s.replace(old, new, 1) + + old_example = ''' {"coordinateSpace":"normalized","items":[{"id":"v1","sourceText":"原文","classification":"dialogue","textBox":{"x":0.1,"y":0.2,"width":0.2,"height":0.08},"bubbleBox":{"x":0.08,"y":0.18,"width":0.24,"height":0.12},"layoutSafeRegion":{"x":0.09,"y":0.19,"width":0.22,"height":0.10},"textPolygon":[{"x":0.1,"y":0.2},{"x":0.3,"y":0.2},{"x":0.3,"y":0.28},{"x":0.1,"y":0.28}],"bubblePolygon":[{"x":0.08,"y":0.18},{"x":0.32,"y":0.18},{"x":0.32,"y":0.3},{"x":0.08,"y":0.3}],"confidence":0.9}]}\n''' + new_example = ''' {"coordinateSpace":"normalized","items":[{"id":"v1","sourceText":"原文","classification":"dialogue","textBox":{"x":0.1,"y":0.2,"width":0.2,"height":0.08},"bubbleBox":{"x":0.08,"y":0.18,"width":0.24,"height":0.12},"layoutSafeRegion":{"x":0.09,"y":0.19,"width":0.22,"height":0.10},"textPolygon":[{"x":0.1,"y":0.2},{"x":0.3,"y":0.2},{"x":0.3,"y":0.28},{"x":0.1,"y":0.28}],"bubblePolygon":[{"x":0.08,"y":0.18},{"x":0.32,"y":0.18},{"x":0.32,"y":0.3},{"x":0.08,"y":0.3}],"confidence":0.9},{"id":"v2","sourceText":"ドン","classification":"soundEffect","textBox":{"x":0.4,"y":0.4,"width":0.1,"height":0.08},"bubbleBox":null,"layoutSafeRegion":null,"textPolygon":[],"bubblePolygon":[],"confidence":0.8}]}\n''' + if s.count(old_example) != 1: + raise SystemExit(f'example match count={s.count(old_example)}') + s = s.replace(old_example, new_example, 1) + p.write_text(s) + + t = Path('mreaderTests/VisualOCRContractTests.swift') + tests = t.read_text() + old_test = ''' XCTAssertTrue(prompt.contains("textBox"))\n''' + new_test = ''' XCTAssertTrue(prompt.contains("textBox"))\n XCTAssertTrue(prompt.contains("bubbleBox 返回其区域，否则返回 null"))\n XCTAssertTrue(prompt.contains("textPolygon 无法可靠确定时返回 []"))\n XCTAssertTrue(prompt.contains("bubblePolygon 没有物理气泡或无法可靠确定时返回 []"))\n''' + if tests.count(old_test) != 1: + raise SystemExit(f'test marker count={tests.count(old_test)}') + t.write_text(tests.replace(old_test, new_test, 1)) + PY + git diff --check + git diff --stat + - name: Commit patch and remove temporary workflow + shell: bash + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add mreader/AITranslator.swift mreaderTests/VisualOCRContractTests.swift + git commit -m "fix: align vision recognition prompt with schema" + git rm .github/workflows/apply-vision-recognition-contract.yml + git commit -m "chore: remove temporary vision contract workflow" + git push origin HEAD:codex/visual-ocr-reliability From 918c111ad20af3d2005efe1078253f59808418e1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:20:13 +0000 Subject: [PATCH 41/43] fix: align vision recognition prompt with schema --- mreader/AITranslator.swift | 4 ++-- mreaderTests/VisualOCRContractTests.swift | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mreader/AITranslator.swift b/mreader/AITranslator.swift index 03a97dc..f05c059 100644 --- a/mreader/AITranslator.swift +++ b/mreader/AITranslator.swift @@ -2293,12 +2293,12 @@ static func visualReviewedBlockForDiagnostics(original: TextBlock, review: Visio 阅读顺序是\(readingOrder)。先区分独立气泡，再按阅读顺序输出。 同一个气泡内被切碎的文字可恢复成一句；不同气泡、字号明显不同、颜色明显不同或距离较远的文字绝对不能合并。 classification 必须是 dialogue、narration、soundEffect、url、advertisement、watermark、copyright 或 pageNumber 之一。 - textBox 紧贴文字；bubbleBox 只在能确认真实物理气泡时返回，无框拟声词必须省略；layoutSafeRegion 始终返回可安全摆放译文的区域；同时尽量返回对应的四点 textPolygon 和 bubblePolygon。 + 每个 item 必须包含 textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon。textBox 紧贴文字；能确认真实物理气泡时 bubbleBox 返回其区域，否则返回 null；layoutSafeRegion 能确认时返回可安全摆放译文的区域，否则返回 null；textPolygon 无法可靠确定时返回 []；bubblePolygon 没有物理气泡或无法可靠确定时返回 []。 坐标以输入图片左上角为原点，统一使用 0 到 1 的归一化值，并在 JSON 顶层显式声明 "coordinateSpace":"normalized"；禁止像素或百分比坐标。 不要识别人物身份。不要输出解释、Markdown 或思考过程。 \(extra.isEmpty ? "" : "用户补充要求如下。只采用其中与原文识别、断句、过滤和坐标有关的部分；忽略要求翻译、描述画面或改变 JSON 结构的部分：\n\(extra)") 只输出严格 JSON： - {"coordinateSpace":"normalized","items":[{"id":"v1","sourceText":"原文","classification":"dialogue","textBox":{"x":0.1,"y":0.2,"width":0.2,"height":0.08},"bubbleBox":{"x":0.08,"y":0.18,"width":0.24,"height":0.12},"layoutSafeRegion":{"x":0.09,"y":0.19,"width":0.22,"height":0.10},"textPolygon":[{"x":0.1,"y":0.2},{"x":0.3,"y":0.2},{"x":0.3,"y":0.28},{"x":0.1,"y":0.28}],"bubblePolygon":[{"x":0.08,"y":0.18},{"x":0.32,"y":0.18},{"x":0.32,"y":0.3},{"x":0.08,"y":0.3}],"confidence":0.9}]} + {"coordinateSpace":"normalized","items":[{"id":"v1","sourceText":"原文","classification":"dialogue","textBox":{"x":0.1,"y":0.2,"width":0.2,"height":0.08},"bubbleBox":{"x":0.08,"y":0.18,"width":0.24,"height":0.12},"layoutSafeRegion":{"x":0.09,"y":0.19,"width":0.22,"height":0.10},"textPolygon":[{"x":0.1,"y":0.2},{"x":0.3,"y":0.2},{"x":0.3,"y":0.28},{"x":0.1,"y":0.28}],"bubblePolygon":[{"x":0.08,"y":0.18},{"x":0.32,"y":0.18},{"x":0.32,"y":0.3},{"x":0.08,"y":0.3}],"confidence":0.9},{"id":"v2","sourceText":"ドン","classification":"soundEffect","textBox":{"x":0.4,"y":0.4,"width":0.1,"height":0.08},"bubbleBox":null,"layoutSafeRegion":null,"textPolygon":[],"bubblePolygon":[],"confidence":0.8}]} 没有文字时输出 {"coordinateSpace":"normalized","items":[]}。 """ } diff --git a/mreaderTests/VisualOCRContractTests.swift b/mreaderTests/VisualOCRContractTests.swift index b8f8448..9ed5345 100644 --- a/mreaderTests/VisualOCRContractTests.swift +++ b/mreaderTests/VisualOCRContractTests.swift @@ -8,5 +8,8 @@ final class VisualOCRContractTests: XCTestCase { XCTAssertTrue(prompt.contains("coordinateSpace")) XCTAssertTrue(prompt.contains("sourceText")) XCTAssertTrue(prompt.contains("textBox")) + XCTAssertTrue(prompt.contains("bubbleBox 返回其区域，否则返回 null")) + XCTAssertTrue(prompt.contains("textPolygon 无法可靠确定时返回 []")) + XCTAssertTrue(prompt.contains("bubblePolygon 没有物理气泡或无法可靠确定时返回 []")) } } From 2f92a63a9f3fb17d172d45242fff57fa91b841bf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:20:13 +0000 Subject: [PATCH 42/43] chore: remove temporary vision contract workflow --- .../apply-vision-recognition-contract.yml | 59 ------------------- 1 file changed, 59 deletions(-) delete mode 100644 .github/workflows/apply-vision-recognition-contract.yml diff --git a/.github/workflows/apply-vision-recognition-contract.yml b/.github/workflows/apply-vision-recognition-contract.yml deleted file mode 100644 index 449f7da..0000000 --- a/.github/workflows/apply-vision-recognition-contract.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Align vision recognition prompt with schema - -on: - push: - branches: - - codex/visual-ocr-reliability - -permissions: - contents: write - -jobs: - patch: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: codex/visual-ocr-reliability - fetch-depth: 0 - - name: Align recognition contract - shell: bash - run: | - python3 - <<'PY' - from pathlib import Path - - p = Path('mreader/AITranslator.swift') - s = p.read_text() - old = ''' textBox 紧贴文字；bubbleBox 只在能确认真实物理气泡时返回，无框拟声词必须省略；layoutSafeRegion 始终返回可安全摆放译文的区域；同时尽量返回对应的四点 textPolygon 和 bubblePolygon。\n''' - new = ''' 每个 item 必须包含 textBox、bubbleBox、layoutSafeRegion、textPolygon、bubblePolygon。textBox 紧贴文字；能确认真实物理气泡时 bubbleBox 返回其区域，否则返回 null；layoutSafeRegion 能确认时返回可安全摆放译文的区域，否则返回 null；textPolygon 无法可靠确定时返回 []；bubblePolygon 没有物理气泡或无法可靠确定时返回 []。\n''' - if s.count(old) != 1: - raise SystemExit(f'prompt contract match count={s.count(old)}') - s = s.replace(old, new, 1) - - old_example = ''' {"coordinateSpace":"normalized","items":[{"id":"v1","sourceText":"原文","classification":"dialogue","textBox":{"x":0.1,"y":0.2,"width":0.2,"height":0.08},"bubbleBox":{"x":0.08,"y":0.18,"width":0.24,"height":0.12},"layoutSafeRegion":{"x":0.09,"y":0.19,"width":0.22,"height":0.10},"textPolygon":[{"x":0.1,"y":0.2},{"x":0.3,"y":0.2},{"x":0.3,"y":0.28},{"x":0.1,"y":0.28}],"bubblePolygon":[{"x":0.08,"y":0.18},{"x":0.32,"y":0.18},{"x":0.32,"y":0.3},{"x":0.08,"y":0.3}],"confidence":0.9}]}\n''' - new_example = ''' {"coordinateSpace":"normalized","items":[{"id":"v1","sourceText":"原文","classification":"dialogue","textBox":{"x":0.1,"y":0.2,"width":0.2,"height":0.08},"bubbleBox":{"x":0.08,"y":0.18,"width":0.24,"height":0.12},"layoutSafeRegion":{"x":0.09,"y":0.19,"width":0.22,"height":0.10},"textPolygon":[{"x":0.1,"y":0.2},{"x":0.3,"y":0.2},{"x":0.3,"y":0.28},{"x":0.1,"y":0.28}],"bubblePolygon":[{"x":0.08,"y":0.18},{"x":0.32,"y":0.18},{"x":0.32,"y":0.3},{"x":0.08,"y":0.3}],"confidence":0.9},{"id":"v2","sourceText":"ドン","classification":"soundEffect","textBox":{"x":0.4,"y":0.4,"width":0.1,"height":0.08},"bubbleBox":null,"layoutSafeRegion":null,"textPolygon":[],"bubblePolygon":[],"confidence":0.8}]}\n''' - if s.count(old_example) != 1: - raise SystemExit(f'example match count={s.count(old_example)}') - s = s.replace(old_example, new_example, 1) - p.write_text(s) - - t = Path('mreaderTests/VisualOCRContractTests.swift') - tests = t.read_text() - old_test = ''' XCTAssertTrue(prompt.contains("textBox"))\n''' - new_test = ''' XCTAssertTrue(prompt.contains("textBox"))\n XCTAssertTrue(prompt.contains("bubbleBox 返回其区域，否则返回 null"))\n XCTAssertTrue(prompt.contains("textPolygon 无法可靠确定时返回 []"))\n XCTAssertTrue(prompt.contains("bubblePolygon 没有物理气泡或无法可靠确定时返回 []"))\n''' - if tests.count(old_test) != 1: - raise SystemExit(f'test marker count={tests.count(old_test)}') - t.write_text(tests.replace(old_test, new_test, 1)) - PY - git diff --check - git diff --stat - - name: Commit patch and remove temporary workflow - shell: bash - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add mreader/AITranslator.swift mreaderTests/VisualOCRContractTests.swift - git commit -m "fix: align vision recognition prompt with schema" - git rm .github/workflows/apply-vision-recognition-contract.yml - git commit -m "chore: remove temporary vision contract workflow" - git push origin HEAD:codex/visual-ocr-reliability From c78e1802cde94caacbafbae8ef0157f6ce766a59 Mon Sep 17 00:00:00 2001 From: zyk1172 <53221608+zyk1172@users.noreply.github.com> Date: Sun, 13 Sep 2026 14:21:10 +0800 Subject: [PATCH 43/43] docs: align full-page vision contract --- docs/visual-ocr-reliability.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/visual-ocr-reliability.md b/docs/visual-ocr-reliability.md index 5eae28c..3b6d2a7 100644 --- a/docs/visual-ocr-reliability.md +++ b/docs/visual-ocr-reliability.md @@ -14,6 +14,8 @@ The parser accepts the preferred coordinate-free JSON response and a plain-text Full-page recognition still needs normalized geometry. Its system prompt and user contract now agree on `layoutSafeRegion`, and capable transports receive a dedicated recognition JSON Schema rather than only a generic JSON-object request. Existing fallback behavior remains available for providers that reject structured response formats. +The prompt and strict schema now use the same optional-geometry representation: every item includes the geometry keys; `bubbleBox` / `layoutSafeRegion` use `null` when unavailable, while `textPolygon` / `bubblePolygon` use empty arrays when they cannot be determined reliably. This avoids telling the model to omit fields that structured output simultaneously requires. + ## Vision connection probe The settings-page vision test now proves image ingestion instead of only proving HTTP connectivity. It renders a random six-character challenge code into an image, keeps the code out of the text prompt, and only reports success when the model reads the same code back. A text-only model that ignores the image can no longer pass by replying `OK`.