From fcd5e8bb1f8aa3c4b74af3bc0d5ebb9b72431336 Mon Sep 17 00:00:00 2001 From: MengzhangLI Date: Fri, 16 Oct 2026 18:43:25 +0800 Subject: [PATCH 01/11] Add Rokid iPhone bridge with local MiniCPM and unsigned IPA build --- .github/workflows/ios-ipa.yml | 74 +++ .gitignore | 4 + ios/Sources/OpenGlassModelSession.swift | 175 ++++++ ios/Sources/OpenGlassSpeechSession.swift | 177 ++++++ ios/Sources/OpenGlassViewController.swift | 695 ++++++++++++++++++++++ ios/Sources/RokidIngressServer.swift | 254 ++++++++ ios/Sources/RokidWireProtocol.swift | 156 +++++ ios/scripts/build_ipa.sh | 100 ++++ ios/scripts/prepare_upstream.py | 224 +++++++ ios/tests/main.swift | 85 +++ ios/tests/run_protocol_tests.sh | 7 + ios/tests/test_prepare_upstream.py | 148 +++++ ios/upstream-lock.json | 19 + ios/windows/.gitignore | 2 + ios/windows/common.ps1 | 67 +++ ios/windows/configure-rokid.ps1 | 31 + ios/windows/diagnose.ps1 | 62 ++ ios/windows/download-artifact.ps1 | 38 ++ ios/windows/requirements.txt | 2 + ios/windows/start-usb-relay.ps1 | 23 + ios/windows/test_usb_relay.py | 134 +++++ ios/windows/usb_relay.py | 175 ++++++ 22 files changed, 2652 insertions(+) create mode 100644 .github/workflows/ios-ipa.yml create mode 100644 ios/Sources/OpenGlassModelSession.swift create mode 100644 ios/Sources/OpenGlassSpeechSession.swift create mode 100644 ios/Sources/OpenGlassViewController.swift create mode 100644 ios/Sources/RokidIngressServer.swift create mode 100644 ios/Sources/RokidWireProtocol.swift create mode 100644 ios/scripts/build_ipa.sh create mode 100644 ios/scripts/prepare_upstream.py create mode 100644 ios/tests/main.swift create mode 100644 ios/tests/run_protocol_tests.sh create mode 100644 ios/tests/test_prepare_upstream.py create mode 100644 ios/upstream-lock.json create mode 100644 ios/windows/.gitignore create mode 100644 ios/windows/common.ps1 create mode 100644 ios/windows/configure-rokid.ps1 create mode 100644 ios/windows/diagnose.ps1 create mode 100644 ios/windows/download-artifact.ps1 create mode 100644 ios/windows/requirements.txt create mode 100644 ios/windows/start-usb-relay.ps1 create mode 100644 ios/windows/test_usb_relay.py create mode 100644 ios/windows/usb_relay.py diff --git a/.github/workflows/ios-ipa.yml b/.github/workflows/ios-ipa.yml new file mode 100644 index 0000000..aa7c87e --- /dev/null +++ b/.github/workflows/ios-ipa.yml @@ -0,0 +1,74 @@ +name: Build OpenGlass iOS IPA + +on: + workflow_dispatch: + push: + branches: [codex/rokid-iphone-minicpm] + paths: ['ios/**', '.github/workflows/ios-ipa.yml'] + pull_request: + paths: ['ios/**', '.github/workflows/ios-ipa.yml'] + +permissions: + contents: read + +concurrency: + group: ios-ipa-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # Xcode 26.1.1 is listed on macos-15 and is the upstream-verified version: + # https://github.com/actions/runner-images/blob/main/images/macos/macos-15-Readme.md + runs-on: macos-15 + timeout-minutes: 90 + env: + DEVELOPER_DIR: /Applications/Xcode_26.1.1.app/Contents/Developer + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: Check build tools + run: | + xcodebuild -version + cmake --version + python3 --version + - name: Test project preparation + run: python3 -m unittest discover -s ios/tests -p 'test_prepare*.py' -v + - name: Test Rokid wire protocol + run: bash ios/tests/run_protocol_tests.sh + - name: Test Windows USB relay without devices + run: python3 -m unittest discover -s ios/windows -p 'test_usb_relay.py' -v + - name: Prepare pinned upstream and OpenGlass sources + run: | + python3 ios/scripts/prepare_upstream.py + plutil -lint tmp/MiniCPM-V-Apps/MiniCPM-V-demo.xcodeproj/project.pbxproj + plutil -lint tmp/MiniCPM-V-Apps/MiniCPM-V-demo/Info.plist + plutil -lint tmp/MiniCPM-V-Apps/MiniCPM-V-demo/MiniCPM-V-demo.entitlements + - name: Cache pinned inference framework and Swift packages + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: | + tmp/MiniCPM-V-Apps/MiniCPM-V-demo/thirdparty/llama.xcframework + tmp/MiniCPM-V-Apps/MiniCPM-V-demo/thirdparty/.openglass-framework-key + tmp/ios-build/SourcePackages + key: ios-device-${{ runner.os }}-${{ runner.arch }}-xcode26.1.1-${{ hashFiles('ios/upstream-lock.json', 'ios/scripts/build_ipa.sh') }} + - name: Build unsigned device IPA + run: bash ios/scripts/build_ipa.sh + - name: Upload IPA for Windows signing + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: OpenGlass-iOS-unsigned-${{ github.run_number }} + path: | + artifacts/OpenGlass-1.0.ipa + artifacts/build-info.json + if-no-files-found: error + retention-days: 14 + compression-level: 0 + - name: Upload build logs on failure + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: OpenGlass-iOS-build-logs-${{ github.run_number }} + path: tmp/ios-build/*.log + if-no-files-found: ignore + retention-days: 7 diff --git a/.gitignore b/.gitignore index 593a3ff..498e69b 100644 --- a/.gitignore +++ b/.gitignore @@ -93,3 +93,7 @@ live_sessions/ *.keystore *.jks *.apk +*.ipa +*.xcframework/ +*.dSYM/ +artifacts/ diff --git a/ios/Sources/OpenGlassModelSession.swift b/ios/Sources/OpenGlassModelSession.swift new file mode 100644 index 0000000..f822d2b --- /dev/null +++ b/ios/Sources/OpenGlassModelSession.swift @@ -0,0 +1,175 @@ +import Foundation +import Combine + +/// Owns exactly one native model context. All entry points run on the main actor; +/// a busy session rejects new work instead of queueing frames or questions. +@MainActor +final class OpenGlassModelSession { + enum Phase: Equatable { + case unloaded, loading, ready, preparing, generating, stopping, failed + } + + enum SessionError: LocalizedError { + case busy, missingFiles, cancelled, restartRequired, invalidFile(String) + var errorDescription: String? { + switch self { + case .busy: return "上一轮尚未结束，请稍候。" + case .missingFiles: return "请先在“模型下载”中下载 MiniCPM-V 4.6 的 LLM 和 VPM 文件。" + case .cancelled: return "已停止本轮。" + case .restartRequired: return "底层推理超时，已锁定模型以保护内存。请关闭并重新打开 App。" + case .invalidFile(let name): return "模型文件无效：\(name)。请在模型下载页重新下载。" + } + } + } + + var onChange: ((Phase, String) -> Void)? + var onOutput: ((String) -> Void)? + private(set) var phase: Phase = .unloaded + private(set) var isBusy = false + private(set) var isLoaded = false + private(set) var requiresAppRestart = false + private var wrapper: MTMDWrapper? + private var tokenSubscription: AnyCancellable? + private var discardResult = false + private var output = "" + + static var modelFiles: [String] { + [MiniCPMModelConst.modelv46_FileName, MiniCPMModelConst.mmprojv46_FileName] + } + + func load() async throws { + guard !requiresAppRestart else { throw SessionError.restartRequired } + guard !isBusy else { throw SessionError.busy } + if isLoaded { return } + isBusy = true + discardResult = false + transition(.loading, "正在加载 MiniCPM-V 4.6…") + defer { isBusy = false } + do { + let documents = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0] + let files = Self.modelFiles.map { documents.appendingPathComponent($0) } + for file in files { + guard FileManager.default.fileExists(atPath: file.path) else { throw SessionError.missingFiles } + // Check only the GGUF header, never hash multi-GB files on the UI thread. + let handle = try FileHandle(forReadingFrom: file) + let header = try handle.read(upToCount: 4) + try handle.close() + guard header == Data("GGUF".utf8) else { throw SessionError.invalidFile(file.lastPathComponent) } + } + let engine = MTMDWrapper() + wrapper = engine + engine.setEnableThinking(false) + let params = MTMDParams(modelPath: files[0].path, mmprojPath: files[1].path, + nPredict: 160, nCtx: 4096, nThreads: 4, + temperature: 0.2, useGPU: true, mmprojUseGPU: true, + warmup: false, nUbatch: 512, imageMaxSliceNums: 1) + try await engine.initialize(with: params) + engine.setModelVersion(46) + engine.decodeTokenLimit = 160 + tokenSubscription = engine.$currentToken.dropFirst().sink { [weak self] token in + guard let self, self.phase == .generating || self.phase == .stopping else { return } + self.output += token.content + if !self.discardResult { self.onOutput?(self.output) } + } + isLoaded = true + transition(.ready, discardResult ? "模型已加载；接收已停止。" : "MiniCPM-V 4.6 已就绪") + } catch { + // initialize has returned; no native worker is left running on this path. + if let engine = wrapper { await engine.reset() } + wrapper = nil + isLoaded = false + transition(.failed, error.localizedDescription) + throw error + } + } + + func ask(question: String, imageURL: URL?, answerLanguage: String = "zh-CN") async throws -> String { + guard !requiresAppRestart else { throw SessionError.restartRequired } + guard !isBusy else { throw SessionError.busy } + guard isLoaded, let engine = wrapper else { throw SessionError.missingFiles } + isBusy = true + discardResult = false + output = "" + engine.decodeTokenLimit = 160 + transition(.preparing, imageURL == nil ? "正在处理问题…" : "正在理解最新画面…") + defer { isBusy = false } + do { + // Previous turn has fully completed before clearing KV. Never call + // upstream stopGeneration(), which publishes completion too early. + engine.clearKVCacheForNewTurn() + if let imageURL { + try await engine.addImageInBackground(imageURL.path, timeoutSeconds: 180) + } + try checkStopped() + let instruction = answerLanguage.hasPrefix("en") + ? "Answer briefly in English, suitable for reading aloud, in at most 80 words. Describe only what you can confirm.\n" + : "请用简洁中文回答，适合语音朗读，不超过100字。只描述你确实能确认的内容。\n" + let prompt = instruction + question + try await engine.addTextInBackground(prompt) + try checkStopped() + transition(.generating, "正在生成回答…") + try await engine.startGeneration() + // startGeneration launches an internal task, so await its terminal + // state before letting another request touch the context. + while engine.generationState == .generating { + try? await Task.sleep(nanoseconds: 30_000_000) + } + if case .failed(let error) = engine.generationState { throw error } + try checkStopped() + let answer = output.trimmingCharacters(in: .whitespacesAndNewlines) + transition(.ready, "回答完成") + return answer + } catch { + if let mtmdError = error as? MTMDError, case .timeout = mtmdError { + // The upstream watchdog can return while C++ still uses ctx. + // Retain the wrapper, never reset/free or start a second call. + requiresAppRestart = true + transition(.failed, SessionError.restartRequired.localizedDescription) + throw SessionError.restartRequired + } + if let sessionError = error as? SessionError, case .cancelled = sessionError { + transition(.ready, "本轮已结束，可重新启动。") + } else { + // The native operation has returned. Reinitializing next time + // is safer than reusing a context after a prefill error. + await engine.reset() + tokenSubscription = nil + wrapper = nil + isLoaded = false + transition(.failed, error.localizedDescription) + } + throw error + } + } + + func requestStop() { + discardResult = true + guard isBusy else { return } + if phase == .generating { + // A Swift-only flag read after mb_mtmd_loop returns. The next token + // finishes normally, including native perf bookkeeping. + wrapper?.decodeTokenLimit = 1 + } + transition(.stopping, "正在结束本轮推理…") + } + + /// Used before entering the download UI, which can replace model files. + func unload() async throws { + guard !requiresAppRestart else { throw SessionError.restartRequired } + guard !isBusy else { throw SessionError.busy } + if let engine = wrapper { await engine.reset() } + wrapper = nil + tokenSubscription = nil + isLoaded = false + transition(.unloaded, "模型未加载") + } + + private func checkStopped() throws { + if discardResult { throw SessionError.cancelled } + } + + private func transition(_ phase: Phase, _ message: String) { + self.phase = phase + onChange?(phase, message) + } +} diff --git a/ios/Sources/OpenGlassSpeechSession.swift b/ios/Sources/OpenGlassSpeechSession.swift new file mode 100644 index 0000000..271aeaa --- /dev/null +++ b/ios/Sources/OpenGlassSpeechSession.swift @@ -0,0 +1,177 @@ +import Foundation +import Speech +import AVFoundation + +/// Recognizes only incoming Rokid PCM16LE, 16 kHz, mono. It never opens the +/// iPhone microphone and never falls back to Apple's cloud recognition. +@MainActor +final class OpenGlassSpeechSession { + var onPartial: ((String) -> Void)? + var onFinal: ((String) -> Void)? + var onError: ((String) -> Void)? + var onStatus: ((String) -> Void)? + private(set) var isListening = false + var isActive: Bool { request != nil } + var isFinishing: Bool { ending } + private var recognizer: SFSpeechRecognizer? + private var request: SFSpeechAudioBufferRecognitionRequest? + private var task: SFSpeechRecognitionTask? + private var timer: Timer? + private var finishTimer: Timer? + private var sessionID = UUID() + private var latestText = "" + private var startedAt = Date() + private var lastVoiceAt: Date? + private var lastPacketAt = Date() + private var pendingByte: UInt8? + private var ending = false + private let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, + sampleRate: 16_000, channels: 1, interleaved: false)! + + static func authorize() async -> Bool { + if SFSpeechRecognizer.authorizationStatus() == .authorized { return true } + return await withCheckedContinuation { continuation in + SFSpeechRecognizer.requestAuthorization { status in + continuation.resume(returning: status == .authorized) + } + } + } + + func start(localeIdentifier: String) throws { + stop() + guard SFSpeechRecognizer.authorizationStatus() == .authorized else { + throw SpeechError.message("未获语音识别权限。可在系统设置中允许，或关闭语音、使用文字提问。") + } + guard let recognizer = SFSpeechRecognizer(locale: Locale(identifier: localeIdentifier)), + recognizer.supportsOnDeviceRecognition else { + throw SpeechError.message("此语言的离线语音识别不可用。请在 iPhone 下载对应语言资源后重试，或使用文字提问；不会转到云端。") + } + guard recognizer.isAvailable else { + throw SpeechError.message("系统离线语音识别暂不可用，请稍后重新启动，或使用文字提问。") + } + let request = SFSpeechAudioBufferRecognitionRequest() + request.requiresOnDeviceRecognition = true + request.shouldReportPartialResults = true + request.taskHint = .dictation + if #available(iOS 16.0, *) { request.addsPunctuation = true } + self.recognizer = recognizer + self.request = request + latestText = "" + ending = false + startedAt = Date() + lastPacketAt = Date() + lastVoiceAt = nil + isListening = true + let id = sessionID + task = recognizer.recognitionTask(with: request) { [weak self] result, error in + Task { @MainActor in + guard let self, self.sessionID == id else { return } + if let result { + self.latestText = result.bestTranscription.formattedString + self.onPartial?(self.latestText) + if result.isFinal { self.complete(); return } + } + if let error { + if self.ending && !self.latestText.isEmpty { + self.complete() + } else { + self.stop() + self.onError?("离线语音识别中断：\(error.localizedDescription)。可重新启动或使用文字提问。") + } + } + } + } + timer = Timer.scheduledTimer(withTimeInterval: 0.2, repeats: true) { [weak self] _ in + Task { @MainActor in self?.tick() } + } + onStatus?("正在听 Rokid 麦克风（离线识别）") + } + + func appendPCM16LE(_ data: Data) { + guard isListening, !ending, let request, !data.isEmpty else { return } + // Preserve a split sample across websocket messages. At most one byte + // is retained; audio received during inference/TTS is simply dropped. + var bytes = [UInt8]() + bytes.reserveCapacity(data.count + 1) + if let pendingByte { bytes.append(pendingByte) } + bytes.append(contentsOf: data) + pendingByte = bytes.count.isMultiple(of: 2) ? nil : bytes.removeLast() + let frames = bytes.count / 2 + guard frames > 0, + let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frames)), + let samples = buffer.floatChannelData?[0] else { return } + buffer.frameLength = AVAudioFrameCount(frames) + var squareSum: Float = 0 + for index in 0.. 0.012 { lastVoiceAt = now } + request.append(buffer) + } + + /// Stop accepting audio and give the recognizer time to finalize its text. + func finishUtterance() { + guard isListening, !ending else { return } + ending = true + isListening = false + timer?.invalidate() + timer = nil + request?.endAudio() + onStatus?("正在结束语句…") + let id = sessionID + finishTimer = Timer.scheduledTimer(withTimeInterval: 2.0, repeats: false) { [weak self] _ in + Task { @MainActor in + guard let self, self.sessionID == id else { return } + self.complete() + } + } + } + + func stop() { + sessionID = UUID() + isListening = false + ending = false + timer?.invalidate() + timer = nil + finishTimer?.invalidate() + finishTimer = nil + request?.endAudio() + task?.cancel() + task = nil + request = nil + recognizer = nil + pendingByte = nil + } + + private func tick() { + guard isListening else { return } + let now = Date() + if let lastVoiceAt, now.timeIntervalSince(lastVoiceAt) >= 1.3 { + finishUtterance() + } else if now.timeIntervalSince(startedAt) >= 25 { + finishUtterance() + } else if now.timeIntervalSince(lastPacketAt) >= 5 { + stop() + onError?("Rokid 音频超过5秒未到达。确认连接后重新启动语音。") + } + } + + private func complete() { + let text = latestText.trimmingCharacters(in: .whitespacesAndNewlines) + stop() + onFinal?(text) + } + + enum SpeechError: LocalizedError { + case message(String) + var errorDescription: String? { + if case .message(let message) = self { return message } + return nil + } + } +} diff --git a/ios/Sources/OpenGlassViewController.swift b/ios/Sources/OpenGlassViewController.swift new file mode 100644 index 0000000..a49a07a --- /dev/null +++ b/ios/Sources/OpenGlassViewController.swift @@ -0,0 +1,695 @@ +import UIKit +import AVFoundation +import Darwin + +/// Foreground Rokid receiver with one-image, one-question local inference. +@MainActor +final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSpeechSynthesizerDelegate { + private let server = RokidIngressServer() + private let model = OpenGlassModelSession() + private let speech = OpenGlassSpeechSession() + private let synthesizer = AVSpeechSynthesizer() + private var activeUtterance: AVSpeechUtterance? + private var isRunning = false + private var isStarting = false + private var isAnswering = false + private var audioConnected = false + private var speechPaused = false + private var suppressAnswerSpeech = false + private var runID = UUID() + private var latestRawImage: Data? + private var latestImage: UIImage? + private var latestImageAt: Date? + private var frameTimer: Timer? + private var backgroundTask: UIBackgroundTaskIdentifier = .invalid + private let maximumFrameAge: TimeInterval = 8 + + private let statusLabel = UILabel() + private let modelLabel = UILabel() + private let connectionLabel = UILabel() + private let frameLabel = UILabel() + private let routeLabel = UILabel() + private let preview = UIImageView() + private let addressView = UITextView() + private let transcriptLabel = UILabel() + private let answerView = UITextView() + private let questionField = UITextField() + private let serviceButton = UIButton(type: .system) + private let askButton = UIButton(type: .system) + private let listenButton = UIButton(type: .system) + private let stopTurnButton = UIButton(type: .system) + private let voiceSwitch = UISwitch() + private let speakSwitch = UISwitch() + private let rotationControl = UISegmentedControl(items: ["0°", "90°", "180°", "270°"]) + private let languageControl = UISegmentedControl(items: ["中文", "English"]) + private var downloadItem: UIBarButtonItem! + + override func viewDidLoad() { + super.viewDidLoad() + title = "OpenGlass · iPhone" + view.backgroundColor = .systemBackground + buildUI() + connectCallbacks() + synthesizer.delegate = self + NotificationCenter.default.addObserver(self, selector: #selector(enteredBackground), + name: UIApplication.didEnterBackgroundNotification, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(enteredForeground), + name: UIApplication.willEnterForegroundNotification, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(audioInterrupted), + name: AVAudioSession.interruptionNotification, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(updateAudioRoute), + name: AVAudioSession.routeChangeNotification, object: nil) + statusLabel.text = "准备就绪。保持 App 在前台，点击启动。" + modelLabel.text = "固定模型：MiniCPM-V 4.6 · 本机推理" + connectionLabel.text = "Rokid 尚未连接" + answerView.text = "回答会显示在这里。" + frameTimer = Timer.scheduledTimer(withTimeInterval: 1, repeats: true) { [weak self] _ in + Task { @MainActor in self?.refreshFrameAge() } + } + refreshAddresses() + refreshFrameAge() + updateAudioRoute() + updateControls() + } + + override func viewDidAppear(_ animated: Bool) { + super.viewDidAppear(animated) + refreshAddresses() + UIApplication.shared.isIdleTimerDisabled = isRunning + } + + deinit { + frameTimer?.invalidate() + NotificationCenter.default.removeObserver(self) + } + + private func buildUI() { + downloadItem = UIBarButtonItem(title: "模型下载", style: .plain, target: self, action: #selector(showDownloads)) + navigationItem.rightBarButtonItem = downloadItem + let scroll = UIScrollView() + scroll.keyboardDismissMode = .interactive + scroll.translatesAutoresizingMaskIntoConstraints = false + view.addSubview(scroll) + let stack = UIStackView() + stack.axis = .vertical + stack.spacing = 12 + stack.translatesAutoresizingMaskIntoConstraints = false + scroll.addSubview(stack) + NSLayoutConstraint.activate([ + scroll.leadingAnchor.constraint(equalTo: view.safeAreaLayoutGuide.leadingAnchor), + scroll.trailingAnchor.constraint(equalTo: view.safeAreaLayoutGuide.trailingAnchor), + scroll.topAnchor.constraint(equalTo: view.safeAreaLayoutGuide.topAnchor), + scroll.bottomAnchor.constraint(equalTo: view.keyboardLayoutGuide.topAnchor), + stack.leadingAnchor.constraint(equalTo: scroll.contentLayoutGuide.leadingAnchor, constant: 18), + stack.trailingAnchor.constraint(equalTo: scroll.contentLayoutGuide.trailingAnchor, constant: -18), + stack.topAnchor.constraint(equalTo: scroll.contentLayoutGuide.topAnchor, constant: 16), + stack.bottomAnchor.constraint(equalTo: scroll.contentLayoutGuide.bottomAnchor, constant: -24), + stack.widthAnchor.constraint(equalTo: scroll.frameLayoutGuide.widthAnchor, constant: -36) + ]) + for label in [statusLabel, modelLabel, connectionLabel, frameLabel, routeLabel, transcriptLabel] { + label.numberOfLines = 0 + label.font = .preferredFont(forTextStyle: .subheadline) + label.adjustsFontForContentSizeCategory = true + } + statusLabel.font = .preferredFont(forTextStyle: .headline) + modelLabel.textColor = .secondaryLabel + frameLabel.textColor = .secondaryLabel + routeLabel.textColor = .secondaryLabel + stack.addArrangedSubview(statusLabel) + stack.addArrangedSubview(modelLabel) + configureButton(serviceButton, title: "启动接收", action: #selector(toggleService), filled: true) + stack.addArrangedSubview(serviceButton) + addressView.isEditable = false + addressView.isScrollEnabled = false + addressView.font = .monospacedSystemFont(ofSize: 13, weight: .regular) + addressView.backgroundColor = .secondarySystemBackground + addressView.layer.cornerRadius = 8 + addressView.accessibilityLabel = "iPhone 接收地址" + stack.addArrangedSubview(addressView) + let addressRow = UIStackView() + addressRow.axis = .horizontal + addressRow.spacing = 12 + let copy = UIButton(type: .system) + configureButton(copy, title: "复制地址", action: #selector(copyAddress)) + let refresh = UIButton(type: .system) + configureButton(refresh, title: "刷新地址", action: #selector(refreshAddresses)) + addressRow.addArrangedSubview(copy) + addressRow.addArrangedSubview(refresh) + stack.addArrangedSubview(addressRow) + stack.addArrangedSubview(connectionLabel) + preview.contentMode = .scaleAspectFit + preview.backgroundColor = .secondarySystemBackground + preview.layer.cornerRadius = 12 + preview.clipsToBounds = true + preview.accessibilityLabel = "最新 Rokid 画面" + preview.heightAnchor.constraint(equalToConstant: 220).isActive = true + stack.addArrangedSubview(preview) + stack.addArrangedSubview(frameLabel) + let savedRotation = UserDefaults.standard.object(forKey: "OpenGlass.imageRotation") as? Int ?? 270 + rotationControl.selectedSegmentIndex = [0, 90, 180, 270].firstIndex(of: savedRotation) ?? 3 + rotationControl.addTarget(self, action: #selector(rotationChanged), for: .valueChanged) + rotationControl.accessibilityLabel = "画面顺时针旋转" + stack.addArrangedSubview(row("画面旋转", rotationControl)) + voiceSwitch.isOn = UserDefaults.standard.object(forKey: "OpenGlass.voiceEnabled") as? Bool ?? true + voiceSwitch.addTarget(self, action: #selector(voiceChanged), for: .valueChanged) + stack.addArrangedSubview(row("Rokid 离线语音提问", voiceSwitch)) + languageControl.selectedSegmentIndex = UserDefaults.standard.integer(forKey: "OpenGlass.speechLanguage") + if !(0...1).contains(languageControl.selectedSegmentIndex) { languageControl.selectedSegmentIndex = 0 } + languageControl.addTarget(self, action: #selector(languageChanged), for: .valueChanged) + stack.addArrangedSubview(row("识别 / 播报语言", languageControl)) + speakSwitch.isOn = UserDefaults.standard.object(forKey: "OpenGlass.speakAnswers") as? Bool ?? true + speakSwitch.addTarget(self, action: #selector(speakChanged), for: .valueChanged) + stack.addArrangedSubview(row("朗读回答", speakSwitch)) + stack.addArrangedSubview(routeLabel) + configureButton(listenButton, title: "开始语音", action: #selector(toggleListening)) + stack.addArrangedSubview(listenButton) + transcriptLabel.text = "识别文字会显示在这里。停顿约1.3秒，或点“结束语句”提问。" + stack.addArrangedSubview(transcriptLabel) + questionField.borderStyle = .roundedRect + questionField.placeholder = "输入问题，例如：我面前有什么？" + questionField.returnKeyType = .send + questionField.delegate = self + questionField.accessibilityLabel = "文字问题" + questionField.heightAnchor.constraint(greaterThanOrEqualToConstant: 44).isActive = true + stack.addArrangedSubview(questionField) + configureButton(askButton, title: "用最新画面提问", action: #selector(askManually), filled: true) + configureButton(stopTurnButton, title: "停止本轮 / 暂停语音", action: #selector(stopCurrentTurn)) + stack.addArrangedSubview(askButton) + stack.addArrangedSubview(stopTurnButton) + answerView.isEditable = false + answerView.isScrollEnabled = false + answerView.font = .preferredFont(forTextStyle: .body) + answerView.backgroundColor = .secondarySystemBackground + answerView.layer.cornerRadius = 8 + answerView.heightAnchor.constraint(greaterThanOrEqualToConstant: 130).isActive = true + answerView.accessibilityLabel = "模型回答" + stack.addArrangedSubview(answerView) + let note = UILabel() + note.numberOfLines = 0 + note.font = .preferredFont(forTextStyle: .footnote) + note.textColor = .secondaryLabel + note.text = "Rokid 与 iPhone 需在同一可互通网络。首次使用请先下载模型。此 App 的模型文件与原 TestFlight App 分开保存。进入后台会停止接收和语音，回到前台后请重新启动。" + stack.addArrangedSubview(note) + } + + private func row(_ title: String, _ control: UIView) -> UIStackView { + let label = UILabel() + label.text = title + label.font = .preferredFont(forTextStyle: .subheadline) + label.numberOfLines = 0 + let stack = UIStackView(arrangedSubviews: [label, control]) + stack.axis = .horizontal + stack.spacing = 12 + stack.alignment = .center + control.setContentHuggingPriority(.required, for: .horizontal) + return stack + } + + private func configureButton(_ button: UIButton, title: String, action: Selector, filled: Bool = false) { + var configuration = filled ? UIButton.Configuration.filled() : UIButton.Configuration.tinted() + configuration.title = title + configuration.cornerStyle = .medium + button.configuration = configuration + button.heightAnchor.constraint(greaterThanOrEqualToConstant: 44).isActive = true + button.addTarget(self, action: action, for: .touchUpInside) + } + + private func connectCallbacks() { + server.onImage = { [weak self] data in + guard let self, self.isRunning, let decoded = UIImage(data: data) else { return } + self.latestRawImage = data + self.latestImage = self.normalizedImage(decoded) + self.latestImageAt = Date() + self.preview.image = self.latestImage + self.refreshFrameAge() + } + server.onAudio = { [weak self] data in + guard let self, self.isRunning, !self.isAnswering, self.activeUtterance == nil, + !self.speechPaused else { return } + self.speech.appendPCM16LE(data) + } + server.onStatus = { [weak self] status in self?.connectionLabel.text = status } + server.onAudioConnectionChanged = { [weak self] connected in + guard let self else { return } + self.audioConnected = connected + if !connected { + self.speech.stop() + if self.isRunning { self.statusLabel.text = "Rokid 音频已断开，等待重新连接。" } + } else { + self.startListeningIfPossible() + } + self.updateControls() + } + model.onChange = { [weak self] _, message in + self?.modelLabel.text = message + self?.updateControls() + } + model.onOutput = { [weak self] output in self?.answerView.text = output } + speech.onPartial = { [weak self] text in self?.transcriptLabel.text = text } + speech.onStatus = { [weak self] message in + self?.statusLabel.text = message + self?.updateControls() + } + speech.onError = { [weak self] message in + guard let self else { return } + self.speechPaused = true + self.statusLabel.text = message + self.updateControls() + } + speech.onFinal = { [weak self] text in + guard let self, self.isRunning, !self.speechPaused else { return } + if text.isEmpty { + self.statusLabel.text = "未识别到问题，继续等待语音。" + self.resumeListeningAfterEchoDelay() + } else { + self.questionField.text = text + self.submit(text) + } + self.updateControls() + } + } + + @objc private func toggleService() { + if isRunning || isStarting { stopService(message: "接收已停止，点击启动可恢复。"); return } + guard !model.isBusy, !model.requiresAppRestart else { return } + isStarting = true + isRunning = true + speechPaused = false + runID = UUID() + let id = runID + latestRawImage = nil + latestImage = nil + latestImageAt = nil + preview.image = nil + do { + try server.start(port: 18080) + } catch { + isRunning = false + isStarting = false + statusLabel.text = "无法启动接收：\(error.localizedDescription)" + updateControls() + return + } + UIApplication.shared.isIdleTimerDisabled = true + statusLabel.text = "接收已启动，正在准备模型…" + refreshAddresses() + refreshFrameAge() + updateControls() + Task { + do { + try await model.load() + guard isRunning, runID == id else { updateControls(); return } + isStarting = false + statusLabel.text = "等待 Rokid 画面和音频；也可以文字提问。" + if voiceSwitch.isOn { + let authorized = await OpenGlassSpeechSession.authorize() + guard isRunning, runID == id else { updateControls(); return } + if !authorized { + speechPaused = true + statusLabel.text = "未获语音识别权限。可用文字提问，或在系统设置中允许。" + } + } + startListeningIfPossible() + } catch { + if runID == id { stopService(message: error.localizedDescription) } + } + isStarting = false + updateControls() + } + } + + private func stopService(message: String) { + isRunning = false + isStarting = false + speechPaused = true + suppressAnswerSpeech = true + runID = UUID() + speech.stop() + stopSpeaking() + server.stop() + audioConnected = false + model.requestStop() + UIApplication.shared.isIdleTimerDisabled = false + statusLabel.text = model.isBusy ? "已停止接收和语音；正在结束本轮推理…" : message + updateControls() + } + + @objc private func stopCurrentTurn() { + speechPaused = true + suppressAnswerSpeech = true + speech.stop() + stopSpeaking() + model.requestStop() + statusLabel.text = model.isBusy ? "正在结束本轮推理…" : "语音已暂停，点击“开始语音”可恢复。" + updateControls() + } + + @objc private func askManually() { + questionField.resignFirstResponder() + submit(questionField.text ?? "") + } + + func textFieldShouldReturn(_ textField: UITextField) -> Bool { + askManually() + return true + } + + private func submit(_ text: String) { + let question = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !question.isEmpty else { statusLabel.text = "请先输入或说出问题。"; return } + guard isRunning, !isStarting, model.isLoaded, !model.isBusy, !isAnswering else { + statusLabel.text = "请先启动接收，并等待当前模型任务结束。" + return + } + guard question.count <= 500 else { statusLabel.text = "问题请控制在500字以内。"; return } + guard let image = latestImage, let receivedAt = latestImageAt, + Date().timeIntervalSince(receivedAt) <= maximumFrameAge else { + speech.stop() + speechPaused = true + statusLabel.text = "没有8秒内的新画面。请恢复 Rokid 图像连接后，再次提问。" + updateControls() + return + } + speech.stop() + stopSpeaking() + isAnswering = true + suppressAnswerSpeech = false + answerView.text = "" + statusLabel.text = "已暂停收音，正在本机理解画面…" + let id = runID + updateControls() + Task { + var imageURL: URL? + do { + let folder = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0] + .appendingPathComponent("OpenGlassFrames", isDirectory: true) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + let file = folder.appendingPathComponent(UUID().uuidString + ".jpg") + guard let jpeg = image.jpegData(compressionQuality: 0.85) else { + throw OpenGlassModelSession.SessionError.invalidFile("当前画面") + } + try jpeg.write(to: file, options: .atomic) + imageURL = file + let answer = try await model.ask(question: question, imageURL: file, answerLanguage: localeIdentifier) + isAnswering = false + if isRunning, runID == id { + answerView.text = answer.isEmpty ? "模型没有返回可朗读的文字，请换个问题重试。" : answer + if !suppressAnswerSpeech, speakSwitch.isOn, !answer.isEmpty { + speak(answer) + } else { + statusLabel.text = "回答完成。" + resumeListeningAfterEchoDelay() + } + } + } catch { + isAnswering = false + if isRunning, runID == id { + statusLabel.text = error.localizedDescription + // Errors are explicit; do not restart recognition in a loop. + speechPaused = true + if model.requiresAppRestart || !model.isLoaded { + stopService(message: error.localizedDescription) + } + } + } + // A timed-out native prefill may still read its file and context. + if !model.requiresAppRestart, let imageURL { try? FileManager.default.removeItem(at: imageURL) } + updateControls() + endBackgroundTaskIfIdle() + } + } + + private var localeIdentifier: String { languageControl.selectedSegmentIndex == 1 ? "en-US" : "zh-CN" } + + private func startListeningIfPossible() { + guard isRunning, !isStarting, !isAnswering, !model.isBusy, model.isLoaded, + !model.requiresAppRestart, voiceSwitch.isOn, !speechPaused, + audioConnected, activeUtterance == nil, !speech.isActive else { return } + do { + try speech.start(localeIdentifier: localeIdentifier) + } catch { + speechPaused = true + statusLabel.text = error.localizedDescription + } + updateControls() + } + + @objc private func toggleListening() { + if speech.isListening { speech.finishUtterance(); updateControls(); return } + if speech.isFinishing { return } + guard isRunning, !model.isBusy, !isAnswering else { return } + voiceSwitch.isOn = true + UserDefaults.standard.set(true, forKey: "OpenGlass.voiceEnabled") + speechPaused = false + let id = runID + Task { + let authorized = await OpenGlassSpeechSession.authorize() + guard isRunning, runID == id else { return } + if authorized { + if !audioConnected { statusLabel.text = "等待 Rokid 音频连接…" } + startListeningIfPossible() + } else { + speechPaused = true + statusLabel.text = "未获语音识别权限，请在系统设置中允许；文字提问仍可使用。" + } + updateControls() + } + } + + @objc private func voiceChanged() { + UserDefaults.standard.set(voiceSwitch.isOn, forKey: "OpenGlass.voiceEnabled") + if voiceSwitch.isOn { + if isRunning { toggleListening() } + } else { + speech.stop() + speechPaused = true + } + updateControls() + } + + @objc private func languageChanged() { + UserDefaults.standard.set(languageControl.selectedSegmentIndex, forKey: "OpenGlass.speechLanguage") + speech.stop() + startListeningIfPossible() + } + + @objc private func speakChanged() { + UserDefaults.standard.set(speakSwitch.isOn, forKey: "OpenGlass.speakAnswers") + if !speakSwitch.isOn { + stopSpeaking() + resumeListeningAfterEchoDelay() + } + } + + private func speak(_ text: String) { + speech.stop() + do { + let audio = AVAudioSession.sharedInstance() + try audio.setCategory(.playback, mode: .spokenAudio, options: []) + try audio.setActive(true) + let utterance = AVSpeechUtterance(string: text) + utterance.voice = AVSpeechSynthesisVoice(language: localeIdentifier) + utterance.rate = AVSpeechUtteranceDefaultSpeechRate + activeUtterance = utterance + statusLabel.text = "正在朗读；收音已暂停。" + updateAudioRoute() + synthesizer.speak(utterance) + } catch { + activeUtterance = nil + speechPaused = true + statusLabel.text = "朗读不可用：\(error.localizedDescription)。回答已保留在屏幕。" + } + updateControls() + } + + private func stopSpeaking() { + activeUtterance = nil + synthesizer.stopSpeaking(at: .immediate) + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + } + + nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { + Task { @MainActor [weak self] in + guard let self, self.activeUtterance === utterance else { return } + self.activeUtterance = nil + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + self.statusLabel.text = "朗读完成。" + self.resumeListeningAfterEchoDelay() + self.updateControls() + } + } + + nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didCancel utterance: AVSpeechUtterance) { + Task { @MainActor [weak self] in + guard let self, self.activeUtterance === utterance else { return } + self.activeUtterance = nil + self.speechPaused = true + self.statusLabel.text = "朗读被中断，点击“开始语音”可继续。" + self.updateControls() + } + } + + private func resumeListeningAfterEchoDelay() { + let id = runID + Task { + try? await Task.sleep(nanoseconds: 700_000_000) + guard isRunning, runID == id else { return } + startListeningIfPossible() + } + } + + @objc private func rotationChanged() { + UserDefaults.standard.set(rotationControl.selectedSegmentIndex * 90, forKey: "OpenGlass.imageRotation") + if let data = latestRawImage, let image = UIImage(data: data) { + latestImage = normalizedImage(image) + preview.image = latestImage + } + } + + private func normalizedImage(_ image: UIImage) -> UIImage { + let quarterTurns = rotationControl.selectedSegmentIndex + let radians = CGFloat(quarterTurns) * .pi / 2 + let original = image.size + let swapped = quarterTurns % 2 != 0 + let width = swapped ? original.height : original.width + let height = swapped ? original.width : original.height + let scale = min(1, 1024 / max(width, height)) + let size = CGSize(width: max(1, width * scale), height: max(1, height * scale)) + let format = UIGraphicsImageRendererFormat() + format.scale = 1 + format.opaque = true + return UIGraphicsImageRenderer(size: size, format: format).image { context in + context.cgContext.translateBy(x: size.width / 2, y: size.height / 2) + context.cgContext.rotate(by: radians) + image.draw(in: CGRect(x: -original.width * scale / 2, y: -original.height * scale / 2, + width: original.width * scale, height: original.height * scale)) + } + } + + private func refreshFrameAge() { + if let date = latestImageAt { + let age = max(0, Date().timeIntervalSince(date)) + frameLabel.text = age <= maximumFrameAge + ? "最新画面：\(Int(age)) 秒前 · 提问时使用这一张" + : "画面已过期（\(Int(age))秒前），等待 Rokid 新画面。" + frameLabel.textColor = age <= maximumFrameAge ? .secondaryLabel : .systemOrange + } else { + frameLabel.text = "尚未收到 Rokid 画面" + frameLabel.textColor = .secondaryLabel + } + } + + private func updateControls() { + let busy = model.isBusy || isAnswering + serviceButton.configuration?.title = isRunning || isStarting ? "停止接收" : "启动接收" + serviceButton.isEnabled = isRunning || (!busy && !model.requiresAppRestart) + askButton.isEnabled = isRunning && !isStarting && model.isLoaded && !busy && !model.requiresAppRestart + listenButton.configuration?.title = speech.isFinishing ? "正在结束语句…" : (speech.isListening ? "结束语句并提问" : "开始语音") + listenButton.isEnabled = isRunning && !isStarting && !busy && activeUtterance == nil + && !model.requiresAppRestart && !speech.isFinishing + stopTurnButton.isEnabled = isRunning || busy || activeUtterance != nil + downloadItem.isEnabled = !busy && !model.requiresAppRestart + languageControl.isEnabled = !busy && activeUtterance == nil + rotationControl.isEnabled = !busy + voiceSwitch.isEnabled = !isStarting + if model.requiresAppRestart { statusLabel.text = OpenGlassModelSession.SessionError.restartRequired.localizedDescription } + } + + @objc private func showDownloads() { + guard !model.isBusy, !isAnswering, !model.requiresAppRestart else { return } + stopService(message: "模型下载完成后，返回本页点击启动。") + Task { + do { + try await model.unload() + navigationController?.pushViewController(MBV46ModelDetailViewController(), animated: true) + } catch { + statusLabel.text = error.localizedDescription + } + updateControls() + } + } + + @objc private func refreshAddresses() { + let addresses = localIPv4Addresses() + addressView.text = addresses.isEmpty ? "尚未取得局域网 IPv4 地址，请连接 Wi-Fi 后刷新。" + : addresses.map { "http://\($0):18080" }.joined(separator: "\n") + } + + @objc private func copyAddress() { + if let first = localIPv4Addresses().first { + UIPasteboard.general.string = "http://\(first):18080" + statusLabel.text = "接收地址已复制。把 Rokid 采集端目标设为这个地址。" + } + } + + private func localIPv4Addresses() -> [String] { + var first: UnsafeMutablePointer? + guard getifaddrs(&first) == 0, let first else { return [] } + defer { freeifaddrs(first) } + var addresses: [(String, String)] = [] + var cursor: UnsafeMutablePointer? = first + while let current = cursor { + let interface = current.pointee + defer { cursor = interface.ifa_next } + guard let address = interface.ifa_addr, address.pointee.sa_family == UInt8(AF_INET) else { continue } + let name = String(cString: interface.ifa_name) + guard name == "en0" || name.hasPrefix("bridge") || name.hasPrefix("ap") else { continue } + var host = [CChar](repeating: 0, count: Int(NI_MAXHOST)) + if getnameinfo(address, socklen_t(address.pointee.sa_len), &host, socklen_t(host.count), + nil, 0, NI_NUMERICHOST) == 0 { + addresses.append((name, String(cString: host))) + } + } + return addresses.sorted { $0.0 < $1.0 }.map { $0.1 } + } + + @objc private func updateAudioRoute() { + let names = AVAudioSession.sharedInstance().currentRoute.outputs.map { $0.portName }.joined(separator: "、") + routeLabel.text = "朗读输出：\(names.isEmpty ? "iPhone 系统音频" : names)。可在系统中选择蓝牙眼镜。" + } + + @objc private func enteredBackground() { + stopService(message: "已进入后台并停止。回到前台后点击启动。") + if model.isBusy, backgroundTask == .invalid { + backgroundTask = UIApplication.shared.beginBackgroundTask(withName: "FinishOpenGlassTurn") { [weak self] in + Task { @MainActor in + guard let self else { return } + self.model.requestStop() + self.endBackgroundTask() + } + } + Task { + while model.isBusy && backgroundTask != .invalid { + try? await Task.sleep(nanoseconds: 200_000_000) + } + endBackgroundTask() + } + } + } + + @objc private func enteredForeground() { + refreshAddresses() + statusLabel.text = model.isBusy ? "正在结束上一轮推理，请稍候。" : "已回到前台，点击“启动接收”恢复。" + updateControls() + } + + @objc private func audioInterrupted(_ notification: Notification) { + guard let type = notification.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt, + type == AVAudioSession.InterruptionType.began.rawValue else { return } + speechPaused = true + speech.stop() + stopSpeaking() + statusLabel.text = "系统音频被打断；恢复后点击“开始语音”，或使用文字提问。" + updateControls() + } + + private func endBackgroundTaskIfIdle() { + if !model.isBusy { endBackgroundTask() } + } + + private func endBackgroundTask() { + guard backgroundTask != .invalid else { return } + UIApplication.shared.endBackgroundTask(backgroundTask) + backgroundTask = .invalid + } +} diff --git a/ios/Sources/RokidIngressServer.swift b/ios/Sources/RokidIngressServer.swift new file mode 100644 index 0000000..415308f --- /dev/null +++ b/ios/Sources/RokidIngressServer.swift @@ -0,0 +1,254 @@ +import Foundation +import Network + +/// Foreground-only local receiver for OpenGlassRokidSensor 0.1.3. +/// Call start/stop on the main queue; all callbacks are delivered on that queue. +final class RokidIngressServer { + var onImage: ((Data) -> Void)? + var onAudio: ((Data) -> Void)? + var onStatus: ((String) -> Void)? + var onAudioConnectionChanged: ((Bool) -> Void)? + + private let queue = DispatchQueue(label: "org.opensqz.rokid.ingress") + private var listener: NWListener? + private var clients: [UUID: RokidIngressConnection] = [:] + private var audioClient: UUID? + private var imageCount = 0 + private var audioPacketCount = 0 + private var epoch = UUID() + + func start(port: UInt16 = 18080) throws { + guard port > 0 else { throw RokidWireError.invalid("Invalid listen port") } + let parameters = NWParameters.tcp + parameters.allowLocalEndpointReuse = true + let newListener = try NWListener(using: parameters, on: NWEndpoint.Port(rawValue: port)!) + stop() + let generation = UUID() + epoch = generation + listener = newListener + newListener.stateUpdateHandler = { [weak self, weak newListener] state in + guard let self else { return } + switch state { + case .ready: self.deliver(generation) { $0.onStatus?("正在接收 Rokid，端口 \(port)") } + case .failed(let error): + self.deliver(generation) { $0.onStatus?("接收服务失败：\(error.localizedDescription)") } + newListener?.cancel() + default: break + } + } + newListener.newConnectionHandler = { [weak self] connection in + guard let self, self.clients.count < 6 else { connection.cancel(); return } + let client = RokidIngressConnection(connection: connection, queue: self.queue) + self.clients[client.id] = client + client.healthJSON = { [weak self] in + guard let self else { return "{}" } + return "{\"service\":\"openglass-ios\",\"protocol\":1,\"image_count\":\(self.imageCount),\"audio_packets_in\":\(self.audioPacketCount),\"audio_connected\":\(self.audioClient != nil)}" + } + client.onImage = { [weak self] data in + self?.imageCount += 1 + self?.deliver(generation) { $0.onImage?(data) } + } + client.onAudioOpen = { [weak self, weak client] in + guard let self, let client else { return } + // A reconnect replaces the previous audio source instead of interleaving PCM. + let previous = self.audioClient + self.audioClient = client.id + if let previous { self.clients[previous]?.close() } + self.deliver(generation) { $0.onAudioConnectionChanged?(true) } + } + client.onAudio = { [weak self, weak client] data in + guard let self, let client, self.audioClient == client.id else { return } + self.audioPacketCount += 1 + self.deliver(generation) { $0.onAudio?(data) } + } + client.onClosed = { [weak self, weak client] in + guard let self, let client else { return } + self.clients.removeValue(forKey: client.id) + if self.audioClient == client.id { + self.audioClient = nil + self.deliver(generation) { $0.onAudioConnectionChanged?(false) } + } + } + client.start() + } + newListener.start(queue: queue) + } + + func stop() { + epoch = UUID() + listener?.cancel() + listener = nil + queue.sync { + let closing = Array(clients.values) + clients.removeAll() + audioClient = nil + imageCount = 0 + audioPacketCount = 0 + closing.forEach { $0.close() } + } + onAudioConnectionChanged?(false) + } + + private func deliver(_ generation: UUID, _ callback: @escaping (RokidIngressServer) -> Void) { + DispatchQueue.main.async { [weak self] in + guard let self, self.epoch == generation else { return } + callback(self) + } + } +} + +private final class RokidIngressConnection { + let id = UUID() + var onImage: ((Data) -> Void)? + var onAudio: ((Data) -> Void)? + var onAudioOpen: (() -> Void)? + var onClosed: (() -> Void)? + var healthJSON: (() -> String)? + private let connection: NWConnection + private let queue: DispatchQueue + private var buffer = Data() + private var head: RokidHTTPHead? + private var decoder = RokidWebSocketDecoder() + private var websocket = false + private var closed = false + private var responding = false + private var deadline: DispatchSourceTimer? + private var lastInput = Date() + private let created = Date() + + init(connection: NWConnection, queue: DispatchQueue) { + self.connection = connection + self.queue = queue + } + + func start() { + connection.stateUpdateHandler = { [weak self] state in + switch state { + case .failed, .cancelled: self?.close() + default: break + } + } + connection.start(queue: queue) + let timer = DispatchSource.makeTimerSource(queue: queue) + timer.schedule(deadline: .now() + 5, repeating: 5) + timer.setEventHandler { [weak self] in + guard let self else { return } + let age = Date().timeIntervalSince(self.websocket ? self.lastInput : self.created) + if age > (self.websocket ? 30 : 15) { self.close() } + } + deadline = timer + timer.resume() + receive() + } + + func close() { + guard !closed else { return } + closed = true + deadline?.cancel() + deadline = nil + connection.cancel() + onClosed?() + } + + private func receive() { + guard !closed, !responding else { return } + connection.receive(minimumIncompleteLength: 1, maximumLength: 64 * 1024) { [weak self] data, _, done, error in + guard let self, !self.closed else { return } + if let data, !data.isEmpty { + self.lastInput = Date() + do { + if self.websocket { try self.consumeAudio(data) } + else { try self.consumeHTTP(data) } + } catch { + if self.websocket { self.closeWebSocket(code: 1002) } + else { self.respond(status: "400 Bad Request", body: "Invalid Rokid request") } + } + } + if error != nil || (done && !self.responding) { self.close() } + else { self.receive() } + } + } + + private func consumeHTTP(_ data: Data) throws { + buffer.append(data) + guard buffer.count <= RokidHTTPHead.maximumImageBytes + RokidHTTPHead.maximumHeaderBytes else { + throw RokidWireError.invalid("Image too large") + } + if head == nil { + guard let parsed = try RokidHTTPHead.parse(buffer) else { return } + head = parsed + buffer = Data(buffer.dropFirst(parsed.bytesConsumed)) + if parsed.method == "GET", parsed.path == "/health" { + respond(status: "200 OK", body: healthJSON?() ?? "{}", type: "application/json") + return + } + if parsed.path == "/rokid/audio" { + let accept = try parsed.webSocketAccept() + websocket = true + let reply = "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: \(accept)\r\n\r\n" + connection.send(content: Data(reply.utf8), completion: .contentProcessed { [weak self] error in + if error != nil { self?.close() } + }) + onAudioOpen?() + let remainder = buffer + buffer.removeAll(keepingCapacity: false) + if !remainder.isEmpty { try consumeAudio(remainder) } + return + } + guard parsed.method == "POST", parsed.path == "/rokid/image" else { + respond(status: "404 Not Found", body: "Unknown route") + return + } + guard let length = parsed.contentLength, length > 3, length <= RokidHTTPHead.maximumImageBytes, + parsed.headers["content-type"]?.lowercased().split(separator: ";").first == "image/jpeg" else { + throw RokidWireError.invalid("Expected a bounded JPEG body") + } + if parsed.headers["expect"]?.lowercased() == "100-continue" { + connection.send(content: Data("HTTP/1.1 100 Continue\r\n\r\n".utf8), completion: .contentProcessed { _ in }) + } + } + guard let count = head?.contentLength, buffer.count >= count else { return } + let jpeg = Data(buffer.prefix(count)) + guard jpeg.starts(with: [0xff, 0xd8]), jpeg.suffix(2) == Data([0xff, 0xd9]) else { + throw RokidWireError.invalid("Invalid JPEG markers") + } + onImage?(jpeg) + respond(status: "200 OK", body: "{\"ok\":true}", type: "application/json") + } + + private func consumeAudio(_ data: Data) throws { + for event in try decoder.append(data) { + switch event { + case .binary(let pcm): + guard pcm.count % 2 == 0 else { throw RokidWireError.invalid("PCM16 requires complete samples") } + if !pcm.isEmpty { onAudio?(pcm) } + case .ping(let payload): + connection.send(content: RokidWebSocketDecoder.serverControlFrame(opcode: 10, payload: payload), + completion: .contentProcessed { [weak self] error in + if error != nil { self?.close() } + }) + case .close(let payload): + responding = true + connection.send(content: RokidWebSocketDecoder.serverControlFrame(opcode: 8, payload: payload), + completion: .contentProcessed { [weak self] _ in self?.close() }) + return + } + } + } + + private func closeWebSocket(code: UInt16) { + responding = true + let payload = Data([UInt8(code >> 8), UInt8(code & 0xff)]) + connection.send(content: RokidWebSocketDecoder.serverControlFrame(opcode: 8, payload: payload), + completion: .contentProcessed { [weak self] _ in self?.close() }) + } + + private func respond(status: String, body: String, type: String = "text/plain") { + guard !responding else { return } + responding = true + let bytes = Data(body.utf8) + var response = Data("HTTP/1.1 \(status)\r\nContent-Type: \(type)\r\nContent-Length: \(bytes.count)\r\nConnection: close\r\n\r\n".utf8) + response.append(bytes) + connection.send(content: response, completion: .contentProcessed { [weak self] _ in self?.close() }) + } +} diff --git a/ios/Sources/RokidWireProtocol.swift b/ios/Sources/RokidWireProtocol.swift new file mode 100644 index 0000000..17a2fe5 --- /dev/null +++ b/ios/Sources/RokidWireProtocol.swift @@ -0,0 +1,156 @@ +import Foundation +import CryptoKit + +enum RokidWireError: Error, LocalizedError { + case invalid(String) + var errorDescription: String? { + if case .invalid(let message) = self { return message } + return nil + } +} + +/// Only the two existing Rokid upload routes and a health probe are supported. +/// Parsing is incremental: TCP packet boundaries are not HTTP/WS message boundaries. +struct RokidHTTPHead: Equatable { + static let maximumHeaderBytes = 16 * 1024 + static let maximumImageBytes = 4 * 1024 * 1024 + let method: String + let path: String + let headers: [String: String] + let bytesConsumed: Int + + var contentLength: Int? { + guard let value = headers["content-length"], !value.isEmpty, + value.allSatisfy({ $0 >= "0" && $0 <= "9" }) else { return nil } + return Int(value) + } + + static func parse(_ data: Data) throws -> RokidHTTPHead? { + guard let boundary = data.range(of: Data("\r\n\r\n".utf8)) else { + if data.count > maximumHeaderBytes { throw RokidWireError.invalid("HTTP header too large") } + return nil + } + let count = data.distance(from: data.startIndex, to: boundary.upperBound) + guard count <= maximumHeaderBytes, + let text = String(data: data.prefix(count - 4), encoding: .ascii) else { + throw RokidWireError.invalid("Invalid HTTP header") + } + let lines = text.components(separatedBy: "\r\n") + let request = lines[0].split(separator: " ", omittingEmptySubsequences: false) + guard request.count == 3, request[2] == "HTTP/1.1" || request[2] == "HTTP/1.0" else { + throw RokidWireError.invalid("Invalid request line") + } + var headers: [String: String] = [:] + for line in lines.dropFirst() { + guard let colon = line.firstIndex(of: ":"), colon != line.startIndex, + line.first != " ", line.first != "\t" else { + throw RokidWireError.invalid("Invalid HTTP field") + } + let name = line[.. String { + let connection = (headers["connection"] ?? "").lowercased() + .split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) } + guard method == "GET", path == "/rokid/audio", + headers["upgrade"]?.lowercased() == "websocket", connection.contains("upgrade"), + headers["sec-websocket-version"] == "13", + let key = headers["sec-websocket-key"], Data(base64Encoded: key)?.count == 16, + (contentLength ?? 0) == 0 else { + throw RokidWireError.invalid("Invalid WebSocket upgrade") + } + let digest = Insecure.SHA1.hash(data: Data((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").utf8)) + return Data(digest).base64EncodedString() + } +} + +enum RokidWebSocketEvent { + case binary(Data) + case ping(Data) + case close(Data) +} + +struct RokidWebSocketDecoder { + static let maximumMessageBytes = 128 * 1024 + private var buffer = Data() + private var fragmented: Data? + + mutating func append(_ data: Data) throws -> [RokidWebSocketEvent] { + buffer.append(data) + var events: [RokidWebSocketEvent] = [] + while buffer.count >= 2 { + let bytes = [UInt8](buffer.prefix(14)) + let final = bytes[0] & 0x80 != 0 + let opcode = bytes[0] & 0x0f + guard bytes[0] & 0x70 == 0, bytes[1] & 0x80 != 0 else { + throw RokidWireError.invalid("Client frames must be masked; extensions are disabled") + } + var offset = 2 + var length = UInt64(bytes[1] & 0x7f) + if length == 126 { + guard buffer.count >= 4 else { break } + length = UInt64(bytes[2]) << 8 | UInt64(bytes[3]) + guard length >= 126 else { throw RokidWireError.invalid("Noncanonical frame length") } + offset = 4 + } else if length == 127 { + guard buffer.count >= 10 else { break } + guard bytes[2] & 0x80 == 0 else { throw RokidWireError.invalid("Invalid frame length") } + length = bytes[2..<10].reduce(UInt64(0)) { ($0 << 8) | UInt64($1) } + guard length >= 65536 else { throw RokidWireError.invalid("Noncanonical frame length") } + offset = 10 + } + guard length <= UInt64(Self.maximumMessageBytes) else { + throw RokidWireError.invalid("Audio frame too large") + } + guard [0, 2, 8, 9, 10].contains(Int(opcode)) else { + throw RokidWireError.invalid("Only binary PCM audio is supported") + } + if opcode >= 8 && (!final || length > 125) { + throw RokidWireError.invalid("Invalid WebSocket control frame") + } + let total = offset + 4 + Int(length) + guard buffer.count >= total else { break } + let mask = Array(buffer.dropFirst(offset).prefix(4)) + let payload = Data(buffer.dropFirst(offset + 4).prefix(Int(length)).enumerated() + .map { $0.element ^ mask[$0.offset % 4] }) + buffer = Data(buffer.dropFirst(total)) + switch opcode { + case 2: + guard fragmented == nil else { throw RokidWireError.invalid("Nested fragmented message") } + if final { events.append(.binary(payload)) } else { fragmented = payload } + case 0: + guard var message = fragmented else { throw RokidWireError.invalid("Unexpected continuation") } + guard message.count + payload.count <= Self.maximumMessageBytes else { + throw RokidWireError.invalid("Audio message too large") + } + message.append(payload) + if final { fragmented = nil; events.append(.binary(message)) } else { fragmented = message } + case 8: + guard payload.count != 1 else { throw RokidWireError.invalid("Invalid close payload") } + events.append(.close(payload)) + return events + case 9: events.append(.ping(payload)) + default: break // pong + } + } + guard buffer.count <= Self.maximumMessageBytes + 14 else { + throw RokidWireError.invalid("Receive buffer exceeded") + } + return events + } + + static func serverControlFrame(opcode: UInt8, payload: Data) -> Data { + precondition(payload.count <= 125) + var data = Data([0x80 | opcode, UInt8(payload.count)]) + data.append(payload) + return data + } +} diff --git a/ios/scripts/build_ipa.sh b/ios/scripts/build_ipa.sh new file mode 100644 index 0000000..c662da3 --- /dev/null +++ b/ios/scripts/build_ipa.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Build an unsigned device IPA for local signing on Windows. No Apple secrets. +set -euo pipefail + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" +UPSTREAM="${ROOT}/tmp/MiniCPM-V-Apps" +BUILD_ROOT="${ROOT}/tmp/ios-build" +ARTIFACTS="${ROOT}/artifacts" +BUILD_NUMBER="${GITHUB_RUN_NUMBER:-1}" +export ROOT UPSTREAM BUILD_ROOT ARTIFACTS + +if [[ "$(uname -s)" != Darwin ]]; then + echo "build_ipa.sh requires macOS with Xcode; run the iOS IPA GitHub Actions workflow from Windows." >&2 + exit 1 +fi +python3 "${ROOT}/ios/scripts/prepare_upstream.py" --build-number "${BUILD_NUMBER}" + +# Check every generated root before upstream's build script cleans its own dirs. +python3 - <<'PY' +import os +from pathlib import Path +root = Path(os.environ['ROOT']).resolve() +upstream = Path(os.environ['UPSTREAM']).resolve() +if not upstream.is_relative_to(root / 'tmp'): + raise SystemExit('Upstream build root escapes repository tmp/') +for relative in ('llama.cpp-omni/build-ios-sim', 'llama.cpp-omni/build-ios-device', + 'llama.cpp-omni/build-macos', 'llama.cpp-omni/build-apple', + 'MiniCPM-V-demo/thirdparty/llama.xcframework'): + if not (upstream / relative).resolve().is_relative_to(upstream): + raise SystemExit('Upstream build output resolves outside its checkout') +for name in ('BUILD_ROOT', 'ARTIFACTS'): + if not Path(os.environ[name]).resolve().is_relative_to(root): + raise SystemExit(name + ' escapes repository') +PY + +FRAMEWORK="${UPSTREAM}/MiniCPM-V-demo/thirdparty/llama.xcframework" +FRAMEWORK_STAMP="${UPSTREAM}/MiniCPM-V-demo/thirdparty/.openglass-framework-key" +FRAMEWORK_KEY="$(python3 - <<'PY' +import hashlib, os, subprocess +from pathlib import Path +payload = (Path(os.environ['ROOT']) / 'ios/upstream-lock.json').read_bytes() +payload += subprocess.check_output(['xcodebuild', '-version']) +payload += subprocess.check_output(['uname', '-m']) +print(hashlib.sha256(payload).hexdigest()) +PY +)" +if [[ ! -f "${FRAMEWORK}/Info.plist" || ! -f "${FRAMEWORK_STAMP}" || "$(cat "${FRAMEWORK_STAMP}")" != "${FRAMEWORK_KEY}" ]]; then + (cd "${UPSTREAM}" && MINIMAL_MODE=ios-device bash scripts/build_xcframework.sh) + printf '%s\n' "${FRAMEWORK_KEY}" > "${FRAMEWORK_STAMP}" +fi + +mkdir -p "${BUILD_ROOT}" "${ARTIFACTS}" +PROJECT="${UPSTREAM}/MiniCPM-V-demo.xcodeproj" +COMMON_ARGS=( + -project "${PROJECT}" -scheme MiniCPM-V-demo + -clonedSourcePackagesDirPath "${BUILD_ROOT}/SourcePackages" + -onlyUsePackageVersionsFromResolvedFile +) +xcodebuild "${COMMON_ARGS[@]}" -resolvePackageDependencies 2>&1 | tee "${BUILD_ROOT}/resolve.log" +xcodebuild "${COMMON_ARGS[@]}" \ + -configuration Release -sdk iphoneos -destination 'generic/platform=iOS' \ + -derivedDataPath "${BUILD_ROOT}/DerivedData" -disableAutomaticPackageResolution \ + CODE_SIGNING_ALLOWED=NO CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY='' \ + DEVELOPMENT_TEAM='' CODE_SIGN_ENTITLEMENTS='' \ + build 2>&1 | tee "${BUILD_ROOT}/xcodebuild.log" + +APP="${BUILD_ROOT}/DerivedData/Build/Products/Release-iphoneos/MiniCPM-V-demo.app" +export APP +python3 - <<'PY' +import json, os, plistlib, subprocess +from pathlib import Path +root, app = Path(os.environ['ROOT']), Path(os.environ['APP']) +lock = json.loads((root / 'ios/upstream-lock.json').read_text()) +info = plistlib.loads((app / 'Info.plist').read_bytes()) +if info.get('CFBundleIdentifier') != lock['bundle_identifier']: + raise SystemExit('Built app has the wrong bundle identifier') +if info.get('CFBundleDisplayName') != lock['display_name']: + raise SystemExit('Built app has the wrong display name') +if info.get('CFBundleShortVersionString') != lock['version']: + raise SystemExit('Built app has the wrong version') +if (app / 'embedded.mobileprovision').exists(): + raise SystemExit('Unsigned build unexpectedly contains a provisioning profile') +if any(app.rglob('*.gguf')) or any(app.rglob('*.safetensors')): + raise SystemExit('Model weights must not be packaged into the IPA') +manifest = json.loads((Path(os.environ['UPSTREAM']) / '.openglass-prepared.json').read_text()) +manifest.update({'signed': False, 'xcode': subprocess.check_output(['xcodebuild', '-version'], text=True).strip(), + 'commit': subprocess.check_output(['git', '-C', str(root), 'rev-parse', 'HEAD'], text=True).strip()}) +(Path(os.environ['ARTIFACTS']) / 'build-info.json').write_text(json.dumps(manifest, indent=2) + '\n') +PY + +# mktemp creates a fresh staging tree; no broad rm -rf or stale Payload merging. +STAGING="$(mktemp -d "${BUILD_ROOT}/package.XXXXXX")" +mkdir -p "${STAGING}/Payload" +ditto "${APP}" "${STAGING}/Payload/OpenGlass.app" +IPA="${ARTIFACTS}/OpenGlass-1.0.ipa" +ditto -c -k --keepParent "${STAGING}/Payload" "${STAGING}/OpenGlass-1.0.ipa" +mv -f "${STAGING}/OpenGlass-1.0.ipa" "${IPA}" +unzip -tq "${IPA}" +echo "Unsigned IPA ready: ${IPA}" +echo "Sign this IPA on your Windows computer before installing on iPhone." diff --git a/ios/scripts/prepare_upstream.py b/ios/scripts/prepare_upstream.py new file mode 100644 index 0000000..3626345 --- /dev/null +++ b/ios/scripts/prepare_upstream.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""Prepare a pinned, disposable iOS build tree. Requires Python 3.10+ and Git. + +Only files below this repository's tmp/ directory may be generated. Each run +derives patched upstream files from the locked Git object, so it is idempotent +and never accumulates edits. The upstream model download UI is left intact. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import plistlib +import re +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[2] +APP = "MiniCPM-V-demo" +PROJECT = f"{APP}.xcodeproj/project.pbxproj" +PACKAGES = f"{APP}.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved" +APP_CONFIG_IDS = ("FA3FA2782C1176E90077F732", "FA3FA2792C1176E90077F732") +LOCAL_NETWORK_DESCRIPTION = "接收同一局域网内 Rokid 眼镜的图像和麦克风音频。" +SPEECH_DESCRIPTION = "在 iPhone 上将眼镜麦克风语音转换为文字；仅使用设备端识别。" + + +class PrepareError(RuntimeError): + pass + + +def git(directory: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(directory), *args], check=True, + stdout=subprocess.PIPE, encoding="utf-8", errors="strict", + ) + return result.stdout.strip() + + +def git_file(directory: Path, revision: str, path: str) -> bytes: + return subprocess.check_output(["git", "-C", str(directory), "show", f"{revision}:{path}"]) + + +def safe_generated_path(root: Path, path: Path) -> Path: + """Reject the repository itself, outside directories, and escaping symlinks.""" + root = root.resolve() + temporary = (root / "tmp").resolve() + if temporary != root / "tmp": + raise PrepareError("tmp/ must be a real directory inside the repository") + resolved = path.resolve() + if resolved == temporary or not resolved.is_relative_to(temporary): + raise PrepareError(f"Generated path must be strictly below {temporary}: {path}") + return resolved + + +def read_lock(root: Path) -> dict: + lock = json.loads((root / "ios/upstream-lock.json").read_text(encoding="utf-8")) + for revision in (lock["commit"], lock["submodule"]["commit"], *lock["swift_packages"].values()): + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise PrepareError("Upstream dependencies must use full 40-character Git SHAs") + if lock["submodule"]["path"] != "llama.cpp-omni": + raise PrepareError("Unexpected inference submodule path") + return lock + + +def ensure_checkout(destination: Path, lock: dict, skip_submodules: bool) -> None: + if not destination.exists(): + destination.mkdir(parents=True) + if not (destination / ".git").exists(): + if any(destination.iterdir()): + raise PrepareError("Refusing to initialize a nonempty, non-Git destination") + git(destination, "init") + git(destination, "remote", "add", "origin", lock["repository"]) + git(destination, "fetch", "--depth", "1", "origin", lock["commit"]) + git(destination, "checkout", "--detach", "FETCH_HEAD") + if Path(git(destination, "rev-parse", "--show-toplevel")).resolve() != destination.resolve(): + raise PrepareError("Destination is not the root of its own Git checkout") + if git(destination, "remote", "get-url", "origin").rstrip("/") != lock["repository"]: + raise PrepareError("Existing checkout has a different origin; choose a fresh tmp/ directory") + if git(destination, "rev-parse", "HEAD") != lock["commit"]: + raise PrepareError("Existing checkout is not at the locked commit; no reset was performed") + managed_files = {PROJECT, PACKAGES, f"{APP}/SceneDelegate.swift", f"{APP}/Info.plist", f"{APP}/{APP}.entitlements"} + changed = set(git(destination, "diff", "HEAD", "--name-only", "--ignore-submodules=all").splitlines()) + unexpected = changed - managed_files + if unexpected: + raise PrepareError("Upstream has edits outside generated files: " + ", ".join(sorted(unexpected))) + recorded = git(destination, "ls-tree", "HEAD", lock["submodule"]["path"]).split() + if len(recorded) != 4 or recorded[0] != "160000" or recorded[2] != lock["submodule"]["commit"]: + raise PrepareError("Upstream Git tree does not match the locked inference submodule") + if not skip_submodules: + git(destination, "submodule", "update", "--init", "--recursive", "--depth", "1", "--checkout") + submodule = destination / lock["submodule"]["path"] + if git(submodule, "rev-parse", "HEAD") != lock["submodule"]["commit"]: + raise PrepareError("Inference submodule checkout does not match its lock") + if git(submodule, "status", "--porcelain", "--untracked-files=no"): + raise PrepareError("Inference submodule has local edits; refusing a non-reproducible build") + + +def replace_once(text: str, old: str, new: str) -> str: + if text.count(old) != 1: + raise PrepareError(f"Expected exactly one upstream patch anchor: {old[:100]!r}") + return text.replace(old, new, 1) + + +def object_id(kind: str, name: str) -> str: + return hashlib.sha256(f"OpenGlass:{kind}:{name}".encode()).hexdigest()[:24].upper() + + +def patch_project(original: str, source_names: list[str], lock: dict, build_number: str) -> str: + if not re.fullmatch(r"[1-9][0-9]{0,8}(?:\.[0-9]{1,3}){0,2}", build_number): + raise PrepareError("Build number must be a positive Apple-compatible numeric version") + if not source_names or len(set(source_names)) != len(source_names): + raise PrepareError("Expected a nonempty list of distinct Swift source files") + if any(not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_+.\-]*\.swift", name) for name in source_names): + raise PrepareError("Swift source filenames must be safe PBX filenames") + source_names = sorted(source_names) + group_id = object_id("group", "OpenGlass") + build_files, references, children, sources = [], [], [], [] + for name in source_names: + reference, build = object_id("file", name), object_id("build", name) + if reference in original or build in original: + raise PrepareError("Generated PBX identifier collides with an upstream object") + build_files.append(f"\t\t{build} /* {name} in Sources */ = {{isa = PBXBuildFile; fileRef = {reference} /* {name} */; }};\n") + references.append(f'\t\t{reference} /* {name} */ = {{isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "{name}"; sourceTree = ""; }};\n') + children.append(f"\t\t\t\t{reference} /* {name} */,\n") + sources.append(f"\t\t\t\t{build} /* {name} in Sources */,\n") + text = replace_once(original, "/* Begin PBXBuildFile section */\n", "/* Begin PBXBuildFile section */\n" + "".join(build_files)) + text = replace_once(text, "/* Begin PBXFileReference section */\n", "/* Begin PBXFileReference section */\n" + "".join(references)) + group = (f"\t\t{group_id} /* OpenGlass */ = {{\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n" + + "".join(children) + f'\t\t\t);\n\t\t\tname = OpenGlass;\n\t\t\tpath = "{APP}/OpenGlass";\n\t\t\tsourceTree = SOURCE_ROOT;\n\t\t}};\n') + text = replace_once(text, "/* Begin PBXGroup section */\n", "/* Begin PBXGroup section */\n" + group) + root_group = "\t\tFA3FA25A2C1176E80077F732 = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n" + text = replace_once(text, root_group, root_group + f"\t\t\t\t{group_id} /* OpenGlass */,\n") + phase = "\t\tFA3FA25F2C1176E80077F732 /* Sources */ = {\n\t\t\tisa = PBXSourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n" + text = replace_once(text, phase, phase + "".join(sources)) + settings = { + "PRODUCT_BUNDLE_IDENTIFIER": lock["bundle_identifier"], + "INFOPLIST_KEY_CFBundleDisplayName": json.dumps(lock["display_name"]), + "INFOPLIST_KEY_NSLocalNetworkUsageDescription": json.dumps(LOCAL_NETWORK_DESCRIPTION, ensure_ascii=False), + "INFOPLIST_KEY_NSSpeechRecognitionUsageDescription": json.dumps(SPEECH_DESCRIPTION, ensure_ascii=False), + "MARKETING_VERSION": lock["version"], + "CURRENT_PROJECT_VERSION": build_number, + } + # Patch only the app's two configurations, retaining upstream test settings. + for config_id in APP_CONFIG_IDS: + pattern = rf"(?ms)^\t\t{config_id} /\* .*? \*/ = \{{\n.*?^\t\t\}};" + match = re.search(pattern, text) + if match is None: + raise PrepareError(f"Missing app build configuration {config_id}") + config = match.group(0) + for key, value in settings.items(): + config, count = re.subn(rf"(?m)^(\t{{4}}{key} = )[^\n;]*;", lambda m: f"{m[1]}{value};", config) + if count != 1: + raise PrepareError(f"Expected one {key} in app build configuration") + text = text[:match.start()] + config + text[match.end():] + return text + + +def write_generated(root: Path, destination: Path, relative: str, data: bytes) -> None: + path = safe_generated_path(root, destination / relative) + if not path.is_relative_to(destination): + raise PrepareError("Generated file escapes its upstream checkout") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + +def prepare(root: Path, destination: Path, lock: dict, build_number: str) -> dict: + destination = safe_generated_path(root, destination) + source_dir = root / "ios/Sources" + sources = sorted(source_dir.glob("*.swift")) + for source in sources: + if source.is_symlink() or source.resolve().parent != source_dir.resolve(): + raise PrepareError("Overlay sources must be regular files in ios/Sources") + revision = lock["commit"] + original = git_file(destination, revision, PROJECT).decode("utf-8").replace("\r\n", "\n") + project = patch_project(original, [source.name for source in sources], lock, build_number) + scene = git_file(destination, revision, f"{APP}/SceneDelegate.swift").decode("utf-8").replace("\r\n", "\n") + scene = replace_once(scene, "let vc = MBHomeViewController()", "let vc = OpenGlassViewController()") + info = plistlib.loads(git_file(destination, revision, f"{APP}/Info.plist")) + info.update({"CFBundleDisplayName": lock["display_name"], + "NSLocalNetworkUsageDescription": LOCAL_NETWORK_DESCRIPTION, + "NSSpeechRecognitionUsageDescription": SPEECH_DESCRIPTION}) + # No background-mode claim: receiving and inference require foreground use. + info.setdefault("NSAppTransportSecurity", {})["NSAllowsLocalNetworking"] = True + packages_data = git_file(destination, revision, PACKAGES) + pins = {item["identity"]: item["state"]["revision"] for item in json.loads(packages_data)["pins"]} + if pins != lock["swift_packages"]: + raise PrepareError("Package.resolved dependencies differ from upstream-lock.json") + outputs = {PROJECT: project.encode("utf-8"), f"{APP}/SceneDelegate.swift": scene.encode("utf-8"), + f"{APP}/Info.plist": plistlib.dumps(info, sort_keys=False), PACKAGES: packages_data, + # Free personal-team signing cannot use increased-memory-limit. + f"{APP}/{APP}.entitlements": plistlib.dumps({})} + outputs.update({f"{APP}/OpenGlass/{source.name}": source.read_bytes() for source in sources}) + for relative, content in outputs.items(): + write_generated(root, destination, relative, content) + manifest = {"upstream_commit": revision, "submodule_commit": lock["submodule"]["commit"], + "bundle_identifier": lock["bundle_identifier"], "version": lock["version"], + "build_number": build_number, "sources": {source.name: hashlib.sha256(source.read_bytes()).hexdigest() for source in sources}} + write_generated(root, destination, ".openglass-prepared.json", (json.dumps(manifest, indent=2) + "\n").encode()) + return manifest + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--upstream-dir", type=Path, default=ROOT / "tmp/MiniCPM-V-Apps") + parser.add_argument("--build-number", default=os.environ.get("GITHUB_RUN_NUMBER", "1")) + parser.add_argument("--skip-submodules", action="store_true", help="Prepare sources only; cannot build the IPA yet") + args = parser.parse_args() + try: + destination = safe_generated_path(ROOT, args.upstream_dir) + lock = read_lock(ROOT) + ensure_checkout(destination, lock, args.skip_submodules) + manifest = prepare(ROOT, destination, lock, args.build_number) + except (PrepareError, subprocess.CalledProcessError, OSError, ValueError) as exc: + print(f"prepare_upstream: {exc}", file=sys.stderr) + return 1 + print(f"Prepared OpenGlass {manifest['version']} ({manifest['build_number']}), {len(manifest['sources'])} Swift sources in {destination}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ios/tests/main.swift b/ios/tests/main.swift new file mode 100644 index 0000000..4083a4b --- /dev/null +++ b/ios/tests/main.swift @@ -0,0 +1,85 @@ +import Foundation + +var checks = 0 +func check(_ condition: @autoclosure () throws -> Bool, _ label: String) { + do { guard try condition() else { fatalError("FAIL: \(label)") } } + catch { fatalError("FAIL: \(label): \(error)") } + checks += 1 +} +func rejects(_ label: String, _ action: () throws -> Void) { + do { try action(); fatalError("FAIL: expected rejection: \(label)") } + catch { checks += 1 } +} +func frame(_ payload: Data, opcode: UInt8 = 2, final: Bool = true) -> Data { + var data = Data([(final ? 0x80 : 0) | opcode]) + if payload.count < 126 { data.append(0x80 | UInt8(payload.count)) } + else if payload.count <= 65535 { + data.append(contentsOf: [0xfe, UInt8(payload.count >> 8), UInt8(payload.count & 0xff)]) + } else { + data.append(0xff) + for shift in stride(from: 56, through: 0, by: -8) { + data.append(UInt8((UInt64(payload.count) >> shift) & 0xff)) + } + } + let mask: [UInt8] = [0x37, 0xfa, 0x21, 0x3d] + data.append(contentsOf: mask) + data.append(contentsOf: payload.enumerated().map { $0.element ^ mask[$0.offset % 4] }) + return data +} +func binary(_ events: [RokidWebSocketEvent]) -> [Data] { + events.compactMap { if case .binary(let data) = $0 { return data }; return nil } +} + +let jpegHeader = Data("POST /rokid/image HTTP/1.1\r\nHost: phone\r\nContent-Type: image/jpeg\r\nContent-Length: 1280\r\n\r\n".utf8) +for split in 0...' + } + return (Resolve-Path -LiteralPath $Adb).Path +} + +function Invoke-OpenGlassAdb { + param([string]$Adb, [string[]]$Arguments) + $output = @(& $Adb @Arguments 2>&1) + if ($LASTEXITCODE -ne 0) { throw "ADB failed ($LASTEXITCODE): $($output -join [Environment]::NewLine)" } + return ($output -join [Environment]::NewLine) +} + +function Get-OpenGlassRokid { + param([string]$Adb, [string]$Serial, [string]$Package, [switch]$UsbOnly) + $rows = Invoke-OpenGlassAdb $Adb @('devices', '-l') + $ready = @($rows -split '\r?\n' | ForEach-Object { + if ($_ -match '^([A-Za-z0-9_.:-]+)\s+device(?:\s|$)') { $Matches[1] } + }) + if ($UsbOnly) { $ready = @($ready | Where-Object { $_ -notmatch ':' }) } + if (-not $Serial) { + if ($ready.Count -ne 1) { throw 'Expected exactly one authorized Android device. Unlock/authorize it or pass -Serial from adb devices -l.' } + $Serial = $ready[0] + } + if ($Serial -notmatch '^[A-Za-z0-9_.:-]+$' -or $Serial -notin $ready) { + throw 'The selected Rokid device is not authorized/online on the requested transport.' + } + $listing = Invoke-OpenGlassAdb $Adb @('-s', $Serial, 'shell', 'pm', 'list', 'packages') + $installed = @($listing -split '\r?\n' | ForEach-Object { + if ($_ -match '^package:(org\.opensqz\.openglass\.rokid(?:\.debug)?)\s*$') { $Matches[1] } + }) + if (-not $Package) { + if ($installed.Count -ne 1) { throw 'Expected exactly one OpenGlass Rokid package. Install the collector or pass -Package for the installed debug/release variant.' } + $Package = $installed[0] + } + if ($Package -notin $installed) { throw 'The requested OpenGlass Rokid package is not installed.' } + $info = Invoke-OpenGlassAdb $Adb @('-s', $Serial, 'shell', 'dumpsys', 'package', $Package) + if ($info -notmatch 'versionName=(0\.1\.3(?:-debug)?)(?:\s|$)') { + throw 'Collector version 0.1.3 is required for pc_base_url. Upgrade or review newer protocol compatibility first.' + } + $version = $Matches[1] + $resolved = Invoke-OpenGlassAdb $Adb @('-s', $Serial, 'shell', 'cmd', 'package', 'resolve-activity', '--brief', '-a', 'android.intent.action.MAIN', '-c', 'android.intent.category.LAUNCHER', $Package) + $activities = @($resolved -split '\r?\n' | Where-Object { $_ -match ('^' + [regex]::Escape($Package) + '/[A-Za-z0-9_.$]+$') }) + if ($activities.Count -ne 1) { throw 'Could not resolve the installed collector launcher activity. No app was started.' } + [pscustomobject]@{ Serial = $Serial; Package = $Package; Version = $version; Activity = $activities[0] } +} + +function Resolve-OpenGlassPython { + param([string]$Python = 'python') + $found = Get-Command $Python -ErrorAction Stop + $version = @(& $found.Source -c 'import sys; assert sys.version_info >= (3, 11); print(sys.executable)' 2>&1) + if ($LASTEXITCODE -ne 0) { throw 'A working Python 3.11+ is required. Pass -Python .' } + return $found.Source +} diff --git a/ios/windows/configure-rokid.ps1 b/ios/windows/configure-rokid.ps1 new file mode 100644 index 0000000..da60fc5 --- /dev/null +++ b/ios/windows/configure-rokid.ps1 @@ -0,0 +1,31 @@ +<# Start the installed Rokid collector against the iPhone on the same LAN. #> +param( + [Parameter(Mandatory = $true)][string]$IPhoneAddress, + [ValidateRange(1, 65535)][int]$Port = 18080, + [string]$Adb, + [string]$Serial, + [string]$Package, + [switch]$SkipReceiverCheck +) +. (Join-Path $PSScriptRoot 'common.ps1') +$address = $null +if (-not [System.Net.IPAddress]::TryParse($IPhoneAddress, [ref]$address) -or + $address.AddressFamily -ne [System.Net.Sockets.AddressFamily]::InterNetwork -or + [System.Net.IPAddress]::IsLoopback($address) -or $address.ToString() -eq '0.0.0.0') { + throw 'Pass the iPhone Wi-Fi IPv4 address, not localhost, a URL, or the glasses address.' +} +$baseUrl = 'http://{0}:{1}' -f $address.ToString(), $Port +$adbPath = Resolve-OpenGlassAdb $Adb +$device = Get-OpenGlassRokid -Adb $adbPath -Serial $Serial -Package $Package +if (-not $SkipReceiverCheck) { + try { $health = Invoke-RestMethod -Uri "$baseUrl/health" -TimeoutSec 5 } + catch { throw "iPhone /health is unreachable at $baseUrl. Open OpenGlass on iPhone and start its receiver on the same LAN. $($_.Exception.Message)" } + if ($health.PSObject.Properties.Name -notcontains 'service' -or $health.service -ne 'openglass-ios') { + throw 'The endpoint is not the OpenGlass iOS receiver. Nothing was changed.' + } +} +$result = Invoke-OpenGlassAdb $adbPath @('-s', $device.Serial, 'shell', 'am', 'start', '-S', '-W', '-n', $device.Activity, '--es', 'pc_base_url', $baseUrl) +if ($result -match 'Error:|Exception') { throw "Collector launch failed: $result" } +Write-Host "Started $($device.Package) $($device.Version) -> $baseUrl" +Write-Host 'Check that image_count and audio_packets_in increase on iPhone. The URL is saved by the collector.' +Write-Host 'Wi-Fi was not enabled or reconfigured by this script. Both devices must already have network access.' diff --git a/ios/windows/diagnose.ps1 b/ios/windows/diagnose.ps1 new file mode 100644 index 0000000..b061b62 --- /dev/null +++ b/ios/windows/diagnose.ps1 @@ -0,0 +1,62 @@ +<# Read-only installation diagnostics. Does not sign, install, pair, or start apps. #> +param([string]$Adb, [string]$Serial, [string]$Package, [string]$Python = 'python', [string]$Ipa) +. (Join-Path $PSScriptRoot 'common.ps1') +Write-Host 'OpenGlass Windows / iPhone preflight' +Write-Host 'Apple credentials are not requested or read by this script.' + +try { + $adbPath = Resolve-OpenGlassAdb $Adb + $device = Get-OpenGlassRokid -Adb $adbPath -Serial $Serial -Package $Package + Write-Host "[OK] Rokid collector: $($device.Package), version $($device.Version)" + Write-Host " Activity: $($device.Activity)" + $wifi = Invoke-OpenGlassAdb $adbPath @('-s', $device.Serial, 'shell', 'settings', 'get', 'global', 'wifi_on') + Write-Host " Wi-Fi setting: $wifi (0=off; USB relay does not require Wi-Fi)" + $reverse = Invoke-OpenGlassAdb $adbPath @('-s', $device.Serial, 'reverse', '--list') + if ($reverse) { Write-Host " Existing reverse mappings (left untouched):`n$reverse" } +} catch { Write-Warning "Rokid: $($_.Exception.Message)" } + +if (Get-Command Get-PnpDevice -ErrorAction SilentlyContinue) { + $phones = @(Get-PnpDevice -PresentOnly -ErrorAction SilentlyContinue | Where-Object { $_.FriendlyName -match 'iPhone|Apple Mobile Device' }) + Write-Host "[INFO] Windows PnP iPhone/Apple Mobile Device entries: $($phones.Count)" + foreach ($phone in $phones) { Write-Host " $($phone.FriendlyName): $($phone.Status)" } + Write-Host ' PnP detection alone does not prove trust, app installation, or a running listener.' +} +$appleService = @(Get-Service -ErrorAction SilentlyContinue | Where-Object { $_.Name -eq 'Apple Mobile Device Service' -or $_.DisplayName -eq 'Apple Mobile Device Service' }) +if ($appleService.Count) { Write-Host "[INFO] Apple Mobile Device Service: $($appleService[0].Status)" } +else { Write-Warning 'Apple Mobile Device Service was not found. Follow the official AltServer Windows prerequisites.' } +foreach ($processName in @('AltServer', 'iTunes', 'iCloud')) { + $running = @(Get-Process -Name $processName -ErrorAction SilentlyContinue).Count -gt 0 + Write-Host "[INFO] $processName running: $running" +} +try { + $pythonPath = Resolve-OpenGlassPython $Python + $moduleVersion = @(& $pythonPath -c 'from importlib.metadata import version; print(version("pymobiledevice3"))' 2>&1) + if ($LASTEXITCODE -ne 0) { throw 'pymobiledevice3 is not installed in this Python environment.' } + Write-Host "[INFO] pymobiledevice3: $($moduleVersion -join ' ')" + Write-Host '[INFO] USB iPhone identifiers (local output; omit these when sharing logs):' + & $pythonPath -m pymobiledevice3 usbmux list --usb --simple + if ($LASTEXITCODE -ne 0) { Write-Warning 'usbmux discovery failed. Check USB, unlock/trust, and Apple Mobile Device Service.' } +} catch { Write-Warning "USB tools: $($_.Exception.Message)" } + +if (Get-Command Get-NetTCPConnection -ErrorAction SilentlyContinue) { + $listeners = @(Get-NetTCPConnection -State Listen -LocalPort 18080 -ErrorAction SilentlyContinue) + if ($listeners.Count) { + Write-Warning 'Windows port 18080 is occupied. Stop its owning session yourself, or use -LocalPort 18081 for the USB relay.' + $listeners | Select-Object LocalAddress, LocalPort, OwningProcess | Format-Table + } else { Write-Host '[OK] No listener currently found on Windows port 18080.' } +} +if ($Ipa) { + Add-Type -AssemblyName System.IO.Compression.FileSystem + $archive = [System.IO.Compression.ZipFile]::OpenRead((Resolve-Path -LiteralPath $Ipa).Path) + try { + $names = @($archive.Entries | ForEach-Object { $_.FullName }) + $appPlists = @($names | Where-Object { $_ -match '^Payload/[^/]+\.app/Info\.plist$' }) + if ($appPlists.Count -ne 1) { throw 'IPA must contain one Payload/.app/Info.plist.' } + $provision = @($names | Where-Object { $_ -match '^Payload/[^/]+\.app/embedded\.mobileprovision$' }).Count -gt 0 + $signature = @($names | Where-Object { $_ -match '^Payload/[^/]+\.app/_CodeSignature/CodeResources$' }).Count -gt 0 + Write-Host "[INFO] IPA layout valid; embedded profile=$provision, signature resources=$signature" + Write-Host ' File presence is not signature verification. CI unsigned IPA must be re-signed using AltStore/AltServer before installation.' + } finally { $archive.Dispose() } +} +Write-Host 'On iPhone: unlock, Trust This Computer, enable Developer Mode, then trust your signing profile when prompted.' +Write-Host 'For a previously unopenable app, check the actual error/expiry first; this script does not uninstall it.' diff --git a/ios/windows/download-artifact.ps1 b/ios/windows/download-artifact.ps1 new file mode 100644 index 0000000..cffaa2d --- /dev/null +++ b/ios/windows/download-artifact.ps1 @@ -0,0 +1,38 @@ +<# Download one successful, explicitly selected GitHub Actions run. Never uploads or starts CI. #> +param( + [Parameter(Mandatory = $true)][ValidatePattern('^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$')][string]$Repository, + [Parameter(Mandatory = $true)][ValidateRange(1, [long]::MaxValue)][long]$RunId, + [string]$ArtifactName, + [string]$Destination +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$gh = (Get-Command gh -ErrorAction Stop).Source +$runJson = @(& $gh api "repos/$Repository/actions/runs/$RunId" 2>&1) +if ($LASTEXITCODE -ne 0) { throw 'Cannot read the run. Check the repository/run ID and use gh auth login yourself if needed.' } +$run = ($runJson -join [Environment]::NewLine) | ConvertFrom-Json +if ($run.status -ne 'completed' -or $run.conclusion -ne 'success') { throw 'The selected workflow run has not completed successfully.' } +if ($run.path -notmatch '(^|/)ios-ipa\.yml(?:@.*)?$') { throw 'The run is not the ios-ipa.yml workflow. Select the iOS build run.' } +$artifactJson = @(& $gh api "repos/$Repository/actions/runs/$RunId/artifacts?per_page=100" 2>&1) +if ($LASTEXITCODE -ne 0) { throw 'Cannot list the run artifacts.' } +$artifactResponse = ($artifactJson -join [Environment]::NewLine) | ConvertFrom-Json +$candidates = @($artifactResponse.artifacts | Where-Object { + -not $_.expired -and $_.name -match '^OpenGlass-iOS-unsigned-[0-9]+$' +}) +if ($ArtifactName) { $candidates = @($candidates | Where-Object { $_.name -eq $ArtifactName }) } +if ($candidates.Count -ne 1) { throw 'Expected exactly one non-expired OpenGlass iOS artifact. Pass -ArtifactName or inspect the selected run.' } +if (-not $Destination) { + $Destination = Join-Path (Split-Path -Parent $PSScriptRoot) "artifacts\run-$RunId" +} +$fullDestination = [System.IO.Path]::GetFullPath($Destination) +if (Test-Path -LiteralPath $fullDestination) { throw "Destination already exists; choose a new empty output path: $fullDestination" } +New-Item -ItemType Directory -Path $fullDestination | Out-Null +& $gh run download "$RunId" --repo $Repository --name $candidates[0].name --dir $fullDestination +if ($LASTEXITCODE -ne 0) { throw "Artifact download failed. Partial output remains for inspection: $fullDestination" } +$ipa = Join-Path $fullDestination 'OpenGlass-1.0.ipa' +if (-not (Test-Path -LiteralPath $ipa -PathType Leaf)) { throw "The artifact is missing OpenGlass-1.0.ipa: $fullDestination" } +Write-Host "Downloaded unsigned IPA: $ipa" +Write-Host "Run: $($run.html_url)" +Write-Host "Commit: $($run.head_sha)" +Get-FileHash -LiteralPath $ipa -Algorithm SHA256 | Format-List +Write-Host 'Next: sign/install with AltStore and AltServer on Windows. Downloading an unsigned IPA does not install or authorize it.' diff --git a/ios/windows/requirements.txt b/ios/windows/requirements.txt new file mode 100644 index 0000000..522ea59 --- /dev/null +++ b/ios/windows/requirements.txt @@ -0,0 +1,2 @@ +# API verified against the upstream v11.32.0 tag; use a dedicated Python 3.11+ venv. +pymobiledevice3==11.32.0 diff --git a/ios/windows/start-usb-relay.ps1 b/ios/windows/start-usb-relay.ps1 new file mode 100644 index 0000000..2a68d50 --- /dev/null +++ b/ios/windows/start-usb-relay.ps1 @@ -0,0 +1,23 @@ +<# Keep this terminal open. Ctrl+C removes only the relay mapping created by this run. #> +param( + [string]$Adb, + [string]$Serial, + [string]$Package, + [string]$IPhoneUdid, + [string]$Python = 'python', + [ValidateRange(1, 65535)][int]$LocalPort = 18080, + [ValidateRange(1, 65535)][int]$IPhonePort = 18080, + [ValidateRange(1, 65535)][int]$RokidPort = 18080 +) +. (Join-Path $PSScriptRoot 'common.ps1') +$adbPath = Resolve-OpenGlassAdb $Adb +$pythonPath = Resolve-OpenGlassPython $Python +$device = Get-OpenGlassRokid -Adb $adbPath -Serial $Serial -Package $Package -UsbOnly +$relayArgs = @( + (Join-Path $PSScriptRoot 'usb_relay.py'), + '--adb', $adbPath, '--rokid-serial', $device.Serial, '--activity', $device.Activity, + '--local-port', "$LocalPort", '--iphone-port', "$IPhonePort", '--rokid-port', "$RokidPort" +) +if ($IPhoneUdid) { $relayArgs += @('--iphone-udid', $IPhoneUdid) } +& $pythonPath @relayArgs +if ($LASTEXITCODE -ne 0) { throw "USB relay exited with code $LASTEXITCODE. Read the diagnostic above; no unrelated processes were stopped." } diff --git a/ios/windows/test_usb_relay.py b/ios/windows/test_usb_relay.py new file mode 100644 index 0000000..54ecb46 --- /dev/null +++ b/ios/windows/test_usb_relay.py @@ -0,0 +1,134 @@ +"""Safety/lifecycle checks with fake devices; never invokes ADB or usbmuxd.""" + +import argparse +import asyncio +import signal +import socket +import types +import unittest +from unittest.mock import patch + +import usb_relay + + +class FakeForwarder: + instances = [] + + def __init__(self, serial, dst_port, src_port, listening_event, usbmux_connection_type): + assert usbmux_connection_type == "USB" + self.src_port = src_port + self.ready = listening_event + self.stopped = asyncio.Event() + self.server = None + self.instances.append(self) + + async def start(self, address): + self.server = await asyncio.start_server(lambda r, w: w.close(), address, self.src_port) + self.ready.set() + try: + await self.stopped.wait() + finally: + self.server.close() + await self.server.wait_closed() + + def stop(self): + self.stopped.set() + + +class RelayTests(unittest.TestCase): + def setUp(self): + FakeForwarder.instances = [] + self.args = argparse.Namespace( + adb="not-called", rokid_serial="TESTDEVICE", activity="org.opensqz.openglass.rokid.debug/.MainActivity", + iphone_udid=None, local_port=0, iphone_port=18080, rokid_port=18080, + ) + self.calls = [] + self.maps = [] + self.change_mapping = False + self.reject_mapping = False + + async def list_devices(self): + return [types.SimpleNamespace(serial="TESTPHONE", is_usb=True)] + + def fake_adb(self, args, *command): + self.calls.append(command) + if command == ("reverse", "--list"): + return "\n".join(f"UsbFfs {remote} {local}" for remote, local in self.maps) + if command[:2] == ("reverse", "--no-rebind"): + if self.reject_mapping: + raise RuntimeError("mapping was claimed by another session") + self.maps = [(command[2], command[3])] + if command[:3] == ("shell", "am", "start"): + if self.change_mapping: + self.maps = [("tcp:18080", "tcp:19000")] + asyncio.get_running_loop().call_soon(signal.getsignal(signal.SIGINT), signal.SIGINT, None) + if command[:2] == ("reverse", "--remove"): + self.maps = [] + return "" + + def execute(self, health=None): + package = types.ModuleType("pymobiledevice3") + package.usbmux = types.SimpleNamespace(list_devices=self.list_devices) + forwarder = types.ModuleType("pymobiledevice3.tcp_forwarder") + forwarder.UsbmuxTcpForwarder = FakeForwarder + modules = {"pymobiledevice3": package, "pymobiledevice3.tcp_forwarder": forwarder} + with patch.dict("sys.modules", modules), patch.object(usb_relay, "adb", self.fake_adb), patch.object( + usb_relay, "read_health", side_effect=health or (lambda port: {"service": "openglass-ios"}) + ): + asyncio.run(usb_relay.run(self.args)) + + def test_preexisting_reverse_aborts_without_changes(self): + self.maps = [("tcp:18080", "tcp:12345")] + with self.assertRaisesRegex(RuntimeError, "already has"): + self.execute() + self.assertEqual(self.calls, [("reverse", "--list")]) + self.assertEqual(FakeForwarder.instances, []) + + def test_busy_port_does_not_touch_mapping(self): + with socket.socket() as occupied: + occupied.bind(("127.0.0.1", 0)) + occupied.listen() + self.args.local_port = occupied.getsockname()[1] + with self.assertRaises(OSError): + self.execute() + self.assertEqual(self.calls, [("reverse", "--list")]) + + def test_unreachable_receiver_closes_listener_without_launch(self): + with self.assertRaisesRegex(RuntimeError, "keep the app|Keep the app"): + self.execute(health=ConnectionRefusedError("app not running")) + self.assertEqual(self.calls, [("reverse", "--list")]) + self.assertFalse(FakeForwarder.instances[0].server.is_serving()) + + def test_success_launch_and_cleanup_only_own_mapping(self): + self.execute() + self.assertIn(("reverse", "--no-rebind", "tcp:18080", "tcp:0"), self.calls) + self.assertIn(("reverse", "--remove", "tcp:18080"), self.calls) + self.assertTrue(any(command[:3] == ("shell", "am", "start") for command in self.calls)) + self.assertEqual(self.maps, []) + self.assertFalse(FakeForwarder.instances[0].server.is_serving()) + + def test_replaced_mapping_is_left_untouched(self): + self.change_mapping = True + self.execute() + self.assertNotIn(("reverse", "--remove", "tcp:18080"), self.calls) + self.assertEqual(self.maps, [("tcp:18080", "tcp:19000")]) + + def test_no_rebind_race_failure_does_not_remove_mapping(self): + self.reject_mapping = True + with self.assertRaisesRegex(RuntimeError, "another session"): + self.execute() + self.assertNotIn(("reverse", "--remove", "tcp:18080"), self.calls) + self.assertFalse(FakeForwarder.instances[0].server.is_serving()) + + def test_non_iphone_health_is_rejected(self): + response = types.SimpleNamespace(read=lambda limit: b'{"service":"other"}') + context = unittest.mock.MagicMock() + context.__enter__.return_value = response + opener = types.SimpleNamespace(open=lambda *a, **kw: context) + with patch.object(usb_relay.urllib.request, "build_opener", return_value=opener): + with self.assertRaisesRegex(RuntimeError, "not the OpenGlass iOS"): + usb_relay.read_health(18080) + + +if __name__ == "__main__": + unittest.main() diff --git a/ios/windows/usb_relay.py b/ios/windows/usb_relay.py new file mode 100644 index 0000000..840dc24 --- /dev/null +++ b/ios/windows/usb_relay.py @@ -0,0 +1,175 @@ +"""Relay a Rokid USB collector to an iPhone TCP listener without Wi-Fi. + +The model runs on iPhone. Windows forwards bytes only. Uses upstream +pymobiledevice3 v11.32.0 UsbmuxTcpForwarder and a scoped ADB reverse mapping. +No signing, Apple account access, automatic pairing, or process-wide cleanup. +""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import json +import logging +import re +import signal +import subprocess +import sys +import urllib.request + + +def adb(args: argparse.Namespace, *command: str) -> str: + result = subprocess.run( + [args.adb, "-s", args.rokid_serial, *command], + capture_output=True, text=True, timeout=20, check=False, + ) + if result.returncode: + raise RuntimeError(f"ADB failed: {result.stderr.strip() or result.stdout.strip()}") + return result.stdout.strip() + + +def reverse_pairs(text: str) -> list[tuple[str, str]]: + """ADB prints one line per mapping: transport remote local.""" + pairs = [] + for line in text.splitlines(): + fields = line.split() + if len(fields) >= 2: + pairs.append((fields[-2], fields[-1])) + return pairs + + +def read_health(port: int) -> dict: + # Ignore system HTTP proxies for a loopback-only request. + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + with opener.open(f"http://127.0.0.1:{port}/health", timeout=5) as response: + payload = json.loads(response.read(64 * 1024)) + if not isinstance(payload, dict) or payload.get("service") != "openglass-ios": + raise RuntimeError("/health is not the OpenGlass iOS receiver.") + return payload + + +async def run(args: argparse.Namespace) -> None: + try: + from pymobiledevice3 import usbmux + from pymobiledevice3.tcp_forwarder import UsbmuxTcpForwarder + except ImportError as exc: + raise RuntimeError("Install ios/windows/requirements.txt into this Python environment first.") from exc + + devices = [device for device in await asyncio.wait_for(usbmux.list_devices(), 10) if device.is_usb] + if args.iphone_udid: + devices = [device for device in devices if device.matches_udid(args.iphone_udid)] + if len(devices) != 1: + raise RuntimeError("Expected one USB iPhone. Unlock/trust it, check Apple Mobile Device Service, or pass --iphone-udid.") + remote, local = f"tcp:{args.rokid_port}", f"tcp:{args.local_port}" + if any(pair[0] == remote for pair in reverse_pairs(adb(args, "reverse", "--list"))): + raise RuntimeError(f"Rokid {remote} already has an ADB reverse mapping. Stop its owning session or choose --rokid-port. No mapping was replaced.") + + ready = asyncio.Event() + stop = asyncio.Event() + forwarder = UsbmuxTcpForwarder( + devices[0].serial, args.iphone_port, args.local_port, + listening_event=ready, usbmux_connection_type="USB", + ) + task = asyncio.create_task(forwarder.start(address="127.0.0.1")) + readiness = asyncio.create_task(ready.wait()) + mapping_created = False + previous_handlers = {} + loop = asyncio.get_running_loop() + # signal.signal works with Windows' default Proactor loop; add_signal_handler does not. + for sig in (signal.SIGINT, signal.SIGTERM): + previous_handlers[sig] = signal.getsignal(sig) + signal.signal(sig, lambda *_: loop.call_soon_threadsafe(stop.set)) + try: + done, _ = await asyncio.wait({task, readiness}, timeout=10, return_when=asyncio.FIRST_COMPLETED) + if task in done: + await task # Surface bind/port-conflict failures without touching ADB state. + raise RuntimeError("USB listener stopped before startup.") + if readiness not in done: + raise RuntimeError("Timed out starting the local USB listener.") + if stop.is_set(): + return + try: + health = await asyncio.to_thread(read_health, args.local_port) + except Exception as exc: + raise RuntimeError("iPhone OpenGlass /health is unavailable. Keep the app in front and start its receiver. USB discovery alone does not mean the app is installed/running.") from exc + if stop.is_set(): + return + # --no-rebind closes the race between the preflight list and mapping creation. + adb(args, "reverse", "--no-rebind", remote, local) + mapping_created = True + result = adb(args, "shell", "am", "start", "-S", "-W", "-n", args.activity, + "--es", "pc_base_url", f"http://127.0.0.1:{args.rokid_port}") + if "Error:" in result or "Exception" in result: + raise RuntimeError(f"Collector launch failed: {result}") + print(f"Rokid:{args.rokid_port} -> Windows loopback:{args.local_port} -> iPhone USB:{args.iphone_port}", flush=True) + print(f"Receiver ready: image_count={health.get('image_count', 0)}, audio_packets_in={health.get('audio_packets_in', 0)}", flush=True) + print("Keep this terminal and the iPhone app open. Ctrl+C stops this relay only.", flush=True) + stopping = asyncio.create_task(stop.wait()) + try: + done, _ = await asyncio.wait({task, stopping}, return_when=asyncio.FIRST_COMPLETED) + if task in done: + await task + finally: + stopping.cancel() + with contextlib.suppress(asyncio.CancelledError): + await stopping + finally: + if mapping_created: + try: + pairs = reverse_pairs(adb(args, "reverse", "--list")) + if (remote, local) in pairs: + adb(args, "reverse", "--remove", remote) + elif any(pair[0] == remote for pair in pairs): + print("Mapping changed by another session; leaving it untouched.", file=sys.stderr) + except Exception as exc: + print(f"Could not remove this session's reverse mapping (device may be unplugged): {exc}", file=sys.stderr) + forwarder.stop() + if not task.done(): + try: + await asyncio.wait_for(asyncio.shield(task), timeout=5) + except asyncio.TimeoutError: + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + readiness.cancel() + with contextlib.suppress(asyncio.CancelledError): + await readiness + for sig, previous in previous_handlers.items(): + signal.signal(sig, previous) + + +def port(value: str) -> int: + number = int(value) + if not 1 <= number <= 65535: + raise argparse.ArgumentTypeError("port must be between 1 and 65535") + return number + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--adb", required=True) + parser.add_argument("--rokid-serial", required=True) + parser.add_argument("--activity", required=True, help="Resolved installed Android launcher component") + parser.add_argument("--iphone-udid") + parser.add_argument("--local-port", type=port, default=18080) + parser.add_argument("--iphone-port", type=port, default=18080) + parser.add_argument("--rokid-port", type=port, default=18080) + args = parser.parse_args() + if not re.fullmatch(r"[A-Za-z0-9_.-]+", args.rokid_serial): + parser.error("Rokid must use an explicit USB serial, not a wireless ADB address") + if not re.fullmatch(r"org\.opensqz\.openglass\.rokid(?:\.debug)?/[A-Za-z0-9_.$]+", args.activity): + parser.error("activity must be the resolved OpenGlass Rokid launcher") + logging.basicConfig(level=logging.WARNING, format="%(levelname)s: %(message)s") + try: + asyncio.run(run(args)) + return 0 + except KeyboardInterrupt: + return 0 + except Exception as exc: + print(f"USB relay stopped: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From 44d85adb28096af1989fb6f1d89873ca0a567e56 Mon Sep 17 00:00:00 2001 From: MengzhangLI Date: Fri, 16 Oct 2026 18:47:12 +0800 Subject: [PATCH 02/11] Harden iOS stream lifecycle and document Windows device setup --- ios/README_zh.md | 180 ++++++++++++++++++++++ ios/Sources/OpenGlassModelSession.swift | 4 + ios/Sources/OpenGlassSpeechSession.swift | 4 +- ios/Sources/OpenGlassViewController.swift | 7 +- ios/Sources/RokidIngressServer.swift | 56 ++++++- ios/windows/common.ps1 | 25 ++- 6 files changed, 261 insertions(+), 15 deletions(-) create mode 100644 ios/README_zh.md diff --git a/ios/README_zh.md b/ios/README_zh.md new file mode 100644 index 0000000..1f561e7 --- /dev/null +++ b/ios/README_zh.md @@ -0,0 +1,180 @@ +# Rokid → iPhone MiniCPM-V 4.6（Windows 开发） + +本目录把现有 Rokid 采集协议接入 iPhone，在手机本机运行 MiniCPM-V 4.6。Windows 可以编辑代码、编译眼镜 APK、下载 iOS 构建和协助安装；iOS 编译交给 GitHub Actions 的 macOS/Xcode runner，安装前在 Windows 通过 AltStore Classic / AltServer 用自己的 Apple 账号重新签名。不要求自有 MacBook。 + +这是开发版本。源码、构建成功、签名安装成功、真实眼镜音视频到达、手机模型回答成功是不同的验证阶段；下方命令不会把“USB 已识别”当成端到端成功。 + +## 运行路径 + +先使用双 USB 路径，眼镜 Wi-Fi 可以保持关闭： + +```text +Rokid 摄像头 / 麦克风 + → 眼镜 OpenGlassRokidSensor 0.1.3 + → adb reverse（眼镜 USB） + → Windows 127.0.0.1:18080 + → pymobiledevice3 usbmux（iPhone USB） + → iPhone OpenGlass :18080 + → MiniCPM-V 4.6 本机视觉推理 → 系统语音播报 +``` + +Windows 在这个路径只转发数据，不运行视觉模型。拔掉任一 USB 线会中断此路径。确认双 USB 稳定后，可用同一 Wi-Fi 下的直接连接，去掉 Windows 数据中继： + +```text +Rokid → HTTP JPEG + WebSocket PCM → iPhone 局域网地址:18080 +``` + +两条路径都要求眼镜本身支持 Android APK、Camera/Mic 权限和 ADB。不能仅根据“Rokid”品牌判断所有型号兼容。原采集 APK 无需改协议；当前采样是 JPEG 约 1 fps、16 kHz 单声道 PCM16LE。模型按问题读取最新画面，不是对每个视频帧连续推理。 + +## 1. 准备 Windows 与手机 + +需要 Android Platform Tools、Python 3.11+、Git；从 Actions 下载产物的脚本还需要 GitHub CLI。Android 采集 APK 已安装时，无需重新编译；需要编译时沿用仓库的 `rokid_app/build_install.cmd`，另备 JDK 17 和 Android SDK。 + +1. 将眼镜和 iPhone 都用支持数据传输的 USB 线连接 Windows。 +2. iPhone 解锁，手动同意“信任此电脑”；眼镜确认 USB 调试授权。 +3. 按 [AltStore 官方 Windows 安装说明](https://faq.altstore.io/altstore-classic/how-to-install-altstore-windows)安装 AltServer 及所要求的 Apple iTunes / iCloud 组件。官方默认流程使用 Apple 直接分发的版本；已安装 Store 版时，先参照官方故障排查，不要随意删除 Apple 驱动或服务。 +4. 在 iPhone 的“隐私与安全性”开启“开发者模式”，按系统要求重启并确认。签名配置的信任按系统提示在“通用 → VPN 与设备管理”中处理。 + +Apple 账号、密码及验证码只由本人输入 AltServer / AltStore 正规界面，不放入脚本、GitHub Secrets、日志或仓库。这里的自动化不读取 Apple 账号、配对记录或签名证书。 + +在仓库根目录创建独立 Python 环境： + +```powershell +python -m venv .\ios\windows\.venv +.\ios\windows\.venv\Scripts\python.exe -m pip install -r .\ios\windows\requirements.txt +``` + +Python 命令未安装或打开 Microsoft Store 时，先安装真正的 Python，或者用其完整 `python.exe` 路径替换命令。 + +运行只读诊断： + +```powershell +.\ios\windows\diagnose.ps1 ` + -Adb "$env:USERPROFILE\Downloads\platform-tools\adb.exe" ` + -Python .\ios\windows\.venv\Scripts\python.exe +``` + +若有多个 Android 设备，在命令后加 `-Serial ''`。若同时安装 debug/release 两个采集 App，加 `-Package 'org.opensqz.openglass.rokid.debug'` 或实际已安装的 release 包。脚本会核验真实安装包、0.1.3 版本和 launcher Activity，不默认猜包名。分享诊断输出前删除设备序列号/UDID。 + +## 2. 获取 iOS IPA + +构建锁定 [MiniCPM-V-Apps](https://github.com/OpenBMB/MiniCPM-V-Apps) 与其 llama.cpp-omni 子模块的具体提交，见 `upstream-lock.json`；模型下载页沿用上游 MiniCPM-V 4.6 页面。iOS 工程由 `scripts/prepare_upstream.py` 在构建目录生成，不覆盖旧 Mac 上的项目。 + +在包含当前分支的 GitHub 仓库里打开 **Actions → Build OpenGlass iOS IPA**，选择该分支运行。工作流是 `.github/workflows/ios-ipa.yml`；首次编译原生推理框架会较慢。等待实际显示成功，再下载名为 `OpenGlass-iOS-unsigned-<运行编号>` 的 artifact，解压得到： + +- `OpenGlass-1.0.ipa`：未签名的真机 arm64 App。 +- `build-info.json`：构建来源信息。 + +也可在 Windows 用指定仓库和运行 ID 下载： + +```powershell +.\ios\windows\download-artifact.ps1 ` + -Repository '你的GitHub用户名/OpenGlass' ` + -RunId 123456789 +``` + +`RunId` 是 Actions 运行 URL 的数值 ID，不是 artifact 后缀的运行编号；示例值必须替换。脚本核验工作流、成功状态和未过期产物，输出源提交与 IPA SHA-256；目标目录存在时退出，避免覆盖旧产物。GitHub CLI 未登录时，由本人先运行 `gh auth login`。 + +## 3. 用免费 Apple 账号签名安装 + +CI 产物不能直接作为已签名 App 安装。`ideviceinstaller`、`pymobiledevice3 apps install` 和 USB 端口转发都不会把 unsigned IPA 变成可运行的签名 App。 + +按 [官方安装流程](https://faq.altstore.io/altstore-classic/how-to-install-altstore-windows)，从 Windows 托盘 AltServer 安装 AltStore Classic 到已解锁的 iPhone，并本人完成账号验证。再把 `OpenGlass-1.0.ipa` 保存到 iPhone 的“文件”App，在 AltStore 的 **My Apps → +** 选择该 IPA，保持 AltServer 可连接，完成签名安装。使用自己选择的文件传输方式；无需把模型权重或私人数据上传到公开链接。 + +源 bundle ID 为 `org.opensqz.openglass.ios`；重签工具可能附加账号相关标识。显示名称是 **OpenGlass**。旧手机 App 若由另一账号/Team 签名，不能保证覆盖安装、共享模型文件或保留原数据；先保留旧 App，核对错误提示与签名来源，避免通过卸载来试错。 + +免费账号需要定期刷新签名，通常 7 天到期，活跃侧载 App 上限为 3 个，AltStore 本身也占名额。使用 AltStore 管理续签；临近到期时连接 Windows 并刷新。官方限制见 [AltStore Classic 开发者说明](https://developer.altstore.io/)及[错误码](https://faq.altstore.io/altstore-classic/error-codes)。如果原 OpenGlass“点不开”，签名过期只是可能原因之一，还应区分未信任、开发者模式、闪退和模型内存问题。 + +## 4. 在 iPhone 准备模型 + +1. 打开新 **OpenGlass**，进入右上角“模型下载”。 +2. 在 MiniCPM-V 4.6 下载页完成 **LLM 与 VPM 两个 GGUF 文件**。这个 App 有独立沙盒；其他 TestFlight/App 已下载的模型不会自动出现在这里。下载与首次资源准备需要网络。 +3. 回到主界面，点“启动接收”，等待模型就绪。初次验证可先关闭“Rokid 离线语音提问”，用文字问题排除语音权限影响。 +4. 保持 OpenGlass 在前台。进入后台会停止接收和语音；返回后需要再次点“启动接收”。本实现不保证锁屏长期运行。 + +模型文件不能混用 MiniCPM-o 4.5、其他尺寸或未经验证的 GGUF 转换。加载会检查文件存在与 GGUF 头；这不能代替完整模型匹配与真机推理验证。文件名由锁定上游的 MiniCPM-V 4.6 常量决定。 + +## 5A. 双 USB 接入（先验证这条） + +确认 iPhone 已启动接收后，在 Windows 仓库根运行： + +```powershell +.\ios\windows\start-usb-relay.ps1 ` + -Adb "$env:USERPROFILE\Downloads\platform-tools\adb.exe" ` + -Python .\ios\windows\.venv\Scripts\python.exe +``` + +仅连接一副授权眼镜和一台 iPhone 时自动选择；多设备时显式加 `-Serial '<眼镜USB序列号>'`、`-IPhoneUdid ''`。用下列只读命令查询 iPhone USB UDID： + +```powershell +.\ios\windows\.venv\Scripts\python.exe -m pymobiledevice3 usbmux list --usb --simple +``` + +脚本先检查既有 ADB reverse 映射，随后在 Windows `127.0.0.1` 绑定端口，通过 iPhone USB 查询 `/health` 并核验 `service: openglass-ios`；通过后才建立独占反向映射并启动眼镜采集 App。App 无需 Wi-Fi 地址，实际保存的采集目标是眼镜自己的 `http://127.0.0.1:18080`。 + +Windows 原 Python 面板占用了 18080 时，先在该面板正常停止；也可以为本次中继换 Windows 端口，不改变 iPhone 监听端口： + +```powershell +.\ios\windows\start-usb-relay.ps1 ` + -Python .\ios\windows\.venv\Scripts\python.exe -LocalPort 18081 +``` + +如果眼镜18080已有其他会话的 reverse 映射，脚本会退出，不抢占。可结束原会话，或显式加 `-RokidPort 18081`；脚本会同步传入正确采集 URL。不要同时启动旧电脑面板去重配同一眼镜。 + +运行期间另开终端查看手机接收计数（修改了 `LocalPort` 就使用对应端口）： + +```powershell +Invoke-RestMethod http://127.0.0.1:18080/health +``` + +`image_count`、`audio_packets_in` 应随采集增长。主终端按 **Ctrl+C** 正常停止，会关闭本次转发器，并移除本次创建、且未被其他会话改变的 reverse 映射。不会杀死所有 Python/ADB 进程，不会卸载 App。眼镜采集 App 可能继续重连，必要时在眼镜界面手动停止。 + +底层使用官方 [pymobiledevice3 v11.32.0 TCP forwarder](https://github.com/doronz88/pymobiledevice3/blob/v11.32.0/pymobiledevice3/tcp_forwarder.py)，限定 USB。其 CLI 对应 `usbmux forward `，这里的包装脚本另外处理设备选择、健康检查和清理。普通 App TCP 转发不使用 iOS 开发调试隧道，不要求将 Apple 配对文件上传到任何服务。 + +## 5B. Wi-Fi 直连（USB 路径通过后） + +在官方 Rokid AI App/设备设置中先把眼镜和 iPhone 放到同一可互通 Wi-Fi。确认网络没有客户端隔离。在 iPhone OpenGlass 点“刷新地址”，取其显示的局域网 IPv4 地址。眼镜暂时保留 USB，仅用于设置采集地址： + +```powershell +.\ios\windows\configure-rokid.ps1 ` + -IPhoneAddress '192.168.1.42' ` + -Adb "$env:USERPROFILE\Downloads\platform-tools\adb.exe" +``` + +替换示例 IP。此脚本不替你打开/修改眼镜 Wi-Fi，只检测手机接收服务并重启眼镜采集 App。确认画面/音频计数增长后再拔掉眼镜 USB。此时即使 Windows 关闭，中间没有 Windows 转发；iPhone 仍需前台运行。 + +iPhone个人热点、眼镜厂商固件对 Wi-Fi/蓝牙共存的影响要另行真机验证，不能将同一 Wi-Fi 成功等同于热点成功。换网或手机 IP 改变后重跑地址配置。若 Windows 暂时不能访问手机但眼镜实际可达，只有明确理解原因时才用 `-SkipReceiverCheck`；它会跳过启动前的手机健康检查。 + +## 6. 验证对话和音频 + +先确认预览方向正确；主界面可选0/90/180/270°旋转，默认270°沿用原电脑路径。输入“我面前有什么？”后点“用最新画面提问”，验证手机能读到当前眼镜画面并生成回答。 + +然后启用“Rokid 离线语音提问”，选择中文或英文，允许系统语音识别权限。输入仍来自眼镜的 PCM 流，App不打开 iPhone 麦克风。Apple 设备端识别资源可用时执行离线识别；不可用会显示错误并保留文字提问，不偷偷改走云端。说完停顿或点“结束语句”后，识别文本与当前图片送给 MiniCPM-V 4.6。回答由系统 TTS 朗读；可在 iPhone 控制中心选择已配对的眼镜作为声音输出，前提是该型号支持相应蓝牙音频功能。 + +当前是“听问题 → 看图推理 → 朗读回答”的分轮交互；推理/播报时暂停送入识别的麦克风片段，避免回答被自己重新听见。它不提供 MiniCPM-o 的原生全双工音频模型，也不承诺打断播报时继续实时看图。 + +## 排障 + +| 现象 | 先检查 | +| --- | --- | +| Windows看得到iPhone，usbmux列表为空/报错 | 数据线、手机解锁与信任、Apple Mobile Device Service；PnP识别不等于配对完成 | +| USB中继提示手机 `/health` 不可达 | 新App是否成功签名安装并已启动接收；是否进入后台；模型加载失败是否导致接收停止 | +| 端口或ADB reverse已存在 | 在原会话中正常停止，或指定其他本机/眼镜端口；脚本不会杀进程抢占 | +| `/health` 成功但计数不增长 | 眼镜相机/麦克风权限、采集App版本、ADB授权、所选设备；Wi-Fi模式另查网络互通 | +| 画面到达但模型未就绪 | 两个正确的4.6模型是否下载完成、磁盘空间、模型加载报错；其他App模型沙盒不共享 | +| 有文本回答但没声音 | “朗读回答”开关、iPhone音量/音频路由、蓝牙配对；看界面实际输出设备 | +| 语音不可用但文字正常 | 系统语音识别权限与该语言的设备端资源；该功能不向云识别降级 | +| 切后台后断流 | 预期行为；回前台重启接收与语音 | +| 免费签名几天后点不开 | 先查AltStore剩余天数并刷新签名；不要先删除App及模型数据 | + +## 验证记录 + +- 本地脚本检查：PowerShell语法解析通过；USB中继使用伪设备测试了端口占用、已有reverse、并发映射冲突、接收端不可达、成功启动与只清理自身映射等情况。 +- 真机状态、CI构建链接、签名安装结果和端到端模型回答需分别记录；不能以以上测试替代真机验证。 +- 不在此文档写入个人设备序列号、UDID、Apple账号或内网凭据。现场更新请用设备类型和验证结果描述。 + +可重复执行无设备测试： + +```powershell +python -m unittest discover -s .\ios\windows -p test_usb_relay.py -v +``` diff --git a/ios/Sources/OpenGlassModelSession.swift b/ios/Sources/OpenGlassModelSession.swift index f822d2b..efa7c1b 100644 --- a/ios/Sources/OpenGlassModelSession.swift +++ b/ios/Sources/OpenGlassModelSession.swift @@ -32,6 +32,9 @@ final class OpenGlassModelSession { private var tokenSubscription: AnyCancellable? private var discardResult = false private var output = "" + // A timed-out worker cannot be joined through the upstream API. Keep its + // context alive even if UIKit later discards the view controller. + private static var quarantinedContexts: [MTMDWrapper] = [] static var modelFiles: [String] { [MiniCPMModelConst.modelv46_FileName, MiniCPMModelConst.mmprojv46_FileName] @@ -124,6 +127,7 @@ final class OpenGlassModelSession { // The upstream watchdog can return while C++ still uses ctx. // Retain the wrapper, never reset/free or start a second call. requiresAppRestart = true + Self.quarantinedContexts.append(engine) transition(.failed, SessionError.restartRequired.localizedDescription) throw SessionError.restartRequired } diff --git a/ios/Sources/OpenGlassSpeechSession.swift b/ios/Sources/OpenGlassSpeechSession.swift index 271aeaa..6a2e42d 100644 --- a/ios/Sources/OpenGlassSpeechSession.swift +++ b/ios/Sources/OpenGlassSpeechSession.swift @@ -67,7 +67,9 @@ final class OpenGlassSpeechSession { Task { @MainActor in guard let self, self.sessionID == id else { return } if let result { - self.latestText = result.bestTranscription.formattedString + let text = result.bestTranscription.formattedString + if !text.isEmpty && text != self.latestText { self.lastVoiceAt = Date() } + self.latestText = text self.onPartial?(self.latestText) if result.isFinal { self.complete(); return } } diff --git a/ios/Sources/OpenGlassViewController.swift b/ios/Sources/OpenGlassViewController.swift index a49a07a..d99e01f 100644 --- a/ios/Sources/OpenGlassViewController.swift +++ b/ios/Sources/OpenGlassViewController.swift @@ -188,7 +188,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp note.numberOfLines = 0 note.font = .preferredFont(forTextStyle: .footnote) note.textColor = .secondaryLabel - note.text = "Rokid 与 iPhone 需在同一可互通网络。首次使用请先下载模型。此 App 的模型文件与原 TestFlight App 分开保存。进入后台会停止接收和语音，回到前台后请重新启动。" + note.text = "Rokid 可通过同一局域网或 Windows 双 USB 中继连接。首次使用请先下载模型。此 App 的模型文件与原 TestFlight App 分开保存。进入后台会停止接收和语音，回到前台后请重新启动。" stack.addArrangedSubview(note) } @@ -229,6 +229,9 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp self.speech.appendPCM16LE(data) } server.onStatus = { [weak self] status in self?.connectionLabel.text = status } + server.onFailure = { [weak self] message in + self?.stopService(message: "接收服务失败：\(message)") + } server.onAudioConnectionChanged = { [weak self] connected in guard let self else { return } self.audioConnected = connected @@ -611,7 +614,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp @objc private func refreshAddresses() { let addresses = localIPv4Addresses() - addressView.text = addresses.isEmpty ? "尚未取得局域网 IPv4 地址，请连接 Wi-Fi 后刷新。" + addressView.text = addresses.isEmpty ? "暂无局域网 IPv4 地址。可使用 Windows 双 USB 中继，或连接 Wi-Fi 后刷新。" : addresses.map { "http://\($0):18080" }.joined(separator: "\n") } diff --git a/ios/Sources/RokidIngressServer.swift b/ios/Sources/RokidIngressServer.swift index 415308f..71f2a50 100644 --- a/ios/Sources/RokidIngressServer.swift +++ b/ios/Sources/RokidIngressServer.swift @@ -7,6 +7,7 @@ final class RokidIngressServer { var onImage: ((Data) -> Void)? var onAudio: ((Data) -> Void)? var onStatus: ((String) -> Void)? + var onFailure: ((String) -> Void)? var onAudioConnectionChanged: ((Bool) -> Void)? private let queue = DispatchQueue(label: "org.opensqz.rokid.ingress") @@ -16,8 +17,15 @@ final class RokidIngressServer { private var imageCount = 0 private var audioPacketCount = 0 private var epoch = UUID() + // Read/written only on queue. epoch belongs to the main queue and only + // filters UI callbacks; this also prevents an old listener accepting work. + private var servingGeneration: UUID? + private var imageDeliveryInFlight = false + private var pendingAudioBytes = 0 + private let maximumPendingAudioBytes = 64_000 // Two seconds at 16 kHz PCM16. func start(port: UInt16 = 18080) throws { + dispatchPrecondition(condition: .onQueue(.main)) guard port > 0 else { throw RokidWireError.invalid("Invalid listen port") } let parameters = NWParameters.tcp parameters.allowLocalEndpointReuse = true @@ -26,18 +34,26 @@ final class RokidIngressServer { let generation = UUID() epoch = generation listener = newListener + queue.sync { servingGeneration = generation } newListener.stateUpdateHandler = { [weak self, weak newListener] state in guard let self else { return } switch state { case .ready: self.deliver(generation) { $0.onStatus?("正在接收 Rokid，端口 \(port)") } case .failed(let error): - self.deliver(generation) { $0.onStatus?("接收服务失败：\(error.localizedDescription)") } + self.deliver(generation) { + let message = "接收服务失败：\(error.localizedDescription)" + $0.onStatus?(message) + $0.onFailure?(message) + } newListener?.cancel() default: break } } newListener.newConnectionHandler = { [weak self] connection in - guard let self, self.clients.count < 6 else { connection.cancel(); return } + guard let self, self.servingGeneration == generation, self.clients.count < 6 else { + connection.cancel() + return + } let client = RokidIngressConnection(connection: connection, queue: self.queue) self.clients[client.id] = client client.healthJSON = { [weak self] in @@ -45,8 +61,15 @@ final class RokidIngressServer { return "{\"service\":\"openglass-ios\",\"protocol\":1,\"image_count\":\(self.imageCount),\"audio_packets_in\":\(self.audioPacketCount),\"audio_connected\":\(self.audioClient != nil)}" } client.onImage = { [weak self] data in - self?.imageCount += 1 - self?.deliver(generation) { $0.onImage?(data) } + guard let self else { return } + self.imageCount += 1 + // A busy UI must not retain an unbounded queue of full JPEGs. + guard !self.imageDeliveryInFlight else { return } + self.imageDeliveryInFlight = true + self.deliver(generation, completionOnQueue: { [weak self] in + guard let self, self.servingGeneration == generation else { return } + self.imageDeliveryInFlight = false + }) { $0.onImage?(data) } } client.onAudioOpen = { [weak self, weak client] in guard let self, let client else { return } @@ -58,8 +81,19 @@ final class RokidIngressServer { } client.onAudio = { [weak self, weak client] data in guard let self, let client, self.audioClient == client.id else { return } + guard self.pendingAudioBytes + data.count <= self.maximumPendingAudioBytes else { + // End the stream explicitly instead of silently introducing + // holes in a recognition request while the UI is blocked. + client.close() + self.deliver(generation) { $0.onStatus?("音频处理未及时跟上，正在等待 Rokid 重连。") } + return + } self.audioPacketCount += 1 - self.deliver(generation) { $0.onAudio?(data) } + self.pendingAudioBytes += data.count + self.deliver(generation, completionOnQueue: { [weak self] in + guard let self, self.servingGeneration == generation else { return } + self.pendingAudioBytes -= data.count + }) { $0.onAudio?(data) } } client.onClosed = { [weak self, weak client] in guard let self, let client else { return } @@ -75,24 +109,30 @@ final class RokidIngressServer { } func stop() { + dispatchPrecondition(condition: .onQueue(.main)) epoch = UUID() listener?.cancel() listener = nil queue.sync { + servingGeneration = nil let closing = Array(clients.values) clients.removeAll() audioClient = nil imageCount = 0 audioPacketCount = 0 + imageDeliveryInFlight = false + pendingAudioBytes = 0 closing.forEach { $0.close() } } onAudioConnectionChanged?(false) } - private func deliver(_ generation: UUID, _ callback: @escaping (RokidIngressServer) -> Void) { + private func deliver(_ generation: UUID, completionOnQueue: (() -> Void)? = nil, + _ callback: @escaping (RokidIngressServer) -> Void) { DispatchQueue.main.async { [weak self] in - guard let self, self.epoch == generation else { return } - callback(self) + guard let self else { return } + if self.epoch == generation { callback(self) } + if let completionOnQueue { self.queue.async(execute: completionOnQueue) } } } } diff --git a/ios/windows/common.ps1 b/ios/windows/common.ps1 index c89dff7..0f0f802 100644 --- a/ios/windows/common.ps1 +++ b/ios/windows/common.ps1 @@ -60,8 +60,25 @@ function Get-OpenGlassRokid { function Resolve-OpenGlassPython { param([string]$Python = 'python') - $found = Get-Command $Python -ErrorAction Stop - $version = @(& $found.Source -c 'import sys; assert sys.version_info >= (3, 11); print(sys.executable)' 2>&1) - if ($LASTEXITCODE -ne 0) { throw 'A working Python 3.11+ is required. Pass -Python .' } - return $found.Source + $candidates = @() + if ($Python -eq 'python') { + $repoRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) + $candidates += Join-Path $PSScriptRoot '.venv\Scripts\python.exe' + $candidates += Join-Path $repoRoot 'tmp\ios-tools\Scripts\python.exe' + $pathPython = Get-Command python.exe -ErrorAction SilentlyContinue + if ($pathPython) { $candidates += $pathPython.Source } + $candidates += Join-Path $env:USERPROFILE 'miniconda3\python.exe' + $candidates += Join-Path $env:USERPROFILE 'anaconda3\python.exe' + } else { + $explicit = Get-Command $Python -ErrorAction SilentlyContinue + if ($explicit) { $candidates += $explicit.Source } + } + foreach ($candidate in ($candidates | Select-Object -Unique)) { + # Do not launch Windows' Store app-execution alias during a diagnostic. + if ($candidate -match '[\\/]Microsoft[\\/]WindowsApps[\\/]' -or + -not (Test-Path -LiteralPath $candidate -PathType Leaf)) { continue } + $version = @(& $candidate -c 'import sys; assert sys.version_info >= (3, 11); print(sys.executable)' 2>&1) + if ($LASTEXITCODE -eq 0) { return (Resolve-Path -LiteralPath $candidate).Path } + } + throw 'No working Python 3.11+ found (WindowsApps Store aliases are skipped). Pass -Python C:\path\to\python.exe, then install ios/windows/requirements.txt into that environment.' } From c11c87b39b5798f8b4218d3a47d5b6cc68e56091 Mon Sep 17 00:00:00 2001 From: MengzhangLI Date: Fri, 16 Oct 2026 18:59:50 +0800 Subject: [PATCH 03/11] Validate model downloads and finish Windows sideload workflow --- .github/workflows/ios-ipa.yml | 45 ++++++++++++++++++++++- ios/README_zh.md | 25 +++++++++---- ios/Sources/OpenGlassModelSession.swift | 20 +++++++++- ios/Sources/OpenGlassViewController.swift | 19 +++++++--- ios/windows/download-artifact.ps1 | 25 ++++++++++++- 5 files changed, 117 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ios-ipa.yml b/.github/workflows/ios-ipa.yml index aa7c87e..d3b867a 100644 --- a/.github/workflows/ios-ipa.yml +++ b/.github/workflows/ios-ipa.yml @@ -44,8 +44,9 @@ jobs: plutil -lint tmp/MiniCPM-V-Apps/MiniCPM-V-demo.xcodeproj/project.pbxproj plutil -lint tmp/MiniCPM-V-Apps/MiniCPM-V-demo/Info.plist plutil -lint tmp/MiniCPM-V-Apps/MiniCPM-V-demo/MiniCPM-V-demo.entitlements - - name: Cache pinned inference framework and Swift packages - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + - name: Restore pinned inference framework and Swift packages + id: restore_build_cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: | tmp/MiniCPM-V-Apps/MiniCPM-V-demo/thirdparty/llama.xcframework @@ -54,6 +55,46 @@ jobs: key: ios-device-${{ runner.os }}-${{ runner.arch }}-xcode26.1.1-${{ hashFiles('ios/upstream-lock.json', 'ios/scripts/build_ipa.sh') }} - name: Build unsigned device IPA run: bash ios/scripts/build_ipa.sh + - name: Validate completed inference framework for caching + id: framework_cache_ready + if: ${{ !cancelled() && steps.restore_build_cache.outcome == 'success' && steps.restore_build_cache.outputs.cache-hit != 'true' }} + run: | + python3 - <<'PY' + import hashlib, os, plistlib, subprocess + from pathlib import Path + root = Path.cwd() + thirdparty = root / 'tmp/MiniCPM-V-Apps/MiniCPM-V-demo/thirdparty' + framework = thirdparty / 'llama.xcframework' + stamp = thirdparty / '.openglass-framework-key' + valid = False + if stamp.is_file() and (framework / 'Info.plist').is_file(): + payload = (root / 'ios/upstream-lock.json').read_bytes() + payload += subprocess.check_output(['xcodebuild', '-version']) + payload += subprocess.check_output(['uname', '-m']) + expected = hashlib.sha256(payload).hexdigest() + if stamp.read_text().strip() == expected: + info = plistlib.loads((framework / 'Info.plist').read_bytes()) + for library in info.get('AvailableLibraries', []): + if (library.get('SupportedPlatform') == 'ios' + and not library.get('SupportedPlatformVariant') + and 'arm64' in library.get('SupportedArchitectures', [])): + binary = framework / library['LibraryIdentifier'] / library['LibraryPath'] / 'llama' + if binary.is_file() and binary.resolve().is_relative_to(framework.resolve()): + valid = subprocess.run(['lipo', '-verify_arch', 'arm64', str(binary)]).returncode == 0 + break + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f"valid={'true' if valid else 'false'}\n") + print('Completed inference framework cacheable:', valid) + PY + - name: Save completed framework even if the app build failed + if: ${{ !cancelled() && steps.framework_cache_ready.outputs.valid == 'true' && steps.restore_build_cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: | + tmp/MiniCPM-V-Apps/MiniCPM-V-demo/thirdparty/llama.xcframework + tmp/MiniCPM-V-Apps/MiniCPM-V-demo/thirdparty/.openglass-framework-key + tmp/ios-build/SourcePackages + key: ${{ steps.restore_build_cache.outputs.cache-primary-key }} - name: Upload IPA for Windows signing uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: diff --git a/ios/README_zh.md b/ios/README_zh.md index 1f561e7..8db0d63 100644 --- a/ios/README_zh.md +++ b/ios/README_zh.md @@ -1,6 +1,6 @@ # Rokid → iPhone MiniCPM-V 4.6（Windows 开发） -本目录把现有 Rokid 采集协议接入 iPhone，在手机本机运行 MiniCPM-V 4.6。Windows 可以编辑代码、编译眼镜 APK、下载 iOS 构建和协助安装；iOS 编译交给 GitHub Actions 的 macOS/Xcode runner，安装前在 Windows 通过 AltStore Classic / AltServer 用自己的 Apple 账号重新签名。不要求自有 MacBook。 +本目录把现有 Rokid 采集协议接入 iPhone，在手机本机运行 MiniCPM-V 4.6。Windows 可以编辑代码、编译眼镜 APK、下载 iOS 构建和协助安装；iOS 编译交给 GitHub Actions 的 macOS/Xcode runner，安装前在 Windows 通过 AltServer 用自己的 Apple 账号重新签名。首次安装优先使用 AltServer 直接侧载 IPA，无需先在手机安装 AltStore。不要求自有 MacBook。 这是开发版本。源码、构建成功、签名安装成功、真实眼镜音视频到达、手机模型回答成功是不同的验证阶段；下方命令不会把“USB 已识别”当成端到端成功。 @@ -73,17 +73,27 @@ Python 命令未安装或打开 Microsoft Store 时，先安装真正的 Python -RunId 123456789 ``` -`RunId` 是 Actions 运行 URL 的数值 ID，不是 artifact 后缀的运行编号；示例值必须替换。脚本核验工作流、成功状态和未过期产物，输出源提交与 IPA SHA-256；目标目录存在时退出，避免覆盖旧产物。GitHub CLI 未登录时，由本人先运行 `gh auth login`。 +`RunId` 是 Actions 运行 URL 的数值 ID，不是 artifact 后缀的运行编号；示例值必须替换。脚本核验工作流、成功状态和未过期产物，再检查 `build-info.json` 的源提交与该运行一致、`signed=false`、bundle ID正确，以及 IPA中存在且可读取 `Payload/OpenGlass.app/Info.plist`。它不把IPA中的任意路径解压到磁盘，并输出源提交与 IPA SHA-256；目标目录存在时退出，避免覆盖旧产物。GitHub CLI 未登录时，由本人先运行 `gh auth login`。 ## 3. 用免费 Apple 账号签名安装 CI 产物不能直接作为已签名 App 安装。`ideviceinstaller`、`pymobiledevice3 apps install` 和 USB 端口转发都不会把 unsigned IPA 变成可运行的签名 App。 -按 [官方安装流程](https://faq.altstore.io/altstore-classic/how-to-install-altstore-windows)，从 Windows 托盘 AltServer 安装 AltStore Classic 到已解锁的 iPhone，并本人完成账号验证。再把 `OpenGlass-1.0.ipa` 保存到 iPhone 的“文件”App，在 AltStore 的 **My Apps → +** 选择该 IPA，保持 AltServer 可连接，完成签名安装。使用自己选择的文件传输方式；无需把模型权重或私人数据上传到公开链接。 +**首装推荐：从 Windows AltServer 直接侧载。** 如果手机已经有旧 OpenGlass 和自己编译的 MiniCPM，先装 AltStore 可能占用第三个免费侧载名额，使新 OpenGlass 成为第四个而无法安装。直接侧载无需增加 AltStore 这个手机 App： + +1. 保持 iPhone 通过 USB 连接、解锁并已信任电脑；确认 Apple Mobile Device Service 可用。Windows能看到iPhone图标或iTunes已安装，不等于该服务已准备好。 +2. 启动 Windows 的 **AltServer**，找到任务栏右下角托盘里的图标。 +3. **按住 Shift，再点击 AltServer 托盘图标**，选择 **Sideload .ipa…**，并选择目标 iPhone（界面有设备子菜单时）。 +4. 在 Windows 文件选择框选中构建下载得到的 **OpenGlass-1.0.ipa**。 +5. 本人在 AltServer 界面输入自己的 Apple 账号并完成验证，等待签名和安装结果；随后在 iPhone 按提示信任签名配置、确认开发者模式，再打开新 App。 + +这条 Windows 直接侧载入口及其无需预装 AltStore 的行为见 [AltServer 官方发布说明的 Direct .ipa Sideloading](https://faq.altstore.io/release-notes/altserver)。保留下载的 IPA；免费签名需要每 7 天用同一账号、同一 IPA 的上述步骤重新签名安装。此路径没有 AltStore 的自动刷新，不需要先把 IPA 转到手机“文件”App。 源 bundle ID 为 `org.opensqz.openglass.ios`；重签工具可能附加账号相关标识。显示名称是 **OpenGlass**。旧手机 App 若由另一账号/Team 签名，不能保证覆盖安装、共享模型文件或保留原数据；先保留旧 App，核对错误提示与签名来源，避免通过卸载来试错。 -免费账号需要定期刷新签名，通常 7 天到期，活跃侧载 App 上限为 3 个，AltStore 本身也占名额。使用 AltStore 管理续签；临近到期时连接 Windows 并刷新。官方限制见 [AltStore Classic 开发者说明](https://developer.altstore.io/)及[错误码](https://faq.altstore.io/altstore-classic/error-codes)。如果原 OpenGlass“点不开”，签名过期只是可能原因之一，还应区分未信任、开发者模式、闪退和模型内存问题。 +免费账号的活跃侧载 App 上限为 3 个，AltStore 本身也占名额；直接侧载只是省去 AltStore 的名额，并不会提高上限。若仍提示超限，先核对当前签名/活跃 App 清单，**不要卸载旧 OpenGlass 或 MiniCPM 来试错**。官方限制见 [AltStore Classic 开发者说明](https://developer.altstore.io/)及[错误码](https://faq.altstore.io/altstore-classic/error-codes)。如果原 OpenGlass“点不开”，签名过期只是可能原因之一，还应区分未信任、开发者模式、闪退和模型内存问题。 + +**后续可选：用 AltStore 管理自动刷新。** 确认有空余名额后，按 [官方安装流程](https://faq.altstore.io/altstore-classic/how-to-install-altstore-windows)从 AltServer 安装 AltStore Classic，再把 IPA 保存到 iPhone“文件”App，在 AltStore **My Apps → +** 中安装并管理。直接用 AltServer 安装的 App 不能假定会自动出现在 AltStore 的管理列表；要以 AltStore 里实际显示的 App 和到期时间为准。自动刷新仍依赖可达的 AltServer及系统后台条件，临近到期应主动检查。 ## 4. 在 iPhone 准备模型 @@ -92,7 +102,7 @@ CI 产物不能直接作为已签名 App 安装。`ideviceinstaller`、`pymobile 3. 回到主界面，点“启动接收”，等待模型就绪。初次验证可先关闭“Rokid 离线语音提问”，用文字问题排除语音权限影响。 4. 保持 OpenGlass 在前台。进入后台会停止接收和语音；返回后需要再次点“启动接收”。本实现不保证锁屏长期运行。 -模型文件不能混用 MiniCPM-o 4.5、其他尺寸或未经验证的 GGUF 转换。加载会检查文件存在与 GGUF 头；这不能代替完整模型匹配与真机推理验证。文件名由锁定上游的 MiniCPM-V 4.6 常量决定。 +模型文件不能混用 MiniCPM-o 4.5、其他尺寸或未经验证的 GGUF 转换。加载会检查文件存在、GGUF 头和锁定上游对应的 MD5；首次完整校验在后台进行，后续复用文件大小/修改时间匹配的校验记录。文件名与预期校验值由锁定上游的 MiniCPM-V 4.6 常量决定；模型通过校验仍需真机推理验证。 ## 5A. 双 USB 接入（先验证这条） @@ -165,12 +175,13 @@ iPhone个人热点、眼镜厂商固件对 Wi-Fi/蓝牙共存的影响要另行 | 有文本回答但没声音 | “朗读回答”开关、iPhone音量/音频路由、蓝牙配对；看界面实际输出设备 | | 语音不可用但文字正常 | 系统语音识别权限与该语言的设备端资源；该功能不向云识别降级 | | 切后台后断流 | 预期行为；回前台重启接收与语音 | -| 免费签名几天后点不开 | 先查AltStore剩余天数并刷新签名；不要先删除App及模型数据 | +| 免费签名几天后点不开 | 直接侧载方式每7天用AltServer重签；AltStore管理方式查看剩余天数并刷新。不要先删除App及模型数据 | ## 验证记录 - 本地脚本检查：PowerShell语法解析通过；USB中继使用伪设备测试了端口占用、已有reverse、并发映射冲突、接收端不可达、成功启动与只清理自身映射等情况。 -- 真机状态、CI构建链接、签名安装结果和端到端模型回答需分别记录；不能以以上测试替代真机验证。 +- [CI运行38889523899](https://github.com/OpenSQZ/OpenGlass/actions/runs/38889523899)成功，源提交为 `44d85adb28096af1989fb6f1d89873ca0a567e56`：1399项wire protocol检查、7项工程准备测试、7项USB中继测试通过，并生成未签名的 `OpenGlass-1.0.ipa`。 +- 现场设备识别为 `iPhone18,2`、iOS `26.6.2`，已完成Windows USB配对。新IPA尚未在手机上完成签名安装，也未完成“眼镜采集 → iPhone模型回答”的端到端验证；CI成功和USB配对不能替代这两步。 - 不在此文档写入个人设备序列号、UDID、Apple账号或内网凭据。现场更新请用设备类型和验证结果描述。 可重复执行无设备测试： diff --git a/ios/Sources/OpenGlassModelSession.swift b/ios/Sources/OpenGlassModelSession.swift index efa7c1b..42d382d 100644 --- a/ios/Sources/OpenGlassModelSession.swift +++ b/ios/Sources/OpenGlassModelSession.swift @@ -53,12 +53,30 @@ final class OpenGlassModelSession { let files = Self.modelFiles.map { documents.appendingPathComponent($0) } for file in files { guard FileManager.default.fileExists(atPath: file.path) else { throw SessionError.missingFiles } - // Check only the GGUF header, never hash multi-GB files on the UI thread. + // Reject obvious truncation before doing full artifact validation. let handle = try FileHandle(forReadingFrom: file) let header = try handle.read(upToCount: 4) try handle.close() guard header == Data("GGUF".utf8) else { throw SessionError.invalidFile(file.lastPathComponent) } } + // The upstream download UI reports completion before its background + // checksum finishes. A GGUF header/filename alone cannot prove that + // these are the matching 4.6 LLM and projector. Reuse its cached + // validation receipts, and keep first-time hashing off the UI actor. + let checksums = [MiniCPMModelConst.modelv46_Q4_K_M_MD5, + MiniCPMModelConst.modelMMProjv46_MD5] + transition(.loading, "正在校验 MiniCPM-V 4.6 模型文件…") + let invalidFile = await Task.detached(priority: .userInitiated) { + for (file, checksum) in zip(files, checksums) { + if !MBModelArtifactStore.validate(file, expectedMD5: checksum) { + return Optional(file.lastPathComponent) + } + } + return Optional.none + }.value + if let invalidFile { throw SessionError.invalidFile(invalidFile) } + try checkStopped() + transition(.loading, "正在加载 MiniCPM-V 4.6…") let engine = MTMDWrapper() wrapper = engine engine.setEnableThinking(false) diff --git a/ios/Sources/OpenGlassViewController.swift b/ios/Sources/OpenGlassViewController.swift index d99e01f..eb784d8 100644 --- a/ios/Sources/OpenGlassViewController.swift +++ b/ios/Sources/OpenGlassViewController.swift @@ -7,11 +7,15 @@ import Darwin final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSpeechSynthesizerDelegate { private let server = RokidIngressServer() private let model = OpenGlassModelSession() + // The upstream downloader requires its ObservableObject for file/progress + // bookkeeping. Never call initialize() on this download-only wrapper. + private lazy var downloadWrapper = MTMDWrapperExample() private let speech = OpenGlassSpeechSession() private let synthesizer = AVSpeechSynthesizer() private var activeUtterance: AVSpeechUtterance? private var isRunning = false private var isStarting = false + private var isAuthorizingSpeech = false private var isAnswering = false private var audioConnected = false private var speechPaused = false @@ -188,7 +192,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp note.numberOfLines = 0 note.font = .preferredFont(forTextStyle: .footnote) note.textColor = .secondaryLabel - note.text = "Rokid 可通过同一局域网或 Windows 双 USB 中继连接。首次使用请先下载模型。此 App 的模型文件与原 TestFlight App 分开保存。进入后台会停止接收和语音，回到前台后请重新启动。" + note.text = "Rokid 可通过同一局域网或 Windows 双 USB 中继连接。首次使用请先下载模型。此 App 的模型文件与原 MiniCPM 示例分开保存。进入后台会停止接收和语音，回到前台后请重新启动。" stack.addArrangedSubview(note) } @@ -302,7 +306,6 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp do { try await model.load() guard isRunning, runID == id else { updateControls(); return } - isStarting = false statusLabel.text = "等待 Rokid 画面和音频；也可以文字提问。" if voiceSwitch.isOn { let authorized = await OpenGlassSpeechSession.authorize() @@ -312,6 +315,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp statusLabel.text = "未获语音识别权限。可用文字提问，或在系统设置中允许。" } } + isStarting = false startListeningIfPossible() } catch { if runID == id { stopService(message: error.localizedDescription) } @@ -324,6 +328,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp private func stopService(message: String) { isRunning = false isStarting = false + isAuthorizingSpeech = false speechPaused = true suppressAnswerSpeech = true runID = UUID() @@ -425,7 +430,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp private var localeIdentifier: String { languageControl.selectedSegmentIndex == 1 ? "en-US" : "zh-CN" } private func startListeningIfPossible() { - guard isRunning, !isStarting, !isAnswering, !model.isBusy, model.isLoaded, + guard isRunning, !isStarting, !isAuthorizingSpeech, !isAnswering, !model.isBusy, model.isLoaded, !model.requiresAppRestart, voiceSwitch.isOn, !speechPaused, audioConnected, activeUtterance == nil, !speech.isActive else { return } do { @@ -440,14 +445,16 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp @objc private func toggleListening() { if speech.isListening { speech.finishUtterance(); updateControls(); return } if speech.isFinishing { return } - guard isRunning, !model.isBusy, !isAnswering else { return } + guard isRunning, !model.isBusy, !isAnswering, !isAuthorizingSpeech else { return } voiceSwitch.isOn = true UserDefaults.standard.set(true, forKey: "OpenGlass.voiceEnabled") speechPaused = false + isAuthorizingSpeech = true let id = runID Task { let authorized = await OpenGlassSpeechSession.authorize() guard isRunning, runID == id else { return } + isAuthorizingSpeech = false if authorized { if !audioConnected { statusLabel.text = "等待 Rokid 音频连接…" } startListeningIfPossible() @@ -589,7 +596,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp askButton.isEnabled = isRunning && !isStarting && model.isLoaded && !busy && !model.requiresAppRestart listenButton.configuration?.title = speech.isFinishing ? "正在结束语句…" : (speech.isListening ? "结束语句并提问" : "开始语音") listenButton.isEnabled = isRunning && !isStarting && !busy && activeUtterance == nil - && !model.requiresAppRestart && !speech.isFinishing + && !model.requiresAppRestart && !speech.isFinishing && !isAuthorizingSpeech stopTurnButton.isEnabled = isRunning || busy || activeUtterance != nil downloadItem.isEnabled = !busy && !model.requiresAppRestart languageControl.isEnabled = !busy && activeUtterance == nil @@ -604,7 +611,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp Task { do { try await model.unload() - navigationController?.pushViewController(MBV46ModelDetailViewController(), animated: true) + navigationController?.pushViewController(MBV46ModelDetailViewController(with: downloadWrapper), animated: true) } catch { statusLabel.text = error.localizedDescription } diff --git a/ios/windows/download-artifact.ps1 b/ios/windows/download-artifact.ps1 index cffaa2d..ebd8c79 100644 --- a/ios/windows/download-artifact.ps1 +++ b/ios/windows/download-artifact.ps1 @@ -31,8 +31,31 @@ New-Item -ItemType Directory -Path $fullDestination | Out-Null if ($LASTEXITCODE -ne 0) { throw "Artifact download failed. Partial output remains for inspection: $fullDestination" } $ipa = Join-Path $fullDestination 'OpenGlass-1.0.ipa' if (-not (Test-Path -LiteralPath $ipa -PathType Leaf)) { throw "The artifact is missing OpenGlass-1.0.ipa: $fullDestination" } +$manifestPath = Join-Path $fullDestination 'build-info.json' +if (-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) { throw 'The artifact is missing build-info.json.' } +$manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json +foreach ($field in @('commit', 'signed', 'bundle_identifier')) { + if ($manifest.PSObject.Properties.Name -notcontains $field) { throw "build-info.json is missing $field." } +} +if ($manifest.commit -cne $run.head_sha) { throw 'Artifact manifest commit does not match the selected Actions run.' } +if ($manifest.signed -isnot [bool] -or $manifest.signed) { throw 'Expected the manifest to declare signed=false (JSON boolean).' } +if ($manifest.bundle_identifier -cne 'org.opensqz.openglass.ios') { throw 'Artifact manifest has an unexpected iOS bundle identifier.' } +Add-Type -AssemblyName System.IO.Compression.FileSystem +$archive = [System.IO.Compression.ZipFile]::OpenRead($ipa) +try { + $infoEntries = @($archive.Entries | Where-Object { $_.FullName -ceq 'Payload/OpenGlass.app/Info.plist' }) + if ($infoEntries.Count -ne 1 -or $infoEntries[0].Length -le 0 -or $infoEntries[0].Length -gt 1MB) { + throw 'IPA must contain one non-empty Payload/OpenGlass.app/Info.plist of reasonable size.' + } + # Read the compressed entry without extracting any archive path to disk. + $infoStream = $infoEntries[0].Open() + try { $infoStream.CopyTo([System.IO.Stream]::Null) } + finally { $infoStream.Dispose() } +} finally { $archive.Dispose() } Write-Host "Downloaded unsigned IPA: $ipa" +Write-Host 'Verified manifest commit, unsigned status, bundle identifier, and readable IPA Info.plist entry.' Write-Host "Run: $($run.html_url)" Write-Host "Commit: $($run.head_sha)" Get-FileHash -LiteralPath $ipa -Algorithm SHA256 | Format-List -Write-Host 'Next: sign/install with AltStore and AltServer on Windows. Downloading an unsigned IPA does not install or authorize it.' +Write-Host 'Next: hold Shift while clicking the Windows AltServer tray icon, choose Sideload .ipa, and sign/install with your own Apple account in its UI.' +Write-Host 'AltStore on iPhone is optional and uses a free-account app slot. Direct AltServer installs require manual re-signing every 7 days.' From 184c0fcde38b30f180f9cc363d61d1b1ceacf70e Mon Sep 17 00:00:00 2001 From: MengzhangLI Date: Fri, 16 Oct 2026 20:34:55 +0800 Subject: [PATCH 04/11] docs: record verified local Windows signing and installation --- ios/README_zh.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/ios/README_zh.md b/ios/README_zh.md index 8db0d63..a3589bd 100644 --- a/ios/README_zh.md +++ b/ios/README_zh.md @@ -1,6 +1,6 @@ # Rokid → iPhone MiniCPM-V 4.6（Windows 开发） -本目录把现有 Rokid 采集协议接入 iPhone，在手机本机运行 MiniCPM-V 4.6。Windows 可以编辑代码、编译眼镜 APK、下载 iOS 构建和协助安装；iOS 编译交给 GitHub Actions 的 macOS/Xcode runner，安装前在 Windows 通过 AltServer 用自己的 Apple 账号重新签名。首次安装优先使用 AltServer 直接侧载 IPA，无需先在手机安装 AltStore。不要求自有 MacBook。 +本目录把现有 Rokid 采集协议接入 iPhone，在手机本机运行 MiniCPM-V 4.6。Windows 可以编辑代码、编译眼镜 APK、下载 iOS 构建和协助安装；iOS 编译交给 GitHub Actions 的 macOS/Xcode runner，安装前在 Windows 用自己的 Apple 账号重新签名。不要求自有 MacBook。签名工具的常规入口、当前登录问题与本机认证测试版见第 3 节。 这是开发版本。源码、构建成功、签名安装成功、真实眼镜音视频到达、手机模型回答成功是不同的验证阶段；下方命令不会把“USB 已识别”当成端到端成功。 @@ -35,7 +35,7 @@ Rokid → HTTP JPEG + WebSocket PCM → iPhone 局域网地址:18080 3. 按 [AltStore 官方 Windows 安装说明](https://faq.altstore.io/altstore-classic/how-to-install-altstore-windows)安装 AltServer 及所要求的 Apple iTunes / iCloud 组件。官方默认流程使用 Apple 直接分发的版本；已安装 Store 版时，先参照官方故障排查，不要随意删除 Apple 驱动或服务。 4. 在 iPhone 的“隐私与安全性”开启“开发者模式”，按系统要求重启并确认。签名配置的信任按系统提示在“通用 → VPN 与设备管理”中处理。 -Apple 账号、密码及验证码只由本人输入 AltServer / AltStore 正规界面，不放入脚本、GitHub Secrets、日志或仓库。这里的自动化不读取 Apple 账号、配对记录或签名证书。 +Apple 账号、密码及验证码只由本人输入所选签名工具的界面，不放入脚本、GitHub Secrets、日志或仓库。安装 iTunes / iCloud 组件与登录它们的云服务是两件事；本机认证测试版使用已安装的 Apple 组件，不要求分别登录 iCloud 和 iTunes。 在仓库根目录创建独立 Python 环境： @@ -79,7 +79,7 @@ Python 命令未安装或打开 Microsoft Store 时，先安装真正的 Python CI 产物不能直接作为已签名 App 安装。`ideviceinstaller`、`pymobiledevice3 apps install` 和 USB 端口转发都不会把 unsigned IPA 变成可运行的签名 App。 -**首装推荐：从 Windows AltServer 直接侧载。** 如果手机已经有旧 OpenGlass 和自己编译的 MiniCPM，先装 AltStore 可能占用第三个免费侧载名额，使新 OpenGlass 成为第四个而无法安装。直接侧载无需增加 AltStore 这个手机 App： +**常规入口：从 Windows AltServer 直接侧载。** 本次现场的 AltServer 1.8 在验证码阶段出现 `-22411`，因此以下入口尚未在本机成功完成安装；不要把它视为已验证的解决办法。手机已经有旧 OpenGlass 和自己编译的 MiniCPM 时，先装 AltStore 可能占用第三个免费侧载名额，使新 OpenGlass 成为第四个而无法安装。直接侧载无需增加 AltStore 这个手机 App： 1. 保持 iPhone 通过 USB 连接、解锁并已信任电脑；确认 Apple Mobile Device Service 可用。Windows能看到iPhone图标或iTunes已安装，不等于该服务已准备好。 2. 启动 Windows 的 **AltServer**，找到任务栏右下角托盘里的图标。 @@ -95,6 +95,17 @@ CI 产物不能直接作为已签名 App 安装。`ideviceinstaller`、`pymobile **后续可选：用 AltStore 管理自动刷新。** 确认有空余名额后，按 [官方安装流程](https://faq.altstore.io/altstore-classic/how-to-install-altstore-windows)从 AltServer 安装 AltStore Classic，再把 IPA 保存到 iPhone“文件”App，在 AltStore **My Apps → +** 中安装并管理。直接用 AltServer 安装的 App 不能假定会自动出现在 AltStore 的管理列表；要以 AltStore 里实际显示的 App 和到期时间为准。自动刷新仍依赖可达的 AltServer及系统后台条件，临近到期应主动检查。 +**本次本地交付：OpenGlass Local Signer 测试版。** Sideloadly 0.70.1 使用 Local Anisette 后可以输入验证码，但本机仍收到 GSA HTTP 503。这个错误本身不能证明密码错误或 TLS 被拦截。为继续只使用本地认证组件，准备了基于 [iloader v2.3.9](https://github.com/nab138/iloader/tree/aa9aa3779e15498180e0205158b7ff91536877c6) 和 isideload 0.4.6 的修改版，保存于本机 `%USERPROFILE%\Downloads\OpenGlass-Local-Signer`。它不是官方 iloader 发布；目录内源码包及 `BUILD_WINDOWS.md` 提供构建输入和说明。 + +主程序旁必须保留 `local-apple-identity.exe` 与 `local-coreadi-helper.exe`。它们核验已安装 Apple DLL 的固定 SHA256 和签名，通过本机管道取得原生身份及认证数据；没有远程 Anisette 备用服务，不自动擦除或重新注册原生身份。当前构建锁定本机 Apple 组件版本，不保证适用于其他版本的电脑。登录、验证码验证、申请签名证书和描述文件仍需要联网到 Apple；“本地认证”不等于离线签名。 + +1. 运行 `OpenGlass-Local-Signer.exe`，本人在 Apple ID 区域登录并填写验证码；保存密码默认不勾选。 +2. 选择 USB 连接的 iPhone，点击 **Import IPA**，选择本次构建的 IPA；选中文件后即开始签名安装。 +3. 若提示开发证书已满，先取消并确认哪些旧 App 依赖已有证书；程序不会默认选择全部撤销。 +4. 用手机安装列表核对新 App 的实际 bundle ID、版本和文件共享状态，再导入模型。仅有“账号登录成功”或 Windows 程序已打开不代表 iPhone 安装完成。 + +测试版已通过 6 项管道协议测试、前后端构建、本机组件调用，以及携带公共客户端描述的 Apple URL-bag GET / TLS 校验。2026-10-16，本人操作后 Apple 账号登录和签名成功；首次安装因已有 3 个免费开发 App 被手机拒绝。用户明确选择卸载旧 OpenGlass 后，复用已签名文件安装新版成功，USB 查询确认版本 `1.0`、build `3`、文件共享可用，MiniCPM 示例与 CyanGlass 均保留。签名工具会给新 bundle ID 追加 Team ID；只改 IPA 的原始 bundle ID 不能假定能覆盖旧 Xcode App。自动更新已关闭，避免被上游的不同认证配置替换。 + ## 4. 在 iPhone 准备模型 1. 打开新 **OpenGlass**，进入右上角“模型下载”。 @@ -180,8 +191,8 @@ iPhone个人热点、眼镜厂商固件对 Wi-Fi/蓝牙共存的影响要另行 ## 验证记录 - 本地脚本检查：PowerShell语法解析通过；USB中继使用伪设备测试了端口占用、已有reverse、并发映射冲突、接收端不可达、成功启动与只清理自身映射等情况。 -- [CI运行38889523899](https://github.com/OpenSQZ/OpenGlass/actions/runs/38889523899)成功，源提交为 `44d85adb28096af1989fb6f1d89873ca0a567e56`：1399项wire protocol检查、7项工程准备测试、7项USB中继测试通过，并生成未签名的 `OpenGlass-1.0.ipa`。 -- 现场设备识别为 `iPhone18,2`、iOS `26.6.2`，已完成Windows USB配对。新IPA尚未在手机上完成签名安装，也未完成“眼镜采集 → iPhone模型回答”的端到端验证；CI成功和USB配对不能替代这两步。 +- [CI运行38890873970](https://github.com/OpenSQZ/OpenGlass/actions/runs/38890873970)成功，源提交为 `c11c87b39b5798f8b4218d3a47d5b6cc68e56091`：1399项wire protocol检查、7项工程准备测试、7项USB中继测试通过，并生成未签名的 `OpenGlass-1.0.ipa`（artifact `OpenGlass-iOS-unsigned-3`）。IPA SHA256 为 `DA76A28A87195C43EA80ECF11366724180480B5D6502DEA27A4D1AF800AD00DB`。 +- 现场设备识别为 `iPhone18,2`、iOS `26.6.2`，已完成 Windows USB 配对。新版 `1.0` / build `3` 已通过本机认证签名并安装；系统返回安装成功，安装列表再次确认。两个对应 4.6 的模型已校验源文件 MD5 并通过 USB 导入，手机端文件大小一致；首次加载仍由 App 完整校验。眼镜图像、ASR、模型回答与 TTS 的端到端结果仍待真机验证。 - 不在此文档写入个人设备序列号、UDID、Apple账号或内网凭据。现场更新请用设备类型和验证结果描述。 可重复执行无设备测试： From 087766314c0d817a78b65c6e6a93d46b281d5576 Mon Sep 17 00:00:00 2001 From: MengzhangLI Date: Sun, 18 Oct 2026 02:23:26 +0800 Subject: [PATCH 05/11] Add automatic iPhone conversation loop and Wi-Fi discovery --- .github/workflows/ios-ipa.yml | 2 + ios/README_zh.md | 58 +++- ios/Sources/OpenGlassSpeechSession.swift | 85 ++++-- ios/Sources/OpenGlassViewController.swift | 330 ++++++++++++++++----- ios/Sources/OpenGlassVoiceLoopPolicy.swift | 79 +++++ ios/Sources/RokidIngressServer.swift | 32 +- ios/scripts/prepare_upstream.py | 1 + ios/tests/run_voice_loop_tests.sh | 7 + ios/tests/test_prepare_upstream.py | 1 + ios/tests/voice_loop/main.swift | 200 +++++++++++++ ios/windows/common.ps1 | 4 +- ios/windows/configure-rokid.ps1 | 20 +- 12 files changed, 698 insertions(+), 121 deletions(-) create mode 100644 ios/Sources/OpenGlassVoiceLoopPolicy.swift create mode 100644 ios/tests/run_voice_loop_tests.sh create mode 100644 ios/tests/voice_loop/main.swift diff --git a/.github/workflows/ios-ipa.yml b/.github/workflows/ios-ipa.yml index d3b867a..e505431 100644 --- a/.github/workflows/ios-ipa.yml +++ b/.github/workflows/ios-ipa.yml @@ -36,6 +36,8 @@ jobs: run: python3 -m unittest discover -s ios/tests -p 'test_prepare*.py' -v - name: Test Rokid wire protocol run: bash ios/tests/run_protocol_tests.sh + - name: Test automatic voice conversation lifecycle + run: bash ios/tests/run_voice_loop_tests.sh - name: Test Windows USB relay without devices run: python3 -m unittest discover -s ios/windows -p 'test_usb_relay.py' -v - name: Prepare pinned upstream and OpenGlass sources diff --git a/ios/README_zh.md b/ios/README_zh.md index a3589bd..37192dd 100644 --- a/ios/README_zh.md +++ b/ios/README_zh.md @@ -6,7 +6,17 @@ ## 运行路径 -先使用双 USB 路径，眼镜 Wi-Fi 可以保持关闭： +无线版本使用 Rokid 采集端 **0.2.0**。眼镜加入 iPhone 个人热点（推荐），或与 iPhone 加入同一可互通的 Wi-Fi： + +```text +Rokid 摄像头 / 麦克风 → Wi-Fi → iPhone OpenGlass :18080 + → iPhone 离线 ASR → MiniCPM-V 4.6 → iPhone TTS + → 自动恢复听音，开始下一轮（每轮清空 KV cache） +``` + +这条路径不需要 Windows 转发；两端连接正常后可拔掉 USB。iPhone App 需要保持前台。首次配置热点 / Wi-Fi、权限和安装更新仍需操作设备。 + +双 USB 路径保留为回退方案，眼镜 Wi-Fi 可以保持关闭： ```text Rokid 摄像头 / 麦克风 @@ -24,7 +34,7 @@ Windows 在这个路径只转发数据，不运行视觉模型。拔掉任一 US Rokid → HTTP JPEG + WebSocket PCM → iPhone 局域网地址:18080 ``` -两条路径都要求眼镜本身支持 Android APK、Camera/Mic 权限和 ADB。不能仅根据“Rokid”品牌判断所有型号兼容。原采集 APK 无需改协议；当前采样是 JPEG 约 1 fps、16 kHz 单声道 PCM16LE。模型按问题读取最新画面，不是对每个视频帧连续推理。 +眼镜本身需要支持 Android APK 和 Camera/Mic 权限；ADB 用于 Windows 安装与可选的首次配置，无线运行不需要 ADB。不能仅根据“Rokid”品牌判断所有型号兼容。0.2.0 延续 JPEG / PCM 协议，新增自动寻找手机和统一的断线重连；当前采样是 JPEG 约 1 fps、16 kHz 单声道 PCM16LE。模型按问题读取最新画面，不是对每个视频帧连续推理。 ## 1. 准备 Windows 与手机 @@ -110,8 +120,8 @@ CI 产物不能直接作为已签名 App 安装。`ideviceinstaller`、`pymobile 1. 打开新 **OpenGlass**，进入右上角“模型下载”。 2. 在 MiniCPM-V 4.6 下载页完成 **LLM 与 VPM 两个 GGUF 文件**。这个 App 有独立沙盒；其他 TestFlight/App 已下载的模型不会自动出现在这里。下载与首次资源准备需要网络。 -3. 回到主界面，点“启动接收”，等待模型就绪。初次验证可先关闭“Rokid 离线语音提问”，用文字问题排除语音权限影响。 -4. 保持 OpenGlass 在前台。进入后台会停止接收和语音；返回后需要再次点“启动接收”。本实现不保证锁屏长期运行。 +3. 模型已准备好时，打开 App 会自动启动接收并加载模型。首次刚从下载页返回时点一次“启动接收”。初次验证可先关闭“Rokid 离线语音提问”，用文字问题排除语音权限影响。 +4. 保持 OpenGlass 在前台。进入后台会挂起接收和语音；返回后自动恢复此前运行状态。主动点“停止接收”或“暂停语音”后不会自动重启。本实现不保证锁屏长期运行。 模型文件不能混用 MiniCPM-o 4.5、其他尺寸或未经验证的 GGUF 转换。加载会检查文件存在、GGUF 头和锁定上游对应的 MD5；首次完整校验在后台进行，后续复用文件大小/修改时间匹配的校验记录。文件名与预期校验值由锁定上游的 MiniCPM-V 4.6 常量决定；模型通过校验仍需真机推理验证。 @@ -125,6 +135,14 @@ CI 产物不能直接作为已签名 App 安装。`ideviceinstaller`、`pymobile -Python .\ios\windows\.venv\Scripts\python.exe ``` +Windows PowerShell 若提示“在此系统上禁止运行脚本 / PSSecurityException”，可仅为这次启动允许本地脚本，使用下面的命令；把路径替换为实际仓库绝对路径。`-ExecutionPolicy RemoteSigned` 只影响新进程，不永久修改用户或系统设置，也不会覆盖组织的组策略： + +```powershell +powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File 'C:\path\to\OpenGlass\ios\windows\start-usb-relay.ps1' +``` + +本项目生成或通过 Git 检出的本地脚本可以使用这条命令。若仍报执行策略错误，先检查 `Get-ExecutionPolicy -List` 和文件的来源标记，不全局降低策略。作用范围见 [Microsoft 官方执行策略说明](https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_execution_policies?view=powershell-5.1)。 + 仅连接一副授权眼镜和一台 iPhone 时自动选择；多设备时显式加 `-Serial '<眼镜USB序列号>'`、`-IPhoneUdid ''`。用下列只读命令查询 iPhone USB UDID： ```powershell @@ -152,9 +170,23 @@ Invoke-RestMethod http://127.0.0.1:18080/health 底层使用官方 [pymobiledevice3 v11.32.0 TCP forwarder](https://github.com/doronz88/pymobiledevice3/blob/v11.32.0/pymobiledevice3/tcp_forwarder.py)，限定 USB。其 CLI 对应 `usbmux forward `，这里的包装脚本另外处理设备选择、健康检查和清理。普通 App TCP 转发不使用 iOS 开发调试隧道，不要求将 Apple 配对文件上传到任何服务。 -## 5B. Wi-Fi 直连（USB 路径通过后） +## 5B. Wi-Fi 直连（推荐 iPhone 个人热点） + +先安装新 iOS IPA 和 Rokid 0.2.0 APK。iOS 使用同一 Apple 账号、同一 bundle ID 覆盖安装，保留现有模型；无需卸载或重新下载模型。 + +1. iPhone“设置 → 个人热点 → 允许其他人加入”。眼镜找不到热点时可开启“最大兼容性”。首次连接时停留在热点页面，直到眼镜已加入。具体开关见 [Apple 个人热点说明](https://support.apple.com/zh-cn/111785)。热点依赖运营商支持；它分享的是蜂窝网络，不是酒店 Wi-Fi 的中继。 +2. 通过 Rokid 官方 App / 设备的 Wi-Fi 设置，让眼镜加入这个热点；密码仅在设备设置内输入。OpenGlass 不修改 Wi-Fi 密码和系统网络。 +3. iPhone 打开 OpenGlass，允许“本地网络”和语音识别；已有模型时自动启动接收，等待模型就绪即可。页面显示固定的 `OpenGlass-xxxxxxxx` 接收名称。请保持“Rokid 离线语音提问”和“朗读回答”开启。 +4. 启动眼镜 OpenGlassRokidSensor，使用“自动连接”。眼镜寻找 `_openglass._tcp` 服务，并探测当前 Wi-Fi 网关的 `:18080/health`；确认是 OpenGlass iOS 接收端后才发送音视频。地址变化后会重新发现，不把热点地址写死。 +5. 确认手机出现新画面、自动语音状态后，拔掉两根 USB，等待十秒并连续问三个问题。回答播完后再说下一个问题，不需要点“结束语句”或“用最新画面提问”。 + +眼镜暂留 USB 时，可在 Windows 手动切到自动发现模式；这条命令只做一次配置与启动，不是数据中继： -在官方 Rokid AI App/设备设置中先把眼镜和 iPhone 放到同一可互通 Wi-Fi。确认网络没有客户端隔离。在 iPhone OpenGlass 点“刷新地址”，取其显示的局域网 IPv4 地址。眼镜暂时保留 USB，仅用于设置采集地址： +```powershell +powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File 'C:\path\to\OpenGlass\ios\windows\configure-rokid.ps1' -AutoDiscover +``` + +普通 Wi-Fi / 酒店 Wi-Fi 也可用：将两台设备加入同一网络。网络必须允许设备互访；若酒店启用了客户端隔离，或眼镜无法完成网页认证，应改用个人热点。Bonjour 被阻止但设备仍能直连时，可从 iPhone 页面取 IPv4 地址手动指定： ```powershell .\ios\windows\configure-rokid.ps1 ` @@ -162,17 +194,19 @@ Invoke-RestMethod http://127.0.0.1:18080/health -Adb "$env:USERPROFILE\Downloads\platform-tools\adb.exe" ``` -替换示例 IP。此脚本不替你打开/修改眼镜 Wi-Fi，只检测手机接收服务并重启眼镜采集 App。确认画面/音频计数增长后再拔掉眼镜 USB。此时即使 Windows 关闭，中间没有 Windows 转发；iPhone 仍需前台运行。 +替换示例 IP。手动地址模式用于明确指定接收设备，换网后可重新选“自动连接”。脚本不打开/修改眼镜 Wi-Fi。若 Windows 未加入手机所在网络而眼镜已加入，Windows 的接收端检查可能失败；可使用自动发现，或确认原因后加 `-SkipReceiverCheck`。 -iPhone个人热点、眼镜厂商固件对 Wi-Fi/蓝牙共存的影响要另行真机验证，不能将同一 Wi-Fi 成功等同于热点成功。换网或手机 IP 改变后重跑地址配置。若 Windows 暂时不能访问手机但眼镜实际可达，只有明确理解原因时才用 `-SkipReceiverCheck`；它会跳过启动前的手机健康检查。 +无线连通性须在这副眼镜和这台 iPhone 上实测，不能把“已编译”当成“热点已跑通”。检查时必须拔掉 USB，并确认新画面仍在更新，防止误把原 USB 链路当成无线成功。 ## 6. 验证对话和音频 先确认预览方向正确；主界面可选0/90/180/270°旋转，默认270°沿用原电脑路径。输入“我面前有什么？”后点“用最新画面提问”，验证手机能读到当前眼镜画面并生成回答。 -然后启用“Rokid 离线语音提问”，选择中文或英文，允许系统语音识别权限。输入仍来自眼镜的 PCM 流，App不打开 iPhone 麦克风。Apple 设备端识别资源可用时执行离线识别；不可用会显示错误并保留文字提问，不偷偷改走云端。说完停顿或点“结束语句”后，识别文本与当前图片送给 MiniCPM-V 4.6。回答由系统 TTS 朗读；可在 iPhone 控制中心选择已配对的眼镜作为声音输出，前提是该型号支持相应蓝牙音频功能。 +然后启用“Rokid 离线语音提问”，选择中文或英文，允许系统语音识别权限。输入仍来自眼镜的 PCM 流，App 不打开 iPhone 麦克风。Apple 设备端识别资源可用时执行离线识别；不可用会显示错误并保留文字提问，不改走云端。说完停顿后，识别文本与当前图片自动送给 MiniCPM-V 4.6。回答由系统 TTS 朗读，结束后自动恢复听音；每轮推理前清空 KV cache，不保留此前问题或图片。可在 iPhone 控制中心选择已配对的眼镜作为声音输出，前提是该型号支持相应蓝牙音频功能。 + +自动循环为“听问题 → 看图推理 → 朗读回答 → 等待回声消退 → 继续听问题”。推理/播报时丢弃输入语音，不积压旧问题；空白识别、短暂识别错误、音频重连与新画面恢复均由状态机处理。权限拒绝、离线语言资源缺失、模型错误仍会明确显示，主动暂停必须由用户恢复。这是逐轮交互，暂不支持边播报边打断。 -当前是“听问题 → 看图推理 → 朗读回答”的分轮交互；推理/播报时暂停送入识别的麦克风片段，避免回答被自己重新听见。它不提供 MiniCPM-o 的原生全双工音频模型，也不承诺打断播报时继续实时看图。 +`/health` 的 `conversation_phase` 显示 `waiting/listening/thinking/speaking/paused`，`completed_turns` 记录本次接收会话中完成的回答数；不返回识别文本或回答正文。测试至少三轮，并留空静音约30秒后再提问，确认没有空问题、重复回答或需要手动唤醒。 ## 排障 @@ -185,14 +219,14 @@ iPhone个人热点、眼镜厂商固件对 Wi-Fi/蓝牙共存的影响要另行 | 画面到达但模型未就绪 | 两个正确的4.6模型是否下载完成、磁盘空间、模型加载报错；其他App模型沙盒不共享 | | 有文本回答但没声音 | “朗读回答”开关、iPhone音量/音频路由、蓝牙配对；看界面实际输出设备 | | 语音不可用但文字正常 | 系统语音识别权限与该语言的设备端资源；该功能不向云识别降级 | -| 切后台后断流 | 预期行为；回前台重启接收与语音 | +| 切后台后断流 | 预期行为；回前台自动恢复此前运行意图；主动停止不会自动恢复 | | 免费签名几天后点不开 | 直接侧载方式每7天用AltServer重签；AltStore管理方式查看剩余天数并刷新。不要先删除App及模型数据 | ## 验证记录 - 本地脚本检查：PowerShell语法解析通过；USB中继使用伪设备测试了端口占用、已有reverse、并发映射冲突、接收端不可达、成功启动与只清理自身映射等情况。 - [CI运行38890873970](https://github.com/OpenSQZ/OpenGlass/actions/runs/38890873970)成功，源提交为 `c11c87b39b5798f8b4218d3a47d5b6cc68e56091`：1399项wire protocol检查、7项工程准备测试、7项USB中继测试通过，并生成未签名的 `OpenGlass-1.0.ipa`（artifact `OpenGlass-iOS-unsigned-3`）。IPA SHA256 为 `DA76A28A87195C43EA80ECF11366724180480B5D6502DEA27A4D1AF800AD00DB`。 -- 现场设备识别为 `iPhone18,2`、iOS `26.6.2`，已完成 Windows USB 配对。新版 `1.0` / build `3` 已通过本机认证签名并安装；系统返回安装成功，安装列表再次确认。两个对应 4.6 的模型已校验源文件 MD5 并通过 USB 导入，手机端文件大小一致；首次加载仍由 App 完整校验。眼镜图像、ASR、模型回答与 TTS 的端到端结果仍待真机验证。 +- 现场设备识别为 `iPhone18,2`、iOS `26.6.2`。新版 `1.0` / build `3` 已通过本机认证签名并安装；两个对应 4.6 模型已校验并导入。2026-10-18 用户反馈：USB 连接下眼镜图像、声音和 iPhone MiniCPM 推理已跑通，仍需手动进行单轮交互。本次无线与自动多轮改动另行验证，不能继承旧版的端到端结论。 - 不在此文档写入个人设备序列号、UDID、Apple账号或内网凭据。现场更新请用设备类型和验证结果描述。 可重复执行无设备测试： diff --git a/ios/Sources/OpenGlassSpeechSession.swift b/ios/Sources/OpenGlassSpeechSession.swift index 6a2e42d..92390db 100644 --- a/ios/Sources/OpenGlassSpeechSession.swift +++ b/ios/Sources/OpenGlassSpeechSession.swift @@ -8,7 +8,7 @@ import AVFoundation final class OpenGlassSpeechSession { var onPartial: ((String) -> Void)? var onFinal: ((String) -> Void)? - var onError: ((String) -> Void)? + var onError: ((Failure) -> Void)? var onStatus: ((String) -> Void)? private(set) var isListening = false var isActive: Bool { request != nil } @@ -20,9 +20,10 @@ final class OpenGlassSpeechSession { private var finishTimer: Timer? private var sessionID = UUID() private var latestText = "" - private var startedAt = Date() - private var lastVoiceAt: Date? - private var lastPacketAt = Date() + private var startedAt = ProcessInfo.processInfo.systemUptime + private var lastVoiceAt: TimeInterval? + private var lastTranscriptAt: TimeInterval? + private var lastPacketAt = ProcessInfo.processInfo.systemUptime private var pendingByte: UInt8? private var ending = false private let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, @@ -40,14 +41,14 @@ final class OpenGlassSpeechSession { func start(localeIdentifier: String) throws { stop() guard SFSpeechRecognizer.authorizationStatus() == .authorized else { - throw SpeechError.message("未获语音识别权限。可在系统设置中允许，或关闭语音、使用文字提问。") + throw Failure.permissionDenied } guard let recognizer = SFSpeechRecognizer(locale: Locale(identifier: localeIdentifier)), recognizer.supportsOnDeviceRecognition else { - throw SpeechError.message("此语言的离线语音识别不可用。请在 iPhone 下载对应语言资源后重试，或使用文字提问；不会转到云端。") + throw Failure.languageUnavailable } guard recognizer.isAvailable else { - throw SpeechError.message("系统离线语音识别暂不可用，请稍后重新启动，或使用文字提问。") + throw Failure.temporarilyUnavailable } let request = SFSpeechAudioBufferRecognitionRequest() request.requiresOnDeviceRecognition = true @@ -58,9 +59,10 @@ final class OpenGlassSpeechSession { self.request = request latestText = "" ending = false - startedAt = Date() - lastPacketAt = Date() + startedAt = ProcessInfo.processInfo.systemUptime + lastPacketAt = startedAt lastVoiceAt = nil + lastTranscriptAt = nil isListening = true let id = sessionID task = recognizer.recognitionTask(with: request) { [weak self] result, error in @@ -68,7 +70,10 @@ final class OpenGlassSpeechSession { guard let self, self.sessionID == id else { return } if let result { let text = result.bestTranscription.formattedString - if !text.isEmpty && text != self.latestText { self.lastVoiceAt = Date() } + if !text.isEmpty && text != self.latestText { + self.lastVoiceAt = ProcessInfo.processInfo.systemUptime + self.lastTranscriptAt = self.lastVoiceAt + } self.latestText = text self.onPartial?(self.latestText) if result.isFinal { self.complete(); return } @@ -77,8 +82,16 @@ final class OpenGlassSpeechSession { if self.ending && !self.latestText.isEmpty { self.complete() } else { + let failure: Failure + if SFSpeechRecognizer.authorizationStatus() != .authorized { + failure = .permissionDenied + } else if self.recognizer?.supportsOnDeviceRecognition == false { + failure = .languageUnavailable + } else { + failure = .recognitionInterrupted(error.localizedDescription) + } self.stop() - self.onError?("离线语音识别中断：\(error.localizedDescription)。可重新启动或使用文字提问。") + self.onError?(failure) } } } @@ -110,7 +123,7 @@ final class OpenGlassSpeechSession { samples[index] = sample squareSum += sample * sample } - let now = Date() + let now = ProcessInfo.processInfo.systemUptime lastPacketAt = now if sqrt(squareSum / Float(frames)) > 0.012 { lastVoiceAt = now } request.append(buffer) @@ -152,28 +165,52 @@ final class OpenGlassSpeechSession { private func tick() { guard isListening else { return } - let now = Date() - if let lastVoiceAt, now.timeIntervalSince(lastVoiceAt) >= 1.3 { - finishUtterance() - } else if now.timeIntervalSince(startedAt) >= 25 { - finishUtterance() - } else if now.timeIntervalSince(lastPacketAt) >= 5 { + let now = ProcessInfo.processInfo.systemUptime + let action = OpenGlassUtteranceTiming.action(now: now, startedAt: startedAt, + lastPacketAt: lastPacketAt, lastVoiceAt: lastVoiceAt, + lastTranscriptAt: lastTranscriptAt, hasText: !latestText.isEmpty) + switch action { + case .finish: finishUtterance() + case .audioTimeout: stop() - onError?("Rokid 音频超过5秒未到达。确认连接后重新启动语音。") + onError?(.audioTimeout) + case .listen: break } } private func complete() { - let text = latestText.trimmingCharacters(in: .whitespacesAndNewlines) + let candidate = latestText.trimmingCharacters(in: .whitespacesAndNewlines) + // Punctuation-only final results are not questions. Silence and failed + // recognition finish normally and let the controller schedule a retry. + let hasWords = candidate.unicodeScalars.contains { CharacterSet.alphanumerics.contains($0) } + let text = hasWords ? candidate : "" stop() onFinal?(text) } - enum SpeechError: LocalizedError { - case message(String) + enum Failure: LocalizedError { + case permissionDenied, languageUnavailable, temporarilyUnavailable, audioTimeout + case recognitionInterrupted(String) + + var requiresUserAction: Bool { + switch self { + case .permissionDenied, .languageUnavailable: return true + default: return false + } + } var errorDescription: String? { - if case .message(let message) = self { return message } - return nil + switch self { + case .permissionDenied: + return "未获语音识别权限。请在系统设置中允许后点击“恢复自动语音”；文字提问仍可使用。" + case .languageUnavailable: + return "此语言的离线识别资源不可用。请下载资源或切换语言后恢复；不会转到云端。" + case .temporarilyUnavailable: + return "系统离线识别暂不可用。" + case .audioTimeout: + return "Rokid 音频暂时没有到达。" + case .recognitionInterrupted(let message): + return "离线识别暂时中断：\(message)" + } } } } diff --git a/ios/Sources/OpenGlassViewController.swift b/ios/Sources/OpenGlassViewController.swift index eb784d8..aba5528 100644 --- a/ios/Sources/OpenGlassViewController.swift +++ b/ios/Sources/OpenGlassViewController.swift @@ -18,9 +18,17 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp private var isAuthorizingSpeech = false private var isAnswering = false private var audioConnected = false - private var speechPaused = false + private var voiceLoop = OpenGlassVoiceLoopPolicy() + private var wantsService = false + private var attemptedAutomaticStart = false + private var isInBackground = false + private var audioInterruptionActive = false + private var listeningWakeTask: Task? + private var speechAuthorizationID = UUID() + private var completedTurns = 0 private var suppressAnswerSpeech = false private var runID = UUID() + private var turnID = UUID() private var latestRawImage: Data? private var latestImage: UIImage? private var latestImageAt: Date? @@ -68,7 +76,11 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp connectionLabel.text = "Rokid 尚未连接" answerView.text = "回答会显示在这里。" frameTimer = Timer.scheduledTimer(withTimeInterval: 1, repeats: true) { [weak self] _ in - Task { @MainActor in self?.refreshFrameAge() } + Task { @MainActor in + self?.refreshFrameAge() + self?.refreshAddresses() + self?.reconcileAutomaticLoop() + } } refreshAddresses() refreshFrameAge() @@ -79,11 +91,26 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp override func viewDidAppear(_ animated: Bool) { super.viewDidAppear(animated) refreshAddresses() + if !attemptedAutomaticStart { + attemptedAutomaticStart = true + let documents = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0] + let hasModelFiles = OpenGlassModelSession.modelFiles.allSatisfy { + FileManager.default.fileExists(atPath: documents.appendingPathComponent($0).path) + } + if hasModelFiles { + wantsService = true + if voiceSwitch.isOn { voiceLoop.requestListening() } + reconcileAutomaticLoop() + } else { + statusLabel.text = "首次使用请在“模型下载”中准备 MiniCPM-V 4.6，然后启动接收。" + } + } UIApplication.shared.isIdleTimerDisabled = isRunning } deinit { frameTimer?.invalidate() + listeningWakeTask?.cancel() NotificationCenter.default.removeObserver(self) } @@ -165,9 +192,9 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp speakSwitch.addTarget(self, action: #selector(speakChanged), for: .valueChanged) stack.addArrangedSubview(row("朗读回答", speakSwitch)) stack.addArrangedSubview(routeLabel) - configureButton(listenButton, title: "开始语音", action: #selector(toggleListening)) + configureButton(listenButton, title: "恢复自动语音", action: #selector(toggleListening)) stack.addArrangedSubview(listenButton) - transcriptLabel.text = "识别文字会显示在这里。停顿约1.3秒，或点“结束语句”提问。" + transcriptLabel.text = "启动后自动听取 Rokid 语音。停顿约1.3秒后提问，朗读结束会自动听下一句。" stack.addArrangedSubview(transcriptLabel) questionField.borderStyle = .roundedRect questionField.placeholder = "输入问题，例如：我面前有什么？" @@ -192,7 +219,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp note.numberOfLines = 0 note.font = .preferredFont(forTextStyle: .footnote) note.textColor = .secondaryLabel - note.text = "Rokid 可通过同一局域网或 Windows 双 USB 中继连接。首次使用请先下载模型。此 App 的模型文件与原 MiniCPM 示例分开保存。进入后台会停止接收和语音，回到前台后请重新启动。" + note.text = "Rokid 可通过同一局域网或 Windows 双 USB 中继连接。首次使用请先下载模型。此 App 的模型文件与原 MiniCPM 示例分开保存。进入后台会停止接收和语音；回到前台会恢复此前运行状态。主动停止后不会自动重启。" stack.addArrangedSubview(note) } @@ -226,10 +253,14 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp self.latestImageAt = Date() self.preview.image = self.latestImage self.refreshFrameAge() + self.reconcileAutomaticLoop() } server.onAudio = { [weak self] data in - guard let self, self.isRunning, !self.isAnswering, self.activeUtterance == nil, - !self.speechPaused else { return } + guard let self else { return } + self.reconcileAutomaticLoop() + guard self.isRunning, !self.isInBackground, !self.audioInterruptionActive, + !self.isAnswering, !self.model.isBusy, self.activeUtterance == nil, + self.voiceLoop.canListen(at: self.monotonicNow) else { return } self.speech.appendPCM16LE(data) } server.onStatus = { [weak self] status in self?.connectionLabel.text = status } @@ -243,7 +274,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp self.speech.stop() if self.isRunning { self.statusLabel.text = "Rokid 音频已断开，等待重新连接。" } } else { - self.startListeningIfPossible() + self.reconcileAutomaticLoop() } self.updateControls() } @@ -257,31 +288,40 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp self?.statusLabel.text = message self?.updateControls() } - speech.onError = { [weak self] message in - guard let self else { return } - self.speechPaused = true - self.statusLabel.text = message - self.updateControls() + speech.onError = { [weak self] failure in + self?.handleSpeechFailure(failure) } speech.onFinal = { [weak self] text in - guard let self, self.isRunning, !self.speechPaused else { return } + guard let self, self.isRunning, !self.isInBackground, + self.voiceLoop.wantsListening, self.voiceLoop.blockedReason == nil else { return } if text.isEmpty { - self.statusLabel.text = "未识别到问题，继续等待语音。" - self.resumeListeningAfterEchoDelay() + self.retryListening(message: "未识别到问题，继续等待语音。") } else { + self.voiceLoop.receivedUtterance() self.questionField.text = text - self.submit(text) + self.submit(text, fromSpeech: true) } self.updateControls() } } @objc private func toggleService() { - if isRunning || isStarting { stopService(message: "接收已停止，点击启动可恢复。"); return } - guard !model.isBusy, !model.requiresAppRestart else { return } + if wantsService || isRunning || isStarting { + stopService(message: "接收已停止，点击启动可恢复。") + return + } + guard !model.requiresAppRestart else { return } + wantsService = true + if voiceSwitch.isOn { voiceLoop.requestListening() } else { voiceLoop.pause() } + reconcileAutomaticLoop() + } + + private func startServiceIfPossible() { + guard wantsService, !isInBackground, !isRunning, !isStarting, + !model.isBusy, !isAnswering, !model.requiresAppRestart else { return } isStarting = true isRunning = true - speechPaused = false + completedTurns = 0 runID = UUID() let id = runID latestRawImage = nil @@ -291,10 +331,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp do { try server.start(port: 18080) } catch { - isRunning = false - isStarting = false - statusLabel.text = "无法启动接收：\(error.localizedDescription)" - updateControls() + stopService(message: "无法启动接收：\(error.localizedDescription)") return } UIApplication.shared.isIdleTimerDisabled = true @@ -303,35 +340,49 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp refreshFrameAge() updateControls() Task { + guard isRunning, runID == id, !isInBackground else { return } do { try await model.load() guard isRunning, runID == id else { updateControls(); return } - statusLabel.text = "等待 Rokid 画面和音频；也可以文字提问。" - if voiceSwitch.isOn { + statusLabel.text = voiceLoop.wantsListening + ? "等待 Rokid 画面和音频；也可以文字提问。" + : "接收已启动；自动语音保持关闭或暂停，仍可文字提问。" + if voiceSwitch.isOn, voiceLoop.wantsListening, voiceLoop.blockedReason == nil { + let authorizationID = UUID() + speechAuthorizationID = authorizationID + isAuthorizingSpeech = true let authorized = await OpenGlassSpeechSession.authorize() guard isRunning, runID == id else { updateControls(); return } - if !authorized { - speechPaused = true - statusLabel.text = "未获语音识别权限。可用文字提问，或在系统设置中允许。" + if speechAuthorizationID == authorizationID { + isAuthorizingSpeech = false + if !authorized, voiceLoop.wantsListening { + handleSpeechFailure(.permissionDenied) + } } } isStarting = false - startListeningIfPossible() + reconcileAutomaticLoop() } catch { if runID == id { stopService(message: error.localizedDescription) } } - isStarting = false updateControls() + endBackgroundTaskIfIdle() } } - private func stopService(message: String) { + private func stopService(message: String, preservingIntent: Bool = false) { + if !preservingIntent { + wantsService = false + voiceLoop.pause() + } isRunning = false isStarting = false isAuthorizingSpeech = false - speechPaused = true + speechAuthorizationID = UUID() + cancelListeningWake() suppressAnswerSpeech = true runID = UUID() + turnID = UUID() speech.stop() stopSpeaking() server.stop() @@ -343,12 +394,16 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp } @objc private func stopCurrentTurn() { - speechPaused = true + turnID = UUID() + voiceLoop.pause() + cancelListeningWake() + isAuthorizingSpeech = false + speechAuthorizationID = UUID() suppressAnswerSpeech = true speech.stop() stopSpeaking() model.requestStop() - statusLabel.text = model.isBusy ? "正在结束本轮推理…" : "语音已暂停，点击“开始语音”可恢复。" + statusLabel.text = model.isBusy ? "正在结束本轮推理…" : "语音已暂停，点击“恢复自动语音”可恢复。" updateControls() } @@ -362,19 +417,22 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp return true } - private func submit(_ text: String) { + private func submit(_ text: String, fromSpeech: Bool = false) { let question = text.trimmingCharacters(in: .whitespacesAndNewlines) guard !question.isEmpty else { statusLabel.text = "请先输入或说出问题。"; return } - guard isRunning, !isStarting, model.isLoaded, !model.isBusy, !isAnswering else { + guard isRunning, !isInBackground, !isStarting, model.isLoaded, !model.isBusy, !isAnswering else { statusLabel.text = "请先启动接收，并等待当前模型任务结束。" return } - guard question.count <= 500 else { statusLabel.text = "问题请控制在500字以内。"; return } + guard question.count <= 500 else { + if fromSpeech { retryListening(message: "问题超过500字，请简短地重新说一次。") } + else { statusLabel.text = "问题请控制在500字以内。" } + return + } guard let image = latestImage, let receivedAt = latestImageAt, Date().timeIntervalSince(receivedAt) <= maximumFrameAge else { speech.stop() - speechPaused = true - statusLabel.text = "没有8秒内的新画面。请恢复 Rokid 图像连接后，再次提问。" + statusLabel.text = "本句未提交：没有8秒内的新画面。收到新画面后会自动继续听，请重新说出问题。" updateControls() return } @@ -385,8 +443,16 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp answerView.text = "" statusLabel.text = "已暂停收音，正在本机理解画面…" let id = runID + turnID = UUID() + let currentTurnID = turnID updateControls() Task { + guard isRunning, runID == id, turnID == currentTurnID, !isInBackground else { + isAnswering = false + updateControls() + reconcileAutomaticLoop() + return + } var imageURL: URL? do { let folder = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0] @@ -400,7 +466,8 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp imageURL = file let answer = try await model.ask(question: question, imageURL: file, answerLanguage: localeIdentifier) isAnswering = false - if isRunning, runID == id { + if isRunning, runID == id, turnID == currentTurnID { + completedTurns += 1 answerView.text = answer.isEmpty ? "模型没有返回可朗读的文字，请换个问题重试。" : answer if !suppressAnswerSpeech, speakSwitch.isOn, !answer.isEmpty { speak(answer) @@ -412,11 +479,10 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp } catch { isAnswering = false if isRunning, runID == id { - statusLabel.text = error.localizedDescription - // Errors are explicit; do not restart recognition in a loop. - speechPaused = true if model.requiresAppRestart || !model.isLoaded { stopService(message: error.localizedDescription) + } else if turnID == currentTurnID, voiceLoop.wantsListening { + retryListening(message: error.localizedDescription) } } } @@ -424,43 +490,110 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp if !model.requiresAppRestart, let imageURL { try? FileManager.default.removeItem(at: imageURL) } updateControls() endBackgroundTaskIfIdle() + reconcileAutomaticLoop() } } private var localeIdentifier: String { languageControl.selectedSegmentIndex == 1 ? "en-US" : "zh-CN" } + private var monotonicNow: TimeInterval { ProcessInfo.processInfo.systemUptime } + private var hasFreshFrame: Bool { + guard latestImage != nil, let date = latestImageAt else { return false } + return Date().timeIntervalSince(date) <= maximumFrameAge + } + + /// All timer, frame, connection and audio callbacks go through the same + /// admission gate. None can bypass a retry or the post-TTS echo deadline. + private func reconcileAutomaticLoop() { + guard !isInBackground else { return } + if !isRunning { startServiceIfPossible() } + startListeningIfPossible() + } private func startListeningIfPossible() { - guard isRunning, !isStarting, !isAuthorizingSpeech, !isAnswering, !model.isBusy, model.isLoaded, - !model.requiresAppRestart, voiceSwitch.isOn, !speechPaused, - audioConnected, activeUtterance == nil, !speech.isActive else { return } + guard isRunning, !isInBackground, !audioInterruptionActive, + !isStarting, !isAuthorizingSpeech, !isAnswering, !model.isBusy, model.isLoaded, + !model.requiresAppRestart, voiceSwitch.isOn, voiceLoop.wantsListening, + voiceLoop.blockedReason == nil, audioConnected, hasFreshFrame, + activeUtterance == nil, !speech.isActive else { return } + guard voiceLoop.canListen(at: monotonicNow) else { + scheduleListeningWake() + return + } do { try speech.start(localeIdentifier: localeIdentifier) + } catch let failure as OpenGlassSpeechSession.Failure { + handleSpeechFailure(failure) } catch { - speechPaused = true - statusLabel.text = error.localizedDescription + retryListening(message: error.localizedDescription) + } + updateControls() + } + + private func handleSpeechFailure(_ failure: OpenGlassSpeechSession.Failure) { + guard isRunning, voiceLoop.wantsListening else { return } + speech.stop() + if failure.requiresUserAction { + voiceLoop.block(reason: failure.localizedDescription) + cancelListeningWake() + statusLabel.text = failure.localizedDescription + updateControls() + } else { + retryListening(message: failure.localizedDescription) } + } + + private func retryListening(message: String) { + guard isRunning, !isInBackground, voiceLoop.wantsListening, + voiceLoop.blockedReason == nil else { return } + speech.stop() + let delay = voiceLoop.retry(now: monotonicNow) + statusLabel.text = "\(message) \(delay.formatted())秒后自动重试。" + scheduleListeningWake() updateControls() } + private func cancelListeningWake() { + listeningWakeTask?.cancel() + listeningWakeTask = nil + } + + private func scheduleListeningWake() { + guard listeningWakeTask == nil, isRunning, !isInBackground, + voiceLoop.wantsListening, voiceLoop.blockedReason == nil else { return } + let delay = voiceLoop.notBefore - monotonicNow + guard delay > 0 else { return } + let id = runID + listeningWakeTask = Task { [weak self] in + do { try await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000)) } + catch { return } + guard let self, !Task.isCancelled, self.runID == id else { return } + self.listeningWakeTask = nil + self.reconcileAutomaticLoop() + } + } + @objc private func toggleListening() { if speech.isListening { speech.finishUtterance(); updateControls(); return } if speech.isFinishing { return } - guard isRunning, !model.isBusy, !isAnswering, !isAuthorizingSpeech else { return } + guard isRunning, !isAuthorizingSpeech else { return } voiceSwitch.isOn = true UserDefaults.standard.set(true, forKey: "OpenGlass.voiceEnabled") - speechPaused = false + voiceLoop.requestListening() isAuthorizingSpeech = true let id = runID + let authorizationID = UUID() + speechAuthorizationID = authorizationID + updateControls() Task { let authorized = await OpenGlassSpeechSession.authorize() - guard isRunning, runID == id else { return } + guard isRunning, runID == id, speechAuthorizationID == authorizationID else { return } isAuthorizingSpeech = false if authorized { if !audioConnected { statusLabel.text = "等待 Rokid 音频连接…" } - startListeningIfPossible() + else if !hasFreshFrame { statusLabel.text = "等待新画面，随后自动听取语音。" } + reconcileAutomaticLoop() } else { - speechPaused = true - statusLabel.text = "未获语音识别权限，请在系统设置中允许；文字提问仍可使用。" + handleSpeechFailure(.permissionDenied) } updateControls() } @@ -472,7 +605,11 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp if isRunning { toggleListening() } } else { speech.stop() - speechPaused = true + voiceLoop.pause() + cancelListeningWake() + isAuthorizingSpeech = false + speechAuthorizationID = UUID() + statusLabel.text = "自动语音已关闭，仍可文字提问。" } updateControls() } @@ -480,7 +617,8 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp @objc private func languageChanged() { UserDefaults.standard.set(languageControl.selectedSegmentIndex, forKey: "OpenGlass.speechLanguage") speech.stop() - startListeningIfPossible() + if voiceLoop.wantsListening { voiceLoop.requestListening() } + reconcileAutomaticLoop() } @objc private func speakChanged() { @@ -506,13 +644,18 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp synthesizer.speak(utterance) } catch { activeUtterance = nil - speechPaused = true statusLabel.text = "朗读不可用：\(error.localizedDescription)。回答已保留在屏幕。" + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + resumeListeningAfterEchoDelay() } updateControls() } private func stopSpeaking() { + if activeUtterance != nil || synthesizer.isSpeaking { + voiceLoop.protectFromEcho(now: monotonicNow) + } + // Invalidate identity before stopSpeaking can dispatch didCancel. activeUtterance = nil synthesizer.stopSpeaking(at: .immediate) try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) @@ -533,19 +676,17 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp Task { @MainActor [weak self] in guard let self, self.activeUtterance === utterance else { return } self.activeUtterance = nil - self.speechPaused = true - self.statusLabel.text = "朗读被中断，点击“开始语音”可继续。" + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + self.statusLabel.text = "朗读被中断，稍后自动继续听取语音。" + self.resumeListeningAfterEchoDelay() self.updateControls() } } private func resumeListeningAfterEchoDelay() { - let id = runID - Task { - try? await Task.sleep(nanoseconds: 700_000_000) - guard isRunning, runID == id else { return } - startListeningIfPossible() - } + voiceLoop.protectFromEcho(now: monotonicNow) + scheduleListeningWake() + reconcileAutomaticLoop() } @objc private func rotationChanged() { @@ -591,10 +732,11 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp private func updateControls() { let busy = model.isBusy || isAnswering - serviceButton.configuration?.title = isRunning || isStarting ? "停止接收" : "启动接收" - serviceButton.isEnabled = isRunning || (!busy && !model.requiresAppRestart) + serviceButton.configuration?.title = wantsService || isRunning || isStarting ? "停止接收" : "启动接收" + serviceButton.isEnabled = wantsService || isRunning || (!busy && !model.requiresAppRestart) askButton.isEnabled = isRunning && !isStarting && model.isLoaded && !busy && !model.requiresAppRestart - listenButton.configuration?.title = speech.isFinishing ? "正在结束语句…" : (speech.isListening ? "结束语句并提问" : "开始语音") + listenButton.configuration?.title = speech.isFinishing ? "正在结束语句…" + : (speech.isListening ? "结束语句并提问" : "恢复自动语音") listenButton.isEnabled = isRunning && !isStarting && !busy && activeUtterance == nil && !model.requiresAppRestart && !speech.isFinishing && !isAuthorizingSpeech stopTurnButton.isEnabled = isRunning || busy || activeUtterance != nil @@ -602,7 +744,16 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp languageControl.isEnabled = !busy && activeUtterance == nil rotationControl.isEnabled = !busy voiceSwitch.isEnabled = !isStarting + if isRunning, let reason = voiceLoop.blockedReason { statusLabel.text = reason } if model.requiresAppRestart { statusLabel.text = OpenGlassModelSession.SessionError.restartRequired.localizedDescription } + let phase: String + if !isRunning || isInBackground { phase = "paused" } + else if busy { phase = "thinking" } + else if activeUtterance != nil { phase = "speaking" } + else if speech.isActive { phase = "listening" } + else if !voiceLoop.wantsListening || voiceLoop.blockedReason != nil { phase = "paused" } + else { phase = "waiting" } + server.setConversationStatus(phase: phase, completedTurns: completedTurns) } @objc private func showDownloads() { @@ -621,8 +772,10 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp @objc private func refreshAddresses() { let addresses = localIPv4Addresses() - addressView.text = addresses.isEmpty ? "暂无局域网 IPv4 地址。可使用 Windows 双 USB 中继，或连接 Wi-Fi 后刷新。" + let endpoints = addresses.isEmpty + ? "等待 Wi-Fi / 热点地址；USB 中继仍可使用。" : addresses.map { "http://\($0):18080" }.joined(separator: "\n") + addressView.text = "无线接收：\(RokidIngressServer.serviceName)\n\(endpoints)\n让眼镜加入此 iPhone 热点或同一 Wi-Fi，选择自动连接。" } @objc private func copyAddress() { @@ -641,7 +794,9 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp while let current = cursor { let interface = current.pointee defer { cursor = interface.ifa_next } - guard let address = interface.ifa_addr, address.pointee.sa_family == UInt8(AF_INET) else { continue } + guard let address = interface.ifa_addr, address.pointee.sa_family == UInt8(AF_INET), + interface.ifa_flags & UInt32(IFF_UP) != 0, + interface.ifa_flags & UInt32(IFF_LOOPBACK) == 0 else { continue } let name = String(cString: interface.ifa_name) guard name == "en0" || name.hasPrefix("bridge") || name.hasPrefix("ap") else { continue } var host = [CChar](repeating: 0, count: Int(NI_MAXHOST)) @@ -659,7 +814,8 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp } @objc private func enteredBackground() { - stopService(message: "已进入后台并停止。回到前台后点击启动。") + isInBackground = true + stopService(message: "已进入后台。回到前台后恢复此前运行状态。", preservingIntent: true) if model.isBusy, backgroundTask == .invalid { backgroundTask = UIApplication.shared.beginBackgroundTask(withName: "FinishOpenGlassTurn") { [weak self] in Task { @MainActor in @@ -678,18 +834,34 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp } @objc private func enteredForeground() { + isInBackground = false + // Background suspension deactivated our audio session; iOS may omit + // its matching interruption-ended notification while suspended. + audioInterruptionActive = false refreshAddresses() - statusLabel.text = model.isBusy ? "正在结束上一轮推理，请稍候。" : "已回到前台，点击“启动接收”恢复。" + if wantsService { + statusLabel.text = model.isBusy ? "正在结束上一轮推理，随后自动恢复接收。" : "已回到前台，正在恢复接收。" + reconcileAutomaticLoop() + } else { + statusLabel.text = "已回到前台，接收保持停止。" + } updateControls() } @objc private func audioInterrupted(_ notification: Notification) { - guard let type = notification.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt, - type == AVAudioSession.InterruptionType.began.rawValue else { return } - speechPaused = true - speech.stop() - stopSpeaking() - statusLabel.text = "系统音频被打断；恢复后点击“开始语音”，或使用文字提问。" + guard let rawType = notification.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt, + let type = AVAudioSession.InterruptionType(rawValue: rawType) else { return } + switch type { + case .began: + audioInterruptionActive = true + speech.stop() + stopSpeaking() + if isRunning { statusLabel.text = "系统音频暂时中断，恢复后继续自动语音。" } + case .ended: + audioInterruptionActive = false + resumeListeningAfterEchoDelay() + @unknown default: break + } updateControls() } diff --git a/ios/Sources/OpenGlassVoiceLoopPolicy.swift b/ios/Sources/OpenGlassVoiceLoopPolicy.swift new file mode 100644 index 0000000..bd48bc0 --- /dev/null +++ b/ios/Sources/OpenGlassVoiceLoopPolicy.swift @@ -0,0 +1,79 @@ +import Foundation + +/// Recovery timing and user intent for the foreground voice loop. +/// Callers supply ProcessInfo.processInfo.systemUptime for every `now` value. +struct OpenGlassVoiceLoopPolicy { + private(set) var wantsListening = false + private(set) var blockedReason: String? + private(set) var retryCount = 0 + private var retryNotBefore: TimeInterval = 0 + private var echoNotBefore: TimeInterval = 0 + + var notBefore: TimeInterval { max(retryNotBefore, echoNotBefore) } + + mutating func requestListening() { + wantsListening = true + blockedReason = nil + receivedUtterance() + } + + mutating func pause() { + wantsListening = false + blockedReason = nil + receivedUtterance() + } + + mutating func block(reason: String) { + blockedReason = reason + } + + /// Returns the retry delay, while `notBefore` also includes any echo hold. + @discardableResult + mutating func retry(now: TimeInterval) -> TimeInterval { + let delays: [TimeInterval] = [0.5, 1, 2, 4, 8] + let delay = delays[min(retryCount, delays.count - 1)] + retryCount = min(retryCount + 1, delays.count) + retryNotBefore = now + delay + return delay + } + + mutating func receivedUtterance() { + retryCount = 0 + retryNotBefore = 0 + } + + mutating func protectFromEcho(now: TimeInterval, delay: TimeInterval = 0.7) { + echoNotBefore = max(echoNotBefore, now + delay) + } + + func canListen(at time: TimeInterval) -> Bool { + wantsListening && blockedReason == nil && time >= notBefore + } +} + +/// Decides when to finalize recognition without depending on Speech or timers. +struct OpenGlassUtteranceTiming { + enum Action: Equatable { + case listen + case finish + case audioTimeout + } + + static func action(now: TimeInterval, startedAt: TimeInterval, + lastPacketAt: TimeInterval, lastVoiceAt: TimeInterval?, + lastTranscriptAt: TimeInterval?, hasText: Bool) -> Action { + if hasText, let lastTranscriptAt, now - lastTranscriptAt >= 2.5 { + return .finish + } + if let lastVoiceAt, now - lastVoiceAt >= 1.3 { + return .finish + } + if now - startedAt >= 25 { + return .finish + } + if now - lastPacketAt >= 5 { + return .audioTimeout + } + return .listen + } +} diff --git a/ios/Sources/RokidIngressServer.swift b/ios/Sources/RokidIngressServer.swift index 71f2a50..8850657 100644 --- a/ios/Sources/RokidIngressServer.swift +++ b/ios/Sources/RokidIngressServer.swift @@ -4,6 +4,18 @@ import Network /// Foreground-only local receiver for OpenGlassRokidSensor 0.1.3. /// Call start/stop on the main queue; all callbacks are delivered on that queue. final class RokidIngressServer { + // A stable, non-personal name lets the glasses find the same phone after + // DHCP changes. Never advertise the user's device name or account. + static let serviceType = "_openglass._tcp" + static let serviceName: String = { + let key = "OpenGlass.receiverServiceName" + if let saved = UserDefaults.standard.string(forKey: key), + saved.hasPrefix("OpenGlass-"), saved.count == 18 { return saved } + let name = "OpenGlass-" + UUID().uuidString.prefix(8) + UserDefaults.standard.set(name, forKey: key) + return name + }() + var onImage: ((Data) -> Void)? var onAudio: ((Data) -> Void)? var onStatus: ((String) -> Void)? @@ -16,6 +28,8 @@ final class RokidIngressServer { private var audioClient: UUID? private var imageCount = 0 private var audioPacketCount = 0 + private var conversationPhase = "waiting" + private var completedTurns = 0 private var epoch = UUID() // Read/written only on queue. epoch belongs to the main queue and only // filters UI callbacks; this also prevents an old listener accepting work. @@ -30,6 +44,7 @@ final class RokidIngressServer { let parameters = NWParameters.tcp parameters.allowLocalEndpointReuse = true let newListener = try NWListener(using: parameters, on: NWEndpoint.Port(rawValue: port)!) + newListener.service = NWListener.Service(name: Self.serviceName, type: Self.serviceType) stop() let generation = UUID() epoch = generation @@ -58,7 +73,7 @@ final class RokidIngressServer { self.clients[client.id] = client client.healthJSON = { [weak self] in guard let self else { return "{}" } - return "{\"service\":\"openglass-ios\",\"protocol\":1,\"image_count\":\(self.imageCount),\"audio_packets_in\":\(self.audioPacketCount),\"audio_connected\":\(self.audioClient != nil)}" + return "{\"service\":\"openglass-ios\",\"protocol\":1,\"receiver_name\":\"\(Self.serviceName)\",\"image_count\":\(self.imageCount),\"audio_packets_in\":\(self.audioPacketCount),\"audio_connected\":\(self.audioClient != nil),\"conversation_phase\":\"\(self.conversationPhase)\",\"completed_turns\":\(self.completedTurns)}" } client.onImage = { [weak self] data in guard let self else { return } @@ -108,6 +123,19 @@ final class RokidIngressServer { newListener.start(queue: queue) } + /// Only operational metadata is exposed; recognized speech and answers + /// never appear in network diagnostics. + func setConversationStatus(phase: String, completedTurns: Int) { + dispatchPrecondition(condition: .onQueue(.main)) + let safePhase = ["paused", "listening", "thinking", "speaking", "waiting"].contains(phase) ? phase : "waiting" + let generation = epoch + queue.async { [weak self] in + guard let self, self.servingGeneration == generation else { return } + self.conversationPhase = safePhase + self.completedTurns = max(0, completedTurns) + } + } + func stop() { dispatchPrecondition(condition: .onQueue(.main)) epoch = UUID() @@ -120,6 +148,8 @@ final class RokidIngressServer { audioClient = nil imageCount = 0 audioPacketCount = 0 + conversationPhase = "waiting" + completedTurns = 0 imageDeliveryInFlight = false pendingAudioBytes = 0 closing.forEach { $0.close() } diff --git a/ios/scripts/prepare_upstream.py b/ios/scripts/prepare_upstream.py index 3626345..f3d0749 100644 --- a/ios/scripts/prepare_upstream.py +++ b/ios/scripts/prepare_upstream.py @@ -181,6 +181,7 @@ def prepare(root: Path, destination: Path, lock: dict, build_number: str) -> dic info = plistlib.loads(git_file(destination, revision, f"{APP}/Info.plist")) info.update({"CFBundleDisplayName": lock["display_name"], "NSLocalNetworkUsageDescription": LOCAL_NETWORK_DESCRIPTION, + "NSBonjourServices": ["_openglass._tcp"], "NSSpeechRecognitionUsageDescription": SPEECH_DESCRIPTION}) # No background-mode claim: receiving and inference require foreground use. info.setdefault("NSAppTransportSecurity", {})["NSAllowsLocalNetworking"] = True diff --git a/ios/tests/run_voice_loop_tests.sh b/ios/tests/run_voice_loop_tests.sh new file mode 100644 index 0000000..75d6413 --- /dev/null +++ b/ios/tests/run_voice_loop_tests.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +OUT="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/openglass-voice-loop-tests-$$" +trap 'rm -f "$OUT"' EXIT +swiftc "$ROOT/ios/Sources/OpenGlassVoiceLoopPolicy.swift" "$ROOT/ios/tests/voice_loop/main.swift" -o "$OUT" +"$OUT" diff --git a/ios/tests/test_prepare_upstream.py b/ios/tests/test_prepare_upstream.py index ad619fd..4fae1c8 100644 --- a/ios/tests/test_prepare_upstream.py +++ b/ios/tests/test_prepare_upstream.py @@ -87,6 +87,7 @@ def test_generates_launch_permissions_signing_and_keeps_official_download_ui(sel self.assertTrue(info["UIFileSharingEnabled"]) self.assertTrue(info["NSAppTransportSecurity"]["NSAllowsLocalNetworking"]) self.assertIn("Rokid", info["NSLocalNetworkUsageDescription"]) + self.assertEqual(info["NSBonjourServices"], ["_openglass._tcp"]) self.assertIn("设备端", info["NSSpeechRecognitionUsageDescription"]) self.assertNotIn("UIBackgroundModes", info) self.assertEqual(plistlib.loads((self.destination / f"{prepare.APP}/{prepare.APP}.entitlements").read_bytes()), {}) diff --git a/ios/tests/voice_loop/main.swift b/ios/tests/voice_loop/main.swift new file mode 100644 index 0000000..b65b257 --- /dev/null +++ b/ios/tests/voice_loop/main.swift @@ -0,0 +1,200 @@ +import Foundation + +var checks = 0 +func check(_ condition: @autoclosure () -> Bool, _ label: String) { + guard condition() else { fatalError("FAIL: \(label)") } + checks += 1 +} +func checkTime(_ actual: TimeInterval, _ expected: TimeInterval, _ label: String) { + check(abs(actual - expected) < 0.000_000_001, label) +} + +// Startup is opt-in, including when a late callback arrives before any start. +var startup = OpenGlassVoiceLoopPolicy() +check(!startup.wantsListening && !startup.canListen(at: 100), "startup waits for user intent") +startup.retry(now: 100) +startup.receivedUtterance() +check(!startup.canListen(at: 200), "callbacks cannot create startup intent") +startup.requestListening() +check(startup.canListen(at: 200), "explicit start opens the gate") + +// Silent recognition failures keep a positive, bounded delay. Wakeups at the +// deadline may retry; repeated audio/frame notifications before it may not. +var silence = OpenGlassVoiceLoopPolicy() +silence.requestListening() +let expectedDelays: [TimeInterval] = [0.5, 1, 2, 4, 8, 8, 8] +var now: TimeInterval = 1_000 +for (index, expected) in expectedDelays.enumerated() { + let delay = silence.retry(now: now) + checkTime(delay, expected, "silence retry \(index + 1) uses bounded backoff") + checkTime(silence.notBefore, now + expected, "retry sets a future deadline") + check(!silence.canListen(at: now), "retry cannot spin immediately") + check(!silence.canListen(at: silence.notBefore - 0.001), "retry waits through its deadline") + check(silence.canListen(at: silence.notBefore), "retry is allowed at its deadline") + now = silence.notBefore +} +for _ in 0..<1_000 { + checkTime(silence.retry(now: now), 8, "long silence stays capped at eight seconds") + check(!silence.canListen(at: now), "capped retries still wait") + now = silence.notBefore +} +check(silence.retryCount == 5, "retry count saturates instead of growing without bound") +silence.receivedUtterance() +check(silence.retryCount == 0 && silence.canListen(at: now), "a recognized question clears backoff") +checkTime(silence.retry(now: now), 0.5, "a later failure starts with the short delay") + +// A retry timer and an echo hold are independent: whichever expires later wins. +var timing = OpenGlassVoiceLoopPolicy() +timing.requestListening() +timing.protectFromEcho(now: 10) +checkTime(timing.retry(now: 10), 0.5, "retry reports its delay independently of echo") +checkTime(timing.notBefore, 10.7, "short retry cannot shorten echo hold") +timing.receivedUtterance() +checkTime(timing.notBefore, 10.7, "successful recognition cannot clear echo hold") +timing.requestListening() +check(!timing.canListen(at: 10.6), "explicit listening request still respects echo") +check(timing.canListen(at: timing.notBefore), "echo hold opens at the boundary") +timing.retry(now: 20) +timing.retry(now: 20) +timing.protectFromEcho(now: 20) +checkTime(timing.notBefore, 21, "longer retry survives a shorter echo hold") +timing.protectFromEcho(now: 20.5) +checkTime(timing.notBefore, 21.2, "a later echo hold extends the gate") +timing.protectFromEcho(now: 20.6, delay: 0.1) +checkTime(timing.notBefore, 21.2, "a stale shorter hold cannot move the deadline backwards") +check(!timing.canListen(at: 20.7), "the first echo wakeup cannot bypass an extended hold") + +// All ingress and completion paths consult the same gate. These arrivals model +// a reconnect, new frame, PCM packet and an old timer while TTS echo can remain. +var arrivals = OpenGlassVoiceLoopPolicy() +arrivals.requestListening() +arrivals.protectFromEcho(now: 30) +for arrival in [30.0, 30.1, 30.4, 30.699] { + check(!arrivals.canListen(at: arrival), "an arrival cannot bypass the common echo gate") +} +arrivals.requestListening() +check(!arrivals.canListen(at: 30.699), "a manual restart during echo also waits") +check(arrivals.canListen(at: arrivals.notBefore), "arrivals become eligible after echo expires") + +// A user pause wins over already queued retry, successful-final and TTS events. +var stopped = OpenGlassVoiceLoopPolicy() +stopped.requestListening() +stopped.retry(now: 40) +stopped.protectFromEcho(now: 40) +stopped.block(reason: "temporary test state") +stopped.pause() +check(!stopped.wantsListening && stopped.blockedReason == nil && stopped.retryCount == 0, + "pause clears failure state while disabling intent") +checkTime(stopped.notBefore, 40.7, "pause preserves the acoustic hold") +stopped.retry(now: 41) +stopped.receivedUtterance() +stopped.protectFromEcho(now: 42) +check(!stopped.canListen(at: 1_000), "late callbacks cannot resurrect a stopped loop") +stopped.requestListening() +check(!stopped.canListen(at: 42.5), "resuming a paused loop still waits for recent echo") +check(stopped.canListen(at: stopped.notBefore), "only explicit resume restores eligibility") + +// Permanent errors preserve user intent but require an explicit request to +// retry. Neither traffic, a successful old utterance nor elapsed time clears it. +var blocked = OpenGlassVoiceLoopPolicy() +blocked.requestListening() +blocked.block(reason: "offline language unavailable") +check(blocked.wantsListening, "blocking preserves intended listening") +blocked.retry(now: 50) +blocked.receivedUtterance() +blocked.protectFromEcho(now: 50) +check(blocked.blockedReason == "offline language unavailable", "late success keeps the permanent block") +check(!blocked.canListen(at: 1_000), "permanent failure does not expire with time") +blocked.requestListening() +check(blocked.blockedReason == nil && blocked.retryCount == 0, "explicit request clears the block") +check(!blocked.canListen(at: 50.6), "clearing a permanent block cannot bypass echo") +check(blocked.canListen(at: blocked.notBefore), "cleared block allows listening after echo") +blocked.pause() +blocked.block(reason: "late permission callback") +blocked.receivedUtterance() +check(!blocked.wantsListening && !blocked.canListen(at: 1_000), "a late permanent error cannot enable intent") + +// Intent survives only when the caller explicitly restores it after background +// cleanup. The policy never enables itself merely because time has passed. +var foreground = OpenGlassVoiceLoopPolicy() +foreground.requestListening() +let wasListening = foreground.wantsListening +foreground.protectFromEcho(now: 60) +foreground.pause() +check(!foreground.canListen(at: 60.2), "background cleanup closes the gate") +if wasListening { foreground.requestListening() } +check(!foreground.canListen(at: 60.2), "foreground restoration preserves the echo deadline") +check(foreground.canListen(at: 61), "restored intent can resume after the deadline") +foreground.pause() +let wasPaused = foreground.wantsListening +if wasPaused { foreground.requestListening() } +check(!foreground.canListen(at: 1_000), "a previously paused loop stays paused on foreground") + +// Steady environmental noise can keep the RMS voice marker fresh. A stable +// transcript still finishes promptly instead of waiting for the 25-second cap. +check(OpenGlassUtteranceTiming.action(now: 4.499, startedAt: 0, + lastPacketAt: 4.499, lastVoiceAt: 4.499, + lastTranscriptAt: 2, hasText: true) == .listen, + "stable text waits the full 2.5 seconds") +check(OpenGlassUtteranceTiming.action(now: 4.5, startedAt: 0, + lastPacketAt: 4.5, lastVoiceAt: 4.5, + lastTranscriptAt: 2, hasText: true) == .finish, + "stable text finishes despite continuous RMS noise") +check(OpenGlassUtteranceTiming.action(now: 4.5, startedAt: 0, + lastPacketAt: 4.5, lastVoiceAt: 4.5, + lastTranscriptAt: 4, hasText: true) == .listen, + "a changed partial transcript extends the text deadline") +check(OpenGlassUtteranceTiming.action(now: 6.5, startedAt: 0, + lastPacketAt: 6.5, lastVoiceAt: 6.5, + lastTranscriptAt: 4, hasText: true) == .finish, + "the updated transcript finishes when it becomes stable") + +// An empty transcript must not reuse the stable-text deadline from old content. +check(OpenGlassUtteranceTiming.action(now: 10, startedAt: 0, + lastPacketAt: 10, lastVoiceAt: 10, + lastTranscriptAt: 0, hasText: false) == .listen, + "old transcript time cannot finalize current empty recognition") +check(OpenGlassUtteranceTiming.action(now: 10, startedAt: 0, + lastPacketAt: 10, lastVoiceAt: 10, + lastTranscriptAt: nil, hasText: true) == .listen, + "text without a stability timestamp does not finish prematurely") +check(OpenGlassUtteranceTiming.action(now: 24.999, startedAt: 0, + lastPacketAt: 24.999, lastVoiceAt: nil, + lastTranscriptAt: nil, hasText: false) == .listen, + "silent packets keep empty recognition alive before the duration cap") +check(OpenGlassUtteranceTiming.action(now: 25, startedAt: 0, + lastPacketAt: 25, lastVoiceAt: nil, + lastTranscriptAt: nil, hasText: false) == .finish, + "continuous silence finishes at 25 seconds") +check(OpenGlassUtteranceTiming.action(now: 1.299, startedAt: 0, + lastPacketAt: 1.299, lastVoiceAt: 0, + lastTranscriptAt: nil, hasText: false) == .listen, + "voice silence waits the full 1.3 seconds") +check(OpenGlassUtteranceTiming.action(now: 1.3, startedAt: 0, + lastPacketAt: 1.3, lastVoiceAt: 0, + lastTranscriptAt: nil, hasText: false) == .finish, + "silence after a voice marker ends even an empty utterance") + +// A stopped transport times out earlier than an otherwise empty utterance. +check(OpenGlassUtteranceTiming.action(now: 4.999, startedAt: 0, + lastPacketAt: 0, lastVoiceAt: nil, + lastTranscriptAt: nil, hasText: false) == .listen, + "packet gap waits the full five seconds") +check(OpenGlassUtteranceTiming.action(now: 5, startedAt: 0, + lastPacketAt: 0, lastVoiceAt: nil, + lastTranscriptAt: nil, hasText: false) == .audioTimeout, + "missing packets time out at five seconds") +check(OpenGlassUtteranceTiming.action(now: 5, startedAt: 0, + lastPacketAt: 0, lastVoiceAt: 5, + lastTranscriptAt: 0, hasText: true) == .finish, + "stable transcript finalization takes precedence over a packet timeout") +check(OpenGlassUtteranceTiming.action(now: 5, startedAt: 0, + lastPacketAt: 0, lastVoiceAt: 0, + lastTranscriptAt: nil, hasText: false) == .finish, + "voice silence finalization takes precedence over a packet timeout") +check(OpenGlassUtteranceTiming.action(now: 25, startedAt: 0, + lastPacketAt: 0, lastVoiceAt: nil, + lastTranscriptAt: nil, hasText: false) == .finish, + "duration cap finalization takes precedence over a packet timeout") + +print("Passed \(checks) OpenGlass voice-loop policy checks") diff --git a/ios/windows/common.ps1 b/ios/windows/common.ps1 index 0f0f802..6272c66 100644 --- a/ios/windows/common.ps1 +++ b/ios/windows/common.ps1 @@ -48,8 +48,8 @@ function Get-OpenGlassRokid { } if ($Package -notin $installed) { throw 'The requested OpenGlass Rokid package is not installed.' } $info = Invoke-OpenGlassAdb $Adb @('-s', $Serial, 'shell', 'dumpsys', 'package', $Package) - if ($info -notmatch 'versionName=(0\.1\.3(?:-debug)?)(?:\s|$)') { - throw 'Collector version 0.1.3 is required for pc_base_url. Upgrade or review newer protocol compatibility first.' + if ($info -notmatch 'versionName=((?:0\.1\.3|0\.2\.0)(?:-debug)?)(?:\s|$)') { + throw 'Collector version 0.1.3 or 0.2.0 is required. Upgrade or review newer protocol compatibility first.' } $version = $Matches[1] $resolved = Invoke-OpenGlassAdb $Adb @('-s', $Serial, 'shell', 'cmd', 'package', 'resolve-activity', '--brief', '-a', 'android.intent.action.MAIN', '-c', 'android.intent.category.LAUNCHER', $Package) diff --git a/ios/windows/configure-rokid.ps1 b/ios/windows/configure-rokid.ps1 index da60fc5..888d4d1 100644 --- a/ios/windows/configure-rokid.ps1 +++ b/ios/windows/configure-rokid.ps1 @@ -1,6 +1,8 @@ <# Start the installed Rokid collector against the iPhone on the same LAN. #> +[CmdletBinding(DefaultParameterSetName = 'Manual')] param( - [Parameter(Mandatory = $true)][string]$IPhoneAddress, + [Parameter(Mandatory = $true, ParameterSetName = 'Manual')][string]$IPhoneAddress, + [Parameter(Mandatory = $true, ParameterSetName = 'Automatic')][switch]$AutoDiscover, [ValidateRange(1, 65535)][int]$Port = 18080, [string]$Adb, [string]$Serial, @@ -8,6 +10,20 @@ param( [switch]$SkipReceiverCheck ) . (Join-Path $PSScriptRoot 'common.ps1') +$adbPath = Resolve-OpenGlassAdb $Adb +$device = Get-OpenGlassRokid -Adb $adbPath -Serial $Serial -Package $Package +if ($AutoDiscover) { + if ($device.Version -notmatch '^(\d+)\.(\d+)\.(\d+)' -or + [version]$Matches[0] -lt [version]'0.2.0') { + throw 'Automatic Wi-Fi discovery requires OpenGlassRokidSensor 0.2.0 or newer. Install the new APK first.' + } + $result = Invoke-OpenGlassAdb $adbPath @('-s', $device.Serial, 'shell', 'am', 'start', '-S', '-W', '-n', $device.Activity, '--ez', 'auto_discover', 'true') + if ($result -match 'Error:|Exception') { throw "Collector launch failed: $result" } + Write-Host 'Automatic Wi-Fi discovery enabled. The glasses verify /health before sending images or audio.' + Write-Host 'Join the iPhone hotspot or the same Wi-Fi in the glasses settings, and keep OpenGlass receiving on iPhone.' + Write-Host 'No Windows relay is used in this mode. Verify both streams before unplugging USB.' + return +} $address = $null if (-not [System.Net.IPAddress]::TryParse($IPhoneAddress, [ref]$address) -or $address.AddressFamily -ne [System.Net.Sockets.AddressFamily]::InterNetwork -or @@ -15,8 +31,6 @@ if (-not [System.Net.IPAddress]::TryParse($IPhoneAddress, [ref]$address) -or throw 'Pass the iPhone Wi-Fi IPv4 address, not localhost, a URL, or the glasses address.' } $baseUrl = 'http://{0}:{1}' -f $address.ToString(), $Port -$adbPath = Resolve-OpenGlassAdb $Adb -$device = Get-OpenGlassRokid -Adb $adbPath -Serial $Serial -Package $Package if (-not $SkipReceiverCheck) { try { $health = Invoke-RestMethod -Uri "$baseUrl/health" -TimeoutSec 5 } catch { throw "iPhone /health is unreachable at $baseUrl. Open OpenGlass on iPhone and start its receiver on the same LAN. $($_.Exception.Message)" } From c439fa3b391f93c746fb02747c74c5092e3237be Mon Sep 17 00:00:00 2001 From: MengzhangLI Date: Sun, 18 Oct 2026 02:35:25 +0800 Subject: [PATCH 06/11] Add standalone Rokid Wi-Fi discovery and reconnecting collector --- ios/README_zh.md | 6 +- ios/windows/common.ps1 | 4 +- rokid_app/.gitignore | 1 + .../OpenGlassRokidSensor/app/build.gradle | 14 +- .../app/src/main/AndroidManifest.xml | 5 +- .../openglass/rokid/ConnectionPolicy.kt | 36 ++ .../openglass/rokid/EndpointController.kt | 330 ++++++++++++++++++ .../opensqz/openglass/rokid/MainActivity.kt | 286 ++++++++++----- .../openglass/rokid/SimpleBinaryWebSocket.kt | 56 ++- .../openglass/rokid/ConnectionPolicyTest.kt | 52 +++ rokid_app/README.md | 32 +- rokid_app/README_zh.md | 34 +- rokid_app/scripts/build_install.ps1 | 10 +- runtime/openglass_omni/rokid_device.py | 4 +- tests/test_rokid_device.py | 12 +- 15 files changed, 744 insertions(+), 138 deletions(-) create mode 100644 rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/ConnectionPolicy.kt create mode 100644 rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/EndpointController.kt create mode 100644 rokid_app/OpenGlassRokidSensor/app/src/test/java/org/opensqz/openglass/rokid/ConnectionPolicyTest.kt diff --git a/ios/README_zh.md b/ios/README_zh.md index 37192dd..d1db0f9 100644 --- a/ios/README_zh.md +++ b/ios/README_zh.md @@ -172,7 +172,7 @@ Invoke-RestMethod http://127.0.0.1:18080/health ## 5B. Wi-Fi 直连（推荐 iPhone 个人热点） -先安装新 iOS IPA 和 Rokid 0.2.0 APK。iOS 使用同一 Apple 账号、同一 bundle ID 覆盖安装，保留现有模型；无需卸载或重新下载模型。 +先安装新 iOS IPA 和 Rokid 0.2.0 APK。iOS 使用同一 Apple 账号、同一 bundle ID 覆盖安装，保留现有模型；无需卸载或重新下载模型。本机旧眼镜 APK 的原签名文件不可用，因此交付并存包 **OpenGlass WiFi**（`org.opensqz.openglass.rokid.wifi`，`0.2.0-wifi`），保留旧采集 App。使用新版前先在旧 App 中停止采集，避免两个程序争用相机和麦克风；新 App 首次打开需允许相机和麦克风。 1. iPhone“设置 → 个人热点 → 允许其他人加入”。眼镜找不到热点时可开启“最大兼容性”。首次连接时停留在热点页面，直到眼镜已加入。具体开关见 [Apple 个人热点说明](https://support.apple.com/zh-cn/111785)。热点依赖运营商支持；它分享的是蜂窝网络，不是酒店 Wi-Fi 的中继。 2. 通过 Rokid 官方 App / 设备的 Wi-Fi 设置，让眼镜加入这个热点；密码仅在设备设置内输入。OpenGlass 不修改 Wi-Fi 密码和系统网络。 @@ -183,7 +183,7 @@ Invoke-RestMethod http://127.0.0.1:18080/health 眼镜暂留 USB 时，可在 Windows 手动切到自动发现模式；这条命令只做一次配置与启动，不是数据中继： ```powershell -powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File 'C:\path\to\OpenGlass\ios\windows\configure-rokid.ps1' -AutoDiscover +powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File 'C:\path\to\OpenGlass\ios\windows\configure-rokid.ps1' -AutoDiscover -Package org.opensqz.openglass.rokid.wifi ``` 普通 Wi-Fi / 酒店 Wi-Fi 也可用：将两台设备加入同一网络。网络必须允许设备互访；若酒店启用了客户端隔离，或眼镜无法完成网页认证，应改用个人热点。Bonjour 被阻止但设备仍能直连时，可从 iPhone 页面取 IPv4 地址手动指定： @@ -227,6 +227,8 @@ powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File 'C:\path\to\OpenGl - 本地脚本检查：PowerShell语法解析通过；USB中继使用伪设备测试了端口占用、已有reverse、并发映射冲突、接收端不可达、成功启动与只清理自身映射等情况。 - [CI运行38890873970](https://github.com/OpenSQZ/OpenGlass/actions/runs/38890873970)成功，源提交为 `c11c87b39b5798f8b4218d3a47d5b6cc68e56091`：1399项wire protocol检查、7项工程准备测试、7项USB中继测试通过，并生成未签名的 `OpenGlass-1.0.ipa`（artifact `OpenGlass-iOS-unsigned-3`）。IPA SHA256 为 `DA76A28A87195C43EA80ECF11366724180480B5D6502DEA27A4D1AF800AD00DB`。 - 现场设备识别为 `iPhone18,2`、iOS `26.6.2`。新版 `1.0` / build `3` 已通过本机认证签名并安装；两个对应 4.6 模型已校验并导入。2026-10-18 用户反馈：USB 连接下眼镜图像、声音和 iPhone MiniCPM 推理已跑通，仍需手动进行单轮交互。本次无线与自动多轮改动另行验证，不能继承旧版的端到端结论。 +- 无线/自动语音 iOS 源提交 `087766314c0d817a78b65c6e6a93d46b281d5576` 通过 [CI 39057876255](https://github.com/OpenSQZ/OpenGlass/actions/runs/39057876255)，生成 build `5`。1399 项协议检查、2087 项语音循环断言、7 项工程准备测试、7 项 USB relay 测试通过。IPA SHA256：`D5A8F689048DFE09E1C4F4AF4EE67E770E5FE01BE3883A0A4B414804B39BFCB7`。Windows 免费签名更新后，USB 查询已确认手机安装 build `5`，两份模型文件仍完整保留。 +- Rokid `0.2.0-wifi` / code `5` 的 `assembleWifi`、7 项 `testWifiUnitTest` 与原 `assembleDebug` 通过；APK SHA256：`A5A64849FEAC796D4688B60833E647FA3DACE1CB3F5A16BD8935AA0EA5623A13`。并存包 `org.opensqz.openglass.rokid.wifi` 已安装，旧 debug 采集 App 保留。眼镜已获得过 iPhone 热点 DHCP 地址；新 App 的权限、无线音视频和拔线后的自动连续对话仍需现场测试。 - 不在此文档写入个人设备序列号、UDID、Apple账号或内网凭据。现场更新请用设备类型和验证结果描述。 可重复执行无设备测试： diff --git a/ios/windows/common.ps1 b/ios/windows/common.ps1 index 6272c66..b0c20c9 100644 --- a/ios/windows/common.ps1 +++ b/ios/windows/common.ps1 @@ -40,7 +40,7 @@ function Get-OpenGlassRokid { } $listing = Invoke-OpenGlassAdb $Adb @('-s', $Serial, 'shell', 'pm', 'list', 'packages') $installed = @($listing -split '\r?\n' | ForEach-Object { - if ($_ -match '^package:(org\.opensqz\.openglass\.rokid(?:\.debug)?)\s*$') { $Matches[1] } + if ($_ -match '^package:(org\.opensqz\.openglass\.rokid(?:\.(?:debug|wifi))?)\s*$') { $Matches[1] } }) if (-not $Package) { if ($installed.Count -ne 1) { throw 'Expected exactly one OpenGlass Rokid package. Install the collector or pass -Package for the installed debug/release variant.' } @@ -48,7 +48,7 @@ function Get-OpenGlassRokid { } if ($Package -notin $installed) { throw 'The requested OpenGlass Rokid package is not installed.' } $info = Invoke-OpenGlassAdb $Adb @('-s', $Serial, 'shell', 'dumpsys', 'package', $Package) - if ($info -notmatch 'versionName=((?:0\.1\.3|0\.2\.0)(?:-debug)?)(?:\s|$)') { + if ($info -notmatch 'versionName=((?:0\.1\.3|0\.2\.0)(?:-(?:debug|wifi))?)(?:\s|$)') { throw 'Collector version 0.1.3 or 0.2.0 is required. Upgrade or review newer protocol compatibility first.' } $version = $Matches[1] diff --git a/rokid_app/.gitignore b/rokid_app/.gitignore index 86e3d4d..4244a57 100644 --- a/rokid_app/.gitignore +++ b/rokid_app/.gitignore @@ -10,3 +10,4 @@ sessions/ *.keystore *.jks *.before-* +.build-tools/ diff --git a/rokid_app/OpenGlassRokidSensor/app/build.gradle b/rokid_app/OpenGlassRokidSensor/app/build.gradle index b0187ec..d2593d8 100644 --- a/rokid_app/OpenGlassRokidSensor/app/build.gradle +++ b/rokid_app/OpenGlassRokidSensor/app/build.gradle @@ -11,8 +11,9 @@ android { applicationId 'org.opensqz.openglass.rokid' minSdk 28 targetSdk 34 - versionCode 4 - versionName '0.1.3' + versionCode 5 + versionName '0.2.0' + manifestPlaceholders = [appLabel: '@string/app_name'] buildConfigField 'String', 'PC_BASE_URL', '"http://127.0.0.1:18080"' buildConfigField 'String', 'PC_WS_URL', '"ws://127.0.0.1:18080/rokid/audio"' @@ -27,6 +28,14 @@ android { applicationIdSuffix '.debug' versionNameSuffix '-debug' } + wifi { + initWith debug + applicationIdSuffix '.wifi' + versionNameSuffix '-wifi' + manifestPlaceholders = [appLabel: 'OpenGlass WiFi'] + signingConfig signingConfigs.debug + matchingFallbacks = ['debug'] + } release { minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' @@ -45,4 +54,5 @@ android { dependencies { implementation 'androidx.core:core-ktx:1.13.1' + testImplementation 'junit:junit:4.13.2' } diff --git a/rokid_app/OpenGlassRokidSensor/app/src/main/AndroidManifest.xml b/rokid_app/OpenGlassRokidSensor/app/src/main/AndroidManifest.xml index 0b38007..1db5f84 100644 --- a/rokid_app/OpenGlassRokidSensor/app/src/main/AndroidManifest.xml +++ b/rokid_app/OpenGlassRokidSensor/app/src/main/AndroidManifest.xml @@ -5,6 +5,8 @@ + + @@ -14,7 +16,7 @@ android:allowBackup="false" android:hardwareAccelerated="true" android:icon="@android:drawable/sym_def_app_icon" - android:label="@string/app_name" + android:label="${appLabel}" android:networkSecurityConfig="@xml/network_security_config" android:roundIcon="@android:drawable/sym_def_app_icon" android:supportsRtl="true" @@ -23,6 +25,7 @@ diff --git a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/ConnectionPolicy.kt b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/ConnectionPolicy.kt new file mode 100644 index 0000000..95a1802 --- /dev/null +++ b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/ConnectionPolicy.kt @@ -0,0 +1,36 @@ +package org.opensqz.openglass.rokid + +import java.net.URI + +internal const val USB_BASE_URL = "http://127.0.0.1:18080" + +/** Keep URL validation independent of Android so the selection rules can be unit tested. */ +internal object ConnectionPolicy { + fun normalizeUrl(value: String): String { + val uri = URI(value.trim().trimEnd('/')) + require(uri.scheme == "http" && !uri.host.isNullOrBlank() && + uri.rawUserInfo == null && uri.rawQuery == null && uri.rawFragment == null && + uri.rawPath.isNullOrEmpty() && (uri.port == -1 || uri.port in 1..65535)) { + "Enter http://:18080 without a path or password" + } + return uri.toASCIIString() + } + + fun isLoopback(url: String): Boolean = + URI(url).host.lowercase() in setOf("127.0.0.1", "localhost", "[::1]", "::1") + + fun websocketUrl(url: String): String = "ws://${URI(url).rawAuthority}/rokid/audio" + + fun receiverMatches(name: String, preferred: String?): Boolean = + name.matches(Regex("OpenGlass-[0-9a-fA-F]{8}")) && + (preferred.isNullOrEmpty() || name == preferred) + + // A previously chosen phone must not silently become a different hotel guest's phone. + fun selectService(available: Set, preferred: String?): String? = when { + !preferred.isNullOrEmpty() -> preferred.takeIf { it in available } + available.size == 1 -> available.first() + else -> null + } + + fun nextDelay(delay: Long): Long = (delay * 2).coerceIn(1_000L, 30_000L) +} diff --git a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/EndpointController.kt b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/EndpointController.kt new file mode 100644 index 0000000..3ad8601 --- /dev/null +++ b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/EndpointController.kt @@ -0,0 +1,330 @@ +package org.opensqz.openglass.rokid + +import android.content.Context +import android.net.ConnectivityManager +import android.net.LinkProperties +import android.net.Network +import android.net.NetworkCapabilities +import android.net.NetworkRequest +import android.net.nsd.NsdManager +import android.net.nsd.NsdServiceInfo +import android.net.wifi.WifiManager +import android.os.Handler +import android.os.Looper +import android.util.Log +import org.json.JSONObject +import java.net.HttpURLConnection +import java.net.Inet4Address +import java.net.URI +import java.net.URL +import java.util.concurrent.Executors + +internal data class ServerEndpoint(val baseUrl: String, val network: Network?, val source: String) { + val websocketUrl: String get() = ConnectionPolicy.websocketUrl(baseUrl) + fun connection(path: String): HttpURLConnection = + (network?.openConnection(URL(baseUrl + path)) ?: URL(baseUrl + path).openConnection()) as HttpURLConnection +} + +/** All mutable state belongs to the main looper; only bounded HTTP probes run on the worker. */ +internal class EndpointController( + context: Context, + private val onEndpoint: (ServerEndpoint?) -> Unit, + private val onStatus: (String) -> Unit, +) { + private val app = context.applicationContext + private val prefs = app.getSharedPreferences("pc_connection", Context.MODE_PRIVATE) + private val main = Handler(Looper.getMainLooper()) + private val connectivity = app.getSystemService(ConnectivityManager::class.java) + private val nsd = app.getSystemService(NsdManager::class.java) + private val worker = Executors.newSingleThreadExecutor { r -> Thread(r, "openglass-discovery") } + private var active = false + private var generation = 0 + private var probing = false + private var fixedUrl: String? = null + private var wifi: Network? = null + private var gateway: String? = null + private var endpoint: ServerEndpoint? = null + private var retryDelay = 1_000L + private var discovery: NsdManager.DiscoveryListener? = null + private var multicast: WifiManager.MulticastLock? = null + private val services = linkedMapOf() + private val knownServices = linkedMapOf() + private val resolveQueue = ArrayDeque() + private var resolving = false + private var discoveryStartedAt = 0L + private val probeTask = Runnable { probe() } + + private val networkCallback = object : ConnectivityManager.NetworkCallback() { + override fun onAvailable(network: Network) { + if (!active || network == wifi) return + wifi = network + gateway = null + networkChanged() + } + override fun onLinkPropertiesChanged(network: Network, properties: LinkProperties) { + if (!active || network != wifi) return + val address = properties.routes.firstOrNull { it.isDefaultRoute && it.gateway is Inet4Address } + ?.gateway?.hostAddress + if (address != gateway) { + gateway = address + scheduleProbe(0) + } + } + override fun onLost(network: Network) { + if (!active || network != wifi) return + wifi = null + gateway = null + networkChanged() + } + } + + fun start(explicitUrl: String?) { + if (active) stop() + active = true + fixedUrl = explicitUrl + retryDelay = 1_000L + wifi = connectivity.allNetworks.firstOrNull { + connectivity.getNetworkCapabilities(it)?.hasTransport(NetworkCapabilities.TRANSPORT_WIFI) == true + } + gateway = wifi?.let { connectivity.getLinkProperties(it) }?.routes + ?.firstOrNull { it.isDefaultRoute && it.gateway is Inet4Address }?.gateway?.hostAddress + connectivity.registerNetworkCallback( + NetworkRequest.Builder().addTransportType(NetworkCapabilities.TRANSPORT_WIFI).build(), + networkCallback, main, + ) + if (fixedUrl != null) publish(endpointFor(fixedUrl!!, "manual / USB")) else { + startDiscovery() + scheduleProbe(0) + } + } + + fun stop() { + if (!active) return + active = false + generation++ + main.removeCallbacks(probeTask) + try { connectivity.unregisterNetworkCallback(networkCallback) } catch (_: IllegalArgumentException) { } + stopDiscovery() + publish(null) + } + + fun destroy() { stop(); worker.shutdownNow() } + + fun retry() { + // A WebSocket failure does not make an unverified host eligible for sensor uploads. + if (active && fixedUrl == null) scheduleProbe(1_000) + } + + private fun endpointFor(url: String, source: String) = + ServerEndpoint(url, if (ConnectionPolicy.isLoopback(url)) null else wifi, source) + + private fun networkChanged() { + generation++ + services.clear() + knownServices.clear() + retryDelay = 1_000L + if (fixedUrl != null) { + publish(endpointFor(fixedUrl!!, "manual / USB")) + } else { + publish(null) + stopDiscovery() + startDiscovery() + scheduleProbe(500) + } + } + + private fun publish(value: ServerEndpoint?) { + if (endpoint == value) return + endpoint = value + onEndpoint(value) + } + + private fun scheduleProbe(delay: Long) { + if (!active || fixedUrl != null) return + main.removeCallbacks(probeTask) + main.postDelayed(probeTask, delay) + } + + private fun startDiscovery() { + if (!active || wifi == null || discovery != null) return + discoveryStartedAt = android.os.SystemClock.elapsedRealtime() + val token = generation + val listener = object : NsdManager.DiscoveryListener { + override fun onDiscoveryStarted(type: String) = Unit + override fun onDiscoveryStopped(type: String) = Unit + override fun onStartDiscoveryFailed(type: String, code: Int) { main.post { + if (active && token == generation) { + stopDiscovery() + onStatus("Bonjour unavailable; checking iPhone hotspot / saved address") + } + } } + override fun onStopDiscoveryFailed(type: String, code: Int) = Unit + override fun onServiceFound(service: NsdServiceInfo) { main.post { + if (active && token == generation && + (knownServices.containsKey(service.serviceName) || knownServices.size < 8) && + resolveQueue.none { it.serviceName == service.serviceName } && + service.serviceType.trimEnd('.') == "_openglass._tcp") { + knownServices[service.serviceName] = service + resolveQueue.addLast(service) + resolveNext(token) + } + } } + override fun onServiceLost(service: NsdServiceInfo) { main.post { + if (active && token == generation) { + services.remove(service.serviceName) + knownServices.remove(service.serviceName) + scheduleProbe(500) + } + } } + } + discovery = listener + try { + multicast = app.getSystemService(WifiManager::class.java).createMulticastLock("openglass-discovery") + .apply { setReferenceCounted(false); acquire() } + nsd.discoverServices("_openglass._tcp.", NsdManager.PROTOCOL_DNS_SD, listener) + } catch (e: Exception) { + stopDiscovery() + Log.w("OpenGlassRokid", "Discovery unavailable: ${e.javaClass.simpleName}") + } + } + + private fun stopDiscovery() { + discovery?.let { try { nsd.stopServiceDiscovery(it) } catch (_: Exception) { } } + discovery = null + resolveQueue.clear() + resolving = false + multicast?.let { if (it.isHeld) it.release() } + multicast = null + } + + @Suppress("DEPRECATION") // Glasses run Android versions predating the API 34 service-info callback. + private fun resolveNext(token: Int) { + if (resolving || resolveQueue.isEmpty() || !active || token != generation) return + resolving = true + val item = resolveQueue.removeFirst() + val listener = object : NsdManager.ResolveListener { + override fun onResolveFailed(service: NsdServiceInfo, code: Int) { main.post { + if (token == generation) { resolving = false; resolveNext(token) } + } } + override fun onServiceResolved(service: NsdServiceInfo) { main.post { + if (active && token == generation) { + resolving = false + val host = service.host?.hostAddress + if (host != null && service.port in 1..65535) { + runCatching { URI("http", null, host, service.port, null, null, null).toASCIIString() } + .getOrNull()?.let { url -> + val changed = services[service.serviceName] != url + services[service.serviceName] = url + if (changed) scheduleProbe(500) + } + } + resolveNext(token) + } + } } + } + try { nsd.resolveService(item, listener) } catch (_: Exception) { + resolving = false + resolveNext(token) + } + } + + private fun probe() { + if (!active || fixedUrl != null) return + if (wifi == null) { + publish(null) + onStatus("Join the iPhone hotspot or shared Wi-Fi first") + scheduleProbe(retryDelay) + retryDelay = ConnectionPolicy.nextDelay(retryDelay) + return + } + if (probing) { scheduleProbe(500); return } + if (discovery == null) startDiscovery() + probing = true + val token = generation + val hotspot = gateway?.let { endpointFor("http://$it:18080", "iPhone hotspot") } + val discovered = services.mapValues { endpointFor(it.value, "Bonjour: ${it.key}") } + val saved = prefs.getString("pc_base_url", null)?.let { + runCatching { ConnectionPolicy.normalizeUrl(it) }.getOrNull() + ?.takeUnless { url -> ConnectionPolicy.isLoopback(url) } + ?.let { url -> endpointFor(url, "saved address") } + } + val preferred = prefs.getString("iphone_service_name", null) + val existing = endpoint + val settled = android.os.SystemClock.elapsedRealtime() - discoveryStartedAt >= 2_500 + worker.execute { + val checked = mutableMapOf() + fun healthy(candidate: ServerEndpoint?): Boolean = candidate != null && + checked.getOrPut(candidate) { health(candidate) }?.let { + ConnectionPolicy.receiverMatches(it.receiverName, preferred) + } == true + // Keep a healthy chosen endpoint; reconnecting never redirects a live session. + var chosen = existing?.takeIf { healthy(it) } + var ambiguous = false + if (chosen == null && healthy(hotspot)) chosen = hotspot + if (chosen == null && settled) { + val available = discovered.filter { (name, candidate) -> + healthy(candidate) && checked[candidate]?.receiverName == name + } + val chosenName = ConnectionPolicy.selectService(available.keys, preferred) + chosen = chosenName?.let { available[it] } + ambiguous = available.isNotEmpty() && chosen == null + } + if (chosen == null && !ambiguous && healthy(saved)) chosen = saved + val chosenName = chosen?.let { checked[it]?.receiverName } + main.post { + probing = false + if (!active) return@post + if (token != generation) { scheduleProbe(0); return@post } + if (chosenName != null) prefs.edit().putString("iphone_service_name", chosenName).apply() + publish(chosen) + if (chosen != null) { + retryDelay = 1_000L + onStatus("${chosen!!.source}: ${chosen!!.baseUrl}") + scheduleProbe(10_000) + } else { + // Resolve again after a failure: a known phone can acquire another DHCP address. + if (!resolving && resolveQueue.isEmpty()) { + resolveQueue.addAll(knownServices.values.take(8)) + resolveNext(generation) + } + onStatus(if (ambiguous) "Phone selection needed: open Connection settings" else + "Waiting for iPhone OpenGlass; join its hotspot or the same Wi-Fi") + scheduleProbe(retryDelay) + retryDelay = ConnectionPolicy.nextDelay(retryDelay) + } + } + } + } + + private data class HealthIdentity(val receiverName: String) + + private fun health(candidate: ServerEndpoint): HealthIdentity? { + var connection: HttpURLConnection? = null + return try { + connection = candidate.connection("/health").apply { + connectTimeout = 1_200 + readTimeout = 1_200 + instanceFollowRedirects = false + } + if (connection.responseCode != 200) null else { + val body = connection.inputStream.use { input -> + val data = ByteArray(4_097) + var count = 0 + val deadline = android.os.SystemClock.elapsedRealtime() + 2_500 + while (count < data.size) { + require(android.os.SystemClock.elapsedRealtime() < deadline) + val read = input.read(data, count, data.size - count) + if (read < 0) break + count += read + } + require(count <= 4_096) + String(data, 0, count, Charsets.UTF_8) + } + val json = JSONObject(body) + val receiverName = json.optString("receiver_name") + if (json.optString("service") == "openglass-ios" && json.optInt("protocol") == 1 && + ConnectionPolicy.receiverMatches(receiverName, null)) HealthIdentity(receiverName) else null + } + } catch (_: Exception) { null } finally { connection?.disconnect() } + } +} diff --git a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt index 65aad1c..8524ced 100644 --- a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt +++ b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt @@ -2,7 +2,9 @@ package org.opensqz.openglass.rokid import android.Manifest import android.app.Activity +import android.app.AlertDialog import android.content.Context +import android.content.Intent import android.content.pm.PackageManager import android.graphics.Bitmap import android.graphics.BitmapFactory @@ -26,14 +28,15 @@ import android.util.Log import android.util.Range import android.util.Size import android.view.KeyEvent -import android.view.MotionEvent import android.view.Surface import android.view.View import android.view.WindowManager import android.widget.TextView +import android.widget.Button +import android.widget.EditText +import android.widget.LinearLayout import java.io.ByteArrayOutputStream import java.net.HttpURLConnection -import java.net.URL import java.nio.ByteBuffer import java.util.Locale import java.util.concurrent.atomic.AtomicBoolean @@ -57,31 +60,42 @@ private const val MANUAL_EXPOSURE_ENABLED = true private const val MANUAL_EXPOSURE_NS = 22_222_222L private const val MANUAL_FRAME_DURATION_NS = 44_444_444L private const val MANUAL_ISO = 1600 -private const val USB_REVERSE_BASE_URL = "http://127.0.0.1:18080" -private const val USB_REVERSE_WS_URL = "ws://127.0.0.1:18080/rokid/audio" private const val ENHANCE_IMAGE_FOR_UPLOAD = false private const val ENHANCE_MIN_LUMA = 55.0 private const val ENHANCE_TARGET_LUMA = 70.0 class MainActivity : Activity() { - private var pcBaseUrl = BuildConfig.PC_BASE_URL - private var pcWsUrl = BuildConfig.PC_WS_URL + private var explicitUrl: String? = null + private lateinit var endpointController: EndpointController + @Volatile private var currentEndpoint: ServerEndpoint? = null + @Volatile private var connectionStatus = "Searching for iPhone" + @Volatile private var imageConnection: HttpURLConnection? = null + @Volatile private var activeRecorder: AudioRecord? = null + private var userStopped = false + private val captureGeneration = AtomicInteger(0) + private val cameraGeneration = AtomicInteger(0) private fun configurePcAddress() { val prefs = getSharedPreferences("pc_connection", Context.MODE_PRIVATE) val supplied = intent.getStringExtra("pc_base_url") - val candidate = supplied ?: prefs.getString("pc_base_url", BuildConfig.PC_BASE_URL)!! + if (intent.getBooleanExtra("auto_discover", false)) { + explicitUrl = null + prefs.edit().putString("connection_mode", "auto").apply() + return + } + val candidate = supplied ?: when (prefs.getString("connection_mode", "auto")) { + "manual" -> prefs.getString("pc_base_url", null) + "usb" -> USB_BASE_URL + else -> null + } + if (candidate == null) { explicitUrl = null; return } try { - val uri = java.net.URI(candidate.trim().trimEnd('/')) - require(uri.scheme == "http" && !uri.host.isNullOrBlank() - && uri.rawUserInfo == null && uri.rawQuery == null && uri.rawFragment == null - && uri.rawPath.isNullOrEmpty() && (uri.port == -1 || uri.port in 1..65535)) - pcBaseUrl = uri.toString() - pcWsUrl = "ws://${uri.rawAuthority}/rokid/audio" - if (supplied != null) prefs.edit().putString("pc_base_url", pcBaseUrl).apply() - Log.i(TAG, "PC_CONFIG applied pcBase=$pcBaseUrl pcWs=$pcWsUrl") + explicitUrl = ConnectionPolicy.normalizeUrl(candidate) + if (supplied != null) prefs.edit().putString("pc_base_url", explicitUrl).apply() + Log.i(TAG, "PC_CONFIG explicit=$explicitUrl") } catch (e: Exception) { - Log.e(TAG, "Invalid PC address; keeping built-in default", e) + explicitUrl = null + Log.e(TAG, "Invalid address; using automatic discovery", e) } } @@ -111,10 +125,11 @@ class MainActivity : Activity() { private val imageUploadInFlight = AtomicBoolean(false) private val cameraRestartPending = AtomicBoolean(false) - private var audioThread: Thread? = null - private var audioWs: SimpleBinaryWebSocket? = null + @Volatile private var audioThread: Thread? = null + @Volatile private var audioWs: SimpleBinaryWebSocket? = null private var reconnectDelayMs = 1_000L - private var audioUrlIndex = 0 + private var webSocketGeneration = 0 + private val reconnectTask = Runnable { connectAudioWebSocket() } private var cameraThread: HandlerThread? = null private var cameraHandler: Handler? = null @@ -146,8 +161,26 @@ class MainActivity : Activity() { typeface = android.graphics.Typeface.MONOSPACE setPadding(18, 18, 18, 18) } - setContentView(hud) - Log.i(TAG, "onCreate pcBase=$pcBaseUrl pcWs=$pcWsUrl") + hud.setOnClickListener { toggleStartStop() } + hud.setOnLongClickListener { showConnectionSettings(); true } + val connectionButton = Button(this).apply { + text = "Connection / 连接设置" + setOnClickListener { showConnectionSettings() } + } + setContentView(LinearLayout(this).apply { + orientation = LinearLayout.VERTICAL + setBackgroundColor(Color.BLACK) + addView(connectionButton) + addView(hud) + }) + endpointController = EndpointController(this, { endpoint -> + currentEndpoint = endpoint + imageConnection?.disconnect() + closeAudioWebSocket() + reconnectDelayMs = 1_000L + if (running && endpoint != null) connectAudioWebSocket() + }, { connectionStatus = it }) + Log.i(TAG, "onCreate explicit=$explicitUrl") mainHandler.post(hudTick) if (hasPermissions()) { @@ -162,13 +195,14 @@ class MainActivity : Activity() { override fun onDestroy() { stopAll() + endpointController.destroy() mainHandler.removeCallbacksAndMessages(null) super.onDestroy() } override fun onResume() { super.onResume() - if (hasPermissions() && !running) { + if (hasPermissions() && !running && !userStopped) { Log.i(TAG, "onResume auto-start") startAll() } else { @@ -176,6 +210,54 @@ class MainActivity : Activity() { } } + override fun onNewIntent(newIntent: Intent) { + super.onNewIntent(newIntent) + setIntent(newIntent) + configurePcAddress() + if (running) endpointController.start(explicitUrl) + else if (hasPermissions()) { userStopped = false; startAll() } + } + + private fun showConnectionSettings() { + val prefs = getSharedPreferences("pc_connection", Context.MODE_PRIVATE) + AlertDialog.Builder(this).setTitle("OpenGlass connection") + .setItems(arrayOf("Auto: iPhone hotspot / Wi-Fi", "Manual address", "USB relay", "Choose phone again")) { _, which -> + fun applyMode(mode: String, url: String?) { + prefs.edit().putString("connection_mode", mode).apply() + explicitUrl = url + if (running) endpointController.start(explicitUrl) + } + when (which) { + 0 -> applyMode("auto", null) + 1 -> { + val input = EditText(this).apply { + setSingleLine(true) + setText(prefs.getString("pc_base_url", "http://172.20.10.1:18080")) + inputType = android.text.InputType.TYPE_CLASS_TEXT or android.text.InputType.TYPE_TEXT_VARIATION_URI + } + val dialog = AlertDialog.Builder(this).setTitle("iPhone / PC HTTP address") + .setView(input).setPositiveButton("Save", null).setNegativeButton("Cancel", null).create() + dialog.setOnShowListener { + dialog.getButton(AlertDialog.BUTTON_POSITIVE).setOnClickListener { + try { + val url = ConnectionPolicy.normalizeUrl(input.text.toString()) + prefs.edit().putString("pc_base_url", url).apply() + applyMode("manual", url) + dialog.dismiss() + } catch (_: Exception) { input.error = "Use http://
:18080" } + } + } + dialog.show() + } + 2 -> applyMode("usb", USB_BASE_URL) + 3 -> { + prefs.edit().remove("iphone_service_name").apply() + applyMode("auto", null) + } + } + }.show() + } + override fun onWindowFocusChanged(hasFocus: Boolean) { super.onWindowFocusChanged(hasFocus) if (hasFocus) { @@ -206,40 +288,36 @@ class MainActivity : Activity() { if (event.action == KeyEvent.ACTION_UP) { when (event.keyCode) { KeyEvent.KEYCODE_DPAD_CENTER, KeyEvent.KEYCODE_ENTER -> { + if (currentFocus is Button) return super.dispatchKeyEvent(event) toggleStartStop() return true } KeyEvent.KEYCODE_BACK -> { if (running) { + userStopped = true stopAll() return true } } + KeyEvent.KEYCODE_MENU -> { showConnectionSettings(); return true } } } return super.dispatchKeyEvent(event) } - override fun dispatchTouchEvent(ev: MotionEvent): Boolean { - Log.i(TAG, "touch action=${ev.action} x=${ev.x} y=${ev.y}") - if (ev.action == MotionEvent.ACTION_UP) { - toggleStartStop() - return true - } - return super.dispatchTouchEvent(ev) - } - private fun hasPermissions(): Boolean = checkSelfPermission(Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED && checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED private fun toggleStartStop() { + userStopped = running if (running) stopAll() else startAll() } private fun startAll() { if (running) return running = true + captureGeneration.incrementAndGet() startMs = System.currentTimeMillis() lastError = "none" audioPackets.set(0) @@ -247,7 +325,7 @@ class MainActivity : Activity() { imagePackets.set(0) imageFailures.set(0) Log.i(TAG, "startAll") - connectAudioWebSocket() + endpointController.start(explicitUrl) startAudioCapture() scheduleCameraStart(1_200) } @@ -265,7 +343,11 @@ class MainActivity : Activity() { private fun stopAll() { if (!running) return running = false + captureGeneration.incrementAndGet() + cameraRestartPending.set(false) Log.i(TAG, "stopAll") + endpointController.stop() + imageConnection?.disconnect() stopAudioCapture() closeAudioWebSocket() stopCameraCapture() @@ -275,8 +357,10 @@ class MainActivity : Activity() { private fun connectAudioWebSocket() { if (!running) return - val urls = pcWsUrls() - val url = urls[audioUrlIndex % urls.size] + val endpoint = currentEndpoint ?: return + closeAudioWebSocket() + val token = webSocketGeneration + val url = endpoint.websocketUrl audioWsUrlText = url audioStatus = "ws connecting" Log.i(TAG, "audio WS connecting $url") @@ -284,54 +368,67 @@ class MainActivity : Activity() { url, object : SimpleBinaryWebSocket.Listener { override fun onOpen() { + mainHandler.post { + if (!running || token != webSocketGeneration) return@post wsOpen = true reconnectDelayMs = 1_000L audioStatus = "ws connected" lastError = "none" Log.i(TAG, "audio WS connected $url") + } } override fun onClosed(reason: String) { + mainHandler.post { + if (!running || token != webSocketGeneration) return@post wsOpen = false audioStatus = "ws closed" Log.w(TAG, "audio WS closed url=$url reason=$reason") scheduleReconnect() + } } override fun onFailure(error: Throwable) { + mainHandler.post { + if (!running || token != webSocketGeneration) return@post wsOpen = false audioStatus = "ws failed" - audioUrlIndex = (audioUrlIndex + 1) % urls.size lastError = "audio ws ${error.javaClass.simpleName}: ${error.message}" - Log.e(TAG, "audio WS failed url=$url; next=${urls[audioUrlIndex % urls.size]}", error) + Log.e(TAG, "audio WS failed url=$url", error) scheduleReconnect() + endpointController.retry() + } } }, + socketFactory = endpoint.network?.socketFactory, ) audioWs = ws ws.start() } - private fun pcBaseUrls(): List = - listOf(pcBaseUrl, USB_REVERSE_BASE_URL).distinct() - - private fun pcWsUrls(): List = - listOf(pcWsUrl, USB_REVERSE_WS_URL).distinct() - private fun scheduleReconnect() { if (!running) return + closeAudioWebSocket() val delay = reconnectDelayMs - reconnectDelayMs = (reconnectDelayMs * 2).coerceAtMost(5_000L) - mainHandler.postDelayed({ connectAudioWebSocket() }, delay) + reconnectDelayMs = ConnectionPolicy.nextDelay(reconnectDelayMs) + mainHandler.postDelayed(reconnectTask, delay) } private fun closeAudioWebSocket() { + mainHandler.removeCallbacks(reconnectTask) + webSocketGeneration++ wsOpen = false audioWs?.close() audioWs = null } private fun startAudioCapture() { + if (!running) return + if (audioThread?.isAlive == true) { + mainHandler.postDelayed({ startAudioCapture() }, 200) + return + } + val token = captureGeneration.get() audioThread = Thread({ val recorder = createAudioRecord() if (recorder == null) { @@ -341,10 +438,12 @@ class MainActivity : Activity() { val buf = ByteArray(CHUNK_BYTES) var lastLog = System.currentTimeMillis() try { + if (!running || token != captureGeneration.get()) return@Thread + activeRecorder = recorder recorder.startRecording() audioStatus = "recording" Log.i(TAG, "AudioRecord started") - while (running) { + while (running && token == captureGeneration.get()) { val n = recorder.read(buf, 0, buf.size) if (n <= 0) { audioDrops.incrementAndGet() @@ -382,6 +481,7 @@ class MainActivity : Activity() { } catch (_: Throwable) { } recorder.release() + if (activeRecorder === recorder) activeRecorder = null Log.i(TAG, "AudioRecord stopped") } }, "rokid-audio").also { it.start() } @@ -440,12 +540,15 @@ class MainActivity : Activity() { } private fun stopAudioCapture() { + try { activeRecorder?.stop() } catch (_: Exception) { } + audioThread?.interrupt() audioThread?.join(700) - audioThread = null + if (audioThread?.isAlive != true) audioThread = null } private fun startCameraCapture() { if (cameraThread != null || cameraDevice != null || imageReader != null) return + val token = cameraGeneration.incrementAndGet() cameraThread = HandlerThread("rokid-camera").also { it.start() } cameraHandler = Handler(cameraThread!!.looper) val manager = getSystemService(Context.CAMERA_SERVICE) as CameraManager @@ -466,7 +569,11 @@ class MainActivity : Activity() { 2, ).also { reader -> reader.setOnImageAvailableListener({ r -> - val image = r.acquireLatestImage() ?: return@setOnImageAvailableListener + val image = runCatching { r.acquireLatestImage() }.getOrNull() ?: return@setOnImageAvailableListener + if (!running || token != cameraGeneration.get()) { + image.close() + return@setOnImageAvailableListener + } try { val buffer: ByteBuffer = image.planes[0].buffer val bytes = ByteArray(buffer.remaining()) @@ -494,11 +601,13 @@ class MainActivity : Activity() { selection.cameraId, object : CameraDevice.StateCallback() { override fun onOpened(camera: CameraDevice) { + if (!running || token != cameraGeneration.get()) { camera.close(); return } cameraDevice = camera createSession(camera) } override fun onDisconnected(camera: CameraDevice) { + if (token != cameraGeneration.get()) { camera.close(); return } cameraStatus = "camera disconnected" lastError = cameraStatus Log.w(TAG, cameraStatus) @@ -507,6 +616,7 @@ class MainActivity : Activity() { } override fun onError(camera: CameraDevice, error: Int) { + if (token != cameraGeneration.get()) { camera.close(); return } cameraStatus = "camera error $error" lastError = cameraStatus Log.e(TAG, cameraStatus) @@ -514,7 +624,7 @@ class MainActivity : Activity() { restartCameraCapture(cameraStatus) } }, - cameraHandler, + mainHandler, ) } catch (t: Throwable) { cameraStatus = "camera failed" @@ -536,6 +646,7 @@ class MainActivity : Activity() { listOf(preview, jpegSurface), object : CameraCaptureSession.StateCallback() { override fun onConfigured(session: CameraCaptureSession) { + if (!running || camera !== cameraDevice) { session.close(); return } captureSession = session cameraStatus = "camera ready $imageSizeText" Log.i(TAG, cameraStatus) @@ -544,12 +655,13 @@ class MainActivity : Activity() { } override fun onConfigureFailed(session: CameraCaptureSession) { + if (!running || camera !== cameraDevice) { session.close(); return } cameraStatus = "session configure failed" lastError = cameraStatus Log.e(TAG, cameraStatus) } }, - cameraHandler, + mainHandler, ) } catch (t: Throwable) { lastError = "session ${t.javaClass.simpleName}: ${t.message}" @@ -631,11 +743,17 @@ class MainActivity : Activity() { private fun restartCameraCapture(reason: String, delayMs: Long = 800L) { if (!running) return + val token = captureGeneration.get() + val cameraToken = cameraGeneration.get() if (!cameraRestartPending.compareAndSet(false, true)) return cameraStatus = "camera restarting" lastError = "camera restart: $reason" Log.w(TAG, "camera restart requested: $reason") mainHandler.post { + if (!running || token != captureGeneration.get() || cameraToken != cameraGeneration.get()) { + cameraRestartPending.set(false) + return@post + } try { stopCameraCapture() } catch (t: Throwable) { @@ -643,7 +761,7 @@ class MainActivity : Activity() { } mainHandler.postDelayed({ cameraRestartPending.set(false) - if (running) { + if (running && token == captureGeneration.get()) { Log.i(TAG, "camera restart now") startCameraCapture() } @@ -757,57 +875,47 @@ class MainActivity : Activity() { private fun uploadJpeg(jpeg: ByteArray) { if (!running) return - if (!imageUploadInFlight.compareAndSet(false, true)) { - imageFailures.incrementAndGet() - Log.w(TAG, "drop image upload: previous POST still in flight") - return - } + val endpoint = currentEndpoint ?: return + val token = captureGeneration.get() + if (!imageUploadInFlight.compareAndSet(false, true)) return Thread({ - val prepareStart = System.currentTimeMillis() - val downsampledJpeg = maybeDownsampleJpegForUpload(jpeg) - val postJpeg = if (ENHANCE_IMAGE_FOR_UPLOAD) enhanceJpegForModel(downsampledJpeg) else downsampledJpeg - val prepareMs = System.currentTimeMillis() - prepareStart - var posted = false - var lastFailure: String? = null - for (baseUrl in pcBaseUrls()) { - imagePostBaseText = baseUrl - try { - val postStart = System.currentTimeMillis() - val conn = (URL("$baseUrl/rokid/image").openConnection() as HttpURLConnection).apply { + var conn: HttpURLConnection? = null + try { + if (!running || token != captureGeneration.get() || endpoint != currentEndpoint) return@Thread + val downsampledJpeg = maybeDownsampleJpegForUpload(jpeg) + val postJpeg = if (ENHANCE_IMAGE_FOR_UPLOAD) enhanceJpegForModel(downsampledJpeg) else downsampledJpeg + conn = endpoint.connection("/rokid/image").apply { requestMethod = "POST" connectTimeout = 1_500 readTimeout = 4_000 + instanceFollowRedirects = false doOutput = true setRequestProperty("Content-Type", "image/jpeg") - setRequestProperty("Content-Length", postJpeg.size.toString()) + setFixedLengthStreamingMode(postJpeg.size) } + imageConnection = conn + if (!running || token != captureGeneration.get() || endpoint != currentEndpoint) return@Thread + imagePostBaseText = endpoint.baseUrl conn.outputStream.use { it.write(postJpeg) } val code = conn.responseCode - conn.disconnect() - if (code in 200..299) { + if (code !in 200..299) throw java.io.IOException("HTTP $code") + if (running && token == captureGeneration.get() && endpoint == currentEndpoint) { val count = imagePackets.incrementAndGet() cameraStatus = "image ok $imageSizeText" - Log.i( - TAG, - "image POST ok base=$baseUrl count=$count bytes=${postJpeg.size} prepareMs=$prepareMs postMs=${System.currentTimeMillis() - postStart}", - ) - posted = true - break - } else { - lastFailure = "HTTP $code at $baseUrl" - Log.e(TAG, "image POST HTTP $code base=$baseUrl") + Log.i(TAG, "image POST ok base=${endpoint.baseUrl} count=$count bytes=${postJpeg.size}") } - } catch (t: Throwable) { - lastFailure = "${t.javaClass.simpleName}: ${t.message} at $baseUrl" - Log.w(TAG, "image POST failed base=$baseUrl bytes=${postJpeg.size}", t) + } catch (t: Exception) { + if (running && token == captureGeneration.get() && endpoint == currentEndpoint) { + imageFailures.incrementAndGet() + lastError = "image post ${t.javaClass.simpleName}: ${t.message}" + cameraStatus = "image post failed" + mainHandler.post { endpointController.retry() } } + } finally { + conn?.disconnect() + if (imageConnection === conn) imageConnection = null + imageUploadInFlight.set(false) } - if (!posted) { - imageFailures.incrementAndGet() - lastError = "image post failed: $lastFailure" - cameraStatus = "image post failed" - } - imageUploadInFlight.set(false) }, "rokid-image-post").start() } @@ -900,6 +1008,7 @@ class MainActivity : Activity() { } private fun stopCameraCapture() { + cameraGeneration.incrementAndGet() cameraHandler?.removeCallbacksAndMessages(null) try { captureSession?.close() @@ -1009,7 +1118,8 @@ class MainActivity : Activity() { hud.text = """ OpenGlass Rokid Sensor MODE: $mode - PC: img=$imagePostBaseText audio=$audioWsUrlText + CONNECTION: $connectionStatus + SERVER: img=$imagePostBaseText audio=$audioWsUrlText AUDIO: $audioStatus source=$audioSourceText wsOpen=$wsOpen pkt=${audioPackets.get()} drop=${audioDrops.get()} rms=${fmt(lastRms)} peak=$lastPeak nz=$lastNonZero IMAGE: $cameraStatus pkt=${imagePackets.get()} fail=${imageFailures.get()} size=$imageSizeText hiResDownsample=$HIGH_RES_CAPTURE_FOR_DOWNSAMPLE UP: ${up}s diff --git a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/SimpleBinaryWebSocket.kt b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/SimpleBinaryWebSocket.kt index 3ea644f..331e37f 100644 --- a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/SimpleBinaryWebSocket.kt +++ b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/SimpleBinaryWebSocket.kt @@ -11,15 +11,23 @@ import java.security.MessageDigest import java.security.SecureRandom import java.util.Locale import java.util.concurrent.ArrayBlockingQueue +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean +import javax.net.SocketFactory private const val WS_TAG = "OpenGlassRokid" private const val WS_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" +private val WS_WATCHDOG = Executors.newSingleThreadScheduledExecutor { r -> + Thread(r, "ws-write-watchdog").apply { isDaemon = true } +} class SimpleBinaryWebSocket( private val url: String, private val listener: Listener, private val queueCapacity: Int = 8, + private val socketFactory: SocketFactory? = null, ) { interface Listener { fun onOpen() @@ -30,12 +38,15 @@ class SimpleBinaryWebSocket( private val random = SecureRandom() private val open = AtomicBoolean(false) private val stopped = AtomicBoolean(false) + private val notified = AtomicBoolean(false) private val queue = ArrayBlockingQueue(queueCapacity) private val writeLock = Object() @Volatile private var socket: Socket? = null @Volatile private var output: BufferedOutputStream? = null private var readerThread: Thread? = null private var writerThread: Thread? = null + @Volatile private var writeStartedNanos = 0L + @Volatile private var watchdog: ScheduledFuture<*>? = null fun start() { readerThread = Thread({ runReader() }, "simple-ws-reader").also { it.start() } @@ -44,15 +55,15 @@ class SimpleBinaryWebSocket( fun close() { stopped.set(true) open.set(false) - try { - writeFrame(0x8, ByteArray(0)) - } catch (_: Throwable) { - } + // Closing a stale network must never block the main thread on a socket write. try { socket?.close() } catch (_: Throwable) { } socket = null + writerThread?.interrupt() + queue.clear() + watchdog?.cancel(false) } fun isOpen(): Boolean = open.get() @@ -74,9 +85,12 @@ class SimpleBinaryWebSocket( append(if (uri.rawPath.isNullOrEmpty()) "/" else uri.rawPath) if (!uri.rawQuery.isNullOrEmpty()) append('?').append(uri.rawQuery) } - val sock = Socket() + val sock = socketFactory?.createSocket() ?: Socket() socket = sock + if (stopped.get()) return sock.connect(java.net.InetSocketAddress(uri.host, port), 3_000) + if (stopped.get()) return + sock.soTimeout = 5_000 val input = BufferedInputStream(sock.getInputStream()) val out = BufferedOutputStream(sock.getOutputStream()) output = out @@ -86,7 +100,7 @@ class SimpleBinaryWebSocket( val key = Base64.encodeToString(keyBytes, Base64.NO_WRAP) val request = ( "GET $path HTTP/1.1\r\n" + - "Host: ${uri.host}:$port\r\n" + + "Host: ${uri.rawAuthority}\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + "Sec-WebSocket-Key: $key\r\n" + @@ -112,7 +126,18 @@ class SimpleBinaryWebSocket( throw IOExceptionCompat("bad websocket accept") } + if (stopped.get()) return + sock.soTimeout = 0 open.set(true) + watchdog = WS_WATCHDOG.scheduleAtFixedRate({ + val started = writeStartedNanos + if (!stopped.get() && started != 0L && System.nanoTime() - started >= TimeUnit.SECONDS.toNanos(10)) { + if (notified.compareAndSet(false, true)) { + listener.onFailure(IOExceptionCompat("websocket write timed out")) + } + close() + } + }, 1, 1, TimeUnit.SECONDS) listener.onOpen() writerThread = Thread({ runWriter() }, "simple-ws-writer").also { it.start() } @@ -121,13 +146,16 @@ class SimpleBinaryWebSocket( if (opcode == 0x8) break } open.set(false) - listener.onClosed("closed") + if (!stopped.get() && notified.compareAndSet(false, true)) listener.onClosed("closed") } catch (t: Throwable) { open.set(false) - if (!stopped.get()) listener.onFailure(t) + if (!stopped.get() && notified.compareAndSet(false, true)) listener.onFailure(t) } finally { stopped.set(true) open.set(false) + writerThread?.interrupt() + queue.clear() + watchdog?.cancel(false) try { socket?.close() } catch (_: Throwable) { @@ -143,7 +171,8 @@ class SimpleBinaryWebSocket( } } catch (t: Throwable) { open.set(false) - if (!stopped.get()) listener.onFailure(t) + if (!stopped.get() && notified.compareAndSet(false, true)) listener.onFailure(t) + try { socket?.close() } catch (_: Throwable) { } } } @@ -159,9 +188,10 @@ class SimpleBinaryWebSocket( } else if (len == 127) { var longLen = 0L repeat(8) { longLen = (longLen shl 8) or readByte(input).toLong() } - if (longLen > Int.MAX_VALUE) throw IOExceptionCompat("websocket frame too large") + if (longLen < 0 || longLen > 1_048_576) throw IOExceptionCompat("websocket frame too large") len = longLen.toInt() } + if (len > 1_048_576 || (opcode >= 0x8 && len > 125)) throw IOExceptionCompat("websocket frame too large") val mask = if (masked) ByteArray(4) { readByte(input).toByte() } else null val payload = ByteArray(len) var off = 0 @@ -184,6 +214,9 @@ class SimpleBinaryWebSocket( private fun writeFrame(opcode: Int, payload: ByteArray) { val out = output ?: return synchronized(writeLock) { + if (stopped.get()) return + writeStartedNanos = System.nanoTime() + try { out.write(0x80 or (opcode and 0x0f)) val len = payload.size when { @@ -204,6 +237,9 @@ class SimpleBinaryWebSocket( out.write(mask) for (i in payload.indices) out.write(payload[i].toInt() xor mask[i % 4].toInt()) out.flush() + } finally { + writeStartedNanos = 0L + } } } diff --git a/rokid_app/OpenGlassRokidSensor/app/src/test/java/org/opensqz/openglass/rokid/ConnectionPolicyTest.kt b/rokid_app/OpenGlassRokidSensor/app/src/test/java/org/opensqz/openglass/rokid/ConnectionPolicyTest.kt new file mode 100644 index 0000000..b38acb0 --- /dev/null +++ b/rokid_app/OpenGlassRokidSensor/app/src/test/java/org/opensqz/openglass/rokid/ConnectionPolicyTest.kt @@ -0,0 +1,52 @@ +package org.opensqz.openglass.rokid + +import org.junit.Assert.* +import org.junit.Test + +class ConnectionPolicyTest { + @Test fun acceptsManualIpv4AndIpv6WithoutChangingPorts() { + assertEquals("http://172.20.10.1:18080", ConnectionPolicy.normalizeUrl(" http://172.20.10.1:18080/ ")) + assertEquals("ws://[fe80::1%25wlan0]:18080/rokid/audio", + ConnectionPolicy.websocketUrl("http://[fe80::1%25wlan0]:18080")) + } + + @Test fun rejectsCredentialsPathsQueriesAndInvalidPorts() { + listOf("https://phone:18080", "http://user:pass@phone", "http://phone/path", + "http://phone?token=x", "http://phone#x", "http://phone:70000", "http://phone:0") + .forEach { value -> + assertTrue("Should reject $value", runCatching { ConnectionPolicy.normalizeUrl(value) }.isFailure) + } + } + + @Test fun loopbackStaysOffWifiBoundSockets() { + assertTrue(ConnectionPolicy.isLoopback(USB_BASE_URL)) + assertTrue(ConnectionPolicy.isLoopback("http://[::1]:18080")) + assertFalse(ConnectionPolicy.isLoopback("http://172.20.10.1:18080")) + } + + @Test fun newSelectionRequiresExactlyOnePhone() { + assertEquals("OpenGlass-one", ConnectionPolicy.selectService(setOf("OpenGlass-one"), null)) + assertNull(ConnectionPolicy.selectService(setOf("OpenGlass-one", "OpenGlass-two"), null)) + assertNull(ConnectionPolicy.selectService(emptySet(), null)) + } + + @Test fun doesNotReplaceAnUnavailablePreviouslySelectedPhone() { + assertNull(ConnectionPolicy.selectService(setOf("OpenGlass-other"), "OpenGlass-mine")) + assertEquals("OpenGlass-mine", ConnectionPolicy.selectService( + setOf("OpenGlass-other", "OpenGlass-mine"), "OpenGlass-mine")) + } + + @Test fun backoffIsBounded() { + assertEquals(2_000L, ConnectionPolicy.nextDelay(1_000)) + assertEquals(30_000L, ConnectionPolicy.nextDelay(16_000)) + assertEquals(30_000L, ConnectionPolicy.nextDelay(30_000)) + } + + @Test fun hotspotAndSavedIpMustStillMatchTheRememberedPhone() { + assertTrue(ConnectionPolicy.receiverMatches("OpenGlass-1234ABcd", null)) + assertTrue(ConnectionPolicy.receiverMatches("OpenGlass-1234ABcd", "OpenGlass-1234ABcd")) + assertFalse(ConnectionPolicy.receiverMatches("OpenGlass-1234ABcd", "OpenGlass-99999999")) + assertFalse(ConnectionPolicy.receiverMatches("", null)) + assertFalse(ConnectionPolicy.receiverMatches("some-other-service", null)) + } +} diff --git a/rokid_app/README.md b/rokid_app/README.md index 6827402..093a037 100644 --- a/rokid_app/README.md +++ b/rokid_app/README.md @@ -2,7 +2,7 @@ [中文说明](README_zh.md) -This directory contains the adapted OpenGlass Rokid sensor app 0.1.3 and a single Android project. It captures JPEG/PCM and accepts a dynamic `pc_base_url`. Use the repository-root `glasses_panel.py` for daily startup. +This directory contains OpenGlass Rokid sensor app 0.2.0 (version code 5). It captures JPEG/PCM and can connect directly to OpenGlass on iPhone over Wi-Fi. The existing PC panel and explicit USB relay modes remain available. ## Companion setup @@ -14,25 +14,37 @@ Prepare JDK 17 (or a JDK compatible with Gradle 8.7 / AGP 8.5.2), Android SDK AP ```powershell # Build only -.\rokid_app\build_install.cmd -Sdk "" +.\rokid_app\build_install.cmd -Sdk "" -Variant Wifi # Build and install after connecting/authorizing USB -.\rokid_app\build_install.cmd -Sdk "" -Install -Serial "" +.\rokid_app\build_install.cmd -Sdk "" -Variant Wifi -Install -Serial "" ``` Use `adb devices -l` to obtain the serial. Optional `-Adb` selects an executable. The CMD delegates to `scripts/build_install.ps1`, which can also be called directly with the same options. A failed build never triggers installation. Multiple online entries require a serial; USB and wireless connections appear separately. Signature mismatches stop installation; the script never uninstalls an existing app. Allow camera/microphone permissions on the device. -Generated APK: `OpenGlassRokidSensor/app/build/outputs/apk/debug/app-debug.apk` (ignored by Git). Equivalent manual commands from the Android project root, with SDK/JDK configured: +The Wi-Fi variant is **OpenGlass WiFi**, package `org.opensqz.openglass.rokid.wifi`, version `0.2.0-wifi`. It can coexist with a collector signed on another computer, without uninstalling it or clearing its data. Run only one collector at a time. Its settings and permissions start separately. The original `-Variant Debug` (default) and release package IDs remain unchanged. + +Generated Wi-Fi APK: `OpenGlassRokidSensor/app/build/outputs/apk/wifi/app-wifi.apk` (ignored by Git). Equivalent manual commands from the Android project root, with SDK/JDK configured: ```powershell -.\gradlew.bat :app:assembleDebug +.\gradlew.bat :app:assembleWifi :app:testWifiUnitTest # Only after a successful build: -adb -s install -r .\app\build\outputs\apk\debug\app-debug.apk +adb -s install -r .\app\build\outputs\apk\wifi\app-wifi.apk ``` -## Configure and run +## iPhone hotspot or shared Wi-Fi + +Join the glasses to the iPhone hotspot using the official Rokid companion app/device settings, or join both devices to the same Wi-Fi. This app does not change Wi-Fi settings. Open iPhone OpenGlass, load its model and start its receiver, then launch this sensor app on the glasses. After camera/microphone permission is granted it starts capturing automatically. Leave both apps in the foreground. No computer or wireless ADB is needed for transport after installation. + +Default **Auto** mode checks the current Wi-Fi default gateway on port 18080 and discovers `_openglass._tcp.` with Bonjour. A saved non-loopback manual address is a fallback. Before sending sensor data, every automatic candidate must return `service=openglass-ios`, `protocol=1`, and its stable `receiver_name` from `/health`. The receiver name is remembered: moving networks may change its address, but does not silently select a different phone. Multiple unknown Bonjour receivers require manual selection/address entry. Hotel networks may isolate clients; use the iPhone hotspot if that happens. + +Auto mode never falls back to USB, and ignores loopback addresses saved by older versions. Audio and images share one selected endpoint. Reconnect delays increase from 1 second to a maximum of 30 seconds while the app remains running; network changes trigger discovery again. This is bounded backoff, not a promise to bypass a hotspot that the system disconnected. + +The visible **Connection / 连接设置** button (also Menu or a long press on the status text) offers Auto, Manual address, USB relay, and Choose phone again. Manual accepts `http://
:18080`; USB selects `http://127.0.0.1:18080` and requires the existing computer USB relay. Choose phone again forgets the remembered receiver. Tap the status text or press Enter/DPAD to stop/start; Back stops, then exits. + +Launch intents remain compatible: an explicit `pc_base_url` fixes the endpoint for that launch and persists the URL as a future fallback. `auto_discover=true` explicitly selects Auto and overrides the old saved USB address. The connection-settings mode persists, while an intent URL alone does not turn a future launcher start into USB mode. -No PC IP is needed at build time. Loopback defaults require an ADB reverse tunnel. The panel supplies and the app persists `pc_base_url` at launch. Configure the shared backend and `rokid_adb`, `rokid_mode`, `rokid_pc_url` in ignored `runtime/openglass_omni/runtime.local.json`; use the PC LAN IPv4 reachable from the glasses. Then run `python glasses_panel.py`. +## Existing PC workflow -Save a working Wi-Fi network on the glasses first. Keep USB attached until the panel establishes wireless ADB and sensor input is confirmed. Later PC/panel restarts can reuse wireless ADB. Glasses power loss may require USB recovery, but does not uninstall the app. A changed PC IP requires a local JSON edit and panel restart, not an APK rebuild. +No PC IP is needed at build time. The PC panel supplies `pc_base_url` at launch, which remains a fixed direct connection without requiring the iPhone health schema. Configure `rokid_adb`, `rokid_mode`, and `rokid_pc_url` in ignored `runtime/openglass_omni/runtime.local.json`, then run `python glasses_panel.py`. The PC workflow may use wireless ADB for launch; it is separate from the direct iPhone Wi-Fi transport above. -See [runtime setup](../runtime/openglass_omni/STARTUP_en.md) for recording and Harness prompts. +See [iPhone setup](../ios/README_zh.md) or [PC runtime setup](../runtime/openglass_omni/STARTUP_en.md). Model inference, speech recognition and multi-turn responses belong to the receiving iPhone/PC app, not this sensor collector. diff --git a/rokid_app/README_zh.md b/rokid_app/README_zh.md index a5a09ce..f3fcf13 100644 --- a/rokid_app/README_zh.md +++ b/rokid_app/README_zh.md @@ -1,6 +1,6 @@ # Rokid 采集 App -此目录保存面板使用的 OpenGlass Rokid 采集 App 0.1.3：发送 JPEG 与 PCM 音频，接受面板传入的 `pc_base_url`。它不是眼镜系统固件。 +此目录保存 OpenGlass Rokid 采集 App 0.2.0（versionCode 5）：发送 JPEG 与 PCM 音频，可通过 Wi-Fi 直接连接 iPhone OpenGlass，也保留电脑面板和显式 USB 中继模式。它不是眼镜系统固件。 ## 文件布局 @@ -8,7 +8,7 @@ - `scripts/build_install.ps1`：构建，可选安装；不启动模型或旧 bridge。 - `build_install.cmd`：同一脚本的 Windows 便捷入口。 -安装完成后，使用仓库根的 `glasses_panel.py` 启动模型与采集服务。 +安装后可以直接打开眼镜上的采集 App；电脑后端仍可使用仓库根的 `glasses_panel.py`。 ## 手机设置 @@ -21,35 +21,45 @@ 推荐在 OpenGlass 根目录运行整理后的脚本。**先只编译**： ```powershell -.\rokid_app\build_install.cmd -Sdk "" +.\rokid_app\build_install.cmd -Sdk "" -Variant Wifi ``` **编译并安装**：连接 USB、确认授权，用 `adb devices -l` 取得设备序列号，再运行： ```powershell -.\rokid_app\build_install.cmd -Sdk "" -Install -Serial "" +.\rokid_app\build_install.cmd -Sdk "" -Variant Wifi -Install -Serial "" ``` 需要指定 ADB 时增加 `-Adb ""`。也可直接调用 `scripts/build_install.ps1`，参数相同。脚本确认编译成功后才安装；设备不唯一时要求指定序列号。USB 和无线 ADB 同时在线会显示两项。 -APK 输出：`OpenGlassRokidSensor/app/build/outputs/apk/debug/app-debug.apk`。这是生成物，不提交 Git。签名不匹配时脚本会停止，**不会自动卸载**。只有确认可以清除旧采集 App 数据和权限后，才由设备所有者处理旧版本。安装后允许相机和麦克风权限。 +Wi-Fi 版显示名为 **OpenGlass WiFi**，包名 `org.opensqz.openglass.rokid.wifi`，版本 `0.2.0-wifi`。它可与其他电脑签名的旧采集 App 并存，不需要卸载或清除旧数据；同一时间只运行一个采集 App。新包的设置和权限需要单独配置。原 `-Variant Debug`（默认）与 release 的包名保留不变。 + +Wi-Fi APK 输出：`OpenGlassRokidSensor/app/build/outputs/apk/wifi/app-wifi.apk`。这是生成物，不提交 Git。签名不匹配时脚本会停止，**不会自动卸载**。安装后允许相机和麦克风权限。 ### 等价手动命令 已配置好 SDK/JDK 时，从 Android 工程目录执行： ```powershell -.\gradlew.bat :app:assembleDebug +.\gradlew.bat :app:assembleWifi :app:testWifiUnitTest # 上一步成功后再安装 -adb -s install -r .\app\build\outputs\apk\debug\app-debug.apk +adb -s install -r .\app\build\outputs\apk\wifi\app-wifi.apk ``` -## 地址配置与日常启动 +## iPhone 热点或同一个 Wi-Fi + +先通过乐奇官方 App 或眼镜网络设置，将眼镜连到 iPhone 个人热点；也可以让两台设备加入同一 Wi-Fi。采集 App 不修改 Wi-Fi 配置。打开 iPhone OpenGlass，加载模型并启动接收服务，再打开眼镜上的采集 App。相机和麦克风授权后自动开始采集。两端 App 保持前台；安装完成后，无线传输不需要电脑或无线 ADB。 + +默认 **Auto 自动模式**会探测当前 Wi-Fi 默认网关的 18080 端口，并用 Bonjour 发现 `_openglass._tcp.` 服务；保存过的非回环手动地址可作为后备。每个自动候选必须先通过 `/health` 检查：`service=openglass-ios`、`protocol=1`，以及稳定的 `receiver_name`。首次选中的手机名称会保存；换网络可以更新地址，但不会自动改连另一台手机。首次发现多台未知手机时需要明确选择地址。酒店网络可能隔离终端，无法互通时使用 iPhone 热点。 + +自动模式**不回退 USB**，会忽略旧版保存的 `127.0.0.1`。图像和音频共用一个已选地址；运行中断线按 1 秒起步、最多 30 秒的间隔重试，换网后重新发现。这不替代系统连接热点的步骤。 + +界面上的 **Connection / 连接设置** 按钮（也可用 Menu 或长按状态文字）提供：Auto、Manual address、USB relay、Choose phone again。手动地址格式为 `http://<地址>:18080`；USB 模式使用 `http://127.0.0.1:18080`，仍需要电脑上的既有 USB 中继。Choose phone again 会忘记已选择的手机。点击状态文字或 Enter/DPAD 可启停；Back 先停止，再退出。 -**编译时不填写电脑 IP。** 工程默认地址是眼镜回环地址，仅用于已有 ADB reverse 的情况；面板每次启动通过 `pc_base_url` 覆盖该地址，采集 App 会保存它。 +启动参数仍兼容：显式 `pc_base_url` 会在本次启动固定连接该地址，并保存为以后自动模式的后备地址；`auto_discover=true` 显式切回自动，覆盖旧保存的 USB 地址。设置界面选择的模式会保存，仅传入一次 URL 不会把以后的普通启动永久绑定到 USB。 -安装完成后按[中文启动指南](../runtime/openglass_omni/STARTUP_zh.md)设置 `runtime.local.json`：填写后端路径、`rokid_adb`、`rokid_mode: "wifi"` 和 `rokid_pc_url: "http://:18080"`。电脑 IP 从 `ipconfig` 中选择与眼镜互通的网卡地址，不是眼镜 IP。 +## 电脑面板工作流 -先在设备/手机端保存 Wi-Fi 网络。首次保持 USB 连接，面板建立无线 ADB，确认音视频传入后拔线；以后从面板启动即可。换电脑 IP 只改本地 JSON，无需再编译 APK。眼镜断电可能重置 Wi-Fi/无线调试，需要 USB 恢复，但不会卸载 App。 +**编译时不填写电脑 IP。** 面板启动时传入的 `pc_base_url` 仍固定直连，不要求电脑提供 iPhone 的 health 字段。按[电脑中文启动指南](../runtime/openglass_omni/STARTUP_zh.md)设置 `runtime.local.json` 中的 `rokid_adb`、`rokid_mode`、`rokid_pc_url`，然后运行 `glasses_panel.py`。电脑面板的无线 ADB 启动流程与上述 iPhone 直连传输是两件事。 -详细录制与 Harness 关键词/提示词配置见启动指南。眼镜采集 App 不承担 Harness、ASR 或模型推理。 +iPhone 模型、签名安装和多轮对话见 [iPhone 中文指南](../ios/README_zh.md)。采集 App 不承担 ASR、TTS 或模型推理。 diff --git a/rokid_app/scripts/build_install.ps1 b/rokid_app/scripts/build_install.ps1 index 90d57e9..ce27a4b 100644 --- a/rokid_app/scripts/build_install.ps1 +++ b/rokid_app/scripts/build_install.ps1 @@ -2,7 +2,8 @@ param( [switch]$Install, [string]$Serial = "", [string]$Adb = $env:ROKID_ADB, - [string]$Sdk = $env:ANDROID_HOME + [string]$Sdk = $env:ANDROID_HOME, + [ValidateSet('Debug', 'Wifi')][string]$Variant = 'Debug' ) $ErrorActionPreference = "Stop" @@ -35,12 +36,13 @@ if ($Install) { Push-Location $projectRoot try { - & .\gradlew.bat :app:assembleDebug + & .\gradlew.bat ":app:assemble$Variant" if ($LASTEXITCODE -ne 0) { throw "Gradle build failed; nothing installed." } } finally { Pop-Location } -$apk = Join-Path $projectRoot "app\build\outputs\apk\debug\app-debug.apk" +$variantFolder = $Variant.ToLowerInvariant() +$apk = Join-Path $projectRoot "app\build\outputs\apk\$variantFolder\app-$variantFolder.apk" if (-not (Test-Path -LiteralPath $apk)) { throw "Build did not produce the expected APK." } Write-Host "Built: $apk" if ($Install) { @@ -50,4 +52,4 @@ if ($Install) { } Write-Host "Installed. Allow Camera and Microphone on the glasses when prompted." } -Write-Host "Configure runtime/openglass_omni/runtime.local.json, then start glasses_panel.py. No build-time PC IP is needed." +Write-Host "For iPhone: join its hotspot/shared Wi-Fi and open both OpenGlass apps (Auto mode). For PC: configure runtime.local.json and start glasses_panel.py." diff --git a/runtime/openglass_omni/rokid_device.py b/runtime/openglass_omni/rokid_device.py index b374e25..d56bfb4 100644 --- a/runtime/openglass_omni/rokid_device.py +++ b/runtime/openglass_omni/rokid_device.py @@ -291,8 +291,8 @@ def launch(self): rc, info = self.run("shell", "dumpsys", "package", self.config.get("rokid_package", "org.opensqz.openglass.rokid.debug")) - if rc or not re.search(r"versionName=0\.1\.3(?:-debug)?(?:\s|$)", info): - raise RuntimeError("尚未确认动态地址版本 0.1.3 已安装；" + if rc or not re.search(r"versionName=(?:0\.1\.3|0\.2\.0)(?:-(?:debug|wifi))?(?:\s|$)", info): + raise RuntimeError("尚未确认动态地址版本 0.1.3 / 0.2.0 已安装；" "旧版会忽略 pc_base_url，请先升级采集程序") self.run("shell", "input", "keyevent", "KEYCODE_WAKEUP") rc, out = self.run("shell", "am", "start", "-S", "-W", "-n", activity, diff --git a/tests/test_rokid_device.py b/tests/test_rokid_device.py index bfc12b1..f3743d7 100644 --- a/tests/test_rokid_device.py +++ b/tests/test_rokid_device.py @@ -30,11 +30,13 @@ def test_ambiguous_rokids_are_rejected(self): with self.assertRaises(RuntimeError): d.select() def test_launch_passes_configured_address(self): - d = self.device() - d.run = Mock(return_value=(0, "versionName=0.1.3-debug\nStatus: ok")) - d.launch() - args = d.run.call_args.args - self.assertEqual(args[-3:], ("--es", "pc_base_url", "http://192.168.1.5:18080")) + for version in ("0.1.3-debug", "0.2.0-debug", "0.2.0-wifi", "0.2.0"): + with self.subTest(version=version): + d = self.device() + d.run = Mock(return_value=(0, f"versionName={version}\nStatus: ok")) + d.launch() + args = d.run.call_args.args + self.assertEqual(args[-3:], ("--es", "pc_base_url", "http://192.168.1.5:18080")) def test_old_app_does_not_silently_ignore_new_address(self): d = self.device();d.run = Mock(return_value=(0, "versionName=0.1.2-debug")) From 66250e40fc65b8719f66d22e3610dc067175c358 Mon Sep 17 00:00:00 2001 From: MengzhangLI Date: Sun, 18 Oct 2026 02:47:54 +0800 Subject: [PATCH 07/11] docs: record unplugged wireless conversation validation [skip ci] --- ios/README_zh.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ios/README_zh.md b/ios/README_zh.md index d1db0f9..f4c00e6 100644 --- a/ios/README_zh.md +++ b/ios/README_zh.md @@ -177,7 +177,7 @@ Invoke-RestMethod http://127.0.0.1:18080/health 1. iPhone“设置 → 个人热点 → 允许其他人加入”。眼镜找不到热点时可开启“最大兼容性”。首次连接时停留在热点页面，直到眼镜已加入。具体开关见 [Apple 个人热点说明](https://support.apple.com/zh-cn/111785)。热点依赖运营商支持；它分享的是蜂窝网络，不是酒店 Wi-Fi 的中继。 2. 通过 Rokid 官方 App / 设备的 Wi-Fi 设置，让眼镜加入这个热点；密码仅在设备设置内输入。OpenGlass 不修改 Wi-Fi 密码和系统网络。 3. iPhone 打开 OpenGlass，允许“本地网络”和语音识别；已有模型时自动启动接收，等待模型就绪即可。页面显示固定的 `OpenGlass-xxxxxxxx` 接收名称。请保持“Rokid 离线语音提问”和“朗读回答”开启。 -4. 启动眼镜 OpenGlassRokidSensor，使用“自动连接”。眼镜寻找 `_openglass._tcp` 服务，并探测当前 Wi-Fi 网关的 `:18080/health`；确认是 OpenGlass iOS 接收端后才发送音视频。地址变化后会重新发现，不把热点地址写死。 +4. 启动眼镜 **OpenGlass WiFi**，使用“自动连接”。眼镜寻找 `_openglass._tcp` 服务，并探测当前 Wi-Fi 网关的 `:18080/health`；确认是 OpenGlass iOS 接收端后才发送音视频。地址变化后会重新发现，不把热点地址写死。等手机显示“正在听 Rokid 麦克风（离线识别）”后开始说话。 5. 确认手机出现新画面、自动语音状态后，拔掉两根 USB，等待十秒并连续问三个问题。回答播完后再说下一个问题，不需要点“结束语句”或“用最新画面提问”。 眼镜暂留 USB 时，可在 Windows 手动切到自动发现模式；这条命令只做一次配置与启动，不是数据中继： @@ -191,6 +191,7 @@ powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File 'C:\path\to\OpenGl ```powershell .\ios\windows\configure-rokid.ps1 ` -IPhoneAddress '192.168.1.42' ` + -Package org.opensqz.openglass.rokid.wifi ` -Adb "$env:USERPROFILE\Downloads\platform-tools\adb.exe" ``` @@ -216,6 +217,7 @@ powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File 'C:\path\to\OpenGl | USB中继提示手机 `/health` 不可达 | 新App是否成功签名安装并已启动接收；是否进入后台；模型加载失败是否导致接收停止 | | 端口或ADB reverse已存在 | 在原会话中正常停止，或指定其他本机/眼镜端口；脚本不会杀进程抢占 | | `/health` 成功但计数不增长 | 眼镜相机/麦克风权限、采集App版本、ADB授权、所选设备；Wi-Fi模式另查网络互通 | +| 无线模式没有画面，眼镜显示 Wi-Fi 已开启 | 已开启不等于已连接。先确认眼镜已加入热点，再切回手机 OpenGlass；切到热点设置期间，接收服务会因进入后台而暂停。两端恢复后自动重连，无需重跑电脑脚本 | | 画面到达但模型未就绪 | 两个正确的4.6模型是否下载完成、磁盘空间、模型加载报错；其他App模型沙盒不共享 | | 有文本回答但没声音 | “朗读回答”开关、iPhone音量/音频路由、蓝牙配对；看界面实际输出设备 | | 语音不可用但文字正常 | 系统语音识别权限与该语言的设备端资源；该功能不向云识别降级 | @@ -228,7 +230,9 @@ powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File 'C:\path\to\OpenGl - [CI运行38890873970](https://github.com/OpenSQZ/OpenGlass/actions/runs/38890873970)成功，源提交为 `c11c87b39b5798f8b4218d3a47d5b6cc68e56091`：1399项wire protocol检查、7项工程准备测试、7项USB中继测试通过，并生成未签名的 `OpenGlass-1.0.ipa`（artifact `OpenGlass-iOS-unsigned-3`）。IPA SHA256 为 `DA76A28A87195C43EA80ECF11366724180480B5D6502DEA27A4D1AF800AD00DB`。 - 现场设备识别为 `iPhone18,2`、iOS `26.6.2`。新版 `1.0` / build `3` 已通过本机认证签名并安装；两个对应 4.6 模型已校验并导入。2026-10-18 用户反馈：USB 连接下眼镜图像、声音和 iPhone MiniCPM 推理已跑通，仍需手动进行单轮交互。本次无线与自动多轮改动另行验证，不能继承旧版的端到端结论。 - 无线/自动语音 iOS 源提交 `087766314c0d817a78b65c6e6a93d46b281d5576` 通过 [CI 39057876255](https://github.com/OpenSQZ/OpenGlass/actions/runs/39057876255)，生成 build `5`。1399 项协议检查、2087 项语音循环断言、7 项工程准备测试、7 项 USB relay 测试通过。IPA SHA256：`D5A8F689048DFE09E1C4F4AF4EE67E770E5FE01BE3883A0A4B414804B39BFCB7`。Windows 免费签名更新后，USB 查询已确认手机安装 build `5`，两份模型文件仍完整保留。 -- Rokid `0.2.0-wifi` / code `5` 的 `assembleWifi`、7 项 `testWifiUnitTest` 与原 `assembleDebug` 通过；APK SHA256：`A5A64849FEAC796D4688B60833E647FA3DACE1CB3F5A16BD8935AA0EA5623A13`。并存包 `org.opensqz.openglass.rokid.wifi` 已安装，旧 debug 采集 App 保留。眼镜已获得过 iPhone 热点 DHCP 地址；新 App 的权限、无线音视频和拔线后的自动连续对话仍需现场测试。 +- Rokid `0.2.0-wifi` / code `5` 的 `assembleWifi`、7 项 `testWifiUnitTest` 与原 `assembleDebug` 通过；APK SHA256：`A5A64849FEAC796D4688B60833E647FA3DACE1CB3F5A16BD8935AA0EA5623A13`。并存包 `org.opensqz.openglass.rokid.wifi` 已安装，旧 debug 采集 App 保留；新 App 的相机和麦克风权限已确认允许。 +- 2026-10-18 热点真机测试：眼镜直接向当前 iPhone 热点网关发送图像和音频，手机接收计数持续增长。拔掉眼镜 USB 后，ADB 已无设备，手机仍持续收流，`completed_turns` 增至 `3`。随后用户确认：无需点按钮即可连续问答，且眼镜和 iPhone 两根 USB 均拔掉后仍正常。已验证本机 ASR → MiniCPM-V 4.6 → TTS 的无线自动多轮流程；酒店 Wi-Fi 尚未实测。 +- 包含 Android 最终改动的提交 `c439fa3` 也通过 [CI 39058773419](https://github.com/OpenSQZ/OpenGlass/actions/runs/39058773419)。其 iOS Swift 源码与已安装的 build `5` 相同，无需再次安装。 - 不在此文档写入个人设备序列号、UDID、Apple账号或内网凭据。现场更新请用设备类型和验证结果描述。 可重复执行无设备测试： From 10edba08c7dde7e9c2ac9495ac820648821aee6b Mon Sep 17 00:00:00 2001 From: MengzhangLI Date: Sun, 18 Oct 2026 02:53:48 +0800 Subject: [PATCH 08/11] Simplify Rokid glasses display to one dim status line --- .../OpenGlassRokidSensor/app/build.gradle | 2 +- .../opensqz/openglass/rokid/MainActivity.kt | 60 +++++++++---------- rokid_app/README.md | 6 +- rokid_app/README_zh.md | 6 +- 4 files changed, 37 insertions(+), 37 deletions(-) diff --git a/rokid_app/OpenGlassRokidSensor/app/build.gradle b/rokid_app/OpenGlassRokidSensor/app/build.gradle index d2593d8..1694858 100644 --- a/rokid_app/OpenGlassRokidSensor/app/build.gradle +++ b/rokid_app/OpenGlassRokidSensor/app/build.gradle @@ -11,7 +11,7 @@ android { applicationId 'org.opensqz.openglass.rokid' minSdk 28 targetSdk 34 - versionCode 5 + versionCode 6 versionName '0.2.0' manifestPlaceholders = [appLabel: '@string/app_name'] diff --git a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt index 8524ced..f243d9b 100644 --- a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt +++ b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt @@ -24,17 +24,19 @@ import android.os.Bundle import android.os.Handler import android.os.HandlerThread import android.os.Looper +import android.text.TextUtils import android.util.Log import android.util.Range import android.util.Size import android.view.KeyEvent +import android.view.Gravity import android.view.Surface import android.view.View +import android.view.ViewGroup import android.view.WindowManager import android.widget.TextView import android.widget.Button import android.widget.EditText -import android.widget.LinearLayout import java.io.ByteArrayOutputStream import java.net.HttpURLConnection import java.nio.ByteBuffer @@ -152,34 +154,35 @@ class MainActivity : Activity() { configurePcAddress() window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) window.decorView.systemUiVisibility = - View.SYSTEM_UI_FLAG_FULLSCREEN or View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY + View.SYSTEM_UI_FLAG_FULLSCREEN or View.SYSTEM_UI_FLAG_HIDE_NAVIGATION or + View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY hud = TextView(this).apply { setBackgroundColor(Color.BLACK) - setTextColor(Color.rgb(40, 255, 110)) - textSize = 15f - typeface = android.graphics.Typeface.MONOSPACE - setPadding(18, 18, 18, 18) + setTextColor(Color.rgb(140, 140, 140)) + textSize = 20f + typeface = android.graphics.Typeface.DEFAULT + gravity = Gravity.CENTER + setSingleLine(true) + ellipsize = TextUtils.TruncateAt.END + val inset = (24 * resources.displayMetrics.density).toInt() + setPadding(inset, 0, inset, 0) } hud.setOnClickListener { toggleStartStop() } hud.setOnLongClickListener { showConnectionSettings(); true } - val connectionButton = Button(this).apply { - text = "Connection / 连接设置" - setOnClickListener { showConnectionSettings() } - } - setContentView(LinearLayout(this).apply { - orientation = LinearLayout.VERTICAL - setBackgroundColor(Color.BLACK) - addView(connectionButton) - addView(hud) - }) + setContentView(hud, ViewGroup.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT, + )) endpointController = EndpointController(this, { endpoint -> currentEndpoint = endpoint imageConnection?.disconnect() closeAudioWebSocket() reconnectDelayMs = 1_000L if (running && endpoint != null) connectAudioWebSocket() - }, { connectionStatus = it }) + }, { status -> + if (connectionStatus != status) Log.i(TAG, "Connection: $status") + connectionStatus = status + }) Log.i(TAG, "onCreate explicit=$explicitUrl") mainHandler.post(hudTick) @@ -1113,21 +1116,14 @@ class MainActivity : Activity() { } private fun updateHud() { - val up = if (running) (System.currentTimeMillis() - startMs) / 1000 else 0 - val mode = if (running) "RUNNING" else "STOPPED" - hud.text = """ - OpenGlass Rokid Sensor - MODE: $mode - CONNECTION: $connectionStatus - SERVER: img=$imagePostBaseText audio=$audioWsUrlText - AUDIO: $audioStatus source=$audioSourceText wsOpen=$wsOpen pkt=${audioPackets.get()} drop=${audioDrops.get()} rms=${fmt(lastRms)} peak=$lastPeak nz=$lastNonZero - IMAGE: $cameraStatus pkt=${imagePackets.get()} fail=${imageFailures.get()} size=$imageSizeText hiResDownsample=$HIGH_RES_CAPTURE_FOR_DOWNSAMPLE - UP: ${up}s - LAST ERROR: $lastError - - Enter/DPAD: start-stop - Back: stop / exit - """.trimIndent() + val status = when { + !hasPermissions() -> "请允许相机和麦克风" + !running -> "OpenGlass · 已暂停" + currentEndpoint == null || !wsOpen -> "OpenGlass · 连接中" + else -> "OpenGlass · 已连接" + } + // Keep the glasses display still; diagnostics remain in logcat. + if (hud.text.toString() != status) hud.text = status } private data class AudioStats(val rms: Double, val peak: Int, val nonZero: Int) diff --git a/rokid_app/README.md b/rokid_app/README.md index 093a037..c2d1727 100644 --- a/rokid_app/README.md +++ b/rokid_app/README.md @@ -2,7 +2,7 @@ [中文说明](README_zh.md) -This directory contains OpenGlass Rokid sensor app 0.2.0 (version code 5). It captures JPEG/PCM and can connect directly to OpenGlass on iPhone over Wi-Fi. The existing PC panel and explicit USB relay modes remain available. +This directory contains OpenGlass Rokid sensor app 0.2.0 (version code 6). It captures JPEG/PCM and can connect directly to OpenGlass on iPhone over Wi-Fi. The existing PC panel and explicit USB relay modes remain available. ## Companion setup @@ -39,7 +39,9 @@ Default **Auto** mode checks the current Wi-Fi default gateway on port 18080 and Auto mode never falls back to USB, and ignores loopback addresses saved by older versions. Audio and images share one selected endpoint. Reconnect delays increase from 1 second to a maximum of 30 seconds while the app remains running; network changes trigger discovery again. This is bounded backoff, not a promise to bypass a hotspot that the system disconnected. -The visible **Connection / 连接设置** button (also Menu or a long press on the status text) offers Auto, Manual address, USB relay, and Choose phone again. Manual accepts `http://
:18080`; USB selects `http://127.0.0.1:18080` and requires the existing computer USB relay. Choose phone again forgets the remembered receiver. Tap the status text or press Enter/DPAD to stop/start; Back stops, then exits. +The default display is one dim, centered status line on black: connected, connecting, or paused. Text changes only with connection state; counters, addresses, and scrolling diagnostics stay off the display. Diagnostics remain available in logcat. Connected means the audio connection is established, not that iPhone ASR is currently ready. + +**Long-press the display or press Menu** to open connection settings: Auto, Manual address, USB relay, and Choose phone again. Manual accepts `http://
:18080`; USB selects `http://127.0.0.1:18080` and requires the existing computer USB relay. Choose phone again forgets the remembered receiver. Tap the display or press Enter/DPAD to stop/start; Back stops, then exits. Launch intents remain compatible: an explicit `pc_base_url` fixes the endpoint for that launch and persists the URL as a future fallback. `auto_discover=true` explicitly selects Auto and overrides the old saved USB address. The connection-settings mode persists, while an intent URL alone does not turn a future launcher start into USB mode. diff --git a/rokid_app/README_zh.md b/rokid_app/README_zh.md index f3fcf13..d4dbf3b 100644 --- a/rokid_app/README_zh.md +++ b/rokid_app/README_zh.md @@ -1,6 +1,6 @@ # Rokid 采集 App -此目录保存 OpenGlass Rokid 采集 App 0.2.0（versionCode 5）：发送 JPEG 与 PCM 音频，可通过 Wi-Fi 直接连接 iPhone OpenGlass，也保留电脑面板和显式 USB 中继模式。它不是眼镜系统固件。 +此目录保存 OpenGlass Rokid 采集 App 0.2.0（versionCode 6）：发送 JPEG 与 PCM 音频，可通过 Wi-Fi 直接连接 iPhone OpenGlass，也保留电脑面板和显式 USB 中继模式。它不是眼镜系统固件。 ## 文件布局 @@ -54,7 +54,9 @@ adb -s install -r .\app\build\outputs\apk\wifi\app-wifi.apk 自动模式**不回退 USB**，会忽略旧版保存的 `127.0.0.1`。图像和音频共用一个已选地址；运行中断线按 1 秒起步、最多 30 秒的间隔重试，换网后重新发现。这不替代系统连接热点的步骤。 -界面上的 **Connection / 连接设置** 按钮（也可用 Menu 或长按状态文字）提供：Auto、Manual address、USB relay、Choose phone again。手动地址格式为 `http://<地址>:18080`；USB 模式使用 `http://127.0.0.1:18080`，仍需要电脑上的既有 USB 中继。Choose phone again 会忘记已选择的手机。点击状态文字或 Enter/DPAD 可启停；Back 先停止，再退出。 +眼镜主界面采用黑底、较暗的居中单行提示：`OpenGlass · 已连接`、`OpenGlass · 连接中` 或 `OpenGlass · 已暂停`。只在状态变化时更新，不显示计数、地址或滚动调试文字；诊断信息仍可从 logcat 查看。“已连接”表示音频连接已建立，不代表 iPhone 此刻已准备好识别。 + +**长按界面或按 Menu** 打开连接设置，提供 Auto、Manual address、USB relay、Choose phone again。手动地址格式为 `http://<地址>:18080`；USB 模式使用 `http://127.0.0.1:18080`，仍需要电脑上的既有 USB 中继。Choose phone again 会忘记已选择的手机。短按界面或 Enter/DPAD 可启停；Back 先停止，再退出。 启动参数仍兼容：显式 `pc_base_url` 会在本次启动固定连接该地址，并保存为以后自动模式的后备地址；`auto_discover=true` 显式切回自动，覆盖旧保存的 USB 地址。设置界面选择的模式会保存，仅传入一次 URL 不会把以后的普通启动永久绑定到 USB。 From 910a38c2545533eaf9743260121b855e75cf28df Mon Sep 17 00:00:00 2001 From: MengzhangLI Date: Sun, 18 Oct 2026 03:02:39 +0800 Subject: [PATCH 09/11] Prevent glasses confirmation keys from pausing capture or opening settings --- .../OpenGlassRokidSensor/app/build.gradle | 2 +- .../opensqz/openglass/rokid/MainActivity.kt | 59 ++++++++++++++----- .../app/src/main/res/values/styles.xml | 8 +++ rokid_app/README.md | 6 +- rokid_app/README_zh.md | 6 +- 5 files changed, 62 insertions(+), 19 deletions(-) diff --git a/rokid_app/OpenGlassRokidSensor/app/build.gradle b/rokid_app/OpenGlassRokidSensor/app/build.gradle index 1694858..b1dbdb2 100644 --- a/rokid_app/OpenGlassRokidSensor/app/build.gradle +++ b/rokid_app/OpenGlassRokidSensor/app/build.gradle @@ -11,7 +11,7 @@ android { applicationId 'org.opensqz.openglass.rokid' minSdk 28 targetSdk 34 - versionCode 6 + versionCode 7 versionName '0.2.0' manifestPlaceholders = [appLabel: '@string/app_name'] diff --git a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt index f243d9b..64eca6d 100644 --- a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt +++ b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt @@ -11,6 +11,7 @@ import android.graphics.BitmapFactory import android.graphics.Color import android.graphics.ImageFormat import android.graphics.SurfaceTexture +import android.graphics.drawable.ColorDrawable import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice @@ -21,6 +22,7 @@ import android.media.AudioRecord import android.media.ImageReader import android.media.MediaRecorder import android.os.Bundle +import android.os.Build import android.os.Handler import android.os.HandlerThread import android.os.Looper @@ -35,7 +37,6 @@ import android.view.View import android.view.ViewGroup import android.view.WindowManager import android.widget.TextView -import android.widget.Button import android.widget.EditText import java.io.ByteArrayOutputStream import java.net.HttpURLConnection @@ -113,6 +114,7 @@ class MainActivity : Activity() { @Volatile private var audioSourceText = "none" @Volatile private var cameraStatus = "idle" @Volatile private var audioStatus = "idle" + @Volatile private var microphoneSilenced: Boolean? = null @Volatile private var imageSizeText = "none" @Volatile private var audioWsUrlText = "none" @Volatile private var imagePostBaseText = "none" @@ -153,12 +155,20 @@ class MainActivity : Activity() { super.onCreate(savedInstanceState) configurePcAddress() window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + // Keep every layer opaque black, including the window before the HUD is drawn. + window.setBackgroundDrawable(ColorDrawable(Color.BLACK)) + window.decorView.setBackgroundColor(Color.BLACK) + window.statusBarColor = Color.BLACK + window.navigationBarColor = Color.BLACK window.decorView.systemUiVisibility = View.SYSTEM_UI_FLAG_FULLSCREEN or View.SYSTEM_UI_FLAG_HIDE_NAVIGATION or View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY hud = TextView(this).apply { setBackgroundColor(Color.BLACK) + // The full-screen clickable TextView receives D-pad focus. The platform's + // default focus highlight otherwise paints a grey rectangle over its black background. + defaultFocusHighlightEnabled = false setTextColor(Color.rgb(140, 140, 140)) textSize = 20f typeface = android.graphics.Typeface.DEFAULT @@ -168,7 +178,7 @@ class MainActivity : Activity() { val inset = (24 * resources.displayMetrics.density).toInt() setPadding(inset, 0, inset, 0) } - hud.setOnClickListener { toggleStartStop() } + hud.setOnClickListener { resumeCapture() } hud.setOnLongClickListener { showConnectionSettings(); true } setContentView(hud, ViewGroup.LayoutParams( ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT, @@ -224,7 +234,8 @@ class MainActivity : Activity() { private fun showConnectionSettings() { val prefs = getSharedPreferences("pc_connection", Context.MODE_PRIVATE) AlertDialog.Builder(this).setTitle("OpenGlass connection") - .setItems(arrayOf("Auto: iPhone hotspot / Wi-Fi", "Manual address", "USB relay", "Choose phone again")) { _, which -> + .setItems(arrayOf("Auto: iPhone hotspot / Wi-Fi", "Manual address", "USB relay", "Choose phone again", + if (running) "Pause capture / 暂停采集" else "Resume capture / 恢复采集")) { _, which -> fun applyMode(mode: String, url: String?) { prefs.edit().putString("connection_mode", mode).apply() explicitUrl = url @@ -257,8 +268,14 @@ class MainActivity : Activity() { prefs.edit().remove("iphone_service_name").apply() applyMode("auto", null) } + 4 -> { + if (running) { + userStopped = true + stopAll() + } else resumeCapture() + } } - }.show() + }.setNegativeButton("关闭", null).show() } override fun onWindowFocusChanged(hasFocus: Boolean) { @@ -288,13 +305,14 @@ class MainActivity : Activity() { TAG, "key action=${event.action} code=${event.keyCode} name=${KeyEvent.keyCodeToString(event.keyCode)}", ) + if (event.keyCode == KeyEvent.KEYCODE_DPAD_CENTER || event.keyCode == KeyEvent.KEYCODE_ENTER) { + // Consume DOWN as well: forwarding it to the focused full-screen TextView can + // arm its long-click timer and unexpectedly open settings while a key is held. + if (event.action == KeyEvent.ACTION_UP && !event.isCanceled) resumeCapture() + return true + } if (event.action == KeyEvent.ACTION_UP) { when (event.keyCode) { - KeyEvent.KEYCODE_DPAD_CENTER, KeyEvent.KEYCODE_ENTER -> { - if (currentFocus is Button) return super.dispatchKeyEvent(event) - toggleStartStop() - return true - } KeyEvent.KEYCODE_BACK -> { if (running) { userStopped = true @@ -312,14 +330,16 @@ class MainActivity : Activity() { checkSelfPermission(Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED && checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED - private fun toggleStartStop() { - userStopped = running - if (running) stopAll() else startAll() + private fun resumeCapture() { + if (running) return + userStopped = false + if (hasPermissions()) startAll() } private fun startAll() { if (running) return running = true + microphoneSilenced = null captureGeneration.incrementAndGet() startMs = System.currentTimeMillis() lastError = "none" @@ -346,6 +366,7 @@ class MainActivity : Activity() { private fun stopAll() { if (!running) return running = false + microphoneSilenced = null captureGeneration.incrementAndGet() cameraRestartPending.set(false) Log.i(TAG, "stopAll") @@ -468,9 +489,15 @@ class MainActivity : Activity() { } val now = System.currentTimeMillis() if (now - lastLog >= 1_000L) { + // Observe this recorder only. An unavailable configuration stays unknown; + // this diagnostic does not alter the audio source, routing or samples. + val silenced = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + runCatching { recorder.activeRecordingConfiguration?.isClientSilenced }.getOrNull() + } else null + if (token == captureGeneration.get()) microphoneSilenced = silenced Log.i( TAG, - "audio source=$audioSourceText pkt=${audioPackets.get()} drop=${audioDrops.get()} rms=${fmt(lastRms)} peak=$lastPeak nonzero=$lastNonZero wsOpen=$wsOpen", + "audio source=$audioSourceText pkt=${audioPackets.get()} drop=${audioDrops.get()} rms=${fmt(lastRms)} peak=$lastPeak nonzero=$lastNonZero wsOpen=$wsOpen silenced=${silenced?.toString() ?: "unknown"}", ) lastLog = now } @@ -484,7 +511,10 @@ class MainActivity : Activity() { } catch (_: Throwable) { } recorder.release() - if (activeRecorder === recorder) activeRecorder = null + if (activeRecorder === recorder) { + activeRecorder = null + microphoneSilenced = null + } Log.i(TAG, "AudioRecord stopped") } }, "rokid-audio").also { it.start() } @@ -1119,6 +1149,7 @@ class MainActivity : Activity() { val status = when { !hasPermissions() -> "请允许相机和麦克风" !running -> "OpenGlass · 已暂停" + microphoneSilenced == true -> "系统提示麦克风静音" currentEndpoint == null || !wsOpen -> "OpenGlass · 连接中" else -> "OpenGlass · 已连接" } diff --git a/rokid_app/OpenGlassRokidSensor/app/src/main/res/values/styles.xml b/rokid_app/OpenGlassRokidSensor/app/src/main/res/values/styles.xml index 41a215d..24291ef 100644 --- a/rokid_app/OpenGlassRokidSensor/app/src/main/res/values/styles.xml +++ b/rokid_app/OpenGlassRokidSensor/app/src/main/res/values/styles.xml @@ -4,7 +4,15 @@ false true true + @android:color/black + @android:color/black + false + false + true + @android:color/black + @android:color/black false + false #33ff77 diff --git a/rokid_app/README.md b/rokid_app/README.md index c2d1727..7d834b5 100644 --- a/rokid_app/README.md +++ b/rokid_app/README.md @@ -2,7 +2,7 @@ [中文说明](README_zh.md) -This directory contains OpenGlass Rokid sensor app 0.2.0 (version code 6). It captures JPEG/PCM and can connect directly to OpenGlass on iPhone over Wi-Fi. The existing PC panel and explicit USB relay modes remain available. +This directory contains OpenGlass Rokid sensor app 0.2.0 (version code 7). It captures JPEG/PCM and can connect directly to OpenGlass on iPhone over Wi-Fi. The existing PC panel and explicit USB relay modes remain available. ## Companion setup @@ -41,7 +41,9 @@ Auto mode never falls back to USB, and ignores loopback addresses saved by older The default display is one dim, centered status line on black: connected, connecting, or paused. Text changes only with connection state; counters, addresses, and scrolling diagnostics stay off the display. Diagnostics remain available in logcat. Connected means the audio connection is established, not that iPhone ASR is currently ready. -**Long-press the display or press Menu** to open connection settings: Auto, Manual address, USB relay, and Choose phone again. Manual accepts `http://
:18080`; USB selects `http://127.0.0.1:18080` and requires the existing computer USB relay. Choose phone again forgets the remembered receiver. Tap the display or press Enter/DPAD to stop/start; Back stops, then exits. +**Long-press the display or press Menu** to open settings: Auto, Manual address, USB relay, Choose phone again, and an explicit Pause/Resume capture item. Use **关闭 (Close)** or Back to dismiss the settings dialog. Manual accepts `http://
:18080`; USB selects `http://127.0.0.1:18080` and requires the existing computer USB relay. Choose phone again forgets the remembered receiver. A tap or Enter/DPAD resumes paused capture and does nothing while capture is running, so a casual touch cannot stop the stream. Holding Enter/DPAD does not open the menu. To pause, choose Pause capture from the menu. Back on the main screen still stops, then exits. The full-screen status view disables Android's default focus highlight, and the opaque window and system-bar backgrounds are black. + +On Android 10+, the audio diagnostic log includes `silenced=true/false/unknown` from this recorder's active configuration. `unknown` means unavailable, not “not silenced.” When the system reports `true`, the single-line display says `系统提示麦克风静音` (system reports microphone silencing). This observation does not change recording parameters or audio routing; a low RMS alone is not treated as proof of system silencing. Launch intents remain compatible: an explicit `pc_base_url` fixes the endpoint for that launch and persists the URL as a future fallback. `auto_discover=true` explicitly selects Auto and overrides the old saved USB address. The connection-settings mode persists, while an intent URL alone does not turn a future launcher start into USB mode. diff --git a/rokid_app/README_zh.md b/rokid_app/README_zh.md index d4dbf3b..bf3e373 100644 --- a/rokid_app/README_zh.md +++ b/rokid_app/README_zh.md @@ -1,6 +1,6 @@ # Rokid 采集 App -此目录保存 OpenGlass Rokid 采集 App 0.2.0（versionCode 6）：发送 JPEG 与 PCM 音频，可通过 Wi-Fi 直接连接 iPhone OpenGlass，也保留电脑面板和显式 USB 中继模式。它不是眼镜系统固件。 +此目录保存 OpenGlass Rokid 采集 App 0.2.0（versionCode 7）：发送 JPEG 与 PCM 音频，可通过 Wi-Fi 直接连接 iPhone OpenGlass，也保留电脑面板和显式 USB 中继模式。它不是眼镜系统固件。 ## 文件布局 @@ -56,7 +56,9 @@ adb -s install -r .\app\build\outputs\apk\wifi\app-wifi.apk 眼镜主界面采用黑底、较暗的居中单行提示：`OpenGlass · 已连接`、`OpenGlass · 连接中` 或 `OpenGlass · 已暂停`。只在状态变化时更新，不显示计数、地址或滚动调试文字；诊断信息仍可从 logcat 查看。“已连接”表示音频连接已建立，不代表 iPhone 此刻已准备好识别。 -**长按界面或按 Menu** 打开连接设置，提供 Auto、Manual address、USB relay、Choose phone again。手动地址格式为 `http://<地址>:18080`；USB 模式使用 `http://127.0.0.1:18080`，仍需要电脑上的既有 USB 中继。Choose phone again 会忘记已选择的手机。短按界面或 Enter/DPAD 可启停；Back 先停止，再退出。 +**长按界面或按 Menu** 打开设置，提供 Auto、Manual address、USB relay、Choose phone again，以及明确的“暂停采集 / 恢复采集”选项。点击**关闭**或按 Back 可关闭设置弹窗。手动地址格式为 `http://<地址>:18080`；USB 模式使用 `http://127.0.0.1:18080`，仍需要电脑上的既有 USB 中继。Choose phone again 会忘记已选择的手机。短按界面或 Enter/DPAD 仅用于恢复已暂停的采集，运行中不会因普通点击而停止；按住 Enter/DPAD 也不会打开菜单。需要暂停时从菜单选择“暂停采集”。主界面 Back 仍先停止，再退出。全屏状态文字关闭系统默认焦点高亮，窗口与系统栏均使用不透明黑底。 + +Android 10 及以上的音频诊断日志会读取当前 AudioRecord 的配置，记录 `silenced=true/false/unknown`；`unknown` 表示配置不可用，不能当作 `false`。仅系统明确返回 `true` 时，单行界面提示“系统提示麦克风静音”。诊断不会改动录音参数或声音路由，也不会仅凭 RMS 很低就断定麦克风被系统静音。 启动参数仍兼容：显式 `pc_base_url` 会在本次启动固定连接该地址，并保存为以后自动模式的后备地址；`auto_discover=true` 显式切回自动，覆盖旧保存的 USB 地址。设置界面选择的模式会保存，仅传入一次 URL 不会把以后的普通启动永久绑定到 USB。 From 57589c0af816310354a2a9bc531ffd88262f6be8 Mon Sep 17 00:00:00 2001 From: MengzhangLI Date: Sun, 18 Oct 2026 03:13:10 +0800 Subject: [PATCH 10/11] Keep automatic speech listening across empty recognition rounds --- ios/README_zh.md | 5 +- ios/Sources/OpenGlassSpeechDiagnostics.swift | 77 +++++ .../OpenGlassSpeechHandoffBuffer.swift | 57 ++++ ios/Sources/OpenGlassSpeechSession.swift | 125 ++++++- ios/Sources/OpenGlassViewController.swift | 90 ++++- ios/Sources/OpenGlassVoiceLoopPolicy.swift | 2 +- ios/Sources/RokidIngressServer.swift | 39 ++- ios/tests/run_voice_loop_tests.sh | 5 +- ios/tests/voice_loop/main.swift | 317 +++++++++++++++++- .../OpenGlassRokidSensor/app/build.gradle | 2 +- .../opensqz/openglass/rokid/MainActivity.kt | 7 +- rokid_app/README.md | 2 +- rokid_app/README_zh.md | 4 +- 13 files changed, 672 insertions(+), 60 deletions(-) create mode 100644 ios/Sources/OpenGlassSpeechDiagnostics.swift create mode 100644 ios/Sources/OpenGlassSpeechHandoffBuffer.swift diff --git a/ios/README_zh.md b/ios/README_zh.md index f4c00e6..4063a19 100644 --- a/ios/README_zh.md +++ b/ios/README_zh.md @@ -207,7 +207,9 @@ powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File 'C:\path\to\OpenGl 自动循环为“听问题 → 看图推理 → 朗读回答 → 等待回声消退 → 继续听问题”。推理/播报时丢弃输入语音，不积压旧问题；空白识别、短暂识别错误、音频重连与新画面恢复均由状态机处理。权限拒绝、离线语言资源缺失、模型错误仍会明确显示，主动暂停必须由用户恢复。这是逐轮交互，暂不支持边播报边打断。 -`/health` 的 `conversation_phase` 显示 `waiting/listening/thinking/speaking/paused`，`completed_turns` 记录本次接收会话中完成的回答数；不返回识别文本或回答正文。测试至少三轮，并留空静音约30秒后再提问，确认没有空问题、重复回答或需要手动唤醒。 +正常空白识别不增加错误退避：有收尾缓冲就立即进入下一轮，否则等下一包声音到达再开始；真实识别错误仍按0.5～8秒退避。仅在 `endAudio` 到空结果之间暂存最多2.2秒的新声音，空结果后须在0.5秒内交给下一轮，避免收尾期间漏掉下一句。已送入识别器的声音不重放；非空结果、识别错误、暂停、断线和后台切换均清空缓冲。没有文字时的短促噪声不再触发1.3秒收尾，安静时按25秒轮换识别任务。 + +手机“自动语音”状态显示实际听音、收尾、等待或播报状态及已完成轮数。`/health` 的 `conversation_phase` 显示 `waiting/listening/finishing/cooldown/thinking/speaking/paused`，`completed_turns` 记录本次接收会话中完成的回答数，`app_build` 用于核对安装版本。`asr` 提供实际送入识别器的样本数、音量、非空结果数和错误域/错误码，每秒刷新；不返回识别文本、音频或回答正文。若音频包持续增长但 `asr.accepted_samples` 不增长，先检查 `asr.gate`；若样本和音量增长但没有非空结果，再检查识别错误。测试至少三轮，并留空静音约30秒后再提问，确认没有空问题、重复回答或需要手动唤醒。 ## 排障 @@ -233,6 +235,7 @@ powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File 'C:\path\to\OpenGl - Rokid `0.2.0-wifi` / code `5` 的 `assembleWifi`、7 项 `testWifiUnitTest` 与原 `assembleDebug` 通过；APK SHA256：`A5A64849FEAC796D4688B60833E647FA3DACE1CB3F5A16BD8935AA0EA5623A13`。并存包 `org.opensqz.openglass.rokid.wifi` 已安装，旧 debug 采集 App 保留；新 App 的相机和麦克风权限已确认允许。 - 2026-10-18 热点真机测试：眼镜直接向当前 iPhone 热点网关发送图像和音频，手机接收计数持续增长。拔掉眼镜 USB 后，ADB 已无设备，手机仍持续收流，`completed_turns` 增至 `3`。随后用户确认：无需点按钮即可连续问答，且眼镜和 iPhone 两根 USB 均拔掉后仍正常。已验证本机 ASR → MiniCPM-V 4.6 → TTS 的无线自动多轮流程；酒店 Wi-Fi 尚未实测。 - 包含 Android 最终改动的提交 `c439fa3` 也通过 [CI 39058773419](https://github.com/OpenSQZ/OpenGlass/actions/runs/39058773419)。其 iOS Swift 源码与已安装的 build `5` 相同，无需再次安装。 +- 后续用户确认单行眼镜界面和首轮 ASR/TTS 正常，但再次提问出现空识别，说明此前成功记录不足以证明持续对话可靠。此次修复空轮退避和收尾期间丢音，并增加 ASR 操作状态诊断；新版端到端多轮结果待安装后复测。 - 不在此文档写入个人设备序列号、UDID、Apple账号或内网凭据。现场更新请用设备类型和验证结果描述。 可重复执行无设备测试： diff --git a/ios/Sources/OpenGlassSpeechDiagnostics.swift b/ios/Sources/OpenGlassSpeechDiagnostics.swift new file mode 100644 index 0000000..322fe76 --- /dev/null +++ b/ios/Sources/OpenGlassSpeechDiagnostics.swift @@ -0,0 +1,77 @@ +/// Operational metadata only: never retains speech, audio or error descriptions. +struct OpenGlassSpeechDiagnostics { + enum State: String { + case idle, listening, finishing + } + + private(set) var state: State = .idle + private(set) var sessions = 0 + private(set) var acceptedSamples = 0 + private(set) var sessionSamples = 0 + private(set) var lastRMS = 0.0 + private(set) var sessionPeakRMS = 0.0 + private(set) var partialCount = 0 + private(set) var nonemptyPartialCount = 0 + private(set) var emptyFinalCount = 0 + private(set) var nonemptyFinalCount = 0 + private(set) var failureCount = 0 + private(set) var lastFailureDomain = "" + private(set) var lastFailureCode: Int? + private(set) var lastEndReason = "not_started" + + mutating func reset() { + self = Self() + } + + mutating func started() { + state = .listening + sessions += 1 + sessionSamples = 0 + lastRMS = 0 + sessionPeakRMS = 0 + lastEndReason = "none" + } + + mutating func appended(samples: Int, rms: Double) { + guard state == .listening, samples > 0 else { return } + acceptedSamples += samples + sessionSamples += samples + // Invalid measurements must not introduce NaN/Infinity into diagnostics. + lastRMS = rms.isFinite ? max(0, rms) : 0 + sessionPeakRMS = max(sessionPeakRMS, lastRMS) + } + + mutating func partial(nonempty: Bool) { + guard state != .idle else { return } + partialCount += 1 + if nonempty { nonemptyPartialCount += 1 } + } + + mutating func finishing(reason: String) { + guard state == .listening else { return } + state = .finishing + lastEndReason = reason + } + + mutating func completed(nonempty: Bool, reason: String) { + guard state != .idle else { return } + if nonempty { nonemptyFinalCount += 1 } + else { emptyFinalCount += 1 } + state = .idle + lastEndReason = reason + } + + mutating func failed(domain: String, code: Int, reason: String) { + // A recognizer can fail synchronously before a session starts. + failureCount += 1 + state = .idle + lastFailureDomain = domain + lastFailureCode = code + lastEndReason = reason + } + + mutating func stopped(reason: String) { + state = .idle + lastEndReason = reason + } +} diff --git a/ios/Sources/OpenGlassSpeechHandoffBuffer.swift b/ios/Sources/OpenGlassSpeechHandoffBuffer.swift new file mode 100644 index 0000000..8c3aaeb --- /dev/null +++ b/ios/Sources/OpenGlassSpeechHandoffBuffer.swift @@ -0,0 +1,57 @@ +import Foundation + +/// In-memory PCM received only after endAudio, for one empty-result handoff. +/// Audio already appended to a recognition request must never enter this buffer. +struct OpenGlassSpeechHandoffBuffer { + enum Phase: Equatable { + case idle, collecting, ready + } + + private(set) var phase: Phase = .idle + private let capacity: Int + private var bytes = Data() + private var readyAt: TimeInterval? + + var byteCount: Int { bytes.count } + + func hasReadyAudio(at now: TimeInterval) -> Bool { + guard phase == .ready, !bytes.isEmpty, let readyAt else { return false } + return now >= readyAt && now - readyAt <= 0.5 + } + + init(capacity: Int = 70_400) { + let nonnegativeCapacity = max(0, capacity) + self.capacity = nonnegativeCapacity - nonnegativeCapacity % 2 + } + + mutating func begin() { + clear() + phase = .collecting + } + + mutating func append(_ data: Data) { + guard phase == .collecting, bytes.count < capacity else { return } + // Preserve the start of an utterance instead of replacing it with its tail. + bytes.append(contentsOf: data.prefix(capacity - bytes.count)) + } + + mutating func complete(empty: Bool, now: TimeInterval) { + guard empty, phase == .collecting else { clear(); return } + phase = .ready + readyAt = now + } + + mutating func takeReadyAudio(now: TimeInterval) -> Data { + guard phase == .ready else { return Data() } + guard hasReadyAudio(at: now) else { clear(); return Data() } + let audio = bytes + clear() + return audio + } + + mutating func clear() { + phase = .idle + bytes.removeAll(keepingCapacity: false) + readyAt = nil + } +} diff --git a/ios/Sources/OpenGlassSpeechSession.swift b/ios/Sources/OpenGlassSpeechSession.swift index 92390db..ffbaf24 100644 --- a/ios/Sources/OpenGlassSpeechSession.swift +++ b/ios/Sources/OpenGlassSpeechSession.swift @@ -11,8 +11,12 @@ final class OpenGlassSpeechSession { var onError: ((Failure) -> Void)? var onStatus: ((String) -> Void)? private(set) var isListening = false + private(set) var diagnostics = OpenGlassSpeechDiagnostics() var isActive: Bool { request != nil } var isFinishing: Bool { ending } + var hasPendingHandoffAudio: Bool { + handoff.hasReadyAudio(at: ProcessInfo.processInfo.systemUptime) + } private var recognizer: SFSpeechRecognizer? private var request: SFSpeechAudioBufferRecognitionRequest? private var task: SFSpeechRecognitionTask? @@ -25,6 +29,7 @@ final class OpenGlassSpeechSession { private var lastTranscriptAt: TimeInterval? private var lastPacketAt = ProcessInfo.processInfo.systemUptime private var pendingByte: UInt8? + private var handoff = OpenGlassSpeechHandoffBuffer() private var ending = false private let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 16_000, channels: 1, interleaved: false)! @@ -39,15 +44,21 @@ final class OpenGlassSpeechSession { } func start(localeIdentifier: String) throws { - stop() + // Only a normal empty final can prepare this one-shot handoff. Taking + // it before cleanup preserves new speech received after endAudio. + let carriedAudio = handoff.takeReadyAudio(now: ProcessInfo.processInfo.systemUptime) + stop(reason: "restart") guard SFSpeechRecognizer.authorizationStatus() == .authorized else { + recordFailure(.permissionDenied) throw Failure.permissionDenied } guard let recognizer = SFSpeechRecognizer(locale: Locale(identifier: localeIdentifier)), recognizer.supportsOnDeviceRecognition else { + recordFailure(.languageUnavailable) throw Failure.languageUnavailable } guard recognizer.isAvailable else { + recordFailure(.temporarilyUnavailable) throw Failure.temporarilyUnavailable } let request = SFSpeechAudioBufferRecognitionRequest() @@ -64,6 +75,8 @@ final class OpenGlassSpeechSession { lastVoiceAt = nil lastTranscriptAt = nil isListening = true + diagnostics.started() + logDiagnostics(event: "started") let id = sessionID task = recognizer.recognitionTask(with: request) { [weak self] result, error in Task { @MainActor in @@ -75,12 +88,18 @@ final class OpenGlassSpeechSession { self.lastTranscriptAt = self.lastVoiceAt } self.latestText = text + if !result.isFinal { + self.diagnostics.partial(nonempty: !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + } self.onPartial?(self.latestText) - if result.isFinal { self.complete(); return } + if result.isFinal { + self.complete(reason: "native_final", allowHandoff: error == nil) + return + } } if let error { if self.ending && !self.latestText.isEmpty { - self.complete() + self.complete(reason: "finalization_error", allowHandoff: false) } else { let failure: Failure if SFSpeechRecognizer.authorizationStatus() != .authorized { @@ -90,7 +109,8 @@ final class OpenGlassSpeechSession { } else { failure = .recognitionInterrupted(error.localizedDescription) } - self.stop() + self.stop(reason: "recognizer_error") + self.recordFailure(failure, underlying: error) self.onError?(failure) } } @@ -99,11 +119,19 @@ final class OpenGlassSpeechSession { timer = Timer.scheduledTimer(withTimeInterval: 0.2, repeats: true) { [weak self] _ in Task { @MainActor in self?.tick() } } + if !carriedAudio.isEmpty { appendPCM16LE(carriedAudio) } onStatus?("正在听 Rokid 麦克风（离线识别）") } func appendPCM16LE(_ data: Data) { - guard isListening, !ending, let request, !data.isEmpty else { return } + guard !data.isEmpty else { return } + if ending { + // This is only the short endAudio/final-result window. Controller + // gates still discard all inference, TTS and user-paused audio. + handoff.append(data) + return + } + guard isListening, let request else { return } // Preserve a split sample across websocket messages. At most one byte // is retained; audio received during inference/TTS is simply dropped. var bytes = [UInt8]() @@ -125,15 +153,24 @@ final class OpenGlassSpeechSession { } let now = ProcessInfo.processInfo.systemUptime lastPacketAt = now - if sqrt(squareSum / Float(frames)) > 0.012 { lastVoiceAt = now } + let rms = sqrt(squareSum / Float(frames)) + if rms > 0.012 { lastVoiceAt = now } request.append(buffer) + diagnostics.appended(samples: frames, rms: Double(rms)) } /// Stop accepting audio and give the recognizer time to finalize its text. - func finishUtterance() { + func finishUtterance(reason: String = "manual") { guard isListening, !ending else { return } ending = true isListening = false + handoff.begin() + // A split sample's first byte has not reached the old request. Keep it + // ahead of the post-endAudio stream so the next request stays aligned. + if let pendingByte { handoff.append(Data([pendingByte])) } + pendingByte = nil + diagnostics.finishing(reason: reason) + logDiagnostics(event: "finishing") timer?.invalidate() timer = nil request?.endAudio() @@ -142,12 +179,20 @@ final class OpenGlassSpeechSession { finishTimer = Timer.scheduledTimer(withTimeInterval: 2.0, repeats: false) { [weak self] _ in Task { @MainActor in guard let self, self.sessionID == id else { return } - self.complete() + self.complete(reason: "finish_timeout") } } } - func stop() { + func stop(reason: String = "cancelled") { + cancelRecognition(reason: reason, preservingHandoff: false) + } + + private func cancelRecognition(reason: String, preservingHandoff: Bool) { + if !preservingHandoff { handoff.clear() } + let wasActive = diagnostics.state != .idle + // No-op cleanup after a final/error must not replace its useful cause. + if wasActive { diagnostics.stopped(reason: reason) } sessionID = UUID() isListening = false ending = false @@ -161,6 +206,13 @@ final class OpenGlassSpeechSession { request = nil recognizer = nil pendingByte = nil + if wasActive { logDiagnostics(event: "stopped") } + } + + /// Reset only when beginning a new receiver run, never between ASR retries. + func resetDiagnostics() { + diagnostics.reset() + logDiagnostics(event: "reset") } private func tick() { @@ -168,26 +220,63 @@ final class OpenGlassSpeechSession { let now = ProcessInfo.processInfo.systemUptime let action = OpenGlassUtteranceTiming.action(now: now, startedAt: startedAt, lastPacketAt: lastPacketAt, lastVoiceAt: lastVoiceAt, - lastTranscriptAt: lastTranscriptAt, hasText: !latestText.isEmpty) + lastTranscriptAt: lastTranscriptAt, hasText: containsWords(latestText)) switch action { - case .finish: finishUtterance() + case .finish: finishUtterance(reason: "timer_finish") case .audioTimeout: - stop() + stop(reason: "audio_timeout") + recordFailure(.audioTimeout) onError?(.audioTimeout) case .listen: break } } - private func complete() { + private func complete(reason: String, allowHandoff: Bool = true) { let candidate = latestText.trimmingCharacters(in: .whitespacesAndNewlines) - // Punctuation-only final results are not questions. Silence and failed - // recognition finish normally and let the controller schedule a retry. - let hasWords = candidate.unicodeScalars.contains { CharacterSet.alphanumerics.contains($0) } - let text = hasWords ? candidate : "" - stop() + // Punctuation-only final results are not questions. A normal empty + // result lets the controller rearm immediately when fresh PCM is ready. + let text = containsWords(candidate) ? candidate : "" + let preserveHandoff = text.isEmpty && allowHandoff + handoff.complete(empty: preserveHandoff, now: ProcessInfo.processInfo.systemUptime) + diagnostics.completed(nonempty: !text.isEmpty, reason: reason) + cancelRecognition(reason: reason, preservingHandoff: preserveHandoff) + logDiagnostics(event: "final") onFinal?(text) } + private func containsWords(_ text: String) -> Bool { + text.unicodeScalars.contains { CharacterSet.alphanumerics.contains($0) } + } + + private func recordFailure(_ failure: Failure, underlying: Error? = nil) { + let reason: String + let localCode: Int + switch failure { + case .permissionDenied: reason = "permission_denied"; localCode = 1 + case .languageUnavailable: reason = "language_unavailable"; localCode = 2 + case .temporarilyUnavailable: reason = "recognizer_unavailable"; localCode = 3 + case .audioTimeout: reason = "audio_timeout"; localCode = 4 + case .recognitionInterrupted: reason = "recognizer_error"; localCode = 5 + } + let nativeError = underlying.map { $0 as NSError } + diagnostics.failed(domain: nativeError?.domain ?? "OpenGlassSpeech", + code: nativeError?.code ?? localCode, reason: reason) + logDiagnostics(event: "failed") + } + + /// State transitions only: never log PCM, recognition text, or error descriptions. + private func logDiagnostics(event: String) { + let d = diagnostics + let summary = "OpenGlassASR event=\(event) state=\(d.state.rawValue) session=\(d.sessions)" + + " accepted_samples=\(d.acceptedSamples) session_samples=\(d.sessionSamples)" + + " rms=\(String(format: "%.5f", d.lastRMS)) peak_rms=\(String(format: "%.5f", d.sessionPeakRMS))" + + " partials=\(d.partialCount) nonempty_partials=\(d.nonemptyPartialCount)" + + " empty_finals=\(d.emptyFinalCount) nonempty_finals=\(d.nonemptyFinalCount)" + + " failures=\(d.failureCount) failure_domain=\(d.lastFailureDomain)" + + " failure_code=\(d.lastFailureCode.map(String.init) ?? "none") end=\(d.lastEndReason)" + NSLog("%@", summary) + } + enum Failure: LocalizedError { case permissionDenied, languageUnavailable, temporarilyUnavailable, audioTimeout case recognitionInterrupted(String) diff --git a/ios/Sources/OpenGlassViewController.swift b/ios/Sources/OpenGlassViewController.swift index aba5528..ab1fc80 100644 --- a/ios/Sources/OpenGlassViewController.swift +++ b/ios/Sources/OpenGlassViewController.swift @@ -18,6 +18,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp private var isAuthorizingSpeech = false private var isAnswering = false private var audioConnected = false + private var waitForFreshAudio = false private var voiceLoop = OpenGlassVoiceLoopPolicy() private var wantsService = false private var attemptedAutomaticStart = false @@ -41,6 +42,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp private let connectionLabel = UILabel() private let frameLabel = UILabel() private let routeLabel = UILabel() + private let voiceStatusLabel = UILabel() private let preview = UIImageView() private let addressView = UITextView() private let transcriptLabel = UILabel() @@ -80,6 +82,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp self?.refreshFrameAge() self?.refreshAddresses() self?.reconcileAutomaticLoop() + self?.refreshVoiceStatus() } } refreshAddresses() @@ -137,7 +140,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp stack.bottomAnchor.constraint(equalTo: scroll.contentLayoutGuide.bottomAnchor, constant: -24), stack.widthAnchor.constraint(equalTo: scroll.frameLayoutGuide.widthAnchor, constant: -36) ]) - for label in [statusLabel, modelLabel, connectionLabel, frameLabel, routeLabel, transcriptLabel] { + for label in [statusLabel, modelLabel, connectionLabel, frameLabel, routeLabel, voiceStatusLabel, transcriptLabel] { label.numberOfLines = 0 label.font = .preferredFont(forTextStyle: .subheadline) label.adjustsFontForContentSizeCategory = true @@ -146,6 +149,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp modelLabel.textColor = .secondaryLabel frameLabel.textColor = .secondaryLabel routeLabel.textColor = .secondaryLabel + voiceStatusLabel.textColor = .secondaryLabel stack.addArrangedSubview(statusLabel) stack.addArrangedSubview(modelLabel) configureButton(serviceButton, title: "启动接收", action: #selector(toggleService), filled: true) @@ -192,6 +196,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp speakSwitch.addTarget(self, action: #selector(speakChanged), for: .valueChanged) stack.addArrangedSubview(row("朗读回答", speakSwitch)) stack.addArrangedSubview(routeLabel) + stack.addArrangedSubview(voiceStatusLabel) configureButton(listenButton, title: "恢复自动语音", action: #selector(toggleListening)) stack.addArrangedSubview(listenButton) transcriptLabel.text = "启动后自动听取 Rokid 语音。停顿约1.3秒后提问，朗读结束会自动听下一句。" @@ -257,6 +262,9 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp } server.onAudio = { [weak self] data in guard let self else { return } + // An empty final with no handoff waits for actual PCM, not a timer + // or image callback. This avoids spinning on immediate empty tasks. + if !data.isEmpty { self.waitForFreshAudio = false } self.reconcileAutomaticLoop() guard self.isRunning, !self.isInBackground, !self.audioInterruptionActive, !self.isAnswering, !self.model.isBusy, self.activeUtterance == nil, @@ -271,7 +279,8 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp guard let self else { return } self.audioConnected = connected if !connected { - self.speech.stop() + self.waitForFreshAudio = false + self.speech.stop(reason: "audio_disconnected") if self.isRunning { self.statusLabel.text = "Rokid 音频已断开，等待重新连接。" } } else { self.reconcileAutomaticLoop() @@ -295,8 +304,15 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp guard let self, self.isRunning, !self.isInBackground, self.voiceLoop.wantsListening, self.voiceLoop.blockedReason == nil else { return } if text.isEmpty { - self.retryListening(message: "未识别到问题，继续等待语音。") + // Silence is normal in a continuous conversation. Backoff is + // reserved for real errors; it otherwise drops the next words. + self.voiceLoop.receivedUtterance() + self.cancelListeningWake() + self.waitForFreshAudio = !self.speech.hasPendingHandoffAudio + self.statusLabel.text = "继续等待 Rokid 语音。" + self.reconcileAutomaticLoop() } else { + self.waitForFreshAudio = false self.voiceLoop.receivedUtterance() self.questionField.text = text self.submit(text, fromSpeech: true) @@ -322,6 +338,8 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp isStarting = true isRunning = true completedTurns = 0 + waitForFreshAudio = false + speech.resetDiagnostics() runID = UUID() let id = runID latestRawImage = nil @@ -380,10 +398,11 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp isAuthorizingSpeech = false speechAuthorizationID = UUID() cancelListeningWake() + waitForFreshAudio = false suppressAnswerSpeech = true runID = UUID() turnID = UUID() - speech.stop() + speech.stop(reason: preservingIntent ? "background" : "receiver_stopped") stopSpeaking() server.stop() audioConnected = false @@ -400,7 +419,8 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp isAuthorizingSpeech = false speechAuthorizationID = UUID() suppressAnswerSpeech = true - speech.stop() + waitForFreshAudio = false + speech.stop(reason: "user_paused") stopSpeaking() model.requestStop() statusLabel.text = model.isBusy ? "正在结束本轮推理…" : "语音已暂停，点击“恢复自动语音”可恢复。" @@ -436,7 +456,8 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp updateControls() return } - speech.stop() + waitForFreshAudio = false + speech.stop(reason: "model_submission") stopSpeaking() isAnswering = true suppressAnswerSpeech = false @@ -514,7 +535,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp !isStarting, !isAuthorizingSpeech, !isAnswering, !model.isBusy, model.isLoaded, !model.requiresAppRestart, voiceSwitch.isOn, voiceLoop.wantsListening, voiceLoop.blockedReason == nil, audioConnected, hasFreshFrame, - activeUtterance == nil, !speech.isActive else { return } + activeUtterance == nil, !speech.isActive, !waitForFreshAudio else { return } guard voiceLoop.canListen(at: monotonicNow) else { scheduleListeningWake() return @@ -579,6 +600,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp voiceSwitch.isOn = true UserDefaults.standard.set(true, forKey: "OpenGlass.voiceEnabled") voiceLoop.requestListening() + waitForFreshAudio = false isAuthorizingSpeech = true let id = runID let authorizationID = UUID() @@ -604,7 +626,8 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp if voiceSwitch.isOn { if isRunning { toggleListening() } } else { - speech.stop() + waitForFreshAudio = false + speech.stop(reason: "voice_disabled") voiceLoop.pause() cancelListeningWake() isAuthorizingSpeech = false @@ -616,7 +639,8 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp @objc private func languageChanged() { UserDefaults.standard.set(languageControl.selectedSegmentIndex, forKey: "OpenGlass.speechLanguage") - speech.stop() + waitForFreshAudio = false + speech.stop(reason: "language_changed") if voiceLoop.wantsListening { voiceLoop.requestListening() } reconcileAutomaticLoop() } @@ -630,7 +654,7 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp } private func speak(_ text: String) { - speech.stop() + speech.stop(reason: "tts_started") do { let audio = AVAudioSession.sharedInstance() try audio.setCategory(.playback, mode: .spokenAudio, options: []) @@ -746,14 +770,43 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp voiceSwitch.isEnabled = !isStarting if isRunning, let reason = voiceLoop.blockedReason { statusLabel.text = reason } if model.requiresAppRestart { statusLabel.text = OpenGlassModelSession.SessionError.restartRequired.localizedDescription } + refreshVoiceStatus() + } + + /// Called on transitions and once per second so health reports samples + /// actually accepted by ASR, not merely packets received by the socket. + private func refreshVoiceStatus() { let phase: String - if !isRunning || isInBackground { phase = "paused" } - else if busy { phase = "thinking" } - else if activeUtterance != nil { phase = "speaking" } - else if speech.isActive { phase = "listening" } - else if !voiceLoop.wantsListening || voiceLoop.blockedReason != nil { phase = "paused" } - else { phase = "waiting" } - server.setConversationStatus(phase: phase, completedTurns: completedTurns) + let gate: String + let label: String + if !isRunning || isInBackground { + (phase, gate, label) = ("paused", "receiver_stopped", "接收已停止") + } else if model.isBusy || isAnswering { + (phase, gate, label) = ("thinking", "model", "正在回答") + } else if activeUtterance != nil { + (phase, gate, label) = ("speaking", "tts", "正在朗读，结束后自动听下一句") + } else if !voiceLoop.wantsListening || !voiceSwitch.isOn || voiceLoop.blockedReason != nil { + (phase, gate, label) = ("paused", "voice_paused", "已暂停") + } else if audioInterruptionActive { + (phase, gate, label) = ("waiting", "audio_interruption", "等待系统音频恢复") + } else if isStarting || isAuthorizingSpeech || !model.isLoaded || model.requiresAppRestart { + (phase, gate, label) = ("waiting", "preparing", "正在准备") + } else if speech.isFinishing { + (phase, gate, label) = ("finishing", "finalizing", "正在确认这一句") + } else if speech.isListening { + (phase, gate, label) = ("listening", "accepting_audio", "正在听 Rokid 麦克风") + } else if !audioConnected { + (phase, gate, label) = ("waiting", "no_audio_connection", "等待眼镜声音连接") + } else if !hasFreshFrame { + (phase, gate, label) = ("waiting", "no_fresh_image", "等待新画面") + } else if !voiceLoop.canListen(at: monotonicNow) { + (phase, gate, label) = ("cooldown", "retry_or_echo_delay", "稍后自动继续听取语音") + } else { + (phase, gate, label) = ("waiting", "fresh_audio", "等待眼镜声音") + } + voiceStatusLabel.text = "自动语音：\(label) · 已完成 \(completedTurns) 轮" + server.setConversationStatus(phase: phase, completedTurns: completedTurns, + asr: speech.diagnostics, gate: gate) } @objc private func showDownloads() { @@ -854,7 +907,8 @@ final class OpenGlassViewController: UIViewController, UITextFieldDelegate, AVSp switch type { case .began: audioInterruptionActive = true - speech.stop() + waitForFreshAudio = false + speech.stop(reason: "audio_interruption") stopSpeaking() if isRunning { statusLabel.text = "系统音频暂时中断，恢复后继续自动语音。" } case .ended: diff --git a/ios/Sources/OpenGlassVoiceLoopPolicy.swift b/ios/Sources/OpenGlassVoiceLoopPolicy.swift index bd48bc0..1fd4a33 100644 --- a/ios/Sources/OpenGlassVoiceLoopPolicy.swift +++ b/ios/Sources/OpenGlassVoiceLoopPolicy.swift @@ -65,7 +65,7 @@ struct OpenGlassUtteranceTiming { if hasText, let lastTranscriptAt, now - lastTranscriptAt >= 2.5 { return .finish } - if let lastVoiceAt, now - lastVoiceAt >= 1.3 { + if hasText, let lastVoiceAt, now - lastVoiceAt >= 1.3 { return .finish } if now - startedAt >= 25 { diff --git a/ios/Sources/RokidIngressServer.swift b/ios/Sources/RokidIngressServer.swift index 8850657..2df3686 100644 --- a/ios/Sources/RokidIngressServer.swift +++ b/ios/Sources/RokidIngressServer.swift @@ -30,6 +30,9 @@ final class RokidIngressServer { private var audioPacketCount = 0 private var conversationPhase = "waiting" private var completedTurns = 0 + private var speechDiagnostics = OpenGlassSpeechDiagnostics() + private var speechGate = "receiver_stopped" + private let appBuild = Bundle.main.object(forInfoDictionaryKey: "CFBundleVersion") as? String ?? "unknown" private var epoch = UUID() // Read/written only on queue. epoch belongs to the main queue and only // filters UI callbacks; this also prevents an old listener accepting work. @@ -73,7 +76,7 @@ final class RokidIngressServer { self.clients[client.id] = client client.healthJSON = { [weak self] in guard let self else { return "{}" } - return "{\"service\":\"openglass-ios\",\"protocol\":1,\"receiver_name\":\"\(Self.serviceName)\",\"image_count\":\(self.imageCount),\"audio_packets_in\":\(self.audioPacketCount),\"audio_connected\":\(self.audioClient != nil),\"conversation_phase\":\"\(self.conversationPhase)\",\"completed_turns\":\(self.completedTurns)}" + return self.healthJSON() } client.onImage = { [weak self] data in guard let self else { return } @@ -125,17 +128,45 @@ final class RokidIngressServer { /// Only operational metadata is exposed; recognized speech and answers /// never appear in network diagnostics. - func setConversationStatus(phase: String, completedTurns: Int) { + func setConversationStatus(phase: String, completedTurns: Int, + asr: OpenGlassSpeechDiagnostics, gate: String) { dispatchPrecondition(condition: .onQueue(.main)) - let safePhase = ["paused", "listening", "thinking", "speaking", "waiting"].contains(phase) ? phase : "waiting" + let safePhase = ["paused", "listening", "finishing", "cooldown", "thinking", "speaking", "waiting"].contains(phase) ? phase : "waiting" let generation = epoch queue.async { [weak self] in guard let self, self.servingGeneration == generation else { return } self.conversationPhase = safePhase self.completedTurns = max(0, completedTurns) + self.speechDiagnostics = asr + self.speechGate = gate } } + private func healthJSON() -> String { + let asr = speechDiagnostics + var details: [String: Any] = [ + "state": asr.state.rawValue, "gate": speechGate, + "sessions": asr.sessions, "accepted_samples": asr.acceptedSamples, + "session_samples": asr.sessionSamples, "last_rms": asr.lastRMS, + "session_peak_rms": asr.sessionPeakRMS, "partial_count": asr.partialCount, + "nonempty_partial_count": asr.nonemptyPartialCount, + "empty_final_count": asr.emptyFinalCount, "nonempty_final_count": asr.nonemptyFinalCount, + "failure_count": asr.failureCount, "last_failure_domain": asr.lastFailureDomain, + "last_end_reason": asr.lastEndReason + ] + if let code = asr.lastFailureCode { details["last_failure_code"] = code } + let status: [String: Any] = [ + "service": "openglass-ios", "protocol": 1, "receiver_name": Self.serviceName, + "app_build": appBuild, "image_count": imageCount, + "audio_packets_in": audioPacketCount, "audio_connected": audioClient != nil, + "conversation_phase": conversationPhase, "completed_turns": completedTurns, + "asr": details + ] + guard let data = try? JSONSerialization.data(withJSONObject: status), + let json = String(data: data, encoding: .utf8) else { return "{}" } + return json + } + func stop() { dispatchPrecondition(condition: .onQueue(.main)) epoch = UUID() @@ -150,6 +181,8 @@ final class RokidIngressServer { audioPacketCount = 0 conversationPhase = "waiting" completedTurns = 0 + speechDiagnostics.reset() + speechGate = "receiver_stopped" imageDeliveryInFlight = false pendingAudioBytes = 0 closing.forEach { $0.close() } diff --git a/ios/tests/run_voice_loop_tests.sh b/ios/tests/run_voice_loop_tests.sh index 75d6413..cb6cd78 100644 --- a/ios/tests/run_voice_loop_tests.sh +++ b/ios/tests/run_voice_loop_tests.sh @@ -3,5 +3,8 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/../.." && pwd)" OUT="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/openglass-voice-loop-tests-$$" trap 'rm -f "$OUT"' EXIT -swiftc "$ROOT/ios/Sources/OpenGlassVoiceLoopPolicy.swift" "$ROOT/ios/tests/voice_loop/main.swift" -o "$OUT" +swiftc "$ROOT/ios/Sources/OpenGlassVoiceLoopPolicy.swift" \ + "$ROOT/ios/Sources/OpenGlassSpeechDiagnostics.swift" \ + "$ROOT/ios/Sources/OpenGlassSpeechHandoffBuffer.swift" \ + "$ROOT/ios/tests/voice_loop/main.swift" -o "$OUT" "$OUT" diff --git a/ios/tests/voice_loop/main.swift b/ios/tests/voice_loop/main.swift index b65b257..9942fc7 100644 --- a/ios/tests/voice_loop/main.swift +++ b/ios/tests/voice_loop/main.swift @@ -18,7 +18,7 @@ check(!startup.canListen(at: 200), "callbacks cannot create startup intent") startup.requestListening() check(startup.canListen(at: 200), "explicit start opens the gate") -// Silent recognition failures keep a positive, bounded delay. Wakeups at the +// Recognition failures keep a positive, bounded delay. Wakeups at the // deadline may retry; repeated audio/frame notifications before it may not. var silence = OpenGlassVoiceLoopPolicy() silence.requestListening() @@ -26,7 +26,7 @@ let expectedDelays: [TimeInterval] = [0.5, 1, 2, 4, 8, 8, 8] var now: TimeInterval = 1_000 for (index, expected) in expectedDelays.enumerated() { let delay = silence.retry(now: now) - checkTime(delay, expected, "silence retry \(index + 1) uses bounded backoff") + checkTime(delay, expected, "recognition failure retry \(index + 1) uses bounded backoff") checkTime(silence.notBefore, now + expected, "retry sets a future deadline") check(!silence.canListen(at: now), "retry cannot spin immediately") check(!silence.canListen(at: silence.notBefore - 0.001), "retry waits through its deadline") @@ -34,7 +34,7 @@ for (index, expected) in expectedDelays.enumerated() { now = silence.notBefore } for _ in 0..<1_000 { - checkTime(silence.retry(now: now), 8, "long silence stays capped at eight seconds") + checkTime(silence.retry(now: now), 8, "repeated recognition failures stay capped at eight seconds") check(!silence.canListen(at: now), "capped retries still wait") now = silence.notBefore } @@ -169,11 +169,27 @@ check(OpenGlassUtteranceTiming.action(now: 25, startedAt: 0, check(OpenGlassUtteranceTiming.action(now: 1.299, startedAt: 0, lastPacketAt: 1.299, lastVoiceAt: 0, lastTranscriptAt: nil, hasText: false) == .listen, - "voice silence waits the full 1.3 seconds") + "voice silence without text does not end an utterance early") check(OpenGlassUtteranceTiming.action(now: 1.3, startedAt: 0, lastPacketAt: 1.3, lastVoiceAt: 0, + lastTranscriptAt: nil, hasText: false) == .listen, + "empty recognition survives the 1.3-second voice silence boundary") +check(OpenGlassUtteranceTiming.action(now: 24.999, startedAt: 0, + lastPacketAt: 24.999, lastVoiceAt: 0, + lastTranscriptAt: nil, hasText: false) == .listen, + "an old RMS voice marker cannot end empty recognition before its cap") +check(OpenGlassUtteranceTiming.action(now: 25, startedAt: 0, + lastPacketAt: 25, lastVoiceAt: 0, lastTranscriptAt: nil, hasText: false) == .finish, - "silence after a voice marker ends even an empty utterance") + "empty recognition with an old voice marker still reaches its cap") +check(OpenGlassUtteranceTiming.action(now: 1.299, startedAt: 0, + lastPacketAt: 1.299, lastVoiceAt: 0, + lastTranscriptAt: 0.5, hasText: true) == .listen, + "recognized text waits the full 1.3 seconds of voice silence") +check(OpenGlassUtteranceTiming.action(now: 1.3, startedAt: 0, + lastPacketAt: 1.3, lastVoiceAt: 0, + lastTranscriptAt: 0.5, hasText: true) == .finish, + "recognized text finishes after 1.3 seconds of voice silence") // A stopped transport times out earlier than an otherwise empty utterance. check(OpenGlassUtteranceTiming.action(now: 4.999, startedAt: 0, @@ -190,11 +206,296 @@ check(OpenGlassUtteranceTiming.action(now: 5, startedAt: 0, "stable transcript finalization takes precedence over a packet timeout") check(OpenGlassUtteranceTiming.action(now: 5, startedAt: 0, lastPacketAt: 0, lastVoiceAt: 0, - lastTranscriptAt: nil, hasText: false) == .finish, - "voice silence finalization takes precedence over a packet timeout") + lastTranscriptAt: nil, hasText: false) == .audioTimeout, + "an empty transcript with an old voice marker still detects missing packets") +check(OpenGlassUtteranceTiming.action(now: 5, startedAt: 0, + lastPacketAt: 0, lastVoiceAt: 0, + lastTranscriptAt: nil, hasText: true) == .finish, + "recognized text can finalize voice silence before a packet timeout") check(OpenGlassUtteranceTiming.action(now: 25, startedAt: 0, lastPacketAt: 0, lastVoiceAt: nil, lastTranscriptAt: nil, hasText: false) == .finish, "duration cap finalization takes precedence over a packet timeout") -print("Passed \(checks) OpenGlass voice-loop policy checks") +// Diagnostics record only accepted samples and callback outcomes. An idle or +// finishing session cannot claim that late audio reached the recognizer. +let checksBeforeDiagnostics = checks +var diagnostics = OpenGlassSpeechDiagnostics() +check(diagnostics.state == .idle && diagnostics.sessions == 0, "diagnostics start idle") +check(diagnostics.lastEndReason == "not_started" && diagnostics.lastFailureCode == nil, + "initial diagnostics have no failure or end event") +diagnostics.appended(samples: 640, rms: 0.13) +diagnostics.partial(nonempty: true) +diagnostics.finishing(reason: "late_finish") +diagnostics.completed(nonempty: false, reason: "late_final") +check(diagnostics.acceptedSamples == 0 && diagnostics.sessionSamples == 0, + "idle audio is not counted as accepted") +check(diagnostics.partialCount == 0 && diagnostics.emptyFinalCount == 0, + "idle callbacks are not recognition results") +check(diagnostics.state == .idle && diagnostics.lastEndReason == "not_started", + "late callbacks cannot create a session") + +diagnostics.started() +check(diagnostics.state == .listening && diagnostics.sessions == 1, + "starting records one listening session") +check(diagnostics.lastEndReason == "none", "starting clears the previous end reason") +diagnostics.appended(samples: 640, rms: 0.13) +diagnostics.appended(samples: 320, rms: 0.05) +check(diagnostics.acceptedSamples == 960 && diagnostics.sessionSamples == 960, + "accepted sample totals count each appended buffer") +checkTime(diagnostics.lastRMS, 0.05, "last RMS follows the latest accepted buffer") +checkTime(diagnostics.sessionPeakRMS, 0.13, "session peak retains the loudest buffer") +diagnostics.appended(samples: 0, rms: 1) +diagnostics.appended(samples: -1, rms: 1) +check(diagnostics.acceptedSamples == 960 && diagnostics.sessionSamples == 960, + "empty and invalid sample counts are ignored") +checkTime(diagnostics.lastRMS, 0.05, "ignored buffers do not change the last RMS") +diagnostics.partial(nonempty: false) +diagnostics.partial(nonempty: true) +check(diagnostics.partialCount == 2 && diagnostics.nonemptyPartialCount == 1, + "partial diagnostics distinguish empty callbacks") +diagnostics.finishing(reason: "silence") +check(diagnostics.state == .finishing && diagnostics.lastEndReason == "silence", + "finishing records the utterance boundary") +diagnostics.finishing(reason: "duplicate_finish") +check(diagnostics.lastEndReason == "silence", "duplicate finishing preserves the original reason") +diagnostics.appended(samples: 640, rms: 0.9) +check(diagnostics.acceptedSamples == 960 && diagnostics.sessionSamples == 960, + "audio after endAudio is not counted as accepted") +checkTime(diagnostics.lastRMS, 0.05, "finishing ignores late RMS values") +diagnostics.partial(nonempty: true) +check(diagnostics.partialCount == 3 && diagnostics.nonemptyPartialCount == 2, + "recognition may report a partial while finishing") +diagnostics.completed(nonempty: false, reason: "empty_final") +check(diagnostics.state == .idle && diagnostics.emptyFinalCount == 1, + "empty final completes one active session") +check(diagnostics.lastEndReason == "empty_final", "final records its end reason") +diagnostics.completed(nonempty: false, reason: "duplicate_final") +diagnostics.completed(nonempty: true, reason: "stale_final") +check(diagnostics.emptyFinalCount == 1 && diagnostics.nonemptyFinalCount == 0, + "duplicate and stale finals cannot count a session twice") +check(diagnostics.lastEndReason == "empty_final", "late final cannot overwrite the completed reason") + +// Invalid RMS values are sanitized independently of the accepted sample count. +diagnostics.started() +check(diagnostics.sessions == 2 && diagnostics.sessionSamples == 0 && diagnostics.acceptedSamples == 960, + "a new session resets its sample count but preserves the cumulative total") +check(diagnostics.lastRMS == 0 && diagnostics.sessionPeakRMS == 0, + "a new session resets both RMS measurements") +check(diagnostics.partialCount == 3 && diagnostics.emptyFinalCount == 1, + "callback totals survive session boundaries") +diagnostics.appended(samples: 10, rms: .nan) +check(diagnostics.lastRMS == 0 && diagnostics.sessionPeakRMS == 0, + "NaN RMS cannot poison diagnostic values") +diagnostics.appended(samples: 10, rms: .infinity) +check(diagnostics.lastRMS == 0 && diagnostics.sessionPeakRMS == 0, + "infinite RMS cannot poison diagnostic values") +diagnostics.appended(samples: 10, rms: -0.2) +check(diagnostics.lastRMS == 0 && diagnostics.sessionPeakRMS == 0, + "negative RMS is clamped to zero") +diagnostics.appended(samples: 10, rms: 0.2) +diagnostics.appended(samples: 10, rms: -.infinity) +check(diagnostics.lastRMS == 0 && diagnostics.sessionPeakRMS == 0.2, + "invalid later RMS cannot erase the finite peak") +check(diagnostics.sessionSamples == 50 && diagnostics.acceptedSamples == 1_010, + "invalid measurements do not discard accepted sample counts") +diagnostics.completed(nonempty: true, reason: "recognized") +check(diagnostics.state == .idle && diagnostics.nonemptyFinalCount == 1 && diagnostics.emptyFinalCount == 1, + "nonempty final can complete directly from listening") + +// Synchronous start failures are visible even when no task became active. +diagnostics.failed(domain: "test.recognizer", code: 42, reason: "start_failed") +check(diagnostics.state == .idle && diagnostics.failureCount == 1 && diagnostics.sessions == 2, + "a synchronous failure counts without inventing a session") +check(diagnostics.lastFailureDomain == "test.recognizer" && diagnostics.lastFailureCode == 42 && + diagnostics.lastEndReason == "start_failed", "failure records only domain, code and reason") +diagnostics.started() +check(diagnostics.failureCount == 1 && diagnostics.lastFailureDomain == "test.recognizer" && + diagnostics.lastFailureCode == 42, "starting preserves the most recent failure metadata") +check(diagnostics.sessions == 3 && diagnostics.lastEndReason == "none", + "retry starts another session with no end reason") +diagnostics.appended(samples: 100, rms: 0.4) +diagnostics.failed(domain: "test.runtime", code: -7, reason: "recognition_failed") +check(diagnostics.state == .idle && diagnostics.failureCount == 2 && diagnostics.sessionSamples == 100, + "asynchronous failure preserves accepted samples for that session") +check(diagnostics.lastFailureDomain == "test.runtime" && diagnostics.lastFailureCode == -7, + "a newer failure replaces previous error metadata") +diagnostics.completed(nonempty: true, reason: "late_failure_final") +check(diagnostics.nonemptyFinalCount == 1 && diagnostics.lastEndReason == "recognition_failed", + "a final arriving after failure is ignored") + +diagnostics.started() +diagnostics.appended(samples: 200, rms: 0.1) +diagnostics.partial(nonempty: true) +diagnostics.stopped(reason: "audio_disconnected") +check(diagnostics.state == .idle && diagnostics.lastEndReason == "audio_disconnected", + "explicit stop records its reason") +check(diagnostics.sessions == 4 && diagnostics.sessionSamples == 200 && diagnostics.acceptedSamples == 1_310, + "explicit stop preserves sample and session totals") +check(diagnostics.partialCount == 4 && diagnostics.nonemptyPartialCount == 3 && + diagnostics.failureCount == 2, "stopping does not change callback totals") +diagnostics.completed(nonempty: true, reason: "late_stop_final") +diagnostics.appended(samples: 640, rms: 1) +diagnostics.partial(nonempty: true) +check(diagnostics.nonemptyFinalCount == 1 && diagnostics.emptyFinalCount == 1 && + diagnostics.acceptedSamples == 1_310 && diagnostics.partialCount == 4, + "stop rejects late final, partial and audio callbacks") +check(diagnostics.lastEndReason == "audio_disconnected", "late callbacks preserve the stop reason") +diagnostics.stopped(reason: "user_pause") +check(diagnostics.lastEndReason == "user_pause" && diagnostics.sessions == 4, + "an idle explicit stop can update its reason without changing counts") + +diagnostics.reset() +check(diagnostics.state == .idle && diagnostics.sessions == 0 && diagnostics.acceptedSamples == 0 && + diagnostics.sessionSamples == 0, "reset clears state and sample totals") +check(diagnostics.lastRMS == 0 && diagnostics.sessionPeakRMS == 0, "reset clears RMS values") +check(diagnostics.partialCount == 0 && diagnostics.nonemptyPartialCount == 0 && + diagnostics.emptyFinalCount == 0 && diagnostics.nonemptyFinalCount == 0 && diagnostics.failureCount == 0, + "reset clears every callback counter") +check(diagnostics.lastFailureDomain.isEmpty && diagnostics.lastFailureCode == nil && + diagnostics.lastEndReason == "not_started", "reset clears failure metadata and the end reason") +print("Passed \(checks - checksBeforeDiagnostics) OpenGlass speech-diagnostics checks") + +// Only audio that arrives after endAudio can be handed to the next request. +// A normal nonempty result, explicit stop or error discards the handoff. +let checksBeforeHandoff = checks +let handoffNow: TimeInterval = 10 +let firstPCM = Data([0, 1, 2, 3]) +let secondPCM = Data([4, 5, 6, 7]) +var handoff = OpenGlassSpeechHandoffBuffer(capacity: 8) +check(handoff.phase == .idle && handoff.byteCount == 0 && !handoff.hasReadyAudio(at: handoffNow), + "handoff begins idle and empty") +handoff.append(firstPCM) +check(handoff.byteCount == 0 && handoff.takeReadyAudio(now: handoffNow).isEmpty, + "audio before finishing cannot enter the handoff") +handoff.begin() +check(handoff.phase == .collecting && !handoff.hasReadyAudio(at: handoffNow), + "begin creates an empty collecting phase") +handoff.append(firstPCM) +check(handoff.byteCount == 4 && !handoff.hasReadyAudio(at: handoffNow), + "collected bytes are not ready until the empty final") +check(handoff.takeReadyAudio(now: handoffNow).isEmpty && handoff.byteCount == 4 && handoff.phase == .collecting, + "taking before completion cannot drain collecting audio") +handoff.complete(empty: true, now: handoffNow) +check(handoff.phase == .ready && handoff.hasReadyAudio(at: handoffNow) && handoff.byteCount == 4, + "empty final makes collected audio available") +handoff.append(secondPCM) +check(handoff.byteCount == 4, "audio after completion cannot enter the handoff") +check(handoff.takeReadyAudio(now: handoffNow) == firstPCM, "empty final hands off the original PCM bytes") +check(handoff.phase == .idle && handoff.byteCount == 0 && !handoff.hasReadyAudio(at: handoffNow), + "taking ready audio clears state and storage") +check(handoff.takeReadyAudio(now: handoffNow).isEmpty, "an empty final can only be drained once") +handoff.append(secondPCM) +check(handoff.byteCount == 0, "audio after draining cannot start another handoff") + +handoff.begin() +handoff.append(Data([0, 1, 2])) +handoff.append(Data([3, 4, 5, 6, 7, 8, 9])) +handoff.append(Data([10, 11])) +check(handoff.byteCount == 8, "collecting never exceeds the byte capacity") +handoff.complete(empty: true, now: handoffNow) +check(handoff.takeReadyAudio(now: handoffNow) == Data([0, 1, 2, 3, 4, 5, 6, 7]), + "overflow retains the stream prefix across packet boundaries") + +var oddCapacity = OpenGlassSpeechHandoffBuffer(capacity: 5) +oddCapacity.begin() +oddCapacity.append(Data([0, 1, 2])) +oddCapacity.append(Data([3, 4, 5])) +check(oddCapacity.byteCount == 4, "odd capacity rounds down to a complete PCM16 sample") +oddCapacity.complete(empty: true, now: handoffNow) +check(oddCapacity.takeReadyAudio(now: handoffNow) == firstPCM, + "odd packet boundaries preserve prefix order and sample alignment") +var tinyCapacity = OpenGlassSpeechHandoffBuffer(capacity: 1) +tinyCapacity.begin() +tinyCapacity.append(firstPCM) +tinyCapacity.complete(empty: true, now: handoffNow) +check(tinyCapacity.phase == .ready && tinyCapacity.byteCount == 0 && !tinyCapacity.hasReadyAudio(at: handoffNow), + "sub-sample capacity can complete without retaining a partial sample") +check(tinyCapacity.takeReadyAudio(now: handoffNow).isEmpty && tinyCapacity.phase == .idle, + "an empty ready buffer can be drained and cleared") +var negativeCapacity = OpenGlassSpeechHandoffBuffer(capacity: -3) +negativeCapacity.begin() +negativeCapacity.append(firstPCM) +check(negativeCapacity.byteCount == 0, "negative capacity is safely treated as zero") +var defaultCapacity = OpenGlassSpeechHandoffBuffer() +defaultCapacity.begin() +defaultCapacity.append(Data(repeating: 9, count: 70_403)) +check(defaultCapacity.byteCount == 70_400, "default capacity holds at most 2.2 seconds of PCM16") +defaultCapacity.clear() + +handoff.begin() +handoff.append(firstPCM) +handoff.complete(empty: false, now: handoffNow) +check(handoff.phase == .idle && handoff.byteCount == 0 && handoff.takeReadyAudio(now: handoffNow).isEmpty, + "a nonempty final discards buffered audio before inference or TTS") +handoff.append(secondPCM) +check(handoff.byteCount == 0, "nonempty completion rejects late audio") +handoff.begin() +handoff.append(firstPCM) +handoff.clear() +handoff.complete(empty: true, now: handoffNow) +check(handoff.phase == .idle && handoff.takeReadyAudio(now: handoffNow).isEmpty, + "stop or error before finalization prevents a late empty final from reviving audio") +handoff.begin() +handoff.append(firstPCM) +handoff.complete(empty: true, now: handoffNow) +handoff.clear() +check(handoff.phase == .idle && handoff.byteCount == 0 && handoff.takeReadyAudio(now: handoffNow).isEmpty, + "stop or error also discards audio already ready for handoff") + +handoff.begin() +handoff.append(firstPCM) +handoff.begin() +check(handoff.phase == .collecting && handoff.byteCount == 0, + "starting a new collection discards an unfinished old collection") +handoff.append(secondPCM) +handoff.complete(empty: true, now: handoffNow) +check(handoff.takeReadyAudio(now: handoffNow) == secondPCM, "a new round never replays the previous round's PCM") +handoff.begin() +handoff.append(firstPCM) +handoff.complete(empty: true, now: handoffNow) +handoff.begin() +handoff.append(secondPCM) +handoff.complete(empty: true, now: handoffNow) +check(handoff.takeReadyAudio(now: handoffNow) == secondPCM, "begin also isolates an old ready handoff from a new round") +handoff.begin() +handoff.append(firstPCM) +handoff.complete(empty: true, now: handoffNow) +handoff.complete(empty: false, now: handoffNow) +check(handoff.takeReadyAudio(now: handoffNow).isEmpty && handoff.phase == .idle, + "a nonempty outcome invalidates a previously ready handoff") + +handoff.begin() +handoff.append(Data([0x34])) +handoff.append(Data([0x12])) +handoff.complete(empty: true, now: handoffNow) +check(handoff.takeReadyAudio(now: handoffNow) == Data([0x34, 0x12]), + "two half-sample packets concatenate without losing either byte") +handoff.begin() +handoff.append(Data([0x34])) +handoff.complete(empty: true, now: handoffNow) +check(handoff.takeReadyAudio(now: handoffNow) == Data([0x34]), + "an incomplete final sample is preserved for the next live packet") + +handoff.begin() +handoff.append(firstPCM) +handoff.complete(empty: true, now: handoffNow) +check(handoff.hasReadyAudio(at: handoffNow + 0.499), "handoff remains ready within its short transfer window") +check(handoff.hasReadyAudio(at: handoffNow + 0.5), "handoff is valid at the half-second deadline") +check(!handoff.hasReadyAudio(at: handoffNow + 0.501), "handoff expires after the half-second deadline") +check(handoff.takeReadyAudio(now: handoffNow + 0.501).isEmpty && handoff.phase == .idle && handoff.byteCount == 0, + "delayed frame or admission gate cannot replay expired PCM") +handoff.begin() +handoff.append(secondPCM) +handoff.complete(empty: true, now: handoffNow + 1) +check(!handoff.hasReadyAudio(at: handoffNow), "a timestamp before completion cannot make a handoff ready") +check(handoff.takeReadyAudio(now: handoffNow + 1.5) == secondPCM, + "new completion owns a fresh deadline and can drain exactly at it") +handoff.begin() +handoff.append(firstPCM) +handoff.complete(empty: true, now: handoffNow) +handoff.clear() +check(!handoff.hasReadyAudio(at: handoffNow + 0.1) && handoff.takeReadyAudio(now: handoffNow + 0.1).isEmpty, + "clear invalidates an unexpired ready handoff") +print("Passed \(checks - checksBeforeHandoff) OpenGlass speech-handoff checks") +print("Passed \(checks) OpenGlass voice-loop, diagnostics and handoff checks") diff --git a/rokid_app/OpenGlassRokidSensor/app/build.gradle b/rokid_app/OpenGlassRokidSensor/app/build.gradle index b1dbdb2..2d97f90 100644 --- a/rokid_app/OpenGlassRokidSensor/app/build.gradle +++ b/rokid_app/OpenGlassRokidSensor/app/build.gradle @@ -11,7 +11,7 @@ android { applicationId 'org.opensqz.openglass.rokid' minSdk 28 targetSdk 34 - versionCode 7 + versionCode 8 versionName '0.2.0' manifestPlaceholders = [appLabel: '@string/app_name'] diff --git a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt index 64eca6d..2faacbd 100644 --- a/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt +++ b/rokid_app/OpenGlassRokidSensor/app/src/main/java/org/opensqz/openglass/rokid/MainActivity.kt @@ -114,7 +114,6 @@ class MainActivity : Activity() { @Volatile private var audioSourceText = "none" @Volatile private var cameraStatus = "idle" @Volatile private var audioStatus = "idle" - @Volatile private var microphoneSilenced: Boolean? = null @Volatile private var imageSizeText = "none" @Volatile private var audioWsUrlText = "none" @Volatile private var imagePostBaseText = "none" @@ -339,7 +338,6 @@ class MainActivity : Activity() { private fun startAll() { if (running) return running = true - microphoneSilenced = null captureGeneration.incrementAndGet() startMs = System.currentTimeMillis() lastError = "none" @@ -366,7 +364,6 @@ class MainActivity : Activity() { private fun stopAll() { if (!running) return running = false - microphoneSilenced = null captureGeneration.incrementAndGet() cameraRestartPending.set(false) Log.i(TAG, "stopAll") @@ -491,10 +488,10 @@ class MainActivity : Activity() { if (now - lastLog >= 1_000L) { // Observe this recorder only. An unavailable configuration stays unknown; // this diagnostic does not alter the audio source, routing or samples. + // Rokid can report true even with audible PCM, so do not infer silence from it. val silenced = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { runCatching { recorder.activeRecordingConfiguration?.isClientSilenced }.getOrNull() } else null - if (token == captureGeneration.get()) microphoneSilenced = silenced Log.i( TAG, "audio source=$audioSourceText pkt=${audioPackets.get()} drop=${audioDrops.get()} rms=${fmt(lastRms)} peak=$lastPeak nonzero=$lastNonZero wsOpen=$wsOpen silenced=${silenced?.toString() ?: "unknown"}", @@ -513,7 +510,6 @@ class MainActivity : Activity() { recorder.release() if (activeRecorder === recorder) { activeRecorder = null - microphoneSilenced = null } Log.i(TAG, "AudioRecord stopped") } @@ -1149,7 +1145,6 @@ class MainActivity : Activity() { val status = when { !hasPermissions() -> "请允许相机和麦克风" !running -> "OpenGlass · 已暂停" - microphoneSilenced == true -> "系统提示麦克风静音" currentEndpoint == null || !wsOpen -> "OpenGlass · 连接中" else -> "OpenGlass · 已连接" } diff --git a/rokid_app/README.md b/rokid_app/README.md index 7d834b5..66c235b 100644 --- a/rokid_app/README.md +++ b/rokid_app/README.md @@ -43,7 +43,7 @@ The default display is one dim, centered status line on black: connected, connec **Long-press the display or press Menu** to open settings: Auto, Manual address, USB relay, Choose phone again, and an explicit Pause/Resume capture item. Use **关闭 (Close)** or Back to dismiss the settings dialog. Manual accepts `http://
:18080`; USB selects `http://127.0.0.1:18080` and requires the existing computer USB relay. Choose phone again forgets the remembered receiver. A tap or Enter/DPAD resumes paused capture and does nothing while capture is running, so a casual touch cannot stop the stream. Holding Enter/DPAD does not open the menu. To pause, choose Pause capture from the menu. Back on the main screen still stops, then exits. The full-screen status view disables Android's default focus highlight, and the opaque window and system-bar backgrounds are black. -On Android 10+, the audio diagnostic log includes `silenced=true/false/unknown` from this recorder's active configuration. `unknown` means unavailable, not “not silenced.” When the system reports `true`, the single-line display says `系统提示麦克风静音` (system reports microphone silencing). This observation does not change recording parameters or audio routing; a low RMS alone is not treated as proof of system silencing. +On Android 10+, the audio diagnostic log includes `silenced=true/false/unknown` from this recorder's active configuration. `unknown` means unavailable, not “not silenced.” Rokid has reported `true` while capturing high-amplitude PCM, so this field is logged only and does not trigger a mute warning or recording restart. This observation does not change recording parameters or audio routing; a low RMS alone is not treated as proof of system silencing. Launch intents remain compatible: an explicit `pc_base_url` fixes the endpoint for that launch and persists the URL as a future fallback. `auto_discover=true` explicitly selects Auto and overrides the old saved USB address. The connection-settings mode persists, while an intent URL alone does not turn a future launcher start into USB mode. diff --git a/rokid_app/README_zh.md b/rokid_app/README_zh.md index bf3e373..61fa8de 100644 --- a/rokid_app/README_zh.md +++ b/rokid_app/README_zh.md @@ -1,6 +1,6 @@ # Rokid 采集 App -此目录保存 OpenGlass Rokid 采集 App 0.2.0（versionCode 7）：发送 JPEG 与 PCM 音频，可通过 Wi-Fi 直接连接 iPhone OpenGlass，也保留电脑面板和显式 USB 中继模式。它不是眼镜系统固件。 +此目录保存 OpenGlass Rokid 采集 App 0.2.0（versionCode 8）：发送 JPEG 与 PCM 音频，可通过 Wi-Fi 直接连接 iPhone OpenGlass，也保留电脑面板和显式 USB 中继模式。它不是眼镜系统固件。 ## 文件布局 @@ -58,7 +58,7 @@ adb -s install -r .\app\build\outputs\apk\wifi\app-wifi.apk **长按界面或按 Menu** 打开设置，提供 Auto、Manual address、USB relay、Choose phone again，以及明确的“暂停采集 / 恢复采集”选项。点击**关闭**或按 Back 可关闭设置弹窗。手动地址格式为 `http://<地址>:18080`；USB 模式使用 `http://127.0.0.1:18080`，仍需要电脑上的既有 USB 中继。Choose phone again 会忘记已选择的手机。短按界面或 Enter/DPAD 仅用于恢复已暂停的采集，运行中不会因普通点击而停止；按住 Enter/DPAD 也不会打开菜单。需要暂停时从菜单选择“暂停采集”。主界面 Back 仍先停止，再退出。全屏状态文字关闭系统默认焦点高亮，窗口与系统栏均使用不透明黑底。 -Android 10 及以上的音频诊断日志会读取当前 AudioRecord 的配置，记录 `silenced=true/false/unknown`；`unknown` 表示配置不可用，不能当作 `false`。仅系统明确返回 `true` 时，单行界面提示“系统提示麦克风静音”。诊断不会改动录音参数或声音路由，也不会仅凭 RMS 很低就断定麦克风被系统静音。 +Android 10 及以上的音频诊断日志会读取当前 AudioRecord 的配置，记录 `silenced=true/false/unknown`；`unknown` 表示配置不可用，不能当作 `false`。实测 Rokid 在有明显音量的 PCM 输入时也可能报告 `true`，因此该字段仅保留在日志中，不用于界面静音提示或触发录音重启。诊断不会改动录音参数或声音路由，也不会仅凭 RMS 很低就断定麦克风被系统静音。 启动参数仍兼容：显式 `pc_base_url` 会在本次启动固定连接该地址，并保存为以后自动模式的后备地址；`auto_discover=true` 显式切回自动，覆盖旧保存的 USB 地址。设置界面选择的模式会保存，仅传入一次 URL 不会把以后的普通启动永久绑定到 USB。 From 3ff7bc6a692ba2f23cd588f43ab6d78a544789bb Mon Sep 17 00:00:00 2001 From: MengzhangLI Date: Sun, 18 Oct 2026 03:17:26 +0800 Subject: [PATCH 11/11] Document automatic speech repair build verification [skip ci] --- ios/README_zh.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ios/README_zh.md b/ios/README_zh.md index 4063a19..7b37e6a 100644 --- a/ios/README_zh.md +++ b/ios/README_zh.md @@ -236,6 +236,7 @@ powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File 'C:\path\to\OpenGl - 2026-10-18 热点真机测试：眼镜直接向当前 iPhone 热点网关发送图像和音频，手机接收计数持续增长。拔掉眼镜 USB 后，ADB 已无设备，手机仍持续收流，`completed_turns` 增至 `3`。随后用户确认：无需点按钮即可连续问答，且眼镜和 iPhone 两根 USB 均拔掉后仍正常。已验证本机 ASR → MiniCPM-V 4.6 → TTS 的无线自动多轮流程；酒店 Wi-Fi 尚未实测。 - 包含 Android 最终改动的提交 `c439fa3` 也通过 [CI 39058773419](https://github.com/OpenSQZ/OpenGlass/actions/runs/39058773419)。其 iOS Swift 源码与已安装的 build `5` 相同，无需再次安装。 - 后续用户确认单行眼镜界面和首轮 ASR/TTS 正常，但再次提问出现空识别，说明此前成功记录不足以证明持续对话可靠。此次修复空轮退避和收尾期间丢音，并增加 ASR 操作状态诊断；新版端到端多轮结果待安装后复测。 +- 语音修复提交 `57589c0` 通过 [CI 39061505661](https://github.com/OpenSQZ/OpenGlass/actions/runs/39061505661)：2176项语音循环/诊断/交接断言、1399项协议检查、7项工程准备测试、7项USB中继测试及真机Release编译通过。未签名IPA build `11` 已下载，manifest、ZIP完整性、Info.plist版本及bundle ID已核验；SHA256为 `1B9839531088356F914B6510C33377A971CFBF6EF162915106DC196413305B6A`。Rokid code `8` 的单行界面更新已覆盖安装并确认相机/麦克风授权；APK SHA256为 `A06D29A45CC2D346A331EFB7AA63B8FA3220963EEC3AABE83C57554A4CFD80F5`。 - 不在此文档写入个人设备序列号、UDID、Apple账号或内网凭据。现场更新请用设备类型和验证结果描述。 可重复执行无设备测试：