From 2759f136e45b904793bb5503c2dc0c7d8f5d4389 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Thu, 15 Jan 2026 15:32:03 +0800 Subject: [PATCH 01/25] =?UTF-8?q?feat(azure=5Fspeech):=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0iOS=E9=A1=B9=E7=9B=AE=E5=9F=BA=E7=A1=80=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=92=8C=E9=9F=B3=E9=A2=91=E6=8E=A5=E6=94=B6=E5=99=A8?= =?UTF-8?q?HFP=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复音频会话配置逻辑，优化蓝牙耳机兼容性 添加iOS Runner项目文件和测试用例 更新音频接收器以支持HFP模式 --- .../Sources/tools/SimpleAudioReceiver.swift | 81 ++++++++++++------- .../ble_service/BleCommandSender.kt | 2 +- .../yunqiinnovation/ble_service/BleService.kt | 2 +- 3 files changed, 52 insertions(+), 33 deletions(-) diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift index 6aebfac1a..3e9030252 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift @@ -55,6 +55,7 @@ public class SimpleAudioReceiver: NSObject { //public var onAudioData: ((Data) -> Void)? public var recordfile: RecordFile? public var isHeadphones = true + public var isReceiverHFP = false // 是否是接收HFP音频 /** * 初始化 @@ -119,6 +120,7 @@ public class SimpleAudioReceiver: NSObject { print("AudioStream音频配置设置完成") } + /** * 开始音频输入 */ @@ -129,9 +131,11 @@ public class SimpleAudioReceiver: NSObject { print("startAudioRecord=audioSourceType\(audioSourceType)") switch audioSourceType { case .microphone: - runMicrophoneCapture() + isReceiverHFP = true + runMicrophoneCapture() case .external: - runExternalCapture() + isReceiverHFP = false + runExternalCapture() } } @@ -225,20 +229,14 @@ public class SimpleAudioReceiver: NSObject { */ private func runExternalCapture() { if audioSourceType == .external { - // do { - // try audioSession.setCategory(.playback, - // mode: .videoChat, - // options: [ .mixWithOthers,.allowBluetoothA2DP // 允许蓝牙耳机，不占用hfp链路 - // ]) // 添加音频优先级控制 - // try audioSession.overrideOutputAudioPort(.none) - // try audioSession.setActive(true) - // } catch { - // print("设置音频会话失败: \(error.localizedDescription)") - // } - + do { + // 根据通话状态进行安全的音频会话配置 + try safeConfigureAudioSessionForMic() + } catch { + print("设置音频会话失败: \(error.localizedDescription)") + } //pushAudioData(data: Data()) micCapture.stopCapture() - } } /** @@ -254,25 +252,46 @@ public class SimpleAudioReceiver: NSObject { let desiredCategory: AVAudioSession.Category let desiredMode: AVAudioSession.Mode let desiredOptions: AVAudioSession.CategoryOptions - - if isTelephonyActive() { - // 通话中：录音优先，避免 A2DP（仅播放），改用 HFP（.allowBluetooth） - desiredCategory = .record - desiredMode = .voiceChat - desiredOptions = [.allowBluetooth] - } else { - if(isHeadphones){ - // 非通话：保持原有逻辑，但使用 .allowBluetooth 以支持录音（而不是 A2DP） - desiredCategory = .playAndRecord - desiredMode = .videoChat - desiredOptions = [.allowBluetooth, .mixWithOthers, .defaultToSpeaker] - //desiredOptions = [.mixWithOthers, .defaultToSpeaker] - }else{ - desiredCategory = .playAndRecord - desiredMode = .videoChat - desiredOptions = [.mixWithOthers, .defaultToSpeaker] + if isReceiverHFP { + if isTelephonyActive() { + // 通话中：录音优先，避免 A2DP（仅播放），改用 HFP（.allowBluetooth） + desiredCategory = .record + desiredMode = .voiceChat + desiredOptions = [.allowBluetooth] + } else { + if(isHeadphones){ + // 非通话：保持原有逻辑，但使用 .allowBluetooth 以支持录音（而不是 A2DP） + desiredCategory = .playAndRecord + desiredMode = .videoChat + desiredOptions = [.allowBluetooth, .mixWithOthers, .defaultToSpeaker] + //desiredOptions = [.mixWithOthers, .defaultToSpeaker] + }else{ + desiredCategory = .playAndRecord + desiredMode = .videoChat + desiredOptions = [.mixWithOthers, .defaultToSpeaker] + } + } + }else{ + if isTelephonyActive() { + // 通话中：录音优先，避免 A2DP（仅播放），改用 HFP（.allowBluetooth） + desiredCategory = .record + desiredMode = .voiceChat + desiredOptions = [.allowBluetoothA2DP] + } else { + if(isHeadphones){ + // 非通话：保持原有逻辑，但使用 .allowBluetooth 以支持录音（而不是 A2DP） + desiredCategory = .playAndRecord + desiredMode = .videoChat + desiredOptions = [.allowBluetoothA2DP, .mixWithOthers, .defaultToSpeaker] + //desiredOptions = [.mixWithOthers, .defaultToSpeaker] + }else{ + desiredCategory = .playAndRecord + desiredMode = .videoChat + desiredOptions = [.mixWithOthers, .defaultToSpeaker] + } } } + // 仅在类别或模式变化时才调用 setCategory，避免触发不必要的 categoryChange let needsSetCategory = (audioSession.category != desiredCategory) || (audioSession.mode != desiredMode) diff --git a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCommandSender.kt b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCommandSender.kt index ed0f64736..e5b52623a 100644 --- a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCommandSender.kt +++ b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCommandSender.kt @@ -456,7 +456,7 @@ class BleCommandSender { fun processDeviceNotification(data: ByteArray) { Log.d( TAG, - "收到设备主动上报: ${data.joinToString(", ") { "0x${(it.toInt() and 0xFF).toString(16)}" }}" + "ble指令----------- 收到设备主动上报: ${data.joinToString(", ") { "0x${(it.toInt() and 0xFF).toString(16)}" }}" ) if (data.size < 3) { Log.e(TAG, "设备主动上报数据格式错误：数据长度过短") diff --git a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt index 2daa1d6e1..b42b08c0a 100644 --- a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt +++ b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt @@ -973,7 +973,7 @@ object BleService { } // 通知特征数据（命令和控制） BleConst.NOTIFY_CHAR_UUID -> { - Log.i(TAG, "收到通知特征数据（命令和控制）") + Log.i(TAG, "ble指令----------- 收到通知特征数据（命令和控制）") // 判断数据类型 when { // 设备响应 (0xBB开头) From 74dd28d11712a346e674454278c3dbc4e0c9419c Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Thu, 15 Jan 2026 17:59:32 +0800 Subject: [PATCH 02/25] =?UTF-8?q?fix(azure=5Fspeech):=20=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E8=93=9D=E7=89=99HFP=E9=BA=A6=E5=85=8B=E9=A3=8E=E6=A3=80?= =?UTF-8?q?=E6=B5=8B=E5=92=8C=E9=9F=B3=E9=A2=91=E8=B7=AF=E7=94=B1=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 动态检测蓝牙HFP输入设备并优先选择 移除默认外放选项，根据输出设备智能路由 添加音频会话配置的兜底逻辑 --- .../agent_service/AgentServiceImpl.swift | 2 +- .../Sources/tools/SimpleAudioReceiver.swift | 44 ++++++++++++++++--- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 1050f1b19..b5abb3e9f 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -191,7 +191,7 @@ class AgentServiceImpl: NSObject { """ } func initialize(config: [String: Any]) -> Bool { - if isInitialized { return true } + // if isInitialized { return true } // os_log("initialize: config=%{public}@", log: logger, type: .info, config) self.config = config diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift index 3e9030252..a3b4380e2 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift @@ -128,10 +128,11 @@ public class SimpleAudioReceiver: NSObject { _isWriting=true self.audioSourceType = audioSourceType; self.audioDataCallback = audioDataCallback - print("startAudioRecord=audioSourceType\(audioSourceType)") + print("音源设制---------- startAudioRecord=audioSourceType:\(audioSourceType)") switch audioSourceType { case .microphone: - isReceiverHFP = true + // 根据当前可用输入设备动态判定是否存在蓝牙HFP麦克风 + isReceiverHFP = hasBluetoothHFPInput() runMicrophoneCapture() case .external: isReceiverHFP = false @@ -139,6 +140,21 @@ public class SimpleAudioReceiver: NSObject { } } + /** + * 检测是否存在蓝牙HFP输入设备 + * 参数：无 + * 返回值：存在返回true，否则返回false + * 异常：无 + */ + private func hasBluetoothHFPInput() -> Bool { + let session = AVAudioSession.sharedInstance() + if let inputs = session.availableInputs { + return inputs.contains { $0.portType == .bluetoothHFP } + } + // 兜底使用当前路由的输入 + return session.currentRoute.inputs.contains { $0.portType == .bluetoothHFP } + } + /** * 开启音频写入线程 * 修复：使用userInitiated QoS避免优先级反转 @@ -263,7 +279,7 @@ public class SimpleAudioReceiver: NSObject { // 非通话：保持原有逻辑，但使用 .allowBluetooth 以支持录音（而不是 A2DP） desiredCategory = .playAndRecord desiredMode = .videoChat - desiredOptions = [.allowBluetooth, .mixWithOthers, .defaultToSpeaker] + desiredOptions = [.allowBluetooth, .mixWithOthers] //desiredOptions = [.mixWithOthers, .defaultToSpeaker] }else{ desiredCategory = .playAndRecord @@ -276,13 +292,13 @@ public class SimpleAudioReceiver: NSObject { // 通话中：录音优先，避免 A2DP（仅播放），改用 HFP（.allowBluetooth） desiredCategory = .record desiredMode = .voiceChat - desiredOptions = [.allowBluetoothA2DP] + desiredOptions = [.allowBluetooth] } else { if(isHeadphones){ // 非通话：保持原有逻辑，但使用 .allowBluetooth 以支持录音（而不是 A2DP） desiredCategory = .playAndRecord desiredMode = .videoChat - desiredOptions = [.allowBluetoothA2DP, .mixWithOthers, .defaultToSpeaker] + desiredOptions = [.allowBluetoothA2DP, .mixWithOthers] //desiredOptions = [.mixWithOthers, .defaultToSpeaker] }else{ desiredCategory = .playAndRecord @@ -307,6 +323,22 @@ public class SimpleAudioReceiver: NSObject { // 保持激活；这里重复 setActive(true) 一般安全，但如果你观察到重复激活也触发路由回调，可在外层控制频率 try audioSession.setActive(true, options: .notifyOthersOnDeactivation) + + // 路由校正：有蓝牙输出时不强制外放，并优先选择蓝牙输入（HFP）或内置麦克风 + let outputs = audioSession.currentRoute.outputs + let hasBtOutput = outputs.contains { out in + out.portType == .bluetoothA2DP || out.portType == .bluetoothHFP || out.portType == .bluetoothLE + } + if hasBtOutput { + try audioSession.overrideOutputAudioPort(.none) + } + if let inputs = audioSession.availableInputs { + if isReceiverHFP, let btInput = inputs.first(where: { $0.portType == .bluetoothHFP }) { + try audioSession.setPreferredInput(btInput) + } else if let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { + try audioSession.setPreferredInput(builtInMic) + } + } } catch { // 兜底：尽量保证能开始录音 print("音频会话配置失败，尝试兜底配置: \(error)") @@ -561,5 +593,3 @@ private class LinkedBlockingQueue { } } - - From 818247411b23238f4fcd6bb262677875969f46c9 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Fri, 16 Jan 2026 19:29:58 +0800 Subject: [PATCH 03/25] =?UTF-8?q?fix(audio):=20=E4=BF=AE=E5=A4=8D=E8=93=9D?= =?UTF-8?q?=E7=89=99=E9=9F=B3=E9=A2=91=E8=B7=AF=E7=94=B1=E9=97=AE=E9=A2=98?= =?UTF-8?q?=E5=B9=B6=E4=BC=98=E5=8C=96=E5=A4=96=E9=83=A8=E9=9F=B3=E9=A2=91?= =?UTF-8?q?=E6=BA=90=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 强制禁用HFP通话音道，改为使用手机麦克风加A2DP输出 新增耳机/蓝牙输出设备检测功能 优化外部音频源的会话配置逻辑 --- .../Sources/tools/SimpleAudioReceiver.swift | 70 +++++++++++++++++-- 1 file changed, 64 insertions(+), 6 deletions(-) diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift index a3b4380e2..0ae23d2e4 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift @@ -123,6 +123,9 @@ public class SimpleAudioReceiver: NSObject { /** * 开始音频输入 + * 参数：audioSourceType 音频源类型（microphone 或 external）；audioDataCallback 音频数据回调 + * 返回值：无 + * 异常：无（内部捕获并打印错误，不抛出） */ public func startAudioRecord(audioSourceType: AudioSourceType = .microphone, audioDataCallback: AudioDataCallback?) { _isWriting=true @@ -131,8 +134,10 @@ public class SimpleAudioReceiver: NSObject { print("音源设制---------- startAudioRecord=audioSourceType:\(audioSourceType)") switch audioSourceType { case .microphone: - // 根据当前可用输入设备动态判定是否存在蓝牙HFP麦克风 - isReceiverHFP = hasBluetoothHFPInput() + // 唤醒后禁用通话音道（HFP），改为手机麦克风 + A2DP 输出 + isReceiverHFP = false + // 动态检测是否存在耳机/蓝牙输出，用于后续路由选项计算 + isHeadphones = detectHeadphonesOutput() runMicrophoneCapture() case .external: isReceiverHFP = false @@ -155,6 +160,23 @@ public class SimpleAudioReceiver: NSObject { return session.currentRoute.inputs.contains { $0.portType == .bluetoothHFP } } + /** + * 检测是否存在耳机/蓝牙输出设备（用于决定是否启用A2DP而非扬声器） + * 参数：无 + * 返回值：存在返回true，否则返回false + * 异常：无 + */ + private func detectHeadphonesOutput() -> Bool { + let session = AVAudioSession.sharedInstance() + let outputs = session.currentRoute.outputs + return outputs.contains { out in + out.portType == .bluetoothA2DP || + out.portType == .bluetoothHFP || + out.portType == .bluetoothLE || + out.portType == .headphones + } + } + /** * 开启音频写入线程 * 修复：使用userInitiated QoS避免优先级反转 @@ -246,13 +268,50 @@ public class SimpleAudioReceiver: NSObject { private func runExternalCapture() { if audioSourceType == .external { do { - // 根据通话状态进行安全的音频会话配置 - try safeConfigureAudioSessionForMic() + // 停止麦克风采集，确保不走通话音道 + micCapture.stopCapture() + // 外部模式会话配置：禁止HFP，仅允许A2DP/耳机输出 + try safeConfigureAudioSessionForExternal() } catch { print("设置音频会话失败: \(error.localizedDescription)") } //pushAudioData(data: Data()) - micCapture.stopCapture() + } + } + + /** + * 外部音频源的安全音频会话配置（禁止HFP通话音道） + * 参数：无 + * 返回值：无 + * 异常：可能抛出AVAudioSession设置错误（内部捕获并兜底） + */ + private func safeConfigureAudioSessionForExternal() throws { + let audioSession = self.audioSession + do { + // 外部模式不录麦，仅做播放路由；有耳机时只允许A2DP，禁止HFP + let outputs = audioSession.currentRoute.outputs + let hasHeadphones = outputs.contains { out in + out.portType == .bluetoothA2DP || + out.portType == .bluetoothLE || + out.portType == .headphones + } + let desiredCategory: AVAudioSession.Category = .playback + let desiredMode: AVAudioSession.Mode = .videoChat + let desiredOptions: AVAudioSession.CategoryOptions = hasHeadphones + ? [.allowBluetoothA2DP, .mixWithOthers] + : [.mixWithOthers, .defaultToSpeaker] + + let needsSetCategory = (audioSession.category != desiredCategory) || (audioSession.mode != desiredMode) + if needsSetCategory { + try audioSession.setCategory(desiredCategory, mode: desiredMode, options: desiredOptions) + } + // 取消扬声器强制，确保耳机/A2DP优先 + try audioSession.overrideOutputAudioPort(.none) + try audioSession.setActive(true, options: .notifyOthersOnDeactivation) + } catch { + print("外部模式音频会话配置失败，兜底到playback: \(error)") + try audioSession.setCategory(.playback, mode: .default, options: [.mixWithOthers, .defaultToSpeaker]) + try audioSession.setActive(true, options: .notifyOthersOnDeactivation) } } /** @@ -592,4 +651,3 @@ private class LinkedBlockingQueue { } } } - From 4e82bb13f6c75204f29bdd13b7129fac55a49d19 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Sat, 17 Jan 2026 14:23:34 +0800 Subject: [PATCH 04/25] =?UTF-8?q?refactor(=E9=9F=B3=E9=A2=91=E5=A4=84?= =?UTF-8?q?=E7=90=86):=20=E4=BF=AE=E6=94=B9=E9=9F=B3=E9=A2=91=E4=BC=9A?= =?UTF-8?q?=E8=AF=9D=E5=92=8C=E8=AF=AD=E9=9F=B3=E8=AF=86=E5=88=AB=E7=9B=B8?= =?UTF-8?q?=E5=85=B3=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将AzureAsrHelper的performAudioStart方法改为public以便外部调用 在AgentServiceImpl中增加音频会话检查和设置逻辑 调整MusicService中音频会话设置的注释 缩短AgentServiceImpl的空闲检测时间至5秒 --- .../Sources/agent_service/AgentServiceImpl.swift | 10 +++++++++- .../Sources/azure_speech/AzureAsrHelper.swift | 2 +- .../Sources/music_service/MusicService.swift | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index b5abb3e9f..4755617ff 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -100,7 +100,7 @@ class AgentServiceImpl: NSObject { private let timerQueue = DispatchQueue(label: "com.example.idleTimer.queue") private var idleTimer: DispatchSourceTimer? - private let maxIdleSeconds: TimeInterval = 10 + private let maxIdleSeconds: TimeInterval = 5 var micCapture: MicrophoneCapture! // 语音识别模式 public var currentRecognitionMode: String = "normal" @@ -311,6 +311,9 @@ class AgentServiceImpl: NSObject { LocationService.shared.getCurrentLocation() { location in // self.locationInfo = location } + + checkAdnSetAudioSession() + return isInitialized } @@ -387,6 +390,11 @@ class AgentServiceImpl: NSObject { return true } + private func checkAdnSetAudioSession() { + let audioSourceType: AzureAsrHelper.AudioSourceType = BleService.shared.isConnected() ? .external : .microphone + azureAsrHelper?.performAudioStart(audioSourceType: audioSourceType, audioDataCallback: nil) + } + private func startIdleCheckForMode(mode: String) { stopIdleCheck() guard isRecognizing else { return } diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift index 9658a20af..3ea1fb6a0 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift @@ -338,7 +338,7 @@ public class AzureAsrHelper: NSObject { /** * 执行音频启动操作 */ - private func performAudioStart( + public func performAudioStart( audioSourceType: AudioSourceType, audioDataCallback: SimpleAudioReceiver.AudioDataCallback? ) -> Bool { diff --git a/local_plugins/music_service/ios/music_service/Sources/music_service/MusicService.swift b/local_plugins/music_service/ios/music_service/Sources/music_service/MusicService.swift index c13217297..0eb558514 100644 --- a/local_plugins/music_service/ios/music_service/Sources/music_service/MusicService.swift +++ b/local_plugins/music_service/ios/music_service/Sources/music_service/MusicService.swift @@ -41,10 +41,10 @@ public class MusicService: NSObject { private override init() { super.init() setupRemoteCommandCenter() - setupAudioSession() + // setupAudioSession() setupNotificationObservers() // 监听通话状态（需主线程队列） -// callObserver.setDelegate(self, queue: DispatchQueue.main) + //callObserver.setDelegate(self, queue: DispatchQueue.main) } public func initialize(serviceUrl: String, userToken: String) { From 8cbd5d98fc1ebb08ae9d2c3683daf509649439b6 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Sat, 17 Jan 2026 14:24:15 +0800 Subject: [PATCH 05/25] =?UTF-8?q?fix:=20=E5=B0=86=E6=9C=80=E5=A4=A7?= =?UTF-8?q?=E7=A9=BA=E9=97=B2=E7=A7=92=E6=95=B0=E4=BB=8E10=E7=A7=92?= =?UTF-8?q?=E8=B0=83=E6=95=B4=E4=B8=BA5=E7=A7=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/yunqiinnovation/agent_service/AgentService.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index 864cd0f5a..44bbb941a 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -147,7 +147,7 @@ object AgentService : CoroutineScope { // 空闲检测相关 private var idleCheckJob: Job? = null - private val maxIdleSeconds = 10 // 最大空闲秒数（默认值） + private val maxIdleSeconds = 5 // 最大空闲秒数（默认值） // 语音识别模式 private var currentRecognitionMode = "normal" // 当前识别模式：normal, ble_wakeup, phone_call, push_to_talk From 3f5173f28fa8dc4c90a414bc84af770138a046f5 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Sat, 17 Jan 2026 23:42:58 +0800 Subject: [PATCH 06/25] =?UTF-8?q?fix(tts):=20=E4=BF=AE=E5=A4=8D=E8=93=9D?= =?UTF-8?q?=E7=89=99=E9=9F=B3=E9=A2=91=E8=B7=AF=E7=94=B1=E5=92=8C=E6=92=AD?= =?UTF-8?q?=E6=94=BE=E7=8A=B6=E6=80=81=E5=90=8C=E6=AD=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 优化音频会话配置逻辑，避免在通话中错误禁用蓝牙音频 重构TTS播放状态管理，确保播放事件与状态同步 修复流式播放完成事件触发时机，避免过早通知 --- .../agent_service/AgentServiceImpl.swift | 19 +- .../Sources/azure_speech/AzureTtsHelper.swift | 738 ++++++++++++++++-- .../Sources/tools/MicrophoneCapture.swift | 64 +- 3 files changed, 698 insertions(+), 123 deletions(-) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 4755617ff..e7bb7a9af 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -407,19 +407,27 @@ class AgentServiceImpl: NSObject { os_log("通话模式：禁用空闲检测", log: logger, type: .info) return case "ble_wakeup": - azureAsrHelper?.disableBluetoothAudio() + if !isSpeaking { + azureAsrHelper?.disableBluetoothAudio() + } // BLE唤醒模式：使用默认的空闲检测 startIdleCheck(idleSeconds: maxIdleSeconds) case "push_to_talk": - azureAsrHelper?.disableBluetoothAudio() + if !isSpeaking { + azureAsrHelper?.disableBluetoothAudio() + } // 按住说话模式：使用默认的空闲检测 startIdleCheck(idleSeconds: maxIdleSeconds) case "normal": - azureAsrHelper?.disableBluetoothAudio() + if !isSpeaking { + azureAsrHelper?.disableBluetoothAudio() + } // 普通模式：使用默认的空闲检测 startIdleCheck(idleSeconds: maxIdleSeconds) default: - azureAsrHelper?.disableBluetoothAudio() + if !isSpeaking { + azureAsrHelper?.disableBluetoothAudio() + } // 未知模式：使用默认的空闲检测 os_log("未知识别模式: %{public}@，使用默认空闲检测", log: logger, type: .error, mode) startIdleCheck(idleSeconds: maxIdleSeconds) @@ -1970,11 +1978,11 @@ extension AgentServiceImpl: TtsEventListener { sendEvent(name: "tts_started", data: ["status": "started"]) case .synthesisCompleted: - isSpeaking = false restartIdleCheck() sendEvent(name: "tts_completed", data: ["status": "completed"]) case .playbackStarted: + isSpeaking = true restartIdleCheck() sendEvent(name: "playback_started", data: ["status": "playback_started"]) audioPlayer?.stopAwaitSound() @@ -1983,6 +1991,7 @@ extension AgentServiceImpl: TtsEventListener { BleService.shared.closeCodec() } case .playbackCompleted: + isSpeaking = false restartIdleCheck() if(isInterrupt){//播报结束。打开解码器 BleService.shared.openB1Encoder() diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index 9e7bb5abc..ae14dab6d 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -5,7 +5,7 @@ import os.log // 自定义语音处理组件，提供音频流处理等功能 import speech -public class AzureTtsHelper: NSObject, ITtsService { +public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { private let tag = "AzureTtsHelper" private let log = OSLog(subsystem: "com.azure.speech", category: "AzureTtsHelper") @@ -48,10 +48,30 @@ public class AzureTtsHelper: NSObject, ITtsService { // 自定义音频输出流 private var customAudioOutputStream: SPXPushAudioOutputStream? - private var useInternalPlayer = true + private var useInternalPlayer = false private var micCapture: MicrophoneCapture! // 记录最后播放的文本 private var lastSpokenText: String? + // 播放相关：队列与播放器 + private var playbackQueue: [Data] = [] + private var audioPlayer: AVAudioPlayer? + private var isPlaying = false + private var currentSynthesisBuffer = Data() + private var currentTempFileURL: URL? + private var isUsingPushStreamCapture = false + + // 流式播放相关：边合成边播放（16kHz/16bit/mono PCM） + private let audioPlaybackQueue = DispatchQueue(label: "com.azure.tts.playback", qos: .userInitiated) + private var audioEngine: AVAudioEngine? + private var playerNode: AVAudioPlayerNode? + private var pcmFormat: AVAudioFormat? + private var pcmPendingData = Data() + private var hasStrippedWavHeader = false + private var hasNotifiedPlaybackStartedForStream = false + private var activeStreamSynthesisCount = 0 + private var scheduledBufferCount = 0 + private let streamChunkBytes = 1600 + private var suppressStreamPlayback = false /** * 初始化语音合成服务 @@ -65,7 +85,7 @@ public class AzureTtsHelper: NSObject, ITtsService { currentLanguage = language // 设置音频输出格式 - 与Android一致 - speechConfig?.setPropertyTo("riff-16khz-16bit-mono-pcm", + speechConfig?.setPropertyTo("raw-16khz-16bit-mono-pcm", byName: "SpeechServiceConnection_SynthOutputFormat") // 设置低延迟属性 @@ -164,6 +184,7 @@ public class AzureTtsHelper: NSObject, ITtsService { } // 异步处理 + os_log("调用链: speakOnce 入队合成 session=%{public}@ 文本长度=%{public}d", log: log, type: .info, sessionid, cleanedText.count) enqueueSynthesisTask { self.performSynthesis(sessionid: sessionid, text: cleanedText) } @@ -258,15 +279,34 @@ public class AzureTtsHelper: NSObject, ITtsService { /** * 停止语音合成 + * - Parameters: 无 + * - Returns: 是否成功发起停止流程 + * - Throws: 无（内部捕获 SDK/音频会话异常并转为 error 事件） */ public func stop() -> Bool { speaking = false sessionid = "" streamBuffer = "" - lastSpokenText = "" - // 重置计数器 - pendingTextCount = 0 + lastSpokenText = nil + + resetStreamingPlaybackState() + + if let player = audioPlayer { + player.stop() + } + audioPlayer = nil + isPlaying = false + playbackQueue.removeAll() + currentSynthesisBuffer = Data() + + if let url = currentTempFileURL { + try? FileManager.default.removeItem(at: url) + currentTempFileURL = nil + } + pendingTextCount = 0 + currentSessionTextCount = 0 + // 清空所有待处理任务并重置处理状态 taskLock.lock() pendingTasks.removeAll() @@ -274,16 +314,8 @@ public class AzureTtsHelper: NSObject, ITtsService { taskLock.unlock() do { - print("liwei-------------停止语音合成播放") - try self.synthesizer?.stopSpeaking() + try synthesizer?.stopSpeaking() self.notifyEvent(eventType: .synthesisCanceled) - - if !micCapture.isCapturing{ - // 停止时设置音频会话为非活跃状态 - // try AVAudioSession.sharedInstance().setActive(false) - } - - print("音频会话已设置为非活跃状态") } catch { os_log("停止语音合成失败: %{public}@", log: self.log, type: .error, error.localizedDescription) self.notifyEvent(eventType: .error, params: [ @@ -293,13 +325,10 @@ public class AzureTtsHelper: NSObject, ITtsService { } synthesisQueue.async { - do { - // self.synthesizer = nil - try self.recreateSynthesizer() - }catch { - os_log("重置语音合成失败: %{public}@", log: self.log, type: .error, error.localizedDescription) - } + self.recreateSynthesizer() } + + deactivateAudioSessionAfterTTSIfIdle() return true } @@ -446,6 +475,14 @@ public class AzureTtsHelper: NSObject, ITtsService { } speaking = true // lastSpokenText = text + if !applyAudioSessionForTTS() { + speaking = false + notifyEvent(eventType: .error, params: [ + "errorCode": "AUDIO_SESSION_FAILED", + "errorMessage": "TTS音频会话激活失败" + ]) + return + } // 生成SSML let ssml = generateOptimizedSsml(text) @@ -479,17 +516,45 @@ public class AzureTtsHelper: NSObject, ITtsService { do { // 强制释放旧的合成器 synthesizer = nil + isUsingPushStreamCapture = false // 创建音频配置 let audioConfig: SPXAudioConfiguration? if !useInternalPlayer || customAudioOutputStream != nil { - audioConfig = try SPXAudioConfiguration(streamOutput: customAudioOutputStream ?? SPXPushAudioOutputStream()) + let stream: SPXPushAudioOutputStream + if let provided = customAudioOutputStream { + stream = provided + } else { + isUsingPushStreamCapture = true + // 使用写入/关闭回调创建推送输出流，避免无效参数错误 + var totalBytes: Int = 0 + let created = SPXPushAudioOutputStream(writeHandler: { [weak self] data -> UInt in + guard let self = self else { return 0 } + totalBytes += data.count + self.handleSynthesizedAudioChunk(data) + return UInt(data.count) + }, closeHandler: { [weak self] in + guard let self = self else { return } + os_log("调用链: 输出流关闭 累计字节=%{public}d", log: self.log, type: .info, totalBytes) + totalBytes = 0 + }) + // 如果创建失败，抛出以进入 catch + guard let nonNilStream = created else { + throw NSError(domain: "AzureTtsHelper", code: -1, userInfo: [NSLocalizedDescriptionKey: "创建推送输出流失败"]) + } + stream = nonNilStream + } + audioConfig = try SPXAudioConfiguration(streamOutput: stream) } else { - audioConfig = nil // 使用默认扬声器 + audioConfig = nil } // 创建合成器 - synthesizer = try SPXSpeechSynthesizer(speechConfig!) + if let audioConfig = audioConfig { + synthesizer = try SPXSpeechSynthesizer(speechConfiguration: speechConfig!, audioConfiguration: audioConfig) + } else { + synthesizer = try SPXSpeechSynthesizer(speechConfig!) + } setupEventListeners() } catch { os_log("重新创建合成器失败: %{public}@", log: log, type: .error, error.localizedDescription) @@ -505,35 +570,53 @@ public class AzureTtsHelper: NSObject, ITtsService { */ private func setupEventListeners() { synthesizer?.addSynthesisStartedEventHandler { [weak self] _, _ in - self?.notifyEvent(eventType: .synthesisStarted) - self?.notifyEvent(eventType: .playbackStarted) + guard let self = self else { return } + self.currentSynthesisBuffer = Data() + if self.suppressStreamPlayback { return } + if self.isUsingPushStreamCapture { + self.audioPlaybackQueue.async { + self.activeStreamSynthesisCount += 1 + self.prepareStreamingQueueForNewSynthesisIfNeeded() + } + } + os_log("调用链: 合成开始 session=%{public}@", log: self.log, type: .info, self.sessionid) + self.notifyEvent(eventType: .synthesisStarted) } synthesizer?.addSynthesizingEventHandler { [weak self] _, event in if let audioData = event.result.audioData { - self?.notifyAudioData(audioData) + guard let self = self else { return } + os_log("调用链: 合成中 接收音频片段=%{public}dB session=%{public}@", log: self.log, type: .debug, audioData.count, self.sessionid) + if !self.isUsingPushStreamCapture { + self.currentSynthesisBuffer.append(audioData) + self.notifyAudioData(audioData) + } } } synthesizer?.addSynthesisCompletedEventHandler { [weak self] _, _ in guard let self = self else { return } self.speaking = false - + if self.suppressStreamPlayback { return } + // 将当前合成的音频加入播放队列 + if !self.currentSynthesisBuffer.isEmpty { + os_log("调用链: 合成完成 入队播放 数据长度=%{public}dB session=%{public}@", log: self.log, type: .info, self.currentSynthesisBuffer.count, self.sessionid) + self.enqueuePlaybackItem(self.currentSynthesisBuffer) + self.currentSynthesisBuffer = Data() + } + if self.isUsingPushStreamCapture { + self.audioPlaybackQueue.async { + self.activeStreamSynthesisCount = max(0, self.activeStreamSynthesisCount - 1) + self.scheduleAvailablePcmBuffers() + self.checkStreamPlaybackCompletedIfNeeded() + } + } // 减少待处理文本计数 self.pendingTextCount = max(0, self.pendingTextCount - 1) - + // 通知合成完成 self.notifyEvent(eventType: .synthesisCompleted) - self.notifyEvent(eventType: .playbackCompleted) - - // 检查是否为最后一个文本播放完毕 - if self.pendingTextCount == 0 && self.pendingTasks.isEmpty && !micCapture.isCapturing{ - do { - // try AVAudioSession.sharedInstance().setActive(false) - print("最后一个文本播放完毕，音频会话已设置为非活跃状态") - } catch { - print("设置音频会话为非活跃状态失败: \(error.localizedDescription)") - } - } + // 尝试开始播放 + self.startPlaybackIfNeeded() } synthesizer?.addSynthesisCanceledEventHandler { [weak self] _, event in @@ -542,6 +625,14 @@ public class AzureTtsHelper: NSObject, ITtsService { // 减少待处理文本计数 self.pendingTextCount = max(0, self.pendingTextCount - 1) + if self.suppressStreamPlayback { return } + + if self.isUsingPushStreamCapture { + self.audioPlaybackQueue.async { + self.activeStreamSynthesisCount = max(0, self.activeStreamSynthesisCount - 1) + self.checkStreamPlaybackCompletedIfNeeded() + } + } var params: [String: Any] = [:] if let details = try? SPXSpeechSynthesisCancellationDetails(fromCanceledSynthesisResult: event.result) { @@ -552,6 +643,390 @@ public class AzureTtsHelper: NSObject, ITtsService { self.notifyEvent(eventType: .synthesisCanceled, params: params) } } + + /** + * 入队播放数据 + * - Parameters: + * - data: 合成的音频数据（RIFF PCM） + * - Returns: 无 + * - Throws: 无 + */ + private func enqueuePlaybackItem(_ data: Data) { + playbackQueue.append(data) + } + + /** + * 处理合成输出的音频片段（用于流式播放） + * - Parameters: + * - data: SDK 推送输出流回调提供的音频片段数据 + * - Returns: 无 + * - Throws: 无 + */ + private func handleSynthesizedAudioChunk(_ data: Data) { + if suppressStreamPlayback { return } + if !speaking { return } + notifyAudioData(data) + guard isUsingPushStreamCapture else { return } + audioPlaybackQueue.async { + if !self.speaking { return } + let pcm = self.stripWavHeaderIfNeeded(data) + if !pcm.isEmpty { + self.pcmPendingData.append(pcm) + self.scheduleAvailablePcmBuffers() + } + } + } + + /** + * 新一轮合成开始时准备流式播放队列（不打断正在播放的内容） + * - Returns: 无 + * - Throws: 无 + */ + private func prepareStreamingQueueForNewSynthesisIfNeeded() { + if pcmPendingData.isEmpty, scheduledBufferCount == 0, playerNode?.isPlaying != true { + hasStrippedWavHeader = false + hasNotifiedPlaybackStartedForStream = false + } + } + + /** + * 停止并清理流式播放资源（用于 stop/dispose） + * - Returns: 无 + * - Throws: 无 + */ + private func resetStreamingPlaybackState() { + audioPlaybackQueue.sync { + self.pcmPendingData = Data() + self.hasStrippedWavHeader = false + self.hasNotifiedPlaybackStartedForStream = false + self.activeStreamSynthesisCount = 0 + self.scheduledBufferCount = 0 + self.playerNode?.stop() + self.audioEngine?.stop() + self.audioEngine = nil + self.playerNode = nil + self.pcmFormat = nil + } + } + + /** + * 将待播 PCM 数据按固定块切分并调度到 AVAudioPlayerNode + * - Returns: 无 + * - Throws: 无 + */ + private func scheduleAvailablePcmBuffers() { + guard ensureStreamingEngineIfNeeded() else { return } + while pcmPendingData.count >= streamChunkBytes { + let chunk = pcmPendingData.prefix(streamChunkBytes) + pcmPendingData.removeFirst(streamChunkBytes) + guard let buffer = makePcmBuffer(from: Data(chunk)) else { continue } + scheduledBufferCount += 1 + if !hasNotifiedPlaybackStartedForStream { + hasNotifiedPlaybackStartedForStream = true + os_log("调用链: 流式播放开始 session=%{public}@", log: log, type: .info, sessionid) + notifyEvent(eventType: .playbackStarted) + } + playerNode?.scheduleBuffer(buffer, completionHandler: { [weak self] in + guard let self = self else { return } + self.audioPlaybackQueue.async { + self.scheduledBufferCount = max(0, self.scheduledBufferCount - 1) + self.checkStreamPlaybackCompletedIfNeeded() + } + }) + } + if activeStreamSynthesisCount == 0, !pcmPendingData.isEmpty { + if pcmPendingData.count % 2 != 0 { + pcmPendingData.removeLast() + } + if !pcmPendingData.isEmpty, let buffer = makePcmBuffer(from: pcmPendingData) { + pcmPendingData.removeAll(keepingCapacity: true) + scheduledBufferCount += 1 + if !hasNotifiedPlaybackStartedForStream { + hasNotifiedPlaybackStartedForStream = true + os_log("调用链: 流式播放开始 session=%{public}@", log: log, type: .info, sessionid) + notifyEvent(eventType: .playbackStarted) + } + playerNode?.scheduleBuffer(buffer, completionHandler: { [weak self] in + guard let self = self else { return } + self.audioPlaybackQueue.async { + self.scheduledBufferCount = max(0, self.scheduledBufferCount - 1) + self.checkStreamPlaybackCompletedIfNeeded() + } + }) + } + } + if playerNode?.isPlaying == false { + playerNode?.play() + } + } + + /** + * 当合成结束且播放缓冲耗尽时触发播放完成事件 + * - Returns: 无 + * - Throws: 无 + */ +private func checkStreamPlaybackCompletedIfNeeded() { + if activeStreamSynthesisCount == 0, pcmPendingData.isEmpty, scheduledBufferCount == 0 { + hasNotifiedPlaybackStartedForStream = false + os_log("调用链: 流式播放完成 session=%{public}@", log: log, type: .info, sessionid) + notifyEvent(eventType: .playbackCompleted) + deactivateAudioSessionAfterTTSIfIdle() + } + } + + /** + * 确保 AVAudioEngine/AVAudioPlayerNode 已准备好用于 16kHz/16bit/mono PCM 播放 + * - Returns: 初始化是否成功 + * - Throws: 无 + */ + private func ensureStreamingEngineIfNeeded() -> Bool { + if !applyAudioSessionForTTS() { + os_log("流式播放初始化: 激活音频会话失败", log: log, type: .error) + return false + } + + if let engine = audioEngine, let node = playerNode, let format = pcmFormat { + if !engine.isRunning { + do { + try engine.start() + } catch { + audioEngine = nil + playerNode = nil + pcmFormat = nil + } + } + if audioEngine != nil, playerNode != nil, pcmFormat != nil { + if node.isPlaying == false, engine.isRunning { + node.play() + } + _ = format + return true + } + } + + let engine = AVAudioEngine() + let node = AVAudioPlayerNode() + guard let format = AVAudioFormat(standardFormatWithSampleRate: 16000, channels: 1) else { + os_log("流式播放初始化失败: 创建音频格式失败", log: log, type: .error) + return false + } + engine.attach(node) + engine.connect(node, to: engine.mainMixerNode, format: format) + do { + try engine.start() + } catch { + os_log("流式播放初始化失败: %{public}@", log: log, type: .error, error.localizedDescription) + return false + } + audioEngine = engine + playerNode = node + pcmFormat = format + return true + } + + /** + * 将 PCM 数据（16kHz/16bit/mono）转换为可调度的 AVAudioPCMBuffer + * - Parameters: + * - data: PCM 裸数据（小端 Int16） + * - Returns: 可用于 scheduleBuffer 的 AVAudioPCMBuffer，失败返回 nil + * - Throws: 无 + */ + private func makePcmBuffer(from data: Data) -> AVAudioPCMBuffer? { + guard let format = pcmFormat else { return nil } + let frameCount = data.count / 2 + guard frameCount > 0 else { return nil } + guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frameCount)) else { return nil } + buffer.frameLength = AVAudioFrameCount(frameCount) + guard let dst = buffer.floatChannelData else { return nil } + + data.withUnsafeBytes { raw in + guard let src = raw.bindMemory(to: Int16.self).baseAddress else { return } + for i in 0.. Data { + if hasStrippedWavHeader { return data } + if data.count >= 12, + String(data: data.subdata(in: 0..<4), encoding: .ascii) == "RIFF", + String(data: data.subdata(in: 8..<12), encoding: .ascii) == "WAVE" { + let marker = Data("data".utf8) + if let range = data.range(of: marker, options: [], in: 0.. 44 { + hasStrippedWavHeader = true + return data.subdata(in: 44.. Data { + if isWavData(data) { + return data + } + os_log("调用链: 音频无WAV头，补WAV头后播放 bytes=%{public}d", log: log, type: .info, data.count) + let header = makeWavHeader(pcmDataSize: UInt32(data.count), sampleRate: 16000, channels: 1, bitsPerSample: 16) + var out = Data() + out.append(header) + out.append(data) + return out + } + + /** + * 判断数据是否为 WAV（RIFF/WAVE） + * - Parameters: + * - data: 音频数据 + * - Returns: 是否为 WAV + * - Throws: 无 + */ + private func isWavData(_ data: Data) -> Bool { + guard data.count >= 12 else { return false } + let riff = data.subdata(in: 0..<4) + let wave = data.subdata(in: 8..<12) + return String(data: riff, encoding: .ascii) == "RIFF" && String(data: wave, encoding: .ascii) == "WAVE" + } + + /** + * 生成 WAV 头（PCM） + * - Parameters: + * - pcmDataSize: PCM 数据长度（字节） + * - sampleRate: 采样率 + * - channels: 声道数 + * - bitsPerSample: 位深 + * - Returns: WAV 文件头数据 + * - Throws: 无 + */ + private func makeWavHeader(pcmDataSize: UInt32, sampleRate: UInt32, channels: UInt16, bitsPerSample: UInt16) -> Data { + let byteRate = sampleRate * UInt32(channels) * UInt32(bitsPerSample) / 8 + let blockAlign = channels * bitsPerSample / 8 + let chunkSize = 36 + pcmDataSize + + var data = Data() + data.append(contentsOf: Array("RIFF".utf8)) + data.append(contentsOf: withUnsafeBytes(of: chunkSize.littleEndian, Array.init)) + data.append(contentsOf: Array("WAVE".utf8)) + data.append(contentsOf: Array("fmt ".utf8)) + var subchunk1Size: UInt32 = 16 + data.append(contentsOf: withUnsafeBytes(of: subchunk1Size.littleEndian, Array.init)) + var audioFormat: UInt16 = 1 + data.append(contentsOf: withUnsafeBytes(of: audioFormat.littleEndian, Array.init)) + data.append(contentsOf: withUnsafeBytes(of: channels.littleEndian, Array.init)) + data.append(contentsOf: withUnsafeBytes(of: sampleRate.littleEndian, Array.init)) + data.append(contentsOf: withUnsafeBytes(of: byteRate.littleEndian, Array.init)) + data.append(contentsOf: withUnsafeBytes(of: blockAlign.littleEndian, Array.init)) + data.append(contentsOf: withUnsafeBytes(of: bitsPerSample.littleEndian, Array.init)) + data.append(contentsOf: Array("data".utf8)) + data.append(contentsOf: withUnsafeBytes(of: pcmDataSize.littleEndian, Array.init)) + return data + } + + /** + * 创建临时WAV文件并返回URL + * - Parameters: + * - data: WAV数据（RIFF 16kHz/16bit/mono） + * - Returns: 成功返回文件URL，失败返回nil + * - Throws: 无 + */ + private func createTempWavFile(_ data: Data) -> URL? { + let tmpDir = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true) + let fileURL = tmpDir.appendingPathComponent("azure_tts_\(UUID().uuidString).wav") + do { + try data.write(to: fileURL, options: .atomic) + os_log("调用链: 写入临时文件 path=%{public}@", log: log, type: .debug, fileURL.path) + return fileURL + } catch { + os_log("写入临时文件失败: %{public}@", log: log, type: .error, error.localizedDescription) + return nil + } + } + + /** + * 播放完成回调 + * - Parameters: + * - player: AVAudioPlayer 实例 + * - successfully: 是否成功播放完成 + * - Returns: 无 + * - Throws: 无 + */ +public func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully flag: Bool) { + // 移除队首 + if !playbackQueue.isEmpty { + playbackQueue.removeFirst() + } + // 清理临时文件 + if let url = currentTempFileURL { + try? FileManager.default.removeItem(at: url) + currentTempFileURL = nil + } + isPlaying = false + os_log("调用链: 播放完成 成功=%{public}@ session=%{public}@", log: log, type: .info, flag.description, sessionid) + notifyEvent(eventType: .playbackCompleted) + // 如果还有剩余，继续播放 + startPlaybackIfNeeded() + deactivateAudioSessionAfterTTSIfIdle() + } /** * 通知事件 @@ -587,6 +1062,8 @@ public class AzureTtsHelper: NSObject, ITtsService { """ synthesisQueue.async { + self.suppressStreamPlayback = true + defer { self.suppressStreamPlayback = false } _ = try? self.synthesizer?.startSpeakingSsml(warmupSsml) } } @@ -688,12 +1165,97 @@ public func resetTextCount() { * 音频会话并发防护和状态 */ private let audioSessionQueue = DispatchQueue(label: "com.azure.tts.audioSession") +private let audioSessionQueueKey = DispatchSpecificKey() +private var audioSessionQueueKeySet = false private var audioInterrupted = false private var audioSessionObserversAdded = false // 新增：路由变化的去抖与配置重入保护标记 private var routeChangeDebounceWorkItem: DispatchWorkItem? private var isApplyingAudioSessionConfig = false +/** + * 确保音频会话队列已设置 specific key（用于避免同队列 sync 死锁） + * - Parameters: 无 + * - Returns: 无 + * - Throws: 无 + */ +private func ensureAudioSessionQueueSpecificKeySet() { + if audioSessionQueueKeySet { return } + audioSessionQueueKeySet = true + audioSessionQueue.setSpecific(key: audioSessionQueueKey, value: 1) +} + +/** + * 同步应用 TTS 音频会话配置（用于需要立即生效的播放/引擎初始化路径） + * - Parameters: 无 + * - Returns: 是否成功激活音频会话 + * - Throws: 无（内部捕获 AVAudioSession 相关异常并记录日志） + */ +private func applyAudioSessionForTTS() -> Bool { + var success = true + ensureAudioSessionQueueSpecificKeySet() + + let applyBody = { + if self.isApplyingAudioSessionConfig { return } + self.isApplyingAudioSessionConfig = true + defer { self.isApplyingAudioSessionConfig = false } + + let audioSession = AVAudioSession.sharedInstance() + let outputs: [AVAudioSessionPortDescription] = audioSession.currentRoute.outputs + let hasBluetoothHFP = outputs.contains(where: { $0.portType == .bluetoothHFP }) + let otherAudioPlaying = audioSession.isOtherAudioPlaying + + let desiredCategory: AVAudioSession.Category + let desiredMode: AVAudioSession.Mode + var desiredOptions: AVAudioSession.CategoryOptions = [] + + if self.isTelephonyActive() || hasBluetoothHFP { + desiredCategory = .playback + desiredMode = .default + desiredOptions.insert(.duckOthers) + } else { + desiredCategory = .playback + desiredMode = .spokenAudio + if otherAudioPlaying { + desiredOptions.insert(.duckOthers) + desiredOptions.insert(.interruptSpokenAudioAndMixWithOthers) + } + } + + do { + for attempt in 0..<3 { + do { + if audioSession.category != desiredCategory || audioSession.mode != desiredMode || audioSession.categoryOptions != desiredOptions { + try audioSession.setCategory(desiredCategory, mode: desiredMode, options: desiredOptions) + } + try audioSession.setActive(true) + break + } catch { + if attempt == 2 { + throw error + } + Thread.sleep(forTimeInterval: 0.05) + } + } + } catch { + do { + try audioSession.setCategory(.playback, mode: .default, options: []) + try audioSession.setActive(true) + } catch { + success = false + os_log("TTS 音频会话配置失败: %{public}@", log: self.log, type: .error, error.localizedDescription) + } + } + } + + if DispatchQueue.getSpecific(key: audioSessionQueueKey) != nil { + applyBody() + } else { + audioSessionQueue.sync(execute: applyBody) + } + return success +} + /** * 监听并处理音频中断与路由变化（通话/Siri/蓝牙切换等） * - 目的：在通话期间或路由变化时安全地停止/重配，避免 AVAudioSession 断言导致崩溃 @@ -729,25 +1291,26 @@ private func isTelephonyActive() -> Bool { } /** - * 安全配置音频会话用于 TTS，兼容通话场景 - * - 通话中：使用 .playback + .duckOthers，避免 .voiceChat/.allowBluetoothA2DP 导致断言 - * - 非通话：使用 .playAndRecord + .voiceChat + .allowBluetooth（不使用 A2DP） + * 安全配置音频会话用于 TTS + * - 播放优先走媒体声道（A2DP），避免切换到通话声道（HFP） + * - 外部音乐播放时：不启用 mix，默认会打断外部音频；同时可 duck 确保可听见 + * - 录音进行中：保持 playAndRecord + A2DP 输出 + 内置麦输入，避免切到 HFP + * - 通话中：保持保守策略，避免触发系统断言或破坏通话链路 + * - Parameters: 无 + * - Returns: 无 + * - Throws: 无（内部捕获 AVAudioSession 相关异常并记录日志） */ /// 为 TTS 安全配置音频会话： /// - 使用异步队列，避免在同一串行队列上 sync 导致死锁 - /// - 根据当前音频路由（蓝牙 A2DP / 蓝牙 HFP / 有线耳机）选择更合适的类别与模式 + /// - 优先使用 .playback + .spokenAudio + .allowBluetoothA2DP（媒体声道），避免进入 HFP + /// - 外部音频播放时启用 spoken audio 策略，确保能在 A2DP 上播放 /// - 仅当类别或模式发生变化时才 setCategory，降低 AVAudioSessionRouteChangeReason.categoryChange 的触发概率 - private func safeConfigureAudioSessionForTTS() { +private func safeConfigureAudioSessionForTTS() { // 使用异步，避免串行队列重入自锁 audioSessionQueue.async { [weak self] in guard let self = self else { return } - // 防重入：配置中不重复进入 - if self.isApplyingAudioSessionConfig { return } - self.isApplyingAudioSessionConfig = true - defer { self.isApplyingAudioSessionConfig = false } - let audioSession = AVAudioSession.sharedInstance() // 显式指定 outputs 的类型，确保闭包参数类型可推断 @@ -757,44 +1320,49 @@ private func isTelephonyActive() -> Bool { // 保持使用 audioSessionQueue.async，防止在串行队列上自锁。 // 仅在类别或模式发生变化时调用 setCategory，减少 categoryChange 的回调风暴。 let hasBluetoothHFP = outputs.contains(where: { $0.portType == .bluetoothHFP }) - let hasBluetoothA2DP = outputs.contains(where: { $0.portType == .bluetoothA2DP }) - let hasWiredHeadphones = outputs.contains(where: { $0.portType == .headphones || $0.portType == .headsetMic }) - - // 计算目标配置 - let desiredCategory: AVAudioSession.Category - let desiredMode: AVAudioSession.Mode - var desiredOptions: AVAudioSession.CategoryOptions = [.mixWithOthers] - - if self.isTelephonyActive() || hasBluetoothHFP { - // 通话活跃或存在 HFP（语音）链路：使用语音聊天模式 - desiredCategory = .playAndRecord - desiredMode = .voiceChat - desiredOptions.insert(.allowBluetooth) // HFP 需 allowBluetooth - } else if hasBluetoothA2DP || hasWiredHeadphones { - // 纯播放链路耳机（A2DP/有线耳机）：避免强制切语音链路 - desiredCategory = .playback - desiredMode = .default - desiredOptions.insert(.allowBluetoothA2DP) - } else { - // 默认：本机扬声器的语音链路 - desiredCategory = .playAndRecord - desiredMode = .voiceChat - desiredOptions.insert(.defaultToSpeaker) - desiredOptions.insert(.allowBluetooth) // 允许外接麦/耳机（非 HFP 也不冲突） - } + _ = hasBluetoothHFP + _ = audioSession - do { - // 仅当类别或模式变化时设置，减少无谓的 categoryChange - if audioSession.category != desiredCategory || audioSession.mode != desiredMode { - try audioSession.setCategory(desiredCategory, mode: desiredMode, options: desiredOptions) - } - // 激活会话（通常不会引发死锁），失败时仅记录错误 - try audioSession.setActive(true) - } catch { - os_log("TTS 音频会话配置失败: %{public}@", log: self.log, type: .error, error.localizedDescription) - } + _ = self.applyAudioSessionForTTS() } } + +/** + * 在 TTS 完全空闲后释放音频会话（用于恢复外部音乐） + * - Parameters: 无 + * - Returns: 无 + * - Throws: 无（内部捕获 AVAudioSession 异常并忽略） + */ +private func deactivateAudioSessionAfterTTSIfIdle() { + ensureAudioSessionQueueSpecificKeySet() + let work = { [weak self] in + guard let self = self else { return } + if self.speaking { return } + if self.isPlaying { return } + if !self.playbackQueue.isEmpty { return } + if self.pendingTextCount > 0 { return } + if !self.pendingTasks.isEmpty { return } + if self.activeStreamSynthesisCount > 0 { return } + if !self.pcmPendingData.isEmpty { return } + if self.scheduledBufferCount > 0 { return } + self.audioPlaybackQueue.async { [weak self] in + guard let self = self else { return } + if self.playerNode?.isPlaying == true { return } + self.playerNode?.stop() + self.audioEngine?.stop() + self.audioEngine = nil + self.playerNode = nil + self.pcmFormat = nil + } + let session = AVAudioSession.sharedInstance() + _ = try? session.setActive(false, options: .notifyOthersOnDeactivation) + } + if DispatchQueue.getSpecific(key: audioSessionQueueKey) != nil { + work() + } else { + audioSessionQueue.async(execute: work) + } +} /** diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift index 2bd895836..df55d1727 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift @@ -94,11 +94,6 @@ public class MicrophoneCapture: NSObject { do { try audioSession.setPreferredSampleRate(sampleRate) try audioSession.setPreferredIOBufferDuration(0.005) // 5ms缓冲 - - // 优先设置蓝牙音频路由 - configureAudioRoute() - - try audioSession.setActive(true) print("音频会话配置成功") } catch { print("音频会话配置失败: \(error.localizedDescription)") @@ -153,11 +148,10 @@ public class MicrophoneCapture: NSObject { // 取消扬声器强制输出，让音频通过蓝牙耳机输出 do { // 设置音频会话参数 - try audioSession.setCategory(.playAndRecord, + try audioSession.setCategory(.playback, mode: .videoChat, - options: [.allowBluetoothA2DP, // 允许蓝牙耳机，不占用hfp链路 - .mixWithOthers, - .allowBluetooth]) // 添加音频优先级控制 + options: [.allowBluetoothA2DP, // 仅允许A2DP，不启用HFP + .mixWithOthers]) try audioSession.overrideOutputAudioPort(.none) print("蓝牙模式：音频输出设置为蓝牙耳机") @@ -214,6 +208,18 @@ public class MicrophoneCapture: NSObject { // 检查麦克风权限 switch audioSession.recordPermission { case .granted: + do { + try audioSession.setCategory(.record, + mode: .default, + options: []) + if let inputs = audioSession.availableInputs, + let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { + try? audioSession.setPreferredInput(builtInMic) + } + try audioSession.setActive(true) + } catch { + throw error + } try setupAudioEngine() case .denied: throw NSError(domain: "麦克风权限被拒绝", code: 0) @@ -221,6 +227,19 @@ public class MicrophoneCapture: NSObject { audioSession.requestRecordPermission { granted in if granted { do { + do { + try audioSession.setCategory(.record, + mode: .default, + options: []) + if let inputs = audioSession.availableInputs, + let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { + try? audioSession.setPreferredInput(builtInMic) + } + try audioSession.setActive(true) + } catch { + print("音频会话配置失败: \(error.localizedDescription)") + return + } try self.setupAudioEngine() } catch { print("音频引擎设置失败: \(error.localizedDescription)") @@ -444,29 +463,8 @@ public class MicrophoneCapture: NSObject { // 安全地处理音频会话 guard let audioSession = audioSession else { return } - do { - try audioSession.setActive(false) - - if self.hasBluetoothDevices { - print("耳机") - // 设置音频会话参数 - try audioSession.setCategory(.playback, - mode: .videoChat, - options: [ - .allowBluetoothA2DP, - .allowBluetooth - ]) // 添加音频优先级控制 - } else { - print("手机") - try audioSession.setCategory(.playback, - mode: .videoChat, - options: [ - .mixWithOthers, - .defaultToSpeaker]) - } - - try audioSession.overrideOutputAudioPort(.none) - try audioSession.setActive(true) + do { + try audioSession.setActive(false, options: .notifyOthersOnDeactivation) } catch { print("音频会话配置失败: \(error.localizedDescription)") } @@ -487,4 +485,4 @@ public class MicrophoneCapture: NSObject { } -} \ No newline at end of file +} From 23f7dbc9f3f0ec8811cca686406553864921cd4c Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Sun, 18 Jan 2026 00:01:25 +0800 Subject: [PATCH 07/25] =?UTF-8?q?fix(azure=5Fspeech):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E6=B5=81=E5=BC=8F=E6=92=AD=E6=94=BE=E6=97=B6speaking=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E6=9C=AA=E6=AD=A3=E7=A1=AE=E6=9B=B4=E6=96=B0=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 确保在流式播放模式下speaking状态在适当的时候更新，避免状态不一致。同时优化了sessionid为空时的处理逻辑。 --- .../Sources/azure_speech/AzureTtsHelper.swift | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index ae14dab6d..664d5ea9b 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -596,8 +596,10 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { synthesizer?.addSynthesisCompletedEventHandler { [weak self] _, _ in guard let self = self else { return } - self.speaking = false if self.suppressStreamPlayback { return } + if !self.isUsingPushStreamCapture { + self.speaking = false + } // 将当前合成的音频加入播放队列 if !self.currentSynthesisBuffer.isEmpty { os_log("调用链: 合成完成 入队播放 数据长度=%{public}dB session=%{public}@", log: self.log, type: .info, self.currentSynthesisBuffer.count, self.sessionid) @@ -621,11 +623,13 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { synthesizer?.addSynthesisCanceledEventHandler { [weak self] _, event in guard let self = self else { return } - self.speaking = false // 减少待处理文本计数 self.pendingTextCount = max(0, self.pendingTextCount - 1) if self.suppressStreamPlayback { return } + if !self.isUsingPushStreamCapture { + self.speaking = false + } if self.isUsingPushStreamCapture { self.audioPlaybackQueue.async { @@ -664,11 +668,11 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { */ private func handleSynthesizedAudioChunk(_ data: Data) { if suppressStreamPlayback { return } - if !speaking { return } + if sessionid.isEmpty { return } notifyAudioData(data) guard isUsingPushStreamCapture else { return } audioPlaybackQueue.async { - if !self.speaking { return } + if self.sessionid.isEmpty { return } let pcm = self.stripWavHeaderIfNeeded(data) if !pcm.isEmpty { self.pcmPendingData.append(pcm) @@ -767,6 +771,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { */ private func checkStreamPlaybackCompletedIfNeeded() { if activeStreamSynthesisCount == 0, pcmPendingData.isEmpty, scheduledBufferCount == 0 { + speaking = false hasNotifiedPlaybackStartedForStream = false os_log("调用链: 流式播放完成 session=%{public}@", log: log, type: .info, sessionid) notifyEvent(eventType: .playbackCompleted) From 4d3e7b9656bf6879dbd732ae97d975224cbe96f8 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Mon, 19 Jan 2026 11:15:00 +0800 Subject: [PATCH 08/25] =?UTF-8?q?feat(tts):=20=E6=94=AF=E6=8C=81=E5=A4=96?= =?UTF-8?q?=E9=83=A8=E9=9F=B3=E9=A2=91=E6=BA=90=E9=85=8D=E7=BD=AE=E5=B9=B6?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=9F=B3=E9=A2=91=E4=BC=9A=E8=AF=9D=E7=AE=A1?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加对蓝牙等外部音频源的支持，扩展TTS初始化接口以接收外部音频源标志 优化音频会话配置逻辑，根据输入源类型和当前音频状态动态调整参数 增加音频流处理稳定性，改进WAV头探测和缓冲区管理 --- .../agent_service/AgentServiceImpl.swift | 11 +- .../Sources/azure_speech/AzureTtsHelper.swift | 265 ++++++++++++++---- 2 files changed, 225 insertions(+), 51 deletions(-) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index e7bb7a9af..08f7f4b7f 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -342,12 +342,12 @@ class AgentServiceImpl: NSObject { sendError("初始化语音识别服务失败", code: "ASR_INIT_ERROR") return false } - guard let ttsSuccess = azureTtsHelper?.initialize( ttsAppId: "", ttsAppToken: azureSpeechKey, ttsResource: azureSpeechRegion, - language: primaryLanguage + language: primaryLanguage, + isExtaudioSource: BleService.shared.isConnected() ), ttsSuccess else { sendError("初始化语音合成服务失败", code: "TTS_INIT_ERROR") return false @@ -619,7 +619,7 @@ class AgentServiceImpl: NSObject { return true } - func speakText(sessionid sessionid:String,_ text: String) -> Bool { + func speakText(sessionid: String, _ text: String) -> Bool { if !isInitialized { sendError("服务未初始化", code: "NOT_INITIALIZED") return false @@ -630,6 +630,7 @@ class AgentServiceImpl: NSObject { } _ = azureTtsHelper?.setSpeechParams(rate: ttsRatePercent, pitch: 0, volume: 100) + _ = azureTtsHelper?.setAudioSourceType(isExtaudioSource: BleService.shared.isConnected()) return azureTtsHelper?.speakOnce(sessionid:sessionid,text) ?? false } @@ -1788,6 +1789,7 @@ class ChatApiStreamCallback: StreamCallback { responseBuilder += token if speakResponse && reply && broadcast{ + _ = agentService.azureTtsHelper?.setAudioSourceType(isExtaudioSource: BleService.shared.isConnected()) try agentService.azureTtsHelper?.speakStream(sessionid:sessionid,token) // 在开始流式TTS时立即停止气泡音 } @@ -1813,6 +1815,7 @@ class ChatApiStreamCallback: StreamCallback { } if speakResponse && reply && broadcast && sessionid == agentService.currsessionId{ + _ = agentService.azureTtsHelper?.setAudioSourceType(isExtaudioSource: BleService.shared.isConnected()) agentService.azureTtsHelper?.flushStream(sessionid:sessionid) } @@ -1864,6 +1867,7 @@ class ChatApiStreamCallback: StreamCallback { "message": message ]) if(code == 2001){ + _ = agentService.azureTtsHelper?.setAudioSourceType(isExtaudioSource: BleService.shared.isConnected()) agentService.azureTtsHelper?.speakStream(sessionid: sessionid,agentService.insufficientIntegralText) } agentService.isAiStreaming = false @@ -1907,6 +1911,7 @@ class ChatApiStreamCallback: StreamCallback { if (iscallingTool) { os_log("收到函数调用: callingToolText:%{public}@", log: agentService.logger, type: .info, agentService.callingToolText) + _ = agentService.azureTtsHelper?.setAudioSourceType(isExtaudioSource: BleService.shared.isConnected()) agentService.azureTtsHelper?.speakStream(sessionid: sessionid,agentService.callingToolText) iscallingTool = false } diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index 664d5ea9b..b59031000 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -48,6 +48,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { // 自定义音频输出流 private var customAudioOutputStream: SPXPushAudioOutputStream? + private var internalAudioOutputStream: SPXPushAudioOutputStream? private var useInternalPlayer = false private var micCapture: MicrophoneCapture! // 记录最后播放的文本 @@ -67,16 +68,51 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { private var pcmFormat: AVAudioFormat? private var pcmPendingData = Data() private var hasStrippedWavHeader = false + private var wavHeaderProbeBuffer = Data() private var hasNotifiedPlaybackStartedForStream = false private var activeStreamSynthesisCount = 0 private var scheduledBufferCount = 0 - private let streamChunkBytes = 1600 + private let streamChunkBytes = 3200 private var suppressStreamPlayback = false + private var pushStreamChunkCount = 0 + private var pushStreamCallbackCount = 0 + private var synthEventAudioCount = 0 + private var isExtaudioSource = false + + /** * 初始化语音合成服务 + * - Parameters: + * - ttsAppId: 服务应用ID（Azure 场景未使用，可为空） + * - ttsAppToken: Azure 语音服务订阅密钥 + * - ttsResource: Azure 语音服务区域 + * - language: 语言代码，如 "zh-CN" + * - Returns: 是否初始化成功 + * - Throws: 无（内部捕获 SDK 异常并记录日志） */ public func initialize(ttsAppId: String, ttsAppToken: String, ttsResource: String, language: String) -> Bool { + return initialize( + ttsAppId: ttsAppId, + ttsAppToken: ttsAppToken, + ttsResource: ttsResource, + language: language, + isExtaudioSource: false + ) + } + + /** + * 初始化语音合成服务（支持外部音频源场景） + * - Parameters: + * - ttsAppId: 服务应用ID（Azure 场景未使用，可为空） + * - ttsAppToken: Azure 语音服务订阅密钥 + * - ttsResource: Azure 语音服务区域 + * - language: 语言代码，如 "zh-CN" + * - isExtaudioSource: 是否为外部音频源（例如 BLE 输入） + * - Returns: 是否初始化成功 + * - Throws: 无（内部捕获 SDK 异常并记录日志） + */ + public func initialize(ttsAppId: String, ttsAppToken: String, ttsResource: String, language: String, isExtaudioSource: Bool) -> Bool { do { micCapture = MicrophoneCapture.shared // 创建语音配置 @@ -94,7 +130,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { // 创建合成器 recreateSynthesizer() - + self.isExtaudioSource = isExtaudioSource // 设置初始化完成 isInitialized = true @@ -147,9 +183,10 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { * @param sessionid 会话ID * @return 是否成功启动 */ - public func startspeak(sessionid sessionid: String) -> Bool { + public func startspeak(sessionid: String) -> Bool { self.sessionid = sessionid print("startspeak sessionid \(sessionid)") + // os_log("调用链: startspeak 设置session=%{public}@", log: log, type: .info, sessionid) // 重置当前会话的文本计数 self.currentSessionTextCount = 0 self.pendingTextCount = 0 @@ -163,7 +200,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { /** * 单次合成并播放 */ - public func speakOnce(sessionid sessionid: String, _ text: String) -> Bool { + public func speakOnce(sessionid: String, _ text: String) -> Bool { if !isInitialized { os_log("语音合成未初始化", log: log, type: .error) notifyEvent(eventType: .error, params: [ @@ -173,20 +210,23 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { return false } - if (self.sessionid != sessionid){ - return false + let effectiveSessionId = sessionid.isEmpty ? self.sessionid : sessionid + if !sessionid.isEmpty { + self.sessionid = sessionid } + // os_log("调用链: speakOnce 接收 session=%{public}@ effective=%{public}@ speaking=%{public}@ pendingTasks=%{public}d", log: log, type: .info, sessionid, effectiveSessionId, speaking.description, pendingTasks.count) // 清理文本 let cleanedText = cleanTextForTTS(text) if cleanedText.isEmpty { + // os_log("调用链: speakOnce 文本清理后为空 session=%{public}@", log: log, type: .info, effectiveSessionId) return false } // 异步处理 - os_log("调用链: speakOnce 入队合成 session=%{public}@ 文本长度=%{public}d", log: log, type: .info, sessionid, cleanedText.count) + // os_log("调用链: speakOnce 入队合成 session=%{public}@ 文本长度=%{public}d", log: log, type: .info, effectiveSessionId, cleanedText.count) enqueueSynthesisTask { - self.performSynthesis(sessionid: sessionid, text: cleanedText) + self.performSynthesis(sessionid: effectiveSessionId, text: cleanedText) } return true @@ -195,7 +235,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { /** * 流式合成文本 */ - public func speakStream(sessionid sessionid: String, _ text: String) -> Bool { + public func speakStream(sessionid: String, _ text: String) -> Bool { if !isInitialized { notifyEvent(eventType: .error, params: [ "errorCode": "NOT_INITIALIZED", @@ -203,12 +243,14 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { ]) return false } - if (self.sessionid != sessionid){ - return false + let effectiveSessionId = sessionid.isEmpty ? self.sessionid : sessionid + if !sessionid.isEmpty { + self.sessionid = sessionid } - self.sessionid = sessionid + // os_log("调用链: speakStream 接收 session=%{public}@ effective=%{public}@ 输入长度=%{public}d bufferLen=%{public}d", log: log, type: .info, sessionid, effectiveSessionId, text.count, streamBuffer.count) if text.isEmpty { + // os_log("调用链: speakStream 空输入直接返回 session=%{public}@", log: log, type: .info, effectiveSessionId) return true } @@ -218,11 +260,17 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { // 清理文本并添加到缓冲区 let cleanedText = cleanTextForTTS(text) + if cleanedText.isEmpty { + // os_log("调用链: speakStream 清理后为空 session=%{public}@ rawLen=%{public}d", log: log, type: .info, effectiveSessionId, text.count) + return true + } streamBuffer.append(cleanedText) + // os_log("调用链: speakStream 入缓冲 session=%{public}@ cleanedLen=%{public}d bufferLen=%{public}d", log: log, type: .info, effectiveSessionId, cleanedText.count, streamBuffer.count) // 防抖逻辑 (150ms) let currentTime = Date().timeIntervalSince1970 if currentTime - lastSpeakTime < 0.15 { + // os_log("调用链: speakStream 防抖跳过 session=%{public}@ deltaMs=%{public}d", log: log, type: .info, effectiveSessionId, Int((currentTime - lastSpeakTime) * 1000.0)) return true } lastSpeakTime = currentTime @@ -248,17 +296,28 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { streamBuffer = String(currentText[startIndex...]) if !textToSpeak.isEmpty { - return speakOnce(sessionid: sessionid, textToSpeak) + // os_log("调用链: speakStream 触发合成 session=%{public}@ speakLen=%{public}d remainLen=%{public}d", log: log, type: .info, effectiveSessionId, textToSpeak.count, streamBuffer.count) + return speakOnce(sessionid: effectiveSessionId, textToSpeak) } } + let maxCharsBeforeForcedSpeak = 120 + if currentText.count >= maxCharsBeforeForcedSpeak { + let textToSpeak = String(currentText.prefix(maxCharsBeforeForcedSpeak)) + let startIndex = currentText.index(currentText.startIndex, offsetBy: maxCharsBeforeForcedSpeak) + streamBuffer = String(currentText[startIndex...]) + // os_log("调用链: speakStream 强制分段合成 session=%{public}@ speakLen=%{public}d remainLen=%{public}d", log: log, type: .info, effectiveSessionId, textToSpeak.count, streamBuffer.count) + return speakOnce(sessionid: effectiveSessionId, textToSpeak) + } + + // os_log("调用链: speakStream 未触发合成 session=%{public}@ bufferLen=%{public}d", log: log, type: .debug, effectiveSessionId, streamBuffer.count) return true } /** * 刷新并播放流式文本 */ - public func flushStream(sessionid sessionid: String) -> Bool { + public func flushStream(sessionid: String) -> Bool { if !isInitialized { notifyEvent(eventType: .error, params: [ "errorCode": "NOT_INITIALIZED", @@ -266,6 +325,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { ]) return false } + // os_log("调用链: flushStream session=%{public}@ bufferLen=%{public}d", log: log, type: .info, sessionid, streamBuffer.count) let remainingText = streamBuffer streamBuffer = "" @@ -327,9 +387,9 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { synthesisQueue.async { self.recreateSynthesizer() } - - deactivateAudioSessionAfterTTSIfIdle() + deactivateAudioSessionAfterTTSIfIdle() + return true } @@ -372,6 +432,14 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { } } + public func setAudioSourceType(isExtaudioSource: Bool) -> Bool { + self.isExtaudioSource = isExtaudioSource + if isInitialized { + safeConfigureAudioSessionForTTS() + } + return true + } + /** * 设置音频输出设备 */ @@ -411,7 +479,8 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { taskLock.lock() pendingTasks.append(task) taskLock.unlock() - + // os_log("调用链: enqueueSynthesisTask 入队 queued=%{public}d processing=%{public}@", log: log, type: .debug, queued, isProcessing.description) + // 确保任务被处理 processTasksIfNeeded() } @@ -430,6 +499,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { isProcessing = true let task = pendingTasks.removeFirst() taskLock.unlock() + // os_log("调用链: processTasksIfNeeded 开始执行 remaining=%{public}d", log: log, type: .info, remaining) synthesisQueue.async { task() @@ -444,6 +514,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { self.processTasksIfNeeded() } else { self.taskLock.unlock() + // os_log("调用链: processTasksIfNeeded 队列清空", log: self.log, type: .info) } } } @@ -471,10 +542,12 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { */ private func performSynthesis(sessionid:String,text: String) { if (self.sessionid != sessionid) { + // os_log("调用链: performSynthesis 丢弃(会话不匹配) current=%{public}@ task=%{public}@", log: log, type: .info, self.sessionid, sessionid) return } speaking = true // lastSpokenText = text + // os_log("调用链: performSynthesis 准备合成 session=%{public}@ pushStream=%{public}@ suppress=%{public}@ textLen=%{public}d", log: log, type: .info, sessionid, isUsingPushStreamCapture.description, suppressStreamPlayback.description, text.count) if !applyAudioSessionForTTS() { speaking = false notifyEvent(eventType: .error, params: [ @@ -488,13 +561,14 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { do { if (self.sessionid != sessionid) { + // os_log("调用链: performSynthesis 中止(会话变化) current=%{public}@ task=%{public}@", log: log, type: .info, self.sessionid, sessionid) return } - os_log("开始合成: %{public}@", log: log, type: .debug, ssml) + // os_log("调用链: performSynthesis startSpeakingSsml session=%{public}@ ssmlLen=%{public}d", log: log, type: .info, sessionid, ssml.count) let result = try synthesizer?.startSpeakingSsml(ssml) if let result = result { - os_log("合成完成，结果: %{public}@", log: log, type: .debug, String(describing: result.reason)) + // os_log("调用链: performSynthesis 返回 reason=%{public}@ session=%{public}@", log: log, type: .debug, String(describing: result.reason), sessionid) } } catch { speaking = false @@ -514,9 +588,11 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { */ private func recreateSynthesizer() { do { + // os_log("调用链: recreateSynthesizer 开始 useInternalPlayer=%{public}@ customStream=%{public}@ initialized=%{public}@", log: log, type: .info, useInternalPlayer.description, (customAudioOutputStream != nil).description, isInitialized.description) // 强制释放旧的合成器 synthesizer = nil isUsingPushStreamCapture = false + internalAudioOutputStream = nil // 创建音频配置 let audioConfig: SPXAudioConfiguration? @@ -535,13 +611,14 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { return UInt(data.count) }, closeHandler: { [weak self] in guard let self = self else { return } - os_log("调用链: 输出流关闭 累计字节=%{public}d", log: self.log, type: .info, totalBytes) + // os_log("调用链: 输出流关闭 累计字节=%{public}d", log: self.log, type: .info, totalBytes) totalBytes = 0 }) // 如果创建失败，抛出以进入 catch guard let nonNilStream = created else { throw NSError(domain: "AzureTtsHelper", code: -1, userInfo: [NSLocalizedDescriptionKey: "创建推送输出流失败"]) } + internalAudioOutputStream = nonNilStream stream = nonNilStream } audioConfig = try SPXAudioConfiguration(streamOutput: stream) @@ -556,6 +633,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { synthesizer = try SPXSpeechSynthesizer(speechConfig!) } setupEventListeners() + // os_log("调用链: recreateSynthesizer 完成 pushStream=%{public}@", log: log, type: .info, isUsingPushStreamCapture.description) } catch { os_log("重新创建合成器失败: %{public}@", log: log, type: .error, error.localizedDescription) notifyEvent(eventType: .error, params: [ @@ -578,16 +656,27 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { self.activeStreamSynthesisCount += 1 self.prepareStreamingQueueForNewSynthesisIfNeeded() } + self.pushStreamChunkCount = 0 + self.pushStreamCallbackCount = 0 + self.synthEventAudioCount = 0 } - os_log("调用链: 合成开始 session=%{public}@", log: self.log, type: .info, self.sessionid) + // os_log("调用链: 合成开始 session=%{public}@ pushStream=%{public}@ active=%{public}d", log: self.log, type: .info, self.sessionid, self.isUsingPushStreamCapture.description, self.activeStreamSynthesisCount) self.notifyEvent(eventType: .synthesisStarted) } synthesizer?.addSynthesizingEventHandler { [weak self] _, event in if let audioData = event.result.audioData { guard let self = self else { return } - os_log("调用链: 合成中 接收音频片段=%{public}dB session=%{public}@", log: self.log, type: .debug, audioData.count, self.sessionid) - if !self.isUsingPushStreamCapture { + // os_log("调用链: 合成中 接收音频片段=%{public}dB session=%{public}@", log: self.log, type: .debug, audioData.count, self.sessionid) + if self.isUsingPushStreamCapture { + if self.pushStreamCallbackCount == 0, !audioData.isEmpty { + self.synthEventAudioCount += 1 + if self.synthEventAudioCount == 1 { + // os_log("调用链: 推送流无回调，使用合成事件音频回退 session=%{public}@", log: self.log, type: .info, self.sessionid) + } + self.handleSynthesizedAudioChunk(audioData, source: "synthEvent") + } + } else { self.currentSynthesisBuffer.append(audioData) self.notifyAudioData(audioData) } @@ -602,7 +691,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { } // 将当前合成的音频加入播放队列 if !self.currentSynthesisBuffer.isEmpty { - os_log("调用链: 合成完成 入队播放 数据长度=%{public}dB session=%{public}@", log: self.log, type: .info, self.currentSynthesisBuffer.count, self.sessionid) + // os_log("调用链: 合成完成 入队播放 数据长度=%{public}dB session=%{public}@", log: self.log, type: .info, self.currentSynthesisBuffer.count, self.sessionid) self.enqueuePlaybackItem(self.currentSynthesisBuffer) self.currentSynthesisBuffer = Data() } @@ -613,6 +702,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { self.checkStreamPlaybackCompletedIfNeeded() } } + // os_log("调用链: 合成完成 session=%{public}@ pushStream=%{public}@ pendingPcm=%{public}d scheduled=%{public}d", log: self.log, type: .info, self.sessionid, self.isUsingPushStreamCapture.description, self.pcmPendingData.count, self.scheduledBufferCount) // 减少待处理文本计数 self.pendingTextCount = max(0, self.pendingTextCount - 1) // 通知合成完成 @@ -667,16 +757,50 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { * - Throws: 无 */ private func handleSynthesizedAudioChunk(_ data: Data) { - if suppressStreamPlayback { return } - if sessionid.isEmpty { return } + handleSynthesizedAudioChunk(data, source: "pushStream") + } + + private func handleSynthesizedAudioChunk(_ data: Data, source: String) { + if suppressStreamPlayback { + // os_log("调用链: 推送音频丢弃(suppress) bytes=%{public}d", log: log, type: .debug, data.count) + return + } + if sessionid.isEmpty, !speaking { + // os_log("调用链: 推送音频丢弃(无会话且未speaking) bytes=%{public}d", log: log, type: .debug, data.count) + return + } notifyAudioData(data) - guard isUsingPushStreamCapture else { return } + guard isUsingPushStreamCapture else { + // os_log("调用链: 推送音频忽略(未启用pushStream) bytes=%{public}d", log: log, type: .debug, data.count) + return + } + if source == "pushStream" { + pushStreamCallbackCount += 1 + } else if source == "synthEvent" { + if pushStreamCallbackCount > 0 { return } + } + pushStreamChunkCount += 1 + if pushStreamChunkCount == 1 { + // os_log("调用链: 推送音频首包 source=%{public}@ session=%{public}@ bytes=%{public}d speaking=%{public}@", log: log, type: .info, source, sessionid, data.count, speaking.description) + } audioPlaybackQueue.async { - if self.sessionid.isEmpty { return } + if self.sessionid.isEmpty, !self.speaking { + // os_log("调用链: 推送音频异步丢弃(无会话且未speaking) bytes=%{public}d", log: self.log, type: .debug, data.count) + return + } let pcm = self.stripWavHeaderIfNeeded(data) if !pcm.isEmpty { self.pcmPendingData.append(pcm) + if self.scheduledBufferCount == 0 { + // os_log("调用链: 推送音频入缓冲 pcm=%{public}dB pendingPcm=%{public}d", log: self.log, type: .debug, pcm.count, self.pcmPendingData.count) + } self.scheduleAvailablePcmBuffers() + } else { + if self.pushStreamChunkCount <= 3 { + // os_log("调用链: 推送音频等待WAV头 session=%{public}@ probe=%{public}dB in=%{public}dB", log: self.log, type: .info, self.sessionid, self.wavHeaderProbeBuffer.count, data.count) + } else { + // os_log("调用链: 推送音频等待WAV头 probe=%{public}dB in=%{public}dB", log: self.log, type: .debug, self.wavHeaderProbeBuffer.count, data.count) + } } } } @@ -689,6 +813,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { private func prepareStreamingQueueForNewSynthesisIfNeeded() { if pcmPendingData.isEmpty, scheduledBufferCount == 0, playerNode?.isPlaying != true { hasStrippedWavHeader = false + wavHeaderProbeBuffer.removeAll(keepingCapacity: true) hasNotifiedPlaybackStartedForStream = false } } @@ -700,8 +825,10 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { */ private func resetStreamingPlaybackState() { audioPlaybackQueue.sync { + // os_log("调用链: resetStreamingPlaybackState 清理 before pendingPcm=%{public}d scheduled=%{public}d active=%{public}d", log: self.log, type: .info, self.pcmPendingData.count, self.scheduledBufferCount, self.activeStreamSynthesisCount) self.pcmPendingData = Data() self.hasStrippedWavHeader = false + self.wavHeaderProbeBuffer = Data() self.hasNotifiedPlaybackStartedForStream = false self.activeStreamSynthesisCount = 0 self.scheduledBufferCount = 0 @@ -710,6 +837,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { self.audioEngine = nil self.playerNode = nil self.pcmFormat = nil + // os_log("调用链: resetStreamingPlaybackState 清理完成", log: self.log, type: .info) } } @@ -720,6 +848,9 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { */ private func scheduleAvailablePcmBuffers() { guard ensureStreamingEngineIfNeeded() else { return } + if !hasNotifiedPlaybackStartedForStream, pcmPendingData.count >= streamChunkBytes { + // os_log("调用链: scheduleAvailablePcmBuffers 准备调度 pendingPcm=%{public}d chunk=%{public}d active=%{public}d", log: log, type: .info, pcmPendingData.count, streamChunkBytes, activeStreamSynthesisCount) + } while pcmPendingData.count >= streamChunkBytes { let chunk = pcmPendingData.prefix(streamChunkBytes) pcmPendingData.removeFirst(streamChunkBytes) @@ -727,7 +858,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { scheduledBufferCount += 1 if !hasNotifiedPlaybackStartedForStream { hasNotifiedPlaybackStartedForStream = true - os_log("调用链: 流式播放开始 session=%{public}@", log: log, type: .info, sessionid) + // os_log("调用链: 流式播放开始 session=%{public}@", log: log, type: .info, sessionid) notifyEvent(eventType: .playbackStarted) } playerNode?.scheduleBuffer(buffer, completionHandler: { [weak self] in @@ -747,7 +878,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { scheduledBufferCount += 1 if !hasNotifiedPlaybackStartedForStream { hasNotifiedPlaybackStartedForStream = true - os_log("调用链: 流式播放开始 session=%{public}@", log: log, type: .info, sessionid) + // os_log("调用链: 流式播放开始 session=%{public}@", log: log, type: .info, sessionid) notifyEvent(eventType: .playbackStarted) } playerNode?.scheduleBuffer(buffer, completionHandler: { [weak self] in @@ -759,7 +890,8 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { }) } } - if playerNode?.isPlaying == false { + if playerNode?.isPlaying == false, scheduledBufferCount > 0 { + // os_log("调用链: playerNode.play scheduled=%{public}d pendingPcm=%{public}d", log: log, type: .debug, scheduledBufferCount, pcmPendingData.count) playerNode?.play() } } @@ -773,7 +905,7 @@ private func checkStreamPlaybackCompletedIfNeeded() { if activeStreamSynthesisCount == 0, pcmPendingData.isEmpty, scheduledBufferCount == 0 { speaking = false hasNotifiedPlaybackStartedForStream = false - os_log("调用链: 流式播放完成 session=%{public}@", log: log, type: .info, sessionid) + // os_log("调用链: 流式播放完成 session=%{public}@", log: log, type: .info, sessionid) notifyEvent(eventType: .playbackCompleted) deactivateAudioSessionAfterTTSIfIdle() } @@ -862,25 +994,36 @@ private func checkStreamPlaybackCompletedIfNeeded() { */ private func stripWavHeaderIfNeeded(_ data: Data) -> Data { if hasStrippedWavHeader { return data } - if data.count >= 12, - String(data: data.subdata(in: 0..<4), encoding: .ascii) == "RIFF", - String(data: data.subdata(in: 8..<12), encoding: .ascii) == "WAVE" { + wavHeaderProbeBuffer.append(data) + if wavHeaderProbeBuffer.count < 12 { return Data() } + if wavHeaderProbeBuffer.count >= 12, + String(data: wavHeaderProbeBuffer.subdata(in: 0..<4), encoding: .ascii) == "RIFF", + String(data: wavHeaderProbeBuffer.subdata(in: 8..<12), encoding: .ascii) == "WAVE" { let marker = Data("data".utf8) - if let range = data.range(of: marker, options: [], in: 0.. 44 { + if wavHeaderProbeBuffer.count > 44 { hasStrippedWavHeader = true - return data.subdata(in: 44.. Bool { let audioSession = AVAudioSession.sharedInstance() let outputs: [AVAudioSessionPortDescription] = audioSession.currentRoute.outputs let hasBluetoothHFP = outputs.contains(where: { $0.portType == .bluetoothHFP }) + let hasWiredHeadphones = outputs.contains(where: { $0.portType == .headphones || $0.portType == .headsetMic }) + let hasBluetoothA2DP = outputs.contains(where: { $0.portType == .bluetoothA2DP }) + let hasHeadphones = hasWiredHeadphones || hasBluetoothA2DP let otherAudioPlaying = audioSession.isOtherAudioPlaying + let isRecording = (self.micCapture?.isCapturing == true) let desiredCategory: AVAudioSession.Category let desiredMode: AVAudioSession.Mode @@ -1218,6 +1365,21 @@ private func applyAudioSessionForTTS() -> Bool { desiredCategory = .playback desiredMode = .default desiredOptions.insert(.duckOthers) + } else if isRecording { + desiredCategory = .playAndRecord + desiredMode = self.isExtaudioSource ? .default : .videoChat + desiredOptions.insert(.allowBluetooth) + desiredOptions.insert(.allowBluetoothA2DP) + if self.isExtaudioSource { + if otherAudioPlaying { + desiredOptions.insert(.duckOthers) + } + } else { + desiredOptions.insert(.mixWithOthers) + if !hasHeadphones { + desiredOptions.insert(.defaultToSpeaker) + } + } } else { desiredCategory = .playback desiredMode = .spokenAudio @@ -1233,6 +1395,9 @@ private func applyAudioSessionForTTS() -> Bool { if audioSession.category != desiredCategory || audioSession.mode != desiredMode || audioSession.categoryOptions != desiredOptions { try audioSession.setCategory(desiredCategory, mode: desiredMode, options: desiredOptions) } + if desiredCategory == .playAndRecord { + _ = try? audioSession.setPreferredIOBufferDuration(0.02) + } try audioSession.setActive(true) break } catch { @@ -1244,11 +1409,15 @@ private func applyAudioSessionForTTS() -> Bool { } } catch { do { - try audioSession.setCategory(.playback, mode: .default, options: []) try audioSession.setActive(true) } catch { - success = false - os_log("TTS 音频会话配置失败: %{public}@", log: self.log, type: .error, error.localizedDescription) + do { + try audioSession.setCategory(.playback, mode: .default, options: []) + try audioSession.setActive(true) + } catch { + success = false + os_log("TTS 音频会话配置失败: %{public}@", log: self.log, type: .error, error.localizedDescription) + } } } } From 337a6f659bbab8d5a93f7d2c5fc9211c86d483d1 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Mon, 19 Jan 2026 21:25:13 +0800 Subject: [PATCH 09/25] =?UTF-8?q?feat(=E8=AF=AD=E9=9F=B3=E5=A4=84=E7=90=86?= =?UTF-8?q?):=20=E5=AE=9E=E7=8E=B0=E7=94=B5=E8=AF=9D=E6=A8=A1=E5=BC=8F?= =?UTF-8?q?=E9=9F=B3=E9=A2=91=E8=B7=AF=E7=94=B1=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加电话模式支持，优化音频路由和麦克风选择逻辑 - 在AgentServiceImpl中新增电话模式设置 - AzureAsrHelper和AzureTtsHelper添加电话模式状态管理 - SimpleAudioReceiver支持优先使用内置麦克风 - 改进音频会话配置和错误处理 - 增强麦克风权限请求和格式转换稳定性 --- .../agent_service/AgentServiceImpl.swift | 11 +- .../Sources/azure_speech/AzureAsrHelper.swift | 21 +- .../Sources/azure_speech/AzureTtsHelper.swift | 30 ++- .../Sources/tools/MicrophoneCapture.swift | 185 +++++++++++------- .../Sources/tools/SimpleAudioReceiver.swift | 46 ++++- 5 files changed, 214 insertions(+), 79 deletions(-) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 08f7f4b7f..8914a89bb 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -402,7 +402,8 @@ class AgentServiceImpl: NSObject { // 根据模式决定空闲检测策略 switch mode { case "phone_call": - azureAsrHelper?.restoreOriginalAudioState() + _ = azureAsrHelper?.setPhoneCallMode(enabled: true) + _ = azureTtsHelper?.setPhoneCallMode(enabled: true) // 通话模式：禁用空闲检测，保持持续激活 os_log("通话模式：禁用空闲检测", log: logger, type: .info) return @@ -523,6 +524,9 @@ class AgentServiceImpl: NSObject { // 设置当前识别模式 currentRecognitionMode = mode os_log("设置语音识别模式: %{public}@", log: logger, type: .info, mode) + _ = azureAsrHelper?.setPhoneCallMode(enabled: mode == "phone_call") + _ = azureTtsHelper?.setPhoneCallMode(enabled: mode == "phone_call") + _ = azureAsrHelper?.setPreferBuiltInMic(enabled: mode == "phone_call" || mode == "push_to_talk") let audioSourceType: AzureAsrHelper.AudioSourceType = useBle ? .external : .microphone @@ -579,6 +583,11 @@ class AgentServiceImpl: NSObject { os_log("停止语音识别，当前模式: %{public}@", log: logger, type: .info, currentRecognitionMode) let success = azureAsrHelper?.stopContinuousRecognition() ?? false + if currentRecognitionMode == "phone_call" { + _ = azureAsrHelper?.setPhoneCallMode(enabled: false) + _ = azureTtsHelper?.setPhoneCallMode(enabled: false) + } + _ = azureAsrHelper?.setPreferBuiltInMic(enabled: false) if !success && isStartingRecognition { // 启动尚未完成，先记录一次待停止请求，onSessionStarted 到来后立即 stop diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift index 3ea1fb6a0..a69b0c385 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift @@ -73,6 +73,7 @@ public class AzureAsrHelper: NSObject { } public var audioSourceType = AudioSourceType.microphone + private var isPhoneCallMode = false // 音频处理 // private var externalAudioStream: ExternalAudioPullStream? @@ -80,6 +81,23 @@ public class AzureAsrHelper: NSObject { // 音频处理 public var audioStream: SimpleAudioReceiver? + /** + * 设置 AI 电话/按住说话场景下是否优先使用手机内置麦克风 + * @param enabled 是否启用 + * @return 是否设置成功 + * @throws 无 + */ + public func setPreferBuiltInMic(enabled: Bool) -> Bool { + audioStream?.setPreferBuiltInMic(enabled) + return true + } + + public func setPhoneCallMode(enabled: Bool) -> Bool { + isPhoneCallMode = enabled + audioStream?.setPhoneCallMode(enabled) + return true + } + public func asrProvider() -> String { return useXunfei ? "xunfei" : "azure" } @@ -237,6 +255,7 @@ public class AzureAsrHelper: NSObject { if audioStream == nil { audioStream = SimpleAudioReceiver() audioStream?.initAudioRecord() + audioStream?.setPhoneCallMode(isPhoneCallMode) } // 预创建音频配置 if let pushStream = audioStream?.pushAudioStream { @@ -768,6 +787,7 @@ public class AzureAsrHelper: NSObject { if audioStream == nil { audioStream = SimpleAudioReceiver() audioStream?.initAudioRecord() // 确保调用初始化 + audioStream?.setPhoneCallMode(isPhoneCallMode) } // 检查音频配置是否已存在 @@ -1208,4 +1228,3 @@ private class AudioDataCallbackProxy: SimpleAudioReceiver.AudioDataCallback { - diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index b59031000..aee7d4930 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -78,6 +78,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { private var pushStreamCallbackCount = 0 private var synthEventAudioCount = 0 private var isExtaudioSource = false + private var isPhoneCallMode = false @@ -440,6 +441,14 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { return true } + public func setPhoneCallMode(enabled: Bool) -> Bool { + isPhoneCallMode = enabled + if isInitialized { + safeConfigureAudioSessionForTTS() + } + return true + } + /** * 设置音频输出设备 */ @@ -1361,15 +1370,20 @@ private func applyAudioSessionForTTS() -> Bool { let desiredMode: AVAudioSession.Mode var desiredOptions: AVAudioSession.CategoryOptions = [] - if self.isTelephonyActive() || hasBluetoothHFP { + if self.isTelephonyActive() { desiredCategory = .playback desiredMode = .default desiredOptions.insert(.duckOthers) } else if isRecording { desiredCategory = .playAndRecord - desiredMode = self.isExtaudioSource ? .default : .videoChat - desiredOptions.insert(.allowBluetooth) - desiredOptions.insert(.allowBluetoothA2DP) + if self.isPhoneCallMode { + desiredMode = .voiceChat + desiredOptions.insert(.allowBluetooth) + } else { + desiredMode = self.isExtaudioSource ? .default : .videoChat + desiredOptions.insert(.allowBluetooth) + desiredOptions.insert(.allowBluetoothA2DP) + } if self.isExtaudioSource { if otherAudioPlaying { desiredOptions.insert(.duckOthers) @@ -1380,6 +1394,10 @@ private func applyAudioSessionForTTS() -> Bool { desiredOptions.insert(.defaultToSpeaker) } } + } else if hasBluetoothHFP { + desiredCategory = .playback + desiredMode = .default + desiredOptions.insert(.duckOthers) } else { desiredCategory = .playback desiredMode = .spokenAudio @@ -1399,6 +1417,9 @@ private func applyAudioSessionForTTS() -> Bool { _ = try? audioSession.setPreferredIOBufferDuration(0.02) } try audioSession.setActive(true) + if self.isPhoneCallMode, desiredCategory == .playAndRecord, !hasHeadphones { + _ = try? audioSession.overrideOutputAudioPort(.speaker) + } break } catch { if attempt == 2 { @@ -1511,6 +1532,7 @@ private func deactivateAudioSessionAfterTTSIfIdle() { ensureAudioSessionQueueSpecificKeySet() let work = { [weak self] in guard let self = self else { return } + if self.micCapture?.isCapturing == true { return } if self.speaking { return } if self.isPlaying { return } if !self.playbackQueue.isEmpty { return } diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift index df55d1727..b379819de 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift @@ -206,49 +206,55 @@ public class MicrophoneCapture: NSObject { } // 检查麦克风权限 + let permission: AVAudioSession.RecordPermission switch audioSession.recordPermission { case .granted: - do { - try audioSession.setCategory(.record, - mode: .default, - options: []) - if let inputs = audioSession.availableInputs, - let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { - try? audioSession.setPreferredInput(builtInMic) - } - try audioSession.setActive(true) - } catch { - throw error - } - try setupAudioEngine() + permission = .granted case .denied: - throw NSError(domain: "麦克风权限被拒绝", code: 0) + permission = .denied case .undetermined: + let semaphore = DispatchSemaphore(value: 0) + var grantedResult = false audioSession.requestRecordPermission { granted in - if granted { - do { - do { - try audioSession.setCategory(.record, - mode: .default, - options: []) - if let inputs = audioSession.availableInputs, - let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { - try? audioSession.setPreferredInput(builtInMic) - } - try audioSession.setActive(true) - } catch { - print("音频会话配置失败: \(error.localizedDescription)") - return - } - try self.setupAudioEngine() - } catch { - print("音频引擎设置失败: \(error.localizedDescription)") - } - } + grantedResult = granted + semaphore.signal() + } + let waitResult = semaphore.wait(timeout: .now() + 60) + if waitResult == .timedOut { + throw NSError(domain: "麦克风权限请求超时", code: 2) } + permission = grantedResult ? .granted : .denied @unknown default: throw NSError(domain: "未知权限状态", code: 1) } + + guard permission == .granted else { + throw NSError(domain: "麦克风权限被拒绝", code: 0) + } + + do { + print("startCapture: 激活前 category=\(audioSession.category.rawValue) mode=\(audioSession.mode.rawValue) options=\(audioSession.categoryOptions)") + print("startCapture: 激活前 sampleRate=\(audioSession.sampleRate) ioBuffer=\(audioSession.ioBufferDuration)") + print("startCapture: 激活前 route inputs=\(audioSession.currentRoute.inputs.map { $0.portType.rawValue }) outputs=\(audioSession.currentRoute.outputs.map { $0.portType.rawValue })") + if let inputs = audioSession.availableInputs, + let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { + try? audioSession.setPreferredInput(builtInMic) + } + try audioSession.setActive(true) + print("startCapture: 激活后 category=\(audioSession.category.rawValue) mode=\(audioSession.mode.rawValue) options=\(audioSession.categoryOptions)") + print("startCapture: 激活后 sampleRate=\(audioSession.sampleRate) ioBuffer=\(audioSession.ioBufferDuration)") + print("startCapture: 激活后 route inputs=\(audioSession.currentRoute.inputs.map { $0.portType.rawValue }) outputs=\(audioSession.currentRoute.outputs.map { $0.portType.rawValue })") + } catch { + throw error + } + isCapturing = true + do { + try setupAudioEngine() + } catch { + isCapturing = false + cleanupAudioEngine() + throw error + } } /** @@ -299,36 +305,51 @@ public class MicrophoneCapture: NSObject { if let audioSession = audioSession { sampleRate = audioSession.sampleRate } - - // 设置音频格式 - let inputFormat = audioInputNode.inputFormat(forBus: 0) - - // iOS 13+ 启用语音处理 + + // iOS 13+ 启用语音处理（可能在部分路由/模式下失败，失败不应阻断录音启动） if #available(iOS 13.0, *) { - try audioInputNode.setVoiceProcessingEnabled(true) + do { + try audioInputNode.setVoiceProcessingEnabled(true) + } catch { + if let nsError = error as NSError? { + print("启用语音处理失败: domain=\(nsError.domain) code=\(nsError.code) desc=\(nsError.localizedDescription)") + } else { + print("启用语音处理失败: \(error.localizedDescription)") + } + } } - - // 检查输入格式是否符合Microsoft要求 - let needsConversion = !isMicrosoftCompatibleFormat(inputFormat) - - // 需要格式转换 - guard let targetFormat = audioFormat, - let converter = AVAudioConverter(from: inputFormat, to: targetFormat) else { + + // 等待并获取有效的输入格式：在路由/类别切换瞬间可能出现 0Hz，直接 installTap 会触发系统断言崩溃 + guard let inputFormat = waitForValidInputFormat(inputNode: audioInputNode, maxAttempts: 10, retryIntervalSeconds: 0.03) else { + let currentSession = AVAudioSession.sharedInstance() + let inputs = currentSession.currentRoute.inputs.map { $0.portType.rawValue }.joined(separator: ",") + let outputs = currentSession.currentRoute.outputs.map { $0.portType.rawValue }.joined(separator: ",") + throw NSError(domain: "AudioSetup", code: 3, userInfo: [ + NSLocalizedDescriptionKey: "输入音频格式无效（sampleRate/channelCount），可能处于路由或类别切换中", + "sessionCategory": currentSession.category.rawValue, + "sessionMode": currentSession.mode.rawValue, + "sessionSampleRate": currentSession.sampleRate, + "routeInputs": inputs, + "routeOutputs": outputs + ]) + } + + // 需要格式转换（输出固定 16k/16bit/mono PCM，匹配 PushStream 默认/常用配置） + guard let targetFormat = audioFormat else { + throw NSError(domain: "AudioSetup", code: 2, userInfo: [NSLocalizedDescriptionKey: "目标音频格式为空"]) + } + guard let converter = AVAudioConverter(from: inputFormat, to: targetFormat) else { throw NSError(domain: "AudioSetup", code: 2, userInfo: [NSLocalizedDescriptionKey: "音频格式转换器创建失败"]) } - - print("输入格式不符合Microsoft要求，进行格式转换") - print("输入格式: \(inputFormat.sampleRate)Hz, \(inputFormat.commonFormat.rawValue)") - print("目标格式: \(targetFormat.sampleRate)Hz, \(targetFormat.commonFormat.rawValue)") - - // 添加tap进行格式转换 + + print("输入格式: \(inputFormat.sampleRate)Hz, \(inputFormat.commonFormat.rawValue), ch=\(inputFormat.channelCount), interleaved=\(inputFormat.isInterleaved)") + print("目标格式: \(targetFormat.sampleRate)Hz, \(targetFormat.commonFormat.rawValue), ch=\(targetFormat.channelCount), interleaved=\(targetFormat.isInterleaved)") + audioInputNode.installTap(onBus: 0, - bufferSize: 1024, - format: inputFormat) { [weak self] buffer, when in - + bufferSize: 1024, + format: inputFormat) { [weak self] buffer, _ in guard let strongSelf = self else { return } - - // 修复：添加安全检查，避免强制解包崩溃 + guard let convertedBuffer = AVAudioPCMBuffer( pcmFormat: targetFormat, frameCapacity: AVAudioFrameCount( @@ -338,36 +359,59 @@ public class MicrophoneCapture: NSObject { print("创建转换缓冲区失败") return } - + var error: NSError? - // 执行音频格式转换 let status = converter.convert( to: convertedBuffer, error: &error, - withInputFrom: { inNumPackets, outStatus in + withInputFrom: { _, outStatus in outStatus.pointee = .haveData return buffer } ) - - // 转换成功且无错误 + if status == .haveData, error == nil { let data = strongSelf.audioBufferToData(convertedBuffer) strongSelf.audioDataHandler?(data) } else if let error = error { - print("音频格式转换失败: \(error.localizedDescription)") + print("音频格式转换失败: domain=\(error.domain) code=\(error.code) desc=\(error.localizedDescription)") } } + audioEngine.prepare() // 启动引擎 do { try audioEngine.start() - isCapturing = true } catch { - print("音频引擎启动失败: \(error.localizedDescription)") + if let nsError = error as NSError? { + print("音频引擎启动失败: domain=\(nsError.domain) code=\(nsError.code) desc=\(nsError.localizedDescription)") + } else { + print("音频引擎启动失败: \(error.localizedDescription)") + } throw error } } + + /** + * 等待并获取有效的输入音频格式 + * - 参数: + * - inputNode: 输入节点（通常为 AVAudioEngine.inputNode） + * - maxAttempts: 最大重试次数 + * - retryIntervalSeconds: 每次重试间隔（秒） + * - 返回值: 若在重试窗口内拿到有效格式则返回 AVAudioFormat，否则返回 nil + * - 异常: 无（内部不抛出异常） + */ + private func waitForValidInputFormat(inputNode: AVAudioInputNode, maxAttempts: Int, retryIntervalSeconds: TimeInterval) -> AVAudioFormat? { + for attempt in 1...maxAttempts { + let format = inputNode.inputFormat(forBus: 0) + if format.sampleRate > 0, format.channelCount > 0 { + return format + } + print("输入格式无效，等待重试 attempt=\(attempt) sampleRate=\(format.sampleRate) ch=\(format.channelCount)") + Thread.sleep(forTimeInterval: retryIntervalSeconds) + } + return nil + } /** * 安全清理音频引擎资源 @@ -401,6 +445,15 @@ public class MicrophoneCapture: NSObject { count: dataLength ) } + // Handle 32-bit integer format + else if let int32Data = buffer.int32ChannelData { + let int32Buffer = int32Data.pointee + var int16Array = [Int16](repeating: 0, count: frameLength) + for i in 0..> 16))) + } + return Data(bytes: int16Array, count: dataLength) + } // Handle float format else if let floatData = buffer.floatChannelData { var int16Array = [Int16](repeating: 0, count: frameLength) diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift index 0ae23d2e4..b5a0a8a5f 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift @@ -56,6 +56,8 @@ public class SimpleAudioReceiver: NSObject { public var recordfile: RecordFile? public var isHeadphones = true public var isReceiverHFP = false // 是否是接收HFP音频 + public var isPhoneCallMode = false // AI电话模式：录音+播放强制走电话链路（voiceChat/HFP） + public var preferBuiltInMic = false // 耳机连接时优先使用手机内置麦克风录音 /** * 初始化 @@ -134,8 +136,7 @@ public class SimpleAudioReceiver: NSObject { print("音源设制---------- startAudioRecord=audioSourceType:\(audioSourceType)") switch audioSourceType { case .microphone: - // 唤醒后禁用通话音道（HFP），改为手机麦克风 + A2DP 输出 - isReceiverHFP = false + isReceiverHFP = isPhoneCallMode // 动态检测是否存在耳机/蓝牙输出，用于后续路由选项计算 isHeadphones = detectHeadphonesOutput() runMicrophoneCapture() @@ -144,6 +145,20 @@ public class SimpleAudioReceiver: NSObject { runExternalCapture() } } + + public func setPhoneCallMode(_ enabled: Bool) { + isPhoneCallMode = enabled + } + + /** + * 设置是否优先使用手机内置麦克风录音 + * @param enabled 是否启用 + * @return 无 + * @throws 无 + */ + public func setPreferBuiltInMic(_ enabled: Bool) { + preferBuiltInMic = enabled + } /** * 检测是否存在蓝牙HFP输入设备 @@ -327,7 +342,15 @@ public class SimpleAudioReceiver: NSObject { let desiredCategory: AVAudioSession.Category let desiredMode: AVAudioSession.Mode let desiredOptions: AVAudioSession.CategoryOptions - if isReceiverHFP { + if isPhoneCallMode { + desiredCategory = .playAndRecord + desiredMode = .voiceChat + if isHeadphones { + desiredOptions = [.allowBluetooth] + } else { + desiredOptions = [.allowBluetooth, .defaultToSpeaker] + } + } else if isReceiverHFP { if isTelephonyActive() { // 通话中：录音优先，避免 A2DP（仅播放），改用 HFP（.allowBluetooth） desiredCategory = .record @@ -357,7 +380,7 @@ public class SimpleAudioReceiver: NSObject { // 非通话：保持原有逻辑，但使用 .allowBluetooth 以支持录音（而不是 A2DP） desiredCategory = .playAndRecord desiredMode = .videoChat - desiredOptions = [.allowBluetoothA2DP, .mixWithOthers] + desiredOptions = [.allowBluetooth, .allowBluetoothA2DP, .mixWithOthers] //desiredOptions = [.mixWithOthers, .defaultToSpeaker] }else{ desiredCategory = .playAndRecord @@ -390,9 +413,13 @@ public class SimpleAudioReceiver: NSObject { } if hasBtOutput { try audioSession.overrideOutputAudioPort(.none) + } else if isPhoneCallMode, !isHeadphones { + try audioSession.overrideOutputAudioPort(.speaker) } if let inputs = audioSession.availableInputs { - if isReceiverHFP, let btInput = inputs.first(where: { $0.portType == .bluetoothHFP }) { + if preferBuiltInMic, let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { + try audioSession.setPreferredInput(builtInMic) + } else if isReceiverHFP, let btInput = inputs.first(where: { $0.portType == .bluetoothHFP }) { try audioSession.setPreferredInput(btInput) } else if let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { try audioSession.setPreferredInput(builtInMic) @@ -429,9 +456,14 @@ public class SimpleAudioReceiver: NSObject { // 启动录音引擎 do { print("resumeRecord") - try micCapture.startCapture() + try safeConfigureAudioSessionForMic() + try micCapture.startCapture() } catch { - os_log("Failed to start audio engine: %@", log: log, type: .error, error.localizedDescription) + if let nsError = error as NSError? { + os_log("Failed to start audio engine: domain=%{public}@ code=%{public}d desc=%{public}@", log: log, type: .error, nsError.domain, nsError.code, nsError.localizedDescription) + } else { + os_log("Failed to start audio engine: %@", log: log, type: .error, error.localizedDescription) + } } } From 68dbadd31e93507f0a221d453088d1ffad34e6b5 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Wed, 21 Jan 2026 21:38:02 +0800 Subject: [PATCH 10/25] =?UTF-8?q?refactor(audio):=20=E9=87=8D=E6=9E=84?= =?UTF-8?q?=E9=9F=B3=E9=A2=91=E5=A4=84=E7=90=86=E9=80=BB=E8=BE=91=EF=BC=8C?= =?UTF-8?q?=E7=AE=80=E5=8C=96=E6=A8=A1=E5=BC=8F=E7=AE=A1=E7=90=86=E5=92=8C?= =?UTF-8?q?=E8=AE=BE=E5=A4=87=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 移除冗余的音频模式设置方法，统一使用currentRecognitionMode管理识别模式 优化音频会话配置逻辑，根据模式和音频源类型动态调整参数 新增耳机设备检测功能，自动适配最佳音频路由 --- .../agent_service/AgentServiceImpl.swift | 25 +- .../Sources/azure_speech/AzureAsrHelper.swift | 32 +- .../Sources/azure_speech/AzureTtsHelper.swift | 63 ++-- .../Sources/tools/SimpleAudioReceiver.swift | 290 +++++++----------- 4 files changed, 153 insertions(+), 257 deletions(-) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 8914a89bb..0f35547e7 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -346,8 +346,7 @@ class AgentServiceImpl: NSObject { ttsAppId: "", ttsAppToken: azureSpeechKey, ttsResource: azureSpeechRegion, - language: primaryLanguage, - isExtaudioSource: BleService.shared.isConnected() + language: primaryLanguage ), ttsSuccess else { sendError("初始化语音合成服务失败", code: "TTS_INIT_ERROR") return false @@ -392,7 +391,7 @@ class AgentServiceImpl: NSObject { private func checkAdnSetAudioSession() { let audioSourceType: AzureAsrHelper.AudioSourceType = BleService.shared.isConnected() ? .external : .microphone - azureAsrHelper?.performAudioStart(audioSourceType: audioSourceType, audioDataCallback: nil) + azureAsrHelper?.performAudioStart(mod: currentRecognitionMode, audioSourceType: audioSourceType, audioDataCallback: nil) } private func startIdleCheckForMode(mode: String) { @@ -402,8 +401,6 @@ class AgentServiceImpl: NSObject { // 根据模式决定空闲检测策略 switch mode { case "phone_call": - _ = azureAsrHelper?.setPhoneCallMode(enabled: true) - _ = azureTtsHelper?.setPhoneCallMode(enabled: true) // 通话模式：禁用空闲检测，保持持续激活 os_log("通话模式：禁用空闲检测", log: logger, type: .info) return @@ -524,9 +521,6 @@ class AgentServiceImpl: NSObject { // 设置当前识别模式 currentRecognitionMode = mode os_log("设置语音识别模式: %{public}@", log: logger, type: .info, mode) - _ = azureAsrHelper?.setPhoneCallMode(enabled: mode == "phone_call") - _ = azureTtsHelper?.setPhoneCallMode(enabled: mode == "phone_call") - _ = azureAsrHelper?.setPreferBuiltInMic(enabled: mode == "phone_call" || mode == "push_to_talk") let audioSourceType: AzureAsrHelper.AudioSourceType = useBle ? .external : .microphone @@ -583,11 +577,6 @@ class AgentServiceImpl: NSObject { os_log("停止语音识别，当前模式: %{public}@", log: logger, type: .info, currentRecognitionMode) let success = azureAsrHelper?.stopContinuousRecognition() ?? false - if currentRecognitionMode == "phone_call" { - _ = azureAsrHelper?.setPhoneCallMode(enabled: false) - _ = azureTtsHelper?.setPhoneCallMode(enabled: false) - } - _ = azureAsrHelper?.setPreferBuiltInMic(enabled: false) if !success && isStartingRecognition { // 启动尚未完成，先记录一次待停止请求，onSessionStarted 到来后立即 stop @@ -639,7 +628,7 @@ class AgentServiceImpl: NSObject { } _ = azureTtsHelper?.setSpeechParams(rate: ttsRatePercent, pitch: 0, volume: 100) - _ = azureTtsHelper?.setAudioSourceType(isExtaudioSource: BleService.shared.isConnected()) + _ = azureTtsHelper?.setPhoneCallMode(mod: currentRecognitionMode) return azureTtsHelper?.speakOnce(sessionid:sessionid,text) ?? false } @@ -1798,7 +1787,7 @@ class ChatApiStreamCallback: StreamCallback { responseBuilder += token if speakResponse && reply && broadcast{ - _ = agentService.azureTtsHelper?.setAudioSourceType(isExtaudioSource: BleService.shared.isConnected()) + _ = agentService.azureTtsHelper?.setPhoneCallMode(mod:agentService.currentRecognitionMode) try agentService.azureTtsHelper?.speakStream(sessionid:sessionid,token) // 在开始流式TTS时立即停止气泡音 } @@ -1824,7 +1813,7 @@ class ChatApiStreamCallback: StreamCallback { } if speakResponse && reply && broadcast && sessionid == agentService.currsessionId{ - _ = agentService.azureTtsHelper?.setAudioSourceType(isExtaudioSource: BleService.shared.isConnected()) + _ = agentService.azureTtsHelper?.setPhoneCallMode(mod: agentService.currentRecognitionMode) agentService.azureTtsHelper?.flushStream(sessionid:sessionid) } @@ -1876,7 +1865,7 @@ class ChatApiStreamCallback: StreamCallback { "message": message ]) if(code == 2001){ - _ = agentService.azureTtsHelper?.setAudioSourceType(isExtaudioSource: BleService.shared.isConnected()) + _ = agentService.azureTtsHelper?.setPhoneCallMode(mod: agentService.currentRecognitionMode) agentService.azureTtsHelper?.speakStream(sessionid: sessionid,agentService.insufficientIntegralText) } agentService.isAiStreaming = false @@ -1920,7 +1909,7 @@ class ChatApiStreamCallback: StreamCallback { if (iscallingTool) { os_log("收到函数调用: callingToolText:%{public}@", log: agentService.logger, type: .info, agentService.callingToolText) - _ = agentService.azureTtsHelper?.setAudioSourceType(isExtaudioSource: BleService.shared.isConnected()) + _ = agentService.azureTtsHelper?.setPhoneCallMode(mod: agentService.currentRecognitionMode) agentService.azureTtsHelper?.speakStream(sessionid: sessionid,agentService.callingToolText) iscallingTool = false } diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift index a69b0c385..f6f89fffa 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift @@ -81,22 +81,9 @@ public class AzureAsrHelper: NSObject { // 音频处理 public var audioStream: SimpleAudioReceiver? - /** - * 设置 AI 电话/按住说话场景下是否优先使用手机内置麦克风 - * @param enabled 是否启用 - * @return 是否设置成功 - * @throws 无 - */ - public func setPreferBuiltInMic(enabled: Bool) -> Bool { - audioStream?.setPreferBuiltInMic(enabled) - return true - } - public func setPhoneCallMode(enabled: Bool) -> Bool { - isPhoneCallMode = enabled - audioStream?.setPhoneCallMode(enabled) - return true - } + + public func asrProvider() -> String { return useXunfei ? "xunfei" : "azure" @@ -255,7 +242,6 @@ public class AzureAsrHelper: NSObject { if audioStream == nil { audioStream = SimpleAudioReceiver() audioStream?.initAudioRecord() - audioStream?.setPhoneCallMode(isPhoneCallMode) } // 预创建音频配置 if let pushStream = audioStream?.pushAudioStream { @@ -283,6 +269,7 @@ public class AzureAsrHelper: NSObject { /// 1) 新任务创建前，取消旧的未执行任务，确保只有“最后一次启动请求”会生效 /// 2) 任务执行前二次校验：必须是当前挂起任务且 audioStream?.isContinuousRecognitionActive 仍为 true 才执行启动 private func startAudioRecordAsync( + mod: String, audioSourceType: AudioSourceType, audioDataCallback: SimpleAudioReceiver.AudioDataCallback? = nil, completion: @escaping (Bool) -> Void @@ -325,7 +312,7 @@ public class AzureAsrHelper: NSObject { // 准备启动 self.isAudioStarting = true self.pendingStopRequest = false // 启动前清空旧的停止请求 - let success = self.performAudioStart(audioSourceType: audioSourceType, audioDataCallback: audioDataCallback) + let success = self.performAudioStart(mod: mod, audioSourceType: audioSourceType, audioDataCallback: audioDataCallback) self.isAudioStarting = false self.isAudioStarted = success @@ -358,12 +345,14 @@ public class AzureAsrHelper: NSObject { * 执行音频启动操作 */ public func performAudioStart( + mod: String, audioSourceType: AudioSourceType, audioDataCallback: SimpleAudioReceiver.AudioDataCallback? ) -> Bool { do { if(!audioStream!.isContinuousRecognitionActive){ audioStream?.startAudioRecord( + mod: mod, audioSourceType: audioSourceType == .microphone ? .microphone : .external, audioDataCallback: audioDataCallback ) @@ -430,6 +419,7 @@ public class AzureAsrHelper: NSObject { * 若 stop 已发生则直接跳过 recognizer 的启动，避免“已停止但仍启动识别”的情况。 */ public func startContinuousRecognition( + mod: String = "normal", audioSourceType: AudioSourceType = .microphone, audioDataCallback: SimpleAudioReceiver.AudioDataCallback? = nil, isRemoveFirstPunctuation: Bool = true @@ -487,7 +477,7 @@ public class AzureAsrHelper: NSObject { } // 异步启动音频处理 - startAudioRecordAsync(audioSourceType: audioSourceType, audioDataCallback: proxyCallback) { [weak self] (success: Bool) in + startAudioRecordAsync(mod: mod, audioSourceType: audioSourceType, audioDataCallback: proxyCallback) { [weak self] (success: Bool) in guard let self = self else { return } if success { @@ -516,7 +506,7 @@ public class AzureAsrHelper: NSObject { // self.audioStream?.isContinuousRecognitionActive = true // 异步启动音频处理 - startAudioRecordAsync(audioSourceType: audioSourceType, audioDataCallback: audioDataCallback) { [weak self] (success: Bool) in + startAudioRecordAsync(mod: mod, audioSourceType: audioSourceType, audioDataCallback: audioDataCallback) { [weak self] (success: Bool) in guard let self = self else { return } if success { @@ -787,7 +777,7 @@ public class AzureAsrHelper: NSObject { if audioStream == nil { audioStream = SimpleAudioReceiver() audioStream?.initAudioRecord() // 确保调用初始化 - audioStream?.setPhoneCallMode(isPhoneCallMode) + } // 检查音频配置是否已存在 @@ -1012,7 +1002,7 @@ public class AzureAsrHelper: NSObject { // 修复：添加类型转换 if(!audioStream!.isContinuousRecognitionActive){ // 异步启动音频处理 - startAudioRecordAsync(audioSourceType: audioSourceType, audioDataCallback: audioDataCallback) { [weak self] (success: Bool) in + startAudioRecordAsync(mod: "normal", audioSourceType: audioSourceType, audioDataCallback: audioDataCallback) { [weak self] (success: Bool) in guard let self = self else { return } } diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index aee7d4930..181fc90c4 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -77,30 +77,12 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { private var pushStreamChunkCount = 0 private var pushStreamCallbackCount = 0 private var synthEventAudioCount = 0 - private var isExtaudioSource = false - private var isPhoneCallMode = false + public var isHeadphones = true + public var currentRecognitionMode: String = "normal" + - /** - * 初始化语音合成服务 - * - Parameters: - * - ttsAppId: 服务应用ID（Azure 场景未使用，可为空） - * - ttsAppToken: Azure 语音服务订阅密钥 - * - ttsResource: Azure 语音服务区域 - * - language: 语言代码，如 "zh-CN" - * - Returns: 是否初始化成功 - * - Throws: 无（内部捕获 SDK 异常并记录日志） - */ - public func initialize(ttsAppId: String, ttsAppToken: String, ttsResource: String, language: String) -> Bool { - return initialize( - ttsAppId: ttsAppId, - ttsAppToken: ttsAppToken, - ttsResource: ttsResource, - language: language, - isExtaudioSource: false - ) - } /** * 初始化语音合成服务（支持外部音频源场景） @@ -109,11 +91,10 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { * - ttsAppToken: Azure 语音服务订阅密钥 * - ttsResource: Azure 语音服务区域 * - language: 语言代码，如 "zh-CN" - * - isExtaudioSource: 是否为外部音频源（例如 BLE 输入） * - Returns: 是否初始化成功 * - Throws: 无（内部捕获 SDK 异常并记录日志） */ - public func initialize(ttsAppId: String, ttsAppToken: String, ttsResource: String, language: String, isExtaudioSource: Bool) -> Bool { + public func initialize(ttsAppId: String, ttsAppToken: String, ttsResource: String, language: String) -> Bool { do { micCapture = MicrophoneCapture.shared // 创建语音配置 @@ -131,7 +112,6 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { // 创建合成器 recreateSynthesizer() - self.isExtaudioSource = isExtaudioSource // 设置初始化完成 isInitialized = true @@ -433,22 +413,31 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { } } - public func setAudioSourceType(isExtaudioSource: Bool) -> Bool { - self.isExtaudioSource = isExtaudioSource + + public func setPhoneCallMode(mod: String) -> Bool { + self.currentRecognitionMode = mod + self.isHeadphones = detectHeadphonesOutput() if isInitialized { safeConfigureAudioSessionForTTS() } return true } - - public func setPhoneCallMode(enabled: Bool) -> Bool { - isPhoneCallMode = enabled - if isInitialized { - safeConfigureAudioSessionForTTS() + /** + * 检测是否存在耳机/蓝牙输出设备（用于决定是否启用A2DP而非扬声器） + * 参数：无 + * 返回值：存在返回true，否则返回false + * 异常：无 + */ + private func detectHeadphonesOutput() -> Bool { + let session = AVAudioSession.sharedInstance() + let outputs = session.currentRoute.outputs + return outputs.contains { out in + out.portType == .bluetoothA2DP || + out.portType == .bluetoothHFP || + out.portType == .bluetoothLE || + out.portType == .headphones } - return true } - /** * 设置音频输出设备 */ @@ -1376,15 +1365,15 @@ private func applyAudioSessionForTTS() -> Bool { desiredOptions.insert(.duckOthers) } else if isRecording { desiredCategory = .playAndRecord - if self.isPhoneCallMode { + if self.currentRecognitionMode == "phone_call" { desiredMode = .voiceChat desiredOptions.insert(.allowBluetooth) } else { - desiredMode = self.isExtaudioSource ? .default : .videoChat + desiredMode = self.isHeadphones ? .default : .videoChat desiredOptions.insert(.allowBluetooth) desiredOptions.insert(.allowBluetoothA2DP) } - if self.isExtaudioSource { + if self.isHeadphones { if otherAudioPlaying { desiredOptions.insert(.duckOthers) } @@ -1417,7 +1406,7 @@ private func applyAudioSessionForTTS() -> Bool { _ = try? audioSession.setPreferredIOBufferDuration(0.02) } try audioSession.setActive(true) - if self.isPhoneCallMode, desiredCategory == .playAndRecord, !hasHeadphones { + if self.currentRecognitionMode == "phone_call", desiredCategory == .playAndRecord, !hasHeadphones { _ = try? audioSession.overrideOutputAudioPort(.speaker) } break diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift index b5a0a8a5f..3944b0b0a 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift @@ -54,10 +54,8 @@ public class SimpleAudioReceiver: NSObject { private var currentRoute: AudioOutputRoute? //public var onAudioData: ((Data) -> Void)? public var recordfile: RecordFile? + private var currentRecognitionMode = "normal" // 当前识别模式：normal, ble_wakeup, phone_call, push_to_talk public var isHeadphones = true - public var isReceiverHFP = false // 是否是接收HFP音频 - public var isPhoneCallMode = false // AI电话模式：录音+播放强制走电话链路（voiceChat/HFP） - public var preferBuiltInMic = false // 耳机连接时优先使用手机内置麦克风录音 /** * 初始化 @@ -125,39 +123,118 @@ public class SimpleAudioReceiver: NSObject { /** * 开始音频输入 - * 参数：audioSourceType 音频源类型（microphone 或 external）；audioDataCallback 音频数据回调 - * 返回值：无 - * 异常：无（内部捕获并打印错误，不抛出） + * - Parameter mod: 识别模式，如 "normal", "phone_call", "push_to_talk", "ble_wakeup"。 + * - Parameter audioSourceType: 音频源类型（.microphone 或 .external）。 + * - Parameter audioDataCallback: 音频数据回调。 */ - public func startAudioRecord(audioSourceType: AudioSourceType = .microphone, audioDataCallback: AudioDataCallback?) { - _isWriting=true - self.audioSourceType = audioSourceType; - self.audioDataCallback = audioDataCallback - print("音源设制---------- startAudioRecord=audioSourceType:\(audioSourceType)") - switch audioSourceType { - case .microphone: - isReceiverHFP = isPhoneCallMode - // 动态检测是否存在耳机/蓝牙输出，用于后续路由选项计算 - isHeadphones = detectHeadphonesOutput() + public func startAudioRecord(mod: String = "normal", audioSourceType: AudioSourceType = .microphone, audioDataCallback: AudioDataCallback?) { + self.currentRecognitionMode = mod + self.audioSourceType = audioSourceType + self.audioDataCallback = audioDataCallback + self.isHeadphones = detectHeadphonesOutput() + + _isWriting = true + + print("音源设置---------- startAudioRecord=mod:\(mod), audioSourceType:\(audioSourceType), isHeadphones:\(isHeadphones)") + + do { + try configureAudioSession() + + let isHybridExternalMode = audioSourceType == .external && (currentRecognitionMode == "phone_call" || currentRecognitionMode == "push_to_talk") + + if audioSourceType == .microphone || isHybridExternalMode { runMicrophoneCapture() - case .external: - isReceiverHFP = false + } else { // True external mode runExternalCapture() } - } - - public func setPhoneCallMode(_ enabled: Bool) { - isPhoneCallMode = enabled + } catch { + print("配置或启动音频捕获失败: \(error.localizedDescription)") + } } /** - * 设置是否优先使用手机内置麦克风录音 - * @param enabled 是否启用 - * @return 无 - * @throws 无 + * 根据当前模式和设备状态配置音频会话 */ - public func setPreferBuiltInMic(_ enabled: Bool) { - preferBuiltInMic = enabled + private func configureAudioSession() throws { + let audioSession = self.audioSession + + var desiredCategory: AVAudioSession.Category + var desiredMode: AVAudioSession.Mode + var desiredOptions: AVAudioSession.CategoryOptions + + let isHybridExternalMode = audioSourceType == .external && (currentRecognitionMode == "phone_call" || currentRecognitionMode == "push_to_talk") + + if audioSourceType == .microphone || isHybridExternalMode { + // --- 麦克风输入逻辑 (包括混合模式) --- + desiredCategory = .playAndRecord + + if currentRecognitionMode == "phone_call" { + desiredMode = .voiceChat + desiredOptions = isHeadphones ? [.allowBluetooth] : [.allowBluetooth, .defaultToSpeaker] + } else { // normal, push_to_talk, ble_wakeup 等 + desiredMode = .videoChat + if isHeadphones { + desiredOptions = [.allowBluetooth, .allowBluetoothA2DP, .mixWithOthers] + } else { + desiredOptions = [.mixWithOthers, .defaultToSpeaker] + } + } + + // 如果在真实通话中，强制使用更适合的录音设置 + if isTelephonyActive() { + desiredCategory = .record + desiredMode = .voiceChat + desiredOptions = [.allowBluetooth] + } + + } else if audioSourceType == .external { + // --- 纯外部源 (仅播放) 逻辑 --- + desiredCategory = .playback + desiredMode = .videoChat + desiredOptions = isHeadphones ? [.allowBluetoothA2DP, .mixWithOthers] : [.mixWithOthers, .defaultToSpeaker] + } else { + // 兜底，理论上不会执行 + desiredCategory = .playAndRecord + desiredMode = .default + desiredOptions = [] + } + + // 仅在需要时更新会话，避免不必要的中断 + if audioSession.category != desiredCategory || audioSession.mode != desiredMode || audioSession.categoryOptions != desiredOptions { + try audioSession.setCategory(desiredCategory, mode: desiredMode, options: desiredOptions) + } + + // 激活会话 + try audioSession.setActive(true, options: .notifyOthersOnDeactivation) + + // --- 路由设置 --- + if audioSourceType == .microphone || isHybridExternalMode { + // 为需要麦克风的场景设置输入输出 + if isHeadphones { + try audioSession.overrideOutputAudioPort(.none) // 允许耳机/蓝牙输出 + } else { + try audioSession.overrideOutputAudioPort(.speaker) // 强制外放 + } + + // 设置首选输入设备 + if let inputs = audioSession.availableInputs { + if isHybridExternalMode { + // 混合模式: 强制使用内置麦克风 + if let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { + try audioSession.setPreferredInput(builtInMic) + print("混合外部模式: 首选输入已设置为内置麦克风") + } + } else if let btInput = inputs.first(where: { $0.portType == .bluetoothHFP }), isHeadphones { + // 标准麦克风模式: 优先使用蓝牙耳机麦克风 + try audioSession.setPreferredInput(btInput) + } else if let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { + // 兜底: 使用内置麦克风 + try audioSession.setPreferredInput(builtInMic) + } + } + } else { // 纯外部源 + try audioSession.overrideOutputAudioPort(.none) + } } /** @@ -255,9 +332,6 @@ public class SimpleAudioReceiver: NSObject { // 确保正在写入状态 guard self._isWriting else { return } - // 根据通话状态进行安全的音频会话配置 - try safeConfigureAudioSessionForMic() - // 启动麦克风采集 try micCapture.startCapture() @@ -282,154 +356,8 @@ public class SimpleAudioReceiver: NSObject { */ private func runExternalCapture() { if audioSourceType == .external { - do { - // 停止麦克风采集，确保不走通话音道 - micCapture.stopCapture() - // 外部模式会话配置：禁止HFP，仅允许A2DP/耳机输出 - try safeConfigureAudioSessionForExternal() - } catch { - print("设置音频会话失败: \(error.localizedDescription)") - } - //pushAudioData(data: Data()) - } - } - - /** - * 外部音频源的安全音频会话配置（禁止HFP通话音道） - * 参数：无 - * 返回值：无 - * 异常：可能抛出AVAudioSession设置错误（内部捕获并兜底） - */ - private func safeConfigureAudioSessionForExternal() throws { - let audioSession = self.audioSession - do { - // 外部模式不录麦，仅做播放路由；有耳机时只允许A2DP，禁止HFP - let outputs = audioSession.currentRoute.outputs - let hasHeadphones = outputs.contains { out in - out.portType == .bluetoothA2DP || - out.portType == .bluetoothLE || - out.portType == .headphones - } - let desiredCategory: AVAudioSession.Category = .playback - let desiredMode: AVAudioSession.Mode = .videoChat - let desiredOptions: AVAudioSession.CategoryOptions = hasHeadphones - ? [.allowBluetoothA2DP, .mixWithOthers] - : [.mixWithOthers, .defaultToSpeaker] - - let needsSetCategory = (audioSession.category != desiredCategory) || (audioSession.mode != desiredMode) - if needsSetCategory { - try audioSession.setCategory(desiredCategory, mode: desiredMode, options: desiredOptions) - } - // 取消扬声器强制，确保耳机/A2DP优先 - try audioSession.overrideOutputAudioPort(.none) - try audioSession.setActive(true, options: .notifyOthersOnDeactivation) - } catch { - print("外部模式音频会话配置失败，兜底到playback: \(error)") - try audioSession.setCategory(.playback, mode: .default, options: [.mixWithOthers, .defaultToSpeaker]) - try audioSession.setActive(true, options: .notifyOthersOnDeactivation) - } - } - /** - * 在通话/非通话场景下安全配置音频会话用于麦克风采集 - * - 通话中：使用 .record + .voiceChat + [.allowBluetooth]，避免 A2DP 和强制外放，兼容电话路由 - * - 非通话：使用 .playAndRecord + .videoChat + [.allowBluetooth, .mixWithOthers, .defaultToSpeaker] - * - 失败兜底：退回到最简单的 .record + .default，确保录音尽可能可用 - */ - private func safeConfigureAudioSessionForMic() throws { - let audioSession = self.audioSession - do { - // 计算目标配置 - let desiredCategory: AVAudioSession.Category - let desiredMode: AVAudioSession.Mode - let desiredOptions: AVAudioSession.CategoryOptions - if isPhoneCallMode { - desiredCategory = .playAndRecord - desiredMode = .voiceChat - if isHeadphones { - desiredOptions = [.allowBluetooth] - } else { - desiredOptions = [.allowBluetooth, .defaultToSpeaker] - } - } else if isReceiverHFP { - if isTelephonyActive() { - // 通话中：录音优先，避免 A2DP（仅播放），改用 HFP（.allowBluetooth） - desiredCategory = .record - desiredMode = .voiceChat - desiredOptions = [.allowBluetooth] - } else { - if(isHeadphones){ - // 非通话：保持原有逻辑，但使用 .allowBluetooth 以支持录音（而不是 A2DP） - desiredCategory = .playAndRecord - desiredMode = .videoChat - desiredOptions = [.allowBluetooth, .mixWithOthers] - //desiredOptions = [.mixWithOthers, .defaultToSpeaker] - }else{ - desiredCategory = .playAndRecord - desiredMode = .videoChat - desiredOptions = [.mixWithOthers, .defaultToSpeaker] - } - } - }else{ - if isTelephonyActive() { - // 通话中：录音优先，避免 A2DP（仅播放），改用 HFP（.allowBluetooth） - desiredCategory = .record - desiredMode = .voiceChat - desiredOptions = [.allowBluetooth] - } else { - if(isHeadphones){ - // 非通话：保持原有逻辑，但使用 .allowBluetooth 以支持录音（而不是 A2DP） - desiredCategory = .playAndRecord - desiredMode = .videoChat - desiredOptions = [.allowBluetooth, .allowBluetoothA2DP, .mixWithOthers] - //desiredOptions = [.mixWithOthers, .defaultToSpeaker] - }else{ - desiredCategory = .playAndRecord - desiredMode = .videoChat - desiredOptions = [.mixWithOthers, .defaultToSpeaker] - } - } - } - - - // 仅在类别或模式变化时才调用 setCategory，避免触发不必要的 categoryChange - let needsSetCategory = (audioSession.category != desiredCategory) || (audioSession.mode != desiredMode) - if needsSetCategory { - try audioSession.setCategory(desiredCategory, - mode: desiredMode, - options: desiredOptions) - } else { - // 类别与模式一致时，也尽量避免重复 setCategory；如需确保 options，可在确有必要时再设置 - // 如果你确实需要校正 options，可取消下一行注释，但这可能再次触发 categoryChange - // try audioSession.setCategory(desiredCategory, mode: desiredMode, options: desiredOptions) - } - - // 保持激活；这里重复 setActive(true) 一般安全，但如果你观察到重复激活也触发路由回调，可在外层控制频率 - try audioSession.setActive(true, options: .notifyOthersOnDeactivation) - - // 路由校正：有蓝牙输出时不强制外放，并优先选择蓝牙输入（HFP）或内置麦克风 - let outputs = audioSession.currentRoute.outputs - let hasBtOutput = outputs.contains { out in - out.portType == .bluetoothA2DP || out.portType == .bluetoothHFP || out.portType == .bluetoothLE - } - if hasBtOutput { - try audioSession.overrideOutputAudioPort(.none) - } else if isPhoneCallMode, !isHeadphones { - try audioSession.overrideOutputAudioPort(.speaker) - } - if let inputs = audioSession.availableInputs { - if preferBuiltInMic, let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { - try audioSession.setPreferredInput(builtInMic) - } else if isReceiverHFP, let btInput = inputs.first(where: { $0.portType == .bluetoothHFP }) { - try audioSession.setPreferredInput(btInput) - } else if let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { - try audioSession.setPreferredInput(builtInMic) - } - } - } catch { - // 兜底：尽量保证能开始录音 - print("音频会话配置失败，尝试兜底配置: \(error)") - try audioSession.setCategory(.record, mode: .default, options: []) - try audioSession.setActive(true, options: .notifyOthersOnDeactivation) + // 停止麦克风采集，确保不走通话音道 + micCapture.stopCapture() } } @@ -451,12 +379,12 @@ public class SimpleAudioReceiver: NSObject { * 继续麦克风捕获 */ public func resumeRecord() { - guard _isWriting==false else { return } + guard _isWriting==false else { return } _isWriting = true // 启动录音引擎 do { print("resumeRecord") - try safeConfigureAudioSessionForMic() + try configureAudioSession() try micCapture.startCapture() } catch { if let nsError = error as NSError? { From bb62534c6698c45e2fd9772176a7b038a9fde096 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Wed, 21 Jan 2026 22:39:51 +0800 Subject: [PATCH 11/25] =?UTF-8?q?fix(audio):=20=E4=BC=98=E5=8C=96=E8=80=B3?= =?UTF-8?q?=E6=9C=BA=E6=A8=A1=E5=BC=8F=E4=B8=8B=E7=9A=84=E9=9F=B3=E9=A2=91?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E5=92=8C=E9=BA=A6=E5=85=8B=E9=A3=8E=E9=80=89?= =?UTF-8?q?=E6=8B=A9=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修正蓝牙耳机模式下音频模式选择问题，增加 A2DP 支持 针对电话和按住说话模式强制使用内置麦克风 优化音频会话配置逻辑，提升不同场景下的音频体验 --- .../Sources/azure_speech/AzureTtsHelper.swift | 45 ++++++++++++------- .../Sources/tools/SimpleAudioReceiver.swift | 25 ++++++++--- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index 181fc90c4..89cba21d4 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -79,6 +79,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { private var synthEventAudioCount = 0 public var isHeadphones = true public var currentRecognitionMode: String = "normal" + private var previousRecognitionMode: String = "normal" @@ -415,6 +416,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { public func setPhoneCallMode(mod: String) -> Bool { + self.previousRecognitionMode = self.currentRecognitionMode self.currentRecognitionMode = mod self.isHeadphones = detectHeadphonesOutput() if isInitialized { @@ -1359,41 +1361,50 @@ private func applyAudioSessionForTTS() -> Bool { let desiredMode: AVAudioSession.Mode var desiredOptions: AVAudioSession.CategoryOptions = [] - if self.isTelephonyActive() { + let isHeadphonesConnected = outputs.contains(where: { $0.portType == .headphones || $0.portType == .headsetMic || $0.portType == .bluetoothA2DP }) + + if self.currentRecognitionMode == "phone_call" || self.currentRecognitionMode == "push_to_talk" { + desiredCategory = .playAndRecord + if isHeadphonesConnected { + desiredMode = .voiceChat + desiredOptions = [.allowBluetooth, .duckOthers] + } else { + desiredMode = .videoChat + desiredOptions = [.allowBluetooth, .defaultToSpeaker] + } + } else if self.isTelephonyActive() { desiredCategory = .playback desiredMode = .default desiredOptions.insert(.duckOthers) } else if isRecording { desiredCategory = .playAndRecord - if self.currentRecognitionMode == "phone_call" { - desiredMode = .voiceChat - desiredOptions.insert(.allowBluetooth) - } else { - desiredMode = self.isHeadphones ? .default : .videoChat + if self.isHeadphones { + desiredMode = .default desiredOptions.insert(.allowBluetooth) desiredOptions.insert(.allowBluetoothA2DP) - } - if self.isHeadphones { if otherAudioPlaying { desiredOptions.insert(.duckOthers) } } else { + desiredMode = .videoChat + desiredOptions.insert(.allowBluetooth) desiredOptions.insert(.mixWithOthers) - if !hasHeadphones { - desiredOptions.insert(.defaultToSpeaker) - } + desiredOptions.insert(.defaultToSpeaker) } } else if hasBluetoothHFP { desiredCategory = .playback - desiredMode = .default - desiredOptions.insert(.duckOthers) - } else { + desiredMode = .voicePrompt + desiredOptions = [.interruptSpokenAudioAndMixWithOthers, .allowBluetooth] + } else if hasHeadphones { desiredCategory = .playback - desiredMode = .spokenAudio + desiredMode = .default if otherAudioPlaying { desiredOptions.insert(.duckOthers) - desiredOptions.insert(.interruptSpokenAudioAndMixWithOthers) } + } else { + desiredCategory = .playback + desiredMode = .default + desiredOptions = [.duckOthers] } do { @@ -1406,7 +1417,7 @@ private func applyAudioSessionForTTS() -> Bool { _ = try? audioSession.setPreferredIOBufferDuration(0.02) } try audioSession.setActive(true) - if self.currentRecognitionMode == "phone_call", desiredCategory == .playAndRecord, !hasHeadphones { + if self.currentRecognitionMode == "phone_call", !self.isHeadphones { _ = try? audioSession.overrideOutputAudioPort(.speaker) } break diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift index 3944b0b0a..a2488a30c 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift @@ -170,12 +170,19 @@ public class SimpleAudioReceiver: NSObject { if currentRecognitionMode == "phone_call" { desiredMode = .voiceChat - desiredOptions = isHeadphones ? [.allowBluetooth] : [.allowBluetooth, .defaultToSpeaker] + if isHeadphones { + // 修正：增加 A2DP 选项以支持高质量蓝牙音频播放 + desiredOptions = [.allowBluetooth, .allowBluetoothA2DP] + } else { + desiredOptions = [.allowBluetooth, .defaultToSpeaker] + } } else { // normal, push_to_talk, ble_wakeup 等 - desiredMode = .videoChat if isHeadphones { + // 修正：使用 .voiceChat 模式以避免 .videoChat 强制外放 + desiredMode = .voiceChat desiredOptions = [.allowBluetooth, .allowBluetoothA2DP, .mixWithOthers] } else { + desiredMode = .videoChat desiredOptions = [.mixWithOthers, .defaultToSpeaker] } } @@ -218,18 +225,26 @@ public class SimpleAudioReceiver: NSObject { // 设置首选输入设备 if let inputs = audioSession.availableInputs { - if isHybridExternalMode { + // 针对电话和按住说话模式，在连接耳机时强制使用内置麦克风 + if isHeadphones && (currentRecognitionMode == "phone_call" || currentRecognitionMode == "push_to_talk") { + if let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { + try audioSession.setPreferredInput(builtInMic) + print("耳机模式下，为 \(currentRecognitionMode) 模式强制设置内置麦克风") + } + } else if isHybridExternalMode { // 混合模式: 强制使用内置麦克风 if let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { try audioSession.setPreferredInput(builtInMic) print("混合外部模式: 首选输入已设置为内置麦克风") } } else if let btInput = inputs.first(where: { $0.portType == .bluetoothHFP }), isHeadphones { - // 标准麦克风模式: 优先使用蓝牙耳机麦克风 + // 其他耳机模式: 优先使用蓝牙耳机麦克风 try audioSession.setPreferredInput(btInput) + print("标准耳机模式: 首选输入已设置为蓝牙麦克风") } else if let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { - // 兜底: 使用内置麦克风 + // 兜底或无耳机: 使用内置麦克风 try audioSession.setPreferredInput(builtInMic) + print("兜底: 首选输入已设置为内置麦克风") } } } else { // 纯外部源 From f7739491046084caef680188877cafef2149dad7 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Thu, 22 Jan 2026 15:16:35 +0800 Subject: [PATCH 12/25] =?UTF-8?q?fix(ble=5Fservice):=20=E6=B7=BB=E5=8A=A0B?= =?UTF-8?q?LE=E9=80=9A=E7=9F=A5=E7=89=B9=E5=BE=81=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=9A=84=E6=97=A5=E5=BF=97=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit refactor(agent_service): 调整音频会话检查逻辑并添加日志 --- .../Sources/agent_service/AgentServiceImpl.swift | 6 ++++-- .../kotlin/com/yunqiinnovation/ble_service/BleService.kt | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 0f35547e7..6cd0dcea6 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -279,6 +279,8 @@ class AgentServiceImpl: NSObject { self.xunfeiAccessKeySecret = xunfeiAccessKeySecret } + + //启动定位服务 NotificationCenter.default.post(name: .locationstartEvent, object: nil) // addlocationmonitor() @@ -311,9 +313,8 @@ class AgentServiceImpl: NSObject { LocationService.shared.getCurrentLocation() { location in // self.locationInfo = location } - + currentRecognitionMode = "normal" checkAdnSetAudioSession() - return isInitialized } @@ -390,6 +391,7 @@ class AgentServiceImpl: NSObject { } private func checkAdnSetAudioSession() { + os_log("liwei------ 检查并设置音频会话", log: logger, type: .info) let audioSourceType: AzureAsrHelper.AudioSourceType = BleService.shared.isConnected() ? .external : .microphone azureAsrHelper?.performAudioStart(mod: currentRecognitionMode, audioSourceType: audioSourceType, audioDataCallback: nil) } diff --git a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt index b42b08c0a..6ed5a084c 100644 --- a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt +++ b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt @@ -953,7 +953,7 @@ object BleService { override fun onCharacteristicChanged(g: BluetoothGatt, c: BluetoothGattCharacteristic) { val data = c.value ?: return - + Log.i(TAG, "ble指令----------- 收到通知特征数据（命令和控制 uuid=${c.uuid}") // 根据特征UUID区分处理 when (c.uuid) { // 音频特征数据 @@ -973,7 +973,7 @@ object BleService { } // 通知特征数据（命令和控制） BleConst.NOTIFY_CHAR_UUID -> { - Log.i(TAG, "ble指令----------- 收到通知特征数据（命令和控制）") + // Log.i(TAG, "ble指令----------- 收到通知特征数据（命令和控制）") // 判断数据类型 when { // 设备响应 (0xBB开头) From cae3352a423954dc90357ba762a53ff3311f4506 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Thu, 22 Jan 2026 17:50:07 +0800 Subject: [PATCH 13/25] =?UTF-8?q?fix(agent=5Fservice):=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E9=80=9A=E8=AF=9D=E6=A8=A1=E5=BC=8F=E7=BB=93=E6=9D=9F?= =?UTF-8?q?=E5=90=8E=E9=9F=B3=E9=A2=91=E4=BC=9A=E8=AF=9D=E6=9C=AA=E9=87=8D?= =?UTF-8?q?=E7=BD=AE=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix(azure_speech): 适配iOS 18以下版本的音频会话配置和播放逻辑 修复iOS 17及以下版本流式播放可能卡住的问题 refactor(chat_storage): 使用串行队列保护数据库访问 确保所有数据库操作线程安全，避免并发问题 --- .../agent_service/AgentServiceImpl.swift | 28 +- .../Sources/azure_speech/AzureTtsHelper.swift | 125 +++- .../chat_storage/ChatStorageHelper.swift | 550 +++++++++--------- 3 files changed, 387 insertions(+), 316 deletions(-) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 6cd0dcea6..0196a1d46 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -576,29 +576,43 @@ class AgentServiceImpl: NSObject { return true } + // 关键修复：如果正在结束的是通话模式，则立即将会话重置为 normal 状态 + if self.currentRecognitionMode == "phone_call" { + os_log("通话模式结束，强制重置音频会话至 'normal' 状态", log: logger, type: .info) + self.currentRecognitionMode = "normal" + // 使用 normal 模式的配置来清理和重置音频会话 + self.checkAdnSetAudioSession() + } + os_log("停止语音识别，当前模式: %{public}@", log: logger, type: .info, currentRecognitionMode) let success = azureAsrHelper?.stopContinuousRecognition() ?? false - - if !success && isStartingRecognition { + if success { + // 识别结束后，立即停用 AudioSession，以便其他模块（如TTS）可以接管 + // 这是解决“按住说话”后无法播报问题的关键 + do { + try AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + os_log("Audio session deactivated successfully after recognition.", log: logger, type: .info) + } catch { + os_log("Failed to deactivate audio session: %{public}@", log: logger, type: .error, error.localizedDescription) + } + } else if isStartingRecognition { // 启动尚未完成，先记录一次待停止请求，onSessionStarted 到来后立即 stop stopRequestedDuringStart = true + // 即使底层 stop 失败，但因为我们已经记录了停止请求，所以对调用方来说，可以认为是成功的 + // 后续的 onSessionStarted 会处理真正的停止逻辑 } stopIdleCheck() // 只有之前在播放时才恢复播放 if wasMusicPlayingBeforeRecognition { - // QPlayAutoManager.shared.play() wasMusicPlayingBeforeRecognition = false MusicService.shared.resume() } - // 重置识别模式 - // currentRecognitionMode = "normal" - // 启动中但已请求停止，也视为已接受停止请求 - return success || isStartingRecognition + return success || stopRequestedDuringStart } func disableBluetoothAudio() { azureAsrHelper?.disableBluetoothAudio() diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index 89cba21d4..cd66fb76d 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -848,33 +848,70 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { */ private func scheduleAvailablePcmBuffers() { guard ensureStreamingEngineIfNeeded() else { return } - if !hasNotifiedPlaybackStartedForStream, pcmPendingData.count >= streamChunkBytes { - // os_log("调用链: scheduleAvailablePcmBuffers 准备调度 pendingPcm=%{public}d chunk=%{public}d active=%{public}d", log: log, type: .info, pcmPendingData.count, streamChunkBytes, activeStreamSynthesisCount) - } - while pcmPendingData.count >= streamChunkBytes { - let chunk = pcmPendingData.prefix(streamChunkBytes) - pcmPendingData.removeFirst(streamChunkBytes) - guard let buffer = makePcmBuffer(from: Data(chunk)) else { continue } - scheduledBufferCount += 1 - if !hasNotifiedPlaybackStartedForStream { - hasNotifiedPlaybackStartedForStream = true - // os_log("调用链: 流式播放开始 session=%{public}@", log: log, type: .info, sessionid) - notifyEvent(eventType: .playbackStarted) + + if #available(iOS 18.0, *) { + // iOS 18及以上版本，使用原始逻辑：先调度，后播放 + if !hasNotifiedPlaybackStartedForStream, pcmPendingData.count >= streamChunkBytes { + // os_log("调用链: scheduleAvailablePcmBuffers 准备调度 pendingPcm=%{public}d chunk=%{public}d active=%{public}d", log: log, type: .info, pcmPendingData.count, streamChunkBytes, activeStreamSynthesisCount) } - playerNode?.scheduleBuffer(buffer, completionHandler: { [weak self] in - guard let self = self else { return } - self.audioPlaybackQueue.async { - self.scheduledBufferCount = max(0, self.scheduledBufferCount - 1) - self.checkStreamPlaybackCompletedIfNeeded() + while pcmPendingData.count >= streamChunkBytes { + let chunk = pcmPendingData.prefix(streamChunkBytes) + pcmPendingData.removeFirst(streamChunkBytes) + guard let buffer = makePcmBuffer(from: Data(chunk)) else { continue } + scheduledBufferCount += 1 + if !hasNotifiedPlaybackStartedForStream { + hasNotifiedPlaybackStartedForStream = true + // os_log("调用链: 流式播放开始 session=%{public}@", log: log, type: .info, sessionid) + notifyEvent(eventType: .playbackStarted) } - }) - } - if activeStreamSynthesisCount == 0, !pcmPendingData.isEmpty { - if pcmPendingData.count % 2 != 0 { - pcmPendingData.removeLast() + playerNode?.scheduleBuffer(buffer, completionHandler: { [weak self] in + guard let self = self else { return } + self.audioPlaybackQueue.async { + self.scheduledBufferCount = max(0, self.scheduledBufferCount - 1) + self.checkStreamPlaybackCompletedIfNeeded() + } + }) } - if !pcmPendingData.isEmpty, let buffer = makePcmBuffer(from: pcmPendingData) { - pcmPendingData.removeAll(keepingCapacity: true) + if activeStreamSynthesisCount == 0, !pcmPendingData.isEmpty { + if pcmPendingData.count % 2 != 0 { + pcmPendingData.removeLast() + } + if !pcmPendingData.isEmpty, let buffer = makePcmBuffer(from: pcmPendingData) { + pcmPendingData.removeAll(keepingCapacity: true) + scheduledBufferCount += 1 + if !hasNotifiedPlaybackStartedForStream { + hasNotifiedPlaybackStartedForStream = true + // os_log("调用链: 流式播放开始 session=%{public}@", log: log, type: .info, sessionid) + notifyEvent(eventType: .playbackStarted) + } + playerNode?.scheduleBuffer(buffer, completionHandler: { [weak self] in + guard let self = self else { return } + self.audioPlaybackQueue.async { + self.scheduledBufferCount = max(0, self.scheduledBufferCount - 1) + self.checkStreamPlaybackCompletedIfNeeded() + } + }) + } + } + if playerNode?.isPlaying == false, scheduledBufferCount > 0 { + // os_log("调用链: playerNode.play scheduled=%{public}d pendingPcm=%{public}d", log: log, type: .debug, scheduledBufferCount, pcmPendingData.count) + playerNode?.play() + } + } else { + // iOS 17及以下版本，使用修复逻辑：先播放，后调度 + if playerNode?.isPlaying == false { + // os_log("调用链: playerNode.play 提前启动播放器", log: log, type: .debug) + playerNode?.play() + } + + if !hasNotifiedPlaybackStartedForStream, pcmPendingData.count >= streamChunkBytes { + // os_log("调用链: scheduleAvailablePcmBuffers 准备调度 pendingPcm=%{public}d chunk=%{public}d active=%{public}d", log: log, type: .info, pcmPendingData.count, streamChunkBytes, activeStreamSynthesisCount) + } + + while pcmPendingData.count >= streamChunkBytes { + let chunk = pcmPendingData.prefix(streamChunkBytes) + pcmPendingData.removeFirst(streamChunkBytes) + guard let buffer = makePcmBuffer(from: Data(chunk)) else { continue } scheduledBufferCount += 1 if !hasNotifiedPlaybackStartedForStream { hasNotifiedPlaybackStartedForStream = true @@ -889,10 +926,27 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { } }) } - } - if playerNode?.isPlaying == false, scheduledBufferCount > 0 { - // os_log("调用链: playerNode.play scheduled=%{public}d pendingPcm=%{public}d", log: log, type: .debug, scheduledBufferCount, pcmPendingData.count) - playerNode?.play() + if activeStreamSynthesisCount == 0, !pcmPendingData.isEmpty { + if pcmPendingData.count % 2 != 0 { + pcmPendingData.removeLast() + } + if !pcmPendingData.isEmpty, let buffer = makePcmBuffer(from: pcmPendingData) { + pcmPendingData.removeAll(keepingCapacity: true) + scheduledBufferCount += 1 + if !hasNotifiedPlaybackStartedForStream { + hasNotifiedPlaybackStartedForStream = true + // os_log("调用链: 流式播放开始 session=%{public}@", log: log, type: .info, sessionid) + notifyEvent(eventType: .playbackStarted) + } + playerNode?.scheduleBuffer(buffer, completionHandler: { [weak self] in + guard let self = self else { return } + self.audioPlaybackQueue.async { + self.scheduledBufferCount = max(0, self.scheduledBufferCount - 1) + self.checkStreamPlaybackCompletedIfNeeded() + } + }) + } + } } } @@ -1367,7 +1421,11 @@ private func applyAudioSessionForTTS() -> Bool { desiredCategory = .playAndRecord if isHeadphonesConnected { desiredMode = .voiceChat - desiredOptions = [.allowBluetooth, .duckOthers] + if #available(iOS 18.0, *) { + desiredOptions = [.allowBluetooth, .duckOthers] + } else { + desiredOptions = [.allowBluetooth, .allowBluetoothA2DP] + } } else { desiredMode = .videoChat desiredOptions = [.allowBluetooth, .defaultToSpeaker] @@ -1404,7 +1462,14 @@ private func applyAudioSessionForTTS() -> Bool { } else { desiredCategory = .playback desiredMode = .default - desiredOptions = [.duckOthers] + if #available(iOS 18.0, *) { + desiredOptions = [] + } else { + desiredOptions = [.defaultToSpeaker] + } + if otherAudioPlaying { + desiredOptions.insert(.duckOthers) + } } do { diff --git a/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift b/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift index 9591dfe11..6fdd60df2 100644 --- a/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift +++ b/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift @@ -15,6 +15,8 @@ public class ChatStorageHelper { private let dbPath: String private let logger = OSLog(subsystem: "com.yunqiinnovation.chat_storage", category: "ChatStorageHelper") + private let dbQueue = DispatchQueue(label: "com.yunqiinnovation.chat_storage.dbQueue") + private init() { // 获取文档目录路径（不变） let fileURL = try! FileManager.default @@ -23,39 +25,43 @@ public class ChatStorageHelper { dbPath = fileURL.path - // 打开数据库（不变） - if sqlite3_open(dbPath, &db) != SQLITE_OK { - let errmsg = String(cString: sqlite3_errmsg(db)!) - os_log("无法打开数据库: %{public}@", log: logger, type: .error, errmsg) - return - } - - // 关键修改：新增 UNIQUE (agent_id, session_id, sender) 联合唯一索引 - let createTableString = """ - CREATE TABLE IF NOT EXISTS messages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - agent_id TEXT NOT NULL, - session_id TEXT NOT NULL, - message TEXT NOT NULL, - sender TEXT NOT NULL, - metadata TEXT, - created_at INTEGER NOT NULL, - UNIQUE (agent_id, session_id, sender) ON CONFLICT REPLACE - ); - CREATE INDEX IF NOT EXISTS idx_agent_id ON messages (agent_id); - CREATE INDEX IF NOT EXISTS idx_session_id ON messages (session_id); - CREATE INDEX IF NOT EXISTS idx_created_at ON messages (created_at); - """ - - if sqlite3_exec(db, createTableString, nil, nil, nil) != SQLITE_OK { - let errmsg = String(cString: sqlite3_errmsg(db)!) - os_log("创建表失败: %{public}@", log: logger, type: .error, errmsg) + dbQueue.sync { + // 打开数据库（不变） + if sqlite3_open(dbPath, &db) != SQLITE_OK { + let errmsg = String(cString: sqlite3_errmsg(db)!) + os_log("无法打开数据库: %{public}@", log: logger, type: .error, errmsg) + return + } + + // 关键修改：新增 UNIQUE (agent_id, session_id, sender) 联合唯一索引 + let createTableString = """ + CREATE TABLE IF NOT EXISTS messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_id TEXT NOT NULL, + session_id TEXT NOT NULL, + message TEXT NOT NULL, + sender TEXT NOT NULL, + metadata TEXT, + created_at INTEGER NOT NULL, + UNIQUE (agent_id, session_id, sender) ON CONFLICT REPLACE + ); + CREATE INDEX IF NOT EXISTS idx_agent_id ON messages (agent_id); + CREATE INDEX IF NOT EXISTS idx_session_id ON messages (session_id); + CREATE INDEX IF NOT EXISTS idx_created_at ON messages (created_at); + """ + + if sqlite3_exec(db, createTableString, nil, nil, nil) != SQLITE_OK { + let errmsg = String(cString: sqlite3_errmsg(db)!) + os_log("创建表失败: %{public}@", log: logger, type: .error, errmsg) + } } } deinit { - if db != nil { - sqlite3_close(db) + dbQueue.sync { + if db != nil { + sqlite3_close(db) + } } } @@ -68,48 +74,48 @@ public class ChatStorageHelper { * @return 插入的消息ID，失败则返回-1 */ public func saveMessage(agentId: String, sessionId: String, message: String, sender: String, metadata: String?) -> Int64 { - // 关键修改：用 INSERT OR REPLACE 替换 INSERT，支持冲突时更新 - let insertStatementString = """ - INSERT OR REPLACE INTO messages - (agent_id, session_id, message, sender, metadata, created_at) - VALUES (?, ?, ?, ?, ?, ?); - """ - var insertStatement: OpaquePointer? - - if sqlite3_prepare_v2(db, insertStatementString, -1, &insertStatement, nil) == SQLITE_OK { - // 绑定参数（逻辑不变，确保三个唯一字段正确传入） - sqlite3_bind_text(insertStatement, 1, (agentId as NSString).utf8String, -1, nil) - sqlite3_bind_text(insertStatement, 2, (sessionId as NSString).utf8String, -1, nil) - sqlite3_bind_text(insertStatement, 3, (message as NSString).utf8String, -1, nil) - sqlite3_bind_text(insertStatement, 4, (sender as NSString).utf8String, -1, nil) - - if let metadata = metadata { - sqlite3_bind_text(insertStatement, 5, (metadata as NSString).utf8String, -1, nil) - } else { - sqlite3_bind_null(insertStatement, 5) - } + var result: Int64 = -1 + dbQueue.sync { + // 关键修改：用 INSERT OR REPLACE 替换 INSERT，支持冲突时更新 + let insertStatementString = """ + INSERT OR REPLACE INTO messages + (agent_id, session_id, message, sender, metadata, created_at) + VALUES (?, ?, ?, ?, ?, ?); + """ + var insertStatement: OpaquePointer? - let currentTime = Int(Date().timeIntervalSince1970) - sqlite3_bind_int(insertStatement, 6, Int32(currentTime)) - - // 执行语句（冲突时会自动替换，返回新的 rowid） - if sqlite3_step(insertStatement) == SQLITE_DONE { - let id = sqlite3_last_insert_rowid(db) // 替换后返回新的 id（原 id 会被删除） + if sqlite3_prepare_v2(db, insertStatementString, -1, &insertStatement, nil) == SQLITE_OK { + // 绑定参数（逻辑不变，确保三个唯一字段正确传入） + sqlite3_bind_text(insertStatement, 1, (agentId as NSString).utf8String, -1, nil) + sqlite3_bind_text(insertStatement, 2, (sessionId as NSString).utf8String, -1, nil) + sqlite3_bind_text(insertStatement, 3, (message as NSString).utf8String, -1, nil) + sqlite3_bind_text(insertStatement, 4, (sender as NSString).utf8String, -1, nil) + + if let metadata = metadata { + sqlite3_bind_text(insertStatement, 5, (metadata as NSString).utf8String, -1, nil) + } else { + sqlite3_bind_null(insertStatement, 5) + } + + let currentTime = Int(Date().timeIntervalSince1970) + sqlite3_bind_int(insertStatement, 6, Int32(currentTime)) + + // 执行语句（冲突时会自动替换，返回新的 rowid） + if sqlite3_step(insertStatement) == SQLITE_DONE { + let id = sqlite3_last_insert_rowid(db) // 替换后返回新的 id（原 id 会被删除） + os_log("消息保存成功（新增/更新），id: %{public}lld", log: logger, type: .info, id) + result = id + } else { + let errmsg = String(cString: sqlite3_errmsg(db)!) + os_log("插入/更新消息失败: %{public}@", log: logger, type: .error, errmsg) + } sqlite3_finalize(insertStatement) - os_log("消息保存成功（新增/更新），id: %{public}lld", log: logger, type: .info, id) - return id } else { let errmsg = String(cString: sqlite3_errmsg(db)!) - os_log("插入/更新消息失败: %{public}@", log: logger, type: .error, errmsg) + os_log("插入/更新消息语句准备失败: %{public}@", log: logger, type: .error, errmsg) } - - sqlite3_finalize(insertStatement) - } else { - let errmsg = String(cString: sqlite3_errmsg(db)!) - os_log("插入/更新消息语句准备失败: %{public}@", log: logger, type: .error, errmsg) } - - return -1 + return result } /** @@ -120,122 +126,106 @@ public class ChatStorageHelper { * @return 消息列表的JSON字符串 */ public func getMessages(agentId: String, page: Int, pageSize: Int) -> String { - let offset = (page - 1) * pageSize - var messagesArray: [[String: Any]] = [] - - // 首先获取总记录数 - let countQuery = "SELECT COUNT(*) FROM messages WHERE agent_id = ?" - var countStatement: OpaquePointer? - var totalCount = 0 - - if sqlite3_prepare_v2(db, countQuery, -1, &countStatement, nil) == SQLITE_OK { - sqlite3_bind_text(countStatement, 1, (agentId as NSString).utf8String, -1, nil) - if sqlite3_step(countStatement) == SQLITE_ROW { - totalCount = Int(sqlite3_column_int(countStatement, 0)) + var result = "{\"messages\":[],\"page\":\(page),\"pageSize\":\(pageSize),\"totalCount\":0,\"totalPages\":0}" + dbQueue.sync { + let offset = (page - 1) * pageSize + var messagesArray: [[String: Any]] = [] + + // 首先获取总记录数 + let countQuery = "SELECT COUNT(*) FROM messages WHERE agent_id = ?" + var countStatement: OpaquePointer? + var totalCount = 0 + + if sqlite3_prepare_v2(db, countQuery, -1, &countStatement, nil) == SQLITE_OK { + sqlite3_bind_text(countStatement, 1, (agentId as NSString).utf8String, -1, nil) + if sqlite3_step(countStatement) == SQLITE_ROW { + totalCount = Int(sqlite3_column_int(countStatement, 0)) + } + sqlite3_finalize(countStatement) } - sqlite3_finalize(countStatement) - } - - // 计算总页数 - let totalPages = totalCount == 0 ? 0 : (totalCount + pageSize - 1) / pageSize - - let queryString = """ - SELECT id, session_id, message, sender, metadata, created_at - FROM messages - WHERE agent_id = ? - ORDER BY created_at DESC - LIMIT ? OFFSET ? - """ - - var queryStatement: OpaquePointer? - - if sqlite3_prepare_v2(db, queryString, -1, &queryStatement, nil) == SQLITE_OK { - sqlite3_bind_text(queryStatement, 1, (agentId as NSString).utf8String, -1, nil) - sqlite3_bind_int(queryStatement, 2, Int32(pageSize)) - sqlite3_bind_int(queryStatement, 3, Int32(offset)) - while sqlite3_step(queryStatement) == SQLITE_ROW { - let id = sqlite3_column_int(queryStatement, 0) - - let sessionIdPtr = sqlite3_column_text(queryStatement, 1) - let sessionId = sessionIdPtr != nil ? String(cString: sessionIdPtr!) : "" - - let messagePtr = sqlite3_column_text(queryStatement, 2) - let message = messagePtr != nil ? String(cString: messagePtr!) : "" - - let senderPtr = sqlite3_column_text(queryStatement, 3) - let sender = senderPtr != nil ? String(cString: senderPtr!) : "" - - let metadataPtr = sqlite3_column_text(queryStatement, 4) - let metadata = metadataPtr != nil ? String(cString: metadataPtr!) : nil + // 计算总页数 + let totalPages = totalCount == 0 ? 0 : (totalCount + pageSize - 1) / pageSize + + let queryString = """ + SELECT id, session_id, message, sender, metadata, created_at + FROM messages + WHERE agent_id = ? + ORDER BY created_at DESC + LIMIT ? OFFSET ? + """ + + var queryStatement: OpaquePointer? + + if sqlite3_prepare_v2(db, queryString, -1, &queryStatement, nil) == SQLITE_OK { + sqlite3_bind_text(queryStatement, 1, (agentId as NSString).utf8String, -1, nil) + sqlite3_bind_int(queryStatement, 2, Int32(pageSize)) + sqlite3_bind_int(queryStatement, 3, Int32(offset)) - let createdAt = sqlite3_column_int(queryStatement, 5) + while sqlite3_step(queryStatement) == SQLITE_ROW { + let id = sqlite3_column_int(queryStatement, 0) + + let sessionIdPtr = sqlite3_column_text(queryStatement, 1) + let sessionId = sessionIdPtr != nil ? String(cString: sessionIdPtr!) : "" + + let messagePtr = sqlite3_column_text(queryStatement, 2) + let message = messagePtr != nil ? String(cString: messagePtr!) : "" + + let senderPtr = sqlite3_column_text(queryStatement, 3) + let sender = senderPtr != nil ? String(cString: senderPtr!) : "" + + let metadataPtr = sqlite3_column_text(queryStatement, 4) + let metadata = metadataPtr != nil ? String(cString: metadataPtr!) : nil + + let createdAt = sqlite3_column_int(queryStatement, 5) + + // 将时间戳转换为ISO 8601格式的字符串 + let date = Date(timeIntervalSince1970: TimeInterval(createdAt)) + let formatter = ISO8601DateFormatter() + let timestamp = formatter.string(from: date) + os_log("getMessages: 读取历史记录: %{public}@", log: logger, type: .info, timestamp) + + var messageDict: [String: Any] = [ + "id": id, + "agentId": agentId, // 添加agentId字段 + "sessionId": sessionId, // 添加sessionId字段 + "message": message, + "sender": sender, + "timestamp": timestamp // 使用timestamp而不是created_at + ] + + if let metadata = metadata { + messageDict["metadata"] = metadata + } + + messagesArray.append(messageDict) + } - // 将时间戳转换为ISO 8601格式的字符串 - let date = Date(timeIntervalSince1970: TimeInterval(createdAt)) - let formatter = ISO8601DateFormatter() - let timestamp = formatter.string(from: date) - os_log("getMessages: 读取历史记录: %{public}@", log: logger, type: .info, timestamp) + sqlite3_finalize(queryStatement) - var messageDict: [String: Any] = [ - "id": id, - "agentId": agentId, // 添加agentId字段 - "sessionId": sessionId, // 添加sessionId字段 - "message": message, - "sender": sender, - "timestamp": timestamp // 使用timestamp而不是created_at + // 构建与Android版本一致的返回格式 + let resultDict: [String: Any] = [ + "messages": messagesArray, + "page": page, + "pageSize": pageSize, + "totalCount": totalCount, + "totalPages": totalPages ] - - if let metadata = metadata { - messageDict["metadata"] = metadata - } - - messagesArray.append(messageDict) - } - - sqlite3_finalize(queryStatement) - - // 构建与Android版本一致的返回格式 - let result: [String: Any] = [ - "messages": messagesArray, - "page": page, - "pageSize": pageSize, - "totalCount": totalCount, - "totalPages": totalPages - ] - os_log("getMessages: 读取历史记录: %{public}@", log: logger, type: .error, messagesArray) - do { - let jsonData = try JSONSerialization.data(withJSONObject: result, options: []) - if let jsonString = String(data: jsonData, encoding: .utf8) { - return jsonString + os_log("getMessages: 读取历史记录: %{public}@", log: logger, type: .error, messagesArray) + do { + let jsonData = try JSONSerialization.data(withJSONObject: resultDict, options: []) + if let jsonString = String(data: jsonData, encoding: .utf8) { + result = jsonString + } + } catch { + os_log("getMessages: JSON转换失败: %{public}@", log: logger, type: .error, error.localizedDescription) } - } catch { - os_log("getMessages: JSON转换失败: %{public}@", log: logger, type: .error, error.localizedDescription) - } - } else { - let errmsg = String(cString: sqlite3_errmsg(db)!) - os_log("getMessages: SQL准备失败: %{public}@", log: logger, type: .error, errmsg) - } - - // 返回空结果，但保持格式一致 - let emptyResult: [String: Any] = [ - "messages": [], - "page": page, - "pageSize": pageSize, - "totalCount": 0, - "totalPages": 0 - ] - - do { - let jsonData = try JSONSerialization.data(withJSONObject: emptyResult, options: []) - if let jsonString = String(data: jsonData, encoding: .utf8) { - return jsonString + } else { + let errmsg = String(cString: sqlite3_errmsg(db)!) + os_log("getMessages: SQL准备失败: %{public}@", log: logger, type: .error, errmsg) } - } catch { - os_log("getMessages: 空结果JSON转换失败: %{public}@", log: logger, type: .error, error.localizedDescription) } - - return "{\"messages\":[],\"page\":\(page),\"pageSize\":\(pageSize),\"totalCount\":0,\"totalPages\":0}" + return result } /** @@ -245,56 +235,55 @@ public class ChatStorageHelper { * @return 是否删除成功 */ public func deleteMessages(agentId: String?, messageIds: [Int]?) -> Bool { - if let agentId = agentId { - let deleteString = "DELETE FROM messages WHERE agent_id = ?;" - var deleteStatement: OpaquePointer? - - if sqlite3_prepare_v2(db, deleteString, -1, &deleteStatement, nil) == SQLITE_OK { - sqlite3_bind_text(deleteStatement, 1, (agentId as NSString).utf8String, -1, nil) + var result = false + dbQueue.sync { + if let agentId = agentId { + let deleteString = "DELETE FROM messages WHERE agent_id = ?;" + var deleteStatement: OpaquePointer? - if sqlite3_step(deleteStatement) == SQLITE_DONE { + if sqlite3_prepare_v2(db, deleteString, -1, &deleteStatement, nil) == SQLITE_OK { + sqlite3_bind_text(deleteStatement, 1, (agentId as NSString).utf8String, -1, nil) + + if sqlite3_step(deleteStatement) == SQLITE_DONE { + sqlite3_finalize(deleteStatement) + result = true + } else { + let errmsg = String(cString: sqlite3_errmsg(db)!) + os_log("删除会话消息失败: %{public}@", log: logger, type: .error, errmsg) + } + sqlite3_finalize(deleteStatement) - return true } else { let errmsg = String(cString: sqlite3_errmsg(db)!) - os_log("删除会话消息失败: %{public}@", log: logger, type: .error, errmsg) - } - - sqlite3_finalize(deleteStatement) - } else { - let errmsg = String(cString: sqlite3_errmsg(db)!) - os_log("删除会话消息语句准备失败: %{public}@", log: logger, type: .error, errmsg) - } - } else if let messageIds = messageIds, !messageIds.isEmpty { - // 构建参数占位符 - let placeholders = String(repeating: "?,", count: messageIds.count).dropLast() - let deleteString = "DELETE FROM messages WHERE id IN (\(placeholders));" - var deleteStatement: OpaquePointer? - - if sqlite3_prepare_v2(db, deleteString, -1, &deleteStatement, nil) == SQLITE_OK { - for (index, id) in messageIds.enumerated() { - sqlite3_bind_int(deleteStatement, Int32(index + 1), Int32(id)) + os_log("删除会话消息语句准备失败: %{public}@", log: logger, type: .error, errmsg) } + } else if let messageIds = messageIds, !messageIds.isEmpty { + // 构建参数占位符 + let placeholders = String(repeating: "?,", count: messageIds.count).dropLast() + let deleteString = "DELETE FROM messages WHERE id IN (\(placeholders));" + var deleteStatement: OpaquePointer? - if sqlite3_step(deleteStatement) == SQLITE_DONE { + if sqlite3_prepare_v2(db, deleteString, -1, &deleteStatement, nil) == SQLITE_OK { + for (index, id) in messageIds.enumerated() { + sqlite3_bind_int(deleteStatement, Int32(index + 1), Int32(id)) + } + + if sqlite3_step(deleteStatement) == SQLITE_DONE { + result = true + } else { + let errmsg = String(cString: sqlite3_errmsg(db)!) + os_log("删除特定消息失败: %{public}@", log: logger, type: .error, errmsg) + } sqlite3_finalize(deleteStatement) - return true } else { let errmsg = String(cString: sqlite3_errmsg(db)!) - os_log("删除特定消息失败: %{public}@", log: logger, type: .error, errmsg) + os_log("删除特定消息语句准备失败: %{public}@", log: logger, type: .error, errmsg) } - - sqlite3_finalize(deleteStatement) } else { - let errmsg = String(cString: sqlite3_errmsg(db)!) - os_log("删除特定消息语句准备失败: %{public}@", log: logger, type: .error, errmsg) + os_log("删除消息参数无效 - agentId和messageIds都为空", log: logger, type: .error) } - } else { - os_log("删除消息参数无效 - agentId和messageIds都为空", log: logger, type: .error) - return false } - - return false + return result } /** @@ -302,15 +291,18 @@ public class ChatStorageHelper { * @return 是否清空成功 */ public func clearDatabase() -> Bool { - let deleteString = "DELETE FROM messages;" - - if sqlite3_exec(db, deleteString, nil, nil, nil) == SQLITE_OK { - return true - } else { - let errmsg = String(cString: sqlite3_errmsg(db)!) - os_log("清空数据库失败: %{public}@", log: logger, type: .error, errmsg) - return false + var result = false + dbQueue.sync { + let deleteString = "DELETE FROM messages;" + + if sqlite3_exec(db, deleteString, nil, nil, nil) == SQLITE_OK { + result = true + } else { + let errmsg = String(cString: sqlite3_errmsg(db)!) + os_log("清空数据库失败: %{public}@", log: logger, type: .error, errmsg) + } } + return result } /** @@ -321,75 +313,75 @@ public class ChatStorageHelper { */ public func getRecentMessages(agentId: String, limit: Int) -> [[String: Any]] { var messages: [[String: Any]] = [] - - // 首先检查数据库中是否有该会话的消息 - let countQuery = "SELECT COUNT(*) FROM messages WHERE agent_id = ?" - var countStatement: OpaquePointer? - - if sqlite3_prepare_v2(db, countQuery, -1, &countStatement, nil) == SQLITE_OK { - sqlite3_bind_text(countStatement, 1, (agentId as NSString).utf8String, -1, nil) - if sqlite3_step(countStatement) == SQLITE_ROW { - _ = Int(sqlite3_column_int(countStatement, 0)) + dbQueue.sync { + // 首先检查数据库中是否有该会话的消息 + let countQuery = "SELECT COUNT(*) FROM messages WHERE agent_id = ?" + var countStatement: OpaquePointer? + + if sqlite3_prepare_v2(db, countQuery, -1, &countStatement, nil) == SQLITE_OK { + sqlite3_bind_text(countStatement, 1, (agentId as NSString).utf8String, -1, nil) + if sqlite3_step(countStatement) == SQLITE_ROW { + _ = Int(sqlite3_column_int(countStatement, 0)) + } + sqlite3_finalize(countStatement) } - sqlite3_finalize(countStatement) - } - - // 构建查询语句 - 按时间倒序获取最近的N条，然后在结果中再按时间正序 - let queryString = """ - SELECT * FROM ( - SELECT id, session_id, message, sender, metadata, created_at - FROM messages - WHERE agent_id = ? - ORDER BY created_at DESC - LIMIT ? - ) tmp ORDER BY created_at ASC - """ - - var queryStatement: OpaquePointer? - - if sqlite3_prepare_v2(db, queryString, -1, &queryStatement, nil) == SQLITE_OK { - sqlite3_bind_text(queryStatement, 1, (agentId as NSString).utf8String, -1, nil) - sqlite3_bind_int(queryStatement, 2, Int32(limit)) - while sqlite3_step(queryStatement) == SQLITE_ROW { - let id = sqlite3_column_int(queryStatement, 0) - - let sessionIdPtr = sqlite3_column_text(queryStatement, 1) - let sessionId = sessionIdPtr != nil ? String(cString: sessionIdPtr!) : "" - - - let messagePtr = sqlite3_column_text(queryStatement, 2) - let message = messagePtr != nil ? String(cString: messagePtr!) : "" - - let senderPtr = sqlite3_column_text(queryStatement, 3) - let sender = senderPtr != nil ? String(cString: senderPtr!) : "" - - let metadataPtr = sqlite3_column_text(queryStatement, 4) - let metadata = metadataPtr != nil ? String(cString: metadataPtr!) : nil - - let createdAt = sqlite3_column_int(queryStatement, 5) - - var messageDict: [String: Any] = [ - "id": id, - "sessionId":sessionId, - "message": message, - "sender": sender, - "created_at": createdAt - ] + // 构建查询语句 - 按时间倒序获取最近的N条，然后在结果中再按时间正序 + let queryString = """ + SELECT * FROM ( + SELECT id, session_id, message, sender, metadata, created_at + FROM messages + WHERE agent_id = ? + ORDER BY created_at DESC + LIMIT ? + ) tmp ORDER BY created_at ASC + """ + + var queryStatement: OpaquePointer? + + if sqlite3_prepare_v2(db, queryString, -1, &queryStatement, nil) == SQLITE_OK { + sqlite3_bind_text(queryStatement, 1, (agentId as NSString).utf8String, -1, nil) + sqlite3_bind_int(queryStatement, 2, Int32(limit)) - if let metadata = metadata { - messageDict["metadata"] = metadata + while sqlite3_step(queryStatement) == SQLITE_ROW { + let id = sqlite3_column_int(queryStatement, 0) + + let sessionIdPtr = sqlite3_column_text(queryStatement, 1) + let sessionId = sessionIdPtr != nil ? String(cString: sessionIdPtr!) : "" + + + let messagePtr = sqlite3_column_text(queryStatement, 2) + let message = messagePtr != nil ? String(cString: messagePtr!) : "" + + let senderPtr = sqlite3_column_text(queryStatement, 3) + let sender = senderPtr != nil ? String(cString: senderPtr!) : "" + + let metadataPtr = sqlite3_column_text(queryStatement, 4) + let metadata = metadataPtr != nil ? String(cString: metadataPtr!) : nil + + let createdAt = sqlite3_column_int(queryStatement, 5) + + var messageDict: [String: Any] = [ + "id": id, + "sessionId":sessionId, + "message": message, + "sender": sender, + "created_at": createdAt + ] + + if let metadata = metadata { + messageDict["metadata"] = metadata + } + + messages.append(messageDict) } - messages.append(messageDict) + sqlite3_finalize(queryStatement) + } else { + let errmsg = String(cString: sqlite3_errmsg(db)!) + os_log("查询最近消息失败: %{public}@", log: logger, type: .error, errmsg) } - - sqlite3_finalize(queryStatement) - } else { - let errmsg = String(cString: sqlite3_errmsg(db)!) - os_log("查询最近消息失败: %{public}@", log: logger, type: .error, errmsg) } - return messages } } From 9874d9c73b335e115ad9e3386be08fd18cd1047e Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Thu, 22 Jan 2026 20:12:18 +0800 Subject: [PATCH 14/25] =?UTF-8?q?fix(agent=5Fservice):=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E8=AF=AD=E9=9F=B3=E8=AF=86=E5=88=AB=E5=92=8C=E6=96=87?= =?UTF-8?q?=E6=9C=AC=E5=A4=84=E7=90=86=E4=B8=AD=E7=9A=84=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复停止语音识别时未取消AI流的问题 调整文本输入处理前的状态重置逻辑 移除重复的TTS参数设置 --- .../agent_service/AgentServiceImpl.swift | 22 ++++++++++++++----- .../chat_storage/ChatStorageHelper.swift | 1 - 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 0196a1d46..bbe269dbc 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -509,6 +509,10 @@ class AgentServiceImpl: NSObject { } } stopRecognition() + if isAiStreaming { + isAiStreaming = false + _ = chatApiService?.cancelCurrentStream() + } stopTts() @@ -537,7 +541,7 @@ class AgentServiceImpl: NSObject { print("ai启动语音=\(audioSourceType)=\(useBle)") guard let success = azureAsrHelper?.startContinuousRecognition( - // callback: self, + mod:self.currentRecognitionMode, audioSourceType: audioSourceType, isRemoveFirstPunctuation: false ), success else { @@ -642,9 +646,6 @@ class AgentServiceImpl: NSObject { if text.isEmpty { return false } - - _ = azureTtsHelper?.setSpeechParams(rate: ttsRatePercent, pitch: 0, volume: 100) - _ = azureTtsHelper?.setPhoneCallMode(mod: currentRecognitionMode) return azureTtsHelper?.speakOnce(sessionid:sessionid,text) ?? false } @@ -706,8 +707,16 @@ class AgentServiceImpl: NSObject { sendError("文本输入不能为空", code: "EMPTY_TEXT") return false } - + stopIdleCheck() + if isAiStreaming { + isAiStreaming = false + _ = chatApiService?.cancelCurrentStream() + } + stopTts() + currentRecognitionMode = "normal" + os_log("开始处理文本输入", log: logger, type: .info) + processWithChatApiService(sessionid: sessionid, text: text, speakResponse: speakResponse) return true } @@ -717,7 +726,8 @@ class AgentServiceImpl: NSObject { sendError("ChatAPI服务未初始化", code: "CHATAPI_NOT_INITIALIZED") return } - + _ = azureTtsHelper?.setSpeechParams(rate: ttsRatePercent, pitch: 0, volume: 100) + _ = azureTtsHelper?.setPhoneCallMode(mod: currentRecognitionMode) let userMessage = chatApiService.createUserMessage(content: text) processWithChatApiServiceInternal(sessionid: sessionid,userMessage: userMessage, displayText: text, speakResponse: speakResponse) } diff --git a/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift b/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift index 6fdd60df2..eb224827b 100644 --- a/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift +++ b/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift @@ -245,7 +245,6 @@ public class ChatStorageHelper { sqlite3_bind_text(deleteStatement, 1, (agentId as NSString).utf8String, -1, nil) if sqlite3_step(deleteStatement) == SQLITE_DONE { - sqlite3_finalize(deleteStatement) result = true } else { let errmsg = String(cString: sqlite3_errmsg(db)!) From fc2afcecd5fda540350c52ff0066f679b80409b8 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Thu, 22 Jan 2026 21:47:43 +0800 Subject: [PATCH 15/25] =?UTF-8?q?fix(agent=5Fservice):=20=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E9=BB=98=E8=AE=A4=E8=AF=86=E5=88=AB=E6=A8=A1=E5=BC=8F?= =?UTF-8?q?=E4=B8=BAnormal=E5=B9=B6=E7=A7=BB=E9=99=A4=E5=86=97=E4=BD=99?= =?UTF-8?q?=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 移除AgentServiceImpl中的冗余日志输出 在AgentServicePlugin中设置currentRecognitionMode默认值 添加AzureTtsHelper的调试日志用于跟踪识别模式 --- .../Sources/agent_service/AgentServiceImpl.swift | 15 ++++++--------- .../agent_service/AgentServicePlugin.swift | 1 + .../Sources/azure_speech/AzureTtsHelper.swift | 4 +++- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index bbe269dbc..b0a7ee5f0 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -713,10 +713,7 @@ class AgentServiceImpl: NSObject { _ = chatApiService?.cancelCurrentStream() } stopTts() - currentRecognitionMode = "normal" - - os_log("开始处理文本输入", log: logger, type: .info) - + processWithChatApiService(sessionid: sessionid, text: text, speakResponse: speakResponse) return true } @@ -749,7 +746,7 @@ class AgentServiceImpl: NSObject { stopTts(); isAiStreaming = true - os_log("设置AI流式状态为true", log: logger, type: .info) + // os_log("设置AI流式状态为true", log: logger, type: .info) var messages: [[String: Any]] = [] @@ -766,17 +763,17 @@ class AgentServiceImpl: NSObject { // print("用户画像数据: \(info)") let _systemPrompt = fillTemplate(systemPrompt, with: info) - print("替换后的系统提示词: \(_systemPrompt)") + // print("替换后的系统提示词: \(_systemPrompt)") messages.append(chatApiService.createSystemMessage(content: _systemPrompt)) - os_log("添加系统提示消息", log: logger, type: .info) + // os_log("添加系统提示消息", log: logger, type: .info) } messages.append(contentsOf: chatHistory) - os_log("添加历史消息: %d条", log: logger, type: .info, chatHistory.count) + // os_log("添加历史消息: %d条", log: logger, type: .info, chatHistory.count) messages.append(userMessage) - os_log("添加用户消息", log: logger, type: .info) + // os_log("添加用户消息", log: logger, type: .info) // 保存用户消息 self.chatStorageHelper.saveMessage( diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift index e8a5d6209..77d9085d9 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift @@ -121,6 +121,7 @@ public class AgentServicePlugin: NSObject, FlutterPlugin { } let speakResponse = arguments["speakResponse"] as? Bool ?? false + impl.currentRecognitionMode = "normal" let success = impl.processTextInput(sessionid: sessionid,text, speakResponse: speakResponse) result(success) diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index cd66fb76d..6734d418f 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -416,6 +416,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { public func setPhoneCallMode(mod: String) -> Bool { + // os_log("[liwei] AzureTtsHelper: setPhoneCallMode: %{public}@", log: log, type: .debug, mod) self.previousRecognitionMode = self.currentRecognitionMode self.currentRecognitionMode = mod self.isHeadphones = detectHeadphonesOutput() @@ -1416,7 +1417,7 @@ private func applyAudioSessionForTTS() -> Bool { var desiredOptions: AVAudioSession.CategoryOptions = [] let isHeadphonesConnected = outputs.contains(where: { $0.portType == .headphones || $0.portType == .headsetMic || $0.portType == .bluetoothA2DP }) - + os_log("[liwei] AzureTtsHelper: applyAudioSessionForTTS: %{public}@", log: self.log, type: .debug, self.currentRecognitionMode) if self.currentRecognitionMode == "phone_call" || self.currentRecognitionMode == "push_to_talk" { desiredCategory = .playAndRecord if isHeadphonesConnected { @@ -1567,6 +1568,7 @@ private func isTelephonyActive() -> Bool { /// - 外部音频播放时启用 spoken audio 策略，确保能在 A2DP 上播放 /// - 仅当类别或模式发生变化时才 setCategory，降低 AVAudioSessionRouteChangeReason.categoryChange 的触发概率 private func safeConfigureAudioSessionForTTS() { + os_log("[liwei] AzureTtsHelper: safeConfigureAudioSessionForTTS - recognitionMode: %{public}@", log: log, type: .debug, currentRecognitionMode) // 使用异步，避免串行队列重入自锁 audioSessionQueue.async { [weak self] in guard let self = self else { return } From 12711b0cf80e1c0ac91b6393abf1c4b1798e998a Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Fri, 23 Jan 2026 21:15:47 +0800 Subject: [PATCH 16/25] =?UTF-8?q?feat:=20=E4=B8=BA=E8=AF=AD=E9=9F=B3?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E6=B7=BB=E5=8A=A0=E6=A8=A1=E5=BC=8F=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在 ASR 和 TTS 服务接口中添加模式参数，支持 normal、phone_call、push_to_talk 等模式 - 修改翻译控制器，根据当前翻译模式设置对应的 TTS 和 ASR 模式 - 重构 iOS 音频会话配置逻辑，优化不同模式下的音频路由和蓝牙设备处理 - 统一 TTS 模式设置方法名，将 setPhoneCallMode 重命名为 setTtsMode --- lib/data/services/asr_service.dart | 1 + .../speech_impl/azure_asr_service.dart | 2 + .../speech_impl/azure_tts_service.dart | 14 ++ .../speech_impl/voice_clone_tts_service.dart | 12 + .../speech_impl/volcano_asr_api_service.dart | 1 + .../speech_impl/volcano_asr_service.dart | 1 + .../speech_impl/volcano_tts_service.dart | 17 ++ lib/data/services/tts_service.dart | 3 + .../controllers/translation_controller.dart | 26 +- .../agent_service/AgentServiceImpl.swift | 12 +- .../azure_speech/AzureSpeechPlugin.swift | 15 +- .../Sources/azure_speech/AzureTtsHelper.swift | 229 +++++++++--------- .../Sources/tools/MicrophoneCapture.swift | 23 +- .../Sources/tools/SimpleAudioReceiver.swift | 42 ++-- 14 files changed, 230 insertions(+), 168 deletions(-) diff --git a/lib/data/services/asr_service.dart b/lib/data/services/asr_service.dart index 8bb7892c6..2b3610f68 100644 --- a/lib/data/services/asr_service.dart +++ b/lib/data/services/asr_service.dart @@ -22,6 +22,7 @@ abstract class AsrService { Future startContinuousRecognition( bool audioSourceType, { bool isRemoveFirstPunctuation = true, + String mode = "normal", }); /// 停止连续语音识别 diff --git a/lib/data/services/speech_impl/azure_asr_service.dart b/lib/data/services/speech_impl/azure_asr_service.dart index a196a5ddc..bdbc6100e 100644 --- a/lib/data/services/speech_impl/azure_asr_service.dart +++ b/lib/data/services/speech_impl/azure_asr_service.dart @@ -207,6 +207,7 @@ class AzureAsrService extends GetxService implements AsrService { Future startContinuousRecognition( bool audioSourceType, { bool isRemoveFirstPunctuation = true, + String mode = "normal", }) async { // 防抖判断：短时间内重复调用直接拦截 final now = DateTime.now(); @@ -229,6 +230,7 @@ class AzureAsrService extends GetxService implements AsrService { await _channel.invokeMethod('startContinuousRecognition', { 'audioSourceType': audioSourceType, 'isRemoveFirstPunctuation': isRemoveFirstPunctuation, + 'mode': mode, }); if (!result) { diff --git a/lib/data/services/speech_impl/azure_tts_service.dart b/lib/data/services/speech_impl/azure_tts_service.dart index cebcb9a10..200002fb4 100644 --- a/lib/data/services/speech_impl/azure_tts_service.dart +++ b/lib/data/services/speech_impl/azure_tts_service.dart @@ -182,6 +182,20 @@ class AzureTtsService extends GetxService implements TtsService { } } + @override + Future setTtsMode(String mod) async { + if (!_isInitialized) await initialize(); + try { + final result = await _channel.invokeMethod('setTtsMode', { + 'mod': mod, + }); + return result; + } catch (e) { + Logger.error('设置TTS模式失败: ${e.toString()}'); + return false; + } + } + @override Future setVoiceFlocking(String speakerProfileId) async { final result = await _channel.invokeMethod('setVoiceFlocking', { diff --git a/lib/data/services/speech_impl/voice_clone_tts_service.dart b/lib/data/services/speech_impl/voice_clone_tts_service.dart index 15ea6803b..97542c02b 100644 --- a/lib/data/services/speech_impl/voice_clone_tts_service.dart +++ b/lib/data/services/speech_impl/voice_clone_tts_service.dart @@ -94,6 +94,18 @@ class VoiceCloneTtsService extends GetxService implements TtsService { } } + @override + Future setTtsMode(String mod) async { + if (!_isInitialized) await initialize(); + try { + Logger.info('设置TTS模式: $mod'); + return true; + } catch (e) { + Logger.error('设置TTS模式失败: ${e.toString()}'); + return false; + } + } + @override Future setVoiceFlocking(String speakerProfileId) async { return true; diff --git a/lib/data/services/speech_impl/volcano_asr_api_service.dart b/lib/data/services/speech_impl/volcano_asr_api_service.dart index 3e5711b38..d105d97ec 100644 --- a/lib/data/services/speech_impl/volcano_asr_api_service.dart +++ b/lib/data/services/speech_impl/volcano_asr_api_service.dart @@ -865,6 +865,7 @@ class VolcanoAsrApiService implements AsrService { Future startContinuousRecognition( bool audioSourceType, { bool isRemoveFirstPunctuation = true, + String mode = "normal", }) { // TODO: implement startContinuousRecognition throw UnimplementedError(); diff --git a/lib/data/services/speech_impl/volcano_asr_service.dart b/lib/data/services/speech_impl/volcano_asr_service.dart index ec36a0385..d1f59648e 100644 --- a/lib/data/services/speech_impl/volcano_asr_service.dart +++ b/lib/data/services/speech_impl/volcano_asr_service.dart @@ -400,6 +400,7 @@ class VolcanoAsrService extends GetxService implements AsrService { Future startContinuousRecognition( bool audioSourceType, { bool isRemoveFirstPunctuation = true, + String mode = "normal", }) { // TODO: implement startContinuousRecognition throw UnimplementedError(); diff --git a/lib/data/services/speech_impl/volcano_tts_service.dart b/lib/data/services/speech_impl/volcano_tts_service.dart index 2bb04ade0..c0dfb8935 100644 --- a/lib/data/services/speech_impl/volcano_tts_service.dart +++ b/lib/data/services/speech_impl/volcano_tts_service.dart @@ -185,6 +185,23 @@ class VolcanoTtsService extends GetxService implements TtsService { } } + @override + Future setTtsMode(String mod) async { + if (!await _ensureInitialized()) return false; + + try { + final result = await _channel.invokeMethod('setTtsMode', { + 'mod': mod, + }) ?? + false; + + return result; + } catch (e) { + debugPrint('设置TTS模式失败：$e'); + return false; + } + } + @override Future setVoiceFlocking(String speakerProfileId) async { return true; diff --git a/lib/data/services/tts_service.dart b/lib/data/services/tts_service.dart index b4bf72625..c5b24ad18 100644 --- a/lib/data/services/tts_service.dart +++ b/lib/data/services/tts_service.dart @@ -19,6 +19,9 @@ abstract class TtsService { /// 设置语音 Future setVoice(String voiceName); + /// 设置TTS模式 + Future setTtsMode(String mod); + /// 设置语音复刻 Future setVoiceFlocking(String speakerProfileId); diff --git a/lib/modules/translation/controllers/translation_controller.dart b/lib/modules/translation/controllers/translation_controller.dart index 994d1c109..c1d57845b 100644 --- a/lib/modules/translation/controllers/translation_controller.dart +++ b/lib/modules/translation/controllers/translation_controller.dart @@ -344,6 +344,15 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } catch (_) {} } + //设置播报的模式 + if (mode == 'simultaneous') { + await _ttsService.setTtsMode('phone_call'); + } else if (mode == 'faceToFace') { + await _ttsService.setTtsMode('push_to_talk'); + } else { + await _ttsService.setTtsMode('normal'); + } + // 检查悬浮窗支持情况 if (isFloatingWindowEnabled.value && !_isFloatingWindowSupportedMode()) { // 当前模式不支持悬浮窗，自动禁用 @@ -437,6 +446,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { await _asrService.startContinuousRecognition( _audioSourceType, isRemoveFirstPunctuation: false, + mode: 'push_to_talk', ); //开启识别 _startAsrActiveTracking(); //开启识别活动跟踪 if (isRecording.value) { @@ -946,7 +956,13 @@ class TranslationController extends GetxController with WidgetsBindingObserver { !bleManager.isCodecActive) { return; } - await _startAsrService(); + if (currentMode.value == 'simultaneous') { + await _startAsrService('push_to_talk'); + } else if (currentMode.value == 'faceToFace') { + await _startAsrService('phone_call'); + } else { + await _startAsrService('normal'); + } await _finalizeRecognitionStart(); } } catch (e) { @@ -1045,8 +1061,12 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } /// 启动语音识别 - Future _startAsrService() async { - await _asrService.startContinuousRecognition(_audioSourceType); + Future _startAsrService(String mode) async { + await _asrService.startContinuousRecognition( + _audioSourceType, + isRemoveFirstPunctuation: false, + mode: mode, + ); } /// 完成识别启动 diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index b0a7ee5f0..61f5e4d8a 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -526,7 +526,7 @@ class AgentServiceImpl: NSObject { // 设置当前识别模式 currentRecognitionMode = mode - os_log("设置语音识别模式: %{public}@", log: logger, type: .info, mode) + os_log("liwei--------------- 设置语音识别模式: %{public}@", log: logger, type: .info, mode) let audioSourceType: AzureAsrHelper.AudioSourceType = useBle ? .external : .microphone @@ -724,7 +724,7 @@ class AgentServiceImpl: NSObject { return } _ = azureTtsHelper?.setSpeechParams(rate: ttsRatePercent, pitch: 0, volume: 100) - _ = azureTtsHelper?.setPhoneCallMode(mod: currentRecognitionMode) + _ = azureTtsHelper?.setTtsMode(mod: currentRecognitionMode) let userMessage = chatApiService.createUserMessage(content: text) processWithChatApiServiceInternal(sessionid: sessionid,userMessage: userMessage, displayText: text, speakResponse: speakResponse) } @@ -1810,7 +1810,7 @@ class ChatApiStreamCallback: StreamCallback { responseBuilder += token if speakResponse && reply && broadcast{ - _ = agentService.azureTtsHelper?.setPhoneCallMode(mod:agentService.currentRecognitionMode) +// _ = agentService.azureTtsHelper?.setTtsMode(mod:agentService.currentRecognitionMode) try agentService.azureTtsHelper?.speakStream(sessionid:sessionid,token) // 在开始流式TTS时立即停止气泡音 } @@ -1836,7 +1836,7 @@ class ChatApiStreamCallback: StreamCallback { } if speakResponse && reply && broadcast && sessionid == agentService.currsessionId{ - _ = agentService.azureTtsHelper?.setPhoneCallMode(mod: agentService.currentRecognitionMode) + // _ = agentService.azureTtsHelper?.setTtsMode(mod: agentService.currentRecognitionMode) agentService.azureTtsHelper?.flushStream(sessionid:sessionid) } @@ -1888,7 +1888,7 @@ class ChatApiStreamCallback: StreamCallback { "message": message ]) if(code == 2001){ - _ = agentService.azureTtsHelper?.setPhoneCallMode(mod: agentService.currentRecognitionMode) + // _ = agentService.azureTtsHelper?.setTtsMode(mod: agentService.currentRecognitionMode) agentService.azureTtsHelper?.speakStream(sessionid: sessionid,agentService.insufficientIntegralText) } agentService.isAiStreaming = false @@ -1932,7 +1932,7 @@ class ChatApiStreamCallback: StreamCallback { if (iscallingTool) { os_log("收到函数调用: callingToolText:%{public}@", log: agentService.logger, type: .info, agentService.callingToolText) - _ = agentService.azureTtsHelper?.setPhoneCallMode(mod: agentService.currentRecognitionMode) + // _ = agentService.azureTtsHelper?.setTtsMode(mod: agentService.currentRecognitionMode) agentService.azureTtsHelper?.speakStream(sessionid: sessionid,agentService.callingToolText) iscallingTool = false } diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift index c74abc624..ce82256b1 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift @@ -382,6 +382,7 @@ private func sendAstEvent(_ event: [String: Any]) { } let removeFirstPunctuation = args["isRemoveFirstPunctuation"] as? Bool ?? true + let mode = args["mode"] as? String ?? "normal" // 根据参数确定音频源类型 let audioSourceType = isExternalActive ? @@ -392,6 +393,7 @@ private func sendAstEvent(_ event: [String: Any]) { // 启动连续识别 let success = azureAsrHelper.startContinuousRecognition( + mod: mode, audioSourceType: audioSourceType, isRemoveFirstPunctuation: removeFirstPunctuation ) @@ -631,19 +633,14 @@ private func sendAstEvent(_ event: [String: Any]) { let success = azureTtsHelper.setVoice(voiceName) result(success) - - case "setSpeechParams": - guard let args = call.arguments as? [String: Any] else { + case "setTtsMode": + guard let args = call.arguments as? [String: Any], + let mod = args["mod"] as? String else { result(FlutterError(code: "INVALID_ARGUMENTS", message: "参数不能为空", details: nil)) return } - - let rate = args["rate"] as? Int ?? 0 - let pitch = args["pitch"] as? Int ?? 0 - let volume = args["volume"] as? Int ?? 100 - let success = azureTtsHelper.setSpeechParams(rate: rate, pitch: pitch, volume: volume) + let success = azureTtsHelper.setTtsMode(mod: mod) result(success) - case "speakText": guard let args = call.arguments as? [String: Any], let sessionid = args["sessionid"] as? String, diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index 6734d418f..68ae36cd2 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -415,14 +415,14 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { } - public func setPhoneCallMode(mod: String) -> Bool { - // os_log("[liwei] AzureTtsHelper: setPhoneCallMode: %{public}@", log: log, type: .debug, mod) + public func setTtsMode(mod: String) -> Bool { + os_log("[liwei] AzureTtsHelper: setTtsMode: %{public}@", log: log, type: .debug, mod) self.previousRecognitionMode = self.currentRecognitionMode self.currentRecognitionMode = mod self.isHeadphones = detectHeadphonesOutput() - if isInitialized { - safeConfigureAudioSessionForTTS() - } + // if isInitialized { + // safeConfigureAudioSessionForTTS() + // } return true } /** @@ -438,6 +438,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { out.portType == .bluetoothA2DP || out.portType == .bluetoothHFP || out.portType == .bluetoothLE || + out.portType == .headsetMic || out.portType == .headphones } } @@ -1376,6 +1377,18 @@ private var audioSessionObserversAdded = false private var routeChangeDebounceWorkItem: DispatchWorkItem? private var isApplyingAudioSessionConfig = false +private func isTtsBusy() -> Bool { + if speaking { return true } + if isPlaying { return true } + if !playbackQueue.isEmpty { return true } + if pendingTextCount > 0 { return true } + if !pendingTasks.isEmpty { return true } + if activeStreamSynthesisCount > 0 { return true } + if !pcmPendingData.isEmpty { return true } + if scheduledBufferCount > 0 { return true } + return false +} + /** * 确保音频会话队列已设置 specific key（用于避免同队列 sync 死锁） * - Parameters: 无 @@ -1388,6 +1401,12 @@ private func ensureAudioSessionQueueSpecificKeySet() { audioSessionQueue.setSpecific(key: audioSessionQueueKey, value: 1) } +/** + * 同步应用 TTS 音频会话配置（用于需要立即生效的播放/引擎初始化路径） + * - Parameters: 无 + * - Returns: 是否成功激活音频会话 + * - Throws: 无（内部捕获 AVAudioSession 相关异常并记录日志） + */ /** * 同步应用 TTS 音频会话配置（用于需要立即生效的播放/引擎初始化路径） * - Parameters: 无 @@ -1397,7 +1416,6 @@ private func ensureAudioSessionQueueSpecificKeySet() { private func applyAudioSessionForTTS() -> Bool { var success = true ensureAudioSessionQueueSpecificKeySet() - let applyBody = { if self.isApplyingAudioSessionConfig { return } self.isApplyingAudioSessionConfig = true @@ -1405,69 +1423,62 @@ private func applyAudioSessionForTTS() -> Bool { let audioSession = AVAudioSession.sharedInstance() let outputs: [AVAudioSessionPortDescription] = audioSession.currentRoute.outputs - let hasBluetoothHFP = outputs.contains(where: { $0.portType == .bluetoothHFP }) let hasWiredHeadphones = outputs.contains(where: { $0.portType == .headphones || $0.portType == .headsetMic }) - let hasBluetoothA2DP = outputs.contains(where: { $0.portType == .bluetoothA2DP }) - let hasHeadphones = hasWiredHeadphones || hasBluetoothA2DP + let hasBluetoothA2DP = outputs.contains(where: { $0.portType == .bluetoothA2DP || $0.portType == .bluetoothLE }) + let hasBluetoothHFP = outputs.contains(where: { $0.portType == .bluetoothHFP }) + let hasHeadphones = hasWiredHeadphones || hasBluetoothA2DP || hasBluetoothHFP let otherAudioPlaying = audioSession.isOtherAudioPlaying let isRecording = (self.micCapture?.isCapturing == true) + self.isHeadphones = hasHeadphones let desiredCategory: AVAudioSession.Category let desiredMode: AVAudioSession.Mode var desiredOptions: AVAudioSession.CategoryOptions = [] - let isHeadphonesConnected = outputs.contains(where: { $0.portType == .headphones || $0.portType == .headsetMic || $0.portType == .bluetoothA2DP }) - os_log("[liwei] AzureTtsHelper: applyAudioSessionForTTS: %{public}@", log: self.log, type: .debug, self.currentRecognitionMode) - if self.currentRecognitionMode == "phone_call" || self.currentRecognitionMode == "push_to_talk" { + + if self.currentRecognitionMode == "phone_call" { desiredCategory = .playAndRecord - if isHeadphonesConnected { - desiredMode = .voiceChat - if #available(iOS 18.0, *) { - desiredOptions = [.allowBluetooth, .duckOthers] - } else { - desiredOptions = [.allowBluetooth, .allowBluetoothA2DP] + if hasHeadphones { + desiredMode = .default + desiredOptions = [.allowBluetoothA2DP] + if let inputs = audioSession.availableInputs, let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { + try? audioSession.setPreferredInput(builtInMic) } } else { - desiredMode = .videoChat + desiredMode = .voiceChat desiredOptions = [.allowBluetooth, .defaultToSpeaker] } - } else if self.isTelephonyActive() { + if otherAudioPlaying { + desiredOptions.insert(.duckOthers) + } + } else if self.currentRecognitionMode == "push_to_talk" || self.currentRecognitionMode == "normal" || self.currentRecognitionMode == "ble_wakeup" { + if isRecording { + self.micCapture?.stopCapture() + Thread.sleep(forTimeInterval: 0.05) + } desiredCategory = .playback - desiredMode = .default - desiredOptions.insert(.duckOthers) + desiredMode = .spokenAudio + if otherAudioPlaying { + desiredOptions.insert(.duckOthers) + } } else if isRecording { desiredCategory = .playAndRecord - if self.isHeadphones { - desiredMode = .default - desiredOptions.insert(.allowBluetooth) + desiredMode = .default + if hasHeadphones { desiredOptions.insert(.allowBluetoothA2DP) - if otherAudioPlaying { - desiredOptions.insert(.duckOthers) + if let inputs = audioSession.availableInputs, let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { + try? audioSession.setPreferredInput(builtInMic) } } else { - desiredMode = .videoChat desiredOptions.insert(.allowBluetooth) - desiredOptions.insert(.mixWithOthers) desiredOptions.insert(.defaultToSpeaker) } - } else if hasBluetoothHFP { - desiredCategory = .playback - desiredMode = .voicePrompt - desiredOptions = [.interruptSpokenAudioAndMixWithOthers, .allowBluetooth] - } else if hasHeadphones { - desiredCategory = .playback - desiredMode = .default if otherAudioPlaying { desiredOptions.insert(.duckOthers) } } else { desiredCategory = .playback - desiredMode = .default - if #available(iOS 18.0, *) { - desiredOptions = [] - } else { - desiredOptions = [.defaultToSpeaker] - } + desiredMode = .spokenAudio if otherAudioPlaying { desiredOptions.insert(.duckOthers) } @@ -1483,7 +1494,7 @@ private func applyAudioSessionForTTS() -> Bool { _ = try? audioSession.setPreferredIOBufferDuration(0.02) } try audioSession.setActive(true) - if self.currentRecognitionMode == "phone_call", !self.isHeadphones { + if self.currentRecognitionMode == "phone_call", !hasHeadphones { _ = try? audioSession.overrideOutputAudioPort(.speaker) } break @@ -1551,43 +1562,20 @@ private func isTelephonyActive() -> Bool { return false } -/** - * 安全配置音频会话用于 TTS - * - 播放优先走媒体声道（A2DP），避免切换到通话声道（HFP） - * - 外部音乐播放时：不启用 mix，默认会打断外部音频；同时可 duck 确保可听见 - * - 录音进行中：保持 playAndRecord + A2DP 输出 + 内置麦输入，避免切到 HFP - * - 通话中：保持保守策略，避免触发系统断言或破坏通话链路 - * - Parameters: 无 - * - Returns: 无 - * - Throws: 无（内部捕获 AVAudioSession 相关异常并记录日志） - */ - - /// 为 TTS 安全配置音频会话： - /// - 使用异步队列，避免在同一串行队列上 sync 导致死锁 - /// - 优先使用 .playback + .spokenAudio + .allowBluetoothA2DP（媒体声道），避免进入 HFP - /// - 外部音频播放时启用 spoken audio 策略，确保能在 A2DP 上播放 - /// - 仅当类别或模式发生变化时才 setCategory，降低 AVAudioSessionRouteChangeReason.categoryChange 的触发概率 -private func safeConfigureAudioSessionForTTS() { - os_log("[liwei] AzureTtsHelper: safeConfigureAudioSessionForTTS - recognitionMode: %{public}@", log: log, type: .debug, currentRecognitionMode) - // 使用异步，避免串行队列重入自锁 - audioSessionQueue.async { [weak self] in - guard let self = self else { return } - - let audioSession = AVAudioSession.sharedInstance() - - // 显式指定 outputs 的类型，确保闭包参数类型可推断 - let outputs: [AVAudioSessionPortDescription] = audioSession.currentRoute.outputs - // 使用 contains(where:) 明确谓词闭包。 - // 保持使用 audioSessionQueue.async，防止在串行队列上自锁。 - // 仅在类别或模式发生变化时调用 setCategory，减少 categoryChange 的回调风暴。 - let hasBluetoothHFP = outputs.contains(where: { $0.portType == .bluetoothHFP }) - _ = hasBluetoothHFP - _ = audioSession - - _ = self.applyAudioSessionForTTS() - } + /// 为 TTS 安全配置音频会话： + /// - 使用异步队列，避免在同一串行队列上 sync 导致死锁 + /// - 优先使用 .playback + .spokenAudio + .allowBluetoothA2DP（媒体声道），避免进入 HFP + /// - 外部音频播放时启用 spoken audio 策略，确保能在 A2DP 上播放 + /// - 仅当类别或模式发生变化时才 setCategory，降低 AVAudioSessionRouteChangeReason.categoryChange 的触发概率 + private func safeConfigureAudioSessionForTTS() { + if !isTtsBusy() { return } + audioSessionQueue.async { [weak self] in + guard let self = self else { return } + if !self.isTtsBusy() { return } + _ = self.applyAudioSessionForTTS() } + } /** * 在 TTS 完全空闲后释放音频会话（用于恢复外部音乐） @@ -1617,8 +1605,6 @@ private func deactivateAudioSessionAfterTTSIfIdle() { self.playerNode = nil self.pcmFormat = nil } - let session = AVAudioSession.sharedInstance() - _ = try? session.setActive(false, options: .notifyOthersOnDeactivation) } if DispatchQueue.getSpecific(key: audioSessionQueueKey) != nil { work() @@ -1645,8 +1631,10 @@ private func deactivateAudioSessionAfterTTSIfIdle() { os_log("音频中断开始（可能为通话/Siri），停止TTS", log: log, type: .info) case .ended: audioInterrupted = false - os_log("音频中断结束，重新配置会话", log: log, type: .info) - safeConfigureAudioSessionForTTS() + if isTtsBusy() { + os_log("音频中断结束，重新配置会话", log: log, type: .info) + safeConfigureAudioSessionForTTS() + } @unknown default: break } @@ -1660,42 +1648,55 @@ private func deactivateAudioSessionAfterTTSIfIdle() { * - 配置过程中（isApplyingAudioSessionConfig = true）直接忽略 */ @objc private func handleRouteChange(_ notification: Notification) { - guard let info = notification.userInfo, - let reasonValue = info[AVAudioSessionRouteChangeReasonKey] as? UInt, - let reason = AVAudioSession.RouteChangeReason(rawValue: reasonValue) else { return } - - os_log("音频路由变化: %{public}@", log: log, type: .info, String(describing: reason)) - - // 配置中触发的路由变化，直接忽略，避免重入导致死循环 - if isApplyingAudioSessionConfig { - os_log("配置进行中，忽略本次路由变化", log: log, type: .debug) - return - } - - // 忽略会造成循环的原因：categoryChange（你的日志 reason=3），以及 routeConfigurationChange 等 - switch reason { - case .categoryChange, .routeConfigurationChange, .wakeFromSleep, .noSuitableRouteForCategory: - os_log("忽略路由变化原因: %{public}@", log: log, type: .debug, String(describing: reason)) - return - default: - break - } - - // 通话期间避免重配，避免与电话路由冲突 - if isTelephonyActive() { - os_log("通话期间检测到路由变化，跳过重配", log: log, type: .info) - return - } + // guard let info = notification.userInfo, + // let reasonValue = info[AVAudioSessionRouteChangeReasonKey] as? UInt, + // let reason = AVAudioSession.RouteChangeReason(rawValue: reasonValue) else { return } + + // if !isTtsBusy() { return } + // os_log("音频路由变化: %{public}@", log: log, type: .info, String(describing: reason)) + + // // 配置中触发的路由变化，直接忽略，避免重入导致死循环 + // if isApplyingAudioSessionConfig { + // os_log("配置进行中，忽略本次路由变化", log: log, type: .debug) + // return + // } + + // // 忽略会造成循环的原因：categoryChange（你的日志 reason=3），以及 routeConfigurationChange 等 + // switch reason { + // case .categoryChange, .routeConfigurationChange, .wakeFromSleep, .noSuitableRouteForCategory: + // os_log("忽略路由变化原因: %{public}@", log: log, type: .debug, String(describing: reason)) + // return + // default: + // break + // } + + // // 通话期间避免重配，避免与电话路由冲突 + // if isTelephonyActive() { + // os_log("通话期间检测到路由变化，跳过重配", log: log, type: .info) + // return + // } + + // let isActivelyPlayingNow = + // speaking || + // isPlaying || + // (audioPlayer?.isPlaying == true) || + // (playerNode?.isPlaying == true) + // if isActivelyPlayingNow { + // routeChangeDebounceWorkItem?.cancel() + // _ = stop() + // os_log("路由变化时正在播报，已停止以避免音源冲突", log: log, type: .info) + // return + // } // 去抖处理：300ms 内只执行最后一次重配 // 忽略 categoryChange 等 // 去抖：在 audioSessionQueue 上延迟调用 - routeChangeDebounceWorkItem?.cancel() - let work = DispatchWorkItem { [weak self] in - self?.safeConfigureAudioSessionForTTS() - } - routeChangeDebounceWorkItem = work - audioSessionQueue.asyncAfter(deadline: .now() + 0.3, execute: work) + // routeChangeDebounceWorkItem?.cancel() + // let work = DispatchWorkItem { [weak self] in + // self?.safeConfigureAudioSessionForTTS() + // } + // routeChangeDebounceWorkItem = work + // audioSessionQueue.asyncAfter(deadline: .now() + 0.3, execute: work) } /** diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift index b379819de..b346d4924 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift @@ -92,8 +92,7 @@ public class MicrophoneCapture: NSObject { } do { - try audioSession.setPreferredSampleRate(sampleRate) - try audioSession.setPreferredIOBufferDuration(0.005) // 5ms缓冲 + try audioSession.setPreferredIOBufferDuration(0.02) print("音频会话配置成功") } catch { print("音频会话配置失败: \(error.localizedDescription)") @@ -306,15 +305,18 @@ public class MicrophoneCapture: NSObject { sampleRate = audioSession.sampleRate } - // iOS 13+ 启用语音处理（可能在部分路由/模式下失败，失败不应阻断录音启动） if #available(iOS 13.0, *) { + let session = self.audioSession ?? AVAudioSession.sharedInstance() + let outputs = session.currentRoute.outputs + let hasBluetoothA2DPOutput = outputs.contains(where: { $0.portType == .bluetoothA2DP || $0.portType == .bluetoothLE }) + let shouldEnableVoiceProcessing = (session.category == .playAndRecord) && (session.mode == .voiceChat || session.mode == .videoChat) && !hasBluetoothA2DPOutput do { - try audioInputNode.setVoiceProcessingEnabled(true) + try audioInputNode.setVoiceProcessingEnabled(shouldEnableVoiceProcessing) } catch { if let nsError = error as NSError? { - print("启用语音处理失败: domain=\(nsError.domain) code=\(nsError.code) desc=\(nsError.localizedDescription)") + print("语音处理设置失败(enable=\(shouldEnableVoiceProcessing)): domain=\(nsError.domain) code=\(nsError.code) desc=\(nsError.localizedDescription)") } else { - print("启用语音处理失败: \(error.localizedDescription)") + print("语音处理设置失败(enable=\(shouldEnableVoiceProcessing)): \(error.localizedDescription)") } } } @@ -514,14 +516,7 @@ public class MicrophoneCapture: NSObject { // 修复：使用安全的清理方法 cleanupAudioEngine() - // 安全地处理音频会话 - guard let audioSession = audioSession else { return } - do { - try audioSession.setActive(false, options: .notifyOthersOnDeactivation) - } catch { - print("音频会话配置失败: \(error.localizedDescription)") - } - + // 安全地处理音频会话：这里不主动 deactivate，避免与后续播报会话切换打架导致卡顿/失败 } /** diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift index a2488a30c..3522a1630 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift @@ -166,25 +166,20 @@ public class SimpleAudioReceiver: NSObject { if audioSourceType == .microphone || isHybridExternalMode { // --- 麦克风输入逻辑 (包括混合模式) --- - desiredCategory = .playAndRecord - if currentRecognitionMode == "phone_call" { - desiredMode = .voiceChat if isHeadphones { - // 修正：增加 A2DP 选项以支持高质量蓝牙音频播放 - desiredOptions = [.allowBluetooth, .allowBluetoothA2DP] + desiredCategory = .playAndRecord + desiredMode = .default + desiredOptions = [.allowBluetoothA2DP] } else { - desiredOptions = [.allowBluetooth, .defaultToSpeaker] - } - } else { // normal, push_to_talk, ble_wakeup 等 - if isHeadphones { - // 修正：使用 .voiceChat 模式以避免 .videoChat 强制外放 + desiredCategory = .playAndRecord desiredMode = .voiceChat - desiredOptions = [.allowBluetooth, .allowBluetoothA2DP, .mixWithOthers] - } else { - desiredMode = .videoChat - desiredOptions = [.mixWithOthers, .defaultToSpeaker] + desiredOptions = [.allowBluetooth, .defaultToSpeaker] } + } else { // normal, push_to_talk, ble_wakeup 等：仅录音，不需要同时播报 + desiredCategory = .record + desiredMode = .default + desiredOptions = [] } // 如果在真实通话中，强制使用更适合的录音设置 @@ -197,8 +192,8 @@ public class SimpleAudioReceiver: NSObject { } else if audioSourceType == .external { // --- 纯外部源 (仅播放) 逻辑 --- desiredCategory = .playback - desiredMode = .videoChat - desiredOptions = isHeadphones ? [.allowBluetoothA2DP, .mixWithOthers] : [.mixWithOthers, .defaultToSpeaker] + desiredMode = .spokenAudio + desiredOptions = isHeadphones ? [.allowBluetoothA2DP, .mixWithOthers] : [.mixWithOthers] } else { // 兜底，理论上不会执行 desiredCategory = .playAndRecord @@ -217,10 +212,12 @@ public class SimpleAudioReceiver: NSObject { // --- 路由设置 --- if audioSourceType == .microphone || isHybridExternalMode { // 为需要麦克风的场景设置输入输出 - if isHeadphones { - try audioSession.overrideOutputAudioPort(.none) // 允许耳机/蓝牙输出 - } else { - try audioSession.overrideOutputAudioPort(.speaker) // 强制外放 + if desiredCategory == .playAndRecord { + if isHeadphones { + try audioSession.overrideOutputAudioPort(.none) // 允许耳机/蓝牙输出 + } else { + try audioSession.overrideOutputAudioPort(.speaker) // 强制外放 + } } // 设置首选输入设备 @@ -248,7 +245,7 @@ public class SimpleAudioReceiver: NSObject { } } } else { // 纯外部源 - try audioSession.overrideOutputAudioPort(.none) + // .overrideOutputAudioPort 仅对 .playAndRecord 有效，.playback 会触发 -50 } } @@ -280,7 +277,8 @@ public class SimpleAudioReceiver: NSObject { out.portType == .bluetoothA2DP || out.portType == .bluetoothHFP || out.portType == .bluetoothLE || - out.portType == .headphones + out.portType == .headphones || + out.portType == .headsetMic } } From e14efd859d6f60109e31c2add6c6daeb3aba5182 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Sat, 24 Jan 2026 10:22:59 +0800 Subject: [PATCH 17/25] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E8=AF=AD?= =?UTF-8?q?=E9=9F=B3=E8=AF=86=E5=88=AB=E4=B8=8ETTS=E6=92=AD=E6=94=BE?= =?UTF-8?q?=E7=9A=84=E9=9F=B3=E9=A2=91=E4=BC=9A=E8=AF=9D=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复iOS端音频会话冲突问题，避免语音识别时TTS仍在播放导致无声或异常。 - 注释掉AgentServiceImpl中初始化时设置识别模式和音频会话的代码，避免干扰 - 调整translation_controller中ASR服务启动逻辑，同时传译模式使用phone_call，面对面模式使用push_to_talk - 修复AzureTtsHelper中播放状态管理，增加播放失败处理和状态重置 - 重新启用音频路由变化处理，当检测到音频会话切换到录音时立即停止TTS播放 - 添加调试日志便于问题追踪 --- .../controllers/translation_controller.dart | 8 +- .../agent_service/AgentServiceImpl.swift | 4 +- .../Sources/azure_speech/AzureTtsHelper.swift | 87 +++++++------------ .../Sources/tools/SimpleAudioReceiver.swift | 9 +- 4 files changed, 46 insertions(+), 62 deletions(-) diff --git a/lib/modules/translation/controllers/translation_controller.dart b/lib/modules/translation/controllers/translation_controller.dart index c1d57845b..808a35bb4 100644 --- a/lib/modules/translation/controllers/translation_controller.dart +++ b/lib/modules/translation/controllers/translation_controller.dart @@ -957,10 +957,14 @@ class TranslationController extends GetxController with WidgetsBindingObserver { return; } if (currentMode.value == 'simultaneous') { - await _startAsrService('push_to_talk'); - } else if (currentMode.value == 'faceToFace') { + Logger.info('liwei-------- 开始语音识别，模式：同声传译，ASR phone_call'); await _startAsrService('phone_call'); + } else if (currentMode.value == 'faceToFace') { + Logger.info('iwei-------- 开始语音识别，模式：面对面，ASR push_to_talk'); + await _startAsrService('push_to_talk'); } else { + Logger.info( + 'iwei-------- 开始语音识别，模式：${currentMode.value}，ASR 模式：normal'); await _startAsrService('normal'); } await _finalizeRecognitionStart(); diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 61f5e4d8a..1f1be4a8f 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -313,8 +313,8 @@ class AgentServiceImpl: NSObject { LocationService.shared.getCurrentLocation() { location in // self.locationInfo = location } - currentRecognitionMode = "normal" - checkAdnSetAudioSession() + // currentRecognitionMode = "normal" + // checkAdnSetAudioSession() return isInitialized } diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index 68ae36cd2..cb1be7642 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -549,7 +549,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { } speaking = true // lastSpokenText = text - // os_log("调用链: performSynthesis 准备合成 session=%{public}@ pushStream=%{public}@ suppress=%{public}@ textLen=%{public}d", log: log, type: .info, sessionid, isUsingPushStreamCapture.description, suppressStreamPlayback.description, text.count) + os_log("liwei----------- 调用链: performSynthesis 准备合成 session=%{public}@ pushStream=%{public}@ suppress=%{public}@ textLen=%{public}d", log: log, type: .info, sessionid, isUsingPushStreamCapture.description, suppressStreamPlayback.description, text.count) if !applyAudioSessionForTTS() { speaking = false notifyEvent(eventType: .error, params: [ @@ -1088,6 +1088,10 @@ private func checkStreamPlaybackCompletedIfNeeded() { * - Throws: 无 */ private func startPlaybackIfNeeded() { + if isPlaying, audioPlayer?.isPlaying != true { + isPlaying = false + audioPlayer = nil + } guard !isPlaying, !playbackQueue.isEmpty else { return } let next = playbackQueue.first! let playableData = ensureWavDataForPlayback(next) @@ -1106,15 +1110,20 @@ private func checkStreamPlaybackCompletedIfNeeded() { } audioPlayer?.delegate = self audioPlayer?.prepareToPlay() - isPlaying = true - // os_log("调用链: 开始播放 队首长度=%{public}dB session=%{public}@", log: log, type: .info, next.count, sessionid) - notifyEvent(eventType: .playbackStarted) - audioPlayer?.play() + let started = audioPlayer?.play() == true + if started { + isPlaying = true + // os_log("调用链: 开始播放 队首长度=%{public}dB session=%{public}@", log: log, type: .info, next.count, sessionid) + notifyEvent(eventType: .playbackStarted) + } else { + throw NSError(domain: "AzureTtsHelper", code: -1, userInfo: [NSLocalizedDescriptionKey: "AVAudioPlayer.play() 返回 false"]) + } } catch { os_log("播放初始化失败: %{public}@", log: log, type: .error, error.localizedDescription) // 异常情况下移除该条并尝试播放下一条 playbackQueue.removeFirst() isPlaying = false + audioPlayer = nil startPlaybackIfNeeded() } } @@ -1415,6 +1424,7 @@ private func ensureAudioSessionQueueSpecificKeySet() { */ private func applyAudioSessionForTTS() -> Bool { var success = true + os_log("liwei-------------TTS applyAudioSessionForTTS currentRecognitionMode:%{public}@", log: self.log, type: .info, currentRecognitionMode) ensureAudioSessionQueueSpecificKeySet() let applyBody = { if self.isApplyingAudioSessionConfig { return } @@ -1648,55 +1658,24 @@ private func deactivateAudioSessionAfterTTSIfIdle() { * - 配置过程中（isApplyingAudioSessionConfig = true）直接忽略 */ @objc private func handleRouteChange(_ notification: Notification) { - // guard let info = notification.userInfo, - // let reasonValue = info[AVAudioSessionRouteChangeReasonKey] as? UInt, - // let reason = AVAudioSession.RouteChangeReason(rawValue: reasonValue) else { return } - - // if !isTtsBusy() { return } - // os_log("音频路由变化: %{public}@", log: log, type: .info, String(describing: reason)) - - // // 配置中触发的路由变化，直接忽略，避免重入导致死循环 - // if isApplyingAudioSessionConfig { - // os_log("配置进行中，忽略本次路由变化", log: log, type: .debug) - // return - // } - - // // 忽略会造成循环的原因：categoryChange（你的日志 reason=3），以及 routeConfigurationChange 等 - // switch reason { - // case .categoryChange, .routeConfigurationChange, .wakeFromSleep, .noSuitableRouteForCategory: - // os_log("忽略路由变化原因: %{public}@", log: log, type: .debug, String(describing: reason)) - // return - // default: - // break - // } - - // // 通话期间避免重配，避免与电话路由冲突 - // if isTelephonyActive() { - // os_log("通话期间检测到路由变化，跳过重配", log: log, type: .info) - // return - // } - - // let isActivelyPlayingNow = - // speaking || - // isPlaying || - // (audioPlayer?.isPlaying == true) || - // (playerNode?.isPlaying == true) - // if isActivelyPlayingNow { - // routeChangeDebounceWorkItem?.cancel() - // _ = stop() - // os_log("路由变化时正在播报，已停止以避免音源冲突", log: log, type: .info) - // return - // } - - // 去抖处理：300ms 内只执行最后一次重配 - // 忽略 categoryChange 等 - // 去抖：在 audioSessionQueue 上延迟调用 - // routeChangeDebounceWorkItem?.cancel() - // let work = DispatchWorkItem { [weak self] in - // self?.safeConfigureAudioSessionForTTS() - // } - // routeChangeDebounceWorkItem = work - // audioSessionQueue.asyncAfter(deadline: .now() + 0.3, execute: work) + guard let info = notification.userInfo, + let reasonValue = info[AVAudioSessionRouteChangeReasonKey] as? UInt, + let reason = AVAudioSession.RouteChangeReason(rawValue: reasonValue) else { return } + + if reason != .categoryChange { return } + + let session = AVAudioSession.sharedInstance() + if session.category != .record { return } + + let isActivelyPlayingNow = + speaking || + isPlaying || + (audioPlayer?.isPlaying == true) || + (playerNode?.isPlaying == true) + if !isActivelyPlayingNow { return } + + _ = stop() + os_log("检测到音频会话切到录音，停止TTS以避免冲突", log: log, type: .info) } /** diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift index 3522a1630..364e0dd9f 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift @@ -135,7 +135,7 @@ public class SimpleAudioReceiver: NSObject { _isWriting = true - print("音源设置---------- startAudioRecord=mod:\(mod), audioSourceType:\(audioSourceType), isHeadphones:\(isHeadphones)") + print("liwei---------- startAudioRecord=mod:\(mod), audioSourceType:\(audioSourceType), isHeadphones:\(isHeadphones)") do { try configureAudioSession() @@ -284,11 +284,12 @@ public class SimpleAudioReceiver: NSObject { /** * 开启音频写入线程 - * 修复：使用userInitiated QoS避免优先级反转 + * 参数：无 + * 返回值：无 + * 异常：无 */ public func startWriteThread() { - // 使用userInitiated QoS匹配音频录制线程的优先级 - writeThread = DispatchQueue(label: "audio.stream.writer", qos: .userInitiated) + writeThread = DispatchQueue(label: "audio.stream.writer", qos: .default) writeThread?.async { [weak self] in guard let self = self else { return } From 7c647e2a8e8e686d1a6b1898599cc21e9e39fd92 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Sat, 24 Jan 2026 14:55:09 +0800 Subject: [PATCH 18/25] =?UTF-8?q?fix(azure=5Fspeech):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E8=AF=AD=E9=9F=B3=E5=90=88=E6=88=90=E4=BC=9A=E8=AF=9D=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复 Azure TTS 助手中的会话管理问题，确保音频合成和播放仅在正确的会话上下文中执行。移除冗余的会话 ID 处理逻辑，并添加会话快照检查以防止跨会话音频数据干扰。 具体修改包括： - 在合成开始和音频处理事件中添加会话 ID 一致性检查 - 移除 `effectiveSessionId` 计算逻辑，直接使用传入的 sessionid - 在停止合成时清空会话快照 - 注释调试日志以减少控制台输出 --- .../azure_speech/AzureSpeechPlugin.swift | 2 +- .../Sources/azure_speech/AzureTtsHelper.swift | 55 +++++++++++++++---- pubspec.yaml | 2 +- 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift index ce82256b1..1a00c6ef8 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift @@ -1116,7 +1116,7 @@ extension AzureSpeechPlugin: BleService.Callback { } } - print("分离音频数据 - 左声道: \(leftBuffer.count) bytes, 右声道: \(rightBuffer.count) bytes") + // print("分离音频数据 - 左声道: \(leftBuffer.count) bytes, 右声道: \(rightBuffer.count) bytes") // 安全解包版本 guard let audioStream = azureAsrHelper.audioStream else { diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index cb1be7642..4160e2345 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -18,6 +18,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { private var isInitialized = false private var speaking = false private var sessionid = "" //语音播报任务id + private var synthesisSessionIdSnapshot = "" // 当前配置 private var currentVoice = DEFAULT_VOICE @@ -192,10 +193,10 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { return false } - let effectiveSessionId = sessionid.isEmpty ? self.sessionid : sessionid - if !sessionid.isEmpty { - self.sessionid = sessionid - } + // let effectiveSessionId = sessionid.isEmpty ? self.sessionid : sessionid + // if !sessionid.isEmpty { + self.sessionid = sessionid + // } // os_log("调用链: speakOnce 接收 session=%{public}@ effective=%{public}@ speaking=%{public}@ pendingTasks=%{public}d", log: log, type: .info, sessionid, effectiveSessionId, speaking.description, pendingTasks.count) // 清理文本 @@ -208,7 +209,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { // 异步处理 // os_log("调用链: speakOnce 入队合成 session=%{public}@ 文本长度=%{public}d", log: log, type: .info, effectiveSessionId, cleanedText.count) enqueueSynthesisTask { - self.performSynthesis(sessionid: effectiveSessionId, text: cleanedText) + self.performSynthesis(sessionid: sessionid, text: cleanedText) } return true @@ -225,9 +226,10 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { ]) return false } - let effectiveSessionId = sessionid.isEmpty ? self.sessionid : sessionid - if !sessionid.isEmpty { - self.sessionid = sessionid + // let effectiveSessionId = sessionid.isEmpty ? self.sessionid : sessionid + // if !sessionid.isEmpty { + if self.sessionid != sessionid { + return false } // os_log("调用链: speakStream 接收 session=%{public}@ effective=%{public}@ 输入长度=%{public}d bufferLen=%{public}d", log: log, type: .info, sessionid, effectiveSessionId, text.count, streamBuffer.count) @@ -279,7 +281,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { if !textToSpeak.isEmpty { // os_log("调用链: speakStream 触发合成 session=%{public}@ speakLen=%{public}d remainLen=%{public}d", log: log, type: .info, effectiveSessionId, textToSpeak.count, streamBuffer.count) - return speakOnce(sessionid: effectiveSessionId, textToSpeak) + return speakOnce(sessionid: sessionid, textToSpeak) } } @@ -289,7 +291,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { let startIndex = currentText.index(currentText.startIndex, offsetBy: maxCharsBeforeForcedSpeak) streamBuffer = String(currentText[startIndex...]) // os_log("调用链: speakStream 强制分段合成 session=%{public}@ speakLen=%{public}d remainLen=%{public}d", log: log, type: .info, effectiveSessionId, textToSpeak.count, streamBuffer.count) - return speakOnce(sessionid: effectiveSessionId, textToSpeak) + return speakOnce(sessionid: sessionid, textToSpeak) } // os_log("调用链: speakStream 未触发合成 session=%{public}@ bufferLen=%{public}d", log: log, type: .debug, effectiveSessionId, streamBuffer.count) @@ -307,6 +309,9 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { ]) return false } + if self.sessionid != sessionid { + return false + } // os_log("调用链: flushStream session=%{public}@ bufferLen=%{public}d", log: log, type: .info, sessionid, streamBuffer.count) let remainingText = streamBuffer @@ -326,8 +331,10 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { * - Throws: 无（内部捕获 SDK/音频会话异常并转为 error 事件） */ public func stop() -> Bool { + os_log("liwei------------- 停止语音合成 session=%{public}@", log: self.log, type: .info, sessionid) speaking = false sessionid = "" + synthesisSessionIdSnapshot = "" streamBuffer = "" lastSpokenText = nil @@ -356,6 +363,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { taskLock.unlock() do { + try synthesizer?.stopSpeaking() self.notifyEvent(eventType: .synthesisCanceled) } catch { @@ -563,9 +571,10 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { do { if (self.sessionid != sessionid) { - // os_log("调用链: performSynthesis 中止(会话变化) current=%{public}@ task=%{public}@", log: log, type: .info, self.sessionid, sessionid) + os_log("liwei----------- 调用链: performSynthesis 中止(会话变化) current=%{public}@ task=%{public}@", log: log, type: .info, self.sessionid, sessionid) return } + synthesisSessionIdSnapshot = sessionid // os_log("调用链: performSynthesis startSpeakingSsml session=%{public}@ ssmlLen=%{public}d", log: log, type: .info, sessionid, ssml.count) let result = try synthesizer?.startSpeakingSsml(ssml) @@ -576,7 +585,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { speaking = false // 减少待处理文本计数 pendingTextCount = max(0, pendingTextCount - text.count) - os_log("语音合成失败: %{public}@", log: log, type: .error, error.localizedDescription) + os_log("liwei-------------- 语音合成失败: %{public}@", log: log, type: .error, error.localizedDescription) notifyEvent(eventType: .error, params: [ "errorCode": "SYNTHESIS_FAILED", "errorMessage": error.localizedDescription @@ -653,6 +662,7 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { guard let self = self else { return } self.currentSynthesisBuffer = Data() if self.suppressStreamPlayback { return } + self.synthesisSessionIdSnapshot = self.sessionid if self.isUsingPushStreamCapture { self.audioPlaybackQueue.async { self.activeStreamSynthesisCount += 1 @@ -669,6 +679,9 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { synthesizer?.addSynthesizingEventHandler { [weak self] _, event in if let audioData = event.result.audioData { guard let self = self else { return } + if self.synthesisSessionIdSnapshot != self.sessionid { + return + } // os_log("调用链: 合成中 接收音频片段=%{public}dB session=%{public}@", log: self.log, type: .debug, audioData.count, self.sessionid) if self.isUsingPushStreamCapture { if self.pushStreamCallbackCount == 0, !audioData.isEmpty { @@ -691,6 +704,18 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { if !self.isUsingPushStreamCapture { self.speaking = false } + if self.synthesisSessionIdSnapshot != self.sessionid { + self.currentSynthesisBuffer = Data() + if self.isUsingPushStreamCapture { + self.audioPlaybackQueue.async { + self.activeStreamSynthesisCount = max(0, self.activeStreamSynthesisCount - 1) + self.checkStreamPlaybackCompletedIfNeeded() + } + } + self.pendingTextCount = max(0, self.pendingTextCount - 1) + self.notifyEvent(eventType: .synthesisCompleted) + return + } // 将当前合成的音频加入播放队列 if !self.currentSynthesisBuffer.isEmpty { // os_log("调用链: 合成完成 入队播放 数据长度=%{public}dB session=%{public}@", log: self.log, type: .info, self.currentSynthesisBuffer.count, self.sessionid) @@ -771,6 +796,9 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { // os_log("调用链: 推送音频丢弃(无会话且未speaking) bytes=%{public}d", log: log, type: .debug, data.count) return } + if synthesisSessionIdSnapshot != sessionid { + return + } notifyAudioData(data) guard isUsingPushStreamCapture else { // os_log("调用链: 推送音频忽略(未启用pushStream) bytes=%{public}d", log: log, type: .debug, data.count) @@ -790,6 +818,9 @@ public class AzureTtsHelper: NSObject, ITtsService, AVAudioPlayerDelegate { // os_log("调用链: 推送音频异步丢弃(无会话且未speaking) bytes=%{public}d", log: self.log, type: .debug, data.count) return } + if self.synthesisSessionIdSnapshot != self.sessionid { + return + } let pcm = self.stripWavHeaderIfNeeded(data) if !pcm.isEmpty { self.pcmPendingData.append(pcm) diff --git a/pubspec.yaml b/pubspec.yaml index e7d6eb828..32e746e95 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,7 +1,7 @@ name: voitrans description: "Voitrans - AI Voice Assistant." publish_to: "none" -version: 1.0.21+71 +version: 1.0.22+71 environment: sdk: ">=3.3.0 <4.0.0" From 7328831a25e4ed964a3f7544fb72efae5560ea67 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Sat, 24 Jan 2026 16:21:07 +0800 Subject: [PATCH 19/25] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E8=AF=AD?= =?UTF-8?q?=E9=9F=B3=E8=AF=86=E5=88=AB=E6=A0=87=E7=82=B9=E5=A4=84=E7=90=86?= =?UTF-8?q?=E5=8F=8AMCP=E8=BF=9E=E6=8E=A5=E7=A8=B3=E5=AE=9A=E6=80=A7?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复语音识别在非push_to_talk模式下未移除首标点的问题，提升听写体验 优化MCP客户端连接管理，增加心跳检测和连接状态主动检查，避免假死连接 移除调试日志语句，避免生产环境日志污染 --- .../controllers/translation_controller.dart | 20 ++-- .../Sources/azure_speech/AzureTtsHelper.swift | 2 +- .../chat_api/Sources/chat_api/MCPClient.swift | 91 +++++++++++++++++-- 3 files changed, 98 insertions(+), 15 deletions(-) diff --git a/lib/modules/translation/controllers/translation_controller.dart b/lib/modules/translation/controllers/translation_controller.dart index 808a35bb4..d8766bd9e 100644 --- a/lib/modules/translation/controllers/translation_controller.dart +++ b/lib/modules/translation/controllers/translation_controller.dart @@ -445,7 +445,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } await _asrService.startContinuousRecognition( _audioSourceType, - isRemoveFirstPunctuation: false, + isRemoveFirstPunctuation: true, mode: 'push_to_talk', ); //开启识别 _startAsrActiveTracking(); //开启识别活动跟踪 @@ -1066,11 +1066,19 @@ class TranslationController extends GetxController with WidgetsBindingObserver { /// 启动语音识别 Future _startAsrService(String mode) async { - await _asrService.startContinuousRecognition( - _audioSourceType, - isRemoveFirstPunctuation: false, - mode: mode, - ); + if (mode == 'push_to_talk') { + await _asrService.startContinuousRecognition( + _audioSourceType, + isRemoveFirstPunctuation: false, + mode: mode, + ); + } else { + await _asrService.startContinuousRecognition( + _audioSourceType, + isRemoveFirstPunctuation: true, + mode: mode, + ); + } } /// 完成识别启动 diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index 4160e2345..6e5d516ab 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -1455,7 +1455,7 @@ private func ensureAudioSessionQueueSpecificKeySet() { */ private func applyAudioSessionForTTS() -> Bool { var success = true - os_log("liwei-------------TTS applyAudioSessionForTTS currentRecognitionMode:%{public}@", log: self.log, type: .info, currentRecognitionMode) + // os_log("liwei-------------TTS applyAudioSessionForTTS currentRecognitionMode:%{public}@", log: self.log, type: .info, currentRecognitionMode) ensureAudioSessionQueueSpecificKeySet() let applyBody = { if self.isApplyingAudioSessionConfig { return } diff --git a/local_plugins/chat_api/ios/chat_api/Sources/chat_api/MCPClient.swift b/local_plugins/chat_api/ios/chat_api/Sources/chat_api/MCPClient.swift index c2da48b8d..5d7146d1a 100644 --- a/local_plugins/chat_api/ios/chat_api/Sources/chat_api/MCPClient.swift +++ b/local_plugins/chat_api/ios/chat_api/Sources/chat_api/MCPClient.swift @@ -36,9 +36,11 @@ public class MCPSubClient { // 使用官方MCP Swift SDK private var mcpClient: Client? + private var transport: CustomSseClientTransport? private var tools: [Tool] = [] private var toolMaps: [[String: Any]] = [] private var isConnectedFlag = false + private let toolsLock = NSLock() // 重连配置 private let maxRetryAttempts = 3 @@ -51,6 +53,10 @@ public class MCPSubClient { // 工具调用配置 - 添加这两行 private let maxToolCallRetries = 3 private let toolCallTimeout: TimeInterval = 30.0 + + private let keepAliveInterval: TimeInterval = 25.0 + private let keepAliveTimeout: TimeInterval = 8.0 + private var keepAliveTask: Task? // 连接状态锁 private let connectionLock = NSLock() @@ -89,6 +95,7 @@ public class MCPSubClient { } } ) + self.transport = transport // 3. 连接到服务器（添加超时） try await withTimeout(seconds: 10) { @@ -106,6 +113,7 @@ public class MCPSubClient { isConnectedFlag = true retryCount = 0 // 重置重试计数 currentReconnectDelay = initialReconnectDelay // 重置延迟 + startKeepAlive() return true @@ -116,6 +124,9 @@ public class MCPSubClient { } private func processTools(_ toolList: [Tool]) { + toolsLock.lock() + defer { toolsLock.unlock() } + tools.removeAll() toolMaps.removeAll() @@ -173,6 +184,13 @@ public class MCPSubClient { /// 检查连接状态 /// 检查连接状态 public func checkConnection() async -> Bool { + if isConnectedFlag, let transport { + let isActive = await transport.isConnectionActive() + if !isActive { + isConnectedFlag = false + } + } + if !isConnectedFlag { return await connect() } @@ -339,14 +357,25 @@ public class MCPSubClient { } public func getToolMaps() -> [[String: Any]] { + toolsLock.lock() + defer { toolsLock.unlock() } + return toolMaps } public func containsTool(name: String) -> Bool { + toolsLock.lock() + defer { toolsLock.unlock() } + return tools.contains { $0.name == name } } public func close() async { + keepAliveTask?.cancel() + keepAliveTask = nil + reconnectTask?.cancel() + reconnectTask = nil + if let client = mcpClient { await client.disconnect() } @@ -359,9 +388,9 @@ public class MCPSubClient { if isConnectedFlag { print("[\(serverId)] 连接已断开，更新状态") isConnectedFlag = false - - // 启动重连逻辑 - // startReconnection() + keepAliveTask?.cancel() + keepAliveTask = nil + startReconnection() } } } @@ -369,7 +398,9 @@ public class MCPSubClient { /// 启动重连 private func startReconnection() { // 取消之前的重连任务 - reconnectTask?.cancel() + if reconnectTask != nil { + return + } reconnectTask = Task { [weak self] in await self?.performReconnection() @@ -378,6 +409,8 @@ public class MCPSubClient { /// 执行重连 private func performReconnection() async { + defer { reconnectTask = nil } + while retryCount < maxRetryAttempts && !Task.isCancelled { retryCount += 1 @@ -403,6 +436,8 @@ public class MCPSubClient { private func handleConnectionError(_ error: Error) async { print("[\(serverId)] 连接错误: \(error.localizedDescription)") isConnectedFlag = false + keepAliveTask?.cancel() + keepAliveTask = nil // 如果是MCP特定错误，可以进行特殊处理 if let mcpError = error as? MCPError { @@ -411,9 +446,49 @@ public class MCPSubClient { } // 如果还有重试机会，启动重连 - // if retryCount < maxRetryAttempts { - // startReconnection() - // } + startReconnection() + } + + private func startKeepAlive() { + keepAliveTask?.cancel() + + keepAliveTask = Task { [weak self] in + guard let self else { return } + + while !Task.isCancelled { + try? await Task.sleep(for: .seconds(self.keepAliveInterval)) + + if Task.isCancelled { + return + } + + if !self.isConnectedFlag { + continue + } + + if let transport = self.transport { + let isActive = await transport.isConnectionActive() + if !isActive { + await self.handleConnectionLost() + continue + } + } + + guard let client = self.mcpClient else { + await self.handleConnectionError(MCPError.internalError("MCP client not initialized")) + continue + } + + do { + let (toolList, _) = try await withTimeout(seconds: self.keepAliveTimeout) { + try await client.listTools() + } + self.processTools(toolList) + } catch { + await self.handleConnectionError(error) + } + } + } } /// 获取连接状态 @@ -665,7 +740,7 @@ public class MCPClient { } public func isConnected() -> Bool { - return !subClients.isEmpty && subClients.values.allSatisfy { $0.getConnectionStatus() } + return !subClients.isEmpty && subClients.values.contains { $0.getConnectionStatus() } } public func disconnectAll() async { From 71353a9bead79101b163da9fe7dcef1ecf756f82 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Mon, 26 Jan 2026 09:29:30 +0800 Subject: [PATCH 20/25] =?UTF-8?q?fix(ios):=20=E4=BF=AE=E5=A4=8D=E8=80=B3?= =?UTF-8?q?=E6=9C=BA=E6=A3=80=E6=B5=8B=E9=80=BB=E8=BE=91=E4=BB=A5=E6=AD=A3?= =?UTF-8?q?=E7=A1=AE=E5=BA=94=E7=94=A8=E9=9F=B3=E9=A2=91=E4=BC=9A=E8=AF=9D?= =?UTF-8?q?=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 重构耳机检测逻辑，使用统一的 `detectHeadphonesOutput` 方法替代分散的端口类型检查，提高代码可维护性并确保音频会话配置在不同耳机类型下的一致性。 --- .../ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index 6e5d516ab..ceb195a59 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -1467,7 +1467,7 @@ private func applyAudioSessionForTTS() -> Bool { let hasWiredHeadphones = outputs.contains(where: { $0.portType == .headphones || $0.portType == .headsetMic }) let hasBluetoothA2DP = outputs.contains(where: { $0.portType == .bluetoothA2DP || $0.portType == .bluetoothLE }) let hasBluetoothHFP = outputs.contains(where: { $0.portType == .bluetoothHFP }) - let hasHeadphones = hasWiredHeadphones || hasBluetoothA2DP || hasBluetoothHFP + let hasHeadphones = self.detectHeadphonesOutput() let otherAudioPlaying = audioSession.isOtherAudioPlaying let isRecording = (self.micCapture?.isCapturing == true) self.isHeadphones = hasHeadphones From a032223676dacce0010618b4465df8a272a51465 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Mon, 26 Jan 2026 11:29:51 +0800 Subject: [PATCH 21/25] =?UTF-8?q?fix(=E8=AF=AD=E9=9F=B3=E8=AF=86=E5=88=AB)?= =?UTF-8?q?:=20=E4=BF=AE=E5=A4=8D=E4=BB=8E=E8=93=9D=E7=89=99=E5=94=A4?= =?UTF-8?q?=E9=86=92=E6=A8=A1=E5=BC=8F=E5=88=87=E6=8D=A2=E6=97=B6=E7=BC=96?= =?UTF-8?q?=E7=A0=81=E5=99=A8=E6=9C=AA=E5=85=B3=E9=97=AD=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在 startRecognition 方法中，当上一次识别模式为 ble_wakeup 而当前模式不是 ble_wakeup 时，主动调用 BleService.shared.closeCodec() 关闭蓝牙编码器，避免资源泄漏和状态不一致。同时移除冗余的 AudioSessionHub 调用并调整代码格式。 --- .../agent_service/AgentServiceImpl.swift | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 1f1be4a8f..8e3bf9d4a 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -103,6 +103,7 @@ class AgentServiceImpl: NSObject { private let maxIdleSeconds: TimeInterval = 5 var micCapture: MicrophoneCapture! // 语音识别模式 + private var lastRecognitionMode: String = "normal" public var currentRecognitionMode: String = "normal" // 识别结果 private var recognitionResult = "" @@ -488,7 +489,7 @@ class AgentServiceImpl: NSObject { startIdleCheckForMode(mode: currentRecognitionMode) } } - func startRecognition(useBle: Bool = false, mode: String = "normal") -> Bool { + func startRecognition(useBle: Bool = false, mode: String = "normal") -> Bool { // 接收到唤醒信号，打开编码器 (设备侧) print("ai启动语音\(useBle))") var openResult = false // 添加openResult变量定义 @@ -507,7 +508,12 @@ class AgentServiceImpl: NSObject { sendError("无麦克风权限", code: "PERMISSION_DENIED") return false } - } + } + lastRecognitionMode = currentRecognitionMode + if (lastRecognitionMode == "ble_wakeup" && mode != "ble_wakeup"){//之前是唤醒模式，现在不是唤醒模式 + os_log("liwei------- 关闭唤醒通道", log: logger, type: .info) + BleService.shared.closeCodec() + } stopRecognition() if isAiStreaming { isAiStreaming = false @@ -523,19 +529,12 @@ class AgentServiceImpl: NSObject { if MusicService.shared.isPlaying { MusicService.shared.pause() } - + // 设置当前识别模式 currentRecognitionMode = mode os_log("liwei--------------- 设置语音识别模式: %{public}@", log: logger, type: .info, mode) let audioSourceType: AzureAsrHelper.AudioSourceType = useBle ? .external : .microphone - - if useBle { - // AudioSessionHub.shared.begin(.playback) - } else { - //AudioSessionHub.shared.begin(.voice) - } - // 新增：标记为"启动中"，用于允许 stop 在启动未完成时也能生效 isStartingRecognition = true @@ -551,8 +550,8 @@ class AgentServiceImpl: NSObject { return false } if(useBle){ - isKeepResult = true - } + isKeepResult = true + } // 根据模式启动相应的空闲检测（此处 guard isRecognizing，会在 onSessionStarted 中再次启动） startIdleCheckForMode(mode: mode) From f7e6d8d130ad219e24efdb3f25d431315f0ca83a Mon Sep 17 00:00:00 2001 From: lxm <1274844347@qq.com> Date: Mon, 26 Jan 2026 20:46:31 +0800 Subject: [PATCH 22/25] =?UTF-8?q?Revert=20"fix(=E8=AF=AD=E9=9F=B3=E8=AF=86?= =?UTF-8?q?=E5=88=AB):=20=E4=BF=AE=E5=A4=8D=E4=BB=8E=E8=93=9D=E7=89=99?= =?UTF-8?q?=E5=94=A4=E9=86=92=E6=A8=A1=E5=BC=8F=E5=88=87=E6=8D=A2=E6=97=B6?= =?UTF-8?q?=E7=BC=96=E7=A0=81=E5=99=A8=E6=9C=AA=E5=85=B3=E9=97=AD=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit e3f0f9bccd748b875005180502c6401085001478. --- .../agent_service/AgentServiceImpl.swift | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 8e3bf9d4a..1f1be4a8f 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -103,7 +103,6 @@ class AgentServiceImpl: NSObject { private let maxIdleSeconds: TimeInterval = 5 var micCapture: MicrophoneCapture! // 语音识别模式 - private var lastRecognitionMode: String = "normal" public var currentRecognitionMode: String = "normal" // 识别结果 private var recognitionResult = "" @@ -489,7 +488,7 @@ class AgentServiceImpl: NSObject { startIdleCheckForMode(mode: currentRecognitionMode) } } - func startRecognition(useBle: Bool = false, mode: String = "normal") -> Bool { + func startRecognition(useBle: Bool = false, mode: String = "normal") -> Bool { // 接收到唤醒信号，打开编码器 (设备侧) print("ai启动语音\(useBle))") var openResult = false // 添加openResult变量定义 @@ -508,12 +507,7 @@ class AgentServiceImpl: NSObject { sendError("无麦克风权限", code: "PERMISSION_DENIED") return false } - } - lastRecognitionMode = currentRecognitionMode - if (lastRecognitionMode == "ble_wakeup" && mode != "ble_wakeup"){//之前是唤醒模式，现在不是唤醒模式 - os_log("liwei------- 关闭唤醒通道", log: logger, type: .info) - BleService.shared.closeCodec() - } + } stopRecognition() if isAiStreaming { isAiStreaming = false @@ -529,12 +523,19 @@ class AgentServiceImpl: NSObject { if MusicService.shared.isPlaying { MusicService.shared.pause() } - + // 设置当前识别模式 currentRecognitionMode = mode os_log("liwei--------------- 设置语音识别模式: %{public}@", log: logger, type: .info, mode) let audioSourceType: AzureAsrHelper.AudioSourceType = useBle ? .external : .microphone + + if useBle { + // AudioSessionHub.shared.begin(.playback) + } else { + //AudioSessionHub.shared.begin(.voice) + } + // 新增：标记为"启动中"，用于允许 stop 在启动未完成时也能生效 isStartingRecognition = true @@ -550,8 +551,8 @@ class AgentServiceImpl: NSObject { return false } if(useBle){ - isKeepResult = true - } + isKeepResult = true + } // 根据模式启动相应的空闲检测（此处 guard isRecognizing，会在 onSessionStarted 中再次启动） startIdleCheckForMode(mode: mode) From 51dedaf383c91af2d4b8e02c42417731aacce8be Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Tue, 27 Jan 2026 10:45:48 +0800 Subject: [PATCH 23/25] =?UTF-8?q?feat(opus):=20=E6=B7=BB=E5=8A=A0=20iOS=20?= =?UTF-8?q?Opus=20=E8=A7=A3=E7=A0=81=E6=94=AF=E6=8C=81=E5=B9=B6=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E9=9F=B3=E9=A2=91=E5=BD=95=E5=88=B6=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 jl_opus iOS 插件，提供 Opus 文件解码功能 - 在 ble_service 中添加 Opus 文件解码方法，支持通过 MethodChannel 调用 - 优化音频录制文件管理，添加延迟停止机制避免数据截断 - 修复 agent_service 中音乐播放状态恢复逻辑 - 调整 opus_test 控制器以支持跨平台解码调用 --- .../controllers/opus_test_controller.dart | 34 +- .../agent_service/AgentServiceImpl.swift | 31 +- .../Sources/ble_service/BleService.swift | 74 ++- .../Sources/ble_service/RecordingFile.swift | 2 +- .../ble_service/SwiftBleServicePlugin.swift | 163 +++++++ .../jl_opus/ios/jl_opus/Package.swift | 22 + .../Sources/jl_opus/JlOpusPlugin.swift | 450 ++++++++++++++++++ 7 files changed, 757 insertions(+), 19 deletions(-) create mode 100644 local_plugins/jl_opus/ios/jl_opus/Package.swift create mode 100644 local_plugins/jl_opus/ios/jl_opus/Sources/jl_opus/JlOpusPlugin.swift diff --git a/lib/modules/opus_test/controllers/opus_test_controller.dart b/lib/modules/opus_test/controllers/opus_test_controller.dart index b22cc4b5e..c93f2bac4 100644 --- a/lib/modules/opus_test/controllers/opus_test_controller.dart +++ b/lib/modules/opus_test/controllers/opus_test_controller.dart @@ -4,6 +4,7 @@ import 'dart:typed_data'; import 'package:file_picker/file_picker.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; import 'package:get/get.dart'; import 'package:path_provider/path_provider.dart'; import 'package:just_audio/just_audio.dart'; @@ -42,8 +43,10 @@ class OpusTestController extends GetxController { String? tempPcmPath; // 杰理OPUS解码器 - late JlOpus jlOpus; + JlOpus? jlOpus; StreamSubscription? _eventSubscription; + static const MethodChannel _bleServiceChannel = + MethodChannel('com.yunqiinnovation.ble_service'); @override void onInit() { @@ -56,16 +59,20 @@ class OpusTestController extends GetxController { void onClose() { player.dispose(); _eventSubscription?.cancel(); - jlOpus.dispose(); + jlOpus?.dispose(); super.onClose(); } // 初始化OPUS解码器 Future _initOpusDecoder() async { + if (!Platform.isAndroid) { + return; + } + jlOpus = JlOpus(); // 监听解码器事件 - _eventSubscription = jlOpus.eventStream.listen((event) { + _eventSubscription = jlOpus!.eventStream.listen((event) { switch (event.event) { case 'onStart': statusMessage.value = '开始${event.type == "file" ? "文件" : "流"}解码...'; @@ -86,12 +93,29 @@ class OpusTestController extends GetxController { }); // 初始化OPUS解码器 - final initialized = await jlOpus.initOpusDecoder(); + final initialized = await jlOpus!.initOpusDecoder(); if (!initialized) { statusMessage.value = 'OPUS解码器初始化失败'; } } + Future _decodeOpusFile( + String inPath, String outPath, OpusOption option) async { + if (Platform.isIOS) { + final result = await _bleServiceChannel.invokeMethod( + 'decodeOpusFile', + { + 'inPath': inPath, + 'outPath': outPath, + ...option.toMap(), + }, + ); + return result; + } + + return jlOpus?.decodeOpusFile(inPath, outPath, option); + } + // 请求必要权限 Future requestPermissions() async { if (Platform.isAndroid) { @@ -241,7 +265,7 @@ class OpusTestController extends GetxController { ); // 解码文件 - final pcmPath = await jlOpus.decodeOpusFile(inPath, outPath, option); + final pcmPath = await _decodeOpusFile(inPath, outPath, option); if (pcmPath == null) { statusMessage.value = '解码失败'; isDecoding.value = false; diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 8e3bf9d4a..6b6e90a2b 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -510,10 +510,6 @@ class AgentServiceImpl: NSObject { } } lastRecognitionMode = currentRecognitionMode - if (lastRecognitionMode == "ble_wakeup" && mode != "ble_wakeup"){//之前是唤醒模式，现在不是唤醒模式 - os_log("liwei------- 关闭唤醒通道", log: logger, type: .info) - BleService.shared.closeCodec() - } stopRecognition() if isAiStreaming { isAiStreaming = false @@ -527,7 +523,7 @@ class AgentServiceImpl: NSObject { // 如果音乐正在播放，则暂停 if MusicService.shared.isPlaying { - MusicService.shared.pause() + MusicService.shared.pause() } // 设置当前识别模式 @@ -591,13 +587,13 @@ class AgentServiceImpl: NSObject { let success = azureAsrHelper?.stopContinuousRecognition() ?? false if success { - // 识别结束后，立即停用 AudioSession，以便其他模块（如TTS）可以接管 - // 这是解决“按住说话”后无法播报问题的关键 do { - try AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) - os_log("Audio session deactivated successfully after recognition.", log: logger, type: .info) + // let audioSession = AVAudioSession.sharedInstance() + // try audioSession.setCategory(.playback, mode: .spokenAudio, options: [.mixWithOthers, .allowBluetoothA2DP]) + // try audioSession.setActive(true) + os_log("Audio session switched to playback successfully after recognition.", log: logger, type: .info) } catch { - os_log("Failed to deactivate audio session: %{public}@", log: logger, type: .error, error.localizedDescription) + os_log("Failed to switch audio session to playback: %{public}@", log: logger, type: .error, error.localizedDescription) } } else if isStartingRecognition { // 启动尚未完成，先记录一次待停止请求，onSessionStarted 到来后立即 stop @@ -2438,6 +2434,11 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { if (currentRecognitionMode == "ble_wakeup") { audioPlayer?.playStopSound() } + + if wasMusicPlayingBeforeRecognition { + wasMusicPlayingBeforeRecognition = false + MusicService.shared.resume() + } } @@ -2459,6 +2460,11 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { isStartingRecognition = false stopRequestedDuringStart = false stopIdleCheck() + + if wasMusicPlayingBeforeRecognition { + wasMusicPlayingBeforeRecognition = false + MusicService.shared.resume() + } } func onError(sessionid:String ,_ errorCode: Int, _ error: String) { @@ -2469,6 +2475,11 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { isStartingRecognition = false stopRequestedDuringStart = false stopIdleCheck() + + if wasMusicPlayingBeforeRecognition { + wasMusicPlayingBeforeRecognition = false + MusicService.shared.resume() + } } // MARK: - isInterrupt 缓存方法 diff --git a/local_plugins/ble_service/ios/ble_service/Sources/ble_service/BleService.swift b/local_plugins/ble_service/ios/ble_service/Sources/ble_service/BleService.swift index ad0326ce2..0691e1e88 100644 --- a/local_plugins/ble_service/ios/ble_service/Sources/ble_service/BleService.swift +++ b/local_plugins/ble_service/ios/ble_service/Sources/ble_service/BleService.swift @@ -55,6 +55,10 @@ public class BleService: NSObject { // 添加录制文件管理器 private var recordingFile: RecordingFile? + + private var pendingStopOpusRecording = false + private var stopOpusRecordingWorkItem: DispatchWorkItem? + private var lastOpusPacketReceivedAt: TimeInterval = 0 private var isScanning = false private var scanTimer: Timer? @@ -471,6 +475,7 @@ private var cmdReplyType: UInt8 = 0 // 停止音频发送定时器 stopAudioSendTimer() + stopOpusRecording() // 清空音频缓冲区和队列 audioQueueLock.lock() @@ -743,9 +748,7 @@ private var cmdReplyType: UInt8 = 0 public func closeCodec() -> Bool { os_log("关闭编解码并停止录制...", log: logger, type: .info) - // 停止录制 - recordingFile?.closeFile() - recordingFile = nil + // 停止音频发送定时器 stopAudioSendTimer() @@ -756,6 +759,9 @@ private var cmdReplyType: UInt8 = 0 audioDataQueue.removeAll() audioQueueLock.unlock() + + requestStopOpusRecording() + // 不停止Opus解码流，只发送命令通知设备关闭编解码 let paramData = Data([BleConst.CODEC_CONTROL_CLOSE, BleConst.AUDIO_CHANNEL_RIGHT]) return sendCommand(BleConst.CMD_CONTROL_CODEC, data: paramData) @@ -906,8 +912,69 @@ private var cmdReplyType: UInt8 = 0 return } + lastOpusPacketReceivedAt = Date().timeIntervalSince1970 + + ensureOpusRecordingStarted(fileName: "耳机端").saveAudioData(data) + // 使用Opus处理器处理音频数据 opusProcessor?.processAudioData(data) + + if pendingStopOpusRecording { + requestStopOpusRecording() + } + } + + /// 确保 Opus 文件已创建并处于可写状态 + /// - Parameter fileName: 文件名前缀（不含时间戳与扩展名） + /// - Returns: 可用于追加写入的 RecordingFile 实例 + /// - Throws: 不抛出；若创建文件失败将导致后续写入无效 + private func ensureOpusRecordingStarted(fileName: String) -> RecordingFile { + if recordingFile == nil { + let recorder = RecordingFile() + recorder.fileName = fileName + recordingFile = recorder + } else if (recordingFile?.fileName.isEmpty ?? true) { + recordingFile?.fileName = fileName + } + + if recordingFile?.isRecording() != true { + recordingFile?.createFile() + } + + return recordingFile! + } + + /// 停止并释放 Opus 落盘资源 + /// - Returns: 无 + /// - Throws: 不抛出 + private func stopOpusRecording() { + stopOpusRecordingWorkItem?.cancel() + stopOpusRecordingWorkItem = nil + pendingStopOpusRecording = false + recordingFile?.closeFile() + recordingFile = nil + } + + /// 请求在短暂静默后停止 Opus 落盘，避免截断尾包导致文件不完整 + /// - Parameters: + /// - graceSeconds: 等待静默的时间窗口 + /// - Returns: 无 + /// - Throws: 不抛出 + private func requestStopOpusRecording(graceSeconds: TimeInterval = 1.0) { + pendingStopOpusRecording = true + + stopOpusRecordingWorkItem?.cancel() + let workItem = DispatchWorkItem { [weak self] in + guard let self = self else { return } + let now = Date().timeIntervalSince1970 + if now - self.lastOpusPacketReceivedAt >= graceSeconds { + self.stopOpusRecording() + } else { + self.requestStopOpusRecording(graceSeconds: graceSeconds) + } + } + stopOpusRecordingWorkItem = workItem + DispatchQueue.main.asyncAfter(deadline: .now() + graceSeconds, execute: workItem) } /// 处理接收到的响应数据 @@ -1786,6 +1853,7 @@ extension BleService: CBCentralManagerDelegate { } updateConnectionState(BleConst.STATE_DISCONNECTED) + stopOpusRecording() // 只有在非主动断开的情况下才重新连接 if !isManualDisconnect { diff --git a/local_plugins/ble_service/ios/ble_service/Sources/ble_service/RecordingFile.swift b/local_plugins/ble_service/ios/ble_service/Sources/ble_service/RecordingFile.swift index 6b5824dfc..1942bcc0c 100644 --- a/local_plugins/ble_service/ios/ble_service/Sources/ble_service/RecordingFile.swift +++ b/local_plugins/ble_service/ios/ble_service/Sources/ble_service/RecordingFile.swift @@ -78,7 +78,7 @@ class RecordingFile { do { try self.fileHandle?.write(contentsOf: buffer) self.totalBytesWritten += buffer.count - os_log("写入音频数据: %d bytes, 总计: %d bytes", log: self.logger, type: .debug, buffer.count, self.totalBytesWritten) + // os_log("写入音频数据: %d bytes, 总计: %d bytes", log: self.logger, type: .debug, buffer.count, self.totalBytesWritten) } catch { os_log("写入音频数据失败: %@", log: self.logger, type: .error, error.localizedDescription) } diff --git a/local_plugins/ble_service/ios/ble_service/Sources/ble_service/SwiftBleServicePlugin.swift b/local_plugins/ble_service/ios/ble_service/Sources/ble_service/SwiftBleServicePlugin.swift index c75f8bf5d..ca6086159 100644 --- a/local_plugins/ble_service/ios/ble_service/Sources/ble_service/SwiftBleServicePlugin.swift +++ b/local_plugins/ble_service/ios/ble_service/Sources/ble_service/SwiftBleServicePlugin.swift @@ -2,6 +2,7 @@ import Flutter import UIKit import CoreBluetooth import os +import opus @available(iOS 13.0, *) @objc(BleServicePlugin) @@ -172,6 +173,40 @@ public class SwiftBleServicePlugin: NSObject, FlutterPlugin { // 移除这个方法调用，让iOS系统自然处理权限 result(FlutterMethodNotImplemented) + case "decodeOpusFile": + guard let args = call.arguments as? [String: Any], + let inPath = args["inPath"] as? String, + let outPath = args["outPath"] as? String else { + result(FlutterError(code: "INVALID_ARGS", message: "缺少必要参数", details: nil)) + return + } + + let hasHeader = (args["hasHeader"] as? Bool) ?? false + let channel = Int32((args["channel"] as? Int) ?? 1) + let sampleRate = Int32((args["sampleRate"] as? Int) ?? 16000) + let packetSize = (args["packetSize"] as? Int) ?? 40 + + DispatchQueue.global(qos: .userInitiated).async { + do { + let output = try OpusFileDecoder.decodeFileToPcm( + inPath: inPath, + outPath: outPath, + hasHeader: hasHeader, + channel: channel, + sampleRate: sampleRate, + packetSize: packetSize + ) + + DispatchQueue.main.async { + result(output) + } + } catch { + DispatchQueue.main.async { + result(FlutterError(code: "DECODE_ERROR", message: "解码失败: \(error)", details: nil)) + } + } + } + default: result(FlutterMethodNotImplemented) } @@ -187,6 +222,134 @@ public class SwiftBleServicePlugin: NSObject, FlutterPlugin { } } +@available(iOS 13.0, *) +private enum OpusFileDecoder { + /// 将 Opus 文件（按固定 packetSize 切分的裸包流）解码为 PCM，并写入 outPath + /// - Parameters: + /// - inPath: 输入 Opus 文件路径 + /// - outPath: 输出 PCM 文件路径 + /// - hasHeader: 是否包含协议头（当前仅支持 false） + /// - channel: 通道数（1 或 2） + /// - sampleRate: 采样率 + /// - packetSize: 固定包长（若不确定，会与 40/80 一起自动尝试） + /// - Returns: 输出文件路径（等于 outPath） + /// - Throws: 参数不合法、文件读写失败、或解码输出为空时抛出 + static func decodeFileToPcm( + inPath: String, + outPath: String, + hasHeader: Bool, + channel: Int32, + sampleRate: Int32, + packetSize: Int + ) throws -> String { + if hasHeader { + throw NSError(domain: "ble_service", code: -10, userInfo: [NSLocalizedDescriptionKey: "暂不支持带协议头的 Opus 文件解码"]) + } + if channel != 1 && channel != 2 { + throw NSError(domain: "ble_service", code: -11, userInfo: [NSLocalizedDescriptionKey: "channel 仅支持 1 或 2"]) + } + if sampleRate <= 0 { + throw NSError(domain: "ble_service", code: -12, userInfo: [NSLocalizedDescriptionKey: "sampleRate 必须大于 0"]) + } + + let inputData = try Data(contentsOf: URL(fileURLWithPath: inPath)) + if inputData.isEmpty { + throw NSError(domain: "ble_service", code: -13, userInfo: [NSLocalizedDescriptionKey: "输入文件为空"]) + } + + let candidates = Array(Set([packetSize, 40, 80].filter { $0 > 0 })) + var best = Data() + var lastError: Error? + + for size in candidates { + do { + let pcm = try decodeFixedPacketSize( + inputData: inputData, + packetSize: size, + sampleRate: sampleRate, + channels: channel + ) + if pcm.count > best.count { + best = pcm + } + } catch { + lastError = error + } + } + + if best.isEmpty { + throw lastError ?? NSError(domain: "ble_service", code: -14, userInfo: [NSLocalizedDescriptionKey: "解码输出为空"]) + } + + try best.write(to: URL(fileURLWithPath: outPath), options: .atomic) + return outPath + } + + /// 按固定 packetSize 切分输入数据并逐包解码为 PCM + /// - Parameters: + /// - inputData: Opus 裸包流（无容器） + /// - packetSize: 固定包长 + /// - sampleRate: 采样率 + /// - channels: 通道数 + /// - Returns: PCM 数据（16-bit little-endian） + /// - Throws: 解码器创建失败或输出为空时抛出 + private static func decodeFixedPacketSize( + inputData: Data, + packetSize: Int, + sampleRate: Int32, + channels: Int32 + ) throws -> Data { + if packetSize <= 0 { + throw NSError(domain: "ble_service", code: -20, userInfo: [NSLocalizedDescriptionKey: "packetSize 必须大于 0"]) + } + + var errorCode: Int32 = 0 + guard let decoder = opus_decoder_create(sampleRate, channels, &errorCode), errorCode == OPUS_OK else { + throw NSError(domain: "ble_service", code: Int(errorCode), userInfo: [NSLocalizedDescriptionKey: "Opus 解码器创建失败: \(errorCode)"]) + } + defer { opus_decoder_destroy(decoder) } + + let maxFrameSize: Int32 = 5760 + var pcmBuffer = [opus_int16](repeating: 0, count: Int(maxFrameSize) * Int(channels)) + var output = Data() + var offset = 0 + + while offset + packetSize <= inputData.count { + let packet = inputData.subdata(in: offset..<(offset + packetSize)) + offset += packetSize + + let decodedSamples: opus_int32 = packet.withUnsafeBytes { packetPtr in + guard let packetBase = packetPtr.bindMemory(to: UInt8.self).baseAddress else { + return opus_int32(-1) + } + return opus_decode( + decoder, + packetBase, + opus_int32(packet.count), + &pcmBuffer, + maxFrameSize, + 0 + ) + } + + if decodedSamples <= 0 { + continue + } + + let bytesCount = Int(decodedSamples) * Int(channels) * MemoryLayout.size + if bytesCount > 0 { + output.append(pcmBuffer.withUnsafeBytes { Data($0.prefix(bytesCount)) }) + } + } + + if output.isEmpty { + throw NSError(domain: "ble_service", code: -21, userInfo: [NSLocalizedDescriptionKey: "解码输出为空（packetSize=\(packetSize)）"]) + } + + return output + } +} + // MARK: - BleService.Callback @available(iOS 13.0, *) extension SwiftBleServicePlugin: BleService.Callback { diff --git a/local_plugins/jl_opus/ios/jl_opus/Package.swift b/local_plugins/jl_opus/ios/jl_opus/Package.swift new file mode 100644 index 000000000..ab0c6c8e8 --- /dev/null +++ b/local_plugins/jl_opus/ios/jl_opus/Package.swift @@ -0,0 +1,22 @@ +// swift-tools-version: 5.9 +import PackageDescription + +let package = Package( + name: "jl_opus", + platforms: [.iOS("16.0")], + products: [ + .library(name: "jl-opus", targets: ["jl_opus"]) + ], + dependencies: [ + .package(path: "../../../ble_service/ios/ble_service") + ], + targets: [ + .target( + name: "jl_opus", + dependencies: [ + .product(name: "ble-service", package: "ble_service") + ], + path: "Sources/jl_opus" + ) + ] +) diff --git a/local_plugins/jl_opus/ios/jl_opus/Sources/jl_opus/JlOpusPlugin.swift b/local_plugins/jl_opus/ios/jl_opus/Sources/jl_opus/JlOpusPlugin.swift new file mode 100644 index 000000000..09734dab1 --- /dev/null +++ b/local_plugins/jl_opus/ios/jl_opus/Sources/jl_opus/JlOpusPlugin.swift @@ -0,0 +1,450 @@ +import Flutter +import Foundation +import opus + +@objc(JlOpusPlugin) +public class JlOpusPlugin: NSObject, FlutterPlugin, FlutterStreamHandler { + private var eventSink: FlutterEventSink? + private var isInitialized: Bool = false + + private var isDecodingStream: Bool = false + private var streamHasHeader: Bool = false + private var streamChannels: Int32 = 1 + private var streamSampleRate: Int32 = 16000 + private var streamDecoder: OpaquePointer? + + private let decodeQueue = DispatchQueue(label: "com.yunqiinnovation.jl_opus.decode", qos: .userInitiated) + + /// Flutter 插件注册入口 + /// - Parameter registrar: FlutterPluginRegistrar + /// - Returns: 无 + /// - Throws: 不抛出 + public static func register(with registrar: FlutterPluginRegistrar) { + let methodChannel = FlutterMethodChannel( + name: "com.yunqiinnovation.jl_opus", + binaryMessenger: registrar.messenger() + ) + + let eventChannel = FlutterEventChannel( + name: "com.yunqiinnovation.jl_opus/events", + binaryMessenger: registrar.messenger() + ) + + let instance = JlOpusPlugin() + registrar.addMethodCallDelegate(instance, channel: methodChannel) + eventChannel.setStreamHandler(instance) + } + + /// 处理 Flutter 侧方法调用 + /// - Parameters: + /// - call: FlutterMethodCall + /// - result: FlutterResult + /// - Returns: 无 + /// - Throws: 不抛出（错误通过 FlutterError 返回） + public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { + switch call.method { + case "initOpusDecoder": + isInitialized = true + result(true) + + case "decodeOpusFile": + decodeOpusFile(arguments: call.arguments, result: result) + + case "startDecodeStream": + startDecodeStream(arguments: call.arguments, result: result) + + case "stopDecodeStream": + stopDecodeStream(result: result) + + case "writeAudioStream": + writeAudioStream(arguments: call.arguments, result: result) + + case "isDecoding": + result(isDecodingStream) + + case "dispose": + dispose(result: result) + + default: + result(FlutterMethodNotImplemented) + } + } + + /// 事件通道开始监听 + /// - Parameters: + /// - arguments: Flutter 传入参数 + /// - eventSink: 事件回调 + /// - Returns: FlutterError?（通常为 nil） + /// - Throws: 不抛出 + public func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { + eventSink = events + return nil + } + + /// 事件通道取消监听 + /// - Parameter arguments: Flutter 传入参数 + /// - Returns: FlutterError?（通常为 nil） + /// - Throws: 不抛出 + public func onCancel(withArguments arguments: Any?) -> FlutterError? { + eventSink = nil + return nil + } + + /// 解码 Opus 文件为 PCM 文件（16-bit little-endian 原始 PCM） + /// - Parameters: + /// - arguments: Flutter 传入参数（需包含 inPath/outPath/hasHeader/channel/sampleRate/packetSize） + /// - result: FlutterResult，成功返回 outPath，失败返回 FlutterError + /// - Returns: 无 + /// - Throws: 不直接抛出；内部错误会通过 FlutterError 与事件通道 onError 返回 + private func decodeOpusFile(arguments: Any?, result: @escaping FlutterResult) { + guard isInitialized else { + result(FlutterError(code: "NOT_INITIALIZED", message: "Opus解码器未初始化", details: nil)) + return + } + + guard let args = arguments as? [String: Any], + let inPath = args["inPath"] as? String, + let outPath = args["outPath"] as? String else { + result(FlutterError(code: "INVALID_ARGS", message: "输入或输出路径不能为空", details: nil)) + return + } + + let hasHeader = (args["hasHeader"] as? Bool) ?? false + let channel = Int32((args["channel"] as? Int) ?? 1) + let sampleRate = Int32((args["sampleRate"] as? Int) ?? 16000) + let packetSize = (args["packetSize"] as? Int) ?? 40 + + sendEvent([ + "event": "onStart", + "type": "file" + ]) + + decodeQueue.async { [weak self] in + guard let self = self else { return } + + do { + let inputUrl = URL(fileURLWithPath: inPath) + let outputUrl = URL(fileURLWithPath: outPath) + + let inputData = try Data(contentsOf: inputUrl) + if inputData.isEmpty { + throw NSError(domain: "jl_opus", code: -2, userInfo: [NSLocalizedDescriptionKey: "输入文件为空"]) + } + + let packets = try self.splitPackets(inputData: inputData, hasHeader: hasHeader, packetSize: packetSize) + let pcmData = try self.decodePacketsToPcm(packets: packets, sampleRate: sampleRate, channels: channel) + + try self.ensureParentDirectoryExists(for: outputUrl) + try pcmData.write(to: outputUrl, options: .atomic) + + self.sendEvent([ + "event": "onComplete", + "type": "file", + "filePath": outPath + ]) + + DispatchQueue.main.async { + result(outPath) + } + } catch { + let message = error.localizedDescription + self.sendEvent([ + "event": "onError", + "type": "file", + "code": -1, + "message": message + ]) + + DispatchQueue.main.async { + result(FlutterError(code: "DECODE_ERROR", message: "解码出错: \(message)", details: nil)) + } + } + } + } + + /// 开始 Opus 数据流解码 + /// - Parameters: + /// - arguments: Flutter 传入参数（需包含 hasHeader/channel/sampleRate） + /// - result: FlutterResult，成功返回 true，失败返回 FlutterError + /// - Returns: 无 + /// - Throws: 不抛出；失败通过 FlutterError/事件 onError 返回 + private func startDecodeStream(arguments: Any?, result: @escaping FlutterResult) { + guard isInitialized else { + result(FlutterError(code: "NOT_INITIALIZED", message: "Opus解码器未初始化", details: nil)) + return + } + + guard let args = arguments as? [String: Any] else { + result(FlutterError(code: "INVALID_ARGS", message: "缺少解码参数", details: nil)) + return + } + + let hasHeader = (args["hasHeader"] as? Bool) ?? false + let channel = Int32((args["channel"] as? Int) ?? 1) + let sampleRate = Int32((args["sampleRate"] as? Int) ?? 16000) + + stopDecodeStreamInternal() + + var error: Int32 = 0 + guard let decoder = opus_decoder_create(sampleRate, channel, &error), error == OPUS_OK else { + sendEvent([ + "event": "onError", + "type": "stream", + "code": Int(error), + "message": "Opus解码器创建失败: \(error)" + ]) + result(FlutterError(code: "DECODE_STREAM_ERROR", message: "数据流解码器创建失败", details: nil)) + return + } + + streamHasHeader = hasHeader + streamChannels = channel + streamSampleRate = sampleRate + streamDecoder = decoder + isDecodingStream = true + + sendEvent([ + "event": "onStart", + "type": "stream" + ]) + result(true) + } + + /// 停止 Opus 数据流解码 + /// - Parameter result: FlutterResult，已停止返回 true，未处于解码返回 false + /// - Returns: 无 + /// - Throws: 不抛出 + private func stopDecodeStream(result: @escaping FlutterResult) { + if isDecodingStream { + stopDecodeStreamInternal() + result(true) + } else { + result(false) + } + } + + /// 停止并释放当前数据流解码器（内部方法） + /// - Returns: 无 + /// - Throws: 不抛出 + private func stopDecodeStreamInternal() { + if let decoder = streamDecoder { + opus_decoder_destroy(decoder) + } + streamDecoder = nil + isDecodingStream = false + } + + /// 写入 Opus 数据到解码流，并通过事件通道返回 PCM 数据 + /// - Parameters: + /// - arguments: Flutter 传入参数（需包含 data） + /// - result: FlutterResult，成功返回 true；不在解码状态返回 FlutterError + /// - Returns: 无 + /// - Throws: 不直接抛出；解码错误通过事件 onError 返回 + private func writeAudioStream(arguments: Any?, result: @escaping FlutterResult) { + guard isDecodingStream, let decoder = streamDecoder else { + result(FlutterError(code: "NOT_DECODING", message: "当前没有处于解码状态", details: nil)) + return + } + + guard let args = arguments as? [String: Any], + let data = args["data"] as? FlutterStandardTypedData else { + result(FlutterError(code: "INVALID_ARGS", message: "音频数据不能为空", details: nil)) + return + } + + let opusData = data.data + if opusData.isEmpty { + result(true) + return + } + + let hasHeader = streamHasHeader + let channels = streamChannels + let sampleRate = streamSampleRate + + decodeQueue.async { [weak self] in + guard let self = self else { return } + + do { + let packets: [Data] + if hasHeader { + packets = try self.splitPacketsByLengthPrefix(inputData: opusData) + } else { + packets = [opusData] + } + + for packet in packets { + let pcm = try self.decodeSinglePacketToPcm(decoder: decoder, packet: packet, sampleRate: sampleRate, channels: channels) + if !pcm.isEmpty { + self.sendEvent([ + "event": "onDecodeStream", + "data": FlutterStandardTypedData(bytes: pcm) + ]) + } + } + } catch { + let message = error.localizedDescription + self.sendEvent([ + "event": "onError", + "type": "stream", + "code": -1, + "message": message + ]) + } + } + + result(true) + } + + /// 释放插件资源 + /// - Parameter result: FlutterResult + /// - Returns: 无 + /// - Throws: 不抛出 + private func dispose(result: @escaping FlutterResult) { + stopDecodeStreamInternal() + isInitialized = false + result(true) + } + + /// 拆分输入文件数据为 Opus 包列表 + /// - Parameters: + /// - inputData: 输入文件二进制数据 + /// - hasHeader: 是否包含 2 字节小端长度前缀 + /// - packetSize: 无协议头时按固定长度切包 + /// - Returns: Opus 包数组 + /// - Throws: 参数错误或解析失败时抛出 NSError + private func splitPackets(inputData: Data, hasHeader: Bool, packetSize: Int) throws -> [Data] { + if hasHeader { + return try splitPacketsByLengthPrefix(inputData: inputData) + } + + guard packetSize > 0 else { + throw NSError(domain: "jl_opus", code: -5, userInfo: [NSLocalizedDescriptionKey: "packetSize 必须大于 0"]) + } + + var packets: [Data] = [] + var offset = 0 + while offset < inputData.count { + let end = min(offset + packetSize, inputData.count) + packets.append(inputData.subdata(in: offset.. [Data] { + var packets: [Data] = [] + var cursor = 0 + while cursor + 2 <= inputData.count { + let lo = Int(inputData[cursor]) + let hi = Int(inputData[cursor + 1]) << 8 + let length = lo | hi + cursor += 2 + + if length <= 0 || cursor + length > inputData.count { + throw NSError(domain: "jl_opus", code: -3, userInfo: [NSLocalizedDescriptionKey: "协议头解析失败"]) + } + + packets.append(inputData.subdata(in: cursor..<(cursor + length))) + cursor += length + } + + if packets.isEmpty { + throw NSError(domain: "jl_opus", code: -4, userInfo: [NSLocalizedDescriptionKey: "未解析到任何Opus数据包"]) + } + + return packets + } + + /// 将 Opus 包列表解码为 PCM 数据 + /// - Parameters: + /// - packets: Opus 包数组 + /// - sampleRate: 采样率 + /// - channels: 通道数 + /// - Returns: PCM 数据（16-bit little-endian） + /// - Throws: 解码器创建失败或任意包解码失败时抛出 NSError + private func decodePacketsToPcm(packets: [Data], sampleRate: Int32, channels: Int32) throws -> Data { + var error: Int32 = 0 + guard let decoder = opus_decoder_create(sampleRate, channels, &error), error == OPUS_OK else { + throw NSError(domain: "jl_opus", code: Int(error), userInfo: [NSLocalizedDescriptionKey: "Opus解码器创建失败: \(error)"]) + } + defer { opus_decoder_destroy(decoder) } + + var pcmOutput = Data() + for packet in packets { + let pcmChunk = try decodeSinglePacketToPcm(decoder: decoder, packet: packet, sampleRate: sampleRate, channels: channels) + pcmOutput.append(pcmChunk) + } + return pcmOutput + } + + /// 解码单个 Opus 包为 PCM + /// - Parameters: + /// - decoder: Opus 解码器指针 + /// - packet: 单个 Opus 包 + /// - sampleRate: 采样率 + /// - channels: 通道数 + /// - Returns: PCM 数据（16-bit little-endian） + /// - Throws: 解码失败时抛出 NSError + private func decodeSinglePacketToPcm(decoder: OpaquePointer, packet: Data, sampleRate: Int32, channels: Int32) throws -> Data { + let maxSamplesPerChannel = Int(sampleRate * 120 / 1000) + var pcmBuffer = [opus_int16](repeating: 0, count: maxSamplesPerChannel * Int(channels)) + + let decodedSamples: Int32 = packet.withUnsafeBytes { packetPtr in + guard let inPtr = packetPtr.bindMemory(to: UInt8.self).baseAddress else { + return OPUS_BAD_ARG + } + + return opus_decode( + decoder, + inPtr, + Int32(packet.count), + &pcmBuffer, + Int32(maxSamplesPerChannel), + 0 + ) + } + + if decodedSamples < 0 { + throw NSError(domain: "jl_opus", code: Int(decodedSamples), userInfo: [NSLocalizedDescriptionKey: "Opus解码失败: \(decodedSamples)"]) + } + + let sampleCount = Int(decodedSamples) * Int(channels) + return pcmBuffer.withUnsafeBytes { rawPtr in + Data(rawPtr.bindMemory(to: UInt8.self).prefix(sampleCount * MemoryLayout.size)) + } + } + + /// 确保输出路径的父目录存在 + /// - Parameter url: 输出文件 URL + /// - Returns: 无 + /// - Throws: 创建目录失败时抛出错误 + private func ensureParentDirectoryExists(for url: URL) throws { + let dir = url.deletingLastPathComponent() + var isDirectory: ObjCBool = false + if FileManager.default.fileExists(atPath: dir.path, isDirectory: &isDirectory), isDirectory.boolValue { + return + } + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true, attributes: nil) + } + + /// 通过事件通道向 Flutter 发送事件 + /// - Parameter event: 事件字典 + /// - Returns: 无 + /// - Throws: 不抛出 + private func sendEvent(_ event: [String: Any]) { + guard let sink = eventSink else { return } + DispatchQueue.main.async { + sink(event) + } + } +} From fa0281c897de52bdc3c96daa5095f722d1e43d11 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Tue, 27 Jan 2026 17:41:41 +0800 Subject: [PATCH 24/25] =?UTF-8?q?chore:=20=E6=B7=BB=E5=8A=A0=E8=B0=83?= =?UTF-8?q?=E8=AF=95=E6=97=A5=E5=BF=97=E4=BB=A5=E8=B7=9F=E8=B8=AA=E8=AF=AD?= =?UTF-8?q?=E9=9F=B3=E5=AF=B9=E8=AF=9D=E6=B5=81=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加开发者前缀的打印语句，便于调试语音对话的开始、结束和设备通信流程 --- .../agent/controllers/agent_controller.dart | 4 ++-- .../agent_service/AgentServicePlugin.swift | 4 ++-- .../agent_service/lib/agent_service.dart | 5 +++-- .../Sources/ble_service/BleService.swift | 22 +++++++++---------- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/lib/modules/agent/controllers/agent_controller.dart b/lib/modules/agent/controllers/agent_controller.dart index c76ab1c54..f16dd703d 100644 --- a/lib/modules/agent/controllers/agent_controller.dart +++ b/lib/modules/agent/controllers/agent_controller.dart @@ -1880,7 +1880,7 @@ class AgentController extends GetxController with WidgetsBindingObserver { // 开始按住说话 void startPushToTalk() async { - Logger.i(TAG, '🎤 开始按住说话'); + print('liwei--------- 🎤 开始按住说话'); // 只有在语音模式且按住说话模式下才能按住说话 if (isTextInputMode.value) return; @@ -1898,7 +1898,7 @@ class AgentController extends GetxController with WidgetsBindingObserver { // 结束按住说话 void endPushToTalk() async { - Logger.i(TAG, '🎤 结束按住说话'); + print('liwei----------🎤 结束按住说话'); // 如果不在按住说话状态，直接返回 if (!isPushToTalkActive.value) return; diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift index 77d9085d9..8963009e1 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift @@ -74,7 +74,7 @@ public class AgentServicePlugin: NSObject, FlutterPlugin { result(true) return } - print("startConversation") + print("liwei----------AgentServicePlugin startConversation") let mode = (call.arguments as? [String: Any])?["mode"] as? String ?? "normal" impl.startRecognition(mode: mode) result(true) @@ -99,7 +99,7 @@ public class AgentServicePlugin: NSObject, FlutterPlugin { result(true) return } - print("stopConversation") + print("liwei-------AgentServicePlugin stopConversation") // 关闭编码器 (设备侧) BleService.shared.closeCodec() impl.stopRecognition() diff --git a/local_plugins/agent_service/lib/agent_service.dart b/local_plugins/agent_service/lib/agent_service.dart index 3ddbed06d..92dd7cafa 100644 --- a/local_plugins/agent_service/lib/agent_service.dart +++ b/local_plugins/agent_service/lib/agent_service.dart @@ -265,8 +265,8 @@ class AgentService { try { // 在启动新对话前，确保之前的对话已完全停止 // 这不仅可以清理状态，还能避免因快速切换导致的资源冲突 - await stopConversation(); - + // await stopConversation(); + print('liwei--------- view startConversation'); final bool result = await _channel.invokeMethod('startConversation', { 'mode': mode, }); @@ -337,6 +337,7 @@ class AgentService { // 即使频繁调用，Native 层也能处理（我们已经修复了 Native 层的并发问题）。 try { + print('liwei--------- view stopConversation'); final bool result = await _channel.invokeMethod('stopConversation'); return result; } on PlatformException catch (e) { diff --git a/local_plugins/ble_service/ios/ble_service/Sources/ble_service/BleService.swift b/local_plugins/ble_service/ios/ble_service/Sources/ble_service/BleService.swift index 0691e1e88..1ec345635 100644 --- a/local_plugins/ble_service/ios/ble_service/Sources/ble_service/BleService.swift +++ b/local_plugins/ble_service/ios/ble_service/Sources/ble_service/BleService.swift @@ -834,7 +834,7 @@ private var cmdReplyType: UInt8 = 0 private func sendCommand(_ cmd: UInt8, data: Data = Data()) -> Bool { guard let characteristic = writeCharacteristic, connectionState == BleConst.STATE_CONNECTED else { - os_log("发送命令失败: 设备未连接", log: logger, type: .error) + os_log("liwei---- 发送命令失败: 设备未连接", log: logger, type: .error) return false } @@ -862,7 +862,7 @@ private var cmdReplyType: UInt8 = 0 if !data.isEmpty { dataHexString = data.map { String(format: "0x%02X", $0) }.joined(separator: ", ") } - os_log("发送命令: CMD=0x%02X, 长度=%d, 数据=[%@], CRC=0x%02X", + os_log("liwei---- 发送命令: CMD=0x%02X, 长度=%d, 数据=[%@], CRC=0x%02X", log: logger, type: .debug, cmd, cmdLength, dataHexString, crc) @@ -881,7 +881,7 @@ private var cmdReplyType: UInt8 = 0 return true } catch { - os_log("发送命令异常: %@", log: logger, type: .error, error.localizedDescription) + os_log("liwei---- 发送命令异常: %@", log: logger, type: .error, error.localizedDescription) return false } } @@ -929,19 +929,17 @@ private var cmdReplyType: UInt8 = 0 /// - Returns: 可用于追加写入的 RecordingFile 实例 /// - Throws: 不抛出；若创建文件失败将导致后续写入无效 private func ensureOpusRecordingStarted(fileName: String) -> RecordingFile { - if recordingFile == nil { - let recorder = RecordingFile() + let recorder = recordingFile ?? RecordingFile() + if recorder.fileName.isEmpty { recorder.fileName = fileName - recordingFile = recorder - } else if (recordingFile?.fileName.isEmpty ?? true) { - recordingFile?.fileName = fileName } - if recordingFile?.isRecording() != true { - recordingFile?.createFile() + if !recorder.isRecording() { + recorder.createFile() } - return recordingFile! + recordingFile = recorder + return recorder } /// 停止并释放 Opus 落盘资源 @@ -1010,7 +1008,7 @@ private var cmdReplyType: UInt8 = 0 // 记录响应详情 var hexString = data.map { String(format: "0x%02X", $0) }.joined(separator: ", ") - os_log("收到设备响应: %@", log: logger, type: .debug, hexString) + os_log("liwei---- 收到设备响应: %@", log: logger, type: .debug, hexString) // 检查数据长度（与 Android 统一失败判定保持一致） if data.count != Int(length) + 4 { // 帧头 + 命令 + 长度 + 数据 + CRC From abc266c177bad3c77de563a92c406ec3baaba962 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Wed, 28 Jan 2026 10:04:23 +0800 Subject: [PATCH 25/25] =?UTF-8?q?fix(agent=5Fservice):=20=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E8=AF=AD=E9=9F=B3=E8=AF=86=E5=88=AB=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E9=80=BB=E8=BE=91=E5=B9=B6=E6=B7=BB=E5=8A=A0=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 调整初始化检查顺序，避免在未初始化时执行蓝牙操作 - 添加 ble_wakeup 模式连续触发保护，防止重复启动 - 增加关键节点调试日志，便于问题排查 - 移除部分冗余日志输出 --- .../agent_service/AgentServiceImpl.swift | 29 ++++++++++++++----- .../Sources/ble_service/BleService.swift | 2 +- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 6b6e90a2b..a3bfb676b 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -442,7 +442,7 @@ class AgentServiceImpl: NSObject { self.idleTimer = nil } guard self.isRecognizing else { return } - os_log("启动空闲检测，超时时间: %.0f秒，模式: %{public}@", + os_log("liwei--------启动空闲检测，超时时间: %.0f秒，模式: %{public}@", log: self.logger, type: .info, idleSeconds, @@ -453,7 +453,7 @@ class AgentServiceImpl: NSObject { guard let self = self else { return } DispatchQueue.main.async { [weak self] in guard let self = self else { return } - os_log("空闲检测触发自动停止，模式: %{public}@", + os_log("liwei--------空闲检测触发自动停止，模式: %{public}@", log: self.logger, type: .info, self.currentRecognitionMode) @@ -490,6 +490,17 @@ class AgentServiceImpl: NSObject { } } func startRecognition(useBle: Bool = false, mode: String = "normal") -> Bool { + if !isInitialized { + sendError("服务未初始化", code: "NOT_INITIALIZED") + return false + } + + if useBle, mode == "ble_wakeup", currentRecognitionMode == "ble_wakeup", (isRecognizing || isStartingRecognition) { + os_log("liwei-------- ble_wakeup 连续触发，识别中/启动中，忽略重复 startRecognition", log: logger, type: .info) + audioPlayer?.playStartSound() + return true + } + // 接收到唤醒信号，打开编码器 (设备侧) print("ai启动语音\(useBle))") var openResult = false // 添加openResult变量定义 @@ -498,10 +509,6 @@ class AgentServiceImpl: NSObject { // 捕获openEncoder的返回值 openResult = BleService.shared.openEncoder() // 正确捕获返回值 } - if !isInitialized { - sendError("服务未初始化", code: "NOT_INITIALIZED") - return false - } if !useBle { guard AVAudioSession.sharedInstance().recordPermission == .granted else { @@ -509,6 +516,7 @@ class AgentServiceImpl: NSObject { return false } } + lastRecognitionMode = currentRecognitionMode stopRecognition() if isAiStreaming { @@ -528,13 +536,13 @@ class AgentServiceImpl: NSObject { // 设置当前识别模式 currentRecognitionMode = mode - os_log("liwei--------------- 设置语音识别模式: %{public}@", log: logger, type: .info, mode) + let audioSourceType: AzureAsrHelper.AudioSourceType = useBle ? .external : .microphone // 新增：标记为"启动中"，用于允许 stop 在启动未完成时也能生效 isStartingRecognition = true - print("ai启动语音=\(audioSourceType)=\(useBle)") + print("ai启动语音=\(audioSourceType)=\(useBle)") guard let success = azureAsrHelper?.startContinuousRecognition( mod:self.currentRecognitionMode, audioSourceType: audioSourceType, @@ -543,11 +551,13 @@ class AgentServiceImpl: NSObject { // 新增：启动失败时复位"启动中"状态 isStartingRecognition = false sendError("启动语音识别失败", code: "RECOGNITION_START_ERROR") + os_log("liwei--------------- 启动语音识别失败: %{public}@", log: logger, type: .info, mode) return false } if(useBle){ isKeepResult = true } + os_log("liwei--------------- 启动语音识别: %{public}@", log: logger, type: .info, mode) // 根据模式启动相应的空闲检测（此处 guard isRecognizing，会在 onSessionStarted 中再次启动） startIdleCheckForMode(mode: mode) @@ -2125,10 +2135,12 @@ class AudioPlayer { } func playStartSound() { + os_log("liwei 播放 playStartSound", log: self.logger, type: .debug) playSound(named: "start", fileType: "mp3") } func playAwaitSound() { + os_log("liwei 播放 playAwaitSound", log: self.logger, type: .debug) // 若已处于等待音效播放状态，则直接返回，避免频繁启停 if isAwaitSoundActive { return } @@ -2146,6 +2158,7 @@ class AudioPlayer { } func playStopSound() { + os_log("liwei 播放 playStopSound", log: self.logger, type: .debug) playSound(named: "stop", fileType: "mp3") } diff --git a/local_plugins/ble_service/ios/ble_service/Sources/ble_service/BleService.swift b/local_plugins/ble_service/ios/ble_service/Sources/ble_service/BleService.swift index 1ec345635..30670648c 100644 --- a/local_plugins/ble_service/ios/ble_service/Sources/ble_service/BleService.swift +++ b/local_plugins/ble_service/ios/ble_service/Sources/ble_service/BleService.swift @@ -1008,7 +1008,7 @@ private var cmdReplyType: UInt8 = 0 // 记录响应详情 var hexString = data.map { String(format: "0x%02X", $0) }.joined(separator: ", ") - os_log("liwei---- 收到设备响应: %@", log: logger, type: .debug, hexString) + // os_log("liwei---- 收到设备响应: %@", log: logger, type: .debug, hexString) // 检查数据长度（与 Android 统一失败判定保持一致） if data.count != Int(length) + 4 { // 帧头 + 命令 + 长度 + 数据 + CRC