diff --git a/lib/data/services/language_manager.dart b/lib/data/services/language_manager.dart index e29a33e95..d6e1c1a20 100644 --- a/lib/data/services/language_manager.dart +++ b/lib/data/services/language_manager.dart @@ -612,8 +612,18 @@ class LanguageManager extends GetxService { .toList(); } + /// 豆包端到端 STS 实际优化良好的语言白名单 + /// 只有源和目标都在白名单内,且至少一个是 zh/en 时,才走豆包; + /// 其余情况退到阿里或微软,避免法/德/西/葡/俄等语种走豆包端到端时 + /// 出现词被拆碎、ASR 误识别、WebSocket 频繁断开等问题 + static const Set _volcanoE2EWhitelist = {'zh', 'en', 'ja', 'ko'}; + /// 查找同时支持源语言和目标语言的最佳提供商(通过 shortCode 匹配) - /// 优先级:豆包(Volcano) > 阿里(Alibaba) > 微软(Azure) > 讯飞(Iflytek) + /// 路由策略: + /// 1. 豆包:源/目标都在白名单(zh/en/ja/ko)且至少一个是 zh/en + /// 2. 阿里:源/目标都在阿里翻译表内(zh/en/ja/ko/fr/de/es/it/pt/ru) + /// 3. 微软:其他所有组合(阿里不覆盖的稀有语种,或两端有任一不在阿里白名单) + /// 4. 讯飞:兜底 BestProviderResult? findBestMatchingProvider( String sourceShortCode, String targetShortCode) { final providers = [ @@ -622,8 +632,11 @@ class LanguageManager extends GetxService { SpeechServiceType.azure, SpeechServiceType.iflytek, ]; + Logger.info( + '[TRANS-INIT] 开始匹配提供商: $sourceShortCode -> $targetShortCode, 策略=豆包(zh/en/ja/ko白名单且含zh|en)>阿里(含zh)>微软>讯飞'); for (final provider in providers) { + final providerName = _providerStringFromType(provider); final transSpecs = _getTranslationSpecs(provider); Map? sourceTransSpec; @@ -641,20 +654,38 @@ class LanguageManager extends GetxService { } if (sourceTransSpec == null || targetTransSpec == null) { + Logger.info( + '[TRANS-INIT] 跳过 ${providerDisplayName(providerName)}: 翻译表缺少${sourceTransSpec == null ? " 源=$sourceShortCode" : ""}${targetTransSpec == null ? " 目标=$targetShortCode" : ""}'); continue; } - // 豆包特殊限制:源语言或目标语言必须有一个是中文或英语 + // 豆包端到端限制: + // (a) 源和目标都必须在白名单 {zh, en, ja, ko} + // (b) 且至少一个是 zh/en(豆包 STS 以中英为 pivot) if (provider == SpeechServiceType.volcano) { - final isSourceZhOrEn = - sourceShortCode == 'zh' || sourceShortCode == 'en'; - final isTargetZhOrEn = - targetShortCode == 'zh' || targetShortCode == 'en'; - if (!isSourceZhOrEn && !isTargetZhOrEn) { + final bothInWhitelist = + _volcanoE2EWhitelist.contains(sourceShortCode) && + _volcanoE2EWhitelist.contains(targetShortCode); + final hasZhOrEn = sourceShortCode == 'zh' || + sourceShortCode == 'en' || + targetShortCode == 'zh' || + targetShortCode == 'en'; + if (!bothInWhitelist) { + Logger.info( + '[TRANS-INIT] 跳过 ${providerDisplayName('volcano')}: 端到端仅对 {zh,en,ja,ko} 稳定,当前 $sourceShortCode/$targetShortCode 不全在白名单'); + continue; + } + if (!hasZhOrEn) { + Logger.info( + '[TRANS-INIT] 跳过 ${providerDisplayName('volcano')}: 需源或目标至少一个为 zh/en(当前 $sourceShortCode/$targetShortCode)'); continue; } } + // 阿里端到端 (qwen3-livetranslate): 覆盖主流 10 语(zh/en/ja/ko/fr/de/es/it/pt/ru), + // 两端都在阿里翻译表内即走阿里;稀有语种交给微软兜底。 + // 这里不加额外白名单——上面 transSpecs 的匹配检查已经做了筛选。 + // 对于非端到端服务(Azure, Iflytek),还需要检查 ASR 和 TTS 支持 if (provider != SpeechServiceType.volcano && provider != SpeechServiceType.alibaba) { @@ -693,12 +724,16 @@ class LanguageManager extends GetxService { } final providerString = _providerStringFromType(provider); + final isEndToEnd = provider == SpeechServiceType.volcano || + provider == SpeechServiceType.alibaba; Logger.info( - 'Found best provider: $providerString for $sourceShortCode -> $targetShortCode, sourceTts=${sourceTransSpec["ttsCode"]}, targetTts=${targetTransSpec["ttsCode"]}'); + '[TRANS-INIT] 命中 ${providerDisplayName(providerString)}[$providerString] for $sourceShortCode -> $targetShortCode (端到端=$isEndToEnd), transSrc=${sourceTransSpec["translationCode"]}, transTgt=${targetTransSpec["translationCode"]}, asrSrc=${sourceTransSpec["asrCode"]}, asrTgt=${targetTransSpec["asrCode"]}, ttsSrc=${sourceTransSpec["ttsCode"]}, ttsTgt=${targetTransSpec["ttsCode"]}'); return BestProviderResult( providerString, sourceTransSpec, targetTransSpec); } + Logger.warning( + '[TRANS-INIT] 所有提供商都不匹配 $sourceShortCode -> $targetShortCode,返回 null'); return null; } @@ -719,6 +754,26 @@ class LanguageManager extends GetxService { } } + /// 将 provider 字符串映射为中文厂商名,用于日志可读性 + static String providerDisplayName(String provider) { + switch (provider) { + case 'volcano': + return '豆包(火山)'; + case 'alibaba': + return '阿里(通义)'; + case 'azure': + return '微软(Azure)'; + case 'iflytek': + return '讯飞'; + case 'google': + return '谷歌'; + case 'flutter': + return '本地(Flutter)'; + default: + return provider; + } + } + // ===== 私有工具方法:加载不同提供商的语言规格 ===== List> _getAsrSpecs(SpeechServiceType provider) { diff --git a/lib/modules/translation/controllers/translation_controller.dart b/lib/modules/translation/controllers/translation_controller.dart index 4ae405b7c..cc8f4656e 100644 --- a/lib/modules/translation/controllers/translation_controller.dart +++ b/lib/modules/translation/controllers/translation_controller.dart @@ -51,6 +51,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver { final TtsService _ttsService = Get.find(); final AstService _astService = Get.find(); final LanguageManager _languageManager = Get.find(); + /// 最近一次通话模式初始化所选中的 AST provider,仅用于日志追踪 + String _lastCallAstProvider = ''; final GetStorage _storage = GetStorage(); final BluetoothManager bluetoothManager = Get.find(); final BleManager bleManager = Get.find(); @@ -152,6 +154,10 @@ class TranslationController extends GetxController with WidgetsBindingObserver { // ==================== 重新初始化守卫 ==================== bool _isReinitializing = false; // 语言切换重新初始化期间为 true,抑制旧会话的错误事件 + // 语言切换互斥:连续切换语言时串行执行,避免并发的 _reinitializeAsrService 导致 + // 原生 AST 已启动但 isRecognizing 仍为 false 的状态不一致。 + Future? _languageSwitchChain; + // 播放防重复机制 String? _lastPlayedItemKey; // 最后播放的项目标识 int _lastPlayTime = 0; // 最后播放时间 @@ -839,8 +845,17 @@ class TranslationController extends GetxController with WidgetsBindingObserver { final sourceShort = _languageManager.getShortCodeByAsrCode(sourceLanguageCode.value) ?? 'zh'; final targetShort = _languageManager.getShortCodeByAsrCode(targetLanguageCode.value) ?? 'en'; + final sourceName = _languageManager.getChineseNameByAsrCode(sourceLanguageCode.value) ?? '?'; + final targetName = _languageManager.getChineseNameByAsrCode(targetLanguageCode.value) ?? '?'; + Logger.info( + '[TRANS-INIT] 通话翻译语言对: $sourceName($sourceShort/${sourceLanguageCode.value}) -> $targetName($targetShort/${targetLanguageCode.value})'); final bestProvider = _languageManager.findBestMatchingProvider(sourceShort, targetShort); final astProvider = bestProvider?.provider ?? 'azure'; + _lastCallAstProvider = astProvider; + if (bestProvider == null) { + Logger.warning( + '[TRANS-INIT] 未找到匹配的提供商,回退到默认 ${LanguageManager.providerDisplayName('azure')}[azure](asrCode=${sourceLanguageCode.value} -> ${targetLanguageCode.value})'); + } // 构建完整的语言参数:[asrCode0, asrCode1, transCode0, transCode1, ttsVoice0, ttsVoice1] final transCode0 = bestProvider?.sourceSpec['translationCode'] ?? sourceLanguageCode.value; @@ -857,7 +872,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver { ttsVoice0, // [4] TTS voice source ttsVoice1, // [5] TTS voice target ]; - Logger.info('1初始化AST服务,支持语言: $callModeLanguages, provider: $astProvider'); + Logger.info( + '[TRANS-INIT] 最终启动服务: ${LanguageManager.providerDisplayName(astProvider)}[$astProvider], asrSrc=${callModeLanguages[0]}, asrTgt=${callModeLanguages[1]}, transSrc=${callModeLanguages[2]}, transTgt=${callModeLanguages[3]}, ttsSrc=${callModeLanguages[4]}, ttsTgt=${callModeLanguages[5]}'); await _astService.initialize(supportedLanguages: callModeLanguages, provider: astProvider); @@ -993,6 +1009,16 @@ class TranslationController extends GetxController with WidgetsBindingObserver { /// 同声翻译、通话翻译、多媒体翻译的语音识别调用入口 Future startRecognition() async { Logger.info('开始语音识别'); + final srcName = _languageManager.getChineseNameByAsrCode(sourceLanguageCode.value) ?? '?'; + final tgtName = _languageManager.getChineseNameByAsrCode(targetLanguageCode.value) ?? '?'; + if (currentMode.value == 'call') { + final p = _lastCallAstProvider.isEmpty ? 'azure(未初始化)' : _lastCallAstProvider; + Logger.info( + '[TRANS-INIT] startRecognition: mode=call, 语言对=$srcName(${sourceLanguageCode.value}) -> $tgtName(${targetLanguageCode.value}), 服务=${LanguageManager.providerDisplayName(p)}[$p]'); + } else { + Logger.info( + '[TRANS-INIT] startRecognition: mode=${currentMode.value}, 语言对=$srcName(${sourceLanguageCode.value}) -> $tgtName(${targetLanguageCode.value})'); + } //检查用户是否登录且积分是否足够,不足则提示充值 if (User.isLoggedIn() && User.instance.tradeintegral <= 0) { showRechargeDialog('translation'); @@ -1017,6 +1043,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } else if (currentMode.value == 'faceToFace') { Logger.info('iwei-------- 开始语音识别,模式:面对面,ASR push_to_talk'); await _startAsrService('push_to_talk'); + } else if (currentMode.value == 'call') { + Logger.info('iwei-------- 通话模式:AST端到端服务已启动,跳过Azure ASR'); } else { Logger.info( 'iwei-------- 开始语音识别,模式:${currentMode.value},ASR 模式:normal'); @@ -1978,61 +2006,84 @@ class TranslationController extends GetxController with WidgetsBindingObserver { /// 获取底部控制栏高度 double get bottomBarHeight => _bottomBarHeight; + /// 串行化语言切换,避免并发 reinit 造成 UI 与原生 AST 状态不一致 + Future _runLanguageSwitch(Future Function() action) { + final previous = _languageSwitchChain; + final completer = Completer(); + final gate = Completer(); + Future run() async { + if (previous != null) { + try { + await previous; + } catch (_) {} + } + try { + completer.complete(await action()); + } catch (e, st) { + completer.completeError(e, st); + } finally { + gate.complete(); + } + } + + _languageSwitchChain = gate.future; + run(); + return completer.future; + } + /// 切换源语言和目标语言 - Future swapLanguages() async { - final wasRecognizing = isRecognizing.value; - await stopAll(); + Future swapLanguages() => _runLanguageSwitch(() async { + final wasRecognizing = isRecognizing.value; + await stopAll(); - final tempName = sourceLanguage.value; - sourceLanguage.value = targetLanguage.value; - targetLanguage.value = tempName; + final tempName = sourceLanguage.value; + sourceLanguage.value = targetLanguage.value; + targetLanguage.value = tempName; - final tempCode = sourceLanguageCode.value; - sourceLanguageCode.value = targetLanguageCode.value; - targetLanguageCode.value = tempCode; + final tempCode = sourceLanguageCode.value; + sourceLanguageCode.value = targetLanguageCode.value; + targetLanguageCode.value = tempCode; - _saveSourceLanguageSetting(isUserSet: true); - _saveTargetLanguageSetting(); - await _reinitializeAsrService(restartRecognition: wasRecognizing); - } + _saveSourceLanguageSetting(isUserSet: true); + _saveTargetLanguageSetting(); + await _reinitializeAsrService(restartRecognition: wasRecognizing); + }); /// 设置源语言 /// [language] 语言名称 - Future setSourceLanguage(String language) async { - if (sourceLanguage.value != language) { - final wasRecognizing = isRecognizing.value; - await stopAll(); + Future setSourceLanguage(String language) => _runLanguageSwitch(() async { + if (sourceLanguage.value == language) return; + final wasRecognizing = isRecognizing.value; + await stopAll(); - sourceLanguage.value = language; + sourceLanguage.value = language; - final asrCode = _languageManager.getAsrCodeByChineseName(language); - if (asrCode != null) { - sourceLanguageCode.value = asrCode; - } + final asrCode = _languageManager.getAsrCodeByChineseName(language); + if (asrCode != null) { + sourceLanguageCode.value = asrCode; + } - _saveSourceLanguageSetting(isUserSet: true); - await _reinitializeAsrService(restartRecognition: wasRecognizing); - } - } + _saveSourceLanguageSetting(isUserSet: true); + await _reinitializeAsrService(restartRecognition: wasRecognizing); + }); /// 设置目标语言 /// [language] 语言名称 - Future setTargetLanguage(String language) async { - if (targetLanguage.value != language) { - final wasRecognizing = isRecognizing.value; - await stopAll(); + Future setTargetLanguage(String language) => _runLanguageSwitch(() async { + if (targetLanguage.value == language) return; + final wasRecognizing = isRecognizing.value; + await stopAll(); - targetLanguage.value = language; + targetLanguage.value = language; - final asrCode = _languageManager.getAsrCodeByChineseName(language); - if (asrCode != null) { - targetLanguageCode.value = asrCode; - } + final asrCode = _languageManager.getAsrCodeByChineseName(language); + if (asrCode != null) { + targetLanguageCode.value = asrCode; + } - _saveTargetLanguageSetting(); - await _reinitializeAsrService(restartRecognition: wasRecognizing); - } - } + _saveTargetLanguageSetting(); + await _reinitializeAsrService(restartRecognition: wasRecognizing); + }); /// 重新初始化ASR服务 /// [restartRecognition] 初始化完成后是否自动重新开始识别 @@ -2070,6 +2121,17 @@ class TranslationController extends GetxController with WidgetsBindingObserver { if (currentMode.value == "call") { await _initializeCallModeTranslationService(); + // _astService.initialize 的原生实现会自动启动 AST(AzureAsrToAsr / DoubaoE2E + // 等在 initialize 中都调用了 startContinuousTranslation)。若切换语言前未处于 + // 识别状态,这里需要立即停止原生 AST,避免 UI 显示未启动但原生仍在识别。 + if (!restartRecognition) { + try { + await _astService.stopContinuousTranslation(); + Logger.info('[STS] 语言切换:非识别状态,已停止新AST保持状态一致'); + } catch (e) { + Logger.info('[STS] 停止新AST(忽略): $e'); + } + } } else { Logger.info('重新初始化ASR服务,支持语言: $asrSupportedLanguages'); diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index 3f381812e..e975037c5 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -321,6 +321,53 @@ object AgentService : CoroutineScope { abandonAiAudioFocus() } + // ========== TTS 轮次结束 debounce ========== + // Azure 流式 TTS 按标点切段合成,每段都会发一对 PLAYBACK_STARTED / PLAYBACK_COMPLETED。 + // 如果每次 PLAYBACK_COMPLETED 都立即 resumeMusic + 释放焦点,就会在段间隙让外部音乐短暂抢回, + // 出现"AI 说到一半音乐就回来了"的现象。这里用 1s debounce: + // - PLAYBACK_COMPLETED → 延迟 1s 再执行真正的 round-end + // - 延迟期内又来 PLAYBACK_STARTED → 取消 pending,视为段间隙 + // - CANCELED / ERROR → 取消 pending,立刻跑 round-end(属于明确结束信号) + private val ttsResumeHandler = Handler(Looper.getMainLooper()) + private var pendingTtsResumeRunnable: Runnable? = null + private const val TTS_RESUME_DEBOUNCE_MS = 1000L + + private fun cancelPendingTtsResume() { + pendingTtsResumeRunnable?.let { ttsResumeHandler.removeCallbacks(it) } + pendingTtsResumeRunnable = null + } + + private fun performTtsRoundEnd(reason: String) { + _isTtsSpeaking.set(false) + _log("INFO", "[Agent][焦点] TTS round end($reason) —— 恢复音乐并尝试释放焦点") + try { + val intent = Intent("com.yunqiinnovation.music_service.AI_SAY_END").apply { + setPackage(context.packageName) + } + context.sendBroadcast(intent) + } catch (e: Exception) { + _log("WARNING", "[Agent] 发送 AI_SAY_END 广播异常: ${e.message}") + } + if (currentRecognitionMode == "ble_wakeup" && _isInterrupt.get()) { + try { BleService.openB1Encoder() } catch (e: Exception) { + _log("WARNING", "[Agent] openB1Encoder 异常: ${e.message}") + } + } + resumeMusic() + tryReleaseAiAudioFocus(reason) + } + + private fun scheduleTtsRoundEnd(reason: String) { + cancelPendingTtsResume() + val runnable = Runnable { + pendingTtsResumeRunnable = null + performTtsRoundEnd(reason) + } + pendingTtsResumeRunnable = runnable + ttsResumeHandler.postDelayed(runnable, TTS_RESUME_DEBOUNCE_MS) + _log("INFO", "[Agent][焦点] 安排 TTS round-end 延迟 ${TTS_RESUME_DEBOUNCE_MS}ms reason=$reason") + } + // 状态 - 使用原子类型确保线程安全 private val _isInitialized = AtomicBoolean(false) val isInitialized: Boolean get() = _isInitialized.get() @@ -611,6 +658,9 @@ object AgentService : CoroutineScope { ttsService?.release() ttsService = null + // 清掉 TTS round-end 的延迟任务,避免服务释放后还回调 + cancelPendingTtsResume() + job.cancel() clearListeners() _isInitialized.set(false) @@ -682,15 +732,21 @@ object AgentService : CoroutineScope { } TtsEventType.SYNTHESIS_CANCELED -> { - _isTtsSpeaking.set(false) + // 明确结束信号:取消 pending debounce 并立即跑 round-end + cancelPendingTtsResume() restartIdleCheck() sendEvent("tts_canceled", mapOf("status" to "canceled")) - // TTS取消时停止气泡音 - // audioPlayer?.stopAudio() - tryReleaseAiAudioFocus("ttsSynthesisCanceled") + performTtsRoundEnd("ttsSynthesisCanceled") } TtsEventType.PLAYBACK_STARTED -> { //开始AI播报内容 + // 新段开始:若有待执行的 round-end(上一段 COMPLETED 安排的 1s 延迟), + // 直接取消——说明只是段间隙,不是真的结束 + val hadPending = pendingTtsResumeRunnable != null + cancelPendingTtsResume() + if (hadPending) { + _log("INFO", "[Agent][焦点] 新段 PLAYBACK_STARTED 到来,取消 pending round-end(段间隙)") + } restartIdleCheck() sendEvent("playback_started", mapOf("status" to "playback_started")) // 确保气泡音已停止,避免与TTS播放重叠 @@ -706,40 +762,25 @@ object AgentService : CoroutineScope { context.sendBroadcast(intent) // Log.e(TAG, "QQ 音乐暂停播放") autopauseMusic() - // QQMusicSingleton.getInstance().autopauseMusic { result -> - // if (result.isSuccess) { - // isplaymusic = true - // } - // } } - TtsEventType.PLAYBACK_COMPLETED -> { //结束AI播报内容 - // 防御性重置:有些 TTS 引擎会把 PLAYBACK_COMPLETED 发在 SYNTHESIS_COMPLETED 之前, - // 导致到这里时 _isTtsSpeaking 还是 true,进而被 tryReleaseAiAudioFocus 的 guard 拦下, - // 结果就是 AI 说完了但外部音乐/焦点一直没恢复 - _isTtsSpeaking.set(false) - _log("INFO", "[Agent][焦点] TTS PLAYBACK_COMPLETED —— AI 对话结束,准备恢复音乐并释放焦点") + TtsEventType.PLAYBACK_COMPLETED -> { //当前段播报内容结束(不一定是整轮结束) restartIdleCheck() sendEvent("playback_completed", mapOf("status" to "playback_completed")) - // 发送AI说话开始广播 - val intent = - Intent("com.yunqiinnovation.music_service.AI_SAY_END").apply { - setPackage(context.packageName) - } - context.sendBroadcast(intent) - if(currentRecognitionMode == "ble_wakeup"&&_isInterrupt.get()){ - BleService.openB1Encoder() - } - resumeMusic() - tryReleaseAiAudioFocus("ttsPlaybackCompleted") + // 流式 TTS 按标点切段,此处只能代表"当前段"播完。 + // 用 1s debounce:若期间没有新 PLAYBACK_STARTED,再真正 resume 音乐 + 释放焦点。 + // 注意:这里不提前把 _isTtsSpeaking 置 false, + // 避免段间隙被 idle check / focus guard 误判为"空闲"。 + scheduleTtsRoundEnd("ttsPlaybackCompleted") } TtsEventType.ERROR -> { - _isTtsSpeaking.set(false) + // 明确结束信号:取消 pending debounce 并立即跑 round-end + cancelPendingTtsResume() restartIdleCheck() // TTS错误时停止气泡音 audioPlayer?.stopAudio() - tryReleaseAiAudioFocus("ttsError") + performTtsRoundEnd("ttsError") val params = event.params val code = params["errorCode"] as? String ?: "UNKNOWN_ERROR" val message = params["errorMessage"] as? String ?: "未知错误" @@ -1743,6 +1784,9 @@ object AgentService : CoroutineScope { if (isTtsSpeaking) { ttsService?.stop() + // 主动停止是明确的结束信号:清掉 debounce 并立即完成 round-end + // (若 ttsService.stop() 之后 SYNTHESIS_CANCELED 事件照常到达,round-end 逻辑是幂等的) + cancelPendingTtsResume() _isTtsSpeaking.set(false) restartIdleCheck() // 状态变化,重启检测 sendEvent("tts_stopped", mapOf("status" to "stopped")) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 6e905b1c5..11de230ff 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -161,7 +161,28 @@ class AgentServiceImpl: NSObject { azureAsrHelper = AzureAsrHelper() azureTtsHelper = AzureTtsHelper() - + // TTS 在段间隙做 deactivate 时,回查业务层状态: + // AI 流式/pendingProcess 工作中时不释放 session,避免外部 App 在 AI 多段合成之间被反复打断-恢复。 + // 注意:ble_wakeup 模式下 ASR 是"持续监听"(不关代表对话结束),不应把它算作 busy—— + // 否则 TTS 播完后外部 App 要等到 5s idle 超时 ASR 自行停掉才释放,中间过长的 + // interruption 会让很多外部 App(Spotify / 网易云 / YouTube)放弃 auto-resume。 + azureTtsHelper?.externalBusyProvider = { [weak self] in + guard let self = self else { return false } + let pendingWork = (self.pendingProcessWorkItem != nil) && !(self.pendingProcessWorkItem?.isCancelled ?? true) + let hasPendingText = !self.recognitionResult.isEmpty + let recogBlocking: Bool + if self.currentRecognitionMode == "ble_wakeup" { + recogBlocking = false + } else { + recogBlocking = self.isRecognizing || self.isStartingRecognition + } + return self.isAiStreaming || recogBlocking || pendingWork || hasPendingText + } + // 把 TTS 音频会话相关的关键日志转发到 app 侧日志通道(EventChannel → Flutter),可通过 app 导出排障 + azureTtsHelper?.exportLog = { [weak self] level, message in + self?._log(level, message) + } + chatApiService = ChatApiService() // chatStorageHelper 现在是单例,不需要初始化 @@ -559,19 +580,42 @@ class AgentServiceImpl: NSObject { } } - // 仅当 ASR / AI / TTS 都空闲、且没有累计识别文本等待触发 AI 时才恢复外部音乐。 + // 仅当 ASR / AI / TTS 都空闲、且没有累计识别文本等待触发 AI 时才恢复音乐。 // 否则会出现"超时中断唤醒→立刻恢复音乐→紧接着 onResult 触发 AI + TTS"同时响的问题。 + // + // 分两路恢复: + // 1) 内部 MusicService(wasMusicPlayingBeforeRecognition)—— 直接 resume + // 2) 外部 App(wasExternalAudioPlayingBeforeRecognition)—— 通过 TTS helper 防抖 deactivate + // + .notifyOthersOnDeactivation 让系统通知外部 App。防抖是因为 Azure 流式 TTS 按标点切段, + // 每段播完都会进到这里,若每次立即 deactivate,段间隙外部 App 会抢回播放。 internal func tryResumeExternalMusicIfIdle(reason: String) { - guard wasMusicPlayingBeforeRecognition else { return } + if !wasMusicPlayingBeforeRecognition && !wasExternalAudioPlayingBeforeRecognition { + return + } let pendingWork = (pendingProcessWorkItem != nil) && !(pendingProcessWorkItem?.isCancelled ?? true) let hasPendingText = !recognitionResult.isEmpty - if isRecognizing || isStartingRecognition || isSpeaking || isAiStreaming || pendingWork || hasPendingText { - _log("INFO", "[Agent] defer music resume (\(reason)) recog=\(isRecognizing) starting=\(isStartingRecognition) tts=\(isSpeaking) ai=\(isAiStreaming) pending=\(pendingWork) text=\(hasPendingText)") + // ble_wakeup 模式下 ASR 是"持续监听等待唤醒",TTS 播完 + AI 完成即视为对话结束, + // 不应为了等 ASR 自然超时而延长外部 App 的 interruption。 + let recogBlocking: Bool + if currentRecognitionMode == "ble_wakeup" { + recogBlocking = false + } else { + recogBlocking = isRecognizing || isStartingRecognition + } + if recogBlocking || isSpeaking || isAiStreaming || pendingWork || hasPendingText { + _log("INFO", "[Agent] defer music resume (\(reason)) mode=\(currentRecognitionMode) recog=\(isRecognizing) starting=\(isStartingRecognition) tts=\(isSpeaking) ai=\(isAiStreaming) pending=\(pendingWork) text=\(hasPendingText)") return } - wasMusicPlayingBeforeRecognition = false - _log("INFO", "[Agent] resume external music (\(reason))") - MusicService.shared.resume() + if wasMusicPlayingBeforeRecognition { + wasMusicPlayingBeforeRecognition = false + _log("INFO", "[Agent] resume internal music (\(reason))") + MusicService.shared.resume() + } + if wasExternalAudioPlayingBeforeRecognition { + wasExternalAudioPlayingBeforeRecognition = false + _log("INFO", "[Agent] schedule deactivate to notify external app (\(reason))") + azureTtsHelper?.scheduleDebouncedDeactivate() + } } func startRecognition(useBle: Bool = false, mode: String = "normal") -> Bool { if agentMode == "sts" { diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AliyunBailianE2EHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AliyunBailianE2EHelper.kt index 65a6e440d..d100a8ad1 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AliyunBailianE2EHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AliyunBailianE2EHelper.kt @@ -93,6 +93,11 @@ class AliyunBailianE2EHelper( private val isStarted = AtomicBoolean(false) private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + // 诊断计数器 + private var pushCount = 0L + private var pushDroppedCount = 0L + private var recvMsgCount = 0L + /** * 初始化助手,设置配置与回调。 * @@ -101,20 +106,29 @@ class AliyunBailianE2EHelper( * @return 是否初始化成功 */ fun initialize(config: Config, cb: Callback): Boolean { + // 切换语言对(或重复初始化)必须彻底销毁旧 WebSocket, + // 否则 startContinuousConversation 的 isStarted 短路会导致新配置不生效。 + Log.d(TAG, "initialize: resetting for new config src=${config.sourceLanguage}->${config.targetLanguage}, prevStarted=${isStarted.get()}") + try { webSocket?.close(1000, "reinit with new config") } catch (_: Exception) {} + webSocket = null + isStarted.set(false) + pushCount = 0 + pushDroppedCount = 0 + recvMsgCount = 0 + audioChunkBuffer = ByteArray(0) + recvTextBuffer.setLength(0) + fullTextBuffer.setLength(0) + fullAudioBuffer.reset() - // 注意:通义千问 LiveTranslate 通常使用 2 位语言代码,但部分方言可能不同。 - // 这里做一个简单的处理,具体需参考文档支持的语种列表。 - - - conf = config.copy(sourceLanguage = config.sourceLanguage, targetLanguage = config.targetLanguage) + conf = config.copy(sourceLanguage = config.sourceLanguage, targetLanguage = config.targetLanguage) callback = cb - Log.d(TAG, "initialize: wsUrl=${conf.wsUrl}, model=${conf.appId}, src=${conf.sourceLanguage}, tgt=${conf.targetLanguage}") - + Log.d(TAG, "initialize: wsUrl=${conf.wsUrl}, model=${conf.appId}, src=${conf.sourceLanguage}, tgt=${conf.targetLanguage}") + client = OkHttpClient.Builder() .pingInterval(30, TimeUnit.SECONDS) .readTimeout(0, TimeUnit.SECONDS) .build() - + startContinuousConversation() return true } @@ -309,8 +323,11 @@ class AliyunBailianE2EHelper( try { val json = JSONObject(text) val type = json.optString("type") - - // Log.d(TAG, "Received event: $type") + + recvMsgCount++ + if (recvMsgCount <= 5 || recvMsgCount % 50 == 1L) { + Log.d(TAG, "Received event #$recvMsgCount: type=$type (src=${conf.sourceLanguage}->${conf.targetLanguage})") + } when (type) { "error" -> { @@ -443,17 +460,28 @@ class AliyunBailianE2EHelper( * 音频数据需 Base64 编码。 */ fun pushAudioData(data: ByteArray): Boolean { - val ws = webSocket ?: return false - if (!isStarted.get()) return false - + val ws = webSocket + val started = isStarted.get() + if (ws == null || !started) { + pushDroppedCount++ + if (pushDroppedCount % 100 == 1L) { + Log.w(TAG, "pushAudioData dropped: ws=${ws != null}, started=$started, dropped=$pushDroppedCount (src=${conf.sourceLanguage}->${conf.targetLanguage})") + } + return false + } + try { // 将 PCM 数据转为 Base64 val base64Audio = data.toByteString().base64() - + val json = JSONObject() json.put("type", "input_audio_buffer.append") json.put("audio", base64Audio) ws.send(json.toString()) + pushCount++ + if (pushCount % 100 == 1L) { + Log.d(TAG, "pushAudioData ok: count=$pushCount, size=${data.size}, src=${conf.sourceLanguage}->${conf.targetLanguage}") + } return true } catch (e: Exception) { Log.e(TAG, "pushAudioData error", e) diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt index 815a067db..90f2aaa97 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt @@ -133,6 +133,14 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, private var astAudioPushCountA = 0L private var astAudioPushCountB = 0L + // 静音门控:连续静音帧数超过阈值后暂停对应通道的推流,防止豆包等端到端 STS 服务端 + // 因持续收到空音频触发 stream is done / session timeout 而断连。 + // 单帧 20ms(16kHz stereo 1280B),50 帧 ≈ 1s;门控阈值 500 足以区分语音与底噪。 + private var leftSilentStreak = 0 + private var rightSilentStreak = 0 + private val gatePeakThreshold = 500 + private val silenceFramesToMute = 50 + private fun pushAstAudioToA(data: ByteArray) { astAudioPushCountA++ if (astAudioPushCountA % 100 == 1L) { @@ -658,13 +666,6 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, val removeFirstPunctuation = call.argument("isRemoveFirstPunctuation") ?: true - // 通话翻译模式(external):AST端到端服务已处理识别+翻译+TTS,无需启动ASR - if (useExternalAudio) { - FileLogger.d(tag, "通话模式,AST端到端服务已激活(provider=$currentAstProvider),跳过ASR") - result.success(true) - return - } - // 确保事件通道已准备好 if (asrEventSink == null) { result.error( @@ -677,7 +678,14 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, try { FileLogger.d(tag, "选择音频源类型: ${useExternalAudio}") - val audioSourceType = AzureAsrHelper.AudioSourceType.MICROPHONE + // external:多媒体翻译(audioVideo)——BLE 对端音频经 channel=1 喂给 Azure ASR + // microphone:同声/面对面——本机麦克风 + // 通话模式(call)不走这里:上层已跳过 ASR,由 AST 端到端服务处理 + val audioSourceType = if (useExternalAudio) { + AzureAsrHelper.AudioSourceType.EXTERNAL + } else { + AzureAsrHelper.AudioSourceType.MICROPHONE + } val success = azureAsrHelper.startContinuousRecognition(audioSourceType) result.success(true) } catch (e: Exception) { @@ -1798,7 +1806,9 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, val leftBuffer = ByteArray(sampleCount * 2) // 左声道缓冲区 val rightBuffer = ByteArray(sampleCount * 2) // 右声道缓冲区 - // 拆分交错的左右声道数据 + // 拆分交错的左右声道数据,同时计算 peak 用于静音门控 + var leftPeak = 0 + var rightPeak = 0 for (i in 0 until sampleCount) { val stereoIndex = i * 4 val monoIndex = i * 2 @@ -1810,9 +1820,42 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, // 右声道 rightBuffer[monoIndex] = data[stereoIndex + 2] rightBuffer[monoIndex + 1] = data[stereoIndex + 3] + + val lSample = ((data[stereoIndex + 1].toInt() shl 8) or (data[stereoIndex].toInt() and 0xFF)).toShort() + val rSample = ((data[stereoIndex + 3].toInt() shl 8) or (data[stereoIndex + 2].toInt() and 0xFF)).toShort() + val lAbs = Math.abs(lSample.toInt()) + val rAbs = Math.abs(rSample.toInt()) + if (lAbs > leftPeak) leftPeak = lAbs + if (rAbs > rightPeak) rightPeak = rAbs } - // 计算左右声道 RMS 音量(每200帧打一次) + // 更新静音连续帧计数 + val leftWasMuted = leftSilentStreak >= silenceFramesToMute + val rightWasMuted = rightSilentStreak >= silenceFramesToMute + if (leftPeak >= gatePeakThreshold) { + if (leftWasMuted) { + FileLogger.i(tag, "[STS] A(己方)恢复推流: peak=$leftPeak (之前静音${leftSilentStreak}帧)") + } + leftSilentStreak = 0 + } else { + leftSilentStreak++ + if (leftSilentStreak == silenceFramesToMute) { + FileLogger.i(tag, "[STS] A(己方)静音超过${silenceFramesToMute}帧,暂停推流") + } + } + if (rightPeak >= gatePeakThreshold) { + if (rightWasMuted) { + FileLogger.i(tag, "[STS] B(对方)恢复推流: peak=$rightPeak (之前静音${rightSilentStreak}帧)") + } + rightSilentStreak = 0 + } else { + rightSilentStreak++ + if (rightSilentStreak == silenceFramesToMute) { + FileLogger.i(tag, "[STS] B(对方)静音超过${silenceFramesToMute}帧,暂停推流") + } + } + + // 计算左右声道 RMS 音量(每200帧打一次,含门控状态) if (bleAudioRecvCount % 200 == 1L) { var leftRms = 0.0 var rightRms = 0.0 @@ -1827,15 +1870,22 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, val samples = leftBuffer.size / 2 leftRms = Math.sqrt(leftRms / samples) rightRms = Math.sqrt(rightRms / samples) - FileLogger.d(tag, "[STS] 声道音量: left(己方/麦克风)=${leftRms.toInt()}, right(对方)=${rightRms.toInt()}, threshold=$lowVolumeThreshold") + val aState = if (leftSilentStreak >= silenceFramesToMute) "muted(streak=$leftSilentStreak)" else "active" + val bState = if (rightSilentStreak >= silenceFramesToMute) "muted(streak=$rightSilentStreak)" else "active" + FileLogger.d(tag, "[STS] 声道音量: left(己方/麦克风)=${leftRms.toInt()}[peak=$leftPeak,$aState], right(对方)=${rightRms.toInt()}[peak=$rightPeak,$bState], threshold=$lowVolumeThreshold, gate=$gatePeakThreshold") } // 过滤低音量音频 val filteredLeftBuffer = filterLowVolumeAudio(leftBuffer, lowVolumeThreshold) val filteredRightBuffer = filterLowVolumeAudio(rightBuffer, lowVolumeThreshold) // 左声道是自己的(麦克风),右声道是对方的 - pushAstAudioToA(filteredLeftBuffer) - pushAstAudioToB(filteredRightBuffer) + // 静音门控:持续静音时不推流,避免服务端(尤其豆包)空流超时断连 + if (leftSilentStreak < silenceFramesToMute) { + pushAstAudioToA(filteredLeftBuffer) + } + if (rightSilentStreak < silenceFramesToMute) { + pushAstAudioToB(filteredRightBuffer) + } } diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt index f2ae5df40..690e45dab 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt @@ -192,8 +192,13 @@ class DoubaoE2ETranslateHelper( val msg = if (resp.hasResponseMeta()) resp.responseMeta.message else "" Log.e(TAG, "onMessage: session error event=${event} msg=${msg}") - if (msg.contains("Timeout waiting next packet", ignoreCase = true)) { - Log.i(TAG, "onMessage: detected timeout error, triggering auto-restart") + // 可自动重连的服务端异常:通常是空闲/长会话导致的 stream 状态机结束,重建 WS 即可恢复 + val isRetryable = msg.contains("Timeout waiting next packet", ignoreCase = true) + || msg.contains("stream is done", ignoreCase = true) + || msg.contains("session timeout", ignoreCase = true) + || msg.contains("current state=Finished", ignoreCase = true) + if (isRetryable) { + Log.i(TAG, "onMessage: retryable server error, triggering auto-restart: '$msg'") restartSession() return } diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index 36eb409d5..20e2d3b29 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -994,7 +994,8 @@ private func checkStreamPlaybackCompletedIfNeeded() { hasNotifiedPlaybackStartedForStream = false // os_log("调用链: 流式播放完成 session=%{public}@", log: log, type: .info, sessionid) notifyEvent(eventType: .playbackCompleted) - deactivateAudioSessionAfterTTSIfIdle() + // 段完成:防抖后再释放 session,避免段间隙被外部 App 抢回 + scheduleDebouncedDeactivate() } } @@ -1269,7 +1270,8 @@ public func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully fl notifyEvent(eventType: .playbackCompleted) // 如果还有剩余,继续播放 startPlaybackIfNeeded() - deactivateAudioSessionAfterTTSIfIdle() + // 段完成:防抖后再释放 session,避免段间隙被外部 App 抢回 + scheduleDebouncedDeactivate() } /** @@ -1418,11 +1420,118 @@ private var shouldDeactivateAudioSessionWhenIdle = false /// 外部调用方(如 AgentServiceImpl)在本轮唤醒/识别开始时设置: /// - true 表示唤醒前有外部或内部音乐在放,TTS 空闲释放 session 时应带上 .notifyOthersOnDeactivation 让被打断的 App 恢复播放 /// - 成功 deactivate 后由本类自动清零 -public var notifyOthersOnNextDeactivate: Bool = false +public var notifyOthersOnNextDeactivate: Bool = false { + didSet { + // 新一轮对话要求通知外部恢复时,重置重试预算,避免上一轮耗尽导致本轮永不重试 + if notifyOthersOnNextDeactivate && !oldValue { + deactivateRetryCount = 0 + } + } +} // 新增:路由变化的去抖与配置重入保护标记 private var routeChangeDebounceWorkItem: DispatchWorkItem? private var isApplyingAudioSessionConfig = false + /// deactivate 重试调度:playerNode 残留 isPlaying / setActive(false) 抛 IsBusy 时, + /// 用退避 timer 兜底再试,避免外部音乐因一次失败永远无法恢复。 + private let deactivateRetryDelays: [TimeInterval] = [0.3, 0.8, 1.5] + private var deactivateRetryCount: Int = 0 + private var pendingDeactivateRetryWorkItem: DispatchWorkItem? + + /// 由业务层(AgentServiceImpl)注入:返回 true 表示 AI/ASR 仍在处理本轮对话, + /// 即使 TTS 段已播完也不应 deactivate——否则外部 App 会在 AI 流式段间被反复 + /// "打断-恢复",多轮折腾后再也不自动恢复。 + public var externalBusyProvider: (() -> Bool)? + + /// 由业务层注入,用于把 TTS 音频会话相关的关键日志转发到 app 侧日志通道(可导出)。 + /// 不注入时仅写 OSLog。 + public var exportLog: ((String, String) -> Void)? + + /// 我们自己追踪的 session 激活状态——setActive(true) 成功置 true,setActive(false) 成功置 false。 + /// 用于 applyAudioSessionForTTS 的快速路径:已激活且 category/mode 匹配时不重复 setActive(true), + /// 避免 PCM chunk 级的重入疯狂触发系统调用(实测单轮 AI 回复可调 250+ 次)。 + private var internalSessionActive: Bool = false + + /// 统一打印:OSLog + app 导出通道双写。 + /// tag 用固定前缀 `[TTS/Audio]` 方便在导出日志里过滤。 + private func dlog(_ level: String = "INFO", _ message: String) { + let line = "[TTS/Audio] " + message + switch level { + case "ERROR": + os_log("%{public}@", log: self.log, type: .error, line) + case "WARNING": + os_log("%{public}@", log: self.log, type: .default, line) + case "DEBUG": + os_log("%{public}@", log: self.log, type: .debug, line) + default: + os_log("%{public}@", log: self.log, type: .info, line) + } + exportLog?(level, line) + } + + /// Azure 流式 TTS 按标点切段,每段都会发 playbackCompleted。 + /// 若每段都立即 deactivate(notifyOthers),段间隙外部 App 会短暂抢回播放 + /// ——体感就是"AI 说到一半外部音乐冒出来"。 + /// 用 1s debounce:playbackCompleted → 延迟释放;期间新段开始则取消 pending。 + private let deactivateDebounceDelay: TimeInterval = 1.0 + private var pendingDeactivateWorkItem: DispatchWorkItem? + + /// 段间防抖 deactivate:延迟 1s 后调用 deactivateAudioSessionAfterTTSIfIdle。 + /// 新段开始播(applyAudioSessionForTTS)会取消 pending。 + /// 注意:业务层(AgentServiceImpl)可能在 AI 终态时调用本方法,而此时 TTS 从未 + /// 实际 apply 过 session(例如 AI 无文本产出),shouldDeactivateAudioSessionWhenIdle 还是 false。 + /// 为了让 ASR 激活但未释放的 session 也能被释放以通知外部 App,这里显式打开该标志。 + public func scheduleDebouncedDeactivate() { + dlog("INFO", "防抖 deactivate 排队 delay=\(String(format: "%.2f", deactivateDebounceDelay))s notifyOthers=\(notifyOthersOnNextDeactivate) externalBusy=\(externalBusyProvider?() ?? false)") + shouldDeactivateAudioSessionWhenIdle = true + cancelPendingDebouncedDeactivate() + let work = DispatchWorkItem { [weak self] in + guard let self = self else { return } + self.pendingDeactivateWorkItem = nil + self.dlog("INFO", "防抖 deactivate 触发执行") + self.deactivateAudioSessionAfterTTSIfIdle() + } + pendingDeactivateWorkItem = work + DispatchQueue.main.asyncAfter(deadline: .now() + deactivateDebounceDelay, execute: work) + } + + /// 取消等待中的防抖 deactivate。 + public func cancelPendingDebouncedDeactivate() { + if pendingDeactivateWorkItem != nil { + dlog("INFO", "取消 pending 防抖 deactivate") + } + pendingDeactivateWorkItem?.cancel() + pendingDeactivateWorkItem = nil + } + + /// 调度下一次 deactivate 重试;若已到上限或已在排队则不重复调度。 + private func scheduleDeactivateRetry(reason: String) { + if pendingDeactivateRetryWorkItem != nil { return } + if deactivateRetryCount >= deactivateRetryDelays.count { + dlog("WARNING", "deactivate 重试已达上限(\(deactivateRetryDelays.count)) reason=\(reason)") + return + } + let delay = deactivateRetryDelays[deactivateRetryCount] + deactivateRetryCount += 1 + dlog("INFO", "安排 deactivate 重试 #\(deactivateRetryCount) delay=\(String(format: "%.2f", delay))s reason=\(reason)") + let work = DispatchWorkItem { [weak self] in + guard let self = self else { return } + self.pendingDeactivateRetryWorkItem = nil + self.deactivateAudioSessionAfterTTSIfIdle() + } + pendingDeactivateRetryWorkItem = work + DispatchQueue.main.asyncAfter(deadline: .now() + delay, execute: work) + } + + /// 取消等待中的 deactivate 重试(新 TTS 段开始时调用)。 + private func cancelPendingDeactivateRetry() { + if pendingDeactivateRetryWorkItem != nil { + dlog("INFO", "取消 pending deactivate 重试") + } + pendingDeactivateRetryWorkItem?.cancel() + pendingDeactivateRetryWorkItem = nil + } + private func isTtsBusy() -> Bool { if speaking { return true } if isPlaying { return true } @@ -1461,7 +1570,22 @@ private func ensureAudioSessionQueueSpecificKeySet() { */ private func applyAudioSessionForTTS() -> Bool { var success = true - // os_log("liwei-------------TTS applyAudioSessionForTTS currentRecognitionMode:%{public}@", log: self.log, type: .info, currentRecognitionMode) + // 新段即将开始播:取消段间隙 pending 的 deactivate + retry,避免中途释放 session + cancelPendingDebouncedDeactivate() + cancelPendingDeactivateRetry() + + // 快速路径:session 已激活且 category/mode 匹配当前模式预期时直接返回, + // 避免 PCM chunk 级调用触发 250+ 次 setActive(true) 对外部 App 的反复干扰 + let sessionSnapshot = AVAudioSession.sharedInstance() + let isBlePlaybackMode = (currentRecognitionMode == "push_to_talk" + || currentRecognitionMode == "normal" + || currentRecognitionMode == "ble_wakeup") + if internalSessionActive && !audioInterrupted && isBlePlaybackMode + && sessionSnapshot.category == .playback + && sessionSnapshot.mode == .spokenAudio { + return true + } + dlog("INFO", "applyAudioSessionForTTS 入口 mode=\(currentRecognitionMode) otherAudioPlaying=\(sessionSnapshot.isOtherAudioPlaying) category=\(sessionSnapshot.category.rawValue) sessionMode=\(sessionSnapshot.mode.rawValue) internalActive=\(internalSessionActive)") ensureAudioSessionQueueSpecificKeySet() let applyBody = { if self.isApplyingAudioSessionConfig { return } @@ -1536,6 +1660,7 @@ private func applyAudioSessionForTTS() -> Bool { _ = try? audioSession.setPreferredIOBufferDuration(0.02) } try audioSession.setActive(true) + self.internalSessionActive = true if self.currentRecognitionMode == "phone_call", !hasHeadphones { _ = try? audioSession.overrideOutputAudioPort(.speaker) } @@ -1627,40 +1752,101 @@ private func isTelephonyActive() -> Bool { */ private func deactivateAudioSessionAfterTTSIfIdle() { ensureAudioSessionQueueSpecificKeySet() + dlog("INFO", "deactivateAudioSessionAfterTTSIfIdle 入口") let work = { [weak self] in guard let self = self else { return } - if self.micCapture?.isCapturing == true { return } - if self.speaking { return } - if self.isPlaying { return } - if !self.playbackQueue.isEmpty { return } - if self.pendingTextCount > 0 { return } - if !self.pendingTasks.isEmpty { return } - if self.activeStreamSynthesisCount > 0 { return } - if !self.pcmPendingData.isEmpty { return } - if self.scheduledBufferCount > 0 { return } + // 注:这里刻意不再用 externalBusy skip。 + // 之前试过"AI 还在 streaming 时段间 skip"——会导致整轮对话只在最终发一次 + // notifyOthersOnDeactivation,有些外部 App(尤其第三方音乐)一次信号来不 + // 及/会被吃掉,表现为"完全不恢复"。 + // 放行后段间会反复 deactivate(notifyOthers),外部 App 能持续收到 ended, + // 虽然有段间短暂冒音的副作用,但能稳定恢复。 + if self.externalBusyProvider?() == true { + self.dlog("INFO", "external busy (AI/ASR 仍忙)——仍允许 deactivate,外部 App 稳定恢复优先") + } + if self.micCapture?.isCapturing == true { + self.dlog("INFO", "deactivate skip: mic capturing,排重试") + self.scheduleDeactivateRetry(reason: "micCapturingOuter") + return + } + if self.speaking { + self.dlog("INFO", "deactivate skip: speaking=true") + return + } + if self.isPlaying { + self.dlog("INFO", "deactivate skip: isPlaying=true") + return + } + if !self.playbackQueue.isEmpty { + self.dlog("INFO", "deactivate skip: playbackQueue=\(self.playbackQueue.count)") + return + } + if self.pendingTextCount > 0 { + self.dlog("INFO", "deactivate skip: pendingTextCount=\(self.pendingTextCount)") + return + } + if !self.pendingTasks.isEmpty { + self.dlog("INFO", "deactivate skip: pendingTasks=\(self.pendingTasks.count)") + return + } + if self.activeStreamSynthesisCount > 0 { + self.dlog("INFO", "deactivate skip: activeStreamSynthesisCount=\(self.activeStreamSynthesisCount)") + return + } + if !self.pcmPendingData.isEmpty { + self.dlog("INFO", "deactivate skip: pcmPendingData=\(self.pcmPendingData.count)") + return + } + if self.scheduledBufferCount > 0 { + self.dlog("INFO", "deactivate skip: scheduledBufferCount=\(self.scheduledBufferCount)") + return + } + dlog("INFO", "deactivate 进入 playbackQueue:准备 stop engine/playerNode") self.audioPlaybackQueue.async { [weak self] in guard let self = self else { return } - if self.playerNode?.isPlaying == true { return } + let wasPlaying = self.playerNode?.isPlaying == true + // A:playerNode 若仍标记 isPlaying(buffer 播完后 flag 未翻转也会落在这), + // 不再直接 return 放任 session 挂住,而是主动 stop 再继续。 self.playerNode?.stop() self.audioEngine?.stop() self.audioEngine = nil self.playerNode = nil self.pcmFormat = nil - if self.shouldDeactivateAudioSessionWhenIdle { - // 麦克风仍在采集时,session 还被 ASR 持有,deactivate 必失败 - // 保留标记等下次 idle 再试,避免标记被清空后永远不再触发 - if self.micCapture?.isCapturing == true { return } + self.dlog("INFO", "deactivate 清理完成 wasPlayerPlaying=\(wasPlaying) shouldDeactivate=\(self.shouldDeactivateAudioSessionWhenIdle) notifyOthers=\(self.notifyOthersOnNextDeactivate)") + guard self.shouldDeactivateAudioSessionWhenIdle else { + self.dlog("INFO", "deactivate 跳过:shouldDeactivateAudioSessionWhenIdle=false") + return + } + // 麦克风仍在采集时,session 还被 ASR 持有,deactivate 必失败 + // 保留标记等下次 idle 再试,避免标记被清空后永远不再触发 + if self.micCapture?.isCapturing == true { + self.dlog("INFO", "deactivate 跳过:mic 仍在采集,排重试") + self.scheduleDeactivateRetry(reason: "micCapturing") + return + } - let opts: AVAudioSession.SetActiveOptions = - self.notifyOthersOnNextDeactivate ? .notifyOthersOnDeactivation : [] - do { - try AVAudioSession.sharedInstance().setActive(false, options: opts) - self.shouldDeactivateAudioSessionWhenIdle = false - self.notifyOthersOnNextDeactivate = false - } catch { - // 失败保留标记,下次 idle 再尝试(常见原因:其他 AudioUnit 仍在运行) - os_log("TTS deactivate 失败,保留标记待下次重试: %{public}@", log: self.log, type: .info, error.localizedDescription) - } + let session = AVAudioSession.sharedInstance() + let otherAudioPlaying = session.isOtherAudioPlaying + // 总是带上 .notifyOthersOnDeactivation: + // - 这是系统 hint,只对本应用实际暂停过的外部 App 生效,未被暂停的 App 不受影响 + // - 之前依赖 notifyOthersOnNextDeactivate 标志,但多轮对话里标志会被 + // "setActive 成功后清零 + 下轮 startRecognition 读到 isOtherAudioPlaying=false 覆盖" + // 组合攻击擦掉,导致外部音乐恢复不了 + let opts: AVAudioSession.SetActiveOptions = .notifyOthersOnDeactivation + self.dlog("INFO", "准备 setActive(false) flagHint=\(self.notifyOthersOnNextDeactivate) otherAudioPlaying=\(otherAudioPlaying) category=\(session.category.rawValue) mode=\(session.mode.rawValue)") + do { + try session.setActive(false, options: opts) + self.dlog("INFO", "✅ setActive(false) 成功(已带 notifyOthers)") + self.shouldDeactivateAudioSessionWhenIdle = false + self.notifyOthersOnNextDeactivate = false + self.internalSessionActive = false + self.deactivateRetryCount = 0 + } catch { + let nsError = error as NSError + // B:失败保留标记,并排一次退避重试(0.3→0.8→1.5s)。 + // 常见原因:其他 AudioUnit 仍在收尾(IsBusy)。 + self.dlog("ERROR", "❌ setActive(false) 失败 domain=\(nsError.domain) code=\(nsError.code) msg=\(error.localizedDescription),安排退避重试") + self.scheduleDeactivateRetry(reason: "setActiveFailed") } } } @@ -1685,6 +1871,8 @@ private func deactivateAudioSessionAfterTTSIfIdle() { switch type { case .began: audioInterrupted = true + // 系统已挂起我们的 session,内部激活状态置 false,让 ended 后的重配能走完整 apply 流程 + internalSessionActive = false _ = stop() os_log("音频中断开始(可能为通话/Siri),停止TTS", log: log, type: .info) case .ended: