Browse Source

上传iOS音乐恢复逻辑

weicu
liwei1dao 6 months ago
parent
commit
933a406d1d
  1. 71
      lib/data/services/language_manager.dart
  2. 142
      lib/modules/translation/controllers/translation_controller.dart
  3. 100
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  4. 60
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift
  5. 56
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AliyunBailianE2EHelper.kt
  6. 76
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt
  7. 9
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt
  8. 244
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift

71
lib/data/services/language_manager.dart

@ -612,8 +612,18 @@ class LanguageManager extends GetxService {
.toList();
}
/// 豆包端到端 STS 实际优化良好的语言白名单
/// 只有源和目标都在白名单内,且至少一个是 zh/en 时,才走豆包;
/// 其余情况退到阿里或微软,避免法/德/西/葡/俄等语种走豆包端到端时
/// 出现词被拆碎、ASR 误识别、WebSocket 频繁断开等问题
static const Set<String> _volcanoE2EWhitelist = {'zh', 'en', 'ja', 'ko'};
/// 查找同时支持源语言和目标语言的最佳提供商(通过 shortCode 匹配)
/// 优先级:豆包(Volcano) > 阿里(Alibaba) > 微软(Azure) > 讯飞(Iflytek)
/// 路由策略:
/// 1. 豆包:源/目标都在白名单(zh/en/ja/ko)且至少一个是 zh/en
/// 2. 阿里:源/目标都在阿里翻译表内(zh/en/ja/ko/fr/de/es/it/pt/ru)
/// 3. 微软:其他所有组合(阿里不覆盖的稀有语种,或两端有任一不在阿里白名单)
/// 4. 讯飞:兜底
BestProviderResult? findBestMatchingProvider(
String sourceShortCode, String targetShortCode) {
final providers = [
@ -622,8 +632,11 @@ class LanguageManager extends GetxService {
SpeechServiceType.azure,
SpeechServiceType.iflytek,
];
Logger.info(
'[TRANS-INIT] 开始匹配提供商: $sourceShortCode -> $targetShortCode, 策略=豆包(zh/en/ja/ko白名单且含zh|en)>阿里(含zh)>微软>讯飞');
for (final provider in providers) {
final providerName = _providerStringFromType(provider);
final transSpecs = _getTranslationSpecs(provider);
Map<String, String>? sourceTransSpec;
@ -641,20 +654,38 @@ class LanguageManager extends GetxService {
}
if (sourceTransSpec == null || targetTransSpec == null) {
Logger.info(
'[TRANS-INIT] 跳过 ${providerDisplayName(providerName)}: 翻译表缺少${sourceTransSpec == null ? " 源=$sourceShortCode" : ""}${targetTransSpec == null ? " 目标=$targetShortCode" : ""}');
continue;
}
// 豆包特殊限制:源语言或目标语言必须有一个是中文或英语
// 豆包端到端限制:
// (a) 源和目标都必须在白名单 {zh, en, ja, ko}
// (b) 且至少一个是 zh/en(豆包 STS 以中英为 pivot)
if (provider == SpeechServiceType.volcano) {
final isSourceZhOrEn =
sourceShortCode == 'zh' || sourceShortCode == 'en';
final isTargetZhOrEn =
targetShortCode == 'zh' || targetShortCode == 'en';
if (!isSourceZhOrEn && !isTargetZhOrEn) {
final bothInWhitelist =
_volcanoE2EWhitelist.contains(sourceShortCode) &&
_volcanoE2EWhitelist.contains(targetShortCode);
final hasZhOrEn = sourceShortCode == 'zh' ||
sourceShortCode == 'en' ||
targetShortCode == 'zh' ||
targetShortCode == 'en';
if (!bothInWhitelist) {
Logger.info(
'[TRANS-INIT] 跳过 ${providerDisplayName('volcano')}: 端到端仅对 {zh,en,ja,ko} 稳定,当前 $sourceShortCode/$targetShortCode 不全在白名单');
continue;
}
if (!hasZhOrEn) {
Logger.info(
'[TRANS-INIT] 跳过 ${providerDisplayName('volcano')}: 需源或目标至少一个为 zh/en(当前 $sourceShortCode/$targetShortCode)');
continue;
}
}
// 阿里端到端 (qwen3-livetranslate): 覆盖主流 10 语(zh/en/ja/ko/fr/de/es/it/pt/ru),
// 两端都在阿里翻译表内即走阿里;稀有语种交给微软兜底。
// 这里不加额外白名单——上面 transSpecs 的匹配检查已经做了筛选。
// 对于非端到端服务(Azure, Iflytek),还需要检查 ASR 和 TTS 支持
if (provider != SpeechServiceType.volcano &&
provider != SpeechServiceType.alibaba) {
@ -693,12 +724,16 @@ class LanguageManager extends GetxService {
}
final providerString = _providerStringFromType(provider);
final isEndToEnd = provider == SpeechServiceType.volcano ||
provider == SpeechServiceType.alibaba;
Logger.info(
'Found best provider: $providerString for $sourceShortCode -> $targetShortCode, sourceTts=${sourceTransSpec["ttsCode"]}, targetTts=${targetTransSpec["ttsCode"]}');
'[TRANS-INIT] 命中 ${providerDisplayName(providerString)}[$providerString] for $sourceShortCode -> $targetShortCode (端到端=$isEndToEnd), transSrc=${sourceTransSpec["translationCode"]}, transTgt=${targetTransSpec["translationCode"]}, asrSrc=${sourceTransSpec["asrCode"]}, asrTgt=${targetTransSpec["asrCode"]}, ttsSrc=${sourceTransSpec["ttsCode"]}, ttsTgt=${targetTransSpec["ttsCode"]}');
return BestProviderResult(
providerString, sourceTransSpec, targetTransSpec);
}
Logger.warning(
'[TRANS-INIT] 所有提供商都不匹配 $sourceShortCode -> $targetShortCode,返回 null');
return null;
}
@ -719,6 +754,26 @@ class LanguageManager extends GetxService {
}
}
/// 将 provider 字符串映射为中文厂商名,用于日志可读性
static String providerDisplayName(String provider) {
switch (provider) {
case 'volcano':
return '豆包(火山)';
case 'alibaba':
return '阿里(通义)';
case 'azure':
return '微软(Azure)';
case 'iflytek':
return '讯飞';
case 'google':
return '谷歌';
case 'flutter':
return '本地(Flutter)';
default:
return provider;
}
}
// ===== 私有工具方法:加载不同提供商的语言规格 =====
List<Map<String, String>> _getAsrSpecs(SpeechServiceType provider) {

142
lib/modules/translation/controllers/translation_controller.dart

@ -51,6 +51,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
final TtsService _ttsService = Get.find<TtsService>();
final AstService _astService = Get.find<AstService>();
final LanguageManager _languageManager = Get.find<LanguageManager>();
/// 最近一次通话模式初始化所选中的 AST provider,仅用于日志追踪
String _lastCallAstProvider = '';
final GetStorage _storage = GetStorage();
final BluetoothManager bluetoothManager = Get.find<BluetoothManager>();
final BleManager bleManager = Get.find<BleManager>();
@ -152,6 +154,10 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
// ==================== 重新初始化守卫 ====================
bool _isReinitializing = false; // 语言切换重新初始化期间为 true,抑制旧会话的错误事件
// 语言切换互斥:连续切换语言时串行执行,避免并发的 _reinitializeAsrService 导致
// 原生 AST 已启动但 isRecognizing 仍为 false 的状态不一致。
Future<void>? _languageSwitchChain;
// 播放防重复机制
String? _lastPlayedItemKey; // 最后播放的项目标识
int _lastPlayTime = 0; // 最后播放时间
@ -839,8 +845,17 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
final sourceShort = _languageManager.getShortCodeByAsrCode(sourceLanguageCode.value) ?? 'zh';
final targetShort = _languageManager.getShortCodeByAsrCode(targetLanguageCode.value) ?? 'en';
final sourceName = _languageManager.getChineseNameByAsrCode(sourceLanguageCode.value) ?? '?';
final targetName = _languageManager.getChineseNameByAsrCode(targetLanguageCode.value) ?? '?';
Logger.info(
'[TRANS-INIT] 通话翻译语言对: $sourceName($sourceShort/${sourceLanguageCode.value}) -> $targetName($targetShort/${targetLanguageCode.value})');
final bestProvider = _languageManager.findBestMatchingProvider(sourceShort, targetShort);
final astProvider = bestProvider?.provider ?? 'azure';
_lastCallAstProvider = astProvider;
if (bestProvider == null) {
Logger.warning(
'[TRANS-INIT] 未找到匹配的提供商,回退到默认 ${LanguageManager.providerDisplayName('azure')}[azure](asrCode=${sourceLanguageCode.value} -> ${targetLanguageCode.value})');
}
// 构建完整的语言参数:[asrCode0, asrCode1, transCode0, transCode1, ttsVoice0, ttsVoice1]
final transCode0 = bestProvider?.sourceSpec['translationCode'] ?? sourceLanguageCode.value;
@ -857,7 +872,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
ttsVoice0, // [4] TTS voice source
ttsVoice1, // [5] TTS voice target
];
Logger.info('1初始化AST服务,支持语言: $callModeLanguages, provider: $astProvider');
Logger.info(
'[TRANS-INIT] 最终启动服务: ${LanguageManager.providerDisplayName(astProvider)}[$astProvider], asrSrc=${callModeLanguages[0]}, asrTgt=${callModeLanguages[1]}, transSrc=${callModeLanguages[2]}, transTgt=${callModeLanguages[3]}, ttsSrc=${callModeLanguages[4]}, ttsTgt=${callModeLanguages[5]}');
await _astService.initialize(supportedLanguages: callModeLanguages, provider: astProvider);
@ -993,6 +1009,16 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
/// 同声翻译、通话翻译、多媒体翻译的语音识别调用入口
Future<void> startRecognition() async {
Logger.info('开始语音识别');
final srcName = _languageManager.getChineseNameByAsrCode(sourceLanguageCode.value) ?? '?';
final tgtName = _languageManager.getChineseNameByAsrCode(targetLanguageCode.value) ?? '?';
if (currentMode.value == 'call') {
final p = _lastCallAstProvider.isEmpty ? 'azure(未初始化)' : _lastCallAstProvider;
Logger.info(
'[TRANS-INIT] startRecognition: mode=call, 语言对=$srcName(${sourceLanguageCode.value}) -> $tgtName(${targetLanguageCode.value}), 服务=${LanguageManager.providerDisplayName(p)}[$p]');
} else {
Logger.info(
'[TRANS-INIT] startRecognition: mode=${currentMode.value}, 语言对=$srcName(${sourceLanguageCode.value}) -> $tgtName(${targetLanguageCode.value})');
}
//检查用户是否登录且积分是否足够,不足则提示充值
if (User.isLoggedIn() && User.instance.tradeintegral <= 0) {
showRechargeDialog('translation');
@ -1017,6 +1043,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
} else if (currentMode.value == 'faceToFace') {
Logger.info('iwei-------- 开始语音识别,模式:面对面,ASR push_to_talk');
await _startAsrService('push_to_talk');
} else if (currentMode.value == 'call') {
Logger.info('iwei-------- 通话模式:AST端到端服务已启动,跳过Azure ASR');
} else {
Logger.info(
'iwei-------- 开始语音识别,模式:${currentMode.value},ASR 模式:normal');
@ -1978,61 +2006,84 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
/// 获取底部控制栏高度
double get bottomBarHeight => _bottomBarHeight;
/// 串行化语言切换,避免并发 reinit 造成 UI 与原生 AST 状态不一致
Future<T> _runLanguageSwitch<T>(Future<T> Function() action) {
final previous = _languageSwitchChain;
final completer = Completer<T>();
final gate = Completer<void>();
Future<void> run() async {
if (previous != null) {
try {
await previous;
} catch (_) {}
}
try {
completer.complete(await action());
} catch (e, st) {
completer.completeError(e, st);
} finally {
gate.complete();
}
}
_languageSwitchChain = gate.future;
run();
return completer.future;
}
/// 切换源语言和目标语言
Future<void> swapLanguages() async {
final wasRecognizing = isRecognizing.value;
await stopAll();
Future<void> swapLanguages() => _runLanguageSwitch(() async {
final wasRecognizing = isRecognizing.value;
await stopAll();
final tempName = sourceLanguage.value;
sourceLanguage.value = targetLanguage.value;
targetLanguage.value = tempName;
final tempName = sourceLanguage.value;
sourceLanguage.value = targetLanguage.value;
targetLanguage.value = tempName;
final tempCode = sourceLanguageCode.value;
sourceLanguageCode.value = targetLanguageCode.value;
targetLanguageCode.value = tempCode;
final tempCode = sourceLanguageCode.value;
sourceLanguageCode.value = targetLanguageCode.value;
targetLanguageCode.value = tempCode;
_saveSourceLanguageSetting(isUserSet: true);
_saveTargetLanguageSetting();
await _reinitializeAsrService(restartRecognition: wasRecognizing);
}
_saveSourceLanguageSetting(isUserSet: true);
_saveTargetLanguageSetting();
await _reinitializeAsrService(restartRecognition: wasRecognizing);
});
/// 设置源语言
/// [language] 语言名称
Future<void> setSourceLanguage(String language) async {
if (sourceLanguage.value != language) {
final wasRecognizing = isRecognizing.value;
await stopAll();
Future<void> setSourceLanguage(String language) => _runLanguageSwitch(() async {
if (sourceLanguage.value == language) return;
final wasRecognizing = isRecognizing.value;
await stopAll();
sourceLanguage.value = language;
sourceLanguage.value = language;
final asrCode = _languageManager.getAsrCodeByChineseName(language);
if (asrCode != null) {
sourceLanguageCode.value = asrCode;
}
final asrCode = _languageManager.getAsrCodeByChineseName(language);
if (asrCode != null) {
sourceLanguageCode.value = asrCode;
}
_saveSourceLanguageSetting(isUserSet: true);
await _reinitializeAsrService(restartRecognition: wasRecognizing);
}
}
_saveSourceLanguageSetting(isUserSet: true);
await _reinitializeAsrService(restartRecognition: wasRecognizing);
});
/// 设置目标语言
/// [language] 语言名称
Future<void> setTargetLanguage(String language) async {
if (targetLanguage.value != language) {
final wasRecognizing = isRecognizing.value;
await stopAll();
Future<void> setTargetLanguage(String language) => _runLanguageSwitch(() async {
if (targetLanguage.value == language) return;
final wasRecognizing = isRecognizing.value;
await stopAll();
targetLanguage.value = language;
targetLanguage.value = language;
final asrCode = _languageManager.getAsrCodeByChineseName(language);
if (asrCode != null) {
targetLanguageCode.value = asrCode;
}
final asrCode = _languageManager.getAsrCodeByChineseName(language);
if (asrCode != null) {
targetLanguageCode.value = asrCode;
}
_saveTargetLanguageSetting();
await _reinitializeAsrService(restartRecognition: wasRecognizing);
}
}
_saveTargetLanguageSetting();
await _reinitializeAsrService(restartRecognition: wasRecognizing);
});
/// 重新初始化ASR服务
/// [restartRecognition] 初始化完成后是否自动重新开始识别
@ -2070,6 +2121,17 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
if (currentMode.value == "call") {
await _initializeCallModeTranslationService();
// _astService.initialize 的原生实现会自动启动 AST(AzureAsrToAsr / DoubaoE2E
// 等在 initialize 中都调用了 startContinuousTranslation)。若切换语言前未处于
// 识别状态,这里需要立即停止原生 AST,避免 UI 显示未启动但原生仍在识别。
if (!restartRecognition) {
try {
await _astService.stopContinuousTranslation();
Logger.info('[STS] 语言切换:非识别状态,已停止新AST保持状态一致');
} catch (e) {
Logger.info('[STS] 停止新AST(忽略): $e');
}
}
} else {
Logger.info('重新初始化ASR服务,支持语言: $asrSupportedLanguages');

100
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt

@ -321,6 +321,53 @@ object AgentService : CoroutineScope {
abandonAiAudioFocus()
}
// ========== TTS 轮次结束 debounce ==========
// Azure 流式 TTS 按标点切段合成,每段都会发一对 PLAYBACK_STARTED / PLAYBACK_COMPLETED。
// 如果每次 PLAYBACK_COMPLETED 都立即 resumeMusic + 释放焦点,就会在段间隙让外部音乐短暂抢回,
// 出现"AI 说到一半音乐就回来了"的现象。这里用 1s debounce:
// - PLAYBACK_COMPLETED → 延迟 1s 再执行真正的 round-end
// - 延迟期内又来 PLAYBACK_STARTED → 取消 pending,视为段间隙
// - CANCELED / ERROR → 取消 pending,立刻跑 round-end(属于明确结束信号)
private val ttsResumeHandler = Handler(Looper.getMainLooper())
private var pendingTtsResumeRunnable: Runnable? = null
private const val TTS_RESUME_DEBOUNCE_MS = 1000L
private fun cancelPendingTtsResume() {
pendingTtsResumeRunnable?.let { ttsResumeHandler.removeCallbacks(it) }
pendingTtsResumeRunnable = null
}
private fun performTtsRoundEnd(reason: String) {
_isTtsSpeaking.set(false)
_log("INFO", "[Agent][焦点] TTS round end($reason) —— 恢复音乐并尝试释放焦点")
try {
val intent = Intent("com.yunqiinnovation.music_service.AI_SAY_END").apply {
setPackage(context.packageName)
}
context.sendBroadcast(intent)
} catch (e: Exception) {
_log("WARNING", "[Agent] 发送 AI_SAY_END 广播异常: ${e.message}")
}
if (currentRecognitionMode == "ble_wakeup" && _isInterrupt.get()) {
try { BleService.openB1Encoder() } catch (e: Exception) {
_log("WARNING", "[Agent] openB1Encoder 异常: ${e.message}")
}
}
resumeMusic()
tryReleaseAiAudioFocus(reason)
}
private fun scheduleTtsRoundEnd(reason: String) {
cancelPendingTtsResume()
val runnable = Runnable {
pendingTtsResumeRunnable = null
performTtsRoundEnd(reason)
}
pendingTtsResumeRunnable = runnable
ttsResumeHandler.postDelayed(runnable, TTS_RESUME_DEBOUNCE_MS)
_log("INFO", "[Agent][焦点] 安排 TTS round-end 延迟 ${TTS_RESUME_DEBOUNCE_MS}ms reason=$reason")
}
// 状态 - 使用原子类型确保线程安全
private val _isInitialized = AtomicBoolean(false)
val isInitialized: Boolean get() = _isInitialized.get()
@ -611,6 +658,9 @@ object AgentService : CoroutineScope {
ttsService?.release()
ttsService = null
// 清掉 TTS round-end 的延迟任务,避免服务释放后还回调
cancelPendingTtsResume()
job.cancel()
clearListeners()
_isInitialized.set(false)
@ -682,15 +732,21 @@ object AgentService : CoroutineScope {
}
TtsEventType.SYNTHESIS_CANCELED -> {
_isTtsSpeaking.set(false)
// 明确结束信号:取消 pending debounce 并立即跑 round-end
cancelPendingTtsResume()
restartIdleCheck()
sendEvent("tts_canceled", mapOf("status" to "canceled"))
// TTS取消时停止气泡音
// audioPlayer?.stopAudio()
tryReleaseAiAudioFocus("ttsSynthesisCanceled")
performTtsRoundEnd("ttsSynthesisCanceled")
}
TtsEventType.PLAYBACK_STARTED -> { //开始AI播报内容
// 新段开始:若有待执行的 round-end(上一段 COMPLETED 安排的 1s 延迟),
// 直接取消——说明只是段间隙,不是真的结束
val hadPending = pendingTtsResumeRunnable != null
cancelPendingTtsResume()
if (hadPending) {
_log("INFO", "[Agent][焦点] 新段 PLAYBACK_STARTED 到来,取消 pending round-end(段间隙)")
}
restartIdleCheck()
sendEvent("playback_started", mapOf("status" to "playback_started"))
// 确保气泡音已停止,避免与TTS播放重叠
@ -706,40 +762,25 @@ object AgentService : CoroutineScope {
context.sendBroadcast(intent)
// Log.e(TAG, "QQ 音乐暂停播放")
autopauseMusic()
// QQMusicSingleton.getInstance().autopauseMusic { result ->
// if (result.isSuccess) {
// isplaymusic = true
// }
// }
}
TtsEventType.PLAYBACK_COMPLETED -> { //结束AI播报内容
// 防御性重置:有些 TTS 引擎会把 PLAYBACK_COMPLETED 发在 SYNTHESIS_COMPLETED 之前,
// 导致到这里时 _isTtsSpeaking 还是 true,进而被 tryReleaseAiAudioFocus 的 guard 拦下,
// 结果就是 AI 说完了但外部音乐/焦点一直没恢复
_isTtsSpeaking.set(false)
_log("INFO", "[Agent][焦点] TTS PLAYBACK_COMPLETED —— AI 对话结束,准备恢复音乐并释放焦点")
TtsEventType.PLAYBACK_COMPLETED -> { //当前段播报内容结束(不一定是整轮结束)
restartIdleCheck()
sendEvent("playback_completed", mapOf("status" to "playback_completed"))
// 发送AI说话开始广播
val intent =
Intent("com.yunqiinnovation.music_service.AI_SAY_END").apply {
setPackage(context.packageName)
}
context.sendBroadcast(intent)
if(currentRecognitionMode == "ble_wakeup"&&_isInterrupt.get()){
BleService.openB1Encoder()
}
resumeMusic()
tryReleaseAiAudioFocus("ttsPlaybackCompleted")
// 流式 TTS 按标点切段,此处只能代表"当前段"播完。
// 用 1s debounce:若期间没有新 PLAYBACK_STARTED,再真正 resume 音乐 + 释放焦点。
// 注意:这里不提前把 _isTtsSpeaking 置 false,
// 避免段间隙被 idle check / focus guard 误判为"空闲"。
scheduleTtsRoundEnd("ttsPlaybackCompleted")
}
TtsEventType.ERROR -> {
_isTtsSpeaking.set(false)
// 明确结束信号:取消 pending debounce 并立即跑 round-end
cancelPendingTtsResume()
restartIdleCheck()
// TTS错误时停止气泡音
audioPlayer?.stopAudio()
tryReleaseAiAudioFocus("ttsError")
performTtsRoundEnd("ttsError")
val params = event.params
val code = params["errorCode"] as? String ?: "UNKNOWN_ERROR"
val message = params["errorMessage"] as? String ?: "未知错误"
@ -1743,6 +1784,9 @@ object AgentService : CoroutineScope {
if (isTtsSpeaking) {
ttsService?.stop()
// 主动停止是明确的结束信号:清掉 debounce 并立即完成 round-end
// (若 ttsService.stop() 之后 SYNTHESIS_CANCELED 事件照常到达,round-end 逻辑是幂等的)
cancelPendingTtsResume()
_isTtsSpeaking.set(false)
restartIdleCheck() // 状态变化,重启检测
sendEvent("tts_stopped", mapOf("status" to "stopped"))

60
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift

@ -161,7 +161,28 @@ class AgentServiceImpl: NSObject {
azureAsrHelper = AzureAsrHelper()
azureTtsHelper = AzureTtsHelper()
// TTS 在段间隙做 deactivate 时,回查业务层状态:
// AI 流式/pendingProcess 工作中时不释放 session,避免外部 App 在 AI 多段合成之间被反复打断-恢复。
// 注意:ble_wakeup 模式下 ASR 是"持续监听"(不关代表对话结束),不应把它算作 busy——
// 否则 TTS 播完后外部 App 要等到 5s idle 超时 ASR 自行停掉才释放,中间过长的
// interruption 会让很多外部 App(Spotify / 网易云 / YouTube)放弃 auto-resume。
azureTtsHelper?.externalBusyProvider = { [weak self] in
guard let self = self else { return false }
let pendingWork = (self.pendingProcessWorkItem != nil) && !(self.pendingProcessWorkItem?.isCancelled ?? true)
let hasPendingText = !self.recognitionResult.isEmpty
let recogBlocking: Bool
if self.currentRecognitionMode == "ble_wakeup" {
recogBlocking = false
} else {
recogBlocking = self.isRecognizing || self.isStartingRecognition
}
return self.isAiStreaming || recogBlocking || pendingWork || hasPendingText
}
// 把 TTS 音频会话相关的关键日志转发到 app 侧日志通道(EventChannel → Flutter),可通过 app 导出排障
azureTtsHelper?.exportLog = { [weak self] level, message in
self?._log(level, message)
}
chatApiService = ChatApiService()
// chatStorageHelper 现在是单例,不需要初始化
@ -559,19 +580,42 @@ class AgentServiceImpl: NSObject {
}
}
// 仅当 ASR / AI / TTS 都空闲、且没有累计识别文本等待触发 AI 时才恢复外部音乐。
// 仅当 ASR / AI / TTS 都空闲、且没有累计识别文本等待触发 AI 时才恢复音乐。
// 否则会出现"超时中断唤醒→立刻恢复音乐→紧接着 onResult 触发 AI + TTS"同时响的问题。
//
// 分两路恢复:
// 1) 内部 MusicService(wasMusicPlayingBeforeRecognition)—— 直接 resume
// 2) 外部 App(wasExternalAudioPlayingBeforeRecognition)—— 通过 TTS helper 防抖 deactivate
// + .notifyOthersOnDeactivation 让系统通知外部 App。防抖是因为 Azure 流式 TTS 按标点切段,
// 每段播完都会进到这里,若每次立即 deactivate,段间隙外部 App 会抢回播放。
internal func tryResumeExternalMusicIfIdle(reason: String) {
guard wasMusicPlayingBeforeRecognition else { return }
if !wasMusicPlayingBeforeRecognition && !wasExternalAudioPlayingBeforeRecognition {
return
}
let pendingWork = (pendingProcessWorkItem != nil) && !(pendingProcessWorkItem?.isCancelled ?? true)
let hasPendingText = !recognitionResult.isEmpty
if isRecognizing || isStartingRecognition || isSpeaking || isAiStreaming || pendingWork || hasPendingText {
_log("INFO", "[Agent] defer music resume (\(reason)) recog=\(isRecognizing) starting=\(isStartingRecognition) tts=\(isSpeaking) ai=\(isAiStreaming) pending=\(pendingWork) text=\(hasPendingText)")
// ble_wakeup 模式下 ASR 是"持续监听等待唤醒",TTS 播完 + AI 完成即视为对话结束,
// 不应为了等 ASR 自然超时而延长外部 App 的 interruption。
let recogBlocking: Bool
if currentRecognitionMode == "ble_wakeup" {
recogBlocking = false
} else {
recogBlocking = isRecognizing || isStartingRecognition
}
if recogBlocking || isSpeaking || isAiStreaming || pendingWork || hasPendingText {
_log("INFO", "[Agent] defer music resume (\(reason)) mode=\(currentRecognitionMode) recog=\(isRecognizing) starting=\(isStartingRecognition) tts=\(isSpeaking) ai=\(isAiStreaming) pending=\(pendingWork) text=\(hasPendingText)")
return
}
wasMusicPlayingBeforeRecognition = false
_log("INFO", "[Agent] resume external music (\(reason))")
MusicService.shared.resume()
if wasMusicPlayingBeforeRecognition {
wasMusicPlayingBeforeRecognition = false
_log("INFO", "[Agent] resume internal music (\(reason))")
MusicService.shared.resume()
}
if wasExternalAudioPlayingBeforeRecognition {
wasExternalAudioPlayingBeforeRecognition = false
_log("INFO", "[Agent] schedule deactivate to notify external app (\(reason))")
azureTtsHelper?.scheduleDebouncedDeactivate()
}
}
func startRecognition(useBle: Bool = false, mode: String = "normal") -> Bool {
if agentMode == "sts" {

56
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AliyunBailianE2EHelper.kt

@ -93,6 +93,11 @@ class AliyunBailianE2EHelper(
private val isStarted = AtomicBoolean(false)
private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob())
// 诊断计数器
private var pushCount = 0L
private var pushDroppedCount = 0L
private var recvMsgCount = 0L
/**
* 初始化助手,设置配置与回调。
*
@ -101,20 +106,29 @@ class AliyunBailianE2EHelper(
* @return 是否初始化成功
*/
fun initialize(config: Config, cb: Callback): Boolean {
// 切换语言对(或重复初始化)必须彻底销毁旧 WebSocket,
// 否则 startContinuousConversation 的 isStarted 短路会导致新配置不生效。
Log.d(TAG, "initialize: resetting for new config src=${config.sourceLanguage}->${config.targetLanguage}, prevStarted=${isStarted.get()}")
try { webSocket?.close(1000, "reinit with new config") } catch (_: Exception) {}
webSocket = null
isStarted.set(false)
pushCount = 0
pushDroppedCount = 0
recvMsgCount = 0
audioChunkBuffer = ByteArray(0)
recvTextBuffer.setLength(0)
fullTextBuffer.setLength(0)
fullAudioBuffer.reset()
// 注意:通义千问 LiveTranslate 通常使用 2 位语言代码,但部分方言可能不同。
// 这里做一个简单的处理,具体需参考文档支持的语种列表。
conf = config.copy(sourceLanguage = config.sourceLanguage, targetLanguage = config.targetLanguage)
conf = config.copy(sourceLanguage = config.sourceLanguage, targetLanguage = config.targetLanguage)
callback = cb
Log.d(TAG, "initialize: wsUrl=${conf.wsUrl}, model=${conf.appId}, src=${conf.sourceLanguage}, tgt=${conf.targetLanguage}")
Log.d(TAG, "initialize: wsUrl=${conf.wsUrl}, model=${conf.appId}, src=${conf.sourceLanguage}, tgt=${conf.targetLanguage}")
client = OkHttpClient.Builder()
.pingInterval(30, TimeUnit.SECONDS)
.readTimeout(0, TimeUnit.SECONDS)
.build()
startContinuousConversation()
return true
}
@ -309,8 +323,11 @@ class AliyunBailianE2EHelper(
try {
val json = JSONObject(text)
val type = json.optString("type")
// Log.d(TAG, "Received event: $type")
recvMsgCount++
if (recvMsgCount <= 5 || recvMsgCount % 50 == 1L) {
Log.d(TAG, "Received event #$recvMsgCount: type=$type (src=${conf.sourceLanguage}->${conf.targetLanguage})")
}
when (type) {
"error" -> {
@ -443,17 +460,28 @@ class AliyunBailianE2EHelper(
* 音频数据需 Base64 编码。
*/
fun pushAudioData(data: ByteArray): Boolean {
val ws = webSocket ?: return false
if (!isStarted.get()) return false
val ws = webSocket
val started = isStarted.get()
if (ws == null || !started) {
pushDroppedCount++
if (pushDroppedCount % 100 == 1L) {
Log.w(TAG, "pushAudioData dropped: ws=${ws != null}, started=$started, dropped=$pushDroppedCount (src=${conf.sourceLanguage}->${conf.targetLanguage})")
}
return false
}
try {
// 将 PCM 数据转为 Base64
val base64Audio = data.toByteString().base64()
val json = JSONObject()
json.put("type", "input_audio_buffer.append")
json.put("audio", base64Audio)
ws.send(json.toString())
pushCount++
if (pushCount % 100 == 1L) {
Log.d(TAG, "pushAudioData ok: count=$pushCount, size=${data.size}, src=${conf.sourceLanguage}->${conf.targetLanguage}")
}
return true
} catch (e: Exception) {
Log.e(TAG, "pushAudioData error", e)

76
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt

@ -133,6 +133,14 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
private var astAudioPushCountA = 0L
private var astAudioPushCountB = 0L
// 静音门控:连续静音帧数超过阈值后暂停对应通道的推流,防止豆包等端到端 STS 服务端
// 因持续收到空音频触发 stream is done / session timeout 而断连。
// 单帧 20ms(16kHz stereo 1280B),50 帧 ≈ 1s;门控阈值 500 足以区分语音与底噪。
private var leftSilentStreak = 0
private var rightSilentStreak = 0
private val gatePeakThreshold = 500
private val silenceFramesToMute = 50
private fun pushAstAudioToA(data: ByteArray) {
astAudioPushCountA++
if (astAudioPushCountA % 100 == 1L) {
@ -658,13 +666,6 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
val removeFirstPunctuation =
call.argument<Boolean>("isRemoveFirstPunctuation") ?: true
// 通话翻译模式(external):AST端到端服务已处理识别+翻译+TTS,无需启动ASR
if (useExternalAudio) {
FileLogger.d(tag, "通话模式,AST端到端服务已激活(provider=$currentAstProvider),跳过ASR")
result.success(true)
return
}
// 确保事件通道已准备好
if (asrEventSink == null) {
result.error(
@ -677,7 +678,14 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
try {
FileLogger.d(tag, "选择音频源类型: ${useExternalAudio}")
val audioSourceType = AzureAsrHelper.AudioSourceType.MICROPHONE
// external:多媒体翻译(audioVideo)——BLE 对端音频经 channel=1 喂给 Azure ASR
// microphone:同声/面对面——本机麦克风
// 通话模式(call)不走这里:上层已跳过 ASR,由 AST 端到端服务处理
val audioSourceType = if (useExternalAudio) {
AzureAsrHelper.AudioSourceType.EXTERNAL
} else {
AzureAsrHelper.AudioSourceType.MICROPHONE
}
val success = azureAsrHelper.startContinuousRecognition(audioSourceType)
result.success(true)
} catch (e: Exception) {
@ -1798,7 +1806,9 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
val leftBuffer = ByteArray(sampleCount * 2) // 左声道缓冲区
val rightBuffer = ByteArray(sampleCount * 2) // 右声道缓冲区
// 拆分交错的左右声道数据
// 拆分交错的左右声道数据,同时计算 peak 用于静音门控
var leftPeak = 0
var rightPeak = 0
for (i in 0 until sampleCount) {
val stereoIndex = i * 4
val monoIndex = i * 2
@ -1810,9 +1820,42 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
// 右声道
rightBuffer[monoIndex] = data[stereoIndex + 2]
rightBuffer[monoIndex + 1] = data[stereoIndex + 3]
val lSample = ((data[stereoIndex + 1].toInt() shl 8) or (data[stereoIndex].toInt() and 0xFF)).toShort()
val rSample = ((data[stereoIndex + 3].toInt() shl 8) or (data[stereoIndex + 2].toInt() and 0xFF)).toShort()
val lAbs = Math.abs(lSample.toInt())
val rAbs = Math.abs(rSample.toInt())
if (lAbs > leftPeak) leftPeak = lAbs
if (rAbs > rightPeak) rightPeak = rAbs
}
// 计算左右声道 RMS 音量(每200帧打一次)
// 更新静音连续帧计数
val leftWasMuted = leftSilentStreak >= silenceFramesToMute
val rightWasMuted = rightSilentStreak >= silenceFramesToMute
if (leftPeak >= gatePeakThreshold) {
if (leftWasMuted) {
FileLogger.i(tag, "[STS] A(己方)恢复推流: peak=$leftPeak (之前静音${leftSilentStreak}帧)")
}
leftSilentStreak = 0
} else {
leftSilentStreak++
if (leftSilentStreak == silenceFramesToMute) {
FileLogger.i(tag, "[STS] A(己方)静音超过${silenceFramesToMute}帧,暂停推流")
}
}
if (rightPeak >= gatePeakThreshold) {
if (rightWasMuted) {
FileLogger.i(tag, "[STS] B(对方)恢复推流: peak=$rightPeak (之前静音${rightSilentStreak}帧)")
}
rightSilentStreak = 0
} else {
rightSilentStreak++
if (rightSilentStreak == silenceFramesToMute) {
FileLogger.i(tag, "[STS] B(对方)静音超过${silenceFramesToMute}帧,暂停推流")
}
}
// 计算左右声道 RMS 音量(每200帧打一次,含门控状态)
if (bleAudioRecvCount % 200 == 1L) {
var leftRms = 0.0
var rightRms = 0.0
@ -1827,15 +1870,22 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
val samples = leftBuffer.size / 2
leftRms = Math.sqrt(leftRms / samples)
rightRms = Math.sqrt(rightRms / samples)
FileLogger.d(tag, "[STS] 声道音量: left(己方/麦克风)=${leftRms.toInt()}, right(对方)=${rightRms.toInt()}, threshold=$lowVolumeThreshold")
val aState = if (leftSilentStreak >= silenceFramesToMute) "muted(streak=$leftSilentStreak)" else "active"
val bState = if (rightSilentStreak >= silenceFramesToMute) "muted(streak=$rightSilentStreak)" else "active"
FileLogger.d(tag, "[STS] 声道音量: left(己方/麦克风)=${leftRms.toInt()}[peak=$leftPeak,$aState], right(对方)=${rightRms.toInt()}[peak=$rightPeak,$bState], threshold=$lowVolumeThreshold, gate=$gatePeakThreshold")
}
// 过滤低音量音频
val filteredLeftBuffer = filterLowVolumeAudio(leftBuffer, lowVolumeThreshold)
val filteredRightBuffer = filterLowVolumeAudio(rightBuffer, lowVolumeThreshold)
// 左声道是自己的(麦克风),右声道是对方的
pushAstAudioToA(filteredLeftBuffer)
pushAstAudioToB(filteredRightBuffer)
// 静音门控:持续静音时不推流,避免服务端(尤其豆包)空流超时断连
if (leftSilentStreak < silenceFramesToMute) {
pushAstAudioToA(filteredLeftBuffer)
}
if (rightSilentStreak < silenceFramesToMute) {
pushAstAudioToB(filteredRightBuffer)
}
}

9
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt

@ -192,8 +192,13 @@ class DoubaoE2ETranslateHelper(
val msg = if (resp.hasResponseMeta()) resp.responseMeta.message else ""
Log.e(TAG, "onMessage: session error event=${event} msg=${msg}")
if (msg.contains("Timeout waiting next packet", ignoreCase = true)) {
Log.i(TAG, "onMessage: detected timeout error, triggering auto-restart")
// 可自动重连的服务端异常:通常是空闲/长会话导致的 stream 状态机结束,重建 WS 即可恢复
val isRetryable = msg.contains("Timeout waiting next packet", ignoreCase = true)
|| msg.contains("stream is done", ignoreCase = true)
|| msg.contains("session timeout", ignoreCase = true)
|| msg.contains("current state=Finished", ignoreCase = true)
if (isRetryable) {
Log.i(TAG, "onMessage: retryable server error, triggering auto-restart: '$msg'")
restartSession()
return
}

244
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift

@ -994,7 +994,8 @@ private func checkStreamPlaybackCompletedIfNeeded() {
hasNotifiedPlaybackStartedForStream = false
// os_log("调用链: 流式播放完成 session=%{public}@", log: log, type: .info, sessionid)
notifyEvent(eventType: .playbackCompleted)
deactivateAudioSessionAfterTTSIfIdle()
// 段完成:防抖后再释放 session,避免段间隙被外部 App 抢回
scheduleDebouncedDeactivate()
}
}
@ -1269,7 +1270,8 @@ public func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully fl
notifyEvent(eventType: .playbackCompleted)
// 如果还有剩余,继续播放
startPlaybackIfNeeded()
deactivateAudioSessionAfterTTSIfIdle()
// 段完成:防抖后再释放 session,避免段间隙被外部 App 抢回
scheduleDebouncedDeactivate()
}
/**
@ -1418,11 +1420,118 @@ private var shouldDeactivateAudioSessionWhenIdle = false
/// 外部调用方(如 AgentServiceImpl)在本轮唤醒/识别开始时设置:
/// - true 表示唤醒前有外部或内部音乐在放,TTS 空闲释放 session 时应带上 .notifyOthersOnDeactivation 让被打断的 App 恢复播放
/// - 成功 deactivate 后由本类自动清零
public var notifyOthersOnNextDeactivate: Bool = false
public var notifyOthersOnNextDeactivate: Bool = false {
didSet {
// 新一轮对话要求通知外部恢复时,重置重试预算,避免上一轮耗尽导致本轮永不重试
if notifyOthersOnNextDeactivate && !oldValue {
deactivateRetryCount = 0
}
}
}
// 新增:路由变化的去抖与配置重入保护标记
private var routeChangeDebounceWorkItem: DispatchWorkItem?
private var isApplyingAudioSessionConfig = false
/// deactivate 重试调度:playerNode 残留 isPlaying / setActive(false) 抛 IsBusy 时,
/// 用退避 timer 兜底再试,避免外部音乐因一次失败永远无法恢复。
private let deactivateRetryDelays: [TimeInterval] = [0.3, 0.8, 1.5]
private var deactivateRetryCount: Int = 0
private var pendingDeactivateRetryWorkItem: DispatchWorkItem?
/// 由业务层(AgentServiceImpl)注入:返回 true 表示 AI/ASR 仍在处理本轮对话,
/// 即使 TTS 段已播完也不应 deactivate——否则外部 App 会在 AI 流式段间被反复
/// "打断-恢复",多轮折腾后再也不自动恢复。
public var externalBusyProvider: (() -> Bool)?
/// 由业务层注入,用于把 TTS 音频会话相关的关键日志转发到 app 侧日志通道(可导出)。
/// 不注入时仅写 OSLog。
public var exportLog: ((String, String) -> Void)?
/// 我们自己追踪的 session 激活状态——setActive(true) 成功置 true,setActive(false) 成功置 false。
/// 用于 applyAudioSessionForTTS 的快速路径:已激活且 category/mode 匹配时不重复 setActive(true),
/// 避免 PCM chunk 级的重入疯狂触发系统调用(实测单轮 AI 回复可调 250+ 次)。
private var internalSessionActive: Bool = false
/// 统一打印:OSLog + app 导出通道双写。
/// tag 用固定前缀 `[TTS/Audio]` 方便在导出日志里过滤。
private func dlog(_ level: String = "INFO", _ message: String) {
let line = "[TTS/Audio] " + message
switch level {
case "ERROR":
os_log("%{public}@", log: self.log, type: .error, line)
case "WARNING":
os_log("%{public}@", log: self.log, type: .default, line)
case "DEBUG":
os_log("%{public}@", log: self.log, type: .debug, line)
default:
os_log("%{public}@", log: self.log, type: .info, line)
}
exportLog?(level, line)
}
/// Azure 流式 TTS 按标点切段,每段都会发 playbackCompleted。
/// 若每段都立即 deactivate(notifyOthers),段间隙外部 App 会短暂抢回播放
/// ——体感就是"AI 说到一半外部音乐冒出来"。
/// 用 1s debounce:playbackCompleted → 延迟释放;期间新段开始则取消 pending。
private let deactivateDebounceDelay: TimeInterval = 1.0
private var pendingDeactivateWorkItem: DispatchWorkItem?
/// 段间防抖 deactivate:延迟 1s 后调用 deactivateAudioSessionAfterTTSIfIdle。
/// 新段开始播(applyAudioSessionForTTS)会取消 pending。
/// 注意:业务层(AgentServiceImpl)可能在 AI 终态时调用本方法,而此时 TTS 从未
/// 实际 apply 过 session(例如 AI 无文本产出),shouldDeactivateAudioSessionWhenIdle 还是 false。
/// 为了让 ASR 激活但未释放的 session 也能被释放以通知外部 App,这里显式打开该标志。
public func scheduleDebouncedDeactivate() {
dlog("INFO", "防抖 deactivate 排队 delay=\(String(format: "%.2f", deactivateDebounceDelay))s notifyOthers=\(notifyOthersOnNextDeactivate) externalBusy=\(externalBusyProvider?() ?? false)")
shouldDeactivateAudioSessionWhenIdle = true
cancelPendingDebouncedDeactivate()
let work = DispatchWorkItem { [weak self] in
guard let self = self else { return }
self.pendingDeactivateWorkItem = nil
self.dlog("INFO", "防抖 deactivate 触发执行")
self.deactivateAudioSessionAfterTTSIfIdle()
}
pendingDeactivateWorkItem = work
DispatchQueue.main.asyncAfter(deadline: .now() + deactivateDebounceDelay, execute: work)
}
/// 取消等待中的防抖 deactivate。
public func cancelPendingDebouncedDeactivate() {
if pendingDeactivateWorkItem != nil {
dlog("INFO", "取消 pending 防抖 deactivate")
}
pendingDeactivateWorkItem?.cancel()
pendingDeactivateWorkItem = nil
}
/// 调度下一次 deactivate 重试;若已到上限或已在排队则不重复调度。
private func scheduleDeactivateRetry(reason: String) {
if pendingDeactivateRetryWorkItem != nil { return }
if deactivateRetryCount >= deactivateRetryDelays.count {
dlog("WARNING", "deactivate 重试已达上限(\(deactivateRetryDelays.count)) reason=\(reason)")
return
}
let delay = deactivateRetryDelays[deactivateRetryCount]
deactivateRetryCount += 1
dlog("INFO", "安排 deactivate 重试 #\(deactivateRetryCount) delay=\(String(format: "%.2f", delay))s reason=\(reason)")
let work = DispatchWorkItem { [weak self] in
guard let self = self else { return }
self.pendingDeactivateRetryWorkItem = nil
self.deactivateAudioSessionAfterTTSIfIdle()
}
pendingDeactivateRetryWorkItem = work
DispatchQueue.main.asyncAfter(deadline: .now() + delay, execute: work)
}
/// 取消等待中的 deactivate 重试(新 TTS 段开始时调用)。
private func cancelPendingDeactivateRetry() {
if pendingDeactivateRetryWorkItem != nil {
dlog("INFO", "取消 pending deactivate 重试")
}
pendingDeactivateRetryWorkItem?.cancel()
pendingDeactivateRetryWorkItem = nil
}
private func isTtsBusy() -> Bool {
if speaking { return true }
if isPlaying { return true }
@ -1461,7 +1570,22 @@ private func ensureAudioSessionQueueSpecificKeySet() {
*/
private func applyAudioSessionForTTS() -> Bool {
var success = true
// os_log("liwei-------------TTS applyAudioSessionForTTS currentRecognitionMode:%{public}@", log: self.log, type: .info, currentRecognitionMode)
// 新段即将开始播:取消段间隙 pending 的 deactivate + retry,避免中途释放 session
cancelPendingDebouncedDeactivate()
cancelPendingDeactivateRetry()
// 快速路径:session 已激活且 category/mode 匹配当前模式预期时直接返回,
// 避免 PCM chunk 级调用触发 250+ 次 setActive(true) 对外部 App 的反复干扰
let sessionSnapshot = AVAudioSession.sharedInstance()
let isBlePlaybackMode = (currentRecognitionMode == "push_to_talk"
|| currentRecognitionMode == "normal"
|| currentRecognitionMode == "ble_wakeup")
if internalSessionActive && !audioInterrupted && isBlePlaybackMode
&& sessionSnapshot.category == .playback
&& sessionSnapshot.mode == .spokenAudio {
return true
}
dlog("INFO", "applyAudioSessionForTTS 入口 mode=\(currentRecognitionMode) otherAudioPlaying=\(sessionSnapshot.isOtherAudioPlaying) category=\(sessionSnapshot.category.rawValue) sessionMode=\(sessionSnapshot.mode.rawValue) internalActive=\(internalSessionActive)")
ensureAudioSessionQueueSpecificKeySet()
let applyBody = {
if self.isApplyingAudioSessionConfig { return }
@ -1536,6 +1660,7 @@ private func applyAudioSessionForTTS() -> Bool {
_ = try? audioSession.setPreferredIOBufferDuration(0.02)
}
try audioSession.setActive(true)
self.internalSessionActive = true
if self.currentRecognitionMode == "phone_call", !hasHeadphones {
_ = try? audioSession.overrideOutputAudioPort(.speaker)
}
@ -1627,40 +1752,101 @@ private func isTelephonyActive() -> Bool {
*/
private func deactivateAudioSessionAfterTTSIfIdle() {
ensureAudioSessionQueueSpecificKeySet()
dlog("INFO", "deactivateAudioSessionAfterTTSIfIdle 入口")
let work = { [weak self] in
guard let self = self else { return }
if self.micCapture?.isCapturing == true { return }
if self.speaking { return }
if self.isPlaying { return }
if !self.playbackQueue.isEmpty { return }
if self.pendingTextCount > 0 { return }
if !self.pendingTasks.isEmpty { return }
if self.activeStreamSynthesisCount > 0 { return }
if !self.pcmPendingData.isEmpty { return }
if self.scheduledBufferCount > 0 { return }
// 注:这里刻意不再用 externalBusy skip。
// 之前试过"AI 还在 streaming 时段间 skip"——会导致整轮对话只在最终发一次
// notifyOthersOnDeactivation,有些外部 App(尤其第三方音乐)一次信号来不
// 及/会被吃掉,表现为"完全不恢复"。
// 放行后段间会反复 deactivate(notifyOthers),外部 App 能持续收到 ended,
// 虽然有段间短暂冒音的副作用,但能稳定恢复。
if self.externalBusyProvider?() == true {
self.dlog("INFO", "external busy (AI/ASR 仍忙)——仍允许 deactivate,外部 App 稳定恢复优先")
}
if self.micCapture?.isCapturing == true {
self.dlog("INFO", "deactivate skip: mic capturing,排重试")
self.scheduleDeactivateRetry(reason: "micCapturingOuter")
return
}
if self.speaking {
self.dlog("INFO", "deactivate skip: speaking=true")
return
}
if self.isPlaying {
self.dlog("INFO", "deactivate skip: isPlaying=true")
return
}
if !self.playbackQueue.isEmpty {
self.dlog("INFO", "deactivate skip: playbackQueue=\(self.playbackQueue.count)")
return
}
if self.pendingTextCount > 0 {
self.dlog("INFO", "deactivate skip: pendingTextCount=\(self.pendingTextCount)")
return
}
if !self.pendingTasks.isEmpty {
self.dlog("INFO", "deactivate skip: pendingTasks=\(self.pendingTasks.count)")
return
}
if self.activeStreamSynthesisCount > 0 {
self.dlog("INFO", "deactivate skip: activeStreamSynthesisCount=\(self.activeStreamSynthesisCount)")
return
}
if !self.pcmPendingData.isEmpty {
self.dlog("INFO", "deactivate skip: pcmPendingData=\(self.pcmPendingData.count)")
return
}
if self.scheduledBufferCount > 0 {
self.dlog("INFO", "deactivate skip: scheduledBufferCount=\(self.scheduledBufferCount)")
return
}
dlog("INFO", "deactivate 进入 playbackQueue:准备 stop engine/playerNode")
self.audioPlaybackQueue.async { [weak self] in
guard let self = self else { return }
if self.playerNode?.isPlaying == true { return }
let wasPlaying = self.playerNode?.isPlaying == true
// A:playerNode 若仍标记 isPlaying(buffer 播完后 flag 未翻转也会落在这),
// 不再直接 return 放任 session 挂住,而是主动 stop 再继续。
self.playerNode?.stop()
self.audioEngine?.stop()
self.audioEngine = nil
self.playerNode = nil
self.pcmFormat = nil
if self.shouldDeactivateAudioSessionWhenIdle {
// 麦克风仍在采集时,session 还被 ASR 持有,deactivate 必失败
// 保留标记等下次 idle 再试,避免标记被清空后永远不再触发
if self.micCapture?.isCapturing == true { return }
self.dlog("INFO", "deactivate 清理完成 wasPlayerPlaying=\(wasPlaying) shouldDeactivate=\(self.shouldDeactivateAudioSessionWhenIdle) notifyOthers=\(self.notifyOthersOnNextDeactivate)")
guard self.shouldDeactivateAudioSessionWhenIdle else {
self.dlog("INFO", "deactivate 跳过:shouldDeactivateAudioSessionWhenIdle=false")
return
}
// 麦克风仍在采集时,session 还被 ASR 持有,deactivate 必失败
// 保留标记等下次 idle 再试,避免标记被清空后永远不再触发
if self.micCapture?.isCapturing == true {
self.dlog("INFO", "deactivate 跳过:mic 仍在采集,排重试")
self.scheduleDeactivateRetry(reason: "micCapturing")
return
}
let opts: AVAudioSession.SetActiveOptions =
self.notifyOthersOnNextDeactivate ? .notifyOthersOnDeactivation : []
do {
try AVAudioSession.sharedInstance().setActive(false, options: opts)
self.shouldDeactivateAudioSessionWhenIdle = false
self.notifyOthersOnNextDeactivate = false
} catch {
// 失败保留标记,下次 idle 再尝试(常见原因:其他 AudioUnit 仍在运行)
os_log("TTS deactivate 失败,保留标记待下次重试: %{public}@", log: self.log, type: .info, error.localizedDescription)
}
let session = AVAudioSession.sharedInstance()
let otherAudioPlaying = session.isOtherAudioPlaying
// 总是带上 .notifyOthersOnDeactivation:
// - 这是系统 hint,只对本应用实际暂停过的外部 App 生效,未被暂停的 App 不受影响
// - 之前依赖 notifyOthersOnNextDeactivate 标志,但多轮对话里标志会被
// "setActive 成功后清零 + 下轮 startRecognition 读到 isOtherAudioPlaying=false 覆盖"
// 组合攻击擦掉,导致外部音乐恢复不了
let opts: AVAudioSession.SetActiveOptions = .notifyOthersOnDeactivation
self.dlog("INFO", "准备 setActive(false) flagHint=\(self.notifyOthersOnNextDeactivate) otherAudioPlaying=\(otherAudioPlaying) category=\(session.category.rawValue) mode=\(session.mode.rawValue)")
do {
try session.setActive(false, options: opts)
self.dlog("INFO", "✅ setActive(false) 成功(已带 notifyOthers)")
self.shouldDeactivateAudioSessionWhenIdle = false
self.notifyOthersOnNextDeactivate = false
self.internalSessionActive = false
self.deactivateRetryCount = 0
} catch {
let nsError = error as NSError
// B:失败保留标记,并排一次退避重试(0.3→0.8→1.5s)。
// 常见原因:其他 AudioUnit 仍在收尾(IsBusy)。
self.dlog("ERROR", "❌ setActive(false) 失败 domain=\(nsError.domain) code=\(nsError.code) msg=\(error.localizedDescription),安排退避重试")
self.scheduleDeactivateRetry(reason: "setActiveFailed")
}
}
}
@ -1685,6 +1871,8 @@ private func deactivateAudioSessionAfterTTSIfIdle() {
switch type {
case .began:
audioInterrupted = true
// 系统已挂起我们的 session,内部激活状态置 false,让 ended 后的重配能走完整 apply 流程
internalSessionActive = false
_ = stop()
os_log("音频中断开始(可能为通话/Siri),停止TTS", log: log, type: .info)
case .ended:

Loading…
Cancel
Save