From 6046df1933fa9f1771e5a8d57a6497ad29b680fc Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Tue, 21 Apr 2026 18:19:05 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=8A=E4=BC=A0iOS=20=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E8=B0=83=E6=95=B4=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .metadata | 2 +- lib/core/bindings/initial_binding.dart | 7 +- lib/data/services/asr_service.dart | 3 + .../microsoft_translation_service.dart | 170 ++++++++++++++++++ .../speech_impl/azure_asr_service.dart | 15 +- .../speech_impl/volcano_asr_api_service.dart | 1 + .../speech_impl/volcano_asr_service.dart | 1 + .../services/volcano_translation_service.dart | 13 +- .../meeting_details_controller.dart | 22 ++- .../controllers/translation_controller.dart | 33 +++- .../agent_service/AgentServiceImpl.swift | 16 +- .../azure_speech/AzureSpeechPlugin.swift | 18 +- .../Sources/azure_speech/AzureTtsHelper.swift | 44 +++-- .../Sources/tools/SimpleAudioReceiver.swift | 12 +- 14 files changed, 312 insertions(+), 45 deletions(-) create mode 100644 lib/data/services/microsoft_translation_service.dart diff --git a/.metadata b/.metadata index 7686b46e4..e7581b54e 100644 --- a/.metadata +++ b/.metadata @@ -15,7 +15,7 @@ migration: - platform: root create_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e base_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e - - platform: android + - platform: ios create_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e base_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e diff --git a/lib/core/bindings/initial_binding.dart b/lib/core/bindings/initial_binding.dart index 94c3ecef5..abc686570 100644 --- a/lib/core/bindings/initial_binding.dart +++ b/lib/core/bindings/initial_binding.dart @@ -14,6 +14,7 @@ import '../../data/services/meeting/meeting_upload_service.dart'; import '../../data/services/novice_guide_service.dart'; import '../../data/services/speech_factory.dart'; import '../../data/services/volcano_translation_service.dart'; +import '../../data/services/microsoft_translation_service.dart'; import '../../data/services/volcano_ai_service.dart'; import '../../data/services/language_manager.dart'; import '../../core/utils/logger.dart'; @@ -43,9 +44,13 @@ class InitialBinding extends Bindings { // 注册 AST 服务 Get.lazyPut(() => AzureAstService(), fenix: true); - // 火山翻译服务 + // 火山翻译服务（图片翻译 / 文本翻译模块仍在使用） Get.lazyPut(() => VolcanoTranslationService(), fenix: true); + // 微软翻译服务（同声翻译 / 面对面翻译 / 音视频翻译 使用） + Get.lazyPut( + () => MicrosoftTranslationService(), + fenix: true); // 蓝牙服务 Get.lazyPut(() => BluetoothManager(), fenix: true); diff --git a/lib/data/services/asr_service.dart b/lib/data/services/asr_service.dart index 294ac7ee3..634d6b602 100644 --- a/lib/data/services/asr_service.dart +++ b/lib/data/services/asr_service.dart @@ -27,10 +27,13 @@ abstract class AsrService { /// 开始连续语音识别 /// /// 返回一个包含识别事件的流 + /// [skipAsrForCallMode] true 时仅占位，不真正启动本地 ASR（用于通话翻译 call 模式： + /// AST 端到端服务已自行处理识别+翻译+TTS）。多媒体翻译等 external 场景保持 false。 Future startContinuousRecognition( bool audioSourceType, { bool isRemoveFirstPunctuation = true, String mode = "normal", + bool skipAsrForCallMode = false, }); /// 停止连续语音识别 diff --git a/lib/data/services/microsoft_translation_service.dart b/lib/data/services/microsoft_translation_service.dart new file mode 100644 index 000000000..2fe05d997 --- /dev/null +++ b/lib/data/services/microsoft_translation_service.dart @@ -0,0 +1,170 @@ +import 'dart:convert'; + +import 'package:get/get.dart'; +import 'package:http/http.dart' as http; + +import '../../core/utils/logger.dart'; +import '../../data/models/appconfig.dart'; +import 'language_configs/azure_language_config.dart'; +import 'language_manager.dart'; + +/// 微软 Azure Translator REST 翻译服务 +/// 端点：https://api.cognitive.microsofttranslator.com +/// 文档：https://learn.microsoft.com/azure/ai-services/translator/reference/v3-0-translate +class MicrosoftTranslationService extends GetxService { + static const String _defaultEndpoint = + 'https://api.cognitive.microsofttranslator.com'; + static const String _apiPath = '/translate'; + static const String _apiVersion = '3.0'; + + late final String _subscriptionKey; + late final String _region; + late final String _endpoint; + + bool _isInitialized = false; + bool get isInitialized => _isInitialized; + + final LanguageManager _languageManager = Get.find(); + + /// shortCode -> 微软 translationCode 缓存（由 getAzureTranslationLanguageSpecs 构建） + final Map _shortCodeToTranslationCode = {}; + + @override + Future onInit() async { + await initialize(); + super.onInit(); + } + + Future initialize() async { + if (_isInitialized) return true; + try { + _subscriptionKey = AppConfig.env('AZURE_TRANSLATION_KEY') ?? ''; + _region = AppConfig.env('AZURE_TRANSLATION_REGION') ?? ''; + _endpoint = + AppConfig.env('AZURE_TRANSLATION_ENDPOINT') ?? _defaultEndpoint; + + if (_subscriptionKey.isEmpty) { + print( + '🟢[MS-TRANS-DBG] AZURE_TRANSLATION_KEY 未配置，服务无法初始化'); + Logger.error('微软翻译服务初始化失败：AZURE_TRANSLATION_KEY 为空'); + return false; + } + + for (final spec in getAzureTranslationLanguageSpecs()) { + final short = spec['shortCode']; + final code = spec['translationCode']; + if (short != null && code != null) { + _shortCodeToTranslationCode[short] = code; + } + } + + _isInitialized = true; + print( + '🟢[MS-TRANS-DBG] 微软翻译服务初始化成功: endpoint=$_endpoint, region=$_region, keyLen=${_subscriptionKey.length}, specCount=${_shortCodeToTranslationCode.length}'); + Logger.info('微软翻译服务初始化成功 region=$_region'); + return true; + } catch (e) { + print('🟢[MS-TRANS-DBG] 微软翻译服务初始化异常: $e'); + Logger.error('微软翻译服务初始化异常: $e'); + return false; + } + } + + /// 把 ASR 代码（如 `zh-CN`）映射为微软 Translator 语言码（如 `zh-Hans`） + /// 找不到返回 null + String? _toTranslationCode(String asrCode) { + if (asrCode.isEmpty) return null; + final short = _languageManager.getShortCodeByAsrCode(asrCode); + if (short == null) return null; + return _shortCodeToTranslationCode[short]; + } + + /// 文本翻译 + /// [text] 待翻译文本 + /// [sourceLanguageCode] 源语言 ASR 码（传空串则让微软自动检测） + /// [targetLanguageCode] 目标语言 ASR 码 + Future translateText({ + required String text, + required String sourceLanguageCode, + required String targetLanguageCode, + }) async { + if (!_isInitialized) { + print('🟢[MS-TRANS-DBG] 未初始化，返回 null'); + Logger.error('微软翻译服务未初始化'); + return null; + } + if (text.isEmpty) return null; + + try { + final String? targetCode = _toTranslationCode(targetLanguageCode); + if (targetCode == null) { + print( + '🟢[MS-TRANS-DBG] 目标语言码映射失败: targetAsr=$targetLanguageCode，返回 null'); + Logger.error('微软翻译失败：目标语言 $targetLanguageCode 不在映射表'); + return null; + } + final String? sourceCode = _toTranslationCode(sourceLanguageCode); + + final queryParams = { + 'api-version': _apiVersion, + 'to': targetCode, + }; + if (sourceCode != null && sourceCode.isNotEmpty) { + queryParams['from'] = sourceCode; + } + + final uri = Uri.parse('$_endpoint$_apiPath') + .replace(queryParameters: queryParams); + final body = json.encode([ + {'text': text} + ]); + final headers = { + 'Content-Type': 'application/json', + 'Ocp-Apim-Subscription-Key': _subscriptionKey, + 'Ocp-Apim-Subscription-Region': _region, + 'X-ClientTraceId': _newTraceId(), + }; + + print( + '🟢[MS-TRANS-DBG] 发起翻译请求: uri=$uri, from=$sourceCode, to=$targetCode, text=$text'); + final response = + await http.post(uri, headers: headers, body: body).timeout( + const Duration(seconds: 10), + onTimeout: () => http.Response('{"error":"timeout"}', 408), + ); + print( + '🟢[MS-TRANS-DBG] 翻译响应: status=${response.statusCode}, bodyLen=${response.body.length}, body=${response.body.length > 500 ? response.body.substring(0, 500) : response.body}'); + + if (response.statusCode != 200) { + Logger.error( + '微软翻译 HTTP 错误: status=${response.statusCode}, body=${response.body}'); + return null; + } + + final decoded = json.decode(response.body); + if (decoded is! List || decoded.isEmpty) { + print('🟢[MS-TRANS-DBG] 响应非 List 或为空，解析失败'); + return null; + } + final first = decoded.first; + if (first is! Map) return null; + final translations = first['translations']; + if (translations is! List || translations.isEmpty) return null; + final translated = translations.first; + if (translated is! Map) return null; + final result = translated['text']; + if (result is! String) return null; + print('🟢[MS-TRANS-DBG] 翻译成功: $result'); + return result; + } catch (e, st) { + print('🟢[MS-TRANS-DBG] 翻译请求异常: $e\n$st'); + Logger.error('微软翻译请求异常: $e'); + return null; + } + } + + String _newTraceId() { + final now = DateTime.now().microsecondsSinceEpoch; + return 'ms-trans-$now'; + } +} diff --git a/lib/data/services/speech_impl/azure_asr_service.dart b/lib/data/services/speech_impl/azure_asr_service.dart index 9ea1a37f5..cbb5ebff9 100644 --- a/lib/data/services/speech_impl/azure_asr_service.dart +++ b/lib/data/services/speech_impl/azure_asr_service.dart @@ -202,6 +202,7 @@ class AzureAsrService extends GetxService implements AsrService { bool audioSourceType, { bool isRemoveFirstPunctuation = true, String mode = "normal", + bool skipAsrForCallMode = false, }) async { // 防抖判断：短时间内重复调用直接拦截 final now = DateTime.now(); @@ -225,6 +226,7 @@ class AzureAsrService extends GetxService implements AsrService { 'audioSourceType': audioSourceType, 'isRemoveFirstPunctuation': isRemoveFirstPunctuation, 'mode': mode, + 'skipAsrForCallMode': skipAsrForCallMode, }); if (!result) { @@ -327,11 +329,22 @@ class AzureAsrService extends GetxService implements AsrService { /// 处理来自原生端的识别事件 void _handleRecognitionEvent(dynamic event) { - if (event is! Map || _eventStreamController == null) return; + if (event is! Map) { + print('🔴[ASR-DBG] 非 Map 事件被丢弃: ${event.runtimeType}'); + return; + } + if (_eventStreamController == null) { + print('🔴[ASR-DBG] _eventStreamController 为 null，事件被丢弃: $event'); + return; + } final Map eventMap = event; final String eventType = eventMap['type'] as String? ?? ''; final String provider = eventMap['provider'] as String? ?? ''; + if (eventType != 'nativeLog' && eventType != 'audioData') { + print( + '🔴[ASR-DBG] 原生事件抵达 azure_asr_service: type=$eventType, provider=$provider, text=${eventMap['text']}'); + } if (provider.isNotEmpty) { if (provider != _latestAsrProvider) { _latestAsrProvider = provider; diff --git a/lib/data/services/speech_impl/volcano_asr_api_service.dart b/lib/data/services/speech_impl/volcano_asr_api_service.dart index b7801e7e7..821a78208 100644 --- a/lib/data/services/speech_impl/volcano_asr_api_service.dart +++ b/lib/data/services/speech_impl/volcano_asr_api_service.dart @@ -869,6 +869,7 @@ class VolcanoAsrApiService implements AsrService { bool audioSourceType, { bool isRemoveFirstPunctuation = true, String mode = "normal", + bool skipAsrForCallMode = false, }) { // TODO: implement startContinuousRecognition throw UnimplementedError(); diff --git a/lib/data/services/speech_impl/volcano_asr_service.dart b/lib/data/services/speech_impl/volcano_asr_service.dart index 504336017..9ba6e4c1a 100644 --- a/lib/data/services/speech_impl/volcano_asr_service.dart +++ b/lib/data/services/speech_impl/volcano_asr_service.dart @@ -404,6 +404,7 @@ class VolcanoAsrService extends GetxService implements AsrService { bool audioSourceType, { bool isRemoveFirstPunctuation = true, String mode = "normal", + bool skipAsrForCallMode = false, }) { // TODO: implement startContinuousRecognition throw UnimplementedError(); diff --git a/lib/data/services/volcano_translation_service.dart b/lib/data/services/volcano_translation_service.dart index 1ce33d71a..9032dc43a 100644 --- a/lib/data/services/volcano_translation_service.dart +++ b/lib/data/services/volcano_translation_service.dart @@ -177,6 +177,7 @@ class VolcanoTranslationService extends GetxService { required String targetLanguageCode, }) async { if (!_isInitialized) { + print('🟡[VOLC-DBG] 翻译服务未初始化，返回 null'); Logger.error('翻译服务未初始化'); return null; } @@ -200,6 +201,7 @@ class VolcanoTranslationService extends GetxService { if (targetShortCode != null) { targetCode = targetShortCode; } + print('🟡[VOLC-DBG] translateText: srcAsr=$sourceLanguageCode->srcShort=$sourceCode, tgtAsr=$targetLanguageCode->tgtShort=$targetCode, text=$text'); // 构建请求体 final requestBody = { @@ -223,11 +225,13 @@ class VolcanoTranslationService extends GetxService { final queryString = Uri(queryParameters: queryParams).query; final url = '$_baseUrl/?$queryString'; + print('🟡[VOLC-DBG] 发起翻译请求: url=$url, body=${json.encode(requestBody)}'); final response = await http.post( Uri.parse(url), headers: headers, body: json.encode(requestBody), ); + print('🟡[VOLC-DBG] 翻译响应: status=${response.statusCode}, bodyLen=${response.body.length}, body=${response.body.length > 500 ? response.body.substring(0, 500) : response.body}'); if (response.statusCode == 200) { final responseData = json.decode(response.body); @@ -279,13 +283,20 @@ class VolcanoTranslationService extends GetxService { } // 返回提取到的翻译结果 + if (translation == null) { + print('🟡[VOLC-DBG] HTTP 200 但未能解析出翻译字段，responseData=$responseData'); + } else { + print('🟡[VOLC-DBG] 翻译成功: $translation'); + } return translation; } + print('🟡[VOLC-DBG] HTTP 错误: status=${response.statusCode}, body=${response.body}'); Logger.error( '翻译HTTP错误: status=${response.statusCode}, body=${response.body}'); return null; - } catch (e) { + } catch (e, st) { + print('🟡[VOLC-DBG] 翻译请求异常: $e\n$st'); Logger.error('翻译请求异常: $e'); return null; } diff --git a/lib/modules/meeting/controllers/meeting_details_controller.dart b/lib/modules/meeting/controllers/meeting_details_controller.dart index d6903030b..f92577083 100644 --- a/lib/modules/meeting/controllers/meeting_details_controller.dart +++ b/lib/modules/meeting/controllers/meeting_details_controller.dart @@ -170,12 +170,22 @@ class MeetingDetailsController extends GetxController if (!await file.exists()) { Logger.error('音频文件不存在: $filepath'); isPlayerReady.value = false; - Get.snackbar( - 'error'.tr, - 'audioFileNotFound'.tr, // 音频文件不存在 - snackPosition: SnackPosition.BOTTOM, - duration: const Duration(seconds: 3), - ); + // 本地文件缺失：若云端存在音频，清空本地路径并持久化， + // 让 UI 自动切换为"下载"按钮，下载完成后即可播放； + // 仅当云端也没有音频时才提示文件不存在 + if (meetingData.value.audiourl.isNotEmpty) { + Logger.d('MeetingDetails', '本地音频缺失，切换为下载态（云端URL可用）'); + meetingData.value.filepath = ''; + meetingData.refresh(); + await SqfliteApi.editMeetingTitle(id, {'filePath': ''}); + } else { + Get.snackbar( + 'error'.tr, + 'audioFileNotFound'.tr, // 音频文件不存在 + snackPosition: SnackPosition.BOTTOM, + duration: const Duration(seconds: 3), + ); + } return; } diff --git a/lib/modules/translation/controllers/translation_controller.dart b/lib/modules/translation/controllers/translation_controller.dart index cc8f4656e..1fc06d0a9 100644 --- a/lib/modules/translation/controllers/translation_controller.dart +++ b/lib/modules/translation/controllers/translation_controller.dart @@ -46,6 +46,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver { // ==================== 服务依赖 ==================== final AsrService _asrService = Get.find(); + // 同声翻译 / 面对面翻译 / 音视频翻译走火山翻译（REST） + // 通话翻译（call 模式）不使用该字段，改走 _astService（AST 端到端） final VolcanoTranslationService _translationService = Get.find(); final TtsService _ttsService = Get.find(); @@ -1172,7 +1174,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver { /// 处理语音识别事件 /// [event] 识别事件 void _handleRecognitionEvent(RecognitionEvent event) { - // Logger.info("处理识别事件:${event}"); + print( + '🟣[TRANS-DBG] _handleRecognitionEvent: type=${event.type}, text=${event.text}, provider=${event.provider}, detectedLang=${event.detectedLanguage}, currentMode=${currentMode.value}'); final String asrProvider = event.provider; if (asrProvider.isNotEmpty) Logger.info('ASR provider: $asrProvider'); switch (event.type) { @@ -1531,12 +1534,17 @@ class TranslationController extends GetxController with WidgetsBindingObserver { // 防抖处理翻译请求 _translationDebounceTimer?.cancel(); _translationDebounceTimer = Timer(const Duration(milliseconds: 300), () { + final lastIsInter = translationHistory.isNotEmpty ? translationHistory.last.isIntermediate : null; + final lastSrc = translationHistory.isNotEmpty ? translationHistory.last.sourceText : ''; + print('🟣[TRANS-DBG] 中间结果防抖触发: histEmpty=${translationHistory.isEmpty}, lastIsInter=$lastIsInter, lastSrcLen=${lastSrc.length}'); if (translationHistory.isNotEmpty && translationHistory.last.isIntermediate && translationHistory.last.sourceText.isNotEmpty) { translateText(translationHistory.last.sourceText, translationHistory.last.timestamp, isFinal: false); + } else { + print('🟣[TRANS-DBG] 中间结果防抖未调用 translateText（条件不满足）'); } }); } @@ -1544,7 +1552,11 @@ class TranslationController extends GetxController with WidgetsBindingObserver { /// 处理最终识别结果 /// [text] 识别的文本 Future handleFinalResult(String text) async { - if (text.isEmpty) return; + print('🟣[TRANS-DBG] handleFinalResult 入口: text=$text, mode=${currentMode.value}, historyLen=${translationHistory.length}, lastIntermediate=${translationHistory.isNotEmpty ? translationHistory.last.isIntermediate : "N/A"}'); + if (text.isEmpty) { + print('🟣[TRANS-DBG] handleFinalResult 因 text 为空直接返回'); + return; + } // 更新当前项 if (translationHistory.isNotEmpty && @@ -1594,22 +1606,30 @@ class TranslationController extends GetxController with WidgetsBindingObserver { /// 返回翻译结果 Future translateText(String sourceText, DateTime timestamp, {bool isFinal = false}) async { - if (sourceText.isEmpty) return null; + print('🟣[TRANS-DBG] translateText 入口: text=$sourceText, isFinal=$isFinal, mode=${currentMode.value}, src=${sourceLanguageCode.value}, tgt=${targetLanguageCode.value}, detected=$detectedLanguageCode'); + if (sourceText.isEmpty) { + print('🟣[TRANS-DBG] translateText 因 sourceText 为空直接返回'); + return null; + } isTranslating.value = true; String? translationResult; try { final languageCodes = _determineLanguageCodes(); + print('🟣[TRANS-DBG] 准备调用翻译 API: src=${languageCodes['source']} tgt=${languageCodes['target']}'); translationResult = await _performTranslation( sourceText, languageCodes['source']!, languageCodes['target']!); + print('🟣[TRANS-DBG] 翻译 API 返回: result=$translationResult'); if (translationResult != null) { await _updateTranslationHistory( sourceText, translationResult, timestamp, languageCodes, isFinal); + } else { + print('🟣[TRANS-DBG] 翻译 API 返回 null，不更新历史也不播报'); } - } catch (e) { - Logger.error('翻译失败: ${e.toString()}'); + } catch (e, st) { + print('🟣[TRANS-DBG] 翻译失败: ${e.toString()}\n$st'); } finally { isTranslating.value = false; } @@ -1785,8 +1805,11 @@ class TranslationController extends GetxController with WidgetsBindingObserver { /// 处理TTS播放 /// [translationResult] 翻译结果 Future _handleTtsPlayback(String translationResult) async { + print('🟣[TRANS-DBG] _handleTtsPlayback 入口: isTtsEnabled=${isTtsEnabled.value}, text=$translationResult'); if (isTtsEnabled.value) { await playTranslatedText(translationResult); + } else { + print('🟣[TRANS-DBG] TTS 已禁用，跳过播报'); } } diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 11de230ff..0db3caec3 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -2623,18 +2623,20 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { sendEvent(name: "recognition_stopped", data: ["sessionid":currsessionAsrId,"status": "stopped"]) }else{ sendEvent(name: "recognition_stopped", data: ["sessionid":sessionid,"status": "stopped"]) - isRecognizing = false - hasSpeechDetected = false - stopIdleCheck() if (currentRecognitionMode == "ble_wakeup") { audioPlayer?.playStopSound() } - - tryResumeExternalMusicIfIdle(reason: "onSessionStopped") } - - + // 统一复位 ASR 状态 + 触发外部音乐恢复判断： + // 之前 PTT 分支漏掉了这几步，isRecognizing 卡在 true， + // 后续 AI/TTS 完成回调里 tryResumeExternalMusicIfIdle 的 recogBlocking 守卫永远为 true， + // 导致 scheduleDebouncedDeactivate 永远进不去、外部音乐收不到 notifyOthersOnDeactivation。 + isRecognizing = false + hasSpeechDetected = false + stopIdleCheck() + _log("INFO", "[Agent] onSessionStopped mode=\(currentRecognitionMode) tryResume") + tryResumeExternalMusicIfIdle(reason: "onSessionStopped") } func onCanceled(sessionid:String ,_ reason: String, _ errorDetails: String) { diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift index 3d53d594e..08fba0f71 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift @@ -415,17 +415,21 @@ private func sendAudioDataEvent(_ event: [String: Any]) { let removeFirstPunctuation = args["isRemoveFirstPunctuation"] as? Bool ?? true let mode = args["mode"] as? String ?? "normal" - + // 通话翻译 call 模式由 Dart 侧显式传 true：AST 端到端服务已跑，占位跳过 ASR。 + // 缺省 false，保证同声/面对面/多媒体翻译（external 音频）都能正常启动 ASR。 + let skipAsrForCallMode = args["skipAsrForCallMode"] as? Bool ?? false + // 根据参数确定音频源类型 - let audioSourceType = isExternalActive ? - AzureAsrHelper.AudioSourceType.external : + let audioSourceType = isExternalActive ? + AzureAsrHelper.AudioSourceType.external : AzureAsrHelper.AudioSourceType.microphone - os_log("startContinuousRecognition: mode=%@, audioSourceType=%@, isExternal=%d, astProvider=%@", - log: ctLog, type: .info, mode, String(describing: audioSourceType), isExternalActive ? 1 : 0, currentAstProvider) + os_log("startContinuousRecognition: mode=%@, audioSourceType=%@, isExternal=%d, skipAsr=%d, astProvider=%@", + log: ctLog, type: .info, mode, String(describing: audioSourceType), isExternalActive ? 1 : 0, skipAsrForCallMode ? 1 : 0, currentAstProvider) - // 通话翻译模式（external）：AST 端到端服务已处理识别+翻译+TTS，无需启动 ASR - if isExternalActive { + // 仅当 Dart 显式标记为 call 模式下的占位调用时才跳过 ASR； + // 之前仅凭 isExternalActive 判断会误伤多媒体翻译（external 音频但需要启动 ASR）。 + if isExternalActive && skipAsrForCallMode { os_log("startContinuousRecognition: 通话模式，AST端到端服务已激活(provider=%@)，跳过ASR", log: ctLog, type: .info, currentAstProvider) result(true) } else { diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift index 20e2d3b29..caf0bcd23 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift @@ -1581,8 +1581,8 @@ private func applyAudioSessionForTTS() -> Bool { || currentRecognitionMode == "normal" || currentRecognitionMode == "ble_wakeup") if internalSessionActive && !audioInterrupted && isBlePlaybackMode - && sessionSnapshot.category == .playback - && sessionSnapshot.mode == .spokenAudio { + && sessionSnapshot.category == .playAndRecord + && sessionSnapshot.mode == .default { return true } dlog("INFO", "applyAudioSessionForTTS 入口 mode=\(currentRecognitionMode) otherAudioPlaying=\(sessionSnapshot.isOtherAudioPlaying) category=\(sessionSnapshot.category.rawValue) sessionMode=\(sessionSnapshot.mode.rawValue) internalActive=\(internalSessionActive)") @@ -1628,9 +1628,13 @@ private func applyAudioSessionForTTS() -> Bool { self.micCapture?.stopCapture() Thread.sleep(forTimeInterval: 0.05) } - desiredCategory = .playback - desiredMode = .spokenAudio - // 不再 mixWithOthers：唤醒/TTS 期间需要打断外部和内部其他音频 + // 和 ASR 阶段保持同 category（.playAndRecord）： + // 换成 .playback + .spokenAudio 会让 iOS 给外部 App 发一次 "interruption ended (no shouldResume)"， + // 最终 setActive(false, notifyOthers) 也没办法让它恢复。 + // 整轮对话只用一种 category，iOS 视角就是单一 interruption，释放时才能正确触发 auto-resume。 + desiredCategory = .playAndRecord + desiredMode = .default + desiredOptions = [.defaultToSpeaker, .allowBluetoothA2DP] } else if isRecording { desiredCategory = .playAndRecord desiredMode = .default @@ -1755,14 +1759,15 @@ private func deactivateAudioSessionAfterTTSIfIdle() { dlog("INFO", "deactivateAudioSessionAfterTTSIfIdle 入口") let work = { [weak self] in guard let self = self else { return } - // 注：这里刻意不再用 externalBusy skip。 - // 之前试过"AI 还在 streaming 时段间 skip"——会导致整轮对话只在最终发一次 - // notifyOthersOnDeactivation，有些外部 App（尤其第三方音乐）一次信号来不 - // 及/会被吃掉，表现为"完全不恢复"。 - // 放行后段间会反复 deactivate(notifyOthers)，外部 App 能持续收到 ended， - // 虽然有段间短暂冒音的副作用，但能稳定恢复。 + // AI/ASR 仍忙时 skip deactivate： + // 实测 PTT 模式多轮工具调用（tool_call → TTS → tool_call → TTS）里，如果段间放行 + // deactivate，外部 App（Apple Music/Spotify 等）会在 1s 内经历 resume→interrupt→resume + // 的快速抖动，最后一次 resume 会被部分 App 判为异常并忽略，表现为"彻底不恢复"。 + // 直接 skip 后等 aiOnComplete/onSessionStopped 的 tryResumeExternalMusicIfIdle + // 重新 scheduleDebouncedDeactivate，保证整轮对话只经历一次 pause→resume。 if self.externalBusyProvider?() == true { - self.dlog("INFO", "external busy (AI/ASR 仍忙)——仍允许 deactivate，外部 App 稳定恢复优先") + self.dlog("INFO", "deactivate skip: external busy (AI/ASR 仍忙)，等回合结束再释放") + return } if self.micCapture?.isCapturing == true { self.dlog("INFO", "deactivate skip: mic capturing，排重试") @@ -1833,6 +1838,21 @@ private func deactivateAudioSessionAfterTTSIfIdle() { // "setActive 成功后清零 + 下轮 startRecognition 读到 isOtherAudioPlaying=false 覆盖" // 组合攻击擦掉，导致外部音乐恢复不了 let opts: AVAudioSession.SetActiveOptions = .notifyOthersOnDeactivation + // 释放前先把 category 切到 .ambient：让 iOS 明确收到"我们彻底不占 session 了"的信号， + // shouldResume 提示才能可靠送达 Apple Music / Spotify 等外部 App。 + // 多个外部 App 对"仍处于 .playAndRecord 的 deactivate"会默认不恢复（视为潜在录音/通话）， + // 切到 .ambient 后再 deactivate 是 Siri 以外多数语音助手实测通用的套路。 + let prevCategory = session.category + let prevMode = session.mode + let prevOptions = session.categoryOptions + if prevCategory != .ambient { + do { + try session.setCategory(.ambient, mode: .default, options: []) + self.dlog("INFO", "deactivate 前先切到 .ambient（原 category=\(prevCategory.rawValue) mode=\(prevMode.rawValue)）") + } catch { + self.dlog("WARNING", "切到 .ambient 失败: \(error.localizedDescription)，继续原 category 释放") + } + } self.dlog("INFO", "准备 setActive(false) flagHint=\(self.notifyOthersOnNextDeactivate) otherAudioPlaying=\(otherAudioPlaying) category=\(session.category.rawValue) mode=\(session.mode.rawValue)") do { try session.setActive(false, options: opts) diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift index fa46f1f52..c620aacfe 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift @@ -177,14 +177,18 @@ public class SimpleAudioReceiver: NSObject { desiredOptions = [.allowBluetooth, .defaultToSpeaker] } } else { // normal, push_to_talk, ble_wakeup 等：仅录音，不需要同时播报 - desiredCategory = .record + // 不使用 .record：该 category 的 interruption 被外部音乐 App（Apple Music/Spotify 等） + // 归为"Siri/录音类打断"，.notifyOthersOnDeactivation 拿不到 auto-resume。 + // 改用 .playAndRecord，和 phone_call 对齐，iOS 会把这类打断归为"媒体打断"， + // 外部 App 在 TTS 结束 session 释放时才能正常恢复。 + desiredCategory = .playAndRecord desiredMode = .default - desiredOptions = [] + desiredOptions = [.defaultToSpeaker, .allowBluetoothA2DP] } - + // 如果在真实通话中，强制使用更适合的录音设置 if isTelephonyActive() { - desiredCategory = .record + desiredCategory = .playAndRecord desiredMode = .voiceChat desiredOptions = [.allowBluetooth] }