From 27b508bd1f313d59bc2235d2c0273fb62b5a04d5 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Tue, 7 Apr 2026 17:48:58 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E6=9C=8D=E5=8A=A1=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/data/services/asr_service.dart | 15 +- lib/data/services/language_manager.dart | 2 +- .../speech_impl/azure_asr_service.dart | 15 +- .../speech_impl/azure_ast_service.dart | 68 +++-- .../speech_impl/volcano_asr_api_service.dart | 7 +- .../speech_impl/volcano_asr_service.dart | 7 +- .../meeting_record_controller.dart | 25 +- .../controllers/translation_controller.dart | 240 +++++++++++++----- .../models/translation_models.dart | 8 + .../azure_speech/AzureSpeechPlugin.kt | 97 ++++--- .../azure_speech/DoubaoE2ETranslateHelper.kt | 27 +- .../yunqiinnovation/ble_service/BleService.kt | 10 +- 12 files changed, 359 insertions(+), 162 deletions(-) diff --git a/lib/data/services/asr_service.dart b/lib/data/services/asr_service.dart index 2b3610f68..294ac7ee3 100644 --- a/lib/data/services/asr_service.dart +++ b/lib/data/services/asr_service.dart @@ -1,6 +1,14 @@ import 'dart:async'; import 'dart:typed_data'; +/// 音频源类型 +enum AudioSourceType { + microphone, // 使用设备麦克风 + systemAudio, // 使用系统音频（麦克风也启动保证后台也能正常运行） + systemAudioPlusMicrophone, // 使用系统音频+麦克风音频 + external // 使用外部提供的音频数据 +} + /// 语音识别服务接口 abstract class AsrService { /// 支持的语言 @@ -47,8 +55,11 @@ abstract class AsrService { /// [filePath] 录音文件路径 /// [audioSourceType] 音频源类型 /// [acceptAudioData] 是否接受音频数据回调，默认为 false - Future enableRecord(bool audioSourceType, String filePath, - {bool acceptAudioData = false}); + Future enableRecord( + AudioSourceType audioSourceType, + String filePath, + bool acceptAudioData, + ); /// 获取音频数据流（如果支持） Stream? getAudioDataStream() => null; diff --git a/lib/data/services/language_manager.dart b/lib/data/services/language_manager.dart index b41b17166..c0d4c97cc 100644 --- a/lib/data/services/language_manager.dart +++ b/lib/data/services/language_manager.dart @@ -319,7 +319,7 @@ class LanguageManager extends GetxService { chineseName: '土耳其语', englishName: 'Turkish', asrCode: 'tr-TR', - ttsVoiceName: 'tr-TR-EmelNeural', + ttsVoiceName: 'tr-TR-SedaNeural', ); _addLanguage( diff --git a/lib/data/services/speech_impl/azure_asr_service.dart b/lib/data/services/speech_impl/azure_asr_service.dart index f75fe01e5..9ea1a37f5 100644 --- a/lib/data/services/speech_impl/azure_asr_service.dart +++ b/lib/data/services/speech_impl/azure_asr_service.dart @@ -7,12 +7,6 @@ import '../../../core/utils/logger.dart'; import 'package:get/get.dart'; import '../asr_service.dart'; -/// 音频源类型 -enum AudioSourceType { - microphone, // 使用设备麦克风 - external // 使用外部提供的音频数据 -} - /// Azure 语音识别服务 /// /// 该服务提供了通过平台通道与原生 Microsoft Speech SDK 交互的接口 @@ -521,11 +515,14 @@ class AzureAsrService extends GetxService implements AsrService { } @override - Future enableRecord(bool audioSourceType, String filePath, - {bool acceptAudioData = false}) async { + Future enableRecord( + AudioSourceType audioSourceType, + String filePath, + bool acceptAudioData, + ) async { try { final bool result = await _channel.invokeMethod('enableRecord', { - 'audioSourceType': audioSourceType, + 'audioSourceType': audioSourceType.index, 'filePath': filePath, 'acceptAudioData': acceptAudioData, // 新增参数 }); diff --git a/lib/data/services/speech_impl/azure_ast_service.dart b/lib/data/services/speech_impl/azure_ast_service.dart index 1fa59abe2..1d2124a08 100644 --- a/lib/data/services/speech_impl/azure_ast_service.dart +++ b/lib/data/services/speech_impl/azure_ast_service.dart @@ -67,9 +67,16 @@ class AzureAstService extends GetxService implements AstService { super.onClose(); } - /// 设置事件通道 + /// 设置事件通道（只在首次调用时订阅 EventChannel，后续复用） void _setupEventChannel() { - _eventSubscription?.cancel(); + // 确保 _eventStreamController 已就绪，避免丢失早期事件 + _eventStreamController ??= StreamController.broadcast(); + // EventChannel.receiveBroadcastStream() 只能活跃一次，复用已有订阅 + if (_eventSubscription != null) { + Logger.info('[STS] _setupEventChannel: 复用已有EventSubscription, streamController=${_eventStreamController.hashCode}'); + return; + } + Logger.info('[STS] _setupEventChannel: 首次订阅EventChannel, streamController=${_eventStreamController.hashCode}'); _eventSubscription = _eventChannel.receiveBroadcastStream().listen((event) { if (event is Map) { _handleRecognitionEvent(event); @@ -78,20 +85,27 @@ class AzureAstService extends GetxService implements AstService { } /// 处理来自原生端的识别事件（AST事件） - /// 将原生侧 AST 事件映射为统一的 ASTEvent，提供给业务层使用。 void _handleRecognitionEvent(dynamic event) { - if (event is! Map || _eventStreamController == null) return; + if (event is! Map) return; + if (_eventStreamController == null) { + Logger.error('[STS] _handleRecognitionEvent: _eventStreamController 为 null，事件被丢弃: ${event['type']}'); + return; + } + if (_eventStreamController!.isClosed) { + Logger.error('[STS] _handleRecognitionEvent: _eventStreamController 已关闭，事件被丢弃: ${event['type']}'); + return; + } final Map eventMap = event; final String eventType = eventMap['type'] as String? ?? ''; switch (eventType) { case 'recognized': - print("Ast处理识别完成事件:${eventMap}"); final String serviceId = eventMap['serviceId'] as String? ?? ''; final String utteranceId = eventMap['utteranceId'] as String? ?? ''; final String text = eventMap['text'] as String? ?? ''; final String detectedLanguage = eventMap['language'] as String? ?? ''; + Logger.info('[STS] 识别完成 [$serviceId]: text=$text, uttId=$utteranceId'); _latestRecognizedText = text; _latestDetectedLanguage = detectedLanguage; _eventStreamController?.add(ASTEvent( @@ -104,11 +118,11 @@ class AzureAstService extends GetxService implements AstService { break; case 'recognizing': - print("Ast处理识别中事件:${eventMap}"); final String serviceId = eventMap['serviceId'] as String? ?? ''; final String utteranceId = eventMap['utteranceId'] as String? ?? ''; final String text = eventMap['text'] as String? ?? ''; final String detectedLanguage = eventMap['language'] as String? ?? ''; + Logger.info('[STS] 识别中 [$serviceId]: text=$text'); _eventStreamController?.add(ASTEvent( type: ASTEventType.intermediateResult, serviceId: serviceId, @@ -118,12 +132,12 @@ class AzureAstService extends GetxService implements AstService { )); break; case 'translatedInterim': - print("Ast处理翻译中事件:${eventMap}"); final String serviceId = eventMap['serviceId'] as String? ?? ''; final String utteranceId = eventMap['utteranceId'] as String? ?? ''; final String text = eventMap['translatedText'] as String? ?? ''; final String detectedLanguage = eventMap['targetLanguage'] as String? ?? ''; + Logger.info('[STS] 翻译中 [$serviceId]: text=$text'); _eventStreamController?.add(ASTEvent( type: ASTEventType.translationInterim, serviceId: serviceId, @@ -133,12 +147,12 @@ class AzureAstService extends GetxService implements AstService { )); break; case 'translated': - print("Ast处理翻译结果事件:${eventMap}"); final String serviceId = eventMap['serviceId'] as String? ?? ''; final String utteranceId = eventMap['utteranceId'] as String? ?? ''; final String text = eventMap['translatedText'] as String? ?? ''; final String detectedLanguage = eventMap['targetLanguage'] as String? ?? ''; + Logger.info('[STS] 翻译完成 [$serviceId]: text=$text, uttId=$utteranceId'); _eventStreamController?.add(ASTEvent( type: ASTEventType.translationResult, serviceId: serviceId, @@ -149,12 +163,14 @@ class AzureAstService extends GetxService implements AstService { break; case 'sessionStarted': + Logger.info('[STS] 会话开始'); _eventStreamController?.add(ASTEvent( type: ASTEventType.sessionStarted, )); break; case 'sessionStopped': + Logger.info('[STS] 会话停止'); _isContinuousRecognitionActive = false; _eventStreamController?.add(ASTEvent( type: ASTEventType.sessionStopped, @@ -165,11 +181,7 @@ class AzureAstService extends GetxService implements AstService { _isContinuousRecognitionActive = false; final String reason = eventMap['reason'] as String? ?? ''; final String errorDetails = eventMap['errorDetails'] as String? ?? ''; - - if (reason.isNotEmpty || errorDetails.isNotEmpty) { - Logger.error('识别取消: $reason - ${errorDetails.toString()}'); - } - + Logger.error('[STS] 识别取消: $reason - $errorDetails'); _eventStreamController?.add(ASTEvent( type: ASTEventType.canceled, error: '$reason: $errorDetails', @@ -185,12 +197,20 @@ class AzureAstService extends GetxService implements AstService { if (code.isNotEmpty) { error = '$error (Code: $code)'; } - Logger.error('识别错误: ${error.toString()}'); + Logger.error('[STS] 识别错误: $error'); _eventStreamController?.add(ASTEvent( type: ASTEventType.error, error: error, )); break; + + case 'serviceInitialized': + Logger.info('[STS] 服务初始化: ${eventMap['serviceId']} ${eventMap['direction']}'); + break; + + default: + Logger.info('[STS] 未处理事件: $eventType'); + break; } } @@ -241,20 +261,23 @@ class AzureAstService extends GetxService implements AstService { @override Future> recognizeCallback() async { if (!_isInitialized) { + Logger.info('[STS] recognizeCallback: 未初始化，先执行initialize'); await initialize(); } try { - _eventStreamController = StreamController.broadcast(); + _eventStreamController ??= StreamController.broadcast(); + Logger.info('[STS] recognizeCallback: streamController=${_eventStreamController.hashCode}, hasListener=${_eventStreamController!.hasListener}'); // 开始连续识别 final bool result = await _channel.invokeMethod('recognizeCallback'); + Logger.info('[STS] recognizeCallback: invokeMethod 结果=$result'); if (!result) { _cleanupEventStream(); } return _eventStreamController!.stream; } catch (e) { - Logger.error('开始连续语音识别失败: ${e.toString()}'); + Logger.error('[STS] recognizeCallback 失败: ${e.toString()}'); rethrow; } } @@ -316,14 +339,15 @@ class AzureAstService extends GetxService implements AstService { @override Future dispose() async { try { - await _eventSubscription?.cancel(); - _eventSubscription = null; + Logger.info('[STS] dispose: 开始释放, streamController=${_eventStreamController?.hashCode}, eventSub=${_eventSubscription?.hashCode}'); + // 不取消 _eventSubscription（EventChannel 只能订阅一次，保持复用） + // 只关闭 _eventStreamController，下次 initialize 时会重新创建 await _eventStreamController?.close(); _eventStreamController = null; await _channel.invokeMethod('dispose'); _isInitialized = false; - Logger.info('Azure AST 资源已释放'); + Logger.info('[STS] dispose: 完成, streamController=null, eventSub保留'); } catch (e) { Logger.error('Azure AST 资源释放失败: ${e.toString()}'); _isInitialized = false; @@ -338,11 +362,16 @@ class AzureAstService extends GetxService implements AstService { try { final List languages = supportedLanguages ?? _defaultSupportedLanguages; + Logger.info('[STS] initialize: provider=$provider, isInitialized=$_isInitialized'); // 底层会初始化前释放 if (_isInitialized) { await dispose(); } + // 先建立事件通道，确保原生端发事件时 astEventSink 已就绪 + _setupEventChannel(); + Logger.info('[STS] initialize: eventChannel已建立, streamController=${_eventStreamController?.hashCode}'); + final bool result = await _channel.invokeMethod('initialize', { 'provider': provider, 'subscriptionKey': _subscriptionKey, @@ -371,7 +400,6 @@ class AzureAstService extends GetxService implements AstService { }); _isInitialized = result; - _setupEventChannel(); print('Azure 语音识别服务初始化${result ? '成功' : '失败'}'); Logger.info('Azure 语音识别服务初始化${result ? '成功' : '失败'}'); return result; diff --git a/lib/data/services/speech_impl/volcano_asr_api_service.dart b/lib/data/services/speech_impl/volcano_asr_api_service.dart index d105d97ec..b7801e7e7 100644 --- a/lib/data/services/speech_impl/volcano_asr_api_service.dart +++ b/lib/data/services/speech_impl/volcano_asr_api_service.dart @@ -813,8 +813,11 @@ class VolcanoAsrApiService implements AsrService { } @override - Future enableRecord(bool audioSourceType, String filePath, - {bool acceptAudioData = false}) { + Future enableRecord( + AudioSourceType audioSourceType, + String filePath, + bool acceptAudioData, + ) { // TODO: implement enableRecord throw UnimplementedError(); } diff --git a/lib/data/services/speech_impl/volcano_asr_service.dart b/lib/data/services/speech_impl/volcano_asr_service.dart index d1f59648e..504336017 100644 --- a/lib/data/services/speech_impl/volcano_asr_service.dart +++ b/lib/data/services/speech_impl/volcano_asr_service.dart @@ -348,8 +348,11 @@ class VolcanoAsrService extends GetxService implements AsrService { } @override - Future enableRecord(bool audioSourceType, String filePath, - {bool acceptAudioData = false}) { + Future enableRecord( + AudioSourceType audioSourceType, + String filePath, + bool acceptAudioData, + ) { // TODO: implement enableRecord throw UnimplementedError(); } diff --git a/lib/modules/meeting/controllers/meeting_record_controller.dart b/lib/modules/meeting/controllers/meeting_record_controller.dart index f03f22bf2..b2c64798c 100644 --- a/lib/modules/meeting/controllers/meeting_record_controller.dart +++ b/lib/modules/meeting/controllers/meeting_record_controller.dart @@ -582,10 +582,31 @@ class MeetingRecordController extends GetxController final fullFileName = "${safeBase}_$formattedTime"; final filePath = path.join(dir.path, '$fullFileName.wav'); + // 根据音频类型选择音频源 + AudioSourceType audioSourceType; + bool acceptAudioData = false; + switch (audioType.value) { + case 0: // Live + audioSourceType = AudioSourceType.microphone; + acceptAudioData = true; + break; + case 1: // Media + audioSourceType = AudioSourceType.systemAudio; + acceptAudioData = false; + break; + case 2: // Call + audioSourceType = AudioSourceType.external; + acceptAudioData = false; + break; + default: + audioSourceType = AudioSourceType.microphone; + acceptAudioData = true; + } + await _asrService.enableRecord( - audioType.value != 0, + audioSourceType, filePath, - acceptAudioData: true, + acceptAudioData, ); // 通过 asrService 获取音频数据流 diff --git a/lib/modules/translation/controllers/translation_controller.dart b/lib/modules/translation/controllers/translation_controller.dart index cffd7cf59..29f267ecf 100644 --- a/lib/modules/translation/controllers/translation_controller.dart +++ b/lib/modules/translation/controllers/translation_controller.dart @@ -834,18 +834,6 @@ class TranslationController extends GetxController with WidgetsBindingObserver { try { Logger.info('开始初始化通话模式语音翻译服务'); - // 初始化 ASR 服务时，明确指定需要支持的语言 - final List asrSupportedLanguages = [targetLanguageCode.value]; - Logger.info('1初始化ASR服务，支持语言: $asrSupportedLanguages'); - - await _asrService.initialize(supportedLanguages: asrSupportedLanguages); - - // 获取识别事件流 - var recognitionStream = await _asrService.recognizeCallback(); - _recognitionSubscription?.cancel(); - _recognitionSubscription = - recognitionStream.listen(_handleRecognitionEvent); - final sourceShort = _languageManager.getShortCodeByAsrCode(sourceLanguageCode.value) ?? 'zh'; final targetShort = _languageManager.getShortCodeByAsrCode(targetLanguageCode.value) ?? 'en'; final bestProvider = _languageManager.findBestMatchingProvider(sourceShort, targetShort); @@ -870,10 +858,15 @@ class TranslationController extends GetxController with WidgetsBindingObserver { await _astService.initialize(supportedLanguages: callModeLanguages, provider: astProvider); - // 获取识别事件流 + // 获取识别事件流（端到端服务已包含 ASR+翻译+TTS，不需要单独启动 ASR） var astStream = await _astService.recognizeCallback(); _astEventSubscription?.cancel(); - _astEventSubscription = astStream.listen(_handleAstEvent); + _astEventSubscription = astStream.listen(_handleAstEvent, onError: (e) { + Logger.error('[STS] astStream 错误: $e'); + }, onDone: () { + Logger.info('[STS] astStream 已关闭'); + }); + Logger.info('[STS] 已订阅 astStream, subscription=${_astEventSubscription.hashCode}'); Logger.info('通话模式语音翻译服务初始化完成'); } catch (e) { @@ -948,13 +941,19 @@ class TranslationController extends GetxController with WidgetsBindingObserver { Future startRecording() async { _timerManager.startTimer(); final formattedTime = DateFormat('yyyyMMdd_HHmmss').format(DateTime.now()); - if (currentMode.value == 'call' || currentMode.value == 'audioVideo') { - await _asrService.enableRecord( - true, "${dir.path}/${currentModeTitle.value.tr}_$formattedTime.wav"); + AudioSourceType audioSourceType; + if (currentMode.value == 'call') { + audioSourceType = AudioSourceType.external; + } else if (currentMode.value == 'audioVideo') { + audioSourceType = AudioSourceType.systemAudio; } else { - await _asrService.enableRecord( - false, "${dir.path}/${currentModeTitle.value.tr}_$formattedTime.wav"); + audioSourceType = AudioSourceType.microphone; } + await _asrService.enableRecord( + audioSourceType, + "${dir.path}/${currentModeTitle.value.tr}_$formattedTime.wav", + false, + ); // if (currentMode.value == 'call') { // await _astService.enableRecord( @@ -1087,28 +1086,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } /// 配置通话模式 + /// 配置通话模式（只做 BLE 配置，AST 已在 _initializeCallModeTranslationService 中初始化） Future _configureCallMode() async { - final sourceShort = _languageManager.getShortCodeByAsrCode(sourceLanguageCode.value) ?? 'zh'; - final targetShort = _languageManager.getShortCodeByAsrCode(targetLanguageCode.value) ?? 'en'; - final bestProvider = _languageManager.findBestMatchingProvider(sourceShort, targetShort); - final astProvider = bestProvider?.provider ?? 'azure'; - - final transCode0 = bestProvider?.sourceSpec['translationCode'] ?? sourceLanguageCode.value; - final transCode1 = bestProvider?.targetSpec['translationCode'] ?? targetLanguageCode.value; - final ttsVoice0 = bestProvider?.sourceSpec['ttsCode'] ?? - (_languageManager.getTtsVoiceNameByAsrCode(sourceLanguageCode.value) ?? 'zh-CN-XiaoxiaoNeural'); - final ttsVoice1 = bestProvider?.targetSpec['ttsCode'] ?? - (_languageManager.getTtsVoiceNameByAsrCode(targetLanguageCode.value) ?? 'en-US-AriaNeural'); - final List callModeLanguages = [ - sourceLanguageCode.value, - targetLanguageCode.value, - transCode0, - transCode1, - ttsVoice0, - ttsVoice1, - ]; - Logger.info('1初始化AST服务，支持语言: $callModeLanguages, provider: $astProvider'); - await _astService.initialize(supportedLanguages: callModeLanguages, provider: astProvider); isPreparing.value = true; await bleManager.openA2DPDecoder(); int attempts = 0; @@ -1232,71 +1211,208 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } /// 处理 AST（语音识别+翻译一体）事件 + /// 按 serviceId 区分双路（A=己方, B=对方），用 utteranceId 匹配同一句话的事件 void _handleAstEvent(ASTEvent event) { + Logger.info('[STS] _handleAstEvent 收到: type=${event.type}, serviceId=${event.serviceId}, text=${event.text}'); + // 根据 serviceId 确定语言方向 + final String srcCode = event.serviceId == "A" + ? sourceLanguageCode.value + : targetLanguageCode.value; + final String tgtCode = event.serviceId == "A" + ? targetLanguageCode.value + : sourceLanguageCode.value; + switch (event.type) { case ASTEventType.intermediateResult: if (event.text.isEmpty) break; Logger.info('AST 识别中 [${event.serviceId}]: ${event.text}'); - handleIntermediateResult(event.text); + currentSessionId ??= DateTime.now().millisecondsSinceEpoch.toString(); + + // 按 utteranceId 查找已有项 + TranslationItem? target; + int targetIndex = -1; + if (event.utteranceId.isNotEmpty) { + for (int i = translationHistory.length - 1; i >= 0; i--) { + if (translationHistory[i].isIntermediate && + translationHistory[i].utteranceId == event.utteranceId) { + target = translationHistory[i]; + targetIndex = i; + break; + } + } + } + + if (target != null) { + target.sourceText = event.text; + target.sourceLanguageCode = srcCode; + target.targetLanguageCode = tgtCode; + target.serviceId = event.serviceId; + translationHistory.refresh(); + } else { + _historyManager.addTranslationItem(TranslationItem( + sourceText: event.text, + translatedText: '', + sourceLanguageCode: srcCode, + targetLanguageCode: tgtCode, + timestamp: DateTime.now(), + sessionId: currentSessionId!, + isFirstInSession: translationHistory.isEmpty || + translationHistory.last.sessionId != currentSessionId, + isIntermediate: true, + utteranceId: event.utteranceId, + serviceId: event.serviceId, + )); + } break; case ASTEventType.finalResult: if (event.text.isEmpty) break; Logger.info('AST 识别完成 [${event.serviceId}]: ${event.text}'); - // AST 已内置翻译，直接更新源文本，不再调用 translateText - if (translationHistory.isNotEmpty && - translationHistory.last.isIntermediate) { - translationHistory.last.sourceText = event.text; - translationHistory.last.isIntermediate = false; - translationHistory.refresh(); - _scrollToBottom(); - saveTranslationHistory(); - } else { - final newItem = TranslationItem( + currentSessionId ??= DateTime.now().millisecondsSinceEpoch.toString(); + + TranslationItem? target; + if (event.utteranceId.isNotEmpty) { + for (int i = translationHistory.length - 1; i >= 0; i--) { + if (translationHistory[i].isIntermediate && + translationHistory[i].utteranceId == event.utteranceId) { + target = translationHistory[i]; + break; + } + } + } + + if (target == null) { + target = TranslationItem( sourceText: event.text, translatedText: '', - sourceLanguageCode: sourceLanguageCode.value, - targetLanguageCode: targetLanguageCode.value, + sourceLanguageCode: srcCode, + targetLanguageCode: tgtCode, timestamp: DateTime.now(), - sessionId: currentSessionId ?? - DateTime.now().millisecondsSinceEpoch.toString(), + sessionId: currentSessionId!, isFirstInSession: translationHistory.isEmpty || translationHistory.last.sessionId != currentSessionId, - isIntermediate: false, + isIntermediate: true, + utteranceId: event.utteranceId, + serviceId: event.serviceId, + isFinalResultReceived: true, ); - translationHistory.add(newItem); + _historyManager.addTranslationItem(target); + } else { + target.sourceText = event.text; + target.sourceLanguageCode = srcCode; + target.targetLanguageCode = tgtCode; + target.serviceId = event.serviceId; + target.isFinalResultReceived = true; + } + + // 识别+翻译都到齐才标记完成 + if (target.isFinalResultReceived && target.isTranslationResultReceived) { + target.isIntermediate = false; translationHistory.refresh(); _scrollToBottom(); saveTranslationHistory(); + } else { + translationHistory.refresh(); } break; case ASTEventType.translationInterim: if (event.text.isEmpty) break; Logger.info('AST 翻译中 [${event.serviceId}]: ${event.text}'); - if (translationHistory.isNotEmpty) { - translationHistory.last.translatedText = event.text; + currentSessionId ??= DateTime.now().millisecondsSinceEpoch.toString(); + + TranslationItem? target; + if (event.utteranceId.isNotEmpty) { + for (int i = translationHistory.length - 1; i >= 0; i--) { + if (translationHistory[i].isIntermediate && + translationHistory[i].utteranceId == event.utteranceId) { + target = translationHistory[i]; + break; + } + } + } + + if (target != null) { + target.translatedText = event.text; + target.sourceLanguageCode = srcCode; + target.targetLanguageCode = tgtCode; translationHistory.refresh(); + } else { + _historyManager.addTranslationItem(TranslationItem( + sourceText: '', + translatedText: event.text, + sourceLanguageCode: srcCode, + targetLanguageCode: tgtCode, + timestamp: DateTime.now(), + sessionId: currentSessionId!, + isFirstInSession: translationHistory.isEmpty || + translationHistory.last.sessionId != currentSessionId, + isIntermediate: true, + utteranceId: event.utteranceId, + serviceId: event.serviceId, + )); } break; case ASTEventType.translationResult: if (event.text.isEmpty) break; Logger.info('AST 翻译完成 [${event.serviceId}]: ${event.text}'); - if (translationHistory.isNotEmpty) { - translationHistory.last.translatedText = event.text; + currentSessionId ??= DateTime.now().millisecondsSinceEpoch.toString(); + + TranslationItem? target; + if (event.utteranceId.isNotEmpty) { + for (int i = translationHistory.length - 1; i >= 0; i--) { + if (translationHistory[i].isIntermediate && + translationHistory[i].utteranceId == event.utteranceId) { + target = translationHistory[i]; + break; + } + } + } + + if (target == null) { + target = TranslationItem( + sourceText: '', + translatedText: event.text, + sourceLanguageCode: srcCode, + targetLanguageCode: tgtCode, + timestamp: DateTime.now(), + sessionId: currentSessionId!, + isFirstInSession: translationHistory.isEmpty || + translationHistory.last.sessionId != currentSessionId, + isIntermediate: true, + utteranceId: event.utteranceId, + serviceId: event.serviceId, + isTranslationResultReceived: true, + ); + translationHistory.add(target); + translationHistory.refresh(); + _scrollToBottom(); + } else { + target.translatedText = event.text; + target.sourceLanguageCode = srcCode; + target.targetLanguageCode = tgtCode; + target.isTranslationResultReceived = true; + } + + if (target.isFinalResultReceived && target.isTranslationResultReceived) { + target.isIntermediate = false; translationHistory.refresh(); _scrollToBottom(); saveTranslationHistory(); + } else { + translationHistory.refresh(); } break; case ASTEventType.error: Logger.error('AST 错误: ${event.error}'); + isRecognizing.value = false; break; case ASTEventType.canceled: Logger.error('AST 取消: ${event.error}'); + isRecognizing.value = false; break; default: @@ -1898,7 +2014,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } if (currentMode.value == "call") { - _initializeCallModeTranslationService(); + await _initializeCallModeTranslationService(); } else { Logger.info('重新初始化ASR服务，支持语言: $asrSupportedLanguages'); diff --git a/lib/modules/translation/models/translation_models.dart b/lib/modules/translation/models/translation_models.dart index d4df1913d..325818b45 100644 --- a/lib/modules/translation/models/translation_models.dart +++ b/lib/modules/translation/models/translation_models.dart @@ -12,6 +12,10 @@ class TranslationItem { String sessionId; final bool isFirstInSession; bool isIntermediate; // 是否为中间结果 + String utteranceId; // AST 事件的 utteranceId，用于匹配同一句话的识别和翻译 + String serviceId; // AST 服务 ID（A=己方, B=对方） + bool isFinalResultReceived; // 是否已收到最终识别结果 + bool isTranslationResultReceived; // 是否已收到最终翻译结果 TranslationItem({ required this.sourceText, @@ -22,6 +26,10 @@ class TranslationItem { required this.sessionId, this.isFirstInSession = false, this.isIntermediate = false, + this.utteranceId = '', + this.serviceId = '', + this.isFinalResultReceived = false, + this.isTranslationResultReceived = false, }); /// 从JSON创建TranslationItem实例 diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt index 72e5f43d9..687135011 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt @@ -130,12 +130,19 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, * 参数：`data` 为单声道 PCM 16kHz 16bit 音频数据 * 返回：无 */ + private var astAudioPushCountA = 0L + private var astAudioPushCountB = 0L + private fun pushAstAudioToA(data: ByteArray) { + astAudioPushCountA++ + if (astAudioPushCountA % 100 == 1L) { + FileLogger.d(tag, "pushAstAudioToA: provider=$currentAstProvider, size=${data.size}, count=$astAudioPushCountA") + } if (currentAstProvider == "iflytek") { iflytekAstHelperA?.pushAudioData(data) } else if (currentAstProvider == "azure") { azureAstHelperA?.pushAudioData(data) - } else if (currentAstProvider == "volcano") { + } else if (currentAstProvider == "volcano") { doubaoAstHelperA?.pushAudioData(data) } else if (currentAstProvider == "alibaba") { bailianAstHelperA?.pushAudioData(data) @@ -149,6 +156,10 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, * 返回：无 */ private fun pushAstAudioToB(data: ByteArray) { + astAudioPushCountB++ + if (astAudioPushCountB % 100 == 1L) { + FileLogger.d(tag, "pushAstAudioToB: provider=$currentAstProvider, size=${data.size}, count=$astAudioPushCountB") + } if (currentAstProvider == "iflytek" ) { iflytekAstHelperB?.pushAudioData(data) } else if (currentAstProvider == "azure") { @@ -1187,7 +1198,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, val azureTranslationKey = call.argument("azureTranslationKey") ?: "" val azureTranslationRegion = - call.argument("azureTranslationRegion") ?: "cn-north-1" + call.argument("azureTranslationServiceRegion") ?: "" val wsUrl = call.argument("wsUrl") ?: "wss://openspeech.bytedance.com/api/v4/ast/v2/translate" @@ -1240,8 +1251,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, { data -> bleWriteScope.launch { bleRightMutex.withLock { - // TODO: 双端翻译时启用右声道写入 - // BleService.writeExternalRightAudioData(data) + BleService.writeExternalAudioData(data) } } } @@ -1249,15 +1259,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, val callbackB = IflytekAstCallback( "B", "$lang1->$lang0", { sendAstEvent(it) }, - { data -> - bleWriteScope.launch { - bleLeftMutex.withLock { - // TODO: 双端翻译时启用左声道分离写入 - // BleService.writeExternalLeftAudioData(data) - BleService.writeExternalAudioData(data) - } - } - } + { data -> } ) GlobalScope.launch(Dispatchers.Main) { @@ -1312,8 +1314,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, { data -> bleWriteScope.launch { bleRightMutex.withLock { - // TODO: 双端翻译时启用右声道写入 - // BleService.writeExternalRightAudioData(data) + BleService.writeExternalAudioData(data) } } } @@ -1321,15 +1322,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, val callbackB = AzureAstCallback( "B", "$lang1->$lang0", { sendAstEvent(it) }, - { data -> - bleWriteScope.launch { - bleLeftMutex.withLock { - // TODO: 双端翻译时启用左声道分离写入 - // BleService.writeExternalLeftAudioData(data) - BleService.writeExternalAudioData(data) - } - } - } + { data -> } ) GlobalScope.launch(Dispatchers.Main) { @@ -1366,8 +1359,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, { data -> bleWriteScope.launch { bleRightMutex.withLock { - // TODO: 双端翻译时启用右声道写入 - // BleService.writeExternalRightAudioData(data) + BleService.writeExternalAudioData(data) } } } @@ -1375,15 +1367,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, val callbackB = DoubaoAstCallback( "B", "$translationLang1->$translationLang0", translationLang0, { sendAstEvent(it) }, - { data -> - bleWriteScope.launch { - bleLeftMutex.withLock { - // TODO: 双端翻译时启用左声道分离写入 - // BleService.writeExternalLeftAudioData(data) - BleService.writeExternalAudioData(data) - } - } - } + { data -> } ) Log.d(tag, "initializeIntegrated:lang0= $translationLang0, lang1=$translationLang1") // 设置会话语言（与 UI 选择一致） @@ -1428,8 +1412,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, { data -> bleWriteScope.launch { bleRightMutex.withLock { - // TODO: 双端翻译时启用右声道写入 - // BleService.writeExternalRightAudioData(data) + BleService.writeExternalAudioData(data) } } } @@ -1437,15 +1420,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, val callbackB = AliyunAstCallback( "B", "$translationLang1->$translationLang0", translationLang0, { sendAstEvent(it) }, - { data -> - bleWriteScope.launch { - bleLeftMutex.withLock { - // TODO: 双端翻译时启用左声道分离写入 - // BleService.writeExternalLeftAudioData(data) - BleService.writeExternalAudioData(data) - } - } - } + { data -> } ) GlobalScope.launch(Dispatchers.Main) { @@ -1773,7 +1748,13 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, override fun onConnectionStateChanged(state: Int) { } + private var bleAudioRecvCount = 0L + override fun onAudioDataReceived(data: ByteArray, channel: Int) { + bleAudioRecvCount++ + if (bleAudioRecvCount % 200 == 1L) { + FileLogger.d(tag, "onAudioDataReceived: channel=$channel, size=${data.size}, count=$bleAudioRecvCount, astProvider=$currentAstProvider") + } if (channel == 0) { return @@ -1829,12 +1810,30 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, rightBuffer[monoIndex + 1] = data[stereoIndex + 3] } + // 计算左右声道 RMS 音量（每200帧打一次） + if (bleAudioRecvCount % 200 == 1L) { + var leftRms = 0.0 + var rightRms = 0.0 + for (i in 0 until leftBuffer.size step 2) { + if (i + 1 < leftBuffer.size) { + val lSample = ((leftBuffer[i + 1].toInt() shl 8) or (leftBuffer[i].toInt() and 0xFF)).toShort() + val rSample = ((rightBuffer[i + 1].toInt() shl 8) or (rightBuffer[i].toInt() and 0xFF)).toShort() + leftRms += lSample.toDouble() * lSample.toDouble() + rightRms += rSample.toDouble() * rSample.toDouble() + } + } + val samples = leftBuffer.size / 2 + leftRms = Math.sqrt(leftRms / samples) + rightRms = Math.sqrt(rightRms / samples) + FileLogger.d(tag, "[STS] 声道音量: left(己方/麦克风)=${leftRms.toInt()}, right(对方)=${rightRms.toInt()}, threshold=$lowVolumeThreshold") + } + // 过滤低音量音频 val filteredLeftBuffer = filterLowVolumeAudio(leftBuffer, lowVolumeThreshold) val filteredRightBuffer = filterLowVolumeAudio(rightBuffer, lowVolumeThreshold) - // 左声道是对方的，右声道是麦的 - pushAstAudioToA(filteredRightBuffer) - pushAstAudioToB(filteredLeftBuffer) + // 左声道是自己的(麦克风)，右声道是对方的 + pushAstAudioToA(filteredLeftBuffer) + pushAstAudioToB(filteredRightBuffer) } diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt index 7003746c6..f2ae5df40 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt @@ -74,15 +74,22 @@ class DoubaoE2ETranslateHelper( /** * 初始化助手，设置配置与回调。 */ + private var instanceTag: String = "" + fun initialize(config: Config, cb: Callback): Boolean { - Log.d(TAG, "initialize: wsUrl=${config.wsUrl}, resourceId=${config.resourceId}") + instanceTag = "${config.sourceLanguage}->${config.targetLanguage}" + Log.d(TAG, "[$instanceTag] initialize: wsUrl=${config.wsUrl}, resourceId=${config.resourceId}") + // 重置状态，确保重新初始化时能正常启动 + isStarted.set(false) + try { webSocket?.close(1000, "reinitialize") } catch (_: Exception) {} + webSocket = null conf = config callback = cb client = OkHttpClient.Builder() .pingInterval(30, TimeUnit.SECONDS) .readTimeout(0, TimeUnit.SECONDS) .build() - Log.d(TAG, "initialize: client created") + Log.d(TAG, "[$instanceTag] initialize: client created") startContinuousTranslation() return true } @@ -118,7 +125,7 @@ class DoubaoE2ETranslateHelper( * 启动会话，建立 WebSocket 并发送 StartSession。 */ fun startContinuousTranslation(): Boolean { - Log.d(TAG, "startContinuousTranslation: isStarted=${isStarted.get()} clientIsNull=${client==null}") + Log.d(TAG, "[$instanceTag] startContinuousTranslation: isStarted=${isStarted.get()} clientIsNull=${client==null}") if (client == null) return false if (isStarted.get()) return true @@ -148,12 +155,12 @@ class DoubaoE2ETranslateHelper( * @param response 握手响应，包含状态码与头信息 */ override fun onOpen(ws: WebSocket, response: Response) { - Log.d(TAG, "onOpen: code=${response.code} logid=${response.header("X-Tt-Logid")}") + Log.d(TAG, "[$instanceTag] onOpen: code=${response.code} logid=${response.header("X-Tt-Logid")}") webSocket = ws - isStarted.set(true) val startReq = makeStartRequest(sessionId) ws.send(ByteString.of(*startReq.toByteArray())) Log.d(TAG, "onOpen: StartSession sent") + isStarted.set(true) callback?.onSessionStarted(sessionId) } @@ -169,7 +176,7 @@ class DoubaoE2ETranslateHelper( * @param bytes 服务端下发的二进制消息 */ override fun onMessage(ws: WebSocket, bytes: ByteString) { - Log.d(TAG, "onMessage: ${bytes.size} bytes") + Log.d(TAG, "[$instanceTag] onMessage: ${bytes.size} bytes") val resp = TranslateResponse.parseFrom(bytes.toByteArray()) val event = resp.event val text = resp.text @@ -322,14 +329,18 @@ class DoubaoE2ETranslateHelper( /** * 推送一段 PCM/WAV 音频数据到服务端。 */ + private var pushCount = 0L + fun pushAudioData(data: ByteArray): Boolean { - // Log.d(TAG, "pushAudioData: size=${data.size} isStarted=${isStarted.get()} wsIsNull=${webSocket==null}") + pushCount++ + if (pushCount % 100 == 1L) { + Log.d(TAG, "pushAudioData: size=${data.size} isStarted=${isStarted.get()} wsIsNull=${webSocket==null} count=$pushCount") + } val ws = webSocket ?: return false if (!isStarted.get()) return false synchronized(sessionLock) { val req = makeChunkRequest(sessionId, data) val ok = ws.send(ByteString.of(*req.toByteArray())) - // Log.d(TAG, "pushAudioData: sent=${ok}") return ok } } diff --git a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt index 857131e9b..777981938 100644 --- a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt +++ b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt @@ -1015,10 +1015,10 @@ object BleService { override fun onCharacteristicWrite( g: BluetoothGatt, c: BluetoothGattCharacteristic, status: Int ) { - Log.i( - TAG, - "写入 ${if (status == BluetoothGatt.GATT_SUCCESS) "成功" else "失败"}" - ) + // Log.i( + // TAG, + // "写入 ${if (status == BluetoothGatt.GATT_SUCCESS) "成功" else "失败"}" + // ) } override fun onDescriptorWrite( @@ -1400,7 +1400,7 @@ object BleService { isWaitingForWriteCallback = false // recordfile1?.saveAudioDataToWav("发送成功，大小: ${chunk.size} 字节") bytesSentInCurrentSecond += chunk.size - Log.d(TAG, "成功发送音频数据块，大小: ${chunk.size} 字节，等待写入回调...") + // Log.d(TAG, "成功发送音频数据块，大小: ${chunk.size} 字节，等待写入回调...") } else { isWaitingForWriteCallback = false //recordfile1?.saveAudioDataToWav("发送失败，大小: ${chunk.size} 字节")