From 804255cf2412803f494a29dd1cb67045ac4e4d53 Mon Sep 17 00:00:00 2001 From: fdp <1286779656@qq.com> Date: Wed, 23 Apr 2025 18:51:51 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8E=A5=E9=80=9A=E7=BF=BB=E8=AF=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/data/services/asr_service.dart | 5 + .../speech_impl/azure_asr_service.dart | 18 + .../speech_impl/flutter_asr_service.dart | 239 +++++++------- .../speech_impl/xunfei_asr_service.dart | 6 + .../controllers/translation_controller.dart | 310 ++++++++++++------ .../azure_speech/AzureAsrHelper.kt | 32 +- .../azure_speech/AzureSpeechPlugin.kt | 16 + .../com/yunqiinnovation/jieli/JieliPlugin.kt | 2 +- local_plugins/jieli/lib/jieli.dart | 2 +- .../yunqiinnovation/jl_opus/JlOpusPlugin.kt | 2 +- 10 files changed, 388 insertions(+), 244 deletions(-) diff --git a/lib/data/services/asr_service.dart b/lib/data/services/asr_service.dart index fee73821c..474b16408 100644 --- a/lib/data/services/asr_service.dart +++ b/lib/data/services/asr_service.dart @@ -16,6 +16,11 @@ abstract class AsrService { /// 返回识别的最终结果 Future recognizeOnce(); + /// 开始连续语音识别 + /// + /// 推送音频数据到连续识别中 + Future pushAudioData(Uint8List data); + /// 开始连续语音识别 /// /// 返回一个包含识别事件的流 diff --git a/lib/data/services/speech_impl/azure_asr_service.dart b/lib/data/services/speech_impl/azure_asr_service.dart index dee08ed5e..878206ef3 100644 --- a/lib/data/services/speech_impl/azure_asr_service.dart +++ b/lib/data/services/speech_impl/azure_asr_service.dart @@ -288,4 +288,22 @@ class AzureAsrService extends GetxService implements AsrService { _isInitialized = false; } } + + @override + Future pushAudioData(Uint8List data) async { + try { + // 开始连续识别 + final bool result = await _channel.invokeMethod('pushAudioData', { + 'AudioData': data // Flutter 会自动将 Uint8List 转换为 ByteBuffer + }); + + if (!result) { + _cleanupEventStream(); + throw Exception('开始pushAudioData失败'); + } + } catch (e) { + Logger.error('开始pushAudioData失败: $e'); + rethrow; + } + } } diff --git a/lib/data/services/speech_impl/flutter_asr_service.dart b/lib/data/services/speech_impl/flutter_asr_service.dart index e17fb9198..7cbf9b88f 100644 --- a/lib/data/services/speech_impl/flutter_asr_service.dart +++ b/lib/data/services/speech_impl/flutter_asr_service.dart @@ -11,26 +11,26 @@ import '../asr_service.dart'; enum AsrState { /// 未初始化 notInitialized, - + /// 已初始化但未开始监听 initialized, - + /// 正在监听 listening, - + /// 正在处理识别结果 processing, - + /// 识别完成 done, - + /// 发生错误 error, } /// Flutter ASR服务，负责本地语音转文本功能 /// 使用speech_to_text插件实现语音识别功能 -class FlutterAsrService extends GetxService implements AsrService { +class FlutterAsrService extends GetxService implements AsrService { static final FlutterAsrService to = Get.put(FlutterAsrService()); // speech_to_text 实例 @@ -40,34 +40,34 @@ class FlutterAsrService extends GetxService implements AsrService { final List _supportedLanguages = []; @override List get supportedLanguages => _supportedLanguages; - + // 当前选择的语言 final RxString _currentLocale = ''.obs; String get currentLocale => _currentLocale.value; - + // 是否可用 final RxBool _isAvailable = false.obs; bool get isAvailable => _isAvailable.value; - + // 是否正在监听 bool _isListening = false; - + // 当前ASR状态 final Rx _asrState = AsrState.notInitialized.obs; AsrState get asrState => _asrState.value; - + // 连续识别模式的结果流控制器 StreamController? _continuousRecognitionController; - + FlutterAsrService() { _initSpeech(); } - + /// 初始化语音识别 Future _initSpeech() async { try { _asrState.value = AsrState.notInitialized; - + // 初始化语音识别，使用较短的最终超时时间，提高响应速度 _isAvailable.value = await _speech.initialize( onStatus: _onSpeechStatus, @@ -75,11 +75,11 @@ class FlutterAsrService extends GetxService implements AsrService { debugLogging: false, finalTimeout: const Duration(milliseconds: 800), // 更短的最终超时时间，加速结果返回 ); - + if (_isAvailable.value) { // 获取支持的语言列表 final systemLocales = await _speech.locales(); - + // 清空并重新填充支持的语言列表 _supportedLanguages.clear(); for (var locale in systemLocales) { @@ -87,7 +87,7 @@ class FlutterAsrService extends GetxService implements AsrService { _supportedLanguages.add(locale.localeId); } } - + // 设置默认语言（优先使用系统语言或中文） var systemLocale = await _speech.systemLocale(); if (systemLocale != null) { @@ -97,7 +97,7 @@ class FlutterAsrService extends GetxService implements AsrService { } else if (_supportedLanguages.isNotEmpty) { _currentLocale.value = _supportedLanguages.first; } - + _asrState.value = AsrState.initialized; Logger.info('语音识别初始化成功，支持 ${_supportedLanguages.length} 种语言'); } else { @@ -110,11 +110,11 @@ class FlutterAsrService extends GetxService implements AsrService { Logger.error('语音识别初始化失败: $e'); } } - + /// 处理语音识别状态变化 void _onSpeechStatus(String status) { Logger.info('语音识别状态: $status'); - + switch (status) { case 'listening': _asrState.value = AsrState.listening; @@ -129,31 +129,33 @@ class FlutterAsrService extends GetxService implements AsrService { case 'done': _asrState.value = AsrState.done; _isListening = false; - + // 在连续识别模式下，当一次识别完成后立即开始下一次识别 - if (_continuousRecognitionController != null && !_continuousRecognitionController!.isClosed) { + if (_continuousRecognitionController != null && + !_continuousRecognitionController!.isClosed) { _restartContinuousRecognition(); } break; } } - + /// 处理语音识别错误 void _onSpeechError(SpeechRecognitionError error) { Logger.error('语音识别错误: ${error.errorMsg} (永久性错误: ${error.permanent})'); _asrState.value = AsrState.error; - + // 向连续识别控制器发送错误事件（如果存在） - _continuousRecognitionController?.add( - RecognitionEvent.error(error.errorMsg) - ); - + _continuousRecognitionController + ?.add(RecognitionEvent.error(error.errorMsg)); + // 在连续识别模式下，尝试从错误中恢复（非永久性错误） - if (!error.permanent && _continuousRecognitionController != null && !_continuousRecognitionController!.isClosed) { + if (!error.permanent && + _continuousRecognitionController != null && + !_continuousRecognitionController!.isClosed) { _restartContinuousRecognition(); } } - + /// 启动语音识别的统一方法 Future _startListening({ required Function(SpeechRecognitionResult) onResult, @@ -168,36 +170,37 @@ class FlutterAsrService extends GetxService implements AsrService { // 延长等待时间，确保资源完全释放 await Future.delayed(const Duration(milliseconds: 800)); } - + // 在启动前再次检查状态 if (_speech.isListening) { Logger.warning('停止识别后，仍处于监听状态，可能有系统冲突'); await _speech.stop(); await Future.delayed(const Duration(milliseconds: 500)); } - + // 启动语音识别前确保状态正确 _asrState.value = AsrState.listening; - + // 启动语音识别 final result = await _speech.listen( - onResult: onResult, - localeId: _currentLocale.value, - listenFor: listenFor, - pauseFor: pauseFor, - listenOptions: stt.SpeechListenOptions( - partialResults: true, - autoPunctuation: autoPunctuation, - // 使用独白模式，减少与系统交互冲突 - listenMode: stt.ListenMode.dictation, - ), - ) ?? false; - + onResult: onResult, + localeId: _currentLocale.value, + listenFor: listenFor, + pauseFor: pauseFor, + listenOptions: stt.SpeechListenOptions( + partialResults: true, + autoPunctuation: autoPunctuation, + // 使用独白模式，减少与系统交互冲突 + listenMode: stt.ListenMode.dictation, + ), + ) ?? + false; + if (!result) { Logger.error('语音识别启动失败'); _asrState.value = AsrState.error; } - + return result; } catch (e) { Logger.error('启动语音识别失败: $e'); @@ -205,33 +208,35 @@ class FlutterAsrService extends GetxService implements AsrService { return false; } } - + /// 重新启动连续识别 Future _restartContinuousRecognition() async { try { // 确保控制器有效 - if (_continuousRecognitionController == null || _continuousRecognitionController!.isClosed) { + if (_continuousRecognitionController == null || + _continuousRecognitionController!.isClosed) { Logger.info('连续识别控制器已关闭，跳过重启'); return; } - + // 检查系统状态，避免频繁重启 if (_speech.isListening) { Logger.info('当前已在监听中，无需重启'); return; } - + // 延长等待时间，避免与系统资源冲突 await Future.delayed(const Duration(milliseconds: 1000)); - + // 再次检查状态 - if (_continuousRecognitionController == null || _continuousRecognitionController!.isClosed) { + if (_continuousRecognitionController == null || + _continuousRecognitionController!.isClosed) { Logger.info('延迟期间控制器已关闭，跳过重启'); return; } - + Logger.info('尝试重新启动连续识别'); - + // 使用统一方法启动识别 final success = await _startListening( onResult: _onSpeechResult, @@ -239,49 +244,42 @@ class FlutterAsrService extends GetxService implements AsrService { listenFor: const Duration(seconds: 30), pauseFor: const Duration(seconds: 4), ); - + if (success) { Logger.info('连续识别重启成功'); } else { Logger.error('连续识别重启失败'); - _continuousRecognitionController?.add( - RecognitionEvent.error('重启语音识别失败，可能存在系统资源冲突') - ); + _continuousRecognitionController + ?.add(RecognitionEvent.error('重启语音识别失败，可能存在系统资源冲突')); } } catch (e) { Logger.error('重启连续识别失败: $e'); - + // 发送错误给控制器 - _continuousRecognitionController?.add( - RecognitionEvent.error('重启失败: $e') - ); + _continuousRecognitionController?.add(RecognitionEvent.error('重启失败: $e')); } } - + /// 处理语音识别结果 void _onSpeechResult(SpeechRecognitionResult result) { // 如果是连续识别，发送到连续识别控制器 if (_continuousRecognitionController != null) { if (result.finalResult) { Logger.info('收到最终识别结果: "${result.recognizedWords}"'); - _continuousRecognitionController!.add( - RecognitionEvent.finalResult( - text: result.recognizedWords, - detectedLanguage: currentLocale, - ) - ); + _continuousRecognitionController!.add(RecognitionEvent.finalResult( + text: result.recognizedWords, + detectedLanguage: currentLocale, + )); } else { - _continuousRecognitionController!.add( - RecognitionEvent( - type: RecognitionEventType.intermediateResult, - text: result.recognizedWords, - detectedLanguage: currentLocale, - ) - ); + _continuousRecognitionController!.add(RecognitionEvent( + type: RecognitionEventType.intermediateResult, + text: result.recognizedWords, + detectedLanguage: currentLocale, + )); } } } - + /// 设置当前语言 void setLocale(String localeId) { if (_supportedLanguages.contains(localeId)) { @@ -291,7 +289,7 @@ class FlutterAsrService extends GetxService implements AsrService { Logger.error('不支持的语言: $localeId'); } } - + @override Future initialize({ List? supportedLanguages, @@ -301,7 +299,7 @@ class FlutterAsrService extends GetxService implements AsrService { _supportedLanguages.clear(); _supportedLanguages.addAll(supportedLanguages); } - + // 初始化语音识别 await _initSpeech(); return _isAvailable.value; @@ -310,24 +308,24 @@ class FlutterAsrService extends GetxService implements AsrService { return false; } } - + @override Future recognizeOnce() async { try { if (!_isAvailable.value) { return RecognitionEvent.error('语音识别不可用'); } - + if (_isListening) { await stopContinuousRecognition(); } - + // 创建Completer来等待最终结果 final completer = Completer(); - + // 开始监听 _asrState.value = AsrState.listening; - + // 定义结果处理函数 void resultListener(SpeechRecognitionResult result) { if (result.finalResult) { @@ -338,21 +336,21 @@ class FlutterAsrService extends GetxService implements AsrService { )); } } - + // 使用统一方法启动识别，单次识别使用更短的时间参数 final success = await _startListening( onResult: resultListener, listenFor: const Duration(seconds: 15), pauseFor: const Duration(seconds: 2), ); - + if (!success) { return RecognitionEvent.error('启动语音识别失败'); } - + // 设置较短的超时时间 return await completer.future.timeout( - const Duration(seconds: 20), // 略长于listenFor以确保获取到结果 + const Duration(seconds: 20), // 略长于listenFor以确保获取到结果 onTimeout: () { _speech.stop(); return RecognitionEvent.error('语音识别超时'); @@ -363,7 +361,7 @@ class FlutterAsrService extends GetxService implements AsrService { return RecognitionEvent.error('$e'); } } - + @override Future> startContinuousRecognition() async { try { @@ -373,43 +371,42 @@ class FlutterAsrService extends GetxService implements AsrService { controller.close(); return controller.stream; } - + // 如果已经在监听，先停止 if (_isListening) { await stopContinuousRecognition(); } - + // 创建新的流控制器 - _continuousRecognitionController = StreamController.broadcast(); - + _continuousRecognitionController = + StreamController.broadcast(); + // 开始会话 - _continuousRecognitionController!.add( - RecognitionEvent(type: RecognitionEventType.sessionStarted) - ); - + _continuousRecognitionController! + .add(RecognitionEvent(type: RecognitionEventType.sessionStarted)); + // 开始连续监听 _asrState.value = AsrState.listening; - + // 使用统一方法启动识别，连续识别使用较长的参数 final success = await _startListening( onResult: _onSpeechResult, ); - + if (!success) { // 添加错误事件，但保留流 - _continuousRecognitionController!.add( - RecognitionEvent.error('启动连续语音识别失败') - ); - + _continuousRecognitionController! + .add(RecognitionEvent.error('启动连续语音识别失败')); + // 返回已有的流控制器 return _continuousRecognitionController!.stream; } - + _isListening = true; return _continuousRecognitionController!.stream; } catch (e) { Logger.error('启动连续语音识别失败: $e'); - + // 如果已经创建了控制器，使用它 if (_continuousRecognitionController != null) { _continuousRecognitionController!.add(RecognitionEvent.error('$e')); @@ -422,7 +419,7 @@ class FlutterAsrService extends GetxService implements AsrService { } } } - + @override Future stopContinuousRecognition() async { try { @@ -430,36 +427,35 @@ class FlutterAsrService extends GetxService implements AsrService { if (_speech.isListening) { await _speech.stop(); } - + _isListening = false; - + // 发送会话结束事件 - _continuousRecognitionController?.add( - RecognitionEvent(type: RecognitionEventType.sessionStopped) - ); - + _continuousRecognitionController + ?.add(RecognitionEvent(type: RecognitionEventType.sessionStopped)); + // 关闭流控制器 await _continuousRecognitionController?.close(); _continuousRecognitionController = null; - + _asrState.value = AsrState.done; return true; } catch (e) { Logger.error('停止连续语音识别失败: $e'); - + // 强制重置状态 _isListening = false; _asrState.value = AsrState.error; - + return false; } } - + @override bool isContinuousRecognitionActive() { return _isListening; } - + @override Future dispose() async { try { @@ -467,15 +463,18 @@ class FlutterAsrService extends GetxService implements AsrService { if (_speech.isListening) { await _speech.stop(); } - + // 关闭控制器 await _continuousRecognitionController?.close(); _continuousRecognitionController = null; - + _isListening = false; _asrState.value = AsrState.notInitialized; } catch (e) { Logger.error('释放语音识别资源失败: $e'); } } -} \ No newline at end of file + + @override + Future pushAudioData(Uint8List data) async {} +} diff --git a/lib/data/services/speech_impl/xunfei_asr_service.dart b/lib/data/services/speech_impl/xunfei_asr_service.dart index 1201f7c66..cadb48b2f 100644 --- a/lib/data/services/speech_impl/xunfei_asr_service.dart +++ b/lib/data/services/speech_impl/xunfei_asr_service.dart @@ -269,4 +269,10 @@ class XunfeiAsrService extends GetxService implements AsrService { _isInitialized = false; } } + + @override + Future pushAudioData(Uint8List data) { + // TODO: implement pushAudioData + throw UnimplementedError(); + } } diff --git a/lib/modules/translation/controllers/translation_controller.dart b/lib/modules/translation/controllers/translation_controller.dart index 59822e45f..d07fe9a22 100644 --- a/lib/modules/translation/controllers/translation_controller.dart +++ b/lib/modules/translation/controllers/translation_controller.dart @@ -10,14 +10,37 @@ import '../../../core/utils/logger.dart'; import '../../../data/models/events/voice_interaction_event.dart'; import '../../../data/services/voice_interaction_service.dart'; import '../../../data/services/asr_service.dart'; +import 'package:jieli/jieli.dart'; + class TranslationController extends GetxController { + late Jieli jieli; + // 事件订阅 + StreamSubscription? _scanSubscription; + StreamSubscription? _connectionSubscription; + StreamSubscription? _wakeupSubscription; + StreamSubscription? _audioSubscription; + StreamSubscription? _commandResultSubscription; + // 状态变量 + var isScanning = false.obs; + var deviceInfo = '未获取'.obs; + var connectStatus = '未连接'.obs; + var currentDevice = Rx(null); + var scannedDevices = RxList([]); + var wakeupReceived = false.obs; + var audioDataSize = 0.obs; + var receivedFrames = 0.obs; + var audioDataReceiving = false.obs; + var volumeCommandResult = ''.obs; + // 服务 final AsrService _asrService = Get.find(); - final VolcanoTranslationService _translationService = Get.find(); + final VolcanoTranslationService _translationService = + Get.find(); final TtsService _ttsService = Get.find(); final LanguageManager _languageManager = Get.find(); final GetStorage _storage = GetStorage(); - final VoiceInteractionService _voiceInteractionService = Get.find(); + final VoiceInteractionService _voiceInteractionService = + Get.find(); // 存储相关 static const String _historyKey = 'translation_history'; @@ -29,75 +52,80 @@ class TranslationController extends GetxController { final isRecognizing = false.obs; final isTranslating = false.obs; final isTtsEnabled = true.obs; - + // 语言相关 final sourceLanguage = '中文(简体)'.obs; final targetLanguage = '英语'.obs; final sourceLanguageCode = 'zh-CN'.obs; final targetLanguageCode = 'en-US'.obs; var detectedLanguageCode = ''; - + // 翻译历史和当前项 final translationHistory = [].obs; String? currentSessionId; - + // 兼容旧版视图的计算属性 Rx get currentItem { - if (translationHistory.isNotEmpty && translationHistory.last.isIntermediate) { + if (translationHistory.isNotEmpty && + translationHistory.last.isIntermediate) { return Rx(translationHistory.last); } return Rx(null); } - + // UI相关 final ScrollController scrollController = ScrollController(); - + // 其他 StreamSubscription? _recognitionSubscription; StreamSubscription? _voiceInteractionSubscription; Timer? _translationDebounceTimer; // 获取支持的语言列表 - Map get supportedLanguages => _languageManager.getChineseNameToAsrCodeMap(); + Map get supportedLanguages => + _languageManager.getChineseNameToAsrCodeMap(); @override void onInit() { super.onInit(); - + jieli = Jieli(); + //_setupEventListeners(); // 加载上次保存的语言设置 _loadSavedLanguages(); - + // 初始化语言代码 - final sourceAsrCode = _languageManager.getAsrCodeByChineseName(sourceLanguage.value); + final sourceAsrCode = + _languageManager.getAsrCodeByChineseName(sourceLanguage.value); if (sourceAsrCode != null) { sourceLanguageCode.value = sourceAsrCode; } - - final targetAsrCode = _languageManager.getAsrCodeByChineseName(targetLanguage.value); + + final targetAsrCode = + _languageManager.getAsrCodeByChineseName(targetLanguage.value); if (targetAsrCode != null) { targetLanguageCode.value = targetAsrCode; } - + // 订阅原生服务事件 _subscribeToVoiceInteractionService(); // 加载历史记录 loadTranslationHistory(); - + // 初始化服务 _initServices(); - + // 使用微任务在布局完成后自动滚动到底部 WidgetsBinding.instance.addPostFrameCallback((_) { _scrollToBottom(); - }); } - + /// 订阅原生服务事件 void _subscribeToVoiceInteractionService() { if (_voiceInteractionService.eventStream != null) { - _voiceInteractionSubscription = _voiceInteractionService.eventStream!.listen((event) { + _voiceInteractionSubscription = + _voiceInteractionService.eventStream!.listen((event) { if (event is RecognitionStartedEvent) { _handleVoiceInteractionStarted(); } @@ -109,35 +137,54 @@ class TranslationController extends GetxController { void _loadSavedLanguages() { final savedSourceLanguage = _storage.read(_sourceLanguageKey); final savedTargetLanguage = _storage.read(_targetLanguageKey); - - if (savedSourceLanguage != null && supportedLanguages.containsKey(savedSourceLanguage)) { + + if (savedSourceLanguage != null && + supportedLanguages.containsKey(savedSourceLanguage)) { sourceLanguage.value = savedSourceLanguage; } - - if (savedTargetLanguage != null && supportedLanguages.containsKey(savedTargetLanguage)) { + + if (savedTargetLanguage != null && + supportedLanguages.containsKey(savedTargetLanguage)) { targetLanguage.value = savedTargetLanguage; } } - + // 保存当前语言设置 void _saveLanguageSettings() { _storage.write(_sourceLanguageKey, sourceLanguage.value); _storage.write(_targetLanguageKey, targetLanguage.value); } - + + // /// 设置事件监听器 + // void _setupEventListeners() { + // // 监听扫描结果 + // _scanSubscription = jieli.scanResults.listen((devices) { + // scannedDevices.assignAll(devices); + // }); + + // // 监听唤醒信号 + // _wakeupSubscription = jieli.wakeupStream.listen((wakeup) { + // wakeupReceived.value = true; + // Get.snackbar('收到唤醒信号', '耳机发送了唤醒信号'); + // }); + // } + // 初始化服务 Future _initServices() async { try { await _translationService.initialize(); - + // 初始化 ASR 服务时，明确指定需要支持的语言 - final List asrSupportedLanguages = [sourceLanguageCode.value, targetLanguageCode.value]; + final List asrSupportedLanguages = [ + sourceLanguageCode.value, + targetLanguageCode.value + ]; Logger.info('初始化ASR服务，支持语言: $asrSupportedLanguages'); - + await _asrService.initialize( supportedLanguages: asrSupportedLanguages, ); - + Logger.info('所有服务初始化完成'); } catch (e) { Logger.error('初始化服务失败: $e'); @@ -159,20 +206,26 @@ class TranslationController extends GetxController { // 开始语音识别 Future startRecognition() async { + Logger.info('开始语音识别'); if (isRecognizing.value) return; _voiceInteractionService.pauseVoiceInteraction(); try { + Logger.info('开始语音识别1'); // 每次开始识别前检查 ASR 服务是否已配置为当前语言对 - final List currentLanguages = [sourceLanguageCode.value, targetLanguageCode.value]; - final bool needsReinit = !_asrService.supportedLanguages.toSet().containsAll(currentLanguages); - + final List currentLanguages = [ + sourceLanguageCode.value, + targetLanguageCode.value + ]; + final bool needsReinit = + !_asrService.supportedLanguages.toSet().containsAll(currentLanguages); + if (needsReinit) { Logger.info('识别开始前发现语言变更，重新初始化ASR服务'); await _reinitializeAsrService(); } - + currentSessionId = DateTime.now().millisecondsSinceEpoch.toString(); // // 初始化新的翻译项 @@ -186,17 +239,25 @@ class TranslationController extends GetxController { // isFirstInSession: true, // isIntermediate: true, // ); - + // // 添加到历史记录 // translationHistory.add(newItem); - + // 开始连续语音识别 final recognitionStream = await _asrService.startContinuousRecognition(); + Logger.info('翻译开启，开始监'); + // 监听音频数据 + _audioSubscription = jieli.audioDataStream.listen((data) async { + // Logger.info('翻译开启，开始监听解码后的数据=${data}'); + final pushAudioDataStream = await _asrService.pushAudioData(data); + }); + isRecognizing.value = true; // 监听识别结果 _recognitionSubscription?.cancel(); - _recognitionSubscription = recognitionStream.listen(_handleRecognitionEvent); + _recognitionSubscription = + recognitionStream.listen(_handleRecognitionEvent); } catch (e) { isRecognizing.value = false; Logger.error('启动语音识别失败: $e'); @@ -205,10 +266,12 @@ class TranslationController extends GetxController { // 处理识别事件 void _handleRecognitionEvent(RecognitionEvent event) { - if (event.type == RecognitionEventType.finalResult && event.text.isNotEmpty) { + if (event.type == RecognitionEventType.finalResult && + event.text.isNotEmpty) { detectedLanguageCode = event.detectedLanguage; handleFinalResult(event.text); - } else if (event.type == RecognitionEventType.intermediateResult && event.text.isNotEmpty) { + } else if (event.type == RecognitionEventType.intermediateResult && + event.text.isNotEmpty) { if (event.detectedLanguage.isNotEmpty) { detectedLanguageCode = event.detectedLanguage; } @@ -231,21 +294,25 @@ class TranslationController extends GetxController { _recognitionSubscription?.cancel(); _recognitionSubscription = null; isRecognizing.value = false; - + // 如果当前有中间结果，将其作为最终结果处理 - if (translationHistory.isNotEmpty && translationHistory.last.isIntermediate && translationHistory.last.sourceText.isNotEmpty) { + if (translationHistory.isNotEmpty && + translationHistory.last.isIntermediate && + translationHistory.last.sourceText.isNotEmpty) { // 标记为最终结果 translationHistory.last.isIntermediate = false; translationHistory.refresh(); - + // 如果还没有翻译结果，尝试翻译 if (translationHistory.last.translatedText.isEmpty) { - await translateText(translationHistory.last.sourceText, translationHistory.last.timestamp, isFinal: true); + await translateText(translationHistory.last.sourceText, + translationHistory.last.timestamp, + isFinal: true); } - + saveTranslationHistory(); } - + currentSessionId = null; } catch (e) { Logger.error('停止语音识别失败: $e'); @@ -265,10 +332,11 @@ class TranslationController extends GetxController { targetLanguageCode: targetLanguageCode.value, timestamp: DateTime.now(), sessionId: currentSessionId!, - isFirstInSession: translationHistory.isEmpty || translationHistory.last.sessionId != currentSessionId, + isFirstInSession: translationHistory.isEmpty || + translationHistory.last.sessionId != currentSessionId, isIntermediate: true, ); - + translationHistory.add(newItem); // 添加新项后滚动到底部 _scrollToBottom(); @@ -283,8 +351,12 @@ class TranslationController extends GetxController { // 防抖处理翻译请求 _translationDebounceTimer?.cancel(); _translationDebounceTimer = Timer(const Duration(milliseconds: 300), () { - if (translationHistory.isNotEmpty && translationHistory.last.isIntermediate && translationHistory.last.sourceText.isNotEmpty) { - translateText(translationHistory.last.sourceText, translationHistory.last.timestamp, isFinal: false); + if (translationHistory.isNotEmpty && + translationHistory.last.isIntermediate && + translationHistory.last.sourceText.isNotEmpty) { + translateText(translationHistory.last.sourceText, + translationHistory.last.timestamp, + isFinal: false); } }); } @@ -301,7 +373,7 @@ class TranslationController extends GetxController { // 更新后滚动到底部 _scrollToBottom(); saveTranslationHistory(); - + // 使用项目的时间戳进行翻译 translateText(text, translationHistory.last.timestamp, isFinal: true); } else { @@ -312,16 +384,17 @@ class TranslationController extends GetxController { sourceLanguageCode: sourceLanguageCode.value, targetLanguageCode: targetLanguageCode.value, timestamp: DateTime.now(), - sessionId: currentSessionId ?? DateTime.now().millisecondsSinceEpoch.toString(), - isFirstInSession: translationHistory.isEmpty || translationHistory.last.sessionId != currentSessionId, + sessionId: currentSessionId ?? + DateTime.now().millisecondsSinceEpoch.toString(), + isFirstInSession: translationHistory.isEmpty || + translationHistory.last.sessionId != currentSessionId, isIntermediate: false, ); - + translationHistory.add(newItem); translationHistory.refresh(); _scrollToBottom(); saveTranslationHistory(); - // 使用新项目的时间戳进行翻译 translateText(text, newItem.timestamp, isFinal: true); @@ -329,7 +402,8 @@ class TranslationController extends GetxController { } // 翻译文本 - Future translateText(String sourceText, DateTime timestamp, {bool isFinal = false}) async { + Future translateText(String sourceText, DateTime timestamp, + {bool isFinal = false}) async { if (sourceText.isEmpty) return null; isTranslating.value = true; @@ -338,8 +412,10 @@ class TranslationController extends GetxController { try { // 根据检测到的语言确定源语言和目标语言 final bool shouldSwap = detectedLanguageCode == sourceLanguageCode.value; - final detectedSourceLanguageCode = shouldSwap ? sourceLanguageCode.value : targetLanguageCode.value; - final detectedTargetLanguageCode = shouldSwap ? targetLanguageCode.value : sourceLanguageCode.value; + final detectedSourceLanguageCode = + shouldSwap ? sourceLanguageCode.value : targetLanguageCode.value; + final detectedTargetLanguageCode = + shouldSwap ? targetLanguageCode.value : sourceLanguageCode.value; // 调用翻译服务 translationResult = await _translationService.translateText( @@ -352,25 +428,28 @@ class TranslationController extends GetxController { // 查找匹配的项目 - 优先使用timestamp查找 int index = -1; - index = translationHistory.indexWhere((item) => - item.timestamp.isAtSameMomentAs(timestamp)); + index = translationHistory + .indexWhere((item) => item.timestamp.isAtSameMomentAs(timestamp)); if (index != -1) { // 更新历史项 translationHistory[index].translatedText = translationResult; - translationHistory[index].sourceLanguageCode = detectedSourceLanguageCode; - translationHistory[index].targetLanguageCode = detectedTargetLanguageCode; + translationHistory[index].sourceLanguageCode = + detectedSourceLanguageCode; + translationHistory[index].targetLanguageCode = + detectedTargetLanguageCode; translationHistory.refresh(); saveTranslationHistory(); // 更新后滚动到底部 _scrollToBottom(); - + // 播放TTS - 只在最终结果时播放 if (isTtsEnabled.value && isFinal) { // Logger.info('播放翻译文本: ${translationResult}'); await playTranslatedText(translationResult); } } else { - Logger.info('未找到匹配的历史项 ${sourceText} ${timestamp.toIso8601String() ?? "无时间戳"}'); + Logger.info( + '未找到匹配的历史项 ${sourceText} ${timestamp.toIso8601String() ?? "无时间戳"}'); } } } catch (e) { @@ -388,9 +467,12 @@ class TranslationController extends GetxController { try { final bool shouldSwap = detectedLanguageCode == targetLanguageCode.value; - final detectedTargetLanguageCode = shouldSwap ? sourceLanguageCode.value : targetLanguageCode.value; - final voiceName = _languageManager.getTtsVoiceNameByAsrCode(detectedTargetLanguageCode) ?? 'en-US-AriaNeural'; - + final detectedTargetLanguageCode = + shouldSwap ? sourceLanguageCode.value : targetLanguageCode.value; + final voiceName = _languageManager + .getTtsVoiceNameByAsrCode(detectedTargetLanguageCode) ?? + 'en-US-AriaNeural'; + await _ttsService.setVoice(voiceName); await _ttsService.speakOnce(text); } catch (e) { @@ -404,7 +486,7 @@ class TranslationController extends GetxController { if (isRecognizing.value) { await stopRecognition(); } - + _translationDebounceTimer?.cancel(); await _ttsService.stop(); } catch (e) { @@ -415,9 +497,11 @@ class TranslationController extends GetxController { // 播放特定翻译项 Future playTranslationItem(TranslationItem item) async { if (item.translatedText.isEmpty) return; - + try { - final voiceName = _languageManager.getTtsVoiceNameByAsrCode(item.targetLanguageCode) ?? 'en-US-AriaNeural'; + final voiceName = + _languageManager.getTtsVoiceNameByAsrCode(item.targetLanguageCode) ?? + 'en-US-AriaNeural'; await _ttsService.setVoice(voiceName); await _ttsService.speakOnce(item.translatedText); } catch (e) { @@ -428,12 +512,15 @@ class TranslationController extends GetxController { // 播放文本 Future playText(String text, {String? languageCode}) async { if (text.isEmpty) return; - + try { - final voiceName = languageCode != null - ? _languageManager.getTtsVoiceNameByAsrCode(languageCode) ?? 'en-US-AriaNeural' - : _languageManager.getTtsVoiceNameByAsrCode(targetLanguageCode.value) ?? 'en-US-AriaNeural'; - + final voiceName = languageCode != null + ? _languageManager.getTtsVoiceNameByAsrCode(languageCode) ?? + 'en-US-AriaNeural' + : _languageManager + .getTtsVoiceNameByAsrCode(targetLanguageCode.value) ?? + 'en-US-AriaNeural'; + await _ttsService.setVoice(voiceName); await _ttsService.speakOnce(text); } catch (e) { @@ -443,7 +530,8 @@ class TranslationController extends GetxController { // 滚动到底部 void _scrollToBottom() { - if (scrollController.hasClients && (translationHistory.isNotEmpty || currentSessionId != null)) { + if (scrollController.hasClients && + (translationHistory.isNotEmpty || currentSessionId != null)) { try { scrollController.animateTo( scrollController.position.maxScrollExtent, @@ -460,33 +548,33 @@ class TranslationController extends GetxController { void setupScrollListener() { // 简化后的空方法，保留接口兼容性 } - + // 底部控制栏高度 double _bottomBarHeight = 0.0; - + // 设置底部控制栏高度 void setBottomBarHeight(double height) { _bottomBarHeight = height; } - + // 获取底部控制栏高度 double get bottomBarHeight => _bottomBarHeight; // 切换源语言和目标语言 void swapLanguages() { stopAll(); - + final tempName = sourceLanguage.value; sourceLanguage.value = targetLanguage.value; targetLanguage.value = tempName; - + final tempCode = sourceLanguageCode.value; sourceLanguageCode.value = targetLanguageCode.value; targetLanguageCode.value = tempCode; - + // 保存语言设置 _saveLanguageSettings(); - + // 更新ASR支持的语言并重新初始化 _reinitializeAsrService(); } @@ -495,17 +583,17 @@ class TranslationController extends GetxController { void setSourceLanguage(String language) { if (sourceLanguage.value != language) { stopAll(); - + sourceLanguage.value = language; - + final asrCode = _languageManager.getAsrCodeByChineseName(language); if (asrCode != null) { sourceLanguageCode.value = asrCode; } - + // 保存语言设置 _saveLanguageSettings(); - + // 更新ASR支持的语言并重新初始化 _reinitializeAsrService(); } @@ -515,35 +603,38 @@ class TranslationController extends GetxController { void setTargetLanguage(String language) { if (targetLanguage.value != language) { stopAll(); - + targetLanguage.value = language; - + final asrCode = _languageManager.getAsrCodeByChineseName(language); if (asrCode != null) { targetLanguageCode.value = asrCode; } - + // 保存语言设置 _saveLanguageSettings(); - + // 更新ASR支持的语言并重新初始化 _reinitializeAsrService(); } } - + // 重新初始化ASR服务以支持新的语言 Future _reinitializeAsrService() async { try { // 更新ASR支持的语言 - final List asrSupportedLanguages = [sourceLanguageCode.value, targetLanguageCode.value]; - + final List asrSupportedLanguages = [ + sourceLanguageCode.value, + targetLanguageCode.value + ]; + Logger.info('重新初始化ASR服务，支持语言: $asrSupportedLanguages'); - + // 重新初始化ASR服务 await _asrService.initialize( supportedLanguages: asrSupportedLanguages, ); - + Logger.info('ASR服务重新初始化成功'); } catch (e) { Logger.error('重新初始化ASR服务失败: $e'); @@ -556,15 +647,17 @@ class TranslationController extends GetxController { if (historyJson != null) { try { final List historyList = jsonDecode(historyJson); - final List allHistory = historyList.map((item) => TranslationItem.fromJson(item)).toList(); - + final List allHistory = + historyList.map((item) => TranslationItem.fromJson(item)).toList(); + // 只保留最新的10条记录 if (allHistory.length > _maxHistoryItems) { - translationHistory.value = allHistory.sublist(allHistory.length - _maxHistoryItems); + translationHistory.value = + allHistory.sublist(allHistory.length - _maxHistoryItems); } else { translationHistory.value = allHistory; } - + // 使用微任务确保在视图完全渲染后滚动到底部 WidgetsBinding.instance.addPostFrameCallback((_) { _scrollToBottom(); @@ -579,11 +672,13 @@ class TranslationController extends GetxController { // 保存翻译历史 void saveTranslationHistory() { if (translationHistory.length > _maxHistoryItems) { - translationHistory.removeRange(0, translationHistory.length - _maxHistoryItems); + translationHistory.removeRange( + 0, translationHistory.length - _maxHistoryItems); } try { - final List> historyJson = translationHistory.map((item) => item.toJson()).toList(); + final List> historyJson = + translationHistory.map((item) => item.toJson()).toList(); _storage.write(_historyKey, jsonEncode(historyJson)); } catch (e) { Logger.error('保存翻译历史失败: $e'); @@ -644,7 +739,8 @@ class TranslationItem { sourceLanguageCode: json['sourceLanguageCode'] ?? 'zh-CN', targetLanguageCode: json['targetLanguageCode'] ?? 'en-US', timestamp: timestamp, - sessionId: json['sessionId'] ?? DateTime.now().millisecondsSinceEpoch.toString(), + sessionId: + json['sessionId'] ?? DateTime.now().millisecondsSinceEpoch.toString(), isFirstInSession: json['isFirstInSession'] ?? false, isIntermediate: false, // 从存储加载的都是最终结果 ); @@ -662,16 +758,18 @@ class TranslationItem { // 不保存isIntermediate，因为存储的都是最终结果 }; } - + // 获取源语言的中文名称 String get sourceLanguageName { final languageManager = Get.find(); - return languageManager.getChineseNameByAsrCode(sourceLanguageCode) ?? '未知语言'; + return languageManager.getChineseNameByAsrCode(sourceLanguageCode) ?? + '未知语言'; } - + // 获取目标语言的中文名称 String get targetLanguageName { final languageManager = Get.find(); - return languageManager.getChineseNameByAsrCode(targetLanguageCode) ?? '未知语言'; + return languageManager.getChineseNameByAsrCode(targetLanguageCode) ?? + '未知语言'; } } diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt index 62ad5685c..26595d9cd 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt @@ -28,7 +28,7 @@ class AzureAsrHelper(private val context: Context) { private var supportedLanguages = arrayOf("zh-CN") // 是否使用回音消除 - 内部控制常量 - private val useEchoCancellation = false + private val useEchoCancellation = true // 自定义音频处理相关 private var customAudioProcessor: CustomAudioProcessor? = null @@ -36,6 +36,7 @@ class AzureAsrHelper(private val context: Context) { private var audioConfig: AudioConfig? = null // 公共方法：允许外部写入音频数据 fun pushAudioData(data: ByteArray) { + FileLogger.d(TAG, "pushAudioData,data=${data},pushStream=${pushStream}") pushStream?.write(data) // 使用 SDK 的正确方法（如 push） } @@ -85,6 +86,7 @@ class AzureAsrHelper(private val context: Context) { // 创建识别器 try { if (useEchoCancellation) { + FileLogger.d(TAG, "如果使用回音消除，创建自定义音频输入流") // 如果使用回音消除，创建自定义音频输入流 setupCustomAudioProcessing() @@ -95,13 +97,13 @@ class AzureAsrHelper(private val context: Context) { recognizer = SpeechRecognizer(speechConfig, audioConfig) } } else { - // 使用默认麦克风输入 - if (isAutoDetectLanguage) { - val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) - recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) - } else { - recognizer = SpeechRecognizer(speechConfig) - } + // // 使用默认麦克风输入 + // if (isAutoDetectLanguage) { + // val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) + // recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) + // } else { + // recognizer = SpeechRecognizer(speechConfig) + // } } FileLogger.d(TAG, "Azure 语音服务初始化成功") @@ -140,13 +142,13 @@ class AzureAsrHelper(private val context: Context) { recognizer = SpeechRecognizer(speechConfig, audioConfig) } } else { - // 使用默认麦克风输入 - if (isAutoDetectLanguage) { - val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) - recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) - } else { - recognizer = SpeechRecognizer(speechConfig) - } + // // 使用默认麦克风输入 + // if (isAutoDetectLanguage) { + // val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) + // recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) + // } else { + // recognizer = SpeechRecognizer(speechConfig) + // } } return true } else { diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt index 090a72efc..e809403ea 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt @@ -4,6 +4,7 @@ import android.content.Context import android.app.Activity import android.os.Handler import android.os.Looper +import android.util.Log import androidx.annotation.NonNull import com.yunqiinnovation.azure_speech.utils.FileLogger @@ -91,6 +92,21 @@ class AzureSpeechPlugin: FlutterPlugin { } catch (e: Exception) { result.error("INITIALIZATION_ERROR", e.message, null) } + } + "pushAudioData" -> { + + val audioData = call.argument("AudioData") + +try { + Log.d("","AsrMethodHandler,pushAudioData${audioData}") + if (audioData != null) { + + val success = azureAsrHelper.pushAudioData(audioData) + result.success(success) + } +} catch (e: Exception) { + result.error("pushAudioData_ERROR", e.message, null) +} } "recognizeOnce" -> { azureAsrHelper.recognizeOnce(object : AzureAsrHelper.RecognizeCallback { diff --git a/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt b/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt index e2f474564..f09c69f50 100644 --- a/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt +++ b/local_plugins/jieli/android/src/main/kotlin/com/yunqiinnovation/jieli/JieliPlugin.kt @@ -92,7 +92,7 @@ class JieliPlugin: FlutterPlugin, MethodCallHandler, ActivityAware, EventChannel override fun onAudioDataReceived(data: ByteArray) { pluginScope.launch(Dispatchers.Main) { - Log.d("","数据流onAudioDataReceived${data}") + // Log.d("","数据流onAudioDataReceived${data}") sendEvent(EVENT_TYPE_AUDIO_DATA, data) } } diff --git a/local_plugins/jieli/lib/jieli.dart b/local_plugins/jieli/lib/jieli.dart index 50e7d99c2..d0d318627 100644 --- a/local_plugins/jieli/lib/jieli.dart +++ b/local_plugins/jieli/lib/jieli.dart @@ -127,7 +127,7 @@ class Jieli { break; case EventType.audioData: if (data is Uint8List) { - print('收到ble通道消息: $data'); + //print('收到ble通道消息: $data'); _audioController.add(data); } break; diff --git a/local_plugins/jl_opus/android/src/main/kotlin/com/yunqiinnovation/jl_opus/JlOpusPlugin.kt b/local_plugins/jl_opus/android/src/main/kotlin/com/yunqiinnovation/jl_opus/JlOpusPlugin.kt index bf47b2917..e8c6db4c9 100644 --- a/local_plugins/jl_opus/android/src/main/kotlin/com/yunqiinnovation/jl_opus/JlOpusPlugin.kt +++ b/local_plugins/jl_opus/android/src/main/kotlin/com/yunqiinnovation/jl_opus/JlOpusPlugin.kt @@ -324,7 +324,7 @@ class JlOpusPlugin: FlutterPlugin, MethodCallHandler { opusManager?.startDecodeStream(option, object : OnDecodeStreamCallback { override fun onDecodeStream(data: ByteArray?) { if (data != null) { - Log.d(TAG, "onDecodeStream${data}") + //Log.d(TAG, "onDecodeStream${data}") callback.onAudioDataReceived1(data) sendEventOnMainThread(mapOf( "event" to "onDecodeStream",