From 460963378bf2d4e1e10a504fd7f27d94060e79cf Mon Sep 17 00:00:00 2001 From: fdp <1286779656@qq.com> Date: Wed, 9 Jul 2025 16:59:08 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=BC=9A=E8=AE=AE?= =?UTF-8?q?=E5=8A=A9=E6=89=8B=E5=85=88=E5=BC=80=E5=BD=95=E9=9F=B3=E5=86=8D?= =?UTF-8?q?=E8=BD=AC=E6=96=87=E6=9C=AC=E4=B8=8D=E7=94=9F=E6=95=88=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../meeting/controllers/meeting_record_controller.dart | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/modules/meeting/controllers/meeting_record_controller.dart b/lib/modules/meeting/controllers/meeting_record_controller.dart index 189422b40..8550ba0ab 100644 --- a/lib/modules/meeting/controllers/meeting_record_controller.dart +++ b/lib/modules/meeting/controllers/meeting_record_controller.dart @@ -189,6 +189,8 @@ class MeetingRecordController extends GetxController _recognitionSubscription?.cancel(); _recognitionSubscription = recognitionStream.listen(_handleRecognitionEvent); + + await _asrService.startContinuousRecognition(_audioSourceType); showTextContent.toggle(); } @@ -278,8 +280,7 @@ class MeetingRecordController extends GetxController /// Resumes paused recording session Future _resumeRecording() async { - final recognitionStream = - await _asrService.startContinuousRecognition(_audioSourceType); + await _asrService.startContinuousRecognition(_audioSourceType); startTimer(); } From 67572565a2fcb6860759cd648a028bbe39c4a1c1 Mon Sep 17 00:00:00 2001 From: fdp <1286779656@qq.com> Date: Wed, 9 Jul 2025 20:04:56 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E8=8E=B7=E5=8F=96=E5=9B=BA=E4=BB=B6?= =?UTF-8?q?=E7=89=88=E6=9C=ACapi=E6=8E=A5=E5=8F=A3=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/ota_upgrade_controller.dart | 12 +++++++----- .../ota_upgrade/model/ota_upgrade_model.dart | 18 +++++++++++------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/lib/modules/ota_upgrade/controllers/ota_upgrade_controller.dart b/lib/modules/ota_upgrade/controllers/ota_upgrade_controller.dart index be56ed535..0df0f9d40 100644 --- a/lib/modules/ota_upgrade/controllers/ota_upgrade_controller.dart +++ b/lib/modules/ota_upgrade/controllers/ota_upgrade_controller.dart @@ -63,7 +63,7 @@ class OtaUpgradeController extends GetxController { final RxDouble progress = 0.0.obs; // 控制器自己的可观察属性 final RxInt type = OtaEvent.NOT_STARTED.obs; final RxBool isUpgrade = false.obs; - + // 新增：详细状态提示 final RxString detailedStatus = ''.obs; final RxBool showDetailedStatus = false.obs; @@ -193,7 +193,7 @@ class OtaUpgradeController extends GetxController { }); // 处理API响应 - if (response['info'] == null) { + if (response['product'] == null) { throw const FormatException('API 响应格式错误'); } @@ -295,7 +295,7 @@ class OtaUpgradeController extends GetxController { isDownloading.value = true; showDetailedStatus.value = true; detailedStatus.value = '准备下载固件包...'; - + int lastIndex = updatepackageaddress.lastIndexOf('/'); String fileName = updatepackageaddress.substring(lastIndex + 1); String directoryName = "ota_upgrades"; @@ -382,7 +382,8 @@ class OtaUpgradeController extends GetxController { onReceiveProgress: (received, total) { if (total != -1) { downloadProgress.value = received / total; - detailedStatus.value = '正在下载固件包... ${(received / total * 100).toStringAsFixed(1)}%'; + detailedStatus.value = + '正在下载固件包... ${(received / total * 100).toStringAsFixed(1)}%'; } }, ); @@ -491,7 +492,8 @@ class OtaUpgradeController extends GetxController { progress.value = event.progress!; type.value = event.type!; isUpgrade.value = true; // 开始升级 - detailedStatus.value = '升级进度: ${(event.progress! * 100).toStringAsFixed(1)}%'; + detailedStatus.value = + '升级进度: ${(event.progress! * 100).toStringAsFixed(1)}%'; } if (event.event == 'onStartOTA') { type.value = START_UPGRADING; diff --git a/lib/modules/ota_upgrade/model/ota_upgrade_model.dart b/lib/modules/ota_upgrade/model/ota_upgrade_model.dart index 4e02b2b9f..d07dbe386 100644 --- a/lib/modules/ota_upgrade/model/ota_upgrade_model.dart +++ b/lib/modules/ota_upgrade/model/ota_upgrade_model.dart @@ -12,8 +12,9 @@ class OtaConfigResp { // 最终属性（不可变） final String? id; - final String? factory; - final String? equipmentname; + final int? factory; + final int? devicetype; + final String? devicename; final String? systemversion; final String? updatepackageaddress; final String? updatedescription; @@ -21,7 +22,8 @@ class OtaConfigResp { OtaConfigResp._({ this.id, this.factory, - this.equipmentname, + this.devicetype, + this.devicename, this.systemversion, this.updatepackageaddress, this.updatedescription, @@ -30,13 +32,14 @@ class OtaConfigResp { factory OtaConfigResp.fromJson(Map json) { print("数据获取fronTokenJson${json}"); // 修复1：安全获取嵌套的 user 字段 - final infoData = json['info'] as Map?; // 允许空值 + final infoData = json['product'] as Map?; // 允许空值 // 更新单例实例 return _instance = OtaConfigResp._( id: (infoData?['id'] as String?) ?? '', - factory: (infoData?['factory'] as String?) ?? '', - equipmentname: (infoData?['equipmentname'] as String?) ?? '', + factory: (infoData?['factoryid'] as int?) ?? 0, + devicetype: (infoData?['devicetype'] as int?) ?? 0, + devicename: (infoData?['devicename'] as String?) ?? '', systemversion: (infoData?['systemversion'] as String?) ?? '', updatepackageaddress: (infoData?['updatepackageaddress'] as String?) ?? '', @@ -49,7 +52,8 @@ class OtaConfigResp { 'user': { 'id': instance.id, 'factory': instance.factory, - 'equipmentname': instance.equipmentname, + 'devicetype': instance.devicetype, + 'devicename': instance.devicename, 'systemversion': instance.systemversion, 'updatepackageaddress': instance.updatepackageaddress, 'updatedescription': instance.updatedescription, From ddad5c07c8b95e9d76c4da69a87a1e6122c1b83c Mon Sep 17 00:00:00 2001 From: fdp <1286779656@qq.com> Date: Thu, 10 Jul 2025 11:00:05 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E9=9F=B3=E9=A2=91?= =?UTF-8?q?=E6=94=BE=E5=9B=9Eflutter,=E5=8A=A0=E5=85=A5=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E9=87=8D=E7=BD=AE=E8=AF=86=E5=88=AB=E5=99=A8=E6=9C=BA=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/data/services/asr_service.dart | 3 + .../speech_impl/azure_asr_service.dart | 8 +- .../meeting_record_controller.dart | 90 +++++++++++- .../meeting/views/meeting_record_view.dart | 111 +++++++++------ .../controllers/speech_test_controller.dart | 3 + .../agent_service/AgentService.kt | 73 +++++----- .../azure_speech/AzureAsrHelper.kt | 70 +++++---- .../azure_speech/AzureSpeechPlugin.kt | 133 +++++++++++------- .../yunqiinnovation/ble_service/BleService.kt | 89 ++++++------ 9 files changed, 373 insertions(+), 207 deletions(-) diff --git a/lib/data/services/asr_service.dart b/lib/data/services/asr_service.dart index 5d1a616d3..e6c67563c 100644 --- a/lib/data/services/asr_service.dart +++ b/lib/data/services/asr_service.dart @@ -67,6 +67,9 @@ enum RecognitionEventType { /// 中间识别结果（实时反馈） intermediateResult, + /// 音频 + onAudio, + /// 会话开始 sessionStarted, diff --git a/lib/data/services/speech_impl/azure_asr_service.dart b/lib/data/services/speech_impl/azure_asr_service.dart index 9d322421f..c853d98f1 100644 --- a/lib/data/services/speech_impl/azure_asr_service.dart +++ b/lib/data/services/speech_impl/azure_asr_service.dart @@ -253,7 +253,13 @@ class AzureAsrService extends GetxService implements AsrService { detectedLanguage: detectedLanguage, )); break; - + case 'onAudio': + final Uint8List data = eventMap['data'] as Uint8List? ?? Uint8List(0); + _eventStreamController?.add(RecognitionEvent( + type: RecognitionEventType.onAudio, + audio: data, + )); + break; case 'sessionStarted': _eventStreamController?.add(RecognitionEvent( type: RecognitionEventType.sessionStarted, diff --git a/lib/modules/meeting/controllers/meeting_record_controller.dart b/lib/modules/meeting/controllers/meeting_record_controller.dart index 8550ba0ab..dc3b33a2e 100644 --- a/lib/modules/meeting/controllers/meeting_record_controller.dart +++ b/lib/modules/meeting/controllers/meeting_record_controller.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:io'; import 'dart:math'; +import 'dart:typed_data'; import 'package:deep_voice/core/utils/logger.dart'; import 'package:deep_voice/core/utils/permission_util.dart'; import 'package:flutter/material.dart'; @@ -56,10 +57,20 @@ class MeetingRecordController extends GetxController Timer? _timer; int _elapsedMilliseconds = 0; int? _lastStartTime; - int _currentPosition = 0; bool _audioSourceType = false; late TabController tabController; + // 添加波形放大倍数变量 + final double waveAmplifyFactor = 100.0; // 增加波动幅度 + final RxDouble ursorPosition = 0.0.obs; // 当前播放位置 + final RxInt _currentPosition = 0.obs; // 原始位置索引 + int maxPoints = 500; // 最大显示点数（根据屏幕宽度优化） + int sampleRate = 16000; // 假设音频采样率（根据实际情况调整） + // 设置最大点数（根据屏幕宽度） + void setMaxPoints(double screenWidth) { + maxPoints = screenWidth ~/ 2; // 每2像素一个数据点 + } + @override void onInit() { super.onInit(); @@ -291,6 +302,10 @@ class MeetingRecordController extends GetxController case RecognitionEventType.intermediateResult: intermediateContent.value = event.text; break; + case RecognitionEventType.onAudio: + // 添加新音频数据到波形 + _processAudioData(event.audio!); + break; case RecognitionEventType.finalResult: finalContent.value += _formatTranscript(event); intermediateContent.value = ''; @@ -300,6 +315,79 @@ class MeetingRecordController extends GetxController } } + /// 处理原始Uint8List音频数据 + void _processAudioData(Uint8List audioData) { + // 将Uint8List转换为Float32List（PCM16 -> 浮点） + final pcmData = _convertToPCM(audioData); + + // 降采样和添加数据 + _addAudioData(pcmData); + } + + /// 转换Uint8List为PCM浮点数组 + Float32List _convertToPCM(Uint8List uint8List) { + // 假设音频为16位PCM小端格式 + final byteData = ByteData.sublistView(uint8List); + final length = uint8List.length ~/ 2; + final pcmData = Float32List(length); + + for (int i = 0; i < length; i++) { + // 读取16位有符号整数 (-32768 到 32767) + final int intValue = byteData.getInt16(i * 2, Endian.little); + // 转换为浮点数 (-1.0 到 1.0) + pcmData[i] = intValue / 32768.0; + } + + return pcmData; + } + + /// 降采样音频数据 + List _downsampleAudio(Float32List audio) { + if (audio.isEmpty) return []; + + final result = []; + final step = (audio.length / maxPoints).ceil(); + + // 计算每个区间的峰值（保留更多细节） + for (int i = 0; i < audio.length; i += step) { + double maxVal = 0.0; + final end = min(i + step, audio.length); + + for (int j = i; j < end; j++) { + final absVal = audio[j].abs(); + if (absVal > maxVal) maxVal = absVal; + } + + // 应用放大倍数增强波动效果 + result.add(maxVal * waveAmplifyFactor); + } + + return result; + } + + // 获取当前光标位置（0.0 - 1.0） + double get cursorPositionValue { + if (waveData.isEmpty) return 0.0; + return _currentPosition.value / waveData.length; + } + + /// 添加处理后的音频数据 + void _addAudioData(Float32List newAudio) { + // 1. 降采样：只保留部分数据点 + final downsampled = _downsampleAudio(newAudio); + + waveData.addAll(downsampled); + + // 3. 更新光标位置到最新点 + _currentPosition.value = waveData.length; + + // 4. 限制数据长度，移除旧数据 + if (waveData.length > maxPoints * 2) { + waveData.removeRange(0, waveData.length - maxPoints); + _currentPosition.value = waveData.length; + } + } + /// Formats transcript with speaker identification String _formatTranscript(RecognitionEvent event) { final buffer = StringBuffer(); diff --git a/lib/modules/meeting/views/meeting_record_view.dart b/lib/modules/meeting/views/meeting_record_view.dart index 10d288604..fab9cdf61 100644 --- a/lib/modules/meeting/views/meeting_record_view.dart +++ b/lib/modules/meeting/views/meeting_record_view.dart @@ -1,3 +1,5 @@ +import 'dart:math'; + import 'package:flutter/material.dart'; import 'package:flutter_screenutil/flutter_screenutil.dart'; import 'package:get/get.dart'; @@ -112,43 +114,53 @@ class MeetingRecordView extends GetView { } Widget _buildAudioTab(BuildContext context, bool isDarkMode) { - return Obx(() => Column( - children: [ - const SizedBox(height: 24), - SizedBox( - height: 120, - child: Stack( - alignment: Alignment.center, - children: [ - CustomPaint( - painter: WaveformPainter( - controller.waveData, - cursorPosition: controller.cursorPosition, - ), - size: Size.infinite, + return Obx(() { + final screenWidth = MediaQuery.of(context).size.width; + // 初始化最大点数 + if (controller.maxPoints == 0) { + controller.setMaxPoints(screenWidth); + } + final cursorPos = controller.cursorPositionValue; + return Column( + children: [ + const SizedBox(height: 24), + SizedBox( + height: 120, + child: Stack( + alignment: Alignment.center, + children: [ + // 波形绘制 + CustomPaint( + painter: WaveformPainter( + controller.waveData, + cursorPosition: cursorPos, + // color: isDarkMode ? AppColors.darkBackground : Colors.grey[50], ), - Positioned( - left: MediaQuery.of(context).size.width * - controller.cursorPosition, - child: Container( - width: 2, - height: 120, - color: Colors.red, - ), + size: Size(screenWidth, 120), + ), + // 光标指示器 + Positioned( + left: MediaQuery.of(context).size.width * cursorPos, + child: Container( + width: 2, + height: 120, + color: Colors.red, ), - ], - ), - ), - const SizedBox(height: 16), - Text( - controller.formatTime(controller.seconds.value), - style: const TextStyle(fontSize: 48, fontWeight: FontWeight.bold), + ), + ], ), - const Spacer(), - _buildBottomBar(context, isDarkMode), - const SizedBox(height: 32), - ], - )); + ), + const SizedBox(height: 16), + Text( + controller.formatTime(controller.seconds.value), + style: const TextStyle(fontSize: 48, fontWeight: FontWeight.bold), + ), + const Spacer(), + _buildBottomBar(context, isDarkMode), + const SizedBox(height: 32), + ], + ); + }); } Widget _buildTextTab(BuildContext context, bool isDarkMode) { @@ -394,16 +406,33 @@ class WaveformPainter extends CustomPainter { ..color = Colors.red ..strokeWidth = 2; - final double barWidth = size.width / data.length; + final double barWidth = size.width / (data.length * 1.2); // 增加条宽 final centerY = size.height / 2; + // 改为绘制矩形条，增加视觉冲击力 for (int i = 0; i < data.length; i++) { - final x = i * barWidth + barWidth / 2; - final height = data[i]; - canvas.drawLine( - Offset(x, centerY - height), - Offset(x, centerY + height), - paint, + final x = i * barWidth; + final height = min(data[i], 50); // 限制最大高度 + final rect = Rect.fromLTWH( + x, + centerY - height / 2, + barWidth * 0.8, // 条宽稍小于间隔 + height.toDouble(), + ); + + // 添加渐变效果增强视觉 + final gradient = LinearGradient( + colors: [Colors.blue[400]!, Colors.blue[200]!], + begin: Alignment.topCenter, + end: Alignment.bottomCenter, + ); + final rectPaint = Paint() + ..shader = gradient.createShader(rect) + ..style = PaintingStyle.fill; + + canvas.drawRRect( + RRect.fromRectAndRadius(rect, const Radius.circular(4)), + rectPaint, ); } diff --git a/lib/modules/speech_test/controllers/speech_test_controller.dart b/lib/modules/speech_test/controllers/speech_test_controller.dart index 541ef58e7..ec1833c90 100644 --- a/lib/modules/speech_test/controllers/speech_test_controller.dart +++ b/lib/modules/speech_test/controllers/speech_test_controller.dart @@ -176,6 +176,9 @@ class SpeechTestController extends GetxController { // 清空中间结果 intermediateResult.value = ''; break; + case RecognitionEventType.onAudio: + // TODO: Handle this case. + break; } }, onError: (error) { isListening.value = false; diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index 113ae86cb..dcc5adb38 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -112,7 +112,7 @@ object AgentService : CoroutineScope { private var lastInterruptTime = 0L private val INTERRUPT_DEBOUNCE_MS = 300L - var audioSourceType = AzureAsrHelper.AudioSourceType.MICROPHONE + var audioSourceType = AzureAsrHelper.AudioSourceType.MICROPHONE // 语气词过滤列表（中英文） private val FILLER_WORDS = setOf( @@ -177,24 +177,24 @@ object AgentService : CoroutineScope { } // 初始化Azure ASR azureAsrHelper = AzureAsrHelper(context).apply { - // 使用协程处理异步初始化 - CoroutineScope(Dispatchers.IO).launch { - val asrInitSuccess = initialize( - subscriptionKey = config["azureSpeechKey"]?.toString() ?: "", - region = config["azureSpeechRegion"]?.toString() ?: "", - audioSourceType = if (isExternalActive) AzureAsrHelper.AudioSourceType.EXTERNAL - else AzureAsrHelper.AudioSourceType.MICROPHONE - ) - - if (asrInitSuccess) { - withContext(Dispatchers.Main) { - recognizeCallback() + // 使用协程处理异步初始化 + CoroutineScope(Dispatchers.IO).launch { + val asrInitSuccess = initialize( + subscriptionKey = config["azureSpeechKey"]?.toString() ?: "", + region = config["azureSpeechRegion"]?.toString() ?: "", + audioSourceType = if (isExternalActive) AzureAsrHelper.AudioSourceType.EXTERNAL + else AzureAsrHelper.AudioSourceType.MICROPHONE + ) + + if (asrInitSuccess) { + withContext(Dispatchers.Main) { + recognizeCallback() + } + } else { + Log.w(TAG, "Azure ASR初始化失败") + } + } } - } else { - Log.w(TAG, "Azure ASR初始化失败") - } - } - } // 初始化TTS服务 initializeTtsEngine(config) @@ -207,9 +207,8 @@ object AgentService : CoroutineScope { config["openaivisionModel"]?.toString() ?: "", config["mcpServer"]?.toString() ?: "" ) - - + // 加载最近的聊天记录 loadChatHistory() @@ -430,14 +429,14 @@ object AgentService : CoroutineScope { Log.d(TAG, "服务未初始化") return false } - Log.d("TAG", "开始语音识别startRecognition?") - stopRecognition() - BleService.openEncoder() + Log.d("TAG", "开始语音识别startRecognition?") + stopRecognition() + BleService.openEncoder() // if (isRecognitionActive&&audioSourceType==AzureAsrHelper.AudioSourceType.MICROPHONE){ // Log.d(TAG, "开始语音识别stopRecognition") // // BleService.closeCodec() // stopRecognition() - + // } // if((isExternalActive&&audioSourceType==AzureAsrHelper.AudioSourceType.MICROPHONE)||!isRecognitionActive) // { @@ -466,9 +465,9 @@ object AgentService : CoroutineScope { } else { AzureAsrHelper.AudioSourceType.MICROPHONE } - Log.d("TAG", "azureAsrHelper?.startContinuousRecognition") + Log.d("TAG", "azureAsrHelper?.startContinuousRecognition") azureAsrHelper?.startContinuousRecognition(audioSourceType) - + return true } catch (e: Exception) { _isRecognitionActive.set(false) @@ -485,7 +484,7 @@ object AgentService : CoroutineScope { fun recognizeCallback(): Boolean { try { - Log.d(TAG, "recognizeCallback") + Log.d(TAG, "recognizeCallback") val success = azureAsrHelper?.setupEventListeners(object : AzureAsrHelper.ContinuousRecognizeCallback { override fun onRecognizing(recognizing: String, detectedLanguage: String) { @@ -560,6 +559,10 @@ object AgentService : CoroutineScope { } + override fun onAudio(data: ByteArray) { + + } + override fun onSessionStopped() { sendEvent("recognition_stopped", mapOf("status" to "stopped")) _isRecognitionActive.set(false) @@ -570,8 +573,7 @@ object AgentService : CoroutineScope { override fun onCanceled(reason: String, errorDetails: String) { _isRecognitionActive.set(false) stopIdleCheck() - Log.d(TAG, "recognition_canceled:") - recognizeCallback() + Log.d(TAG, "recognition_canceled:") BleService.closeCodec() sendEvent( "recognition_canceled", mapOf( @@ -579,12 +581,12 @@ object AgentService : CoroutineScope { "details" to errorDetails ) ) + recognizeCallback() } override fun onError(error: String) { _isRecognitionActive.set(false) stopIdleCheck() -recognizeCallback() BleService.closeCodec() Log.d(TAG, "语音识别错误: $error") sendEvent( @@ -593,6 +595,7 @@ recognizeCallback() "message" to error ) ) + recognizeCallback() } }) return true @@ -609,19 +612,19 @@ recognizeCallback() * 停止语音识别 */ fun stopRecognition() { - if (!isRecognitionActive) return + if (!isRecognitionActive) return try { azureAsrHelper?.stopContinuousRecognition() BleService.closeCodec() _isRecognitionActive.set(false) stopIdleCheck() - return + return } catch (e: Exception) { Log.e(TAG, "停止语音识别异常: ${e.message}") _isRecognitionActive.set(false) stopIdleCheck() - return + return } } @@ -857,7 +860,7 @@ recognizeCallback() override fun onComplete(sessionid: String) { try { // 视情况决定是否朗读回复 - if (speakResponse && reply && broadcast && sessionid == currsessionId){ + if (speakResponse && reply && broadcast && sessionid == currsessionId) { ttsService?.flushStream() audioPlayer?.stopAudio() } @@ -870,7 +873,7 @@ recognizeCallback() if (hasImage) { responseData["hasImage"] = true } - if (reply){ + if (reply) { sendEvent("assistant_response", responseData) } diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt index 415bbae21..d5f98efca 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt @@ -38,6 +38,7 @@ class AzureAsrHelper(private val context: Context) { private var speechConfig: SpeechConfig? = null private var recognizer: SpeechRecognizer? = null private var audioConfig: AudioConfig? = null + private var continuousCallback: ContinuousRecognizeCallback? = null // 状态管理 private var isContinuousRecognitionActive = false @@ -145,22 +146,24 @@ class AzureAsrHelper(private val context: Context) { // 录音文件类 recordfile = RecordFile; - + return true } catch (e: Exception) { Log.e(tag, "初始化失败: ${e.message}") return false } } -private fun isRecognizerValid(): Boolean { - return recognizer != null -} + + private fun isRecognizerValid(): Boolean { + return recognizer != null + } + /** * 设置识别器 */ private fun setupRecognizer(): Boolean { try { - // 如果正在进行连续识别，先停止 + // 如果正在进行连续识别，先停止 if (isContinuousRecognitionActive) { // 直接停止，不等待结果 recognizer?.stopContinuousRecognitionAsync() @@ -192,7 +195,7 @@ private fun isRecognizerValid(): Boolean { private fun setupMicrophoneStream() { try { Log.d(tag, "设置麦克风流 - 使用拉流方式: }") - + if (audioStream == null) { // 创建外部音频拉流对象 audioStream = AudioStream() @@ -222,7 +225,7 @@ private fun isRecognizerValid(): Boolean { callback.onError("语音服务未初始化") return } - + this.audioSourceType = audioSourceType // 确保不在连续识别中 if (isContinuousRecognitionActive) { @@ -270,26 +273,26 @@ private fun isRecognizerValid(): Boolean { return true } if (!isRecognizerValid()) { - Log.w(tag, "识别器已失效，正在重新创建...") - if (!setupRecognizer()) { - Log.e(tag, "重新创建识别器失败") - return false + Log.w(tag, "识别器已失效，正在重新创建...") + if (!setupRecognizer()) { + Log.e(tag, "重新创建识别器失败") + return false + } } - } this.audioSourceType = audioSourceType try { - + Log.d(tag, "startContinuousRecognition: ") // 启动音频处理 //startAudioProcessing() // 开始连续识别 - + // 启动音频处理 audioStream!!.startAudioRecord() - recognizer?.startContinuousRecognitionAsync() + recognizer?.startContinuousRecognitionAsync() isContinuousRecognitionActive = true @@ -306,19 +309,20 @@ private fun isRecognizerValid(): Boolean { * 设置事件监听器 */ fun setupEventListeners(callback: ContinuousRecognizeCallback): Boolean { - Log.d(tag, "设置ssssss监听器:${speechConfig} ") + Log.d(tag, "设置ssssss监听器:${speechConfig} ") if (speechConfig == null) { callback.onError("语音服务未初始化") return false } - + // 重设识别器 if (!setupRecognizer()) { Log.d(tag, "初始化失败: ") return false } try { - Log.d(tag, "设置ssssss监听器: ") + continuousCallback = callback; + Log.d(tag, "设置ssssss监听器: ") // 优化：识别中事件 - 添加文本长度检查 recognizer?.recognizing?.addEventListener( EventHandler { _, event -> @@ -367,7 +371,7 @@ private fun isRecognizerValid(): Boolean { callback.onSessionStopped() isContinuousRecognitionActive = false - // stopAudioProcessing() + // stopAudioProcessing() } } ) @@ -378,10 +382,10 @@ private fun isRecognizerValid(): Boolean { val errorDetails = event.errorDetails ?: "未知错误" val reason = event.reason.toString() callback.onCanceled(reason, errorDetails) - + } ) - + } catch (e: Exception) { callback.onError("启动连续识别失败: ${e.message}") return false @@ -414,7 +418,7 @@ private fun isRecognizerValid(): Boolean { } Log.d(tag, "停止连续语音识别: ") // 直接停止连续识别（SDK内部已是异步操作） -recognizer?.stopContinuousRecognitionAsync()?.get(1000, TimeUnit.MILLISECONDS) + recognizer?.stopContinuousRecognitionAsync()?.get(1000, TimeUnit.MILLISECONDS) // 停止音频处理 audioStream!!.stopMicrophoneCapture() @@ -424,7 +428,7 @@ recognizer?.stopContinuousRecognitionAsync()?.get(1000, TimeUnit.MILLISECONDS) // 强制重置状态 isContinuousRecognitionActive = false Log.e(tag, "停止连续识别失败: ${e.message}") -recognizer?.stopContinuousRecognitionAsync() + recognizer?.stopContinuousRecognitionAsync() // 停止音频处理 audioStream!!.stopMicrophoneCapture() // 尝试强制关闭识别器 @@ -455,7 +459,7 @@ recognizer?.stopContinuousRecognitionAsync() recognizer?.stopContinuousRecognitionAsync() isContinuousRecognitionActive = false } - // recognizer?.stopContinuousRecognitionAsync()?.get(1000, TimeUnit.MILLISECONDS) + // recognizer?.stopContinuousRecognitionAsync()?.get(1000, TimeUnit.MILLISECONDS) // 停止音频处理 stopAudioProcessing() // 停止录音 @@ -499,7 +503,7 @@ recognizer?.stopContinuousRecognitionAsync() private fun stopAudioProcessing() { audioStream?.let { try { - Log.e(tag, "停止音频处理: }") + Log.e(tag, "停止音频处理: }") it.releaseAudioResources() it.pushAudioStream?.close() @@ -695,7 +699,7 @@ recognizer?.stopContinuousRecognitionAsync() // 情况2：不在录制但队列有数据 -> 从队列获取 else if (writeQueue.isNotEmpty()) { data = writeQueue.poll() - Log.d("tag", "写入数据: ${ data?.size}") + Log.d("tag", "写入数据: ${data?.size}") bytesToWrite = data?.size ?: 0 } @@ -706,6 +710,7 @@ recognizer?.stopContinuousRecognitionAsync() if (bytesToWrite < data.size) data.copyOf(bytesToWrite) else data try { + continuousCallback?.onAudio(finalData) pushAudioStream?.write(finalData) recordfile?.saveAudioDataToWav(finalData) } catch (e: Exception) { @@ -732,7 +737,7 @@ recognizer?.stopContinuousRecognitionAsync() * 外部音频输入 */ fun saveAudioDataTo(buffer: ByteArray) { - + if (audioSourceType == AudioSourceType.MICROPHONE) return // 放入队列，由写线程写入 writeQueue.offer(buffer.copyOf()) @@ -817,7 +822,7 @@ recognizer?.stopContinuousRecognitionAsync() try { // TODO: 实现外部音频源捕获逻辑 // 停止录音 if (audioRecord?.recordingState == AudioRecord.STATE_INITIALIZED) { - Log.d("TAG", "外部音频捕获启动 释放audioRecord") + Log.d("TAG", "外部音频捕获启动 释放audioRecord") audioRecord?.stop() // 释放录音实例 audioRecord?.release() @@ -826,7 +831,7 @@ recognizer?.stopContinuousRecognitionAsync() } catch (e: Exception) { Log.e("TAG", "外部音频捕获异常: ${e.message}") } finally { - + } } @@ -947,6 +952,13 @@ recognizer?.stopContinuousRecognitionAsync() */ fun onSessionStopped() + /** + * 返回音频 + * + * @param data 识别的音频 + */ + fun onAudio(data: ByteArray) + /** * 识别取消时调用 * diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt index c20214038..b415262e8 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt @@ -132,6 +132,83 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin { } } + //设置识别器 + private fun recognizeCallback(): Boolean { + try { + azureAsrHelper.setupEventListeners(object : + AzureAsrHelper.ContinuousRecognizeCallback { + override fun onResult(text: String, detectedLanguage: String) { + + sendAsrEvent( + mapOf( + "type" to "result", + "text" to text, + "detectedLanguage" to detectedLanguage + ) + ) + + } + + override fun onRecognizing( + recognizing: String, + detectedLanguage: String + ) { + + sendAsrEvent( + mapOf( + "type" to "recognizing", + "text" to recognizing, + "detectedLanguage" to detectedLanguage + ) + ) + } + + override fun onSessionStarted() { + sendAsrEvent(mapOf("type" to "sessionStarted")) + } + + override fun onSessionStopped() { + sendAsrEvent(mapOf("type" to "sessionStopped")) + } + + override fun onAudio(data: ByteArray) { + sendAsrEvent( + mapOf( + "type" to "onAudio", + "data" to data + ) + ) + } + + override fun onCanceled(reason: String, errorDetails: String) { + + sendAsrEvent( + mapOf( + "type" to "canceled", + "reason" to reason, + "errorDetails" to errorDetails + ) + ) + recognizeCallback() + + } + + override fun onError(error: String) { + sendAsrEvent( + mapOf( + "type" to "error", + "message" to error + ) + ) + recognizeCallback() + } + }) + return true + } catch (e: Exception) { + return false + } + } + /** * 设置TTS事件监听器 */ @@ -296,60 +373,8 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin { "recognizeCallback" -> { - val success = azureAsrHelper.setupEventListeners(object : - AzureAsrHelper.ContinuousRecognizeCallback { - override fun onResult(text: String, detectedLanguage: String) { - sendAsrEvent( - mapOf( - "type" to "result", - "text" to text, - "detectedLanguage" to detectedLanguage - ) - ) - } - - override fun onRecognizing( - recognizing: String, - detectedLanguage: String - ) { - sendAsrEvent( - mapOf( - "type" to "recognizing", - "text" to recognizing, - "detectedLanguage" to detectedLanguage - ) - ) - } - - override fun onSessionStarted() { - sendAsrEvent(mapOf("type" to "sessionStarted")) - } - - override fun onSessionStopped() { - sendAsrEvent(mapOf("type" to "sessionStopped")) - } - - override fun onCanceled(reason: String, errorDetails: String) { - - sendAsrEvent( - mapOf( - "type" to "canceled", - "reason" to reason, - "errorDetails" to errorDetails - ) - ) - } - - override fun onError(error: String) { - sendAsrEvent( - mapOf( - "type" to "error", - "message" to error - ) - ) - } - }) - FileLogger.d(tag, "recognizeCallback: ${success}") // + var success = recognizeCallback() + FileLogger.d(tag, "recognizeCallback: ${success}") // result.success(success) } diff --git a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt index 3a39d7237..af78e6d49 100644 --- a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt +++ b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt @@ -64,7 +64,8 @@ object BleService { // 上下文 private lateinit var context: Context -private val commandQueue: Queue = LinkedList() + private val commandQueue: Queue = LinkedList() + // 蓝牙相关组件 private lateinit var bluetoothManager: BluetoothManager private var bluetoothAdapter: BluetoothAdapter? = null @@ -75,7 +76,7 @@ private val commandQueue: Queue = LinkedList() private var notifyChar: BluetoothGattCharacteristic? = null private var writeChar: BluetoothGattCharacteristic? = null private var audioChar: BluetoothGattCharacteristic? = null - private var audioChar1: BluetoothGattCharacteristic? = null + private var audioChar1: BluetoothGattCharacteristic? = null var recordfile: RecordingFile? = null // 扫描相关 @@ -88,8 +89,10 @@ private val commandQueue: Queue = LinkedList() // 是否回复指令 var isReply = true + // 是否回复指令码 - var cmdReplyType =0xff + var cmdReplyType = 0xff + // 连接状态 val connectionState = MutableLiveData(BleConst.STATE_DISCONNECTED) @@ -198,12 +201,13 @@ private val commandQueue: Queue = LinkedList() bluetoothAdapter = manager.adapter // 重新获取新引用 bluetoothLeScanner = bluetoothAdapter?.bluetoothLeScanner } + /** * 开始主动扫描BLE设备 * @param timeout 扫描超时时间（毫秒） * @return 是否成功开始扫描 */ - fun startScan(deviceName:String,timeout: Long = 15000): Boolean { + fun startScan(deviceName: String, timeout: Long = 15000): Boolean { // Log.i(TAG, "liwei---------startScan 1 $deviceName") if (!isInitialized) { Log.e(TAG, "BLE服务未初始化") @@ -332,12 +336,12 @@ private val commandQueue: Queue = LinkedList() } } - fun getDeviceInfo(address: String): Map? { + fun getDeviceInfo(address: String): Map? { // 更新缓存 val idx = scanResults.indexOfFirst { it.device.address == address } Log.d(TAG, "获取设备信息:${address} idx:$idx") - if (idx >= 0){ + if (idx >= 0) { stopScan() val result = scanResults[idx]; return printDeviceInfo(result) @@ -348,7 +352,7 @@ private val commandQueue: Queue = LinkedList() /** * 打印设备信息和服务UUID */ - private fun printDeviceInfo(result: ScanResult):Map?{ + private fun printDeviceInfo(result: ScanResult): Map? { val device = result.device val scanRecord = result.scanRecord val serviceUuids = @@ -545,7 +549,7 @@ private val commandQueue: Queue = LinkedList() audioChar = audioSvc?.getCharacteristic(BleConst.RECEIVE_AUDIO_CHAR_UUID) - // 获取音频服务1特征 + // 获取音频服务1特征 val audioSvc1 = g.getService(BleConst.AUDIO_SERVICE_UUID1) audioChar1 = audioSvc1?.getCharacteristic(BleConst.RECEIVE_AUDIO_CHAR_UUID1) if (notifyChar == null || writeChar == null) { @@ -563,7 +567,7 @@ private val commandQueue: Queue = LinkedList() } else { Log.w(TAG, "音频服务特征未找到") } - if (audioChar1 != null) { + if (audioChar1 != null) { setupNotifications(g, audioChar1) Log.i(TAG, "音频服务特征找到并设置通知") } else { @@ -581,7 +585,7 @@ private val commandQueue: Queue = LinkedList() // 根据特征UUID区分处理 when (c.uuid) { - // 音频特征数据 + // 音频特征数据 // BleConst.RECEIVE_AUDIO_CHAR_UUID1 -> { // Log.i(TAG, "RECEIVE_AUDIO_CHAR_UUID1") @@ -667,7 +671,7 @@ private val commandQueue: Queue = LinkedList() private fun processAudioData(data: ByteArray) { try { if (opusManager?.isDecodeStream == true) { - + recordfile?.saveAudioDataToWav(data) opusManager?.writeAudioStream(data) } else { @@ -748,21 +752,19 @@ private val commandQueue: Queue = LinkedList() } }" ) -isReply=true; -cmdReplyType=cmdType; -if(!commandQueue.isEmpty()) -{ - val nextCmd = commandQueue.poll() - writeChar!!.value=nextCmd - var isSeccess = bluetoothGatt?.writeCharacteristic(writeChar) - if(isSeccess==true) - { - Log.d( - TAG, - "收到设备响应:" - ) - } -} + isReply = true; + cmdReplyType = cmdType; + if (!commandQueue.isEmpty()) { + val nextCmd = commandQueue.poll() + writeChar!!.value = nextCmd + var isSeccess = bluetoothGatt?.writeCharacteristic(writeChar) + if (isSeccess == true) { + Log.d( + TAG, + "收到设备响应:" + ) + } + } // 根据命令类型处理不同的响应 when (cmdType) { BleConst.CMD_GET_VERSION -> { @@ -1026,7 +1028,7 @@ if(!commandQueue.isEmpty()) BleConst.CMD_WAKE_UP -> { Log.i(TAG, "收到唤醒指令") // 接收到唤醒信号，打开编码器 (设备侧) - + notifyWakeupSignalReceived() } @@ -1380,26 +1382,21 @@ if(!commandQueue.isEmpty()) // 发送数据 writeChar!!.value = cmdPacket - // 加入队列并触发发送 - - if(isReply) - { - var isSeccess = bluetoothGatt?.writeCharacteristic(writeChar) - if(isSeccess == true) - { - isReply=false; - cmdType; - return true - } - else - return false - } - else - { + // 加入队列并触发发送 + + if (isReply) { + var isSeccess = bluetoothGatt?.writeCharacteristic(writeChar) + if (isSeccess == true) { + isReply = false; + cmdType; + return true + } else + return false + } else { commandQueue.offer(cmdPacket) - } - return true - + } + return true + } catch (e: Exception) { Log.e(TAG, "发送命令异常: ${e.message}", e) return false From 7e45cd68a0d954c05b83afd6c3fab990aef368c5 Mon Sep 17 00:00:00 2001 From: fdp <1286779656@qq.com> Date: Thu, 10 Jul 2025 11:18:39 +0800 Subject: [PATCH 4/4] add --- lib/modules/agent/controllers/agent_controller.dart | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/modules/agent/controllers/agent_controller.dart b/lib/modules/agent/controllers/agent_controller.dart index 29a95eeb1..c179f6c74 100644 --- a/lib/modules/agent/controllers/agent_controller.dart +++ b/lib/modules/agent/controllers/agent_controller.dart @@ -1029,11 +1029,11 @@ class AgentController extends GetxController { return; } - // try { - - // } catch (e) { - // Logger.e(TAG, '启动语音输入失败: $e'); - // } + try { + await AgentService.recognizeCallback(); + } catch (e) { + Logger.e(TAG, '启动识别器失败: $e'); + } } // 停止语音输入