加入音频放回flutter,加入异常重置识别器机制

lib/data/services/asr_service.dart

@@ -67,6 +67,9 @@ enum RecognitionEventType {
   /// 中间识别结果（实时反馈）
   intermediateResult,
 
+  /// 音频
+  onAudio,
+
   /// 会话开始
   sessionStarted,
 

lib/data/services/speech_impl/azure_asr_service.dart

@@ -253,7 +253,13 @@ class AzureAsrService extends GetxService implements AsrService {
           detectedLanguage: detectedLanguage,
         ));
         break;
-
+      case 'onAudio':
+        final Uint8List data = eventMap['data'] as Uint8List? ?? Uint8List(0);
+        _eventStreamController?.add(RecognitionEvent(
+          type: RecognitionEventType.onAudio,
+          audio: data,
+        ));
+        break;
       case 'sessionStarted':
         _eventStreamController?.add(RecognitionEvent(
           type: RecognitionEventType.sessionStarted,

lib/modules/meeting/controllers/meeting_record_controller.dart

@@ -2,6 +2,7 @@
 import 'dart:async';
 import 'dart:io';
 import 'dart:math';
+import 'dart:typed_data';
 import 'package:deep_voice/core/utils/logger.dart';
 import 'package:deep_voice/core/utils/permission_util.dart';
 import 'package:flutter/material.dart';
@@ -56,10 +57,20 @@ class MeetingRecordController extends GetxController
   Timer? _timer;
   int _elapsedMilliseconds = 0;
   int? _lastStartTime;
-  int _currentPosition = 0;
   bool _audioSourceType = false;
   late TabController tabController;
 
+  // 添加波形放大倍数变量
+  final double waveAmplifyFactor = 100.0; // 增加波动幅度
+  final RxDouble ursorPosition = 0.0.obs; // 当前播放位置
+  final RxInt _currentPosition = 0.obs; // 原始位置索引
+  int maxPoints = 500; // 最大显示点数（根据屏幕宽度优化）
+  int sampleRate = 16000; // 假设音频采样率（根据实际情况调整）
+  // 设置最大点数（根据屏幕宽度）
+  void setMaxPoints(double screenWidth) {
+    maxPoints = screenWidth ~/ 2; // 每2像素一个数据点
+  }
+
   @override
   void onInit() {
     super.onInit();
@@ -291,6 +302,10 @@ class MeetingRecordController extends GetxController
       case RecognitionEventType.intermediateResult:
         intermediateContent.value = event.text;
         break;
+      case RecognitionEventType.onAudio:
+        // 添加新音频数据到波形
+        _processAudioData(event.audio!);
+        break;
       case RecognitionEventType.finalResult:
         finalContent.value += _formatTranscript(event);
         intermediateContent.value = '';
@@ -300,6 +315,79 @@ class MeetingRecordController extends GetxController
     }
   }
 
+  /// 处理原始Uint8List音频数据
+  void _processAudioData(Uint8List audioData) {
+    // 将Uint8List转换为Float32List（PCM16 -> 浮点）
+    final pcmData = _convertToPCM(audioData);
+
+    // 降采样和添加数据
+    _addAudioData(pcmData);
+  }
+
+  /// 转换Uint8List为PCM浮点数组
+  Float32List _convertToPCM(Uint8List uint8List) {
+    // 假设音频为16位PCM小端格式
+    final byteData = ByteData.sublistView(uint8List);
+    final length = uint8List.length ~/ 2;
+    final pcmData = Float32List(length);
+
+    for (int i = 0; i < length; i++) {
+      // 读取16位有符号整数 (-32768 到 32767)
+      final int intValue = byteData.getInt16(i * 2, Endian.little);
+      // 转换为浮点数 (-1.0 到 1.0)
+      pcmData[i] = intValue / 32768.0;
+    }
+
+    return pcmData;
+  }
+
+  /// 降采样音频数据
+  List<double> _downsampleAudio(Float32List audio) {
+    if (audio.isEmpty) return [];
+
+    final result = <double>[];
+    final step = (audio.length / maxPoints).ceil();
+
+    // 计算每个区间的峰值（保留更多细节）
+    for (int i = 0; i < audio.length; i += step) {
+      double maxVal = 0.0;
+      final end = min(i + step, audio.length);
+
+      for (int j = i; j < end; j++) {
+        final absVal = audio[j].abs();
+        if (absVal > maxVal) maxVal = absVal;
+      }
+
+      // 应用放大倍数增强波动效果
+      result.add(maxVal * waveAmplifyFactor);
+    }
+
+    return result;
+  }
+
+  // 获取当前光标位置（0.0 - 1.0）
+  double get cursorPositionValue {
+    if (waveData.isEmpty) return 0.0;
+    return _currentPosition.value / waveData.length;
+  }
+
+  /// 添加处理后的音频数据
+  void _addAudioData(Float32List newAudio) {
+    // 1. 降采样：只保留部分数据点
+    final downsampled = _downsampleAudio(newAudio);
+
+    waveData.addAll(downsampled);
+
+    // 3. 更新光标位置到最新点
+    _currentPosition.value = waveData.length;
+
+    // 4. 限制数据长度，移除旧数据
+    if (waveData.length > maxPoints * 2) {
+      waveData.removeRange(0, waveData.length - maxPoints);
+      _currentPosition.value = waveData.length;
+    }
+  }
+
   /// Formats transcript with speaker identification
   String _formatTranscript(RecognitionEvent event) {
     final buffer = StringBuffer();

lib/modules/meeting/views/meeting_record_view.dart

@@ -1,3 +1,5 @@
+import 'dart:math';
+
 import 'package:flutter/material.dart';
 import 'package:flutter_screenutil/flutter_screenutil.dart';
 import 'package:get/get.dart';
@@ -112,7 +114,14 @@ class MeetingRecordView extends GetView<MeetingRecordController> {
   }
 
   Widget _buildAudioTab(BuildContext context, bool isDarkMode) {
-    return Obx(() => Column(
+    return Obx(() {
+      final screenWidth = MediaQuery.of(context).size.width;
+      // 初始化最大点数
+      if (controller.maxPoints == 0) {
+        controller.setMaxPoints(screenWidth);
+      }
+      final cursorPos = controller.cursorPositionValue;
+      return Column(
         children: [
           const SizedBox(height: 24),
           SizedBox(
@@ -120,16 +129,18 @@ class MeetingRecordView extends GetView<MeetingRecordController> {
             child: Stack(
               alignment: Alignment.center,
               children: [
+                // 波形绘制
                 CustomPaint(
                   painter: WaveformPainter(
                     controller.waveData,
-                      cursorPosition: controller.cursorPosition,
+                    cursorPosition: cursorPos,
+                    //   color: isDarkMode ? AppColors.darkBackground : Colors.grey[50],
                   ),
-                    size: Size.infinite,
+                  size: Size(screenWidth, 120),
                 ),
+                // 光标指示器
                 Positioned(
-                    left: MediaQuery.of(context).size.width *
+                  left: MediaQuery.of(context).size.width * cursorPos,
-                        controller.cursorPosition,
                   child: Container(
                     width: 2,
                     height: 120,
@@ -148,7 +159,8 @@ class MeetingRecordView extends GetView<MeetingRecordController> {
           _buildBottomBar(context, isDarkMode),
           const SizedBox(height: 32),
         ],
-        ));
+      );
+    });
   }
 
   Widget _buildTextTab(BuildContext context, bool isDarkMode) {
@@ -394,16 +406,33 @@ class WaveformPainter extends CustomPainter {
       ..color = Colors.red
       ..strokeWidth = 2;
 
-    final double barWidth = size.width / data.length;
+    final double barWidth = size.width / (data.length * 1.2); // 增加条宽
     final centerY = size.height / 2;
 
+    // 改为绘制矩形条，增加视觉冲击力
     for (int i = 0; i < data.length; i++) {
-      final x = i * barWidth + barWidth / 2;
+      final x = i * barWidth;
-      final height = data[i];
+      final height = min(data[i], 50); // 限制最大高度
-      canvas.drawLine(
+      final rect = Rect.fromLTWH(
-        Offset(x, centerY - height),
+        x,
-        Offset(x, centerY + height),
+        centerY - height / 2,
-        paint,
+        barWidth * 0.8, // 条宽稍小于间隔
+        height.toDouble(),
+      );
+
+      // 添加渐变效果增强视觉
+      final gradient = LinearGradient(
+        colors: [Colors.blue[400]!, Colors.blue[200]!],
+        begin: Alignment.topCenter,
+        end: Alignment.bottomCenter,
+      );
+      final rectPaint = Paint()
+        ..shader = gradient.createShader(rect)
+        ..style = PaintingStyle.fill;
+
+      canvas.drawRRect(
+        RRect.fromRectAndRadius(rect, const Radius.circular(4)),
+        rectPaint,
       );
     }
 

lib/modules/speech_test/controllers/speech_test_controller.dart

@@ -176,6 +176,9 @@ class SpeechTestController extends GetxController {
           // 清空中间结果
           intermediateResult.value = '';
           break;
+        case RecognitionEventType.onAudio:
+          // TODO: Handle this case.
+          break;
       }
     }, onError: (error) {
       isListening.value = false;

local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt

@@ -209,7 +209,6 @@ object AgentService : CoroutineScope {
             )
 
 
-            
             // 加载最近的聊天记录
             loadChatHistory()
 
@@ -560,6 +559,10 @@ object AgentService : CoroutineScope {
 
                 }
 
+                override fun onAudio(data: ByteArray) {
+
+                }
+
                 override fun onSessionStopped() {
                     sendEvent("recognition_stopped", mapOf("status" to "stopped"))
                     _isRecognitionActive.set(false)
@@ -571,7 +574,6 @@ object AgentService : CoroutineScope {
                     _isRecognitionActive.set(false)
                     stopIdleCheck()
                     Log.d(TAG, "recognition_canceled:")
-                    recognizeCallback()
                     BleService.closeCodec()
                     sendEvent(
                         "recognition_canceled", mapOf(
@@ -579,12 +581,12 @@ object AgentService : CoroutineScope {
                             "details" to errorDetails
                         )
                     )
+                    recognizeCallback()
                 }
 
                 override fun onError(error: String) {
                     _isRecognitionActive.set(false)
                     stopIdleCheck()
-recognizeCallback()
                     BleService.closeCodec()
                     Log.d(TAG, "语音识别错误: $error")
                     sendEvent(
@@ -593,6 +595,7 @@ recognizeCallback()
                             "message" to error
                         )
                     )
+                    recognizeCallback()
                 }
             })
             return true
@@ -857,7 +860,7 @@ recognizeCallback()
                         override fun onComplete(sessionid: String) {
                             try {
                                 // 视情况决定是否朗读回复
-                                if (speakResponse && reply && broadcast &&  sessionid == currsessionId){
+                                if (speakResponse && reply && broadcast && sessionid == currsessionId) {
                                     ttsService?.flushStream()
                                     audioPlayer?.stopAudio()
                                 }
@@ -870,7 +873,7 @@ recognizeCallback()
                                 if (hasImage) {
                                     responseData["hasImage"] = true
                                 }
-                                if (reply){
+                                if (reply) {
                                     sendEvent("assistant_response", responseData)
                                 }
 

local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt

@@ -38,6 +38,7 @@ class AzureAsrHelper(private val context: Context) {
     private var speechConfig: SpeechConfig? = null
     private var recognizer: SpeechRecognizer? = null
     private var audioConfig: AudioConfig? = null
+    private var continuousCallback: ContinuousRecognizeCallback? = null
 
     // 状态管理
     private var isContinuousRecognitionActive = false
@@ -152,9 +153,11 @@ class AzureAsrHelper(private val context: Context) {
             return false
         }
     }
-private fun isRecognizerValid(): Boolean {
+
+    private fun isRecognizerValid(): Boolean {
         return recognizer != null
-}
+    }
+
     /**
      * 设置识别器
      */
@@ -318,6 +321,7 @@ private fun isRecognizerValid(): Boolean {
             return false
         }
         try {
+            continuousCallback = callback;
             Log.d(tag, "设置ssssss监听器: ")
             // 优化：识别中事件 - 添加文本长度检查
             recognizer?.recognizing?.addEventListener(
@@ -414,7 +418,7 @@ private fun isRecognizerValid(): Boolean {
             }
             Log.d(tag, "停止连续语音识别: ")
             // 直接停止连续识别（SDK内部已是异步操作）
-recognizer?.stopContinuousRecognitionAsync()?.get(1000, TimeUnit.MILLISECONDS)
+            recognizer?.stopContinuousRecognitionAsync()?.get(1000, TimeUnit.MILLISECONDS)
 
             // 停止音频处理
             audioStream!!.stopMicrophoneCapture()
@@ -424,7 +428,7 @@ recognizer?.stopContinuousRecognitionAsync()?.get(1000, TimeUnit.MILLISECONDS)
             // 强制重置状态
             isContinuousRecognitionActive = false
             Log.e(tag, "停止连续识别失败: ${e.message}")
-recognizer?.stopContinuousRecognitionAsync()
+            recognizer?.stopContinuousRecognitionAsync()
             // 停止音频处理
             audioStream!!.stopMicrophoneCapture()
             // 尝试强制关闭识别器
@@ -695,7 +699,7 @@ recognizer?.stopContinuousRecognitionAsync()
                         // 情况2：不在录制但队列有数据 -> 从队列获取
                         else if (writeQueue.isNotEmpty()) {
                             data = writeQueue.poll()
-                               Log.d("tag", "写入数据: ${ data?.size}")
+                            Log.d("tag", "写入数据: ${data?.size}")
                             bytesToWrite = data?.size ?: 0
                         }
 
@@ -706,6 +710,7 @@ recognizer?.stopContinuousRecognitionAsync()
                                 if (bytesToWrite < data.size) data.copyOf(bytesToWrite) else data
 
                             try {
+                                continuousCallback?.onAudio(finalData)
                                 pushAudioStream?.write(finalData)
                                 recordfile?.saveAudioDataToWav(finalData)
                             } catch (e: Exception) {
@@ -947,6 +952,13 @@ recognizer?.stopContinuousRecognitionAsync()
          */
         fun onSessionStopped()
 
+        /**
+         * 返回音频
+         *
+         * @param data 识别的音频
+         */
+        fun onAudio(data: ByteArray)
+
         /**
          * 识别取消时调用
          *

local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt

@@ -132,6 +132,83 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
         }
     }
 
+    //设置识别器
+    private fun recognizeCallback(): Boolean {
+        try {
+            azureAsrHelper.setupEventListeners(object :
+                AzureAsrHelper.ContinuousRecognizeCallback {
+                override fun onResult(text: String, detectedLanguage: String) {
+
+                    sendAsrEvent(
+                        mapOf(
+                            "type" to "result",
+                            "text" to text,
+                            "detectedLanguage" to detectedLanguage
+                        )
+                    )
+
+                }
+
+                override fun onRecognizing(
+                    recognizing: String,
+                    detectedLanguage: String
+                ) {
+
+                    sendAsrEvent(
+                        mapOf(
+                            "type" to "recognizing",
+                            "text" to recognizing,
+                            "detectedLanguage" to detectedLanguage
+                        )
+                    )
+                }
+
+                override fun onSessionStarted() {
+                    sendAsrEvent(mapOf("type" to "sessionStarted"))
+                }
+
+                override fun onSessionStopped() {
+                    sendAsrEvent(mapOf("type" to "sessionStopped"))
+                }
+
+                override fun onAudio(data: ByteArray) {
+                    sendAsrEvent(
+                        mapOf(
+                            "type" to "onAudio",
+                            "data" to data
+                        )
+                    )
+                }
+
+                override fun onCanceled(reason: String, errorDetails: String) {
+
+                    sendAsrEvent(
+                        mapOf(
+                            "type" to "canceled",
+                            "reason" to reason,
+                            "errorDetails" to errorDetails
+                        )
+                    )
+                    recognizeCallback()
+
+                }
+
+                override fun onError(error: String) {
+                    sendAsrEvent(
+                        mapOf(
+                            "type" to "error",
+                            "message" to error
+                        )
+                    )
+                    recognizeCallback()
+                }
+            })
+            return true
+        } catch (e: Exception) {
+            return false
+        }
+    }
+
     /**
      * 设置TTS事件监听器
      */
@@ -296,59 +373,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
 
                 "recognizeCallback" -> {
 
-                    val success = azureAsrHelper.setupEventListeners(object :
+                    var success = recognizeCallback()
-                        AzureAsrHelper.ContinuousRecognizeCallback {
-                        override fun onResult(text: String, detectedLanguage: String) {
-                            sendAsrEvent(
-                                mapOf(
-                                    "type" to "result",
-                                    "text" to text,
-                                    "detectedLanguage" to detectedLanguage
-                                )
-                            )
-                        }
-
-                        override fun onRecognizing(
-                            recognizing: String,
-                            detectedLanguage: String
-                        ) {
-                            sendAsrEvent(
-                                mapOf(
-                                    "type" to "recognizing",
-                                    "text" to recognizing,
-                                    "detectedLanguage" to detectedLanguage
-                                )
-                            )
-                        }
-
-                        override fun onSessionStarted() {
-                            sendAsrEvent(mapOf("type" to "sessionStarted"))
-                        }
-
-                        override fun onSessionStopped() {
-                            sendAsrEvent(mapOf("type" to "sessionStopped"))
-                        }
-
-                        override fun onCanceled(reason: String, errorDetails: String) {
-  
-                            sendAsrEvent(
-                                mapOf(
-                                    "type" to "canceled",
-                                    "reason" to reason,
-                                    "errorDetails" to errorDetails
-                                )
-                            )
-                        }
-
-                        override fun onError(error: String) {
-                            sendAsrEvent(
-                                mapOf(
-                                    "type" to "error",
-                                    "message" to error
-                                )
-                            )
-                        }
-                    })
                     FileLogger.d(tag, "recognizeCallback: ${success}")  //
                     result.success(success)
                 }

local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt

@@ -64,7 +64,8 @@ object BleService {
 
     // 上下文
     private lateinit var context: Context
-private val commandQueue: Queue<ByteArray> = LinkedList()
+    private val commandQueue: Queue<ByteArray> = LinkedList()
+
     // 蓝牙相关组件
     private lateinit var bluetoothManager: BluetoothManager
     private var bluetoothAdapter: BluetoothAdapter? = null
@@ -88,8 +89,10 @@ private val commandQueue: Queue<ByteArray> = LinkedList()
 
     // 是否回复指令
     var isReply = true
+
     // 是否回复指令码
-    var cmdReplyType =0xff
+    var cmdReplyType = 0xff
+
     // 连接状态
     val connectionState = MutableLiveData(BleConst.STATE_DISCONNECTED)
 
@@ -198,12 +201,13 @@ private val commandQueue: Queue<ByteArray> = LinkedList()
         bluetoothAdapter = manager.adapter // 重新获取新引用
         bluetoothLeScanner = bluetoothAdapter?.bluetoothLeScanner
     }
+
     /**
      * 开始主动扫描BLE设备
      * @param timeout 扫描超时时间（毫秒）
      * @return 是否成功开始扫描
      */
-    fun startScan(deviceName:String,timeout: Long = 15000): Boolean {
+    fun startScan(deviceName: String, timeout: Long = 15000): Boolean {
 //        Log.i(TAG, "liwei---------startScan 1 $deviceName")
         if (!isInitialized) {
             Log.e(TAG, "BLE服务未初始化")
@@ -332,12 +336,12 @@ private val commandQueue: Queue<ByteArray> = LinkedList()
         }
     }
 
-    fun  getDeviceInfo(address: String): Map<String,Any>? {
+    fun getDeviceInfo(address: String): Map<String, Any>? {
 
         // 更新缓存
         val idx = scanResults.indexOfFirst { it.device.address == address }
         Log.d(TAG, "获取设备信息:${address} idx:$idx")
-        if (idx >= 0){
+        if (idx >= 0) {
             stopScan()
             val result = scanResults[idx];
             return printDeviceInfo(result)
@@ -348,7 +352,7 @@ private val commandQueue: Queue<ByteArray> = LinkedList()
     /**
      * 打印设备信息和服务UUID
      */
-    private fun printDeviceInfo(result: ScanResult):Map<String,Any>?{
+    private fun printDeviceInfo(result: ScanResult): Map<String, Any>? {
         val device = result.device
         val scanRecord = result.scanRecord
         val serviceUuids =
@@ -748,21 +752,19 @@ private val commandQueue: Queue<ByteArray> = LinkedList()
                 }
             }"
         )
-isReply=true;
+        isReply = true;
-cmdReplyType=cmdType;
+        cmdReplyType = cmdType;
-if(!commandQueue.isEmpty())
+        if (!commandQueue.isEmpty()) {
-{
             val nextCmd = commandQueue.poll()
-        writeChar!!.value=nextCmd 
+            writeChar!!.value = nextCmd
             var isSeccess = bluetoothGatt?.writeCharacteristic(writeChar)
-     if(isSeccess==true)
+            if (isSeccess == true) {
-     {
                 Log.d(
                     TAG,
                     "收到设备响应:"
                 )
             }
-}
+        }
         // 根据命令类型处理不同的响应
         when (cmdType) {
             BleConst.CMD_GET_VERSION -> {
@@ -1382,20 +1384,15 @@ if(!commandQueue.isEmpty())
             writeChar!!.value = cmdPacket
             // 加入队列并触发发送
 
-        if(isReply)
+            if (isReply) {
-        {
                 var isSeccess = bluetoothGatt?.writeCharacteristic(writeChar)
-            if(isSeccess == true)
+                if (isSeccess == true) {
-            {
+                    isReply = false;
-                isReply=false;
                     cmdType;
                     return true
-            }
+                } else
-            else
                     return false
-        }
+            } else {
-        else
-        {
                 commandQueue.offer(cmdPacket)
             }
             return true