Browse Source

修复ai声音时大时小的问题,加入识别超时机制

weicu
fdp 1 year ago
parent
commit
d33330eda0
  1. 20
      lib/modules/agent/controllers/agent_controller.dart
  2. 9
      lib/modules/translation/controllers/translation_controller.dart
  3. 10
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  4. 22
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift
  5. 9
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift
  6. 29
      local_plugins/agent_service/lib/agent_service.dart
  7. 2
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
  8. 36
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift
  9. 15
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift

20
lib/modules/agent/controllers/agent_controller.dart

@ -3,6 +3,8 @@ import 'dart:convert';
import 'dart:ffi';
import 'dart:io';
import 'package:amap_service/amap_service.dart';
import '../../../data/services/ble_manager.dart';
import '../../../data/services/bluetooth_manager.dart';
import '../../../data/services/novice_guide_service.dart';
import '/core/utils/upload_oss.dart';
import '/data/models/user_Info.dart';
@ -41,7 +43,10 @@ class AgentController extends GetxController {
final ScrollController scrollController = ScrollController();
final noviceGuideService = Get.find<NoviceGuideService>();
// 确保服务已初始化
// ========== 依赖服务 ==========
final BluetoothManager _bluetoothManager = Get.find<BluetoothManager>();
final BleManager _bleManager = Get.find<BleManager>();
//音乐播放管理
final _musiceManager = Get.find<MusicManager>();
@ -558,16 +563,16 @@ class AgentController extends GetxController {
break;
case AgentServiceEventType.ttsStarted:
// isSpeaking.value = true;
// Logger.i(TAG, 'TTS开始播放');
isSpeaking.value = true;
print('TTS开始播放');
break;
case AgentServiceEventType.ttsCompleted:
case AgentServiceEventType.ttsStopped:
case AgentServiceEventType.ttsCanceled:
// isSpeaking.value = false;
isSpeaking.value = false;
// Logger.i(TAG, 'TTS停止播放, $event.type');
Logger.i(TAG, 'TTS停止播放, $event.type');
break;
case AgentServiceEventType.playbackStarted:
@ -1035,6 +1040,11 @@ class AgentController extends GetxController {
try {
await AgentService.recognizeCallback();
if (_bluetoothManager.currentDeviceRx.value != null) {
await AgentService.restoreOriginalAudioState();
} else {
await AgentService.disableBluetoothAudio();
}
} catch (e) {
Logger.e(TAG, '启动识别器失败: $e');
}

9
lib/modules/translation/controllers/translation_controller.dart

@ -9,6 +9,7 @@ import 'package:get_storage/get_storage.dart';
import 'package:intl/intl.dart';
import 'package:path_provider/path_provider.dart';
import 'package:permission_handler/permission_handler.dart';
import '../../../data/services/bluetooth_manager.dart';
import '../../../data/services/music_manager.dart';
import '../../../data/services/volcano_translation_service.dart';
import '../../../data/services/tts_service.dart';
@ -30,6 +31,8 @@ class TranslationController extends GetxController {
static const String TAG = 'TranslationController';
// 服务
final BluetoothManager _bluetoothManager = Get.find<BluetoothManager>();
final BleManager _bleManager = Get.find<BleManager>();
final AsrService _asrService = Get.find<AsrService>();
final VolcanoTranslationService _translationService =
Get.find<VolcanoTranslationService>();
@ -483,7 +486,11 @@ class TranslationController extends GetxController {
if (currentMode.value == 'faceToFace') {
_asrService.stopContinuousRecognition();
} else {
await _ttsService.setAudioOutputDevice(0);
if (_bluetoothManager.currentDeviceRx.value != null) {
restoreOriginalAudioState();
} else {
disableBluetoothAudio();
}
await _asrService.startContinuousRecognition(_audioSourceType);
// 开始ASR活跃时长计时
_startAsrActiveTracking();

10
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt

@ -420,7 +420,7 @@ object AgentService : CoroutineScope {
try {
delay(maxIdleSeconds * 1000L)
if (isActive && isRecognitionActive && !hasSpeechDetected && !isTtsSpeaking && !isAiStreaming) {
if (isActive && isRecognitionActive && !isTtsSpeaking && !isAiStreaming) {
stopRecognition()
sendEvent(
"auto_stop",
@ -522,12 +522,12 @@ object AgentService : CoroutineScope {
override fun onRecognizing(recognizing: String, detectedLanguage: String) {
if (recognizing.isNotEmpty()) {
// 检测到语音,更新状态
val previousHasSpeech = hasSpeechDetected
_hasSpeechDetected.set(true)
val previousHasSpeech = hasSpeechDetected
_hasSpeechDetected.set(true)
if (!previousHasSpeech) {
restartIdleCheck()
}
restartIdleCheck()
}
sendEvent(
"recognizing", mapOf(

22
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift

@ -291,8 +291,8 @@ class AgentServiceImpl: NSObject {
// 使用更安全的方式检查self是否存在
DispatchQueue.main.async { [weak self] in
guard let self = self else { return }
if self.isRecognizing && !self.hasSpeechDetected && !self.isSpeaking && !self.isAiStreaming {
if self.isRecognizing && !self.isSpeaking && !self.isAiStreaming {
self.stopRecognition()
self.sendEvent(name: "auto_stop", data: [
"reason": "idle_timeout",
@ -349,8 +349,14 @@ print("ai启动语音=\(audioSourceType)=\(useBle)")
return true
}
func recognizeCallback(useBle: Bool = false) -> Bool {
func recognizeCallback() -> Bool {
print("重新设置识别回调=")
guard let asrsetupSuccess = azureAsrHelper?.setupEventListeners(
callback: self
), asrsetupSuccess else {
sendError("初始化语音识别服务失败", code: "ASR_INIT_ERROR")
return false
}
return true
}
@ -365,7 +371,12 @@ print("ai启动语音=\(audioSourceType)=\(useBle)")
return true
}
func disableBluetoothAudio() {
azureAsrHelper?.disableBluetoothAudio()
}
func restoreOriginalAudioState() {
azureAsrHelper?.restoreOriginalAudioState()
}
func pushAudioData(_ audioData: Data) -> Bool {
if !isRecognizing || !isInitialized {
return false
@ -1572,6 +1583,7 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback {
hasSpeechDetected = true
if !previousHasSpeech {
print("onRecognizing restartIdleCheck")
restartIdleCheck()
}

9
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift

@ -51,7 +51,14 @@ public class AgentServicePlugin: NSObject, FlutterPlugin {
case "stopConversation":
impl.stopRecognition()
result(true)
case "disableBluetoothAudio":
impl.disableBluetoothAudio();
return
case "restoreOriginalAudioState":
impl.restoreOriginalAudioState();
return
case "processTextInput":
guard let arguments = call.arguments as? [String: Any],
let text = arguments["text"] as? String else {

29
local_plugins/agent_service/lib/agent_service.dart

@ -261,6 +261,33 @@ class AgentService {
}
}
/// 恢复原始音频状态
///
/// 恢复原始音频状态
/// 返回是否成功恢复原始音频状态
static Future<bool> restoreOriginalAudioState() async {
try {
final bool result =
await _channel.invokeMethod('restoreOriginalAudioState');
return result;
} on PlatformException catch (e) {
throw AgentServiceException(e.code, e.message ?? '恢复原始音频状态失败', e.details);
}
}
/// 禁用蓝牙音频
///
/// 禁用蓝牙音频
/// 返回是否成功禁用蓝牙音频
static Future<bool> disableBluetoothAudio() async {
try {
final bool result = await _channel.invokeMethod('disableBluetoothAudio');
return result;
} on PlatformException catch (e) {
throw AgentServiceException(e.code, e.message ?? '禁用蓝牙音频失败', e.details);
}
}
/// 停止对话
///
/// 停止语音识别
@ -332,8 +359,6 @@ class AgentService {
}
}
/// 停止语音合成
static Future<bool> stopAiSteam() async {
try {

2
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt

@ -745,7 +745,7 @@ class AzureAsrHelper(private val context: Context) {
*/
fun saveAudioDataTo(buffer: ByteArray) {
if (audioSourceType == AudioSourceType.MICROPHONE) return
if (audioSourceType == AudioSourceType.MICROPHONE||!isContinuousRecognitionActive) return
// 放入队列,由写线程写入
writeQueue.offer(buffer.copyOf())

36
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift

@ -131,7 +131,7 @@ public class AzureAsrHelper: NSObject {
*/
private func preInitializeAudioComponents() {
if audioStream == nil {
audioStream = AudioStream()
audioStream = AudioStream(parentHelper: self)
audioStream?.initAudioRecord()
audioStream?.onAudioData = { [weak self] data in
self?.recordfile?.saveAudioDataToWav(data)
@ -354,7 +354,7 @@ public class AzureAsrHelper: NSObject {
do {
// 【优化】检查是否已经初始化,避免重复创建
if audioStream == nil {
audioStream = AudioStream()
audioStream = AudioStream(parentHelper: self)
audioStream?.initAudioRecord()
audioStream?.onAudioData = { [weak self] data in
self?.recordfile?.saveAudioDataToWav(data)
@ -548,6 +548,7 @@ public class AzureAsrHelper: NSObject {
public func disableBluetoothAudio() {
do {
print("禁用蓝牙音频功能,切换回正常音频模式")
audioStream?.setAudioOutputRoute(.speaker)
} catch {
print("disableBluetoothAudio")
@ -559,6 +560,7 @@ public class AzureAsrHelper: NSObject {
public func restoreOriginalAudioState() {
do {
print("恢复原始音频设备状态(通常是重新启用蓝牙)")
audioStream?.setAudioOutputRoute(.receiver)
} catch {
print("restoreOriginalAudioState")
@ -632,9 +634,18 @@ public class AzureAsrHelper: NSObject {
private var isRunning = false
public var isWriting = false
private let bufferSize: Int = 4096
private var writeThread: DispatchQueue?
private var currentRoute: AudioOutputRoute?
public var onAudioData: ((Data) -> Void)?
private var writeThread: DispatchQueue?
private var currentRoute: AudioOutputRoute?
public var onAudioData: ((Data) -> Void)?
// 添加对外部类的弱引用
private weak var parentHelper: AzureAsrHelper?
// 添加初始化方法,接收外部类引用
init(parentHelper: AzureAsrHelper) {
self.parentHelper = parentHelper
super.init()
}
/**
* 初始化
*/
@ -654,7 +665,7 @@ public class AzureAsrHelper: NSObject {
// 创建新的写线程
writeThread = DispatchQueue(label: "audio.stream.writer")
startWriteThread()
self.setAudioOutputRoute(.receiver)
//self.setAudioOutputRoute(.receiver)
}
/// 获取最佳音频格式 (iOS 通常支持标准采样率)
@ -717,7 +728,8 @@ public class AzureAsrHelper: NSObject {
* @param data 音频数据字节数组
*/
public func saveAudioDataTo(data: Data) {
if audioSourceType != .external {
// 通过父类引用调用方法
if audioSourceType != .external || !(parentHelper?.isContinuousRecognitionActive() ?? false) {
return
}
// print("外部data=\(data)")
@ -879,8 +891,7 @@ private func runMicrophoneCapture() {
case bluetooth
}
public func setAudioOutputRoute(_ route: AudioOutputRoute) {
guard currentRoute != route else { return }
print("setAudioOutputRoute")
do {
//
if self.isWriting {
@ -890,14 +901,15 @@ private func runMicrophoneCapture() {
try audioSession.setActive(false)
}
print("setAudioOutputRoutecurrent,route=\(route)")
switch route {
case .speaker:
print("setAudioOutputRoutecurrent,speaker")
// 使用扬声器时必须用videoChat模式
try audioSession.setCategory(
.playAndRecord,
mode: .videoChat,
options: [.allowBluetooth, .mixWithOthers, .defaultToSpeaker]
options: [.defaultToSpeaker]
)
try audioSession.overrideOutputAudioPort(.speaker)
@ -906,7 +918,7 @@ private func runMicrophoneCapture() {
try audioSession.setCategory(
.playAndRecord,
mode: .voiceChat,
options: [.allowBluetooth, .mixWithOthers, .defaultToSpeaker]
options: [.allowBluetooth]
)
try audioSession.overrideOutputAudioPort(.none)

15
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift

@ -184,6 +184,15 @@ import os.log
callback: currentAsrCallback!
)
result(success)
case "disableBluetoothAudio":
azureAsrHelper.disableBluetoothAudio();
return
case "restoreOriginalAudioState":
azureAsrHelper.restoreOriginalAudioState();
return
case "stopContinuousRecognition":
let success = azureAsrHelper.stopContinuousRecognition()
@ -224,7 +233,7 @@ import os.log
return
}
do {
print("音频文件名称为: \(filePath)")
print("音频文件名称为: (filePath)")
try azureAsrHelper.renameFile(filePath: filePath, newName: newName)
result(true)
} catch {
@ -239,7 +248,7 @@ import os.log
return
}
do {
print("音频文件名称为: \(sourcePath)")
print("音频文件名称为: (sourcePath)")
try azureAsrHelper.moveFile(sourcePath: sourcePath, destPath: destPath)
result(true)
} catch {
@ -253,7 +262,7 @@ import os.log
return
}
do {
print("音频文件名称为: \(filePath)")
print("音频文件名称为: (filePath)")
try azureAsrHelper.enableRecord(filePath: filePath)
result(true)
} catch {

Loading…
Cancel
Save