Browse Source

feat(语音识别): 添加多模式语音识别支持

为语音识别服务添加多种模式支持,包括普通模式、BLE唤醒模式、通话模式和按住说话模式。根据不同的模式调整空闲检测策略,优化用户体验。

修改涉及跨平台实现,包括Dart、Kotlin和Swift代码,确保各平台行为一致。主要变更包括:
- 在startConversation方法中添加mode参数
- 根据不同模式调整空闲检测逻辑
- 在自动停止事件中添加模式信息
- 修复BLE唤醒模式下的识别问题
weicu
lxm 1 year ago
parent
commit
991ecc4955
  1. 2
      lib/data/services/novice_guide_service.dart
  2. 7
      lib/modules/agent/controllers/agent_controller.dart
  3. 76
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  4. 3
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt
  5. 8
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt
  6. 86
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift
  7. 3
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift
  8. 7
      local_plugins/agent_service/lib/agent_service.dart

2
lib/data/services/novice_guide_service.dart

@ -14,7 +14,7 @@ class NoviceGuideService extends GetxService {
StreamSubscription<dynamic>? _statusSubscription;
bool isHomeGuide = true;
bool isAIGuide = false;
bool isAIGuide = true;
RxBool isWakeup = false.obs;

7
lib/modules/agent/controllers/agent_controller.dart

@ -160,6 +160,7 @@ class AgentController extends GetxController {
_setupScrollListener(); //设置滚动监听器
_loadChatHistory(); //加载聊天历史记录
_subscribeToEvents(); //订阅代理服务事件
_subscribeToBleEvents(); //订阅BLE事件
_initMusicStatus(); //初始化音乐状态
_setupMediaSessionCallbacks(); //设置媒体会话回调
_setupTextInputListener(); //设置文本输入监听器
@ -1341,7 +1342,7 @@ class AgentController extends GetxController {
// 打电话模式
isTextInputMode.value = false;
Logger.i(TAG, '打电话模式已启动,开始语音输入');
await AgentService.startConversation();
await AgentService.startConversation(mode: 'phone_call');
}
// 结束打电话模式
@ -1393,7 +1394,7 @@ class AgentController extends GetxController {
} else {
// 当前未录音,开始录音
Logger.i(TAG, '开始打电话模式录音');
final result = await AgentService.startConversation();
final result = await AgentService.startConversation(mode: 'phone_call');
// 手动更新状态,确保UI同步
if (result) {
@ -1497,7 +1498,7 @@ class AgentController extends GetxController {
isPushToTalkActive.value = true;
//await AgentService.recognizeCallback();
// 开始语音识别
await AgentService.startConversation();
await AgentService.startConversation(mode: 'push_to_talk');
// await startVoiceInput();
}

76
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt

@ -117,7 +117,10 @@ object AgentService : CoroutineScope {
// 空闲检测相关
private var idleCheckJob: Job? = null
private val maxIdleSeconds = 10 // 最大空闲秒数
private val maxIdleSeconds = 10 // 最大空闲秒数(默认值)
// 语音识别模式
private var currentRecognitionMode = "normal" // 当前识别模式:normal, ble_wakeup, phone_call, push_to_talk
// 打断防抖相关
private var lastInterruptTime = 0L
@ -409,22 +412,63 @@ object AgentService : CoroutineScope {
listeners.clear()
}
/**
* 根据模式启动空闲检测
*/
private fun startIdleCheckForMode(mode: String) {
stopIdleCheck() // 先停止现有的检查
if (!isRecognitionActive) return
// 根据模式决定空闲检测策略
when (mode) {
"phone_call" -> {
// 通话模式:禁用空闲检测,保持持续激活
Log.d(TAG, "通话模式:禁用空闲检测")
return
}
"ble_wakeup" -> {
// BLE唤醒模式:使用默认的空闲检测
startIdleCheck(maxIdleSeconds)
}
"push_to_talk" -> {
// 按住说话模式:使用较长的空闲检测时间
startIdleCheck(maxIdleSeconds)
}
"normal" -> {
// 普通模式:使用默认的空闲检测
startIdleCheck(maxIdleSeconds)
}
else -> {
// 未知模式:使用默认的空闲检测
Log.w(TAG, "未知识别模式: $mode,使用默认空闲检测")
startIdleCheck(maxIdleSeconds)
}
}
}
/**
* 启动空闲检测
*/
private fun startIdleCheck() {
private fun startIdleCheck(idleSeconds: Int = maxIdleSeconds) {
stopIdleCheck() // 先停止现有的检查
if (!isRecognitionActive) return
Log.d(TAG, "启动空闲检测,超时时间: ${idleSeconds}秒,模式: $currentRecognitionMode")
idleCheckJob = launch {
try {
delay(maxIdleSeconds * 1000L)
delay(idleSeconds * 1000L)
if (isActive && isRecognitionActive && !isTtsSpeaking && !isAiStreaming) {
Log.d(TAG, "空闲检测触发自动停止,模式: $currentRecognitionMode")
stopRecognition()
sendEvent(
"auto_stop",
mapOf("reason" to "idle_timeout", "seconds" to maxIdleSeconds)
mapOf(
"reason" to "idle_timeout",
"seconds" to idleSeconds,
"mode" to currentRecognitionMode
)
)
}
} catch (e: CancellationException) {
@ -449,14 +493,16 @@ object AgentService : CoroutineScope {
*/
private fun restartIdleCheck() {
if (isRecognitionActive) {
startIdleCheck()
startIdleCheckForMode(currentRecognitionMode)
}
}
/**
* 开始语音识别
* @param isExternalActive 是否使用外部音频源(蓝牙)
* @param mode 识别模式:normal, ble_wakeup, phone_call, push_to_talk
*/
fun startRecognition(isExternalActive: Boolean): Boolean {
fun startRecognition(isExternalActive: Boolean, mode: String = "normal"): Boolean {
if (!isInitialized) {
Log.d(TAG, "服务未初始化")
return false
@ -488,6 +534,10 @@ object AgentService : CoroutineScope {
return false
}
// 设置当前识别模式
currentRecognitionMode = mode
Log.d(TAG, "设置语音识别模式: $mode")
_isRecognitionActive.set(true)
_hasSpeechDetected.set(false)
@ -500,6 +550,9 @@ object AgentService : CoroutineScope {
Log.d("TAG", "azureAsrHelper?.startContinuousRecognition")
azureAsrHelper?.startContinuousRecognition(audioSourceType)
// 根据模式启动相应的空闲检测
startIdleCheckForMode(mode)
return true
} catch (e: Exception) {
_isRecognitionActive.set(false)
@ -586,7 +639,8 @@ object AgentService : CoroutineScope {
override fun onSessionStarted() {
sendEvent("recognition_started", mapOf("status" to "started"))
startIdleCheck()
// 使用模式感知的空闲检测启动方法
startIdleCheckForMode(currentRecognitionMode)
audioPlayer?.playAudio(R.raw.start)
}
@ -644,16 +698,24 @@ object AgentService : CoroutineScope {
fun stopRecognition() {
if (!isRecognitionActive) return
Log.d(TAG, "停止语音识别,当前模式: $currentRecognitionMode")
try {
azureAsrHelper?.stopContinuousRecognition()
BleService.closeCodec()
_isRecognitionActive.set(false)
stopIdleCheck()
// 重置识别模式
currentRecognitionMode = "normal"
return
} catch (e: Exception) {
Log.e(TAG, "停止语音识别异常: ${e.message}")
_isRecognitionActive.set(false)
stopIdleCheck()
// 重置识别模式
currentRecognitionMode = "normal"
return
}
}

3
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt

@ -115,7 +115,8 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
"startConversation" -> {
try {
val isStarted = AgentService.startRecognition(false)
val mode = call.argument<String>("mode") ?: "normal"
val isStarted = AgentService.startRecognition(false, mode)
result.success(isStarted)
} catch (e: Exception) {
Log.e(TAG, "启动对话失败", e)

8
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt

@ -66,7 +66,7 @@ object BleAgent : BleService.Callback, AgentServiceListener {
return false
}
Log.d(TAG, "手动启动语音识别: ")
return AgentService.startRecognition(true)
return AgentService.startRecognition(true, "ble_wakeup")
}
/**
@ -148,11 +148,11 @@ Log.d(TAG, "手动启动语音识别: ")
// 停止当前TTS,避免冲突
AgentService.interruptCurrentResponse()
// AgentService.speakText("我在!")
// 启动语音识别
// 启动语音识别,使用BLE唤醒模式
// 如果已经在识别,则重新启动
//stopListening()
//stopListening()
startListening()

86
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift

@ -65,6 +65,9 @@ class AgentServiceImpl: NSObject {
private var idleTimer: DispatchSourceTimer?
private let maxIdleSeconds: TimeInterval = 10
// 语音识别模式
private var currentRecognitionMode: String = "normal"
internal var audioPlayer: AudioPlayer?
// private var locationInfo:[String: String]?
@ -280,26 +283,56 @@ class AgentServiceImpl: NSObject {
return true
}
private func startIdleCheck() {
private func startIdleCheckForMode(mode: String) {
stopIdleCheck()
guard isRecognizing else { return }
// 根据模式决定空闲检测策略
switch mode {
case "phone_call":
// 通话模式:禁用空闲检测,保持持续激活
os_log("通话模式:禁用空闲检测", log: logger, type: .info)
return
case "ble_wakeup":
// BLE唤醒模式:使用默认的空闲检测
startIdleCheck(idleSeconds: maxIdleSeconds)
case "push_to_talk":
// 按住说话模式:使用默认的空闲检测
startIdleCheck(idleSeconds: maxIdleSeconds)
case "normal":
// 普通模式:使用默认的空闲检测
startIdleCheck(idleSeconds: maxIdleSeconds)
default:
// 未知模式:使用默认的空闲检测
os_log("未知识别模式: %{public}@,使用默认空闲检测", log: logger, type: .error, mode)
startIdleCheck(idleSeconds: maxIdleSeconds)
}
}
private func startIdleCheck(idleSeconds: TimeInterval = 10) {
stopIdleCheck()
guard isRecognizing else { return }
os_log("启动空闲检测,超时时间: %.0f秒,模式: %{public}@", log: logger, type: .info, idleSeconds, currentRecognitionMode)
// 使用全局队列而不是主队列,避免后台挂起问题
let timer = DispatchSource.makeTimerSource(queue: DispatchQueue.global(qos: .utility))
idleTimer = timer
timer.schedule(deadline: .now() + maxIdleSeconds)
timer.schedule(deadline: .now() + idleSeconds)
timer.setEventHandler { [weak self] in
// 使用更安全的方式检查self是否存在
DispatchQueue.main.async { [weak self] in
guard let self = self else { return }
if self.isRecognizing && !self.isSpeaking && !self.isAiStreaming {
os_log("空闲检测触发自动停止,模式: %{public}@", log: self.logger, type: .info, self.currentRecognitionMode)
self.stopRecognition()
self.sendEvent(name: "auto_stop", data: [
"reason": "idle_timeout",
"seconds": self.maxIdleSeconds
"reason": "idle_timeout",
"seconds": idleSeconds,
"mode": self.currentRecognitionMode
])
}
}
@ -314,10 +347,10 @@ class AgentServiceImpl: NSObject {
private func restartIdleCheck() {
if isRecognizing {
startIdleCheck()
startIdleCheckForMode(mode: currentRecognitionMode)
}
}
func startRecognition(useBle: Bool = false) -> Bool {
func startRecognition(useBle: Bool = false, mode: String = "normal") -> Bool {
if !isInitialized {
sendError("服务未初始化", code: "NOT_INITIALIZED")
return false
@ -333,9 +366,13 @@ class AgentServiceImpl: NSObject {
if isRecognizing {
stopRecognition()
}
// 设置当前识别模式
currentRecognitionMode = mode
os_log("设置语音识别模式: %{public}@", log: logger, type: .info, mode)
let audioSourceType: AzureAsrHelper.AudioSourceType = useBle ? .external : .microphone
if useBle {
AudioSessionHub.shared.begin(.playback)
} else {
@ -349,7 +386,10 @@ print("ai启动语音=\(audioSourceType)=\(useBle)")
sendError("启动语音识别失败", code: "RECOGNITION_START_ERROR")
return false
}
// 根据模式启动相应的空闲检测
startIdleCheckForMode(mode: mode)
return true
}
func recognizeCallback() -> Bool {
@ -367,11 +407,18 @@ print("ai启动语音=\(audioSourceType)=\(useBle)")
if !isRecognizing {
return true
}
os_log("停止语音识别,当前模式: %{public}@", log: logger, type: .info, currentRecognitionMode)
guard let success = azureAsrHelper?.stopContinuousRecognition(), success else {
return false
}
stopIdleCheck()
// 重置识别模式
currentRecognitionMode = "normal"
return true
}
func disableBluetoothAudio() {
@ -1388,10 +1435,10 @@ extension AgentServiceImpl: BleService.Callback {
func onWakeupSignalReceived() {
stopTts()
stopRecognition()
startRecognition(useBle: true)
startRecognition(useBle: true, mode: "ble_wakeup")
}
func onDeviceInfoReceived(infoType: Int, infoData: [String: Any]) {
@ -1620,7 +1667,8 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback {
sendEvent(name: "recognition_started", data: ["status": "started"])
isRecognizing = true
hasSpeechDetected = false
startIdleCheck()
// 使用模式感知的空闲检测启动方法
startIdleCheckForMode(mode: currentRecognitionMode)
audioPlayer?.playStartSound()
}

3
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift

@ -41,7 +41,8 @@ public class AgentServicePlugin: NSObject, FlutterPlugin {
result(success)
case "startConversation":
impl.startRecognition()
let mode = (call.arguments as? [String: Any])?["mode"] as? String ?? "normal"
impl.startRecognition(mode: mode)
result(true)
case "recognizeCallback":

7
local_plugins/agent_service/lib/agent_service.dart

@ -238,10 +238,13 @@ class AgentService {
/// 开始对话
///
/// 启动语音识别,开始监听用户语音输入
/// [mode] 语音识别模式:'ble_wakeup'(BLE唤醒), 'phone_call'(通话模式), 'push_to_talk'(按住说话), 'normal'(普通模式)
/// 返回是否成功开始对话
static Future<bool> startConversation() async {
static Future<bool> startConversation({String mode = 'normal'}) async {
try {
final bool result = await _channel.invokeMethod('startConversation');
final bool result = await _channel.invokeMethod('startConversation', {
'mode': mode,
});
return result;
} on PlatformException catch (e) {
throw AgentServiceException(e.code, e.message ?? '启动对话失败', e.details);

Loading…
Cancel
Save