Browse Source

feat(语音识别): 优化按住说话模式的识别结果处理

修改AzureAsrHelper中会话结束事件的条件判断,移除外部音频源限制
在AgentService中添加识别结果缓存,改进按住说话模式下的结果拼接和事件发送
newdev_shunjiawei
liwei1dao 11 months ago
parent
commit
5b4d80cc0a
  1. 55
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  2. 4
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt

55
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt

@ -149,6 +149,8 @@ object AgentService : CoroutineScope {
private var isplaymusic = false; private var isplaymusic = false;
//是否保留最终结果 //是否保留最终结果
private var isKeepResult = true; private var isKeepResult = true;
// 识别结果
private var recognitionResult = ""
// 打断防抖相关 // 打断防抖相关
private var lastInterruptTime = 0L private var lastInterruptTime = 0L
@ -666,20 +668,33 @@ object AgentService : CoroutineScope {
val success = azureAsrHelper?.setupEventListeners(object : val success = azureAsrHelper?.setupEventListeners(object :
AzureAsrHelper.ContinuousRecognizeCallback { AzureAsrHelper.ContinuousRecognizeCallback {
override fun onRecognizing(sessionid:String, recognizing: String, detectedLanguage: String) { override fun onRecognizing(sessionid:String, recognizing: String, detectedLanguage: String) {
Log.d(TAG, "liwei-------- onRecognizing: $sessionid")
if (recognizing.isNotEmpty()&&isKeepResult==true) { if (recognizing.isNotEmpty()&&isKeepResult==true) {
// 检测到语音,更新状态 // 检测到语音,更新状态
_hasSpeechDetected.set(true) _hasSpeechDetected.set(true)
Log.d(TAG, "ai识别中重置定时器") Log.d(TAG, "ai识别中重置定时器")
restartIdleCheck() restartIdleCheck()
var text = ""
if (currentRecognitionMode == "push_to_talk") {
text = recognitionResult + recognizing
sendEvent(
"recognizing", mapOf(
"sessionid" to currsessionId,
"text" to text,
"language" to detectedLanguage
)
)
}else{
text = recognizing
sendEvent( sendEvent(
"recognizing", mapOf( "recognizing", mapOf(
"sessionid" to sessionid, "sessionid" to sessionid,
"text" to recognizing, "text" to text,
"language" to detectedLanguage "language" to detectedLanguage
) )
) )
}
// 如果TTS正在播放或AI正在生成,检查是否需要打断 // 如果TTS正在播放或AI正在生成,检查是否需要打断
if (isTtsSpeaking || isAiStreaming) { if (isTtsSpeaking || isAiStreaming) {
@ -709,7 +724,12 @@ object AgentService : CoroutineScope {
} }
override fun onResult(sessionid:String,result: String, detectedLanguage: String) { override fun onResult(sessionid:String,result: String, detectedLanguage: String) {
Log.d(TAG, "liwei-------- onResult: $sessionid ")
if (result.isNotEmpty()&&isKeepResult==true) {//加入是否结束要输出最后的结果判读 if (result.isNotEmpty()&&isKeepResult==true) {//加入是否结束要输出最后的结果判读
// 保存识别结果
if (currentRecognitionMode == "push_to_talk") { //按住说话
recognitionResult += result
}else{
sendEvent( sendEvent(
"recognition_result", mapOf( "recognition_result", mapOf(
"sessionid" to sessionid, "sessionid" to sessionid,
@ -717,10 +737,9 @@ object AgentService : CoroutineScope {
"language" to detectedLanguage "language" to detectedLanguage
) )
) )
processWithChatApi(sessionid,result) processWithChatApi(sessionid,result)
} }
}
_hasSpeechDetected.set(false) _hasSpeechDetected.set(false)
@ -729,25 +748,42 @@ object AgentService : CoroutineScope {
} }
override fun onSessionStarted(sessionid:String) { override fun onSessionStarted(sessionid:String) {
Log.d(TAG, "liwei-------- onSessionStarted: $sessionid")
sendEvent("recognition_started", mapOf("sessionid" to sessionid, "status" to "started")) sendEvent("recognition_started", mapOf("sessionid" to sessionid, "status" to "started"))
// 使用模式感知的空闲检测启动方法 // 使用模式感知的空闲检测启动方法
recognitionResult = ""
currsessionId = sessionid
audioPlayer?.playAudio(R.raw.start) audioPlayer?.playAudio(R.raw.start)
} }
override fun onSessionStopped(sessionid:String) { override fun onSessionStopped(sessionid:String) {
Log.d(TAG, "liwei-------- onSessionStopped: $sessionid")
if (currentRecognitionMode == "push_to_talk") { //按住说话
sendEvent(
"recognition_result", mapOf(
"sessionid" to currsessionId,
"text" to recognitionResult
)
)
processWithChatApi(currsessionId,recognitionResult)
sendEvent("recognition_stopped", mapOf("sessionid" to currsessionId, "status" to "stopped"))
}else{
sendEvent("recognition_stopped", mapOf("sessionid" to sessionid, "status" to "stopped")) sendEvent("recognition_stopped", mapOf("sessionid" to sessionid, "status" to "stopped"))
_isRecognitionActive.set(false)
stopIdleCheck()
audioPlayer?.playAudio(R.raw.stop) audioPlayer?.playAudio(R.raw.stop)
} }
stopIdleCheck()
_isRecognitionActive.set(false)
}
override fun onCanceled(sessionid: String,reason: String, errorDetails: String) { override fun onCanceled(sessionid: String,reason: String, errorDetails: String) {
Log.d(TAG, "liwei-------- onCanceled: $sessionid $reason $errorDetails")
_isRecognitionActive.set(false) _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
Log.d(TAG, "recognition_canceled:")
BleService.closeCodec() BleService.closeCodec()
sendEvent( sendEvent(
"recognition_canceled", mapOf( "recognition_canceled", mapOf(
@ -759,6 +795,7 @@ object AgentService : CoroutineScope {
} }
override fun onError(sessionid:String, code: Int, error: String) { override fun onError(sessionid:String, code: Int, error: String) {
Log.d(TAG, "liwei-------- onError: $sessionid $code $error")
_isRecognitionActive.set(false) _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
BleService.closeCodec() BleService.closeCodec()

4
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt

@ -421,10 +421,10 @@ class AzureAsrHelper(private val context: Context) {
EventHandler<SessionEventArgs> { _, _ -> EventHandler<SessionEventArgs> { _, _ ->
// 直接在当前线程调用回调 // 直接在当前线程调用回调
Log.d(tag, "会话结束事件") Log.d(tag, "会话结束事件")
if (audioSourceType == AudioSourceType.EXTERNAL) { // if (audioSourceType == AudioSourceType.EXTERNAL) {
callback.onSessionStopped(currsessionid) callback.onSessionStopped(currsessionid)
audioStream?.isContinuousRecognitionActive = false audioStream?.isContinuousRecognitionActive = false
} // }
} }
) )

Loading…
Cancel
Save