liwei1dao
/
deepvoice_client
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

修复识别结果处理和会话管理问题

weicu
tanlongsheng 10 months ago
parent
f8f16bde1d
commit
7129016063
6 changed files with 125 additions and 75 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 56
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  2. 84
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift
  3. 2
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
  4. 8
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift
  5. 5
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift
  6. 45
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/XunFeiAsrHelper.swift

56
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
View File

@ -677,32 +677,18 @@ object AgentService : CoroutineScope {
_hasSpeechDetected.set(true) _hasSpeechDetected.set(true)
Log.d(TAG, "ai识别中重置定时器") Log.d(TAG, "ai识别中重置定时器")
restartIdleCheck() restartIdleCheck()
var text = "" if(currsessionAsrId.isEmpty()) {
if (currentRecognitionMode == "push_to_talk") { currsessionAsrId = sessionid
text = recognitionResult + recognizing
sendEvent(
"recognizing", mapOf(
"sessionid" to currsessionId,
"text" to text,
"language" to detectedLanguage
)
)
}else{
if(currsessionAsrId.isEmpty()) {
currsessionAsrId = sessionid
}
text = recognitionResult + recognizing
sendEvent(
"recognizing", mapOf(
"sessionid" to currsessionAsrId,
"text" to text,
"language" to detectedLanguage
)
)
} }
var text = recognitionResult + recognizing
sendEvent(
"recognizing", mapOf(
"sessionid" to currsessionAsrId,
"text" to text,
"language" to detectedLanguage
)
)
// 如果TTS正在播放或AI正在生成，检查是否需要打断 // 如果TTS正在播放或AI正在生成，检查是否需要打断
if (isTtsSpeaking || isAiStreaming) { if (isTtsSpeaking || isAiStreaming) {
val currentTime = System.currentTimeMillis() val currentTime = System.currentTimeMillis()
@ -734,20 +720,19 @@ object AgentService : CoroutineScope {
Log.d(TAG, "liwei-------- onResult: $sessionid ") Log.d(TAG, "liwei-------- onResult: $sessionid ")
if (result.isNotEmpty()&&isKeepResult==true) {//加入是否结束要输出最后的结果判读 if (result.isNotEmpty()&&isKeepResult==true) {//加入是否结束要输出最后的结果判读
// 保存识别结果 // 保存识别结果
if (currsessionAsrId.isEmpty()) {
currsessionAsrId = sessionid
}
recognitionResult += result
if (currentRecognitionMode == "push_to_talk") { //按住说话 if (currentRecognitionMode == "push_to_talk") { //按住说话
recognitionResult += result
sendEvent( sendEvent(
"recognizing", mapOf( "recognizing", mapOf(
"sessionid" to currsessionId, "sessionid" to currsessionAsrId,
"text" to recognitionResult, "text" to recognitionResult,
"language" to detectedLanguage "language" to detectedLanguage
) )
) )
}else{ }else{
if (currsessionAsrId.isEmpty()) {
currsessionAsrId = sessionid
}
recognitionResult += result
pendingProcessJob?.cancel() pendingProcessJob?.cancel()
sendEvent( sendEvent(
"recognizing", mapOf( "recognizing", mapOf(
@ -757,7 +742,7 @@ object AgentService : CoroutineScope {
) )
) )
pendingProcessJob = launch { pendingProcessJob = launch {
delay(2000) delay(1500)
if (recognitionResult.isNotEmpty()) { if (recognitionResult.isNotEmpty()) {
sendEvent( sendEvent(
"recognition_result", mapOf( "recognition_result", mapOf(
@ -785,7 +770,8 @@ object AgentService : CoroutineScope {
sendEvent("recognition_started", mapOf("sessionid" to sessionid, "status" to "started")) sendEvent("recognition_started", mapOf("sessionid" to sessionid, "status" to "started"))
// 使用模式感知的空闲检测启动方法 // 使用模式感知的空闲检测启动方法
recognitionResult = "" recognitionResult = ""
currsessionId = sessionid currsessionAsrId = ""
pendingProcessJob?.cancel()
audioPlayer?.playAudio(R.raw.start) audioPlayer?.playAudio(R.raw.start)
} }
@ -797,13 +783,13 @@ object AgentService : CoroutineScope {
if (recognitionResult.isNotEmpty()){ if (recognitionResult.isNotEmpty()){
sendEvent( sendEvent(
"recognition_result", mapOf( "recognition_result", mapOf(
"sessionid" to currsessionId, "sessionid" to currsessionAsrId,
"text" to recognitionResult "text" to recognitionResult
) )
) )
processWithChatApi(currsessionId,recognitionResult) processWithChatApi(currsessionAsrId, recognitionResult)
} }
sendEvent("recognition_stopped", mapOf("sessionid" to currsessionId, "status" to "stopped")) sendEvent("recognition_stopped", mapOf("sessionid" to currsessionAsrId, "status" to "stopped"))
}else{ }else{
sendEvent("recognition_stopped", mapOf("sessionid" to sessionid, "status" to "stopped")) sendEvent("recognition_stopped", mapOf("sessionid" to sessionid, "status" to "stopped"))

84
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift
View File

@ -100,6 +100,8 @@ class AgentServiceImpl: NSObject {
public var currentRecognitionMode: String = "normal" public var currentRecognitionMode: String = "normal"
// 识别结果 // 识别结果
private var recognitionResult = "" private var recognitionResult = ""
private var currsessionAsrId: String = ""
private var pendingProcessWorkItem: DispatchWorkItem?
internal var audioPlayer: AudioPlayer? internal var audioPlayer: AudioPlayer?
// private var locationInfo:[String: String]? // private var locationInfo:[String: String]?
@ -486,7 +488,8 @@ class AgentServiceImpl: NSObject {
print("ai启动语音=\(audioSourceType)=\(useBle)") print("ai启动语音=\(audioSourceType)=\(useBle)")
guard let success = azureAsrHelper?.startContinuousRecognition( guard let success = azureAsrHelper?.startContinuousRecognition(
// callback: self, // callback: self,
audioSourceType: audioSourceType audioSourceType: audioSourceType,
isRemoveFirstPunctuation: false
), success else { ), success else {
// 新增：启动失败时复位"启动中"状态 // 新增：启动失败时复位"启动中"状态
isStartingRecognition = false isStartingRecognition = false
@ -2166,15 +2169,49 @@ class AudioPlayer {
extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback {
func onResult(sessionid:String ,_ text: String, _ detectedLanguage: String) { func onResult(sessionid:String ,_ text: String, _ detectedLanguage: String) {
if !text.isEmpty && isKeepResult { if !text.isEmpty && isKeepResult {
if currsessionAsrId.isEmpty {
currsessionAsrId = sessionid
}
recognitionResult += text
if (currentRecognitionMode == "push_to_talk") { //按住说话 if (currentRecognitionMode == "push_to_talk") { //按住说话
recognitionResult += text var data: [String: Any] = [
"sessionid": currsessionAsrId,
"text": recognitionResult
]
if !detectedLanguage.isEmpty {
data["language"] = detectedLanguage
}
sendEvent(name: "recognizing", data: data)
}else{ }else{
var data: [String: Any] = ["sessionid":sessionid,"text": text] pendingProcessWorkItem?.cancel()
var data: [String: Any] = [
"sessionid": currsessionAsrId,
"text": recognitionResult
]
if !detectedLanguage.isEmpty { if !detectedLanguage.isEmpty {
data["language"] = detectedLanguage data["language"] = detectedLanguage
} }
sendEvent(name: "recognition_result", data: data) sendEvent(name: "recognizing", data: data)
processTextInput(sessionid: sessionid,text, speakResponse: true)
let workItem = DispatchWorkItem { [weak self] in
guard let self = self else { return }
if !self.recognitionResult.isEmpty {
var resultData: [String: Any] = [
"sessionid": self.currsessionAsrId,
"text": self.recognitionResult
]
if !detectedLanguage.isEmpty {
resultData["language"] = detectedLanguage
}
self.sendEvent(name: "recognition_result", data: resultData)
self.processTextInput(sessionid: self.currsessionAsrId, self.recognitionResult, speakResponse: true)
self.recognitionResult = ""
self.currsessionAsrId = ""
}
}
pendingProcessWorkItem = workItem
DispatchQueue.main.asyncAfter(deadline: .now() + 1.5, execute: workItem)
} }
} }
@ -2187,6 +2224,7 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback {
} }
func onRecognizing(sessionid:String ,_ recognizing: String, _ detectedLanguage: String) { func onRecognizing(sessionid:String ,_ recognizing: String, _ detectedLanguage: String) {
pendingProcessWorkItem?.cancel()
if !recognizing.isEmpty && isKeepResult { if !recognizing.isEmpty && isKeepResult {
let previousHasSpeech = hasSpeechDetected let previousHasSpeech = hasSpeechDetected
hasSpeechDetected = true hasSpeechDetected = true
@ -2195,21 +2233,19 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback {
print("onRecognizing restartIdleCheck") print("onRecognizing restartIdleCheck")
restartIdleCheck() restartIdleCheck()
} }
if (currentRecognitionMode == "push_to_talk") { //按住说话 if currsessionAsrId.isEmpty {
var data: [String: Any] = ["sessionid":currsessionId,"text": recognitionResult+recognizing] currsessionAsrId = sessionid
if !detectedLanguage.isEmpty { }
data["language"] = detectedLanguage let text = recognitionResult + recognizing
} var data: [String: Any] = [
sendEvent(name: "recognizing", data: data) "sessionid": currsessionAsrId,
}else{ "text": text
var data: [String: Any] = ["sessionid":sessionid,"text": recognizing] ]
if !detectedLanguage.isEmpty { if !detectedLanguage.isEmpty {
data["language"] = detectedLanguage data["language"] = detectedLanguage
}
sendEvent(name: "recognizing", data: data)
} }
sendEvent(name: "recognizing", data: data)
if isSpeaking || isAiStreaming { if isSpeaking || isAiStreaming {
interruptCurrentResponse() interruptCurrentResponse()
} }
@ -2219,7 +2255,8 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback {
func onSessionStarted(sessionid:String) { func onSessionStarted(sessionid:String) {
sendEvent(name: "recognition_started", data: ["sessionid":sessionid,"status": "started"]) sendEvent(name: "recognition_started", data: ["sessionid":sessionid,"status": "started"])
recognitionResult = "" recognitionResult = ""
currsessionId = sessionid currsessionAsrId = ""
pendingProcessWorkItem?.cancel()
isRecognizing = true isRecognizing = true
hasSpeechDetected = false hasSpeechDetected = false
// 新增：会话真正启动，复位"启动中"状态 // 新增：会话真正启动，复位"启动中"状态
@ -2242,11 +2279,14 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback {
func onSessionStopped(sessionid:String) { func onSessionStopped(sessionid:String) {
if (currentRecognitionMode == "push_to_talk") { //按住说话 if (currentRecognitionMode == "push_to_talk") { //按住说话
if (!recognitionResult.isEmpty){ if (!recognitionResult.isEmpty){
var data: [String: Any] = ["sessionid":currsessionId,"text": recognitionResult] var data: [String: Any] = [
"sessionid": currsessionAsrId,
"text": recognitionResult
]
sendEvent(name: "recognition_result", data: data) sendEvent(name: "recognition_result", data: data)
processTextInput(sessionid: currsessionId, recognitionResult, speakResponse: true) processTextInput(sessionid: currsessionAsrId, recognitionResult, speakResponse: true)
} }
sendEvent(name: "recognition_stopped", data: ["sessionid":currsessionId,"status": "stopped"]) sendEvent(name: "recognition_stopped", data: ["sessionid":currsessionAsrId,"status": "stopped"])
}else{ }else{
sendEvent(name: "recognition_stopped", data: ["sessionid":sessionid,"status": "stopped"]) sendEvent(name: "recognition_stopped", data: ["sessionid":sessionid,"status": "stopped"])
isRecognizing = false isRecognizing = false

2
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
View File

@ -114,7 +114,7 @@ class AzureAsrHelper(private val context: Context) {
this.supportedLanguages = supportedLanguages this.supportedLanguages = supportedLanguages
} }
// this.useXunfei = supportedLanguages.all { it == "zh-CN" || it == "en-US" } this.useXunfei = supportedLanguages.all { it == "zh-CN" || it == "en-US" }
// 根据支持的语言数量决定是否启用自动语言检测 // 根据支持的语言数量决定是否启用自动语言检测
this.isAutoDetectLanguage = supportedLanguages.size >= 2 this.isAutoDetectLanguage = supportedLanguages.size >= 2

8
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift
View File

@ -116,7 +116,7 @@ public class AzureAsrHelper: NSObject {
} }
// 检查是否只包含中文或英文，如果是则使用讯飞 // 检查是否只包含中文或英文，如果是则使用讯飞
// self.useXunfei = supportedLanguages.allSatisfy { $0 == "zh-CN" || $0 == "en-US" } self.useXunfei = supportedLanguages.allSatisfy { $0 == "zh-CN" || $0 == "en-US" }
// 根据支持的语言数量决定是否启用自动语言检测 // 根据支持的语言数量决定是否启用自动语言检测
self.isAutoDetectLanguage = supportedLanguages.count >= 2 self.isAutoDetectLanguage = supportedLanguages.count >= 2
@ -388,6 +388,7 @@ public class AzureAsrHelper: NSObject {
* *
* @param audioSourceType 音频源类型 * @param audioSourceType 音频源类型
* @param audioDataCallback 音频数据回调 * @param audioDataCallback 音频数据回调
* @param isRemoveFirstPunctuation 是否移除第一个标点符号
* @return 是否成功开始识别（调度成功即返回 true） * @return 是否成功开始识别（调度成功即返回 true）
* *
* 修复点：在音频启动成功后的闭包中，先判断 audioStream?.isContinuousRecognitionActive 是否仍然为 true， * 修复点：在音频启动成功后的闭包中，先判断 audioStream?.isContinuousRecognitionActive 是否仍然为 true，
@ -395,7 +396,8 @@ public class AzureAsrHelper: NSObject {
*/ */
public func startContinuousRecognition( public func startContinuousRecognition(
audioSourceType: AudioSourceType = .microphone, audioSourceType: AudioSourceType = .microphone,
audioDataCallback: SimpleAudioReceiver.AudioDataCallback? = nil audioDataCallback: SimpleAudioReceiver.AudioDataCallback? = nil,
isRemoveFirstPunctuation: Bool = true
) -> Bool { ) -> Bool {
guard speechConfig != nil else { guard speechConfig != nil else {
print("语音服务未初始化") print("语音服务未初始化")
@ -432,7 +434,7 @@ public class AzureAsrHelper: NSObject {
} }
// 启动讯飞识别 // 启动讯飞识别
xunFeiAsrHelper?.start(callback: callback, language: currentLanguage, isAutoDetect: isAutoDetectLanguage) xunFeiAsrHelper?.start(callback: callback, language: currentLanguage, isAutoDetect: isAutoDetectLanguage, isRemoveFirstPunctuation: isRemoveFirstPunctuation)
// 创建代理回调，将音频数据转发给讯飞 // 创建代理回调，将音频数据转发给讯飞
let proxyCallback = AudioDataCallbackProxy(original: audioDataCallback) { [weak self] data in let proxyCallback = AudioDataCallbackProxy(original: audioDataCallback) { [weak self] data in

5
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift
View File

@ -371,6 +371,8 @@ private func sendAstEvent(_ event: [String: Any]) {
return return
} }
let removeFirstPunctuation = args["isRemoveFirstPunctuation"] as? Bool ?? true
// 根据参数确定音频源类型 // 根据参数确定音频源类型
let audioSourceType = isExternalActive ? let audioSourceType = isExternalActive ?
AzureAsrHelper.AudioSourceType.external : AzureAsrHelper.AudioSourceType.external :
@ -380,7 +382,8 @@ private func sendAstEvent(_ event: [String: Any]) {
// 启动连续识别 // 启动连续识别
let success = azureAsrHelper.startContinuousRecognition( let success = azureAsrHelper.startContinuousRecognition(
audioSourceType: audioSourceType audioSourceType: audioSourceType,
isRemoveFirstPunctuation: removeFirstPunctuation
) )
result(success) result(success)
case "recognizeCallback": case "recognizeCallback":

45
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/XunFeiAsrHelper.swift
View File

@ -20,15 +20,19 @@ class XunFeiAsrHelper: NSObject, URLSessionWebSocketDelegate {
private var currentLanguage = "zh-CN" private var currentLanguage = "zh-CN"
private var isAutoDetectLanguage = false private var isAutoDetectLanguage = false
// 是否第去除一个文字的标点符号
private var removeFirstPunctuation = true
private var callback: AzureAsrHelper.ContinuousRecognizeCallback? private var callback: AzureAsrHelper.ContinuousRecognizeCallback?
func start(callback: AzureAsrHelper.ContinuousRecognizeCallback, language: String, isAutoDetect: Bool) { func start(callback: AzureAsrHelper.ContinuousRecognizeCallback, language: String, isAutoDetect: Bool, isRemoveFirstPunctuation: Bool) {
stop() stop()
self.callback = callback self.callback = callback
self.currentLanguage = language self.currentLanguage = language
self.isAutoDetectLanguage = isAutoDetect self.isAutoDetectLanguage = isAutoDetect
self.removeFirstPunctuation = isRemoveFirstPunctuation
self.currentSessionId = UUID().uuidString self.currentSessionId = UUID().uuidString
self.lastIntermediateResult = "" self.lastIntermediateResult = ""
self.sessionId = "" self.sessionId = ""
@ -115,7 +119,6 @@ class XunFeiAsrHelper: NSObject, URLSessionWebSocketDelegate {
do { do {
guard let json = try JSONSerialization.jsonObject(with: data, options: []) as? [String: Any] else { return } guard let json = try JSONSerialization.jsonObject(with: data, options: []) as? [String: Any] else { return }
print("\(tag): -------------------\(json)")
let msgType = json["msg_type"] as? String let msgType = json["msg_type"] as? String
@ -183,19 +186,35 @@ class XunFeiAsrHelper: NSObject, URLSessionWebSocketDelegate {
for kObj in ws { for kObj in ws {
guard let cw = kObj["cw"] as? [[String: Any]] else { continue } guard let cw = kObj["cw"] as? [[String: Any]] else { continue }
for lObj in cw { if(removeFirstPunctuation) {
let w = lObj["w"] as? String ?? "" // 如果resultTextTemp中第一个文字是标点符号，则去除第一个文字的标点符号
let wp = lObj["wp"] as? String ?? "" for lObj in cw {
let w = lObj["w"] as? String ?? ""
if wp == "p" { let wp = lObj["wp"] as? String ?? ""
punctuationCount += 1
if resultTextTemp.isEmpty { if wp == "p" {
continue punctuationCount += 1
if resultTextTemp.isEmpty {
continue
}
} else {
wordCount += 1
}
resultTextTemp += w
}
} else {
// 不去除resultTextTemp第一个文字的标点符号
for lObj in cw {
let w = lObj["w"] as? String ?? ""
let wp = lObj["wp"] as? String ?? ""
if wp == "p" {
punctuationCount += 1
} else {
wordCount += 1
} }
} else { resultTextTemp += w
wordCount += 1
} }
resultTextTemp += w
} }
} }
} }

Write Preview
Loading…
Cancel
Save