From 71290160636c7306949c552cda4751db274794c8 Mon Sep 17 00:00:00 2001 From: tanlongsheng <252620078@qq.com> Date: Tue, 16 Dec 2025 16:35:27 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=AF=86=E5=88=AB=E7=BB=93?= =?UTF-8?q?=E6=9E=9C=E5=A4=84=E7=90=86=E5=92=8C=E4=BC=9A=E8=AF=9D=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agent_service/AgentService.kt | 56 +++++-------- .../agent_service/AgentServiceImpl.swift | 84 ++++++++++++++----- .../azure_speech/AzureAsrHelper.kt | 2 +- .../Sources/azure_speech/AzureAsrHelper.swift | 8 +- .../azure_speech/AzureSpeechPlugin.swift | 5 +- .../azure_speech/XunFeiAsrHelper.swift | 45 +++++++--- 6 files changed, 125 insertions(+), 75 deletions(-) diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index 70018fff2..cb9265409 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -677,32 +677,18 @@ object AgentService : CoroutineScope { _hasSpeechDetected.set(true) Log.d(TAG, "ai识别中重置定时器") restartIdleCheck() - var text = "" - if (currentRecognitionMode == "push_to_talk") { - text = recognitionResult + recognizing - sendEvent( - "recognizing", mapOf( - "sessionid" to currsessionId, - "text" to text, - "language" to detectedLanguage - ) - ) - }else{ - if(currsessionAsrId.isEmpty()) { - currsessionAsrId = sessionid - } - text = recognitionResult + recognizing - - sendEvent( - "recognizing", mapOf( - "sessionid" to currsessionAsrId, - "text" to text, - "language" to detectedLanguage - ) - ) + if(currsessionAsrId.isEmpty()) { + currsessionAsrId = sessionid } + var text = recognitionResult + recognizing + sendEvent( + "recognizing", mapOf( + "sessionid" to currsessionAsrId, + "text" to text, + "language" to detectedLanguage + ) + ) - // 如果TTS正在播放或AI正在生成，检查是否需要打断 if (isTtsSpeaking || isAiStreaming) { val currentTime = System.currentTimeMillis() @@ -734,20 +720,19 @@ object AgentService : CoroutineScope { Log.d(TAG, "liwei-------- onResult: $sessionid ") if (result.isNotEmpty()&&isKeepResult==true) {//加入是否结束要输出最后的结果判读 // 保存识别结果 + if (currsessionAsrId.isEmpty()) { + currsessionAsrId = sessionid + } + recognitionResult += result if (currentRecognitionMode == "push_to_talk") { //按住说话 - recognitionResult += result sendEvent( "recognizing", mapOf( - "sessionid" to currsessionId, + "sessionid" to currsessionAsrId, "text" to recognitionResult, "language" to detectedLanguage ) ) }else{ - if (currsessionAsrId.isEmpty()) { - currsessionAsrId = sessionid - } - recognitionResult += result pendingProcessJob?.cancel() sendEvent( "recognizing", mapOf( @@ -757,7 +742,7 @@ object AgentService : CoroutineScope { ) ) pendingProcessJob = launch { - delay(2000) + delay(1500) if (recognitionResult.isNotEmpty()) { sendEvent( "recognition_result", mapOf( @@ -785,7 +770,8 @@ object AgentService : CoroutineScope { sendEvent("recognition_started", mapOf("sessionid" to sessionid, "status" to "started")) // 使用模式感知的空闲检测启动方法 recognitionResult = "" - currsessionId = sessionid + currsessionAsrId = "" + pendingProcessJob?.cancel() audioPlayer?.playAudio(R.raw.start) } @@ -797,13 +783,13 @@ object AgentService : CoroutineScope { if (recognitionResult.isNotEmpty()){ sendEvent( "recognition_result", mapOf( - "sessionid" to currsessionId, + "sessionid" to currsessionAsrId, "text" to recognitionResult ) ) - processWithChatApi(currsessionId,recognitionResult) + processWithChatApi(currsessionAsrId, recognitionResult) } - sendEvent("recognition_stopped", mapOf("sessionid" to currsessionId, "status" to "stopped")) + sendEvent("recognition_stopped", mapOf("sessionid" to currsessionAsrId, "status" to "stopped")) }else{ sendEvent("recognition_stopped", mapOf("sessionid" to sessionid, "status" to "stopped")) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index bd9a17a1e..51ad8c897 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -100,6 +100,8 @@ class AgentServiceImpl: NSObject { public var currentRecognitionMode: String = "normal" // 识别结果 private var recognitionResult = "" + private var currsessionAsrId: String = "" + private var pendingProcessWorkItem: DispatchWorkItem? internal var audioPlayer: AudioPlayer? // private var locationInfo:[String: String]? @@ -486,7 +488,8 @@ class AgentServiceImpl: NSObject { print("ai启动语音=\(audioSourceType)=\(useBle)") guard let success = azureAsrHelper?.startContinuousRecognition( // callback: self, - audioSourceType: audioSourceType + audioSourceType: audioSourceType, + isRemoveFirstPunctuation: false ), success else { // 新增：启动失败时复位"启动中"状态 isStartingRecognition = false @@ -2166,15 +2169,49 @@ class AudioPlayer { extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { func onResult(sessionid:String ,_ text: String, _ detectedLanguage: String) { if !text.isEmpty && isKeepResult { + if currsessionAsrId.isEmpty { + currsessionAsrId = sessionid + } + recognitionResult += text if (currentRecognitionMode == "push_to_talk") { //按住说话 - recognitionResult += text + var data: [String: Any] = [ + "sessionid": currsessionAsrId, + "text": recognitionResult + ] + if !detectedLanguage.isEmpty { + data["language"] = detectedLanguage + } + sendEvent(name: "recognizing", data: data) }else{ - var data: [String: Any] = ["sessionid":sessionid,"text": text] + pendingProcessWorkItem?.cancel() + + var data: [String: Any] = [ + "sessionid": currsessionAsrId, + "text": recognitionResult + ] if !detectedLanguage.isEmpty { data["language"] = detectedLanguage } - sendEvent(name: "recognition_result", data: data) - processTextInput(sessionid: sessionid,text, speakResponse: true) + sendEvent(name: "recognizing", data: data) + + let workItem = DispatchWorkItem { [weak self] in + guard let self = self else { return } + if !self.recognitionResult.isEmpty { + var resultData: [String: Any] = [ + "sessionid": self.currsessionAsrId, + "text": self.recognitionResult + ] + if !detectedLanguage.isEmpty { + resultData["language"] = detectedLanguage + } + self.sendEvent(name: "recognition_result", data: resultData) + self.processTextInput(sessionid: self.currsessionAsrId, self.recognitionResult, speakResponse: true) + self.recognitionResult = "" + self.currsessionAsrId = "" + } + } + pendingProcessWorkItem = workItem + DispatchQueue.main.asyncAfter(deadline: .now() + 1.5, execute: workItem) } } @@ -2187,6 +2224,7 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { } func onRecognizing(sessionid:String ,_ recognizing: String, _ detectedLanguage: String) { + pendingProcessWorkItem?.cancel() if !recognizing.isEmpty && isKeepResult { let previousHasSpeech = hasSpeechDetected hasSpeechDetected = true @@ -2195,21 +2233,19 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { print("onRecognizing restartIdleCheck") restartIdleCheck() } - if (currentRecognitionMode == "push_to_talk") { //按住说话 - var data: [String: Any] = ["sessionid":currsessionId,"text": recognitionResult+recognizing] - if !detectedLanguage.isEmpty { - data["language"] = detectedLanguage - } - sendEvent(name: "recognizing", data: data) - }else{ - var data: [String: Any] = ["sessionid":sessionid,"text": recognizing] - if !detectedLanguage.isEmpty { - data["language"] = detectedLanguage - } - sendEvent(name: "recognizing", data: data) + if currsessionAsrId.isEmpty { + currsessionAsrId = sessionid + } + let text = recognitionResult + recognizing + var data: [String: Any] = [ + "sessionid": currsessionAsrId, + "text": text + ] + if !detectedLanguage.isEmpty { + data["language"] = detectedLanguage } + sendEvent(name: "recognizing", data: data) - if isSpeaking || isAiStreaming { interruptCurrentResponse() } @@ -2219,7 +2255,8 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { func onSessionStarted(sessionid:String) { sendEvent(name: "recognition_started", data: ["sessionid":sessionid,"status": "started"]) recognitionResult = "" - currsessionId = sessionid + currsessionAsrId = "" + pendingProcessWorkItem?.cancel() isRecognizing = true hasSpeechDetected = false // 新增：会话真正启动，复位"启动中"状态 @@ -2242,11 +2279,14 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { func onSessionStopped(sessionid:String) { if (currentRecognitionMode == "push_to_talk") { //按住说话 if (!recognitionResult.isEmpty){ - var data: [String: Any] = ["sessionid":currsessionId,"text": recognitionResult] + var data: [String: Any] = [ + "sessionid": currsessionAsrId, + "text": recognitionResult + ] sendEvent(name: "recognition_result", data: data) - processTextInput(sessionid: currsessionId, recognitionResult, speakResponse: true) + processTextInput(sessionid: currsessionAsrId, recognitionResult, speakResponse: true) } - sendEvent(name: "recognition_stopped", data: ["sessionid":currsessionId,"status": "stopped"]) + sendEvent(name: "recognition_stopped", data: ["sessionid":currsessionAsrId,"status": "stopped"]) }else{ sendEvent(name: "recognition_stopped", data: ["sessionid":sessionid,"status": "stopped"]) isRecognizing = false diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt index e96133295..343c5ca00 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt @@ -114,7 +114,7 @@ class AzureAsrHelper(private val context: Context) { this.supportedLanguages = supportedLanguages } - // this.useXunfei = supportedLanguages.all { it == "zh-CN" || it == "en-US" } + this.useXunfei = supportedLanguages.all { it == "zh-CN" || it == "en-US" } // 根据支持的语言数量决定是否启用自动语言检测 this.isAutoDetectLanguage = supportedLanguages.size >= 2 diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift index 25447f8a9..f6e66af2a 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift @@ -116,7 +116,7 @@ public class AzureAsrHelper: NSObject { } // 检查是否只包含中文或英文，如果是则使用讯飞 - // self.useXunfei = supportedLanguages.allSatisfy { $0 == "zh-CN" || $0 == "en-US" } + self.useXunfei = supportedLanguages.allSatisfy { $0 == "zh-CN" || $0 == "en-US" } // 根据支持的语言数量决定是否启用自动语言检测 self.isAutoDetectLanguage = supportedLanguages.count >= 2 @@ -388,6 +388,7 @@ public class AzureAsrHelper: NSObject { * * @param audioSourceType 音频源类型 * @param audioDataCallback 音频数据回调 + * @param isRemoveFirstPunctuation 是否移除第一个标点符号 * @return 是否成功开始识别（调度成功即返回 true） * * 修复点：在音频启动成功后的闭包中，先判断 audioStream?.isContinuousRecognitionActive 是否仍然为 true， @@ -395,7 +396,8 @@ public class AzureAsrHelper: NSObject { */ public func startContinuousRecognition( audioSourceType: AudioSourceType = .microphone, - audioDataCallback: SimpleAudioReceiver.AudioDataCallback? = nil + audioDataCallback: SimpleAudioReceiver.AudioDataCallback? = nil, + isRemoveFirstPunctuation: Bool = true ) -> Bool { guard speechConfig != nil else { print("语音服务未初始化") @@ -432,7 +434,7 @@ public class AzureAsrHelper: NSObject { } // 启动讯飞识别 - xunFeiAsrHelper?.start(callback: callback, language: currentLanguage, isAutoDetect: isAutoDetectLanguage) + xunFeiAsrHelper?.start(callback: callback, language: currentLanguage, isAutoDetect: isAutoDetectLanguage, isRemoveFirstPunctuation: isRemoveFirstPunctuation) // 创建代理回调，将音频数据转发给讯飞 let proxyCallback = AudioDataCallbackProxy(original: audioDataCallback) { [weak self] data in diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift index e55663cc4..194208bd0 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift @@ -371,6 +371,8 @@ private func sendAstEvent(_ event: [String: Any]) { return } + let removeFirstPunctuation = args["isRemoveFirstPunctuation"] as? Bool ?? true + // 根据参数确定音频源类型 let audioSourceType = isExternalActive ? AzureAsrHelper.AudioSourceType.external : @@ -380,7 +382,8 @@ private func sendAstEvent(_ event: [String: Any]) { // 启动连续识别 let success = azureAsrHelper.startContinuousRecognition( - audioSourceType: audioSourceType + audioSourceType: audioSourceType, + isRemoveFirstPunctuation: removeFirstPunctuation ) result(success) case "recognizeCallback": diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/XunFeiAsrHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/XunFeiAsrHelper.swift index b9f703831..9f8dbdde2 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/XunFeiAsrHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/XunFeiAsrHelper.swift @@ -20,15 +20,19 @@ class XunFeiAsrHelper: NSObject, URLSessionWebSocketDelegate { private var currentLanguage = "zh-CN" private var isAutoDetectLanguage = false + + // 是否第去除一个文字的标点符号 + private var removeFirstPunctuation = true private var callback: AzureAsrHelper.ContinuousRecognizeCallback? - func start(callback: AzureAsrHelper.ContinuousRecognizeCallback, language: String, isAutoDetect: Bool) { + func start(callback: AzureAsrHelper.ContinuousRecognizeCallback, language: String, isAutoDetect: Bool, isRemoveFirstPunctuation: Bool) { stop() self.callback = callback self.currentLanguage = language self.isAutoDetectLanguage = isAutoDetect + self.removeFirstPunctuation = isRemoveFirstPunctuation self.currentSessionId = UUID().uuidString self.lastIntermediateResult = "" self.sessionId = "" @@ -115,7 +119,6 @@ class XunFeiAsrHelper: NSObject, URLSessionWebSocketDelegate { do { guard let json = try JSONSerialization.jsonObject(with: data, options: []) as? [String: Any] else { return } - print("\(tag): -------------------\(json)") let msgType = json["msg_type"] as? String @@ -183,19 +186,35 @@ class XunFeiAsrHelper: NSObject, URLSessionWebSocketDelegate { for kObj in ws { guard let cw = kObj["cw"] as? [[String: Any]] else { continue } - for lObj in cw { - let w = lObj["w"] as? String ?? "" - let wp = lObj["wp"] as? String ?? "" - - if wp == "p" { - punctuationCount += 1 - if resultTextTemp.isEmpty { - continue + if(removeFirstPunctuation) { + // 如果resultTextTemp中第一个文字是标点符号，则去除第一个文字的标点符号 + for lObj in cw { + let w = lObj["w"] as? String ?? "" + let wp = lObj["wp"] as? String ?? "" + + if wp == "p" { + punctuationCount += 1 + if resultTextTemp.isEmpty { + continue + } + } else { + wordCount += 1 + } + resultTextTemp += w + } + } else { + // 不去除resultTextTemp第一个文字的标点符号 + for lObj in cw { + let w = lObj["w"] as? String ?? "" + let wp = lObj["wp"] as? String ?? "" + + if wp == "p" { + punctuationCount += 1 + } else { + wordCount += 1 } - } else { - wordCount += 1 + resultTextTemp += w } - resultTextTemp += w } } }