liwei1dao
/
deepvoice_client
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

修复识别结果处理和会话管理问题

weicu
tanlongsheng 10 months ago
parent
f8f16bde1d
commit
7129016063
6 changed files with 125 additions and 75 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 56
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  2. 84
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift
  3. 2
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
  4. 8
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift
  5. 5
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift
  6. 45
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/XunFeiAsrHelper.swift

56
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
View File

@ -677,32 +677,18 @@ object AgentService : CoroutineScope {
_hasSpeechDetected.set(true)
Log.d(TAG, "ai识别中重置定时器")
restartIdleCheck()
var text = ""
if (currentRecognitionMode == "push_to_talk") {
text = recognitionResult + recognizing
sendEvent(
"recognizing", mapOf(
"sessionid" to currsessionId,
"text" to text,
"language" to detectedLanguage
)
)
}else{
if(currsessionAsrId.isEmpty()) {
currsessionAsrId = sessionid
}
text = recognitionResult + recognizing
sendEvent(
"recognizing", mapOf(
"sessionid" to currsessionAsrId,
"text" to text,
"language" to detectedLanguage
)
)
if(currsessionAsrId.isEmpty()) {
currsessionAsrId = sessionid
}
var text = recognitionResult + recognizing
sendEvent(
"recognizing", mapOf(
"sessionid" to currsessionAsrId,
"text" to text,
"language" to detectedLanguage
)
)
// 如果TTS正在播放或AI正在生成，检查是否需要打断
if (isTtsSpeaking || isAiStreaming) {
val currentTime = System.currentTimeMillis()
@ -734,20 +720,19 @@ object AgentService : CoroutineScope {
Log.d(TAG, "liwei-------- onResult: $sessionid ")
if (result.isNotEmpty()&&isKeepResult==true) {//加入是否结束要输出最后的结果判读
// 保存识别结果
if (currsessionAsrId.isEmpty()) {
currsessionAsrId = sessionid
}
recognitionResult += result
if (currentRecognitionMode == "push_to_talk") { //按住说话
recognitionResult += result
sendEvent(
"recognizing", mapOf(
"sessionid" to currsessionId,
"sessionid" to currsessionAsrId,
"text" to recognitionResult,
"language" to detectedLanguage
)
)
}else{
if (currsessionAsrId.isEmpty()) {
currsessionAsrId = sessionid
}
recognitionResult += result
pendingProcessJob?.cancel()
sendEvent(
"recognizing", mapOf(
@ -757,7 +742,7 @@ object AgentService : CoroutineScope {
)
)
pendingProcessJob = launch {
delay(2000)
delay(1500)
if (recognitionResult.isNotEmpty()) {
sendEvent(
"recognition_result", mapOf(
@ -785,7 +770,8 @@ object AgentService : CoroutineScope {
sendEvent("recognition_started", mapOf("sessionid" to sessionid, "status" to "started"))
// 使用模式感知的空闲检测启动方法
recognitionResult = ""
currsessionId = sessionid
currsessionAsrId = ""
pendingProcessJob?.cancel()
audioPlayer?.playAudio(R.raw.start)
}
@ -797,13 +783,13 @@ object AgentService : CoroutineScope {
if (recognitionResult.isNotEmpty()){
sendEvent(
"recognition_result", mapOf(
"sessionid" to currsessionId,
"sessionid" to currsessionAsrId,
"text" to recognitionResult
)
)
processWithChatApi(currsessionId,recognitionResult)
processWithChatApi(currsessionAsrId, recognitionResult)
}
sendEvent("recognition_stopped", mapOf("sessionid" to currsessionId, "status" to "stopped"))
sendEvent("recognition_stopped", mapOf("sessionid" to currsessionAsrId, "status" to "stopped"))
}else{
sendEvent("recognition_stopped", mapOf("sessionid" to sessionid, "status" to "stopped"))

84
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift
View File

@ -100,6 +100,8 @@ class AgentServiceImpl: NSObject {
public var currentRecognitionMode: String = "normal"
// 识别结果
private var recognitionResult = ""
private var currsessionAsrId: String = ""
private var pendingProcessWorkItem: DispatchWorkItem?
internal var audioPlayer: AudioPlayer?
// private var locationInfo:[String: String]?
@ -486,7 +488,8 @@ class AgentServiceImpl: NSObject {
print("ai启动语音=\(audioSourceType)=\(useBle)")
guard let success = azureAsrHelper?.startContinuousRecognition(
// callback: self,
audioSourceType: audioSourceType
audioSourceType: audioSourceType,
isRemoveFirstPunctuation: false
), success else {
// 新增：启动失败时复位"启动中"状态
isStartingRecognition = false
@ -2166,15 +2169,49 @@ class AudioPlayer {
extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback {
func onResult(sessionid:String ,_ text: String, _ detectedLanguage: String) {
if !text.isEmpty && isKeepResult {
if currsessionAsrId.isEmpty {
currsessionAsrId = sessionid
}
recognitionResult += text
if (currentRecognitionMode == "push_to_talk") { //按住说话
recognitionResult += text
var data: [String: Any] = [
"sessionid": currsessionAsrId,
"text": recognitionResult
]
if !detectedLanguage.isEmpty {
data["language"] = detectedLanguage
}
sendEvent(name: "recognizing", data: data)
}else{
var data: [String: Any] = ["sessionid":sessionid,"text": text]
pendingProcessWorkItem?.cancel()
var data: [String: Any] = [
"sessionid": currsessionAsrId,
"text": recognitionResult
]
if !detectedLanguage.isEmpty {
data["language"] = detectedLanguage
}
sendEvent(name: "recognition_result", data: data)
processTextInput(sessionid: sessionid,text, speakResponse: true)
sendEvent(name: "recognizing", data: data)
let workItem = DispatchWorkItem { [weak self] in
guard let self = self else { return }
if !self.recognitionResult.isEmpty {
var resultData: [String: Any] = [
"sessionid": self.currsessionAsrId,
"text": self.recognitionResult
]
if !detectedLanguage.isEmpty {
resultData["language"] = detectedLanguage
}
self.sendEvent(name: "recognition_result", data: resultData)
self.processTextInput(sessionid: self.currsessionAsrId, self.recognitionResult, speakResponse: true)
self.recognitionResult = ""
self.currsessionAsrId = ""
}
}
pendingProcessWorkItem = workItem
DispatchQueue.main.asyncAfter(deadline: .now() + 1.5, execute: workItem)
}
}
@ -2187,6 +2224,7 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback {
}
func onRecognizing(sessionid:String ,_ recognizing: String, _ detectedLanguage: String) {
pendingProcessWorkItem?.cancel()
if !recognizing.isEmpty && isKeepResult {
let previousHasSpeech = hasSpeechDetected
hasSpeechDetected = true
@ -2195,21 +2233,19 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback {
print("onRecognizing restartIdleCheck")
restartIdleCheck()
}
if (currentRecognitionMode == "push_to_talk") { //按住说话
var data: [String: Any] = ["sessionid":currsessionId,"text": recognitionResult+recognizing]
if !detectedLanguage.isEmpty {
data["language"] = detectedLanguage
}
sendEvent(name: "recognizing", data: data)
}else{
var data: [String: Any] = ["sessionid":sessionid,"text": recognizing]
if !detectedLanguage.isEmpty {
data["language"] = detectedLanguage
}
sendEvent(name: "recognizing", data: data)
if currsessionAsrId.isEmpty {
currsessionAsrId = sessionid
}
let text = recognitionResult + recognizing
var data: [String: Any] = [
"sessionid": currsessionAsrId,
"text": text
]
if !detectedLanguage.isEmpty {
data["language"] = detectedLanguage
}
sendEvent(name: "recognizing", data: data)
if isSpeaking || isAiStreaming {
interruptCurrentResponse()
}
@ -2219,7 +2255,8 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback {
func onSessionStarted(sessionid:String) {
sendEvent(name: "recognition_started", data: ["sessionid":sessionid,"status": "started"])
recognitionResult = ""
currsessionId = sessionid
currsessionAsrId = ""
pendingProcessWorkItem?.cancel()
isRecognizing = true
hasSpeechDetected = false
// 新增：会话真正启动，复位"启动中"状态
@ -2242,11 +2279,14 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback {
func onSessionStopped(sessionid:String) {
if (currentRecognitionMode == "push_to_talk") { //按住说话
if (!recognitionResult.isEmpty){
var data: [String: Any] = ["sessionid":currsessionId,"text": recognitionResult]
var data: [String: Any] = [
"sessionid": currsessionAsrId,
"text": recognitionResult
]
sendEvent(name: "recognition_result", data: data)
processTextInput(sessionid: currsessionId, recognitionResult, speakResponse: true)
processTextInput(sessionid: currsessionAsrId, recognitionResult, speakResponse: true)
}
sendEvent(name: "recognition_stopped", data: ["sessionid":currsessionId,"status": "stopped"])
sendEvent(name: "recognition_stopped", data: ["sessionid":currsessionAsrId,"status": "stopped"])
}else{
sendEvent(name: "recognition_stopped", data: ["sessionid":sessionid,"status": "stopped"])
isRecognizing = false

2
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
View File

@ -114,7 +114,7 @@ class AzureAsrHelper(private val context: Context) {
this.supportedLanguages = supportedLanguages
}
// this.useXunfei = supportedLanguages.all { it == "zh-CN" || it == "en-US" }
this.useXunfei = supportedLanguages.all { it == "zh-CN" || it == "en-US" }
// 根据支持的语言数量决定是否启用自动语言检测
this.isAutoDetectLanguage = supportedLanguages.size >= 2

8
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift
View File

@ -116,7 +116,7 @@ public class AzureAsrHelper: NSObject {
}
// 检查是否只包含中文或英文，如果是则使用讯飞
// self.useXunfei = supportedLanguages.allSatisfy { $0 == "zh-CN" || $0 == "en-US" }
self.useXunfei = supportedLanguages.allSatisfy { $0 == "zh-CN" || $0 == "en-US" }
// 根据支持的语言数量决定是否启用自动语言检测
self.isAutoDetectLanguage = supportedLanguages.count >= 2
@ -388,6 +388,7 @@ public class AzureAsrHelper: NSObject {
*
* @param audioSourceType 音频源类型
* @param audioDataCallback 音频数据回调
* @param isRemoveFirstPunctuation 是否移除第一个标点符号
* @return 是否成功开始识别（调度成功即返回 true）
*
* 修复点：在音频启动成功后的闭包中，先判断 audioStream?.isContinuousRecognitionActive 是否仍然为 true，
@ -395,7 +396,8 @@ public class AzureAsrHelper: NSObject {
*/
public func startContinuousRecognition(
audioSourceType: AudioSourceType = .microphone,
audioDataCallback: SimpleAudioReceiver.AudioDataCallback? = nil
audioDataCallback: SimpleAudioReceiver.AudioDataCallback? = nil,
isRemoveFirstPunctuation: Bool = true
) -> Bool {
guard speechConfig != nil else {
print("语音服务未初始化")
@ -432,7 +434,7 @@ public class AzureAsrHelper: NSObject {
}
// 启动讯飞识别
xunFeiAsrHelper?.start(callback: callback, language: currentLanguage, isAutoDetect: isAutoDetectLanguage)
xunFeiAsrHelper?.start(callback: callback, language: currentLanguage, isAutoDetect: isAutoDetectLanguage, isRemoveFirstPunctuation: isRemoveFirstPunctuation)
// 创建代理回调，将音频数据转发给讯飞
let proxyCallback = AudioDataCallbackProxy(original: audioDataCallback) { [weak self] data in

5
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift
View File

@ -371,6 +371,8 @@ private func sendAstEvent(_ event: [String: Any]) {
return
}
let removeFirstPunctuation = args["isRemoveFirstPunctuation"] as? Bool ?? true
// 根据参数确定音频源类型
let audioSourceType = isExternalActive ?
AzureAsrHelper.AudioSourceType.external :
@ -380,7 +382,8 @@ private func sendAstEvent(_ event: [String: Any]) {
// 启动连续识别
let success = azureAsrHelper.startContinuousRecognition(
audioSourceType: audioSourceType
audioSourceType: audioSourceType,
isRemoveFirstPunctuation: removeFirstPunctuation
)
result(success)
case "recognizeCallback":

45
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/XunFeiAsrHelper.swift
View File

@ -20,15 +20,19 @@ class XunFeiAsrHelper: NSObject, URLSessionWebSocketDelegate {
private var currentLanguage = "zh-CN"
private var isAutoDetectLanguage = false
// 是否第去除一个文字的标点符号
private var removeFirstPunctuation = true
private var callback: AzureAsrHelper.ContinuousRecognizeCallback?
func start(callback: AzureAsrHelper.ContinuousRecognizeCallback, language: String, isAutoDetect: Bool) {
func start(callback: AzureAsrHelper.ContinuousRecognizeCallback, language: String, isAutoDetect: Bool, isRemoveFirstPunctuation: Bool) {
stop()
self.callback = callback
self.currentLanguage = language
self.isAutoDetectLanguage = isAutoDetect
self.removeFirstPunctuation = isRemoveFirstPunctuation
self.currentSessionId = UUID().uuidString
self.lastIntermediateResult = ""
self.sessionId = ""
@ -115,7 +119,6 @@ class XunFeiAsrHelper: NSObject, URLSessionWebSocketDelegate {
do {
guard let json = try JSONSerialization.jsonObject(with: data, options: []) as? [String: Any] else { return }
print("\(tag): -------------------\(json)")
let msgType = json["msg_type"] as? String
@ -183,19 +186,35 @@ class XunFeiAsrHelper: NSObject, URLSessionWebSocketDelegate {
for kObj in ws {
guard let cw = kObj["cw"] as? [[String: Any]] else { continue }
for lObj in cw {
let w = lObj["w"] as? String ?? ""
let wp = lObj["wp"] as? String ?? ""
if wp == "p" {
punctuationCount += 1
if resultTextTemp.isEmpty {
continue
if(removeFirstPunctuation) {
// 如果resultTextTemp中第一个文字是标点符号，则去除第一个文字的标点符号
for lObj in cw {
let w = lObj["w"] as? String ?? ""
let wp = lObj["wp"] as? String ?? ""
if wp == "p" {
punctuationCount += 1
if resultTextTemp.isEmpty {
continue
}
} else {
wordCount += 1
}
resultTextTemp += w
}
} else {
// 不去除resultTextTemp第一个文字的标点符号
for lObj in cw {
let w = lObj["w"] as? String ?? ""
let wp = lObj["wp"] as? String ?? ""
if wp == "p" {
punctuationCount += 1
} else {
wordCount += 1
}
} else {
wordCount += 1
resultTextTemp += w
}
resultTextTemp += w
}
}
}

Write Preview
Loading…
Cancel
Save