diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift index 8fe11e856..a2a5063bd 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift @@ -57,6 +57,9 @@ public class AzureAsrHelper: NSObject { private var subscriptionKey = "" private var region = "" + private var useXunfei = false + private var xunFeiAsrHelper: XunFeiAsrHelper? + // 音频源配置 public enum AudioSourceType { /** 使用设备麦克风 */ @@ -112,6 +115,9 @@ public class AzureAsrHelper: NSObject { self.supportedLanguages = supportedLanguages } + // 检查是否只包含中文或英文,如果是则使用讯飞 + self.useXunfei = supportedLanguages.allSatisfy { $0 == "zh-CN" || $0 == "en-US" } + // 根据支持的语言数量决定是否启用自动语言检测 self.isAutoDetectLanguage = supportedLanguages.count >= 2 @@ -413,6 +419,43 @@ public class AzureAsrHelper: NSObject { self.audioSourceType = audioSourceType print("startContinuousRecognition:准备启动") + // 如果使用讯飞识别 + if useXunfei { + print("Using Xunfei Recognition") + guard let callback = self.continuousCallback else { + print("无法启动连续识别:回调为空") + return false + } + + if xunFeiAsrHelper == nil { + xunFeiAsrHelper = XunFeiAsrHelper() + } + + // 启动讯飞识别 + xunFeiAsrHelper?.start(callback: callback, language: currentLanguage, isAutoDetect: isAutoDetectLanguage) + + // 创建代理回调,将音频数据转发给讯飞 + let proxyCallback = AudioDataCallbackProxy(original: audioDataCallback) { [weak self] data in + self?.xunFeiAsrHelper?.sendAudio(frameBuffer: data) + } + + // 异步启动音频处理 + startAudioRecordAsync(audioSourceType: audioSourceType, audioDataCallback: proxyCallback) { [weak self] (success: Bool) in + guard let self = self else { return } + + if success { + self.audioStream?.isContinuousRecognitionActive = true + os_log("讯飞连续识别已启动", log: self.log, type: .info) + } else { + os_log("音频启动失败,无法开始讯飞识别", log: self.log, type: .error) + self.xunFeiAsrHelper?.stop() + self.audioStream?.isContinuousRecognitionActive = false + } + } + + return true + } + do { if recognizer == nil { // 检查continuousCallback是否为空,只有在不为空时才设置事件监听器 @@ -464,6 +507,23 @@ public class AzureAsrHelper: NSObject { * - 返回: 停止操作是否成功 */ public func stopContinuousRecognition() -> Bool { + if useXunfei { + xunFeiAsrHelper?.stop() + + // 关键:取消尚未执行的启动任务,防止 stop 后音频又被启动 + cancelPendingAudioStart() + + stopAudioRecordAsync { [weak self] (success: Bool) in + guard let self = self else { return } + if success { + self.audioStream?.isContinuousRecognitionActive = false + os_log("讯飞音频停止成功", log: self.log, type: .info) + } + } + self.audioStream?.isContinuousRecognitionActive = false + return true + } + guard speechConfig != nil else { os_log("语音服务未初始化", log: log, type: .error) return false @@ -545,8 +605,12 @@ public class AzureAsrHelper: NSObject { print("释放所有资源:") // 如果正在进行连续识别,先停止 if audioStream?.isContinuousRecognitionActive == true { - // 直接停止,不等待结果 - try? recognizer?.stopContinuousRecognition() + if useXunfei { + xunFeiAsrHelper?.stop() + } else { + // 直接停止,不等待结果 + try? recognizer?.stopContinuousRecognition() + } audioStream?.isContinuousRecognitionActive = false } // 取消防抖任务 - 只取消开始操作的防抖 @@ -560,6 +624,8 @@ public class AzureAsrHelper: NSObject { speechConfig = nil audioConfig = nil audioStream = nil + xunFeiAsrHelper?.stop() + xunFeiAsrHelper = nil // 确保状态被重置 audioStream?.isContinuousRecognitionActive = false //externalAudioStream = nil @@ -739,6 +805,9 @@ public class AzureAsrHelper: NSObject { self.supportedLanguages = [validatedSourceLang, validatedTargetLang] self.currentLanguage = validatedSourceLang + // 更新是否使用讯飞 + self.useXunfei = supportedLanguages.allSatisfy { $0 == "zh-CN" || $0 == "en-US" } + // 重新配置语言检测 self.isAutoDetectLanguage = (validatedSourceLang != validatedTargetLang) @@ -1084,7 +1153,7 @@ extension AzureAsrHelper: NetworkStateMonitor.NetworkStateListener { self.handleNetworkLost() } - // .MARK: - 启动任务管理 + // MARK: - 启动任务管理 /// 取消尚未执行的音频启动任务,防止 stop 后仍然触发启动 private func cancelPendingAudioStart() { audioStartWorkItem?.cancel() @@ -1093,6 +1162,22 @@ extension AzureAsrHelper: NetworkStateMonitor.NetworkStateListener { } +// MARK: - AudioDataCallback Proxy +private class AudioDataCallbackProxy: SimpleAudioReceiver.AudioDataCallback { + private let originalCallback: SimpleAudioReceiver.AudioDataCallback? + private let xunfeiCallback: ((Data) -> Void)? + + init(original: SimpleAudioReceiver.AudioDataCallback?, xunfei: ((Data) -> Void)?) { + self.originalCallback = original + self.xunfeiCallback = xunfei + } + + func onAudio(_ data: Data) { + originalCallback?.onAudio(data) + xunfeiCallback?(data) + } +} + diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/XunFeiAsrHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/XunFeiAsrHelper.swift new file mode 100644 index 000000000..19aa162fe --- /dev/null +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/XunFeiAsrHelper.swift @@ -0,0 +1,280 @@ +import Foundation +import CryptoKit + +class XunFeiAsrHelper: NSObject, URLSessionWebSocketDelegate { + private let tag = "XunFeiAsrHelper" + + private let appId = "43d87d22" + private let accessKeyId = "2bd01ce06ac35957fc0c082f9b6284ec" + private let accessKeySecret = "ODU3OTUyYjhjMjY5NDhkMmMxYWY2Zjlm" + + private var webSocketTask: URLSessionWebSocketTask? + private lazy var session: URLSession = { + let configuration = URLSessionConfiguration.default + return URLSession(configuration: configuration, delegate: self, delegateQueue: OperationQueue()) + }() + + private var currentSessionId = "" + private var sessionId = "" + private var lastIntermediateResult = "" + + private var currentLanguage = "zh-CN" + private var isAutoDetectLanguage = false + + private var callback: AzureAsrHelper.ContinuousRecognizeCallback? + + func start(callback: AzureAsrHelper.ContinuousRecognizeCallback, language: String, isAutoDetect: Bool) { + stop() + + self.callback = callback + self.currentLanguage = language + self.isAutoDetectLanguage = isAutoDetect + self.currentSessionId = UUID().uuidString + self.lastIntermediateResult = "" + self.sessionId = "" + + let urlString = getWebSocketUrl() + guard let url = URL(string: urlString) else { + print("\(tag): Invalid URL") + return + } + + let request = URLRequest(url: url) + webSocketTask = session.webSocketTask(with: request) + webSocketTask?.resume() + + print("\(tag): WebSocket Connecting...") + receiveMessage() + } + + func sendAudio(frameBuffer: Data) { + guard let webSocketTask = webSocketTask else { return } + let message = URLSessionWebSocketTask.Message.data(frameBuffer) + webSocketTask.send(message) { error in + if let error = error { + print("\(self.tag): Send audio failed: \(error)") + } + } + } + + func stop() { + if let webSocketTask = webSocketTask { + let endJson = "{\"end\": true, \"sessionId\": \"\(sessionId)\"}" + let message = URLSessionWebSocketTask.Message.string(endJson) + webSocketTask.send(message) { _ in } + webSocketTask.cancel(with: .normalClosure, reason: "User stopped".data(using: .utf8)) + } + webSocketTask = nil + } + + private func receiveMessage() { + webSocketTask?.receive { [weak self] result in + guard let self = self else { return } + + switch result { + case .failure(let error): + print("\(self.tag): WebSocket Receive Error: \(error)") + self.callback?.onError(sessionid: self.currentSessionId, 1004, "Xunfei connection failed: \(error.localizedDescription)") + + case .success(let message): + switch message { + case .string(let text): + self.processMessage(text) + case .data(let data): + if let text = String(data: data, encoding: .utf8) { + self.processMessage(text) + } + @unknown default: + break + } + + // Continue receiving messages + self.receiveMessage() + } + } + } + + func urlSession(_ session: URLSession, webSocketTask: URLSessionWebSocketTask, didOpenWithProtocol protocol: String?) { + print("\(tag): WebSocket Opened") + callback?.onSessionStarted(sessionid: currentSessionId) + } + + func urlSession(_ session: URLSession, webSocketTask: URLSessionWebSocketTask, didCloseWith closeCode: URLSessionWebSocketTask.CloseCode, reason: Data?) { + let reasonString = reason.flatMap { String(data: $0, encoding: .utf8) } ?? "Unknown" + print("\(tag): WebSocket Closed: \(closeCode) / \(reasonString)") + + if !currentSessionId.isEmpty && !lastIntermediateResult.isEmpty { + let detectedLanguage = detectLanguage(from: lastIntermediateResult) + callback?.onResult(sessionid: currentSessionId, lastIntermediateResult, detectedLanguage) + currentSessionId = "" + } + callback?.onSessionStopped(sessionid: currentSessionId) + } + + private func processMessage(_ text: String) { + guard let data = text.data(using: .utf8) else { return } + + do { + guard let json = try JSONSerialization.jsonObject(with: data, options: []) as? [String: Any] else { return } + print("\(tag): -------------------\(json)") + + let msgType = json["msg_type"] as? String + + switch msgType { + case "action": + if let sid = json["sessionId"] as? String, !sid.isEmpty { + self.sessionId = sid + } + print("\(tag): Handshake success") + + case "result": + processResult(json) + + case "error": + let errorMsg = "\(json)" + print("\(tag): Error: \(errorMsg)") + callback?.onError(sessionid: currentSessionId, 1005, "Xunfei error: \(errorMsg)") + + default: + print("\(tag): Unknown msg_type: \(msgType ?? "nil")") + callback?.onError(sessionid: currentSessionId, 1006, "Xunfei unknown msg_type: \(msgType ?? "nil")") + } + + } catch { + print("\(tag): Parse error: \(error)") + } + } + + private func processResult(_ json: [String: Any]) { + guard let dataStr = json["data"] as? String, + let dataData = dataStr.data(using: .utf8), + let data = try? JSONSerialization.jsonObject(with: dataData, options: []) as? [String: Any] else { + return + } + + let ls = data["ls"] as? Bool ?? false + guard let cn = data["cn"] as? [String: Any], + let st = cn["st"] as? [String: Any], + let rt = st["rt"] as? [[String: Any]] else { + return + } + + var resultTextTemp = "" + var punctuationCount = 0 + var wordCount = 0 + + for j in rt { + guard let ws = j["ws"] as? [[String: Any]] else { continue } + for kObj in ws { + guard let cw = kObj["cw"] as? [[String: Any]] else { continue } + + for lObj in cw { + let w = lObj["w"] as? String ?? "" + let wp = lObj["wp"] as? String ?? "" + + if wp == "p" { + punctuationCount += 1 + if resultTextTemp.isEmpty { + continue + } + } else { + wordCount += 1 + } + resultTextTemp += w + } + } + } + + if ls && wordCount == 0 && punctuationCount == 1 { + return + } + + let detectedLanguage = detectLanguage(from: resultTextTemp) + + let type = st["type"] as? String + let typeInt = st["type"] as? Int ?? Int(type ?? "") ?? -1 + + if typeInt == 0 { + callback?.onResult(sessionid: currentSessionId, resultTextTemp, detectedLanguage) + currentSessionId = "" + lastIntermediateResult = "" + } else { + if currentSessionId.isEmpty { + currentSessionId = UUID().uuidString + callback?.onSessionStarted(sessionid: currentSessionId) + } + lastIntermediateResult = resultTextTemp + callback?.onRecognizing(sessionid: currentSessionId, resultTextTemp, detectedLanguage) + } + } + + private func detectLanguage(from text: String) -> String { + if isAutoDetectLanguage { + if text.range(of: "[\\u4e00-\\u9fa5]", options: .regularExpression) != nil { + return "zh-CN" + } else { + return "en-US" + } + } else { + return currentLanguage + } + } + + private func getWebSocketUrl() -> String { + let baseWsUrl = "wss://office-api-ast-dx.iflyaisol.com/ast/communicate/v1" + + var params: [String: String] = [:] + params["audio_encode"] = "pcm_s16le" + params["lang"] = "autodialect" + params["samplerate"] = "16000" + params["accessKeyId"] = accessKeyId + params["appId"] = appId + params["utc"] = getUtcTime() + + let signature = calculateSignature(params: params) + params["signature"] = signature + + var urlComponents = URLComponents(string: baseWsUrl) + urlComponents?.queryItems = params.map { URLQueryItem(name: $0.key, value: $0.value) } + + return urlComponents?.url?.absoluteString ?? baseWsUrl + } + + private func getUtcTime() -> String { + let dateFormatter = DateFormatter() + dateFormatter.dateFormat = "yyyy-MM-dd'T'HH:mm:ssZ" + dateFormatter.timeZone = TimeZone(secondsFromGMT: 8 * 3600) // GMT+8 + return dateFormatter.string(from: Date()) + } + + private func calculateSignature(params: [String: String]) -> String { + let sortedKeys = params.keys.sorted() + var baseStr = "" + var first = true + + for key in sortedKeys { + if key == "signature" { continue } + guard let value = params[key], !value.isEmpty else { continue } + + if !first { + baseStr += "&" + } + + let encodedKey = key.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? key + let encodedValue = value.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? value + + baseStr += "\(encodedKey)=\(encodedValue)" + first = false + } + + let keyData = accessKeySecret.data(using: .utf8)! + let msgData = baseStr.data(using: .utf8)! + + if #available(iOS 13.0, *) { + let authentication = HMAC.authenticationCode(for: msgData, using: SymmetricKey(data: keyData)) + return Data(authentication).base64EncodedString() + } else { + return "" + } + } +}