From 867942a65fecae40089876769187c6019a5b3df2 Mon Sep 17 00:00:00 2001 From: fdp <1286779656@qq.com> Date: Thu, 28 Aug 2025 12:09:10 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=BC=80=E9=BA=A6=E7=9A=84?= =?UTF-8?q?=E6=93=8D=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agent_service/AudioSessionHub.swift | 14 +- .../azure_speech/AzureSpeechPlugin.swift | 2 +- .../Sources/tools/MicrophoneCapture.swift | 317 ++++++++++++++ .../Sources/tools/SimpleAudioReceiver.swift | 395 ++++++------------ 4 files changed, 452 insertions(+), 276 deletions(-) create mode 100644 local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AudioSessionHub.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AudioSessionHub.swift index 3f4e5f67b..75d886488 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AudioSessionHub.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AudioSessionHub.swift @@ -639,13 +639,13 @@ public class AudioSessionHub { } private func performVolumeControlWithMPVolumeView(action: VolumeControlAction, value: Float?, completion: @escaping (Bool, String) -> Void) { - // 确保在有音频会话的情况下进行音量控制 - do { - try audioSession.setCategory(.playback, mode: .videoChat, options: [.defaultToSpeaker, .allowBluetooth,.mixWithOthers]) - try audioSession.setActive(true) - } catch { - os_log("设置音频会话失败: %{public}@", log: logger, type: .error, error.localizedDescription) - } + // // 确保在有音频会话的情况下进行音量控制 + // do { + // try audioSession.setCategory(.playback, mode: .videoChat, options: [.defaultToSpeaker, .allowBluetooth,.mixWithOthers]) + // try audioSession.setActive(true) + // } catch { + // os_log("设置音频会话失败: %{public}@", log: logger, type: .error, error.localizedDescription) + // } let volumeView = MPVolumeView(frame: CGRect(x: 0, y: 0, width: 100, height: 100)) volumeView.showsVolumeSlider = true diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift index f068f476f..bc0277785 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift @@ -411,6 +411,7 @@ private func sendAstEvent(_ event: [String: Any]) { return } do { + try azureAsrHelper.stopRecord(isSave: isSave) result(true) } catch { @@ -1041,7 +1042,6 @@ private class AudioDataCallbackImpl: SimpleAudioReceiver.AudioDataCallback { * @param data 音频数据 */ func onAudio(_ data: Data) { - print("音频数据回调方法: \(data.count)") callback(data) } } diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift new file mode 100644 index 000000000..b65d12fd4 --- /dev/null +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/MicrophoneCapture.swift @@ -0,0 +1,317 @@ +import AVFoundation + +class MicrophoneCapture: NSObject { + // MARK: - 单例实例 + /** + * 单例实例，确保整个应用中只有一个音频捕获对象 + * 避免多个实例同时访问麦克风资源导致冲突 + */ + static let shared = MicrophoneCapture() + + private var audioEngine: AVAudioEngine! + private var audioSession: AVAudioSession! + private var audioFormat: AVAudioFormat? + private var audioInputNode: AVAudioInputNode! + private var sampleRate: Double = 16000 // 修改为16000采样率，符合Microsoft要求 + private var isCapturing = false + + // 音频数据回调 + var audioDataHandler: ((Data) -> Void)? + + /** + * 私有初始化方法，防止外部直接创建实例 + * 只能通过shared单例访问 + */ + private override init() { + super.init() + setupAudioSession() + } + + /// 获取最佳音频格式 (iOS 通常支持标准采样率) + public func getOptimalAudioFormat() -> AVAudioFormat? { + let sampleRate: Double = 16000 // Microsoft要求最低16kHz + return AVAudioFormat( + commonFormat: .pcmFormatInt16, // 16位little endian有符号整数 + sampleRate: sampleRate, + channels: 1, + interleaved: true + ) + } + + // MARK: - 音频会话配置 + /** + * 配置音频会话参数 + * 设置音频类别、采样率、缓冲区大小等 + */ + public func setupAudioSession() { + audioFormat = getOptimalAudioFormat() + audioSession = AVAudioSession.sharedInstance() + + do { + // 设置音频会话参数 + try audioSession.setCategory(.playAndRecord, + mode: .videoChat, + options: [.allowBluetooth, + // .allowAirPlay, + // .allowBluetoothA2DP, + //.mixWithOthers, + .defaultToSpeaker + ]) + + try audioSession.setPreferredSampleRate(sampleRate) + try audioSession.setPreferredIOBufferDuration(0.005) // 5ms缓冲 + + // 设置蓝牙设备优先级 + if let bluetoothInput = audioSession.availableInputs?.first(where: { $0.portType == .bluetoothHFP }) { + try audioSession.setPreferredInput(bluetoothInput) + } + + try audioSession.setActive(true) + print("音频会话配置成功") + } catch { + print("音频会话配置失败: \(error.localizedDescription)") + } + } + + // MARK: - 开始捕获 + /** + * 开始音频捕获 + * 检查麦克风权限并启动音频引擎 + */ + public func startCapture() throws { + guard !isCapturing else { return } + print("开始捕获") + // 检查麦克风权限 + switch audioSession.recordPermission { + case .granted: + try setupAudioEngine() + case .denied: + throw NSError(domain: "麦克风权限被拒绝", code: 0) + case .undetermined: + audioSession.requestRecordPermission { granted in + if granted { + do { + try self.setupAudioEngine() + } catch { + print("音频引擎设置失败: \(error.localizedDescription)") + } + } + } + @unknown default: + throw NSError(domain: "未知权限状态", code: 1) + } + } + + /** + * 检查音频格式是否符合Microsoft音频堆栈要求 + * 如果符合则无需转换，直接使用原始格式 + */ + private func isMicrosoftCompatibleFormat(_ format: AVAudioFormat) -> Bool { + // Microsoft支持的采样率：16kHz的整数倍，最低16kHz + let sampleRate = format.sampleRate + guard sampleRate >= 16000 && sampleRate.truncatingRemainder(dividingBy: 16000) == 0 else { + return false + } + + // Microsoft支持的格式 + switch format.commonFormat { + case .pcmFormatFloat32: // 32位IEEE little endian浮点 + return true + case .pcmFormatInt32: // 32位little endian有符号整数 + return true + case .pcmFormatInt16: // 16位little endian有符号整数 + return true + default: + return false + } + } + + /** + * 设置音频引擎和音频处理管道 + * 根据Microsoft音频堆栈要求优化格式转换 + */ + private func setupAudioEngine() throws { + audioEngine = AVAudioEngine() + + audioInputNode = audioEngine.inputNode + + // 获取实际硬件采样率 + sampleRate = audioSession.sampleRate + + // 移除之前的音频处理块，避免重复添加 + audioInputNode.removeTap(onBus: 0) + + // 设置音频格式 + let inputFormat = audioInputNode.inputFormat(forBus: 0) + + // iOS 13+ 启用语音处理 + if #available(iOS 13.0, *) { + try audioInputNode.setVoiceProcessingEnabled(true) + } + + // 检查输入格式是否符合Microsoft要求 + let needsConversion = !isMicrosoftCompatibleFormat(inputFormat) + + // if needsConversion { + // 需要格式转换 + guard let targetFormat = audioFormat, + let converter = AVAudioConverter(from: inputFormat, to: targetFormat) else { + throw NSError(domain: "AudioSetup", code: 2) + } + + print("输入格式不符合Microsoft要求，进行格式转换") + print("输入格式: \(inputFormat.sampleRate)Hz, \(inputFormat.commonFormat.rawValue)") + print("目标格式: \(targetFormat.sampleRate)Hz, \(targetFormat.commonFormat.rawValue)") + + // 添加tap进行格式转换 + audioInputNode.installTap(onBus: 0, + bufferSize: 1024, + format: inputFormat) { [weak self] buffer, when in + + guard let strongSelf = self else { return } + + // 创建目标格式的音频缓冲区 + let convertedBuffer = AVAudioPCMBuffer( + pcmFormat: targetFormat, + frameCapacity: AVAudioFrameCount( + targetFormat.sampleRate * Double(buffer.frameLength) / buffer.format.sampleRate + ) + )! + + var error: NSError? + // 执行音频格式转换 + let status = converter.convert( + to: convertedBuffer, + error: &error, + withInputFrom: { inNumPackets, outStatus in + outStatus.pointee = .haveData + return buffer + } + ) + + // 转换成功且无错误 + if status == .haveData, error == nil { + let data = strongSelf.audioBufferToData(convertedBuffer) + print("转换后数据长度: \(data.count)") + strongSelf.audioDataHandler?(data) + } + } + // } else { + // // 格式已符合Microsoft要求，直接使用 + // print("输入格式符合Microsoft要求，无需转换") + // print("格式: \(inputFormat.sampleRate)Hz, \(inputFormat.commonFormat.rawValue)") + + // audioInputNode.installTap(onBus: 0, + // bufferSize: 1024, + // format: inputFormat) { [weak self] buffer, when in + + // guard let strongSelf = self else { return } + //strongSelf.processAudioBuffer(buffer) + // // 直接处理原始音频数据 + // let data = strongSelf.audioBufferToData(convertedBuffer) + // print("转换后数据长度: \(data.count)") + // strongSelf.audioDataHandler?(data) + + // } + // } + + // 启动引擎 + do { + try audioEngine.start() + isCapturing = true + } catch { + print("音频引擎启动失败: \(error.localizedDescription)") + } + } + + /** + * 将音频缓冲区转换为Data格式 + * 支持16位整数和32位浮点格式 + */ + private func audioBufferToData(_ buffer: AVAudioPCMBuffer) -> Data { + let frameLength = Int(buffer.frameLength) + let channelCount = 1 + let dataLength = frameLength * channelCount * MemoryLayout.size + + // Handle 16-bit integer format + if let int16Data = buffer.int16ChannelData { + return Data( + bytes: int16Data.pointee, + count: dataLength + ) + } + // Handle float format + else if let floatData = buffer.floatChannelData { + var int16Array = [Int16](repeating: 0, count: frameLength) + let floatBuffer = floatData.pointee + + for i in 0...size)) + } + + // 通过回调返回数据 + DispatchQueue.main.async { + print("音频数据回调\(pcmData.count)") + self.audioDataHandler?(pcmData) + } + } + + // MARK: - 停止捕获 + /** + * 停止音频捕获 + * 清理音频引擎和会话资源 + */ + func stopCapture() { + guard isCapturing else { return } + + audioEngine.stop() + audioInputNode.removeTap(onBus: 0) + // try? audioSession.setActive(false) + isCapturing = false + } + + /** + * 重置音频捕获状态 + * 用于单例模式下的状态清理 + */ + func reset() { + stopCapture() + audioDataHandler = nil + } + + deinit { + stopCapture() + } +} \ No newline at end of file diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift index f7e6bced6..e5e4cb2a6 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift @@ -10,24 +10,17 @@ import os.log */ public class SimpleAudioReceiver: NSObject { - // // MARK: - 单例实现 - - // /** - // * 单例实例 - // */ - // public static let shared = SimpleAudioReceiver() - - // /** - // * 私有初始化方法，防止外部创建实例 - // */ - // private override init() { - // super.init() - // initAudioRecord() - // } + private let tag = "SimpleAudioReceiver" private let log = OSLog(subsystem: "com.azure.speech", category: "SimpleAudioReceiver") + /** + * 麦克风捕获实例 + * 使用 MicrophoneCapture 的单例模式 + */ + private var micCapture: MicrophoneCapture! + /** * 音频来源类型 */ @@ -65,18 +58,17 @@ public class SimpleAudioReceiver: NSObject { * 注意：由于是单例模式，此方法可以被多次调用但只会初始化一次 */ public func initAudioRecord() { - // 防止重复初始化 - guard audioEngine == nil else { - print("音频录制组件已初始化，跳过重复初始化") - return - } + + micCapture = MicrophoneCapture.shared + // .audioDataHandler = { data in + // self.writeQueue.offer(data) + // } + print("初始化了") - audioFormat = getOptimalAudioFormat() pushAudioStream = SPXPushAudioInputStream() - // 初始化音频引擎 - audioEngine = AVAudioEngine() + isRunning = true // 创建新的写线程 @@ -84,22 +76,7 @@ public class SimpleAudioReceiver: NSObject { startWriteThread() } - /// 获取最佳音频格式 (iOS 通常支持标准采样率) - public func getOptimalAudioFormat() -> AVAudioFormat? { - let sampleRate: Double = 16000 // iOS 通常支持 16kHz - return AVAudioFormat( - commonFormat: .pcmFormatInt16, - sampleRate: sampleRate, - channels: 1, - interleaved: true - ) - } - - /** - * 设置音频配置 - * @param sampleRate 采样率，默认16000 - * @param channels 声道数，默认1 - */ + /** * 设置音频配置参数 * @param sampleRate 采样率 @@ -165,10 +142,10 @@ public class SimpleAudioReceiver: NSObject { do { if self.audioDataCallback != nil { - print("写入数据长度: \(dataToWrite.count)") self.audioDataCallback!.onAudio(dataToWrite) } if self.pushAudioStream != nil && self.audioDataCallback == nil { + try self.pushAudioStream?.write(dataToWrite) } if self.recordfile != nil { @@ -199,104 +176,21 @@ public class SimpleAudioReceiver: NSObject { writeQueue.put(data) } - // 私有方法：启动麦克风捕获 /** - * 启动麦克风捕获 - * 优化：复用已初始化的音频引擎，减少启动延迟 + * 运行麦克风捕获 */ private func runMicrophoneCapture() { do { - // 【新增】首先配置音频会话 - try audioSession.setCategory( - .playAndRecord, - mode: .videoChat, - options: [.defaultToSpeaker,.allowBluetooth,.mixWithOthers] - ) - - // 【新增】请求麦克风权限（如果尚未授权） - if audioSession.recordPermission != .granted { - audioSession.requestRecordPermission { granted in - if !granted { - print("麦克风权限被拒绝") - } - } - } - - // 【新增】激活音频会话（在配置音频引擎之前） - try audioSession.setActive(true) - - // 检查音频引擎是否已初始化，避免重复创建 - if audioEngine == nil { - audioEngine = AVAudioEngine() - } - - // 如果音频引擎正在运行，先停止 - if audioEngine?.isRunning == true { - audioEngine?.stop() - } - - // 获取音频输入节点（麦克风） - guard let inputNode = audioEngine?.inputNode else { - throw NSError(domain: "AudioSetup", code: 1, userInfo: [NSLocalizedDescriptionKey: "无法获取音频输入节点"]) - } - - // 移除之前的音频处理块，避免重复添加 - inputNode.removeTap(onBus: 0) - - // 获取硬件支持的原始音频格式 - let hardwareFormat = inputNode.inputFormat(forBus: 0) - - // iOS 13+ 启用语音处理 - if #available(iOS 13.0, *) { - try inputNode.setVoiceProcessingEnabled(true) - } - - // 检查目标音频格式和转换器是否可用 - guard let targetFormat = audioFormat, // 外部定义的期望音频格式 - let converter = AVAudioConverter(from: hardwareFormat, to: targetFormat) else { - throw NSError(domain: "AudioSetup", code: 2) - } - - // 在输入节点上安装录音回调 - inputNode.installTap(onBus: 0, - bufferSize: UInt32(bufferSize), // 每次回调的缓冲区大小 - format: hardwareFormat) { // 使用原始硬件格式 - [weak self] buffer, time in // 弱引用避免循环引用 - - // 确保实例存在且正在写入状态 - guard let self = self, self._isWriting else { return } - - // 创建目标格式的音频缓冲区 - let convertedBuffer = AVAudioPCMBuffer( - pcmFormat: targetFormat, - // 计算转换后的帧容量（考虑采样率差异） - frameCapacity: AVAudioFrameCount( - targetFormat.sampleRate * Double(buffer.frameLength) / buffer.format.sampleRate - ) - )! - - var error: NSError? - // 执行音频格式转换 - let status = converter.convert( - to: convertedBuffer, - error: &error, - withInputFrom: { inNumPackets, outStatus in - outStatus.pointee = .haveData // 标记有数据可用 - return buffer // 返回原始音频数据 - } - ) - - // 转换成功且无错误 - if status == .haveData, error == nil { - // 将音频缓冲区转换为二进制数据 - let data = self.audioBufferToData(convertedBuffer) - // 将数据放入写入队列（后续处理） - self.writeQueue.put(data) - } + // 确保正在写入状态 + guard self._isWriting else { return } + try micCapture.startCapture() + try micCapture.audioDataHandler = { data in + self.writeQueue.put(data) } - - - try audioEngine?.start() + // // 将数据放入写入队列（后续处理） + // self.writeQueue.put(data) + + } catch { print("麦克风启动失败: \(error)") // 【新增】添加详细错误处理 @@ -306,44 +200,12 @@ public class SimpleAudioReceiver: NSObject { } } } - private func audioBufferToData(_ buffer: AVAudioPCMBuffer) -> Data { - let frameLength = Int(buffer.frameLength) - let channelCount = 1 - let dataLength = frameLength * channelCount * MemoryLayout.size - - // Handle 16-bit integer format - if let int16Data = buffer.int16ChannelData { - return Data( - bytes: int16Data.pointee, - count: dataLength - ) - } - // Handle float format - else if let floatData = buffer.floatChannelData { - var int16Array = [Int16](repeating: 0, count: frameLength) - let floatBuffer = floatData.pointee - - for i in 0..