liwei1dao
/
deepvoice_client
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Merge commit '2954cafed87c5ad0713752704aa9759003c4a808' into new_dev

newdev_shunjiawei
lxm 1 year ago
parent
70e3c7a9c4 2954cafed8
commit
d4518dd59f
3 changed files with 222 additions and 97 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 18
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift
  2. 8
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/IntegratedSpeechTranslationService.swift
  3. 293
      local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift

18
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift
View File

@ -203,8 +203,8 @@ public class AzureAsrHelper: NSObject {
*/ */
private func preInitializeAudioComponents() { private func preInitializeAudioComponents() {
if audioStream == nil { if audioStream == nil {
audioStream = SimpleAudioReceiver() audioStream = SimpleAudioReceiver.shared
audioStream?.initAudioRecord() //audioStream?.initAudioRecord()
} }
// 预创建音频配置 // 预创建音频配置
if let pushStream = audioStream?.pushAudioStream { if let pushStream = audioStream?.pushAudioStream {
@ -346,14 +346,14 @@ public class AzureAsrHelper: NSObject {
} }
// 停止音频处理 // 停止音频处理
audioStream?.releaseAudioResources() //audioStream?.releaseAudioResources()
// 释放网络监听资源 // 释放网络监听资源
networkMonitor.dispose() networkMonitor.dispose()
// 释放资源 // 释放资源
recognizer = nil recognizer = nil
speechConfig = nil speechConfig = nil
audioConfig = nil audioConfig = nil
audioStream = nil //audioStream = nil
// 确保状态被重置 // 确保状态被重置
_isContinuousRecognitionActive = false _isContinuousRecognitionActive = false
//externalAudioStream = nil //externalAudioStream = nil
@ -466,8 +466,8 @@ public class AzureAsrHelper: NSObject {
private func setupMicrophoneStream() { private func setupMicrophoneStream() {
// 只有在 audioStream 为 nil 时才创建新实例 // 只有在 audioStream 为 nil 时才创建新实例
if audioStream == nil { if audioStream == nil {
audioStream = SimpleAudioReceiver() audioStream = SimpleAudioReceiver.shared
audioStream?.initAudioRecord() // 确保调用初始化 //audioStream?.initAudioRecord() // 确保调用初始化
} }
// 检查音频配置是否已存在 // 检查音频配置是否已存在
@ -669,14 +669,14 @@ public class AzureAsrHelper: NSObject {
if audioStream == nil { if audioStream == nil {
// 创建外部音频拉流对象 // 创建外部音频拉流对象
audioStream = SimpleAudioReceiver() audioStream = SimpleAudioReceiver.shared
audioStream?.initAudioRecord() //audioStream?.initAudioRecord()
} }
if audioStream?.recordfile == nil { if audioStream?.recordfile == nil {
audioStream?.recordfile = RecordFile() audioStream?.recordfile = RecordFile()
} }
// 修复：移除多余的 audioDataCallback 参数 // 修复：移除多余的 audioDataCallback 参数
audioStream?.startAudioRecord( audioStream?.startAudioRecord(

8
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/IntegratedSpeechTranslationService.swift
View File

@ -329,8 +329,8 @@ import os.log
* 初始化音频处理器 * 初始化音频处理器
*/ */
private func initializeAudioProcessor() { private func initializeAudioProcessor() {
audioProcessor = SimpleAudioReceiver() audioProcessor = SimpleAudioReceiver.shared
audioProcessor?.initAudioRecord() // audioProcessor?.initAudioRecord()
// 检查音频配置是否已存在 // 检查音频配置是否已存在
if audioConfig == nil, let pushStream = audioProcessor?.pushAudioStream { if audioConfig == nil, let pushStream = audioProcessor?.pushAudioStream {
audioConfig = SPXAudioConfiguration(streamInput: pushStream) audioConfig = SPXAudioConfiguration(streamInput: pushStream)
@ -913,8 +913,8 @@ import os.log
audioConfig = nil audioConfig = nil
// 释放其他组件 // 释放其他组件
audioProcessor?.releaseAudioResources() //audioProcessor?.releaseAudioResources()
audioProcessor = nil //audioProcessor = nil
translationService?.dispose() translationService?.dispose()
translationService = nil translationService = nil

293
local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift
View File

@ -9,9 +9,32 @@ import os.log
*/ */
public class SimpleAudioReceiver: NSObject { public class SimpleAudioReceiver: NSObject {
/// 单例实例（线程安全）
/// 通过 SimpleAudioReceiver.shared 获取全局唯一实例
public static let shared = SimpleAudioReceiver()
/// 私有化构造函数，防止外部直接实例化
/// 使用方式：通过 SimpleAudioReceiver.shared 访问实例
private override init() {
super.init()
initAudioRecord()
}
private let tag = "SimpleAudioReceiver" private let tag = "SimpleAudioReceiver"
private let log = OSLog(subsystem: "com.azure.speech", category: "SimpleAudioReceiver") private let log = OSLog(subsystem: "com.azure.speech", category: "SimpleAudioReceiver")
/// 音频格式转换专用队列（避免在音频渲染线程上做重操作）
private let conversionQueue = DispatchQueue(label: "audio.stream.convert", qos: .userInitiated)
/// 缓存的音频转换器，避免每个缓冲区重复创建
private var cachedConverter: AVAudioConverter?
/// 统一的目标格式（16kHz/单声道/Float32，用于后续再转 Int16）
private lazy var targetFormat16000Mono: AVAudioFormat = {
return AVAudioFormat(commonFormat: .pcmFormatFloat32,
sampleRate: 16000,
channels: 1,
interleaved: false)!
}()
/** /**
* 音频来源类型 * 音频来源类型
*/ */
@ -25,7 +48,7 @@ public class SimpleAudioReceiver: NSObject {
// MARK: - 音频流类 // MARK: - 音频流类
public private(set) var pushAudioStream: SPXPushAudioInputStream? public private(set) var pushAudioStream: SPXPushAudioInputStream?
private let writeQueue = LinkedBlockingQueue<Data>() private var writeQueue = LinkedBlockingQueue<Data>()
private var audioEngine: AVAudioEngine? private var audioEngine: AVAudioEngine?
private var audioFormat: AVAudioFormat? private var audioFormat: AVAudioFormat?
private let audioSession = AVAudioSession.sharedInstance() private let audioSession = AVAudioSession.sharedInstance()
@ -48,13 +71,15 @@ public class SimpleAudioReceiver: NSObject {
* 包括音频格式、推流、音频引擎等核心组件的初始化 * 包括音频格式、推流、音频引擎等核心组件的初始化
*/ */
public func initAudioRecord() { public func initAudioRecord() {
// 每次初始化前，先释放上一次的资源，避免重复引擎/线程/tap
releaseAudioResources()
print("初始化了") print("初始化了")
audioFormat = getOptimalAudioFormat() audioFormat = getOptimalAudioFormat()
pushAudioStream = SPXPushAudioInputStream() pushAudioStream = SPXPushAudioInputStream()
// 初始化音频引擎 // 初始化音频引擎
audioEngine = AVAudioEngine() audioEngine = AVAudioEngine()
isRunning = true isRunning = true
// 创建新的写线程 // 创建新的写线程
@ -66,15 +91,13 @@ public class SimpleAudioReceiver: NSObject {
// 处理实时音频数据 // 处理实时音频数据
let channelData = buffer.floatChannelData?[0] let channelData = buffer.floatChannelData?[0]
let frameCount = buffer.frameLength let frameCount = buffer.frameLength
print("麦克风启动: \(frameCount)") // print("麦克风启动: \(frameCount)")
print("🎵 音频数据: 帧数=\(frameCount), 声道数据=\(channelData != nil ? "有效" : "无效")") // print("🎵 音频数据: 帧数=\(frameCount), 声道数据=\(channelData != nil ? "有效" : "无效")")
self.writeQueue.put(self.audioBufferToData(buffer)) self.writeQueue.put(self.audioBufferToData(buffer))
// 分析或传输数据... // 分析或传输数据...
} }
} catch { } catch {
print("麦克风启动失败: \(error)") print("麦克风启动失败: \(error)")
} }
} }
@ -192,83 +215,149 @@ public class SimpleAudioReceiver: NSObject {
writeQueue.put(data) writeQueue.put(data)
} }
/** /**
* 开始采集音频数据 * 开始音频捕获
* @param bufferHandler 音频缓冲区处理回调 * 1) 重置音频引擎并配置音频会话
* @throws 音频配置或启动错误 * 2) 安装 tap 时不指定格式（format: nil），由系统使用硬件实际格式
*/ * 3) 在回调内将硬件格式转换为统一的 16kHz 单声道后回调上层
func startCapture(bufferHandler: @escaping (AVAudioPCMBuffer) -> Void) throws { * @param bufferHandler 音频缓冲区处理回调（已转换为目标格式）
* @throws 音频引擎启动失败时抛出错误
*/
guard let audioEngine = audioEngine else { func startCapture(bufferHandler: @escaping (AVAudioPCMBuffer) -> Void) throws {
throw NSError(domain: "SimpleAudioReceiver", code: -1, userInfo: [NSLocalizedDescriptionKey: "Audio engine not initialized"]) guard let audioEngine = audioEngine else {
} throw NSError(domain: "SimpleAudioReceiver", code: -1, userInfo: [NSLocalizedDescriptionKey: "Audio engine not initialized"])
}
// 确保音频引擎完全停止
if audioEngine.isRunning { // 确保音频引擎完全停止
audioEngine.stop() if audioEngine.isRunning {
// 等待一小段时间确保完全停止 audioEngine.stop()
Thread.sleep(forTimeInterval: 0.1) // 等待一小段时间确保完全停止
} Thread.sleep(forTimeInterval: 0.1)
}
// 先安全地移除已存在的 tap
removeTapSafely() // 先安全地移除已存在的 tap
removeTapSafely()
// 重置音频引擎（在获取格式之前）
audioEngine.reset() // 重置音频引擎（在获取格式之前）
try setupAudioSession() audioEngine.reset()
// 重新获取输入节点和格式（reset后需要重新获取）
let inputNode = audioEngine.inputNode // 配置音频会话
let inputFormat = inputNode.outputFormat(forBus: 0) do {
try setupAudioSession()
print("硬件输入格式: 采样率=\(inputFormat.sampleRate)Hz, 声道数=\(inputFormat.channelCount), 格式=\(inputFormat.commonFormat.rawValue)") } catch {
print("音频会话设置失败: \(error.localizedDescription)")
// 验证格式是否有效 throw error
if inputFormat.sampleRate <= 0 || inputFormat.channelCount <= 0 { }
// 如果格式无效，使用默认格式
let defaultFormat = AVAudioFormat(standardFormatWithSampleRate: 48000, channels: 1)! // 重新获取输入节点（此处获取的输出格式在未启动时可能为 0Hz，仅用于日志）
print("使用默认音频格式: 采样率=\(defaultFormat.sampleRate)Hz, 声道数=\(defaultFormat.channelCount)") let inputNode = audioEngine.inputNode
let preFormat = inputNode.outputFormat(forBus: 0)
inputNode.installTap(onBus: 0, print("当前节点(启动前)输出格式: 采样率=\(preFormat.sampleRate)Hz, 声道数=\(preFormat.channelCount), 格式=\(preFormat.commonFormat.rawValue)")
bufferSize: 1024,
format: defaultFormat) { (buffer, time) in // 目标格式 - 统一使用16000Hz单声道（Float32），后续再转 Int16
bufferHandler(buffer) let targetFormat = self.targetFormat16000Mono
} print("使用目标音频格式: 采样率=\(targetFormat.sampleRate)Hz, 声道数=\(targetFormat.channelCount)")
} else {
// 使用硬件原生格式 // 使用硬件原生格式安装 tap（format: nil），回调中异步转换
inputNode.installTap(onBus: 0, inputNode.installTap(onBus: 0,
bufferSize: 1024, bufferSize: 1024,
format: inputFormat) { (buffer, time) in format: nil) { [weak self] (buffer, time) in
bufferHandler(buffer) guard let self = self else { return }
} let srcFormat = buffer.format
// 将重采样与格式转换放到专用队列，避免阻塞音频渲染线程
self.conversionQueue.async {
// 如果源格式与目标格式不同，复用/重建转换器
var outputBuffer: AVAudioPCMBuffer? = buffer
if srcFormat.sampleRate != targetFormat.sampleRate ||
srcFormat.channelCount != targetFormat.channelCount ||
srcFormat.commonFormat != targetFormat.commonFormat {
if self.cachedConverter == nil ||
self.cachedConverter?.inputFormat != srcFormat ||
self.cachedConverter?.outputFormat != targetFormat {
self.cachedConverter = AVAudioConverter(from: srcFormat, to: targetFormat)
}
if let converter = self.cachedConverter {
let capacity = AVAudioFrameCount(Double(buffer.frameLength) * targetFormat.sampleRate / srcFormat.sampleRate)
if let convertedBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) {
var error: NSError?
let status = converter.convert(to: convertedBuffer, error: &error) { _, outStatus in
outStatus.pointee = .haveData
return buffer
}
if status == .error {
// 限制日志：仅在错误时打印，避免频繁输出
print("音频转换失败: \(error?.localizedDescription ?? "未知错误")")
return
}
outputBuffer = convertedBuffer
} else {
print("无法创建转换后的音频缓冲区")
return
}
} else {
print("无法创建音频转换器: 从\(srcFormat.sampleRate)Hz到\(targetFormat.sampleRate)Hz")
return
}
}
// 回调上层（注意：此时不在实时渲染线程）
if let out = outputBuffer {
bufferHandler(out)
}
} }
}
if #available(iOS 13.0, *) {
// 启用语音处理（如果支持）
enableVoiceProcessingIfAvailable(inputNode: inputNode)
// 准备并启动音频引擎
audioEngine.prepare()
// print("音频引擎启动成功。Tap使用硬件格式: 采样率=\(postFormat.sampleRate)Hz, 声道数=\(postFormat.channelCount)")
//print("音频引擎启动成功，使用格式: 采样率=\(tapFormat.sampleRate)Hz, 声道数=\(tapFormat.channelCount)")
}
/**
* 启用语音处理（如果设备支持）
* @param inputNode 输入节点
*/
private func enableVoiceProcessingIfAvailable(inputNode: AVAudioInputNode) {
if #available(iOS 13.0, *) {
do {
try inputNode.setVoiceProcessingEnabled(true) try inputNode.setVoiceProcessingEnabled(true)
print("语音处理已启用")
} catch {
print("启用语音处理失败: \(error.localizedDescription)")
// 不抛出错误，因为这不是关键功能
} }
} else {
audioEngine.prepare() print("当前iOS版本不支持语音处理")
print("音频引擎启动成功")
} }
}
/**
* 安全地移除音频tap
* 避免在移除tap时出现崩溃
*/
/** /**
* 安全地移除音频tap * 安全地移除音频tap
* 避免在移除不存在的tap时出错 * 避免在移除tap时出现崩溃
*/ */
private func removeTapSafely() { private func removeTapSafely() {
guard let audioEngine = audioEngine else { return } guard let audioEngine = audioEngine else { return }
let inputNode = audioEngine.inputNode let inputNode = audioEngine.inputNode
// 尝试移除tap，如果没有tap也不会出错 // 检查是否有tap需要移除
do { if inputNode.numberOfInputs > 0 {
inputNode.removeTap(onBus: 0) do {
print("成功移除音频tap") inputNode.removeTap(onBus: 0)
} catch { print("成功移除音频tap")
// 如果没有tap可移除，这是正常的 } catch {
print("移除tap时出现错误（可能没有tap存在）: \(error)") print("移除音频tap时出错: \(error.localizedDescription)")
}
} }
} }
private func runMicrophoneCapture() { private func runMicrophoneCapture() {
@ -290,8 +379,8 @@ public class SimpleAudioReceiver: NSObject {
// 处理实时音频数据 // 处理实时音频数据
let channelData = buffer.floatChannelData?[0] let channelData = buffer.floatChannelData?[0]
let frameCount = buffer.frameLength let frameCount = buffer.frameLength
print("麦克风启动: \(frameCount)") // print("麦克风启动: \(frameCount)")
print("🎵 音频数据: 帧数=\(frameCount), 声道数据=\(channelData != nil ? "有效" : "无效")") // print("🎵 音频数据: 帧数=\(frameCount), 声道数据=\(channelData != nil ? "有效" : "无效")")
self.writeQueue.put(self.audioBufferToData(buffer)) self.writeQueue.put(self.audioBufferToData(buffer))
// 分析或传输数据... // 分析或传输数据...
} }
@ -311,7 +400,7 @@ public class SimpleAudioReceiver: NSObject {
private func runExternalCapture() { private func runExternalCapture() {
if audioSourceType == .external { if audioSourceType == .external {
//pushAudioData(data: Data()) //pushAudioData(data: Data())
//stopMicrophoneCapture()
audioEngine?.pause() audioEngine?.pause()
//audioEngine?.inputNode.removeTap(onBus: 0) //audioEngine?.inputNode.removeTap(onBus: 0)
} }
@ -344,23 +433,26 @@ public class SimpleAudioReceiver: NSObject {
public func releaseAudioResources() { public func releaseAudioResources() {
print("释放了") print("释放了")
// 停止麦克风捕获 // 停止麦克风捕获（含 _isWriting=false 与移除 tap）
stopMicrophoneCapture() stopMicrophoneCapture()
// 停止音频引擎 // 停止音频引擎
audioEngine?.stop() audioEngine?.stop()
// 关闭写入队列 // 安全关闭旧的写队列以唤醒阻塞的 take()，然后重建一个新的队列
let oldQueue = self.writeQueue
isRunning = false
writeThread?.async { writeThread?.async {
self.writeQueue.close() oldQueue.close()
} }
writeThread = nil writeThread = nil
// 重建队列，避免旧数据残留以及被 close 影响
writeQueue = LinkedBlockingQueue<Data>()
// 安全地移除 tap // 再次安全地移除 tap（幂等）
removeTapSafely() removeTapSafely()
// 重置状态 // 重置状态
isRunning = false
audioEngine = nil audioEngine = nil
} }
// MARK: - 协议定义 // MARK: - 协议定义
@ -429,6 +521,40 @@ private func audioBufferToData(_ buffer: AVAudioPCMBuffer) -> Data {
} }
} }
/**
* 转换音频缓冲区格式
* @param buffer 原始音频缓冲区
* @param targetFormat 目标音频格式
* @return 转换后的音频缓冲区，转换失败时返回nil
*/
private func convertAudioBuffer(_ buffer: AVAudioPCMBuffer, to targetFormat: AVAudioFormat) -> AVAudioPCMBuffer? {
guard let converter = AVAudioConverter(from: buffer.format, to: targetFormat) else {
print("无法创建音频转换器: 从\(buffer.format.sampleRate)Hz到\(targetFormat.sampleRate)Hz")
return nil
}
// 计算转换后的帧数
let capacity = AVAudioFrameCount(Double(buffer.frameLength) * targetFormat.sampleRate / buffer.format.sampleRate)
guard let convertedBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {
print("无法创建转换后的音频缓冲区")
return nil
}
var error: NSError?
let status = converter.convert(to: convertedBuffer, error: &error) { _, outStatus in
outStatus.pointee = .haveData
return buffer
}
if status == .error {
print("音频转换失败: \(error?.localizedDescription ?? "未知错误")")
return nil
}
return convertedBuffer
}
/** /**
* 配置音频会话 * 配置音频会话
* 设置录音和播放所需的音频会话参数 * 设置录音和播放所需的音频会话参数
@ -438,10 +564,10 @@ private func setupAudioSession() throws {
try audioSession.setCategory(.playAndRecord, mode: .default, options: [.defaultToSpeaker, .allowBluetooth]) try audioSession.setCategory(.playAndRecord, mode: .default, options: [.defaultToSpeaker, .allowBluetooth])
try audioSession.setActive(true) try audioSession.setActive(true)
// 设置首选的音频参数 // // 设置首选的音频参数 - 修改为16000Hz以匹配Azure Speech要求
try audioSession.setPreferredSampleRate(48000) // try audioSession.setPreferredSampleRate(16000)
// 设置输入声道数 // // 设置输入声道数
try audioSession.setPreferredInputNumberOfChannels(1) // try audioSession.setPreferredInputNumberOfChannels(1)
print("音频会话配置成功: 采样率=\(audioSession.sampleRate)Hz, 输入声道=\(audioSession.inputNumberOfChannels)") print("音频会话配置成功: 采样率=\(audioSession.sampleRate)Hz, 输入声道=\(audioSession.inputNumberOfChannels)")
} }
@ -574,4 +700,3 @@ private class LinkedBlockingQueue<T> {
} }
} }
} }

Write Preview
Loading…
Cancel
Save