Browse Source

add

newdev_shunjiawei
fdp 1 year ago
parent
commit
c8331fe741
  1. 2
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift
  2. 3
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
  3. 274
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift
  4. 4
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift
  5. 0
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/RecordFile.swift

2
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift

@ -347,7 +347,7 @@ class AgentServiceImpl: NSObject {
return false return false
} }
azureAsrHelper?.audioStream?.saveAudioDataTo(data) azureAsrHelper?.pushAudioData(data: audioData)
return true return true
} }

3
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt

@ -171,7 +171,8 @@ class AzureAsrHelper(private val context: Context) {
} }
setupMicrophoneStream() setupMicrophoneStream()
recognizer?.close()
recognizer = null
//创建识别器 //创建识别器
recognizer = if (isAutoDetectLanguage) { recognizer = if (isAutoDetectLanguage) {
val autoDetectConfig = val autoDetectConfig =

274
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift

@ -31,13 +31,7 @@ public class AzureAsrHelper: NSObject {
private var isAutoDetectLanguage = false private var isAutoDetectLanguage = false
private var subscriptionKey = "" private var subscriptionKey = ""
private var region = "" private var region = ""
// 音频处理
private let audioQueue = DispatchQueue(label: "audio.processing.queue")
private var audioEngine: AVAudioEngine?
private var audioBuffer: Data?
// 录音文件处理
private let recordFile = RecordFile.shared
// 音频源配置 // 音频源配置
public enum AudioSourceType { public enum AudioSourceType {
/** 使用设备麦克风 */ /** 使用设备麦克风 */
@ -51,11 +45,6 @@ public class AzureAsrHelper: NSObject {
// 音频处理 // 音频处理
private var externalAudioStream: ExternalAudioPullStream? private var externalAudioStream: ExternalAudioPullStream?
private var audioStream: AudioStream?
// 当前回调
private weak var currentContinuousCallback: ContinuousRecognizeCallback?
/** /**
* 初始化Azure语音服务 * 初始化Azure语音服务
@ -117,16 +106,28 @@ public class AzureAsrHelper: NSObject {
} }
// 创建识别器 // 创建识别器
return true return setupRecognizer()
} catch { } catch {
os_log("初始化失败: %{public}@", log: log, type: .error, error.localizedDescription) os_log("初始化失败: %{public}@", log: log, type: .error, error.localizedDescription)
return false return false
} }
} }
private fun isRecognizerValid(): Boolean { /**
return recognizer != null * 向音频流写入音频数据
* 仅当音频源设置为external时有效
*
* @param data 音频数据字节数组
*/
public func pushAudioData(data: Data) {
if audioSourceType != .external {
return
}
// 使用拉流模式,将数据推入队列
externalAudioStream?.pushAudio(data)
} }
/** /**
* 开始连续语音识别 * 开始连续语音识别
* *
@ -135,7 +136,7 @@ public class AzureAsrHelper: NSObject {
* @return 是否成功开始识别 * @return 是否成功开始识别
*/ */
public func startContinuousRecognition( public func startContinuousRecognition(
callback: ContinuousRecognizeCallback,
audioSourceType: AudioSourceType = .microphone audioSourceType: AudioSourceType = .microphone
) -> Bool { ) -> Bool {
guard speechConfig != nil else { guard speechConfig != nil else {
@ -146,24 +147,22 @@ public class AzureAsrHelper: NSObject {
if _isContinuousRecognitionActive { if _isContinuousRecognitionActive {
return true return true
} }
if (!isRecognizerValid()) {
Log.w(tag, "识别器已失效,正在重新创建...")
if (!setupRecognizer()) {
Log.e(tag, "重新创建识别器失败")
return false
}
}
self.audioSourceType = audioSourceType self.audioSourceType = audioSourceType
// 重置识别器
if !setupRecognizer() {
callback.onError("重置识别器失败")
return false
}
do { do {
// 设置各种事件监听 // 设置各种事件监听
//setupEventListeners(callback: callback) setupEventListeners(callback: callback)
// 启动音频处理 // 启动音频处理
// startAudioProcessing() startAudioProcessing()
audioStream!!.startAudioRecord()
// 开始连续识别 // 开始连续识别
try recognizer?.startContinuousRecognition() try recognizer?.startContinuousRecognition()
_isContinuousRecognitionActive = true _isContinuousRecognitionActive = true
@ -202,7 +201,7 @@ public class AzureAsrHelper: NSObject {
} }
if audioSourceType == .external { if audioSourceType == .external {
//pushAudioData(data: Data()) pushAudioData(data: Data())
} }
// 停止连续识别 // 停止连续识别
@ -267,38 +266,6 @@ public class AzureAsrHelper: NSObject {
speechConfig = nil speechConfig = nil
} }
} }
// MARK: - 录音功能
/// 开启录音
func enableRecord(filePath: String) {
recordFile.stopRecord(isSave: false)
recordFile.creatingFiles(atPath: filePath)
}
/// 移动文件
func moveFile(sourcePath: String, destPath: String) -> Bool {
return recordFile.moveFile(from: sourcePath, to: destPath)
}
/// 重命名文件
func renameFile(at sourcePath: String, to newName: String) -> Bool {
return recordFile.renameFile(at: sourcePath, to: newName)
}
/// 暂停录音
func pauseRecord() {
recordFile.isPause = true
}
/// 停止录音
func stopRecord(isSave: Bool) {
recordFile.stopRecord(isSave: isSave)
}
/// 输入外部音频数据
func saveAudioDataTo(_ data: Data) {
guard audioSourceType == .external else { return }
writeQueue.offer(data.copyOf())
}
// MARK: - 私有方法 // MARK: - 私有方法
@ -358,32 +325,13 @@ public class AzureAsrHelper: NSObject {
externalAudioStream = nil externalAudioStream = nil
} }
} }
/**
* 设置麦克风流 - 使用推流方式
*/
private func setupMicrophoneStream() {
do {
// 创建麦克风流对象
audioStream = AudioStream()
// 创建音频配置
audioConfig = SPXAudioConfiguration(streamInput: audioStream!.pushAudioStream)
} catch {
os_log("设置麦克风流失败: %{public}@", log: log, type: .error, error.localizedDescription)
audioStream = nil
}
}
/** /**
* 设置事件监听器 * 设置事件监听器
*/ */
private func setupEventListeners(callback: ContinuousRecognizeCallback) { private func setupEventListeners(callback: ContinuousRecognizeCallback) {
guard let recognizer = recognizer else { return } guard let recognizer = recognizer else { return }
// 重置识别器
if !setupRecognizer() {
callback.onError("重置识别器失败")
return false
}
// 识别中事件 // 识别中事件
recognizer.addRecognizingEventHandler { [weak self] (sender, event) in recognizer.addRecognizingEventHandler { [weak self] (sender, event) in
guard let self = self else { return } guard let self = self else { return }
@ -484,171 +432,7 @@ public class AzureAsrHelper: NSObject {
} }
} }
/**
* 麦克风音频流处理
*/
private class AudioStream: NSObject {
private var audioEngine: AVAudioEngine?
private var pushAudioStream: SPXPushAudioInputStream?
private var audioFile: AVAudioFile?
private var isRecording = false
private var recordFilePath: URL?
private let writeQueue = LinkedBlockingQueue<Data>()
override init() {
super.init()
// 创建推流对象
let format = SPXAudioStreamFormat(usingPCMWithSampleRate: 16000, bitsPerSample: 16, channels: 1)!
pushAudioStream = SPXPushAudioInputStream(audioFormat: format)
}
/**
* 开始音频捕获
*/
func startAudioRecord() {
do {
// 配置音频会话
let session = AVAudioSession.sharedInstance()
try session.setCategory(.playAndRecord, mode: .default, options: [.defaultToSpeaker, .allowBluetooth])
try session.setActive(true)
// 初始化音频引擎
audioEngine = AVAudioEngine()
guard let engine = audioEngine else { return }
// 获取输入节点
let inputNode = engine.inputNode
let inputFormat = inputNode.outputFormat(forBus: 0)
// 配置音频格式
let recordingFormat = AVAudioFormat(
commonFormat: .pcmFormatInt16,
sampleRate: 16000,
channels: 1,
interleaved: true
)!
// 安装Tap
inputNode.installTap(
onBus: 0,
bufferSize: 1024,
format: inputFormat
) { [weak self] (buffer, time) in
self?.processAudioBuffer(buffer, format: recordingFormat)
}
// 启动引擎
try engine.start()
} catch {
os_log("音频捕获启动失败: %{public}@", type: .error, error.localizedDescription)
}
}
/**
* 停止音频捕获
*/
func stopCapture() {
audioEngine?.stop()
audioEngine?.inputNode.removeTap(onBus: 0)
audioEngine = nil
stopRecord(isSave: true)
}
/**
* 处理音频缓冲区
*/
private func processAudioBuffer(_ buffer: AVAudioPCMBuffer, format: AVAudioFormat) {
guard let converter = AVAudioConverter(from: buffer.format, to: format) else { return }
// 设置音频配置
switch audioSourceType {
case .microphone:
// 使用默认麦克风输入配置
// 创建目标缓冲区
let targetFrameCapacity = AVAudioFrameCount(
(Double(buffer.frameCapacity) * format.sampleRate / buffer.format.sampleRate
)
guard let targetBuffer = AVAudioPCMBuffer(
pcmFormat: format,
frameCapacity: targetFrameCapacity
) else { return }
// 转换音频格式
var error: NSError?
let inputBlock: AVAudioConverterInputBlock = { inNumPackets, outStatus in
outStatus.pointee = .haveData
return buffer
}
converter.convert(to: targetBuffer, error: &error, withInputFrom: inputBlock)
// 获取音频数据
guard let int16Data = targetBuffer.int16ChannelData else { return }
let data = Data(
bytes: int16Data[0],
count: Int(targetBuffer.frameLength) * MemoryLayout<Int16>.size
)
case .external:
// 使用拉流方式处理外部音频
if (writeQueue.isNotEmpty()) {
let data = writeQueue.poll()
Log.d("tag", "写入数据: ${data?.size}")
bytesToWrite = data?.size ?: 0
}
}
// 推送到Azure流
pushAudioStream?.write(data)
// 保存到录音文件
// 触发音频回调
if let callback = AzureAsrHelper.shared?.currentContinuousCallback {
callback.onAudio(data)
}
}
/**
* 开启录音
*/
func enableRecord(filePath: String) {
do {
recordFilePath = URL(fileURLWithPath: filePath)
let settings: [String: Any] = [
AVFormatIDKey: kAudioFormatLinearPCM,
AVSampleRateKey: 16000.0,
AVNumberOfChannelsKey: 1,
AVEncoderBitDepthHintKey: 16,
AVEncoderAudioQualityKey: AVAudioQuality.high.rawValue
]
audioFile = try AVAudioFile(
forWriting: recordFilePath!,
settings: settings
)
isRecording = true
} catch {
os_log("录音文件创建失败: %{public}@", type: .error, error.localizedDescription)
}
}
/**
* 停止录音
*/
func stopRecord(isSave: Bool) {
isRecording = false
audioFile = nil
if !isSave, let path = recordFilePath {
try? FileManager.default.removeItem(at: path)
}
recordFilePath = nil
}
}
/** /**
* 外部音频拉流 * 外部音频拉流
* 实现PullAudioInputStreamCallback,将外部推送的音频数据转换为SDK可拉取的形式 * 实现PullAudioInputStreamCallback,将外部推送的音频数据转换为SDK可拉取的形式

4
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift

@ -183,8 +183,8 @@ import os.log
result(FlutterError(code: "INVALID_ARGUMENTS", message: "音频数据不能为空", details: nil)) result(FlutterError(code: "INVALID_ARGUMENTS", message: "音频数据不能为空", details: nil))
return return
} }
azureAsrHelper?.audioStream?.saveAudioDataTo(data: audioBytes.data)
azureAsrHelper.pushAudioData(data: audioBytes.data)
result(true) result(true)
default: default:

0
local_plugins/azure_speech/ios/azure_speech/Sources/tools/RecordFile.swift → local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/RecordFile.swift

Loading…
Cancel
Save