liwei1dao
/
deepvoice_client
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

加入录音音波动画

weicu
fdp 1 year ago
parent
eeeca0e993
commit
929f5caf18
3 changed files with 227 additions and 158 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/SimpleAudioReceiver.kt
  2. 376
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift
  3. 6
      local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift

3
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/SimpleAudioReceiver.kt
View File

@ -145,8 +145,9 @@ class SimpleAudioReceiver(private val context: Context) {
}
pushAudioStream?.write(finalData)
if(recordfile!=null){
recordfile?.saveAudioDataToWav(finalData)
}
} catch (e: Exception) {
Log.e(TAG, "写入失败: ${e.message}")

376
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift
View File

@ -20,16 +20,40 @@ import ble_service
// 日志对象
private let log = OSLog(subsystem: "com.azure.speech", category: "AzureSpeechPlugin")
// 替换原有的 StreamHandler 类
private typealias AsrEventStreamHandler = BaseEventStreamHandler
private typealias TtsEventStreamHandler = BaseEventStreamHandler
private typealias AudioDataEventStreamHandler = BaseEventStreamHandler
private typealias AstEventStreamHandler = BaseEventStreamHandler
// 事件类型枚举
public enum EventType {
case asr
case tts
case audioData
case ast
}
/**
* 通道配置结构体
*/
private struct ChannelConfig {
let methodChannelName: String
let eventChannelName: String
let eventType: EventType
let methodHandler: (FlutterMethodCall, @escaping FlutterResult) -> Void
}
// ASR相关
private var asrChannel: FlutterMethodChannel?
private var asrEventChannel: FlutterEventChannel?
private var asrEventSink: FlutterEventSink?
internal var asrEventSink: FlutterEventSink?
private let azureAsrHelper = AzureAsrHelper()
// TTS相关
private var ttsChannel: FlutterMethodChannel?
private var ttsEventChannel: FlutterEventChannel?
private var ttsEventSink: FlutterEventSink?
internal var ttsEventSink: FlutterEventSink?
private let azureTtsHelper = AzureTtsHelper()
// 是否已添加TTS事件监听器
@ -37,98 +61,152 @@ import ble_service
// AST相关
private var astChannel: FlutterMethodChannel?
private var astEventChannel: FlutterEventChannel?
internal var astEventSink: FlutterEventSink?
private let azureAstHelper = IntegratedSpeechTranslationService()
// 音频数据相关
private var audioDataEventChannel: FlutterEventChannel?
internal var audioDataEventSink: FlutterEventSink?
// 创建事件回调
private var astEventCallback: AstEventCallback?
// 是否已添加AST事件监听器
private var isAstListenerAdded = false
// 当前的连续识别回调
private var currentAsrCallback: AsrCallbackWrapper?
// 插件注册
public static func register(with registrar: FlutterPluginRegistrar) {
let instance = AzureSpeechPlugin()
// 2. 初始化BleService并注册回调
BleService.shared.addCallback(instance)
// 初始化ASR通道
let asrChannel = FlutterMethodChannel(name: "azure_speech/asr", binaryMessenger: registrar.messenger())
registrar.addMethodCallDelegate(instance, channel: asrChannel)
instance.asrChannel = asrChannel
// 设置ASR通道处理器 - 使用实例方法
asrChannel.setMethodCallHandler { [weak instance] (call, result) in
instance?.handleAsrMethodCall(call, result: result)
// 添加音频数据回调实例作为类属性
private lazy var audioDataCallback: AudioDataCallbackImpl = {
return AudioDataCallbackImpl { [weak self] data in
self?.handleAudioData(data)
}
// 初始化TTS通道
let ttsChannel = FlutterMethodChannel(name: "azure_speech/tts", binaryMessenger: registrar.messenger())
registrar.addMethodCallDelegate(instance, channel: ttsChannel)
instance.ttsChannel = ttsChannel
// 设置TTS通道处理器 - 使用实例方法
ttsChannel.setMethodCallHandler { [weak instance] (call, result) in
instance?.handleTtsMethodCall(call, result: result)
}()
/**
* 初始化通道的通用方法
* @param registrar Flutter插件注册器
* @param config 通道配置
*/
private func setupChannel(registrar: FlutterPluginRegistrar, config: ChannelConfig) {
// 初始化方法通道
let methodChannel = FlutterMethodChannel(name: config.methodChannelName, binaryMessenger: registrar.messenger())
methodChannel.setMethodCallHandler { [weak self] (call, result) in
config.methodHandler(call, result)
}
// 初始化事件通道
let eventChannel = FlutterEventChannel(name: config.eventChannelName, binaryMessenger: registrar.messenger())
eventChannel.setStreamHandler(BaseEventStreamHandler(instance: self, eventType: config.eventType))
// 初始化AST通道
let astChannel = FlutterMethodChannel(name: "azure_speech/ast", binaryMessenger: registrar.messenger())
registrar.addMethodCallDelegate(instance, channel: astChannel)
instance.astChannel = astChannel
// 设置AST通道处理器 - 使用实例方法
astChannel.setMethodCallHandler { [weak instance] (call, result) in
instance?.handleAstMethodCall(call, result: result)
// 保存通道引用
switch config.eventType {
case .asr:
self.asrChannel = methodChannel
self.asrEventChannel = eventChannel
case .tts:
self.ttsChannel = methodChannel
self.ttsEventChannel = eventChannel
case .audioData:
self.audioDataEventChannel = eventChannel
case .ast:
self.astChannel = methodChannel
self.astEventChannel = eventChannel
}
// 初始化ASR事件通道
let asrEventChannel = FlutterEventChannel(name: "azure_speech/asr_events", binaryMessenger: registrar.messenger())
asrEventChannel.setStreamHandler(instance)
instance.asrEventChannel = asrEventChannel
// 初始化TTS事件通道
let ttsEventChannel = FlutterEventChannel(name: "azure_speech/tts_events", binaryMessenger: registrar.messenger())
ttsEventChannel.setStreamHandler(instance)
instance.ttsEventChannel = ttsEventChannel
// 初始化TTS事件通道
let astEventChannel = FlutterEventChannel(name: "azure_speech/asr_events", binaryMessenger: registrar.messenger())
astEventChannel.setStreamHandler(instance)
instance.astEventChannel = astEventChannel
}
// 发送ASR事件方法
internal func sendAsrEvent(_ event: [String: Any]) {
if asrEventSink == nil {
os_log("无法发送ASR事件：事件通道未准备好", log: log, type: .error)
return
}
DispatchQueue.main.async { [weak self] in
guard let self = self else { return }
self.asrEventSink?(event)
}
/**
* 统一的事件发送方法
* @param event 事件数据
* @param type 事件类型
*/
private func sendEvent(_ event: [String: Any], type: EventType) {
let eventSink: FlutterEventSink?
let logMessage: String
switch type {
case .asr:
eventSink = asrEventSink
logMessage = "无法发送ASR事件：事件通道未准备好"
case .tts:
eventSink = ttsEventSink
logMessage = "无法发送TTS事件：事件通道未准备好"
case .audioData:
eventSink = audioDataEventSink
logMessage = "无法发送音频数据事件：事件通道未准备好"
case .ast:
eventSink = astEventSink
logMessage = "无法发送AST事件：事件通道未准备好"
}
// 发送TTS事件方法
private func sendTtsEvent(_ event: [String: Any]) {
if ttsEventSink == nil {
os_log("无法发送TTS事件：事件通道未准备好", log: log, type: .error)
return
}
guard eventSink != nil else {
os_log("%@", log: log, type: .error, logMessage)
return
}
DispatchQueue.main.async { [weak self] in
guard let self = self else { return }
eventSink?(event)
}
}
// 插件注册
public static func register(with registrar: FlutterPluginRegistrar) {
let instance = AzureSpeechPlugin()
BleService.shared.addCallback(instance)
DispatchQueue.main.async { [weak self] in
guard let self = self else { return }
self.ttsEventSink?(event)
// 配置所有通道
let channelConfigs = [
ChannelConfig(
methodChannelName: "azure_speech/asr",
eventChannelName: "azure_speech/asr_events",
eventType: .asr,
methodHandler: instance.handleAsrMethodCall
),
ChannelConfig(
methodChannelName: "azure_speech/tts",
eventChannelName: "azure_speech/tts_events",
eventType: .tts,
methodHandler: instance.handleTtsMethodCall
),
ChannelConfig(
methodChannelName: "azure_speech/ast",
eventChannelName: "azure_speech/ast_events",
eventType: .ast,
methodHandler: instance.handleAstMethodCall
)
]
// 批量初始化通道
for config in channelConfigs {
instance.setupChannel(registrar: registrar, config: config)
}
// 单独处理音频数据事件通道（只有事件通道）
let audioDataEventChannel = FlutterEventChannel(name: "azure_speech/audio_data_events", binaryMessenger: registrar.messenger())
audioDataEventChannel.setStreamHandler(BaseEventStreamHandler(instance: instance, eventType: .audioData))
instance.audioDataEventChannel = audioDataEventChannel
}
// 简化的事件发送方法
internal func sendAsrEvent(_ event: [String: Any]) {
sendEvent(event, type: .asr)
}
private func sendTtsEvent(_ event: [String: Any]) {
sendEvent(event, type: .tts)
}
private func sendAudioDataEvent(_ event: [String: Any]) {
sendEvent(event, type: .audioData)
}
private func sendAstEvent(_ event: [String: Any]) {
sendEvent(event, type: .ast)
}
// 设置TTS事件监听器
private func setupTtsEventListener() {
internal func setupTtsEventListener() {
if !isTtsListenerAdded {
azureTtsHelper.addListener(self)
isTtsListenerAdded = true
@ -136,18 +214,25 @@ import ble_service
}
// // 处理Flutter方法调用
// public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) {
// switch call.method {
// case "azure_speech/asr":
// handleAsrMethodCall(call, result)
// case "azure_speech/tts":
// handleTtsMethodCall(call, result)
// default:
// // 处理未识别的通道
// result(FlutterMethodNotImplemented)
// }
// }
/**
* 处理音频数据
* @param data 音频数据
*/
private func handleAudioData(_ data: Data) {
guard audioDataEventSink != nil else {
print("音频数据事件通道未准备好")
return
}
let eventMap: [String: Any] = [
"type": "audioData",
"data": FlutterStandardTypedData(bytes: data),
"timestamp": Int(Date().timeIntervalSince1970 * 1000),
"size": data.count
]
print("音频数据大小: \(data.count)")
sendAudioDataEvent(eventMap)
}
// MARK: - ASR 方法处理
@ -299,19 +384,8 @@ import ble_service
do {
if acceptAudioData {
// 创建音频数据回调实现类
let audioCallback = AudioDataCallbackImpl { [weak self] audioData in
// 构建音频数据事件映射
let audioEvent: [String: Any] = [
"type": "audioData",
"data": audioData,
"timestamp": Date().timeIntervalSince1970 * 1000,
"size": audioData.count
]
// 发送音频数据事件到 Flutter 层
self?.sendAsrEvent(audioEvent)
}
try azureAsrHelper.enableRecord(filePath: filePath, audioDataCallback: audioCallback)
// 使用复用的音频数据回调实例
try azureAsrHelper.enableRecord(filePath: filePath, audioDataCallback: audioDataCallback)
} else {
try azureAsrHelper.enableRecord(filePath: filePath)
}
@ -325,9 +399,10 @@ import ble_service
case "pauseRecord":
azureAsrHelper.pauseRecord()
result(true)
// case "resumeRecord":
// azureAsrHelper.resumeRecord()
// result(true)
case "resumeRecord":
azureAsrHelper.resumeRecord()
result(true)
case "stopRecord":
guard let args = call.arguments as? [String: Any],
@ -851,57 +926,60 @@ extension AzureSpeechPlugin: BleService.Callback {
public func onDeviceInfoReceived(infoType: Int, infoData: [String: Any]) {
}
}
// MARK: - 事件处理
extension AzureSpeechPlugin: FlutterStreamHandler {
public func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? {
// 根据通道类型设置事件接收器
if let args = arguments as? [String: Any],
let channel = args["channel"] as? String {
if channel == "asr" {
asrEventSink = events
} else if channel == "tts" {
ttsEventSink = events
setupTtsEventListener()
}
} else if arguments == nil {
// 如果没有指定通道，尝试弄清楚是哪个通道在监听
if asrEventSink == nil && ttsEventSink != nil {
asrEventSink = events
} else if asrEventSink != nil && ttsEventSink == nil {
ttsEventSink = events
setupTtsEventListener()
} else {
// 无法确定哪个通道，默认设置为ASR
asrEventSink = events
}
/**
* 通用事件流处理器基类
*/
private class BaseEventStreamHandler: NSObject, FlutterStreamHandler {
private weak var instance: AzureSpeechPlugin?
private let eventType: AzureSpeechPlugin.EventType
/**
* 初始化事件流处理器
* @param instance AzureSpeechPlugin实例
* @param eventType 事件类型
*/
init(instance: AzureSpeechPlugin, eventType: AzureSpeechPlugin.EventType) {
self.instance = instance
self.eventType = eventType
super.init()
}
func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? {
switch eventType {
case .asr:
instance?.asrEventSink = events
case .tts:
instance?.ttsEventSink = events
instance?.setupTtsEventListener()
case .audioData:
instance?.audioDataEventSink = events
print("音频数据事件通道已准备好")
case .ast:
instance?.astEventSink = events
}
return nil
}
public func onCancel(withArguments arguments: Any?) -> FlutterError? {
// 根据通道类型清除事件接收器
if let args = arguments as? [String: Any],
let channel = args["channel"] as? String {
if channel == "asr" {
asrEventSink = nil
} else if channel == "tts" {
ttsEventSink = nil
}
} else {
// 如果没有指定通道，清除所有通道
asrEventSink = nil
ttsEventSink = nil
func onCancel(withArguments arguments: Any?) -> FlutterError? {
switch eventType {
case .asr:
instance?.asrEventSink = nil
case .tts:
instance?.ttsEventSink = nil
case .audioData:
instance?.audioDataEventSink = nil
print("音频数据事件通道已取消")
case .ast:
instance?.astEventSink = nil
}
return nil
}
}
// MARK: - TTS 事件监听实现
extension AzureSpeechPlugin: TtsEventListener {
// 使用@objc特性为方法提供一个不同的Objective-C选择器名称
@ -942,19 +1020,6 @@ extension AzureSpeechPlugin: TtsEventListener {
}
}
// MARK: - 音频数据监听实现
extension AzureSpeechPlugin: AudioDataListener {
public func onAudioData(_ data: Data) {
if ttsEventSink == nil { return }
let eventMap: [String: Any] = [
"type": "audio_data",
"data": FlutterStandardTypedData(bytes: data)
]
sendTtsEvent(eventMap)
}
}
// MARK: - 音频数据回调实现
@ -978,6 +1043,7 @@ private class AudioDataCallbackImpl: SimpleAudioReceiver.AudioDataCallback {
* @param data 音频数据
*/
func onAudio(_ data: Data) {
print("音频数据回调方法: \(data.count)")
callback(data)
}
}

6
local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift
View File

@ -140,8 +140,10 @@ public class SimpleAudioReceiver: NSObject {
guard let dataToWrite = self.writeQueue.take() else { continue }
do {
if let callback = self.audioDataCallback {
callback.onAudio(dataToWrite)
if self.audioDataCallback != nil {
print("写入数据长度: \(dataToWrite.count)")
self.audioDataCallback!.onAudio(dataToWrite)
}
try self.pushAudioStream?.write(dataToWrite)
if self.recordfile != nil {

Write Preview
Loading…
Cancel
Save