Browse Source

Revert "perf(agent_service): 优化音频处理和事件分发性能"

This reverts commit 89e99cbe67.
weicu
lxm 1 year ago
parent
commit
18e98e2fdc
  1. 316
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  2. 42
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt
  3. 112
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift
  4. 35
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
  5. 40
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift

316
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt

@ -15,7 +15,6 @@ import com.yunqiinnovation.user_portrait.UserPortrait
import com.yunqiinnovation.location_service.LocationService import com.yunqiinnovation.location_service.LocationService
import android.util.Log import android.util.Log
import kotlinx.coroutines.* import kotlinx.coroutines.*
import kotlinx.coroutines.channels.Channel
import java.util.Collections import java.util.Collections
import kotlin.coroutines.CoroutineContext import kotlin.coroutines.CoroutineContext
import com.yunqiinnovation.ble_service.BleService import com.yunqiinnovation.ble_service.BleService
@ -117,7 +116,7 @@ object AgentService : CoroutineScope {
// 空闲检测相关 // 空闲检测相关
private var idleCheckJob: Job? = null private var idleCheckJob: Job? = null
private val maxIdleSeconds = 10 // 最大空闲秒数(保持原有设置) private val maxIdleSeconds = 10 // 最大空闲秒数
// 打断防抖相关 // 打断防抖相关
private var lastInterruptTime = 0L private var lastInterruptTime = 0L
@ -193,31 +192,27 @@ object AgentService : CoroutineScope {
} }
// 初始化Azure ASR // 初始化Azure ASR
azureAsrHelper = AzureAsrHelper(context).apply { azureAsrHelper = AzureAsrHelper(context).apply {
// 使用AgentService的协程作用域,确保生命周期管理 // 使用协程处理异步初始化
launch(Dispatchers.IO) { CoroutineScope(Dispatchers.IO).launch {
try { //获取语音识别的语种
//获取语音识别的语种 val supportedLanguagesConfig = config["supportedLanguages"] as? List<String>
val supportedLanguagesConfig = config["supportedLanguages"] as? List<String> val supportedLanguages = supportedLanguagesConfig?.toTypedArray()
val supportedLanguages = supportedLanguagesConfig?.toTypedArray() ?: arrayOf("zh-CN", "en-US") // 默认支持多语言
?: arrayOf("zh-CN", "en-US") // 默认支持多语言
val asrInitSuccess = initialize(
val asrInitSuccess = initialize( subscriptionKey = config["azureSpeechKey"]?.toString() ?: "",
subscriptionKey = config["azureSpeechKey"]?.toString() ?: "", region = config["azureSpeechRegion"]?.toString() ?: "",
region = config["azureSpeechRegion"]?.toString() ?: "", supportedLanguages = supportedLanguages,
supportedLanguages = supportedLanguages, audioSourceType = if (isExternalActive) AzureAsrHelper.AudioSourceType.EXTERNAL
audioSourceType = if (isExternalActive) AzureAsrHelper.AudioSourceType.EXTERNAL else AzureAsrHelper.AudioSourceType.MICROPHONE
else AzureAsrHelper.AudioSourceType.MICROPHONE )
)
if (asrInitSuccess) { if (asrInitSuccess) {
withContext(Dispatchers.Main) { withContext(Dispatchers.Main) {
recognizeCallback() recognizeCallback()
}
} else {
Log.w(TAG, "Azure ASR初始化失败")
} }
} catch (e: Exception) { } else {
Log.e(TAG, "Azure ASR初始化异常", e) Log.w(TAG, "Azure ASR初始化失败")
} }
} }
} }
@ -249,71 +244,35 @@ object AgentService : CoroutineScope {
} }
/** /**
* 释放所有资源 - 增强版本 * 释放所有资源
*/ */
fun dispose() { fun dispose() {
try { try {
Log.i(TAG, "开始释放AgentService资源...")
// 1. 停止所有活动
stopRecognition() stopRecognition()
stopTts() stopTts()
stopAiStream() stopAiStream()
stopIdleCheck() stopIdleCheck()
// 2. 停止事件处理器
eventProcessorJob?.cancel()
eventProcessorJob = null
eventQueue.close()
// 3. 释放音频播放器
audioPlayer?.release() audioPlayer?.release()
audioPlayer = null audioPlayer = null
// 4. 释放Azure ASR(确保完全清理) azureAsrHelper?.dispose()
azureAsrHelper?.let { helper ->
try {
helper.dispose()
} catch (e: Exception) {
Log.e(TAG, "释放Azure ASR异常", e)
}
}
azureAsrHelper = null
// 5. 释放ChatAPI服务 // 释放ChatAPI服务
if (::chatApiService.isInitialized) { if (::chatApiService.isInitialized) {
try { chatApiService.cancelCurrentStream()
chatApiService.cancelCurrentStream() chatApiService.dispose()
chatApiService.dispose()
} catch (e: Exception) {
Log.e(TAG, "释放ChatAPI服务异常", e)
}
} }
// 6. 释放TTS服务 // 释放TTS服务
ttsService?.let { tts -> ttsService?.release()
try {
tts.release()
} catch (e: Exception) {
Log.e(TAG, "释放TTS服务异常", e)
}
}
ttsService = null ttsService = null
// 7. 取消所有协程并清理监听器
job.cancel() job.cancel()
clearListeners() clearListeners()
// 8. 重置状态
_isInitialized.set(false) _isInitialized.set(false)
_isRecognitionActive.set(false)
_hasSpeechDetected.set(false)
_isTtsSpeaking.set(false)
_isAiStreaming.set(false)
Log.i(TAG, "AgentService资源释放完成")
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "释放资源异常: ${e.message}", e) Log.e(TAG, "释放资源异常: ${e.message}")
} }
} }
@ -456,31 +415,19 @@ object AgentService : CoroutineScope {
stopIdleCheck() // 先停止现有的检查 stopIdleCheck() // 先停止现有的检查
if (!isRecognitionActive) return if (!isRecognitionActive) return
// BLE模式下使用稍长的超时时间,但不影响正常使用
val timeoutSeconds = if (audioSourceType == AzureAsrHelper.AudioSourceType.EXTERNAL) {
maxIdleSeconds + 3 // BLE模式额外增加3秒,避免过度延长
} else {
maxIdleSeconds
}
idleCheckJob = launch { idleCheckJob = launch {
try { try {
Log.d(TAG, "启动空闲检测,超时时间: ${timeoutSeconds}秒,音频源: $audioSourceType") delay(maxIdleSeconds * 1000L)
delay(timeoutSeconds * 1000L)
if (isActive && isRecognitionActive && !hasSpeechDetected && !isTtsSpeaking && !isAiStreaming) { if (isActive && isRecognitionActive && !hasSpeechDetected && !isTtsSpeaking && !isAiStreaming) {
Log.i(TAG, "空闲超时,停止识别 - 超时时间: ${timeoutSeconds}秒")
stopRecognition() stopRecognition()
sendEvent( sendEvent(
"auto_stop", "auto_stop",
mapOf("reason" to "idle_timeout", "seconds" to timeoutSeconds) mapOf("reason" to "idle_timeout", "seconds" to maxIdleSeconds)
) )
} else {
Log.d(TAG, "空闲检测结束,但条件不满足停止要求")
} }
} catch (e: CancellationException) { } catch (e: CancellationException) {
// 协程被取消,正常行为 // 协程被取消,正常行为
Log.d(TAG, "空闲检测被取消")
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "空闲检测异常: ${e.message}") Log.e(TAG, "空闲检测异常: ${e.message}")
} }
@ -513,12 +460,26 @@ object AgentService : CoroutineScope {
Log.d(TAG, "服务未初始化") Log.d(TAG, "服务未初始化")
return false return false
} }
Log.d("TAG", "开始语音识别startRecognition?")
Log.i(TAG, "开始语音识别 - 模式: ${if (isExternalActive) "BLE外部音频" else "设备麦克风"}")
// 先停止当前识别
stopRecognition() stopRecognition()
BleService.openEncoder()
// if (isRecognitionActive&&audioSourceType==AzureAsrHelper.AudioSourceType.MICROPHONE){
// Log.d(TAG, "开始语音识别stopRecognition")
// // BleService.closeCodec()
// stopRecognition()
// }
// if((isExternalActive&&audioSourceType==AzureAsrHelper.AudioSourceType.MICROPHONE)||!isRecognitionActive)
// {
// Log.d(TAG, "开始语音识别BleService.openEncoder")
// BleService.openEncoder()
// }
// if(isRecognitionActive)
// {
// Log.d(TAG, "开始语音识别azureAsrHelper?.startContinuousRecognition(audioSourceType)")
// return true
// }
// 检查录音权限 // 检查录音权限
if (!isExternalActive && !checkRecordAudioPermission()) { if (!isExternalActive && !checkRecordAudioPermission()) {
Log.e(TAG, "需要录音权限") Log.e(TAG, "需要录音权限")
@ -526,45 +487,22 @@ object AgentService : CoroutineScope {
return false return false
} }
// 设置音频源类型 _isRecognitionActive.set(true)
audioSourceType = if (isExternalActive) { _hasSpeechDetected.set(false)
AzureAsrHelper.AudioSourceType.EXTERNAL
} else {
AzureAsrHelper.AudioSourceType.MICROPHONE
}
Log.i(TAG, "音频源类型设置为: $audioSourceType")
try { try {
// 启动Azure ASR audioSourceType = if (isExternalActive) {
val asrStarted = azureAsrHelper?.startContinuousRecognition(audioSourceType) ?: false AzureAsrHelper.AudioSourceType.EXTERNAL
if (!asrStarted) { } else {
Log.e(TAG, "Azure ASR启动失败") AzureAsrHelper.AudioSourceType.MICROPHONE
return false
}
// 如果是BLE模式,在ASR启动后再打开编码器
if (isExternalActive) {
Log.i(TAG, "BLE模式:启动编码器")
// 使用协程延迟而不是阻塞主线程
launch {
delay(50) // 给ASR一些启动时间
val encoderOpened = BleService.openEncoder()
if (!encoderOpened) {
Log.w(TAG, "BLE编码器启动失败,但继续尝试识别")
}
}
} }
Log.d("TAG", "azureAsrHelper?.startContinuousRecognition")
azureAsrHelper?.startContinuousRecognition(audioSourceType)
_isRecognitionActive.set(true)
_hasSpeechDetected.set(false)
Log.i(TAG, "语音识别启动成功")
return true return true
} catch (e: Exception) { } catch (e: Exception) {
_isRecognitionActive.set(false) _isRecognitionActive.set(false)
Log.e(TAG, "启动语音识别失败: ${e.message}", e) Log.e(TAG, "启动语音识别失败: ${e.message}")
sendEvent( sendEvent(
"error", mapOf( "error", mapOf(
"code" to "RECOGNITION_START_ERROR", "code" to "RECOGNITION_START_ERROR",
@ -587,17 +525,13 @@ object AgentService : CoroutineScope {
_hasSpeechDetected.set(true) _hasSpeechDetected.set(true)
if (!previousHasSpeech) { if (!previousHasSpeech) {
Log.i(TAG, "首次检测到语音输入: $recognizing (音频源: $audioSourceType)")
restartIdleCheck() restartIdleCheck()
} }
// 实时发送识别结果,确保UI及时更新
Log.v(TAG, "实时识别: $recognizing")
sendEvent( sendEvent(
"recognizing", mapOf( "recognizing", mapOf(
"text" to recognizing, "text" to recognizing,
"language" to detectedLanguage, "language" to detectedLanguage
"audioSource" to audioSourceType.toString()
) )
) )
@ -670,42 +604,28 @@ object AgentService : CoroutineScope {
override fun onCanceled(reason: String, errorDetails: String) { override fun onCanceled(reason: String, errorDetails: String) {
_isRecognitionActive.set(false) _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
Log.w(TAG, "语音识别被取消: $reason, 详情: $errorDetails") Log.d(TAG, "recognition_canceled:")
BleService.closeCodec()
// 如果是BLE模式,关闭编码器
if (audioSourceType == AzureAsrHelper.AudioSourceType.EXTERNAL) {
BleService.closeCodec()
}
sendEvent( sendEvent(
"recognition_canceled", mapOf( "recognition_canceled", mapOf(
"reason" to reason, "reason" to reason,
"details" to errorDetails "details" to errorDetails
) )
) )
// 重新设置回调,但不自动重启识别
recognizeCallback() recognizeCallback()
} }
override fun onError(error: String) { override fun onError(error: String) {
_isRecognitionActive.set(false) _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
Log.e(TAG, "语音识别错误: $error") BleService.closeCodec()
Log.d(TAG, "语音识别错误: $error")
// 如果是BLE模式,关闭编码器
if (audioSourceType == AzureAsrHelper.AudioSourceType.EXTERNAL) {
BleService.closeCodec()
}
sendEvent( sendEvent(
"error", mapOf( "error", mapOf(
"code" to "RECOGNITION_ERROR", "code" to "RECOGNITION_ERROR",
"message" to error "message" to error
) )
) )
// 重新设置回调,但不自动重启识别
recognizeCallback() recognizeCallback()
} }
}) })
@ -715,68 +635,10 @@ object AgentService : CoroutineScope {
} }
} }
// 音频数据推送统计
private var audioDataCount = 0
private var lastAudioDataTime = 0L
// 性能监控
private var lastMemoryCheckTime = 0L
private val MEMORY_CHECK_INTERVAL = 30000L // 30秒检查一次内存
fun pushAudioData(data: ByteArray) { fun pushAudioData(data: ByteArray) {
val currentTime = System.currentTimeMillis()
audioDataCount++
// 每100个数据包或每5秒输出一次统计
if (audioDataCount % 100 == 0 || (currentTime - lastAudioDataTime) > 5000) {
Log.i(TAG, "音频数据统计: 已推送${audioDataCount}包, 当前包大小: ${data.size} bytes, 音频源: $audioSourceType")
lastAudioDataTime = currentTime
// 定期检查内存使用情况
checkMemoryUsage(currentTime)
}
// 添加调试日志,但保持原有逻辑的兼容性
if (!isRecognitionActive) {
Log.w(TAG, "语音识别未激活,但仍尝试推送音频数据: ${data.size} bytes")
}
if (audioSourceType != AzureAsrHelper.AudioSourceType.EXTERNAL) {
Log.w(TAG, "音频源类型不匹配: $audioSourceType,推送音频数据: ${data.size} bytes")
}
// 保持原有逻辑,确保兼容性
azureAsrHelper?.audioStream?.saveAudioDataTo(data) azureAsrHelper?.audioStream?.saveAudioDataTo(data)
} }
/**
* 检查内存使用情况
*/
private fun checkMemoryUsage(currentTime: Long) {
if (currentTime - lastMemoryCheckTime > MEMORY_CHECK_INTERVAL) {
lastMemoryCheckTime = currentTime
val runtime = Runtime.getRuntime()
val usedMemory = runtime.totalMemory() - runtime.freeMemory()
val maxMemory = runtime.maxMemory()
val memoryUsagePercent = (usedMemory * 100 / maxMemory).toInt()
Log.i(TAG, "内存使用情况: ${usedMemory / 1024 / 1024}MB / ${maxMemory / 1024 / 1024}MB (${memoryUsagePercent}%)")
// 如果内存使用超过80%,发出警告
if (memoryUsagePercent > 80) {
Log.w(TAG, "内存使用率过高: ${memoryUsagePercent}%,建议检查内存泄漏")
// 发送内存警告事件
sendEvent("memory_warning", mapOf(
"usedMemoryMB" to (usedMemory / 1024 / 1024),
"maxMemoryMB" to (maxMemory / 1024 / 1024),
"usagePercent" to memoryUsagePercent
))
}
}
}
/** /**
* 停止语音识别 * 停止语音识别
*/ */
@ -1374,53 +1236,19 @@ object AgentService : CoroutineScope {
} }
} }
// 事件发送优化:使用单一协程处理事件队列
private val eventQueue = Channel<Pair<String, Map<String, Any>>>(capacity = Channel.UNLIMITED)
private var eventProcessorJob: Job? = null
init {
// 启动事件处理协程
startEventProcessor()
}
private fun startEventProcessor() {
eventProcessorJob = launch(Dispatchers.Main) {
for ((eventName, data) in eventQueue) {
try {
// 创建监听器快照,避免并发修改
val currentListeners = synchronized(listeners) { listeners.toList() }
currentListeners.forEach { listener ->
try {
listener.onEvent(eventName, data)
} catch (e: Exception) {
Log.e(TAG, "向监听器发送事件失败: $eventName", e)
}
}
} catch (e: Exception) {
Log.e(TAG, "处理事件失败: $eventName", e)
}
}
}
}
/** /**
* 发送事件 - 优化版本,使用事件队列避免主线程阻塞 * 发送事件
*/ */
private fun sendEvent(eventName: String, data: Map<String, Any>) { private fun sendEvent(eventName: String, data: Map<String, Any>) {
try { // 切换到主线程执行监听器回调,避免 UI 更新问题
// 对于高频事件,进行限流处理 launch(Dispatchers.Main) {
if (eventName == "recognizing" && eventQueue.trySend(Pair(eventName, data)).isFailure) { listeners.forEach { listener ->
Log.v(TAG, "识别事件队列已满,跳过当前事件") try {
return listener.onEvent(eventName, data)
} } catch (e: Exception) {
Log.e(TAG, "向监听器发送事件失败", e)
// 非阻塞发送到事件队列 }
if (!eventQueue.trySend(Pair(eventName, data)).isSuccess) {
Log.w(TAG, "事件队列已满,丢弃事件: $eventName")
} }
} catch (e: Exception) {
Log.e(TAG, "发送事件异常: $eventName", e)
} }
} }

42
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt

@ -286,43 +286,19 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
} }
/** /**
* 实现AgentServiceListener接口 - 优化版本 * 实现AgentServiceListener接口
* 接收来自AgentService的事件,并转发到Flutter * 接收来自AgentService的事件,并转发到Flutter
*/ */
override fun onEvent(eventName: String, data: Map<String, Any>) { override fun onEvent(eventName: String, data: Map<String, Any>) {
try { // 将事件转换为JSON字符串
// 对于高频事件,进行优化处理 val eventData = JSONObject().apply {
val eventData = when (eventName) { put("event", eventName)
"recognizing" -> { put("data", JSONObject(data))
// 实时识别事件,简化数据结构 }
JSONObject().apply {
put("event", eventName)
put("data", JSONObject().apply {
put("text", data["text"] ?: "")
put("language", data["language"] ?: "")
put("audioSource", data["audioSource"] ?: "")
})
}
}
else -> {
// 其他事件,正常处理
JSONObject().apply {
put("event", eventName)
put("data", JSONObject(data))
}
}
}
// 使用协程在主线程发送事件 // 在主线程发送事件到Flutter
MainScope().launch { MainScope().launch {
try { eventSink?.success(eventData.toString())
eventSink?.success(eventData.toString())
} catch (e: Exception) {
Log.e(TAG, "发送事件到Flutter失败: $eventName", e)
}
}
} catch (e: Exception) {
Log.e(TAG, "处理事件异常: $eventName", e)
} }
} }

112
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift

@ -104,34 +104,13 @@ class AgentServiceImpl: NSObject {
listenersLock.unlock() listenersLock.unlock()
} }
// 事件发送优化:使用队列避免主线程阻塞
private let eventQueue = DispatchQueue(label: "com.yunqiinnovation.agent_service.events", qos: .userInitiated)
internal func sendEvent(name eventName: String, data: [String: Any]) { internal func sendEvent(name eventName: String, data: [String: Any]) {
// 对于高频事件,使用异步处理
if eventName == "recognizing" {
eventQueue.async { [weak self] in
self?.dispatchEventToListeners(eventName: eventName, data: data)
}
} else {
// 其他事件在主线程处理
DispatchQueue.main.async { [weak self] in
self?.dispatchEventToListeners(eventName: eventName, data: data)
}
}
}
private func dispatchEventToListeners(eventName: String, data: [String: Any]) {
listenersLock.lock() listenersLock.lock()
let currentListeners = self.listeners let currentListeners = self.listeners
listenersLock.unlock() listenersLock.unlock()
for listener in currentListeners { for listener in currentListeners {
do { listener.onEvent(eventName: eventName, data: data)
listener.onEvent(eventName: eventName, data: data)
} catch {
os_log("发送事件失败: %{public}@", log: logger, type: .error, eventName)
}
} }
} }
@ -296,39 +275,25 @@ class AgentServiceImpl: NSObject {
private func startIdleCheck() { private func startIdleCheck() {
stopIdleCheck() stopIdleCheck()
guard isRecognizing else { return } guard isRecognizing else { return }
// BLE模式下使用稍长的超时时间
let timeoutSeconds: TimeInterval
if let azureHelper = azureAsrHelper,
azureHelper.audioSourceType == .external {
timeoutSeconds = maxIdleSeconds + 3 // BLE模式额外增加3秒
} else {
timeoutSeconds = maxIdleSeconds
}
os_log("启动空闲检测,超时时间: %.0f秒", log: logger, type: .debug, timeoutSeconds)
// 使用全局队列而不是主队列,避免后台挂起问题 // 使用全局队列而不是主队列,避免后台挂起问题
let timer = DispatchSource.makeTimerSource(queue: DispatchQueue.global(qos: .utility)) let timer = DispatchSource.makeTimerSource(queue: DispatchQueue.global(qos: .utility))
idleTimer = timer idleTimer = timer
timer.schedule(deadline: .now() + timeoutSeconds) timer.schedule(deadline: .now() + maxIdleSeconds)
timer.setEventHandler { [weak self] in timer.setEventHandler { [weak self] in
// 使用更安全的方式检查self是否存在 // 使用更安全的方式检查self是否存在
DispatchQueue.main.async { [weak self] in DispatchQueue.main.async { [weak self] in
guard let self = self else { return } guard let self = self else { return }
if self.isRecognizing && !self.hasSpeechDetected && !self.isSpeaking && !self.isAiStreaming { if self.isRecognizing && !self.hasSpeechDetected && !self.isSpeaking && !self.isAiStreaming {
os_log("空闲超时,停止识别 - 超时时间: %.0f秒", log: self.logger, type: .info, timeoutSeconds)
self.stopRecognition() self.stopRecognition()
self.sendEvent(name: "auto_stop", data: [ self.sendEvent(name: "auto_stop", data: [
"reason": "idle_timeout", "reason": "idle_timeout",
"seconds": timeoutSeconds "seconds": self.maxIdleSeconds
]) ])
} else {
os_log("空闲检测结束,但条件不满足停止要求", log: self.logger, type: .debug)
} }
} }
} }
@ -350,49 +315,34 @@ class AgentServiceImpl: NSObject {
sendError("服务未初始化", code: "NOT_INITIALIZED") sendError("服务未初始化", code: "NOT_INITIALIZED")
return false return false
} }
os_log("开始语音识别 - 模式: %{public}@", log: logger, type: .info, useBle ? "BLE外部音频" : "设备麦克风")
if !useBle { if !useBle {
guard AVAudioSession.sharedInstance().recordPermission == .granted else { guard AVAudioSession.sharedInstance().recordPermission == .granted else {
sendError("无麦克风权限", code: "PERMISSION_DENIED") sendError("无麦克风权限", code: "PERMISSION_DENIED")
return false return false
} }
} }
if isRecognizing { if isRecognizing {
stopRecognition() stopRecognition()
} }
let audioSourceType: AzureAsrHelper.AudioSourceType = useBle ? .external : .microphone let audioSourceType: AzureAsrHelper.AudioSourceType = useBle ? .external : .microphone
if useBle { if useBle {
AudioSessionHub.shared.begin(.playback) AudioSessionHub.shared.begin(.playback)
} else { } else {
AudioSessionHub.shared.begin(.voice) AudioSessionHub.shared.begin(.voice)
} }
print("ai启动语音=\(audioSourceType)=\(useBle)")
os_log("音频源类型设置为: %{public}@", log: logger, type: .info, audioSourceType == .external ? "EXTERNAL" : "MICROPHONE")
guard let success = azureAsrHelper?.startContinuousRecognition( guard let success = azureAsrHelper?.startContinuousRecognition(
// callback: self,
audioSourceType: audioSourceType audioSourceType: audioSourceType
), success else { ), success else {
sendError("启动语音识别失败", code: "RECOGNITION_START_ERROR") sendError("启动语音识别失败", code: "RECOGNITION_START_ERROR")
return false return false
} }
// 如果是BLE模式,在ASR启动后再打开编码器
if useBle {
os_log("BLE模式:启动编码器", log: logger, type: .info)
DispatchQueue.global().asyncAfter(deadline: .now() + 0.05) {
let encoderOpened = BleService.shared.openEncoder()
if !encoderOpened {
os_log("BLE编码器启动失败,但继续尝试识别", log: self.logger, type: .warning)
}
}
}
os_log("语音识别启动成功", log: logger, type: .info)
return true return true
} }
func recognizeCallback(useBle: Bool = false) -> Bool { func recognizeCallback(useBle: Bool = false) -> Bool {
@ -413,15 +363,13 @@ class AgentServiceImpl: NSObject {
} }
func pushAudioData(_ audioData: Data) -> Bool { func pushAudioData(_ audioData: Data) -> Bool {
// 添加调试日志,但保持兼容性
if !isRecognizing || !isInitialized { if !isRecognizing || !isInitialized {
os_log("语音识别未激活或服务未初始化,但仍尝试推送音频数据: %d bytes", log: logger, type: .debug, audioData.count)
}
guard let audioStream = azureAsrHelper?.audioStream else {
os_log("音频流未初始化", log: logger, type: .error)
return false return false
} }
guard let audioStream = azureAsrHelper?.audioStream else {
os_log("音频流未初始化", log: logger,type: .error)
return false
}
audioStream.saveAudioDataTo(data: audioData) audioStream.saveAudioDataTo(data: audioData)
return true return true
@ -1450,8 +1398,6 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback {
} }
func onCanceled(_ reason: String, _ errorDetails: String) { func onCanceled(_ reason: String, _ errorDetails: String) {
os_log("语音识别被取消: %{public}@, 详情: %{public}@", log: logger, type: .warning, reason, errorDetails)
var data: [String: Any] = [:] var data: [String: Any] = [:]
if !reason.isEmpty { if !reason.isEmpty {
data["reason"] = reason data["reason"] = reason
@ -1459,27 +1405,13 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback {
if !errorDetails.isEmpty { if !errorDetails.isEmpty {
data["details"] = errorDetails data["details"] = errorDetails
} }
// 如果是BLE模式,关闭编码器
if let azureHelper = azureAsrHelper,
azureHelper.audioSourceType == .external {
BleService.shared.closeCodec()
}
sendEvent(name: "recognition_canceled", data: data) sendEvent(name: "recognition_canceled", data: data)
isRecognizing = false isRecognizing = false
stopIdleCheck() stopIdleCheck()
} }
func onError(_ error: String) { func onError(_ error: String) {
os_log("语音识别错误: %{public}@", log: logger, type: .error, error)
// 如果是BLE模式,关闭编码器
if let azureHelper = azureAsrHelper,
azureHelper.audioSourceType == .external {
BleService.shared.closeCodec()
}
let data: [String: Any] = ["message": error.isEmpty ? "未知错误" : error] let data: [String: Any] = ["message": error.isEmpty ? "未知错误" : error]
sendEvent(name: "error", data: data) sendEvent(name: "error", data: data)
isRecognizing = false isRecognizing = false

35
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt

@ -146,17 +146,6 @@ class AzureAsrHelper(private val context: Context) {
// 设置分段策略为时间模式 // 设置分段策略为时间模式
setProperty("Speech_SegmentationStrategy", "Time") setProperty("Speech_SegmentationStrategy", "Time")
// 针对EXTERNAL音频源的实时性优化
if (audioSourceType == AudioSourceType.EXTERNAL) {
// 减少缓冲延迟
setProperty("SpeechServiceConnection_ReceiveTimeout", "1000")
setProperty("SpeechServiceConnection_SendTimeout", "1000")
// 启用实时模式
setProperty("SpeechServiceConnection_EnableRealTimeRecognition", "true")
// 减少音频缓冲
setProperty("SpeechServiceConnection_AudioBufferSizeMs", "100")
}
} }
// 录音文件类 // 录音文件类
recordfile = RecordFile; recordfile = RecordFile;
@ -717,9 +706,7 @@ class AzureAsrHelper(private val context: Context) {
// 情况2:不在录制但队列有数据 -> 从队列获取 // 情况2:不在录制但队列有数据 -> 从队列获取
else if (writeQueue.isNotEmpty()) { else if (writeQueue.isNotEmpty()) {
data = writeQueue.poll() data = writeQueue.poll()
if (data != null) { Log.d("tag", "写入数据: ${data?.size}")
Log.v("tag", "从队列写入数据: ${data.size} bytes, 队列剩余: ${writeQueue.size}")
}
bytesToWrite = data?.size ?: 0 bytesToWrite = data?.size ?: 0
} }
@ -737,8 +724,7 @@ class AzureAsrHelper(private val context: Context) {
Log.e("tag", "写入失败: ${e.message}") Log.e("tag", "写入失败: ${e.message}")
} }
} else { } else {
// 使用更短的等待时间,提高响应性 Thread.yield() // 避免空转消耗CPU
Thread.sleep(1) // 1ms等待,避免空转但保持响应性
} }
} }
} catch (e: Exception) { } catch (e: Exception) {
@ -758,22 +744,11 @@ class AzureAsrHelper(private val context: Context) {
* 外部音频输入 * 外部音频输入
*/ */
fun saveAudioDataTo(buffer: ByteArray) { fun saveAudioDataTo(buffer: ByteArray) {
if (audioSourceType == AudioSourceType.MICROPHONE) return
// 检查队列大小,避免积压
if (writeQueue.size > 50) { // 限制队列大小
Log.w("AudioStream", "音频队列积压严重(${writeQueue.size}),清理旧数据")
// 清理一半的旧数据,保持实时性
repeat(writeQueue.size / 2) {
writeQueue.poll()
}
}
if (audioSourceType == AudioSourceType.MICROPHONE) return
// 放入队列,由写线程写入 // 放入队列,由写线程写入
val offered = writeQueue.offer(buffer.copyOf()) writeQueue.offer(buffer.copyOf())
if (!offered) {
Log.w("AudioStream", "音频队列已满,丢弃数据包: ${buffer.size} bytes")
}
} }
/** /**

40
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift

@ -690,27 +690,22 @@ public class AzureAsrHelper: NSObject {
guard let self = self else { return } guard let self = self else { return }
while self.isRunning { while self.isRunning {
// print("是否写入: \(self.isWriting)")
if !self.isWriting { if !self.isWriting {
usleep(1_000) // 减少等待时间,提高响应性 usleep(10_000)
continue continue
} }
guard let dataToWrite = self.writeQueue.take() else { continue }
// print("写入数据长度: \(dataToWrite.count)")
do {
try self.pushAudioStream?.write(dataToWrite)
self.onAudioData?(dataToWrite)
} catch {
print("推送音频数据失败: \(error.localizedDescription)")
}
guard let dataToWrite = self.writeQueue.take() else {
usleep(1_000) // 短暂等待,避免空转
continue
}
// 添加队列状态日志
if self.writeQueue.count > 10 {
print("音频写入队列积压: \(self.writeQueue.count)")
}
do {
try self.pushAudioStream?.write(dataToWrite)
self.onAudioData?(dataToWrite)
} catch {
print("推送音频数据失败: \(error.localizedDescription)")
}
} }
} }
} }
@ -725,16 +720,7 @@ public class AzureAsrHelper: NSObject {
if audioSourceType != .external { if audioSourceType != .external {
return return
} }
// print("外部data=\(data)")
// 检查队列大小,避免积压
if writeQueue.count > 50 {
os_log("音频队列积压严重(%d),清理旧数据", log: OSLog.default, type: .warning, writeQueue.count)
// 清理一半的旧数据,保持实时性
for _ in 0..<(writeQueue.count / 2) {
_ = writeQueue.take()
}
}
// 放入队列,由写线程写入 // 放入队列,由写线程写入
writeQueue.put(data) writeQueue.put(data)
} }

Loading…
Cancel
Save