From 70d9dd222de564bee87de6c2fbd4084869a9ce33 Mon Sep 17 00:00:00 2001 From: fdp <1286779656@qq.com> Date: Mon, 18 Aug 2025 16:49:29 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=89=E5=8D=93=E8=AF=AD=E8=A8=80=E8=AF=86?= =?UTF-8?q?=E5=88=AB=E5=8A=A0=E5=85=A5=E7=BD=91=E7=BB=9C=E6=A3=80=E6=B5=8B?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agent/controllers/agent_controller.dart | 12 +- .../controllers/opus_test_controller.dart | 27 +- .../agent_service/AgentService.kt | 38 +-- .../azure_speech/AzureAsrHelper.kt | 257 ++++++++++++++++-- 4 files changed, 274 insertions(+), 60 deletions(-) diff --git a/lib/modules/agent/controllers/agent_controller.dart b/lib/modules/agent/controllers/agent_controller.dart index e778b2a5b..04b8e7d8e 100644 --- a/lib/modules/agent/controllers/agent_controller.dart +++ b/lib/modules/agent/controllers/agent_controller.dart @@ -649,11 +649,13 @@ class AgentController extends GetxController { break; case AgentServiceEventType.error: - isListening.value = false; - isSpeaking.value = false; - isProcessing.value = false; - isImageProcessing.value = false; - + //验证错误码 + if (event.data['code'] == 1000) { + isListening.value = false; + isSpeaking.value = false; + isProcessing.value = false; + isImageProcessing.value = false; + } // 移除临时的识别消息 final index = messages.lastIndexWhere((msg) => msg.isRecognizing && msg.isUser); diff --git a/lib/modules/opus_test/controllers/opus_test_controller.dart b/lib/modules/opus_test/controllers/opus_test_controller.dart index 4fad692e9..81ee98093 100644 --- a/lib/modules/opus_test/controllers/opus_test_controller.dart +++ b/lib/modules/opus_test/controllers/opus_test_controller.dart @@ -103,16 +103,31 @@ class OpusTestController extends GetxController { Future loadExternalFiles() async { try { externalFiles.clear(); - statusMessage.value = '正在扫描外部存储...'; + statusMessage.value = '正在扫描存储...'; + print('正在扫描存储...'); - // 获取外部存储目录 - final directories = await getExternalStorageDirectories(); - if (directories == null || directories.isEmpty) { - statusMessage.value = '未找到外部存储目录'; + List directories = []; + + if (Platform.isAndroid) { + // Android: 使用外部存储目录 + final androidDirs = await getExternalStorageDirectories(); + if (androidDirs != null && androidDirs.isNotEmpty) { + directories.addAll(androidDirs); + } + } else if (Platform.isIOS) { + // iOS: 使用Documents目录 + final iosDir = await getApplicationDocumentsDirectory(); + directories.add(iosDir); + } + + if (directories.isEmpty) { + statusMessage.value = '未找到存储目录'; return; } - // 遍历所有外部存储目录 + print('存储目录: $directories'); + + // 遍历所有存储目录 for (final directory in directories) { final dirPath = directory.path; final dir = Directory(dirPath); diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index 423ed7fc6..6fc1400c2 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -435,18 +435,7 @@ object AgentService : CoroutineScope { Log.d(TAG, "通话模式：禁用空闲检测") return } - "ble_wakeup" -> { - // BLE唤醒模式：使用默认的空闲检测 - startIdleCheck(maxIdleSeconds) - } - "push_to_talk" -> { - // 按住说话模式：使用较长的空闲检测时间 - startIdleCheck(maxIdleSeconds) - } - "normal" -> { - // 普通模式：使用默认的空闲检测 - startIdleCheck(maxIdleSeconds) - } + else -> { // 未知模式：使用默认的空闲检测 Log.w(TAG, "未知识别模式: $mode，使用默认空闲检测") @@ -543,9 +532,7 @@ object AgentService : CoroutineScope { return false } - // 设置当前识别模式 - currentRecognitionMode = mode - Log.d(TAG, "设置语音识别模式: $mode") + _isRecognitionActive.set(true) _hasSpeechDetected.set(false) @@ -561,7 +548,9 @@ object AgentService : CoroutineScope { // 根据模式启动相应的空闲检测 startIdleCheckForMode(mode) - + // 设置当前识别模式 + currentRecognitionMode = mode + Log.d(TAG, "设置语音识别模式: $mode") return true } catch (e: Exception) { _isRecognitionActive.set(false) @@ -584,12 +573,12 @@ object AgentService : CoroutineScope { override fun onRecognizing(recognizing: String, detectedLanguage: String) { if (recognizing.isNotEmpty()) { // 检测到语音，更新状态 - val previousHasSpeech = hasSpeechDetected + _hasSpeechDetected.set(true) - if (!previousHasSpeech) { + restartIdleCheck() - } + sendEvent( "recognizing", mapOf( @@ -637,19 +626,16 @@ object AgentService : CoroutineScope { processWithChatApi(result) } - // 重置状态，继续识别 - val previousHasSpeech = hasSpeechDetected + _hasSpeechDetected.set(false) - if (previousHasSpeech) { restartIdleCheck() - } + } override fun onSessionStarted() { sendEvent("recognition_started", mapOf("status" to "started")) // 使用模式感知的空闲检测启动方法 - startIdleCheckForMode(currentRecognitionMode) audioPlayer?.playAudio(R.raw.start) } @@ -677,14 +663,14 @@ object AgentService : CoroutineScope { recognizeCallback() } - override fun onError(error: String) { + override fun onError(code: Int, error: String) { _isRecognitionActive.set(false) stopIdleCheck() BleService.closeCodec() Log.d(TAG, "语音识别错误: $error") sendEvent( "error", mapOf( - "code" to "RECOGNITION_ERROR", + "code" to code, "message" to error ) ) diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt index 519dd42e8..8f1a75393 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt @@ -35,6 +35,12 @@ import java.util.concurrent.atomic.AtomicBoolean * 针对回音消除的场景，使用拉流模式 (PullAudioInputStreamCallback) 实现，避免复杂的协程处理 * 通过检测扬声器和耳机状态，自动决定是否启用回音消除 */ +import android.net.ConnectivityManager +import android.net.Network +import android.net.NetworkCapabilities +import android.net.NetworkRequest +import android.os.Build + class AzureAsrHelper(private val context: Context) { private val tag = "AzureAsrHelper" @@ -60,7 +66,11 @@ class AzureAsrHelper(private val context: Context) { // 音频处理 var audioStream: SimpleAudioReceiver? = null - + // 网络状态监听相关 + private var connectivityManager: ConnectivityManager? = null + private var networkCallback: ConnectivityManager.NetworkCallback? = null + private var isNetworkAvailable = true + private var isNetworkRecovering = false @@ -142,6 +152,9 @@ class AzureAsrHelper(private val context: Context) { } + // 初始化网络监听 + initNetworkMonitoring() + return true } catch (e: Exception) { Log.e(tag, "初始化失败: ${e.message}") @@ -233,7 +246,7 @@ class AzureAsrHelper(private val context: Context) { audioSourceType: AudioSourceType = AudioSourceType.MICROPHONE ) { if (speechConfig == null) { - callback.onError("语音服务未初始化") + callback.onError(1002,"语音服务未初始化") return } @@ -268,26 +281,59 @@ class AzureAsrHelper(private val context: Context) { ?: supportedLanguages[0] callback.onResult(result.text, detectedLanguage) } else { - callback.onError("未能识别语音") + callback.onError(1004,"未能识别语音") } } catch (e: Exception) { stopAudioProcessing() - callback.onError("识别异常: ${e.message}") + callback.onError(1003,"识别异常: ${e.message}") + } + } + + /** + * 检查当前网络状态 + * @return true表示网络可用，false表示网络不可用 + */ + private fun checkNetworkStatus(): Boolean { + try { + val connectivityManager = context.getSystemService(Context.CONNECTIVITY_SERVICE) as ConnectivityManager + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + val network = connectivityManager.activeNetwork ?: return false + val networkCapabilities = connectivityManager.getNetworkCapabilities(network) ?: return false + + return networkCapabilities.hasCapability(NetworkCapabilities.NET_CAPABILITY_INTERNET) && + networkCapabilities.hasCapability(NetworkCapabilities.NET_CAPABILITY_VALIDATED) + } else { + @Suppress("DEPRECATION") + val networkInfo = connectivityManager.activeNetworkInfo + return networkInfo?.isConnectedOrConnecting == true + } + } catch (e: Exception) { + Log.e(tag, "检查网络状态失败: ${e.message}") + return false } } /** * 开始连续语音识别 * @param audioSourceType 音频源类型 - * @param audioDataCallback 音频数据回调接口 - * @return 是否成功启动 + * @param audioDataCallback 音频数据回调 + * @return 是否成功开始识别 */ fun startContinuousRecognition( audioSourceType: AudioSourceType = AudioSourceType.MICROPHONE, audioDataCallback: SimpleAudioReceiver.AudioDataCallback? = null ): Boolean { - + Log.d(tag, "startContinuousRecognition:$isContinuousRecognitionActive ") + + // 检查网络状态 + if (!checkNetworkStatus()) { + Log.e(tag, "网络不可用，无法开始语音识别") + continuousCallback?.onError(1000,"网络连接不可用，请检查网络设置") + return false + } + if (isContinuousRecognitionActive) { return true } @@ -299,15 +345,15 @@ class AzureAsrHelper(private val context: Context) { } } this.audioSourceType = audioSourceType - + try { - - - Log.d(tag, "startContinuousRecognition: ") + Log.d(tag, "startContinuousRecognition: ${audioStream!!.isWriting()}") // 启动音频处理（若已启动则跳过） if (!audioStream!!.isWriting()) { + Log.d(tag, "startAudioRecord:$audioSourceType") audioStream!!.startAudioRecord( when (audioSourceType) { + AudioSourceType.MICROPHONE -> SimpleAudioReceiver.AudioSourceType.MICROPHONE AudioSourceType.EXTERNAL -> SimpleAudioReceiver.AudioSourceType.EXTERNAL }, @@ -316,7 +362,7 @@ class AzureAsrHelper(private val context: Context) { } else { Log.d(tag, "音频已启动，跳过重复启动") } - + recognizer?.startContinuousRecognitionAsync() isContinuousRecognitionActive = true return true @@ -334,7 +380,7 @@ class AzureAsrHelper(private val context: Context) { fun setupEventListeners(callback: ContinuousRecognizeCallback): Boolean { Log.d(tag, "设置ssssss监听器:${speechConfig} ") if (speechConfig == null) { - callback.onError("语音服务未初始化") + callback.onError(1002,"语音服务未初始化") return false } @@ -399,18 +445,31 @@ class AzureAsrHelper(private val context: Context) { } ) - // 取消事件 + // 取消事件 recognizer?.canceled?.addEventListener( EventHandler { _, event -> val errorDetails = event.errorDetails ?: "未知错误" val reason = event.reason.toString() - callback.onCanceled(reason, errorDetails) - + + Log.d(tag, "识别被取消: reason=$reason, details=$errorDetails") + + // 检查是否是网络相关错误 + if (isNetworkRelatedError(reason, errorDetails)) { + Log.d(tag, "检测到网络相关错误，停止音频处理") + stopAudioProcessingImmediately() + + // 如果不是正在恢复网络，通知上层网络错误 + if (!isNetworkRecovering) { + callback.onError(1000,"启动连续识别失败，网络连接已断开") + } + } else { + // 非网络错误，正常处理 + callback.onCanceled(reason, errorDetails) + } } ) - } catch (e: Exception) { - callback.onError("启动连续识别失败: ${e.message}") + callback.onError(1001,"启动连续识别失败: ${e.message}") return false } return true @@ -485,6 +544,14 @@ class AzureAsrHelper(private val context: Context) { // recognizer?.stopContinuousRecognitionAsync()?.get(1000, TimeUnit.MILLISECONDS) // 停止音频处理 stopAudioProcessing() + + // 清理网络监听 + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N && networkCallback != null) { + connectivityManager?.unregisterNetworkCallback(networkCallback!!) + networkCallback = null + } + connectivityManager = null + // 停止录音 if (audioStream!!.recordfile != null) { audioStream!!.recordfile!!.closeFile(true) @@ -647,9 +714,10 @@ class AzureAsrHelper(private val context: Context) { /** * 识别错误时调用 * + * @param code 错误码 * @param error 错误信息 */ - fun onError(error: String) + fun onError(code: Int,error: String) } /** @@ -696,10 +764,153 @@ class AzureAsrHelper(private val context: Context) { * * @param error 错误信息 */ - fun onError(error: String) + fun onError(code: Int,error: String) } - - + /** + * 检查是否是网络相关错误 + */ + private fun isNetworkRelatedError(reason: String, errorDetails: String): Boolean { + val networkErrorKeywords = arrayOf( + "network", "connection", "timeout", "unreachable", + "dns", "socket", "ssl", "tls", "certificate", + "网络", "连接", "超时", "不可达" + ) + + val combinedText = "$reason $errorDetails".lowercase() + return networkErrorKeywords.any { keyword -> + combinedText.contains(keyword.lowercase()) + } + } + /** + * 初始化网络状态监听 + */ + private fun initNetworkMonitoring() { + try { + connectivityManager = context.getSystemService(Context.CONNECTIVITY_SERVICE) as ConnectivityManager + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + networkCallback = object : ConnectivityManager.NetworkCallback() { + override fun onAvailable(network: Network) { + super.onAvailable(network) + Log.d(tag, "网络连接可用") + handleNetworkAvailable() + } + + override fun onLost(network: Network) { + super.onLost(network) + Log.d(tag, "网络连接丢失") + handleNetworkLost() + } + + override fun onCapabilitiesChanged(network: Network, networkCapabilities: NetworkCapabilities) { + super.onCapabilitiesChanged(network, networkCapabilities) + val hasInternet = networkCapabilities.hasCapability(NetworkCapabilities.NET_CAPABILITY_INTERNET) + Log.d(tag, "网络能力变化，是否有网络: $hasInternet") + if (hasInternet && !isNetworkAvailable) { + handleNetworkAvailable() + } else if (!hasInternet && isNetworkAvailable) { + handleNetworkLost() + } + } + } + + val networkRequest = NetworkRequest.Builder() + .addCapability(NetworkCapabilities.NET_CAPABILITY_INTERNET) + .build() + + connectivityManager?.registerNetworkCallback(networkRequest, networkCallback!!) + } + } catch (e: Exception) { + Log.e(tag, "初始化网络监听失败: ${e.message}") + } + } + + /** + * 处理网络连接可用 + */ + private fun handleNetworkAvailable() { + if (!isNetworkAvailable) { + isNetworkAvailable = true + isNetworkRecovering = true + Log.d(tag, "网络恢复，准备重新启动识别") + + // 如果之前在进行连续识别，重新启动 + + Handler(Looper.getMainLooper()).postDelayed({ + restartContinuousRecognition() + }, 1000) // 延迟1秒确保网络稳定 + + } + } + + /** + * 处理网络连接丢失 + */ + private fun handleNetworkLost() { + if (isNetworkAvailable) { + isNetworkAvailable = false + Log.d(tag, "网络断开，停止音频处理") + + + stopAudioProcessingImmediately() + + } + } + + /** + * 立即停止音频处理（网络断开时使用） + */ + private fun stopAudioProcessingImmediately() { + try { + Log.d(tag, "立即停止音频处理") + stopContinuousRecognition() + // 不改变isContinuousRecognitionActive状态，保持识别意图 + } catch (e: Exception) { + Log.e(tag, "立即停止音频处理失败: ${e.message}") + } + } + + /** + * 重新启动连续识别（网络恢复时使用） + */ + private fun restartContinuousRecognition() { + try { + Log.d(tag, "重新启动连续识别") + // 检查continuousCallback是否为空 + val callback = continuousCallback + if (callback == null) { + Log.e(tag, "无法重新启动连续识别：回调为空") + return + } + + setupEventListeners(callback) + // 重新启动音频处理 + if (audioStream?.isWriting() != true) { + audioStream?.startAudioRecord( + when (audioSourceType) { + AudioSourceType.MICROPHONE -> SimpleAudioReceiver.AudioSourceType.MICROPHONE + AudioSourceType.EXTERNAL -> SimpleAudioReceiver.AudioSourceType.EXTERNAL + }, + null + ) + } + + // 重新启动连续识别 + recognizer?.startContinuousRecognitionAsync() + + isNetworkRecovering = false + Log.d(tag, "连续识别重新启动成功") + + // 通知上层网络已恢复 + //continuousCallback?.onError("网络已恢复") + + } catch (e: Exception) { + Log.e(tag, "重新启动连续识别失败: ${e.message}") + isNetworkRecovering = false + } + } + } - \ No newline at end of file + + \ No newline at end of file