From a8a902b86cced2b50976ebce44ccbdd288098539 Mon Sep 17 00:00:00 2001 From: wolfplus Date: Mon, 24 Mar 2025 18:18:14 +0000 Subject: [PATCH] add --- android/app/build.gradle.kts | 8 +- .../com/example/deep_voice/AzureAsrHelper.kt | 392 +++++++++++++++++- ios/Runner/AzureAsrHelper.swift | 2 +- .../home/controllers/home_controller.dart | 1 + 4 files changed, 389 insertions(+), 14 deletions(-) diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 6879ee469..5f45a8a3f 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -1,9 +1,8 @@ repositories { google() mavenCentral() - maven { - url = uri("https://artifact.bytedance.com/repository/Volcengine/") - } + maven { url = uri("https://artifact.bytedance.com/repository/Volcengine/") } + maven { url = uri("https://oss.sonatype.org/content/repositories/snapshots/") } } plugins { @@ -50,6 +49,9 @@ android { } dependencies { + // WebRTC依赖已移除 + // implementation("io.antmedia:webrtc-android-framework:2.8.0-SNAPSHOT") + // 添加OkHttp依赖 implementation("com.squareup.okhttp3:okhttp:4.9.3") diff --git a/android/app/src/main/kotlin/com/example/deep_voice/AzureAsrHelper.kt b/android/app/src/main/kotlin/com/example/deep_voice/AzureAsrHelper.kt index b9f5d6718..ba5bac143 100644 --- a/android/app/src/main/kotlin/com/example/deep_voice/AzureAsrHelper.kt +++ b/android/app/src/main/kotlin/com/example/deep_voice/AzureAsrHelper.kt @@ -1,11 +1,21 @@ package com.example.deep_voice import android.content.Context +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioRecord +import android.media.MediaRecorder +import android.media.audiofx.AcousticEchoCanceler +import android.media.audiofx.NoiseSuppressor +import android.media.audiofx.AutomaticGainControl +import android.os.Process import com.example.deep_voice.core.utils.FileLogger import com.microsoft.cognitiveservices.speech.* import com.microsoft.cognitiveservices.speech.audio.* import com.microsoft.cognitiveservices.speech.util.EventHandler import java.util.concurrent.ExecutionException +import java.util.concurrent.atomic.AtomicBoolean +// 移除WebRTC相关导入 class AzureAsrHelper(private val context: Context) { private var recognizer: SpeechRecognizer? = null @@ -17,6 +27,14 @@ class AzureAsrHelper(private val context: Context) { private var serviceRegion = "" private var isAutoDetectLanguage = false private var supportedLanguages = arrayOf("zh-CN", "en-US") + + // 是否使用回音消除 - 内部控制常量 + private val useEchoCancellation = true + + // 自定义音频处理相关 + private var customAudioProcessor: CustomAudioProcessor? = null + private var pushStream: PushAudioInputStream? = null + private var audioConfig: AudioConfig? = null // 初始化SDK并创建recognizer fun initialize(subscriptionKey: String, serviceRegion: String, @@ -63,17 +81,31 @@ class AzureAsrHelper(private val context: Context) { // 创建识别器 try { - if (isAutoDetectLanguage) { - val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) - recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) + if (useEchoCancellation) { + // 如果使用回音消除，创建自定义音频输入流 + setupCustomAudioProcessing() + + if (isAutoDetectLanguage) { + val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) + recognizer = SpeechRecognizer(speechConfig, autoDetectConfig, audioConfig) + } else { + recognizer = SpeechRecognizer(speechConfig, audioConfig) + } } else { - recognizer = SpeechRecognizer(speechConfig) + // 使用默认麦克风输入 + if (isAutoDetectLanguage) { + val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) + recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) + } else { + recognizer = SpeechRecognizer(speechConfig) + } } FileLogger.d(TAG, "Azure 语音服务初始化成功") return true } catch (e: Exception) { FileLogger.e(TAG, "创建识别器失败: ${e.message}") + stopCustomAudioProcessing() return false } } catch (e: Exception) { @@ -89,13 +121,29 @@ class AzureAsrHelper(private val context: Context) { recognizer?.close() recognizer = null + // 停止当前的音频处理 + stopCustomAudioProcessing() + // 使用现有配置重新创建 recognizer if (speechConfig != null) { - if (isAutoDetectLanguage) { - val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) - recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) + if (useEchoCancellation) { + // 如果使用回音消除，创建自定义音频输入流 + setupCustomAudioProcessing() + + if (isAutoDetectLanguage) { + val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) + recognizer = SpeechRecognizer(speechConfig, autoDetectConfig, audioConfig) + } else { + recognizer = SpeechRecognizer(speechConfig, audioConfig) + } } else { - recognizer = SpeechRecognizer(speechConfig) + // 使用默认麦克风输入 + if (isAutoDetectLanguage) { + val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList()) + recognizer = SpeechRecognizer(speechConfig, autoDetectConfig) + } else { + recognizer = SpeechRecognizer(speechConfig) + } } return true } else { @@ -122,17 +170,24 @@ class AzureAsrHelper(private val context: Context) { } try { + // 启动音频处理 + startCustomAudioProcessing() + // 执行识别 val result = recognizer?.recognizeOnceAsync()?.get() + // 停止音频处理 + stopCustomAudioProcessing() + if (result != null && result.reason == ResultReason.RecognizedSpeech) { val detectedLanguage = AutoDetectSourceLanguageResult.fromResult(result)?.language - callback.onResult(result.text, detectedLanguage ?: "") } else { callback.onError("未能识别语音") } } catch (e: Exception) { + // 停止音频处理 + stopCustomAudioProcessing() callback.onError("识别异常: ${e.message}") } } @@ -156,6 +211,9 @@ class AzureAsrHelper(private val context: Context) { } try { + // 启动音频处理 + startCustomAudioProcessing() + // 设置识别事件处理 // 最终识别结果 recognizer?.recognized?.addEventListener( @@ -219,6 +277,7 @@ class AzureAsrHelper(private val context: Context) { } catch (e: Exception) { callback.onError("开始连续识别失败: ${e.message}") isContinuousRecognitionActive = false + stopCustomAudioProcessing() return false } } @@ -233,10 +292,15 @@ class AzureAsrHelper(private val context: Context) { recognizer?.stopContinuousRecognitionAsync() isContinuousRecognitionActive = false callback.onSessionStopped() + + // 停止音频处理 + stopCustomAudioProcessing() + return true } catch (e: Exception) { callback.onError("停止连续识别失败: ${e.message}") isContinuousRecognitionActive = false + stopCustomAudioProcessing() return false } } @@ -245,10 +309,90 @@ class AzureAsrHelper(private val context: Context) { fun isContinuousRecognitionActive(): Boolean { return isContinuousRecognitionActive } + + // 设置自定义音频处理 + private fun setupCustomAudioProcessing() { + if (!useEchoCancellation) { + return + } + + try { + // 1. 创建PushAudioInputStream + pushStream = PushAudioInputStream.create() + + // 2. 创建AudioConfig + audioConfig = AudioConfig.fromStreamInput(pushStream) + + // 3. 创建自定义音频处理器 + customAudioProcessor = CustomAudioProcessor(pushStream) + + FileLogger.d(TAG, "自定义音频处理设置完成") + } catch (e: Exception) { + FileLogger.e(TAG, "设置自定义音频处理失败: ${e.message}") + releaseCustomAudioProcessing() + + // 降级处理：如果自定义处理设置失败，尝试使用默认麦克风 + try { + FileLogger.d(TAG, "尝试降级到默认麦克风输入") + audioConfig = AudioConfig.fromDefaultMicrophoneInput() + } catch (e2: Exception) { + FileLogger.e(TAG, "默认麦克风输入设置也失败: ${e2.message}") + audioConfig = null + } + } + } + + // 启动自定义音频处理 + private fun startCustomAudioProcessing() { + if (!useEchoCancellation || customAudioProcessor == null) { + return + } + + try { + customAudioProcessor?.startRecording() + FileLogger.d(TAG, "自定义音频处理已启动") + } catch (e: Exception) { + FileLogger.e(TAG, "启动自定义音频处理失败: ${e.message}") + } + } + + // 停止自定义音频处理 + private fun stopCustomAudioProcessing() { + if (!useEchoCancellation || customAudioProcessor == null) { + return + } + + try { + customAudioProcessor?.stopRecording() + FileLogger.d(TAG, "自定义音频处理已停止") + } catch (e: Exception) { + FileLogger.e(TAG, "停止自定义音频处理失败: ${e.message}") + } + } + + // 释放自定义音频处理资源 + private fun releaseCustomAudioProcessing() { + stopCustomAudioProcessing() + + try { + customAudioProcessor = null + pushStream?.close() + pushStream = null + audioConfig?.close() + audioConfig = null + + FileLogger.d(TAG, "自定义音频处理资源已释放") + } catch (e: Exception) { + FileLogger.e(TAG, "释放自定义音频处理资源时出错: ${e.message}") + } + } // 释放所有资源 fun dispose() { try { + // 停止和释放音频处理 + releaseCustomAudioProcessing() + recognizer?.close() recognizer = null @@ -263,6 +407,235 @@ class AzureAsrHelper(private val context: Context) { } } + // 自定义音频处理器 - 使用Android原生回音消除 + private inner class CustomAudioProcessor(private val pushStream: PushAudioInputStream?) { + private val SAMPLE_RATE = 16000 + private val CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO + private val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT + private val BUFFER_SIZE = SAMPLE_RATE * 2 // 简化缓冲区大小计算，更稳定 + + private var audioRecord: AudioRecord? = null + private var echoCanceler: AcousticEchoCanceler? = null + private val isRecording = AtomicBoolean(false) + private var recordingThread: Thread? = null + + // 启动录音并处理音频数据 + fun startRecording() { + if (isRecording.get() || pushStream == null) { + return + } + + try { + // 使用Builder模式构建AudioFormat + val audioFormat = AudioFormat.Builder() + .setSampleRate(SAMPLE_RATE) + .setEncoding(AUDIO_FORMAT) + .setChannelMask(CHANNEL_CONFIG) + .build() + + // 使用Builder模式创建AudioRecord实例 + audioRecord = AudioRecord.Builder() + .setAudioSource(MediaRecorder.AudioSource.VOICE_COMMUNICATION) + .setAudioFormat(audioFormat) + .setBufferSizeInBytes(BUFFER_SIZE) + .build() + + // 检查AudioRecord初始化状态 + if (audioRecord?.state != AudioRecord.STATE_INITIALIZED) { + FileLogger.e(TAG, "AudioRecord初始化失败，状态: ${audioRecord?.state}") + // 尝试使用DEFAULT音频源重试一次 + audioRecord?.release() + audioRecord = AudioRecord.Builder() + .setAudioSource(MediaRecorder.AudioSource.DEFAULT) + .setAudioFormat(audioFormat) + .setBufferSizeInBytes(BUFFER_SIZE) + .build() + + if (audioRecord?.state != AudioRecord.STATE_INITIALIZED) { + FileLogger.e(TAG, "AudioRecord初始化第二次尝试也失败，放弃") + releaseAudioResources() + return + } else { + FileLogger.d(TAG, "使用默认音频源成功初始化AudioRecord") + } + } + + // 启用音频效果（回音消除、噪声抑制等） + enableAudioEffects() + + // 启动录音 + audioRecord?.startRecording() + isRecording.set(true) + + // 创建录音线程 + recordingThread = Thread({ + val buffer = ByteArray(BUFFER_SIZE) + + while (isRecording.get()) { + try { + val readSize = audioRecord?.read(buffer, 0, BUFFER_SIZE) ?: 0 + + if (readSize > 0) { + try { + // 将处理后的音频数据推送到流 + if (readSize == buffer.size) { + // 如果读取的大小等于buffer的大小，直接写入整个buffer + pushStream.write(buffer) + } else { + // 如果只读取了部分数据，创建新的数组只包含有效数据 + val validData = buffer.copyOfRange(0, readSize) + pushStream.write(validData) + } + } catch (e: Exception) { + FileLogger.e(TAG, "写入音频数据失败: ${e.message}") + break + } + } else if (readSize == 0) { + // 读取为0，可能是临时的，等待一下继续尝试 + Thread.sleep(10) + } else { + // 负值表示错误 + FileLogger.e(TAG, "读取音频数据失败，错误码: $readSize") + break + } + } catch (e: Exception) { + FileLogger.e(TAG, "录音线程异常: ${e.message}") + break + } + } + }, "AudioRecordingThread") + + // 设置线程优先级并启动 + recordingThread?.priority = Thread.MAX_PRIORITY + recordingThread?.start() + + FileLogger.d(TAG, "音频录制已启动" + (if(echoCanceler?.enabled == true) "，回音消除已启用" else "")) + } catch (e: Exception) { + FileLogger.e(TAG, "启动音频录制失败: ${e.message}") + releaseAudioResources() + } + } + + // 启用音频效果（回音消除、噪声抑制等） + private fun enableAudioEffects() { + try { + val audioSessionId = audioRecord?.audioSessionId ?: -1 + + if (audioSessionId != -1) { + // 启用回音消除 + if (AcousticEchoCanceler.isAvailable()) { + try { + echoCanceler = AcousticEchoCanceler.create(audioSessionId) + if (echoCanceler != null) { + echoCanceler?.enabled = true + FileLogger.d(TAG, "回音消除已启用，会话ID: $audioSessionId") + } else { + FileLogger.w(TAG, "回音消除器创建返回null") + } + } catch (e: Exception) { + FileLogger.e(TAG, "创建回音消除器时出错: ${e.message}") + } + } else { + FileLogger.d(TAG, "设备不支持回音消除") + } + + // 以下功能暂时不启用，可根据需要取消注释 + /* + // 启用噪声抑制 + if (NoiseSuppressor.isAvailable()) { + try { + val ns = NoiseSuppressor.create(audioSessionId) + ns?.enabled = true + FileLogger.d(TAG, "噪声抑制已启用") + } catch (e: Exception) { + FileLogger.e(TAG, "创建噪声抑制器时出错: ${e.message}") + } + } + + // 启用自动增益控制 + if (AutomaticGainControl.isAvailable()) { + try { + val agc = AutomaticGainControl.create(audioSessionId) + agc?.enabled = true + FileLogger.d(TAG, "自动增益控制已启用") + } catch (e: Exception) { + FileLogger.e(TAG, "创建自动增益控制时出错: ${e.message}") + } + } + */ + } else { + FileLogger.w(TAG, "无效的音频会话ID，无法启用音频效果") + } + } catch (e: Exception) { + FileLogger.e(TAG, "启用音频效果时出错: ${e.message}") + } + } + + // 停止录音 + fun stopRecording() { + if (!isRecording.get()) { + return + } + + isRecording.set(false) + + try { + // 等待录音线程结束 + recordingThread?.join(1000) + + // 释放资源 + releaseAudioResources() + + FileLogger.d(TAG, "音频录制已停止") + } catch (e: Exception) { + FileLogger.e(TAG, "停止音频录制失败: ${e.message}") + } + } + + // 释放音频资源 + private fun releaseAudioResources() { + try { + // 停止录音 + try { + if (audioRecord?.state == AudioRecord.STATE_INITIALIZED) { + audioRecord?.stop() + } + } catch (e: Exception) { + // 忽略可能的IllegalStateException + FileLogger.w(TAG, "停止AudioRecord时出错: ${e.message}") + } + + // 释放回音消除器 + try { + if (echoCanceler != null) { + echoCanceler?.enabled = false + echoCanceler?.release() + echoCanceler = null + } + } catch (e: Exception) { + FileLogger.w(TAG, "释放回音消除器时出错: ${e.message}") + } finally { + echoCanceler = null + } + + // 释放音频记录器 + try { + audioRecord?.release() + } catch (e: Exception) { + FileLogger.w(TAG, "释放AudioRecord时出错: ${e.message}") + } finally { + audioRecord = null + } + + // 重置线程 + recordingThread = null + + } catch (e: Exception) { + FileLogger.e(TAG, "释放音频资源失败: ${e.message}") + } + } + } + // 一次性识别回调接口 interface RecognizeCallback { fun onResult(result: String, detectedLanguage: String = "") @@ -278,5 +651,4 @@ class AzureAsrHelper(private val context: Context) { fun onCanceled(reason: String, errorDetails: String) fun onError(error: String) } - } \ No newline at end of file diff --git a/ios/Runner/AzureAsrHelper.swift b/ios/Runner/AzureAsrHelper.swift index 7eb272c50..5a2b73189 100644 --- a/ios/Runner/AzureAsrHelper.swift +++ b/ios/Runner/AzureAsrHelper.swift @@ -9,7 +9,7 @@ class AzureAsrHelper: NSObject { // MARK: - 属性 /// 是否使用自定义音频处理器 - 设置为false时将使用系统默认麦克风输入 - private let useCustomAudioProcessor = false + private let useCustomAudioProcessor = true /// 事件处理回调 private var eventHandler: (([String: Any]) -> Void)? diff --git a/lib/modules/home/controllers/home_controller.dart b/lib/modules/home/controllers/home_controller.dart index c15c5eb42..7e5eef94a 100644 --- a/lib/modules/home/controllers/home_controller.dart +++ b/lib/modules/home/controllers/home_controller.dart @@ -205,6 +205,7 @@ class HomeController extends GetxController { // 刷新设备信息 Future refreshDeviceInfo() async { + return; // 如果正在更新，则跳过 if (_isUpdatingStatus) return;