liwei1dao
/
deepvoice_client
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

add

weicu
wolfplus 2 years ago
parent
a1c3903524
commit
c42c0f6bab
4 changed files with 389 additions and 14 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 8
      android/app/build.gradle.kts
  2. 392
      android/app/src/main/kotlin/com/example/deep_voice/AzureAsrHelper.kt
  3. 2
      ios/Runner/AzureAsrHelper.swift
  4. 1
      lib/modules/home/controllers/home_controller.dart

8
android/app/build.gradle.kts
View File

@ -1,9 +1,8 @@
repositories {
google()
mavenCentral()
maven {
url = uri("https://artifact.bytedance.com/repository/Volcengine/")
}
maven { url = uri("https://artifact.bytedance.com/repository/Volcengine/") }
maven { url = uri("https://oss.sonatype.org/content/repositories/snapshots/") }
}
plugins {
@ -50,6 +49,9 @@ android {
}
dependencies {
// WebRTC依赖已移除
// implementation("io.antmedia:webrtc-android-framework:2.8.0-SNAPSHOT")
// 添加OkHttp依赖
implementation("com.squareup.okhttp3:okhttp:4.9.3")

392
android/app/src/main/kotlin/com/example/deep_voice/AzureAsrHelper.kt
View File

@ -1,11 +1,21 @@
package com.example.deep_voice
import android.content.Context
import android.media.AudioAttributes
import android.media.AudioFormat
import android.media.AudioRecord
import android.media.MediaRecorder
import android.media.audiofx.AcousticEchoCanceler
import android.media.audiofx.NoiseSuppressor
import android.media.audiofx.AutomaticGainControl
import android.os.Process
import com.example.deep_voice.core.utils.FileLogger
import com.microsoft.cognitiveservices.speech.*
import com.microsoft.cognitiveservices.speech.audio.*
import com.microsoft.cognitiveservices.speech.util.EventHandler
import java.util.concurrent.ExecutionException
import java.util.concurrent.atomic.AtomicBoolean
// 移除WebRTC相关导入
class AzureAsrHelper(private val context: Context) {
private var recognizer: SpeechRecognizer? = null
@ -18,6 +28,14 @@ class AzureAsrHelper(private val context: Context) {
private var isAutoDetectLanguage = false
private var supportedLanguages = arrayOf("zh-CN", "en-US")
// 是否使用回音消除 - 内部控制常量
private val useEchoCancellation = true
// 自定义音频处理相关
private var customAudioProcessor: CustomAudioProcessor? = null
private var pushStream: PushAudioInputStream? = null
private var audioConfig: AudioConfig? = null
// 初始化SDK并创建recognizer
fun initialize(subscriptionKey: String, serviceRegion: String,
supportedLanguages: Array<String> = arrayOf("zh-CN", "en-US")): Boolean {
@ -63,17 +81,31 @@ class AzureAsrHelper(private val context: Context) {
// 创建识别器
try {
if (isAutoDetectLanguage) {
val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList())
recognizer = SpeechRecognizer(speechConfig, autoDetectConfig)
if (useEchoCancellation) {
// 如果使用回音消除，创建自定义音频输入流
setupCustomAudioProcessing()
if (isAutoDetectLanguage) {
val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList())
recognizer = SpeechRecognizer(speechConfig, autoDetectConfig, audioConfig)
} else {
recognizer = SpeechRecognizer(speechConfig, audioConfig)
}
} else {
recognizer = SpeechRecognizer(speechConfig)
// 使用默认麦克风输入
if (isAutoDetectLanguage) {
val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList())
recognizer = SpeechRecognizer(speechConfig, autoDetectConfig)
} else {
recognizer = SpeechRecognizer(speechConfig)
}
}
FileLogger.d(TAG, "Azure 语音服务初始化成功")
return true
} catch (e: Exception) {
FileLogger.e(TAG, "创建识别器失败: ${e.message}")
stopCustomAudioProcessing()
return false
}
} catch (e: Exception) {
@ -89,13 +121,29 @@ class AzureAsrHelper(private val context: Context) {
recognizer?.close()
recognizer = null
// 停止当前的音频处理
stopCustomAudioProcessing()
// 使用现有配置重新创建 recognizer
if (speechConfig != null) {
if (isAutoDetectLanguage) {
val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList())
recognizer = SpeechRecognizer(speechConfig, autoDetectConfig)
if (useEchoCancellation) {
// 如果使用回音消除，创建自定义音频输入流
setupCustomAudioProcessing()
if (isAutoDetectLanguage) {
val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList())
recognizer = SpeechRecognizer(speechConfig, autoDetectConfig, audioConfig)
} else {
recognizer = SpeechRecognizer(speechConfig, audioConfig)
}
} else {
recognizer = SpeechRecognizer(speechConfig)
// 使用默认麦克风输入
if (isAutoDetectLanguage) {
val autoDetectConfig = AutoDetectSourceLanguageConfig.fromLanguages(supportedLanguages.toList())
recognizer = SpeechRecognizer(speechConfig, autoDetectConfig)
} else {
recognizer = SpeechRecognizer(speechConfig)
}
}
return true
} else {
@ -122,17 +170,24 @@ class AzureAsrHelper(private val context: Context) {
}
try {
// 启动音频处理
startCustomAudioProcessing()
// 执行识别
val result = recognizer?.recognizeOnceAsync()?.get()
// 停止音频处理
stopCustomAudioProcessing()
if (result != null && result.reason == ResultReason.RecognizedSpeech) {
val detectedLanguage = AutoDetectSourceLanguageResult.fromResult(result)?.language
callback.onResult(result.text, detectedLanguage ?: "")
} else {
callback.onError("未能识别语音")
}
} catch (e: Exception) {
// 停止音频处理
stopCustomAudioProcessing()
callback.onError("识别异常: ${e.message}")
}
}
@ -156,6 +211,9 @@ class AzureAsrHelper(private val context: Context) {
}
try {
// 启动音频处理
startCustomAudioProcessing()
// 设置识别事件处理
// 最终识别结果
recognizer?.recognized?.addEventListener(
@ -219,6 +277,7 @@ class AzureAsrHelper(private val context: Context) {
} catch (e: Exception) {
callback.onError("开始连续识别失败: ${e.message}")
isContinuousRecognitionActive = false
stopCustomAudioProcessing()
return false
}
}
@ -233,10 +292,15 @@ class AzureAsrHelper(private val context: Context) {
recognizer?.stopContinuousRecognitionAsync()
isContinuousRecognitionActive = false
callback.onSessionStopped()
// 停止音频处理
stopCustomAudioProcessing()
return true
} catch (e: Exception) {
callback.onError("停止连续识别失败: ${e.message}")
isContinuousRecognitionActive = false
stopCustomAudioProcessing()
return false
}
}
@ -246,9 +310,89 @@ class AzureAsrHelper(private val context: Context) {
return isContinuousRecognitionActive
}
// 设置自定义音频处理
private fun setupCustomAudioProcessing() {
if (!useEchoCancellation) {
return
}
try {
// 1. 创建PushAudioInputStream
pushStream = PushAudioInputStream.create()
// 2. 创建AudioConfig
audioConfig = AudioConfig.fromStreamInput(pushStream)
// 3. 创建自定义音频处理器
customAudioProcessor = CustomAudioProcessor(pushStream)
FileLogger.d(TAG, "自定义音频处理设置完成")
} catch (e: Exception) {
FileLogger.e(TAG, "设置自定义音频处理失败: ${e.message}")
releaseCustomAudioProcessing()
// 降级处理：如果自定义处理设置失败，尝试使用默认麦克风
try {
FileLogger.d(TAG, "尝试降级到默认麦克风输入")
audioConfig = AudioConfig.fromDefaultMicrophoneInput()
} catch (e2: Exception) {
FileLogger.e(TAG, "默认麦克风输入设置也失败: ${e2.message}")
audioConfig = null
}
}
}
// 启动自定义音频处理
private fun startCustomAudioProcessing() {
if (!useEchoCancellation || customAudioProcessor == null) {
return
}
try {
customAudioProcessor?.startRecording()
FileLogger.d(TAG, "自定义音频处理已启动")
} catch (e: Exception) {
FileLogger.e(TAG, "启动自定义音频处理失败: ${e.message}")
}
}
// 停止自定义音频处理
private fun stopCustomAudioProcessing() {
if (!useEchoCancellation || customAudioProcessor == null) {
return
}
try {
customAudioProcessor?.stopRecording()
FileLogger.d(TAG, "自定义音频处理已停止")
} catch (e: Exception) {
FileLogger.e(TAG, "停止自定义音频处理失败: ${e.message}")
}
}
// 释放自定义音频处理资源
private fun releaseCustomAudioProcessing() {
stopCustomAudioProcessing()
try {
customAudioProcessor = null
pushStream?.close()
pushStream = null
audioConfig?.close()
audioConfig = null
FileLogger.d(TAG, "自定义音频处理资源已释放")
} catch (e: Exception) {
FileLogger.e(TAG, "释放自定义音频处理资源时出错: ${e.message}")
}
}
// 释放所有资源
fun dispose() {
try {
// 停止和释放音频处理
releaseCustomAudioProcessing()
recognizer?.close()
recognizer = null
@ -263,6 +407,235 @@ class AzureAsrHelper(private val context: Context) {
}
}
// 自定义音频处理器 - 使用Android原生回音消除
private inner class CustomAudioProcessor(private val pushStream: PushAudioInputStream?) {
private val SAMPLE_RATE = 16000
private val CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO
private val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT
private val BUFFER_SIZE = SAMPLE_RATE * 2 // 简化缓冲区大小计算，更稳定
private var audioRecord: AudioRecord? = null
private var echoCanceler: AcousticEchoCanceler? = null
private val isRecording = AtomicBoolean(false)
private var recordingThread: Thread? = null
// 启动录音并处理音频数据
fun startRecording() {
if (isRecording.get() || pushStream == null) {
return
}
try {
// 使用Builder模式构建AudioFormat
val audioFormat = AudioFormat.Builder()
.setSampleRate(SAMPLE_RATE)
.setEncoding(AUDIO_FORMAT)
.setChannelMask(CHANNEL_CONFIG)
.build()
// 使用Builder模式创建AudioRecord实例
audioRecord = AudioRecord.Builder()
.setAudioSource(MediaRecorder.AudioSource.VOICE_COMMUNICATION)
.setAudioFormat(audioFormat)
.setBufferSizeInBytes(BUFFER_SIZE)
.build()
// 检查AudioRecord初始化状态
if (audioRecord?.state != AudioRecord.STATE_INITIALIZED) {
FileLogger.e(TAG, "AudioRecord初始化失败，状态: ${audioRecord?.state}")
// 尝试使用DEFAULT音频源重试一次
audioRecord?.release()
audioRecord = AudioRecord.Builder()
.setAudioSource(MediaRecorder.AudioSource.DEFAULT)
.setAudioFormat(audioFormat)
.setBufferSizeInBytes(BUFFER_SIZE)
.build()
if (audioRecord?.state != AudioRecord.STATE_INITIALIZED) {
FileLogger.e(TAG, "AudioRecord初始化第二次尝试也失败，放弃")
releaseAudioResources()
return
} else {
FileLogger.d(TAG, "使用默认音频源成功初始化AudioRecord")
}
}
// 启用音频效果（回音消除、噪声抑制等）
enableAudioEffects()
// 启动录音
audioRecord?.startRecording()
isRecording.set(true)
// 创建录音线程
recordingThread = Thread({
val buffer = ByteArray(BUFFER_SIZE)
while (isRecording.get()) {
try {
val readSize = audioRecord?.read(buffer, 0, BUFFER_SIZE) ?: 0
if (readSize > 0) {
try {
// 将处理后的音频数据推送到流
if (readSize == buffer.size) {
// 如果读取的大小等于buffer的大小，直接写入整个buffer
pushStream.write(buffer)
} else {
// 如果只读取了部分数据，创建新的数组只包含有效数据
val validData = buffer.copyOfRange(0, readSize)
pushStream.write(validData)
}
} catch (e: Exception) {
FileLogger.e(TAG, "写入音频数据失败: ${e.message}")
break
}
} else if (readSize == 0) {
// 读取为0，可能是临时的，等待一下继续尝试
Thread.sleep(10)
} else {
// 负值表示错误
FileLogger.e(TAG, "读取音频数据失败，错误码: $readSize")
break
}
} catch (e: Exception) {
FileLogger.e(TAG, "录音线程异常: ${e.message}")
break
}
}
}, "AudioRecordingThread")
// 设置线程优先级并启动
recordingThread?.priority = Thread.MAX_PRIORITY
recordingThread?.start()
FileLogger.d(TAG, "音频录制已启动" + (if(echoCanceler?.enabled == true) "，回音消除已启用" else ""))
} catch (e: Exception) {
FileLogger.e(TAG, "启动音频录制失败: ${e.message}")
releaseAudioResources()
}
}
// 启用音频效果（回音消除、噪声抑制等）
private fun enableAudioEffects() {
try {
val audioSessionId = audioRecord?.audioSessionId ?: -1
if (audioSessionId != -1) {
// 启用回音消除
if (AcousticEchoCanceler.isAvailable()) {
try {
echoCanceler = AcousticEchoCanceler.create(audioSessionId)
if (echoCanceler != null) {
echoCanceler?.enabled = true
FileLogger.d(TAG, "回音消除已启用，会话ID: $audioSessionId")
} else {
FileLogger.w(TAG, "回音消除器创建返回null")
}
} catch (e: Exception) {
FileLogger.e(TAG, "创建回音消除器时出错: ${e.message}")
}
} else {
FileLogger.d(TAG, "设备不支持回音消除")
}
// 以下功能暂时不启用，可根据需要取消注释
/*
// 启用噪声抑制
if (NoiseSuppressor.isAvailable()) {
try {
val ns = NoiseSuppressor.create(audioSessionId)
ns?.enabled = true
FileLogger.d(TAG, "噪声抑制已启用")
} catch (e: Exception) {
FileLogger.e(TAG, "创建噪声抑制器时出错: ${e.message}")
}
}
// 启用自动增益控制
if (AutomaticGainControl.isAvailable()) {
try {
val agc = AutomaticGainControl.create(audioSessionId)
agc?.enabled = true
FileLogger.d(TAG, "自动增益控制已启用")
} catch (e: Exception) {
FileLogger.e(TAG, "创建自动增益控制时出错: ${e.message}")
}
}
*/
} else {
FileLogger.w(TAG, "无效的音频会话ID，无法启用音频效果")
}
} catch (e: Exception) {
FileLogger.e(TAG, "启用音频效果时出错: ${e.message}")
}
}
// 停止录音
fun stopRecording() {
if (!isRecording.get()) {
return
}
isRecording.set(false)
try {
// 等待录音线程结束
recordingThread?.join(1000)
// 释放资源
releaseAudioResources()
FileLogger.d(TAG, "音频录制已停止")
} catch (e: Exception) {
FileLogger.e(TAG, "停止音频录制失败: ${e.message}")
}
}
// 释放音频资源
private fun releaseAudioResources() {
try {
// 停止录音
try {
if (audioRecord?.state == AudioRecord.STATE_INITIALIZED) {
audioRecord?.stop()
}
} catch (e: Exception) {
// 忽略可能的IllegalStateException
FileLogger.w(TAG, "停止AudioRecord时出错: ${e.message}")
}
// 释放回音消除器
try {
if (echoCanceler != null) {
echoCanceler?.enabled = false
echoCanceler?.release()
echoCanceler = null
}
} catch (e: Exception) {
FileLogger.w(TAG, "释放回音消除器时出错: ${e.message}")
} finally {
echoCanceler = null
}
// 释放音频记录器
try {
audioRecord?.release()
} catch (e: Exception) {
FileLogger.w(TAG, "释放AudioRecord时出错: ${e.message}")
} finally {
audioRecord = null
}
// 重置线程
recordingThread = null
} catch (e: Exception) {
FileLogger.e(TAG, "释放音频资源失败: ${e.message}")
}
}
}
// 一次性识别回调接口
interface RecognizeCallback {
fun onResult(result: String, detectedLanguage: String = "")
@ -278,5 +651,4 @@ class AzureAsrHelper(private val context: Context) {
fun onCanceled(reason: String, errorDetails: String)
fun onError(error: String)
}
}

2
ios/Runner/AzureAsrHelper.swift
View File

@ -9,7 +9,7 @@ class AzureAsrHelper: NSObject {
// MARK: - 属性
/// 是否使用自定义音频处理器 - 设置为false时将使用系统默认麦克风输入
private let useCustomAudioProcessor = false
private let useCustomAudioProcessor = true
/// 事件处理回调
private var eventHandler: (([String: Any]) -> Void)?

1
lib/modules/home/controllers/home_controller.dart
View File

@ -205,6 +205,7 @@ class HomeController extends GetxController {
// 刷新设备信息
Future<void> refreshDeviceInfo() async {
return;
// 如果正在更新，则跳过
if (_isUpdatingStatus) return;

Write Preview
Loading…
Cancel
Save