优化面对面翻译，加入降噪

1 year ago · 47739817ff
11 changed files with 115 additions and 74 deletions
--- a/lib/data/services/asr_service.dart
+++ b/lib/data/services/asr_service.dart
@ -26,11 +26,16 @@ abstract class AsrService {
  /// 停止连续语音识别
  Future<bool> stopContinuousRecognition();

+  /// 停止音频输入
  Future<void> setPause();

-  Future<void> setcontinue();
+  /// 继续音频输入
+  Future<void> setContinue();
+
+  /// 恢复原始音频状态
  Future<void> restoreOriginalAudioState();

+  /// 禁用蓝牙音频
  Future<void> disableBluetoothAudio();

  /// 检查连续识别是否活跃
--- a/lib/data/services/speech_impl/azure_asr_service.dart
+++ b/lib/data/services/speech_impl/azure_asr_service.dart
@ -429,7 +429,7 @@ class AzureAsrService extends GetxService implements AsrService {
  }

  @override
-  Future<void> setcontinue() async {
+  Future<void> setContinue() async {
    // TODO: implement setcontinue
    try {
      final bool result = await _channel.invokeMethod('setcontinue');
--- a/lib/data/services/speech_impl/volcano_asr_api_service.dart
+++ b/lib/data/services/speech_impl/volcano_asr_api_service.dart
@ -849,7 +849,7 @@ class VolcanoAsrApiService implements AsrService {
  }

  @override
-  Future<void> setcontinue() {
+  Future<void> setContinue() {
    // TODO: implement setcontinue
    throw UnimplementedError();
  }
--- a/lib/data/services/speech_impl/volcano_asr_service.dart
+++ b/lib/data/services/speech_impl/volcano_asr_service.dart
@ -384,7 +384,7 @@ class VolcanoAsrService extends GetxService implements AsrService {
  }

  @override
-  Future<void> setcontinue() {
+  Future<void> setContinue() {
    // TODO: implement setcontinue
    throw UnimplementedError();
  }
--- a/lib/data/services/speech_impl/xunfei_asr_service.dart
+++ b/lib/data/services/speech_impl/xunfei_asr_service.dart
@ -308,7 +308,7 @@ class XunfeiAsrService extends GetxService implements AsrService {
  }

  @override
-  Future<void> setcontinue() {
+  Future<void> setContinue() {
    // TODO: implement setcontinue
    throw UnimplementedError();
  }
--- a/lib/modules/translation/controllers/translation_controller.dart
+++ b/lib/modules/translation/controllers/translation_controller.dart
@ -37,10 +37,13 @@ class TranslationController extends GetxController {
  final bleManager = Get.find<BleManager>();
  // 音频输入源
  bool _audioSourceType = false;
-  // 倒计时器
-  bool istts = false;
+//是否开启播放
+  bool isPlayback = false;
+  //录音计时器
  final RxString recordDurationText = '00:00'.obs;
+  //记录计时器
  Timer? _recordTimer;
+  //秒
  int _seconds = 0;
  // 存储相关
  static const String _historyKey = 'translation_history';
@ -99,23 +102,24 @@ class TranslationController extends GetxController {
    activeSpeaker.value = id;
    // 监听识别结果
    if (activeSpeaker.value != 0) {
-      istts = false;
-      _asrService.setcontinue();
+      isPlayback = false;
+      _asrService.setContinue();
      lastActiveSpeaker.value = id;
    } else {
      _asrService.setPause();
-      if (fTFTranslationResult != '' && !istts) {
-        istts = true;
+      if (fTFTranslationResult != '' && !isPlayback) {
+        isPlayback = true;
        await _ttsService.setAudioOutputDevice(lastActiveSpeaker.value);
+        await Future.delayed(const Duration(milliseconds: 1300));
        await playTranslatedText(fTFTranslationResult!);
        fTFTranslationResult = '';
      }
    }
  }

-  // 切换TTS功能
+  //  禁用蓝牙音频
  void disableBluetoothAudio() => _asrService.disableBluetoothAudio();
-  // 切换TTS功能
+  //恢复原始音频状态
  void restoreOriginalAudioState() => _asrService.restoreOriginalAudioState();

  // 兼容旧版视图的计算属性
@ -379,7 +383,8 @@ class TranslationController extends GetxController {
      if (currentMode.value == 'faceToFace') {
        _asrService.setPause();
      } else {
-        _asrService.setcontinue();
+        await _ttsService.setAudioOutputDevice(0);
+        _asrService.setContinue();
      }
      isRecognizing.value = true;
      if (isRecording.value && isRecognizing.value) {
@ -577,19 +582,20 @@ class TranslationController extends GetxController {
            fTFTranslationResult =
                (fTFTranslationResult ?? '') + translationResult;

-            if (activeSpeaker.value == 0 && !istts) {
-              istts = true;
+            if (activeSpeaker.value == 0 &&
+                !isPlayback &&
+                currentMode.value == 'faceToFace') {
+              isPlayback = true;
              await _ttsService.setAudioOutputDevice(lastActiveSpeaker.value);
+              await Future.delayed(const Duration(milliseconds: 1300));
              await playTranslatedText(fTFTranslationResult!);
              fTFTranslationResult = '';
-              ;
            }
          }
          // 播放TTS - 只在最终结果时播放
          if (isTtsEnabled.value &&
              isFinal &&
              currentMode.value != 'faceToFace') {
-            _ttsService.setAudioOutputDevice(activeSpeaker.value);
            await playTranslatedText(translationResult);
          }
        } else {
--- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
+++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
@ -80,6 +80,7 @@ class AzureAsrHelper(private val context: Context) {
        /** 使用外部提供的音频数据 */
        EXTERNAL
    }
+
    /**
     * 获取设备支持的最佳音频格式
     * 优先选择16000Hz，若不支持则降级到8000Hz
@ -103,6 +104,7 @@ class AzureAsrHelper(private val context: Context) {
        // 创建对应的音频格式
        return AudioStreamFormat.getWaveFormatPCM(sampleRate.toLong(), 16, 1)
    }
+
    /**
     * 初始化Azure语音服务
     *
@ -391,8 +393,7 @@ class AzureAsrHelper(private val context: Context) {
            audioRecord?.startRecording()
            startCaptureThread()          // 再启动数据读取线程
            isContinuousRecognitionActive = true
-Log.d(tag, "识别器创建成功: ${recognizer != null}")
-Log.d(tag, "开始识别任务: recognizeOnce 或 startContinuousRecognition 被调用")
+

            return true
        } catch (e: Exception) {
@ -751,7 +752,21 @@ Log.d(tag, "开始识别任务: recognizeOnce 或 startContinuousRecognition 被
            channelConfig,
            audioFormat
        )
-
+// 创建录音对象
+        if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.M) {
+            Log.d("AzureASR", "android.os.Build.VERSION.SDK_INT")
+            val format = AudioFormat.Builder()
+                .setSampleRate(sampleRate)
+                .setEncoding(audioFormat)
+                .setChannelMask(channelConfig)
+                .build()
+
+            audioRecord = AudioRecord.Builder()
+                .setAudioSource(MediaRecorder.AudioSource.VOICE_COMMUNICATION)
+                .setAudioFormat(format)
+                .setBufferSizeInBytes(minBufferSize * 2)
+                .build()
+        } else {
            // 创建 AudioRecord 对象
            audioRecord = AudioRecord(
                MediaRecorder.AudioSource.VOICE_RECOGNITION,  // 语音识别专用音源（自带降噪/回声消除）
@ -761,9 +776,15 @@ Log.d(tag, "开始识别任务: recognizeOnce 或 startContinuousRecognition 被
                minBufferSize * 2  // 使用两倍缓冲区防止溢出
            )

+        }
+
+
+        // 检查初始化状态
+        if (audioRecord?.state != AudioRecord.STATE_INITIALIZED) {
+            throw IllegalStateException("AudioRecord初始化失败")
+        }
+

-        // 应用音频效果
-        applyAudioEffects()
    }

    // 3. 单独封装线程启动逻辑
@ -781,7 +802,6 @@ Log.d(tag, "开始识别任务: recognizeOnce 或 startContinuousRecognition 被
                )
                bufferSize > 0 // 返回正值表示支持
            } ?: 16000 // 默认回退值
-     // (sampleRate * 0.032).toInt() * 2 // 32ms数据量

            val buffer = ByteArray(bufferSize)

@ -789,8 +809,6 @@ Log.d(tag, "开始识别任务: recognizeOnce 或 startContinuousRecognition 被

                // 读取音频
                val bytesRead = audioRecord?.read(buffer, 0, buffer.size) ?: 0
-Log.i("AzureASR", "AudioRecord state: ${audioRecord?.state}, recordingState: ${audioRecord?.recordingState}")
-Log.i("AzureASR", "BytesRead = $bytesRead, buffer.isSilent = ${buffer.all { it == 0.toByte() }}")

                if (!isPaused && bytesRead > 0) {

--- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt
+++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt
@ -257,20 +257,24 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {

                "setPause" -> {
                    azureAsrHelper.stopAudioPause();
+                     return
                }

                "setcontinue" -> {
                    //  azureAsrHelper.stopAudioPause();
                    azureAsrHelper.setAudiocontinue();
+                     return
                }

                "disableBluetoothAudio" -> {
                    azureAsrHelper.disableBluetoothAudio();
+                     return
                }

                "restoreOriginalAudioState" -> {
                    //  azureAsrHelper.stopAudioPause();
                    azureAsrHelper.restoreOriginalAudioState();
+                     return
                }

                "startContinuousRecognition" -> {
@ -523,19 +527,20 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {

                "setAudioOutputDevice" -> {
                    val type = call.argument<Int>("type")
+                   var success =false
                    if (type == 0) {
                        // 默认（如果有耳机选耳机，否则使用系统扬声器）
-                        azureTtsHelper.setAudioOutputDevice(AudioOutputDevice.DEFAULT)
+                        success=azureTtsHelper.setAudioOutputDevice(AudioOutputDevice.DEFAULT)
                    } else if (type == 1) {
                        // 强制使用声器
-                        azureTtsHelper.setAudioOutputDevice(AudioOutputDevice.SPEAKER)
+                        success=azureTtsHelper.setAudioOutputDevice(AudioOutputDevice.SPEAKER)
                    } else if (type == 2) {
                        // 强制使用耳机
-                        azureTtsHelper.setAudioOutputDevice(AudioOutputDevice.HEADPHONES)
+                        success=azureTtsHelper.setAudioOutputDevice(AudioOutputDevice.HEADPHONES)
                    }


-                    result.success(true)
+                    result.success(success)
                }

                else -> {
--- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt
+++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt
@ -210,9 +210,10 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
     *
     * @param device 音频输出设备类型
     */
-    override fun setAudioOutputDevice(device: AudioOutputDevice) {
+    override fun setAudioOutputDevice(device: AudioOutputDevice): Boolean{
        // Azure TTS使用系统默认的音频路由
        // 音频输出设备的控制需要通过Android的AudioManager实现
+          try {
        FileLogger.e(TAG, "音频输出设备类型device=: ${device}")
        val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager
        audioManager?.let { manager ->
@ -244,6 +245,11 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
                }
            }
        }
+        return true
+         } catch (e: Exception) {
+            FileLogger.e(TAG, "设置音频输出设备失败: ${e.message}")
+            return false
+        }
    }

    /**
--- a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt
+++ b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt
@ -425,10 +425,11 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
    /**
     * 设置音频输出设备
     */
-    override fun setAudioOutputDevice(device: AudioOutputDevice) {
+    override fun setAudioOutputDevice(device: AudioOutputDevice) : Boolean{
        if (useInternalPlayer) {
            audioPlayer.setAudioOutputDevice(device)
        }
+        return true;
    }
    
    /**
--- a/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/ITtsService.kt
+++ b/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/ITtsService.kt
@ -103,5 +103,5 @@ interface ITtsService {
     * 
     * @param device 音频输出设备类型
     */
-    fun setAudioOutputDevice(device: AudioOutputDevice)
+    fun setAudioOutputDevice(device: AudioOutputDevice): Boolean
 }