Browse Source

优化音视频翻译内容

weicu
liwei1dao 5 months ago
parent
commit
c1c45608b8
  1. 15
      lib/core/services/floating_window_manager.dart
  2. 2
      lib/core/translations/language/en_us.dart
  3. 2
      lib/core/translations/language/zh_cn.dart
  4. 16
      lib/data/services/asr_service.dart
  5. 39
      lib/data/services/speech_impl/azure_asr_service.dart
  6. 9
      lib/data/services/speech_impl/volcano_asr_api_service.dart
  7. 9
      lib/data/services/speech_impl/volcano_asr_service.dart
  8. 43
      lib/modules/meeting/controllers/meeting_record_controller.dart
  9. 94
      lib/modules/translation/controllers/translation_controller.dart
  10. 7
      local_plugins/azure_speech/android/src/main/AndroidManifest.xml
  11. 57
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt

15
lib/core/services/floating_window_manager.dart

@ -59,10 +59,15 @@ class FloatingWindowManager extends GetxController {
final _currentState = FloatingWindowState.hidden.obs;
final _hasPermission = false.obs;
// 用户主动点击悬浮窗关闭按钮的事件流(区别于应用代码主动 disable)。
// 让上层(如 TranslationController)能在用户叉掉浮窗时停掉对应的翻译会话。
final _onClosedByUser = StreamController<void>.broadcast();
// Getters
bool get isEnabled => _isEnabled.value;
FloatingWindowState get currentState => _currentState.value;
bool get hasPermission => _hasPermission.value;
Stream<void> get onClosedByUser => _onClosedByUser.stream;
@override
void onInit() {
@ -71,6 +76,12 @@ class FloatingWindowManager extends GetxController {
_checkPermissions();
}
@override
void onClose() {
_onClosedByUser.close();
super.onClose();
}
/// 设置方法调用处理器
void _setupMethodCallHandler() {
_channel.setMethodCallHandler((call) async {
@ -82,6 +93,10 @@ class FloatingWindowManager extends GetxController {
case 'onFloatingWindowClosed':
_currentState.value = FloatingWindowState.hidden;
_isEnabled.value = false;
// 通知监听者:用户点了悬浮窗的关闭按钮(X)
if (!_onClosedByUser.isClosed) {
_onClosedByUser.add(null);
}
break;
case 'onFloatingWindowClicked':
// 悬浮窗被点击,可以执行相应操作(如返回主应用)

2
lib/core/translations/language/en_us.dart

@ -542,6 +542,8 @@ const Map<String, String> enUS = {
'Failed to stop the recording. Please try again.', // 结束录音失败,请重试
'meetingRecordSaveFailed':
'Failed to save the recording. Please try again.', // 保存录音失败,请重试
'meetingRecordScreenCapturePermissionDenied':
'Screen recording permission is required to capture system audio. Please grant it and try again.', // 需要"录屏"权限才能录制系统音频,请同意后重试
'meetingRecordRecordingNotificationTitle': 'Recording in progress', // 录音进行中
'meetingRecordRecordingNotificationBody':
'Recording in background. Tap to return to the app.', // 正在后台录音,点击返回应用

2
lib/core/translations/language/zh_cn.dart

@ -505,6 +505,8 @@ const Map<String, String> zhCN = {
'meetingRecordStopTimeout': '结束录音超时,请重试',
'meetingRecordStopFailed': '结束录音失败,请重试',
'meetingRecordSaveFailed': '保存录音失败,请重试',
'meetingRecordScreenCapturePermissionDenied':
'需要"录屏"权限才能录制系统音频,请同意后重试',
'meetingRecordRecordingNotificationTitle': '录音进行中',
'meetingRecordRecordingNotificationBody': '正在后台录音,点击返回应用',
'overlayPermissionTitle': '悬浮窗权限',

16
lib/data/services/asr_service.dart

@ -88,6 +88,22 @@ abstract class AsrService {
/// 结束录音
Future<bool> stopRecord(bool isSave);
/// 申请系统音频捕获授权(Android MediaProjection)
///
/// 仅在 Android 上有真正的实现:会弹出系统"录屏"授权弹窗,
/// 用户授权后内部启动前台服务,开始捕获系统播放音频并通过
/// 平台层把 PCM 直接写入 [enableRecord] 创建的 WAV 文件。
/// 返回 true 表示授权成功且捕获已启动;false 表示用户拒绝或失败。
/// 其他平台默认返回 false(不支持)。
Future<bool> requestScreenCapture() async => false;
/// 停止系统音频捕获(停掉 MediaProjection 前台服务)
/// 仅 Android 上有真正实现;其他平台默认返回 true(无副作用)。
Future<bool> stopScreenCapture() async => true;
/// 查询当前是否在系统音频捕获中
Future<bool> isScreenCapturing() async => false;
/// 释放资源
Future<void> dispose();
}

39
lib/data/services/speech_impl/azure_asr_service.dart

@ -1,4 +1,5 @@
import 'dart:async';
import 'dart:io';
import 'dart:typed_data';
import '../../../data/models/appconfig.dart';
import 'package:flutter/services.dart';
@ -557,6 +558,44 @@ class AzureAsrService extends GetxService implements AsrService {
}
}
@override
Future<bool> requestScreenCapture() async {
if (!Platform.isAndroid) return false;
try {
final bool ok =
(await _channel.invokeMethod<bool>('requestScreenCapture')) ?? false;
return ok;
} catch (e) {
Logger.error('请求屏幕音频捕获授权失败: ${e.toString()}');
return false;
}
}
@override
Future<bool> stopScreenCapture() async {
if (!Platform.isAndroid) return true;
try {
final bool ok =
(await _channel.invokeMethod<bool>('stopScreenCapture')) ?? true;
return ok;
} catch (e) {
Logger.error('停止屏幕音频捕获失败: ${e.toString()}');
return false;
}
}
@override
Future<bool> isScreenCapturing() async {
if (!Platform.isAndroid) return false;
try {
final bool running =
(await _channel.invokeMethod<bool>('isScreenCapturing')) ?? false;
return running;
} catch (_) {
return false;
}
}
/// 设置音频配置
///
/// [sampleRate] 采样率,默认16000

9
lib/data/services/speech_impl/volcano_asr_api_service.dart

@ -892,4 +892,13 @@ class VolcanoAsrApiService implements AsrService {
// TODO: implement getAudioDataStream
throw UnimplementedError();
}
@override
Future<bool> requestScreenCapture() async => false;
@override
Future<bool> stopScreenCapture() async => true;
@override
Future<bool> isScreenCapturing() async => false;
}

9
lib/data/services/speech_impl/volcano_asr_service.dart

@ -427,4 +427,13 @@ class VolcanoAsrService extends GetxService implements AsrService {
// TODO: implement getAudioDataStream
throw UnimplementedError();
}
@override
Future<bool> requestScreenCapture() async => false;
@override
Future<bool> stopScreenCapture() async => true;
@override
Future<bool> isScreenCapturing() async => false;
}

43
lib/modules/meeting/controllers/meeting_record_controller.dart

@ -480,7 +480,12 @@ class MeetingRecordController extends GetxController
titleEditingController.dispose();
titleFocusNode.dispose();
if (audioType.value == 1) {
_bleManager.closeCodec();
if (Platform.isAndroid) {
// Android 关掉 MediaProjection 前台服务;onClose 不能 await,所以 fire-and-forget。
unawaited(_asrService.stopScreenCapture());
} else {
_bleManager.closeCodec();
}
}
final pendingJieli = _jieliCallRecorder;
_jieliCallRecorder = null;
@ -598,8 +603,11 @@ class MeetingRecordController extends GetxController
acceptAudioData = true;
break;
case 1: // Media
// Android 走 MediaProjection + AudioPlaybackCaptureConfiguration,
// azure_speech 插件的 audioDataListener 在写 WAV 同时也会把 PCM
// 通过 EventChannel 推回 Dart,开启 acceptAudioData 让波形可见。
audioSourceType = AudioSourceType.systemAudio;
acceptAudioData = false;
acceptAudioData = Platform.isAndroid;
break;
case 2: // Call
audioSourceType = AudioSourceType.external;
@ -624,6 +632,19 @@ class MeetingRecordController extends GetxController
return;
}
} else {
// Android 上音视频录音先要拿到系统的"录屏"授权,由 MediaProjection
// 抓取 USAGE_MEDIA / USAGE_GAME 等播放流。用户拒绝则直接返回,不进入录音状态。
if (audioType.value == 1 && Platform.isAndroid) {
final granted = await _asrService.requestScreenCapture();
if (!granted) {
Get.snackbar(
'error'.tr,
'meetingRecordScreenCapturePermissionDenied'.tr,
);
return;
}
}
await _asrService.enableRecord(
audioSourceType,
filePath,
@ -667,7 +688,11 @@ class MeetingRecordController extends GetxController
break;
case 1:
_asrService.setAudioConfig(sampleRate: 16000, channels: 1);
_bleManager.openDecoder();
// Android 上音视频录音改走 MediaProjection(在 _startRecording 已发起
// 系统授权 + 启动前台服务),不再依赖蓝牙耳机的 A2DP 解码通路。
if (!Platform.isAndroid) {
_bleManager.openDecoder();
}
break;
case 2:
// 通话录音走 device_jieli AI 助理通路(assistantStart),
@ -798,7 +823,11 @@ class MeetingRecordController extends GetxController
}
if (audioType.value == 1) {
_bleManager.closeCodec();
if (Platform.isAndroid) {
await _asrService.stopScreenCapture();
} else {
_bleManager.closeCodec();
}
}
try {
@ -862,7 +891,11 @@ class MeetingRecordController extends GetxController
}
if (audioType.value == 1) {
_bleManager.closeCodec();
if (Platform.isAndroid) {
await _asrService.stopScreenCapture();
} else {
_bleManager.closeCodec();
}
}
try {

94
lib/modules/translation/controllers/translation_controller.dart

@ -161,6 +161,9 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
StreamSubscription? _jieliCallInUplinkSub;
StreamSubscription? _jieliCallInDownlinkSub;
StreamSubscription? _jieliCallAstTtsSub;
// Android 悬浮窗"被用户叉掉"的事件订阅 — 用于在 audioVideo 翻译中
// 把"关掉浮窗"语义升级为"结束翻译",避免后台还在跑识别 + 录屏
StreamSubscription<void>? _floatingWindowClosedSubscription;
Timer? _translationDebounceTimer;
// ==================== 重新初始化守卫 ====================
@ -187,6 +190,21 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
Get.put(FloatingWindowManager(), permanent: true);
}
_floatingWindowManager = Get.find<FloatingWindowManager>();
// 监听用户主动关闭浮窗:在 Android 音视频翻译里浮窗就是用户唯一能看
// 翻译的入口,叉掉它意味着"结束翻译",顺势停掉识别 + 录屏前台服务,
// 避免后台空跑。其它模式(call 等)保持原行为,仅同步本地标志位。
_floatingWindowClosedSubscription?.cancel();
_floatingWindowClosedSubscription =
_floatingWindowManager.onClosedByUser.listen((_) {
Logger.info('悬浮窗被用户关闭,同步翻译会话状态');
isFloatingWindowEnabled.value = false;
if (currentMode.value == 'audioVideo' &&
(isRecognizing.value || isPreparing.value)) {
stopAll();
}
});
Logger.info('悬浮窗管理器初始化完成');
} catch (e) {
Logger.error('悬浮窗管理器初始化失败: $e');
@ -969,6 +987,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
_voiceInteractionSubscription?.cancel();
_deviceCallTranslationSubscription?.cancel();
_deviceCallTranslationSubscription = null;
_floatingWindowClosedSubscription?.cancel();
_floatingWindowClosedSubscription = null;
_detachJieliCallTranslationBridge();
_translationDebounceTimer?.cancel();
restoreOtherServices();
@ -1101,10 +1121,26 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
try {
await _initializeSession();
// Android 音视频翻译:先把悬浮窗起来,让用户切到其他 App 后还能看到翻译。
// 浮窗权限被拒绝则直接放弃启动 — 不开浮窗就没意义(用户切走就看不见结果)。
final isAndroidAudioVideo =
Platform.isAndroid && currentMode.value == 'audioVideo';
if (isAndroidAudioVideo && !isFloatingWindowEnabled.value) {
await _enableFloatingWindow();
if (!isFloatingWindowEnabled.value) {
return;
}
}
await _configureAudioMode();
if (currentMode.value != 'faceToFace') {
// Android 上音视频翻译改走 MediaProjection,PCM 由系统播放流提供,
// 不再需要 BLE codec 处于激活状态。其它平台/模式仍按旧逻辑校验:
// 拿不到 BLE 编解码就放弃启动识别。
if ((currentMode.value == 'audioVideo' ||
currentMode.value == 'call') &&
!isAndroidAudioVideo &&
!bleManager.isCodecActive) {
return;
}
@ -1123,6 +1159,24 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
await _startAsrService('normal');
}
await _finalizeRecognitionStart();
// Android 音视频翻译:识别就绪后才弹"录屏"授权 — 用户同意的瞬间
// 系统音频开始流入已经准备好的 ASR 通路,可以立即切到目标 App。
if (isAndroidAudioVideo) {
final captured = await _asrService.requestScreenCapture();
if (!captured) {
// 拒绝则回滚已经启动的识别 + 关掉浮窗,避免空浮窗悬空在桌面
await stopRecognition();
if (isFloatingWindowEnabled.value) {
await _disableFloatingWindow();
}
Get.snackbar(
'tip'.tr,
'meetingRecordScreenCapturePermissionDenied'.tr,
);
return;
}
}
}
} catch (e) {
isPreparing.value = false;
@ -1172,16 +1226,23 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
/// 配置音视频模式
Future<void> _configureAudioVideoMode() async {
isPreparing.value = true;
await bleManager.openDecoder();
int attempts = 0;
while (!bleManager.isCodecActive && attempts < 10) {
await Future.delayed(const Duration(milliseconds: 150));
attempts++;
}
isPreparing.value = false;
if (!bleManager.isCodecActive) {
Get.snackbar('tip'.tr, 'operationFailedPleaseRetry'.tr);
throw Exception('lxm--BLE编解码打开失败');
if (Platform.isAndroid) {
// Android 改走 MediaProjection。这里只设置音频源标志,真正的录屏授权
// 推迟到 startRecognition 末尾申请:先把浮窗、识别都准备好,最后弹授权
// 让用户立刻就能切到 YouTube 等 App 看翻译结果。
isPreparing.value = false;
} else {
await bleManager.openDecoder();
int attempts = 0;
while (!bleManager.isCodecActive && attempts < 10) {
await Future.delayed(const Duration(milliseconds: 150));
attempts++;
}
isPreparing.value = false;
if (!bleManager.isCodecActive) {
Get.snackbar('tip'.tr, 'operationFailedPleaseRetry'.tr);
throw Exception('lxm--BLE编解码打开失败');
}
}
_audioSourceType = true;
isTtsEnabled.value = false;
@ -2002,7 +2063,12 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
isPreparing.value = false;
if (currentMode.value == "audioVideo" || currentMode.value == "call") {
_detachJieliCallTranslationBridge();
bleManager.closeCodec();
if (currentMode.value == "audioVideo" && Platform.isAndroid) {
// Android 音视频翻译走 MediaProjection,关掉对应前台服务即可。
await _asrService.stopScreenCapture();
} else {
bleManager.closeCodec();
}
Logger.info('关闭ble编解码器');
}
if (isRecognizing.value) {
@ -2032,7 +2098,11 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
_translationDebounceTimer?.cancel();
await _ttsService.stop();
if (currentMode.value == "call" || currentMode.value == "audioVideo") {
bleManager.closeCodec();
if (currentMode.value == "audioVideo" && Platform.isAndroid) {
await _asrService.stopScreenCapture();
} else {
bleManager.closeCodec();
}
}
} catch (e) {
Logger.error('[STS] 终止通话服务失败: $e');

7
local_plugins/azure_speech/android/src/main/AndroidManifest.xml

@ -6,6 +6,7 @@
<!-- 添加前台服务权限 -->
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" />
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
<application>
@ -15,5 +16,11 @@
android:enabled="true"
android:exported="false"
android:foregroundServiceType="microphone" />
<!-- 注册屏幕/系统音频捕获前台服务(Android 14+ 必须 mediaProjection 类型) -->
<service
android:name=".tools.ScreenCaptureForegroundService"
android:enabled="true"
android:exported="false"
android:foregroundServiceType="mediaProjection" />
</application>
</manifest>

57
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt

@ -171,6 +171,11 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
private var captureQuality: Int = 80
private val REQUEST_SCREEN_CAPTURE = 9001
// 等待系统授权弹窗结果的 MethodChannel.Result。授权流程是:
// Dart invoke "requestScreenCapture" -> 这里持有 result,发起 startActivityForResult ->
// onActivityResult 拿到用户选择后再回包。Dart 侧因此可以 await 出真实结果。
private var pendingScreenCaptureResult: MethodChannel.Result? = null
private fun nextAstInitGeneration(): Int = astInitGeneration.incrementAndGet()
private fun isAstInitGenerationCurrent(generation: Int): Boolean =
astInitGeneration.get() == generation
@ -501,24 +506,41 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
activityBinding?.removeActivityResultListener(this)
activityBinding = null
activity = null
// Activity 解绑时若仍有未回包的屏幕捕获请求,回 false 避免 Dart 端永久挂起
pendingScreenCaptureResult?.success(false)
pendingScreenCaptureResult = null
}
override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?): Boolean {
if (requestCode == REQUEST_SCREEN_CAPTURE && resultCode == Activity.RESULT_OK && data != null) {
if (requestCode != REQUEST_SCREEN_CAPTURE) return false
val pending = pendingScreenCaptureResult
pendingScreenCaptureResult = null
if (resultCode == Activity.RESULT_OK && data != null) {
// 使用前台服务启动屏幕捕获,满足 Android 14 的安全要求
val ctx = activity ?: context
ScreenCaptureForegroundService.startService(
ctx,
resultCode,
data,
captureWidth,
captureHeight,
captureDpi,
captureQuality
)
try {
ScreenCaptureForegroundService.startService(
ctx,
resultCode,
data,
captureWidth,
captureHeight,
captureDpi,
captureQuality
)
pending?.success(true)
} catch (e: Exception) {
Log.e(tag, "启动屏幕捕获前台服务失败: ${e.message}")
pending?.success(false)
}
return true
}
return false
// 用户拒绝或取消授权
pending?.success(false)
return true
}
//设置识别器
@ -854,14 +876,25 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
result.error("NO_ACTIVITY", "Activity 未附着,无法请求屏幕捕获权限", null)
return
}
// 同时只允许一个 pending 授权请求,避免 onActivityResult 回包错位
if (pendingScreenCaptureResult != null) {
result.error(
"REQUEST_CAPTURE_BUSY",
"上一个屏幕捕获授权请求尚未完成",
null,
)
return
}
try {
val pm =
act.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as android.media.projection.MediaProjectionManager
val intent = pm.createScreenCaptureIntent()
// 先存住 result,由 onActivityResult 在用户选择后再回包
pendingScreenCaptureResult = result
act.startActivityForResult(intent, REQUEST_SCREEN_CAPTURE)
result.success(true)
} catch (e: Exception) {
pendingScreenCaptureResult = null
result.error("REQUEST_CAPTURE_ERROR", e.message, null)
}
}

Loading…
Cancel
Save