From 7b38a975fae3bc8b1d9a71f4e27035465d861511 Mon Sep 17 00:00:00 2001 From: Rodger-Wang <1367893453@qq.com> Date: Tue, 13 Oct 2026 21:34:34 +0800 Subject: [PATCH] =?UTF-8?q?client=EF=BC=9A=E5=88=87=E5=90=8E=E5=8F=B0?= =?UTF-8?q?=E6=97=B6=E5=90=8C=E5=A3=B0=E3=80=81=E9=80=9A=E8=AF=9D=E5=92=8C?= =?UTF-8?q?=E8=80=B3=E6=9C=BA=20AI=20=E4=B8=8D=E5=86=8D=E6=96=AD=E8=BF=9E?= =?UTF-8?q?=EF=BC=8C=E5=90=8C=E5=A3=B0=E6=94=B9=E8=B5=B0=E8=80=B3=E6=9C=BA?= =?UTF-8?q?=E9=BA=A6=EF=BC=8C=E7=BB=93=E6=9D=9F=E5=AF=B9=E8=AF=9D=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E6=8A=8A=E6=B2=A1=E5=9C=A8=E6=92=AD=E7=9A=84=E9=9F=B3?= =?UTF-8?q?=E4=B9=90=E6=8B=89=E8=B5=B7=E6=9D=A5=E3=80=82=E6=9E=84=E5=BB=BA?= =?UTF-8?q?=E5=8F=B7=201.0.3+37?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- .../core/utils/audio_foreground_service.dart | 15 +- .../client/lib/data/services/asr_service.dart | 4 + .../services/bailian_multimodal_service.dart | 20 +- .../data/services/bailian_native_dialog.dart | 7 +- .../services/device_ai_session_service.dart | 142 ++++++-- .../services/device_location_service.dart | 26 +- .../speech_impl/azure_asr_service.dart | 9 +- .../lib/data/utils/background_keepalive.dart | 13 + .../lib/data/utils/emai_audio_session.dart | 130 +++++++- .../devices/bes/bes_bluetooth_service.dart | 6 + .../lib/devices/bes/bes_device_plugin.dart | 20 +- .../devices/eaicar/eaicar_device_plugin.dart | 25 +- .../emai/controllers/emai_controller.dart | 43 ++- .../controllers/translation_controller.dart | 136 +++++++- .../android/src/main/AndroidManifest.xml | 5 +- .../azure_speech/AzureSpeechPlugin.kt | 13 +- .../azure_speech/DeviceDialogFocus.kt | 303 ++++++++++++++++++ .../tools/AudioRecordingForegroundService.kt | 201 ++++++++++-- .../azure_speech/AzureSpeechPlugin.swift | 37 +++ apps/client/pubspec.yaml | 2 +- .../devices/bes_recorder_sources_test.dart | 5 +- .../test/emai_device_dialog_audio_test.dart | 20 ++ 22 files changed, 1061 insertions(+), 121 deletions(-) create mode 100644 apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DeviceDialogFocus.kt create mode 100644 apps/client/test/emai_device_dialog_audio_test.dart diff --git a/apps/client/lib/core/utils/audio_foreground_service.dart b/apps/client/lib/core/utils/audio_foreground_service.dart index fb4981fd..0c05c3d1 100644 --- a/apps/client/lib/core/utils/audio_foreground_service.dart +++ b/apps/client/lib/core/utils/audio_foreground_service.dart @@ -30,10 +30,21 @@ class AudioForegroundService { static const MethodChannel _channel = MethodChannel('azure_speech/ast'); static const String _tag = 'AudioForegroundService'; - static Future start() async { + /// [keepalive] 为真时，服务里再挂静音播放和唤醒锁。 + /// 耳机 AI 的声音走耳机私有链路，手机侧没有音频，华为会把后台网络掐掉。 + /// + /// [pinSpeaker] 为真时静音钉在手机扬声器，避免把耳机 A2DP 抢走。 + /// 同声翻译的译文本身就走这条媒体通道，钉住会让译文改从扬声器出来，这时要传 false。 + static Future start({ + bool keepalive = false, + bool pinSpeaker = true, + }) async { if (!Platform.isAndroid) return; try { - await _channel.invokeMethod('startAudioForegroundService'); + await _channel.invokeMethod('startAudioForegroundService', { + 'keepalive': keepalive, + 'pinSpeaker': pinSpeaker, + }); } catch (e) { // 起不来不该阻断录音本身：前台是为了「切后台还能录」， // 停在前台的那部分照样是好的。 diff --git a/apps/client/lib/data/services/asr_service.dart b/apps/client/lib/data/services/asr_service.dart index d37b471d..9e9f95c4 100644 --- a/apps/client/lib/data/services/asr_service.dart +++ b/apps/client/lib/data/services/asr_service.dart @@ -39,6 +39,10 @@ abstract class AsrService { /// 停止连续语音识别 Future stopContinuousRecognition(); + /// 外部音频模式下推进一块 PCM16/16k/单声道。 + /// 手机麦由原生自己采集，默认实现不做事。 + Future pushAudioData(Uint8List data) async => false; + /// 设置识别回掉 /// /// 返回一个包含识别事件的流 diff --git a/apps/client/lib/data/services/bailian_multimodal_service.dart b/apps/client/lib/data/services/bailian_multimodal_service.dart index fa6e7fc2..fe2d7553 100644 --- a/apps/client/lib/data/services/bailian_multimodal_service.dart +++ b/apps/client/lib/data/services/bailian_multimodal_service.dart @@ -213,8 +213,22 @@ class BailianMultimodalService extends GetxService { return appId.isNotEmpty && appId == resolveAppId(); } + /// 能念出来的地址。空和「未知」都不算，避免写进提示词。 + static String? spokenPlace(dynamic raw) { + final text = raw?.toString().trim() ?? ''; + if (text.isEmpty || text == '未知' || text == 'null') return null; + return text; + } + + /// 这个应用要带地址、而缓存里还没有能念的地址时，先刷新一次。 + static Future ensurePromptPlace(String appId) async { + if (!wantsClientContext(appId)) return; + await DeviceLocationService.maybe?.ensureSpokenPlace(); + } + /// `${location}` / `${date}` 与 `client_info.location`。只读缓存，不等定位。 /// 这个应用不需要时返回 null。没有缓存时地址是「未知」，日期照样给。 + /// 「未知」不要写进提示词，否则模型会把这两个字当成用户所在的地方。 static Map? promptContext(String appId) { if (!wantsClientContext(appId)) return null; DeviceLocationService? loc; @@ -251,13 +265,15 @@ class BailianMultimodalService extends GetxService { _downstreamRate = downstreamRate; directiveSource = source; _taskId = _uuid(); - // 提示词「系统条件」里的 ${location} / ${date}：只读缓存，不等定位 + // 提示词「系统条件」里的 ${location} / ${date}。没有路名时先补一次定位。 + await ensurePromptPlace(_appId); final ctx = promptContext(_appId); final clientLocation = ctx?['clientLocation'] as Map?; + final spoken = spokenPlace(ctx?['location']); final promptParams = ctx == null ? const {} : { - 'location': '${ctx['location']}', + if (spoken != null) 'location': spoken, 'date': '${ctx['date']}', }; if (ctx != null) { diff --git a/apps/client/lib/data/services/bailian_native_dialog.dart b/apps/client/lib/data/services/bailian_native_dialog.dart index 28191e07..10a9f17f 100644 --- a/apps/client/lib/data/services/bailian_native_dialog.dart +++ b/apps/client/lib/data/services/bailian_native_dialog.dart @@ -120,7 +120,8 @@ class BailianNativeDialog { ); try { final appId = BailianMultimodalService.resolveAppId(_appIdOverride); - // 地址和日期的文案只在 Dart 算一份。原生负责写进 Start，不在这里等定位。 + // 地址和日期的文案只在 Dart 算一份。缓存没有路名时先补一次定位。 + await BailianMultimodalService.ensurePromptPlace(appId); final ctx = BailianMultimodalService.promptContext(appId); final rawLoc = ctx?['clientLocation']; final locMap = rawLoc is Map @@ -145,7 +146,9 @@ class BailianNativeDialog { if (mcpToken != null && mcpToken.trim().isNotEmpty) 'mcpToken': mcpToken, // 工具名只在 Dart 维护一份：服务端加工具时改那一处就够了 'mcpAuthTools': BailianMultimodalService.mcpAuthTools, - if (ctx != null) 'promptLocation': ctx['location'], + if (BailianMultimodalService.spokenPlace(ctx?['location']) != null) + 'promptLocation': + BailianMultimodalService.spokenPlace(ctx?['location']), if (ctx != null) 'promptDate': ctx['date'], if (locMap != null) 'latitude': locMap['latitude'], if (locMap != null) 'longitude': locMap['longitude'], diff --git a/apps/client/lib/data/services/device_ai_session_service.dart b/apps/client/lib/data/services/device_ai_session_service.dart index 39d95da9..a27dbe0f 100644 --- a/apps/client/lib/data/services/device_ai_session_service.dart +++ b/apps/client/lib/data/services/device_ai_session_service.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:math'; import 'dart:typed_data'; import 'package:device_plugin_interface/device_plugin_interface.dart'; @@ -12,6 +13,7 @@ import '../models/agent_module.dart'; import '../models/emai_message.dart'; import '../models/user_Info.dart' as user_info; import '../utils/background_keepalive.dart'; +import '../utils/emai_audio_session.dart'; import '../utils/cue_tone.dart'; import '../utils/peer_speech_gate.dart' show rmsOfPcm16; import 'agent_module_defaults.dart'; @@ -167,7 +169,7 @@ class DeviceAiSessionService extends GetxService { /// 百炼 WebSocket 掉了以后的自动重连（蜂窝网切换、弱网抖动）。 /// 原来 `closed` 一到就整个会话拆掉——用户还戴着耳机在说话。 - static const int _wsReconnectMax = 2; + static const int _wsReconnectMax = 8; static const List _wsReconnectDelays = [ Duration(seconds: 1), Duration(seconds: 3), @@ -191,6 +193,26 @@ class DeviceAiSessionService extends GetxService { EmaiMessage? _streamingUser; EmaiMessage? _streamingAssistant; + /// 最近一次助手回复文本到达的时刻。用来分辨 requestAccepted 是不是 + /// 「这一轮还在播」而不是「服务端放弃了回复」。 + DateTime? _lastAssistantContentAt; + + /// 流式指针被清掉之后，同一句又来一帧时别再新建气泡。 + /// 只认末尾那条、且文本完全一样；新的一句对不上就照常追加。 + EmaiMessage _adoptRecent({ + required bool isUser, + required String text, + required EmaiMessage Function() create, + }) { + if (text.isNotEmpty && _store.messages.isNotEmpty) { + final last = _store.messages.last; + if (last.isUser == isUser && !last.isError && last.text == text) { + return last; + } + } + return create(); + } + @override void onInit() { super.onInit(); @@ -428,6 +450,7 @@ class DeviceAiSessionService extends GetxService { _wsReconnectAttempts = 0; // 换了设备就重新试一次 ai.state（上一台不支持不代表这台也不支持） _aiStateUnsupported = false; + _leaveDeviceAi = false; _quitAfterPlayout = false; _greetPending = false; _ownRequestPending = false; @@ -441,18 +464,19 @@ class DeviceAiSessionService extends GetxService { // iOS：会话期间保持一个静音的后台音频播放，否则锁屏几秒后进程被挂起， // BLE 上行与 WebSocket 一起停摆（见 BackgroundKeepalive）。Android 空操作。 await BackgroundKeepalive.acquire(_tag); + // 先看外面有没有音乐在播，再开静音保活。顺序反了的话，自己的音轨会让 + // isMusicActive 变成 true，结束时交还焦点就把没在播的音乐拉起来。 + await EmaiAudioSession.holdForDeviceDialog(); + // 华为把退到后台的进程网络掐掉，日志是 Software caused connection abort。 + // 用户那句已经写进记录，回复还在这条 WebSocket 上，连接一断就没有回答。 + // 前台服务把进程钉住。香薰和耳机都要，不能只给走手机播放器的香薰。 + await AudioForegroundService.start(keepalive: true); + _holdsForeground = true; + Logger.w(_tag, '设备对话已开前台服务'); // ★ 先试原生直连：上行 PCM 与合成音频都不经过 Dart。 // 插件老 / 设备不是恒玄 → 返回 null，整条链路退回下面的 Dart 路径。 _native = await _tryOpenNativeDialog(session); - // Android 的等价物：前台服务把进程钉住。 - // ⚠️ 2026-09-25 华为 ALN-AL80 实测：车载香薰对话中按 Home，约 2s 后进程被 Pged-Freezer - // 冻住（上行 PCM 攒了 2s 才一次性冲出来），百炼 WS 随即断开，重连两次又各活 3s， - // 最后放弃——用户看到的就是「切后台就聊不了了」。恒玄耳机走原生链路，不在这里管。 - if (_native == null && _isEaiCar) { - unawaited(AudioForegroundService.start()); - _holdsForeground = true; - } try { if (_useNative) { _eventSub = _native!.events.listen(_onBailianEvent); @@ -479,9 +503,10 @@ class DeviceAiSessionService extends GetxService { await _playCue(CueTone.ding(sampleRate: _deviceRate)); _touchActivity(); - // 握手一完成就让助手用自己的音色说一句「我在」（见 _greetOnWake）。 - // 只在 Dart 链路做：原生直连（恒玄耳机）不经过 _svc，行为保持不变。 - _greetPending = _native == null && _isEaiCar; + // 握手一完成就让助手用自己的音色说一句。 + // 耳机：我在 / 回来了 / 有什么可以帮到你，三句里随机一句。 + // 香薰仍走原来那组，而且只在 Dart 链路（原生直连不经过这里的问候）。 + _greetPending = !_isEaiCar || _native == null; final ok = await _startDialog(); final elapsed = DateTime.now().difference(ackAt); if (elapsed > _ackDeadline) { @@ -530,7 +555,18 @@ class DeviceAiSessionService extends GetxService { // respondingEnded）。门若还开着，上行就一直是静音、服务端在 listening // 里干等，双方互等到 _gateSafety 的 90s 才解开（2026-09-25 车载香薰实测）。 // 所以：丢掉那一轮没播完的音频，立刻关门，把耳朵还给用户。 + // + // 回复文本刚到的那一次不算放弃。真机 19:08:54 答复帧之后 1 秒就来了 + // requestAccepted，按放弃处理会把正在播的音频停掉并打开麦克风， + // 麦把这句回复又收进去，列表里同一轮的问和答各出现两份。 if (_uplinkGated && _isEaiCar) { + final contentAt = _lastAssistantContentAt; + if (contentAt != null && + DateTime.now().difference(contentAt) < + const Duration(seconds: 3)) { + Logger.w(_tag, '回复进行中收到 requestAccepted，保持门控，不停播'); + break; + } Logger.w(_tag, '服务端放弃了本轮回复（requestAccepted），停播并解除门控'); _spk?.discardPending(); _cancelPlayoutTimer(); @@ -550,8 +586,12 @@ class DeviceAiSessionService extends GetxService { break; case BailianEventType.speechContent: - _streamingUser ??= _store - .append(EmaiMessage(isUser: true, text: '', fromEarphone: true)); + _streamingUser ??= _adoptRecent( + isUser: true, + text: e.text, + create: () => _store.append( + EmaiMessage(isUser: true, text: '', fromEarphone: true)), + ); _store.applyText(_streamingUser!, e.text); if (e.isFinal) { _store.finalize(_streamingUser); @@ -574,23 +614,32 @@ class DeviceAiSessionService extends GetxService { _respondingStartedAt = DateTime.now(); _cancelPlayoutTimer(); _streamingAssistant = null; + // 第一包 TTS 到达之前麦还开着，外放的开头会被收回去，服务端当成下一句。 + // 手机页半双工是在 respondingStarted 就关麦的，设备这边对齐。 + if (_isEaiCar) _openGate(); break; case BailianEventType.respondingContent: + if (e.text.isNotEmpty) _lastAssistantContentAt = DateTime.now(); if (e.directives.isNotEmpty && Get.isRegistered()) { AssistantDirectiveService.to.handle(e.directives); } - // 「关闭对话」：模型会下发 quit 指令并同时回一句告别语。 - // 不能立刻结束——那样告别语还没播完就被掐了；标记下来，等播完再收尾。 + // 「关闭对话」：模型下发 quit。立刻收尾。 + // 先前等告别语播完，但这一轮的播完计时会被后面的插话清掉， + // 会话就一直留在聆听，直到空闲超时（真机 18:52:49 收到 quit，18:53:17 才交还焦点）。 if (e.directives.any((d) => d.name == _quitDirective)) { - Logger.i(_tag, '收到 quit 指令，本轮播完后结束会话'); - _quitAfterPlayout = true; + Logger.w(_tag, '收到 quit 指令，立即结束会话'); + unawaited(requestStop()); } // 纯指令帧没有文本，别建空气泡 if (e.text.isEmpty && e.finishReason == 'command_calls') break; - _streamingAssistant ??= _store - .append(EmaiMessage(isUser: false, text: '', fromEarphone: true)); + _streamingAssistant ??= _adoptRecent( + isUser: false, + text: e.text, + create: () => _store.append( + EmaiMessage(isUser: false, text: '', fromEarphone: true)), + ); _store.applyText(_streamingAssistant!, e.text); break; @@ -694,6 +743,10 @@ class DeviceAiSessionService extends GetxService { /// 设备是否支持 [DeviceFeatures.aiState]。不支持就别每次都试、别每次都打日志。 bool _aiStateUnsupported = false; + /// 本轮已经决定退出。结束音还在播、事件订阅还没拆的时候， + /// 迟到的「聆听中」会把刚退出的设备又拉回 AI 模式。 + bool _leaveDeviceAi = false; + /// 模型下发的「结束对话」指令名（`extra_info.commands` 里的 `name`） static const String _quitDirective = 'quit'; @@ -746,6 +799,11 @@ class DeviceAiSessionService extends GetxService { /// 刚发出问候、等它的 requestAccepted——那一条不能当成「服务端放弃了本轮回复」。 bool _ownRequestPending = false; + static const List _earphoneGreetingsZh = [ + '我在', + '回来了', + '有什么可以帮到你', + ]; static const List _greetingsZh = [ '我在', '我在呢', @@ -760,12 +818,19 @@ class DeviceAiSessionService extends GetxService { ]; void _greetOnWake() { + final earphone = !_isEaiCar; final zh = (Get.locale?.languageCode ?? 'zh') == 'zh'; - final list = zh ? _greetingsZh : _greetingsEn; - final text = list[DateTime.now().microsecond % list.length]; - Logger.i(_tag, '唤醒问候: $text'); + final list = earphone + ? (zh ? _earphoneGreetingsZh : _greetingsEn) + : (zh ? _greetingsZh : _greetingsEn); + final text = list[Random().nextInt(list.length)]; + Logger.w(_tag, '唤醒问候: $text'); _ownRequestPending = true; - _svc.sendText(text, speakDirectly: true); + if (_useNative && _native != null) { + unawaited(_native!.sendText(text, speakDirectly: true)); + } else { + _svc.sendText(text, speakDirectly: true); + } // 服务端万一不回 requestAccepted，别让这个标记吞掉之后真正的那一次 Timer(const Duration(seconds: 3), () => _ownRequestPending = false); } @@ -778,7 +843,7 @@ class DeviceAiSessionService extends GetxService { /// **fire-and-forget**：设备不支持、或这条命令失败，都不该影响对话本身， /// 所以既不 await 也不往上抛。 void _pushAiState(DeviceAiState state) { - if (_aiStateUnsupported) return; + if (_leaveDeviceAi || _aiStateUnsupported) return; final s = _session; if (s == null || s.state != DeviceConnectionState.ready) return; s.invokeFeature(DeviceFeatures.aiState, {'state': state}).catchError( @@ -813,7 +878,8 @@ class DeviceAiSessionService extends GetxService { return false; } if (_wsReconnectTimer != null) return true; // 已在排队 - final delay = _wsReconnectDelays[_wsReconnectAttempts]; + final delay = _wsReconnectDelays[ + _wsReconnectAttempts.clamp(0, _wsReconnectDelays.length - 1)]; _wsReconnectAttempts++; Logger.w(_tag, '百炼 WS 掉线（$reason），${delay.inSeconds}s 后第 $_wsReconnectAttempts 次重连'); // 重连期间别让空闲计时把会话判死 @@ -935,14 +1001,21 @@ class DeviceAiSessionService extends GetxService { // ---------------- 结束 ---------------- /// 主动结束：发 ai.exit，等设备回 hangup；超时就本地强制收尾。 + /// + /// 车载香薰的退出是单向 0x07 03，设备不回应答。先发的话，收尾期间的结束音 + /// 和迟到的状态帧会再推一次「聆听中」，设备又回到 AI 模式。所以香薰的退出 + /// 指令放到本地音频停掉、事件订阅拆掉之后再发，并且从这一刻起不再推状态。 Future requestStop() async { if (!isActive.value) return; - Logger.i(_tag, '请求设备退出 AI 模式'); + _leaveDeviceAi = true; + Logger.w(_tag, '请求设备退出 AI 模式'); + if (_isEaiCar) { + await _teardown(notifyDevice: true); + return; + } final s = _session; final noAck = s != null && await _sendExit(s); if (noAck) { - // 车载香薰的「退出」是单向的 0x07 03，设备从不回应答—— - // 干等 2s 只会让 App 比设备晚 2s 结束（2026-09-25 日志里每次都是超时收尾） await _teardown(notifyDevice: false); return; } @@ -957,9 +1030,13 @@ class DeviceAiSessionService extends GetxService { /// 发 ai.exit。返回 true 表示该设备**不会回结束应答**（插件回了 `noAck`）， /// 调用方可以直接收尾。 Future _sendExit(DeviceSession s) async { - if (s.state != DeviceConnectionState.ready) return false; + if (s.state != DeviceConnectionState.ready) { + Logger.w(_tag, 'ai.exit 未发送：设备状态 ${s.state}'); + return false; + } try { final r = await s.invokeFeature(DeviceFeatures.aiExit); + Logger.w(_tag, '已发送 ai.exit → $r'); return r['noAck'] == true; } catch (e) { Logger.w(_tag, '发送 ai.exit 失败: $e'); @@ -986,6 +1063,7 @@ class DeviceAiSessionService extends GetxService { required bool notifyDevice, bool playEndTone = true, }) async { + if (notifyDevice) _leaveDeviceAi = true; _stopAckTimer?.cancel(); _stopAckTimer = null; _idleTimer?.cancel(); @@ -1015,6 +1093,7 @@ class DeviceAiSessionService extends GetxService { _streamingUser = null; _store.dropIfEmpty(_streamingAssistant); _streamingAssistant = null; + _lastAssistantContentAt = null; if (wasActive) _store.persist(); await _eventSub?.cancel(); @@ -1061,7 +1140,10 @@ class DeviceAiSessionService extends GetxService { } if (wasActive && notifyDevice && s != null) await _sendExit(s); _session = null; + // 先停静音保活，再交还会话。保活还在播时 deactivate 会被系统拒绝， + // 外部音乐收不到继续播放的通知。 await BackgroundKeepalive.release(_tag); + await EmaiAudioSession.releaseDeviceDialog(); if (_holdsForeground) { _holdsForeground = false; unawaited(AudioForegroundService.stop()); diff --git a/apps/client/lib/data/services/device_location_service.dart b/apps/client/lib/data/services/device_location_service.dart index e4fca8ef..a4f8aec0 100644 --- a/apps/client/lib/data/services/device_location_service.dart +++ b/apps/client/lib/data/services/device_location_service.dart @@ -113,8 +113,22 @@ class DeviceLocationService extends GetxService with WidgetsBindingObserver { } } + /// 缓存里没有能念出来的地址时，建会话前再定一次位。 + /// 已经有地址就马上返回，不拖唤醒。 + Future ensureSpokenPlace() async { + if (_hasSpokenPlace()) return; + Logger.w(_tag, '缓存没有详细地址，重新定位'); + await refresh(force: true, timeoutMs: 4000); + Logger.w(_tag, '定位结果: ${placeText()} 坐标=${clientLocation() != null}'); + } + + bool _hasSpokenPlace() { + final text = placeText(); + return text.isNotEmpty && text != '未知'; + } + /// 后台刷新一次位置（不弹权限框、不抛异常）。 - Future refresh({bool force = false}) async { + Future refresh({bool force = false, int timeoutMs = 15000}) async { if (_refreshing) return; final last = _lastRefreshAt; if (!force && @@ -126,18 +140,20 @@ class DeviceLocationService extends GetxService with WidgetsBindingObserver { try { final st = await Permission.locationWhenInUse.status; if (!(st.isGranted || st.isLimited)) { - Logger.i(_tag, '没有定位权限（$st），不取位置'); + Logger.w(_tag, '没有定位权限（$st），不取位置'); return; } - final r = await LocationService().currentPlace(); + final r = await LocationService().currentPlace(timeoutMs: timeoutMs); if (r['error'] != null) { Logger.w(_tag, '定位未成功: ${r['error']}（沿用缓存: ${placeText()}）'); return; } _place = r; - _lastRefreshAt = DateTime.now(); + final spoken = placeText(); await _storage.write(_cacheKey, r); - Logger.i(_tag, '位置已更新: ${placeText()}（原始: $r）'); + // 只有市名街道都空、念出来是「未知」时，不记冷却，下次唤醒再试。 + if (spoken != '未知') _lastRefreshAt = DateTime.now(); + Logger.w(_tag, '位置已更新: $spoken'); } catch (e) { Logger.w(_tag, '刷新位置失败: $e'); } finally { diff --git a/apps/client/lib/data/services/speech_impl/azure_asr_service.dart b/apps/client/lib/data/services/speech_impl/azure_asr_service.dart index b3f4bf03..7d52c42a 100644 --- a/apps/client/lib/data/services/speech_impl/azure_asr_service.dart +++ b/apps/client/lib/data/services/speech_impl/azure_asr_service.dart @@ -218,6 +218,12 @@ class AzureAsrService extends GetxService implements AsrService { await stopContinuousRecognition(); } + // 与这次启动的音源对齐。推流入口只认这个标志， + // 只在 initialize 里设的话，同声翻译改走耳机麦时会被拒。 + _audioSourceType = audioSourceType + ? AudioSourceType.external + : AudioSourceType.microphone; + try { // 开始连续识别 final bool result = @@ -492,7 +498,8 @@ class AzureAsrService extends GetxService implements AsrService { try { final bool result = await _channel.invokeMethod('pushAudioData', { - 'audioData': data // Flutter 会自动将 Uint8List 转换为 ByteBuffer + // 原生两端读的键都是 data。写成 audioData 会直接被判空丢掉。 + 'data': data, }); if (!result) { diff --git a/apps/client/lib/data/utils/background_keepalive.dart b/apps/client/lib/data/utils/background_keepalive.dart index 6b9022b7..d454dfea 100644 --- a/apps/client/lib/data/utils/background_keepalive.dart +++ b/apps/client/lib/data/utils/background_keepalive.dart @@ -78,6 +78,19 @@ class BackgroundKeepalive { } } + /// 类别被切成独占之后，静音循环可能被停掉。会话还在就要它继续播， + /// 否则后台不再算「正在播放」，系统会把进程挂起。 + static Future keepPlaying() async { + if (!_apply) return; + final p = _player; + if (p == null || p.playing) return; + try { + await p.play(); + } catch (e) { + Logger.w(_tag, '静音保活续播失败: $e'); + } + } + static Future release(String owner) async { if (!_apply) return; if (_refs == 0) return; diff --git a/apps/client/lib/data/utils/emai_audio_session.dart b/apps/client/lib/data/utils/emai_audio_session.dart index 4bfe9ed5..82ddc760 100644 --- a/apps/client/lib/data/utils/emai_audio_session.dart +++ b/apps/client/lib/data/utils/emai_audio_session.dart @@ -1,9 +1,11 @@ import 'dart:io' show Platform; import 'package:audio_session/audio_session.dart'; -import 'package:flutter/foundation.dart' show kIsWeb; +import 'package:flutter/foundation.dart' show kIsWeb, visibleForTesting; +import 'package:flutter/services.dart'; import '../../core/utils/logger.dart'; +import 'background_keepalive.dart'; /// 这条语音链路要用手机的哪些能力。决定 AVAudioSession 的类别， /// 而类别直接决定**声音从哪儿出来**。 @@ -34,6 +36,10 @@ class EmaiAudioSession { static const String _tag = 'EmaiAudioSession'; + /// 设备对话占住期间登记 Now Playing、查询外部播放状态。 + /// 不在这里改会话类别，类别仍由本类用 audio_session 配。 + static const MethodChannel _probe = MethodChannel('azure_speech/pcm_player'); + /// 最近一次配置的档位，只用于日志。 /// /// ⚠️ 不能拿它当「已配好就跳过」的缓存：这个值是 Dart 进程内的，而 AVAudioSession @@ -45,8 +51,104 @@ class EmaiAudioSession { /// 每次都配没有代价。 static EmaiAudioProfile? _configured; + /// 这场设备对话已经占住了外部播放。句间、打断、重连都不交还。 + static bool _deviceDialogHeld = false; + + /// iOS：进来时别的 App 确实在播，结束才通知它继续。 + /// 没在播就通知的话，系统会把暂停着的音乐拉起来。 + static bool _iosInterruptedOthers = false; + static bool get _apply => !kIsWeb && (Platform.isIOS || Platform.isAndroid); + /// 设备按键唤醒的对话（EaiCar 与耳机 EMAI）一开始就占住手机上的媒体播放。 + /// + /// 已经占住时直接返回，同一次唤醒不重复激活会话。失败只记日志，对话照常进行。 + static Future holdForDeviceDialog() async { + if (!_apply || _deviceDialogHeld) return; + _deviceDialogHeld = true; + try { + if (Platform.isAndroid) { + // 不走 audio_session.setActive：它在已有焦点对象时直接返回 true， + // 并且申请的是 GAIN_TRANSIENT，华为上音乐可以继续播。 + final snap = await _probe.invokeMapMethod('holdDeviceFocus'); + _configured = EmaiAudioProfile.playbackOnly; + Logger.i(_tag, '占住媒体播放 $snap'); + return; + } + final before = await _externalPlaying(); + if (before != true) { + Logger.w(_tag, '系统没在播音乐，不独占会话 before=$before'); + return; + } + await _setNowPlaying(true); + final session = await AudioSession.instance; + await session.configure(_playbackOnlyConfig()); + _configured = EmaiAudioProfile.playbackOnly; + final ok = await session.setActive(true); + _iosInterruptedOthers = ok == true; + await BackgroundKeepalive.keepPlaying(); + final after = await _externalPlaying(); + Logger.w(_tag, '占住媒体播放 ok=$ok 外部播放 $before → $after'); + } catch (e) { + Logger.w(_tag, '占住媒体播放失败（对话继续）: $e'); + } + } + + /// 结束音播完、扬声器关掉、静音保活停掉之后调用。 + /// 没占住过就什么都不做。 + static Future releaseDeviceDialog() async { + if (!_apply || !_deviceDialogHeld) return; + _deviceDialogHeld = false; + try { + if (Platform.isAndroid) { + final snap = + await _probe.invokeMapMethod('releaseDeviceFocus'); + _configured = null; + // 没暂停过外部播放时，不要再走 audio_session 的 setActive(false)。 + // 那一下会 abandon 焦点，华为把暂停着的音乐当成可以续播，直接开起来。 + Logger.w(_tag, '交还媒体播放 $snap'); + return; + } + if (!_iosInterruptedOthers) { + Logger.w(_tag, '进来时系统没在播，交还不通知其它 App'); + return; + } + _iosInterruptedOthers = false; + final before = await _externalPlaying(); + final session = await AudioSession.instance; + final ok = await session.setActive( + false, + avAudioSessionSetActiveOptions: + AVAudioSessionSetActiveOptions.notifyOthersOnDeactivation, + ); + // 交还之后回到混音，翻译和保活后面不会停在独占类别上。 + // 这里不再 setActive，否则外部音乐刚被通知继续又会被打断。 + await session.configure(_mixPlaybackConfig()); + _configured = null; + await _setNowPlaying(false); + final after = await _externalPlaying(); + Logger.i(_tag, '交还媒体播放 ok=$ok 外部播放 $before → $after'); + } catch (e) { + Logger.w(_tag, '交还媒体播放失败: $e'); + } + } + + static Future _externalPlaying() async { + try { + return await _probe.invokeMethod('musicActive'); + } catch (_) { + return null; + } + } + + static Future _setNowPlaying(bool active) async { + try { + await _probe.invokeMethod('setNowPlaying', {'active': active}); + } catch (e) { + Logger.w(_tag, 'Now Playing 更新失败: $e'); + } + } + /// 按档位配置并激活会话。失败只记日志，不抛—— /// 会话配不上时让调用方继续尝试，总比整条链路直接不启动好。 /// @@ -86,6 +188,9 @@ class EmaiAudioSession { } _configured = profile; if (profile == EmaiAudioProfile.playbackOnly) { + // 设备对话已经用独占焦点停过音乐。这里再 setActive 会向 audio_session + // 申请更弱的 GAIN_TRANSIENT，华为上音乐就跟着响回来。 + if (Platform.isAndroid && _deviceDialogHeld) return false; await session.configure(_playbackOnlyConfig()); await session.setActive(true); return false; @@ -127,22 +232,39 @@ class EmaiAudioSession { } /// 只放音：输出跟随系统路由（蓝牙 A2DP / AirPlay / 扬声器）。 + /// + /// ⚠️ `.playback` 自带 A2DP。再写 `allowBluetoothA2dp` 会让 `setCategory` + /// 返回 -50，整次配置失败，会话留在保活设的混音上，外部音乐停不下来。 + /// 选项里没有 `mixWithOthers`，激活之后系统会把别的媒体停掉。 static AudioSessionConfiguration _playbackOnlyConfig() => AudioSessionConfiguration( avAudioSessionCategory: AVAudioSessionCategory.playback, // ⚠️ 不要写 defaultToSpeaker：它只在 playAndRecord 下合法，而且正是 // 「声音不从蓝牙设备出来」的那个元凶。 - avAudioSessionCategoryOptions: - AVAudioSessionCategoryOptions.allowBluetoothA2dp | - AVAudioSessionCategoryOptions.allowAirPlay, + avAudioSessionCategoryOptions: AVAudioSessionCategoryOptions.allowAirPlay, avAudioSessionMode: AVAudioSessionMode.defaultMode, androidAudioAttributes: const AndroidAudioAttributes( contentType: AndroidAudioContentType.speech, usage: AndroidAudioUsage.media, ), androidAudioFocusGainType: AndroidAudioFocusGainType.gainTransient, + androidWillPauseWhenDucked: true, ); + /// 设备对话结束、焦点交还之后的类别。混音，并且不再激活。 + static AudioSessionConfiguration _mixPlaybackConfig() => + const AudioSessionConfiguration( + avAudioSessionCategory: AVAudioSessionCategory.playback, + avAudioSessionCategoryOptions: + AVAudioSessionCategoryOptions.mixWithOthers, + avAudioSessionMode: AVAudioSessionMode.defaultMode, + androidAudioFocusGainType: AndroidAudioFocusGainType.gainTransientMayDuck, + ); + + @visibleForTesting + static AudioSessionConfiguration playbackOnlyConfigForTest() => + _playbackOnlyConfig(); + static AudioSessionConfiguration _phoneMicConfig({required bool a2dpOutput}) => AudioSessionConfiguration( // iOS 必须 playAndRecord：上行要录音、下行要外放，缺一不可。 diff --git a/apps/client/lib/devices/bes/bes_bluetooth_service.dart b/apps/client/lib/devices/bes/bes_bluetooth_service.dart index 3505eb1a..48dd7588 100644 --- a/apps/client/lib/devices/bes/bes_bluetooth_service.dart +++ b/apps/client/lib/devices/bes/bes_bluetooth_service.dart @@ -717,6 +717,12 @@ class BesBluetoothService extends GetxService { /// 本端麦克风 PCM（通话模式 mode=0x03 的 mic 路），16k/16bit/mono Stream get micPcmStream => _typedPcm('micData'); + /// 现场录音主麦（`AA 59` 之后的 mSBC 主麦帧），16k/16bit/mono。 + /// + /// 原生按帧标了 `mainMicData` / `ffMicData`。裸流隔帧取会在订阅相位 + /// 不对时永远吃到前馈麦，同声翻译和现场录音都用这一条。 + Stream get mainMicPcmStream => _typedPcm('mainMicData'); + /// 对端通话音 PCM（通话模式 mode=0x03 的 spk 路），16k/16bit/mono Stream get spkPcmStream => _typedPcm('spkData'); diff --git a/apps/client/lib/devices/bes/bes_device_plugin.dart b/apps/client/lib/devices/bes/bes_device_plugin.dart index b21b4ffb..2cae0d7f 100644 --- a/apps/client/lib/devices/bes/bes_device_plugin.dart +++ b/apps/client/lib/devices/bes/bes_device_plugin.dart @@ -460,8 +460,9 @@ class BesDeviceSession extends DeviceSession { _requireReady(); switch (route) { case AudioRoute.mic: - // 现场录音：主麦与前馈麦交替上行，隔帧取主麦 - return _BesSource(route, _alternate(_link.audioStream)); + // 现场录音 / 同声翻译：原生已把主麦标成 mainMicData。 + // 裸流隔帧取会在中途订阅时锁死前馈麦。 + return _BesSource(route, _link.mainMicPcmStream); case AudioRoute.callLocal: return _BesSource(route, _link.micPcmStream); case AudioRoute.callPeer: @@ -487,10 +488,8 @@ class BesDeviceSession extends DeviceSession { /// | callLocal | `micData` | 通话本端麦 | /// | callPeer | `spkData` | 通话对端声 | /// - /// ⚠️ `mic` 这里取的是**带类型的主麦帧**，而不是 [openMic] 那条 - /// `_alternate(audioStream)`（裸流隔帧取）。隔帧取靠的是"订阅时恰好对上相位"， - /// 中途开始订阅就会永久取到前馈麦——原生早就把两路标好了 mainMicData / - /// ffMicData，按类型取顺带把这类错相消掉。 + /// ⚠️ `mic` 与 [openMic] 都取原生标好的 `mainMicData`。 + /// 裸流隔帧取会在订阅相位不对时锁死前馈麦。 /// 金测试 `test/devices/bes_recorder_sources_test.dart` 逐条钉着这张表： /// 串要与原生 `BluetoothManager` 发布的类型名逐字一致，改错不报任何错， /// 只表现为「录音起来了但一个字节都没写进去」。 @@ -526,15 +525,6 @@ class BesDeviceSession extends DeviceSession { return _BesFileRecorder(id, path, _link); } - static Stream _alternate(Stream raw) { - var take = true; - return raw.where((_) { - final t = take; - take = !take; - return t; - }); - } - @override Future openSpeaker( {AudioRoute route = AudioRoute.ai}) async { diff --git a/apps/client/lib/devices/eaicar/eaicar_device_plugin.dart b/apps/client/lib/devices/eaicar/eaicar_device_plugin.dart index 2940f1ba..c3e4fd1e 100644 --- a/apps/client/lib/devices/eaicar/eaicar_device_plugin.dart +++ b/apps/client/lib/devices/eaicar/eaicar_device_plugin.dart @@ -119,8 +119,9 @@ class EaiCarDevicePlugin implements DevicePlugin { // // 2026-09-25 以前 [tryAutoConnect] 写好了但**全项目没有调用方**（DeviceHub 的 // tryAutoConnectAll 也没人调），于是香薰断了就断了，冷启动也不连。现在两处触发： - // 1. 冷启动：最后一次用的设备是香薰（[DeviceHub.lastUsedVendor]），且用户同意过 - // 后台自动连接（与恒玄同一个开关）； + // 1. 冷启动：最后一次用的设备是香薰（[DeviceHub.lastUsedVendor]）。 + // 扫描页不弹「后台自动连接」授权，没问过的用户 granted 恒为 false， + // 按那个开关会让杀掉进程后再打开永远不回连。只有设置里明确关掉才停。 // 2. 意外断开：不是用户点断开 / 切到别的设备 / 解绑造成的，就按退避重试。 // 两个平台都能按 id 直连已知外设，不需要先扫描（iOS retrievePeripherals、 // Android getRemoteDevice），设备回到范围内就能接上。 @@ -145,16 +146,24 @@ class EaiCarDevicePlugin implements DevicePlugin { void _startupReconnect() { if (_disposed || _session != null || link.isConnected) return; - if (rememberedDevices.isEmpty) return; + if (rememberedDevices.isEmpty) { + Logger.w(_tag, '没有记住的香薰，跳过冷启动回连'); + return; + } String last = ''; try { last = DeviceHub.to.lastUsedVendor; } catch (_) {} - if (last.isNotEmpty && last != vendorKey) return; - if (!BackgroundLaunchConsent.granted) { - Logger.i(_tag, '用户未开启设备自动连接，跳过香薰冷启动回连'); + if (last.isNotEmpty && last != vendorKey) { + Logger.w(_tag, '上次使用的是 $last，跳过香薰冷启动回连'); + return; + } + if (BackgroundLaunchConsent.declined) { + Logger.w(_tag, '用户已关闭设备自动连接，跳过香薰冷启动回连'); return; } + final first = rememberedDevices.first; + Logger.w(_tag, '冷启动回连香薰 ${first.name} ${first.id}'); _reconnectAttempt = 0; _scheduleReconnect('冷启动', immediate: true); } @@ -186,7 +195,7 @@ class EaiCarDevicePlugin implements DevicePlugin { : _reconnectDelaysSec[ math.min(_reconnectAttempt, _reconnectDelaysSec.length - 1)]; _reconnectAttempt++; - Logger.i(_tag, '$why，${sec}s 后第 $_reconnectAttempt 次自动回连'); + Logger.w(_tag, '$why，${sec}s 后第 $_reconnectAttempt 次自动回连'); _reconnectTimer = Timer(Duration(seconds: sec), () async { _reconnectTimer = null; if (_disposed || @@ -211,7 +220,7 @@ class EaiCarDevicePlugin implements DevicePlugin { timeout: const Duration(seconds: 45), extra: {if (mac is String && mac.isNotEmpty) 'mac': mac}, )); - Logger.i(_tag, '香薰自动回连成功（第 $_reconnectAttempt 次）'); + Logger.w(_tag, '香薰自动回连成功（第 $_reconnectAttempt 次）'); _reconnectAttempt = 0; } catch (e) { _scheduleReconnect('自动回连未成功（$e）'); diff --git a/apps/client/lib/modules/emai/controllers/emai_controller.dart b/apps/client/lib/modules/emai/controllers/emai_controller.dart index b6c114f5..e5438dba 100644 --- a/apps/client/lib/modules/emai/controllers/emai_controller.dart +++ b/apps/client/lib/modules/emai/controllers/emai_controller.dart @@ -853,11 +853,15 @@ class EmaiController extends GetxController break; case BailianEventType.speechContent: - _streamingUser ??= () { - final m = EmaiMessage(isUser: true, text: ''); - _append(m); - return m; - }(); + _streamingUser ??= _adoptRecent( + isUser: true, + text: e.text, + create: () { + final m = EmaiMessage(isUser: true, text: ''); + _append(m); + return m; + }, + ); _mergeInto(_streamingUser!, e.text); if (e.isFinal) { _streamingUser!.isFinal = true; @@ -896,11 +900,15 @@ class EmaiController extends GetxController // 不会再来 RespondingStarted/Ended。照常建气泡的话会在列表里留一个 // 永远填不上的空泡（就是聊天记录里那几个孤零零的 “• • •”）。 if (e.text.isEmpty && e.finishReason == 'command_calls') break; - _streamingAssistant ??= () { - final m = EmaiMessage(isUser: false, text: ''); - _append(m); - return m; - }(); + _streamingAssistant ??= _adoptRecent( + isUser: false, + text: e.text, + create: () { + final m = EmaiMessage(isUser: false, text: ''); + _append(m); + return m; + }, + ); _mergeInto(_streamingAssistant!, e.text); messages.refresh(); _scrollToBottom(); @@ -1004,6 +1012,21 @@ class EmaiController extends GetxController void _mergeInto(EmaiMessage m, String incoming) => _store.applyText(m, incoming); + /// 同一句的收尾帧如果流式指针已经空了，复用末尾那条完全相同的气泡。 + EmaiMessage _adoptRecent({ + required bool isUser, + required String text, + required EmaiMessage Function() create, + }) { + if (text.isNotEmpty && messages.isNotEmpty) { + final last = messages.last; + if (last.isUser == isUser && !last.isError && last.text == text) { + return last; + } + } + return create(); + } + void _dropIfEmpty(EmaiMessage? m) => _store.dropIfEmpty(m); void _append(EmaiMessage m) { diff --git a/apps/client/lib/modules/translation/controllers/translation_controller.dart b/apps/client/lib/modules/translation/controllers/translation_controller.dart index d143f686..e77de5d2 100644 --- a/apps/client/lib/modules/translation/controllers/translation_controller.dart +++ b/apps/client/lib/modules/translation/controllers/translation_controller.dart @@ -227,6 +227,11 @@ class TranslationController extends GetxController with WidgetsBindingObserver { // openMic(callLocal)(本端) / openMic(callPeer)(对端) → AstService.pushExternalAudio // AstService.astTtsFrames → openSpeaker(callLegA / callLegB) DeviceSession? _callSession; + /// 同声翻译正在用的耳机麦。空 = 本轮走手机麦克风。 + DeviceSession? _simHeadsetSession; + DeviceAudioSource? _simHeadsetMic; + StreamSubscription? _simHeadsetMicSub; + DeviceFileRecorder? _simHeadsetRec; DeviceAudioSource? _callLocalSrc; DeviceAudioSource? _callPeerSrc; DeviceAudioSink? _callSinkA; @@ -1300,13 +1305,26 @@ class TranslationController extends GetxController with WidgetsBindingObserver { if (currentMode.value == 'call') { await _astService.enableRecord(filePath); } else { - AudioSourceType audioSourceType; - if (currentMode.value == 'audioVideo') { - audioSourceType = AudioSourceType.systemAudio; + if (_simHeadsetSession != null) { + // 识别已经在吃耳机麦。这里再开手机麦录音会把手机麦一起打开。 + _simHeadsetRec = await _simHeadsetSession!.openFileRecorder( + path: filePath, + routes: const [AudioRoute.mic], + ); + Logger.w( + 'Translation', + _simHeadsetRec != null + ? '同声翻译录音走耳机麦' + : '同声翻译耳机麦落盘未开，识别仍走耳机麦'); } else { - audioSourceType = AudioSourceType.microphone; + AudioSourceType audioSourceType; + if (currentMode.value == 'audioVideo') { + audioSourceType = AudioSourceType.systemAudio; + } else { + audioSourceType = AudioSourceType.microphone; + } + await _asrService.enableRecord(audioSourceType, filePath, false); } - await _asrService.enableRecord(audioSourceType, filePath, false); } isCreateRecord.value = true; _recordingStartedAt = DateTime.now(); @@ -1335,7 +1353,14 @@ class TranslationController extends GetxController with WidgetsBindingObserver { _recordingStartedAt = null; _timerManager.stopTimer(); - if (currentMode.value == 'call') { + if (_simHeadsetRec != null) { + try { + await _simHeadsetRec!.stop(save: true); + } catch (e) { + Logger.w('Translation', '停止耳机麦录音失败: $e'); + } + _simHeadsetRec = null; + } else if (currentMode.value == 'call') { await _astService.stopRecord(true); } else { await _asrService.stopRecord(true); @@ -1563,8 +1588,20 @@ class TranslationController extends GetxController with WidgetsBindingObserver { return; } if (currentMode.value == 'simultaneous') { - Logger.i('Translation', '开始语音识别，模式：同声传译，ASR phone_call'); + // 耳机麦不经过手机录音，只挂麦克风型前台服务时华为仍会掐后台网络。 + // 保活必须在还处于前台时拉起。 + await AudioForegroundService.start(keepalive: true, pinSpeaker: false); + // iOS 没有安卓那条前台服务。同声翻译的收音走耳机、译文走系统播放， + // 切后台要靠静音循环把 audio 后台模式撑住。 + if (Platform.isIOS) { + await BackgroundKeepalive.acquire('simultaneous'); + } + await _armSimultaneousInput(); + Logger.w( + 'Translation', + '开始语音识别，模式：同声传译，收音=${_simHeadsetSession != null ? "耳机麦" : "手机麦克风"}'); await _startAsrService('phone_call'); + await _attachSimultaneousHeadsetMic(); } else if (currentMode.value == 'faceToFace') { final bes = _besFaceToFaceSession(); if (bes != null) { @@ -1657,6 +1694,13 @@ class TranslationController extends GetxController with WidgetsBindingObserver { Logger.error('异常终止时停止录音失败(忽略): $e'); } } + await _stopSimultaneousHeadsetMic(); + if (currentMode.value == 'simultaneous') { + await AudioForegroundService.stop(); + if (Platform.isIOS) { + await BackgroundKeepalive.release('simultaneous'); + } + } // 音视频/通话模式：识别挂了，设备侧/录屏广播那条链路也得收掉， // 否则 socket 接收端还开着、没人消费（正常 stopAll 走的是同一个收尾，幂等）。 if (currentMode.value == "call" || currentMode.value == "audioVideo") { @@ -1877,7 +1921,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { // `Pged-Freezer: Freeze process` 冻结进程 → 两条 WebSocket 当场 // `Software caused connection abort` → Dart 当成 AST 故障 → // _terminateCallOnError 把整个通话翻译关掉。用户看到的是「切个后台功能就没了」。 - await AudioForegroundService.start(); + await AudioForegroundService.start(keepalive: true); _callSession = await _awaitCallAudioSession(); _callModeUsingBes = _callSession != null; _callAudioTornDown = false; @@ -2157,6 +2201,74 @@ class TranslationController extends GetxController with WidgetsBindingObserver { isTtsEnabled.value = true; } + /// 同声翻译的收音：连着耳机就用耳机主麦，否则用手机麦。 + /// + /// 只认恒玄耳机的现场录音麦（`AA 59` 之后的 `mainMicData`）。 + /// 面对面那条双耳 BLE、通话双路都不走这里。 + Future _armSimultaneousInput() async { + _audioSourceType = false; + final session = _connectedHeadsetMic(); + if (session == null) { + Logger.w('Translation', '同声翻译收音：手机麦克风'); + return; + } + try { + await session.invokeFeature(DeviceFeatures.recordStart); + _simHeadsetSession = session; + _audioSourceType = true; + Logger.w('Translation', '同声翻译收音：耳机麦'); + } catch (e) { + _simHeadsetSession = null; + _audioSourceType = false; + Logger.w('Translation', '同声翻译耳机麦开启失败，改用手机麦: $e'); + } + } + + DeviceSession? _connectedHeadsetMic() { + for (final session in DeviceHub.to.sessions) { + if (session.vendor == DeviceVendors.bes && + session.supports(DeviceCapability.micRecording)) { + return session; + } + } + return null; + } + + Future _attachSimultaneousHeadsetMic() async { + final session = _simHeadsetSession; + if (session == null) return; + final src = await session.openMic(route: AudioRoute.mic); + _simHeadsetMic = src; + var logged = false; + _simHeadsetMicSub = src.pcm.listen((pcm) { + if (!logged) { + logged = true; + Logger.w('Translation', '同声翻译耳机麦首帧 ${pcm.length} 字节'); + } + unawaited(_asrService.pushAudioData(pcm)); + }); + } + + Future _stopSimultaneousHeadsetMic() async { + await _simHeadsetMicSub?.cancel(); + _simHeadsetMicSub = null; + try { + await _simHeadsetMic?.close(); + } catch (e) { + Logger.w('Translation', '关闭耳机麦订阅失败: $e'); + } + _simHeadsetMic = null; + final session = _simHeadsetSession; + _simHeadsetSession = null; + if (session == null) return; + try { + await session.invokeFeature(DeviceFeatures.recordStop); + Logger.w('Translation', '同声翻译已关闭耳机麦'); + } catch (e) { + Logger.w('Translation', '关闭耳机麦失败: $e'); + } + } + /// 启动语音识别 /// /// 这一段是「首次进同传拉不起识别」的另一半嫌疑：原生 @@ -2555,6 +2667,14 @@ class TranslationController extends GetxController with WidgetsBindingObserver { currentSessionId = null; } catch (e) { Logger.error('停止语音识别失败: ${e.toString()}'); + } finally { + await _stopSimultaneousHeadsetMic(); + if (currentMode.value == 'simultaneous') { + await AudioForegroundService.stop(); + if (Platform.isIOS) { + await BackgroundKeepalive.release('simultaneous'); + } + } } } diff --git a/apps/client/local_plugins/azure_speech/android/src/main/AndroidManifest.xml b/apps/client/local_plugins/azure_speech/android/src/main/AndroidManifest.xml index 5de36e3c..b32f5831 100644 --- a/apps/client/local_plugins/azure_speech/android/src/main/AndroidManifest.xml +++ b/apps/client/local_plugins/azure_speech/android/src/main/AndroidManifest.xml @@ -6,6 +6,9 @@ + + + @@ -15,7 +18,7 @@ android:name=".tools.AudioRecordingForegroundService" android:enabled="true" android:exported="false" - android:foregroundServiceType="microphone" /> + android:foregroundServiceType="microphone|mediaPlayback|connectedDevice" /> { pcmSpeakerPlayer.end(); result.success(null) } "stop" -> { pcmSpeakerPlayer.stop(); result.success(null) } "dispose" -> { pcmSpeakerPlayer.dispose(); result.success(null) } + // 设备对话占住前后打日志用。真正停音乐走 holdDeviceFocus， + // 不经过 audio_session（那条会短路，而且只申请可被忽略的 TRANSIENT）。 + "musicActive" -> { + val am = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager + result.success(am.isMusicActive) + } + "holdDeviceFocus" -> DeviceDialogFocus.hold(context, mainHandler, result) + "releaseDeviceFocus" -> result.success(DeviceDialogFocus.release(context)) else -> result.notImplemented() } } @@ -1401,7 +1410,9 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, // 否则切后台后录音会在国内 ROM 上被悄悄掐断。 "startAudioForegroundService" -> { try { - AudioRecordingForegroundService.startService(context) + val keepalive = call.argument("keepalive") ?: false + val pinSpeaker = call.argument("pinSpeaker") ?: true + AudioRecordingForegroundService.startService(context, keepalive, pinSpeaker) result.success(true) } catch (e: Exception) { result.error("START_FGS_ERROR", e.message, null) diff --git a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DeviceDialogFocus.kt b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DeviceDialogFocus.kt new file mode 100644 index 00000000..2f04234c --- /dev/null +++ b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DeviceDialogFocus.kt @@ -0,0 +1,303 @@ +package com.yunqiinnovation.azure_speech + +import android.app.ActivityManager +import android.content.Context +import android.media.AudioAttributes +import android.media.AudioFocusRequest +import android.media.AudioManager +import android.media.AudioPlaybackConfiguration +import android.os.Build +import android.os.Handler +import android.os.Process +import android.os.SystemClock +import android.util.Log +import android.view.KeyEvent +import io.flutter.plugin.common.MethodChannel + +/** + * 设备按键唤醒的对话，在 Android 上自己申请音频焦点。 + * + * audio_session 的 setActive 有两个对不上的地方： + * 1. 进程里已经有一份 AudioFocusRequest 时，它直接返回 true，不再向系统申请； + * 2. 它申请的是 GAIN_TRANSIENT + USAGE_MEDIA，华为/ColorOS 上音乐 App 可以继续播， + * 只压低音量，甚至完全不理会。 + * + * 华为 ALN-AL80 上独占焦点能拿到，但 300ms 内 isMusicActive 仍为 true， + * 音乐 App 不暂停。中途放弃再改抢 ASSISTANT 会给对方一次 LOSS， + * 结束时再 abandon 它也不自动续播。 + * + * 所以整段对话只持有一次 MEDIA 独占焦点。进来时**别的应用**正在播，才暂停， + * 结束时再补一次播放键。系统本来没在播，就不抢焦点、也不发播放键—— + * 交还焦点或 MEDIA_PLAY 都会把华为音乐从暂停状态拉起来。 + */ +object DeviceDialogFocus { + private const val TAG = "EmaiFocus" + + /** AudioPlaybackConfiguration.PLAYER_STATE_STARTED，公开 SDK 不导出这个常量。 */ + private const val PLAYER_STATE_STARTED = 2 + + private var audioManager: AudioManager? = null + private var request: AudioFocusRequest? = null + private var pausedExternal = false + + private val listener = AudioManager.OnAudioFocusChangeListener { change -> + val music = try { + audioManager?.isMusicActive + } catch (_: Throwable) { + null + } + Log.i(TAG, "焦点变化=${focusName(change)} isMusicActive=$music") + } + + fun hold(context: Context, handler: Handler, result: MethodChannel.Result) { + val am = try { + manager(context) + } catch (e: Throwable) { + Log.e(TAG, "拿不到 AudioManager: ${e.message}") + result.success(mapOf("error" to (e.message ?: "no AudioManager"))) + return + } + val before = music(am) + val external = externalStarted(am) + val mode = try { + am.mode + } catch (_: Throwable) { + -1 + } + val importance = importance() + pausedExternal = false + if (!external) { + Log.i( + TAG, + "系统没在播音乐，不抢焦点、不发媒体键 isMusicActive=$before " + + "importance=$importance ${Build.BRAND}/${Build.MODEL}" + ) + result.success( + mapOf( + "gain" to "NONE", + "usage" to "MEDIA", + "result" to "SKIPPED", + "mediaPause" to false, + "externalPlaying" to false, + "musicBefore" to before, + "importance" to importance, + ) + ) + return + } + abandon(am, "重抢前") + val outcome = request(am, AudioAttributes.USAGE_MEDIA, "MEDIA") + mediaKey(am, KeyEvent.KEYCODE_MEDIA_PAUSE) + pausedExternal = true + val after = music(am) + Log.i( + TAG, + "申请 EXCLUSIVE/MEDIA 结果=${outcome.name} 媒体暂停键=$pausedExternal " + + "mode=$mode importance=$importance 音乐 $before → $after " + + "sdk=${Build.VERSION.SDK_INT} ${Build.BRAND}/${Build.MODEL}" + ) + result.success( + mapOf( + "gain" to "EXCLUSIVE", + "usage" to "MEDIA", + "result" to outcome.name, + "mediaPause" to pausedExternal, + "musicBefore" to before, + "musicAfter" to after, + "mode" to mode, + "importance" to importance, + "brand" to Build.BRAND, + "model" to Build.MODEL, + "sdk" to Build.VERSION.SDK_INT, + ) + ) + handler.postDelayed({ + Log.i(TAG, "300ms 复查 音乐=${music(am)} 已发暂停键=$pausedExternal") + }, 300) + } + + fun release(context: Context): Map { + val am = manager(context) + val before = music(am) + val had = request != null + val shouldResume = pausedExternal + if (!shouldResume) { + Log.i(TAG, "进来时系统没在播，交还不发播放键 isMusicActive=$before") + return mapOf( + "had" to had, + "mediaPlay" to false, + "musicBefore" to before, + "musicAfter" to before, + ) + } + abandon(am, "交还") + pausedExternal = false + val afterAbandon = music(am) + mediaKey(am, KeyEvent.KEYCODE_MEDIA_PLAY) + val sentPlay = true + val after = music(am) + Log.i( + TAG, + "交还 had=$had 播放键=$sentPlay 音乐 $before → $afterAbandon → $after" + ) + return mapOf( + "had" to had, + "mediaPlay" to sentPlay, + "musicBefore" to before, + "musicAfter" to after, + ) + } + + private fun manager(context: Context): AudioManager { + val existing = audioManager + if (existing != null) return existing + val created = context.applicationContext + .getSystemService(Context.AUDIO_SERVICE) as AudioManager + audioManager = created + return created + } + + private fun request(am: AudioManager, usage: Int, usageName: String): RequestOutcome { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { + @Suppress("DEPRECATION") + val status = am.requestAudioFocus( + listener, + AudioManager.STREAM_MUSIC, + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE, + ) + val outcome = RequestOutcome(status) + Log.i(TAG, "legacy 申请 EXCLUSIVE/$usageName 结果=${outcome.name}") + return outcome + } + val attrs = AudioAttributes.Builder() + .setUsage(usage) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + val req = AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE) + .setAudioAttributes(attrs) + .setAcceptsDelayedFocusGain(false) + .setWillPauseWhenDucked(true) + .setOnAudioFocusChangeListener(listener) + .build() + val status = am.requestAudioFocus(req) + if (status == AudioManager.AUDIOFOCUS_REQUEST_GRANTED) { + request = req + } else { + request = null + Log.w(TAG, "申请 EXCLUSIVE/$usageName 失败 结果=${RequestOutcome(status).name}") + } + return RequestOutcome(status) + } + + private fun abandon(am: AudioManager, why: String) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val prev = request + request = null + if (prev == null) return + try { + val status = am.abandonAudioFocusRequest(prev) + Log.i(TAG, "$why abandon=${RequestOutcome(status).name}") + } catch (e: Throwable) { + Log.w(TAG, "$why abandon 异常: ${e.message}") + } + } else { + @Suppress("DEPRECATION") + try { + am.abandonAudioFocus(listener) + } catch (e: Throwable) { + Log.w(TAG, "$why abandon legacy 异常: ${e.message}") + } + } + } + + private fun mediaKey(am: AudioManager, keyCode: Int) { + val now = SystemClock.uptimeMillis() + try { + am.dispatchMediaKeyEvent(KeyEvent(now, now, KeyEvent.ACTION_DOWN, keyCode, 0)) + am.dispatchMediaKeyEvent(KeyEvent(now, now, KeyEvent.ACTION_UP, keyCode, 0)) + Log.i(TAG, "媒体键 ${if (keyCode == KeyEvent.KEYCODE_MEDIA_PAUSE) "PAUSE" else "PLAY"}") + } catch (e: Throwable) { + Log.w(TAG, "媒体键失败 key=$keyCode: ${e.message}") + } + } + + /** + * 别的应用有一条正在出声的媒体播放。自己的 AudioTrack 不算，暂停着的也不算。 + * + * 必须同时满足「播放器状态是 STARTED」和「isMusicActive」。只看状态会把 + * 暂停着、音轨还留在列表里的播放器算成在播，结束时一交还焦点，华为就把 + * 它拉起来。只看 isMusicActive 又会被我们自己后启动的静音保活污染。 + * + * getClientUid / getPlayerState 在公开 android.jar 里是隐藏接口，编译期调不到， + * 运行时再反射。读不到就退回 isMusicActive：调用方要在自己的音轨起来之前问。 + */ + private fun externalStarted(am: AudioManager): Boolean { + val audible = music(am) + if (!audible) { + Log.i(TAG, "isMusicActive=false，没有正在出声的播放") + return false + } + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) return true + return try { + val mine = Process.myUid() + val uidOf = AudioPlaybackConfiguration::class.java.getMethod("getClientUid") + val stateOf = AudioPlaybackConfiguration::class.java.getMethod("getPlayerState") + val rows = am.activePlaybackConfigurations.map { cfg -> + Triple( + uidOf.invoke(cfg) as Int, + stateOf.invoke(cfg) as Int, + cfg.audioAttributes.usage, + ) + } + val started = rows.any { + it.first != mine && + it.second == PLAYER_STATE_STARTED && + it.third == AudioAttributes.USAGE_MEDIA + } + Log.i( + TAG, + "外部播放器 " + rows.joinToString { + "uid=${it.first}/state=${it.second}/usage=${it.third}" + } + " mediaStarted=$started audible=$audible" + ) + started + } catch (e: Throwable) { + Log.w(TAG, "读播放器列表失败，退回 isMusicActive=$audible: ${e.message}") + audible + } + } + + private fun music(am: AudioManager): Boolean = try { + am.isMusicActive + } catch (_: Throwable) { + false + } + + private fun importance(): Int = try { + val info = ActivityManager.RunningAppProcessInfo() + ActivityManager.getMyMemoryState(info) + info.importance + } catch (_: Throwable) { + -1 + } + + private fun focusName(code: Int): String = when (code) { + AudioManager.AUDIOFOCUS_GAIN -> "GAIN" + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT -> "GAIN_TRANSIENT" + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE -> "GAIN_TRANSIENT_EXCLUSIVE" + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK -> "GAIN_TRANSIENT_MAY_DUCK" + AudioManager.AUDIOFOCUS_LOSS -> "LOSS" + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT -> "LOSS_TRANSIENT" + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK -> "LOSS_TRANSIENT_CAN_DUCK" + else -> "UNKNOWN($code)" + } + + private class RequestOutcome(val code: Int) { + val name: String = when (code) { + AudioManager.AUDIOFOCUS_REQUEST_GRANTED -> "GRANTED" + AudioManager.AUDIOFOCUS_REQUEST_FAILED -> "FAILED" + AudioManager.AUDIOFOCUS_REQUEST_DELAYED -> "DELAYED" + else -> "UNKNOWN($code)" + } + } +} diff --git a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/AudioRecordingForegroundService.kt b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/AudioRecordingForegroundService.kt index 16c768d5..49a183ef 100644 --- a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/AudioRecordingForegroundService.kt +++ b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/AudioRecordingForegroundService.kt @@ -8,11 +8,18 @@ import android.app.Service import android.content.Context import android.content.Intent import android.content.pm.ServiceInfo +import android.media.AudioAttributes +import android.media.AudioDeviceInfo +import android.media.AudioFormat +import android.media.AudioManager +import android.media.AudioTrack import android.os.Build import android.os.IBinder +import android.os.PowerManager import androidx.core.app.NotificationCompat import android.util.Log import org.json.JSONObject +import kotlin.math.max /** * 音频录制前台服务 @@ -20,11 +27,23 @@ import org.json.JSONObject */ class AudioRecordingForegroundService : Service() { + private var wakeLock: PowerManager.WakeLock? = null + private var silenceTrack: AudioTrack? = null + /** 一旦某次启动要了保活，后续不带这个参数的启动（ASR / 通话翻译原生侧）不能把它降回去。 */ + private var keepaliveLatched = false + /** 同声翻译的译文走手机媒体通道，钉在扬声器会把耳机的 A2DP 一起拽走。 */ + private var pinSilenceToSpeaker = true + @Volatile private var silenceRun = false + private var silenceThread: Thread? = null + companion object { private const val TAG = "AudioRecordingService" private const val NOTIFICATION_ID = 1001 private const val CHANNEL_ID = "audio_recording_channel" private const val CHANNEL_NAME = "音频录制服务" + private const val KEEPALIVE_CHANNEL_ID = "session_keepalive" + private const val EXTRA_KEEPALIVE = "keepalive" + private const val EXTRA_PIN_SPEAKER = "pinSpeaker" private const val ACTION_STOP = "com.yunqiinnovation.azure_speech.tools.STOP_SERVICE" private const val NATIVE_LOCALIZATION_PREF_NAME = "native_localization" private const val NATIVE_LOCALIZATION_PREF_KEY_STRINGS_JSON = "strings_json" @@ -33,14 +52,16 @@ class AudioRecordingForegroundService : Service() { * 启动前台服务 * @param context 上下文 */ - fun startService(context: Context) { + fun startService(context: Context, keepalive: Boolean = false, pinSpeaker: Boolean = true) { val intent = Intent(context, AudioRecordingForegroundService::class.java) + intent.putExtra(EXTRA_KEEPALIVE, keepalive) + intent.putExtra(EXTRA_PIN_SPEAKER, pinSpeaker) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { context.startForegroundService(intent) } else { context.startService(intent) } - Log.d(TAG, "音频录制前台服务已启动") + Log.i("EmaiFocus", "前台服务启动请求 keepalive=$keepalive") } /** @@ -72,34 +93,48 @@ class AudioRecordingForegroundService : Service() { /** * 以麦克风类型进入前台。通知权限被关或 ROM 拒绝时只记日志，不把进程带崩。 */ - private fun startForegroundCompat(notification: android.app.Notification) { + private fun startForegroundCompat(notification: android.app.Notification, keepalive: Boolean) { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { startForeground(NOTIFICATION_ID, notification) + Log.i("EmaiFocus", "前台服务已进入前台") return } + val mic = ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE + val typed = if (keepalive) { + mic or ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PLAYBACK or + ServiceInfo.FOREGROUND_SERVICE_TYPE_CONNECTED_DEVICE + } else { + mic + } try { - startForeground( - NOTIFICATION_ID, - notification, - ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE, - ) - Log.d(TAG, "前台服务已启动(类型=麦克风)") + startForeground(NOTIFICATION_ID, notification, typed) + Log.i("EmaiFocus", "前台服务已进入前台 type=$typed keepalive=$keepalive") } catch (e: Exception) { - Log.e(TAG, "前台服务启动失败: ${e.message}") + Log.e("EmaiFocus", "带类型前台失败，退回麦克风: ${e.message}") + try { + startForeground(NOTIFICATION_ID, notification, mic) + } catch (e2: Exception) { + Log.e("EmaiFocus", "前台服务启动失败: ${e2.message}") + } } } override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { Log.d(TAG, "AudioRecordingForegroundService onStartCommand action=${intent?.action}") - + if (intent?.getBooleanExtra(EXTRA_KEEPALIVE, false) == true) { + keepaliveLatched = true + if (silenceTrack == null) { + pinSilenceToSpeaker = + intent?.getBooleanExtra(EXTRA_PIN_SPEAKER, true) ?: true + } + } + val keepalive = keepaliveLatched + // 创建前台通知 - val notification = createNotification() - // 前台通知显示条件： - // - 只要该 Service 被启动（AzureSpeechPlugin/AzureAsrHelper 在 enableRecord 时会启动它），就会在 onStartCommand 里立刻调用 startForeground()； - // - 渠道为 IMPORTANCE_LOW，通常不会弹出“抬头横幅(Heads-up)”，但会在下拉通知栏显示常驻通知； - // - Android 13+ 如果未授予通知权限 / 用户关闭通知，可能出现通知不可见或启动前台服务失败（不同 ROM 行为可能不同）。 + val notification = createNotification(keepalive) // 必须立即调用 startForeground 以满足 Android 8.0+ 的要求 - startForegroundCompat(notification) + startForegroundCompat(notification, keepalive) + if (keepalive) startDialogKeepalive() // 检查是否是停止指令 if (intent?.action == ACTION_STOP) { @@ -118,9 +153,101 @@ class AudioRecordingForegroundService : Service() { } override fun onDestroy() { + keepaliveLatched = false + pinSilenceToSpeaker = true + stopDialogKeepalive() super.onDestroy() Log.d(TAG, "AudioRecordingForegroundService onDestroy") - stopForeground(true) + stopForeground(STOP_FOREGROUND_REMOVE) + } + + /// 耳机 AI 的声音不经过手机。华为看到「没有音频的后台进程」会掐掉网络， + /// 百炼 WebSocket 就是 Software caused connection abort。 + /// 这里不申请音频焦点、不改 AudioManager 模式，音量是 0，输出钉在手机听筒外的扬声器。 + private fun startDialogKeepalive() { + if (silenceTrack != null) return + try { + val pm = getSystemService(POWER_SERVICE) as PowerManager + val lock = pm.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, "eaimar:device-dialog") + lock.setReferenceCounted(false) + lock.acquire(30 * 60 * 1000L) + wakeLock = lock + } catch (e: Exception) { + Log.e("EmaiFocus", "唤醒锁失败: ${e.message}") + } + try { + val sampleRate = 16000 + val min = AudioTrack.getMinBufferSize( + sampleRate, + AudioFormat.CHANNEL_OUT_MONO, + AudioFormat.ENCODING_PCM_16BIT, + ) + val track = AudioTrack.Builder() + .setAudioAttributes( + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_MEDIA) + .setContentType(AudioAttributes.CONTENT_TYPE_MUSIC) + .build() + ) + .setAudioFormat( + AudioFormat.Builder() + .setSampleRate(sampleRate) + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setChannelMask(AudioFormat.CHANNEL_OUT_MONO) + .build() + ) + .setBufferSizeInBytes(max(min, 640)) + .setTransferMode(AudioTrack.MODE_STREAM) + .build() + if (pinSilenceToSpeaker && Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + val am = getSystemService(AUDIO_SERVICE) as AudioManager + val speaker = am.getDevices(AudioManager.GET_DEVICES_OUTPUTS) + .firstOrNull { it.type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER } + val routed = speaker != null && track.setPreferredDevice(speaker) + Log.i("EmaiFocus", "静音保活输出到扬声器=$routed") + } else { + Log.i("EmaiFocus", "静音保活跟随当前输出") + } + track.setVolume(0f) + track.play() + silenceTrack = track + silenceRun = true + val silence = ByteArray(640) + silenceThread = Thread { + val player = track + while (silenceRun) { + val n = player.write(silence, 0, silence.size) + if (n < 0) break + } + }.apply { + name = "device-dialog-silence" + start() + } + Log.i("EmaiFocus", "设备对话静音保活已开") + } catch (e: Exception) { + Log.e("EmaiFocus", "静音保活失败: ${e.message}") + } + } + + private fun stopDialogKeepalive() { + silenceRun = false + try { + silenceThread?.join(200) + } catch (_: InterruptedException) { + } + silenceThread = null + try { + silenceTrack?.pause() + silenceTrack?.flush() + silenceTrack?.release() + } catch (_: Exception) { + } + silenceTrack = null + try { + if (wakeLock?.isHeld == true) wakeLock?.release() + } catch (_: Exception) { + } + wakeLock = null } /** @@ -128,7 +255,8 @@ class AudioRecordingForegroundService : Service() { */ private fun createNotificationChannel() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - val channel = NotificationChannel( + val notificationManager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + val recording = NotificationChannel( CHANNEL_ID, CHANNEL_NAME, NotificationManager.IMPORTANCE_LOW @@ -138,9 +266,19 @@ class AudioRecordingForegroundService : Service() { setSound(null, null) enableVibration(false) } - - val notificationManager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager - notificationManager.createNotificationChannel(channel) + notificationManager.createNotificationChannel(recording) + // 设备对话这条要能在通知栏看见。华为对「看不见的前台服务」照样掐后台网络。 + val dialog = NotificationChannel( + KEEPALIVE_CHANNEL_ID, + "EAIMAR", + NotificationManager.IMPORTANCE_DEFAULT + ).apply { + description = "同声翻译、通话翻译、AI 对话切到后台时保持连接" + setShowBadge(false) + setSound(null, null) + enableVibration(false) + } + notificationManager.createNotificationChannel(dialog) } } @@ -148,7 +286,7 @@ class AudioRecordingForegroundService : Service() { * 创建前台服务通知 * @return 通知对象 */ - private fun createNotification(): Notification { + private fun createNotification(keepalive: Boolean = false): Notification { val intent = (packageManager.getLaunchIntentForPackage(packageName) ?: Intent()).apply { addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_CLEAR_TOP) putExtra("com.yimai.eaimar.extra.NAVIGATE_TO", "meetingRecord") @@ -161,11 +299,18 @@ class AudioRecordingForegroundService : Service() { ) val title = getNativeLocalizedString("native.notification.appName", "EAIMAR") - val content = getNativeLocalizedString( - "native.notification.audioRecording.recording", - "Recording audio, tap to return to app" - ) - return NotificationCompat.Builder(this, CHANNEL_ID) + val content = if (keepalive) { + getNativeLocalizedString( + "native.notification.bleAgent.running", + "语音助手正在运行" + ) + } else { + getNativeLocalizedString( + "native.notification.audioRecording.recording", + "Recording audio, tap to return to app" + ) + } + return NotificationCompat.Builder(this, if (keepalive) KEEPALIVE_CHANNEL_ID else CHANNEL_ID) .setContentTitle(title) .setContentText(content) .setSmallIcon(applicationInfo.icon) diff --git a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift index daa355c3..6a64eea5 100644 --- a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift +++ b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift @@ -1,6 +1,7 @@ import Flutter import UIKit import AVFoundation +import MediaPlayer import ReplayKit import MicrosoftCognitiveServicesSpeech // 自定义语音处理组件，提供音频流处理等功能 @@ -605,6 +606,13 @@ internal func handlePcmPlayerMethodCall(_ call: FlutterMethodCall, result: @esca case "dispose": pcmSpeakerPlayer.dispose() result(nil) + case "musicActive": + result(AVAudioSession.sharedInstance().isOtherAudioPlaying) + case "setNowPlaying": + let args = call.arguments as? [String: Any] + let active = args?["active"] as? Bool ?? false + setDeviceDialogNowPlaying(active) + result(nil) case "inputFormat": // 给 Dart 侧 MicPcmStreamer 探路：record 插件会拿输入格式去 installTap， // 0Hz 时那是 ObjC 断言必崩，所以开麦前先在这里看一眼。 @@ -624,6 +632,35 @@ internal func handlePcmPlayerMethodCall(_ call: FlutterMethodCall, result: @esca } } +private static var deviceDialogNowPlayingReady = false +private static let deviceDialogNowPlayingTitle = "AI 助手" + +/// 让系统把这次设备对话认成正在播放。不改会话类别。 +/// 只清自己写上去的那条，避免抹掉别的模块的 Now Playing。 +private func setDeviceDialogNowPlaying(_ active: Bool) { + if active { + if !Self.deviceDialogNowPlayingReady { + let commands = MPRemoteCommandCenter.shared() + commands.playCommand.isEnabled = true + commands.playCommand.addTarget { _ in .success } + commands.pauseCommand.isEnabled = true + commands.pauseCommand.addTarget { _ in .success } + Self.deviceDialogNowPlayingReady = true + } + MPNowPlayingInfoCenter.default().nowPlayingInfo = [ + MPMediaItemPropertyTitle: Self.deviceDialogNowPlayingTitle, + MPNowPlayingInfoPropertyPlaybackRate: 1.0, + MPNowPlayingInfoPropertyElapsedPlaybackTime: 0.0, + ] + return + } + let center = MPNowPlayingInfoCenter.default() + if let info = center.nowPlayingInfo, + (info[MPMediaItemPropertyTitle] as? String) == Self.deviceDialogNowPlayingTitle { + center.nowPlayingInfo = nil + } +} + // 简化的事件发送方法 internal func sendAsrEvent(_ event: [String: Any]) { sendEvent(event, type: .asr) diff --git a/apps/client/pubspec.yaml b/apps/client/pubspec.yaml index 76240f76..42e9a11d 100644 --- a/apps/client/pubspec.yaml +++ b/apps/client/pubspec.yaml @@ -1,7 +1,7 @@ name: eaimar description: "Eaimar - AI Voice Assistant." publish_to: "none" -version: 1.0.2+27 +version: 1.0.3+37 environment: sdk: ">=3.3.0 <4.0.0" diff --git a/apps/client/test/devices/bes_recorder_sources_test.dart b/apps/client/test/devices/bes_recorder_sources_test.dart index 0ecb8b42..5bb2bc5d 100644 --- a/apps/client/test/devices/bes_recorder_sources_test.dart +++ b/apps/client/test/devices/bes_recorder_sources_test.dart @@ -19,9 +19,8 @@ void main() { }); test('现场录音取主麦而不是前馈麦', () { - // 裸流隔帧取（openMic 那条 _alternate）靠的是"订阅时恰好对上相位"， - // 中途开始订阅就会永久取到前馈麦。原生早就把两路标好了，按类型取 - // 顺带把这类错相消掉——所以这一条只能是 mainMicData，不能是 ffMicData。 + // openMic(mic) 与落盘都取原生标好的 mainMicData。 + // 裸流隔帧取会在中途订阅时锁死前馈麦，所以这一条只能是 mainMicData。 expect(BesDeviceSession.recorderSources[AudioRoute.mic], 'mainMicData'); }); diff --git a/apps/client/test/emai_device_dialog_audio_test.dart b/apps/client/test/emai_device_dialog_audio_test.dart new file mode 100644 index 00000000..4ce97a8c --- /dev/null +++ b/apps/client/test/emai_device_dialog_audio_test.dart @@ -0,0 +1,20 @@ +import 'package:audio_session/audio_session.dart'; +import 'package:eaimar/data/utils/emai_audio_session.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('设备对话的放音会话独占，且不带 playback 上非法的 A2DP 选项', () { + final config = EmaiAudioSession.playbackOnlyConfigForTest(); + final options = config.avAudioSessionCategoryOptions!; + + expect(config.avAudioSessionCategory, AVAudioSessionCategory.playback); + expect(config.avAudioSessionMode, AVAudioSessionMode.defaultMode); + expect(options.contains(AVAudioSessionCategoryOptions.mixWithOthers), isFalse); + expect(options.contains(AVAudioSessionCategoryOptions.allowBluetoothA2dp), isFalse); + expect(options.contains(AVAudioSessionCategoryOptions.defaultToSpeaker), isFalse); + expect(config.androidAudioFocusGainType, AndroidAudioFocusGainType.gainTransient); + expect(config.androidAudioAttributes!.usage, AndroidAudioUsage.media); + expect(config.androidAudioAttributes!.contentType, AndroidAudioContentType.speech); + expect(config.androidWillPauseWhenDucked, isTrue); + }); +}