Browse Source
iOS 上恒玄走的是 GATT over BR/EDR 而非 BLE，之前按 BLE 模型处理导致一连串问题。 本次改动都有真机数据支撑（2026-09-21 压测）。 GATT 写入（bluetooth_manager） - 写类型改为优先 .withoutResponse。ATT 是请求-响应协议，带应答写每包等 ACK， 追不上下行 50 包/秒，表现为音频越听越滞后。 - canSendWriteWithoutResponse 只作观测、不作门槛。它是 CoreBluetooth 为 BLE 设计的提示，在 BR/EDR GATT 上不反映真实能力（实测 100% 为 false 却零丢弃）； 拿它当闸门会死锁：不写就不回调，不回调就永不翻真，实测 548 包只出去 11 个。 - 控制命令无条件直发、不排队。排到积压音频后面会让 AA 09/AA 06/queryCallState 全部发不出去，曾被误判成「固件换了命令号」。 - 不再对标准电池服务 0x180F 做任何 ATT 操作：它在 Echo-one 上是空壳（值恒为 0）， 而挂在它上面的读/订阅会把整条 ATT 通道堵死。 后台存活（translation） - 通话翻译期间持有 BackgroundKeepalive。两路音频全走耳机 GATT，手机侧没有 AVAudioSession 的实际 I/O，UIBackgroundModes=audio 不成立，切后台几十秒后 上行被掐断：WebSocket 还连着，但服务端收不到语音，靠默认 VAD 每 30 秒强切一句， 吐出空译文或幻觉 —— 表现为「切后台后只剩译文、原文全没」。 - BackgroundKeepalive 不再无条件把音频类别改成 .playback，避免踩掉通话翻译 正在跑的 .playAndRecord 录音链路。 状态竞态（azure_speech / device_hub） - markReady 的就绪状态记在插件上、建桥时继承。建桥与 AST 异步初始化是两条独立 路径，初始化先完成时那句 optional chaining 打在 nil 上静默丢失，之后 ready 恒为 false、上行帧全积在环形缓冲里，表现为「打开通话翻译没反应，再开一次才好」。 - DeviceHub.inCall 改以会话快照为准。事件会随 worker 在断连时被 dispose 而丢失， 重连后 ever 又不回调当前值，且后续重复赋同值不触发，永远不会自愈。 可观测性 - 下行发送器与 GATT 写入的日志从 NSLog 改为 os_log（release/profile 包里 idevicesyslog 只收得到 os_log），并经 DeviceSession.diagnostics() 暴露给 Dart， 写入文件日志后可用 devicectl 直接拉取，不依赖 usbmuxd。 - 新增 inCall 变化、闸门拦截、通话状态翻转、前后台切换四处观测点。 其它 - DeviceSession 新增 diagnostics()（默认空表）；SmartcarSession 因是 implements 而非 extends，同步补了实现。 - CLAUDE.md 记录本次全部实测结论，含 BB 8E 为通话状态权威、BB 02/03 是噪声 （已知问题，本次未改判定）。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>main
Rodger-Wang
3 weeks ago
104 changed files with 6866 additions and 235 deletions
Binary file not shown.
@ -0,0 +1,324 @@ |
|||
import 'dart:async'; |
|||
import 'dart:convert'; |
|||
import 'dart:io' show Platform; |
|||
|
|||
import 'package:flutter/foundation.dart' show kIsWeb; |
|||
import 'package:flutter/services.dart'; |
|||
|
|||
import '../../core/utils/logger.dart'; |
|||
import '../models/assistant_directive.dart'; |
|||
import 'bailian_multimodal_service.dart'; |
|||
|
|||
/// 耳机唤醒那条 EMAI 链路的**原生**百炼客户端句柄。 |
|||
/// |
|||
/// 真正的 WebSocket 与音频收发在 `azure_speech` 插件里 |
|||
/// （`BailianDialogClient.swift` / `.kt`）；上行 PCM 由 `bluetooth_manager` 直接 |
|||
/// 交过去，合成音频原路回耳机，**一个字节都不经过 Dart**。这里只做三件事： |
|||
/// 起停、把原生事件翻译成 [BailianEvent]、把门控开关推下去。 |
|||
/// |
|||
/// ## 为什么保留 Dart 版 |
|||
/// [BailianMultimodalService] 仍然服务两种情况： |
|||
/// 1. **手机麦那条链路**（EMAI 页按住说话）——音频本来就在 Dart 侧采集，没有下沉的意义； |
|||
/// 2. **退路**——插件是老版本（`MissingPluginException`）、或设备不是恒玄（AI PCM 桥 |
|||
/// 只存在于 bluetooth_manager）时，整条链路退回 Dart，功能不受影响。 |
|||
/// |
|||
/// ## ⚠️ 这不修「锁屏断连」 |
|||
/// iOS 挂起的是整个进程，原生和 Dart 一起停。锁屏能不能活着取决于 |
|||
/// `UIBackgroundModes=audio` + 确实有音频在播（`BackgroundKeepalive` 干的事）， |
|||
/// 与代码在哪一层无关。本类解决的是 event loop 抖动与跨界开销。 |
|||
class BailianNativeDialog { |
|||
BailianNativeDialog({String? appId, String? workspaceId}) |
|||
: _appIdOverride = appId, |
|||
_workspaceIdOverride = workspaceId; |
|||
|
|||
static const String _tag = 'BailianNative'; |
|||
|
|||
static const MethodChannel _channel = |
|||
MethodChannel('azure_speech/bailian_dialog'); |
|||
static const EventChannel _events = |
|||
EventChannel('azure_speech/bailian_dialog_events'); |
|||
|
|||
final String? _appIdOverride; |
|||
final String? _workspaceIdOverride; |
|||
|
|||
final _eventCtl = StreamController<BailianEvent>.broadcast(); |
|||
final _downBytesCtl = StreamController<int>.broadcast(); |
|||
final _micActivityCtl = StreamController<void>.broadcast(); |
|||
|
|||
StreamSubscription? _sub; |
|||
bool _running = false; |
|||
|
|||
/// 本次会话解出来的指令记到哪个来源名下（与 Dart 版同义） |
|||
String directiveSource = 'earphone'; |
|||
|
|||
/// 业务事件流。与 [BailianMultimodalService.events] 同一套 [BailianEvent]， |
|||
/// 所以 `DeviceAiSessionService` 的事件处理一行都不用改。 |
|||
Stream<BailianEvent> get events => _eventCtl.stream; |
|||
|
|||
/// 下行已写入的**累计**字节数（约 100ms 一条）。 |
|||
/// |
|||
/// 原生直连之后 Dart 拿不到下行 PCM，但它要靠「首包到达」开门控、靠累计字节数 |
|||
/// 估算播完时刻。这两件事都只依赖长度，所以只报数字。 |
|||
Stream<int> get downstreamBytes => _downBytesCtl.stream; |
|||
|
|||
/// 设备上行有声（原生按 500ms 节流上报）。 |
|||
/// |
|||
/// ⚠️ **不能把它当成 `speechStarted`**：那个事件在会话服务里会丢弃待播队列、 |
|||
/// 取消播完计时（「真人插话」的语义）。这里只是给空闲计时续命， |
|||
/// 混在一起会让 AI 每说一句就被自己的回声打断一次。 |
|||
Stream<void> get micActivity => _micActivityCtl.stream; |
|||
|
|||
bool get isRunning => _running; |
|||
|
|||
bool get isConfigured => |
|||
BailianMultimodalService.resolveApiKey().trim().isNotEmpty; |
|||
|
|||
String get configHint => isConfigured ? '' : '未配置 ALIBABA_OPENSPEECH_APP_KEY'; |
|||
|
|||
/// 这台机器/这个包能不能用原生链路。 |
|||
/// |
|||
/// 老包没有这个方法通道 → `MissingPluginException` → 返回 false，调用方退回 Dart。 |
|||
/// Web 直接排除（压根没有原生插件）。 |
|||
static Future<bool> isAvailable() async { |
|||
if (kIsWeb || !(Platform.isIOS || Platform.isAndroid)) return false; |
|||
try { |
|||
await _channel.invokeMethod<Map>('stats'); |
|||
return true; |
|||
} on MissingPluginException { |
|||
return false; |
|||
} catch (e) { |
|||
Logger.w(_tag, '原生百炼客户端探测失败（退回 Dart）: $e'); |
|||
return false; |
|||
} |
|||
} |
|||
|
|||
/// 建立会话。参数与 [BailianMultimodalService.start] 一一对应。 |
|||
Future<bool> start({ |
|||
String mode = 'duplex', |
|||
String? voice, |
|||
String? userId, |
|||
bool enableWebSearch = true, |
|||
int downstreamRate = 16000, |
|||
String source = 'earphone', |
|||
String? mcpToken, |
|||
}) async { |
|||
if (!isConfigured) { |
|||
Logger.e(_tag, '配置不全，无法启动会话：$configHint'); |
|||
_eventCtl.add(BailianEvent(BailianEventType.error, |
|||
text: configHint, code: 'NOT_CONFIGURED', fatal: true)); |
|||
return false; |
|||
} |
|||
await stop(); |
|||
directiveSource = source; |
|||
_sub = _events.receiveBroadcastStream().listen( |
|||
_onNativeEvent, |
|||
onError: (e) { |
|||
Logger.e(_tag, '原生事件流错误: $e'); |
|||
_eventCtl.add( |
|||
BailianEvent(BailianEventType.error, text: '$e', fatal: true)); |
|||
}, |
|||
); |
|||
try { |
|||
final ok = await _channel.invokeMethod<bool>('start', { |
|||
'apiKey': BailianMultimodalService.resolveApiKey(), |
|||
'appId': BailianMultimodalService.resolveAppId(_appIdOverride), |
|||
'workspaceId': |
|||
BailianMultimodalService.resolveWorkspaceId(_workspaceIdOverride), |
|||
'mode': mode, |
|||
if (voice != null && voice.trim().isNotEmpty) 'voice': voice, |
|||
'userId': userId ?? '', |
|||
'enableWebSearch': enableWebSearch, |
|||
'downstreamRate': downstreamRate, |
|||
if (mcpToken != null && mcpToken.trim().isNotEmpty) 'mcpToken': mcpToken, |
|||
// 工具名只在 Dart 维护一份：服务端加工具时改那一处就够了 |
|||
'mcpAuthTools': BailianMultimodalService.mcpAuthTools, |
|||
}); |
|||
_running = ok == true; |
|||
if (!_running) await _cancelSub(); |
|||
return _running; |
|||
} on MissingPluginException { |
|||
Logger.w(_tag, '原生不支持，退回 Dart 链路'); |
|||
await _cancelSub(); |
|||
return false; |
|||
} catch (e) { |
|||
Logger.e(_tag, '启动原生会话失败: $e'); |
|||
await _cancelSub(); |
|||
return false; |
|||
} |
|||
} |
|||
|
|||
Future<void> stop() async { |
|||
if (!_running) { |
|||
await _cancelSub(); |
|||
return; |
|||
} |
|||
_running = false; |
|||
try { |
|||
await _channel.invokeMethod<void>('stop'); |
|||
} catch (e) { |
|||
Logger.w(_tag, '停止原生会话失败: $e'); |
|||
} |
|||
await _cancelSub(); |
|||
} |
|||
|
|||
/// AI 是否正在耳机里播报。播报期间原生把上行换成等长静音。 |
|||
/// |
|||
/// **什么时候开关门由 Dart 决定**（`DeviceAiSessionService` 里那套带兜底计时、 |
|||
/// 非致命错误也要关门的逻辑），这里只是把结论推下去。 |
|||
Future<void> setGated(bool gated) => _invoke('setGated', {'gated': gated}); |
|||
|
|||
Future<void> notifyLocalRespondingStarted() => |
|||
_invoke('localRespondingStarted'); |
|||
|
|||
Future<void> notifyLocalRespondingEnded() => _invoke('localRespondingEnded'); |
|||
|
|||
Future<void> interrupt() => _invoke('interrupt'); |
|||
|
|||
Future<void> beginSpeech() => _invoke('beginSpeech'); |
|||
|
|||
Future<void> endSpeech() => _invoke('endSpeech'); |
|||
|
|||
Future<void> cancelSpeech() => _invoke('cancelSpeech'); |
|||
|
|||
Future<void> sendText(String text, {bool speakDirectly = false}) => |
|||
_invoke('sendText', {'text': text, 'speakDirectly': speakDirectly}); |
|||
|
|||
/// 诊断：上/下行字节、出包、丢包、门控状态。桥开着却没声音时先问它。 |
|||
Future<Map<String, dynamic>> stats() async { |
|||
try { |
|||
final raw = await _channel.invokeMethod<Map>('stats'); |
|||
return raw == null ? const {} : Map<String, dynamic>.from(raw); |
|||
} catch (e) { |
|||
return {'error': '$e'}; |
|||
} |
|||
} |
|||
|
|||
Future<void> _invoke(String method, [Map<String, dynamic>? args]) async { |
|||
if (!_running) return; |
|||
try { |
|||
await _channel.invokeMethod<void>(method, args); |
|||
} catch (e) { |
|||
Logger.w(_tag, '$method 失败: $e'); |
|||
} |
|||
} |
|||
|
|||
Future<void> _cancelSub() async { |
|||
await _sub?.cancel(); |
|||
_sub = null; |
|||
} |
|||
|
|||
// ---------- 事件翻译 ---------- |
|||
|
|||
void _onNativeEvent(dynamic raw) { |
|||
if (raw is! Map) return; |
|||
final name = raw['event']?.toString() ?? ''; |
|||
final text = raw['text']?.toString() ?? ''; |
|||
final isFinal = raw['isFinal'] == true; |
|||
|
|||
switch (name) { |
|||
case 'started': |
|||
_emit(const BailianEvent(BailianEventType.started)); |
|||
break; |
|||
case 'stateChanged': |
|||
final s = raw['state']?.toString() ?? ''; |
|||
_emit(BailianEvent(BailianEventType.stateChanged, state: _mapState(s))); |
|||
break; |
|||
case 'speechStarted': |
|||
_emit(const BailianEvent(BailianEventType.speechStarted)); |
|||
break; |
|||
case 'speechContent': |
|||
_emit(BailianEvent(BailianEventType.speechContent, |
|||
text: text, isFinal: isFinal)); |
|||
break; |
|||
case 'speechEnded': |
|||
_emit(BailianEvent(BailianEventType.speechEnded, |
|||
text: text, isFinal: true)); |
|||
break; |
|||
case 'requestAccepted': |
|||
_emit(const BailianEvent(BailianEventType.requestAccepted)); |
|||
break; |
|||
case 'respondingStarted': |
|||
_emit(const BailianEvent(BailianEventType.respondingStarted)); |
|||
break; |
|||
case 'respondingContent': |
|||
_emit(BailianEvent( |
|||
BailianEventType.respondingContent, |
|||
text: text, |
|||
isFinal: isFinal, |
|||
finishReason: raw['finishReason']?.toString() ?? '', |
|||
directives: _parseDirectives(raw['extraInfo']?.toString(), text), |
|||
)); |
|||
break; |
|||
case 'respondingEnded': |
|||
_emit(BailianEvent(BailianEventType.respondingEnded, |
|||
text: text, isFinal: true)); |
|||
break; |
|||
case 'error': |
|||
_emit(BailianEvent(BailianEventType.error, |
|||
text: text, |
|||
code: raw['code']?.toString(), |
|||
fatal: raw['fatal'] == true)); |
|||
break; |
|||
case 'closed': |
|||
_running = false; |
|||
_emit(const BailianEvent(BailianEventType.closed)); |
|||
break; |
|||
case 'downlink': |
|||
final n = (raw['bytes'] as num?)?.toInt() ?? 0; |
|||
if (!_downBytesCtl.isClosed) _downBytesCtl.add(n); |
|||
break; |
|||
case 'micActivity': |
|||
// 只走自己的流，不冒充任何协议事件（见 micActivity 的说明） |
|||
if (!_micActivityCtl.isClosed) _micActivityCtl.add(null); |
|||
break; |
|||
case 'overflow': |
|||
// 上行拥塞丢包。warn 级：这是「说了没反应」时唯一的证据 |
|||
Logger.w(_tag, '[AI-BRIDGE] 上行拥塞，原生累计丢包=${raw['dropped']}'); |
|||
break; |
|||
default: |
|||
Logger.d(_tag, '未处理的原生事件: $name'); |
|||
} |
|||
} |
|||
|
|||
/// 指令解析**仍然走 Dart 这一份**：`extra_info.tool_calls` 的那些坑 |
|||
/// （参数是字符串套 JSON、值恒为中文、下午时间要校正）有一批真实 payload 的 |
|||
/// 金测试守着，原生再写一遍只会让那批测试失去意义。原生把 `extra_info` |
|||
/// 原样转成字符串带上来，这里拼回 output 的形状交给同一个解析器。 |
|||
List<AssistantDirective> _parseDirectives(String? extraInfoJson, String text) { |
|||
if (extraInfoJson == null || extraInfoJson.isEmpty) return const []; |
|||
try { |
|||
final extra = json.decode(extraInfoJson); |
|||
final parsed = AssistantDirective.parseFromOutput( |
|||
{'extra_info': extra, 'text': text}, |
|||
source: directiveSource, |
|||
); |
|||
if (parsed.isNotEmpty) { |
|||
// warn 级、且只在真的带了指令时打——「闹钟没定上」只有两种可能 |
|||
// （百炼没下发 / 端侧没落地），这一行是把两者分开的证据 |
|||
Logger.w(_tag, '[指令帧] 原生链路解析出 ${parsed.length} 条'); |
|||
} |
|||
return parsed; |
|||
} catch (e) { |
|||
Logger.w(_tag, 'extra_info 解析失败: $e'); |
|||
return const []; |
|||
} |
|||
} |
|||
|
|||
static BailianDialogState _mapState(String s) => switch (s) { |
|||
'listening' => BailianDialogState.listening, |
|||
'thinking' => BailianDialogState.thinking, |
|||
'responding' => BailianDialogState.responding, |
|||
_ => BailianDialogState.idle, |
|||
}; |
|||
|
|||
void _emit(BailianEvent e) { |
|||
if (!_eventCtl.isClosed) _eventCtl.add(e); |
|||
} |
|||
|
|||
Future<void> dispose() async { |
|||
await stop(); |
|||
await _eventCtl.close(); |
|||
await _downBytesCtl.close(); |
|||
await _micActivityCtl.close(); |
|||
} |
|||
} |
|||
@ -0,0 +1,92 @@ |
|||
import 'dart:async' show unawaited; |
|||
import 'dart:io' show Platform; |
|||
|
|||
import 'package:audio_session/audio_session.dart'; |
|||
import 'package:flutter/foundation.dart' show kIsWeb; |
|||
import 'package:just_audio/just_audio.dart'; |
|||
|
|||
import '../../core/utils/logger.dart'; |
|||
|
|||
/// iOS 后台保活：循环播一段**静音**，让 `UIBackgroundModes=audio` 有个成立的理由。 |
|||
/// |
|||
/// ## 为什么需要它 |
|||
/// 设备按键唤醒的 AI 会话，音频全走耳机（BLE），手机侧一个 AVAudioSession 都没激活； |
|||
/// 而 Info.plist 里只有 `bluetooth-central / audio / location`——光靠 bluetooth-central， |
|||
/// 锁屏后系统只给短暂的唤醒窗口，撑不住一段几十秒、每秒 50 包上行 + 一条 WebSocket |
|||
/// 的会话。Android 有 `AudioForegroundService`，iOS 没有等价物，这是 iOS 上让进程 |
|||
/// 在锁屏后活下去的标准做法（2026-09-20 苹果测试「锁屏后耳机对话很容易中断」）。 |
|||
/// |
|||
/// - 用引用计数：多条链路同时要保活时，最后一个 release 才真的停； |
|||
/// - 类别 `playback` + `mixWithOthers`：不打断别的 App 的声音，也不抢路由； |
|||
/// - 音量 0 + 文件本身就是静音，用户听不到任何东西； |
|||
/// - Android / Web 全是空操作。 |
|||
class BackgroundKeepalive { |
|||
BackgroundKeepalive._(); |
|||
|
|||
static const String _tag = 'BackgroundKeepalive'; |
|||
static const String _asset = 'assets/audio/silence_1s.wav'; |
|||
|
|||
static AudioPlayer? _player; |
|||
static int _refs = 0; |
|||
|
|||
static bool get _apply => !kIsWeb && Platform.isIOS; |
|||
|
|||
static Future<void> acquire(String owner) async { |
|||
if (!_apply) return; |
|||
_refs++; |
|||
if (_refs > 1) return; |
|||
try { |
|||
final session = await AudioSession.instance; |
|||
// ⚠️ **只在当前类别撑不住后台音频时才改它**。 |
|||
// |
|||
// 无条件 configure 成 .playback 会把通话翻译/多媒体翻译正在跑的 |
|||
// .playAndRecord 录音链路踩掉（同 SubtitlePip 的 prepareAudioSessionForPip， |
|||
// 那边栽过一次）。而 .playAndRecord / .multiRoute 本身就满足 |
|||
// UIBackgroundModes=audio 的要求，不需要降级——这时只要把静音循环播起来， |
|||
// 让「确实有音频在播」这个条件成立就够了。 |
|||
final cur = session.configuration?.avAudioSessionCategory; |
|||
final alreadyOk = cur == AVAudioSessionCategory.playback || |
|||
cur == AVAudioSessionCategory.playAndRecord || |
|||
cur == AVAudioSessionCategory.multiRoute; |
|||
if (!alreadyOk) { |
|||
await session.configure(const AudioSessionConfiguration( |
|||
avAudioSessionCategory: AVAudioSessionCategory.playback, |
|||
avAudioSessionCategoryOptions: |
|||
AVAudioSessionCategoryOptions.mixWithOthers, |
|||
avAudioSessionMode: AVAudioSessionMode.defaultMode, |
|||
)); |
|||
} else { |
|||
Logger.i(_tag, '当前音频类别 $cur 已支持后台音频，不改类别'); |
|||
} |
|||
await session.setActive(true); |
|||
final p = AudioPlayer(); |
|||
await p.setAsset(_asset); |
|||
await p.setLoopMode(LoopMode.one); |
|||
await p.setVolume(0); |
|||
unawaited(p.play()); |
|||
_player = p; |
|||
Logger.i(_tag, '后台保活已开启（$owner）'); |
|||
} catch (e) { |
|||
// 起不来不阻断会话本身：只是锁屏后可能被挂起，前台使用不受影响 |
|||
Logger.w(_tag, '后台保活开启失败（$owner）: $e'); |
|||
_refs--; |
|||
} |
|||
} |
|||
|
|||
static Future<void> release(String owner) async { |
|||
if (!_apply) return; |
|||
if (_refs == 0) return; |
|||
_refs--; |
|||
if (_refs > 0) return; |
|||
final p = _player; |
|||
_player = null; |
|||
if (p == null) return; |
|||
try { |
|||
await p.stop(); |
|||
await p.dispose(); |
|||
Logger.i(_tag, '后台保活已关闭（$owner）'); |
|||
} catch (e) { |
|||
Logger.w(_tag, '后台保活关闭失败: $e'); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,442 @@ |
|||
package com.yunqiinnovation.azure_speech |
|||
|
|||
import android.os.Handler |
|||
import android.os.Looper |
|||
import android.util.Log |
|||
import okhttp3.OkHttpClient |
|||
import okhttp3.Request |
|||
import okhttp3.Response |
|||
import okhttp3.WebSocket |
|||
import okhttp3.WebSocketListener |
|||
import okio.ByteString |
|||
import org.json.JSONArray |
|||
import org.json.JSONObject |
|||
import java.util.UUID |
|||
import java.util.concurrent.Executors |
|||
import java.util.concurrent.TimeUnit |
|||
|
|||
/** |
|||
* 阿里云百炼「多模态交互协议」的**原生**客户端（EMAI 助手）。 |
|||
* |
|||
* 协议文档：https://help.aliyun.com/zh/model-studio/multimodal-interaction-protocol |
|||
* 端点：`wss://dashscope.aliyuncs.com/api-ws/v1/inference` |
|||
* |
|||
* ## 为什么有原生这一份 |
|||
* Dart 侧的 `BailianMultimodalService` 仍在（手机麦那条链路、以及本类起不来时的退路）。 |
|||
* 耳机按键唤醒那条链路原先是 |
|||
* `bluetooth_manager → EventChannel → Dart → WS(Dart) → Dart → MethodChannel → bluetooth_manager`， |
|||
* 上行 20ms 一帧、下行每个音频块一次，全压在 Dart 的单 isolate 上。本类把 WS 连同音频 |
|||
* 收发搬到原生：PCM 经 [BesAiPcmBridge] 在原生之间直连，**Dart 只收文本与状态事件**。 |
|||
* |
|||
* ## ⚠️ 这不修「锁屏断连」 |
|||
* 进程被系统挂起时原生和 Dart 一起停。锁屏能不能活着取决于前台服务 / 后台音频， |
|||
* 与代码在哪一层无关。本类解决的是 event loop 抖动与跨界开销。 |
|||
* |
|||
* ## ⚠️ 与 Dart 版、iOS 版**逐条对齐** |
|||
* 握手参数、事件名映射、心跳间隔照 `lib/data/services/bailian_multimodal_service.dart` |
|||
* 与 `BailianDialogClient.swift` 写。改一处必须同改另两处，否则「用耳机唤醒」和 |
|||
* 「在 App 里按住说话」会变成两种行为，而且都不报错。 |
|||
* |
|||
* ## 指令解析刻意**留在 Dart** |
|||
* `extra_info.tool_calls` 的解析有一批真实 payload 的金测试守着 |
|||
* （`test/assistant_directive_test.dart`），解析结果又要落进 GetStorage 和「代办」页。 |
|||
* 在原生解析再序列化回 Dart 毫无收益，只会把那批测试作废。这里把 `extra_info` |
|||
* 原样转成字符串带上去，Dart 照旧解析。 |
|||
*/ |
|||
class BailianDialogClient( |
|||
private val onEvent: (Map<String, Any?>) -> Unit, |
|||
private val onAudio: (ByteArray) -> Unit, |
|||
) { |
|||
companion object { |
|||
private const val TAG = "BailianDialog" |
|||
private const val ENDPOINT = "wss://dashscope.aliyuncs.com/api-ws/v1/inference" |
|||
|
|||
/** 上行音频：16k / 16bit / 单声道 / 小端 */ |
|||
private const val UPSTREAM_SAMPLE_RATE = 16000 |
|||
|
|||
/** 服务端要求任意连续 60s 内必须有消息往来；文档建议 50s 一次心跳 */ |
|||
private const val HEARTBEAT_INTERVAL_MS = 50_000L |
|||
} |
|||
|
|||
data class Config( |
|||
val apiKey: String, |
|||
val appId: String, |
|||
val workspaceId: String, |
|||
val mode: String, |
|||
val voice: String?, |
|||
val userId: String, |
|||
val enableWebSearch: Boolean, |
|||
val downstreamRate: Int, |
|||
val mcpToken: String?, |
|||
/** |
|||
* 需要用户身份的 MCP 工具名。**由 Dart 传下来**，不在原生再抄一份： |
|||
* 服务端加工具时只改 Dart 那一处，漏改的表现是那个工具永远「身份校验失败」。 |
|||
*/ |
|||
val mcpAuthTools: List<String>, |
|||
) |
|||
|
|||
private val exec = Executors.newSingleThreadExecutor { r -> Thread(r, "azure_speech.bailian") } |
|||
private val main = Handler(Looper.getMainLooper()) |
|||
|
|||
private var http: OkHttpClient? = null |
|||
private var ws: WebSocket? = null |
|||
private var config: Config? = null |
|||
private var taskId = "" |
|||
private var dialogId = "" |
|||
|
|||
@Volatile private var running = false |
|||
private var heartbeat: Runnable? = null |
|||
|
|||
private var uplinkBytes = 0L |
|||
private var downlinkBytes = 0L |
|||
|
|||
/** |
|||
* 下行字节数的上报节流。Dart 侧不再拿到 PCM，但它要靠「首包到达」开门控、 |
|||
* 靠累计字节数估算播完时刻（门控的收尾计时），所以这两件事必须报上去。 |
|||
* 按 100ms 汇总一次，别每块都报——那就把省下来的跨界又加回去了。 |
|||
*/ |
|||
private var lastDownlinkReportAt = 0L |
|||
|
|||
val isRunning: Boolean get() = running |
|||
|
|||
// ---------- 生命周期 ---------- |
|||
|
|||
/** 建立会话。返回 false 表示配置不全。连接是异步的，失败经 onEvent 上报。 */ |
|||
fun start(c: Config): Boolean { |
|||
if (c.apiKey.isBlank()) { |
|||
Log.e(TAG, "未配置 API Key，不尝试连接") |
|||
// Key 缺失时不要尝试连接——服务端会先回 task-started 再立刻 task-failed， |
|||
// 日志上看起来像"连上了又掉"，容易误判成网络问题 |
|||
emit(mapOf("event" to "error", "text" to "未配置 ALIBABA_OPENSPEECH_APP_KEY", |
|||
"code" to "NOT_CONFIGURED", "fatal" to true)) |
|||
return false |
|||
} |
|||
stop() |
|||
config = c |
|||
taskId = UUID.randomUUID().toString() |
|||
dialogId = "" |
|||
uplinkBytes = 0L |
|||
downlinkBytes = 0L |
|||
lastDownlinkReportAt = 0L |
|||
|
|||
val client = OkHttpClient.Builder() |
|||
.pingInterval(20, TimeUnit.SECONDS) // 与 Dart 的 pingInterval 对齐 |
|||
.readTimeout(0, TimeUnit.MILLISECONDS) |
|||
.build() |
|||
http = client |
|||
val req = Request.Builder() |
|||
.url(ENDPOINT) |
|||
.addHeader("Authorization", "Bearer ${c.apiKey}") |
|||
.addHeader("X-DashScope-DataInspection", "enable") |
|||
.build() |
|||
running = true |
|||
ws = client.newWebSocket(req, Listener()) |
|||
sendJson(startPayload(c)) |
|||
startHeartbeat() |
|||
Log.i(TAG, "会话启动 app_id=${c.appId} workspace=${c.workspaceId} mode=${c.mode} downstream=${c.downstreamRate}Hz") |
|||
return true |
|||
} |
|||
|
|||
/** 结束会话：先发 Stop 让服务端收尾，再 finish-task 结束整个 WS 任务 */ |
|||
fun stop() { |
|||
if (!running) { |
|||
cleanup() |
|||
return |
|||
} |
|||
try { |
|||
sendDirective("Stop") |
|||
sendJson( |
|||
JSONObject() |
|||
.put("header", JSONObject() |
|||
.put("action", "finish-task").put("task_id", taskId).put("streaming", "duplex")) |
|||
.put("payload", JSONObject().put("input", JSONObject())) |
|||
) |
|||
} catch (_: Exception) { |
|||
} |
|||
Log.i(TAG, "会话结束 上行=$uplinkBytes 字节 下行=$downlinkBytes 字节") |
|||
try { |
|||
ws?.close(1000, null) |
|||
} catch (_: Exception) { |
|||
} |
|||
cleanup() |
|||
} |
|||
|
|||
// ---------- 上行 ---------- |
|||
|
|||
/** 推一包上行音频（PCM 16k/16bit/mono）。由 [BesAiPcmBridge] 的串行执行器调用。 */ |
|||
fun pushAudio(pcm: ByteArray) { |
|||
if (pcm.isEmpty() || !running) return |
|||
exec.execute { |
|||
val w = ws ?: return@execute |
|||
uplinkBytes += pcm.size |
|||
if (!w.send(ByteString.of(*pcm))) { |
|||
Log.w(TAG, "推送音频失败（发送队列已满）") |
|||
} |
|||
} |
|||
} |
|||
|
|||
fun beginSpeech() = sendDirective("SendSpeech") |
|||
fun endSpeech() = sendDirective("StopSpeech") |
|||
fun cancelSpeech() = sendDirective("CancelSpeech") |
|||
fun interrupt() = sendDirective("RequestToSpeak") |
|||
fun notifyLocalRespondingStarted() = sendDirective("LocalRespondingStarted") |
|||
fun notifyLocalRespondingEnded() = sendDirective("LocalRespondingEnded") |
|||
|
|||
/** 文本对话：交给应用的 LLM 处理（prompt），或让它直接朗读（transcript） */ |
|||
fun sendText(text: String, speakDirectly: Boolean) { |
|||
val t = text.trim() |
|||
if (t.isEmpty()) return |
|||
sendDirective("RequestToRespond", JSONObject() |
|||
.put("type", if (speakDirectly) "transcript" else "prompt") |
|||
.put("text", t)) |
|||
} |
|||
|
|||
fun stats(): Map<String, Any> = mapOf( |
|||
"running" to running, |
|||
"uplinkBytes" to uplinkBytes, |
|||
"downlinkBytes" to downlinkBytes, |
|||
"dialogId" to dialogId, |
|||
) |
|||
|
|||
// ---------- 协议帧 ---------- |
|||
|
|||
private fun startPayload(c: Config): JSONObject { |
|||
val downstream = JSONObject() |
|||
.put("audio_format", "pcm") |
|||
.put("sample_rate", c.downstreamRate) |
|||
// transcript：中间态只回朗读文本，跟 TTS 出声一致， |
|||
// 免得 UI 上先闪一段 dialog 思考态文本再被覆盖 |
|||
.put("intermediate_text", "transcript") |
|||
// 显式钉死"每帧回全文"：Dart 收到 RespondingContent 是整体替换气泡内容的 |
|||
.put("incremental_response", false) |
|||
// 不传 voice：音色属于百炼应用的配置，硬编码只会跟应用实际用的 TTS 模型对不上 |
|||
if (!c.voice.isNullOrBlank()) downstream.put("voice", c.voice) |
|||
|
|||
val parameters = JSONObject() |
|||
.put("upstream", JSONObject() |
|||
.put("type", "AudioOnly") |
|||
.put("mode", c.mode) |
|||
.put("audio_format", "pcm") |
|||
.put("sample_rate", UPSTREAM_SAMPLE_RATE)) |
|||
.put("downstream", downstream) |
|||
.put("client_info", JSONObject().put("user_id", safeUserId(c.userId))) |
|||
|
|||
// biz_params 是可选块，两件事都没有时整块不发——万一应用侧没开这些能力， |
|||
// 多带一个字段可能让 Start 直接被拒，那样整个功能都起不来 |
|||
val hasToken = !c.mcpToken.isNullOrBlank() |
|||
if (c.enableWebSearch || hasToken) { |
|||
val userDefined = JSONObject() |
|||
if (hasToken) { |
|||
// ⚠️ **必须按工具名嵌套**，平铺写 auth_token 会被静默丢弃 |
|||
for (tool in c.mcpAuthTools) { |
|||
userDefined.put(tool, JSONObject().put("auth_token", c.mcpToken)) |
|||
} |
|||
} |
|||
if (c.enableWebSearch) { |
|||
userDefined.put("extra_config", JSONObject().put("enable_web_search", true)) |
|||
} |
|||
parameters.put("biz_params", JSONObject().put("user_defined_params", userDefined)) |
|||
} |
|||
|
|||
return JSONObject() |
|||
.put("header", JSONObject() |
|||
.put("action", "run-task").put("task_id", taskId).put("streaming", "duplex")) |
|||
.put("payload", JSONObject() |
|||
.put("task_group", "aigc") |
|||
.put("task", "multimodal-generation") |
|||
.put("function", "generation") |
|||
.put("model", "multimodal-dialog") |
|||
.put("input", JSONObject() |
|||
.put("directive", "Start") |
|||
.put("workspace_id", c.workspaceId) |
|||
.put("app_id", c.appId)) |
|||
.put("parameters", parameters)) |
|||
} |
|||
|
|||
private fun sendDirective(directive: String, extra: JSONObject? = null) { |
|||
val c = config ?: return |
|||
if (!running) return |
|||
val input = JSONObject() |
|||
.put("directive", directive) |
|||
.put("workspace_id", c.workspaceId) |
|||
.put("app_id", c.appId) |
|||
if (dialogId.isNotEmpty()) input.put("dialog_id", dialogId) |
|||
extra?.keys()?.forEach { k -> input.put(k, extra.get(k)) } |
|||
sendJson(JSONObject() |
|||
.put("header", JSONObject() |
|||
.put("action", "continue-task").put("task_id", taskId).put("streaming", "duplex")) |
|||
.put("payload", JSONObject() |
|||
.put("task_group", "aigc") |
|||
.put("task", "multimodal-generation") |
|||
.put("function", "generation") |
|||
.put("model", "multimodal-dialog") |
|||
.put("input", input))) |
|||
} |
|||
|
|||
private fun sendJson(msg: JSONObject) { |
|||
exec.execute { |
|||
val w = ws ?: return@execute |
|||
try { |
|||
w.send(msg.toString()) |
|||
} catch (e: Exception) { |
|||
Log.w(TAG, "发送失败: ${e.message}") |
|||
} |
|||
} |
|||
} |
|||
|
|||
// ---------- 下行 ---------- |
|||
|
|||
private inner class Listener : WebSocketListener() { |
|||
override fun onMessage(webSocket: WebSocket, bytes: ByteString) { |
|||
// 二进制帧就是下行合成音频 |
|||
val b = bytes.toByteArray() |
|||
downlinkBytes += b.size |
|||
val now = System.currentTimeMillis() |
|||
// 首块立刻报（Dart 要用它开门控），之后按间隔汇总 |
|||
if (lastDownlinkReportAt == 0L || now - lastDownlinkReportAt >= 100L) { |
|||
lastDownlinkReportAt = now |
|||
emit(mapOf("event" to "downlink", "bytes" to downlinkBytes)) |
|||
} |
|||
if (b.isNotEmpty()) onAudio(b) |
|||
} |
|||
|
|||
override fun onMessage(webSocket: WebSocket, text: String) { |
|||
handleText(text) |
|||
} |
|||
|
|||
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { |
|||
if (!running) return |
|||
Log.e(TAG, "WebSocket 错误: ${t.message}") |
|||
emit(mapOf("event" to "error", "text" to (t.message ?: "WebSocket error"), "fatal" to true)) |
|||
cleanup() |
|||
} |
|||
|
|||
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { |
|||
Log.i(TAG, "WebSocket 关闭 code=$code") |
|||
emit(mapOf("event" to "closed")) |
|||
cleanup() |
|||
} |
|||
} |
|||
|
|||
private fun handleText(raw: String) { |
|||
val obj = try { |
|||
JSONObject(raw) |
|||
} catch (e: Exception) { |
|||
Log.w(TAG, "无法解析的文本帧: ${e.message}") |
|||
return |
|||
} |
|||
val header = obj.optJSONObject("header") ?: JSONObject() |
|||
val payload = obj.optJSONObject("payload") ?: JSONObject() |
|||
when (val event = header.optString("event", "")) { |
|||
"task-started" -> Log.i(TAG, "task-started") |
|||
"task-failed" -> { |
|||
val code = header.optString("error_code", "") |
|||
val msg = header.optString("error_message", "") |
|||
Log.e(TAG, "task-failed: $code $msg") |
|||
// header 级 = 整个 task 结束，必须重连 |
|||
emit(mapOf("event" to "error", "text" to msg, "code" to code, "fatal" to true)) |
|||
} |
|||
"task-finished" -> { |
|||
Log.i(TAG, "task-finished") |
|||
emit(mapOf("event" to "closed")) |
|||
} |
|||
else -> handleDirectiveEvent(event, payload) |
|||
} |
|||
} |
|||
|
|||
/** 业务事件都在 payload.output.event 里 */ |
|||
private fun handleDirectiveEvent(outerEvent: String, payload: JSONObject) { |
|||
val output = payload.optJSONObject("output") ?: JSONObject() |
|||
val name = output.optString("event", outerEvent) |
|||
val text = output.optString("text", "") |
|||
val finished = output.optBoolean("finished", false) |
|||
|
|||
when (name) { |
|||
"Started" -> { |
|||
dialogId = output.optString("dialog_id", "") |
|||
Log.i(TAG, "Started dialog_id=$dialogId") |
|||
emit(mapOf("event" to "started")) |
|||
} |
|||
"DialogStateChanged" -> { |
|||
// 协议里是首字母大写（Listening/Thinking/Responding），统一小写再交给 Dart |
|||
emit(mapOf("event" to "stateChanged", |
|||
"state" to output.optString("state", "").lowercase())) |
|||
} |
|||
"SpeechStarted" -> emit(mapOf("event" to "speechStarted")) |
|||
"SpeechContent" -> emit(mapOf("event" to "speechContent", "text" to text, "isFinal" to finished)) |
|||
"SpeechEnded" -> emit(mapOf("event" to "speechEnded", "text" to text, "isFinal" to true)) |
|||
"RequestAccepted" -> emit(mapOf("event" to "requestAccepted")) |
|||
"RespondingStarted" -> emit(mapOf("event" to "respondingStarted")) |
|||
"RespondingContent" -> { |
|||
val reason = output.optString("finish_reason", "") |
|||
// 指令解析留在 Dart（见类注释）：把 extra_info 原样带上去。 |
|||
// 它只在整轮收尾那一帧出现，不是每个流式分片都有，频率可控。 |
|||
val e = mutableMapOf<String, Any?>( |
|||
"event" to "respondingContent", |
|||
"text" to text, |
|||
"isFinal" to finished, |
|||
"finishReason" to reason, |
|||
) |
|||
val extra = output.opt("extra_info") |
|||
if (extra is JSONObject || extra is JSONArray) e["extraInfo"] = extra.toString() |
|||
emit(e) |
|||
} |
|||
"RespondingEnded" -> emit(mapOf("event" to "respondingEnded", "text" to text, "isFinal" to true)) |
|||
"Error" -> { |
|||
val code = output.optString("error_code", "") |
|||
val msg = output.optString("error_message", "") |
|||
val name0 = output.optString("error_name", "") |
|||
Log.e(TAG, "Error: $code/$name0 $msg") |
|||
// payload 级 Error 是"这一轮不行"，连接通常还在，不标 fatal |
|||
val shown = if (msg.isNotEmpty()) msg else if (text.isNotEmpty()) text else name0 |
|||
emit(mapOf("event" to "error", "text" to shown, "code" to code, "fatal" to false)) |
|||
} |
|||
"Stopped" -> emit(mapOf("event" to "closed")) |
|||
"HeartBeat" -> Unit |
|||
else -> Log.d(TAG, "未处理事件: $name") |
|||
} |
|||
} |
|||
|
|||
// ---------- 其它 ---------- |
|||
|
|||
private fun startHeartbeat() { |
|||
stopHeartbeat() |
|||
val r = object : Runnable { |
|||
override fun run() { |
|||
if (!running) return |
|||
sendDirective("HeartBeat") |
|||
main.postDelayed(this, HEARTBEAT_INTERVAL_MS) |
|||
} |
|||
} |
|||
heartbeat = r |
|||
main.postDelayed(r, HEARTBEAT_INTERVAL_MS) |
|||
} |
|||
|
|||
private fun stopHeartbeat() { |
|||
heartbeat?.let { main.removeCallbacks(it) } |
|||
heartbeat = null |
|||
} |
|||
|
|||
private fun cleanup() { |
|||
stopHeartbeat() |
|||
running = false |
|||
ws = null |
|||
dialogId = "" |
|||
try { |
|||
http?.dispatcher?.executorService?.shutdown() |
|||
} catch (_: Exception) { |
|||
} |
|||
http = null |
|||
} |
|||
|
|||
private fun emit(e: Map<String, Any?>) { |
|||
main.post { onEvent(e) } |
|||
} |
|||
|
|||
/** 协议限制 user_id 不超过 36 字符 */ |
|||
private fun safeUserId(raw: String): String { |
|||
val v = raw.trim() |
|||
if (v.isEmpty()) return "eaimar-guest" |
|||
return if (v.length <= 36) v else v.substring(0, 36) |
|||
} |
|||
} |
|||
@ -0,0 +1,182 @@ |
|||
package com.yunqiinnovation.azure_speech |
|||
|
|||
import android.util.Log |
|||
import com.example.bluetooth_manager.AiAudioDownlink |
|||
import com.example.bluetooth_manager.BluetoothManager |
|||
import java.util.concurrent.Executors |
|||
import java.util.concurrent.atomic.AtomicInteger |
|||
import kotlin.math.sqrt |
|||
|
|||
/** |
|||
* EMAI 会话 PCM 的**原生直连桥**（azure_speech 这一端）。 |
|||
* |
|||
* 对端是 bluetooth_manager 的 `BluetoothManager.aiPcmTap`：耳机解码出来的 AI 上行 PCM 直接 |
|||
* 到这里（不再经过 Dart），攒成 100ms 一包推给 [BailianDialogClient]；百炼合成回来的 PCM 由 |
|||
* [postDownlink] 直接喂给 [AiAudioDownlink]。 |
|||
* |
|||
* 本模块 `compileOnly project(":bluetooth_manager")`，运行时两者都在 APK 里。 |
|||
* **与 iOS 端 BesAiPcmBridge.swift 语义逐条对齐**，改一端要同改另一端。 |
|||
* |
|||
* ## 门控为什么是从 Dart 推下来的布尔量 |
|||
* 「AI 在耳机里播报时把上行换成等长静音」这件事本身必须在这里做（逐帧）；但**什么时候 |
|||
* 开关门**留在 Dart 的 `DeviceAiSessionService`：那套逻辑有兜底计时、非致命错误要关门、 |
|||
* 纯指令帧收尾也要关门等一串来之不易的分支（门卡开着的后果是「会话活着但是聋的」）。 |
|||
* 门一轮只切换几次，一次 MethodChannel 调用的开销可以忽略。 |
|||
* |
|||
* ## 线程与背压 |
|||
* tap 在耳机解码线程上同步到达，这里**立刻转到自己的单线程执行器**，不让 WS 发送拖住解码线程。 |
|||
* 排队超过 [MAX_PENDING_PACKETS]（约 2s）就丢新包并计数。 |
|||
*/ |
|||
class BesAiPcmBridge( |
|||
private val onPacket: (ByteArray) -> Unit, |
|||
private val onMicActivity: () -> Unit, |
|||
private val onOverflow: (dropped: Long) -> Unit, |
|||
) { |
|||
companion object { |
|||
private const val TAG = "BesAiPcmBridge" |
|||
|
|||
/** |
|||
* 上行攒包大小：16k/16bit/mono 的 100ms = 3200 字节。 |
|||
* ⚠️ 与 Dart `DeviceAiSessionService._micBatchBytes`、iOS 的 batchBytes 相同，三处同改。 |
|||
*/ |
|||
private const val BATCH_BYTES = 3200 |
|||
|
|||
/** |
|||
* 说话人能量门限与上报节流。 |
|||
* ⚠️ 与 Dart 的 `_micActivityRms` / `_micActivityThrottle`、iOS 版相同，三处同改。 |
|||
* 只用来给 Dart 续空闲计时，不参与切句。 |
|||
*/ |
|||
private const val ACTIVITY_RMS = 600.0 |
|||
private const val ACTIVITY_THROTTLE_MS = 500L |
|||
|
|||
/** 约 2 秒的积压上限（100ms 一包） */ |
|||
private const val MAX_PENDING_PACKETS = 20 |
|||
} |
|||
|
|||
private val ingress = Executors.newSingleThreadExecutor { r -> Thread(r, "azure_speech.aipcm.ingress") } |
|||
private val pending = AtomicInteger(0) |
|||
|
|||
@Volatile private var enabled = false |
|||
@Volatile private var gated = false |
|||
|
|||
private var accum = ByteArray(0) |
|||
private var lastActivityAt = 0L |
|||
|
|||
private var framesIn = 0L |
|||
private var packetsOut = 0L |
|||
private var dropped = 0L |
|||
private var downlinkFrames = 0L |
|||
|
|||
val isEnabled: Boolean get() = enabled |
|||
|
|||
fun enable() { |
|||
if (enabled) return |
|||
enabled = true |
|||
ingress.execute { |
|||
accum = ByteArray(0) |
|||
gated = false |
|||
lastActivityAt = 0L |
|||
framesIn = 0L |
|||
packetsOut = 0L |
|||
dropped = 0L |
|||
downlinkFrames = 0L |
|||
} |
|||
pending.set(0) |
|||
BluetoothManager.aiPcmTap = { pcm -> ingest(pcm) } |
|||
BluetoothManager.setAiPcmBridge(true) |
|||
Log.d(TAG, "AI 上行直连已开启") |
|||
} |
|||
|
|||
fun disable() { |
|||
if (!enabled) return |
|||
enabled = false |
|||
BluetoothManager.setAiPcmBridge(false) |
|||
BluetoothManager.aiPcmTap = null |
|||
ingress.execute { |
|||
accum = ByteArray(0) |
|||
Log.d(TAG, "AI 上行直连已关闭 上行=$framesIn 帧 出包=$packetsOut 丢包=$dropped 下行=$downlinkFrames 帧") |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* AI 是否正在耳机里播报。播报期间上行换成**等长静音**而不是断流： |
|||
* 上行保持连续，服务端 VAD 收到的是"真的没人说话"，而不是一段数据缺口。 |
|||
*/ |
|||
fun setGated(v: Boolean) { |
|||
ingress.execute { |
|||
gated = v |
|||
// 关门时把攒了一半的静音丢掉，下一包从真实音频开始， |
|||
// 免得用户开口的头 100ms 里混着上一轮的静音 |
|||
if (!v) accum = ByteArray(0) |
|||
} |
|||
} |
|||
|
|||
// ---------- 上行 ---------- |
|||
|
|||
private fun ingest(pcm: ByteArray) { |
|||
if (pcm.isEmpty() || !enabled) return |
|||
// 背压：排队太多就丢新包。判断放在入队前，不然队列本身就是那个积压 |
|||
if (pending.get() >= MAX_PENDING_PACKETS) { |
|||
ingress.execute { |
|||
dropped++ |
|||
if (dropped == 1L || dropped % 50L == 0L) Log.w(TAG, "上行拥塞，已丢 $dropped 包") |
|||
onOverflow(dropped) |
|||
} |
|||
return |
|||
} |
|||
pending.incrementAndGet() |
|||
ingress.execute { |
|||
pending.decrementAndGet() |
|||
framesIn++ |
|||
accum = if (gated) { |
|||
accum + ByteArray(pcm.size) // 等长静音 |
|||
} else { |
|||
if (rms(pcm) >= ACTIVITY_RMS) { |
|||
val now = System.currentTimeMillis() |
|||
if (now - lastActivityAt >= ACTIVITY_THROTTLE_MS) { |
|||
lastActivityAt = now |
|||
onMicActivity() |
|||
} |
|||
} |
|||
accum + pcm |
|||
} |
|||
while (accum.size >= BATCH_BYTES) { |
|||
val pkt = accum.copyOfRange(0, BATCH_BYTES) |
|||
accum = accum.copyOfRange(BATCH_BYTES, accum.size) |
|||
packetsOut++ |
|||
onPacket(pkt) |
|||
} |
|||
} |
|||
} |
|||
|
|||
// ---------- 下行 ---------- |
|||
|
|||
/** 把一块合成音频交回耳机下行发送器（G.722 编码与节流在 bluetooth_manager 那边做） */ |
|||
fun postDownlink(pcm: ByteArray) { |
|||
if (pcm.isEmpty()) return |
|||
AiAudioDownlink.pushPcm(pcm) |
|||
ingress.execute { downlinkFrames++ } |
|||
} |
|||
|
|||
fun stats(): Map<String, Any> = mapOf( |
|||
"enabled" to enabled, |
|||
"framesIn" to framesIn, |
|||
"packetsOut" to packetsOut, |
|||
"dropped" to dropped, |
|||
"downlinkFrames" to downlinkFrames, |
|||
"gated" to gated, |
|||
) |
|||
|
|||
/** 16bit 小端 PCM 的 RMS（0~32767 量纲，与 Dart 的 `rmsOfPcm16` 同口径） */ |
|||
private fun rms(pcm: ByteArray): Double { |
|||
val count = pcm.size / 2 |
|||
if (count == 0) return 0.0 |
|||
var sum = 0.0 |
|||
for (i in 0 until count) { |
|||
// 高字节 toInt() 自带符号扩展，or 上无符号低字节即为 -32768..32767 |
|||
val v = (pcm[i * 2 + 1].toInt() shl 8) or (pcm[i * 2].toInt() and 0xFF) |
|||
sum += v.toDouble() * v.toDouble() |
|||
} |
|||
return sqrt(sum / count) |
|||
} |
|||
} |
|||
@ -0,0 +1,176 @@ |
|||
package com.yunqiinnovation.azure_speech |
|||
|
|||
import android.util.Log |
|||
import com.example.bluetooth_manager.BluetoothManager |
|||
import com.example.bluetooth_manager.CallTranslationDownlink |
|||
import java.util.concurrent.Executors |
|||
import java.util.concurrent.atomic.AtomicInteger |
|||
|
|||
/** |
|||
* 通话翻译 PCM 的**原生直连桥**（azure_speech 这一端）。 |
|||
* |
|||
* 对端是 bluetooth_manager 的 `BluetoothManager.callPcmTap`：耳机解码出来的双路 PCM 直接到这里 |
|||
* （不再经过 Dart），这里过 B 路说话人门限后推给 AST helper；AST 合成出来的译文 PCM 由 |
|||
* [postDownlink] 直接喂给 [CallTranslationDownlink]。 |
|||
* |
|||
* 本模块 `compileOnly project(":bluetooth_manager")`（与 agent_service→azure_speech 同一种先例）， |
|||
* 运行时两者都在 APK 里。**与 iOS 端 BesCallPcmBridge.swift 语义逐条对齐**，改一端要同改另一端。 |
|||
* |
|||
* ## 线程与背压 |
|||
* tap 在耳机解码线程上同步到达，这里**立刻转到自己的单线程执行器** [ingress]，不让 WS 发送或 |
|||
* base64 编码拖住解码线程。排队超过 [MAX_PENDING_FRAMES]（2s）就丢新帧并计数——这就是原来 |
|||
* Dart 链路上「AudioSendSlow 1011」的替代品：同样的拥塞，现在有数字、有日志，丢的是帧而不是整个会话。 |
|||
* |
|||
* ## 重连期间不丢音频 |
|||
* AST helper 在 initialize 期间会被重建，它自己的缓冲随之清空。「未就绪」期间的帧留在**这里** |
|||
* 的环形缓冲（每路 [HOLD_FRAMES]，3s），[markReady] 时整段补推——弱网重连的那 1~4 秒不再是纯丢音频。 |
|||
*/ |
|||
class BesCallPcmBridge( |
|||
private val onLegA: (ByteArray) -> Unit, |
|||
private val onLegB: (ByteArray) -> Unit, |
|||
private val onGateOpened: (rms: Int, fallback: Boolean) -> Unit, |
|||
private val onOverflow: (dropped: Long) -> Unit, |
|||
) { |
|||
companion object { |
|||
private const val TAG = "BesCallPcmBridge" |
|||
private const val MAX_PENDING_FRAMES = 100 |
|||
private const val HOLD_FRAMES = 150 |
|||
} |
|||
|
|||
private val ingress = Executors.newSingleThreadExecutor { r -> Thread(r, "azure_speech.callpcm.ingress") } |
|||
private val pending = AtomicInteger(0) |
|||
|
|||
@Volatile var isEnabled: Boolean = false |
|||
private set |
|||
|
|||
// 以下只在 ingress 线程上读写 |
|||
private var ready = false |
|||
private val gate = PeerSpeechGate() |
|||
private val holdA = ArrayDeque<ByteArray>() |
|||
private val holdB = ArrayDeque<ByteArray>() |
|||
private var framesA = 0L |
|||
private var framesB = 0L |
|||
@Volatile private var dropped = 0L |
|||
private var gatedB = 0L |
|||
@Volatile private var downlinkFrames = 0L |
|||
@Volatile private var maxPendingSeen = 0 |
|||
|
|||
fun enable() { |
|||
if (isEnabled) return |
|||
isEnabled = true |
|||
ingress.execute { |
|||
framesA = 0; framesB = 0; dropped = 0; gatedB = 0; downlinkFrames = 0; maxPendingSeen = 0 |
|||
} |
|||
BluetoothManager.callPcmTap = { leg, pcm -> enqueue(leg, pcm) } |
|||
BluetoothManager.setCallPcmBridge(true) |
|||
Log.d(TAG, "已开启：上行经 tap 直达，下行直喂发送器") |
|||
} |
|||
|
|||
fun disable() { |
|||
if (!isEnabled) return |
|||
isEnabled = false |
|||
BluetoothManager.setCallPcmBridge(false) |
|||
BluetoothManager.callPcmTap = null |
|||
ingress.execute { |
|||
Log.d(TAG, "已关闭 A=$framesA B=$framesB 丢帧=$dropped B门限前=$gatedB 下行=$downlinkFrames 峰值积压=$maxPendingSeen") |
|||
holdA.clear(); holdB.clear() |
|||
ready = false |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* AST 会话（重）建立开始：新会话要重新刻音色，门限重新关上；帧先攒着等 [markReady]。 |
|||
* |
|||
* ⚠️ 只清 B 路的缓存：那些帧是在**旧**门限状态下放行的，门限复位后必须由现场音频 |
|||
* 重新判定「对端开口了没有」，否则重连后照样可能刻到串音。A 路是机主自己的麦， |
|||
* 没有门限，攒下的整段都要补推——这正是「重连那几秒不丢音频」的价值所在。 |
|||
*/ |
|||
fun onSessionInit() { |
|||
ingress.execute { |
|||
ready = false |
|||
gate.reset() |
|||
holdB.clear() |
|||
} |
|||
} |
|||
|
|||
/** AST 会话销毁：帧继续攒着（重连场景），直到下一次 markReady。 */ |
|||
fun onSessionDisposed() { |
|||
ingress.execute { ready = false } |
|||
} |
|||
|
|||
/** A/B 两路 helper 都起来了：把攒下的整段补推出去，之后直通。 */ |
|||
fun markReady() { |
|||
ingress.execute { |
|||
val a = holdA.toList(); val b = holdB.toList() |
|||
holdA.clear(); holdB.clear() |
|||
ready = true |
|||
if (a.isNotEmpty() || b.isNotEmpty()) Log.d(TAG, "就绪，补推积攒的 A=${a.size} B=${b.size} 帧") |
|||
a.forEach(onLegA) |
|||
b.forEach(onLegB) |
|||
} |
|||
} |
|||
|
|||
/** 由 AST 的 TTS 输出调用：译文 PCM 直接喂给耳机下行发送器。 */ |
|||
fun postDownlink(leg: String, pcm: ByteArray) { |
|||
if (pcm.isEmpty()) return |
|||
CallTranslationDownlink.pushPcm(leg, pcm) |
|||
downlinkFrames++ |
|||
} |
|||
|
|||
fun stats(): Map<String, Any> = mapOf( |
|||
"enabled" to isEnabled, "ready" to ready, |
|||
"framesA" to framesA, "framesB" to framesB, |
|||
"dropped" to dropped, "gatedB" to gatedB, |
|||
"downlinkFrames" to downlinkFrames, "maxPending" to maxPendingSeen, |
|||
"gateOpen" to gate.isOpen, "gateFallback" to gate.openedByFallback, |
|||
) |
|||
|
|||
private fun enqueue(leg: String, pcm: ByteArray) { |
|||
if (pcm.isEmpty()) return |
|||
val p = pending.incrementAndGet() |
|||
if (p > MAX_PENDING_FRAMES) { |
|||
pending.decrementAndGet() |
|||
val d = ++dropped |
|||
if (d == 1L || d % 100L == 0L) { |
|||
Log.w(TAG, "上行积压超过 $MAX_PENDING_FRAMES 帧，丢帧累计 $d") |
|||
onOverflow(d) |
|||
} |
|||
return |
|||
} |
|||
if (p > maxPendingSeen) maxPendingSeen = p |
|||
ingress.execute { |
|||
pending.decrementAndGet() |
|||
handle(leg, pcm) |
|||
} |
|||
} |
|||
|
|||
/** 只在 ingress 线程上调用 */ |
|||
private fun handle(leg: String, pcm: ByteArray) { |
|||
if (leg == "A") { |
|||
framesA++ |
|||
deliverA(pcm) |
|||
return |
|||
} |
|||
framesB++ |
|||
val wasOpen = gate.isOpen |
|||
val frames = gate.accept(pcm) |
|||
if (frames.isEmpty()) gatedB++ |
|||
if (!wasOpen && gate.isOpen) { |
|||
Log.w(TAG, "B 路门限开启 rms=${gate.openRms} fallback=${gate.openedByFallback}") |
|||
onGateOpened(gate.openRms, gate.openedByFallback) |
|||
} |
|||
frames.forEach { deliverB(it) } |
|||
} |
|||
|
|||
private fun deliverA(pcm: ByteArray) { |
|||
if (ready) { onLegA(pcm); return } |
|||
holdA.addLast(pcm) |
|||
if (holdA.size > HOLD_FRAMES) holdA.removeFirst() |
|||
} |
|||
|
|||
private fun deliverB(pcm: ByteArray) { |
|||
if (ready) { onLegB(pcm); return } |
|||
holdB.addLast(pcm) |
|||
if (holdB.size > HOLD_FRAMES) holdB.removeFirst() |
|||
} |
|||
} |
|||
@ -0,0 +1,320 @@ |
|||
package com.yunqiinnovation.azure_speech |
|||
|
|||
import android.media.AudioAttributes |
|||
import android.media.AudioFormat |
|||
import android.media.AudioManager |
|||
import android.media.AudioTrack |
|||
import android.os.Build |
|||
import android.util.Log |
|||
import java.util.concurrent.LinkedBlockingQueue |
|||
import java.util.concurrent.TimeUnit |
|||
|
|||
/** |
|||
* 流式 PCM 外放播放器（16bit 小端、单声道），Android 侧。 |
|||
* |
|||
* ## 为什么不用 just_audio |
|||
* Dart 的 [PcmStreamPlayer] 在 Android 上原来是给 just_audio 喂一个「长度未知的 |
|||
* StreamAudioSource」，而 just_audio 内部**会起一个本地代理 HTTP 服务**，把 PCM |
|||
* 包成 chunked WAV 再交给 ExoPlayer。也就是说一条实时语音流要先走一遍 localhost |
|||
* TCP + HTTP 分块编码，再进解码器。能用，但这条路上白白多了一次内存拷贝、一次 |
|||
* HTTP 分帧和 ExoPlayer 自己的起播缓冲，首字延迟比直接喂 AudioTrack 高一截。 |
|||
* |
|||
* iOS 那边早就绕开了（`PcmSpeakerPlayer.swift`，那边是被迫的：AVPlayer 压根放不了 |
|||
* 长度未知的直播流）。这个类把 Android 也对齐过来，**方法名与 iOS 逐字一致** |
|||
* （start / feed / end / stop / dispose + 回调 onCompleted），Dart 侧两端共用一条 |
|||
* 代码路径。 |
|||
* |
|||
* ## ⚠️ 不碰 AudioManager 的焦点与模式 |
|||
* 音频会话由 Dart 侧的 audio_session 统一配置（`EmaiAudioSession`）。这里只按它 |
|||
* 声明的属性建 AudioTrack（`USAGE_MEDIA` + `CONTENT_TYPE_SPEECH`，与 |
|||
* `emai_audio_session.dart` 里那份 `AndroidAudioAttributes` 对齐）。 |
|||
* 自己再去 requestAudioFocus / setMode 会和录音链路互相掀桌子。 |
|||
* |
|||
* ## 播完的判定 |
|||
* AudioTrack 没有 iOS 那种逐 buffer 的完成回调，所以用「写进去的帧数」对比 |
|||
* `playbackHeadPosition`（已经真正推给硬件的帧数）。**不能在队列排空时就报完成**： |
|||
* 那时最后一两百毫秒还在 AudioTrack 的缓冲里没出声，调用方会据此给服务端回 |
|||
* `LocalRespondingEnded`，尾音被切掉。 |
|||
*/ |
|||
class PcmSpeakerPlayer(private val onCompleted: () -> Unit) { |
|||
|
|||
companion object { |
|||
private const val TAG = "PcmSpeakerPlayer" |
|||
|
|||
/** 队列轮询间隔：既是写线程的空转节奏，也是播完判定的精度 */ |
|||
private const val POLL_MS = 20L |
|||
|
|||
/** |
|||
* 播完判定的兜底上限。正常情况下缓冲里最多就几百毫秒， |
|||
* 走到这个上限说明 AudioTrack 卡住了（拔耳机、路由切换）， |
|||
* 这时也必须报完成，否则服务端会一直停在 Responding。 |
|||
*/ |
|||
private const val DRAIN_TIMEOUT_MS = 5_000L |
|||
} |
|||
|
|||
private val lock = Any() |
|||
private val queue = LinkedBlockingQueue<ByteArray>() |
|||
|
|||
private var track: AudioTrack? = null |
|||
private var trackRate = 0 |
|||
private var writer: Thread? = null |
|||
|
|||
@Volatile private var running = false |
|||
@Volatile private var ended = false |
|||
|
|||
/** 本轮代际。stop() 之后迟到的写线程按代际自行退出 */ |
|||
@Volatile private var generation = 0 |
|||
|
|||
/** 已写进 AudioTrack 的帧数，只在写线程上累加 */ |
|||
private var framesWritten = 0 |
|||
|
|||
// ---------- 对外接口（与 iOS 逐字对齐）---------- |
|||
|
|||
fun start(sampleRate: Int): Boolean { |
|||
synchronized(lock) { |
|||
generation++ |
|||
val gen = generation |
|||
stopWriterLocked() |
|||
queue.clear() |
|||
ended = false |
|||
framesWritten = 0 |
|||
|
|||
if (!ensureTrackLocked(sampleRate)) return false |
|||
val t = track ?: return false |
|||
try { |
|||
t.play() |
|||
} catch (e: Exception) { |
|||
Log.e(TAG, "AudioTrack.play 失败: ${e.message}") |
|||
return false |
|||
} |
|||
running = true |
|||
val th = Thread({ writeLoop(gen) }, "azure_speech.pcm_speaker") |
|||
th.isDaemon = true |
|||
writer = th |
|||
th.start() |
|||
return true |
|||
} |
|||
} |
|||
|
|||
fun feed(pcm: ByteArray) { |
|||
if (!running || pcm.isEmpty()) return |
|||
queue.offer(pcm) |
|||
} |
|||
|
|||
/** 上层不再喂数据了：已排队的播完就报完成 */ |
|||
fun end() { |
|||
ended = true |
|||
} |
|||
|
|||
/** 打断：立刻停声，不报完成 */ |
|||
fun stop() { |
|||
synchronized(lock) { |
|||
generation++ |
|||
running = false |
|||
ended = false |
|||
queue.clear() |
|||
stopWriterLocked() |
|||
val t = track |
|||
if (t != null) { |
|||
try { |
|||
// pause + flush 才是"立刻不出声"；stop() 会把缓冲里的先放完 |
|||
t.pause() |
|||
t.flush() |
|||
} catch (e: Exception) { |
|||
Log.w(TAG, "停止失败: ${e.message}") |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
fun dispose() { |
|||
synchronized(lock) { |
|||
generation++ |
|||
running = false |
|||
queue.clear() |
|||
stopWriterLocked() |
|||
val t = track |
|||
track = null |
|||
trackRate = 0 |
|||
try { |
|||
t?.pause() |
|||
t?.flush() |
|||
t?.release() |
|||
} catch (e: Exception) { |
|||
Log.w(TAG, "释放失败: ${e.message}") |
|||
} |
|||
} |
|||
} |
|||
|
|||
// ---------- 内部 ---------- |
|||
|
|||
private fun writeLoop(gen: Int) { |
|||
try { |
|||
while (running && gen == generation) { |
|||
val chunk = queue.poll(POLL_MS, TimeUnit.MILLISECONDS) |
|||
if (chunk != null) { |
|||
writeFully(chunk, gen) |
|||
continue |
|||
} |
|||
// 队列空了。没收到 end() 就继续等下一块（服务端还在合成） |
|||
if (!ended) continue |
|||
if (awaitDrained(gen)) { |
|||
finish(gen) |
|||
return |
|||
} |
|||
} |
|||
} catch (e: InterruptedException) { |
|||
Thread.currentThread().interrupt() |
|||
} catch (e: Exception) { |
|||
Log.e(TAG, "写线程异常: ${e.message}") |
|||
// 异常也要收尾，否则调用方永远等不到 onCompleted， |
|||
// 服务端会一直停在 Responding |
|||
finish(gen) |
|||
} |
|||
} |
|||
|
|||
private fun writeFully(chunk: ByteArray, gen: Int) { |
|||
val t = track ?: return |
|||
var off = 0 |
|||
while (off < chunk.size && running && gen == generation) { |
|||
val n = try { |
|||
t.write(chunk, off, chunk.size - off) |
|||
} catch (e: Exception) { |
|||
Log.e(TAG, "写入失败: ${e.message}") |
|||
return |
|||
} |
|||
if (n <= 0) return // ERROR_INVALID_OPERATION / ERROR_DEAD_OBJECT |
|||
off += n |
|||
} |
|||
framesWritten += chunk.size / 2 |
|||
} |
|||
|
|||
/** |
|||
* 等已写入的帧真正播完。返回 false 表示这一轮被 stop() 掀掉了。 |
|||
* |
|||
* `playbackHeadPosition` 是 Int 且会回绕，但一轮回复远到不了 2^31 帧 |
|||
* （16k 下约 37 小时），直接比就行。 |
|||
*/ |
|||
private fun awaitDrained(gen: Int): Boolean { |
|||
val t = track ?: return true |
|||
val deadline = System.currentTimeMillis() + DRAIN_TIMEOUT_MS |
|||
while (running && gen == generation) { |
|||
// 期间又来了新数据（服务端接着说）：回到主循环继续写 |
|||
if (queue.isNotEmpty()) return false |
|||
val head = try { |
|||
t.playbackHeadPosition |
|||
} catch (e: Exception) { |
|||
return true |
|||
} |
|||
if (head >= framesWritten) return true |
|||
if (System.currentTimeMillis() > deadline) { |
|||
Log.w(TAG, "等待播完超时（已写 $framesWritten 帧，已播 $head 帧），按播完处理") |
|||
return true |
|||
} |
|||
try { |
|||
Thread.sleep(POLL_MS) |
|||
} catch (e: InterruptedException) { |
|||
Thread.currentThread().interrupt() |
|||
return false |
|||
} |
|||
} |
|||
return false |
|||
} |
|||
|
|||
private fun finish(gen: Int) { |
|||
synchronized(lock) { |
|||
if (gen != generation) return |
|||
running = false |
|||
ended = false |
|||
framesWritten = 0 |
|||
try { |
|||
// 一轮播完就把 track 停掉，别让它空跑着占住输出。 |
|||
// 这时缓冲已经排空（awaitDrained 保证），stop() 不会切掉任何声音。 |
|||
track?.stop() |
|||
} catch (e: Exception) { |
|||
Log.w(TAG, "停止 track 失败: ${e.message}") |
|||
} |
|||
} |
|||
onCompleted() |
|||
} |
|||
|
|||
private fun stopWriterLocked() { |
|||
val th = writer |
|||
writer = null |
|||
th?.interrupt() |
|||
} |
|||
|
|||
/** 采样率变了要重建：AudioTrack 的采样率是构造时定死的 */ |
|||
private fun ensureTrackLocked(sampleRate: Int): Boolean { |
|||
val t = track |
|||
if (t != null && trackRate == sampleRate && |
|||
t.state == AudioTrack.STATE_INITIALIZED |
|||
) { |
|||
try { |
|||
t.pause() |
|||
t.flush() // 上一轮的残留不能带到这一轮 |
|||
} catch (_: Exception) { |
|||
} |
|||
return true |
|||
} |
|||
try { |
|||
t?.release() |
|||
} catch (_: Exception) { |
|||
} |
|||
track = null |
|||
trackRate = 0 |
|||
|
|||
val minBuf = AudioTrack.getMinBufferSize( |
|||
sampleRate, AudioFormat.CHANNEL_OUT_MONO, AudioFormat.ENCODING_PCM_16BIT |
|||
) |
|||
if (minBuf <= 0) { |
|||
Log.e(TAG, "不支持的采样率 $sampleRate（getMinBufferSize=$minBuf）") |
|||
return false |
|||
} |
|||
// 给到 ~200ms，留足抖动余量。太小会在弱网分块到达不均时出现断音。 |
|||
val bufSize = maxOf(minBuf, sampleRate * 2 / 5) |
|||
|
|||
val attrs = AudioAttributes.Builder() |
|||
// 与 emai_audio_session.dart 里声明的 AndroidAudioAttributes 对齐， |
|||
// 改这两行会改变输出路由（比如 A2DP 还走不走） |
|||
.setUsage(AudioAttributes.USAGE_MEDIA) |
|||
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) |
|||
.build() |
|||
val fmt = AudioFormat.Builder() |
|||
.setEncoding(AudioFormat.ENCODING_PCM_16BIT) |
|||
.setSampleRate(sampleRate) |
|||
.setChannelMask(AudioFormat.CHANNEL_OUT_MONO) |
|||
.build() |
|||
|
|||
val newTrack = try { |
|||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { |
|||
AudioTrack.Builder() |
|||
.setAudioAttributes(attrs) |
|||
.setAudioFormat(fmt) |
|||
.setBufferSizeInBytes(bufSize) |
|||
.setTransferMode(AudioTrack.MODE_STREAM) |
|||
.build() |
|||
} else { |
|||
@Suppress("DEPRECATION") |
|||
AudioTrack( |
|||
AudioManager.STREAM_MUSIC, sampleRate, AudioFormat.CHANNEL_OUT_MONO, |
|||
AudioFormat.ENCODING_PCM_16BIT, bufSize, AudioTrack.MODE_STREAM |
|||
) |
|||
} |
|||
} catch (e: Exception) { |
|||
Log.e(TAG, "创建 AudioTrack 失败: ${e.message}") |
|||
return false |
|||
} |
|||
if (newTrack.state != AudioTrack.STATE_INITIALIZED) { |
|||
Log.e(TAG, "AudioTrack 未初始化成功 state=${newTrack.state}") |
|||
try { |
|||
newTrack.release() |
|||
} catch (_: Exception) { |
|||
} |
|||
return false |
|||
} |
|||
track = newTrack |
|||
trackRate = sampleRate |
|||
return true |
|||
} |
|||
} |
|||
@ -0,0 +1,81 @@ |
|||
package com.yunqiinnovation.azure_speech |
|||
|
|||
import kotlin.math.roundToInt |
|||
import kotlin.math.sqrt |
|||
|
|||
/** |
|||
* 对端（B 路）说话人门限——Dart 版 `lib/data/utils/peer_speech_gate.dart` 的逐条移植。 |
|||
* |
|||
* **在对端真的开口之前，不要把音频推给端到端翻译。** 阿里的实时声音复刻是 |
|||
* `frequency = "once"`，只在会话开头刻一次音色；此时对端往往还没开口，B 路先收到的是 |
|||
* 机主声音的串音，刻错之后用户听到的对端译文用的是自己的声音。 |
|||
* |
|||
* - 未开门期间音频进环形缓冲（[prerollFrames]），开门瞬间整段补发——对端先开口也不吃掉开头； |
|||
* - 连续 [openFrames] 帧 RMS ≥ [rmsThreshold] 才开门，断续的串音尖峰不算； |
|||
* - 收满 [fallbackFrames] 仍未开门就**强行开门**（门限太严=整通电话不翻译，比刻错音色严重得多）； |
|||
* - 每次会话重建（含断线重连）必须 [reset]。 |
|||
* |
|||
* ⚠️ 参数默认值与 Dart 版、Swift 版三处相同，调阈值要三处同改。 |
|||
* 线程：调用方保证串行（BesCallPcmBridge 的 ingress 线程）。 |
|||
*/ |
|||
class PeerSpeechGate( |
|||
val rmsThreshold: Int = 600, |
|||
val openFrames: Int = 8, |
|||
val prerollFrames: Int = 25, |
|||
val fallbackFrames: Int = 500, |
|||
) { |
|||
var isOpen = false |
|||
private set |
|||
var openRms = 0 |
|||
private set |
|||
var openedByFallback = false |
|||
private set |
|||
private var run = 0 |
|||
private var seen = 0 |
|||
private val preroll = ArrayDeque<ByteArray>() |
|||
|
|||
/** 喂一帧，返回应当推给翻译服务的帧（0 帧、1 帧，或开门瞬间的一整串）。 */ |
|||
fun accept(frame: ByteArray): List<ByteArray> { |
|||
if (isOpen) return listOf(frame) |
|||
seen++ |
|||
val rms = rmsOfPcm16(frame) |
|||
if (rms >= rmsThreshold) run++ else run = 0 |
|||
preroll.addLast(frame) |
|||
while (preroll.size > prerollFrames) preroll.removeFirst() |
|||
val hitThreshold = run >= openFrames |
|||
val hitFallback = seen >= fallbackFrames |
|||
if (!hitThreshold && !hitFallback) return emptyList() |
|||
isOpen = true |
|||
openRms = rms |
|||
openedByFallback = !hitThreshold |
|||
val out = preroll.toList() |
|||
preroll.clear() |
|||
return out |
|||
} |
|||
|
|||
fun reset() { |
|||
isOpen = false |
|||
run = 0 |
|||
seen = 0 |
|||
openRms = 0 |
|||
openedByFallback = false |
|||
preroll.clear() |
|||
} |
|||
|
|||
companion object { |
|||
/** PCM16 小端单声道的 RMS；字节数为奇数时忽略末尾半个样本。 */ |
|||
fun rmsOfPcm16(frame: ByteArray): Int { |
|||
val n = frame.size / 2 |
|||
if (n == 0) return 0 |
|||
var sum = 0.0 |
|||
for (i in 0 until n) { |
|||
val lo = frame[i * 2].toInt() and 0xFF |
|||
val hi = frame[i * 2 + 1].toInt() |
|||
val s = ((hi shl 8) or lo).toDouble() |
|||
sum += s * s |
|||
} |
|||
val mean = sum / n |
|||
return if (mean.isFinite()) sqrt(mean).roundToInt() else 0 |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,487 @@ |
|||
import Foundation |
|||
|
|||
/// 阿里云百炼「多模态交互协议」的**原生**客户端（EMAI 助手）。 |
|||
/// |
|||
/// 协议文档：https://help.aliyun.com/zh/model-studio/multimodal-interaction-protocol |
|||
/// 端点：`wss://dashscope.aliyuncs.com/api-ws/v1/inference` |
|||
/// |
|||
/// ## 为什么有原生这一份 |
|||
/// Dart 侧的 `BailianMultimodalService` 仍在（手机麦那条链路、以及本类起不来时的退路）。 |
|||
/// 耳机按键唤醒那条链路原先是 |
|||
/// `bluetooth_manager → EventChannel → Dart → WS(Dart) → Dart → MethodChannel → bluetooth_manager`， |
|||
/// 上行 20ms 一帧、下行每个音频块一次，全压在 Dart 的单 isolate 上，和页面布局、GC |
|||
/// 排同一个 event loop。本类把 WS 连同音频收发搬到原生：PCM 经 `BesAiPcmBridge` |
|||
/// 在原生之间直连，**Dart 只收文本与状态事件**。 |
|||
/// |
|||
/// ## ⚠️ 这不修「锁屏断连」 |
|||
/// iOS 挂起的是整个进程，原生和 Dart 一起停。锁屏后能不能活着取决于 |
|||
/// `UIBackgroundModes=audio` + 确实有音频在播（Dart 侧 `BackgroundKeepalive` 干的事）， |
|||
/// 与代码在哪一层无关。本类解决的是 event loop 抖动与跨界开销，别把它当保活手段。 |
|||
/// |
|||
/// ## ⚠️ 与 Dart 版**逐条对齐** |
|||
/// 握手参数（upstream/downstream/client_info/biz_params）、事件名映射、心跳间隔都照 |
|||
/// `lib/data/services/bailian_multimodal_service.dart` 写。改一边必须同改另一边， |
|||
/// 否则「用耳机唤醒」和「在 App 里按住说话」会变成两种行为，而且两边都不报错。 |
|||
/// |
|||
/// ## 指令解析刻意**留在 Dart** |
|||
/// `extra_info.tool_calls` 的解析（参数是字符串套 JSON、值恒为中文、下午时间要校正…） |
|||
/// 有一批真实 payload 的金测试守着（`test/assistant_directive_test.dart`）， |
|||
/// 而解析结果最终要落进 GetStorage 和「代办」页——在原生解析再序列化回 Dart 毫无收益， |
|||
/// 只会把那批测试作废。这里把整个 `output` 原样上抛，Dart 照旧解析。 |
|||
final class BailianDialogClient: NSObject { |
|||
|
|||
// MARK: - 配置 |
|||
|
|||
struct Config { |
|||
let apiKey: String |
|||
let appId: String |
|||
let workspaceId: String |
|||
let mode: String // push2talk / tap2talk / duplex |
|||
let voice: String? // 留空用百炼应用自己配置的音色 |
|||
let userId: String |
|||
let enableWebSearch: Bool |
|||
let downstreamRate: Int |
|||
let mcpToken: String? |
|||
/// 需要用户身份的 MCP 工具名。**由 Dart 传下来**，不在原生再抄一份： |
|||
/// 服务端加工具时只改 Dart 那一处，漏改的表现是那个工具永远「身份校验失败」。 |
|||
let mcpAuthTools: [String] |
|||
} |
|||
|
|||
private static let endpoint = "wss://dashscope.aliyuncs.com/api-ws/v1/inference" |
|||
|
|||
/// 上行音频：16k / 16bit / 单声道 / 小端 |
|||
private static let upstreamSampleRate = 16000 |
|||
|
|||
/// 服务端要求任意连续 60s 内必须有消息往来；文档建议 50s 一次心跳 |
|||
private static let heartbeatInterval: TimeInterval = 50 |
|||
|
|||
/// WS 层 ping，与业务心跳是两回事（对齐 Dart 的 pingInterval: 20s） |
|||
private static let pingInterval: TimeInterval = 20 |
|||
|
|||
// MARK: - 回调 |
|||
|
|||
/// 业务事件（已拍平成 Dart 能直接用的字典），在主队列上回调 |
|||
var onEvent: (([String: Any]) -> Void)? |
|||
|
|||
/// 下行合成音频（PCM，采样率 = config.downstreamRate），在接收队列上回调 |
|||
var onAudio: ((Data) -> Void)? |
|||
|
|||
// MARK: - 状态 |
|||
|
|||
private let queue = DispatchQueue(label: "azure_speech.bailian.dialog") |
|||
private var session: URLSession? |
|||
private var task: URLSessionWebSocketTask? |
|||
private var heartbeatTimer: DispatchSourceTimer? |
|||
private var pingTimer: DispatchSourceTimer? |
|||
private var config: Config? |
|||
private var taskId = "" |
|||
private var dialogId = "" |
|||
private var running = false |
|||
|
|||
private var uplinkBytes: Int64 = 0 |
|||
private var downlinkBytes: Int64 = 0 |
|||
/// 下行字节数的上报节流。Dart 侧不再拿到 PCM，但它要靠「首包到达」开门控、 |
|||
/// 靠累计字节数估算播完时刻（门控的收尾计时），所以这两件事必须报上去。 |
|||
/// 按 100ms 汇总一次，别每块都报——那就把省下来的跨界又加回去了。 |
|||
private var lastDownlinkReportAt: TimeInterval = 0 |
|||
private static let downlinkReportInterval: TimeInterval = 0.1 |
|||
|
|||
var isRunning: Bool { |
|||
queue.sync { running } |
|||
} |
|||
|
|||
// MARK: - 生命周期 |
|||
|
|||
/// 建立会话。返回 false 表示配置不全或连接没起来。 |
|||
func start(config: Config) -> Bool { |
|||
guard !config.apiKey.trimmingCharacters(in: .whitespaces).isEmpty else { |
|||
NSLog("[BailianDialog] 未配置 API Key，不尝试连接") |
|||
// Key 缺失时不要尝试连接——服务端会先回 task-started 再立刻 task-failed， |
|||
// 日志上看起来像"连上了又掉"，容易误判成网络问题 |
|||
emit(["event": "error", "text": "未配置 ALIBABA_OPENSPEECH_APP_KEY", |
|||
"code": "NOT_CONFIGURED", "fatal": true]) |
|||
return false |
|||
} |
|||
stop() |
|||
return queue.sync { |
|||
self.config = config |
|||
taskId = Self.uuid() |
|||
dialogId = "" |
|||
uplinkBytes = 0 |
|||
downlinkBytes = 0 |
|||
lastDownlinkReportAt = 0 |
|||
|
|||
guard let url = URL(string: Self.endpoint) else { return false } |
|||
var req = URLRequest(url: url) |
|||
req.setValue("Bearer \(config.apiKey)", forHTTPHeaderField: "Authorization") |
|||
req.setValue("enable", forHTTPHeaderField: "X-DashScope-DataInspection") |
|||
|
|||
let cfg = URLSessionConfiguration.default |
|||
let s = URLSession(configuration: cfg) |
|||
let t = s.webSocketTask(with: req) |
|||
session = s |
|||
task = t |
|||
running = true |
|||
t.resume() |
|||
receiveLoop() |
|||
sendJson(startPayload(config)) |
|||
startTimers() |
|||
NSLog("[BailianDialog] 会话启动 app_id=%@ workspace=%@ mode=%@ downstream=%dHz", |
|||
config.appId, config.workspaceId, config.mode, config.downstreamRate) |
|||
return true |
|||
} |
|||
} |
|||
|
|||
/// 结束会话：先发 Stop 让服务端收尾，再 finish-task 结束整个 WS 任务 |
|||
func stop() { |
|||
queue.sync { |
|||
guard running, let t = task else { |
|||
cleanupLocked() |
|||
return |
|||
} |
|||
sendDirectiveLocked("Stop") |
|||
sendJsonLocked([ |
|||
"header": ["action": "finish-task", "task_id": taskId, "streaming": "duplex"], |
|||
"payload": ["input": [String: Any]()], |
|||
]) |
|||
t.cancel(with: .normalClosure, reason: nil) |
|||
NSLog("[BailianDialog] 会话结束 上行=%lld 字节 下行=%lld 字节", uplinkBytes, downlinkBytes) |
|||
cleanupLocked() |
|||
} |
|||
} |
|||
|
|||
// MARK: - 上行 |
|||
|
|||
/// 推一包上行音频（PCM 16k/16bit/mono）。由 `BesAiPcmBridge` 的串行队列调用。 |
|||
func pushAudio(_ pcm: Data) { |
|||
guard !pcm.isEmpty else { return } |
|||
queue.async { |
|||
guard self.running, let t = self.task else { return } |
|||
self.uplinkBytes &+= Int64(pcm.count) |
|||
t.send(.data(pcm)) { err in |
|||
if let e = err { NSLog("[BailianDialog] 推送音频失败: %@", e.localizedDescription) } |
|||
} |
|||
} |
|||
} |
|||
|
|||
/// push2talk：开始/结束/放弃一段说话 |
|||
func beginSpeech() { sendDirective("SendSpeech") } |
|||
func endSpeech() { sendDirective("StopSpeech") } |
|||
func cancelSpeech() { sendDirective("CancelSpeech") } |
|||
|
|||
/// 打断当前回复并请求发言权 |
|||
func interrupt() { sendDirective("RequestToSpeak") } |
|||
|
|||
/// 本地播报开始/结束（服务端据此推进对话状态） |
|||
func notifyLocalRespondingStarted() { sendDirective("LocalRespondingStarted") } |
|||
func notifyLocalRespondingEnded() { sendDirective("LocalRespondingEnded") } |
|||
|
|||
/// 文本对话：交给应用的 LLM 处理（prompt），或让它直接朗读（transcript） |
|||
func sendText(_ text: String, speakDirectly: Bool) { |
|||
let t = text.trimmingCharacters(in: .whitespacesAndNewlines) |
|||
guard !t.isEmpty else { return } |
|||
sendDirective("RequestToRespond", extra: [ |
|||
"type": speakDirectly ? "transcript" : "prompt", |
|||
"text": t, |
|||
]) |
|||
} |
|||
|
|||
func stats() -> [String: Any] { |
|||
queue.sync { |
|||
["running": running, "uplinkBytes": uplinkBytes, |
|||
"downlinkBytes": downlinkBytes, "dialogId": dialogId] |
|||
} |
|||
} |
|||
|
|||
// MARK: - 协议帧 |
|||
|
|||
private func startPayload(_ c: Config) -> [String: Any] { |
|||
var downstream: [String: Any] = [ |
|||
"audio_format": "pcm", |
|||
"sample_rate": c.downstreamRate, |
|||
// transcript：中间态只回朗读文本，跟 TTS 出声一致， |
|||
// 免得 UI 上先闪一段 dialog 思考态文本再被覆盖 |
|||
"intermediate_text": "transcript", |
|||
// 显式钉死"每帧回全文"：Dart 收到 RespondingContent 是整体替换气泡内容的， |
|||
// 改成增量会导致只显示最后一个片段 |
|||
"incremental_response": false, |
|||
] |
|||
// 不传 voice：音色属于百炼应用的配置，客户端硬编码一个名字只会跟应用实际用的 |
|||
// TTS 模型对不上（实测传 longxiaochun_v2 会被回 "tts voice error"） |
|||
if let v = c.voice?.trimmingCharacters(in: .whitespaces), !v.isEmpty { |
|||
downstream["voice"] = v |
|||
} |
|||
|
|||
var parameters: [String: Any] = [ |
|||
"upstream": [ |
|||
"type": "AudioOnly", |
|||
"mode": c.mode, |
|||
"audio_format": "pcm", |
|||
"sample_rate": Self.upstreamSampleRate, |
|||
], |
|||
"downstream": downstream, |
|||
"client_info": ["user_id": Self.safeUserId(c.userId)], |
|||
] |
|||
|
|||
// biz_params 是可选块，两件事都没有时整块不发——万一应用侧没开这些能力， |
|||
// 多带一个字段可能让 Start 直接被拒，那样整个功能都起不来 |
|||
let hasToken = !(c.mcpToken ?? "").trimmingCharacters(in: .whitespaces).isEmpty |
|||
if c.enableWebSearch || hasToken { |
|||
var userDefined: [String: Any] = [:] |
|||
if hasToken, let tok = c.mcpToken { |
|||
// ⚠️ **必须按工具名嵌套**，平铺写 `auth_token` 会被静默丢弃 |
|||
for tool in c.mcpAuthTools { |
|||
userDefined[tool] = ["auth_token": tok] |
|||
} |
|||
} |
|||
if c.enableWebSearch { |
|||
userDefined["extra_config"] = ["enable_web_search": true] |
|||
} |
|||
parameters["biz_params"] = ["user_defined_params": userDefined] |
|||
} |
|||
|
|||
return [ |
|||
"header": ["action": "run-task", "task_id": taskId, "streaming": "duplex"], |
|||
"payload": [ |
|||
"task_group": "aigc", |
|||
"task": "multimodal-generation", |
|||
"function": "generation", |
|||
"model": "multimodal-dialog", |
|||
"input": [ |
|||
"directive": "Start", |
|||
"workspace_id": c.workspaceId, |
|||
"app_id": c.appId, |
|||
], |
|||
"parameters": parameters, |
|||
], |
|||
] |
|||
} |
|||
|
|||
private func sendDirective(_ directive: String, extra: [String: Any]? = nil) { |
|||
queue.async { self.sendDirectiveLocked(directive, extra: extra) } |
|||
} |
|||
|
|||
private func sendDirectiveLocked(_ directive: String, extra: [String: Any]? = nil) { |
|||
guard running, let c = config else { return } |
|||
var input: [String: Any] = [ |
|||
"directive": directive, |
|||
"workspace_id": c.workspaceId, |
|||
"app_id": c.appId, |
|||
] |
|||
if !dialogId.isEmpty { input["dialog_id"] = dialogId } |
|||
if let e = extra { input.merge(e) { _, new in new } } |
|||
sendJsonLocked([ |
|||
"header": ["action": "continue-task", "task_id": taskId, "streaming": "duplex"], |
|||
"payload": [ |
|||
"task_group": "aigc", |
|||
"task": "multimodal-generation", |
|||
"function": "generation", |
|||
"model": "multimodal-dialog", |
|||
"input": input, |
|||
], |
|||
]) |
|||
} |
|||
|
|||
private func sendJson(_ msg: [String: Any]) { |
|||
queue.async { self.sendJsonLocked(msg) } |
|||
} |
|||
|
|||
private func sendJsonLocked(_ msg: [String: Any]) { |
|||
guard let t = task else { return } |
|||
guard let data = try? JSONSerialization.data(withJSONObject: msg), |
|||
let s = String(data: data, encoding: .utf8) else { |
|||
NSLog("[BailianDialog] 序列化失败") |
|||
return |
|||
} |
|||
t.send(.string(s)) { err in |
|||
if let e = err { NSLog("[BailianDialog] 发送失败: %@", e.localizedDescription) } |
|||
} |
|||
} |
|||
|
|||
// MARK: - 下行 |
|||
|
|||
private func receiveLoop() { |
|||
guard let t = task else { return } |
|||
t.receive { [weak self] result in |
|||
guard let self = self else { return } |
|||
switch result { |
|||
case .failure(let err): |
|||
self.queue.async { |
|||
guard self.running else { return } |
|||
NSLog("[BailianDialog] WebSocket 错误: %@", err.localizedDescription) |
|||
self.emit(["event": "error", "text": err.localizedDescription, "fatal": true]) |
|||
self.cleanupLocked() |
|||
} |
|||
case .success(let msg): |
|||
switch msg { |
|||
case .data(let d): |
|||
// 二进制帧就是下行合成音频 |
|||
self.queue.async { |
|||
self.downlinkBytes &+= Int64(d.count) |
|||
let now = Date().timeIntervalSince1970 |
|||
// 首块立刻报（Dart 要用它开门控），之后按间隔汇总 |
|||
if self.lastDownlinkReportAt == 0 |
|||
|| now - self.lastDownlinkReportAt >= Self.downlinkReportInterval { |
|||
self.lastDownlinkReportAt = now |
|||
self.emit(["event": "downlink", "bytes": self.downlinkBytes]) |
|||
} |
|||
} |
|||
if !d.isEmpty { self.onAudio?(d) } |
|||
case .string(let s): |
|||
self.handleText(s) |
|||
@unknown default: |
|||
break |
|||
} |
|||
self.receiveLoop() |
|||
} |
|||
} |
|||
} |
|||
|
|||
private func handleText(_ raw: String) { |
|||
guard let data = raw.data(using: .utf8), |
|||
let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { |
|||
NSLog("[BailianDialog] 无法解析的文本帧") |
|||
return |
|||
} |
|||
let header = obj["header"] as? [String: Any] ?? [:] |
|||
let payload = obj["payload"] as? [String: Any] ?? [:] |
|||
let event = (header["event"] as? String) ?? "" |
|||
|
|||
switch event { |
|||
case "task-started": |
|||
NSLog("[BailianDialog] task-started") |
|||
case "task-failed": |
|||
let code = (header["error_code"] as? String) ?? "" |
|||
let msg = (header["error_message"] as? String) ?? "" |
|||
NSLog("[BailianDialog] task-failed: %@ %@", code, msg) |
|||
// header 级 = 整个 task 结束，必须重连 |
|||
emit(["event": "error", "text": msg, "code": code, "fatal": true]) |
|||
case "task-finished": |
|||
NSLog("[BailianDialog] task-finished") |
|||
emit(["event": "closed"]) |
|||
default: |
|||
handleDirectiveEvent(event, payload) |
|||
} |
|||
} |
|||
|
|||
/// 业务事件都在 payload.output.event 里 |
|||
private func handleDirectiveEvent(_ outerEvent: String, _ payload: [String: Any]) { |
|||
let output = payload["output"] as? [String: Any] ?? [:] |
|||
let name = (output["event"] as? String) ?? outerEvent |
|||
let text = (output["text"] as? String) ?? "" |
|||
let finished = (output["finished"] as? Bool) ?? false |
|||
|
|||
switch name { |
|||
case "Started": |
|||
let did = (output["dialog_id"] as? String) ?? "" |
|||
queue.async { self.dialogId = did } |
|||
NSLog("[BailianDialog] Started dialog_id=%@", did) |
|||
emit(["event": "started"]) |
|||
case "DialogStateChanged": |
|||
// 协议里是首字母大写（Listening/Thinking/Responding），统一小写再交给 Dart |
|||
let s = ((output["state"] as? String) ?? "").lowercased() |
|||
emit(["event": "stateChanged", "state": s]) |
|||
case "SpeechStarted": |
|||
emit(["event": "speechStarted"]) |
|||
case "SpeechContent": |
|||
emit(["event": "speechContent", "text": text, "isFinal": finished]) |
|||
case "SpeechEnded": |
|||
emit(["event": "speechEnded", "text": text, "isFinal": true]) |
|||
case "RequestAccepted": |
|||
emit(["event": "requestAccepted"]) |
|||
case "RespondingStarted": |
|||
emit(["event": "respondingStarted"]) |
|||
case "RespondingContent": |
|||
let reason = (output["finish_reason"] as? String) ?? "" |
|||
// 指令解析留在 Dart（见类注释）：这里把 output 原样带上去。 |
|||
// extra_info 只在整轮收尾那一帧出现，不是每个流式分片都有，频率可控。 |
|||
var e: [String: Any] = [ |
|||
"event": "respondingContent", |
|||
"text": text, |
|||
"isFinal": finished, |
|||
"finishReason": reason, |
|||
] |
|||
if let extra = output["extra_info"] { |
|||
e["extraInfo"] = Self.jsonString(extra) |
|||
} |
|||
emit(e) |
|||
case "RespondingEnded": |
|||
emit(["event": "respondingEnded", "text": text, "isFinal": true]) |
|||
case "Error": |
|||
let code = (output["error_code"] as? String) ?? "" |
|||
let msg = (output["error_message"] as? String) ?? "" |
|||
let name0 = (output["error_name"] as? String) ?? "" |
|||
NSLog("[BailianDialog] Error: %@/%@ %@", code, name0, msg) |
|||
// payload 级 Error 是"这一轮不行"，连接通常还在，不标 fatal |
|||
let shown = !msg.isEmpty ? msg : (!text.isEmpty ? text : name0) |
|||
emit(["event": "error", "text": shown, "code": code, "fatal": false]) |
|||
case "Stopped": |
|||
emit(["event": "closed"]) |
|||
case "HeartBeat": |
|||
break |
|||
default: |
|||
NSLog("[BailianDialog] 未处理事件: %@", name) |
|||
} |
|||
} |
|||
|
|||
// MARK: - 定时器 |
|||
|
|||
private func startTimers() { |
|||
let hb = DispatchSource.makeTimerSource(queue: queue) |
|||
hb.schedule(deadline: .now() + Self.heartbeatInterval, repeating: Self.heartbeatInterval) |
|||
hb.setEventHandler { [weak self] in self?.sendDirectiveLocked("HeartBeat") } |
|||
hb.resume() |
|||
heartbeatTimer = hb |
|||
|
|||
let pg = DispatchSource.makeTimerSource(queue: queue) |
|||
pg.schedule(deadline: .now() + Self.pingInterval, repeating: Self.pingInterval) |
|||
pg.setEventHandler { [weak self] in |
|||
self?.task?.sendPing { err in |
|||
if let e = err { NSLog("[BailianDialog] ping 失败: %@", e.localizedDescription) } |
|||
} |
|||
} |
|||
pg.resume() |
|||
pingTimer = pg |
|||
} |
|||
|
|||
private func cleanupLocked() { |
|||
heartbeatTimer?.cancel() |
|||
heartbeatTimer = nil |
|||
pingTimer?.cancel() |
|||
pingTimer = nil |
|||
task = nil |
|||
session?.invalidateAndCancel() |
|||
session = nil |
|||
dialogId = "" |
|||
running = false |
|||
} |
|||
|
|||
// MARK: - 工具 |
|||
|
|||
private func emit(_ e: [String: Any]) { |
|||
let cb = onEvent |
|||
DispatchQueue.main.async { cb?(e) } |
|||
} |
|||
|
|||
/// 把任意 JSON 结构转回字符串交给 Dart 解析（保持与 Dart 版同一份解析代码） |
|||
private static func jsonString(_ v: Any) -> String { |
|||
guard JSONSerialization.isValidJSONObject(v), |
|||
let d = try? JSONSerialization.data(withJSONObject: v), |
|||
let s = String(data: d, encoding: .utf8) else { return "" } |
|||
return s |
|||
} |
|||
|
|||
/// 协议限制 user_id 不超过 36 字符 |
|||
private static func safeUserId(_ raw: String) -> String { |
|||
let v = raw.trimmingCharacters(in: .whitespaces) |
|||
if v.isEmpty { return "eaimar-guest" } |
|||
return v.count <= 36 ? v : String(v.prefix(36)) |
|||
} |
|||
|
|||
private static func uuid() -> String { |
|||
func hex(_ n: Int) -> String { |
|||
(0..<n).map { _ in String(Int.random(in: 0..<16), radix: 16) }.joined() |
|||
} |
|||
return "\(hex(8))-\(hex(4))-4\(hex(3))-a\(hex(3))-\(hex(12))" |
|||
} |
|||
} |
|||
@ -0,0 +1,206 @@ |
|||
import Foundation |
|||
|
|||
/// EMAI 会话 PCM 的**原生直连桥**（azure_speech 这一端）。 |
|||
/// |
|||
/// 对端是 bluetooth_manager 的 `AiPcmBridge.swift`：耳机解码出来的 AI 上行 PCM 经 |
|||
/// NotificationCenter 直接到这里（不再经过 Dart），这里攒成 100ms 一包推给 |
|||
/// [BailianDialogClient]；百炼合成回来的 PCM 由 [postDownlink] 直接回给 |
|||
/// bluetooth_manager 的下行发送器。 |
|||
/// |
|||
/// ## ⚠️ 与 bluetooth_manager 手工同步的契约（通知名 / userInfo 键） |
|||
/// - 上行 `com.eaimar.bes.aipcm.uplink` `pcm`(Data, 16k/16bit/mono) |
|||
/// - 下行 `com.eaimar.bes.aipcm.downlink` `pcm`(Data) |
|||
/// 改一处必须同改另一处；改漏了不会编译报错，只会「桥开着但一帧都到不了」。 |
|||
/// |
|||
/// ## 门控为什么是从 Dart 推下来的布尔量 |
|||
/// 「AI 在耳机里播报时把上行换成等长静音」这件事本身必须在这里做（逐帧）； |
|||
/// 但**什么时候开关门**留在 Dart 的 `DeviceAiSessionService`：那套逻辑有兜底计时、 |
|||
/// 非致命错误要关门、纯指令帧收尾也要关门等一串来之不易的分支（门卡开着的后果是 |
|||
/// 「会话活着但是聋的」）。门一轮只切换几次，一次 MethodChannel 调用的开销可以忽略， |
|||
/// 没必要把那套状态机也搬过来、再在两端各维护一份。 |
|||
/// |
|||
/// ## 线程与背压 |
|||
/// 通知在耳机解码线程上同步到达，这里**立刻转到自己的串行队列**，不让 WS 发送拖住 |
|||
/// 解码线程。排队超过 [maxPendingPackets]（约 2s）就丢新包并计数——同样的拥塞， |
|||
/// 现在有数字、有日志，丢的是包而不是整个会话。 |
|||
final class BesAiPcmBridge { |
|||
static let uplinkNotification = Notification.Name("com.eaimar.bes.aipcm.uplink") |
|||
static let downlinkNotification = Notification.Name("com.eaimar.bes.aipcm.downlink") |
|||
static let pcmKey = "pcm" |
|||
|
|||
/// 上行攒包大小：16k/16bit/mono 的 100ms = 3200 字节。 |
|||
/// ⚠️ 与 Dart `DeviceAiSessionService._micBatchBytes` 相同，两处要同改—— |
|||
/// 百炼建议每包 ~100ms，改小了包多、改大了首字延迟上升。 |
|||
private static let batchBytes = 3200 |
|||
|
|||
/// 说话人能量门限与上报节流。 |
|||
/// ⚠️ 与 Dart 的 `_micActivityRms` / `_micActivityThrottle` 相同，三处（Dart/本类/ |
|||
/// Kotlin 版）要同改。它只用来给 Dart 续空闲计时，不参与切句。 |
|||
private static let activityRms = 600.0 |
|||
private static let activityThrottle: TimeInterval = 0.5 |
|||
|
|||
/// 约 2 秒的积压上限（100ms 一包） |
|||
private static let maxPendingPackets = 20 |
|||
|
|||
private let ingress = DispatchQueue(label: "azure_speech.aipcm.ingress", qos: .userInitiated) |
|||
private let lock = NSLock() |
|||
|
|||
private var enabled = false |
|||
private var observer: NSObjectProtocol? |
|||
private var accum = Data() |
|||
private var pending = 0 |
|||
private var gated = false |
|||
private var lastActivityAt: TimeInterval = 0 |
|||
|
|||
// 统计（只在 ingress 队列上写） |
|||
private var framesIn: Int64 = 0 |
|||
private var packetsOut: Int64 = 0 |
|||
private var dropped: Int64 = 0 |
|||
private var downlinkFrames: Int64 = 0 |
|||
|
|||
/// 攒够一包就交给它（百炼 WS 客户端） |
|||
var onPacket: ((Data) -> Void)? |
|||
|
|||
/// 用户在说话（能量过门限、且不在门控中），已按 [activityThrottle] 节流。 |
|||
/// Dart 用它给空闲计时续命——设备上行有声也算活动，不然锁屏 + 蓝牙延迟下 |
|||
/// 很容易被判「双方无声」而提前收尾。 |
|||
var onMicActivity: (() -> Void)? |
|||
|
|||
/// 背压丢包时上报一次（累计丢包数） |
|||
var onOverflow: ((Int64) -> Void)? |
|||
|
|||
var isEnabled: Bool { |
|||
lock.lock(); defer { lock.unlock() } |
|||
return enabled |
|||
} |
|||
|
|||
func enable() { |
|||
lock.lock() |
|||
if enabled { lock.unlock(); return } |
|||
enabled = true |
|||
lock.unlock() |
|||
ingress.async { |
|||
self.accum.removeAll(keepingCapacity: true) |
|||
self.pending = 0 |
|||
self.gated = false |
|||
self.lastActivityAt = 0 |
|||
self.framesIn = 0 |
|||
self.packetsOut = 0 |
|||
self.dropped = 0 |
|||
self.downlinkFrames = 0 |
|||
} |
|||
observer = NotificationCenter.default.addObserver( |
|||
forName: Self.uplinkNotification, object: nil, queue: nil |
|||
) { [weak self] note in |
|||
guard let self = self, |
|||
let pcm = note.userInfo?[Self.pcmKey] as? Data, !pcm.isEmpty else { return } |
|||
self.ingest(pcm) |
|||
} |
|||
NSLog("[BesAiPcmBridge] AI 上行直连已开启") |
|||
} |
|||
|
|||
func disable() { |
|||
lock.lock() |
|||
if !enabled { lock.unlock(); return } |
|||
enabled = false |
|||
lock.unlock() |
|||
if let o = observer { |
|||
NotificationCenter.default.removeObserver(o) |
|||
observer = nil |
|||
} |
|||
ingress.async { |
|||
self.accum.removeAll(keepingCapacity: false) |
|||
NSLog("[BesAiPcmBridge] AI 上行直连已关闭 上行=%lld 帧 出包=%lld 丢包=%lld 下行=%lld 帧", |
|||
self.framesIn, self.packetsOut, self.dropped, self.downlinkFrames) |
|||
} |
|||
} |
|||
|
|||
/// AI 是否正在耳机里播报。播报期间上行换成**等长静音**而不是断流： |
|||
/// 上行保持连续，服务端 VAD 收到的是"真的没人说话"，而不是一段数据缺口。 |
|||
func setGated(_ v: Bool) { |
|||
ingress.async { |
|||
self.gated = v |
|||
// 关门时把攒了一半的静音丢掉，下一包从真实音频开始， |
|||
// 免得用户开口的头 100ms 里混着上一轮的静音 |
|||
if !v { self.accum.removeAll(keepingCapacity: true) } |
|||
} |
|||
} |
|||
|
|||
// MARK: - 上行 |
|||
|
|||
private func ingest(_ pcm: Data) { |
|||
// 背压：排队太多就丢新包。判断放在入队前，不然队列本身就是那个积压 |
|||
lock.lock() |
|||
let p = pending |
|||
lock.unlock() |
|||
if p >= Self.maxPendingPackets { |
|||
ingress.async { |
|||
self.dropped &+= 1 |
|||
let n = self.dropped |
|||
if n == 1 || n % 50 == 0 { |
|||
NSLog("[BesAiPcmBridge] 上行拥塞，已丢 %lld 包", n) |
|||
} |
|||
self.onOverflow?(n) |
|||
} |
|||
return |
|||
} |
|||
lock.lock(); pending += 1; lock.unlock() |
|||
ingress.async { |
|||
self.lock.lock(); self.pending -= 1; self.lock.unlock() |
|||
self.framesIn &+= 1 |
|||
if self.gated { |
|||
// 等长静音 |
|||
self.accum.append(Data(count: pcm.count)) |
|||
} else { |
|||
if Self.rms(pcm) >= Self.activityRms { |
|||
let now = Date().timeIntervalSince1970 |
|||
if now - self.lastActivityAt >= Self.activityThrottle { |
|||
self.lastActivityAt = now |
|||
self.onMicActivity?() |
|||
} |
|||
} |
|||
self.accum.append(pcm) |
|||
} |
|||
while self.accum.count >= Self.batchBytes { |
|||
let pkt = self.accum.prefix(Self.batchBytes) |
|||
self.accum = Data(self.accum.dropFirst(Self.batchBytes)) |
|||
self.packetsOut &+= 1 |
|||
self.onPacket?(Data(pkt)) |
|||
} |
|||
} |
|||
} |
|||
|
|||
// MARK: - 下行 |
|||
|
|||
/// 把一块合成音频交回 bluetooth_manager 的下行发送器（G.722 编码与节流在那边做） |
|||
func postDownlink(_ pcm: Data) { |
|||
guard !pcm.isEmpty else { return } |
|||
NotificationCenter.default.post(name: Self.downlinkNotification, object: nil, |
|||
userInfo: [Self.pcmKey: pcm]) |
|||
ingress.async { self.downlinkFrames &+= 1 } |
|||
} |
|||
|
|||
func stats() -> [String: Any] { |
|||
ingress.sync { |
|||
["enabled": isEnabled, "framesIn": framesIn, "packetsOut": packetsOut, |
|||
"dropped": dropped, "downlinkFrames": downlinkFrames, "gated": gated] |
|||
} |
|||
} |
|||
|
|||
/// 16bit 小端 PCM 的 RMS（0~32767 量纲，与 Dart 的 `rmsOfPcm16` 同口径） |
|||
private static func rms(_ pcm: Data) -> Double { |
|||
let count = pcm.count / 2 |
|||
guard count > 0 else { return 0 } |
|||
var sum = 0.0 |
|||
pcm.withUnsafeBytes { raw in |
|||
guard let base = raw.baseAddress?.assumingMemoryBound(to: UInt8.self) else { return } |
|||
for i in 0..<count { |
|||
// 逐字节拼，不用 Int16 对齐读：上游 Data 可能是带奇数偏移的切片 |
|||
var v = Int(base[i * 2 + 1]) << 8 | Int(base[i * 2]) |
|||
if v >= 0x8000 { v -= 0x10000 } |
|||
sum += Double(v) * Double(v) |
|||
} |
|||
} |
|||
return (sum / Double(count)).squareRoot() |
|||
} |
|||
} |
|||
@ -0,0 +1,208 @@ |
|||
import Foundation |
|||
|
|||
/// 通话翻译 PCM 的**原生直连桥**（azure_speech 这一端）。 |
|||
/// |
|||
/// 对端是 bluetooth_manager 的 `CallPcmBridge.swift`：耳机解码出来的双路 PCM 经 |
|||
/// NotificationCenter 直接到这里（不再经过 Dart），这里过 B 路说话人门限后推给 AST helper； |
|||
/// AST 合成出来的译文 PCM 由 [postDownlink] 直接回给 bluetooth_manager 的下行发送器。 |
|||
/// |
|||
/// ## ⚠️ 与 bluetooth_manager 手工同步的契约（通知名 / userInfo 键） |
|||
/// - 上行 `com.eaimar.bes.callpcm.uplink` `leg`("A"=本端麦 / "B"=对端通话音), `pcm`(Data) |
|||
/// - 下行 `com.eaimar.bes.callpcm.downlink` `leg`("A"=对端听 / "B"=己方听), `pcm`(Data) |
|||
/// 改一处必须同改另一处；改漏了不会编译报错，只会「桥开着但一帧都到不了」。 |
|||
/// |
|||
/// ## 线程与背压 |
|||
/// 通知在耳机解码线程上同步到达，这里**立刻转到自己的串行队列** [ingress]，不让 WS 发送 |
|||
/// 或 base64 编码拖住解码线程。队列里排队超过 [maxPendingFrames]（2s）就丢新帧并计数： |
|||
/// 这就是原来 Dart 链路上「AudioSendSlow 1011」的替代品——同样的拥塞，现在有数字、有日志、 |
|||
/// 而且丢的是帧而不是整个会话。 |
|||
/// |
|||
/// ## 重连期间不丢音频 |
|||
/// AST helper 在 initialize 期间会被 dispose 重建，它自己的 pending 缓冲随之清空。 |
|||
/// 所以「未就绪」期间的帧留在**这里**的环形缓冲（每路 [holdFrames]，3s）， |
|||
/// [markReady] 时整段补推——弱网重连的那 1~4 秒不再是纯丢音频。 |
|||
final class BesCallPcmBridge { |
|||
static let uplinkNotification = Notification.Name("com.eaimar.bes.callpcm.uplink") |
|||
static let downlinkNotification = Notification.Name("com.eaimar.bes.callpcm.downlink") |
|||
static let legKey = "leg" |
|||
static let pcmKey = "pcm" |
|||
|
|||
private static let maxPendingFrames = 100 |
|||
private static let holdFrames = 150 |
|||
|
|||
private let ingress = DispatchQueue(label: "azure_speech.callpcm.ingress", qos: .userInitiated) |
|||
private let lock = NSLock() |
|||
|
|||
private var enabled = false |
|||
private var ready = false |
|||
private var observer: NSObjectProtocol? |
|||
private var pending = 0 |
|||
private let gate = PeerSpeechGate() |
|||
private var holdA: [Data] = [] |
|||
private var holdB: [Data] = [] |
|||
|
|||
// 统计（只在 ingress 队列上写；读走 stats()） |
|||
private var framesA: Int64 = 0 |
|||
private var framesB: Int64 = 0 |
|||
private var dropped: Int64 = 0 |
|||
private var gatedB: Int64 = 0 |
|||
private var downlinkFrames: Int64 = 0 |
|||
private var maxPendingSeen = 0 |
|||
|
|||
/// 上行落点：A = 本端麦 → A 路 AST；B = 对端通话音 → B 路 AST（已过门限） |
|||
var onLegA: ((Data) -> Void)? |
|||
var onLegB: ((Data) -> Void)? |
|||
/// 门限开启（rms, 是否兜底强开），供上抛给 Dart 记日志——真机调阈值的唯一依据 |
|||
var onGateOpened: ((Int, Bool) -> Void)? |
|||
/// 背压丢帧时上报一次（累计丢帧数） |
|||
var onOverflow: ((Int64) -> Void)? |
|||
|
|||
var isEnabled: Bool { |
|||
lock.lock(); defer { lock.unlock() } |
|||
return enabled |
|||
} |
|||
|
|||
func enable() { |
|||
lock.lock() |
|||
if enabled { lock.unlock(); return } |
|||
enabled = true |
|||
lock.unlock() |
|||
ingress.async { self.resetCountersLocked() } |
|||
observer = NotificationCenter.default.addObserver( |
|||
forName: Self.uplinkNotification, object: nil, queue: nil |
|||
) { [weak self] note in |
|||
guard let self = self, |
|||
let leg = note.userInfo?[Self.legKey] as? String, |
|||
let pcm = note.userInfo?[Self.pcmKey] as? Data, !pcm.isEmpty else { return } |
|||
self.enqueue(leg: leg, pcm: pcm) |
|||
} |
|||
NSLog("[BesCallPcmBridge] 已开启：上行经通知直达，下行直回发送器") |
|||
} |
|||
|
|||
func disable() { |
|||
lock.lock() |
|||
if !enabled { lock.unlock(); return } |
|||
enabled = false |
|||
lock.unlock() |
|||
if let o = observer { |
|||
NotificationCenter.default.removeObserver(o) |
|||
observer = nil |
|||
} |
|||
ingress.async { |
|||
NSLog("[BesCallPcmBridge] 已关闭 A=\(self.framesA) B=\(self.framesB) 丢帧=\(self.dropped) B门限前=\(self.gatedB) 下行=\(self.downlinkFrames) 峰值积压=\(self.maxPendingSeen)") |
|||
self.holdA.removeAll(); self.holdB.removeAll() |
|||
self.ready = false |
|||
} |
|||
} |
|||
|
|||
/// AST 会话（重）建立开始：新会话要重新刻音色，门限重新关上；帧先攒着等 [markReady]。 |
|||
/// |
|||
/// ⚠️ 只清 B 路的缓存：那些帧是在**旧**门限状态下放行的，门限复位后必须由现场音频 |
|||
/// 重新判定「对端开口了没有」，否则重连后照样可能刻到串音。A 路是机主自己的麦， |
|||
/// 没有门限，攒下的整段都要补推——这正是「重连那几秒不丢音频」的价值所在。 |
|||
func onSessionInit() { |
|||
ingress.async { |
|||
self.ready = false |
|||
self.gate.reset() |
|||
self.holdB.removeAll() |
|||
} |
|||
} |
|||
|
|||
/// AST 会话销毁：帧继续攒着（重连场景），直到下一次 markReady。 |
|||
func onSessionDisposed() { |
|||
ingress.async { self.ready = false } |
|||
} |
|||
|
|||
/// A/B 两路 helper 都起来了：把攒下的整段补推出去，之后直通。 |
|||
func markReady() { |
|||
ingress.async { |
|||
let a = self.holdA, b = self.holdB |
|||
self.holdA.removeAll(); self.holdB.removeAll() |
|||
self.ready = true |
|||
if !a.isEmpty || !b.isEmpty { |
|||
NSLog("[BesCallPcmBridge] 就绪，补推积攒的 A=\(a.count) B=\(b.count) 帧") |
|||
} |
|||
for f in a { self.onLegA?(f) } |
|||
for f in b { self.onLegB?(f) } |
|||
} |
|||
} |
|||
|
|||
/// 由 AST 的 TTS 输出调用：译文 PCM 直接回给 bluetooth_manager 的下行发送器。 |
|||
func postDownlink(leg: String, pcm: Data) { |
|||
guard !pcm.isEmpty else { return } |
|||
NotificationCenter.default.post(name: Self.downlinkNotification, object: nil, |
|||
userInfo: [Self.legKey: leg, Self.pcmKey: pcm]) |
|||
ingress.async { self.downlinkFrames &+= 1 } |
|||
} |
|||
|
|||
func stats() -> [String: Any] { |
|||
var out: [String: Any] = [:] |
|||
ingress.sync { |
|||
out = [ |
|||
"enabled": self.enabled, "ready": self.ready, |
|||
"framesA": self.framesA, "framesB": self.framesB, |
|||
"dropped": self.dropped, "gatedB": self.gatedB, |
|||
"downlinkFrames": self.downlinkFrames, "maxPending": self.maxPendingSeen, |
|||
"gateOpen": self.gate.isOpen, "gateFallback": self.gate.openedByFallback, |
|||
] |
|||
} |
|||
return out |
|||
} |
|||
|
|||
// MARK: - 私有 |
|||
|
|||
private func resetCountersLocked() { |
|||
framesA = 0; framesB = 0; dropped = 0; gatedB = 0; downlinkFrames = 0; maxPendingSeen = 0 |
|||
} |
|||
|
|||
private func enqueue(leg: String, pcm: Data) { |
|||
lock.lock() |
|||
if pending >= Self.maxPendingFrames { |
|||
dropped &+= 1 |
|||
let d = dropped |
|||
lock.unlock() |
|||
if d == 1 || d % 100 == 0 { |
|||
NSLog("[BesCallPcmBridge] 上行积压超过 \(Self.maxPendingFrames) 帧，丢帧累计 \(d)") |
|||
onOverflow?(d) |
|||
} |
|||
return |
|||
} |
|||
pending += 1 |
|||
if pending > maxPendingSeen { maxPendingSeen = pending } |
|||
lock.unlock() |
|||
ingress.async { [self] in |
|||
lock.lock(); pending -= 1; lock.unlock() |
|||
handle(leg: leg, pcm: pcm) |
|||
} |
|||
} |
|||
|
|||
/// 只在 ingress 队列上调用 |
|||
private func handle(leg: String, pcm: Data) { |
|||
if leg == "A" { |
|||
framesA &+= 1 |
|||
deliver(a: pcm) |
|||
return |
|||
} |
|||
framesB &+= 1 |
|||
let wasOpen = gate.isOpen |
|||
let frames = gate.accept(pcm) |
|||
if frames.isEmpty { gatedB &+= 1 } |
|||
if !wasOpen && gate.isOpen { |
|||
NSLog("[BesCallPcmBridge] B 路门限开启 rms=\(gate.openRms) fallback=\(gate.openedByFallback)") |
|||
onGateOpened?(gate.openRms, gate.openedByFallback) |
|||
} |
|||
for f in frames { deliver(b: f) } |
|||
} |
|||
|
|||
private func deliver(a pcm: Data) { |
|||
if ready { onLegA?(pcm); return } |
|||
holdA.append(pcm) |
|||
if holdA.count > Self.holdFrames { holdA.removeFirst() } |
|||
} |
|||
|
|||
private func deliver(b pcm: Data) { |
|||
if ready { onLegB?(pcm); return } |
|||
holdB.append(pcm) |
|||
if holdB.count > Self.holdFrames { holdB.removeFirst() } |
|||
} |
|||
} |
|||
@ -0,0 +1,79 @@ |
|||
import Foundation |
|||
|
|||
/// 对端（B 路）说话人门限——Dart 版 `lib/data/utils/peer_speech_gate.dart` 的逐条移植。 |
|||
/// |
|||
/// **在对端真的开口之前，不要把音频推给端到端翻译。** 阿里的实时声音复刻是 |
|||
/// `frequency = "once"`，只在会话开头刻一次音色；此时对端往往还没开口，B 路先收到的是 |
|||
/// 机主声音的串音，刻错之后用户听到的对端译文用的是自己的声音。 |
|||
/// |
|||
/// - 未开门期间音频进环形缓冲（[prerollFrames]），开门瞬间整段补发——对端先开口也不吃掉开头； |
|||
/// - 连续 [openFrames] 帧 RMS ≥ [rmsThreshold] 才开门，断续的串音尖峰不算； |
|||
/// - 收满 [fallbackFrames] 仍未开门就**强行开门**（门限太严=整通电话不翻译，比刻错音色严重得多）； |
|||
/// - 每次会话重建（含断线重连）必须 [reset]。 |
|||
/// |
|||
/// ⚠️ 参数默认值与 Dart 版、Kotlin 版三处相同，调阈值要三处同改。 |
|||
/// 线程：调用方保证串行（BesCallPcmBridge 的 ingress 队列）。 |
|||
final class PeerSpeechGate { |
|||
let rmsThreshold: Int |
|||
let openFrames: Int |
|||
let prerollFrames: Int |
|||
let fallbackFrames: Int |
|||
|
|||
private(set) var isOpen = false |
|||
private(set) var openRms = 0 |
|||
private(set) var openedByFallback = false |
|||
private var run = 0 |
|||
private var seen = 0 |
|||
private var preroll: [Data] = [] |
|||
|
|||
init(rmsThreshold: Int = 600, openFrames: Int = 8, prerollFrames: Int = 25, fallbackFrames: Int = 500) { |
|||
self.rmsThreshold = rmsThreshold |
|||
self.openFrames = openFrames |
|||
self.prerollFrames = prerollFrames |
|||
self.fallbackFrames = fallbackFrames |
|||
} |
|||
|
|||
/// 喂一帧，返回应当推给翻译服务的帧（0 帧、1 帧，或开门瞬间的一整串）。 |
|||
func accept(_ frame: Data) -> [Data] { |
|||
if isOpen { return [frame] } |
|||
seen += 1 |
|||
let rms = PeerSpeechGate.rms(ofPcm16: frame) |
|||
if rms >= rmsThreshold { run += 1 } else { run = 0 } |
|||
preroll.append(frame) |
|||
while preroll.count > prerollFrames { preroll.removeFirst() } |
|||
let hitThreshold = run >= openFrames |
|||
let hitFallback = seen >= fallbackFrames |
|||
if !hitThreshold && !hitFallback { return [] } |
|||
isOpen = true |
|||
openRms = rms |
|||
openedByFallback = !hitThreshold |
|||
let out = preroll |
|||
preroll.removeAll() |
|||
return out |
|||
} |
|||
|
|||
func reset() { |
|||
isOpen = false |
|||
run = 0 |
|||
seen = 0 |
|||
openRms = 0 |
|||
openedByFallback = false |
|||
preroll.removeAll() |
|||
} |
|||
|
|||
/// PCM16 小端单声道的 RMS；字节数为奇数时忽略末尾半个样本。 |
|||
static func rms(ofPcm16 frame: Data) -> Int { |
|||
let n = frame.count / 2 |
|||
if n == 0 { return 0 } |
|||
var sum: Double = 0 |
|||
frame.withUnsafeBytes { (raw: UnsafeRawBufferPointer) in |
|||
let base = raw.baseAddress! |
|||
for i in 0..<n { |
|||
let s = Double(Int16(littleEndian: base.load(fromByteOffset: i * 2, as: Int16.self))) |
|||
sum += s * s |
|||
} |
|||
} |
|||
let mean = sum / Double(n) |
|||
return mean.isFinite ? Int(mean.squareRoot().rounded()) : 0 |
|||
} |
|||
} |
|||
@ -0,0 +1,335 @@ |
|||
package com.example.bluetooth_manager |
|||
|
|||
import android.util.Log |
|||
import java.io.File |
|||
import java.io.RandomAccessFile |
|||
import java.util.concurrent.ConcurrentHashMap |
|||
import java.util.concurrent.Executors |
|||
import kotlin.math.min |
|||
import kotlin.math.sqrt |
|||
|
|||
/** |
|||
* 设备音频直接落盘的 WAV 录制器（16bit 小端，单声道或双声道交织）。 |
|||
* |
|||
* **为什么在原生**：此前是每帧 PCM 经 EventChannel 到 Dart、由 `WavWriter` 用 |
|||
* `RandomAccessFile` 排队写盘。恒玄 20ms 一帧 = 单路 50 帧/s、通话双路 100 帧/s， |
|||
* 每帧都要跨一次平台通道并在 Dart 的单 isolate 上排队——event loop 一被页面布局 |
|||
* 或 GC 占住，写入队列就堆积。`WavWriter` 的注释里记着这条链路出过的事故： |
|||
* 写入重叠导致丢帧，**等效采样率只剩 1587Hz**。现在帧在原生解码完就地写文件。 |
|||
* |
|||
* 与 iOS 的 `PcmWavRecorder.swift` 逐条对齐，改一边必须同改另一边。 |
|||
*/ |
|||
class PcmWavRecorder( |
|||
val id: String, |
|||
private val path: String, |
|||
channels: Int, |
|||
private val sampleRate: Int, |
|||
) { |
|||
companion object { |
|||
private const val TAG = "PcmWavRecorder" |
|||
|
|||
/** 统计上报间隔，与 Dart 侧波形采样同频 */ |
|||
private const val STATS_INTERVAL_MS = 100L |
|||
|
|||
/** 一路长期缺席时另一路的最大堆积：10 秒 @16k/16bit */ |
|||
private const val MAX_PENDING = 16000 * 2 * 10 |
|||
|
|||
/** |
|||
* 与 Dart `WavWriter.rmsLevel` 同口径：归一化 RMS 再开一次根 |
|||
* （语音 RMS 常年在 0.02~0.3，不拉一下动态范围界面几乎没动静）。 |
|||
*/ |
|||
fun rmsLevel(pcm: ByteArray, length: Int = pcm.size): Double { |
|||
val count = length / 2 |
|||
if (count == 0) return 0.0 |
|||
var sum = 0.0 |
|||
for (i in 0 until count) { |
|||
// 高字节 toInt() 本身就带符号扩展，`shl 8` 再 or 上无符号低字节， |
|||
// 结果直接落在 -32768..32767，不需要再减 0x10000（Dart 那边因为 |
|||
// Uint8List 的元素是无符号的，才要手工补那一步） |
|||
val v = (pcm[i * 2 + 1].toInt() shl 8) or (pcm[i * 2].toInt() and 0xFF) |
|||
val n = v / 32768.0 |
|||
sum += n * n |
|||
} |
|||
val rms = sqrt(sum / count) |
|||
return min(1.0, sqrt(rms)) |
|||
} |
|||
|
|||
/** 两路单声道按样本交织成双声道（左在前） */ |
|||
fun interleave(left: ByteArray, leftLen: Int, right: ByteArray, rightLen: Int): ByteArray { |
|||
val samples = min(leftLen, rightLen) / 2 |
|||
val out = ByteArray(samples * 4) |
|||
for (i in 0 until samples) { |
|||
val si = i * 2 |
|||
val di = i * 4 |
|||
out[di] = left[si] |
|||
out[di + 1] = left[si + 1] |
|||
out[di + 2] = right[si] |
|||
out[di + 3] = right[si + 1] |
|||
} |
|||
return out |
|||
} |
|||
|
|||
/** 44 字节标准 WAV 头 */ |
|||
fun wavHeader(dataLength: Int, channels: Int, sampleRate: Int): ByteArray { |
|||
val bits = 16 |
|||
val byteRate = sampleRate * channels * bits / 8 |
|||
val blockAlign = channels * bits / 8 |
|||
val h = ByteArray(44) |
|||
var p = 0 |
|||
fun ascii(s: String) { for (c in s) h[p++] = c.code.toByte() } |
|||
fun u32(v: Int) { |
|||
h[p++] = (v and 0xFF).toByte() |
|||
h[p++] = ((v shr 8) and 0xFF).toByte() |
|||
h[p++] = ((v shr 16) and 0xFF).toByte() |
|||
h[p++] = ((v shr 24) and 0xFF).toByte() |
|||
} |
|||
fun u16(v: Int) { |
|||
h[p++] = (v and 0xFF).toByte() |
|||
h[p++] = ((v shr 8) and 0xFF).toByte() |
|||
} |
|||
ascii("RIFF"); u32(36 + dataLength); ascii("WAVE") |
|||
ascii("fmt "); u32(16); u16(1); u16(channels) |
|||
u32(sampleRate); u32(byteRate); u16(blockAlign); u16(bits) |
|||
ascii("data"); u32(dataLength) |
|||
return h |
|||
} |
|||
} |
|||
|
|||
val channels: Int = channels.coerceIn(1, 2) |
|||
|
|||
/** 串行写线程。文件句柄只在这上面碰。 */ |
|||
private val exec = Executors.newSingleThreadExecutor { r -> Thread(r, "bes-pcmwav-$id") } |
|||
|
|||
private var raf: RandomAccessFile? = null |
|||
|
|||
@Volatile var dataBytes: Int = 0; private set |
|||
@Volatile var writtenFrames: Int = 0; private set |
|||
@Volatile var droppedFrames: Int = 0; private set |
|||
@Volatile private var closed = false |
|||
|
|||
/** 双声道的左右暂存（见 flushPairs 的移植说明） */ |
|||
private var leftBuf = ByteArray(0) |
|||
private var rightBuf = ByteArray(0) |
|||
|
|||
private var levelPeak = 0.0 |
|||
private var lastStatsAt = 0L |
|||
|
|||
/** 统计回调（在写线程上调用，接收方自己派发到主线程） */ |
|||
var onStats: ((String, Int, Double) -> Unit)? = null |
|||
|
|||
// ---------- 生命周期 ---------- |
|||
|
|||
/** 建文件并写占位头。返回 false 表示建不出来（目录不可写等）。 */ |
|||
fun start(): Boolean { |
|||
return try { |
|||
val f = File(path) |
|||
f.parentFile?.mkdirs() |
|||
val r = RandomAccessFile(f, "rw") |
|||
r.setLength(0) |
|||
r.write(wavHeader(0, channels, sampleRate)) |
|||
raf = r |
|||
dataBytes = 0 |
|||
writtenFrames = 0 |
|||
droppedFrames = 0 |
|||
lastStatsAt = System.currentTimeMillis() |
|||
true |
|||
} catch (e: Exception) { |
|||
Log.e(TAG, "建文件失败: ${e.message}") |
|||
false |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 停止。[save] = false 时删除文件（没收到任何音频那种）。 |
|||
* 会等写队列排空再回填 WAV 头，否则最后几帧会丢、头里的长度也对不上。 |
|||
*/ |
|||
fun stop(save: Boolean): Map<String, Any> { |
|||
if (closed) { |
|||
return mapOf("bytes" to dataBytes, "written" to writtenFrames, "dropped" to droppedFrames) |
|||
} |
|||
closed = true |
|||
// 排空：submit 一个空任务并等它跑完，前面排队的写入就都落盘了 |
|||
try { |
|||
exec.submit { }.get() |
|||
} catch (_: Exception) { |
|||
} |
|||
try { |
|||
val r = raf |
|||
raf = null |
|||
if (r != null) { |
|||
if (save) { |
|||
try { |
|||
r.seek(0) |
|||
r.write(wavHeader(dataBytes, channels, sampleRate)) |
|||
} catch (e: Exception) { |
|||
Log.e(TAG, "回填 WAV 头失败: ${e.message}") |
|||
} |
|||
} |
|||
r.close() |
|||
} |
|||
if (!save) File(path).delete() |
|||
} catch (e: Exception) { |
|||
Log.e(TAG, "关闭失败: ${e.message}") |
|||
} finally { |
|||
exec.shutdown() |
|||
} |
|||
return mapOf("bytes" to dataBytes, "written" to writtenFrames, "dropped" to droppedFrames) |
|||
} |
|||
|
|||
// ---------- 进料 ---------- |
|||
|
|||
/** 单声道：直接落盘 */ |
|||
fun append(pcm: ByteArray) { |
|||
if (pcm.isEmpty() || closed) return |
|||
submit { |
|||
trackLevel(pcm, pcm.size) |
|||
write(pcm) |
|||
emitStatsIfDue() |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* stop() 里 shutdown 之后仍可能有在途帧进来（解码线程与调用方不同线程）， |
|||
* 那时 execute 会抛 RejectedExecutionException。这是正常收尾，不是错误。 |
|||
*/ |
|||
private fun submit(task: () -> Unit) { |
|||
try { |
|||
exec.execute { |
|||
if (!closed) task() |
|||
} |
|||
} catch (_: Exception) { |
|||
} |
|||
} |
|||
|
|||
/** 双声道左路（本端麦） */ |
|||
fun appendLeft(pcm: ByteArray) = appendStereo(pcm, true) |
|||
|
|||
/** 双声道右路（对端） */ |
|||
fun appendRight(pcm: ByteArray) = appendStereo(pcm, false) |
|||
|
|||
private fun appendStereo(pcm: ByteArray, isLeft: Boolean) { |
|||
if (pcm.isEmpty() || closed) return |
|||
submit { |
|||
if (isLeft) { |
|||
leftBuf = pushCapped(leftBuf, pcm) |
|||
} else { |
|||
rightBuf = pushCapped(rightBuf, pcm) |
|||
} |
|||
flushPairs() |
|||
emitStatsIfDue() |
|||
} |
|||
} |
|||
|
|||
private fun pushCapped(buf: ByteArray, add: ByteArray): ByteArray { |
|||
val merged = buf + add |
|||
return if (merged.size > MAX_PENDING) merged.copyOfRange(merged.size - MAX_PENDING, merged.size) |
|||
else merged |
|||
} |
|||
|
|||
/** |
|||
* StereoPairer 的移植：两路都排队，谁都不丢，每次取出两边都已有的那一段 |
|||
* （长度取较短者、对齐到整样本）。 |
|||
* |
|||
* ⚠️ 不能写成「只留最后一帧对端、本端来一帧配一帧」：两路在 BLE 上是同一个帧里 |
|||
* 交替的 40 字节块，但到达这里已经是两次独立回调，顺序不保证严格交替。按 |
|||
* 「最后一帧」配对会把右声道断续抽掉，转写出来只剩自己这一方。Dart 侧的金测试 |
|||
* `test/stereo_pairer_test.dart` 定的就是这个行为，三处（Dart/Swift/Kotlin）同改。 |
|||
*/ |
|||
private fun flushPairs() { |
|||
val n = min(leftBuf.size, rightBuf.size) and 1.inv() |
|||
if (n <= 0) return |
|||
val l = leftBuf.copyOfRange(0, n) |
|||
val r = rightBuf.copyOfRange(0, n) |
|||
leftBuf = leftBuf.copyOfRange(n, leftBuf.size) |
|||
rightBuf = rightBuf.copyOfRange(n, rightBuf.size) |
|||
trackLevel(l, n) // 电平按本端（左声道）算，与 Dart 原行为一致 |
|||
write(interleave(l, n, r, n)) |
|||
} |
|||
|
|||
private fun write(data: ByteArray) { |
|||
val r = raf ?: return |
|||
try { |
|||
r.write(data) |
|||
dataBytes += data.size |
|||
writtenFrames++ |
|||
} catch (e: Exception) { |
|||
droppedFrames++ |
|||
Log.e(TAG, "写入失败(已丢 $droppedFrames): ${e.message}") |
|||
} |
|||
} |
|||
|
|||
// ---------- 电平与统计 ---------- |
|||
|
|||
private fun trackLevel(pcm: ByteArray, length: Int) { |
|||
val v = rmsLevel(pcm, length) |
|||
if (v > levelPeak) levelPeak = v |
|||
} |
|||
|
|||
private fun emitStatsIfDue() { |
|||
val now = System.currentTimeMillis() |
|||
if (now - lastStatsAt < STATS_INTERVAL_MS) return |
|||
lastStatsAt = now |
|||
val peak = levelPeak |
|||
levelPeak = 0.0 |
|||
onStats?.invoke(id, dataBytes, peak) |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 活跃录制器登记处。 |
|||
* |
|||
* `BluetoothManager` 解码完一帧就按类型喂给这里的录制器——**只是多挂一个 sink， |
|||
* 不拦截原有的 EventChannel 回调**。这点和原生直连桥（callPcmBridgeEnabled）不同： |
|||
* 那条是「桥开着就不再上抛」，因为通话翻译的 PCM 有唯一消费者；录音则可能与别的 |
|||
* 订阅方（比如正在跑的通话翻译）并存，拦了会把对方饿死。 |
|||
*/ |
|||
object PcmRecorderRegistry { |
|||
private val byId = ConcurrentHashMap<String, PcmWavRecorder>() |
|||
|
|||
/** |
|||
* 音频类型 → 录制器。类型串与 `BluetoothManager` 里发的一致： |
|||
* `mainMicData`（现场录音主麦）/ `micData`（通话本端）/ `spkData`（通话对端）。 |
|||
* value 的 Boolean：null=单声道，true=左路，false=右路。 |
|||
*/ |
|||
private val sinks = ConcurrentHashMap<String, MutableList<Pair<PcmWavRecorder, Boolean?>>>() |
|||
|
|||
val isEmpty: Boolean get() = byId.isEmpty() |
|||
|
|||
fun register(rec: PcmWavRecorder, monoSource: String?, left: String?, right: String?) { |
|||
byId[rec.id] = rec |
|||
monoSource?.let { sinks.getOrPut(it) { mutableListOf() }.add(rec to null) } |
|||
left?.let { sinks.getOrPut(it) { mutableListOf() }.add(rec to true) } |
|||
right?.let { sinks.getOrPut(it) { mutableListOf() }.add(rec to false) } |
|||
} |
|||
|
|||
fun unregister(id: String): PcmWavRecorder? { |
|||
val rec = byId.remove(id) ?: return null |
|||
val empties = mutableListOf<String>() |
|||
for ((k, v) in sinks) { |
|||
v.removeAll { it.first === rec } |
|||
if (v.isEmpty()) empties.add(k) |
|||
} |
|||
for (k in empties) sinks.remove(k) |
|||
return rec |
|||
} |
|||
|
|||
/** 由 `BluetoothManager` 在解码线程上调用。没有录制器时立刻返回。 */ |
|||
fun feed(type: String, pcm: ByteArray) { |
|||
val list = sinks[type] ?: return |
|||
for ((rec, side) in list) { |
|||
when (side) { |
|||
null -> rec.append(pcm) |
|||
true -> rec.appendLeft(pcm) |
|||
false -> rec.appendRight(pcm) |
|||
} |
|||
} |
|||
} |
|||
|
|||
/** 断连/停录时把还开着的全停掉，避免留下没回填头的半截文件 */ |
|||
fun stopAll(save: Boolean) { |
|||
val all = byId.values.toList() |
|||
byId.clear() |
|||
sinks.clear() |
|||
for (r in all) r.stop(save) |
|||
} |
|||
} |
|||
@ -0,0 +1,94 @@ |
|||
import Foundation |
|||
|
|||
/// EMAI 会话 PCM 的**原生直连桥**（bluetooth_manager 这一端）。 |
|||
/// |
|||
/// ## 为什么有这个 |
|||
/// 耳机按键唤醒的 AI 会话，音频原来是 |
|||
/// `本插件 → EventChannel → Dart → 百炼 WebSocket（Dart）→ Dart → MethodChannel → 本插件`。 |
|||
/// 上行 20ms 一帧（50 次/s），下行每个 WS 音频块一次，全都压在 Dart 的单 isolate 上， |
|||
/// 和页面布局、GC 排同一个 event loop。桥开着时上行 PCM 直接交给 azure_speech 里的 |
|||
/// 原生百炼客户端，合成回来的 PCM 也直接回到 [AiAudioDownlink]，**Dart 只收文本与状态事件**。 |
|||
/// |
|||
/// ## 与 [CallPcmBridge] 的关系 |
|||
/// 结构逐条对称，但**刻意分成两个文件、两组通知名**：通话翻译那条是已量产、已冻结的 |
|||
/// 链路，它的 leg A/B 有明确语义（本端/对端）；AI 只有一路。混用会让通话的 leg 语义 |
|||
/// 多出一个不属于它的取值，排查时分不清一帧到底属于哪条链路。 |
|||
/// |
|||
/// ## ⚠️ 传输为什么是 NotificationCenter |
|||
/// 本插件是 CocoaPods 包、azure_speech 是 SwiftPM 包，两边**不能互相 import** |
|||
/// （Android 那边是 Gradle 子工程，可以直接依赖，走的是类型化的静态槽）。 |
|||
/// NotificationCenter 的 post 是**同步、在发帖线程上**执行的，没有序列化、没有跨线程。 |
|||
/// |
|||
/// ## ⚠️ 两边手工同步的契约 |
|||
/// 通知名与 userInfo 键在 azure_speech 的 `BesAiPcmBridge.swift` 里各写了一份， |
|||
/// **改一处必须同改另一处**，改漏了不会编译报错，只会表现为「桥开着但一帧都到不了」。 |
|||
/// 契约： |
|||
/// - 上行 `com.eaimar.bes.aipcm.uplink` userInfo: `pcm`(Data, 16k/16bit/mono) |
|||
/// - 下行 `com.eaimar.bes.aipcm.downlink` userInfo: `pcm`(Data, 16k/16bit/mono) |
|||
public enum AiPcmBridge { |
|||
public static let uplinkNotification = Notification.Name("com.eaimar.bes.aipcm.uplink") |
|||
public static let downlinkNotification = Notification.Name("com.eaimar.bes.aipcm.downlink") |
|||
public static let pcmKey = "pcm" |
|||
|
|||
private static let lock = NSLock() |
|||
private static var enabled = false |
|||
private static var downlinkObserver: NSObjectProtocol? |
|||
private static var uplinkFrames: Int64 = 0 |
|||
private static var downlinkFrames: Int64 = 0 |
|||
|
|||
/// 桥是否开着。解码线程每帧都读它。 |
|||
public static var isEnabled: Bool { |
|||
lock.lock(); defer { lock.unlock() } |
|||
return enabled |
|||
} |
|||
|
|||
/// 开桥：之后 AI 模式的 micData 只走通知，不再进 Dart 的 ai EventChannel； |
|||
/// 同时开始收 azure_speech 发来的合成 PCM 喂给下行发送器。 |
|||
public static func enable() { |
|||
lock.lock() |
|||
if enabled { lock.unlock(); return } |
|||
enabled = true |
|||
uplinkFrames = 0 |
|||
downlinkFrames = 0 |
|||
lock.unlock() |
|||
downlinkObserver = NotificationCenter.default.addObserver( |
|||
forName: downlinkNotification, object: nil, queue: nil |
|||
) { note in |
|||
guard let pcm = note.userInfo?[pcmKey] as? Data, !pcm.isEmpty else { return } |
|||
AiAudioDownlink.shared.pushPcm(pcm) |
|||
lock.lock(); downlinkFrames &+= 1; lock.unlock() |
|||
} |
|||
NSLog("[AiPcmBridge] AI 原生直连桥已开启（上行走通知，下行直喂发送器）") |
|||
} |
|||
|
|||
public static func disable() { |
|||
lock.lock() |
|||
if !enabled { lock.unlock(); return } |
|||
enabled = false |
|||
let up = uplinkFrames, down = downlinkFrames |
|||
lock.unlock() |
|||
if let o = downlinkObserver { |
|||
NotificationCenter.default.removeObserver(o) |
|||
downlinkObserver = nil |
|||
} |
|||
NSLog("[AiPcmBridge] AI 原生直连桥已关闭 上行=\(up) 帧 下行=\(down) 帧") |
|||
} |
|||
|
|||
/// 由 BluetoothManager 的解码线程调用：把一帧上行 PCM 交给 azure_speech。 |
|||
static func postUplink(pcm: Data) { |
|||
NotificationCenter.default.post(name: uplinkNotification, object: nil, |
|||
userInfo: [pcmKey: pcm]) |
|||
lock.lock() |
|||
uplinkFrames &+= 1 |
|||
let n = uplinkFrames |
|||
lock.unlock() |
|||
if n == 1 || n % 1000 == 0 { |
|||
NSLog("[AiPcmBridge] 上行已直连转发 \(n) 帧") |
|||
} |
|||
} |
|||
|
|||
public static func stats() -> [String: Any] { |
|||
lock.lock(); defer { lock.unlock() } |
|||
return ["enabled": enabled, "uplinkFrames": uplinkFrames, "downlinkFrames": downlinkFrames] |
|||
} |
|||
} |
|||
@ -0,0 +1,94 @@ |
|||
import Foundation |
|||
|
|||
/// 通话翻译 PCM 的**原生直连桥**（bluetooth_manager 这一端）。 |
|||
/// |
|||
/// ## 为什么有这个 |
|||
/// 通话翻译的音频原来是 `本插件 → EventChannel → Dart → MethodChannel → azure_speech`， |
|||
/// 20ms 一帧、双路，**每秒约 200 次跨界**。2026-09-20 苹果测试反馈「WiFi 下断断续续、 |
|||
/// 长句基本不行」，日志里就是 Dart event loop 被拖住之后的 `AudioSendSlow 1011`。 |
|||
/// 桥开着时，解码出来的 micData/spkData **不再经过 Dart**，直接交给 azure_speech； |
|||
/// azure_speech 合成出来的译文 PCM 也直接回到 [CallTranslationDownlink]。 |
|||
/// |
|||
/// ## ⚠️ 传输为什么是 NotificationCenter |
|||
/// 本插件是 CocoaPods 包、azure_speech 是 SwiftPM 包，两边**不能互相 import** |
|||
/// （Android 那边是 Gradle 子工程，可以直接依赖，走的是类型化的静态槽）。 |
|||
/// NotificationCenter 的 post 是**同步、在发帖线程上**执行的，没有序列化、没有跨线程， |
|||
/// 开销只是一次字典分配，对 100 帧/秒来说可以忽略。 |
|||
/// |
|||
/// ## ⚠️ 两边手工同步的契约 |
|||
/// 通知名与 userInfo 键在 azure_speech 的 `BesCallPcmBridge.swift` 里各写了一份， |
|||
/// **改一处必须同改另一处**，改漏了不会编译报错，只会表现为「桥开着但一帧都到不了」。 |
|||
/// 契约： |
|||
/// - 上行 `com.eaimar.bes.callpcm.uplink` userInfo: `leg`("A"=本端麦 / "B"=对端通话音), `pcm`(Data, 16k/16bit/mono) |
|||
/// - 下行 `com.eaimar.bes.callpcm.downlink` userInfo: `leg`("A"=对端听 / "B"=己方听), `pcm`(Data) |
|||
public enum CallPcmBridge { |
|||
public static let uplinkNotification = Notification.Name("com.eaimar.bes.callpcm.uplink") |
|||
public static let downlinkNotification = Notification.Name("com.eaimar.bes.callpcm.downlink") |
|||
public static let legKey = "leg" |
|||
public static let pcmKey = "pcm" |
|||
|
|||
private static let lock = NSLock() |
|||
private static var enabled = false |
|||
private static var downlinkObserver: NSObjectProtocol? |
|||
private static var uplinkFrames: Int64 = 0 |
|||
private static var downlinkFrames: Int64 = 0 |
|||
|
|||
/// 桥是否开着。解码线程每帧都读它，用锁而不是 atomic 是因为 Swift 没有内建 atomic， |
|||
/// 而 NSLock 的无争用开销是纳秒级。 |
|||
public static var isEnabled: Bool { |
|||
lock.lock(); defer { lock.unlock() } |
|||
return enabled |
|||
} |
|||
|
|||
/// 开桥：之后 micData/spkData 只走通知，不再进 Dart 的 audio EventChannel； |
|||
/// 同时开始收 azure_speech 发来的译文 PCM 喂给下行发送器。 |
|||
public static func enable() { |
|||
lock.lock() |
|||
if enabled { lock.unlock(); return } |
|||
enabled = true |
|||
uplinkFrames = 0 |
|||
downlinkFrames = 0 |
|||
lock.unlock() |
|||
downlinkObserver = NotificationCenter.default.addObserver( |
|||
forName: downlinkNotification, object: nil, queue: nil |
|||
) { note in |
|||
guard let leg = note.userInfo?[legKey] as? String, |
|||
let pcm = note.userInfo?[pcmKey] as? Data, !pcm.isEmpty else { return } |
|||
CallTranslationDownlink.shared.pushPcm(leg: leg, pcm: pcm) |
|||
lock.lock(); downlinkFrames &+= 1; lock.unlock() |
|||
} |
|||
NSLog("[CallPcmBridge] 原生直连桥已开启（上行走通知，下行直喂发送器）") |
|||
} |
|||
|
|||
public static func disable() { |
|||
lock.lock() |
|||
if !enabled { lock.unlock(); return } |
|||
enabled = false |
|||
let up = uplinkFrames, down = downlinkFrames |
|||
lock.unlock() |
|||
if let o = downlinkObserver { |
|||
NotificationCenter.default.removeObserver(o) |
|||
downlinkObserver = nil |
|||
} |
|||
NSLog("[CallPcmBridge] 原生直连桥已关闭 上行=\(up) 帧 下行=\(down) 帧") |
|||
} |
|||
|
|||
/// 由 BluetoothManager 的解码线程调用：把一帧上行 PCM 交给 azure_speech。 |
|||
/// [leg] "A" = 本端麦（micData）；"B" = 对端通话音（spkData）。 |
|||
static func postUplink(leg: String, pcm: Data) { |
|||
NotificationCenter.default.post(name: uplinkNotification, object: nil, |
|||
userInfo: [legKey: leg, pcmKey: pcm]) |
|||
lock.lock() |
|||
uplinkFrames &+= 1 |
|||
let n = uplinkFrames |
|||
lock.unlock() |
|||
if n == 1 || n % 1000 == 0 { |
|||
NSLog("[CallPcmBridge] 上行已直连转发 \(n) 帧") |
|||
} |
|||
} |
|||
|
|||
public static func stats() -> [String: Any] { |
|||
lock.lock(); defer { lock.unlock() } |
|||
return ["enabled": enabled, "uplinkFrames": uplinkFrames, "downlinkFrames": downlinkFrames] |
|||
} |
|||
} |
|||
@ -0,0 +1,365 @@ |
|||
import Foundation |
|||
|
|||
/// 设备音频直接落盘的 WAV 录制器（16bit 小端，单声道或双声道交织）。 |
|||
/// |
|||
/// **为什么在原生**：此前是每帧 PCM 经 EventChannel 到 Dart、由 `WavWriter` |
|||
/// 用 `RandomAccessFile` 排队写盘。恒玄 20ms 一帧 = 单路 50 帧/s、通话双路 |
|||
/// 100 帧/s，每帧都要跨一次平台通道并在 Dart 的单 isolate 上排队——event loop |
|||
/// 一被页面布局或 GC 占住，写入队列就堆积。`WavWriter` 的注释里记着这条链路 |
|||
/// 出过的事故：写入重叠导致丢帧，**等效采样率只剩 1587Hz**（播放又快又糊）。 |
|||
/// 现在帧在原生解码完就地写文件，Dart 只收路径、字节数和电平。 |
|||
/// |
|||
/// ⚠️ 这个类**不是**单例。azure_speech 里的 `RecordFile` 是单例且已被通话翻译 |
|||
/// 的立体声录音占着，复用它会让现场录音和翻译录音互相踩。每次录音一个实例。 |
|||
final class PcmWavRecorder { |
|||
|
|||
/// 一次统计上报 |
|||
struct Stats { |
|||
let bytes: Int // 已写入的音频字节数（不含 44 字节头） |
|||
let level: Double // 本窗口电平峰值 0~1 |
|||
} |
|||
|
|||
let id: String |
|||
let path: String |
|||
let channels: Int |
|||
let sampleRate: Int |
|||
|
|||
/// 统计上报间隔。与 Dart 侧波形采样同频；原来是每帧 `level.value=`， |
|||
/// 50~100 次/s 的 Rx 通知里绝大多数会被下一帧覆盖掉。 |
|||
private static let statsInterval: TimeInterval = 0.1 |
|||
|
|||
/// 串行写队列。文件句柄只在这个队列上碰。 |
|||
private let queue: DispatchQueue |
|||
private var handle: FileHandle? |
|||
private(set) var dataBytes: Int = 0 |
|||
private(set) var writtenFrames: Int = 0 |
|||
private(set) var droppedFrames: Int = 0 |
|||
private var closed = false |
|||
|
|||
/// 双声道的左右暂存（见 StereoPairer 的移植说明） |
|||
private var leftBuf = Data() |
|||
private var rightBuf = Data() |
|||
/// 一路长期缺席时另一路的最大堆积：10 秒 @16k/16bit。 |
|||
/// 超了丢最旧的，避免对端根本没数据时本端无限涨。 |
|||
private let maxPending = 16000 * 2 * 10 |
|||
|
|||
private var levelPeak: Double = 0 |
|||
private var lastStatsAt: TimeInterval = 0 |
|||
|
|||
/// 统计回调（在写队列上调用，接收方自己派发） |
|||
var onStats: ((String, Stats) -> Void)? |
|||
|
|||
init(id: String, path: String, channels: Int, sampleRate: Int) { |
|||
self.id = id |
|||
self.path = path |
|||
self.channels = max(1, min(2, channels)) |
|||
self.sampleRate = sampleRate |
|||
self.queue = DispatchQueue(label: "bes.pcmwav.\(id)") |
|||
} |
|||
|
|||
// MARK: - 生命周期 |
|||
|
|||
/// 建文件并写占位头。返回 false 表示建不出来（目录不可写等）。 |
|||
func start() -> Bool { |
|||
var ok = false |
|||
queue.sync { |
|||
let url = URL(fileURLWithPath: path) |
|||
do { |
|||
try FileManager.default.createDirectory( |
|||
at: url.deletingLastPathComponent(), |
|||
withIntermediateDirectories: true) |
|||
} catch { |
|||
NSLog("[PcmWavRecorder] 建目录失败: %@", error.localizedDescription) |
|||
return |
|||
} |
|||
FileManager.default.createFile(atPath: path, contents: nil) |
|||
do { |
|||
let h = try FileHandle(forWritingTo: url) |
|||
try h.write(contentsOf: Self.wavHeader( |
|||
dataLength: 0, channels: channels, sampleRate: sampleRate)) |
|||
handle = h |
|||
dataBytes = 0 |
|||
writtenFrames = 0 |
|||
droppedFrames = 0 |
|||
lastStatsAt = Date().timeIntervalSince1970 |
|||
ok = true |
|||
} catch { |
|||
NSLog("[PcmWavRecorder] 建文件失败: %@", error.localizedDescription) |
|||
} |
|||
} |
|||
return ok |
|||
} |
|||
|
|||
/// 停止。[save] = false 时删除文件（没收到任何音频那种）。 |
|||
/// 返回统计，给 Dart 算时长与排查用。 |
|||
@discardableResult |
|||
func stop(save: Bool) -> [String: Any] { |
|||
var result: [String: Any] = [:] |
|||
queue.sync { |
|||
guard !closed else { return } |
|||
closed = true |
|||
let h = handle |
|||
handle = nil |
|||
if let h = h { |
|||
if save { |
|||
// 回填 RIFF/data 长度 |
|||
do { |
|||
try h.seek(toOffset: 0) |
|||
try h.write(contentsOf: Self.wavHeader( |
|||
dataLength: dataBytes, channels: channels, sampleRate: sampleRate)) |
|||
} catch { |
|||
NSLog("[PcmWavRecorder] 回填 WAV 头失败: %@", error.localizedDescription) |
|||
} |
|||
} |
|||
try? h.close() |
|||
} |
|||
if !save { |
|||
try? FileManager.default.removeItem(atPath: path) |
|||
} |
|||
result = [ |
|||
"bytes": dataBytes, |
|||
"written": writtenFrames, |
|||
"dropped": droppedFrames, |
|||
] |
|||
} |
|||
return result |
|||
} |
|||
|
|||
// MARK: - 进料 |
|||
|
|||
/// 单声道：直接落盘 |
|||
func append(_ pcm: Data) { |
|||
guard !pcm.isEmpty else { return } |
|||
queue.async { [weak self] in |
|||
guard let self = self, !self.closed else { return } |
|||
self.trackLevel(pcm) |
|||
self.writeLocked(pcm) |
|||
self.emitStatsIfDue() |
|||
} |
|||
} |
|||
|
|||
/// 双声道左路（本端麦） |
|||
func appendLeft(_ pcm: Data) { appendStereo(pcm, isLeft: true) } |
|||
|
|||
/// 双声道右路（对端） |
|||
func appendRight(_ pcm: Data) { appendStereo(pcm, isLeft: false) } |
|||
|
|||
private func appendStereo(_ pcm: Data, isLeft: Bool) { |
|||
guard !pcm.isEmpty else { return } |
|||
queue.async { [weak self] in |
|||
guard let self = self, !self.closed else { return } |
|||
if isLeft { |
|||
self.leftBuf.append(pcm) |
|||
if self.leftBuf.count > self.maxPending { |
|||
self.leftBuf = Data(self.leftBuf.suffix(self.maxPending)) |
|||
} |
|||
} else { |
|||
self.rightBuf.append(pcm) |
|||
if self.rightBuf.count > self.maxPending { |
|||
self.rightBuf = Data(self.rightBuf.suffix(self.maxPending)) |
|||
} |
|||
} |
|||
self.flushPairsLocked() |
|||
self.emitStatsIfDue() |
|||
} |
|||
} |
|||
|
|||
/// StereoPairer 的移植：两路都排队，谁都不丢，每次取出两边都已有的那一段 |
|||
/// （长度取较短者、对齐到整样本）。 |
|||
/// |
|||
/// ⚠️ 不能写成「只留最后一帧对端、本端来一帧配一帧」：两路在 BLE 上是同一 |
|||
/// 个帧里交替的 40 字节块，但到达这里已经是两次独立回调，顺序不保证严格交替。 |
|||
/// 按「最后一帧」配对会把右声道断续抽掉，转写出来只剩自己这一方。 |
|||
/// Dart 侧的金测试 `test/stereo_pairer_test.dart` 定的就是这个行为，两边同改。 |
|||
private func flushPairsLocked() { |
|||
let n = min(leftBuf.count, rightBuf.count) & ~1 |
|||
guard n > 0 else { return } |
|||
let l = leftBuf.prefix(n) |
|||
let r = rightBuf.prefix(n) |
|||
leftBuf = Data(leftBuf.dropFirst(n)) |
|||
rightBuf = Data(rightBuf.dropFirst(n)) |
|||
trackLevel(Data(l)) // 电平按本端（左声道）算，与 Dart 原行为一致 |
|||
writeLocked(Self.interleave(Data(l), Data(r))) |
|||
} |
|||
|
|||
private func writeLocked(_ data: Data) { |
|||
guard let h = handle else { return } |
|||
do { |
|||
try h.write(contentsOf: data) |
|||
dataBytes += data.count |
|||
writtenFrames += 1 |
|||
} catch { |
|||
droppedFrames += 1 |
|||
NSLog("[PcmWavRecorder] 写入失败(已丢 %d): %@", droppedFrames, error.localizedDescription) |
|||
} |
|||
} |
|||
|
|||
// MARK: - 电平与统计 |
|||
|
|||
private func trackLevel(_ pcm: Data) { |
|||
let v = Self.rmsLevel(pcm) |
|||
if v > levelPeak { levelPeak = v } |
|||
} |
|||
|
|||
private func emitStatsIfDue() { |
|||
let now = Date().timeIntervalSince1970 |
|||
guard now - lastStatsAt >= Self.statsInterval else { return } |
|||
lastStatsAt = now |
|||
let s = Stats(bytes: dataBytes, level: levelPeak) |
|||
levelPeak = 0 |
|||
onStats?(id, s) |
|||
} |
|||
|
|||
// MARK: - 工具 |
|||
|
|||
/// 与 Dart `WavWriter.rmsLevel` 同口径：归一化 RMS 再开一次根 |
|||
/// （语音 RMS 常年在 0.02~0.3，不拉一下动态范围界面几乎没动静）。 |
|||
static func rmsLevel(_ pcm: Data) -> Double { |
|||
let count = pcm.count / 2 |
|||
guard count > 0 else { return 0 } |
|||
var sum = 0.0 |
|||
pcm.withUnsafeBytes { raw in |
|||
guard let base = raw.baseAddress?.assumingMemoryBound(to: UInt8.self) else { return } |
|||
for i in 0..<count { |
|||
// 逐字节拼，不用 assumingMemoryBound(to: Int16.self)： |
|||
// 上游 Data 可能是带奇数偏移的切片，按 Int16 对齐会踩未对齐读取 |
|||
var v = Int(base[i * 2 + 1]) << 8 | Int(base[i * 2]) |
|||
if v >= 0x8000 { v -= 0x10000 } |
|||
let n = Double(v) / 32768.0 |
|||
sum += n * n |
|||
} |
|||
} |
|||
let rms = (sum / Double(count)).squareRoot() |
|||
return min(1.0, rms.squareRoot()) |
|||
} |
|||
|
|||
/// 两路单声道按样本交织成双声道（左在前） |
|||
static func interleave(_ left: Data, _ right: Data) -> Data { |
|||
let samples = min(left.count, right.count) / 2 |
|||
var out = Data(count: samples * 4) |
|||
out.withUnsafeMutableBytes { o in |
|||
left.withUnsafeBytes { l in |
|||
right.withUnsafeBytes { r in |
|||
guard let op = o.baseAddress?.assumingMemoryBound(to: UInt8.self), |
|||
let lp = l.baseAddress?.assumingMemoryBound(to: UInt8.self), |
|||
let rp = r.baseAddress?.assumingMemoryBound(to: UInt8.self) else { return } |
|||
for i in 0..<samples { |
|||
let si = i * 2, di = i * 4 |
|||
op[di] = lp[si] |
|||
op[di + 1] = lp[si + 1] |
|||
op[di + 2] = rp[si] |
|||
op[di + 3] = rp[si + 1] |
|||
} |
|||
} |
|||
} |
|||
} |
|||
return out |
|||
} |
|||
|
|||
/// 44 字节标准 WAV 头 |
|||
static func wavHeader(dataLength: Int, channels: Int, sampleRate: Int) -> Data { |
|||
let bits = 16 |
|||
let byteRate = sampleRate * channels * bits / 8 |
|||
let blockAlign = channels * bits / 8 |
|||
var d = Data(capacity: 44) |
|||
func ascii(_ s: String) { d.append(contentsOf: Array(s.utf8)) } |
|||
func u32(_ v: Int) { |
|||
let x = UInt32(truncatingIfNeeded: v) |
|||
d.append(contentsOf: [UInt8(x & 0xFF), UInt8((x >> 8) & 0xFF), |
|||
UInt8((x >> 16) & 0xFF), UInt8((x >> 24) & 0xFF)]) |
|||
} |
|||
func u16(_ v: Int) { |
|||
let x = UInt16(truncatingIfNeeded: v) |
|||
d.append(contentsOf: [UInt8(x & 0xFF), UInt8((x >> 8) & 0xFF)]) |
|||
} |
|||
ascii("RIFF"); u32(36 + dataLength); ascii("WAVE") |
|||
ascii("fmt "); u32(16); u16(1); u16(channels) |
|||
u32(sampleRate); u32(byteRate); u16(blockAlign); u16(bits) |
|||
ascii("data"); u32(dataLength) |
|||
return d |
|||
} |
|||
} |
|||
|
|||
/// 活跃录制器登记处。 |
|||
/// |
|||
/// `BluetoothManager` 解码完一帧就按类型喂给这里的录制器——**只是多挂一个 |
|||
/// sink，不拦截原有的 EventChannel 回调**。这点和 `CallPcmBridge` 不同： |
|||
/// 那条是「桥开着就不再上抛」，因为通话翻译的 PCM 有唯一消费者；录音则可能 |
|||
/// 与别的订阅方（比如正在跑的通话翻译）并存，拦了会把对方饿死。 |
|||
enum PcmRecorderRegistry { |
|||
|
|||
/// 音频类型 → 录制器的投递表。 |
|||
/// 类型串与 `BluetoothManager.publishCallTranslation` 里发的一致： |
|||
/// `mainMicData`（现场录音主麦）/ `micData`（通话本端）/ `spkData`（通话对端）。 |
|||
private static var monoSinks: [String: [PcmWavRecorder]] = [:] |
|||
private static var stereoSinks: [String: [(rec: PcmWavRecorder, isLeft: Bool)]] = [:] |
|||
private static var byId: [String: PcmWavRecorder] = [:] |
|||
private static let lock = NSLock() |
|||
|
|||
static var isEmpty: Bool { |
|||
lock.lock(); defer { lock.unlock() } |
|||
return byId.isEmpty |
|||
} |
|||
|
|||
/// 登记一个录制器。[monoSource] 与 [stereoSources] 二选一。 |
|||
static func register(_ rec: PcmWavRecorder, |
|||
monoSource: String?, |
|||
stereoSources: (left: String, right: String)?) { |
|||
lock.lock(); defer { lock.unlock() } |
|||
byId[rec.id] = rec |
|||
if let s = monoSource { |
|||
monoSinks[s, default: []].append(rec) |
|||
} |
|||
if let s = stereoSources { |
|||
stereoSinks[s.left, default: []].append((rec, true)) |
|||
stereoSinks[s.right, default: []].append((rec, false)) |
|||
} |
|||
} |
|||
|
|||
static func unregister(id: String) -> PcmWavRecorder? { |
|||
lock.lock(); defer { lock.unlock() } |
|||
guard let rec = byId.removeValue(forKey: id) else { return nil } |
|||
for (k, v) in monoSinks { |
|||
let left = v.filter { $0 !== rec } |
|||
if left.isEmpty { monoSinks.removeValue(forKey: k) } else { monoSinks[k] = left } |
|||
} |
|||
for (k, v) in stereoSinks { |
|||
let left = v.filter { $0.rec !== rec } |
|||
if left.isEmpty { stereoSinks.removeValue(forKey: k) } else { stereoSinks[k] = left } |
|||
} |
|||
return rec |
|||
} |
|||
|
|||
static func recorder(id: String) -> PcmWavRecorder? { |
|||
lock.lock(); defer { lock.unlock() } |
|||
return byId[id] |
|||
} |
|||
|
|||
/// 由 `BluetoothManager` 在解码线程上调用。没有录制器时立刻返回， |
|||
/// 开销就是一次字典查空。 |
|||
static func feed(type: String, pcm: Data) { |
|||
lock.lock() |
|||
let mono = monoSinks[type] |
|||
let stereo = stereoSinks[type] |
|||
lock.unlock() |
|||
if let mono = mono { |
|||
for r in mono { r.append(pcm) } |
|||
} |
|||
if let stereo = stereo { |
|||
for e in stereo { |
|||
if e.isLeft { e.rec.appendLeft(pcm) } else { e.rec.appendRight(pcm) } |
|||
} |
|||
} |
|||
} |
|||
|
|||
/// 进程级兜底：断连/停录时把还开着的全停掉，避免留下没回填头的半截文件 |
|||
static func stopAll(save: Bool) { |
|||
lock.lock() |
|||
let all = Array(byId.values) |
|||
byId.removeAll() |
|||
monoSinks.removeAll() |
|||
stereoSinks.removeAll() |
|||
lock.unlock() |
|||
for r in all { r.stop(save: save) } |
|||
} |
|||
} |
|||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue