diff --git a/lib/modules/meeting/controllers/meeting_record_controller.dart b/lib/modules/meeting/controllers/meeting_record_controller.dart index a2f7e512e..f3827c7bd 100644 --- a/lib/modules/meeting/controllers/meeting_record_controller.dart +++ b/lib/modules/meeting/controllers/meeting_record_controller.dart @@ -619,9 +619,9 @@ class MeetingRecordController extends GetxController } if (audioType.value == 2) { - // 通话录音:走 device_jieli AI 助理通路(assistantStart → - // MODE_RECORD + STRATEGY_DEVICE_ALWAYS_RECORDING),单声道 - // PCM 直写 WAV,不依赖手机真实通话状态。 + // 通话录音:走 device_jieli 设备录音通路(deviceRecordStart → + // MODE_CALL_TRANSLATION_WITH_STEREO + ch=2 + STRATEGY_DEVICE_ALWAYS_RECORDING), + // 双声道交织 PCM(L=本端 / R=对端)直写 stereo WAV,不依赖手机真实通话状态。 _jieliCallRecorder = JieliCallRecorder( onWavePcm: _processAudioData, ); @@ -695,8 +695,9 @@ class MeetingRecordController extends GetxController } break; case 2: - // 通话录音走 device_jieli AI 助理通路(assistantStart), - // 由 JieliCallRecorder 在 Dart 层接管 PCM 写入与 WAV 文件生成, + // 通话录音走 device_jieli 设备录音通路(deviceRecordStart, + // MODE_CALL_TRANSLATION_WITH_STEREO + ch=2 + ALWAYS_RECORDING), + // 由 JieliCallRecorder 在 Dart 层接管 stereo PCM 写入与 WAV 文件生成, // 不再依赖 BleManager.openCallRecordDecoder(当前为空壳实现)。 break; } diff --git a/lib/modules/meeting/utils/jieli_call_recorder.dart b/lib/modules/meeting/utils/jieli_call_recorder.dart index a03a476dd..e611ef9d9 100644 --- a/lib/modules/meeting/utils/jieli_call_recorder.dart +++ b/lib/modules/meeting/utils/jieli_call_recorder.dart @@ -1,15 +1,20 @@ -// 通话录音 Dart 端 PCM 写入器(AI 助理通路) +// 通话录音 Dart 端 PCM 写入器(设备录音通路 / stereo) // -// 之前的实现走 device_jieli 的 deviceRecordStart(MODE_CALL_TRANSLATION -// + STRATEGY_DEVICE_ALWAYS_RECORDING),监听 uplink + downlink 双流配对 -// 写立体声 WAV。问题:那条路是为真实 SCO 电话通话准备的,没有通话期间 -// 设备不会推任何 PCM,本地测试无法采样。 +// 早先曾走过两版: +// 1) deviceRecordStart (MODE_CALL_TRANSLATION + ch=1, ALWAYS_RECORDING): +// 监听 uplink + downlink 两路 mono 配对写 stereo WAV。 +// 问题:那条路是为真实 SCO 电话通话准备的,没有通话期间设备不会推任何 +// PCM,本地测试无法采样。 +// 2) assistantStart (MODE_RECORD + ch=1, ALWAYS_RECORDING): +// 只能拿到本端单声道,对端音丢失,无法满足"会议记录"对双方音轨的需求。 // -// 现在改走 AI 助理通路(assistantStart → MODE_RECORD + STRATEGY_ -// DEVICE_ALWAYS_RECORDING),接收 [AssistantAudioEvent] 拿单路 -// 16k/16bit/mono/20ms PCM 帧,直接写成标准单声道 WAV,与通话状态解耦。 +// 当前实现回到 deviceRecordStart,但 native 已经把 mode 切到 +// MODE_CALL_TRANSLATION_WITH_STEREO + ch=2 + STRATEGY_DEVICE_ALWAYS_RECORDING。 +// 耳机持续上推 SOURCE_E_SCO_MIX 双声道交织 OPUS(与真实通话状态解耦), +// 由 device_jieli 解码成 16k/16bit/stereo PCM(L=本端 R=对端)后,按 +// [DeviceRecordAudioEvent] 推到 Dart。本类直接写成标准 stereo WAV。 // -// 注意:device_jieli AI 助理通路无原生 pause API,本类的暂停语义通过 +// 注意:device_jieli 设备录音通路无原生 pause API,本类的暂停语义通过 // 在 Dart 层丢弃接收到的 PCM 帧实现。 import 'dart:async'; import 'dart:io'; @@ -26,17 +31,18 @@ class JieliCallRecorder { this.onWavePcm, }); - /// 采样率(Hz)。device_jieli AI 助理通路固定 16000。 + /// 采样率(Hz)。device_jieli 设备录音通路固定 16000。 final int sampleRate; /// 位深,固定 16bit。 final int bitsPerSample; - /// 写入文件后的 mono PCM 通知,用于驱动上层波形/振幅。 + /// 写入文件后的 stereo PCM 通知(与 WAV 同步),用于驱动上层波形/振幅。 + /// payload 是 stereo 交织 PCM,调用方如需做 mono 振幅可自取一边或求平均。 final void Function(Uint8List pcm)? onWavePcm; static const String _tag = 'JieliCallRecorder'; - static const int _channels = 1; // AI 助理通路是 mono + static const int _channels = 2; // stereo: L=本端 R=对端 StreamSubscription? _eventSub; RandomAccessFile? _raf; @@ -48,7 +54,7 @@ class JieliCallRecorder { bool get isRunning => _running; String? get filePath => _filePath; - /// 启动录音:写 WAV 头占位、订阅事件、调起 AI 助理通路。 + /// 启动录音:写 WAV 头占位、订阅事件、调起设备录音通路。 Future start(String filePath) async { if (_running) { Logger.w(_tag, '重复 start 调用,已忽略'); @@ -73,25 +79,20 @@ class JieliCallRecorder { _eventSub = Jielihome.instance.events.listen(_onEvent); try { - // 防御性先 stop,避免外部已有 AI 助理通路在跑导致 start 行为不确定 + // 防御性先 stop,避免外部已有录音通路在跑导致 start 行为不确定 try { - await Jielihome.instance.assistantStop(); + await Jielihome.instance.deviceRecordStop(); } catch (_) {} - final ok = await Jielihome.instance.assistantStart(); - if (!ok) { - Logger.error('$_tag assistantStart 返回 false'); - await _shutdown(saveFile: false); - return false; - } + await Jielihome.instance.deviceRecordStart(sampleRate: sampleRate); } catch (e) { - Logger.error('$_tag assistantStart 失败: $e'); + Logger.error('$_tag deviceRecordStart 失败: $e'); await _shutdown(saveFile: false); return false; } _running = true; Logger.info( - '$_tag 启动通话录音 path=$filePath sr=$sampleRate (AI 助理通路)'); + '$_tag 启动通话录音 path=$filePath sr=$sampleRate ch=$_channels (设备录音通路 / stereo)'); return true; } catch (e) { Logger.error('$_tag 启动失败: $e'); @@ -122,9 +123,9 @@ class JieliCallRecorder { _running = false; try { try { - await Jielihome.instance.assistantStop(); + await Jielihome.instance.deviceRecordStop(); } catch (e) { - Logger.w(_tag, 'assistantStop 失败(忽略): $e'); + Logger.w(_tag, 'deviceRecordStop 失败(忽略): $e'); } await _shutdown(saveFile: save); return true; @@ -175,14 +176,13 @@ class JieliCallRecorder { } void _onEvent(JieliEvent event) { - if (event is AssistantAudioEvent) { + if (event is DeviceRecordAudioEvent) { if (_paused) return; - if (event.encoding != 'pcm16') return; if (event.pcm.isEmpty) return; _writePcm(event.pcm); - } else if (event is AssistantErrorEvent) { + } else if (event is DeviceRecordErrorEvent) { Logger.error( - '$_tag AI 助理通路错误: code=${event.code} msg=${event.message ?? ''}'); + '$_tag 设备录音通路错误: code=${event.code} msg=${event.message ?? ''}'); } } @@ -200,6 +200,7 @@ class JieliCallRecorder { } /// 标准 PCM WAV 头(44 字节)。dataLength=0 用于占位,stop 时回填实际值。 + /// 头里的 channels=2 / blockAlign=4 反映 stereo 交织 PCM 格式。 Uint8List _makeWavHeader(int dataLength) { final byteRate = sampleRate * _channels * (bitsPerSample ~/ 8); final blockAlign = _channels * (bitsPerSample ~/ 8); diff --git a/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/core/JieliHomeServer.kt b/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/core/JieliHomeServer.kt index 13d9703b2..fc8d371cd 100644 --- a/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/core/JieliHomeServer.kt +++ b/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/core/JieliHomeServer.kt @@ -110,14 +110,13 @@ class JieliHomeServer private constructor() { val option = BluetoothOption.createDefaultOption() option.setUseMultiDevice(multiDevice) option.setSkipNoNameDev(skipNoNameDev) - // 关闭设备认证:SDK 默认 isUseDeviceAuth=true,会要求设备走 RCSP 授权流程 - // (`DeviceStatusManager.isAuthBtDevice`),未授权设备会被拒绝 → 表现为 - // 连上后 RCSP init 完成立刻断开。授权需要服务端发签名密钥的整套基础 - // 设施,目前没接,统一关掉。 + // 启用 RCSP 设备认证(SDK 默认行为):连接成功后 SDK 会通过 + // `DeviceStatusManager.isAuthBtDevice` 走授权握手;未授权设备会被拒掉 + // → 表现为连上后 RCSP init 完成立刻断开。本项目耳机已发签名授权,保持开启。 option.setUseDeviceAuth(true) android.util.Log.d( "JieliHome", - "init multiDev=$multiDevice skipNoName=$skipNoNameDev useAuth=false" + "init multiDev=$multiDevice skipNoName=$skipNoNameDev useAuth=true defaultProtocol=SPP" ) RCSPController.init(context, option) @@ -138,7 +137,7 @@ class JieliHomeServer private constructor() { translationFeature = TranslationFeature(context.applicationContext, btManager, connectFeature) speechFeature = SpeechFeature(btManager, connectFeature, dispatcher).also { it.attach() } otaFeature = OtaFeature(btManager, connectFeature, dispatcher) - deviceRecordFeature = DeviceRecordFeature(context.applicationContext, btManager, connectFeature, dispatcher) + deviceRecordFeature = DeviceRecordFeature(this) // 默认音频桥:EventChannel;injector 把 Dart 推过来的 PCM 路由回当前 ModeHandler defaultBridge = EventChannelAudioBridge(dispatcher) { _, streamId, pcm, fmt, isFinal -> diff --git a/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/feature/ConnectFeature.kt b/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/feature/ConnectFeature.kt index f6f807eda..2ce086f87 100644 --- a/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/feature/ConnectFeature.kt +++ b/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/feature/ConnectFeature.kt @@ -6,6 +6,20 @@ import com.jieli.bluetooth.constant.JL_DeviceType import com.jieli.bluetooth.impl.JL_BluetoothManager import com.jieli.bluetooth.utils.BluetoothUtil +/** + * 连接功能。 + * + * # 协议选择策略([resolveTarget]) + * 默认**走 SPP(经典蓝牙 RFCOMM)**——只要拿得到 EDR 地址、且不是手表设备: + * - 有 [edrAddress] 且 [deviceType] != WATCH: + * * [connectWay] 显式指定 `GATT_OVER_BR_EDR(2)`:用 GATT_OVER_BR_EDR + * * 其余情况(包括 SDK 默认 0 / 上层未指定 / 扫描解析回 SPP):用 **SPP(1)** + * - 否则(无 edrAddr 或 WATCH 设备):**fallback BLE(0)** + * + * 这个策略把"默认 SPP"作为成年男耳机/带 EDR 设备的首选——这类设备本身设计走经典 + * 蓝牙 + RCSP over SPP,BLE-only 通路在 OPUS 高速下行场景下吞吐受限。手表 / 仅 BLE + * 的设备会自动 fallback BLE,不影响其工作。 + */ class ConnectFeature(private val btManager: JL_BluetoothManager) { fun connect( @@ -44,13 +58,23 @@ class ConnectFeature(private val btManager: JL_BluetoothManager) { connectWay: Int, ): Pair? { val ble = BluetoothUtil.getRemoteDevice(bleAddress) ?: return null - if (deviceType != JL_DeviceType.JL_DEVICE_TYPE_WATCH && - !edrAddress.isNullOrEmpty() && - (connectWay == BluetoothConstant.PROTOCOL_TYPE_SPP || - connectWay == BluetoothConstant.PROTOCOL_TYPE_GATT_OVER_BR_EDR) - ) { - BluetoothUtil.getRemoteDevice(edrAddress)?.let { return it to connectWay } + + // 手表强制 BLE:JL Watch 系列只暴露 GATT,没有 SPP profile + if (deviceType == JL_DeviceType.JL_DEVICE_TYPE_WATCH) { + return ble to BluetoothConstant.PROTOCOL_TYPE_BLE + } + + // 默认走 SPP;调用方显式要求 GATT_OVER_BR_EDR 时尊重之 + if (!edrAddress.isNullOrEmpty()) { + val protocol = if (connectWay == BluetoothConstant.PROTOCOL_TYPE_GATT_OVER_BR_EDR) { + BluetoothConstant.PROTOCOL_TYPE_GATT_OVER_BR_EDR + } else { + BluetoothConstant.PROTOCOL_TYPE_SPP + } + BluetoothUtil.getRemoteDevice(edrAddress)?.let { return it to protocol } } + + // 没有 EDR 地址,回退 BLE return ble to BluetoothConstant.PROTOCOL_TYPE_BLE } } diff --git a/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/feature/record/DeviceRecordFeature.kt b/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/feature/record/DeviceRecordFeature.kt index 6c60e4350..9feb165da 100644 --- a/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/feature/record/DeviceRecordFeature.kt +++ b/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/feature/record/DeviceRecordFeature.kt @@ -1,26 +1,27 @@ package com.jielihome.jielihome.feature.record -import android.content.Context -import com.jieli.bluetooth.bean.translation.AudioData -import com.jieli.bluetooth.bean.translation.TranslationMode -import com.jieli.bluetooth.constant.Constants -import com.jieli.bluetooth.impl.JL_BluetoothManager import com.jielihome.jielihome.bridge.EventDispatcher -import com.jielihome.jielihome.feature.ConnectFeature -import com.jielihome.jielihome.feature.translation.runtime.RcspTranslationRuntime -import java.io.File +import com.jielihome.jielihome.core.JieliHomeServer +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.onEach /** - * 设备录音功能。 + * 设备录音功能(MethodChannel/EventChannel 适配层)。 * - * 通过 RCSP [TranslationMode.MODE_CALL_TRANSLATION] + STRATEGY_DEVICE_ALWAYS_RECORDING - * 让耳机持续上推双通道 PCM: - * - [AudioData.SOURCE_E_SCO_UP_LINK] → streamId = "in.uplink" (本端/耳机麦克风) - * - [AudioData.SOURCE_E_SCO_DOWN_LINK] → streamId = "in.downlink" (对端/通话对方) + * 实现下沉到 [JieliDeviceRecordPort]:MODE_CALL_TRANSLATION_WITH_STEREO(=6) + * + STRATEGY_DEVICE_ALWAYS_RECORDING + 双声道 OPUS(SOURCE_E_SCO_MIX)。耳机 + * 持续上推 stereo OPUS,Port 解码成 16k/16bit/stereo 交织 PCM;本类把 Flow + * 事件转发到 [EventDispatcher] 给 Dart 侧。 * * 事件类型: - * - `deviceRecordStart` — 上行启动成功 - * - `deviceRecordAudio` — PCM 帧(含 streamId 区分上下行) + * - `deviceRecordStart` — 上行启动成功(payload: address, sampleRate, tsMs) + * - `deviceRecordAudio` — PCM 帧 + * payload: address, streamId="in.stereo", sampleRate, channels=2, + * bitsPerSample=16, tsMs, pcm(ByteArray, 交织 LR) * - `deviceRecordStop` — 上行已停止 * - `deviceRecordError` — 错误 * @@ -29,12 +30,13 @@ import java.io.File * 调用方(MethodRouter)负责在启动前 stop 另一个。 */ class DeviceRecordFeature( - private val context: Context, - private val btManager: JL_BluetoothManager, - private val connectFeature: ConnectFeature, - private val dispatcher: EventDispatcher, + private val server: JieliHomeServer, ) { - @Volatile private var runtime: RcspTranslationRuntime? = null + private val dispatcher: EventDispatcher get() = server.dispatcher + + private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + @Volatile private var audioJob: Job? = null + @Volatile private var errorJob: Job? = null @Volatile private var working = false @Volatile private var deviceAddress: String? = null @@ -51,64 +53,54 @@ class DeviceRecordFeature( if (working) return Result.failure(IllegalStateException("already recording")) val address = args["address"] as? String - val device = address?.let { connectFeature.deviceByAddress(it) } - ?: connectFeature.connectedDevice() - ?: return Result.failure(IllegalStateException("no connected device; pass args.address or connect first")) - val sampleRate = (args["sampleRate"] as? Int) ?: 16000 - val sdkMode = TranslationMode( - TranslationMode.MODE_CALL_TRANSLATION, - Constants.AUDIO_TYPE_OPUS, - 1, - sampleRate, - ).setRecordingStrategy(TranslationMode.STRATEGY_DEVICE_ALWAYS_RECORDING) + // 解析地址用于 deviceRecordStart/Stop 事件携带(即使 args.address 省略,也要把 + // 实际连接到的设备地址回传给 Dart,便于上层做多设备路由) + val device = address?.let { server.connectFeature.deviceByAddress(it) } + ?: server.connectFeature.connectedDevice() + ?: return Result.failure( + IllegalStateException("no connected device; pass args.address or connect first") + ) + val resolvedAddress = device.address - val rt = RcspTranslationRuntime( - btManager = btManager, - device = device, - mode = sdkMode, - tempDir = File(context.cacheDir, "jieli_device_record_tmp"), - onPcm = { source, pcm -> - val streamId = when (source) { - AudioData.SOURCE_E_SCO_UP_LINK -> "in.uplink" - AudioData.SOURCE_E_SCO_DOWN_LINK -> "in.downlink" - else -> return@RcspTranslationRuntime - } - dispatcher.send( - mapOf( - "type" to "deviceRecordAudio", - "address" to device.address, - "streamId" to streamId, - "sampleRate" to sampleRate, - "channels" to 1, - "bitsPerSample" to 16, - "tsMs" to System.currentTimeMillis(), - "pcm" to pcm, - ) + val port = server.deviceRecordPort + + // 先订阅再 start,避免首帧落在订阅建立之前被 SharedFlow 丢掉(replay=0) + audioJob = port.audioFrames.onEach { f -> + dispatcher.send( + mapOf( + "type" to "deviceRecordAudio", + "address" to f.address, + "streamId" to f.streamId, + "sampleRate" to f.sampleRate, + "channels" to f.channels, + "bitsPerSample" to f.bitsPerSample, + "tsMs" to f.tsMs, + "pcm" to f.pcm, ) - }, - onError = { code, msg -> - dispatcher.send( - mapOf( - "type" to "deviceRecordError", - "address" to device.address, - "code" to code, - "message" to msg, - ) + ) + }.launchIn(scope) + + errorJob = port.errors.onEach { e -> + dispatcher.send( + mapOf( + "type" to "deviceRecordError", + "address" to e.address, + "code" to e.code, + "message" to e.message, ) - }, - ) + ) + }.launchIn(scope) - return rt.start().fold( + return port.start(address = resolvedAddress, sampleRate = sampleRate).fold( onSuccess = { - runtime = rt working = true - deviceAddress = device.address + deviceAddress = resolvedAddress dispatcher.send( mapOf( "type" to "deviceRecordStart", - "address" to device.address, + "address" to resolvedAddress, "sampleRate" to sampleRate, "tsMs" to System.currentTimeMillis(), ) @@ -116,7 +108,10 @@ class DeviceRecordFeature( Result.success(Unit) }, onFailure = { err -> - rt.stop() + runCatching { audioJob?.cancel() } + runCatching { errorJob?.cancel() } + audioJob = null + errorJob = null Result.failure(err) }, ) @@ -128,8 +123,11 @@ class DeviceRecordFeature( working = false val addr = deviceAddress deviceAddress = null - runtime?.stop() - runtime = null + runCatching { audioJob?.cancel() } + runCatching { errorJob?.cancel() } + audioJob = null + errorJob = null + runCatching { server.deviceRecordPort.stop() } dispatcher.send( mapOf( "type" to "deviceRecordStop", diff --git a/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/feature/record/JieliDeviceRecordPort.kt b/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/feature/record/JieliDeviceRecordPort.kt index 1d3dc1a84..19b53a29a 100644 --- a/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/feature/record/JieliDeviceRecordPort.kt +++ b/local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/feature/record/JieliDeviceRecordPort.kt @@ -1,25 +1,33 @@ package com.jielihome.jielihome.feature.record +import android.bluetooth.BluetoothDevice import android.util.Log -import com.jielihome.jielihome.api.JieliEventAdapter +import com.jieli.bluetooth.bean.translation.AudioData +import com.jieli.bluetooth.bean.translation.TranslationMode +import com.jieli.bluetooth.constant.Constants +import com.jieli.bluetooth.impl.rcsp.translation.TranslationImpl +import com.jieli.bluetooth.interfaces.rcsp.callback.OnRcspActionCallback +import com.jieli.bluetooth.interfaces.rcsp.translation.TranslationCallback +import com.jielihome.jielihome.audio.OpusStreamDecoder import com.jielihome.jielihome.core.JieliHomeServer +import com.jielihome.jielihome.feature.translation.runtime.NoOpAITranslationApi import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.asSharedFlow +import java.util.concurrent.atomic.AtomicLong -/** 录音音频帧(已解码为 PCM_S16LE)。*/ +/** 录音音频帧(已解码为 PCM_S16LE,双声道交织 L/R/L/R …)。 */ data class DeviceRecordFrame( val address: String, - /** [DeviceRecordFeature] 的 "in.uplink"(本端)或 "in.downlink"(对端)*/ + /** 当前固定为 [JieliDeviceRecordPort.STREAM_STEREO]。双声道交织 PCM;左=本端,右=对端 */ val streamId: String, val pcm: ByteArray, val sampleRate: Int, - val channels: Int = 1, + val channels: Int = 2, val bitsPerSample: Int = 16, val tsMs: Long, ) { - // ByteArray 默认 equals/hashCode 按内容比,这里改为引用比避免大帧逐字节比 override fun equals(other: Any?): Boolean = this === other override fun hashCode(): Int = System.identityHashCode(this) } @@ -28,34 +36,39 @@ data class DeviceRecordFrame( data class DeviceRecordError(val address: String?, val code: Int, val message: String?) /** - * 设备录音端口 —— 原生层编排器的直连入口。 + * 设备录音端口 —— **立体声通话翻译模式(MODE_CALL_TRANSLATION_WITH_STEREO=6)+ 双声道**版。 * - * 与 [com.jielihome.jielihome.feature.translation.JieliCallTranslationPort] 对称: - * - [start] 启动耳机音频上行 - * - [audioFrames] 是 Kotlin Flow,收集即可获取上行/下行 PCM 帧 - * - [stop] 停止上行 + * # 设计要点 + * 与 [com.jielihome.jielihome.feature.assistant.JieliAssistantPort] 是同构关系,只是 + * 一个单声道、一个双声道: + * - JieliAssistantPort:mode=MODE_RECORD(1) + ch=1,耳机麦单通道上行 + * - 本类: mode=MODE_CALL_TRANSLATION_WITH_STEREO(6) + ch=2,耳机持续 + * 上推双声道(L=本端 UPLINK / R=对端 DOWNLINK)OPUS(source = + * SOURCE_E_SCO_MIX),由 [OpusStreamDecoder] 解码成 16k/16bit + * /stereo 交织 PCM 抛给 [audioFrames] * - * Flutter 路径(EventChannel)不受影响:同一帧会同时推给 native flow 和 Dart EventChannel。 + * 通路: + * 耳机双麦 → 耳机固件 OPUS 编码(stereo, MIX) → onReceiveAudioData + * → OpusStreamDecoder(channel=2, packetSize=80) → 16k/16bit/stereo 交织 PCM + * → SharedFlow * - * # 典型用法 - * ```kotlin - * val port = server.deviceRecordPort - * val job = scope.launch { - * port.audioFrames.collect { frame -> - * // frame.streamId == "in.uplink" → 本端说话 - * // frame.streamId == "in.downlink" → 对端说话 - * writeToFile(frame.streamId, frame.pcm) - * } - * } - * port.start(address = device.address) - * // ... - * port.stop() - * job.cancel() - * ``` + * # 与 [DeviceRecordFeature] 的关系 + * 两者都是录音通路实现,但走两条独立的 TranslationImpl: + * - [DeviceRecordFeature]:MODE_CALL_TRANSLATION + ch=1,单声道分两路(uplink/downlink) + * - 本类: MODE_CALL_TRANSLATION_WITH_STEREO + ch=2,双声道一路 stereo + * SDK enterMode 是设备级互斥的,两者不能同时活跃。Port 内部不再依赖 feature; + * MethodRouter / Flutter 侧仍走 feature,保持兼容。 * - * # 生命周期 - * Port 是 [JieliHomeServer] 的懒加载单例(`server.deviceRecordPort`), - * 与 server 共存亡,无需手动 [release]。若在 server 生命周期外独立使用则需调 [release]。 + * # 设备能力 + * 进入前会校验 `TranslationImpl.isSupportCallTranslationWithStereo`;不支持的耳机直接 + * 失败返回,避免下发后耳机状态机卡 IDLE 静默丢帧。 + * + * # 格式 + * PCM_S16LE / 16 kHz / **stereo** / 20 ms。stereo 16k/16bit/20ms ≈ 80 B/OPUS 帧。 + * + * # 线程安全 + * `start` / `stop` 通过 `synchronized(this)` 互斥; + * SharedFlow.tryEmit 自身线程安全;SDK 回调线程不固定,所有共享状态用 @Volatile 标记。 */ class JieliDeviceRecordPort( private val server: JieliHomeServer, @@ -63,8 +76,8 @@ class JieliDeviceRecordPort( companion object { private const val TAG = "JieliDeviceRecordPort" - const val STREAM_UPLINK = "in.uplink" - const val STREAM_DOWNLINK = "in.downlink" + /** 双声道交织 PCM,左=本端(耳机麦),右=对端(通话对方/参考) */ + const val STREAM_STEREO = "in.stereo" } private val _audioFrames = MutableSharedFlow( @@ -78,74 +91,226 @@ class JieliDeviceRecordPort( onBufferOverflow = BufferOverflow.DROP_OLDEST, ) - /** 录音 PCM 帧流。每帧包含 streamId 区分上行/下行。 */ + /** 录音 PCM 帧流(双声道交织)。 */ val audioFrames: Flow = _audioFrames.asSharedFlow() /** 错误流。 */ val errors: Flow = _errors.asSharedFlow() - val isRecording: Boolean get() = server.deviceRecordFeature.isRecording() - - private val eventListener = object : JieliEventAdapter() { - override fun onDeviceRecordAudio(payload: Map) { - val address = payload["address"] as? String ?: return - val streamId = payload["streamId"] as? String ?: return - val pcm = payload["pcm"] as? ByteArray ?: return - val sampleRate = (payload["sampleRate"] as? Number)?.toInt() ?: 16000 - val channels = (payload["channels"] as? Number)?.toInt() ?: 1 - val bitsPerSample = (payload["bitsPerSample"] as? Number)?.toInt() ?: 16 - val tsMs = (payload["tsMs"] as? Number)?.toLong() ?: System.currentTimeMillis() - val frame = DeviceRecordFrame( - address = address, - streamId = streamId, - pcm = pcm, - sampleRate = sampleRate, - channels = channels, - bitsPerSample = bitsPerSample, - tsMs = tsMs, - ) - if (!_audioFrames.tryEmit(frame)) { - Log.w(TAG, "audioFrames buffer overflow; dropped streamId=$streamId") - } - } + @Volatile private var entered = false + @Volatile private var device: BluetoothDevice? = null + @Volatile private var sampleRateHz = 16000 + @Volatile private var decoder: OpusStreamDecoder? = null + @Volatile private var translationImpl: TranslationImpl? = null + @Volatile private var translationCallback: TranslationCallback? = null - override fun onDeviceRecordError(payload: Map) { - _errors.tryEmit( - DeviceRecordError( - address = payload["address"] as? String, - code = (payload["code"] as? Number)?.toInt() ?: 0, - message = payload["message"] as? String, - ) - ) - } - } + val isRecording: Boolean get() = entered - init { - server.addEventListener(eventListener) - } + /** 调试统计:每秒打一次 SDK PCM 帧产出量 */ + private val upCount = AtomicLong(0) + @Volatile private var lastReportMs = 0L + /** 首帧 flag:首次 onModeChange / onReceiveAudioData / 首次 PCM 帧发射都各打一条 INFO */ + @Volatile private var rxFirstModeChangeLogged = false + @Volatile private var rxFirstAudioLogged = false /** - * 启动设备录音上行。若翻译功能正在运行,会先自动停止。 + * 启动设备录音上行(MODE_RECORDING_TRANSLATION + stereo + DEVICE_ALWAYS_RECORDING)。 * * @param address 目标设备 MAC;null 取当前已连设备 * @param sampleRate 采样率(Hz),默认 16000 */ + @Synchronized fun start(address: String? = null, sampleRate: Int = 16000): Result { + if (entered) return Result.failure(IllegalStateException("already recording")) + + // 与 TranslationFeature 互斥:两者都调用 enterMode,同时只能一个活跃 if (server.translationFeature.isWorking()) server.translationFeature.stop() - val args = buildMap { - if (address != null) put("address", address) - put("sampleRate", sampleRate) + + val dev = address?.let { server.connectFeature.deviceByAddress(it) } + ?: server.connectFeature.connectedDevice() + ?: return Result.failure( + IllegalStateException("device.record.no_device: pass args.address or connect first") + ) + + sampleRateHz = sampleRate + + // 1. 启动 Opus 解码器:耳机推上来的录音流是 stereo OPUS。 + // packetSize 是单帧真实大小:stereo 16k/16bit/20ms ≈ 80 B(mono 是 40 B)。 + // 详见 OpusStreamDecoder 注释。 + val od = OpusStreamDecoder( + channel = 2, + packetSize = 80, + sampleRate = sampleRate, + onPcm = { stereoPcm -> emitStereoFrame(dev.address, stereoPcm) }, + onError = { c, m -> + _errors.tryEmit( + DeviceRecordError( + address = dev.address, + code = c, + message = "opus stereo decode: ${m ?: ""}", + ) + ) + }, + ).also { it.start() } + decoder = od + + // 2. 构造 TranslationImpl + 前置校验 + val impl = TranslationImpl(server.internalBtManager, NoOpAITranslationApi(), dev) + if (!impl.isInit) { + runCatching { impl.destroy() } + cleanup() + return Result.failure( + IllegalStateException("device.record.rcsp_not_init: RCSP not init for ${dev.address}") + ) + } + if (!impl.isSupportTranslation) { + runCatching { impl.destroy() } + cleanup() + return Result.failure( + IllegalStateException("device.record.not_supported: device does not support translation") + ) + } + if (!impl.isSupportCallTranslationWithStereo) { + runCatching { impl.destroy() } + cleanup() + return Result.failure( + IllegalStateException("device.record.stereo_not_supported: device does not support stereo call translation") + ) + } + + // 3. 注册 TranslationCallback + val cb = object : TranslationCallback { + override fun onModeChange(d: BluetoothDevice, m: TranslationMode) { + if (!rxFirstModeChangeLogged) { + rxFirstModeChangeLogged = true + Log.i(TAG, "[SDK<-DEV] onModeChange FIRST addr=${d.address} mode=${m.mode} type=${m.audioType} ch=${m.channel} sr=${m.sampleRate} strategy=${m.recordingStrategy}") + } else { + Log.i(TAG, "[SDK<-DEV] onModeChange addr=${d.address} mode=${m.mode} strategy=${m.recordingStrategy}") + } + if (m.mode == TranslationMode.MODE_IDLE) { + _errors.tryEmit( + DeviceRecordError( + address = d.address, + code = -1, + message = "headset exited MODE_CALL_TRANSLATION_WITH_STEREO → MODE_IDLE", + ) + ) + } + } + + override fun onReceiveAudioData(d: BluetoothDevice, data: AudioData) { + val payload = data.audioData + if (payload == null || payload.isEmpty()) { + Log.w(TAG, "[SDK<-DEV] onReceiveAudioData null/empty payload source=${data.source} type=${data.type}") + return + } + if (!rxFirstAudioLogged) { + rxFirstAudioLogged = true + Log.i(TAG, "[SDK<-DEV] onReceiveAudioData FIRST FRAME addr=${d.address} source=${data.source} type=${data.type} size=${payload.size}") + } + when (data.type) { + Constants.AUDIO_TYPE_PCM -> emitStereoFrame(d.address, payload) + Constants.AUDIO_TYPE_OPUS -> decoder?.feedEncoded(payload) + else -> Log.w(TAG, "[SDK<-DEV] onReceiveAudioData unknown type=${data.type} (skip)") + } + } + + override fun onError(d: BluetoothDevice, code: Int, msg: String) { + Log.e(TAG, "[SDK<-DEV] TranslationCallback.onError code=$code msg=$msg") + _errors.tryEmit( + DeviceRecordError( + address = d.address, + code = code, + message = msg, + ) + ) + } + } + impl.addTranslationCallback(cb) + translationImpl = impl + translationCallback = cb + device = dev + + // 4. 下发 enterMode(MODE_CALL_TRANSLATION_WITH_STEREO=6, OPUS, ch=2, sr, STRATEGY_DEVICE_ALWAYS_RECORDING) + // - mode=6:立体声通话翻译模式,耳机 RCSP 状态机进入 STEREO 状态,固件按 + // SOURCE_E_SCO_MIX 上推双声道交织 OPUS + // - ch=2: 双声道(L=本端 UPLINK 耳机麦,R=对端 DOWNLINK 通话音/参考) + // - STRATEGY_DEVICE_ALWAYS_RECORDING=1:耳机固件自主采集并持续上推,APP 不用手机麦 + val sdkMode = TranslationMode( + TranslationMode.MODE_CALL_TRANSLATION_WITH_STEREO, + Constants.AUDIO_TYPE_OPUS, + 2, + sampleRate, + ).setRecordingStrategy(TranslationMode.STRATEGY_DEVICE_ALWAYS_RECORDING) + Log.i(TAG, "[APP->SDK] enterMode addr=${dev.address} mode=${sdkMode.mode}(MODE_CALL_TRANSLATION_WITH_STEREO) type=${sdkMode.audioType}(OPUS) ch=${sdkMode.channel} sr=${sdkMode.sampleRate} strategy=${sdkMode.recordingStrategy}(DEVICE_ALWAYS_RECORDING)") + impl.enterMode(sdkMode, cb) + + entered = true + Log.d(TAG, "entered device record mode (MODE_CALL_TRANSLATION_WITH_STEREO=6, ch=2, DEVICE_ALWAYS_RECORDING)") + return Result.success(Unit) + } + + private fun emitStereoFrame(address: String, pcm: ByteArray) { + upCount.incrementAndGet() + val now = System.currentTimeMillis() + if (now - lastReportMs >= 1000L) { + Log.d(TAG, "uplink stereo PCM stats (last 1s): frames=${upCount.getAndSet(0)} bytes/frame=${pcm.size}") + lastReportMs = now + } + val frame = DeviceRecordFrame( + address = address, + streamId = STREAM_STEREO, + pcm = pcm, + sampleRate = sampleRateHz, + channels = 2, + bitsPerSample = 16, + tsMs = now, + ) + if (!_audioFrames.tryEmit(frame)) { + Log.w(TAG, "audioFrames buffer overflow; dropped streamId=$STREAM_STEREO") } - return server.deviceRecordFeature.start(args) } /** 停止设备录音上行。幂等。 */ + @Synchronized fun stop() { - server.deviceRecordFeature.stop() + if (!entered) return + entered = false + cleanup() + Log.d(TAG, "exited device record mode") + } + + /** 释放 enterMode / decoder / callback;可被 start 失败路径调用,幂等 */ + private fun cleanup() { + val impl = translationImpl + val cb = translationCallback + val addr = device?.address + if (impl != null) { + runCatching { + Log.i(TAG, "[APP->SDK] exitMode addr=$addr mode=${TranslationMode.MODE_CALL_TRANSLATION_WITH_STEREO}") + impl.exitMode(object : OnRcspActionCallback { + override fun onSuccess(d: BluetoothDevice?, t: Int?) { + Log.i(TAG, "[APP->SDK] exitMode onSuccess addr=${d?.address} t=$t") + } + override fun onError(d: BluetoothDevice?, e: com.jieli.bluetooth.bean.base.BaseError?) { + Log.w(TAG, "[APP->SDK] exitMode onError addr=${d?.address} code=${e?.code} msg=${e?.message}") + } + }) + } + if (cb != null) runCatching { impl.removeTranslationCallback(cb) } + runCatching { impl.destroy() } + } + runCatching { decoder?.stop() } + translationImpl = null + translationCallback = null + decoder = null + device = null + rxFirstModeChangeLogged = false + rxFirstAudioLogged = false } - /** 释放事件监听。Port 作为 server 单例使用时无需调用;独立使用时须在不再需要时调用。 */ + /** 兼容旧签名:本类不再依赖 EventDispatcher,无需 release。 */ fun release() { - server.removeEventListener(eventListener) + stop() } } diff --git a/local_plugins/device_jieli/lib/device_jieli.dart b/local_plugins/device_jieli/lib/device_jieli.dart index 4c269d178..4f60b7d76 100644 --- a/local_plugins/device_jieli/lib/device_jieli.dart +++ b/local_plugins/device_jieli/lib/device_jieli.dart @@ -299,10 +299,12 @@ class SpeechStopReason { // ───── 设备录音事件 ───── -/// 上行/下行通道 ID 常量(与 Kotlin DeviceRecordFeature 保持一致) +/// 通道 ID 常量(与 Kotlin DeviceRecordFeature 保持一致) +/// +/// 当前实现走 MODE_CALL_TRANSLATION_WITH_STEREO + ch=2,耳机一次推一帧 +/// **双声道交织 PCM**(L=本端 / R=对端),统一用 [inStereo] 标识。 class DeviceRecordStreams { - static const String inUplink = 'in.uplink'; - static const String inDownlink = 'in.downlink'; + static const String inStereo = 'in.stereo'; } class DeviceRecordStartEvent extends JieliEvent { @@ -319,7 +321,7 @@ class DeviceRecordStartEvent extends JieliEvent { class DeviceRecordAudioEvent extends JieliEvent { final String? address; - /// [DeviceRecordStreams.inUplink] 或 [DeviceRecordStreams.inDownlink] + /// [DeviceRecordStreams.inStereo](双声道交织:L=本端 / R=对端) final String streamId; final int sampleRate; final int channels; diff --git a/local_plugins/device_plugin_interface/android/src/main/kotlin/com/aiagent/device_plugin_interface/CallRecording.kt b/local_plugins/device_plugin_interface/android/src/main/kotlin/com/aiagent/device_plugin_interface/CallRecording.kt new file mode 100644 index 000000000..50c11d912 --- /dev/null +++ b/local_plugins/device_plugin_interface/android/src/main/kotlin/com/aiagent/device_plugin_interface/CallRecording.kt @@ -0,0 +1,126 @@ +package com.aiagent.device_plugin_interface + +import kotlinx.coroutines.flow.Flow + +/** + * 通话录音两条腿(leg)的方向: + * - [UPLINK]: 本端说出去(戴耳机的人 / 会议本地参会者) → 录音 + * - [DOWNLINK]: 对端说过来(远端通话音 / 会议参会对方) → 录音 + * + * 与 [CallTranslationLeg] 同义;区分独立枚举仅为防止跨场景误用。 + */ +enum class CallRecordingLeg { UPLINK, DOWNLINK } + +/** 通话录音音频载荷的编码。 */ +enum class CallRecordingAudioCodec { OPUS, PCM_S16LE } + +/** 通话录音协商时使用的音频格式描述。 */ +data class CallRecordingAudioFormat( + val codec: CallRecordingAudioCodec, + val sampleRate: Int, + val channels: Int, + val frameMs: Int, +) { + companion object { + /** 16 kHz / mono / 20 ms 一帧 OPUS(耳机原生格式) */ + val OPUS_16K_MONO_20MS = CallRecordingAudioFormat(CallRecordingAudioCodec.OPUS, 16000, 1, 20) + + /** 16 kHz / mono / 20 ms 一帧 PCM_S16LE = 640 字节 */ + val PCM_S16LE_16K_MONO_20MS = CallRecordingAudioFormat(CallRecordingAudioCodec.PCM_S16LE, 16000, 1, 20) + } +} + +/** + * device → 编排器:一帧通话录音音频。 + * + * 与 [CallAudioFrame] 同形但语义独立 —— 通话录音不需要 TTS 回灌,因此对应的端口里 + * 没有 [DeviceCallTranslationPort.reportTranslated] 这种回写接口。 + */ +data class CallRecordingFrame( + val leg: CallRecordingLeg, + val codec: CallRecordingAudioCodec, + val sampleRate: Int, + val channels: Int, + val bytes: ByteArray, + /** 单调递增帧序号;丢帧后**不得**回填。 */ + val sequence: Long, + /** 单调递增的捕获时间戳(微秒)。 */ + val timestampUs: Long, +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is CallRecordingFrame) return false + return leg == other.leg && sequence == other.sequence && bytes.contentEquals(other.bytes) + } + override fun hashCode(): Int { + var r = leg.hashCode() + r = 31 * r + sequence.hashCode() + r = 31 * r + bytes.contentHashCode() + return r + } +} + +/** 通话录音过程中由 device 抛出的错误(不致命:编排器决定是否 stop)。 */ +data class CallRecordingError( + val code: String, + val message: String, +) + +/** + * 通话录音能力端口(device 侧)。 + * + * 应用场景:会议记录 / 通话存证 / 通话转写。耳机持续把**本端 + 对端**两路音频 + * 同步上推给 APP,APP 落盘或交给 ASR/纪要服务做后续处理。 + * + * 由设备厂商插件按需实现,例如杰理基于 RCSP `MODE_CALL_TRANSLATION_WITH_STEREO`(=6) + * + `STRATEGY_DEVICE_ALWAYS_RECORDING` + 双声道 OPUS(SOURCE_E_SCO_MIX)落地一份 + * `JieliCallRecordingPort`(左=UPLINK、右=DOWNLINK 反交织成 mono 双流)。 + * + * # 与 [DeviceCallTranslationPort] 的区别 + * 两者都按 leg 派发音频,但通话录音是**单向上报**: + * - 没有 TTS 回灌(无 `reportTranslated` / `reportPlayback`),编排器只读不写 + * - 一般也不依赖真实通话(`MODE_RECORDING_TRANSLATION` 不需要 eSCO 通话存在), + * 适合「会议中的环境录音 + 远端通话录音混合」场景 + * + * # 与 [DeviceAssistantPort] 的区别 + * AI 助理是单路上行(用户麦),通话录音是**双路上行**(按 leg)。 + * + * 生命周期: + * ``` + * idle → enter(format) → active → exit() → idle + * ``` + * + * 铁律: + * - 同一设备同一时刻**至多一个** active 端口(与 callTranslation / assistant 共用底层 + * RCSP 状态机互斥);重复 `enter` 抛 `IllegalStateException("device.call_recording.busy")` + * - 协商格式([enter] 入参)必须出自 [supportedSourceFormats] 列出的范围; + * - [exit] 后 [audioFrames]/[errors] 仍可订阅,但不再产新事件。 + */ +interface DeviceCallRecordingPort { + /** 该端口能向上派发的音频源格式集合(per-leg mono)。 */ + fun supportedSourceFormats(): Set + + /** + * 进入通话录音模式。 + * + * @param sourceFormat 协商好的源音频格式(必须 ∈ [supportedSourceFormats])。 + * device 侧据此决定是否做 OPUS↔PCM 转换:例如耳机原生发 stereo OPUS, + * APP 只吃 mono PCM 时 device 内部解码 + 反交织后再按 leg 派发到 [audioFrames]。 + */ + fun enter(sourceFormat: CallRecordingAudioFormat) + + /** + * 上行/下行通话录音帧。`enter` 之后开始派发,`exit` 后停止。 + * 必须是 broadcast / replayable 不变的实现,编排器多次订阅安全。 + * + * 帧到达顺序(UPLINK vs DOWNLINK)由 device 实现保证「时序基本对齐」即可, + * 不要求严格交替。消费方应以各 leg 自己的 `timestampUs` 做时间轴。 + */ + val audioFrames: Flow + + /** 退出通话录音模式,归还设备 RCSP / 音频通道。幂等。 */ + fun exit() + + /** 错误流。**不致命**——编排器自行决定是否 [exit]。 */ + val errors: Flow +}