Browse Source

优化接力通话效果

weicu
liwei1dao 4 months ago
parent
commit
b111124e39
  1. 137
      lib/modules/devices/controllers/devices_controller.dart
  2. 35
      lib/modules/devices/views/devices_view.dart
  3. 39
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AliyunBailianE2EHelper.kt
  4. 47
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt
  5. 54
      local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/feature/translation/runtime/JieliAITranslationBridge.kt

137
lib/modules/devices/controllers/devices_controller.dart

@ -1,8 +1,10 @@
import 'dart:async';
import 'dart:io';
import 'package:app_settings/app_settings.dart';
import 'package:device_jieli/device_jieli.dart';
import 'package:flutter/material.dart';
import 'package:flutter_blue_plus/flutter_blue_plus.dart' as fbp;
import 'package:flutter_screenutil/flutter_screenutil.dart';
import 'package:get/get.dart';
import 'package:permission_handler/permission_handler.dart';
@ -37,24 +39,68 @@ class DevicesController extends GetxController {
/// 直接读 user 对象不响应式,所以加一个 tick。
final RxInt boundDevicesTick = 0.obs;
/// 蓝牙适配器是否已开启(用于 UI 提示)。
final RxBool isBluetoothOn = true.obs;
/// 蓝牙开关状态监听。
StreamSubscription<fbp.BluetoothAdapterState>? _adapterStateSub;
@override
void onInit() {
super.onInit();
_bleManager = Get.find<BleManager>();
Future.microtask(startScan);
_init();
}
@override
void onClose() {
_adapterStateSub?.cancel();
_bleManager.stopScan();
super.onClose();
}
Future<void> _init() async {
// 1. 已存在已连接的设备 → 不进扫描流程,直接进入主界面。
try {
if (await _bleManager.hasActiveConnection()) {
Logger.i(_tag, '已存在已连接设备,直接进入主界面');
Get.offAllNamed(Routes.home);
return;
}
} catch (e) {
Logger.w(_tag, '检查已连接设备失败: $e');
}
// 2. 监听蓝牙开关:关闭→停扫并刷新提示;从关到开→自动重新扫描。
_listenAdapterState();
// 3. 开始扫描(内部包含蓝牙开关 / 权限检查与引导)。
await startScan();
}
void _listenAdapterState() {
_adapterStateSub?.cancel();
_adapterStateSub = fbp.FlutterBluePlus.adapterState.listen((state) {
final on = state == fbp.BluetoothAdapterState.on;
final wasOn = isBluetoothOn.value;
isBluetoothOn.value = on;
if (!on) {
_bleManager.stopScan();
} else if (!wasOn && !isScanning.value) {
// 蓝牙刚被打开 → 自动重新扫描。
startScan();
}
});
}
Future<void> startScan() async {
Logger.i(_tag, 'startScan 被调用');
final granted = await _ensureBluetoothPermission();
if (!granted) {
Get.snackbar('tip'.tr, 'pleaseGrantBluetoothPermission'.tr);
// 1. 蓝牙是否开启,未开启则弹引导对话框。
if (!await _ensureBluetoothOn()) {
Logger.w(_tag, '蓝牙未开启,已弹出开启引导');
return;
}
// 2. 蓝牙相关权限,未授权则弹引导对话框。
if (!await _ensureBluetoothPermission()) {
Logger.w(_tag, '蓝牙权限不足,已弹出权限引导');
return;
}
final uuids =
@ -310,6 +356,60 @@ class DevicesController extends GetxController {
_bleManager.currentDeviceAddress == device.address;
}
/// 检查蓝牙是否开启。未开启则弹出引导对话框(Android 可直接开启,
/// iOS 引导去系统设置),并返回 false。
Future<bool> _ensureBluetoothOn() async {
var state = fbp.FlutterBluePlus.adapterStateNow;
// 首帧可能还是 unknown,等首个确定状态(带超时兜底)。
if (state == fbp.BluetoothAdapterState.unknown) {
try {
state = await fbp.FlutterBluePlus.adapterState
.firstWhere((s) => s != fbp.BluetoothAdapterState.unknown)
.timeout(const Duration(seconds: 2));
} catch (_) {
state = fbp.FlutterBluePlus.adapterStateNow;
}
}
isBluetoothOn.value = state == fbp.BluetoothAdapterState.on;
if (isBluetoothOn.value) return true;
_showEnableBluetoothDialog();
return false;
}
void _showEnableBluetoothDialog() {
if (Get.isDialogOpen ?? false) return;
Get.dialog(
AlertDialog(
title: Text('needEnableBluetooth'.tr),
content: Text('enableBluetoothDialogContent'.tr),
actions: [
TextButton(
onPressed: () => Get.back(),
child: Text('cancel'.tr),
),
TextButton(
onPressed: () async {
Get.back();
// Android 可直接请求系统开启蓝牙;iOS 不允许,引导去系统设置。
if (Platform.isAndroid) {
try {
await fbp.FlutterBluePlus.turnOn();
return;
} catch (e) {
Logger.w(_tag, 'turnOn 失败,转去系统蓝牙设置: $e');
}
}
await AppSettings.openAppSettings(
type: AppSettingsType.bluetooth);
},
child: Text('enableBluetooth'.tr),
),
],
),
barrierDismissible: false,
);
}
Future<bool> _ensureBluetoothPermission() async {
// iOS: CoreBluetooth 首次扫描时由系统自动弹出蓝牙权限对话框
// (依赖 Info.plist 的 NSBluetoothAlwaysUsageDescription),
@ -324,10 +424,37 @@ class DevicesController extends GetxController {
].request();
final scanOk = results[Permission.bluetoothScan]?.isGranted ?? false;
final connectOk = results[Permission.bluetoothConnect]?.isGranted ?? false;
return scanOk && connectOk;
if (scanOk && connectOk) return true;
// 被拒(含永久拒绝)→ 弹引导对话框,带用户去系统设置手动开启。
_showPermissionDialog();
return false;
} catch (e) {
Logger.error('请求蓝牙权限失败: $e');
return false;
}
}
void _showPermissionDialog() {
if (Get.isDialogOpen ?? false) return;
Get.dialog(
AlertDialog(
title: Text('needPermission'.tr),
content: Text('blePermissionDialogContent'.tr),
actions: [
TextButton(
onPressed: () => Get.back(),
child: Text('cancel'.tr),
),
TextButton(
onPressed: () {
Get.back();
AppSettings.openAppSettings();
},
child: Text('goToSettings'.tr),
),
],
),
barrierDismissible: false,
);
}
}

35
lib/modules/devices/views/devices_view.dart

@ -92,6 +92,16 @@ class _DevicesViewState extends State<DevicesView>
_buildScanAnimation(isDark),
SizedBox(height: 16.h),
Obx(() {
if (!controller.isBluetoothOn.value) {
return Text(
'pleaseEnableBluetooth'.tr,
style: TextStyle(
fontSize: 14.sp,
fontWeight: FontWeight.w600,
color: const Color(0xFFFF9500),
),
);
}
final scanning = controller.isScanning.value;
return Text(
scanning ? 'scanning'.tr : 'scanIdle'.tr,
@ -457,19 +467,36 @@ class _DevicesViewState extends State<DevicesView>
return Padding(
padding: EdgeInsets.fromLTRB(24.w, 12.h, 24.w, 20.h),
child: Obx(() {
final bluetoothOff = !controller.isBluetoothOn.value;
final scanning = controller.isScanning.value;
final IconData icon;
final String label;
if (bluetoothOff) {
icon = Icons.bluetooth_disabled;
label = 'enableBluetooth'.tr;
} else if (scanning) {
icon = Icons.stop;
label = 'stopScan'.tr;
} else {
icon = Icons.refresh;
label = 'startScan'.tr;
}
return SizedBox(
width: double.infinity,
height: 48.h,
child: ElevatedButton.icon(
onPressed: scanning ? controller.stopScan : controller.startScan,
icon: Icon(scanning ? Icons.stop : Icons.refresh, size: 18.sp),
onPressed: (!bluetoothOff && scanning)
? controller.stopScan
: controller.startScan,
icon: Icon(icon, size: 18.sp),
label: Text(
scanning ? 'stopScan'.tr : 'startScan'.tr,
label,
style: TextStyle(fontSize: 14.sp, fontWeight: FontWeight.w600),
),
style: ElevatedButton.styleFrom(
backgroundColor: const Color(0xFF0066CC),
backgroundColor: bluetoothOff
? const Color(0xFFFF9500)
: const Color(0xFF0066CC),
foregroundColor: Colors.white,
shape: RoundedRectangleBorder(
borderRadius: BorderRadius.circular(14.r),

39
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AliyunBailianE2EHelper.kt

@ -78,6 +78,16 @@ class AliyunBailianE2EHelper(
private var webSocket: WebSocket? = null
private var sessionId: String = ""
// 字幕段独立序号:source / translation 各自累加(同 DoubaoE2ETranslateHelper 的修复)。
// 整条会话共用一个 sessionId,若直接拿它当 utteranceId,Flutter 端无法区分句子,连续语音下会
// 出现原文/译文跨句错位、迟到译文单独成泡。这里给每段 source/translation 各自计数,第 K 段
// source 与第 K 段 translation 对应同一句话 → 用 "$sessionId#$K" 作为该句唯一 utteranceId。
// 实时协议无显式段起始事件,用 active 标志在每段首个 partial 时自增、在 final 时复位。
private var srcSubtitleSeq = 0
private var tgtSubtitleSeq = 0
private var srcSegActive = false
private var tgtSegActive = false
// 用于累积文本,以便在结束时返回完整内容
private val fullTextBuffer = StringBuilder()
// 用于累积音频
@ -143,6 +153,11 @@ class AliyunBailianE2EHelper(
if (isStarted.get()) return true
sessionId = UUID.randomUUID().toString()
// 新会话:字幕段序号与段活跃标志归零
srcSubtitleSeq = 0
tgtSubtitleSeq = 0
srcSegActive = false
tgtSegActive = false
Log.d(TAG, "startContinuousConversation: sessionId=${sessionId}")
recvTextBuffer.setLength(0)
@ -348,8 +363,13 @@ class AliyunBailianE2EHelper(
"conversation.item.input_audio_transcription.text" -> {
val txt = json.optString("text", "")
if (txt.isNotEmpty()) {
// 新源段首个 partial:分配新的源段序号
if (!srcSegActive) {
srcSubtitleSeq++
srcSegActive = true
}
Log.d(TAG, "onPartialSourceText: $txt")
callback?.onPartialSourceText(sessionId, txt)
callback?.onPartialSourceText("$sessionId#$srcSubtitleSeq", txt)
}
}
// 源语言识别结果 (Final)
@ -373,18 +393,26 @@ class AliyunBailianE2EHelper(
}
if (finalTxt.isNotEmpty()) {
// 该源段无 partial 时(直接 completed)也要保证序号推进,避免与上一段共用 id
if (!srcSegActive) srcSubtitleSeq++
Log.d(TAG, "sessionId: ${sessionId}, onFinalSourceText: $finalTxt")
callback?.onFinalSourceText(sessionId, finalTxt)
callback?.onFinalSourceText("$sessionId#$srcSubtitleSeq", finalTxt)
}
srcSegActive = false
}
// 翻译/生成结果 (Partial Text)
"response.audio_transcript.delta" -> {
val delta = json.optString("delta", "")
if (delta.isNotEmpty()) {
// 新译段首个 delta:分配新的译段序号
if (!tgtSegActive) {
tgtSubtitleSeq++
tgtSegActive = true
}
recvTextBuffer.append(delta)
Log.d(TAG, "onPartialText: $delta")
callback?.onPartialText(sessionId, delta)
callback?.onPartialText("$sessionId#$tgtSubtitleSeq", delta)
}
}
@ -392,8 +420,11 @@ class AliyunBailianE2EHelper(
"response.audio_transcript.done" -> {
val transcript = json.optString("transcript", "")
val finalText = if (transcript.isNotEmpty()) transcript else recvTextBuffer.toString()
// 该译段无 delta 时也要保证序号推进
if (!tgtSegActive) tgtSubtitleSeq++
Log.d(TAG, "sessionId: ${sessionId}, onFinalTranslatedText: $finalText")
callback?.onFinalTranslatedText(sessionId, finalText)
callback?.onFinalTranslatedText("$sessionId#$tgtSubtitleSeq", finalText)
tgtSegActive = false
if (fullTextBuffer.isNotEmpty()) fullTextBuffer.append(" ")
fullTextBuffer.append(finalText)

47
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt

@ -96,6 +96,16 @@ class DoubaoE2ETranslateHelper(
private val recvText = mutableListOf<String>()
private val recvSourceText = mutableListOf<String>()
// 字幕段独立序号:source / translation 各自累加。
// 服务端在同一条 WS 会话(sessionId)上按顺序 1:1 产出 source/translation 段,但整条会话共用一个
// sessionId。若直接拿 sessionId 当 utteranceId 传给 Flutter,上层无法区分不同句子,只能靠
// “最后一个未完成 item” 配对,连续语音下会出现:上一句译文晚到时被下一句源文覆盖、原文/译文
// 跨句错位、迟到译文单独成泡(无原文)。
// 这里给每段 source/translation 各自计数,第 K 段 source 与第 K 段 translation 对应同一句话,
// 故用 "$sessionId#$K" 作为该句唯一 utteranceId(即使源/译段交错下发也能正确配对)。
private var srcSubtitleSeq = 0
private var tgtSubtitleSeq = 0
private var audioChunkBuffer = ByteArray(0)
private val isStarted = AtomicBoolean(false)
@ -177,9 +187,13 @@ class DoubaoE2ETranslateHelper(
synchronized(sessionLock) {
sessionId = UUID.randomUUID().toString()
// 新会话:字幕段序号归零(sessionId 也已变化,组合 id 不会与上次会话冲突)
srcSubtitleSeq = 0
tgtSubtitleSeq = 0
Log.d(TAG, "startContinuousTranslation: sessionId=${sessionId}")
recvAudio.reset()
recvText.clear()
recvSourceText.clear()
}
val connId = UUID.randomUUID().toString()
@ -313,35 +327,44 @@ class DoubaoE2ETranslateHelper(
Log.d(TAG, "onMessage: partial audio appended size=${data.size}")
processAudioChunk(data)
}
// 段边界事件(Start)不带文本(textLen=0),必须在 text 判空之前处理:
// 每个 SourceSubtitleStart / TranslationSubtitleStart 各自累加段序号并清空累积缓冲,
// 避免源/译段交错下发时跨段串文,并为该段事件分配唯一 utteranceId。
when (event) {
Type.SourceSubtitleStart -> {
srcSubtitleSeq++
recvSourceText.clear()
Log.d(TAG, "onMessage: SourceSubtitleStart seq=$srcSubtitleSeq")
}
Type.TranslationSubtitleStart -> {
tgtSubtitleSeq++
recvText.clear()
Log.d(TAG, "onMessage: TranslationSubtitleStart seq=$tgtSubtitleSeq")
}
else -> {}
}
if (!text.isNullOrBlank()) {
when (event) {
Type.SourceSubtitleStart -> {
Log.d(TAG, "onMessage: SourceSubtitleStart")
recvSourceText.clear()
}
Type.SourceSubtitleResponse -> {
Log.d(TAG, "onMessage: SourceSubtitleResponse text='${text}'")
recvSourceText.add(text)
callback?.onPartialSourceText(sessionId, recvSourceText.joinToString(" "))
callback?.onPartialSourceText("$sessionId#$srcSubtitleSeq", recvSourceText.joinToString(" "))
}
Type.SourceSubtitleEnd -> {
Log.d(TAG, "onMessage: SourceSubtitleEnd")
val finalSrc = recvSourceText.joinToString(" ")
callback?.onFinalSourceText(sessionId, finalSrc)
}
Type.TranslationSubtitleStart -> {
Log.d(TAG, "onMessage: TranslationSubtitleStart")
recvText.clear()
callback?.onFinalSourceText("$sessionId#$srcSubtitleSeq", finalSrc)
}
Type.TranslationSubtitleResponse -> {
Log.d(TAG, "onMessage: TranslationSubtitleResponse text='${text}'")
recvText.add(text)
callback?.onPartialText(sessionId, recvText.joinToString(" "))
callback?.onPartialText("$sessionId#$tgtSubtitleSeq", recvText.joinToString(" "))
}
Type.TranslationSubtitleEnd -> {
Log.d(TAG, "onMessage: TranslationSubtitleEnd")
val finalTgt = recvText.joinToString(" ")
callback?.onFinalTranslatedText(sessionId, finalTgt)
callback?.onFinalTranslatedText("$sessionId#$tgtSubtitleSeq", finalTgt)
// 一句话结束后,重新生成 sessionId 并在同一连接上发送新的 StartSession
startNewSubSession(ws)
}

54
local_plugins/device_jieli/android/src/main/kotlin/com/jielihome/jielihome/feature/translation/runtime/JieliAITranslationBridge.kt

@ -1,5 +1,6 @@
package com.jielihome.jielihome.feature.translation.runtime
import android.os.SystemClock
import android.util.Log
import com.jieli.bluetooth.bean.translation.AudioData
import com.jieli.bluetooth.bean.translation.TranslationMode
@ -147,6 +148,28 @@ class JieliAITranslationBridge(
private val idleFutures =
java.util.concurrent.ConcurrentHashMap<String, java.util.concurrent.ScheduledFuture<*>>()
/**
* Per-leg:该腿上一次写入音频内容的时刻([SystemClock.elapsedRealtime] 单调毫秒)。
* 用来判断"这一组语音是不是停顿后的新一组"——见 [groupStartPending]。
*/
private val lastFeedAt =
java.util.concurrent.ConcurrentHashMap<String, Long>()
/**
* Per-leg:当前正在累积的这一组,是否为"停顿后的新一组"(其首段应立即下发)。
*
* 背景:合并逻辑会让短句(短于 COALESCE_TARGET_MS)卡在空闲去抖里等 COALESCE_IDLE_MS
* 才下发。会话最开始、以及每次停顿后重新说话的"那一句"都会因此延迟。需求是:
* **每一组语音的首句立即下发,只有连续说话时才合并等待**。
*
* 判定时机:在 buffer 从空开始累积新一组的那一刻锁定(不能在 isFinal 时再看 buffer 是否
* 为空——首句若由多块组成,到 isFinal 时 buffer 已非空;首句立即发后 buffer 又会变空,
* 连续说话的第二句会被误判为新组而退化成"每段都立即发")。所以唯一可靠的判据是
* **距上次音频活动的空闲间隔 ≥ COALESCE_IDLE_MS**:超过空闲窗口=停顿后的新一组。
*/
private val groupStartPending =
java.util.concurrent.ConcurrentHashMap<String, Boolean>()
/** 调试落地:整句 TTS 音频目录(懒创建)。null = 关闭。 */
private val resolvedDumpDir: File? = dumpDir?.also { runCatching { if (!it.exists()) it.mkdirs() } }
@ -177,6 +200,9 @@ class JieliAITranslationBridge(
}
}
sdkCallback = callback
// 新会话:清空 per-leg 组状态,保证每通通话开始的首句立即下发(不等合并)。
lastFeedAt.clear()
groupStartPending.clear()
if (!tempDir.exists()) tempDir.mkdirs()
upDecoder?.start()
downDecoder?.start()
@ -191,6 +217,8 @@ class JieliAITranslationBridge(
// 取消所有未触发的合并定时器(utterance 已被打断,剩余直接丢)。
idleFutures.values.forEach { it.cancel(false) }
idleFutures.clear()
lastFeedAt.clear()
groupStartPending.clear()
runCatching { upDecoder?.stop() }
runCatching { downDecoder?.stop() }
runCatching { stereoDecoder?.stop() }
@ -246,6 +274,9 @@ class JieliAITranslationBridge(
* - OPUS 模式:累积 per-leg buffer;`isFinal=true` 或缓冲到 [PCM_BUFFER_HARD_LIMIT_BYTES]
* 时整段编码 → 通过 [AITranslationCallback.onTranslateResult] 交给 SDK,SDK 内部
* 完成 cmd=52 切包 / 发送 / 速率控制。
* 例外:**每一组语音的首段立即下发**(见 [groupStartPending]),跳过合并空闲等待,
* 消除"会话开始 / 每次停顿后首句"的起播延迟;连续说话时(段间隔 < COALESCE_IDLE_MS)
* 首段过后非新组,仍走合并保证连贯。
*
* @return 是否接受本帧(false 表示丢弃,例如未启动 / 缓冲爆 / SDK callback 缺失)
*/
@ -261,30 +292,47 @@ class JieliAITranslationBridge(
// 合并目标字节(按当前采样率换算):累计达到就立刻下发。
val targetBytes = COALESCE_TARGET_MS * (sampleRateHz / 1000 * 2).coerceAtLeast(1)
val now = SystemClock.elapsedRealtime()
var pendingFlush: ByteArray? = null
var armIdle = false
synchronized(bufferLock) {
val buf = pcmBuffers.getOrPut(outputStreamId) { ByteArrayOutputStream() }
// 新一组的判定要在 buffer 从空开始累积的那一刻锁定:距上次音频活动空闲
// ≥ COALESCE_IDLE_MS(含本腿首次,last==null)说明是停顿后的新一组,其首段立即下发。
if (buf.size() == 0) {
val last = lastFeedAt[outputStreamId]
val idleGap = if (last == null) Long.MAX_VALUE else now - last
groupStartPending[outputStreamId] = idleGap >= COALESCE_IDLE_MS
}
if (pcm.isNotEmpty()) {
buf.write(pcm)
lastFeedAt[outputStreamId] = now
}
val isGroupStart = groupStartPending[outputStreamId] ?: true
when {
// 硬上限兜底:无论如何先放出去,防止内存爆。
buf.size() > PCM_BUFFER_HARD_LIMIT_BYTES -> {
Log.w(TAG, "feedTtsPcm leg=$outputStreamId hit hard limit, force flush")
pendingFlush = buf.toByteArray(); buf.reset()
groupStartPending.remove(outputStreamId)
}
// 不合并:老行为,isFinal 立即下发。
!COALESCE_ENABLED && isFinal -> {
val out = buf.toByteArray(); buf.reset()
if (out.isNotEmpty()) pendingFlush = out
}
// 合并:累计够长了立刻发,否则等空闲去抖(期待后续段拼进来)。
// 合并:每组首段、或累计够长,立刻下发;否则等空闲去抖(期待后续段拼进来)。
// 首段立即下发 → 消除"会话开始 / 每次停顿后首句"的起播延迟;
// 连续说话时(段间隔 < COALESCE_IDLE_MS)首段过后非新组,仍走合并保证连贯。
COALESCE_ENABLED && isFinal -> {
if (buf.size() >= targetBytes) {
if (isGroupStart || buf.size() >= targetBytes) {
val out = buf.toByteArray(); buf.reset()
if (out.isNotEmpty()) pendingFlush = out
if (out.isNotEmpty()) {
pendingFlush = out
groupStartPending.remove(outputStreamId)
}
} else if (buf.size() > 0) {
armIdle = true
}

Loading…
Cancel
Save