Browse Source

上传可以播报对方翻译原生

newdev_shunjiawei
liwei1dao 6 months ago
parent
commit
a11df3bcfb
  1. 2
      lib/data/models/appconfig_model.dart
  2. 2
      lib/data/models/appconfig_model.g.dart
  3. 6
      lib/data/services/ast_service.dart
  4. 14
      lib/data/services/speech_impl/azure_ast_service.dart
  5. 30
      lib/modules/translation/controllers/translation_controller.dart
  6. 66
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt
  7. 96
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/CallBPcmPlayer.kt
  8. 6
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AliyunBailianE2EHelper.swift
  9. 196
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift
  10. 13
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/DoubaoE2ETranslateHelper.swift

2
lib/data/models/appconfig_model.dart

@ -84,6 +84,7 @@ class DBProduct {
bool? isforceupdate;
final bool? iscontinuouschat;
final bool? wakeupenable; //该产品是否拥有唤醒开关控制
final bool? broadcastpeertranslate; //通话翻译是否本地播报对方翻译内容
final String? updatedescription;
final String? updatepackageaddress;
@ -98,6 +99,7 @@ class DBProduct {
required this.isforceupdate, //该产品固件版本是否强制更新
required this.iscontinuouschat,
required this.wakeupenable,
required this.broadcastpeertranslate,
required this.updatedescription,
required this.updatepackageaddress,
});

2
lib/data/models/appconfig_model.g.dart

@ -84,6 +84,7 @@ DBProduct _$DBProductFromJson(Map<String, dynamic> json) => DBProduct(
isforceupdate: json['isforceupdate'] as bool?,
iscontinuouschat: json['iscontinuouschat'] as bool?,
wakeupenable: json['wakeupenable'] as bool?,
broadcastpeertranslate: json['broadcastpeertranslate'] as bool?,
updatedescription: json['updatedescription'] as String?,
updatepackageaddress: json['updatepackageaddress'] as String?,
);
@ -99,6 +100,7 @@ Map<String, dynamic> _$DBProductToJson(DBProduct instance) => <String, dynamic>{
'isforceupdate': instance.isforceupdate,
'iscontinuouschat': instance.iscontinuouschat,
'wakeupenable': instance.wakeupenable,
'broadcastpeertranslate': instance.broadcastpeertranslate,
'updatedescription': instance.updatedescription,
'updatepackageaddress': instance.updatepackageaddress,
};

6
lib/data/services/ast_service.dart

@ -30,6 +30,12 @@ abstract class AstService {
/// 开始录音
Future<bool> path(String filePath);
/// 通话翻译:是否本地播报对方(B)翻译后的音频。
/// true → 原生层把 B 路 E2E PCM 直接播出,同时启用播放期 gate 防回环
/// false → 原生层丢弃 B 路播放,gate 永不触发
/// 未实现的原生端应 no-op 并返回 true
Future<void> setBroadcastPeerTranslate(bool enabled);
/// 释放资源
Future<void> dispose();
}

14
lib/data/services/speech_impl/azure_ast_service.dart

@ -410,6 +410,20 @@ class AzureAstService extends GetxService implements AstService {
}
}
@override
Future<void> setBroadcastPeerTranslate(bool enabled) async {
try {
await _channel.invokeMethod('setBroadcastPeerTranslate', {
'enabled': enabled,
});
Logger.info('[STS] setBroadcastPeerTranslate=$enabled');
} on MissingPluginException {
// 原生未实现:忽略(默认原生层会按自己的默认行为处理)
} catch (e) {
Logger.error('setBroadcastPeerTranslate 失败: $e');
}
}
@override
void setLowVolumeThreshold(int threshold) {
try {

30
lib/modules/translation/controllers/translation_controller.dart

@ -30,6 +30,8 @@ import 'record_timer_controller.dart';
import 'translation_history_controller.dart';
import '../models/translation_models.dart';
import '../views/dialogs/translation_mode_selector.dart';
import '../../../data/models/appconfig.dart';
import '../../../data/models/appconfig_model.dart';
import '../../../data/models/user_Info.dart';
import '../../common/widgets/recharge_dialog.dart';
@ -1183,6 +1185,22 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
}
_audioSourceType = true;
isTtsEnabled.value = false;
// 根据设备产品配置决定是否由原生层本地播报对方(B)翻译音频
// 只有 product.broadcastpeertranslate == true 才播报;null / false / 读取异常 → 不播报
bool broadcastPeer = false;
try {
final deviceName = bleManager.getDeviceName();
if (deviceName.isNotEmpty) {
final DBProduct? product = AppConfig.getproduct(deviceName);
broadcastPeer = product?.broadcastpeertranslate == true;
}
} catch (e) {
Logger.error('读取 broadcastpeertranslate 失败,回退默认 false: $e');
}
await _astService.setBroadcastPeerTranslate(broadcastPeer);
Logger.info('[CALL] broadcastPeerTranslate=$broadcastPeer');
Logger.info('发送ble系统mic和dac(音乐或者通话远端)声音');
}
@ -1294,8 +1312,14 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
/// 处理 AST(语音识别+翻译一体)事件
/// 按 serviceId 区分双路(A=己方, B=对方),用 utteranceId 匹配同一句话的事件
void _handleAstEvent(ASTEvent event) {
Logger.d('Translation',
'[STS] _handleAstEvent: type=${event.type}, serviceId=${event.serviceId}');
// 仅对终态事件打 info 日志,避免中间事件高频写 I/O 阻塞 event loop(会导致 AudioSendSlow 1011)
if (event.type == ASTEventType.finalResult ||
event.type == ASTEventType.translationResult ||
event.type == ASTEventType.error ||
event.type == ASTEventType.canceled) {
Logger.info(
'[STS] _handleAstEvent 收到: type=${event.type}, serviceId=${event.serviceId}, text=${event.text}');
}
// 根据 serviceId 确定语言方向
final String srcCode = event.serviceId == "A"
? sourceLanguageCode.value
@ -1307,7 +1331,6 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
switch (event.type) {
case ASTEventType.intermediateResult:
if (event.text.isEmpty) break;
Logger.d('Translation', 'AST 识别中 [${event.serviceId}]: ${event.text}');
currentSessionId ??= DateTime.now().millisecondsSinceEpoch.toString();
// 按 utteranceId 查找已有项
@ -1400,7 +1423,6 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
case ASTEventType.translationInterim:
if (event.text.isEmpty) break;
Logger.d('Translation', 'AST 翻译中 [${event.serviceId}]: ${event.text}');
currentSessionId ??= DateTime.now().millisecondsSinceEpoch.toString();
TranslationItem? target;

66
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt

@ -70,6 +70,14 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
private lateinit var astEventChannel: EventChannel
private var astEventSink: EventChannel.EventSink? = null
// 通话翻译:对方(B)路 E2E 翻译音频(16kHz PCM16 mono)本地播放器
private val callBPcmPlayer = CallBPcmPlayer()
// 设备产品配置 broadcastpeertranslate:true=播报并启用 gate;false/未下发=不播报且 gate 永不触发。默认 false。
@Volatile private var broadcastPeerTranslate: Boolean = false
private val bPcmWriter: AudioWriter = AudioWriter { data ->
if (broadcastPeerTranslate) callBPcmPlayer.feed(data)
}
// 翻译(Volcano)相关
private lateinit var translationChannel: MethodChannel
private lateinit var translationEventChannel: EventChannel
@ -141,19 +149,27 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
private val gatePeakThreshold = 500
private val silenceFramesToMute = 50
// B 路本地播放时,A/B 两路都改推静音帧,避免 TTS 回放被麦克风回采形成闭环
// 若开关关闭,gate 永不触发(也不会有 isPlaying,因为 feed 已被 bPcmWriter 拦截)
private fun gateAgainstBPlayback(data: ByteArray): ByteArray {
if (!broadcastPeerTranslate) return data
return if (callBPcmPlayer.isPlaying()) ByteArray(data.size) else data
}
private fun pushAstAudioToA(data: ByteArray) {
astAudioPushCountA++
if (astAudioPushCountA % 100 == 1L) {
FileLogger.d(tag, "pushAstAudioToA: provider=$currentAstProvider, size=${data.size}, count=$astAudioPushCountA")
}
val payload = gateAgainstBPlayback(data)
if (currentAstProvider == "iflytek") {
iflytekAstHelperA?.pushAudioData(data)
iflytekAstHelperA?.pushAudioData(payload)
} else if (currentAstProvider == "azure") {
azureAstHelperA?.pushAudioData(data)
azureAstHelperA?.pushAudioData(payload)
} else if (currentAstProvider == "volcano") {
doubaoAstHelperA?.pushAudioData(data)
doubaoAstHelperA?.pushAudioData(payload)
} else if (currentAstProvider == "alibaba") {
bailianAstHelperA?.pushAudioData(data)
bailianAstHelperA?.pushAudioData(payload)
}
}
@ -168,14 +184,15 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
if (astAudioPushCountB % 100 == 1L) {
FileLogger.d(tag, "pushAstAudioToB: provider=$currentAstProvider, size=${data.size}, count=$astAudioPushCountB")
}
val payload = gateAgainstBPlayback(data)
if (currentAstProvider == "iflytek" ) {
iflytekAstHelperB?.pushAudioData(data)
iflytekAstHelperB?.pushAudioData(payload)
} else if (currentAstProvider == "azure") {
azureAstHelperB?.pushAudioData(data)
azureAstHelperB?.pushAudioData(payload)
} else if (currentAstProvider == "volcano") {
doubaoAstHelperB?.pushAudioData(data)
doubaoAstHelperB?.pushAudioData(payload)
} else if (currentAstProvider == "alibaba") {
bailianAstHelperB?.pushAudioData(data)
bailianAstHelperB?.pushAudioData(payload)
}
}
@ -1120,6 +1137,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
"dispose" -> {
try {
FileLogger.d(tag, "释放双向 AST 资源(异步)")
callBPcmPlayer.stop()
GlobalScope.launch(Dispatchers.IO) {
try {
if (currentAstProvider == "iflytek" ) {
@ -1178,6 +1196,16 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
result.success(true)
}
"setBroadcastPeerTranslate" -> {
val enabled = call.argument<Boolean>("enabled") ?: false
broadcastPeerTranslate = enabled
if (!enabled) {
callBPcmPlayer.stop()
}
FileLogger.d(tag, "[CALL] setBroadcastPeerTranslate=$enabled")
result.success(true)
}
"initialize" -> {
val provider = call.argument<String>("provider") ?: ""
val supportedLanguages =
@ -1269,7 +1297,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
val callbackB = IflytekAstCallback(
"B", "$lang1->$lang0",
{ sendAstEvent(it) },
{ data -> }
bPcmWriter
)
GlobalScope.launch(Dispatchers.Main) {
@ -1332,7 +1360,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
val callbackB = AzureAstCallback(
"B", "$lang1->$lang0",
{ sendAstEvent(it) },
{ data -> }
bPcmWriter
)
GlobalScope.launch(Dispatchers.Main) {
@ -1377,7 +1405,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
val callbackB = DoubaoAstCallback(
"B", "$translationLang1->$translationLang0", translationLang0,
{ sendAstEvent(it) },
{ data -> }
bPcmWriter
)
Log.d(tag, "initializeIntegrated:lang0= $translationLang0, lang1=$translationLang1")
// 设置会话语言(与 UI 选择一致)
@ -1430,7 +1458,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
val callbackB = AliyunAstCallback(
"B", "$translationLang1->$translationLang0", translationLang0,
{ sendAstEvent(it) },
{ data -> }
bPcmWriter
)
GlobalScope.launch(Dispatchers.Main) {
@ -1879,13 +1907,13 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
val filteredLeftBuffer = filterLowVolumeAudio(leftBuffer, lowVolumeThreshold)
val filteredRightBuffer = filterLowVolumeAudio(rightBuffer, lowVolumeThreshold)
// 左声道是自己的(麦克风),右声道是对方的
// 静音门控:持续静音时不推流,避免服务端(尤其豆包)空流超时断连
if (leftSilentStreak < silenceFramesToMute) {
pushAstAudioToA(filteredLeftBuffer)
}
if (rightSilentStreak < silenceFramesToMute) {
pushAstAudioToB(filteredRightBuffer)
}
// 持续静音时推零 PCM 保活(不断流 → 不触发豆包 audio not enough 1011)
pushAstAudioToA(
if (leftSilentStreak >= silenceFramesToMute) ByteArray(filteredLeftBuffer.size) else filteredLeftBuffer
)
pushAstAudioToB(
if (rightSilentStreak >= silenceFramesToMute) ByteArray(filteredRightBuffer.size) else filteredRightBuffer
)
}

96
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/CallBPcmPlayer.kt

@ -0,0 +1,96 @@
package com.yunqiinnovation.azure_speech
import android.media.AudioAttributes
import android.media.AudioFormat
import android.media.AudioManager
import android.media.AudioTrack
import android.util.Log
import java.util.concurrent.Executors
/**
* 通话翻译 B 路 PCM 本地播放器
* 豆包/阿里 E2E onPartialAudio 回调拿到的 16kHz PCM16 mono 音频,直接流式写到 AudioTrack
* 单线程顺序 write,避免被 WebSocket 线程阻塞
*/
class CallBPcmPlayer {
private val tag = "CallBPcmPlayer"
private val sampleRate = 16000
private val channelCfg = AudioFormat.CHANNEL_OUT_MONO
private val encoding = AudioFormat.ENCODING_PCM_16BIT
private val executor = Executors.newSingleThreadExecutor { r ->
Thread(r, "CallBPcmPlayer").apply { isDaemon = true }
}
@Volatile private var track: AudioTrack? = null
// 排空时间点(elapsedRealtime, ms)—— 小于此时间认为 B 路 PCM 还在播,此时上游应对 A/B 路推静音防回环
@Volatile var playbackUntilMs: Long = 0L
private set
private val tailPaddingMs = 200L
fun isPlaying(): Boolean = android.os.SystemClock.elapsedRealtime() < playbackUntilMs
private fun ensureTrack(): AudioTrack {
track?.let { return it }
val minBuf = AudioTrack.getMinBufferSize(sampleRate, channelCfg, encoding)
val bufferSize = (minBuf.coerceAtLeast(1) * 4)
val attrs = AudioAttributes.Builder()
.setUsage(AudioAttributes.USAGE_MEDIA)
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
.build()
val format = AudioFormat.Builder()
.setSampleRate(sampleRate)
.setChannelMask(channelCfg)
.setEncoding(encoding)
.build()
val t = AudioTrack(
attrs,
format,
bufferSize,
AudioTrack.MODE_STREAM,
AudioManager.AUDIO_SESSION_ID_GENERATE
)
t.play()
track = t
return t
}
fun feed(data: ByteArray) {
if (data.isEmpty()) return
// 16bit mono 16kHz → 每字节 = 1/32ms;+ 尾部静默余量避免回环残尾
val incomingMs = data.size.toLong() * 1000L / (sampleRate.toLong() * 2L)
val now = android.os.SystemClock.elapsedRealtime()
val base = if (playbackUntilMs > now) playbackUntilMs else now
playbackUntilMs = base + incomingMs + tailPaddingMs
executor.execute {
try {
val t = ensureTrack()
var off = 0
while (off < data.size) {
val written = t.write(data, off, data.size - off)
if (written <= 0) break
off += written
}
} catch (e: Exception) {
Log.e(tag, "feed failed: ${e.message}")
}
}
}
fun stop() {
playbackUntilMs = 0L
executor.execute {
try {
track?.apply {
if (playState == AudioTrack.PLAYSTATE_PLAYING) stop()
flush()
release()
}
track = null
} catch (e: Exception) {
Log.e(tag, "stop failed: ${e.message}")
}
}
}
}

6
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AliyunBailianE2EHelper.swift

@ -242,7 +242,7 @@ class AliyunBailianE2EHelper: NSObject, URLSessionWebSocketDelegate {
os_log("Session updated", log: log, type: .info)
case "conversation.item.input_audio_transcription.text":
if let txt = obj["text"] as? String, !txt.isEmpty {
os_log("onPartialSourceText: %{public}@", log: log, type: .info, txt)
// 高频事件,取消 info 日志
callback?.onPartialSourceText(sessionId: sessionId, text: txt)
}
case "conversation.item.input_audio_transcription.completed":
@ -266,7 +266,7 @@ class AliyunBailianE2EHelper: NSObject, URLSessionWebSocketDelegate {
case "response.audio_transcript.delta":
if let delta = obj["delta"] as? String, !delta.isEmpty {
recvTextBuffer.append(delta)
os_log("onPartialText: %{public}@", log: log, type: .info, delta)
// 高频事件,取消 info 日志
callback?.onPartialText(sessionId: sessionId, text: delta)
}
case "response.audio_transcript.done":
@ -286,7 +286,7 @@ class AliyunBailianE2EHelper: NSObject, URLSessionWebSocketDelegate {
let resampled = resample24kTo16k(audioBytes)
if !resampled.isEmpty {
fullAudioBuffer.append(resampled)
os_log("onPartialAudio: rawSize=%{public}d resampledSize=%{public}d", log: log, type: .info, audioBytes.count, resampled.count)
// 极高频(音频帧),取消 info 日志
processAudioChunk(incoming: resampled, multiple: 1280)
}
}

196
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift

@ -110,6 +110,28 @@ import ble_service
private var bailianCallbackA: AliyunCallbackProxy?
private var bailianCallbackB: AliyunCallbackProxy?
// 通话翻译:对方(B)路 E2E 返回的翻译音频(16kHz PCM16 mono)直接用 AVAudioEngine 播到系统默认路由
private let callBPcmPlayer = CallBPcmPlayer()
// 设备产品配置 broadcastpeertranslate:true=播报并启用 gate;false/未下发=不播报且 gate 永不触发。默认 false。
private var broadcastPeerTranslate: Bool = false
fileprivate func pushCallBPcm(_ data: Data) {
guard broadcastPeerTranslate else { return }
callBPcmPlayer.feed(data)
}
fileprivate func resetCallBPlayer() {
callBPcmPlayer.stop()
}
/// 若 B 路本地播放未启用或当前不在播放窗口内,返回原始 data;
/// 否则返回同长度的零字节 PCM,既保持 ws 连续推流又阻断声学回环。
fileprivate func gateAgainstBPlayback(_ data: Data) -> Data {
if !broadcastPeerTranslate { return data }
if !callBPcmPlayer.isPlaying { return data }
return Data(count: data.count)
}
// 音频数据相关
private var audioDataEventChannel: FlutterEventChannel?
internal var audioDataEventSink: FlutterEventSink?
@ -830,6 +852,16 @@ private func sendAudioDataEvent(_ event: [String: Any]) {
}
}
case "setBroadcastPeerTranslate":
let args = call.arguments as? [String: Any]
let enabled = (args?["enabled"] as? Bool) ?? false
broadcastPeerTranslate = enabled
if !enabled {
callBPcmPlayer.stop()
}
os_log("[CALL] setBroadcastPeerTranslate=%{public}@", log: ctLog, type: .info, enabled ? "true" : "false")
result(true)
case "dispose":
os_log("[AST-DISPOSE] Releasing all AST resources, provider=%{public}@", log: ctLog, type: .info, currentAstProvider)
azureAstHelperA.dispose()
@ -838,6 +870,7 @@ private func sendAudioDataEvent(_ event: [String: Any]) {
doubaoAstHelperB.dispose()
bailianAstHelperA.dispose()
bailianAstHelperB.dispose()
resetCallBPlayer()
os_log("[AST-DISPOSE] All AST resources released", log: ctLog, type: .info)
result(true)
@ -1281,26 +1314,29 @@ extension AzureSpeechPlugin: BleService.Callback {
// 根据当前 AST 提供商路由音频数据
// 左声道 -> A(己方语音),右声道 -> B(对方语音)
// B 路本地播放期间,A/B 两路都替换为零字节 PCM 防止 TTS 回放被麦克风回采形成闭环
let leftPayload = gateAgainstBPlayback(leftBuffer)
let rightPayload = gateAgainstBPlayback(rightBuffer)
if currentAstProvider == "volcano" {
doubaoAstHelperA.pushAudioData(leftBuffer)
doubaoAstHelperB.pushAudioData(rightBuffer)
doubaoAstHelperA.pushAudioData(leftPayload)
doubaoAstHelperB.pushAudioData(rightPayload)
if bleAudioRecvCount % 200 == 1 {
os_log("[STS] pushAudio: volcano A(left)=%d bytes, B(right)=%d bytes, count=%lld",
log: ctLog, type: .info, leftBuffer.count, rightBuffer.count, bleAudioRecvCount)
log: ctLog, type: .info, leftPayload.count, rightPayload.count, bleAudioRecvCount)
}
} else if currentAstProvider == "alibaba" {
bailianAstHelperA.pushAudioData(leftBuffer)
bailianAstHelperB.pushAudioData(rightBuffer)
bailianAstHelperA.pushAudioData(leftPayload)
bailianAstHelperB.pushAudioData(rightPayload)
if bleAudioRecvCount % 200 == 1 {
os_log("[STS] pushAudio: alibaba A(left)=%d bytes, B(right)=%d bytes, count=%lld",
log: ctLog, type: .info, leftBuffer.count, rightBuffer.count, bleAudioRecvCount)
log: ctLog, type: .info, leftPayload.count, rightPayload.count, bleAudioRecvCount)
}
} else {
azureAstHelperA.pushAudioData(audioData: leftBuffer)
azureAstHelperB.pushAudioData(audioData: rightBuffer)
azureAstHelperA.pushAudioData(audioData: leftPayload)
azureAstHelperB.pushAudioData(audioData: rightPayload)
if bleAudioRecvCount % 200 == 1 {
os_log("[STS] pushAudio: azure A(left)=%d bytes, B(right)=%d bytes, count=%lld",
log: ctLog, type: .info, leftBuffer.count, rightBuffer.count, bleAudioRecvCount)
log: ctLog, type: .info, leftPayload.count, rightPayload.count, bleAudioRecvCount)
}
}
}
@ -1425,8 +1461,7 @@ private class DoubaoCallbackProxy: DoubaoE2ETranslateHelper.Callback {
}
func onPartialSourceText(sessionId: String, text: String) {
os_log("[DoubaoCallback-%{public}@] onPartialSourceText text=%{public}@",
log: ctLog, type: .info, serviceId, text)
// 高频事件,日志改 debug;避免 event loop 拥塞引发豆包 AudioSendSlow(1011)
plugin?.sendAstEvent([
"type": "recognizing",
"serviceId": serviceId,
@ -1452,8 +1487,7 @@ private class DoubaoCallbackProxy: DoubaoE2ETranslateHelper.Callback {
}
func onPartialText(sessionId: String, text: String) {
os_log("[DoubaoCallback-%{public}@] onPartialText text=%{public}@",
log: ctLog, type: .info, serviceId, text)
// 高频事件,取消 info 日志
plugin?.sendAstEvent([
"type": "translatedInterim",
"serviceId": serviceId,
@ -1466,11 +1500,13 @@ private class DoubaoCallbackProxy: DoubaoE2ETranslateHelper.Callback {
}
func onPartialAudio(sessionId: String, data: Data) {
let isServiceA = (serviceId == "A")
if data.count > 0 && isServiceA {
os_log("[DoubaoCallback-%{public}@] onPartialAudio writing %d bytes to BLE",
log: ctLog, type: .debug, serviceId, data.count)
if data.isEmpty { return }
if serviceId == "A" {
// A 路:翻译后的音频通过 BLE 写到耳机外放给对方
BleService.shared.writeExternalAudioData(data: data)
} else {
// B 路:对方的翻译音频,本地 PCM 播放器直接播
plugin?.pushCallBPcm(data)
}
}
@ -1558,8 +1594,7 @@ private class AliyunCallbackProxy: AliyunBailianE2EHelper.Callback {
}
func onPartialSourceText(sessionId: String, text: String) {
os_log("[AliyunCallback-%{public}@] onPartialSourceText text=%{public}@",
log: ctLog, type: .info, serviceId, text)
// 高频事件,取消 info 日志
plugin?.sendAstEvent([
"type": "recognizing",
"serviceId": serviceId,
@ -1585,8 +1620,7 @@ private class AliyunCallbackProxy: AliyunBailianE2EHelper.Callback {
}
func onPartialText(sessionId: String, text: String) {
os_log("[AliyunCallback-%{public}@] onPartialText text=%{public}@",
log: ctLog, type: .info, serviceId, text)
// 高频事件,取消 info 日志
plugin?.sendAstEvent([
"type": "translatedInterim",
"serviceId": serviceId,
@ -1599,11 +1633,11 @@ private class AliyunCallbackProxy: AliyunBailianE2EHelper.Callback {
}
func onPartialAudio(sessionId: String, data: Data) {
let isServiceA = (serviceId == "A")
if data.count > 0 && isServiceA {
os_log("[AliyunCallback-%{public}@] onPartialAudio writing %d bytes to BLE",
log: ctLog, type: .debug, serviceId, data.count)
if data.isEmpty { return }
if serviceId == "A" {
BleService.shared.writeExternalAudioData(data: data)
} else {
plugin?.pushCallBPcm(data)
}
}
@ -1821,3 +1855,115 @@ private class AsrCallbackWrapper: AzureAsrHelper.ContinuousRecognizeCallback {
])
}
}
// MARK: - 通话翻译 B 路 PCM 本地播放器
/// 16kHz / Int16 / mono PCM 流式播放;由豆包/阿里 E2E onPartialAudio 逐帧灌入
final class CallBPcmPlayer {
private let log = OSLog(subsystem: "com.azure.speech", category: "CallBPcmPlayer")
private let queue = DispatchQueue(label: "com.azure.speech.call.b.pcm")
private let engine = AVAudioEngine()
private let player = AVAudioPlayerNode()
private let sourceFormat = AVAudioFormat(commonFormat: .pcmFormatInt16,
sampleRate: 16000,
channels: 1,
interleaved: true)!
private lazy var outputFormat: AVAudioFormat = engine.mainMixerNode.outputFormat(forBus: 0)
private var converter: AVAudioConverter?
private var attached = false
// 排空时间点(CACurrentMediaTime 秒)—— 小于此时间视为 B 路仍在播,上游应对 A/B 路推静音防回环
private let untilLock = NSLock()
private var playbackUntil: CFTimeInterval = 0
private let tailPadding: CFTimeInterval = 0.2
/// 当前是否还在播放窗口内(供 pushAudioData 入口查询决定是否替换为静音帧)
var isPlaying: Bool {
untilLock.lock(); defer { untilLock.unlock() }
return CACurrentMediaTime() < playbackUntil
}
private func ensureAttached() {
guard !attached else { return }
engine.attach(player)
engine.connect(player, to: engine.mainMixerNode, format: outputFormat)
converter = AVAudioConverter(from: sourceFormat, to: outputFormat)
attached = true
}
private func ensureRunning() {
if engine.isRunning { return }
do {
try engine.start()
if !player.isPlaying { player.play() }
} catch {
os_log("engine start failed: %{public}@", log: log, type: .error, error.localizedDescription)
}
}
func feed(_ data: Data) {
if data.isEmpty { return }
// 16bit mono 16kHz → 每字节 = 1/32ms;+ 尾部静默余量避免回环残尾
let durSec = Double(data.count) / (16000.0 * 2.0)
untilLock.lock()
let now = CACurrentMediaTime()
let base = max(now, playbackUntil)
playbackUntil = base + durSec + tailPadding
untilLock.unlock()
queue.async { [weak self] in
guard let self = self else { return }
self.ensureAttached()
self.ensureRunning()
guard let converter = self.converter else { return }
let frameCount = AVAudioFrameCount(data.count / 2)
guard frameCount > 0,
let inBuf = AVAudioPCMBuffer(pcmFormat: self.sourceFormat, frameCapacity: frameCount) else {
return
}
inBuf.frameLength = frameCount
if let dst = inBuf.int16ChannelData?[0] {
data.withUnsafeBytes { raw in
if let src = raw.baseAddress {
memcpy(dst, src, Int(frameCount) * 2)
}
}
}
let ratio = self.outputFormat.sampleRate / self.sourceFormat.sampleRate
let outCap = AVAudioFrameCount(Double(frameCount) * ratio + 1024)
guard let outBuf = AVAudioPCMBuffer(pcmFormat: self.outputFormat, frameCapacity: outCap) else {
return
}
var supplied = false
var err: NSError?
let status = converter.convert(to: outBuf, error: &err) { _, flag in
if supplied {
flag.pointee = .noDataNow
return nil
}
supplied = true
flag.pointee = .haveData
return inBuf
}
if status == .error || err != nil {
os_log("convert failed: %{public}@", log: self.log, type: .error, err?.localizedDescription ?? "?")
return
}
if outBuf.frameLength > 0 {
self.player.scheduleBuffer(outBuf, completionHandler: nil)
}
}
}
func stop() {
untilLock.lock()
playbackUntil = 0
untilLock.unlock()
queue.async { [weak self] in
guard let self = self else { return }
if self.player.isPlaying { self.player.stop() }
if self.engine.isRunning { self.engine.stop() }
}
}
}

13
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/DoubaoE2ETranslateHelper.swift

@ -438,18 +438,16 @@ class DoubaoE2ETranslateHelper: NSObject, URLSessionWebSocketDelegate {
if !audioData.isEmpty {
recvAudio.append(audioData)
os_log("Append audio chunk: chunkSize=%{public}d totalSize=%{public}d",
log: log, type: .info, audioData.count, recvAudio.count)
// 极高频(音频帧),取消 info 日志
processAudioChunk(incoming: audioData, multiple: 1280)
}
if !text.isEmpty {
switch event {
case .sourceSubtitleStart:
os_log("SourceSubtitleStart", log: log, type: .info)
recvSourceText.removeAll()
case .sourceSubtitleResponse:
os_log("SourceSubtitleResponse text=%{public}@", log: log, type: .info, text)
// 高频事件,取消 info 日志
recvSourceText.append(text)
callback?.onPartialSourceText(sessionId: sessionId, text: text)
case .sourceSubtitleEnd:
@ -457,10 +455,9 @@ class DoubaoE2ETranslateHelper: NSObject, URLSessionWebSocketDelegate {
log: log, type: .info, recvSourceText.joined(separator: " ").count)
callback?.onFinalSourceText(sessionId: sessionId, finalText: recvSourceText.joined(separator: " "))
case .translationSubtitleStart:
os_log("TranslationSubtitleStart", log: log, type: .info)
recvText.removeAll()
case .translationSubtitleResponse:
os_log("TranslationSubtitleResponse text=%{public}@", log: log, type: .info, text)
// 高频事件,取消 info 日志
recvText.append(text)
callback?.onPartialText(sessionId: sessionId, text: text)
case .translationSubtitleEnd:
@ -469,9 +466,7 @@ class DoubaoE2ETranslateHelper: NSObject, URLSessionWebSocketDelegate {
callback?.onFinalTranslatedText(sessionId: sessionId, finalText: recvText.joined(separator: " "))
startNewSubSession()
default:
os_log("Other text event=%{public}d text=%{public}@", log: log, type: .info, event.rawValue, text)
// recvText.append(text)
// callback?.onPartialText(sessionId: sessionId, text: text)
break
}
}
}

Loading…
Cancel
Save