From a11df3bcfb7c3189e7e3e9fd81cfbcf38ef3233d Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Thu, 23 Apr 2026 09:27:32 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E5=8F=AF=E4=BB=A5=E6=92=AD?= =?UTF-8?q?=E6=8A=A5=E5=AF=B9=E6=96=B9=E7=BF=BB=E8=AF=91=E5=8E=9F=E7=94=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/data/models/appconfig_model.dart | 2 + lib/data/models/appconfig_model.g.dart | 2 + lib/data/services/ast_service.dart | 6 + .../speech_impl/azure_ast_service.dart | 14 ++ .../controllers/translation_controller.dart | 30 ++- .../azure_speech/AzureSpeechPlugin.kt | 66 ++++-- .../azure_speech/CallBPcmPlayer.kt | 96 +++++++++ .../azure_speech/AliyunBailianE2EHelper.swift | 6 +- .../azure_speech/AzureSpeechPlugin.swift | 196 +++++++++++++++--- .../DoubaoE2ETranslateHelper.swift | 13 +- 10 files changed, 371 insertions(+), 60 deletions(-) create mode 100644 local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/CallBPcmPlayer.kt diff --git a/lib/data/models/appconfig_model.dart b/lib/data/models/appconfig_model.dart index 9487db683..88c0a9661 100644 --- a/lib/data/models/appconfig_model.dart +++ b/lib/data/models/appconfig_model.dart @@ -84,6 +84,7 @@ class DBProduct { bool? isforceupdate; final bool? iscontinuouschat; final bool? wakeupenable; //该产品是否拥有唤醒开关控制 + final bool? broadcastpeertranslate; //通话翻译是否本地播报对方翻译内容 final String? updatedescription; final String? updatepackageaddress; @@ -98,6 +99,7 @@ class DBProduct { required this.isforceupdate, //该产品固件版本是否强制更新 required this.iscontinuouschat, required this.wakeupenable, + required this.broadcastpeertranslate, required this.updatedescription, required this.updatepackageaddress, }); diff --git a/lib/data/models/appconfig_model.g.dart b/lib/data/models/appconfig_model.g.dart index d369dc7a5..4227611e4 100644 --- a/lib/data/models/appconfig_model.g.dart +++ b/lib/data/models/appconfig_model.g.dart @@ -84,6 +84,7 @@ DBProduct _$DBProductFromJson(Map json) => DBProduct( isforceupdate: json['isforceupdate'] as bool?, iscontinuouschat: json['iscontinuouschat'] as bool?, wakeupenable: json['wakeupenable'] as bool?, + broadcastpeertranslate: json['broadcastpeertranslate'] as bool?, updatedescription: json['updatedescription'] as String?, updatepackageaddress: json['updatepackageaddress'] as String?, ); @@ -99,6 +100,7 @@ Map _$DBProductToJson(DBProduct instance) => { 'isforceupdate': instance.isforceupdate, 'iscontinuouschat': instance.iscontinuouschat, 'wakeupenable': instance.wakeupenable, + 'broadcastpeertranslate': instance.broadcastpeertranslate, 'updatedescription': instance.updatedescription, 'updatepackageaddress': instance.updatepackageaddress, }; diff --git a/lib/data/services/ast_service.dart b/lib/data/services/ast_service.dart index 8692e2b45..992ab3545 100644 --- a/lib/data/services/ast_service.dart +++ b/lib/data/services/ast_service.dart @@ -30,6 +30,12 @@ abstract class AstService { /// 开始录音 Future path(String filePath); + /// 通话翻译：是否本地播报对方(B)翻译后的音频。 + /// true → 原生层把 B 路 E2E PCM 直接播出，同时启用播放期 gate 防回环 + /// false → 原生层丢弃 B 路播放，gate 永不触发 + /// 未实现的原生端应 no-op 并返回 true + Future setBroadcastPeerTranslate(bool enabled); + /// 释放资源 Future dispose(); } diff --git a/lib/data/services/speech_impl/azure_ast_service.dart b/lib/data/services/speech_impl/azure_ast_service.dart index cbb3364b7..e75215a59 100644 --- a/lib/data/services/speech_impl/azure_ast_service.dart +++ b/lib/data/services/speech_impl/azure_ast_service.dart @@ -410,6 +410,20 @@ class AzureAstService extends GetxService implements AstService { } } + @override + Future setBroadcastPeerTranslate(bool enabled) async { + try { + await _channel.invokeMethod('setBroadcastPeerTranslate', { + 'enabled': enabled, + }); + Logger.info('[STS] setBroadcastPeerTranslate=$enabled'); + } on MissingPluginException { + // 原生未实现：忽略（默认原生层会按自己的默认行为处理） + } catch (e) { + Logger.error('setBroadcastPeerTranslate 失败: $e'); + } + } + @override void setLowVolumeThreshold(int threshold) { try { diff --git a/lib/modules/translation/controllers/translation_controller.dart b/lib/modules/translation/controllers/translation_controller.dart index 756b932f9..3ddaf957c 100644 --- a/lib/modules/translation/controllers/translation_controller.dart +++ b/lib/modules/translation/controllers/translation_controller.dart @@ -30,6 +30,8 @@ import 'record_timer_controller.dart'; import 'translation_history_controller.dart'; import '../models/translation_models.dart'; import '../views/dialogs/translation_mode_selector.dart'; +import '../../../data/models/appconfig.dart'; +import '../../../data/models/appconfig_model.dart'; import '../../../data/models/user_Info.dart'; import '../../common/widgets/recharge_dialog.dart'; @@ -1183,6 +1185,22 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } _audioSourceType = true; isTtsEnabled.value = false; + + // 根据设备产品配置决定是否由原生层本地播报对方(B)翻译音频 + // 只有 product.broadcastpeertranslate == true 才播报；null / false / 读取异常 → 不播报 + bool broadcastPeer = false; + try { + final deviceName = bleManager.getDeviceName(); + if (deviceName.isNotEmpty) { + final DBProduct? product = AppConfig.getproduct(deviceName); + broadcastPeer = product?.broadcastpeertranslate == true; + } + } catch (e) { + Logger.error('读取 broadcastpeertranslate 失败，回退默认 false: $e'); + } + await _astService.setBroadcastPeerTranslate(broadcastPeer); + Logger.info('[CALL] broadcastPeerTranslate=$broadcastPeer'); + Logger.info('发送ble系统mic和dac（音乐或者通话远端）声音'); } @@ -1294,8 +1312,14 @@ class TranslationController extends GetxController with WidgetsBindingObserver { /// 处理 AST（语音识别+翻译一体）事件 /// 按 serviceId 区分双路（A=己方, B=对方），用 utteranceId 匹配同一句话的事件 void _handleAstEvent(ASTEvent event) { - Logger.d('Translation', - '[STS] _handleAstEvent: type=${event.type}, serviceId=${event.serviceId}'); + // 仅对终态事件打 info 日志，避免中间事件高频写 I/O 阻塞 event loop（会导致 AudioSendSlow 1011） + if (event.type == ASTEventType.finalResult || + event.type == ASTEventType.translationResult || + event.type == ASTEventType.error || + event.type == ASTEventType.canceled) { + Logger.info( + '[STS] _handleAstEvent 收到: type=${event.type}, serviceId=${event.serviceId}, text=${event.text}'); + } // 根据 serviceId 确定语言方向 final String srcCode = event.serviceId == "A" ? sourceLanguageCode.value @@ -1307,7 +1331,6 @@ class TranslationController extends GetxController with WidgetsBindingObserver { switch (event.type) { case ASTEventType.intermediateResult: if (event.text.isEmpty) break; - Logger.d('Translation', 'AST 识别中 [${event.serviceId}]: ${event.text}'); currentSessionId ??= DateTime.now().millisecondsSinceEpoch.toString(); // 按 utteranceId 查找已有项 @@ -1400,7 +1423,6 @@ class TranslationController extends GetxController with WidgetsBindingObserver { case ASTEventType.translationInterim: if (event.text.isEmpty) break; - Logger.d('Translation', 'AST 翻译中 [${event.serviceId}]: ${event.text}'); currentSessionId ??= DateTime.now().millisecondsSinceEpoch.toString(); TranslationItem? target; diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt index 90f2aaa97..a03772888 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt @@ -70,6 +70,14 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, private lateinit var astEventChannel: EventChannel private var astEventSink: EventChannel.EventSink? = null + // 通话翻译：对方(B)路 E2E 翻译音频（16kHz PCM16 mono）本地播放器 + private val callBPcmPlayer = CallBPcmPlayer() + // 设备产品配置 broadcastpeertranslate：true=播报并启用 gate；false/未下发=不播报且 gate 永不触发。默认 false。 + @Volatile private var broadcastPeerTranslate: Boolean = false + private val bPcmWriter: AudioWriter = AudioWriter { data -> + if (broadcastPeerTranslate) callBPcmPlayer.feed(data) + } + // 翻译（Volcano）相关 private lateinit var translationChannel: MethodChannel private lateinit var translationEventChannel: EventChannel @@ -141,19 +149,27 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, private val gatePeakThreshold = 500 private val silenceFramesToMute = 50 + // B 路本地播放时，A/B 两路都改推静音帧，避免 TTS 回放被麦克风回采形成闭环 + // 若开关关闭，gate 永不触发（也不会有 isPlaying，因为 feed 已被 bPcmWriter 拦截） + private fun gateAgainstBPlayback(data: ByteArray): ByteArray { + if (!broadcastPeerTranslate) return data + return if (callBPcmPlayer.isPlaying()) ByteArray(data.size) else data + } + private fun pushAstAudioToA(data: ByteArray) { astAudioPushCountA++ if (astAudioPushCountA % 100 == 1L) { FileLogger.d(tag, "pushAstAudioToA: provider=$currentAstProvider, size=${data.size}, count=$astAudioPushCountA") } + val payload = gateAgainstBPlayback(data) if (currentAstProvider == "iflytek") { - iflytekAstHelperA?.pushAudioData(data) + iflytekAstHelperA?.pushAudioData(payload) } else if (currentAstProvider == "azure") { - azureAstHelperA?.pushAudioData(data) + azureAstHelperA?.pushAudioData(payload) } else if (currentAstProvider == "volcano") { - doubaoAstHelperA?.pushAudioData(data) + doubaoAstHelperA?.pushAudioData(payload) } else if (currentAstProvider == "alibaba") { - bailianAstHelperA?.pushAudioData(data) + bailianAstHelperA?.pushAudioData(payload) } } @@ -168,14 +184,15 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, if (astAudioPushCountB % 100 == 1L) { FileLogger.d(tag, "pushAstAudioToB: provider=$currentAstProvider, size=${data.size}, count=$astAudioPushCountB") } + val payload = gateAgainstBPlayback(data) if (currentAstProvider == "iflytek" ) { - iflytekAstHelperB?.pushAudioData(data) + iflytekAstHelperB?.pushAudioData(payload) } else if (currentAstProvider == "azure") { - azureAstHelperB?.pushAudioData(data) + azureAstHelperB?.pushAudioData(payload) } else if (currentAstProvider == "volcano") { - doubaoAstHelperB?.pushAudioData(data) + doubaoAstHelperB?.pushAudioData(payload) } else if (currentAstProvider == "alibaba") { - bailianAstHelperB?.pushAudioData(data) + bailianAstHelperB?.pushAudioData(payload) } } @@ -1120,6 +1137,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, "dispose" -> { try { FileLogger.d(tag, "释放双向 AST 资源(异步)") + callBPcmPlayer.stop() GlobalScope.launch(Dispatchers.IO) { try { if (currentAstProvider == "iflytek" ) { @@ -1178,6 +1196,16 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, result.success(true) } + "setBroadcastPeerTranslate" -> { + val enabled = call.argument("enabled") ?: false + broadcastPeerTranslate = enabled + if (!enabled) { + callBPcmPlayer.stop() + } + FileLogger.d(tag, "[CALL] setBroadcastPeerTranslate=$enabled") + result.success(true) + } + "initialize" -> { val provider = call.argument("provider") ?: "" val supportedLanguages = @@ -1269,7 +1297,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, val callbackB = IflytekAstCallback( "B", "$lang1->$lang0", { sendAstEvent(it) }, - { data -> } + bPcmWriter ) GlobalScope.launch(Dispatchers.Main) { @@ -1332,7 +1360,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, val callbackB = AzureAstCallback( "B", "$lang1->$lang0", { sendAstEvent(it) }, - { data -> } + bPcmWriter ) GlobalScope.launch(Dispatchers.Main) { @@ -1377,7 +1405,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, val callbackB = DoubaoAstCallback( "B", "$translationLang1->$translationLang0", translationLang0, { sendAstEvent(it) }, - { data -> } + bPcmWriter ) Log.d(tag, "initializeIntegrated:lang0= $translationLang0, lang1=$translationLang1") // 设置会话语言（与 UI 选择一致） @@ -1430,7 +1458,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, val callbackB = AliyunAstCallback( "B", "$translationLang1->$translationLang0", translationLang0, { sendAstEvent(it) }, - { data -> } + bPcmWriter ) GlobalScope.launch(Dispatchers.Main) { @@ -1879,13 +1907,13 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, val filteredLeftBuffer = filterLowVolumeAudio(leftBuffer, lowVolumeThreshold) val filteredRightBuffer = filterLowVolumeAudio(rightBuffer, lowVolumeThreshold) // 左声道是自己的(麦克风)，右声道是对方的 - // 静音门控：持续静音时不推流，避免服务端（尤其豆包）空流超时断连 - if (leftSilentStreak < silenceFramesToMute) { - pushAstAudioToA(filteredLeftBuffer) - } - if (rightSilentStreak < silenceFramesToMute) { - pushAstAudioToB(filteredRightBuffer) - } + // 持续静音时推零 PCM 保活（不断流 → 不触发豆包 audio not enough 1011） + pushAstAudioToA( + if (leftSilentStreak >= silenceFramesToMute) ByteArray(filteredLeftBuffer.size) else filteredLeftBuffer + ) + pushAstAudioToB( + if (rightSilentStreak >= silenceFramesToMute) ByteArray(filteredRightBuffer.size) else filteredRightBuffer + ) } diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/CallBPcmPlayer.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/CallBPcmPlayer.kt new file mode 100644 index 000000000..ac4478fd9 --- /dev/null +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/CallBPcmPlayer.kt @@ -0,0 +1,96 @@ +package com.yunqiinnovation.azure_speech + +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioManager +import android.media.AudioTrack +import android.util.Log +import java.util.concurrent.Executors + +/** + * 通话翻译 B 路 PCM 本地播放器 + * 豆包/阿里 E2E onPartialAudio 回调拿到的 16kHz PCM16 mono 音频，直接流式写到 AudioTrack + * 单线程顺序 write，避免被 WebSocket 线程阻塞 + */ +class CallBPcmPlayer { + private val tag = "CallBPcmPlayer" + private val sampleRate = 16000 + private val channelCfg = AudioFormat.CHANNEL_OUT_MONO + private val encoding = AudioFormat.ENCODING_PCM_16BIT + private val executor = Executors.newSingleThreadExecutor { r -> + Thread(r, "CallBPcmPlayer").apply { isDaemon = true } + } + + @Volatile private var track: AudioTrack? = null + + // 排空时间点（elapsedRealtime, ms）—— 小于此时间认为 B 路 PCM 还在播，此时上游应对 A/B 路推静音防回环 + @Volatile var playbackUntilMs: Long = 0L + private set + private val tailPaddingMs = 200L + + fun isPlaying(): Boolean = android.os.SystemClock.elapsedRealtime() < playbackUntilMs + + private fun ensureTrack(): AudioTrack { + track?.let { return it } + val minBuf = AudioTrack.getMinBufferSize(sampleRate, channelCfg, encoding) + val bufferSize = (minBuf.coerceAtLeast(1) * 4) + val attrs = AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_MEDIA) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + val format = AudioFormat.Builder() + .setSampleRate(sampleRate) + .setChannelMask(channelCfg) + .setEncoding(encoding) + .build() + val t = AudioTrack( + attrs, + format, + bufferSize, + AudioTrack.MODE_STREAM, + AudioManager.AUDIO_SESSION_ID_GENERATE + ) + t.play() + track = t + return t + } + + fun feed(data: ByteArray) { + if (data.isEmpty()) return + // 16bit mono 16kHz → 每字节 = 1/32ms；+ 尾部静默余量避免回环残尾 + val incomingMs = data.size.toLong() * 1000L / (sampleRate.toLong() * 2L) + val now = android.os.SystemClock.elapsedRealtime() + val base = if (playbackUntilMs > now) playbackUntilMs else now + playbackUntilMs = base + incomingMs + tailPaddingMs + + executor.execute { + try { + val t = ensureTrack() + var off = 0 + while (off < data.size) { + val written = t.write(data, off, data.size - off) + if (written <= 0) break + off += written + } + } catch (e: Exception) { + Log.e(tag, "feed failed: ${e.message}") + } + } + } + + fun stop() { + playbackUntilMs = 0L + executor.execute { + try { + track?.apply { + if (playState == AudioTrack.PLAYSTATE_PLAYING) stop() + flush() + release() + } + track = null + } catch (e: Exception) { + Log.e(tag, "stop failed: ${e.message}") + } + } + } +} diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AliyunBailianE2EHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AliyunBailianE2EHelper.swift index 41ecacc17..4c9a16c6b 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AliyunBailianE2EHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AliyunBailianE2EHelper.swift @@ -242,7 +242,7 @@ class AliyunBailianE2EHelper: NSObject, URLSessionWebSocketDelegate { os_log("Session updated", log: log, type: .info) case "conversation.item.input_audio_transcription.text": if let txt = obj["text"] as? String, !txt.isEmpty { - os_log("onPartialSourceText: %{public}@", log: log, type: .info, txt) + // 高频事件，取消 info 日志 callback?.onPartialSourceText(sessionId: sessionId, text: txt) } case "conversation.item.input_audio_transcription.completed": @@ -266,7 +266,7 @@ class AliyunBailianE2EHelper: NSObject, URLSessionWebSocketDelegate { case "response.audio_transcript.delta": if let delta = obj["delta"] as? String, !delta.isEmpty { recvTextBuffer.append(delta) - os_log("onPartialText: %{public}@", log: log, type: .info, delta) + // 高频事件，取消 info 日志 callback?.onPartialText(sessionId: sessionId, text: delta) } case "response.audio_transcript.done": @@ -286,7 +286,7 @@ class AliyunBailianE2EHelper: NSObject, URLSessionWebSocketDelegate { let resampled = resample24kTo16k(audioBytes) if !resampled.isEmpty { fullAudioBuffer.append(resampled) - os_log("onPartialAudio: rawSize=%{public}d resampledSize=%{public}d", log: log, type: .info, audioBytes.count, resampled.count) + // 极高频（音频帧），取消 info 日志 processAudioChunk(incoming: resampled, multiple: 1280) } } diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift index 08fba0f71..b304f6d35 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift @@ -110,6 +110,28 @@ import ble_service private var bailianCallbackA: AliyunCallbackProxy? private var bailianCallbackB: AliyunCallbackProxy? + // 通话翻译：对方(B)路 E2E 返回的翻译音频（16kHz PCM16 mono）直接用 AVAudioEngine 播到系统默认路由 + private let callBPcmPlayer = CallBPcmPlayer() + // 设备产品配置 broadcastpeertranslate：true=播报并启用 gate；false/未下发=不播报且 gate 永不触发。默认 false。 + private var broadcastPeerTranslate: Bool = false + + fileprivate func pushCallBPcm(_ data: Data) { + guard broadcastPeerTranslate else { return } + callBPcmPlayer.feed(data) + } + + fileprivate func resetCallBPlayer() { + callBPcmPlayer.stop() + } + + /// 若 B 路本地播放未启用或当前不在播放窗口内，返回原始 data； + /// 否则返回同长度的零字节 PCM，既保持 ws 连续推流又阻断声学回环。 + fileprivate func gateAgainstBPlayback(_ data: Data) -> Data { + if !broadcastPeerTranslate { return data } + if !callBPcmPlayer.isPlaying { return data } + return Data(count: data.count) + } + // 音频数据相关 private var audioDataEventChannel: FlutterEventChannel? internal var audioDataEventSink: FlutterEventSink? @@ -830,6 +852,16 @@ private func sendAudioDataEvent(_ event: [String: Any]) { } } + case "setBroadcastPeerTranslate": + let args = call.arguments as? [String: Any] + let enabled = (args?["enabled"] as? Bool) ?? false + broadcastPeerTranslate = enabled + if !enabled { + callBPcmPlayer.stop() + } + os_log("[CALL] setBroadcastPeerTranslate=%{public}@", log: ctLog, type: .info, enabled ? "true" : "false") + result(true) + case "dispose": os_log("[AST-DISPOSE] Releasing all AST resources, provider=%{public}@", log: ctLog, type: .info, currentAstProvider) azureAstHelperA.dispose() @@ -838,6 +870,7 @@ private func sendAudioDataEvent(_ event: [String: Any]) { doubaoAstHelperB.dispose() bailianAstHelperA.dispose() bailianAstHelperB.dispose() + resetCallBPlayer() os_log("[AST-DISPOSE] All AST resources released", log: ctLog, type: .info) result(true) @@ -1281,26 +1314,29 @@ extension AzureSpeechPlugin: BleService.Callback { // 根据当前 AST 提供商路由音频数据 // 左声道 -> A（己方语音），右声道 -> B（对方语音） + // B 路本地播放期间，A/B 两路都替换为零字节 PCM 防止 TTS 回放被麦克风回采形成闭环 + let leftPayload = gateAgainstBPlayback(leftBuffer) + let rightPayload = gateAgainstBPlayback(rightBuffer) if currentAstProvider == "volcano" { - doubaoAstHelperA.pushAudioData(leftBuffer) - doubaoAstHelperB.pushAudioData(rightBuffer) + doubaoAstHelperA.pushAudioData(leftPayload) + doubaoAstHelperB.pushAudioData(rightPayload) if bleAudioRecvCount % 200 == 1 { os_log("[STS] pushAudio: volcano A(left)=%d bytes, B(right)=%d bytes, count=%lld", - log: ctLog, type: .info, leftBuffer.count, rightBuffer.count, bleAudioRecvCount) + log: ctLog, type: .info, leftPayload.count, rightPayload.count, bleAudioRecvCount) } } else if currentAstProvider == "alibaba" { - bailianAstHelperA.pushAudioData(leftBuffer) - bailianAstHelperB.pushAudioData(rightBuffer) + bailianAstHelperA.pushAudioData(leftPayload) + bailianAstHelperB.pushAudioData(rightPayload) if bleAudioRecvCount % 200 == 1 { os_log("[STS] pushAudio: alibaba A(left)=%d bytes, B(right)=%d bytes, count=%lld", - log: ctLog, type: .info, leftBuffer.count, rightBuffer.count, bleAudioRecvCount) + log: ctLog, type: .info, leftPayload.count, rightPayload.count, bleAudioRecvCount) } } else { - azureAstHelperA.pushAudioData(audioData: leftBuffer) - azureAstHelperB.pushAudioData(audioData: rightBuffer) + azureAstHelperA.pushAudioData(audioData: leftPayload) + azureAstHelperB.pushAudioData(audioData: rightPayload) if bleAudioRecvCount % 200 == 1 { os_log("[STS] pushAudio: azure A(left)=%d bytes, B(right)=%d bytes, count=%lld", - log: ctLog, type: .info, leftBuffer.count, rightBuffer.count, bleAudioRecvCount) + log: ctLog, type: .info, leftPayload.count, rightPayload.count, bleAudioRecvCount) } } } @@ -1425,8 +1461,7 @@ private class DoubaoCallbackProxy: DoubaoE2ETranslateHelper.Callback { } func onPartialSourceText(sessionId: String, text: String) { - os_log("[DoubaoCallback-%{public}@] onPartialSourceText text=%{public}@", - log: ctLog, type: .info, serviceId, text) + // 高频事件，日志改 debug；避免 event loop 拥塞引发豆包 AudioSendSlow(1011) plugin?.sendAstEvent([ "type": "recognizing", "serviceId": serviceId, @@ -1452,8 +1487,7 @@ private class DoubaoCallbackProxy: DoubaoE2ETranslateHelper.Callback { } func onPartialText(sessionId: String, text: String) { - os_log("[DoubaoCallback-%{public}@] onPartialText text=%{public}@", - log: ctLog, type: .info, serviceId, text) + // 高频事件，取消 info 日志 plugin?.sendAstEvent([ "type": "translatedInterim", "serviceId": serviceId, @@ -1466,11 +1500,13 @@ private class DoubaoCallbackProxy: DoubaoE2ETranslateHelper.Callback { } func onPartialAudio(sessionId: String, data: Data) { - let isServiceA = (serviceId == "A") - if data.count > 0 && isServiceA { - os_log("[DoubaoCallback-%{public}@] onPartialAudio writing %d bytes to BLE", - log: ctLog, type: .debug, serviceId, data.count) + if data.isEmpty { return } + if serviceId == "A" { + // A 路：翻译后的音频通过 BLE 写到耳机外放给对方 BleService.shared.writeExternalAudioData(data: data) + } else { + // B 路：对方的翻译音频，本地 PCM 播放器直接播 + plugin?.pushCallBPcm(data) } } @@ -1558,8 +1594,7 @@ private class AliyunCallbackProxy: AliyunBailianE2EHelper.Callback { } func onPartialSourceText(sessionId: String, text: String) { - os_log("[AliyunCallback-%{public}@] onPartialSourceText text=%{public}@", - log: ctLog, type: .info, serviceId, text) + // 高频事件，取消 info 日志 plugin?.sendAstEvent([ "type": "recognizing", "serviceId": serviceId, @@ -1585,8 +1620,7 @@ private class AliyunCallbackProxy: AliyunBailianE2EHelper.Callback { } func onPartialText(sessionId: String, text: String) { - os_log("[AliyunCallback-%{public}@] onPartialText text=%{public}@", - log: ctLog, type: .info, serviceId, text) + // 高频事件，取消 info 日志 plugin?.sendAstEvent([ "type": "translatedInterim", "serviceId": serviceId, @@ -1599,11 +1633,11 @@ private class AliyunCallbackProxy: AliyunBailianE2EHelper.Callback { } func onPartialAudio(sessionId: String, data: Data) { - let isServiceA = (serviceId == "A") - if data.count > 0 && isServiceA { - os_log("[AliyunCallback-%{public}@] onPartialAudio writing %d bytes to BLE", - log: ctLog, type: .debug, serviceId, data.count) + if data.isEmpty { return } + if serviceId == "A" { BleService.shared.writeExternalAudioData(data: data) + } else { + plugin?.pushCallBPcm(data) } } @@ -1821,3 +1855,115 @@ private class AsrCallbackWrapper: AzureAsrHelper.ContinuousRecognizeCallback { ]) } } + +// MARK: - 通话翻译 B 路 PCM 本地播放器 +/// 16kHz / Int16 / mono PCM 流式播放；由豆包/阿里 E2E onPartialAudio 逐帧灌入 +final class CallBPcmPlayer { + private let log = OSLog(subsystem: "com.azure.speech", category: "CallBPcmPlayer") + private let queue = DispatchQueue(label: "com.azure.speech.call.b.pcm") + private let engine = AVAudioEngine() + private let player = AVAudioPlayerNode() + private let sourceFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, + sampleRate: 16000, + channels: 1, + interleaved: true)! + private lazy var outputFormat: AVAudioFormat = engine.mainMixerNode.outputFormat(forBus: 0) + private var converter: AVAudioConverter? + private var attached = false + + // 排空时间点（CACurrentMediaTime 秒）—— 小于此时间视为 B 路仍在播，上游应对 A/B 路推静音防回环 + private let untilLock = NSLock() + private var playbackUntil: CFTimeInterval = 0 + private let tailPadding: CFTimeInterval = 0.2 + + /// 当前是否还在播放窗口内（供 pushAudioData 入口查询决定是否替换为静音帧） + var isPlaying: Bool { + untilLock.lock(); defer { untilLock.unlock() } + return CACurrentMediaTime() < playbackUntil + } + + private func ensureAttached() { + guard !attached else { return } + engine.attach(player) + engine.connect(player, to: engine.mainMixerNode, format: outputFormat) + converter = AVAudioConverter(from: sourceFormat, to: outputFormat) + attached = true + } + + private func ensureRunning() { + if engine.isRunning { return } + do { + try engine.start() + if !player.isPlaying { player.play() } + } catch { + os_log("engine start failed: %{public}@", log: log, type: .error, error.localizedDescription) + } + } + + func feed(_ data: Data) { + if data.isEmpty { return } + // 16bit mono 16kHz → 每字节 = 1/32ms；+ 尾部静默余量避免回环残尾 + let durSec = Double(data.count) / (16000.0 * 2.0) + untilLock.lock() + let now = CACurrentMediaTime() + let base = max(now, playbackUntil) + playbackUntil = base + durSec + tailPadding + untilLock.unlock() + + queue.async { [weak self] in + guard let self = self else { return } + self.ensureAttached() + self.ensureRunning() + guard let converter = self.converter else { return } + + let frameCount = AVAudioFrameCount(data.count / 2) + guard frameCount > 0, + let inBuf = AVAudioPCMBuffer(pcmFormat: self.sourceFormat, frameCapacity: frameCount) else { + return + } + inBuf.frameLength = frameCount + if let dst = inBuf.int16ChannelData?[0] { + data.withUnsafeBytes { raw in + if let src = raw.baseAddress { + memcpy(dst, src, Int(frameCount) * 2) + } + } + } + + let ratio = self.outputFormat.sampleRate / self.sourceFormat.sampleRate + let outCap = AVAudioFrameCount(Double(frameCount) * ratio + 1024) + guard let outBuf = AVAudioPCMBuffer(pcmFormat: self.outputFormat, frameCapacity: outCap) else { + return + } + var supplied = false + var err: NSError? + let status = converter.convert(to: outBuf, error: &err) { _, flag in + if supplied { + flag.pointee = .noDataNow + return nil + } + supplied = true + flag.pointee = .haveData + return inBuf + } + if status == .error || err != nil { + os_log("convert failed: %{public}@", log: self.log, type: .error, err?.localizedDescription ?? "?") + return + } + if outBuf.frameLength > 0 { + self.player.scheduleBuffer(outBuf, completionHandler: nil) + } + } + } + + func stop() { + untilLock.lock() + playbackUntil = 0 + untilLock.unlock() + queue.async { [weak self] in + guard let self = self else { return } + if self.player.isPlaying { self.player.stop() } + if self.engine.isRunning { self.engine.stop() } + } + } +} diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/DoubaoE2ETranslateHelper.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/DoubaoE2ETranslateHelper.swift index fcc55baa3..27591a0e9 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/DoubaoE2ETranslateHelper.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/DoubaoE2ETranslateHelper.swift @@ -438,18 +438,16 @@ class DoubaoE2ETranslateHelper: NSObject, URLSessionWebSocketDelegate { if !audioData.isEmpty { recvAudio.append(audioData) - os_log("Append audio chunk: chunkSize=%{public}d totalSize=%{public}d", - log: log, type: .info, audioData.count, recvAudio.count) + // 极高频（音频帧），取消 info 日志 processAudioChunk(incoming: audioData, multiple: 1280) } if !text.isEmpty { switch event { case .sourceSubtitleStart: - os_log("SourceSubtitleStart", log: log, type: .info) recvSourceText.removeAll() case .sourceSubtitleResponse: - os_log("SourceSubtitleResponse text=%{public}@", log: log, type: .info, text) + // 高频事件，取消 info 日志 recvSourceText.append(text) callback?.onPartialSourceText(sessionId: sessionId, text: text) case .sourceSubtitleEnd: @@ -457,10 +455,9 @@ class DoubaoE2ETranslateHelper: NSObject, URLSessionWebSocketDelegate { log: log, type: .info, recvSourceText.joined(separator: " ").count) callback?.onFinalSourceText(sessionId: sessionId, finalText: recvSourceText.joined(separator: " ")) case .translationSubtitleStart: - os_log("TranslationSubtitleStart", log: log, type: .info) recvText.removeAll() case .translationSubtitleResponse: - os_log("TranslationSubtitleResponse text=%{public}@", log: log, type: .info, text) + // 高频事件，取消 info 日志 recvText.append(text) callback?.onPartialText(sessionId: sessionId, text: text) case .translationSubtitleEnd: @@ -469,9 +466,7 @@ class DoubaoE2ETranslateHelper: NSObject, URLSessionWebSocketDelegate { callback?.onFinalTranslatedText(sessionId: sessionId, finalText: recvText.joined(separator: " ")) startNewSubSession() default: - os_log("Other text event=%{public}d text=%{public}@", log: log, type: .info, event.rawValue, text) - // recvText.append(text) - // callback?.onPartialText(sessionId: sessionId, text: text) + break } } }