Browse Source

优化 翻译尾音 的丢音频问题

newdev_chengguofeng
liwei1dao 3 months ago
parent
commit
137494e105
  1. 11
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AstCallbacks.kt
  2. 18
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt
  3. 32
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt
  4. 20
      local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt
  5. 31
      local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/OpusAudioManager.kt

11
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AstCallbacks.kt

@ -521,7 +521,10 @@ class DoubaoAstCallback(
private val direction: String, private val direction: String,
private val targetLanguage: String, private val targetLanguage: String,
private val eventSender: AstEventSender, private val eventSender: AstEventSender,
private val audioWriter: AudioWriter? = null private val audioWriter: AudioWriter? = null,
// 句尾补偿静音回调:本句音频经 onPartialAudio 全部下发后触发,由外部按声道走与音频同一 mutex 补一段静音,
// 把 jl_opus 编码器内滞留的不足一帧真实尾音顶出、并给耳机 pipeline 排空余量。
private val onSentenceFinished: (() -> Unit)? = null
) : DoubaoE2ETranslateHelper.Callback { ) : DoubaoE2ETranslateHelper.Callback {
private val tag = "DoubaoAstCallback" private val tag = "DoubaoAstCallback"
private val doubaoFinalSourceTextCache: MutableMap<String, String> = mutableMapOf() private val doubaoFinalSourceTextCache: MutableMap<String, String> = mutableMapOf()
@ -603,6 +606,12 @@ class DoubaoAstCallback(
audioWriter?.write(data) audioWriter?.write(data)
} }
override fun onTtsSentenceEnd(sessionId: String) {
// 单句译音结束(TTSSentenceEnd):触发句尾补偿静音。
// 走与音频同一 mutex,保证补在本句最后一段真实音频之后,避免耳机端句尾最后一个字被截断。
onSentenceFinished?.invoke()
}
override fun onSessionFinished( override fun onSessionFinished(
sessionId: String, sessionId: String,
finalText: String, finalText: String,

18
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt

@ -1488,12 +1488,28 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
BleService.writeExternalLeftAudioData(data) BleService.writeExternalLeftAudioData(data)
} }
} }
},
{
// 句尾补偿静音:与本端音频同一 bleLeftMutex,保证补在最后一段真实音频之后
bleWriteScope.launch {
bleLeftMutex.withLock {
BleService.feedLeftTailSilence()
}
}
} }
) )
val callbackB = DoubaoAstCallback( val callbackB = DoubaoAstCallback(
"B", "$translationLang1->$translationLang0", translationLang0, "B", "$translationLang1->$translationLang0", translationLang0,
{ sendAstEvent(it) }, { sendAstEvent(it) },
bPcmWriter bPcmWriter,
{
// 句尾补偿静音:与对端音频同一 bleRightMutex
bleWriteScope.launch {
bleRightMutex.withLock {
BleService.feedRightTailSilence()
}
}
}
) )
Log.d(tag, "initializeIntegrated:lang0= $translationLang0, lang1=$translationLang1") Log.d(tag, "initializeIntegrated:lang0= $translationLang0, lang1=$translationLang1")
// 设置会话语言(与 UI 选择一致) // 设置会话语言(与 UI 选择一致)

32
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt

@ -48,6 +48,9 @@ class DoubaoE2ETranslateHelper(
/** 增量音频片段到达(服务端返回的目标音频) */ /** 增量音频片段到达(服务端返回的目标音频) */
fun onPartialAudio(sessionId: String, data: ByteArray) fun onPartialAudio(sessionId: String, data: ByteArray)
/** 单句 TTS 音频结束(TTSSentenceEnd):本句合成音频已全部经 onPartialAudio 下发。默认空实现,可选处理。 */
fun onTtsSentenceEnd(sessionId: String) {}
/** 会话结束,返回完整文本与完整合成音频 */ /** 会话结束,返回完整文本与完整合成音频 */
fun onSessionFinished(sessionId: String, finalText: String, finalAudio: ByteArray) fun onSessionFinished(sessionId: String, finalText: String, finalAudio: ByteArray)
@ -162,6 +165,8 @@ class DoubaoE2ETranslateHelper(
Log.d(TAG, "startContinuousTranslation: sessionId=${sessionId}") Log.d(TAG, "startContinuousTranslation: sessionId=${sessionId}")
recvAudio.reset() recvAudio.reset()
recvText.clear() recvText.clear()
// 丢弃上一 session 未冲刷的残留(mid-sentence 重连时),避免旧尾音串到新 session 头部
audioChunkBuffer = ByteArray(0)
} }
val connId = UUID.randomUUID().toString() val connId = UUID.randomUUID().toString()
@ -257,6 +262,8 @@ class DoubaoE2ETranslateHelper(
if (event == Type.SessionFinished) { if (event == Type.SessionFinished) {
Log.d(TAG, "onMessage: SessionFinished") Log.d(TAG, "onMessage: SessionFinished")
// 兜底:本句若缺 TTSSentenceEnd(事件不完整),会话结束时把残留真实尾音一并冲刷出去,避免整段被截。
flushAudioChunkBuffer()
val finalText = recvText.joinToString(" ") val finalText = recvText.joinToString(" ")
val finalAudio = recvAudio.toByteArray() val finalAudio = recvAudio.toByteArray()
callback?.onSessionFinished(sessionId, finalText, finalAudio) callback?.onSessionFinished(sessionId, finalText, finalAudio)
@ -264,6 +271,16 @@ class DoubaoE2ETranslateHelper(
return return
} }
// 单句 TTS 音频结束:本句 TTSResponse 已全部到达(经下方 onPartialAudio 下发),
// 通知上层做句尾补偿(往编码器补静音,把滞留的不足一帧真实尾音顶出)。TTSSentenceEnd 不带音频/文本。
if (event == Type.TTSSentenceEnd) {
Log.d(TAG, "onMessage: TTSSentenceEnd")
// 先冲刷 WS 层滞留的不足一块(1280B)真实尾音进编码器,再触发句尾补偿静音,避免句末被截。
flushAudioChunkBuffer()
callback?.onTtsSentenceEnd(sessionId)
return
}
if (data.isNotEmpty()) { if (data.isNotEmpty()) {
recvAudio.write(data) recvAudio.write(data)
Log.d(TAG, "onMessage: partial audio appended size=${data.size}") Log.d(TAG, "onMessage: partial audio appended size=${data.size}")
@ -387,6 +404,21 @@ class DoubaoE2ETranslateHelper(
} }
} }
/**
* 冲刷 [audioChunkBuffer] 里不足对齐粒度(multiple=1280)的残留字节:原样(不补零)全部经 onPartialAudio 下发。
* [processAudioChunk] 只发送 1280 整数倍,句末最后不足一块的真实尾音(最多 <1280B ≈ 40ms@16k/16bit)会滞留在缓冲里;
* 若不冲刷,这段尾音要么串到下一句开头、要么在句末/会话末彻底丢失——表现为「部分音频结尾被截」,
* 且与句尾补偿静音(feedTailSilence)无关(那段真实尾音根本没进编码器,加大 padding 也补不回来)。
* 须在句尾补偿静音之前调用,保证真实尾音先进编码器、静音再顶在其后。
*/
private fun flushAudioChunkBuffer() {
val remain = audioChunkBuffer
if (remain.isEmpty()) return
audioChunkBuffer = ByteArray(0)
Log.d(TAG, "flushAudioChunkBuffer: 冲刷残留尾音 size=${remain.size}")
callback?.onPartialAudio(sessionId, remain)
}
/** /**
* 推送一段 PCM/WAV 音频数据到服务端。 * 推送一段 PCM/WAV 音频数据到服务端。
*/ */

20
local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt

@ -576,6 +576,26 @@ object BleService {
opusAudioManager.writeRightPcm(data) opusAudioManager.writeRightPcm(data)
} }
/**
* 句尾补偿静音(本端/左声道)。
* 在一句翻译音频的 PCM 全部喂完(豆包端到端「单句音频结束」信号)后调用:
* 往左声道编码器补一段静音 PCM,把 jl_opus 内部滞留的不足一帧真实尾音顶出成整帧,
* 避免耳机端句尾被截断;多出的静音帧一并下发给耳机,给解码/DAC pipeline 排空余量。
* 必须在本句最后一段真实 PCM 之后、同一喂入线程上调用,保证编码顺序。
* @param paddingMs 补偿静音时长(ms),默认 400
*/
fun feedLeftTailSilence(paddingMs: Int = 400) {
opusAudioManager.feedTailSilence(OpusAudioManager.CHANNEL_LEFT, paddingMs)
}
/**
* 句尾补偿静音(对端/右声道),语义同 [feedLeftTailSilence]。
* @param paddingMs 补偿静音时长(ms),默认 400
*/
fun feedRightTailSilence(paddingMs: Int = 400) {
opusAudioManager.feedTailSilence(OpusAudioManager.CHANNEL_RIGHT, paddingMs)
}
// ====================================================================================================== // ======================================================================================================
// 扫描功能 // 扫描功能
// ====================================================================================================== // ======================================================================================================

31
local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/OpusAudioManager.kt

@ -594,6 +594,37 @@ class OpusAudioManager {
rightEncodeManager?.writeEncodeStream(pcm) rightEncodeManager?.writeEncodeStream(pcm)
} }
/**
* 句尾补偿静音:一句翻译音频的真实 PCM 全部喂完后调用。
* 往指定声道编码器再补一段零 PCM,把 jl_opus 内部滞留的不足一帧真实尾音顶出成整帧,
* 避免耳机端句尾被截断;多出的静音帧一并编码下发,给解码/DAC pipeline 排空余量。
* writeEncodeStream 异步,尾音的实际打包仍由编码器在后续静默中完成,此处只负责喂入补偿 PCM。
* 必须在本句最后一段真实 PCM 之后、同一喂入线程上调用,保证编码顺序。
* @param channel CHANNEL_LEFT(0=本端/左) / CHANNEL_RIGHT(1=对端/右)
* @param paddingMs 补偿静音时长(ms),默认 400
*/
fun feedTailSilence(channel: Byte, paddingMs: Int = 400) {
if (!isDualEncoding) return
if (paddingMs <= 0) return
// 16kHz / 16bit / 单声道:32 字节 = 1ms(与 SILENCE_PCM_BYTES=100ms 的换算一致)
val silence = ByteArray(paddingMs * 32)
when (channel) {
CHANNEL_LEFT -> {
leftPcmWavWriter?.writePcmData(silence)
leftEncodeManager?.writeEncodeStream(silence)
}
CHANNEL_RIGHT -> {
rightPcmWavWriter?.writePcmData(silence)
rightEncodeManager?.writeEncodeStream(silence)
}
else -> {
Log.w(TAG, "feedTailSilence 未知声道: $channel")
return
}
}
CallLog.i(TAG, "句尾补偿静音已喂入 channel=${if (channel == CHANNEL_LEFT) "左" else "右"} paddingMs=$paddingMs bytes=${silence.size}")
}
private fun openPcmWavWriters(sampleRate: Int) { private fun openPcmWavWriters(sampleRate: Int) {
val ctx = appContext ?: return val ctx = appContext ?: return

Loading…
Cancel
Save