|
|
|
@ -110,6 +110,28 @@ import ble_service |
|
|
|
private var bailianCallbackA: AliyunCallbackProxy? |
|
|
|
private var bailianCallbackB: AliyunCallbackProxy? |
|
|
|
|
|
|
|
// 通话翻译：对方(B)路 E2E 返回的翻译音频（16kHz PCM16 mono）直接用 AVAudioEngine 播到系统默认路由 |
|
|
|
private let callBPcmPlayer = CallBPcmPlayer() |
|
|
|
// 设备产品配置 broadcastpeertranslate：true=播报并启用 gate；false/未下发=不播报且 gate 永不触发。默认 false。 |
|
|
|
private var broadcastPeerTranslate: Bool = false |
|
|
|
|
|
|
|
fileprivate func pushCallBPcm(_ data: Data) { |
|
|
|
guard broadcastPeerTranslate else { return } |
|
|
|
callBPcmPlayer.feed(data) |
|
|
|
} |
|
|
|
|
|
|
|
fileprivate func resetCallBPlayer() { |
|
|
|
callBPcmPlayer.stop() |
|
|
|
} |
|
|
|
|
|
|
|
/// 若 B 路本地播放未启用或当前不在播放窗口内，返回原始 data； |
|
|
|
/// 否则返回同长度的零字节 PCM，既保持 ws 连续推流又阻断声学回环。 |
|
|
|
fileprivate func gateAgainstBPlayback(_ data: Data) -> Data { |
|
|
|
if !broadcastPeerTranslate { return data } |
|
|
|
if !callBPcmPlayer.isPlaying { return data } |
|
|
|
return Data(count: data.count) |
|
|
|
} |
|
|
|
|
|
|
|
// 音频数据相关 |
|
|
|
private var audioDataEventChannel: FlutterEventChannel? |
|
|
|
internal var audioDataEventSink: FlutterEventSink? |
|
|
|
@ -830,6 +852,16 @@ private func sendAudioDataEvent(_ event: [String: Any]) { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
case "setBroadcastPeerTranslate": |
|
|
|
let args = call.arguments as? [String: Any] |
|
|
|
let enabled = (args?["enabled"] as? Bool) ?? false |
|
|
|
broadcastPeerTranslate = enabled |
|
|
|
if !enabled { |
|
|
|
callBPcmPlayer.stop() |
|
|
|
} |
|
|
|
os_log("[CALL] setBroadcastPeerTranslate=%{public}@", log: ctLog, type: .info, enabled ? "true" : "false") |
|
|
|
result(true) |
|
|
|
|
|
|
|
case "dispose": |
|
|
|
os_log("[AST-DISPOSE] Releasing all AST resources, provider=%{public}@", log: ctLog, type: .info, currentAstProvider) |
|
|
|
azureAstHelperA.dispose() |
|
|
|
@ -838,6 +870,7 @@ private func sendAudioDataEvent(_ event: [String: Any]) { |
|
|
|
doubaoAstHelperB.dispose() |
|
|
|
bailianAstHelperA.dispose() |
|
|
|
bailianAstHelperB.dispose() |
|
|
|
resetCallBPlayer() |
|
|
|
os_log("[AST-DISPOSE] All AST resources released", log: ctLog, type: .info) |
|
|
|
result(true) |
|
|
|
|
|
|
|
@ -1285,26 +1318,29 @@ extension AzureSpeechPlugin: BleService.Callback { |
|
|
|
|
|
|
|
// 根据当前 AST 提供商路由音频数据 |
|
|
|
// 左声道 -> A（己方语音），右声道 -> B（对方语音） |
|
|
|
// B 路本地播放期间，A/B 两路都替换为零字节 PCM 防止 TTS 回放被麦克风回采形成闭环 |
|
|
|
let leftPayload = gateAgainstBPlayback(leftBuffer) |
|
|
|
let rightPayload = gateAgainstBPlayback(rightBuffer) |
|
|
|
if currentAstProvider == "volcano" { |
|
|
|
doubaoAstHelperA.pushAudioData(leftBuffer) |
|
|
|
doubaoAstHelperB.pushAudioData(rightBuffer) |
|
|
|
doubaoAstHelperA.pushAudioData(leftPayload) |
|
|
|
doubaoAstHelperB.pushAudioData(rightPayload) |
|
|
|
if bleAudioRecvCount % 200 == 1 { |
|
|
|
os_log("[STS] pushAudio: volcano A(left)=%d bytes, B(right)=%d bytes, count=%lld", |
|
|
|
log: ctLog, type: .info, leftBuffer.count, rightBuffer.count, bleAudioRecvCount) |
|
|
|
log: ctLog, type: .info, leftPayload.count, rightPayload.count, bleAudioRecvCount) |
|
|
|
} |
|
|
|
} else if currentAstProvider == "alibaba" { |
|
|
|
bailianAstHelperA.pushAudioData(leftBuffer) |
|
|
|
bailianAstHelperB.pushAudioData(rightBuffer) |
|
|
|
bailianAstHelperA.pushAudioData(leftPayload) |
|
|
|
bailianAstHelperB.pushAudioData(rightPayload) |
|
|
|
if bleAudioRecvCount % 200 == 1 { |
|
|
|
os_log("[STS] pushAudio: alibaba A(left)=%d bytes, B(right)=%d bytes, count=%lld", |
|
|
|
log: ctLog, type: .info, leftBuffer.count, rightBuffer.count, bleAudioRecvCount) |
|
|
|
log: ctLog, type: .info, leftPayload.count, rightPayload.count, bleAudioRecvCount) |
|
|
|
} |
|
|
|
} else { |
|
|
|
azureAstHelperA.pushAudioData(audioData: leftBuffer) |
|
|
|
azureAstHelperB.pushAudioData(audioData: rightBuffer) |
|
|
|
azureAstHelperA.pushAudioData(audioData: leftPayload) |
|
|
|
azureAstHelperB.pushAudioData(audioData: rightPayload) |
|
|
|
if bleAudioRecvCount % 200 == 1 { |
|
|
|
os_log("[STS] pushAudio: azure A(left)=%d bytes, B(right)=%d bytes, count=%lld", |
|
|
|
log: ctLog, type: .info, leftBuffer.count, rightBuffer.count, bleAudioRecvCount) |
|
|
|
log: ctLog, type: .info, leftPayload.count, rightPayload.count, bleAudioRecvCount) |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
@ -1429,8 +1465,7 @@ private class DoubaoCallbackProxy: DoubaoE2ETranslateHelper.Callback { |
|
|
|
} |
|
|
|
|
|
|
|
func onPartialSourceText(sessionId: String, text: String) { |
|
|
|
os_log("[DoubaoCallback-%{public}@] onPartialSourceText text=%{public}@", |
|
|
|
log: ctLog, type: .info, serviceId, text) |
|
|
|
// 高频事件，日志改 debug；避免 event loop 拥塞引发豆包 AudioSendSlow(1011) |
|
|
|
plugin?.sendAstEvent([ |
|
|
|
"type": "recognizing", |
|
|
|
"serviceId": serviceId, |
|
|
|
@ -1456,8 +1491,7 @@ private class DoubaoCallbackProxy: DoubaoE2ETranslateHelper.Callback { |
|
|
|
} |
|
|
|
|
|
|
|
func onPartialText(sessionId: String, text: String) { |
|
|
|
os_log("[DoubaoCallback-%{public}@] onPartialText text=%{public}@", |
|
|
|
log: ctLog, type: .info, serviceId, text) |
|
|
|
// 高频事件，取消 info 日志 |
|
|
|
plugin?.sendAstEvent([ |
|
|
|
"type": "translatedInterim", |
|
|
|
"serviceId": serviceId, |
|
|
|
@ -1470,11 +1504,13 @@ private class DoubaoCallbackProxy: DoubaoE2ETranslateHelper.Callback { |
|
|
|
} |
|
|
|
|
|
|
|
func onPartialAudio(sessionId: String, data: Data) { |
|
|
|
let isServiceA = (serviceId == "A") |
|
|
|
if data.count > 0 && isServiceA { |
|
|
|
os_log("[DoubaoCallback-%{public}@] onPartialAudio writing %d bytes to BLE", |
|
|
|
log: ctLog, type: .debug, serviceId, data.count) |
|
|
|
if data.isEmpty { return } |
|
|
|
if serviceId == "A" { |
|
|
|
// A 路：翻译后的音频通过 BLE 写到耳机外放给对方 |
|
|
|
BleService.shared.writeExternalAudioData(data: data) |
|
|
|
} else { |
|
|
|
// B 路：对方的翻译音频，本地 PCM 播放器直接播 |
|
|
|
plugin?.pushCallBPcm(data) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
@ -1562,8 +1598,7 @@ private class AliyunCallbackProxy: AliyunBailianE2EHelper.Callback { |
|
|
|
} |
|
|
|
|
|
|
|
func onPartialSourceText(sessionId: String, text: String) { |
|
|
|
os_log("[AliyunCallback-%{public}@] onPartialSourceText text=%{public}@", |
|
|
|
log: ctLog, type: .info, serviceId, text) |
|
|
|
// 高频事件，取消 info 日志 |
|
|
|
plugin?.sendAstEvent([ |
|
|
|
"type": "recognizing", |
|
|
|
"serviceId": serviceId, |
|
|
|
@ -1589,8 +1624,7 @@ private class AliyunCallbackProxy: AliyunBailianE2EHelper.Callback { |
|
|
|
} |
|
|
|
|
|
|
|
func onPartialText(sessionId: String, text: String) { |
|
|
|
os_log("[AliyunCallback-%{public}@] onPartialText text=%{public}@", |
|
|
|
log: ctLog, type: .info, serviceId, text) |
|
|
|
// 高频事件，取消 info 日志 |
|
|
|
plugin?.sendAstEvent([ |
|
|
|
"type": "translatedInterim", |
|
|
|
"serviceId": serviceId, |
|
|
|
@ -1603,11 +1637,11 @@ private class AliyunCallbackProxy: AliyunBailianE2EHelper.Callback { |
|
|
|
} |
|
|
|
|
|
|
|
func onPartialAudio(sessionId: String, data: Data) { |
|
|
|
let isServiceA = (serviceId == "A") |
|
|
|
if data.count > 0 && isServiceA { |
|
|
|
os_log("[AliyunCallback-%{public}@] onPartialAudio writing %d bytes to BLE", |
|
|
|
log: ctLog, type: .debug, serviceId, data.count) |
|
|
|
if data.isEmpty { return } |
|
|
|
if serviceId == "A" { |
|
|
|
BleService.shared.writeExternalAudioData(data: data) |
|
|
|
} else { |
|
|
|
plugin?.pushCallBPcm(data) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
@ -1825,3 +1859,115 @@ private class AsrCallbackWrapper: AzureAsrHelper.ContinuousRecognizeCallback { |
|
|
|
]) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// MARK: - 通话翻译 B 路 PCM 本地播放器 |
|
|
|
/// 16kHz / Int16 / mono PCM 流式播放；由豆包/阿里 E2E onPartialAudio 逐帧灌入 |
|
|
|
final class CallBPcmPlayer { |
|
|
|
private let log = OSLog(subsystem: "com.azure.speech", category: "CallBPcmPlayer") |
|
|
|
private let queue = DispatchQueue(label: "com.azure.speech.call.b.pcm") |
|
|
|
private let engine = AVAudioEngine() |
|
|
|
private let player = AVAudioPlayerNode() |
|
|
|
private let sourceFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, |
|
|
|
sampleRate: 16000, |
|
|
|
channels: 1, |
|
|
|
interleaved: true)! |
|
|
|
private lazy var outputFormat: AVAudioFormat = engine.mainMixerNode.outputFormat(forBus: 0) |
|
|
|
private var converter: AVAudioConverter? |
|
|
|
private var attached = false |
|
|
|
|
|
|
|
// 排空时间点（CACurrentMediaTime 秒）—— 小于此时间视为 B 路仍在播，上游应对 A/B 路推静音防回环 |
|
|
|
private let untilLock = NSLock() |
|
|
|
private var playbackUntil: CFTimeInterval = 0 |
|
|
|
private let tailPadding: CFTimeInterval = 0.2 |
|
|
|
|
|
|
|
/// 当前是否还在播放窗口内（供 pushAudioData 入口查询决定是否替换为静音帧） |
|
|
|
var isPlaying: Bool { |
|
|
|
untilLock.lock(); defer { untilLock.unlock() } |
|
|
|
return CACurrentMediaTime() < playbackUntil |
|
|
|
} |
|
|
|
|
|
|
|
private func ensureAttached() { |
|
|
|
guard !attached else { return } |
|
|
|
engine.attach(player) |
|
|
|
engine.connect(player, to: engine.mainMixerNode, format: outputFormat) |
|
|
|
converter = AVAudioConverter(from: sourceFormat, to: outputFormat) |
|
|
|
attached = true |
|
|
|
} |
|
|
|
|
|
|
|
private func ensureRunning() { |
|
|
|
if engine.isRunning { return } |
|
|
|
do { |
|
|
|
try engine.start() |
|
|
|
if !player.isPlaying { player.play() } |
|
|
|
} catch { |
|
|
|
os_log("engine start failed: %{public}@", log: log, type: .error, error.localizedDescription) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
func feed(_ data: Data) { |
|
|
|
if data.isEmpty { return } |
|
|
|
// 16bit mono 16kHz → 每字节 = 1/32ms；+ 尾部静默余量避免回环残尾 |
|
|
|
let durSec = Double(data.count) / (16000.0 * 2.0) |
|
|
|
untilLock.lock() |
|
|
|
let now = CACurrentMediaTime() |
|
|
|
let base = max(now, playbackUntil) |
|
|
|
playbackUntil = base + durSec + tailPadding |
|
|
|
untilLock.unlock() |
|
|
|
|
|
|
|
queue.async { [weak self] in |
|
|
|
guard let self = self else { return } |
|
|
|
self.ensureAttached() |
|
|
|
self.ensureRunning() |
|
|
|
guard let converter = self.converter else { return } |
|
|
|
|
|
|
|
let frameCount = AVAudioFrameCount(data.count / 2) |
|
|
|
guard frameCount > 0, |
|
|
|
let inBuf = AVAudioPCMBuffer(pcmFormat: self.sourceFormat, frameCapacity: frameCount) else { |
|
|
|
return |
|
|
|
} |
|
|
|
inBuf.frameLength = frameCount |
|
|
|
if let dst = inBuf.int16ChannelData?[0] { |
|
|
|
data.withUnsafeBytes { raw in |
|
|
|
if let src = raw.baseAddress { |
|
|
|
memcpy(dst, src, Int(frameCount) * 2) |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
let ratio = self.outputFormat.sampleRate / self.sourceFormat.sampleRate |
|
|
|
let outCap = AVAudioFrameCount(Double(frameCount) * ratio + 1024) |
|
|
|
guard let outBuf = AVAudioPCMBuffer(pcmFormat: self.outputFormat, frameCapacity: outCap) else { |
|
|
|
return |
|
|
|
} |
|
|
|
var supplied = false |
|
|
|
var err: NSError? |
|
|
|
let status = converter.convert(to: outBuf, error: &err) { _, flag in |
|
|
|
if supplied { |
|
|
|
flag.pointee = .noDataNow |
|
|
|
return nil |
|
|
|
} |
|
|
|
supplied = true |
|
|
|
flag.pointee = .haveData |
|
|
|
return inBuf |
|
|
|
} |
|
|
|
if status == .error || err != nil { |
|
|
|
os_log("convert failed: %{public}@", log: self.log, type: .error, err?.localizedDescription ?? "?") |
|
|
|
return |
|
|
|
} |
|
|
|
if outBuf.frameLength > 0 { |
|
|
|
self.player.scheduleBuffer(outBuf, completionHandler: nil) |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
func stop() { |
|
|
|
untilLock.lock() |
|
|
|
playbackUntil = 0 |
|
|
|
untilLock.unlock() |
|
|
|
queue.async { [weak self] in |
|
|
|
guard let self = self else { return } |
|
|
|
if self.player.isPlaying { self.player.stop() } |
|
|
|
if self.engine.isRunning { self.engine.stop() } |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
