Browse Source

修复通话状态下,播报tts导致奔溃 问题

weicu
fdp 1 year ago
parent
commit
0beab4c8f3
  1. 281
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift
  2. 91
      local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift

281
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureTtsHelper.swift

@ -4,6 +4,7 @@ import MicrosoftCognitiveServicesSpeech
import os.log import os.log
// 自定义语音处理组件,提供音频流处理等功能 // 自定义语音处理组件,提供音频流处理等功能
import speech import speech
public class AzureTtsHelper: NSObject, ITtsService { public class AzureTtsHelper: NSObject, ITtsService {
private let tag = "AzureTtsHelper" private let tag = "AzureTtsHelper"
private let log = OSLog(subsystem: "com.azure.speech", category: "AzureTtsHelper") private let log = OSLog(subsystem: "com.azure.speech", category: "AzureTtsHelper")
@ -16,7 +17,7 @@ public class AzureTtsHelper: NSObject, ITtsService {
private var synthesizer: SPXSpeechSynthesizer? private var synthesizer: SPXSpeechSynthesizer?
private var isInitialized = false private var isInitialized = false
private var speaking = false private var speaking = false
private var sessionid = "" //语音播报任务idsetAudioOutputRoute private var sessionid = "" //语音播报任务id
// 当前配置 // 当前配置
private var currentVoice = DEFAULT_VOICE private var currentVoice = DEFAULT_VOICE
@ -51,12 +52,13 @@ public class AzureTtsHelper: NSObject, ITtsService {
private var micCapture: MicrophoneCapture! private var micCapture: MicrophoneCapture!
// 记录最后播放的文本 // 记录最后播放的文本
private var lastSpokenText: String? private var lastSpokenText: String?
/** /**
* 初始化语音合成服务 * 初始化语音合成服务
*/ */
public func initialize(ttsAppId: String, ttsAppToken: String, ttsResource: String, language: String) -> Bool { public func initialize(ttsAppId: String, ttsAppToken: String, ttsResource: String, language: String) -> Bool {
do { do {
micCapture = MicrophoneCapture.shared micCapture = MicrophoneCapture.shared
// 创建语音配置 // 创建语音配置
speechConfig = try SPXSpeechConfiguration(subscription: ttsAppToken, region: ttsResource) speechConfig = try SPXSpeechConfiguration(subscription: ttsAppToken, region: ttsResource)
speechConfig?.speechSynthesisLanguage = language speechConfig?.speechSynthesisLanguage = language
@ -76,6 +78,9 @@ public class AzureTtsHelper: NSObject, ITtsService {
// 设置初始化完成 // 设置初始化完成
isInitialized = true isInitialized = true
// 新增:监听音频会话中断和路由变化,避免通话时崩溃
setupAudioSessionObservers()
// 预热TTS引擎 // 预热TTS引擎
warmupSynthesizer() warmupSynthesizer()
@ -122,34 +127,15 @@ public class AzureTtsHelper: NSObject, ITtsService {
* @param sessionid 会话ID * @param sessionid 会话ID
* @return 是否成功启动 * @return 是否成功启动
*/ */
public func startspeak(sessionid sessionid: String)-> Bool { public func startspeak(sessionid sessionid: String) -> Bool {
self.sessionid = sessionid self.sessionid = sessionid
print("startspeak sessionid \(sessionid)") print("startspeak sessionid \(sessionid)")
// 重置当前会话的文本计数 // 重置当前会话的文本计数
self.currentSessionTextCount = 0 self.currentSessionTextCount = 0
self.pendingTextCount = 0 self.pendingTextCount = 0
do { // 使用安全的音频会话配置,兼容通话场景
// 新增:使用 AVAudioSession.sharedInstance 配置为“通话模式 + 扬声器外放” safeConfigureAudioSessionForTTS()
// - 使用 .playAndRecord 以保证通话上下行链路
// - 使用 .voiceChat 获得通话处理(AEC 等)
// - 使用 .defaultToSpeaker 默认走外放
// - 通过 overrideOutputAudioPort(.speaker) 强制切到扬声器
let audioSession = AVAudioSession.sharedInstance()
//try audioSession.setActive(false)
try audioSession.setCategory(.playAndRecord,
mode: .voiceChat,
options: [.defaultToSpeaker,
.mixWithOthers,
.defaultToSpeaker,
.allowBluetoothA2DP,
.allowBluetooth])
try audioSession.overrideOutputAudioPort(.none)
try audioSession.setActive(true)
} catch {
print("音频会话配置失败: \(error.localizedDescription)")
}
return true return true
} }
@ -157,7 +143,7 @@ public class AzureTtsHelper: NSObject, ITtsService {
/** /**
* 单次合成并播放 * 单次合成并播放
*/ */
public func speakOnce(sessionid sessionid: String,_ text: String) -> Bool { public func speakOnce(sessionid sessionid: String, _ text: String) -> Bool {
if !isInitialized { if !isInitialized {
os_log("语音合成未初始化", log: log, type: .error) os_log("语音合成未初始化", log: log, type: .error)
notifyEvent(eventType: .error, params: [ notifyEvent(eventType: .error, params: [
@ -171,7 +157,6 @@ public class AzureTtsHelper: NSObject, ITtsService {
return false return false
} }
// 清理文本 // 清理文本
let cleanedText = cleanTextForTTS(text) let cleanedText = cleanTextForTTS(text)
if cleanedText.isEmpty { if cleanedText.isEmpty {
@ -189,7 +174,7 @@ public class AzureTtsHelper: NSObject, ITtsService {
/** /**
* 流式合成文本 * 流式合成文本
*/ */
public func speakStream(sessionid sessionid: String,_ text: String) -> Bool { public func speakStream(sessionid sessionid: String, _ text: String) -> Bool {
if !isInitialized { if !isInitialized {
notifyEvent(eventType: .error, params: [ notifyEvent(eventType: .error, params: [
"errorCode": "NOT_INITIALIZED", "errorCode": "NOT_INITIALIZED",
@ -210,8 +195,6 @@ public class AzureTtsHelper: NSObject, ITtsService {
totalTextCount += text.count totalTextCount += text.count
currentSessionTextCount += text.count currentSessionTextCount += text.count
//print("流式合成文本\(text.count),总计文本数量:\(totalTextCount),当前会话:\(currentSessionTextCount)")
// 清理文本并添加到缓冲区 // 清理文本并添加到缓冲区
let cleanedText = cleanTextForTTS(text) let cleanedText = cleanTextForTTS(text)
streamBuffer.append(cleanedText) streamBuffer.append(cleanedText)
@ -226,7 +209,7 @@ public class AzureTtsHelper: NSObject, ITtsService {
let currentText = streamBuffer let currentText = streamBuffer
// 标点符号列表 (与Android一致) // 标点符号列表 (与Android一致)
let punctuationMarks: [Character] = [".", "。", "!", "!","*", "?", "?", ";", ";", ",", ",", ":", ":", "\n"] let punctuationMarks: [Character] = [".", "。", "!", "!", "*", "?", "?", ";", ";", ",", ",", ":", ":", "\n"]
// 从后往前查找最后一个标点符号 // 从后往前查找最后一个标点符号
var lastPunctuationIndex = -1 var lastPunctuationIndex = -1
@ -270,7 +253,7 @@ public class AzureTtsHelper: NSObject, ITtsService {
return true return true
} }
return speakOnce(sessionid:sessionid,remainingText) return speakOnce(sessionid: sessionid, remainingText)
} }
/** /**
@ -317,25 +300,7 @@ public class AzureTtsHelper: NSObject, ITtsService {
return true return true
} }
/**
* 释放资源
*/
public func dispose() {
_ = stop()
synthesisGroup.wait()
synthesisQueue.sync {
synthesizer = nil
speechConfig = nil
customAudioOutputStream = nil
isInitialized = false
speaking = false
taskLock.lock()
pendingTasks.removeAll()
taskLock.unlock()
}
}
/** /**
* 添加事件监听器 * 添加事件监听器
@ -374,6 +339,7 @@ public class AzureTtsHelper: NSObject, ITtsService {
recreateSynthesizer() recreateSynthesizer()
} }
} }
/** /**
* 设置音频输出设备 * 设置音频输出设备
*/ */
@ -384,6 +350,12 @@ public class AzureTtsHelper: NSObject, ITtsService {
_ = stop() _ = stop()
} }
// 新增:在通话期间不进行音频设备切换,避免崩溃
if isTelephonyActive() {
os_log("检测到通话进行中,跳过音频设备切换以避免崩溃", log: log, type: .info)
return true
}
// 重新创建合成器以应用新设备 // 重新创建合成器以应用新设备
if isInitialized { if isInitialized {
print(" 重新创建合成器以应用新设备") print(" 重新创建合成器以应用新设备")
@ -411,6 +383,7 @@ public class AzureTtsHelper: NSObject, ITtsService {
// 确保任务被处理 // 确保任务被处理
processTasksIfNeeded() processTasksIfNeeded()
} }
/** /**
* 处理任务队列 * 处理任务队列
*/ */
@ -704,5 +677,215 @@ public func resetTextCount() {
} }
/**
* 音频会话并发防护和状态
*/
private let audioSessionQueue = DispatchQueue(label: "com.azure.tts.audioSession")
private var audioInterrupted = false
private var audioSessionObserversAdded = false
// 新增:路由变化的去抖与配置重入保护标记
private var routeChangeDebounceWorkItem: DispatchWorkItem?
private var isApplyingAudioSessionConfig = false
/**
* 监听并处理音频中断与路由变化(通话/Siri/蓝牙切换等)
* - 目的:在通话期间或路由变化时安全地停止/重配,避免 AVAudioSession 断言导致崩溃
*/
private func setupAudioSessionObservers() {
if audioSessionObserversAdded { return }
audioSessionObserversAdded = true
NotificationCenter.default.addObserver(self,
selector: #selector(handleAudioInterruption(_:)),
name: AVAudioSession.interruptionNotification,
object: AVAudioSession.sharedInstance())
NotificationCenter.default.addObserver(self,
selector: #selector(handleRouteChange(_:)),
name: AVAudioSession.routeChangeNotification,
object: AVAudioSession.sharedInstance())
}
/**
* 判断当前是否处于通话相关音频占用(电话或蓝牙 HFP)
* - 使用 secondaryAudioShouldBeSilencedHint 与蓝牙 HFP 路由作为判断依据
*/
private func isTelephonyActive() -> Bool {
let session = AVAudioSession.sharedInstance()
if session.secondaryAudioShouldBeSilencedHint { return true }
// 检查是否使用蓝牙 HFP(通话用的 Hands-Free Profile)
for out in session.currentRoute.outputs {
if out.portType == .bluetoothHFP {
return true
}
}
return false
}
/**
* 安全配置音频会话用于 TTS,兼容通话场景
* - 通话中:使用 .playback + .duckOthers,避免 .voiceChat/.allowBluetoothA2DP 导致断言
* - 非通话:使用 .playAndRecord + .voiceChat + .allowBluetooth(不使用 A2DP)
*/
/// 为 TTS 安全配置音频会话:
/// - 使用异步队列,避免在同一串行队列上 sync 导致死锁
/// - 根据当前音频路由(蓝牙 A2DP / 蓝牙 HFP / 有线耳机)选择更合适的类别与模式
/// - 仅当类别或模式发生变化时才 setCategory,降低 AVAudioSessionRouteChangeReason.categoryChange 的触发概率
private func safeConfigureAudioSessionForTTS() {
// 使用异步,避免串行队列重入自锁
audioSessionQueue.async { [weak self] in
guard let self = self else { return }
// 防重入:配置中不重复进入
if self.isApplyingAudioSessionConfig { return }
self.isApplyingAudioSessionConfig = true
defer { self.isApplyingAudioSessionConfig = false }
let audioSession = AVAudioSession.sharedInstance()
// 显式指定 outputs 的类型,确保闭包参数类型可推断
let outputs: [AVAudioSessionPortDescription] = audioSession.currentRoute.outputs
// 使用 contains(where:) 明确谓词闭包。
// 保持使用 audioSessionQueue.async,防止在串行队列上自锁。
// 仅在类别或模式发生变化时调用 setCategory,减少 categoryChange 的回调风暴。
let hasBluetoothHFP = outputs.contains(where: { $0.portType == .bluetoothHFP })
let hasBluetoothA2DP = outputs.contains(where: { $0.portType == .bluetoothA2DP })
let hasWiredHeadphones = outputs.contains(where: { $0.portType == .headphones || $0.portType == .headsetMic })
// 计算目标配置
let desiredCategory: AVAudioSession.Category
let desiredMode: AVAudioSession.Mode
var desiredOptions: AVAudioSession.CategoryOptions = [.mixWithOthers]
if self.isTelephonyActive() || hasBluetoothHFP {
// 通话活跃或存在 HFP(语音)链路:使用语音聊天模式
desiredCategory = .playAndRecord
desiredMode = .voiceChat
desiredOptions.insert(.allowBluetooth) // HFP 需 allowBluetooth
} else if hasBluetoothA2DP || hasWiredHeadphones {
// 纯播放链路耳机(A2DP/有线耳机):避免强制切语音链路
desiredCategory = .playback
desiredMode = .default
desiredOptions.insert(.allowBluetoothA2DP)
} else {
// 默认:本机扬声器的语音链路
desiredCategory = .playAndRecord
desiredMode = .voiceChat
desiredOptions.insert(.defaultToSpeaker)
desiredOptions.insert(.allowBluetooth) // 允许外接麦/耳机(非 HFP 也不冲突)
}
do {
// 仅当类别或模式变化时设置,减少无谓的 categoryChange
if audioSession.category != desiredCategory || audioSession.mode != desiredMode {
try audioSession.setCategory(desiredCategory, mode: desiredMode, options: desiredOptions)
}
// 激活会话(通常不会引发死锁),失败时仅记录错误
try audioSession.setActive(true)
} catch {
os_log("TTS 音频会话配置失败: %{public}@", log: self.log, type: .error, error.localizedDescription)
}
}
}
/**
* 处理系统音频中断(来电/通话开始、Siri等)
* - began:标记中断并停止当前合成播放
* - ended:清除标记并按场景重配音频会话
*/
@objc private func handleAudioInterruption(_ notification: Notification) {
guard let info = notification.userInfo,
let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt,
let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return }
switch type {
case .began:
audioInterrupted = true
_ = stop()
os_log("音频中断开始(可能为通话/Siri),停止TTS", log: log, type: .info)
case .ended:
audioInterrupted = false
os_log("音频中断结束,重新配置会话", log: log, type: .info)
safeConfigureAudioSessionForTTS()
@unknown default:
break
}
}
/**
* 处理音频路由变化(设备插拔、蓝牙切换、扬声器/听筒切换等)
* - 忽略 .categoryChange/.routeConfigurationChange 等由我们设置类别引发的变化,避免死循环
* - 配置进行中(isApplyingAudioSessionConfig)直接忽略,防止重入
* - 加入 300ms 去抖,汇聚瞬时多次变化后再重配
* - 配置过程中(isApplyingAudioSessionConfig = true)直接忽略
*/
@objc private func handleRouteChange(_ notification: Notification) {
guard let info = notification.userInfo,
let reasonValue = info[AVAudioSessionRouteChangeReasonKey] as? UInt,
let reason = AVAudioSession.RouteChangeReason(rawValue: reasonValue) else { return }
os_log("音频路由变化: %{public}@", log: log, type: .info, String(describing: reason))
// 配置中触发的路由变化,直接忽略,避免重入导致死循环
if isApplyingAudioSessionConfig {
os_log("配置进行中,忽略本次路由变化", log: log, type: .debug)
return
}
// 忽略会造成循环的原因:categoryChange(你的日志 reason=3),以及 routeConfigurationChange 等
switch reason {
case .categoryChange, .routeConfigurationChange, .wakeFromSleep, .noSuitableRouteForCategory:
os_log("忽略路由变化原因: %{public}@", log: log, type: .debug, String(describing: reason))
return
default:
break
}
// 通话期间避免重配,避免与电话路由冲突
if isTelephonyActive() {
os_log("通话期间检测到路由变化,跳过重配", log: log, type: .info)
return
}
// 去抖处理:300ms 内只执行最后一次重配
// 忽略 categoryChange 等
// 去抖:在 audioSessionQueue 上延迟调用
routeChangeDebounceWorkItem?.cancel()
let work = DispatchWorkItem { [weak self] in
self?.safeConfigureAudioSessionForTTS()
}
routeChangeDebounceWorkItem = work
audioSessionQueue.asyncAfter(deadline: .now() + 0.3, execute: work)
}
/**
* 释放资源
*/
public func dispose() {
_ = stop()
synthesisGroup.wait()
synthesisQueue.sync {
synthesizer = nil
speechConfig = nil
customAudioOutputStream = nil
isInitialized = false
speaking = false
taskLock.lock()
pendingTasks.removeAll()
taskLock.unlock()
}
// 新增:移除音频会话观察者,防止泄漏
if audioSessionObserversAdded {
NotificationCenter.default.removeObserver(self, name: AVAudioSession.interruptionNotification, object: AVAudioSession.sharedInstance())
NotificationCenter.default.removeObserver(self, name: AVAudioSession.routeChangeNotification, object: AVAudioSession.sharedInstance())
audioSessionObserversAdded = false
}
}
} }

91
local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift

@ -2,6 +2,7 @@ import Foundation
import AVFoundation import AVFoundation
import MicrosoftCognitiveServicesSpeech import MicrosoftCognitiveServicesSpeech
import os.log import os.log
import CallKit
/** /**
* 简单音频接收器类,用于处理音频录制和流传输 * 简单音频接收器类,用于处理音频录制和流传输
@ -15,6 +16,9 @@ public class SimpleAudioReceiver: NSObject {
private let tag = "SimpleAudioReceiver" private let tag = "SimpleAudioReceiver"
private let log = OSLog(subsystem: "com.azure.speech", category: "SimpleAudioReceiver") private let log = OSLog(subsystem: "com.azure.speech", category: "SimpleAudioReceiver")
// 用于检测系统电话通话状态(iOS10+)
private let callObserver = CXCallObserver()
/** /**
* 麦克风捕获实例 * 麦克风捕获实例
* 使用 MicrophoneCapture 的单例模式 * 使用 MicrophoneCapture 的单例模式
@ -187,30 +191,27 @@ public class SimpleAudioReceiver: NSObject {
/** /**
* 运行麦克风捕获 * 运行麦克风捕获
* 针对通话/非通话场景做安全的音频会话配置,避免因会话类别/模式不兼容导致启动录音失败
*/ */
private func runMicrophoneCapture() { private func runMicrophoneCapture() {
do { do {
// 确保正在写入状态 // 确保正在写入状态
guard self._isWriting else { return } guard self._isWriting else { return }
// 设置音频会话参数
try audioSession.setCategory(.playAndRecord, // 根据通话状态进行安全的音频会话配置
mode: .videoChat, try safeConfigureAudioSessionForMic()
options: [.allowBluetoothA2DP, // 允许蓝牙耳机,不占用hfp链路
.mixWithOthers, // 启动麦克风采集
.defaultToSpeaker]) // 添加音频优先级控制 try micCapture.startCapture()
try audioSession.setActive(true) // 捕获到的数据放入写入队列(后续处理)
try micCapture.startCapture() try micCapture.audioDataHandler = { data in
try micCapture.audioDataHandler = { data in
self.writeQueue.put(data) self.writeQueue.put(data)
} }
// // 将数据放入写入队列(后续处理)
// self.writeQueue.put(data)
} catch { } catch {
print("麦克风启动失败: \(error)") print("麦克风启动失败: \(error)")
// 【新增】添加详细错误处理 // 详细错误处理
if let nsError = error as NSError? { if let nsError = error as NSError? {
print("错误域: \(nsError.domain), 错误代码: \(nsError.code)") print("错误域: \(nsError.domain), 错误代码: \(nsError.code)")
print("错误描述: \(nsError.localizedDescription)") print("错误描述: \(nsError.localizedDescription)")
@ -241,8 +242,68 @@ public class SimpleAudioReceiver: NSObject {
} }
} }
/** /**
* 继续麦克风捕获 * 在通话/非通话场景下安全配置音频会话用于麦克风采集
* - 通话中:使用 .record + .voiceChat + [.allowBluetooth],避免 A2DP 和强制外放,兼容电话路由
* - 非通话:使用 .playAndRecord + .videoChat + [.allowBluetooth, .mixWithOthers, .defaultToSpeaker]
* - 失败兜底:退回到最简单的 .record + .default,确保录音尽可能可用
*/ */
private func safeConfigureAudioSessionForMic() throws {
let audioSession = self.audioSession
do {
// 计算目标配置
let desiredCategory: AVAudioSession.Category
let desiredMode: AVAudioSession.Mode
let desiredOptions: AVAudioSession.CategoryOptions
if isTelephonyActive() {
// 通话中:录音优先,避免 A2DP(仅播放),改用 HFP(.allowBluetooth)
desiredCategory = .record
desiredMode = .voiceChat
desiredOptions = [.allowBluetooth]
} else {
// 非通话:保持原有逻辑,但使用 .allowBluetooth 以支持录音(而不是 A2DP)
desiredCategory = .playAndRecord
desiredMode = .videoChat
//desiredOptions = [.allowBluetooth, .mixWithOthers, .defaultToSpeaker]
desiredOptions = [.mixWithOthers, .defaultToSpeaker]
}
// 仅在类别或模式变化时才调用 setCategory,避免触发不必要的 categoryChange
let needsSetCategory = (audioSession.category != desiredCategory) || (audioSession.mode != desiredMode)
if needsSetCategory {
try audioSession.setCategory(desiredCategory,
mode: desiredMode,
options: desiredOptions)
} else {
// 类别与模式一致时,也尽量避免重复 setCategory;如需确保 options,可在确有必要时再设置
// 如果你确实需要校正 options,可取消下一行注释,但这可能再次触发 categoryChange
// try audioSession.setCategory(desiredCategory, mode: desiredMode, options: desiredOptions)
}
// 保持激活;这里重复 setActive(true) 一般安全,但如果你观察到重复激活也触发路由回调,可在外层控制频率
try audioSession.setActive(true, options: .notifyOthersOnDeactivation)
} catch {
// 兜底:尽量保证能开始录音
print("音频会话配置失败,尝试兜底配置: \(error)")
try audioSession.setCategory(.record, mode: .default, options: [])
try audioSession.setActive(true, options: .notifyOthersOnDeactivation)
}
}
/**
* 检测是否存在正在进行的系统电话通话
* - iOS10+:使用 CallKit 的 CXCallObserver
* - 旧系统:使用 secondaryAudioShouldBeSilencedHint 近似判断
*/
private func isTelephonyActive() -> Bool {
if #available(iOS 10.0, *) {
return callObserver.calls.contains { !$0.hasEnded }
} else {
// 非精确,但可作为兜底判断:若需要静音则可能有电话或系统占用
return AVAudioSession.sharedInstance().secondaryAudioShouldBeSilencedHint
}
}
/** /**
* 继续麦克风捕获 * 继续麦克风捕获
*/ */

Loading…
Cancel
Save