|
|
@ -59,6 +59,7 @@ class AgentServiceImpl: NSObject { |
|
|
internal var isAiStreaming: Bool = false |
|
|
internal var isAiStreaming: Bool = false |
|
|
|
|
|
|
|
|
var currsessionId: String = "" |
|
|
var currsessionId: String = "" |
|
|
|
|
|
var language: String = "" |
|
|
|
|
|
|
|
|
private var idleTimer: DispatchSourceTimer? |
|
|
private var idleTimer: DispatchSourceTimer? |
|
|
private let maxIdleSeconds: TimeInterval = 10 |
|
|
private let maxIdleSeconds: TimeInterval = 10 |
|
|
@ -158,6 +159,9 @@ class AgentServiceImpl: NSObject { |
|
|
if let openaivisionModel = config["openaivisionModel"] as? String { |
|
|
if let openaivisionModel = config["openaivisionModel"] as? String { |
|
|
self.visionModel = openaivisionModel |
|
|
self.visionModel = openaivisionModel |
|
|
} |
|
|
} |
|
|
|
|
|
if let supportedLanguages = config["supportedLanguages"] as? [String] { |
|
|
|
|
|
self.language = supportedLanguages[0] |
|
|
|
|
|
} |
|
|
if let customSystemPrompt = config["systemPrompt"] as? String, !customSystemPrompt.isEmpty { |
|
|
if let customSystemPrompt = config["systemPrompt"] as? String, !customSystemPrompt.isEmpty { |
|
|
self.systemPrompt = customSystemPrompt |
|
|
self.systemPrompt = customSystemPrompt |
|
|
} else { |
|
|
} else { |
|
|
@ -208,7 +212,7 @@ class AgentServiceImpl: NSObject { |
|
|
azureTtsHelper?.addListener(self) |
|
|
azureTtsHelper?.addListener(self) |
|
|
|
|
|
|
|
|
// 获取支持的语言列表，支持从配置中读取 |
|
|
// 获取支持的语言列表，支持从配置中读取 |
|
|
let supportedLanguages = config["supportedLanguages"] as? [String] ?? ["zh-CN", "en-US"] |
|
|
let supportedLanguages = config["supportedLanguages"] as? [String] ?? ["zh-CN"] |
|
|
|
|
|
|
|
|
guard let asrSuccess = azureAsrHelper?.initialize( |
|
|
guard let asrSuccess = azureAsrHelper?.initialize( |
|
|
subscriptionKey: azureSpeechKey, |
|
|
subscriptionKey: azureSpeechKey, |
|
|
@ -287,8 +291,8 @@ class AgentServiceImpl: NSObject { |
|
|
// 使用更安全的方式检查self是否存在 |
|
|
// 使用更安全的方式检查self是否存在 |
|
|
DispatchQueue.main.async { [weak self] in |
|
|
DispatchQueue.main.async { [weak self] in |
|
|
guard let self = self else { return } |
|
|
guard let self = self else { return } |
|
|
|
|
|
|
|
|
if self.isRecognizing && !self.hasSpeechDetected && !self.isSpeaking && !self.isAiStreaming { |
|
|
if self.isRecognizing && !self.isSpeaking && !self.isAiStreaming { |
|
|
self.stopRecognition() |
|
|
self.stopRecognition() |
|
|
self.sendEvent(name: "auto_stop", data: [ |
|
|
self.sendEvent(name: "auto_stop", data: [ |
|
|
"reason": "idle_timeout", |
|
|
"reason": "idle_timeout", |
|
|
@ -345,8 +349,14 @@ print("ai启动语音=\(audioSourceType)=\(useBle)") |
|
|
|
|
|
|
|
|
return true |
|
|
return true |
|
|
} |
|
|
} |
|
|
func recognizeCallback(useBle: Bool = false) -> Bool { |
|
|
func recognizeCallback() -> Bool { |
|
|
|
|
|
print("重新设置识别回调=") |
|
|
|
|
|
guard let asrsetupSuccess = azureAsrHelper?.setupEventListeners( |
|
|
|
|
|
callback: self |
|
|
|
|
|
), asrsetupSuccess else { |
|
|
|
|
|
sendError("初始化语音识别服务失败", code: "ASR_INIT_ERROR") |
|
|
|
|
|
return false |
|
|
|
|
|
} |
|
|
return true |
|
|
return true |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
@ -361,7 +371,12 @@ print("ai启动语音=\(audioSourceType)=\(useBle)") |
|
|
|
|
|
|
|
|
return true |
|
|
return true |
|
|
} |
|
|
} |
|
|
|
|
|
func disableBluetoothAudio() { |
|
|
|
|
|
azureAsrHelper?.disableBluetoothAudio() |
|
|
|
|
|
} |
|
|
|
|
|
func restoreOriginalAudioState() { |
|
|
|
|
|
azureAsrHelper?.restoreOriginalAudioState() |
|
|
|
|
|
} |
|
|
func pushAudioData(_ audioData: Data) -> Bool { |
|
|
func pushAudioData(_ audioData: Data) -> Bool { |
|
|
if !isRecognizing || !isInitialized { |
|
|
if !isRecognizing || !isInitialized { |
|
|
return false |
|
|
return false |
|
|
@ -454,8 +469,8 @@ audioStream.saveAudioDataTo(data: audioData) |
|
|
processWithChatApiServiceInternal(userMessage: userMessage, displayText: text, speakResponse: speakResponse) |
|
|
processWithChatApiServiceInternal(userMessage: userMessage, displayText: text, speakResponse: speakResponse) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
private func processWithChatApiServiceInternal(userMessage: [String: Any], displayText: String, speakResponse: Bool = true, hasImage: Bool = false) { |
|
|
private func processWithChatApiServiceInternal(userMessage: [String: Any], displayText: String, speakResponse: Bool = true, hasImage: Bool = false,imagePath:String = "") { |
|
|
os_log("开始处理ChatAPI请求: displayText=%{public}@, speakResponse=%{public}@, hasImage=%{public}@", |
|
|
os_log("开始处理ChatAPI请求: displayText=%{public}@, speakResponse=%{public}@, hasImage=%{public}@", |
|
|
log: logger, type: .info, displayText, speakResponse ? "是" : "否", hasImage ? "是" : "否") |
|
|
log: logger, type: .info, displayText, speakResponse ? "是" : "否", hasImage ? "是" : "否") |
|
|
|
|
|
|
|
|
guard let chatApiService = chatApiService else { |
|
|
guard let chatApiService = chatApiService else { |
|
|
@ -476,9 +491,16 @@ audioStream.saveAudioDataTo(data: audioData) |
|
|
var messages: [[String: Any]] = [] |
|
|
var messages: [[String: Any]] = [] |
|
|
|
|
|
|
|
|
if !systemPrompt.isEmpty { |
|
|
if !systemPrompt.isEmpty { |
|
|
|
|
|
var info = UserPortrait.shared.getAll() |
|
|
let info: [String: String] = UserPortrait.shared.getAll() |
|
|
info["language"] = self.language |
|
|
|
|
|
|
|
|
|
|
|
// 添加调试日志 |
|
|
|
|
|
print("系统提示词模板: \(systemPrompt)") |
|
|
|
|
|
print("用户画像数据: \(info)") |
|
|
|
|
|
|
|
|
let _systemPrompt = fillTemplate(systemPrompt, with: info) |
|
|
let _systemPrompt = fillTemplate(systemPrompt, with: info) |
|
|
|
|
|
print("替换后的系统提示词: \(_systemPrompt)") |
|
|
|
|
|
|
|
|
messages.append(chatApiService.createSystemMessage(content: _systemPrompt)) |
|
|
messages.append(chatApiService.createSystemMessage(content: _systemPrompt)) |
|
|
os_log("添加系统提示消息", log: logger, type: .info) |
|
|
os_log("添加系统提示消息", log: logger, type: .info) |
|
|
} |
|
|
} |
|
|
@ -495,7 +517,8 @@ audioStream.saveAudioDataTo(data: audioData) |
|
|
agentService: self, |
|
|
agentService: self, |
|
|
speakResponse: speakResponse, |
|
|
speakResponse: speakResponse, |
|
|
displayText: displayText, |
|
|
displayText: displayText, |
|
|
hasImage: hasImage |
|
|
hasImage: hasImage, |
|
|
|
|
|
imagePath: imagePath |
|
|
) |
|
|
) |
|
|
currsessionId = UUID().uuidString |
|
|
currsessionId = UUID().uuidString |
|
|
os_log("设置回调并调用sendMessageStream", log: logger, type: .info) |
|
|
os_log("设置回调并调用sendMessageStream", log: logger, type: .info) |
|
|
@ -552,7 +575,8 @@ audioStream.saveAudioDataTo(data: audioData) |
|
|
userMessage: userMessage, |
|
|
userMessage: userMessage, |
|
|
displayText: displayText, |
|
|
displayText: displayText, |
|
|
speakResponse: speakResponse, |
|
|
speakResponse: speakResponse, |
|
|
hasImage: true |
|
|
hasImage: true, |
|
|
|
|
|
imagePath: imagePath |
|
|
) |
|
|
) |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
@ -879,53 +903,201 @@ private func jsonToString(_ json: [String: Any]) -> String? { |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
/** |
|
|
/** |
|
|
* 设置支持的语音识别语言 |
|
|
* 设置支持的语音识别语言，同时更新TTS语言设置 |
|
|
* |
|
|
* |
|
|
* @param languages 支持的语言列表，如 ["zh-CN", "en-US"] |
|
|
* @param languages 支持的语言列表，如 ["zh-CN", "en-US"] |
|
|
* @return 是否设置成功 |
|
|
* @return 是否设置成功 |
|
|
*/ |
|
|
*/ |
|
|
func setSupportedLanguages(_ languages: [String]) -> Bool { |
|
|
func setSupportedLanguages(_ languages: [String]) -> Bool { |
|
|
os_log("设置支持的语言: %{public}@", log: logger, type: .info, languages.joined(separator: ", ")) |
|
|
os_log("设置支持的语言: %{public}@", log: logger, type: .info, languages.joined(separator: ", ")) |
|
|
|
|
|
|
|
|
if !isInitialized { |
|
|
if !isInitialized { |
|
|
os_log("服务未初始化，无法设置语言", log: logger, type: .error) |
|
|
os_log("服务未初始化，无法设置语言", log: logger, type: .error) |
|
|
sendError("服务未初始化", code: "NOT_INITIALIZED") |
|
|
sendError("服务未初始化", code: "NOT_INITIALIZED") |
|
|
return false |
|
|
return false |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
if languages.isEmpty { |
|
|
if languages.isEmpty { |
|
|
os_log("语言列表为空，使用默认语言", log: logger, type: .info) |
|
|
os_log("语言列表为空，使用默认语言", log: logger, type: .info) |
|
|
return false |
|
|
return false |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
self.language = languages[0] |
|
|
|
|
|
|
|
|
// 如果正在识别，需要先停止 |
|
|
// 如果正在识别，需要先停止 |
|
|
let wasRecognizing = isRecognizing |
|
|
let wasRecognizing = isRecognizing |
|
|
if wasRecognizing { |
|
|
if wasRecognizing { |
|
|
_ = stopRecognition() |
|
|
_ = stopRecognition() |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
// 重新初始化ASR引擎，使用新的语言设置 |
|
|
// 重新初始化ASR引擎，使用新的语言设置 |
|
|
guard let success = azureAsrHelper?.initialize( |
|
|
guard let asrSuccess = azureAsrHelper?.initialize( |
|
|
subscriptionKey: azureSpeechKey, |
|
|
subscriptionKey: azureSpeechKey, |
|
|
region: azureSpeechRegion, |
|
|
region: azureSpeechRegion, |
|
|
supportedLanguages: languages, |
|
|
supportedLanguages: languages, |
|
|
audioSourceType: .microphone |
|
|
audioSourceType: .microphone |
|
|
), success else { |
|
|
), asrSuccess else { |
|
|
os_log("重新初始化ASR引擎失败", log: logger, type: .error) |
|
|
sendError("初始化语音识别服务失败", code: "ASR_INIT_ERROR") |
|
|
sendError("重新初始化ASR引擎失败", code: "ASR_INIT_ERROR") |
|
|
return false |
|
|
|
|
|
} |
|
|
|
|
|
guard let asrsetupSuccess = azureAsrHelper?.setupEventListeners( |
|
|
|
|
|
callback: self |
|
|
|
|
|
), asrsetupSuccess else { |
|
|
|
|
|
sendError("初始化语音识别服务失败", code: "ASR_INIT_ERROR") |
|
|
return false |
|
|
return false |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
os_log("成功设置支持的语言: %{public}@", log: logger, type: .info, languages.joined(separator: ", ")) |
|
|
// 同时更新TTS语言设置 |
|
|
|
|
|
let ttsSuccess = updateTtsLanguage(languages.first ?? "zh-CN") |
|
|
|
|
|
|
|
|
|
|
|
let overallSuccess = asrSuccess && ttsSuccess |
|
|
|
|
|
|
|
|
|
|
|
if overallSuccess { |
|
|
|
|
|
os_log("ASR和TTS语言设置更新成功: %{public}@", log: logger, type: .info, languages.joined(separator: ", ")) |
|
|
|
|
|
} else { |
|
|
|
|
|
os_log("语言设置更新失败 - ASR: %{public}@, TTS: %{public}@", log: logger, type: .error, String(asrSuccess), String(ttsSuccess)) |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
// 如果之前正在识别，重新开始 |
|
|
// 如果之前正在识别，重新开始 |
|
|
if wasRecognizing { |
|
|
if wasRecognizing { |
|
|
_ = startRecognition() |
|
|
_ = startRecognition() |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
return true |
|
|
return overallSuccess |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
* 更新TTS语言设置 |
|
|
|
|
|
* |
|
|
|
|
|
* @param languageCode ASR语言代码，如 "zh-CN", "en-US" |
|
|
|
|
|
* @return 是否更新成功 |
|
|
|
|
|
*/ |
|
|
|
|
|
private func updateTtsLanguage(_ languageCode: String) -> Bool { |
|
|
|
|
|
os_log("更新TTS语言设置: %{public}@", log: logger, type: .info, languageCode) |
|
|
|
|
|
|
|
|
|
|
|
// 根据ASR语言代码映射到对应的TTS语音角色 |
|
|
|
|
|
let voiceName = getTtsVoiceNameByAsrCode(languageCode) |
|
|
|
|
|
|
|
|
|
|
|
if !voiceName.isEmpty { |
|
|
|
|
|
// 设置TTS语音角色 |
|
|
|
|
|
let success = azureTtsHelper?.setVoice(voiceName) ?? false |
|
|
|
|
|
|
|
|
|
|
|
if success { |
|
|
|
|
|
os_log("TTS语音角色设置成功: %{public}@", log: logger, type: .info, voiceName) |
|
|
|
|
|
} else { |
|
|
|
|
|
os_log("TTS语音角色设置失败: %{public}@", log: logger, type: .error, voiceName) |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
return success |
|
|
|
|
|
} else { |
|
|
|
|
|
os_log("未找到语言代码 %{public}@ 对应的TTS语音角色", log: logger, type: .error, languageCode) |
|
|
|
|
|
return false |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
* 根据ASR语言代码获取对应的TTS语音角色名称 |
|
|
|
|
|
* 与LanguageManager中的映射保持一致 |
|
|
|
|
|
* |
|
|
|
|
|
* @param asrCode ASR语言代码，如 "zh-CN", "en-US" |
|
|
|
|
|
* @return TTS语音角色名称 |
|
|
|
|
|
*/ |
|
|
|
|
|
private func getTtsVoiceNameByAsrCode(_ asrCode: String) -> String { |
|
|
|
|
|
switch asrCode { |
|
|
|
|
|
// 中文相关 |
|
|
|
|
|
case "zh-CN": |
|
|
|
|
|
return "zh-CN-XiaoxiaoNeural" |
|
|
|
|
|
case "zh-TW": |
|
|
|
|
|
return "zh-TW-HsiaoChenNeural" |
|
|
|
|
|
case "zh-HK": |
|
|
|
|
|
return "zh-HK-HiuMaanNeural" |
|
|
|
|
|
|
|
|
|
|
|
// 英语相关 |
|
|
|
|
|
case "en-US": |
|
|
|
|
|
return "en-US-AriaNeural" |
|
|
|
|
|
|
|
|
|
|
|
// 亚洲语言 |
|
|
|
|
|
case "ja-JP": |
|
|
|
|
|
return "ja-JP-NanamiNeural" |
|
|
|
|
|
case "ko-KR": |
|
|
|
|
|
return "ko-KR-SunHiNeural" |
|
|
|
|
|
case "th-TH": |
|
|
|
|
|
return "th-TH-PremwadeeNeural" |
|
|
|
|
|
case "vi-VN": |
|
|
|
|
|
return "vi-VN-HoaiMyNeural" |
|
|
|
|
|
case "id-ID": |
|
|
|
|
|
return "id-ID-GadisNeural" |
|
|
|
|
|
case "hi-IN": |
|
|
|
|
|
return "hi-IN-SwaraNeural" |
|
|
|
|
|
case "ms-MY": |
|
|
|
|
|
return "ms-MY-YasminNeural" |
|
|
|
|
|
case "bn-IN": |
|
|
|
|
|
return "bn-IN-TanishaaNeural" |
|
|
|
|
|
case "ta-IN": |
|
|
|
|
|
return "ta-IN-PallaviNeural" |
|
|
|
|
|
case "te-IN": |
|
|
|
|
|
return "te-IN-ShrutiNeural" |
|
|
|
|
|
case "mr-IN": |
|
|
|
|
|
return "mr-IN-AarohiNeural" |
|
|
|
|
|
case "ur-IN": |
|
|
|
|
|
return "ur-IN-GulNeural" |
|
|
|
|
|
case "tr-TR": |
|
|
|
|
|
return "tr-TR-EmelNeural" |
|
|
|
|
|
case "fa-IR": |
|
|
|
|
|
return "fa-IR-DilaraNeural" |
|
|
|
|
|
|
|
|
|
|
|
// 欧洲语言 |
|
|
|
|
|
case "fr-FR": |
|
|
|
|
|
return "fr-FR-DeniseNeural" |
|
|
|
|
|
case "es-ES": |
|
|
|
|
|
return "es-ES-ElviraNeural" |
|
|
|
|
|
case "pt-BR": |
|
|
|
|
|
return "pt-BR-FranciscaNeural" |
|
|
|
|
|
case "it-IT": |
|
|
|
|
|
return "it-IT-ElsaNeural" |
|
|
|
|
|
case "de-DE": |
|
|
|
|
|
return "de-DE-KatjaNeural" |
|
|
|
|
|
case "ru-RU": |
|
|
|
|
|
return "ru-RU-SvetlanaNeural" |
|
|
|
|
|
case "pl-PL": |
|
|
|
|
|
return "pl-PL-AgnieszkaNeural" |
|
|
|
|
|
case "nl-NL": |
|
|
|
|
|
return "nl-NL-ColetteNeural" |
|
|
|
|
|
case "sv-SE": |
|
|
|
|
|
return "sv-SE-SofieNeural" |
|
|
|
|
|
case "cs-CZ": |
|
|
|
|
|
return "cs-CZ-VlastaNeural" |
|
|
|
|
|
case "el-GR": |
|
|
|
|
|
return "el-GR-AthinaNeural" |
|
|
|
|
|
case "ro-RO": |
|
|
|
|
|
return "ro-RO-AlinaNeural" |
|
|
|
|
|
case "hu-HU": |
|
|
|
|
|
return "hu-HU-NoemiNeural" |
|
|
|
|
|
case "uk-UA": |
|
|
|
|
|
return "uk-UA-PolinaNeural" |
|
|
|
|
|
case "da-DK": |
|
|
|
|
|
return "da-DK-ChristelNeural" |
|
|
|
|
|
case "fi-FI": |
|
|
|
|
|
return "fi-FI-NooraNeural" |
|
|
|
|
|
case "nb-NO": |
|
|
|
|
|
return "nb-NO-IselinNeural" |
|
|
|
|
|
case "hr-HR": |
|
|
|
|
|
return "hr-HR-GabrijelaNeural" |
|
|
|
|
|
case "ca-ES": |
|
|
|
|
|
return "ca-ES-JoanaNeural" |
|
|
|
|
|
|
|
|
|
|
|
// 阿拉伯语和非洲语言 |
|
|
|
|
|
case "ar-EG": |
|
|
|
|
|
return "ar-EG-SalmaNeural" |
|
|
|
|
|
case "sw-KE": |
|
|
|
|
|
return "sw-KE-ZuriNeural" |
|
|
|
|
|
|
|
|
|
|
|
default: |
|
|
|
|
|
os_log("未支持的语言代码: %{public}@，使用默认英语语音", log: logger, type: .error, asrCode) |
|
|
|
|
|
return "en-US-AriaNeural" |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
func dispose() -> Bool { |
|
|
func dispose() -> Bool { |
|
|
print("ai释放资源") |
|
|
print("ai释放资源") |
|
|
if isRecognizing { |
|
|
if isRecognizing { |
|
|
@ -967,19 +1139,22 @@ class ChatApiStreamCallback: StreamCallback { |
|
|
private let speakResponse: Bool |
|
|
private let speakResponse: Bool |
|
|
private let displayText: String |
|
|
private let displayText: String |
|
|
private let hasImage: Bool |
|
|
private let hasImage: Bool |
|
|
|
|
|
private let imagePath: String |
|
|
private var responseBuilder = "" |
|
|
private var responseBuilder = "" |
|
|
private var metadata = "" |
|
|
private var metadata = "" |
|
|
private var userMetadata:[String: Any] = [:] |
|
|
private var userMetadata:[String: Any] = [:] |
|
|
private var reply = true; //是否回复 |
|
|
private var reply = true; //是否回复 |
|
|
private var broadcast = true //是否播发 |
|
|
private var broadcast = true //是否播发 |
|
|
|
|
|
|
|
|
init(agentService: AgentServiceImpl, speakResponse: Bool, displayText: String, hasImage: Bool) { |
|
|
init(agentService: AgentServiceImpl, speakResponse: Bool, displayText: String, hasImage: Bool,imagePath:String) { |
|
|
self.agentService = agentService |
|
|
self.agentService = agentService |
|
|
self.speakResponse = speakResponse |
|
|
self.speakResponse = speakResponse |
|
|
self.displayText = displayText |
|
|
self.displayText = displayText |
|
|
self.hasImage = hasImage |
|
|
self.hasImage = hasImage |
|
|
|
|
|
self.imagePath = imagePath |
|
|
|
|
|
self.userMetadata["hasImage"] = hasImage |
|
|
|
|
|
self.userMetadata["imagePath"] = imagePath |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
} |
|
|
func onUsage(_ sessionid: String,_ prompt_tokens: Int?,_ completion_tokens: Int?,_ total_tokens: Int?){ |
|
|
func onUsage(_ sessionid: String,_ prompt_tokens: Int?,_ completion_tokens: Int?,_ total_tokens: Int?){ |
|
|
|
|
|
|
|
|
@ -1418,6 +1593,7 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { |
|
|
hasSpeechDetected = true |
|
|
hasSpeechDetected = true |
|
|
|
|
|
|
|
|
if !previousHasSpeech { |
|
|
if !previousHasSpeech { |
|
|
|
|
|
print("onRecognizing restartIdleCheck") |
|
|
restartIdleCheck() |
|
|
restartIdleCheck() |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
