|
|
|
@ -37,6 +37,7 @@ class AgentServiceImpl: NSObject { |
|
|
|
private var apiKey: String = "" |
|
|
|
private var baseUrl: String = "https://api.openai.com/v1/chat/completions" |
|
|
|
private var model: String = "gpt-3.5-turbo" |
|
|
|
private var visionModel = "gpt-4-vision-preview" |
|
|
|
private var systemPrompt: String = "" |
|
|
|
private var chatHistory: [[String: Any]] = [] |
|
|
|
|
|
|
|
@ -55,6 +56,8 @@ class AgentServiceImpl: NSObject { |
|
|
|
private var hasSpeechDetected: Bool = false |
|
|
|
internal var isAiStreaming: Bool = false |
|
|
|
|
|
|
|
var currsessionId: String = "" |
|
|
|
|
|
|
|
private var idleTimer: DispatchSourceTimer? |
|
|
|
private let maxIdleSeconds: TimeInterval = 10 |
|
|
|
|
|
|
|
@ -245,6 +248,7 @@ class AgentServiceImpl: NSObject { |
|
|
|
apiKey: apiKey, |
|
|
|
baseUrl: baseUrl, |
|
|
|
model: model, |
|
|
|
visionModel:visionModel, |
|
|
|
mcpServer: mcpServer |
|
|
|
) |
|
|
|
|
|
|
|
@ -482,10 +486,10 @@ audioStream.saveAudioDataTo(data: audioData) |
|
|
|
displayText: displayText, |
|
|
|
hasImage: hasImage |
|
|
|
) |
|
|
|
|
|
|
|
currsessionId = UUID().uuidString |
|
|
|
os_log("设置回调并调用sendMessageStream", log: logger, type: .info) |
|
|
|
chatApiService.setStreamCallback(callback) |
|
|
|
chatApiService.sendMessageStream(messages: messages) |
|
|
|
chatApiService.setStreamCallback(currsessionId,callback) |
|
|
|
chatApiService.sendMessageStream(currsessionId,messages: messages) |
|
|
|
os_log("sendMessageStream调用完成", log: logger, type: .info) |
|
|
|
} |
|
|
|
|
|
|
|
@ -592,27 +596,28 @@ audioStream.saveAudioDataTo(data: audioData) |
|
|
|
/** |
|
|
|
* 保存聊天消息 |
|
|
|
*/ |
|
|
|
internal func saveChatMessage(userMessage: String, assistantMessage: String, metadata: String = "") { |
|
|
|
internal func saveChatMessage(userMessage: String, assistantMessage: String, metadata: String = "",userMetadata: String) { |
|
|
|
DispatchQueue.global(qos: .utility).async { |
|
|
|
// 保存用户消息 |
|
|
|
let userMessageId = self.chatStorageHelper.saveMessage( |
|
|
|
sessionId: self.sessionId, |
|
|
|
message: userMessage, |
|
|
|
sender: "user", |
|
|
|
metadata: nil as String? |
|
|
|
metadata: userMetadata |
|
|
|
) |
|
|
|
|
|
|
|
if userMessageId != -1 { |
|
|
|
// 保存AI回复 |
|
|
|
let assistantMessageId = self.chatStorageHelper.saveMessage( |
|
|
|
sessionId: self.sessionId, |
|
|
|
message: assistantMessage, |
|
|
|
sender: "assistant", |
|
|
|
metadata: metadata.isEmpty ? nil : metadata |
|
|
|
) |
|
|
|
|
|
|
|
if assistantMessageId == -1 { |
|
|
|
os_log("保存助手消息失败", log: self.logger, type: .error) |
|
|
|
if !assistantMessage.isEmpty || !metadata.isEmpty { |
|
|
|
// 保存AI回复 |
|
|
|
let assistantMessageId = self.chatStorageHelper.saveMessage( |
|
|
|
sessionId: self.sessionId, |
|
|
|
message: assistantMessage, |
|
|
|
sender: "assistant", |
|
|
|
metadata: metadata.isEmpty ? nil : metadata |
|
|
|
) |
|
|
|
if assistantMessageId == -1 { |
|
|
|
os_log("保存助手消息失败", log: self.logger, type: .error) |
|
|
|
} |
|
|
|
} |
|
|
|
} else { |
|
|
|
os_log("保存用户消息失败", log: self.logger, type: .error) |
|
|
|
@ -622,16 +627,26 @@ audioStream.saveAudioDataTo(data: audioData) |
|
|
|
|
|
|
|
//解析卡片数据 |
|
|
|
@discardableResult |
|
|
|
internal func autoHandleFunctionCallResult(toolName: String, functionCallResult: [String: Any]) -> (String, Bool) { |
|
|
|
internal func autoHandleFunctionCallResult(toolName: String, functionCallResult: [String: Any]) -> [String:Any] { |
|
|
|
if let contentArray = functionCallResult["content"] as? [[String: Any]], |
|
|
|
let firstContent = contentArray.first, |
|
|
|
let textStr = firstContent["text"] as? String, |
|
|
|
let jsonData = textStr.data(using: .utf8) { |
|
|
|
do { |
|
|
|
var broadcast = true; |
|
|
|
var notreply = false; |
|
|
|
if let jsonObject = try JSONSerialization.jsonObject(with: jsonData, options: []) as? [String: Any] { |
|
|
|
print("解析成功:\(jsonObject)") |
|
|
|
var metadata: [String: Any] = [:] |
|
|
|
metadata[toolName] = jsonObject |
|
|
|
|
|
|
|
if let _broadcast = jsonObject["broadcast"] as? Bool { |
|
|
|
broadcast = _broadcast |
|
|
|
} |
|
|
|
if let _notreply = jsonObject["notreply"] as? Bool { |
|
|
|
notreply = _notreply |
|
|
|
} |
|
|
|
metadata["notreply"] = notreply |
|
|
|
// 将 metadata 转为 JSON 字符串 |
|
|
|
let metaStr: String |
|
|
|
if let jsonData = try? JSONSerialization.data(withJSONObject: metadata, options: [.prettyPrinted]), |
|
|
|
@ -642,7 +657,7 @@ audioStream.saveAudioDataTo(data: audioData) |
|
|
|
} |
|
|
|
// 处理 card_music |
|
|
|
if let cardMusic = jsonObject["card_music"] as? [String: Any] { |
|
|
|
let id = cardMusic["id"] as? String ?? "" |
|
|
|
let id = cardMusic["id"] as? Int ?? 0 |
|
|
|
let url = cardMusic["url"] as? String ?? "" |
|
|
|
let title = cardMusic["name"] as? String ?? "" |
|
|
|
let artist = cardMusic["sgener"] as? String ?? "" |
|
|
|
@ -655,21 +670,21 @@ audioStream.saveAudioDataTo(data: audioData) |
|
|
|
"artist": artist, |
|
|
|
"coverUrl": coverUrl |
|
|
|
]) |
|
|
|
return (metaStr, false) |
|
|
|
// 处理 card_musiclist |
|
|
|
} else if let cardMusiclist = jsonObject["card_musiclist"] as? [String: Any], |
|
|
|
let musics = cardMusiclist["musics"] as? [[String: Any]] { |
|
|
|
|
|
|
|
var playlist: [[String: String]] = [] |
|
|
|
var playlist: [[String: Any]] = [] |
|
|
|
|
|
|
|
for item in musics { |
|
|
|
let id = item["id"] as? String ?? "" |
|
|
|
let id = item["id"] as? Int ?? 0 |
|
|
|
let url = item["url"] as? String ?? "" |
|
|
|
let title = item["name"] as? String ?? "" |
|
|
|
let artist = item["sgener"] as? String ?? "" |
|
|
|
let coverUrl = item["image"] as? String ?? "" |
|
|
|
let ts = item["ts"] as? Int ?? 0 |
|
|
|
|
|
|
|
let song: [String: String] = [ |
|
|
|
let song: [String: Any] = [ |
|
|
|
"id": id, |
|
|
|
"url": url, |
|
|
|
"title": title, |
|
|
|
@ -682,33 +697,38 @@ audioStream.saveAudioDataTo(data: audioData) |
|
|
|
if !playlist.isEmpty { |
|
|
|
print("自动播放音乐列表: \(playlist)") |
|
|
|
processMusicPlayList(playlist) |
|
|
|
return (metaStr, false) |
|
|
|
} else { |
|
|
|
print("card_musiclist 中没有有效的音乐条目") |
|
|
|
} |
|
|
|
}else if let cardNavigation = jsonObject["card_navigation"] as? [String: Any] { //导航 |
|
|
|
let start:String = cardNavigation["start"] as? String ?? "" |
|
|
|
let end:String = cardNavigation["end"] as? String ?? "" |
|
|
|
// processNavigation(start,end) |
|
|
|
return (metaStr, false) |
|
|
|
} |
|
|
|
return (metaStr, true) |
|
|
|
return [ |
|
|
|
"meta": metaStr, |
|
|
|
"broadcast": broadcast, |
|
|
|
"notreply": notreply |
|
|
|
] |
|
|
|
} |
|
|
|
} catch { |
|
|
|
print("JSON 解析失败: \(error)") |
|
|
|
} |
|
|
|
} |
|
|
|
return ("", true) |
|
|
|
return [ |
|
|
|
"meta": "", |
|
|
|
"broadcast": true, |
|
|
|
"notreply": false, |
|
|
|
] |
|
|
|
} |
|
|
|
|
|
|
|
// 后台播放单曲 |
|
|
|
private func processMusicPlay(_ song: [String: String]) { |
|
|
|
private func processMusicPlay(_ song: [String: Any]) { |
|
|
|
sendEvent(name: "music_play", data: song) |
|
|
|
MusicService.shared.play(song: song) |
|
|
|
} |
|
|
|
|
|
|
|
// 后台播放音乐列表 |
|
|
|
private func processMusicPlayList(_ songs: [[String: String]]) { |
|
|
|
private func processMusicPlayList(_ songs: [[String: Any]]) { |
|
|
|
// sendEvent(name: "music_playlist", data: songs) |
|
|
|
MusicService.shared.playPlaylist(songs: songs) |
|
|
|
} |
|
|
|
@ -795,7 +815,9 @@ class ChatApiStreamCallback: StreamCallback { |
|
|
|
private let hasImage: Bool |
|
|
|
private var responseBuilder = "" |
|
|
|
private var metadata = "" |
|
|
|
private var nobroadcast = true |
|
|
|
private var userMetadata:[String: Any] = [:] |
|
|
|
private var reply = true; //是否回复 |
|
|
|
private var broadcast = true //是否播发 |
|
|
|
|
|
|
|
init(agentService: AgentServiceImpl, speakResponse: Bool, displayText: String, hasImage: Bool) { |
|
|
|
self.agentService = agentService |
|
|
|
@ -805,31 +827,39 @@ class ChatApiStreamCallback: StreamCallback { |
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
func onToken(_ token: String) { |
|
|
|
guard let agentService = agentService else { return } |
|
|
|
func onUsage(_ sessionid: String,_ prompt_tokens: Int?,_ completion_tokens: Int?,_ total_tokens: Int?){ |
|
|
|
|
|
|
|
|
|
|
|
responseBuilder += token |
|
|
|
|
|
|
|
agentService.sendEvent(name: "assistant_token", data: ["token": token]) |
|
|
|
|
|
|
|
if speakResponse && nobroadcast{ |
|
|
|
agentService.azureTtsHelper?.speakStream(token) |
|
|
|
} |
|
|
|
func onToken(_ sessionId:String,_ token: String) { |
|
|
|
do { |
|
|
|
guard let agentService = try agentService else { return } |
|
|
|
if (sessionId != agentService.currsessionId) { |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
responseBuilder += token |
|
|
|
|
|
|
|
if speakResponse && reply && broadcast{ |
|
|
|
try agentService.azureTtsHelper?.speakStream(token) |
|
|
|
} |
|
|
|
if (reply && token != "") { |
|
|
|
try agentService.sendEvent(name: "assistant_token", data: ["token": token]) |
|
|
|
} |
|
|
|
}catch { |
|
|
|
print("liwei--------------- AI Call onToken 异常: \(error)") |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
func onComplete() { |
|
|
|
guard let agentService = agentService else { return } |
|
|
|
|
|
|
|
|
|
|
|
if speakResponse && nobroadcast{ |
|
|
|
agentService.azureTtsHelper?.flushStream() |
|
|
|
} |
|
|
|
|
|
|
|
let response = responseBuilder |
|
|
|
|
|
|
|
if !response.isEmpty { |
|
|
|
func onComplete(_ sessionId:String) { |
|
|
|
do { |
|
|
|
guard let agentService = try agentService else { return } |
|
|
|
|
|
|
|
if speakResponse && reply && broadcast && sessionId == agentService.currsessionId{ |
|
|
|
agentService.azureTtsHelper?.flushStream() |
|
|
|
} |
|
|
|
|
|
|
|
let response = responseBuilder |
|
|
|
|
|
|
|
var responseData: [String: Any] = [ |
|
|
|
"text": response, |
|
|
|
"userInput": displayText |
|
|
|
@ -837,77 +867,109 @@ class ChatApiStreamCallback: StreamCallback { |
|
|
|
if hasImage { |
|
|
|
responseData["hasImage"] = true |
|
|
|
} |
|
|
|
agentService.sendEvent(name: "assistant_response", data: responseData) |
|
|
|
if (reply) { |
|
|
|
try agentService.sendEvent(name: "assistant_response", data: responseData) |
|
|
|
} |
|
|
|
|
|
|
|
let assistantMessage: [String: Any] = [ |
|
|
|
"role": "assistant", |
|
|
|
"content": response |
|
|
|
] |
|
|
|
agentService.addToHistoryMessages(assistantMessage) |
|
|
|
|
|
|
|
if !response.isEmpty { |
|
|
|
try agentService.addToHistoryMessages(assistantMessage) |
|
|
|
} |
|
|
|
// 保存聊天记录 |
|
|
|
agentService.saveChatMessage(userMessage: displayText, assistantMessage: response, metadata: metadata) |
|
|
|
try agentService.saveChatMessage(userMessage: displayText, assistantMessage: response, metadata: metadata,userMetadata:toJSONString(userMetadata)) |
|
|
|
agentService.isAiStreaming = false |
|
|
|
os_log("设置AI流式状态为false", log: agentService.logger, type: .info) |
|
|
|
}catch{ |
|
|
|
print("liwei--------------- AI Call onComplete 异常: \(error)") |
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
func onError(_ sessionId:String,_ error: Error) { |
|
|
|
do { |
|
|
|
guard let agentService = try agentService else { return } |
|
|
|
|
|
|
|
os_log("保存聊天记录完成", log: agentService.logger, type: .info) |
|
|
|
} else { |
|
|
|
os_log("警告: 响应内容为空", log: agentService.logger, type: .error) |
|
|
|
os_log("ChatAPI错误: %{public}@", log: agentService.logger, type: .error, error.localizedDescription) |
|
|
|
if (sessionId != agentService.currsessionId) { |
|
|
|
return |
|
|
|
} |
|
|
|
try agentService.sendEvent(name: "error", data: [ |
|
|
|
"code": "AI_ERROR", |
|
|
|
"message": error.localizedDescription |
|
|
|
]) |
|
|
|
|
|
|
|
agentService.isAiStreaming = false |
|
|
|
os_log("因错误设置AI流式状态为false", log: agentService.logger, type: .info) |
|
|
|
}catch{ |
|
|
|
print("liwei--------------- AI Call onError 异常: \(error)") |
|
|
|
} |
|
|
|
|
|
|
|
agentService.isAiStreaming = false |
|
|
|
os_log("设置AI流式状态为false", log: agentService.logger, type: .info) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
func onError(_ error: Error) { |
|
|
|
guard let agentService = agentService else { return } |
|
|
|
|
|
|
|
os_log("ChatAPI错误: %{public}@", log: agentService.logger, type: .error, error.localizedDescription) |
|
|
|
|
|
|
|
agentService.sendEvent(name: "error", data: [ |
|
|
|
"code": "AI_ERROR", |
|
|
|
"message": error.localizedDescription |
|
|
|
]) |
|
|
|
|
|
|
|
agentService.isAiStreaming = false |
|
|
|
os_log("因错误设置AI流式状态为false", log: agentService.logger, type: .info) |
|
|
|
func onFunctionCall(_ sessionId:String,_ functionCall: [String: Any]) { |
|
|
|
do { |
|
|
|
guard let agentService = try agentService else { return } |
|
|
|
if (sessionId != agentService.currsessionId) { |
|
|
|
return |
|
|
|
} |
|
|
|
let name = functionCall["name"] as? String ?? "" |
|
|
|
os_log("收到函数调用: %{public}@", log: agentService.logger, type: .info, name) |
|
|
|
|
|
|
|
try agentService.audioPlayer?.playCallingSound() |
|
|
|
|
|
|
|
try agentService.sendEvent(name: "function_call", data: [ |
|
|
|
"name": name, |
|
|
|
"arguments": functionCall |
|
|
|
]) |
|
|
|
|
|
|
|
if name == "exit_interaction" { |
|
|
|
os_log("执行退出交互", log: agentService.logger, type: .info) |
|
|
|
agentService.stopRecognition() |
|
|
|
} |
|
|
|
|
|
|
|
}catch{ |
|
|
|
print("liwei--------------- AI Call onFunctionCall 异常: \(error)") |
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
func onFunctionCall(_ functionCall: [String: Any]) { |
|
|
|
guard let agentService = agentService else { return } |
|
|
|
|
|
|
|
let name = functionCall["name"] as? String ?? "" |
|
|
|
os_log("收到函数调用: %{public}@", log: agentService.logger, type: .info, name) |
|
|
|
|
|
|
|
agentService.audioPlayer?.playCallingSound() |
|
|
|
|
|
|
|
agentService.sendEvent(name: "function_call", data: [ |
|
|
|
"name": name, |
|
|
|
"arguments": functionCall |
|
|
|
]) |
|
|
|
|
|
|
|
if name == "exit_interaction" { |
|
|
|
os_log("执行退出交互", log: agentService.logger, type: .info) |
|
|
|
agentService.stopRecognition() |
|
|
|
func onFunctionCallResult(_ sessionId:String,_ functionCall: [String: Any], _ functionCallResult: [String: Any]) { |
|
|
|
do { |
|
|
|
guard let agentService = agentService else { return } |
|
|
|
|
|
|
|
let name = functionCall["name"] as? String ?? "" |
|
|
|
os_log("收到函数调用结果: %{public}@", log: agentService.logger, type: .info, functionCallResult) |
|
|
|
|
|
|
|
agentService.audioPlayer?.stopCallingSound() |
|
|
|
let resultmap = agentService.autoHandleFunctionCallResult(toolName:name,functionCallResult:functionCallResult) |
|
|
|
metadata = resultmap["meta"] as? String ?? "" |
|
|
|
broadcast = resultmap["broadcast"] as? Bool ?? true |
|
|
|
reply = !(resultmap["notreply"] as? Bool ?? true) |
|
|
|
os_log("卡片数据: %{public}@", log: agentService.logger, type: .info, metadata) |
|
|
|
agentService.sendEvent(name: "function_call_result", data: [ |
|
|
|
"function_call": functionCall, |
|
|
|
"result": functionCallResult, |
|
|
|
"meta":metadata, |
|
|
|
]) |
|
|
|
os_log("处理函数调用结果完成", log: agentService.logger, type: .info) |
|
|
|
}catch{ |
|
|
|
print("liwei--------------- AI Call onFunctionCallResult 异常: \(error)") |
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
func onFunctionCallResult(_ functionCall: [String: Any], _ functionCallResult: [String: Any]) { |
|
|
|
guard let agentService = agentService else { return } |
|
|
|
|
|
|
|
let name = functionCall["name"] as? String ?? "" |
|
|
|
os_log("收到函数调用结果: %{public}@", log: agentService.logger, type: .info, functionCallResult) |
|
|
|
|
|
|
|
agentService.audioPlayer?.stopCallingSound() |
|
|
|
let (metestr, broadcast) = agentService.autoHandleFunctionCallResult(toolName:name,functionCallResult:functionCallResult) |
|
|
|
metadata = metestr; |
|
|
|
nobroadcast = broadcast |
|
|
|
os_log("卡片数据: %{public}@", log: agentService.logger, type: .info, metadata) |
|
|
|
agentService.sendEvent(name: "function_call_result", data: [ |
|
|
|
"function_call": functionCall, |
|
|
|
"result": functionCallResult, |
|
|
|
"meta":metadata, |
|
|
|
]) |
|
|
|
os_log("处理函数调用结果完成", log: agentService.logger, type: .info) |
|
|
|
func toJSONString(_ dict: [String: Any]) -> String { |
|
|
|
guard JSONSerialization.isValidJSONObject(dict), |
|
|
|
let data = try? JSONSerialization.data(withJSONObject: dict), |
|
|
|
let json = String(data: data, encoding: .utf8) else { |
|
|
|
return "" |
|
|
|
} |
|
|
|
return json |
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
extension AgentServiceImpl: TtsEventListener { |
|
|
|
|