diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 3e7edd752..e37032b85 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -625,95 +625,11 @@ audioStream.saveAudioDataTo(data: audioData) } } - //解析卡片数据 - @discardableResult - internal func autoHandleFunctionCallResult(toolName: String, functionCallResult: [String: Any]) -> [String:Any] { - if let contentArray = functionCallResult["content"] as? [[String: Any]], - let firstContent = contentArray.first, - let textStr = firstContent["text"] as? String, - let jsonData = textStr.data(using: .utf8) { - do { - var broadcast = true; - var notreply = false; - if let jsonObject = try JSONSerialization.jsonObject(with: jsonData, options: []) as? [String: Any] { - print("解析成功：\(jsonObject)") - var metadata: [String: Any] = [:] - metadata[toolName] = jsonObject - - if let _broadcast = jsonObject["broadcast"] as? Bool { - broadcast = _broadcast - } - if let _notreply = jsonObject["notreply"] as? Bool { - notreply = _notreply - } - metadata["notreply"] = notreply - // 将 metadata 转为 JSON 字符串 - let metaStr: String - if let jsonData = try? JSONSerialization.data(withJSONObject: metadata, options: [.prettyPrinted]), - let jsonString = String(data: jsonData, encoding: .utf8) { - metaStr = jsonString - } else { - metaStr = "" - } - // 处理 card_music - if let cardMusic = jsonObject["card_music"] as? [String: Any] { - let id = cardMusic["id"] as? Int ?? 0 - let url = cardMusic["url"] as? String ?? "" - let title = cardMusic["name"] as? String ?? "" - let artist = cardMusic["sgener"] as? String ?? "" - let coverUrl = cardMusic["image"] as? String ?? "" - - processMusicPlay([ - "id": id, - "url": url, - "title": title, - "artist": artist, - "coverUrl": coverUrl - ]) - // 处理 card_musiclist - } else if let cardMusiclist = jsonObject["card_musiclist"] as? [String: Any], - let musics = cardMusiclist["musics"] as? [[String: Any]] { - - var playlist: [[String: Any]] = [] - - for item in musics { - let id = item["id"] as? Int ?? 0 - let url = item["url"] as? String ?? "" - let title = item["name"] as? String ?? "" - let artist = item["sgener"] as? String ?? "" - let coverUrl = item["image"] as? String ?? "" - let ts = item["ts"] as? Int ?? 0 - - let song: [String: Any] = [ - "id": id, - "url": url, - "title": title, - "artist": artist, - "coverUrl": coverUrl - ] - playlist.append(song) - } - - if !playlist.isEmpty { - print("自动播放音乐列表: \(playlist)") - processMusicPlayList(playlist) - } else { - print("card_musiclist 中没有有效的音乐条目") - } - }else if let cardNavigation = jsonObject["card_navigation"] as? [String: Any] { //导航 - let start:String = cardNavigation["start"] as? String ?? "" - let end:String = cardNavigation["end"] as? String ?? "" - } - return [ - "meta": metaStr, - "broadcast": broadcast, - "notreply": notreply - ] - } - } catch { - print("JSON 解析失败: \(error)") - } - } + // 解析卡片数据 +@discardableResult +internal func autoHandleFunctionCallResult(toolName: String, functionCallResult: [String: Any]) -> [String:Any] { + // 1. 从functionCallResult获取context + guard let contextValue = functionCallResult["context"] else { return [ "meta": "", "broadcast": true, @@ -721,6 +637,162 @@ audioStream.saveAudioDataTo(data: audioData) ] } + var contextStr = "" + + // 处理不同类型的context值 + if let strValue = contextValue as? String { + contextStr = strValue + } else if let data = try? JSONSerialization.data(withJSONObject: contextValue, options: []), + let str = String(data: data, encoding: .utf8) { + contextStr = str + } + + // 尝试解析JSON + if let jsonData = contextStr.data(using: .utf8), + let jsonObject = try? JSONSerialization.jsonObject(with: jsonData, options: .allowFragments) { + + // 处理字典类型的JSON + if let jsonDict = jsonObject as? [String: Any] { + return handleJSONDictionary(jsonDict, toolName: toolName) + } + // 处理数组类型的JSON + else if let jsonArray = jsonObject as? [[String: Any]] { + return handleJSONDictionary(["array": jsonArray], toolName: toolName) + } + // 处理基础类型 + else { + return handlePrimitiveType(jsonObject, toolName: toolName) + } + } + + // 纯文本内容 + return handlePlainText(contextStr, toolName: toolName) +} + +// 处理字典类型的JSON +private func handleJSONDictionary(_ jsonDict: [String: Any], toolName: String) -> [String: Any] { + var broadcast = true + var notreply = false + var metadata: [String: Any] = [:] + metadata[toolName] = jsonDict + + if let _broadcast = jsonDict["broadcast"] as? Bool { + broadcast = _broadcast + } + if let _notreply = jsonDict["notreply"] as? Bool { + notreply = _notreply + } + metadata["notreply"] = notreply + + // 处理卡片数据 + if let cardMusic = jsonDict["card_music"] as? [String: Any] { + processMusicCard(cardMusic) + } else if let cardMusiclist = jsonDict["card_musiclist"] as? [String: Any], + let musics = cardMusiclist["musics"] as? [[String: Any]] { + processMusicListCard(musics) + } else if let cardNavigation = jsonDict["card_navigation"] as? [String: Any] { + processNavigationCard(cardNavigation) + } + + let metaStr: String + if let jsonData = try? JSONSerialization.data(withJSONObject: metadata, options: []), + let jsonString = String(data: jsonData, encoding: .utf8) { + metaStr = jsonString + } else { + metaStr = "" + } + + return [ + "meta": metaStr, + "broadcast": broadcast, + "notreply": notreply + ] +} + +// 处理基本类型（字符串、数字等） +private func handlePrimitiveType(_ value: Any, toolName: String) -> [String: Any] { + let metaStr = "{\"\(toolName)\": \"\(value)\"}" + return [ + "meta": metaStr, + "broadcast": true, + "notreply": false + ] +} + +// 处理纯文本内容 +private func handlePlainText(_ text: String, toolName: String) -> [String: Any] { + let metaStr = "{\"\(toolName)\": \"\(text)\"}" + return [ + "meta": metaStr, + "broadcast": true, + "notreply": false + ] +} + +/// 辅助方法：将字典转换为JSON字符串 +private func jsonToString(_ json: [String: Any]) -> String? { + do { + let jsonData = try JSONSerialization.data(withJSONObject: json, options: []) + return String(data: jsonData, encoding: .utf8) + } catch { + print("JSON序列化失败: \(error)") + return nil + } +} + +// 处理音乐卡片 +private func processMusicCard(_ cardMusic: [String: Any]) { + let id = cardMusic["id"] as? Int ?? 0 + let url = cardMusic["url"] as? String ?? "" + let title = cardMusic["name"] as? String ?? "" + let artist = cardMusic["sgener"] as? String ?? "" + let coverUrl = cardMusic["image"] as? String ?? "" + let ts = cardMusic["ts"] as? Int ?? 0 + + processMusicPlay([ + "id": id, + "url": url, + "title": title, + "artist": artist, + "coverUrl": coverUrl, + "ts": ts + ]) +} + +// 处理音乐列表卡片 +private func processMusicListCard(_ musics: [[String: Any]]) { + var playlist: [[String: Any]] = [] + + for item in musics { + let id = item["id"] as? Int ?? 0 + let url = item["url"] as? String ?? "" + let title = item["name"] as? String ?? "" + let artist = item["sgener"] as? String ?? "" + let coverUrl = item["image"] as? String ?? "" + let ts = item["ts"] as? Int ?? 0 + + playlist.append([ + "id": id, + "url": url, + "title": title, + "artist": artist, + "coverUrl": coverUrl, + "ts": ts + ]) + } + + if !playlist.isEmpty { + processMusicPlayList(playlist) + } +} + +// 处理导航卡片 +private func processNavigationCard(_ cardNavigation: [String: Any]) { + let start = cardNavigation["start"] as? String ?? "" + let end = cardNavigation["end"] as? String ?? "" + // 实际处理导航逻辑 + print("启动导航: 从 \(start) 到 \(end)") +} // 后台播放单曲 private func processMusicPlay(_ song: [String: Any]) { sendEvent(name: "music_play", data: song) @@ -917,8 +989,8 @@ class ChatApiStreamCallback: StreamCallback { } let name = functionCall["name"] as? String ?? "" os_log("收到函数调用: %{public}@", log: agentService.logger, type: .info, name) - - try agentService.audioPlayer?.playCallingSound() + print("onFunctionCall=functionCall\(functionCall)") + // try agentService.audioPlayer?.playCallingSound() try agentService.sendEvent(name: "function_call", data: [ "name": name, @@ -943,7 +1015,7 @@ class ChatApiStreamCallback: StreamCallback { let name = functionCall["name"] as? String ?? "" os_log("收到函数调用结果: %{public}@", log: agentService.logger, type: .info, functionCallResult) - agentService.audioPlayer?.stopCallingSound() + // agentService.audioPlayer?.stopCallingSound() let resultmap = agentService.autoHandleFunctionCallResult(toolName:name,functionCallResult:functionCallResult) metadata = resultmap["meta"] as? String ?? "" broadcast = resultmap["broadcast"] as? Bool ?? true diff --git a/local_plugins/chat_api/ios/chat_api/Sources/chat_api/ChatApiService.swift b/local_plugins/chat_api/ios/chat_api/Sources/chat_api/ChatApiService.swift index a63e6a74a..ad6b53219 100644 --- a/local_plugins/chat_api/ios/chat_api/Sources/chat_api/ChatApiService.swift +++ b/local_plugins/chat_api/ios/chat_api/Sources/chat_api/ChatApiService.swift @@ -252,192 +252,313 @@ public class ChatApiService: NSObject { } /// 发送消息（流式输出） - public func sendMessageStream(_ sessionId:String,messages: [[String: Any]]) { - // 1. 立即中止旧会话 - abortCurrentSession() - - // 2. 创建新会话 - currSessionId = sessionId - - guard isInitialized && !apiKey.isEmpty, let openAI = openAI else { - let error = ChatApiException("ChatAPI服务未初始化") - getSessionCallback(sessionId)?.onError(sessionId,error) - return - } - - - // 重置状态 - currentMessages = messages - toolCalls = [:] - - // 检查是否包含图片，决定使用哪个模型 - var currentModel = model - if containsImage(in: messages) { - currentModel = visionModel - } + // MARK: - 流式请求实现 +public func sendMessageStream(_ sessionId: String, messages: [[String: Any]]) { + // 1. 立即中止旧会话 + abortCurrentSession() + + // 2. 创建新会话 + currSessionId = sessionId + + guard isInitialized && !apiKey.isEmpty, let openAI = openAI else { + let error = ChatApiException("ChatAPI服务未初始化") + getSessionCallback(sessionId)?.onError(sessionId, error) + return + } + + // 保存当前消息和回调 + currentMessages = messages + toolCalls = [:] + sessionCallbacks[sessionId] = getSessionCallback(sessionId) + + // 检查是否包含图片，决定模型 + let currentModel = containsImage(in: messages) ? visionModel : model + + currentStreamTask = Task { [weak self] in + guard let self = self else { return } + var replyContext = "" // 收集AI回复内容 - currentStreamTask = Task { [weak self] in - guard let self = self else { return } + do { + // 转换消息格式 + let chatMessages = try self.convertToChatMessages(messages) - do { - // 转换消息格式 - let chatMessages = try self.convertToChatMessages(messages) - - // 获取MCP工具列表 - let tools = self.mcpClient?.getToolMaps() ?? [] - - // 构建请求 - let query: ChatQuery - if !tools.isEmpty { - // 转换工具格式 - let chatTools = self.convertToOpenAITools(tools) - query = ChatQuery( - messages: chatMessages, - model: currentModel, - maxTokens: 2000, - temperature: 0.7, - tools: chatTools - ) - } else { - query = ChatQuery( - messages: chatMessages, - model: currentModel, - maxTokens: 2000, - temperature: 0.7 - ) + // 获取工具列表 + let tools = self.mcpClient?.getToolMaps() ?? [] + + // 构建请求 + let query: ChatQuery + if !tools.isEmpty { + let chatTools = self.convertToOpenAITools(tools) + query = ChatQuery( + messages: chatMessages, + model: currentModel, + maxTokens: 2000, + temperature: 0.7, + tools: chatTools + ) + } else { + query = ChatQuery( + messages: chatMessages, + model: currentModel, + maxTokens: 2000, + temperature: 0.7 + ) + } + + let chatsStream: AsyncThrowingStream = openAI.chatsStream(query: query) + + for try await result in chatsStream { + // 检查会话是否已变更 + if sessionId != self.currSessionId { + return } - let chatsStream: AsyncThrowingStream = openAI.chatsStream(query: query) + guard let choice = result.choices.first else { continue } + let delta = choice.delta + + // 处理文本内容 + if let content = delta.content { + replyContext += content + self.getSessionCallback(sessionId)?.onToken(sessionId, content) + } - for try await result in chatsStream { - if sessionId != self.currSessionId { - return - } - - if let choice = result.choices.first { - let delta = choice.delta + // 收集工具调用信息 + if let toolCalls = delta.toolCalls { + for toolCall in toolCalls { + let index = toolCall.index - // 处理普通文本内容 - if let content = delta.content { - getSessionCallback(sessionId)?.onToken(sessionId,content) + // 初始化工具调用信息 + if self.toolCalls[index] == nil { + self.toolCalls[index] = ToolCallInfo() } - // 收集工具调用信息 - if let toolCalls = delta.toolCalls { - for toolCall in toolCalls { - let index = toolCall.index - - // 创建或获取现有的工具调用信息 - if self.toolCalls[index] == nil { - self.toolCalls[index] = ToolCallInfo() - } - - let toolCallInfo = self.toolCalls[index]! - - // 更新ID - if let id = toolCall.id { - toolCallInfo.id = id - } - - // 更新函数信息 - if let function = toolCall.function { - if let name = function.name { - toolCallInfo.name = name - } - - if let arguments = function.arguments { - toolCallInfo.arguments += arguments - } - } + guard let toolCallInfo = self.toolCalls[index] else { continue } + + // 更新ID + if let id = toolCall.id { + toolCallInfo.id = id + } + + // 更新函数信息 + if let function = toolCall.function { + if let name = function.name { + toolCallInfo.name = name + } + + if let arguments = function.arguments { + toolCallInfo.arguments += arguments } } } } + } + + // 流式响应结束 + if sessionId == self.currSessionId { + print("发送消息（流式输出）\(replyContext)") + // 计算token使用量 + let promptTokens = self.estimatePromptTokens(messages: self.currentMessages) + let completionTokens = self.roughTokenCount(text: replyContext) + let totalTokens = promptTokens + completionTokens + self.getSessionCallback(sessionId)?.onUsage(sessionId, promptTokens, completionTokens, totalTokens) - if sessionId == self.currSessionId { - // 处理工具调用或完成 - let hasToolCalls = await self.processToolCalls(sessionId) - if !hasToolCalls { - getSessionCallback(sessionId)?.onComplete(sessionId) - } + // 处理工具调用 + let hasToolCalls = await self.processToolCalls(sessionId) + if !hasToolCalls { + self.getSessionCallback(sessionId)?.onComplete(sessionId) + self.clearCurrentSession() } - - } catch { - if sessionId == self.currSessionId { - let chatApiError = ChatApiException("流式请求失败: \(error.localizedDescription)") - getSessionCallback(sessionId)?.onError(sessionId,chatApiError) + } + + } catch { + if sessionId == self.currSessionId { + let errorMessage = "流式请求失败: \(error.localizedDescription)" + let chatApiError = ChatApiException(errorMessage) + self.getSessionCallback(sessionId)?.onError(sessionId, chatApiError) + self.clearCurrentSession() + } + } + } +} + +// MARK: - 新增辅助方法 +private func estimatePromptTokens(messages: [[String: Any]]) -> Int { + var totalText = "" + for message in messages { + if let content = message["content"] as? String { + totalText += content + " " + } else if let contentArray = message["content"] as? [[String: Any]] { + for item in contentArray { + if let type = item["type"] as? String, + type == "text", + let text = item["text"] as? String { + totalText += text + " " } } } } + return roughTokenCount(text: totalText) +} + +private func roughTokenCount(text: String) -> Int { + let separators = CharacterSet.whitespacesAndNewlines.union(.punctuationCharacters) + let tokens = text.components(separatedBy: separators).filter { !$0.isEmpty } + return tokens.count +} /// 处理工具调用 - private func processToolCalls(_ sessionId:String) async -> Bool { - guard let firstToolCall = toolCalls.values.first, firstToolCall.isValid() else { - return false - } - - // 创建函数调用字典 - let functionCall: [String: Any] = [ - "name": firstToolCall.name, - "arguments": firstToolCall.arguments, - "id": firstToolCall.id - ] - - // 通知上层工具调用事件 - getSessionCallback(sessionId)?.onFunctionCall(sessionId,functionCall) - - // 在后台队列处理工具调用 - Task { - do { - if sessionId == self.currSessionId { - // 调用工具 - let result: [String: Any] - let args = try self.parseJsonArguments(firstToolCall.arguments) - if let client = self.mcpClient { - result = await client.callTool(name: firstToolCall.name, arguments: args) ?? ["context": "工具调用失败"] - } else { - result = ["context": "MCP客户端未初始化"] - } - - if sessionId == self.currSessionId { - // 处理结果 - getSessionCallback(sessionId)?.onFunctionCallResult(sessionId,functionCall, result) + /// 处理工具调用 +private func processToolCalls(_ sessionId:String) async -> Bool { + // 验证工具调用集合不为空 + if toolCalls.isEmpty { + print("[Session: \(sessionId)] 没有工具调用需要处理") + return false + } + + // 输出所有收集到的工具调用信息（用于调试） + print("[Session: \(sessionId)] 收集到 \(toolCalls.count) 个工具调用") + for (index, toolCall) in toolCalls { + print("[Session: \(sessionId)] 工具调用[\(index)]: name=\(toolCall.name), args.length=\(toolCall.arguments.count), args=\(toolCall.arguments.prefix(100))...") + } + + // 只处理有效的工具调用 + let validToolCalls = toolCalls.values.filter { $0.isValid() } + if validToolCalls.isEmpty { + print("[Session: \(sessionId)] 没有有效的工具调用（参数可能不完整）") + return false + } + + // 处理第一个有效的工具调用 + let firstToolCall = validToolCalls.first! + print("[Session: \(sessionId)] 处理工具调用: \(firstToolCall.name)，参数: \(firstToolCall.arguments)") + + // 创建函数调用字典 + let functionCall: [String: Any] = [ + "name": firstToolCall.name, + "arguments": firstToolCall.arguments, + "id": firstToolCall.id + ] + // 转换为JSON对象格式 +let jsonFunctionCall = convertMapToJsonObject(functionCall) +getSessionCallback(sessionId)?.onFunctionCall(sessionId, jsonFunctionCall) + // 通知上层工具调用事件 + // getSessionCallback(sessionId)?.onFunctionCall(sessionId, functionCall) + + // 在后台处理工具调用 + Task { [weak self] in + guard let self = self else { return } + + do { + if sessionId == self.currSessionId { + // 解析参数 + let args = try self.parseJsonArguments(firstToolCall.arguments) + + // 通过MCP客户端处理工具调用 + var result: [String: Any] = [:] + + if let client = self.mcpClient { + // 调用MCP工具 + if let toolResult = await client.callTool(name: firstToolCall.name, arguments: args) { + print("[Session: \(sessionId)] AI调用工具结果: \(firstToolCall.name), 参数: \(args), 结果: \(toolResult)") - // 将result转换为JSON字符串 - let context: String - do { - let jsonData = try JSONSerialization.data(withJSONObject: result, options: .prettyPrinted) - context = String(data: jsonData, encoding: .utf8) ?? "工具调用失败" - } catch { - context = "工具调用失败" + // 统一结果格式 + if toolResult["isError"] as? Bool == true { + if let content = toolResult["content"] as? [[String: Any]], + let firstContent = content.first, + let errorText = firstContent["text"] as? String { + result = ["context": errorText] + } else { + result = ["context": "Tool execution failed"] + } + } else if let context = toolResult["context"] { + // 本地函数结果 + result = ["context": context] + } else if let content = toolResult["content"] as? [[String: Any]], + let firstContent = content.first, + let text = firstContent["text"] as? String { + // MCP工具结果 + result = ["context": text] + } else { + print("[Session: \(sessionId)] MCP工具调用返回无法解析的结果") + result = ["context": "Tool call failed"] } - - // 将结果发送回OpenAI继续对话 - await self.sendFunctionCallResultInternal( - sessionId:sessionId, - messages: self.currentMessages, - functionCall: functionCall, - functionResult: context - ) + } else { + print("[Session: \(sessionId)] MCP工具调用返回nil") + result = ["context": "Tool call failed"] } + } else { + // 工具不存在 + result = ["context": "Tool not found: \(firstToolCall.name)"] } - } catch { + if sessionId == self.currSessionId { - let errorMessage = "工具调用处理失败: \(error.localizedDescription)" + // 处理结果 + self.getSessionCallback(sessionId)?.onFunctionCallResult( + sessionId, + functionCall, + result + ) + + // 将结果发送回OpenAI继续对话 await self.sendFunctionCallResultInternal( - sessionId:sessionId, + sessionId: sessionId, messages: self.currentMessages, functionCall: functionCall, - functionResult: errorMessage + functionResult: self.jsonToString(result) ?? "{}" ) } } + } catch { + print("[Session: \(sessionId)] 工具调用处理过程中出错: \(error.localizedDescription)") + if sessionId == self.currSessionId { + let errorMessage = "工具调用处理失败: \(error.localizedDescription)" + await self.sendFunctionCallResultInternal( + sessionId: sessionId, + messages: self.currentMessages, + functionCall: functionCall, + functionResult: errorMessage + ) + } } - - return true } + return true +} + +/// 辅助方法：将字典转换为JSON字符串 +private func jsonToString(_ json: [String: Any]) -> String? { + do { + let jsonData = try JSONSerialization.data(withJSONObject: json, options: []) + return String(data: jsonData, encoding: .utf8) + } catch { + print("JSON序列化失败: \(error)") + return nil + } +} + // MARK: - JSON转换工具 +private func convertMapToJsonObject(_ map: [String: Any]) -> [String: Any] { + var result = [String: Any]() + + for (key, value) in map { + if let nestedMap = value as? [String: Any] { + result[key] = convertMapToJsonObject(nestedMap) + } else if let array = value as? [Any] { + var jsonArray = [Any]() + for item in array { + if let itemMap = item as? [String: Any] { + jsonArray.append(convertMapToJsonObject(itemMap)) + } else { + jsonArray.append(item) + } + } + result[key] = jsonArray + } else { + result[key] = value + } + } + + return result +} /// 发送函数调用结果（内部方法） private func sendFunctionCallResultInternal( sessionId:String,