From b91a28753d515f6fc3536fcef69cc13ab509220c Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Sat, 26 Jul 2025 15:56:20 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=8A=E4=BC=A0iOS=20=E7=9A=84=E5=9B=BE?= =?UTF-8?q?=E7=89=87=E8=AF=86=E5=88=AB=E9=80=BB=E8=BE=91=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agent_service/AgentServiceImpl.swift | 112 +++++++++--------- .../agent_service/AgentServicePlugin.swift | 3 +- .../Sources/chat_api/ChatApiService.swift | 11 +- 3 files changed, 65 insertions(+), 61 deletions(-) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index ce456f1e3..d1f5c4c1e 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -154,7 +154,9 @@ class AgentServiceImpl: NSObject { if let openaiModel = config["openaiModel"] as? String { self.model = openaiModel } - + if let openaivisionModel = config["openaivisionModel"] as? String { + self.visionModel = openaivisionModel + } if let customSystemPrompt = config["systemPrompt"] as? String, !customSystemPrompt.isEmpty { self.systemPrompt = customSystemPrompt } else { @@ -501,7 +503,7 @@ audioStream.saveAudioDataTo(data: audioData) os_log("sendMessageStream调用完成", log: logger, type: .info) } - func processImageInput(imagePath: String, text: String, speakResponse: Bool) -> Bool { + func processImageInput(imagePath: String,imageUrl: String, text: String, speakResponse: Bool) -> Bool { if !isInitialized { sendError("服务未初始化", code: "NOT_INITIALIZED") return false @@ -525,15 +527,15 @@ audioStream.saveAudioDataTo(data: audioData) DispatchQueue.global(qos: .userInitiated).async { [weak self] in guard let self = self else { return } - guard let imageBase64 = chatApiService.fileToBase64(filePath: imagePath) else { - DispatchQueue.main.async { - self.sendEvent(name: "error", data: [ - "code": "IMAGE_CONVERSION_FAILED", - "message": "图片转换失败" - ]) - } - return - } +// guard let imageBase64 = chatApiService.fileToBase64(filePath: imagePath) else { +// DispatchQueue.main.async { +// self.sendEvent(name: "error", data: [ +// "code": "IMAGE_CONVERSION_FAILED", +// "message": "图片转换失败" +// ]) +// } +// return +// } DispatchQueue.main.async { self.sendEvent(name: "image_ready", data: [ @@ -541,7 +543,7 @@ audioStream.saveAudioDataTo(data: audioData) "imagePath": imagePath ]) - let userMessage = chatApiService.createUserMessageWithImage(text: text, imageBase64: imageBase64) + let userMessage = chatApiService.createUserMessageWithImage(text: text, imageUrl: imageUrl) let displayText = text.isEmpty ? "[图片]" : text self.processWithChatApiServiceInternal( @@ -747,38 +749,16 @@ private func jsonToString(_ json: [String: Any]) -> String? { } } -// 处理音乐卡片 -private func processMusicCard(_ cardMusic: [String: Any]) { - let id = cardMusic["id"] as? Int ?? 0 - let url = cardMusic["url"] as? String ?? "" - let title = cardMusic["name"] as? String ?? "" - let artist = cardMusic["sgener"] as? String ?? "" - let coverUrl = cardMusic["image"] as? String ?? "" - let ts = cardMusic["ts"] as? Int ?? 0 - - processMusicPlay([ - "id": id, - "url": url, - "title": title, - "artist": artist, - "coverUrl": coverUrl, - "ts": ts - ]) -} - -// 处理音乐列表卡片 -private func processMusicListCard(_ musics: [[String: Any]]) { - var playlist: [[String: Any]] = [] - - for item in musics { - let id = item["id"] as? Int ?? 0 - let url = item["url"] as? String ?? "" - let title = item["name"] as? String ?? "" - let artist = item["sgener"] as? String ?? "" - let coverUrl = item["image"] as? String ?? "" - let ts = item["ts"] as? Int ?? 0 - - playlist.append([ + // 处理音乐卡片 + private func processMusicCard(_ cardMusic: [String: Any]) { + let id = cardMusic["id"] as? Int ?? 0 + let url = cardMusic["url"] as? String ?? "" + let title = cardMusic["name"] as? String ?? "" + let artist = cardMusic["sgener"] as? String ?? "" + let coverUrl = cardMusic["image"] as? String ?? "" + let ts = cardMusic["ts"] as? Int ?? 0 + + processMusicPlay([ "id": id, "url": url, "title": title, @@ -787,19 +767,41 @@ private func processMusicListCard(_ musics: [[String: Any]]) { "ts": ts ]) } - - if !playlist.isEmpty { - processMusicPlayList(playlist) + + // 处理音乐列表卡片 + private func processMusicListCard(_ musics: [[String: Any]]) { + var playlist: [[String: Any]] = [] + + for item in musics { + let id = item["id"] as? Int ?? 0 + let url = item["url"] as? String ?? "" + let title = item["name"] as? String ?? "" + let artist = item["sgener"] as? String ?? "" + let coverUrl = item["image"] as? String ?? "" + let ts = item["ts"] as? Int ?? 0 + + playlist.append([ + "id": id, + "url": url, + "title": title, + "artist": artist, + "coverUrl": coverUrl, + "ts": ts + ]) + } + + if !playlist.isEmpty { + processMusicPlayList(playlist) + } } -} -// 处理导航卡片 -private func processNavigationCard(_ cardNavigation: [String: Any]) { - let start = cardNavigation["start"] as? String ?? "" - let end = cardNavigation["end"] as? String ?? "" - // 实际处理导航逻辑 - print("启动导航: 从 \(start) 到 \(end)") -} + // 处理导航卡片 + private func processNavigationCard(_ cardNavigation: [String: Any]) { + let start = cardNavigation["start"] as? String ?? "" + let end = cardNavigation["end"] as? String ?? "" + // 实际处理导航逻辑 + print("启动导航: 从 \(start) 到 \(end)") + } // 后台播放单曲 private func processMusicPlay(_ song: [String: Any]) { sendEvent(name: "music_play", data: song) diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift index 1e6be6288..1c8a6062c 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift @@ -65,6 +65,7 @@ public class AgentServicePlugin: NSObject, FlutterPlugin { case "processImageInput": guard let arguments = call.arguments as? [String: Any], + let imageUrl = arguments["imageUrl"] as? String, let imagePath = arguments["imagePath"] as? String else { result(FlutterError(code: "INVALID_ARGUMENTS", message: "图片路径不能为空", details: nil)) return @@ -72,7 +73,7 @@ public class AgentServicePlugin: NSObject, FlutterPlugin { let text = arguments["text"] as? String ?? "" let speakResponse = arguments["speakResponse"] as? Bool ?? false - let success = impl.processImageInput(imagePath: imagePath, text: text, speakResponse: speakResponse) + let success = impl.processImageInput(imagePath: imagePath,imageUrl: imageUrl, text: text, speakResponse: speakResponse) result(success) case "speakText": diff --git a/local_plugins/chat_api/ios/chat_api/Sources/chat_api/ChatApiService.swift b/local_plugins/chat_api/ios/chat_api/Sources/chat_api/ChatApiService.swift index cdf5cfa3a..689117876 100644 --- a/local_plugins/chat_api/ios/chat_api/Sources/chat_api/ChatApiService.swift +++ b/local_plugins/chat_api/ios/chat_api/Sources/chat_api/ChatApiService.swift @@ -169,7 +169,7 @@ public class ChatApiService: NSObject { } /// 创建带图片的用户消息 - public func createUserMessageWithImage(text: String, imageBase64: String) -> [String: Any] { + public func createUserMessageWithImage(text: String, imageUrl: String) -> [String: Any] { var contentArray: [[String: Any]] = [] // 添加文本部分 @@ -184,7 +184,8 @@ public class ChatApiService: NSObject { contentArray.append([ "type": "image_url", "image_url": [ - "url": "data:image/jpeg;base64,\(imageBase64)" +// "url": "data:image/jpeg;base64,\(imageBase64)" + "url": imageUrl ] ]) @@ -202,9 +203,9 @@ public class ChatApiService: NSObject { // 检查是否包含图片，决定使用哪个模型 var currentModel = model - // if containsImage(in: messages) { - // currentModel = visionModel - // } + if containsImage(in: messages) { + currentModel = visionModel + } // 转换消息格式 let chatMessages = try convertToChatMessages(messages)