liwei1dao
/
deepvoice_client
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

上传iOS 的图片识别逻辑代码

weicu
liwei1dao 1 year ago
parent
c4d8662ce7
commit
b91a28753d
3 changed files with 65 additions and 61 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 112
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift
  2. 3
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift
  3. 11
      local_plugins/chat_api/ios/chat_api/Sources/chat_api/ChatApiService.swift

112
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift
View File

@ -154,7 +154,9 @@ class AgentServiceImpl: NSObject {
if let openaiModel = config["openaiModel"] as? String { if let openaiModel = config["openaiModel"] as? String {
self.model = openaiModel self.model = openaiModel
} }
if let openaivisionModel = config["openaivisionModel"] as? String {
self.visionModel = openaivisionModel
}
if let customSystemPrompt = config["systemPrompt"] as? String, !customSystemPrompt.isEmpty { if let customSystemPrompt = config["systemPrompt"] as? String, !customSystemPrompt.isEmpty {
self.systemPrompt = customSystemPrompt self.systemPrompt = customSystemPrompt
} else { } else {
@ -501,7 +503,7 @@ audioStream.saveAudioDataTo(data: audioData)
os_log("sendMessageStream调用完成", log: logger, type: .info) os_log("sendMessageStream调用完成", log: logger, type: .info)
} }
func processImageInput(imagePath: String, text: String, speakResponse: Bool) -> Bool { func processImageInput(imagePath: String,imageUrl: String, text: String, speakResponse: Bool) -> Bool {
if !isInitialized { if !isInitialized {
sendError("服务未初始化", code: "NOT_INITIALIZED") sendError("服务未初始化", code: "NOT_INITIALIZED")
return false return false
@ -525,15 +527,15 @@ audioStream.saveAudioDataTo(data: audioData)
DispatchQueue.global(qos: .userInitiated).async { [weak self] in DispatchQueue.global(qos: .userInitiated).async { [weak self] in
guard let self = self else { return } guard let self = self else { return }
guard let imageBase64 = chatApiService.fileToBase64(filePath: imagePath) else { // guard let imageBase64 = chatApiService.fileToBase64(filePath: imagePath) else {
DispatchQueue.main.async { // DispatchQueue.main.async {
self.sendEvent(name: "error", data: [ // self.sendEvent(name: "error", data: [
"code": "IMAGE_CONVERSION_FAILED", // "code": "IMAGE_CONVERSION_FAILED",
"message": "图片转换失败" // "message": "图片转换失败"
]) // ])
} // }
return // return
} // }
DispatchQueue.main.async { DispatchQueue.main.async {
self.sendEvent(name: "image_ready", data: [ self.sendEvent(name: "image_ready", data: [
@ -541,7 +543,7 @@ audioStream.saveAudioDataTo(data: audioData)
"imagePath": imagePath "imagePath": imagePath
]) ])
let userMessage = chatApiService.createUserMessageWithImage(text: text, imageBase64: imageBase64) let userMessage = chatApiService.createUserMessageWithImage(text: text, imageUrl: imageUrl)
let displayText = text.isEmpty ? "[图片]" : text let displayText = text.isEmpty ? "[图片]" : text
self.processWithChatApiServiceInternal( self.processWithChatApiServiceInternal(
@ -747,38 +749,16 @@ private func jsonToString(_ json: [String: Any]) -> String? {
} }
} }
// 处理音乐卡片 // 处理音乐卡片
private func processMusicCard(_ cardMusic: [String: Any]) { private func processMusicCard(_ cardMusic: [String: Any]) {
let id = cardMusic["id"] as? Int ?? 0 let id = cardMusic["id"] as? Int ?? 0
let url = cardMusic["url"] as? String ?? "" let url = cardMusic["url"] as? String ?? ""
let title = cardMusic["name"] as? String ?? "" let title = cardMusic["name"] as? String ?? ""
let artist = cardMusic["sgener"] as? String ?? "" let artist = cardMusic["sgener"] as? String ?? ""
let coverUrl = cardMusic["image"] as? String ?? "" let coverUrl = cardMusic["image"] as? String ?? ""
let ts = cardMusic["ts"] as? Int ?? 0 let ts = cardMusic["ts"] as? Int ?? 0
processMusicPlay([ processMusicPlay([
"id": id,
"url": url,
"title": title,
"artist": artist,
"coverUrl": coverUrl,
"ts": ts
])
}
// 处理音乐列表卡片
private func processMusicListCard(_ musics: [[String: Any]]) {
var playlist: [[String: Any]] = []
for item in musics {
let id = item["id"] as? Int ?? 0
let url = item["url"] as? String ?? ""
let title = item["name"] as? String ?? ""
let artist = item["sgener"] as? String ?? ""
let coverUrl = item["image"] as? String ?? ""
let ts = item["ts"] as? Int ?? 0
playlist.append([
"id": id, "id": id,
"url": url, "url": url,
"title": title, "title": title,
@ -787,19 +767,41 @@ private func processMusicListCard(_ musics: [[String: Any]]) {
"ts": ts "ts": ts
]) ])
} }
if !playlist.isEmpty { // 处理音乐列表卡片
processMusicPlayList(playlist) private func processMusicListCard(_ musics: [[String: Any]]) {
var playlist: [[String: Any]] = []
for item in musics {
let id = item["id"] as? Int ?? 0
let url = item["url"] as? String ?? ""
let title = item["name"] as? String ?? ""
let artist = item["sgener"] as? String ?? ""
let coverUrl = item["image"] as? String ?? ""
let ts = item["ts"] as? Int ?? 0
playlist.append([
"id": id,
"url": url,
"title": title,
"artist": artist,
"coverUrl": coverUrl,
"ts": ts
])
}
if !playlist.isEmpty {
processMusicPlayList(playlist)
}
} }
}
// 处理导航卡片 // 处理导航卡片
private func processNavigationCard(_ cardNavigation: [String: Any]) { private func processNavigationCard(_ cardNavigation: [String: Any]) {
let start = cardNavigation["start"] as? String ?? "" let start = cardNavigation["start"] as? String ?? ""
let end = cardNavigation["end"] as? String ?? "" let end = cardNavigation["end"] as? String ?? ""
// 实际处理导航逻辑 // 实际处理导航逻辑
print("启动导航: 从 \(start) 到 \(end)") print("启动导航: 从 \(start) 到 \(end)")
} }
// 后台播放单曲 // 后台播放单曲
private func processMusicPlay(_ song: [String: Any]) { private func processMusicPlay(_ song: [String: Any]) {
sendEvent(name: "music_play", data: song) sendEvent(name: "music_play", data: song)

3
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift
View File

@ -65,6 +65,7 @@ public class AgentServicePlugin: NSObject, FlutterPlugin {
case "processImageInput": case "processImageInput":
guard let arguments = call.arguments as? [String: Any], guard let arguments = call.arguments as? [String: Any],
let imageUrl = arguments["imageUrl"] as? String,
let imagePath = arguments["imagePath"] as? String else { let imagePath = arguments["imagePath"] as? String else {
result(FlutterError(code: "INVALID_ARGUMENTS", message: "图片路径不能为空", details: nil)) result(FlutterError(code: "INVALID_ARGUMENTS", message: "图片路径不能为空", details: nil))
return return
@ -72,7 +73,7 @@ public class AgentServicePlugin: NSObject, FlutterPlugin {
let text = arguments["text"] as? String ?? "" let text = arguments["text"] as? String ?? ""
let speakResponse = arguments["speakResponse"] as? Bool ?? false let speakResponse = arguments["speakResponse"] as? Bool ?? false
let success = impl.processImageInput(imagePath: imagePath, text: text, speakResponse: speakResponse) let success = impl.processImageInput(imagePath: imagePath,imageUrl: imageUrl, text: text, speakResponse: speakResponse)
result(success) result(success)
case "speakText": case "speakText":

11
local_plugins/chat_api/ios/chat_api/Sources/chat_api/ChatApiService.swift
View File

@ -169,7 +169,7 @@ public class ChatApiService: NSObject {
} }
/// 创建带图片的用户消息 /// 创建带图片的用户消息
public func createUserMessageWithImage(text: String, imageBase64: String) -> [String: Any] { public func createUserMessageWithImage(text: String, imageUrl: String) -> [String: Any] {
var contentArray: [[String: Any]] = [] var contentArray: [[String: Any]] = []
// 添加文本部分 // 添加文本部分
@ -184,7 +184,8 @@ public class ChatApiService: NSObject {
contentArray.append([ contentArray.append([
"type": "image_url", "type": "image_url",
"image_url": [ "image_url": [
"url": "data:image/jpeg;base64,\(imageBase64)" // "url": "data:image/jpeg;base64,\(imageBase64)"
"url": imageUrl
] ]
]) ])
@ -202,9 +203,9 @@ public class ChatApiService: NSObject {
// 检查是否包含图片，决定使用哪个模型 // 检查是否包含图片，决定使用哪个模型
var currentModel = model var currentModel = model
// if containsImage(in: messages) { if containsImage(in: messages) {
// currentModel = visionModel currentModel = visionModel
// } }
// 转换消息格式 // 转换消息格式
let chatMessages = try convertToChatMessages(messages) let chatMessages = try convertToChatMessages(messages)

Write Preview
Loading…
Cancel
Save