wolfplus 1 year ago
parent
commit
407c224c9e
  1. 163
      lib/modules/agent/controllers/agent_controller.dart
  2. 388
      lib/modules/agent/views/agent_view.dart
  3. 49
      lib/modules/agent/views/message_bubble.dart
  4. 184
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  5. 42
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt
  6. 29
      local_plugins/agent_service/lib/agent_service.dart
  7. 112
      local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt

163
lib/modules/agent/controllers/agent_controller.dart

@ -1,22 +1,28 @@
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:flutter/material.dart';
import 'package:get/get.dart';
import 'package:agent_service/agent_service.dart';
import 'package:logger/logger.dart';
import 'package:chat_storage/chat_storage.dart';
import 'package:image_picker/image_picker.dart';
class Message {
final bool isUser;
final String text;
final DateTime timestamp;
final bool isRecognizing; // 是否为语音识别中的临时消息
final bool hasImage; // 是否包含图片
final String? imagePath; // 图片路径
Message({
required this.isUser,
required this.text,
DateTime? timestamp,
this.isRecognizing = false,
this.hasImage = false,
this.imagePath,
}) : this.timestamp = timestamp ?? DateTime.now();
}
@ -38,12 +44,19 @@ class AgentController extends GetxController {
final isSpeaking = false.obs;
final isProcessing = false.obs;
// 图片处理状态
final isImageProcessing = false.obs;
// 输入模式控制
final isTextInputMode = true.obs;
// 当前输入的文本内容
final currentText = ''.obs;
// 图片输入相关
final isImageInputActive = false.obs; // 是否处于图片输入模式
final selectedImagePath = Rx<String?>(null); // 当前选择的图片路径
// 流事件订阅
StreamSubscription<AgentServiceEvent>? _eventSubscription;
@ -217,6 +230,14 @@ class AgentController extends GetxController {
case AgentServiceEventType.ttsCanceled:
isSpeaking.value = false;
break;
case AgentServiceEventType.imageProcessing:
isImageProcessing.value = true;
break;
case AgentServiceEventType.imageReady:
// 图片准备就绪,但AI还没有开始处理,保持processing状态
break;
case AgentServiceEventType.assistantToken:
if (!isProcessing.value) isProcessing.value = true;
@ -249,6 +270,7 @@ class AgentController extends GetxController {
case AgentServiceEventType.assistantResponse:
isProcessing.value = false;
isImageProcessing.value = false;
// 标记当前回复完成
_isNewAssistantResponse = true;
break;
@ -257,6 +279,7 @@ class AgentController extends GetxController {
isListening.value = false;
isSpeaking.value = false;
isProcessing.value = false;
isImageProcessing.value = false;
// 移除临时的识别消息
final index = messages.lastIndexWhere((msg) => msg.isRecognizing && msg.isUser);
@ -331,6 +354,17 @@ class AgentController extends GetxController {
// 发送文本消息
Future<void> sendTextMessage() async {
// 如果图片输入是活跃的,使用选中的图片和文本框的内容
if (isImageInputActive.value && selectedImagePath.value != null) {
final text = textController.text.trim();
final imagePath = selectedImagePath.value!; // 使用!强制断言非空
await sendImageMessage(imagePath, text: text);
// 清除图片输入状态
clearImageInput();
return;
}
final text = textController.text.trim();
if (text.isEmpty) return;
@ -352,6 +386,50 @@ class AgentController extends GetxController {
}
}
// 发送图片消息
Future<void> sendImageMessage(String imagePath, {String? text}) async {
if (imagePath.isEmpty || !File(imagePath).existsSync()) {
logger.e('图片不存在: $imagePath');
return;
}
// 显示的文本,如果没有提供则使用默认值
final displayText = text?.isNotEmpty == true ? text! : '[图片]';
// 添加用户消息
final message = Message(
isUser: true,
text: displayText,
hasImage: true,
imagePath: imagePath,
);
messages.add(message);
// 清空文本输入
textController.clear();
// 滚动到底部
_scrollToBottom();
try {
isProcessing.value = true;
isImageProcessing.value = true;
// 标记为新的AI回复
_isNewAssistantResponse = true;
// 调用Agent Service处理图片
await AgentService.processImageInput(
imagePath,
text: text ?? '',
speakResponse: true,
);
} catch (e) {
logger.e('处理图片失败: $e');
isProcessing.value = false;
isImageProcessing.value = false;
}
}
// 开始语音输入
Future<void> startVoiceInput() async {
if (isListening.value) return;
@ -395,6 +473,12 @@ class AgentController extends GetxController {
// 切换输入模式
void toggleInputMode() {
// 如果当前在图片输入模式,先清除
if (isImageInputActive.value) {
clearImageInput();
return;
}
isTextInputMode.toggle();
// 切换到语音模式时,直接开始语音输入
@ -406,4 +490,83 @@ class AgentController extends GetxController {
stopVoiceInput();
}
}
// 从相册选择图片
Future<void> pickImage() async {
try {
final ImagePicker picker = ImagePicker();
final XFile? image = await picker.pickImage(
source: ImageSource.gallery,
imageQuality: 80, // 设置图片质量
maxWidth: 1024, // 限制最大宽度
maxHeight: 1024, // 限制最大高度
);
if (image != null) {
// 设置已选择的图片
selectedImagePath.value = image.path;
// 激活图片输入模式
isImageInputActive.value = true;
}
} catch (e) {
logger.e('选择图片失败: $e');
}
}
// 拍照获取图片
Future<void> takePhoto() async {
try {
final ImagePicker picker = ImagePicker();
final XFile? photo = await picker.pickImage(
source: ImageSource.camera,
imageQuality: 80, // 设置图片质量
maxWidth: 1024, // 限制最大宽度
maxHeight: 1024, // 限制最大高度
);
if (photo != null) {
// 设置已选择的图片
selectedImagePath.value = photo.path;
// 激活图片输入模式
isImageInputActive.value = true;
}
} catch (e) {
logger.e('拍照失败: $e');
}
}
// 清除图片输入状态
void clearImageInput() {
selectedImagePath.value = null;
isImageInputActive.value = false;
}
// 使用预设提示词处理图片
void useImagePrompt(String prompt) {
// 确保selectedImagePath.value不为null
if (selectedImagePath.value == null) return;
// 填充文本框同时直接发送消息
textController.text = prompt;
// 立即发送图片和提示词
final imagePath = selectedImagePath.value!;
sendImageMessage(imagePath, text: prompt);
// 发送后清除图片输入状态
clearImageInput();
}
// 预设提示词:总结图片内容
void summarizeImage() {
useImagePrompt('请总结这张图片的内容');
}
// 预设提示词:抽取图像文字
void extractTextFromImage() {
useImagePrompt('请提取这张图片中的所有文字');
}
// 预设提示词:翻译图像文字
void translateImageText() {
useImagePrompt('请翻译这张图片中的文字');
}
}

388
lib/modules/agent/views/agent_view.dart

@ -1,3 +1,4 @@
import 'dart:io';
import 'package:flutter/material.dart';
import 'package:get/get.dart';
import '../controllers/agent_controller.dart';
@ -90,6 +91,8 @@ class AgentView extends StatelessWidget {
message: message.text,
timestamp: message.timestamp,
isRecognizing: message.isRecognizing,
hasImage: message.hasImage,
imagePath: message.imagePath,
);
},
),
@ -109,145 +112,265 @@ class AgentView extends StatelessWidget {
),
],
),
child: Row(
child: Column(
mainAxisSize: MainAxisSize.min,
children: [
// 键盘/语音切换按钮
// 图片预览区域 - 仅在图片输入模式下显示
Obx(() {
final isTextMode = controller.isTextInputMode.value;
return GestureDetector(
onTap: () {
controller.toggleInputMode();
},
child: Container(
width: 48,
height: 48,
decoration: BoxDecoration(
color: Colors.grey[200],
borderRadius: BorderRadius.circular(24),
),
child: Icon(
isTextMode ? Icons.mic_none : Icons.keyboard,
color: Colors.grey[600],
),
),
);
}),
const SizedBox(width: 8),
// 文本输入框或语音波形图
Expanded(
child: Obx(() {
final isTextMode = controller.isTextInputMode.value;
if (isTextMode) {
// 文本输入模式
return Container(
padding: const EdgeInsets.symmetric(horizontal: 16),
decoration: BoxDecoration(
color: Colors.grey[200],
borderRadius: BorderRadius.circular(24),
),
child: Row(
if (controller.isImageInputActive.value && controller.selectedImagePath.value != null) {
return Column(
mainAxisSize: MainAxisSize.min,
children: [
// 图片预览和关闭按钮
Stack(
children: [
Expanded(
child: TextField(
controller: controller.textController,
decoration: const InputDecoration(
hintText: '输入消息...',
border: InputBorder.none,
),
onSubmitted: (_) => controller.sendTextMessage(),
ClipRRect(
borderRadius: BorderRadius.circular(12),
child: Image.file(
File(controller.selectedImagePath.value!),
height: 150,
width: double.infinity,
fit: BoxFit.cover,
),
),
GestureDetector(
onTap: () => controller.sendTextMessage(),
child: Icon(
Icons.send,
color: Theme.of(context).primaryColor,
Positioned(
top: 8,
right: 8,
child: GestureDetector(
onTap: () => controller.clearImageInput(),
child: Container(
padding: EdgeInsets.all(4),
decoration: BoxDecoration(
color: Colors.black.withOpacity(0.6),
shape: BoxShape.circle,
),
child: Icon(
Icons.close,
color: Colors.white,
size: 16,
),
),
),
),
],
),
);
} else {
// 语音输入模式 - 始终显示波形图
// 预设提示选项
SizedBox(height: 8),
SingleChildScrollView(
scrollDirection: Axis.horizontal,
child: Row(
children: [
_buildPromptChip(
context: context,
label: '总结图片内容',
onTap: () => controller.summarizeImage(),
),
SizedBox(width: 8),
_buildPromptChip(
context: context,
label: '抽取图像文字',
onTap: () => controller.extractTextFromImage(),
),
SizedBox(width: 8),
_buildPromptChip(
context: context,
label: '翻译图像文字',
onTap: () => controller.translateImageText(),
),
],
),
),
SizedBox(height: 12),
],
);
} else {
return SizedBox.shrink();
}
}),
// 输入控件行
Row(
children: [
// 键盘/语音切换按钮
Obx(() {
final isTextMode = controller.isTextInputMode.value;
return GestureDetector(
onTap: () {
if (controller.isListening.value) {
controller.stopVoiceInput();
} else {
controller.startVoiceInput();
}
controller.toggleInputMode();
},
child: Container(
width: 48,
height: 48,
padding: const EdgeInsets.symmetric(horizontal: 16),
decoration: BoxDecoration(
color: Colors.grey[200],
borderRadius: BorderRadius.circular(24),
),
child: Row(
children: [
// 语音波形图 - 始终显示
Expanded(
child: Row(
mainAxisAlignment: MainAxisAlignment.start,
children: [
...List.generate(10, (index) {
return _buildSoundBar(index);
}),
const Spacer(),
// "正在聆听"文字
Obx(() {
final isListening = controller.isListening.value;
return Row(
children: [
Icon(
Icons.mic,
size: 14,
color: isListening
? Theme.of(context).primaryColor
: Colors.grey,
),
const SizedBox(width: 4),
Text(
'正在聆听',
style: TextStyle(
fontSize: 12,
color: isListening
? Theme.of(context).primaryColor
: Colors.grey,
),
),
],
);
}),
],
),
),
],
child: Icon(
isTextMode ? Icons.mic_none : Icons.keyboard,
color: Colors.grey[600],
),
),
);
}
}),
),
const SizedBox(width: 8),
// 只显示添加按钮,不再显示停止按钮
Container(
width: 48,
height: 48,
decoration: BoxDecoration(
color: Colors.grey[200],
borderRadius: BorderRadius.circular(24),
),
child: Icon(
Icons.add,
color: Colors.grey[600],
),
}),
const SizedBox(width: 8),
// 文本输入框或语音波形图
Expanded(
child: Obx(() {
final isTextMode = controller.isTextInputMode.value;
if (isTextMode) {
// 文本输入模式 - 注意观察图片模式时的高度变化
return Container(
padding: const EdgeInsets.symmetric(horizontal: 16),
decoration: BoxDecoration(
color: Colors.grey[200],
borderRadius: BorderRadius.circular(24),
),
child: Row(
children: [
Expanded(
child: Obx(() {
// 图片模式下使用更高的输入框
final isImageActive = controller.isImageInputActive.value;
return TextField(
controller: controller.textController,
maxLines: isImageActive ? 2 : 1,
minLines: isImageActive ? 2 : 1,
decoration: InputDecoration(
hintText: isImageActive ? '添加对图片的问题或描述...' : '输入消息...',
border: InputBorder.none,
contentPadding: EdgeInsets.symmetric(vertical: isImageActive ? 12 : 0),
),
onSubmitted: (_) => controller.sendTextMessage(),
);
}),
),
GestureDetector(
onTap: () => controller.sendTextMessage(),
child: Icon(
Icons.send,
color: Theme.of(context).primaryColor,
),
),
],
),
);
} else {
// 语音输入模式 - 始终显示波形图
return GestureDetector(
onTap: () {
if (controller.isListening.value) {
controller.stopVoiceInput();
} else {
controller.startVoiceInput();
}
},
child: Container(
height: 48,
padding: const EdgeInsets.symmetric(horizontal: 16),
decoration: BoxDecoration(
color: Colors.grey[200],
borderRadius: BorderRadius.circular(24),
),
child: Row(
children: [
// 语音波形图 - 始终显示
Expanded(
child: Row(
mainAxisAlignment: MainAxisAlignment.start,
children: [
...List.generate(10, (index) {
return _buildSoundBar(index);
}),
const Spacer(),
// "正在聆听"文字
Obx(() {
final isListening = controller.isListening.value;
return Row(
children: [
Icon(
Icons.mic,
size: 14,
color: isListening
? Theme.of(context).primaryColor
: Colors.grey,
),
const SizedBox(width: 4),
Text(
'正在聆听',
style: TextStyle(
fontSize: 12,
color: isListening
? Theme.of(context).primaryColor
: Colors.grey,
),
),
],
);
}),
],
),
),
],
),
),
);
}
}),
),
const SizedBox(width: 8),
// 图片选择按钮
Container(
width: 48,
height: 48,
decoration: BoxDecoration(
color: Colors.grey[200],
borderRadius: BorderRadius.circular(24),
),
child: PopupMenuButton<String>(
icon: Icon(
Icons.add_photo_alternate_outlined,
color: Colors.grey[600],
),
padding: EdgeInsets.zero,
onSelected: (value) async {
if (value == 'camera') {
await controller.takePhoto();
} else if (value == 'gallery') {
await controller.pickImage();
}
},
itemBuilder: (context) => [
PopupMenuItem<String>(
value: 'camera',
child: Row(
children: [
Icon(Icons.camera_alt, color: Theme.of(context).primaryColor),
SizedBox(width: 8),
Text('拍照'),
],
),
),
PopupMenuItem<String>(
value: 'gallery',
child: Row(
children: [
Icon(Icons.photo_library, color: Theme.of(context).primaryColor),
SizedBox(width: 8),
Text('从相册选择'),
],
),
),
],
),
),
],
),
],
),
@ -257,6 +380,35 @@ class AgentView extends StatelessWidget {
);
}
// 构建提示选项芯片
Widget _buildPromptChip({
required BuildContext context,
required String label,
required VoidCallback onTap,
}) {
return GestureDetector(
onTap: onTap,
child: Container(
padding: EdgeInsets.symmetric(horizontal: 12, vertical: 6),
decoration: BoxDecoration(
color: Theme.of(context).primaryColor.withOpacity(0.1),
borderRadius: BorderRadius.circular(16),
border: Border.all(
color: Theme.of(context).primaryColor.withOpacity(0.3),
),
),
child: Text(
label,
style: TextStyle(
fontSize: 13,
color: Theme.of(context).primaryColor,
fontWeight: FontWeight.w500,
),
),
),
);
}
// 构建声音柱
Widget _buildSoundBar(int index) {
return Padding(

49
lib/modules/agent/views/message_bubble.dart

@ -1,5 +1,6 @@
import 'package:flutter/material.dart';
import 'package:intl/intl.dart';
import 'dart:io';
/// 消息气泡组件
class MessageBubble extends StatelessWidget {
@ -7,6 +8,8 @@ class MessageBubble extends StatelessWidget {
final String message;
final DateTime timestamp;
final bool isRecognizing;
final bool hasImage;
final String? imagePath;
const MessageBubble({
Key? key,
@ -14,6 +17,8 @@ class MessageBubble extends StatelessWidget {
required this.message,
required this.timestamp,
this.isRecognizing = false,
this.hasImage = false,
this.imagePath,
}) : super(key: key);
@override
@ -33,7 +38,7 @@ class MessageBubble extends StatelessWidget {
crossAxisAlignment: isUser ? CrossAxisAlignment.end : CrossAxisAlignment.start,
children: [
Container(
padding: const EdgeInsets.all(12),
padding: hasImage ? (message == '[图片]' ? EdgeInsets.zero : const EdgeInsets.all(12)) : const EdgeInsets.all(12),
decoration: BoxDecoration(
color: _getBubbleColor(context),
borderRadius: BorderRadius.circular(18).copyWith(
@ -131,6 +136,48 @@ class MessageBubble extends StatelessWidget {
);
}
// 如果是图片消息,显示图片
if (hasImage && imagePath != null) {
// 如果消息是默认的[图片]标记,只显示图片
if (message == '[图片]') {
return ClipRRect(
borderRadius: BorderRadius.circular(16),
child: Image.file(
File(imagePath!),
width: 200,
height: 200,
fit: BoxFit.cover,
),
);
}
// 如果消息是自定义文本,显示文本+图片
return Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text(
message,
style: TextStyle(
color: isUser ? Colors.white : Colors.black87,
fontSize: 14,
height: 1.4,
fontWeight: FontWeight.w400,
),
),
const SizedBox(height: 8),
ClipRRect(
borderRadius: BorderRadius.circular(12),
child: Image.file(
File(imagePath!),
width: 200,
height: 200,
fit: BoxFit.cover,
),
),
],
);
}
// 判断消息内容,显示特殊卡片
if (!isUser) {
if (message.contains('播放列表') || message.contains('音乐')) {

184
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt

@ -99,7 +99,7 @@ object AgentService : CoroutineScope {
4. 避免过长的列表,尽量将信息分成小段
5. 不要使用需要视觉展示的元素(如表格、图表或代码块)
6. 记住用户之前的对话内容,保持对话连贯
7. 如果用户发送了图片,请根据图片内容和文字要求回答问题
你不仅可以回答知识性问题,还可以帮助用户设置提醒、提供建议,或进行轻松愉快的对话。
无论遇到什么问题,都要尽力以温暖、贴心的语气提供最佳帮助。
""".trimIndent()
@ -467,12 +467,6 @@ object AgentService : CoroutineScope {
return false
}
// 发送与语音识别结果相同格式的事件
sendEvent("recognition_result", mapOf(
"text" to text,
"language" to "zh-CN", // 默认语言,未来可以从配置或检测中获取
"inputType" to "text"
))
// 使用OpenAI处理文本
processWithOpenAI(text, speakResponse)
@ -499,6 +493,46 @@ object AgentService : CoroutineScope {
) {
FileLogger.d(TAG, "用户问题: $text")
// 创建用户文本消息并处理
val userMessage = openAIService.createUserMessage(text)
processWithOpenAIInternal(userMessage, text, speakResponse)
}
/**
* 使用OpenAI处理图片
*
* @param imageBase64 Base64编码的图片数据
* @param text 可选的文本描述或问题
* @param speakResponse 是否朗读回复
*/
private fun processImageWithOpenAI(
imageBase64: String,
text: String = "",
speakResponse: Boolean = false
) {
FileLogger.d(TAG, "处理图片输入: ${if (text.isEmpty()) "无附加文本" else "附带文本: $text"}")
// 创建带图片的用户消息并处理
val userMessage = openAIService.createUserMessageWithImage(text, imageBase64)
// 图片描述用于存储
val displayText = text.ifEmpty { "[图片]" }
processWithOpenAIInternal(userMessage, displayText, speakResponse, true)
}
/**
* 内部方法:通用的OpenAI处理逻辑
*
* @param userMessage 用户消息(可以是文本或图片格式)
* @param displayText 用于显示和存储的文本
* @param speakResponse 是否朗读回复
* @param hasImage 是否包含图片
*/
private fun processWithOpenAIInternal(
userMessage: JSONObject,
displayText: String,
speakResponse: Boolean = true,
hasImage: Boolean = false
) {
// 如果有正在进行的AI流式输出,先停止它
stopAiStream()
@ -508,12 +542,7 @@ object AgentService : CoroutineScope {
// 设置状态为正在流式输出
isAiStreaming = true
// 创建用户消息
val userMessage = openAIService.createUserMessage(text)
// 将用户消息添加到历史记录
addToHistoryMessages(userMessage)
// 使用历史记录作为上下文发送到OpenAI
val responseBuilder = StringBuilder()
@ -529,6 +558,36 @@ object AgentService : CoroutineScope {
for (i in 0 until historyMessages.length()) {
messagesWithSystemPrompt.put(historyMessages.getJSONObject(i))
}
messagesWithSystemPrompt.put(userMessage)
// 将用户消息添加到历史记录(注意:要去掉图片数据再保存)
if (userMessage.has("content")) {
val content = userMessage.get("content")
// 检查是否是带图片的消息(即content是JSONArray而不是String)
if (content is JSONArray) {
// 提取文本内容
var textContent = ""
for (i in 0 until content.length()) {
val item = content.getJSONObject(i)
if (item.getString("type") == "text") {
textContent = item.getString("text")
break
}
}
// 创建新的只含文本的消息
val textOnlyMessage = JSONObject(userMessage.toString())
textOnlyMessage.put("content", textContent.ifEmpty { "[图片]" })
// 添加到历史记录
addToHistoryMessages(textOnlyMessage)
} else {
// 普通文本消息,直接添加
addToHistoryMessages(userMessage)
}
} else {
// 兜底处理,直接添加原消息
addToHistoryMessages(userMessage)
}
openAIService.sendMessageStream(
messages = messagesWithSystemPrompt,
@ -548,20 +607,23 @@ object AgentService : CoroutineScope {
azureTtsHelper?.flushStream()
}
val response = responseBuilder.toString()
// FileLogger.d(TAG, "AI完整回复: $response, $speakResponse")
if (response.isNotEmpty()) {
// 发送完整回复
sendEvent("assistant_response", mapOf(
// 发送完整回复,包含是否有图片的标记
val responseData = mutableMapOf<String, Any>(
"text" to response,
"userInput" to text
))
"userInput" to displayText
)
if (hasImage) {
responseData["hasImage"] = true
}
sendEvent("assistant_response", responseData)
// 添加AI回复到历史记录
addToHistoryMessages(openAIService.createAssistantMessage(response))
// 保存聊天记录
saveChatMessage(text, response)
saveChatMessage(displayText, response)
}
// 标记AI流式输出已完成
@ -602,9 +664,10 @@ object AgentService : CoroutineScope {
)
} catch (e: Exception) {
val errorType = if (hasImage) "AI_IMAGE_PROCESS_ERROR" else "AI_PROCESS_ERROR"
FileLogger.e(TAG, "AI处理出错", e)
sendEvent("error", mapOf(
"code" to "AI_PROCESS_ERROR",
"code" to errorType,
"message" to e.message.toString()
))
@ -712,11 +775,11 @@ object AgentService : CoroutineScope {
sender = "assistant"
)
if (assistantMessageId != -1L) {
FileLogger.d(TAG, "聊天记录已保存:用户消息ID=$userMessageId, 助手消息ID=$assistantMessageId")
} else {
FileLogger.e(TAG, "保存助手消息失败")
}
// if (assistantMessageId != -1L) {
// FileLogger.d(TAG, "聊天记录已保存:用户消息ID=$userMessageId, 助手消息ID=$assistantMessageId")
// } else {
// FileLogger.e(TAG, "保存助手消息失败")
// }
} else {
FileLogger.e(TAG, "保存用户消息失败")
}
@ -803,4 +866,73 @@ object AgentService : CoroutineScope {
val result = context.checkCallingOrSelfPermission(permission)
return result == android.content.pm.PackageManager.PERMISSION_GRANTED
}
/**
* 处理图片输入
* 将图片与文本一起发送给AI进行处理
*
* @param imagePath 图片文件路径
* @param text 可选的文本描述或问题,默认为空
* @param speakResponse 是否朗读回复,默认为false
* @return 是否成功开始处理
*/
fun processImageInput(imagePath: String, text: String = "", speakResponse: Boolean = false): Boolean {
if (!isInitialized) {
FileLogger.e(TAG, "服务未初始化")
sendEvent("error", mapOf("code" to "NOT_INITIALIZED", "message" to "服务未初始化"))
return false
}
if (imagePath.isEmpty()) {
FileLogger.e(TAG, "图片路径不能为空")
sendEvent("error", mapOf("code" to "EMPTY_IMAGE_PATH", "message" to "图片路径不能为空"))
return false
}
// 通知开始处理图片
sendEvent("image_processing", mapOf(
"status" to "processing",
"imagePath" to imagePath
))
// 使用协程处理耗时的图片转换操作
launch(Dispatchers.IO) {
try {
// 将图片转换为Base64格式
val imageBase64 = openAIService.fileToBase64(imagePath)
if (imageBase64 == null) {
withContext(Dispatchers.Main) {
FileLogger.e(TAG, "图片转换失败: $imagePath")
sendEvent("error", mapOf(
"code" to "IMAGE_CONVERSION_FAILED",
"message" to "图片转换失败"
))
}
return@launch
}
// 通知图片准备完成
withContext(Dispatchers.Main) {
sendEvent("image_ready", mapOf(
"status" to "ready",
"imagePath" to imagePath
))
// 使用OpenAI处理包含图片的消息
processImageWithOpenAI(imageBase64, text, speakResponse)
}
} catch (e: Exception) {
withContext(Dispatchers.Main) {
FileLogger.e(TAG, "处理图片输入失败", e)
sendEvent("error", mapOf(
"code" to "IMAGE_PROCESSING_ERROR",
"message" to e.message.toString()
))
}
}
}
return true
}
}

42
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt

@ -76,6 +76,29 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("TEXT_PROCESS_ERROR", "处理文本输入失败: ${e.message}", null)
}
}
"processImageInput" -> {
val imagePath = call.argument<String>("imagePath")
if (imagePath.isNullOrEmpty()) {
result.error("INVALID_ARGUMENTS", "图片路径不能为空", null)
return
}
val text = call.argument<String>("text") ?: ""
val speakResponse = call.argument<Boolean>("speakResponse") ?: false
try {
val isProcessing = AgentService.processImageInput(
imagePath = imagePath,
text = text,
speakResponse = speakResponse
)
result.success(isProcessing)
} catch (e: Exception) {
FileLogger.e(TAG, "处理图片输入失败", e)
result.error("IMAGE_PROCESS_ERROR", "处理图片输入失败: ${e.message}", null)
}
}
"speakText" -> {
val text = call.argument<String>("text")
if (text.isNullOrEmpty()) {
@ -110,7 +133,24 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("CLEAR_HISTORY_ERROR", "清除聊天历史失败: ${e.message}", null)
}
}
"interruptCurrentResponse" -> {
try {
AgentService.interruptCurrentResponse()
result.success(true)
} catch (e: Exception) {
FileLogger.e(TAG, "中断响应失败", e)
result.error("INTERRUPT_ERROR", "中断响应失败: ${e.message}", null)
}
}
"dispose" -> {
try {
AgentService.dispose()
result.success(true)
} catch (e: Exception) {
FileLogger.e(TAG, "释放资源失败", e)
result.error("DISPOSE_ERROR", "释放资源失败: ${e.message}", null)
}
}
else -> {
result.notImplemented()
}

29
local_plugins/agent_service/lib/agent_service.dart

@ -53,6 +53,12 @@ enum AgentServiceEventType {
/// 函数调用结果
functionCallResult,
/// 图片处理中
imageProcessing,
/// 图片准备就绪
imageReady,
/// 错误
error,
@ -151,6 +157,10 @@ class AgentService {
return AgentServiceEventType.functionCall;
case 'function_call_result':
return AgentServiceEventType.functionCallResult;
case 'image_processing':
return AgentServiceEventType.imageProcessing;
case 'image_ready':
return AgentServiceEventType.imageReady;
default:
return AgentServiceEventType.unknown;
}
@ -273,6 +283,25 @@ class AgentService {
}
}
/// 处理图片输入
///
/// [imagePath] 图片文件路径
/// [text] 可选的文本描述或问题
/// [speakResponse] 是否朗读响应
/// 返回是否成功处理图片
static Future<bool> processImageInput(String imagePath, {String text = "", bool speakResponse = false}) async {
try {
final bool result = await _channel.invokeMethod('processImageInput', {
'imagePath': imagePath,
'text': text,
'speakResponse': speakResponse,
});
return result;
} on PlatformException catch (e) {
throw AgentServiceException(e.code, e.message ?? '处理图片输入失败', e.details);
}
}
/// 释放资源
static Future<bool> dispose() async {
try {

112
local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt

@ -1,6 +1,9 @@
package com.yunqiinnovation.open_ai_service
import android.content.Context
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.util.Base64
import android.util.Log
import com.fasterxml.jackson.core.type.TypeReference
import com.fasterxml.jackson.databind.ObjectMapper
@ -14,6 +17,8 @@ import okhttp3.RequestBody.Companion.toRequestBody
import okio.Buffer
import org.json.JSONArray
import org.json.JSONObject
import java.io.ByteArrayOutputStream
import java.io.File
import java.io.IOException
import java.util.concurrent.TimeUnit
@ -34,6 +39,7 @@ class OpenAIService(private val context: Context? = null) {
private var apiKey: String = ""
private var isInitialized = false
private var model: String = "" // 默认模型
private var visionModel: String = "doubao-1-5-vision-pro-32k-250115" // 默认模型
// MCP客户端
private var mcpClient: MCPClient? = null
@ -75,6 +81,101 @@ class OpenAIService(private val context: Context? = null) {
return command.toString()
}
/**
* 将文件转换为Base64字符串
*/
fun fileToBase64(filePath: String, maxSizeKB: Int = 20480): String? {
try {
val file = File(filePath)
if (!file.exists() || !file.isFile) {
Log.e(TAG, "文件不存在: $filePath")
return null
}
// 读取文件并压缩(如果需要)
val originalBitmap = BitmapFactory.decodeFile(filePath)
if (originalBitmap == null) {
Log.e(TAG, "无法解码图片: $filePath")
return null
}
val outputStream = ByteArrayOutputStream()
var quality = 100
var compressedBitmap = originalBitmap
// 检查图片尺寸,限制最大为1024*1024
val maxDimension = 1024
if (originalBitmap.width > maxDimension || originalBitmap.height > maxDimension) {
Log.d(TAG, "图片尺寸超过限制,进行缩放: ${originalBitmap.width}x${originalBitmap.height} -> ${maxDimension}x${maxDimension}")
// 计算缩放比例,保持纵横比
val widthRatio = maxDimension.toFloat() / originalBitmap.width
val heightRatio = maxDimension.toFloat() / originalBitmap.height
val ratio = Math.min(widthRatio, heightRatio)
val newWidth = (originalBitmap.width * ratio).toInt()
val newHeight = (originalBitmap.height * ratio).toInt()
compressedBitmap = Bitmap.createScaledBitmap(originalBitmap, newWidth, newHeight, true)
Log.d(TAG, "缩放后图片尺寸: ${newWidth}x${newHeight}")
}
// 如果原始图片太大,继续优化文件大小
var fileSize = file.length() / 1024 // 转为KB
if (fileSize > maxSizeKB) {
val scale = Math.sqrt(maxSizeKB.toDouble() / fileSize)
val newWidth = (compressedBitmap.width * scale).toInt()
val newHeight = (compressedBitmap.height * scale).toInt()
compressedBitmap = Bitmap.createScaledBitmap(compressedBitmap, newWidth, newHeight, true)
quality = 85
}
// 压缩图片
compressedBitmap.compress(Bitmap.CompressFormat.JPEG, quality, outputStream)
val imageBytes = outputStream.toByteArray()
// 检查压缩后大小
if (imageBytes.size / 1024 > maxSizeKB) {
Log.w(TAG, "压缩后图片仍然超出大小限制: ${imageBytes.size / 1024}KB > ${maxSizeKB}KB")
}
// 转为Base64
return Base64.encodeToString(imageBytes, Base64.NO_WRAP)
} catch (e: Exception) {
Log.e(TAG, "转换文件到Base64失败: ${e.message}", e)
return null
}
}
/**
* 创建带图片的用户消息
*/
fun createUserMessageWithImage(text: String, imageBase64: String): JSONObject {
// 创建包含文本和图片的内容数组
val contentArray = JSONArray().apply {
// 添加文本部分
if (text.isNotEmpty()) {
put(JSONObject().apply {
put("type", "text")
put("text", text)
})
}
// 添加图片部分
put(JSONObject().apply {
put("type", "image_url")
put("image_url", JSONObject().apply {
put("url", "data:image/jpeg;base64,$imageBase64")
})
})
}
return JSONObject().apply {
put("role", "user")
put("content", contentArray)
}
}
/**
* 创建用户消息
*/
@ -371,8 +472,17 @@ class OpenAIService(private val context: Context? = null) {
// 重置取消状态
isCanceled = false
// 检查最后一条消息内容中是否包含 "image_url",决定使用哪个模型
var currentModel = model
if (messages.length() > 0) {
val lastMessage = messages.getJSONObject(messages.length() - 1)
if (lastMessage.has("content")) {
val content = lastMessage.getString("content")
currentModel = visionModel
}
}
val requestBody = JSONObject().apply {
put("model", model)
put("model", currentModel)
put("messages", messages)
put("temperature", 0.7)
put("max_tokens", 2000)

Loading…
Cancel
Save