liwei1dao
/
deepvoice_client
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

feat(agent): 支持图片消息持久化并优化聊天记录加载 

- 在AgentService中新增图片路径参数，用于处理带图片的聊天消息
- 优化聊天记录加载逻辑，支持从元数据中提取图片信息
- 将图片复制到应用永久存储目录，确保图片持久化
- 更新欢迎消息中的助手名称从Emma改为小言
newdev_shunjiawei
lxm 1 year ago
parent
2fd6147954
commit
a89ef59a13
4 changed files with 197 additions and 81 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 111
      lib/modules/agent/controllers/agent_controller.dart
  2. 2
      lib/modules/agent/views/agent_view.dart
  3. 38
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  4. 127
      local_plugins/agent_service/lib/agent_service.dart

111
lib/modules/agent/controllers/agent_controller.dart
View File

@ -11,6 +11,7 @@ import 'package:get_storage/get_storage.dart';
import 'package:permission_handler/permission_handler.dart';
import 'package:app_settings/app_settings.dart';
import 'package:deep_voice/core/utils/logger.dart';
import 'package:path_provider/path_provider.dart';
class Message {
final bool isUser; //标识信息是否来自用户还是AI
@ -136,10 +137,11 @@ class AgentController extends GetxController {
Future<void> _loadChatHistory() async {
try {
Logger.i(TAG, '正在加载聊天历史...');
//通过本地插件ChatStorage获取聊天记录
final result = await ChatStorage.getMessages(
sessionId: sessionId,
page: 1,
pageSize: 20,
sessionId: sessionId, //会话ID
page: 1, //从第一页开始
pageSize: 20, //每页20条记录
);
if (result.messages.isNotEmpty) {
@ -148,11 +150,22 @@ class AgentController extends GetxController {
final meta = msg.metadata != null && msg.metadata != ""
? jsonDecode(msg.metadata!)
: null; // 如果为空，使用空对象
// 从元数据中提取图片信息
final hasImage = meta != null && meta.containsKey('hasImage')
? meta['hasImage'] as bool
: false;
final imagePath = meta != null && meta.containsKey('imagePath')
? meta['imagePath'] as String?
: null;
return Message(
isUser: msg.sender == 'user',
text: msg.message,
meta: meta,
timestamp: DateTime.parse(msg.timestamp),
hasImage: hasImage,
imagePath: imagePath,
);
}).toList();
@ -161,6 +174,22 @@ class AgentController extends GetxController {
messages.assignAll(historyMessages);
Logger.i(TAG, '成功加载 ${historyMessages.length} 条聊天记录');
// 详细打印每条消息的内容和属性，便于调试
Logger.d(TAG, '===== 历史消息详情开始 =====');
for (int i = 0; i < historyMessages.length; i++) {
final msg = historyMessages[i];
Logger.d(
TAG,
'消息[$i]: {'
'isUser: ${msg.isUser}, '
'text: "${msg.text.length > 50 ? msg.text.substring(0, 50) + "..." : msg.text}", '
'time: ${msg.timestamp}, '
'hasImage: ${msg.hasImage}, '
'imagePath: ${msg.imagePath ?? "无"}, '
'meta: ${msg.meta != null ? "有" : "无"}'
'}');
}
Logger.d(TAG, '===== 历史消息详情结束 =====');
} else {
// 没有历史记录时添加欢迎消息
_addWelcomeMessage();
@ -180,7 +209,7 @@ class AgentController extends GetxController {
void _addWelcomeMessage() {
messages.add(Message(
isUser: false,
text: '你好！我是**Emma**，我能帮你:\n\n'
text: '你好！我是**小言**，我能帮你:\n\n'
'• 答问题和提供信息\n\n'
'• 实时语音翻译\n\n'
'• 推荐音乐和播客\n\n'
@ -424,7 +453,7 @@ class AgentController extends GetxController {
});
}
// 发送文本消息
// 发送消息（包括所有支持的消息类型）
Future<void> sendTextMessage() async {
// 如果图片输入是活跃的，使用选中的图片和文本框的内容
if (isImageInputActive.value && selectedImagePath.value != null) {
@ -466,6 +495,9 @@ class AgentController extends GetxController {
return;
}
// 复制图片到应用永久存储目录
final permanentImagePath = await _copyImageToPermanentStorage(imagePath);
// 显示的文本，如果没有提供则使用默认值
final displayText = text?.isNotEmpty == true ? text! : '[图片]';
@ -474,7 +506,7 @@ class AgentController extends GetxController {
isUser: true,
text: displayText,
hasImage: true,
imagePath: imagePath,
imagePath: permanentImagePath,
);
messages.add(message);
@ -489,9 +521,12 @@ class AgentController extends GetxController {
// 标记为新的AI回复
_isNewAssistantResponse = true;
// 保存消息到持久化存储
//await _saveMessageToStorage(message);
// 调用Agent Service处理图片，考虑TTS状态
await AgentService.processImageInput(
imagePath,
permanentImagePath,
text: text ?? '',
speakResponse: isTtsEnabled.value,
);
@ -502,6 +537,68 @@ class AgentController extends GetxController {
}
}
// 将图片复制到应用的永久存储目录
Future<String> _copyImageToPermanentStorage(String sourcePath) async {
try {
final File sourceFile = File(sourcePath);
if (!await sourceFile.exists()) {
Logger.e(TAG, '源图片文件不存在: $sourcePath');
return sourcePath; // 如果源文件不存在，返回原路径
}
// 创建应用文档目录下的chatImages文件夹
final appDir = await getApplicationDocumentsDirectory();
final imagesDir = Directory('${appDir.path}/chatImages');
if (!await imagesDir.exists()) {
await imagesDir.create(recursive: true);
}
// 生成唯一文件名
final fileName =
'${DateTime.now().millisecondsSinceEpoch}_${sourcePath.split('/').last}';
final targetPath = '${imagesDir.path}/$fileName';
// 复制文件
await sourceFile.copy(targetPath);
Logger.i(TAG, '图片已复制到永久存储: $targetPath');
return targetPath;
} catch (e) {
Logger.e(TAG, '复制图片到永久存储失败: $e');
return sourcePath; // 出错时返回原路径
}
}
// 保存消息到持久化存储
// Future<void> _saveMessageToStorage(Message message) async {
// try {
// // 准备元数据，包含图片信息
// Map<String, dynamic> metadata = {};
// if (message.meta != null) {
// metadata = Map<String, dynamic>.from(message.meta!);
// }
// // 添加图片相关信息到元数据
// if (message.hasImage) {
// metadata['hasImage'] = true;
// metadata['imagePath'] = message.imagePath;
// }
// // 保存消息
// await ChatStorage.saveMessage(
// sessionId: sessionId,
// sender: message.isUser ? 'user' : 'assistant',
// message: message.text,
// metadata: metadata.isNotEmpty ? jsonEncode(metadata) : null,
// //timestamp: message.timestamp.toIso8601String(),
// );
// Logger.i(TAG, '消息已保存到持久化存储');
// } catch (e) {
// Logger.e(TAG, '保存消息到持久化存储失败: $e');
// }
// }
// 切换输入模式
void toggleInputMode() async {
// 如果当前在图片输入模式，先清除

2
lib/modules/agent/views/agent_view.dart
View File

@ -52,7 +52,7 @@ class AgentView extends StatelessWidget {
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text(
'Emma',
'小言'.tr,
style: TextStyle(
fontSize: 16,
fontWeight: FontWeight.bold,

38
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
View File

@ -91,7 +91,6 @@ object AgentService : CoroutineScope {
// 空闲检测相关
private var idleCheckJob: Job? = null
private val maxIdleSeconds = 10 // 最大空闲秒数
/**
* 初始化系统提示词
*/
@ -644,12 +643,14 @@ object AgentService : CoroutineScope {
* @param displayText 用于显示和存储的文本
* @param speakResponse 是否朗读回复
* @param hasImage 是否包含图片
* @param imagePath 图片路径（如果有）
*/
private fun processWithOpenAIInternal(
userMessage: JSONObject,
displayText: String,
speakResponse: Boolean = true,
hasImage: Boolean = false
hasImage: Boolean = false,
imagePath: String? = null
) {
// 如果有正在进行的AI流式输出，先停止它
stopAiStream()
@ -662,8 +663,15 @@ object AgentService : CoroutineScope {
// 使用历史记录作为上下文发送到OpenAI
val responseBuilder = StringBuilder()
var metadata:String = ""
var aiMetadata:String = ""
var userMetadata = JSONObject()
// 如果有图片，添加图片信息到元数据
if (hasImage) {
userMetadata.put("hasImage", true)
userMetadata.put("imagePath", imagePath)
}
// 添加系统提示词到历史消息的副本中
val messagesWithSystemPrompt = JSONArray()
@ -741,7 +749,9 @@ object AgentService : CoroutineScope {
addToHistoryMessages(openAIService.createAssistantMessage(response))
// 保存聊天记录
saveChatMessage(displayText, response,metadata)
saveChatMessage(displayText, response,aiMetadata,userMetadata.toString())
}
// 标记AI流式输出已完成
@ -784,7 +794,7 @@ object AgentService : CoroutineScope {
"result" to functionCallResult.toString(),
))
Log.d(TAG, "mcp调用结果: $functionCallResult")
metadata = autoHandleFcunCallResult(functionCallResult);
aiMetadata = autoHandleFcunCallResult(functionCallResult);
}
}
)
@ -887,7 +897,7 @@ object AgentService : CoroutineScope {
/**
* 保存聊天消息
*/
private fun saveChatMessage(userMessage: String, assistantMessage: String,metadata : String) {
private fun saveChatMessage(userMessage: String, assistantMessage: String,aiMetadata : String,userMetadata : String) {
// 显式指定IO调度器，数据库操作应在后台线程执行，即使父协程在Main上下文中
launch(Dispatchers.IO) {
try {
@ -895,7 +905,8 @@ object AgentService : CoroutineScope {
val userMessageId = chatStorageHelper.saveMessage(
sessionId = sessionId,
message = userMessage,
sender = "user"
sender = "user",
metadata = userMetadata
)
if (userMessageId != -1L) {
@ -904,7 +915,7 @@ object AgentService : CoroutineScope {
sessionId = sessionId,
message = assistantMessage,
sender = "assistant",
metadata = metadata
metadata = aiMetadata
)
// if (assistantMessageId != -1L) {
@ -1009,6 +1020,11 @@ object AgentService : CoroutineScope {
))
return@launch
}
// 创建带图片的用户消息并处理
val userMessage = openAIService.createUserMessageWithImage(text, imageBase64)
// 图片描述用于存储
val displayText = text.ifEmpty { "[图片]" }
// 通知图片准备完成
sendEvent("image_ready", mapOf(
@ -1016,8 +1032,8 @@ object AgentService : CoroutineScope {
"imagePath" to imagePath
))
// 处理包含图片的消息
processImageWithOpenAI(imageBase64, text, speakResponse)
// 处理消息并传入存储路径
processWithOpenAIInternal(userMessage, displayText, speakResponse, true, imagePath)
} catch (e: Exception) {
Log.e(TAG, "处理图片失败: ${e.message}")
sendEvent("error", mapOf(

127
local_plugins/agent_service/lib/agent_service.dart
View File

@ -7,61 +7,61 @@ import 'package:flutter/services.dart';
enum AgentServiceEventType {
/// 识别开始
recognitionStarted,
/// 识别中
recognizing,
/// 识别最终结果
recognitionResult,
/// 识别停止
recognitionStopped,
/// 识别取消
recognitionCanceled,
/// 自动停止
autoStop,
/// 识别错误
recognitionError,
/// TTS开始
ttsStarted,
/// TTS完成
ttsCompleted,
/// TTS取消
ttsCanceled,
/// TTS停止
ttsStopped,
/// 响应中断
responseInterrupted,
/// AI助手Token
assistantToken,
/// AI助手完整响应
assistantResponse,
/// 函数调用
functionCall,
/// 函数调用结果
functionCallResult,
/// 图片处理中
imageProcessing,
/// 图片准备就绪
imageReady,
/// 错误
error,
/// 未知事件
unknown
}
@ -70,12 +70,12 @@ enum AgentServiceEventType {
class AgentServiceEvent {
/// 事件类型
final AgentServiceEventType type;
/// 事件数据
final Map<String, dynamic> data;
AgentServiceEvent({required this.type, required this.data});
@override
String toString() => 'AgentServiceEvent(type: $type, data: $data)';
}
@ -85,20 +85,22 @@ class AgentServiceException implements Exception {
final String code;
final String message;
final dynamic details;
AgentServiceException(this.code, this.message, [this.details]);
@override
String toString() => '代理服务异常($code): $message';
}
/// 代理服务插件
class AgentService {
static const MethodChannel _channel = MethodChannel('com.yunqiinnovation.agent_service');
static const EventChannel _eventChannel = EventChannel('com.yunqiinnovation.agent_service/events');
static const MethodChannel _channel =
MethodChannel('com.yunqiinnovation.agent_service');
static const EventChannel _eventChannel =
EventChannel('com.yunqiinnovation.agent_service/events');
static Stream<AgentServiceEvent>? _eventStream;
/// 获取事件流
static Stream<AgentServiceEvent> get events {
_eventStream ??= _eventChannel.receiveBroadcastStream().map((event) {
@ -106,22 +108,21 @@ class AgentService {
final Map<String, dynamic> eventMap = jsonDecode(event);
final String eventName = eventMap['event'];
final Map<String, dynamic> eventData = eventMap['data'];
// 将事件名称转换为枚举
final AgentServiceEventType type = _stringToEventType(eventName);
return AgentServiceEvent(type: type, data: eventData);
} catch (e) {
return AgentServiceEvent(
type: AgentServiceEventType.unknown,
data: {'error': e.toString(), 'rawEvent': event}
);
type: AgentServiceEventType.unknown,
data: {'error': e.toString(), 'rawEvent': event});
}
});
return _eventStream!;
}
/// 将字符串事件名转换为枚举类型
static AgentServiceEventType _stringToEventType(String eventName) {
switch (eventName) {
@ -165,9 +166,9 @@ class AgentService {
return AgentServiceEventType.unknown;
}
}
/// 初始化代理服务
///
///
/// [config] 初始化配置，应包含以下参数：
/// - azureSpeechKey: Azure语音服务密钥
/// - azureSpeechRegion: Azure语音服务区域
@ -176,7 +177,7 @@ class AgentService {
/// - openaiModel: (可选) OpenAI模型名称，默认为"gpt-3.5-turbo"
/// - mcpServer: (可选) MCP服务器地址
/// - systemPrompt: (可选) 自定义系统提示词，用于设置AI语音助手的行为和风格
///
///
/// 返回是否初始化成功
static Future<bool> initialize(Map<String, dynamic> config) async {
try {
@ -188,9 +189,9 @@ class AgentService {
throw AgentServiceException(e.code, e.message ?? '初始化失败', e.details);
}
}
/// 启动语音助手服务
///
///
/// 启动常驻的语音助手服务，支持后台蓝牙设备唤醒和媒体按钮唤醒
/// [config] 配置参数，应包含以下参数：
/// - azureSpeechKey: Azure语音服务密钥
@ -199,7 +200,7 @@ class AgentService {
/// - openaiBaseUrl: (可选) OpenAI API 基础URL
/// - openaiModel: (可选) OpenAI模型名称，默认为"gpt-3.5-turbo"
/// - mcpServer: (可选) MCP服务器地址
///
///
/// 返回是否成功启动服务
static Future<bool> startAgentService(Map<String, dynamic> config) async {
try {
@ -211,9 +212,9 @@ class AgentService {
throw AgentServiceException(e.code, e.message ?? '启动语音助手服务失败', e.details);
}
}
/// 停止语音助手服务
///
///
/// 停止常驻的语音助手服务
/// 返回是否成功停止服务
static Future<bool> stopAgentService() async {
@ -224,9 +225,9 @@ class AgentService {
throw AgentServiceException(e.code, e.message ?? '停止语音助手服务失败', e.details);
}
}
/// 开始对话
///
///
/// 启动语音识别，开始监听用户语音输入
/// 返回是否成功开始对话
static Future<bool> startConversation() async {
@ -237,9 +238,9 @@ class AgentService {
throw AgentServiceException(e.code, e.message ?? '启动对话失败', e.details);
}
}
/// 停止对话
///
///
/// 停止语音识别
/// 返回是否成功停止对话
static Future<bool> stopConversation() async {
@ -250,13 +251,14 @@ class AgentService {
throw AgentServiceException(e.code, e.message ?? '停止对话失败', e.details);
}
}
/// 处理文本输入
///
///
/// [text] 文本内容
/// [speakResponse] 是否朗读响应
/// 返回是否成功处理文本
static Future<bool> processTextInput(String text, {bool speakResponse = false}) async {
static Future<bool> processTextInput(String text,
{bool speakResponse = false}) async {
try {
final bool result = await _channel.invokeMethod('processTextInput', {
'text': text,
@ -267,9 +269,9 @@ class AgentService {
throw AgentServiceException(e.code, e.message ?? '处理文本输入失败', e.details);
}
}
/// 朗读文本
///
///
/// [text] 要朗读的文本
static Future<bool> speakText(String text) async {
try {
@ -281,7 +283,7 @@ class AgentService {
throw AgentServiceException(e.code, e.message ?? '语音合成失败', e.details);
}
}
/// 停止语音合成
static Future<bool> stopTts() async {
try {
@ -291,7 +293,7 @@ class AgentService {
throw AgentServiceException(e.code, e.message ?? '停止语音合成失败', e.details);
}
}
/// 清除聊天历史
static Future<bool> clearChatHistory() async {
try {
@ -301,15 +303,15 @@ class AgentService {
throw AgentServiceException(e.code, e.message ?? '清除聊天历史失败', e.details);
}
}
/// 中断当前响应
///
///
/// 停止语音合成和AI流输出
static Future<bool> interruptCurrentResponse() async {
try {
// 首先尝试停止TTS播放
await stopTts();
// 根据原生层的实现，processTextInput可以被用来中断当前的AI流输出
// 这里我们通过发送一个特殊的空字符串来实现这一点
// 然后立即返回true，因为我们不需要真正处理任何文本
@ -318,14 +320,15 @@ class AgentService {
throw AgentServiceException(e.code, e.message ?? '中断响应失败', e.details);
}
}
/// 处理图片输入
///
///
/// [imagePath] 图片文件路径
/// [text] 可选的文本描述或问题
/// [speakResponse] 是否朗读响应
/// 返回是否成功处理图片
static Future<bool> processImageInput(String imagePath, {String text = "", bool speakResponse = false}) async {
static Future<bool> processImageInput(String imagePath,
{String text = "", bool speakResponse = false}) async {
try {
final bool result = await _channel.invokeMethod('processImageInput', {
'imagePath': imagePath,
@ -337,7 +340,7 @@ class AgentService {
throw AgentServiceException(e.code, e.message ?? '处理图片输入失败', e.details);
}
}
/// 释放资源
static Future<bool> dispose() async {
try {
@ -347,4 +350,4 @@ class AgentService {
throw AgentServiceException(e.code, e.message ?? '释放资源失败', e.details);
}
}
}
}

Write Preview
Loading…
Cancel
Save