diff --git a/lib/modules/agent/controllers/agent_controller.dart b/lib/modules/agent/controllers/agent_controller.dart index 97d17f807..446845002 100644 --- a/lib/modules/agent/controllers/agent_controller.dart +++ b/lib/modules/agent/controllers/agent_controller.dart @@ -11,6 +11,7 @@ import 'package:get_storage/get_storage.dart'; import 'package:permission_handler/permission_handler.dart'; import 'package:app_settings/app_settings.dart'; import 'package:deep_voice/core/utils/logger.dart'; +import 'package:path_provider/path_provider.dart'; class Message { final bool isUser; //标识信息是否来自用户还是AI @@ -136,10 +137,11 @@ class AgentController extends GetxController { Future _loadChatHistory() async { try { Logger.i(TAG, '正在加载聊天历史...'); + //通过本地插件ChatStorage获取聊天记录 final result = await ChatStorage.getMessages( - sessionId: sessionId, - page: 1, - pageSize: 20, + sessionId: sessionId, //会话ID + page: 1, //从第一页开始 + pageSize: 20, //每页20条记录 ); if (result.messages.isNotEmpty) { @@ -148,11 +150,22 @@ class AgentController extends GetxController { final meta = msg.metadata != null && msg.metadata != "" ? jsonDecode(msg.metadata!) : null; // 如果为空，使用空对象 + + // 从元数据中提取图片信息 + final hasImage = meta != null && meta.containsKey('hasImage') + ? meta['hasImage'] as bool + : false; + final imagePath = meta != null && meta.containsKey('imagePath') + ? meta['imagePath'] as String? + : null; + return Message( isUser: msg.sender == 'user', text: msg.message, meta: meta, timestamp: DateTime.parse(msg.timestamp), + hasImage: hasImage, + imagePath: imagePath, ); }).toList(); @@ -161,6 +174,22 @@ class AgentController extends GetxController { messages.assignAll(historyMessages); Logger.i(TAG, '成功加载 ${historyMessages.length} 条聊天记录'); + // 详细打印每条消息的内容和属性，便于调试 + Logger.d(TAG, '===== 历史消息详情开始 ====='); + for (int i = 0; i < historyMessages.length; i++) { + final msg = historyMessages[i]; + Logger.d( + TAG, + '消息[$i]: {' + 'isUser: ${msg.isUser}, ' + 'text: "${msg.text.length > 50 ? msg.text.substring(0, 50) + "..." : msg.text}", ' + 'time: ${msg.timestamp}, ' + 'hasImage: ${msg.hasImage}, ' + 'imagePath: ${msg.imagePath ?? "无"}, ' + 'meta: ${msg.meta != null ? "有" : "无"}' + '}'); + } + Logger.d(TAG, '===== 历史消息详情结束 ====='); } else { // 没有历史记录时添加欢迎消息 _addWelcomeMessage(); @@ -180,7 +209,7 @@ class AgentController extends GetxController { void _addWelcomeMessage() { messages.add(Message( isUser: false, - text: '你好！我是**Emma**，我能帮你:\n\n' + text: '你好！我是**小言**，我能帮你:\n\n' '• 答问题和提供信息\n\n' '• 实时语音翻译\n\n' '• 推荐音乐和播客\n\n' @@ -424,7 +453,7 @@ class AgentController extends GetxController { }); } - // 发送文本消息 + // 发送消息（包括所有支持的消息类型） Future sendTextMessage() async { // 如果图片输入是活跃的，使用选中的图片和文本框的内容 if (isImageInputActive.value && selectedImagePath.value != null) { @@ -466,6 +495,9 @@ class AgentController extends GetxController { return; } + // 复制图片到应用永久存储目录 + final permanentImagePath = await _copyImageToPermanentStorage(imagePath); + // 显示的文本，如果没有提供则使用默认值 final displayText = text?.isNotEmpty == true ? text! : '[图片]'; @@ -474,7 +506,7 @@ class AgentController extends GetxController { isUser: true, text: displayText, hasImage: true, - imagePath: imagePath, + imagePath: permanentImagePath, ); messages.add(message); @@ -489,9 +521,12 @@ class AgentController extends GetxController { // 标记为新的AI回复 _isNewAssistantResponse = true; + // 保存消息到持久化存储 + //await _saveMessageToStorage(message); + // 调用Agent Service处理图片，考虑TTS状态 await AgentService.processImageInput( - imagePath, + permanentImagePath, text: text ?? '', speakResponse: isTtsEnabled.value, ); @@ -502,6 +537,68 @@ class AgentController extends GetxController { } } +// 将图片复制到应用的永久存储目录 + Future _copyImageToPermanentStorage(String sourcePath) async { + try { + final File sourceFile = File(sourcePath); + if (!await sourceFile.exists()) { + Logger.e(TAG, '源图片文件不存在: $sourcePath'); + return sourcePath; // 如果源文件不存在，返回原路径 + } + + // 创建应用文档目录下的chatImages文件夹 + final appDir = await getApplicationDocumentsDirectory(); + final imagesDir = Directory('${appDir.path}/chatImages'); + if (!await imagesDir.exists()) { + await imagesDir.create(recursive: true); + } + + // 生成唯一文件名 + final fileName = + '${DateTime.now().millisecondsSinceEpoch}_${sourcePath.split('/').last}'; + final targetPath = '${imagesDir.path}/$fileName'; + + // 复制文件 + await sourceFile.copy(targetPath); + Logger.i(TAG, '图片已复制到永久存储: $targetPath'); + + return targetPath; + } catch (e) { + Logger.e(TAG, '复制图片到永久存储失败: $e'); + return sourcePath; // 出错时返回原路径 + } + } + + // 保存消息到持久化存储 + // Future _saveMessageToStorage(Message message) async { + // try { + // // 准备元数据，包含图片信息 + // Map metadata = {}; + // if (message.meta != null) { + // metadata = Map.from(message.meta!); + // } + + // // 添加图片相关信息到元数据 + // if (message.hasImage) { + // metadata['hasImage'] = true; + // metadata['imagePath'] = message.imagePath; + // } + + // // 保存消息 + // await ChatStorage.saveMessage( + // sessionId: sessionId, + // sender: message.isUser ? 'user' : 'assistant', + // message: message.text, + // metadata: metadata.isNotEmpty ? jsonEncode(metadata) : null, + // //timestamp: message.timestamp.toIso8601String(), + // ); + + // Logger.i(TAG, '消息已保存到持久化存储'); + // } catch (e) { + // Logger.e(TAG, '保存消息到持久化存储失败: $e'); + // } + // } + // 切换输入模式 void toggleInputMode() async { // 如果当前在图片输入模式，先清除 diff --git a/lib/modules/agent/views/agent_view.dart b/lib/modules/agent/views/agent_view.dart index 9d9aec1d7..54cb541d3 100644 --- a/lib/modules/agent/views/agent_view.dart +++ b/lib/modules/agent/views/agent_view.dart @@ -52,7 +52,7 @@ class AgentView extends StatelessWidget { crossAxisAlignment: CrossAxisAlignment.start, children: [ Text( - 'Emma', + '小言'.tr, style: TextStyle( fontSize: 16, fontWeight: FontWeight.bold, diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index a3ef740d6..cda7c7df7 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -91,7 +91,6 @@ object AgentService : CoroutineScope { // 空闲检测相关 private var idleCheckJob: Job? = null private val maxIdleSeconds = 10 // 最大空闲秒数 - /** * 初始化系统提示词 */ @@ -644,12 +643,14 @@ object AgentService : CoroutineScope { * @param displayText 用于显示和存储的文本 * @param speakResponse 是否朗读回复 * @param hasImage 是否包含图片 + * @param imagePath 图片路径（如果有） */ private fun processWithOpenAIInternal( userMessage: JSONObject, displayText: String, speakResponse: Boolean = true, - hasImage: Boolean = false + hasImage: Boolean = false, + imagePath: String? = null ) { // 如果有正在进行的AI流式输出，先停止它 stopAiStream() @@ -662,8 +663,15 @@ object AgentService : CoroutineScope { // 使用历史记录作为上下文发送到OpenAI val responseBuilder = StringBuilder() - var metadata:String = "" - + var aiMetadata:String = "" + var userMetadata = JSONObject() + + // 如果有图片，添加图片信息到元数据 + if (hasImage) { + userMetadata.put("hasImage", true) + userMetadata.put("imagePath", imagePath) + } + // 添加系统提示词到历史消息的副本中 val messagesWithSystemPrompt = JSONArray() @@ -741,7 +749,9 @@ object AgentService : CoroutineScope { addToHistoryMessages(openAIService.createAssistantMessage(response)) // 保存聊天记录 - saveChatMessage(displayText, response,metadata) + saveChatMessage(displayText, response,aiMetadata,userMetadata.toString()) + + } // 标记AI流式输出已完成 @@ -784,7 +794,7 @@ object AgentService : CoroutineScope { "result" to functionCallResult.toString(), )) Log.d(TAG, "mcp调用结果: $functionCallResult") - metadata = autoHandleFcunCallResult(functionCallResult); + aiMetadata = autoHandleFcunCallResult(functionCallResult); } } ) @@ -887,7 +897,7 @@ object AgentService : CoroutineScope { /** * 保存聊天消息 */ - private fun saveChatMessage(userMessage: String, assistantMessage: String,metadata : String) { + private fun saveChatMessage(userMessage: String, assistantMessage: String,aiMetadata : String,userMetadata : String) { // 显式指定IO调度器，数据库操作应在后台线程执行，即使父协程在Main上下文中 launch(Dispatchers.IO) { try { @@ -895,7 +905,8 @@ object AgentService : CoroutineScope { val userMessageId = chatStorageHelper.saveMessage( sessionId = sessionId, message = userMessage, - sender = "user" + sender = "user", + metadata = userMetadata ) if (userMessageId != -1L) { @@ -904,7 +915,7 @@ object AgentService : CoroutineScope { sessionId = sessionId, message = assistantMessage, sender = "assistant", - metadata = metadata + metadata = aiMetadata ) // if (assistantMessageId != -1L) { @@ -1009,6 +1020,11 @@ object AgentService : CoroutineScope { )) return@launch } + + // 创建带图片的用户消息并处理 + val userMessage = openAIService.createUserMessageWithImage(text, imageBase64) + // 图片描述用于存储 + val displayText = text.ifEmpty { "[图片]" } // 通知图片准备完成 sendEvent("image_ready", mapOf( @@ -1016,8 +1032,8 @@ object AgentService : CoroutineScope { "imagePath" to imagePath )) - // 处理包含图片的消息 - processImageWithOpenAI(imageBase64, text, speakResponse) + // 处理消息并传入存储路径 + processWithOpenAIInternal(userMessage, displayText, speakResponse, true, imagePath) } catch (e: Exception) { Log.e(TAG, "处理图片失败: ${e.message}") sendEvent("error", mapOf( diff --git a/local_plugins/agent_service/lib/agent_service.dart b/local_plugins/agent_service/lib/agent_service.dart index ffcdd5fbb..4397f4a85 100644 --- a/local_plugins/agent_service/lib/agent_service.dart +++ b/local_plugins/agent_service/lib/agent_service.dart @@ -7,61 +7,61 @@ import 'package:flutter/services.dart'; enum AgentServiceEventType { /// 识别开始 recognitionStarted, - + /// 识别中 recognizing, - + /// 识别最终结果 recognitionResult, - + /// 识别停止 recognitionStopped, - + /// 识别取消 recognitionCanceled, - + /// 自动停止 autoStop, - + /// 识别错误 recognitionError, - + /// TTS开始 ttsStarted, - + /// TTS完成 ttsCompleted, - + /// TTS取消 ttsCanceled, - + /// TTS停止 ttsStopped, - + /// 响应中断 responseInterrupted, - + /// AI助手Token assistantToken, - + /// AI助手完整响应 assistantResponse, - + /// 函数调用 functionCall, - + /// 函数调用结果 functionCallResult, - + /// 图片处理中 imageProcessing, - + /// 图片准备就绪 imageReady, - + /// 错误 error, - + /// 未知事件 unknown } @@ -70,12 +70,12 @@ enum AgentServiceEventType { class AgentServiceEvent { /// 事件类型 final AgentServiceEventType type; - + /// 事件数据 final Map data; - + AgentServiceEvent({required this.type, required this.data}); - + @override String toString() => 'AgentServiceEvent(type: $type, data: $data)'; } @@ -85,20 +85,22 @@ class AgentServiceException implements Exception { final String code; final String message; final dynamic details; - + AgentServiceException(this.code, this.message, [this.details]); - + @override String toString() => '代理服务异常($code): $message'; } /// 代理服务插件 class AgentService { - static const MethodChannel _channel = MethodChannel('com.yunqiinnovation.agent_service'); - static const EventChannel _eventChannel = EventChannel('com.yunqiinnovation.agent_service/events'); - + static const MethodChannel _channel = + MethodChannel('com.yunqiinnovation.agent_service'); + static const EventChannel _eventChannel = + EventChannel('com.yunqiinnovation.agent_service/events'); + static Stream? _eventStream; - + /// 获取事件流 static Stream get events { _eventStream ??= _eventChannel.receiveBroadcastStream().map((event) { @@ -106,22 +108,21 @@ class AgentService { final Map eventMap = jsonDecode(event); final String eventName = eventMap['event']; final Map eventData = eventMap['data']; - + // 将事件名称转换为枚举 final AgentServiceEventType type = _stringToEventType(eventName); - + return AgentServiceEvent(type: type, data: eventData); } catch (e) { return AgentServiceEvent( - type: AgentServiceEventType.unknown, - data: {'error': e.toString(), 'rawEvent': event} - ); + type: AgentServiceEventType.unknown, + data: {'error': e.toString(), 'rawEvent': event}); } }); - + return _eventStream!; } - + /// 将字符串事件名转换为枚举类型 static AgentServiceEventType _stringToEventType(String eventName) { switch (eventName) { @@ -165,9 +166,9 @@ class AgentService { return AgentServiceEventType.unknown; } } - + /// 初始化代理服务 - /// + /// /// [config] 初始化配置，应包含以下参数： /// - azureSpeechKey: Azure语音服务密钥 /// - azureSpeechRegion: Azure语音服务区域 @@ -176,7 +177,7 @@ class AgentService { /// - openaiModel: (可选) OpenAI模型名称，默认为"gpt-3.5-turbo" /// - mcpServer: (可选) MCP服务器地址 /// - systemPrompt: (可选) 自定义系统提示词，用于设置AI语音助手的行为和风格 - /// + /// /// 返回是否初始化成功 static Future initialize(Map config) async { try { @@ -188,9 +189,9 @@ class AgentService { throw AgentServiceException(e.code, e.message ?? '初始化失败', e.details); } } - + /// 启动语音助手服务 - /// + /// /// 启动常驻的语音助手服务，支持后台蓝牙设备唤醒和媒体按钮唤醒 /// [config] 配置参数，应包含以下参数： /// - azureSpeechKey: Azure语音服务密钥 @@ -199,7 +200,7 @@ class AgentService { /// - openaiBaseUrl: (可选) OpenAI API 基础URL /// - openaiModel: (可选) OpenAI模型名称，默认为"gpt-3.5-turbo" /// - mcpServer: (可选) MCP服务器地址 - /// + /// /// 返回是否成功启动服务 static Future startAgentService(Map config) async { try { @@ -211,9 +212,9 @@ class AgentService { throw AgentServiceException(e.code, e.message ?? '启动语音助手服务失败', e.details); } } - + /// 停止语音助手服务 - /// + /// /// 停止常驻的语音助手服务 /// 返回是否成功停止服务 static Future stopAgentService() async { @@ -224,9 +225,9 @@ class AgentService { throw AgentServiceException(e.code, e.message ?? '停止语音助手服务失败', e.details); } } - + /// 开始对话 - /// + /// /// 启动语音识别，开始监听用户语音输入 /// 返回是否成功开始对话 static Future startConversation() async { @@ -237,9 +238,9 @@ class AgentService { throw AgentServiceException(e.code, e.message ?? '启动对话失败', e.details); } } - + /// 停止对话 - /// + /// /// 停止语音识别 /// 返回是否成功停止对话 static Future stopConversation() async { @@ -250,13 +251,14 @@ class AgentService { throw AgentServiceException(e.code, e.message ?? '停止对话失败', e.details); } } - + /// 处理文本输入 - /// + /// /// [text] 文本内容 /// [speakResponse] 是否朗读响应 /// 返回是否成功处理文本 - static Future processTextInput(String text, {bool speakResponse = false}) async { + static Future processTextInput(String text, + {bool speakResponse = false}) async { try { final bool result = await _channel.invokeMethod('processTextInput', { 'text': text, @@ -267,9 +269,9 @@ class AgentService { throw AgentServiceException(e.code, e.message ?? '处理文本输入失败', e.details); } } - + /// 朗读文本 - /// + /// /// [text] 要朗读的文本 static Future speakText(String text) async { try { @@ -281,7 +283,7 @@ class AgentService { throw AgentServiceException(e.code, e.message ?? '语音合成失败', e.details); } } - + /// 停止语音合成 static Future stopTts() async { try { @@ -291,7 +293,7 @@ class AgentService { throw AgentServiceException(e.code, e.message ?? '停止语音合成失败', e.details); } } - + /// 清除聊天历史 static Future clearChatHistory() async { try { @@ -301,15 +303,15 @@ class AgentService { throw AgentServiceException(e.code, e.message ?? '清除聊天历史失败', e.details); } } - + /// 中断当前响应 - /// + /// /// 停止语音合成和AI流输出 static Future interruptCurrentResponse() async { try { // 首先尝试停止TTS播放 await stopTts(); - + // 根据原生层的实现，processTextInput可以被用来中断当前的AI流输出 // 这里我们通过发送一个特殊的空字符串来实现这一点 // 然后立即返回true，因为我们不需要真正处理任何文本 @@ -318,14 +320,15 @@ class AgentService { throw AgentServiceException(e.code, e.message ?? '中断响应失败', e.details); } } - + /// 处理图片输入 - /// + /// /// [imagePath] 图片文件路径 /// [text] 可选的文本描述或问题 /// [speakResponse] 是否朗读响应 /// 返回是否成功处理图片 - static Future processImageInput(String imagePath, {String text = "", bool speakResponse = false}) async { + static Future processImageInput(String imagePath, + {String text = "", bool speakResponse = false}) async { try { final bool result = await _channel.invokeMethod('processImageInput', { 'imagePath': imagePath, @@ -337,7 +340,7 @@ class AgentService { throw AgentServiceException(e.code, e.message ?? '处理图片输入失败', e.details); } } - + /// 释放资源 static Future dispose() async { try { @@ -347,4 +350,4 @@ class AgentService { throw AgentServiceException(e.code, e.message ?? '释放资源失败', e.details); } } -} \ No newline at end of file +}