diff --git a/lib/core/translations/language/zh_cn.dart b/lib/core/translations/language/zh_cn.dart index d41a69255..f9f4adb26 100644 --- a/lib/core/translations/language/zh_cn.dart +++ b/lib/core/translations/language/zh_cn.dart @@ -511,6 +511,7 @@ const Map zhCN = { "loadMoreMessages": "加载更多消息...", "loadFailedClickRetry": "加载失败，点击重试", "swipeUpForMore": "向上滑动查看更多", + "callingTool": "正在为您调用工具查询相关信息", // 预设提示词 "presetMusic": "来点音乐", // 来点音乐 diff --git a/lib/data/models/message_model.dart b/lib/data/models/message_model.dart index e2c534200..e15c07314 100644 --- a/lib/data/models/message_model.dart +++ b/lib/data/models/message_model.dart @@ -11,7 +11,7 @@ class Message { final bool hasImage; // 是否包含图片 final String? imagePath; // 图片路径 final Map? meta; //卡片数据 - final List? functionCalls; // 函数调用 + final Map>? functionCalls; // 函数调用 bool isToolCallsExpanded; // 工具调用是否展开 Message({ @@ -39,7 +39,8 @@ class Message { hasImage: json['hasImage'] as bool? ?? false, imagePath: json['imagePath'] as String?, meta: json['meta'] as Map?, - functionCalls: json['functionCalls'] as List?, + functionCalls: + json['functionCalls'] as Map>?, isToolCallsExpanded: json['isToolCallsExpanded'] as bool? ?? false, ); } @@ -80,7 +81,7 @@ class Message { bool? hasImage, String? imagePath, Map? meta, - List? functionCalls, + Map>? functionCalls, bool? isToolCallsExpanded, }) { return Message( diff --git a/lib/data/services/ble_manager.dart b/lib/data/services/ble_manager.dart index 1a3b2921c..ef78d9290 100644 --- a/lib/data/services/ble_manager.dart +++ b/lib/data/services/ble_manager.dart @@ -736,7 +736,8 @@ class BleManager extends GetxService { 'usertoken': jsonString, 'supportedLanguages': supportedLanguages, 'musicServiceType': musicServiceType, - "navigationMode": navigationMode + "navigationMode": navigationMode, + "callingTool": "callingTool".tr }; // 记录日志（不记录敏感信息） diff --git a/lib/modules/agent/controllers/agent_controller.dart b/lib/modules/agent/controllers/agent_controller.dart index 5021bab94..f37f56608 100644 --- a/lib/modules/agent/controllers/agent_controller.dart +++ b/lib/modules/agent/controllers/agent_controller.dart @@ -790,7 +790,9 @@ class AgentController extends GetxController with WidgetsBindingObserver { sessionid: sessionid, isUser: false, text: '', - functionCalls: [functionName])); + functionCalls: { + functionName: {'arguments': arguments} + })); _isNewAssistantResponse = false; } else { // 否则更新最后一条消息 @@ -803,8 +805,10 @@ class AgentController extends GetxController with WidgetsBindingObserver { text: messages[lastIndex].text, meta: messages[lastIndex].meta, timestamp: messages[lastIndex].timestamp, - functionCalls: messages[lastIndex].functionCalls ?? [] - ..add(functionName), + functionCalls: messages[lastIndex].functionCalls ?? {} + ..addAll({ + functionName: {'arguments': arguments} + }), ); } } @@ -812,10 +816,42 @@ class AgentController extends GetxController with WidgetsBindingObserver { case AgentServiceEventType.functionCallResult: final sessionid = event.data['sessionid'] ?? ''; - // final result = event.data['result']; + final function_name = event.data['function_name'] ?? ''; + final function_result = event.data['result'] ?? ''; final metaStr = event.data['meta'] ?? ''; - final functionCall = event.data['function_call'] ?? ''; + final index = messages.lastIndexWhere( + (msg) => msg.sessionid == sessionid && !msg.isUser); + if (index != -1) { + // 获取当前的 functionCalls，如果为空则创建新的 Map + final functionCalls = Map>.from( + messages[index].functionCalls?.map((key, value) => MapEntry( + key, Map.from(value as Map))) ?? + {}); + + // 更新对应工具的调用结果 + if (function_name.isNotEmpty) { + // 如果该工具已存在，更新其结果；如果不存在，创建新的条目 + if (functionCalls.containsKey(function_name)) { + functionCalls[function_name]!['result'] = function_result; + } else { + functionCalls[function_name] = { + 'result': function_result, + }; + } + } + + messages[index] = Message( + sessionid: sessionid, + isUser: false, + text: messages[index].text, + meta: messages[index].meta, + timestamp: messages[index].timestamp, + functionCalls: functionCalls, + isToolCallsExpanded: messages[index].isToolCallsExpanded, + ); + } Logger.i(TAG, 'Flutter 结束调用 $sessionid mcp: $metaStr'); + if (metaStr.isNotEmpty) { Map metaResult = {}; metaResult = jsonDecode(metaStr) as Map; @@ -830,8 +866,7 @@ class AgentController extends GetxController with WidgetsBindingObserver { //不回复直接返回 break; } - final index = messages.lastIndexWhere( - (msg) => msg.sessionid == sessionid && !msg.isUser); + Logger.i(TAG, 'Flutter 结束调用 卡片数据 $index $metaResult'); // 判断是否为新的回复或响应ID是否改变 if (index == -1) { @@ -917,7 +952,7 @@ class AgentController extends GetxController with WidgetsBindingObserver { Logger.i(TAG, 'Flutter 结束调用 解析失败 $metaStr'); } } - if (functionCall.contains("music")) { + if (function_name.contains("music")) { Future.delayed(Duration(seconds: 1), () { _qqmusicManager.syncCurrentPlayInfo(isOpenPlay: false); }); diff --git a/lib/modules/agent/views/message_bubble.dart b/lib/modules/agent/views/message_bubble.dart index 7feb47f07..bf2b04983 100644 --- a/lib/modules/agent/views/message_bubble.dart +++ b/lib/modules/agent/views/message_bubble.dart @@ -9,6 +9,7 @@ import '../../../modules/agent/views/navigation_card.dart'; import 'weather_card.dart'; import 'travel_card.dart'; import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; // 添加这个导入 import 'package:get/get.dart'; import 'package:intl/intl.dart'; import 'dart:io'; @@ -26,7 +27,7 @@ class MessageBubble extends GetView { final bool hasImage; final String? imagePath; final Map? meta; - final List? functionCalls; + final Map>? functionCalls; final bool isToolCallsExpanded; const MessageBubble({ Key? key, @@ -86,7 +87,7 @@ class MessageBubble extends GetView { padding: (!isUser && functionCalls != null && functionCalls!.isNotEmpty) - ? const EdgeInsets.only(top: 20) + ? const EdgeInsets.only(top: 30) : EdgeInsets.zero, child: _buildMessageContent(context), ), @@ -612,43 +613,50 @@ class MessageBubble extends GetView { child: Column( crossAxisAlignment: CrossAxisAlignment.start, children: [ - // 可点击的标题行 - GestureDetector( - onTap: () async { - await controller.updateToolCallsExpanded( - sessionid, - !isToolCallsExpanded, - ); - }, - child: Container( - height: 30, // 增加固定高度，给字体更多空间 - padding: EdgeInsets.all(8), - decoration: BoxDecoration( + // 标题行 + Container( + height: 40, // 增加固定高度，给字体更多空间 + padding: EdgeInsets.all(8), + decoration: BoxDecoration( + color: Theme.of(context).brightness == Brightness.dark + ? const Color.fromARGB(255, 71, 68, 68)?.withOpacity(0.8) + : Colors.grey[100]?.withOpacity(0.9), + borderRadius: const BorderRadius.only( + topLeft: Radius.circular(6), + topRight: Radius.circular(6), + ), + border: Border.all( color: Theme.of(context).brightness == Brightness.dark - ? const Color.fromARGB(255, 71, 68, 68) - ?.withOpacity(0.8) - : Colors.grey[100]?.withOpacity(0.9), - borderRadius: const BorderRadius.only( - topLeft: Radius.circular(6), - topRight: Radius.circular(6), - ), - border: Border.all( + ? Colors.white.withOpacity(0.1) + : Colors.black.withOpacity(0.1), + width: 0.5, + ), + ), + child: Row( + children: [ + Icon( + Icons.build, // 工具icon + size: 20, // 减小图标大小 color: Theme.of(context).brightness == Brightness.dark - ? Colors.white.withOpacity(0.1) - : Colors.black.withOpacity(0.1), - width: 0.5, + ? Colors.white.withOpacity(0.7) + : Colors.black54, ), - ), - child: Row( - children: [ - // 显示最后一个工具名称 - Expanded( + const SizedBox(width: 4), + // 显示最后一个工具名称 - 可点击展开 + Expanded( + child: GestureDetector( + onTap: () async { + await controller.updateToolCallsExpanded( + sessionid, + !isToolCallsExpanded, + ); + }, child: Text( functionCalls != null && functionCalls!.isNotEmpty - ? functionCalls!.last + ? functionCalls!.keys.last : '工具调用', style: TextStyle( - fontSize: 12, // 适当增大字体 + fontSize: 16, // 适当增大字体 color: Theme.of(context).brightness == Brightness.dark ? Colors.white.withOpacity(0.7) @@ -659,19 +667,47 @@ class MessageBubble extends GetView { overflow: TextOverflow.ellipsis, ), ), - const SizedBox(width: 4), - // 箭头放在右侧 - Icon( - isToolCallsExpanded - ? Icons.keyboard_arrow_up // 展开时显示向上箭头 - : Icons.keyboard_arrow_down, // 收起时显示向下箭头 - size: 14, // 减小图标大小 - color: Theme.of(context).brightness == Brightness.dark - ? Colors.white.withOpacity(0.7) - : Colors.black54, + ), + const SizedBox(width: 4), + // 拷贝按钮 + GestureDetector( + onTap: () { + _copyFunctionCallsToClipboard(); + }, + child: Container( + padding: const EdgeInsets.all(2), + child: Icon( + Icons.copy, + size: 20, + color: Theme.of(context).brightness == Brightness.dark + ? Colors.white.withOpacity(0.7) + : Colors.black54, + ), ), - ], - ), + ), + const SizedBox(width: 4), + // 展开按钮 + GestureDetector( + onTap: () async { + await controller.updateToolCallsExpanded( + sessionid, + !isToolCallsExpanded, + ); + }, + child: Container( + padding: const EdgeInsets.all(2), // 增加 padding 从 2 到 8 + child: Icon( + isToolCallsExpanded + ? Icons.keyboard_arrow_down // 展开时显示向上箭头 + : Icons.keyboard_arrow_up, // 收起时显示向下箭头 + size: 20, // 增加图标大小从 14 到 16 + color: Theme.of(context).brightness == Brightness.dark + ? Colors.white.withOpacity(0.7) + : Colors.black54, + ), + ), + ), + ], ), ), // 展开的工具调用列表 @@ -710,33 +746,38 @@ class MessageBubble extends GetView { // 添加滚动，防止超界 child: Column( crossAxisAlignment: CrossAxisAlignment.start, - children: functionCalls!.asMap().entries.map((entry) { - int index = entry.key; - String call = entry.value; + children: functionCalls!.entries.map((entry) { + String toolName = entry.key; + dynamic toolResult = entry.value; return Padding( - padding: EdgeInsets.only( - bottom: index < functionCalls!.length - 1 - ? 2 - : 0), // 减少间距 + padding: const EdgeInsets.only(bottom: 8), // 减少间距 child: Row( crossAxisAlignment: CrossAxisAlignment.start, children: [ - Text( - '${index + 1}. ', - style: TextStyle( - fontSize: 12, // 修改为更小的字体 - color: Theme.of(context).brightness == - Brightness.dark - ? Colors.white.withOpacity(0.6) - : Colors.black45, - fontWeight: FontWeight.w500, + // 工具名称 - 固定宽度 + SizedBox( + width: 80, // 固定宽度 + child: Text( + toolName, + style: TextStyle( + fontSize: 12, + color: Theme.of(context).brightness == + Brightness.dark + ? Colors.white.withOpacity(0.6) + : Colors.black45, + fontWeight: FontWeight.w500, + ), + maxLines: 2, + overflow: TextOverflow.ellipsis, ), ), + const SizedBox(width: 12), // 增加间距 + // 工具结果 - 剩余空间 Expanded( child: Text( - call, + toolResult?.toString() ?? '无结果', style: TextStyle( - fontSize: 12, // 修改为更小的字体 + fontSize: 12, color: Theme.of(context).brightness == Brightness.dark ? Colors.white.withOpacity(0.8) @@ -756,6 +797,44 @@ class MessageBubble extends GetView { ), )); } + + /// 拷贝工具调用内容到剪贴板 + void _copyFunctionCallsToClipboard() { + if (functionCalls == null || functionCalls!.isEmpty) { + Get.snackbar( + '提示', + '没有工具调用内容可以拷贝', + snackPosition: SnackPosition.BOTTOM, + duration: const Duration(seconds: 2), + ); + return; + } + + StringBuffer buffer = StringBuffer(); + buffer.writeln('工具调用详情：'); + buffer.writeln('=' * 30); + + functionCalls!.forEach((toolName, toolData) { + buffer.writeln('🔧 工具名称: $toolName'); + if (toolData is Map) { + toolData.forEach((key, value) { + buffer.writeln(' $key: $value'); + }); + } else { + buffer.writeln(' 结果: $toolData'); + } + buffer.writeln('-' * 20); + }); + + Clipboard.setData(ClipboardData(text: buffer.toString())); + + Get.snackbar( + '成功', + '工具调用内容已复制到剪贴板', + snackPosition: SnackPosition.BOTTOM, + duration: const Duration(seconds: 2), + ); + } } /// 动画点组件 diff --git a/lib/modules/home/views/home_view.dart b/lib/modules/home/views/home_view.dart index 5e5a72be4..745d19381 100644 --- a/lib/modules/home/views/home_view.dart +++ b/lib/modules/home/views/home_view.dart @@ -92,7 +92,6 @@ class HomeView extends GetView { // 设备状态卡片 IntroStepBuilder( - key: const ValueKey('home_device_status'), // 添加唯一key group: 'home', order: 1, onWidgetLoad: () { @@ -121,7 +120,6 @@ class HomeView extends GetView { SizedBox(height: 20.h), IntroStepBuilder( - key: const ValueKey('home_wakeup_guide'), // 添加唯一key group: 'home', order: 2, getOverlayPosition: IntroOverlayBuilder.overlayPosition, @@ -797,7 +795,6 @@ class HomeView extends GetView { Expanded( flex: 1, child: IntroStepBuilder( - key: const ValueKey('home_ai_assistant'), // 添加唯一key group: 'home', order: 3, getOverlayPosition: IntroOverlayBuilder.overlayPosition, diff --git a/lib/modules/pairing/controllers/pairing_controller.dart b/lib/modules/pairing/controllers/pairing_controller.dart index 954d636b6..5f548f303 100644 --- a/lib/modules/pairing/controllers/pairing_controller.dart +++ b/lib/modules/pairing/controllers/pairing_controller.dart @@ -803,13 +803,13 @@ class PairingController extends GetxController with WidgetsBindingObserver { } } else { //未绑定设备 - + _bleManager.clearPairedDevices(); Logger.i(_tag, '验证时BLE状态: $currentBleState'); if (currentBleState == 2) { //已连接 清理 // BleConnectionState.connected Logger.i(_tag, 'BLE已连接，断开并清理缓存'); - _bleManager.clearPairedDevices(); + _bleManager.disconnect(); } } diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index 995ee6935..21aaba574 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -91,6 +91,8 @@ object AgentService : CoroutineScope { // 会话ID，用于区分不同聊天上下文 private val agentId = "default_agent" + //工具调用提示 + private var callingTool = "" // 服务端连接 private var serverurl = "" private var usertoken = "" @@ -195,6 +197,7 @@ object AgentService : CoroutineScope { try { serverurl = config["serverurl"]?.toString() ?: ""; usertoken = config["usertoken"]?.toString() ?: ""; + callingTool = config["callingTool"]?.toString() ?: ""; // 初始化上下文和服务 this.context = appContext.applicationContext this.chatApiService = ChatApiService(context) @@ -620,7 +623,7 @@ object AgentService : CoroutineScope { Log.d(TAG, "recognizeCallback") val success = azureAsrHelper?.setupEventListeners(object : AzureAsrHelper.ContinuousRecognizeCallback { - override fun onRecognizing(sessiondid:String, recognizing: String, detectedLanguage: String) { + override fun onRecognizing(sessionid:String, recognizing: String, detectedLanguage: String) { if (recognizing.isNotEmpty()&&isKeepResult==true) { // 检测到语音，更新状态 @@ -664,17 +667,17 @@ object AgentService : CoroutineScope { } } - override fun onResult(sessiondid:String,result: String, detectedLanguage: String) { + override fun onResult(sessionid:String,result: String, detectedLanguage: String) { if (result.isNotEmpty()&&isKeepResult==true) {//加入是否结束要输出最后的结果判读 sendEvent( "recognition_result", mapOf( - "sessiondid" to sessiondid, + "sessionid" to sessionid, "text" to result, "language" to detectedLanguage ) ) - processWithChatApi(sessiondid,result) + processWithChatApi(sessionid,result) } @@ -684,8 +687,8 @@ object AgentService : CoroutineScope { } - override fun onSessionStarted(sessiondid:String) { - sendEvent("recognition_started", mapOf("sessiondid" to sessiondid, "status" to "started")) + override fun onSessionStarted(sessionid:String) { + sendEvent("recognition_started", mapOf("sessionid" to sessionid, "status" to "started")) // 使用模式感知的空闲检测启动方法 audioPlayer?.playAudio(R.raw.start) @@ -693,35 +696,35 @@ object AgentService : CoroutineScope { - override fun onSessionStopped(sessiondid:String) { - sendEvent("recognition_stopped", mapOf("sessiondid" to sessiondid, "status" to "stopped")) + override fun onSessionStopped(sessionid:String) { + sendEvent("recognition_stopped", mapOf("sessionid" to sessionid, "status" to "stopped")) _isRecognitionActive.set(false) stopIdleCheck() audioPlayer?.playAudio(R.raw.stop) } - override fun onCanceled(sessiondid: String,reason: String, errorDetails: String) { + override fun onCanceled(sessionid: String,reason: String, errorDetails: String) { _isRecognitionActive.set(false) stopIdleCheck() Log.d(TAG, "recognition_canceled:") BleService.closeCodec() sendEvent( "recognition_canceled", mapOf( - "sessiondid" to sessiondid, + "sessionid" to sessionid, "reason" to reason, "details" to errorDetails ) ) } - override fun onError(sessiondid:String, code: Int, error: String) { + override fun onError(sessionid:String, code: Int, error: String) { _isRecognitionActive.set(false) stopIdleCheck() BleService.closeCodec() Log.d(TAG, "语音识别错误: $error") sendEvent( "error", mapOf( - "sessiondid" to sessiondid, + "sessionid" to sessionid, "code" to code, "message" to error ) @@ -921,7 +924,8 @@ object AgentService : CoroutineScope { var userMetadata = JSONObject() var reply = true; //是否回复 var broadcast = true //是否播发 - var iscard = false; + var iscard = false; //是否是卡片 + var iscallingTool = true; //是否调用工具 // 如果有图片，添加图片信息到元数据 if (hasImage) { @@ -1090,21 +1094,24 @@ object AgentService : CoroutineScope { try { // audioPlayer?.playAudio(R.raw.calling, true) val name = call.getString("name") + val arguments = call.getString("arguments") sendEvent( "function_call", mapOf( "sessionid" to sessionid, "name" to name, - "arguments" to call.toString() + "arguments" to arguments ) ) if (name == "exit_interaction") { BleService.closeCodec() stopRecognition() } - - //播报工具调用 - ttsService?.speakStream("正在调用工具:${name}") - + Log.d(TAG, "liwei--------------- AI Call onFunctionCall iscallingTool:$iscallingTool callingTool:$callingTool") + if (iscallingTool) { + + ttsService?.speakStream(callingTool) + iscallingTool = false + } } catch (e: Exception) { Log.e(TAG, "liwei--------------- AI Call onFunctionCall 异常", e) } @@ -1132,6 +1139,7 @@ object AgentService : CoroutineScope { sendEvent( "function_call_result", mapOf( "sessionid" to sessionid, + "function_name" to name, "function_call" to functionCall.toString(), "result" to functionCallResult.toString(), "meta" to aiMetadata, @@ -1141,6 +1149,7 @@ object AgentService : CoroutineScope { sendEvent( "function_call_result", mapOf( "sessionid" to sessionid, + "function_name" to name, "function_call" to functionCall.toString(), "result" to functionCallResult.toString(), ) diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt index ea6e8ad14..27ecd2105 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt @@ -77,6 +77,8 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream config["usertoken"]?.toString()?.let { intent.putExtra("usertoken", it) } config["musicServiceType"]?.toString()?.let { intent.putExtra("musicServiceType", it) } config["navigationMode"]?.toString()?.let { intent.putExtra("navigationMode", it) } + config["callingTool"]?.toString()?.let { intent.putExtra("callingTool", it) } + // 添加 supportedLanguages 参数处理 config["supportedLanguages"]?.let { languages -> diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt index 9898312af..08b079a23 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt @@ -67,6 +67,7 @@ class BleMediaService : Service(), CoroutineScope { private const val KEY_SUPPORTED_LANGUAGES = "supportedLanguages" private const val KEY_MUSICSERVICE_TYPE = "musicServiceType" private const val APP_NAVIGATION_MODE = "navigationMode" + private const val APP_CALLING_TOOL = "callingTool" // 用于跟踪服务是否正在运行 private val isRunning = AtomicBoolean(false) @@ -569,6 +570,7 @@ private var companionManager: BleCompanionManager? = null it.getStringExtra(KEY_USER_TOKEN)?.let { value -> config[KEY_USER_TOKEN] = value } it.getStringExtra(KEY_MUSICSERVICE_TYPE)?.let { value -> config[KEY_MUSICSERVICE_TYPE] = value } it.getStringExtra(APP_NAVIGATION_MODE)?.let { value -> config[APP_NAVIGATION_MODE] = value } + it.getStringExtra(APP_CALLING_TOOL)?.let { value -> config[APP_CALLING_TOOL] = value } // 提取 supportedLanguages 参数 it.getStringArrayListExtra(KEY_SUPPORTED_LANGUAGES)?.let { languages -> if (languages.isNotEmpty()) { @@ -629,6 +631,7 @@ private var companionManager: BleCompanionManager? = null config[KEY_USER_TOKEN] = getConfigString(prefs, KEY_USER_TOKEN, "") config[KEY_MUSICSERVICE_TYPE] = getConfigString(prefs, KEY_MUSICSERVICE_TYPE, "") config[APP_NAVIGATION_MODE] = getConfigString(prefs, APP_NAVIGATION_MODE, "") + config[APP_CALLING_TOOL] = getConfigString(prefs, APP_CALLING_TOOL, "") // 添加 supportedLanguages 参数处理 val supportedLanguagesStr = getConfigString(prefs, KEY_SUPPORTED_LANGUAGES, "") diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index cc11359ee..a5e25b57d 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -60,6 +60,8 @@ class AgentServiceImpl: NSObject { // 会话ID，用于区分不同聊天上下文 private let agentId = "default_agent" + //工具调用提示 + public var callingTool = "" private var isInitialized: Bool = false private var isRecognizing: Bool = false @@ -162,6 +164,11 @@ class AgentServiceImpl: NSObject { // os_log("initialize: config=%{public}@", log: logger, type: .info, config) self.config = config + + if let callingTool = config["callingTool"] as? String { + self.callingTool = callingTool + } + if let serverurl = config["serverurl"] as? String { self.serverurl = serverurl } @@ -642,7 +649,7 @@ audioStream.saveAudioDataTo(data: audioData) os_log("添加用户消息", log: logger, type: .info) addToHistoryMessages(chatApiService.createUserMessage(content: displayText)) - + currsessionId = sessionid let callback = ChatApiStreamCallback( agentService: self, speakResponse: speakResponse, @@ -650,11 +657,10 @@ audioStream.saveAudioDataTo(data: audioData) hasImage: hasImage, imagePath: imagePath ) - currsessionId = sessionid - azureTtsHelper?.startspeak(sessionid: currsessionId) + azureTtsHelper?.startspeak(sessionid: sessionid) os_log("设置回调并调用sendMessageStream", log: logger, type: .info) - chatApiService.setStreamCallback(currsessionId,callback) - chatApiService.sendMessageStream(currsessionId,messages: messages) + chatApiService.setStreamCallback(sessionid,callback) + chatApiService.sendMessageStream(sessionid,messages: messages) audioPlayer?.playAwaitSound() os_log("sendMessageStream调用完成", log: logger, type: .info) } @@ -1613,6 +1619,7 @@ class ChatApiStreamCallback: StreamCallback { private var iscard = false; private var reply = true; //是否回复 private var broadcast = true //是否播发 + private var iscallingTool = true; //是否调用工具 init(agentService: AgentServiceImpl, speakResponse: Bool, displayText: String, hasImage: Bool,imagePath:String) { self.agentService = agentService @@ -1627,45 +1634,46 @@ class ChatApiStreamCallback: StreamCallback { func onUsage(_ sessionid: String,_ prompt_tokens: Int?,_ completion_tokens: Int?,_ total_tokens: Int?){ } - func onToken(_ sessionId:String,_ token: String) { + func onToken(_ sessionid:String,_ token: String) { do { -// print("liwei--------------- AI Call onToken! sessionId:\(sessionId) token:\(token)") - guard let agentService = try agentService else { return } - if (sessionId != agentService.currsessionId) { + guard let agentService = try agentService else { return } +// print("liwei--------------- AI Call onToken! sessionId:\(sessionid) currsessionId:\(agentService.currsessionId) token:\(token)") + if (sessionid != agentService.currsessionId) { return } responseBuilder += token if speakResponse && reply && broadcast{ - try agentService.azureTtsHelper?.speakStream(sessionid:sessionId,token) + try agentService.azureTtsHelper?.speakStream(sessionid:sessionid,token) // 在开始流式TTS时立即停止气泡音 } if (reply && token != "") { - try agentService.sendEvent(name: "assistant_token", data: ["token": token]) + try agentService.sendEvent(name: "assistant_token", data: ["sessionid":sessionid,"token": token]) } }catch { print("liwei--------------- AI Call onToken 异常: \(error)") } } - func onComplete(_ sessionId:String) { + func onComplete(_ sessionid:String) { do { guard let agentService = try agentService else { return } - if sessionId == agentService.currsessionId { + if sessionid == agentService.currsessionId { // 停止等待音效 agentService.audioPlayer?.stopAwaitSound() } - if speakResponse && reply && broadcast && sessionId == agentService.currsessionId{ - agentService.azureTtsHelper?.flushStream(sessionid:sessionId) + if speakResponse && reply && broadcast && sessionid == agentService.currsessionId{ + agentService.azureTtsHelper?.flushStream(sessionid:sessionid) } let response = responseBuilder var responseData: [String: Any] = [ + "sessionid":sessionid, "text": response, "userInput": displayText ] @@ -1684,8 +1692,9 @@ class ChatApiStreamCallback: StreamCallback { try agentService.addToHistoryMessages(assistantMessage) } // 保存聊天记录 - try agentService.saveChatMessage(sessionId: sessionId,userMessage: displayText, assistantMessage: response, metadata: metadata,userMetadata:toJSONString(userMetadata)) + try agentService.saveChatMessage(sessionId: sessionid,userMessage: displayText, assistantMessage: response, metadata: metadata,userMetadata:toJSONString(userMetadata)) agentService.isAiStreaming = false + iscallingTool = true os_log("设置AI流式状态为false", log: agentService.logger, type: .info) }catch{ print("liwei--------------- AI Call onComplete 异常: \(error)") @@ -1693,20 +1702,21 @@ class ChatApiStreamCallback: StreamCallback { } - func onError(_ sessionId:String,_ error: Error) { + func onError(_ sessionid:String,_ error: Error) { do { guard let agentService = try agentService else { return } - if sessionId == agentService.currsessionId { + if sessionid == agentService.currsessionId { // 停止等待音效 agentService.audioPlayer?.stopAwaitSound() } os_log("ChatAPI错误: %{public}@", log: agentService.logger, type: .error, error.localizedDescription) - if (sessionId != agentService.currsessionId) { + if (sessionid != agentService.currsessionId) { return } try agentService.sendEvent(name: "error", data: [ + "sessionid":sessionid, "code": "AI_ERROR", "message": error.localizedDescription ]) @@ -1719,27 +1729,34 @@ class ChatApiStreamCallback: StreamCallback { } - func onFunctionCall(_ sessionId:String,_ functionCall: [String: Any]) { + func onFunctionCall(_ sessionid:String,_ functionCall: [String: Any]) { do { guard let agentService = try agentService else { return } - if (sessionId != agentService.currsessionId) { + if (sessionid != agentService.currsessionId) { return } let name = functionCall["name"] as? String ?? "" - os_log("收到函数调用: %{public}@", log: agentService.logger, type: .info, name) + let arguments = functionCall["arguments"] as? String ?? "" + os_log("收到函数调用:%{public}@", log: agentService.logger, type: .info, functionCall) print("onFunctionCall=functionCall\(functionCall)") if (name == "exit_interaction") { // 关闭编码器 (设备侧) - BleService.shared.closeCodec() - agentService.stopRecognition() - } + BleService.shared.closeCodec() + agentService.stopRecognition() + } try agentService.sendEvent(name: "function_call", data: [ + "sessionid":sessionid, "name": name, - "arguments": functionCall + "arguments": arguments ]) - + + if (iscallingTool) { + os_log("收到函数调用: callingTool:%{public}@", log: agentService.logger, type: .info, agentService.callingTool) + agentService.azureTtsHelper?.speakStream(sessionid: sessionid,agentService.callingTool) + iscallingTool = false + } }catch{ @@ -1748,7 +1765,7 @@ class ChatApiStreamCallback: StreamCallback { } - func onFunctionCallResult(_ sessionId:String,_ functionCall: [String: Any], _ functionCallResult: [String: Any]) { + func onFunctionCallResult(_ sessionid:String,_ functionCall: [String: Any], _ functionCallResult: [String: Any]) { do { guard let agentService = agentService else { return } @@ -1763,6 +1780,8 @@ class ChatApiStreamCallback: StreamCallback { iscard = !(resultmap["iscard"] as? Bool ?? false) if (iscard){ agentService.sendEvent(name: "function_call_result", data: [ + "sessionid":sessionid, + "function_name": name, "function_call": functionCall, "result": functionCallResult, "meta":metadata, @@ -2066,7 +2085,7 @@ class AudioPlayer { extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { func onResult(sessionid:String ,_ text: String, _ detectedLanguage: String) { if !text.isEmpty && isKeepResult { - var data: [String: Any] = ["text": text] + var data: [String: Any] = ["sessionid":sessionid,"text": text] if !detectedLanguage.isEmpty { data["language"] = detectedLanguage } @@ -2093,7 +2112,7 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { restartIdleCheck() } - var data: [String: Any] = ["text": recognizing] + var data: [String: Any] = ["sessionid":sessionid,"text": recognizing] if !detectedLanguage.isEmpty { data["language"] = detectedLanguage } @@ -2106,7 +2125,7 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { } func onSessionStarted(sessionid:String) { - sendEvent(name: "recognition_started", data: ["status": "started"]) + sendEvent(name: "recognition_started", data: ["sessionid":sessionid,"status": "started"]) isRecognizing = true hasSpeechDetected = false // 新增：会话真正启动，复位"启动中"状态 @@ -2127,7 +2146,7 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { } func onSessionStopped(sessionid:String) { - sendEvent(name: "recognition_stopped", data: ["status": "stopped"]) + sendEvent(name: "recognition_stopped", data: ["sessionid":sessionid,"status": "stopped"]) isRecognizing = false hasSpeechDetected = false stopIdleCheck() @@ -2137,7 +2156,7 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { } func onCanceled(sessionid:String ,_ reason: String, _ errorDetails: String) { - var data: [String: Any] = [:] + var data: [String: Any] = ["sessionid":sessionid] if !reason.isEmpty { data["reason"] = reason } @@ -2154,7 +2173,7 @@ extension AgentServiceImpl: AzureAsrHelper.ContinuousRecognizeCallback { } func onError(sessionid:String ,_ errorCode: Int, _ error: String) { - let data: [String: Any] = ["message": error.isEmpty ? "未知错误" : error] + let data: [String: Any] = ["sessionid":sessionid,"message": error.isEmpty ? "未知错误" : error] sendEvent(name: "error", data: data) isRecognizing = false // 新增：出错时复位"启动中/待停止"状态 diff --git a/local_plugins/amap_service/ios/Classes/AmapViewController.swift b/local_plugins/amap_service/ios/Classes/AmapViewController.swift index c6bda6a36..c421818bf 100644 --- a/local_plugins/amap_service/ios/Classes/AmapViewController.swift +++ b/local_plugins/amap_service/ios/Classes/AmapViewController.swift @@ -56,9 +56,15 @@ class AmapViewController: UIViewController { cleanupResources() } + /** + * 清理资源，包括移除通知观察者 + */ private func cleanupResources() { print("开始清理导航资源") + // 移除通知观察者，防止内存泄漏 + NotificationCenter.default.removeObserver(self, name: NSNotification.Name("ExitNavigation"), object: nil) + // 停止语音合成 speechSynthesizer.stopSpeaking(at: .immediate) @@ -144,21 +150,36 @@ class AmapViewController: UIViewController { print("导航------------------ startNavigation_4") } + /** + * 退出导航并清理资源 + * 添加了nil检查以防止崩溃 + */ @objc private func exitNavigation() { print("导航------------------ 退出导航，清理资源") + + // 安全检查：确保driveManager存在再进行操作 + guard let manager = driveManager else { + print("⚠️ driveManager已为nil，跳过导航清理") + return + } + // 停止导航 - driveManager.stopNavi() - driveManager.removeDataRepresentative(driveView) - driveManager.delegate = nil + manager.stopNavi() - // 移除导航视图 - driveView.delegate = nil - driveView.removeFromSuperview() - driveView = nil + // 安全检查：确保driveView存在再进行操作 + if let view = driveView { + manager.removeDataRepresentative(view) + view.delegate = nil + view.removeFromSuperview() + } else { + print("⚠️ driveView已为nil，跳过视图清理") + } - // 销毁导航管理器 + // 清理管理器 + manager.delegate = nil driveManager = nil - + driveView = nil + // 关闭当前页面 - 修复：使用正确的退出方式 if let navigationController = navigationController { // 如果是在导航控制器中，使用 pop @@ -168,13 +189,19 @@ class AmapViewController: UIViewController { dismiss(animated: true, completion: nil) } } + + /** + * 添加通知监听器 + * 使用弱引用避免循环引用和对象销毁后的调用 + */ private func addNSNotification(){ NotificationCenter.default.addObserver( forName: NSNotification.Name("ExitNavigation"), object: nil, queue: .main - ) { _ in - self.exitNavigation() + ) { [weak self] _ in + // 使用弱引用避免在对象销毁后调用方法 + self?.exitNavigation() } } } diff --git a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift index 05a6424d1..6bf1caab1 100644 --- a/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift +++ b/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift @@ -1211,6 +1211,7 @@ private class AsrCallbackWrapper: AzureAsrHelper.ContinuousRecognizeCallback { func onResult(sessionid:String,_ text: String, _ detectedLanguage: String) { plugin?.sendAsrEvent([ "type": "result", + "sessionid": sessionid, "text": text, "detectedLanguage": detectedLanguage ]) @@ -1224,6 +1225,7 @@ private class AsrCallbackWrapper: AzureAsrHelper.ContinuousRecognizeCallback { func onRecognizing(sessionid:String,_ text: String, _ detectedLanguage: String) { plugin?.sendAsrEvent([ "type": "recognizing", + "sessionid": sessionid, "text": text, "detectedLanguage": detectedLanguage ]) @@ -1234,7 +1236,8 @@ private class AsrCallbackWrapper: AzureAsrHelper.ContinuousRecognizeCallback { */ func onSessionStarted(sessionid:String) { plugin?.sendAsrEvent([ - "type": "sessionStarted" + "type": "sessionStarted", + "sessionid": sessionid ]) } @@ -1243,7 +1246,8 @@ private class AsrCallbackWrapper: AzureAsrHelper.ContinuousRecognizeCallback { */ func onSessionStopped(sessionid:String) { plugin?.sendAsrEvent([ - "type": "sessionStopped" + "type": "sessionStopped", + "sessionid": sessionid ]) } @@ -1254,7 +1258,8 @@ private class AsrCallbackWrapper: AzureAsrHelper.ContinuousRecognizeCallback { */ func onCanceled(sessionid:String,_ reason: String, _ errorDetails: String) { plugin?.sendAsrEvent([ - "type": "canceled", + "type": "canceled", + "sessionid": sessionid, "reason": reason, "errorDetails": errorDetails ]) @@ -1268,6 +1273,7 @@ private class AsrCallbackWrapper: AzureAsrHelper.ContinuousRecognizeCallback { func onError(sessionid:String,_ errorCode: Int, _ error: String) { plugin?.sendAsrEvent([ "type": "error", + "sessionid": sessionid, "code": errorCode, "message": error ]) diff --git a/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift b/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift index acf2b4588..e1f0515e8 100644 --- a/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift +++ b/local_plugins/chat_storage/ios/chat_storage/Sources/chat_storage/ChatStorageHelper.swift @@ -276,7 +276,7 @@ public class ChatStorageHelper { os_log("删除特定消息语句准备失败: %{public}@", log: logger, type: .error, errmsg) } } else { - os_log("删除消息参数无效 - sessionId和messageIds都为空", log: logger, type: .error) + os_log("删除消息参数无效 - agentId和messageIds都为空", log: logger, type: .error) return false } @@ -309,7 +309,7 @@ public class ChatStorageHelper { var messages: [[String: Any]] = [] // 首先检查数据库中是否有该会话的消息 - let countQuery = "SELECT COUNT(*) FROM messages WHERE session_id = ?" + let countQuery = "SELECT COUNT(*) FROM messages WHERE agent_id = ?" var countStatement: OpaquePointer? if sqlite3_prepare_v2(db, countQuery, -1, &countStatement, nil) == SQLITE_OK { @@ -325,7 +325,7 @@ public class ChatStorageHelper { SELECT * FROM ( SELECT id, message, sender, metadata, created_at FROM messages - WHERE session_id = ? + WHERE agent_id = ? ORDER BY created_at DESC LIMIT ? ) tmp ORDER BY created_at ASC