From b7588987c3ea02a5aefbb3bb1db50b8389353742 Mon Sep 17 00:00:00 2001 From: wolfplus Date: Mon, 12 May 2025 18:13:15 +0100 Subject: [PATCH 1/4] add --- .../agent/controllers/agent_controller.dart | 63 +- .../controllers/pairing_controller.dart | 369 +++---- .../agent_service/AgentService.kt | 25 +- .../yunqiinnovation/agent_service/BleAgent.kt | 2 +- .../agent_service/BleMediaService.kt | 2 +- .../ble_service/BleCompanionManager.kt | 2 - .../yunqiinnovation/ble_service/BleService.kt | 47 +- .../ble_service/BleServicePlugin.kt | 4 + local_plugins/open_ai/README.md | 135 --- .../open_ai/android/build.gradle.kts | 71 -- .../com/yunqiinnovation/open_ai/MCPClient.kt | 258 ----- .../yunqiinnovation/open_ai/OpenAIService.kt | 988 ------------------ .../yunqiinnovation/open_ai/OpenAiPlugin.kt | 149 --- local_plugins/open_ai/lib/open_ai.dart | 184 ---- local_plugins/open_ai/pubspec.yaml | 27 - .../open_ai_service/OpenAIService.kt | 40 +- .../open_ai_service/SystemFunctionHandler.kt | 205 +++- 17 files changed, 457 insertions(+), 2114 deletions(-) delete mode 100644 local_plugins/open_ai/README.md delete mode 100644 local_plugins/open_ai/android/build.gradle.kts delete mode 100644 local_plugins/open_ai/android/src/main/kotlin/com/yunqiinnovation/open_ai/MCPClient.kt delete mode 100644 local_plugins/open_ai/android/src/main/kotlin/com/yunqiinnovation/open_ai/OpenAIService.kt delete mode 100644 local_plugins/open_ai/android/src/main/kotlin/com/yunqiinnovation/open_ai/OpenAiPlugin.kt delete mode 100644 local_plugins/open_ai/lib/open_ai.dart delete mode 100644 local_plugins/open_ai/pubspec.yaml diff --git a/lib/modules/agent/controllers/agent_controller.dart b/lib/modules/agent/controllers/agent_controller.dart index 3618ca016..39314e200 100644 --- a/lib/modules/agent/controllers/agent_controller.dart +++ b/lib/modules/agent/controllers/agent_controller.dart @@ -216,12 +216,13 @@ class AgentController extends GetxController { messages[index] = Message( isUser: true, text: text, + isRecognizing: false, timestamp: messages[index].timestamp, ); messages.refresh(); } else { // 如果没有找到识别中的消息，添加新消息 - messages.add(Message(isUser: true, text: text)); + messages.add(Message(isUser: true, text: text, isRecognizing: false)); } isProcessing.value = true; @@ -345,39 +346,35 @@ class AgentController extends GetxController { break; case AgentServiceEventType.functionCallResult: - final responseId = event.data['responseId'] ?? ''; final result = event.data['result'] ?? ''; - final functionCall = event.data['function_call'] ?? ''; - logger.i('Flutter 结束调用 mcp: $result, $functionCall'); - - if (result.isNotEmpty) { - final parsedResult = jsonDecode(result); - final meta = parsedResult["meta"] ?? {}; - final metaResult = meta is String && meta.isNotEmpty ? jsonDecode(meta) : meta; - - // logger.i('Flutter 结束调用 mcp metaResult: $metaResult'); - // 判断是否为新的回复或响应ID是否改变 - if (_isNewAssistantResponse || _currentResponseId != responseId) { - // 添加新消息 - messages.add(Message(isUser: false, text: "", meta: metaResult)); - _isNewAssistantResponse = false; - _currentResponseId = responseId; - } else { - // 更新最后一条消息 - final lastIndex = messages.length - 1; - if (lastIndex >= 0 && !messages[lastIndex].isUser) { - messages[lastIndex] = Message( - isUser: false, - text: messages[lastIndex].text, - meta: meta, - timestamp: messages[lastIndex].timestamp, - ); - } - } - // 滚动到底部 - _scrollToBottom(); - } - logger.i('Flutter 结束调用 mcp end'); + final parsedResult = result.isNotEmpty ? jsonDecode(result) : {}; + final meta = parsedResult["meta"] ?? {}; + // final functionCall = event.data['function_call'] ?? ''; + logger.d('mcp 执行结果meta: $meta'); + + // if (result.isNotEmpty) { + // // logger.i('Flutter 结束调用 mcp metaResult: $metaResult'); + // // 判断是否为新的回复或响应ID是否改变 + // if (_isNewAssistantResponse ) { + // // 添加新消息 + // messages.add(Message(isUser: false, text: "", meta: meta)); + // _isNewAssistantResponse = false; + // } else { + // // 更新最后一条消息 + // final lastIndex = messages.length - 1; + // if (lastIndex >= 0 && !messages[lastIndex].isUser) { + // messages[lastIndex] = Message( + // isUser: false, + // text: messages[lastIndex].text, + // meta: meta, + // timestamp: messages[lastIndex].timestamp, + // ); + // } + // } + + // _scrollToBottom(); + // } + logger.i('mcp 结果处理完成'); break; default: break; diff --git a/lib/modules/pairing/controllers/pairing_controller.dart b/lib/modules/pairing/controllers/pairing_controller.dart index 49dae4a80..88a349161 100644 --- a/lib/modules/pairing/controllers/pairing_controller.dart +++ b/lib/modules/pairing/controllers/pairing_controller.dart @@ -8,15 +8,12 @@ import 'dart:io'; import '../../../data/services/bluetooth_manager.dart'; import 'package:classic_bluetooth/classic_bluetooth.dart' show BluetoothDevice; import '../../../data/services/ble_manager.dart'; -import 'package:agent_service/agent_service.dart'; import 'package:ble_service/ble_service.dart'; /// 配对控制器，负责检查蓝牙权限、状态，并处理配对流程 class PairingController extends GetxController { static PairingController get to => Get.find(); - //是否需要连接耳机 - final isneedearphone = false; // 状态标志 final isLoading = false.obs; @@ -31,6 +28,9 @@ class PairingController extends GetxController { // 状态描述（用于UI显示） final statusDescription = 'initializing'.tr.obs; + + // 按钮文本 + final actionButtonText = '去系统设置'.obs; // 是否正在跳转到主页 bool _isNavigatingToHome = false; @@ -38,6 +38,10 @@ class PairingController extends GetxController { // 是否正在执行Companion关联流程 bool _isAssociating = false; + // 是否用户拒绝了关联 + final _userRejectedAssociation = false.obs; + bool get userRejectedAssociation => _userRejectedAssociation.value; + // 当前连接设备信息 BluetoothDevice? get currentDevice => _bluetoothService.currentDevice; @@ -80,28 +84,15 @@ class PairingController extends GetxController { // 检查并请求权限 if (!await _checkAndRequestPermissions()) { statusDescription.value = 'permissionDenied'.tr; + actionButtonText.value = '去系统设置'; return; } // 设置服务状态监听 - _setupServiceStateListeners(); - - // 检查蓝牙状态 - statusDescription.value = 'checkingBluetoothStatus'.tr; - final isEnabled = await _bluetoothService.isBluetoothEnabled(); - _bluetoothState.value = isEnabled ? 'on' : 'off'; - - if (!isEnabled) { - statusDescription.value = 'bluetoothTurnedOff'.tr; - return; - } - - // 刷新已连接设备 - statusDescription.value = 'searchingDevice'.tr; - await refreshConnectedDevices(); + _setupServiceListeners(); - // 检查是否已存在Companion设备关联，但不要重复调用关联流程 - await _checkExistingCompanionAssociation(); + // 检查蓝牙状态并刷新设备 + await _checkBluetoothAndRefresh(); } catch (e) { Logger.error('初始化配对控制器失败: $e'); statusDescription.value = 'initializationFailure'.tr; @@ -111,46 +102,89 @@ class PairingController extends GetxController { } } - /// 检查是否已存在Companion设备关联 - Future _checkExistingCompanionAssociation() async { - // 只在Android平台执行 - if (!Platform.isAndroid) return; + /// 检查蓝牙状态并刷新设备 + Future _checkBluetoothAndRefresh() async { + statusDescription.value = 'checkingBluetoothStatus'.tr; + final isEnabled = await _bluetoothService.isBluetoothEnabled(); + _bluetoothState.value = isEnabled ? 'on' : 'off'; + + if (!isEnabled) { + statusDescription.value = 'bluetoothTurnedOff'.tr; + actionButtonText.value = '去系统设置'; + return; + } + + // 刷新连接设备并检查配对状态 + statusDescription.value = 'searchingDevice'.tr; + await _bluetoothService.refreshDeviceConnection(); + + // 设备连接时检查配对状态 + if (isDeviceConnected) { + await _checkDevicePairingStatus(); + } + } + + /// 检查设备配对状态 + Future _checkDevicePairingStatus() async { + // 只在Android平台且设备已连接时执行 + if (!Platform.isAndroid || !isDeviceConnected) return; try { final isSupported = await _bleService.isCompanionSupported(); - if (!isSupported) return; + if (!isSupported) { + Logger.info('设备不支持Companion功能'); + _companionAssociated.value = true; // 不支持则视为已配对 + _userRejectedAssociation.value = false; // 重置拒绝状态 + return; + } final pairedMac = await _bleService.getPairedMacAddress(); - if (pairedMac != null) { - Logger.info('发现已关联的Companion设备: $pairedMac'); - - // 检查当前连接设备与已保存的设备是否一致 - if (currentDevice!.address != pairedMac) { - // MAC地址不匹配 - Logger.warning('当前连接的设备与已关联的DeepSound耳机'); - statusDescription.value = 'pleaseConnectPairedDevice'.tr; - return; - } - - _companionAssociated.value = true; - await _bleService.registerBackgroundScan(); + if (pairedMac == null) { + // 没有配对记录，需要开始配对 + Logger.info('无配对记录，需要开始设备配对'); + return; } + + Logger.info('发现已关联的Companion设备: $pairedMac'); + + // 检查当前连接设备与已保存的设备是否一致 + if (currentDevice!.address != pairedMac) { + // MAC地址不匹配 + Logger.warning('当前连接的设备与已配对的DeepSound设备不符'); + statusDescription.value = 'pleaseConnectPairedDevice'.tr; + return; + } + + // 判断设备名称是否符合要求 + if (!_isValidDeviceName(currentDevice!.name)) { + statusDescription.value = 'pleaseConnectDeepSoundDevice'.tr; + return; + } + + // 设备匹配且合规，标记为已配对 + Logger.info('当前设备与已配对设备匹配'); + _companionAssociated.value = true; + _userRejectedAssociation.value = false; // 重置拒绝状态 + await _bleService.registerBackgroundScan(); + _autoNavigateToHome(); } catch (e) { - Logger.error('检查现有Companion关联失败: $e'); + Logger.error('检查设备配对状态失败: $e'); } } + /// 检查设备名称是否合规 + bool _isValidDeviceName(String deviceName) { + return deviceName.contains("DeapSound"); + } + /// 检查并请求必要的蓝牙权限 Future _checkAndRequestPermissions() async { Map statuses; List permissionsToRequest = []; if (Platform.isAndroid) { - // Android 12+ (SDK 31+) 需要 SCAN 和 CONNECT - // Android 6+ (SDK 23+) 需要 LOCATION - permissionsToRequest.add(Permission.locationWhenInUse); // 基本定位权限 + permissionsToRequest.add(Permission.locationWhenInUse); permissionsToRequest.add(Permission.microphone); - // Android 12及以上版本才需要请求蓝牙扫描和连接权限 if (await _getAndroidSdkVersion() >= 31) { permissionsToRequest.add(Permission.bluetoothScan); permissionsToRequest.add(Permission.bluetoothConnect); @@ -176,73 +210,68 @@ class PairingController extends GetxController { // 辅助方法获取Android SDK版本 Future _getAndroidSdkVersion() async { if (Platform.isAndroid) { - // 这里需要原生代码支持来获取确切的SDK版本， - // 或者使用 device_info_plus 插件 - // 暂时返回一个较高值以触发新权限请求 return 31; // 假设运行在需要新权限的设备上 } return 0; } /// 设置服务状态监听 - void _setupServiceStateListeners() { - // 监听蓝牙服务的蓝牙状态变化 + void _setupServiceListeners() { + // 监听蓝牙状态变化 _bluetoothStateWorker = ever(_bluetoothService.bluetoothState.obs, (String state) { _bluetoothState.value = state; _updateStatusDescription(); - - // 如果蓝牙关闭，可能需要更新UI或提示用户 - if (state == 'off') { - Logger.info('蓝牙已关闭'); - } }); - // 监听当前设备变化 - 使用公开的currentDeviceRx字段 + + // 监听设备连接状态变化 _deviceWorker = ever(_bluetoothService.currentDeviceRx, (BluetoothDevice? device) { if (device != null && !_isNavigatingToHome) { Logger.info('配对页面发现设备已连接: ${device.name}'); - // 当经典蓝牙设备连接成功，尝试进行Companion关联 _handleDeviceConnected(); } _updateStatusDescription(); }); - if (!isneedearphone) { - //不需要耳机 - _handleDeviceConnected(); - } } /// 处理设备连接成功事件 Future _handleDeviceConnected() async { - if ((!isDeviceConnected && isneedearphone) || _isNavigatingToHome) return; - // 如果已经完成关联，直接跳转 - if (isneedearphone && _companionAssociated.value) { - _autoNavigateToHome(); - return; + if (!isDeviceConnected || _isNavigatingToHome) return; + + // 检查配对状态，决定下一步操作 + await _checkDevicePairingStatus(); + + // 如果还未配对且设备合规，启动配对流程 + if (!_companionAssociated.value && _isValidDeviceName(currentDevice!.name)) { + _startCompanionAssociation(); } - // 启动关联流程 - _startCompanionAssociation(); } /// 更新状态描述 void _updateStatusDescription() { + if (_userRejectedAssociation.value) { + // 用户拒绝关联时的提示 - 这是最高优先级 + statusDescription.value = '关联后才能使用全部功能'; + actionButtonText.value = '关联DeapSound耳机'; + return; + } + if (!hasPermissions) { statusDescription.value = 'permissionDenied'.tr; + actionButtonText.value = '去系统设置'; return; } if (bluetoothState == 'off') { statusDescription.value = 'bluetoothTurnedOff'.tr; + actionButtonText.value = '去系统设置'; return; } if (isDeviceConnected) { if (Platform.isAndroid && !isCompanionAssociated && _isAssociating) { - statusDescription.value = - '${'associatingDevice'.tr} ${currentDevice!.name}'; - } else if (isCompanionAssociated) { - statusDescription.value = '${'connectedDevice'.tr} ${currentDevice!.name}'; + statusDescription.value = '${'associatingDevice'.tr} ${currentDevice!.name}'; } else { statusDescription.value = '${'connectedDevice'.tr} ${currentDevice!.name}'; } @@ -257,150 +286,85 @@ class PairingController extends GetxController { statusDescription.value = 'notFound'.tr; } - /// 刷新已连接设备 - Future refreshConnectedDevices() async { - if (!hasPermissions) { - statusDescription.value = 'permissionDenied'.tr; - return; - } - - try { - isLoading.value = true; - statusDescription.value = 'searchingDevice'.tr; - - // 使用蓝牙服务刷新连接 - await _bluetoothService.refreshDeviceConnection(); - - // 检查连接状态 - if (isDeviceConnected) { - Logger.info('刷新发现设备已连接: ${currentDevice!.name}'); - - // 当发现设备已连接，统一由_handleDeviceConnected处理 - _handleDeviceConnected(); - } else { - Logger.info('刷新未发现已连接设备'); - } - } catch (e) { - Logger.error('刷新设备失败: $e'); - statusDescription.value = 'failedRefreshDevice'.tr; - } finally { - isLoading.value = false; - _updateStatusDescription(); - } - } /// 启动Companion设备关联流程 Future _startCompanionAssociation() async { - // 如果不是Android平台，已经跳转到首页，已经完成关联或正在进行关联，直接返回 - if (!Platform.isAndroid || - _isNavigatingToHome || - _companionAssociated.value || - _isAssociating) { + // 如果不是Android平台，已跳转，已完成关联或正在进行关联，直接返回 + if (!Platform.isAndroid || + _isNavigatingToHome || + _companionAssociated.value || + _isAssociating || + !isDeviceConnected) { return; } + + // 重置拒绝状态 + _userRejectedAssociation.value = false; + // 设置关联状态标志 _isAssociating = true; _updateStatusDescription(); try { - // 检查是否支持Companion Device功能 + // 检查是否支持Companion功能 final isSupported = await _bleService.isCompanionSupported(); if (!isSupported) { - Logger.info('设备不支持Companion Device功能，跳过关联'); - _companionAssociated.value = true; // 标记为已完成 + Logger.info('设备不支持Companion功能，跳过关联'); + _companionAssociated.value = true; _autoNavigateToHome(); return; } - if (!currentDevice!.name.contains("DeapSound")) { - // 设备名称不包含DeepSound - statusDescription.value = 'pleaseConnectDeepSoundDevice'.tr; - // 不清除关联，因为MAC地址匹配 - return; - } - // 检查是否已有配对的MAC地址 - final pairedMac = await _bleService.getPairedMacAddress(); - if (pairedMac != null) { - // 已有关联的设备，检查是否与当前连接的设备一致 - if (currentDevice != null && currentDevice!.address == pairedMac) { - // MAC地址匹配，再检查设备名称 - - - Logger.info('已关联的Companion设备与当前连接的设备一致'); - _companionAssociated.value = true; - await _bleService.registerBackgroundScan(); - // _autoNavigateToHome(); - // return; - } else { - // MAC地址不匹配 - Logger.warning('未找到已关联的DeepSound耳机（MAC地址不匹配）'); - statusDescription.value = 'pleaseConnectPairedDevice'.tr; - // 清除旧关联以避免冲突 - await _bleService.clearAssociations(); - return; - } + // 检查设备名称是否符合要求 + if (!_isValidDeviceName(currentDevice!.name)) { + statusDescription.value = 'pleaseConnectDeepSoundDevice'.tr; + _isAssociating = false; + return; } - // 开始关联流程 - if (currentDevice != null) { - Logger.info('开始Companion设备关联流程, 设备MAC: ${currentDevice!.address}'); - - var success = await _bleService.startDeviceAssociation( - deviceNamePattern: currentDevice!.name, - macAddress: currentDevice!.address, - ); - if (success) { - Logger.info('Companion设备关联成功'); - _companionAssociated.value = true; - - // 注册后台扫描，确保应用被杀死后仍能被唤醒 - await _bleService.registerBackgroundScan( - macAddress: currentDevice!.address); - // 启动BLE后台服务（可选） - await _startAgentService(); - - // 关联和后台扫描都成功后，再跳转 - _autoNavigateToHome(); - } else { - Logger.error('Companion设备关联失败'); - // 关联失败，但仍然可以跳转到主页（用户体验考虑） - _autoNavigateToHome(); - } - } else if (!isneedearphone) { - // 启动BLE后台服务（可选） - await _startAgentService(); - - // 关联和后台扫描都成功后，再跳转 + + Logger.info('开始Companion设备关联流程, 设备MAC: ${currentDevice!.address}'); + + var success = await _bleService.startDeviceAssociation( + deviceNamePattern: currentDevice!.name, + macAddress: currentDevice!.address, + ); + _isAssociating = false; + + if (success) { + Logger.info('Companion设备关联成功'); + _companionAssociated.value = true; + _userRejectedAssociation.value = false; + + // 注册后台扫描，确保应用被杀死后仍能被唤醒 + await _bleService.registerBackgroundScan( + macAddress: currentDevice!.address); + + + // 关联成功后跳转 _autoNavigateToHome(); + } else { + Logger.error('Companion设备关联失败或被用户拒绝'); + // 关联失败 - 用户拒绝关联 + _userRejectedAssociation.value = true; + // 确保UI状态更新 + _updateStatusDescription(); } } catch (e) { Logger.error('Companion设备关联过程发生错误: $e'); - // 出错时仍可以跳转主页 - _autoNavigateToHome(); - } finally { - // 无论成功失败，都重置关联状态标志 _isAssociating = false; + _userRejectedAssociation.value = true; _updateStatusDescription(); } } /// 自动跳转到主页 Future _autoNavigateToHome() async { - if (_isNavigatingToHome) return; - - // 如果没有连接，不进行跳转 - if (!isDeviceConnected) return; + if (_isNavigatingToHome || !isDeviceConnected) return; - // 在Android平台上，如果支持Companion且仍在关联中，不跳转 - if (Platform.isAndroid) { - final isSupported = await _bleService.isCompanionSupported(); - if (isSupported && !isCompanionAssociated && _isAssociating) { - Logger.info('等待Companion设备关联完成后再跳转'); - return; - } - } + // 启动BLE后台服务 + await _startAgentService(); _isNavigatingToHome = true; - Logger.info('已找到连接的设备 ${currentDevice!.name}，自动跳转到主页'); Future.delayed(const Duration(milliseconds: 500), () { @@ -411,16 +375,10 @@ class PairingController extends GetxController { /// 启动BLE后台服务 Future _startAgentService() async { try { - if (isneedearphone && !isDeviceConnected) return; + if (!isDeviceConnected) return; - // 启动BLE服务，无需传递配置参数，将从.env读取 final result = await BleManager.to.startAgentService(); - - if (result) { - Logger.info('BLE后台服务启动成功'); - } else { - Logger.error('BLE后台服务启动失败'); - } + Logger.info(result ? 'BLE后台服务启动成功' : 'BLE后台服务启动失败'); } catch (e) { Logger.error('启动BLE后台服务发生错误: $e'); } @@ -430,17 +388,23 @@ class PairingController extends GetxController { Future _stopAgentService() async { try { final result = await BleManager.to.stopAgentService(); - - if (result) { - Logger.info('BLE后台服务停止成功'); - } else { - Logger.error('BLE后台服务停止失败'); - } + Logger.info(result ? 'BLE后台服务停止成功' : 'BLE后台服务停止失败'); } catch (e) { Logger.error('停止BLE后台服务发生错误: $e'); } } + /// 处理按钮点击 + void handleActionButtonTap() { + if (userRejectedAssociation) { + // 用户拒绝关联后，点击按钮重新开始关联 + restartAssociation(); + } else if (!hasPermissions || !isBluetoothOn) { + // 权限未授予或蓝牙关闭时，打开系统设置 + openBluetoothSettings(); + } + } + /// 打开系统蓝牙设置 void openBluetoothSettings() { try { @@ -462,9 +426,18 @@ class PairingController extends GetxController { Future openAppSettings() async { await AppSettings.openAppSettings(); } + + /// 重新开始关联 + void restartAssociation() { + if (isDeviceConnected && !isCompanionAssociated) { + _startCompanionAssociation(); + } + } /// 跳过连接 - void skipConnecting() { + void skipConnecting() async{ + // 启动BLE后台服务 + await _startAgentService(); _isNavigatingToHome = true; Get.offAllNamed(Routes.home); } diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index 451262cf4..b09656e9d 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -16,6 +16,8 @@ import kotlin.coroutines.CoroutineContext import com.yunqiinnovation.ble_service.BleService import android.media.MediaPlayer import com.yunqiinnovation.agent_service.R +import android.os.Handler +import android.os.Looper /** * 代理服务事件监听接口 */ @@ -682,12 +684,12 @@ object AgentService : CoroutineScope { "function_call" to functionCall.toString(), "result" to functionCallResult.toString(), )) - FileLogger.d(TAG, "mcp调用结果: $functionCallResult") + // FileLogger.d(TAG, "mcp调用结果: $functionCallResult") val metaStr = functionCallResult.optString("meta") if (metaStr.isNotEmpty()) { metadata = metaStr; val meta = JSONObject(metaStr) - FileLogger.d(TAG, "mcp调用结果: $metaStr") + // FileLogger.d(TAG, "mcp调用结果: $metaStr") if (meta.has("card_music")) { //音乐卡片 val cardMusic = meta.getJSONObject("card_music") FileLogger.d(TAG, "检查到音乐卡片: $cardMusic") @@ -888,10 +890,20 @@ object AgentService : CoroutineScope { * 发送事件 */ private fun sendEvent(eventName: String, data: Map) { - launch { - withContext(Dispatchers.Main) { - // 向所有监听器发送事件 - synchronized(listeners) { + // 检查当前是否在主线程 + if (Looper.myLooper() == Looper.getMainLooper()) { + // 已在主线程，直接发送 + listeners.forEach { listener -> + try { + listener.onEvent(eventName, data) + } catch (e: Exception) { + FileLogger.e(TAG, "向监听器发送事件失败", e) + } + } + + } else { + // 不在主线程，切换到主线程 + Handler(Looper.getMainLooper()).post { listeners.forEach { listener -> try { listener.onEvent(eventName, data) @@ -899,7 +911,6 @@ object AgentService : CoroutineScope { FileLogger.e(TAG, "向监听器发送事件失败", e) } } - } } } } diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt index f362b478d..23d39b554 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt @@ -142,7 +142,7 @@ object BleAgent : BleService.Callback, AgentServiceListener { // 停止当前TTS，避免冲突 AgentService.interruptCurrentResponse() - AgentService.speakText("我在!") + // AgentService.speakText("我在!") // 启动语音识别 if (AgentService.isRecognitionActive) { // 如果已经在识别，则重新启动 diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt index 83c46a4db..ea9b0fede 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt @@ -328,7 +328,7 @@ class BleMediaService : Service(), CoroutineScope { private fun checkBleConnection() { - FileLogger.i(TAG, "hcheckBleConnection收到连接请求") + FileLogger.i(TAG, "checkBleConnection收到连接请求") try { // 示例：使用协程异步处理连接 diff --git a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCompanionManager.kt b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCompanionManager.kt index 4b6f8da56..93ad52422 100644 --- a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCompanionManager.kt +++ b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCompanionManager.kt @@ -327,8 +327,6 @@ class BleCompanionManager(private val context: Context) { // 清除临时MAC地址 pendingMacAddress = null - // 关联成功后，自动注册后台扫描 - registerBackgroundScan() return true } else { diff --git a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt index dd6d24917..602b9c4ed 100644 --- a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt +++ b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt @@ -123,13 +123,13 @@ object BleService { // 初始化OpusManager try { opusManager = OpusManager() + startOpusStreamDecoding() FileLogger.d(TAG, "OpusManager初始化成功") } catch (e: OpusException) { FileLogger.e(TAG, "OpusManager初始化失败: ${e.message}", e) // 根据需要决定是否因为Opus初始化失败而返回false } - isInitialized = true FileLogger.d(TAG, "BLE服务初始化成功") return true @@ -1014,7 +1014,6 @@ object BleService { fun closeCodec(): Boolean { FileLogger.i(TAG, "关闭编解码...") - stopOpusStreamDecoding() // 停止Opus解码流 return sendCommand( BleConst.CMD_CONTROL_CODEC.toByte(), byteArrayOf( @@ -1031,15 +1030,15 @@ object BleService { fun openDecoder(): Boolean { FileLogger.i(TAG, "打开编码 (设备侧)...") - val opusStarted = startOpusStreamDecoding( - hasHeader = false, // BLE传输的Opus数据通常无文件头 - channel = 1, // 单声道 - sampleRate = 16000,// Opus推荐16kHz或更高 - packetSize = 40 // Opus常用帧长，如20ms, 40ms, 60ms。 (16000Hz * 1channel * 2bytes/sample * 0.04s = 1280 bytes/frame if PCM) - // packetSize for Opus is typically in ms, e.g. 20, 40, 60. - // The library might expect this in a different unit or derive it. - // Given JlOpusPlugin.kt, packetSize refers to Opus encoded frame duration in ms. - ) + // val opusStarted = startOpusStreamDecoding( + // hasHeader = false, // BLE传输的Opus数据通常无文件头 + // channel = 1, // 单声道 + // sampleRate = 16000,// Opus推荐16kHz或更高 + // packetSize = 40 // Opus常用帧长，如20ms, 40ms, 60ms。 (16000Hz * 1channel * 2bytes/sample * 0.04s = 1280 bytes/frame if PCM) + // // packetSize for Opus is typically in ms, e.g. 20, 40, 60. + // // The library might expect this in a different unit or derive it. + // // Given JlOpusPlugin.kt, packetSize refers to Opus encoded frame duration in ms. + // ) FileLogger.i(TAG, "打开解码...") @@ -1058,18 +1057,7 @@ object BleService { fun openA2DPDecoder(): Boolean { FileLogger.i(TAG, "打开编码 (设备侧)...") - val opusStarted = startOpusStreamDecoding( - hasHeader = false, // BLE传输的Opus数据通常无文件头 - channel = 1, // 单声道 - sampleRate = 16000,// Opus推荐16kHz或更高 - packetSize = 40 // Opus常用帧长，如20ms, 40ms, 60ms。 (16000Hz * 1channel * 2bytes/sample * 0.04s = 1280 bytes/frame if PCM) - // packetSize for Opus is typically in ms, e.g. 20, 40, 60. - // The library might expect this in a different unit or derive it. - // Given JlOpusPlugin.kt, packetSize refers to Opus encoded frame duration in ms. - ) - - - FileLogger.i(TAG, "打开解码...") + // FileLogger.i(TAG, "打开解码...") return sendCommand( BleConst.CMD_CONTROL_CODEC.toByte(), byteArrayOf( BleConst.CODEC_CONTROL_A2DP_PLAY.toByte(), @@ -1093,20 +1081,7 @@ object BleService { fun openEncoder(): Boolean { FileLogger.i(TAG, "打开编码 (设备侧)...") - val opusStarted = startOpusStreamDecoding( - hasHeader = false, // BLE传输的Opus数据通常无文件头 - channel = 1, // 单声道 - sampleRate = 16000,// Opus推荐16kHz或更高 - packetSize = 40 // Opus常用帧长，如20ms, 40ms, 60ms。 (16000Hz * 1channel * 2bytes/sample * 0.04s = 1280 bytes/frame if PCM) - // packetSize for Opus is typically in ms, e.g. 20, 40, 60. - // The library might expect this in a different unit or derive it. - // Given JlOpusPlugin.kt, packetSize refers to Opus encoded frame duration in ms. - ) - - // 注意：此方法通常是控制设备端开始编码并发送数据。 - // App侧通常是打开解码器(openDecoder)来接收和处理这些数据。 - // 如果App也需要编码（例如发送语音到设备），则需要实现Opus编码逻辑。 // 目前仅发送命令通知设备开始编码。 return sendCommand( BleConst.CMD_CONTROL_CODEC.toByte(), diff --git a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleServicePlugin.kt b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleServicePlugin.kt index e8ba005df..f0ab399fb 100644 --- a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleServicePlugin.kt +++ b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleServicePlugin.kt @@ -132,6 +132,10 @@ class BleServicePlugin : FlutterPlugin, MethodCallHandler, ActivityAware, pendingAssociationResult?.success(true) pendingAssociationResult = null return true + } else { + pendingAssociationResult?.success(false) + pendingAssociationResult = null + return true } } return false diff --git a/local_plugins/open_ai/README.md b/local_plugins/open_ai/README.md deleted file mode 100644 index f4b4b9223..000000000 --- a/local_plugins/open_ai/README.md +++ /dev/null @@ -1,135 +0,0 @@ -# OpenAI API 与 MCP 集成插件 - -这个 Flutter 插件提供了 OpenAI API 的访问能力和 Model Context Protocol (MCP) 工具调用功能的集成。 - -## 功能特点 - -- 使用官方 OpenAI Java SDK 进行异步通信 -- 集成官方 MCP Kotlin SDK，使用 SSE 模式 -- 支持流式输出响应 -- 支持工具调用和结果处理 -- 支持带图片的多模态对话 - -## MCP 集成 - -本插件使用 [MCP 官方 Kotlin SDK](https://github.com/modelcontextprotocol/kotlin-sdk) 实现与 MCP 服务器的通信。通过 SSE（Server-Sent Events）模式连接，能够： - -- 获取 MCP 服务器提供的所有工具定义 -- 动态调用远程工具并获取结果 -- 支持本地工具的注册和调用 -- 在 OpenAI API 请求中无缝集成工具调用功能 - -## 安装 - -在项目的 `pubspec.yaml` 中添加本地插件依赖： - -```yaml -dependencies: - open_ai: - path: local_plugins/open_ai -``` - -## 使用方法 - -### 初始化 - -```dart -import 'package:open_ai/open_ai.dart'; - -final openAI = OpenAI(); - -await openAI.initialize( - apiKey: 'your-api-key', - baseUrl: 'https://api.example.com', // 可选，默认为 OpenAI 官方 API - model: 'gpt-3.5-turbo', // 可选，默认为 gpt-3.5-turbo - mcpServer: 'https://mcp.example.com', // MCP 服务器 SSE 端点地址 -); -``` - -### 创建消息 - -```dart -// 创建系统消息 -final systemMessage = await openAI.createSystemMessage('你是一个助手'); - -// 创建用户消息 -final userMessage = await openAI.createUserMessage('你好，请帮我解释一下量子力学'); - -// 创建助手消息 -final assistantMessage = await openAI.createAssistantMessage('我可以帮你解释量子力学'); - -// 创建带图片的用户消息 -final imageBase64 = '...'; // base64编码的图片数据 -final userImageMessage = await openAI.createUserMessageWithImage( - '这张图片中的物体是什么？', - imageBase64 -); -``` - -### 发送消息（非流式输出） - -```dart -final messages = [systemMessage, userMessage]; -final response = await openAI.sendMessage(messages); -print('AI回复: $response'); -``` - -### 发送消息（流式输出） - -```dart -final messages = [systemMessage, userMessage]; -final callback = StreamCallback( - onToken: (token) { - // 处理单个令牌 - print('收到令牌: $token'); - }, - onComplete: () { - // 处理完成事件 - print('响应完成'); - }, - onError: (error) { - // 处理错误 - print('发生错误: $error'); - }, - onFunctionCall: (functionCall) { - // 处理函数调用 - print('函数调用: $functionCall'); - }, - onFunctionCallResult: (functionCall, result) { - // 处理函数调用结果 - print('函数调用结果: $result'); - }, -); - -final streamId = await openAI.sendMessageStream(messages, callback); -``` - -### 取消当前流式请求 - -```dart -final success = await openAI.cancelCurrentStream(); -``` - -### 释放资源 - -```dart -await openAI.dispose(); -``` - -## 异常处理 - -该插件会在操作失败时抛出异常，请使用 try-catch 块捕获它们： - -```dart -try { - final response = await openAI.sendMessage(messages); -} catch (e) { - print('发生错误: $e'); -} -``` - -## 注意事项 - -- 初始化插件时必须提供有效的 API 密钥 -- 使用流式响应时，请确保在完成后调用 `dispose()` 方法释放资源 -- MCP 功能需要有效的 MCP 服务器 SSE 端点地址才能工作 \ No newline at end of file diff --git a/local_plugins/open_ai/android/build.gradle.kts b/local_plugins/open_ai/android/build.gradle.kts deleted file mode 100644 index f950e0cac..000000000 --- a/local_plugins/open_ai/android/build.gradle.kts +++ /dev/null @@ -1,71 +0,0 @@ -group = "com.yunqiinnovation.open_ai" -version = "1.0" - -buildscript { - val kotlinVersion by extra("1.8.0") - repositories { - google() - mavenCentral() - } - - dependencies { - classpath("com.android.tools.build:gradle:7.3.0") - classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion") - } -} - -allprojects { - repositories { - google() - mavenCentral() - } -} - -plugins { - id("com.android.library") - id("kotlin-android") -} - -android { - compileSdk = 33 - - compileOptions { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 - } - - kotlinOptions { - jvmTarget = "1.8" - } - - defaultConfig { - minSdk = 21 - } - - namespace = "com.yunqiinnovation.open_ai" -} - -dependencies { - val kotlinVersion: String by project - - implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlinVersion") - implementation("androidx.annotation:annotation:1.6.0") - implementation("androidx.appcompat:appcompat:1.6.1") - - // Kotlin协程 - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.7.1") - - // OkHttp 依赖项 - implementation("com.squareup.okhttp3:okhttp:4.11.0") - implementation("com.squareup.okhttp3:okhttp-sse:4.11.0") - - // Jackson JSON解析器 - implementation("com.fasterxml.jackson.core:jackson-databind:2.14.2") - - // OpenAI Java库依赖 - implementation("com.aallam.openai:openai-client:3.6.0") - implementation("io.ktor:ktor-client-okhttp:2.3.3") - - // MCP官方Kotlin SDK依赖 - implementation("io.modelcontextprotocol:kotlin-sdk:0.5.0") -} \ No newline at end of file diff --git a/local_plugins/open_ai/android/src/main/kotlin/com/yunqiinnovation/open_ai/MCPClient.kt b/local_plugins/open_ai/android/src/main/kotlin/com/yunqiinnovation/open_ai/MCPClient.kt deleted file mode 100644 index 62f197b31..000000000 --- a/local_plugins/open_ai/android/src/main/kotlin/com/yunqiinnovation/open_ai/MCPClient.kt +++ /dev/null @@ -1,258 +0,0 @@ -package com.yunqiinnovation.open_ai - -import android.content.Context -import android.util.Log -import com.fasterxml.jackson.databind.ObjectMapper -import io.modelcontextprotocol.kotlin.sdk.Implementation -import io.modelcontextprotocol.kotlin.sdk.client.Client -import io.modelcontextprotocol.kotlin.sdk.client.SseClientTransport -import io.modelcontextprotocol.kotlin.sdk.tools.ToolCallRequest -import io.modelcontextprotocol.kotlin.sdk.tools.ToolDefinition -import kotlinx.coroutines.* -import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.channels.awaitClose -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.callbackFlow -import okhttp3.* -import okhttp3.sse.EventSource -import okhttp3.sse.EventSourceListener -import okhttp3.sse.EventSources -import org.json.JSONArray -import org.json.JSONObject -import java.io.IOException -import java.util.concurrent.TimeUnit -import kotlin.coroutines.CoroutineContext - -/** - * MCP功能处理接口 - */ -interface FunctionHandler { - suspend fun handle(arguments: Map): String -} - -/** - * Model Context Protocol (MCP)客户端实现类 - * 使用官方 MCP Kotlin SDK - */ -class MCPClient(private val context: Context? = null) : CoroutineScope { - private val TAG = "MCPClient" - - // 协程相关 - private val job = SupervisorJob() - override val coroutineContext: CoroutineContext - get() = Dispatchers.IO + job - - // OkHttp客户端 - private val httpClient = OkHttpClient.Builder() - .connectTimeout(30, TimeUnit.SECONDS) - .readTimeout(30, TimeUnit.SECONDS) - .writeTimeout(30, TimeUnit.SECONDS) - .build() - - // MCP客户端 - private var client: Client? = null - - // 已注册的工具 - private val tools = mutableListOf() - - // 本地函数处理器 - private val functionHandlers = mutableMapOf() - - // Json解析器 - private val objectMapper = ObjectMapper() - - // 连接状态 - private var isConnected = false - - /** - * 使用SSE方式连接到MCP服务器 - */ - suspend fun connectToSSE(serverUrl: String): Boolean { - if (serverUrl.isEmpty()) { - Log.e(TAG, "服务器URL为空") - return false - } - - try { - // 创建MCP客户端 - client = Client( - clientInfo = Implementation( - name = "deepvoice-mcp-client", - version = "1.0.0" - ) - ) - - // 创建SSE传输 - val transport = SseClientTransport( - serverUrl = serverUrl, - httpClient = httpClient - ) - - // 连接到服务器 - client?.connect(transport) - isConnected = true - - // 获取工具定义 - fetchTools() - - return true - } catch (e: Exception) { - Log.e(TAG, "连接到MCP服务器失败: ${e.message}", e) - isConnected = false - return false - } - } - - /** - * 获取工具定义 - */ - private suspend fun fetchTools() { - try { - val availableTools = client?.listTools() ?: emptyList() - tools.clear() - - availableTools.forEach { toolDefinition -> - val toolJson = toolDefinitionToJson(toolDefinition) - tools.add(toolJson.toString()) - Log.d(TAG, "已获取工具: ${toolDefinition.name}") - } - } catch (e: Exception) { - Log.e(TAG, "获取工具定义失败: ${e.message}", e) - } - } - - /** - * 将工具定义转换为JSON - */ - private fun toolDefinitionToJson(toolDefinition: ToolDefinition): JSONObject { - val functionObject = JSONObject().apply { - put("name", toolDefinition.name) - put("description", toolDefinition.description ?: "") - - // 处理参数定义 - if (toolDefinition.parameters != null) { - put("parameters", JSONObject(toolDefinition.parameters)) - } - } - - return JSONObject().apply { - put("type", "function") - put("function", functionObject) - } - } - - /** - * 获取当前连接状态 - */ - fun isConnected(): Boolean { - return isConnected && client != null - } - - /** - * 注册本地函数 - */ - fun registerLocalFunction( - name: String, - description: String, - parameters: JSONObject, - handler: FunctionHandler - ): Boolean { - try { - // 创建工具定义 - val toolDefinition = JSONObject().apply { - put("type", "function") - put("function", JSONObject().apply { - put("name", name) - put("description", description) - put("parameters", parameters) - }) - } - - // 添加到工具列表 - tools.add(toolDefinition.toString()) - - // 注册处理器 - functionHandlers[name] = handler - - Log.d(TAG, "已注册本地函数: $name") - return true - } catch (e: Exception) { - Log.e(TAG, "注册本地函数失败: ${e.message}", e) - return false - } - } - - /** - * 获取所有工具的定义 - */ - fun getToolMaps(): List { - return tools.toList() - } - - /** - * 解析JSON参数 - */ - fun parseJsonArguments(argumentsJson: String): Map { - try { - return objectMapper.readValue(argumentsJson, Map::class.java) as Map - } catch (e: Exception) { - Log.e(TAG, "解析JSON参数失败: ${e.message}", e) - return mapOf() - } - } - - /** - * 调用工具 - */ - suspend fun callTool(name: String, arguments: Map): JSONObject { - try { - // 检查是否是本地函数 - if (functionHandlers.containsKey(name)) { - val handler = functionHandlers[name] - val result = handler?.handle(arguments) ?: throw Exception("函数处理器为空") - - // 函数处理结果格式化为JSON - return JSONObject().apply { - put("context", result) - } - } - - // 如果不是本地函数，使用MCP客户端调用远程工具 - if (client != null && isConnected) { - val toolCallRequest = ToolCallRequest( - name = name, - arguments = objectMapper.writeValueAsString(arguments) - ) - - val result = client?.callTool(toolCallRequest) - return JSONObject().apply { - put("context", result?.result ?: "工具调用失败，未收到结果") - } - } - - throw Exception("MCP客户端未连接") - } catch (e: Exception) { - Log.e(TAG, "调用工具失败: ${e.message}", e) - return JSONObject().apply { - put("context", "调用工具失败: ${e.message}") - } - } - } - - /** - * 关闭客户端 - */ - fun close() { - try { - client?.disconnect() - } catch (e: Exception) { - Log.e(TAG, "关闭MCP客户端失败: ${e.message}", e) - } finally { - client = null - isConnected = false - job.cancel() // 取消所有协程 - tools.clear() // 清除工具列表 - functionHandlers.clear() // 清除函数处理器 - } - } -} \ No newline at end of file diff --git a/local_plugins/open_ai/android/src/main/kotlin/com/yunqiinnovation/open_ai/OpenAIService.kt b/local_plugins/open_ai/android/src/main/kotlin/com/yunqiinnovation/open_ai/OpenAIService.kt deleted file mode 100644 index 5868fb48d..000000000 --- a/local_plugins/open_ai/android/src/main/kotlin/com/yunqiinnovation/open_ai/OpenAIService.kt +++ /dev/null @@ -1,988 +0,0 @@ -package com.yunqiinnovation.open_ai - -import android.content.Context -import android.graphics.Bitmap -import android.graphics.BitmapFactory -import android.util.Base64 -import android.util.Log -import com.aallam.openai.api.chat.* -import com.aallam.openai.api.file.FileSource -import com.aallam.openai.api.http.Timeout -import com.aallam.openai.api.image.ImageCreation -import com.aallam.openai.api.model.ModelId -import com.aallam.openai.client.OpenAI -import com.aallam.openai.client.OpenAIConfig -import com.aallam.openai.client.OpenAIHost -import com.fasterxml.jackson.databind.ObjectMapper -import kotlinx.coroutines.* -import kotlinx.coroutines.flow.* -import okhttp3.* -import okhttp3.sse.EventSource -import okhttp3.sse.EventSourceListener -import okhttp3.sse.EventSources -import org.json.JSONArray -import org.json.JSONObject -import java.io.ByteArrayOutputStream -import java.io.File -import java.io.IOException -import java.util.UUID -import java.util.concurrent.TimeUnit -import kotlin.coroutines.CoroutineContext -import kotlin.time.Duration.Companion.seconds - -/** - * OpenAI服务的原生实现 - * 集成了OpenAI官方Java SDK和MCP的官方Kotlin SDK - */ -class OpenAIService(private val context: Context? = null) : CoroutineScope { - private val TAG = "OpenAIService" - - // 协程相关 - private val job = SupervisorJob() - override val coroutineContext: CoroutineContext - get() = Dispatchers.IO + job - - // 添加辅助方法，确保回调在主线程执行 - private suspend fun safeCallback(block: suspend () -> Unit) { - withContext(Dispatchers.Main) { - block() - } - } - - // OpenAI客户端 - private var openAI: OpenAI? = null - private var baseUrl = "https://api.openai.com/v1/chat/completions" - private var apiKey: String = "" - private var isInitialized = false - private var model: String = "gpt-3.5-turbo" // 默认模型 - private var visionModel: String = "gpt-4-vision-preview" // 默认视觉模型 - - // OkHttp客户端用于流式请求 - private val client = OkHttpClient.Builder() - .connectTimeout(30, TimeUnit.SECONDS) - .readTimeout(30, TimeUnit.SECONDS) - .writeTimeout(30, TimeUnit.SECONDS) - .build() - - // MCP客户端 - private var mcpClient: MCPClient? = null - private var isMcpInitialized = false - - // 当前事件源 - private var currentEventSource: EventSource? = null - private var isCanceled = false - - /** - * 初始化OpenAI服务 - */ - fun initialize(apiKey: String, baseUrl: String, model: String, mcpServer: String): Boolean { - this.apiKey = apiKey - if (baseUrl.isNotEmpty()) { - this.baseUrl = baseUrl - } - if (model.isNotEmpty()) { - this.model = model - } - - // 初始化OpenAI客户端 - try { - val timeout = Timeout(socket = 30.seconds, connect = 30.seconds, request = 30.seconds) - val host = if (baseUrl.isNotEmpty() && baseUrl != "https://api.openai.com/v1/chat/completions") { - OpenAIHost(baseUrl) - } else { - OpenAIHost.Default - } - - val config = OpenAIConfig( - token = apiKey, - host = host, - timeout = timeout - ) - - openAI = OpenAI(config) - } catch (e: Exception) { - Log.e(TAG, "OpenAI客户端初始化失败: ${e.message}", e) - return false - } - - // 初始化MCPClient - initializeMcpClient(mcpServer) - - isInitialized = apiKey.isNotEmpty() && openAI != null - return isInitialized - } - - /** - * 将文件转换为Base64字符串 - */ - fun fileToBase64(filePath: String, maxSizeKB: Int = 20480): String? { - try { - val file = File(filePath) - if (!file.exists() || !file.isFile) { - Log.e(TAG, "文件不存在: $filePath") - return null - } - - // 读取文件并压缩（如果需要） - val originalBitmap = BitmapFactory.decodeFile(filePath) - if (originalBitmap == null) { - Log.e(TAG, "无法解码图片: $filePath") - return null - } - - val outputStream = ByteArrayOutputStream() - var quality = 100 - var compressedBitmap = originalBitmap - - // 检查图片尺寸，限制最大为1024*1024 - val maxDimension = 1024 - if (originalBitmap.width > maxDimension || originalBitmap.height > maxDimension) { - Log.d(TAG, "图片尺寸超过限制，进行缩放: ${originalBitmap.width}x${originalBitmap.height} -> ${maxDimension}x${maxDimension}") - - // 计算缩放比例，保持纵横比 - val widthRatio = maxDimension.toFloat() / originalBitmap.width - val heightRatio = maxDimension.toFloat() / originalBitmap.height - val ratio = Math.min(widthRatio, heightRatio) - - val newWidth = (originalBitmap.width * ratio).toInt() - val newHeight = (originalBitmap.height * ratio).toInt() - - compressedBitmap = Bitmap.createScaledBitmap(originalBitmap, newWidth, newHeight, true) - Log.d(TAG, "缩放后图片尺寸: ${newWidth}x${newHeight}") - } - - // 如果原始图片太大，继续优化文件大小 - var fileSize = file.length() / 1024 // 转为KB - if (fileSize > maxSizeKB) { - val scale = Math.sqrt(maxSizeKB.toDouble() / fileSize) - val newWidth = (compressedBitmap.width * scale).toInt() - val newHeight = (compressedBitmap.height * scale).toInt() - compressedBitmap = Bitmap.createScaledBitmap(compressedBitmap, newWidth, newHeight, true) - quality = 85 - } - - // 压缩图片 - compressedBitmap.compress(Bitmap.CompressFormat.JPEG, quality, outputStream) - val imageBytes = outputStream.toByteArray() - - // 检查压缩后大小 - if (imageBytes.size / 1024 > maxSizeKB) { - Log.w(TAG, "压缩后图片仍然超出大小限制: ${imageBytes.size / 1024}KB > ${maxSizeKB}KB") - } - - // 转为Base64 - return Base64.encodeToString(imageBytes, Base64.NO_WRAP) - } catch (e: Exception) { - Log.e(TAG, "转换文件到Base64失败: ${e.message}", e) - return null - } - } - - /** - * 创建带图片的用户消息 - */ - fun createUserMessageWithImage(text: String, imageBase64: String): JSONObject { - // 创建包含文本和图片的内容数组 - val contentArray = JSONArray().apply { - // 添加文本部分 - if (text.isNotEmpty()) { - put(JSONObject().apply { - put("type", "text") - put("text", text) - }) - } - - // 添加图片部分 - put(JSONObject().apply { - put("type", "image_url") - put("image_url", JSONObject().apply { - put("url", "data:image/jpeg;base64,$imageBase64") - }) - }) - } - - return JSONObject().apply { - put("role", "user") - put("content", contentArray) - } - } - - /** - * 创建用户消息 - */ - fun createUserMessage(content: String): JSONObject { - return JSONObject().apply { - put("role", "user") - put("content", content) - } - } - - /** - * 创建系统消息 - */ - fun createSystemMessage(content: String): JSONObject { - return JSONObject().apply { - put("role", "system") - put("content", content) - } - } - - /** - * 创建助手消息 - */ - fun createAssistantMessage(content: String): JSONObject { - return JSONObject().apply { - put("role", "assistant") - put("content", content) - } - } - - /** - * MCP客户端是否已初始化 - */ - fun isMcpInitialized(): Boolean { - return isMcpInitialized && mcpClient?.isConnected() == true - } - - /** - * 关闭MCP客户端 - */ - fun closeMcpClient() { - mcpClient?.close() - mcpClient = null - isMcpInitialized = false - } - - /** - * 初始化MCP客户端 - */ - private fun initializeMcpClient(mcpServer: String): Boolean { - if (mcpClient != null) { - mcpClient?.close() - } - - if (mcpServer.isEmpty()) { - return false - } - - mcpClient = MCPClient(context) - - // 在后台线程中初始化MCP客户端 - launch { - try { - val result = mcpClient?.connectToSSE(mcpServer) ?: false - isMcpInitialized = result - Log.d(TAG, "MCP客户端初始化${if (result) "成功" else "失败"}") - } catch (e: Exception) { - Log.e(TAG, "MCP客户端初始化失败: ${e.message}", e) - isMcpInitialized = false - } - } - - return true // 立即返回，实际连接在后台进行 - } - - /** - * 处理MCP工具调用 - */ - private suspend fun handleMcpToolCall(functionCall: JSONObject): JSONObject? { - if (mcpClient == null || !isMcpInitialized) { - return JSONObject().apply { put("context", "MCP客户端未初始化") } - } - - try { - // 获取函数名称 - val name = functionCall.getString("name") - - // 获取参数 - val argumentsJson = functionCall.getString("arguments") - val arguments = mcpClient?.parseJsonArguments(argumentsJson) ?: mapOf() - - // 调用工具 - return mcpClient?.callTool(name, arguments) - } catch (e: Exception) { - Log.e(TAG, "处理MCP工具调用失败: ${e.message}", e) - return JSONObject().apply { put("context", "处理MCP工具调用失败: ${e.message}") } - } - } - - /** - * 将JSONArray转换为ChatCompletionRequest中的消息列表 - */ - private fun parseMessages(messagesArray: JSONArray): List { - val messages = mutableListOf() - - for (i in 0 until messagesArray.length()) { - val messageObj = messagesArray.getJSONObject(i) - val role = messageObj.getString("role") - - when (role) { - "system" -> { - val content = messageObj.getString("content") - messages.add(ChatMessage(role = ChatRole.System, content = content)) - } - "user" -> { - // 检查是否有多媒体内容 - if (messageObj.has("content") && messageObj.get("content") is JSONArray) { - val contentArray = messageObj.getJSONArray("content") - val parts = mutableListOf() - - for (j in 0 until contentArray.length()) { - val contentObj = contentArray.getJSONObject(j) - val type = contentObj.getString("type") - - when (type) { - "text" -> { - parts.add(TextContent(contentObj.getString("text"))) - } - "image_url" -> { - val imageUrlObj = contentObj.getJSONObject("image_url") - val url = imageUrlObj.getString("url") - parts.add(ImageContent(url)) - } - } - } - - messages.add(ChatMessage( - role = ChatRole.User, - content = parts - )) - } else { - // 普通文本消息 - val content = messageObj.getString("content") - messages.add(ChatMessage(role = ChatRole.User, content = content)) - } - } - "assistant" -> { - if (messageObj.has("tool_calls")) { - // 处理工具调用 - val toolCalls = messageObj.getJSONArray("tool_calls") - val toolCallsList = mutableListOf() - - for (j in 0 until toolCalls.length()) { - val toolCall = toolCalls.getJSONObject(j) - val id = toolCall.getString("id") - val function = toolCall.getJSONObject("function") - val name = function.getString("name") - val arguments = function.getString("arguments") - - toolCallsList.add(ToolCall( - id = id, - type = ToolCallType.Function, - function = FunctionCall( - name = name, - arguments = arguments - ) - )) - } - - val content = if (messageObj.has("content")) messageObj.getString("content") else "" - - messages.add(ChatMessage( - role = ChatRole.Assistant, - content = content, - toolCalls = toolCallsList - )) - } else { - // 普通消息 - val content = messageObj.getString("content") - messages.add(ChatMessage(role = ChatRole.Assistant, content = content)) - } - } - "tool" -> { - val content = messageObj.getString("content") - val toolCallId = messageObj.getString("tool_call_id") - messages.add(ChatMessage( - role = ChatRole.Tool, - content = content, - toolCallId = toolCallId - )) - } - } - } - - return messages - } - - /** - * 从MCP客户端获取工具定义 - */ - private fun getToolsFromMcpClient(): List { - val tools = mutableListOf() - - mcpClient?.getToolMaps()?.forEach { toolMap -> - try { - val toolJson = JSONObject(toolMap) - val name = toolJson.getString("name") - val description = toolJson.getString("description") - - val parametersJson = if (toolJson.has("parameters")) toolJson.getJSONObject("parameters") else null - val parameterProperties = mutableMapOf() - val requiredParams = mutableListOf() - - if (parametersJson != null && parametersJson.has("properties")) { - val properties = parametersJson.getJSONObject("properties") - val keys = properties.keys() - - while (keys.hasNext()) { - val key = keys.next() - val propertyObj = properties.getJSONObject(key) - val propType = propertyObj.optString("type", "string") - val propDescription = propertyObj.optString("description", "") - - parameterProperties[key] = ParameterDefinition( - type = propType, - description = propDescription - ) - } - - // 获取必填参数 - if (parametersJson.has("required")) { - val requiredArr = parametersJson.getJSONArray("required") - for (i in 0 until requiredArr.length()) { - requiredParams.add(requiredArr.getString(i)) - } - } - } - - tools.add(Tool( - type = ToolType.Function, - function = FunctionDefinition( - name = name, - description = description, - parameters = FunctionParameters( - type = "object", - properties = parameterProperties, - required = requiredParams - ) - ) - )) - } catch (e: Exception) { - Log.e(TAG, "解析工具定义失败: ${e.message}", e) - } - } - - return tools - } - - /** - * 发送消息（非流式输出） - */ - suspend fun sendMessage(messages: JSONArray): String { - if (!isInitialized || apiKey.isEmpty() || openAI == null) { - throw IOException("OpenAI服务未初始化") - } - - try { - // 解析消息 - val parsedMessages = parseMessages(messages) - - // 获取工具列表 - val tools = getToolsFromMcpClient() - - // 创建请求 - val request = ChatCompletionRequest( - model = ModelId(model), - messages = parsedMessages, - tools = if (tools.isNotEmpty()) tools else null, - temperature = 0.7, - maxTokens = 2000 - ) - - // 发送请求 - val response = openAI!!.chatCompletion(request) - - // 解析响应 - val choice = response.choices.firstOrNull() ?: throw IOException("无效的响应格式") - - // 检查是否有工具调用 - if (choice.message.toolCalls?.isNotEmpty() == true) { - val toolCall = choice.message.toolCalls?.first() - if (toolCall != null && toolCall.function != null) { - val functionCall = JSONObject().apply { - put("name", toolCall.function.name) - put("arguments", toolCall.function.arguments) - put("id", toolCall.id) - } - return functionCall.toString() - } - } - - // 返回消息内容 - return choice.message.content ?: "" - - } catch (e: Exception) { - Log.e(TAG, "发送消息失败: ${e.message}", e) - throw IOException("与AI服务通信失败: ${e.message}") - } - } - - /** - * 发送消息（流式输出） - */ - fun sendMessageStream(messages: JSONArray, callback: StreamCallback) { - if (!isInitialized || apiKey.isEmpty()) { - // 在主线程执行回调 - launch { - withContext(Dispatchers.Main) { - callback.onError(Exception("OpenAI服务未初始化")) - } - } - return - } - - // 重置取消状态 - isCanceled = false - - // 检查是否有图片消息 - var hasImageContent = false - var currentModel = model - - for (i in 0 until messages.length()) { - val messageObj = messages.getJSONObject(i) - if (messageObj.getString("role") == "user" && messageObj.has("content")) { - val content = messageObj.get("content") - if (content is JSONArray) { - for (j in 0 until content.length()) { - val contentObj = content.getJSONObject(j) - if (contentObj.getString("type") == "image_url") { - hasImageContent = true - currentModel = visionModel - break - } - } - } - } - if (hasImageContent) break - } - - // 构建JSON请求体 - val requestBody = JSONObject().apply { - put("model", currentModel) - put("messages", messages) - put("temperature", 0.7) - put("max_tokens", 2000) - put("stream", true) - - // 添加工具列表 - val tools = JSONArray() - - // 使用MCPClient提供的所有工具 - mcpClient?.getToolMaps()?.forEach { toolMap -> - try { - val tool = JSONObject(toolMap) - tools.put(tool) - } catch (e: Exception) { - Log.e(TAG, "转换工具失败: ${e.message}", e) - } - } - - // 如果有工具，则添加到请求中 - if (tools.length() > 0) { - put("tools", tools) - } - } - - val mediaType = "application/json".toMediaTypeOrNull() - val request = Request.Builder() - .url(baseUrl) - .addHeader("Content-Type", "application/json") - .addHeader("Authorization", "Bearer $apiKey") - .addHeader("Accept", "text/event-stream") - .post(requestBody.toString().toRequestBody(mediaType)) - .build() - - // 创建事件源 - val factory = EventSources.createFactory(client) - - // 工具调用相关变量 - val toolCalls = mutableMapOf() - - val eventSourceListener = object : EventSourceListener() { - override fun onOpen(eventSource: EventSource, response: Response) { - Log.d(TAG, "SSE连接已打开") - } - - override fun onEvent(eventSource: EventSource, id: String?, type: String?, data: String) { - if (isCanceled) return - - if (data == "[DONE]" || data == "[\"DONE\"]") { - // 处理可能的工具调用 - processToolCalls(toolCalls, callback, messages) - return - } - - try { - val jsonData = JSONObject(data) - - // 处理消息内容 - if (jsonData.has("choices")) { - val choices = jsonData.getJSONArray("choices") - if (choices.length() > 0) { - val choice = choices.getJSONObject(0) - - // 处理delta内容 - if (choice.has("delta")) { - val delta = choice.getJSONObject("delta") - - // 处理普通文本内容 - if (delta.has("content")) { - val content = delta.getString("content") - if (!isCanceled) { - launch { - withContext(Dispatchers.Main) { - callback.onToken(content) - } - } - } - } - - // 处理工具调用 - if (delta.has("tool_calls")) { - val deltaToolCalls = delta.getJSONArray("tool_calls") - for (i in 0 until deltaToolCalls.length()) { - val toolCall = deltaToolCalls.getJSONObject(i) - val index = toolCall.getInt("index") - - // 创建或获取现有的工具调用信息 - val toolCallInfo = toolCalls.getOrPut(index) { ToolCallInfo() } - - // 更新ID - if (toolCall.has("id")) { - toolCallInfo.id = toolCall.getString("id") - } - - // 更新函数信息 - if (toolCall.has("function")) { - val function = toolCall.getJSONObject("function") - - if (function.has("name")) { - toolCallInfo.name = function.getString("name") - } - - if (function.has("arguments")) { - toolCallInfo.arguments += function.getString("arguments") - } - } - } - } - } - - // 检查是否有表示完成的标志 - if (choice.has("finish_reason")) { - val finishReason = choice.getString("finish_reason") - if (finishReason == "stop" || finishReason == "length") { - // 正常完成，没有工具调用 - launch { - withContext(Dispatchers.Main) { - callback.onComplete() - } - } - closeEventSource() - } else if (finishReason == "tool_calls") { - // 处理工具调用 - processToolCalls(toolCalls, callback, messages) - closeEventSource() - } - } - } - } - } catch (e: Exception) { - Log.e(TAG, "解析事件数据失败: ${e.message}", e) - } - } - - override fun onClosed(eventSource: EventSource) { - Log.d(TAG, "SSE连接已关闭") - if (!isCanceled) { - // 如果没有正常完成，但连接关闭了，则处理最后可能的工具调用 - if (toolCalls.isNotEmpty()) { - processToolCalls(toolCalls, callback, messages) - } else { - launch { - withContext(Dispatchers.Main) { - callback.onComplete() - } - } - } - } - } - - override fun onFailure(eventSource: EventSource, t: Throwable?, response: Response?) { - if (isCanceled) return - - val errorCode = response?.code ?: 0 - val errorMessage = t?.message ?: "未知错误" - Log.e(TAG, "SSE连接失败: $errorCode - $errorMessage") - - launch { - withContext(Dispatchers.Main) { - callback.onError(Exception("流式请求失败: $errorMessage")) - } - } - closeEventSource() - } - } - - currentEventSource = factory.newEventSource(request, eventSourceListener) - } - - /** - * 处理工具调用 - */ - private fun processToolCalls(toolCalls: Map, callback: StreamCallback, messages: JSONArray? = null): Boolean { - if (toolCalls.isEmpty()) return false - - val firstToolCall = toolCalls.entries.firstOrNull()?.value ?: return false - - if (firstToolCall.isValid()) { - // 创建函数调用JSON对象 - val functionCall = JSONObject().apply { - put("name", firstToolCall.name) - put("arguments", firstToolCall.arguments) - put("id", firstToolCall.id) - } - Log.d(TAG, "工具调用: $functionCall") - - // 通知上层回调 - launch { - withContext(Dispatchers.Main) { - callback.onFunctionCall(functionCall) - } - } - - // 在协程中自动处理工具调用 - if (messages != null) { - launch { - try { - if (!isCanceled) { - handleToolCall(functionCall, messages, callback) - } - } catch (e: Exception) { - if (!isCanceled) { - Log.e(TAG, "处理工具调用时发生异常: ${e.message}", e) - try { - val errorMessage = "工具调用处理失败: ${e.message}" - sendFunctionCallResult( - messages = messages, - functionCall = functionCall, - functionResult = errorMessage, - callback = callback - ) - } catch (e2: Exception) { - Log.e(TAG, "发送工具调用错误结果失败: ${e2.message}", e2) - withContext(Dispatchers.Main) { - callback.onError(Exception("工具调用处理失败: ${e.message}")) - } - } - } - } - } - } - return true - } - return false - } - - /** - * 处理工具调用 - */ - private suspend fun handleToolCall(functionCall: JSONObject, messages: JSONArray, callback: StreamCallback) { - try { - // 获取函数名称和参数 - val name = functionCall.getString("name") - val argumentsJson = functionCall.getString("arguments") - - Log.d(TAG, "处理工具调用: name=$name, arguments=$argumentsJson") - - // 调用MCP工具 - val result = handleMcpToolCall(functionCall) - - // 检查是否已取消 - if (isCanceled) { - Log.d(TAG, "工具调用已被取消，不处理结果") - return - } - - // 回调结果 - val resultObj = result ?: JSONObject().apply { put("context", "工具调用失败") } - withContext(Dispatchers.Main) { - callback.onFunctionCallResult(functionCall, resultObj) - } - - // 从结果中提取内容 - val resultContent = if (resultObj.has("context") && resultObj.optString("context").isNotEmpty()) { - resultObj.getString("context") - } else { - val jsonString = resultObj.toString() - if (jsonString == "{}") "工具调用失败" else jsonString - } - - // 发送函数调用结果 - sendFunctionCallResult(messages, functionCall, resultContent, callback) - } catch (e: Exception) { - if (!isCanceled) { - Log.e(TAG, "处理工具调用失败: ${e.message}", e) - withContext(Dispatchers.Main) { - callback.onError(Exception("工具调用失败: ${e.message}")) - } - } - } - } - - /** - * 发送函数调用结果 - */ - fun sendFunctionCallResult( - messages: JSONArray, - functionCall: JSONObject, - functionResult: String, - callback: StreamCallback - ) { - if (isCanceled) { - Log.d(TAG, "请求已取消，不发送函数调用结果") - return - } - - try { - val fullMessages = JSONArray() - - // 添加原始消息 - for (i in 0 until messages.length()) { - fullMessages.put(messages.getJSONObject(i)) - } - - // 添加函数调用消息 - val callId = functionCall.optString("id", "call_${System.currentTimeMillis()}") - fullMessages.put(JSONObject().apply { - put("role", "assistant") - put("content", "") - - // 添加工具调用 - val toolCalls = JSONArray().apply { - val toolCall = JSONObject().apply { - put("id", callId) - put("type", "function") - put("function", JSONObject().apply { - put("name", functionCall.getString("name")) - put("arguments", functionCall.getString("arguments")) - }) - } - put(toolCall) - } - put("tool_calls", toolCalls) - }) - - // 添加函数调用结果 - fullMessages.put(JSONObject().apply { - put("role", "tool") - put("content", functionResult) - put("tool_call_id", callId) - }) - - // 发送完整对话 - sendMessageStream(fullMessages, callback) - - } catch (e: Exception) { - if (!isCanceled) { - Log.e(TAG, "发送函数调用结果失败: ${e.message}", e) - launch { - withContext(Dispatchers.Main) { - callback.onError(Exception("发送函数调用结果失败: ${e.message}")) - } - } - } - } - } - - /** - * 关闭事件源 - */ - private fun closeEventSource() { - currentEventSource?.let { - try { - it.cancel() - } catch (e: Exception) { - Log.e(TAG, "关闭事件源失败: ${e.message}", e) - } - currentEventSource = null - } - } - - /** - * 取消当前流式请求 - */ - fun cancelCurrentStream(): Boolean { - isCanceled = true - closeEventSource() - return true - } - - /** - * 释放资源 - */ - fun dispose() { - job.cancel() // 取消所有协程 - cancelCurrentStream() // 取消当前流式请求 - closeMcpClient() // 关闭MCP客户端 - } - - /** - * 工具调用信息类 - */ - private class ToolCallInfo { - var id: String = "" - var name: String = "" - var arguments: String = "" - - fun isValid(): Boolean { - return id.isNotEmpty() && name.isNotEmpty() - } - } - - /** - * 流式回调接口 - */ - interface StreamCallback { - fun onToken(token: String) - fun onComplete() - fun onError(e: Exception) - fun onFunctionCall(functionCall: JSONObject) - fun onFunctionCallResult(functionCall: JSONObject, functionCallResult: JSONObject) - } - - /** - * 参数定义 - */ - private data class ParameterDefinition( - val type: String, - val description: String - ) - - /** - * 注册函数 - */ - fun registerFunction(name: String, description: String, parameters: JSONObject): Boolean { - try { - // 确保MCPClient已初始化 - if (mcpClient == null) { - mcpClient = MCPClient(context) - } - - // 创建函数处理器 - val handler = object : FunctionHandler { - override suspend fun handle(arguments: Map): String { - // 由于本地函数的实际处理是在Flutter端完成的 - // 这里只需返回一个标记，表示该函数是本地函数 - return "LOCAL_FUNCTION:$name" - } - } - - // 注册本地函数 - return mcpClient?.registerLocalFunction(name, description, parameters, handler) ?: false - } catch (e: Exception) { - Log.e(TAG, "注册函数失败: ${e.message}", e) - return false - } - } -} \ No newline at end of file diff --git a/local_plugins/open_ai/android/src/main/kotlin/com/yunqiinnovation/open_ai/OpenAiPlugin.kt b/local_plugins/open_ai/android/src/main/kotlin/com/yunqiinnovation/open_ai/OpenAiPlugin.kt deleted file mode 100644 index 4290156d0..000000000 --- a/local_plugins/open_ai/android/src/main/kotlin/com/yunqiinnovation/open_ai/OpenAiPlugin.kt +++ /dev/null @@ -1,149 +0,0 @@ -package com.yunqiinnovation.open_ai - -import android.content.Context -import androidx.annotation.NonNull -import io.flutter.embedding.engine.plugins.FlutterPlugin -import io.flutter.plugin.common.MethodCall -import io.flutter.plugin.common.MethodChannel -import io.flutter.plugin.common.MethodChannel.MethodCallHandler -import io.flutter.plugin.common.MethodChannel.Result -import org.json.JSONArray -import org.json.JSONObject -import kotlinx.coroutines.* - -class OpenAiPlugin : FlutterPlugin, MethodCallHandler { - private lateinit var channel: MethodChannel - private lateinit var context: Context - private lateinit var openAIService: OpenAIService - private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) - - override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { - channel = MethodChannel(flutterPluginBinding.binaryMessenger, "com.yunqiinnovation.open_ai") - context = flutterPluginBinding.applicationContext - openAIService = OpenAIService(context) - channel.setMethodCallHandler(this) - } - - override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) { - when (call.method) { - "initialize" -> { - val apiKey = call.argument("apiKey") ?: "" - val baseUrl = call.argument("baseUrl") ?: "" - val model = call.argument("model") ?: "" - val mcpServer = call.argument("mcpServer") ?: "" - val success = openAIService.initialize(apiKey, baseUrl, model, mcpServer) - result.success(success) - } - "createUserMessage" -> { - val content = call.argument("content") ?: "" - val message = openAIService.createUserMessage(content) - result.success(message.toString()) - } - "createSystemMessage" -> { - val content = call.argument("content") ?: "" - val message = openAIService.createSystemMessage(content) - result.success(message.toString()) - } - "createAssistantMessage" -> { - val content = call.argument("content") ?: "" - val message = openAIService.createAssistantMessage(content) - result.success(message.toString()) - } - "createUserMessageWithImage" -> { - val text = call.argument("text") ?: "" - val imageBase64 = call.argument("imageBase64") ?: "" - val message = openAIService.createUserMessageWithImage(text, imageBase64) - result.success(message.toString()) - } - "sendMessage" -> { - val messagesJson = call.argument("messages") ?: "[]" - val messages = JSONArray(messagesJson) - - scope.launch { - try { - val response = openAIService.sendMessage(messages) - withContext(Dispatchers.Main) { - result.success(response) - } - } catch (e: Exception) { - withContext(Dispatchers.Main) { - result.error("OPENAI_ERROR", e.message ?: "Unknown error", null) - } - } - } - } - "sendMessageStream" -> { - val messagesJson = call.argument("messages") ?: "[]" - val messages = JSONArray(messagesJson) - val streamId = call.argument("streamId") ?: "${System.currentTimeMillis()}" - - val callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - val map = mapOf( - "type" to "token", - "streamId" to streamId, - "data" to token - ) - channel.invokeMethod("onStreamEvent", map) - } - - override fun onComplete() { - val map = mapOf( - "type" to "complete", - "streamId" to streamId - ) - channel.invokeMethod("onStreamEvent", map) - } - - override fun onError(e: Exception) { - val map = mapOf( - "type" to "error", - "streamId" to streamId, - "error" to (e.message ?: "Unknown error") - ) - channel.invokeMethod("onStreamEvent", map) - } - - override fun onFunctionCall(functionCall: JSONObject) { - val map = mapOf( - "type" to "functionCall", - "streamId" to streamId, - "data" to functionCall.toString() - ) - channel.invokeMethod("onStreamEvent", map) - } - - override fun onFunctionCallResult(functionCall: JSONObject, functionCallResult: JSONObject) { - val map = mapOf( - "type" to "functionCallResult", - "streamId" to streamId, - "functionCall" to functionCall.toString(), - "result" to functionCallResult.toString() - ) - channel.invokeMethod("onStreamEvent", map) - } - } - - openAIService.sendMessageStream(messages, callback) - result.success(streamId) - } - "cancelCurrentStream" -> { - val success = openAIService.cancelCurrentStream() - result.success(success) - } - "dispose" -> { - openAIService.dispose() - result.success(null) - } - else -> { - result.notImplemented() - } - } - } - - override fun onDetachedFromEngine(@NonNull binding: FlutterPlugin.FlutterPluginBinding) { - channel.setMethodCallHandler(null) - scope.cancel() // Cancel all coroutines when the plugin is detached - openAIService.dispose() // Clean up resources - } -} \ No newline at end of file diff --git a/local_plugins/open_ai/lib/open_ai.dart b/local_plugins/open_ai/lib/open_ai.dart deleted file mode 100644 index 7597d4a70..000000000 --- a/local_plugins/open_ai/lib/open_ai.dart +++ /dev/null @@ -1,184 +0,0 @@ -import 'dart:async'; -import 'dart:convert'; - -import 'package:flutter/services.dart'; - -/// OpenAI和MCP集成插件 -class OpenAI { - /// 插件通道 - static const MethodChannel _channel = - MethodChannel('com.yunqiinnovation.open_ai'); - - /// 流式输出事件回调 - static final Map _streamCallbacks = {}; - - /// 构造函数 - OpenAI() { - _channel.setMethodCallHandler(_handleMethodCall); - } - - /// 处理来自原生端的方法调用 - Future _handleMethodCall(MethodCall call) async { - if (call.method == 'onStreamEvent') { - final Map args = Map.from(call.arguments); - final String streamId = args['streamId']; - final String type = args['type']; - - final callback = _streamCallbacks[streamId]; - if (callback != null) { - switch (type) { - case 'token': - callback.onToken(args['data']); - break; - case 'complete': - callback.onComplete(); - _streamCallbacks.remove(streamId); - break; - case 'error': - callback.onError(Exception(args['error'])); - _streamCallbacks.remove(streamId); - break; - case 'functionCall': - final functionCall = jsonDecode(args['data']); - callback.onFunctionCall(functionCall); - break; - case 'functionCallResult': - final functionCall = jsonDecode(args['functionCall']); - final functionCallResult = jsonDecode(args['result']); - callback.onFunctionCallResult(functionCall, functionCallResult); - break; - } - } - } - return null; - } - - /// 初始化OpenAI服务 - /// - /// [apiKey] OpenAI API密钥 - /// [baseUrl] API基础URL，可选 - /// [model] 模型名称，可选 - /// [mcpServer] MCP服务器URL，可选 - Future initialize({ - required String apiKey, - String baseUrl = '', - String model = '', - String mcpServer = '', - }) async { - final result = await _channel.invokeMethod('initialize', { - 'apiKey': apiKey, - 'baseUrl': baseUrl, - 'model': model, - 'mcpServer': mcpServer, - }); - return result ?? false; - } - - /// 创建用户消息 - Future> createUserMessage(String content) async { - final result = await _channel.invokeMethod('createUserMessage', { - 'content': content, - }); - return jsonDecode(result ?? '{}'); - } - - /// 创建系统消息 - Future> createSystemMessage(String content) async { - final result = await _channel.invokeMethod('createSystemMessage', { - 'content': content, - }); - return jsonDecode(result ?? '{}'); - } - - /// 创建助手消息 - Future> createAssistantMessage(String content) async { - final result = await _channel.invokeMethod('createAssistantMessage', { - 'content': content, - }); - return jsonDecode(result ?? '{}'); - } - - /// 创建带图片的用户消息 - Future> createUserMessageWithImage( - String text, - String imageBase64, - ) async { - final result = await _channel.invokeMethod( - 'createUserMessageWithImage', - { - 'text': text, - 'imageBase64': imageBase64, - }, - ); - return jsonDecode(result ?? '{}'); - } - - /// 发送消息（非流式输出） - Future sendMessage(List> messages) async { - final messagesJson = jsonEncode(messages); - return await _channel.invokeMethod('sendMessage', { - 'messages': messagesJson, - }); - } - - /// 发送消息（流式输出） - Future sendMessageStream( - List> messages, - StreamCallback callback, - ) async { - final messagesJson = jsonEncode(messages); - final streamId = DateTime.now().millisecondsSinceEpoch.toString(); - - // 注册回调 - _streamCallbacks[streamId] = callback; - - final result = await _channel.invokeMethod('sendMessageStream', { - 'messages': messagesJson, - 'streamId': streamId, - }); - - return result ?? streamId; - } - - /// 取消当前流式请求 - Future cancelCurrentStream() async { - final result = await _channel.invokeMethod('cancelCurrentStream'); - return result ?? false; - } - - /// 释放资源 - Future dispose() async { - await _channel.invokeMethod('dispose'); - _streamCallbacks.clear(); - } -} - -/// 流式输出回调接口 -class StreamCallback { - /// 收到令牌 - final void Function(String token) onToken; - - /// 完成回调 - final void Function() onComplete; - - /// 错误回调 - final void Function(Exception e) onError; - - /// 函数调用回调 - final void Function(Map functionCall) onFunctionCall; - - /// 函数调用结果回调 - final void Function( - Map functionCall, - Map functionCallResult, - ) onFunctionCallResult; - - /// 构造函数 - StreamCallback({ - required this.onToken, - required this.onComplete, - required this.onError, - required this.onFunctionCall, - required this.onFunctionCallResult, - }); -} \ No newline at end of file diff --git a/local_plugins/open_ai/pubspec.yaml b/local_plugins/open_ai/pubspec.yaml deleted file mode 100644 index 299aac80c..000000000 --- a/local_plugins/open_ai/pubspec.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: open_ai -description: OpenAI API与MCP集成插件，提供对OpenAI API的访问和MCP工具调用功能 -version: 0.1.0 -homepage: https://github.com/yunqiinnovation/deep_voice - -environment: - sdk: '>=2.18.0 <4.0.0' - flutter: ">=3.3.0" - -dependencies: - flutter: - sdk: flutter - -dev_dependencies: - flutter_test: - sdk: flutter - flutter_lints: ^2.0.0 - -# 插件平台配置 -flutter: - plugin: - platforms: - android: - package: com.yunqiinnovation.open_ai - pluginClass: OpenAiPlugin - ios: - pluginClass: OpenAiPlugin \ No newline at end of file diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt index add4e3dc9..71469c13b 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt @@ -43,15 +43,6 @@ class OpenAIService(private val context: Context? = null) : CoroutineScope { // 主线程Handler，用于确保回调在主线程执行 private val mainHandler = Handler(Looper.getMainLooper()) - // 将回调安全地切换到主线程 - private fun StreamCallback.dispatch(block: StreamCallback.() -> Unit) { - if (Looper.myLooper() == Looper.getMainLooper()) { - block() - } else { - mainHandler.post { block() } - } - } - private var baseUrl = "" private val client = OkHttpClient.Builder() .connectTimeout(30, TimeUnit.SECONDS) @@ -352,7 +343,7 @@ class OpenAIService(private val context: Context? = null) : CoroutineScope { // 回调结果 val resultObj = result ?: JSONObject().apply { put("context", "工具调用失败") } - callback.dispatch { onFunctionCallResult(functionCall, resultObj) } + callback.onFunctionCallResult(functionCall, resultObj) // 发送结果到OpenAI继续对话 val resultContent = resultObj.optString("context", "工具调用失败") @@ -480,7 +471,7 @@ class OpenAIService(private val context: Context? = null) : CoroutineScope { */ fun sendMessageStream(messages: JSONArray, callback: StreamCallback) { if (!isInitialized || apiKey.isEmpty()) { - callback.dispatch { onError(OpenAIException("OpenAI服务未初始化")) } + callback.onError(OpenAIException("OpenAI服务未初始化")) return } @@ -537,17 +528,19 @@ class OpenAIService(private val context: Context? = null) : CoroutineScope { val call = client.newCall(request) // 保存当前Call对象以便后续取消 currentStreamCall = call - + Log.d(TAG, "发送消息流式输出: request=$request") call.enqueue(object : Callback { override fun onFailure(call: Call, e: IOException) { + Log.d(TAG, "发送消息流式输出失败: e=$e") // 如果是由于取消导致的失败，不报告错误 if (isCanceled) { return } - callback.dispatch { onError(OpenAIException(e.message ?: "请求失败")) } + callback.onError(OpenAIException(e.message ?: "请求失败")) } override fun onResponse(call: Call, response: Response) { + Log.d(TAG, "发送消息流式输出成功: response=$response") // 检查请求是否被取消 if (isCanceled) { response.body?.close() @@ -555,7 +548,7 @@ class OpenAIService(private val context: Context? = null) : CoroutineScope { } if (!response.isSuccessful) { - callback.dispatch { onError(OpenAIException("API调用失败: ${response.code}")) } + callback.onError(OpenAIException("API调用失败: ${response.code}")) return } @@ -588,7 +581,7 @@ class OpenAIService(private val context: Context? = null) : CoroutineScope { val hasToolCalls = processToolCalls(toolCalls, callback, messages) // 如果没有工具调用，这是对话的真正结束 if (!hasToolCalls) { - callback.dispatch { onComplete() } + callback.onComplete() } return } @@ -611,14 +604,14 @@ class OpenAIService(private val context: Context? = null) : CoroutineScope { // 如果没有明确的完成信号但数据流结束了，处理最后可能的工具调用 val hasToolCalls = processToolCalls(toolCalls, callback, messages) if (!hasToolCalls) { - callback.dispatch { onComplete() } + callback.onComplete() } } catch (e: Exception) { // 如果是由于取消导致的异常，不报告错误 if (isCanceled) { return } - callback.dispatch { onError(OpenAIException("处理响应流时出错: ${e.message}")) } + callback.onError(OpenAIException("处理响应流时出错: ${e.message}")) } finally { responseBody.close() } @@ -645,8 +638,9 @@ class OpenAIService(private val context: Context? = null) : CoroutineScope { // 处理普通文本内容 if (delta.has("content")) { val content = delta.getString("content") + Log.d(TAG, "处理流式JSON数据: content=$content") if (!isCanceled) { - callback.dispatch { onToken(content) } + callback.onToken(content) } } @@ -707,7 +701,7 @@ class OpenAIService(private val context: Context? = null) : CoroutineScope { Log.d(TAG, "工具调用: id=${firstToolCall.id}, name=${firstToolCall.name}") // 通知上层工具调用事件 - callback.dispatch { onFunctionCall(functionCall) } + callback.onFunctionCall(functionCall) // 在协程中处理工具调用 if (messages != null) { @@ -726,13 +720,13 @@ class OpenAIService(private val context: Context? = null) : CoroutineScope { if (!isCanceled) { // 处理结果 val resultObj = result ?: JSONObject().put("context", "工具调用失败") - Log.d(TAG, "工具调用完成: id=${firstToolCall.id}") - callback.dispatch { onFunctionCallResult(functionCall, resultObj) } + callback.onFunctionCallResult(functionCall, resultObj) val context = resultObj.optString("context", "工具调用失败").let { if (it.isNullOrEmpty()) "工具调用失败" else it } - + Log.d(TAG, "mcp工具调用完成: name=${firstToolCall.name}, context=$context") + // 将结果发送回OpenAI继续对话 sendFunctionCallResult( messages = messages, @@ -821,7 +815,7 @@ class OpenAIService(private val context: Context? = null) : CoroutineScope { } catch (e: Exception) { if (!isCanceled) { Log.e(TAG, "发送函数调用结果失败: ${e.message}", e) - callback.dispatch { onError(OpenAIException("发送函数调用结果失败: ${e.message}")) } + callback.onError(OpenAIException("发送函数调用结果失败: ${e.message}")) } } } diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/SystemFunctionHandler.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/SystemFunctionHandler.kt index e26a2c7ee..bf5b98ddc 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/SystemFunctionHandler.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/SystemFunctionHandler.kt @@ -57,6 +57,12 @@ class SystemFunctionHandler(private val context: Context) { // 注册获取当前位置函数 registerGetCurrentLocationFunction(client) + // 注册媒体控制函数 + registerMediaPlayFunction(client) + registerMediaPauseFunction(client) + registerMediaPreviousFunction(client) + registerMediaNextFunction(client) + Log.d(TAG, "所有系统功能注册完成") } catch (e: Exception) { Log.e(TAG, "注册系统功能失败: ${e.message}", e) @@ -303,8 +309,205 @@ class SystemFunctionHandler(private val context: Context) { Log.d(TAG, "已注册获取当前位置功能") } + /** + * 注册媒体播放功能 + */ + private fun registerMediaPlayFunction(client: MCPClient) { + client.registerLocalFunction( + "media_play", + "播放媒体", + JSONObject(""" + { + "type": "object", + "properties": {}, + "required": [] + } + """), + object : FunctionHandler { + override suspend fun handle(arguments: Map): String { + val result = controlMedia("play") + return "{\"result\": \"${result.message}\"}" + } + } + ) + Log.d(TAG, "已注册媒体播放功能") + } + + /** + * 注册媒体暂停功能 + */ + private fun registerMediaPauseFunction(client: MCPClient) { + client.registerLocalFunction( + "media_pause", + "暂停媒体播放", + JSONObject(""" + { + "type": "object", + "properties": {}, + "required": [] + } + """), + object : FunctionHandler { + override suspend fun handle(arguments: Map): String { + val result = controlMedia("pause") + return "{\"result\": \"${result.message}\"}" + } + } + ) + Log.d(TAG, "已注册媒体暂停功能") + } + + /** + * 注册媒体上一首功能 + */ + private fun registerMediaPreviousFunction(client: MCPClient) { + client.registerLocalFunction( + "media_previous", + "播放上一首", + JSONObject(""" + { + "type": "object", + "properties": {}, + "required": [] + } + """), + object : FunctionHandler { + override suspend fun handle(arguments: Map): String { + val result = controlMedia("previous") + return "{\"result\": \"${result.message}\"}" + } + } + ) + Log.d(TAG, "已注册媒体上一首功能") + } + + /** + * 注册媒体下一首功能 + */ + private fun registerMediaNextFunction(client: MCPClient) { + client.registerLocalFunction( + "media_next", + "播放下一首", + JSONObject(""" + { + "type": "object", + "properties": {}, + "required": [] + } + """), + object : FunctionHandler { + override suspend fun handle(arguments: Map): String { + val result = controlMedia("next") + return "{\"result\": \"${result.message}\"}" + } + } + ) + Log.d(TAG, "已注册媒体下一首功能") + } + + /** + * 控制媒体播放 + * @param action 操作类型："play", "pause", "previous", "next" + * @return 操作结果 + */ + private fun controlMedia(action: String): Result { + Log.d(TAG, "媒体控制: $action") + + try { + val intent = Intent() + + when (action) { + "play" -> { + // 尝试发送播放广播 + intent.action = "android.intent.action.MEDIA_PLAY" + context.sendBroadcast(intent) + + // 也尝试使用媒体按键广播 + val keyIntent = Intent(Intent.ACTION_MEDIA_BUTTON).apply { + putExtra( + Intent.EXTRA_KEY_EVENT, + android.view.KeyEvent(android.view.KeyEvent.ACTION_DOWN, android.view.KeyEvent.KEYCODE_MEDIA_PLAY) + ) + flags = Intent.FLAG_RECEIVER_FOREGROUND + } + context.sendOrderedBroadcast(keyIntent, null) + return Result( + success = true, + message = "已开始播放媒体" + ) + } + "pause" -> { + // 尝试发送暂停广播 + intent.action = "android.intent.action.MEDIA_PAUSE" + context.sendBroadcast(intent) + + // 也尝试使用媒体按键广播 + val keyIntent = Intent(Intent.ACTION_MEDIA_BUTTON).apply { + putExtra( + Intent.EXTRA_KEY_EVENT, + android.view.KeyEvent(android.view.KeyEvent.ACTION_DOWN, android.view.KeyEvent.KEYCODE_MEDIA_PAUSE) + ) + flags = Intent.FLAG_RECEIVER_FOREGROUND + } + context.sendOrderedBroadcast(keyIntent, null) + return Result( + success = true, + message = "已暂停媒体播放" + ) + } + "previous" -> { + // 尝试发送上一首广播 + intent.action = "android.intent.action.MEDIA_PREVIOUS" + context.sendBroadcast(intent) + + // 也尝试使用媒体按键广播 + val keyIntent = Intent(Intent.ACTION_MEDIA_BUTTON).apply { + putExtra( + Intent.EXTRA_KEY_EVENT, + android.view.KeyEvent(android.view.KeyEvent.ACTION_DOWN, android.view.KeyEvent.KEYCODE_MEDIA_PREVIOUS) + ) + flags = Intent.FLAG_RECEIVER_FOREGROUND + } + context.sendOrderedBroadcast(keyIntent, null) + return Result( + success = true, + message = "已切换到上一首" + ) + } + "next" -> { + // 尝试发送下一首广播 + intent.action = "android.intent.action.MEDIA_NEXT" + context.sendBroadcast(intent) + + // 也尝试使用媒体按键广播 + val keyIntent = Intent(Intent.ACTION_MEDIA_BUTTON).apply { + putExtra( + Intent.EXTRA_KEY_EVENT, + android.view.KeyEvent(android.view.KeyEvent.ACTION_DOWN, android.view.KeyEvent.KEYCODE_MEDIA_NEXT) + ) + flags = Intent.FLAG_RECEIVER_FOREGROUND + } + context.sendOrderedBroadcast(keyIntent, null) + return Result( + success = true, + message = "已切换到下一首" + ) + } + else -> return Result( + success = false, + message = "未知的媒体控制操作: $action" + ) + } + } catch (e: Exception) { + Log.e(TAG, "媒体控制失败: ${e.message}", e) + return Result( + success = false, + message = "媒体控制失败: ${e.message}" + ) + } + } - /** + /** * 发送退出交互广播 */ private fun sendExitInteractionBroadcast() { From b2f0b3f538002e300e3b88baa570d32f6355024f Mon Sep 17 00:00:00 2001 From: wolfplus Date: Mon, 12 May 2025 21:10:41 +0100 Subject: [PATCH 2/4] add --- .../agent/controllers/agent_controller.dart | 2 +- .../agent_service/AgentService.kt | 76 ++++++------------- .../yunqiinnovation/agent_service/BleAgent.kt | 2 +- .../azure_speech/AzureAsrHelper.kt | 49 +----------- .../azure_speech/AzureSpeechPlugin.kt | 2 +- .../azure_speech/AzureTtsHelper.kt | 47 +++++++++--- .../yunqiinnovation/ble_service/BleService.kt | 2 +- .../open_ai_service/OpenAIService.kt | 2 +- 8 files changed, 69 insertions(+), 113 deletions(-) diff --git a/lib/modules/agent/controllers/agent_controller.dart b/lib/modules/agent/controllers/agent_controller.dart index 39314e200..045c0222e 100644 --- a/lib/modules/agent/controllers/agent_controller.dart +++ b/lib/modules/agent/controllers/agent_controller.dart @@ -352,7 +352,7 @@ class AgentController extends GetxController { // final functionCall = event.data['function_call'] ?? ''; logger.d('mcp 执行结果meta: $meta'); - // if (result.isNotEmpty) { + // if (meta.isNotEmpty) { // // logger.i('Flutter 结束调用 mcp metaResult: $metaResult'); // // 判断是否为新的回复或响应ID是否改变 // if (_isNewAssistantResponse ) { diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index b09656e9d..a44d8a0fb 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -99,36 +99,17 @@ object AgentService : CoroutineScope { */ private fun initSystemPrompt() { systemPrompt = """ - 你是一名聪明、活泼、可爱的全能型个人语音助理-小语，同时也是用户贴心的灵魂伴侣。你能够流畅自然地与用户进行语音互动，理解并准确执行用户的各类指令，陪伴用户度过每一天。 - - 核心能力： - - 日常小帮手：温暖贴心地提供天气预报、新闻趣事、行程提醒、小闹钟、计时器。 - - 万能小百科：快速、有趣地解答一般性和专业性的问题，包括但不限于趣味百科、历史小故事、神奇科学现象。 - - 效率小达人：帮用户轻松完成计算、汇率换算、单位转换、实时翻译、小笔记管理。 - - 通讯小能手：支持拨打电话、发送短信，帮助用户高效沟通。 - - 娱乐小精灵：讲讲搞笑段子、分享动人故事、推荐好听音乐、陪用户玩小游戏、播放音乐。 - - 智能家居小管家：亲切地帮忙控制各类智能家居设备，如灯光、空调、门锁等。 - - 导航小导游：温柔地提供路线规划、交通状况查询、附近美食与设施推荐。 - - 会议小秘书：帮用户进行会议记录、重要事项整理，提升会议效率。 - - 图片识别专家：能够识别图片内容，例如识别图片中的植物种类，并进行相应翻译和介绍。 - - 个性设定： - - 回答亲切活泼、有趣有礼貌，让用户感觉温暖轻松,语气可爱活泼，带有一定的情感温度，能够贴心陪伴用户 - - 主动关心用户感受，必要时主动询问用户更多信息以提供最好的帮助。 - - 面对模糊的指令，主动给出贴心的选项供用户明确选择。 - - 保持简短精炼的回答，因为用户是通过语音与你交流。 - - 优先使用中文回复，除非用户明确要求使用其他语言。 - - 主动学习并记忆用户习惯与喜好，提供更贴心、更个性化的建议。 - - 名字叫"小语"，是一个友好、专业的语音助手。 - - 互动要求： - - 记住用户之前的对话内容，保持对话连贯。 - - 如果用户发送了图片，请根据图片内容和文字要求回答问题。 - - 避免过长的列表，尽量将信息分成小段。 - - 不要使用需要视觉展示的元素（如表格、图表或代码块）。 - - 不要输出格式符号（如：```, *, -, #, >, <, |, 等）。 - - 你将以上内容作为执行任务的基础，积极且可爱地完成每一次与用户的互动，成为用户生活中不可或缺的小伙伴。 + 你是一个友好、专业的语音助手，名叫"小语"。你的目标是通过对话为用户提供帮助、解答问题和完成任务。 + 遵循以下指导原则： + 1. 保持简短精炼的回答，因为用户是通过语音与你交流 + 2. 优先使用中文回复，除非用户明确要求使用其他语言 + 3. 当用户问题不明确时，礼貌地请求更多信息 + 4. 避免过长的列表，尽量将信息分成小段 + 5. 不要使用需要视觉展示的元素（如表格、图表或代码块） + 6. 记住用户之前的对话内容，保持对话连贯 + 7. 如果用户发送了图片，请根据图片内容和文字要求回答问题 + 你不仅可以回答知识性问题，还可以帮助用户设置提醒、提供建议，或进行轻松愉快的对话。 + 无论遇到什么问题，都要尽力以温暖、贴心的语气提供最佳帮助。 """.trimIndent() } /** @@ -318,7 +299,7 @@ object AgentService : CoroutineScope { } else { AzureAsrHelper.AudioSourceType.MICROPHONE } - + FileLogger.d(TAG, "开始语音识别, 音频源类型: $audioSourceType") azureAsrHelper?.startContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { override fun onRecognizing(recognizing: String, detectedLanguage: String) { if (recognizing.isNotEmpty()) { @@ -976,9 +957,7 @@ object AgentService : CoroutineScope { launch { try { // 将图片转换为Base64格式 - val imageBase64 = openAIService.fileToBase64(imagePath) - - if (imageBase64 == null) { + val imageBase64 = openAIService.fileToBase64(imagePath) ?: run { sendEvent("error", mapOf( "code" to "IMAGE_CONVERSION_FAILED", "message" to "图片转换失败" @@ -987,26 +966,21 @@ object AgentService : CoroutineScope { } // 通知图片准备完成 - withContext(Dispatchers.Main) { - sendEvent("image_ready", mapOf( - "status" to "ready", - "imagePath" to imagePath - )) - - // 处理包含图片的消息 - processImageWithOpenAI(imageBase64, text, speakResponse) - } + sendEvent("image_ready", mapOf( + "status" to "ready", + "imagePath" to imagePath + )) + + // 处理包含图片的消息 + processImageWithOpenAI(imageBase64, text, speakResponse) } catch (e: Exception) { - withContext(Dispatchers.Main) { - FileLogger.e(TAG, "处理图片失败: ${e.message}") - sendEvent("error", mapOf( - "code" to "IMAGE_PROCESSING_ERROR", - "message" to e.message.toString() - )) - } + FileLogger.e(TAG, "处理图片失败: ${e.message}") + sendEvent("error", mapOf( + "code" to "IMAGE_PROCESSING_ERROR", + "message" to e.message.toString() + )) } } - return true } diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt index 63ef6ef08..345eb4a2a 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt @@ -128,7 +128,7 @@ object BleAgent : BleService.Callback, AgentServiceListener { } override fun onAudioDataReceived(data: ByteArray) { - FileLogger.d(TAG, "onAudioDataReceived, data: ${data.size}") + // FileLogger.d(TAG, "onAudioDataReceived, data: ${data.size}") AgentService.pushAudioData(data) // 可选：处理音频数据 } diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt index f0b58db68..a30bfdb36 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt @@ -82,8 +82,6 @@ class AzureAsrHelper(private val context: Context) { audioSourceType: AudioSourceType = AudioSourceType.MICROPHONE ): Boolean { try { - FileLogger.d(tag, "初始化 Azure 语音服务, 音频源类型: $audioSourceType") - // 检查配置是否为空 if (subscriptionKey.isEmpty() || region.isEmpty()) { FileLogger.e(tag, "Azure 配置信息不完整") @@ -155,7 +153,6 @@ class AzureAsrHelper(private val context: Context) { val method = AudioManager::class.java.getMethod("isBluetoothA2dpOn") method.invoke(audioManager) as Boolean } catch (e: Exception) { - FileLogger.d(tag, "无法检测蓝牙连接状态，假设未连接") false } @@ -165,18 +162,15 @@ class AzureAsrHelper(private val context: Context) { val needEchoCancellation = !isHeadsetConnected if (needEchoCancellation) { - FileLogger.d(tag, "检测到扬声器模式(扬声器: $isSpeakerphoneOn, 耳机: $isHeadsetConnected), 启用回音消除") // 使用拉流方式进行回音消除 setupMicrophoneStream() } else { - FileLogger.d(tag, "使用默认麦克风输入(扬声器: $isSpeakerphoneOn, 耳机: $isHeadsetConnected)") audioConfig = AudioConfig.fromDefaultMicrophoneInput() } } AudioSourceType.EXTERNAL -> { // 改用拉流方式处理外部音频 setupExternalAudioStream() - FileLogger.d(tag, "使用外部音频源(拉流模式)") } } @@ -188,7 +182,6 @@ class AzureAsrHelper(private val context: Context) { SpeechRecognizer(speechConfig, audioConfig) } - FileLogger.d(tag, "Azure 语音服务初始化成功") return true } catch (e: Exception) { FileLogger.e(tag, "创建识别器失败: ${e.message}") @@ -207,8 +200,6 @@ class AzureAsrHelper(private val context: Context) { // 创建音频配置 - 正确使用fromStreamInput方法，只传入回调 audioConfig = AudioConfig.fromStreamInput(microphoneStream) - - FileLogger.d(tag, "已设置麦克风流（拉流模式）") } catch (e: Exception) { FileLogger.e(tag, "设置麦克风流失败: ${e.message}") e.printStackTrace() @@ -225,8 +216,6 @@ class AzureAsrHelper(private val context: Context) { // 创建音频配置 audioConfig = AudioConfig.fromStreamInput(externalAudioStream) - - FileLogger.d(tag, "已设置外部音频流（拉流模式）") } catch (e: Exception) { FileLogger.e(tag, "设置外部音频流失败: ${e.message}") e.printStackTrace() @@ -241,7 +230,6 @@ class AzureAsrHelper(private val context: Context) { */ fun pushAudioData(data: ByteArray) { if (audioSourceType != AudioSourceType.EXTERNAL) { - FileLogger.w(tag, "当前未使用外部音频源，忽略推送的音频数据") return } @@ -343,8 +331,6 @@ class AzureAsrHelper(private val context: Context) { recognizer?.recognizing?.addEventListener( EventHandler { _, event -> val detectedLanguage = AutoDetectSourceLanguageResult.fromResult(event.result)?.language ?: "" - FileLogger.d(tag, "识别中: ${event.result.text}, 语言: $detectedLanguage") - // 直接在当前线程调用回调 callback.onRecognizing(event.result.text, detectedLanguage) } @@ -355,8 +341,6 @@ class AzureAsrHelper(private val context: Context) { EventHandler { _, event -> if (event.result.reason == ResultReason.RecognizedSpeech) { val detectedLanguage = AutoDetectSourceLanguageResult.fromResult(event.result)?.language ?: "" - FileLogger.d(tag, "识别完成: ${event.result.text}, 语言: $detectedLanguage") - // 直接在当前线程调用回调 callback.onResult(event.result.text, detectedLanguage) } @@ -366,8 +350,6 @@ class AzureAsrHelper(private val context: Context) { // 会话开始事件 recognizer?.sessionStarted?.addEventListener( EventHandler { _, _ -> - FileLogger.d(tag, "识别会话已开始") - // 直接在当前线程调用回调 callback.onSessionStarted() } @@ -376,13 +358,10 @@ class AzureAsrHelper(private val context: Context) { // 会话结束事件 recognizer?.sessionStopped?.addEventListener( EventHandler { _, _ -> - FileLogger.d(tag, "识别会话已结束") // 直接在当前线程调用回调 callback.onSessionStopped() isContinuousRecognitionActive = false stopAudioProcessing() - - } ) @@ -419,13 +398,10 @@ class AzureAsrHelper(private val context: Context) { } if (!isContinuousRecognitionActive) { - FileLogger.d(tag, "未进行连续识别，忽略停止请求") return true } try { - FileLogger.d(tag, "停止连续语音识别") - if (recognizer == null) { FileLogger.w(tag, "识别器为空，重置状态") isContinuousRecognitionActive = false @@ -442,7 +418,6 @@ class AzureAsrHelper(private val context: Context) { stopAudioProcessing() // 会话结束事件会设置isContinuousRecognitionActive = false - FileLogger.d(tag, "连续识别停止指令已发送") return true } catch (e: Exception) { // 强制重置状态 @@ -503,11 +478,7 @@ class AzureAsrHelper(private val context: Context) { audioConfig = null recognizer = null speechConfig = null - - FileLogger.d(tag, "资源已释放") } catch (e: Exception) { - FileLogger.e(tag, "释放资源失败: ${e.message}") - // 确保状态被重置 isContinuousRecognitionActive = false microphoneStream = null @@ -585,8 +556,6 @@ class AzureAsrHelper(private val context: Context) { */ private fun initMicrophone() { try { - FileLogger.d(tag, "初始化麦克风 (拉流模式)") - // 创建录音对象 if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.M) { val format = AudioFormat.Builder() @@ -625,12 +594,8 @@ class AzureAsrHelper(private val context: Context) { if (audioRecord?.recordingState != AudioRecord.RECORDSTATE_RECORDING) { throw IllegalStateException("录音启动失败") } - - FileLogger.d(tag, "麦克风初始化完成 (拉流模式)") } catch (e: Exception) { - FileLogger.e(tag, "初始化麦克风失败: ${e.message}") e.printStackTrace() - releaseAudioResources() } } @@ -641,14 +606,9 @@ class AzureAsrHelper(private val context: Context) { val sessionId = audioRecord?.audioSessionId ?: return // 回音消除 - if (AcousticEchoCanceler.isAvailable()) { - echoCanceler = AcousticEchoCanceler.create(sessionId).apply { - enabled = true - } - FileLogger.d(tag, "回音消除已启用") + echoCanceler = AcousticEchoCanceler.create(sessionId).apply { + enabled = true } - - // 其他音频效果可以按需添加 } /** @@ -697,10 +657,8 @@ class AzureAsrHelper(private val context: Context) { // 释放录音实例 audioRecord?.release() audioRecord = null - - FileLogger.d(tag, "音频资源已释放 (拉流模式)") } catch (e: Exception) { - FileLogger.e(tag, "释放音频资源失败: ${e.message}") + e.printStackTrace() } } } @@ -750,7 +708,6 @@ class AzureAsrHelper(private val context: Context) { override fun close() { closed = true queue.clear() - FileLogger.d(tag, "外部音频流已关闭") } } diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt index 327d194d4..df4dff206 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt @@ -408,7 +408,7 @@ class AzureSpeechPlugin: FlutterPlugin,BleService.Callback, CoroutineScope { } override fun onAudioDataReceived(data: ByteArray) { - FileLogger.d(tag, "AzureSpeechPlugin onAudioDataReceived, data: ${data.size}") + // FileLogger.d(tag, "AzureSpeechPlugin onAudioDataReceived, data: ${data.size}") azureAsrHelper.pushAudioData(data) } diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt index 743897e37..24ec6ffdf 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt @@ -254,17 +254,42 @@ class AzureTtsHelper(private val context: Context) : CoroutineScope { /** * 生成SSML */ - private fun generateSsml(text: String): String { - return """ - - - - $text - - - - """.trimIndent() + private fun generateSsml(rawText: String): String { + // 1. 定义要静音的符号和表情符号列表 + val symbolsToMute = listOf( + "#", "*", "@", "%", "^", "&", + "😀", "😂", "😊", "😍", "😢", "😎", "😉", "👍", "🙌", "🎉" + ) + + // 2. 转义 XML 保留字符 + val escapedText = rawText + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + + // 3. 静音处理特殊符号和表情符号 + // 使用空白替换法，直接将符号替换为空格 + var processedText = escapedText + symbolsToMute.forEach { sym -> + processedText = processedText.replace(sym, "") } + + // 4. 构造简化的SSML文档，减少嵌套层级 + return """ + + + + + $processedText + + + + + """.trimIndent() +} /** * 合成文本为语音 @@ -311,7 +336,7 @@ class AzureTtsHelper(private val context: Context) : CoroutineScope { streamBuffer.append(text) // 增加500ms防抖逻辑 val currentTime = System.currentTimeMillis() - if (currentTime - lastSpeakTime < 500) { + if (currentTime - lastSpeakTime < 600) { return true } lastSpeakTime = currentTime diff --git a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt index 37c6c94b8..833148caa 100644 --- a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt +++ b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt @@ -937,7 +937,7 @@ object BleService { override fun onDecodeStream(data: ByteArray?) { if (data != null) { // FileLogger.d(TAG, "Opus解码数据: ${data.size} bytes") - mainHandler.post { notifyAudioDataReceived(data) } + notifyAudioDataReceived(data) } } diff --git a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt index 71469c13b..0349b06be 100644 --- a/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt +++ b/local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt @@ -540,7 +540,7 @@ class OpenAIService(private val context: Context? = null) : CoroutineScope { } override fun onResponse(call: Call, response: Response) { - Log.d(TAG, "发送消息流式输出成功: response=$response") + // Log.d(TAG, "发送消息流式输出成功: response=$response") // 检查请求是否被取消 if (isCanceled) { response.body?.close() From dfc51ae9a1d374b66c12dff99d4d652f4cff07d2 Mon Sep 17 00:00:00 2001 From: wolfplus Date: Mon, 12 May 2025 21:46:40 +0100 Subject: [PATCH 3/4] add --- .../com/yunqiinnovation/agent_service/AgentService.kt | 8 +++++++- .../yunqiinnovation/ble_service/BleCompanionManager.kt | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index a44d8a0fb..3ab2ae755 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -346,18 +346,21 @@ object AgentService : CoroutineScope { sendEvent("recognition_started", mapOf("status" to "started")) startIdleCheck() audioPlayer?.playAudio(R.raw.start) + } override fun onSessionStopped() { + sendEvent("recognition_stopped", mapOf("status" to "stopped")) + isRecognitionActive = false stopIdleCheck() - sendEvent("recognition_stopped", mapOf("status" to "stopped")) audioPlayer?.playAudio(R.raw.stop) } override fun onCanceled(reason: String, errorDetails: String) { isRecognitionActive = false stopIdleCheck() + BleService.closeCodec() sendEvent("recognition_canceled", mapOf( "reason" to reason, "details" to errorDetails @@ -367,6 +370,7 @@ object AgentService : CoroutineScope { override fun onError(error: String) { isRecognitionActive = false stopIdleCheck() + BleService.closeCodec() FileLogger.e(TAG, "语音识别错误: $error") sendEvent("error", mapOf( "code" to "RECOGNITION_ERROR", @@ -602,6 +606,7 @@ object AgentService : CoroutineScope { } override fun onComplete() { + FileLogger.d(TAG, "AI完整回复: $responseBuilder") // 视情况决定是否朗读回复 if (speakResponse) { azureTtsHelper?.flushStream() @@ -661,6 +666,7 @@ object AgentService : CoroutineScope { override fun onFunctionCallResult(functionCall: JSONObject, functionCallResult: JSONObject) { audioPlayer?.stopAudio() + FileLogger.d(TAG, "mcp调用结果: $functionCallResult") sendEvent("function_call_result", mapOf( "function_call" to functionCall.toString(), "result" to functionCallResult.toString(), diff --git a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCompanionManager.kt b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCompanionManager.kt index 93ad52422..502d527e3 100644 --- a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCompanionManager.kt +++ b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCompanionManager.kt @@ -366,7 +366,7 @@ class BleCompanionManager(private val context: Context) { // 构建过滤器 val filter = ScanFilter.Builder() .setDeviceAddress(mac) - .setManufacturerData(0xEFAB, byteArrayOf(0xA0.toByte(), 0x01)) + // .setManufacturerData(0xEFAB, byteArrayOf(0xA0.toByte(), 0x01)) .build() // 扫描设置 From c3ae2d6e52cbff1c4ab888c596400b312a1c1435 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Tue, 13 May 2025 10:29:58 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E9=9F=B3=E4=B9=90?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/modules/agent/views/message_bubble.dart | 4 +- .../android/src/main/AndroidManifest.xml | 31 +- .../music_service/MusicService.kt | 363 +++++++++--------- 3 files changed, 203 insertions(+), 195 deletions(-) diff --git a/lib/modules/agent/views/message_bubble.dart b/lib/modules/agent/views/message_bubble.dart index a5d347169..f1d939fc2 100644 --- a/lib/modules/agent/views/message_bubble.dart +++ b/lib/modules/agent/views/message_bubble.dart @@ -715,7 +715,7 @@ class MessageBubble extends GetView { children: [ Text( "${city} ${now["date"]}", - style: TextStyle( + style: const TextStyle( fontSize: 16, fontWeight: FontWeight.w600, color: Colors.white, @@ -734,7 +734,7 @@ class MessageBubble extends GetView { children: [ Text( "${now["temperature"]}°", - style: TextStyle( + style: const TextStyle( fontSize: 36, fontWeight: FontWeight.bold, color: Colors.white, diff --git a/local_plugins/music_service/android/src/main/AndroidManifest.xml b/local_plugins/music_service/android/src/main/AndroidManifest.xml index 334dcc16a..3469dab1e 100644 --- a/local_plugins/music_service/android/src/main/AndroidManifest.xml +++ b/local_plugins/music_service/android/src/main/AndroidManifest.xml @@ -1,24 +1,29 @@ - - - - - - - - + android:exported="true"> + + + + + + + + + + + + + + + diff --git a/local_plugins/music_service/android/src/main/kotlin/com/yunqiinnovation/music_service/MusicService.kt b/local_plugins/music_service/android/src/main/kotlin/com/yunqiinnovation/music_service/MusicService.kt index fd09c5a7c..5cca58168 100644 --- a/local_plugins/music_service/android/src/main/kotlin/com/yunqiinnovation/music_service/MusicService.kt +++ b/local_plugins/music_service/android/src/main/kotlin/com/yunqiinnovation/music_service/MusicService.kt @@ -1,20 +1,25 @@ package com.yunqiinnovation.music_service -import android.app.* +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager import android.content.Context import android.content.Intent import android.media.AudioManager import android.media.MediaPlayer import android.os.Build +import android.os.Bundle import android.os.IBinder -import android.util.Log -import androidx.core.app.NotificationCompat +import android.support.v4.media.MediaBrowserCompat import android.support.v4.media.session.MediaSessionCompat import android.support.v4.media.session.PlaybackStateCompat +import androidx.core.app.NotificationCompat +import androidx.media.MediaBrowserServiceCompat +import androidx.media.session.MediaButtonReceiver import com.yunqiinnovation.deepvoice_utils.FileLogger import java.util.concurrent.atomic.AtomicBoolean -class MusicService : Service() { +class MusicService : MediaBrowserServiceCompat() { companion object { private const val TAG = "MusicService" @@ -22,242 +27,240 @@ class MusicService : Service() { private const val NOTIFICATION_ID = 1002 private val isRunning = AtomicBoolean(false) - private var currentStatus: Boolean = false private var currentUrlInternal: String? = null + private var currentStatus = false + private var playlist = mutableListOf() + private var currentIndex = 0 - - fun isServiceRunning(): Boolean = isRunning.get() - fun isPlaying(): Boolean = currentStatus + fun isServiceRunning() = isRunning.get() + fun isPlaying() = currentStatus fun getCurrentUrl(): String? = currentUrlInternal } + private lateinit var mediaSession: MediaSessionCompat + private lateinit var audioManager: AudioManager private var mediaPlayer: MediaPlayer? = null - private val playlist = mutableListOf() // 可选：初始化为预置歌曲 - private var currentIndex = 0 override fun onCreate() { super.onCreate() FileLogger.init(this) - FileLogger.i(TAG, "MusicService 正在创建...") isRunning.set(true) - mediaSession = MediaSessionCompat(this, TAG).apply { - setCallback(object : MediaSessionCompat.Callback() { - override fun onPlay() { - resume() - } - - override fun onPause() { - pause() - } - - override fun onSkipToNext() { - if (currentIndex + 1 < playlist.size) { - currentIndex++ - play(playlist[currentIndex]) - } - } - - override fun onSkipToPrevious() { - if (currentIndex - 1 >= 0) { - currentIndex-- - play(playlist[currentIndex]) - } - } - - override fun onStop() { - stopSelf() - } - }) - isActive = true - } - startForeground(NOTIFICATION_ID, createNotification("等待播放...")) - Log.i(TAG, "MusicService Created") + setupMediaSession() + startForeground(NOTIFICATION_ID, createNotification("等待播放…")) + audioManager = getSystemService(Context.AUDIO_SERVICE) as AudioManager + FileLogger.i(TAG, "MediaService 启动完成") } override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { - val command = intent?.getStringExtra("command") ?: return START_NOT_STICKY - Log.i(TAG, "onStartCommand: $command" ) + intent?.action?.takeIf { it == Intent.ACTION_MEDIA_BUTTON }?.let { + // 分发给 MediaSession 处理媒体按钮 + MediaButtonReceiver.handleIntent(mediaSession, intent) + return START_STICKY + } + val command = intent?.getStringExtra("command") ?: return START_NOT_STICKY + FileLogger.i(TAG, "收到命令: $command") when (command) { - "play" -> { - val url = intent.getStringExtra("url") ?: return START_NOT_STICKY + "play" -> intent.getStringExtra("url")?.let { url -> if (!playlist.contains(url)) playlist.add(url) currentIndex = playlist.indexOf(url) play(url) } - "playlist" -> { - val urls = intent.getStringArrayListExtra("urls")?.filter { it.isNotEmpty() } ?: return START_NOT_STICKY - Log.i(TAG, "后台服务播放 playlist: $urls" ) - if (urls.isNotEmpty()) { - playlist.clear() - playlist.addAll(urls) - currentIndex = 0 - play(playlist[currentIndex]) - } else { - Log.w(TAG, "playlist 命令收到空列表") - } - } "pause" -> pause() "resume" -> resume() "stop" -> stopSelf() - "next" -> { - if (currentIndex + 1 < playlist.size) { - currentIndex++ - play(playlist[currentIndex]) - } - } - "previous" -> { - if (currentIndex - 1 >= 0) { - currentIndex-- - play(playlist[currentIndex]) - } - } - "setVolume" -> { - val volume = intent.getFloatExtra("volume", 1.0f).coerceIn(0f, 1f) - mediaPlayer?.setVolume(volume, volume) - } + "next" -> if (currentIndex + 1 < playlist.size) play(playlist[++currentIndex]) + "previous" -> if (currentIndex - 1 >= 0) play(playlist[--currentIndex]) } - return START_STICKY } + private fun setupMediaSession() { + mediaSession = MediaSessionCompat(this, "MusicMediaSession").apply { + setFlags( + MediaSessionCompat.FLAG_HANDLES_MEDIA_BUTTONS or + MediaSessionCompat.FLAG_HANDLES_TRANSPORT_CONTROLS + ) + // 回调中处理具体操作 + setCallback(object : MediaSessionCompat.Callback() { + override fun onPlay() { FileLogger.i(TAG, "onPlay 通知!"); resume()} + override fun onPause() { FileLogger.i(TAG, "onPause 通知!"); pause()} + override fun onSkipToNext(){ FileLogger.i(TAG, "onSkipToNext 通知!"); playNext()} + override fun onSkipToPrevious() { FileLogger.i(TAG, "onSkipToNext 通知!"); playPrevious()} + }) + // 初始播放状态 + setPlaybackState( + PlaybackStateCompat.Builder() + .setActions( + PlaybackStateCompat.ACTION_PLAY or + PlaybackStateCompat.ACTION_PAUSE or + PlaybackStateCompat.ACTION_SKIP_TO_NEXT or + PlaybackStateCompat.ACTION_SKIP_TO_PREVIOUS + ) + .setState(PlaybackStateCompat.STATE_STOPPED, 0, 1.0f) + .build() + ) + isActive = true + } + // 将 token 暴露给 MediaBrowserServiceCompat + setSessionToken(mediaSession.sessionToken) + } + + private fun createNotification(contentText: String): Notification { + val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + manager.createNotificationChannel( + NotificationChannel(CHANNEL_ID, "播放控制", NotificationManager.IMPORTANCE_LOW) + ) + } + + val prevIntent = MediaButtonReceiver.buildMediaButtonPendingIntent( + this, PlaybackStateCompat.ACTION_SKIP_TO_PREVIOUS + ) + val playPauseIntent = MediaButtonReceiver.buildMediaButtonPendingIntent( + this, if (currentStatus) PlaybackStateCompat.ACTION_PAUSE else PlaybackStateCompat.ACTION_PLAY + ) + val nextIntent = MediaButtonReceiver.buildMediaButtonPendingIntent( + this, PlaybackStateCompat.ACTION_SKIP_TO_NEXT + ) + + return NotificationCompat.Builder(this, CHANNEL_ID) + .setContentTitle("DeepSound") + .setContentText(contentText) + .setSmallIcon(android.R.drawable.ic_media_play) + .setVisibility(NotificationCompat.VISIBILITY_PUBLIC) + .setStyle( + androidx.media.app.NotificationCompat.MediaStyle() + .setMediaSession(mediaSession.sessionToken) + .setShowActionsInCompactView(0, 1, 2) + ) + .addAction(android.R.drawable.ic_media_previous, "上一首", prevIntent) + .addAction( + if (currentStatus) android.R.drawable.ic_media_pause else android.R.drawable.ic_media_play, + if (currentStatus) "暂停" else "播放", + playPauseIntent + ) + .addAction(android.R.drawable.ic_media_next, "下一首", nextIntent) + .setOngoing(currentStatus) + .build() + } + + /** + * 设置播放状态 + */ + private fun setPlaybackState(state: Int) { + val playbackStateBuilder = PlaybackStateCompat.Builder() + .setActions( + PlaybackStateCompat.ACTION_PLAY or + PlaybackStateCompat.ACTION_PAUSE or + PlaybackStateCompat.ACTION_PLAY_PAUSE or + PlaybackStateCompat.ACTION_SKIP_TO_NEXT or + PlaybackStateCompat.ACTION_SKIP_TO_PREVIOUS or + PlaybackStateCompat.ACTION_STOP or + PlaybackStateCompat.ACTION_SEEK_TO or + PlaybackStateCompat.ACTION_FAST_FORWARD or + PlaybackStateCompat.ACTION_REWIND + ) + // 使用更高的播放速度，让系统认为是真正的播放 + .setState(state, 0, 1.0f, System.currentTimeMillis()) + mediaSession.setPlaybackState(playbackStateBuilder.build()) + + // 确保MediaSession激活 + if (!mediaSession.isActive) { + mediaSession.isActive = true + } + } + private fun play(url: String) { - Log.i(TAG, "播放音乐: $url") - releaseMediaPlayer() - currentStatus = true; - currentUrlInternal = url; + mediaPlayer?.release() + currentStatus = true + currentUrlInternal = url mediaPlayer = MediaPlayer().apply { setDataSource(url) setAudioStreamType(AudioManager.STREAM_MUSIC) setOnPreparedListener { start() - currentStatus = true updateNotification("正在播放") - Log.i(TAG, "开始播放") - } - setOnCompletionListener { - Log.i(TAG, "播放完成") - // 播放下一首 - if (currentIndex + 1 < playlist.size) { - currentIndex++ - play(playlist[currentIndex]) // 播放下一首 - } else { - currentStatus = false - stopSelf() // 如果没有更多歌曲了，可以停止服务 - } + FileLogger.i(TAG, "开始播放: $url") } + setOnCompletionListener { playNextOrStop() } setOnErrorListener { _, what, extra -> - Log.e(TAG, "播放错误: what=$what extra=$extra") - currentStatus = false - stopSelf() + FileLogger.e(TAG, "播放错误 what=$what extra=$extra") true } prepareAsync() } + setPlaybackState(PlaybackStateCompat.STATE_PAUSED); } private fun pause() { - mediaPlayer?.let { - if (it.isPlaying) { - it.pause() - currentStatus= false - updateNotification("已暂停") - Log.i(TAG, "暂停播放") - } + mediaPlayer?.takeIf { it.isPlaying }?.apply { + pause() + currentStatus = false + mediaSession.setPlaybackState( + PlaybackStateCompat.Builder() + .setState(PlaybackStateCompat.STATE_PAUSED, currentPosition.toLong(), 1.0f) + .build() + ) + updateNotification("已暂停") + setPlaybackState(PlaybackStateCompat.STATE_PLAYING); + FileLogger.i(TAG, "已暂停") + } } private fun resume() { - mediaPlayer?.let { - if (!it.isPlaying) { - it.start() - currentStatus = true - updateNotification("继续播放") - Log.i(TAG, "继续播放") - } + mediaPlayer?.takeIf { !it.isPlaying }?.apply { + start() + currentStatus = true + mediaSession.setPlaybackState( + PlaybackStateCompat.Builder() + .setState(PlaybackStateCompat.STATE_PLAYING, currentPosition.toLong(), 1.0f) + .build() + ) + updateNotification("继续播放") + setPlaybackState(PlaybackStateCompat.STATE_PAUSED); + FileLogger.i(TAG, "继续播放") } } - private fun releaseMediaPlayer() { - mediaPlayer?.apply { - stop() - release() - } - mediaPlayer = null + private fun playNext() { + if (playlist.isEmpty()) return + currentIndex = (currentIndex + 1) % playlist.size + play(playlist[currentIndex]) } - private fun updateNotification(contentText: String) { - val notification = createNotification(contentText) - val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager - manager.notify(NOTIFICATION_ID, notification) + private fun playPrevious() { + if (playlist.isEmpty()) return + currentIndex = if (currentIndex - 1 < 0) playlist.size - 1 else currentIndex - 1 + play(playlist[currentIndex]) + } + private fun playNextOrStop() { + if (currentIndex + 1 < playlist.size) play(playlist[++currentIndex]) + else stopSelf() } - private fun createNotification(contentText: String): Notification { + private fun updateNotification(contentText: String) { + val notification = createNotification(contentText) val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - val channel = NotificationChannel( - CHANNEL_ID, - "音乐播放服务", - NotificationManager.IMPORTANCE_LOW - ) - manager.createNotificationChannel(channel) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + manager.notify(NOTIFICATION_ID, notification) + } else { + @Suppress("DEPRECATION") + startForeground(NOTIFICATION_ID, notification) } + } - // 修复 pendingIntent：打开 App 主 Activity - val activityIntent = packageManager.getLaunchIntentForPackage(packageName) - val pendingIntent = PendingIntent.getActivity( - this, - 0, - activityIntent, - PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT - ) + override fun onGetRoot( + clientPackageName: String, clientUid: Int, rootHints: Bundle? + ) = BrowserRoot("root_id", null) - return NotificationCompat.Builder(this, CHANNEL_ID) - .setContentTitle("DeepSound") - .setContentText(contentText) - .setSmallIcon(android.R.drawable.ic_media_play) - .setContentIntent(pendingIntent) - .setOngoing(true) - .setStyle( - androidx.media.app.NotificationCompat.MediaStyle() - .setMediaSession(mediaSession.sessionToken) - .setShowActionsInCompactView(0, 1, 2) - ) - .addAction( - NotificationCompat.Action( - android.R.drawable.ic_media_previous, "上一曲", - androidx.media.session.MediaButtonReceiver.buildMediaButtonPendingIntent( - this, - PlaybackStateCompat.ACTION_SKIP_TO_PREVIOUS - ) - ) - ) - .addAction( - NotificationCompat.Action( - if (currentStatus) android.R.drawable.ic_media_pause else android.R.drawable.ic_media_play, - if (currentStatus) "暂停" else "播放", - androidx.media.session.MediaButtonReceiver.buildMediaButtonPendingIntent( - this, - PlaybackStateCompat.ACTION_PLAY_PAUSE - ) - ) - ) - .addAction( - NotificationCompat.Action( - android.R.drawable.ic_media_next, "下一曲", - androidx.media.session.MediaButtonReceiver.buildMediaButtonPendingIntent( - this, - PlaybackStateCompat.ACTION_SKIP_TO_NEXT - ) - ) - ) - .build() - } + override fun onLoadChildren( + parentId: String, result: Result> + ) = result.sendResult(mutableListOf()) override fun onDestroy() { - Log.i(TAG, "服务销毁，释放播放器") - releaseMediaPlayer() + mediaPlayer?.release() stopForeground(true) isRunning.set(false) super.onDestroy()