diff --git a/lib/data/models/user_Info.dart b/lib/data/models/user_Info.dart index 6d64c1533..279102fd3 100644 --- a/lib/data/models/user_Info.dart +++ b/lib/data/models/user_Info.dart @@ -218,34 +218,42 @@ class User { class UserDevice { int id; String uid; + int productid; int devicetype; String devicename; String devicemac; String license; + String cmei; UserDevice({ required this.id, required this.uid, + required this.productid, required this.devicetype, required this.devicename, required this.devicemac, required this.license, + this.cmei = '', }); factory UserDevice.fromJson(Map json) => UserDevice( id: json['id'] as int, uid: json['uid'] as String, + productid: (json['productid'] as num).toInt(), devicetype: (json['devicetype'] as num).toInt(), devicename: json['devicename'] as String, devicemac: json['devicemac'] as String, license: json['license'] as String, + cmei: json['cmei'] as String? ?? '', ); toJson() => { 'id': id, 'uid': uid, + 'productid': productid, 'devicetype': devicetype, 'devicename': devicename, 'devicemac': devicemac, 'license': license, + 'cmei': cmei, }; } diff --git a/lib/data/services/ble_manager.dart b/lib/data/services/ble_manager.dart index 3550b523b..a71d35e4d 100644 --- a/lib/data/services/ble_manager.dart +++ b/lib/data/services/ble_manager.dart @@ -601,23 +601,17 @@ class BleManager extends GetxService { } } - /// 更新设备id + /// 更新设备id，并检测是否为移动精灵设备 void _updateProductIdFromData(Map data) { try { - // 确保数据中包含版本信息 if (!data.containsKey('productId')) { Logger.warning('版本信息数据不完整'); return; } - // 提取版本信息 final String productId = data['productId'] as String? ?? ''; - // 更新设备信息 - _updateDeviceInfoIfNeeded( - firmwareProductId: productId, - ); - + _updateDeviceInfoIfNeeded(firmwareProductId: productId); Logger.info('设备设备id已更新: 设备id=$productId'); } catch (e) { Logger.error('更新设备id失败: ${e.toString()}'); @@ -1035,7 +1029,42 @@ class BleManager extends GetxService { supportedLanguages = ['zh-CN', 'en-US']; } + // STS 模式参数（移动精灵） + // 优先使用设备检测结果（YIDONG_PID 匹配时写入），否则用服务端 AGENT_TYPE + final agentType = _storage.read('device_agent_type') + ?? AppConfig.env('AGENT_TYPE') + ?? 'stt_llm_tts'; + final agentMode = (agentType == 'yidong') ? 'sts' : 'stt_llm_tts'; + final agentId = (agentType == 'yidong') + ? (AppConfig.env('MOBILE_ELF_AGENT_ID') ?? 'mobile_elf') + : 'default_agent'; + final stsProductId = AppConfig.env('MOBILE_ELF_PRODUCT_ID') ?? ''; + final stsProductKey = AppConfig.env('MOBILE_ELF_PRODUCT_KEY') ?? ''; + // 使用当前连接设备的 SN（配对时写入），认证失败由原生层上报 credential_failed 事件 + final stsDeviceNo = _storage.read('device_sts_code') ?? ''; + final stsDeviceId = _storage.read('sts_device_id') ?? ''; + final stsDeviceSecret = _storage.read('sts_device_secret') ?? ''; + // 从 UserDevice 获取当前连接设备的 CMEI + final mac = currentDeviceAddress?.toUpperCase() ?? ''; + String stsCmei = ''; + try { + final matchedDevice = userInfo.User.instance.devices.firstWhere( + (d) => d.devicemac.toUpperCase() == mac, + ); + stsCmei = matchedDevice.cmei; + } catch (_) {} + Logger.info('lxm--当前设备 CMEI: $stsCmei (MAC=$mac)'); + final config = { + // STS 模式路由 + 'agentMode': agentMode, + 'agentId': agentId, + 'stsProductId': stsProductId, + 'stsProductKey': stsProductKey, + 'stsDeviceNo': stsDeviceNo, + 'stsDeviceId': stsDeviceId, + 'stsDeviceSecret': stsDeviceSecret, + 'stsCmei': stsCmei, // 语音代理配置参数 'azureSpeechKey': azureSpeechKey, 'azureSpeechRegion': azureSpeechRegion, @@ -1068,7 +1097,7 @@ class BleManager extends GetxService { Logger.warning('XUNFEI_ASR_* 未完整配置，ASR 将不会切到讯飞'); } - if (azureSpeechKey.isEmpty || openaiApiKey.isEmpty) { + if (agentMode != 'sts' && (azureSpeechKey.isEmpty || openaiApiKey.isEmpty)) { Logger.warning('缺少必要的API密钥，请检查.env文件配置'); // 检查.env中相关密钥是否存在 @@ -1097,6 +1126,19 @@ class BleManager extends GetxService { } }; + // STS 模式：监听凭证获取事件并缓存到 GetStorage + if (agentMode == 'sts') { + AgentService.events.where((e) => e.type == AgentServiceEventType.stsCredentials).listen((e) { + final deviceId = e.data['deviceId']?.toString() ?? ''; + final deviceSecret = e.data['deviceSecret']?.toString() ?? ''; + final deviceNo = e.data['deviceNo']?.toString() ?? ''; + if (deviceId.isNotEmpty) _storage.write('sts_device_id', deviceId); + if (deviceSecret.isNotEmpty) _storage.write('sts_device_secret', deviceSecret); + if (deviceNo.isNotEmpty) _storage.write('sts_device_no', deviceNo); + Logger.info('STS 凭证已缓存: deviceId=$deviceId deviceNo=$deviceNo'); + }); + } + Logger.info('lxm--准备调用AgentService.startAgentService启动BLE服务'); final result = await AgentService.startAgentService(config); Logger.info('lxm--AgentService.startAgentService调用结果: $result'); diff --git a/lib/modules/devices/views/devices_view.dart b/lib/modules/devices/views/devices_view.dart index 7a69dd5b0..30b5570ee 100644 --- a/lib/modules/devices/views/devices_view.dart +++ b/lib/modules/devices/views/devices_view.dart @@ -1,12 +1,10 @@ -import 'dart:ui'; - import '../../../routes/app_routes.dart'; -import '/modules/devices/controllers/devices_controller.dart'; import 'package:flutter/material.dart'; import 'package:flutter_screenutil/flutter_screenutil.dart'; import 'package:get/get.dart'; import '/core/theme/app_colors.dart'; import '/data/models/user_Info.dart'; +import '/data/services/ble_manager.dart'; import '../controllers/devices_controller.dart'; // 设备类型枚举 @@ -20,7 +18,6 @@ enum DeviceType { const DeviceType(this.value); final int value; - // 使用getter获取多语言显示名称 String get displayName { switch (this) { case DeviceType.Unknown: @@ -61,112 +58,240 @@ enum DeviceType { class DevicesView extends GetView { const DevicesView({super.key}); + bool _isConnected(UserDevice device) { + try { + final bleManager = Get.find(); + return bleManager.currentDeviceAddress != null && + bleManager.currentDeviceAddress == device.devicemac; + } catch (_) { + return false; + } + } + @override Widget build(BuildContext context) { - final isDarkMode = Theme.of(context).brightness == Brightness.dark; + final isDark = Theme.of(context).brightness == Brightness.dark; + final bg = isDark ? const Color(0xFF0A0A0F) : const Color(0xFFF5F6FA); + final topBg = isDark ? const Color(0xFF0F1018) : const Color(0xFFF0F1F6); return Scaffold( - backgroundColor: isDarkMode ? AppColors.darkBackground : Colors.white, + backgroundColor: bg, body: Column( children: [ - // 顶部标题栏 - Container( - padding: EdgeInsets.only( - top: MediaQuery.of(context).padding.top + 10.h, - left: 20.w, - right: 20.w, - bottom: 20.h, - ), - decoration: BoxDecoration( - color: isDarkMode ? Colors.black : Colors.white, - boxShadow: [ - BoxShadow( - color: Colors.black.withOpacity(0.05), - blurRadius: 10, - offset: const Offset(0, 2), - ), - ], - ), - child: Row( - children: [ - GestureDetector( - onTap: () => Get.back(), + // 顶部区域 + _buildTopArea(context, isDark, bg, topBg), + // 设备列表 + Expanded(child: _buildDevicesList(isDark, topBg)), + ], + ), + ); + } + + Widget _buildTopArea( + BuildContext context, bool isDark, Color bg, Color topBg) { + return Container( + padding: EdgeInsets.only( + top: MediaQuery.of(context).padding.top + 8.h, + left: 20.w, + right: 20.w, + bottom: 24.h, + ), + decoration: BoxDecoration(color: bg), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + // 导航栏 + Row( + children: [ + GestureDetector( + onTap: () => Get.back(), + child: Container( + width: 40.w, + height: 40.w, + decoration: BoxDecoration( + shape: BoxShape.circle, + color: isDark + ? Colors.white.withOpacity(0.06) + : Colors.black.withOpacity(0.04), + ), child: Icon( - Icons.arrow_back_ios, - color: isDarkMode ? Colors.white : Colors.black, - size: 24.sp, + Icons.arrow_back_ios_new, + size: 18.sp, + color: isDark ? Colors.white : const Color(0xFF222222), ), ), - SizedBox(width: 16.w), - Text( - "deviceManagement".tr, - style: TextStyle( - fontSize: 20.sp, - fontWeight: FontWeight.w600, - color: isDarkMode ? Colors.white : Colors.black, - ), + ), + SizedBox(width: 16.w), + Text( + "deviceManagement".tr, + style: TextStyle( + fontSize: 26.sp, + fontWeight: FontWeight.w800, + color: isDark ? Colors.white : const Color(0xFF111111), + letterSpacing: -0.5, ), - ], - ), - ), - // 设备列表内容 - Expanded( - child: _buildDevicesList(isDarkMode), + ), + ], ), + SizedBox(height: 20.h), + // 摘要卡片 + Obx(() { + final totalDevices = controller.devices.values + .fold(0, (sum, list) => sum + list.length); + final connectedCount = controller.devices.values + .expand((list) => list) + .where((d) => _isConnected(d)) + .length; + return Row( + children: [ + _buildSummaryCard( + number: totalDevices.toString(), + label: "boundDevices".tr, + isActive: true, + isDark: isDark, + ), + SizedBox(width: 12.w), + _buildSummaryCard( + number: connectedCount.toString(), + label: "currentConnection".tr, + isActive: false, + isDark: isDark, + ), + ], + ); + }), ], ), ); } - Widget _buildDevicesList(bool isDarkMode) { + Widget _buildSummaryCard({ + required String number, + required String label, + required bool isActive, + required bool isDark, + }) { + final activeGradient = isDark + ? [ + const Color(0xFF6366F1).withOpacity(0.15), + const Color(0xFF8B5CF6).withOpacity(0.1), + ] + : [const Color(0xFFEEF2FF), const Color(0xFFF5F3FF)]; + + return Expanded( + child: Container( + padding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 14.h), + decoration: BoxDecoration( + borderRadius: BorderRadius.circular(16.r), + gradient: isActive ? LinearGradient(colors: activeGradient) : null, + color: isActive + ? null + : (isDark + ? Colors.white.withOpacity(0.04) + : Colors.white), + border: Border.all( + color: isActive + ? (isDark + ? const Color(0xFF6366F1).withOpacity(0.2) + : const Color(0xFF6366F1).withOpacity(0.15)) + : (isDark + ? Colors.white.withOpacity(0.06) + : Colors.black.withOpacity(0.04)), + ), + boxShadow: isDark + ? null + : [ + BoxShadow( + color: Colors.black.withOpacity(0.04), + blurRadius: 4, + offset: const Offset(0, 1), + ), + ], + ), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + number, + style: TextStyle( + fontSize: 24.sp, + fontWeight: FontWeight.w700, + color: isActive + ? (isDark + ? const Color(0xFFA5B4FC) + : const Color(0xFF6366F1)) + : (isDark ? Colors.white : const Color(0xFF222222)), + ), + ), + SizedBox(height: 2.h), + Text( + label, + style: TextStyle( + fontSize: 11.sp, + fontWeight: FontWeight.w500, + color: isActive + ? (isDark + ? const Color(0xFFA5B4FC).withOpacity(0.6) + : const Color(0xFF818CF8)) + : (isDark + ? Colors.white.withOpacity(0.35) + : const Color(0xFF999999)), + ), + ), + ], + ), + ), + ); + } + + Widget _buildDevicesList(bool isDark, Color topBg) { return Obx(() { - // 使用控制器的响应式数据，这会触发控制器的实例化 final groupedDevices = controller.devices; if (groupedDevices.isEmpty) { - return _buildEmptyState(isDarkMode); + return _buildEmptyState(isDark); } return ListView( - padding: EdgeInsets.all(20.w), + padding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 20.h), children: [ - // 遍历每个设备类型 ...groupedDevices.entries.map((entry) { - final deviceTypeValue = entry.key; + final deviceType = DeviceType.fromValue(entry.key); final deviceList = entry.value; - final deviceType = DeviceType.fromValue(deviceTypeValue); return Column( crossAxisAlignment: CrossAxisAlignment.start, children: [ - // 设备类型标题 + // 分类标题 Padding( - padding: EdgeInsets.only(bottom: 12.h), + padding: EdgeInsets.only(left: 8.w, bottom: 12.h), child: Text( - deviceType.displayName, + deviceType.displayName.toUpperCase(), style: TextStyle( - fontSize: 16.sp, + fontSize: 12.sp, fontWeight: FontWeight.w600, - color: isDarkMode ? Colors.white : Colors.black87, + color: isDark + ? Colors.white.withOpacity(0.25) + : const Color(0xFFAAAAAA), + letterSpacing: 1.5, ), ), ), - // 该类型下的设备列表 ...deviceList.map((device) => _buildDeviceCard( device: device, deviceType: deviceType, - isDarkMode: isDarkMode, + isDark: isDark, )), - SizedBox(height: 20.h), + SizedBox(height: 16.h), ], ); - }).toList(), + }), ], ); }); } - Widget _buildEmptyState(bool isDarkMode) { + Widget _buildEmptyState(bool isDark) { return Center( child: Column( mainAxisAlignment: MainAxisAlignment.center, @@ -174,14 +299,14 @@ class DevicesView extends GetView { Icon( Icons.devices_other, size: 80.sp, - color: isDarkMode ? Colors.white54 : Colors.grey[400], + color: isDark ? Colors.white54 : Colors.grey[400], ), SizedBox(height: 20.h), Text( "noDevicesBound".tr, style: TextStyle( fontSize: 16.sp, - color: isDarkMode ? Colors.white70 : Colors.grey[600], + color: isDark ? Colors.white70 : Colors.grey[600], ), ), SizedBox(height: 12.h), @@ -189,7 +314,7 @@ class DevicesView extends GetView { "pleaseBindDeviceFirst".tr, style: TextStyle( fontSize: 14.sp, - color: isDarkMode ? Colors.white54 : Colors.grey[500], + color: isDark ? Colors.white54 : Colors.grey[500], ), ), ], @@ -200,121 +325,263 @@ class DevicesView extends GetView { Widget _buildDeviceCard({ required UserDevice device, required DeviceType deviceType, - required bool isDarkMode, + required bool isDark, }) { + final connected = _isConnected(device); + final connectedColor = const Color(0xFF22C55E); + return GestureDetector( onTap: () => Get.toNamed(Routes.eqSettings), child: Container( - margin: EdgeInsets.only(bottom: 12.h), - padding: EdgeInsets.all(16.w), + margin: EdgeInsets.only(bottom: 10.h), + padding: EdgeInsets.all(20.w), decoration: BoxDecoration( - color: isDarkMode ? AppColors.cardBackground : Colors.white, - borderRadius: BorderRadius.circular(12.r), - boxShadow: [ - BoxShadow( - color: Colors.black.withOpacity(0.05), - blurRadius: 8, - offset: const Offset(0, 2), - ), - ], + borderRadius: BorderRadius.circular(20.r), + color: isDark + ? (connected + ? connectedColor.withOpacity(0.04) + : Colors.white.withOpacity(0.03)) + : (connected + ? connectedColor.withOpacity(0.03) + : Colors.white), + border: Border.all( + color: connected + ? connectedColor.withOpacity(0.2) + : (isDark + ? Colors.white.withOpacity(0.05) + : Colors.black.withOpacity(0.04)), + ), + boxShadow: isDark + ? null + : [ + BoxShadow( + color: Colors.black.withOpacity(0.03), + blurRadius: 12, + offset: const Offset(0, 2), + ), + ], ), - child: Row( + child: Column( children: [ - // 设备图标 - Container( - width: 48.w, - height: 48.w, - decoration: BoxDecoration( - color: isDarkMode - ? Colors.white.withOpacity(0.1) - : Colors.grey[100], - borderRadius: BorderRadius.circular(12.r), - ), - child: Icon( - deviceType.icon, - color: isDarkMode ? Colors.white : Colors.black87, - size: 24.sp, - ), - ), - SizedBox(width: 16.w), - // 设备信息 - Expanded( - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Text( - device.devicename.isNotEmpty - ? device.devicename - : "deviceTypeUnknown".tr, - style: TextStyle( - fontSize: 16.sp, - fontWeight: FontWeight.w500, - color: isDarkMode ? Colors.white : Colors.black87, + // 上部：图标 + 名称/MAC + 解绑 + Row( + children: [ + // 设备图标 + Container( + width: 46.w, + height: 46.w, + decoration: BoxDecoration( + borderRadius: BorderRadius.circular(14.r), + gradient: LinearGradient( + begin: Alignment.topLeft, + end: Alignment.bottomRight, + colors: connected + ? [connectedColor, const Color(0xFF16A34A)] + : [const Color(0xFF6366F1), const Color(0xFF8B5CF6)], ), + boxShadow: [ + BoxShadow( + color: (connected ? connectedColor : const Color(0xFF6366F1)) + .withOpacity(0.3), + blurRadius: 16, + offset: const Offset(0, 4), + ), + ], ), - SizedBox(height: 4.h), - Text( - device.devicemac.isNotEmpty - ? device.devicemac - : "noMacAddress".tr, - style: TextStyle( - fontSize: 14.sp, - color: isDarkMode ? Colors.white70 : Colors.grey[600], - ), + child: Icon( + deviceType.icon, + color: Colors.white, + size: 22.sp, ), - if (device.license.isNotEmpty) ...[ - SizedBox(height: 4.h), - Text( - "${"licenseCode".tr}: ${device.license}", + ), + SizedBox(width: 14.w), + // 设备名 + MAC + Expanded( + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Row( + children: [ + Text( + device.devicename.isNotEmpty + ? device.devicename + : "deviceTypeUnknown".tr, + style: TextStyle( + fontSize: 16.sp, + fontWeight: FontWeight.w700, + color: isDark + ? Colors.white + : const Color(0xFF1A1A2E), + ), + ), + if (connected) ...[ + SizedBox(width: 8.w), + Container( + width: 6.w, + height: 6.w, + decoration: BoxDecoration( + shape: BoxShape.circle, + color: connectedColor, + boxShadow: [ + BoxShadow( + color: connectedColor.withOpacity(0.6), + blurRadius: 8, + ), + ], + ), + ), + SizedBox(width: 6.w), + Container( + padding: EdgeInsets.symmetric( + horizontal: 8.w, vertical: 3.h), + decoration: BoxDecoration( + borderRadius: BorderRadius.circular(6.r), + color: connectedColor.withOpacity(0.1), + ), + child: Text( + "connected".tr, + style: TextStyle( + fontSize: 10.sp, + fontWeight: FontWeight.w600, + color: connectedColor, + ), + ), + ), + ], + ], + ), + SizedBox(height: 3.h), + Text( + device.devicemac.isNotEmpty + ? device.devicemac + : "noMacAddress".tr, + style: TextStyle( + fontSize: 12.sp, + fontWeight: FontWeight.w500, + color: isDark + ? Colors.white.withOpacity(0.3) + : const Color(0xFF999999), + fontFamily: 'Menlo', + letterSpacing: 0.5, + ), + ), + ], + ), + ), + // 解绑按钮 + GestureDetector( + onTap: () => _showUnbindDialog(device, isDark), + child: Container( + padding: EdgeInsets.symmetric( + horizontal: 14.w, vertical: 8.h), + decoration: BoxDecoration( + borderRadius: BorderRadius.circular(10.r), + border: Border.all( + color: isDark + ? Colors.white.withOpacity(0.08) + : Colors.black.withOpacity(0.08), + ), + ), + child: Text( + "unbind".tr, style: TextStyle( fontSize: 12.sp, - color: isDarkMode ? Colors.white54 : Colors.grey[500], + fontWeight: FontWeight.w600, + color: isDark + ? Colors.white.withOpacity(0.4) + : const Color(0xFF999999), ), ), - ], - ], - ), - ), - // 解绑按钮 - GestureDetector( - onTap: () => _showUnbindDialog(device, isDarkMode), - child: Container( - padding: EdgeInsets.symmetric( - horizontal: 12.w, - vertical: 6.h, + ), ), + ], + ), + // 下部：授权码 / CMEI + if (device.license.isNotEmpty || device.cmei.isNotEmpty) ...[ + SizedBox(height: 16.h), + Container( + padding: EdgeInsets.all(12.w), decoration: BoxDecoration( - color: Colors.red.withOpacity(0.1), - borderRadius: BorderRadius.circular(6.r), - border: Border.all( - color: Colors.red.withOpacity(0.3), - width: 1, - ), + borderRadius: BorderRadius.circular(12.r), + color: isDark + ? Colors.white.withOpacity(0.03) + : const Color(0xFFF7F7FA), ), - child: Text( - "unbind".tr, - style: TextStyle( - fontSize: 12.sp, - color: Colors.red, - fontWeight: FontWeight.w500, - ), + child: Column( + children: [ + if (device.license.isNotEmpty) + _buildFieldRow( + label: "licenseCode".tr, + value: device.license, + isDark: isDark, + ), + if (device.license.isNotEmpty && device.cmei.isNotEmpty) + SizedBox(height: 8.h), + if (device.cmei.isNotEmpty) + _buildFieldRow( + label: "CMEI", + value: device.cmei, + isDark: isDark, + ), + ], ), ), - ), + ], ], ), ), ); } - void _showUnbindDialog(UserDevice device, bool isDarkMode) { + Widget _buildFieldRow({ + required String label, + required String value, + required bool isDark, + }) { + return Row( + children: [ + SizedBox( + width: 44.w, + child: Text( + label, + style: TextStyle( + fontSize: 10.sp, + fontWeight: FontWeight.w600, + color: isDark + ? Colors.white.withOpacity(0.2) + : const Color(0xFFBBBBBB), + letterSpacing: 0.5, + ), + ), + ), + SizedBox(width: 8.w), + Expanded( + child: Text( + value, + style: TextStyle( + fontSize: 11.sp, + fontWeight: FontWeight.w500, + color: isDark + ? Colors.white.withOpacity(0.5) + : const Color(0xFF666666), + fontFamily: 'Menlo', + letterSpacing: 0.3, + ), + overflow: TextOverflow.ellipsis, + ), + ), + ], + ); + } + + void _showUnbindDialog(UserDevice device, bool isDark) { Get.dialog( AlertDialog( - backgroundColor: isDarkMode ? AppColors.cardBackground : Colors.white, + backgroundColor: isDark ? AppColors.cardBackground : Colors.white, title: Text( "confirmUnbind".tr, style: TextStyle( - color: isDarkMode ? Colors.white : Colors.black87, + color: isDark ? Colors.white : Colors.black87, ), ), content: Text( @@ -322,7 +589,7 @@ class DevicesView extends GetView { .tr .replaceAll('{deviceName}', device.devicename), style: TextStyle( - color: isDarkMode ? Colors.white70 : Colors.black54, + color: isDark ? Colors.white70 : Colors.black54, ), ), actions: [ @@ -331,7 +598,7 @@ class DevicesView extends GetView { child: Text( "cancel".tr, style: TextStyle( - color: isDarkMode ? Colors.white70 : Colors.grey[600], + color: isDark ? Colors.white70 : Colors.grey[600], ), ), ), diff --git a/lib/modules/home/controllers/home_controller.dart b/lib/modules/home/controllers/home_controller.dart index 0dede3002..55d431d59 100644 --- a/lib/modules/home/controllers/home_controller.dart +++ b/lib/modules/home/controllers/home_controller.dart @@ -256,8 +256,16 @@ class HomeController extends GetxController { } void exploreAIFeatures() { - Get.toNamed(Routes.agent); - AgentService.interruptCurrentResponse(); + // 优先读设备检测写入的 storage（YIDONG_PID 匹配时），否则降级到服务端 AGENT_TYPE + final agentType = GetStorage().read('device_agent_type') + ?? AppConfig.env('AGENT_TYPE') + ?? 'stt_llm_tts'; + if (agentType == 'yidong') { + Get.toNamed(Routes.mobileElf); + } else { + Get.toNamed(Routes.agent); + AgentService.interruptCurrentResponse(); + } } Future _checkConnectionStatus() async { diff --git a/lib/modules/home/views/home_view.dart b/lib/modules/home/views/home_view.dart index dab865a45..cc2c15243 100644 --- a/lib/modules/home/views/home_view.dart +++ b/lib/modules/home/views/home_view.dart @@ -145,6 +145,7 @@ class HomeView extends GetView { _buildSectionTitle(isDarkMode), SizedBox(height: 16.h), _buildPremiumAILayout(), + ], ), ), diff --git a/lib/modules/mobile_elf/bindings/mobile_elf_binding.dart b/lib/modules/mobile_elf/bindings/mobile_elf_binding.dart new file mode 100644 index 000000000..193e9fbe3 --- /dev/null +++ b/lib/modules/mobile_elf/bindings/mobile_elf_binding.dart @@ -0,0 +1,9 @@ +import 'package:get/get.dart'; +import '../controllers/mobile_elf_controller.dart'; + +class MobileElfBinding extends Bindings { + @override + void dependencies() { + Get.lazyPut(() => MobileElfController()); + } +} diff --git a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart new file mode 100644 index 000000000..356b0f726 --- /dev/null +++ b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart @@ -0,0 +1,459 @@ +import 'dart:async'; +import 'package:agent_service/agent_service.dart'; +import 'package:chat_storage/chat_storage.dart'; +import 'package:flutter/widgets.dart'; +import 'package:get/get.dart'; +import 'package:get_storage/get_storage.dart'; +import 'package:permission_handler/permission_handler.dart'; +import '../../../data/models/appconfig.dart'; +import '../../../core/utils/logger.dart'; +import '../../../core/utils/permission_util.dart'; + +enum MobileElfStatus { idle, connecting, recording, thinking, speaking, error } + +class MobileElfMessage { + final bool isUser; + final String text; + final DateTime time; + MobileElfMessage( + {required this.isUser, required this.text, required this.time}); +} + +class MobileElfController extends GetxController { + static const _tag = 'MobileElfCtrl'; + + // ── 可观察状态 ───────────────────────────────────────────────────────────── + final status = MobileElfStatus.idle.obs; + final messages = [].obs; + final realtimeText = ''.obs; + final errorMsg = ''.obs; + final inCall = false.obs; + final callSeconds = 0.obs; + final recordingSeconds = 0.obs; + final micRms = 0.0.obs; + final micAlive = false.obs; + final pttRecording = false.obs; + final pttSeconds = 0.obs; + final ttsEnabled = true.obs; // 是否播报 AI 回复 + final ScrollController scrollController = ScrollController(); + + // 凭证状态（原生层负责，Dart 层始终无错） + final credentialReady = true.obs; + final credentialError = ''.obs; + // 设备认证失败（STS credential_failed 事件触发） + final credentialFailed = false.obs; + final credentialFailedReason = ''.obs; + + // ── 内部 ─────────────────────────────────────────────────────────────────── + StreamSubscription? _eventSub; + Timer? _callTimer; + Timer? _pttTimer; + Timer? _pttSafetyTimer; + bool _inCallMode = false; + bool _callStarting = false; // 防止并发 startCall + // PTT 松开后仍有异步事件待处理（recognition_result / assistant_response / tts） + bool _pttResultPending = false; + // PTT 正在异步启动（await 权限期间用户已松手） + bool _pttStarting = false; + // BLE/AI 唤醒触发的会话（非通话、非PTT模式下的唤醒） + bool _bleWakeupActive = false; + // PTT 被中止（轻点松手），用于忽略原生层的残留事件 + bool _pttAborted = false; + final _storage = GetStorage(); + // 滚动：layout 变化前是否已在底部 + bool _atBottom = true; + + // ═══════════════════════════════════════════════════════════════════════════ + // 生命周期 + // ═══════════════════════════════════════════════════════════════════════════ + + String get _agentId => AppConfig.env('MOBILE_ELF_AGENT_ID') ?? 'mobile_elf'; + + @override + void onInit() { + super.onInit(); + _subscribeEvents(); + _loadChatHistory(); + // 同步播报开关到原生层 + ever(ttsEnabled, (bool enabled) { + AgentService.setTtsMuted(!enabled); + }); + // 主动触发认证检查（页面打开时，确保 credential_failed 事件不被错过） + AgentService.fetchCredentials(); + } + + Future _loadChatHistory() async { + try { + final result = await ChatStorage.getMessages( + agentId: _agentId, + page: 1, + pageSize: 50, + ); + if (result.messages.isNotEmpty) { + final history = result.messages.map((m) => MobileElfMessage( + isUser: m.sender == 'user', + text: m.message, + time: DateTime.tryParse(m.timestamp) ?? DateTime.now(), + )).toList(); + // 按时间正序排列 + history.sort((a, b) => a.time.compareTo(b.time)); + messages.addAll(history); + _scrollToBottom(); + } + } catch (e) { + Logger.error('$_tag 加载聊天记录失败: $e'); + } + } + + @override + void onClose() { + _eventSub?.cancel(); + _callTimer?.cancel(); + _pttTimer?.cancel(); + _pttSafetyTimer?.cancel(); + AgentService.stopConversation(); + scrollController.dispose(); + super.onClose(); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 事件订阅（来自原生 StsAgent） + // ═══════════════════════════════════════════════════════════════════════════ + + void _subscribeEvents() { + _eventSub = AgentService.events.listen(_onAgentEvent); + } + + void _onAgentEvent(AgentServiceEvent event) { + Logger.info('$_tag event: ${event.type} data: ${event.data}'); + // PTT 松开后 pttRecording=false，但 fin_result/assistant_response/tts 仍异步到达 + // 用 _pttResultPending 保持会话激活，直到 ttsCompleted/error 才关闭 + final sessionActive = _inCallMode || pttRecording.value || _pttResultPending || _bleWakeupActive; + switch (event.type) { + case AgentServiceEventType.recognitionStarted: + // 忽略刚刚中止的 PTT 会话的残留事件，避免误触发 BLE 唤醒 + if (_pttAborted && !sessionActive) { + _pttAborted = false; + break; + } + // BLE/AI 唤醒：即使没有主动会话也激活 + if (!sessionActive) _bleWakeupActive = true; + status.value = MobileElfStatus.recording; + realtimeText.value = ''; + micAlive.value = true; + break; + + case AgentServiceEventType.recognizing: + if (!sessionActive) break; + realtimeText.value = event.data['text']?.toString() ?? ''; + status.value = MobileElfStatus.recording; + break; + + case AgentServiceEventType.recognitionResult: + if (!sessionActive) break; + final text = event.data['text']?.toString() ?? ''; + realtimeText.value = ''; + micAlive.value = false; + if (text.isNotEmpty) _addMessage(isUser: true, text: text); + status.value = MobileElfStatus.thinking; + break; + + case AgentServiceEventType.recognitionStopped: + case AgentServiceEventType.recognitionCanceled: + if (!sessionActive) break; + realtimeText.value = ''; + micAlive.value = false; + break; + + case AgentServiceEventType.assistantResponse: + if (!sessionActive) break; + final text = event.data['response']?.toString() ?? ''; + if (text.isNotEmpty) _updateOrAddAssistantMessage(text); + if (!ttsEnabled.value) { + status.value = + _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; + micAlive.value = _inCallMode; + } + break; + + case AgentServiceEventType.ttsStarted: + if (!sessionActive) break; + status.value = MobileElfStatus.speaking; + break; + + case AgentServiceEventType.ttsCompleted: + if (!sessionActive) break; + _pttResultPending = false; + _pttSafetyTimer?.cancel(); + _endBleWakeup(); // BLE 唤醒一轮结束，清理 native PTT 状态 + if (_inCallMode) { + status.value = MobileElfStatus.recording; + micAlive.value = true; + } else { + status.value = MobileElfStatus.idle; + micAlive.value = false; + } + break; + + case AgentServiceEventType.ttsCanceled: + case AgentServiceEventType.ttsStopped: + if (!sessionActive) break; + _pttResultPending = false; + _pttSafetyTimer?.cancel(); + _endBleWakeup(); + status.value = _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; + micAlive.value = _inCallMode; + break; + + case AgentServiceEventType.error: + if (!sessionActive) break; + _pttResultPending = false; + _pttSafetyTimer?.cancel(); + _endBleWakeup(); + final msg = event.data['message']?.toString() ?? '未知错误'; + status.value = MobileElfStatus.error; + errorMsg.value = msg; + Logger.error('$_tag error: $msg'); + break; + + case AgentServiceEventType.micRms: + final rms = (event.data['rms'] as num?)?.toDouble() ?? 0.0; + micRms.value = rms; + break; + + case AgentServiceEventType.heartbeatReported: + // 记录心跳上报时间 + final t = (event.data['time'] as num?)?.toInt() ?? DateTime.now().millisecondsSinceEpoch; + _storage.write('sts_heartbeat_time', t); + Logger.info('$_tag 心跳上报成功，时间: $t'); + break; + + case AgentServiceEventType.credentialFailed: + final reason = event.data['reason']?.toString() ?? '设备未认证'; + Logger.error('$_tag 设备认证失败: $reason'); + credentialFailed.value = true; + credentialFailedReason.value = reason; + // 确保停止任何进行中的会话 + if (_inCallMode || pttRecording.value) { + disconnect(); + } + break; + + default: + break; + } + _scrollToBottom(); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 公共接口（View 调用） + // ═══════════════════════════════════════════════════════════════════════════ + + Future startCall() async { + if (inCall.value || _callStarting) return; + _callStarting = true; + _pttAborted = false; + _checkHeartbeat(); + try { + // 若有正在播报的 PTT/BLE 唤醒会话，先强制中断 TTS/AI 流， + // 使原生层立刻进入可停止状态，避免 stopConversation 长时间等待 + if (_pttResultPending || _bleWakeupActive) { + _pttResultPending = false; + _bleWakeupActive = false; + await AgentService.interruptCurrentResponse(); + } + // 再确保原生层完全停止 + await AgentService.stopConversation(); + + final granted = await PermissionUtil.instance.requestPermission( + permissionType: Permission.microphone, + permissionName: '麦克风', + explanationText: '移动精灵需要麦克风权限才能进行语音对话', + permanentDenialText: '请在设置中开启麦克风权限以使用语音功能', + ); + if (!granted || inCall.value) return; // 权限等待期间可能已被挂断或重复调用 + _inCallMode = true; + inCall.value = true; + callSeconds.value = 0; + micAlive.value = true; + status.value = MobileElfStatus.connecting; + _callTimer?.cancel(); + _callTimer = Timer.periodic( + const Duration(seconds: 1), (_) => callSeconds.value++); + await AgentService.startConversation(mode: 'phone_call'); + // startCall 返回即表示麦克风已打开、VAD 已在监听，直接显示聆听状态 + // （call 模式下 recognition_started 只在 VAD 检测到语音时才发出，不能等它来转状态） + if (_inCallMode) status.value = MobileElfStatus.recording; + } finally { + _callStarting = false; + } + } + + Future disconnect() async { + _callStarting = false; + _inCallMode = false; + _pttStarting = false; + _pttResultPending = false; // 强制结束所有等待 + _pttSafetyTimer?.cancel(); + _bleWakeupActive = false; + inCall.value = false; + _callTimer?.cancel(); + _callTimer = null; + callSeconds.value = 0; + pttRecording.value = false; + _pttTimer?.cancel(); + _pttTimer = null; + status.value = MobileElfStatus.idle; + micAlive.value = false; + realtimeText.value = ''; + await AgentService.stopConversation(); + } + + /// 超过1小时未上报心跳则补报一次 + void _checkHeartbeat() { + final last = _storage.read('sts_heartbeat_time') ?? 0; + if (DateTime.now().millisecondsSinceEpoch - last > 3600000) { + AgentService.reportHeartbeat().catchError((_) {}); + } + } + + Future startPtt() async { + if (pttRecording.value || _pttStarting) return; + _pttStarting = true; + _pttAborted = false; + _checkHeartbeat(); + // BLE 唤醒/PTT 等待结果期间用户按下 PTT，先中断当前会话（同 startCall 逻辑） + if (_bleWakeupActive || _pttResultPending) { + _pttResultPending = false; + _bleWakeupActive = false; + await AgentService.interruptCurrentResponse(); + await AgentService.stopConversation(); + } + final granted = await PermissionUtil.instance.requestPermission( + permissionType: Permission.microphone, + permissionName: '麦克风', + explanationText: '移动精灵需要麦克风权限才能进行语音对话', + permanentDenialText: '请在设置中开启麦克风权限以使用语音功能', + ); + // 权限等待期间用户已松手（stopPtt 将 _pttStarting 置 false），直接放弃 + if (!granted || !_pttStarting) { + _pttStarting = false; + return; + } + _pttStarting = false; + pttRecording.value = true; + pttSeconds.value = 0; + status.value = MobileElfStatus.connecting; + _pttTimer?.cancel(); + _pttTimer = Timer.periodic( + const Duration(seconds: 1), (_) => pttSeconds.value++); + AgentService.startConversation(mode: 'push_to_talk'); + } + + void stopPtt() { + // 若仍在等待权限授予，取消本次启动 + if (_pttStarting) { + _pttStarting = false; + return; + } + if (!pttRecording.value) return; + pttRecording.value = false; + _pttTimer?.cancel(); + _pttTimer = null; + + // 如果还在连接中（还未开始识别），说明用户轻点了一下就松手， + // 直接恢复空闲，不需要等待异步事件回来 + if (status.value == MobileElfStatus.connecting) { + _pttAborted = true; // 标记中止，忽略原生层的残留事件 + _pttResultPending = false; + status.value = MobileElfStatus.idle; + micAlive.value = false; + AgentService.stopConversation(); + return; + } + + _pttResultPending = true; // 等待 fin_result/assistant_response/tts 异步回来 + AgentService.stopConversation(); + + // 安全超时：防止原生层未返回事件导致状态永久卡住 + _pttSafetyTimer?.cancel(); + _pttSafetyTimer = Timer(const Duration(seconds: 8), () { + if (_pttResultPending && !_inCallMode) { + Logger.info('$_tag PTT safety timeout — force reset to idle'); + _pttResultPending = false; + status.value = MobileElfStatus.idle; + micAlive.value = false; + } + }); + } + + void interruptSpeaking() { + AgentService.interruptCurrentResponse(); + status.value = + _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; + micAlive.value = _inCallMode; + } + + /// 原生层已自动获取凭证，此处保留方法供 View 调用不报错 + Future retryCredentials() async { + credentialError.value = ''; + } + + /// 清空聊天记录 + Future clearMessages() async { + messages.clear(); + try { + await AgentService.clearChatHistory(); + } catch (e) { + Logger.error('$_tag 清空聊天记录失败: $e'); + } + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 工具 + // ═══════════════════════════════════════════════════════════════════════════ + + void _addMessage({required bool isUser, required String text}) { + messages.add(MobileElfMessage( + isUser: isUser, text: text, time: DateTime.now())); + _scrollToBottom(); + } + + /// 更新最后一条 AI 消息（如果存在且更短），否则添加新消息 + void _updateOrAddAssistantMessage(String text) { + final lastIdx = messages.length - 1; + if (lastIdx >= 0 && !messages[lastIdx].isUser) { + // 用更完整的文本替换 + messages[lastIdx] = MobileElfMessage( + isUser: false, text: text, time: messages[lastIdx].time); + messages.refresh(); + } else { + _addMessage(isUser: false, text: text); + } + _scrollToBottom(); + } + + /// BLE 唤醒会话标志清除 + /// Native 状态（pttRecording 等）由 StsAgent.startPtt() 的重入逻辑懒清理， + /// 不在此调用 stopConversation()，避免与新会话产生竞态。 + void _endBleWakeup() { + _bleWakeupActive = false; + } + + void _scrollToBottom() { + // layout 变化前捕获是否在底部，避免新内容撑开后差值变大误判为"已滚离底部" + if (scrollController.hasClients) { + final pos = scrollController.position; + _atBottom = pos.maxScrollExtent - pos.pixels <= 80; + } + WidgetsBinding.instance.addPostFrameCallback((_) { + if (scrollController.hasClients && _atBottom) { + scrollController.animateTo( + scrollController.position.maxScrollExtent, + duration: const Duration(milliseconds: 300), + curve: Curves.easeOut, + ); + } + }); + } +} diff --git a/lib/modules/mobile_elf/views/mobile_elf_view.dart b/lib/modules/mobile_elf/views/mobile_elf_view.dart new file mode 100644 index 000000000..dad19841f --- /dev/null +++ b/lib/modules/mobile_elf/views/mobile_elf_view.dart @@ -0,0 +1,988 @@ +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_screenutil/flutter_screenutil.dart'; +import 'package:get/get.dart'; +import 'package:intl/intl.dart'; // 用于 DateFormat +import '../controllers/mobile_elf_controller.dart'; + +class MobileElfView extends GetView { + const MobileElfView({Key? key}) : super(key: key); + + // ── 颜色常量 ────────────────────────────────────────────────────────────── + static const _blue = Color(0xFF2E7CF6); + static const _green = Color(0xFF34C759); + static const _greenDark = Color(0xFF16A34A); + static const _red = Color(0xFFEF4444); + static const _orange = Color(0xFFF79009); + static const _bgLight = Color(0xFFF4F6F9); + static const _surfaceLight = Colors.white; + static const _textPrimary = Color(0xFF101828); + static const _textSecondary = Color(0xFF667085); + static const _border = Color(0xFFEAECF0); + + // 渐变复用 + static const _avatarGradient = LinearGradient( + colors: [Color(0xFF667EEA), Color(0xFF764BA2)], + begin: Alignment.topLeft, + end: Alignment.bottomRight, + ); + + @override + Widget build(BuildContext context) { + final isDark = Theme.of(context).brightness == Brightness.dark; + return Scaffold( + backgroundColor: isDark ? const Color(0xFF0D0D0D) : _bgLight, + appBar: _buildAppBar(isDark), + // 顶层 Listener 捕获 PTT 松开事件（确保 widget 树重建后仍能接收 pointerUp） + body: Listener( + behavior: HitTestBehavior.translucent, + onPointerUp: (_) { + if (controller.pttRecording.value) controller.stopPtt(); + }, + child: Obx(() { + if (controller.credentialError.value.isNotEmpty) { + return _buildCredentialError(); + } + return Column( + children: [ + Expanded(child: _buildMessageList(isDark)), + _buildBottomBar(isDark), + ], + ); + }), + ), + ); + } + + // ══════════════════════════════════════════════════════════════════════════ + // AppBar + // ══════════════════════════════════════════════════════════════════════════ + + PreferredSizeWidget _buildAppBar(bool isDark) { + return AppBar( + backgroundColor: isDark ? const Color(0xFF1A1A1A) : _surfaceLight, + elevation: 0, + surfaceTintColor: Colors.transparent, + bottom: PreferredSize( + preferredSize: const Size.fromHeight(1), + child: Container( + color: isDark ? const Color(0xFF222222) : _border, height: 1), + ), + leading: IconButton( + icon: Icon(Icons.arrow_back_ios, + size: 18.sp, color: isDark ? Colors.white : _textSecondary), + onPressed: () => Get.back(), + ), + titleSpacing: 0, + title: Row( + mainAxisSize: MainAxisSize.min, + children: [ + _gradientCircle(32.w, '晓', 13.sp), + SizedBox(width: 10.w), + Text('移动精灵', + style: TextStyle( + fontSize: 16.sp, + fontWeight: FontWeight.w600, + color: isDark ? Colors.white : _textPrimary)), + SizedBox(width: 8.w), + Obx(() => _buildStatusPill()), + ], + ), + actions: [ + // 播报开关 + Obx(() => IconButton( + icon: Icon( + controller.ttsEnabled.value + ? Icons.volume_up_rounded + : Icons.volume_off_rounded, + size: 20.sp, + color: controller.ttsEnabled.value + ? _blue + : (isDark ? Colors.grey[500] : _textSecondary), + ), + onPressed: () => controller.ttsEnabled.toggle(), + tooltip: controller.ttsEnabled.value ? '关闭播报' : '开启播报', + )), + PopupMenuButton( + icon: Icon(Icons.more_vert, + size: 20.sp, color: isDark ? Colors.white : _textSecondary), + onSelected: (value) { + if (value == 'clear') { + Get.dialog( + AlertDialog( + title: Text('clearChatHistory'.tr.isNotEmpty ? 'clearChatHistory'.tr : '清空记录'), + content: Text('confirmClearChat'.tr.isNotEmpty ? 'confirmClearChat'.tr : '确定清空所有聊天记录吗？'), + actions: [ + TextButton(onPressed: () => Get.back(), child: Text('cancel'.tr.isNotEmpty ? 'cancel'.tr : '取消')), + TextButton( + onPressed: () { + controller.clearMessages(); + Get.back(); + }, + child: Text('confirm'.tr.isNotEmpty ? 'confirm'.tr : '确定', style: const TextStyle(color: Colors.red)), + ), + ], + ), + ); + } + }, + itemBuilder: (_) => [ + PopupMenuItem( + value: 'clear', + child: Row(children: [ + Icon(Icons.delete_outline, size: 18.sp, color: Colors.red), + SizedBox(width: 8.w), + Text('清空记录', style: TextStyle(fontSize: 14.sp, color: Colors.red)), + ]), + ), + ], + ), + ], + ); + } + + /// 右上角状态胶囊 + 通话计时 + Widget _buildStatusPill() { + if (controller.credentialFailed.value) { + return Container( + padding: EdgeInsets.symmetric(horizontal: 10.w, vertical: 3.h), + decoration: BoxDecoration(color: const Color(0xFFFEF2F2), borderRadius: BorderRadius.circular(20.r)), + child: Row(mainAxisSize: MainAxisSize.min, children: [ + Icon(Icons.lock_outline_rounded, size: 11.sp, color: _red), + SizedBox(width: 4.w), + Text('认证失败', style: TextStyle(fontSize: 11.sp, color: _red, fontWeight: FontWeight.w500)), + ]), + ); + } + final s = controller.status.value; + final inCall = controller.inCall.value; + final cfg = _pillConfig(s, inCall); + return Container( + padding: EdgeInsets.symmetric(horizontal: 10.w, vertical: 3.h), + decoration: BoxDecoration( + color: cfg.bg, + borderRadius: BorderRadius.circular(20.r), + ), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Text(cfg.label, + style: TextStyle( + fontSize: 11.sp, + color: cfg.fg, + fontWeight: FontWeight.w500)), + if (inCall) ...[ + SizedBox(width: 4.w), + Obx(() { + final sec = controller.callSeconds.value; + return Text(_fmtDuration(sec), + style: TextStyle( + fontSize: 11.sp, + color: cfg.fg, + fontWeight: FontWeight.w400)); + }), + ], + ], + ), + ); + } + + _PillCfg _pillConfig(MobileElfStatus s, bool inCall) { + switch (s) { + case MobileElfStatus.connecting: + return const _PillCfg(Color(0xFFFFFAEB), Color(0xFFB54708), '连接中…'); + case MobileElfStatus.recording: + return const _PillCfg(Color(0xFFECFDF3), _greenDark, '聆听中'); + case MobileElfStatus.thinking: + return const _PillCfg(Color(0xFFEFF8FF), Color(0xFF175CD3), '处理中'); + case MobileElfStatus.speaking: + return const _PillCfg(Color(0xFFECFDF3), _greenDark, '朗读中'); + case MobileElfStatus.error: + return const _PillCfg(Color(0xFFFEF2F2), _red, '错误'); + case MobileElfStatus.idle: + return inCall + ? const _PillCfg(Color(0xFFECFDF3), _greenDark, '通话中') + : const _PillCfg(Color(0xFFF2F4F7), _textSecondary, '空闲'); + } + } + + // ══════════════════════════════════════════════════════════════════════════ + // 凭证错误 + // ══════════════════════════════════════════════════════════════════════════ + + Widget _buildCredentialError() { + return Center( + child: Padding( + padding: EdgeInsets.all(32.w), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + Icon(Icons.wifi_off_rounded, size: 56.sp, color: Colors.red[300]), + SizedBox(height: 16.h), + Obx(() => Text(controller.credentialError.value, + textAlign: TextAlign.center, + style: TextStyle(fontSize: 13.sp, color: Colors.red[400]))), + SizedBox(height: 20.h), + ElevatedButton.icon( + onPressed: controller.retryCredentials, + icon: const Icon(Icons.refresh), + label: const Text('重试'), + ), + ], + ), + ), + ); + } + + // ══════════════════════════════════════════════════════════════════════════ + // 消息列表 + // ══════════════════════════════════════════════════════════════════════════ + + Widget _buildMessageList(bool isDark) { + return Obx(() { + final msgs = controller.messages; + final rt = controller.realtimeText.value; + if (msgs.isEmpty && rt.isEmpty) return _buildEmptyState(isDark); + return ListView.builder( + controller: controller.scrollController, + padding: EdgeInsets.fromLTRB(16.w, 16.h, 16.w, 8.h), + itemCount: msgs.length + (rt.isNotEmpty ? 1 : 0), + itemBuilder: (_, i) { + if (i < msgs.length) return _buildBubble(msgs[i], isDark); + return _buildRealtimeBubble(rt); + }, + ); + }); + } + + Widget _buildEmptyState(bool isDark) { + return Center( + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + _gradientCircle(80.w, '晓', 32.sp), + SizedBox(height: 20.h), + Text('移动精灵', + style: TextStyle( + fontSize: 20.sp, + fontWeight: FontWeight.bold, + color: isDark ? Colors.white : _textPrimary)), + SizedBox(height: 8.h), + Text('点击右下角按钮，即可与晓晓对话', + style: TextStyle( + fontSize: 13.sp, + color: isDark ? Colors.grey[500] : _textSecondary)), + ], + ), + ); + } + + // ── 消息气泡 ────────────────────────────────────────────────────────────── + + Widget _buildBubble(MobileElfMessage msg, bool isDark) { + final isUser = msg.isUser; + return Padding( + padding: EdgeInsets.only(bottom: 14.h), + child: Column( + crossAxisAlignment: + isUser ? CrossAxisAlignment.end : CrossAxisAlignment.start, + children: [ + // AI 头像 + 名字 + if (!isUser) ...[ + Row(children: [ + _gradientCircle(28.w, '晓', 11.sp), + SizedBox(width: 6.w), + Text('晓晓', + style: TextStyle(fontSize: 11.sp, color: _textSecondary)), + ]), + SizedBox(height: 4.h), + ], + // 气泡体（长按弹出菜单） + Row( + mainAxisAlignment: + isUser ? MainAxisAlignment.end : MainAxisAlignment.start, + children: [ + if (!isUser) SizedBox(width: 34.w), + Flexible( + child: GestureDetector( + onLongPressStart: (details) => _showMessageMenu(details, msg, isDark), + child: Container( + padding: + EdgeInsets.symmetric(horizontal: 14.w, vertical: 10.h), + decoration: BoxDecoration( + color: isUser + ? _blue + : (isDark ? const Color(0xFF2A2A2A) : _surfaceLight), + borderRadius: BorderRadius.only( + topLeft: Radius.circular(18.r), + topRight: Radius.circular(18.r), + bottomLeft: Radius.circular(isUser ? 18.r : 4.r), + bottomRight: Radius.circular(isUser ? 4.r : 18.r), + ), + boxShadow: [ + BoxShadow( + color: Colors.black.withValues(alpha: 0.05), + blurRadius: 6, + offset: const Offset(0, 2), + ), + ], + ), + child: Text(msg.text, + style: TextStyle( + fontSize: 15.sp, + height: 1.55, + color: isUser + ? Colors.white + : (isDark ? Colors.white : _textPrimary))), + ), + ), + ), + ], + ), + // 时间 + AI 标签 + SizedBox(height: 3.h), + Padding( + padding: EdgeInsets.only(left: isUser ? 0 : 34.w), + child: Row( + mainAxisAlignment: + isUser ? MainAxisAlignment.end : MainAxisAlignment.start, + children: [ + Text(_formatTimestamp(msg.time), + style: TextStyle( + fontSize: 10.sp, color: isDark ? Colors.grey[500] : const Color(0xFFD0D5DD))), + if (!isUser) ...[ + SizedBox(width: 8.w), + Text('generatedByAi'.tr.isNotEmpty ? 'generatedByAi'.tr : '由AI生成', + style: TextStyle( + fontSize: 10.sp, + color: Colors.blue[600], + fontWeight: FontWeight.bold, + fontStyle: FontStyle.italic)), + ], + ], + ), + ), + ], + ), + ); + } + + /// 长按消息弹出菜单 + void _showMessageMenu(LongPressStartDetails details, MobileElfMessage msg, bool isDark) { + HapticFeedback.mediumImpact(); + final overlay = Overlay.of(Get.context!).context.findRenderObject() as RenderBox?; + if (overlay == null) return; + final position = RelativeRect.fromRect( + Rect.fromLTWH(details.globalPosition.dx, details.globalPosition.dy, 0, 0), + Offset.zero & overlay.size, + ); + showMenu( + context: Get.context!, + position: position, + shape: RoundedRectangleBorder(borderRadius: BorderRadius.circular(12.r)), + color: isDark ? const Color(0xFF2A2A2A) : Colors.white, + items: [ + PopupMenuItem( + onTap: () { + Clipboard.setData(ClipboardData(text: msg.text)); + Get.snackbar('success'.tr.isNotEmpty ? 'success'.tr : '成功', + 'copySuccess'.tr.isNotEmpty ? 'copySuccess'.tr : '已复制', + backgroundColor: Colors.blue.withValues(alpha: 0.8), colorText: Colors.white); + }, + child: Row(children: [ + Icon(Icons.copy, size: 18.sp, color: isDark ? Colors.white : Colors.black87), + SizedBox(width: 8.w), + Text('copy'.tr.isNotEmpty ? 'copy'.tr : '复制', + style: TextStyle(fontSize: 14.sp, color: isDark ? Colors.white : Colors.black87)), + ]), + ), + ], + ); + } + + /// 智能时间格式化（与 agent 模块一致） + String _formatTimestamp(DateTime t) { + final now = DateTime.now(); + final diff = now.difference(t); + final today = DateTime(now.year, now.month, now.day); + final msgDate = DateTime(t.year, t.month, t.day); + + if (diff.inMinutes < 1) return 'justNow'.tr.isNotEmpty ? 'justNow'.tr : '刚刚'; + if (diff.inHours < 1) { + final m = diff.inMinutes.toString(); + final text = 'minutesAgo'.trParams({'minutes': m}); + return text.isNotEmpty && text != 'minutesAgo' ? text : '$m分钟前'; + } + if (msgDate == today) return DateFormat('HH:mm').format(t); + if (msgDate == today.subtract(const Duration(days: 1))) { + return '${('yesterday'.tr.isNotEmpty ? 'yesterday'.tr : '昨天')} ${DateFormat('HH:mm').format(t)}'; + } + if (diff.inDays < 7) { + return '${_weekdayName(t.weekday)} ${DateFormat('HH:mm').format(t)}'; + } + if (t.year == now.year) return DateFormat('MM-dd HH:mm').format(t); + return DateFormat('yyyy-MM-dd HH:mm').format(t); + } + + String _weekdayName(int w) { + const names = ['', '星期一', '星期二', '星期三', '星期四', '星期五', '星期六', '星期日']; + return names[w]; + } + + Widget _buildRealtimeBubble(String text) { + return Padding( + padding: EdgeInsets.only(bottom: 14.h), + child: Row(mainAxisAlignment: MainAxisAlignment.end, children: [ + Flexible( + child: Container( + padding: EdgeInsets.symmetric(horizontal: 14.w, vertical: 10.h), + decoration: BoxDecoration( + color: _blue.withValues(alpha: 0.6), + borderRadius: BorderRadius.circular(18.r), + ), + child: Text(text, + style: TextStyle( + fontSize: 15.sp, color: Colors.white, height: 1.55)), + ), + ), + ]), + ); + } + + // ══════════════════════════════════════════════════════════════════════════ + // 底部控制栏 + // ══════════════════════════════════════════════════════════════════════════ + + Widget _buildBottomBar(bool isDark) { + return Obx(() { + + final inCall = controller.inCall.value; + final s = controller.status.value; + final ptt = controller.pttRecording.value; + + // 状态栏显示条件： + // - 通话模式：recording/thinking/speaking/connecting 都显示 + // - 非通话非PTT：thinking/speaking 显示 + // - PTT 中：不显示（状态在中间按钮体现） + final showStatusBar = !ptt && ( + (inCall && s != MobileElfStatus.idle && s != MobileElfStatus.error) || + (!inCall && (s == MobileElfStatus.speaking || s == MobileElfStatus.thinking)) + ); + + return Container( + width: double.infinity, + decoration: BoxDecoration( + color: isDark ? const Color(0xFF1A1A1A) : _surfaceLight, + border: Border(top: BorderSide(color: isDark ? const Color(0xFF222222) : _border)), + boxShadow: [ + BoxShadow(color: Colors.black.withValues(alpha: 0.05), blurRadius: 12, offset: const Offset(0, -2)), + ], + ), + child: SafeArea( + top: false, + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + if (showStatusBar) _buildStatusBar(s, inCall, isDark), + if (inCall) + _buildInCallInputBar(isDark) + else + _buildNormalInputBar(isDark), + ], + ), + ), + ); + }); + } + + // ── 状态提示栏 ──────────────────────────────────────────────────────────── + + Widget _buildStatusBar(MobileElfStatus s, bool inCall, bool isDark) { + Widget? content; + + switch (s) { + case MobileElfStatus.speaking: + content = GestureDetector( + onTap: controller.interruptSpeaking, + behavior: HitTestBehavior.opaque, + child: Container( + padding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 8.h), + color: const Color(0xFFECFDF3), + child: Row( + children: [ + _MiniWave(color: _greenDark), + SizedBox(width: 8.w), + Expanded(child: Text('晓晓正在朗读...', style: TextStyle(fontSize: 13.sp, color: _greenDark, fontWeight: FontWeight.w500))), + Container( + padding: EdgeInsets.symmetric(horizontal: 12.w, vertical: 4.h), + decoration: BoxDecoration( + border: Border.all(color: _red), + borderRadius: BorderRadius.circular(16.r), + ), + child: Text('× 打断', style: TextStyle(fontSize: 12.sp, color: _red, fontWeight: FontWeight.w500)), + ), + ], + ), + ), + ); + break; + case MobileElfStatus.recording: + content = Container( + padding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 8.h), + color: const Color(0xFFECFDF3), + child: Row( + children: [ + _MiniWave(color: _greenDark), + SizedBox(width: 8.w), + Expanded(child: Obx(() { + final rt = controller.realtimeText.value; + return Text(rt.isNotEmpty ? rt : '聆听中...', style: TextStyle(fontSize: 13.sp, color: _greenDark), maxLines: 1, overflow: TextOverflow.ellipsis); + })), + ], + ), + ); + break; + case MobileElfStatus.connecting: + content = Container( + padding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 8.h), + color: const Color(0xFFFFFAEB), + child: Row( + children: [ + SizedBox(width: 14.w, height: 14.w, child: CircularProgressIndicator(strokeWidth: 2, color: _orange)), + SizedBox(width: 8.w), + Text('正在连接...', style: TextStyle(fontSize: 13.sp, color: const Color(0xFFB54708))), + ], + ), + ); + break; + case MobileElfStatus.thinking: + content = Container( + padding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 8.h), + color: const Color(0xFFEFF8FF), + child: Row( + children: [ + SizedBox(width: 14.w, height: 14.w, child: CircularProgressIndicator(strokeWidth: 2, color: const Color(0xFF175CD3))), + SizedBox(width: 8.w), + Text('AI 思考中...', style: TextStyle(fontSize: 13.sp, color: const Color(0xFF175CD3))), + ], + ), + ); + break; + default: + return const SizedBox.shrink(); + } + return content; + } + + // ── 通话模式底栏 ────────────────────────────────────────────────────────── + + Widget _buildInCallInputBar(bool isDark) { + return Padding( + padding: EdgeInsets.fromLTRB(20.w, 12.h, 20.w, 14.h), + child: Row( + children: [ + // 通话状态 + 计时 + Expanded( + child: Obx(() { + final s = controller.status.value; + final idle = s == MobileElfStatus.idle; + return Row( + children: [ + Container( + width: 8.w, height: 8.w, + decoration: BoxDecoration(shape: BoxShape.circle, color: idle ? _green : _orange), + ), + SizedBox(width: 6.w), + Text('通话中', style: TextStyle(fontSize: 13.sp, color: isDark ? Colors.white70 : _textSecondary)), + SizedBox(width: 4.w), + Obx(() => Text(_fmtDuration(controller.callSeconds.value), + style: TextStyle(fontSize: 13.sp, color: isDark ? Colors.white70 : _textSecondary))), + ], + ); + }), + ), + // 音量指示器 + Obx(() => _VolumeIndicator( + rms: controller.micRms.value, + alive: controller.micAlive.value, + isDark: isDark, + )), + SizedBox(width: 12.w), + // 挂断按钮 + GestureDetector( + onTap: controller.disconnect, + child: Container( + width: 52.w, height: 52.w, + decoration: BoxDecoration( + shape: BoxShape.circle, + color: _red, + boxShadow: [BoxShadow(color: _red.withValues(alpha: 0.3), blurRadius: 14, offset: const Offset(0, 4))], + ), + child: Icon(Icons.call_end_rounded, color: Colors.white, size: 26.sp), + ), + ), + ], + ), + ); + } + + // ── 普通输入栏 (按住说话) ────────────────────────────────────────────────── + + Widget _buildNormalInputBar(bool isDark) { + return Obx(() { + final s = controller.status.value; + final ptt = controller.pttRecording.value; + + final locked = controller.credentialFailed.value; + final disabled = locked || (!ptt && s != MobileElfStatus.idle && s != MobileElfStatus.error); + final pttClickable = !locked && s == MobileElfStatus.speaking; + final sideDisabled = locked || ptt || (disabled && !pttClickable); + + return Padding( + padding: EdgeInsets.fromLTRB(12.w, 6.h, 12.w, 14.h), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + // ── PTT 实时识别文字（居中 + 渐变背景） ── + if (ptt) + Obx(() { + final rt = controller.realtimeText.value; + if (rt.isEmpty) return const SizedBox.shrink(); + return Container( + width: double.infinity, + padding: EdgeInsets.symmetric(horizontal: 20.w, vertical: 12.h), + margin: EdgeInsets.only(bottom: 8.h), + decoration: BoxDecoration( + gradient: LinearGradient( + begin: Alignment.topLeft, + end: Alignment.bottomRight, + colors: isDark + ? [const Color(0xFF2A2A2A), const Color(0xFF1A2332)] + : [const Color(0xFFF0F4FF), const Color(0xFFE8F5E9)], + ), + borderRadius: BorderRadius.circular(16.r), + border: Border.all( + color: isDark ? const Color(0xFF334155) : const Color(0xFFE2E8F0), + width: 0.5, + ), + ), + child: Text(rt, + style: TextStyle(fontSize: 15.sp, color: isDark ? Colors.white : _textPrimary, height: 1.4), + textAlign: TextAlign.center, + maxLines: 3, overflow: TextOverflow.ellipsis, + ), + ); + }), + // ── 按钮行 ── + Row( + children: [ + // 中间：PTT 按钮 + Expanded( + child: ptt + ? _buildPttRecordingCenter(isDark) + : IgnorePointer( + ignoring: disabled && !pttClickable, + child: Opacity( + opacity: (disabled && !pttClickable) ? 0.5 : 1.0, + child: _buildPttIdleButton(isDark), + ), + ), + ), + SizedBox(width: 8.w), + // 右侧：电话按钮 + IgnorePointer( + ignoring: sideDisabled, + child: Opacity( + opacity: sideDisabled ? 0.4 : 1.0, + child: _roundButton( + onTap: controller.startCall, + gradient: const LinearGradient( + colors: [Color(0xFF34C759), Color(0xFF28A745)], + begin: Alignment.topLeft, end: Alignment.bottomRight, + ), + shadow: _green.withValues(alpha: 0.4), + size: 44, + child: Icon(Icons.phone, color: Colors.white, size: 20.sp), + ), + ), + ), + ], + ), + ], + ), + ); + }); + } + + /// 按住说话按钮（空闲状态）— 参考微信风格 + Widget _buildPttIdleButton(bool isDark) { + return Listener( + onPointerDown: (_) => controller.startPtt(), + child: Container( + height: 42.h, + decoration: BoxDecoration( + color: isDark ? const Color(0xFF2A2A2A) : Colors.white, + borderRadius: BorderRadius.circular(21.r), + border: Border.all(color: isDark ? const Color(0xFF444444) : const Color(0xFFD0D5DD), width: 1), + ), + child: Center( + child: Text('按住 说话', + style: TextStyle( + fontSize: 15.sp, + color: isDark ? Colors.white70 : _textPrimary, + fontWeight: FontWeight.w600, + letterSpacing: 2.w, + ), + ), + ), + ), + ); + } + + /// PTT 录音中：仅中间部分 + /// 连接中=与按住说话相同样式，录音中=橙色 + Widget _buildPttRecordingCenter(bool isDark) { + return Obx(() { + final s = controller.status.value; + final isConnecting = s == MobileElfStatus.connecting; + + if (isConnecting) { + // 连接中：与按住说话空闲状态一样的外观 + 加载指示 + return Container( + height: 42.h, + decoration: BoxDecoration( + color: isDark ? const Color(0xFF2A2A2A) : Colors.white, + borderRadius: BorderRadius.circular(21.r), + border: Border.all(color: isDark ? const Color(0xFF444444) : const Color(0xFFD0D5DD), width: 1), + ), + child: Row( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + SizedBox(width: 14.w, height: 14.w, + child: CircularProgressIndicator(strokeWidth: 2, color: isDark ? Colors.white70 : _textSecondary)), + SizedBox(width: 8.w), + Text('连接中...', style: TextStyle(fontSize: 14.sp, color: isDark ? Colors.white70 : _textSecondary, fontWeight: FontWeight.w500)), + ], + ), + ); + } + + // 录音中：橙色 + return Container( + height: 42.h, + decoration: BoxDecoration( + gradient: const LinearGradient(colors: [Color(0xFFF79009), Color(0xFFDC6803)]), + borderRadius: BorderRadius.circular(21.r), + boxShadow: [BoxShadow( + color: _orange.withValues(alpha: 0.3), + blurRadius: 8, offset: const Offset(0, 2), + )], + ), + child: Row( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Obx(() => _RmsBars(rms: controller.micRms.value, color: Colors.white)), + SizedBox(width: 8.w), + Obx(() => Text('松开发送 · ${controller.pttSeconds.value}s', + style: TextStyle(fontSize: 14.sp, color: Colors.white, fontWeight: FontWeight.w500))), + ], + ), + ); + }); + } + + // ── 通用圆形按钮 ────────────────────────────────────────────────────────── + + Widget _roundButton({ + required VoidCallback onTap, + required Widget child, + Color? color, + Gradient? gradient, + Color? shadow, + Color? border, + double size = 56, + }) { + return GestureDetector( + onTap: onTap, + child: Container( + width: size.w, + height: size.w, + decoration: BoxDecoration( + shape: BoxShape.circle, + color: gradient == null ? color : null, + gradient: gradient, + border: border != null ? Border.all(color: border, width: 2) : null, + boxShadow: shadow != null + ? [BoxShadow(color: shadow, blurRadius: 14, offset: const Offset(0, 4))] + : null, + ), + child: Center(child: child), + ), + ); + } + + // ══════════════════════════════════════════════════════════════════════════ + // 工具方法 + // ══════════════════════════════════════════════════════════════════════════ + + /// 渐变圆形头像 + Widget _gradientCircle(double size, String text, double fontSize) { + return Container( + width: size, + height: size, + decoration: const BoxDecoration( + shape: BoxShape.circle, + gradient: _avatarGradient, + ), + child: Center( + child: Text(text, + style: TextStyle( + color: Colors.white, + fontSize: fontSize, + fontWeight: FontWeight.w600)), + ), + ); + } + + + String _fmtDuration(int secs) { + final m = (secs ~/ 60).toString().padLeft(2, '0'); + final s = (secs % 60).toString().padLeft(2, '0'); + return '$m:$s'; + } +} + +// ═══════════════════════════════════════════════════════════════════════════ +// 辅助类 +// ═══════════════════════════════════════════════════════════════════════════ + +class _PillCfg { + final Color bg, fg; + final String label; + const _PillCfg(this.bg, this.fg, this.label); +} + +/// 3 条小波浪动画 +class _MiniWave extends StatefulWidget { + final Color color; + const _MiniWave({required this.color}); + + @override + State<_MiniWave> createState() => _MiniWaveState(); +} + +class _MiniWaveState extends State<_MiniWave> + with SingleTickerProviderStateMixin { + late final AnimationController _ctrl; + + @override + void initState() { + super.initState(); + _ctrl = AnimationController( + vsync: this, duration: const Duration(milliseconds: 800)) + ..repeat(); + } + + @override + void dispose() { + _ctrl.dispose(); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + return AnimatedBuilder( + animation: _ctrl, + builder: (_, __) => Row( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.center, + children: List.generate(3, (i) { + final phase = (_ctrl.value + i * 0.2) % 1.0; + final h = 3.0 + 7.0 * (0.5 - (phase - 0.5).abs()) * 2; + return Container( + width: 2.5, + height: h.clamp(3.0, 10.0), + margin: const EdgeInsets.symmetric(horizontal: 1), + decoration: BoxDecoration( + color: widget.color, + borderRadius: BorderRadius.circular(2), + ), + ); + }), + ), + ); + } +} + +/// 麦克风音量指示器（通话模式用） +class _VolumeIndicator extends StatelessWidget { + final double rms; + final bool alive; + final bool isDark; + const _VolumeIndicator({required this.rms, required this.alive, required this.isDark}); + + @override + Widget build(BuildContext context) { + // rms 范围大约 0~6000，归一化到 0~1 + final level = alive ? (rms / 5000).clamp(0.0, 1.0) : 0.0; + const barCount = 5; + return Container( + width: 44.w, height: 44.w, + decoration: BoxDecoration( + shape: BoxShape.circle, + color: isDark ? const Color(0xFF2A2A2A) : const Color(0xFFF2F4F7), + ), + child: Row( + mainAxisAlignment: MainAxisAlignment.center, + crossAxisAlignment: CrossAxisAlignment.center, + children: List.generate(barCount, (i) { + final threshold = (i + 1) / barCount; + final active = level >= threshold; + return Container( + width: 3.w, + height: (8 + i * 4).h, + margin: EdgeInsets.symmetric(horizontal: 1.w), + decoration: BoxDecoration( + color: active + ? const Color(0xFF34C759) + : (isDark ? const Color(0xFF444444) : const Color(0xFFD0D5DD)), + borderRadius: BorderRadius.circular(1.5.r), + ), + ); + }), + ), + ); + } +} + +/// PTT 录音时响应麦克风音量的波形条 +class _RmsBars extends StatelessWidget { + final double rms; + final Color color; + const _RmsBars({required this.rms, required this.color}); + + @override + Widget build(BuildContext context) { + // rms 范围约 0~6000，归一化到 0~1 + final level = (rms / 5000).clamp(0.0, 1.0); + const barCount = 3; + return Row( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.center, + children: List.generate(barCount, (i) { + final threshold = (i + 1) / barCount; + final h = level >= threshold ? 10.0 : 3.0; + return AnimatedContainer( + duration: const Duration(milliseconds: 80), + width: 2.5, + height: h, + margin: const EdgeInsets.symmetric(horizontal: 1), + decoration: BoxDecoration( + color: color, + borderRadius: BorderRadius.circular(2), + ), + ); + }), + ); + } +} diff --git a/lib/modules/pairing/controllers/pairing_controller.dart b/lib/modules/pairing/controllers/pairing_controller.dart index 2b63098c6..5156ea8b4 100644 --- a/lib/modules/pairing/controllers/pairing_controller.dart +++ b/lib/modules/pairing/controllers/pairing_controller.dart @@ -810,6 +810,7 @@ class PairingController extends GetxController with WidgetsBindingObserver { orElse: () => UserDevice( id: -1, uid: '', + productid: -1, devicetype: -1, devicename: '', devicemac: '', @@ -840,15 +841,21 @@ class PairingController extends GetxController with WidgetsBindingObserver { orElse: () => UserDevice( id: -1, uid: '', + productid: -1, devicetype: -1, devicename: '', devicemac: '', license: ''), ); if (existingDevice.id != -1) { + _verifiedDevicePid = existingDevice.productid; _verifiedDeviceName = device.name; _verifiedDeviceAddress = device.address; - // 设备时ble 蓝牙 且已绑定 + // 已绑定设备直接用 license 作为 deviceCode（即绑定时扫到的 SN） + if (existingDevice.license.isNotEmpty) { + deviceCode.value = existingDevice.license; + Logger.i(_tag, '[YIDONG] 已绑定设备，从license提取 deviceCode=${existingDevice.license}'); + } return 10; } } @@ -885,14 +892,15 @@ class PairingController extends GetxController with WidgetsBindingObserver { code = extractAndReverseBytes(hexData, 13, 22); _verifiedDevicePid = extractAndReverseBytesToInt(hexData, 4, 5); - print('找到匹配设备，获取到 code: $code'); + final pidHexIos = _verifiedDevicePid?.toRadixString(16).toUpperCase() ?? ''; + Logger.i(_tag, '[YIDONG] iOS BLE扫描 MAC匹配 device=${device.name} pidHex=$pidHexIos code=$code'); if (code != "") { _bleManager.stopScan(); deviceCode.value = code; String? uuids = deviceInfo["id"]; _verifiedDeviceName = device.name; _verifiedDeviceAddress = uuids; - print('找到匹配设备，获取到 uuids: $uuids'); + Logger.i(_tag, '[YIDONG] iOS deviceCode提取成功 code=$code uuid=$uuids'); //检查是否已经绑定 final existingDevice = User.instance.devices.firstWhere( (d) => @@ -902,6 +910,7 @@ class PairingController extends GetxController with WidgetsBindingObserver { orElse: () => UserDevice( id: -1, uid: '', + productid: -1, devicetype: -1, devicename: '', devicemac: '', @@ -921,12 +930,14 @@ class PairingController extends GetxController with WidgetsBindingObserver { code = extractAndReverseBytes(hexData, 15, 24); _verifiedDevicePid = extractAndReverseBytesToInt(hexData, 6, 7); - print('找到匹配设备，获取到 code: $code'); + final pidHexAndroid = _verifiedDevicePid?.toRadixString(16).toUpperCase() ?? ''; + Logger.i(_tag, '[YIDONG] Android BLE扫描 MAC匹配 device=${device.name} mac=${device.address} pidHex=$pidHexAndroid code=$code'); if (code != "") { _bleManager.stopScan(); deviceCode.value = code; _verifiedDeviceName = device.name; _verifiedDeviceAddress = device.address; + Logger.i(_tag, '[YIDONG] Android deviceCode提取成功 code=$code mac=${device.address}'); //检查是否已经绑定 final existingDevice = User.instance.devices.firstWhere( (d) => @@ -936,6 +947,7 @@ class PairingController extends GetxController with WidgetsBindingObserver { orElse: () => UserDevice( id: -1, uid: '', + productid: -1, devicetype: -1, devicename: '', devicemac: '', @@ -1178,26 +1190,92 @@ class PairingController extends GetxController with WidgetsBindingObserver { } } + /// 检测是否为移动精灵（YIDONG）设备，并写入 GetStorage + void _detectAndSaveYidongDevice() { + final yidongPid = AppConfig.env('YIDONG_PID') ?? ''; + if (yidongPid.isEmpty) { + Logger.i(_tag, '[YIDONG] 未配置 YIDONG_PID，跳过检测'); + return; + } + + // 优先使用本次扫描解析的 pid，否则从已绑定设备列表中查找 + int? pid = _verifiedDevicePid; + String pidSource = 'BLE扫描'; + if (pid == null) { + final mac = _verifiedDeviceAddress ?? _currentClassicDevice?.address ?? ''; + Logger.i(_tag, '[YIDONG] 本次未扫描到pid，从已绑定设备查找 mac=$mac'); + if (mac.isNotEmpty) { + final matched = User.instance.devices.firstWhere( + (d) => + d.devicemac.toUpperCase().replaceAll(':', '') == + mac.toUpperCase().replaceAll(':', ''), + orElse: () => UserDevice( + id: -1, uid: '', productid: -1, + devicetype: -1, devicename: '', devicemac: '', license: ''), + ); + if (matched.id != -1) { + pid = matched.productid; + pidSource = '已绑定设备'; + } + } + } + + if (pid == null) { + Logger.i(_tag, '[YIDONG] 无法获取设备pid，跳过检测'); + return; + } + + // 将 int pid 转为十六进制字符串（不补零，大写）与 YIDONG_PID 比对 + final pidHex = pid.toRadixString(16).toUpperCase(); + final isYidong = pidHex == yidongPid.toUpperCase(); + + Logger.i(_tag, '[YIDONG] 设备检测: pidHex=$pidHex yidongPid=${yidongPid.toUpperCase()} source=$pidSource matched=$isYidong'); + + if (isYidong) { + _storage.write('device_agent_type', 'yidong'); + // deviceCode 来源： + // 新设备 → BLE 扫描从广播包提取 + // 已绑定设备 → authbindingDevice 从 existingDevice.license 赋值 + if (deviceCode.value.isNotEmpty) { + final prevCode = _storage.read('device_sts_code') ?? ''; + // SN 变了说明换了设备，清除旧凭证，强制重新向移动平台认证 + if (prevCode != deviceCode.value) { + _storage.remove('sts_device_id'); + _storage.remove('sts_device_secret'); + Logger.i(_tag, '[YIDONG] SN 变更（$prevCode → ${deviceCode.value}），已清除旧凭证，强制重新认证'); + } + _storage.write('device_sts_code', deviceCode.value); + Logger.i(_tag, '[YIDONG] 识别成功，写入 device_sts_code=${deviceCode.value}'); + } else { + // 既无扫描结果也无 license，清除所有凭证，认证失败会提示用户 + _storage.remove('device_sts_code'); + _storage.remove('sts_device_id'); + _storage.remove('sts_device_secret'); + Logger.i(_tag, '[YIDONG] deviceCode 为空，已清除所有凭证'); + } + } else { + _storage.remove('device_agent_type'); + _storage.remove('device_sts_code'); + _storage.remove('sts_device_id'); + _storage.remove('sts_device_secret'); + Logger.i(_tag, '[YIDONG] 非移动精灵设备，已清除相关存储'); + } + } + /// 导航到主页 void _navigateToHome() { - // Future.delayed(const Duration(milliseconds: 500), () async { + // 检测是否为移动精灵（YIDONG）设备 + _detectAndSaveYidongDevice(); // 启动AgentService服务 try { Logger.i(_tag, '启动AgentService服务'); _bleManager.startAgentService(); - // _bleManager.registerBackgroundScan(macAddress: _verifiedDeviceAddress); } catch (e) { Logger.e(_tag, '启动AgentService失败: $e'); } - //print('lxm-上一个路由: ${Get.previousRoute.toString()}'); - // if (Get.previousRoute == Routes.home) { - // 如果是从home页过来的，则直接返回 - // Get.back(); - // } else { - // 否则，跳转到home页并清空之前的路由栈 + // 已在 Home 页时不重复导航，避免两个 HomeView 同时存在导致 Duplicate GlobalKey + if (Get.currentRoute == Routes.home) return; Get.offAllNamed(Routes.home); - // } - // }); } // ==================== 用户操作 ==================== @@ -1259,6 +1337,7 @@ class PairingController extends GetxController with WidgetsBindingObserver { UserDevice( id: 0, // 临时设备ID uid: User.instance.uid, + productid: _verifiedDevicePid ?? -1, devicetype: 1, devicemac: _currentClassicDevice?.address ?? '', devicename: _currentClassicDevice?.name ?? 'Unknown Device', diff --git a/lib/routes/app_pages.dart b/lib/routes/app_pages.dart index 5cc429486..d24555e14 100644 --- a/lib/routes/app_pages.dart +++ b/lib/routes/app_pages.dart @@ -82,6 +82,8 @@ import '../modules/goods/bindings/goodsmeet_binding.dart'; import '../modules/goods/views/goodsmeet_view.dart'; import '../modules/goods/bindings/goodsai_binding.dart'; import '../modules/goods/views/goodsai_view.dart'; +import '../modules/mobile_elf/bindings/mobile_elf_binding.dart'; +import '../modules/mobile_elf/views/mobile_elf_view.dart'; abstract class AppPages { static final pages = [ @@ -378,5 +380,12 @@ abstract class AppPages { transition: Transition.rightToLeft, transitionDuration: Duration(milliseconds: 250), ), + GetPage( + name: Routes.mobileElf, + page: () => const MobileElfView(), + binding: MobileElfBinding(), + transition: Transition.rightToLeft, + transitionDuration: Duration(milliseconds: 250), + ), ]; } diff --git a/lib/routes/app_routes.dart b/lib/routes/app_routes.dart index e36f89565..e9bcfdc48 100644 --- a/lib/routes/app_routes.dart +++ b/lib/routes/app_routes.dart @@ -61,6 +61,9 @@ abstract class Routes { // 实时语音对话 static const realtime = '/realtime'; + // 移动精灵 + static const mobileElf = '/mobile_elf'; + // 设备管理 static const devices = '/devices'; diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index 2d0dde0a3..a2d002a07 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -131,8 +131,12 @@ object AgentService : CoroutineScope { // 聊天存储服务 private lateinit var chatStorageHelper: ChatStorageHelper + // Agent模式: stt_llm_tts（默认）| sts（移动精灵） + private var agentMode = "stt_llm_tts" + private var stsAgent: StsAgent? = null + // 会话ID，用于区分不同聊天上下文 - private val agentId = "default_agent" + private var agentId = "default_agent" //工具调用提示 private var callingToolText = "" private var insufficientIntegralText = "" // 积分不足提示 @@ -246,6 +250,44 @@ object AgentService : CoroutineScope { if (isInitialized) return true try { + agentMode = config["agentMode"]?.toString() ?: "stt_llm_tts" + agentId = config["agentId"]?.toString()?.takeIf { it.isNotBlank() } ?: "default_agent" + + // STS 模式：跳过 Azure/OpenAI 初始化，直接启动 StsAgent + if (agentMode == "sts") { + this.context = appContext.applicationContext + audioPlayer = AudioPlayer(context) // STS 模式也需要音频播放器（提示音等） + val stsConfig = StsConfig( + productId = config["stsProductId"]?.toString() ?: "", + productKey = config["stsProductKey"]?.toString() ?: "", + deviceNo = config["stsDeviceNo"]?.toString() ?: "", + deviceId = config["stsDeviceId"]?.toString() ?: "", + deviceSecret = config["stsDeviceSecret"]?.toString() ?: "", + agentId = agentId, + cmei = config["stsCmei"]?.toString() ?: "", + ) + stsAgent = StsAgent( + context = context, + config = stsConfig, + onEvent = { name, data -> + // BLE 唤醒：AI 开始回复时关闭 BLE 编解码器（说明用户已说完） + if (currentRecognitionMode == "ble_wakeup" && name == "assistant_response") { + BleService.closeCodec() + } + sendEvent(name, data) + }, + onCredentialsFetched = { deviceId, deviceSecret, deviceNo -> + sendEvent("sts_credentials", mapOf( + "deviceId" to deviceId, + "deviceSecret" to deviceSecret, + "deviceNo" to deviceNo, + )) + } + ) + _isInitialized.set(true) + return true + } + serverurl = config["serverurl"]?.toString() ?: ""; usertoken = config["usertoken"]?.toString() ?: ""; callingToolText = config["callingToolText"]?.toString() ?: ""; @@ -358,6 +400,11 @@ object AgentService : CoroutineScope { config["mcpServer"]?.toString() ?: "", config["musicServiceType"]?.toString() ?: "" ) + // 更新 STS 模式参数（cmei 等可能在首次初始化时尚未获取到） + val newCmei = config["stsCmei"]?.toString() ?: "" + if (newCmei.isNotEmpty()) { + stsAgent?.updateCmei(newCmei) + } } catch (e: Exception) { _log("ERROR", "[Agent] 应用配置更新失败: ${e.message}") } @@ -369,6 +416,12 @@ object AgentService : CoroutineScope { */ fun dispose() { try { + if (agentMode == "sts") { + stsAgent?.dispose() + stsAgent = null + _isInitialized.set(false) + return + } BleService.closeCodec() stopRecognition() stopTts() @@ -670,6 +723,19 @@ object AgentService : CoroutineScope { * @param mode 识别模式：normal, ble_wakeup, phone_call, push_to_talk */ fun startRecognition(isExternalActive: Boolean, mode: String = "normal"): Boolean { + if (agentMode == "sts") { + currentRecognitionMode = mode // 记录模式供 onEvent 回调使用 + when (mode) { + "push_to_talk" -> stsAgent?.startPtt() + "phone_call" -> stsAgent?.startCall() + "ble_wakeup" -> { + audioPlayer?.playAudio(R.raw.start) // 立即播提示音，不等 WS 连接完成 + stsAgent?.startPtt() + } + else -> stsAgent?.startPtt() // 普通模式默认 PTT + } + return true + } if(isExternalActive){ isKeepResult = false BleService.openEncoder()//不发指令。只是录制opus @@ -928,6 +994,11 @@ object AgentService : CoroutineScope { * 停止语音识别 */ fun stopRecognition() { + if (agentMode == "sts") { + // StsAgent 内部根据 inCall / pttRecording 状态自行决定停止方式 + stsAgent?.stop() + return + } if (!isRecognitionActive) return _log("INFO", "[Agent] 停止语音识别，模式: $currentRecognitionMode") @@ -954,7 +1025,26 @@ object AgentService : CoroutineScope { * 打断当前响应 * 停止TTS播放和AI流输出 */ + /** 设置 STS 模式 TTS 静音开关 */ + fun setTtsMuted(muted: Boolean) { + stsAgent?.ttsMuted = muted + } + + /** 手动触发心跳上报（Flutter 层调用） */ + fun reportHeartbeat() { + stsAgent?.reportHeartbeat() + } + + /** 主动触发认证检查（移动精灵页面打开时调用，确保认证事件被 Flutter 层接收） */ + fun fetchCredentials() { + stsAgent?.fetchCredentials() + } + fun interruptCurrentResponse() { + if (agentMode == "sts") { + stsAgent?.interruptSpeaking() + return + } if (isAiStreaming || isTtsSpeaking) { val startTime = System.currentTimeMillis() stopRecognition() @@ -1689,6 +1779,11 @@ object AgentService : CoroutineScope { * 清除聊天历史 */ fun clearChatHistory(callback: (Boolean) -> Unit) { + if (agentMode == "sts") { + stsAgent?.clearHistory() + callback(true) + return + } launch { try { val success = chatStorageHelper.deleteMessages(agentId) diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt index c3f37c7fe..f4078607c 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt @@ -97,6 +97,15 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream config["xunfeiAppId"]?.toString()?.let { intent.putExtra("xunfeiAppId", it) } config["xunfeiAccessKeyId"]?.toString()?.let { intent.putExtra("xunfeiAccessKeyId", it) } config["xunfeiAccessKeySecret"]?.toString()?.let { intent.putExtra("xunfeiAccessKeySecret", it) } + // STS 模式参数 + config["agentMode"]?.toString()?.let { intent.putExtra("agentMode", it) } + config["agentId"]?.toString()?.let { intent.putExtra("agentId", it) } + config["stsProductId"]?.toString()?.let { intent.putExtra("stsProductId", it) } + config["stsProductKey"]?.toString()?.let { intent.putExtra("stsProductKey", it) } + config["stsDeviceNo"]?.toString()?.let { intent.putExtra("stsDeviceNo", it) } + config["stsDeviceId"]?.toString()?.let { intent.putExtra("stsDeviceId", it) } + config["stsDeviceSecret"]?.toString()?.let { intent.putExtra("stsDeviceSecret", it) } + config["stsCmei"]?.toString()?.let { intent.putExtra("stsCmei", it) } config["vipexptime"]?.toString()?.let { intent.putExtra("vipexptime", it) //打印日志 @@ -344,6 +353,37 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream } } + "setTtsMuted" -> { + try { + val muted = call.argument("muted") ?: false + AgentService.setTtsMuted(muted) + result.success(true) + } catch (e: Exception) { + Log.e(TAG, "设置TTS静音失败", e) + result.error("SET_TTS_MUTED_ERROR", "设置TTS静音失败: ${e.message}", null) + } + } + + "reportHeartbeat" -> { + try { + AgentService.reportHeartbeat() + result.success(true) + } catch (e: Exception) { + Log.e(TAG, "心跳上报失败", e) + result.error("HEARTBEAT_ERROR", "心跳上报失败: ${e.message}", null) + } + } + + "fetchCredentials" -> { + try { + AgentService.fetchCredentials() + result.success(true) + } catch (e: Exception) { + Log.e(TAG, "触发认证检查失败", e) + result.error("FETCH_CREDENTIALS_ERROR", "触发认证检查失败: ${e.message}", null) + } + } + "setCanWakeUp" -> { try { val canWakeUp = call.argument("canWakeUp") diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt index d2111244b..574af96ff 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt @@ -77,6 +77,15 @@ class BleMediaService : Service(), CoroutineScope { private const val KEY_XUNFEI_APP_ID = "xunfeiAppId" // 讯飞AppID private const val KEY_XUNFEI_ACCESS_KEY_ID = "xunfeiAccessKeyId" // 讯飞AccessKeyID private const val KEY_XUNFEI_ACCESS_KEY_SECRET = "xunfeiAccessKeySecret" // 讯飞AccessKeySecret + // STS 模式参数键 + private const val KEY_AGENT_MODE = "agentMode" + private const val KEY_AGENT_ID = "agentId" + private const val KEY_STS_PRODUCT_ID = "stsProductId" + private const val KEY_STS_PRODUCT_KEY = "stsProductKey" + private const val KEY_STS_DEVICE_NO = "stsDeviceNo" + private const val KEY_STS_DEVICE_ID = "stsDeviceId" + private const val KEY_STS_DEVICE_SECRET = "stsDeviceSecret" + private const val KEY_STS_CMEI = "stsCmei" // 用于跟踪服务是否正在运行 private val isRunning = AtomicBoolean(false) @@ -546,13 +555,15 @@ private var companionManager: BleCompanionManager? = null val config = getConfigFromPreferences(prefs) Log.d(TAG, "当前配置参数: $config") - // 检查必要的配置是否存在 - val azureSpeechKey = config[KEY_AZURE_SPEECH_KEY]?.toString() ?: "" - val openaiApiKey = config[KEY_OPENAI_API_KEY]?.toString() ?: "" - - if (azureSpeechKey.isEmpty() || openaiApiKey.isEmpty()) { - Log.w(TAG, "语音代理初始化失败: API密钥未配置") - return + // 检查必要的配置是否存在（STS 模式不需要 Azure/OpenAI 密钥） + val agentMode = config[KEY_AGENT_MODE]?.toString() ?: "stt_llm_tts" + if (agentMode != "sts") { + val azureSpeechKey = config[KEY_AZURE_SPEECH_KEY]?.toString() ?: "" + val openaiApiKey = config[KEY_OPENAI_API_KEY]?.toString() ?: "" + if (azureSpeechKey.isEmpty() || openaiApiKey.isEmpty()) { + Log.w(TAG, "语音代理初始化失败: API密钥未配置") + return + } } if (isChatAgentInitialized) { @@ -604,6 +615,14 @@ private var companionManager: BleCompanionManager? = null it.getStringExtra(KEY_XUNFEI_APP_ID)?.let { value -> config[KEY_XUNFEI_APP_ID] = value } it.getStringExtra(KEY_XUNFEI_ACCESS_KEY_ID)?.let { value -> config[KEY_XUNFEI_ACCESS_KEY_ID] = value } it.getStringExtra(KEY_XUNFEI_ACCESS_KEY_SECRET)?.let { value -> config[KEY_XUNFEI_ACCESS_KEY_SECRET] = value } + it.getStringExtra(KEY_AGENT_MODE)?.let { value -> config[KEY_AGENT_MODE] = value } + it.getStringExtra(KEY_AGENT_ID)?.let { value -> config[KEY_AGENT_ID] = value } + it.getStringExtra(KEY_STS_PRODUCT_ID)?.let { value -> config[KEY_STS_PRODUCT_ID] = value } + it.getStringExtra(KEY_STS_PRODUCT_KEY)?.let { value -> config[KEY_STS_PRODUCT_KEY] = value } + it.getStringExtra(KEY_STS_DEVICE_NO)?.let { value -> config[KEY_STS_DEVICE_NO] = value } + it.getStringExtra(KEY_STS_DEVICE_ID)?.let { value -> config[KEY_STS_DEVICE_ID] = value } + it.getStringExtra(KEY_STS_DEVICE_SECRET)?.let { value -> config[KEY_STS_DEVICE_SECRET] = value } + it.getStringExtra(KEY_STS_CMEI)?.let { value -> config[KEY_STS_CMEI] = value } Log.d(TAG, "liwei-------vipexptime 3-1 ${ config[KEY_VIP_EXPTIME]}") // 提取 supportedLanguages 参数 it.getStringArrayListExtra(KEY_SUPPORTED_LANGUAGES)?.let { languages -> @@ -671,6 +690,14 @@ private var companionManager: BleCompanionManager? = null config[KEY_XUNFEI_APP_ID] = getConfigString(prefs, KEY_XUNFEI_APP_ID, "") config[KEY_XUNFEI_ACCESS_KEY_ID] = getConfigString(prefs, KEY_XUNFEI_ACCESS_KEY_ID, "") config[KEY_XUNFEI_ACCESS_KEY_SECRET] = getConfigString(prefs, KEY_XUNFEI_ACCESS_KEY_SECRET, "") + config[KEY_AGENT_MODE] = getConfigString(prefs, KEY_AGENT_MODE, "stt_llm_tts") + config[KEY_AGENT_ID] = getConfigString(prefs, KEY_AGENT_ID, "default_agent") + config[KEY_STS_PRODUCT_ID] = getConfigString(prefs, KEY_STS_PRODUCT_ID, "") + config[KEY_STS_PRODUCT_KEY] = getConfigString(prefs, KEY_STS_PRODUCT_KEY, "") + config[KEY_STS_DEVICE_NO] = getConfigString(prefs, KEY_STS_DEVICE_NO, "") + config[KEY_STS_DEVICE_ID] = getConfigString(prefs, KEY_STS_DEVICE_ID, "") + config[KEY_STS_DEVICE_SECRET] = getConfigString(prefs, KEY_STS_DEVICE_SECRET, "") + config[KEY_STS_CMEI] = getConfigString(prefs, KEY_STS_CMEI, "") Log.d(TAG, "liwei-------vipexptime 3-0 ${ config[KEY_VIP_EXPTIME]}") // 添加 supportedLanguages 参数处理 val supportedLanguagesStr = getConfigString(prefs, KEY_SUPPORTED_LANGUAGES, "") diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt new file mode 100644 index 000000000..b76a16a58 --- /dev/null +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt @@ -0,0 +1,935 @@ +package com.yunqiinnovation.agent_service + +import android.content.Context +import android.media.AudioFormat +import android.media.AudioManager +import android.media.AudioRecord +import android.media.MediaPlayer +import android.media.MediaRecorder +import android.media.ToneGenerator +import android.os.Handler +import android.os.Looper +import android.util.Log +import com.yunqiinnovation.chat_storage.ChatStorageHelper +import kotlinx.coroutines.* +import okhttp3.* +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.RequestBody.Companion.toRequestBody +import okio.ByteString.Companion.toByteString +import org.json.JSONArray +import org.json.JSONObject +import java.io.IOException +import java.security.MessageDigest +import java.security.SecureRandom +import java.security.cert.X509Certificate +import java.util.UUID +import java.util.concurrent.TimeUnit +import javax.net.ssl.SSLContext +import javax.net.ssl.TrustManager +import javax.net.ssl.X509TrustManager +import kotlin.coroutines.resumeWithException +import kotlin.math.sqrt + +/** + * STS 配置参数（移动精灵 A80 平台） + */ +data class StsConfig( + val productId: String, + val productKey: String, + var deviceNo: String, + var deviceId: String, + var deviceSecret: String, + val agentId: String, + val cmei: String = "", +) + +/** + * StsAgent — 中国移动 A80 WS 一体化语音对话实现 + * + * 通过一条 WSS 连接完成 ASR + AI + TTS，事件以与标准 stt_llm_tts 相同的格式 + * 通过 [onEvent] 回调发出，供 AgentService 转发到 Flutter EventChannel。 + */ +class StsAgent( + private val context: Context, + private var config: StsConfig, + private val onEvent: (eventName: String, data: Map) -> Unit, + private val onCredentialsFetched: ((deviceId: String, deviceSecret: String, deviceNo: String) -> Unit)? = null, +) : CoroutineScope { + + private val TAG = "StsAgent" + private val job = SupervisorJob() + override val coroutineContext = Dispatchers.IO + job + + init { + Log.i(TAG, "[YIDONG] StsAgent初始化: productId=${config.productId} deviceNo=${config.deviceNo} deviceId=${config.deviceId} agentId=${config.agentId} cmei=${config.cmei} hasSecret=${config.deviceSecret.isNotEmpty()}") + fetchCredentials() + } + + /** 动态更新 CMEI（首次初始化时可能尚未获取到） */ + fun updateCmei(cmei: String) { + if (cmei.isNotEmpty() && cmei != config.cmei) { + config = config.copy(cmei = cmei) + Log.i(TAG, "[YIDONG] updateCmei: cmei已更新为 $cmei") + } + } + + // ── WebSocket ───────────────────────────────────────────────────────────── + private val okHttpClient: OkHttpClient = buildTrustAllClient() + private var webSocket: WebSocket? = null + private var readyReceived = false + private var sessionEnded = true + private var dialogRequestId = "" + + // ── 录音 ────────────────────────────────────────────────────────────────── + private val SAMPLE_RATE = 16000 + private val FRAME_SIZE = 5120 // 160ms @ 16kHz 16bit mono + private val RECORD_CHUNK = FRAME_SIZE / 2 // 读取块大小 + private val RECORD_BUFFER = maxOf( + AudioRecord.getMinBufferSize( + SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT + ), FRAME_SIZE * 2 + ) + private var audioRecord: AudioRecord? = null + private var recordingJob: Job? = null + private var micChunkCount = 0 + + // ── 发送缓冲 ────────────────────────────────────────────────────────────── + private val sendBuffer = ArrayDeque() + private var sendBufferOffset = 0 + + // ── 预缓冲（VAD 触发前 ~1.5s）──────────────────────────────────────────── + private val preBuffer = ArrayDeque() + private val preBufferLock = Any() + + // ── 麦克风读写锁（防止 read() 与 stop()/release() 并发导致 native SIGABRT）── + private val micLock = Any() + private val PRE_BUFFER_MAX_CHUNKS = 75 + + // ── 本地 VAD ────────────────────────────────────────────────────────────── + private var speechDetected = false + private val VAD_THRESHOLD = 800.0 + private val VAD_ONSET_CHUNKS = 3 + private var vadOnsetCount = 0 + + // ── 状态标记 ────────────────────────────────────────────────────────────── + private var forwardingAudio = false + private var wsConnecting = false + private var inCall = false + private var pttRecording = false + + // ── TTS 播放队列 ────────────────────────────────────────────────────────── + private val ttsQueue = ArrayDeque() + private var ttsPlaying = false + private var ttsInterrupted = false + private var currentPlayer: MediaPlayer? = null + private val mainHandler = Handler(Looper.getMainLooper()) + /** 静音开关：设为 true 时停止当前播放、跳过后续 TTS，文本接收完后主动关闭连接 */ + var ttsMuted = false + set(value) { + field = value + if (value) { + // 立即停止当前播放 + ttsQueue.clear() + ttsPlaying = false + mainHandler.post { + try { currentPlayer?.stop(); currentPlayer?.release() } catch (_: Exception) {} + currentPlayer = null + } + emit("tts_canceled", emptyMap()) + // 如果会话已结束（文本已全部接收），主动关闭连接 + if (sessionEnded) { + closeWs() + if (inCall) { resetVad(); startIdleTimer() } + } + } + } + + // ── 消息历史（WS 上下文） ────────────────────────────────────────────────── + private var messages = JSONArray() + private val chatStorageHelper = ChatStorageHelper(context) + private var currentSessionId = "" + private var pendingAnswer = "" // 累积最完整的回复文本，会话结束时统一发送 + + // ── 计时器 ──────────────────────────────────────────────────────────────── + private var micWatchdogJob: Job? = null + private var lastWatchdogChunkCount = 0 + private var idleTimerJob: Job? = null + private val IDLE_TIMEOUT_MS = 15_000L + + // ── 凭证就绪 ────────────────────────────────────────────────────────────── + var credentialReady = false + private set + + // ── 心跳冷却 ────────────────────────────────────────────────────────────── + private val HEARTBEAT_COOLDOWN_MS = 3_600_000L // 1 小时 + + // ═════════════════════════════════════════════════════════════════════════ + // 凭证获取 + // ═════════════════════════════════════════════════════════════════════════ + + fun fetchCredentials() { + if (config.deviceSecret.isNotEmpty()) { + credentialReady = true + Log.i(TAG, "[YIDONG] fetchCredentials: 凭证已存在，跳过请求 deviceNo=${config.deviceNo} deviceId=${config.deviceId}") + reportHeartbeat() // 凭证已有，直接上报心跳 + return + } + Log.i(TAG, "[YIDONG] fetchCredentials: 开始请求密钥 productId=${config.productId} deviceNo=${config.deviceNo}") + launch { + try { + val body = JSONObject().apply { + put("deviceNoType", "SN") + put("deviceNo", config.deviceNo) + put("productId", config.productId) + put("productKey", config.productKey) + }.toString().toRequestBody("application/json".toMediaType()) + + val req = Request.Builder() + .url("https://ivs.chinamobiledevice.com:11443/v2/customer/device/secret/info") + .post(body) + .build() + + okHttpClient.newCall(req).execute().use { resp -> + val json = JSONObject(resp.body!!.string()) + if (json.getString("code") == "200" && json.getBoolean("success")) { + val d = json.getJSONObject("data") + config = config.copy( + deviceNo = d.optString("deviceNo", config.deviceNo), + deviceId = d.optString("deviceId", ""), + deviceSecret = d.optString("deviceSecret", ""), + ) + credentialReady = config.deviceSecret.isNotEmpty() + if (credentialReady) { + Log.i(TAG, "[YIDONG] fetchCredentials: 密钥获取成功 deviceId=${config.deviceId} deviceNo=${config.deviceNo} cmei=${config.cmei}") + onCredentialsFetched?.invoke(config.deviceId, config.deviceSecret, config.deviceNo) + reportHeartbeat() // 凭证获取成功后立即上报心跳 + } else { + Log.e(TAG, "[YIDONG] fetchCredentials: 响应成功但deviceSecret为空") + emit("credential_failed", mapOf("reason" to "设备未认证，deviceSecret为空")) + } + } else { + val msg = json.optString("message", "未知错误") + Log.e(TAG, "[YIDONG] fetchCredentials: 密钥获取失败 code=${json.optString("code")} msg=$msg") + emit("credential_failed", mapOf("reason" to "设备未认证：$msg")) + } + } + } catch (e: Exception) { + Log.e(TAG, "[YIDONG] fetchCredentials: 请求异常 ${e.message}") + emit("credential_failed", mapOf("reason" to "设备认证请求失败：${e.message}")) + } + } + } + + /** 向中国移动平台上报心跳 */ + fun reportHeartbeat() { + if (!credentialReady) { + Log.w(TAG, "[YIDONG] reportHeartbeat: 凭证未就绪，跳过上报") + return + } + // 1 小时冷却：StsAgent 会被反复重建，init/fetchCredentials 都会调到这里， + // 必须在原生层拦，否则每次重建都会打一发心跳 + val prefs = context.getSharedPreferences("sts_agent", Context.MODE_PRIVATE) + val last = prefs.getLong("last_heartbeat_ts", 0L) + val now = System.currentTimeMillis() + if (last > 0L && now - last < HEARTBEAT_COOLDOWN_MS) { + Log.i(TAG, "[YIDONG] reportHeartbeat: 距上次 ${(now - last) / 1000}s，冷却中跳过") + return + } + prefs.edit().putLong("last_heartbeat_ts", now).apply() + Log.i(TAG, "[YIDONG] reportHeartbeat: 开始上报 deviceId=${config.deviceId} deviceNo=${config.deviceNo} cmei=${config.cmei}") + launch { + try { + val netType = try { + val cm = context.getSystemService(android.content.Context.CONNECTIVITY_SERVICE) + as android.net.ConnectivityManager + val net = cm.activeNetworkInfo + when (net?.type) { + android.net.ConnectivityManager.TYPE_WIFI -> "WiFi" + android.net.ConnectivityManager.TYPE_MOBILE -> "5G" + else -> "WiFi" + } + } catch (_: Exception) { "WiFi" } + + val body = JSONObject().apply { + put("deviceId", config.deviceId) + put("deviceSecret", config.deviceSecret) + put("productId", config.productId) + put("productKey", config.productKey) + put("params", JSONObject().apply { + put("innerIp", JSONArray().put("127.0.0.1")) + put("netSpeed", "0Mbps") + put("netType", netType) + put("platform", "Android${android.os.Build.VERSION.RELEASE}") + put("sdkVersion", "ai_http_1.0") + put("firmwareVersion", "1.0.0") + put("imei", "") + put("cmei", config.cmei) // 优先使用 UserDevice 的 CMEI + put("mac", "") + }) + }.toString().toRequestBody("application/json".toMediaType()) + + val req = Request.Builder() + .url("https://ivs.chinamobiledevice.com:11443/v2/customer/device/report") + .post(body) + .build() + + okHttpClient.newCall(req).execute().use { resp -> + val json = JSONObject(resp.body!!.string()) + val code = json.optString("code") + val msg = json.optString("message") + if (code == "200") { + Log.i(TAG, "[YIDONG] reportHeartbeat: 上报成功 code=$code msg=$msg") + emit("heartbeat_reported", mapOf("time" to System.currentTimeMillis())) + } else { + Log.e(TAG, "[YIDONG] reportHeartbeat: 上报失败 code=$code msg=$msg") + // 上报失败回滚时间戳，避免卡 1 小时无法重试 + prefs.edit().putLong("last_heartbeat_ts", last).apply() + } + } + } catch (e: Exception) { + Log.e(TAG, "[YIDONG] reportHeartbeat: 请求异常 ${e.message}") + // 异常同样回滚时间戳 + prefs.edit().putLong("last_heartbeat_ts", last).apply() + } + } + } + + // ═════════════════════════════════════════════════════════════════════════ + // 公开接口 + // ═════════════════════════════════════════════════════════════════════════ + + /** 通话模式：开启麦克风，VAD 自动触发 WS */ + fun startCall() { + if (inCall) return + if (ttsPlaying) interruptTts() + if (pttRecording) stopPtt(cancel = true) + if (!credentialReady) { emitError("移动精灵凭证未就绪，请稍后重试"); return } + Log.i(TAG, "[YIDONG] startCall: 启动通话模式 deviceNo=${config.deviceNo} agentId=${config.agentId}") + openMic() + startMicWatchdog() + inCall = true + startIdleTimer() + } + + /** PTT 模式：立即连 WS 并开始录音 */ + fun startPtt() { + if (pttRecording) { + if (forwardingAudio) return // 真正在录音中，忽略重复调用 + // forwardingAudio=false：WS 已关闭，正处于 TTS 播报等待阶段（如 BLE 唤醒等待 AI 回复期间） + // 调用方（BleAgent）已通过 interruptCurrentResponse() 中断了 TTS， + // 此处只做静默状态重置，不再重复发出 tts_canceled 事件 + Log.i(TAG, "[YIDONG] startPtt: 静默重置旧会话，重新启动") + stopMicHard() + sessionEnded = false + pttRecording = false + } + if (ttsPlaying) interruptTts() + if (!credentialReady) { emitError("移动精灵凭证未就绪"); return } + Log.i(TAG, "[YIDONG] startPtt: 启动PTT模式 deviceNo=${config.deviceNo} agentId=${config.agentId}") + + pttRecording = true + openMic() + startMicWatchdog() + ttsInterrupted = false + closeWs() + sessionEnded = false + wsConnecting = true + dialogRequestId = UUID.randomUUID().toString() + clearSendBuffer() + pendingAnswer = "" + + launch { + try { + connectWs() + forwardingAudio = true + wsConnecting = false + emit("recognition_started", emptyMap()) + } catch (e: Exception) { + wsConnecting = false + pttRecording = false + stopMicWatchdog() + emitError("连接失败: ${e.message}") + } + } + } + + /** PTT 松开：发送剩余音频 + finish */ + fun stopPtt(cancel: Boolean = false) { + pttRecording = false + stopMicWatchdog() + if (cancel) { + forwardingAudio = false + closeWs() + stopMicHard() + return + } + if (forwardingAudio) { + forwardingAudio = false + synchronized(sendBufferLock) { + val remain = sendBuffer.size - sendBufferOffset + if (remain > 0) { + sendAudioChunk(sendBuffer.toMutableList().subList(sendBufferOffset, sendBuffer.size).toByteArray()) + } + sendBuffer.clear(); sendBufferOffset = 0 + } + sendFinish() + emit("recognition_stopped", emptyMap()) + } + stopMicHard() + } + + /** 统一停止：根据当前状态自动选择 stopPtt 或 disconnect */ + fun stop() { + if (pttRecording) stopPtt() + else disconnect() + } + + /** 挂断通话 */ + fun disconnect() { + stopMicWatchdog() + cancelIdleTimer() + forwardingAudio = false + wsConnecting = false + speechDetected = false + stopMicHard() + ttsInterrupted = true + ttsQueue.clear() + ttsPlaying = false + mainHandler.post { + try { currentPlayer?.stop(); currentPlayer?.release() } catch (_: Exception) {} + currentPlayer = null + } + closeWs() + synchronized(preBufferLock) { preBuffer.clear() } + clearSendBuffer() + flushPendingAnswer() + inCall = false + pttRecording = false + } + + /** 打断 AI 播报 */ + fun interruptSpeaking() { + interruptTts() + ttsInterrupted = false + if (inCall) resetVad() + } + + /** 清空消息历史 */ + fun clearHistory() { + messages = JSONArray() + } + + /** 释放资源 */ + fun dispose() { + disconnect() + job.cancel() + } + + // ═════════════════════════════════════════════════════════════════════════ + // 麦克风 & 本地 VAD + // ═════════════════════════════════════════════════════════════════════════ + + private fun openMic() { + stopMicHard() + speechDetected = false; vadOnsetCount = 0; micChunkCount = 0 + try { + audioRecord = AudioRecord( + MediaRecorder.AudioSource.VOICE_COMMUNICATION, + SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + RECORD_BUFFER + ) + audioRecord?.startRecording() + recordingJob = launch { + val buf = ByteArray(RECORD_CHUNK) + while (isActive && (inCall || pttRecording)) { + // 使用非阻塞读 + micLock，确保与 stopMicHard() 中的 stop()/release() 互斥。 + // READ_NON_BLOCKING 让 read() 立即返回（微秒级），锁持有时间极短， + // 彻底消除 releaseBuffer 并发导致的 native SIGABRT。 + val read = synchronized(micLock) { + audioRecord?.read(buf, 0, buf.size, AudioRecord.READ_NON_BLOCKING) ?: -1 + } + if (read < 0) break + if (read > 0) { micChunkCount++; onMicAudio(buf.copyOf(read)) } + else delay(5) // 无数据时释放协程让出线程，避免忙等 + } + } + } catch (e: Exception) { + Log.e(TAG, "打开麦克风失败: ${e.message}") + } + } + + private fun stopMicHard() { + recordingJob?.cancel(); recordingJob = null + // 拿 micLock 后再 stop()/release()，与录音循环的 read() 互斥， + // 保证 stop() 永远不会在 read() 内部 releaseBuffer 期间被调用。 + synchronized(micLock) { + val ar = audioRecord + audioRecord = null + try { ar?.stop(); ar?.release() } catch (_: Exception) {} + } + } + + private fun startMicWatchdog() { + stopMicWatchdog() + lastWatchdogChunkCount = micChunkCount + micWatchdogJob = launch { + while (isActive && (inCall || pttRecording)) { + delay(1500) + if (micChunkCount == lastWatchdogChunkCount && !ttsPlaying) openMic() + lastWatchdogChunkCount = micChunkCount + } + } + } + + private fun stopMicWatchdog() { micWatchdogJob?.cancel(); micWatchdogJob = null } + + private fun onMicAudio(chunk: ByteArray) { + if (!inCall && !pttRecording) return + if (forwardingAudio && webSocket != null) { + if (micChunkCount % 3 == 0) emit("mic_rms", mapOf("rms" to calcRms(chunk))) + feedSendBuffer(chunk); return + } + if (wsConnecting) { addToPreBuffer(chunk); return } + if (ttsPlaying) return + + addToPreBuffer(chunk) + val rms = calcRms(chunk) + // 发送麦克风音量给 Dart 层 + if (micChunkCount % 3 == 0) { + emit("mic_rms", mapOf("rms" to rms)) + } + if (rms >= VAD_THRESHOLD) { + vadOnsetCount++ + if (vadOnsetCount >= VAD_ONSET_CHUNKS && !speechDetected) { + speechDetected = true + cancelIdleTimer() + launch { onSpeechDetected() } + } + } else { + vadOnsetCount = 0 + } + } + + private fun calcRms(pcm: ByteArray): Double { + if (pcm.size < 2) return 0.0 + val samples = pcm.size / 2 + var sum = 0.0 + for (i in 0 until pcm.size - 1 step 2) { + var s = (pcm[i].toInt() and 0xFF) or (pcm[i + 1].toInt() shl 8) + if (s >= 32768) s -= 65536 + sum += s.toLong() * s.toLong() + } + return sqrt(sum / samples) + } + + private fun addToPreBuffer(chunk: ByteArray) { + synchronized(preBufferLock) { + preBuffer.addLast(chunk.copyOf()) + while (preBuffer.size > PRE_BUFFER_MAX_CHUNKS) preBuffer.removeFirst() + } + } + + // ═════════════════════════════════════════════════════════════════════════ + // VAD 触发 → WS 连接 + // ═════════════════════════════════════════════════════════════════════════ + + private suspend fun onSpeechDetected() { + if (wsConnecting) return + ttsInterrupted = false + closeWs() + sessionEnded = false + wsConnecting = true + dialogRequestId = UUID.randomUUID().toString() + clearSendBuffer() + pendingAnswer = "" + + try { + connectWs() + } catch (e: Exception) { + wsConnecting = false; speechDetected = false + emitError("连接失败: ${e.message}") + return + } + + forwardingAudio = true; wsConnecting = false + playReadyBeep() + emit("recognition_started", emptyMap()) + + val cached = synchronized(preBufferLock) { preBuffer.toList().also { preBuffer.clear() } } + for (c in cached) feedSendBuffer(c) + } + + /** 唤醒成功提示音：通知用户可以说话了 */ + private fun playReadyBeep() { + try { + val toneGen = ToneGenerator(AudioManager.STREAM_MUSIC, 60) + toneGen.startTone(ToneGenerator.TONE_PROP_BEEP, 180) + mainHandler.postDelayed({ toneGen.release() }, 300) + } catch (e: Exception) { + Log.w(TAG, "playReadyBeep failed: ${e.message}") + } + } + + // ═════════════════════════════════════════════════════════════════════════ + // WebSocket + // ═════════════════════════════════════════════════════════════════════════ + + private suspend fun connectWs(): Unit = suspendCancellableCoroutine { cont -> + val ts = System.currentTimeMillis() + val sign = calcSign(config.deviceSecret, ts.toString()) + val sn = UUID.randomUUID().toString() + val url = "wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr" + + "?deviceNo=${config.deviceNo}&deviceId=${config.deviceId}" + + "&productId=${config.productId}&productKey=${config.productKey}" + + "&sign=$sign&sn=$sn&ts=$ts" + Log.i(TAG, "[YIDONG] connectWs: 连接WSS deviceNo=${config.deviceNo} deviceId=${config.deviceId} sn=$sn") + + val req = Request.Builder().url(url) + .header("Origin", "https://ivs.chinamobiledevice.com:11443") + .header("Host", "ivs.chinamobiledevice.com:11443") + .build() + + var resolved = false + + webSocket = okHttpClient.newWebSocket(req, object : WebSocketListener() { + override fun onOpen(ws: WebSocket, response: Response) { + sendStartFrame() + } + + override fun onMessage(ws: WebSocket, text: String) { + try { + val msg = JSONObject(text) + if (msg.optString("type") == "ready") { + Log.i(TAG, "[YIDONG] connectWs: 收到ready，WSS连接成功") + if (!resolved) { resolved = true; cont.resume(Unit) {} } + } else { + launch { handleMessage(msg) } + } + } catch (_: Exception) {} + } + + override fun onMessage(ws: WebSocket, bytes: okio.ByteString) {} + + override fun onFailure(ws: WebSocket, t: Throwable, response: Response?) { + Log.e(TAG, "[YIDONG] connectWs: WSS连接失败 ${t.message}") + if (webSocket === ws) webSocket = null // 只清除自己的引用，避免覆盖新 WS + if (!resolved) { resolved = true; cont.resumeWithException(t) } + else launch { handleWsDone() } + } + + override fun onClosing(ws: WebSocket, code: Int, reason: String) { ws.close(1000, null) } + + override fun onClosed(ws: WebSocket, code: Int, reason: String) { + if (webSocket === ws) webSocket = null // 只清除自己的引用，避免覆盖新 WS + if (!resolved) { resolved = true; cont.resumeWithException(IOException("WS closed")) } + else launch { handleWsDone() } + } + }) + + cont.invokeOnCancellation { closeWs() } + + // 15s 超时 + launch { + delay(15_000) + if (!resolved) { + resolved = true; closeWs() + cont.resumeWithException(IOException("等待 ready 超时")) + } + } + } + + private fun handleWsDone() { + forwardingAudio = false; wsConnecting = false + when { + sessionEnded && inCall -> if (!ttsPlaying) { resetVad(); startIdleTimer() } + sessionEnded -> { + // PTT 结束；若无 TTS 排队或播放（如无语音输入场景），立即补发 tts_completed + // 避免 Flutter 侧 _pttResultPending 永久无法清除导致按钮锁死 + if (!ttsPlaying && ttsQueue.isEmpty()) emit("tts_completed", emptyMap()) + } + inCall -> resetVad() + // WS 意外关闭（session 未正常结束，如极速松手音频未发出），同样补发保底 + else -> if (!ttsPlaying && ttsQueue.isEmpty() && !ttsInterrupted) emit("tts_completed", emptyMap()) + } + } + + private fun resetVad() { + speechDetected = false; vadOnsetCount = 0 + synchronized(preBufferLock) { preBuffer.clear() } + openMic() + } + + private fun closeWs() { + try { webSocket?.cancel() } catch (_: Exception) {} + webSocket = null + } + + // ═════════════════════════════════════════════════════════════════════════ + // 发送帧 + // ═════════════════════════════════════════════════════════════════════════ + + private fun sendStartFrame() { + currentSessionId = UUID.randomUUID().toString() + val payload = JSONObject().apply { + put("type", "start") + put("data", JSONObject().apply { + put("cuid", config.deviceId) + put("format", "pcm") + put("sample", 16000) + put("support_dcs", 2) + put("chunk_size", 10240) + put("support_tts", true) + put("support_text2dcs", true) + put("user_agent", "voitrans/1.0") + put("dialog_request_id", dialogRequestId) + put("access_rc", true) + put("support_part_tts", true) + put("need_stoplisten", true) + put("need_dialogue_finish", true) + put("result_trans2directive", false) + put("rc_version", "5") + put("messages", messages) + put("client_context", JSONArray()) + }) + } + webSocket?.send(payload.toString()) + } + + private val sendBufferLock = Any() + + private fun clearSendBuffer() { + synchronized(sendBufferLock) { + sendBuffer.clear(); sendBufferOffset = 0 + } + } + + private fun feedSendBuffer(chunk: ByteArray) { + synchronized(sendBufferLock) { + sendBuffer.addAll(chunk.toList()) + while (sendBuffer.size - sendBufferOffset >= FRAME_SIZE) { + val frame = ByteArray(FRAME_SIZE) { i -> sendBuffer[sendBufferOffset + i] } + sendBufferOffset += FRAME_SIZE + sendAudioChunk(frame) + } + // 收缩缓冲 + if (sendBufferOffset >= FRAME_SIZE * 8) { + val toRemove = sendBufferOffset + repeat(toRemove) { if (sendBuffer.isNotEmpty()) sendBuffer.removeFirst() } + sendBufferOffset = 0 + } + } + } + + private fun sendAudioChunk(data: ByteArray) { + try { webSocket?.send(data.toByteString()) } catch (_: Exception) {} + } + + private fun sendFinish() { + try { webSocket?.send("""{"type":"finish"}""") } catch (_: Exception) {} + } + + // ═════════════════════════════════════════════════════════════════════════ + // 消息处理 + // ═════════════════════════════════════════════════════════════════════════ + + private suspend fun handleMessage(msg: JSONObject) { + when (msg.optString("type")) { + "stoplisten" -> { + forwardingAudio = false; speechDetected = false; vadOnsetCount = 0 + synchronized(sendBufferLock) { + val remain = sendBuffer.size - sendBufferOffset + if (remain > 0) { + sendAudioChunk(ByteArray(remain) { i -> sendBuffer[sendBufferOffset + i] }) + } + sendBuffer.clear(); sendBufferOffset = 0 + } + sendFinish() + emit("recognition_stopped", emptyMap()) + } + "mid_result" -> { + val text = msg.optString("result", "") + if (text.isNotEmpty()) emit("recognizing", mapOf("text" to text)) + } + "fin_result" -> { + val result = msg.optString("result", "") + if (result.isNotEmpty()) { + emit("recognition_result", mapOf("text" to result)) + messages.put(JSONObject(mapOf("role" to "user", "content" to result))) + withContext(Dispatchers.IO) { + try { + chatStorageHelper.saveMessage(config.agentId, currentSessionId, result, "user") + } catch (_: Exception) {} + } + } + if (msg.optInt("end", 0) == 1) markSessionEnded() + } + "inside_rc" -> handleInsideRc(msg) + "dcs_decide" -> if (msg.optInt("end", 0) == 1) markSessionEnded() + } + } + + private suspend fun handleInsideRc(msg: JSONObject) { + val data = msg.optJSONObject("data") ?: return + if (data.optInt("code", -1) != 0) return + + val rawAnswer = data.optString("assistant_answer", "") + val cleanAnswer = try { + JSONObject(rawAnswer).optString("content", rawAnswer) + } catch (_: Exception) { rawAnswer } + + var ttsUrl: String? = null + val items = data.optJSONArray("data") + if (items != null) { + for (i in 0 until items.length()) { + val item = items.optJSONObject(i) ?: continue + val h = item.optJSONObject("header") ?: continue + if (h.optString("namespace") == "ai.fxzsos.device_interface.voice_output" && + h.optString("name") == "Speak" + ) { + ttsUrl = item.optJSONObject("payload")?.optString("url") + break + } + } + } + + // 每次取更完整的回复文本，立即发送给 UI + if (cleanAnswer.isNotEmpty() && cleanAnswer.length > pendingAnswer.length) { + pendingAnswer = cleanAnswer + emit("assistant_response", mapOf("response" to cleanAnswer, "sessionid" to currentSessionId)) + } + ttsUrl?.takeIf { it.isNotEmpty() }?.let { enqueueTts(it) } + if (msg.optInt("end", 0) == 1) markSessionEnded() + } + + /** 将 pendingAnswer 保存到历史和存储，并清空 */ + private fun flushPendingAnswer() { + if (pendingAnswer.isEmpty()) return + val answer = pendingAnswer + val sid = currentSessionId + messages.put(JSONObject(mapOf("role" to "assistant", "content" to answer))) + launch(Dispatchers.IO) { + try { + chatStorageHelper.saveMessage(config.agentId, sid, answer, "assistant") + } catch (_: Exception) {} + } + pendingAnswer = "" + } + + private fun markSessionEnded() { + if (sessionEnded) return + sessionEnded = true; forwardingAudio = false; speechDetected = false; vadOnsetCount = 0 + clearSendBuffer() + flushPendingAnswer() + // 静音模式下文本接收完毕，发送完成事件并关闭连接 + if (ttsMuted) { + emit("tts_completed", emptyMap()) + closeWs() + if (inCall) { resetVad(); startIdleTimer() } + } + } + + // ═════════════════════════════════════════════════════════════════════════ + // TTS 播放队列 + // ═════════════════════════════════════════════════════════════════════════ + + private fun enqueueTts(url: String) { + if (ttsMuted) return // 静音模式跳过 TTS + ttsQueue.addLast(url) + if (!ttsPlaying) { + stopMicHard() + mainHandler.post { playNextTts() } + } + } + + private fun playNextTts() { + if (ttsQueue.isEmpty() || ttsInterrupted) { + ttsPlaying = false + if (!ttsInterrupted) { + emit("tts_completed", emptyMap()) + if (inCall) { resetVad(); startIdleTimer() } + } + return + } + ttsPlaying = true + emit("tts_started", emptyMap()) + val url = ttsQueue.removeFirst() + try { + currentPlayer?.stop(); currentPlayer?.release() + currentPlayer = MediaPlayer().apply { + setDataSource(url) + setOnCompletionListener { + if (!ttsInterrupted) mainHandler.post { playNextTts() } + } + setOnErrorListener { _, _, _ -> + if (!ttsInterrupted) mainHandler.post { playNextTts() } + true + } + prepare() + start() + } + } catch (e: Exception) { + Log.e(TAG, "TTS 播放失败: ${e.message}") + if (!ttsInterrupted) mainHandler.post { playNextTts() } + } + } + + private fun interruptTts() { + ttsInterrupted = true; ttsQueue.clear(); ttsPlaying = false + mainHandler.post { + try { currentPlayer?.stop(); currentPlayer?.release() } catch (_: Exception) {} + currentPlayer = null + } + emit("tts_canceled", emptyMap()) + } + + // ═════════════════════════════════════════════════════════════════════════ + // 超时 & 工具 + // ═════════════════════════════════════════════════════════════════════════ + + private fun startIdleTimer() { + cancelIdleTimer() + idleTimerJob = launch { + delay(IDLE_TIMEOUT_MS) + if (inCall && !speechDetected) disconnect() + } + } + + private fun cancelIdleTimer() { idleTimerJob?.cancel(); idleTimerJob = null } + + private fun calcSign(secret: String, ts: String): String { + val digest = MessageDigest.getInstance("MD5").digest("$secret$ts".toByteArray()) + return digest.joinToString("") { "%02x".format(it) } + } + + private fun emit(eventName: String, data: Map) { + launch(Dispatchers.Main) { onEvent(eventName, data) } + } + + private fun emitError(msg: String) = emit("error", mapOf("message" to msg)) + + private fun buildTrustAllClient(): OkHttpClient { + return try { + val trustAll = arrayOf(object : X509TrustManager { + override fun checkClientTrusted(c: Array, t: String) {} + override fun checkServerTrusted(c: Array, t: String) {} + override fun getAcceptedIssuers(): Array = emptyArray() + }) + val sc = SSLContext.getInstance("TLS") + sc.init(null, trustAll, SecureRandom()) + OkHttpClient.Builder() + .sslSocketFactory(sc.socketFactory, trustAll[0] as X509TrustManager) + .hostnameVerifier { _, _ -> true } + .connectTimeout(15, TimeUnit.SECONDS) + .readTimeout(90, TimeUnit.SECONDS) + .writeTimeout(90, TimeUnit.SECONDS) + .build() + } catch (e: Exception) { + Log.e(TAG, "buildTrustAllClient 失败，使用默认客户端", e) + OkHttpClient() + } + } +} diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index af68fa34f..219f19208 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -101,8 +101,12 @@ class AgentServiceImpl: NSObject { return ChatStorageHelper.shared } + // Agent 模式 + private var agentMode = "stt_llm_tts" + private var stsAgent: StsAgent? + // 会话ID，用于区分不同聊天上下文 - private let agentId = "default_agent" + private var agentId = "default_agent" //工具调用提示 public var callingToolText = "" // 工具调用提示 public var insufficientIntegralText = "" // 积分不足提示 @@ -229,9 +233,39 @@ class AgentServiceImpl: NSObject { func initialize(config: [String: Any]) -> Bool { // if isInitialized { return true } // os_log("initialize: config=%{public}@", log: logger, type: .info, config) - + + agentMode = config["agentMode"] as? String ?? "stt_llm_tts" + agentId = (config["agentId"] as? String)?.isEmpty == false + ? config["agentId"] as! String : "default_agent" + + // STS 模式：跳过 Azure/OpenAI 初始化 + if agentMode == "sts" { + let stsConfig = StsConfig( + productId: config["stsProductId"] as? String ?? "", + productKey: config["stsProductKey"] as? String ?? "", + deviceNo: config["stsDeviceNo"] as? String ?? "", + deviceId: config["stsDeviceId"] as? String ?? "", + deviceSecret: config["stsDeviceSecret"] as? String ?? "", + agentId: agentId, + cmei: config["stsCmei"] as? String ?? "" + ) + stsAgent = StsAgent( + config: stsConfig, + onEvent: { [weak self] name, data in self?.sendEvent(name: name, data: data) }, + onCredentialsFetched: { [weak self] deviceId, deviceSecret, deviceNo in + self?.sendEvent(name: "sts_credentials", data: [ + "deviceId": deviceId, + "deviceSecret": deviceSecret, + "deviceNo": deviceNo, + ]) + } + ) + isInitialized = true + return true + } + self.config = config - + if let navigationMode = config["navigationMode"] as? String { self.navigationMode = navigationMode } @@ -522,6 +556,17 @@ class AgentServiceImpl: NSObject { } } func startRecognition(useBle: Bool = false, mode: String = "normal") -> Bool { + if agentMode == "sts" { + switch mode { + case "push_to_talk": + stsAgent?.startPtt() + case "phone_call", "ble_wakeup": + stsAgent?.startCall() + default: + stsAgent?.startPtt() + } + return true + } if !isInitialized { sendError("服务未初始化", code: "NOT_INITIALIZED") return false @@ -607,6 +652,10 @@ class AgentServiceImpl: NSObject { } func stopRecognition() -> Bool { + if agentMode == "sts" { + stsAgent?.stop() + return true + } /// 函数功能：停止语音识别 /// 特性： /// 1) 即便处于"启动中"（isStartingRecognition）也会打印日志并尝试停止 @@ -710,7 +759,21 @@ class AgentServiceImpl: NSObject { return success } + /// STS 模式下主动触发中国移动平台的凭证获取（Dart AgentService.fetchCredentials 会调到这里） + func fetchCredentials() { + stsAgent?.fetchCredentials() + } + + /// STS 模式下主动上报一次心跳（1 小时冷却在原生层拦截） + func reportHeartbeat() { + stsAgent?.reportHeartbeat() + } + func interruptCurrentResponse() -> Bool { + if agentMode == "sts" { + stsAgent?.interruptSpeaking() + return true + } var interrupted = false if isSpeaking { @@ -1530,8 +1593,12 @@ private func jsonToString(_ json: [String: Any]) -> String? { return result } func clearChatHistory() -> Bool { + if agentMode == "sts" { + stsAgent?.clearHistory() + return true + } chatHistory.removeAll() - + DispatchQueue.global(qos: .utility).async { let success = self.chatStorageHelper.deleteMessages(agentId:self.agentId, messageIds: nil as [Int]?) if !success { @@ -1778,6 +1845,12 @@ private func jsonToString(_ json: [String: Any]) -> String? { func dispose() -> Bool { _log("INFO", "[Agent] dispose") + if agentMode == "sts" { + stsAgent?.dispose() + stsAgent = nil + isInitialized = false + return true + } if isRecognizing { // 关闭编码器 (设备侧) BleService.shared.closeCodec() diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift index 82f53326b..b8d998ee4 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift @@ -203,6 +203,17 @@ public class AgentServicePlugin: NSObject, FlutterPlugin { } impl.setCanWakeUp(canWakeUp) result(true) + + case "reportHeartbeat": + // 中国移动精灵心跳上报，原生层持久化 1 小时冷却 + impl.reportHeartbeat() + result(nil) + + case "fetchCredentials": + // 中国移动精灵凭证拉取，成功后会自动触发心跳（受冷却保护） + impl.fetchCredentials() + result(nil) + default: result(FlutterMethodNotImplemented) } diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/StsAgent.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/StsAgent.swift new file mode 100644 index 000000000..477eb78cb --- /dev/null +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/StsAgent.swift @@ -0,0 +1,613 @@ +import Foundation +import UIKit +import AVFoundation +import CryptoKit +import os.log +import chat_storage + +// MARK: - StsConfig + +struct StsConfig { + let productId: String + let productKey: String + var deviceNo: String + var deviceId: String + var deviceSecret: String + let agentId: String + var cmei: String = "" +} + +// MARK: - StsAgent + +/// StsAgent — 中国移动 A80 WS 一体化语音对话实现（iOS） +/// +/// 通过一条 WSS 连接完成 ASR + AI + TTS，事件格式与标准 stt_llm_tts 完全一致， +/// 通过 onEvent 回调发出，供 AgentServiceImpl 转发到 Flutter EventChannel。 +class StsAgent: NSObject { + + private let TAG = "StsAgent" + private let logger = OSLog(subsystem: "com.yunqiinnovation.agent_service", category: "StsAgent") + + private var config: StsConfig + private let onEvent: (String, [String: Any]) -> Void + private let onCredentialsFetched: ((String, String, String) -> Void)? + + // MARK: WebSocket + private var webSocketTask: URLSessionWebSocketTask? + private var urlSession: URLSession! + private var readyReceived = false + private var sessionEnded = true + private var dialogRequestId = "" + + // MARK: 录音 + private let SAMPLE_RATE: Double = 16000 + private let FRAME_SIZE = 5120 // 160ms @ 16kHz 16bit mono + private var audioEngine: AVAudioEngine? + private var inputNode: AVAudioInputNode? + private var recordingActive = false + private var sendBuffer = Data() + private var micChunkCount = 0 + + // MARK: VAD + private let VAD_THRESHOLD: Float = 800 + private let VAD_ONSET_CHUNKS = 3 + private var vadOnsetCount = 0 + private var vadActive = false + private var preBuffer: [[UInt8]] = [] + private let PRE_BUFFER_MAX = 75 + + // MARK: TTS + private var ttsQueue: [String] = [] + private var ttsPlayer: AVPlayer? + private var ttsPlaying = false + private var ttsPlayerObserver: Any? + + // MARK: 历史 + private var messages: [[String: String]] = [] + private var currentSessionId = "" + private let chatStorage = ChatStorageHelper.shared + + // MARK: 模式 + private var isPttMode = false + + // MARK: 凭证请求 + private var credentialsFetched = false + + // MARK: 心跳冷却 + /// 1 小时冷却（毫秒），避免 StsAgent 反复重建 / fetchCredentials 反复触发时频繁上报 + private let HEARTBEAT_COOLDOWN_MS: Int64 = 3_600_000 + private let HEARTBEAT_PREFS_KEY = "sts_agent_last_heartbeat_ts" + + init(config: StsConfig, + onEvent: @escaping (String, [String: Any]) -> Void, + onCredentialsFetched: ((String, String, String) -> Void)? = nil) { + self.config = config + self.onEvent = onEvent + self.onCredentialsFetched = onCredentialsFetched + super.init() + let sessionConfig = URLSessionConfiguration.default + urlSession = URLSession(configuration: sessionConfig, delegate: nil, delegateQueue: OperationQueue()) + os_log("StsAgent 初始化: productId=%{public}@ deviceNo=%{public}@ deviceId=%{public}@ cmei=%{public}@ hasSecret=%{public}@", + log: logger, type: .info, + config.productId, config.deviceNo, config.deviceId, config.cmei, + config.deviceSecret.isEmpty ? "false" : "true") + // 与 Android 行为对齐：凭证已有时立即尝试上报一次心跳（受 1 小时冷却保护） + if !config.deviceSecret.isEmpty && !config.deviceId.isEmpty { + credentialsFetched = true + reportHeartbeat() + } + } + + /// 动态更新 CMEI（首次初始化时可能尚未获取到） + func updateCmei(_ cmei: String) { + guard !cmei.isEmpty, cmei != config.cmei else { return } + config.cmei = cmei + os_log("updateCmei: cmei 已更新为 %{public}@", log: logger, type: .info, cmei) + } + + // MARK: - 公共接口 + + func startCall() { + isPttMode = false + os_log("StsAgent startCall", log: logger, type: .info) + fetchCredentialsIfNeeded { [weak self] in + self?.connect() + } + } + + func startPtt() { + isPttMode = true + os_log("StsAgent startPtt", log: logger, type: .info) + fetchCredentialsIfNeeded { [weak self] in + self?.connect() + } + } + + func stopPtt() { + os_log("StsAgent stopPtt", log: logger, type: .info) + guard isPttMode else { return } + sendFinishFrame() + } + + /// 统一停止：根据当前模式自动选择 stopPtt 或 dispose + func stop() { + if isPttMode { + stopPtt() + } else { + dispose() + } + } + + func interruptSpeaking() { + os_log("StsAgent interruptSpeaking", log: logger, type: .info) + stopTts() + emit("tts_canceled") + } + + func clearHistory() { + messages.removeAll() + DispatchQueue.global().async { [weak self] in + guard let self = self else { return } + _ = self.chatStorage.deleteMessages(agentId: self.config.agentId, messageIds: nil as [Int]?) + } + } + + func dispose() { + os_log("StsAgent dispose", log: logger, type: .info) + stopRecording() + webSocketTask?.cancel(with: .normalClosure, reason: nil) + webSocketTask = nil + stopTts() + } + + // MARK: - 凭证获取 + + /// 公开入口：主动触发认证检查（Dart 层 AgentService.fetchCredentials 会调用） + func fetchCredentials() { + fetchCredentialsIfNeeded { [weak self] in + // 成功与否都不做额外处理；fetchCredentialsIfNeeded 内部会在命中凭证时触发心跳 + _ = self + } + } + + private func fetchCredentialsIfNeeded(completion: @escaping () -> Void) { + if credentialsFetched && !config.deviceId.isEmpty && !config.deviceSecret.isEmpty { + // 凭证已有，直接上报心跳（受冷却保护） + reportHeartbeat() + completion() + return + } + let ts = Int64(Date().timeIntervalSince1970 * 1000) + let sign = calcSign(secret: config.productKey, ts: ts) + let urlStr = "https://ivs.chinamobiledevice.com:11443/v2/customer/device/secret/info" + guard var components = URLComponents(string: urlStr) else { completion(); return } + components.queryItems = [ + URLQueryItem(name: "productId", value: config.productId), + URLQueryItem(name: "deviceNo", value: config.deviceNo), + URLQueryItem(name: "ts", value: "\(ts)"), + URLQueryItem(name: "sign", value: sign), + ] + guard let url = components.url else { completion(); return } + var req = URLRequest(url: url) + req.timeoutInterval = 10 + URLSession.shared.dataTask(with: req) { [weak self] data, _, error in + guard let self = self else { return } + if let error = error { + os_log("凭证请求失败: %{public}@", log: self.logger, type: .error, error.localizedDescription) + completion() + return + } + guard let data = data, + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let body = json["body"] as? [String: Any] else { + completion() + return + } + if let deviceId = body["deviceId"] as? String { self.config.deviceId = deviceId } + if let deviceSecret = body["deviceSecret"] as? String { self.config.deviceSecret = deviceSecret } + if let deviceNo = body["deviceNo"] as? String { self.config.deviceNo = deviceNo } + self.credentialsFetched = true + self.onCredentialsFetched?(self.config.deviceId, self.config.deviceSecret, self.config.deviceNo) + // 凭证获取成功后立即上报心跳（受冷却保护） + self.reportHeartbeat() + completion() + }.resume() + } + + // MARK: - 心跳上报 + + /// 向中国移动平台上报心跳；1 小时冷却通过 UserDefaults 持久化，所有调用路径共享同一把闸门 + func reportHeartbeat() { + guard credentialsFetched, + !config.deviceId.isEmpty, + !config.deviceSecret.isEmpty else { + os_log("reportHeartbeat: 凭证未就绪，跳过上报", log: logger, type: .info) + return + } + let defaults = UserDefaults.standard + // 用 Double 存毫秒时间戳，避开 Int64 的 UserDefaults 桥接问题 + let last = defaults.double(forKey: HEARTBEAT_PREFS_KEY) + let now = Date().timeIntervalSince1970 * 1000 + if last > 0, now - last < Double(HEARTBEAT_COOLDOWN_MS) { + os_log("reportHeartbeat: 距上次 %{public}ds，冷却中跳过", log: logger, type: .info, Int((now - last) / 1000)) + return + } + defaults.set(now, forKey: HEARTBEAT_PREFS_KEY) + os_log("reportHeartbeat: 开始上报 deviceId=%{public}@ deviceNo=%{public}@ cmei=%{public}@", + log: logger, type: .info, config.deviceId, config.deviceNo, config.cmei) + + let urlStr = "https://ivs.chinamobiledevice.com:11443/v2/customer/device/report" + guard let url = URL(string: urlStr) else { return } + var req = URLRequest(url: url) + req.httpMethod = "POST" + req.setValue("application/json", forHTTPHeaderField: "Content-Type") + req.timeoutInterval = 10 + + let systemVersion = UIDevice.current.systemVersion + let params: [String: Any] = [ + "innerIp": ["127.0.0.1"], + "netSpeed": "0Mbps", + "netType": "WiFi", + "platform": "iOS\(systemVersion)", + "sdkVersion": "ai_http_1.0", + "firmwareVersion": "1.0.0", + "imei": "", + "cmei": config.cmei, + "mac": "", + ] + let bodyDict: [String: Any] = [ + "deviceId": config.deviceId, + "deviceSecret": config.deviceSecret, + "productId": config.productId, + "productKey": config.productKey, + "params": params, + ] + guard let bodyData = try? JSONSerialization.data(withJSONObject: bodyDict, options: []) else { return } + req.httpBody = bodyData + + URLSession.shared.dataTask(with: req) { [weak self] data, _, error in + guard let self = self else { return } + if let error = error { + os_log("reportHeartbeat: 请求异常 %{public}@", log: self.logger, type: .error, error.localizedDescription) + // 异常回滚时间戳，避免卡 1 小时无法重试 + UserDefaults.standard.set(last, forKey: self.HEARTBEAT_PREFS_KEY) + return + } + guard let data = data, + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + os_log("reportHeartbeat: 响应解析失败", log: self.logger, type: .error) + UserDefaults.standard.set(last, forKey: self.HEARTBEAT_PREFS_KEY) + return + } + // code 可能是 String 或 NSNumber，统一转成字符串 + let code: String = { + if let s = json["code"] as? String { return s } + if let n = json["code"] as? NSNumber { return n.stringValue } + return "" + }() + let msg = (json["message"] as? String) ?? "" + if code == "200" { + os_log("reportHeartbeat: 上报成功 code=%{public}@ msg=%{public}@", log: self.logger, type: .info, code, msg) + self.emit("heartbeat_reported", data: ["time": Int64(now)]) + } else { + os_log("reportHeartbeat: 上报失败 code=%{public}@ msg=%{public}@", log: self.logger, type: .error, code, msg) + // 上报失败回滚时间戳 + UserDefaults.standard.set(last, forKey: self.HEARTBEAT_PREFS_KEY) + } + }.resume() + } + + // MARK: - WebSocket 连接 + + private func connect() { + let ts = Int64(Date().timeIntervalSince1970 * 1000) + let sign = calcSign(secret: config.deviceSecret, ts: ts) + let urlStr = "wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr" + guard var components = URLComponents(string: urlStr) else { return } + components.queryItems = [ + URLQueryItem(name: "productId", value: config.productId), + URLQueryItem(name: "deviceId", value: config.deviceId), + URLQueryItem(name: "ts", value: "\(ts)"), + URLQueryItem(name: "sign", value: sign), + ] + guard let url = components.url else { return } + webSocketTask = urlSession.webSocketTask(with: url) + webSocketTask?.resume() + receiveMessages() + + // 发送 start 帧 + sendStartFrame() + } + + private func sendStartFrame() { + dialogRequestId = UUID().uuidString.replacingOccurrences(of: "-", with: "").lowercased() + currentSessionId = dialogRequestId + let payload: [String: Any] = [ + "msgType": "start", + "dialogRequestId": dialogRequestId, + "params": [ + "asrParams": [ + "format": "pcm", + "sampleRate": 16000, + "lang": "zh-CN", + "vadEnable": false, + ], + "contextParams": [ + "history": messages.map { ["role": $0["role"] ?? "", "content": $0["content"] ?? ""] } + ] + ] + ] + sendJson(payload) + } + + private func receiveMessages() { + webSocketTask?.receive { [weak self] result in + guard let self = self else { return } + switch result { + case .success(let msg): + switch msg { + case .string(let text): + self.handleMessage(text) + default: + break + } + self.receiveMessages() + case .failure(let err): + os_log("WS error: %{public}@", log: self.logger, type: .error, err.localizedDescription) + self.emit("error", data: ["message": err.localizedDescription]) + } + } + } + + // MARK: - 消息处理 + + private func handleMessage(_ text: String) { + guard let data = text.data(using: .utf8), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return } + let msgType = json["msgType"] as? String ?? "" + os_log("WS recv: %{public}@", log: logger, type: .debug, msgType) + + switch msgType { + case "ready": + readyReceived = true + sessionEnded = false + emit("recognition_started") + startRecording() + if isPttMode { + // PTT 模式等待 stopPtt() 调用发送 finish + } + + case "mid_result": + let result = (json["data"] as? [String: Any])?["result"] as? String ?? "" + if !result.isEmpty { emit("recognizing", data: ["text": result]) } + + case "fin_result": + let result = (json["data"] as? [String: Any])?["result"] as? String ?? "" + emit("recognition_result", data: ["text": result]) + emit("recognition_stopped") + stopRecording() + if !result.isEmpty { + messages.append(["role": "user", "content": result]) + saveToChatStorage(text: result, role: "user") + } + + case "stoplisten": + stopRecording() + emit("recognition_stopped") + + case "inside_rc": + handleInsideRc(json["data"] as? [String: Any]) + + case "dcs_decide": + // 语义决策，忽略 + break + + default: + os_log("未处理的消息类型: %{public}@", log: logger, type: .info, msgType) + } + } + + private func handleInsideRc(_ data: [String: Any]?) { + guard let data = data else { return } + guard let rcType = data["rcType"] as? String else { return } + if rcType == "assistant_answer" { + let answer = (data["answer"] as? [String: Any])?["text"] as? String ?? "" + let cleanAnswer = answer.trimmingCharacters(in: .whitespacesAndNewlines) + let sid = data["sessionId"] as? String ?? currentSessionId + if !cleanAnswer.isEmpty { + messages.append(["role": "assistant", "content": cleanAnswer]) + saveToChatStorage(text: cleanAnswer, role: "assistant") + emit("assistant_response", data: ["response": cleanAnswer, "sessionid": sid]) + } + // TTS URLs + if let ttsData = data["tts"] as? [[String: Any]] { + for item in ttsData { + if let url = item["url"] as? String, !url.isEmpty { + ttsQueue.append(url) + } + } + if !ttsPlaying { playNextTts() } + } + } + } + + // MARK: - TTS 播放 + + private func playNextTts() { + guard !ttsQueue.isEmpty else { + ttsPlaying = false + emit("tts_completed") + return + } + let urlStr = ttsQueue.removeFirst() + guard let url = URL(string: urlStr) else { playNextTts(); return } + + ttsPlaying = true + if ttsQueue.isEmpty && !ttsPlaying { emit("tts_started") } + emit("tts_started") + + let item = AVPlayerItem(url: url) + ttsPlayer = AVPlayer(playerItem: item) + if let observer = ttsPlayerObserver { + NotificationCenter.default.removeObserver(observer) + } + ttsPlayerObserver = NotificationCenter.default.addObserver( + forName: .AVPlayerItemDidPlayToEndTime, object: item, queue: .main + ) { [weak self] _ in + self?.playNextTts() + } + ttsPlayer?.play() + } + + private func stopTts() { + ttsPlayer?.pause() + ttsPlayer = nil + ttsQueue.removeAll() + ttsPlaying = false + if let observer = ttsPlayerObserver { + NotificationCenter.default.removeObserver(observer) + ttsPlayerObserver = nil + } + } + + // MARK: - 录音 + + private func startRecording() { + guard !recordingActive else { return } + let engine = AVAudioEngine() + audioEngine = engine + inputNode = engine.inputNode + let fmt = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: SAMPLE_RATE, channels: 1, interleaved: true)! + let hwFmt = inputNode!.outputFormat(forBus: 0) + let converterFmt = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: SAMPLE_RATE, channels: 1, interleaved: true)! + + inputNode!.installTap(onBus: 0, bufferSize: 4096, format: hwFmt) { [weak self] buffer, _ in + guard let self = self, self.recordingActive else { return } + // 转换为 16kHz PCM16 + guard let cvt = AVAudioConverter(from: hwFmt, to: converterFmt) else { return } + let frameCount = AVAudioFrameCount(Double(buffer.frameLength) * self.SAMPLE_RATE / hwFmt.sampleRate) + guard let outBuf = AVAudioPCMBuffer(pcmFormat: converterFmt, frameCapacity: frameCount) else { return } + var error: NSError? + cvt.convert(to: outBuf, error: &error) { _, outStatus in + outStatus.pointee = .haveData + return buffer + } + if error != nil { return } + let byteCount = Int(outBuf.frameLength) * 2 + let ptr = outBuf.int16ChannelData![0] + let data = Data(bytes: ptr, count: byteCount) + self.onMicAudio(data) + } + + do { + try AVAudioSession.sharedInstance().setCategory(.playAndRecord, mode: .voiceChat, options: [.defaultToSpeaker]) + try AVAudioSession.sharedInstance().setActive(true) + try engine.start() + recordingActive = true + } catch { + os_log("录音启动失败: %{public}@", log: logger, type: .error, error.localizedDescription) + } + } + + private func stopRecording() { + guard recordingActive else { return } + recordingActive = false + inputNode?.removeTap(onBus: 0) + audioEngine?.stop() + audioEngine = nil + inputNode = nil + sendBuffer.removeAll() + micChunkCount = 0 + vadActive = false + vadOnsetCount = 0 + preBuffer.removeAll() + } + + private func onMicAudio(_ pcm: Data) { + micChunkCount += 1 + let rms = calcRms(pcm) + let chunk = [UInt8](pcm) + + if !vadActive { + if preBuffer.count >= PRE_BUFFER_MAX { preBuffer.removeFirst() } + preBuffer.append(chunk) + } + + if !vadActive && rms > VAD_THRESHOLD { + vadOnsetCount += 1 + if vadOnsetCount >= VAD_ONSET_CHUNKS { + vadActive = true + // 先发预缓存 + for c in preBuffer { feedSendBuffer(Data(c)) } + preBuffer.removeAll() + } + } else if vadActive { + feedSendBuffer(pcm) + } + } + + private func calcRms(_ pcm: Data) -> Float { + let count = pcm.count / 2 + guard count > 0 else { return 0 } + var sum: Float = 0 + pcm.withUnsafeBytes { ptr in + let samples = ptr.bindMemory(to: Int16.self) + for i in 0..= FRAME_SIZE { + let frame = sendBuffer.prefix(FRAME_SIZE) + sendBuffer.removeFirst(FRAME_SIZE) + webSocketTask?.send(.data(Data(frame))) { _ in } + } + } + + private func sendFinishFrame() { + let payload: [String: Any] = [ + "msgType": "finish", + "dialogRequestId": dialogRequestId, + ] + sendJson(payload) + } + + // MARK: - 工具 + + private func sendJson(_ dict: [String: Any]) { + guard let data = try? JSONSerialization.data(withJSONObject: dict), + let str = String(data: data, encoding: .utf8) else { return } + webSocketTask?.send(.string(str)) { _ in } + } + + private func calcSign(secret: String, ts: Int64) -> String { + let raw = "\(secret)\(ts)" + let data = Data(raw.utf8) + let digest = Insecure.MD5.hash(data: data) + return digest.map { String(format: "%02x", $0) }.joined() + } + + private func emit(_ name: String, data: [String: Any] = [:]) { + DispatchQueue.main.async { [weak self] in + self?.onEvent(name, data) + } + } + + private func saveToChatStorage(text: String, role: String) { + DispatchQueue.global().async { [weak self] in + guard let self = self else { return } + _ = self.chatStorage.saveMessage( + agentId: self.config.agentId, + sessionId: self.currentSessionId, + message: text, + sender: role, + metadata: nil + ) + } + } +} diff --git a/local_plugins/agent_service/lib/agent_service.dart b/local_plugins/agent_service/lib/agent_service.dart index bb2c58425..533f76ddf 100644 --- a/local_plugins/agent_service/lib/agent_service.dart +++ b/local_plugins/agent_service/lib/agent_service.dart @@ -71,6 +71,18 @@ enum AgentServiceEventType { /// 错误 error, + /// STS 凭证获取成功（设备 deviceId/deviceSecret） + stsCredentials, + + /// 麦克风音量（RMS） + micRms, + + /// 心跳上报成功 + heartbeatReported, + + /// STS 设备认证失败 + credentialFailed, + /// 未知事件 unknown } @@ -193,6 +205,14 @@ class AgentService { return AgentServiceEventType.imageProcessing; case 'image_ready': return AgentServiceEventType.imageReady; + case 'sts_credentials': + return AgentServiceEventType.stsCredentials; + case 'mic_rms': + return AgentServiceEventType.micRms; + case 'heartbeat_reported': + return AgentServiceEventType.heartbeatReported; + case 'credential_failed': + return AgentServiceEventType.credentialFailed; default: return AgentServiceEventType.unknown; } @@ -463,6 +483,37 @@ class AgentService { } } + /// 设置 TTS 静音开关（STS 模式） + static Future setTtsMuted(bool muted) async { + try { + final bool result = await _channel.invokeMethod('setTtsMuted', { + 'muted': muted, + }); + return result; + } on PlatformException catch (e) { + throw AgentServiceException(e.code, e.message ?? '设置TTS静音失败', e.details); + } + } + + /// 手动触发心跳上报（移动精灵） + static Future reportHeartbeat() async { + try { + await _channel.invokeMethod('reportHeartbeat'); + } on PlatformException catch (e) { + throw AgentServiceException(e.code, e.message ?? '心跳上报失败', e.details); + } + } + + /// 主动触发认证检查（移动精灵页面打开时调用） + /// 若凭证不存在则重新请求，结果通过事件流返回（credentialFailed / heartbeatReported） + static Future fetchCredentials() async { + try { + await _channel.invokeMethod('fetchCredentials'); + } on PlatformException catch (_) { + // 忽略异常，认证结果通过事件流反馈 + } + } + /// 中断当前响应 /// /// 停止语音合成和AI流输出 diff --git a/local_plugins/ble_service/lib/ble_service.dart b/local_plugins/ble_service/lib/ble_service.dart index 0127bfe4b..52c1ada13 100644 --- a/local_plugins/ble_service/lib/ble_service.dart +++ b/local_plugins/ble_service/lib/ble_service.dart @@ -329,7 +329,7 @@ class BleService { /// 断开连接 Future stopScan() async { try { - final result = await _methodChannel.invokeMethod('stopscan'); + final result = await _methodChannel.invokeMethod('stopScan'); return result ?? false; } catch (e) { print('开启扫描ble设备失败: $e'); diff --git a/preview/yidong.html b/preview/yidong.html new file mode 100644 index 000000000..c8be59f76 --- /dev/null +++ b/preview/yidong.html @@ -0,0 +1,563 @@ + + + + + +移动精灵 UI 预览 + + + + + +
+

① 文本输入模式

+
+
+ ‹ +
晓
+ 移动精灵 + 空闲 +
+
+
+
晓
+
移动精灵
+
点击右下角按钮，即可与晓晓对话
+
+
+
+
+ +
+ + +
+ +
+
+
+
+ + +
+

② 语音模式 - 按住说话

+
+
+ ‹ +
晓
+ 移动精灵 + 空闲 +
+
+
+
晓
晓晓
+
你好！我是晓晓，有什么可以帮你的吗？
+ 14:23 +
+
+
+
+ +
按住 说话
+ +
+
+
+
+ + +
+

③ 按住说话 - 连接中（同空闲样式）

+
+
+ ‹ +
晓
+ 移动精灵 + 连接中… +
+
+
+
晓
晓晓
+
你好！我是晓晓，有什么可以帮你的吗？
+ 14:23 +
+
+
+
+ +
+ + 连接中... +
+ +
+
+
+
+ + +
+

④ 按住说话 - 录音中（橙色）

+
+
+ ‹ +
晓
+ 移动精灵 + 识别中 +
+
+
+
晓
晓晓
+
你好！我是晓晓，有什么可以帮你的吗？
+ 14:23 +
+
+
+ +
今天天气怎么样？
+
+ +
+
+ 松开发送 · 2s +
+ +
+
+
+
+ + +
+

⑤ AI 思考中（输入栏禁用）

+
+
+ ‹ +
晓
+ 移动精灵 + 处理中 +
+
+
+
晓
晓晓
+
你好！我是晓晓，有什么可以帮你的吗？
+ 14:23 +
+
+
今天天气怎么样？
+ 14:24 +
+
+
+
+ + AI 思考中... +
+
+ +
按住 说话
+ +
+
+
+
+ + +
+

⑥ AI 播报中（可打断）

+
+
+ ‹ +
晓
+ 移动精灵 + 朗读中 +
+
+
+
今天天气怎么样？
+ 14:24 +
+
+
晓
晓晓
+
今天晴天，气温 18°C，微风，空气质量优，很适合出行哦
+ 14:24 +
+
+
+
+
+ 晓晓正在朗读... + +
+
+ +
按住 说话
+ +
+
+
+
+ + +
+

⑦ 通话模式 - 等待说话

+
+
+ ‹ +
晓
+ 移动精灵 + 通话中 01:23 +
+
+
+
今天天气怎么样？
+ 14:24 +
+
+
晓
晓晓
+
今天晴天，气温 18°C，微风，空气质量优
+ 14:24 +
+
+
+
+
+
+ 通话中 + 01:23 +
+ +
+
+
+
+
+
+
+ + +
+
+
+
+ + +
+

⑧ 通话模式 - AI 播报

+
+
+ ‹ +
晓
+ 移动精灵 + 朗读中 01:48 +
+
+
+
Flutter 有什么优势？
+ 14:41 +
+
+
晓
晓晓
+
Flutter 的核心优势在于跨平台能力强、渲染性能高，以及 Dart 语言的高效开发体验...
+ 14:41 +
+
+
+
+
+ 晓晓正在朗读... + +
+
+
+
+ 通话中 + 01:48 +
+
+
+
+
+
+
+
+ +
+
+
+
+ + + diff --git a/test_mobile_elf.py b/test_mobile_elf.py new file mode 100644 index 000000000..77d4449c4 --- /dev/null +++ b/test_mobile_elf.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +import ssl, time, hashlib, json, websocket + +DEVICE_NO = '199839C0AAAE99653D2E' +DEVICE_ID = '2032013791868043266' +PRODUCT_ID = '2031271967792472065' +PRODUCT_KEY = 'hotepZZwvouWQjHz' +DEVICE_SECRET = 'ijdthZGQELhkmigf' + +ts = str(int(time.time() * 1000)) +sign = hashlib.md5((DEVICE_SECRET + ts).encode()).hexdigest() +url = (f'wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr' + f'?deviceNo={DEVICE_NO}&deviceId={DEVICE_ID}' + f'&productId={PRODUCT_ID}&productKey={PRODUCT_KEY}' + f'&sign={sign}&sn=py-test&ts={ts}') + +print(f'connecting...\n{url}') +ws = websocket.WebSocket(sslopt={'cert_reqs': ssl.CERT_NONE}) +ws.settimeout(15) +ws.connect(url) +print('connected! sending start frame immediately...') + +import uuid +start_frame = json.dumps({ + 'type': 'start', + 'data': { + 'cuid': DEVICE_ID, + 'format': 'pcm', + 'sample': 16000, + 'support_dcs': 2, + 'chunk_size': 10240, + 'support_tts': True, + 'support_text2dcs': True, + 'user_agent': 'voitrans/1.0', + 'dialog_request_id': str(uuid.uuid4()), + 'access_rc': True, + 'support_part_tts': True, + 'need_stoplisten': True, + 'need_dialogue_finish': True, + 'result_trans2directive': False, + 'rc_version': '5', + 'messages': [], + 'client_context': [], + } +}) +ws.send(start_frame) +print('start frame sent, waiting for ready...') + +# 等待 ready +while True: + msg = ws.recv() + print('received:', msg) + data = json.loads(msg) + if data.get('type') == 'ready': + print('--- ready received, sending text "你好" ---') + break + +# 发送文本 +text_frame = json.dumps({'type': 'text', 'data': {'text': '你好'}}) +ws.send(text_frame) +print('text frame sent') + +# 发送 finish +finish_frame = json.dumps({'type': 'finish'}) +ws.send(finish_frame) +print('finish frame sent, waiting for AI response...') + +# 接收所有回包直到 end=1 +while True: + msg = ws.recv() + print('received:', msg) + try: + data = json.loads(msg) + if data.get('end') == 1: + print('--- session end (end=1), closing ---') + break + except Exception: + pass + +ws.close() +print('done.')