From cc069113ed03e5eae89a3361e0da7af75f0a03a4 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Thu, 12 Mar 2026 09:58:01 +0800 Subject: [PATCH 01/14] =?UTF-8?q?feat(mobile=5Felf):=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E7=A7=BB=E5=8A=A8=E7=B2=BE=E7=81=B5=E6=A8=A1=E5=9D=97=E5=8F=8A?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/modules/home/views/home_view.dart | 24 + .../bindings/mobile_elf_binding.dart | 9 + .../controllers/mobile_elf_controller.dart | 572 ++++++++++++++++++ .../mobile_elf/views/mobile_elf_view.dart | 508 ++++++++++++++++ lib/routes/app_pages.dart | 9 + lib/routes/app_routes.dart | 3 + 6 files changed, 1125 insertions(+) create mode 100644 lib/modules/mobile_elf/bindings/mobile_elf_binding.dart create mode 100644 lib/modules/mobile_elf/controllers/mobile_elf_controller.dart create mode 100644 lib/modules/mobile_elf/views/mobile_elf_view.dart diff --git a/lib/modules/home/views/home_view.dart b/lib/modules/home/views/home_view.dart index dab865a45..b9f7d8eeb 100644 --- a/lib/modules/home/views/home_view.dart +++ b/lib/modules/home/views/home_view.dart @@ -145,6 +145,9 @@ class HomeView extends GetView { _buildSectionTitle(isDarkMode), SizedBox(height: 16.h), _buildPremiumAILayout(), + + SizedBox(height: 16.h), + _buildMobileElfCard(isDarkMode), ], ), ), @@ -899,6 +902,27 @@ class HomeView extends GetView { ); } + // 移动精灵卡片 + Widget _buildMobileElfCard(bool isDarkMode) { + return _buildFeatureCard( + title: '移动精灵', + description: '智能语音助手，移动AI全场景对话', + icon: Icons.cell_tower, + gradient: LinearGradient( + begin: Alignment.topLeft, + end: Alignment.bottomRight, + colors: [ + const Color(0xFF0091EA).withOpacity(0.3), + const Color(0xFF00BFA5).withOpacity(0.3), + ], + ), + onTap: () => Get.toNamed(Routes.mobileElf), + height: 70.h, + isDarkMode: isDarkMode, + iconColor: const Color(0xFF0091EA), + ); + } + // 特性卡片构建方法 Widget _buildFeatureCard({ required String title, diff --git a/lib/modules/mobile_elf/bindings/mobile_elf_binding.dart b/lib/modules/mobile_elf/bindings/mobile_elf_binding.dart new file mode 100644 index 000000000..193e9fbe3 --- /dev/null +++ b/lib/modules/mobile_elf/bindings/mobile_elf_binding.dart @@ -0,0 +1,9 @@ +import 'package:get/get.dart'; +import '../controllers/mobile_elf_controller.dart'; + +class MobileElfBinding extends Bindings { + @override + void dependencies() { + Get.lazyPut(() => MobileElfController()); + } +} diff --git a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart new file mode 100644 index 000000000..1525e5731 --- /dev/null +++ b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart @@ -0,0 +1,572 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:typed_data'; +import 'package:crypto/crypto.dart'; +import 'package:dio/dio.dart'; +import 'package:flutter/foundation.dart' show kIsWeb; +import 'package:flutter_dotenv/flutter_dotenv.dart'; +import 'package:get/get.dart'; +import 'package:get_storage/get_storage.dart'; +import 'package:just_audio/just_audio.dart'; +import 'package:record/record.dart'; +import 'package:uuid/uuid.dart'; +import 'package:web_socket_channel/io.dart'; +import 'package:web_socket_channel/web_socket_channel.dart'; +import '../../../core/utils/logger.dart'; +import '../../../data/models/user_Info.dart'; +import '../../../data/services/bluetooth_manager.dart'; + +enum MobileElfStatus { idle, connecting, recording, thinking, speaking, error } + +class MobileElfMessage { + final bool isUser; + final String text; + final DateTime time; + final String? ttsUrl; + + MobileElfMessage({ + required this.isUser, + required this.text, + required this.time, + this.ttsUrl, + }); +} + +class MobileElfController extends GetxController { + // 状态 + final status = MobileElfStatus.idle.obs; + final messages = [].obs; + final realtimeText = ''.obs; + final errorMsg = ''.obs; + + // WebSocket + WebSocketChannel? _channel; + StreamSubscription? _wsSubscription; + + // 录音 + final AudioRecorder _recorder = AudioRecorder(); + StreamSubscription? _audioSubscription; + final List _audioBuffer = []; + static const int _frameSize = 5120; // 160ms @ 16kHz 16bit mono + + // 播放 + final AudioPlayer _player = AudioPlayer(); + + // 配置 + late String _deviceNo; + late String _deviceId; + late String _productId; + late String _productKey; + late String _deviceSecret; + + // A80 凭证状态 + final credentialReady = false.obs; + final credentialError = ''.obs; + + final _storage = GetStorage(); + static const _kDeviceId = 'mobile_elf_device_id'; + static const _kDeviceSecret = 'mobile_elf_device_secret'; + static const _kDeviceNo = 'mobile_elf_device_no'; + + // 当前对话 ID + String _dialogRequestId = ''; + + final _uuid = const Uuid(); + + @override + void onInit() { + super.onInit(); + _loadConfig(); + _fetchA80CredentialsIfNeeded(); + } + + @override + void onClose() { + disconnect(); + _recorder.dispose(); + _player.dispose(); + super.onClose(); + } + + void _loadConfig() { + _productId = dotenv.env['MOBILE_ELF_PRODUCT_ID'] ?? '1899007766144614402'; + _productKey = dotenv.env['MOBILE_ELF_PRODUCT_KEY'] ?? 'd0aI2yidbUUXS'; + // 优先使用缓存的 A80 凭证，否则降级到 .env 默认值 + _deviceNo = _storage.read(_kDeviceNo) ?? + dotenv.env['MOBILE_ELF_DEVICE_NO'] ?? '869700070000570'; + _deviceId = _storage.read(_kDeviceId) ?? + dotenv.env['MOBILE_ELF_DEVICE_ID'] ?? '1899013964170694658'; + _deviceSecret = _storage.read(_kDeviceSecret) ?? + dotenv.env['MOBILE_ELF_DEVICE_SECRET'] ?? ''; + credentialReady.value = _deviceSecret.isNotEmpty; + } + + /// 检测连接的耳机是否为 A80，如是则用其 license 换取平台凭证 + Future _fetchA80CredentialsIfNeeded() async { + try { + final btManager = Get.find(); + final deviceName = btManager.currentDevice?.name ?? ''; + final deviceMac = btManager.currentDevice?.address ?? ''; + final license = _findLicense(deviceMac); + + Logger.i('[MobileElf]', '=== 设备信息 ==='); + Logger.i('[MobileElf]', '设备名称: $deviceName'); + Logger.i('[MobileElf]', '设备MAC: $deviceMac'); + Logger.i('[MobileElf]', '匹配license: $license'); + Logger.i('[MobileElf]', '==============='); + + if (!deviceName.toUpperCase().contains('A80')) { + credentialReady.value = _deviceSecret.isNotEmpty; + return; + } + + // 已有缓存凭证且设备号未变则直接使用 + final cachedNo = _storage.read(_kDeviceNo) ?? ''; + final cachedSecret = _storage.read(_kDeviceSecret) ?? ''; + + if (license == null || license.isEmpty) { + Logger.w('[MobileElf]', 'A80 设备未找到 license，使用默认凭证'); + credentialReady.value = _deviceSecret.isNotEmpty; + return; + } + + if (cachedNo == license && cachedSecret.isNotEmpty) { + Logger.i('[MobileElf]', 'A80 凭证命中缓存，跳过请求'); + credentialReady.value = true; + return; + } + + credentialError.value = ''; + Logger.i('[MobileElf]', 'A80 设备 license=$license，开始请求平台凭证'); + + final dio = Dio(BaseOptions( + baseUrl: 'https://ivs.chinamobiledevice.com:11443', + connectTimeout: const Duration(seconds: 10), + receiveTimeout: const Duration(seconds: 10), + )); + + final resp = await dio.post( + '/v2/customer/device/secret/info', + data: { + 'deviceNoType': 'SN', + 'deviceNo': license, + 'productId': _productId, + 'productKey': _productKey, + }, + ); + + final body = resp.data as Map; + if (body['code'].toString() == '200' && body['success'] == true) { + final data = body['data'] as Map; + _deviceNo = data['deviceNo'] as String? ?? license; + _deviceId = data['deviceId'] as String? ?? _deviceId; + _deviceSecret = data['deviceSecret'] as String? ?? ''; + + await _storage.write(_kDeviceNo, _deviceNo); + await _storage.write(_kDeviceId, _deviceId); + await _storage.write(_kDeviceSecret, _deviceSecret); + + credentialReady.value = _deviceSecret.isNotEmpty; + Logger.i('[MobileElf]', 'A80 凭证获取成功 deviceId=$_deviceId'); + } else { + credentialError.value = body['message']?.toString() ?? '凭证请求失败'; + Logger.e('[MobileElf]', '平台返回错误: ${body['message']}'); + } + } catch (e) { + credentialError.value = '凭证请求异常: $e'; + Logger.e('[MobileElf]', '获取 A80 凭证失败: $e'); + } + } + + /// 从用户设备列表中按蓝牙 MAC 匹配 license + String? _findLicense(String? bluetoothMac) { + if (!User.isLoggedIn()) return null; + final devices = User.instance.devices; + if (bluetoothMac != null && bluetoothMac.isNotEmpty) { + final matched = devices.where((d) => + d.devicemac.toUpperCase() == bluetoothMac.toUpperCase()).toList(); + if (matched.isNotEmpty) return matched.first.license; + } + // 按设备名兜底匹配 + final byName = devices.where((d) => + d.devicename.toUpperCase().contains('A80')).toList(); + return byName.isNotEmpty ? byName.first.license : null; + } + + // ── 连接 ────────────────────────────────────────────────────────────────── + + Future _connect() async { + if (_channel != null) return; + + status.value = MobileElfStatus.connecting; + errorMsg.value = ''; + + try { + final ts = DateTime.now().millisecondsSinceEpoch.toString(); + final sn = _uuid.v4(); + final sign = _calcSign(_deviceSecret, ts); + + final url = 'wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr' + '?deviceNo=$_deviceNo' + '&deviceId=$_deviceId' + '&productId=$_productId' + '&productKey=$_productKey' + '&sign=$sign' + '&sn=$sn' + '&ts=$ts'; + + if (kIsWeb) { + _channel = WebSocketChannel.connect(Uri.parse(url)); + } else { + _channel = IOWebSocketChannel.connect(Uri.parse(url)); + } + + _wsSubscription = _channel!.stream.listen( + _handleMessage, + onError: _handleWsError, + onDone: _handleWsDone, + ); + + Logger.info('[MobileElf] WebSocket 连接中: $url'); + } catch (e) { + Logger.error('[MobileElf] 连接失败: $e'); + _setError('连接失败: $e'); + } + } + + // ── 开始对话（录音） ───────────────────────────────────────────────────── + + Future startRecording() async { + if (status.value == MobileElfStatus.recording) return; + if (status.value == MobileElfStatus.thinking) return; + + await _connect(); + + final hasPermission = await _recorder.hasPermission(); + if (!hasPermission) { + _setError('无麦克风权限'); + return; + } + + _dialogRequestId = _uuid.v4(); + realtimeText.value = ''; + + try { + final stream = await _recorder.startStream( + const RecordConfig( + encoder: AudioEncoder.pcm16bits, + sampleRate: 16000, + numChannels: 1, + ), + ); + + _audioBuffer.clear(); + _audioSubscription = stream.listen((chunk) { + _audioBuffer.addAll(chunk); + while (_audioBuffer.length >= _frameSize) { + _sendAudioChunk( + Uint8List.fromList(_audioBuffer.sublist(0, _frameSize))); + _audioBuffer.removeRange(0, _frameSize); + } + }); + + status.value = MobileElfStatus.recording; + Logger.info('[MobileElf] 开始录音'); + } catch (e) { + Logger.error('[MobileElf] 录音失败: $e'); + _setError('录音失败: $e'); + } + } + + Future stopRecording() async { + if (status.value != MobileElfStatus.recording) return; + + await _audioSubscription?.cancel(); + _audioSubscription = null; + + // 发送剩余缓冲数据 + if (_audioBuffer.isNotEmpty) { + _sendAudioChunk(Uint8List.fromList(_audioBuffer)); + _audioBuffer.clear(); + } + + await _recorder.stop(); + _sendFinish(); + + status.value = MobileElfStatus.thinking; + Logger.info('[MobileElf] 停止录音，等待 AI 回答'); + } + + // ── 发送帧 ──────────────────────────────────────────────────────────────── + + void _sendStartFrame() { + if (_channel == null) return; + + final startFrame = { + 'type': 'start', + 'data': { + 'cuid': _deviceId, + 'format': 'pcm', + 'sample': 16000, + 'support_dcs': 2, + 'chunk_size': 10240, + 'support_tts': true, + 'support_text2dcs': true, + 'user_agent': 'voitrans/1.0', + 'dialog_request_id': _dialogRequestId, + 'access_rc': true, + 'support_part_tts': true, + 'need_stoplisten': true, + 'need_dialogue_finish': true, + 'result_trans2directive': false, + 'rc_version': '5', + 'messages': _buildContextMessages(), + 'client_context': [], + }, + }; + + _channel!.sink.add(jsonEncode(startFrame)); + Logger.info('[MobileElf] 已发送 start 帧'); + } + + void _sendAudioChunk(Uint8List data) { + try { + _channel?.sink.add(data); + } catch (e) { + Logger.error('[MobileElf] 发送音频帧失败: $e'); + } + } + + void _sendFinish() { + try { + _channel?.sink.add(jsonEncode({'type': 'finish'})); + Logger.info('[MobileElf] 已发送 finish 帧'); + } catch (e) { + Logger.error('[MobileElf] 发送 finish 帧失败: $e'); + } + } + + // ── 接收消息处理 ────────────────────────────────────────────────────────── + + void _handleMessage(dynamic raw) { + try { + if (raw is! String) return; + final Map msg = jsonDecode(raw); + final type = msg['type'] as String? ?? ''; + + switch (type) { + case 'ready': + Logger.info('[MobileElf] 收到 ready，发送 start 帧'); + _sendStartFrame(); + break; + + case 'mid_result': + final result = msg['result'] as String? ?? ''; + realtimeText.value = result; + break; + + case 'fin_result': + final result = msg['result'] as String? ?? ''; + if (result.isNotEmpty) { + realtimeText.value = result; + // 用户消息在 fin_result 时确认写入 + _addUserMessage(result); + } + final end = msg['end'] as int? ?? 0; + if (end == 1) _handleSessionEnd(); + break; + + case 'inside_rc': + _handleInsideRc(msg); + break; + + case 'dcs_decide': + final end = msg['end'] as int? ?? 0; + if (end == 1) _handleSessionEnd(); + break; + + default: + Logger.info('[MobileElf] 未知消息类型: $type'); + } + } catch (e) { + Logger.error('[MobileElf] 处理消息失败: $e'); + } + } + + void _handleInsideRc(Map msg) { + final data = msg['data'] as Map?; + if (data == null) return; + + final code = data['code'] as int? ?? -1; + if (code != 0) { + Logger.error('[MobileElf] inside_rc 错误: ${data['msg']}'); + return; + } + + // 提取 AI 文本回答 + final assistantAnswer = data['assistant_answer'] as String? ?? ''; + String cleanAnswer = assistantAnswer; + // assistant_answer 可能是 JSON 字符串 + try { + final decoded = jsonDecode(assistantAnswer) as Map; + cleanAnswer = decoded['content'] as String? ?? assistantAnswer; + } catch (_) {} + + // 提取 TTS URL（Speak 指令） + String? ttsUrl; + final directives = data['data'] as List? ?? []; + for (final item in directives) { + final directive = item as Map; + final header = directive['header'] as Map?; + final ns = header?['namespace'] as String? ?? ''; + final name = header?['name'] as String? ?? ''; + if (ns == 'ai.fxzsos.device_interface.voice_output' && name == 'Speak') { + final payload = directive['payload'] as Map?; + ttsUrl = payload?['url'] as String?; + break; + } + } + + if (cleanAnswer.isNotEmpty) { + _addAiMessage(cleanAnswer, ttsUrl: ttsUrl); + } + + // 播放 TTS + if (ttsUrl != null && ttsUrl.isNotEmpty) { + playTts(ttsUrl); + } + + final end = msg['end'] as int? ?? 0; + if (end == 1) _handleSessionEnd(); + } + + void _handleSessionEnd() { + if (status.value != MobileElfStatus.speaking) { + status.value = MobileElfStatus.idle; + } + realtimeText.value = ''; + // 断开当前 WebSocket，下次对话重新连接（协议要求每次新对话重连） + _closeWs(); + } + + void _handleWsError(dynamic error) { + Logger.error('[MobileElf] WebSocket 错误: $error'); + _setError('连接错误: $error'); + _closeWs(); + } + + void _handleWsDone() { + Logger.info('[MobileElf] WebSocket 关闭'); + if (status.value == MobileElfStatus.connecting || + status.value == MobileElfStatus.recording) { + _setError('连接已断开'); + } + _channel = null; + _wsSubscription = null; + } + + // ── TTS 播放 ────────────────────────────────────────────────────────────── + + Future playTts(String url) async { + try { + status.value = MobileElfStatus.speaking; + await _player.setUrl(url); + await _player.play(); + await _player.processingStateStream + .firstWhere((s) => s == ProcessingState.completed); + status.value = MobileElfStatus.idle; + Logger.info('[MobileElf] TTS 播放完成'); + } catch (e) { + Logger.error('[MobileElf] TTS 播放失败: $e'); + status.value = MobileElfStatus.idle; + } + } + + Future stopSpeaking() async { + await _player.stop(); + status.value = MobileElfStatus.idle; + } + + // ── 断开连接 ────────────────────────────────────────────────────────────── + + void disconnect() { + _audioSubscription?.cancel(); + _audioSubscription = null; + _recorder.cancel(); + _closeWs(); + status.value = MobileElfStatus.idle; + } + + void _closeWs() { + _wsSubscription?.cancel(); + _wsSubscription = null; + _channel?.sink.close(); + _channel = null; + } + + // ── 辅助方法 ────────────────────────────────────────────────────────────── + + String _calcSign(String deviceSecret, String ts) { + final bytes = utf8.encode('$deviceSecret$ts'); + return md5.convert(bytes).toString(); + } + + List> _buildContextMessages() { + return messages.map((m) => { + 'role': m.isUser ? 'user' : 'assistant', + 'content': m.text, + }).toList(); + } + + void _addUserMessage(String text) { + // 避免重复添加 + if (messages.isNotEmpty && + messages.last.isUser && + messages.last.text == text) { + return; + } + messages.add(MobileElfMessage( + isUser: true, + text: text, + time: DateTime.now(), + )); + } + + void _addAiMessage(String text, {String? ttsUrl}) { + messages.add(MobileElfMessage( + isUser: false, + text: text, + time: DateTime.now(), + ttsUrl: ttsUrl, + )); + } + + void _setError(String msg) { + errorMsg.value = msg; + status.value = MobileElfStatus.error; + } + + /// 重试获取 A80 凭证 + Future retryCredentials() async { + credentialError.value = ''; + await _fetchA80CredentialsIfNeeded(); + } + + /// 按钮主操作：空闲/错误 → 开始录音；录音中 → 停止；播放中 → 停止 + Future onMicTap() async { + switch (status.value) { + case MobileElfStatus.idle: + case MobileElfStatus.error: + await startRecording(); + break; + case MobileElfStatus.recording: + await stopRecording(); + break; + case MobileElfStatus.speaking: + await stopSpeaking(); + break; + default: + break; + } + } +} diff --git a/lib/modules/mobile_elf/views/mobile_elf_view.dart b/lib/modules/mobile_elf/views/mobile_elf_view.dart new file mode 100644 index 000000000..51db8c287 --- /dev/null +++ b/lib/modules/mobile_elf/views/mobile_elf_view.dart @@ -0,0 +1,508 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_screenutil/flutter_screenutil.dart'; +import 'package:get/get.dart'; +import '../controllers/mobile_elf_controller.dart'; + +class MobileElfView extends GetView { + const MobileElfView({Key? key}) : super(key: key); + + @override + Widget build(BuildContext context) { + final bool isDark = Theme.of(context).brightness == Brightness.dark; + return Scaffold( + backgroundColor: isDark ? Colors.black : const Color(0xFFF9FAFC), + appBar: _buildAppBar(isDark), + body: Obx(() { + final err = controller.credentialError.value; + if (err.isNotEmpty) { + return Center( + child: Padding( + padding: EdgeInsets.all(24.w), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + Icon(Icons.error_outline, size: 48.sp, color: Colors.red), + SizedBox(height: 12.h), + Text(err, + textAlign: TextAlign.center, + style: TextStyle(fontSize: 13.sp, color: Colors.red)), + SizedBox(height: 16.h), + ElevatedButton( + onPressed: controller.retryCredentials, + child: const Text('重试'), + ), + ], + ), + ), + ); + } + return Column( + children: [ + _buildStatusBanner(), + Expanded(child: _buildMessageList(isDark)), + _buildBottomBar(isDark), + ], + ); + }), + ); + } + + PreferredSizeWidget _buildAppBar(bool isDark) { + return AppBar( + backgroundColor: isDark ? const Color(0xFF1A1A1A) : Colors.white, + elevation: 0.5, + leading: IconButton( + icon: Icon(Icons.arrow_back_ios, + size: 18.sp, + color: isDark ? Colors.white : Colors.grey[800]), + onPressed: () => Get.back(), + ), + title: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Container( + width: 32.w, + height: 32.w, + decoration: BoxDecoration( + gradient: const LinearGradient( + colors: [Color(0xFF0091EA), Color(0xFF00BFA5)], + ), + borderRadius: BorderRadius.circular(8.r), + ), + child: Icon(Icons.cell_tower, color: Colors.white, size: 18.sp), + ), + SizedBox(width: 8.w), + Text( + '移动精灵', + style: TextStyle( + fontSize: 16.sp, + fontWeight: FontWeight.bold, + color: isDark ? Colors.white : Colors.grey[800], + ), + ), + ], + ), + actions: [ + Obx(() => Padding( + padding: EdgeInsets.only(right: 16.w), + child: _buildStatusDot(), + )), + ], + ); + } + + Widget _buildStatusDot() { + final color = _statusColor(controller.status.value); + return Row( + children: [ + Container( + width: 8.w, + height: 8.w, + decoration: BoxDecoration( + color: color, + shape: BoxShape.circle, + boxShadow: [ + BoxShadow(color: color.withOpacity(0.4), blurRadius: 4) + ], + ), + ), + SizedBox(width: 4.w), + Text( + _statusLabel(controller.status.value), + style: TextStyle(fontSize: 11.sp, color: color), + ), + ], + ); + } + + Widget _buildStatusBanner() { + return Obx(() { + final s = controller.status.value; + if (s == MobileElfStatus.idle) return const SizedBox.shrink(); + + final color = _statusColor(s); + final label = _statusBannerLabel(s); + + return Container( + width: double.infinity, + padding: EdgeInsets.symmetric(vertical: 6.h), + color: color.withOpacity(0.12), + child: Row( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + if (s == MobileElfStatus.thinking || s == MobileElfStatus.connecting) + SizedBox( + width: 12.w, + height: 12.w, + child: CircularProgressIndicator( + strokeWidth: 1.5, + color: color, + ), + ), + if (s == MobileElfStatus.recording) + Icon(Icons.mic, size: 14.sp, color: color), + if (s == MobileElfStatus.speaking) + Icon(Icons.volume_up, size: 14.sp, color: color), + if (s == MobileElfStatus.error) + Icon(Icons.error_outline, size: 14.sp, color: color), + SizedBox(width: 6.w), + Text(label, + style: TextStyle( + fontSize: 12.sp, + color: color, + fontWeight: FontWeight.w500)), + ], + ), + ); + }); + } + + Widget _buildMessageList(bool isDark) { + return Obx(() { + if (controller.messages.isEmpty) { + return _buildEmptyHint(isDark); + } + return ListView.builder( + padding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 12.h), + reverse: false, + itemCount: controller.messages.length, + itemBuilder: (_, i) => _buildBubble(controller.messages[i], isDark), + ); + }); + } + + Widget _buildEmptyHint(bool isDark) { + return Center( + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + Container( + width: 72.w, + height: 72.w, + decoration: BoxDecoration( + gradient: const LinearGradient( + colors: [Color(0xFF0091EA), Color(0xFF00BFA5)], + ), + shape: BoxShape.circle, + ), + child: Icon(Icons.cell_tower, color: Colors.white, size: 36.sp), + ), + SizedBox(height: 16.h), + Text( + '移动精灵', + style: TextStyle( + fontSize: 18.sp, + fontWeight: FontWeight.bold, + color: isDark ? Colors.white : Colors.grey[800], + ), + ), + SizedBox(height: 8.h), + Text( + '按住麦克风按钮开始语音对话', + style: TextStyle( + fontSize: 13.sp, + color: isDark ? Colors.grey[400] : Colors.grey[500], + ), + ), + ], + ), + ); + } + + Widget _buildBubble(MobileElfMessage msg, bool isDark) { + final isUser = msg.isUser; + return Padding( + padding: EdgeInsets.only(bottom: 12.h), + child: Row( + mainAxisAlignment: + isUser ? MainAxisAlignment.end : MainAxisAlignment.start, + crossAxisAlignment: CrossAxisAlignment.end, + children: [ + if (!isUser) ...[ + _buildAvatar(isUser: false), + SizedBox(width: 8.w), + ], + Flexible( + child: Container( + padding: + EdgeInsets.symmetric(horizontal: 14.w, vertical: 10.h), + decoration: BoxDecoration( + color: isUser + ? const Color(0xFF0091EA) + : (isDark + ? const Color(0xFF2A2A2A) + : Colors.white), + borderRadius: BorderRadius.only( + topLeft: Radius.circular(16.r), + topRight: Radius.circular(16.r), + bottomLeft: + isUser ? Radius.circular(16.r) : Radius.circular(4.r), + bottomRight: + isUser ? Radius.circular(4.r) : Radius.circular(16.r), + ), + boxShadow: [ + BoxShadow( + color: Colors.black.withOpacity(0.06), + blurRadius: 6, + offset: const Offset(0, 2), + ), + ], + ), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + msg.text, + style: TextStyle( + fontSize: 14.sp, + color: isUser + ? Colors.white + : (isDark ? Colors.white : Colors.grey[800]), + height: 1.4, + ), + ), + if (!isUser && msg.ttsUrl != null) ...[ + SizedBox(height: 6.h), + GestureDetector( + onTap: () => controller.playTts(msg.ttsUrl!), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Icon( + Icons.play_circle_outline, + size: 16.sp, + color: const Color(0xFF0091EA), + ), + SizedBox(width: 4.w), + Text( + '重播语音', + style: TextStyle( + fontSize: 11.sp, + color: const Color(0xFF0091EA), + ), + ), + ], + ), + ), + ], + ], + ), + ), + ), + if (isUser) ...[ + SizedBox(width: 8.w), + _buildAvatar(isUser: true), + ], + ], + ), + ); + } + + Widget _buildAvatar({required bool isUser}) { + return Container( + width: 32.w, + height: 32.w, + decoration: BoxDecoration( + color: isUser + ? const Color(0xFF0091EA).withOpacity(0.15) + : const Color(0xFF00BFA5).withOpacity(0.15), + shape: BoxShape.circle, + ), + child: Icon( + isUser ? Icons.person : Icons.cell_tower, + size: 16.sp, + color: isUser ? const Color(0xFF0091EA) : const Color(0xFF00BFA5), + ), + ); + } + + Widget _buildBottomBar(bool isDark) { + return Container( + padding: EdgeInsets.fromLTRB(16.w, 8.h, 16.w, 24.h), + decoration: BoxDecoration( + color: isDark ? const Color(0xFF1A1A1A) : Colors.white, + boxShadow: [ + BoxShadow( + color: Colors.black.withOpacity(0.05), + blurRadius: 8, + offset: const Offset(0, -2), + ), + ], + ), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + // 实时识别字幕 + Obx(() { + final text = controller.realtimeText.value; + if (text.isEmpty) return const SizedBox.shrink(); + return Padding( + padding: EdgeInsets.only(bottom: 8.h), + child: Text( + text, + style: TextStyle( + fontSize: 13.sp, + color: isDark ? Colors.grey[400] : Colors.grey[500], + fontStyle: FontStyle.italic, + ), + textAlign: TextAlign.center, + maxLines: 2, + overflow: TextOverflow.ellipsis, + ), + ); + }), + + // 错误提示 + Obx(() { + final err = controller.errorMsg.value; + if (err.isEmpty) return const SizedBox.shrink(); + return Padding( + padding: EdgeInsets.only(bottom: 8.h), + child: Text( + err, + style: TextStyle( + fontSize: 12.sp, color: Colors.red[400]), + textAlign: TextAlign.center, + ), + ); + }), + + // 麦克风按钮 + Obx(() => _buildMicButton(controller.status.value)), + ], + ), + ); + } + + Widget _buildMicButton(MobileElfStatus status) { + final bool isThinking = status == MobileElfStatus.thinking || + status == MobileElfStatus.connecting; + final Color btnColor = _statusColor(status); + final IconData icon = _micIcon(status); + final String label = _micLabel(status); + + return Column( + children: [ + GestureDetector( + onTap: isThinking ? null : controller.onMicTap, + child: AnimatedContainer( + duration: const Duration(milliseconds: 200), + width: 68.w, + height: 68.w, + decoration: BoxDecoration( + shape: BoxShape.circle, + color: isThinking + ? Colors.grey.withOpacity(0.2) + : btnColor.withOpacity(0.15), + border: Border.all( + color: isThinking ? Colors.grey.withOpacity(0.3) : btnColor, + width: 2, + ), + ), + child: isThinking + ? Center( + child: SizedBox( + width: 24.w, + height: 24.w, + child: CircularProgressIndicator( + strokeWidth: 2.5, color: Colors.grey), + ), + ) + : Icon(icon, size: 30.sp, color: btnColor), + ), + ), + SizedBox(height: 6.h), + Text( + label, + style: TextStyle( + fontSize: 11.sp, + color: isThinking ? Colors.grey : btnColor, + ), + ), + ], + ); + } + + // ── 辅助 ────────────────────────────────────────────────────────────────── + + Color _statusColor(MobileElfStatus s) { + switch (s) { + case MobileElfStatus.connecting: + return Colors.orange; + case MobileElfStatus.recording: + return const Color(0xFF0066CC); + case MobileElfStatus.thinking: + return Colors.orange; + case MobileElfStatus.speaking: + return const Color(0xFF4CAF50); + case MobileElfStatus.error: + return Colors.red; + case MobileElfStatus.idle: + return const Color(0xFF0091EA); + } + } + + String _statusLabel(MobileElfStatus s) { + switch (s) { + case MobileElfStatus.connecting: + return '连接中'; + case MobileElfStatus.recording: + return '录音中'; + case MobileElfStatus.thinking: + return '思考中'; + case MobileElfStatus.speaking: + return '播报中'; + case MobileElfStatus.error: + return '错误'; + case MobileElfStatus.idle: + return '就绪'; + } + } + + String _statusBannerLabel(MobileElfStatus s) { + switch (s) { + case MobileElfStatus.connecting: + return '正在连接移动精灵...'; + case MobileElfStatus.recording: + return '正在聆听，请说话'; + case MobileElfStatus.thinking: + return 'AI 思考中，请稍候'; + case MobileElfStatus.speaking: + return '正在播报回答'; + case MobileElfStatus.error: + return controller.errorMsg.value; + case MobileElfStatus.idle: + return ''; + } + } + + IconData _micIcon(MobileElfStatus s) { + switch (s) { + case MobileElfStatus.recording: + return Icons.stop_circle_outlined; + case MobileElfStatus.speaking: + return Icons.stop_circle_outlined; + case MobileElfStatus.error: + return Icons.refresh; + default: + return Icons.mic; + } + } + + String _micLabel(MobileElfStatus s) { + switch (s) { + case MobileElfStatus.recording: + return '点击停止'; + case MobileElfStatus.speaking: + return '点击停止播报'; + case MobileElfStatus.error: + return '重试'; + case MobileElfStatus.thinking: + return '思考中...'; + case MobileElfStatus.connecting: + return '连接中...'; + default: + return '点击说话'; + } + } +} diff --git a/lib/routes/app_pages.dart b/lib/routes/app_pages.dart index de6142ab3..c90f995b7 100644 --- a/lib/routes/app_pages.dart +++ b/lib/routes/app_pages.dart @@ -80,6 +80,8 @@ import '../modules/goods/bindings/goodsmeet_binding.dart'; import '../modules/goods/views/goodsmeet_view.dart'; import '../modules/goods/bindings/goodsai_binding.dart'; import '../modules/goods/views/goodsai_view.dart'; +import '../modules/mobile_elf/bindings/mobile_elf_binding.dart'; +import '../modules/mobile_elf/views/mobile_elf_view.dart'; abstract class AppPages { static final pages = [ @@ -369,5 +371,12 @@ abstract class AppPages { transition: Transition.rightToLeft, transitionDuration: Duration(milliseconds: 250), ), + GetPage( + name: Routes.mobileElf, + page: () => const MobileElfView(), + binding: MobileElfBinding(), + transition: Transition.rightToLeft, + transitionDuration: Duration(milliseconds: 250), + ), ]; } diff --git a/lib/routes/app_routes.dart b/lib/routes/app_routes.dart index be5b03701..0d69db214 100644 --- a/lib/routes/app_routes.dart +++ b/lib/routes/app_routes.dart @@ -60,6 +60,9 @@ abstract class Routes { // 实时语音对话 static const realtime = '/realtime'; + // 移动精灵 + static const mobileElf = '/mobile_elf'; + // 设备管理 static const devices = '/devices'; From eeda106629021d871e34a17d2c4cdfc0124db078 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Wed, 18 Mar 2026 11:46:28 +0800 Subject: [PATCH 02/14] Add mobile ELF test script and initial UI layout for Yidong application --- .../controllers/mobile_elf_controller.dart | 1038 ++++++++++----- .../mobile_elf/views/mobile_elf_view.dart | 1108 +++++++++++------ test_mobile_elf.py | 81 ++ ui/yidong.html | 563 +++++++++ 4 files changed, 2075 insertions(+), 715 deletions(-) create mode 100644 test_mobile_elf.py create mode 100644 ui/yidong.html diff --git a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart index 1525e5731..cfb1e2802 100644 --- a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart +++ b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart @@ -1,9 +1,13 @@ import 'dart:async'; import 'dart:convert'; +import 'dart:io'; +import 'dart:math' as math; import 'dart:typed_data'; import 'package:crypto/crypto.dart'; import 'package:dio/dio.dart'; import 'package:flutter/foundation.dart' show kIsWeb; +import 'package:flutter/widgets.dart'; +import 'package:audio_session/audio_session.dart'; import 'package:flutter_dotenv/flutter_dotenv.dart'; import 'package:get/get.dart'; import 'package:get_storage/get_storage.dart'; @@ -13,8 +17,6 @@ import 'package:uuid/uuid.dart'; import 'package:web_socket_channel/io.dart'; import 'package:web_socket_channel/web_socket_channel.dart'; import '../../../core/utils/logger.dart'; -import '../../../data/models/user_Info.dart'; -import '../../../data/services/bluetooth_manager.dart'; enum MobileElfStatus { idle, connecting, recording, thinking, speaking, error } @@ -23,55 +25,93 @@ class MobileElfMessage { final String text; final DateTime time; final String? ttsUrl; - - MobileElfMessage({ - required this.isUser, - required this.text, - required this.time, - this.ttsUrl, - }); + MobileElfMessage({required this.isUser, required this.text, required this.time, this.ttsUrl}); } class MobileElfController extends GetxController { - // 状态 + // ── 可观察状态 ───────────────────────────────────────────────────────────── final status = MobileElfStatus.idle.obs; final messages = [].obs; final realtimeText = ''.obs; final errorMsg = ''.obs; - - // WebSocket + final inCall = false.obs; + final callSeconds = 0.obs; + final recordingSeconds = 0.obs; + final micRms = 0.0.obs; // 实时麦克风音量（RMS），供 UI 显示 + final micAlive = false.obs; // 麦克风流是否存活 + final pttRecording = false.obs; // 按住说话录音中 + final pttSeconds = 0.obs; // 按住说话计时 + final ScrollController scrollController = ScrollController(); + + // ── WebSocket ────────────────────────────────────────────────────────────── WebSocketChannel? _channel; StreamSubscription? _wsSubscription; + Completer? _readyCompleter; + bool _sessionEnded = true; + String _dialogRequestId = ''; + int _currentAiMsgIndex = -1; - // 录音 + // ── 录音 & 本地 VAD ──────────────────────────────────────────────────────── final AudioRecorder _recorder = AudioRecorder(); - StreamSubscription? _audioSubscription; - final List _audioBuffer = []; - static const int _frameSize = 5120; // 160ms @ 16kHz 16bit mono + StreamSubscription? _micSubscription; - // 播放 - final AudioPlayer _player = AudioPlayer(); - - // 配置 - late String _deviceNo; - late String _deviceId; - late String _productId; - late String _productKey; - late String _deviceSecret; + // 音频发送缓冲（凑帧用） + final List _sendBuffer = []; + int _sendBufferOffset = 0; + static const int _frameSize = 5120; // 160ms @ 16kHz 16bit mono - // A80 凭证状态 + // 预缓存：滚动保留最近 ~1.5 秒音频，用于在 WS 连接前缓存用户已说的话 + final List _preBuffer = []; + static const int _preBufferMaxChunks = 75; // ~1.5s (按每 chunk 20ms 算) + + // 本地 VAD（仅在空闲等待状态工作，TTS 播放中不做 VAD） + bool _speechDetected = false; + static const int _vadThreshold = 800; + static const int _vadOnsetChunks = 3; // 连续 3 chunk (~60ms) → 检测到说话 + int _vadOnsetCount = 0; + + // 状态标记 + bool _forwardingAudio = false; // 是否正在转发音频到 WS + bool _wsConnecting = false; // WS 正在连接中（防止重复触发） + int _micChunkCount = 0; // 麦克风 chunk 计数（调试用） + + // ── 播放 & TTS 队列 ────────────────────────────────────────────────────── + // 关闭 just_audio 的音频会话管理，防止它与 record 争抢音频硬件： + // - handleInterruptions: false → 不自动暂停/恢复 + // - handleAudioSessionActivation: false → 不自动调用 session.setActive() + // - androidApplyAudioAttributes: false → 不监听 audio_session 配置变更 + final AudioPlayer _player = AudioPlayer( + handleInterruptions: false, + handleAudioSessionActivation: false, + androidApplyAudioAttributes: false, + ); + final List _ttsQueue = []; + bool _ttsPlaying = false; + bool _ttsInterrupted = false; + Completer? _ttsPlayCompleter; // 用于等待当前段播完或被打断 + + // ── 配置 ─────────────────────────────────────────────────────────────────── + late String _deviceNo, _deviceId, _productId, _productKey, _deviceSecret; final credentialReady = false.obs; final credentialError = ''.obs; - final _storage = GetStorage(); static const _kDeviceId = 'mobile_elf_device_id'; static const _kDeviceSecret = 'mobile_elf_device_secret'; static const _kDeviceNo = 'mobile_elf_device_no'; + final _uuid = const Uuid(); - // 当前对话 ID - String _dialogRequestId = ''; + // 计时器 + Timer? _callTimer; + Timer? _recordingTimer; + Timer? _micWatchdog; + int _lastWatchdogChunkCount = 0; + Timer? _idleTimer; + Timer? _pttTimer; + static const int _idleTimeoutSeconds = 15; // TTS 播完后等待说话超时秒数 - final _uuid = const Uuid(); + // ═══════════════════════════════════════════════════════════════════════════ + // 生命周期 + // ═══════════════════════════════════════════════════════════════════════════ @override void onInit() { @@ -85,224 +125,457 @@ class MobileElfController extends GetxController { disconnect(); _recorder.dispose(); _player.dispose(); + scrollController.dispose(); super.onClose(); } void _loadConfig() { _productId = dotenv.env['MOBILE_ELF_PRODUCT_ID'] ?? '1899007766144614402'; _productKey = dotenv.env['MOBILE_ELF_PRODUCT_KEY'] ?? 'd0aI2yidbUUXS'; - // 优先使用缓存的 A80 凭证，否则降级到 .env 默认值 - _deviceNo = _storage.read(_kDeviceNo) ?? - dotenv.env['MOBILE_ELF_DEVICE_NO'] ?? '869700070000570'; - _deviceId = _storage.read(_kDeviceId) ?? - dotenv.env['MOBILE_ELF_DEVICE_ID'] ?? '1899013964170694658'; - _deviceSecret = _storage.read(_kDeviceSecret) ?? - dotenv.env['MOBILE_ELF_DEVICE_SECRET'] ?? ''; + _deviceNo = _storage.read(_kDeviceNo) ?? dotenv.env['MOBILE_ELF_DEVICE_NO'] ?? ''; + _deviceId = _storage.read(_kDeviceId) ?? ''; + _deviceSecret = _storage.read(_kDeviceSecret) ?? ''; credentialReady.value = _deviceSecret.isNotEmpty; } - /// 检测连接的耳机是否为 A80，如是则用其 license 换取平台凭证 Future _fetchA80CredentialsIfNeeded() async { try { - final btManager = Get.find(); - final deviceName = btManager.currentDevice?.name ?? ''; - final deviceMac = btManager.currentDevice?.address ?? ''; - final license = _findLicense(deviceMac); - - Logger.i('[MobileElf]', '=== 设备信息 ==='); - Logger.i('[MobileElf]', '设备名称: $deviceName'); - Logger.i('[MobileElf]', '设备MAC: $deviceMac'); - Logger.i('[MobileElf]', '匹配license: $license'); - Logger.i('[MobileElf]', '==============='); - - if (!deviceName.toUpperCase().contains('A80')) { - credentialReady.value = _deviceSecret.isNotEmpty; - return; - } - - // 已有缓存凭证且设备号未变则直接使用 - final cachedNo = _storage.read(_kDeviceNo) ?? ''; - final cachedSecret = _storage.read(_kDeviceSecret) ?? ''; - - if (license == null || license.isEmpty) { - Logger.w('[MobileElf]', 'A80 设备未找到 license，使用默认凭证'); - credentialReady.value = _deviceSecret.isNotEmpty; - return; - } - - if (cachedNo == license && cachedSecret.isNotEmpty) { - Logger.i('[MobileElf]', 'A80 凭证命中缓存，跳过请求'); - credentialReady.value = true; - return; - } - + final deviceNo = dotenv.env['MOBILE_ELF_DEVICE_NO'] ?? ''; + if (deviceNo.isEmpty) return; credentialError.value = ''; - Logger.i('[MobileElf]', 'A80 设备 license=$license，开始请求平台凭证'); - final dio = Dio(BaseOptions( baseUrl: 'https://ivs.chinamobiledevice.com:11443', connectTimeout: const Duration(seconds: 10), receiveTimeout: const Duration(seconds: 10), )); - - final resp = await dio.post( - '/v2/customer/device/secret/info', - data: { - 'deviceNoType': 'SN', - 'deviceNo': license, - 'productId': _productId, - 'productKey': _productKey, - }, - ); - + final resp = await dio.post('/v2/customer/device/secret/info', data: { + 'deviceNoType': 'SN', 'deviceNo': deviceNo, + 'productId': _productId, 'productKey': _productKey, + }); final body = resp.data as Map; if (body['code'].toString() == '200' && body['success'] == true) { final data = body['data'] as Map; - _deviceNo = data['deviceNo'] as String? ?? license; - _deviceId = data['deviceId'] as String? ?? _deviceId; + _deviceNo = data['deviceNo'] as String? ?? deviceNo; + _deviceId = data['deviceId'] as String? ?? ''; _deviceSecret = data['deviceSecret'] as String? ?? ''; - await _storage.write(_kDeviceNo, _deviceNo); await _storage.write(_kDeviceId, _deviceId); await _storage.write(_kDeviceSecret, _deviceSecret); - credentialReady.value = _deviceSecret.isNotEmpty; - Logger.i('[MobileElf]', 'A80 凭证获取成功 deviceId=$_deviceId'); } else { credentialError.value = body['message']?.toString() ?? '凭证请求失败'; - Logger.e('[MobileElf]', '平台返回错误: ${body['message']}'); } } catch (e) { credentialError.value = '凭证请求异常: $e'; - Logger.e('[MobileElf]', '获取 A80 凭证失败: $e'); } } - /// 从用户设备列表中按蓝牙 MAC 匹配 license - String? _findLicense(String? bluetoothMac) { - if (!User.isLoggedIn()) return null; - final devices = User.instance.devices; - if (bluetoothMac != null && bluetoothMac.isNotEmpty) { - final matched = devices.where((d) => - d.devicemac.toUpperCase() == bluetoothMac.toUpperCase()).toList(); - if (matched.isNotEmpty) return matched.first.license; + Future retryCredentials() async { + credentialError.value = ''; + await _fetchA80CredentialsIfNeeded(); + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 开始/结束通话 + // ═══════════════════════════════════════════════════════════════════════════ + + /// 配置音频会话：playAndRecord 模式，允许录音和播放同时进行 + Future _configureAudioSession() async { + try { + final session = await AudioSession.instance; + await session.configure(AudioSessionConfiguration( + avAudioSessionCategory: AVAudioSessionCategory.playAndRecord, + avAudioSessionCategoryOptions: AVAudioSessionCategoryOptions.defaultToSpeaker | + AVAudioSessionCategoryOptions.allowBluetooth, + avAudioSessionMode: AVAudioSessionMode.voiceChat, + androidAudioAttributes: const AndroidAudioAttributes( + contentType: AndroidAudioContentType.speech, + usage: AndroidAudioUsage.voiceCommunication, + ), + androidAudioFocusGainType: AndroidAudioFocusGainType.gain, + )); + await session.setActive(true); + Logger.info('yiding----------- [音频会话] 已配置 playAndRecord 模式'); + } catch (e) { + Logger.error('yiding----------- [音频会话] 配置失败: $e'); + } + } + + /// 点击通话按钮 → 开麦，进入等待说话状态（不连接服务器） + Future startCall() async { + if (inCall.value) return; + // PTT 模式下 AI 正在播报 → 先打断 TTS，再进入通话 + if (_ttsPlaying || status.value == MobileElfStatus.speaking) { + _interruptTts(); + _ttsInterrupted = false; + // 等一帧让 player.stop() 完成 + await Future.delayed(const Duration(milliseconds: 50)); + } + // 如果 PTT 正在录音中 → 先取消 + if (pttRecording.value) { + await stopPtt(cancel: true); + } + final hasPermission = await _recorder.hasPermission(); + if (!hasPermission) { _setError('无麦克风权限'); return; } + + await _configureAudioSession(); + // 为播放器单独设置 voiceCommunication 音频属性，与录音器使用同一音频路由 + // 这样 ExoPlayer 和 AudioRecord 可以像 VoIP 应用一样同时工作 + if (!kIsWeb) { + await _player.setAndroidAudioAttributes(const AndroidAudioAttributes( + contentType: AndroidAudioContentType.speech, + usage: AndroidAudioUsage.voiceCommunication, + )); } - // 按设备名兜底匹配 - final byName = devices.where((d) => - d.devicename.toUpperCase().contains('A80')).toList(); - return byName.isNotEmpty ? byName.first.license : null; + await _openMic(); + _startMicWatchdog(); + inCall.value = true; + status.value = MobileElfStatus.idle; // 等待用户说话 + callSeconds.value = 0; + _callTimer?.cancel(); + _callTimer = Timer.periodic(const Duration(seconds: 1), (_) => callSeconds.value++); + _startIdleTimer(); // 开始等待说话超时 + Logger.info('yiding----------- [通话] 已开始，等待用户说话'); } - // ── 连接 ────────────────────────────────────────────────────────────────── + /// 挂断 + void disconnect() { + Logger.info('yiding----------- [通话] 挂断'); + _stopRecordingTimer(); + _stopMicWatchdog(); + _cancelIdleTimer(); + _callTimer?.cancel(); _callTimer = null; + _forwardingAudio = false; + _wsConnecting = false; + _speechDetected = false; + _micSubscription?.cancel(); _micSubscription = null; + _recorder.stop(); + // 清空 TTS 队列 & 停播 + _ttsInterrupted = true; + _ttsQueue.clear(); + _ttsPlaying = false; + if (!(_ttsPlayCompleter?.isCompleted ?? true)) _ttsPlayCompleter?.complete(); + _player.stop(); + _closeWs(); + _preBuffer.clear(); + _sendBuffer.clear(); _sendBufferOffset = 0; + micRms.value = 0; micAlive.value = false; + inCall.value = false; + status.value = MobileElfStatus.idle; + } - Future _connect() async { - if (_channel != null) return; + // ═══════════════════════════════════════════════════════════════════════════ + // 麦克风 & 本地 VAD + // ═══════════════════════════════════════════════════════════════════════════ + + Future _openMic() async { + _micSubscription?.cancel(); + _micSubscription = null; + try { await _recorder.stop(); } catch (_) {} + final stream = await _recorder.startStream( + const RecordConfig( + encoder: AudioEncoder.pcm16bits, + sampleRate: 16000, + numChannels: 1, + echoCancel: true, + noiseSuppress: true, + androidConfig: AndroidRecordConfig( + audioSource: AndroidAudioSource.voiceCommunication, + ), + ), + ); + _speechDetected = false; + _vadOnsetCount = 0; + _micChunkCount = 0; + micAlive.value = false; + _micSubscription = stream.listen(_onMicAudio); + Logger.info('yiding----------- [麦克风] 已开启 (voiceCommunication + echoCancel)'); + } + + /// 启动麦克风看门狗：每 1.5 秒检测一次，如果无新 chunk 则标记 micAlive=false + void _startMicWatchdog() { + _micWatchdog?.cancel(); + _lastWatchdogChunkCount = _micChunkCount; + _micWatchdog = Timer.periodic(const Duration(milliseconds: 1500), (_) { + if (!inCall.value && !pttRecording.value) { _micWatchdog?.cancel(); return; } + if (_micChunkCount == _lastWatchdogChunkCount) { + micAlive.value = false; + if (status.value == MobileElfStatus.speaking || _ttsPlaying) { + // TTS 播放中麦克风已关闭，不需要重启 + } else { + Logger.info('yiding----------- [看门狗] 麦克风无数据! 尝试重启'); + _openMic(); + } + } + _lastWatchdogChunkCount = _micChunkCount; + }); + } + + void _stopMicWatchdog() { + _micWatchdog?.cancel(); + _micWatchdog = null; + } + + void _onMicAudio(Uint8List chunk) { + if (!inCall.value && !pttRecording.value) return; + _micChunkCount++; + + // 计算 RMS（PCM 16-bit little-endian） + final rms = _calcRms(chunk); + + // 更新 UI 可观察值 + micAlive.value = true; + micRms.value = rms; + + // 每 100 个 chunk (~2s) 输出一次调试日志，确认麦克风存活 + if (_micChunkCount % 100 == 0) { + Logger.info('yiding----------- [麦克风] chunk#$_micChunkCount rms=${rms.toStringAsFixed(0)} ' + 'status=${status.value} speechDetected=$_speechDetected fwd=$_forwardingAudio wsConn=$_wsConnecting'); + } + + // ── 当前正在转发音频到 WS ── + if (_forwardingAudio && _channel != null) { + _feedSendBuffer(chunk); + return; + } + + // ── WS 正在连接中，先存入预缓存 ── + if (_wsConnecting) { + _addToPreBuffer(chunk); + return; + } + + // ── TTS 播放中 / thinking 中不做 VAD ── + if (status.value == MobileElfStatus.speaking || _ttsPlaying || + status.value == MobileElfStatus.thinking) { + return; + } + + // ── 空闲/等待状态：连续 chunk VAD ── + _addToPreBuffer(chunk); + + if (rms >= _vadThreshold) { + _vadOnsetCount++; + if (_vadOnsetCount >= _vadOnsetChunks && !_speechDetected) { + _speechDetected = true; + _cancelIdleTimer(); + Logger.info('yiding----------- [VAD] 检测到说话 rms=${rms.toStringAsFixed(0)} ' + 'onset=$_vadOnsetCount/$_vadOnsetChunks，启动连接'); + _onSpeechDetected(); + } + } else { + _vadOnsetCount = 0; + } + } + + double _calcRms(Uint8List pcm) { + if (pcm.length < 2) return 0; + final samples = pcm.length ~/ 2; + double sum = 0; + for (int i = 0; i < pcm.length - 1; i += 2) { + int sample = pcm[i] | (pcm[i + 1] << 8); + if (sample >= 32768) sample -= 65536; // signed + sum += sample * sample; + } + return math.sqrt(sum / samples); + } + + void _addToPreBuffer(Uint8List chunk) { + _preBuffer.add(Uint8List.fromList(chunk)); + while (_preBuffer.length > _preBufferMaxChunks) { + _preBuffer.removeAt(0); + } + } + + // ═══════════════════════════════════════════════════════════════════════════ + // 检测到说话 → 连接服务器 → 发送预缓存 → 实时转发 + // ═══════════════════════════════════════════════════════════════════════════ + + Future _onSpeechDetected() async { + if (_wsConnecting) return; + + _ttsInterrupted = false; + _closeWs(); + _sessionEnded = false; + _wsConnecting = true; status.value = MobileElfStatus.connecting; + _dialogRequestId = _uuid.v4(); + _currentAiMsgIndex = -1; + realtimeText.value = ''; + _sendBuffer.clear(); + _sendBufferOffset = 0; errorMsg.value = ''; + Logger.info('yiding----------- [新轮] 说话触发连接 dialogId=$_dialogRequestId preBuffer=${_preBuffer.length}chunks'); + + try { + await _connect(); + Logger.info('yiding----------- [新轮] WS ready，开始发送预缓存'); + } catch (e) { + Logger.error('yiding----------- [新轮] 连接失败: $e'); + _wsConnecting = false; + _speechDetected = false; + _setError('连接失败: $e'); + return; + } + + // 发送预缓存音频 + _forwardingAudio = true; + _wsConnecting = false; + status.value = MobileElfStatus.recording; + recordingSeconds.value = 0; + _stopRecordingTimer(); + _recordingTimer = Timer.periodic(const Duration(seconds: 1), (_) => recordingSeconds.value++); + + final cached = List.from(_preBuffer); + _preBuffer.clear(); + int cachedBytes = 0; + for (final c in cached) { + _feedSendBuffer(c); + cachedBytes += c.length; + } + Logger.info('yiding----------- [新轮] 预缓存已发送 ${cached.length}chunks ${cachedBytes}bytes，实时转发中'); + } + + /// 将 chunk 加入发送缓冲，凑满 _frameSize 就发 + void _feedSendBuffer(Uint8List chunk) { + _sendBuffer.addAll(chunk); + while (_sendBuffer.length - _sendBufferOffset >= _frameSize) { + final frame = Uint8List.fromList( + _sendBuffer.sublist(_sendBufferOffset, _sendBufferOffset + _frameSize)); + _sendBufferOffset += _frameSize; + _sendAudioChunk(frame); + } + // 紧缩 + if (_sendBufferOffset >= _frameSize * 8) { + _sendBuffer.removeRange(0, _sendBufferOffset); + _sendBufferOffset = 0; + } + } + + // ═══════════════════════════════════════════════════════════════════════════ + // WebSocket 连接 + // ═══════════════════════════════════════════════════════════════════════════ + + Future _connect() async { + _readyCompleter = Completer(); try { final ts = DateTime.now().millisecondsSinceEpoch.toString(); final sn = _uuid.v4(); final sign = _calcSign(_deviceSecret, ts); - final url = 'wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr' - '?deviceNo=$_deviceNo' - '&deviceId=$_deviceId' - '&productId=$_productId' - '&productKey=$_productKey' - '&sign=$sign' - '&sn=$sn' - '&ts=$ts'; + '?deviceNo=$_deviceNo&deviceId=$_deviceId&productId=$_productId' + '&productKey=$_productKey&sign=$sign&sn=$sn&ts=$ts'; if (kIsWeb) { _channel = WebSocketChannel.connect(Uri.parse(url)); } else { - _channel = IOWebSocketChannel.connect(Uri.parse(url)); + final httpClient = HttpClient()..badCertificateCallback = (_, __, ___) => true; + final ws = await WebSocket.connect(url, customClient: httpClient, headers: { + 'Origin': 'https://ivs.chinamobiledevice.com:11443', + 'Host': 'ivs.chinamobiledevice.com:11443', + }); + _channel = IOWebSocketChannel(ws); } _wsSubscription = _channel!.stream.listen( _handleMessage, - onError: _handleWsError, - onDone: _handleWsDone, + onError: (e) { + Logger.error('yiding----------- [WS] 错误: $e'); + if (!(_readyCompleter?.isCompleted ?? true)) _readyCompleter?.completeError(e); + _handleWsDone(); + }, + onDone: () { + Logger.info('yiding----------- [WS] onDone sessionEnded=$_sessionEnded status=${status.value}'); + if (!(_readyCompleter?.isCompleted ?? true)) _readyCompleter?.completeError('WS closed'); + _handleWsDone(); + }, ); - Logger.info('[MobileElf] WebSocket 连接中: $url'); + _sendStartFrame(); + await _readyCompleter!.future.timeout( + const Duration(seconds: 15), + onTimeout: () => throw TimeoutException('等待 ready 超时'), + ); } catch (e) { - Logger.error('[MobileElf] 连接失败: $e'); - _setError('连接失败: $e'); + _closeWs(); + rethrow; } } - // ── 开始对话（录音） ───────────────────────────────────────────────────── - - Future startRecording() async { - if (status.value == MobileElfStatus.recording) return; - if (status.value == MobileElfStatus.thinking) return; - - await _connect(); - - final hasPermission = await _recorder.hasPermission(); - if (!hasPermission) { - _setError('无麦克风权限'); - return; - } - - _dialogRequestId = _uuid.v4(); - realtimeText.value = ''; - - try { - final stream = await _recorder.startStream( - const RecordConfig( - encoder: AudioEncoder.pcm16bits, - sampleRate: 16000, - numChannels: 1, - ), - ); - - _audioBuffer.clear(); - _audioSubscription = stream.listen((chunk) { - _audioBuffer.addAll(chunk); - while (_audioBuffer.length >= _frameSize) { - _sendAudioChunk( - Uint8List.fromList(_audioBuffer.sublist(0, _frameSize))); - _audioBuffer.removeRange(0, _frameSize); - } - }); + void _handleWsDone() { + _channel = null; + _wsSubscription = null; + _forwardingAudio = false; + _wsConnecting = false; - status.value = MobileElfStatus.recording; - Logger.info('[MobileElf] 开始录音'); - } catch (e) { - Logger.error('[MobileElf] 录音失败: $e'); - _setError('录音失败: $e'); + if (_sessionEnded && inCall.value) { + Logger.info('yiding----------- [WS] 正常关闭 (通话模式)'); + if (status.value == MobileElfStatus.speaking) { + // TTS 播放中，等 playTts 完成后自动回到 idle 等待说话 + } else { + status.value = MobileElfStatus.idle; + _resetVad(); + Logger.info('yiding----------- [WS] 回到等待说话'); + } + } else if (_sessionEnded && !inCall.value) { + // PTT/文本模式：WS 正常关闭 + Logger.info('yiding----------- [WS] 正常关闭 (PTT/文本模式)'); + if (status.value != MobileElfStatus.speaking) { + status.value = MobileElfStatus.idle; + } + } else if (inCall.value) { + Logger.info('yiding----------- [WS] 非预期关闭，回到等待说话'); + status.value = MobileElfStatus.idle; + _resetVad(); + } else { + Logger.info('yiding----------- [WS] 非预期关闭'); + status.value = MobileElfStatus.idle; } } - Future stopRecording() async { - if (status.value != MobileElfStatus.recording) return; + void _resetVad() { + _speechDetected = false; + _vadOnsetCount = 0; + _preBuffer.clear(); + _openMic(); + } - await _audioSubscription?.cancel(); - _audioSubscription = null; + // ═══════════════════════════════════════════════════════════════════════════ + // 空闲超时：TTS 播完后等待说话，超时自动挂断 + // ═══════════════════════════════════════════════════════════════════════════ - // 发送剩余缓冲数据 - if (_audioBuffer.isNotEmpty) { - _sendAudioChunk(Uint8List.fromList(_audioBuffer)); - _audioBuffer.clear(); - } + void _startIdleTimer() { + _cancelIdleTimer(); + _idleTimer = Timer(Duration(seconds: _idleTimeoutSeconds), () { + if (inCall.value && status.value == MobileElfStatus.idle && !_speechDetected) { + Logger.info('yiding----------- [超时] ${_idleTimeoutSeconds}秒无说话，自动挂断'); + disconnect(); + } + }); + Logger.info('yiding----------- [超时] 开始${_idleTimeoutSeconds}秒等待说话计时'); + } - await _recorder.stop(); - _sendFinish(); + void _cancelIdleTimer() { + _idleTimer?.cancel(); + _idleTimer = null; + } - status.value = MobileElfStatus.thinking; - Logger.info('[MobileElf] 停止录音，等待 AI 回答'); + void _closeWs() { + final sub = _wsSubscription; _wsSubscription = null; + sub?.cancel(); + final ch = _channel; _channel = null; + try { ch?.sink.close(); } catch (_) {} } - // ── 发送帧 ──────────────────────────────────────────────────────────────── + // ═══════════════════════════════════════════════════════════════════════════ + // 发送帧 + // ═══════════════════════════════════════════════════════════════════════════ void _sendStartFrame() { if (_channel == null) return; - - final startFrame = { + _channel!.sink.add(jsonEncode({ 'type': 'start', 'data': { 'cuid': _deviceId, @@ -323,57 +596,57 @@ class MobileElfController extends GetxController { 'messages': _buildContextMessages(), 'client_context': [], }, - }; - - _channel!.sink.add(jsonEncode(startFrame)); - Logger.info('[MobileElf] 已发送 start 帧'); + })); } void _sendAudioChunk(Uint8List data) { - try { - _channel?.sink.add(data); - } catch (e) { - Logger.error('[MobileElf] 发送音频帧失败: $e'); - } + try { _channel?.sink.add(data); } catch (_) {} } void _sendFinish() { - try { - _channel?.sink.add(jsonEncode({'type': 'finish'})); - Logger.info('[MobileElf] 已发送 finish 帧'); - } catch (e) { - Logger.error('[MobileElf] 发送 finish 帧失败: $e'); - } + try { _channel?.sink.add(jsonEncode({'type': 'finish'})); } catch (_) {} } - // ── 接收消息处理 ────────────────────────────────────────────────────────── + // ═══════════════════════════════════════════════════════════════════════════ + // 接收消息处理 + // ═══════════════════════════════════════════════════════════════════════════ void _handleMessage(dynamic raw) { try { if (raw is! String) return; - final Map msg = jsonDecode(raw); + final msg = jsonDecode(raw) as Map; final type = msg['type'] as String? ?? ''; switch (type) { case 'ready': - Logger.info('[MobileElf] 收到 ready，发送 start 帧'); - _sendStartFrame(); + if (!(_readyCompleter?.isCompleted ?? true)) _readyCompleter?.complete(); + break; + + case 'stoplisten': + Logger.info('yiding----------- [服务端] stoplisten'); + _stopRecordingTimer(); + _forwardingAudio = false; + _speechDetected = false; // 允许 VAD 在 thinking/speaking 期间检测新语音 + _vadOnsetCount = 0; + // 发送剩余 + final remain = _sendBuffer.length - _sendBufferOffset; + if (remain > 0) _sendAudioChunk(Uint8List.fromList(_sendBuffer.sublist(_sendBufferOffset))); + _sendBuffer.clear(); _sendBufferOffset = 0; + _sendFinish(); + status.value = MobileElfStatus.thinking; break; case 'mid_result': - final result = msg['result'] as String? ?? ''; - realtimeText.value = result; + realtimeText.value = msg['result'] as String? ?? ''; break; case 'fin_result': final result = msg['result'] as String? ?? ''; if (result.isNotEmpty) { - realtimeText.value = result; - // 用户消息在 fin_result 时确认写入 + realtimeText.value = ''; _addUserMessage(result); } - final end = msg['end'] as int? ?? 0; - if (end == 1) _handleSessionEnd(); + if (msg['end'] == 1) _markSessionEnded(); break; case 'inside_rc': @@ -381,12 +654,11 @@ class MobileElfController extends GetxController { break; case 'dcs_decide': - final end = msg['end'] as int? ?? 0; - if (end == 1) _handleSessionEnd(); + if (msg['end'] == 1) { + Logger.info('yiding----------- [服务端] dcs_decide end:1'); + _markSessionEnded(); + } break; - - default: - Logger.info('[MobileElf] 未知消息类型: $type'); } } catch (e) { Logger.error('[MobileElf] 处理消息失败: $e'); @@ -396,177 +668,313 @@ class MobileElfController extends GetxController { void _handleInsideRc(Map msg) { final data = msg['data'] as Map?; if (data == null) return; + if ((data['code'] as int? ?? -1) != 0) return; - final code = data['code'] as int? ?? -1; - if (code != 0) { - Logger.error('[MobileElf] inside_rc 错误: ${data['msg']}'); - return; - } - - // 提取 AI 文本回答 final assistantAnswer = data['assistant_answer'] as String? ?? ''; String cleanAnswer = assistantAnswer; - // assistant_answer 可能是 JSON 字符串 try { - final decoded = jsonDecode(assistantAnswer) as Map; - cleanAnswer = decoded['content'] as String? ?? assistantAnswer; + cleanAnswer = (jsonDecode(assistantAnswer) as Map)['content'] as String? ?? assistantAnswer; } catch (_) {} - // 提取 TTS URL（Speak 指令） String? ttsUrl; - final directives = data['data'] as List? ?? []; - for (final item in directives) { - final directive = item as Map; - final header = directive['header'] as Map?; - final ns = header?['namespace'] as String? ?? ''; - final name = header?['name'] as String? ?? ''; - if (ns == 'ai.fxzsos.device_interface.voice_output' && name == 'Speak') { - final payload = directive['payload'] as Map?; - ttsUrl = payload?['url'] as String?; + for (final item in (data['data'] as List? ?? [])) { + final d = item as Map; + final h = d['header'] as Map?; + if (h?['namespace'] == 'ai.fxzsos.device_interface.voice_output' && h?['name'] == 'Speak') { + ttsUrl = (d['payload'] as Map?)?['url'] as String?; break; } } if (cleanAnswer.isNotEmpty) { - _addAiMessage(cleanAnswer, ttsUrl: ttsUrl); - } - - // 播放 TTS - if (ttsUrl != null && ttsUrl.isNotEmpty) { - playTts(ttsUrl); + if (_currentAiMsgIndex >= 0 && _currentAiMsgIndex < messages.length) { + messages[_currentAiMsgIndex] = MobileElfMessage( + isUser: false, text: cleanAnswer, + time: messages[_currentAiMsgIndex].time, + ttsUrl: ttsUrl ?? messages[_currentAiMsgIndex].ttsUrl, + ); + } else { + _currentAiMsgIndex = messages.length; + _addAiMessage(cleanAnswer, ttsUrl: ttsUrl); + } } - final end = msg['end'] as int? ?? 0; - if (end == 1) _handleSessionEnd(); + if (ttsUrl != null && ttsUrl.isNotEmpty) _enqueueTts(ttsUrl); + if (msg['end'] == 1) _markSessionEnded(); } - void _handleSessionEnd() { - if (status.value != MobileElfStatus.speaking) { - status.value = MobileElfStatus.idle; - } + void _markSessionEnded() { + if (_sessionEnded) return; + _sessionEnded = true; + Logger.info('yiding----------- [会话] 本轮结束 status=${status.value}'); + _stopRecordingTimer(); + _forwardingAudio = false; + _speechDetected = false; // 允许 VAD 在 TTS 播放期间检测新语音 + _vadOnsetCount = 0; + _sendBuffer.clear(); _sendBufferOffset = 0; realtimeText.value = ''; - // 断开当前 WebSocket，下次对话重新连接（协议要求每次新对话重连） - _closeWs(); } - void _handleWsError(dynamic error) { - Logger.error('[MobileElf] WebSocket 错误: $error'); - _setError('连接错误: $error'); - _closeWs(); + // ═══════════════════════════════════════════════════════════════════════════ + // TTS 播放队列 + // ═══════════════════════════════════════════════════════════════════════════ + + /// 将 TTS URL 加入队列，如果没有正在播放则关闭麦克风并开始播放 + void _enqueueTts(String url) { + _ttsQueue.add(url); + Logger.info('yiding----------- [TTS] 入队 queue=${_ttsQueue.length}'); + if (!_ttsPlaying) { + // 关闭麦克风，避免音频冲突，TTS 播完后会重开 + _micSubscription?.cancel(); _micSubscription = null; + try { _recorder.stop(); } catch (_) {} + micAlive.value = false; micRms.value = 0; + Logger.info('yiding----------- [TTS] 关闭麦克风，开始播放'); + _playNextTts(); + } } - void _handleWsDone() { - Logger.info('[MobileElf] WebSocket 关闭'); - if (status.value == MobileElfStatus.connecting || - status.value == MobileElfStatus.recording) { - _setError('连接已断开'); + /// 顺序播放队列中的 TTS + Future _playNextTts() async { + if (_ttsQueue.isEmpty || _ttsInterrupted) { + _ttsPlaying = false; + // 队列播完 → 重开麦克风，回到等待说话，启动超时计时 + if (!_ttsInterrupted && status.value == MobileElfStatus.speaking) { + status.value = MobileElfStatus.idle; + Logger.info('yiding----------- [TTS] 队列全部播完，重开麦克风，等待新一轮说话'); + if (inCall.value) { + _resetVad(); + _startIdleTimer(); + } + } + return; } - _channel = null; - _wsSubscription = null; - } - // ── TTS 播放 ────────────────────────────────────────────────────────────── + _ttsPlaying = true; + status.value = MobileElfStatus.speaking; + final url = _ttsQueue.removeAt(0); + + // 用 Completer 等待播放完成或被打断 + _ttsPlayCompleter = Completer(); + StreamSubscription? stateSub; - Future playTts(String url) async { try { - status.value = MobileElfStatus.speaking; + await _player.stop(); + // 不再重新配置音频会话！startCall() 已配置好 playAndRecord 模式， + // 重复配置会导致 audio_session 重新请求音频焦点，杀死录音流 await _player.setUrl(url); + + // 监听 ProcessingState: completed=正常播完, idle=被打断或意外停止 + stateSub = _player.processingStateStream.listen((s) { + if ((s == ProcessingState.completed || s == ProcessingState.idle) && + !(_ttsPlayCompleter?.isCompleted ?? true)) { + Logger.info('yiding----------- [TTS] 当前段结束 state=$s'); + _ttsPlayCompleter?.complete(); + } + }); + await _player.play(); - await _player.processingStateStream - .firstWhere((s) => s == ProcessingState.completed); - status.value = MobileElfStatus.idle; - Logger.info('[MobileElf] TTS 播放完成'); + Logger.info('yiding----------- [TTS] 开始播放 剩余队列=${_ttsQueue.length} micAlive=${micAlive.value} micChunk=$_micChunkCount'); + + // 等待：正常播完 complete() 或被打断 _interruptTts() 也会 complete() + await _ttsPlayCompleter!.future.timeout(const Duration(seconds: 60)); } catch (e) { - Logger.error('[MobileElf] TTS 播放失败: $e'); - status.value = MobileElfStatus.idle; + Logger.error('yiding----------- [TTS] 播放异常: $e'); + } finally { + stateSub?.cancel(); + } + + // 当前段结束，继续下一段（如果没被打断） + if (!_ttsInterrupted) { + _playNextTts(); } } - Future stopSpeaking() async { - await _player.stop(); + /// 立即打断 TTS：停播 + 清队列 + 解除等待 + void _interruptTts() { + Logger.info('yiding----------- [TTS] 打断! 清空队列=${_ttsQueue.length}'); + _ttsInterrupted = true; + _ttsQueue.clear(); + _ttsPlaying = false; + // 先解除 _playNextTts 的等待 + if (!(_ttsPlayCompleter?.isCompleted ?? true)) { + _ttsPlayCompleter?.complete(); + } + _player.stop(); + } + + /// 手动打断 AI 播报（界面按钮调用） + Future interruptSpeaking() async { + _interruptTts(); + _ttsInterrupted = false; // 手动打断后重置，不影响后续 status.value = MobileElfStatus.idle; + if (inCall.value) _resetVad(); } - // ── 断开连接 ────────────────────────────────────────────────────────────── + // ═══════════════════════════════════════════════════════════════════════════ + // 手动停止录音 + // ═══════════════════════════════════════════════════════════════════════════ - void disconnect() { - _audioSubscription?.cancel(); - _audioSubscription = null; - _recorder.cancel(); - _closeWs(); - status.value = MobileElfStatus.idle; + Future stopRecording() async { + if (!_forwardingAudio) return; + _stopRecordingTimer(); + _forwardingAudio = false; + final remain = _sendBuffer.length - _sendBufferOffset; + if (remain > 0) _sendAudioChunk(Uint8List.fromList(_sendBuffer.sublist(_sendBufferOffset))); + _sendBuffer.clear(); _sendBufferOffset = 0; + _sendFinish(); + status.value = MobileElfStatus.thinking; } - void _closeWs() { - _wsSubscription?.cancel(); - _wsSubscription = null; - _channel?.sink.close(); - _channel = null; + void _stopRecordingTimer() { + _recordingTimer?.cancel(); _recordingTimer = null; } - // ── 辅助方法 ────────────────────────────────────────────────────────────── + // ═══════════════════════════════════════════════════════════════════════════ + // 按住说话 (PTT) + // ═══════════════════════════════════════════════════════════════════════════ + + /// 按住说话 → 开始录音并连接 WS + Future startPtt() async { + if (pttRecording.value) return; + // 如果正在播放 AI 回复 → 立即打断 + if (_ttsPlaying || status.value == MobileElfStatus.speaking) { + _interruptTts(); + _ttsInterrupted = false; + } + final hasPermission = await _recorder.hasPermission(); + if (!hasPermission) { _setError('无麦克风权限'); return; } - String _calcSign(String deviceSecret, String ts) { - final bytes = utf8.encode('$deviceSecret$ts'); - return md5.convert(bytes).toString(); - } + // 先设置 connecting 状态，再设置 pttRecording，避免中间按钮橙色闪烁 + status.value = MobileElfStatus.connecting; + pttRecording.value = true; + pttSeconds.value = 0; + _pttTimer = Timer.periodic(const Duration(seconds: 1), (_) => pttSeconds.value++); + + await _configureAudioSession(); + if (!kIsWeb) { + await _player.setAndroidAudioAttributes(const AndroidAudioAttributes( + contentType: AndroidAudioContentType.speech, + usage: AndroidAudioUsage.voiceCommunication, + )); + } + await _openMic(); + _startMicWatchdog(); - List> _buildContextMessages() { - return messages.map((m) => { - 'role': m.isUser ? 'user' : 'assistant', - 'content': m.text, - }).toList(); - } + _ttsInterrupted = false; + _closeWs(); + _sessionEnded = false; + _wsConnecting = true; + _dialogRequestId = _uuid.v4(); + _currentAiMsgIndex = -1; + realtimeText.value = ''; + _sendBuffer.clear(); _sendBufferOffset = 0; + errorMsg.value = ''; - void _addUserMessage(String text) { - // 避免重复添加 - if (messages.isNotEmpty && - messages.last.isUser && - messages.last.text == text) { - return; + try { + await _connect(); + _forwardingAudio = true; + _wsConnecting = false; + status.value = MobileElfStatus.recording; + Logger.info('yiding----------- [PTT] 录音开始'); + } catch (e) { + _wsConnecting = false; + pttRecording.value = false; + _pttTimer?.cancel(); _pttTimer = null; + _stopMicWatchdog(); + _setError('连接失败: $e'); } - messages.add(MobileElfMessage( - isUser: true, - text: text, - time: DateTime.now(), - )); } - void _addAiMessage(String text, {String? ttsUrl}) { - messages.add(MobileElfMessage( - isUser: false, - text: text, - time: DateTime.now(), - ttsUrl: ttsUrl, - )); - } + /// 松开 → 停止录音，发送 finish + Future stopPtt({bool cancel = false}) async { + pttRecording.value = false; + _pttTimer?.cancel(); _pttTimer = null; + _stopMicWatchdog(); + + if (cancel) { + _forwardingAudio = false; + _closeWs(); + _micSubscription?.cancel(); _micSubscription = null; + try { _recorder.stop(); } catch (_) {} + micAlive.value = false; micRms.value = 0; + realtimeText.value = ''; + status.value = MobileElfStatus.idle; + Logger.info('yiding----------- [PTT] 已取消'); + return; + } - void _setError(String msg) { - errorMsg.value = msg; - status.value = MobileElfStatus.error; + // 正常结束：发送剩余音频 + finish + if (_forwardingAudio) { + _forwardingAudio = false; + final remain = _sendBuffer.length - _sendBufferOffset; + if (remain > 0) _sendAudioChunk(Uint8List.fromList(_sendBuffer.sublist(_sendBufferOffset))); + _sendBuffer.clear(); _sendBufferOffset = 0; + _sendFinish(); + status.value = MobileElfStatus.thinking; + } + _micSubscription?.cancel(); _micSubscription = null; + try { _recorder.stop(); } catch (_) {} + micAlive.value = false; micRms.value = 0; + Logger.info('yiding----------- [PTT] 录音结束，等待AI回复'); } - /// 重试获取 A80 凭证 - Future retryCredentials() async { - credentialError.value = ''; - await _fetchA80CredentialsIfNeeded(); - } + // ═══════════════════════════════════════════════════════════════════════════ + // 按钮操作 + // ═══════════════════════════════════════════════════════════════════════════ - /// 按钮主操作：空闲/错误 → 开始录音；录音中 → 停止；播放中 → 停止 Future onMicTap() async { + Logger.info('yiding----------- [按钮] status=${status.value} inCall=${inCall.value}'); switch (status.value) { case MobileElfStatus.idle: case MobileElfStatus.error: - await startRecording(); + if (!inCall.value) { + await startCall(); + } break; case MobileElfStatus.recording: await stopRecording(); break; case MobileElfStatus.speaking: - await stopSpeaking(); + await interruptSpeaking(); break; default: break; } } + + // ═══════════════════════════════════════════════════════════════════════════ + // 工具方法 + // ═══════════════════════════════════════════════════════════════════════════ + + String _calcSign(String secret, String ts) => md5.convert(utf8.encode('$secret$ts')).toString(); + + List> _buildContextMessages() => + messages.map((m) => {'role': m.isUser ? 'user' : 'assistant', 'content': m.text}).toList(); + + void _addUserMessage(String text) { + if (messages.isNotEmpty && messages.last.isUser && messages.last.text == text) return; + messages.add(MobileElfMessage(isUser: true, text: text, time: DateTime.now())); + scrollToBottom(); + } + + void _addAiMessage(String text, {String? ttsUrl}) { + messages.add(MobileElfMessage(isUser: false, text: text, time: DateTime.now(), ttsUrl: ttsUrl)); + scrollToBottom(); + } + + void _setError(String msg) { + errorMsg.value = msg; + status.value = MobileElfStatus.error; + } + + void scrollToBottom() { + if (scrollController.hasClients) { + WidgetsBinding.instance.addPostFrameCallback((_) { + if (scrollController.hasClients) { + scrollController.animateTo(scrollController.position.maxScrollExtent, + duration: const Duration(milliseconds: 200), curve: Curves.easeOut); + } + }); + } + } } diff --git a/lib/modules/mobile_elf/views/mobile_elf_view.dart b/lib/modules/mobile_elf/views/mobile_elf_view.dart index 51db8c287..7aaa114b1 100644 --- a/lib/modules/mobile_elf/views/mobile_elf_view.dart +++ b/lib/modules/mobile_elf/views/mobile_elf_view.dart @@ -6,503 +6,811 @@ import '../controllers/mobile_elf_controller.dart'; class MobileElfView extends GetView { const MobileElfView({Key? key}) : super(key: key); + // ── 颜色常量 ────────────────────────────────────────────────────────────── + static const _blue = Color(0xFF2E7CF6); + static const _green = Color(0xFF34C759); + static const _greenDark = Color(0xFF16A34A); + static const _red = Color(0xFFEF4444); + static const _orange = Color(0xFFF79009); + static const _bgLight = Color(0xFFF4F6F9); + static const _surfaceLight = Colors.white; + static const _textPrimary = Color(0xFF101828); + static const _textSecondary = Color(0xFF667085); + static const _border = Color(0xFFEAECF0); + + // 渐变复用 + static const _avatarGradient = LinearGradient( + colors: [Color(0xFF667EEA), Color(0xFF764BA2)], + begin: Alignment.topLeft, + end: Alignment.bottomRight, + ); + @override Widget build(BuildContext context) { - final bool isDark = Theme.of(context).brightness == Brightness.dark; + final isDark = Theme.of(context).brightness == Brightness.dark; return Scaffold( - backgroundColor: isDark ? Colors.black : const Color(0xFFF9FAFC), + backgroundColor: isDark ? const Color(0xFF0D0D0D) : _bgLight, appBar: _buildAppBar(isDark), - body: Obx(() { - final err = controller.credentialError.value; - if (err.isNotEmpty) { - return Center( - child: Padding( - padding: EdgeInsets.all(24.w), - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - Icon(Icons.error_outline, size: 48.sp, color: Colors.red), - SizedBox(height: 12.h), - Text(err, - textAlign: TextAlign.center, - style: TextStyle(fontSize: 13.sp, color: Colors.red)), - SizedBox(height: 16.h), - ElevatedButton( - onPressed: controller.retryCredentials, - child: const Text('重试'), - ), - ], - ), - ), + // 顶层 Listener 捕获 PTT 松开事件（确保 widget 树重建后仍能接收 pointerUp） + body: Listener( + behavior: HitTestBehavior.translucent, + onPointerUp: (_) { + if (controller.pttRecording.value) controller.stopPtt(); + }, + child: Obx(() { + if (controller.credentialError.value.isNotEmpty) { + return _buildCredentialError(); + } + return Column( + children: [ + Expanded(child: _buildMessageList(isDark)), + _buildBottomBar(isDark), + ], ); - } - return Column( - children: [ - _buildStatusBanner(), - Expanded(child: _buildMessageList(isDark)), - _buildBottomBar(isDark), - ], - ); - }), + }), + ), ); } + // ══════════════════════════════════════════════════════════════════════════ + // AppBar + // ══════════════════════════════════════════════════════════════════════════ + PreferredSizeWidget _buildAppBar(bool isDark) { return AppBar( - backgroundColor: isDark ? const Color(0xFF1A1A1A) : Colors.white, - elevation: 0.5, + backgroundColor: isDark ? const Color(0xFF1A1A1A) : _surfaceLight, + elevation: 0, + surfaceTintColor: Colors.transparent, + bottom: PreferredSize( + preferredSize: const Size.fromHeight(1), + child: Container( + color: isDark ? const Color(0xFF222222) : _border, height: 1), + ), leading: IconButton( icon: Icon(Icons.arrow_back_ios, - size: 18.sp, - color: isDark ? Colors.white : Colors.grey[800]), + size: 18.sp, color: isDark ? Colors.white : _textSecondary), onPressed: () => Get.back(), ), title: Row( mainAxisSize: MainAxisSize.min, children: [ - Container( - width: 32.w, - height: 32.w, - decoration: BoxDecoration( - gradient: const LinearGradient( - colors: [Color(0xFF0091EA), Color(0xFF00BFA5)], - ), - borderRadius: BorderRadius.circular(8.r), - ), - child: Icon(Icons.cell_tower, color: Colors.white, size: 18.sp), - ), - SizedBox(width: 8.w), - Text( - '移动精灵', - style: TextStyle( - fontSize: 16.sp, - fontWeight: FontWeight.bold, - color: isDark ? Colors.white : Colors.grey[800], - ), - ), + _gradientCircle(32.w, '晓', 13.sp), + SizedBox(width: 10.w), + Text('移动精灵', + style: TextStyle( + fontSize: 16.sp, + fontWeight: FontWeight.w600, + color: isDark ? Colors.white : _textPrimary)), ], ), actions: [ Obx(() => Padding( padding: EdgeInsets.only(right: 16.w), - child: _buildStatusDot(), + child: _buildStatusPill(), )), ], ); } - Widget _buildStatusDot() { - final color = _statusColor(controller.status.value); - return Row( - children: [ - Container( - width: 8.w, - height: 8.w, - decoration: BoxDecoration( - color: color, - shape: BoxShape.circle, - boxShadow: [ - BoxShadow(color: color.withOpacity(0.4), blurRadius: 4) - ], - ), - ), - SizedBox(width: 4.w), - Text( - _statusLabel(controller.status.value), - style: TextStyle(fontSize: 11.sp, color: color), - ), - ], + /// 右上角状态胶囊 + 通话计时 + Widget _buildStatusPill() { + final s = controller.status.value; + final inCall = controller.inCall.value; + final cfg = _pillConfig(s, inCall); + return Container( + padding: EdgeInsets.symmetric(horizontal: 10.w, vertical: 3.h), + decoration: BoxDecoration( + color: cfg.bg, + borderRadius: BorderRadius.circular(20.r), + ), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Text(cfg.label, + style: TextStyle( + fontSize: 11.sp, + color: cfg.fg, + fontWeight: FontWeight.w500)), + if (inCall) ...[ + SizedBox(width: 4.w), + Obx(() { + final sec = controller.callSeconds.value; + return Text(_fmtDuration(sec), + style: TextStyle( + fontSize: 11.sp, + color: cfg.fg, + fontWeight: FontWeight.w400)); + }), + ], + ], + ), ); } - Widget _buildStatusBanner() { - return Obx(() { - final s = controller.status.value; - if (s == MobileElfStatus.idle) return const SizedBox.shrink(); + _PillCfg _pillConfig(MobileElfStatus s, bool inCall) { + switch (s) { + case MobileElfStatus.connecting: + return const _PillCfg(Color(0xFFFFFAEB), Color(0xFFB54708), '连接中…'); + case MobileElfStatus.recording: + return const _PillCfg(Color(0xFFECFDF3), _greenDark, '聆听中'); + case MobileElfStatus.thinking: + return const _PillCfg(Color(0xFFEFF8FF), Color(0xFF175CD3), '处理中'); + case MobileElfStatus.speaking: + return const _PillCfg(Color(0xFFECFDF3), _greenDark, '朗读中'); + case MobileElfStatus.error: + return const _PillCfg(Color(0xFFFEF2F2), _red, '错误'); + case MobileElfStatus.idle: + return inCall + ? const _PillCfg(Color(0xFFECFDF3), _greenDark, '通话中') + : const _PillCfg(Color(0xFFF2F4F7), _textSecondary, '空闲'); + } + } - final color = _statusColor(s); - final label = _statusBannerLabel(s); + // ══════════════════════════════════════════════════════════════════════════ + // 凭证错误 + // ══════════════════════════════════════════════════════════════════════════ - return Container( - width: double.infinity, - padding: EdgeInsets.symmetric(vertical: 6.h), - color: color.withOpacity(0.12), - child: Row( - mainAxisAlignment: MainAxisAlignment.center, + Widget _buildCredentialError() { + return Center( + child: Padding( + padding: EdgeInsets.all(32.w), + child: Column( + mainAxisSize: MainAxisSize.min, children: [ - if (s == MobileElfStatus.thinking || s == MobileElfStatus.connecting) - SizedBox( - width: 12.w, - height: 12.w, - child: CircularProgressIndicator( - strokeWidth: 1.5, - color: color, - ), - ), - if (s == MobileElfStatus.recording) - Icon(Icons.mic, size: 14.sp, color: color), - if (s == MobileElfStatus.speaking) - Icon(Icons.volume_up, size: 14.sp, color: color), - if (s == MobileElfStatus.error) - Icon(Icons.error_outline, size: 14.sp, color: color), - SizedBox(width: 6.w), - Text(label, - style: TextStyle( - fontSize: 12.sp, - color: color, - fontWeight: FontWeight.w500)), + Icon(Icons.wifi_off_rounded, size: 56.sp, color: Colors.red[300]), + SizedBox(height: 16.h), + Obx(() => Text(controller.credentialError.value, + textAlign: TextAlign.center, + style: TextStyle(fontSize: 13.sp, color: Colors.red[400]))), + SizedBox(height: 20.h), + ElevatedButton.icon( + onPressed: controller.retryCredentials, + icon: const Icon(Icons.refresh), + label: const Text('重试'), + ), ], ), - ); - }); + ), + ); } + // ══════════════════════════════════════════════════════════════════════════ + // 消息列表 + // ══════════════════════════════════════════════════════════════════════════ + Widget _buildMessageList(bool isDark) { return Obx(() { - if (controller.messages.isEmpty) { - return _buildEmptyHint(isDark); - } + final msgs = controller.messages; + final rt = controller.realtimeText.value; + if (msgs.isEmpty && rt.isEmpty) return _buildEmptyState(isDark); return ListView.builder( - padding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 12.h), - reverse: false, - itemCount: controller.messages.length, - itemBuilder: (_, i) => _buildBubble(controller.messages[i], isDark), + controller: controller.scrollController, + padding: EdgeInsets.fromLTRB(16.w, 16.h, 16.w, 8.h), + itemCount: msgs.length + (rt.isNotEmpty ? 1 : 0), + itemBuilder: (_, i) { + if (i < msgs.length) return _buildBubble(msgs[i], isDark); + return _buildRealtimeBubble(rt); + }, ); }); } - Widget _buildEmptyHint(bool isDark) { + Widget _buildEmptyState(bool isDark) { return Center( child: Column( mainAxisSize: MainAxisSize.min, children: [ - Container( - width: 72.w, - height: 72.w, - decoration: BoxDecoration( - gradient: const LinearGradient( - colors: [Color(0xFF0091EA), Color(0xFF00BFA5)], - ), - shape: BoxShape.circle, - ), - child: Icon(Icons.cell_tower, color: Colors.white, size: 36.sp), - ), - SizedBox(height: 16.h), - Text( - '移动精灵', - style: TextStyle( - fontSize: 18.sp, - fontWeight: FontWeight.bold, - color: isDark ? Colors.white : Colors.grey[800], - ), - ), + _gradientCircle(80.w, '晓', 32.sp), + SizedBox(height: 20.h), + Text('移动精灵', + style: TextStyle( + fontSize: 20.sp, + fontWeight: FontWeight.bold, + color: isDark ? Colors.white : _textPrimary)), SizedBox(height: 8.h), - Text( - '按住麦克风按钮开始语音对话', - style: TextStyle( - fontSize: 13.sp, - color: isDark ? Colors.grey[400] : Colors.grey[500], - ), - ), + Text('点击右下角按钮，即可与晓晓对话', + style: TextStyle( + fontSize: 13.sp, + color: isDark ? Colors.grey[500] : _textSecondary)), ], ), ); } + // ── 消息气泡 ────────────────────────────────────────────────────────────── + Widget _buildBubble(MobileElfMessage msg, bool isDark) { final isUser = msg.isUser; return Padding( - padding: EdgeInsets.only(bottom: 12.h), - child: Row( - mainAxisAlignment: - isUser ? MainAxisAlignment.end : MainAxisAlignment.start, - crossAxisAlignment: CrossAxisAlignment.end, + padding: EdgeInsets.only(bottom: 14.h), + child: Column( + crossAxisAlignment: + isUser ? CrossAxisAlignment.end : CrossAxisAlignment.start, children: [ + // AI 头像 + 名字 if (!isUser) ...[ - _buildAvatar(isUser: false), - SizedBox(width: 8.w), + Row(children: [ + _gradientCircle(28.w, '晓', 11.sp), + SizedBox(width: 6.w), + Text('晓晓', + style: TextStyle(fontSize: 11.sp, color: _textSecondary)), + ]), + SizedBox(height: 4.h), ], - Flexible( - child: Container( - padding: - EdgeInsets.symmetric(horizontal: 14.w, vertical: 10.h), - decoration: BoxDecoration( - color: isUser - ? const Color(0xFF0091EA) - : (isDark - ? const Color(0xFF2A2A2A) - : Colors.white), - borderRadius: BorderRadius.only( - topLeft: Radius.circular(16.r), - topRight: Radius.circular(16.r), - bottomLeft: - isUser ? Radius.circular(16.r) : Radius.circular(4.r), - bottomRight: - isUser ? Radius.circular(4.r) : Radius.circular(16.r), - ), - boxShadow: [ - BoxShadow( - color: Colors.black.withOpacity(0.06), - blurRadius: 6, - offset: const Offset(0, 2), - ), - ], - ), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Text( - msg.text, - style: TextStyle( - fontSize: 14.sp, - color: isUser - ? Colors.white - : (isDark ? Colors.white : Colors.grey[800]), - height: 1.4, + // 气泡体 + Row( + mainAxisAlignment: + isUser ? MainAxisAlignment.end : MainAxisAlignment.start, + children: [ + if (!isUser) SizedBox(width: 34.w), + Flexible( + child: Container( + padding: + EdgeInsets.symmetric(horizontal: 14.w, vertical: 10.h), + decoration: BoxDecoration( + color: isUser + ? _blue + : (isDark ? const Color(0xFF2A2A2A) : _surfaceLight), + borderRadius: BorderRadius.only( + topLeft: Radius.circular(18.r), + topRight: Radius.circular(18.r), + bottomLeft: Radius.circular(isUser ? 18.r : 4.r), + bottomRight: Radius.circular(isUser ? 4.r : 18.r), ), - ), - if (!isUser && msg.ttsUrl != null) ...[ - SizedBox(height: 6.h), - GestureDetector( - onTap: () => controller.playTts(msg.ttsUrl!), - child: Row( - mainAxisSize: MainAxisSize.min, - children: [ - Icon( - Icons.play_circle_outline, - size: 16.sp, - color: const Color(0xFF0091EA), - ), - SizedBox(width: 4.w), - Text( - '重播语音', - style: TextStyle( - fontSize: 11.sp, - color: const Color(0xFF0091EA), - ), - ), - ], + boxShadow: [ + BoxShadow( + color: Colors.black.withValues(alpha: 0.05), + blurRadius: 6, + offset: const Offset(0, 2), ), - ), - ], - ], + ], + ), + child: Text(msg.text, + style: TextStyle( + fontSize: 15.sp, + height: 1.55, + color: isUser + ? Colors.white + : (isDark ? Colors.white : _textPrimary))), + ), ), - ), + ], + ), + // 时间 + SizedBox(height: 3.h), + Padding( + padding: EdgeInsets.only(left: isUser ? 0 : 34.w), + child: Text(_fmtTime(msg.time), + style: TextStyle( + fontSize: 11.sp, color: const Color(0xFFD0D5DD))), ), - if (isUser) ...[ - SizedBox(width: 8.w), - _buildAvatar(isUser: true), - ], ], ), ); } - Widget _buildAvatar({required bool isUser}) { - return Container( - width: 32.w, - height: 32.w, - decoration: BoxDecoration( - color: isUser - ? const Color(0xFF0091EA).withOpacity(0.15) - : const Color(0xFF00BFA5).withOpacity(0.15), - shape: BoxShape.circle, - ), - child: Icon( - isUser ? Icons.person : Icons.cell_tower, - size: 16.sp, - color: isUser ? const Color(0xFF0091EA) : const Color(0xFF00BFA5), - ), + Widget _buildRealtimeBubble(String text) { + return Padding( + padding: EdgeInsets.only(bottom: 14.h), + child: Row(mainAxisAlignment: MainAxisAlignment.end, children: [ + Flexible( + child: Container( + padding: EdgeInsets.symmetric(horizontal: 14.w, vertical: 10.h), + decoration: BoxDecoration( + color: _blue.withValues(alpha: 0.6), + borderRadius: BorderRadius.circular(18.r), + ), + child: Text(text, + style: TextStyle( + fontSize: 15.sp, color: Colors.white, height: 1.55)), + ), + ), + ]), ); } + // ══════════════════════════════════════════════════════════════════════════ + // 底部控制栏 + // ══════════════════════════════════════════════════════════════════════════ + Widget _buildBottomBar(bool isDark) { - return Container( - padding: EdgeInsets.fromLTRB(16.w, 8.h, 16.w, 24.h), - decoration: BoxDecoration( - color: isDark ? const Color(0xFF1A1A1A) : Colors.white, - boxShadow: [ - BoxShadow( - color: Colors.black.withOpacity(0.05), - blurRadius: 8, - offset: const Offset(0, -2), + return Obx(() { + final inCall = controller.inCall.value; + final s = controller.status.value; + final ptt = controller.pttRecording.value; + + // 状态栏显示条件： + // - 通话模式：recording/thinking/speaking/connecting 都显示 + // - 非通话非PTT：thinking/speaking 显示 + // - PTT 中：不显示（状态在中间按钮体现） + final showStatusBar = !ptt && ( + (inCall && s != MobileElfStatus.idle && s != MobileElfStatus.error) || + (!inCall && (s == MobileElfStatus.speaking || s == MobileElfStatus.thinking)) + ); + + return Container( + width: double.infinity, + decoration: BoxDecoration( + color: isDark ? const Color(0xFF1A1A1A) : _surfaceLight, + border: Border(top: BorderSide(color: isDark ? const Color(0xFF222222) : _border)), + boxShadow: [ + BoxShadow(color: Colors.black.withValues(alpha: 0.05), blurRadius: 12, offset: const Offset(0, -2)), + ], + ), + child: SafeArea( + top: false, + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + if (showStatusBar) _buildStatusBar(s, inCall, isDark), + if (inCall) + _buildInCallInputBar(isDark) + else + _buildNormalInputBar(isDark), + ], ), - ], - ), - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - // 实时识别字幕 - Obx(() { - final text = controller.realtimeText.value; - if (text.isEmpty) return const SizedBox.shrink(); - return Padding( - padding: EdgeInsets.only(bottom: 8.h), - child: Text( - text, - style: TextStyle( - fontSize: 13.sp, - color: isDark ? Colors.grey[400] : Colors.grey[500], - fontStyle: FontStyle.italic, + ), + ); + }); + } + + // ── 状态提示栏 ──────────────────────────────────────────────────────────── + + Widget _buildStatusBar(MobileElfStatus s, bool inCall, bool isDark) { + Widget? content; + + switch (s) { + case MobileElfStatus.speaking: + content = GestureDetector( + onTap: controller.interruptSpeaking, + behavior: HitTestBehavior.opaque, + child: Container( + padding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 8.h), + color: const Color(0xFFECFDF3), + child: Row( + children: [ + _MiniWave(color: _greenDark), + SizedBox(width: 8.w), + Expanded(child: Text('晓晓正在朗读...', style: TextStyle(fontSize: 13.sp, color: _greenDark, fontWeight: FontWeight.w500))), + Container( + padding: EdgeInsets.symmetric(horizontal: 12.w, vertical: 4.h), + decoration: BoxDecoration( + border: Border.all(color: _red), + borderRadius: BorderRadius.circular(16.r), + ), + child: Text('× 打断', style: TextStyle(fontSize: 12.sp, color: _red, fontWeight: FontWeight.w500)), ), - textAlign: TextAlign.center, - maxLines: 2, - overflow: TextOverflow.ellipsis, - ), - ); - }), - - // 错误提示 - Obx(() { - final err = controller.errorMsg.value; - if (err.isEmpty) return const SizedBox.shrink(); - return Padding( - padding: EdgeInsets.only(bottom: 8.h), - child: Text( - err, - style: TextStyle( - fontSize: 12.sp, color: Colors.red[400]), - textAlign: TextAlign.center, - ), - ); - }), + ], + ), + ), + ); + break; + case MobileElfStatus.recording: + content = Container( + padding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 8.h), + color: const Color(0xFFECFDF3), + child: Row( + children: [ + _MiniWave(color: _greenDark), + SizedBox(width: 8.w), + Expanded(child: Obx(() { + final rt = controller.realtimeText.value; + return Text(rt.isNotEmpty ? rt : '聆听中...', style: TextStyle(fontSize: 13.sp, color: _greenDark), maxLines: 1, overflow: TextOverflow.ellipsis); + })), + ], + ), + ); + break; + case MobileElfStatus.connecting: + content = Container( + padding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 8.h), + color: const Color(0xFFFFFAEB), + child: Row( + children: [ + SizedBox(width: 14.w, height: 14.w, child: CircularProgressIndicator(strokeWidth: 2, color: _orange)), + SizedBox(width: 8.w), + Text('正在连接...', style: TextStyle(fontSize: 13.sp, color: const Color(0xFFB54708))), + ], + ), + ); + break; + case MobileElfStatus.thinking: + content = Container( + padding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 8.h), + color: const Color(0xFFEFF8FF), + child: Row( + children: [ + SizedBox(width: 14.w, height: 14.w, child: CircularProgressIndicator(strokeWidth: 2, color: const Color(0xFF175CD3))), + SizedBox(width: 8.w), + Text('AI 思考中...', style: TextStyle(fontSize: 13.sp, color: const Color(0xFF175CD3))), + ], + ), + ); + break; + default: + return const SizedBox.shrink(); + } + return content; + } + + // ── 通话模式底栏 ────────────────────────────────────────────────────────── - // 麦克风按钮 - Obx(() => _buildMicButton(controller.status.value)), + Widget _buildInCallInputBar(bool isDark) { + return Padding( + padding: EdgeInsets.fromLTRB(20.w, 12.h, 20.w, 14.h), + child: Row( + children: [ + // 通话状态 + 计时 + Expanded( + child: Obx(() { + final s = controller.status.value; + final idle = s == MobileElfStatus.idle; + return Row( + children: [ + Container( + width: 8.w, height: 8.w, + decoration: BoxDecoration(shape: BoxShape.circle, color: idle ? _green : _orange), + ), + SizedBox(width: 6.w), + Text('通话中', style: TextStyle(fontSize: 13.sp, color: isDark ? Colors.white70 : _textSecondary)), + SizedBox(width: 4.w), + Obx(() => Text(_fmtDuration(controller.callSeconds.value), + style: TextStyle(fontSize: 13.sp, color: isDark ? Colors.white70 : _textSecondary))), + ], + ); + }), + ), + // 音量指示器 + Obx(() => _VolumeIndicator( + rms: controller.micRms.value, + alive: controller.micAlive.value, + isDark: isDark, + )), + SizedBox(width: 12.w), + // 挂断按钮 + GestureDetector( + onTap: controller.disconnect, + child: Container( + width: 52.w, height: 52.w, + decoration: BoxDecoration( + shape: BoxShape.circle, + color: _red, + boxShadow: [BoxShadow(color: _red.withValues(alpha: 0.3), blurRadius: 14, offset: const Offset(0, 4))], + ), + child: Icon(Icons.call_end_rounded, color: Colors.white, size: 26.sp), + ), + ), ], ), ); } - Widget _buildMicButton(MobileElfStatus status) { - final bool isThinking = status == MobileElfStatus.thinking || - status == MobileElfStatus.connecting; - final Color btnColor = _statusColor(status); - final IconData icon = _micIcon(status); - final String label = _micLabel(status); - - return Column( - children: [ - GestureDetector( - onTap: isThinking ? null : controller.onMicTap, - child: AnimatedContainer( - duration: const Duration(milliseconds: 200), - width: 68.w, - height: 68.w, - decoration: BoxDecoration( - shape: BoxShape.circle, - color: isThinking - ? Colors.grey.withOpacity(0.2) - : btnColor.withOpacity(0.15), - border: Border.all( - color: isThinking ? Colors.grey.withOpacity(0.3) : btnColor, - width: 2, - ), - ), - child: isThinking - ? Center( - child: SizedBox( - width: 24.w, - height: 24.w, - child: CircularProgressIndicator( - strokeWidth: 2.5, color: Colors.grey), + // ── 普通输入栏 (按住说话) ────────────────────────────────────────────────── + + Widget _buildNormalInputBar(bool isDark) { + return Obx(() { + final s = controller.status.value; + final ptt = controller.pttRecording.value; + + final disabled = !ptt && s != MobileElfStatus.idle && s != MobileElfStatus.error; + final pttClickable = s == MobileElfStatus.speaking; + final sideDisabled = ptt || (disabled && !pttClickable); + + return Padding( + padding: EdgeInsets.fromLTRB(12.w, 6.h, 12.w, 14.h), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + // ── PTT 实时识别文字（居中 + 渐变背景） ── + if (ptt) + Obx(() { + final rt = controller.realtimeText.value; + if (rt.isEmpty) return const SizedBox.shrink(); + return Container( + width: double.infinity, + padding: EdgeInsets.symmetric(horizontal: 20.w, vertical: 12.h), + margin: EdgeInsets.only(bottom: 8.h), + decoration: BoxDecoration( + gradient: LinearGradient( + begin: Alignment.topLeft, + end: Alignment.bottomRight, + colors: isDark + ? [const Color(0xFF2A2A2A), const Color(0xFF1A2332)] + : [const Color(0xFFF0F4FF), const Color(0xFFE8F5E9)], ), - ) - : Icon(icon, size: 30.sp, color: btnColor), + borderRadius: BorderRadius.circular(16.r), + border: Border.all( + color: isDark ? const Color(0xFF334155) : const Color(0xFFE2E8F0), + width: 0.5, + ), + ), + child: Text(rt, + style: TextStyle(fontSize: 15.sp, color: isDark ? Colors.white : _textPrimary, height: 1.4), + textAlign: TextAlign.center, + maxLines: 3, overflow: TextOverflow.ellipsis, + ), + ); + }), + // ── 按钮行 ── + Row( + children: [ + // 中间：PTT 按钮 + Expanded( + child: ptt + ? _buildPttRecordingCenter(isDark) + : IgnorePointer( + ignoring: disabled && !pttClickable, + child: Opacity( + opacity: (disabled && !pttClickable) ? 0.5 : 1.0, + child: _buildPttIdleButton(isDark), + ), + ), + ), + SizedBox(width: 8.w), + // 右侧：电话按钮 + IgnorePointer( + ignoring: sideDisabled, + child: Opacity( + opacity: sideDisabled ? 0.4 : 1.0, + child: _roundButton( + onTap: controller.startCall, + gradient: const LinearGradient( + colors: [Color(0xFF34C759), Color(0xFF28A745)], + begin: Alignment.topLeft, end: Alignment.bottomRight, + ), + shadow: _green.withValues(alpha: 0.4), + size: 44, + child: Icon(Icons.phone, color: Colors.white, size: 20.sp), + ), + ), + ), + ], + ), + ], + ), + ); + }); + } + + /// 按住说话按钮（空闲状态）— 参考微信风格 + Widget _buildPttIdleButton(bool isDark) { + return Listener( + onPointerDown: (_) => controller.startPtt(), + child: Container( + height: 42.h, + decoration: BoxDecoration( + color: isDark ? const Color(0xFF2A2A2A) : Colors.white, + borderRadius: BorderRadius.circular(21.r), + border: Border.all(color: isDark ? const Color(0xFF444444) : const Color(0xFFD0D5DD), width: 1), + ), + child: Center( + child: Text('按住 说话', + style: TextStyle( + fontSize: 15.sp, + color: isDark ? Colors.white70 : _textPrimary, + fontWeight: FontWeight.w600, + letterSpacing: 2.w, + ), ), ), - SizedBox(height: 6.h), - Text( - label, - style: TextStyle( - fontSize: 11.sp, - color: isThinking ? Colors.grey : btnColor, + ), + ); + } + + /// PTT 录音中：仅中间部分 + /// 连接中=与按住说话相同样式，录音中=橙色 + Widget _buildPttRecordingCenter(bool isDark) { + return Obx(() { + final s = controller.status.value; + final isConnecting = s == MobileElfStatus.connecting; + + if (isConnecting) { + // 连接中：与按住说话空闲状态一样的外观 + 加载指示 + return Container( + height: 42.h, + decoration: BoxDecoration( + color: isDark ? const Color(0xFF2A2A2A) : Colors.white, + borderRadius: BorderRadius.circular(21.r), + border: Border.all(color: isDark ? const Color(0xFF444444) : const Color(0xFFD0D5DD), width: 1), + ), + child: Row( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + SizedBox(width: 14.w, height: 14.w, + child: CircularProgressIndicator(strokeWidth: 2, color: isDark ? Colors.white70 : _textSecondary)), + SizedBox(width: 8.w), + Text('连接中...', style: TextStyle(fontSize: 14.sp, color: isDark ? Colors.white70 : _textSecondary, fontWeight: FontWeight.w500)), + ], ), + ); + } + + // 录音中：橙色 + return Container( + height: 42.h, + decoration: BoxDecoration( + gradient: const LinearGradient(colors: [Color(0xFFF79009), Color(0xFFDC6803)]), + borderRadius: BorderRadius.circular(21.r), + boxShadow: [BoxShadow( + color: _orange.withValues(alpha: 0.3), + blurRadius: 8, offset: const Offset(0, 2), + )], ), - ], + child: Row( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + const _MiniWave(color: Colors.white), + SizedBox(width: 8.w), + Obx(() => Text('松开发送 · ${controller.pttSeconds.value}s', + style: TextStyle(fontSize: 14.sp, color: Colors.white, fontWeight: FontWeight.w500))), + ], + ), + ); + }); + } + + // ── 通用圆形按钮 ────────────────────────────────────────────────────────── + + Widget _roundButton({ + required VoidCallback onTap, + required Widget child, + Color? color, + Gradient? gradient, + Color? shadow, + Color? border, + double size = 56, + }) { + return GestureDetector( + onTap: onTap, + child: Container( + width: size.w, + height: size.w, + decoration: BoxDecoration( + shape: BoxShape.circle, + color: gradient == null ? color : null, + gradient: gradient, + border: border != null ? Border.all(color: border, width: 2) : null, + boxShadow: shadow != null + ? [BoxShadow(color: shadow, blurRadius: 14, offset: const Offset(0, 4))] + : null, + ), + child: Center(child: child), + ), ); } - // ── 辅助 ────────────────────────────────────────────────────────────────── + // ══════════════════════════════════════════════════════════════════════════ + // 工具方法 + // ══════════════════════════════════════════════════════════════════════════ - Color _statusColor(MobileElfStatus s) { - switch (s) { - case MobileElfStatus.connecting: - return Colors.orange; - case MobileElfStatus.recording: - return const Color(0xFF0066CC); - case MobileElfStatus.thinking: - return Colors.orange; - case MobileElfStatus.speaking: - return const Color(0xFF4CAF50); - case MobileElfStatus.error: - return Colors.red; - case MobileElfStatus.idle: - return const Color(0xFF0091EA); - } + /// 渐变圆形头像 + Widget _gradientCircle(double size, String text, double fontSize) { + return Container( + width: size, + height: size, + decoration: const BoxDecoration( + shape: BoxShape.circle, + gradient: _avatarGradient, + ), + child: Center( + child: Text(text, + style: TextStyle( + color: Colors.white, + fontSize: fontSize, + fontWeight: FontWeight.w600)), + ), + ); } - String _statusLabel(MobileElfStatus s) { - switch (s) { - case MobileElfStatus.connecting: - return '连接中'; - case MobileElfStatus.recording: - return '录音中'; - case MobileElfStatus.thinking: - return '思考中'; - case MobileElfStatus.speaking: - return '播报中'; - case MobileElfStatus.error: - return '错误'; - case MobileElfStatus.idle: - return '就绪'; - } + String _fmtTime(DateTime t) => + '${t.hour.toString().padLeft(2, '0')}:${t.minute.toString().padLeft(2, '0')}'; + + String _fmtDuration(int secs) { + final m = (secs ~/ 60).toString().padLeft(2, '0'); + final s = (secs % 60).toString().padLeft(2, '0'); + return '$m:$s'; } +} - String _statusBannerLabel(MobileElfStatus s) { - switch (s) { - case MobileElfStatus.connecting: - return '正在连接移动精灵...'; - case MobileElfStatus.recording: - return '正在聆听，请说话'; - case MobileElfStatus.thinking: - return 'AI 思考中，请稍候'; - case MobileElfStatus.speaking: - return '正在播报回答'; - case MobileElfStatus.error: - return controller.errorMsg.value; - case MobileElfStatus.idle: - return ''; - } +// ═══════════════════════════════════════════════════════════════════════════ +// 辅助类 +// ═══════════════════════════════════════════════════════════════════════════ + +class _PillCfg { + final Color bg, fg; + final String label; + const _PillCfg(this.bg, this.fg, this.label); +} + +/// 3 条小波浪动画 +class _MiniWave extends StatefulWidget { + final Color color; + const _MiniWave({required this.color}); + + @override + State<_MiniWave> createState() => _MiniWaveState(); +} + +class _MiniWaveState extends State<_MiniWave> + with SingleTickerProviderStateMixin { + late final AnimationController _ctrl; + + @override + void initState() { + super.initState(); + _ctrl = AnimationController( + vsync: this, duration: const Duration(milliseconds: 800)) + ..repeat(); } - IconData _micIcon(MobileElfStatus s) { - switch (s) { - case MobileElfStatus.recording: - return Icons.stop_circle_outlined; - case MobileElfStatus.speaking: - return Icons.stop_circle_outlined; - case MobileElfStatus.error: - return Icons.refresh; - default: - return Icons.mic; - } + @override + void dispose() { + _ctrl.dispose(); + super.dispose(); } - String _micLabel(MobileElfStatus s) { - switch (s) { - case MobileElfStatus.recording: - return '点击停止'; - case MobileElfStatus.speaking: - return '点击停止播报'; - case MobileElfStatus.error: - return '重试'; - case MobileElfStatus.thinking: - return '思考中...'; - case MobileElfStatus.connecting: - return '连接中...'; - default: - return '点击说话'; - } + @override + Widget build(BuildContext context) { + return AnimatedBuilder( + animation: _ctrl, + builder: (_, __) => Row( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.center, + children: List.generate(3, (i) { + final phase = (_ctrl.value + i * 0.2) % 1.0; + final h = 3.0 + 7.0 * (0.5 - (phase - 0.5).abs()) * 2; + return Container( + width: 2.5, + height: h.clamp(3.0, 10.0), + margin: const EdgeInsets.symmetric(horizontal: 1), + decoration: BoxDecoration( + color: widget.color, + borderRadius: BorderRadius.circular(2), + ), + ); + }), + ), + ); + } +} + +/// 麦克风音量指示器（通话模式用） +class _VolumeIndicator extends StatelessWidget { + final double rms; + final bool alive; + final bool isDark; + const _VolumeIndicator({required this.rms, required this.alive, required this.isDark}); + + @override + Widget build(BuildContext context) { + // rms 范围大约 0~6000，归一化到 0~1 + final level = alive ? (rms / 5000).clamp(0.0, 1.0) : 0.0; + const barCount = 5; + return Container( + width: 44.w, height: 44.w, + decoration: BoxDecoration( + shape: BoxShape.circle, + color: isDark ? const Color(0xFF2A2A2A) : const Color(0xFFF2F4F7), + ), + child: Row( + mainAxisAlignment: MainAxisAlignment.center, + crossAxisAlignment: CrossAxisAlignment.center, + children: List.generate(barCount, (i) { + final threshold = (i + 1) / barCount; + final active = level >= threshold; + return Container( + width: 3.w, + height: (8 + i * 4).h, + margin: EdgeInsets.symmetric(horizontal: 1.w), + decoration: BoxDecoration( + color: active + ? const Color(0xFF34C759) + : (isDark ? const Color(0xFF444444) : const Color(0xFFD0D5DD)), + borderRadius: BorderRadius.circular(1.5.r), + ), + ); + }), + ), + ); } } diff --git a/test_mobile_elf.py b/test_mobile_elf.py new file mode 100644 index 000000000..77d4449c4 --- /dev/null +++ b/test_mobile_elf.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +import ssl, time, hashlib, json, websocket + +DEVICE_NO = '199839C0AAAE99653D2E' +DEVICE_ID = '2032013791868043266' +PRODUCT_ID = '2031271967792472065' +PRODUCT_KEY = 'hotepZZwvouWQjHz' +DEVICE_SECRET = 'ijdthZGQELhkmigf' + +ts = str(int(time.time() * 1000)) +sign = hashlib.md5((DEVICE_SECRET + ts).encode()).hexdigest() +url = (f'wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr' + f'?deviceNo={DEVICE_NO}&deviceId={DEVICE_ID}' + f'&productId={PRODUCT_ID}&productKey={PRODUCT_KEY}' + f'&sign={sign}&sn=py-test&ts={ts}') + +print(f'connecting...\n{url}') +ws = websocket.WebSocket(sslopt={'cert_reqs': ssl.CERT_NONE}) +ws.settimeout(15) +ws.connect(url) +print('connected! sending start frame immediately...') + +import uuid +start_frame = json.dumps({ + 'type': 'start', + 'data': { + 'cuid': DEVICE_ID, + 'format': 'pcm', + 'sample': 16000, + 'support_dcs': 2, + 'chunk_size': 10240, + 'support_tts': True, + 'support_text2dcs': True, + 'user_agent': 'voitrans/1.0', + 'dialog_request_id': str(uuid.uuid4()), + 'access_rc': True, + 'support_part_tts': True, + 'need_stoplisten': True, + 'need_dialogue_finish': True, + 'result_trans2directive': False, + 'rc_version': '5', + 'messages': [], + 'client_context': [], + } +}) +ws.send(start_frame) +print('start frame sent, waiting for ready...') + +# 等待 ready +while True: + msg = ws.recv() + print('received:', msg) + data = json.loads(msg) + if data.get('type') == 'ready': + print('--- ready received, sending text "你好" ---') + break + +# 发送文本 +text_frame = json.dumps({'type': 'text', 'data': {'text': '你好'}}) +ws.send(text_frame) +print('text frame sent') + +# 发送 finish +finish_frame = json.dumps({'type': 'finish'}) +ws.send(finish_frame) +print('finish frame sent, waiting for AI response...') + +# 接收所有回包直到 end=1 +while True: + msg = ws.recv() + print('received:', msg) + try: + data = json.loads(msg) + if data.get('end') == 1: + print('--- session end (end=1), closing ---') + break + except Exception: + pass + +ws.close() +print('done.') diff --git a/ui/yidong.html b/ui/yidong.html new file mode 100644 index 000000000..c8be59f76 --- /dev/null +++ b/ui/yidong.html @@ -0,0 +1,563 @@ + + + + + +移动精灵 UI 预览 + + + + + +
+

① 文本输入模式

+
+
+ ‹ +
晓
+ 移动精灵 + 空闲 +
+
+
+
晓
+
移动精灵
+
点击右下角按钮，即可与晓晓对话
+
+
+
+
+ +
+ + +
+ +
+
+
+
+ + +
+

② 语音模式 - 按住说话

+
+
+ ‹ +
晓
+ 移动精灵 + 空闲 +
+
+
+
晓
晓晓
+
你好！我是晓晓，有什么可以帮你的吗？
+ 14:23 +
+
+
+
+ +
按住 说话
+ +
+
+
+
+ + +
+

③ 按住说话 - 连接中（同空闲样式）

+
+
+ ‹ +
晓
+ 移动精灵 + 连接中… +
+
+
+
晓
晓晓
+
你好！我是晓晓，有什么可以帮你的吗？
+ 14:23 +
+
+
+
+ +
+ + 连接中... +
+ +
+
+
+
+ + +
+

④ 按住说话 - 录音中（橙色）

+
+
+ ‹ +
晓
+ 移动精灵 + 识别中 +
+
+
+
晓
晓晓
+
你好！我是晓晓，有什么可以帮你的吗？
+ 14:23 +
+
+
+ +
今天天气怎么样？
+
+ +
+
+ 松开发送 · 2s +
+ +
+
+
+
+ + +
+

⑤ AI 思考中（输入栏禁用）

+
+
+ ‹ +
晓
+ 移动精灵 + 处理中 +
+
+
+
晓
晓晓
+
你好！我是晓晓，有什么可以帮你的吗？
+ 14:23 +
+
+
今天天气怎么样？
+ 14:24 +
+
+
+
+ + AI 思考中... +
+
+ +
按住 说话
+ +
+
+
+
+ + +
+

⑥ AI 播报中（可打断）

+
+
+ ‹ +
晓
+ 移动精灵 + 朗读中 +
+
+
+
今天天气怎么样？
+ 14:24 +
+
+
晓
晓晓
+
今天晴天，气温 18°C，微风，空气质量优，很适合出行哦
+ 14:24 +
+
+
+
+
+ 晓晓正在朗读... + +
+
+ +
按住 说话
+ +
+
+
+
+ + +
+

⑦ 通话模式 - 等待说话

+
+
+ ‹ +
晓
+ 移动精灵 + 通话中 01:23 +
+
+
+
今天天气怎么样？
+ 14:24 +
+
+
晓
晓晓
+
今天晴天，气温 18°C，微风，空气质量优
+ 14:24 +
+
+
+
+
+
+ 通话中 + 01:23 +
+ +
+
+
+
+
+
+
+ + +
+
+
+
+ + +
+

⑧ 通话模式 - AI 播报

+
+
+ ‹ +
晓
+ 移动精灵 + 朗读中 01:48 +
+
+
+
Flutter 有什么优势？
+ 14:41 +
+
+
晓
晓晓
+
Flutter 的核心优势在于跨平台能力强、渲染性能高，以及 Dart 语言的高效开发体验...
+ 14:41 +
+
+
+
+
+ 晓晓正在朗读... + +
+
+
+
+ 通话中 + 01:48 +
+
+
+
+
+
+
+
+ +
+
+
+
+ + + From 6c4e9e6e8b396531039b51577eb63c38dec47851 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Thu, 19 Mar 2026 18:17:58 +0800 Subject: [PATCH 03/14] =?UTF-8?q?=E7=A7=BB=E5=8A=A8=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E7=9A=84=E5=A4=A7=E6=A8=A1=E5=9E=8B=E6=8E=A5=E5=85=A5=E6=88=90?= =?UTF-8?q?=E5=8A=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/data/services/ble_manager.dart | 36 +- .../home/controllers/home_controller.dart | 9 +- lib/modules/home/views/home_view.dart | 23 - .../controllers/mobile_elf_controller.dart | 1076 +++-------------- .../mobile_elf/views/mobile_elf_view.dart | 199 ++- .../agent_service/AgentService.kt | 70 +- .../agent_service/AgentServicePlugin.kt | 19 + .../agent_service/BleMediaService.kt | 38 +- .../yunqiinnovation/agent_service/StsAgent.kt | 783 ++++++++++++ .../agent_service/AgentServiceImpl.swift | 72 +- .../Sources/agent_service/StsAgent.swift | 495 ++++++++ .../agent_service/lib/agent_service.dart | 22 + .../ble_service/lib/ble_service.dart | 2 +- 13 files changed, 1885 insertions(+), 959 deletions(-) create mode 100644 local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt create mode 100644 local_plugins/agent_service/ios/agent_service/Sources/agent_service/StsAgent.swift diff --git a/lib/data/services/ble_manager.dart b/lib/data/services/ble_manager.dart index a98989bdb..ddf05eedc 100644 --- a/lib/data/services/ble_manager.dart +++ b/lib/data/services/ble_manager.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +import 'package:flutter_dotenv/flutter_dotenv.dart'; import 'package:get_storage/get_storage.dart'; import 'package:get/get.dart'; import 'package:ble_service/ble_service.dart'; @@ -957,7 +958,27 @@ class BleManager extends GetxService { supportedLanguages = ['zh-CN', 'en-US']; } + // STS 模式参数（移动精灵） + final agentType = AppConfig.env('AGENT_TYPE') ?? 'yidong'; + final agentMode = (agentType == 'yidong') ? 'sts' : 'stt_llm_tts'; + final agentId = (agentType == 'yidong') + ? (AppConfig.env('MOBILE_ELF_AGENT_ID') ?? 'mobile_elf') + : 'default_agent'; + final stsProductId = AppConfig.env('MOBILE_ELF_PRODUCT_ID') ?? ''; + final stsProductKey = AppConfig.env('MOBILE_ELF_PRODUCT_KEY') ?? ''; + final stsDeviceNo = dotenv.env['MOBILE_ELF_DEVICE_NO'] ?? ''; + final stsDeviceId = _storage.read('sts_device_id') ?? ''; + final stsDeviceSecret = _storage.read('sts_device_secret') ?? ''; + final config = { + // STS 模式路由 + 'agentMode': agentMode, + 'agentId': agentId, + 'stsProductId': stsProductId, + 'stsProductKey': stsProductKey, + 'stsDeviceNo': stsDeviceNo, + 'stsDeviceId': stsDeviceId, + 'stsDeviceSecret': stsDeviceSecret, // 语音代理配置参数 'azureSpeechKey': azureSpeechKey, 'azureSpeechRegion': azureSpeechRegion, @@ -990,7 +1011,7 @@ class BleManager extends GetxService { Logger.warning('XUNFEI_ASR_* 未完整配置，ASR 将不会切到讯飞'); } - if (azureSpeechKey.isEmpty || openaiApiKey.isEmpty) { + if (agentMode != 'sts' && (azureSpeechKey.isEmpty || openaiApiKey.isEmpty)) { Logger.warning('缺少必要的API密钥，请检查.env文件配置'); // 检查.env中相关密钥是否存在 @@ -1002,6 +1023,19 @@ class BleManager extends GetxService { } } + // STS 模式：监听凭证获取事件并缓存到 GetStorage + if (agentMode == 'sts') { + AgentService.events.where((e) => e.type == AgentServiceEventType.stsCredentials).listen((e) { + final deviceId = e.data['deviceId']?.toString() ?? ''; + final deviceSecret = e.data['deviceSecret']?.toString() ?? ''; + final deviceNo = e.data['deviceNo']?.toString() ?? ''; + if (deviceId.isNotEmpty) _storage.write('sts_device_id', deviceId); + if (deviceSecret.isNotEmpty) _storage.write('sts_device_secret', deviceSecret); + if (deviceNo.isNotEmpty) _storage.write('sts_device_no', deviceNo); + Logger.info('STS 凭证已缓存: deviceId=$deviceId deviceNo=$deviceNo'); + }); + } + Logger.info('lxm--准备调用AgentService.startAgentService启动BLE服务'); final result = await AgentService.startAgentService(config); Logger.info('lxm--AgentService.startAgentService调用结果: $result'); diff --git a/lib/modules/home/controllers/home_controller.dart b/lib/modules/home/controllers/home_controller.dart index 0dede3002..7df97989b 100644 --- a/lib/modules/home/controllers/home_controller.dart +++ b/lib/modules/home/controllers/home_controller.dart @@ -256,8 +256,13 @@ class HomeController extends GetxController { } void exploreAIFeatures() { - Get.toNamed(Routes.agent); - AgentService.interruptCurrentResponse(); + final agentType = AppConfig.env('AGENT_TYPE') ?? 'yidong'; + if (agentType == 'yidong') { + Get.toNamed(Routes.mobileElf); + } else { + Get.toNamed(Routes.agent); + AgentService.interruptCurrentResponse(); + } } Future _checkConnectionStatus() async { diff --git a/lib/modules/home/views/home_view.dart b/lib/modules/home/views/home_view.dart index b9f7d8eeb..cc2c15243 100644 --- a/lib/modules/home/views/home_view.dart +++ b/lib/modules/home/views/home_view.dart @@ -146,8 +146,6 @@ class HomeView extends GetView { SizedBox(height: 16.h), _buildPremiumAILayout(), - SizedBox(height: 16.h), - _buildMobileElfCard(isDarkMode), ], ), ), @@ -902,27 +900,6 @@ class HomeView extends GetView { ); } - // 移动精灵卡片 - Widget _buildMobileElfCard(bool isDarkMode) { - return _buildFeatureCard( - title: '移动精灵', - description: '智能语音助手，移动AI全场景对话', - icon: Icons.cell_tower, - gradient: LinearGradient( - begin: Alignment.topLeft, - end: Alignment.bottomRight, - colors: [ - const Color(0xFF0091EA).withOpacity(0.3), - const Color(0xFF00BFA5).withOpacity(0.3), - ], - ), - onTap: () => Get.toNamed(Routes.mobileElf), - height: 70.h, - isDarkMode: isDarkMode, - iconColor: const Color(0xFF0091EA), - ); - } - // 特性卡片构建方法 Widget _buildFeatureCard({ required String title, diff --git a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart index cfb1e2802..a673cf443 100644 --- a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart +++ b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart @@ -1,21 +1,9 @@ import 'dart:async'; -import 'dart:convert'; -import 'dart:io'; -import 'dart:math' as math; -import 'dart:typed_data'; -import 'package:crypto/crypto.dart'; -import 'package:dio/dio.dart'; -import 'package:flutter/foundation.dart' show kIsWeb; +import 'package:agent_service/agent_service.dart'; +import 'package:chat_storage/chat_storage.dart'; import 'package:flutter/widgets.dart'; -import 'package:audio_session/audio_session.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; import 'package:get/get.dart'; -import 'package:get_storage/get_storage.dart'; -import 'package:just_audio/just_audio.dart'; -import 'package:record/record.dart'; -import 'package:uuid/uuid.dart'; -import 'package:web_socket_channel/io.dart'; -import 'package:web_socket_channel/web_socket_channel.dart'; +import '../../../data/models/appconfig.dart'; import '../../../core/utils/logger.dart'; enum MobileElfStatus { idle, connecting, recording, thinking, speaking, error } @@ -24,11 +12,13 @@ class MobileElfMessage { final bool isUser; final String text; final DateTime time; - final String? ttsUrl; - MobileElfMessage({required this.isUser, required this.text, required this.time, this.ttsUrl}); + MobileElfMessage( + {required this.isUser, required this.text, required this.time}); } class MobileElfController extends GetxController { + static const _tag = 'MobileElfCtrl'; + // ── 可观察状态 ───────────────────────────────────────────────────────────── final status = MobileElfStatus.idle.obs; final messages = [].obs; @@ -37,944 +27,264 @@ class MobileElfController extends GetxController { final inCall = false.obs; final callSeconds = 0.obs; final recordingSeconds = 0.obs; - final micRms = 0.0.obs; // 实时麦克风音量（RMS），供 UI 显示 - final micAlive = false.obs; // 麦克风流是否存活 - final pttRecording = false.obs; // 按住说话录音中 - final pttSeconds = 0.obs; // 按住说话计时 + final micRms = 0.0.obs; + final micAlive = false.obs; + final pttRecording = false.obs; + final pttSeconds = 0.obs; + final ttsEnabled = true.obs; // 是否播报 AI 回复 final ScrollController scrollController = ScrollController(); - // ── WebSocket ────────────────────────────────────────────────────────────── - WebSocketChannel? _channel; - StreamSubscription? _wsSubscription; - Completer? _readyCompleter; - bool _sessionEnded = true; - String _dialogRequestId = ''; - int _currentAiMsgIndex = -1; - - // ── 录音 & 本地 VAD ──────────────────────────────────────────────────────── - final AudioRecorder _recorder = AudioRecorder(); - StreamSubscription? _micSubscription; - - // 音频发送缓冲（凑帧用） - final List _sendBuffer = []; - int _sendBufferOffset = 0; - static const int _frameSize = 5120; // 160ms @ 16kHz 16bit mono - - // 预缓存：滚动保留最近 ~1.5 秒音频，用于在 WS 连接前缓存用户已说的话 - final List _preBuffer = []; - static const int _preBufferMaxChunks = 75; // ~1.5s (按每 chunk 20ms 算) - - // 本地 VAD（仅在空闲等待状态工作，TTS 播放中不做 VAD） - bool _speechDetected = false; - static const int _vadThreshold = 800; - static const int _vadOnsetChunks = 3; // 连续 3 chunk (~60ms) → 检测到说话 - int _vadOnsetCount = 0; - - // 状态标记 - bool _forwardingAudio = false; // 是否正在转发音频到 WS - bool _wsConnecting = false; // WS 正在连接中（防止重复触发） - int _micChunkCount = 0; // 麦克风 chunk 计数（调试用） - - // ── 播放 & TTS 队列 ────────────────────────────────────────────────────── - // 关闭 just_audio 的音频会话管理，防止它与 record 争抢音频硬件： - // - handleInterruptions: false → 不自动暂停/恢复 - // - handleAudioSessionActivation: false → 不自动调用 session.setActive() - // - androidApplyAudioAttributes: false → 不监听 audio_session 配置变更 - final AudioPlayer _player = AudioPlayer( - handleInterruptions: false, - handleAudioSessionActivation: false, - androidApplyAudioAttributes: false, - ); - final List _ttsQueue = []; - bool _ttsPlaying = false; - bool _ttsInterrupted = false; - Completer? _ttsPlayCompleter; // 用于等待当前段播完或被打断 - - // ── 配置 ─────────────────────────────────────────────────────────────────── - late String _deviceNo, _deviceId, _productId, _productKey, _deviceSecret; - final credentialReady = false.obs; + // 凭证状态（原生层负责，Dart 层始终无错） + final credentialReady = true.obs; final credentialError = ''.obs; - final _storage = GetStorage(); - static const _kDeviceId = 'mobile_elf_device_id'; - static const _kDeviceSecret = 'mobile_elf_device_secret'; - static const _kDeviceNo = 'mobile_elf_device_no'; - final _uuid = const Uuid(); - // 计时器 + // ── 内部 ─────────────────────────────────────────────────────────────────── + StreamSubscription? _eventSub; Timer? _callTimer; - Timer? _recordingTimer; - Timer? _micWatchdog; - int _lastWatchdogChunkCount = 0; - Timer? _idleTimer; Timer? _pttTimer; - static const int _idleTimeoutSeconds = 15; // TTS 播完后等待说话超时秒数 + bool _inCallMode = false; // ═══════════════════════════════════════════════════════════════════════════ // 生命周期 // ═══════════════════════════════════════════════════════════════════════════ + String get _agentId => AppConfig.env('MOBILE_ELF_AGENT_ID') ?? 'mobile_elf'; + @override void onInit() { super.onInit(); - _loadConfig(); - _fetchA80CredentialsIfNeeded(); - } - - @override - void onClose() { - disconnect(); - _recorder.dispose(); - _player.dispose(); - scrollController.dispose(); - super.onClose(); - } - - void _loadConfig() { - _productId = dotenv.env['MOBILE_ELF_PRODUCT_ID'] ?? '1899007766144614402'; - _productKey = dotenv.env['MOBILE_ELF_PRODUCT_KEY'] ?? 'd0aI2yidbUUXS'; - _deviceNo = _storage.read(_kDeviceNo) ?? dotenv.env['MOBILE_ELF_DEVICE_NO'] ?? ''; - _deviceId = _storage.read(_kDeviceId) ?? ''; - _deviceSecret = _storage.read(_kDeviceSecret) ?? ''; - credentialReady.value = _deviceSecret.isNotEmpty; - } - - Future _fetchA80CredentialsIfNeeded() async { - try { - final deviceNo = dotenv.env['MOBILE_ELF_DEVICE_NO'] ?? ''; - if (deviceNo.isEmpty) return; - credentialError.value = ''; - final dio = Dio(BaseOptions( - baseUrl: 'https://ivs.chinamobiledevice.com:11443', - connectTimeout: const Duration(seconds: 10), - receiveTimeout: const Duration(seconds: 10), - )); - final resp = await dio.post('/v2/customer/device/secret/info', data: { - 'deviceNoType': 'SN', 'deviceNo': deviceNo, - 'productId': _productId, 'productKey': _productKey, - }); - final body = resp.data as Map; - if (body['code'].toString() == '200' && body['success'] == true) { - final data = body['data'] as Map; - _deviceNo = data['deviceNo'] as String? ?? deviceNo; - _deviceId = data['deviceId'] as String? ?? ''; - _deviceSecret = data['deviceSecret'] as String? ?? ''; - await _storage.write(_kDeviceNo, _deviceNo); - await _storage.write(_kDeviceId, _deviceId); - await _storage.write(_kDeviceSecret, _deviceSecret); - credentialReady.value = _deviceSecret.isNotEmpty; - } else { - credentialError.value = body['message']?.toString() ?? '凭证请求失败'; - } - } catch (e) { - credentialError.value = '凭证请求异常: $e'; - } - } - - Future retryCredentials() async { - credentialError.value = ''; - await _fetchA80CredentialsIfNeeded(); - } - - // ═══════════════════════════════════════════════════════════════════════════ - // 开始/结束通话 - // ═══════════════════════════════════════════════════════════════════════════ - - /// 配置音频会话：playAndRecord 模式，允许录音和播放同时进行 - Future _configureAudioSession() async { - try { - final session = await AudioSession.instance; - await session.configure(AudioSessionConfiguration( - avAudioSessionCategory: AVAudioSessionCategory.playAndRecord, - avAudioSessionCategoryOptions: AVAudioSessionCategoryOptions.defaultToSpeaker | - AVAudioSessionCategoryOptions.allowBluetooth, - avAudioSessionMode: AVAudioSessionMode.voiceChat, - androidAudioAttributes: const AndroidAudioAttributes( - contentType: AndroidAudioContentType.speech, - usage: AndroidAudioUsage.voiceCommunication, - ), - androidAudioFocusGainType: AndroidAudioFocusGainType.gain, - )); - await session.setActive(true); - Logger.info('yiding----------- [音频会话] 已配置 playAndRecord 模式'); - } catch (e) { - Logger.error('yiding----------- [音频会话] 配置失败: $e'); - } - } - - /// 点击通话按钮 → 开麦，进入等待说话状态（不连接服务器） - Future startCall() async { - if (inCall.value) return; - // PTT 模式下 AI 正在播报 → 先打断 TTS，再进入通话 - if (_ttsPlaying || status.value == MobileElfStatus.speaking) { - _interruptTts(); - _ttsInterrupted = false; - // 等一帧让 player.stop() 完成 - await Future.delayed(const Duration(milliseconds: 50)); - } - // 如果 PTT 正在录音中 → 先取消 - if (pttRecording.value) { - await stopPtt(cancel: true); - } - final hasPermission = await _recorder.hasPermission(); - if (!hasPermission) { _setError('无麦克风权限'); return; } - - await _configureAudioSession(); - // 为播放器单独设置 voiceCommunication 音频属性，与录音器使用同一音频路由 - // 这样 ExoPlayer 和 AudioRecord 可以像 VoIP 应用一样同时工作 - if (!kIsWeb) { - await _player.setAndroidAudioAttributes(const AndroidAudioAttributes( - contentType: AndroidAudioContentType.speech, - usage: AndroidAudioUsage.voiceCommunication, - )); - } - await _openMic(); - _startMicWatchdog(); - inCall.value = true; - status.value = MobileElfStatus.idle; // 等待用户说话 - callSeconds.value = 0; - _callTimer?.cancel(); - _callTimer = Timer.periodic(const Duration(seconds: 1), (_) => callSeconds.value++); - _startIdleTimer(); // 开始等待说话超时 - Logger.info('yiding----------- [通话] 已开始，等待用户说话'); - } - - /// 挂断 - void disconnect() { - Logger.info('yiding----------- [通话] 挂断'); - _stopRecordingTimer(); - _stopMicWatchdog(); - _cancelIdleTimer(); - _callTimer?.cancel(); _callTimer = null; - _forwardingAudio = false; - _wsConnecting = false; - _speechDetected = false; - _micSubscription?.cancel(); _micSubscription = null; - _recorder.stop(); - // 清空 TTS 队列 & 停播 - _ttsInterrupted = true; - _ttsQueue.clear(); - _ttsPlaying = false; - if (!(_ttsPlayCompleter?.isCompleted ?? true)) _ttsPlayCompleter?.complete(); - _player.stop(); - _closeWs(); - _preBuffer.clear(); - _sendBuffer.clear(); _sendBufferOffset = 0; - micRms.value = 0; micAlive.value = false; - inCall.value = false; - status.value = MobileElfStatus.idle; - } - - // ═══════════════════════════════════════════════════════════════════════════ - // 麦克风 & 本地 VAD - // ═══════════════════════════════════════════════════════════════════════════ - - Future _openMic() async { - _micSubscription?.cancel(); - _micSubscription = null; - try { await _recorder.stop(); } catch (_) {} - final stream = await _recorder.startStream( - const RecordConfig( - encoder: AudioEncoder.pcm16bits, - sampleRate: 16000, - numChannels: 1, - echoCancel: true, - noiseSuppress: true, - androidConfig: AndroidRecordConfig( - audioSource: AndroidAudioSource.voiceCommunication, - ), - ), - ); - _speechDetected = false; - _vadOnsetCount = 0; - _micChunkCount = 0; - micAlive.value = false; - _micSubscription = stream.listen(_onMicAudio); - Logger.info('yiding----------- [麦克风] 已开启 (voiceCommunication + echoCancel)'); - } - - /// 启动麦克风看门狗：每 1.5 秒检测一次，如果无新 chunk 则标记 micAlive=false - void _startMicWatchdog() { - _micWatchdog?.cancel(); - _lastWatchdogChunkCount = _micChunkCount; - _micWatchdog = Timer.periodic(const Duration(milliseconds: 1500), (_) { - if (!inCall.value && !pttRecording.value) { _micWatchdog?.cancel(); return; } - if (_micChunkCount == _lastWatchdogChunkCount) { - micAlive.value = false; - if (status.value == MobileElfStatus.speaking || _ttsPlaying) { - // TTS 播放中麦克风已关闭，不需要重启 - } else { - Logger.info('yiding----------- [看门狗] 麦克风无数据! 尝试重启'); - _openMic(); - } - } - _lastWatchdogChunkCount = _micChunkCount; + _subscribeEvents(); + _loadChatHistory(); + // 同步播报开关到原生层 + ever(ttsEnabled, (bool enabled) { + AgentService.setTtsMuted(!enabled); }); } - void _stopMicWatchdog() { - _micWatchdog?.cancel(); - _micWatchdog = null; - } - - void _onMicAudio(Uint8List chunk) { - if (!inCall.value && !pttRecording.value) return; - _micChunkCount++; - - // 计算 RMS（PCM 16-bit little-endian） - final rms = _calcRms(chunk); - - // 更新 UI 可观察值 - micAlive.value = true; - micRms.value = rms; - - // 每 100 个 chunk (~2s) 输出一次调试日志，确认麦克风存活 - if (_micChunkCount % 100 == 0) { - Logger.info('yiding----------- [麦克风] chunk#$_micChunkCount rms=${rms.toStringAsFixed(0)} ' - 'status=${status.value} speechDetected=$_speechDetected fwd=$_forwardingAudio wsConn=$_wsConnecting'); - } - - // ── 当前正在转发音频到 WS ── - if (_forwardingAudio && _channel != null) { - _feedSendBuffer(chunk); - return; - } - - // ── WS 正在连接中，先存入预缓存 ── - if (_wsConnecting) { - _addToPreBuffer(chunk); - return; - } - - // ── TTS 播放中 / thinking 中不做 VAD ── - if (status.value == MobileElfStatus.speaking || _ttsPlaying || - status.value == MobileElfStatus.thinking) { - return; - } - - // ── 空闲/等待状态：连续 chunk VAD ── - _addToPreBuffer(chunk); - - if (rms >= _vadThreshold) { - _vadOnsetCount++; - if (_vadOnsetCount >= _vadOnsetChunks && !_speechDetected) { - _speechDetected = true; - _cancelIdleTimer(); - Logger.info('yiding----------- [VAD] 检测到说话 rms=${rms.toStringAsFixed(0)} ' - 'onset=$_vadOnsetCount/$_vadOnsetChunks，启动连接'); - _onSpeechDetected(); - } - } else { - _vadOnsetCount = 0; - } - } - - double _calcRms(Uint8List pcm) { - if (pcm.length < 2) return 0; - final samples = pcm.length ~/ 2; - double sum = 0; - for (int i = 0; i < pcm.length - 1; i += 2) { - int sample = pcm[i] | (pcm[i + 1] << 8); - if (sample >= 32768) sample -= 65536; // signed - sum += sample * sample; - } - return math.sqrt(sum / samples); - } - - void _addToPreBuffer(Uint8List chunk) { - _preBuffer.add(Uint8List.fromList(chunk)); - while (_preBuffer.length > _preBufferMaxChunks) { - _preBuffer.removeAt(0); - } - } - - // ═══════════════════════════════════════════════════════════════════════════ - // 检测到说话 → 连接服务器 → 发送预缓存 → 实时转发 - // ═══════════════════════════════════════════════════════════════════════════ - - Future _onSpeechDetected() async { - if (_wsConnecting) return; - - _ttsInterrupted = false; - _closeWs(); - _sessionEnded = false; - _wsConnecting = true; - - status.value = MobileElfStatus.connecting; - _dialogRequestId = _uuid.v4(); - _currentAiMsgIndex = -1; - realtimeText.value = ''; - _sendBuffer.clear(); - _sendBufferOffset = 0; - errorMsg.value = ''; - - Logger.info('yiding----------- [新轮] 说话触发连接 dialogId=$_dialogRequestId preBuffer=${_preBuffer.length}chunks'); - + Future _loadChatHistory() async { try { - await _connect(); - Logger.info('yiding----------- [新轮] WS ready，开始发送预缓存'); - } catch (e) { - Logger.error('yiding----------- [新轮] 连接失败: $e'); - _wsConnecting = false; - _speechDetected = false; - _setError('连接失败: $e'); - return; - } - - // 发送预缓存音频 - _forwardingAudio = true; - _wsConnecting = false; - status.value = MobileElfStatus.recording; - recordingSeconds.value = 0; - _stopRecordingTimer(); - _recordingTimer = Timer.periodic(const Duration(seconds: 1), (_) => recordingSeconds.value++); - - final cached = List.from(_preBuffer); - _preBuffer.clear(); - int cachedBytes = 0; - for (final c in cached) { - _feedSendBuffer(c); - cachedBytes += c.length; - } - Logger.info('yiding----------- [新轮] 预缓存已发送 ${cached.length}chunks ${cachedBytes}bytes，实时转发中'); - } - - /// 将 chunk 加入发送缓冲，凑满 _frameSize 就发 - void _feedSendBuffer(Uint8List chunk) { - _sendBuffer.addAll(chunk); - while (_sendBuffer.length - _sendBufferOffset >= _frameSize) { - final frame = Uint8List.fromList( - _sendBuffer.sublist(_sendBufferOffset, _sendBufferOffset + _frameSize)); - _sendBufferOffset += _frameSize; - _sendAudioChunk(frame); - } - // 紧缩 - if (_sendBufferOffset >= _frameSize * 8) { - _sendBuffer.removeRange(0, _sendBufferOffset); - _sendBufferOffset = 0; - } - } - - // ═══════════════════════════════════════════════════════════════════════════ - // WebSocket 连接 - // ═══════════════════════════════════════════════════════════════════════════ - - Future _connect() async { - _readyCompleter = Completer(); - try { - final ts = DateTime.now().millisecondsSinceEpoch.toString(); - final sn = _uuid.v4(); - final sign = _calcSign(_deviceSecret, ts); - final url = 'wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr' - '?deviceNo=$_deviceNo&deviceId=$_deviceId&productId=$_productId' - '&productKey=$_productKey&sign=$sign&sn=$sn&ts=$ts'; - - if (kIsWeb) { - _channel = WebSocketChannel.connect(Uri.parse(url)); - } else { - final httpClient = HttpClient()..badCertificateCallback = (_, __, ___) => true; - final ws = await WebSocket.connect(url, customClient: httpClient, headers: { - 'Origin': 'https://ivs.chinamobiledevice.com:11443', - 'Host': 'ivs.chinamobiledevice.com:11443', - }); - _channel = IOWebSocketChannel(ws); - } - - _wsSubscription = _channel!.stream.listen( - _handleMessage, - onError: (e) { - Logger.error('yiding----------- [WS] 错误: $e'); - if (!(_readyCompleter?.isCompleted ?? true)) _readyCompleter?.completeError(e); - _handleWsDone(); - }, - onDone: () { - Logger.info('yiding----------- [WS] onDone sessionEnded=$_sessionEnded status=${status.value}'); - if (!(_readyCompleter?.isCompleted ?? true)) _readyCompleter?.completeError('WS closed'); - _handleWsDone(); - }, + final result = await ChatStorage.getMessages( + agentId: _agentId, + page: 1, + pageSize: 50, ); - - _sendStartFrame(); - await _readyCompleter!.future.timeout( - const Duration(seconds: 15), - onTimeout: () => throw TimeoutException('等待 ready 超时'), - ); - } catch (e) { - _closeWs(); - rethrow; - } - } - - void _handleWsDone() { - _channel = null; - _wsSubscription = null; - _forwardingAudio = false; - _wsConnecting = false; - - if (_sessionEnded && inCall.value) { - Logger.info('yiding----------- [WS] 正常关闭 (通话模式)'); - if (status.value == MobileElfStatus.speaking) { - // TTS 播放中，等 playTts 完成后自动回到 idle 等待说话 - } else { - status.value = MobileElfStatus.idle; - _resetVad(); - Logger.info('yiding----------- [WS] 回到等待说话'); - } - } else if (_sessionEnded && !inCall.value) { - // PTT/文本模式：WS 正常关闭 - Logger.info('yiding----------- [WS] 正常关闭 (PTT/文本模式)'); - if (status.value != MobileElfStatus.speaking) { - status.value = MobileElfStatus.idle; + if (result.messages.isNotEmpty) { + final history = result.messages.map((m) => MobileElfMessage( + isUser: m.sender == 'user', + text: m.message, + time: DateTime.tryParse(m.timestamp) ?? DateTime.now(), + )).toList(); + // 按时间正序排列 + history.sort((a, b) => a.time.compareTo(b.time)); + messages.addAll(history); + _scrollToBottom(); } - } else if (inCall.value) { - Logger.info('yiding----------- [WS] 非预期关闭，回到等待说话'); - status.value = MobileElfStatus.idle; - _resetVad(); - } else { - Logger.info('yiding----------- [WS] 非预期关闭'); - status.value = MobileElfStatus.idle; + } catch (e) { + Logger.error('$_tag 加载聊天记录失败: $e'); } } - void _resetVad() { - _speechDetected = false; - _vadOnsetCount = 0; - _preBuffer.clear(); - _openMic(); - } - - // ═══════════════════════════════════════════════════════════════════════════ - // 空闲超时：TTS 播完后等待说话，超时自动挂断 - // ═══════════════════════════════════════════════════════════════════════════ - - void _startIdleTimer() { - _cancelIdleTimer(); - _idleTimer = Timer(Duration(seconds: _idleTimeoutSeconds), () { - if (inCall.value && status.value == MobileElfStatus.idle && !_speechDetected) { - Logger.info('yiding----------- [超时] ${_idleTimeoutSeconds}秒无说话，自动挂断'); - disconnect(); - } - }); - Logger.info('yiding----------- [超时] 开始${_idleTimeoutSeconds}秒等待说话计时'); - } - - void _cancelIdleTimer() { - _idleTimer?.cancel(); - _idleTimer = null; - } - - void _closeWs() { - final sub = _wsSubscription; _wsSubscription = null; - sub?.cancel(); - final ch = _channel; _channel = null; - try { ch?.sink.close(); } catch (_) {} - } - - // ═══════════════════════════════════════════════════════════════════════════ - // 发送帧 - // ═══════════════════════════════════════════════════════════════════════════ - - void _sendStartFrame() { - if (_channel == null) return; - _channel!.sink.add(jsonEncode({ - 'type': 'start', - 'data': { - 'cuid': _deviceId, - 'format': 'pcm', - 'sample': 16000, - 'support_dcs': 2, - 'chunk_size': 10240, - 'support_tts': true, - 'support_text2dcs': true, - 'user_agent': 'voitrans/1.0', - 'dialog_request_id': _dialogRequestId, - 'access_rc': true, - 'support_part_tts': true, - 'need_stoplisten': true, - 'need_dialogue_finish': true, - 'result_trans2directive': false, - 'rc_version': '5', - 'messages': _buildContextMessages(), - 'client_context': [], - }, - })); - } - - void _sendAudioChunk(Uint8List data) { - try { _channel?.sink.add(data); } catch (_) {} - } - - void _sendFinish() { - try { _channel?.sink.add(jsonEncode({'type': 'finish'})); } catch (_) {} + @override + void onClose() { + _eventSub?.cancel(); + _callTimer?.cancel(); + _pttTimer?.cancel(); + AgentService.stopConversation(); + scrollController.dispose(); + super.onClose(); } // ═══════════════════════════════════════════════════════════════════════════ - // 接收消息处理 + // 事件订阅（来自原生 StsAgent） // ═══════════════════════════════════════════════════════════════════════════ - void _handleMessage(dynamic raw) { - try { - if (raw is! String) return; - final msg = jsonDecode(raw) as Map; - final type = msg['type'] as String? ?? ''; - - switch (type) { - case 'ready': - if (!(_readyCompleter?.isCompleted ?? true)) _readyCompleter?.complete(); - break; - - case 'stoplisten': - Logger.info('yiding----------- [服务端] stoplisten'); - _stopRecordingTimer(); - _forwardingAudio = false; - _speechDetected = false; // 允许 VAD 在 thinking/speaking 期间检测新语音 - _vadOnsetCount = 0; - // 发送剩余 - final remain = _sendBuffer.length - _sendBufferOffset; - if (remain > 0) _sendAudioChunk(Uint8List.fromList(_sendBuffer.sublist(_sendBufferOffset))); - _sendBuffer.clear(); _sendBufferOffset = 0; - _sendFinish(); - status.value = MobileElfStatus.thinking; - break; - - case 'mid_result': - realtimeText.value = msg['result'] as String? ?? ''; - break; - - case 'fin_result': - final result = msg['result'] as String? ?? ''; - if (result.isNotEmpty) { - realtimeText.value = ''; - _addUserMessage(result); - } - if (msg['end'] == 1) _markSessionEnded(); - break; - - case 'inside_rc': - _handleInsideRc(msg); - break; - - case 'dcs_decide': - if (msg['end'] == 1) { - Logger.info('yiding----------- [服务端] dcs_decide end:1'); - _markSessionEnded(); - } - break; - } - } catch (e) { - Logger.error('[MobileElf] 处理消息失败: $e'); - } + void _subscribeEvents() { + _eventSub = AgentService.events.listen(_onAgentEvent); } - void _handleInsideRc(Map msg) { - final data = msg['data'] as Map?; - if (data == null) return; - if ((data['code'] as int? ?? -1) != 0) return; - - final assistantAnswer = data['assistant_answer'] as String? ?? ''; - String cleanAnswer = assistantAnswer; - try { - cleanAnswer = (jsonDecode(assistantAnswer) as Map)['content'] as String? ?? assistantAnswer; - } catch (_) {} - - String? ttsUrl; - for (final item in (data['data'] as List? ?? [])) { - final d = item as Map; - final h = d['header'] as Map?; - if (h?['namespace'] == 'ai.fxzsos.device_interface.voice_output' && h?['name'] == 'Speak') { - ttsUrl = (d['payload'] as Map?)?['url'] as String?; + void _onAgentEvent(AgentServiceEvent event) { + Logger.info('$_tag event: ${event.type} data: ${event.data}'); + switch (event.type) { + case AgentServiceEventType.recognitionStarted: + status.value = MobileElfStatus.recording; + realtimeText.value = ''; + micAlive.value = true; break; - } - } - if (cleanAnswer.isNotEmpty) { - if (_currentAiMsgIndex >= 0 && _currentAiMsgIndex < messages.length) { - messages[_currentAiMsgIndex] = MobileElfMessage( - isUser: false, text: cleanAnswer, - time: messages[_currentAiMsgIndex].time, - ttsUrl: ttsUrl ?? messages[_currentAiMsgIndex].ttsUrl, - ); - } else { - _currentAiMsgIndex = messages.length; - _addAiMessage(cleanAnswer, ttsUrl: ttsUrl); - } - } - - if (ttsUrl != null && ttsUrl.isNotEmpty) _enqueueTts(ttsUrl); - if (msg['end'] == 1) _markSessionEnded(); - } - - void _markSessionEnded() { - if (_sessionEnded) return; - _sessionEnded = true; - Logger.info('yiding----------- [会话] 本轮结束 status=${status.value}'); - _stopRecordingTimer(); - _forwardingAudio = false; - _speechDetected = false; // 允许 VAD 在 TTS 播放期间检测新语音 - _vadOnsetCount = 0; - _sendBuffer.clear(); _sendBufferOffset = 0; - realtimeText.value = ''; - } + case AgentServiceEventType.recognizing: + realtimeText.value = event.data['text']?.toString() ?? ''; + status.value = MobileElfStatus.recording; + break; - // ═══════════════════════════════════════════════════════════════════════════ - // TTS 播放队列 - // ═══════════════════════════════════════════════════════════════════════════ + case AgentServiceEventType.recognitionResult: + final text = event.data['text']?.toString() ?? ''; + realtimeText.value = ''; + micAlive.value = false; + if (text.isNotEmpty) _addMessage(isUser: true, text: text); + status.value = MobileElfStatus.thinking; + break; - /// 将 TTS URL 加入队列，如果没有正在播放则关闭麦克风并开始播放 - void _enqueueTts(String url) { - _ttsQueue.add(url); - Logger.info('yiding----------- [TTS] 入队 queue=${_ttsQueue.length}'); - if (!_ttsPlaying) { - // 关闭麦克风，避免音频冲突，TTS 播完后会重开 - _micSubscription?.cancel(); _micSubscription = null; - try { _recorder.stop(); } catch (_) {} - micAlive.value = false; micRms.value = 0; - Logger.info('yiding----------- [TTS] 关闭麦克风，开始播放'); - _playNextTts(); - } - } + case AgentServiceEventType.recognitionStopped: + case AgentServiceEventType.recognitionCanceled: + realtimeText.value = ''; + micAlive.value = false; + break; - /// 顺序播放队列中的 TTS - Future _playNextTts() async { - if (_ttsQueue.isEmpty || _ttsInterrupted) { - _ttsPlaying = false; - // 队列播完 → 重开麦克风，回到等待说话，启动超时计时 - if (!_ttsInterrupted && status.value == MobileElfStatus.speaking) { - status.value = MobileElfStatus.idle; - Logger.info('yiding----------- [TTS] 队列全部播完，重开麦克风，等待新一轮说话'); - if (inCall.value) { - _resetVad(); - _startIdleTimer(); + case AgentServiceEventType.assistantResponse: + final text = event.data['response']?.toString() ?? ''; + if (text.isNotEmpty) _updateOrAddAssistantMessage(text); + // 静音模式下收到回复后直接恢复状态（不等 TTS 事件） + if (!ttsEnabled.value) { + status.value = + _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; + micAlive.value = _inCallMode; } - } - return; - } - - _ttsPlaying = true; - status.value = MobileElfStatus.speaking; - final url = _ttsQueue.removeAt(0); + break; - // 用 Completer 等待播放完成或被打断 - _ttsPlayCompleter = Completer(); - StreamSubscription? stateSub; + case AgentServiceEventType.ttsStarted: + status.value = MobileElfStatus.speaking; + break; - try { - await _player.stop(); - // 不再重新配置音频会话！startCall() 已配置好 playAndRecord 模式， - // 重复配置会导致 audio_session 重新请求音频焦点，杀死录音流 - await _player.setUrl(url); - - // 监听 ProcessingState: completed=正常播完, idle=被打断或意外停止 - stateSub = _player.processingStateStream.listen((s) { - if ((s == ProcessingState.completed || s == ProcessingState.idle) && - !(_ttsPlayCompleter?.isCompleted ?? true)) { - Logger.info('yiding----------- [TTS] 当前段结束 state=$s'); - _ttsPlayCompleter?.complete(); + case AgentServiceEventType.ttsCompleted: + if (_inCallMode) { + // 通话模式：TTS 播完后自动回到录音等待状态 + status.value = MobileElfStatus.recording; + micAlive.value = true; + } else { + status.value = MobileElfStatus.idle; + micAlive.value = false; } - }); + break; - await _player.play(); - Logger.info('yiding----------- [TTS] 开始播放 剩余队列=${_ttsQueue.length} micAlive=${micAlive.value} micChunk=$_micChunkCount'); + case AgentServiceEventType.ttsCanceled: + case AgentServiceEventType.ttsStopped: + status.value = _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; + micAlive.value = _inCallMode; + break; - // 等待：正常播完 complete() 或被打断 _interruptTts() 也会 complete() - await _ttsPlayCompleter!.future.timeout(const Duration(seconds: 60)); - } catch (e) { - Logger.error('yiding----------- [TTS] 播放异常: $e'); - } finally { - stateSub?.cancel(); - } + case AgentServiceEventType.error: + final msg = event.data['message']?.toString() ?? '未知错误'; + status.value = MobileElfStatus.error; + errorMsg.value = msg; + Logger.error('$_tag error: $msg'); + break; - // 当前段结束，继续下一段（如果没被打断） - if (!_ttsInterrupted) { - _playNextTts(); - } - } + case AgentServiceEventType.micRms: + final rms = (event.data['rms'] as num?)?.toDouble() ?? 0.0; + micRms.value = rms; + break; - /// 立即打断 TTS：停播 + 清队列 + 解除等待 - void _interruptTts() { - Logger.info('yiding----------- [TTS] 打断! 清空队列=${_ttsQueue.length}'); - _ttsInterrupted = true; - _ttsQueue.clear(); - _ttsPlaying = false; - // 先解除 _playNextTts 的等待 - if (!(_ttsPlayCompleter?.isCompleted ?? true)) { - _ttsPlayCompleter?.complete(); + default: + break; } - _player.stop(); - } - - /// 手动打断 AI 播报（界面按钮调用） - Future interruptSpeaking() async { - _interruptTts(); - _ttsInterrupted = false; // 手动打断后重置，不影响后续 - status.value = MobileElfStatus.idle; - if (inCall.value) _resetVad(); + _scrollToBottom(); } // ═══════════════════════════════════════════════════════════════════════════ - // 手动停止录音 + // 公共接口（View 调用） // ═══════════════════════════════════════════════════════════════════════════ - Future stopRecording() async { - if (!_forwardingAudio) return; - _stopRecordingTimer(); - _forwardingAudio = false; - final remain = _sendBuffer.length - _sendBufferOffset; - if (remain > 0) _sendAudioChunk(Uint8List.fromList(_sendBuffer.sublist(_sendBufferOffset))); - _sendBuffer.clear(); _sendBufferOffset = 0; - _sendFinish(); - status.value = MobileElfStatus.thinking; + Future startCall() async { + if (inCall.value) return; + _inCallMode = true; + inCall.value = true; + callSeconds.value = 0; + status.value = MobileElfStatus.connecting; + _callTimer?.cancel(); + _callTimer = Timer.periodic( + const Duration(seconds: 1), (_) => callSeconds.value++); + await AgentService.startConversation(mode: 'phone_call'); } - void _stopRecordingTimer() { - _recordingTimer?.cancel(); _recordingTimer = null; + Future disconnect() async { + _inCallMode = false; + inCall.value = false; + _callTimer?.cancel(); + _callTimer = null; + callSeconds.value = 0; + pttRecording.value = false; + _pttTimer?.cancel(); + _pttTimer = null; + status.value = MobileElfStatus.idle; + micAlive.value = false; + realtimeText.value = ''; + await AgentService.stopConversation(); } - // ═══════════════════════════════════════════════════════════════════════════ - // 按住说话 (PTT) - // ═══════════════════════════════════════════════════════════════════════════ - - /// 按住说话 → 开始录音并连接 WS - Future startPtt() async { + void startPtt() { if (pttRecording.value) return; - // 如果正在播放 AI 回复 → 立即打断 - if (_ttsPlaying || status.value == MobileElfStatus.speaking) { - _interruptTts(); - _ttsInterrupted = false; - } - final hasPermission = await _recorder.hasPermission(); - if (!hasPermission) { _setError('无麦克风权限'); return; } - - // 先设置 connecting 状态，再设置 pttRecording，避免中间按钮橙色闪烁 - status.value = MobileElfStatus.connecting; pttRecording.value = true; pttSeconds.value = 0; - _pttTimer = Timer.periodic(const Duration(seconds: 1), (_) => pttSeconds.value++); - - await _configureAudioSession(); - if (!kIsWeb) { - await _player.setAndroidAudioAttributes(const AndroidAudioAttributes( - contentType: AndroidAudioContentType.speech, - usage: AndroidAudioUsage.voiceCommunication, - )); - } - await _openMic(); - _startMicWatchdog(); - - _ttsInterrupted = false; - _closeWs(); - _sessionEnded = false; - _wsConnecting = true; - _dialogRequestId = _uuid.v4(); - _currentAiMsgIndex = -1; - realtimeText.value = ''; - _sendBuffer.clear(); _sendBufferOffset = 0; - errorMsg.value = ''; - - try { - await _connect(); - _forwardingAudio = true; - _wsConnecting = false; - status.value = MobileElfStatus.recording; - Logger.info('yiding----------- [PTT] 录音开始'); - } catch (e) { - _wsConnecting = false; - pttRecording.value = false; - _pttTimer?.cancel(); _pttTimer = null; - _stopMicWatchdog(); - _setError('连接失败: $e'); - } + status.value = MobileElfStatus.connecting; + _pttTimer?.cancel(); + _pttTimer = Timer.periodic( + const Duration(seconds: 1), (_) => pttSeconds.value++); + AgentService.startConversation(mode: 'push_to_talk'); } - /// 松开 → 停止录音，发送 finish - Future stopPtt({bool cancel = false}) async { + void stopPtt() { + if (!pttRecording.value) return; pttRecording.value = false; - _pttTimer?.cancel(); _pttTimer = null; - _stopMicWatchdog(); - - if (cancel) { - _forwardingAudio = false; - _closeWs(); - _micSubscription?.cancel(); _micSubscription = null; - try { _recorder.stop(); } catch (_) {} - micAlive.value = false; micRms.value = 0; - realtimeText.value = ''; - status.value = MobileElfStatus.idle; - Logger.info('yiding----------- [PTT] 已取消'); - return; - } + _pttTimer?.cancel(); + _pttTimer = null; + AgentService.stopConversation(); + } - // 正常结束：发送剩余音频 + finish - if (_forwardingAudio) { - _forwardingAudio = false; - final remain = _sendBuffer.length - _sendBufferOffset; - if (remain > 0) _sendAudioChunk(Uint8List.fromList(_sendBuffer.sublist(_sendBufferOffset))); - _sendBuffer.clear(); _sendBufferOffset = 0; - _sendFinish(); - status.value = MobileElfStatus.thinking; - } - _micSubscription?.cancel(); _micSubscription = null; - try { _recorder.stop(); } catch (_) {} - micAlive.value = false; micRms.value = 0; - Logger.info('yiding----------- [PTT] 录音结束，等待AI回复'); + void interruptSpeaking() { + AgentService.interruptCurrentResponse(); + status.value = + _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; + micAlive.value = _inCallMode; } - // ═══════════════════════════════════════════════════════════════════════════ - // 按钮操作 - // ═══════════════════════════════════════════════════════════════════════════ + /// 原生层已自动获取凭证，此处保留方法供 View 调用不报错 + Future retryCredentials() async { + credentialError.value = ''; + } - Future onMicTap() async { - Logger.info('yiding----------- [按钮] status=${status.value} inCall=${inCall.value}'); - switch (status.value) { - case MobileElfStatus.idle: - case MobileElfStatus.error: - if (!inCall.value) { - await startCall(); - } - break; - case MobileElfStatus.recording: - await stopRecording(); - break; - case MobileElfStatus.speaking: - await interruptSpeaking(); - break; - default: - break; + /// 清空聊天记录 + Future clearMessages() async { + messages.clear(); + try { + await AgentService.clearChatHistory(); + } catch (e) { + Logger.error('$_tag 清空聊天记录失败: $e'); } } // ═══════════════════════════════════════════════════════════════════════════ - // 工具方法 + // 工具 // ═══════════════════════════════════════════════════════════════════════════ - String _calcSign(String secret, String ts) => md5.convert(utf8.encode('$secret$ts')).toString(); - - List> _buildContextMessages() => - messages.map((m) => {'role': m.isUser ? 'user' : 'assistant', 'content': m.text}).toList(); - - void _addUserMessage(String text) { - if (messages.isNotEmpty && messages.last.isUser && messages.last.text == text) return; - messages.add(MobileElfMessage(isUser: true, text: text, time: DateTime.now())); - scrollToBottom(); - } - - void _addAiMessage(String text, {String? ttsUrl}) { - messages.add(MobileElfMessage(isUser: false, text: text, time: DateTime.now(), ttsUrl: ttsUrl)); - scrollToBottom(); + void _addMessage({required bool isUser, required String text}) { + messages.add(MobileElfMessage( + isUser: isUser, text: text, time: DateTime.now())); + _scrollToBottom(); } - void _setError(String msg) { - errorMsg.value = msg; - status.value = MobileElfStatus.error; + /// 更新最后一条 AI 消息（如果存在且更短），否则添加新消息 + void _updateOrAddAssistantMessage(String text) { + final lastIdx = messages.length - 1; + if (lastIdx >= 0 && !messages[lastIdx].isUser) { + // 用更完整的文本替换 + messages[lastIdx] = MobileElfMessage( + isUser: false, text: text, time: messages[lastIdx].time); + messages.refresh(); + } else { + _addMessage(isUser: false, text: text); + } + _scrollToBottom(); } - void scrollToBottom() { - if (scrollController.hasClients) { - WidgetsBinding.instance.addPostFrameCallback((_) { - if (scrollController.hasClients) { - scrollController.animateTo(scrollController.position.maxScrollExtent, - duration: const Duration(milliseconds: 200), curve: Curves.easeOut); - } - }); - } + void _scrollToBottom() { + WidgetsBinding.instance.addPostFrameCallback((_) { + if (scrollController.hasClients) { + scrollController.animateTo( + scrollController.position.maxScrollExtent, + duration: const Duration(milliseconds: 300), + curve: Curves.easeOut, + ); + } + }); } } diff --git a/lib/modules/mobile_elf/views/mobile_elf_view.dart b/lib/modules/mobile_elf/views/mobile_elf_view.dart index 7aaa114b1..5dc2d89a1 100644 --- a/lib/modules/mobile_elf/views/mobile_elf_view.dart +++ b/lib/modules/mobile_elf/views/mobile_elf_view.dart @@ -1,6 +1,8 @@ import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; import 'package:flutter_screenutil/flutter_screenutil.dart'; import 'package:get/get.dart'; +import 'package:intl/intl.dart'; // 用于 DateFormat import '../controllers/mobile_elf_controller.dart'; class MobileElfView extends GetView { @@ -71,6 +73,7 @@ class MobileElfView extends GetView { size: 18.sp, color: isDark ? Colors.white : _textSecondary), onPressed: () => Get.back(), ), + titleSpacing: 0, title: Row( mainAxisSize: MainAxisSize.min, children: [ @@ -81,13 +84,59 @@ class MobileElfView extends GetView { fontSize: 16.sp, fontWeight: FontWeight.w600, color: isDark ? Colors.white : _textPrimary)), + SizedBox(width: 8.w), + Obx(() => _buildStatusPill()), ], ), actions: [ - Obx(() => Padding( - padding: EdgeInsets.only(right: 16.w), - child: _buildStatusPill(), + // 播报开关 + Obx(() => IconButton( + icon: Icon( + controller.ttsEnabled.value + ? Icons.volume_up_rounded + : Icons.volume_off_rounded, + size: 20.sp, + color: controller.ttsEnabled.value + ? _blue + : (isDark ? Colors.grey[500] : _textSecondary), + ), + onPressed: () => controller.ttsEnabled.toggle(), + tooltip: controller.ttsEnabled.value ? '关闭播报' : '开启播报', )), + PopupMenuButton( + icon: Icon(Icons.more_vert, + size: 20.sp, color: isDark ? Colors.white : _textSecondary), + onSelected: (value) { + if (value == 'clear') { + Get.dialog( + AlertDialog( + title: Text('clearChatHistory'.tr.isNotEmpty ? 'clearChatHistory'.tr : '清空记录'), + content: Text('confirmClearChat'.tr.isNotEmpty ? 'confirmClearChat'.tr : '确定清空所有聊天记录吗？'), + actions: [ + TextButton(onPressed: () => Get.back(), child: Text('cancel'.tr.isNotEmpty ? 'cancel'.tr : '取消')), + TextButton( + onPressed: () { + controller.clearMessages(); + Get.back(); + }, + child: Text('confirm'.tr.isNotEmpty ? 'confirm'.tr : '确定', style: const TextStyle(color: Colors.red)), + ), + ], + ), + ); + } + }, + itemBuilder: (_) => [ + PopupMenuItem( + value: 'clear', + child: Row(children: [ + Icon(Icons.delete_outline, size: 18.sp, color: Colors.red), + SizedBox(width: 8.w), + Text('清空记录', style: TextStyle(fontSize: 14.sp, color: Colors.red)), + ]), + ), + ], + ), ], ); } @@ -237,58 +286,138 @@ class MobileElfView extends GetView { ]), SizedBox(height: 4.h), ], - // 气泡体 + // 气泡体（长按弹出菜单） Row( mainAxisAlignment: isUser ? MainAxisAlignment.end : MainAxisAlignment.start, children: [ if (!isUser) SizedBox(width: 34.w), Flexible( - child: Container( - padding: - EdgeInsets.symmetric(horizontal: 14.w, vertical: 10.h), - decoration: BoxDecoration( - color: isUser - ? _blue - : (isDark ? const Color(0xFF2A2A2A) : _surfaceLight), - borderRadius: BorderRadius.only( - topLeft: Radius.circular(18.r), - topRight: Radius.circular(18.r), - bottomLeft: Radius.circular(isUser ? 18.r : 4.r), - bottomRight: Radius.circular(isUser ? 4.r : 18.r), - ), - boxShadow: [ - BoxShadow( - color: Colors.black.withValues(alpha: 0.05), - blurRadius: 6, - offset: const Offset(0, 2), + child: GestureDetector( + onLongPressStart: (details) => _showMessageMenu(details, msg, isDark), + child: Container( + padding: + EdgeInsets.symmetric(horizontal: 14.w, vertical: 10.h), + decoration: BoxDecoration( + color: isUser + ? _blue + : (isDark ? const Color(0xFF2A2A2A) : _surfaceLight), + borderRadius: BorderRadius.only( + topLeft: Radius.circular(18.r), + topRight: Radius.circular(18.r), + bottomLeft: Radius.circular(isUser ? 18.r : 4.r), + bottomRight: Radius.circular(isUser ? 4.r : 18.r), ), - ], + boxShadow: [ + BoxShadow( + color: Colors.black.withValues(alpha: 0.05), + blurRadius: 6, + offset: const Offset(0, 2), + ), + ], + ), + child: Text(msg.text, + style: TextStyle( + fontSize: 15.sp, + height: 1.55, + color: isUser + ? Colors.white + : (isDark ? Colors.white : _textPrimary))), ), - child: Text(msg.text, - style: TextStyle( - fontSize: 15.sp, - height: 1.55, - color: isUser - ? Colors.white - : (isDark ? Colors.white : _textPrimary))), ), ), ], ), - // 时间 + // 时间 + AI 标签 SizedBox(height: 3.h), Padding( padding: EdgeInsets.only(left: isUser ? 0 : 34.w), - child: Text(_fmtTime(msg.time), - style: TextStyle( - fontSize: 11.sp, color: const Color(0xFFD0D5DD))), + child: Row( + mainAxisAlignment: + isUser ? MainAxisAlignment.end : MainAxisAlignment.start, + children: [ + Text(_formatTimestamp(msg.time), + style: TextStyle( + fontSize: 10.sp, color: isDark ? Colors.grey[500] : const Color(0xFFD0D5DD))), + if (!isUser) ...[ + SizedBox(width: 8.w), + Text('generatedByAi'.tr.isNotEmpty ? 'generatedByAi'.tr : '由AI生成', + style: TextStyle( + fontSize: 10.sp, + color: Colors.blue[600], + fontWeight: FontWeight.bold, + fontStyle: FontStyle.italic)), + ], + ], + ), ), ], ), ); } + /// 长按消息弹出菜单 + void _showMessageMenu(LongPressStartDetails details, MobileElfMessage msg, bool isDark) { + HapticFeedback.mediumImpact(); + final overlay = Overlay.of(Get.context!).context.findRenderObject() as RenderBox?; + if (overlay == null) return; + final position = RelativeRect.fromRect( + Rect.fromLTWH(details.globalPosition.dx, details.globalPosition.dy, 0, 0), + Offset.zero & overlay.size, + ); + showMenu( + context: Get.context!, + position: position, + shape: RoundedRectangleBorder(borderRadius: BorderRadius.circular(12.r)), + color: isDark ? const Color(0xFF2A2A2A) : Colors.white, + items: [ + PopupMenuItem( + onTap: () { + Clipboard.setData(ClipboardData(text: msg.text)); + Get.snackbar('success'.tr.isNotEmpty ? 'success'.tr : '成功', + 'copySuccess'.tr.isNotEmpty ? 'copySuccess'.tr : '已复制', + backgroundColor: Colors.blue.withValues(alpha: 0.8), colorText: Colors.white); + }, + child: Row(children: [ + Icon(Icons.copy, size: 18.sp, color: isDark ? Colors.white : Colors.black87), + SizedBox(width: 8.w), + Text('copy'.tr.isNotEmpty ? 'copy'.tr : '复制', + style: TextStyle(fontSize: 14.sp, color: isDark ? Colors.white : Colors.black87)), + ]), + ), + ], + ); + } + + /// 智能时间格式化（与 agent 模块一致） + String _formatTimestamp(DateTime t) { + final now = DateTime.now(); + final diff = now.difference(t); + final today = DateTime(now.year, now.month, now.day); + final msgDate = DateTime(t.year, t.month, t.day); + + if (diff.inMinutes < 1) return 'justNow'.tr.isNotEmpty ? 'justNow'.tr : '刚刚'; + if (diff.inHours < 1) { + final m = diff.inMinutes.toString(); + final text = 'minutesAgo'.trParams({'minutes': m}); + return text.isNotEmpty && text != 'minutesAgo' ? text : '$m分钟前'; + } + if (msgDate == today) return DateFormat('HH:mm').format(t); + if (msgDate == today.subtract(const Duration(days: 1))) { + return '${('yesterday'.tr.isNotEmpty ? 'yesterday'.tr : '昨天')} ${DateFormat('HH:mm').format(t)}'; + } + if (diff.inDays < 7) { + return '${_weekdayName(t.weekday)} ${DateFormat('HH:mm').format(t)}'; + } + if (t.year == now.year) return DateFormat('MM-dd HH:mm').format(t); + return DateFormat('yyyy-MM-dd HH:mm').format(t); + } + + String _weekdayName(int w) { + const names = ['', '星期一', '星期二', '星期三', '星期四', '星期五', '星期六', '星期日']; + return names[w]; + } + Widget _buildRealtimeBubble(String text) { return Padding( padding: EdgeInsets.only(bottom: 14.h), @@ -702,8 +831,6 @@ class MobileElfView extends GetView { ); } - String _fmtTime(DateTime t) => - '${t.hour.toString().padLeft(2, '0')}:${t.minute.toString().padLeft(2, '0')}'; String _fmtDuration(int secs) { final m = (secs ~/ 60).toString().padLeft(2, '0'); diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index 3a1dddbd6..3a56af051 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -100,8 +100,12 @@ object AgentService : CoroutineScope { // 聊天存储服务 private lateinit var chatStorageHelper: ChatStorageHelper + // Agent模式: stt_llm_tts（默认）| sts（移动精灵） + private var agentMode = "stt_llm_tts" + private var stsAgent: StsAgent? = null + // 会话ID，用于区分不同聊天上下文 - private val agentId = "default_agent" + private var agentId = "default_agent" //工具调用提示 private var callingToolText = "" private var insufficientIntegralText = "" // 积分不足提示 @@ -215,6 +219,36 @@ object AgentService : CoroutineScope { if (isInitialized) return true try { + agentMode = config["agentMode"]?.toString() ?: "stt_llm_tts" + agentId = config["agentId"]?.toString()?.takeIf { it.isNotBlank() } ?: "default_agent" + + // STS 模式：跳过 Azure/OpenAI 初始化，直接启动 StsAgent + if (agentMode == "sts") { + this.context = appContext.applicationContext + val stsConfig = StsConfig( + productId = config["stsProductId"]?.toString() ?: "", + productKey = config["stsProductKey"]?.toString() ?: "", + deviceNo = config["stsDeviceNo"]?.toString() ?: "", + deviceId = config["stsDeviceId"]?.toString() ?: "", + deviceSecret = config["stsDeviceSecret"]?.toString() ?: "", + agentId = agentId, + ) + stsAgent = StsAgent( + context = context, + config = stsConfig, + onEvent = { name, data -> sendEvent(name, data) }, + onCredentialsFetched = { deviceId, deviceSecret, deviceNo -> + sendEvent("sts_credentials", mapOf( + "deviceId" to deviceId, + "deviceSecret" to deviceSecret, + "deviceNo" to deviceNo, + )) + } + ) + _isInitialized.set(true) + return true + } + serverurl = config["serverurl"]?.toString() ?: ""; usertoken = config["usertoken"]?.toString() ?: ""; callingToolText = config["callingToolText"]?.toString() ?: ""; @@ -341,6 +375,12 @@ object AgentService : CoroutineScope { */ fun dispose() { try { + if (agentMode == "sts") { + stsAgent?.dispose() + stsAgent = null + _isInitialized.set(false) + return + } BleService.closeCodec() stopRecognition() stopTts() @@ -644,6 +684,15 @@ object AgentService : CoroutineScope { * @param mode 识别模式：normal, ble_wakeup, phone_call, push_to_talk */ fun startRecognition(isExternalActive: Boolean, mode: String = "normal"): Boolean { + if (agentMode == "sts") { + when (mode) { + "push_to_talk" -> stsAgent?.startPtt() + "phone_call" -> stsAgent?.startCall() + "ble_wakeup" -> stsAgent?.startCall() // BLE 唤醒：进入通话模式（VAD 自动检测） + else -> stsAgent?.startPtt() // 普通模式默认 PTT + } + return true + } if(isExternalActive){ isKeepResult = false BleService.openEncoder()//不发指令。只是录制opus @@ -902,6 +951,11 @@ object AgentService : CoroutineScope { * 停止语音识别 */ fun stopRecognition() { + if (agentMode == "sts") { + // StsAgent 内部根据 inCall / pttRecording 状态自行决定停止方式 + stsAgent?.stop() + return + } if (!isRecognitionActive) return Log.d(TAG, "停止语音识别，当前模式: $currentRecognitionMode") @@ -928,7 +982,16 @@ object AgentService : CoroutineScope { * 打断当前响应 * 停止TTS播放和AI流输出 */ + /** 设置 STS 模式 TTS 静音开关 */ + fun setTtsMuted(muted: Boolean) { + stsAgent?.ttsMuted = muted + } + fun interruptCurrentResponse() { + if (agentMode == "sts") { + stsAgent?.interruptSpeaking() + return + } if (isAiStreaming || isTtsSpeaking) { val startTime = System.currentTimeMillis() stopRecognition() @@ -1661,6 +1724,11 @@ object AgentService : CoroutineScope { * 清除聊天历史 */ fun clearChatHistory(callback: (Boolean) -> Unit) { + if (agentMode == "sts") { + stsAgent?.clearHistory() + callback(true) + return + } launch { try { val success = chatStorageHelper.deleteMessages(agentId) diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt index 4e17d7172..e6a08f893 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt @@ -83,6 +83,14 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream config["xunfeiAppId"]?.toString()?.let { intent.putExtra("xunfeiAppId", it) } config["xunfeiAccessKeyId"]?.toString()?.let { intent.putExtra("xunfeiAccessKeyId", it) } config["xunfeiAccessKeySecret"]?.toString()?.let { intent.putExtra("xunfeiAccessKeySecret", it) } + // STS 模式参数 + config["agentMode"]?.toString()?.let { intent.putExtra("agentMode", it) } + config["agentId"]?.toString()?.let { intent.putExtra("agentId", it) } + config["stsProductId"]?.toString()?.let { intent.putExtra("stsProductId", it) } + config["stsProductKey"]?.toString()?.let { intent.putExtra("stsProductKey", it) } + config["stsDeviceNo"]?.toString()?.let { intent.putExtra("stsDeviceNo", it) } + config["stsDeviceId"]?.toString()?.let { intent.putExtra("stsDeviceId", it) } + config["stsDeviceSecret"]?.toString()?.let { intent.putExtra("stsDeviceSecret", it) } config["vipexptime"]?.toString()?.let { intent.putExtra("vipexptime", it) //打印日志 @@ -330,6 +338,17 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream } } + "setTtsMuted" -> { + try { + val muted = call.argument("muted") ?: false + AgentService.setTtsMuted(muted) + result.success(true) + } catch (e: Exception) { + Log.e(TAG, "设置TTS静音失败", e) + result.error("SET_TTS_MUTED_ERROR", "设置TTS静音失败: ${e.message}", null) + } + } + "setCanWakeUp" -> { try { val canWakeUp = call.argument("canWakeUp") diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt index d2111244b..20f0ac498 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt @@ -77,6 +77,14 @@ class BleMediaService : Service(), CoroutineScope { private const val KEY_XUNFEI_APP_ID = "xunfeiAppId" // 讯飞AppID private const val KEY_XUNFEI_ACCESS_KEY_ID = "xunfeiAccessKeyId" // 讯飞AccessKeyID private const val KEY_XUNFEI_ACCESS_KEY_SECRET = "xunfeiAccessKeySecret" // 讯飞AccessKeySecret + // STS 模式参数键 + private const val KEY_AGENT_MODE = "agentMode" + private const val KEY_AGENT_ID = "agentId" + private const val KEY_STS_PRODUCT_ID = "stsProductId" + private const val KEY_STS_PRODUCT_KEY = "stsProductKey" + private const val KEY_STS_DEVICE_NO = "stsDeviceNo" + private const val KEY_STS_DEVICE_ID = "stsDeviceId" + private const val KEY_STS_DEVICE_SECRET = "stsDeviceSecret" // 用于跟踪服务是否正在运行 private val isRunning = AtomicBoolean(false) @@ -546,13 +554,15 @@ private var companionManager: BleCompanionManager? = null val config = getConfigFromPreferences(prefs) Log.d(TAG, "当前配置参数: $config") - // 检查必要的配置是否存在 - val azureSpeechKey = config[KEY_AZURE_SPEECH_KEY]?.toString() ?: "" - val openaiApiKey = config[KEY_OPENAI_API_KEY]?.toString() ?: "" - - if (azureSpeechKey.isEmpty() || openaiApiKey.isEmpty()) { - Log.w(TAG, "语音代理初始化失败: API密钥未配置") - return + // 检查必要的配置是否存在（STS 模式不需要 Azure/OpenAI 密钥） + val agentMode = config[KEY_AGENT_MODE]?.toString() ?: "stt_llm_tts" + if (agentMode != "sts") { + val azureSpeechKey = config[KEY_AZURE_SPEECH_KEY]?.toString() ?: "" + val openaiApiKey = config[KEY_OPENAI_API_KEY]?.toString() ?: "" + if (azureSpeechKey.isEmpty() || openaiApiKey.isEmpty()) { + Log.w(TAG, "语音代理初始化失败: API密钥未配置") + return + } } if (isChatAgentInitialized) { @@ -604,6 +614,13 @@ private var companionManager: BleCompanionManager? = null it.getStringExtra(KEY_XUNFEI_APP_ID)?.let { value -> config[KEY_XUNFEI_APP_ID] = value } it.getStringExtra(KEY_XUNFEI_ACCESS_KEY_ID)?.let { value -> config[KEY_XUNFEI_ACCESS_KEY_ID] = value } it.getStringExtra(KEY_XUNFEI_ACCESS_KEY_SECRET)?.let { value -> config[KEY_XUNFEI_ACCESS_KEY_SECRET] = value } + it.getStringExtra(KEY_AGENT_MODE)?.let { value -> config[KEY_AGENT_MODE] = value } + it.getStringExtra(KEY_AGENT_ID)?.let { value -> config[KEY_AGENT_ID] = value } + it.getStringExtra(KEY_STS_PRODUCT_ID)?.let { value -> config[KEY_STS_PRODUCT_ID] = value } + it.getStringExtra(KEY_STS_PRODUCT_KEY)?.let { value -> config[KEY_STS_PRODUCT_KEY] = value } + it.getStringExtra(KEY_STS_DEVICE_NO)?.let { value -> config[KEY_STS_DEVICE_NO] = value } + it.getStringExtra(KEY_STS_DEVICE_ID)?.let { value -> config[KEY_STS_DEVICE_ID] = value } + it.getStringExtra(KEY_STS_DEVICE_SECRET)?.let { value -> config[KEY_STS_DEVICE_SECRET] = value } Log.d(TAG, "liwei-------vipexptime 3-1 ${ config[KEY_VIP_EXPTIME]}") // 提取 supportedLanguages 参数 it.getStringArrayListExtra(KEY_SUPPORTED_LANGUAGES)?.let { languages -> @@ -671,6 +688,13 @@ private var companionManager: BleCompanionManager? = null config[KEY_XUNFEI_APP_ID] = getConfigString(prefs, KEY_XUNFEI_APP_ID, "") config[KEY_XUNFEI_ACCESS_KEY_ID] = getConfigString(prefs, KEY_XUNFEI_ACCESS_KEY_ID, "") config[KEY_XUNFEI_ACCESS_KEY_SECRET] = getConfigString(prefs, KEY_XUNFEI_ACCESS_KEY_SECRET, "") + config[KEY_AGENT_MODE] = getConfigString(prefs, KEY_AGENT_MODE, "stt_llm_tts") + config[KEY_AGENT_ID] = getConfigString(prefs, KEY_AGENT_ID, "default_agent") + config[KEY_STS_PRODUCT_ID] = getConfigString(prefs, KEY_STS_PRODUCT_ID, "") + config[KEY_STS_PRODUCT_KEY] = getConfigString(prefs, KEY_STS_PRODUCT_KEY, "") + config[KEY_STS_DEVICE_NO] = getConfigString(prefs, KEY_STS_DEVICE_NO, "") + config[KEY_STS_DEVICE_ID] = getConfigString(prefs, KEY_STS_DEVICE_ID, "") + config[KEY_STS_DEVICE_SECRET] = getConfigString(prefs, KEY_STS_DEVICE_SECRET, "") Log.d(TAG, "liwei-------vipexptime 3-0 ${ config[KEY_VIP_EXPTIME]}") // 添加 supportedLanguages 参数处理 val supportedLanguagesStr = getConfigString(prefs, KEY_SUPPORTED_LANGUAGES, "") diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt new file mode 100644 index 000000000..01f938ac8 --- /dev/null +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt @@ -0,0 +1,783 @@ +package com.yunqiinnovation.agent_service + +import android.content.Context +import android.media.AudioFormat +import android.media.AudioRecord +import android.media.MediaPlayer +import android.media.MediaRecorder +import android.os.Handler +import android.os.Looper +import android.util.Log +import com.yunqiinnovation.chat_storage.ChatStorageHelper +import kotlinx.coroutines.* +import okhttp3.* +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.RequestBody.Companion.toRequestBody +import okio.ByteString.Companion.toByteString +import org.json.JSONArray +import org.json.JSONObject +import java.io.IOException +import java.security.MessageDigest +import java.security.SecureRandom +import java.security.cert.X509Certificate +import java.util.UUID +import java.util.concurrent.TimeUnit +import javax.net.ssl.SSLContext +import javax.net.ssl.TrustManager +import javax.net.ssl.X509TrustManager +import kotlin.coroutines.resumeWithException +import kotlin.math.sqrt + +/** + * STS 配置参数（移动精灵 A80 平台） + */ +data class StsConfig( + val productId: String, + val productKey: String, + var deviceNo: String, + var deviceId: String, + var deviceSecret: String, + val agentId: String, +) + +/** + * StsAgent — 中国移动 A80 WS 一体化语音对话实现 + * + * 通过一条 WSS 连接完成 ASR + AI + TTS，事件以与标准 stt_llm_tts 相同的格式 + * 通过 [onEvent] 回调发出，供 AgentService 转发到 Flutter EventChannel。 + */ +class StsAgent( + private val context: Context, + private var config: StsConfig, + private val onEvent: (eventName: String, data: Map) -> Unit, + private val onCredentialsFetched: ((deviceId: String, deviceSecret: String, deviceNo: String) -> Unit)? = null, +) : CoroutineScope { + + private val TAG = "StsAgent" + private val job = SupervisorJob() + override val coroutineContext = Dispatchers.IO + job + + init { + fetchCredentials() + } + + // ── WebSocket ───────────────────────────────────────────────────────────── + private val okHttpClient: OkHttpClient = buildTrustAllClient() + private var webSocket: WebSocket? = null + private var readyReceived = false + private var sessionEnded = true + private var dialogRequestId = "" + + // ── 录音 ────────────────────────────────────────────────────────────────── + private val SAMPLE_RATE = 16000 + private val FRAME_SIZE = 5120 // 160ms @ 16kHz 16bit mono + private val RECORD_CHUNK = FRAME_SIZE / 2 // 读取块大小 + private val RECORD_BUFFER = maxOf( + AudioRecord.getMinBufferSize( + SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT + ), FRAME_SIZE * 2 + ) + private var audioRecord: AudioRecord? = null + private var recordingJob: Job? = null + private var micChunkCount = 0 + + // ── 发送缓冲 ────────────────────────────────────────────────────────────── + private val sendBuffer = ArrayDeque() + private var sendBufferOffset = 0 + + // ── 预缓冲（VAD 触发前 ~1.5s）──────────────────────────────────────────── + private val preBuffer = ArrayDeque() + private val PRE_BUFFER_MAX_CHUNKS = 75 + + // ── 本地 VAD ────────────────────────────────────────────────────────────── + private var speechDetected = false + private val VAD_THRESHOLD = 800.0 + private val VAD_ONSET_CHUNKS = 3 + private var vadOnsetCount = 0 + + // ── 状态标记 ────────────────────────────────────────────────────────────── + private var forwardingAudio = false + private var wsConnecting = false + private var inCall = false + private var pttRecording = false + + // ── TTS 播放队列 ────────────────────────────────────────────────────────── + private val ttsQueue = ArrayDeque() + private var ttsPlaying = false + private var ttsInterrupted = false + private var currentPlayer: MediaPlayer? = null + private val mainHandler = Handler(Looper.getMainLooper()) + /** 静音开关：设为 true 时停止当前播放、跳过后续 TTS，文本接收完后主动关闭连接 */ + var ttsMuted = false + set(value) { + field = value + if (value) { + // 立即停止当前播放 + ttsQueue.clear() + ttsPlaying = false + mainHandler.post { + try { currentPlayer?.stop(); currentPlayer?.release() } catch (_: Exception) {} + currentPlayer = null + } + emit("tts_canceled", emptyMap()) + // 如果会话已结束（文本已全部接收），主动关闭连接 + if (sessionEnded) { + closeWs() + if (inCall) { resetVad(); startIdleTimer() } + } + } + } + + // ── 消息历史（WS 上下文） ────────────────────────────────────────────────── + private var messages = JSONArray() + private val chatStorageHelper = ChatStorageHelper(context) + private var currentSessionId = "" + private var pendingAnswer = "" // 累积最完整的回复文本，会话结束时统一发送 + + // ── 计时器 ──────────────────────────────────────────────────────────────── + private var micWatchdogJob: Job? = null + private var lastWatchdogChunkCount = 0 + private var idleTimerJob: Job? = null + private val IDLE_TIMEOUT_MS = 15_000L + + // ── 凭证就绪 ────────────────────────────────────────────────────────────── + var credentialReady = false + private set + + // ═════════════════════════════════════════════════════════════════════════ + // 凭证获取 + // ═════════════════════════════════════════════════════════════════════════ + + fun fetchCredentials() { + if (config.deviceSecret.isNotEmpty()) { + credentialReady = true + return + } + launch { + try { + val body = JSONObject().apply { + put("deviceNoType", "SN") + put("deviceNo", config.deviceNo) + put("productId", config.productId) + put("productKey", config.productKey) + }.toString().toRequestBody("application/json".toMediaType()) + + val req = Request.Builder() + .url("https://ivs.chinamobiledevice.com:11443/v2/customer/device/secret/info") + .post(body) + .build() + + okHttpClient.newCall(req).execute().use { resp -> + val json = JSONObject(resp.body!!.string()) + if (json.getString("code") == "200" && json.getBoolean("success")) { + val d = json.getJSONObject("data") + config = config.copy( + deviceNo = d.optString("deviceNo", config.deviceNo), + deviceId = d.optString("deviceId", ""), + deviceSecret = d.optString("deviceSecret", ""), + ) + credentialReady = config.deviceSecret.isNotEmpty() + if (credentialReady) { + onCredentialsFetched?.invoke(config.deviceId, config.deviceSecret, config.deviceNo) + } + } else { + Log.e(TAG, "凭证获取失败: ${json.optString("message")}") + } + } + } catch (e: Exception) { + Log.e(TAG, "凭证获取异常: ${e.message}") + } + } + } + + // ═════════════════════════════════════════════════════════════════════════ + // 公开接口 + // ═════════════════════════════════════════════════════════════════════════ + + /** 通话模式：开启麦克风，VAD 自动触发 WS */ + fun startCall() { + if (inCall) return + if (ttsPlaying) interruptTts() + if (pttRecording) stopPtt(cancel = true) + if (!credentialReady) { emitError("移动精灵凭证未就绪，请稍后重试"); return } + openMic() + startMicWatchdog() + inCall = true + startIdleTimer() + } + + /** PTT 模式：立即连 WS 并开始录音 */ + fun startPtt() { + if (pttRecording) return + if (ttsPlaying) interruptTts() + if (!credentialReady) { emitError("移动精灵凭证未就绪"); return } + + pttRecording = true + openMic() + startMicWatchdog() + ttsInterrupted = false + closeWs() + sessionEnded = false + wsConnecting = true + dialogRequestId = UUID.randomUUID().toString() + clearSendBuffer() + pendingAnswer = "" + + launch { + try { + connectWs() + forwardingAudio = true + wsConnecting = false + emit("recognition_started", emptyMap()) + } catch (e: Exception) { + wsConnecting = false + pttRecording = false + stopMicWatchdog() + emitError("连接失败: ${e.message}") + } + } + } + + /** PTT 松开：发送剩余音频 + finish */ + fun stopPtt(cancel: Boolean = false) { + pttRecording = false + stopMicWatchdog() + if (cancel) { + forwardingAudio = false + closeWs() + stopMicHard() + return + } + if (forwardingAudio) { + forwardingAudio = false + synchronized(sendBufferLock) { + val remain = sendBuffer.size - sendBufferOffset + if (remain > 0) { + sendAudioChunk(sendBuffer.toMutableList().subList(sendBufferOffset, sendBuffer.size).toByteArray()) + } + sendBuffer.clear(); sendBufferOffset = 0 + } + sendFinish() + emit("recognition_stopped", emptyMap()) + } + stopMicHard() + } + + /** 统一停止：根据当前状态自动选择 stopPtt 或 disconnect */ + fun stop() { + if (pttRecording) stopPtt() + else disconnect() + } + + /** 挂断通话 */ + fun disconnect() { + stopMicWatchdog() + cancelIdleTimer() + forwardingAudio = false + wsConnecting = false + speechDetected = false + stopMicHard() + ttsInterrupted = true + ttsQueue.clear() + ttsPlaying = false + mainHandler.post { + try { currentPlayer?.stop(); currentPlayer?.release() } catch (_: Exception) {} + currentPlayer = null + } + closeWs() + preBuffer.clear() + clearSendBuffer() + flushPendingAnswer() + inCall = false + pttRecording = false + } + + /** 打断 AI 播报 */ + fun interruptSpeaking() { + interruptTts() + ttsInterrupted = false + if (inCall) resetVad() + } + + /** 清空消息历史 */ + fun clearHistory() { + messages = JSONArray() + } + + /** 释放资源 */ + fun dispose() { + disconnect() + job.cancel() + } + + // ═════════════════════════════════════════════════════════════════════════ + // 麦克风 & 本地 VAD + // ═════════════════════════════════════════════════════════════════════════ + + private fun openMic() { + stopMicHard() + speechDetected = false; vadOnsetCount = 0; micChunkCount = 0 + try { + audioRecord = AudioRecord( + MediaRecorder.AudioSource.VOICE_COMMUNICATION, + SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + RECORD_BUFFER + ) + audioRecord?.startRecording() + recordingJob = launch { + val buf = ByteArray(RECORD_CHUNK) + while (isActive && (inCall || pttRecording)) { + val read = audioRecord?.read(buf, 0, buf.size) ?: break + if (read > 0) { micChunkCount++; onMicAudio(buf.copyOf(read)) } + } + } + } catch (e: Exception) { + Log.e(TAG, "打开麦克风失败: ${e.message}") + } + } + + private fun stopMicHard() { + recordingJob?.cancel(); recordingJob = null + try { audioRecord?.stop(); audioRecord?.release() } catch (_: Exception) {} + audioRecord = null + } + + private fun startMicWatchdog() { + stopMicWatchdog() + lastWatchdogChunkCount = micChunkCount + micWatchdogJob = launch { + while (isActive && (inCall || pttRecording)) { + delay(1500) + if (micChunkCount == lastWatchdogChunkCount && !ttsPlaying) openMic() + lastWatchdogChunkCount = micChunkCount + } + } + } + + private fun stopMicWatchdog() { micWatchdogJob?.cancel(); micWatchdogJob = null } + + private fun onMicAudio(chunk: ByteArray) { + if (!inCall && !pttRecording) return + if (forwardingAudio && webSocket != null) { + if (micChunkCount % 3 == 0) emit("mic_rms", mapOf("rms" to calcRms(chunk))) + feedSendBuffer(chunk); return + } + if (wsConnecting) { addToPreBuffer(chunk); return } + if (ttsPlaying) return + + addToPreBuffer(chunk) + val rms = calcRms(chunk) + // 发送麦克风音量给 Dart 层 + if (micChunkCount % 3 == 0) { + emit("mic_rms", mapOf("rms" to rms)) + } + if (rms >= VAD_THRESHOLD) { + vadOnsetCount++ + if (vadOnsetCount >= VAD_ONSET_CHUNKS && !speechDetected) { + speechDetected = true + cancelIdleTimer() + launch { onSpeechDetected() } + } + } else { + vadOnsetCount = 0 + } + } + + private fun calcRms(pcm: ByteArray): Double { + if (pcm.size < 2) return 0.0 + val samples = pcm.size / 2 + var sum = 0.0 + for (i in 0 until pcm.size - 1 step 2) { + var s = (pcm[i].toInt() and 0xFF) or (pcm[i + 1].toInt() shl 8) + if (s >= 32768) s -= 65536 + sum += s.toLong() * s.toLong() + } + return sqrt(sum / samples) + } + + private fun addToPreBuffer(chunk: ByteArray) { + preBuffer.addLast(chunk.copyOf()) + while (preBuffer.size > PRE_BUFFER_MAX_CHUNKS) preBuffer.removeFirst() + } + + // ═════════════════════════════════════════════════════════════════════════ + // VAD 触发 → WS 连接 + // ═════════════════════════════════════════════════════════════════════════ + + private suspend fun onSpeechDetected() { + if (wsConnecting) return + ttsInterrupted = false + closeWs() + sessionEnded = false + wsConnecting = true + dialogRequestId = UUID.randomUUID().toString() + clearSendBuffer() + pendingAnswer = "" + + try { + connectWs() + } catch (e: Exception) { + wsConnecting = false; speechDetected = false + emitError("连接失败: ${e.message}") + return + } + + forwardingAudio = true; wsConnecting = false + emit("recognition_started", emptyMap()) + + val cached = preBuffer.toList(); preBuffer.clear() + for (c in cached) feedSendBuffer(c) + } + + // ═════════════════════════════════════════════════════════════════════════ + // WebSocket + // ═════════════════════════════════════════════════════════════════════════ + + private suspend fun connectWs(): Unit = suspendCancellableCoroutine { cont -> + val ts = System.currentTimeMillis() + val sign = calcSign(config.deviceSecret, ts.toString()) + val sn = UUID.randomUUID().toString() + val url = "wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr" + + "?deviceNo=${config.deviceNo}&deviceId=${config.deviceId}" + + "&productId=${config.productId}&productKey=${config.productKey}" + + "&sign=$sign&sn=$sn&ts=$ts" + + val req = Request.Builder().url(url) + .header("Origin", "https://ivs.chinamobiledevice.com:11443") + .header("Host", "ivs.chinamobiledevice.com:11443") + .build() + + var resolved = false + + webSocket = okHttpClient.newWebSocket(req, object : WebSocketListener() { + override fun onOpen(ws: WebSocket, response: Response) { + sendStartFrame() + } + + override fun onMessage(ws: WebSocket, text: String) { + try { + val msg = JSONObject(text) + if (msg.optString("type") == "ready") { + if (!resolved) { resolved = true; cont.resume(Unit) {} } + } else { + launch { handleMessage(msg) } + } + } catch (_: Exception) {} + } + + override fun onMessage(ws: WebSocket, bytes: okio.ByteString) {} + + override fun onFailure(ws: WebSocket, t: Throwable, response: Response?) { + webSocket = null + if (!resolved) { resolved = true; cont.resumeWithException(t) } + else launch { handleWsDone() } + } + + override fun onClosing(ws: WebSocket, code: Int, reason: String) { ws.close(1000, null) } + + override fun onClosed(ws: WebSocket, code: Int, reason: String) { + webSocket = null + if (!resolved) { resolved = true; cont.resumeWithException(IOException("WS closed")) } + else launch { handleWsDone() } + } + }) + + cont.invokeOnCancellation { closeWs() } + + // 15s 超时 + launch { + delay(15_000) + if (!resolved) { + resolved = true; closeWs() + cont.resumeWithException(IOException("等待 ready 超时")) + } + } + } + + private fun handleWsDone() { + webSocket = null; forwardingAudio = false; wsConnecting = false + when { + sessionEnded && inCall -> if (!ttsPlaying) { resetVad(); startIdleTimer() } + sessionEnded -> { /* PTT 结束，等 TTS 播完 */ } + inCall -> resetVad() + } + } + + private fun resetVad() { + speechDetected = false; vadOnsetCount = 0; preBuffer.clear(); openMic() + } + + private fun closeWs() { + try { webSocket?.cancel() } catch (_: Exception) {} + webSocket = null + } + + // ═════════════════════════════════════════════════════════════════════════ + // 发送帧 + // ═════════════════════════════════════════════════════════════════════════ + + private fun sendStartFrame() { + currentSessionId = UUID.randomUUID().toString() + val payload = JSONObject().apply { + put("type", "start") + put("data", JSONObject().apply { + put("cuid", config.deviceId) + put("format", "pcm") + put("sample", 16000) + put("support_dcs", 2) + put("chunk_size", 10240) + put("support_tts", true) + put("support_text2dcs", true) + put("user_agent", "voitrans/1.0") + put("dialog_request_id", dialogRequestId) + put("access_rc", true) + put("support_part_tts", true) + put("need_stoplisten", true) + put("need_dialogue_finish", true) + put("result_trans2directive", false) + put("rc_version", "5") + put("messages", messages) + put("client_context", JSONArray()) + }) + } + webSocket?.send(payload.toString()) + } + + private val sendBufferLock = Any() + + private fun clearSendBuffer() { + synchronized(sendBufferLock) { + sendBuffer.clear(); sendBufferOffset = 0 + } + } + + private fun feedSendBuffer(chunk: ByteArray) { + synchronized(sendBufferLock) { + sendBuffer.addAll(chunk.toList()) + while (sendBuffer.size - sendBufferOffset >= FRAME_SIZE) { + val frame = ByteArray(FRAME_SIZE) { i -> sendBuffer[sendBufferOffset + i] } + sendBufferOffset += FRAME_SIZE + sendAudioChunk(frame) + } + // 收缩缓冲 + if (sendBufferOffset >= FRAME_SIZE * 8) { + val toRemove = sendBufferOffset + repeat(toRemove) { if (sendBuffer.isNotEmpty()) sendBuffer.removeFirst() } + sendBufferOffset = 0 + } + } + } + + private fun sendAudioChunk(data: ByteArray) { + try { webSocket?.send(data.toByteString()) } catch (_: Exception) {} + } + + private fun sendFinish() { + try { webSocket?.send("""{"type":"finish"}""") } catch (_: Exception) {} + } + + // ═════════════════════════════════════════════════════════════════════════ + // 消息处理 + // ═════════════════════════════════════════════════════════════════════════ + + private suspend fun handleMessage(msg: JSONObject) { + when (msg.optString("type")) { + "stoplisten" -> { + forwardingAudio = false; speechDetected = false; vadOnsetCount = 0 + synchronized(sendBufferLock) { + val remain = sendBuffer.size - sendBufferOffset + if (remain > 0) { + sendAudioChunk(ByteArray(remain) { i -> sendBuffer[sendBufferOffset + i] }) + } + sendBuffer.clear(); sendBufferOffset = 0 + } + sendFinish() + emit("recognition_stopped", emptyMap()) + } + "mid_result" -> { + val text = msg.optString("result", "") + if (text.isNotEmpty()) emit("recognizing", mapOf("text" to text)) + } + "fin_result" -> { + val result = msg.optString("result", "") + if (result.isNotEmpty()) { + emit("recognition_result", mapOf("text" to result)) + messages.put(JSONObject(mapOf("role" to "user", "content" to result))) + withContext(Dispatchers.IO) { + try { + chatStorageHelper.saveMessage(config.agentId, currentSessionId, result, "user") + } catch (_: Exception) {} + } + } + if (msg.optInt("end", 0) == 1) markSessionEnded() + } + "inside_rc" -> handleInsideRc(msg) + "dcs_decide" -> if (msg.optInt("end", 0) == 1) markSessionEnded() + } + } + + private suspend fun handleInsideRc(msg: JSONObject) { + val data = msg.optJSONObject("data") ?: return + if (data.optInt("code", -1) != 0) return + + val rawAnswer = data.optString("assistant_answer", "") + val cleanAnswer = try { + JSONObject(rawAnswer).optString("content", rawAnswer) + } catch (_: Exception) { rawAnswer } + + var ttsUrl: String? = null + val items = data.optJSONArray("data") + if (items != null) { + for (i in 0 until items.length()) { + val item = items.optJSONObject(i) ?: continue + val h = item.optJSONObject("header") ?: continue + if (h.optString("namespace") == "ai.fxzsos.device_interface.voice_output" && + h.optString("name") == "Speak" + ) { + ttsUrl = item.optJSONObject("payload")?.optString("url") + break + } + } + } + + // 每次取更完整的回复文本，立即发送给 UI + if (cleanAnswer.isNotEmpty() && cleanAnswer.length > pendingAnswer.length) { + pendingAnswer = cleanAnswer + emit("assistant_response", mapOf("response" to cleanAnswer, "sessionid" to currentSessionId)) + } + ttsUrl?.takeIf { it.isNotEmpty() }?.let { enqueueTts(it) } + if (msg.optInt("end", 0) == 1) markSessionEnded() + } + + /** 将 pendingAnswer 保存到历史和存储，并清空 */ + private fun flushPendingAnswer() { + if (pendingAnswer.isEmpty()) return + val answer = pendingAnswer + val sid = currentSessionId + messages.put(JSONObject(mapOf("role" to "assistant", "content" to answer))) + launch(Dispatchers.IO) { + try { + chatStorageHelper.saveMessage(config.agentId, sid, answer, "assistant") + } catch (_: Exception) {} + } + pendingAnswer = "" + } + + private fun markSessionEnded() { + if (sessionEnded) return + sessionEnded = true; forwardingAudio = false; speechDetected = false; vadOnsetCount = 0 + clearSendBuffer() + flushPendingAnswer() + // 静音模式下文本接收完毕，发送完成事件并关闭连接 + if (ttsMuted) { + emit("tts_completed", emptyMap()) + closeWs() + if (inCall) { resetVad(); startIdleTimer() } + } + } + + // ═════════════════════════════════════════════════════════════════════════ + // TTS 播放队列 + // ═════════════════════════════════════════════════════════════════════════ + + private fun enqueueTts(url: String) { + if (ttsMuted) return // 静音模式跳过 TTS + ttsQueue.addLast(url) + if (!ttsPlaying) { + stopMicHard() + mainHandler.post { playNextTts() } + } + } + + private fun playNextTts() { + if (ttsQueue.isEmpty() || ttsInterrupted) { + ttsPlaying = false + if (!ttsInterrupted) { + emit("tts_completed", emptyMap()) + if (inCall) { resetVad(); startIdleTimer() } + } + return + } + ttsPlaying = true + emit("tts_started", emptyMap()) + val url = ttsQueue.removeFirst() + try { + currentPlayer?.stop(); currentPlayer?.release() + currentPlayer = MediaPlayer().apply { + setDataSource(url) + setOnCompletionListener { + if (!ttsInterrupted) mainHandler.post { playNextTts() } + } + setOnErrorListener { _, _, _ -> + if (!ttsInterrupted) mainHandler.post { playNextTts() } + true + } + prepare() + start() + } + } catch (e: Exception) { + Log.e(TAG, "TTS 播放失败: ${e.message}") + if (!ttsInterrupted) mainHandler.post { playNextTts() } + } + } + + private fun interruptTts() { + ttsInterrupted = true; ttsQueue.clear(); ttsPlaying = false + mainHandler.post { + try { currentPlayer?.stop(); currentPlayer?.release() } catch (_: Exception) {} + currentPlayer = null + } + emit("tts_canceled", emptyMap()) + } + + // ═════════════════════════════════════════════════════════════════════════ + // 超时 & 工具 + // ═════════════════════════════════════════════════════════════════════════ + + private fun startIdleTimer() { + cancelIdleTimer() + idleTimerJob = launch { + delay(IDLE_TIMEOUT_MS) + if (inCall && !speechDetected) disconnect() + } + } + + private fun cancelIdleTimer() { idleTimerJob?.cancel(); idleTimerJob = null } + + private fun calcSign(secret: String, ts: String): String { + val digest = MessageDigest.getInstance("MD5").digest("$secret$ts".toByteArray()) + return digest.joinToString("") { "%02x".format(it) } + } + + private fun emit(eventName: String, data: Map) { + launch(Dispatchers.Main) { onEvent(eventName, data) } + } + + private fun emitError(msg: String) = emit("error", mapOf("message" to msg)) + + private fun buildTrustAllClient(): OkHttpClient { + return try { + val trustAll = arrayOf(object : X509TrustManager { + override fun checkClientTrusted(c: Array, t: String) {} + override fun checkServerTrusted(c: Array, t: String) {} + override fun getAcceptedIssuers(): Array = emptyArray() + }) + val sc = SSLContext.getInstance("TLS") + sc.init(null, trustAll, SecureRandom()) + OkHttpClient.Builder() + .sslSocketFactory(sc.socketFactory, trustAll[0] as X509TrustManager) + .hostnameVerifier { _, _ -> true } + .connectTimeout(15, TimeUnit.SECONDS) + .readTimeout(90, TimeUnit.SECONDS) + .writeTimeout(90, TimeUnit.SECONDS) + .build() + } catch (e: Exception) { + Log.e(TAG, "buildTrustAllClient 失败，使用默认客户端", e) + OkHttpClient() + } + } +} diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index fd135db40..034f6b72d 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -66,8 +66,12 @@ class AgentServiceImpl: NSObject { return ChatStorageHelper.shared } + // Agent 模式 + private var agentMode = "stt_llm_tts" + private var stsAgent: StsAgent? + // 会话ID，用于区分不同聊天上下文 - private let agentId = "default_agent" + private var agentId = "default_agent" //工具调用提示 public var callingToolText = "" // 工具调用提示 public var insufficientIntegralText = "" // 积分不足提示 @@ -194,9 +198,38 @@ class AgentServiceImpl: NSObject { func initialize(config: [String: Any]) -> Bool { // if isInitialized { return true } // os_log("initialize: config=%{public}@", log: logger, type: .info, config) - + + agentMode = config["agentMode"] as? String ?? "stt_llm_tts" + agentId = (config["agentId"] as? String)?.isEmpty == false + ? config["agentId"] as! String : "default_agent" + + // STS 模式：跳过 Azure/OpenAI 初始化 + if agentMode == "sts" { + let stsConfig = StsConfig( + productId: config["stsProductId"] as? String ?? "", + productKey: config["stsProductKey"] as? String ?? "", + deviceNo: config["stsDeviceNo"] as? String ?? "", + deviceId: config["stsDeviceId"] as? String ?? "", + deviceSecret: config["stsDeviceSecret"] as? String ?? "", + agentId: agentId + ) + stsAgent = StsAgent( + config: stsConfig, + onEvent: { [weak self] name, data in self?.sendEvent(name: name, data: data) }, + onCredentialsFetched: { [weak self] deviceId, deviceSecret, deviceNo in + self?.sendEvent(name: "sts_credentials", data: [ + "deviceId": deviceId, + "deviceSecret": deviceSecret, + "deviceNo": deviceNo, + ]) + } + ) + isInitialized = true + return true + } + self.config = config - + if let navigationMode = config["navigationMode"] as? String { self.navigationMode = navigationMode } @@ -490,6 +523,17 @@ class AgentServiceImpl: NSObject { } } func startRecognition(useBle: Bool = false, mode: String = "normal") -> Bool { + if agentMode == "sts" { + switch mode { + case "push_to_talk": + stsAgent?.startPtt() + case "phone_call", "ble_wakeup": + stsAgent?.startCall() + default: + stsAgent?.startPtt() + } + return true + } if !isInitialized { sendError("服务未初始化", code: "NOT_INITIALIZED") return false @@ -575,6 +619,10 @@ class AgentServiceImpl: NSObject { } func stopRecognition() -> Bool { + if agentMode == "sts" { + stsAgent?.stop() + return true + } /// 函数功能：停止语音识别 /// 特性： /// 1) 即便处于"启动中"（isStartingRecognition）也会打印日志并尝试停止 @@ -679,8 +727,12 @@ class AgentServiceImpl: NSObject { } func interruptCurrentResponse() -> Bool { + if agentMode == "sts" { + stsAgent?.interruptSpeaking() + return true + } var interrupted = false - + if isSpeaking { interrupted = stopTts() || interrupted } @@ -1490,8 +1542,12 @@ private func jsonToString(_ json: [String: Any]) -> String? { return result } func clearChatHistory() -> Bool { + if agentMode == "sts" { + stsAgent?.clearHistory() + return true + } chatHistory.removeAll() - + DispatchQueue.global(qos: .utility).async { let success = self.chatStorageHelper.deleteMessages(agentId:self.agentId, messageIds: nil as [Int]?) if !success { @@ -1738,6 +1794,12 @@ private func jsonToString(_ json: [String: Any]) -> String? { func dispose() -> Bool { print("ai释放资源") + if agentMode == "sts" { + stsAgent?.dispose() + stsAgent = nil + isInitialized = false + return true + } if isRecognizing { // 关闭编码器 (设备侧) BleService.shared.closeCodec() diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/StsAgent.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/StsAgent.swift new file mode 100644 index 000000000..a5e50eb78 --- /dev/null +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/StsAgent.swift @@ -0,0 +1,495 @@ +import Foundation +import AVFoundation +import CryptoKit +import os.log +import chat_storage + +// MARK: - StsConfig + +struct StsConfig { + let productId: String + let productKey: String + var deviceNo: String + var deviceId: String + var deviceSecret: String + let agentId: String +} + +// MARK: - StsAgent + +/// StsAgent — 中国移动 A80 WS 一体化语音对话实现（iOS） +/// +/// 通过一条 WSS 连接完成 ASR + AI + TTS，事件格式与标准 stt_llm_tts 完全一致， +/// 通过 onEvent 回调发出，供 AgentServiceImpl 转发到 Flutter EventChannel。 +class StsAgent: NSObject { + + private let TAG = "StsAgent" + private let logger = OSLog(subsystem: "com.yunqiinnovation.agent_service", category: "StsAgent") + + private var config: StsConfig + private let onEvent: (String, [String: Any]) -> Void + private let onCredentialsFetched: ((String, String, String) -> Void)? + + // MARK: WebSocket + private var webSocketTask: URLSessionWebSocketTask? + private var urlSession: URLSession! + private var readyReceived = false + private var sessionEnded = true + private var dialogRequestId = "" + + // MARK: 录音 + private let SAMPLE_RATE: Double = 16000 + private let FRAME_SIZE = 5120 // 160ms @ 16kHz 16bit mono + private var audioEngine: AVAudioEngine? + private var inputNode: AVAudioInputNode? + private var recordingActive = false + private var sendBuffer = Data() + private var micChunkCount = 0 + + // MARK: VAD + private let VAD_THRESHOLD: Float = 800 + private let VAD_ONSET_CHUNKS = 3 + private var vadOnsetCount = 0 + private var vadActive = false + private var preBuffer: [[UInt8]] = [] + private let PRE_BUFFER_MAX = 75 + + // MARK: TTS + private var ttsQueue: [String] = [] + private var ttsPlayer: AVPlayer? + private var ttsPlaying = false + private var ttsPlayerObserver: Any? + + // MARK: 历史 + private var messages: [[String: String]] = [] + private var currentSessionId = "" + private let chatStorage = ChatStorageHelper.shared + + // MARK: 模式 + private var isPttMode = false + + // MARK: 凭证请求 + private var credentialsFetched = false + + init(config: StsConfig, + onEvent: @escaping (String, [String: Any]) -> Void, + onCredentialsFetched: ((String, String, String) -> Void)? = nil) { + self.config = config + self.onEvent = onEvent + self.onCredentialsFetched = onCredentialsFetched + super.init() + let sessionConfig = URLSessionConfiguration.default + urlSession = URLSession(configuration: sessionConfig, delegate: nil, delegateQueue: OperationQueue()) + } + + // MARK: - 公共接口 + + func startCall() { + isPttMode = false + os_log("StsAgent startCall", log: logger, type: .info) + fetchCredentialsIfNeeded { [weak self] in + self?.connect() + } + } + + func startPtt() { + isPttMode = true + os_log("StsAgent startPtt", log: logger, type: .info) + fetchCredentialsIfNeeded { [weak self] in + self?.connect() + } + } + + func stopPtt() { + os_log("StsAgent stopPtt", log: logger, type: .info) + guard isPttMode else { return } + sendFinishFrame() + } + + /// 统一停止：根据当前模式自动选择 stopPtt 或 dispose + func stop() { + if isPttMode { + stopPtt() + } else { + dispose() + } + } + + func interruptSpeaking() { + os_log("StsAgent interruptSpeaking", log: logger, type: .info) + stopTts() + emit("tts_canceled") + } + + func clearHistory() { + messages.removeAll() + DispatchQueue.global().async { [weak self] in + guard let self = self else { return } + _ = self.chatStorage.deleteMessages(agentId: self.config.agentId, messageIds: nil as [Int]?) + } + } + + func dispose() { + os_log("StsAgent dispose", log: logger, type: .info) + stopRecording() + webSocketTask?.cancel(with: .normalClosure, reason: nil) + webSocketTask = nil + stopTts() + } + + // MARK: - 凭证获取 + + private func fetchCredentialsIfNeeded(completion: @escaping () -> Void) { + if credentialsFetched && !config.deviceId.isEmpty && !config.deviceSecret.isEmpty { + completion() + return + } + let ts = Int64(Date().timeIntervalSince1970 * 1000) + let sign = calcSign(secret: config.productKey, ts: ts) + let urlStr = "https://ivs.chinamobiledevice.com:11443/v2/customer/device/secret/info" + guard var components = URLComponents(string: urlStr) else { completion(); return } + components.queryItems = [ + URLQueryItem(name: "productId", value: config.productId), + URLQueryItem(name: "deviceNo", value: config.deviceNo), + URLQueryItem(name: "ts", value: "\(ts)"), + URLQueryItem(name: "sign", value: sign), + ] + guard let url = components.url else { completion(); return } + var req = URLRequest(url: url) + req.timeoutInterval = 10 + URLSession.shared.dataTask(with: req) { [weak self] data, _, error in + guard let self = self else { return } + if let error = error { + os_log("凭证请求失败: %{public}@", log: self.logger, type: .error, error.localizedDescription) + completion() + return + } + guard let data = data, + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let body = json["body"] as? [String: Any] else { + completion() + return + } + if let deviceId = body["deviceId"] as? String { self.config.deviceId = deviceId } + if let deviceSecret = body["deviceSecret"] as? String { self.config.deviceSecret = deviceSecret } + if let deviceNo = body["deviceNo"] as? String { self.config.deviceNo = deviceNo } + self.credentialsFetched = true + self.onCredentialsFetched?(self.config.deviceId, self.config.deviceSecret, self.config.deviceNo) + completion() + }.resume() + } + + // MARK: - WebSocket 连接 + + private func connect() { + let ts = Int64(Date().timeIntervalSince1970 * 1000) + let sign = calcSign(secret: config.deviceSecret, ts: ts) + let urlStr = "wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr" + guard var components = URLComponents(string: urlStr) else { return } + components.queryItems = [ + URLQueryItem(name: "productId", value: config.productId), + URLQueryItem(name: "deviceId", value: config.deviceId), + URLQueryItem(name: "ts", value: "\(ts)"), + URLQueryItem(name: "sign", value: sign), + ] + guard let url = components.url else { return } + webSocketTask = urlSession.webSocketTask(with: url) + webSocketTask?.resume() + receiveMessages() + + // 发送 start 帧 + sendStartFrame() + } + + private func sendStartFrame() { + dialogRequestId = UUID().uuidString.replacingOccurrences(of: "-", with: "").lowercased() + currentSessionId = dialogRequestId + let payload: [String: Any] = [ + "msgType": "start", + "dialogRequestId": dialogRequestId, + "params": [ + "asrParams": [ + "format": "pcm", + "sampleRate": 16000, + "lang": "zh-CN", + "vadEnable": false, + ], + "contextParams": [ + "history": messages.map { ["role": $0["role"] ?? "", "content": $0["content"] ?? ""] } + ] + ] + ] + sendJson(payload) + } + + private func receiveMessages() { + webSocketTask?.receive { [weak self] result in + guard let self = self else { return } + switch result { + case .success(let msg): + switch msg { + case .string(let text): + self.handleMessage(text) + default: + break + } + self.receiveMessages() + case .failure(let err): + os_log("WS error: %{public}@", log: self.logger, type: .error, err.localizedDescription) + self.emit("error", data: ["message": err.localizedDescription]) + } + } + } + + // MARK: - 消息处理 + + private func handleMessage(_ text: String) { + guard let data = text.data(using: .utf8), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return } + let msgType = json["msgType"] as? String ?? "" + os_log("WS recv: %{public}@", log: logger, type: .debug, msgType) + + switch msgType { + case "ready": + readyReceived = true + sessionEnded = false + emit("recognition_started") + startRecording() + if isPttMode { + // PTT 模式等待 stopPtt() 调用发送 finish + } + + case "mid_result": + let result = (json["data"] as? [String: Any])?["result"] as? String ?? "" + if !result.isEmpty { emit("recognizing", data: ["text": result]) } + + case "fin_result": + let result = (json["data"] as? [String: Any])?["result"] as? String ?? "" + emit("recognition_result", data: ["text": result]) + emit("recognition_stopped") + stopRecording() + if !result.isEmpty { + messages.append(["role": "user", "content": result]) + saveToChatStorage(text: result, role: "user") + } + + case "stoplisten": + stopRecording() + emit("recognition_stopped") + + case "inside_rc": + handleInsideRc(json["data"] as? [String: Any]) + + case "dcs_decide": + // 语义决策，忽略 + break + + default: + os_log("未处理的消息类型: %{public}@", log: logger, type: .info, msgType) + } + } + + private func handleInsideRc(_ data: [String: Any]?) { + guard let data = data else { return } + guard let rcType = data["rcType"] as? String else { return } + if rcType == "assistant_answer" { + let answer = (data["answer"] as? [String: Any])?["text"] as? String ?? "" + let cleanAnswer = answer.trimmingCharacters(in: .whitespacesAndNewlines) + let sid = data["sessionId"] as? String ?? currentSessionId + if !cleanAnswer.isEmpty { + messages.append(["role": "assistant", "content": cleanAnswer]) + saveToChatStorage(text: cleanAnswer, role: "assistant") + emit("assistant_response", data: ["response": cleanAnswer, "sessionid": sid]) + } + // TTS URLs + if let ttsData = data["tts"] as? [[String: Any]] { + for item in ttsData { + if let url = item["url"] as? String, !url.isEmpty { + ttsQueue.append(url) + } + } + if !ttsPlaying { playNextTts() } + } + } + } + + // MARK: - TTS 播放 + + private func playNextTts() { + guard !ttsQueue.isEmpty else { + ttsPlaying = false + emit("tts_completed") + return + } + let urlStr = ttsQueue.removeFirst() + guard let url = URL(string: urlStr) else { playNextTts(); return } + + ttsPlaying = true + if ttsQueue.isEmpty && !ttsPlaying { emit("tts_started") } + emit("tts_started") + + let item = AVPlayerItem(url: url) + ttsPlayer = AVPlayer(playerItem: item) + if let observer = ttsPlayerObserver { + NotificationCenter.default.removeObserver(observer) + } + ttsPlayerObserver = NotificationCenter.default.addObserver( + forName: .AVPlayerItemDidPlayToEndTime, object: item, queue: .main + ) { [weak self] _ in + self?.playNextTts() + } + ttsPlayer?.play() + } + + private func stopTts() { + ttsPlayer?.pause() + ttsPlayer = nil + ttsQueue.removeAll() + ttsPlaying = false + if let observer = ttsPlayerObserver { + NotificationCenter.default.removeObserver(observer) + ttsPlayerObserver = nil + } + } + + // MARK: - 录音 + + private func startRecording() { + guard !recordingActive else { return } + let engine = AVAudioEngine() + audioEngine = engine + inputNode = engine.inputNode + let fmt = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: SAMPLE_RATE, channels: 1, interleaved: true)! + let hwFmt = inputNode!.outputFormat(forBus: 0) + let converterFmt = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: SAMPLE_RATE, channels: 1, interleaved: true)! + + inputNode!.installTap(onBus: 0, bufferSize: 4096, format: hwFmt) { [weak self] buffer, _ in + guard let self = self, self.recordingActive else { return } + // 转换为 16kHz PCM16 + guard let cvt = AVAudioConverter(from: hwFmt, to: converterFmt) else { return } + let frameCount = AVAudioFrameCount(Double(buffer.frameLength) * self.SAMPLE_RATE / hwFmt.sampleRate) + guard let outBuf = AVAudioPCMBuffer(pcmFormat: converterFmt, frameCapacity: frameCount) else { return } + var error: NSError? + cvt.convert(to: outBuf, error: &error) { _, outStatus in + outStatus.pointee = .haveData + return buffer + } + if error != nil { return } + let byteCount = Int(outBuf.frameLength) * 2 + let ptr = outBuf.int16ChannelData![0] + let data = Data(bytes: ptr, count: byteCount) + self.onMicAudio(data) + } + + do { + try AVAudioSession.sharedInstance().setCategory(.playAndRecord, mode: .voiceChat, options: [.defaultToSpeaker]) + try AVAudioSession.sharedInstance().setActive(true) + try engine.start() + recordingActive = true + } catch { + os_log("录音启动失败: %{public}@", log: logger, type: .error, error.localizedDescription) + } + } + + private func stopRecording() { + guard recordingActive else { return } + recordingActive = false + inputNode?.removeTap(onBus: 0) + audioEngine?.stop() + audioEngine = nil + inputNode = nil + sendBuffer.removeAll() + micChunkCount = 0 + vadActive = false + vadOnsetCount = 0 + preBuffer.removeAll() + } + + private func onMicAudio(_ pcm: Data) { + micChunkCount += 1 + let rms = calcRms(pcm) + let chunk = [UInt8](pcm) + + if !vadActive { + if preBuffer.count >= PRE_BUFFER_MAX { preBuffer.removeFirst() } + preBuffer.append(chunk) + } + + if !vadActive && rms > VAD_THRESHOLD { + vadOnsetCount += 1 + if vadOnsetCount >= VAD_ONSET_CHUNKS { + vadActive = true + // 先发预缓存 + for c in preBuffer { feedSendBuffer(Data(c)) } + preBuffer.removeAll() + } + } else if vadActive { + feedSendBuffer(pcm) + } + } + + private func calcRms(_ pcm: Data) -> Float { + let count = pcm.count / 2 + guard count > 0 else { return 0 } + var sum: Float = 0 + pcm.withUnsafeBytes { ptr in + let samples = ptr.bindMemory(to: Int16.self) + for i in 0..= FRAME_SIZE { + let frame = sendBuffer.prefix(FRAME_SIZE) + sendBuffer.removeFirst(FRAME_SIZE) + webSocketTask?.send(.data(Data(frame))) { _ in } + } + } + + private func sendFinishFrame() { + let payload: [String: Any] = [ + "msgType": "finish", + "dialogRequestId": dialogRequestId, + ] + sendJson(payload) + } + + // MARK: - 工具 + + private func sendJson(_ dict: [String: Any]) { + guard let data = try? JSONSerialization.data(withJSONObject: dict), + let str = String(data: data, encoding: .utf8) else { return } + webSocketTask?.send(.string(str)) { _ in } + } + + private func calcSign(secret: String, ts: Int64) -> String { + let raw = "\(secret)\(ts)" + let data = Data(raw.utf8) + let digest = Insecure.MD5.hash(data: data) + return digest.map { String(format: "%02x", $0) }.joined() + } + + private func emit(_ name: String, data: [String: Any] = [:]) { + DispatchQueue.main.async { [weak self] in + self?.onEvent(name, data) + } + } + + private func saveToChatStorage(text: String, role: String) { + DispatchQueue.global().async { [weak self] in + guard let self = self else { return } + _ = self.chatStorage.saveMessage( + agentId: self.config.agentId, + sessionId: self.currentSessionId, + message: text, + sender: role, + metadata: nil + ) + } + } +} diff --git a/local_plugins/agent_service/lib/agent_service.dart b/local_plugins/agent_service/lib/agent_service.dart index 92dd7cafa..1cdec8d65 100644 --- a/local_plugins/agent_service/lib/agent_service.dart +++ b/local_plugins/agent_service/lib/agent_service.dart @@ -71,6 +71,12 @@ enum AgentServiceEventType { /// 错误 error, + /// STS 凭证获取成功（设备 deviceId/deviceSecret） + stsCredentials, + + /// 麦克风音量（RMS） + micRms, + /// 未知事件 unknown } @@ -176,6 +182,10 @@ class AgentService { return AgentServiceEventType.imageProcessing; case 'image_ready': return AgentServiceEventType.imageReady; + case 'sts_credentials': + return AgentServiceEventType.stsCredentials; + case 'mic_rms': + return AgentServiceEventType.micRms; default: return AgentServiceEventType.unknown; } @@ -446,6 +456,18 @@ class AgentService { } } + /// 设置 TTS 静音开关（STS 模式） + static Future setTtsMuted(bool muted) async { + try { + final bool result = await _channel.invokeMethod('setTtsMuted', { + 'muted': muted, + }); + return result; + } on PlatformException catch (e) { + throw AgentServiceException(e.code, e.message ?? '设置TTS静音失败', e.details); + } + } + /// 中断当前响应 /// /// 停止语音合成和AI流输出 diff --git a/local_plugins/ble_service/lib/ble_service.dart b/local_plugins/ble_service/lib/ble_service.dart index 995ab4280..89e166651 100644 --- a/local_plugins/ble_service/lib/ble_service.dart +++ b/local_plugins/ble_service/lib/ble_service.dart @@ -329,7 +329,7 @@ class BleService { /// 断开连接 Future stopScan() async { try { - final result = await _methodChannel.invokeMethod('stopscan'); + final result = await _methodChannel.invokeMethod('stopScan'); return result ?? false; } catch (e) { print('开启扫描ble设备失败: $e'); From 3cb4dc95eb153edd32225767b4db5db3d105ab93 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Fri, 20 Mar 2026 09:45:40 +0800 Subject: [PATCH 04/14] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=A7=BB=E5=8A=A8?= =?UTF-8?q?=E7=B2=BE=E7=81=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/mobile_elf_controller.dart | 57 +++++++++++++++---- .../mobile_elf/views/mobile_elf_view.dart | 34 ++++++++++- .../yunqiinnovation/agent_service/StsAgent.kt | 6 +- 3 files changed, 81 insertions(+), 16 deletions(-) diff --git a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart index a673cf443..6393fc975 100644 --- a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart +++ b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart @@ -3,8 +3,10 @@ import 'package:agent_service/agent_service.dart'; import 'package:chat_storage/chat_storage.dart'; import 'package:flutter/widgets.dart'; import 'package:get/get.dart'; +import 'package:permission_handler/permission_handler.dart'; import '../../../data/models/appconfig.dart'; import '../../../core/utils/logger.dart'; +import '../../../core/utils/permission_util.dart'; enum MobileElfStatus { idle, connecting, recording, thinking, speaking, error } @@ -43,6 +45,7 @@ class MobileElfController extends GetxController { Timer? _callTimer; Timer? _pttTimer; bool _inCallMode = false; + bool _callStarting = false; // 防止并发 startCall // ═══════════════════════════════════════════════════════════════════════════ // 生命周期 @@ -104,19 +107,24 @@ class MobileElfController extends GetxController { void _onAgentEvent(AgentServiceEvent event) { Logger.info('$_tag event: ${event.type} data: ${event.data}'); + // 已挂断/停止时忽略原生层残留的状态事件 + final sessionActive = _inCallMode || pttRecording.value; switch (event.type) { case AgentServiceEventType.recognitionStarted: + if (!sessionActive) break; status.value = MobileElfStatus.recording; realtimeText.value = ''; micAlive.value = true; break; case AgentServiceEventType.recognizing: + if (!sessionActive) break; realtimeText.value = event.data['text']?.toString() ?? ''; status.value = MobileElfStatus.recording; break; case AgentServiceEventType.recognitionResult: + if (!sessionActive) break; final text = event.data['text']?.toString() ?? ''; realtimeText.value = ''; micAlive.value = false; @@ -126,14 +134,15 @@ class MobileElfController extends GetxController { case AgentServiceEventType.recognitionStopped: case AgentServiceEventType.recognitionCanceled: + if (!sessionActive) break; realtimeText.value = ''; micAlive.value = false; break; case AgentServiceEventType.assistantResponse: + if (!sessionActive) break; final text = event.data['response']?.toString() ?? ''; if (text.isNotEmpty) _updateOrAddAssistantMessage(text); - // 静音模式下收到回复后直接恢复状态（不等 TTS 事件） if (!ttsEnabled.value) { status.value = _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; @@ -142,12 +151,13 @@ class MobileElfController extends GetxController { break; case AgentServiceEventType.ttsStarted: + if (!sessionActive) break; status.value = MobileElfStatus.speaking; break; case AgentServiceEventType.ttsCompleted: + if (!sessionActive) break; if (_inCallMode) { - // 通话模式：TTS 播完后自动回到录音等待状态 status.value = MobileElfStatus.recording; micAlive.value = true; } else { @@ -158,11 +168,13 @@ class MobileElfController extends GetxController { case AgentServiceEventType.ttsCanceled: case AgentServiceEventType.ttsStopped: + if (!sessionActive) break; status.value = _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; micAlive.value = _inCallMode; break; case AgentServiceEventType.error: + if (!sessionActive) break; final msg = event.data['message']?.toString() ?? '未知错误'; status.value = MobileElfStatus.error; errorMsg.value = msg; @@ -185,18 +197,32 @@ class MobileElfController extends GetxController { // ═══════════════════════════════════════════════════════════════════════════ Future startCall() async { - if (inCall.value) return; - _inCallMode = true; - inCall.value = true; - callSeconds.value = 0; - status.value = MobileElfStatus.connecting; - _callTimer?.cancel(); - _callTimer = Timer.periodic( - const Duration(seconds: 1), (_) => callSeconds.value++); - await AgentService.startConversation(mode: 'phone_call'); + if (inCall.value || _callStarting) return; + _callStarting = true; + try { + final granted = await PermissionUtil.instance.requestPermission( + permissionType: Permission.microphone, + permissionName: '麦克风', + explanationText: '移动精灵需要麦克风权限才能进行语音对话', + permanentDenialText: '请在设置中开启麦克风权限以使用语音功能', + ); + if (!granted || inCall.value) return; // 权限等待期间可能已被挂断或重复调用 + _inCallMode = true; + inCall.value = true; + callSeconds.value = 0; + micAlive.value = true; + status.value = MobileElfStatus.connecting; + _callTimer?.cancel(); + _callTimer = Timer.periodic( + const Duration(seconds: 1), (_) => callSeconds.value++); + await AgentService.startConversation(mode: 'phone_call'); + } finally { + _callStarting = false; + } } Future disconnect() async { + _callStarting = false; _inCallMode = false; inCall.value = false; _callTimer?.cancel(); @@ -211,8 +237,15 @@ class MobileElfController extends GetxController { await AgentService.stopConversation(); } - void startPtt() { + Future startPtt() async { if (pttRecording.value) return; + final granted = await PermissionUtil.instance.requestPermission( + permissionType: Permission.microphone, + permissionName: '麦克风', + explanationText: '移动精灵需要麦克风权限才能进行语音对话', + permanentDenialText: '请在设置中开启麦克风权限以使用语音功能', + ); + if (!granted) return; pttRecording.value = true; pttSeconds.value = 0; status.value = MobileElfStatus.connecting; diff --git a/lib/modules/mobile_elf/views/mobile_elf_view.dart b/lib/modules/mobile_elf/views/mobile_elf_view.dart index 5dc2d89a1..da76aa701 100644 --- a/lib/modules/mobile_elf/views/mobile_elf_view.dart +++ b/lib/modules/mobile_elf/views/mobile_elf_view.dart @@ -768,7 +768,7 @@ class MobileElfView extends GetView { child: Row( mainAxisAlignment: MainAxisAlignment.center, children: [ - const _MiniWave(color: Colors.white), + Obx(() => _RmsBars(rms: controller.micRms.value, color: Colors.white)), SizedBox(width: 8.w), Obx(() => Text('松开发送 · ${controller.pttSeconds.value}s', style: TextStyle(fontSize: 14.sp, color: Colors.white, fontWeight: FontWeight.w500))), @@ -941,3 +941,35 @@ class _VolumeIndicator extends StatelessWidget { ); } } + +/// PTT 录音时响应麦克风音量的波形条 +class _RmsBars extends StatelessWidget { + final double rms; + final Color color; + const _RmsBars({required this.rms, required this.color}); + + @override + Widget build(BuildContext context) { + // rms 范围约 0~6000，归一化到 0~1 + final level = (rms / 5000).clamp(0.0, 1.0); + const barCount = 3; + return Row( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.center, + children: List.generate(barCount, (i) { + final threshold = (i + 1) / barCount; + final h = level >= threshold ? 10.0 : 3.0; + return AnimatedContainer( + duration: const Duration(milliseconds: 80), + width: 2.5, + height: h, + margin: const EdgeInsets.symmetric(horizontal: 1), + decoration: BoxDecoration( + color: color, + borderRadius: BorderRadius.circular(2), + ), + ); + }), + ); + } +} diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt index 01f938ac8..8454d3eab 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt @@ -472,7 +472,7 @@ class StsAgent( override fun onMessage(ws: WebSocket, bytes: okio.ByteString) {} override fun onFailure(ws: WebSocket, t: Throwable, response: Response?) { - webSocket = null + if (webSocket === ws) webSocket = null // 只清除自己的引用，避免覆盖新 WS if (!resolved) { resolved = true; cont.resumeWithException(t) } else launch { handleWsDone() } } @@ -480,7 +480,7 @@ class StsAgent( override fun onClosing(ws: WebSocket, code: Int, reason: String) { ws.close(1000, null) } override fun onClosed(ws: WebSocket, code: Int, reason: String) { - webSocket = null + if (webSocket === ws) webSocket = null // 只清除自己的引用，避免覆盖新 WS if (!resolved) { resolved = true; cont.resumeWithException(IOException("WS closed")) } else launch { handleWsDone() } } @@ -499,7 +499,7 @@ class StsAgent( } private fun handleWsDone() { - webSocket = null; forwardingAudio = false; wsConnecting = false + forwardingAudio = false; wsConnecting = false when { sessionEnded && inCall -> if (!ttsPlaying) { resetVad(); startIdleTimer() } sessionEnded -> { /* PTT 结束，等 TTS 播完 */ } From 02533577a358c482f1ceb9bc80f4566b1c690561 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Fri, 20 Mar 2026 17:23:35 +0800 Subject: [PATCH 05/14] =?UTF-8?q?=E7=A1=AE=E5=AE=9A=E7=A7=BB=E5=8A=A8?= =?UTF-8?q?=E8=AE=BE=E5=A4=87=E7=BB=91=E5=AE=9A=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/data/models/user_Info.dart | 4 + lib/data/services/ble_manager.dart | 19 ++-- .../home/controllers/home_controller.dart | 5 +- .../controllers/mobile_elf_controller.dart | 50 +++++++- .../mobile_elf/views/mobile_elf_view.dart | 19 +++- .../controllers/pairing_controller.dart | 107 +++++++++++++++--- .../agent_service/AgentService.kt | 10 ++ .../agent_service/AgentServicePlugin.kt | 20 ++++ .../yunqiinnovation/agent_service/StsAgent.kt | 81 ++++++++++++- .../agent_service/lib/agent_service.dart | 29 +++++ 10 files changed, 311 insertions(+), 33 deletions(-) diff --git a/lib/data/models/user_Info.dart b/lib/data/models/user_Info.dart index 87e2cfbce..2467ba7bd 100644 --- a/lib/data/models/user_Info.dart +++ b/lib/data/models/user_Info.dart @@ -212,6 +212,7 @@ class User { class UserDevice { int id; String uid; + int productid; int devicetype; String devicename; String devicemac; @@ -219,6 +220,7 @@ class UserDevice { UserDevice({ required this.id, required this.uid, + required this.productid, required this.devicetype, required this.devicename, required this.devicemac, @@ -228,6 +230,7 @@ class UserDevice { factory UserDevice.fromJson(Map json) => UserDevice( id: json['id'] as int, uid: json['uid'] as String, + productid: (json['productid'] as num).toInt(), devicetype: (json['devicetype'] as num).toInt(), devicename: json['devicename'] as String, devicemac: json['devicemac'] as String, @@ -237,6 +240,7 @@ class UserDevice { toJson() => { 'id': id, 'uid': uid, + 'productid': productid, 'devicetype': devicetype, 'devicename': devicename, 'devicemac': devicemac, diff --git a/lib/data/services/ble_manager.dart b/lib/data/services/ble_manager.dart index ddf05eedc..7327929b6 100644 --- a/lib/data/services/ble_manager.dart +++ b/lib/data/services/ble_manager.dart @@ -2,7 +2,6 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; import 'package:get_storage/get_storage.dart'; import 'package:get/get.dart'; import 'package:ble_service/ble_service.dart'; @@ -587,23 +586,17 @@ class BleManager extends GetxService { } } - /// 更新设备id + /// 更新设备id，并检测是否为移动精灵设备 void _updateProductIdFromData(Map data) { try { - // 确保数据中包含版本信息 if (!data.containsKey('productId')) { Logger.warning('版本信息数据不完整'); return; } - // 提取版本信息 final String productId = data['productId'] as String? ?? ''; - // 更新设备信息 - _updateDeviceInfoIfNeeded( - firmwareProductId: productId, - ); - + _updateDeviceInfoIfNeeded(firmwareProductId: productId); Logger.info('设备设备id已更新: 设备id=$productId'); } catch (e) { Logger.error('更新设备id失败: ${e.toString()}'); @@ -959,14 +952,18 @@ class BleManager extends GetxService { } // STS 模式参数（移动精灵） - final agentType = AppConfig.env('AGENT_TYPE') ?? 'yidong'; + // 优先使用设备检测结果（YIDONG_PID 匹配时写入），否则用服务端 AGENT_TYPE + final agentType = _storage.read('device_agent_type') + ?? AppConfig.env('AGENT_TYPE') + ?? 'stt_llm_tts'; final agentMode = (agentType == 'yidong') ? 'sts' : 'stt_llm_tts'; final agentId = (agentType == 'yidong') ? (AppConfig.env('MOBILE_ELF_AGENT_ID') ?? 'mobile_elf') : 'default_agent'; final stsProductId = AppConfig.env('MOBILE_ELF_PRODUCT_ID') ?? ''; final stsProductKey = AppConfig.env('MOBILE_ELF_PRODUCT_KEY') ?? ''; - final stsDeviceNo = dotenv.env['MOBILE_ELF_DEVICE_NO'] ?? ''; + // 使用当前连接设备的 SN（配对时写入），认证失败由原生层上报 credential_failed 事件 + final stsDeviceNo = _storage.read('device_sts_code') ?? ''; final stsDeviceId = _storage.read('sts_device_id') ?? ''; final stsDeviceSecret = _storage.read('sts_device_secret') ?? ''; diff --git a/lib/modules/home/controllers/home_controller.dart b/lib/modules/home/controllers/home_controller.dart index 7df97989b..55d431d59 100644 --- a/lib/modules/home/controllers/home_controller.dart +++ b/lib/modules/home/controllers/home_controller.dart @@ -256,7 +256,10 @@ class HomeController extends GetxController { } void exploreAIFeatures() { - final agentType = AppConfig.env('AGENT_TYPE') ?? 'yidong'; + // 优先读设备检测写入的 storage（YIDONG_PID 匹配时），否则降级到服务端 AGENT_TYPE + final agentType = GetStorage().read('device_agent_type') + ?? AppConfig.env('AGENT_TYPE') + ?? 'stt_llm_tts'; if (agentType == 'yidong') { Get.toNamed(Routes.mobileElf); } else { diff --git a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart index 6393fc975..c39399405 100644 --- a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart +++ b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart @@ -3,6 +3,7 @@ import 'package:agent_service/agent_service.dart'; import 'package:chat_storage/chat_storage.dart'; import 'package:flutter/widgets.dart'; import 'package:get/get.dart'; +import 'package:get_storage/get_storage.dart'; import 'package:permission_handler/permission_handler.dart'; import '../../../data/models/appconfig.dart'; import '../../../core/utils/logger.dart'; @@ -39,6 +40,9 @@ class MobileElfController extends GetxController { // 凭证状态（原生层负责，Dart 层始终无错） final credentialReady = true.obs; final credentialError = ''.obs; + // 设备认证失败（STS credential_failed 事件触发） + final credentialFailed = false.obs; + final credentialFailedReason = ''.obs; // ── 内部 ─────────────────────────────────────────────────────────────────── StreamSubscription? _eventSub; @@ -46,6 +50,9 @@ class MobileElfController extends GetxController { Timer? _pttTimer; bool _inCallMode = false; bool _callStarting = false; // 防止并发 startCall + // PTT 松开后仍有异步事件待处理（recognition_result / assistant_response / tts） + bool _pttResultPending = false; + final _storage = GetStorage(); // ═══════════════════════════════════════════════════════════════════════════ // 生命周期 @@ -62,6 +69,8 @@ class MobileElfController extends GetxController { ever(ttsEnabled, (bool enabled) { AgentService.setTtsMuted(!enabled); }); + // 主动触发认证检查（页面打开时，确保 credential_failed 事件不被错过） + AgentService.fetchCredentials(); } Future _loadChatHistory() async { @@ -107,8 +116,9 @@ class MobileElfController extends GetxController { void _onAgentEvent(AgentServiceEvent event) { Logger.info('$_tag event: ${event.type} data: ${event.data}'); - // 已挂断/停止时忽略原生层残留的状态事件 - final sessionActive = _inCallMode || pttRecording.value; + // PTT 松开后 pttRecording=false，但 fin_result/assistant_response/tts 仍异步到达 + // 用 _pttResultPending 保持会话激活，直到 ttsCompleted/error 才关闭 + final sessionActive = _inCallMode || pttRecording.value || _pttResultPending; switch (event.type) { case AgentServiceEventType.recognitionStarted: if (!sessionActive) break; @@ -157,6 +167,7 @@ class MobileElfController extends GetxController { case AgentServiceEventType.ttsCompleted: if (!sessionActive) break; + _pttResultPending = false; // PTT 一轮结束 if (_inCallMode) { status.value = MobileElfStatus.recording; micAlive.value = true; @@ -169,12 +180,14 @@ class MobileElfController extends GetxController { case AgentServiceEventType.ttsCanceled: case AgentServiceEventType.ttsStopped: if (!sessionActive) break; + _pttResultPending = false; // TTS 被打断也视为结束 status.value = _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; micAlive.value = _inCallMode; break; case AgentServiceEventType.error: if (!sessionActive) break; + _pttResultPending = false; // 出错时结束等待 final msg = event.data['message']?.toString() ?? '未知错误'; status.value = MobileElfStatus.error; errorMsg.value = msg; @@ -186,6 +199,24 @@ class MobileElfController extends GetxController { micRms.value = rms; break; + case AgentServiceEventType.heartbeatReported: + // 记录心跳上报时间 + final t = (event.data['time'] as num?)?.toInt() ?? DateTime.now().millisecondsSinceEpoch; + _storage.write('sts_heartbeat_time', t); + Logger.info('$_tag 心跳上报成功，时间: $t'); + break; + + case AgentServiceEventType.credentialFailed: + final reason = event.data['reason']?.toString() ?? '设备未认证'; + Logger.error('$_tag 设备认证失败: $reason'); + credentialFailed.value = true; + credentialFailedReason.value = reason; + // 确保停止任何进行中的会话 + if (_inCallMode || pttRecording.value) { + disconnect(); + } + break; + default: break; } @@ -199,7 +230,11 @@ class MobileElfController extends GetxController { Future startCall() async { if (inCall.value || _callStarting) return; _callStarting = true; + _checkHeartbeat(); try { + // 先确保原生层完全停止（处理 onClose 未 await 导致的残留状态） + await AgentService.stopConversation(); + final granted = await PermissionUtil.instance.requestPermission( permissionType: Permission.microphone, permissionName: '麦克风', @@ -224,6 +259,7 @@ class MobileElfController extends GetxController { Future disconnect() async { _callStarting = false; _inCallMode = false; + _pttResultPending = false; // 强制结束所有等待 inCall.value = false; _callTimer?.cancel(); _callTimer = null; @@ -237,8 +273,17 @@ class MobileElfController extends GetxController { await AgentService.stopConversation(); } + /// 超过1小时未上报心跳则补报一次 + void _checkHeartbeat() { + final last = _storage.read('sts_heartbeat_time') ?? 0; + if (DateTime.now().millisecondsSinceEpoch - last > 3600000) { + AgentService.reportHeartbeat().catchError((_) {}); + } + } + Future startPtt() async { if (pttRecording.value) return; + _checkHeartbeat(); final granted = await PermissionUtil.instance.requestPermission( permissionType: Permission.microphone, permissionName: '麦克风', @@ -258,6 +303,7 @@ class MobileElfController extends GetxController { void stopPtt() { if (!pttRecording.value) return; pttRecording.value = false; + _pttResultPending = true; // 等待 fin_result/assistant_response/tts 异步回来 _pttTimer?.cancel(); _pttTimer = null; AgentService.stopConversation(); diff --git a/lib/modules/mobile_elf/views/mobile_elf_view.dart b/lib/modules/mobile_elf/views/mobile_elf_view.dart index da76aa701..dad19841f 100644 --- a/lib/modules/mobile_elf/views/mobile_elf_view.dart +++ b/lib/modules/mobile_elf/views/mobile_elf_view.dart @@ -143,6 +143,17 @@ class MobileElfView extends GetView { /// 右上角状态胶囊 + 通话计时 Widget _buildStatusPill() { + if (controller.credentialFailed.value) { + return Container( + padding: EdgeInsets.symmetric(horizontal: 10.w, vertical: 3.h), + decoration: BoxDecoration(color: const Color(0xFFFEF2F2), borderRadius: BorderRadius.circular(20.r)), + child: Row(mainAxisSize: MainAxisSize.min, children: [ + Icon(Icons.lock_outline_rounded, size: 11.sp, color: _red), + SizedBox(width: 4.w), + Text('认证失败', style: TextStyle(fontSize: 11.sp, color: _red, fontWeight: FontWeight.w500)), + ]), + ); + } final s = controller.status.value; final inCall = controller.inCall.value; final cfg = _pillConfig(s, inCall); @@ -444,6 +455,7 @@ class MobileElfView extends GetView { Widget _buildBottomBar(bool isDark) { return Obx(() { + final inCall = controller.inCall.value; final s = controller.status.value; final ptt = controller.pttRecording.value; @@ -621,9 +633,10 @@ class MobileElfView extends GetView { final s = controller.status.value; final ptt = controller.pttRecording.value; - final disabled = !ptt && s != MobileElfStatus.idle && s != MobileElfStatus.error; - final pttClickable = s == MobileElfStatus.speaking; - final sideDisabled = ptt || (disabled && !pttClickable); + final locked = controller.credentialFailed.value; + final disabled = locked || (!ptt && s != MobileElfStatus.idle && s != MobileElfStatus.error); + final pttClickable = !locked && s == MobileElfStatus.speaking; + final sideDisabled = locked || ptt || (disabled && !pttClickable); return Padding( padding: EdgeInsets.fromLTRB(12.w, 6.h, 12.w, 14.h), diff --git a/lib/modules/pairing/controllers/pairing_controller.dart b/lib/modules/pairing/controllers/pairing_controller.dart index 2b63098c6..5156ea8b4 100644 --- a/lib/modules/pairing/controllers/pairing_controller.dart +++ b/lib/modules/pairing/controllers/pairing_controller.dart @@ -810,6 +810,7 @@ class PairingController extends GetxController with WidgetsBindingObserver { orElse: () => UserDevice( id: -1, uid: '', + productid: -1, devicetype: -1, devicename: '', devicemac: '', @@ -840,15 +841,21 @@ class PairingController extends GetxController with WidgetsBindingObserver { orElse: () => UserDevice( id: -1, uid: '', + productid: -1, devicetype: -1, devicename: '', devicemac: '', license: ''), ); if (existingDevice.id != -1) { + _verifiedDevicePid = existingDevice.productid; _verifiedDeviceName = device.name; _verifiedDeviceAddress = device.address; - // 设备时ble 蓝牙 且已绑定 + // 已绑定设备直接用 license 作为 deviceCode（即绑定时扫到的 SN） + if (existingDevice.license.isNotEmpty) { + deviceCode.value = existingDevice.license; + Logger.i(_tag, '[YIDONG] 已绑定设备，从license提取 deviceCode=${existingDevice.license}'); + } return 10; } } @@ -885,14 +892,15 @@ class PairingController extends GetxController with WidgetsBindingObserver { code = extractAndReverseBytes(hexData, 13, 22); _verifiedDevicePid = extractAndReverseBytesToInt(hexData, 4, 5); - print('找到匹配设备，获取到 code: $code'); + final pidHexIos = _verifiedDevicePid?.toRadixString(16).toUpperCase() ?? ''; + Logger.i(_tag, '[YIDONG] iOS BLE扫描 MAC匹配 device=${device.name} pidHex=$pidHexIos code=$code'); if (code != "") { _bleManager.stopScan(); deviceCode.value = code; String? uuids = deviceInfo["id"]; _verifiedDeviceName = device.name; _verifiedDeviceAddress = uuids; - print('找到匹配设备，获取到 uuids: $uuids'); + Logger.i(_tag, '[YIDONG] iOS deviceCode提取成功 code=$code uuid=$uuids'); //检查是否已经绑定 final existingDevice = User.instance.devices.firstWhere( (d) => @@ -902,6 +910,7 @@ class PairingController extends GetxController with WidgetsBindingObserver { orElse: () => UserDevice( id: -1, uid: '', + productid: -1, devicetype: -1, devicename: '', devicemac: '', @@ -921,12 +930,14 @@ class PairingController extends GetxController with WidgetsBindingObserver { code = extractAndReverseBytes(hexData, 15, 24); _verifiedDevicePid = extractAndReverseBytesToInt(hexData, 6, 7); - print('找到匹配设备，获取到 code: $code'); + final pidHexAndroid = _verifiedDevicePid?.toRadixString(16).toUpperCase() ?? ''; + Logger.i(_tag, '[YIDONG] Android BLE扫描 MAC匹配 device=${device.name} mac=${device.address} pidHex=$pidHexAndroid code=$code'); if (code != "") { _bleManager.stopScan(); deviceCode.value = code; _verifiedDeviceName = device.name; _verifiedDeviceAddress = device.address; + Logger.i(_tag, '[YIDONG] Android deviceCode提取成功 code=$code mac=${device.address}'); //检查是否已经绑定 final existingDevice = User.instance.devices.firstWhere( (d) => @@ -936,6 +947,7 @@ class PairingController extends GetxController with WidgetsBindingObserver { orElse: () => UserDevice( id: -1, uid: '', + productid: -1, devicetype: -1, devicename: '', devicemac: '', @@ -1178,26 +1190,92 @@ class PairingController extends GetxController with WidgetsBindingObserver { } } + /// 检测是否为移动精灵（YIDONG）设备，并写入 GetStorage + void _detectAndSaveYidongDevice() { + final yidongPid = AppConfig.env('YIDONG_PID') ?? ''; + if (yidongPid.isEmpty) { + Logger.i(_tag, '[YIDONG] 未配置 YIDONG_PID，跳过检测'); + return; + } + + // 优先使用本次扫描解析的 pid，否则从已绑定设备列表中查找 + int? pid = _verifiedDevicePid; + String pidSource = 'BLE扫描'; + if (pid == null) { + final mac = _verifiedDeviceAddress ?? _currentClassicDevice?.address ?? ''; + Logger.i(_tag, '[YIDONG] 本次未扫描到pid，从已绑定设备查找 mac=$mac'); + if (mac.isNotEmpty) { + final matched = User.instance.devices.firstWhere( + (d) => + d.devicemac.toUpperCase().replaceAll(':', '') == + mac.toUpperCase().replaceAll(':', ''), + orElse: () => UserDevice( + id: -1, uid: '', productid: -1, + devicetype: -1, devicename: '', devicemac: '', license: ''), + ); + if (matched.id != -1) { + pid = matched.productid; + pidSource = '已绑定设备'; + } + } + } + + if (pid == null) { + Logger.i(_tag, '[YIDONG] 无法获取设备pid，跳过检测'); + return; + } + + // 将 int pid 转为十六进制字符串（不补零，大写）与 YIDONG_PID 比对 + final pidHex = pid.toRadixString(16).toUpperCase(); + final isYidong = pidHex == yidongPid.toUpperCase(); + + Logger.i(_tag, '[YIDONG] 设备检测: pidHex=$pidHex yidongPid=${yidongPid.toUpperCase()} source=$pidSource matched=$isYidong'); + + if (isYidong) { + _storage.write('device_agent_type', 'yidong'); + // deviceCode 来源： + // 新设备 → BLE 扫描从广播包提取 + // 已绑定设备 → authbindingDevice 从 existingDevice.license 赋值 + if (deviceCode.value.isNotEmpty) { + final prevCode = _storage.read('device_sts_code') ?? ''; + // SN 变了说明换了设备，清除旧凭证，强制重新向移动平台认证 + if (prevCode != deviceCode.value) { + _storage.remove('sts_device_id'); + _storage.remove('sts_device_secret'); + Logger.i(_tag, '[YIDONG] SN 变更（$prevCode → ${deviceCode.value}），已清除旧凭证，强制重新认证'); + } + _storage.write('device_sts_code', deviceCode.value); + Logger.i(_tag, '[YIDONG] 识别成功，写入 device_sts_code=${deviceCode.value}'); + } else { + // 既无扫描结果也无 license，清除所有凭证，认证失败会提示用户 + _storage.remove('device_sts_code'); + _storage.remove('sts_device_id'); + _storage.remove('sts_device_secret'); + Logger.i(_tag, '[YIDONG] deviceCode 为空，已清除所有凭证'); + } + } else { + _storage.remove('device_agent_type'); + _storage.remove('device_sts_code'); + _storage.remove('sts_device_id'); + _storage.remove('sts_device_secret'); + Logger.i(_tag, '[YIDONG] 非移动精灵设备，已清除相关存储'); + } + } + /// 导航到主页 void _navigateToHome() { - // Future.delayed(const Duration(milliseconds: 500), () async { + // 检测是否为移动精灵（YIDONG）设备 + _detectAndSaveYidongDevice(); // 启动AgentService服务 try { Logger.i(_tag, '启动AgentService服务'); _bleManager.startAgentService(); - // _bleManager.registerBackgroundScan(macAddress: _verifiedDeviceAddress); } catch (e) { Logger.e(_tag, '启动AgentService失败: $e'); } - //print('lxm-上一个路由: ${Get.previousRoute.toString()}'); - // if (Get.previousRoute == Routes.home) { - // 如果是从home页过来的，则直接返回 - // Get.back(); - // } else { - // 否则，跳转到home页并清空之前的路由栈 + // 已在 Home 页时不重复导航，避免两个 HomeView 同时存在导致 Duplicate GlobalKey + if (Get.currentRoute == Routes.home) return; Get.offAllNamed(Routes.home); - // } - // }); } // ==================== 用户操作 ==================== @@ -1259,6 +1337,7 @@ class PairingController extends GetxController with WidgetsBindingObserver { UserDevice( id: 0, // 临时设备ID uid: User.instance.uid, + productid: _verifiedDevicePid ?? -1, devicetype: 1, devicemac: _currentClassicDevice?.address ?? '', devicename: _currentClassicDevice?.name ?? 'Unknown Device', diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index 3a56af051..bbad03eaa 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -987,6 +987,16 @@ object AgentService : CoroutineScope { stsAgent?.ttsMuted = muted } + /** 手动触发心跳上报（Flutter 层调用） */ + fun reportHeartbeat() { + stsAgent?.reportHeartbeat() + } + + /** 主动触发认证检查（移动精灵页面打开时调用，确保认证事件被 Flutter 层接收） */ + fun fetchCredentials() { + stsAgent?.fetchCredentials() + } + fun interruptCurrentResponse() { if (agentMode == "sts") { stsAgent?.interruptSpeaking() diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt index e6a08f893..b513800fa 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt @@ -349,6 +349,26 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream } } + "reportHeartbeat" -> { + try { + AgentService.reportHeartbeat() + result.success(true) + } catch (e: Exception) { + Log.e(TAG, "心跳上报失败", e) + result.error("HEARTBEAT_ERROR", "心跳上报失败: ${e.message}", null) + } + } + + "fetchCredentials" -> { + try { + AgentService.fetchCredentials() + result.success(true) + } catch (e: Exception) { + Log.e(TAG, "触发认证检查失败", e) + result.error("FETCH_CREDENTIALS_ERROR", "触发认证检查失败: ${e.message}", null) + } + } + "setCanWakeUp" -> { try { val canWakeUp = call.argument("canWakeUp") diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt index 8454d3eab..111cd2006 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt @@ -58,6 +58,7 @@ class StsAgent( override val coroutineContext = Dispatchers.IO + job init { + Log.i(TAG, "[YIDONG] StsAgent初始化: productId=${config.productId} deviceNo=${config.deviceNo} deviceId=${config.deviceId} agentId=${config.agentId} hasSecret=${config.deviceSecret.isNotEmpty()}") fetchCredentials() } @@ -153,8 +154,11 @@ class StsAgent( fun fetchCredentials() { if (config.deviceSecret.isNotEmpty()) { credentialReady = true + Log.i(TAG, "[YIDONG] fetchCredentials: 凭证已存在，跳过请求 deviceNo=${config.deviceNo} deviceId=${config.deviceId}") + reportHeartbeat() // 凭证已有，直接上报心跳 return } + Log.i(TAG, "[YIDONG] fetchCredentials: 开始请求密钥 productId=${config.productId} deviceNo=${config.deviceNo}") launch { try { val body = JSONObject().apply { @@ -180,14 +184,82 @@ class StsAgent( ) credentialReady = config.deviceSecret.isNotEmpty() if (credentialReady) { + Log.i(TAG, "[YIDONG] fetchCredentials: 密钥获取成功 deviceId=${config.deviceId} deviceNo=${config.deviceNo}") onCredentialsFetched?.invoke(config.deviceId, config.deviceSecret, config.deviceNo) + reportHeartbeat() // 凭证获取成功后立即上报心跳 + } else { + Log.e(TAG, "[YIDONG] fetchCredentials: 响应成功但deviceSecret为空") + emit("credential_failed", mapOf("reason" to "设备未认证，deviceSecret为空")) } } else { - Log.e(TAG, "凭证获取失败: ${json.optString("message")}") + val msg = json.optString("message", "未知错误") + Log.e(TAG, "[YIDONG] fetchCredentials: 密钥获取失败 code=${json.optString("code")} msg=$msg") + emit("credential_failed", mapOf("reason" to "设备未认证：$msg")) } } } catch (e: Exception) { - Log.e(TAG, "凭证获取异常: ${e.message}") + Log.e(TAG, "[YIDONG] fetchCredentials: 请求异常 ${e.message}") + emit("credential_failed", mapOf("reason" to "设备认证请求失败：${e.message}")) + } + } + } + + /** 向中国移动平台上报心跳 */ + fun reportHeartbeat() { + if (!credentialReady) { + Log.w(TAG, "[YIDONG] reportHeartbeat: 凭证未就绪，跳过上报") + return + } + Log.i(TAG, "[YIDONG] reportHeartbeat: 开始上报 deviceId=${config.deviceId} deviceNo=${config.deviceNo}") + launch { + try { + val netType = try { + val cm = context.getSystemService(android.content.Context.CONNECTIVITY_SERVICE) + as android.net.ConnectivityManager + val net = cm.activeNetworkInfo + when (net?.type) { + android.net.ConnectivityManager.TYPE_WIFI -> "WiFi" + android.net.ConnectivityManager.TYPE_MOBILE -> "5G" + else -> "WiFi" + } + } catch (_: Exception) { "WiFi" } + + val body = JSONObject().apply { + put("deviceId", config.deviceId) + put("deviceSecret", config.deviceSecret) + put("productId", config.productId) + put("productKey", config.productKey) + put("params", JSONObject().apply { + put("innerIp", JSONArray().put("127.0.0.1")) + put("netSpeed", "0Mbps") + put("netType", netType) + put("platform", "Android${android.os.Build.VERSION.RELEASE}") + put("sdkVersion", "ai_http_1.0") + put("firmwareVersion", "1.0.0") + put("imei", "") + put("cmei", config.deviceNo) // 非蜂窝设备使用 SN 作为 cmei + put("mac", "") + }) + }.toString().toRequestBody("application/json".toMediaType()) + + val req = Request.Builder() + .url("https://ivs.chinamobiledevice.com:11443/v2/customer/device/report") + .post(body) + .build() + + okHttpClient.newCall(req).execute().use { resp -> + val json = JSONObject(resp.body!!.string()) + val code = json.optString("code") + val msg = json.optString("message") + if (code == "200") { + Log.i(TAG, "[YIDONG] reportHeartbeat: 上报成功 code=$code msg=$msg") + emit("heartbeat_reported", mapOf("time" to System.currentTimeMillis())) + } else { + Log.e(TAG, "[YIDONG] reportHeartbeat: 上报失败 code=$code msg=$msg") + } + } + } catch (e: Exception) { + Log.e(TAG, "[YIDONG] reportHeartbeat: 请求异常 ${e.message}") } } } @@ -202,6 +274,7 @@ class StsAgent( if (ttsPlaying) interruptTts() if (pttRecording) stopPtt(cancel = true) if (!credentialReady) { emitError("移动精灵凭证未就绪，请稍后重试"); return } + Log.i(TAG, "[YIDONG] startCall: 启动通话模式 deviceNo=${config.deviceNo} agentId=${config.agentId}") openMic() startMicWatchdog() inCall = true @@ -213,6 +286,7 @@ class StsAgent( if (pttRecording) return if (ttsPlaying) interruptTts() if (!credentialReady) { emitError("移动精灵凭证未就绪"); return } + Log.i(TAG, "[YIDONG] startPtt: 启动PTT模式 deviceNo=${config.deviceNo} agentId=${config.agentId}") pttRecording = true openMic() @@ -445,6 +519,7 @@ class StsAgent( "?deviceNo=${config.deviceNo}&deviceId=${config.deviceId}" + "&productId=${config.productId}&productKey=${config.productKey}" + "&sign=$sign&sn=$sn&ts=$ts" + Log.i(TAG, "[YIDONG] connectWs: 连接WSS deviceNo=${config.deviceNo} deviceId=${config.deviceId} sn=$sn") val req = Request.Builder().url(url) .header("Origin", "https://ivs.chinamobiledevice.com:11443") @@ -462,6 +537,7 @@ class StsAgent( try { val msg = JSONObject(text) if (msg.optString("type") == "ready") { + Log.i(TAG, "[YIDONG] connectWs: 收到ready，WSS连接成功") if (!resolved) { resolved = true; cont.resume(Unit) {} } } else { launch { handleMessage(msg) } @@ -472,6 +548,7 @@ class StsAgent( override fun onMessage(ws: WebSocket, bytes: okio.ByteString) {} override fun onFailure(ws: WebSocket, t: Throwable, response: Response?) { + Log.e(TAG, "[YIDONG] connectWs: WSS连接失败 ${t.message}") if (webSocket === ws) webSocket = null // 只清除自己的引用，避免覆盖新 WS if (!resolved) { resolved = true; cont.resumeWithException(t) } else launch { handleWsDone() } diff --git a/local_plugins/agent_service/lib/agent_service.dart b/local_plugins/agent_service/lib/agent_service.dart index 1cdec8d65..9478c66fb 100644 --- a/local_plugins/agent_service/lib/agent_service.dart +++ b/local_plugins/agent_service/lib/agent_service.dart @@ -77,6 +77,12 @@ enum AgentServiceEventType { /// 麦克风音量（RMS） micRms, + /// 心跳上报成功 + heartbeatReported, + + /// STS 设备认证失败 + credentialFailed, + /// 未知事件 unknown } @@ -186,6 +192,10 @@ class AgentService { return AgentServiceEventType.stsCredentials; case 'mic_rms': return AgentServiceEventType.micRms; + case 'heartbeat_reported': + return AgentServiceEventType.heartbeatReported; + case 'credential_failed': + return AgentServiceEventType.credentialFailed; default: return AgentServiceEventType.unknown; } @@ -468,6 +478,25 @@ class AgentService { } } + /// 手动触发心跳上报（移动精灵） + static Future reportHeartbeat() async { + try { + await _channel.invokeMethod('reportHeartbeat'); + } on PlatformException catch (e) { + throw AgentServiceException(e.code, e.message ?? '心跳上报失败', e.details); + } + } + + /// 主动触发认证检查（移动精灵页面打开时调用） + /// 若凭证不存在则重新请求，结果通过事件流返回（credentialFailed / heartbeatReported） + static Future fetchCredentials() async { + try { + await _channel.invokeMethod('fetchCredentials'); + } on PlatformException catch (_) { + // 忽略异常，认证结果通过事件流反馈 + } + } + /// 中断当前响应 /// /// 停止语音合成和AI流输出 From a51d56ebf80166613407b293282f12ab75a43055 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Fri, 20 Mar 2026 18:01:27 +0800 Subject: [PATCH 06/14] =?UTF-8?q?=E5=94=A4=E9=86=92=E4=B8=8D=E7=8E=B0?= =?UTF-8?q?=E5=AE=9E=E6=96=87=E6=9C=AC=20bug=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mobile_elf/controllers/mobile_elf_controller.dart | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart index c39399405..eb711cc44 100644 --- a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart +++ b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart @@ -52,6 +52,8 @@ class MobileElfController extends GetxController { bool _callStarting = false; // 防止并发 startCall // PTT 松开后仍有异步事件待处理（recognition_result / assistant_response / tts） bool _pttResultPending = false; + // BLE/AI 唤醒触发的会话（非通话、非PTT模式下的唤醒） + bool _bleWakeupActive = false; final _storage = GetStorage(); // ═══════════════════════════════════════════════════════════════════════════ @@ -118,10 +120,11 @@ class MobileElfController extends GetxController { Logger.info('$_tag event: ${event.type} data: ${event.data}'); // PTT 松开后 pttRecording=false，但 fin_result/assistant_response/tts 仍异步到达 // 用 _pttResultPending 保持会话激活，直到 ttsCompleted/error 才关闭 - final sessionActive = _inCallMode || pttRecording.value || _pttResultPending; + final sessionActive = _inCallMode || pttRecording.value || _pttResultPending || _bleWakeupActive; switch (event.type) { case AgentServiceEventType.recognitionStarted: - if (!sessionActive) break; + // BLE/AI 唤醒：即使没有主动会话也激活 + if (!sessionActive) _bleWakeupActive = true; status.value = MobileElfStatus.recording; realtimeText.value = ''; micAlive.value = true; @@ -168,6 +171,7 @@ class MobileElfController extends GetxController { case AgentServiceEventType.ttsCompleted: if (!sessionActive) break; _pttResultPending = false; // PTT 一轮结束 + _bleWakeupActive = false; // BLE 唤醒一轮结束 if (_inCallMode) { status.value = MobileElfStatus.recording; micAlive.value = true; @@ -181,6 +185,7 @@ class MobileElfController extends GetxController { case AgentServiceEventType.ttsStopped: if (!sessionActive) break; _pttResultPending = false; // TTS 被打断也视为结束 + _bleWakeupActive = false; status.value = _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; micAlive.value = _inCallMode; break; @@ -188,6 +193,7 @@ class MobileElfController extends GetxController { case AgentServiceEventType.error: if (!sessionActive) break; _pttResultPending = false; // 出错时结束等待 + _bleWakeupActive = false; final msg = event.data['message']?.toString() ?? '未知错误'; status.value = MobileElfStatus.error; errorMsg.value = msg; @@ -260,6 +266,7 @@ class MobileElfController extends GetxController { _callStarting = false; _inCallMode = false; _pttResultPending = false; // 强制结束所有等待 + _bleWakeupActive = false; inCall.value = false; _callTimer?.cancel(); _callTimer = null; From c6fe1b26b3b4b493c0702ea0b09af19144f11082 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Fri, 20 Mar 2026 20:05:01 +0800 Subject: [PATCH 07/14] =?UTF-8?q?=E7=A1=AE=E5=AE=9A=E7=A7=BB=E5=8A=A8?= =?UTF-8?q?=E8=AE=BE=E5=A4=87=E7=BB=91=E5=AE=9A=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/data/models/user_Info.dart | 4 + lib/data/services/ble_manager.dart | 19 ++-- .../home/controllers/home_controller.dart | 5 +- .../controllers/mobile_elf_controller.dart | 87 +++++++++++++- .../mobile_elf/views/mobile_elf_view.dart | 19 +++- .../controllers/pairing_controller.dart | 107 +++++++++++++++--- .../agent_service/AgentService.kt | 10 ++ .../agent_service/AgentServicePlugin.kt | 20 ++++ .../yunqiinnovation/agent_service/StsAgent.kt | 106 +++++++++++++++-- .../agent_service/lib/agent_service.dart | 29 +++++ 10 files changed, 363 insertions(+), 43 deletions(-) diff --git a/lib/data/models/user_Info.dart b/lib/data/models/user_Info.dart index 87e2cfbce..2467ba7bd 100644 --- a/lib/data/models/user_Info.dart +++ b/lib/data/models/user_Info.dart @@ -212,6 +212,7 @@ class User { class UserDevice { int id; String uid; + int productid; int devicetype; String devicename; String devicemac; @@ -219,6 +220,7 @@ class UserDevice { UserDevice({ required this.id, required this.uid, + required this.productid, required this.devicetype, required this.devicename, required this.devicemac, @@ -228,6 +230,7 @@ class UserDevice { factory UserDevice.fromJson(Map json) => UserDevice( id: json['id'] as int, uid: json['uid'] as String, + productid: (json['productid'] as num).toInt(), devicetype: (json['devicetype'] as num).toInt(), devicename: json['devicename'] as String, devicemac: json['devicemac'] as String, @@ -237,6 +240,7 @@ class UserDevice { toJson() => { 'id': id, 'uid': uid, + 'productid': productid, 'devicetype': devicetype, 'devicename': devicename, 'devicemac': devicemac, diff --git a/lib/data/services/ble_manager.dart b/lib/data/services/ble_manager.dart index ddf05eedc..7327929b6 100644 --- a/lib/data/services/ble_manager.dart +++ b/lib/data/services/ble_manager.dart @@ -2,7 +2,6 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; import 'package:get_storage/get_storage.dart'; import 'package:get/get.dart'; import 'package:ble_service/ble_service.dart'; @@ -587,23 +586,17 @@ class BleManager extends GetxService { } } - /// 更新设备id + /// 更新设备id，并检测是否为移动精灵设备 void _updateProductIdFromData(Map data) { try { - // 确保数据中包含版本信息 if (!data.containsKey('productId')) { Logger.warning('版本信息数据不完整'); return; } - // 提取版本信息 final String productId = data['productId'] as String? ?? ''; - // 更新设备信息 - _updateDeviceInfoIfNeeded( - firmwareProductId: productId, - ); - + _updateDeviceInfoIfNeeded(firmwareProductId: productId); Logger.info('设备设备id已更新: 设备id=$productId'); } catch (e) { Logger.error('更新设备id失败: ${e.toString()}'); @@ -959,14 +952,18 @@ class BleManager extends GetxService { } // STS 模式参数（移动精灵） - final agentType = AppConfig.env('AGENT_TYPE') ?? 'yidong'; + // 优先使用设备检测结果（YIDONG_PID 匹配时写入），否则用服务端 AGENT_TYPE + final agentType = _storage.read('device_agent_type') + ?? AppConfig.env('AGENT_TYPE') + ?? 'stt_llm_tts'; final agentMode = (agentType == 'yidong') ? 'sts' : 'stt_llm_tts'; final agentId = (agentType == 'yidong') ? (AppConfig.env('MOBILE_ELF_AGENT_ID') ?? 'mobile_elf') : 'default_agent'; final stsProductId = AppConfig.env('MOBILE_ELF_PRODUCT_ID') ?? ''; final stsProductKey = AppConfig.env('MOBILE_ELF_PRODUCT_KEY') ?? ''; - final stsDeviceNo = dotenv.env['MOBILE_ELF_DEVICE_NO'] ?? ''; + // 使用当前连接设备的 SN（配对时写入），认证失败由原生层上报 credential_failed 事件 + final stsDeviceNo = _storage.read('device_sts_code') ?? ''; final stsDeviceId = _storage.read('sts_device_id') ?? ''; final stsDeviceSecret = _storage.read('sts_device_secret') ?? ''; diff --git a/lib/modules/home/controllers/home_controller.dart b/lib/modules/home/controllers/home_controller.dart index 7df97989b..55d431d59 100644 --- a/lib/modules/home/controllers/home_controller.dart +++ b/lib/modules/home/controllers/home_controller.dart @@ -256,7 +256,10 @@ class HomeController extends GetxController { } void exploreAIFeatures() { - final agentType = AppConfig.env('AGENT_TYPE') ?? 'yidong'; + // 优先读设备检测写入的 storage（YIDONG_PID 匹配时），否则降级到服务端 AGENT_TYPE + final agentType = GetStorage().read('device_agent_type') + ?? AppConfig.env('AGENT_TYPE') + ?? 'stt_llm_tts'; if (agentType == 'yidong') { Get.toNamed(Routes.mobileElf); } else { diff --git a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart index 6393fc975..e11b599fa 100644 --- a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart +++ b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart @@ -3,6 +3,7 @@ import 'package:agent_service/agent_service.dart'; import 'package:chat_storage/chat_storage.dart'; import 'package:flutter/widgets.dart'; import 'package:get/get.dart'; +import 'package:get_storage/get_storage.dart'; import 'package:permission_handler/permission_handler.dart'; import '../../../data/models/appconfig.dart'; import '../../../core/utils/logger.dart'; @@ -39,6 +40,9 @@ class MobileElfController extends GetxController { // 凭证状态（原生层负责，Dart 层始终无错） final credentialReady = true.obs; final credentialError = ''.obs; + // 设备认证失败（STS credential_failed 事件触发） + final credentialFailed = false.obs; + final credentialFailedReason = ''.obs; // ── 内部 ─────────────────────────────────────────────────────────────────── StreamSubscription? _eventSub; @@ -46,6 +50,13 @@ class MobileElfController extends GetxController { Timer? _pttTimer; bool _inCallMode = false; bool _callStarting = false; // 防止并发 startCall + // PTT 松开后仍有异步事件待处理（recognition_result / assistant_response / tts） + bool _pttResultPending = false; + // PTT 正在异步启动（await 权限期间用户已松手） + bool _pttStarting = false; + // BLE/AI 唤醒触发的会话（非通话、非PTT模式下的唤醒） + bool _bleWakeupActive = false; + final _storage = GetStorage(); // ═══════════════════════════════════════════════════════════════════════════ // 生命周期 @@ -62,6 +73,8 @@ class MobileElfController extends GetxController { ever(ttsEnabled, (bool enabled) { AgentService.setTtsMuted(!enabled); }); + // 主动触发认证检查（页面打开时，确保 credential_failed 事件不被错过） + AgentService.fetchCredentials(); } Future _loadChatHistory() async { @@ -107,11 +120,13 @@ class MobileElfController extends GetxController { void _onAgentEvent(AgentServiceEvent event) { Logger.info('$_tag event: ${event.type} data: ${event.data}'); - // 已挂断/停止时忽略原生层残留的状态事件 - final sessionActive = _inCallMode || pttRecording.value; + // PTT 松开后 pttRecording=false，但 fin_result/assistant_response/tts 仍异步到达 + // 用 _pttResultPending 保持会话激活，直到 ttsCompleted/error 才关闭 + final sessionActive = _inCallMode || pttRecording.value || _pttResultPending || _bleWakeupActive; switch (event.type) { case AgentServiceEventType.recognitionStarted: - if (!sessionActive) break; + // BLE/AI 唤醒：即使没有主动会话也激活 + if (!sessionActive) _bleWakeupActive = true; status.value = MobileElfStatus.recording; realtimeText.value = ''; micAlive.value = true; @@ -157,6 +172,8 @@ class MobileElfController extends GetxController { case AgentServiceEventType.ttsCompleted: if (!sessionActive) break; + _pttResultPending = false; // PTT 一轮结束 + _bleWakeupActive = false; // BLE 唤醒一轮结束 if (_inCallMode) { status.value = MobileElfStatus.recording; micAlive.value = true; @@ -169,12 +186,16 @@ class MobileElfController extends GetxController { case AgentServiceEventType.ttsCanceled: case AgentServiceEventType.ttsStopped: if (!sessionActive) break; + _pttResultPending = false; // TTS 被打断也视为结束 + _bleWakeupActive = false; status.value = _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; micAlive.value = _inCallMode; break; case AgentServiceEventType.error: if (!sessionActive) break; + _pttResultPending = false; // 出错时结束等待 + _bleWakeupActive = false; final msg = event.data['message']?.toString() ?? '未知错误'; status.value = MobileElfStatus.error; errorMsg.value = msg; @@ -186,6 +207,24 @@ class MobileElfController extends GetxController { micRms.value = rms; break; + case AgentServiceEventType.heartbeatReported: + // 记录心跳上报时间 + final t = (event.data['time'] as num?)?.toInt() ?? DateTime.now().millisecondsSinceEpoch; + _storage.write('sts_heartbeat_time', t); + Logger.info('$_tag 心跳上报成功，时间: $t'); + break; + + case AgentServiceEventType.credentialFailed: + final reason = event.data['reason']?.toString() ?? '设备未认证'; + Logger.error('$_tag 设备认证失败: $reason'); + credentialFailed.value = true; + credentialFailedReason.value = reason; + // 确保停止任何进行中的会话 + if (_inCallMode || pttRecording.value) { + disconnect(); + } + break; + default: break; } @@ -199,7 +238,18 @@ class MobileElfController extends GetxController { Future startCall() async { if (inCall.value || _callStarting) return; _callStarting = true; + _checkHeartbeat(); try { + // 若有正在播报的 PTT/BLE 唤醒会话，先强制中断 TTS/AI 流， + // 使原生层立刻进入可停止状态，避免 stopConversation 长时间等待 + if (_pttResultPending || _bleWakeupActive) { + _pttResultPending = false; + _bleWakeupActive = false; + await AgentService.interruptCurrentResponse(); + } + // 再确保原生层完全停止 + await AgentService.stopConversation(); + final granted = await PermissionUtil.instance.requestPermission( permissionType: Permission.microphone, permissionName: '麦克风', @@ -216,6 +266,9 @@ class MobileElfController extends GetxController { _callTimer = Timer.periodic( const Duration(seconds: 1), (_) => callSeconds.value++); await AgentService.startConversation(mode: 'phone_call'); + // startCall 返回即表示麦克风已打开、VAD 已在监听，直接显示聆听状态 + // （call 模式下 recognition_started 只在 VAD 检测到语音时才发出，不能等它来转状态） + if (_inCallMode) status.value = MobileElfStatus.recording; } finally { _callStarting = false; } @@ -224,6 +277,9 @@ class MobileElfController extends GetxController { Future disconnect() async { _callStarting = false; _inCallMode = false; + _pttStarting = false; + _pttResultPending = false; // 强制结束所有等待 + _bleWakeupActive = false; inCall.value = false; _callTimer?.cancel(); _callTimer = null; @@ -237,15 +293,30 @@ class MobileElfController extends GetxController { await AgentService.stopConversation(); } + /// 超过1小时未上报心跳则补报一次 + void _checkHeartbeat() { + final last = _storage.read('sts_heartbeat_time') ?? 0; + if (DateTime.now().millisecondsSinceEpoch - last > 3600000) { + AgentService.reportHeartbeat().catchError((_) {}); + } + } + Future startPtt() async { - if (pttRecording.value) return; + if (pttRecording.value || _pttStarting) return; + _pttStarting = true; + _checkHeartbeat(); final granted = await PermissionUtil.instance.requestPermission( permissionType: Permission.microphone, permissionName: '麦克风', explanationText: '移动精灵需要麦克风权限才能进行语音对话', permanentDenialText: '请在设置中开启麦克风权限以使用语音功能', ); - if (!granted) return; + // 权限等待期间用户已松手（stopPtt 将 _pttStarting 置 false），直接放弃 + if (!granted || !_pttStarting) { + _pttStarting = false; + return; + } + _pttStarting = false; pttRecording.value = true; pttSeconds.value = 0; status.value = MobileElfStatus.connecting; @@ -256,8 +327,14 @@ class MobileElfController extends GetxController { } void stopPtt() { + // 若仍在等待权限授予，取消本次启动 + if (_pttStarting) { + _pttStarting = false; + return; + } if (!pttRecording.value) return; pttRecording.value = false; + _pttResultPending = true; // 等待 fin_result/assistant_response/tts 异步回来 _pttTimer?.cancel(); _pttTimer = null; AgentService.stopConversation(); diff --git a/lib/modules/mobile_elf/views/mobile_elf_view.dart b/lib/modules/mobile_elf/views/mobile_elf_view.dart index da76aa701..dad19841f 100644 --- a/lib/modules/mobile_elf/views/mobile_elf_view.dart +++ b/lib/modules/mobile_elf/views/mobile_elf_view.dart @@ -143,6 +143,17 @@ class MobileElfView extends GetView { /// 右上角状态胶囊 + 通话计时 Widget _buildStatusPill() { + if (controller.credentialFailed.value) { + return Container( + padding: EdgeInsets.symmetric(horizontal: 10.w, vertical: 3.h), + decoration: BoxDecoration(color: const Color(0xFFFEF2F2), borderRadius: BorderRadius.circular(20.r)), + child: Row(mainAxisSize: MainAxisSize.min, children: [ + Icon(Icons.lock_outline_rounded, size: 11.sp, color: _red), + SizedBox(width: 4.w), + Text('认证失败', style: TextStyle(fontSize: 11.sp, color: _red, fontWeight: FontWeight.w500)), + ]), + ); + } final s = controller.status.value; final inCall = controller.inCall.value; final cfg = _pillConfig(s, inCall); @@ -444,6 +455,7 @@ class MobileElfView extends GetView { Widget _buildBottomBar(bool isDark) { return Obx(() { + final inCall = controller.inCall.value; final s = controller.status.value; final ptt = controller.pttRecording.value; @@ -621,9 +633,10 @@ class MobileElfView extends GetView { final s = controller.status.value; final ptt = controller.pttRecording.value; - final disabled = !ptt && s != MobileElfStatus.idle && s != MobileElfStatus.error; - final pttClickable = s == MobileElfStatus.speaking; - final sideDisabled = ptt || (disabled && !pttClickable); + final locked = controller.credentialFailed.value; + final disabled = locked || (!ptt && s != MobileElfStatus.idle && s != MobileElfStatus.error); + final pttClickable = !locked && s == MobileElfStatus.speaking; + final sideDisabled = locked || ptt || (disabled && !pttClickable); return Padding( padding: EdgeInsets.fromLTRB(12.w, 6.h, 12.w, 14.h), diff --git a/lib/modules/pairing/controllers/pairing_controller.dart b/lib/modules/pairing/controllers/pairing_controller.dart index 2b63098c6..5156ea8b4 100644 --- a/lib/modules/pairing/controllers/pairing_controller.dart +++ b/lib/modules/pairing/controllers/pairing_controller.dart @@ -810,6 +810,7 @@ class PairingController extends GetxController with WidgetsBindingObserver { orElse: () => UserDevice( id: -1, uid: '', + productid: -1, devicetype: -1, devicename: '', devicemac: '', @@ -840,15 +841,21 @@ class PairingController extends GetxController with WidgetsBindingObserver { orElse: () => UserDevice( id: -1, uid: '', + productid: -1, devicetype: -1, devicename: '', devicemac: '', license: ''), ); if (existingDevice.id != -1) { + _verifiedDevicePid = existingDevice.productid; _verifiedDeviceName = device.name; _verifiedDeviceAddress = device.address; - // 设备时ble 蓝牙 且已绑定 + // 已绑定设备直接用 license 作为 deviceCode（即绑定时扫到的 SN） + if (existingDevice.license.isNotEmpty) { + deviceCode.value = existingDevice.license; + Logger.i(_tag, '[YIDONG] 已绑定设备，从license提取 deviceCode=${existingDevice.license}'); + } return 10; } } @@ -885,14 +892,15 @@ class PairingController extends GetxController with WidgetsBindingObserver { code = extractAndReverseBytes(hexData, 13, 22); _verifiedDevicePid = extractAndReverseBytesToInt(hexData, 4, 5); - print('找到匹配设备，获取到 code: $code'); + final pidHexIos = _verifiedDevicePid?.toRadixString(16).toUpperCase() ?? ''; + Logger.i(_tag, '[YIDONG] iOS BLE扫描 MAC匹配 device=${device.name} pidHex=$pidHexIos code=$code'); if (code != "") { _bleManager.stopScan(); deviceCode.value = code; String? uuids = deviceInfo["id"]; _verifiedDeviceName = device.name; _verifiedDeviceAddress = uuids; - print('找到匹配设备，获取到 uuids: $uuids'); + Logger.i(_tag, '[YIDONG] iOS deviceCode提取成功 code=$code uuid=$uuids'); //检查是否已经绑定 final existingDevice = User.instance.devices.firstWhere( (d) => @@ -902,6 +910,7 @@ class PairingController extends GetxController with WidgetsBindingObserver { orElse: () => UserDevice( id: -1, uid: '', + productid: -1, devicetype: -1, devicename: '', devicemac: '', @@ -921,12 +930,14 @@ class PairingController extends GetxController with WidgetsBindingObserver { code = extractAndReverseBytes(hexData, 15, 24); _verifiedDevicePid = extractAndReverseBytesToInt(hexData, 6, 7); - print('找到匹配设备，获取到 code: $code'); + final pidHexAndroid = _verifiedDevicePid?.toRadixString(16).toUpperCase() ?? ''; + Logger.i(_tag, '[YIDONG] Android BLE扫描 MAC匹配 device=${device.name} mac=${device.address} pidHex=$pidHexAndroid code=$code'); if (code != "") { _bleManager.stopScan(); deviceCode.value = code; _verifiedDeviceName = device.name; _verifiedDeviceAddress = device.address; + Logger.i(_tag, '[YIDONG] Android deviceCode提取成功 code=$code mac=${device.address}'); //检查是否已经绑定 final existingDevice = User.instance.devices.firstWhere( (d) => @@ -936,6 +947,7 @@ class PairingController extends GetxController with WidgetsBindingObserver { orElse: () => UserDevice( id: -1, uid: '', + productid: -1, devicetype: -1, devicename: '', devicemac: '', @@ -1178,26 +1190,92 @@ class PairingController extends GetxController with WidgetsBindingObserver { } } + /// 检测是否为移动精灵（YIDONG）设备，并写入 GetStorage + void _detectAndSaveYidongDevice() { + final yidongPid = AppConfig.env('YIDONG_PID') ?? ''; + if (yidongPid.isEmpty) { + Logger.i(_tag, '[YIDONG] 未配置 YIDONG_PID，跳过检测'); + return; + } + + // 优先使用本次扫描解析的 pid，否则从已绑定设备列表中查找 + int? pid = _verifiedDevicePid; + String pidSource = 'BLE扫描'; + if (pid == null) { + final mac = _verifiedDeviceAddress ?? _currentClassicDevice?.address ?? ''; + Logger.i(_tag, '[YIDONG] 本次未扫描到pid，从已绑定设备查找 mac=$mac'); + if (mac.isNotEmpty) { + final matched = User.instance.devices.firstWhere( + (d) => + d.devicemac.toUpperCase().replaceAll(':', '') == + mac.toUpperCase().replaceAll(':', ''), + orElse: () => UserDevice( + id: -1, uid: '', productid: -1, + devicetype: -1, devicename: '', devicemac: '', license: ''), + ); + if (matched.id != -1) { + pid = matched.productid; + pidSource = '已绑定设备'; + } + } + } + + if (pid == null) { + Logger.i(_tag, '[YIDONG] 无法获取设备pid，跳过检测'); + return; + } + + // 将 int pid 转为十六进制字符串（不补零，大写）与 YIDONG_PID 比对 + final pidHex = pid.toRadixString(16).toUpperCase(); + final isYidong = pidHex == yidongPid.toUpperCase(); + + Logger.i(_tag, '[YIDONG] 设备检测: pidHex=$pidHex yidongPid=${yidongPid.toUpperCase()} source=$pidSource matched=$isYidong'); + + if (isYidong) { + _storage.write('device_agent_type', 'yidong'); + // deviceCode 来源： + // 新设备 → BLE 扫描从广播包提取 + // 已绑定设备 → authbindingDevice 从 existingDevice.license 赋值 + if (deviceCode.value.isNotEmpty) { + final prevCode = _storage.read('device_sts_code') ?? ''; + // SN 变了说明换了设备，清除旧凭证，强制重新向移动平台认证 + if (prevCode != deviceCode.value) { + _storage.remove('sts_device_id'); + _storage.remove('sts_device_secret'); + Logger.i(_tag, '[YIDONG] SN 变更（$prevCode → ${deviceCode.value}），已清除旧凭证，强制重新认证'); + } + _storage.write('device_sts_code', deviceCode.value); + Logger.i(_tag, '[YIDONG] 识别成功，写入 device_sts_code=${deviceCode.value}'); + } else { + // 既无扫描结果也无 license，清除所有凭证，认证失败会提示用户 + _storage.remove('device_sts_code'); + _storage.remove('sts_device_id'); + _storage.remove('sts_device_secret'); + Logger.i(_tag, '[YIDONG] deviceCode 为空，已清除所有凭证'); + } + } else { + _storage.remove('device_agent_type'); + _storage.remove('device_sts_code'); + _storage.remove('sts_device_id'); + _storage.remove('sts_device_secret'); + Logger.i(_tag, '[YIDONG] 非移动精灵设备，已清除相关存储'); + } + } + /// 导航到主页 void _navigateToHome() { - // Future.delayed(const Duration(milliseconds: 500), () async { + // 检测是否为移动精灵（YIDONG）设备 + _detectAndSaveYidongDevice(); // 启动AgentService服务 try { Logger.i(_tag, '启动AgentService服务'); _bleManager.startAgentService(); - // _bleManager.registerBackgroundScan(macAddress: _verifiedDeviceAddress); } catch (e) { Logger.e(_tag, '启动AgentService失败: $e'); } - //print('lxm-上一个路由: ${Get.previousRoute.toString()}'); - // if (Get.previousRoute == Routes.home) { - // 如果是从home页过来的，则直接返回 - // Get.back(); - // } else { - // 否则，跳转到home页并清空之前的路由栈 + // 已在 Home 页时不重复导航，避免两个 HomeView 同时存在导致 Duplicate GlobalKey + if (Get.currentRoute == Routes.home) return; Get.offAllNamed(Routes.home); - // } - // }); } // ==================== 用户操作 ==================== @@ -1259,6 +1337,7 @@ class PairingController extends GetxController with WidgetsBindingObserver { UserDevice( id: 0, // 临时设备ID uid: User.instance.uid, + productid: _verifiedDevicePid ?? -1, devicetype: 1, devicemac: _currentClassicDevice?.address ?? '', devicename: _currentClassicDevice?.name ?? 'Unknown Device', diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index 3a56af051..bbad03eaa 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -987,6 +987,16 @@ object AgentService : CoroutineScope { stsAgent?.ttsMuted = muted } + /** 手动触发心跳上报（Flutter 层调用） */ + fun reportHeartbeat() { + stsAgent?.reportHeartbeat() + } + + /** 主动触发认证检查（移动精灵页面打开时调用，确保认证事件被 Flutter 层接收） */ + fun fetchCredentials() { + stsAgent?.fetchCredentials() + } + fun interruptCurrentResponse() { if (agentMode == "sts") { stsAgent?.interruptSpeaking() diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt index e6a08f893..b513800fa 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt @@ -349,6 +349,26 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream } } + "reportHeartbeat" -> { + try { + AgentService.reportHeartbeat() + result.success(true) + } catch (e: Exception) { + Log.e(TAG, "心跳上报失败", e) + result.error("HEARTBEAT_ERROR", "心跳上报失败: ${e.message}", null) + } + } + + "fetchCredentials" -> { + try { + AgentService.fetchCredentials() + result.success(true) + } catch (e: Exception) { + Log.e(TAG, "触发认证检查失败", e) + result.error("FETCH_CREDENTIALS_ERROR", "触发认证检查失败: ${e.message}", null) + } + } + "setCanWakeUp" -> { try { val canWakeUp = call.argument("canWakeUp") diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt index 8454d3eab..1fbe7ec72 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt @@ -58,6 +58,7 @@ class StsAgent( override val coroutineContext = Dispatchers.IO + job init { + Log.i(TAG, "[YIDONG] StsAgent初始化: productId=${config.productId} deviceNo=${config.deviceNo} deviceId=${config.deviceId} agentId=${config.agentId} hasSecret=${config.deviceSecret.isNotEmpty()}") fetchCredentials() } @@ -89,6 +90,7 @@ class StsAgent( // ── 预缓冲（VAD 触发前 ~1.5s）──────────────────────────────────────────── private val preBuffer = ArrayDeque() + private val preBufferLock = Any() private val PRE_BUFFER_MAX_CHUNKS = 75 // ── 本地 VAD ────────────────────────────────────────────────────────────── @@ -153,8 +155,11 @@ class StsAgent( fun fetchCredentials() { if (config.deviceSecret.isNotEmpty()) { credentialReady = true + Log.i(TAG, "[YIDONG] fetchCredentials: 凭证已存在，跳过请求 deviceNo=${config.deviceNo} deviceId=${config.deviceId}") + reportHeartbeat() // 凭证已有，直接上报心跳 return } + Log.i(TAG, "[YIDONG] fetchCredentials: 开始请求密钥 productId=${config.productId} deviceNo=${config.deviceNo}") launch { try { val body = JSONObject().apply { @@ -180,14 +185,82 @@ class StsAgent( ) credentialReady = config.deviceSecret.isNotEmpty() if (credentialReady) { + Log.i(TAG, "[YIDONG] fetchCredentials: 密钥获取成功 deviceId=${config.deviceId} deviceNo=${config.deviceNo}") onCredentialsFetched?.invoke(config.deviceId, config.deviceSecret, config.deviceNo) + reportHeartbeat() // 凭证获取成功后立即上报心跳 + } else { + Log.e(TAG, "[YIDONG] fetchCredentials: 响应成功但deviceSecret为空") + emit("credential_failed", mapOf("reason" to "设备未认证，deviceSecret为空")) } } else { - Log.e(TAG, "凭证获取失败: ${json.optString("message")}") + val msg = json.optString("message", "未知错误") + Log.e(TAG, "[YIDONG] fetchCredentials: 密钥获取失败 code=${json.optString("code")} msg=$msg") + emit("credential_failed", mapOf("reason" to "设备未认证：$msg")) } } } catch (e: Exception) { - Log.e(TAG, "凭证获取异常: ${e.message}") + Log.e(TAG, "[YIDONG] fetchCredentials: 请求异常 ${e.message}") + emit("credential_failed", mapOf("reason" to "设备认证请求失败：${e.message}")) + } + } + } + + /** 向中国移动平台上报心跳 */ + fun reportHeartbeat() { + if (!credentialReady) { + Log.w(TAG, "[YIDONG] reportHeartbeat: 凭证未就绪，跳过上报") + return + } + Log.i(TAG, "[YIDONG] reportHeartbeat: 开始上报 deviceId=${config.deviceId} deviceNo=${config.deviceNo}") + launch { + try { + val netType = try { + val cm = context.getSystemService(android.content.Context.CONNECTIVITY_SERVICE) + as android.net.ConnectivityManager + val net = cm.activeNetworkInfo + when (net?.type) { + android.net.ConnectivityManager.TYPE_WIFI -> "WiFi" + android.net.ConnectivityManager.TYPE_MOBILE -> "5G" + else -> "WiFi" + } + } catch (_: Exception) { "WiFi" } + + val body = JSONObject().apply { + put("deviceId", config.deviceId) + put("deviceSecret", config.deviceSecret) + put("productId", config.productId) + put("productKey", config.productKey) + put("params", JSONObject().apply { + put("innerIp", JSONArray().put("127.0.0.1")) + put("netSpeed", "0Mbps") + put("netType", netType) + put("platform", "Android${android.os.Build.VERSION.RELEASE}") + put("sdkVersion", "ai_http_1.0") + put("firmwareVersion", "1.0.0") + put("imei", "") + put("cmei", config.deviceNo) // 非蜂窝设备使用 SN 作为 cmei + put("mac", "") + }) + }.toString().toRequestBody("application/json".toMediaType()) + + val req = Request.Builder() + .url("https://ivs.chinamobiledevice.com:11443/v2/customer/device/report") + .post(body) + .build() + + okHttpClient.newCall(req).execute().use { resp -> + val json = JSONObject(resp.body!!.string()) + val code = json.optString("code") + val msg = json.optString("message") + if (code == "200") { + Log.i(TAG, "[YIDONG] reportHeartbeat: 上报成功 code=$code msg=$msg") + emit("heartbeat_reported", mapOf("time" to System.currentTimeMillis())) + } else { + Log.e(TAG, "[YIDONG] reportHeartbeat: 上报失败 code=$code msg=$msg") + } + } + } catch (e: Exception) { + Log.e(TAG, "[YIDONG] reportHeartbeat: 请求异常 ${e.message}") } } } @@ -202,6 +275,7 @@ class StsAgent( if (ttsPlaying) interruptTts() if (pttRecording) stopPtt(cancel = true) if (!credentialReady) { emitError("移动精灵凭证未就绪，请稍后重试"); return } + Log.i(TAG, "[YIDONG] startCall: 启动通话模式 deviceNo=${config.deviceNo} agentId=${config.agentId}") openMic() startMicWatchdog() inCall = true @@ -213,6 +287,7 @@ class StsAgent( if (pttRecording) return if (ttsPlaying) interruptTts() if (!credentialReady) { emitError("移动精灵凭证未就绪"); return } + Log.i(TAG, "[YIDONG] startPtt: 启动PTT模式 deviceNo=${config.deviceNo} agentId=${config.agentId}") pttRecording = true openMic() @@ -287,7 +362,7 @@ class StsAgent( currentPlayer = null } closeWs() - preBuffer.clear() + synchronized(preBufferLock) { preBuffer.clear() } clearSendBuffer() flushPendingAnswer() inCall = false @@ -400,8 +475,10 @@ class StsAgent( } private fun addToPreBuffer(chunk: ByteArray) { - preBuffer.addLast(chunk.copyOf()) - while (preBuffer.size > PRE_BUFFER_MAX_CHUNKS) preBuffer.removeFirst() + synchronized(preBufferLock) { + preBuffer.addLast(chunk.copyOf()) + while (preBuffer.size > PRE_BUFFER_MAX_CHUNKS) preBuffer.removeFirst() + } } // ═════════════════════════════════════════════════════════════════════════ @@ -429,7 +506,7 @@ class StsAgent( forwardingAudio = true; wsConnecting = false emit("recognition_started", emptyMap()) - val cached = preBuffer.toList(); preBuffer.clear() + val cached = synchronized(preBufferLock) { preBuffer.toList().also { preBuffer.clear() } } for (c in cached) feedSendBuffer(c) } @@ -445,6 +522,7 @@ class StsAgent( "?deviceNo=${config.deviceNo}&deviceId=${config.deviceId}" + "&productId=${config.productId}&productKey=${config.productKey}" + "&sign=$sign&sn=$sn&ts=$ts" + Log.i(TAG, "[YIDONG] connectWs: 连接WSS deviceNo=${config.deviceNo} deviceId=${config.deviceId} sn=$sn") val req = Request.Builder().url(url) .header("Origin", "https://ivs.chinamobiledevice.com:11443") @@ -462,6 +540,7 @@ class StsAgent( try { val msg = JSONObject(text) if (msg.optString("type") == "ready") { + Log.i(TAG, "[YIDONG] connectWs: 收到ready，WSS连接成功") if (!resolved) { resolved = true; cont.resume(Unit) {} } } else { launch { handleMessage(msg) } @@ -472,6 +551,7 @@ class StsAgent( override fun onMessage(ws: WebSocket, bytes: okio.ByteString) {} override fun onFailure(ws: WebSocket, t: Throwable, response: Response?) { + Log.e(TAG, "[YIDONG] connectWs: WSS连接失败 ${t.message}") if (webSocket === ws) webSocket = null // 只清除自己的引用，避免覆盖新 WS if (!resolved) { resolved = true; cont.resumeWithException(t) } else launch { handleWsDone() } @@ -502,13 +582,21 @@ class StsAgent( forwardingAudio = false; wsConnecting = false when { sessionEnded && inCall -> if (!ttsPlaying) { resetVad(); startIdleTimer() } - sessionEnded -> { /* PTT 结束，等 TTS 播完 */ } - inCall -> resetVad() + sessionEnded -> { + // PTT 结束；若无 TTS 排队或播放（如无语音输入场景），立即补发 tts_completed + // 避免 Flutter 侧 _pttResultPending 永久无法清除导致按钮锁死 + if (!ttsPlaying && ttsQueue.isEmpty()) emit("tts_completed", emptyMap()) + } + inCall -> resetVad() + // WS 意外关闭（session 未正常结束，如极速松手音频未发出），同样补发保底 + else -> if (!ttsPlaying && ttsQueue.isEmpty() && !ttsInterrupted) emit("tts_completed", emptyMap()) } } private fun resetVad() { - speechDetected = false; vadOnsetCount = 0; preBuffer.clear(); openMic() + speechDetected = false; vadOnsetCount = 0 + synchronized(preBufferLock) { preBuffer.clear() } + openMic() } private fun closeWs() { diff --git a/local_plugins/agent_service/lib/agent_service.dart b/local_plugins/agent_service/lib/agent_service.dart index 1cdec8d65..9478c66fb 100644 --- a/local_plugins/agent_service/lib/agent_service.dart +++ b/local_plugins/agent_service/lib/agent_service.dart @@ -77,6 +77,12 @@ enum AgentServiceEventType { /// 麦克风音量（RMS） micRms, + /// 心跳上报成功 + heartbeatReported, + + /// STS 设备认证失败 + credentialFailed, + /// 未知事件 unknown } @@ -186,6 +192,10 @@ class AgentService { return AgentServiceEventType.stsCredentials; case 'mic_rms': return AgentServiceEventType.micRms; + case 'heartbeat_reported': + return AgentServiceEventType.heartbeatReported; + case 'credential_failed': + return AgentServiceEventType.credentialFailed; default: return AgentServiceEventType.unknown; } @@ -468,6 +478,25 @@ class AgentService { } } + /// 手动触发心跳上报（移动精灵） + static Future reportHeartbeat() async { + try { + await _channel.invokeMethod('reportHeartbeat'); + } on PlatformException catch (e) { + throw AgentServiceException(e.code, e.message ?? '心跳上报失败', e.details); + } + } + + /// 主动触发认证检查（移动精灵页面打开时调用） + /// 若凭证不存在则重新请求，结果通过事件流返回（credentialFailed / heartbeatReported） + static Future fetchCredentials() async { + try { + await _channel.invokeMethod('fetchCredentials'); + } on PlatformException catch (_) { + // 忽略异常，认证结果通过事件流反馈 + } + } + /// 中断当前响应 /// /// 停止语音合成和AI流输出 From f599ed768eeabbcaf4e2f0ec948243c0be50c43e Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Fri, 20 Mar 2026 20:57:14 +0800 Subject: [PATCH 08/14] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E9=97=AA=E9=80=80?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../yunqiinnovation/agent_service/StsAgent.kt | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt index 1fbe7ec72..b0bd08d0c 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt @@ -2,9 +2,11 @@ package com.yunqiinnovation.agent_service import android.content.Context import android.media.AudioFormat +import android.media.AudioManager import android.media.AudioRecord import android.media.MediaPlayer import android.media.MediaRecorder +import android.media.ToneGenerator import android.os.Handler import android.os.Looper import android.util.Log @@ -91,6 +93,9 @@ class StsAgent( // ── 预缓冲（VAD 触发前 ~1.5s）──────────────────────────────────────────── private val preBuffer = ArrayDeque() private val preBufferLock = Any() + + // ── 麦克风读写锁（防止 read() 与 stop()/release() 并发导致 native SIGABRT）── + private val micLock = Any() private val PRE_BUFFER_MAX_CHUNKS = 75 // ── 本地 VAD ────────────────────────────────────────────────────────────── @@ -406,8 +411,15 @@ class StsAgent( recordingJob = launch { val buf = ByteArray(RECORD_CHUNK) while (isActive && (inCall || pttRecording)) { - val read = audioRecord?.read(buf, 0, buf.size) ?: break + // 使用非阻塞读 + micLock，确保与 stopMicHard() 中的 stop()/release() 互斥。 + // READ_NON_BLOCKING 让 read() 立即返回（微秒级），锁持有时间极短， + // 彻底消除 releaseBuffer 并发导致的 native SIGABRT。 + val read = synchronized(micLock) { + audioRecord?.read(buf, 0, buf.size, AudioRecord.READ_NON_BLOCKING) ?: -1 + } + if (read < 0) break if (read > 0) { micChunkCount++; onMicAudio(buf.copyOf(read)) } + else delay(5) // 无数据时释放协程让出线程，避免忙等 } } } catch (e: Exception) { @@ -417,8 +429,13 @@ class StsAgent( private fun stopMicHard() { recordingJob?.cancel(); recordingJob = null - try { audioRecord?.stop(); audioRecord?.release() } catch (_: Exception) {} - audioRecord = null + // 拿 micLock 后再 stop()/release()，与录音循环的 read() 互斥， + // 保证 stop() 永远不会在 read() 内部 releaseBuffer 期间被调用。 + synchronized(micLock) { + val ar = audioRecord + audioRecord = null + try { ar?.stop(); ar?.release() } catch (_: Exception) {} + } } private fun startMicWatchdog() { @@ -504,12 +521,24 @@ class StsAgent( } forwardingAudio = true; wsConnecting = false + playReadyBeep() emit("recognition_started", emptyMap()) val cached = synchronized(preBufferLock) { preBuffer.toList().also { preBuffer.clear() } } for (c in cached) feedSendBuffer(c) } + /** 唤醒成功提示音：通知用户可以说话了 */ + private fun playReadyBeep() { + try { + val toneGen = ToneGenerator(AudioManager.STREAM_MUSIC, 60) + toneGen.startTone(ToneGenerator.TONE_PROP_BEEP, 180) + mainHandler.postDelayed({ toneGen.release() }, 300) + } catch (e: Exception) { + Log.w(TAG, "playReadyBeep failed: ${e.message}") + } + } + // ═════════════════════════════════════════════════════════════════════════ // WebSocket // ═════════════════════════════════════════════════════════════════════════ From ac69a14296124b46292e7ba2e3c913abef1c4f27 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Fri, 20 Mar 2026 22:00:00 +0800 Subject: [PATCH 09/14] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=A7=BB=E5=8A=A8?= =?UTF-8?q?=E8=81=8A=E5=A4=A9=E7=95=8C=E9=9D=A2=20=E7=9A=84=E7=BB=86?= =?UTF-8?q?=E8=8A=82=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/mobile_elf_controller.dart | 35 +++++++++++++++---- .../agent_service/AgentService.kt | 15 ++++++-- .../yunqiinnovation/agent_service/StsAgent.kt | 11 +++++- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart index e11b599fa..b59fc43aa 100644 --- a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart +++ b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart @@ -57,6 +57,8 @@ class MobileElfController extends GetxController { // BLE/AI 唤醒触发的会话（非通话、非PTT模式下的唤醒） bool _bleWakeupActive = false; final _storage = GetStorage(); + // 滚动：layout 变化前是否已在底部 + bool _atBottom = true; // ═══════════════════════════════════════════════════════════════════════════ // 生命周期 @@ -172,8 +174,8 @@ class MobileElfController extends GetxController { case AgentServiceEventType.ttsCompleted: if (!sessionActive) break; - _pttResultPending = false; // PTT 一轮结束 - _bleWakeupActive = false; // BLE 唤醒一轮结束 + _pttResultPending = false; + _endBleWakeup(); // BLE 唤醒一轮结束，清理 native PTT 状态 if (_inCallMode) { status.value = MobileElfStatus.recording; micAlive.value = true; @@ -186,16 +188,16 @@ class MobileElfController extends GetxController { case AgentServiceEventType.ttsCanceled: case AgentServiceEventType.ttsStopped: if (!sessionActive) break; - _pttResultPending = false; // TTS 被打断也视为结束 - _bleWakeupActive = false; + _pttResultPending = false; + _endBleWakeup(); status.value = _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; micAlive.value = _inCallMode; break; case AgentServiceEventType.error: if (!sessionActive) break; - _pttResultPending = false; // 出错时结束等待 - _bleWakeupActive = false; + _pttResultPending = false; + _endBleWakeup(); final msg = event.data['message']?.toString() ?? '未知错误'; status.value = MobileElfStatus.error; errorMsg.value = msg; @@ -305,6 +307,13 @@ class MobileElfController extends GetxController { if (pttRecording.value || _pttStarting) return; _pttStarting = true; _checkHeartbeat(); + // BLE 唤醒/PTT 等待结果期间用户按下 PTT，先中断当前会话（同 startCall 逻辑） + if (_bleWakeupActive || _pttResultPending) { + _pttResultPending = false; + _bleWakeupActive = false; + await AgentService.interruptCurrentResponse(); + await AgentService.stopConversation(); + } final granted = await PermissionUtil.instance.requestPermission( permissionType: Permission.microphone, permissionName: '麦克风', @@ -386,9 +395,21 @@ class MobileElfController extends GetxController { _scrollToBottom(); } + /// BLE 唤醒会话标志清除 + /// Native 状态（pttRecording 等）由 StsAgent.startPtt() 的重入逻辑懒清理， + /// 不在此调用 stopConversation()，避免与新会话产生竞态。 + void _endBleWakeup() { + _bleWakeupActive = false; + } + void _scrollToBottom() { + // layout 变化前捕获是否在底部，避免新内容撑开后差值变大误判为"已滚离底部" + if (scrollController.hasClients) { + final pos = scrollController.position; + _atBottom = pos.maxScrollExtent - pos.pixels <= 80; + } WidgetsBinding.instance.addPostFrameCallback((_) { - if (scrollController.hasClients) { + if (scrollController.hasClients && _atBottom) { scrollController.animateTo( scrollController.position.maxScrollExtent, duration: const Duration(milliseconds: 300), diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index bbad03eaa..eacbecf4b 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -225,6 +225,7 @@ object AgentService : CoroutineScope { // STS 模式：跳过 Azure/OpenAI 初始化，直接启动 StsAgent if (agentMode == "sts") { this.context = appContext.applicationContext + audioPlayer = AudioPlayer(context) // STS 模式也需要音频播放器（提示音等） val stsConfig = StsConfig( productId = config["stsProductId"]?.toString() ?: "", productKey = config["stsProductKey"]?.toString() ?: "", @@ -236,7 +237,13 @@ object AgentService : CoroutineScope { stsAgent = StsAgent( context = context, config = stsConfig, - onEvent = { name, data -> sendEvent(name, data) }, + onEvent = { name, data -> + // BLE 唤醒：AI 开始回复时关闭 BLE 编解码器（说明用户已说完） + if (currentRecognitionMode == "ble_wakeup" && name == "assistant_response") { + BleService.closeCodec() + } + sendEvent(name, data) + }, onCredentialsFetched = { deviceId, deviceSecret, deviceNo -> sendEvent("sts_credentials", mapOf( "deviceId" to deviceId, @@ -685,10 +692,14 @@ object AgentService : CoroutineScope { */ fun startRecognition(isExternalActive: Boolean, mode: String = "normal"): Boolean { if (agentMode == "sts") { + currentRecognitionMode = mode // 记录模式供 onEvent 回调使用 when (mode) { "push_to_talk" -> stsAgent?.startPtt() "phone_call" -> stsAgent?.startCall() - "ble_wakeup" -> stsAgent?.startCall() // BLE 唤醒：进入通话模式（VAD 自动检测） + "ble_wakeup" -> { + audioPlayer?.playAudio(R.raw.start) // 立即播提示音，不等 WS 连接完成 + stsAgent?.startPtt() + } else -> stsAgent?.startPtt() // 普通模式默认 PTT } return true diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt index b0bd08d0c..5c28b2c5e 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt @@ -289,7 +289,16 @@ class StsAgent( /** PTT 模式：立即连 WS 并开始录音 */ fun startPtt() { - if (pttRecording) return + if (pttRecording) { + if (forwardingAudio) return // 真正在录音中，忽略重复调用 + // forwardingAudio=false：WS 已关闭，正处于 TTS 播报等待阶段（如 BLE 唤醒等待 AI 回复期间） + // 调用方（BleAgent）已通过 interruptCurrentResponse() 中断了 TTS， + // 此处只做静默状态重置，不再重复发出 tts_canceled 事件 + Log.i(TAG, "[YIDONG] startPtt: 静默重置旧会话，重新启动") + stopMicHard() + sessionEnded = false + pttRecording = false + } if (ttsPlaying) interruptTts() if (!credentialReady) { emitError("移动精灵凭证未就绪"); return } Log.i(TAG, "[YIDONG] startPtt: 启动PTT模式 deviceNo=${config.deviceNo} agentId=${config.agentId}") From 6ac8b2a9c347486739f023224418f738cb110201 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Mon, 23 Mar 2026 18:15:35 +0800 Subject: [PATCH 10/14] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E7=A7=BB=E5=8A=A8cmei?= =?UTF-8?q?=E7=A0=81=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/data/models/user_Info.dart | 4 + lib/modules/devices/views/devices_view.dart | 571 ++++++++++++++------ {ui => preview}/yidong.html | 0 3 files changed, 423 insertions(+), 152 deletions(-) rename {ui => preview}/yidong.html (100%) diff --git a/lib/data/models/user_Info.dart b/lib/data/models/user_Info.dart index 2467ba7bd..e6125b672 100644 --- a/lib/data/models/user_Info.dart +++ b/lib/data/models/user_Info.dart @@ -217,6 +217,7 @@ class UserDevice { String devicename; String devicemac; String license; + String cmei; UserDevice({ required this.id, required this.uid, @@ -225,6 +226,7 @@ class UserDevice { required this.devicename, required this.devicemac, required this.license, + this.cmei = '', }); factory UserDevice.fromJson(Map json) => UserDevice( @@ -235,6 +237,7 @@ class UserDevice { devicename: json['devicename'] as String, devicemac: json['devicemac'] as String, license: json['license'] as String, + cmei: json['cmei'] as String? ?? '', ); toJson() => { @@ -245,5 +248,6 @@ class UserDevice { 'devicename': devicename, 'devicemac': devicemac, 'license': license, + 'cmei': cmei, }; } diff --git a/lib/modules/devices/views/devices_view.dart b/lib/modules/devices/views/devices_view.dart index 7a69dd5b0..30b5570ee 100644 --- a/lib/modules/devices/views/devices_view.dart +++ b/lib/modules/devices/views/devices_view.dart @@ -1,12 +1,10 @@ -import 'dart:ui'; - import '../../../routes/app_routes.dart'; -import '/modules/devices/controllers/devices_controller.dart'; import 'package:flutter/material.dart'; import 'package:flutter_screenutil/flutter_screenutil.dart'; import 'package:get/get.dart'; import '/core/theme/app_colors.dart'; import '/data/models/user_Info.dart'; +import '/data/services/ble_manager.dart'; import '../controllers/devices_controller.dart'; // 设备类型枚举 @@ -20,7 +18,6 @@ enum DeviceType { const DeviceType(this.value); final int value; - // 使用getter获取多语言显示名称 String get displayName { switch (this) { case DeviceType.Unknown: @@ -61,112 +58,240 @@ enum DeviceType { class DevicesView extends GetView { const DevicesView({super.key}); + bool _isConnected(UserDevice device) { + try { + final bleManager = Get.find(); + return bleManager.currentDeviceAddress != null && + bleManager.currentDeviceAddress == device.devicemac; + } catch (_) { + return false; + } + } + @override Widget build(BuildContext context) { - final isDarkMode = Theme.of(context).brightness == Brightness.dark; + final isDark = Theme.of(context).brightness == Brightness.dark; + final bg = isDark ? const Color(0xFF0A0A0F) : const Color(0xFFF5F6FA); + final topBg = isDark ? const Color(0xFF0F1018) : const Color(0xFFF0F1F6); return Scaffold( - backgroundColor: isDarkMode ? AppColors.darkBackground : Colors.white, + backgroundColor: bg, body: Column( children: [ - // 顶部标题栏 - Container( - padding: EdgeInsets.only( - top: MediaQuery.of(context).padding.top + 10.h, - left: 20.w, - right: 20.w, - bottom: 20.h, - ), - decoration: BoxDecoration( - color: isDarkMode ? Colors.black : Colors.white, - boxShadow: [ - BoxShadow( - color: Colors.black.withOpacity(0.05), - blurRadius: 10, - offset: const Offset(0, 2), - ), - ], - ), - child: Row( - children: [ - GestureDetector( - onTap: () => Get.back(), + // 顶部区域 + _buildTopArea(context, isDark, bg, topBg), + // 设备列表 + Expanded(child: _buildDevicesList(isDark, topBg)), + ], + ), + ); + } + + Widget _buildTopArea( + BuildContext context, bool isDark, Color bg, Color topBg) { + return Container( + padding: EdgeInsets.only( + top: MediaQuery.of(context).padding.top + 8.h, + left: 20.w, + right: 20.w, + bottom: 24.h, + ), + decoration: BoxDecoration(color: bg), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + // 导航栏 + Row( + children: [ + GestureDetector( + onTap: () => Get.back(), + child: Container( + width: 40.w, + height: 40.w, + decoration: BoxDecoration( + shape: BoxShape.circle, + color: isDark + ? Colors.white.withOpacity(0.06) + : Colors.black.withOpacity(0.04), + ), child: Icon( - Icons.arrow_back_ios, - color: isDarkMode ? Colors.white : Colors.black, - size: 24.sp, + Icons.arrow_back_ios_new, + size: 18.sp, + color: isDark ? Colors.white : const Color(0xFF222222), ), ), - SizedBox(width: 16.w), - Text( - "deviceManagement".tr, - style: TextStyle( - fontSize: 20.sp, - fontWeight: FontWeight.w600, - color: isDarkMode ? Colors.white : Colors.black, - ), + ), + SizedBox(width: 16.w), + Text( + "deviceManagement".tr, + style: TextStyle( + fontSize: 26.sp, + fontWeight: FontWeight.w800, + color: isDark ? Colors.white : const Color(0xFF111111), + letterSpacing: -0.5, ), - ], - ), - ), - // 设备列表内容 - Expanded( - child: _buildDevicesList(isDarkMode), + ), + ], ), + SizedBox(height: 20.h), + // 摘要卡片 + Obx(() { + final totalDevices = controller.devices.values + .fold(0, (sum, list) => sum + list.length); + final connectedCount = controller.devices.values + .expand((list) => list) + .where((d) => _isConnected(d)) + .length; + return Row( + children: [ + _buildSummaryCard( + number: totalDevices.toString(), + label: "boundDevices".tr, + isActive: true, + isDark: isDark, + ), + SizedBox(width: 12.w), + _buildSummaryCard( + number: connectedCount.toString(), + label: "currentConnection".tr, + isActive: false, + isDark: isDark, + ), + ], + ); + }), ], ), ); } - Widget _buildDevicesList(bool isDarkMode) { + Widget _buildSummaryCard({ + required String number, + required String label, + required bool isActive, + required bool isDark, + }) { + final activeGradient = isDark + ? [ + const Color(0xFF6366F1).withOpacity(0.15), + const Color(0xFF8B5CF6).withOpacity(0.1), + ] + : [const Color(0xFFEEF2FF), const Color(0xFFF5F3FF)]; + + return Expanded( + child: Container( + padding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 14.h), + decoration: BoxDecoration( + borderRadius: BorderRadius.circular(16.r), + gradient: isActive ? LinearGradient(colors: activeGradient) : null, + color: isActive + ? null + : (isDark + ? Colors.white.withOpacity(0.04) + : Colors.white), + border: Border.all( + color: isActive + ? (isDark + ? const Color(0xFF6366F1).withOpacity(0.2) + : const Color(0xFF6366F1).withOpacity(0.15)) + : (isDark + ? Colors.white.withOpacity(0.06) + : Colors.black.withOpacity(0.04)), + ), + boxShadow: isDark + ? null + : [ + BoxShadow( + color: Colors.black.withOpacity(0.04), + blurRadius: 4, + offset: const Offset(0, 1), + ), + ], + ), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + number, + style: TextStyle( + fontSize: 24.sp, + fontWeight: FontWeight.w700, + color: isActive + ? (isDark + ? const Color(0xFFA5B4FC) + : const Color(0xFF6366F1)) + : (isDark ? Colors.white : const Color(0xFF222222)), + ), + ), + SizedBox(height: 2.h), + Text( + label, + style: TextStyle( + fontSize: 11.sp, + fontWeight: FontWeight.w500, + color: isActive + ? (isDark + ? const Color(0xFFA5B4FC).withOpacity(0.6) + : const Color(0xFF818CF8)) + : (isDark + ? Colors.white.withOpacity(0.35) + : const Color(0xFF999999)), + ), + ), + ], + ), + ), + ); + } + + Widget _buildDevicesList(bool isDark, Color topBg) { return Obx(() { - // 使用控制器的响应式数据，这会触发控制器的实例化 final groupedDevices = controller.devices; if (groupedDevices.isEmpty) { - return _buildEmptyState(isDarkMode); + return _buildEmptyState(isDark); } return ListView( - padding: EdgeInsets.all(20.w), + padding: EdgeInsets.symmetric(horizontal: 16.w, vertical: 20.h), children: [ - // 遍历每个设备类型 ...groupedDevices.entries.map((entry) { - final deviceTypeValue = entry.key; + final deviceType = DeviceType.fromValue(entry.key); final deviceList = entry.value; - final deviceType = DeviceType.fromValue(deviceTypeValue); return Column( crossAxisAlignment: CrossAxisAlignment.start, children: [ - // 设备类型标题 + // 分类标题 Padding( - padding: EdgeInsets.only(bottom: 12.h), + padding: EdgeInsets.only(left: 8.w, bottom: 12.h), child: Text( - deviceType.displayName, + deviceType.displayName.toUpperCase(), style: TextStyle( - fontSize: 16.sp, + fontSize: 12.sp, fontWeight: FontWeight.w600, - color: isDarkMode ? Colors.white : Colors.black87, + color: isDark + ? Colors.white.withOpacity(0.25) + : const Color(0xFFAAAAAA), + letterSpacing: 1.5, ), ), ), - // 该类型下的设备列表 ...deviceList.map((device) => _buildDeviceCard( device: device, deviceType: deviceType, - isDarkMode: isDarkMode, + isDark: isDark, )), - SizedBox(height: 20.h), + SizedBox(height: 16.h), ], ); - }).toList(), + }), ], ); }); } - Widget _buildEmptyState(bool isDarkMode) { + Widget _buildEmptyState(bool isDark) { return Center( child: Column( mainAxisAlignment: MainAxisAlignment.center, @@ -174,14 +299,14 @@ class DevicesView extends GetView { Icon( Icons.devices_other, size: 80.sp, - color: isDarkMode ? Colors.white54 : Colors.grey[400], + color: isDark ? Colors.white54 : Colors.grey[400], ), SizedBox(height: 20.h), Text( "noDevicesBound".tr, style: TextStyle( fontSize: 16.sp, - color: isDarkMode ? Colors.white70 : Colors.grey[600], + color: isDark ? Colors.white70 : Colors.grey[600], ), ), SizedBox(height: 12.h), @@ -189,7 +314,7 @@ class DevicesView extends GetView { "pleaseBindDeviceFirst".tr, style: TextStyle( fontSize: 14.sp, - color: isDarkMode ? Colors.white54 : Colors.grey[500], + color: isDark ? Colors.white54 : Colors.grey[500], ), ), ], @@ -200,121 +325,263 @@ class DevicesView extends GetView { Widget _buildDeviceCard({ required UserDevice device, required DeviceType deviceType, - required bool isDarkMode, + required bool isDark, }) { + final connected = _isConnected(device); + final connectedColor = const Color(0xFF22C55E); + return GestureDetector( onTap: () => Get.toNamed(Routes.eqSettings), child: Container( - margin: EdgeInsets.only(bottom: 12.h), - padding: EdgeInsets.all(16.w), + margin: EdgeInsets.only(bottom: 10.h), + padding: EdgeInsets.all(20.w), decoration: BoxDecoration( - color: isDarkMode ? AppColors.cardBackground : Colors.white, - borderRadius: BorderRadius.circular(12.r), - boxShadow: [ - BoxShadow( - color: Colors.black.withOpacity(0.05), - blurRadius: 8, - offset: const Offset(0, 2), - ), - ], + borderRadius: BorderRadius.circular(20.r), + color: isDark + ? (connected + ? connectedColor.withOpacity(0.04) + : Colors.white.withOpacity(0.03)) + : (connected + ? connectedColor.withOpacity(0.03) + : Colors.white), + border: Border.all( + color: connected + ? connectedColor.withOpacity(0.2) + : (isDark + ? Colors.white.withOpacity(0.05) + : Colors.black.withOpacity(0.04)), + ), + boxShadow: isDark + ? null + : [ + BoxShadow( + color: Colors.black.withOpacity(0.03), + blurRadius: 12, + offset: const Offset(0, 2), + ), + ], ), - child: Row( + child: Column( children: [ - // 设备图标 - Container( - width: 48.w, - height: 48.w, - decoration: BoxDecoration( - color: isDarkMode - ? Colors.white.withOpacity(0.1) - : Colors.grey[100], - borderRadius: BorderRadius.circular(12.r), - ), - child: Icon( - deviceType.icon, - color: isDarkMode ? Colors.white : Colors.black87, - size: 24.sp, - ), - ), - SizedBox(width: 16.w), - // 设备信息 - Expanded( - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Text( - device.devicename.isNotEmpty - ? device.devicename - : "deviceTypeUnknown".tr, - style: TextStyle( - fontSize: 16.sp, - fontWeight: FontWeight.w500, - color: isDarkMode ? Colors.white : Colors.black87, + // 上部：图标 + 名称/MAC + 解绑 + Row( + children: [ + // 设备图标 + Container( + width: 46.w, + height: 46.w, + decoration: BoxDecoration( + borderRadius: BorderRadius.circular(14.r), + gradient: LinearGradient( + begin: Alignment.topLeft, + end: Alignment.bottomRight, + colors: connected + ? [connectedColor, const Color(0xFF16A34A)] + : [const Color(0xFF6366F1), const Color(0xFF8B5CF6)], ), + boxShadow: [ + BoxShadow( + color: (connected ? connectedColor : const Color(0xFF6366F1)) + .withOpacity(0.3), + blurRadius: 16, + offset: const Offset(0, 4), + ), + ], ), - SizedBox(height: 4.h), - Text( - device.devicemac.isNotEmpty - ? device.devicemac - : "noMacAddress".tr, - style: TextStyle( - fontSize: 14.sp, - color: isDarkMode ? Colors.white70 : Colors.grey[600], - ), + child: Icon( + deviceType.icon, + color: Colors.white, + size: 22.sp, ), - if (device.license.isNotEmpty) ...[ - SizedBox(height: 4.h), - Text( - "${"licenseCode".tr}: ${device.license}", + ), + SizedBox(width: 14.w), + // 设备名 + MAC + Expanded( + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Row( + children: [ + Text( + device.devicename.isNotEmpty + ? device.devicename + : "deviceTypeUnknown".tr, + style: TextStyle( + fontSize: 16.sp, + fontWeight: FontWeight.w700, + color: isDark + ? Colors.white + : const Color(0xFF1A1A2E), + ), + ), + if (connected) ...[ + SizedBox(width: 8.w), + Container( + width: 6.w, + height: 6.w, + decoration: BoxDecoration( + shape: BoxShape.circle, + color: connectedColor, + boxShadow: [ + BoxShadow( + color: connectedColor.withOpacity(0.6), + blurRadius: 8, + ), + ], + ), + ), + SizedBox(width: 6.w), + Container( + padding: EdgeInsets.symmetric( + horizontal: 8.w, vertical: 3.h), + decoration: BoxDecoration( + borderRadius: BorderRadius.circular(6.r), + color: connectedColor.withOpacity(0.1), + ), + child: Text( + "connected".tr, + style: TextStyle( + fontSize: 10.sp, + fontWeight: FontWeight.w600, + color: connectedColor, + ), + ), + ), + ], + ], + ), + SizedBox(height: 3.h), + Text( + device.devicemac.isNotEmpty + ? device.devicemac + : "noMacAddress".tr, + style: TextStyle( + fontSize: 12.sp, + fontWeight: FontWeight.w500, + color: isDark + ? Colors.white.withOpacity(0.3) + : const Color(0xFF999999), + fontFamily: 'Menlo', + letterSpacing: 0.5, + ), + ), + ], + ), + ), + // 解绑按钮 + GestureDetector( + onTap: () => _showUnbindDialog(device, isDark), + child: Container( + padding: EdgeInsets.symmetric( + horizontal: 14.w, vertical: 8.h), + decoration: BoxDecoration( + borderRadius: BorderRadius.circular(10.r), + border: Border.all( + color: isDark + ? Colors.white.withOpacity(0.08) + : Colors.black.withOpacity(0.08), + ), + ), + child: Text( + "unbind".tr, style: TextStyle( fontSize: 12.sp, - color: isDarkMode ? Colors.white54 : Colors.grey[500], + fontWeight: FontWeight.w600, + color: isDark + ? Colors.white.withOpacity(0.4) + : const Color(0xFF999999), ), ), - ], - ], - ), - ), - // 解绑按钮 - GestureDetector( - onTap: () => _showUnbindDialog(device, isDarkMode), - child: Container( - padding: EdgeInsets.symmetric( - horizontal: 12.w, - vertical: 6.h, + ), ), + ], + ), + // 下部：授权码 / CMEI + if (device.license.isNotEmpty || device.cmei.isNotEmpty) ...[ + SizedBox(height: 16.h), + Container( + padding: EdgeInsets.all(12.w), decoration: BoxDecoration( - color: Colors.red.withOpacity(0.1), - borderRadius: BorderRadius.circular(6.r), - border: Border.all( - color: Colors.red.withOpacity(0.3), - width: 1, - ), + borderRadius: BorderRadius.circular(12.r), + color: isDark + ? Colors.white.withOpacity(0.03) + : const Color(0xFFF7F7FA), ), - child: Text( - "unbind".tr, - style: TextStyle( - fontSize: 12.sp, - color: Colors.red, - fontWeight: FontWeight.w500, - ), + child: Column( + children: [ + if (device.license.isNotEmpty) + _buildFieldRow( + label: "licenseCode".tr, + value: device.license, + isDark: isDark, + ), + if (device.license.isNotEmpty && device.cmei.isNotEmpty) + SizedBox(height: 8.h), + if (device.cmei.isNotEmpty) + _buildFieldRow( + label: "CMEI", + value: device.cmei, + isDark: isDark, + ), + ], ), ), - ), + ], ], ), ), ); } - void _showUnbindDialog(UserDevice device, bool isDarkMode) { + Widget _buildFieldRow({ + required String label, + required String value, + required bool isDark, + }) { + return Row( + children: [ + SizedBox( + width: 44.w, + child: Text( + label, + style: TextStyle( + fontSize: 10.sp, + fontWeight: FontWeight.w600, + color: isDark + ? Colors.white.withOpacity(0.2) + : const Color(0xFFBBBBBB), + letterSpacing: 0.5, + ), + ), + ), + SizedBox(width: 8.w), + Expanded( + child: Text( + value, + style: TextStyle( + fontSize: 11.sp, + fontWeight: FontWeight.w500, + color: isDark + ? Colors.white.withOpacity(0.5) + : const Color(0xFF666666), + fontFamily: 'Menlo', + letterSpacing: 0.3, + ), + overflow: TextOverflow.ellipsis, + ), + ), + ], + ); + } + + void _showUnbindDialog(UserDevice device, bool isDark) { Get.dialog( AlertDialog( - backgroundColor: isDarkMode ? AppColors.cardBackground : Colors.white, + backgroundColor: isDark ? AppColors.cardBackground : Colors.white, title: Text( "confirmUnbind".tr, style: TextStyle( - color: isDarkMode ? Colors.white : Colors.black87, + color: isDark ? Colors.white : Colors.black87, ), ), content: Text( @@ -322,7 +589,7 @@ class DevicesView extends GetView { .tr .replaceAll('{deviceName}', device.devicename), style: TextStyle( - color: isDarkMode ? Colors.white70 : Colors.black54, + color: isDark ? Colors.white70 : Colors.black54, ), ), actions: [ @@ -331,7 +598,7 @@ class DevicesView extends GetView { child: Text( "cancel".tr, style: TextStyle( - color: isDarkMode ? Colors.white70 : Colors.grey[600], + color: isDark ? Colors.white70 : Colors.grey[600], ), ), ), diff --git a/ui/yidong.html b/preview/yidong.html similarity index 100% rename from ui/yidong.html rename to preview/yidong.html From 7e6744134376bf86ac92c959ba702f50c8fe97bd Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Mon, 23 Mar 2026 19:07:54 +0800 Subject: [PATCH 11/14] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=8C=89=E9=92=AE?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E9=94=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/mobile_elf_controller.dart | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart index b59fc43aa..356b0f726 100644 --- a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart +++ b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart @@ -48,6 +48,7 @@ class MobileElfController extends GetxController { StreamSubscription? _eventSub; Timer? _callTimer; Timer? _pttTimer; + Timer? _pttSafetyTimer; bool _inCallMode = false; bool _callStarting = false; // 防止并发 startCall // PTT 松开后仍有异步事件待处理（recognition_result / assistant_response / tts） @@ -56,6 +57,8 @@ class MobileElfController extends GetxController { bool _pttStarting = false; // BLE/AI 唤醒触发的会话（非通话、非PTT模式下的唤醒） bool _bleWakeupActive = false; + // PTT 被中止（轻点松手），用于忽略原生层的残留事件 + bool _pttAborted = false; final _storage = GetStorage(); // 滚动：layout 变化前是否已在底部 bool _atBottom = true; @@ -107,6 +110,7 @@ class MobileElfController extends GetxController { _eventSub?.cancel(); _callTimer?.cancel(); _pttTimer?.cancel(); + _pttSafetyTimer?.cancel(); AgentService.stopConversation(); scrollController.dispose(); super.onClose(); @@ -127,6 +131,11 @@ class MobileElfController extends GetxController { final sessionActive = _inCallMode || pttRecording.value || _pttResultPending || _bleWakeupActive; switch (event.type) { case AgentServiceEventType.recognitionStarted: + // 忽略刚刚中止的 PTT 会话的残留事件，避免误触发 BLE 唤醒 + if (_pttAborted && !sessionActive) { + _pttAborted = false; + break; + } // BLE/AI 唤醒：即使没有主动会话也激活 if (!sessionActive) _bleWakeupActive = true; status.value = MobileElfStatus.recording; @@ -175,6 +184,7 @@ class MobileElfController extends GetxController { case AgentServiceEventType.ttsCompleted: if (!sessionActive) break; _pttResultPending = false; + _pttSafetyTimer?.cancel(); _endBleWakeup(); // BLE 唤醒一轮结束，清理 native PTT 状态 if (_inCallMode) { status.value = MobileElfStatus.recording; @@ -189,6 +199,7 @@ class MobileElfController extends GetxController { case AgentServiceEventType.ttsStopped: if (!sessionActive) break; _pttResultPending = false; + _pttSafetyTimer?.cancel(); _endBleWakeup(); status.value = _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; micAlive.value = _inCallMode; @@ -197,6 +208,7 @@ class MobileElfController extends GetxController { case AgentServiceEventType.error: if (!sessionActive) break; _pttResultPending = false; + _pttSafetyTimer?.cancel(); _endBleWakeup(); final msg = event.data['message']?.toString() ?? '未知错误'; status.value = MobileElfStatus.error; @@ -240,6 +252,7 @@ class MobileElfController extends GetxController { Future startCall() async { if (inCall.value || _callStarting) return; _callStarting = true; + _pttAborted = false; _checkHeartbeat(); try { // 若有正在播报的 PTT/BLE 唤醒会话，先强制中断 TTS/AI 流， @@ -281,6 +294,7 @@ class MobileElfController extends GetxController { _inCallMode = false; _pttStarting = false; _pttResultPending = false; // 强制结束所有等待 + _pttSafetyTimer?.cancel(); _bleWakeupActive = false; inCall.value = false; _callTimer?.cancel(); @@ -306,6 +320,7 @@ class MobileElfController extends GetxController { Future startPtt() async { if (pttRecording.value || _pttStarting) return; _pttStarting = true; + _pttAborted = false; _checkHeartbeat(); // BLE 唤醒/PTT 等待结果期间用户按下 PTT，先中断当前会话（同 startCall 逻辑） if (_bleWakeupActive || _pttResultPending) { @@ -343,10 +358,33 @@ class MobileElfController extends GetxController { } if (!pttRecording.value) return; pttRecording.value = false; - _pttResultPending = true; // 等待 fin_result/assistant_response/tts 异步回来 _pttTimer?.cancel(); _pttTimer = null; + + // 如果还在连接中（还未开始识别），说明用户轻点了一下就松手， + // 直接恢复空闲，不需要等待异步事件回来 + if (status.value == MobileElfStatus.connecting) { + _pttAborted = true; // 标记中止，忽略原生层的残留事件 + _pttResultPending = false; + status.value = MobileElfStatus.idle; + micAlive.value = false; + AgentService.stopConversation(); + return; + } + + _pttResultPending = true; // 等待 fin_result/assistant_response/tts 异步回来 AgentService.stopConversation(); + + // 安全超时：防止原生层未返回事件导致状态永久卡住 + _pttSafetyTimer?.cancel(); + _pttSafetyTimer = Timer(const Duration(seconds: 8), () { + if (_pttResultPending && !_inCallMode) { + Logger.info('$_tag PTT safety timeout — force reset to idle'); + _pttResultPending = false; + status.value = MobileElfStatus.idle; + micAlive.value = false; + } + }); } void interruptSpeaking() { From e8425dc8a79f958d5bf4905dbb9f2412c3eecf59 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Tue, 24 Mar 2026 14:55:24 +0800 Subject: [PATCH 12/14] =?UTF-8?q?=E4=B8=8A=E6=8A=A5cmei=20=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/data/services/ble_manager.dart | 11 +++++++++++ .../agent_service/AgentService.kt | 6 ++++++ .../agent_service/AgentServicePlugin.kt | 1 + .../agent_service/BleMediaService.kt | 3 +++ .../yunqiinnovation/agent_service/StsAgent.kt | 17 +++++++++++++---- 5 files changed, 34 insertions(+), 4 deletions(-) diff --git a/lib/data/services/ble_manager.dart b/lib/data/services/ble_manager.dart index 7327929b6..ede3f510b 100644 --- a/lib/data/services/ble_manager.dart +++ b/lib/data/services/ble_manager.dart @@ -966,6 +966,16 @@ class BleManager extends GetxService { final stsDeviceNo = _storage.read('device_sts_code') ?? ''; final stsDeviceId = _storage.read('sts_device_id') ?? ''; final stsDeviceSecret = _storage.read('sts_device_secret') ?? ''; + // 从 UserDevice 获取当前连接设备的 CMEI + final mac = currentDeviceAddress?.toUpperCase() ?? ''; + String stsCmei = ''; + try { + final matchedDevice = userInfo.User.instance.devices.firstWhere( + (d) => d.devicemac.toUpperCase() == mac, + ); + stsCmei = matchedDevice.cmei; + } catch (_) {} + Logger.info('lxm--当前设备 CMEI: $stsCmei (MAC=$mac)'); final config = { // STS 模式路由 @@ -976,6 +986,7 @@ class BleManager extends GetxService { 'stsDeviceNo': stsDeviceNo, 'stsDeviceId': stsDeviceId, 'stsDeviceSecret': stsDeviceSecret, + 'stsCmei': stsCmei, // 语音代理配置参数 'azureSpeechKey': azureSpeechKey, 'azureSpeechRegion': azureSpeechRegion, diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index eacbecf4b..997fb74a7 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -233,6 +233,7 @@ object AgentService : CoroutineScope { deviceId = config["stsDeviceId"]?.toString() ?: "", deviceSecret = config["stsDeviceSecret"]?.toString() ?: "", agentId = agentId, + cmei = config["stsCmei"]?.toString() ?: "", ) stsAgent = StsAgent( context = context, @@ -371,6 +372,11 @@ object AgentService : CoroutineScope { config["mcpServer"]?.toString() ?: "", config["musicServiceType"]?.toString() ?: "" ) + // 更新 STS 模式参数（cmei 等可能在首次初始化时尚未获取到） + val newCmei = config["stsCmei"]?.toString() ?: "" + if (newCmei.isNotEmpty()) { + stsAgent?.updateCmei(newCmei) + } } catch (e: Exception) { Log.e(TAG, "应用配置更新失败 config: $config Exception:${e.message}") } diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt index b513800fa..1b0bf0cd1 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt @@ -91,6 +91,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream config["stsDeviceNo"]?.toString()?.let { intent.putExtra("stsDeviceNo", it) } config["stsDeviceId"]?.toString()?.let { intent.putExtra("stsDeviceId", it) } config["stsDeviceSecret"]?.toString()?.let { intent.putExtra("stsDeviceSecret", it) } + config["stsCmei"]?.toString()?.let { intent.putExtra("stsCmei", it) } config["vipexptime"]?.toString()?.let { intent.putExtra("vipexptime", it) //打印日志 diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt index 20f0ac498..574af96ff 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt @@ -85,6 +85,7 @@ class BleMediaService : Service(), CoroutineScope { private const val KEY_STS_DEVICE_NO = "stsDeviceNo" private const val KEY_STS_DEVICE_ID = "stsDeviceId" private const val KEY_STS_DEVICE_SECRET = "stsDeviceSecret" + private const val KEY_STS_CMEI = "stsCmei" // 用于跟踪服务是否正在运行 private val isRunning = AtomicBoolean(false) @@ -621,6 +622,7 @@ private var companionManager: BleCompanionManager? = null it.getStringExtra(KEY_STS_DEVICE_NO)?.let { value -> config[KEY_STS_DEVICE_NO] = value } it.getStringExtra(KEY_STS_DEVICE_ID)?.let { value -> config[KEY_STS_DEVICE_ID] = value } it.getStringExtra(KEY_STS_DEVICE_SECRET)?.let { value -> config[KEY_STS_DEVICE_SECRET] = value } + it.getStringExtra(KEY_STS_CMEI)?.let { value -> config[KEY_STS_CMEI] = value } Log.d(TAG, "liwei-------vipexptime 3-1 ${ config[KEY_VIP_EXPTIME]}") // 提取 supportedLanguages 参数 it.getStringArrayListExtra(KEY_SUPPORTED_LANGUAGES)?.let { languages -> @@ -695,6 +697,7 @@ private var companionManager: BleCompanionManager? = null config[KEY_STS_DEVICE_NO] = getConfigString(prefs, KEY_STS_DEVICE_NO, "") config[KEY_STS_DEVICE_ID] = getConfigString(prefs, KEY_STS_DEVICE_ID, "") config[KEY_STS_DEVICE_SECRET] = getConfigString(prefs, KEY_STS_DEVICE_SECRET, "") + config[KEY_STS_CMEI] = getConfigString(prefs, KEY_STS_CMEI, "") Log.d(TAG, "liwei-------vipexptime 3-0 ${ config[KEY_VIP_EXPTIME]}") // 添加 supportedLanguages 参数处理 val supportedLanguagesStr = getConfigString(prefs, KEY_SUPPORTED_LANGUAGES, "") diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt index 5c28b2c5e..76938f508 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt @@ -40,6 +40,7 @@ data class StsConfig( var deviceId: String, var deviceSecret: String, val agentId: String, + val cmei: String = "", ) /** @@ -60,10 +61,18 @@ class StsAgent( override val coroutineContext = Dispatchers.IO + job init { - Log.i(TAG, "[YIDONG] StsAgent初始化: productId=${config.productId} deviceNo=${config.deviceNo} deviceId=${config.deviceId} agentId=${config.agentId} hasSecret=${config.deviceSecret.isNotEmpty()}") + Log.i(TAG, "[YIDONG] StsAgent初始化: productId=${config.productId} deviceNo=${config.deviceNo} deviceId=${config.deviceId} agentId=${config.agentId} cmei=${config.cmei} hasSecret=${config.deviceSecret.isNotEmpty()}") fetchCredentials() } + /** 动态更新 CMEI（首次初始化时可能尚未获取到） */ + fun updateCmei(cmei: String) { + if (cmei.isNotEmpty() && cmei != config.cmei) { + config = config.copy(cmei = cmei) + Log.i(TAG, "[YIDONG] updateCmei: cmei已更新为 $cmei") + } + } + // ── WebSocket ───────────────────────────────────────────────────────────── private val okHttpClient: OkHttpClient = buildTrustAllClient() private var webSocket: WebSocket? = null @@ -190,7 +199,7 @@ class StsAgent( ) credentialReady = config.deviceSecret.isNotEmpty() if (credentialReady) { - Log.i(TAG, "[YIDONG] fetchCredentials: 密钥获取成功 deviceId=${config.deviceId} deviceNo=${config.deviceNo}") + Log.i(TAG, "[YIDONG] fetchCredentials: 密钥获取成功 deviceId=${config.deviceId} deviceNo=${config.deviceNo} cmei=${config.cmei}") onCredentialsFetched?.invoke(config.deviceId, config.deviceSecret, config.deviceNo) reportHeartbeat() // 凭证获取成功后立即上报心跳 } else { @@ -216,7 +225,7 @@ class StsAgent( Log.w(TAG, "[YIDONG] reportHeartbeat: 凭证未就绪，跳过上报") return } - Log.i(TAG, "[YIDONG] reportHeartbeat: 开始上报 deviceId=${config.deviceId} deviceNo=${config.deviceNo}") + Log.i(TAG, "[YIDONG] reportHeartbeat: 开始上报 deviceId=${config.deviceId} deviceNo=${config.deviceNo} cmei=${config.cmei}") launch { try { val netType = try { @@ -243,7 +252,7 @@ class StsAgent( put("sdkVersion", "ai_http_1.0") put("firmwareVersion", "1.0.0") put("imei", "") - put("cmei", config.deviceNo) // 非蜂窝设备使用 SN 作为 cmei + put("cmei", config.cmei.ifEmpty { config.deviceNo }) // 优先使用 UserDevice 的 CMEI put("mac", "") }) }.toString().toRequestBody("application/json".toMediaType()) From fdb8a89502a06885c38dd26c24e8d6da3028fb9b Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Wed, 25 Mar 2026 17:49:45 +0800 Subject: [PATCH 13/14] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=A7=BB=E5=8A=A8?= =?UTF-8?q?=E5=AE=A2=E6=88=B7=E7=AB=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .metadata | 25 ++++--------------- .../yunqiinnovation/agent_service/StsAgent.kt | 2 +- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/.metadata b/.metadata index 369b5ecec..7686b46e4 100644 --- a/.metadata +++ b/.metadata @@ -4,7 +4,7 @@ # This file should be version controlled and should not be manually edited. version: - revision: "8defaa71a77c16e8547abdbfad2053ce3a6e2d5b" + revision: "30ade5e937e6afbd2d2c38ec53175b89b556447e" channel: "stable" project_type: app @@ -13,26 +13,11 @@ project_type: app migration: platforms: - platform: root - create_revision: 8defaa71a77c16e8547abdbfad2053ce3a6e2d5b - base_revision: 8defaa71a77c16e8547abdbfad2053ce3a6e2d5b + create_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e + base_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e - platform: android - create_revision: 8defaa71a77c16e8547abdbfad2053ce3a6e2d5b - base_revision: 8defaa71a77c16e8547abdbfad2053ce3a6e2d5b - - platform: ios - create_revision: 8defaa71a77c16e8547abdbfad2053ce3a6e2d5b - base_revision: 8defaa71a77c16e8547abdbfad2053ce3a6e2d5b - - platform: linux - create_revision: 8defaa71a77c16e8547abdbfad2053ce3a6e2d5b - base_revision: 8defaa71a77c16e8547abdbfad2053ce3a6e2d5b - - platform: macos - create_revision: 8defaa71a77c16e8547abdbfad2053ce3a6e2d5b - base_revision: 8defaa71a77c16e8547abdbfad2053ce3a6e2d5b - - platform: web - create_revision: 8defaa71a77c16e8547abdbfad2053ce3a6e2d5b - base_revision: 8defaa71a77c16e8547abdbfad2053ce3a6e2d5b - - platform: windows - create_revision: 8defaa71a77c16e8547abdbfad2053ce3a6e2d5b - base_revision: 8defaa71a77c16e8547abdbfad2053ce3a6e2d5b + create_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e + base_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e # User provided section diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt index 76938f508..b5b27c4bf 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt @@ -252,7 +252,7 @@ class StsAgent( put("sdkVersion", "ai_http_1.0") put("firmwareVersion", "1.0.0") put("imei", "") - put("cmei", config.cmei.ifEmpty { config.deviceNo }) // 优先使用 UserDevice 的 CMEI + put("cmei", config.cmei) // 优先使用 UserDevice 的 CMEI put("mac", "") }) }.toString().toRequestBody("application/json".toMediaType()) From 42e7755ffc2e7079bc28fa2430ff3ba0d51b5fe6 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Thu, 9 Apr 2026 21:01:50 +0800 Subject: [PATCH 14/14] =?UTF-8?q?=E4=BC=98=E5=8C=96=20=E7=A7=BB=E5=8A=A8?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../yunqiinnovation/agent_service/StsAgent.kt | 17 +++ .../agent_service/AgentServiceImpl.swift | 13 +- .../agent_service/AgentServicePlugin.swift | 11 ++ .../Sources/agent_service/StsAgent.swift | 118 ++++++++++++++++++ 4 files changed, 158 insertions(+), 1 deletion(-) diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt index b5b27c4bf..b76a16a58 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt @@ -162,6 +162,9 @@ class StsAgent( var credentialReady = false private set + // ── 心跳冷却 ────────────────────────────────────────────────────────────── + private val HEARTBEAT_COOLDOWN_MS = 3_600_000L // 1 小时 + // ═════════════════════════════════════════════════════════════════════════ // 凭证获取 // ═════════════════════════════════════════════════════════════════════════ @@ -225,6 +228,16 @@ class StsAgent( Log.w(TAG, "[YIDONG] reportHeartbeat: 凭证未就绪，跳过上报") return } + // 1 小时冷却：StsAgent 会被反复重建，init/fetchCredentials 都会调到这里， + // 必须在原生层拦，否则每次重建都会打一发心跳 + val prefs = context.getSharedPreferences("sts_agent", Context.MODE_PRIVATE) + val last = prefs.getLong("last_heartbeat_ts", 0L) + val now = System.currentTimeMillis() + if (last > 0L && now - last < HEARTBEAT_COOLDOWN_MS) { + Log.i(TAG, "[YIDONG] reportHeartbeat: 距上次 ${(now - last) / 1000}s，冷却中跳过") + return + } + prefs.edit().putLong("last_heartbeat_ts", now).apply() Log.i(TAG, "[YIDONG] reportHeartbeat: 开始上报 deviceId=${config.deviceId} deviceNo=${config.deviceNo} cmei=${config.cmei}") launch { try { @@ -271,10 +284,14 @@ class StsAgent( emit("heartbeat_reported", mapOf("time" to System.currentTimeMillis())) } else { Log.e(TAG, "[YIDONG] reportHeartbeat: 上报失败 code=$code msg=$msg") + // 上报失败回滚时间戳，避免卡 1 小时无法重试 + prefs.edit().putLong("last_heartbeat_ts", last).apply() } } } catch (e: Exception) { Log.e(TAG, "[YIDONG] reportHeartbeat: 请求异常 ${e.message}") + // 异常同样回滚时间戳 + prefs.edit().putLong("last_heartbeat_ts", last).apply() } } } diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index 034f6b72d..13dc6b0af 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -211,7 +211,8 @@ class AgentServiceImpl: NSObject { deviceNo: config["stsDeviceNo"] as? String ?? "", deviceId: config["stsDeviceId"] as? String ?? "", deviceSecret: config["stsDeviceSecret"] as? String ?? "", - agentId: agentId + agentId: agentId, + cmei: config["stsCmei"] as? String ?? "" ) stsAgent = StsAgent( config: stsConfig, @@ -726,6 +727,16 @@ class AgentServiceImpl: NSObject { return success } + /// STS 模式下主动触发中国移动平台的凭证获取（Dart AgentService.fetchCredentials 会调到这里） + func fetchCredentials() { + stsAgent?.fetchCredentials() + } + + /// STS 模式下主动上报一次心跳（1 小时冷却在原生层拦截） + func reportHeartbeat() { + stsAgent?.reportHeartbeat() + } + func interruptCurrentResponse() -> Bool { if agentMode == "sts" { stsAgent?.interruptSpeaking() diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift index 8963009e1..fe0e3a60e 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift @@ -203,6 +203,17 @@ public class AgentServicePlugin: NSObject, FlutterPlugin { } impl.setCanWakeUp(canWakeUp) result(true) + + case "reportHeartbeat": + // 中国移动精灵心跳上报，原生层持久化 1 小时冷却 + impl.reportHeartbeat() + result(nil) + + case "fetchCredentials": + // 中国移动精灵凭证拉取，成功后会自动触发心跳（受冷却保护） + impl.fetchCredentials() + result(nil) + default: result(FlutterMethodNotImplemented) } diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/StsAgent.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/StsAgent.swift index a5e50eb78..477eb78cb 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/StsAgent.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/StsAgent.swift @@ -1,4 +1,5 @@ import Foundation +import UIKit import AVFoundation import CryptoKit import os.log @@ -13,6 +14,7 @@ struct StsConfig { var deviceId: String var deviceSecret: String let agentId: String + var cmei: String = "" } // MARK: - StsAgent @@ -71,6 +73,11 @@ class StsAgent: NSObject { // MARK: 凭证请求 private var credentialsFetched = false + // MARK: 心跳冷却 + /// 1 小时冷却（毫秒），避免 StsAgent 反复重建 / fetchCredentials 反复触发时频繁上报 + private let HEARTBEAT_COOLDOWN_MS: Int64 = 3_600_000 + private let HEARTBEAT_PREFS_KEY = "sts_agent_last_heartbeat_ts" + init(config: StsConfig, onEvent: @escaping (String, [String: Any]) -> Void, onCredentialsFetched: ((String, String, String) -> Void)? = nil) { @@ -80,6 +87,22 @@ class StsAgent: NSObject { super.init() let sessionConfig = URLSessionConfiguration.default urlSession = URLSession(configuration: sessionConfig, delegate: nil, delegateQueue: OperationQueue()) + os_log("StsAgent 初始化: productId=%{public}@ deviceNo=%{public}@ deviceId=%{public}@ cmei=%{public}@ hasSecret=%{public}@", + log: logger, type: .info, + config.productId, config.deviceNo, config.deviceId, config.cmei, + config.deviceSecret.isEmpty ? "false" : "true") + // 与 Android 行为对齐：凭证已有时立即尝试上报一次心跳（受 1 小时冷却保护） + if !config.deviceSecret.isEmpty && !config.deviceId.isEmpty { + credentialsFetched = true + reportHeartbeat() + } + } + + /// 动态更新 CMEI（首次初始化时可能尚未获取到） + func updateCmei(_ cmei: String) { + guard !cmei.isEmpty, cmei != config.cmei else { return } + config.cmei = cmei + os_log("updateCmei: cmei 已更新为 %{public}@", log: logger, type: .info, cmei) } // MARK: - 公共接口 @@ -139,8 +162,18 @@ class StsAgent: NSObject { // MARK: - 凭证获取 + /// 公开入口：主动触发认证检查（Dart 层 AgentService.fetchCredentials 会调用） + func fetchCredentials() { + fetchCredentialsIfNeeded { [weak self] in + // 成功与否都不做额外处理；fetchCredentialsIfNeeded 内部会在命中凭证时触发心跳 + _ = self + } + } + private func fetchCredentialsIfNeeded(completion: @escaping () -> Void) { if credentialsFetched && !config.deviceId.isEmpty && !config.deviceSecret.isEmpty { + // 凭证已有，直接上报心跳（受冷却保护） + reportHeartbeat() completion() return } @@ -175,10 +208,95 @@ class StsAgent: NSObject { if let deviceNo = body["deviceNo"] as? String { self.config.deviceNo = deviceNo } self.credentialsFetched = true self.onCredentialsFetched?(self.config.deviceId, self.config.deviceSecret, self.config.deviceNo) + // 凭证获取成功后立即上报心跳（受冷却保护） + self.reportHeartbeat() completion() }.resume() } + // MARK: - 心跳上报 + + /// 向中国移动平台上报心跳；1 小时冷却通过 UserDefaults 持久化，所有调用路径共享同一把闸门 + func reportHeartbeat() { + guard credentialsFetched, + !config.deviceId.isEmpty, + !config.deviceSecret.isEmpty else { + os_log("reportHeartbeat: 凭证未就绪，跳过上报", log: logger, type: .info) + return + } + let defaults = UserDefaults.standard + // 用 Double 存毫秒时间戳，避开 Int64 的 UserDefaults 桥接问题 + let last = defaults.double(forKey: HEARTBEAT_PREFS_KEY) + let now = Date().timeIntervalSince1970 * 1000 + if last > 0, now - last < Double(HEARTBEAT_COOLDOWN_MS) { + os_log("reportHeartbeat: 距上次 %{public}ds，冷却中跳过", log: logger, type: .info, Int((now - last) / 1000)) + return + } + defaults.set(now, forKey: HEARTBEAT_PREFS_KEY) + os_log("reportHeartbeat: 开始上报 deviceId=%{public}@ deviceNo=%{public}@ cmei=%{public}@", + log: logger, type: .info, config.deviceId, config.deviceNo, config.cmei) + + let urlStr = "https://ivs.chinamobiledevice.com:11443/v2/customer/device/report" + guard let url = URL(string: urlStr) else { return } + var req = URLRequest(url: url) + req.httpMethod = "POST" + req.setValue("application/json", forHTTPHeaderField: "Content-Type") + req.timeoutInterval = 10 + + let systemVersion = UIDevice.current.systemVersion + let params: [String: Any] = [ + "innerIp": ["127.0.0.1"], + "netSpeed": "0Mbps", + "netType": "WiFi", + "platform": "iOS\(systemVersion)", + "sdkVersion": "ai_http_1.0", + "firmwareVersion": "1.0.0", + "imei": "", + "cmei": config.cmei, + "mac": "", + ] + let bodyDict: [String: Any] = [ + "deviceId": config.deviceId, + "deviceSecret": config.deviceSecret, + "productId": config.productId, + "productKey": config.productKey, + "params": params, + ] + guard let bodyData = try? JSONSerialization.data(withJSONObject: bodyDict, options: []) else { return } + req.httpBody = bodyData + + URLSession.shared.dataTask(with: req) { [weak self] data, _, error in + guard let self = self else { return } + if let error = error { + os_log("reportHeartbeat: 请求异常 %{public}@", log: self.logger, type: .error, error.localizedDescription) + // 异常回滚时间戳，避免卡 1 小时无法重试 + UserDefaults.standard.set(last, forKey: self.HEARTBEAT_PREFS_KEY) + return + } + guard let data = data, + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + os_log("reportHeartbeat: 响应解析失败", log: self.logger, type: .error) + UserDefaults.standard.set(last, forKey: self.HEARTBEAT_PREFS_KEY) + return + } + // code 可能是 String 或 NSNumber，统一转成字符串 + let code: String = { + if let s = json["code"] as? String { return s } + if let n = json["code"] as? NSNumber { return n.stringValue } + return "" + }() + let msg = (json["message"] as? String) ?? "" + if code == "200" { + os_log("reportHeartbeat: 上报成功 code=%{public}@ msg=%{public}@", log: self.logger, type: .info, code, msg) + self.emit("heartbeat_reported", data: ["time": Int64(now)]) + } else { + os_log("reportHeartbeat: 上报失败 code=%{public}@ msg=%{public}@", log: self.logger, type: .error, code, msg) + // 上报失败回滚时间戳 + UserDefaults.standard.set(last, forKey: self.HEARTBEAT_PREFS_KEY) + } + }.resume() + } + // MARK: - WebSocket 连接 private func connect() {