diff --git a/lib/data/services/ble_manager.dart b/lib/data/services/ble_manager.dart index a98989bdb..ddf05eedc 100644 --- a/lib/data/services/ble_manager.dart +++ b/lib/data/services/ble_manager.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +import 'package:flutter_dotenv/flutter_dotenv.dart'; import 'package:get_storage/get_storage.dart'; import 'package:get/get.dart'; import 'package:ble_service/ble_service.dart'; @@ -957,7 +958,27 @@ class BleManager extends GetxService { supportedLanguages = ['zh-CN', 'en-US']; } + // STS 模式参数（移动精灵） + final agentType = AppConfig.env('AGENT_TYPE') ?? 'yidong'; + final agentMode = (agentType == 'yidong') ? 'sts' : 'stt_llm_tts'; + final agentId = (agentType == 'yidong') + ? (AppConfig.env('MOBILE_ELF_AGENT_ID') ?? 'mobile_elf') + : 'default_agent'; + final stsProductId = AppConfig.env('MOBILE_ELF_PRODUCT_ID') ?? ''; + final stsProductKey = AppConfig.env('MOBILE_ELF_PRODUCT_KEY') ?? ''; + final stsDeviceNo = dotenv.env['MOBILE_ELF_DEVICE_NO'] ?? ''; + final stsDeviceId = _storage.read('sts_device_id') ?? ''; + final stsDeviceSecret = _storage.read('sts_device_secret') ?? ''; + final config = { + // STS 模式路由 + 'agentMode': agentMode, + 'agentId': agentId, + 'stsProductId': stsProductId, + 'stsProductKey': stsProductKey, + 'stsDeviceNo': stsDeviceNo, + 'stsDeviceId': stsDeviceId, + 'stsDeviceSecret': stsDeviceSecret, // 语音代理配置参数 'azureSpeechKey': azureSpeechKey, 'azureSpeechRegion': azureSpeechRegion, @@ -990,7 +1011,7 @@ class BleManager extends GetxService { Logger.warning('XUNFEI_ASR_* 未完整配置，ASR 将不会切到讯飞'); } - if (azureSpeechKey.isEmpty || openaiApiKey.isEmpty) { + if (agentMode != 'sts' && (azureSpeechKey.isEmpty || openaiApiKey.isEmpty)) { Logger.warning('缺少必要的API密钥，请检查.env文件配置'); // 检查.env中相关密钥是否存在 @@ -1002,6 +1023,19 @@ class BleManager extends GetxService { } } + // STS 模式：监听凭证获取事件并缓存到 GetStorage + if (agentMode == 'sts') { + AgentService.events.where((e) => e.type == AgentServiceEventType.stsCredentials).listen((e) { + final deviceId = e.data['deviceId']?.toString() ?? ''; + final deviceSecret = e.data['deviceSecret']?.toString() ?? ''; + final deviceNo = e.data['deviceNo']?.toString() ?? ''; + if (deviceId.isNotEmpty) _storage.write('sts_device_id', deviceId); + if (deviceSecret.isNotEmpty) _storage.write('sts_device_secret', deviceSecret); + if (deviceNo.isNotEmpty) _storage.write('sts_device_no', deviceNo); + Logger.info('STS 凭证已缓存: deviceId=$deviceId deviceNo=$deviceNo'); + }); + } + Logger.info('lxm--准备调用AgentService.startAgentService启动BLE服务'); final result = await AgentService.startAgentService(config); Logger.info('lxm--AgentService.startAgentService调用结果: $result'); diff --git a/lib/modules/home/controllers/home_controller.dart b/lib/modules/home/controllers/home_controller.dart index 0dede3002..7df97989b 100644 --- a/lib/modules/home/controllers/home_controller.dart +++ b/lib/modules/home/controllers/home_controller.dart @@ -256,8 +256,13 @@ class HomeController extends GetxController { } void exploreAIFeatures() { - Get.toNamed(Routes.agent); - AgentService.interruptCurrentResponse(); + final agentType = AppConfig.env('AGENT_TYPE') ?? 'yidong'; + if (agentType == 'yidong') { + Get.toNamed(Routes.mobileElf); + } else { + Get.toNamed(Routes.agent); + AgentService.interruptCurrentResponse(); + } } Future _checkConnectionStatus() async { diff --git a/lib/modules/home/views/home_view.dart b/lib/modules/home/views/home_view.dart index b9f7d8eeb..cc2c15243 100644 --- a/lib/modules/home/views/home_view.dart +++ b/lib/modules/home/views/home_view.dart @@ -146,8 +146,6 @@ class HomeView extends GetView { SizedBox(height: 16.h), _buildPremiumAILayout(), - SizedBox(height: 16.h), - _buildMobileElfCard(isDarkMode), ], ), ), @@ -902,27 +900,6 @@ class HomeView extends GetView { ); } - // 移动精灵卡片 - Widget _buildMobileElfCard(bool isDarkMode) { - return _buildFeatureCard( - title: '移动精灵', - description: '智能语音助手，移动AI全场景对话', - icon: Icons.cell_tower, - gradient: LinearGradient( - begin: Alignment.topLeft, - end: Alignment.bottomRight, - colors: [ - const Color(0xFF0091EA).withOpacity(0.3), - const Color(0xFF00BFA5).withOpacity(0.3), - ], - ), - onTap: () => Get.toNamed(Routes.mobileElf), - height: 70.h, - isDarkMode: isDarkMode, - iconColor: const Color(0xFF0091EA), - ); - } - // 特性卡片构建方法 Widget _buildFeatureCard({ required String title, diff --git a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart index cfb1e2802..a673cf443 100644 --- a/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart +++ b/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart @@ -1,21 +1,9 @@ import 'dart:async'; -import 'dart:convert'; -import 'dart:io'; -import 'dart:math' as math; -import 'dart:typed_data'; -import 'package:crypto/crypto.dart'; -import 'package:dio/dio.dart'; -import 'package:flutter/foundation.dart' show kIsWeb; +import 'package:agent_service/agent_service.dart'; +import 'package:chat_storage/chat_storage.dart'; import 'package:flutter/widgets.dart'; -import 'package:audio_session/audio_session.dart'; -import 'package:flutter_dotenv/flutter_dotenv.dart'; import 'package:get/get.dart'; -import 'package:get_storage/get_storage.dart'; -import 'package:just_audio/just_audio.dart'; -import 'package:record/record.dart'; -import 'package:uuid/uuid.dart'; -import 'package:web_socket_channel/io.dart'; -import 'package:web_socket_channel/web_socket_channel.dart'; +import '../../../data/models/appconfig.dart'; import '../../../core/utils/logger.dart'; enum MobileElfStatus { idle, connecting, recording, thinking, speaking, error } @@ -24,11 +12,13 @@ class MobileElfMessage { final bool isUser; final String text; final DateTime time; - final String? ttsUrl; - MobileElfMessage({required this.isUser, required this.text, required this.time, this.ttsUrl}); + MobileElfMessage( + {required this.isUser, required this.text, required this.time}); } class MobileElfController extends GetxController { + static const _tag = 'MobileElfCtrl'; + // ── 可观察状态 ───────────────────────────────────────────────────────────── final status = MobileElfStatus.idle.obs; final messages = [].obs; @@ -37,944 +27,264 @@ class MobileElfController extends GetxController { final inCall = false.obs; final callSeconds = 0.obs; final recordingSeconds = 0.obs; - final micRms = 0.0.obs; // 实时麦克风音量（RMS），供 UI 显示 - final micAlive = false.obs; // 麦克风流是否存活 - final pttRecording = false.obs; // 按住说话录音中 - final pttSeconds = 0.obs; // 按住说话计时 + final micRms = 0.0.obs; + final micAlive = false.obs; + final pttRecording = false.obs; + final pttSeconds = 0.obs; + final ttsEnabled = true.obs; // 是否播报 AI 回复 final ScrollController scrollController = ScrollController(); - // ── WebSocket ────────────────────────────────────────────────────────────── - WebSocketChannel? _channel; - StreamSubscription? _wsSubscription; - Completer? _readyCompleter; - bool _sessionEnded = true; - String _dialogRequestId = ''; - int _currentAiMsgIndex = -1; - - // ── 录音 & 本地 VAD ──────────────────────────────────────────────────────── - final AudioRecorder _recorder = AudioRecorder(); - StreamSubscription? _micSubscription; - - // 音频发送缓冲（凑帧用） - final List _sendBuffer = []; - int _sendBufferOffset = 0; - static const int _frameSize = 5120; // 160ms @ 16kHz 16bit mono - - // 预缓存：滚动保留最近 ~1.5 秒音频，用于在 WS 连接前缓存用户已说的话 - final List _preBuffer = []; - static const int _preBufferMaxChunks = 75; // ~1.5s (按每 chunk 20ms 算) - - // 本地 VAD（仅在空闲等待状态工作，TTS 播放中不做 VAD） - bool _speechDetected = false; - static const int _vadThreshold = 800; - static const int _vadOnsetChunks = 3; // 连续 3 chunk (~60ms) → 检测到说话 - int _vadOnsetCount = 0; - - // 状态标记 - bool _forwardingAudio = false; // 是否正在转发音频到 WS - bool _wsConnecting = false; // WS 正在连接中（防止重复触发） - int _micChunkCount = 0; // 麦克风 chunk 计数（调试用） - - // ── 播放 & TTS 队列 ────────────────────────────────────────────────────── - // 关闭 just_audio 的音频会话管理，防止它与 record 争抢音频硬件： - // - handleInterruptions: false → 不自动暂停/恢复 - // - handleAudioSessionActivation: false → 不自动调用 session.setActive() - // - androidApplyAudioAttributes: false → 不监听 audio_session 配置变更 - final AudioPlayer _player = AudioPlayer( - handleInterruptions: false, - handleAudioSessionActivation: false, - androidApplyAudioAttributes: false, - ); - final List _ttsQueue = []; - bool _ttsPlaying = false; - bool _ttsInterrupted = false; - Completer? _ttsPlayCompleter; // 用于等待当前段播完或被打断 - - // ── 配置 ─────────────────────────────────────────────────────────────────── - late String _deviceNo, _deviceId, _productId, _productKey, _deviceSecret; - final credentialReady = false.obs; + // 凭证状态（原生层负责，Dart 层始终无错） + final credentialReady = true.obs; final credentialError = ''.obs; - final _storage = GetStorage(); - static const _kDeviceId = 'mobile_elf_device_id'; - static const _kDeviceSecret = 'mobile_elf_device_secret'; - static const _kDeviceNo = 'mobile_elf_device_no'; - final _uuid = const Uuid(); - // 计时器 + // ── 内部 ─────────────────────────────────────────────────────────────────── + StreamSubscription? _eventSub; Timer? _callTimer; - Timer? _recordingTimer; - Timer? _micWatchdog; - int _lastWatchdogChunkCount = 0; - Timer? _idleTimer; Timer? _pttTimer; - static const int _idleTimeoutSeconds = 15; // TTS 播完后等待说话超时秒数 + bool _inCallMode = false; // ═══════════════════════════════════════════════════════════════════════════ // 生命周期 // ═══════════════════════════════════════════════════════════════════════════ + String get _agentId => AppConfig.env('MOBILE_ELF_AGENT_ID') ?? 'mobile_elf'; + @override void onInit() { super.onInit(); - _loadConfig(); - _fetchA80CredentialsIfNeeded(); - } - - @override - void onClose() { - disconnect(); - _recorder.dispose(); - _player.dispose(); - scrollController.dispose(); - super.onClose(); - } - - void _loadConfig() { - _productId = dotenv.env['MOBILE_ELF_PRODUCT_ID'] ?? '1899007766144614402'; - _productKey = dotenv.env['MOBILE_ELF_PRODUCT_KEY'] ?? 'd0aI2yidbUUXS'; - _deviceNo = _storage.read(_kDeviceNo) ?? dotenv.env['MOBILE_ELF_DEVICE_NO'] ?? ''; - _deviceId = _storage.read(_kDeviceId) ?? ''; - _deviceSecret = _storage.read(_kDeviceSecret) ?? ''; - credentialReady.value = _deviceSecret.isNotEmpty; - } - - Future _fetchA80CredentialsIfNeeded() async { - try { - final deviceNo = dotenv.env['MOBILE_ELF_DEVICE_NO'] ?? ''; - if (deviceNo.isEmpty) return; - credentialError.value = ''; - final dio = Dio(BaseOptions( - baseUrl: 'https://ivs.chinamobiledevice.com:11443', - connectTimeout: const Duration(seconds: 10), - receiveTimeout: const Duration(seconds: 10), - )); - final resp = await dio.post('/v2/customer/device/secret/info', data: { - 'deviceNoType': 'SN', 'deviceNo': deviceNo, - 'productId': _productId, 'productKey': _productKey, - }); - final body = resp.data as Map; - if (body['code'].toString() == '200' && body['success'] == true) { - final data = body['data'] as Map; - _deviceNo = data['deviceNo'] as String? ?? deviceNo; - _deviceId = data['deviceId'] as String? ?? ''; - _deviceSecret = data['deviceSecret'] as String? ?? ''; - await _storage.write(_kDeviceNo, _deviceNo); - await _storage.write(_kDeviceId, _deviceId); - await _storage.write(_kDeviceSecret, _deviceSecret); - credentialReady.value = _deviceSecret.isNotEmpty; - } else { - credentialError.value = body['message']?.toString() ?? '凭证请求失败'; - } - } catch (e) { - credentialError.value = '凭证请求异常: $e'; - } - } - - Future retryCredentials() async { - credentialError.value = ''; - await _fetchA80CredentialsIfNeeded(); - } - - // ═══════════════════════════════════════════════════════════════════════════ - // 开始/结束通话 - // ═══════════════════════════════════════════════════════════════════════════ - - /// 配置音频会话：playAndRecord 模式，允许录音和播放同时进行 - Future _configureAudioSession() async { - try { - final session = await AudioSession.instance; - await session.configure(AudioSessionConfiguration( - avAudioSessionCategory: AVAudioSessionCategory.playAndRecord, - avAudioSessionCategoryOptions: AVAudioSessionCategoryOptions.defaultToSpeaker | - AVAudioSessionCategoryOptions.allowBluetooth, - avAudioSessionMode: AVAudioSessionMode.voiceChat, - androidAudioAttributes: const AndroidAudioAttributes( - contentType: AndroidAudioContentType.speech, - usage: AndroidAudioUsage.voiceCommunication, - ), - androidAudioFocusGainType: AndroidAudioFocusGainType.gain, - )); - await session.setActive(true); - Logger.info('yiding----------- [音频会话] 已配置 playAndRecord 模式'); - } catch (e) { - Logger.error('yiding----------- [音频会话] 配置失败: $e'); - } - } - - /// 点击通话按钮 → 开麦，进入等待说话状态（不连接服务器） - Future startCall() async { - if (inCall.value) return; - // PTT 模式下 AI 正在播报 → 先打断 TTS，再进入通话 - if (_ttsPlaying || status.value == MobileElfStatus.speaking) { - _interruptTts(); - _ttsInterrupted = false; - // 等一帧让 player.stop() 完成 - await Future.delayed(const Duration(milliseconds: 50)); - } - // 如果 PTT 正在录音中 → 先取消 - if (pttRecording.value) { - await stopPtt(cancel: true); - } - final hasPermission = await _recorder.hasPermission(); - if (!hasPermission) { _setError('无麦克风权限'); return; } - - await _configureAudioSession(); - // 为播放器单独设置 voiceCommunication 音频属性，与录音器使用同一音频路由 - // 这样 ExoPlayer 和 AudioRecord 可以像 VoIP 应用一样同时工作 - if (!kIsWeb) { - await _player.setAndroidAudioAttributes(const AndroidAudioAttributes( - contentType: AndroidAudioContentType.speech, - usage: AndroidAudioUsage.voiceCommunication, - )); - } - await _openMic(); - _startMicWatchdog(); - inCall.value = true; - status.value = MobileElfStatus.idle; // 等待用户说话 - callSeconds.value = 0; - _callTimer?.cancel(); - _callTimer = Timer.periodic(const Duration(seconds: 1), (_) => callSeconds.value++); - _startIdleTimer(); // 开始等待说话超时 - Logger.info('yiding----------- [通话] 已开始，等待用户说话'); - } - - /// 挂断 - void disconnect() { - Logger.info('yiding----------- [通话] 挂断'); - _stopRecordingTimer(); - _stopMicWatchdog(); - _cancelIdleTimer(); - _callTimer?.cancel(); _callTimer = null; - _forwardingAudio = false; - _wsConnecting = false; - _speechDetected = false; - _micSubscription?.cancel(); _micSubscription = null; - _recorder.stop(); - // 清空 TTS 队列 & 停播 - _ttsInterrupted = true; - _ttsQueue.clear(); - _ttsPlaying = false; - if (!(_ttsPlayCompleter?.isCompleted ?? true)) _ttsPlayCompleter?.complete(); - _player.stop(); - _closeWs(); - _preBuffer.clear(); - _sendBuffer.clear(); _sendBufferOffset = 0; - micRms.value = 0; micAlive.value = false; - inCall.value = false; - status.value = MobileElfStatus.idle; - } - - // ═══════════════════════════════════════════════════════════════════════════ - // 麦克风 & 本地 VAD - // ═══════════════════════════════════════════════════════════════════════════ - - Future _openMic() async { - _micSubscription?.cancel(); - _micSubscription = null; - try { await _recorder.stop(); } catch (_) {} - final stream = await _recorder.startStream( - const RecordConfig( - encoder: AudioEncoder.pcm16bits, - sampleRate: 16000, - numChannels: 1, - echoCancel: true, - noiseSuppress: true, - androidConfig: AndroidRecordConfig( - audioSource: AndroidAudioSource.voiceCommunication, - ), - ), - ); - _speechDetected = false; - _vadOnsetCount = 0; - _micChunkCount = 0; - micAlive.value = false; - _micSubscription = stream.listen(_onMicAudio); - Logger.info('yiding----------- [麦克风] 已开启 (voiceCommunication + echoCancel)'); - } - - /// 启动麦克风看门狗：每 1.5 秒检测一次，如果无新 chunk 则标记 micAlive=false - void _startMicWatchdog() { - _micWatchdog?.cancel(); - _lastWatchdogChunkCount = _micChunkCount; - _micWatchdog = Timer.periodic(const Duration(milliseconds: 1500), (_) { - if (!inCall.value && !pttRecording.value) { _micWatchdog?.cancel(); return; } - if (_micChunkCount == _lastWatchdogChunkCount) { - micAlive.value = false; - if (status.value == MobileElfStatus.speaking || _ttsPlaying) { - // TTS 播放中麦克风已关闭，不需要重启 - } else { - Logger.info('yiding----------- [看门狗] 麦克风无数据! 尝试重启'); - _openMic(); - } - } - _lastWatchdogChunkCount = _micChunkCount; + _subscribeEvents(); + _loadChatHistory(); + // 同步播报开关到原生层 + ever(ttsEnabled, (bool enabled) { + AgentService.setTtsMuted(!enabled); }); } - void _stopMicWatchdog() { - _micWatchdog?.cancel(); - _micWatchdog = null; - } - - void _onMicAudio(Uint8List chunk) { - if (!inCall.value && !pttRecording.value) return; - _micChunkCount++; - - // 计算 RMS（PCM 16-bit little-endian） - final rms = _calcRms(chunk); - - // 更新 UI 可观察值 - micAlive.value = true; - micRms.value = rms; - - // 每 100 个 chunk (~2s) 输出一次调试日志，确认麦克风存活 - if (_micChunkCount % 100 == 0) { - Logger.info('yiding----------- [麦克风] chunk#$_micChunkCount rms=${rms.toStringAsFixed(0)} ' - 'status=${status.value} speechDetected=$_speechDetected fwd=$_forwardingAudio wsConn=$_wsConnecting'); - } - - // ── 当前正在转发音频到 WS ── - if (_forwardingAudio && _channel != null) { - _feedSendBuffer(chunk); - return; - } - - // ── WS 正在连接中，先存入预缓存 ── - if (_wsConnecting) { - _addToPreBuffer(chunk); - return; - } - - // ── TTS 播放中 / thinking 中不做 VAD ── - if (status.value == MobileElfStatus.speaking || _ttsPlaying || - status.value == MobileElfStatus.thinking) { - return; - } - - // ── 空闲/等待状态：连续 chunk VAD ── - _addToPreBuffer(chunk); - - if (rms >= _vadThreshold) { - _vadOnsetCount++; - if (_vadOnsetCount >= _vadOnsetChunks && !_speechDetected) { - _speechDetected = true; - _cancelIdleTimer(); - Logger.info('yiding----------- [VAD] 检测到说话 rms=${rms.toStringAsFixed(0)} ' - 'onset=$_vadOnsetCount/$_vadOnsetChunks，启动连接'); - _onSpeechDetected(); - } - } else { - _vadOnsetCount = 0; - } - } - - double _calcRms(Uint8List pcm) { - if (pcm.length < 2) return 0; - final samples = pcm.length ~/ 2; - double sum = 0; - for (int i = 0; i < pcm.length - 1; i += 2) { - int sample = pcm[i] | (pcm[i + 1] << 8); - if (sample >= 32768) sample -= 65536; // signed - sum += sample * sample; - } - return math.sqrt(sum / samples); - } - - void _addToPreBuffer(Uint8List chunk) { - _preBuffer.add(Uint8List.fromList(chunk)); - while (_preBuffer.length > _preBufferMaxChunks) { - _preBuffer.removeAt(0); - } - } - - // ═══════════════════════════════════════════════════════════════════════════ - // 检测到说话 → 连接服务器 → 发送预缓存 → 实时转发 - // ═══════════════════════════════════════════════════════════════════════════ - - Future _onSpeechDetected() async { - if (_wsConnecting) return; - - _ttsInterrupted = false; - _closeWs(); - _sessionEnded = false; - _wsConnecting = true; - - status.value = MobileElfStatus.connecting; - _dialogRequestId = _uuid.v4(); - _currentAiMsgIndex = -1; - realtimeText.value = ''; - _sendBuffer.clear(); - _sendBufferOffset = 0; - errorMsg.value = ''; - - Logger.info('yiding----------- [新轮] 说话触发连接 dialogId=$_dialogRequestId preBuffer=${_preBuffer.length}chunks'); - + Future _loadChatHistory() async { try { - await _connect(); - Logger.info('yiding----------- [新轮] WS ready，开始发送预缓存'); - } catch (e) { - Logger.error('yiding----------- [新轮] 连接失败: $e'); - _wsConnecting = false; - _speechDetected = false; - _setError('连接失败: $e'); - return; - } - - // 发送预缓存音频 - _forwardingAudio = true; - _wsConnecting = false; - status.value = MobileElfStatus.recording; - recordingSeconds.value = 0; - _stopRecordingTimer(); - _recordingTimer = Timer.periodic(const Duration(seconds: 1), (_) => recordingSeconds.value++); - - final cached = List.from(_preBuffer); - _preBuffer.clear(); - int cachedBytes = 0; - for (final c in cached) { - _feedSendBuffer(c); - cachedBytes += c.length; - } - Logger.info('yiding----------- [新轮] 预缓存已发送 ${cached.length}chunks ${cachedBytes}bytes，实时转发中'); - } - - /// 将 chunk 加入发送缓冲，凑满 _frameSize 就发 - void _feedSendBuffer(Uint8List chunk) { - _sendBuffer.addAll(chunk); - while (_sendBuffer.length - _sendBufferOffset >= _frameSize) { - final frame = Uint8List.fromList( - _sendBuffer.sublist(_sendBufferOffset, _sendBufferOffset + _frameSize)); - _sendBufferOffset += _frameSize; - _sendAudioChunk(frame); - } - // 紧缩 - if (_sendBufferOffset >= _frameSize * 8) { - _sendBuffer.removeRange(0, _sendBufferOffset); - _sendBufferOffset = 0; - } - } - - // ═══════════════════════════════════════════════════════════════════════════ - // WebSocket 连接 - // ═══════════════════════════════════════════════════════════════════════════ - - Future _connect() async { - _readyCompleter = Completer(); - try { - final ts = DateTime.now().millisecondsSinceEpoch.toString(); - final sn = _uuid.v4(); - final sign = _calcSign(_deviceSecret, ts); - final url = 'wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr' - '?deviceNo=$_deviceNo&deviceId=$_deviceId&productId=$_productId' - '&productKey=$_productKey&sign=$sign&sn=$sn&ts=$ts'; - - if (kIsWeb) { - _channel = WebSocketChannel.connect(Uri.parse(url)); - } else { - final httpClient = HttpClient()..badCertificateCallback = (_, __, ___) => true; - final ws = await WebSocket.connect(url, customClient: httpClient, headers: { - 'Origin': 'https://ivs.chinamobiledevice.com:11443', - 'Host': 'ivs.chinamobiledevice.com:11443', - }); - _channel = IOWebSocketChannel(ws); - } - - _wsSubscription = _channel!.stream.listen( - _handleMessage, - onError: (e) { - Logger.error('yiding----------- [WS] 错误: $e'); - if (!(_readyCompleter?.isCompleted ?? true)) _readyCompleter?.completeError(e); - _handleWsDone(); - }, - onDone: () { - Logger.info('yiding----------- [WS] onDone sessionEnded=$_sessionEnded status=${status.value}'); - if (!(_readyCompleter?.isCompleted ?? true)) _readyCompleter?.completeError('WS closed'); - _handleWsDone(); - }, + final result = await ChatStorage.getMessages( + agentId: _agentId, + page: 1, + pageSize: 50, ); - - _sendStartFrame(); - await _readyCompleter!.future.timeout( - const Duration(seconds: 15), - onTimeout: () => throw TimeoutException('等待 ready 超时'), - ); - } catch (e) { - _closeWs(); - rethrow; - } - } - - void _handleWsDone() { - _channel = null; - _wsSubscription = null; - _forwardingAudio = false; - _wsConnecting = false; - - if (_sessionEnded && inCall.value) { - Logger.info('yiding----------- [WS] 正常关闭 (通话模式)'); - if (status.value == MobileElfStatus.speaking) { - // TTS 播放中，等 playTts 完成后自动回到 idle 等待说话 - } else { - status.value = MobileElfStatus.idle; - _resetVad(); - Logger.info('yiding----------- [WS] 回到等待说话'); - } - } else if (_sessionEnded && !inCall.value) { - // PTT/文本模式：WS 正常关闭 - Logger.info('yiding----------- [WS] 正常关闭 (PTT/文本模式)'); - if (status.value != MobileElfStatus.speaking) { - status.value = MobileElfStatus.idle; + if (result.messages.isNotEmpty) { + final history = result.messages.map((m) => MobileElfMessage( + isUser: m.sender == 'user', + text: m.message, + time: DateTime.tryParse(m.timestamp) ?? DateTime.now(), + )).toList(); + // 按时间正序排列 + history.sort((a, b) => a.time.compareTo(b.time)); + messages.addAll(history); + _scrollToBottom(); } - } else if (inCall.value) { - Logger.info('yiding----------- [WS] 非预期关闭，回到等待说话'); - status.value = MobileElfStatus.idle; - _resetVad(); - } else { - Logger.info('yiding----------- [WS] 非预期关闭'); - status.value = MobileElfStatus.idle; + } catch (e) { + Logger.error('$_tag 加载聊天记录失败: $e'); } } - void _resetVad() { - _speechDetected = false; - _vadOnsetCount = 0; - _preBuffer.clear(); - _openMic(); - } - - // ═══════════════════════════════════════════════════════════════════════════ - // 空闲超时：TTS 播完后等待说话，超时自动挂断 - // ═══════════════════════════════════════════════════════════════════════════ - - void _startIdleTimer() { - _cancelIdleTimer(); - _idleTimer = Timer(Duration(seconds: _idleTimeoutSeconds), () { - if (inCall.value && status.value == MobileElfStatus.idle && !_speechDetected) { - Logger.info('yiding----------- [超时] ${_idleTimeoutSeconds}秒无说话，自动挂断'); - disconnect(); - } - }); - Logger.info('yiding----------- [超时] 开始${_idleTimeoutSeconds}秒等待说话计时'); - } - - void _cancelIdleTimer() { - _idleTimer?.cancel(); - _idleTimer = null; - } - - void _closeWs() { - final sub = _wsSubscription; _wsSubscription = null; - sub?.cancel(); - final ch = _channel; _channel = null; - try { ch?.sink.close(); } catch (_) {} - } - - // ═══════════════════════════════════════════════════════════════════════════ - // 发送帧 - // ═══════════════════════════════════════════════════════════════════════════ - - void _sendStartFrame() { - if (_channel == null) return; - _channel!.sink.add(jsonEncode({ - 'type': 'start', - 'data': { - 'cuid': _deviceId, - 'format': 'pcm', - 'sample': 16000, - 'support_dcs': 2, - 'chunk_size': 10240, - 'support_tts': true, - 'support_text2dcs': true, - 'user_agent': 'voitrans/1.0', - 'dialog_request_id': _dialogRequestId, - 'access_rc': true, - 'support_part_tts': true, - 'need_stoplisten': true, - 'need_dialogue_finish': true, - 'result_trans2directive': false, - 'rc_version': '5', - 'messages': _buildContextMessages(), - 'client_context': [], - }, - })); - } - - void _sendAudioChunk(Uint8List data) { - try { _channel?.sink.add(data); } catch (_) {} - } - - void _sendFinish() { - try { _channel?.sink.add(jsonEncode({'type': 'finish'})); } catch (_) {} + @override + void onClose() { + _eventSub?.cancel(); + _callTimer?.cancel(); + _pttTimer?.cancel(); + AgentService.stopConversation(); + scrollController.dispose(); + super.onClose(); } // ═══════════════════════════════════════════════════════════════════════════ - // 接收消息处理 + // 事件订阅（来自原生 StsAgent） // ═══════════════════════════════════════════════════════════════════════════ - void _handleMessage(dynamic raw) { - try { - if (raw is! String) return; - final msg = jsonDecode(raw) as Map; - final type = msg['type'] as String? ?? ''; - - switch (type) { - case 'ready': - if (!(_readyCompleter?.isCompleted ?? true)) _readyCompleter?.complete(); - break; - - case 'stoplisten': - Logger.info('yiding----------- [服务端] stoplisten'); - _stopRecordingTimer(); - _forwardingAudio = false; - _speechDetected = false; // 允许 VAD 在 thinking/speaking 期间检测新语音 - _vadOnsetCount = 0; - // 发送剩余 - final remain = _sendBuffer.length - _sendBufferOffset; - if (remain > 0) _sendAudioChunk(Uint8List.fromList(_sendBuffer.sublist(_sendBufferOffset))); - _sendBuffer.clear(); _sendBufferOffset = 0; - _sendFinish(); - status.value = MobileElfStatus.thinking; - break; - - case 'mid_result': - realtimeText.value = msg['result'] as String? ?? ''; - break; - - case 'fin_result': - final result = msg['result'] as String? ?? ''; - if (result.isNotEmpty) { - realtimeText.value = ''; - _addUserMessage(result); - } - if (msg['end'] == 1) _markSessionEnded(); - break; - - case 'inside_rc': - _handleInsideRc(msg); - break; - - case 'dcs_decide': - if (msg['end'] == 1) { - Logger.info('yiding----------- [服务端] dcs_decide end:1'); - _markSessionEnded(); - } - break; - } - } catch (e) { - Logger.error('[MobileElf] 处理消息失败: $e'); - } + void _subscribeEvents() { + _eventSub = AgentService.events.listen(_onAgentEvent); } - void _handleInsideRc(Map msg) { - final data = msg['data'] as Map?; - if (data == null) return; - if ((data['code'] as int? ?? -1) != 0) return; - - final assistantAnswer = data['assistant_answer'] as String? ?? ''; - String cleanAnswer = assistantAnswer; - try { - cleanAnswer = (jsonDecode(assistantAnswer) as Map)['content'] as String? ?? assistantAnswer; - } catch (_) {} - - String? ttsUrl; - for (final item in (data['data'] as List? ?? [])) { - final d = item as Map; - final h = d['header'] as Map?; - if (h?['namespace'] == 'ai.fxzsos.device_interface.voice_output' && h?['name'] == 'Speak') { - ttsUrl = (d['payload'] as Map?)?['url'] as String?; + void _onAgentEvent(AgentServiceEvent event) { + Logger.info('$_tag event: ${event.type} data: ${event.data}'); + switch (event.type) { + case AgentServiceEventType.recognitionStarted: + status.value = MobileElfStatus.recording; + realtimeText.value = ''; + micAlive.value = true; break; - } - } - if (cleanAnswer.isNotEmpty) { - if (_currentAiMsgIndex >= 0 && _currentAiMsgIndex < messages.length) { - messages[_currentAiMsgIndex] = MobileElfMessage( - isUser: false, text: cleanAnswer, - time: messages[_currentAiMsgIndex].time, - ttsUrl: ttsUrl ?? messages[_currentAiMsgIndex].ttsUrl, - ); - } else { - _currentAiMsgIndex = messages.length; - _addAiMessage(cleanAnswer, ttsUrl: ttsUrl); - } - } - - if (ttsUrl != null && ttsUrl.isNotEmpty) _enqueueTts(ttsUrl); - if (msg['end'] == 1) _markSessionEnded(); - } - - void _markSessionEnded() { - if (_sessionEnded) return; - _sessionEnded = true; - Logger.info('yiding----------- [会话] 本轮结束 status=${status.value}'); - _stopRecordingTimer(); - _forwardingAudio = false; - _speechDetected = false; // 允许 VAD 在 TTS 播放期间检测新语音 - _vadOnsetCount = 0; - _sendBuffer.clear(); _sendBufferOffset = 0; - realtimeText.value = ''; - } + case AgentServiceEventType.recognizing: + realtimeText.value = event.data['text']?.toString() ?? ''; + status.value = MobileElfStatus.recording; + break; - // ═══════════════════════════════════════════════════════════════════════════ - // TTS 播放队列 - // ═══════════════════════════════════════════════════════════════════════════ + case AgentServiceEventType.recognitionResult: + final text = event.data['text']?.toString() ?? ''; + realtimeText.value = ''; + micAlive.value = false; + if (text.isNotEmpty) _addMessage(isUser: true, text: text); + status.value = MobileElfStatus.thinking; + break; - /// 将 TTS URL 加入队列，如果没有正在播放则关闭麦克风并开始播放 - void _enqueueTts(String url) { - _ttsQueue.add(url); - Logger.info('yiding----------- [TTS] 入队 queue=${_ttsQueue.length}'); - if (!_ttsPlaying) { - // 关闭麦克风，避免音频冲突，TTS 播完后会重开 - _micSubscription?.cancel(); _micSubscription = null; - try { _recorder.stop(); } catch (_) {} - micAlive.value = false; micRms.value = 0; - Logger.info('yiding----------- [TTS] 关闭麦克风，开始播放'); - _playNextTts(); - } - } + case AgentServiceEventType.recognitionStopped: + case AgentServiceEventType.recognitionCanceled: + realtimeText.value = ''; + micAlive.value = false; + break; - /// 顺序播放队列中的 TTS - Future _playNextTts() async { - if (_ttsQueue.isEmpty || _ttsInterrupted) { - _ttsPlaying = false; - // 队列播完 → 重开麦克风，回到等待说话，启动超时计时 - if (!_ttsInterrupted && status.value == MobileElfStatus.speaking) { - status.value = MobileElfStatus.idle; - Logger.info('yiding----------- [TTS] 队列全部播完，重开麦克风，等待新一轮说话'); - if (inCall.value) { - _resetVad(); - _startIdleTimer(); + case AgentServiceEventType.assistantResponse: + final text = event.data['response']?.toString() ?? ''; + if (text.isNotEmpty) _updateOrAddAssistantMessage(text); + // 静音模式下收到回复后直接恢复状态（不等 TTS 事件） + if (!ttsEnabled.value) { + status.value = + _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; + micAlive.value = _inCallMode; } - } - return; - } - - _ttsPlaying = true; - status.value = MobileElfStatus.speaking; - final url = _ttsQueue.removeAt(0); + break; - // 用 Completer 等待播放完成或被打断 - _ttsPlayCompleter = Completer(); - StreamSubscription? stateSub; + case AgentServiceEventType.ttsStarted: + status.value = MobileElfStatus.speaking; + break; - try { - await _player.stop(); - // 不再重新配置音频会话！startCall() 已配置好 playAndRecord 模式， - // 重复配置会导致 audio_session 重新请求音频焦点，杀死录音流 - await _player.setUrl(url); - - // 监听 ProcessingState: completed=正常播完, idle=被打断或意外停止 - stateSub = _player.processingStateStream.listen((s) { - if ((s == ProcessingState.completed || s == ProcessingState.idle) && - !(_ttsPlayCompleter?.isCompleted ?? true)) { - Logger.info('yiding----------- [TTS] 当前段结束 state=$s'); - _ttsPlayCompleter?.complete(); + case AgentServiceEventType.ttsCompleted: + if (_inCallMode) { + // 通话模式：TTS 播完后自动回到录音等待状态 + status.value = MobileElfStatus.recording; + micAlive.value = true; + } else { + status.value = MobileElfStatus.idle; + micAlive.value = false; } - }); + break; - await _player.play(); - Logger.info('yiding----------- [TTS] 开始播放 剩余队列=${_ttsQueue.length} micAlive=${micAlive.value} micChunk=$_micChunkCount'); + case AgentServiceEventType.ttsCanceled: + case AgentServiceEventType.ttsStopped: + status.value = _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; + micAlive.value = _inCallMode; + break; - // 等待：正常播完 complete() 或被打断 _interruptTts() 也会 complete() - await _ttsPlayCompleter!.future.timeout(const Duration(seconds: 60)); - } catch (e) { - Logger.error('yiding----------- [TTS] 播放异常: $e'); - } finally { - stateSub?.cancel(); - } + case AgentServiceEventType.error: + final msg = event.data['message']?.toString() ?? '未知错误'; + status.value = MobileElfStatus.error; + errorMsg.value = msg; + Logger.error('$_tag error: $msg'); + break; - // 当前段结束，继续下一段（如果没被打断） - if (!_ttsInterrupted) { - _playNextTts(); - } - } + case AgentServiceEventType.micRms: + final rms = (event.data['rms'] as num?)?.toDouble() ?? 0.0; + micRms.value = rms; + break; - /// 立即打断 TTS：停播 + 清队列 + 解除等待 - void _interruptTts() { - Logger.info('yiding----------- [TTS] 打断! 清空队列=${_ttsQueue.length}'); - _ttsInterrupted = true; - _ttsQueue.clear(); - _ttsPlaying = false; - // 先解除 _playNextTts 的等待 - if (!(_ttsPlayCompleter?.isCompleted ?? true)) { - _ttsPlayCompleter?.complete(); + default: + break; } - _player.stop(); - } - - /// 手动打断 AI 播报（界面按钮调用） - Future interruptSpeaking() async { - _interruptTts(); - _ttsInterrupted = false; // 手动打断后重置，不影响后续 - status.value = MobileElfStatus.idle; - if (inCall.value) _resetVad(); + _scrollToBottom(); } // ═══════════════════════════════════════════════════════════════════════════ - // 手动停止录音 + // 公共接口（View 调用） // ═══════════════════════════════════════════════════════════════════════════ - Future stopRecording() async { - if (!_forwardingAudio) return; - _stopRecordingTimer(); - _forwardingAudio = false; - final remain = _sendBuffer.length - _sendBufferOffset; - if (remain > 0) _sendAudioChunk(Uint8List.fromList(_sendBuffer.sublist(_sendBufferOffset))); - _sendBuffer.clear(); _sendBufferOffset = 0; - _sendFinish(); - status.value = MobileElfStatus.thinking; + Future startCall() async { + if (inCall.value) return; + _inCallMode = true; + inCall.value = true; + callSeconds.value = 0; + status.value = MobileElfStatus.connecting; + _callTimer?.cancel(); + _callTimer = Timer.periodic( + const Duration(seconds: 1), (_) => callSeconds.value++); + await AgentService.startConversation(mode: 'phone_call'); } - void _stopRecordingTimer() { - _recordingTimer?.cancel(); _recordingTimer = null; + Future disconnect() async { + _inCallMode = false; + inCall.value = false; + _callTimer?.cancel(); + _callTimer = null; + callSeconds.value = 0; + pttRecording.value = false; + _pttTimer?.cancel(); + _pttTimer = null; + status.value = MobileElfStatus.idle; + micAlive.value = false; + realtimeText.value = ''; + await AgentService.stopConversation(); } - // ═══════════════════════════════════════════════════════════════════════════ - // 按住说话 (PTT) - // ═══════════════════════════════════════════════════════════════════════════ - - /// 按住说话 → 开始录音并连接 WS - Future startPtt() async { + void startPtt() { if (pttRecording.value) return; - // 如果正在播放 AI 回复 → 立即打断 - if (_ttsPlaying || status.value == MobileElfStatus.speaking) { - _interruptTts(); - _ttsInterrupted = false; - } - final hasPermission = await _recorder.hasPermission(); - if (!hasPermission) { _setError('无麦克风权限'); return; } - - // 先设置 connecting 状态，再设置 pttRecording，避免中间按钮橙色闪烁 - status.value = MobileElfStatus.connecting; pttRecording.value = true; pttSeconds.value = 0; - _pttTimer = Timer.periodic(const Duration(seconds: 1), (_) => pttSeconds.value++); - - await _configureAudioSession(); - if (!kIsWeb) { - await _player.setAndroidAudioAttributes(const AndroidAudioAttributes( - contentType: AndroidAudioContentType.speech, - usage: AndroidAudioUsage.voiceCommunication, - )); - } - await _openMic(); - _startMicWatchdog(); - - _ttsInterrupted = false; - _closeWs(); - _sessionEnded = false; - _wsConnecting = true; - _dialogRequestId = _uuid.v4(); - _currentAiMsgIndex = -1; - realtimeText.value = ''; - _sendBuffer.clear(); _sendBufferOffset = 0; - errorMsg.value = ''; - - try { - await _connect(); - _forwardingAudio = true; - _wsConnecting = false; - status.value = MobileElfStatus.recording; - Logger.info('yiding----------- [PTT] 录音开始'); - } catch (e) { - _wsConnecting = false; - pttRecording.value = false; - _pttTimer?.cancel(); _pttTimer = null; - _stopMicWatchdog(); - _setError('连接失败: $e'); - } + status.value = MobileElfStatus.connecting; + _pttTimer?.cancel(); + _pttTimer = Timer.periodic( + const Duration(seconds: 1), (_) => pttSeconds.value++); + AgentService.startConversation(mode: 'push_to_talk'); } - /// 松开 → 停止录音，发送 finish - Future stopPtt({bool cancel = false}) async { + void stopPtt() { + if (!pttRecording.value) return; pttRecording.value = false; - _pttTimer?.cancel(); _pttTimer = null; - _stopMicWatchdog(); - - if (cancel) { - _forwardingAudio = false; - _closeWs(); - _micSubscription?.cancel(); _micSubscription = null; - try { _recorder.stop(); } catch (_) {} - micAlive.value = false; micRms.value = 0; - realtimeText.value = ''; - status.value = MobileElfStatus.idle; - Logger.info('yiding----------- [PTT] 已取消'); - return; - } + _pttTimer?.cancel(); + _pttTimer = null; + AgentService.stopConversation(); + } - // 正常结束：发送剩余音频 + finish - if (_forwardingAudio) { - _forwardingAudio = false; - final remain = _sendBuffer.length - _sendBufferOffset; - if (remain > 0) _sendAudioChunk(Uint8List.fromList(_sendBuffer.sublist(_sendBufferOffset))); - _sendBuffer.clear(); _sendBufferOffset = 0; - _sendFinish(); - status.value = MobileElfStatus.thinking; - } - _micSubscription?.cancel(); _micSubscription = null; - try { _recorder.stop(); } catch (_) {} - micAlive.value = false; micRms.value = 0; - Logger.info('yiding----------- [PTT] 录音结束，等待AI回复'); + void interruptSpeaking() { + AgentService.interruptCurrentResponse(); + status.value = + _inCallMode ? MobileElfStatus.recording : MobileElfStatus.idle; + micAlive.value = _inCallMode; } - // ═══════════════════════════════════════════════════════════════════════════ - // 按钮操作 - // ═══════════════════════════════════════════════════════════════════════════ + /// 原生层已自动获取凭证，此处保留方法供 View 调用不报错 + Future retryCredentials() async { + credentialError.value = ''; + } - Future onMicTap() async { - Logger.info('yiding----------- [按钮] status=${status.value} inCall=${inCall.value}'); - switch (status.value) { - case MobileElfStatus.idle: - case MobileElfStatus.error: - if (!inCall.value) { - await startCall(); - } - break; - case MobileElfStatus.recording: - await stopRecording(); - break; - case MobileElfStatus.speaking: - await interruptSpeaking(); - break; - default: - break; + /// 清空聊天记录 + Future clearMessages() async { + messages.clear(); + try { + await AgentService.clearChatHistory(); + } catch (e) { + Logger.error('$_tag 清空聊天记录失败: $e'); } } // ═══════════════════════════════════════════════════════════════════════════ - // 工具方法 + // 工具 // ═══════════════════════════════════════════════════════════════════════════ - String _calcSign(String secret, String ts) => md5.convert(utf8.encode('$secret$ts')).toString(); - - List> _buildContextMessages() => - messages.map((m) => {'role': m.isUser ? 'user' : 'assistant', 'content': m.text}).toList(); - - void _addUserMessage(String text) { - if (messages.isNotEmpty && messages.last.isUser && messages.last.text == text) return; - messages.add(MobileElfMessage(isUser: true, text: text, time: DateTime.now())); - scrollToBottom(); - } - - void _addAiMessage(String text, {String? ttsUrl}) { - messages.add(MobileElfMessage(isUser: false, text: text, time: DateTime.now(), ttsUrl: ttsUrl)); - scrollToBottom(); + void _addMessage({required bool isUser, required String text}) { + messages.add(MobileElfMessage( + isUser: isUser, text: text, time: DateTime.now())); + _scrollToBottom(); } - void _setError(String msg) { - errorMsg.value = msg; - status.value = MobileElfStatus.error; + /// 更新最后一条 AI 消息（如果存在且更短），否则添加新消息 + void _updateOrAddAssistantMessage(String text) { + final lastIdx = messages.length - 1; + if (lastIdx >= 0 && !messages[lastIdx].isUser) { + // 用更完整的文本替换 + messages[lastIdx] = MobileElfMessage( + isUser: false, text: text, time: messages[lastIdx].time); + messages.refresh(); + } else { + _addMessage(isUser: false, text: text); + } + _scrollToBottom(); } - void scrollToBottom() { - if (scrollController.hasClients) { - WidgetsBinding.instance.addPostFrameCallback((_) { - if (scrollController.hasClients) { - scrollController.animateTo(scrollController.position.maxScrollExtent, - duration: const Duration(milliseconds: 200), curve: Curves.easeOut); - } - }); - } + void _scrollToBottom() { + WidgetsBinding.instance.addPostFrameCallback((_) { + if (scrollController.hasClients) { + scrollController.animateTo( + scrollController.position.maxScrollExtent, + duration: const Duration(milliseconds: 300), + curve: Curves.easeOut, + ); + } + }); } } diff --git a/lib/modules/mobile_elf/views/mobile_elf_view.dart b/lib/modules/mobile_elf/views/mobile_elf_view.dart index 7aaa114b1..5dc2d89a1 100644 --- a/lib/modules/mobile_elf/views/mobile_elf_view.dart +++ b/lib/modules/mobile_elf/views/mobile_elf_view.dart @@ -1,6 +1,8 @@ import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; import 'package:flutter_screenutil/flutter_screenutil.dart'; import 'package:get/get.dart'; +import 'package:intl/intl.dart'; // 用于 DateFormat import '../controllers/mobile_elf_controller.dart'; class MobileElfView extends GetView { @@ -71,6 +73,7 @@ class MobileElfView extends GetView { size: 18.sp, color: isDark ? Colors.white : _textSecondary), onPressed: () => Get.back(), ), + titleSpacing: 0, title: Row( mainAxisSize: MainAxisSize.min, children: [ @@ -81,13 +84,59 @@ class MobileElfView extends GetView { fontSize: 16.sp, fontWeight: FontWeight.w600, color: isDark ? Colors.white : _textPrimary)), + SizedBox(width: 8.w), + Obx(() => _buildStatusPill()), ], ), actions: [ - Obx(() => Padding( - padding: EdgeInsets.only(right: 16.w), - child: _buildStatusPill(), + // 播报开关 + Obx(() => IconButton( + icon: Icon( + controller.ttsEnabled.value + ? Icons.volume_up_rounded + : Icons.volume_off_rounded, + size: 20.sp, + color: controller.ttsEnabled.value + ? _blue + : (isDark ? Colors.grey[500] : _textSecondary), + ), + onPressed: () => controller.ttsEnabled.toggle(), + tooltip: controller.ttsEnabled.value ? '关闭播报' : '开启播报', )), + PopupMenuButton( + icon: Icon(Icons.more_vert, + size: 20.sp, color: isDark ? Colors.white : _textSecondary), + onSelected: (value) { + if (value == 'clear') { + Get.dialog( + AlertDialog( + title: Text('clearChatHistory'.tr.isNotEmpty ? 'clearChatHistory'.tr : '清空记录'), + content: Text('confirmClearChat'.tr.isNotEmpty ? 'confirmClearChat'.tr : '确定清空所有聊天记录吗？'), + actions: [ + TextButton(onPressed: () => Get.back(), child: Text('cancel'.tr.isNotEmpty ? 'cancel'.tr : '取消')), + TextButton( + onPressed: () { + controller.clearMessages(); + Get.back(); + }, + child: Text('confirm'.tr.isNotEmpty ? 'confirm'.tr : '确定', style: const TextStyle(color: Colors.red)), + ), + ], + ), + ); + } + }, + itemBuilder: (_) => [ + PopupMenuItem( + value: 'clear', + child: Row(children: [ + Icon(Icons.delete_outline, size: 18.sp, color: Colors.red), + SizedBox(width: 8.w), + Text('清空记录', style: TextStyle(fontSize: 14.sp, color: Colors.red)), + ]), + ), + ], + ), ], ); } @@ -237,58 +286,138 @@ class MobileElfView extends GetView { ]), SizedBox(height: 4.h), ], - // 气泡体 + // 气泡体（长按弹出菜单） Row( mainAxisAlignment: isUser ? MainAxisAlignment.end : MainAxisAlignment.start, children: [ if (!isUser) SizedBox(width: 34.w), Flexible( - child: Container( - padding: - EdgeInsets.symmetric(horizontal: 14.w, vertical: 10.h), - decoration: BoxDecoration( - color: isUser - ? _blue - : (isDark ? const Color(0xFF2A2A2A) : _surfaceLight), - borderRadius: BorderRadius.only( - topLeft: Radius.circular(18.r), - topRight: Radius.circular(18.r), - bottomLeft: Radius.circular(isUser ? 18.r : 4.r), - bottomRight: Radius.circular(isUser ? 4.r : 18.r), - ), - boxShadow: [ - BoxShadow( - color: Colors.black.withValues(alpha: 0.05), - blurRadius: 6, - offset: const Offset(0, 2), + child: GestureDetector( + onLongPressStart: (details) => _showMessageMenu(details, msg, isDark), + child: Container( + padding: + EdgeInsets.symmetric(horizontal: 14.w, vertical: 10.h), + decoration: BoxDecoration( + color: isUser + ? _blue + : (isDark ? const Color(0xFF2A2A2A) : _surfaceLight), + borderRadius: BorderRadius.only( + topLeft: Radius.circular(18.r), + topRight: Radius.circular(18.r), + bottomLeft: Radius.circular(isUser ? 18.r : 4.r), + bottomRight: Radius.circular(isUser ? 4.r : 18.r), ), - ], + boxShadow: [ + BoxShadow( + color: Colors.black.withValues(alpha: 0.05), + blurRadius: 6, + offset: const Offset(0, 2), + ), + ], + ), + child: Text(msg.text, + style: TextStyle( + fontSize: 15.sp, + height: 1.55, + color: isUser + ? Colors.white + : (isDark ? Colors.white : _textPrimary))), ), - child: Text(msg.text, - style: TextStyle( - fontSize: 15.sp, - height: 1.55, - color: isUser - ? Colors.white - : (isDark ? Colors.white : _textPrimary))), ), ), ], ), - // 时间 + // 时间 + AI 标签 SizedBox(height: 3.h), Padding( padding: EdgeInsets.only(left: isUser ? 0 : 34.w), - child: Text(_fmtTime(msg.time), - style: TextStyle( - fontSize: 11.sp, color: const Color(0xFFD0D5DD))), + child: Row( + mainAxisAlignment: + isUser ? MainAxisAlignment.end : MainAxisAlignment.start, + children: [ + Text(_formatTimestamp(msg.time), + style: TextStyle( + fontSize: 10.sp, color: isDark ? Colors.grey[500] : const Color(0xFFD0D5DD))), + if (!isUser) ...[ + SizedBox(width: 8.w), + Text('generatedByAi'.tr.isNotEmpty ? 'generatedByAi'.tr : '由AI生成', + style: TextStyle( + fontSize: 10.sp, + color: Colors.blue[600], + fontWeight: FontWeight.bold, + fontStyle: FontStyle.italic)), + ], + ], + ), ), ], ), ); } + /// 长按消息弹出菜单 + void _showMessageMenu(LongPressStartDetails details, MobileElfMessage msg, bool isDark) { + HapticFeedback.mediumImpact(); + final overlay = Overlay.of(Get.context!).context.findRenderObject() as RenderBox?; + if (overlay == null) return; + final position = RelativeRect.fromRect( + Rect.fromLTWH(details.globalPosition.dx, details.globalPosition.dy, 0, 0), + Offset.zero & overlay.size, + ); + showMenu( + context: Get.context!, + position: position, + shape: RoundedRectangleBorder(borderRadius: BorderRadius.circular(12.r)), + color: isDark ? const Color(0xFF2A2A2A) : Colors.white, + items: [ + PopupMenuItem( + onTap: () { + Clipboard.setData(ClipboardData(text: msg.text)); + Get.snackbar('success'.tr.isNotEmpty ? 'success'.tr : '成功', + 'copySuccess'.tr.isNotEmpty ? 'copySuccess'.tr : '已复制', + backgroundColor: Colors.blue.withValues(alpha: 0.8), colorText: Colors.white); + }, + child: Row(children: [ + Icon(Icons.copy, size: 18.sp, color: isDark ? Colors.white : Colors.black87), + SizedBox(width: 8.w), + Text('copy'.tr.isNotEmpty ? 'copy'.tr : '复制', + style: TextStyle(fontSize: 14.sp, color: isDark ? Colors.white : Colors.black87)), + ]), + ), + ], + ); + } + + /// 智能时间格式化（与 agent 模块一致） + String _formatTimestamp(DateTime t) { + final now = DateTime.now(); + final diff = now.difference(t); + final today = DateTime(now.year, now.month, now.day); + final msgDate = DateTime(t.year, t.month, t.day); + + if (diff.inMinutes < 1) return 'justNow'.tr.isNotEmpty ? 'justNow'.tr : '刚刚'; + if (diff.inHours < 1) { + final m = diff.inMinutes.toString(); + final text = 'minutesAgo'.trParams({'minutes': m}); + return text.isNotEmpty && text != 'minutesAgo' ? text : '$m分钟前'; + } + if (msgDate == today) return DateFormat('HH:mm').format(t); + if (msgDate == today.subtract(const Duration(days: 1))) { + return '${('yesterday'.tr.isNotEmpty ? 'yesterday'.tr : '昨天')} ${DateFormat('HH:mm').format(t)}'; + } + if (diff.inDays < 7) { + return '${_weekdayName(t.weekday)} ${DateFormat('HH:mm').format(t)}'; + } + if (t.year == now.year) return DateFormat('MM-dd HH:mm').format(t); + return DateFormat('yyyy-MM-dd HH:mm').format(t); + } + + String _weekdayName(int w) { + const names = ['', '星期一', '星期二', '星期三', '星期四', '星期五', '星期六', '星期日']; + return names[w]; + } + Widget _buildRealtimeBubble(String text) { return Padding( padding: EdgeInsets.only(bottom: 14.h), @@ -702,8 +831,6 @@ class MobileElfView extends GetView { ); } - String _fmtTime(DateTime t) => - '${t.hour.toString().padLeft(2, '0')}:${t.minute.toString().padLeft(2, '0')}'; String _fmtDuration(int secs) { final m = (secs ~/ 60).toString().padLeft(2, '0'); diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index 3a1dddbd6..3a56af051 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -100,8 +100,12 @@ object AgentService : CoroutineScope { // 聊天存储服务 private lateinit var chatStorageHelper: ChatStorageHelper + // Agent模式: stt_llm_tts（默认）| sts（移动精灵） + private var agentMode = "stt_llm_tts" + private var stsAgent: StsAgent? = null + // 会话ID，用于区分不同聊天上下文 - private val agentId = "default_agent" + private var agentId = "default_agent" //工具调用提示 private var callingToolText = "" private var insufficientIntegralText = "" // 积分不足提示 @@ -215,6 +219,36 @@ object AgentService : CoroutineScope { if (isInitialized) return true try { + agentMode = config["agentMode"]?.toString() ?: "stt_llm_tts" + agentId = config["agentId"]?.toString()?.takeIf { it.isNotBlank() } ?: "default_agent" + + // STS 模式：跳过 Azure/OpenAI 初始化，直接启动 StsAgent + if (agentMode == "sts") { + this.context = appContext.applicationContext + val stsConfig = StsConfig( + productId = config["stsProductId"]?.toString() ?: "", + productKey = config["stsProductKey"]?.toString() ?: "", + deviceNo = config["stsDeviceNo"]?.toString() ?: "", + deviceId = config["stsDeviceId"]?.toString() ?: "", + deviceSecret = config["stsDeviceSecret"]?.toString() ?: "", + agentId = agentId, + ) + stsAgent = StsAgent( + context = context, + config = stsConfig, + onEvent = { name, data -> sendEvent(name, data) }, + onCredentialsFetched = { deviceId, deviceSecret, deviceNo -> + sendEvent("sts_credentials", mapOf( + "deviceId" to deviceId, + "deviceSecret" to deviceSecret, + "deviceNo" to deviceNo, + )) + } + ) + _isInitialized.set(true) + return true + } + serverurl = config["serverurl"]?.toString() ?: ""; usertoken = config["usertoken"]?.toString() ?: ""; callingToolText = config["callingToolText"]?.toString() ?: ""; @@ -341,6 +375,12 @@ object AgentService : CoroutineScope { */ fun dispose() { try { + if (agentMode == "sts") { + stsAgent?.dispose() + stsAgent = null + _isInitialized.set(false) + return + } BleService.closeCodec() stopRecognition() stopTts() @@ -644,6 +684,15 @@ object AgentService : CoroutineScope { * @param mode 识别模式：normal, ble_wakeup, phone_call, push_to_talk */ fun startRecognition(isExternalActive: Boolean, mode: String = "normal"): Boolean { + if (agentMode == "sts") { + when (mode) { + "push_to_talk" -> stsAgent?.startPtt() + "phone_call" -> stsAgent?.startCall() + "ble_wakeup" -> stsAgent?.startCall() // BLE 唤醒：进入通话模式（VAD 自动检测） + else -> stsAgent?.startPtt() // 普通模式默认 PTT + } + return true + } if(isExternalActive){ isKeepResult = false BleService.openEncoder()//不发指令。只是录制opus @@ -902,6 +951,11 @@ object AgentService : CoroutineScope { * 停止语音识别 */ fun stopRecognition() { + if (agentMode == "sts") { + // StsAgent 内部根据 inCall / pttRecording 状态自行决定停止方式 + stsAgent?.stop() + return + } if (!isRecognitionActive) return Log.d(TAG, "停止语音识别，当前模式: $currentRecognitionMode") @@ -928,7 +982,16 @@ object AgentService : CoroutineScope { * 打断当前响应 * 停止TTS播放和AI流输出 */ + /** 设置 STS 模式 TTS 静音开关 */ + fun setTtsMuted(muted: Boolean) { + stsAgent?.ttsMuted = muted + } + fun interruptCurrentResponse() { + if (agentMode == "sts") { + stsAgent?.interruptSpeaking() + return + } if (isAiStreaming || isTtsSpeaking) { val startTime = System.currentTimeMillis() stopRecognition() @@ -1661,6 +1724,11 @@ object AgentService : CoroutineScope { * 清除聊天历史 */ fun clearChatHistory(callback: (Boolean) -> Unit) { + if (agentMode == "sts") { + stsAgent?.clearHistory() + callback(true) + return + } launch { try { val success = chatStorageHelper.deleteMessages(agentId) diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt index 4e17d7172..e6a08f893 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt @@ -83,6 +83,14 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream config["xunfeiAppId"]?.toString()?.let { intent.putExtra("xunfeiAppId", it) } config["xunfeiAccessKeyId"]?.toString()?.let { intent.putExtra("xunfeiAccessKeyId", it) } config["xunfeiAccessKeySecret"]?.toString()?.let { intent.putExtra("xunfeiAccessKeySecret", it) } + // STS 模式参数 + config["agentMode"]?.toString()?.let { intent.putExtra("agentMode", it) } + config["agentId"]?.toString()?.let { intent.putExtra("agentId", it) } + config["stsProductId"]?.toString()?.let { intent.putExtra("stsProductId", it) } + config["stsProductKey"]?.toString()?.let { intent.putExtra("stsProductKey", it) } + config["stsDeviceNo"]?.toString()?.let { intent.putExtra("stsDeviceNo", it) } + config["stsDeviceId"]?.toString()?.let { intent.putExtra("stsDeviceId", it) } + config["stsDeviceSecret"]?.toString()?.let { intent.putExtra("stsDeviceSecret", it) } config["vipexptime"]?.toString()?.let { intent.putExtra("vipexptime", it) //打印日志 @@ -330,6 +338,17 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream } } + "setTtsMuted" -> { + try { + val muted = call.argument("muted") ?: false + AgentService.setTtsMuted(muted) + result.success(true) + } catch (e: Exception) { + Log.e(TAG, "设置TTS静音失败", e) + result.error("SET_TTS_MUTED_ERROR", "设置TTS静音失败: ${e.message}", null) + } + } + "setCanWakeUp" -> { try { val canWakeUp = call.argument("canWakeUp") diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt index d2111244b..20f0ac498 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt @@ -77,6 +77,14 @@ class BleMediaService : Service(), CoroutineScope { private const val KEY_XUNFEI_APP_ID = "xunfeiAppId" // 讯飞AppID private const val KEY_XUNFEI_ACCESS_KEY_ID = "xunfeiAccessKeyId" // 讯飞AccessKeyID private const val KEY_XUNFEI_ACCESS_KEY_SECRET = "xunfeiAccessKeySecret" // 讯飞AccessKeySecret + // STS 模式参数键 + private const val KEY_AGENT_MODE = "agentMode" + private const val KEY_AGENT_ID = "agentId" + private const val KEY_STS_PRODUCT_ID = "stsProductId" + private const val KEY_STS_PRODUCT_KEY = "stsProductKey" + private const val KEY_STS_DEVICE_NO = "stsDeviceNo" + private const val KEY_STS_DEVICE_ID = "stsDeviceId" + private const val KEY_STS_DEVICE_SECRET = "stsDeviceSecret" // 用于跟踪服务是否正在运行 private val isRunning = AtomicBoolean(false) @@ -546,13 +554,15 @@ private var companionManager: BleCompanionManager? = null val config = getConfigFromPreferences(prefs) Log.d(TAG, "当前配置参数: $config") - // 检查必要的配置是否存在 - val azureSpeechKey = config[KEY_AZURE_SPEECH_KEY]?.toString() ?: "" - val openaiApiKey = config[KEY_OPENAI_API_KEY]?.toString() ?: "" - - if (azureSpeechKey.isEmpty() || openaiApiKey.isEmpty()) { - Log.w(TAG, "语音代理初始化失败: API密钥未配置") - return + // 检查必要的配置是否存在（STS 模式不需要 Azure/OpenAI 密钥） + val agentMode = config[KEY_AGENT_MODE]?.toString() ?: "stt_llm_tts" + if (agentMode != "sts") { + val azureSpeechKey = config[KEY_AZURE_SPEECH_KEY]?.toString() ?: "" + val openaiApiKey = config[KEY_OPENAI_API_KEY]?.toString() ?: "" + if (azureSpeechKey.isEmpty() || openaiApiKey.isEmpty()) { + Log.w(TAG, "语音代理初始化失败: API密钥未配置") + return + } } if (isChatAgentInitialized) { @@ -604,6 +614,13 @@ private var companionManager: BleCompanionManager? = null it.getStringExtra(KEY_XUNFEI_APP_ID)?.let { value -> config[KEY_XUNFEI_APP_ID] = value } it.getStringExtra(KEY_XUNFEI_ACCESS_KEY_ID)?.let { value -> config[KEY_XUNFEI_ACCESS_KEY_ID] = value } it.getStringExtra(KEY_XUNFEI_ACCESS_KEY_SECRET)?.let { value -> config[KEY_XUNFEI_ACCESS_KEY_SECRET] = value } + it.getStringExtra(KEY_AGENT_MODE)?.let { value -> config[KEY_AGENT_MODE] = value } + it.getStringExtra(KEY_AGENT_ID)?.let { value -> config[KEY_AGENT_ID] = value } + it.getStringExtra(KEY_STS_PRODUCT_ID)?.let { value -> config[KEY_STS_PRODUCT_ID] = value } + it.getStringExtra(KEY_STS_PRODUCT_KEY)?.let { value -> config[KEY_STS_PRODUCT_KEY] = value } + it.getStringExtra(KEY_STS_DEVICE_NO)?.let { value -> config[KEY_STS_DEVICE_NO] = value } + it.getStringExtra(KEY_STS_DEVICE_ID)?.let { value -> config[KEY_STS_DEVICE_ID] = value } + it.getStringExtra(KEY_STS_DEVICE_SECRET)?.let { value -> config[KEY_STS_DEVICE_SECRET] = value } Log.d(TAG, "liwei-------vipexptime 3-1 ${ config[KEY_VIP_EXPTIME]}") // 提取 supportedLanguages 参数 it.getStringArrayListExtra(KEY_SUPPORTED_LANGUAGES)?.let { languages -> @@ -671,6 +688,13 @@ private var companionManager: BleCompanionManager? = null config[KEY_XUNFEI_APP_ID] = getConfigString(prefs, KEY_XUNFEI_APP_ID, "") config[KEY_XUNFEI_ACCESS_KEY_ID] = getConfigString(prefs, KEY_XUNFEI_ACCESS_KEY_ID, "") config[KEY_XUNFEI_ACCESS_KEY_SECRET] = getConfigString(prefs, KEY_XUNFEI_ACCESS_KEY_SECRET, "") + config[KEY_AGENT_MODE] = getConfigString(prefs, KEY_AGENT_MODE, "stt_llm_tts") + config[KEY_AGENT_ID] = getConfigString(prefs, KEY_AGENT_ID, "default_agent") + config[KEY_STS_PRODUCT_ID] = getConfigString(prefs, KEY_STS_PRODUCT_ID, "") + config[KEY_STS_PRODUCT_KEY] = getConfigString(prefs, KEY_STS_PRODUCT_KEY, "") + config[KEY_STS_DEVICE_NO] = getConfigString(prefs, KEY_STS_DEVICE_NO, "") + config[KEY_STS_DEVICE_ID] = getConfigString(prefs, KEY_STS_DEVICE_ID, "") + config[KEY_STS_DEVICE_SECRET] = getConfigString(prefs, KEY_STS_DEVICE_SECRET, "") Log.d(TAG, "liwei-------vipexptime 3-0 ${ config[KEY_VIP_EXPTIME]}") // 添加 supportedLanguages 参数处理 val supportedLanguagesStr = getConfigString(prefs, KEY_SUPPORTED_LANGUAGES, "") diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt new file mode 100644 index 000000000..01f938ac8 --- /dev/null +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/StsAgent.kt @@ -0,0 +1,783 @@ +package com.yunqiinnovation.agent_service + +import android.content.Context +import android.media.AudioFormat +import android.media.AudioRecord +import android.media.MediaPlayer +import android.media.MediaRecorder +import android.os.Handler +import android.os.Looper +import android.util.Log +import com.yunqiinnovation.chat_storage.ChatStorageHelper +import kotlinx.coroutines.* +import okhttp3.* +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.RequestBody.Companion.toRequestBody +import okio.ByteString.Companion.toByteString +import org.json.JSONArray +import org.json.JSONObject +import java.io.IOException +import java.security.MessageDigest +import java.security.SecureRandom +import java.security.cert.X509Certificate +import java.util.UUID +import java.util.concurrent.TimeUnit +import javax.net.ssl.SSLContext +import javax.net.ssl.TrustManager +import javax.net.ssl.X509TrustManager +import kotlin.coroutines.resumeWithException +import kotlin.math.sqrt + +/** + * STS 配置参数（移动精灵 A80 平台） + */ +data class StsConfig( + val productId: String, + val productKey: String, + var deviceNo: String, + var deviceId: String, + var deviceSecret: String, + val agentId: String, +) + +/** + * StsAgent — 中国移动 A80 WS 一体化语音对话实现 + * + * 通过一条 WSS 连接完成 ASR + AI + TTS，事件以与标准 stt_llm_tts 相同的格式 + * 通过 [onEvent] 回调发出，供 AgentService 转发到 Flutter EventChannel。 + */ +class StsAgent( + private val context: Context, + private var config: StsConfig, + private val onEvent: (eventName: String, data: Map) -> Unit, + private val onCredentialsFetched: ((deviceId: String, deviceSecret: String, deviceNo: String) -> Unit)? = null, +) : CoroutineScope { + + private val TAG = "StsAgent" + private val job = SupervisorJob() + override val coroutineContext = Dispatchers.IO + job + + init { + fetchCredentials() + } + + // ── WebSocket ───────────────────────────────────────────────────────────── + private val okHttpClient: OkHttpClient = buildTrustAllClient() + private var webSocket: WebSocket? = null + private var readyReceived = false + private var sessionEnded = true + private var dialogRequestId = "" + + // ── 录音 ────────────────────────────────────────────────────────────────── + private val SAMPLE_RATE = 16000 + private val FRAME_SIZE = 5120 // 160ms @ 16kHz 16bit mono + private val RECORD_CHUNK = FRAME_SIZE / 2 // 读取块大小 + private val RECORD_BUFFER = maxOf( + AudioRecord.getMinBufferSize( + SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT + ), FRAME_SIZE * 2 + ) + private var audioRecord: AudioRecord? = null + private var recordingJob: Job? = null + private var micChunkCount = 0 + + // ── 发送缓冲 ────────────────────────────────────────────────────────────── + private val sendBuffer = ArrayDeque() + private var sendBufferOffset = 0 + + // ── 预缓冲（VAD 触发前 ~1.5s）──────────────────────────────────────────── + private val preBuffer = ArrayDeque() + private val PRE_BUFFER_MAX_CHUNKS = 75 + + // ── 本地 VAD ────────────────────────────────────────────────────────────── + private var speechDetected = false + private val VAD_THRESHOLD = 800.0 + private val VAD_ONSET_CHUNKS = 3 + private var vadOnsetCount = 0 + + // ── 状态标记 ────────────────────────────────────────────────────────────── + private var forwardingAudio = false + private var wsConnecting = false + private var inCall = false + private var pttRecording = false + + // ── TTS 播放队列 ────────────────────────────────────────────────────────── + private val ttsQueue = ArrayDeque() + private var ttsPlaying = false + private var ttsInterrupted = false + private var currentPlayer: MediaPlayer? = null + private val mainHandler = Handler(Looper.getMainLooper()) + /** 静音开关：设为 true 时停止当前播放、跳过后续 TTS，文本接收完后主动关闭连接 */ + var ttsMuted = false + set(value) { + field = value + if (value) { + // 立即停止当前播放 + ttsQueue.clear() + ttsPlaying = false + mainHandler.post { + try { currentPlayer?.stop(); currentPlayer?.release() } catch (_: Exception) {} + currentPlayer = null + } + emit("tts_canceled", emptyMap()) + // 如果会话已结束（文本已全部接收），主动关闭连接 + if (sessionEnded) { + closeWs() + if (inCall) { resetVad(); startIdleTimer() } + } + } + } + + // ── 消息历史（WS 上下文） ────────────────────────────────────────────────── + private var messages = JSONArray() + private val chatStorageHelper = ChatStorageHelper(context) + private var currentSessionId = "" + private var pendingAnswer = "" // 累积最完整的回复文本，会话结束时统一发送 + + // ── 计时器 ──────────────────────────────────────────────────────────────── + private var micWatchdogJob: Job? = null + private var lastWatchdogChunkCount = 0 + private var idleTimerJob: Job? = null + private val IDLE_TIMEOUT_MS = 15_000L + + // ── 凭证就绪 ────────────────────────────────────────────────────────────── + var credentialReady = false + private set + + // ═════════════════════════════════════════════════════════════════════════ + // 凭证获取 + // ═════════════════════════════════════════════════════════════════════════ + + fun fetchCredentials() { + if (config.deviceSecret.isNotEmpty()) { + credentialReady = true + return + } + launch { + try { + val body = JSONObject().apply { + put("deviceNoType", "SN") + put("deviceNo", config.deviceNo) + put("productId", config.productId) + put("productKey", config.productKey) + }.toString().toRequestBody("application/json".toMediaType()) + + val req = Request.Builder() + .url("https://ivs.chinamobiledevice.com:11443/v2/customer/device/secret/info") + .post(body) + .build() + + okHttpClient.newCall(req).execute().use { resp -> + val json = JSONObject(resp.body!!.string()) + if (json.getString("code") == "200" && json.getBoolean("success")) { + val d = json.getJSONObject("data") + config = config.copy( + deviceNo = d.optString("deviceNo", config.deviceNo), + deviceId = d.optString("deviceId", ""), + deviceSecret = d.optString("deviceSecret", ""), + ) + credentialReady = config.deviceSecret.isNotEmpty() + if (credentialReady) { + onCredentialsFetched?.invoke(config.deviceId, config.deviceSecret, config.deviceNo) + } + } else { + Log.e(TAG, "凭证获取失败: ${json.optString("message")}") + } + } + } catch (e: Exception) { + Log.e(TAG, "凭证获取异常: ${e.message}") + } + } + } + + // ═════════════════════════════════════════════════════════════════════════ + // 公开接口 + // ═════════════════════════════════════════════════════════════════════════ + + /** 通话模式：开启麦克风，VAD 自动触发 WS */ + fun startCall() { + if (inCall) return + if (ttsPlaying) interruptTts() + if (pttRecording) stopPtt(cancel = true) + if (!credentialReady) { emitError("移动精灵凭证未就绪，请稍后重试"); return } + openMic() + startMicWatchdog() + inCall = true + startIdleTimer() + } + + /** PTT 模式：立即连 WS 并开始录音 */ + fun startPtt() { + if (pttRecording) return + if (ttsPlaying) interruptTts() + if (!credentialReady) { emitError("移动精灵凭证未就绪"); return } + + pttRecording = true + openMic() + startMicWatchdog() + ttsInterrupted = false + closeWs() + sessionEnded = false + wsConnecting = true + dialogRequestId = UUID.randomUUID().toString() + clearSendBuffer() + pendingAnswer = "" + + launch { + try { + connectWs() + forwardingAudio = true + wsConnecting = false + emit("recognition_started", emptyMap()) + } catch (e: Exception) { + wsConnecting = false + pttRecording = false + stopMicWatchdog() + emitError("连接失败: ${e.message}") + } + } + } + + /** PTT 松开：发送剩余音频 + finish */ + fun stopPtt(cancel: Boolean = false) { + pttRecording = false + stopMicWatchdog() + if (cancel) { + forwardingAudio = false + closeWs() + stopMicHard() + return + } + if (forwardingAudio) { + forwardingAudio = false + synchronized(sendBufferLock) { + val remain = sendBuffer.size - sendBufferOffset + if (remain > 0) { + sendAudioChunk(sendBuffer.toMutableList().subList(sendBufferOffset, sendBuffer.size).toByteArray()) + } + sendBuffer.clear(); sendBufferOffset = 0 + } + sendFinish() + emit("recognition_stopped", emptyMap()) + } + stopMicHard() + } + + /** 统一停止：根据当前状态自动选择 stopPtt 或 disconnect */ + fun stop() { + if (pttRecording) stopPtt() + else disconnect() + } + + /** 挂断通话 */ + fun disconnect() { + stopMicWatchdog() + cancelIdleTimer() + forwardingAudio = false + wsConnecting = false + speechDetected = false + stopMicHard() + ttsInterrupted = true + ttsQueue.clear() + ttsPlaying = false + mainHandler.post { + try { currentPlayer?.stop(); currentPlayer?.release() } catch (_: Exception) {} + currentPlayer = null + } + closeWs() + preBuffer.clear() + clearSendBuffer() + flushPendingAnswer() + inCall = false + pttRecording = false + } + + /** 打断 AI 播报 */ + fun interruptSpeaking() { + interruptTts() + ttsInterrupted = false + if (inCall) resetVad() + } + + /** 清空消息历史 */ + fun clearHistory() { + messages = JSONArray() + } + + /** 释放资源 */ + fun dispose() { + disconnect() + job.cancel() + } + + // ═════════════════════════════════════════════════════════════════════════ + // 麦克风 & 本地 VAD + // ═════════════════════════════════════════════════════════════════════════ + + private fun openMic() { + stopMicHard() + speechDetected = false; vadOnsetCount = 0; micChunkCount = 0 + try { + audioRecord = AudioRecord( + MediaRecorder.AudioSource.VOICE_COMMUNICATION, + SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + RECORD_BUFFER + ) + audioRecord?.startRecording() + recordingJob = launch { + val buf = ByteArray(RECORD_CHUNK) + while (isActive && (inCall || pttRecording)) { + val read = audioRecord?.read(buf, 0, buf.size) ?: break + if (read > 0) { micChunkCount++; onMicAudio(buf.copyOf(read)) } + } + } + } catch (e: Exception) { + Log.e(TAG, "打开麦克风失败: ${e.message}") + } + } + + private fun stopMicHard() { + recordingJob?.cancel(); recordingJob = null + try { audioRecord?.stop(); audioRecord?.release() } catch (_: Exception) {} + audioRecord = null + } + + private fun startMicWatchdog() { + stopMicWatchdog() + lastWatchdogChunkCount = micChunkCount + micWatchdogJob = launch { + while (isActive && (inCall || pttRecording)) { + delay(1500) + if (micChunkCount == lastWatchdogChunkCount && !ttsPlaying) openMic() + lastWatchdogChunkCount = micChunkCount + } + } + } + + private fun stopMicWatchdog() { micWatchdogJob?.cancel(); micWatchdogJob = null } + + private fun onMicAudio(chunk: ByteArray) { + if (!inCall && !pttRecording) return + if (forwardingAudio && webSocket != null) { + if (micChunkCount % 3 == 0) emit("mic_rms", mapOf("rms" to calcRms(chunk))) + feedSendBuffer(chunk); return + } + if (wsConnecting) { addToPreBuffer(chunk); return } + if (ttsPlaying) return + + addToPreBuffer(chunk) + val rms = calcRms(chunk) + // 发送麦克风音量给 Dart 层 + if (micChunkCount % 3 == 0) { + emit("mic_rms", mapOf("rms" to rms)) + } + if (rms >= VAD_THRESHOLD) { + vadOnsetCount++ + if (vadOnsetCount >= VAD_ONSET_CHUNKS && !speechDetected) { + speechDetected = true + cancelIdleTimer() + launch { onSpeechDetected() } + } + } else { + vadOnsetCount = 0 + } + } + + private fun calcRms(pcm: ByteArray): Double { + if (pcm.size < 2) return 0.0 + val samples = pcm.size / 2 + var sum = 0.0 + for (i in 0 until pcm.size - 1 step 2) { + var s = (pcm[i].toInt() and 0xFF) or (pcm[i + 1].toInt() shl 8) + if (s >= 32768) s -= 65536 + sum += s.toLong() * s.toLong() + } + return sqrt(sum / samples) + } + + private fun addToPreBuffer(chunk: ByteArray) { + preBuffer.addLast(chunk.copyOf()) + while (preBuffer.size > PRE_BUFFER_MAX_CHUNKS) preBuffer.removeFirst() + } + + // ═════════════════════════════════════════════════════════════════════════ + // VAD 触发 → WS 连接 + // ═════════════════════════════════════════════════════════════════════════ + + private suspend fun onSpeechDetected() { + if (wsConnecting) return + ttsInterrupted = false + closeWs() + sessionEnded = false + wsConnecting = true + dialogRequestId = UUID.randomUUID().toString() + clearSendBuffer() + pendingAnswer = "" + + try { + connectWs() + } catch (e: Exception) { + wsConnecting = false; speechDetected = false + emitError("连接失败: ${e.message}") + return + } + + forwardingAudio = true; wsConnecting = false + emit("recognition_started", emptyMap()) + + val cached = preBuffer.toList(); preBuffer.clear() + for (c in cached) feedSendBuffer(c) + } + + // ═════════════════════════════════════════════════════════════════════════ + // WebSocket + // ═════════════════════════════════════════════════════════════════════════ + + private suspend fun connectWs(): Unit = suspendCancellableCoroutine { cont -> + val ts = System.currentTimeMillis() + val sign = calcSign(config.deviceSecret, ts.toString()) + val sn = UUID.randomUUID().toString() + val url = "wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr" + + "?deviceNo=${config.deviceNo}&deviceId=${config.deviceId}" + + "&productId=${config.productId}&productKey=${config.productKey}" + + "&sign=$sign&sn=$sn&ts=$ts" + + val req = Request.Builder().url(url) + .header("Origin", "https://ivs.chinamobiledevice.com:11443") + .header("Host", "ivs.chinamobiledevice.com:11443") + .build() + + var resolved = false + + webSocket = okHttpClient.newWebSocket(req, object : WebSocketListener() { + override fun onOpen(ws: WebSocket, response: Response) { + sendStartFrame() + } + + override fun onMessage(ws: WebSocket, text: String) { + try { + val msg = JSONObject(text) + if (msg.optString("type") == "ready") { + if (!resolved) { resolved = true; cont.resume(Unit) {} } + } else { + launch { handleMessage(msg) } + } + } catch (_: Exception) {} + } + + override fun onMessage(ws: WebSocket, bytes: okio.ByteString) {} + + override fun onFailure(ws: WebSocket, t: Throwable, response: Response?) { + webSocket = null + if (!resolved) { resolved = true; cont.resumeWithException(t) } + else launch { handleWsDone() } + } + + override fun onClosing(ws: WebSocket, code: Int, reason: String) { ws.close(1000, null) } + + override fun onClosed(ws: WebSocket, code: Int, reason: String) { + webSocket = null + if (!resolved) { resolved = true; cont.resumeWithException(IOException("WS closed")) } + else launch { handleWsDone() } + } + }) + + cont.invokeOnCancellation { closeWs() } + + // 15s 超时 + launch { + delay(15_000) + if (!resolved) { + resolved = true; closeWs() + cont.resumeWithException(IOException("等待 ready 超时")) + } + } + } + + private fun handleWsDone() { + webSocket = null; forwardingAudio = false; wsConnecting = false + when { + sessionEnded && inCall -> if (!ttsPlaying) { resetVad(); startIdleTimer() } + sessionEnded -> { /* PTT 结束，等 TTS 播完 */ } + inCall -> resetVad() + } + } + + private fun resetVad() { + speechDetected = false; vadOnsetCount = 0; preBuffer.clear(); openMic() + } + + private fun closeWs() { + try { webSocket?.cancel() } catch (_: Exception) {} + webSocket = null + } + + // ═════════════════════════════════════════════════════════════════════════ + // 发送帧 + // ═════════════════════════════════════════════════════════════════════════ + + private fun sendStartFrame() { + currentSessionId = UUID.randomUUID().toString() + val payload = JSONObject().apply { + put("type", "start") + put("data", JSONObject().apply { + put("cuid", config.deviceId) + put("format", "pcm") + put("sample", 16000) + put("support_dcs", 2) + put("chunk_size", 10240) + put("support_tts", true) + put("support_text2dcs", true) + put("user_agent", "voitrans/1.0") + put("dialog_request_id", dialogRequestId) + put("access_rc", true) + put("support_part_tts", true) + put("need_stoplisten", true) + put("need_dialogue_finish", true) + put("result_trans2directive", false) + put("rc_version", "5") + put("messages", messages) + put("client_context", JSONArray()) + }) + } + webSocket?.send(payload.toString()) + } + + private val sendBufferLock = Any() + + private fun clearSendBuffer() { + synchronized(sendBufferLock) { + sendBuffer.clear(); sendBufferOffset = 0 + } + } + + private fun feedSendBuffer(chunk: ByteArray) { + synchronized(sendBufferLock) { + sendBuffer.addAll(chunk.toList()) + while (sendBuffer.size - sendBufferOffset >= FRAME_SIZE) { + val frame = ByteArray(FRAME_SIZE) { i -> sendBuffer[sendBufferOffset + i] } + sendBufferOffset += FRAME_SIZE + sendAudioChunk(frame) + } + // 收缩缓冲 + if (sendBufferOffset >= FRAME_SIZE * 8) { + val toRemove = sendBufferOffset + repeat(toRemove) { if (sendBuffer.isNotEmpty()) sendBuffer.removeFirst() } + sendBufferOffset = 0 + } + } + } + + private fun sendAudioChunk(data: ByteArray) { + try { webSocket?.send(data.toByteString()) } catch (_: Exception) {} + } + + private fun sendFinish() { + try { webSocket?.send("""{"type":"finish"}""") } catch (_: Exception) {} + } + + // ═════════════════════════════════════════════════════════════════════════ + // 消息处理 + // ═════════════════════════════════════════════════════════════════════════ + + private suspend fun handleMessage(msg: JSONObject) { + when (msg.optString("type")) { + "stoplisten" -> { + forwardingAudio = false; speechDetected = false; vadOnsetCount = 0 + synchronized(sendBufferLock) { + val remain = sendBuffer.size - sendBufferOffset + if (remain > 0) { + sendAudioChunk(ByteArray(remain) { i -> sendBuffer[sendBufferOffset + i] }) + } + sendBuffer.clear(); sendBufferOffset = 0 + } + sendFinish() + emit("recognition_stopped", emptyMap()) + } + "mid_result" -> { + val text = msg.optString("result", "") + if (text.isNotEmpty()) emit("recognizing", mapOf("text" to text)) + } + "fin_result" -> { + val result = msg.optString("result", "") + if (result.isNotEmpty()) { + emit("recognition_result", mapOf("text" to result)) + messages.put(JSONObject(mapOf("role" to "user", "content" to result))) + withContext(Dispatchers.IO) { + try { + chatStorageHelper.saveMessage(config.agentId, currentSessionId, result, "user") + } catch (_: Exception) {} + } + } + if (msg.optInt("end", 0) == 1) markSessionEnded() + } + "inside_rc" -> handleInsideRc(msg) + "dcs_decide" -> if (msg.optInt("end", 0) == 1) markSessionEnded() + } + } + + private suspend fun handleInsideRc(msg: JSONObject) { + val data = msg.optJSONObject("data") ?: return + if (data.optInt("code", -1) != 0) return + + val rawAnswer = data.optString("assistant_answer", "") + val cleanAnswer = try { + JSONObject(rawAnswer).optString("content", rawAnswer) + } catch (_: Exception) { rawAnswer } + + var ttsUrl: String? = null + val items = data.optJSONArray("data") + if (items != null) { + for (i in 0 until items.length()) { + val item = items.optJSONObject(i) ?: continue + val h = item.optJSONObject("header") ?: continue + if (h.optString("namespace") == "ai.fxzsos.device_interface.voice_output" && + h.optString("name") == "Speak" + ) { + ttsUrl = item.optJSONObject("payload")?.optString("url") + break + } + } + } + + // 每次取更完整的回复文本，立即发送给 UI + if (cleanAnswer.isNotEmpty() && cleanAnswer.length > pendingAnswer.length) { + pendingAnswer = cleanAnswer + emit("assistant_response", mapOf("response" to cleanAnswer, "sessionid" to currentSessionId)) + } + ttsUrl?.takeIf { it.isNotEmpty() }?.let { enqueueTts(it) } + if (msg.optInt("end", 0) == 1) markSessionEnded() + } + + /** 将 pendingAnswer 保存到历史和存储，并清空 */ + private fun flushPendingAnswer() { + if (pendingAnswer.isEmpty()) return + val answer = pendingAnswer + val sid = currentSessionId + messages.put(JSONObject(mapOf("role" to "assistant", "content" to answer))) + launch(Dispatchers.IO) { + try { + chatStorageHelper.saveMessage(config.agentId, sid, answer, "assistant") + } catch (_: Exception) {} + } + pendingAnswer = "" + } + + private fun markSessionEnded() { + if (sessionEnded) return + sessionEnded = true; forwardingAudio = false; speechDetected = false; vadOnsetCount = 0 + clearSendBuffer() + flushPendingAnswer() + // 静音模式下文本接收完毕，发送完成事件并关闭连接 + if (ttsMuted) { + emit("tts_completed", emptyMap()) + closeWs() + if (inCall) { resetVad(); startIdleTimer() } + } + } + + // ═════════════════════════════════════════════════════════════════════════ + // TTS 播放队列 + // ═════════════════════════════════════════════════════════════════════════ + + private fun enqueueTts(url: String) { + if (ttsMuted) return // 静音模式跳过 TTS + ttsQueue.addLast(url) + if (!ttsPlaying) { + stopMicHard() + mainHandler.post { playNextTts() } + } + } + + private fun playNextTts() { + if (ttsQueue.isEmpty() || ttsInterrupted) { + ttsPlaying = false + if (!ttsInterrupted) { + emit("tts_completed", emptyMap()) + if (inCall) { resetVad(); startIdleTimer() } + } + return + } + ttsPlaying = true + emit("tts_started", emptyMap()) + val url = ttsQueue.removeFirst() + try { + currentPlayer?.stop(); currentPlayer?.release() + currentPlayer = MediaPlayer().apply { + setDataSource(url) + setOnCompletionListener { + if (!ttsInterrupted) mainHandler.post { playNextTts() } + } + setOnErrorListener { _, _, _ -> + if (!ttsInterrupted) mainHandler.post { playNextTts() } + true + } + prepare() + start() + } + } catch (e: Exception) { + Log.e(TAG, "TTS 播放失败: ${e.message}") + if (!ttsInterrupted) mainHandler.post { playNextTts() } + } + } + + private fun interruptTts() { + ttsInterrupted = true; ttsQueue.clear(); ttsPlaying = false + mainHandler.post { + try { currentPlayer?.stop(); currentPlayer?.release() } catch (_: Exception) {} + currentPlayer = null + } + emit("tts_canceled", emptyMap()) + } + + // ═════════════════════════════════════════════════════════════════════════ + // 超时 & 工具 + // ═════════════════════════════════════════════════════════════════════════ + + private fun startIdleTimer() { + cancelIdleTimer() + idleTimerJob = launch { + delay(IDLE_TIMEOUT_MS) + if (inCall && !speechDetected) disconnect() + } + } + + private fun cancelIdleTimer() { idleTimerJob?.cancel(); idleTimerJob = null } + + private fun calcSign(secret: String, ts: String): String { + val digest = MessageDigest.getInstance("MD5").digest("$secret$ts".toByteArray()) + return digest.joinToString("") { "%02x".format(it) } + } + + private fun emit(eventName: String, data: Map) { + launch(Dispatchers.Main) { onEvent(eventName, data) } + } + + private fun emitError(msg: String) = emit("error", mapOf("message" to msg)) + + private fun buildTrustAllClient(): OkHttpClient { + return try { + val trustAll = arrayOf(object : X509TrustManager { + override fun checkClientTrusted(c: Array, t: String) {} + override fun checkServerTrusted(c: Array, t: String) {} + override fun getAcceptedIssuers(): Array = emptyArray() + }) + val sc = SSLContext.getInstance("TLS") + sc.init(null, trustAll, SecureRandom()) + OkHttpClient.Builder() + .sslSocketFactory(sc.socketFactory, trustAll[0] as X509TrustManager) + .hostnameVerifier { _, _ -> true } + .connectTimeout(15, TimeUnit.SECONDS) + .readTimeout(90, TimeUnit.SECONDS) + .writeTimeout(90, TimeUnit.SECONDS) + .build() + } catch (e: Exception) { + Log.e(TAG, "buildTrustAllClient 失败，使用默认客户端", e) + OkHttpClient() + } + } +} diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index fd135db40..034f6b72d 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -66,8 +66,12 @@ class AgentServiceImpl: NSObject { return ChatStorageHelper.shared } + // Agent 模式 + private var agentMode = "stt_llm_tts" + private var stsAgent: StsAgent? + // 会话ID，用于区分不同聊天上下文 - private let agentId = "default_agent" + private var agentId = "default_agent" //工具调用提示 public var callingToolText = "" // 工具调用提示 public var insufficientIntegralText = "" // 积分不足提示 @@ -194,9 +198,38 @@ class AgentServiceImpl: NSObject { func initialize(config: [String: Any]) -> Bool { // if isInitialized { return true } // os_log("initialize: config=%{public}@", log: logger, type: .info, config) - + + agentMode = config["agentMode"] as? String ?? "stt_llm_tts" + agentId = (config["agentId"] as? String)?.isEmpty == false + ? config["agentId"] as! String : "default_agent" + + // STS 模式：跳过 Azure/OpenAI 初始化 + if agentMode == "sts" { + let stsConfig = StsConfig( + productId: config["stsProductId"] as? String ?? "", + productKey: config["stsProductKey"] as? String ?? "", + deviceNo: config["stsDeviceNo"] as? String ?? "", + deviceId: config["stsDeviceId"] as? String ?? "", + deviceSecret: config["stsDeviceSecret"] as? String ?? "", + agentId: agentId + ) + stsAgent = StsAgent( + config: stsConfig, + onEvent: { [weak self] name, data in self?.sendEvent(name: name, data: data) }, + onCredentialsFetched: { [weak self] deviceId, deviceSecret, deviceNo in + self?.sendEvent(name: "sts_credentials", data: [ + "deviceId": deviceId, + "deviceSecret": deviceSecret, + "deviceNo": deviceNo, + ]) + } + ) + isInitialized = true + return true + } + self.config = config - + if let navigationMode = config["navigationMode"] as? String { self.navigationMode = navigationMode } @@ -490,6 +523,17 @@ class AgentServiceImpl: NSObject { } } func startRecognition(useBle: Bool = false, mode: String = "normal") -> Bool { + if agentMode == "sts" { + switch mode { + case "push_to_talk": + stsAgent?.startPtt() + case "phone_call", "ble_wakeup": + stsAgent?.startCall() + default: + stsAgent?.startPtt() + } + return true + } if !isInitialized { sendError("服务未初始化", code: "NOT_INITIALIZED") return false @@ -575,6 +619,10 @@ class AgentServiceImpl: NSObject { } func stopRecognition() -> Bool { + if agentMode == "sts" { + stsAgent?.stop() + return true + } /// 函数功能：停止语音识别 /// 特性： /// 1) 即便处于"启动中"（isStartingRecognition）也会打印日志并尝试停止 @@ -679,8 +727,12 @@ class AgentServiceImpl: NSObject { } func interruptCurrentResponse() -> Bool { + if agentMode == "sts" { + stsAgent?.interruptSpeaking() + return true + } var interrupted = false - + if isSpeaking { interrupted = stopTts() || interrupted } @@ -1490,8 +1542,12 @@ private func jsonToString(_ json: [String: Any]) -> String? { return result } func clearChatHistory() -> Bool { + if agentMode == "sts" { + stsAgent?.clearHistory() + return true + } chatHistory.removeAll() - + DispatchQueue.global(qos: .utility).async { let success = self.chatStorageHelper.deleteMessages(agentId:self.agentId, messageIds: nil as [Int]?) if !success { @@ -1738,6 +1794,12 @@ private func jsonToString(_ json: [String: Any]) -> String? { func dispose() -> Bool { print("ai释放资源") + if agentMode == "sts" { + stsAgent?.dispose() + stsAgent = nil + isInitialized = false + return true + } if isRecognizing { // 关闭编码器 (设备侧) BleService.shared.closeCodec() diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/StsAgent.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/StsAgent.swift new file mode 100644 index 000000000..a5e50eb78 --- /dev/null +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/StsAgent.swift @@ -0,0 +1,495 @@ +import Foundation +import AVFoundation +import CryptoKit +import os.log +import chat_storage + +// MARK: - StsConfig + +struct StsConfig { + let productId: String + let productKey: String + var deviceNo: String + var deviceId: String + var deviceSecret: String + let agentId: String +} + +// MARK: - StsAgent + +/// StsAgent — 中国移动 A80 WS 一体化语音对话实现（iOS） +/// +/// 通过一条 WSS 连接完成 ASR + AI + TTS，事件格式与标准 stt_llm_tts 完全一致， +/// 通过 onEvent 回调发出，供 AgentServiceImpl 转发到 Flutter EventChannel。 +class StsAgent: NSObject { + + private let TAG = "StsAgent" + private let logger = OSLog(subsystem: "com.yunqiinnovation.agent_service", category: "StsAgent") + + private var config: StsConfig + private let onEvent: (String, [String: Any]) -> Void + private let onCredentialsFetched: ((String, String, String) -> Void)? + + // MARK: WebSocket + private var webSocketTask: URLSessionWebSocketTask? + private var urlSession: URLSession! + private var readyReceived = false + private var sessionEnded = true + private var dialogRequestId = "" + + // MARK: 录音 + private let SAMPLE_RATE: Double = 16000 + private let FRAME_SIZE = 5120 // 160ms @ 16kHz 16bit mono + private var audioEngine: AVAudioEngine? + private var inputNode: AVAudioInputNode? + private var recordingActive = false + private var sendBuffer = Data() + private var micChunkCount = 0 + + // MARK: VAD + private let VAD_THRESHOLD: Float = 800 + private let VAD_ONSET_CHUNKS = 3 + private var vadOnsetCount = 0 + private var vadActive = false + private var preBuffer: [[UInt8]] = [] + private let PRE_BUFFER_MAX = 75 + + // MARK: TTS + private var ttsQueue: [String] = [] + private var ttsPlayer: AVPlayer? + private var ttsPlaying = false + private var ttsPlayerObserver: Any? + + // MARK: 历史 + private var messages: [[String: String]] = [] + private var currentSessionId = "" + private let chatStorage = ChatStorageHelper.shared + + // MARK: 模式 + private var isPttMode = false + + // MARK: 凭证请求 + private var credentialsFetched = false + + init(config: StsConfig, + onEvent: @escaping (String, [String: Any]) -> Void, + onCredentialsFetched: ((String, String, String) -> Void)? = nil) { + self.config = config + self.onEvent = onEvent + self.onCredentialsFetched = onCredentialsFetched + super.init() + let sessionConfig = URLSessionConfiguration.default + urlSession = URLSession(configuration: sessionConfig, delegate: nil, delegateQueue: OperationQueue()) + } + + // MARK: - 公共接口 + + func startCall() { + isPttMode = false + os_log("StsAgent startCall", log: logger, type: .info) + fetchCredentialsIfNeeded { [weak self] in + self?.connect() + } + } + + func startPtt() { + isPttMode = true + os_log("StsAgent startPtt", log: logger, type: .info) + fetchCredentialsIfNeeded { [weak self] in + self?.connect() + } + } + + func stopPtt() { + os_log("StsAgent stopPtt", log: logger, type: .info) + guard isPttMode else { return } + sendFinishFrame() + } + + /// 统一停止：根据当前模式自动选择 stopPtt 或 dispose + func stop() { + if isPttMode { + stopPtt() + } else { + dispose() + } + } + + func interruptSpeaking() { + os_log("StsAgent interruptSpeaking", log: logger, type: .info) + stopTts() + emit("tts_canceled") + } + + func clearHistory() { + messages.removeAll() + DispatchQueue.global().async { [weak self] in + guard let self = self else { return } + _ = self.chatStorage.deleteMessages(agentId: self.config.agentId, messageIds: nil as [Int]?) + } + } + + func dispose() { + os_log("StsAgent dispose", log: logger, type: .info) + stopRecording() + webSocketTask?.cancel(with: .normalClosure, reason: nil) + webSocketTask = nil + stopTts() + } + + // MARK: - 凭证获取 + + private func fetchCredentialsIfNeeded(completion: @escaping () -> Void) { + if credentialsFetched && !config.deviceId.isEmpty && !config.deviceSecret.isEmpty { + completion() + return + } + let ts = Int64(Date().timeIntervalSince1970 * 1000) + let sign = calcSign(secret: config.productKey, ts: ts) + let urlStr = "https://ivs.chinamobiledevice.com:11443/v2/customer/device/secret/info" + guard var components = URLComponents(string: urlStr) else { completion(); return } + components.queryItems = [ + URLQueryItem(name: "productId", value: config.productId), + URLQueryItem(name: "deviceNo", value: config.deviceNo), + URLQueryItem(name: "ts", value: "\(ts)"), + URLQueryItem(name: "sign", value: sign), + ] + guard let url = components.url else { completion(); return } + var req = URLRequest(url: url) + req.timeoutInterval = 10 + URLSession.shared.dataTask(with: req) { [weak self] data, _, error in + guard let self = self else { return } + if let error = error { + os_log("凭证请求失败: %{public}@", log: self.logger, type: .error, error.localizedDescription) + completion() + return + } + guard let data = data, + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let body = json["body"] as? [String: Any] else { + completion() + return + } + if let deviceId = body["deviceId"] as? String { self.config.deviceId = deviceId } + if let deviceSecret = body["deviceSecret"] as? String { self.config.deviceSecret = deviceSecret } + if let deviceNo = body["deviceNo"] as? String { self.config.deviceNo = deviceNo } + self.credentialsFetched = true + self.onCredentialsFetched?(self.config.deviceId, self.config.deviceSecret, self.config.deviceNo) + completion() + }.resume() + } + + // MARK: - WebSocket 连接 + + private func connect() { + let ts = Int64(Date().timeIntervalSince1970 * 1000) + let sign = calcSign(secret: config.deviceSecret, ts: ts) + let urlStr = "wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr" + guard var components = URLComponents(string: urlStr) else { return } + components.queryItems = [ + URLQueryItem(name: "productId", value: config.productId), + URLQueryItem(name: "deviceId", value: config.deviceId), + URLQueryItem(name: "ts", value: "\(ts)"), + URLQueryItem(name: "sign", value: sign), + ] + guard let url = components.url else { return } + webSocketTask = urlSession.webSocketTask(with: url) + webSocketTask?.resume() + receiveMessages() + + // 发送 start 帧 + sendStartFrame() + } + + private func sendStartFrame() { + dialogRequestId = UUID().uuidString.replacingOccurrences(of: "-", with: "").lowercased() + currentSessionId = dialogRequestId + let payload: [String: Any] = [ + "msgType": "start", + "dialogRequestId": dialogRequestId, + "params": [ + "asrParams": [ + "format": "pcm", + "sampleRate": 16000, + "lang": "zh-CN", + "vadEnable": false, + ], + "contextParams": [ + "history": messages.map { ["role": $0["role"] ?? "", "content": $0["content"] ?? ""] } + ] + ] + ] + sendJson(payload) + } + + private func receiveMessages() { + webSocketTask?.receive { [weak self] result in + guard let self = self else { return } + switch result { + case .success(let msg): + switch msg { + case .string(let text): + self.handleMessage(text) + default: + break + } + self.receiveMessages() + case .failure(let err): + os_log("WS error: %{public}@", log: self.logger, type: .error, err.localizedDescription) + self.emit("error", data: ["message": err.localizedDescription]) + } + } + } + + // MARK: - 消息处理 + + private func handleMessage(_ text: String) { + guard let data = text.data(using: .utf8), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return } + let msgType = json["msgType"] as? String ?? "" + os_log("WS recv: %{public}@", log: logger, type: .debug, msgType) + + switch msgType { + case "ready": + readyReceived = true + sessionEnded = false + emit("recognition_started") + startRecording() + if isPttMode { + // PTT 模式等待 stopPtt() 调用发送 finish + } + + case "mid_result": + let result = (json["data"] as? [String: Any])?["result"] as? String ?? "" + if !result.isEmpty { emit("recognizing", data: ["text": result]) } + + case "fin_result": + let result = (json["data"] as? [String: Any])?["result"] as? String ?? "" + emit("recognition_result", data: ["text": result]) + emit("recognition_stopped") + stopRecording() + if !result.isEmpty { + messages.append(["role": "user", "content": result]) + saveToChatStorage(text: result, role: "user") + } + + case "stoplisten": + stopRecording() + emit("recognition_stopped") + + case "inside_rc": + handleInsideRc(json["data"] as? [String: Any]) + + case "dcs_decide": + // 语义决策，忽略 + break + + default: + os_log("未处理的消息类型: %{public}@", log: logger, type: .info, msgType) + } + } + + private func handleInsideRc(_ data: [String: Any]?) { + guard let data = data else { return } + guard let rcType = data["rcType"] as? String else { return } + if rcType == "assistant_answer" { + let answer = (data["answer"] as? [String: Any])?["text"] as? String ?? "" + let cleanAnswer = answer.trimmingCharacters(in: .whitespacesAndNewlines) + let sid = data["sessionId"] as? String ?? currentSessionId + if !cleanAnswer.isEmpty { + messages.append(["role": "assistant", "content": cleanAnswer]) + saveToChatStorage(text: cleanAnswer, role: "assistant") + emit("assistant_response", data: ["response": cleanAnswer, "sessionid": sid]) + } + // TTS URLs + if let ttsData = data["tts"] as? [[String: Any]] { + for item in ttsData { + if let url = item["url"] as? String, !url.isEmpty { + ttsQueue.append(url) + } + } + if !ttsPlaying { playNextTts() } + } + } + } + + // MARK: - TTS 播放 + + private func playNextTts() { + guard !ttsQueue.isEmpty else { + ttsPlaying = false + emit("tts_completed") + return + } + let urlStr = ttsQueue.removeFirst() + guard let url = URL(string: urlStr) else { playNextTts(); return } + + ttsPlaying = true + if ttsQueue.isEmpty && !ttsPlaying { emit("tts_started") } + emit("tts_started") + + let item = AVPlayerItem(url: url) + ttsPlayer = AVPlayer(playerItem: item) + if let observer = ttsPlayerObserver { + NotificationCenter.default.removeObserver(observer) + } + ttsPlayerObserver = NotificationCenter.default.addObserver( + forName: .AVPlayerItemDidPlayToEndTime, object: item, queue: .main + ) { [weak self] _ in + self?.playNextTts() + } + ttsPlayer?.play() + } + + private func stopTts() { + ttsPlayer?.pause() + ttsPlayer = nil + ttsQueue.removeAll() + ttsPlaying = false + if let observer = ttsPlayerObserver { + NotificationCenter.default.removeObserver(observer) + ttsPlayerObserver = nil + } + } + + // MARK: - 录音 + + private func startRecording() { + guard !recordingActive else { return } + let engine = AVAudioEngine() + audioEngine = engine + inputNode = engine.inputNode + let fmt = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: SAMPLE_RATE, channels: 1, interleaved: true)! + let hwFmt = inputNode!.outputFormat(forBus: 0) + let converterFmt = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: SAMPLE_RATE, channels: 1, interleaved: true)! + + inputNode!.installTap(onBus: 0, bufferSize: 4096, format: hwFmt) { [weak self] buffer, _ in + guard let self = self, self.recordingActive else { return } + // 转换为 16kHz PCM16 + guard let cvt = AVAudioConverter(from: hwFmt, to: converterFmt) else { return } + let frameCount = AVAudioFrameCount(Double(buffer.frameLength) * self.SAMPLE_RATE / hwFmt.sampleRate) + guard let outBuf = AVAudioPCMBuffer(pcmFormat: converterFmt, frameCapacity: frameCount) else { return } + var error: NSError? + cvt.convert(to: outBuf, error: &error) { _, outStatus in + outStatus.pointee = .haveData + return buffer + } + if error != nil { return } + let byteCount = Int(outBuf.frameLength) * 2 + let ptr = outBuf.int16ChannelData![0] + let data = Data(bytes: ptr, count: byteCount) + self.onMicAudio(data) + } + + do { + try AVAudioSession.sharedInstance().setCategory(.playAndRecord, mode: .voiceChat, options: [.defaultToSpeaker]) + try AVAudioSession.sharedInstance().setActive(true) + try engine.start() + recordingActive = true + } catch { + os_log("录音启动失败: %{public}@", log: logger, type: .error, error.localizedDescription) + } + } + + private func stopRecording() { + guard recordingActive else { return } + recordingActive = false + inputNode?.removeTap(onBus: 0) + audioEngine?.stop() + audioEngine = nil + inputNode = nil + sendBuffer.removeAll() + micChunkCount = 0 + vadActive = false + vadOnsetCount = 0 + preBuffer.removeAll() + } + + private func onMicAudio(_ pcm: Data) { + micChunkCount += 1 + let rms = calcRms(pcm) + let chunk = [UInt8](pcm) + + if !vadActive { + if preBuffer.count >= PRE_BUFFER_MAX { preBuffer.removeFirst() } + preBuffer.append(chunk) + } + + if !vadActive && rms > VAD_THRESHOLD { + vadOnsetCount += 1 + if vadOnsetCount >= VAD_ONSET_CHUNKS { + vadActive = true + // 先发预缓存 + for c in preBuffer { feedSendBuffer(Data(c)) } + preBuffer.removeAll() + } + } else if vadActive { + feedSendBuffer(pcm) + } + } + + private func calcRms(_ pcm: Data) -> Float { + let count = pcm.count / 2 + guard count > 0 else { return 0 } + var sum: Float = 0 + pcm.withUnsafeBytes { ptr in + let samples = ptr.bindMemory(to: Int16.self) + for i in 0..= FRAME_SIZE { + let frame = sendBuffer.prefix(FRAME_SIZE) + sendBuffer.removeFirst(FRAME_SIZE) + webSocketTask?.send(.data(Data(frame))) { _ in } + } + } + + private func sendFinishFrame() { + let payload: [String: Any] = [ + "msgType": "finish", + "dialogRequestId": dialogRequestId, + ] + sendJson(payload) + } + + // MARK: - 工具 + + private func sendJson(_ dict: [String: Any]) { + guard let data = try? JSONSerialization.data(withJSONObject: dict), + let str = String(data: data, encoding: .utf8) else { return } + webSocketTask?.send(.string(str)) { _ in } + } + + private func calcSign(secret: String, ts: Int64) -> String { + let raw = "\(secret)\(ts)" + let data = Data(raw.utf8) + let digest = Insecure.MD5.hash(data: data) + return digest.map { String(format: "%02x", $0) }.joined() + } + + private func emit(_ name: String, data: [String: Any] = [:]) { + DispatchQueue.main.async { [weak self] in + self?.onEvent(name, data) + } + } + + private func saveToChatStorage(text: String, role: String) { + DispatchQueue.global().async { [weak self] in + guard let self = self else { return } + _ = self.chatStorage.saveMessage( + agentId: self.config.agentId, + sessionId: self.currentSessionId, + message: text, + sender: role, + metadata: nil + ) + } + } +} diff --git a/local_plugins/agent_service/lib/agent_service.dart b/local_plugins/agent_service/lib/agent_service.dart index 92dd7cafa..1cdec8d65 100644 --- a/local_plugins/agent_service/lib/agent_service.dart +++ b/local_plugins/agent_service/lib/agent_service.dart @@ -71,6 +71,12 @@ enum AgentServiceEventType { /// 错误 error, + /// STS 凭证获取成功（设备 deviceId/deviceSecret） + stsCredentials, + + /// 麦克风音量（RMS） + micRms, + /// 未知事件 unknown } @@ -176,6 +182,10 @@ class AgentService { return AgentServiceEventType.imageProcessing; case 'image_ready': return AgentServiceEventType.imageReady; + case 'sts_credentials': + return AgentServiceEventType.stsCredentials; + case 'mic_rms': + return AgentServiceEventType.micRms; default: return AgentServiceEventType.unknown; } @@ -446,6 +456,18 @@ class AgentService { } } + /// 设置 TTS 静音开关（STS 模式） + static Future setTtsMuted(bool muted) async { + try { + final bool result = await _channel.invokeMethod('setTtsMuted', { + 'muted': muted, + }); + return result; + } on PlatformException catch (e) { + throw AgentServiceException(e.code, e.message ?? '设置TTS静音失败', e.details); + } + } + /// 中断当前响应 /// /// 停止语音合成和AI流输出 diff --git a/local_plugins/ble_service/lib/ble_service.dart b/local_plugins/ble_service/lib/ble_service.dart index 995ab4280..89e166651 100644 --- a/local_plugins/ble_service/lib/ble_service.dart +++ b/local_plugins/ble_service/lib/ble_service.dart @@ -329,7 +329,7 @@ class BleService { /// 断开连接 Future stopScan() async { try { - final result = await _methodChannel.invokeMethod('stopscan'); + final result = await _methodChannel.invokeMethod('stopScan'); return result ?? false; } catch (e) { print('开启扫描ble设备失败: $e');