You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
980 lines
38 KiB
980 lines
38 KiB
import 'dart:async';
|
|
import 'dart:convert';
|
|
import 'dart:io';
|
|
import 'dart:math' as math;
|
|
import 'dart:typed_data';
|
|
import 'package:crypto/crypto.dart';
|
|
import 'package:dio/dio.dart';
|
|
import 'package:flutter/foundation.dart' show kIsWeb;
|
|
import 'package:flutter/widgets.dart';
|
|
import 'package:audio_session/audio_session.dart';
|
|
import 'package:flutter_dotenv/flutter_dotenv.dart';
|
|
import 'package:get/get.dart';
|
|
import 'package:get_storage/get_storage.dart';
|
|
import 'package:just_audio/just_audio.dart';
|
|
import 'package:record/record.dart';
|
|
import 'package:uuid/uuid.dart';
|
|
import 'package:web_socket_channel/io.dart';
|
|
import 'package:web_socket_channel/web_socket_channel.dart';
|
|
import '../../../core/utils/logger.dart';
|
|
|
|
enum MobileElfStatus { idle, connecting, recording, thinking, speaking, error }
|
|
|
|
class MobileElfMessage {
|
|
final bool isUser;
|
|
final String text;
|
|
final DateTime time;
|
|
final String? ttsUrl;
|
|
MobileElfMessage({required this.isUser, required this.text, required this.time, this.ttsUrl});
|
|
}
|
|
|
|
class MobileElfController extends GetxController {
|
|
// ── 可观察状态 ─────────────────────────────────────────────────────────────
|
|
final status = MobileElfStatus.idle.obs;
|
|
final messages = <MobileElfMessage>[].obs;
|
|
final realtimeText = ''.obs;
|
|
final errorMsg = ''.obs;
|
|
final inCall = false.obs;
|
|
final callSeconds = 0.obs;
|
|
final recordingSeconds = 0.obs;
|
|
final micRms = 0.0.obs; // 实时麦克风音量（RMS），供 UI 显示
|
|
final micAlive = false.obs; // 麦克风流是否存活
|
|
final pttRecording = false.obs; // 按住说话录音中
|
|
final pttSeconds = 0.obs; // 按住说话计时
|
|
final ScrollController scrollController = ScrollController();
|
|
|
|
// ── WebSocket ──────────────────────────────────────────────────────────────
|
|
WebSocketChannel? _channel;
|
|
StreamSubscription? _wsSubscription;
|
|
Completer<void>? _readyCompleter;
|
|
bool _sessionEnded = true;
|
|
String _dialogRequestId = '';
|
|
int _currentAiMsgIndex = -1;
|
|
|
|
// ── 录音 & 本地 VAD ────────────────────────────────────────────────────────
|
|
final AudioRecorder _recorder = AudioRecorder();
|
|
StreamSubscription<Uint8List>? _micSubscription;
|
|
|
|
// 音频发送缓冲（凑帧用）
|
|
final List<int> _sendBuffer = [];
|
|
int _sendBufferOffset = 0;
|
|
static const int _frameSize = 5120; // 160ms @ 16kHz 16bit mono
|
|
|
|
// 预缓存：滚动保留最近 ~1.5 秒音频，用于在 WS 连接前缓存用户已说的话
|
|
final List<Uint8List> _preBuffer = [];
|
|
static const int _preBufferMaxChunks = 75; // ~1.5s (按每 chunk 20ms 算)
|
|
|
|
// 本地 VAD（仅在空闲等待状态工作，TTS 播放中不做 VAD）
|
|
bool _speechDetected = false;
|
|
static const int _vadThreshold = 800;
|
|
static const int _vadOnsetChunks = 3; // 连续 3 chunk (~60ms) → 检测到说话
|
|
int _vadOnsetCount = 0;
|
|
|
|
// 状态标记
|
|
bool _forwardingAudio = false; // 是否正在转发音频到 WS
|
|
bool _wsConnecting = false; // WS 正在连接中（防止重复触发）
|
|
int _micChunkCount = 0; // 麦克风 chunk 计数（调试用）
|
|
|
|
// ── 播放 & TTS 队列 ──────────────────────────────────────────────────────
|
|
// 关闭 just_audio 的音频会话管理，防止它与 record 争抢音频硬件：
|
|
// - handleInterruptions: false → 不自动暂停/恢复
|
|
// - handleAudioSessionActivation: false → 不自动调用 session.setActive()
|
|
// - androidApplyAudioAttributes: false → 不监听 audio_session 配置变更
|
|
final AudioPlayer _player = AudioPlayer(
|
|
handleInterruptions: false,
|
|
handleAudioSessionActivation: false,
|
|
androidApplyAudioAttributes: false,
|
|
);
|
|
final List<String> _ttsQueue = [];
|
|
bool _ttsPlaying = false;
|
|
bool _ttsInterrupted = false;
|
|
Completer<void>? _ttsPlayCompleter; // 用于等待当前段播完或被打断
|
|
|
|
// ── 配置 ───────────────────────────────────────────────────────────────────
|
|
late String _deviceNo, _deviceId, _productId, _productKey, _deviceSecret;
|
|
final credentialReady = false.obs;
|
|
final credentialError = ''.obs;
|
|
final _storage = GetStorage();
|
|
static const _kDeviceId = 'mobile_elf_device_id';
|
|
static const _kDeviceSecret = 'mobile_elf_device_secret';
|
|
static const _kDeviceNo = 'mobile_elf_device_no';
|
|
final _uuid = const Uuid();
|
|
|
|
// 计时器
|
|
Timer? _callTimer;
|
|
Timer? _recordingTimer;
|
|
Timer? _micWatchdog;
|
|
int _lastWatchdogChunkCount = 0;
|
|
Timer? _idleTimer;
|
|
Timer? _pttTimer;
|
|
static const int _idleTimeoutSeconds = 15; // TTS 播完后等待说话超时秒数
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
// 生命周期
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
@override
|
|
void onInit() {
|
|
super.onInit();
|
|
_loadConfig();
|
|
_fetchA80CredentialsIfNeeded();
|
|
}
|
|
|
|
@override
|
|
void onClose() {
|
|
disconnect();
|
|
_recorder.dispose();
|
|
_player.dispose();
|
|
scrollController.dispose();
|
|
super.onClose();
|
|
}
|
|
|
|
void _loadConfig() {
|
|
_productId = dotenv.env['MOBILE_ELF_PRODUCT_ID'] ?? '1899007766144614402';
|
|
_productKey = dotenv.env['MOBILE_ELF_PRODUCT_KEY'] ?? 'd0aI2yidbUUXS';
|
|
_deviceNo = _storage.read(_kDeviceNo) ?? dotenv.env['MOBILE_ELF_DEVICE_NO'] ?? '';
|
|
_deviceId = _storage.read(_kDeviceId) ?? '';
|
|
_deviceSecret = _storage.read(_kDeviceSecret) ?? '';
|
|
credentialReady.value = _deviceSecret.isNotEmpty;
|
|
}
|
|
|
|
Future<void> _fetchA80CredentialsIfNeeded() async {
|
|
try {
|
|
final deviceNo = dotenv.env['MOBILE_ELF_DEVICE_NO'] ?? '';
|
|
if (deviceNo.isEmpty) return;
|
|
credentialError.value = '';
|
|
final dio = Dio(BaseOptions(
|
|
baseUrl: 'https://ivs.chinamobiledevice.com:11443',
|
|
connectTimeout: const Duration(seconds: 10),
|
|
receiveTimeout: const Duration(seconds: 10),
|
|
));
|
|
final resp = await dio.post('/v2/customer/device/secret/info', data: {
|
|
'deviceNoType': 'SN', 'deviceNo': deviceNo,
|
|
'productId': _productId, 'productKey': _productKey,
|
|
});
|
|
final body = resp.data as Map<String, dynamic>;
|
|
if (body['code'].toString() == '200' && body['success'] == true) {
|
|
final data = body['data'] as Map<String, dynamic>;
|
|
_deviceNo = data['deviceNo'] as String? ?? deviceNo;
|
|
_deviceId = data['deviceId'] as String? ?? '';
|
|
_deviceSecret = data['deviceSecret'] as String? ?? '';
|
|
await _storage.write(_kDeviceNo, _deviceNo);
|
|
await _storage.write(_kDeviceId, _deviceId);
|
|
await _storage.write(_kDeviceSecret, _deviceSecret);
|
|
credentialReady.value = _deviceSecret.isNotEmpty;
|
|
} else {
|
|
credentialError.value = body['message']?.toString() ?? '凭证请求失败';
|
|
}
|
|
} catch (e) {
|
|
credentialError.value = '凭证请求异常: $e';
|
|
}
|
|
}
|
|
|
|
Future<void> retryCredentials() async {
|
|
credentialError.value = '';
|
|
await _fetchA80CredentialsIfNeeded();
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
// 开始/结束通话
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
/// 配置音频会话：playAndRecord 模式，允许录音和播放同时进行
|
|
Future<void> _configureAudioSession() async {
|
|
try {
|
|
final session = await AudioSession.instance;
|
|
await session.configure(AudioSessionConfiguration(
|
|
avAudioSessionCategory: AVAudioSessionCategory.playAndRecord,
|
|
avAudioSessionCategoryOptions: AVAudioSessionCategoryOptions.defaultToSpeaker |
|
|
AVAudioSessionCategoryOptions.allowBluetooth,
|
|
avAudioSessionMode: AVAudioSessionMode.voiceChat,
|
|
androidAudioAttributes: const AndroidAudioAttributes(
|
|
contentType: AndroidAudioContentType.speech,
|
|
usage: AndroidAudioUsage.voiceCommunication,
|
|
),
|
|
androidAudioFocusGainType: AndroidAudioFocusGainType.gain,
|
|
));
|
|
await session.setActive(true);
|
|
Logger.info('yiding----------- [音频会话] 已配置 playAndRecord 模式');
|
|
} catch (e) {
|
|
Logger.error('yiding----------- [音频会话] 配置失败: $e');
|
|
}
|
|
}
|
|
|
|
/// 点击通话按钮 → 开麦，进入等待说话状态（不连接服务器）
|
|
Future<void> startCall() async {
|
|
if (inCall.value) return;
|
|
// PTT 模式下 AI 正在播报 → 先打断 TTS，再进入通话
|
|
if (_ttsPlaying || status.value == MobileElfStatus.speaking) {
|
|
_interruptTts();
|
|
_ttsInterrupted = false;
|
|
// 等一帧让 player.stop() 完成
|
|
await Future.delayed(const Duration(milliseconds: 50));
|
|
}
|
|
// 如果 PTT 正在录音中 → 先取消
|
|
if (pttRecording.value) {
|
|
await stopPtt(cancel: true);
|
|
}
|
|
final hasPermission = await _recorder.hasPermission();
|
|
if (!hasPermission) { _setError('无麦克风权限'); return; }
|
|
|
|
await _configureAudioSession();
|
|
// 为播放器单独设置 voiceCommunication 音频属性，与录音器使用同一音频路由
|
|
// 这样 ExoPlayer 和 AudioRecord 可以像 VoIP 应用一样同时工作
|
|
if (!kIsWeb) {
|
|
await _player.setAndroidAudioAttributes(const AndroidAudioAttributes(
|
|
contentType: AndroidAudioContentType.speech,
|
|
usage: AndroidAudioUsage.voiceCommunication,
|
|
));
|
|
}
|
|
await _openMic();
|
|
_startMicWatchdog();
|
|
inCall.value = true;
|
|
status.value = MobileElfStatus.idle; // 等待用户说话
|
|
callSeconds.value = 0;
|
|
_callTimer?.cancel();
|
|
_callTimer = Timer.periodic(const Duration(seconds: 1), (_) => callSeconds.value++);
|
|
_startIdleTimer(); // 开始等待说话超时
|
|
Logger.info('yiding----------- [通话] 已开始，等待用户说话');
|
|
}
|
|
|
|
/// 挂断
|
|
void disconnect() {
|
|
Logger.info('yiding----------- [通话] 挂断');
|
|
_stopRecordingTimer();
|
|
_stopMicWatchdog();
|
|
_cancelIdleTimer();
|
|
_callTimer?.cancel(); _callTimer = null;
|
|
_forwardingAudio = false;
|
|
_wsConnecting = false;
|
|
_speechDetected = false;
|
|
_micSubscription?.cancel(); _micSubscription = null;
|
|
_recorder.stop();
|
|
// 清空 TTS 队列 & 停播
|
|
_ttsInterrupted = true;
|
|
_ttsQueue.clear();
|
|
_ttsPlaying = false;
|
|
if (!(_ttsPlayCompleter?.isCompleted ?? true)) _ttsPlayCompleter?.complete();
|
|
_player.stop();
|
|
_closeWs();
|
|
_preBuffer.clear();
|
|
_sendBuffer.clear(); _sendBufferOffset = 0;
|
|
micRms.value = 0; micAlive.value = false;
|
|
inCall.value = false;
|
|
status.value = MobileElfStatus.idle;
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
// 麦克风 & 本地 VAD
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
Future<void> _openMic() async {
|
|
_micSubscription?.cancel();
|
|
_micSubscription = null;
|
|
try { await _recorder.stop(); } catch (_) {}
|
|
final stream = await _recorder.startStream(
|
|
const RecordConfig(
|
|
encoder: AudioEncoder.pcm16bits,
|
|
sampleRate: 16000,
|
|
numChannels: 1,
|
|
echoCancel: true,
|
|
noiseSuppress: true,
|
|
androidConfig: AndroidRecordConfig(
|
|
audioSource: AndroidAudioSource.voiceCommunication,
|
|
),
|
|
),
|
|
);
|
|
_speechDetected = false;
|
|
_vadOnsetCount = 0;
|
|
_micChunkCount = 0;
|
|
micAlive.value = false;
|
|
_micSubscription = stream.listen(_onMicAudio);
|
|
Logger.info('yiding----------- [麦克风] 已开启 (voiceCommunication + echoCancel)');
|
|
}
|
|
|
|
/// 启动麦克风看门狗：每 1.5 秒检测一次，如果无新 chunk 则标记 micAlive=false
|
|
void _startMicWatchdog() {
|
|
_micWatchdog?.cancel();
|
|
_lastWatchdogChunkCount = _micChunkCount;
|
|
_micWatchdog = Timer.periodic(const Duration(milliseconds: 1500), (_) {
|
|
if (!inCall.value && !pttRecording.value) { _micWatchdog?.cancel(); return; }
|
|
if (_micChunkCount == _lastWatchdogChunkCount) {
|
|
micAlive.value = false;
|
|
if (status.value == MobileElfStatus.speaking || _ttsPlaying) {
|
|
// TTS 播放中麦克风已关闭，不需要重启
|
|
} else {
|
|
Logger.info('yiding----------- [看门狗] 麦克风无数据! 尝试重启');
|
|
_openMic();
|
|
}
|
|
}
|
|
_lastWatchdogChunkCount = _micChunkCount;
|
|
});
|
|
}
|
|
|
|
void _stopMicWatchdog() {
|
|
_micWatchdog?.cancel();
|
|
_micWatchdog = null;
|
|
}
|
|
|
|
void _onMicAudio(Uint8List chunk) {
|
|
if (!inCall.value && !pttRecording.value) return;
|
|
_micChunkCount++;
|
|
|
|
// 计算 RMS（PCM 16-bit little-endian）
|
|
final rms = _calcRms(chunk);
|
|
|
|
// 更新 UI 可观察值
|
|
micAlive.value = true;
|
|
micRms.value = rms;
|
|
|
|
// 每 100 个 chunk (~2s) 输出一次调试日志，确认麦克风存活
|
|
if (_micChunkCount % 100 == 0) {
|
|
Logger.info('yiding----------- [麦克风] chunk#$_micChunkCount rms=${rms.toStringAsFixed(0)} '
|
|
'status=${status.value} speechDetected=$_speechDetected fwd=$_forwardingAudio wsConn=$_wsConnecting');
|
|
}
|
|
|
|
// ── 当前正在转发音频到 WS ──
|
|
if (_forwardingAudio && _channel != null) {
|
|
_feedSendBuffer(chunk);
|
|
return;
|
|
}
|
|
|
|
// ── WS 正在连接中，先存入预缓存 ──
|
|
if (_wsConnecting) {
|
|
_addToPreBuffer(chunk);
|
|
return;
|
|
}
|
|
|
|
// ── TTS 播放中 / thinking 中不做 VAD ──
|
|
if (status.value == MobileElfStatus.speaking || _ttsPlaying ||
|
|
status.value == MobileElfStatus.thinking) {
|
|
return;
|
|
}
|
|
|
|
// ── 空闲/等待状态：连续 chunk VAD ──
|
|
_addToPreBuffer(chunk);
|
|
|
|
if (rms >= _vadThreshold) {
|
|
_vadOnsetCount++;
|
|
if (_vadOnsetCount >= _vadOnsetChunks && !_speechDetected) {
|
|
_speechDetected = true;
|
|
_cancelIdleTimer();
|
|
Logger.info('yiding----------- [VAD] 检测到说话 rms=${rms.toStringAsFixed(0)} '
|
|
'onset=$_vadOnsetCount/$_vadOnsetChunks，启动连接');
|
|
_onSpeechDetected();
|
|
}
|
|
} else {
|
|
_vadOnsetCount = 0;
|
|
}
|
|
}
|
|
|
|
double _calcRms(Uint8List pcm) {
|
|
if (pcm.length < 2) return 0;
|
|
final samples = pcm.length ~/ 2;
|
|
double sum = 0;
|
|
for (int i = 0; i < pcm.length - 1; i += 2) {
|
|
int sample = pcm[i] | (pcm[i + 1] << 8);
|
|
if (sample >= 32768) sample -= 65536; // signed
|
|
sum += sample * sample;
|
|
}
|
|
return math.sqrt(sum / samples);
|
|
}
|
|
|
|
void _addToPreBuffer(Uint8List chunk) {
|
|
_preBuffer.add(Uint8List.fromList(chunk));
|
|
while (_preBuffer.length > _preBufferMaxChunks) {
|
|
_preBuffer.removeAt(0);
|
|
}
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
// 检测到说话 → 连接服务器 → 发送预缓存 → 实时转发
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
Future<void> _onSpeechDetected() async {
|
|
if (_wsConnecting) return;
|
|
|
|
_ttsInterrupted = false;
|
|
_closeWs();
|
|
_sessionEnded = false;
|
|
_wsConnecting = true;
|
|
|
|
status.value = MobileElfStatus.connecting;
|
|
_dialogRequestId = _uuid.v4();
|
|
_currentAiMsgIndex = -1;
|
|
realtimeText.value = '';
|
|
_sendBuffer.clear();
|
|
_sendBufferOffset = 0;
|
|
errorMsg.value = '';
|
|
|
|
Logger.info('yiding----------- [新轮] 说话触发连接 dialogId=$_dialogRequestId preBuffer=${_preBuffer.length}chunks');
|
|
|
|
try {
|
|
await _connect();
|
|
Logger.info('yiding----------- [新轮] WS ready，开始发送预缓存');
|
|
} catch (e) {
|
|
Logger.error('yiding----------- [新轮] 连接失败: $e');
|
|
_wsConnecting = false;
|
|
_speechDetected = false;
|
|
_setError('连接失败: $e');
|
|
return;
|
|
}
|
|
|
|
// 发送预缓存音频
|
|
_forwardingAudio = true;
|
|
_wsConnecting = false;
|
|
status.value = MobileElfStatus.recording;
|
|
recordingSeconds.value = 0;
|
|
_stopRecordingTimer();
|
|
_recordingTimer = Timer.periodic(const Duration(seconds: 1), (_) => recordingSeconds.value++);
|
|
|
|
final cached = List<Uint8List>.from(_preBuffer);
|
|
_preBuffer.clear();
|
|
int cachedBytes = 0;
|
|
for (final c in cached) {
|
|
_feedSendBuffer(c);
|
|
cachedBytes += c.length;
|
|
}
|
|
Logger.info('yiding----------- [新轮] 预缓存已发送 ${cached.length}chunks ${cachedBytes}bytes，实时转发中');
|
|
}
|
|
|
|
/// 将 chunk 加入发送缓冲，凑满 _frameSize 就发
|
|
void _feedSendBuffer(Uint8List chunk) {
|
|
_sendBuffer.addAll(chunk);
|
|
while (_sendBuffer.length - _sendBufferOffset >= _frameSize) {
|
|
final frame = Uint8List.fromList(
|
|
_sendBuffer.sublist(_sendBufferOffset, _sendBufferOffset + _frameSize));
|
|
_sendBufferOffset += _frameSize;
|
|
_sendAudioChunk(frame);
|
|
}
|
|
// 紧缩
|
|
if (_sendBufferOffset >= _frameSize * 8) {
|
|
_sendBuffer.removeRange(0, _sendBufferOffset);
|
|
_sendBufferOffset = 0;
|
|
}
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
// WebSocket 连接
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
Future<void> _connect() async {
|
|
_readyCompleter = Completer<void>();
|
|
try {
|
|
final ts = DateTime.now().millisecondsSinceEpoch.toString();
|
|
final sn = _uuid.v4();
|
|
final sign = _calcSign(_deviceSecret, ts);
|
|
final url = 'wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr'
|
|
'?deviceNo=$_deviceNo&deviceId=$_deviceId&productId=$_productId'
|
|
'&productKey=$_productKey&sign=$sign&sn=$sn&ts=$ts';
|
|
|
|
if (kIsWeb) {
|
|
_channel = WebSocketChannel.connect(Uri.parse(url));
|
|
} else {
|
|
final httpClient = HttpClient()..badCertificateCallback = (_, __, ___) => true;
|
|
final ws = await WebSocket.connect(url, customClient: httpClient, headers: {
|
|
'Origin': 'https://ivs.chinamobiledevice.com:11443',
|
|
'Host': 'ivs.chinamobiledevice.com:11443',
|
|
});
|
|
_channel = IOWebSocketChannel(ws);
|
|
}
|
|
|
|
_wsSubscription = _channel!.stream.listen(
|
|
_handleMessage,
|
|
onError: (e) {
|
|
Logger.error('yiding----------- [WS] 错误: $e');
|
|
if (!(_readyCompleter?.isCompleted ?? true)) _readyCompleter?.completeError(e);
|
|
_handleWsDone();
|
|
},
|
|
onDone: () {
|
|
Logger.info('yiding----------- [WS] onDone sessionEnded=$_sessionEnded status=${status.value}');
|
|
if (!(_readyCompleter?.isCompleted ?? true)) _readyCompleter?.completeError('WS closed');
|
|
_handleWsDone();
|
|
},
|
|
);
|
|
|
|
_sendStartFrame();
|
|
await _readyCompleter!.future.timeout(
|
|
const Duration(seconds: 15),
|
|
onTimeout: () => throw TimeoutException('等待 ready 超时'),
|
|
);
|
|
} catch (e) {
|
|
_closeWs();
|
|
rethrow;
|
|
}
|
|
}
|
|
|
|
void _handleWsDone() {
|
|
_channel = null;
|
|
_wsSubscription = null;
|
|
_forwardingAudio = false;
|
|
_wsConnecting = false;
|
|
|
|
if (_sessionEnded && inCall.value) {
|
|
Logger.info('yiding----------- [WS] 正常关闭 (通话模式)');
|
|
if (status.value == MobileElfStatus.speaking) {
|
|
// TTS 播放中，等 playTts 完成后自动回到 idle 等待说话
|
|
} else {
|
|
status.value = MobileElfStatus.idle;
|
|
_resetVad();
|
|
Logger.info('yiding----------- [WS] 回到等待说话');
|
|
}
|
|
} else if (_sessionEnded && !inCall.value) {
|
|
// PTT/文本模式：WS 正常关闭
|
|
Logger.info('yiding----------- [WS] 正常关闭 (PTT/文本模式)');
|
|
if (status.value != MobileElfStatus.speaking) {
|
|
status.value = MobileElfStatus.idle;
|
|
}
|
|
} else if (inCall.value) {
|
|
Logger.info('yiding----------- [WS] 非预期关闭，回到等待说话');
|
|
status.value = MobileElfStatus.idle;
|
|
_resetVad();
|
|
} else {
|
|
Logger.info('yiding----------- [WS] 非预期关闭');
|
|
status.value = MobileElfStatus.idle;
|
|
}
|
|
}
|
|
|
|
void _resetVad() {
|
|
_speechDetected = false;
|
|
_vadOnsetCount = 0;
|
|
_preBuffer.clear();
|
|
_openMic();
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
// 空闲超时：TTS 播完后等待说话，超时自动挂断
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
void _startIdleTimer() {
|
|
_cancelIdleTimer();
|
|
_idleTimer = Timer(Duration(seconds: _idleTimeoutSeconds), () {
|
|
if (inCall.value && status.value == MobileElfStatus.idle && !_speechDetected) {
|
|
Logger.info('yiding----------- [超时] ${_idleTimeoutSeconds}秒无说话，自动挂断');
|
|
disconnect();
|
|
}
|
|
});
|
|
Logger.info('yiding----------- [超时] 开始${_idleTimeoutSeconds}秒等待说话计时');
|
|
}
|
|
|
|
void _cancelIdleTimer() {
|
|
_idleTimer?.cancel();
|
|
_idleTimer = null;
|
|
}
|
|
|
|
void _closeWs() {
|
|
final sub = _wsSubscription; _wsSubscription = null;
|
|
sub?.cancel();
|
|
final ch = _channel; _channel = null;
|
|
try { ch?.sink.close(); } catch (_) {}
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
// 发送帧
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
void _sendStartFrame() {
|
|
if (_channel == null) return;
|
|
_channel!.sink.add(jsonEncode({
|
|
'type': 'start',
|
|
'data': {
|
|
'cuid': _deviceId,
|
|
'format': 'pcm',
|
|
'sample': 16000,
|
|
'support_dcs': 2,
|
|
'chunk_size': 10240,
|
|
'support_tts': true,
|
|
'support_text2dcs': true,
|
|
'user_agent': 'voitrans/1.0',
|
|
'dialog_request_id': _dialogRequestId,
|
|
'access_rc': true,
|
|
'support_part_tts': true,
|
|
'need_stoplisten': true,
|
|
'need_dialogue_finish': true,
|
|
'result_trans2directive': false,
|
|
'rc_version': '5',
|
|
'messages': _buildContextMessages(),
|
|
'client_context': [],
|
|
},
|
|
}));
|
|
}
|
|
|
|
void _sendAudioChunk(Uint8List data) {
|
|
try { _channel?.sink.add(data); } catch (_) {}
|
|
}
|
|
|
|
void _sendFinish() {
|
|
try { _channel?.sink.add(jsonEncode({'type': 'finish'})); } catch (_) {}
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
// 接收消息处理
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
void _handleMessage(dynamic raw) {
|
|
try {
|
|
if (raw is! String) return;
|
|
final msg = jsonDecode(raw) as Map<String, dynamic>;
|
|
final type = msg['type'] as String? ?? '';
|
|
|
|
switch (type) {
|
|
case 'ready':
|
|
if (!(_readyCompleter?.isCompleted ?? true)) _readyCompleter?.complete();
|
|
break;
|
|
|
|
case 'stoplisten':
|
|
Logger.info('yiding----------- [服务端] stoplisten');
|
|
_stopRecordingTimer();
|
|
_forwardingAudio = false;
|
|
_speechDetected = false; // 允许 VAD 在 thinking/speaking 期间检测新语音
|
|
_vadOnsetCount = 0;
|
|
// 发送剩余
|
|
final remain = _sendBuffer.length - _sendBufferOffset;
|
|
if (remain > 0) _sendAudioChunk(Uint8List.fromList(_sendBuffer.sublist(_sendBufferOffset)));
|
|
_sendBuffer.clear(); _sendBufferOffset = 0;
|
|
_sendFinish();
|
|
status.value = MobileElfStatus.thinking;
|
|
break;
|
|
|
|
case 'mid_result':
|
|
realtimeText.value = msg['result'] as String? ?? '';
|
|
break;
|
|
|
|
case 'fin_result':
|
|
final result = msg['result'] as String? ?? '';
|
|
if (result.isNotEmpty) {
|
|
realtimeText.value = '';
|
|
_addUserMessage(result);
|
|
}
|
|
if (msg['end'] == 1) _markSessionEnded();
|
|
break;
|
|
|
|
case 'inside_rc':
|
|
_handleInsideRc(msg);
|
|
break;
|
|
|
|
case 'dcs_decide':
|
|
if (msg['end'] == 1) {
|
|
Logger.info('yiding----------- [服务端] dcs_decide end:1');
|
|
_markSessionEnded();
|
|
}
|
|
break;
|
|
}
|
|
} catch (e) {
|
|
Logger.error('[MobileElf] 处理消息失败: $e');
|
|
}
|
|
}
|
|
|
|
void _handleInsideRc(Map<String, dynamic> msg) {
|
|
final data = msg['data'] as Map<String, dynamic>?;
|
|
if (data == null) return;
|
|
if ((data['code'] as int? ?? -1) != 0) return;
|
|
|
|
final assistantAnswer = data['assistant_answer'] as String? ?? '';
|
|
String cleanAnswer = assistantAnswer;
|
|
try {
|
|
cleanAnswer = (jsonDecode(assistantAnswer) as Map<String, dynamic>)['content'] as String? ?? assistantAnswer;
|
|
} catch (_) {}
|
|
|
|
String? ttsUrl;
|
|
for (final item in (data['data'] as List<dynamic>? ?? [])) {
|
|
final d = item as Map<String, dynamic>;
|
|
final h = d['header'] as Map<String, dynamic>?;
|
|
if (h?['namespace'] == 'ai.fxzsos.device_interface.voice_output' && h?['name'] == 'Speak') {
|
|
ttsUrl = (d['payload'] as Map<String, dynamic>?)?['url'] as String?;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (cleanAnswer.isNotEmpty) {
|
|
if (_currentAiMsgIndex >= 0 && _currentAiMsgIndex < messages.length) {
|
|
messages[_currentAiMsgIndex] = MobileElfMessage(
|
|
isUser: false, text: cleanAnswer,
|
|
time: messages[_currentAiMsgIndex].time,
|
|
ttsUrl: ttsUrl ?? messages[_currentAiMsgIndex].ttsUrl,
|
|
);
|
|
} else {
|
|
_currentAiMsgIndex = messages.length;
|
|
_addAiMessage(cleanAnswer, ttsUrl: ttsUrl);
|
|
}
|
|
}
|
|
|
|
if (ttsUrl != null && ttsUrl.isNotEmpty) _enqueueTts(ttsUrl);
|
|
if (msg['end'] == 1) _markSessionEnded();
|
|
}
|
|
|
|
void _markSessionEnded() {
|
|
if (_sessionEnded) return;
|
|
_sessionEnded = true;
|
|
Logger.info('yiding----------- [会话] 本轮结束 status=${status.value}');
|
|
_stopRecordingTimer();
|
|
_forwardingAudio = false;
|
|
_speechDetected = false; // 允许 VAD 在 TTS 播放期间检测新语音
|
|
_vadOnsetCount = 0;
|
|
_sendBuffer.clear(); _sendBufferOffset = 0;
|
|
realtimeText.value = '';
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
// TTS 播放队列
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
/// 将 TTS URL 加入队列，如果没有正在播放则关闭麦克风并开始播放
|
|
void _enqueueTts(String url) {
|
|
_ttsQueue.add(url);
|
|
Logger.info('yiding----------- [TTS] 入队 queue=${_ttsQueue.length}');
|
|
if (!_ttsPlaying) {
|
|
// 关闭麦克风，避免音频冲突，TTS 播完后会重开
|
|
_micSubscription?.cancel(); _micSubscription = null;
|
|
try { _recorder.stop(); } catch (_) {}
|
|
micAlive.value = false; micRms.value = 0;
|
|
Logger.info('yiding----------- [TTS] 关闭麦克风，开始播放');
|
|
_playNextTts();
|
|
}
|
|
}
|
|
|
|
/// 顺序播放队列中的 TTS
|
|
Future<void> _playNextTts() async {
|
|
if (_ttsQueue.isEmpty || _ttsInterrupted) {
|
|
_ttsPlaying = false;
|
|
// 队列播完 → 重开麦克风，回到等待说话，启动超时计时
|
|
if (!_ttsInterrupted && status.value == MobileElfStatus.speaking) {
|
|
status.value = MobileElfStatus.idle;
|
|
Logger.info('yiding----------- [TTS] 队列全部播完，重开麦克风，等待新一轮说话');
|
|
if (inCall.value) {
|
|
_resetVad();
|
|
_startIdleTimer();
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
|
|
_ttsPlaying = true;
|
|
status.value = MobileElfStatus.speaking;
|
|
final url = _ttsQueue.removeAt(0);
|
|
|
|
// 用 Completer 等待播放完成或被打断
|
|
_ttsPlayCompleter = Completer<void>();
|
|
StreamSubscription? stateSub;
|
|
|
|
try {
|
|
await _player.stop();
|
|
// 不再重新配置音频会话！startCall() 已配置好 playAndRecord 模式，
|
|
// 重复配置会导致 audio_session 重新请求音频焦点，杀死录音流
|
|
await _player.setUrl(url);
|
|
|
|
// 监听 ProcessingState: completed=正常播完, idle=被打断或意外停止
|
|
stateSub = _player.processingStateStream.listen((s) {
|
|
if ((s == ProcessingState.completed || s == ProcessingState.idle) &&
|
|
!(_ttsPlayCompleter?.isCompleted ?? true)) {
|
|
Logger.info('yiding----------- [TTS] 当前段结束 state=$s');
|
|
_ttsPlayCompleter?.complete();
|
|
}
|
|
});
|
|
|
|
await _player.play();
|
|
Logger.info('yiding----------- [TTS] 开始播放 剩余队列=${_ttsQueue.length} micAlive=${micAlive.value} micChunk=$_micChunkCount');
|
|
|
|
// 等待：正常播完 complete() 或被打断 _interruptTts() 也会 complete()
|
|
await _ttsPlayCompleter!.future.timeout(const Duration(seconds: 60));
|
|
} catch (e) {
|
|
Logger.error('yiding----------- [TTS] 播放异常: $e');
|
|
} finally {
|
|
stateSub?.cancel();
|
|
}
|
|
|
|
// 当前段结束，继续下一段（如果没被打断）
|
|
if (!_ttsInterrupted) {
|
|
_playNextTts();
|
|
}
|
|
}
|
|
|
|
/// 立即打断 TTS：停播 + 清队列 + 解除等待
|
|
void _interruptTts() {
|
|
Logger.info('yiding----------- [TTS] 打断! 清空队列=${_ttsQueue.length}');
|
|
_ttsInterrupted = true;
|
|
_ttsQueue.clear();
|
|
_ttsPlaying = false;
|
|
// 先解除 _playNextTts 的等待
|
|
if (!(_ttsPlayCompleter?.isCompleted ?? true)) {
|
|
_ttsPlayCompleter?.complete();
|
|
}
|
|
_player.stop();
|
|
}
|
|
|
|
/// 手动打断 AI 播报（界面按钮调用）
|
|
Future<void> interruptSpeaking() async {
|
|
_interruptTts();
|
|
_ttsInterrupted = false; // 手动打断后重置，不影响后续
|
|
status.value = MobileElfStatus.idle;
|
|
if (inCall.value) _resetVad();
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
// 手动停止录音
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
Future<void> stopRecording() async {
|
|
if (!_forwardingAudio) return;
|
|
_stopRecordingTimer();
|
|
_forwardingAudio = false;
|
|
final remain = _sendBuffer.length - _sendBufferOffset;
|
|
if (remain > 0) _sendAudioChunk(Uint8List.fromList(_sendBuffer.sublist(_sendBufferOffset)));
|
|
_sendBuffer.clear(); _sendBufferOffset = 0;
|
|
_sendFinish();
|
|
status.value = MobileElfStatus.thinking;
|
|
}
|
|
|
|
void _stopRecordingTimer() {
|
|
_recordingTimer?.cancel(); _recordingTimer = null;
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
// 按住说话 (PTT)
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
/// 按住说话 → 开始录音并连接 WS
|
|
Future<void> startPtt() async {
|
|
if (pttRecording.value) return;
|
|
// 如果正在播放 AI 回复 → 立即打断
|
|
if (_ttsPlaying || status.value == MobileElfStatus.speaking) {
|
|
_interruptTts();
|
|
_ttsInterrupted = false;
|
|
}
|
|
final hasPermission = await _recorder.hasPermission();
|
|
if (!hasPermission) { _setError('无麦克风权限'); return; }
|
|
|
|
// 先设置 connecting 状态，再设置 pttRecording，避免中间按钮橙色闪烁
|
|
status.value = MobileElfStatus.connecting;
|
|
pttRecording.value = true;
|
|
pttSeconds.value = 0;
|
|
_pttTimer = Timer.periodic(const Duration(seconds: 1), (_) => pttSeconds.value++);
|
|
|
|
await _configureAudioSession();
|
|
if (!kIsWeb) {
|
|
await _player.setAndroidAudioAttributes(const AndroidAudioAttributes(
|
|
contentType: AndroidAudioContentType.speech,
|
|
usage: AndroidAudioUsage.voiceCommunication,
|
|
));
|
|
}
|
|
await _openMic();
|
|
_startMicWatchdog();
|
|
|
|
_ttsInterrupted = false;
|
|
_closeWs();
|
|
_sessionEnded = false;
|
|
_wsConnecting = true;
|
|
_dialogRequestId = _uuid.v4();
|
|
_currentAiMsgIndex = -1;
|
|
realtimeText.value = '';
|
|
_sendBuffer.clear(); _sendBufferOffset = 0;
|
|
errorMsg.value = '';
|
|
|
|
try {
|
|
await _connect();
|
|
_forwardingAudio = true;
|
|
_wsConnecting = false;
|
|
status.value = MobileElfStatus.recording;
|
|
Logger.info('yiding----------- [PTT] 录音开始');
|
|
} catch (e) {
|
|
_wsConnecting = false;
|
|
pttRecording.value = false;
|
|
_pttTimer?.cancel(); _pttTimer = null;
|
|
_stopMicWatchdog();
|
|
_setError('连接失败: $e');
|
|
}
|
|
}
|
|
|
|
/// 松开 → 停止录音，发送 finish
|
|
Future<void> stopPtt({bool cancel = false}) async {
|
|
pttRecording.value = false;
|
|
_pttTimer?.cancel(); _pttTimer = null;
|
|
_stopMicWatchdog();
|
|
|
|
if (cancel) {
|
|
_forwardingAudio = false;
|
|
_closeWs();
|
|
_micSubscription?.cancel(); _micSubscription = null;
|
|
try { _recorder.stop(); } catch (_) {}
|
|
micAlive.value = false; micRms.value = 0;
|
|
realtimeText.value = '';
|
|
status.value = MobileElfStatus.idle;
|
|
Logger.info('yiding----------- [PTT] 已取消');
|
|
return;
|
|
}
|
|
|
|
// 正常结束：发送剩余音频 + finish
|
|
if (_forwardingAudio) {
|
|
_forwardingAudio = false;
|
|
final remain = _sendBuffer.length - _sendBufferOffset;
|
|
if (remain > 0) _sendAudioChunk(Uint8List.fromList(_sendBuffer.sublist(_sendBufferOffset)));
|
|
_sendBuffer.clear(); _sendBufferOffset = 0;
|
|
_sendFinish();
|
|
status.value = MobileElfStatus.thinking;
|
|
}
|
|
_micSubscription?.cancel(); _micSubscription = null;
|
|
try { _recorder.stop(); } catch (_) {}
|
|
micAlive.value = false; micRms.value = 0;
|
|
Logger.info('yiding----------- [PTT] 录音结束，等待AI回复');
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
// 按钮操作
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
Future<void> onMicTap() async {
|
|
Logger.info('yiding----------- [按钮] status=${status.value} inCall=${inCall.value}');
|
|
switch (status.value) {
|
|
case MobileElfStatus.idle:
|
|
case MobileElfStatus.error:
|
|
if (!inCall.value) {
|
|
await startCall();
|
|
}
|
|
break;
|
|
case MobileElfStatus.recording:
|
|
await stopRecording();
|
|
break;
|
|
case MobileElfStatus.speaking:
|
|
await interruptSpeaking();
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
// 工具方法
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
String _calcSign(String secret, String ts) => md5.convert(utf8.encode('$secret$ts')).toString();
|
|
|
|
List<Map<String, String>> _buildContextMessages() =>
|
|
messages.map((m) => {'role': m.isUser ? 'user' : 'assistant', 'content': m.text}).toList();
|
|
|
|
void _addUserMessage(String text) {
|
|
if (messages.isNotEmpty && messages.last.isUser && messages.last.text == text) return;
|
|
messages.add(MobileElfMessage(isUser: true, text: text, time: DateTime.now()));
|
|
scrollToBottom();
|
|
}
|
|
|
|
void _addAiMessage(String text, {String? ttsUrl}) {
|
|
messages.add(MobileElfMessage(isUser: false, text: text, time: DateTime.now(), ttsUrl: ttsUrl));
|
|
scrollToBottom();
|
|
}
|
|
|
|
void _setError(String msg) {
|
|
errorMsg.value = msg;
|
|
status.value = MobileElfStatus.error;
|
|
}
|
|
|
|
void scrollToBottom() {
|
|
if (scrollController.hasClients) {
|
|
WidgetsBinding.instance.addPostFrameCallback((_) {
|
|
if (scrollController.hasClients) {
|
|
scrollController.animateTo(scrollController.position.maxScrollExtent,
|
|
duration: const Duration(milliseconds: 200), curve: Curves.easeOut);
|
|
}
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|