From 6902972aa2880115f9406cc27aabbdeaa8eea2a3 Mon Sep 17 00:00:00 2001 From: wolfplus Date: Mon, 9 Jun 2025 22:23:30 +0100 Subject: [PATCH] add --- VOLCANO_VOICE_README.md | 195 --------- .../controllers/explore_controller.dart | 2 +- lib/modules/realtime/README.md | 136 ++++++ .../realtime/bindings/realtime_binding.dart | 10 + .../controllers/realtime_controller.dart | 409 ++++++++++++++++++ lib/modules/realtime/views/realtime_view.dart | 240 ++++++++++ .../views/widgets/voice_mic_flow.dart | 89 ++++ .../controllers/settings_controller.dart | 2 +- .../settings/views/permissions_view.dart | 8 +- lib/modules/settings/views/settings_view.dart | 34 +- lib/routes/app_pages.dart | 7 + lib/routes/app_routes.dart | 9 +- local_plugins/realtime/.gitignore | 51 +++ local_plugins/realtime/IMPLEMENTATION.md | 190 ++++++++ local_plugins/realtime/README.md | 125 ++++++ .../realtime/android/build.gradle.kts | 47 ++ .../realtime/android/settings.gradle.kts | 1 + .../android/src/main/AndroidManifest.xml | 11 + .../realtime/RealtimeAudioManager.kt | 355 +++++++++++++++ .../realtime/RealtimePlugin.kt | 309 +++++++++++++ .../realtime/RealtimeWebSocketManager.kt | 218 ++++++++++ local_plugins/realtime/example.md | 274 ++++++++++++ .../realtime/ios/Classes/RealtimePlugin.h | 4 + .../realtime/ios/Classes/RealtimePlugin.m | 8 + local_plugins/realtime/ios/realtime.podspec | 20 + .../realtime/ios/realtime/Package.swift | 18 + .../realtime/RealtimeAudioManager.swift | 314 ++++++++++++++ .../Sources/realtime/RealtimePlugin.swift | 274 ++++++++++++ .../realtime/RealtimeWebSocketManager.swift | 264 +++++++++++ local_plugins/realtime/lib/realtime.dart | 243 +++++++++++ local_plugins/realtime/pubspec.yaml | 29 ++ pubspec.yaml | 16 +- shaders/mic_flow.frag | 70 +++ shaders/mic_flow2.frag | 67 +++ 34 files changed, 3837 insertions(+), 212 deletions(-) delete mode 100644 VOLCANO_VOICE_README.md create mode 100644 lib/modules/realtime/README.md create mode 100644 lib/modules/realtime/bindings/realtime_binding.dart create mode 100644 lib/modules/realtime/controllers/realtime_controller.dart create mode 100644 lib/modules/realtime/views/realtime_view.dart create mode 100644 lib/modules/realtime/views/widgets/voice_mic_flow.dart create mode 100644 local_plugins/realtime/.gitignore create mode 100644 local_plugins/realtime/IMPLEMENTATION.md create mode 100644 local_plugins/realtime/README.md create mode 100644 local_plugins/realtime/android/build.gradle.kts create mode 100644 local_plugins/realtime/android/settings.gradle.kts create mode 100644 local_plugins/realtime/android/src/main/AndroidManifest.xml create mode 100644 local_plugins/realtime/android/src/main/kotlin/com/yunqiinnovation/realtime/RealtimeAudioManager.kt create mode 100644 local_plugins/realtime/android/src/main/kotlin/com/yunqiinnovation/realtime/RealtimePlugin.kt create mode 100644 local_plugins/realtime/android/src/main/kotlin/com/yunqiinnovation/realtime/RealtimeWebSocketManager.kt create mode 100644 local_plugins/realtime/example.md create mode 100644 local_plugins/realtime/ios/Classes/RealtimePlugin.h create mode 100644 local_plugins/realtime/ios/Classes/RealtimePlugin.m create mode 100644 local_plugins/realtime/ios/realtime.podspec create mode 100644 local_plugins/realtime/ios/realtime/Package.swift create mode 100644 local_plugins/realtime/ios/realtime/Sources/realtime/RealtimeAudioManager.swift create mode 100644 local_plugins/realtime/ios/realtime/Sources/realtime/RealtimePlugin.swift create mode 100644 local_plugins/realtime/ios/realtime/Sources/realtime/RealtimeWebSocketManager.swift create mode 100644 local_plugins/realtime/lib/realtime.dart create mode 100644 local_plugins/realtime/pubspec.yaml create mode 100644 shaders/mic_flow.frag create mode 100644 shaders/mic_flow2.frag diff --git a/VOLCANO_VOICE_README.md b/VOLCANO_VOICE_README.md deleted file mode 100644 index 4fe8c2122..000000000 --- a/VOLCANO_VOICE_README.md +++ /dev/null @@ -1,195 +0,0 @@ -# 火山语音服务配置指南 - -本文档提供了关于火山语音服务的配置和常见问题解决方案。 - -## 环境变量配置 - -火山语音服务需要以下环境变量： - -``` -# 火山语音服务配置 -VOLCANO_APP_ID=your_volcano_app_id_here # 应用ID -VOLCANO_APP_KEY=your_volcano_app_key_here # 应用密钥/Token -VOLCANO_CLUSTER=your_volcano_cluster_here # 集群区域，例如: cn-beijing - -# 语音合成配置 -VOLCANO_VOICE_TYPE=zh_female_wanqudashu_moon_bigtts # 默认语音类型 - 湾区大叔 -``` - -请确保在 `.env` 文件中正确设置这些变量。 - -## 常见问题解决 - -### 1. TTS资源授权错误 - -如果遇到以下错误： - -``` -语音类型授权错误: 您可能没有权限使用当前选择的语音类型 -``` - -**解决方案**： -- 尝试使用基础语音类型，如 `zh_male_qingse_common` 或 `zh_female_qingse_common` -- 确保您的火山引擎账户已开通语音合成服务 -- 检查应用ID和密钥是否正确 - -### 2. 语音识别认证错误 - -如果遇到以下错误： - -``` -authentication signature from request: 'Authorization' header: invalid auth token -``` - -**解决方案**： - -#### 标准语音识别SDK (API v2) -- 确保 `VOLCANO_APP_KEY` 格式正确，这是一个完整的令牌 -- **必须**在Token前添加 `Bearer;` 前缀（注意使用分号而非空格） -- 检查应用ID和密钥是否匹配 -- 确保您的火山引擎账户已开通语音识别服务 -- 使用正确的API路径: `/api/v2/asr` - -#### 大模型流式识别SDK (API v3) -- 使用正确的API路径: `/api/v3/sauc/bigmodel` -- **不要**在Token前添加Bearer前缀 -- 设置正确的资源ID (`RESOURCE_ID_STRING`) -- 设置协议类型为 `PROTOCOL_TYPE_SEED` -- 确保您的火山引擎账户已开通大模型流式语音识别服务 - -### 3. WebSocket连接错误 - -如果遇到以下错误： - -``` -Error during WebSocket handshake: Unexpected response code: 400 -``` - -**解决方案**： -- 确保网络连接正常，可以访问 `openspeech.bytedance.com` -- **集群区域设置非常重要**，必须设置正确的 `VOLCANO_CLUSTER` 环境变量 -- 默认使用 `cn-beijing`，但您的账户可能需要使用其他区域，如 `cn-shanghai` 或 `cn-guangzhou` -- 如果使用默认区域出现错误，请尝试切换到其他区域 -- 确保您的账户已开通相应的语音识别服务 -- 检查请求参数格式是否正确 -- 对于标准语音识别SDK (API v2)，确保Token前添加了 `Bearer;` 前缀 -- 如果问题仍然存在，请联系火山引擎技术支持 - -### 4. 检查配置工具 - -我们提供了两个工具来检查火山语音服务的配置： - -1. **检查TTS配置**： - ``` - flutter run lib/tools/check_volcano_config.dart - ``` - -2. **检查语音识别配置**： - ``` - flutter run lib/tools/check_volcano_asr_config.dart - ``` - -这些工具将帮助您验证环境变量、网络连接和服务授权是否正确。 - -## 离线TTS支持 - -我们的应用支持离线TTS功能，当在线TTS失败时会自动切换到离线模式。离线模式支持基础语音类型： -- `zh_male_qingse_common`（基础男声） -- `zh_female_qingse_common`（基础女声） - -要使用离线TTS，您可以： -1. 在TTS测试页面选择基础语音类型 -2. 当在线合成失败时，系统会自动尝试使用离线合成 - -## 标准语音识别SDK配置 (API v2) - -标准语音识别SDK是火山语音服务的基础版本，配置相对简单。 - -### 关键配置点 -1. **API路径**：使用 `/api/v2/asr` -2. **认证方式**： - - 必须在Token前添加 `Bearer;` 前缀（注意使用分号而非空格） - - 不需要设置资源ID -3. **集群区域**：确保设置正确的集群区域，如 `cn-beijing` - - 集群区域必须与您的账户配置匹配 - - 如果遇到WebSocket握手错误，尝试切换到其他区域 - -### 配置示例 -```kotlin -// 设置API路径 -engine.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_ASR_URI_STRING, "/api/v2/asr"); - -// 设置认证信息 -engine.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_APP_ID_STRING, "YOUR_APP_ID"); -engine.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_APP_TOKEN_STRING, "Bearer;YOUR_APP_KEY"); // 必须添加Bearer;前缀 - -// 设置集群区域 -engine.setOptionString(engineHandler, SpeechEngineDefines.PARAMS_KEY_ASR_CLUSTER_STRING, "cn-beijing"); -``` - -## 大模型流式识别SDK配置 (API v3) - -从2024年2月26日起，火山语音服务提供了新的大模型流式识别SDK。如果您使用的是这个新版本，请注意以下配置差异： - -### 版本信息 -- Android: `com.bytedance.speechengine:speechengine_asr_tob:1.1.7` -- iOS: `pod 'SpeechEngineAsrToB', '1.1.7'` - -### 关键配置差异 -1. **API路径**：使用 `/api/v3/sauc/bigmodel` 而非旧版的 `/api/v2/asr` -2. **认证方式**： - - 不需要在Token前添加 `Bearer` 前缀 - - 需要设置资源ID (`RESOURCE_ID_STRING`) -3. **协议类型**：需要设置为 `PROTOCOL_TYPE_SEED` -4. **集群区域**：确保设置正确的集群区域，如 `cn-beijing` - - 集群区域必须与您的账户配置匹配 - - 如果遇到WebSocket握手错误，尝试切换到其他区域 - -### 配置示例 -```kotlin -// 设置API路径 -mSpeechEngine.setOptionString(SpeechEngineDefines.PARAMS_KEY_ASR_URI_STRING, "/api/v3/sauc/bigmodel"); - -// 设置认证信息 -mSpeechEngine.setOptionString(SpeechEngineDefines.PARAMS_KEY_APP_ID_STRING, "YOUR_APP_ID"); -mSpeechEngine.setOptionString(SpeechEngineDefines.PARAMS_KEY_APP_TOKEN_STRING, "YOUR_APP_KEY"); // 不需要Bearer前缀 - -// 设置资源ID -mSpeechEngine.setOptionString(SpeechEngineDefines.PARAMS_KEY_RESOURCE_ID_STRING, "YOUR_RESOURCE_ID"); - -// 设置协议类型 -mSpeechEngine.setOptionInt(SpeechEngineDefines.PARAMS_KEY_PROTOCOL_TYPE_INT, SpeechEngineDefines.PROTOCOL_TYPE_SEED); - -// 设置集群区域 -mSpeechEngine.setOptionString(SpeechEngineDefines.PARAMS_KEY_ASR_CLUSTER_STRING, "cn-beijing"); - -// 设置ASR请求参数 -mSpeechEngine.setOptionString(SpeechEngineDefines.PARAMS_KEY_ASR_REQ_PARAMS_STRING, - "{"force_to_speech_time":0, "end_window_size":800}"); -``` - -## 支持的语音类型 - -我们支持多种语音类型，包括： - -### 趣味方言 -- 湾区大叔 (`zh_female_wanqudashu_moon_bigtts`) -- 呆萌川妹 (`zh_female_daimengchuanmei_moon_bigtts`) -- 广州德哥 (`zh_male_guozhoudege_moon_bigtts`) -- 北京小爷 (`zh_male_beijingxiaoye_moon_bigtts`) -- 浩宇小哥 (`zh_male_haoyuxiaoge_moon_bigtts`) - -### 通用场景 -- 少年梓辛/Brayan (`zh_male_shaonianzixin_moon_bigtts`) - -### 角色扮演 -- 魅力女友 (`zh_female_meilinvyou_moon_bigtts`) -- 深夜播客 (`zh_male_shenyeboke_moon_bigtts`) -- 柔美女友 (`zh_female_sajiaonvyou_moon_bigtts`) -- 撒娇学妹 (`zh_female_yuanqinvyou_moon_bigtts`) - -### 基础语音类型 -- 基础男声 (`zh_male_qingse_common`) -- 基础女声 (`zh_female_qingse_common`) -- 高级男声 (`zh_male_M392_conversation_wvae_bigtts`) -- 高级女声 (`zh_female_F392_conversation_wvae_bigtts`) \ No newline at end of file diff --git a/lib/modules/explore/controllers/explore_controller.dart b/lib/modules/explore/controllers/explore_controller.dart index 1cb448ecd..067e4b6af 100644 --- a/lib/modules/explore/controllers/explore_controller.dart +++ b/lib/modules/explore/controllers/explore_controller.dart @@ -117,7 +117,7 @@ class ExploreController extends GetxController { startColor: const Color(0xFF4CAF50), endColor: const Color(0xFF009688), iconColor: const Color(0xFF4CAF50), - targetRoute: '', + targetRoute: Routes.realtime, ), FunctionScene( title: 'travelTranslator'.tr, // 旅行翻译助手 diff --git a/lib/modules/realtime/README.md b/lib/modules/realtime/README.md new file mode 100644 index 000000000..1962860f0 --- /dev/null +++ b/lib/modules/realtime/README.md @@ -0,0 +1,136 @@ +# Realtime 实时语音聊天模块 + +## ⚠️ 当前状态 + +**此模块为功能预览版本，语音对话功能暂未实现，仅提供界面效果演示。** + +后续将通过插件方式实现完整的语音识别、AI对话和语音合成功能。 + +## 概述 + +Realtime模块预留了类似ChatGPT app的实时语音对话功能框架，当前版本提供： + +- ✅ **界面框架** - 完整的UI布局和视觉效果 +- ✅ **状态管理** - 聆听、思考、回答状态的可视化 +- ✅ **动画效果** - 圆形波形动画和状态指示器 +- ✅ **交互体验** - 按钮点击和模拟状态切换 +- ⏳ **语音功能** - 待通过插件实现 + +## 界面设计 + +### 布局结构 +- **顶部状态区域** - 显示当前状态和提示信息 +- **中间可视化区域** - 圆形波形动画，类似ChatGPT风格 +- **底部控制区域** - 主控制按钮和状态指示器 + +### 状态指示 +- 🔵 **聆听状态** - 蓝色动画，模拟语音输入 +- 🟠 **思考状态** - 橙色动画，模拟AI处理 +- 🟢 **回答状态** - 绿色动画，模拟语音输出 +- ⚪ **空闲状态** - 灰色静态，等待交互 + +## 使用方法 + +### 路由导航 + +```dart +// 跳转到实时语音对话页面（模拟功能） +Get.toNamed(Routes.realtime); +``` + +### 体验模拟效果 + +1. 点击中央按钮开始模拟聆听 +2. 3秒后自动切换到AI思考状态 +3. 2秒后切换到语音播放状态 +4. 3秒后回到初始状态 + +## 技术架构 + +### 控制器 (RealtimeController) +- **状态管理**: `isListening`, `isSpeaking`, `isProcessing` +- **模拟逻辑**: 定时器控制状态切换 +- **动画控制**: 波形数据生成和更新 +- **生命周期**: 基础资源管理 + +### 视图组件 +- **RealtimeView** - 主视图容器 +- **VoiceVisualizer** - 语音可视化组件（简化版） +- **ControlButton** - 交互控制按钮（模拟功能） + +### 待实现功能 +- **AsrService** - 语音识别服务（插件） +- **TtsService** - 语音合成服务（插件） +- **OpenAIServiceAdapter** - AI对话服务（插件） + +## 模拟流程 + +1. **点击开始** → 模拟语音识别启动 +2. **动态波形** → 显示模拟的语音输入效果 +3. **自动停止** → 3秒后自动结束聆听 +4. **AI处理** → 模拟AI思考过程 +5. **语音输出** → 模拟语音播放效果 +6. **完成循环** → 返回待机状态 + +## 开发计划 + +### 阶段一：界面框架 ✅ +- [x] 基础UI布局 +- [x] 状态管理 +- [x] 动画效果 +- [x] 模拟交互 + +### 阶段二：插件集成 ⏳ +- [ ] 语音识别插件集成 +- [ ] AI对话服务集成 +- [ ] 语音合成插件集成 +- [ ] 实时流式处理 + +### 阶段三：功能完善 ⏳ +- [ ] 错误处理优化 +- [ ] 性能优化 +- [ ] 用户体验增强 +- [ ] 多语言支持 + +## 注意事项 + +1. **当前为演示版** - 所有功能均为模拟效果 +2. **无需权限** - 暂不涉及麦克风等硬件权限 +3. **无网络依赖** - 模拟功能不需要网络连接 +4. **保留扩展性** - 架构设计支持后续插件集成 + +## 自定义扩展 + +### 修改模拟参数 + +```dart +// 在RealtimeController中调整时间参数 +Timer(const Duration(seconds: 3), () { // 聆听时长 + stopListening(); +}); + +await Future.delayed(const Duration(seconds: 2)); // AI处理时长 +await Future.delayed(const Duration(seconds: 3)); // 播放时长 +``` + +### 添加新的视觉效果 + +```dart +// 在VoiceWavePainter中添加新的绘制方法 +void _drawCustomEffect(Canvas canvas, Offset center, double radius) { + // 自定义绘制逻辑 +} +``` + +## 技术栈 + +- **Flutter SDK**: >= 3.0.0 +- **GetX**: >= 4.6.5 (状态管理) +- **Material Design**: UI组件 +- **CustomPainter**: 自定义绘制 + +## 版本信息 + +- **当前版本**: v0.1.0 (Preview) +- **最后更新**: 2024年 +- **状态**: 模拟功能演示 \ No newline at end of file diff --git a/lib/modules/realtime/bindings/realtime_binding.dart b/lib/modules/realtime/bindings/realtime_binding.dart new file mode 100644 index 000000000..a9b6b65ba --- /dev/null +++ b/lib/modules/realtime/bindings/realtime_binding.dart @@ -0,0 +1,10 @@ +import 'package:get/get.dart'; +import '../controllers/realtime_controller.dart'; + +class RealtimeBinding extends Bindings { + @override + void dependencies() { + // 注册实时语音聊天控制器 + Get.lazyPut(() => RealtimeController()); + } +} \ No newline at end of file diff --git a/lib/modules/realtime/controllers/realtime_controller.dart b/lib/modules/realtime/controllers/realtime_controller.dart new file mode 100644 index 000000000..5022a813c --- /dev/null +++ b/lib/modules/realtime/controllers/realtime_controller.dart @@ -0,0 +1,409 @@ +import 'dart:async'; +import 'dart:math'; +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:get/get.dart'; +import 'package:realtime/realtime.dart'; +import '../../../core/utils/logger.dart'; + +class RealtimeController extends GetxController with GetTickerProviderStateMixin { + // Realtime服务实例 + final RealtimeService _realtimeService = RealtimeService(); + + // 连接状态 + final RxBool isConnected = false.obs; + final RxBool isConnecting = false.obs; + + // 语音状态 + final RxBool isListening = false.obs; + final RxBool isSpeaking = false.obs; + + // 状态和字幕显示 + final RxString currentStatus = '正在初始化...'.obs; + final RxString userSpeechText = ''.obs; + final RxString aiResponseText = ''.obs; + + // 音频可视化数据 + final RxList audioLevels = [].obs; + final RxDouble userVoiceLevel = 0.0.obs; + final RxDouble aiVoiceLevel = 0.0.obs; + + // 动画控制器 + late AnimationController pulseAnimationController; + late Animation pulseAnimation; + + // 状态监听 + StreamSubscription? _eventSubscription; + Timer? _audioUpdateTimer; + + @override + void onInit() { + super.onInit(); + _initializeAnimations(); + _initializeAudioLevels(); + _setupEventListener(); + Logger.info('Realtime模块初始化完成'); + + // 自动开始连接 + WidgetsBinding.instance.addPostFrameCallback((_) { + _autoConnect(); + }); + } + + @override + void onClose() { + _cleanup(); + super.onClose(); + } + + /// 初始化动画 + void _initializeAnimations() { + pulseAnimationController = AnimationController( + duration: const Duration(milliseconds: 1000), + vsync: this, + ); + + pulseAnimation = Tween( + begin: 0.8, + end: 1.2, + ).animate(CurvedAnimation( + parent: pulseAnimationController, + curve: Curves.easeInOut, + )); + } + + /// 初始化音频级别数据 + void _initializeAudioLevels() { + audioLevels.value = List.generate(20, (index) => 0.0); + } + + /// 设置事件监听器 + void _setupEventListener() { + _eventSubscription = _realtimeService.eventStream.listen( + (event) { + _handleRealtimeEvent(event); + }, + onError: (error) { + Logger.error('Realtime事件流错误: $error'); + }, + ); + } + + /// 处理Realtime事件 + void _handleRealtimeEvent(RealtimeEvent event) { + switch (event.type) { + case 'connectionStatusChanged': + _handleConnectionStatusChanged(event.data); + break; + case 'voiceStatusChanged': + _handleVoiceStatusChanged(event.data); + break; + case 'textReceived': + _handleTextReceived(event.data); + break; + case 'error': + _handleError(event.data); + break; + } + } + + /// 自动连接 + Future _autoConnect() async { + currentStatus.value = '正在连接AI服务器...'; + await connect(); + } + + /// 处理连接状态变化 + void _handleConnectionStatusChanged(String status) { + switch (status) { + case 'connected': + isConnected.value = true; + isConnecting.value = false; + currentStatus.value = '已连接，正在开始录音...'; + _startAudioUpdates(); + Logger.info('已连接到Vocode服务器'); + // 连接成功后自动开始录音 + Timer(const Duration(milliseconds: 500), () { + startListening(); + }); + break; + case 'connecting': + isConnecting.value = true; + currentStatus.value = '正在连接AI服务器...'; + Logger.info('正在连接Vocode服务器...'); + break; + case 'disconnected': + isConnected.value = false; + isConnecting.value = false; + currentStatus.value = '已断开连接'; + _resetState(); + Logger.info('已断开Vocode服务器连接'); + break; + case 'error': + isConnected.value = false; + isConnecting.value = false; + currentStatus.value = '连接失败，请检查网络'; + _resetState(); + Logger.error('Vocode服务器连接错误'); + break; + } + } + + /// 处理语音状态变化 + void _handleVoiceStatusChanged(String status) { + switch (status) { + case 'recording': + isListening.value = true; + isSpeaking.value = false; + currentStatus.value = '正在聆听，请说话...'; + pulseAnimationController.repeat(reverse: true); + Logger.info('开始录音'); + break; + case 'processing': + // 用户说话结束，等待AI回应 + isListening.value = false; + currentStatus.value = 'AI正在思考中...'; + break; + case 'playing': + isListening.value = false; + isSpeaking.value = true; + currentStatus.value = 'AI正在回复...'; + pulseAnimationController.repeat(reverse: true); + Logger.info('AI开始说话'); + break; + case 'idle': + isListening.value = false; + isSpeaking.value = false; + currentStatus.value = isConnected.value ? '等待用户说话...' : '已断开连接'; + pulseAnimationController.stop(); + Logger.info('语音状态空闲'); + break; + } + } + + /// 处理收到的文本消息 + void _handleTextReceived(String text) { + Logger.info('收到文本消息: $text'); + + // 根据消息来源更新相应的文本 + if (text.startsWith('[USER]')) { + userSpeechText.value = text.replaceFirst('[USER]', '').trim(); + } else if (text.startsWith('[AI]')) { + aiResponseText.value = text.replaceFirst('[AI]', '').trim(); + } else { + // 默认认为是AI回复 + aiResponseText.value = text; + } + } + + /// 处理错误 + void _handleError(String error) { + Logger.error('Realtime错误: $error'); + // 可以显示错误提示 + } + + /// 切换连接状态 + Future toggleConnection() async { + if (isConnected.value) { + await disconnect(); + } else { + await connect(); + } + } + + /// 连接到AI服务 + Future connect() async { + if (isConnecting.value || isConnected.value) return; + + try { + HapticFeedback.lightImpact(); + + // 初始化Realtime服务 + final initialized = await _realtimeService.initialize( + serverUrl: 'ws://192.168.1.11:8000/ws', // 本地开发服务器使用ws协议 + sampleRate: 16000, + channels: 1, + bitsPerSample: 16, + ); + + if (!initialized) { + throw Exception('初始化Realtime服务失败'); + } + + // 连接服务器 + final connected = await _realtimeService.connect(); + if (!connected) { + throw Exception('连接Vocode服务器失败'); + } + + } catch (e) { + Logger.error('连接失败: $e'); + isConnecting.value = false; + _resetState(); + } + } + + /// 断开连接 + Future disconnect() async { + try { + HapticFeedback.mediumImpact(); + + // 停止录音和播放 + if (isListening.value) { + await _realtimeService.stopRecording(); + } + + await _realtimeService.stopPlaying(); + + // 断开连接 + await _realtimeService.disconnect(); + + // 停止本地更新 + _stopAudioUpdates(); + pulseAnimationController.stop(); + + } catch (e) { + Logger.error('断开连接失败: $e'); + } + } + + /// 开始/停止聆听 + void toggleListening() { + if (!isConnected.value) return; + + if (isListening.value) { + stopListening(); + } else { + startListening(); + } + } + + /// 开始聆听 + void startListening() { + if (!isConnected.value || isListening.value) return; + + HapticFeedback.lightImpact(); + + _realtimeService.startRecording().then((success) { + if (!success) { + Logger.error('开始录音失败'); + } + }); + } + + /// 停止聆听 + void stopListening() { + if (!isListening.value) return; + + _realtimeService.stopRecording().then((success) { + if (!success) { + Logger.error('停止录音失败'); + } + }); + } + + /// 开始音频数据更新 + void _startAudioUpdates() { + _audioUpdateTimer = Timer.periodic(const Duration(milliseconds: 100), (timer) { + if (!isConnected.value) { + timer.cancel(); + return; + } + + _updateAudioLevels(); + }); + } + + /// 停止音频数据更新 + void _stopAudioUpdates() { + _audioUpdateTimer?.cancel(); + _audioUpdateTimer = null; + } + + /// 更新音频级别 + void _updateAudioLevels() { + final random = Random(); + + // 更新用户语音级别 + if (isListening.value) { + userVoiceLevel.value = 0.3 + random.nextDouble() * 0.7; + } else { + userVoiceLevel.value = userVoiceLevel.value * 0.8; // 衰减 + } + + // 更新AI语音级别 + if (isSpeaking.value) { + aiVoiceLevel.value = 0.2 + random.nextDouble() * 0.8; + } else { + aiVoiceLevel.value = aiVoiceLevel.value * 0.9; // 衰减 + } + + // 更新音频级别数组 + final newLevels = List.generate(20, (index) { + if (isListening.value || isSpeaking.value) { + return random.nextDouble() * (isListening.value ? userVoiceLevel.value : aiVoiceLevel.value); + } + return audioLevels[index] * 0.95; // 衰减 + }); + + audioLevels.value = newLevels; + } + + /// 重置音频级别 + void _resetAudioLevels() { + userVoiceLevel.value = 0.0; + aiVoiceLevel.value = 0.0; + audioLevels.value = List.generate(20, (index) => 0.0); + } + + /// 关闭并返回上一页 + Future closeAndGoBack() async { + // 立即返回上一页，避免阻塞UI + Get.back(); + + // 之后异步进行清理，不阻塞页面切换 + unawaited(_performShutdown()); + } + + /// 实际执行关闭逻辑（后台） + Future _performShutdown() async { + try { + currentStatus.value = '正在关闭连接...'; + + if (isListening.value) { + await _realtimeService.stopRecording(); + } + await _realtimeService.stopPlaying(); + await _realtimeService.disconnect(); + + _resetState(); + Logger.info('实时语音连接已关闭'); + } catch (e) { + Logger.error('关闭连接失败: $e'); + } + } + + /// 重置状态 + void _resetState() { + isListening.value = false; + isSpeaking.value = false; + userSpeechText.value = ''; + aiResponseText.value = ''; + _resetAudioLevels(); + pulseAnimationController.stop(); + _stopAudioUpdates(); + } + + /// 清理资源 + void _cleanup() async { + _eventSubscription?.cancel(); + _stopAudioUpdates(); + pulseAnimationController.dispose(); + + try { + await _realtimeService.dispose(); + } catch (e) { + Logger.error('清理Realtime服务失败: $e'); + } + } +} \ No newline at end of file diff --git a/lib/modules/realtime/views/realtime_view.dart b/lib/modules/realtime/views/realtime_view.dart new file mode 100644 index 000000000..693caf091 --- /dev/null +++ b/lib/modules/realtime/views/realtime_view.dart @@ -0,0 +1,240 @@ +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:get/get.dart'; +import '../controllers/realtime_controller.dart'; +import 'widgets/voice_mic_flow.dart'; + +class RealtimeView extends GetView { + const RealtimeView({Key? key}) : super(key: key); + + @override + Widget build(BuildContext context) { + // 设置状态栏为黑色文字 + SystemChrome.setSystemUIOverlayStyle( + const SystemUiOverlayStyle( + statusBarColor: Colors.transparent, + statusBarIconBrightness: Brightness.dark, + statusBarBrightness: Brightness.light, + ), + ); + + return Scaffold( + backgroundColor: Colors.white, + body: SafeArea( + child: Stack( + children: [ + // 右上角图标按钮 + Positioned( + top: 16, + right: 16, + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + // 分享按钮 + Icon( + Icons.ios_share, + color: Colors.grey[600], + size: 24, + ), + const SizedBox(width: 20), + // 设置按钮 + Icon( + Icons.tune, + color: Colors.grey[600], + size: 24, + ), + ], + ), + ), + + // 状态和字幕显示区域 + Positioned( + top: 100, + left: 20, + right: 20, + child: Column( + children: [ + // 当前状态显示 + Obx(() => Container( + width: double.infinity, + padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 12), + margin: const EdgeInsets.only(bottom: 12), + decoration: BoxDecoration( + color: Colors.black.withOpacity(0.1), + borderRadius: BorderRadius.circular(20), + ), + child: Text( + controller.currentStatus.value, + textAlign: TextAlign.center, + style: const TextStyle( + fontSize: 16, + fontWeight: FontWeight.w500, + color: Colors.black87, + ), + ), + )), + + // 用户ASR内容显示 + Obx(() => AnimatedContainer( + duration: const Duration(milliseconds: 300), + width: double.infinity, + height: controller.userSpeechText.value.isNotEmpty ? null : 0, + padding: controller.userSpeechText.value.isNotEmpty + ? const EdgeInsets.symmetric(horizontal: 16, vertical: 12) + : EdgeInsets.zero, + margin: controller.userSpeechText.value.isNotEmpty + ? const EdgeInsets.only(bottom: 8) + : EdgeInsets.zero, + decoration: BoxDecoration( + color: Colors.blue.withOpacity(0.1), + borderRadius: BorderRadius.circular(16), + border: Border.all( + color: Colors.blue.withOpacity(0.3), + width: 1, + ), + ), + child: controller.userSpeechText.value.isNotEmpty + ? Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + const Text( + '你说：', + style: TextStyle( + fontSize: 12, + fontWeight: FontWeight.w600, + color: Colors.blue, + ), + ), + const SizedBox(height: 4), + Text( + controller.userSpeechText.value, + style: const TextStyle( + fontSize: 14, + color: Colors.black87, + ), + ), + ], + ) + : null, + )), + + // AI回复内容显示 + Obx(() => AnimatedContainer( + duration: const Duration(milliseconds: 300), + width: double.infinity, + height: controller.aiResponseText.value.isNotEmpty ? null : 0, + padding: controller.aiResponseText.value.isNotEmpty + ? const EdgeInsets.symmetric(horizontal: 16, vertical: 12) + : EdgeInsets.zero, + margin: controller.aiResponseText.value.isNotEmpty + ? const EdgeInsets.only(bottom: 20) + : EdgeInsets.zero, + decoration: BoxDecoration( + color: Colors.green.withOpacity(0.1), + borderRadius: BorderRadius.circular(16), + border: Border.all( + color: Colors.green.withOpacity(0.3), + width: 1, + ), + ), + child: controller.aiResponseText.value.isNotEmpty + ? Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + const Text( + 'AI回复：', + style: TextStyle( + fontSize: 12, + fontWeight: FontWeight.w600, + color: Colors.green, + ), + ), + const SizedBox(height: 4), + Text( + controller.aiResponseText.value, + style: const TextStyle( + fontSize: 14, + color: Colors.black87, + ), + ), + ], + ) + : null, + )), + ], + ), + ), + + // 中间会呼吸的云彩动画 + Positioned.fill( + child: Align( + alignment: const Alignment(0, 0.1), + child: const VoiceMicFlow(size: 280), + ), + ), + + // 底部控制按钮 + Positioned( + bottom: 80, + left: 0, + right: 0, + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 40), + child: Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + // 摄像头按钮 + _buildBottomButton(Icons.videocam, null), + // 麦克风按钮 + _buildBottomButton(Icons.mic_none, controller.toggleListening), + // 更多按钮 + _buildBottomButton(Icons.more_horiz, null), + // 关闭按钮 + _buildBottomButton(Icons.close, controller.closeAndGoBack), + ], + ), + ), + ), + + // Home Indicator + Positioned( + bottom: 8, + left: 0, + right: 0, + child: Center( + child: Container( + width: 134, + height: 5, + decoration: BoxDecoration( + color: Colors.black, + borderRadius: BorderRadius.circular(2.5), + ), + ), + ), + ), + ], + ), + ), + ); + } + + /// 构建底部圆形按钮 + Widget _buildBottomButton(IconData icon, VoidCallback? onTap) { + return GestureDetector( + onTap: onTap, + child: Container( + width: 60, + height: 60, + decoration: BoxDecoration( + color: Colors.grey[200], + shape: BoxShape.circle, + ), + child: Icon( + icon, + color: Colors.black, + size: 26, + ), + ), + ); + } +} \ No newline at end of file diff --git a/lib/modules/realtime/views/widgets/voice_mic_flow.dart b/lib/modules/realtime/views/widgets/voice_mic_flow.dart new file mode 100644 index 000000000..1716cad47 --- /dev/null +++ b/lib/modules/realtime/views/widgets/voice_mic_flow.dart @@ -0,0 +1,89 @@ +import 'dart:ui'; +import 'package:flutter/material.dart'; +import 'package:flutter_shaders/flutter_shaders.dart'; + +class VoiceMicFlow extends StatefulWidget { + const VoiceMicFlow({super.key, this.size = 280, this.amp = 0.0}); + final double size; + final double amp; // 音量振幅 0.0-1.0 + + @override + State createState() => _VoiceMicFlowState(); +} + +class _VoiceMicFlowState extends State + with SingleTickerProviderStateMixin { + late final AnimationController _ctl; + + @override + void initState() { + super.initState(); + _ctl = AnimationController( + duration: const Duration(seconds: 100), // 100秒的长周期 + vsync: this, + )..repeat(); + } + + @override + void dispose() { + _ctl.dispose(); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + return ClipOval( + child: ShaderBuilder( + assetKey: 'shaders/mic_flow.frag', + (context, shader, child) { + return AnimatedBuilder( + animation: _ctl, + builder: (_, __) { + // 将0.0-1.0的动画值映射到更大的时间范围 + final time = _ctl.value * 100.0; + shader + ..setFloat(0, widget.size) // u_resolution.x + ..setFloat(1, widget.size) // u_resolution.y + ..setFloat(2, time) // u_time + ..setFloat(3, widget.amp); // u_amp (音量振幅) + return CustomPaint( + size: Size.square(widget.size), + painter: _ShaderPainter(shader), + ); + }, + ); + }, + // 着色器编译时的占位符 + child: Container( + width: widget.size, + height: widget.size, + decoration: const BoxDecoration( + shape: BoxShape.circle, + gradient: LinearGradient( + begin: Alignment.topLeft, + end: Alignment.bottomRight, + colors: [ + Color(0xFF1565C0), // 深蓝 + Color(0xFFE3F2FD), // 浅蓝 + ], + ), + ), + ), + ), + ); + } +} + +class _ShaderPainter extends CustomPainter { + _ShaderPainter(this.shader); + final FragmentShader shader; + + @override + void paint(Canvas canvas, Size size) { + final paint = Paint()..shader = shader; + canvas.drawRect(Offset.zero & size, paint); + } + + @override + bool shouldRepaint(covariant _ShaderPainter oldDelegate) => true; +} \ No newline at end of file diff --git a/lib/modules/settings/controllers/settings_controller.dart b/lib/modules/settings/controllers/settings_controller.dart index aa8ac4d54..b892932ce 100644 --- a/lib/modules/settings/controllers/settings_controller.dart +++ b/lib/modules/settings/controllers/settings_controller.dart @@ -121,7 +121,7 @@ class SettingsController extends GetxController { 'name': '日程权限', 'icon': Icons.calendar_today, 'permissions': [ - {'type': Permission.calendarWriteOnly, 'name': '日历写入', 'description': '用于创建日历事件和提醒'}, + {'type': Permission.calendarFullAccess, 'name': '日历写入', 'description': '用于创建日历事件和提醒'}, {'type': Permission.calendarFullAccess, 'name': '日历完全访问', 'description': '用于读取和修改日历事件'} ] }, diff --git a/lib/modules/settings/views/permissions_view.dart b/lib/modules/settings/views/permissions_view.dart index ccc94bfb1..abfaa2ee7 100644 --- a/lib/modules/settings/views/permissions_view.dart +++ b/lib/modules/settings/views/permissions_view.dart @@ -150,7 +150,7 @@ class PermissionsView extends GetView { return '电话'; case Permission.sms: return '短信'; - case Permission.calendarWriteOnly: + case Permission.calendarFullAccess: return '日历写入'; case Permission.calendarFullAccess: return '日历完整访问'; @@ -168,7 +168,7 @@ class PermissionsView extends GetView { return '用于拨打电话功能'; case Permission.sms: return '用于发送短信功能'; - case Permission.calendarWriteOnly: + case Permission.calendarFullAccess: return '用于创建日历事件和提醒'; case Permission.calendarFullAccess: return '用于读取和管理日历事件及提醒'; @@ -342,7 +342,7 @@ class PermissionsView extends GetView { return '电话'; case Permission.sms: return '短信'; - case Permission.calendarWriteOnly: + case Permission.calendarFullAccess: return '日历写入'; case Permission.calendarFullAccess: return '日历完整访问'; @@ -360,7 +360,7 @@ class PermissionsView extends GetView { return '拨打电话功能需要电话权限'; case Permission.sms: return '发送短信功能需要短信权限'; - case Permission.calendarWriteOnly: + case Permission.calendarFullAccess: return '创建日历事件需要日历写入权限'; case Permission.calendarFullAccess: return '管理日历事件需要完整日历权限'; diff --git a/lib/modules/settings/views/settings_view.dart b/lib/modules/settings/views/settings_view.dart index 9fe16c6dc..faa531560 100644 --- a/lib/modules/settings/views/settings_view.dart +++ b/lib/modules/settings/views/settings_view.dart @@ -339,6 +339,28 @@ class SettingsView extends GetView { isDarkMode: isDarkMode, ), ), + Divider( + height: 1, + color: isDarkMode + ? Colors.white.withOpacity(0.1) + : Colors.grey[200]), + // 实时对话入口 + _buildSimpleNavigationSetting( + title: '实时语音对话', + subtitle: '体验智能语音交互功能（功能预览）', + icon: Icons.chat_bubble_outline, + iconBgColor: isDarkMode + ? Colors.purple[900]!.withOpacity(0.3) + : Colors.purple[100]!, + iconColor: + isDarkMode ? Colors.purple[300]! : Colors.purple[600]!, + titleColor: isDarkMode ? Colors.white : null, + subtitleColor: isDarkMode ? Colors.white70 : null, + onTap: () { + Get.toNamed(Routes.realtime); + }, + isDarkMode: isDarkMode, + ), ], ), ), @@ -599,11 +621,13 @@ class SettingsView extends GetView { Get.toNamed(Routes.permissions); }, ), - // Divider( - // height: 1, - // color: isDarkMode - // ? Colors.white.withOpacity(0.1) - // : Colors.grey[200]), + Divider( + height: 1, + color: isDarkMode + ? Colors.white.withOpacity(0.1) + : Colors.grey[200]), + + // 退出登录 // _buildSimpleNavigationSetting( diff --git a/lib/routes/app_pages.dart b/lib/routes/app_pages.dart index d54329ac2..849c81551 100644 --- a/lib/routes/app_pages.dart +++ b/lib/routes/app_pages.dart @@ -37,6 +37,8 @@ import '../modules/agent/views/agent_view.dart'; import '../modules/agent/bindings/agent_binding.dart'; import '../modules/ota_upgrade/views/ota_upgrade_view.dart'; import '../modules/ota_upgrade/bindings/ota_upgrade_binding.dart'; +import '../modules/realtime/bindings/realtime_binding.dart'; +import '../modules/realtime/views/realtime_view.dart'; abstract class AppPages { static final pages = [ @@ -189,5 +191,10 @@ abstract class AppPages { transition: Transition.rightToLeft, transitionDuration: Duration(milliseconds: 250), ), + GetPage( + name: Routes.realtime, + page: () => const RealtimeView(), + binding: RealtimeBinding(), + ), ]; } diff --git a/lib/routes/app_routes.dart b/lib/routes/app_routes.dart index 3388ca4af..e054da0f3 100644 --- a/lib/routes/app_routes.dart +++ b/lib/routes/app_routes.dart @@ -1,6 +1,6 @@ abstract class Routes { // 认证相关 - static const splash = '/splash'; + static const splash = '/'; static const login = '/login'; static const pairing = '/pairing'; @@ -21,11 +21,11 @@ abstract class Routes { // 会议相关 static const meeting = '/meeting'; - static const meetingDetails = '/meetingDetails'; + static const meetingDetails = '/meeting/details'; // 语言功能 static const translation = '/translation'; // 实时翻译 - static const translationHistory = '/translation_history'; // 翻译历史 + static const translationHistory = '/translation/history'; // 翻译历史 static const speechDemo = '/speech_demo'; // 语音演示 // 测试路由（生产环境建议移除） @@ -45,4 +45,7 @@ abstract class Routes { static const feedback = '/feedback'; // 用户反馈 static const agent = '/agent'; // 在线客服 static const navigation = '/navigation'; // 实时导航 + + // 实时语音对话 + static const realtime = '/realtime'; } diff --git a/local_plugins/realtime/.gitignore b/local_plugins/realtime/.gitignore new file mode 100644 index 000000000..f5af98475 --- /dev/null +++ b/local_plugins/realtime/.gitignore @@ -0,0 +1,51 @@ +# Miscellaneous +*.class +*.log +*.pyc +*.swp +.DS_Store +.atom/ +.buildlog/ +.history +.svn/ + +# IntelliJ related +*.iml +*.ipr +*.iws +.idea/ + +# Visual Studio Code related +.vscode/ + +# Flutter/Dart/Pub related +**/doc/api/ +.dart_tool/ +.flutter-plugins +.flutter-plugins-dependencies +.packages +.pub-cache/ +.pub/ +build/ + +# iOS related +**/ios/.generated/ +**/ios/Flutter/Flutter.framework +**/ios/Flutter/Flutter.podspec +**/ios/Flutter/Generated.xcconfig +**/ios/Flutter/app.flx +**/ios/Flutter/app.zip +**/ios/Flutter/flutter_assets/ +**/ios/Flutter/flutter_export_environment.sh +**/ios/ServiceDefinitions.json +**/ios/Runner/GeneratedPluginRegistrant.* +.build/ + +# Coverage +coverage/ + +# Exceptions to above rules. +!**/ios/**/default.mode1v3 +!**/ios/**/default.mode2v3 +!**/ios/**/default.pbxuser +!**/ios/**/default.perspectivev3 \ No newline at end of file diff --git a/local_plugins/realtime/IMPLEMENTATION.md b/local_plugins/realtime/IMPLEMENTATION.md new file mode 100644 index 000000000..a5d49a976 --- /dev/null +++ b/local_plugins/realtime/IMPLEMENTATION.md @@ -0,0 +1,190 @@ +# Realtime Plugin 实现总结 + +## 项目概述 + +基于您提供的Android原生示例，我为DeepVoice项目创建了一个realtime插件，用于通过WebSocket接入Vocode服务器实现实时语音聊天功能。该插件已完成Android和iOS双平台实现。 + +## 架构设计 + +### 插件结构 +``` +local_plugins/realtime/ +├── android/ # Android实现 +│ ├── build.gradle # Android构建配置 +│ ├── src/main/ +│ │ ├── AndroidManifest.xml # Android权限配置 +│ │ └── kotlin/com/yunqiinnovation/realtime/ +│ │ ├── RealtimePlugin.kt # 主插件入口 +│ │ ├── RealtimeAudioManager.kt # 音频管理器（AudioRecord/AudioTrack） +│ │ └── RealtimeWebSocketManager.kt # WebSocket管理器（OkHttp） +├── ios/ # iOS实现 +│ ├── Classes/ # Objective-C桥接文件 +│ │ ├── RealtimePlugin.h +│ │ └── RealtimePlugin.m +│ ├── realtime/ # Swift Package +│ │ ├── Package.swift +│ │ └── Sources/realtime/ +│ │ ├── RealtimePlugin.swift # 主插件入口 +│ │ ├── RealtimeAudioManager.swift # 音频管理器（AVAudioEngine） +│ │ └── RealtimeWebSocketManager.swift # WebSocket管理器（URLSession） +│ └── realtime.podspec +├── lib/ +│ └── realtime.dart # Flutter接口 +├── pubspec.yaml +└── README.md +``` + +### 核心组件 + +1. **RealtimePlugin (SwiftRealtimePlugin)** + - 主插件入口，处理Flutter方法调用 + - 管理各组件间的协调 + - 处理事件分发 + +2. **RealtimeAudioManager** + - 使用AVAudioEngine进行音频录制 + - 使用AVAudioPlayerNode进行音频播放 + - 支持16kHz/16-bit/单声道格式 + - 实现20ms帧长的音频处理 + +3. **RealtimeWebSocketManager** + - 使用URLSessionWebSocketTask实现WebSocket通信 + - 支持音频数据和文本消息的收发 + - 自动连接状态管理 + +## 技术实现对照 + +| 功能 | Android示例 | iOS实现 | +|------|-------------|---------| +| 音频录制 | AudioRecord | AVAudioEngine + AVAudioInputNode | +| 音频播放 | AudioTrack | AVAudioPlayerNode | +| WebSocket | OkHttp WebSocket | URLSessionWebSocketTask | +| 线程管理 | Kotlin协程 | DispatchQueue | +| 音频格式 | 16kHz/16-bit/单声道 | 16kHz/16-bit/单声道 | +| 帧长 | 20ms (320字节) | 20ms (320字节) | + +## Flutter接口 + +### 主要类 +- `RealtimeService`: 主服务类 +- `RealtimeEvent`: 事件类 +- `ConnectionStatus`: 连接状态枚举 +- `VoiceStatus`: 语音状态枚举 +- `RealtimeException`: 异常类 + +### 主要方法 +- `initialize()`: 初始化服务 +- `connect()`: 连接服务器 +- `disconnect()`: 断开连接 +- `startRecording()`: 开始录音 +- `stopRecording()`: 停止录音 +- `eventStream`: 事件流 + +## 使用方式 + +### 1. 在RealtimeController中集成 +已更新`lib/modules/realtime/controllers/realtime_controller.dart`使用真实的realtime插件，取代了原来的模拟实现。 + +### 2. 事件监听 +```dart +_realtimeService.eventStream.listen((event) { + switch (event.type) { + case 'connectionStatusChanged': + // 处理连接状态变化 + case 'voiceStatusChanged': + // 处理语音状态变化 + case 'textReceived': + // 处理文本消息 + case 'error': + // 处理错误 + } +}); +``` + +## 配置要求 + +### 依赖配置 +已添加到主项目的`pubspec.yaml`: +```yaml +realtime: + path: local_plugins/realtime +``` + +### 权限配置 +iOS的`Info.plist`已包含必要的麦克风权限： +```xml +NSMicrophoneUsageDescription +需要麦克风权限用于语音识别和录音功能 +``` + +## 与Android示例的对应关系 + +1. **初始化对应** + - Android: 创建AudioRecord, AudioTrack, OkHttpClient + - iOS: 创建AVAudioEngine, AVAudioPlayerNode, URLSession + +2. **录音线程对应** + - Android: `loopRecordSend()` 协程 + - iOS: AVAudioInputNode的installTap回调 + +3. **播放线程对应** + - Android: `loopPlayback()` 协程 + LinkedBlockingQueue + - iOS: DispatchQueue + AVAudioPCMBuffer队列 + +4. **WebSocket对应** + - Android: OkHttp WebSocketListener + - iOS: URLSessionWebSocketDelegate + +## 特性支持 + +✅ **已实现** +- Android和iOS双平台支持 +- 实时音频录制和播放（16kHz/16-bit/单声道） +- WebSocket双向通信 +- 状态管理和事件通知 +- 错误处理 +- 资源管理和清理 +- 20ms帧长处理 +- Kotlin协程和DispatchQueue线程管理 + +❌ **未实现（可扩展）** +- 音频编码（Opus等） +- 噪声消除/回声抑制 +- 自动重连机制 +- 音频质量自适应 + +## 使用注意事项 + +1. **服务器地址配置** + ```dart + // 需要替换为实际的Vocode服务器地址 + serverUrl: 'wss://your-vocode-server/ws' + ``` + +2. **音频格式一致性** + - 确保服务器支持16kHz/16-bit/单声道格式 + - 帧长固定为20ms (320字节) + +3. **错误处理** + - 监听事件流中的错误 + - 处理网络断线和重连 + +4. **资源管理** + - 及时调用dispose()释放资源 + - 避免内存泄漏 + +## 测试建议 + +1. **本地测试** + - 先用echo服务器测试WebSocket连接 + - 验证音频录制和播放功能 + +2. **集成测试** + - 与真实Vocode服务器集成 + - 测试端到端语音交互 + +3. **性能测试** + - 测试长时间使用的稳定性 + - 监控内存和CPU使用 + +这个实现为DeepVoice项目提供了完整的实时语音交互能力，可以直接与Vocode服务器进行通信，实现类似Android示例的功能。 \ No newline at end of file diff --git a/local_plugins/realtime/README.md b/local_plugins/realtime/README.md new file mode 100644 index 000000000..a5bc746f0 --- /dev/null +++ b/local_plugins/realtime/README.md @@ -0,0 +1,125 @@ +# Realtime Plugin + +实时语音聊天插件，通过WebSocket连接Vocode服务器实现实时语音交互。 + +## 功能特性 + +- **实时音频录制**: 16kHz/16-bit/单声道格式录音 +- **WebSocket通信**: 与Vocode服务器进行实时数据传输 +- **音频播放**: 播放服务器返回的语音数据 +- **状态管理**: 连接状态和语音状态监控 +- **事件回调**: 支持各种事件的监听和处理 + +## 技术实现 + +### Android端实现 +- **音频录制**: 使用AudioRecord进行16kHz/16-bit/单声道录音 +- **音频播放**: 使用AudioTrack进行音频播放 +- **WebSocket**: 使用OkHttp WebSocket客户端 +- **线程管理**: 使用Kotlin协程处理录音和播放 + +### iOS端实现 +- **音频录制**: 使用AVAudioEngine和AVAudioInputNode +- **音频播放**: 使用AVAudioPlayerNode进行播放 +- **WebSocket**: 使用URLSessionWebSocketTask +- **线程管理**: 使用DispatchQueue管理录音和播放线程 + +### 核心组件 +- `RealtimePlugin`: 主插件入口，处理Flutter方法调用 +- `RealtimeAudioManager`: 音频管理器，负责录音和播放 +- `RealtimeWebSocketManager`: WebSocket管理器，负责网络通信 + +**Android端组件**: +- AudioRecord + AudioTrack + OkHttp + Kotlin协程 +- 对应您提供的Android原生示例功能 + +**iOS端组件**: +- AVAudioEngine + AVAudioPlayerNode + URLSessionWebSocketTask + DispatchQueue + +## 使用方法 + +```dart +import 'package:realtime/realtime.dart'; + +final realtimeService = RealtimeService(); + +// 初始化 +await realtimeService.initialize( + serverUrl: 'wss://your-server/ws', + sampleRate: 16000, + channels: 1, + bitsPerSample: 16, +); + +// 监听事件 +realtimeService.eventStream.listen((event) { + switch (event.type) { + case 'connectionStatusChanged': + // 处理连接状态变化 + break; + case 'voiceStatusChanged': + // 处理语音状态变化 + break; + case 'textReceived': + // 处理收到的文本消息 + break; + case 'error': + // 处理错误 + break; + } +}); + +// 连接服务器 +await realtimeService.connect(); + +// 开始录音 +await realtimeService.startRecording(); + +// 停止录音 +await realtimeService.stopRecording(); + +// 断开连接 +await realtimeService.disconnect(); + +// 释放资源 +await realtimeService.dispose(); +``` + +## 音频格式 + +- **采样率**: 16000 Hz +- **声道数**: 1（单声道） +- **位深**: 16-bit +- **帧长**: 20ms (320字节) + +## 权限要求 + +### Android +权限已自动包含在插件中： +- `RECORD_AUDIO`: 音频录制权限 +- `INTERNET`: 网络访问权限 +- `ACCESS_NETWORK_STATE`: 网络状态权限 + +### iOS +在Info.plist中添加麦克风权限： +```xml +NSMicrophoneUsageDescription +应用需要麦克风权限进行语音录制 +``` + +## 注意事项 + +1. 确保服务器地址正确且可访问 +2. 网络环境良好，避免频繁断线 +3. 音频格式与服务器保持一致 +4. 及时释放资源，避免内存泄漏 + +## 错误处理 + +插件会自动处理常见错误： +- 网络连接失败 +- 音频设备不可用 +- 权限被拒绝 +- 服务器断开连接 + +通过事件流可以监听这些错误并进行相应处理。 \ No newline at end of file diff --git a/local_plugins/realtime/android/build.gradle.kts b/local_plugins/realtime/android/build.gradle.kts new file mode 100644 index 000000000..b787e4124 --- /dev/null +++ b/local_plugins/realtime/android/build.gradle.kts @@ -0,0 +1,47 @@ +plugins { + id("com.android.library") + id("org.jetbrains.kotlin.android") + kotlin("plugin.serialization") version "1.9.24" +} + +android { + namespace = "com.yunqiinnovation.realtime" + compileSdk = 35 + + defaultConfig { + minSdk = 21 + targetSdk = 33 + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 + } + + kotlinOptions { + jvmTarget = "11" + } + + packaging { + resources { + excludes.add("META-INF/DEPENDENCIES") + excludes.add("META-INF/LICENSE") + excludes.add("META-INF/LICENSE.txt") + excludes.add("META-INF/license.txt") + excludes.add("META-INF/NOTICE") + excludes.add("META-INF/NOTICE.txt") + excludes.add("META-INF/notice.txt") + excludes.add("META-INF/*.kotlin_module") + } + } +} + +dependencies { + implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.7.3") + + // OkHttp for WebSocket + implementation(platform("com.squareup.okhttp3:okhttp-bom:4.12.0")) + implementation("com.squareup.okhttp3:okhttp") +} \ No newline at end of file diff --git a/local_plugins/realtime/android/settings.gradle.kts b/local_plugins/realtime/android/settings.gradle.kts new file mode 100644 index 000000000..dd4a80ad2 --- /dev/null +++ b/local_plugins/realtime/android/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "realtime" \ No newline at end of file diff --git a/local_plugins/realtime/android/src/main/AndroidManifest.xml b/local_plugins/realtime/android/src/main/AndroidManifest.xml new file mode 100644 index 000000000..64df6a0ae --- /dev/null +++ b/local_plugins/realtime/android/src/main/AndroidManifest.xml @@ -0,0 +1,11 @@ + + + + + + + + + + \ No newline at end of file diff --git a/local_plugins/realtime/android/src/main/kotlin/com/yunqiinnovation/realtime/RealtimeAudioManager.kt b/local_plugins/realtime/android/src/main/kotlin/com/yunqiinnovation/realtime/RealtimeAudioManager.kt new file mode 100644 index 000000000..b954a2d6d --- /dev/null +++ b/local_plugins/realtime/android/src/main/kotlin/com/yunqiinnovation/realtime/RealtimeAudioManager.kt @@ -0,0 +1,355 @@ +package com.yunqiinnovation.realtime + +import android.content.Context +import android.media.* +import android.util.Log +import kotlinx.coroutines.* +import java.util.concurrent.LinkedBlockingQueue + +/** + * 实时音频管理器 + * + * 负责音频的录制和播放，对应Android示例中的AudioRecord和AudioTrack功能 + */ +class RealtimeAudioManager(private val context: Context) { + companion object { + private const val TAG = "RealtimeAudioManager" + } + + // 音频配置 + private var audioConfig: AudioConfig? = null + + // 录音相关 + private var audioRecord: AudioRecord? = null + private var isRecording = false + private var recordJob: Job? = null + + // 播放相关 + private var audioTrack: AudioTrack? = null + private var isPlaying = false + private var playJob: Job? = null + private val playbackQueue = LinkedBlockingQueue() + + // 回调 + var onAudioData: ((ByteArray) -> Unit)? = null + var onStatusChanged: ((VoiceStatus) -> Unit)? = null + + // 协程作用域 + private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + + /// 初始化音频管理器 + fun initialize(config: AudioConfig): Boolean { + this.audioConfig = config + + return setupAudioRecord(config) && setupAudioTrack(config) + } + + /// 设置AudioRecord（录音） + private fun setupAudioRecord(config: AudioConfig): Boolean { + try { + val channelConfig = if (config.channels == 1) { + AudioFormat.CHANNEL_IN_MONO + } else { + AudioFormat.CHANNEL_IN_STEREO + } + + val audioFormat = when (config.bitsPerSample) { + 16 -> AudioFormat.ENCODING_PCM_16BIT + 8 -> AudioFormat.ENCODING_PCM_8BIT + else -> AudioFormat.ENCODING_PCM_16BIT + } + + val bufferSize = AudioRecord.getMinBufferSize( + config.sampleRate, + channelConfig, + audioFormat + ) + + if (bufferSize == AudioRecord.ERROR || bufferSize == AudioRecord.ERROR_BAD_VALUE) { + Log.e(TAG, "无法获取音频录制缓冲区大小") + return false + } + + audioRecord = AudioRecord( + MediaRecorder.AudioSource.MIC, + config.sampleRate, + channelConfig, + audioFormat, + bufferSize + ) + + if (audioRecord?.state != AudioRecord.STATE_INITIALIZED) { + Log.e(TAG, "AudioRecord初始化失败") + return false + } + + Log.i(TAG, "AudioRecord初始化成功，采样率: ${config.sampleRate}, 声道: ${config.channels}, 位深: ${config.bitsPerSample}") + return true + } catch (e: Exception) { + Log.e(TAG, "设置AudioRecord失败: ${e.message}", e) + return false + } + } + + /// 设置AudioTrack（播放） + private fun setupAudioTrack(config: AudioConfig): Boolean { + try { + val channelConfig = if (config.channels == 1) { + AudioFormat.CHANNEL_OUT_MONO + } else { + AudioFormat.CHANNEL_OUT_STEREO + } + + val audioFormat = when (config.bitsPerSample) { + 16 -> AudioFormat.ENCODING_PCM_16BIT + 8 -> AudioFormat.ENCODING_PCM_8BIT + else -> AudioFormat.ENCODING_PCM_16BIT + } + + val bufferSize = AudioTrack.getMinBufferSize( + config.sampleRate, + channelConfig, + audioFormat + ) + + if (bufferSize == AudioTrack.ERROR || bufferSize == AudioTrack.ERROR_BAD_VALUE) { + Log.e(TAG, "无法获取音频播放缓冲区大小") + return false + } + + audioTrack = AudioTrack.Builder() + .setAudioAttributes( + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_MEDIA) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + ) + .setAudioFormat( + AudioFormat.Builder() + .setSampleRate(config.sampleRate) + .setChannelMask(channelConfig) + .setEncoding(audioFormat) + .build() + ) + .setBufferSizeInBytes(bufferSize) + .setTransferMode(AudioTrack.MODE_STREAM) + .build() + + if (audioTrack?.state != AudioTrack.STATE_INITIALIZED) { + Log.e(TAG, "AudioTrack初始化失败") + return false + } + + Log.i(TAG, "AudioTrack初始化成功") + return true + } catch (e: Exception) { + Log.e(TAG, "设置AudioTrack失败: ${e.message}", e) + return false + } + } + + /// 开始录音 + fun startRecording(): Boolean { + if (isRecording || audioRecord == null) { + return false + } + + try { + audioRecord?.startRecording() + isRecording = true + onStatusChanged?.invoke(VoiceStatus.RECORDING) + + // 启动录音协程 + recordJob = scope.launch { + loopRecordSend() + } + + Log.i(TAG, "开始录音") + return true + } catch (e: Exception) { + Log.e(TAG, "开始录音失败: ${e.message}", e) + return false + } + } + + /// 录音循环（对应Android示例中的loopRecordSend） + private suspend fun loopRecordSend() { + val config = audioConfig ?: return + val frameSize = config.frameSize + val buffer = ByteArray(frameSize) + + while (isRecording && audioRecord != null) { + try { + val bytesRead = audioRecord!!.read(buffer, 0, frameSize) + + if (bytesRead > 0) { + // 确保读取的数据长度正确 + val audioData = if (bytesRead == frameSize) { + buffer.copyOf() + } else { + buffer.copyOf(bytesRead) + } + + // 回调音频数据 + onAudioData?.invoke(audioData) + } else { + Log.w(TAG, "录音读取数据失败: $bytesRead") + } + } catch (e: Exception) { + Log.e(TAG, "录音循环异常: ${e.message}", e) + break + } + } + } + + /// 停止录音 + fun stopRecording(): Boolean { + if (!isRecording) { + return false + } + + try { + isRecording = false + recordJob?.cancel() + audioRecord?.stop() + onStatusChanged?.invoke(VoiceStatus.IDLE) + + Log.i(TAG, "停止录音") + return true + } catch (e: Exception) { + Log.e(TAG, "停止录音失败: ${e.message}", e) + return false + } + } + + /// 播放音频数据 + fun playAudioData(data: ByteArray) { + if (data.isNotEmpty()) { + playbackQueue.offer(data) + + // 如果没在播放，开始播放 + if (!isPlaying) { + startPlayback() + } + } + } + + /// 开始播放 + private fun startPlayback() { + if (isPlaying || audioTrack == null) { + return + } + + try { + audioTrack?.play() + isPlaying = true + onStatusChanged?.invoke(VoiceStatus.PLAYING) + + // 启动播放协程 + playJob = scope.launch { + loopPlayback() + } + + Log.i(TAG, "开始播放") + } catch (e: Exception) { + Log.e(TAG, "开始播放失败: ${e.message}", e) + } + } + + /// 播放循环（对应Android示例中的loopPlayback） + private suspend fun loopPlayback() { + while (isPlaying && audioTrack != null) { + try { + val audioData = playbackQueue.poll() + + if (audioData != null) { + val bytesWritten = audioTrack!!.write(audioData, 0, audioData.size) + + if (bytesWritten < 0) { + Log.w(TAG, "音频播放写入失败: $bytesWritten") + } + } else { + // 队列为空，检查是否应该停止播放 + delay(10) // 短暂等待 + + if (playbackQueue.isEmpty()) { + // 队列持续为空，停止播放 + break + } + } + } catch (e: Exception) { + Log.e(TAG, "播放循环异常: ${e.message}", e) + break + } + } + + // 播放结束 + stopPlayback() + } + + /// 停止播放 + fun stopPlaying(): Boolean { + return stopPlayback() + } + + private fun stopPlayback(): Boolean { + if (!isPlaying) { + return false + } + + try { + isPlaying = false + playJob?.cancel() + audioTrack?.stop() + playbackQueue.clear() + onStatusChanged?.invoke(VoiceStatus.IDLE) + + Log.i(TAG, "停止播放") + return true + } catch (e: Exception) { + Log.e(TAG, "停止播放失败: ${e.message}", e) + return false + } + } + + /// 更新配置 + fun updateConfig(config: AudioConfig): Boolean { + // 如果正在录音或播放，先停止 + if (isRecording) { + stopRecording() + } + if (isPlaying) { + stopPlayback() + } + + // 释放旧的音频对象 + releaseAudioObjects() + + return initialize(config) + } + + /// 释放音频对象 + private fun releaseAudioObjects() { + try { + audioRecord?.release() + audioRecord = null + + audioTrack?.release() + audioTrack = null + } catch (e: Exception) { + Log.e(TAG, "释放音频对象失败: ${e.message}", e) + } + } + + /// 释放资源 + fun dispose() { + stopRecording() + stopPlayback() + + scope.cancel() + releaseAudioObjects() + playbackQueue.clear() + + Log.i(TAG, "音频管理器已释放") + } +} \ No newline at end of file diff --git a/local_plugins/realtime/android/src/main/kotlin/com/yunqiinnovation/realtime/RealtimePlugin.kt b/local_plugins/realtime/android/src/main/kotlin/com/yunqiinnovation/realtime/RealtimePlugin.kt new file mode 100644 index 000000000..a8e787656 --- /dev/null +++ b/local_plugins/realtime/android/src/main/kotlin/com/yunqiinnovation/realtime/RealtimePlugin.kt @@ -0,0 +1,309 @@ +package com.yunqiinnovation.realtime + +import android.content.Context +import android.util.Log +import io.flutter.embedding.engine.plugins.FlutterPlugin +import io.flutter.plugin.common.EventChannel +import io.flutter.plugin.common.MethodCall +import io.flutter.plugin.common.MethodChannel +import io.flutter.plugin.common.MethodChannel.MethodCallHandler +import io.flutter.plugin.common.MethodChannel.Result +import kotlinx.coroutines.* + +/** + * 实时语音聊天插件 + * + * 通过WebSocket连接Vocode服务器实现实时语音交互 + * 包含录音、播放、WebSocket通信等功能 + */ +class RealtimePlugin: FlutterPlugin, MethodCallHandler, EventChannel.StreamHandler { + companion object { + private const val TAG = "RealtimePlugin" + private const val METHOD_CHANNEL = "realtime/methods" + private const val EVENT_CHANNEL = "realtime/events" + } + + private lateinit var context: Context + private lateinit var methodChannel: MethodChannel + private lateinit var eventChannel: EventChannel + private var eventSink: EventChannel.EventSink? = null + + // 核心组件 + private lateinit var audioManager: RealtimeAudioManager + private lateinit var webSocketManager: RealtimeWebSocketManager + + // 协程作用域 + private val scope = CoroutineScope(Dispatchers.Main + SupervisorJob()) + + // 配置参数 + private var serverUrl: String = "" + private var sampleRate: Int = 16000 + private var channels: Int = 1 + private var bitsPerSample: Int = 16 + + // 状态 + @Volatile + private var connectionStatus: ConnectionStatus = ConnectionStatus.DISCONNECTED + @Volatile + private var voiceStatus: VoiceStatus = VoiceStatus.IDLE + + override fun onAttachedToEngine(flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { + context = flutterPluginBinding.applicationContext + + methodChannel = MethodChannel(flutterPluginBinding.binaryMessenger, METHOD_CHANNEL) + methodChannel.setMethodCallHandler(this) + + eventChannel = EventChannel(flutterPluginBinding.binaryMessenger, EVENT_CHANNEL) + eventChannel.setStreamHandler(this) + + // 初始化核心组件 + audioManager = RealtimeAudioManager(context) + webSocketManager = RealtimeWebSocketManager() + + setupCallbacks() + + Log.i(TAG, "Realtime插件已附加到引擎") + } + + override fun onDetachedFromEngine(binding: FlutterPlugin.FlutterPluginBinding) { + methodChannel.setMethodCallHandler(null) + eventChannel.setStreamHandler(null) + + // 清理资源 + scope.cancel() + audioManager.dispose() + webSocketManager.dispose() + + Log.i(TAG, "Realtime插件已从引擎分离") + } + + /// 设置各组件的回调 + private fun setupCallbacks() { + // 音频管理器回调 + audioManager.onAudioData = { data -> + webSocketManager.sendAudioData(data) + } + + audioManager.onStatusChanged = { status -> + voiceStatus = status + sendEvent("voiceStatusChanged", status.value) + } + + // WebSocket管理器回调 + webSocketManager.onConnectionStatusChanged = { status -> + connectionStatus = status + sendEvent("connectionStatusChanged", status.value) + } + + webSocketManager.onAudioReceived = { data -> + audioManager.playAudioData(data) + } + + webSocketManager.onTextReceived = { text -> + sendEvent("textReceived", text) + } + + webSocketManager.onError = { error -> + sendEvent("error", error) + } + } + + /// 发送事件到Flutter端 + private fun sendEvent(type: String, data: Any?) { + val event = mapOf( + "type" to type, + "data" to data + ) + + scope.launch { + eventSink?.success(event) + } + } + + override fun onMethodCall(call: MethodCall, result: Result) { + when (call.method) { + "initialize" -> handleInitialize(call, result) + "connect" -> handleConnect(result) + "disconnect" -> handleDisconnect(result) + "startRecording" -> handleStartRecording(result) + "stopRecording" -> handleStopRecording(result) + "stopPlaying" -> handleStopPlaying(result) + "getConnectionStatus" -> result.success(connectionStatus.value) + "getVoiceStatus" -> result.success(voiceStatus.value) + "sendTextMessage" -> handleSendTextMessage(call, result) + "setAudioConfig" -> handleSetAudioConfig(call, result) + "dispose" -> handleDispose(result) + else -> result.notImplemented() + } + } + + private fun handleInitialize(call: MethodCall, result: Result) { + try { + val args = call.arguments as? Map + val serverUrl = args?.get("serverUrl") as? String + ?: return result.error("INVALID_ARGUMENTS", "服务器地址不能为空", null) + + this.serverUrl = serverUrl + this.sampleRate = args["sampleRate"] as? Int ?: 16000 + this.channels = args["channels"] as? Int ?: 1 + this.bitsPerSample = args["bitsPerSample"] as? Int ?: 16 + + // 初始化音频管理器 + val audioConfig = AudioConfig(sampleRate, channels, bitsPerSample) + val success = audioManager.initialize(audioConfig) + + if (success) { + webSocketManager.initialize(serverUrl) + Log.i(TAG, "实时语音插件初始化成功") + } else { + Log.e(TAG, "实时语音插件初始化失败") + } + + result.success(success) + } catch (e: Exception) { + Log.e(TAG, "初始化失败: ${e.message}", e) + result.error("INITIALIZATION_ERROR", "初始化失败: ${e.message}", null) + } + } + + private fun handleConnect(result: Result) { + scope.launch { + try { + val success = webSocketManager.connect() + result.success(success) + } catch (e: Exception) { + Log.e(TAG, "连接失败: ${e.message}", e) + result.error("CONNECTION_ERROR", "连接失败: ${e.message}", null) + } + } + } + + private fun handleDisconnect(result: Result) { + scope.launch { + try { + webSocketManager.disconnect() + result.success(true) + } catch (e: Exception) { + Log.e(TAG, "断开连接失败: ${e.message}", e) + result.error("DISCONNECTION_ERROR", "断开连接失败: ${e.message}", null) + } + } + } + + private fun handleStartRecording(result: Result) { + scope.launch { + try { + val success = audioManager.startRecording() + result.success(success) + } catch (e: Exception) { + Log.e(TAG, "开始录音失败: ${e.message}", e) + result.error("RECORDING_ERROR", "开始录音失败: ${e.message}", null) + } + } + } + + private fun handleStopRecording(result: Result) { + scope.launch { + try { + val success = audioManager.stopRecording() + result.success(success) + } catch (e: Exception) { + Log.e(TAG, "停止录音失败: ${e.message}", e) + result.error("RECORDING_ERROR", "停止录音失败: ${e.message}", null) + } + } + } + + private fun handleStopPlaying(result: Result) { + scope.launch { + try { + val success = audioManager.stopPlaying() + result.success(success) + } catch (e: Exception) { + Log.e(TAG, "停止播放失败: ${e.message}", e) + result.error("PLAYBACK_ERROR", "停止播放失败: ${e.message}", null) + } + } + } + + private fun handleSendTextMessage(call: MethodCall, result: Result) { + try { + val args = call.arguments as? Map + val message = args?.get("message") as? String + ?: return result.error("INVALID_ARGUMENTS", "消息内容不能为空", null) + + val success = webSocketManager.sendTextMessage(message) + result.success(success) + } catch (e: Exception) { + Log.e(TAG, "发送文本消息失败: ${e.message}", e) + result.error("MESSAGE_ERROR", "发送文本消息失败: ${e.message}", null) + } + } + + private fun handleSetAudioConfig(call: MethodCall, result: Result) { + try { + val args = call.arguments as? Map + ?: return result.error("INVALID_ARGUMENTS", "参数无效", null) + + args["sampleRate"]?.let { this.sampleRate = it as Int } + args["channels"]?.let { this.channels = it as Int } + args["bitsPerSample"]?.let { this.bitsPerSample = it as Int } + + val audioConfig = AudioConfig(sampleRate, channels, bitsPerSample) + val success = audioManager.updateConfig(audioConfig) + result.success(success) + } catch (e: Exception) { + Log.e(TAG, "设置音频配置失败: ${e.message}", e) + result.error("CONFIG_ERROR", "设置音频配置失败: ${e.message}", null) + } + } + + private fun handleDispose(result: Result) { + scope.launch { + try { + audioManager.dispose() + webSocketManager.dispose() + result.success(null) + } catch (e: Exception) { + Log.e(TAG, "释放资源失败: ${e.message}", e) + result.error("DISPOSE_ERROR", "释放资源失败: ${e.message}", null) + } + } + } + + // EventChannel.StreamHandler 实现 + override fun onListen(arguments: Any?, events: EventChannel.EventSink?) { + eventSink = events + Log.i(TAG, "事件流监听已开始") + } + + override fun onCancel(arguments: Any?) { + eventSink = null + Log.i(TAG, "事件流监听已取消") + } +} + +// 枚举定义 +enum class ConnectionStatus(val value: String) { + DISCONNECTED("disconnected"), + CONNECTING("connecting"), + CONNECTED("connected"), + ERROR("error") +} + +enum class VoiceStatus(val value: String) { + IDLE("idle"), + RECORDING("recording"), + PROCESSING("processing"), + PLAYING("playing") +} + +// 音频配置 +data class AudioConfig( + val sampleRate: Int, + val channels: Int, + val bitsPerSample: Int +) { + val frameSize: Int + get() = (sampleRate * channels * bitsPerSample / 8 * 20) / 1000 // 20ms帧 +} \ No newline at end of file diff --git a/local_plugins/realtime/android/src/main/kotlin/com/yunqiinnovation/realtime/RealtimeWebSocketManager.kt b/local_plugins/realtime/android/src/main/kotlin/com/yunqiinnovation/realtime/RealtimeWebSocketManager.kt new file mode 100644 index 000000000..77a6ebb49 --- /dev/null +++ b/local_plugins/realtime/android/src/main/kotlin/com/yunqiinnovation/realtime/RealtimeWebSocketManager.kt @@ -0,0 +1,218 @@ +package com.yunqiinnovation.realtime + +import android.util.Log +import kotlinx.coroutines.* +import okhttp3.* +import okio.ByteString +import java.util.concurrent.TimeUnit + +/** + * 实时WebSocket管理器 + * + * 负责与Vocode服务器的WebSocket通信，对应Android示例中的OkHttp WebSocket功能 + */ +class RealtimeWebSocketManager : WebSocketListener() { + companion object { + private const val TAG = "RealtimeWebSocketManager" + private const val CONNECT_TIMEOUT = 30L + private const val READ_TIMEOUT = 60L + private const val WRITE_TIMEOUT = 30L + } + + // OkHttp客户端和WebSocket + private var okHttpClient: OkHttpClient? = null + private var webSocket: WebSocket? = null + + // 服务器URL + private var serverUrl: String = "" + + // 连接状态 + @Volatile + private var isConnected = false + @Volatile + private var isConnecting = false + + // 回调 + var onConnectionStatusChanged: ((ConnectionStatus) -> Unit)? = null + var onAudioReceived: ((ByteArray) -> Unit)? = null + var onTextReceived: ((String) -> Unit)? = null + var onError: ((String) -> Unit)? = null + + // 协程作用域 + private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + + /// 初始化WebSocket管理器 + fun initialize(serverUrl: String) { + this.serverUrl = serverUrl + + // 创建OkHttp客户端 + okHttpClient = OkHttpClient.Builder() + .connectTimeout(CONNECT_TIMEOUT, TimeUnit.SECONDS) + .readTimeout(READ_TIMEOUT, TimeUnit.SECONDS) + .writeTimeout(WRITE_TIMEOUT, TimeUnit.SECONDS) + .retryOnConnectionFailure(true) + .build() + + Log.i(TAG, "WebSocket管理器初始化，服务器地址: $serverUrl") + } + + /// 连接到服务器 + suspend fun connect(): Boolean = withContext(Dispatchers.IO) { + if (isConnected || isConnecting) { + return@withContext false + } + + val client = okHttpClient ?: run { + Log.e(TAG, "OkHttp客户端未初始化") + onError?.invoke("客户端未初始化") + return@withContext false + } + + try { + isConnecting = true + onConnectionStatusChanged?.invoke(ConnectionStatus.CONNECTING) + + val request = Request.Builder() + .url(serverUrl) + .build() + + webSocket = client.newWebSocket(request, this@RealtimeWebSocketManager) + + // 等待连接建立 + var waitTime = 0 + while (isConnecting && waitTime < CONNECT_TIMEOUT * 1000) { + delay(100) + waitTime += 100 + } + + if (isConnected) { + Log.i(TAG, "WebSocket连接成功") + return@withContext true + } else { + Log.e(TAG, "WebSocket连接超时") + isConnecting = false + onConnectionStatusChanged?.invoke(ConnectionStatus.ERROR) + return@withContext false + } + } catch (e: Exception) { + Log.e(TAG, "WebSocket连接失败: ${e.message}", e) + isConnecting = false + onConnectionStatusChanged?.invoke(ConnectionStatus.ERROR) + onError?.invoke("连接失败: ${e.message}") + return@withContext false + } + } + + /// 断开连接 + fun disconnect() { + try { + isConnected = false + isConnecting = false + + webSocket?.close(1000, "正常关闭") + webSocket = null + + onConnectionStatusChanged?.invoke(ConnectionStatus.DISCONNECTED) + Log.i(TAG, "WebSocket已断开连接") + } catch (e: Exception) { + Log.e(TAG, "断开WebSocket连接失败: ${e.message}", e) + } + } + + /// 发送音频数据 + fun sendAudioData(data: ByteArray) { + if (!isConnected) { + return + } + + try { + val byteString = ByteString.of(*data) + val success = webSocket?.send(byteString) ?: false + + if (!success) { + Log.w(TAG, "发送音频数据失败") + } + } catch (e: Exception) { + Log.e(TAG, "发送音频数据异常: ${e.message}", e) + } + } + + /// 发送文本消息 + fun sendTextMessage(message: String): Boolean { + if (!isConnected) { + Log.w(TAG, "WebSocket未连接，无法发送文本消息") + return false + } + + return try { + val success = webSocket?.send(message) ?: false + if (!success) { + Log.w(TAG, "发送文本消息失败") + } + success + } catch (e: Exception) { + Log.e(TAG, "发送文本消息异常: ${e.message}", e) + false + } + } + + /// 释放资源 + fun dispose() { + disconnect() + + scope.cancel() + + okHttpClient?.dispatcher?.executorService?.shutdown() + okHttpClient = null + + Log.i(TAG, "WebSocket管理器已释放") + } + + // WebSocketListener 回调实现 + override fun onOpen(webSocket: WebSocket, response: Response) { + Log.i(TAG, "WebSocket连接已建立") + isConnected = true + isConnecting = false + onConnectionStatusChanged?.invoke(ConnectionStatus.CONNECTED) + } + + override fun onMessage(webSocket: WebSocket, text: String) { + Log.d(TAG, "收到文本消息: $text") + onTextReceived?.invoke(text) + } + + override fun onMessage(webSocket: WebSocket, bytes: ByteString) { + Log.d(TAG, "收到音频数据: ${bytes.size} 字节") + onAudioReceived?.invoke(bytes.toByteArray()) + } + + override fun onClosing(webSocket: WebSocket, code: Int, reason: String) { + Log.i(TAG, "WebSocket正在关闭，代码: $code, 原因: $reason") + webSocket.close(1000, null) + } + + override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { + Log.i(TAG, "WebSocket已关闭，代码: $code, 原因: $reason") + isConnected = false + isConnecting = false + onConnectionStatusChanged?.invoke(ConnectionStatus.DISCONNECTED) + } + + override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { + val errorMessage = when { + t.message?.contains("Unable to parse TLS packet header") == true -> + "SSL错误：请检查服务器是否支持wss://，本地开发请使用ws://" + t.message?.contains("Connection refused") == true -> + "连接被拒绝：请检查服务器是否运行在指定端口" + t.message?.contains("timeout") == true -> + "连接超时：请检查网络连接和服务器地址" + else -> "连接失败: ${t.message}" + } + + Log.e(TAG, "WebSocket连接失败: ${t.message}", t) + isConnected = false + isConnecting = false + onConnectionStatusChanged?.invoke(ConnectionStatus.ERROR) + onError?.invoke(errorMessage) + } +} \ No newline at end of file diff --git a/local_plugins/realtime/example.md b/local_plugins/realtime/example.md new file mode 100644 index 000000000..74b8ebd44 --- /dev/null +++ b/local_plugins/realtime/example.md @@ -0,0 +1,274 @@ +# Realtime Plugin 使用示例 + +## 基本用法 + +```dart +import 'package:realtime/realtime.dart'; + +class VoiceCallPage extends StatefulWidget { + @override + _VoiceCallPageState createState() => _VoiceCallPageState(); +} + +class _VoiceCallPageState extends State { + final RealtimeService _realtimeService = RealtimeService(); + StreamSubscription? _eventSubscription; + + ConnectionStatus _connectionStatus = ConnectionStatus.disconnected; + VoiceStatus _voiceStatus = VoiceStatus.idle; + + @override + void initState() { + super.initState(); + _setupEventListener(); + _initializeService(); + } + + void _setupEventListener() { + _eventSubscription = _realtimeService.eventStream.listen((event) { + switch (event.type) { + case 'connectionStatusChanged': + setState(() { + _connectionStatus = _parseConnectionStatus(event.data); + }); + break; + case 'voiceStatusChanged': + setState(() { + _voiceStatus = _parseVoiceStatus(event.data); + }); + break; + case 'textReceived': + print('收到文本: ${event.data}'); + break; + case 'error': + print('错误: ${event.data}'); + break; + } + }); + } + + Future _initializeService() async { + try { + final success = await _realtimeService.initialize( + serverUrl: 'wss://your-vocode-server.com/ws', + sampleRate: 16000, + channels: 1, + bitsPerSample: 16, + ); + + if (success) { + print('Realtime服务初始化成功'); + } else { + print('Realtime服务初始化失败'); + } + } catch (e) { + print('初始化错误: $e'); + } + } + + Future _connect() async { + try { + final success = await _realtimeService.connect(); + if (!success) { + print('连接失败'); + } + } catch (e) { + print('连接错误: $e'); + } + } + + Future _disconnect() async { + try { + await _realtimeService.disconnect(); + } catch (e) { + print('断开连接错误: $e'); + } + } + + Future _startRecording() async { + try { + final success = await _realtimeService.startRecording(); + if (!success) { + print('开始录音失败'); + } + } catch (e) { + print('录音错误: $e'); + } + } + + Future _stopRecording() async { + try { + final success = await _realtimeService.stopRecording(); + if (!success) { + print('停止录音失败'); + } + } catch (e) { + print('停止录音错误: $e'); + } + } + + ConnectionStatus _parseConnectionStatus(String status) { + switch (status) { + case 'connected': return ConnectionStatus.connected; + case 'connecting': return ConnectionStatus.connecting; + case 'disconnected': return ConnectionStatus.disconnected; + case 'error': return ConnectionStatus.error; + default: return ConnectionStatus.disconnected; + } + } + + VoiceStatus _parseVoiceStatus(String status) { + switch (status) { + case 'recording': return VoiceStatus.recording; + case 'processing': return VoiceStatus.processing; + case 'playing': return VoiceStatus.playing; + case 'idle': return VoiceStatus.idle; + default: return VoiceStatus.idle; + } + } + + @override + Widget build(BuildContext context) { + return Scaffold( + appBar: AppBar( + title: Text('实时语音聊天'), + ), + body: Center( + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + // 连接状态显示 + Text('连接状态: ${_connectionStatus.name}'), + SizedBox(height: 20), + + // 语音状态显示 + Text('语音状态: ${_voiceStatus.name}'), + SizedBox(height: 40), + + // 连接按钮 + ElevatedButton( + onPressed: _connectionStatus == ConnectionStatus.disconnected + ? _connect + : _disconnect, + child: Text(_connectionStatus == ConnectionStatus.disconnected + ? '连接' + : '断开'), + ), + SizedBox(height: 20), + + // 录音按钮 + ElevatedButton( + onPressed: _connectionStatus == ConnectionStatus.connected + ? (_voiceStatus == VoiceStatus.recording + ? _stopRecording + : _startRecording) + : null, + child: Text(_voiceStatus == VoiceStatus.recording + ? '停止录音' + : '开始录音'), + ), + ], + ), + ), + ); + } + + @override + void dispose() { + _eventSubscription?.cancel(); + _realtimeService.dispose(); + super.dispose(); + } +} +``` + +## GetX控制器中的使用 + +```dart +import 'package:get/get.dart'; +import 'package:realtime/realtime.dart'; + +class VoiceController extends GetxController { + final RealtimeService _realtimeService = RealtimeService(); + + final RxBool isConnected = false.obs; + final RxBool isRecording = false.obs; + + StreamSubscription? _eventSubscription; + + @override + void onInit() { + super.onInit(); + _setupEventListener(); + _initializeService(); + } + + void _setupEventListener() { + _eventSubscription = _realtimeService.eventStream.listen((event) { + switch (event.type) { + case 'connectionStatusChanged': + isConnected.value = event.data == 'connected'; + break; + case 'voiceStatusChanged': + isRecording.value = event.data == 'recording'; + break; + } + }); + } + + Future _initializeService() async { + await _realtimeService.initialize( + serverUrl: 'wss://your-server/ws', + ); + } + + Future toggleConnection() async { + if (isConnected.value) { + await _realtimeService.disconnect(); + } else { + await _realtimeService.connect(); + } + } + + Future toggleRecording() async { + if (isRecording.value) { + await _realtimeService.stopRecording(); + } else { + await _realtimeService.startRecording(); + } + } + + @override + void onClose() { + _eventSubscription?.cancel(); + _realtimeService.dispose(); + super.onClose(); + } +} +``` + +## 权限配置 + +### iOS +在 `ios/Runner/Info.plist` 中添加： + +```xml +NSMicrophoneUsageDescription +应用需要麦克风权限进行语音录制 +``` + +## 服务器配置示例 + +需要一个支持WebSocket的Vocode服务器，具体实现可参考Vocode官方文档。 + +服务器需要： +1. 接收16kHz/16-bit/单声道的PCM音频数据 +2. 返回相同格式的音频数据 +3. 支持文本消息交换 + +## 注意事项 + +1. 确保网络连接稳定 +2. 音频格式必须与服务器保持一致 +3. 及时处理事件流中的错误 +4. 在适当时机释放资源 \ No newline at end of file diff --git a/local_plugins/realtime/ios/Classes/RealtimePlugin.h b/local_plugins/realtime/ios/Classes/RealtimePlugin.h new file mode 100644 index 000000000..c3ae05d49 --- /dev/null +++ b/local_plugins/realtime/ios/Classes/RealtimePlugin.h @@ -0,0 +1,4 @@ +#import + +@interface RealtimePlugin : NSObject +@end \ No newline at end of file diff --git a/local_plugins/realtime/ios/Classes/RealtimePlugin.m b/local_plugins/realtime/ios/Classes/RealtimePlugin.m new file mode 100644 index 000000000..32ac567a9 --- /dev/null +++ b/local_plugins/realtime/ios/Classes/RealtimePlugin.m @@ -0,0 +1,8 @@ +#import "RealtimePlugin.h" +@import realtime; + +@implementation RealtimePlugin ++ (void)registerWithRegistrar:(NSObject*)registrar { + [SwiftRealtimePlugin registerWithRegistrar:registrar]; +} +@end \ No newline at end of file diff --git a/local_plugins/realtime/ios/realtime.podspec b/local_plugins/realtime/ios/realtime.podspec new file mode 100644 index 000000000..b231b1206 --- /dev/null +++ b/local_plugins/realtime/ios/realtime.podspec @@ -0,0 +1,20 @@ +Pod::Spec.new do |s| + s.name = 'realtime' + s.version = '0.0.1' + s.summary = '实时语音聊天插件' + s.description = <<-DESC +实时语音聊天插件，通过WebSocket连接Vocode服务器实现实时语音交互 + DESC + s.homepage = 'http://example.com' + s.license = { :file => '../LICENSE' } + s.author = { 'YunQi Innovation' => 'dev@yunqiinnovation.com' } + s.source = { :path => '.' } + s.source_files = 'Classes/**/*' + s.public_header_files = 'Classes/**/*.h' + s.dependency 'Flutter' + s.platform = :ios, '12.0' + + # Flutter.framework does not contain a i386 slice. + s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386' } + s.swift_version = '5.0' +end \ No newline at end of file diff --git a/local_plugins/realtime/ios/realtime/Package.swift b/local_plugins/realtime/ios/realtime/Package.swift new file mode 100644 index 000000000..514fbc1e1 --- /dev/null +++ b/local_plugins/realtime/ios/realtime/Package.swift @@ -0,0 +1,18 @@ +// swift-tools-version: 5.9 +import PackageDescription + +let package = Package( + name: "realtime", + platforms: [.iOS("18.0")], + products: [ + .library(name: "realtime", targets: ["realtime"]) + ], + dependencies: [], + targets: [ + .target( + name: "realtime", + dependencies: [], + path: "Sources/realtime" + ) + ] +) \ No newline at end of file diff --git a/local_plugins/realtime/ios/realtime/Sources/realtime/RealtimeAudioManager.swift b/local_plugins/realtime/ios/realtime/Sources/realtime/RealtimeAudioManager.swift new file mode 100644 index 000000000..281907ce5 --- /dev/null +++ b/local_plugins/realtime/ios/realtime/Sources/realtime/RealtimeAudioManager.swift @@ -0,0 +1,314 @@ +import AVFoundation +import os.log +import Accelerate + +/** + * 实时音频管理器 + * + * 负责音频的录制和播放，对应Android示例中的AudioRecord和AudioTrack功能 + */ +class RealtimeAudioManager { + + private let log = OSLog(subsystem: "com.yunqiinnovation.realtime", category: "RealtimeAudioManager") + + // 音频配置 + private var audioConfig: AudioConfig? + + // 录音相关 + private var audioEngine: AVAudioEngine? + private var inputNode: AVAudioInputNode? + private var recordingFormat: AVAudioFormat? + private var isRecording = false + + // 播放相关 + private var audioPlayer: AVAudioPlayerNode? + private var playbackFormat: AVAudioFormat? + private var playbackQueue: DispatchQueue + private var audioBufferQueue: [AVAudioPCMBuffer] = [] + private var isPlaying = false + + // 回调 + var onAudioData: ((Data) -> Void)? + var onStatusChanged: ((VoiceStatus) -> Void)? + + // 录音缓冲区 + private var recordingBuffer: AVAudioPCMBuffer? + private var frameSize: Int = 320 // 20ms @ 16kHz + + init() { + playbackQueue = DispatchQueue(label: "com.realtime.playback", qos: .userInitiated) + setupAudioSession() + } + + /// 设置音频会话 + private func setupAudioSession() { + do { + let session = AVAudioSession.sharedInstance() + try session.setCategory(.playAndRecord, + mode: .default, + options: [.defaultToSpeaker, .allowBluetooth]) + try session.setActive(true) + os_log("音频会话设置成功", log: log, type: .info) + } catch { + os_log("音频会话设置失败: %@", log: log, type: .error, error.localizedDescription) + } + } + + /// 初始化音频管理器 + func initialize(config: AudioConfig) -> Bool { + self.audioConfig = config + self.frameSize = config.frameSize + + guard setupAudioEngine(config: config) else { + os_log("音频引擎初始化失败", log: log, type: .error) + return false + } + + guard setupAudioPlayer(config: config) else { + os_log("音频播放器初始化失败", log: log, type: .error) + return false + } + + os_log("音频管理器初始化成功", log: log, type: .info) + return true + } + + /// 设置音频引擎（录音） + private func setupAudioEngine(config: AudioConfig) -> Bool { + audioEngine = AVAudioEngine() + + guard let audioEngine = audioEngine else { return false } + + inputNode = audioEngine.inputNode + + // 创建录音格式：16kHz, 16-bit, 单声道 + recordingFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, + sampleRate: Double(config.sampleRate), + channels: AVAudioChannelCount(config.channels), + interleaved: true) + + guard let recordingFormat = recordingFormat else { + os_log("无法创建录音格式", log: log, type: .error) + return false + } + + // 创建录音缓冲区 + let frameCount = AVAudioFrameCount(config.frameSize / (config.bitsPerSample / 8)) + recordingBuffer = AVAudioPCMBuffer(pcmFormat: recordingFormat, frameCapacity: frameCount) + + return true + } + + /// 设置音频播放器 + private func setupAudioPlayer(config: AudioConfig) -> Bool { + guard let audioEngine = audioEngine else { return false } + + audioPlayer = AVAudioPlayerNode() + + // 创建播放格式：16kHz, 16-bit, 单声道 + playbackFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, + sampleRate: Double(config.sampleRate), + channels: AVAudioChannelCount(config.channels), + interleaved: true) + + guard let audioPlayer = audioPlayer, + let playbackFormat = playbackFormat else { + os_log("无法创建播放格式", log: log, type: .error) + return false + } + + // 连接播放器到音频引擎 + audioEngine.attach(audioPlayer) + audioEngine.connect(audioPlayer, to: audioEngine.outputNode, format: playbackFormat) + + return true + } + + /// 开始录音 + func startRecording() -> Bool { + guard let audioEngine = audioEngine, + let inputNode = inputNode, + let recordingFormat = recordingFormat, + !isRecording else { + return false + } + + // 安装录音tap + inputNode.installTap(onBus: 0, bufferSize: AVAudioFrameCount(frameSize / 2), format: recordingFormat) { [weak self] buffer, time in + self?.processRecordedAudio(buffer: buffer) + } + + do { + try audioEngine.start() + isRecording = true + onStatusChanged?(.recording) + os_log("开始录音", log: log, type: .info) + return true + } catch { + os_log("开始录音失败: %@", log: log, type: .error, error.localizedDescription) + return false + } + } + + /// 处理录制的音频数据 + private func processRecordedAudio(buffer: AVAudioPCMBuffer) { + guard let channelData = buffer.int16ChannelData?[0], + buffer.frameLength > 0 else { + return + } + + // 确保数据长度匹配frameSize + let dataLength = min(Int(buffer.frameLength) * 2, frameSize) // 2 bytes per sample + + // 转换为Data + let data = Data(bytes: channelData, count: dataLength) + + // 回调音频数据 + onAudioData?(data) + } + + /// 停止录音 + func stopRecording() -> Bool { + guard let audioEngine = audioEngine, + let inputNode = inputNode, + isRecording else { + return false + } + + inputNode.removeTap(onBus: 0) + audioEngine.stop() + isRecording = false + onStatusChanged?(.idle) + os_log("停止录音", log: log, type: .info) + return true + } + + /// 播放音频数据 + func playAudioData(_ data: Data) { + guard let playbackFormat = playbackFormat, + let audioPlayer = audioPlayer else { + return + } + + playbackQueue.async { [weak self] in + self?.enqueueAudioData(data, format: playbackFormat, player: audioPlayer) + } + } + + /// 将音频数据加入播放队列 + private func enqueueAudioData(_ data: Data, format: AVAudioFormat, player: AVAudioPlayerNode) { + // 创建PCM缓冲区 + let frameCount = AVAudioFrameCount(data.count / 2) // 16-bit = 2 bytes per sample + + guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameCount) else { + os_log("无法创建播放缓冲区", log: log, type: .error) + return + } + + buffer.frameLength = frameCount + + // 复制数据到缓冲区 + guard let channelData = buffer.int16ChannelData?[0] else { + return + } + + data.withUnsafeBytes { bytes in + let int16Pointer = bytes.bindMemory(to: Int16.self) + channelData.update(from: int16Pointer.baseAddress!, count: Int(frameCount)) + } + + // 加入播放队列 + audioBufferQueue.append(buffer) + + // 如果没在播放，开始播放 + if !isPlaying { + startPlayback() + } + } + + /// 开始播放 + private func startPlayback() { + guard let audioEngine = audioEngine, + let audioPlayer = audioPlayer, + !audioBufferQueue.isEmpty else { + return + } + + if !audioEngine.isRunning { + do { + try audioEngine.start() + } catch { + os_log("启动音频引擎失败: %@", log: log, type: .error, error.localizedDescription) + return + } + } + + if !audioPlayer.isPlaying { + audioPlayer.play() + } + + isPlaying = true + onStatusChanged?(.playing) + + // 播放队列中的音频 + playNextBuffer() + } + + /// 播放下一个缓冲区 + private func playNextBuffer() { + guard let audioPlayer = audioPlayer, + !audioBufferQueue.isEmpty else { + isPlaying = false + onStatusChanged?(.idle) + return + } + + let buffer = audioBufferQueue.removeFirst() + + audioPlayer.scheduleBuffer(buffer) { [weak self] in + DispatchQueue.main.async { + self?.playNextBuffer() + } + } + } + + /// 停止播放 + func stopPlaying() -> Bool { + guard let audioPlayer = audioPlayer else { + return false + } + + audioPlayer.stop() + audioBufferQueue.removeAll() + isPlaying = false + onStatusChanged?(.idle) + os_log("停止播放", log: log, type: .info) + return true + } + + /// 更新配置 + func updateConfig(config: AudioConfig) -> Bool { + // 如果正在录音或播放，先停止 + if isRecording { + _ = stopRecording() + } + if isPlaying { + _ = stopPlaying() + } + + return initialize(config: config) + } + + /// 释放资源 + func dispose() { + _ = stopRecording() + _ = stopPlaying() + + audioEngine?.stop() + audioEngine = nil + audioPlayer = nil + audioBufferQueue.removeAll() + + os_log("音频管理器已释放", log: log, type: .info) + } +} \ No newline at end of file diff --git a/local_plugins/realtime/ios/realtime/Sources/realtime/RealtimePlugin.swift b/local_plugins/realtime/ios/realtime/Sources/realtime/RealtimePlugin.swift new file mode 100644 index 000000000..746ba7d73 --- /dev/null +++ b/local_plugins/realtime/ios/realtime/Sources/realtime/RealtimePlugin.swift @@ -0,0 +1,274 @@ +import Flutter +import UIKit +import AVFoundation +import os.log + +/** + * 实时语音聊天插件 + * + * 通过WebSocket连接Vocode服务器实现实时语音交互 + * 包含录音、播放、WebSocket通信等功能 + */ +@objc public class SwiftRealtimePlugin: NSObject, FlutterPlugin { + + private let log = OSLog(subsystem: "com.yunqiinnovation.realtime", category: "RealtimePlugin") + + // Flutter通道 + private var methodChannel: FlutterMethodChannel? + private var eventChannel: FlutterEventChannel? + private var eventSink: FlutterEventSink? + + // 核心组件 + private let audioManager = RealtimeAudioManager() + private let websocketManager = RealtimeWebSocketManager() + + // 配置参数 + private var serverUrl: String = "" + private var sampleRate: Int = 16000 + private var channels: Int = 1 + private var bitsPerSample: Int = 16 + + // 状态 + private var connectionStatus: ConnectionStatus = .disconnected + private var voiceStatus: VoiceStatus = .idle + + /// 插件注册 + public static func register(with registrar: FlutterPluginRegistrar) { + let instance = SwiftRealtimePlugin() + + // 方法通道 + let methodChannel = FlutterMethodChannel(name: "realtime/methods", binaryMessenger: registrar.messenger()) + registrar.addMethodCallDelegate(instance, channel: methodChannel) + instance.methodChannel = methodChannel + + // 事件通道 + let eventChannel = FlutterEventChannel(name: "realtime/events", binaryMessenger: registrar.messenger()) + eventChannel.setStreamHandler(instance) + instance.eventChannel = eventChannel + + // 设置组件回调 + instance.setupCallbacks() + } + + /// 设置各组件的回调 + private func setupCallbacks() { + // 音频管理器回调 + audioManager.onAudioData = { [weak self] data in + self?.websocketManager.sendAudioData(data) + } + + audioManager.onStatusChanged = { [weak self] status in + self?.voiceStatus = status + self?.sendEvent(type: "voiceStatusChanged", data: status.rawValue) + } + + // WebSocket管理器回调 + websocketManager.onConnectionStatusChanged = { [weak self] status in + self?.connectionStatus = status + self?.sendEvent(type: "connectionStatusChanged", data: status.rawValue) + } + + websocketManager.onAudioReceived = { [weak self] data in + self?.audioManager.playAudioData(data) + } + + websocketManager.onTextReceived = { [weak self] text in + self?.sendEvent(type: "textReceived", data: text) + } + + websocketManager.onError = { [weak self] error in + self?.sendEvent(type: "error", data: error) + } + } + + /// 发送事件到Flutter端 + private func sendEvent(type: String, data: Any?) { + guard let eventSink = eventSink else { return } + + let event: [String: Any] = [ + "type": type, + "data": data ?? NSNull() + ] + + DispatchQueue.main.async { + eventSink(event) + } + } +} + +// MARK: - FlutterPlugin +extension RealtimePlugin { + + public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { + switch call.method { + case "initialize": + handleInitialize(call, result) + case "connect": + handleConnect(result) + case "disconnect": + handleDisconnect(result) + case "startRecording": + handleStartRecording(result) + case "stopRecording": + handleStopRecording(result) + case "stopPlaying": + handleStopPlaying(result) + case "getConnectionStatus": + result(connectionStatus.rawValue) + case "getVoiceStatus": + result(voiceStatus.rawValue) + case "sendTextMessage": + handleSendTextMessage(call, result) + case "setAudioConfig": + handleSetAudioConfig(call, result) + case "dispose": + handleDispose(result) + default: + result(FlutterMethodNotImplemented) + } + } + + private func handleInitialize(_ call: FlutterMethodCall, _ result: @escaping FlutterResult) { + guard let args = call.arguments as? [String: Any], + let serverUrl = args["serverUrl"] as? String else { + result(FlutterError(code: "INVALID_ARGUMENTS", message: "服务器地址不能为空", details: nil)) + return + } + + self.serverUrl = serverUrl + self.sampleRate = args["sampleRate"] as? Int ?? 16000 + self.channels = args["channels"] as? Int ?? 1 + self.bitsPerSample = args["bitsPerSample"] as? Int ?? 16 + + // 初始化音频管理器 + let audioConfig = AudioConfig( + sampleRate: sampleRate, + channels: channels, + bitsPerSample: bitsPerSample + ) + + let success = audioManager.initialize(config: audioConfig) + if success { + websocketManager.initialize(serverUrl: serverUrl) + os_log("实时语音插件初始化成功", log: log, type: .info) + } else { + os_log("实时语音插件初始化失败", log: log, type: .error) + } + + result(success) + } + + private func handleConnect(_ result: @escaping FlutterResult) { + websocketManager.connect { [weak self] success in + DispatchQueue.main.async { + result(success) + } + } + } + + private func handleDisconnect(_ result: @escaping FlutterResult) { + websocketManager.disconnect() + result(true) + } + + private func handleStartRecording(_ result: @escaping FlutterResult) { + let success = audioManager.startRecording() + result(success) + } + + private func handleStopRecording(_ result: @escaping FlutterResult) { + let success = audioManager.stopRecording() + result(success) + } + + private func handleStopPlaying(_ result: @escaping FlutterResult) { + let success = audioManager.stopPlaying() + result(success) + } + + private func handleSendTextMessage(_ call: FlutterMethodCall, _ result: @escaping FlutterResult) { + guard let args = call.arguments as? [String: Any], + let message = args["message"] as? String else { + result(FlutterError(code: "INVALID_ARGUMENTS", message: "消息内容不能为空", details: nil)) + return + } + + let success = websocketManager.sendTextMessage(message) + result(success) + } + + private func handleSetAudioConfig(_ call: FlutterMethodCall, _ result: @escaping FlutterResult) { + guard let args = call.arguments as? [String: Any] else { + result(FlutterError(code: "INVALID_ARGUMENTS", message: "参数无效", details: nil)) + return + } + + if let sampleRate = args["sampleRate"] as? Int { + self.sampleRate = sampleRate + } + if let channels = args["channels"] as? Int { + self.channels = channels + } + if let bitsPerSample = args["bitsPerSample"] as? Int { + self.bitsPerSample = bitsPerSample + } + + let audioConfig = AudioConfig( + sampleRate: sampleRate, + channels: channels, + bitsPerSample: bitsPerSample + ) + + let success = audioManager.updateConfig(config: audioConfig) + result(success) + } + + private func handleDispose(_ result: @escaping FlutterResult) { + audioManager.dispose() + websocketManager.dispose() + result(nil) + } +} + +// MARK: - FlutterStreamHandler +extension RealtimePlugin: FlutterStreamHandler { + + public func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { + self.eventSink = events + return nil + } + + public func onCancel(withArguments arguments: Any?) -> FlutterError? { + self.eventSink = nil + return nil + } +} + +// MARK: - 枚举定义 + +enum ConnectionStatus: String { + case disconnected = "disconnected" + case connecting = "connecting" + case connected = "connected" + case error = "error" +} + +enum VoiceStatus: String { + case idle = "idle" + case recording = "recording" + case processing = "processing" + case playing = "playing" +} + +// MARK: - 音频配置 + +struct AudioConfig { + let sampleRate: Int + let channels: Int + let bitsPerSample: Int + + var frameSize: Int { + // 20ms的帧大小 + return (sampleRate * channels * bitsPerSample / 8 * 20) / 1000 + } +} \ No newline at end of file diff --git a/local_plugins/realtime/ios/realtime/Sources/realtime/RealtimeWebSocketManager.swift b/local_plugins/realtime/ios/realtime/Sources/realtime/RealtimeWebSocketManager.swift new file mode 100644 index 000000000..98c69c076 --- /dev/null +++ b/local_plugins/realtime/ios/realtime/Sources/realtime/RealtimeWebSocketManager.swift @@ -0,0 +1,264 @@ +import Foundation +import Network +import os.log + +/** + * 实时WebSocket管理器 + * + * 负责与Vocode服务器的WebSocket通信，对应Android示例中的OkHttp WebSocket功能 + */ +class RealtimeWebSocketManager: NSObject { + + private let log = OSLog(subsystem: "com.yunqiinnovation.realtime", category: "RealtimeWebSocketManager") + + // WebSocket连接 + private var webSocketTask: URLSessionWebSocketTask? + private var urlSession: URLSession? + + // 服务器URL + private var serverUrl: String = "" + + // 连接状态 + private var isConnected = false + private var isConnecting = false + + // 回调 + var onConnectionStatusChanged: ((ConnectionStatus) -> Void)? + var onAudioReceived: ((Data) -> Void)? + var onTextReceived: ((String) -> Void)? + var onError: ((String) -> Void)? + + // 发送队列 + private let sendQueue = DispatchQueue(label: "com.realtime.websocket.send", qos: .userInitiated) + private let receiveQueue = DispatchQueue(label: "com.realtime.websocket.receive", qos: .userInitiated) + + override init() { + super.init() + setupURLSession() + } + + /// 设置URL会话 + private func setupURLSession() { + let config = URLSessionConfiguration.default + config.timeoutIntervalForRequest = 30 + config.timeoutIntervalForResource = 60 + urlSession = URLSession(configuration: config, delegate: self, delegateQueue: nil) + } + + /// 初始化WebSocket管理器 + func initialize(serverUrl: String) { + self.serverUrl = serverUrl + os_log("WebSocket管理器初始化，服务器地址: %@", log: log, type: .info, serverUrl) + } + + /// 连接到服务器 + func connect(completion: @escaping (Bool) -> Void) { + guard !isConnected && !isConnecting else { + completion(false) + return + } + + guard let url = URL(string: serverUrl) else { + os_log("无效的服务器地址: %@", log: log, type: .error, serverUrl) + onError?("无效的服务器地址") + completion(false) + return + } + + isConnecting = true + onConnectionStatusChanged?(.connecting) + + // 创建WebSocket任务 + var request = URLRequest(url: url) + request.timeoutInterval = 30 + + webSocketTask = urlSession?.webSocketTask(with: request) + + // 开始监听消息 + startListening() + + // 开始连接 + webSocketTask?.resume() + + // 等待连接建立 + DispatchQueue.global().asyncAfter(deadline: .now() + 1.0) { [weak self] in + self?.checkConnection(completion: completion) + } + } + + /// 检查连接状态 + private func checkConnection(completion: @escaping (Bool) -> Void) { + guard let webSocketTask = webSocketTask else { + isConnecting = false + onConnectionStatusChanged?(.error) + completion(false) + return + } + + switch webSocketTask.state { + case .running: + isConnected = true + isConnecting = false + onConnectionStatusChanged?(.connected) + os_log("WebSocket连接成功", log: log, type: .info) + completion(true) + case .suspended: + // 等待连接完成 + DispatchQueue.global().asyncAfter(deadline: .now() + 0.5) { [weak self] in + self?.checkConnection(completion: completion) + } + default: + isConnecting = false + onConnectionStatusChanged?(.error) + os_log("WebSocket连接失败", log: log, type: .error) + completion(false) + } + } + + /// 开始监听消息 + private func startListening() { + receiveMessage() + } + + /// 接收消息 + private func receiveMessage() { + webSocketTask?.receive { [weak self] result in + switch result { + case .success(let message): + self?.handleReceivedMessage(message) + // 继续监听下一条消息 + self?.receiveMessage() + case .failure(let error): + self?.handleReceiveError(error) + } + } + } + + /// 处理接收到的消息 + private func handleReceivedMessage(_ message: URLSessionWebSocketTask.Message) { + switch message { + case .data(let data): + // 音频数据 + onAudioReceived?(data) + case .string(let text): + // 文本数据 + onTextReceived?(text) + @unknown default: + os_log("收到未知类型消息", log: log, type: .warning) + } + } + + /// 处理接收错误 + private func handleReceiveError(_ error: Error) { + os_log("WebSocket接收错误: %@", log: log, type: .error, error.localizedDescription) + + // 如果连接断开，尝试重连 + if isConnected { + isConnected = false + onConnectionStatusChanged?(.error) + onError?("连接断开: \(error.localizedDescription)") + } + } + + /// 发送音频数据 + func sendAudioData(_ data: Data) { + guard isConnected else { return } + + sendQueue.async { [weak self] in + self?.webSocketTask?.send(.data(data)) { error in + if let error = error { + self?.os_log("发送音频数据失败: %@", log: self?.log ?? OSLog.default, type: .error, error.localizedDescription) + } + } + } + } + + /// 发送文本消息 + func sendTextMessage(_ message: String) -> Bool { + guard isConnected else { + os_log("WebSocket未连接，无法发送文本消息", log: log, type: .warning) + return false + } + + sendQueue.async { [weak self] in + self?.webSocketTask?.send(.string(message)) { error in + if let error = error { + self?.os_log("发送文本消息失败: %@", log: self?.log ?? OSLog.default, type: .error, error.localizedDescription) + } + } + } + + return true + } + + /// 断开连接 + func disconnect() { + guard isConnected || isConnecting else { return } + + webSocketTask?.cancel(with: .goingAway, reason: nil) + webSocketTask = nil + + isConnected = false + isConnecting = false + onConnectionStatusChanged?(.disconnected) + os_log("WebSocket已断开连接", log: log, type: .info) + } + + /// 释放资源 + func dispose() { + disconnect() + urlSession?.invalidateAndCancel() + urlSession = nil + os_log("WebSocket管理器已释放", log: log, type: .info) + } + + // 修复方法访问性问题 + private func os_log(_ message: String, log: OSLog, type: OSLogType, _ args: CVarArg...) { + if args.isEmpty { + os_log("%@", log: log, type: type, message) + } else { + os_log(message, log: log, type: type, args) + } + } +} + +// MARK: - URLSessionWebSocketDelegate +extension RealtimeWebSocketManager: URLSessionWebSocketDelegate { + + func urlSession(_ session: URLSession, webSocketTask: URLSessionWebSocketTask, didOpenWithProtocol protocol: String?) { + os_log("WebSocket连接已建立", log: log, type: .info) + isConnected = true + isConnecting = false + onConnectionStatusChanged?(.connected) + } + + func urlSession(_ session: URLSession, webSocketTask: URLSessionWebSocketTask, didCloseWith closeCode: URLSessionWebSocketTask.CloseCode, reason: Data?) { + os_log("WebSocket连接已关闭，关闭码: %d", log: log, type: .info, closeCode.rawValue) + isConnected = false + isConnecting = false + onConnectionStatusChanged?(.disconnected) + } +} + +// MARK: - URLSessionDelegate +extension RealtimeWebSocketManager: URLSessionDelegate { + + func urlSession(_ session: URLSession, didBecomeInvalidWithError error: Error?) { + if let error = error { + os_log("URLSession失效: %@", log: log, type: .error, error.localizedDescription) + onError?("会话失效: \(error.localizedDescription)") + } + } + + func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) { + if let error = error { + os_log("URLSessionTask完成时出错: %@", log: log, type: .error, error.localizedDescription) + if isConnected || isConnecting { + isConnected = false + isConnecting = false + onConnectionStatusChanged?(.error) + onError?("连接错误: \(error.localizedDescription)") + } + } + } +} \ No newline at end of file diff --git a/local_plugins/realtime/lib/realtime.dart b/local_plugins/realtime/lib/realtime.dart new file mode 100644 index 000000000..237e9e3e8 --- /dev/null +++ b/local_plugins/realtime/lib/realtime.dart @@ -0,0 +1,243 @@ +import 'dart:async'; +import 'dart:typed_data'; + +import 'package:flutter/services.dart'; + +/// 实时语音服务异常 +class RealtimeException implements Exception { + final String message; + + RealtimeException(this.message); + + @override + String toString() => 'RealtimeException: $message'; +} + +/// 连接状态 +enum ConnectionStatus { + disconnected, + connecting, + connected, + error, +} + +/// 语音状态 +enum VoiceStatus { + idle, + recording, + processing, + playing, +} + +/// 实时语音事件 +class RealtimeEvent { + final String type; + final dynamic data; + + RealtimeEvent({required this.type, this.data}); + + factory RealtimeEvent.fromMap(Map map) { + return RealtimeEvent( + type: map['type'] as String, + data: map['data'], + ); + } +} + +/// 实时语音聊天服务 +class RealtimeService { + static const MethodChannel _channel = MethodChannel('realtime/methods'); + static const EventChannel _eventChannel = EventChannel('realtime/events'); + + StreamController? _eventStreamController; + Stream? _eventStream; + + /// 获取事件流 + Stream get eventStream { + if (_eventStream == null) { + _eventStreamController = StreamController.broadcast(); + _eventStream = _eventStreamController!.stream; + + // 监听原生事件 + _eventChannel.receiveBroadcastStream().listen( + (dynamic event) { + if (event is Map) { + final eventMap = Map.from(event); + final realtimeEvent = RealtimeEvent.fromMap(eventMap); + _eventStreamController!.add(realtimeEvent); + } + }, + onError: (error) { + _eventStreamController!.addError(RealtimeException('事件流错误: $error')); + }, + ); + } + + return _eventStream!; + } + + /// 初始化实时语音服务 + /// + /// [serverUrl] WebSocket服务器地址 + /// [sampleRate] 采样率，默认16000 + /// [channels] 声道数，默认1（单声道） + /// [bitsPerSample] 位深，默认16 + Future initialize({ + required String serverUrl, + int sampleRate = 16000, + int channels = 1, + int bitsPerSample = 16, + }) async { + try { + final result = await _channel.invokeMethod( + 'initialize', + { + 'serverUrl': serverUrl, + 'sampleRate': sampleRate, + 'channels': channels, + 'bitsPerSample': bitsPerSample, + }, + ); + + return result ?? false; + } catch (e) { + throw RealtimeException('初始化失败: $e'); + } + } + + /// 连接到服务器 + Future connect() async { + try { + final result = await _channel.invokeMethod('connect'); + return result ?? false; + } catch (e) { + throw RealtimeException('连接失败: $e'); + } + } + + /// 断开连接 + Future disconnect() async { + try { + final result = await _channel.invokeMethod('disconnect'); + return result ?? false; + } catch (e) { + throw RealtimeException('断开连接失败: $e'); + } + } + + /// 开始录音 + Future startRecording() async { + try { + final result = await _channel.invokeMethod('startRecording'); + return result ?? false; + } catch (e) { + throw RealtimeException('开始录音失败: $e'); + } + } + + /// 停止录音 + Future stopRecording() async { + try { + final result = await _channel.invokeMethod('stopRecording'); + return result ?? false; + } catch (e) { + throw RealtimeException('停止录音失败: $e'); + } + } + + /// 停止播放 + Future stopPlaying() async { + try { + final result = await _channel.invokeMethod('stopPlaying'); + return result ?? false; + } catch (e) { + throw RealtimeException('停止播放失败: $e'); + } + } + + /// 获取连接状态 + Future getConnectionStatus() async { + try { + final result = await _channel.invokeMethod('getConnectionStatus'); + switch (result) { + case 'disconnected': + return ConnectionStatus.disconnected; + case 'connecting': + return ConnectionStatus.connecting; + case 'connected': + return ConnectionStatus.connected; + case 'error': + return ConnectionStatus.error; + default: + return ConnectionStatus.disconnected; + } + } catch (e) { + throw RealtimeException('获取连接状态失败: $e'); + } + } + + /// 获取语音状态 + Future getVoiceStatus() async { + try { + final result = await _channel.invokeMethod('getVoiceStatus'); + switch (result) { + case 'idle': + return VoiceStatus.idle; + case 'recording': + return VoiceStatus.recording; + case 'processing': + return VoiceStatus.processing; + case 'playing': + return VoiceStatus.playing; + default: + return VoiceStatus.idle; + } + } catch (e) { + throw RealtimeException('获取语音状态失败: $e'); + } + } + + /// 发送文本消息 + Future sendTextMessage(String message) async { + try { + final result = await _channel.invokeMethod( + 'sendTextMessage', + {'message': message}, + ); + return result ?? false; + } catch (e) { + throw RealtimeException('发送文本消息失败: $e'); + } + } + + /// 设置音频参数 + Future setAudioConfig({ + int? sampleRate, + int? channels, + int? bitsPerSample, + }) async { + try { + final params = {}; + if (sampleRate != null) params['sampleRate'] = sampleRate; + if (channels != null) params['channels'] = channels; + if (bitsPerSample != null) params['bitsPerSample'] = bitsPerSample; + + final result = await _channel.invokeMethod('setAudioConfig', params); + return result ?? false; + } catch (e) { + throw RealtimeException('设置音频参数失败: $e'); + } + } + + /// 释放资源 + Future dispose() async { + try { + await _channel.invokeMethod('dispose'); + _eventStreamController?.close(); + _eventStreamController = null; + _eventStream = null; + } catch (e) { + throw RealtimeException('释放资源失败: $e'); + } + } +} \ No newline at end of file diff --git a/local_plugins/realtime/pubspec.yaml b/local_plugins/realtime/pubspec.yaml new file mode 100644 index 000000000..0eed88a7d --- /dev/null +++ b/local_plugins/realtime/pubspec.yaml @@ -0,0 +1,29 @@ +name: realtime +description: 实时语音聊天插件，通过WebSocket连接Vocode服务器实现语音交互 +version: 0.0.1 +homepage: + +environment: + sdk: ">=2.17.0 <3.0.0" + flutter: ">=2.5.0" + +dependencies: + flutter: + sdk: flutter + +dev_dependencies: + flutter_test: + sdk: flutter + flutter_lints: ^2.0.0 + +# Flutter插件配置 +flutter: + plugin: + platforms: + android: + package: com.yunqiinnovation.realtime + pluginClass: RealtimePlugin + ios: + pluginClass: RealtimePlugin + swiftPackage: + path: ios/realtime \ No newline at end of file diff --git a/pubspec.yaml b/pubspec.yaml index 58beb5f36..fe2bd0ce1 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -12,6 +12,8 @@ dependencies: flutter_localizations: sdk: flutter cupertino_icons: ^1.0.6 + flutter_shaders: ^0.0.6 + get: ^4.7.2 json_annotation: ^4.9.0 get_storage: ^2.1.1 @@ -21,8 +23,8 @@ dependencies: audio_session: ^0.1.18 flutter_markdown: ^0.7.7+1 markdown: ^7.1.1 - permission_handler: ^12.0.0+1 - web_socket_channel: ^3.0.2 + permission_handler: ^10.4.5 + web_socket_channel: ^2.4.0 flutter_dotenv: ^5.1.0 synchronized: ^3.1.0 intl: ^0.20.2 @@ -89,6 +91,8 @@ dependencies: path: local_plugins/music_service amap_service: path: local_plugins/amap_service + realtime: + path: local_plugins/realtime test: path: local_plugins/test @@ -100,9 +104,15 @@ dev_dependencies: build_runner: ^2.3.3 # 用于生成代码 flutter pub run build_runner build json_serializable: ^6.6.0 # 用于自动生成代码 +dependency_overrides: + permission_handler: ^12.0.0 + permission_handler_android: ^13.0.0 + permission_handler_apple: ^9.4.0 + flutter: uses-material-design: true - + shaders: + - shaders/mic_flow.frag assets: - assets/images/headphone_light.png - assets/images/headphone_dark.png diff --git a/shaders/mic_flow.frag b/shaders/mic_flow.frag new file mode 100644 index 000000000..0b462df59 --- /dev/null +++ b/shaders/mic_flow.frag @@ -0,0 +1,70 @@ +// shaders/dialog_wave_core.frag +#version 300 es +precision highp float; + +#include + +uniform vec2 u_resolution; // setFloat(0, w) & (1, h) +uniform float u_time; // setFloat(2, t) +uniform float u_amp; // setFloat(3, amp) —— 建议传音量 RMS + +out vec4 fragColor; + +// ─────────── Noise Helpers ─────────── +float hash(vec2 p){return fract(sin(dot(p,vec2(127.1,311.7)))*43758.545);} +float noise(vec2 p){ + vec2 i=floor(p), f=fract(p); + vec2 u=f*f*(3.0-2.0*f); + return mix( + mix(hash(i), hash(i+vec2(1,0)),u.x), + mix(hash(i+vec2(0,1)), hash(i+vec2(1,1)),u.x),u.y); +} +float fbm(vec2 p){ + float a=.5, v=0.; + mat2 m=mat2(1.6,1.2,-1.2,1.6); + for(int i=0;i<3;i++){ + v+=a*noise(p); p=m*p*1.6; a*=.55; + } + return v; +} + +// ─────────── Palette ─────────── +vec3 palette(float t){ + vec3 deep = vec3(0.05,0.10,0.30); // 靛蓝 + vec3 mid = vec3(0.15,0.00,0.55); // 暗紫 + vec3 glow = vec3(0.00,0.60,1.00); // 电蓝 + return mix(deep, mix(mid, glow, t*t), t); // t² 让亮部偏蓝 +} + +void main(){ + // ① 归一化到 [-1,1]（保持比例） + vec2 uv = (FlutterFragCoord().xy / u_resolution)*2.0 - 1.0; + uv.x *= u_resolution.x / u_resolution.y; + float r = length(uv); // 半径 0~√2 + + // ② “呼吸”——音量驱动整体缩放（amp ∈ [0,1]） + float breath = 1.0 - (.08 + .05*u_amp) * sin(u_time*1.5); + vec2 p = uv / breath; // 越大越收缩 + p *= 3.0; // 纹理密度 + p += u_time * 0.35; // 缓慢横移 + + // ③ 旋转扭曲 + fBm 云雾 + float angle = atan(p.y,p.x) + 0.25*r; // 小角度漩涡 + float radius = length(p); + vec2 q = vec2(cos(angle),sin(angle))*radius; + float n = fbm(q) * .7 + fbm(q*2.0) * .3; + n = smoothstep(.3,.7,n); + + vec3 col = palette(n); + + // ④ 亮度脉冲（随 amp 加强） + col += (.05 + .08*u_amp) * sin(u_time*2.0 + n*6.0); + + // ⑤ 外环光晕（amp 加深） + float halo = smoothstep(.55,.05,r) * (0.15 + 0.5*u_amp); + col += halo; + + // ⑥ 羽化：可见区半径 ≈ 0.96 +float alpha = step(r, 0.75); +fragColor = vec4(col, alpha); +} \ No newline at end of file diff --git a/shaders/mic_flow2.frag b/shaders/mic_flow2.frag new file mode 100644 index 000000000..63e23dfe2 --- /dev/null +++ b/shaders/mic_flow2.frag @@ -0,0 +1,67 @@ +// shaders/mic_flow.frag +#version 300 es +precision highp float; + +#include + +uniform vec2 u_resolution; // 由 FlutterShaders 自动注入 +uniform float u_time; // AnimationController -> setFloat(2, ...) +uniform float u_amp; // 可选：实时音量 (0~1)，建议 setFloat(3, rms) + +out vec4 fragColor; + +// ——— 经典 hash & noise ——— +float hash(vec2 p) { + return fract(sin(dot(p, vec2(127.1,311.7))) * 43758.5453123); +} +float noise(vec2 p) { + vec2 i = floor(p), f = fract(p); + vec2 u = f * f * (3.0 - 2.0 * f); + return mix(mix(hash(i), hash(i + vec2(1.0,0.0)), u.x), + mix(hash(i + vec2(0.0,1.0)), hash(i + vec2(1.0,1.0)), u.x), + u.y); +} +// fBm：3-oct & 微 Domain Warp +float fbm(vec2 p) { + float sum = 0.0, amp = 0.5; + mat2 m = mat2(1.6, 1.2, + -1.2, 1.6); // 轻微旋转，防止轴纹 + for (int i = 0; i < 3; ++i) { + sum += amp * noise(p); + p = m * p * 1.6; + amp *= 0.55; + } + return sum; +} + +// 调色盘：深蓝 → 云白 +vec3 palette(float t) { + vec3 deep = vec3(0.08, 0.42, 1.00); + vec3 light= vec3(0.94, 0.98, 1.00); + return mix(deep, light, t); +} + +void main() { + // 0‒1 -> -0.5‒0.5，保持圆心在 Shader 0,0 + vec2 uv = (FlutterFragCoord().xy / u_resolution) - .5; + + // 呼吸缩放（≈ 0.92‒1.08） + float breath = 1.0 + (0.05 + 0.03 * u_amp) * sin(u_time * 1.4); + vec2 p = uv * 3.2 / breath; // 越大云越细腻 + p += u_time * 0.35; // 横向漂移 + + // 多层 fBm + 微距 domain-warp + float n = fbm(p) * 0.65 + fbm(p * 2.0 + vec2(5.2)) * 0.35; + float t = smoothstep(0.32, 0.68, n); // 阈值决定明暗分布 + vec3 col= palette(t); + + // ─── 边缘羽化 ─── + float r = length(uv); // 半径 0-0.5 + float alpha = smoothstep(0.48, 0.46, r); // 92% 处开始衰减 + + // 轻量亮度脉冲：让中心更有“呼吸”感 + float pulse = 0.04 * sin(u_time * 2.0 + n * 4.0); + col += pulse; + + fragColor = vec4(col, alpha); +} \ No newline at end of file