From 906234c4686f8443e66a967ca60c28e6070afdd5 Mon Sep 17 00:00:00 2001 From: wolfplus Date: Thu, 19 Jun 2025 17:21:56 +0100 Subject: [PATCH 01/10] add --- android/settings.gradle.kts | 4 +++- .../controllers/pairing_controller.dart | 21 ++++++++++++++-- .../agent_service/android/build.gradle.kts | 1 + .../agent_service/AgentService.kt | 24 +++++++++---------- 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/android/settings.gradle.kts b/android/settings.gradle.kts index cf7591c68..acc6518c9 100644 --- a/android/settings.gradle.kts +++ b/android/settings.gradle.kts @@ -40,6 +40,7 @@ include(":classic_bluetooth") include(":agent_service") include(":speech") include(":chat_api") +include(":bytedance_speech") // 设置azure_speech项目的路径 project(":azure_speech").projectDir = file("../local_plugins/azure_speech/android") @@ -50,4 +51,5 @@ project(":ble_service").projectDir = file("../local_plugins/ble_service/android" project(":classic_bluetooth").projectDir = file("../local_plugins/classic_bluetooth/android") project(":agent_service").projectDir = file("../local_plugins/agent_service/android") project(":speech").projectDir = file("../local_plugins/speech/android") -project(":chat_api").projectDir = file("../local_plugins/chat_api/android") \ No newline at end of file +project(":chat_api").projectDir = file("../local_plugins/chat_api/android") +project(":bytedance_speech").projectDir = file("../local_plugins/bytedance_speech/android") \ No newline at end of file diff --git a/lib/modules/pairing/controllers/pairing_controller.dart b/lib/modules/pairing/controllers/pairing_controller.dart index 34574e0f0..47e2a00b6 100644 --- a/lib/modules/pairing/controllers/pairing_controller.dart +++ b/lib/modules/pairing/controllers/pairing_controller.dart @@ -506,7 +506,15 @@ class PairingController extends GetxController { /// 导航到主页 void _navigateToHome() { - Future.delayed(const Duration(seconds: 1), () { + Future.delayed(const Duration(seconds: 1), () async { + // 启动AgentService服务 + try { + Logger.i(_tag, '启动AgentService服务'); + await _bleManager.startAgentService(); + } catch (e) { + Logger.e(_tag, '启动AgentService失败: $e'); + } + Get.offAllNamed(Routes.home); }); } @@ -563,8 +571,17 @@ class PairingController extends GetxController { } /// 用户点击跳过按钮 - void onSkipTap() { + void onSkipTap() async { Logger.i(_tag, '用户选择跳过配对'); + + // 启动AgentService服务 + try { + Logger.i(_tag, '启动AgentService服务'); + await _bleManager.startAgentService(); + } catch (e) { + Logger.e(_tag, '启动AgentService失败: $e'); + } + Get.offAllNamed(Routes.home); } diff --git a/local_plugins/agent_service/android/build.gradle.kts b/local_plugins/agent_service/android/build.gradle.kts index 9d8dea1bb..d193bfbae 100644 --- a/local_plugins/agent_service/android/build.gradle.kts +++ b/local_plugins/agent_service/android/build.gradle.kts @@ -50,4 +50,5 @@ dependencies { add("compileOnly", project(":speech")) add("compileOnly", project(":chat_api")) add("compileOnly", project(":location_service")) + add("compileOnly", project(":bytedance_speech")) } \ No newline at end of file diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index 00055f5eb..009ccd4ba 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -19,7 +19,7 @@ import android.media.MediaPlayer import com.deep_voice.speech.tts.TtsEvent import com.deep_voice.speech.tts.TtsEventListener import com.deep_voice.speech.tts.TtsEventType -import com.yunqiinnovation.azure_speech.AzureTtsHelper +import com.deep_voice.bytedance_speech.BytedanceTTS /** * 代理服务事件监听接口 */ @@ -52,8 +52,8 @@ object AgentService : CoroutineScope { // Azure服务 private var azureAsrHelper: AzureAsrHelper? = null - // 使用AzureTtsHelper作为唯一的TTS实现 - private var ttsService: AzureTtsHelper? = null + // 使用BytedanceTTS作为唯一的TTS实现 + private var ttsService: BytedanceTTS? = null // ChatAPI服务 - 使用新的ChatApiService private lateinit var chatApiService: ChatApiService @@ -216,20 +216,20 @@ object AgentService : CoroutineScope { val ttsAppToken = config["volcanoToken"]?.toString() ?: "" val ttsLanguage = config["ttsLanguage"]?.toString() ?: "zh-CN" - // 创建并初始化AzureTtsHelper - val azureTtsHelper = AzureTtsHelper(context) - ttsService = azureTtsHelper + // 创建并初始化BytedanceTTS + val bytedanceTts = BytedanceTTS(context) + ttsService = bytedanceTts - // 初始化Azure TTS - val success = azureTtsHelper.initialize( - ttsAppId = "", // Azure TTS不需要appId - ttsAppToken = config["azureSpeechKey"]?.toString() ?: "", // Azure需要subscription key - ttsResource = config["azureSpeechRegion"]?.toString() ?: "", // Azure需要region信息 + // 初始化Bytedance TTS + val success = bytedanceTts.initialize( + ttsAppId = ttsAppId, + ttsAppToken = ttsAppToken, + ttsResource = "", // Bytedance TTS不需要resource参数 language = ttsLanguage ) if (!success) { - Log.e(TAG, "Azure TTS服务初始化失败") + Log.e(TAG, "Bytedance TTS服务初始化失败") } // 添加TTS事件监听 From 87d5fbaa937e9cfe509e4a10df638e04fe582060 Mon Sep 17 00:00:00 2001 From: fdp <1286779656@qq.com> Date: Sat, 21 Jun 2025 19:23:19 +0800 Subject: [PATCH 02/10] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BC=9A=E8=AE=AE?= =?UTF-8?q?=E5=8A=A9=E6=89=8B=EF=BC=8C=E7=BF=BB=E8=AF=91=E4=B8=8D=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=E6=96=87=E4=BB=B6=E5=8D=A1=E4=BD=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/modules/FTFTranslation/README.md | 160 ++++++++++++ .../bindings/FTFTranslation_binding.dart | 10 + .../FTFTranslation_controller.dart | 241 ++++++++++++++++++ .../views/FTFTranslation_view.dart | 212 +++++++++++++++ .../controllers/meeting_controller.dart | 7 +- .../meeting_record_controller.dart | 5 +- lib/modules/meeting/views/meeting_view.dart | 88 ++++--- lib/modules/settings/views/settings_view.dart | 22 ++ .../controllers/translation_controller.dart | 8 +- .../translation/views/translation_view.dart | 4 +- lib/routes/app_pages.dart | 9 + lib/routes/app_routes.dart | 1 + .../yunqiinnovation/ble_service/BleService.kt | 129 +++++----- 13 files changed, 780 insertions(+), 116 deletions(-) create mode 100644 lib/modules/FTFTranslation/README.md create mode 100644 lib/modules/FTFTranslation/bindings/FTFTranslation_binding.dart create mode 100644 lib/modules/FTFTranslation/controllers/FTFTranslation_controller.dart create mode 100644 lib/modules/FTFTranslation/views/FTFTranslation_view.dart diff --git a/lib/modules/FTFTranslation/README.md b/lib/modules/FTFTranslation/README.md new file mode 100644 index 000000000..c41004a52 --- /dev/null +++ b/lib/modules/FTFTranslation/README.md @@ -0,0 +1,160 @@ +# Realtime 实时语音聊天模块 + +## ⚠️ 当前状态 + +**此模块为功能预览版本，提供完整的实时语音对话功能。** + +通过原生插件实现语音识别、AI对话和语音合成的完整语音交互体验。 + +## 概述 + +Realtime模块实现了类似ChatGPT app的实时语音对话功能，当前版本提供： + +- ✅ **完整语音交互** - 实时语音识别、AI对话和语音合成 +- ✅ **状态管理** - 聆听、思考、回答状态的可视化 +- ✅ **动画效果** - 基于Shader的流动云效果，支持音频振幅驱动 +- ✅ **字幕显示** - 可选择开启/关闭的字幕模式 +- ✅ **智能UI** - 根据连接状态自动调整按钮可用性 + +## 界面设计 + +### 布局结构 +- **顶部状态区域** - 显示当前状态和字幕模式切换按钮 +- **中间可视化区域** - 圆形Shader动画，根据音频振幅呈现呼吸效果 +- **底部控制区域** - 主控制按钮和状态指示器 + +### 状态指示 +- 🔵 **聆听状态** - 蓝色动画，接收用户语音输入 +- 🟠 **思考状态** - 橙色动画，AI处理中 +- 🟢 **回答状态** - 绿色动画，AI语音输出 +- ⚪ **空闲状态** - 灰色静态，等待交互 + +### 字幕模式 +- **默认关闭** - 初始状态下不显示任何字幕内容 +- **手动切换** - 通过右上角按钮手动开启/关闭字幕 +- **完整隐藏** - 关闭时字幕区域完全不显示，不占用界面空间 +- **智能布局** - 开启时圆形动画自动上移，字幕区域占据下半屏 + +### 交互控制 +- **录音按钮** - 仅在成功连接AI服务器且开始录音后可用 +- **连接状态** - 未连接时录音按钮显示不可用状态 +- **状态反馈** - 按钮颜色和可点击性根据连接状态动态调整 + +## 使用方法 + +### 路由导航 + +```dart +// 跳转到实时语音对话页面 +Get.toNamed(Routes.realtime); +``` + +### 交互流程 + +1. **自动连接** - 页面加载后自动尝试连接AI服务器 +2. **等待连接** - 录音按钮在连接成功前保持不可用状态 +3. **开始录音** - 连接成功后自动开始录音，录音按钮变为可用 +4. **语音对话** - 用户说话 → AI处理 → 语音回复 +5. **字幕控制** - 可随时通过右上角按钮切换字幕显示 + +## 技术架构 + +### 控制器 (RealtimeController) +- **连接管理**: WebSocket连接到AI服务器 +- **状态同步**: `isConnected`, `isListening`, `isSpeaking` +- **字幕控制**: `isSubtitleMode` 默认为false，用户手动控制 +- **消息管理**: 智能去重，避免流式响应重复显示 +- **音频处理**: 原生层计算RMS值，Flutter层接收并控制动画 + +### 视图组件 +- **RealtimeView** - 主视图容器，响应式布局 +- **ShaderMicFlow** - 基于GLSL Shader的流动云动画组件 +- **AutoScrollList** - 自动滚动的字幕列表 + +### 原生集成 +- **RealtimeService** - 原生音频处理和WebSocket通信 +- **音频格式** - 16kHz, 1声道, 16位PCM +- **实时传输** - 低延迟音频流处理 +- **RMS计算** - 原生层实时计算音频振幅 + +## 默认行为 + +### 字幕模式 +- **初始状态**: 关闭 (`isSubtitleMode.value = false`) +- **显示逻辑**: 仅在用户手动开启时显示字幕内容 +- **空间占用**: 关闭时字幕区域高度为0，不影响其他UI元素 +- **切换效果**: 平滑动画过渡，圆形动画位置和大小同步调整 + +### 按钮状态 +- **录音按钮**: 默认不可用，连接成功且开始录音后才可点击 +- **状态指示**: 通过颜色和图标变化反映当前可用性 +- **交互反馈**: 不可用时点击无效果，避免误操作 + +## 开发计划 + +### 已完成功能 ✅ +- [x] 完整语音交互流程 +- [x] 实时WebSocket通信 +- [x] Shader流动云动画 +- [x] RMS音频振幅计算 +- [x] 智能UI状态管理 +- [x] 字幕模式控制 +- [x] 消息去重优化 + +### 计划增强 ⏳ +- [ ] 多语言语音识别 +- [ ] 语音中断检测 +- [ ] 音频质量自适应 +- [ ] 离线降级处理 +- [ ] 语音情感识别 + +## 注意事项 + +1. **网络依赖** - 需要连接到AI服务器 (ws://192.168.1.11:8000/ws) +2. **麦克风权限** - 需要获取麦克风使用权限 +3. **音频会话** - 自动管理音频会话，支持蓝牙设备 +4. **资源管理** - 页面关闭时自动清理音频资源和网络连接 + +## 自定义配置 + +### 修改服务器地址 + +```dart +// 在RealtimeController中修改连接URL +await _realtimeService.initialize( + serverUrl: 'ws://your-server:port/ws', + // ... 其他参数 +); +``` + +### 调整UI布局 + +```dart +// 修改字幕模式下圆形动画位置 +alignment: controller.isSubtitleMode.value + ? const Alignment(0, -0.9) // 调整Y轴位置 + : const Alignment(0, -0.25), +``` + +### 音频参数调优 + +```dart +// 在初始化时调整音频参数 +sampleRate: 16000, // 采样率 +channels: 1, // 声道数 +bitsPerSample: 16, // 位深度 +``` + +## 技术栈 + +- **Flutter SDK**: >= 3.0.0 +- **GetX**: >= 4.6.5 (状态管理) +- **Flutter Shaders**: >= 0.0.6 (Shader动画) +- **原生插件**: Realtime服务 (音频处理) +- **WebSocket**: 实时通信协议 + +## 版本信息 + +- **当前版本**: v1.0.0 (Production) +- **最后更新**: 2024年 +- **状态**: 完整功能实现 \ No newline at end of file diff --git a/lib/modules/FTFTranslation/bindings/FTFTranslation_binding.dart b/lib/modules/FTFTranslation/bindings/FTFTranslation_binding.dart new file mode 100644 index 000000000..85e44791c --- /dev/null +++ b/lib/modules/FTFTranslation/bindings/FTFTranslation_binding.dart @@ -0,0 +1,10 @@ +import 'package:get/get.dart'; +import '../controllers/FTFTranslation_controller.dart'; + +class FTFTranslationBinding extends Bindings { + @override + void dependencies() { + // 注册实时语音聊天控制器 + Get.lazyPut(() => FTFTranslationController()); + } +} diff --git a/lib/modules/FTFTranslation/controllers/FTFTranslation_controller.dart b/lib/modules/FTFTranslation/controllers/FTFTranslation_controller.dart new file mode 100644 index 000000000..53cffe68d --- /dev/null +++ b/lib/modules/FTFTranslation/controllers/FTFTranslation_controller.dart @@ -0,0 +1,241 @@ +import 'dart:async'; +import 'dart:typed_data'; + +import 'package:get/get.dart'; +import '../../../core/utils/logger.dart'; +import '../../../data/services/asr_service.dart'; +import '../../../data/services/speech_impl/xunfei_asr_service.dart'; +import '../../../data/services/tts_service.dart'; +import '../../../data/services/language_manager.dart'; +import '../../../data/services/volcano_translation_service.dart'; + +class FTFTranslationController extends GetxController + with GetSingleTickerProviderStateMixin { + final messages = [].obs; + final isChineseInput = true.obs; + final leftLanguage = '中文'.obs; + final rightLanguage = 'English'.obs; + final isLeftInput = true.obs; + + final supportedLanguages = const [ + '中文', + 'English', + '日本語', + '한국어', + 'Español', + ]; + + late final AsrService _asrService; + + StreamSubscription? _recognitionSubscription; + + RxString intermediateContent = ''.obs; + RxString finalContent = ''.obs; + BytesBuilder _originaBytes = BytesBuilder(); //原始音频 + + final VolcanoTranslationService _translationService = + Get.find(); + final LanguageManager _languageManager = Get.find(); + // 当前使用的TTS服务 + late TtsService _ttsService; + final isTranslating = false.obs; + final sourceLanguageCode = 'zh-CN'.obs; + final targetLanguageCode = 'en-US'.obs; + final isTtsEnabled = true.obs; + + @override + void onInit() { + super.onInit(); + Get.put(XunfeiAsrService()); + final TtsService _ttsService = Get.find(); + _asrService = Get.find(); + } + + @override + void onClose() { + _asrService.dispose(); + _recognitionSubscription?.cancel(); + super.onClose(); + } + + //开始录音 + Future _startRecorder() async { + final recognitionStream = + await _asrService.startContinuousRecognition(false); + _recognitionSubscription = recognitionStream.listen( + _handleRecognitionEvent, + ); + } + + //结束录音 + Future _stopRecorder() async { + await _asrService.stopContinuousRecognition(); + } + + // 处理语音识别事件 + void _handleRecognitionEvent(RecognitionEvent event) { + switch (event.type) { + case RecognitionEventType.intermediateResult: + intermediateContent.value = event.text; + handleFinalResult(event.text); + break; + case RecognitionEventType.finalResult: + finalContent.value += _spliceText(event); + intermediateContent.value = ''; + _originaBytes.add(event.audio!); + handleIntermediateResult(event.text); + break; + default: + // 不做任何处理 + break; + } + } + + // 翻译文本 + Future translateText(String sourceText, + {bool isFinal = false}) async { + if (sourceText.isEmpty) return null; + + isTranslating.value = true; + String? translationResult; + + try { + // 根据检测到的语言确定源语言和目标语言 + final bool shouldSwap = + intermediateContent.value == sourceLanguageCode.value; + final detectedSourceLanguageCode = + shouldSwap ? sourceLanguageCode.value : targetLanguageCode.value; + final detectedTargetLanguageCode = + shouldSwap ? targetLanguageCode.value : sourceLanguageCode.value; + + // 调用翻译服务 + translationResult = await _translationService.translateText( + text: sourceText, + sourceLanguageCode: detectedSourceLanguageCode, + targetLanguageCode: detectedTargetLanguageCode, + ); + + if (translationResult != null) { + // 查找匹配的项目 - 优先使用timestamp查找 + int index = -1; + + // 播放TTS - 只在最终结果时播放 + if (isTtsEnabled.value && isFinal) { + // Logger.info('播放翻译文本: ${translationResult}'); + await playTranslatedText(translationResult); + } + } + } catch (e) { + Logger.error('翻译失败: ${e.toString()}'); + } finally { + isTranslating.value = false; + } + + return translationResult; + } + + // 处理中间识别结果 + void handleIntermediateResult(String text) { + if (text.isEmpty) return; + + translateText(text, isFinal: false); + } + + // 处理最终识别结果 + Future handleFinalResult(String text) async { + if (text.isEmpty) return; + + // 使用项目的时间戳进行翻译 + translateText(text, isFinal: true); + } + + // 播放翻译文本 + Future playTranslatedText(String text) async { + if (text.isEmpty) return; + + try { + final bool shouldSwap = + intermediateContent.value == targetLanguageCode.value; + final detectedTargetLanguageCode = + shouldSwap ? sourceLanguageCode.value : targetLanguageCode.value; + final voiceName = _languageManager + .getTtsVoiceNameByAsrCode(detectedTargetLanguageCode) ?? + 'en-US-AriaNeural'; + + await _ttsService.setVoice(voiceName); + await _ttsService.speakOnce(text); + } catch (e) { + Logger.error('播放翻译失败: ${e.toString()}'); + } + } + + String _spliceText(RecognitionEvent event) { + String eventText = event.text; + String text = ''; + if (finalContent.value.isNotEmpty) { + if (event.role != '') { + final punctuations = { + '，', + '。', + '！', + '？', + '；', + '：', + ',', + '.', + ':', + ';', + '!', + '?' + }; + final firstChar = eventText[0]; + if (punctuations.contains(firstChar)) { + text += '$firstChar\n'; + eventText = eventText.substring(1); + } else { + text += '\n'; + } + text += '说话人${event.role}：'; + } + } else { + text += '说话人${event.role}：'; + } + text += eventText; + return text; + } + + void addMessage(String text) { + final fromLang = + isLeftInput.value ? leftLanguage.value : rightLanguage.value; + final toLang = isLeftInput.value ? rightLanguage.value : leftLanguage.value; + + messages.add( + FTFTranslationMessage( + original: text, + translated: '[$fromLang → $toLang] $text (翻译模拟)', + isChineseSpeaker: isLeftInput.value, + ), + ); + } + + void swapLanguages() { + final temp = leftLanguage.value; + leftLanguage.value = rightLanguage.value; + rightLanguage.value = temp; + isLeftInput.value = !isLeftInput.value; + } + + void toggleInputSide(bool isLeft) => isLeftInput.value = isLeft; +} + +class FTFTranslationMessage { + final String original; + final String translated; + final bool isChineseSpeaker; + + FTFTranslationMessage({ + required this.original, + required this.translated, + required this.isChineseSpeaker, + }); +} diff --git a/lib/modules/FTFTranslation/views/FTFTranslation_view.dart b/lib/modules/FTFTranslation/views/FTFTranslation_view.dart new file mode 100644 index 000000000..84d8c1c6a --- /dev/null +++ b/lib/modules/FTFTranslation/views/FTFTranslation_view.dart @@ -0,0 +1,212 @@ +import 'package:flutter/material.dart'; +import 'package:get/get.dart'; +import '../controllers/FTFTranslation_controller.dart'; + +class FTFTranslationView extends GetView { + const FTFTranslationView({Key? key}) : super(key: key); + + @override + Widget build(BuildContext context) { + final TextEditingController inputController = TextEditingController(); + + return Scaffold( + backgroundColor: const Color(0xFF0E0F12), + appBar: AppBar( + backgroundColor: Colors.black, + title: const Text('面对面翻译'), + centerTitle: true, + ), + body: Column( + children: [ + Expanded( + child: Obx(() => ListView.builder( + padding: const EdgeInsets.all(16), + itemCount: controller.messages.length, + itemBuilder: (context, index) { + final msg = controller.messages[index]; + final isChinese = msg.isChineseSpeaker; + return Align( + alignment: isChinese + ? Alignment.centerLeft + : Alignment.centerRight, + child: Container( + margin: const EdgeInsets.symmetric(vertical: 6), + padding: const EdgeInsets.all(12), + decoration: BoxDecoration( + color: isChinese ? Colors.blue : Colors.grey[800], + borderRadius: BorderRadius.circular(12), + ), + constraints: const BoxConstraints(maxWidth: 300), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + msg.original, + style: const TextStyle( + color: Colors.white, + fontWeight: FontWeight.bold, + ), + ), + const SizedBox(height: 4), + Text( + msg.translated, + style: TextStyle( + color: Colors.white.withOpacity(0.9), + fontSize: 14, + ), + ), + ], + ), + ), + ); + }, + )), + ), + _buildBottomInput(inputController), + ], + ), + ); + } + + Widget _buildBottomInput(TextEditingController inputController) { + return Obx(() { + final isLeftInput = controller.isLeftInput.value; + final inputLang = isLeftInput + ? controller.leftLanguage.value + : controller.rightLanguage.value; + + return Container( + color: Colors.black, + padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 8), + child: Column( + children: [ + Row( + children: [ + Expanded( + child: TextField( + controller: inputController, + style: const TextStyle(color: Colors.white), + decoration: InputDecoration( + filled: true, + fillColor: Colors.grey[900], + hintText: '请输入 $inputLang 内容', + hintStyle: const TextStyle(color: Colors.grey), + border: OutlineInputBorder( + borderRadius: BorderRadius.circular(8), + borderSide: BorderSide.none, + ), + contentPadding: + const EdgeInsets.symmetric(horizontal: 12), + ), + ), + ), + IconButton( + onPressed: () { + final text = inputController.text.trim(); + if (text.isNotEmpty) { + controller.addMessage(text); + inputController.clear(); + } + }, + icon: const Icon(Icons.send, color: Colors.white), + ), + ], + ), + const SizedBox(height: 8), + Row( + children: [ + Expanded( + child: _buildLanguageDropdown( + selected: controller.leftLanguage.value, + items: controller.supportedLanguages, + onChanged: (value) { + if (value != null) controller.leftLanguage.value = value; + }, + icon: Icons.headphones, + isSelected: isLeftInput, + label: '左方语言', + onTapInputSide: () => controller.toggleInputSide(true), + ), + ), + const SizedBox(width: 12), + GestureDetector( + onTap: controller.swapLanguages, + child: Container( + padding: const EdgeInsets.all(8), + decoration: BoxDecoration( + color: Colors.white10, + shape: BoxShape.circle, + ), + child: + const Icon(Icons.compare_arrows, color: Colors.white), + ), + ), + const SizedBox(width: 12), + Expanded( + child: _buildLanguageDropdown( + selected: controller.rightLanguage.value, + items: controller.supportedLanguages, + onChanged: (value) { + if (value != null) controller.rightLanguage.value = value; + }, + icon: Icons.smartphone, + isSelected: !isLeftInput, + label: '右方语言', + onTapInputSide: () => controller.toggleInputSide(false), + ), + ), + ], + ), + ], + ), + ); + }); + } + + Widget _buildLanguageDropdown({ + required String selected, + required List items, + required void Function(String?) onChanged, + required IconData icon, + required bool isSelected, + required String label, + required VoidCallback onTapInputSide, + }) { + return GestureDetector( + onTap: onTapInputSide, + child: Container( + padding: const EdgeInsets.symmetric(vertical: 8, horizontal: 12), + decoration: BoxDecoration( + border: Border.all( + color: isSelected ? Colors.white : Colors.grey, + ), + borderRadius: BorderRadius.circular(24), + color: isSelected ? Colors.white10 : Colors.transparent, + ), + child: Row( + children: [ + Icon(icon, color: Colors.white), + const SizedBox(width: 8), + Expanded( + child: DropdownButton( + dropdownColor: Colors.grey[900], + value: selected, + isExpanded: true, + underline: const SizedBox(), + iconEnabledColor: Colors.white, + style: const TextStyle(color: Colors.white), + items: items.map((lang) { + return DropdownMenuItem( + value: lang, + child: Text(lang), + ); + }).toList(), + onChanged: onChanged, + ), + ), + ], + ), + ), + ); + } +} diff --git a/lib/modules/meeting/controllers/meeting_controller.dart b/lib/modules/meeting/controllers/meeting_controller.dart index e38cba917..0058b31d8 100644 --- a/lib/modules/meeting/controllers/meeting_controller.dart +++ b/lib/modules/meeting/controllers/meeting_controller.dart @@ -14,6 +14,7 @@ import 'package:permission_handler/permission_handler.dart'; import '../../../core/utils/logger.dart'; import '../../../core/utils/synchrodata.dart'; import '../../../core/utils/upload_oss.dart'; +import '../../../data/services/ble_manager.dart'; import '../../../data/services/db/sqflite_api.dart'; import '../../../core/utils/permission_util.dart'; @@ -37,7 +38,8 @@ class MeetingController extends GetxController { TextEditingController titleController = TextEditingController(); // 录音标题 RxList dataList = [].obs; - + // 蓝牙服务 + final bleManager = Get.find(); List dataListTask = []; bool isDisposed = false; String filePath = ''; @@ -313,6 +315,9 @@ class MeetingController extends GetxController { PlatformFile file = result.files.first; filePath = await _saveFile(file); fileName = file.name; + } else { + EasyLoading.dismiss(); + return; } } diff --git a/lib/modules/meeting/controllers/meeting_record_controller.dart b/lib/modules/meeting/controllers/meeting_record_controller.dart index 2fea38ea5..ee4640b18 100644 --- a/lib/modules/meeting/controllers/meeting_record_controller.dart +++ b/lib/modules/meeting/controllers/meeting_record_controller.dart @@ -113,12 +113,15 @@ class MeetingRecordController extends GetxController switch (audioType.value) { case 0: fileName.value = "现场录音"; + _audioSourceType = false; break; case 1: fileName.value = "音、视频录音"; + _audioSourceType = true; break; case 2: fileName.value = "通话录音"; + _audioSourceType = true; break; } newName.value = fileName.value; @@ -234,7 +237,7 @@ class MeetingRecordController extends GetxController void _startHardwareServices() { switch (audioType.value) { case 0: - _bleManager.openEncoder(); + // _bleManager.openEncoder(); break; case 1: _bleManager.openDecoder(); diff --git a/lib/modules/meeting/views/meeting_view.dart b/lib/modules/meeting/views/meeting_view.dart index 35d34e6d8..2a1c2c7e9 100644 --- a/lib/modules/meeting/views/meeting_view.dart +++ b/lib/modules/meeting/views/meeting_view.dart @@ -454,43 +454,43 @@ class MeetingView extends GetView { // TODO: 实现从文件导入逻辑 }, ), - SizedBox(height: 8.w), - _importAudioItem1( - context, - isDarkMode, - icon: Image.asset('assets/images/explore.png', - width: 32.w, height: 32.w), - title: '从翻译里导入', - onTap: () { - Navigator.of(context).pop(); - _showLocalAudioVideoPicker(context, isDarkMode); - // TODO: 实现从相册导入逻辑 - }, - ), - SizedBox(height: 8.w), - _importAudioItem1( - context, - isDarkMode, - icon: Row( - mainAxisSize: MainAxisSize.min, - children: [ - Image.asset('assets/images/explore.png', - width: 20.w, height: 20.w), - SizedBox(width: 4.w), - Image.asset('assets/images/explore.png', - width: 20.w, height: 20.w), - SizedBox(width: 4.w), - Image.asset('assets/images/explore.png', - width: 20.w, height: 20.w), - ], - ), - title: '从opus导出', - onTap: () { - Navigator.of(context).pop(); - // TODO: 实现从其他APP导入逻辑 - _showLocalOpusPicker(context, isDarkMode); - }, - ), + // SizedBox(height: 8.w), + // _importAudioItem1( + // context, + // isDarkMode, + // icon: Image.asset('assets/images/explore.png', + // width: 32.w, height: 32.w), + // title: '从翻译里导入', + // onTap: () { + // Navigator.of(context).pop(); + // _showLocalAudioVideoPicker(context, isDarkMode); + // // TODO: 实现从相册导入逻辑 + // }, + // ), + // SizedBox(height: 8.w), + // _importAudioItem1( + // context, + // isDarkMode, + // icon: Row( + // mainAxisSize: MainAxisSize.min, + // children: [ + // Image.asset('assets/images/explore.png', + // width: 20.w, height: 20.w), + // SizedBox(width: 4.w), + // Image.asset('assets/images/explore.png', + // width: 20.w, height: 20.w), + // SizedBox(width: 4.w), + // Image.asset('assets/images/explore.png', + // width: 20.w, height: 20.w), + // ], + // ), + // title: '从opus导出', + // onTap: () { + // Navigator.of(context).pop(); + // // TODO: 实现从其他APP导入逻辑 + // _showLocalOpusPicker(context, isDarkMode); + // }, + // ), ], ), ), @@ -578,13 +578,12 @@ class MeetingView extends GetView { SizedBox(height: 8.w), // 音、视频录音 - 根据控制器状态禁用 _importAudioItem( - context, - isDarkMode, + context, isDarkMode, icon: Image.asset('assets/images/explore.png', width: 32.w, height: 32.w), title: '音、视频录音', - enabled: false, // 依赖控制器状态 - onTap: false + enabled: controller.bleManager.isConnected, // 依赖控制器状态 + onTap: controller.bleManager.isConnected ? () async { // 条件启用 Navigator.of(context).pop(); @@ -599,14 +598,13 @@ class MeetingView extends GetView { SizedBox(height: 8.w), // 通话录音 - 根据控制器状态禁用 _importAudioItem( - context, - isDarkMode, + context, isDarkMode, icon: Row( // ... 图标组合保持不变 ), title: '通话录音', - enabled: false, // 依赖控制器状态 - onTap: false + enabled: controller.bleManager.isConnected, // 依赖控制器状态 + onTap: controller.bleManager.isConnected ? () async { // 条件启用 Navigator.of(context).pop(); diff --git a/lib/modules/settings/views/settings_view.dart b/lib/modules/settings/views/settings_view.dart index 35bb6c90b..777f8c48f 100644 --- a/lib/modules/settings/views/settings_view.dart +++ b/lib/modules/settings/views/settings_view.dart @@ -543,6 +543,28 @@ class SettingsView extends GetView { }, isDarkMode: isDarkMode, ), + Divider( + height: 1, + color: isDarkMode + ? Colors.white.withOpacity(0.1) + : Colors.grey[200]), + // 面对面翻译测试 + _buildSimpleNavigationSetting( + title: '面对面翻译测试', + subtitle: '测试面对面翻译', + icon: Icons.bluetooth_searching, + iconBgColor: isDarkMode + ? Colors.green[900]!.withOpacity(0.3) + : Colors.green[100]!, + iconColor: + isDarkMode ? Colors.green[300]! : Colors.green[600]!, + titleColor: isDarkMode ? Colors.white : null, + subtitleColor: isDarkMode ? Colors.white70 : null, + onTap: () { + Get.toNamed(Routes.ftftranslation); + }, + isDarkMode: isDarkMode, + ), Divider( height: 1, color: isDarkMode diff --git a/lib/modules/translation/controllers/translation_controller.dart b/lib/modules/translation/controllers/translation_controller.dart index 9a911accf..d83b04bdd 100644 --- a/lib/modules/translation/controllers/translation_controller.dart +++ b/lib/modules/translation/controllers/translation_controller.dart @@ -26,7 +26,7 @@ class TranslationController extends GetxController { final LanguageManager _languageManager = Get.find(); final GetStorage _storage = GetStorage(); // 蓝牙服务 - final _bleManager = Get.find(); + final bleManager = Get.find(); // 音频输入源 bool _audioSourceType = false; // 存储相关 @@ -308,14 +308,14 @@ class TranslationController extends GetxController { // translationHistory.add(newItem); // 开始连续语音识别 if (currentMode.value == "audioVideo") { - _bleManager.openDecoder(); + bleManager.openDecoder(); // 发送ble音乐或者通话远端声音 _audioSourceType = true; isTtsEnabled.value = false; // 禁用TTS以避免干扰 Logger.info('发送ble音乐或者通话远端声音'); } else if (currentMode.value == "call") { // 发送ble系统mic和dac（音乐或者通话远端）声音 - _bleManager.openA2DPDecoder(); + bleManager.openA2DPDecoder(); _audioSourceType = true; isTtsEnabled.value = true; Logger.info('发送ble系统mic和dac（音乐或者通话远端）声音'); @@ -563,7 +563,7 @@ class TranslationController extends GetxController { Future stopAll() async { try { if (currentMode.value == "audioVideo" || currentMode.value == "call") { - _bleManager.closeCodec(); + bleManager.closeCodec(); // 发送ble音乐或者通话远端声音 Logger.info('发送ble音乐或者通话远端声音'); } diff --git a/lib/modules/translation/views/translation_view.dart b/lib/modules/translation/views/translation_view.dart index e8eaf3d3e..048f5f463 100644 --- a/lib/modules/translation/views/translation_view.dart +++ b/lib/modules/translation/views/translation_view.dart @@ -67,12 +67,12 @@ class TranslationView extends GetView { _buildPopupMenuItem( 'audioVideo', 'audioVideoTranslation'.tr, - enabled: true, // 根据连接状态启用/禁用 + enabled: controller.bleManager.isConnected, // 根据连接状态启用/禁用 ), _buildPopupMenuItem( 'call', 'callTranslation'.tr, - enabled: true, // 根据连接状态启用/禁用 + enabled: controller.bleManager.isConnected, // 根据连接状态启用/禁用 ), ], child: Row( diff --git a/lib/routes/app_pages.dart b/lib/routes/app_pages.dart index 66c5f2516..8f3a14799 100644 --- a/lib/routes/app_pages.dart +++ b/lib/routes/app_pages.dart @@ -46,6 +46,8 @@ import '../modules/realtime/bindings/realtime_binding.dart'; import '../modules/realtime/views/realtime_view.dart'; import '../modules/music/views/music_playlist_view.dart'; import '../modules/music/bindings/music_playlist_binding.dart'; +import '../modules/FTFTranslation/bindings/FTFTranslation_binding.dart'; +import '../modules/FTFTranslation/views/FTFTranslation_view.dart'; abstract class AppPages { static final pages = [ @@ -191,6 +193,13 @@ abstract class AppPages { transition: Transition.rightToLeft, transitionDuration: Duration(milliseconds: 250), ), + GetPage( + name: Routes.ftftranslation, + page: () => const FTFTranslationView(), + binding: FTFTranslationBinding(), + transition: Transition.noTransition, + transitionDuration: Duration(milliseconds: 250), + ), GetPage( name: Routes.opusTest, page: () => const OpusTestView(), diff --git a/lib/routes/app_routes.dart b/lib/routes/app_routes.dart index 3e2e1be89..550ab16cc 100644 --- a/lib/routes/app_routes.dart +++ b/lib/routes/app_routes.dart @@ -28,6 +28,7 @@ abstract class Routes { static const translationHistory = '/translation/history'; // 翻译历史 static const speechDemo = '/speech_demo'; // 语音演示 static const translationList = '/translation/List'; // 翻译历史 + static const ftftranslation = '/ftftranslation'; // 面对面翻译 // 测试路由（生产环境建议移除） static const ttsTest = '/tts_test'; // 文字转语音测试 static const asrTest = '/asr_test'; // 语音识别测试 diff --git a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt index 55eb87911..2e9d93f87 100644 --- a/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt +++ b/local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt @@ -22,6 +22,7 @@ import java.util.concurrent.LinkedBlockingQueue import android.os.Handler import java.util.concurrent.atomic.AtomicBoolean import android.util.Log + /** * BLE服务类：提供蓝牙低功耗设备的扫描、连接和通信功能 * 支持： @@ -67,17 +68,20 @@ object BleService { private var bluetoothLeScanner: BluetoothLeScanner? = null // GATT连接相关 - var bluetoothGatt: BluetoothGatt? = null + var bluetoothGatt: BluetoothGatt? = null private var notifyChar: BluetoothGattCharacteristic? = null private var writeChar: BluetoothGattCharacteristic? = null private var audioChar: BluetoothGattCharacteristic? = null var recordfile: RecordingFile? = null + // 扫描相关 private lateinit var scanHandler: Handler private val scanResults = ArrayList() private var isScanning = false -// 是否是ota模式 + + // 是否是ota模式 var isEnterOta = false + // 连接状态 val connectionState = MutableLiveData(BleConst.STATE_DISCONNECTED) @@ -136,7 +140,7 @@ object BleService { Log.e(TAG, "OpusManager初始化失败: ${e.message}", e) // 根据需要决定是否因为Opus初始化失败而返回false } -recordfile = RecordingFile(this.context) + recordfile = RecordingFile(this.context) isInitialized = true Log.d(TAG, "BLE服务初始化成功") return true @@ -198,7 +202,6 @@ recordfile = RecordingFile(this.context) // 获取一个局部引用，避免并发访问问题 val scanner = bluetoothLeScanner ?: return false if (isScanning) return false - Log.i(TAG, "开始主动扫描BLE设备...") scanResults.clear() @@ -341,13 +344,12 @@ recordfile = RecordingFile(this.context) Log.w(TAG, "未找到已配对的设备 MAC 地址") return } - - - if(!isEnterOta) - { - Log.i(TAG, "检测到已配对设备，MAC: $pairedMac，尝试连接") - // 调用现有的 connect 方法 - connect(pairedMac) + + + if (!isEnterOta) { + Log.i(TAG, "检测到已配对设备，MAC: $pairedMac，尝试连接") + // 调用现有的 connect 方法 + connect(pairedMac) } } catch (e: Exception) { @@ -518,7 +520,7 @@ recordfile = RecordingFile(this.context) when (c.uuid) { // 音频特征数据 BleConst.RECEIVE_AUDIO_CHAR_UUID -> { - + // Log.i(TAG, "收到音频特征数据") processAudioData(data) } @@ -597,7 +599,7 @@ recordfile = RecordingFile(this.context) private fun processAudioData(data: ByteArray) { try { if (opusManager?.isDecodeStream == true) { - recordfile?.saveAudioDataToWav(data) + recordfile?.saveAudioDataToWav(data) opusManager?.writeAudioStream(data) } else { // Log.d(TAG, "Opus解码流未启动，忽略音频数据") @@ -795,17 +797,16 @@ recordfile = RecordingFile(this.context) BleConst.CODEC_CONTROL_ENCODE_ON -> "已打开编码" else -> "未知状态($codecStatus)" } -if (codecStatus == BleConst.CODEC_CONTROL_DECODE_ON || - codecStatus == BleConst.CODEC_CONTROL_A2DP_PLAY || - codecStatus == BleConst.CODEC_CONTROL_ENCODE_ON) { - - recordfile!!.closeFile() - recordfile!!.creatingFiles() -} -else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE) -{ - recordfile!!.closeFile() -} + if (codecStatus == BleConst.CODEC_CONTROL_DECODE_ON || + codecStatus == BleConst.CODEC_CONTROL_A2DP_PLAY || + codecStatus == BleConst.CODEC_CONTROL_ENCODE_ON + ) { + + recordfile!!.closeFile() + recordfile!!.creatingFiles() + } else if (codecStatus == BleConst.CODEC_CONTROL_CLOSE) { + recordfile!!.closeFile() + } // 声道模式描述 val channelDesc = when (channelMode) { BleConst.AUDIO_CHANNEL_LEFT -> "左声道" @@ -1093,7 +1094,7 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE) // // The library might expect this in a different unit or derive it. // // Given JlOpusPlugin.kt, packetSize refers to Opus encoded frame duration in ms. // ) - startOpusStreamDecoding(false, 1, 16000, 40) + startOpusStreamDecoding(false, 1, 16000, 40) Log.i(TAG, "打开解码0xA2") return sendCommand( @@ -1110,7 +1111,7 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE) */ fun openA2DPDecoder(): Boolean { Log.i(TAG, "打开编码 0xA1") - startOpusStreamDecoding(false, 1, 16000, 80) + startOpusStreamDecoding(false, 1, 16000, 80) // Log.i(TAG, "打开解码...") return sendCommand( BleConst.CMD_CONTROL_CODEC.toByte(), byteArrayOf( @@ -1135,7 +1136,7 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE) fun openEncoder(): Boolean { Log.i(TAG, "打开编码0xB1") - startOpusStreamDecoding(false, 1, 16000, 40) + startOpusStreamDecoding(false, 1, 16000, 40) // 目前仅发送命令通知设备开始编码。 return sendCommand( BleConst.CMD_CONTROL_CODEC.toByte(), @@ -1385,9 +1386,10 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE) } } - + } - class RecordingFile(private val context: Context) { + +class RecordingFile(private val context: Context) { private var currentAudioFile: File? = null private var fos: FileOutputStream? = null @@ -1408,8 +1410,9 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE) // 创建新的音频文件 val dateFormat = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.getDefault()) val timestamp = dateFormat.format(Date()) - - val filePath = File(context.getExternalFilesDir(null), "${fileName}_${timestamp}.opus").absolutePath + + val filePath = + File(context.getExternalFilesDir(null), "${fileName}_${timestamp}.opus").absolutePath currentAudioFile = File(filePath) currentAudioFile?.createNewFile() // 追加音频数据到文件 @@ -1417,42 +1420,42 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE) startWriteThread() // 新增：启动写入线程 } - // 新增：启动写入线程 - private fun startWriteThread() { - if (isWriting.get()) return - isWriting.set(true) - writeThread = Thread { - try { - while (isWriting.get() || writeQueue.isNotEmpty()) { - val data = writeQueue.poll() ?: continue - fos?.write(data) - } - } catch (e: Exception) { - Log.e("", "异步写入音频数据失败: ${e.message}") + // 新增：启动写入线程 + private fun startWriteThread() { + if (isWriting.get()) return + isWriting.set(true) + writeThread = Thread { + try { + while (isWriting.get() || writeQueue.isNotEmpty()) { + val data = writeQueue.poll() ?: continue + fos?.write(data) } + } catch (e: Exception) { + Log.e("", "异步写入音频数据失败: ${e.message}") } - writeThread?.start() } + writeThread?.start() + } - /** - * 保存音频数据到 WAV 文件（异步） - */ - internal fun saveAudioDataToWav(buffer: ByteArray) { - if (fos == null || currentAudioFile == null) return - // 放入队列，由写线程写入 - // 1. 将字节数据转换为十六进制字符串 - // val hexData = buildString { - // buffer.forEachIndexed { index, byte -> - // append("%02X".format(byte)) - - // } - // } - - // 2. 写入十六进制字符串 - - writeQueue.offer(buffer) - totalBytesWritten += buffer.size - } + /** + * 保存音频数据到 WAV 文件（异步） + */ + internal fun saveAudioDataToWav(buffer: ByteArray) { + if (fos == null || currentAudioFile == null) return + // 放入队列，由写线程写入 + // 1. 将字节数据转换为十六进制字符串 + // val hexData = buildString { + // buffer.forEachIndexed { index, byte -> + // append("%02X".format(byte)) + + // } + // } + + // 2. 写入十六进制字符串 + + writeQueue.offer(buffer) + totalBytesWritten += buffer.size + } internal fun closeFile() { From 47432987635afe8493ee96cfe41c3f4430cb6f2e Mon Sep 17 00:00:00 2001 From: wolfplus2048 <30993207+wolfplus2048@users.noreply.github.com> Date: Sat, 21 Jun 2025 21:15:15 +0100 Subject: [PATCH 03/10] add --- .../azure_speech/AzureTtsHelper.kt | 34 ++ .../bytedance_speech/BytedanceAudioPlayer.kt | 342 +++++++----------- .../bytedance_speech/BytedanceTTS.kt | 177 ++++----- .../chat_api/ChatApiService.kt | 3 +- .../com/deep_voice/speech/ITtsService.kt | 8 + 5 files changed, 240 insertions(+), 324 deletions(-) diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt index 62ac83d49..9b3d8ccfe 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt @@ -169,6 +169,40 @@ class AzureTtsHelper(private val context: Context) : ITtsService { audioDataListeners.remove(listener) } + /** + * 设置是否使用内部播放器 + * + * @param useInternalPlayer true: 使用内部播放器自动播放音频 + * false: 仅通过音频数据监听器输出数据，不播放 + */ + override fun setUseInternalPlayer(useInternalPlayer: Boolean) { + // Azure TTS 通过 AudioConfig 控制播放器 + // 如果不使用内部播放器，需要重新配置 synthesizer + if (!useInternalPlayer && customAudioOutputStream == null) { + // 创建自定义音频输出流以便捕获音频数据 + customAudioOutputStream = SimpleAudioPlayer(context).getAudioOutputStream() + + // 如果已经初始化，需要重新创建 synthesizer + if (isInitialized && speechConfig != null) { + synthesizer?.close() + val audioConfig = AudioConfig.fromStreamOutput(customAudioOutputStream) + synthesizer = SpeechSynthesizer(speechConfig, audioConfig) + setupEventListeners() + } + } else if (useInternalPlayer && customAudioOutputStream != null) { + // 切换回使用内部播放器 + customAudioOutputStream = null + + // 如果已经初始化，需要重新创建 synthesizer + if (isInitialized && speechConfig != null) { + synthesizer?.close() + val audioConfig = AudioConfig.fromDefaultSpeakerOutput() + synthesizer = SpeechSynthesizer(speechConfig, audioConfig) + setupEventListeners() + } + } + } + /** * 触发事件通知 */ diff --git a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt index 9587f401f..84e73b1be 100644 --- a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt +++ b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt @@ -7,112 +7,134 @@ import android.media.AudioTrack import android.os.Build import android.util.Log import com.deep_voice.speech.tts.AudioDataListener -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.cancelChildren -import kotlinx.coroutines.delay -import kotlinx.coroutines.isActive -import kotlinx.coroutines.launch -import kotlinx.coroutines.withContext -import java.util.concurrent.ConcurrentLinkedQueue -import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent.CancellationException + /** - * 字节跳动语音合成音频播放器 - * - * 使用AudioTrack播放PCM格式音频流，基于Kotlin协程实现异步处理 + * 简化的音频流播放器 + * 基于位置标记触发播放完成回调 */ -class BytedanceAudioPlayer : AudioDataListener, CoroutineScope { +class BytedanceAudioPlayer : AudioDataListener { companion object { private const val TAG = "BytedanceAudioPlayer" - private const val SAMPLE_RATE = 24000 // 样本率 - private const val CHANNEL_CONFIG = AudioFormat.CHANNEL_OUT_MONO // 单声道 - private const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT // 16位PCM - private const val IDLE_TIMEOUT_MS = 800L // 无数据超时时间 + private const val SAMPLE_RATE = 24000 + private const val CHANNEL_CONFIG = AudioFormat.CHANNEL_OUT_MONO + private const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT } - // 协程相关 - private val job = SupervisorJob() - override val coroutineContext = Dispatchers.IO + job - - // 音频处理相关 private var audioTrack: AudioTrack? = null - private val isPlaying = AtomicBoolean(false) - private val isPaused = AtomicBoolean(false) - - // 播放状态 - private val playStarted = AtomicBoolean(false) - private val completionReported = AtomicBoolean(false) + private var isFirstData = true // 是否是第一次接收数据 + private var totalBytesWritten = 0 // 总共写入的字节数 + private var sessionActive = true // 会话是否活跃 - // 使用简单的并发队列，保证线程安全 - private val audioDataQueue = ConcurrentLinkedQueue() - - // 播放状态监听 + // 回调 private var onPlayStarted: (() -> Unit)? = null private var onPlayCompleted: (() -> Unit)? = null - private var onError: ((String) -> Unit)? = null - // 处理协程 - private var processingJob: Job? = null - private var lastDataTime = 0L + // 播放位置监听器 + private val playbackListener = object : AudioTrack.OnPlaybackPositionUpdateListener { + override fun onMarkerReached(track: AudioTrack) { + Log.d(TAG, "播放到达标记位置: ${track.playbackHeadPosition}") + onPlayCompleted?.invoke() + } + + override fun onPeriodicNotification(track: AudioTrack) { + // 不使用周期性通知 + } + } - init { - Log.d(TAG, "BytedanceAudioPlayer初始化") + + /** + * 开始新的播放会话 + */ + fun startSession() { + Log.d(TAG, "开始新会话") + + // 重置状态 + isFirstData = true + totalBytesWritten = 0 + sessionActive = true + + // 重置 AudioTrack + audioTrack?.let { track -> + if (track.state == AudioTrack.STATE_INITIALIZED) { + // 停止并清空缓冲区 + track.pause() + track.flush() + // 重新开始播放 + track.play() + Log.d(TAG, "AudioTrack 已重置") + } + } ?: initAudioTrack() // 如果没有初始化，则初始化 } /** - * 开始播放 + * 标记数据流结束 */ - fun start() { - if (isPlaying.getAndSet(true)) return + fun endSession() { + if (!sessionActive) return - isPaused.set(false) - playStarted.set(false) - completionReported.set(false) - lastDataTime = System.currentTimeMillis() + Log.d(TAG, "数据流结束，总共写入字节数: $totalBytesWritten") + sessionActive = false - processingJob = launch { - try { - initAudioTrack() + audioTrack?.let { track -> + if (totalBytesWritten > 0) { + // 计算总帧数（16-bit 单声道，每帧2字节） + val totalFrames = totalBytesWritten / 2 - // 处理队列中的音频数据 - while (isActive && isPlaying.get()) { - processQueuedAudio() - - // 检查是否播放完成 - checkPlaybackCompletion() - - // 队列为空时短暂延迟 - if (audioDataQueue.isEmpty()) { - delay(10) - } + // 设置标记位置 + try { + track.setNotificationMarkerPosition(totalFrames) + Log.d(TAG, "设置播放完成标记位置: $totalFrames") + } catch (e: Exception) { + Log.e(TAG, "设置标记失败: ${e.message}") + // 设置失败时，使用延迟触发作为后备 + val durationMs = (totalFrames * 1000L) / SAMPLE_RATE + android.os.Handler(android.os.Looper.getMainLooper()).postDelayed({ + onPlayCompleted?.invoke() + }, durationMs + 500) } - } catch (e: CancellationException) { - // 协程被取消，正常行为 - } catch (e: Exception) { - Log.e(TAG, "播放错误: ${e.message}") - onError?.invoke("播放错误: ${e.message}") - stopInternal(false) + } else { + // 没有数据，直接触发完成 + onPlayCompleted?.invoke() } } } /** - * 暂停播放 + * 接收音频数据 */ - fun pause() { - if (!isPlaying.get() || isPaused.getAndSet(true)) return - audioTrack?.pause() - } - - /** - * 恢复播放 - */ - fun resume() { - if (!isPlaying.get() || !isPaused.getAndSet(false)) return - audioTrack?.play() + override fun onAudioData(data: ByteArray) { + if (data.isEmpty()) return + if (!sessionActive) { + Log.w(TAG, "收到音频数据但会话未激活，忽略数据") + return + } + if (audioTrack == null || audioTrack?.state != AudioTrack.STATE_INITIALIZED) { + Log.w(TAG, "收到音频数据但 AudioTrack 未初始化或状态不正确，忽略数据") + return + } + try { + audioTrack?.let { track -> + if (track.state == AudioTrack.STATE_INITIALIZED) { + val bytesWritten = track.write(data, 0, data.size) + + if (bytesWritten > 0) { + // 第一次写入数据时自动触发开始回调 + if (isFirstData) { + isFirstData = false + onPlayStarted?.invoke() + Log.d(TAG, "播放开始") + } + + // 累计写入字节数 + if (sessionActive) { + totalBytesWritten += bytesWritten + } + } + } + } + } catch (e: Exception) { + Log.e(TAG, "写入音频数据失败: ${e.message}") + } } /** @@ -120,44 +142,32 @@ class BytedanceAudioPlayer : AudioDataListener, CoroutineScope { */ fun stop() { Log.d(TAG, "停止播放") - stopInternal(true) - } - - /** - * 内部停止处理 - */ - private fun stopInternal(reportCompletion: Boolean) { - if (!isPlaying.getAndSet(false)) return - - isPaused.set(false) - processingJob?.cancel() - - audioTrack?.stop() - // 不在stopInternal中释放资源，只停止播放 - // 清空队列 - audioDataQueue.clear() + sessionActive = false - // 播放完成通知 - if (reportCompletion && playStarted.get() && !completionReported.getAndSet(true)) { - launch(Dispatchers.Main) { - onPlayCompleted?.invoke() + audioTrack?.let { track -> + if (track.state == AudioTrack.STATE_INITIALIZED) { + track.pause() + track.flush() } } + + // 如果已经开始播放，触发完成回调 + if (!isFirstData) { + onPlayCompleted?.invoke() + } } /** * 释放资源 */ fun release() { - stopInternal(false) + Log.d(TAG, "释放资源") + + stop() - // 在release方法中释放AudioTrack资源 audioTrack?.release() audioTrack = null - - // 释放协程资源 - job.cancel() } /** @@ -175,120 +185,32 @@ class BytedanceAudioPlayer : AudioDataListener, CoroutineScope { } /** - * 设置错误回调 + * 设置错误回调（兼容接口） */ fun setOnError(listener: (String) -> Unit) { - onError = listener - } - - /** - * 接收音频数据（实现AudioDataListener接口） - */ - override fun onAudioData(data: ByteArray) { - if (data.isEmpty()) return - - lastDataTime = System.currentTimeMillis() - - if (!isPlaying.get()) { - start() - } - - // 添加到队列 - 简单有效，保证FIFO顺序 - audioDataQueue.add(data.copyOf()) + // 不实现，仅为兼容 } /** * 检查是否正在播放 */ - fun isPlaying(): Boolean = isPlaying.get() && !isPaused.get() - - /** - * 处理队列中的音频数据 - */ - private suspend fun processQueuedAudio() { - if (isPaused.get() || !isPlaying.get() || !isActive) return - - // 获取并处理队列中的数据 - val data = audioDataQueue.poll() ?: return - - // 写入音频数据 - val result = audioTrack?.write(data, 0, data.size) ?: 0 - - if (result > 0) { - // 标记播放开始 - if (!playStarted.getAndSet(true)) { - withContext(Dispatchers.Main) { - onPlayStarted?.invoke() - } - } - } else if (result < 0) { - // 处理错误 - handleAudioTrackError(result) - } + fun isPlaying(): Boolean { + return audioTrack?.playState == AudioTrack.PLAYSTATE_PLAYING } /** - * 检查播放是否完成（超时无数据） - */ - private suspend fun checkPlaybackCompletion() { - val currentTime = System.currentTimeMillis() - - // 超时判断 - 队列为空且超过超时时间 - if (playStarted.get() && audioDataQueue.isEmpty() && - currentTime - lastDataTime > IDLE_TIMEOUT_MS && isPlaying.get()) { - - // 播放完成 - if (!completionReported.getAndSet(true)) { - withContext(Dispatchers.Main) { - onPlayCompleted?.invoke() - } - stopInternal(false) // 已报告，不需要再次报告 - } - } - } - - /** - * 处理AudioTrack错误 - */ - private fun handleAudioTrackError(result: Int) { - val errorMsg = when(result) { - AudioTrack.ERROR_INVALID_OPERATION -> "无效操作" - AudioTrack.ERROR_BAD_VALUE -> "参数错误" - AudioTrack.ERROR_DEAD_OBJECT -> "对象已销毁" - else -> "未知错误" - } - - // 对象已销毁时尝试重建 - if (result == AudioTrack.ERROR_DEAD_OBJECT) { - try { - audioTrack?.release() - initAudioTrack() - } catch (e: Exception) { - Log.e(TAG, "重建AudioTrack失败: ${e.message}") - throw e - } - } else { - Log.e(TAG, "AudioTrack错误: $errorMsg") - } - } - - /** - * 初始化AudioTrack + * 初始化 AudioTrack */ private fun initAudioTrack() { - val minBufferSize = AudioTrack.getMinBufferSize( - SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT - ) - - if (minBufferSize == AudioTrack.ERROR || minBufferSize == AudioTrack.ERROR_BAD_VALUE) { - throw IllegalStateException("无法获取有效的音频缓冲区大小") - } + // 释放旧实例 + audioTrack?.release() - // 使用较大的缓冲区提高稳定性 - val bufferSize = minBufferSize * 4 + // 计算缓冲区大小 + val minBufferSize = AudioTrack.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT) + val bufferSize = minBufferSize * 2 - // 创建AudioTrack - audioTrack = if (Build.VERSION.SDK_INT >= 23) { // Android M (6.0) + // 创建 AudioTrack + audioTrack = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { AudioTrack.Builder() .setAudioAttributes( AudioAttributes.Builder() @@ -318,12 +240,12 @@ class BytedanceAudioPlayer : AudioDataListener, CoroutineScope { ) } - // 检查初始化状态 - if (audioTrack?.state != AudioTrack.STATE_INITIALIZED) { - throw IllegalStateException("AudioTrack初始化失败") - } + // 设置播放位置监听器 + audioTrack?.setPlaybackPositionUpdateListener(playbackListener) // 开始播放 audioTrack?.play() + + Log.d(TAG, "AudioTrack 初始化成功") } -} \ No newline at end of file +} \ No newline at end of file diff --git a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt index d202a36e4..49d83efcc 100644 --- a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt +++ b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt @@ -100,6 +100,9 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { // 内部音频播放器 private val audioPlayer = BytedanceAudioPlayer() + // 是否使用内部播放器 + private var useInternalPlayer = true + // WebSocket连接 private var webSocket: WebSocket? = null private val client: OkHttpClient @@ -119,9 +122,7 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { private var language: String = "zh-CN" // 流式处理状态 - private var isStreamMode = false private var currentStatus = STATUS_STOPPED - private var isSpeaking = false // 连接相关状态 private var connectionAttempts = 0 @@ -155,12 +156,8 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { * 初始化音频播放器回调 */ private fun initAudioPlayerCallbacks() { - // 添加播放器作为音频数据监听器 - addAudioDataListener(audioPlayer) - audioPlayer.setOnPlayStarted { Log.d(TAG, "播放开始") - isSpeaking = true updateStatus(STATUS_SPEAKING) notifyEvent(TtsEventType.SYNTHESIS_STARTED) } @@ -168,17 +165,18 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { // 设置播放完成回调 audioPlayer.setOnPlayCompleted { Log.d(TAG, "播放结束") - isSpeaking = false - updateStatus(if (isStreamMode) STATUS_READY else STATUS_STOPPED) + updateStatus(STATUS_READY) notifyEvent(TtsEventType.SYNTHESIS_COMPLETED) } // 设置错误回调 audioPlayer.setOnError { errorMsg -> - isSpeaking = false updateStatus(STATUS_ERROR) notifyEvent(TtsEventType.ERROR, mapOf("errorCode" to "PLAYER_ERROR", "errorMessage" to errorMsg)) } + + // 根据设置决定是否使用内部播放器 + updateInternalPlayerUsage() } /** @@ -208,16 +206,19 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { */ override fun stop(): Boolean { try { - audioPlayer.stop() + Log.i(TAG, ">stop()") + + if (useInternalPlayer) { + audioPlayer.stop() + } textProcessingJob?.cancel() - launch { - if (isSessionStarted) { - finishSession() - } - isSpeaking = false // 显式设置，因为这是强制停止 - updateStatus(STATUS_STOPPED) + + if (isSessionStarted) { + finishSession() } + updateStatus(STATUS_STOPPED) + return true } catch (e: Exception) { Log.e(TAG, "停止合成失败: ${e.message}") @@ -277,7 +278,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { isConnected = false isSessionStarted = false - isSpeaking = false updateStatus(STATUS_STOPPED) startTextProcessing() @@ -304,29 +304,9 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { * 单次播放(非流式) */ override fun speakOnce(text: String): Boolean { - try { - if (text.isEmpty()) { - return false - } - - isStreamMode = false - - launch { - if (!ensureSessionReady()) { - notifyEvent(TtsEventType.ERROR, mapOf("errorCode" to "SESSION_ERROR", - "errorMessage" to "无法建立会话")) - return@launch - } - - sendTextRequest(text) - } - - return true - } catch (e: Exception) { - Log.e(TAG, "语音合成失败: ${e.message}") - updateStatus(STATUS_ERROR) - return false - } + // 未实现：只支持流式模式 + Log.w(TAG, "speakOnce未实现，请使用speakStream") + return false } /** @@ -337,7 +317,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { if (text.isEmpty()) { return true } - isStreamMode = true // 确保会话已就绪 if (!isSessionStarted && !ensureSessionReady()) { @@ -365,17 +344,11 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { */ override fun flushStream(): Boolean { try { - if (!isStreamMode) return true // Log.d(TAG, "flushStream: $sessionId") - launch { - withContext(Dispatchers.IO) { - delay(300) // 确保现有文本处理完成 - } - - if (isSessionStarted) { - isSessionStarted = false - finishSession() - } + + if (isSessionStarted) { + isSessionStarted = false + finishSession() } return true @@ -440,6 +413,29 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { audioDataListeners.remove(listener) } + /** + * 设置是否使用内部播放器 + */ + override fun setUseInternalPlayer(useInternalPlayer: Boolean) { + this.useInternalPlayer = useInternalPlayer + updateInternalPlayerUsage() + } + + /** + * 更新内部播放器的使用状态 + */ + private fun updateInternalPlayerUsage() { + if (useInternalPlayer) { + // 使用内部播放器 + if (!audioDataListeners.contains(audioPlayer)) { + audioDataListeners.add(audioPlayer) + } + } else { + // 不使用内部播放器 + audioDataListeners.remove(audioPlayer) + } + } + /** * 通知事件处理 */ @@ -461,15 +457,14 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { * 通知音频数据 */ private fun notifyAudioData(data: ByteArray) { - launch(Dispatchers.Main) { - for (listener in audioDataListeners) { - try { - listener.onAudioData(data) - } catch (e: Exception) { - Log.e(TAG, "音频数据回调异常: ${e.message}") - } + for (listener in audioDataListeners) { + try { + listener.onAudioData(data) + } catch (e: Exception) { + Log.e(TAG, "音频数据回调异常: ${e.message}") } } + } /** @@ -483,7 +478,7 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { // 如果已连接但会话未开始，只需要开始会话 if (isConnected && !isSessionStarted) { - Log.d(TAG, "WebSocket已连接，正在启动新会话...") + // Log.d(TAG, "WebSocket已连接，正在启动新会话...") return startSessionOnly() } @@ -648,7 +643,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { "errorMessage" to errorMsg)) webSocket.close(1000, "Error") updateStatus(STATUS_ERROR) - isSpeaking = false // 错误情况下显式设置 isSessionStarted = false isConnected = false isConnecting.set(false) @@ -656,6 +650,10 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { EVENT_SESSION_STARTED -> { connectionAttempts = 0 + // 会话开始时，如果使用内部播放器则启动播放器会话 + if (useInternalPlayer) { + audioPlayer.startSession() + } } EVENT_TTS_RESPONSE -> { @@ -673,16 +671,18 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { EVENT_SESSION_FINISHED -> { isSessionStarted = false - - if (!isStreamMode) { - finishConnection(webSocket) + Log.i(TAG, "EVENT_SESSION_FINISHED, ${sessionId}") + // 如果使用内部播放器则结束播放器会话 + if (useInternalPlayer) { + audioPlayer.endSession() } + + // 流式模式下不自动关闭连接 } } } catch (e: Exception) { Log.e(TAG, "解析响应失败: ${e.message}") updateStatus(STATUS_ERROR) - isSpeaking = false // 错误情况下显式设置 isConnecting.set(false) } } @@ -690,7 +690,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { isConnected = false isSessionStarted = false - isSpeaking = false isConnecting.set(false) Log.e(TAG, "WebSocket连接失败: ${t.message}") @@ -702,7 +701,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { isConnected = false isSessionStarted = false - isSpeaking = false isConnecting.set(false) updateStatus(STATUS_STOPPED) notifyEvent(TtsEventType.SYNTHESIS_CANCELED) @@ -798,53 +796,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { sendEvent(webSocket, header, optional, payload) } - /** - * 发送文本进行合成 (非流式) - */ - private fun sendTextRequest(text: String) { - webSocket?.let { ws -> - val header = Header( - protocolVersion = PROTOCOL_VERSION, - headerSize = DEFAULT_HEADER_SIZE, - messageType = FULL_CLIENT_REQUEST, - messageTypeSpecificFlags = MSG_TYPE_FLAG_WITH_EVENT, - serializationMethod = JSON, - messageCompression = COMPRESSION_NO, - reserved = 0 - ) - - val optional = Optional( - event = EVENT_TASK_REQUEST, - sessionId = sessionId - ) - - val jsonObject = JSONObject() - val user = JSONObject() - user.put("uid", "123456") - jsonObject.put("user", user) - jsonObject.put("event", EVENT_TASK_REQUEST) - jsonObject.put("namespace", "BidirectionalTTS") - - val reqParams = JSONObject() - // 将#和*替换为空格 - val processedText = text.replace("#", " ").replace("*", " ") - reqParams.put("text", processedText) - reqParams.put("speaker", speaker) - - val audioParams = JSONObject() - audioParams.put("format", format) - audioParams.put("sample_rate", sampleRate) - audioParams.put("emotion", "happy") - - reqParams.put("audio_params", audioParams) - jsonObject.put("req_params", reqParams) - - val payload = jsonObject.toString().toByteArray() - Log.d(TAG, "发送文本合成请求: ${jsonObject.toString()}") - - sendEvent(ws, header, optional, payload) - } - } /** * 发送文本进行合成 (流式) diff --git a/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt b/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt index 3e3d056db..13f74a898 100644 --- a/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt +++ b/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt @@ -157,7 +157,8 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor currentStreamJob = null // 2. 通知旧会话被中止 - currSessionCallback?.onError(ChatApiException("Session aborted by new request")) + // currSessionCallback?.onError(ChatApiException("Session aborted by new request")) + currSessionCallback?.onComplete() // 直接完成当前会话 // 3. 清理状态 currSessionId = "" diff --git a/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/ITtsService.kt b/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/ITtsService.kt index 91c8945d6..d43604582 100644 --- a/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/ITtsService.kt +++ b/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/ITtsService.kt @@ -89,4 +89,12 @@ interface ITtsService { * @param listener 要移除的音频数据监听器 */ fun removeAudioDataListener(listener: AudioDataListener) + + /** + * 设置是否使用内部播放器 + * + * @param useInternalPlayer true: 使用内部播放器自动播放音频 + * false: 仅通过音频数据监听器输出数据，不播放 + */ + fun setUseInternalPlayer(useInternalPlayer: Boolean) } \ No newline at end of file From a63217e5cad512ae813f758badad16df26f4e54f Mon Sep 17 00:00:00 2001 From: wolfplus2048 <30993207+wolfplus2048@users.noreply.github.com> Date: Sat, 21 Jun 2025 22:02:59 +0100 Subject: [PATCH 04/10] add --- .../azure_speech/AzureSpeechPlugin.kt | 8 ++ .../azure_speech/AzureTtsHelper.kt | 44 +++++++- .../bytedance_speech/BytedanceAudioPlayer.kt | 100 ++++++++++++++++-- .../bytedance_speech/BytedanceSpeechPlugin.kt | 6 ++ .../bytedance_speech/BytedanceTTS.kt | 24 ++++- .../com/deep_voice/speech/ITtsService.kt | 7 ++ .../kotlin/com/deep_voice/speech/TtsEvents.kt | 16 +++ 7 files changed, 193 insertions(+), 12 deletions(-) diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt index e19579cc3..54e960092 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt @@ -158,6 +158,14 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin { ) } + TtsEventType.PLAYBACK_STARTED -> mapOf( + "type" to "playback_started" + ) + + TtsEventType.PLAYBACK_COMPLETED -> mapOf( + "type" to "playback_completed" + ) + TtsEventType.ERROR -> { val params = event.params mapOf( diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt index 9b3d8ccfe..0465e5e4c 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt @@ -1,14 +1,16 @@ package com.yunqiinnovation.azure_speech import android.content.Context +import android.media.AudioManager import com.microsoft.cognitiveservices.speech.* import com.microsoft.cognitiveservices.speech.audio.* import com.yunqiinnovation.azure_speech.utils.FileLogger +import com.deep_voice.speech.tts.AudioDataListener +import com.deep_voice.speech.tts.AudioOutputDevice import com.deep_voice.speech.tts.ITtsService import com.deep_voice.speech.tts.TtsEvent import com.deep_voice.speech.tts.TtsEventListener import com.deep_voice.speech.tts.TtsEventType -import com.deep_voice.speech.tts.AudioDataListener import kotlinx.coroutines.* import java.io.ByteArrayInputStream import java.io.InputStream @@ -203,6 +205,42 @@ class AzureTtsHelper(private val context: Context) : ITtsService { } } + /** + * 设置音频输出设备 + * + * @param device 音频输出设备类型 + */ + override fun setAudioOutputDevice(device: AudioOutputDevice) { + // Azure TTS使用系统默认的音频路由 + // 音频输出设备的控制需要通过Android的AudioManager实现 + val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager + audioManager?.let { manager -> + when (device) { + AudioOutputDevice.DEFAULT -> { + // 默认模式：系统自动选择 + manager.mode = AudioManager.MODE_NORMAL + manager.isSpeakerphoneOn = false + } + AudioOutputDevice.SPEAKER -> { + // 强制使用扬声器 + manager.mode = AudioManager.MODE_NORMAL + manager.isSpeakerphoneOn = true + } + AudioOutputDevice.HEADPHONES -> { + // 强制使用耳机（如果已连接） + manager.mode = AudioManager.MODE_NORMAL + manager.isSpeakerphoneOn = false + // 注意：Android不能强制路由到耳机，只能在耳机已连接时使用 + } + AudioOutputDevice.EARPIECE -> { + // 强制使用听筒 + manager.mode = AudioManager.MODE_IN_COMMUNICATION + manager.isSpeakerphoneOn = false + } + } + } + } + /** * 触发事件通知 */ @@ -228,6 +266,8 @@ class AzureTtsHelper(private val context: Context) : ITtsService { SynthesisStarted?.addEventListener { _, eventArgs -> FileLogger.d(TAG, "语音合成开始: resultId=${eventArgs.result.resultId}") notifyEvent(TtsEventType.SYNTHESIS_STARTED) + // Azure TTS 在使用默认音频输出时会立即开始播放 + notifyEvent(TtsEventType.PLAYBACK_STARTED) } // 合成中事件（接收音频数据） @@ -253,6 +293,8 @@ class AzureTtsHelper(private val context: Context) : ITtsService { FileLogger.d(TAG, "语音合成完成: resultId=${eventArgs.result.resultId}, 音频长度=${eventArgs.result.audioLength} 字节") isSpeaking = false notifyEvent(TtsEventType.SYNTHESIS_COMPLETED) + // Azure TTS 合成完成即播放完成 + notifyEvent(TtsEventType.PLAYBACK_COMPLETED) } // 合成取消事件 diff --git a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt index 84e73b1be..a3f2c972b 100644 --- a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt +++ b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt @@ -1,5 +1,6 @@ package com.deep_voice.bytedance_speech +import android.content.Context import android.media.AudioAttributes import android.media.AudioFormat import android.media.AudioManager @@ -7,12 +8,13 @@ import android.media.AudioTrack import android.os.Build import android.util.Log import com.deep_voice.speech.tts.AudioDataListener +import com.deep_voice.speech.tts.AudioOutputDevice /** * 简化的音频流播放器 * 基于位置标记触发播放完成回调 */ -class BytedanceAudioPlayer : AudioDataListener { +class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { companion object { private const val TAG = "BytedanceAudioPlayer" private const val SAMPLE_RATE = 24000 @@ -24,6 +26,8 @@ class BytedanceAudioPlayer : AudioDataListener { private var isFirstData = true // 是否是第一次接收数据 private var totalBytesWritten = 0 // 总共写入的字节数 private var sessionActive = true // 会话是否活跃 + private var audioOutputDevice = AudioOutputDevice.DEFAULT // 音频输出设备 + private var audioManager: AudioManager? = null // 回调 private var onPlayStarted: (() -> Unit)? = null @@ -40,6 +44,12 @@ class BytedanceAudioPlayer : AudioDataListener { // 不使用周期性通知 } } + + init { + Log.d(TAG, "BytedanceAudioPlayer 初始化") + audioManager = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager + initAudioTrack() + } /** @@ -198,6 +208,23 @@ class BytedanceAudioPlayer : AudioDataListener { return audioTrack?.playState == AudioTrack.PLAYSTATE_PLAYING } + /** + * 设置音频输出设备 + */ + fun setAudioOutputDevice(device: AudioOutputDevice) { + if (audioOutputDevice != device) { + audioOutputDevice = device + // 如果AudioTrack已初始化，需要重新创建以应用新的输出设备设置 + if (audioTrack != null) { + val wasPlaying = isPlaying() + initAudioTrack() + if (wasPlaying) { + audioTrack?.play() + } + } + } + } + /** * 初始化 AudioTrack */ @@ -209,15 +236,36 @@ class BytedanceAudioPlayer : AudioDataListener { val minBufferSize = AudioTrack.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT) val bufferSize = minBufferSize * 2 - // 创建 AudioTrack - audioTrack = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - AudioTrack.Builder() - .setAudioAttributes( + // 根据输出设备配置AudioAttributes + val audioAttributes = when (audioOutputDevice) { + AudioOutputDevice.EARPIECE -> { + // 听筒模式 + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + } else { + null + } + } + else -> { + // 默认、耳机、扬声器模式 + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { AudioAttributes.Builder() .setUsage(AudioAttributes.USAGE_MEDIA) .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) .build() - ) + } else { + null + } + } + } + + // 创建 AudioTrack + audioTrack = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && audioAttributes != null) { + AudioTrack.Builder() + .setAudioAttributes(audioAttributes) .setAudioFormat( AudioFormat.Builder() .setEncoding(AUDIO_FORMAT) @@ -230,8 +278,12 @@ class BytedanceAudioPlayer : AudioDataListener { .build() } else { @Suppress("DEPRECATION") + val streamType = when (audioOutputDevice) { + AudioOutputDevice.EARPIECE -> AudioManager.STREAM_VOICE_CALL + else -> AudioManager.STREAM_MUSIC + } AudioTrack( - AudioManager.STREAM_MUSIC, + streamType, SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT, @@ -243,9 +295,43 @@ class BytedanceAudioPlayer : AudioDataListener { // 设置播放位置监听器 audioTrack?.setPlaybackPositionUpdateListener(playbackListener) + // 配置音频路由 + configureAudioRouting() + // 开始播放 audioTrack?.play() Log.d(TAG, "AudioTrack 初始化成功") } + + /** + * 配置音频路由 + */ + private fun configureAudioRouting() { + audioManager?.let { manager -> + when (audioOutputDevice) { + AudioOutputDevice.DEFAULT -> { + // 默认模式：系统自动选择 + manager.mode = AudioManager.MODE_NORMAL + manager.isSpeakerphoneOn = false + } + AudioOutputDevice.SPEAKER -> { + // 强制使用扬声器 + manager.mode = AudioManager.MODE_NORMAL + manager.isSpeakerphoneOn = true + } + AudioOutputDevice.HEADPHONES -> { + // 强制使用耳机（如果已连接） + manager.mode = AudioManager.MODE_NORMAL + manager.isSpeakerphoneOn = false + // 注意：Android不能强制路由到耳机，只能在耳机已连接时使用 + } + AudioOutputDevice.EARPIECE -> { + // 强制使用听筒 + manager.mode = AudioManager.MODE_IN_COMMUNICATION + manager.isSpeakerphoneOn = false + } + } + } + } } \ No newline at end of file diff --git a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceSpeechPlugin.kt b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceSpeechPlugin.kt index d86faa71f..a6003e050 100644 --- a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceSpeechPlugin.kt +++ b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceSpeechPlugin.kt @@ -255,6 +255,12 @@ class BytedanceSpeechPlugin : FlutterPlugin { TtsEventType.SYNTHESIS_CANCELED -> { sendTTSEvent("canceled", null) } + TtsEventType.PLAYBACK_STARTED -> { + sendTTSEvent("playback_started", null) + } + TtsEventType.PLAYBACK_COMPLETED -> { + sendTTSEvent("playback_completed", null) + } TtsEventType.ERROR -> { val params = HashMap() params["code"] = event.params["errorCode"] ?: "UNKNOWN_ERROR" diff --git a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt index 49d83efcc..b3cd7e75f 100644 --- a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt +++ b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt @@ -3,6 +3,7 @@ package com.deep_voice.bytedance_speech import android.content.Context import android.util.Log import com.deep_voice.speech.tts.AudioDataListener +import com.deep_voice.speech.tts.AudioOutputDevice import com.deep_voice.speech.tts.ITtsService import com.deep_voice.speech.tts.TtsEvent import com.deep_voice.speech.tts.TtsEventListener @@ -98,7 +99,7 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { private val audioDataListeners = mutableListOf() // 内部音频播放器 - private val audioPlayer = BytedanceAudioPlayer() + private val audioPlayer = BytedanceAudioPlayer(context) // 是否使用内部播放器 private var useInternalPlayer = true @@ -159,14 +160,14 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { audioPlayer.setOnPlayStarted { Log.d(TAG, "播放开始") updateStatus(STATUS_SPEAKING) - notifyEvent(TtsEventType.SYNTHESIS_STARTED) + notifyEvent(TtsEventType.PLAYBACK_STARTED) } // 设置播放完成回调 audioPlayer.setOnPlayCompleted { Log.d(TAG, "播放结束") updateStatus(STATUS_READY) - notifyEvent(TtsEventType.SYNTHESIS_COMPLETED) + notifyEvent(TtsEventType.PLAYBACK_COMPLETED) } // 设置错误回调 @@ -421,6 +422,15 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { updateInternalPlayerUsage() } + /** + * 设置音频输出设备 + */ + override fun setAudioOutputDevice(device: AudioOutputDevice) { + if (useInternalPlayer) { + audioPlayer.setAudioOutputDevice(device) + } + } + /** * 更新内部播放器的使用状态 */ @@ -650,7 +660,9 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { EVENT_SESSION_STARTED -> { connectionAttempts = 0 - // 会话开始时，如果使用内部播放器则启动播放器会话 + // 会话开始时触发合成开始事件 + notifyEvent(TtsEventType.SYNTHESIS_STARTED) + // 如果使用内部播放器则启动播放器会话 if (useInternalPlayer) { audioPlayer.startSession() } @@ -672,6 +684,10 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { EVENT_SESSION_FINISHED -> { isSessionStarted = false Log.i(TAG, "EVENT_SESSION_FINISHED, ${sessionId}") + + // 触发合成完成事件 + notifyEvent(TtsEventType.SYNTHESIS_COMPLETED) + // 如果使用内部播放器则结束播放器会话 if (useInternalPlayer) { audioPlayer.endSession() diff --git a/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/ITtsService.kt b/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/ITtsService.kt index d43604582..00db2f2b3 100644 --- a/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/ITtsService.kt +++ b/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/ITtsService.kt @@ -97,4 +97,11 @@ interface ITtsService { * false: 仅通过音频数据监听器输出数据，不播放 */ fun setUseInternalPlayer(useInternalPlayer: Boolean) + + /** + * 设置音频输出设备 + * + * @param device 音频输出设备类型 + */ + fun setAudioOutputDevice(device: AudioOutputDevice) } \ No newline at end of file diff --git a/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/TtsEvents.kt b/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/TtsEvents.kt index cb69a849d..6758c6f05 100644 --- a/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/TtsEvents.kt +++ b/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/TtsEvents.kt @@ -13,6 +13,12 @@ enum class TtsEventType { /** 合成取消 */ SYNTHESIS_CANCELED, + /** 播放开始 */ + PLAYBACK_STARTED, + + /** 播放结束 */ + PLAYBACK_COMPLETED, + /** 发生错误 */ ERROR } @@ -52,4 +58,14 @@ interface AudioDataListener { * @param data 音频数据字节数组 */ fun onAudioData(data: ByteArray) +} + +/** + * 音频输出设备类型 + */ +enum class AudioOutputDevice { + DEFAULT, // 默认（如果有耳机选耳机，否则使用系统扬声器） + HEADPHONES, // 强制使用耳机 + SPEAKER, // 强制使用扬声器 + EARPIECE // 强制使用听筒 } \ No newline at end of file From e79f6ccf3b650e50cfaaefed8220a8bdc8c249c5 Mon Sep 17 00:00:00 2001 From: wolfplus2048 <30993207+wolfplus2048@users.noreply.github.com> Date: Sun, 22 Jun 2025 13:16:24 +0100 Subject: [PATCH 05/10] add --- .../azure_speech/AzureTtsHelper.kt | 5 - .../bytedance_speech/BytedanceAudioPlayer.kt | 315 ++++++++++++------ .../chat_api/ChatApiService.kt | 3 +- .../kotlin/com/deep_voice/speech/TtsEvents.kt | 1 - 4 files changed, 214 insertions(+), 110 deletions(-) diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt index 0465e5e4c..ea9d42588 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt @@ -232,11 +232,6 @@ class AzureTtsHelper(private val context: Context) : ITtsService { manager.isSpeakerphoneOn = false // 注意：Android不能强制路由到耳机，只能在耳机已连接时使用 } - AudioOutputDevice.EARPIECE -> { - // 强制使用听筒 - manager.mode = AudioManager.MODE_IN_COMMUNICATION - manager.isSpeakerphoneOn = false - } } } } diff --git a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt index a3f2c972b..ac7cf0afd 100644 --- a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt +++ b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt @@ -1,12 +1,16 @@ package com.deep_voice.bytedance_speech +import android.content.BroadcastReceiver import android.content.Context +import android.content.Intent +import android.content.IntentFilter import android.media.AudioAttributes +import android.media.AudioDeviceCallback +import android.media.AudioDeviceInfo import android.media.AudioFormat import android.media.AudioManager import android.media.AudioTrack import android.os.Build -import android.util.Log import com.deep_voice.speech.tts.AudioDataListener import com.deep_voice.speech.tts.AudioOutputDevice @@ -29,6 +33,11 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { private var audioOutputDevice = AudioOutputDevice.DEFAULT // 音频输出设备 private var audioManager: AudioManager? = null + // 设备监听相关 + private var deviceCallback: AudioDeviceCallback? = null + private var noisyReceiver: BroadcastReceiver? = null + private var isMonitoringDevices = false + // 回调 private var onPlayStarted: (() -> Unit)? = null private var onPlayCompleted: (() -> Unit)? = null @@ -36,7 +45,6 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { // 播放位置监听器 private val playbackListener = object : AudioTrack.OnPlaybackPositionUpdateListener { override fun onMarkerReached(track: AudioTrack) { - Log.d(TAG, "播放到达标记位置: ${track.playbackHeadPosition}") onPlayCompleted?.invoke() } @@ -46,7 +54,6 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { } init { - Log.d(TAG, "BytedanceAudioPlayer 初始化") audioManager = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager initAudioTrack() } @@ -56,24 +63,19 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { * 开始新的播放会话 */ fun startSession() { - Log.d(TAG, "开始新会话") - - // 重置状态 isFirstData = true totalBytesWritten = 0 sessionActive = true - // 重置 AudioTrack audioTrack?.let { track -> if (track.state == AudioTrack.STATE_INITIALIZED) { - // 停止并清空缓冲区 track.pause() track.flush() - // 重新开始播放 track.play() - Log.d(TAG, "AudioTrack 已重置") } - } ?: initAudioTrack() // 如果没有初始化，则初始化 + } ?: initAudioTrack() + + startDeviceMonitoring() } /** @@ -82,28 +84,20 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { fun endSession() { if (!sessionActive) return - Log.d(TAG, "数据流结束，总共写入字节数: $totalBytesWritten") sessionActive = false audioTrack?.let { track -> if (totalBytesWritten > 0) { - // 计算总帧数（16-bit 单声道，每帧2字节） val totalFrames = totalBytesWritten / 2 - - // 设置标记位置 try { track.setNotificationMarkerPosition(totalFrames) - Log.d(TAG, "设置播放完成标记位置: $totalFrames") } catch (e: Exception) { - Log.e(TAG, "设置标记失败: ${e.message}") - // 设置失败时，使用延迟触发作为后备 val durationMs = (totalFrames * 1000L) / SAMPLE_RATE android.os.Handler(android.os.Looper.getMainLooper()).postDelayed({ onPlayCompleted?.invoke() }, durationMs + 500) } } else { - // 没有数据，直接触发完成 onPlayCompleted?.invoke() } } @@ -113,37 +107,26 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { * 接收音频数据 */ override fun onAudioData(data: ByteArray) { - if (data.isEmpty()) return - if (!sessionActive) { - Log.w(TAG, "收到音频数据但会话未激活，忽略数据") - return - } - if (audioTrack == null || audioTrack?.state != AudioTrack.STATE_INITIALIZED) { - Log.w(TAG, "收到音频数据但 AudioTrack 未初始化或状态不正确，忽略数据") - return - } + if (data.isEmpty() || !sessionActive) return + if (audioTrack?.state != AudioTrack.STATE_INITIALIZED) return + try { audioTrack?.let { track -> - if (track.state == AudioTrack.STATE_INITIALIZED) { - val bytesWritten = track.write(data, 0, data.size) + val bytesWritten = track.write(data, 0, data.size) + + if (bytesWritten > 0) { + if (isFirstData) { + isFirstData = false + onPlayStarted?.invoke() + } - if (bytesWritten > 0) { - // 第一次写入数据时自动触发开始回调 - if (isFirstData) { - isFirstData = false - onPlayStarted?.invoke() - Log.d(TAG, "播放开始") - } - - // 累计写入字节数 - if (sessionActive) { - totalBytesWritten += bytesWritten - } + if (sessionActive) { + totalBytesWritten += bytesWritten } } } } catch (e: Exception) { - Log.e(TAG, "写入音频数据失败: ${e.message}") + // 忽略写入失败 } } @@ -151,8 +134,6 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { * 停止播放 */ fun stop() { - Log.d(TAG, "停止播放") - sessionActive = false audioTrack?.let { track -> @@ -162,7 +143,8 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { } } - // 如果已经开始播放，触发完成回调 + stopDeviceMonitoring() + if (!isFirstData) { onPlayCompleted?.invoke() } @@ -172,12 +154,10 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { * 释放资源 */ fun release() { - Log.d(TAG, "释放资源") - stop() - audioTrack?.release() audioTrack = null + stopDeviceMonitoring() } /** @@ -214,13 +194,14 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { fun setAudioOutputDevice(device: AudioOutputDevice) { if (audioOutputDevice != device) { audioOutputDevice = device - // 如果AudioTrack已初始化，需要重新创建以应用新的输出设备设置 - if (audioTrack != null) { - val wasPlaying = isPlaying() - initAudioTrack() - if (wasPlaying) { - audioTrack?.play() - } + + // 应用新的音频路由设置 + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + // API 31+ 可以动态更改设备 + setPreferredDeviceForTrack() + } else { + // API 30 及以下需要重新配置 + configureAudioRouting() } } } @@ -229,43 +210,19 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { * 初始化 AudioTrack */ private fun initAudioTrack() { - // 释放旧实例 audioTrack?.release() - // 计算缓冲区大小 val minBufferSize = AudioTrack.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT) val bufferSize = minBufferSize * 2 - // 根据输出设备配置AudioAttributes - val audioAttributes = when (audioOutputDevice) { - AudioOutputDevice.EARPIECE -> { - // 听筒模式 - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - AudioAttributes.Builder() - .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) - .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) - .build() - } else { - null - } - } - else -> { - // 默认、耳机、扬声器模式 - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + audioTrack = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + AudioTrack.Builder() + .setAudioAttributes( AudioAttributes.Builder() .setUsage(AudioAttributes.USAGE_MEDIA) .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) .build() - } else { - null - } - } - } - - // 创建 AudioTrack - audioTrack = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && audioAttributes != null) { - AudioTrack.Builder() - .setAudioAttributes(audioAttributes) + ) .setAudioFormat( AudioFormat.Builder() .setEncoding(AUDIO_FORMAT) @@ -278,12 +235,8 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { .build() } else { @Suppress("DEPRECATION") - val streamType = when (audioOutputDevice) { - AudioOutputDevice.EARPIECE -> AudioManager.STREAM_VOICE_CALL - else -> AudioManager.STREAM_MUSIC - } AudioTrack( - streamType, + AudioManager.STREAM_MUSIC, SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT, @@ -292,46 +245,202 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { ) } - // 设置播放位置监听器 audioTrack?.setPlaybackPositionUpdateListener(playbackListener) - // 配置音频路由 - configureAudioRouting() + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + setPreferredDeviceForTrack() + } else { + configureAudioRouting() + } - // 开始播放 audioTrack?.play() + } + + /** + * API 31+ 使用 setPreferredDevice 设置音频输出设备 + */ + private fun setPreferredDeviceForTrack() { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) return - Log.d(TAG, "AudioTrack 初始化成功") + audioManager?.let { manager -> + audioTrack?.let { track -> + when (audioOutputDevice) { + AudioOutputDevice.DEFAULT -> { + track.setPreferredDevice(null) + manager.isSpeakerphoneOn = false + manager.mode = AudioManager.MODE_NORMAL + } + + AudioOutputDevice.SPEAKER -> { + val speaker = manager.getDevices(AudioManager.GET_DEVICES_OUTPUTS) + .firstOrNull { it.type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER } + speaker?.let { track.setPreferredDevice(it) } + } + + AudioOutputDevice.HEADPHONES -> { + val headphones = manager.getDevices(AudioManager.GET_DEVICES_OUTPUTS) + .firstOrNull { device -> + device.type == AudioDeviceInfo.TYPE_WIRED_HEADSET || + device.type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES || + device.type == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP || + device.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO + } + headphones?.let { track.setPreferredDevice(it) } + ?: track.setPreferredDevice(null) + } + } + } + } } /** - * 配置音频路由 + * 配置音频路由（API 30 及以下） */ private fun configureAudioRouting() { audioManager?.let { manager -> when (audioOutputDevice) { AudioOutputDevice.DEFAULT -> { - // 默认模式：系统自动选择 manager.mode = AudioManager.MODE_NORMAL manager.isSpeakerphoneOn = false } + AudioOutputDevice.SPEAKER -> { - // 强制使用扬声器 manager.mode = AudioManager.MODE_NORMAL manager.isSpeakerphoneOn = true } + AudioOutputDevice.HEADPHONES -> { - // 强制使用耳机（如果已连接） manager.mode = AudioManager.MODE_NORMAL manager.isSpeakerphoneOn = false - // 注意：Android不能强制路由到耳机，只能在耳机已连接时使用 } - AudioOutputDevice.EARPIECE -> { - // 强制使用听筒 - manager.mode = AudioManager.MODE_IN_COMMUNICATION - manager.isSpeakerphoneOn = false + } + } + } + + /** + * 开始监听设备变化 + */ + private fun startDeviceMonitoring() { + if (isMonitoringDevices) return + + // API 23+ 使用 AudioDeviceCallback + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + deviceCallback = object : AudioDeviceCallback() { + override fun onAudioDevicesAdded(addedDevices: Array) { + super.onAudioDevicesAdded(addedDevices) + handleDeviceAdded(addedDevices) + } + + override fun onAudioDevicesRemoved(removedDevices: Array) { + super.onAudioDevicesRemoved(removedDevices) + handleDeviceRemoved(removedDevices) } } + audioManager?.registerAudioDeviceCallback(deviceCallback, null) + } + + // 所有版本都监听 NOISY 广播（耳机拔出） + noisyReceiver = object : BroadcastReceiver() { + override fun onReceive(context: Context, intent: Intent) { + if (intent.action == AudioManager.ACTION_AUDIO_BECOMING_NOISY) { + handleNoisyAudioEvent() + } + } + } + context.registerReceiver(noisyReceiver, IntentFilter(AudioManager.ACTION_AUDIO_BECOMING_NOISY)) + + isMonitoringDevices = true + } + + /** + * 停止监听设备变化 + */ + private fun stopDeviceMonitoring() { + if (!isMonitoringDevices) return + + // 注销 AudioDeviceCallback + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + deviceCallback?.let { + audioManager?.unregisterAudioDeviceCallback(it) + } + deviceCallback = null + } + + // 注销广播接收器 + noisyReceiver?.let { + try { + context.unregisterReceiver(it) + } catch (e: Exception) { + // 忽略已注销的异常 + } + } + noisyReceiver = null + + isMonitoringDevices = false + } + + /** + * 处理设备添加 + */ + private fun handleDeviceAdded(devices: Array) { + val hasHeadphones = devices.any { device -> + device.type == AudioDeviceInfo.TYPE_WIRED_HEADSET || + device.type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES || + device.type == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP || + device.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO + } + + if (hasHeadphones) { + switchToDefaultMode() + } + } + + /** + * 处理设备移除 + */ + private fun handleDeviceRemoved(devices: Array) { + val hasHeadphones = devices.any { device -> + device.type == AudioDeviceInfo.TYPE_WIRED_HEADSET || + device.type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES || + device.type == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP || + device.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO + } + + if (hasHeadphones) { + switchToSpeakerMode() + } + } + + /** + * 处理音频变得嘈杂事件（通常是耳机拔出） + */ + private fun handleNoisyAudioEvent() { + switchToSpeakerMode() + } + + /** + * 切换到 DEFAULT 模式 + */ + private fun switchToDefaultMode() { + audioOutputDevice = AudioOutputDevice.DEFAULT + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + setPreferredDeviceForTrack() + } else { + configureAudioRouting() + } + } + + /** + * 切换到扬声器模式 + */ + private fun switchToSpeakerMode() { + audioOutputDevice = AudioOutputDevice.SPEAKER + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + setPreferredDeviceForTrack() + } else { + configureAudioRouting() } } } \ No newline at end of file diff --git a/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt b/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt index 13f74a898..55485623b 100644 --- a/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt +++ b/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt @@ -218,7 +218,8 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor mcpConfigJson = mcpServer // 异步初始化MCP客户端 launch { - initializeMcpClient(mcpServer) + // initializeMcpClient(mcpServer) + initializeMcpClient("{}") } isInitialized = apiKey.isNotEmpty() diff --git a/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/TtsEvents.kt b/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/TtsEvents.kt index 6758c6f05..47bea05bf 100644 --- a/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/TtsEvents.kt +++ b/local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/TtsEvents.kt @@ -67,5 +67,4 @@ enum class AudioOutputDevice { DEFAULT, // 默认（如果有耳机选耳机，否则使用系统扬声器） HEADPHONES, // 强制使用耳机 SPEAKER, // 强制使用扬声器 - EARPIECE // 强制使用听筒 } \ No newline at end of file From 2f4744357fa40edc7af741e03450397c956f1063 Mon Sep 17 00:00:00 2001 From: wolfplus2048 <30993207+wolfplus2048@users.noreply.github.com> Date: Sun, 22 Jun 2025 14:03:52 +0100 Subject: [PATCH 06/10] before job --- .../agent/controllers/agent_controller.dart | 22 ++++++-- .../agent_service/AgentService.kt | 53 +++++++++++-------- .../agent_service/AgentServiceImpl.swift | 8 +++ .../agent_service/lib/agent_service.dart | 10 ++++ .../chat_api/SystemFunctionHandler.kt | 22 ++++---- 5 files changed, 76 insertions(+), 39 deletions(-) diff --git a/lib/modules/agent/controllers/agent_controller.dart b/lib/modules/agent/controllers/agent_controller.dart index 2bbaa3d78..cc98107c4 100644 --- a/lib/modules/agent/controllers/agent_controller.dart +++ b/lib/modules/agent/controllers/agent_controller.dart @@ -253,6 +253,7 @@ class AgentController extends GetxController { case AgentServiceEventType.recognizing: final text = event.data['text'] ?? ''; currentText.value = text; // 保留当前文本，以便其他地方使用 + Logger.i(TAG, '识别中间结果: $text'); if (text.isNotEmpty) { // 查找是否有正在识别中的消息 final index = messages @@ -280,7 +281,7 @@ class AgentController extends GetxController { case AgentServiceEventType.recognitionResult: final text = event.data['text'] ?? ''; - Logger.i(TAG, '识别结果: $text'); + Logger.i(TAG, '识别最终结果: $text'); // 查找是否有正在识别中的消息 final index = messages.lastIndexWhere((msg) => msg.isRecognizing && msg.isUser); @@ -313,16 +314,27 @@ class AgentController extends GetxController { break; case AgentServiceEventType.ttsStarted: - isSpeaking.value = true; - Logger.i(TAG, 'TTS开始播放'); + // isSpeaking.value = true; + // Logger.i(TAG, 'TTS开始播放'); break; case AgentServiceEventType.ttsCompleted: case AgentServiceEventType.ttsStopped: case AgentServiceEventType.ttsCanceled: + // isSpeaking.value = false; + + // Logger.i(TAG, 'TTS停止播放, $event.type'); + break; + + case AgentServiceEventType.playbackStarted: + isSpeaking.value = true; + Logger.i(TAG, 'TTS播放开始'); + break; + + case AgentServiceEventType.playbackCompleted: isSpeaking.value = false; - Logger.i(TAG, 'TTS停止播放, $event.type'); + Logger.i(TAG, 'TTS播放完成'); break; case AgentServiceEventType.imageProcessing: @@ -338,7 +350,7 @@ class AgentController extends GetxController { final token = event.data['token'] ?? ''; final responseId = event.data['responseId'] ?? ''; - + Logger.i(TAG, 'AI回复Token: $token, responseId: $responseId'); if (token.isNotEmpty) { // 如果是新的回复或者响应ID改变，创建新消息 if (_isNewAssistantResponse || diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index d9d2c9700..ea0a1b19a 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -266,6 +266,14 @@ object AgentService : CoroutineScope { restartIdleCheck() sendEvent("tts_canceled", mapOf("status" to "canceled")) } + TtsEventType.PLAYBACK_STARTED -> { + restartIdleCheck() + sendEvent("playback_started", mapOf("status" to "playback_started")) + } + TtsEventType.PLAYBACK_COMPLETED -> { + restartIdleCheck() + sendEvent("playback_completed", mapOf("status" to "playback_completed")) + } TtsEventType.ERROR -> { isTtsSpeaking = false restartIdleCheck() @@ -400,21 +408,21 @@ object AgentService : CoroutineScope { if (isTtsSpeaking || isAiStreaming) { val currentTime = System.currentTimeMillis() - // 防抖处理：避免过于频繁的打断 - if (currentTime - lastInterruptTime < INTERRUPT_DEBOUNCE_MS) { - return - } + // // 防抖处理：避免过于频繁的打断 + // if (currentTime - lastInterruptTime < INTERRUPT_DEBOUNCE_MS) { + // return + // } - // 简单过滤：太短的内容可能是噪音 - if (recognizing.trim().length < 2) { - return - } + // // 简单过滤：太短的内容可能是噪音 + // if (recognizing.trim().length < 2) { + // return + // } - // 过滤纯语气词 - val trimmedText = recognizing.trim().lowercase() - if (FILLER_WORDS.contains(trimmedText)) { - return - } + // // 过滤纯语气词 + // val trimmedText = recognizing.trim().lowercase() + // if (FILLER_WORDS.contains(trimmedText)) { + // return + // } // 执行打断 lastInterruptTime = currentTime @@ -520,18 +528,17 @@ object AgentService : CoroutineScope { val startTime = System.currentTimeMillis() // 并行执行停止操作，加快响应速度 - launch { stopTts() } - launch { stopAiStream() } + stopTts() + stopAiStream() // 记录打断耗时 - launch { - delay(100) // 短暂延迟后计算耗时 - val duration = System.currentTimeMillis() - startTime - sendEvent("response_interrupted", mapOf( - "status" to "interrupted", - "duration_ms" to duration - )) - } + + val duration = System.currentTimeMillis() - startTime + sendEvent("response_interrupted", mapOf( + "status" to "interrupted", + "duration_ms" to duration + )) + } } diff --git a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift index e5035bf17..0dc9119f6 100644 --- a/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift +++ b/local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift @@ -909,6 +909,14 @@ extension AgentServiceImpl: TtsEventListener { restartIdleCheck() sendEvent(name: "tts_completed", data: ["status": "completed"]) + case .playbackStarted: + restartIdleCheck() + sendEvent(name: "playback_started", data: ["status": "playback_started"]) + + case .playbackCompleted: + restartIdleCheck() + sendEvent(name: "playback_completed", data: ["status": "playback_completed"]) + case .synthesisCanceled: isSpeaking = false restartIdleCheck() diff --git a/local_plugins/agent_service/lib/agent_service.dart b/local_plugins/agent_service/lib/agent_service.dart index 4ee6907cb..fc1579a05 100644 --- a/local_plugins/agent_service/lib/agent_service.dart +++ b/local_plugins/agent_service/lib/agent_service.dart @@ -38,6 +38,12 @@ enum AgentServiceEventType { /// TTS停止 ttsStopped, + /// 播放开始 + playbackStarted, + + /// 播放完成 + playbackCompleted, + /// 响应中断 responseInterrupted, @@ -147,6 +153,10 @@ class AgentService { return AgentServiceEventType.ttsCanceled; case 'tts_stopped': return AgentServiceEventType.ttsStopped; + case 'playback_started': + return AgentServiceEventType.playbackStarted; + case 'playback_completed': + return AgentServiceEventType.playbackCompleted; case 'response_interrupted': return AgentServiceEventType.responseInterrupted; case 'assistant_token': diff --git a/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/SystemFunctionHandler.kt b/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/SystemFunctionHandler.kt index 1c5b7de75..24f56af0d 100644 --- a/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/SystemFunctionHandler.kt +++ b/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/SystemFunctionHandler.kt @@ -47,17 +47,17 @@ class SystemFunctionHandler(private val context: Context? = null) { handler = ExitInteractionHandler(context) ) - // 注册翻译模式函数 - client.registerLocalFunction( - name = "enter_translation_mode", - description = "用户请求进入实时翻译模式时，启动实时翻译功能", - parameters = mapOf( - "type" to "object", - "properties" to emptyMap(), - "required" to emptyList() - ), - handler = TranslationModeHandler(context) - ) + // // 注册翻译模式函数 + // client.registerLocalFunction( + // name = "enter_translation_mode", + // description = "用户请求进入实时翻译模式时，启动实时翻译功能", + // parameters = mapOf( + // "type" to "object", + // "properties" to emptyMap(), + // "required" to emptyList() + // ), + // handler = TranslationModeHandler(context) + // ) // 注册发送短信函数 client.registerLocalFunction( From bcda9422e8d6ee5a49caec1b5ff031b369611bb0 Mon Sep 17 00:00:00 2001 From: wolfplus2048 <30993207+wolfplus2048@users.noreply.github.com> Date: Sun, 22 Jun 2025 15:47:04 +0100 Subject: [PATCH 07/10] add --- .../agent/controllers/agent_controller.dart | 2 +- .../agent_service/AgentService.kt | 113 ++++++++++-------- .../bytedance_speech/BytedanceAudioPlayer.kt | 40 +++++-- .../chat_api/ChatApiService.kt | 47 +++++++- .../yunqiinnovation/chat_api/MCPSubClient.kt | 4 +- 5 files changed, 140 insertions(+), 66 deletions(-) diff --git a/lib/modules/agent/controllers/agent_controller.dart b/lib/modules/agent/controllers/agent_controller.dart index cc98107c4..7cdebb046 100644 --- a/lib/modules/agent/controllers/agent_controller.dart +++ b/lib/modules/agent/controllers/agent_controller.dart @@ -350,7 +350,7 @@ class AgentController extends GetxController { final token = event.data['token'] ?? ''; final responseId = event.data['responseId'] ?? ''; - Logger.i(TAG, 'AI回复Token: $token, responseId: $responseId'); + // Logger.i(TAG, 'AI回复Token: $token, responseId: $responseId'); if (token.isNotEmpty) { // 如果是新的回复或者响应ID改变，创建新消息 if (_isNewAssistantResponse || diff --git a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt index ea0a1b19a..e18de276f 100644 --- a/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt +++ b/local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt @@ -16,6 +16,9 @@ import java.util.Collections import kotlin.coroutines.CoroutineContext import com.yunqiinnovation.ble_service.BleService import android.media.MediaPlayer +import java.util.concurrent.atomic.AtomicBoolean +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import com.deep_voice.speech.tts.TtsEvent import com.deep_voice.speech.tts.TtsEventListener import com.deep_voice.speech.tts.TtsEventType @@ -43,7 +46,7 @@ object AgentService : CoroutineScope { // 协程相关 private val job = SupervisorJob() override val coroutineContext: CoroutineContext - get() = Dispatchers.Main + job + get() = Dispatchers.IO + job // 上下文和监听器 private lateinit var context: Context @@ -73,16 +76,24 @@ object AgentService : CoroutineScope { // 音频播放器 private var audioPlayer: AudioPlayer? = null // 初始化音频播放器 - // 状态 - private var isInitialized = false - var isRecognitionActive = false - private set - var isTtsSpeaking = false - private set - var hasSpeechDetected = false - private set - var isAiStreaming = false - private set + // 状态 - 使用原子类型确保线程安全 + private val _isInitialized = AtomicBoolean(false) + val isInitialized: Boolean get() = _isInitialized.get() + + private val _isRecognitionActive = AtomicBoolean(false) + val isRecognitionActive: Boolean get() = _isRecognitionActive.get() + + private val _isTtsSpeaking = AtomicBoolean(false) + val isTtsSpeaking: Boolean get() = _isTtsSpeaking.get() + + private val _hasSpeechDetected = AtomicBoolean(false) + val hasSpeechDetected: Boolean get() = _hasSpeechDetected.get() + + private val _isAiStreaming = AtomicBoolean(false) + val isAiStreaming: Boolean get() = _isAiStreaming.get() + + // 用于保护复杂状态操作的互斥锁 + private val stateMutex = Mutex() // AI流生成相关 private var currentAiJob: Job? = null @@ -175,7 +186,7 @@ object AgentService : CoroutineScope { // 加载最近的聊天记录 loadChatHistory() - isInitialized = true + _isInitialized.set(true) return true } catch (e: Exception) { Log.e(TAG, "初始化失败: ${e.message}") @@ -201,6 +212,7 @@ object AgentService : CoroutineScope { // 释放ChatAPI服务 if (::chatApiService.isInitialized) { chatApiService.cancelCurrentStream() + chatApiService.dispose() } // 释放TTS服务 @@ -209,7 +221,7 @@ object AgentService : CoroutineScope { job.cancel() clearListeners() - isInitialized = false + _isInitialized.set(false) } catch (e: Exception) { Log.e(TAG, "释放资源异常: ${e.message}") } @@ -252,17 +264,17 @@ object AgentService : CoroutineScope { override fun onEvent(event: TtsEvent) { when (event.type) { TtsEventType.SYNTHESIS_STARTED -> { - isTtsSpeaking = true + _isTtsSpeaking.set(true) restartIdleCheck() sendEvent("tts_started", mapOf("status" to "started")) } TtsEventType.SYNTHESIS_COMPLETED -> { - isTtsSpeaking = false + _isTtsSpeaking.set(false) restartIdleCheck() sendEvent("tts_completed", mapOf("status" to "completed")) } TtsEventType.SYNTHESIS_CANCELED -> { - isTtsSpeaking = false + _isTtsSpeaking.set(false) restartIdleCheck() sendEvent("tts_canceled", mapOf("status" to "canceled")) } @@ -275,7 +287,7 @@ object AgentService : CoroutineScope { sendEvent("playback_completed", mapOf("status" to "playback_completed")) } TtsEventType.ERROR -> { - isTtsSpeaking = false + _isTtsSpeaking.set(false) restartIdleCheck() val params = event.params val code = params["errorCode"] as? String ?: "UNKNOWN_ERROR" @@ -379,8 +391,8 @@ object AgentService : CoroutineScope { return false } - isRecognitionActive = true - hasSpeechDetected = false + _isRecognitionActive.set(true) + _hasSpeechDetected.set(false) try { val audioSourceType = if (isExternalActive) { @@ -393,7 +405,7 @@ object AgentService : CoroutineScope { if (recognizing.isNotEmpty()) { // 检测到语音，更新状态 val previousHasSpeech = hasSpeechDetected - hasSpeechDetected = true + _hasSpeechDetected.set(true) if (!previousHasSpeech) { restartIdleCheck() @@ -408,21 +420,21 @@ object AgentService : CoroutineScope { if (isTtsSpeaking || isAiStreaming) { val currentTime = System.currentTimeMillis() - // // 防抖处理：避免过于频繁的打断 - // if (currentTime - lastInterruptTime < INTERRUPT_DEBOUNCE_MS) { - // return - // } + // 防抖处理：避免过于频繁的打断 + if (currentTime - lastInterruptTime < INTERRUPT_DEBOUNCE_MS) { + return + } - // // 简单过滤：太短的内容可能是噪音 - // if (recognizing.trim().length < 2) { - // return - // } + // 简单过滤：太短的内容可能是噪音 + if (recognizing.trim().length < 2) { + return + } - // // 过滤纯语气词 - // val trimmedText = recognizing.trim().lowercase() - // if (FILLER_WORDS.contains(trimmedText)) { - // return - // } + // 过滤纯语气词 + val trimmedText = recognizing.trim().lowercase() + if (FILLER_WORDS.contains(trimmedText)) { + return + } // 执行打断 lastInterruptTime = currentTime @@ -443,7 +455,7 @@ object AgentService : CoroutineScope { // 重置状态，继续识别 val previousHasSpeech = hasSpeechDetected - hasSpeechDetected = false + _hasSpeechDetected.set(false) if (previousHasSpeech) { restartIdleCheck() @@ -460,13 +472,13 @@ object AgentService : CoroutineScope { override fun onSessionStopped() { sendEvent("recognition_stopped", mapOf("status" to "stopped")) - isRecognitionActive = false + _isRecognitionActive.set(false) stopIdleCheck() audioPlayer?.playAudio(R.raw.stop) } override fun onCanceled(reason: String, errorDetails: String) { - isRecognitionActive = false + _isRecognitionActive.set(false) stopIdleCheck() BleService.closeCodec() sendEvent("recognition_canceled", mapOf( @@ -476,7 +488,7 @@ object AgentService : CoroutineScope { } override fun onError(error: String) { - isRecognitionActive = false + _isRecognitionActive.set(false) stopIdleCheck() BleService.closeCodec() Log.e(TAG, "语音识别错误: $error") @@ -488,7 +500,7 @@ object AgentService : CoroutineScope { }, audioSourceType) return true } catch (e: Exception) { - isRecognitionActive = false + _isRecognitionActive.set(false) Log.e(TAG, "启动语音识别失败: ${e.message}") sendEvent("error", mapOf( "code" to "RECOGNITION_START_ERROR", @@ -510,11 +522,11 @@ object AgentService : CoroutineScope { try { azureAsrHelper?.stopContinuousRecognition() BleService.closeCodec() - isRecognitionActive = false + _isRecognitionActive.set(false) stopIdleCheck() } catch (e: Exception) { Log.e(TAG, "停止语音识别异常: ${e.message}") - isRecognitionActive = false + _isRecognitionActive.set(false) stopIdleCheck() } } @@ -549,7 +561,7 @@ object AgentService : CoroutineScope { if (isAiStreaming) { try { // 先更新状态，避免回调时的状态不一致 - isAiStreaming = false + _isAiStreaming.set(false) // 取消当前AI生成任务 currentAiJob?.cancel() @@ -561,7 +573,7 @@ object AgentService : CoroutineScope { } catch (e: Exception) { Log.e(TAG, "停止AI流输出异常", e) // 确保状态被重置，即使发生异常 - isAiStreaming = false + _isAiStreaming.set(false) currentAiJob = null } } @@ -655,7 +667,7 @@ object AgentService : CoroutineScope { currentAiJob = launch { try { // 设置状态为正在流式输出 - isAiStreaming = true + _isAiStreaming.set(true) // 使用历史记录作为上下文发送到OpenAI val responseBuilder = StringBuilder() @@ -753,7 +765,7 @@ object AgentService : CoroutineScope { // 保存聊天记录 saveChatMessage(displayText, response,aiMetadata,userMetadata.toString()) // 标记AI流式输出已完成 - isAiStreaming = false + _isAiStreaming.set(false) currentAiJob = null } @@ -765,7 +777,7 @@ object AgentService : CoroutineScope { )) // 标记AI流式输出已完成 - isAiStreaming = false + _isAiStreaming.set(false) currentAiJob = null } @@ -809,7 +821,7 @@ object AgentService : CoroutineScope { )) // 确保状态被重置 - isAiStreaming = false + _isAiStreaming.set(false) currentAiJob = null } } @@ -873,7 +885,7 @@ object AgentService : CoroutineScope { if (text.isEmpty()) return // 更新状态 - isTtsSpeaking = true + _isTtsSpeaking.set(true) restartIdleCheck() // 状态变化，重启检测 // 直接调用TTS，无需协程包装 @@ -887,7 +899,7 @@ object AgentService : CoroutineScope { if (isTtsSpeaking) { ttsService?.stop() - isTtsSpeaking = false + _isTtsSpeaking.set(false) restartIdleCheck() // 状态变化，重启检测 sendEvent("tts_stopped", mapOf("status" to "stopped")) } @@ -964,9 +976,8 @@ object AgentService : CoroutineScope { * 发送事件 */ private fun sendEvent(eventName: String, data: Map) { - // 使用协程确保在主线程上执行 - launch { - // 我们已在主线程上下文中启动协程，无需再切换线程 + // 切换到主线程执行监听器回调，避免 UI 更新问题 + launch(Dispatchers.Main) { listeners.forEach { listener -> try { listener.onEvent(eventName, data) diff --git a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt index ac7cf0afd..b3cffcbd1 100644 --- a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt +++ b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt @@ -24,6 +24,7 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { private const val SAMPLE_RATE = 24000 private const val CHANNEL_CONFIG = AudioFormat.CHANNEL_OUT_MONO private const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT + private const val VOLUME_REDUCTION = 0.7f // 降低音量以减少回音 } private var audioTrack: AudioTrack? = null @@ -206,6 +207,7 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { } } + /** * 初始化 AudioTrack */ @@ -213,13 +215,18 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { audioTrack?.release() val minBufferSize = AudioTrack.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT) - val bufferSize = minBufferSize * 2 + val bufferSize = minBufferSize * 2 // 使用较小的缓冲区以降低延迟 audioTrack = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - AudioTrack.Builder() + val builder = AudioTrack.Builder() .setAudioAttributes( AudioAttributes.Builder() - .setUsage(AudioAttributes.USAGE_MEDIA) + .setUsage( + // 扬声器模式下默认使用语音通信模式以启用回声消除 + if (audioOutputDevice == AudioOutputDevice.SPEAKER) + AudioAttributes.USAGE_VOICE_COMMUNICATION + else AudioAttributes.USAGE_MEDIA + ) .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) .build() ) @@ -232,7 +239,13 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { ) .setBufferSizeInBytes(bufferSize) .setTransferMode(AudioTrack.MODE_STREAM) - .build() + + // API 26+ 设置低延迟模式 + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + builder.setPerformanceMode(AudioTrack.PERFORMANCE_MODE_LOW_LATENCY) + } + + builder.build() } else { @Suppress("DEPRECATION") AudioTrack( @@ -247,6 +260,9 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { audioTrack?.setPlaybackPositionUpdateListener(playbackListener) + // 设置音量以减少回音 + audioTrack?.setVolume(VOLUME_REDUCTION) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { setPreferredDeviceForTrack() } else { @@ -268,13 +284,17 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { AudioOutputDevice.DEFAULT -> { track.setPreferredDevice(null) manager.isSpeakerphoneOn = false - manager.mode = AudioManager.MODE_NORMAL + // 默认模式下使用通信模式以获得更好的音频处理 + manager.mode = AudioManager.MODE_IN_COMMUNICATION } AudioOutputDevice.SPEAKER -> { val speaker = manager.getDevices(AudioManager.GET_DEVICES_OUTPUTS) .firstOrNull { it.type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER } speaker?.let { track.setPreferredDevice(it) } + // 扬声器模式下始终启用通信模式以获得回声消除 + manager.mode = AudioManager.MODE_IN_COMMUNICATION + manager.isSpeakerphoneOn = true } AudioOutputDevice.HEADPHONES -> { @@ -287,6 +307,9 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { } headphones?.let { track.setPreferredDevice(it) } ?: track.setPreferredDevice(null) + // 耳机模式下可以使用普通模式 + manager.mode = AudioManager.MODE_NORMAL + manager.isSpeakerphoneOn = false } } } @@ -300,16 +323,19 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { audioManager?.let { manager -> when (audioOutputDevice) { AudioOutputDevice.DEFAULT -> { - manager.mode = AudioManager.MODE_NORMAL + // 默认模式下使用通信模式以获得更好的音频处理 + manager.mode = AudioManager.MODE_IN_COMMUNICATION manager.isSpeakerphoneOn = false } AudioOutputDevice.SPEAKER -> { - manager.mode = AudioManager.MODE_NORMAL + // 扬声器模式下始终启用通信模式以获得回声消除 + manager.mode = AudioManager.MODE_IN_COMMUNICATION manager.isSpeakerphoneOn = true } AudioOutputDevice.HEADPHONES -> { + // 耳机模式下可以使用普通模式 manager.mode = AudioManager.MODE_NORMAL manager.isSpeakerphoneOn = false } diff --git a/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt b/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt index 55485623b..ee7fae90a 100644 --- a/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt +++ b/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt @@ -111,6 +111,10 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor override val coroutineContext: CoroutineContext = Dispatchers.IO + SupervisorJob() + companion object { + private const val TAG = "ChatApiService" + } + // MARK: - 属性 private var baseUrl = "https://api.openai.com/v1/" private var apiKey = "" @@ -218,8 +222,8 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor mcpConfigJson = mcpServer // 异步初始化MCP客户端 launch { - // initializeMcpClient(mcpServer) - initializeMcpClient("{}") + initializeMcpClient(mcpServer) + // initializeMcpClient("{}") } isInitialized = apiKey.isNotEmpty() @@ -514,8 +518,8 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor ) // 通知上层工具调用事件 currSessionCallback?.onFunctionCall(convertMapToJsonObject(functionCall)) - // 在后台队列处理工具调用 - launch { + // 在当前协程作用域内处理工具调用，使用async确保生命周期管理 + val toolCallDeferred = async { try { if (sessionid == currSessionId) { // 通过MCP客户端处理工具调用 @@ -587,6 +591,15 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor } } + // 等待工具调用完成，确保生命周期管理 + try { + toolCallDeferred.await() + } catch (e: CancellationException) { + // 协程被取消，确保子任务也被取消 + toolCallDeferred.cancel() + throw e + } + return true } @@ -780,7 +793,7 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor description = description, parameters = parameters ) - Log.d("ChatApiService", "AI携带工具: $name, 参数定义: $parameters") + // Log.d("ChatApiService", "AI携带工具: $name, 参数定义: $parameters") tools.add(tool) } catch (e: Exception) { @@ -813,6 +826,30 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor } } + /** + * 释放所有资源 + * 统一的资源管理方法，确保所有协程被正确取消 + */ + fun dispose() { + runBlocking { + // 取消当前流式任务 + currentStreamJob?.cancelAndJoin() + currentStreamJob = null + + // 关闭MCP客户端 + closeMcpClient() + + // 取消所有子协程 + coroutineContext[Job]?.cancelChildren() + + // 清理其他资源 + currSessionCallback = null + currSessionId = "" + currentMessages = emptyList() + toolCalls.clear() + } + } + /** * 处理MCP工具调用 * diff --git a/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/MCPSubClient.kt b/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/MCPSubClient.kt index 3fe6b926a..77393a44a 100644 --- a/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/MCPSubClient.kt +++ b/local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/MCPSubClient.kt @@ -129,7 +129,7 @@ class MCPSubClient( "properties" to emptyMap(), "required" to emptyList() ) - Log.w(TAG, "解析工具数据: ${tool.name} ${parametersMap}") + // Log.w(TAG, "解析工具数据: ${tool.name} ${parametersMap}") val toolMap = mapOf( "type" to "function", "function" to mapOf( @@ -175,7 +175,7 @@ class MCPSubClient( name = name, arguments = argumentsJson ) - + Log.d(TAG, "[$serverId] 调用工具 '$name'，参数: ${argumentsJson.toString()}") // 调用工具 val result = mcpClient?.callTool(request) From f5b7c7a2117040634afacaa8666b2bf6eb126691 Mon Sep 17 00:00:00 2001 From: wolfplus2048 <30993207+wolfplus2048@users.noreply.github.com> Date: Sun, 22 Jun 2025 22:32:36 +0100 Subject: [PATCH 08/10] add --- .../bytedance_speech/BytedanceAudioPlayer.kt | 444 ++++++------------ .../bytedance_speech/BytedanceTTS.kt | 4 +- 2 files changed, 150 insertions(+), 298 deletions(-) diff --git a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt index b3cffcbd1..a761c015c 100644 --- a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt +++ b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt @@ -1,9 +1,6 @@ package com.deep_voice.bytedance_speech -import android.content.BroadcastReceiver import android.content.Context -import android.content.Intent -import android.content.IntentFilter import android.media.AudioAttributes import android.media.AudioDeviceCallback import android.media.AudioDeviceInfo @@ -11,38 +8,46 @@ import android.media.AudioFormat import android.media.AudioManager import android.media.AudioTrack import android.os.Build +import android.os.Handler +import android.os.Looper +import android.util.Log import com.deep_voice.speech.tts.AudioDataListener import com.deep_voice.speech.tts.AudioOutputDevice /** - * 简化的音频流播放器 - * 基于位置标记触发播放完成回调 + * 高性能音频播放器 - 16kHz采样率 + * 设计要点： + * • 永远使用 USAGE_MEDIA + MODE_NORMAL 媒体播放通路 + * • 利用 AudioDeviceCallback 监听设备热插拔 + * • API 31+ 使用 setPreferredDevice() 精准路由 + * • 有耳机自动从耳机播放，无耳机自动切到扬声器 */ -class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { +class BytedanceAudioPlayer(private val context: Context) : AudioDataListener, AudioDeviceCallback() { companion object { private const val TAG = "BytedanceAudioPlayer" - private const val SAMPLE_RATE = 24000 + private const val SAMPLE_RATE = 16000 // 固定16kHz采样率 private const val CHANNEL_CONFIG = AudioFormat.CHANNEL_OUT_MONO private const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT - private const val VOLUME_REDUCTION = 0.7f // 降低音量以减少回音 } + private val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager private var audioTrack: AudioTrack? = null - private var isFirstData = true // 是否是第一次接收数据 - private var totalBytesWritten = 0 // 总共写入的字节数 - private var sessionActive = true // 会话是否活跃 - private var audioOutputDevice = AudioOutputDevice.DEFAULT // 音频输出设备 - private var audioManager: AudioManager? = null - - // 设备监听相关 - private var deviceCallback: AudioDeviceCallback? = null - private var noisyReceiver: BroadcastReceiver? = null - private var isMonitoringDevices = false + private var isFirstData = true + private var totalBytesWritten = 0 + private var sessionActive = true // 回调 private var onPlayStarted: (() -> Unit)? = null private var onPlayCompleted: (() -> Unit)? = null + // 耳机设备类型（优先级顺序） + private val headsetTypes = intArrayOf( + AudioDeviceInfo.TYPE_WIRED_HEADSET, + AudioDeviceInfo.TYPE_WIRED_HEADPHONES, + AudioDeviceInfo.TYPE_BLUETOOTH_A2DP + // TYPE_BLUETOOTH_LE 在较新的 API 中才有，这里先不包含 + ) + // 播放位置监听器 private val playbackListener = object : AudioTrack.OnPlaybackPositionUpdateListener { override fun onMarkerReached(track: AudioTrack) { @@ -55,10 +60,113 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { } init { - audioManager = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager - initAudioTrack() + // 注册设备热插拔监听 + audioManager.registerAudioDeviceCallback(this, Handler(Looper.getMainLooper())) + buildTrack(selectOutputDevice()) // 首次创建 + } + + /** + * 监听输出设备变化 + */ + override fun onAudioDevicesAdded(addedDevices: Array) { + Log.d(TAG, "音频设备连接: ${addedDevices.joinToString { "${it.productName}(${it.type})" }}") + rebuildIfNeeded() + } + + override fun onAudioDevicesRemoved(removedDevices: Array) { + Log.d(TAG, "音频设备断开: ${removedDevices.joinToString { "${it.productName}(${it.type})" }}") + rebuildIfNeeded() + } + + /** + * 检查设备变化，必要时重建AudioTrack + */ + private fun rebuildIfNeeded() { + val newDevice = selectOutputDevice() + val currentDevice = audioTrack?.preferredDevice + + // 设备发生变化时重建 + if (newDevice?.id != currentDevice?.id) { + Log.d(TAG, "设备切换: ${currentDevice?.productName} -> ${newDevice?.productName}") + + // 保存当前状态 + val wasPlaying = audioTrack?.playState == AudioTrack.PLAYSTATE_PLAYING + val savedFirstData = isFirstData + val savedTotalBytes = totalBytesWritten + val savedSessionActive = sessionActive + + // 重建AudioTrack + buildTrack(newDevice) + + // 恢复状态 + if (wasPlaying) { + isFirstData = savedFirstData + totalBytesWritten = savedTotalBytes + sessionActive = savedSessionActive + } + } + } + + /** + * 选择当前优先输出设备（耳机优先） + */ + private fun selectOutputDevice(): AudioDeviceInfo? { + return audioManager.getDevices(AudioManager.GET_DEVICES_OUTPUTS) + .firstOrNull { it.type in headsetTypes } + } + + /** + * 构建16kHz单声道AudioTrack + */ + private fun buildTrack(device: AudioDeviceInfo?) { + audioTrack?.release() + + val minBufferSize = AudioTrack.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT) + + audioTrack = AudioTrack.Builder() + .setAudioAttributes( + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_MEDIA) // 统一媒体通路 + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + ) + .setAudioFormat( + AudioFormat.Builder() + .setSampleRate(SAMPLE_RATE) + .setEncoding(AUDIO_FORMAT) + .setChannelMask(CHANNEL_CONFIG) + .build() + ) + .setTransferMode(AudioTrack.MODE_STREAM) + .setBufferSizeInBytes(minBufferSize * 2) // 小缓冲区保证低延迟 + .apply { + // 低延迟模式 + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + setPerformanceMode(AudioTrack.PERFORMANCE_MODE_LOW_LATENCY) + } + } + .build().also { track -> + // API 31+ 精准路由到指定设备 + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && device != null) { + track.preferredDevice = device + Log.d(TAG, "设置首选设备: ${device.productName} (${device.type})") + } + + // 设置播放监听器 + track.setPlaybackPositionUpdateListener(playbackListener) + + // 开始播放 + track.play() + } + + // 配置音频管理器 - 统一使用 MODE_NORMAL + audioManager.mode = AudioManager.MODE_NORMAL + // 切换扬声器开关仅在无耳机时打开 + audioManager.isSpeakerphoneOn = (device == null) + + val deviceName = device?.productName ?: "扬声器" + Log.d(TAG, "AudioTrack已创建 - 输出设备: $deviceName, 采样率: ${SAMPLE_RATE}Hz") } - /** * 开始新的播放会话 @@ -74,9 +182,7 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { track.flush() track.play() } - } ?: initAudioTrack() - - startDeviceMonitoring() + } ?: buildTrack(selectOutputDevice()) } /** @@ -94,9 +200,9 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { track.setNotificationMarkerPosition(totalFrames) } catch (e: Exception) { val durationMs = (totalFrames * 1000L) / SAMPLE_RATE - android.os.Handler(android.os.Looper.getMainLooper()).postDelayed({ + Handler(Looper.getMainLooper()).postDelayed({ onPlayCompleted?.invoke() - }, durationMs + 500) + }, durationMs + 100) // 更短的延迟 } } else { onPlayCompleted?.invoke() @@ -127,7 +233,7 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { } } } catch (e: Exception) { - // 忽略写入失败 + Log.w(TAG, "音频数据写入失败: ${e.message}") } } @@ -144,8 +250,6 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { } } - stopDeviceMonitoring() - if (!isFirstData) { onPlayCompleted?.invoke() } @@ -158,7 +262,7 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { stop() audioTrack?.release() audioTrack = null - stopDeviceMonitoring() + audioManager.unregisterAudioDeviceCallback(this) } /** @@ -190,283 +294,31 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener { } /** - * 设置音频输出设备 + * 设置音频输出设备（兼容接口） + * 注意：此实现依赖自动设备切换，手动设置可能被热插拔覆盖 */ fun setAudioOutputDevice(device: AudioOutputDevice) { - if (audioOutputDevice != device) { - audioOutputDevice = device - - // 应用新的音频路由设置 - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - // API 31+ 可以动态更改设备 - setPreferredDeviceForTrack() - } else { - // API 30 及以下需要重新配置 - configureAudioRouting() - } - } + // 本实现专注于自动设备切换 + // 手动设置会在下次设备变化时被自动逻辑覆盖 + Log.d(TAG, "setAudioOutputDevice被调用，但本实现依赖自动设备切换") } - /** - * 初始化 AudioTrack + * 获取当前路由设备信息（用于调试） */ - private fun initAudioTrack() { - audioTrack?.release() - - val minBufferSize = AudioTrack.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT) - val bufferSize = minBufferSize * 2 // 使用较小的缓冲区以降低延迟 - - audioTrack = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - val builder = AudioTrack.Builder() - .setAudioAttributes( - AudioAttributes.Builder() - .setUsage( - // 扬声器模式下默认使用语音通信模式以启用回声消除 - if (audioOutputDevice == AudioOutputDevice.SPEAKER) - AudioAttributes.USAGE_VOICE_COMMUNICATION - else AudioAttributes.USAGE_MEDIA - ) - .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) - .build() - ) - .setAudioFormat( - AudioFormat.Builder() - .setEncoding(AUDIO_FORMAT) - .setSampleRate(SAMPLE_RATE) - .setChannelMask(CHANNEL_CONFIG) - .build() - ) - .setBufferSizeInBytes(bufferSize) - .setTransferMode(AudioTrack.MODE_STREAM) - - // API 26+ 设置低延迟模式 - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - builder.setPerformanceMode(AudioTrack.PERFORMANCE_MODE_LOW_LATENCY) - } - - builder.build() + fun getCurrentDevice(): String { + val routedDevice = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + audioTrack?.routedDevice } else { - @Suppress("DEPRECATION") - AudioTrack( - AudioManager.STREAM_MUSIC, - SAMPLE_RATE, - CHANNEL_CONFIG, - AUDIO_FORMAT, - bufferSize, - AudioTrack.MODE_STREAM - ) - } - - audioTrack?.setPlaybackPositionUpdateListener(playbackListener) - - // 设置音量以减少回音 - audioTrack?.setVolume(VOLUME_REDUCTION) - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - setPreferredDeviceForTrack() - } else { - configureAudioRouting() - } - - audioTrack?.play() - } - - /** - * API 31+ 使用 setPreferredDevice 设置音频输出设备 - */ - private fun setPreferredDeviceForTrack() { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) return - - audioManager?.let { manager -> - audioTrack?.let { track -> - when (audioOutputDevice) { - AudioOutputDevice.DEFAULT -> { - track.setPreferredDevice(null) - manager.isSpeakerphoneOn = false - // 默认模式下使用通信模式以获得更好的音频处理 - manager.mode = AudioManager.MODE_IN_COMMUNICATION - } - - AudioOutputDevice.SPEAKER -> { - val speaker = manager.getDevices(AudioManager.GET_DEVICES_OUTPUTS) - .firstOrNull { it.type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER } - speaker?.let { track.setPreferredDevice(it) } - // 扬声器模式下始终启用通信模式以获得回声消除 - manager.mode = AudioManager.MODE_IN_COMMUNICATION - manager.isSpeakerphoneOn = true - } - - AudioOutputDevice.HEADPHONES -> { - val headphones = manager.getDevices(AudioManager.GET_DEVICES_OUTPUTS) - .firstOrNull { device -> - device.type == AudioDeviceInfo.TYPE_WIRED_HEADSET || - device.type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES || - device.type == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP || - device.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO - } - headphones?.let { track.setPreferredDevice(it) } - ?: track.setPreferredDevice(null) - // 耳机模式下可以使用普通模式 - manager.mode = AudioManager.MODE_NORMAL - manager.isSpeakerphoneOn = false - } - } - } - } - } - - /** - * 配置音频路由（API 30 及以下） - */ - private fun configureAudioRouting() { - audioManager?.let { manager -> - when (audioOutputDevice) { - AudioOutputDevice.DEFAULT -> { - // 默认模式下使用通信模式以获得更好的音频处理 - manager.mode = AudioManager.MODE_IN_COMMUNICATION - manager.isSpeakerphoneOn = false - } - - AudioOutputDevice.SPEAKER -> { - // 扬声器模式下始终启用通信模式以获得回声消除 - manager.mode = AudioManager.MODE_IN_COMMUNICATION - manager.isSpeakerphoneOn = true - } - - AudioOutputDevice.HEADPHONES -> { - // 耳机模式下可以使用普通模式 - manager.mode = AudioManager.MODE_NORMAL - manager.isSpeakerphoneOn = false - } - } - } - } - - /** - * 开始监听设备变化 - */ - private fun startDeviceMonitoring() { - if (isMonitoringDevices) return - - // API 23+ 使用 AudioDeviceCallback - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - deviceCallback = object : AudioDeviceCallback() { - override fun onAudioDevicesAdded(addedDevices: Array) { - super.onAudioDevicesAdded(addedDevices) - handleDeviceAdded(addedDevices) - } - - override fun onAudioDevicesRemoved(removedDevices: Array) { - super.onAudioDevicesRemoved(removedDevices) - handleDeviceRemoved(removedDevices) - } - } - audioManager?.registerAudioDeviceCallback(deviceCallback, null) + null } - // 所有版本都监听 NOISY 广播（耳机拔出） - noisyReceiver = object : BroadcastReceiver() { - override fun onReceive(context: Context, intent: Intent) { - if (intent.action == AudioManager.ACTION_AUDIO_BECOMING_NOISY) { - handleNoisyAudioEvent() - } - } - } - context.registerReceiver(noisyReceiver, IntentFilter(AudioManager.ACTION_AUDIO_BECOMING_NOISY)) - - isMonitoringDevices = true - } - - /** - * 停止监听设备变化 - */ - private fun stopDeviceMonitoring() { - if (!isMonitoringDevices) return - - // 注销 AudioDeviceCallback - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - deviceCallback?.let { - audioManager?.unregisterAudioDeviceCallback(it) - } - deviceCallback = null - } - - // 注销广播接收器 - noisyReceiver?.let { - try { - context.unregisterReceiver(it) - } catch (e: Exception) { - // 忽略已注销的异常 - } - } - noisyReceiver = null - - isMonitoringDevices = false - } - - /** - * 处理设备添加 - */ - private fun handleDeviceAdded(devices: Array) { - val hasHeadphones = devices.any { device -> - device.type == AudioDeviceInfo.TYPE_WIRED_HEADSET || - device.type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES || - device.type == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP || - device.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO - } - - if (hasHeadphones) { - switchToDefaultMode() - } - } - - /** - * 处理设备移除 - */ - private fun handleDeviceRemoved(devices: Array) { - val hasHeadphones = devices.any { device -> - device.type == AudioDeviceInfo.TYPE_WIRED_HEADSET || - device.type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES || - device.type == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP || - device.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO - } - - if (hasHeadphones) { - switchToSpeakerMode() - } - } - - /** - * 处理音频变得嘈杂事件（通常是耳机拔出） - */ - private fun handleNoisyAudioEvent() { - switchToSpeakerMode() - } - - /** - * 切换到 DEFAULT 模式 - */ - private fun switchToDefaultMode() { - audioOutputDevice = AudioOutputDevice.DEFAULT - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - setPreferredDeviceForTrack() + val preferredDevice = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + audioTrack?.preferredDevice } else { - configureAudioRouting() + null } - } - - /** - * 切换到扬声器模式 - */ - private fun switchToSpeakerMode() { - audioOutputDevice = AudioOutputDevice.SPEAKER - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - setPreferredDeviceForTrack() - } else { - configureAudioRouting() - } + return "路由设备: ${routedDevice?.productName}, 首选设备: ${preferredDevice?.productName}" } } \ No newline at end of file diff --git a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt index b3cd7e75f..bb446a994 100644 --- a/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt +++ b/local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt @@ -117,8 +117,8 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope { private var appId: String = "" private var accessToken: String = "" private var speaker: String = "zh_female_shuangkuaisisi_emo_v2_mars_bigtts" // 默认语音，使用Java示例的音色 - private var format: String = "pcm" // 使用mp3格式 - private var sampleRate: Int = 24000 // 使用24kHz采样率 + private var format: String = "pcm" // pcm 格式 + private var sampleRate: Int = 16000 // 使用16kHz采样率 private var enableTimestamp: Boolean = false private var language: String = "zh-CN" From 9e0017b268f285ecc8432dd3bb34e1cd26253f56 Mon Sep 17 00:00:00 2001 From: liwei1dao Date: Mon, 23 Jun 2025 10:55:21 +0800 Subject: [PATCH 09/10] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E8=AE=BE=E5=A4=87?= =?UTF-8?q?=E4=BF=9D=E5=AE=9A=E6=9C=BA=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/data/models/user_Info.dart | 28 ++-- lib/data/services/network/api.dart | 9 ++ .../controllers/pairing_controller.dart | 129 +++++++++++++----- lib/modules/pairing/views/pairing_view.dart | 51 ++++++- .../pairing/views/pairing_view_new.dart | 73 +++++++--- lib/modules/settings/views/settings_view.dart | 3 +- 6 files changed, 223 insertions(+), 70 deletions(-) diff --git a/lib/data/models/user_Info.dart b/lib/data/models/user_Info.dart index cf5853ee6..63f6d60be 100644 --- a/lib/data/models/user_Info.dart +++ b/lib/data/models/user_Info.dart @@ -14,14 +14,15 @@ class User { String name; String uid; String avatar; - String deviceid; String language; String token; String mail; String phone; String phonemac; String password; - int membershipLevel = 0; //添加会员等级字段,0表示普通用户 + int viplv = 0; //添加会员等级字段,0表示普通用户 + String earphonemac; + String earphoneauthcode; // final Int64 lastbettime; // final Int64 createtime; // final String wxopenid; @@ -33,14 +34,15 @@ class User { required this.name, required this.uid, required this.avatar, - required this.deviceid, required this.language, required this.token, required this.mail, required this.phone, required this.phonemac, required this.password, - this.membershipLevel = 0, // 添加默认值 + required this.viplv, // 添加默认值 + required this.earphonemac, + required this.earphoneauthcode, // required this.lastbettime, // required this.createtime, // required this.wxopenid, @@ -60,14 +62,16 @@ class User { name: (userData?['name'] as String?) ?? '默认名称', // 提供默认值 uid: (userData?['uid'] as String?) ?? '', avatar: (userData?['avatar'] as String?) ?? '', - deviceid: (userData?['deviceid'] as String?) ?? '', + language: (userData?['language'] as String?) ?? 'en', mail: (userData?['mail'] as String?) ?? '', phone: (userData?['phone'] as String?) ?? '', phonemac: (userData?['phonemac'] as String?) ?? '', password: (userData?['password'] as String?) ?? '', - membershipLevel: (userData?['membershipLevel'] as int?) ?? 0, // 提供默认值 + viplv: (userData?['viplv'] as int?) ?? 0, // 提供默认值 + earphonemac: (userData?['earphonemac'] as String?) ?? '', + earphoneauthcode: (userData?['earphoneauthcode'] as String?) ?? '', // lastbettime: _parseTimestamp(userData['lastbettime']), // createtime: _parseTimestamp(userData['createtime']), // wxopenid: (userData?['wxopenid'] as String?) ?? '', @@ -85,13 +89,16 @@ class User { name: (userData?['name'] as String?) ?? '默认名称', // 提供默认值 uid: (userData?['uid'] as String?) ?? '', avatar: (userData?['avatar'] as String?) ?? '', - deviceid: (userData?['deviceid'] as String?) ?? '', language: (userData?['language'] as String?) ?? 'en', mail: (userData?['mail'] as String?) ?? '', phone: (userData?['phone'] as String?) ?? '', phonemac: (userData?['phonemac'] as String?) ?? '', password: (userData?['password'] as String?) ?? '', + + viplv: (userData?['viplv'] as int?) ?? 0, // 提供默认值 + earphonemac: (userData?['earphonemac'] as String?) ?? '', + earphoneauthcode: (userData?['earphoneauthcode'] as String?) ?? '', // lastbettime: _parseTimestamp(userData['lastbettime']), // createtime: _parseTimestamp(userData['createtime']), // wxopenid: (userData?['wxopenid'] as String?) ?? '', @@ -106,14 +113,17 @@ class User { 'name': name, 'uid': uid, 'avatar': avatar, - 'deviceid': deviceid, + 'language': language, 'token': token, 'mail': mail, 'phone': phone, 'phonemac': phonemac, 'password': password, - 'membershipLevel': membershipLevel, + 'viplv': viplv, + 'earphonemac': earphonemac, + 'earphoneauthcode': earphoneauthcode, + // 'lastbettime': lastbettime, // 'createtime': createtime, // 'wxopenid': wxopenid, diff --git a/lib/data/services/network/api.dart b/lib/data/services/network/api.dart index 07e265b4e..1eb3f41a0 100644 --- a/lib/data/services/network/api.dart +++ b/lib/data/services/network/api.dart @@ -126,4 +126,13 @@ class Api { params: params, ); } + + //绑定设备 + static bindearphone([params]) { + return DioManager().request( + NWMethod.post, + '/api/home/user_bindearphone', + params: params, + ); + } } diff --git a/lib/modules/pairing/controllers/pairing_controller.dart b/lib/modules/pairing/controllers/pairing_controller.dart index ed62d4ae9..c6fd22ae9 100644 --- a/lib/modules/pairing/controllers/pairing_controller.dart +++ b/lib/modules/pairing/controllers/pairing_controller.dart @@ -1,4 +1,7 @@ import 'package:app_settings/app_settings.dart'; +import 'package:deep_voice/data/models/user_Info.dart'; +import 'package:deep_voice/data/services/network/api.dart'; +import 'package:flutter/material.dart'; import 'package:get/get.dart'; import 'dart:async'; import 'package:permission_handler/permission_handler.dart'; @@ -15,9 +18,10 @@ enum PairingState { checking, // 检查中（蓝牙、权限等） waitingClassic, // 等待经典蓝牙连接 verifying, // 验证设备 + manualbinding, //手动绑定设备 associating, // 正在进行BLE设备关联 - waitingBle, // 等待BLE连接 - completed, // 配对完成 + waitingBle, //等待BLE连接 + completed, //配对完成 error, // 错误状态 } @@ -31,6 +35,7 @@ enum ErrorType { deviceBound, // 设备已被绑定 associationFailed, // 关联失败 bleTimeout, // BLE连接超时 + authnFailed, // 授权失败 networkError, // 网络错误 } @@ -53,6 +58,7 @@ class PairingController extends GetxController { final isLoading = false.obs; final showConnectButton = false.obs; final showRetryButton = false.obs; + final showAutoCodeInput = false.obs; final showSkipButton = false.obs; final connectButtonText = '连接设备'.obs; final retryButtonText = '重试'.obs; @@ -68,6 +74,7 @@ class PairingController extends GetxController { // ========== 当前设备信息 ========== BluetoothDevice? _currentClassicDevice; String? _verifiedDeviceAddress; + TextEditingController codeInputController = TextEditingController(); // Public getter for view access BluetoothDevice? get currentDevice => _currentClassicDevice; @@ -236,6 +243,7 @@ class PairingController extends GetxController { showConnectButton.value = false; showRetryButton.value = false; showSkipButton.value = false; + showAutoCodeInput.value = false; switch (state) { case PairingState.idle: @@ -276,7 +284,12 @@ class PairingController extends GetxController { _startBleConnection(); }); break; - + case PairingState.manualbinding: + statusText.value = '需要手动版定授权码...'; + isLoading.value = false; + showAutoCodeInput.value = true; + showSkipButton.value = true; + break; case PairingState.completed: statusText.value = '配对成功！'; isLoading.value = false; @@ -306,6 +319,10 @@ class PairingController extends GetxController { showRetryButton.value = true; retryButtonText.value = '重新关联'; break; + case ErrorType.authnFailed: + showRetryButton.value = true; + retryButtonText.value = '重新授权'; + break; default: showRetryButton.value = true; retryButtonText.value = '重试'; @@ -393,44 +410,48 @@ class PairingController extends GetxController { Future _verifyDevice(BluetoothDevice device) async { try { Logger.i(_tag, '开始验证设备: ${device.name} (${device.address})'); - // TODO: 实现API验证逻辑 // 暂时模拟验证通过 - await Future.delayed(const Duration(seconds: 2)); - - // 检查设备名称是否包含 DeepSound 或 DeapSound - final deviceName = device.name ?? ''; - if (deviceName.contains('DeepSound') || - deviceName.contains('DeapSound')) { - Logger.i(_tag, '设备验证通过'); - _verifiedDeviceAddress = device.address; - - // 检查是否已有BLE关联 - final pairedMacAddress = await _bleManager.getPairedMacAddress(); - Logger.i(_tag, '已关联的MAC地址: $pairedMacAddress'); - - if (pairedMacAddress != null && - pairedMacAddress.toUpperCase() == device.address.toUpperCase()) { - // 已有关联，检查BLE连接状态 - Logger.i(_tag, '设备已关联，检查BLE连接状态'); - final currentBleState = await _bleManager.getConnectionState(); - Logger.i(_tag, '验证时BLE状态: $currentBleState'); - - if (currentBleState == 2) { - // BleConnectionState.connected - Logger.i(_tag, 'BLE已连接，直接完成配对'); - _transition(PairingState.completed); + // await Future.delayed(const Duration(seconds: 2));; + if (await authbindingDevice()) { + // 检查设备名称是否包含 DeepSound 或 DeapSound + final deviceName = device.name ?? ''; + if (deviceName.contains('DeepSound') || + deviceName.contains('DeapSound')) { + Logger.i(_tag, '设备验证通过'); + _verifiedDeviceAddress = device.address; + + // 检查是否已有BLE关联 + final pairedMacAddress = await _bleManager.getPairedMacAddress(); + Logger.i(_tag, '已关联的MAC地址: $pairedMacAddress'); + + if (pairedMacAddress != null && + pairedMacAddress.toUpperCase() == device.address.toUpperCase()) { + // 已有关联，检查BLE连接状态 + Logger.i(_tag, '设备已关联，检查BLE连接状态'); + final currentBleState = await _bleManager.getConnectionState(); + Logger.i(_tag, '验证时BLE状态: $currentBleState'); + + if (currentBleState == 2) { + // BleConnectionState.connected + Logger.i(_tag, 'BLE已连接，直接完成配对'); + _transition(PairingState.completed); + } else { + Logger.i(_tag, 'BLE未连接，等待BLE连接'); + _transition(PairingState.waitingBle); + } } else { - Logger.i(_tag, 'BLE未连接，等待BLE连接'); - _transition(PairingState.waitingBle); + // 没有关联或关联的是其他设备，需要进行关联 + Logger.i(_tag, '需要进行BLE设备关联'); + _transition(PairingState.associating); } } else { - // 没有关联或关联的是其他设备，需要进行关联 - Logger.i(_tag, '需要进行BLE设备关联'); - _transition(PairingState.associating); + // _setError(ErrorType.deviceNotMatch, '请连接 DeepSound 设备'); + _transition(PairingState.completed); } } else { - _setError(ErrorType.deviceNotMatch, '请连接 DeepSound 设备'); + //验证失败 进入手动授权 + _transition(PairingState.manualbinding); } } catch (e) { Logger.e(_tag, '设备验证异常: $e'); @@ -504,6 +525,46 @@ class PairingController extends GetxController { } } + //自动绑定设备 + Future authbindingDevice() async { + if (User.instance.earphonemac == currentDevice?.address) { + // 已经绑定 + return true; + } + + // 调用BLE管理器的自动绑定方法 + final response = await Api.bindearphone({ + "earphonemac": currentDevice?.address, + "earphoneauthcode": "", + }); + Logger.i(_tag, '手动授权结果 ${response}'); + if (response != null) { + Logger.i(_tag, '设备自动绑定成功'); + User.instance.earphonemac = currentDevice?.address ?? ''; + return true; + } else { + Logger.e(_tag, '手动绑定失败!'); + return false; + } + } + + /// 用户点击授权码输入框 + void onAutoCodeInputTap() async { + // 调用BLE管理器的自动绑定方法 + final response = await Api.bindearphone({ + "earphonemac": currentDevice?.address, + "earphoneauthcode": codeInputController.text.trim(), + }); + Logger.i(_tag, '手动授权结果 ${response}'); + if (response != null) { + _transition(PairingState.verifying); + _verifyDevice(_currentClassicDevice!); + } else { + Logger.e(_tag, '手动绑定失败!'); + _setError(ErrorType.authnFailed, '设备授权失败,请检查授权码'); + } + } + /// 导航到主页 void _navigateToHome() async { await BleManager.to.startAgentService(); diff --git a/lib/modules/pairing/views/pairing_view.dart b/lib/modules/pairing/views/pairing_view.dart index b0cf500c3..542e22ec9 100644 --- a/lib/modules/pairing/views/pairing_view.dart +++ b/lib/modules/pairing/views/pairing_view.dart @@ -336,6 +336,8 @@ class PairingView extends GetView { return '扫描超时，请确保设备在附近'; case ErrorType.networkError: return '网络错误，请检查网络连接'; + case ErrorType.authnFailed: + return '授权失败,请检查授权码是否正确'; default: return '配对失败，请重试'; } @@ -433,7 +435,7 @@ class PairingView extends GetView { final showConnectButton = controller.showConnectButton.value; final showRetryButton = controller.showRetryButton.value; final showSkipButton = controller.showSkipButton.value; - + final showAutoCodeInput = controller.showAutoCodeInput.value; return Container( width: double.infinity, child: Column( @@ -490,6 +492,51 @@ class PairingView extends GetView { ), ), ), + if (showAutoCodeInput) + Column( + children: [ + // 输入框 + SizedBox( + width: double.infinity, + height: 48.h, + child: TextField( + controller: controller + .codeInputController, // 你可以在 Controller 中声明这个 + decoration: InputDecoration( + hintText: '请输入代码', + border: OutlineInputBorder( + borderRadius: BorderRadius.circular(12.r), + ), + contentPadding: EdgeInsets.symmetric(horizontal: 16.w), + ), + ), + ), + SizedBox(height: 12.h), + SizedBox( + width: double.infinity, + height: 48.h, + child: ElevatedButton( + onPressed: controller.onAutoCodeInputTap, + style: ElevatedButton.styleFrom( + backgroundColor: Theme.of(Get.context!).primaryColor, + shape: RoundedRectangleBorder( + borderRadius: BorderRadius.circular(12.r), + ), + elevation: 0, + ), + child: Text( + '授权绑定', + style: TextStyle( + fontSize: 16.sp, + fontWeight: FontWeight.w600, + color: Colors.white, + ), + ), + ), + ), + ], + ), + // 自动输入代码按钮 // 跳过按钮 if (showSkipButton) ...[ @@ -510,11 +557,9 @@ class PairingView extends GetView { ), ), ], - ], ), ); }); } - } diff --git a/lib/modules/pairing/views/pairing_view_new.dart b/lib/modules/pairing/views/pairing_view_new.dart index eae654a1b..d4b843744 100644 --- a/lib/modules/pairing/views/pairing_view_new.dart +++ b/lib/modules/pairing/views/pairing_view_new.dart @@ -21,17 +21,17 @@ class PairingView extends GetView { children: [ // 顶部标题 _buildHeader(isDarkMode), - + // 中间内容 Expanded( child: Center( child: Obx(() => _buildContent(isDarkMode)), ), ), - + // 底部按钮 Obx(() => _buildButtons(isDarkMode)), - + SizedBox(height: 20.h), ], ), @@ -69,15 +69,15 @@ class PairingView extends GetView { Widget _buildContent(bool isDarkMode) { final state = controller.currentState; final isLoading = controller.isLoading.value; - + return Column( mainAxisSize: MainAxisSize.min, children: [ // 状态图标 _buildStateIcon(state, isLoading, isDarkMode), - + SizedBox(height: 40.h), - + // 状态文本 Text( controller.statusText.value, @@ -87,7 +87,7 @@ class PairingView extends GetView { ), textAlign: TextAlign.center, ), - + // 错误详情 if (state == PairingState.error) ...[ SizedBox(height: 16.h), @@ -132,48 +132,51 @@ class PairingView extends GetView { ), ); } - + IconData iconData; Color iconColor; - + switch (state) { case PairingState.idle: case PairingState.checking: iconData = Icons.bluetooth_searching; iconColor = Colors.blue; break; - + case PairingState.waitingClassic: iconData = Icons.bluetooth; iconColor = Colors.grey; break; - + case PairingState.verifying: iconData = Icons.verified_user; iconColor = Colors.orange; break; - + case PairingState.associating: iconData = Icons.link; iconColor = Colors.blue; break; - + case PairingState.waitingBle: iconData = Icons.sync; iconColor = Colors.blue; break; - + case PairingState.manualbinding: + iconData = Icons.blind; + iconColor = Colors.blue; + break; case PairingState.completed: iconData = Icons.check_circle; iconColor = Colors.green; break; - + case PairingState.error: iconData = Icons.error; iconColor = Colors.red; break; } - + return Container( width: 100.w, height: 100.w, @@ -193,8 +196,9 @@ class PairingView extends GetView { Widget _buildButtons(bool isDarkMode) { final showConnectButton = controller.showConnectButton.value; final showRetryButton = controller.showRetryButton.value; + final showAutoCodeInput = controller.showAutoCodeInput.value; final showSkipButton = controller.showSkipButton.value; - + return Column( children: [ // 主要操作按钮 @@ -204,16 +208,40 @@ class PairingView extends GetView { controller.onConnectTap, isDarkMode, ), - + if (showRetryButton) _buildPrimaryButton( '重试', controller.onRetryTap, isDarkMode, ), - + SizedBox(height: 12.h), - + if (showAutoCodeInput) + // 自动输入代码按钮 + SizedBox( + width: double.infinity, + height: 48.h, + child: ElevatedButton( + onPressed: controller.onAutoCodeInputTap, + style: ElevatedButton.styleFrom( + backgroundColor: Theme.of(Get.context!).primaryColor, + shape: RoundedRectangleBorder( + borderRadius: BorderRadius.circular(12.r), + ), + elevation: 0, + ), + child: Text( + '自动输入代码', + style: TextStyle( + fontSize: 16.sp, + fontWeight: FontWeight.w600, + color: Colors.white, + ), + ), + ), + ), + // 跳过按钮 if (showSkipButton) TextButton( @@ -231,7 +259,8 @@ class PairingView extends GetView { } /// 构建主要按钮 - Widget _buildPrimaryButton(String text, VoidCallback onPressed, bool isDarkMode) { + Widget _buildPrimaryButton( + String text, VoidCallback onPressed, bool isDarkMode) { return SizedBox( width: double.infinity, height: 48.h, @@ -255,4 +284,4 @@ class PairingView extends GetView { ), ); } -} \ No newline at end of file +} diff --git a/lib/modules/settings/views/settings_view.dart b/lib/modules/settings/views/settings_view.dart index 35bb6c90b..4f16e2b33 100644 --- a/lib/modules/settings/views/settings_view.dart +++ b/lib/modules/settings/views/settings_view.dart @@ -200,8 +200,7 @@ class SettingsView extends GetView { 2: 'proUser', //'专业用户' }; final membershipString = membershipMap[ - controller - .userInfo.value.membershipLevel] ?? + controller.userInfo.value.viplv] ?? 'Unknown'; return Text( membershipString, From 759c2d3ed6174619cc44188e56e81e6686923aa0 Mon Sep 17 00:00:00 2001 From: fdp <1286779656@qq.com> Date: Mon, 23 Jun 2025 11:24:55 +0800 Subject: [PATCH 10/10] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=BC=BA=E5=88=B6?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E6=89=AC=E5=A3=B0=E5=99=A8=E4=B8=8D=E7=94=9F?= =?UTF-8?q?=E6=95=88=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../speech_impl/azure_tts_service.dart | 14 ++++++- .../speech_impl/voice_clone_tts_service.dart | 8 +++- lib/data/services/tts_service.dart | 39 +++++++++---------- .../azure_speech/AzureSpeechPlugin.kt | 6 ++- .../azure_speech/AzureTtsHelper.kt | 6 ++- 5 files changed, 49 insertions(+), 24 deletions(-) diff --git a/lib/data/services/speech_impl/azure_tts_service.dart b/lib/data/services/speech_impl/azure_tts_service.dart index d6eccdf9b..8b2512260 100644 --- a/lib/data/services/speech_impl/azure_tts_service.dart +++ b/lib/data/services/speech_impl/azure_tts_service.dart @@ -413,4 +413,16 @@ class AzureTtsService extends GetxService implements TtsService { Logger.error('释放资源失败: ${e.toString()}'); } } -} \ No newline at end of file + + @override + Future setAudioOutputDevice() async { + try { + // 开始合成，传递参数 + final result = await _channel.invokeMethod('setAudioOutputDevice'); + return result; + } catch (e) { + Logger.error('音频输出设备类型失败: ${e.toString()}'); + return false; + } + } +} diff --git a/lib/data/services/speech_impl/voice_clone_tts_service.dart b/lib/data/services/speech_impl/voice_clone_tts_service.dart index 43079325f..656fe4bcb 100644 --- a/lib/data/services/speech_impl/voice_clone_tts_service.dart +++ b/lib/data/services/speech_impl/voice_clone_tts_service.dart @@ -390,4 +390,10 @@ class VoiceCloneTtsService extends GetxService implements TtsService { Logger.error('释放资源失败: ${e.toString()}'); } } -} \ No newline at end of file + + @override + Future setAudioOutputDevice() { + // TODO: implement setAudioOutputDevice + throw UnimplementedError(); + } +} diff --git a/lib/data/services/tts_service.dart b/lib/data/services/tts_service.dart index 62d938b56..4e2ec3ef4 100644 --- a/lib/data/services/tts_service.dart +++ b/lib/data/services/tts_service.dart @@ -2,52 +2,51 @@ import 'dart:async'; /// 语音合成服务接口 abstract class TtsService { - /// 当前选择的语音 String get currentVoice; - + /// 当前是否正在播放 bool get isSpeaking; - + /// TTS事件流，用于监听语音合成事件 Stream get onEvent; - + /// 初始化语音合成服务 Future initialize({ List? supportedLanguages, }); - + /// 设置语音 Future setVoice(String voiceName); - - + /// 单次播放文本，完成后返回 Future speakOnce(String text); - + /// 将文本加入播放队列，支持连续播放多段文本 Future speakStream(String text); - + /// 播放队列中剩余文本 Future flushStream(); - + /// 停止播放 Future stop(); - + /// 释放资源 Future dispose(); + Future setAudioOutputDevice(); } /// TTS事件类型 enum TtsEventType { /// 开始播放 started, - + /// 播放完成 completed, - + /// 播放取消 canceled, - + /// 播放错误 error, } @@ -56,15 +55,15 @@ enum TtsEventType { class TtsEvent { /// 事件类型 final TtsEventType type; - + /// 错误信息（仅在 error 类型中有效） final String error; - + TtsEvent({ required this.type, this.error = '', }); - + /// 创建错误事件的快捷构造函数 factory TtsEvent.error(String errorMessage) { return TtsEvent( @@ -72,12 +71,12 @@ class TtsEvent { error: errorMessage, ); } - + /// 检查是否为错误 bool get isError => type == TtsEventType.error; - + @override String toString() { return 'TtsEvent{type: $type, error: $error}'; } -} \ No newline at end of file +} diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt index 54e960092..498613b8c 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt @@ -15,7 +15,7 @@ import com.deep_voice.speech.tts.TtsEvent import com.deep_voice.speech.tts.TtsEventListener import com.deep_voice.speech.tts.TtsEventType import com.yunqiinnovation.ble_service.BleService - +import com.deep_voice.speech.tts.AudioOutputDevice /** AzureSpeechPlugin */ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin { private val tag = "AzureSpeechPlugin" @@ -501,6 +501,10 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin { azureTtsHelper.release() result.success(true) } + "setAudioOutputDevice" -> { + azureTtsHelper.setAudioOutputDevice(AudioOutputDevice.SPEAKER) + result.success(true) + } else -> { result.notImplemented() diff --git a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt index ea9d42588..d5444ffb2 100644 --- a/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt +++ b/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt @@ -223,7 +223,11 @@ class AzureTtsHelper(private val context: Context) : ITtsService { } AudioOutputDevice.SPEAKER -> { // 强制使用扬声器 - manager.mode = AudioManager.MODE_NORMAL + /* MODE_NORMAL 默认媒体模式 ❌ 不能强制扬声器（会被耳机覆盖） + MODE_IN_COMMUNICATION VoIP 通信模式 ✅ 可以强制使用扬声器或听筒 + MODE_IN_CALL 电话模式（仅限系统） ✅ 更强但风险大，不推荐使用 + */ + manager.mode = AudioManager.MODE_IN_COMMUNICATION manager.isSpeakerphoneOn = true } AudioOutputDevice.HEADPHONES -> {