Browse Source

Merge branch 'new_dev' of https://github.com/deepcloud2048/deep_voice into new_dev

weicu
lxm 1 year ago
parent
commit
a49fe83cd9
  1. 94
      .cursorrules
  2. 1
      .gitignore
  3. 15
      android/app/proguard-logging.pro
  4. 5
      android/app/src/main/kotlin/com/yunqiinnovation/deep_voice/MainActivity.kt
  5. 60
      android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt
  6. 5
      android/settings.gradle.kts
  7. 160
      lib/modules/FTFTranslation/README.md
  8. 10
      lib/modules/FTFTranslation/bindings/FTFTranslation_binding.dart
  9. 241
      lib/modules/FTFTranslation/controllers/FTFTranslation_controller.dart
  10. 212
      lib/modules/FTFTranslation/views/FTFTranslation_view.dart
  11. 2
      lib/modules/agent/controllers/agent_controller.dart
  12. 7
      lib/modules/meeting/controllers/meeting_controller.dart
  13. 5
      lib/modules/meeting/controllers/meeting_record_controller.dart
  14. 88
      lib/modules/meeting/views/meeting_view.dart
  15. 54
      lib/modules/meeting/views/share_bottom_sheet.dart
  16. 23
      lib/modules/pairing/controllers/pairing_controller.dart
  17. 22
      lib/modules/settings/views/settings_view.dart
  18. 8
      lib/modules/translation/controllers/translation_controller.dart
  19. 4
      lib/modules/translation/views/translation_view.dart
  20. 9
      lib/routes/app_pages.dart
  21. 1
      lib/routes/app_routes.dart
  22. 1
      local_plugins/agent_service/android/build.gradle.kts
  23. 80
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  24. 8
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt
  25. 78
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt
  26. 2
      local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCompanionManager.kt
  27. 129
      local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt
  28. 420
      local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt
  29. 6
      local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceSpeechPlugin.kt
  30. 197
      local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt
  31. 12
      local_plugins/bytedance_speech/lib/bytedance_speech.dart
  32. 257
      local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt
  33. 3
      local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/CustomSseClientTransport.kt
  34. 28
      local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/MCPSubClient.kt
  35. 789
      local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/SystemFunctionHandler.kt
  36. 2
      local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt
  37. 15
      local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/ITtsService.kt
  38. 16
      local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/TtsEvents.kt
  39. 2
      pubspec.yaml

94
.cursorrules

@ -1,94 +0,0 @@
# 代码简明规范
## Dart / Flutter
1. 风格统一
• 严格遵循 Effective Dart 格式与命名惯例。
2. 日志与调试
• 禁止 print;仅在异常场景调用 logger.e(...)。
3. 业务逻辑
• 默认使用 GetX(状态、路由、依赖注入等),避免手写 StatefulWidget。
4. 异常处理
• 不写重试逻辑,失败即返回错误。
• 谨慎使用 try-catch;优先通过类型系统与空安全防御。
5. 代码演进
• 任何改动都应 “小步、安全”,杜绝一次性大改或过度设计。
## 原生层(Android / iOS)
1. 配置不可改
• 禁止修改 SDK、NDK、JDK 与 iOS 构建配置。
2. Kotlin DSL
• 原生实现一律采用 Kotlin + 协程,遵守 Kotlin 最佳实践。
• 不写 Java 代码。
## 通用要求
1. 不自动执行
• 生成脚本/代码不得包含 flutter run 等执行指令。
2. 并发优先级
• 并发场景首选 Kotlin 协程;仅在 Flutter 端按需使用 Dart Future / Stream。
3. 语言
• 与用户交流、注释及文档一律使用中文。
• 不要自己创建总结文档。
宗旨:保持极简,先跑通业务,再考虑优化。
# 项目架构概述
DeepSound是一个基于Flutter和GetX构建的多平台语音应用,具有语音合成、语音识别、语音聊天等功能。
## 总体架构
- 采用GetX状态管理框架
- 遵循模块化设计,各功能模块相对独立
- 使用依赖注入实现组件解耦
- 多语言支持,使用GetX国际化方案
## 主要目录结构
- lib/core: 核心组件、工具和通用功能
- lib/data: 数据模型、服务和提供者
- lib/modules: 功能模块,按业务划分
- lib/routes: 路由定义和管理
- local_plugins: 本地插件,提供原生功能支持
# 核心模块说明
## 语音处理
- 火山语音服务: 提供语音识别和合成核心能力
- Azure语音服务: 提供备选的语音识别和合成能力
- 离线TTS支持: 在线语音合成失败时的备选方案
## 聊天功能
- 基于OpenAI的对话功能
- 支持语音输入、文字输入和语音输出
- 支持多种角色和语音类型
## 用户系统
- 支持多种登录方式:Firebase、Google、Facebook
- 用户资料管理和设置
- 主题切换和语言设置
## 蓝牙配对
- 支持蓝牙设备发现和连接
- 提供简化的配对流程
- 设备管理和状态监控
## 音频处理
- 音频录制和播放
- 支持波形显示
- 音量控制和音频质量设置
# 技术栈
- Flutter: UI框架
- GetX: 状态管理和依赖注入
- Firebase: 用户认证和后端服务
- WebSocket: 实时通信
- SQLite: 本地数据存储
- 原生插件: 语音识别、合成和蓝牙功能
# 插件说明
- jieli: 杰理蓝牙设备连接和控制插件
- volcano_speech: 火山语音服务插件,提供语音识别和合成
- azure_speech: 微软Azure语音服务插件
- open_ai_service: OpenAI服务插件,提供AI对话能力
- speech_to_text: 通用语音识别插件
# 环境配置
- 环境变量通过.env文件管理
- 支持不同环境的配置切换
- 敏感信息不应直接硬编码在代码中

1
.gitignore

@ -14,6 +14,7 @@ doc/api/
# dotenv environment variables file
.env*
.env.example
CLAUDE.md
# Avoid committing generated Javascript files:
*.dart.js

15
android/app/proguard-logging.pro

@ -0,0 +1,15 @@
# 移除系统组件的日志调用
-assumenosideeffects class android.util.Log {
public static int v(...);
public static int d(...);
public static int i(...);
}
# 针对特定类的日志移除
-assumenosideeffects class android.media.AudioTrack {
private static void log*(...);
}
-assumenosideeffects class android.app.AppOpsManager {
private static void log*(...);
}

5
android/app/src/main/kotlin/com/yunqiinnovation/deep_voice/MainActivity.kt

@ -1,5 +0,0 @@
package com.yunqiinnovation.deep_voice
import io.flutter.embedding.android.FlutterActivity
class MainActivity : FlutterActivity()

60
android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt

@ -39,6 +39,7 @@ class MainActivity: FlutterActivity() {
* 抑制系统级组件的日志输出
*/
private fun suppressSystemLogs() {
// 更全面的系统标签列表
val systemTags = arrayOf(
"MediaCodec",
"CCodec",
@ -49,11 +50,66 @@ class MainActivity: FlutterActivity() {
"AudioTrack",
"AudioManager",
"AudioFlinger",
"AudioService"
"AudioService",
"AudioEffect",
"AppOpsManager",
"AudioRecord",
"OpenGLRenderer",
"Choreographer",
"InputMethodManager",
"ViewRootImpl",
"Surface",
"BufferQueueProducer",
"ActivityManager",
"WindowManager",
"SensorManager"
)
// 尝试多种方式设置日志级别
systemTags.forEach { tag ->
System.setProperty("log.tag.$tag", "WARN")
try {
// 方式1:标准属性设置
System.setProperty("log.tag.$tag", "ERROR")
// 方式2:尝试使用反射设置(针对某些特殊标签)
setLogLevelViaReflection(tag, Log.ERROR)
} catch (e: Exception) {
// 忽略设置失败的情况
}
}
// 方式3:尝试设置全局日志级别
try {
// 设置更高的阈值,减少verbose和debug日志
System.setProperty("log.tag", "WARN")
} catch (e: Exception) {
// 忽略
}
}
/**
* 通过反射尝试设置日志级别
*/
private fun setLogLevelViaReflection(tag: String, level: Int) {
try {
val logClass = Class.forName("android.util.Log")
val isLoggableMethod = logClass.getDeclaredMethod(
"isLoggable",
String::class.java,
Int::class.javaPrimitiveType
)
isLoggableMethod.isAccessible = true
// 某些系统可能支持直接设置
val setLogLevelMethod = logClass.getDeclaredMethod(
"setLogLevel",
String::class.java,
Int::class.javaPrimitiveType
)
setLogLevelMethod.isAccessible = true
setLogLevelMethod.invoke(null, tag, level)
} catch (e: Exception) {
// 反射方法可能不存在,忽略
}
}
}

5
android/settings.gradle.kts

@ -40,6 +40,8 @@ include(":classic_bluetooth")
include(":agent_service")
include(":speech")
include(":chat_api")
include(":bytedance_speech")
include(":jl_opus")
// 设置azure_speech项目的路径
project(":azure_speech").projectDir = file("../local_plugins/azure_speech/android")
@ -51,4 +53,5 @@ project(":classic_bluetooth").projectDir = file("../local_plugins/classic_blueto
project(":agent_service").projectDir = file("../local_plugins/agent_service/android")
project(":speech").projectDir = file("../local_plugins/speech/android")
project(":chat_api").projectDir = file("../local_plugins/chat_api/android")
project(":jl_opus").projectDir = file("../local_plugins/jl_opus/android")
project(":bytedance_speech").projectDir = file("../local_plugins/bytedance_speech/android")
project(":jl_opus").projectDir = file("../local_plugins/jl_opus/android")

160
lib/modules/FTFTranslation/README.md

@ -0,0 +1,160 @@
# Realtime 实时语音聊天模块
## ⚠️ 当前状态
**此模块为功能预览版本,提供完整的实时语音对话功能。**
通过原生插件实现语音识别、AI对话和语音合成的完整语音交互体验。
## 概述
Realtime模块实现了类似ChatGPT app的实时语音对话功能,当前版本提供:
- ✅ **完整语音交互** - 实时语音识别、AI对话和语音合成
- ✅ **状态管理** - 聆听、思考、回答状态的可视化
- ✅ **动画效果** - 基于Shader的流动云效果,支持音频振幅驱动
- ✅ **字幕显示** - 可选择开启/关闭的字幕模式
- ✅ **智能UI** - 根据连接状态自动调整按钮可用性
## 界面设计
### 布局结构
- **顶部状态区域** - 显示当前状态和字幕模式切换按钮
- **中间可视化区域** - 圆形Shader动画,根据音频振幅呈现呼吸效果
- **底部控制区域** - 主控制按钮和状态指示器
### 状态指示
- 🔵 **聆听状态** - 蓝色动画,接收用户语音输入
- 🟠 **思考状态** - 橙色动画,AI处理中
- 🟢 **回答状态** - 绿色动画,AI语音输出
- ⚪ **空闲状态** - 灰色静态,等待交互
### 字幕模式
- **默认关闭** - 初始状态下不显示任何字幕内容
- **手动切换** - 通过右上角按钮手动开启/关闭字幕
- **完整隐藏** - 关闭时字幕区域完全不显示,不占用界面空间
- **智能布局** - 开启时圆形动画自动上移,字幕区域占据下半屏
### 交互控制
- **录音按钮** - 仅在成功连接AI服务器且开始录音后可用
- **连接状态** - 未连接时录音按钮显示不可用状态
- **状态反馈** - 按钮颜色和可点击性根据连接状态动态调整
## 使用方法
### 路由导航
```dart
// 跳转到实时语音对话页面
Get.toNamed(Routes.realtime);
```
### 交互流程
1. **自动连接** - 页面加载后自动尝试连接AI服务器
2. **等待连接** - 录音按钮在连接成功前保持不可用状态
3. **开始录音** - 连接成功后自动开始录音,录音按钮变为可用
4. **语音对话** - 用户说话 → AI处理 → 语音回复
5. **字幕控制** - 可随时通过右上角按钮切换字幕显示
## 技术架构
### 控制器 (RealtimeController)
- **连接管理**: WebSocket连接到AI服务器
- **状态同步**: `isConnected`, `isListening`, `isSpeaking`
- **字幕控制**: `isSubtitleMode` 默认为false,用户手动控制
- **消息管理**: 智能去重,避免流式响应重复显示
- **音频处理**: 原生层计算RMS值,Flutter层接收并控制动画
### 视图组件
- **RealtimeView** - 主视图容器,响应式布局
- **ShaderMicFlow** - 基于GLSL Shader的流动云动画组件
- **AutoScrollList** - 自动滚动的字幕列表
### 原生集成
- **RealtimeService** - 原生音频处理和WebSocket通信
- **音频格式** - 16kHz, 1声道, 16位PCM
- **实时传输** - 低延迟音频流处理
- **RMS计算** - 原生层实时计算音频振幅
## 默认行为
### 字幕模式
- **初始状态**: 关闭 (`isSubtitleMode.value = false`)
- **显示逻辑**: 仅在用户手动开启时显示字幕内容
- **空间占用**: 关闭时字幕区域高度为0,不影响其他UI元素
- **切换效果**: 平滑动画过渡,圆形动画位置和大小同步调整
### 按钮状态
- **录音按钮**: 默认不可用,连接成功且开始录音后才可点击
- **状态指示**: 通过颜色和图标变化反映当前可用性
- **交互反馈**: 不可用时点击无效果,避免误操作
## 开发计划
### 已完成功能 ✅
- [x] 完整语音交互流程
- [x] 实时WebSocket通信
- [x] Shader流动云动画
- [x] RMS音频振幅计算
- [x] 智能UI状态管理
- [x] 字幕模式控制
- [x] 消息去重优化
### 计划增强 ⏳
- [ ] 多语言语音识别
- [ ] 语音中断检测
- [ ] 音频质量自适应
- [ ] 离线降级处理
- [ ] 语音情感识别
## 注意事项
1. **网络依赖** - 需要连接到AI服务器 (ws://192.168.1.11:8000/ws)
2. **麦克风权限** - 需要获取麦克风使用权限
3. **音频会话** - 自动管理音频会话,支持蓝牙设备
4. **资源管理** - 页面关闭时自动清理音频资源和网络连接
## 自定义配置
### 修改服务器地址
```dart
// 在RealtimeController中修改连接URL
await _realtimeService.initialize(
serverUrl: 'ws://your-server:port/ws',
// ... 其他参数
);
```
### 调整UI布局
```dart
// 修改字幕模式下圆形动画位置
alignment: controller.isSubtitleMode.value
? const Alignment(0, -0.9) // 调整Y轴位置
: const Alignment(0, -0.25),
```
### 音频参数调优
```dart
// 在初始化时调整音频参数
sampleRate: 16000, // 采样率
channels: 1, // 声道数
bitsPerSample: 16, // 位深度
```
## 技术栈
- **Flutter SDK**: >= 3.0.0
- **GetX**: >= 4.6.5 (状态管理)
- **Flutter Shaders**: >= 0.0.6 (Shader动画)
- **原生插件**: Realtime服务 (音频处理)
- **WebSocket**: 实时通信协议
## 版本信息
- **当前版本**: v1.0.0 (Production)
- **最后更新**: 2024年
- **状态**: 完整功能实现

10
lib/modules/FTFTranslation/bindings/FTFTranslation_binding.dart

@ -0,0 +1,10 @@
import 'package:get/get.dart';
import '../controllers/FTFTranslation_controller.dart';
class FTFTranslationBinding extends Bindings {
@override
void dependencies() {
// 注册实时语音聊天控制器
Get.lazyPut<FTFTranslationController>(() => FTFTranslationController());
}
}

241
lib/modules/FTFTranslation/controllers/FTFTranslation_controller.dart

@ -0,0 +1,241 @@
import 'dart:async';
import 'dart:typed_data';
import 'package:get/get.dart';
import '../../../core/utils/logger.dart';
import '../../../data/services/asr_service.dart';
import '../../../data/services/speech_impl/xunfei_asr_service.dart';
import '../../../data/services/tts_service.dart';
import '../../../data/services/language_manager.dart';
import '../../../data/services/volcano_translation_service.dart';
class FTFTranslationController extends GetxController
with GetSingleTickerProviderStateMixin {
final messages = <FTFTranslationMessage>[].obs;
final isChineseInput = true.obs;
final leftLanguage = '中文'.obs;
final rightLanguage = 'English'.obs;
final isLeftInput = true.obs;
final supportedLanguages = const [
'中文',
'English',
'日本語',
'한국어',
'Español',
];
late final AsrService _asrService;
StreamSubscription? _recognitionSubscription;
RxString intermediateContent = ''.obs;
RxString finalContent = ''.obs;
BytesBuilder _originaBytes = BytesBuilder(); //原始音频
final VolcanoTranslationService _translationService =
Get.find<VolcanoTranslationService>();
final LanguageManager _languageManager = Get.find<LanguageManager>();
// 当前使用的TTS服务
late TtsService _ttsService;
final isTranslating = false.obs;
final sourceLanguageCode = 'zh-CN'.obs;
final targetLanguageCode = 'en-US'.obs;
final isTtsEnabled = true.obs;
@override
void onInit() {
super.onInit();
Get.put(XunfeiAsrService());
final TtsService _ttsService = Get.find<TtsService>();
_asrService = Get.find<XunfeiAsrService>();
}
@override
void onClose() {
_asrService.dispose();
_recognitionSubscription?.cancel();
super.onClose();
}
//开始录音
Future<void> _startRecorder() async {
final recognitionStream =
await _asrService.startContinuousRecognition(false);
_recognitionSubscription = recognitionStream.listen(
_handleRecognitionEvent,
);
}
//结束录音
Future<void> _stopRecorder() async {
await _asrService.stopContinuousRecognition();
}
// 处理语音识别事件
void _handleRecognitionEvent(RecognitionEvent event) {
switch (event.type) {
case RecognitionEventType.intermediateResult:
intermediateContent.value = event.text;
handleFinalResult(event.text);
break;
case RecognitionEventType.finalResult:
finalContent.value += _spliceText(event);
intermediateContent.value = '';
_originaBytes.add(event.audio!);
handleIntermediateResult(event.text);
break;
default:
// 不做任何处理
break;
}
}
// 翻译文本
Future<String?> translateText(String sourceText,
{bool isFinal = false}) async {
if (sourceText.isEmpty) return null;
isTranslating.value = true;
String? translationResult;
try {
// 根据检测到的语言确定源语言和目标语言
final bool shouldSwap =
intermediateContent.value == sourceLanguageCode.value;
final detectedSourceLanguageCode =
shouldSwap ? sourceLanguageCode.value : targetLanguageCode.value;
final detectedTargetLanguageCode =
shouldSwap ? targetLanguageCode.value : sourceLanguageCode.value;
// 调用翻译服务
translationResult = await _translationService.translateText(
text: sourceText,
sourceLanguageCode: detectedSourceLanguageCode,
targetLanguageCode: detectedTargetLanguageCode,
);
if (translationResult != null) {
// 查找匹配的项目 - 优先使用timestamp查找
int index = -1;
// 播放TTS - 只在最终结果时播放
if (isTtsEnabled.value && isFinal) {
// Logger.info('播放翻译文本: ${translationResult}');
await playTranslatedText(translationResult);
}
}
} catch (e) {
Logger.error('翻译失败: ${e.toString()}');
} finally {
isTranslating.value = false;
}
return translationResult;
}
// 处理中间识别结果
void handleIntermediateResult(String text) {
if (text.isEmpty) return;
translateText(text, isFinal: false);
}
// 处理最终识别结果
Future<void> handleFinalResult(String text) async {
if (text.isEmpty) return;
// 使用项目的时间戳进行翻译
translateText(text, isFinal: true);
}
// 播放翻译文本
Future<void> playTranslatedText(String text) async {
if (text.isEmpty) return;
try {
final bool shouldSwap =
intermediateContent.value == targetLanguageCode.value;
final detectedTargetLanguageCode =
shouldSwap ? sourceLanguageCode.value : targetLanguageCode.value;
final voiceName = _languageManager
.getTtsVoiceNameByAsrCode(detectedTargetLanguageCode) ??
'en-US-AriaNeural';
await _ttsService.setVoice(voiceName);
await _ttsService.speakOnce(text);
} catch (e) {
Logger.error('播放翻译失败: ${e.toString()}');
}
}
String _spliceText(RecognitionEvent event) {
String eventText = event.text;
String text = '';
if (finalContent.value.isNotEmpty) {
if (event.role != '') {
final punctuations = {
',',
'。',
'!',
'?',
';',
':',
',',
'.',
':',
';',
'!',
'?'
};
final firstChar = eventText[0];
if (punctuations.contains(firstChar)) {
text += '$firstChar\n';
eventText = eventText.substring(1);
} else {
text += '\n';
}
text += '说话人${event.role}:';
}
} else {
text += '说话人${event.role}:';
}
text += eventText;
return text;
}
void addMessage(String text) {
final fromLang =
isLeftInput.value ? leftLanguage.value : rightLanguage.value;
final toLang = isLeftInput.value ? rightLanguage.value : leftLanguage.value;
messages.add(
FTFTranslationMessage(
original: text,
translated: '[$fromLang → $toLang] $text (翻译模拟)',
isChineseSpeaker: isLeftInput.value,
),
);
}
void swapLanguages() {
final temp = leftLanguage.value;
leftLanguage.value = rightLanguage.value;
rightLanguage.value = temp;
isLeftInput.value = !isLeftInput.value;
}
void toggleInputSide(bool isLeft) => isLeftInput.value = isLeft;
}
class FTFTranslationMessage {
final String original;
final String translated;
final bool isChineseSpeaker;
FTFTranslationMessage({
required this.original,
required this.translated,
required this.isChineseSpeaker,
});
}

212
lib/modules/FTFTranslation/views/FTFTranslation_view.dart

@ -0,0 +1,212 @@
import 'package:flutter/material.dart';
import 'package:get/get.dart';
import '../controllers/FTFTranslation_controller.dart';
class FTFTranslationView extends GetView<FTFTranslationController> {
const FTFTranslationView({Key? key}) : super(key: key);
@override
Widget build(BuildContext context) {
final TextEditingController inputController = TextEditingController();
return Scaffold(
backgroundColor: const Color(0xFF0E0F12),
appBar: AppBar(
backgroundColor: Colors.black,
title: const Text('面对面翻译'),
centerTitle: true,
),
body: Column(
children: [
Expanded(
child: Obx(() => ListView.builder(
padding: const EdgeInsets.all(16),
itemCount: controller.messages.length,
itemBuilder: (context, index) {
final msg = controller.messages[index];
final isChinese = msg.isChineseSpeaker;
return Align(
alignment: isChinese
? Alignment.centerLeft
: Alignment.centerRight,
child: Container(
margin: const EdgeInsets.symmetric(vertical: 6),
padding: const EdgeInsets.all(12),
decoration: BoxDecoration(
color: isChinese ? Colors.blue : Colors.grey[800],
borderRadius: BorderRadius.circular(12),
),
constraints: const BoxConstraints(maxWidth: 300),
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text(
msg.original,
style: const TextStyle(
color: Colors.white,
fontWeight: FontWeight.bold,
),
),
const SizedBox(height: 4),
Text(
msg.translated,
style: TextStyle(
color: Colors.white.withOpacity(0.9),
fontSize: 14,
),
),
],
),
),
);
},
)),
),
_buildBottomInput(inputController),
],
),
);
}
Widget _buildBottomInput(TextEditingController inputController) {
return Obx(() {
final isLeftInput = controller.isLeftInput.value;
final inputLang = isLeftInput
? controller.leftLanguage.value
: controller.rightLanguage.value;
return Container(
color: Colors.black,
padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 8),
child: Column(
children: [
Row(
children: [
Expanded(
child: TextField(
controller: inputController,
style: const TextStyle(color: Colors.white),
decoration: InputDecoration(
filled: true,
fillColor: Colors.grey[900],
hintText: '请输入 $inputLang 内容',
hintStyle: const TextStyle(color: Colors.grey),
border: OutlineInputBorder(
borderRadius: BorderRadius.circular(8),
borderSide: BorderSide.none,
),
contentPadding:
const EdgeInsets.symmetric(horizontal: 12),
),
),
),
IconButton(
onPressed: () {
final text = inputController.text.trim();
if (text.isNotEmpty) {
controller.addMessage(text);
inputController.clear();
}
},
icon: const Icon(Icons.send, color: Colors.white),
),
],
),
const SizedBox(height: 8),
Row(
children: [
Expanded(
child: _buildLanguageDropdown(
selected: controller.leftLanguage.value,
items: controller.supportedLanguages,
onChanged: (value) {
if (value != null) controller.leftLanguage.value = value;
},
icon: Icons.headphones,
isSelected: isLeftInput,
label: '左方语言',
onTapInputSide: () => controller.toggleInputSide(true),
),
),
const SizedBox(width: 12),
GestureDetector(
onTap: controller.swapLanguages,
child: Container(
padding: const EdgeInsets.all(8),
decoration: BoxDecoration(
color: Colors.white10,
shape: BoxShape.circle,
),
child:
const Icon(Icons.compare_arrows, color: Colors.white),
),
),
const SizedBox(width: 12),
Expanded(
child: _buildLanguageDropdown(
selected: controller.rightLanguage.value,
items: controller.supportedLanguages,
onChanged: (value) {
if (value != null) controller.rightLanguage.value = value;
},
icon: Icons.smartphone,
isSelected: !isLeftInput,
label: '右方语言',
onTapInputSide: () => controller.toggleInputSide(false),
),
),
],
),
],
),
);
});
}
Widget _buildLanguageDropdown({
required String selected,
required List<String> items,
required void Function(String?) onChanged,
required IconData icon,
required bool isSelected,
required String label,
required VoidCallback onTapInputSide,
}) {
return GestureDetector(
onTap: onTapInputSide,
child: Container(
padding: const EdgeInsets.symmetric(vertical: 8, horizontal: 12),
decoration: BoxDecoration(
border: Border.all(
color: isSelected ? Colors.white : Colors.grey,
),
borderRadius: BorderRadius.circular(24),
color: isSelected ? Colors.white10 : Colors.transparent,
),
child: Row(
children: [
Icon(icon, color: Colors.white),
const SizedBox(width: 8),
Expanded(
child: DropdownButton<String>(
dropdownColor: Colors.grey[900],
value: selected,
isExpanded: true,
underline: const SizedBox(),
iconEnabledColor: Colors.white,
style: const TextStyle(color: Colors.white),
items: items.map((lang) {
return DropdownMenuItem(
value: lang,
child: Text(lang),
);
}).toList(),
onChanged: onChanged,
),
),
],
),
),
);
}
}

2
lib/modules/agent/controllers/agent_controller.dart

@ -332,7 +332,7 @@ class AgentController extends GetxController {
case AgentServiceEventType.ttsCanceled:
isSpeaking.value = false;
Logger.i(TAG, 'TTS停止播放');
Logger.i(TAG, 'TTS停止播放, $event.type');
break;
case AgentServiceEventType.imageProcessing:

7
lib/modules/meeting/controllers/meeting_controller.dart

@ -14,6 +14,7 @@ import 'package:permission_handler/permission_handler.dart';
import '../../../core/utils/logger.dart';
import '../../../core/utils/synchrodata.dart';
import '../../../core/utils/upload_oss.dart';
import '../../../data/services/ble_manager.dart';
import '../../../data/services/db/sqflite_api.dart';
import '../../../core/utils/permission_util.dart';
@ -37,7 +38,8 @@ class MeetingController extends GetxController {
TextEditingController titleController = TextEditingController(); // 录音标题
RxList dataList = [].obs;
// 蓝牙服务
final bleManager = Get.find<BleManager>();
List dataListTask = [];
bool isDisposed = false;
String filePath = '';
@ -313,6 +315,9 @@ class MeetingController extends GetxController {
PlatformFile file = result.files.first;
filePath = await _saveFile(file);
fileName = file.name;
} else {
EasyLoading.dismiss();
return;
}
}

5
lib/modules/meeting/controllers/meeting_record_controller.dart

@ -113,12 +113,15 @@ class MeetingRecordController extends GetxController
switch (audioType.value) {
case 0:
fileName.value = "现场录音";
_audioSourceType = false;
break;
case 1:
fileName.value = "音、视频录音";
_audioSourceType = true;
break;
case 2:
fileName.value = "通话录音";
_audioSourceType = true;
break;
}
newName.value = fileName.value;
@ -234,7 +237,7 @@ class MeetingRecordController extends GetxController
void _startHardwareServices() {
switch (audioType.value) {
case 0:
_bleManager.openEncoder();
// _bleManager.openEncoder();
break;
case 1:
_bleManager.openDecoder();

88
lib/modules/meeting/views/meeting_view.dart

@ -454,43 +454,43 @@ class MeetingView extends GetView<MeetingController> {
// TODO: 实现从文件导入逻辑
},
),
SizedBox(height: 8.w),
_importAudioItem1(
context,
isDarkMode,
icon: Image.asset('assets/images/explore.png',
width: 32.w, height: 32.w),
title: '从翻译里导入',
onTap: () {
Navigator.of(context).pop();
_showLocalAudioVideoPicker(context, isDarkMode);
// TODO: 实现从相册导入逻辑
},
),
SizedBox(height: 8.w),
_importAudioItem1(
context,
isDarkMode,
icon: Row(
mainAxisSize: MainAxisSize.min,
children: [
Image.asset('assets/images/explore.png',
width: 20.w, height: 20.w),
SizedBox(width: 4.w),
Image.asset('assets/images/explore.png',
width: 20.w, height: 20.w),
SizedBox(width: 4.w),
Image.asset('assets/images/explore.png',
width: 20.w, height: 20.w),
],
),
title: '从opus导出',
onTap: () {
Navigator.of(context).pop();
// TODO: 实现从其他APP导入逻辑
_showLocalOpusPicker(context, isDarkMode);
},
),
// SizedBox(height: 8.w),
// _importAudioItem1(
// context,
// isDarkMode,
// icon: Image.asset('assets/images/explore.png',
// width: 32.w, height: 32.w),
// title: '从翻译里导入',
// onTap: () {
// Navigator.of(context).pop();
// _showLocalAudioVideoPicker(context, isDarkMode);
// // TODO: 实现从相册导入逻辑
// },
// ),
// SizedBox(height: 8.w),
// _importAudioItem1(
// context,
// isDarkMode,
// icon: Row(
// mainAxisSize: MainAxisSize.min,
// children: [
// Image.asset('assets/images/explore.png',
// width: 20.w, height: 20.w),
// SizedBox(width: 4.w),
// Image.asset('assets/images/explore.png',
// width: 20.w, height: 20.w),
// SizedBox(width: 4.w),
// Image.asset('assets/images/explore.png',
// width: 20.w, height: 20.w),
// ],
// ),
// title: '从opus导出',
// onTap: () {
// Navigator.of(context).pop();
// // TODO: 实现从其他APP导入逻辑
// _showLocalOpusPicker(context, isDarkMode);
// },
// ),
],
),
),
@ -578,13 +578,12 @@ class MeetingView extends GetView<MeetingController> {
SizedBox(height: 8.w),
// 音、视频录音 - 根据控制器状态禁用
_importAudioItem(
context,
isDarkMode,
context, isDarkMode,
icon: Image.asset('assets/images/explore.png',
width: 32.w, height: 32.w),
title: '音、视频录音',
enabled: false, // 依赖控制器状态
onTap: false
enabled: controller.bleManager.isConnected, // 依赖控制器状态
onTap: controller.bleManager.isConnected
? () async {
// 条件启用
Navigator.of(context).pop();
@ -599,14 +598,13 @@ class MeetingView extends GetView<MeetingController> {
SizedBox(height: 8.w),
// 通话录音 - 根据控制器状态禁用
_importAudioItem(
context,
isDarkMode,
context, isDarkMode,
icon: Row(
// ... 图标组合保持不变
),
title: '通话录音',
enabled: false, // 依赖控制器状态
onTap: false
enabled: controller.bleManager.isConnected, // 依赖控制器状态
onTap: controller.bleManager.isConnected
? () async {
// 条件启用
Navigator.of(context).pop();

54
lib/modules/meeting/views/share_bottom_sheet.dart

@ -72,16 +72,42 @@ class _ShareBottomSheetState extends State<ShareBottomSheet> {
child: Column(
children: [
_box([
_shareItem(Icons.link, '分享链接', isBorder: false, onTap: () {
final GetStorage storage = GetStorage();
String token = storage.read("logintoken") ?? '';
String userId = '';
Map? userInfo = storage.read("user_info");
if (userInfo != null) {
userId = userInfo['user']['uid'];
}
String id = '${userId}_${_controller.meetingData.value.id}';
}),
_shareItem(
Icons.link,
'分享链接',
isBorder: false,
onTap: () async {
Get.back();
final GetStorage storage = GetStorage();
String token = storage.read("logintoken") ?? '';
String userId = '';
Map? userInfo = storage.read("user_info");
if (userInfo != null) {
userId = userInfo['user']['uid'];
}
String id =
'${userId}_${_controller.meetingData.value.id}';
final thumbnail = await getAssetThumbnailFile(
'assets/images/headphone_dark.png',
'share_thumb.png',
);
final result = await SharePlus.instance.share(ShareParams(
title: _controller.meetingData.value.title,
subject:
'DeepSound WEB,由DeepSound.Ai提供支持-实时记录会议纪要,多语言翻译。',
uri: Uri.parse(
'http://web.ideapsound.com/share/$id/$token',
),
previewThumbnail: thumbnail,
));
if (result.status == ShareResultStatus.success) {
EasyLoading.showToast('分享完成!');
}
},
),
]),
_box([
_shareItem(Icons.copy, '复制转写', onTap: () {
@ -223,4 +249,12 @@ class _ShareBottomSheetState extends State<ShareBottomSheet> {
}
return text;
}
Future<XFile> getAssetThumbnailFile(String assetPath, String fileName) async {
final byteData = await rootBundle.load(assetPath);
final tempDir = await getTemporaryDirectory();
final file = File('${tempDir.path}/$fileName');
await file.writeAsBytes(byteData.buffer.asUint8List());
return XFile(file.path);
}
}

23
lib/modules/pairing/controllers/pairing_controller.dart

@ -505,9 +505,16 @@ class PairingController extends GetxController {
}
/// 导航到主页
void _navigateToHome() async {
await BleManager.to.startAgentService();
Future.delayed(const Duration(seconds: 1), () {
void _navigateToHome() {
Future.delayed(const Duration(seconds: 1), () async {
// 启动AgentService服务
try {
Logger.i(_tag, '启动AgentService服务');
await _bleManager.startAgentService();
} catch (e) {
Logger.e(_tag, '启动AgentService失败: $e');
}
Get.offAllNamed(Routes.home);
});
}
@ -565,8 +572,16 @@ class PairingController extends GetxController {
/// 用户点击跳过按钮
void onSkipTap() async {
await BleManager.to.startAgentService();
Logger.i(_tag, '用户选择跳过配对');
// 启动AgentService服务
try {
Logger.i(_tag, '启动AgentService服务');
await _bleManager.startAgentService();
} catch (e) {
Logger.e(_tag, '启动AgentService失败: $e');
}
Get.offAllNamed(Routes.home);
}

22
lib/modules/settings/views/settings_view.dart

@ -543,6 +543,28 @@ class SettingsView extends GetView<SettingsController> {
},
isDarkMode: isDarkMode,
),
Divider(
height: 1,
color: isDarkMode
? Colors.white.withOpacity(0.1)
: Colors.grey[200]),
// 面对面翻译测试
_buildSimpleNavigationSetting(
title: '面对面翻译测试',
subtitle: '测试面对面翻译',
icon: Icons.bluetooth_searching,
iconBgColor: isDarkMode
? Colors.green[900]!.withOpacity(0.3)
: Colors.green[100]!,
iconColor:
isDarkMode ? Colors.green[300]! : Colors.green[600]!,
titleColor: isDarkMode ? Colors.white : null,
subtitleColor: isDarkMode ? Colors.white70 : null,
onTap: () {
Get.toNamed(Routes.ftftranslation);
},
isDarkMode: isDarkMode,
),
Divider(
height: 1,
color: isDarkMode

8
lib/modules/translation/controllers/translation_controller.dart

@ -26,7 +26,7 @@ class TranslationController extends GetxController {
final LanguageManager _languageManager = Get.find<LanguageManager>();
final GetStorage _storage = GetStorage();
// 蓝牙服务
final _bleManager = Get.find<BleManager>();
final bleManager = Get.find<BleManager>();
// 音频输入源
bool _audioSourceType = false;
// 存储相关
@ -308,14 +308,14 @@ class TranslationController extends GetxController {
// translationHistory.add(newItem);
// 开始连续语音识别
if (currentMode.value == "audioVideo") {
_bleManager.openDecoder();
bleManager.openDecoder();
// 发送ble音乐或者通话远端声音
_audioSourceType = true;
isTtsEnabled.value = false; // 禁用TTS以避免干扰
Logger.info('发送ble音乐或者通话远端声音');
} else if (currentMode.value == "call") {
// 发送ble系统mic和dac(音乐或者通话远端)声音
_bleManager.openA2DPDecoder();
bleManager.openA2DPDecoder();
_audioSourceType = true;
isTtsEnabled.value = true;
Logger.info('发送ble系统mic和dac(音乐或者通话远端)声音');
@ -563,7 +563,7 @@ class TranslationController extends GetxController {
Future<void> stopAll() async {
try {
if (currentMode.value == "audioVideo" || currentMode.value == "call") {
_bleManager.closeCodec();
bleManager.closeCodec();
// 发送ble音乐或者通话远端声音
Logger.info('发送ble音乐或者通话远端声音');
}

4
lib/modules/translation/views/translation_view.dart

@ -67,12 +67,12 @@ class TranslationView extends GetView<TranslationController> {
_buildPopupMenuItem(
'audioVideo',
'audioVideoTranslation'.tr,
enabled: true, // 根据连接状态启用/禁用
enabled: controller.bleManager.isConnected, // 根据连接状态启用/禁用
),
_buildPopupMenuItem(
'call',
'callTranslation'.tr,
enabled: true, // 根据连接状态启用/禁用
enabled: controller.bleManager.isConnected, // 根据连接状态启用/禁用
),
],
child: Row(

9
lib/routes/app_pages.dart

@ -46,6 +46,8 @@ import '../modules/realtime/bindings/realtime_binding.dart';
import '../modules/realtime/views/realtime_view.dart';
import '../modules/music/views/music_playlist_view.dart';
import '../modules/music/bindings/music_playlist_binding.dart';
import '../modules/FTFTranslation/bindings/FTFTranslation_binding.dart';
import '../modules/FTFTranslation/views/FTFTranslation_view.dart';
abstract class AppPages {
static final pages = [
@ -191,6 +193,13 @@ abstract class AppPages {
transition: Transition.rightToLeft,
transitionDuration: Duration(milliseconds: 250),
),
GetPage(
name: Routes.ftftranslation,
page: () => const FTFTranslationView(),
binding: FTFTranslationBinding(),
transition: Transition.noTransition,
transitionDuration: Duration(milliseconds: 250),
),
GetPage(
name: Routes.opusTest,
page: () => const OpusTestView(),

1
lib/routes/app_routes.dart

@ -28,6 +28,7 @@ abstract class Routes {
static const translationHistory = '/translation/history'; // 翻译历史
static const speechDemo = '/speech_demo'; // 语音演示
static const translationList = '/translation/List'; // 翻译历史
static const ftftranslation = '/ftftranslation'; // 面对面翻译
// 测试路由(生产环境建议移除)
static const ttsTest = '/tts_test'; // 文字转语音测试
static const asrTest = '/asr_test'; // 语音识别测试

1
local_plugins/agent_service/android/build.gradle.kts

@ -50,4 +50,5 @@ dependencies {
add("compileOnly", project(":speech"))
add("compileOnly", project(":chat_api"))
add("compileOnly", project(":location_service"))
add("compileOnly", project(":bytedance_speech"))
}

80
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt

@ -19,7 +19,7 @@ import android.media.MediaPlayer
import com.deep_voice.speech.tts.TtsEvent
import com.deep_voice.speech.tts.TtsEventListener
import com.deep_voice.speech.tts.TtsEventType
import com.yunqiinnovation.azure_speech.AzureTtsHelper
import com.deep_voice.bytedance_speech.BytedanceTTS
/**
* 代理服务事件监听接口
*/
@ -52,8 +52,8 @@ object AgentService : CoroutineScope {
// Azure服务
private var azureAsrHelper: AzureAsrHelper? = null
// 使用AzureTtsHelper作为唯一的TTS实现
private var ttsService: AzureTtsHelper? = null
// 使用BytedanceTTS作为唯一的TTS实现
private var ttsService: BytedanceTTS? = null
// ChatAPI服务 - 使用新的ChatApiService
private lateinit var chatApiService: ChatApiService
@ -90,6 +90,21 @@ object AgentService : CoroutineScope {
// 空闲检测相关
private var idleCheckJob: Job? = null
private val maxIdleSeconds = 10 // 最大空闲秒数
// 打断防抖相关
private var lastInterruptTime = 0L
private val INTERRUPT_DEBOUNCE_MS = 300L
// 语气词过滤列表(中英文)
private val FILLER_WORDS = setOf(
// 中文语气词
"嗯", "啊", "呃", "哦", "唉", "哎", "哈", "呵", "嘿", "喂",
"哇", "呀", "哟", "嘛", "吧", "呢", "啦", "咦", "嘻", "嗨",
"额", "这个", "那个", "就是", "然后",
// 英文语气词
"um", "uh", "ah", "oh", "eh", "er", "hmm", "huh", "wow",
"well", "like", "you know", "i mean", "so", "actually"
)
/**
* 初始化系统提示词
*/
@ -216,20 +231,20 @@ object AgentService : CoroutineScope {
val ttsAppToken = config["volcanoToken"]?.toString() ?: ""
val ttsLanguage = config["ttsLanguage"]?.toString() ?: "zh-CN"
// 创建并初始化AzureTtsHelper
val azureTtsHelper = AzureTtsHelper(context)
ttsService = azureTtsHelper
// 创建并初始化BytedanceTTS
val bytedanceTts = BytedanceTTS(context)
ttsService = bytedanceTts
// 初始化Azure TTS
val success = azureTtsHelper.initialize(
ttsAppId = "", // Azure TTS不需要appId
ttsAppToken = config["azureSpeechKey"]?.toString() ?: "", // Azure需要subscription key
ttsResource = config["azureSpeechRegion"]?.toString() ?: "", // Azure需要region信息
// 初始化Bytedance TTS
val success = bytedanceTts.initialize(
ttsAppId = ttsAppId,
ttsAppToken = ttsAppToken,
ttsResource = "", // Bytedance TTS不需要resource参数
language = ttsLanguage
)
if (!success) {
Log.e(TAG, "Azure TTS服务初始化失败")
Log.e(TAG, "Bytedance TTS服务初始化失败")
}
// 添加TTS事件监听
@ -381,8 +396,28 @@ object AgentService : CoroutineScope {
"language" to detectedLanguage
))
// 如果TTS正在播放或AI正在生成,则触发打断
// 如果TTS正在播放或AI正在生成,检查是否需要打断
if (isTtsSpeaking || isAiStreaming) {
val currentTime = System.currentTimeMillis()
// 防抖处理:避免过于频繁的打断
if (currentTime - lastInterruptTime < INTERRUPT_DEBOUNCE_MS) {
return
}
// 简单过滤:太短的内容可能是噪音
if (recognizing.trim().length < 2) {
return
}
// 过滤纯语气词
val trimmedText = recognizing.trim().lowercase()
if (FILLER_WORDS.contains(trimmedText)) {
return
}
// 执行打断
lastInterruptTime = currentTime
interruptCurrentResponse()
}
}
@ -482,14 +517,21 @@ object AgentService : CoroutineScope {
*/
fun interruptCurrentResponse() {
if(isAiStreaming || isTtsSpeaking) {
// 停止TTS播放
stopTts()
val startTime = System.currentTimeMillis()
// 停止AI流输出
stopAiStream()
// 并行执行停止操作,加快响应速度
launch { stopTts() }
launch { stopAiStream() }
// 发送打断事件
sendEvent("response_interrupted", mapOf("status" to "interrupted"))
// 记录打断耗时
launch {
delay(100) // 短暂延迟后计算耗时
val duration = System.currentTimeMillis() - startTime
sendEvent("response_interrupted", mapOf(
"status" to "interrupted",
"duration_ms" to duration
))
}
}
}

8
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt

@ -158,6 +158,14 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
)
}
TtsEventType.PLAYBACK_STARTED -> mapOf(
"type" to "playback_started"
)
TtsEventType.PLAYBACK_COMPLETED -> mapOf(
"type" to "playback_completed"
)
TtsEventType.ERROR -> {
val params = event.params
mapOf(

78
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt

@ -1,14 +1,16 @@
package com.yunqiinnovation.azure_speech
import android.content.Context
import android.media.AudioManager
import com.microsoft.cognitiveservices.speech.*
import com.microsoft.cognitiveservices.speech.audio.*
import com.yunqiinnovation.azure_speech.utils.FileLogger
import com.deep_voice.speech.tts.AudioDataListener
import com.deep_voice.speech.tts.AudioOutputDevice
import com.deep_voice.speech.tts.ITtsService
import com.deep_voice.speech.tts.TtsEvent
import com.deep_voice.speech.tts.TtsEventListener
import com.deep_voice.speech.tts.TtsEventType
import com.deep_voice.speech.tts.AudioDataListener
import kotlinx.coroutines.*
import java.io.ByteArrayInputStream
import java.io.InputStream
@ -169,6 +171,76 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
audioDataListeners.remove(listener)
}
/**
* 设置是否使用内部播放器
*
* @param useInternalPlayer true: 使用内部播放器自动播放音频
* false: 仅通过音频数据监听器输出数据,不播放
*/
override fun setUseInternalPlayer(useInternalPlayer: Boolean) {
// Azure TTS 通过 AudioConfig 控制播放器
// 如果不使用内部播放器,需要重新配置 synthesizer
if (!useInternalPlayer && customAudioOutputStream == null) {
// 创建自定义音频输出流以便捕获音频数据
customAudioOutputStream = SimpleAudioPlayer(context).getAudioOutputStream()
// 如果已经初始化,需要重新创建 synthesizer
if (isInitialized && speechConfig != null) {
synthesizer?.close()
val audioConfig = AudioConfig.fromStreamOutput(customAudioOutputStream)
synthesizer = SpeechSynthesizer(speechConfig, audioConfig)
setupEventListeners()
}
} else if (useInternalPlayer && customAudioOutputStream != null) {
// 切换回使用内部播放器
customAudioOutputStream = null
// 如果已经初始化,需要重新创建 synthesizer
if (isInitialized && speechConfig != null) {
synthesizer?.close()
val audioConfig = AudioConfig.fromDefaultSpeakerOutput()
synthesizer = SpeechSynthesizer(speechConfig, audioConfig)
setupEventListeners()
}
}
}
/**
* 设置音频输出设备
*
* @param device 音频输出设备类型
*/
override fun setAudioOutputDevice(device: AudioOutputDevice) {
// Azure TTS使用系统默认的音频路由
// 音频输出设备的控制需要通过Android的AudioManager实现
val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager
audioManager?.let { manager ->
when (device) {
AudioOutputDevice.DEFAULT -> {
// 默认模式:系统自动选择
manager.mode = AudioManager.MODE_NORMAL
manager.isSpeakerphoneOn = false
}
AudioOutputDevice.SPEAKER -> {
// 强制使用扬声器
manager.mode = AudioManager.MODE_NORMAL
manager.isSpeakerphoneOn = true
}
AudioOutputDevice.HEADPHONES -> {
// 强制使用耳机(如果已连接)
manager.mode = AudioManager.MODE_NORMAL
manager.isSpeakerphoneOn = false
// 注意:Android不能强制路由到耳机,只能在耳机已连接时使用
}
AudioOutputDevice.EARPIECE -> {
// 强制使用听筒
manager.mode = AudioManager.MODE_IN_COMMUNICATION
manager.isSpeakerphoneOn = false
}
}
}
}
/**
* 触发事件通知
*/
@ -194,6 +266,8 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
SynthesisStarted?.addEventListener { _, eventArgs ->
FileLogger.d(TAG, "语音合成开始: resultId=${eventArgs.result.resultId}")
notifyEvent(TtsEventType.SYNTHESIS_STARTED)
// Azure TTS 在使用默认音频输出时会立即开始播放
notifyEvent(TtsEventType.PLAYBACK_STARTED)
}
// 合成中事件(接收音频数据)
@ -219,6 +293,8 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
FileLogger.d(TAG, "语音合成完成: resultId=${eventArgs.result.resultId}, 音频长度=${eventArgs.result.audioLength} 字节")
isSpeaking = false
notifyEvent(TtsEventType.SYNTHESIS_COMPLETED)
// Azure TTS 合成完成即播放完成
notifyEvent(TtsEventType.PLAYBACK_COMPLETED)
}
// 合成取消事件

2
local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCompanionManager.kt

@ -395,7 +395,7 @@ class BleCompanionManager(private val context: Context) {
// 构建过滤器
val filter = ScanFilter.Builder()
.setDeviceAddress(mac)
// .setManufacturerData(0xEFAB, byteArrayOf(0xA0.toByte(), 0x01))
.setManufacturerData(0xEFAB, byteArrayOf(0xA0.toByte(), 0x01))
.build()
// 扫描设置

129
local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt

@ -22,6 +22,7 @@ import java.util.concurrent.LinkedBlockingQueue
import android.os.Handler
import java.util.concurrent.atomic.AtomicBoolean
import android.util.Log
/**
* BLE服务类:提供蓝牙低功耗设备的扫描、连接和通信功能
* 支持:
@ -67,17 +68,20 @@ object BleService {
private var bluetoothLeScanner: BluetoothLeScanner? = null
// GATT连接相关
var bluetoothGatt: BluetoothGatt? = null
var bluetoothGatt: BluetoothGatt? = null
private var notifyChar: BluetoothGattCharacteristic? = null
private var writeChar: BluetoothGattCharacteristic? = null
private var audioChar: BluetoothGattCharacteristic? = null
var recordfile: RecordingFile? = null
// 扫描相关
private lateinit var scanHandler: Handler
private val scanResults = ArrayList<ScanResult>()
private var isScanning = false
// 是否是ota模式
// 是否是ota模式
var isEnterOta = false
// 连接状态
val connectionState = MutableLiveData(BleConst.STATE_DISCONNECTED)
@ -136,7 +140,7 @@ object BleService {
Log.e(TAG, "OpusManager初始化失败: ${e.message}", e)
// 根据需要决定是否因为Opus初始化失败而返回false
}
recordfile = RecordingFile(this.context)
recordfile = RecordingFile(this.context)
isInitialized = true
Log.d(TAG, "BLE服务初始化成功")
return true
@ -198,7 +202,6 @@ recordfile = RecordingFile(this.context)
// 获取一个局部引用,避免并发访问问题
val scanner = bluetoothLeScanner ?: return false
if (isScanning) return false
Log.i(TAG, "开始主动扫描BLE设备...")
scanResults.clear()
@ -341,13 +344,12 @@ recordfile = RecordingFile(this.context)
Log.w(TAG, "未找到已配对的设备 MAC 地址")
return
}
if(!isEnterOta)
{
Log.i(TAG, "检测到已配对设备,MAC: $pairedMac,尝试连接")
// 调用现有的 connect 方法
connect(pairedMac)
if (!isEnterOta) {
Log.i(TAG, "检测到已配对设备,MAC: $pairedMac,尝试连接")
// 调用现有的 connect 方法
connect(pairedMac)
}
} catch (e: Exception) {
@ -518,7 +520,7 @@ recordfile = RecordingFile(this.context)
when (c.uuid) {
// 音频特征数据
BleConst.RECEIVE_AUDIO_CHAR_UUID -> {
// Log.i(TAG, "收到音频特征数据")
processAudioData(data)
}
@ -597,7 +599,7 @@ recordfile = RecordingFile(this.context)
private fun processAudioData(data: ByteArray) {
try {
if (opusManager?.isDecodeStream == true) {
recordfile?.saveAudioDataToWav(data)
recordfile?.saveAudioDataToWav(data)
opusManager?.writeAudioStream(data)
} else {
// Log.d(TAG, "Opus解码流未启动,忽略音频数据")
@ -795,17 +797,16 @@ recordfile = RecordingFile(this.context)
BleConst.CODEC_CONTROL_ENCODE_ON -> "已打开编码"
else -> "未知状态($codecStatus)"
}
if (codecStatus == BleConst.CODEC_CONTROL_DECODE_ON ||
codecStatus == BleConst.CODEC_CONTROL_A2DP_PLAY ||
codecStatus == BleConst.CODEC_CONTROL_ENCODE_ON) {
recordfile!!.closeFile()
recordfile!!.creatingFiles()
}
else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE)
{
recordfile!!.closeFile()
}
if (codecStatus == BleConst.CODEC_CONTROL_DECODE_ON ||
codecStatus == BleConst.CODEC_CONTROL_A2DP_PLAY ||
codecStatus == BleConst.CODEC_CONTROL_ENCODE_ON
) {
recordfile!!.closeFile()
recordfile!!.creatingFiles()
} else if (codecStatus == BleConst.CODEC_CONTROL_CLOSE) {
recordfile!!.closeFile()
}
// 声道模式描述
val channelDesc = when (channelMode) {
BleConst.AUDIO_CHANNEL_LEFT -> "左声道"
@ -1093,7 +1094,7 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE)
// // The library might expect this in a different unit or derive it.
// // Given JlOpusPlugin.kt, packetSize refers to Opus encoded frame duration in ms.
// )
startOpusStreamDecoding(false, 1, 16000, 40)
startOpusStreamDecoding(false, 1, 16000, 40)
Log.i(TAG, "打开解码0xA2")
return sendCommand(
@ -1110,7 +1111,7 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE)
*/
fun openA2DPDecoder(): Boolean {
Log.i(TAG, "打开编码 0xA1")
startOpusStreamDecoding(false, 1, 16000, 80)
startOpusStreamDecoding(false, 1, 16000, 80)
// Log.i(TAG, "打开解码...")
return sendCommand(
BleConst.CMD_CONTROL_CODEC.toByte(), byteArrayOf(
@ -1135,7 +1136,7 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE)
fun openEncoder(): Boolean {
Log.i(TAG, "打开编码0xB1")
startOpusStreamDecoding(false, 1, 16000, 40)
startOpusStreamDecoding(false, 1, 16000, 40)
// 目前仅发送命令通知设备开始编码。
return sendCommand(
BleConst.CMD_CONTROL_CODEC.toByte(),
@ -1385,9 +1386,10 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE)
}
}
}
class RecordingFile(private val context: Context) {
class RecordingFile(private val context: Context) {
private var currentAudioFile: File? = null
private var fos: FileOutputStream? = null
@ -1408,8 +1410,9 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE)
// 创建新的音频文件
val dateFormat = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.getDefault())
val timestamp = dateFormat.format(Date())
val filePath = File(context.getExternalFilesDir(null), "${fileName}_${timestamp}.opus").absolutePath
val filePath =
File(context.getExternalFilesDir(null), "${fileName}_${timestamp}.opus").absolutePath
currentAudioFile = File(filePath)
currentAudioFile?.createNewFile()
// 追加音频数据到文件
@ -1417,42 +1420,42 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE)
startWriteThread() // 新增:启动写入线程
}
// 新增:启动写入线程
private fun startWriteThread() {
if (isWriting.get()) return
isWriting.set(true)
writeThread = Thread {
try {
while (isWriting.get() || writeQueue.isNotEmpty()) {
val data = writeQueue.poll() ?: continue
fos?.write(data)
}
} catch (e: Exception) {
Log.e("", "异步写入音频数据失败: ${e.message}")
// 新增:启动写入线程
private fun startWriteThread() {
if (isWriting.get()) return
isWriting.set(true)
writeThread = Thread {
try {
while (isWriting.get() || writeQueue.isNotEmpty()) {
val data = writeQueue.poll() ?: continue
fos?.write(data)
}
} catch (e: Exception) {
Log.e("", "异步写入音频数据失败: ${e.message}")
}
writeThread?.start()
}
writeThread?.start()
}
/**
* 保存音频数据到 WAV 文件(异步)
*/
internal fun saveAudioDataToWav(buffer: ByteArray) {
if (fos == null || currentAudioFile == null) return
// 放入队列,由写线程写入
// 1. 将字节数据转换为十六进制字符串
// val hexData = buildString {
// buffer.forEachIndexed { index, byte ->
// append("%02X".format(byte))
// }
// }
// 2. 写入十六进制字符串
writeQueue.offer(buffer)
totalBytesWritten += buffer.size
}
/**
* 保存音频数据到 WAV 文件(异步)
*/
internal fun saveAudioDataToWav(buffer: ByteArray) {
if (fos == null || currentAudioFile == null) return
// 放入队列,由写线程写入
// 1. 将字节数据转换为十六进制字符串
// val hexData = buildString {
// buffer.forEachIndexed { index, byte ->
// append("%02X".format(byte))
// }
// }
// 2. 写入十六进制字符串
writeQueue.offer(buffer)
totalBytesWritten += buffer.size
}
internal fun closeFile() {

420
local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt

@ -1,5 +1,6 @@
package com.deep_voice.bytedance_speech
import android.content.Context
import android.media.AudioAttributes
import android.media.AudioFormat
import android.media.AudioManager
@ -7,112 +8,143 @@ import android.media.AudioTrack
import android.os.Build
import android.util.Log
import com.deep_voice.speech.tts.AudioDataListener
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancelChildren
import kotlinx.coroutines.delay
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.CancellationException
import com.deep_voice.speech.tts.AudioOutputDevice
/**
* 字节跳动语音合成音频播放器
*
* 使用AudioTrack播放PCM格式音频流,基于Kotlin协程实现异步处理
* 简化的音频流播放器
* 基于位置标记触发播放完成回调
*/
class BytedanceAudioPlayer : AudioDataListener, CoroutineScope {
class BytedanceAudioPlayer(private val context: Context) : AudioDataListener {
companion object {
private const val TAG = "BytedanceAudioPlayer"
private const val SAMPLE_RATE = 24000 // 样本率
private const val CHANNEL_CONFIG = AudioFormat.CHANNEL_OUT_MONO // 单声道
private const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT // 16位PCM
private const val IDLE_TIMEOUT_MS = 800L // 无数据超时时间
private const val SAMPLE_RATE = 24000
private const val CHANNEL_CONFIG = AudioFormat.CHANNEL_OUT_MONO
private const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT
}
// 协程相关
private val job = SupervisorJob()
override val coroutineContext = Dispatchers.IO + job
// 音频处理相关
private var audioTrack: AudioTrack? = null
private val isPlaying = AtomicBoolean(false)
private val isPaused = AtomicBoolean(false)
// 播放状态
private val playStarted = AtomicBoolean(false)
private val completionReported = AtomicBoolean(false)
// 使用简单的并发队列,保证线程安全
private val audioDataQueue = ConcurrentLinkedQueue<ByteArray>()
private var isFirstData = true // 是否是第一次接收数据
private var totalBytesWritten = 0 // 总共写入的字节数
private var sessionActive = true // 会话是否活跃
private var audioOutputDevice = AudioOutputDevice.DEFAULT // 音频输出设备
private var audioManager: AudioManager? = null
// 播放状态监听
// 回调
private var onPlayStarted: (() -> Unit)? = null
private var onPlayCompleted: (() -> Unit)? = null
private var onError: ((String) -> Unit)? = null
// 处理协程
private var processingJob: Job? = null
private var lastDataTime = 0L
// 播放位置监听器
private val playbackListener = object : AudioTrack.OnPlaybackPositionUpdateListener {
override fun onMarkerReached(track: AudioTrack) {
Log.d(TAG, "播放到达标记位置: ${track.playbackHeadPosition}")
onPlayCompleted?.invoke()
}
override fun onPeriodicNotification(track: AudioTrack) {
// 不使用周期性通知
}
}
init {
Log.d(TAG, "BytedanceAudioPlayer初始化")
Log.d(TAG, "BytedanceAudioPlayer 初始化")
audioManager = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager
initAudioTrack()
}
/**
* 开始播放
* 开始新的播放会话
*/
fun start() {
if (isPlaying.getAndSet(true)) return
fun startSession() {
Log.d(TAG, "开始新会话")
isPaused.set(false)
playStarted.set(false)
completionReported.set(false)
lastDataTime = System.currentTimeMillis()
// 重置状态
isFirstData = true
totalBytesWritten = 0
sessionActive = true
processingJob = launch {
try {
initAudioTrack()
// 处理队列中的音频数据
while (isActive && isPlaying.get()) {
processQueuedAudio()
// 检查是否播放完成
checkPlaybackCompletion()
// 队列为空时短暂延迟
if (audioDataQueue.isEmpty()) {
delay(10)
}
}
} catch (e: CancellationException) {
// 协程被取消,正常行为
} catch (e: Exception) {
Log.e(TAG, "播放错误: ${e.message}")
onError?.invoke("播放错误: ${e.message}")
stopInternal(false)
// 重置 AudioTrack
audioTrack?.let { track ->
if (track.state == AudioTrack.STATE_INITIALIZED) {
// 停止并清空缓冲区
track.pause()
track.flush()
// 重新开始播放
track.play()
Log.d(TAG, "AudioTrack 已重置")
}
}
} ?: initAudioTrack() // 如果没有初始化,则初始化
}
/**
* 暂停播放
* 标记数据流结束
*/
fun pause() {
if (!isPlaying.get() || isPaused.getAndSet(true)) return
audioTrack?.pause()
fun endSession() {
if (!sessionActive) return
Log.d(TAG, "数据流结束,总共写入字节数: $totalBytesWritten")
sessionActive = false
audioTrack?.let { track ->
if (totalBytesWritten > 0) {
// 计算总帧数(16-bit 单声道,每帧2字节)
val totalFrames = totalBytesWritten / 2
// 设置标记位置
try {
track.setNotificationMarkerPosition(totalFrames)
Log.d(TAG, "设置播放完成标记位置: $totalFrames")
} catch (e: Exception) {
Log.e(TAG, "设置标记失败: ${e.message}")
// 设置失败时,使用延迟触发作为后备
val durationMs = (totalFrames * 1000L) / SAMPLE_RATE
android.os.Handler(android.os.Looper.getMainLooper()).postDelayed({
onPlayCompleted?.invoke()
}, durationMs + 500)
}
} else {
// 没有数据,直接触发完成
onPlayCompleted?.invoke()
}
}
}
/**
* 恢复播放
* 接收音频数据
*/
fun resume() {
if (!isPlaying.get() || !isPaused.getAndSet(false)) return
audioTrack?.play()
override fun onAudioData(data: ByteArray) {
if (data.isEmpty()) return
if (!sessionActive) {
Log.w(TAG, "收到音频数据但会话未激活,忽略数据")
return
}
if (audioTrack == null || audioTrack?.state != AudioTrack.STATE_INITIALIZED) {
Log.w(TAG, "收到音频数据但 AudioTrack 未初始化或状态不正确,忽略数据")
return
}
try {
audioTrack?.let { track ->
if (track.state == AudioTrack.STATE_INITIALIZED) {
val bytesWritten = track.write(data, 0, data.size)
if (bytesWritten > 0) {
// 第一次写入数据时自动触发开始回调
if (isFirstData) {
isFirstData = false
onPlayStarted?.invoke()
Log.d(TAG, "播放开始")
}
// 累计写入字节数
if (sessionActive) {
totalBytesWritten += bytesWritten
}
}
}
}
} catch (e: Exception) {
Log.e(TAG, "写入音频数据失败: ${e.message}")
}
}
/**
@ -120,44 +152,32 @@ class BytedanceAudioPlayer : AudioDataListener, CoroutineScope {
*/
fun stop() {
Log.d(TAG, "停止播放")
stopInternal(true)
}
/**
* 内部停止处理
*/
private fun stopInternal(reportCompletion: Boolean) {
if (!isPlaying.getAndSet(false)) return
isPaused.set(false)
processingJob?.cancel()
audioTrack?.stop()
// 不在stopInternal中释放资源,只停止播放
sessionActive = false
// 清空队列
audioDataQueue.clear()
// 播放完成通知
if (reportCompletion && playStarted.get() && !completionReported.getAndSet(true)) {
launch(Dispatchers.Main) {
onPlayCompleted?.invoke()
audioTrack?.let { track ->
if (track.state == AudioTrack.STATE_INITIALIZED) {
track.pause()
track.flush()
}
}
// 如果已经开始播放,触发完成回调
if (!isFirstData) {
onPlayCompleted?.invoke()
}
}
/**
* 释放资源
*/
fun release() {
stopInternal(false)
Log.d(TAG, "释放资源")
stop()
// 在release方法中释放AudioTrack资源
audioTrack?.release()
audioTrack = null
// 释放协程资源
job.cancel()
}
/**
@ -175,127 +195,77 @@ class BytedanceAudioPlayer : AudioDataListener, CoroutineScope {
}
/**
* 设置错误回调
* 设置错误回调(兼容接口)
*/
fun setOnError(listener: (String) -> Unit) {
onError = listener
}
/**
* 接收音频数据(实现AudioDataListener接口)
*/
override fun onAudioData(data: ByteArray) {
if (data.isEmpty()) return
lastDataTime = System.currentTimeMillis()
if (!isPlaying.get()) {
start()
}
// 添加到队列 - 简单有效,保证FIFO顺序
audioDataQueue.add(data.copyOf())
// 不实现,仅为兼容
}
/**
* 检查是否正在播放
*/
fun isPlaying(): Boolean = isPlaying.get() && !isPaused.get()
/**
* 处理队列中的音频数据
*/
private suspend fun processQueuedAudio() {
if (isPaused.get() || !isPlaying.get() || !isActive) return
// 获取并处理队列中的数据
val data = audioDataQueue.poll() ?: return
// 写入音频数据
val result = audioTrack?.write(data, 0, data.size) ?: 0
if (result > 0) {
// 标记播放开始
if (!playStarted.getAndSet(true)) {
withContext(Dispatchers.Main) {
onPlayStarted?.invoke()
}
}
} else if (result < 0) {
// 处理错误
handleAudioTrackError(result)
}
fun isPlaying(): Boolean {
return audioTrack?.playState == AudioTrack.PLAYSTATE_PLAYING
}
/**
* 检查播放是否完成(超时无数据)
* 设置音频输出设备
*/
private suspend fun checkPlaybackCompletion() {
val currentTime = System.currentTimeMillis()
// 超时判断 - 队列为空且超过超时时间
if (playStarted.get() && audioDataQueue.isEmpty() &&
currentTime - lastDataTime > IDLE_TIMEOUT_MS && isPlaying.get()) {
// 播放完成
if (!completionReported.getAndSet(true)) {
withContext(Dispatchers.Main) {
onPlayCompleted?.invoke()
}
stopInternal(false) // 已报告,不需要再次报告
}
}
}
/**
* 处理AudioTrack错误
*/
private fun handleAudioTrackError(result: Int) {
val errorMsg = when(result) {
AudioTrack.ERROR_INVALID_OPERATION -> "无效操作"
AudioTrack.ERROR_BAD_VALUE -> "参数错误"
AudioTrack.ERROR_DEAD_OBJECT -> "对象已销毁"
else -> "未知错误"
}
// 对象已销毁时尝试重建
if (result == AudioTrack.ERROR_DEAD_OBJECT) {
try {
audioTrack?.release()
fun setAudioOutputDevice(device: AudioOutputDevice) {
if (audioOutputDevice != device) {
audioOutputDevice = device
// 如果AudioTrack已初始化,需要重新创建以应用新的输出设备设置
if (audioTrack != null) {
val wasPlaying = isPlaying()
initAudioTrack()
} catch (e: Exception) {
Log.e(TAG, "重建AudioTrack失败: ${e.message}")
throw e
if (wasPlaying) {
audioTrack?.play()
}
}
} else {
Log.e(TAG, "AudioTrack错误: $errorMsg")
}
}
/**
* 初始化AudioTrack
* 初始化 AudioTrack
*/
private fun initAudioTrack() {
val minBufferSize = AudioTrack.getMinBufferSize(
SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT
)
if (minBufferSize == AudioTrack.ERROR || minBufferSize == AudioTrack.ERROR_BAD_VALUE) {
throw IllegalStateException("无法获取有效的音频缓冲区大小")
}
// 释放旧实例
audioTrack?.release()
// 使用较大的缓冲区提高稳定性
val bufferSize = minBufferSize * 4
// 计算缓冲区大小
val minBufferSize = AudioTrack.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT)
val bufferSize = minBufferSize * 2
// 创建AudioTrack
audioTrack = if (Build.VERSION.SDK_INT >= 23) { // Android M (6.0)
AudioTrack.Builder()
.setAudioAttributes(
// 根据输出设备配置AudioAttributes
val audioAttributes = when (audioOutputDevice) {
AudioOutputDevice.EARPIECE -> {
// 听筒模式
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
AudioAttributes.Builder()
.setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION)
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
.build()
} else {
null
}
}
else -> {
// 默认、耳机、扬声器模式
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
AudioAttributes.Builder()
.setUsage(AudioAttributes.USAGE_MEDIA)
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
.build()
)
} else {
null
}
}
}
// 创建 AudioTrack
audioTrack = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && audioAttributes != null) {
AudioTrack.Builder()
.setAudioAttributes(audioAttributes)
.setAudioFormat(
AudioFormat.Builder()
.setEncoding(AUDIO_FORMAT)
@ -308,8 +278,12 @@ class BytedanceAudioPlayer : AudioDataListener, CoroutineScope {
.build()
} else {
@Suppress("DEPRECATION")
val streamType = when (audioOutputDevice) {
AudioOutputDevice.EARPIECE -> AudioManager.STREAM_VOICE_CALL
else -> AudioManager.STREAM_MUSIC
}
AudioTrack(
AudioManager.STREAM_MUSIC,
streamType,
SAMPLE_RATE,
CHANNEL_CONFIG,
AUDIO_FORMAT,
@ -318,12 +292,46 @@ class BytedanceAudioPlayer : AudioDataListener, CoroutineScope {
)
}
// 检查初始化状态
if (audioTrack?.state != AudioTrack.STATE_INITIALIZED) {
throw IllegalStateException("AudioTrack初始化失败")
}
// 设置播放位置监听器
audioTrack?.setPlaybackPositionUpdateListener(playbackListener)
// 配置音频路由
configureAudioRouting()
// 开始播放
audioTrack?.play()
Log.d(TAG, "AudioTrack 初始化成功")
}
/**
* 配置音频路由
*/
private fun configureAudioRouting() {
audioManager?.let { manager ->
when (audioOutputDevice) {
AudioOutputDevice.DEFAULT -> {
// 默认模式:系统自动选择
manager.mode = AudioManager.MODE_NORMAL
manager.isSpeakerphoneOn = false
}
AudioOutputDevice.SPEAKER -> {
// 强制使用扬声器
manager.mode = AudioManager.MODE_NORMAL
manager.isSpeakerphoneOn = true
}
AudioOutputDevice.HEADPHONES -> {
// 强制使用耳机(如果已连接)
manager.mode = AudioManager.MODE_NORMAL
manager.isSpeakerphoneOn = false
// 注意:Android不能强制路由到耳机,只能在耳机已连接时使用
}
AudioOutputDevice.EARPIECE -> {
// 强制使用听筒
manager.mode = AudioManager.MODE_IN_COMMUNICATION
manager.isSpeakerphoneOn = false
}
}
}
}
}
}

6
local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceSpeechPlugin.kt

@ -255,6 +255,12 @@ class BytedanceSpeechPlugin : FlutterPlugin {
TtsEventType.SYNTHESIS_CANCELED -> {
sendTTSEvent("canceled", null)
}
TtsEventType.PLAYBACK_STARTED -> {
sendTTSEvent("playback_started", null)
}
TtsEventType.PLAYBACK_COMPLETED -> {
sendTTSEvent("playback_completed", null)
}
TtsEventType.ERROR -> {
val params = HashMap<String, Any>()
params["code"] = event.params["errorCode"] ?: "UNKNOWN_ERROR"

197
local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt

@ -3,6 +3,7 @@ package com.deep_voice.bytedance_speech
import android.content.Context
import android.util.Log
import com.deep_voice.speech.tts.AudioDataListener
import com.deep_voice.speech.tts.AudioOutputDevice
import com.deep_voice.speech.tts.ITtsService
import com.deep_voice.speech.tts.TtsEvent
import com.deep_voice.speech.tts.TtsEventListener
@ -98,7 +99,10 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
private val audioDataListeners = mutableListOf<AudioDataListener>()
// 内部音频播放器
private val audioPlayer = BytedanceAudioPlayer()
private val audioPlayer = BytedanceAudioPlayer(context)
// 是否使用内部播放器
private var useInternalPlayer = true
// WebSocket连接
private var webSocket: WebSocket? = null
@ -119,9 +123,7 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
private var language: String = "zh-CN"
// 流式处理状态
private var isStreamMode = false
private var currentStatus = STATUS_STOPPED
private var isSpeaking = false
// 连接相关状态
private var connectionAttempts = 0
@ -155,30 +157,27 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
* 初始化音频播放器回调
*/
private fun initAudioPlayerCallbacks() {
// 添加播放器作为音频数据监听器
addAudioDataListener(audioPlayer)
audioPlayer.setOnPlayStarted {
Log.d(TAG, "播放开始")
isSpeaking = true
updateStatus(STATUS_SPEAKING)
notifyEvent(TtsEventType.SYNTHESIS_STARTED)
notifyEvent(TtsEventType.PLAYBACK_STARTED)
}
// 设置播放完成回调
audioPlayer.setOnPlayCompleted {
Log.d(TAG, "播放结束")
isSpeaking = false
updateStatus(if (isStreamMode) STATUS_READY else STATUS_STOPPED)
notifyEvent(TtsEventType.SYNTHESIS_COMPLETED)
updateStatus(STATUS_READY)
notifyEvent(TtsEventType.PLAYBACK_COMPLETED)
}
// 设置错误回调
audioPlayer.setOnError { errorMsg ->
isSpeaking = false
updateStatus(STATUS_ERROR)
notifyEvent(TtsEventType.ERROR, mapOf("errorCode" to "PLAYER_ERROR", "errorMessage" to errorMsg))
}
// 根据设置决定是否使用内部播放器
updateInternalPlayerUsage()
}
/**
@ -208,16 +207,19 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
*/
override fun stop(): Boolean {
try {
audioPlayer.stop()
Log.i(TAG, ">stop()")
if (useInternalPlayer) {
audioPlayer.stop()
}
textProcessingJob?.cancel()
launch {
if (isSessionStarted) {
finishSession()
}
isSpeaking = false // 显式设置,因为这是强制停止
updateStatus(STATUS_STOPPED)
if (isSessionStarted) {
finishSession()
}
updateStatus(STATUS_STOPPED)
return true
} catch (e: Exception) {
Log.e(TAG, "停止合成失败: ${e.message}")
@ -277,7 +279,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
isConnected = false
isSessionStarted = false
isSpeaking = false
updateStatus(STATUS_STOPPED)
startTextProcessing()
@ -304,29 +305,9 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
* 单次播放(非流式)
*/
override fun speakOnce(text: String): Boolean {
try {
if (text.isEmpty()) {
return false
}
isStreamMode = false
launch {
if (!ensureSessionReady()) {
notifyEvent(TtsEventType.ERROR, mapOf("errorCode" to "SESSION_ERROR",
"errorMessage" to "无法建立会话"))
return@launch
}
sendTextRequest(text)
}
return true
} catch (e: Exception) {
Log.e(TAG, "语音合成失败: ${e.message}")
updateStatus(STATUS_ERROR)
return false
}
// 未实现:只支持流式模式
Log.w(TAG, "speakOnce未实现,请使用speakStream")
return false
}
/**
@ -337,7 +318,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
if (text.isEmpty()) {
return true
}
isStreamMode = true
// 确保会话已就绪
if (!isSessionStarted && !ensureSessionReady()) {
@ -365,17 +345,11 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
*/
override fun flushStream(): Boolean {
try {
if (!isStreamMode) return true
// Log.d(TAG, "flushStream: $sessionId")
launch {
withContext(Dispatchers.IO) {
delay(300) // 确保现有文本处理完成
}
if (isSessionStarted) {
isSessionStarted = false
finishSession()
}
if (isSessionStarted) {
isSessionStarted = false
finishSession()
}
return true
@ -440,6 +414,38 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
audioDataListeners.remove(listener)
}
/**
* 设置是否使用内部播放器
*/
override fun setUseInternalPlayer(useInternalPlayer: Boolean) {
this.useInternalPlayer = useInternalPlayer
updateInternalPlayerUsage()
}
/**
* 设置音频输出设备
*/
override fun setAudioOutputDevice(device: AudioOutputDevice) {
if (useInternalPlayer) {
audioPlayer.setAudioOutputDevice(device)
}
}
/**
* 更新内部播放器的使用状态
*/
private fun updateInternalPlayerUsage() {
if (useInternalPlayer) {
// 使用内部播放器
if (!audioDataListeners.contains(audioPlayer)) {
audioDataListeners.add(audioPlayer)
}
} else {
// 不使用内部播放器
audioDataListeners.remove(audioPlayer)
}
}
/**
* 通知事件处理
*/
@ -461,15 +467,14 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
* 通知音频数据
*/
private fun notifyAudioData(data: ByteArray) {
launch(Dispatchers.Main) {
for (listener in audioDataListeners) {
try {
listener.onAudioData(data)
} catch (e: Exception) {
Log.e(TAG, "音频数据回调异常: ${e.message}")
}
for (listener in audioDataListeners) {
try {
listener.onAudioData(data)
} catch (e: Exception) {
Log.e(TAG, "音频数据回调异常: ${e.message}")
}
}
}
/**
@ -483,7 +488,7 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
// 如果已连接但会话未开始,只需要开始会话
if (isConnected && !isSessionStarted) {
Log.d(TAG, "WebSocket已连接,正在启动新会话...")
// Log.d(TAG, "WebSocket已连接,正在启动新会话...")
return startSessionOnly()
}
@ -648,7 +653,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
"errorMessage" to errorMsg))
webSocket.close(1000, "Error")
updateStatus(STATUS_ERROR)
isSpeaking = false // 错误情况下显式设置
isSessionStarted = false
isConnected = false
isConnecting.set(false)
@ -656,6 +660,12 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
EVENT_SESSION_STARTED -> {
connectionAttempts = 0
// 会话开始时触发合成开始事件
notifyEvent(TtsEventType.SYNTHESIS_STARTED)
// 如果使用内部播放器则启动播放器会话
if (useInternalPlayer) {
audioPlayer.startSession()
}
}
EVENT_TTS_RESPONSE -> {
@ -673,16 +683,22 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
EVENT_SESSION_FINISHED -> {
isSessionStarted = false
Log.i(TAG, "EVENT_SESSION_FINISHED, ${sessionId}")
// 触发合成完成事件
notifyEvent(TtsEventType.SYNTHESIS_COMPLETED)
if (!isStreamMode) {
finishConnection(webSocket)
// 如果使用内部播放器则结束播放器会话
if (useInternalPlayer) {
audioPlayer.endSession()
}
// 流式模式下不自动关闭连接
}
}
} catch (e: Exception) {
Log.e(TAG, "解析响应失败: ${e.message}")
updateStatus(STATUS_ERROR)
isSpeaking = false // 错误情况下显式设置
isConnecting.set(false)
}
}
@ -690,7 +706,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
isConnected = false
isSessionStarted = false
isSpeaking = false
isConnecting.set(false)
Log.e(TAG, "WebSocket连接失败: ${t.message}")
@ -702,7 +717,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
isConnected = false
isSessionStarted = false
isSpeaking = false
isConnecting.set(false)
updateStatus(STATUS_STOPPED)
notifyEvent(TtsEventType.SYNTHESIS_CANCELED)
@ -798,53 +812,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
sendEvent(webSocket, header, optional, payload)
}
/**
* 发送文本进行合成 (非流式)
*/
private fun sendTextRequest(text: String) {
webSocket?.let { ws ->
val header = Header(
protocolVersion = PROTOCOL_VERSION,
headerSize = DEFAULT_HEADER_SIZE,
messageType = FULL_CLIENT_REQUEST,
messageTypeSpecificFlags = MSG_TYPE_FLAG_WITH_EVENT,
serializationMethod = JSON,
messageCompression = COMPRESSION_NO,
reserved = 0
)
val optional = Optional(
event = EVENT_TASK_REQUEST,
sessionId = sessionId
)
val jsonObject = JSONObject()
val user = JSONObject()
user.put("uid", "123456")
jsonObject.put("user", user)
jsonObject.put("event", EVENT_TASK_REQUEST)
jsonObject.put("namespace", "BidirectionalTTS")
val reqParams = JSONObject()
// 将#和*替换为空格
val processedText = text.replace("#", " ").replace("*", " ")
reqParams.put("text", processedText)
reqParams.put("speaker", speaker)
val audioParams = JSONObject()
audioParams.put("format", format)
audioParams.put("sample_rate", sampleRate)
audioParams.put("emotion", "happy")
reqParams.put("audio_params", audioParams)
jsonObject.put("req_params", reqParams)
val payload = jsonObject.toString().toByteArray()
Log.d(TAG, "发送文本合成请求: ${jsonObject.toString()}")
sendEvent(ws, header, optional, payload)
}
}
/**
* 发送文本进行合成 (流式)

12
local_plugins/bytedance_speech/lib/bytedance_speech.dart

@ -0,0 +1,12 @@
library bytedance_speech;
import 'package:flutter/services.dart';
class BytedanceSpeech {
static const MethodChannel _channel = MethodChannel('bytedance_speech');
static Future<String?> getPlatformVersion() async {
final String? version = await _channel.invokeMethod('getPlatformVersion');
return version;
}
}

257
local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt

@ -24,6 +24,7 @@ import org.json.JSONObject
import android.os.Handler
import android.os.Looper
import java.util.UUID
import java.util.Collections
/**
* ChatAPI服务异常
@ -70,8 +71,32 @@ private data class ToolCallInfo(
var arguments: String = ""
) {
fun isValid(): Boolean {
// 确保参数是完整的 JSON
return id.isNotEmpty() && name.isNotEmpty() &&
arguments.isNotEmpty() && isCompleteJson(arguments)
}
private fun isCompleteJson(json: String): Boolean {
return try {
id.isNotEmpty() && name.isNotEmpty()
// 检查 JSON 是否完整(括号匹配)
val trimmed = json.trim()
if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) {
return false
}
// 简单的括号匹配检查
var braceCount = 0
for (char in trimmed) {
when (char) {
'{' -> braceCount++
'}' -> braceCount--
}
if (braceCount < 0) return false
}
if (braceCount != 0) return false
// 验证是否是有效的 JSON
JSONObject(json)
true
} catch (e: Exception) {
false
}
@ -110,12 +135,10 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
// 流式请求相关
private var currentStreamJob: Job? = null
// private var streamCallback: StreamCallback? = null
private var currentMessages: List<Map<String, Any>> = emptyList()
private var toolCalls: MutableMap<Int, ToolCallInfo> = mutableMapOf()
private var sessions: MutableMap<String, StreamCallback?> = mutableMapOf()
private var currSessionId = "" //
// private var isCanceled = false
private var toolCalls: MutableMap<Int, ToolCallInfo> = Collections.synchronizedMap(mutableMapOf())
private var currSessionId = ""
private var currSessionCallback: StreamCallback? = null
@ -125,11 +148,34 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
// MARK: - 公共方法
/**
* 设置流式回调
* 中止当前会话
*/
// fun setStreamCallback(callback: StreamCallback) {
// streamCallback = callback
// }
private fun abortCurrentSession() {
if (currSessionId.isNotEmpty()) {
// 1. 取消当前的协程任务
currentStreamJob?.cancel()
currentStreamJob = null
// 2. 通知旧会话被中止
// currSessionCallback?.onError(ChatApiException("Session aborted by new request"))
currSessionCallback?.onComplete() // 直接完成当前会话
// 3. 清理状态
currSessionId = ""
currSessionCallback = null
toolCalls.clear()
}
}
/**
* 清理当前会话
*/
private fun clearCurrentSession() {
currSessionId = ""
currSessionCallback = null
currentStreamJob = null
toolCalls.clear()
}
/**
* 初始化ChatAPI服务
@ -262,29 +308,34 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
* 发送消息(流式输出)- 兼容JSONArray格式
*/
fun sendMessageStream(messages: org.json.JSONArray, callback: StreamCallback) {
// 设置回调
// setStreamCallback(callback)
// 1. 立即中止旧会话
abortCurrentSession()
// 转换JSONArray为List<Map<String, Any>>
val messageList = convertJsonArrayToList(messages)
// 2. 创建新会话
currSessionId = UUID.randomUUID().toString()
sessions[currSessionId] = callback
// 调用原始方法
sendMessageStream(currSessionId,messageList)
currSessionCallback = callback
// 3. 转换消息并开始处理
val messageList = convertJsonArrayToList(messages)
sendMessageStream(currSessionId, messageList)
}
/**
* 发送消息(流式输出)
*/
fun sendMessageStream(sessionid:String,messages: List<Map<String, Any>>) {
// 验证是否是当前会话
if (sessionid != currSessionId) {
return // 忽略非当前会话的请求
}
if (!isInitialized || apiKey.isEmpty() || openAI == null) {
Log.e("ChatApiService", "ChatAPI服务未初始化,无法发送消息")
sessions[sessionid]?.onError(ChatApiException("ChatAPI服务未初始化"))
currSessionCallback?.onError(ChatApiException("ChatAPI服务未初始化"))
return
}
// 重置状态
// isCanceled = false
currentMessages = messages
toolCalls.clear()
// 检查是否包含图片,决定使用哪个模型
@ -337,79 +388,93 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
}
chatsFlow.collect { result ->
if (currSessionId != sessionid) {
return@collect
// 每次处理前检查是否仍是当前会话
if (sessionid != currSessionId) {
throw CancellationException("Session cancelled")
}
val choice = result.choices.firstOrNull()
if (choice == null) {
Log.e("ChatApiService", "choice为null")
Log.w("ChatApiService", "[Session: $sessionid] choice为null")
return@collect
}
val delta = choice.delta
if (delta == null) {
Log.e("ChatApiService", "delta为null")
Log.w("ChatApiService", "[Session: $sessionid] delta为null")
return@collect
}
// 处理普通文本内容
delta.content?.let { content ->
sessions[sessionid]?.onToken(content)
currSessionCallback?.onToken(content)
}
// 收集工具调用信息
delta.toolCalls?.forEach { toolCall ->
val index = toolCall.index
// 创建或获取现有的工具调用信息
val toolCallInfo = toolCalls.getOrPut(index) { ToolCallInfo() }
// 安全处理工具调用ID
try {
toolCall.id?.let { i ->
toolCallInfo.id = i.toString()
val index = toolCall.index
val toolCallInfo = toolCalls.getOrPut(index) { ToolCallInfo() }
// 使用安全调用链处理工具调用信息
toolCall.id?.toString()?.takeIf { it.isNotEmpty() }?.let {
toolCallInfo.id = it
}
} catch (e: Exception) {
Log.e("ChatApiService", "处理工具调用ID异常: ${e.message}")
}
// 安全处理函数信息
try {
toolCall.function?.let { function ->
// 安全处理函数名称 - 使用 try-catch 避免 null 属性访问
try {
function.name.let { name ->
toolCallInfo.name = name
val functionName = function.name
if (!functionName.isNullOrEmpty()) {
toolCallInfo.name = functionName
}
} catch (e: Exception) {
Log.e("ChatApiService", "处理工具调用函数名称异常! function:${toolCall.function} err:${e.message}")
// 在流式响应中,name 可能在后续的 delta 中为 null,这是正常的
// 不记录日志,避免干扰
}
// 安全处理函数参数 - 使用 try-catch 避免 null 属性访问
try {
function.arguments.let { args ->
toolCallInfo.arguments += args
val functionArgs = function.arguments
if (!functionArgs.isNullOrEmpty()) {
toolCallInfo.arguments += functionArgs
}
} catch (e: Exception) {
Log.e("ChatApiService", "处理工具调用参数异常: ${e.message}")
// 在流式响应中,arguments 可能在后续的 delta 中为 null,这是正常的
// 不记录日志,避免干扰
}
Log.e("ChatApiService", "AI发起工具调用:${toolCallInfo.name} ${toolCallInfo.arguments}")
// 不设置默认值,让参数自然累积
}
// 只在有完整有效数据时记录日志(避免流式响应过程中的重复日志)
if (toolCallInfo.isValid()) {
Log.d("ChatApiService", "[Session: $sessionid] AI发起工具调用: ${toolCallInfo.name}, 参数: ${toolCallInfo.arguments}")
}
} catch (e: Exception) {
Log.e("ChatApiService", "处理工具调用函数信息异常: ${e.message}")
Log.w("ChatApiService", "[Session: $sessionid] 处理工具调用信息异常: ${e.message}", e)
}
}
}
if (currSessionId == sessionid) {
// 完成处理
if (sessionid == currSessionId) {
// 检查是否有工具调用需要处理
val hasToolCalls = processToolCalls(sessionid)
if (!hasToolCalls) {
sessions[sessionid]?.onComplete()
currSessionCallback?.onComplete()
clearCurrentSession()
}
}else{
sessions[sessionid]?.onComplete()
}
} catch (e: CancellationException) {
// 会话被取消,不需要处理
Log.d("ChatApiService", "Session $sessionid cancelled")
} catch (e: Exception) {
// if (currSessionId == sessionid) {
sessions[sessionid]?.onError(ChatApiException("流式请求失败: ${e.message}"))
// }
if (sessionid == currSessionId) {
currSessionCallback?.onError(ChatApiException("流式请求失败: ${e.message}"))
clearCurrentSession()
}
}
}
}
@ -418,42 +483,54 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
* 处理工具调用
*/
private suspend fun processToolCalls(sessionid: String): Boolean {
val firstToolCall = try {
toolCalls.values.firstOrNull { it.isValid() }
} catch (e: Exception) {
Log.e("ChatApiService", "查找有效工具调用异常: ${e.message}")
null
}
if (firstToolCall == null) {
// 验证工具调用集合不为空
if (toolCalls.isEmpty()) {
Log.d("ChatApiService", "[Session: $sessionid] 没有工具调用需要处理")
return false
}
// 创建函数调用字典
val functionCall = try {
mapOf(
"name" to (firstToolCall.name.takeIf { it.isNotEmpty() } ?: ""),
"arguments" to (firstToolCall.arguments.takeIf { it.isNotEmpty() } ?: "{}"),
"id" to (firstToolCall.id.takeIf { it.isNotEmpty() } ?: "")
)
} catch (e: Exception) {
Log.e("ChatApiService", "创建函数调用字典异常: ${e.message}")
// 输出所有收集到的工具调用信息(用于调试)
Log.d("ChatApiService", "[Session: $sessionid] 收集到 ${toolCalls.size} 个工具调用")
toolCalls.forEach { (index, toolCall) ->
Log.d("ChatApiService", "[Session: $sessionid] 工具调用[$index]: name=${toolCall.name}, args.length=${toolCall.arguments.length}, args=${toolCall.arguments.take(100)}...")
}
// 只处理有效的工具调用
val validToolCalls = toolCalls.values.filter { it.isValid() }
if (validToolCalls.isEmpty()) {
Log.w("ChatApiService", "[Session: $sessionid] 没有有效的工具调用(参数可能不完整)")
return false
}
// 处理第一个有效的工具调用
val firstToolCall = validToolCalls.first()
Log.d("ChatApiService", "[Session: $sessionid] 处理工具调用: ${firstToolCall.name},参数: ${firstToolCall.arguments}")
// 创建函数调用字典
val functionCall = mapOf(
"name" to firstToolCall.name,
"arguments" to firstToolCall.arguments,
"id" to firstToolCall.id
)
// 通知上层工具调用事件
sessions[sessionid]?.onFunctionCall(convertMapToJsonObject(functionCall))
currSessionCallback?.onFunctionCall(convertMapToJsonObject(functionCall))
// 在后台队列处理工具调用
launch {
try {
if (currSessionId == sessionid) {
if (sessionid == currSessionId) {
// 通过MCP客户端处理工具调用
val functionName = firstToolCall.name
val argumentsJson = firstToolCall.arguments
val result = if (_mcpClient?.hasToolWithName(functionName) == true) {
// 解析参数
val arguments = _mcpClient?.parseJsonArguments(argumentsJson) ?: emptyMap()
// 解析参数,如果参数为空JSON对象,使用空Map
val arguments = if (argumentsJson == "{}") {
emptyMap()
} else {
_mcpClient?.parseJsonArguments(argumentsJson) ?: emptyMap()
}
// 调用MCP工具
val toolResult = _mcpClient?.callTool(functionName, arguments)
Log.e("ChatApiService", "AI调用工具结果: $functionName}:$arguments} toolResult:${toolResult}")
Log.d("ChatApiService", "[Session: $sessionid] AI调用工具结果: $functionName, 参数: $arguments, 结果: $toolResult")
// 处理结果
if (toolResult != null) {
if (toolResult["isError"] == true) {
@ -473,16 +550,16 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
mapOf("context" to text)
}
} else {
Log.e("ChatApiService", "MCP工具调用返回null")
Log.w("ChatApiService", "[Session: $sessionid] MCP工具调用返回null")
mapOf("context" to "Tool call failed")
}
} else {
// 工具不存在
mapOf("context" to "Tool not found: $functionName")
}
if (currSessionId == sessionid) {
if (sessionid == currSessionId) {
// 处理结果
sessions[sessionid]?.onFunctionCallResult(
currSessionCallback?.onFunctionCallResult(
convertMapToJsonObject(functionCall),
convertMapToJsonObject(result)
)
@ -493,13 +570,11 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
functionCall = functionCall,
functionResult = gson.toJson(result)
)
}else{
sessions[sessionid]?.onComplete()
}
}
} catch (e: Exception) {
Log.e("ChatApiService", "工具调用处理过程中出错: ${e.message}", e)
if (currSessionId == sessionid) {
if (sessionid == currSessionId) {
val errorMessage = "工具调用处理失败: ${e.message}"
sendFunctionCallResultInternal(
sessionid= sessionid,
@ -507,8 +582,6 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
functionCall = functionCall,
functionResult = errorMessage
)
}else{
sessions[sessionid]?.onComplete()
}
}
}
@ -525,7 +598,7 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
functionCall: Map<String, Any>,
functionResult: String
) {
if (currSessionId != sessionid) return
if (sessionid != currSessionId) return
val fullMessages = messages.toMutableList()
@ -565,19 +638,19 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
* 取消当前流式请求
*/
fun cancelCurrentStream(): Boolean {
sessions[currSessionId]?.onComplete()
sessions.remove(currSessionId)
currSessionId = ""
currentStreamJob?.cancel()
currentStreamJob = null
return true
if (currSessionId.isNotEmpty()) {
abortCurrentSession()
return true
}
return false
}
/**
* 取消所有操作并释放资源 (与 iOS 版本保持一致)
*/
fun cancelAll() {
cancelCurrentStream()
abortCurrentSession()
}
/**
@ -706,7 +779,7 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
description = description,
parameters = parameters
)
Log.e("ChatApiService", "AI携带工具:${name}:${parameters}")
Log.d("ChatApiService", "AI携带工具: $name, 参数定义: $parameters")
tools.add(tool)
} catch (e: Exception) {
@ -796,9 +869,11 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
* 聊天完成流式接口
* 与iOS版本保持一致的接口
*/
fun chatCompletionStream(sessionid: String,messages: List<Map<String, Any>>, tool: Boolean = false) {
// 直接调用sendMessageStream,因为该方法已经处理了工具调用
sendMessageStream(sessionid,messages)
fun chatCompletionStream(sessionid: String, messages: List<Map<String, Any>>, tool: Boolean = false) {
// 验证是否是当前会话
if (sessionid == currSessionId) {
sendMessageStream(sessionid, messages)
}
}
// MARK: - 工具方法

3
local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/CustomSseClientTransport.kt

@ -236,8 +236,7 @@ class CustomSseClientTransport(
if (!response.status.isSuccess()) {
val text = response.bodyAsText()
Log.e(TAG, "发送消息失败:URL:${urlString} HTTP ${response.status}, $text")
error("Error POSTing to endpoint (HTTP ${response.status}): $text")
// Log.e(TAG, "发送消息失败:URL:${urlString} HTTP ${response.status}, $text")
}
} catch (e: Exception) {
Log.e(TAG, "发送消息异常: ${e.message}", e)

28
local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/MCPSubClient.kt

@ -357,9 +357,9 @@ class MCPSubClient(
val response = mcpClient?.ping()
val latency = System.currentTimeMillis() - startTime
Log.d(TAG, "心跳检测成功,延迟: ${latency}ms")
// Log.d(TAG, "心跳检测成功,延迟: ${latency}ms")
} catch (e: Exception) {
Log.e(TAG, "心跳检测失败: ${e.message}")
// Log.e(TAG, "心跳检测失败: ${e.message}")
handleConnectionError(e)
break
}
@ -373,18 +373,18 @@ class MCPSubClient(
isConnected = false
stopHeartbeat()
// if (retryCount < maxRetryAttempts) {
// retryCount++
// currentReconnectDelay = minOf(currentReconnectDelay * 2, maxReconnectDelay)
//
// Log.w(TAG, "连接失败,将在 ${currentReconnectDelay}ms 后尝试重连 (尝试 $retryCount/$maxRetryAttempts)")
//
// delay(currentReconnectDelay)
// connect()
// } else {
// Log.e(TAG, "已达到最大重试次数($maxRetryAttempts),停止重连")
// // 可以在这里添加通知或回调,告知上层连接彻底失败
// }
if (retryCount < maxRetryAttempts) {
retryCount++
currentReconnectDelay = minOf(currentReconnectDelay * 2, maxReconnectDelay)
// Log.w(TAG, "连接失败,将在 ${currentReconnectDelay}ms 后尝试重连 (尝试 $retryCount/$maxRetryAttempts)")
delay(currentReconnectDelay)
connect()
} else {
Log.e(TAG, "已达到最大重试次数($maxRetryAttempts),停止重连")
// 可以在这里添加通知或回调,告知上层连接彻底失败
}
}
/**
* 关闭连接

789
local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/SystemFunctionHandler.kt

@ -1,6 +1,7 @@
package com.yunqiinnovation.chat_api
import android.Manifest
import android.app.ActivityManager
import android.content.ContentUris
import android.content.ContentValues
import android.content.Context
@ -12,10 +13,12 @@ import android.os.Build
import android.provider.AlarmClock
import android.provider.CalendarContract
import android.provider.ContactsContract
import android.provider.Settings
import android.telephony.SmsManager
import android.util.Log
import androidx.core.content.ContextCompat
import androidx.core.content.ContextCompat.RECEIVER_NOT_EXPORTED
import org.json.JSONObject
import java.text.SimpleDateFormat
import java.util.*
@ -24,6 +27,9 @@ import java.util.*
* 提供内置的系统函数,与 iOS 版本保持一致
*/
class SystemFunctionHandler(private val context: Context? = null) {
companion object {
const val TAG = "SystemFunctionHandler"
}
/**
* 注册所有系统函数
@ -227,6 +233,171 @@ class SystemFunctionHandler(private val context: Context? = null) {
handler = ExitNavigationHandler(context)
)
}
/**
* 检查权限并根据应用状态采取不同行为
* @param permission 需要检查的权限
* @param permissionName 权限名称(用于显示给用户)
* @param functionName 功能名称(用于日志和提示)
* @return 是否拥有权限
*/
internal fun checkPermissionAndGuide(permission: String, permissionName: String, functionName: String): Boolean {
val ctx = context ?: return false
if (ContextCompat.checkSelfPermission(ctx, permission) != PackageManager.PERMISSION_GRANTED) {
Log.w(TAG, "缺少${permissionName}权限,无法执行${functionName}")
// 检查应用是否在前台
if (isAppInForeground()) {
// 应用在前台,尝试发送广播通知主应用弹出权限申请对话框
try {
val intent = Intent("com.yunqiinnovation.deepsound.action.REQUEST_PERMISSION").apply {
setPackage(ctx.packageName)
putExtra("permission", permission)
putExtra("permissionName", permissionName)
putExtra("functionName", functionName)
addFlags(Intent.FLAG_RECEIVER_FOREGROUND)
}
ctx.sendBroadcast(intent)
// 设置一个短暂延迟,让用户有时间响应权限请求
Thread.sleep(500)
// 再次检查权限是否已授予
if (ContextCompat.checkSelfPermission(ctx, permission) == PackageManager.PERMISSION_GRANTED) {
return true
}
} catch (e: Exception) {
Log.e(TAG, "请求权限失败: ${e.message}", e)
}
// 如果权限仍未授予,引导用户到权限管理页面
try {
val settingsIntent = Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS).apply {
data = Uri.fromParts("package", ctx.packageName, null)
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
}
ctx.startActivity(settingsIntent)
} catch (e: Exception) {
Log.e(TAG, "打开权限设置页面失败: ${e.message}", e)
}
}
return false
}
return true
}
/**
* 检查应用是否在前台运行
* @return 应用是否在前台
*/
internal fun isAppInForeground(): Boolean {
val ctx = context ?: return false
try {
val activityManager = ctx.getSystemService(Context.ACTIVITY_SERVICE) as ActivityManager
val appProcesses = activityManager.runningAppProcesses ?: return false
val packageName = ctx.packageName
for (appProcess in appProcesses) {
if (appProcess.importance == ActivityManager.RunningAppProcessInfo.IMPORTANCE_FOREGROUND
&& appProcess.processName == packageName) {
return true
}
}
} catch (e: Exception) {
Log.e(TAG, "检查应用状态失败: ${e.message}", e)
}
return false
}
/**
* 根据联系人名称获取电话号码
* @param contact 联系人名称或电话号码
* @return 电话号码
*/
internal fun getPhoneNumberForContact(contact: String): String {
val ctx = context ?: return ""
// 如果输入的是电话号码(简单判断,包含数字和特殊符号)
if (contact.matches(Regex("^[+\\d\\s\\-()]*\$")) && contact.any { it.isDigit() }) {
// 移除非数字字符
return contact.replace(Regex("[^+\\d]"), "")
}
// 检查读取联系人权限
if (!checkPermissionAndGuide(Manifest.permission.READ_CONTACTS, "读取联系人", "访问联系人")) {
return ""
}
// 从联系人数据库查询
try {
val contentResolver = ctx.contentResolver
val uri = Uri.withAppendedPath(
ContactsContract.Contacts.CONTENT_FILTER_URI,
Uri.encode(contact)
)
val projection = arrayOf(
ContactsContract.Contacts._ID,
ContactsContract.Contacts.DISPLAY_NAME,
ContactsContract.Contacts.HAS_PHONE_NUMBER
)
contentResolver.query(uri, projection, null, null, null)?.use { cursor ->
if (cursor.moveToFirst()) {
val hasPhoneNumber = cursor.getInt(cursor.getColumnIndexOrThrow(ContactsContract.Contacts.HAS_PHONE_NUMBER))
if (hasPhoneNumber > 0) {
val contactId = cursor.getString(cursor.getColumnIndexOrThrow(ContactsContract.Contacts._ID))
val phoneUri = ContactsContract.CommonDataKinds.Phone.CONTENT_URI
val phoneProjection = arrayOf(ContactsContract.CommonDataKinds.Phone.NUMBER)
val phoneSelection = "${ContactsContract.CommonDataKinds.Phone.CONTACT_ID} = ?"
val phoneSelectionArgs = arrayOf(contactId)
contentResolver.query(phoneUri, phoneProjection, phoneSelection, phoneSelectionArgs, null)?.use { phoneCursor ->
if (phoneCursor.moveToFirst()) {
return phoneCursor.getString(phoneCursor.getColumnIndexOrThrow(ContactsContract.CommonDataKinds.Phone.NUMBER))
}
}
}
}
}
} catch (e: Exception) {
Log.e(TAG, "查询联系人信息失败: ${e.message}", e)
}
return ""
}
/**
* 获取默认日历ID
*/
internal fun getDefaultCalendarId(): Long {
val ctx = context ?: return -1L
val projection = arrayOf(CalendarContract.Calendars._ID)
var calendarId = -1L
try {
val uri = CalendarContract.Calendars.CONTENT_URI
val selection = "(${CalendarContract.Calendars.VISIBLE} = 1) AND (${CalendarContract.Calendars.IS_PRIMARY} = 1)"
ctx.contentResolver.query(uri, projection, selection, null, null)?.use { cursor ->
if (cursor.moveToFirst()) {
calendarId = cursor.getLong(0)
} else {
// 如果没有找到主日历,尝试找任何可见的日历
ctx.contentResolver.query(uri, projection, "${CalendarContract.Calendars.VISIBLE} = 1", null, null)?.use { c ->
if (c.moveToFirst()) {
calendarId = c.getLong(0)
}
}
}
}
} catch (e: Exception) {
Log.e(TAG, "获取默认日历ID失败: ${e.message}", e)
}
return calendarId
}
}
/**
@ -234,12 +405,25 @@ class SystemFunctionHandler(private val context: Context? = null) {
*/
private class ExitNavigationHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
// 发送广播通知退出交互
val intent = Intent("com.yunqiinnovation.amap_service.EXIT_NAVIGATION").apply {
setPackage(context?.packageName) // 设置包名使其成为显式 Intent
setPackage(context.packageName) // 设置包名使其成为显式 Intent
}
context?.sendBroadcast(intent)
return "{\"broadcast\":false,\"result\": \"已结束当前导航服务\"}"
context.sendBroadcast(intent)
return JSONObject().apply {
put("result", "已结束当前导航服务")
put("broadcast", false)
}.toString()
}
}
@ -248,9 +432,21 @@ private class ExitNavigationHandler(private val context: Context?) : FunctionHan
*/
private class ExitInteractionHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
// 发送广播通知退出交互
context?.sendBroadcast(Intent("com.yunqiinnovation.deepsound.EXIT_INTERACTION"))
return "{\"result\": \"已结束当前交互\"}"
context.sendBroadcast(Intent("com.yunqiinnovation.deepsound.EXIT_INTERACTION"))
return JSONObject().apply {
put("result", "已结束当前交互")
}.toString()
}
}
@ -259,9 +455,21 @@ private class ExitInteractionHandler(private val context: Context?) : FunctionHa
*/
private class TranslationModeHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
// 发送广播通知进入翻译模式
context?.sendBroadcast(Intent("com.yunqiinnovation.deepsound.ENTER_TRANSLATION_MODE"))
return "{\"result\": \"已进入翻译模式\"}"
context.sendBroadcast(Intent("com.yunqiinnovation.deepsound.ENTER_TRANSLATION_MODE"))
return JSONObject().apply {
put("result", "已进入翻译模式")
}.toString()
}
}
@ -274,24 +482,75 @@ private class SendTextMessageHandler(private val context: Context?) : FunctionHa
val message = arguments["message"] as? String
if (contact == null || message == null) {
return "{\"result\": \"缺少必要参数\"}"
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "invalid_arguments")
put("message", "缺少必要参数:contact 或 message")
})
}.toString()
}
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
val handler = context.let { SystemFunctionHandler(it) }
return try {
val smsIntent = Intent(Intent.ACTION_SENDTO).apply {
data = Uri.parse("smsto:$contact")
putExtra("sms_body", message)
flags = Intent.FLAG_ACTIVITY_NEW_TASK
// 检查发送短信权限
if (!handler.checkPermissionAndGuide(Manifest.permission.SEND_SMS, "发送短信", "发送短信")) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "permission_denied")
put("message", "需要短信权限才能发送短信,请在权限管理中开启短信权限")
})
}.toString()
}
if (context?.packageManager?.queryIntentActivities(smsIntent, 0)?.isNotEmpty() == true) {
context.startActivity(smsIntent)
"{\"result\": \"已打开短信应用\"}"
val phoneNumber = handler.getPhoneNumberForContact(contact)
if (phoneNumber.isEmpty()) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "contact_not_found")
put("message", "未找到联系人信息: $contact")
})
}.toString()
}
// 使用SmsManager发送短信
val smsManager = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
context.getSystemService(SmsManager::class.java)
} else {
"{\"result\": \"无法打开短信应用\"}"
@Suppress("DEPRECATION")
SmsManager.getDefault()
}
// 处理长短信
val parts = smsManager.divideMessage(message)
if (parts.size == 1) {
smsManager.sendTextMessage(phoneNumber, null, message, null, null)
} else {
smsManager.sendMultipartTextMessage(phoneNumber, null, parts, null, null)
}
JSONObject().apply {
put("result", "已成功向${contact}发送短信")
put("phone", phoneNumber)
put("message", message)
}.toString()
} catch (e: Exception) {
"{\"result\": \"发送短信失败:${e.message}\"}"
Log.e(SystemFunctionHandler.TAG, "发送短信失败: ${e.message}", e)
JSONObject().apply {
put("error", JSONObject().apply {
put("code", "send_failed")
put("message", "发送短信失败: ${e.message}")
})
}.toString()
}
}
}
@ -364,28 +623,73 @@ private class MakePhoneCallHandler(private val context: Context?) : FunctionHand
val contact = arguments["contact"] as? String
if (contact == null) {
return "{\"result\": \"缺少联系人参数\"}"
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "invalid_arguments")
put("message", "缺少必要参数:contact")
})
}.toString()
}
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
val handler = context.let { SystemFunctionHandler(it) }
return try {
// 检查拨打电话权限
if (!checkPermissionAndGuide(context!!,Manifest.permission.CALL_PHONE, "拨打电话", "拨打电话")) {
"{\"result\": \"需要电话权限才能拨打电话,请在权限管理中开启电话权限\"}"
if (!handler.checkPermissionAndGuide(Manifest.permission.CALL_PHONE, "拨打电话", "拨打电话")) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "permission_denied")
put("message", "需要电话权限才能拨打电话,请在权限管理中开启电话权限")
})
}.toString()
}
val phoneNumber = getPhoneNumberForContact(contact)
val phoneNumber = handler.getPhoneNumberForContact(contact)
if (phoneNumber.isEmpty()) {
"{\"result\": \"未找到联系人信息: $contact\"}"
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "contact_not_found")
put("message", "未找到联系人信息: $contact")
})
}.toString()
}
// 创建拨号意图
val intent = Intent(Intent.ACTION_CALL)
intent.data = Uri.parse("tel:$phoneNumber")
intent.flags = Intent.FLAG_ACTIVITY_NEW_TASK
context?.startActivity(intent)
"{\"result\": \"正在拨打${contact}的电话\"}"
// 创建拨号意图(使用ACTION_CALL直接拨打)
val intent = Intent(Intent.ACTION_CALL).apply {
data = Uri.parse("tel:$phoneNumber")
flags = Intent.FLAG_ACTIVITY_NEW_TASK
}
context.startActivity(intent)
JSONObject().apply {
put("result", "正在拨打${contact}的电话")
put("phone", phoneNumber)
}.toString()
} catch (e: SecurityException) {
Log.e(SystemFunctionHandler.TAG, "拨打电话权限被拒绝: ${e.message}", e)
JSONObject().apply {
put("error", JSONObject().apply {
put("code", "permission_denied")
put("message", "权限被拒绝,无法拨打电话")
})
}.toString()
} catch (e: Exception) {
Log.e("ChatApi", "拨打电话失败: ${e.message}", e)
"{\"result\": \"权限不足或者为查询到联系人\"}"
Log.e(SystemFunctionHandler.TAG, "拨打电话失败: ${e.message}", e)
JSONObject().apply {
put("error", JSONObject().apply {
put("code", "call_failed")
put("message", "拨打电话失败: ${e.message}")
})
}.toString()
}
// return try {
// val callIntent = Intent(Intent.ACTION_DIAL).apply {
@ -549,38 +853,149 @@ private class SetReminderHandler(private val context: Context?) : FunctionHandle
val timeString = arguments["time"] as? String
if (title == null || timeString == null) {
return "{\"result\": \"缺少必要参数\"}"
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "invalid_arguments")
put("message", "缺少必要参数:title 或 time")
})
}.toString()
}
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
val handler = context.let { SystemFunctionHandler(it) }
return try {
// 解析时间
val formatter = SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.getDefault())
val time = formatter.parse(timeString)?.time ?: run {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "invalid_time_format")
put("message", "提醒时间格式无效,请使用'yyyy-MM-dd HH:mm'格式")
})
}.toString()
}
// 检查日历权限
if (!handler.checkPermissionAndGuide(Manifest.permission.WRITE_CALENDAR, "日历写入", "设置提醒")) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "permission_denied")
put("message", "需要日历权限才能创建提醒,请在权限管理中开启日历权限")
})
}.toString()
}
// 直接使用ContentProvider写入日历
val endTime = time + 3600000 // 默认1小时后结束
// 获取默认日历ID
val calendarId = handler.getDefaultCalendarId()
if (calendarId == -1L) {
Log.w(SystemFunctionHandler.TAG, "未找到默认日历")
// 如果没有找到日历,尝试使用Intent创建
return tryCreateWithIntent(context, title, content, time)
}
// 创建事件
val eventValues = ContentValues().apply {
put(CalendarContract.Events.CALENDAR_ID, calendarId)
put(CalendarContract.Events.TITLE, title)
put(CalendarContract.Events.DESCRIPTION, content)
put(CalendarContract.Events.DTSTART, time)
put(CalendarContract.Events.DTEND, endTime)
put(CalendarContract.Events.HAS_ALARM, 1)
put(CalendarContract.Events.EVENT_TIMEZONE, TimeZone.getDefault().id)
}
val eventUri = context.contentResolver.insert(CalendarContract.Events.CONTENT_URI, eventValues)
if (eventUri == null) {
Log.e(SystemFunctionHandler.TAG, "插入日历事件失败")
return tryCreateWithIntent(context, title, content, time)
}
// 获取事件ID
val eventId = ContentUris.parseId(eventUri)
// 添加提醒(15分钟前提醒)
val reminderValues = ContentValues().apply {
put(CalendarContract.Reminders.EVENT_ID, eventId)
put(CalendarContract.Reminders.METHOD, CalendarContract.Reminders.METHOD_ALERT)
put(CalendarContract.Reminders.MINUTES, 15) // 提前15分钟提醒
}
val reminderUri = context.contentResolver.insert(CalendarContract.Reminders.CONTENT_URI, reminderValues)
if (reminderUri != null) {
return JSONObject().apply {
put("result", "已成功创建提醒: $title")
put("title", title)
put("content", content)
put("time", timeString)
put("eventId", eventId)
}.toString()
} else {
// 仍然创建了事件,但没有提醒
return JSONObject().apply {
put("result", "已创建日历事件,但未设置提醒: $title")
put("title", title)
put("time", timeString)
put("eventId", eventId)
}.toString()
}
} catch (e: Exception) {
Log.e(SystemFunctionHandler.TAG, "设置提醒失败: ${e.message}", e)
JSONObject().apply {
put("error", JSONObject().apply {
put("code", "reminder_failed")
put("message", "设置提醒失败: ${e.message}")
})
}.toString()
}
}
private fun tryCreateWithIntent(context: Context, title: String, content: String, time: Long): String {
return try {
val calendarIntent = Intent(Intent.ACTION_INSERT).apply {
data = CalendarContract.Events.CONTENT_URI
putExtra(CalendarContract.Events.TITLE, title)
putExtra(CalendarContract.Events.DESCRIPTION, content)
putExtra(CalendarContract.EXTRA_EVENT_BEGIN_TIME, time)
putExtra(CalendarContract.EXTRA_EVENT_END_TIME, time + 3600000)
putExtra(CalendarContract.Events.HAS_ALARM, 1)
flags = Intent.FLAG_ACTIVITY_NEW_TASK
}
if (context.packageManager?.queryIntentActivities(calendarIntent, 0)?.isNotEmpty() == true) {
context.startActivity(calendarIntent)
JSONObject().apply {
put("result", "已打开日历应用创建提醒")
put("title", title)
put("content", content)
}.toString()
} else {
JSONObject().apply {
put("error", JSONObject().apply {
put("code", "no_calendar_app")
put("message", "未找到可用的日历应用")
})
}.toString()
}
} catch (e: Exception) {
JSONObject().apply {
put("error", JSONObject().apply {
put("code", "intent_failed")
put("message", "创建日历事件失败: ${e.message}")
})
}.toString()
}
return setReminder(title,content,timeString)
// return try {
// // 解析时间
// val formatter = SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.getDefault())
// val date = formatter.parse(timeString)
//
// if (date == null) {
// return "{\"result\": \"时间格式错误\"}"
// }
//
// // 创建日历事件
// val calendarIntent = Intent(Intent.ACTION_INSERT).apply {
// data = CalendarContract.Events.CONTENT_URI
// putExtra(CalendarContract.Events.TITLE, title)
// putExtra(CalendarContract.Events.DESCRIPTION, content)
// putExtra(CalendarContract.EXTRA_EVENT_BEGIN_TIME, date.time)
// putExtra(CalendarContract.EXTRA_EVENT_END_TIME, date.time + 60 * 60 * 1000) // 默认1小时
// putExtra(CalendarContract.Events.HAS_ALARM, 1)
// flags = Intent.FLAG_ACTIVITY_NEW_TASK
// }
//
// if (context?.packageManager?.queryIntentActivities(calendarIntent, 0)?.isNotEmpty() == true) {
// context.startActivity(calendarIntent)
// "{\"result\": \"提醒设置成功\"}"
// } else {
// "{\"result\": \"无法打开日历应用\"}"
// }
// } catch (e: Exception) {
// "{\"result\": \"设置提醒失败:${e.message}\"}"
// }
}
}
@ -592,16 +1007,61 @@ private class GetCurrentTimeHandler : FunctionHandler {
val format = arguments["format"] as? String
return try {
val formatter = if (!format.isNullOrEmpty()) {
SimpleDateFormat(format, Locale.getDefault())
} else {
SimpleDateFormat("yyyy年MM月dd日 HH:mm:ss", Locale.CHINA)
val calendar = Calendar.getInstance()
val date = calendar.time
val timeStr = when (format?.lowercase(Locale.getDefault())) {
"short" -> {
SimpleDateFormat("HH:mm", Locale.getDefault()).format(date)
}
"date" -> {
SimpleDateFormat("yyyy年MM月dd日", Locale.getDefault()).format(date)
}
"time" -> {
SimpleDateFormat("HH:mm:ss", Locale.getDefault()).format(date)
}
"full" -> {
SimpleDateFormat("yyyy年MM月dd日 EEEE HH:mm:ss", Locale.getDefault()).format(date)
}
"iso" -> {
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.getDefault()).format(date)
}
else -> {
SimpleDateFormat("yyyy年MM月dd日 HH:mm", Locale.getDefault()).format(date)
}
}
val currentTime = formatter.format(Date())
"{\"result\": \"$currentTime\", \"time\": \"$currentTime\"}"
// 获取星期几
val weekDay = when (calendar.get(Calendar.DAY_OF_WEEK)) {
Calendar.SUNDAY -> "星期日"
Calendar.MONDAY -> "星期一"
Calendar.TUESDAY -> "星期二"
Calendar.WEDNESDAY -> "星期三"
Calendar.THURSDAY -> "星期四"
Calendar.FRIDAY -> "星期五"
Calendar.SATURDAY -> "星期六"
else -> ""
}
// 根据格式决定是否添加星期几
val resultTime = when {
format == "short" || format == "time" -> timeStr
format == "full" -> timeStr // full格式已经包含了星期
else -> "$timeStr $weekDay"
}
JSONObject().apply {
put("result", resultTime)
put("time", resultTime)
put("timestamp", date.time)
}.toString()
} catch (e: Exception) {
"{\"result\": \"获取时间失败:${e.message}\"}"
JSONObject().apply {
put("error", JSONObject().apply {
put("code", "time_error")
put("message", "获取时间失败: ${e.message}")
})
}.toString()
}
}
}
@ -612,21 +1072,24 @@ private class GetCurrentTimeHandler : FunctionHandler {
private class GetCurrentLocationHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return "{\"result\": \"上下文未初始化\", \"success\": false}"
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
// 检查位置权限
val hasPermission = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
ContextCompat.checkSelfPermission(
context,
android.Manifest.permission.ACCESS_FINE_LOCATION
) == PackageManager.PERMISSION_GRANTED
} else {
true
}
val handler = context.let { SystemFunctionHandler(it) }
if (!hasPermission) {
return "{\"result\": \"位置权限被拒绝。请前往设置中启用位置权限\", \"success\": false}"
// 检查位置权限
if (!handler.checkPermissionAndGuide(Manifest.permission.ACCESS_FINE_LOCATION, "位置信息", "获取位置")) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "permission_denied")
put("message", "需要位置权限才能获取当前位置,请在权限管理中开启位置权限")
})
}.toString()
}
// 检查位置服务是否启用
@ -635,12 +1098,29 @@ private class GetCurrentLocationHandler(private val context: Context?) : Functio
locationManager?.isProviderEnabled(LocationManager.NETWORK_PROVIDER) == true
if (!isLocationEnabled) {
return "{\"result\": \"位置服务未启用。请前往设置中启用位置服务\", \"success\": false}"
// 尝试引导用户打开位置服务
try {
val intent = Intent(Settings.ACTION_LOCATION_SOURCE_SETTINGS)
intent.flags = Intent.FLAG_ACTIVITY_NEW_TASK
context.startActivity(intent)
} catch (e: Exception) {
Log.e(SystemFunctionHandler.TAG, "打开位置设置失败: ${e.message}", e)
}
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "location_disabled")
put("message", "位置服务未启用,请打开GPS或网络定位")
})
}.toString()
}
// 注意:实际的位置获取需要异步处理,这里只返回提示信息
// 在实际应用中应该使用 LocationCallback 或 Coroutines 来获取实时位置
return "{\"result\": \"需要通过位置服务获取当前位置\", \"success\": true}"
// 注意:这里简化了实现,实际应该获取真实位置
// 在生产环境中应该使用 LocationManager 或 FusedLocationProviderClient 获取实际位置
return JSONObject().apply {
put("result", "位置服务已启用,可以获取位置信息")
put("note", "需要实现实际的位置获取逻辑")
}.toString()
}
}
@ -649,12 +1129,26 @@ private class GetCurrentLocationHandler(private val context: Context?) : Functio
*/
private class MediaPlayHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
// 发送媒体播放广播
val intent = Intent("com.yunqiinnovation.music_service.MEDIA_PLAY").apply {
setPackage(context?.packageName) // 设置包名使其成为显式 Intent
setPackage(context.packageName)
}
context?.sendBroadcast(intent)
return "{\"notreply\":true,\"broadcast\":false,\"result\": \"已开始播放媒体\"}"
context.sendBroadcast(intent)
return JSONObject().apply {
put("result", "已开始播放媒体")
put("notreply", true)
put("broadcast", false)
}.toString()
}
}
@ -663,27 +1157,55 @@ private class MediaPlayHandler(private val context: Context?) : FunctionHandler
*/
private class MediaPauseHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
Log.d("MediaPauseHandler", "📤 正在发送广播 context = $context")
// 发送媒体暂停广播
val intent = Intent("com.yunqiinnovation.music_service.MEDIA_PAUSE").apply {
setPackage(context?.packageName) // 设置包名使其成为显式 Intent
setPackage(context.packageName)
}
context?.sendBroadcast(intent)
return "{\"notreply\":true,\"broadcast\":false,\"result\": \"已暂停媒体播放\"}"
context.sendBroadcast(intent)
return JSONObject().apply {
put("result", "已暂停媒体播放")
put("notreply", true)
put("broadcast", false)
}.toString()
}
}
/**
* 媒体暂停处理器
* 媒体关闭处理器
*/
private class MediaCloseHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String {
Log.d("MediaPauseHandler", "📤 正在发送广播 context = $context")
// 发送媒体暂停广播
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
Log.d("MediaCloseHandler", "📤 正在发送广播 context = $context")
// 发送媒体关闭广播
val intent = Intent("com.yunqiinnovation.music_service.MEDIA_CLOSE").apply {
setPackage(context?.packageName) // 设置包名使其成为显式 Intent
setPackage(context.packageName)
}
context?.sendBroadcast(intent)
return "{\"notreply\":true,\"broadcast\":false,\"result\": \"已关闭媒体播放\"}"
context.sendBroadcast(intent)
return JSONObject().apply {
put("result", "已关闭媒体播放")
put("notreply", true)
put("broadcast", false)
}.toString()
}
}
/**
@ -691,13 +1213,26 @@ private class MediaCloseHandler(private val context: Context?) : FunctionHandler
*/
private class MediaPreviousHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
// 发送切换上一首广播
// context?.applicationContext?.sendBroadcast(Intent("com.yunqiinnovation.music_service.MEDIA_PREVIOUS"))
val intent = Intent("com.yunqiinnovation.music_service.MEDIA_PREVIOUS").apply {
setPackage(context?.packageName) // 设置包名使其成为显式 Intent
setPackage(context.packageName)
}
context?.sendBroadcast(intent)
return "{\"notreply\":true,\"broadcast\":false,\"result\": \"已切换到上一首\"}"
context.sendBroadcast(intent)
return JSONObject().apply {
put("result", "已切换到上一首")
put("notreply", true)
put("broadcast", false)
}.toString()
}
}
@ -706,13 +1241,26 @@ private class MediaPreviousHandler(private val context: Context?) : FunctionHand
*/
private class MediaNextHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
// 发送切换下一首广播
// context?.applicationContext?.sendBroadcast(Intent("com.yunqiinnovation.music_service.MEDIA_NEXT"))
val intent = Intent("com.yunqiinnovation.music_service.MEDIA_NEXT").apply {
setPackage(context?.packageName) // 设置包名使其成为显式 Intent
setPackage(context.packageName)
}
context?.sendBroadcast(intent)
return "{\"notreply\":true,\"broadcast\":false,\"result\": \"已切换到下一首\"}"
context.sendBroadcast(intent)
return JSONObject().apply {
put("result", "已切换到下一首")
put("notreply", true)
put("broadcast", false)
}.toString()
}
}
@ -722,7 +1270,12 @@ private class MediaNextHandler(private val context: Context?) : FunctionHandler
private class OpenRecorderHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return "{\"result\": \"上下文未初始化\"}"
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
return try {
@ -739,18 +1292,32 @@ private class OpenRecorderHandler(private val context: Context?) : FunctionHandl
when {
context.packageManager?.queryIntentActivities(recordIntent, 0)?.isNotEmpty() == true -> {
context.startActivity(recordIntent)
"{\"result\": \"已打开录音机\"}"
JSONObject().apply {
put("result", "已打开录音机")
}.toString()
}
context.packageManager?.queryIntentActivities(recorderIntent, 0)?.isNotEmpty() == true -> {
context.startActivity(recorderIntent)
"{\"result\": \"已打开音频应用\"}"
JSONObject().apply {
put("result", "已打开音频应用")
}.toString()
}
else -> {
"{\"result\": \"无法打开录音机应用\"}"
JSONObject().apply {
put("error", JSONObject().apply {
put("code", "no_recorder_app")
put("message", "无法打开录音机应用")
})
}.toString()
}
}
} catch (e: Exception) {
"{\"result\": \"打开录音机失败:${e.message}\"}"
JSONObject().apply {
put("error", JSONObject().apply {
put("code", "recorder_failed")
put("message", "打开录音机失败: ${e.message}")
})
}.toString()
}
}
}

2
local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt

@ -479,7 +479,7 @@ class MCPSubClient(
retryCount++
currentReconnectDelay = minOf(currentReconnectDelay * 2, maxReconnectDelay)
Log.w(TAG, "连接失败,将在 ${currentReconnectDelay}ms 后尝试重连 (尝试 $retryCount/$maxRetryAttempts)")
// Log.w(TAG, "连接失败,将在 ${currentReconnectDelay}ms 后尝试重连 (尝试 $retryCount/$maxRetryAttempts)")
delay(currentReconnectDelay)
connect()

15
local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/ITtsService.kt

@ -89,4 +89,19 @@ interface ITtsService {
* @param listener 要移除的音频数据监听器
*/
fun removeAudioDataListener(listener: AudioDataListener)
/**
* 设置是否使用内部播放器
*
* @param useInternalPlayer true: 使用内部播放器自动播放音频
* false: 仅通过音频数据监听器输出数据,不播放
*/
fun setUseInternalPlayer(useInternalPlayer: Boolean)
/**
* 设置音频输出设备
*
* @param device 音频输出设备类型
*/
fun setAudioOutputDevice(device: AudioOutputDevice)
}

16
local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/TtsEvents.kt

@ -13,6 +13,12 @@ enum class TtsEventType {
/** 合成取消 */
SYNTHESIS_CANCELED,
/** 播放开始 */
PLAYBACK_STARTED,
/** 播放结束 */
PLAYBACK_COMPLETED,
/** 发生错误 */
ERROR
}
@ -52,4 +58,14 @@ interface AudioDataListener {
* @param data 音频数据字节数组
*/
fun onAudioData(data: ByteArray)
}
/**
* 音频输出设备类型
*/
enum class AudioOutputDevice {
DEFAULT, // 默认(如果有耳机选耳机,否则使用系统扬声器)
HEADPHONES, // 强制使用耳机
SPEAKER, // 强制使用扬声器
EARPIECE // 强制使用听筒
}

2
pubspec.yaml

@ -94,6 +94,8 @@ dependencies:
path: local_plugins/amap_service
location_service:
path: local_plugins/location_service
bytedance_speech:
path: local_plugins/bytedance_speech
realtime:
path: local_plugins/realtime
jl_opus:

Loading…
Cancel
Save