Browse Source

Merge branch 'new_dev' of https://github.com/deepcloud2048/deep_voice into new_dev

weicu
tanlongsheng 1 year ago
parent
commit
c01213751a
  1. 94
      .cursorrules
  2. 1
      .gitignore
  3. 15
      android/app/proguard-logging.pro
  4. 5
      android/app/src/main/kotlin/com/yunqiinnovation/deep_voice/MainActivity.kt
  5. 60
      android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt
  6. 5
      android/settings.gradle.kts
  7. 28
      lib/data/models/user_Info.dart
  8. 9
      lib/data/services/network/api.dart
  9. 14
      lib/data/services/speech_impl/azure_tts_service.dart
  10. 8
      lib/data/services/speech_impl/voice_clone_tts_service.dart
  11. 39
      lib/data/services/tts_service.dart
  12. 160
      lib/modules/FTFTranslation/README.md
  13. 10
      lib/modules/FTFTranslation/bindings/FTFTranslation_binding.dart
  14. 241
      lib/modules/FTFTranslation/controllers/FTFTranslation_controller.dart
  15. 212
      lib/modules/FTFTranslation/views/FTFTranslation_view.dart
  16. 22
      lib/modules/agent/controllers/agent_controller.dart
  17. 7
      lib/modules/meeting/controllers/meeting_controller.dart
  18. 5
      lib/modules/meeting/controllers/meeting_record_controller.dart
  19. 88
      lib/modules/meeting/views/meeting_view.dart
  20. 152
      lib/modules/pairing/controllers/pairing_controller.dart
  21. 51
      lib/modules/pairing/views/pairing_view.dart
  22. 73
      lib/modules/pairing/views/pairing_view_new.dart
  23. 25
      lib/modules/settings/views/settings_view.dart
  24. 8
      lib/modules/translation/controllers/translation_controller.dart
  25. 4
      lib/modules/translation/views/translation_view.dart
  26. 9
      lib/routes/app_pages.dart
  27. 1
      lib/routes/app_routes.dart
  28. 1
      local_plugins/agent_service/android/build.gradle.kts
  29. 172
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  30. 8
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift
  31. 10
      local_plugins/agent_service/lib/agent_service.dart
  32. 14
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt
  33. 77
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt
  34. 2
      local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCompanionManager.kt
  35. 129
      local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt
  36. 459
      local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt
  37. 6
      local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceSpeechPlugin.kt
  38. 201
      local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt
  39. 12
      local_plugins/bytedance_speech/lib/bytedance_speech.dart
  40. 299
      local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt
  41. 3
      local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/CustomSseClientTransport.kt
  42. 32
      local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/MCPSubClient.kt
  43. 811
      local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/SystemFunctionHandler.kt
  44. 2
      local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt
  45. 15
      local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/ITtsService.kt
  46. 15
      local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/TtsEvents.kt
  47. 2
      pubspec.yaml

94
.cursorrules

@ -1,94 +0,0 @@
# 代码简明规范
## Dart / Flutter
1. 风格统一
• 严格遵循 Effective Dart 格式与命名惯例。
2. 日志与调试
• 禁止 print;仅在异常场景调用 logger.e(...)。
3. 业务逻辑
• 默认使用 GetX(状态、路由、依赖注入等),避免手写 StatefulWidget。
4. 异常处理
• 不写重试逻辑,失败即返回错误。
• 谨慎使用 try-catch;优先通过类型系统与空安全防御。
5. 代码演进
• 任何改动都应 “小步、安全”,杜绝一次性大改或过度设计。
## 原生层(Android / iOS)
1. 配置不可改
• 禁止修改 SDK、NDK、JDK 与 iOS 构建配置。
2. Kotlin DSL
• 原生实现一律采用 Kotlin + 协程,遵守 Kotlin 最佳实践。
• 不写 Java 代码。
## 通用要求
1. 不自动执行
• 生成脚本/代码不得包含 flutter run 等执行指令。
2. 并发优先级
• 并发场景首选 Kotlin 协程;仅在 Flutter 端按需使用 Dart Future / Stream。
3. 语言
• 与用户交流、注释及文档一律使用中文。
• 不要自己创建总结文档。
宗旨:保持极简,先跑通业务,再考虑优化。
# 项目架构概述
DeepSound是一个基于Flutter和GetX构建的多平台语音应用,具有语音合成、语音识别、语音聊天等功能。
## 总体架构
- 采用GetX状态管理框架
- 遵循模块化设计,各功能模块相对独立
- 使用依赖注入实现组件解耦
- 多语言支持,使用GetX国际化方案
## 主要目录结构
- lib/core: 核心组件、工具和通用功能
- lib/data: 数据模型、服务和提供者
- lib/modules: 功能模块,按业务划分
- lib/routes: 路由定义和管理
- local_plugins: 本地插件,提供原生功能支持
# 核心模块说明
## 语音处理
- 火山语音服务: 提供语音识别和合成核心能力
- Azure语音服务: 提供备选的语音识别和合成能力
- 离线TTS支持: 在线语音合成失败时的备选方案
## 聊天功能
- 基于OpenAI的对话功能
- 支持语音输入、文字输入和语音输出
- 支持多种角色和语音类型
## 用户系统
- 支持多种登录方式:Firebase、Google、Facebook
- 用户资料管理和设置
- 主题切换和语言设置
## 蓝牙配对
- 支持蓝牙设备发现和连接
- 提供简化的配对流程
- 设备管理和状态监控
## 音频处理
- 音频录制和播放
- 支持波形显示
- 音量控制和音频质量设置
# 技术栈
- Flutter: UI框架
- GetX: 状态管理和依赖注入
- Firebase: 用户认证和后端服务
- WebSocket: 实时通信
- SQLite: 本地数据存储
- 原生插件: 语音识别、合成和蓝牙功能
# 插件说明
- jieli: 杰理蓝牙设备连接和控制插件
- volcano_speech: 火山语音服务插件,提供语音识别和合成
- azure_speech: 微软Azure语音服务插件
- open_ai_service: OpenAI服务插件,提供AI对话能力
- speech_to_text: 通用语音识别插件
# 环境配置
- 环境变量通过.env文件管理
- 支持不同环境的配置切换
- 敏感信息不应直接硬编码在代码中

1
.gitignore

@ -14,6 +14,7 @@ doc/api/
# dotenv environment variables file # dotenv environment variables file
.env* .env*
.env.example .env.example
CLAUDE.md
# Avoid committing generated Javascript files: # Avoid committing generated Javascript files:
*.dart.js *.dart.js

15
android/app/proguard-logging.pro

@ -0,0 +1,15 @@
# 移除系统组件的日志调用
-assumenosideeffects class android.util.Log {
public static int v(...);
public static int d(...);
public static int i(...);
}
# 针对特定类的日志移除
-assumenosideeffects class android.media.AudioTrack {
private static void log*(...);
}
-assumenosideeffects class android.app.AppOpsManager {
private static void log*(...);
}

5
android/app/src/main/kotlin/com/yunqiinnovation/deep_voice/MainActivity.kt

@ -1,5 +0,0 @@
package com.yunqiinnovation.deep_voice
import io.flutter.embedding.android.FlutterActivity
class MainActivity : FlutterActivity()

60
android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt

@ -39,6 +39,7 @@ class MainActivity: FlutterActivity() {
* 抑制系统级组件的日志输出 * 抑制系统级组件的日志输出
*/ */
private fun suppressSystemLogs() { private fun suppressSystemLogs() {
// 更全面的系统标签列表
val systemTags = arrayOf( val systemTags = arrayOf(
"MediaCodec", "MediaCodec",
"CCodec", "CCodec",
@ -49,11 +50,66 @@ class MainActivity: FlutterActivity() {
"AudioTrack", "AudioTrack",
"AudioManager", "AudioManager",
"AudioFlinger", "AudioFlinger",
"AudioService" "AudioService",
"AudioEffect",
"AppOpsManager",
"AudioRecord",
"OpenGLRenderer",
"Choreographer",
"InputMethodManager",
"ViewRootImpl",
"Surface",
"BufferQueueProducer",
"ActivityManager",
"WindowManager",
"SensorManager"
) )
// 尝试多种方式设置日志级别
systemTags.forEach { tag -> systemTags.forEach { tag ->
System.setProperty("log.tag.$tag", "WARN") try {
// 方式1:标准属性设置
System.setProperty("log.tag.$tag", "ERROR")
// 方式2:尝试使用反射设置(针对某些特殊标签)
setLogLevelViaReflection(tag, Log.ERROR)
} catch (e: Exception) {
// 忽略设置失败的情况
}
}
// 方式3:尝试设置全局日志级别
try {
// 设置更高的阈值,减少verbose和debug日志
System.setProperty("log.tag", "WARN")
} catch (e: Exception) {
// 忽略
}
}
/**
* 通过反射尝试设置日志级别
*/
private fun setLogLevelViaReflection(tag: String, level: Int) {
try {
val logClass = Class.forName("android.util.Log")
val isLoggableMethod = logClass.getDeclaredMethod(
"isLoggable",
String::class.java,
Int::class.javaPrimitiveType
)
isLoggableMethod.isAccessible = true
// 某些系统可能支持直接设置
val setLogLevelMethod = logClass.getDeclaredMethod(
"setLogLevel",
String::class.java,
Int::class.javaPrimitiveType
)
setLogLevelMethod.isAccessible = true
setLogLevelMethod.invoke(null, tag, level)
} catch (e: Exception) {
// 反射方法可能不存在,忽略
} }
} }
} }

5
android/settings.gradle.kts

@ -40,6 +40,8 @@ include(":classic_bluetooth")
include(":agent_service") include(":agent_service")
include(":speech") include(":speech")
include(":chat_api") include(":chat_api")
include(":bytedance_speech")
include(":jl_opus") include(":jl_opus")
// 设置azure_speech项目的路径 // 设置azure_speech项目的路径
project(":azure_speech").projectDir = file("../local_plugins/azure_speech/android") project(":azure_speech").projectDir = file("../local_plugins/azure_speech/android")
@ -51,4 +53,5 @@ project(":classic_bluetooth").projectDir = file("../local_plugins/classic_blueto
project(":agent_service").projectDir = file("../local_plugins/agent_service/android") project(":agent_service").projectDir = file("../local_plugins/agent_service/android")
project(":speech").projectDir = file("../local_plugins/speech/android") project(":speech").projectDir = file("../local_plugins/speech/android")
project(":chat_api").projectDir = file("../local_plugins/chat_api/android") project(":chat_api").projectDir = file("../local_plugins/chat_api/android")
project(":jl_opus").projectDir = file("../local_plugins/jl_opus/android") project(":bytedance_speech").projectDir = file("../local_plugins/bytedance_speech/android")
project(":jl_opus").projectDir = file("../local_plugins/jl_opus/android")

28
lib/data/models/user_Info.dart

@ -14,14 +14,15 @@ class User {
String name; String name;
String uid; String uid;
String avatar; String avatar;
String deviceid;
String language; String language;
String token; String token;
String mail; String mail;
String phone; String phone;
String phonemac; String phonemac;
String password; String password;
int membershipLevel = 0; //添加会员等级字段,0表示普通用户 int viplv = 0; //添加会员等级字段,0表示普通用户
String earphonemac;
String earphoneauthcode;
// final Int64 lastbettime; // final Int64 lastbettime;
// final Int64 createtime; // final Int64 createtime;
// final String wxopenid; // final String wxopenid;
@ -33,14 +34,15 @@ class User {
required this.name, required this.name,
required this.uid, required this.uid,
required this.avatar, required this.avatar,
required this.deviceid,
required this.language, required this.language,
required this.token, required this.token,
required this.mail, required this.mail,
required this.phone, required this.phone,
required this.phonemac, required this.phonemac,
required this.password, required this.password,
this.membershipLevel = 0, // 添加默认值 required this.viplv, // 添加默认值
required this.earphonemac,
required this.earphoneauthcode,
// required this.lastbettime, // required this.lastbettime,
// required this.createtime, // required this.createtime,
// required this.wxopenid, // required this.wxopenid,
@ -60,14 +62,16 @@ class User {
name: (userData?['name'] as String?) ?? '默认名称', // 提供默认值 name: (userData?['name'] as String?) ?? '默认名称', // 提供默认值
uid: (userData?['uid'] as String?) ?? '', uid: (userData?['uid'] as String?) ?? '',
avatar: (userData?['avatar'] as String?) ?? '', avatar: (userData?['avatar'] as String?) ?? '',
deviceid: (userData?['deviceid'] as String?) ?? '',
language: (userData?['language'] as String?) ?? 'en', language: (userData?['language'] as String?) ?? 'en',
mail: (userData?['mail'] as String?) ?? '', mail: (userData?['mail'] as String?) ?? '',
phone: (userData?['phone'] as String?) ?? '', phone: (userData?['phone'] as String?) ?? '',
phonemac: (userData?['phonemac'] as String?) ?? '', phonemac: (userData?['phonemac'] as String?) ?? '',
password: (userData?['password'] as String?) ?? '', password: (userData?['password'] as String?) ?? '',
membershipLevel: (userData?['membershipLevel'] as int?) ?? 0, // 提供默认值 viplv: (userData?['viplv'] as int?) ?? 0, // 提供默认值
earphonemac: (userData?['earphonemac'] as String?) ?? '',
earphoneauthcode: (userData?['earphoneauthcode'] as String?) ?? '',
// lastbettime: _parseTimestamp(userData['lastbettime']), // lastbettime: _parseTimestamp(userData['lastbettime']),
// createtime: _parseTimestamp(userData['createtime']), // createtime: _parseTimestamp(userData['createtime']),
// wxopenid: (userData?['wxopenid'] as String?) ?? '', // wxopenid: (userData?['wxopenid'] as String?) ?? '',
@ -85,13 +89,16 @@ class User {
name: (userData?['name'] as String?) ?? '默认名称', // 提供默认值 name: (userData?['name'] as String?) ?? '默认名称', // 提供默认值
uid: (userData?['uid'] as String?) ?? '', uid: (userData?['uid'] as String?) ?? '',
avatar: (userData?['avatar'] as String?) ?? '', avatar: (userData?['avatar'] as String?) ?? '',
deviceid: (userData?['deviceid'] as String?) ?? '',
language: (userData?['language'] as String?) ?? 'en', language: (userData?['language'] as String?) ?? 'en',
mail: (userData?['mail'] as String?) ?? '', mail: (userData?['mail'] as String?) ?? '',
phone: (userData?['phone'] as String?) ?? '', phone: (userData?['phone'] as String?) ?? '',
phonemac: (userData?['phonemac'] as String?) ?? '', phonemac: (userData?['phonemac'] as String?) ?? '',
password: (userData?['password'] as String?) ?? '', password: (userData?['password'] as String?) ?? '',
viplv: (userData?['viplv'] as int?) ?? 0, // 提供默认值
earphonemac: (userData?['earphonemac'] as String?) ?? '',
earphoneauthcode: (userData?['earphoneauthcode'] as String?) ?? '',
// lastbettime: _parseTimestamp(userData['lastbettime']), // lastbettime: _parseTimestamp(userData['lastbettime']),
// createtime: _parseTimestamp(userData['createtime']), // createtime: _parseTimestamp(userData['createtime']),
// wxopenid: (userData?['wxopenid'] as String?) ?? '', // wxopenid: (userData?['wxopenid'] as String?) ?? '',
@ -106,14 +113,17 @@ class User {
'name': name, 'name': name,
'uid': uid, 'uid': uid,
'avatar': avatar, 'avatar': avatar,
'deviceid': deviceid,
'language': language, 'language': language,
'token': token, 'token': token,
'mail': mail, 'mail': mail,
'phone': phone, 'phone': phone,
'phonemac': phonemac, 'phonemac': phonemac,
'password': password, 'password': password,
'membershipLevel': membershipLevel, 'viplv': viplv,
'earphonemac': earphonemac,
'earphoneauthcode': earphoneauthcode,
// 'lastbettime': lastbettime, // 'lastbettime': lastbettime,
// 'createtime': createtime, // 'createtime': createtime,
// 'wxopenid': wxopenid, // 'wxopenid': wxopenid,

9
lib/data/services/network/api.dart

@ -126,4 +126,13 @@ class Api {
params: params, params: params,
); );
} }
//绑定设备
static bindearphone([params]) {
return DioManager().request(
NWMethod.post,
'/api/home/user_bindearphone',
params: params,
);
}
} }

14
lib/data/services/speech_impl/azure_tts_service.dart

@ -413,4 +413,16 @@ class AzureTtsService extends GetxService implements TtsService {
Logger.error('释放资源失败: ${e.toString()}'); Logger.error('释放资源失败: ${e.toString()}');
} }
} }
}
@override
Future<bool> setAudioOutputDevice() async {
try {
// 开始合成,传递参数
final result = await _channel.invokeMethod('setAudioOutputDevice');
return result;
} catch (e) {
Logger.error('音频输出设备类型失败: ${e.toString()}');
return false;
}
}
}

8
lib/data/services/speech_impl/voice_clone_tts_service.dart

@ -390,4 +390,10 @@ class VoiceCloneTtsService extends GetxService implements TtsService {
Logger.error('释放资源失败: ${e.toString()}'); Logger.error('释放资源失败: ${e.toString()}');
} }
} }
}
@override
Future<bool> setAudioOutputDevice() {
// TODO: implement setAudioOutputDevice
throw UnimplementedError();
}
}

39
lib/data/services/tts_service.dart

@ -2,52 +2,51 @@ import 'dart:async';
/// 语音合成服务接口 /// 语音合成服务接口
abstract class TtsService { abstract class TtsService {
/// 当前选择的语音 /// 当前选择的语音
String get currentVoice; String get currentVoice;
/// 当前是否正在播放 /// 当前是否正在播放
bool get isSpeaking; bool get isSpeaking;
/// TTS事件流,用于监听语音合成事件 /// TTS事件流,用于监听语音合成事件
Stream<TtsEvent> get onEvent; Stream<TtsEvent> get onEvent;
/// 初始化语音合成服务 /// 初始化语音合成服务
Future<bool> initialize({ Future<bool> initialize({
List<String>? supportedLanguages, List<String>? supportedLanguages,
}); });
/// 设置语音 /// 设置语音
Future<bool> setVoice(String voiceName); Future<bool> setVoice(String voiceName);
/// 单次播放文本,完成后返回 /// 单次播放文本,完成后返回
Future<bool> speakOnce(String text); Future<bool> speakOnce(String text);
/// 将文本加入播放队列,支持连续播放多段文本 /// 将文本加入播放队列,支持连续播放多段文本
Future<bool> speakStream(String text); Future<bool> speakStream(String text);
/// 播放队列中剩余文本 /// 播放队列中剩余文本
Future<bool> flushStream(); Future<bool> flushStream();
/// 停止播放 /// 停止播放
Future<bool> stop(); Future<bool> stop();
/// 释放资源 /// 释放资源
Future<void> dispose(); Future<void> dispose();
Future<bool> setAudioOutputDevice();
} }
/// TTS事件类型 /// TTS事件类型
enum TtsEventType { enum TtsEventType {
/// 开始播放 /// 开始播放
started, started,
/// 播放完成 /// 播放完成
completed, completed,
/// 播放取消 /// 播放取消
canceled, canceled,
/// 播放错误 /// 播放错误
error, error,
} }
@ -56,15 +55,15 @@ enum TtsEventType {
class TtsEvent { class TtsEvent {
/// 事件类型 /// 事件类型
final TtsEventType type; final TtsEventType type;
/// 错误信息(仅在 error 类型中有效) /// 错误信息(仅在 error 类型中有效)
final String error; final String error;
TtsEvent({ TtsEvent({
required this.type, required this.type,
this.error = '', this.error = '',
}); });
/// 创建错误事件的快捷构造函数 /// 创建错误事件的快捷构造函数
factory TtsEvent.error(String errorMessage) { factory TtsEvent.error(String errorMessage) {
return TtsEvent( return TtsEvent(
@ -72,12 +71,12 @@ class TtsEvent {
error: errorMessage, error: errorMessage,
); );
} }
/// 检查是否为错误 /// 检查是否为错误
bool get isError => type == TtsEventType.error; bool get isError => type == TtsEventType.error;
@override @override
String toString() { String toString() {
return 'TtsEvent{type: $type, error: $error}'; return 'TtsEvent{type: $type, error: $error}';
} }
} }

160
lib/modules/FTFTranslation/README.md

@ -0,0 +1,160 @@
# Realtime 实时语音聊天模块
## ⚠️ 当前状态
**此模块为功能预览版本,提供完整的实时语音对话功能。**
通过原生插件实现语音识别、AI对话和语音合成的完整语音交互体验。
## 概述
Realtime模块实现了类似ChatGPT app的实时语音对话功能,当前版本提供:
- ✅ **完整语音交互** - 实时语音识别、AI对话和语音合成
- ✅ **状态管理** - 聆听、思考、回答状态的可视化
- ✅ **动画效果** - 基于Shader的流动云效果,支持音频振幅驱动
- ✅ **字幕显示** - 可选择开启/关闭的字幕模式
- ✅ **智能UI** - 根据连接状态自动调整按钮可用性
## 界面设计
### 布局结构
- **顶部状态区域** - 显示当前状态和字幕模式切换按钮
- **中间可视化区域** - 圆形Shader动画,根据音频振幅呈现呼吸效果
- **底部控制区域** - 主控制按钮和状态指示器
### 状态指示
- 🔵 **聆听状态** - 蓝色动画,接收用户语音输入
- 🟠 **思考状态** - 橙色动画,AI处理中
- 🟢 **回答状态** - 绿色动画,AI语音输出
- ⚪ **空闲状态** - 灰色静态,等待交互
### 字幕模式
- **默认关闭** - 初始状态下不显示任何字幕内容
- **手动切换** - 通过右上角按钮手动开启/关闭字幕
- **完整隐藏** - 关闭时字幕区域完全不显示,不占用界面空间
- **智能布局** - 开启时圆形动画自动上移,字幕区域占据下半屏
### 交互控制
- **录音按钮** - 仅在成功连接AI服务器且开始录音后可用
- **连接状态** - 未连接时录音按钮显示不可用状态
- **状态反馈** - 按钮颜色和可点击性根据连接状态动态调整
## 使用方法
### 路由导航
```dart
// 跳转到实时语音对话页面
Get.toNamed(Routes.realtime);
```
### 交互流程
1. **自动连接** - 页面加载后自动尝试连接AI服务器
2. **等待连接** - 录音按钮在连接成功前保持不可用状态
3. **开始录音** - 连接成功后自动开始录音,录音按钮变为可用
4. **语音对话** - 用户说话 → AI处理 → 语音回复
5. **字幕控制** - 可随时通过右上角按钮切换字幕显示
## 技术架构
### 控制器 (RealtimeController)
- **连接管理**: WebSocket连接到AI服务器
- **状态同步**: `isConnected`, `isListening`, `isSpeaking`
- **字幕控制**: `isSubtitleMode` 默认为false,用户手动控制
- **消息管理**: 智能去重,避免流式响应重复显示
- **音频处理**: 原生层计算RMS值,Flutter层接收并控制动画
### 视图组件
- **RealtimeView** - 主视图容器,响应式布局
- **ShaderMicFlow** - 基于GLSL Shader的流动云动画组件
- **AutoScrollList** - 自动滚动的字幕列表
### 原生集成
- **RealtimeService** - 原生音频处理和WebSocket通信
- **音频格式** - 16kHz, 1声道, 16位PCM
- **实时传输** - 低延迟音频流处理
- **RMS计算** - 原生层实时计算音频振幅
## 默认行为
### 字幕模式
- **初始状态**: 关闭 (`isSubtitleMode.value = false`)
- **显示逻辑**: 仅在用户手动开启时显示字幕内容
- **空间占用**: 关闭时字幕区域高度为0,不影响其他UI元素
- **切换效果**: 平滑动画过渡,圆形动画位置和大小同步调整
### 按钮状态
- **录音按钮**: 默认不可用,连接成功且开始录音后才可点击
- **状态指示**: 通过颜色和图标变化反映当前可用性
- **交互反馈**: 不可用时点击无效果,避免误操作
## 开发计划
### 已完成功能 ✅
- [x] 完整语音交互流程
- [x] 实时WebSocket通信
- [x] Shader流动云动画
- [x] RMS音频振幅计算
- [x] 智能UI状态管理
- [x] 字幕模式控制
- [x] 消息去重优化
### 计划增强 ⏳
- [ ] 多语言语音识别
- [ ] 语音中断检测
- [ ] 音频质量自适应
- [ ] 离线降级处理
- [ ] 语音情感识别
## 注意事项
1. **网络依赖** - 需要连接到AI服务器 (ws://192.168.1.11:8000/ws)
2. **麦克风权限** - 需要获取麦克风使用权限
3. **音频会话** - 自动管理音频会话,支持蓝牙设备
4. **资源管理** - 页面关闭时自动清理音频资源和网络连接
## 自定义配置
### 修改服务器地址
```dart
// 在RealtimeController中修改连接URL
await _realtimeService.initialize(
serverUrl: 'ws://your-server:port/ws',
// ... 其他参数
);
```
### 调整UI布局
```dart
// 修改字幕模式下圆形动画位置
alignment: controller.isSubtitleMode.value
? const Alignment(0, -0.9) // 调整Y轴位置
: const Alignment(0, -0.25),
```
### 音频参数调优
```dart
// 在初始化时调整音频参数
sampleRate: 16000, // 采样率
channels: 1, // 声道数
bitsPerSample: 16, // 位深度
```
## 技术栈
- **Flutter SDK**: >= 3.0.0
- **GetX**: >= 4.6.5 (状态管理)
- **Flutter Shaders**: >= 0.0.6 (Shader动画)
- **原生插件**: Realtime服务 (音频处理)
- **WebSocket**: 实时通信协议
## 版本信息
- **当前版本**: v1.0.0 (Production)
- **最后更新**: 2024年
- **状态**: 完整功能实现

10
lib/modules/FTFTranslation/bindings/FTFTranslation_binding.dart

@ -0,0 +1,10 @@
import 'package:get/get.dart';
import '../controllers/FTFTranslation_controller.dart';
class FTFTranslationBinding extends Bindings {
@override
void dependencies() {
// 注册实时语音聊天控制器
Get.lazyPut<FTFTranslationController>(() => FTFTranslationController());
}
}

241
lib/modules/FTFTranslation/controllers/FTFTranslation_controller.dart

@ -0,0 +1,241 @@
import 'dart:async';
import 'dart:typed_data';
import 'package:get/get.dart';
import '../../../core/utils/logger.dart';
import '../../../data/services/asr_service.dart';
import '../../../data/services/speech_impl/xunfei_asr_service.dart';
import '../../../data/services/tts_service.dart';
import '../../../data/services/language_manager.dart';
import '../../../data/services/volcano_translation_service.dart';
class FTFTranslationController extends GetxController
with GetSingleTickerProviderStateMixin {
final messages = <FTFTranslationMessage>[].obs;
final isChineseInput = true.obs;
final leftLanguage = '中文'.obs;
final rightLanguage = 'English'.obs;
final isLeftInput = true.obs;
final supportedLanguages = const [
'中文',
'English',
'日本語',
'한국어',
'Español',
];
late final AsrService _asrService;
StreamSubscription? _recognitionSubscription;
RxString intermediateContent = ''.obs;
RxString finalContent = ''.obs;
BytesBuilder _originaBytes = BytesBuilder(); //原始音频
final VolcanoTranslationService _translationService =
Get.find<VolcanoTranslationService>();
final LanguageManager _languageManager = Get.find<LanguageManager>();
// 当前使用的TTS服务
late TtsService _ttsService;
final isTranslating = false.obs;
final sourceLanguageCode = 'zh-CN'.obs;
final targetLanguageCode = 'en-US'.obs;
final isTtsEnabled = true.obs;
@override
void onInit() {
super.onInit();
Get.put(XunfeiAsrService());
final TtsService _ttsService = Get.find<TtsService>();
_asrService = Get.find<XunfeiAsrService>();
}
@override
void onClose() {
_asrService.dispose();
_recognitionSubscription?.cancel();
super.onClose();
}
//开始录音
Future<void> _startRecorder() async {
final recognitionStream =
await _asrService.startContinuousRecognition(false);
_recognitionSubscription = recognitionStream.listen(
_handleRecognitionEvent,
);
}
//结束录音
Future<void> _stopRecorder() async {
await _asrService.stopContinuousRecognition();
}
// 处理语音识别事件
void _handleRecognitionEvent(RecognitionEvent event) {
switch (event.type) {
case RecognitionEventType.intermediateResult:
intermediateContent.value = event.text;
handleFinalResult(event.text);
break;
case RecognitionEventType.finalResult:
finalContent.value += _spliceText(event);
intermediateContent.value = '';
_originaBytes.add(event.audio!);
handleIntermediateResult(event.text);
break;
default:
// 不做任何处理
break;
}
}
// 翻译文本
Future<String?> translateText(String sourceText,
{bool isFinal = false}) async {
if (sourceText.isEmpty) return null;
isTranslating.value = true;
String? translationResult;
try {
// 根据检测到的语言确定源语言和目标语言
final bool shouldSwap =
intermediateContent.value == sourceLanguageCode.value;
final detectedSourceLanguageCode =
shouldSwap ? sourceLanguageCode.value : targetLanguageCode.value;
final detectedTargetLanguageCode =
shouldSwap ? targetLanguageCode.value : sourceLanguageCode.value;
// 调用翻译服务
translationResult = await _translationService.translateText(
text: sourceText,
sourceLanguageCode: detectedSourceLanguageCode,
targetLanguageCode: detectedTargetLanguageCode,
);
if (translationResult != null) {
// 查找匹配的项目 - 优先使用timestamp查找
int index = -1;
// 播放TTS - 只在最终结果时播放
if (isTtsEnabled.value && isFinal) {
// Logger.info('播放翻译文本: ${translationResult}');
await playTranslatedText(translationResult);
}
}
} catch (e) {
Logger.error('翻译失败: ${e.toString()}');
} finally {
isTranslating.value = false;
}
return translationResult;
}
// 处理中间识别结果
void handleIntermediateResult(String text) {
if (text.isEmpty) return;
translateText(text, isFinal: false);
}
// 处理最终识别结果
Future<void> handleFinalResult(String text) async {
if (text.isEmpty) return;
// 使用项目的时间戳进行翻译
translateText(text, isFinal: true);
}
// 播放翻译文本
Future<void> playTranslatedText(String text) async {
if (text.isEmpty) return;
try {
final bool shouldSwap =
intermediateContent.value == targetLanguageCode.value;
final detectedTargetLanguageCode =
shouldSwap ? sourceLanguageCode.value : targetLanguageCode.value;
final voiceName = _languageManager
.getTtsVoiceNameByAsrCode(detectedTargetLanguageCode) ??
'en-US-AriaNeural';
await _ttsService.setVoice(voiceName);
await _ttsService.speakOnce(text);
} catch (e) {
Logger.error('播放翻译失败: ${e.toString()}');
}
}
String _spliceText(RecognitionEvent event) {
String eventText = event.text;
String text = '';
if (finalContent.value.isNotEmpty) {
if (event.role != '') {
final punctuations = {
',',
'。',
'!',
'?',
';',
':',
',',
'.',
':',
';',
'!',
'?'
};
final firstChar = eventText[0];
if (punctuations.contains(firstChar)) {
text += '$firstChar\n';
eventText = eventText.substring(1);
} else {
text += '\n';
}
text += '说话人${event.role}:';
}
} else {
text += '说话人${event.role}:';
}
text += eventText;
return text;
}
void addMessage(String text) {
final fromLang =
isLeftInput.value ? leftLanguage.value : rightLanguage.value;
final toLang = isLeftInput.value ? rightLanguage.value : leftLanguage.value;
messages.add(
FTFTranslationMessage(
original: text,
translated: '[$fromLang → $toLang] $text (翻译模拟)',
isChineseSpeaker: isLeftInput.value,
),
);
}
void swapLanguages() {
final temp = leftLanguage.value;
leftLanguage.value = rightLanguage.value;
rightLanguage.value = temp;
isLeftInput.value = !isLeftInput.value;
}
void toggleInputSide(bool isLeft) => isLeftInput.value = isLeft;
}
class FTFTranslationMessage {
final String original;
final String translated;
final bool isChineseSpeaker;
FTFTranslationMessage({
required this.original,
required this.translated,
required this.isChineseSpeaker,
});
}

212
lib/modules/FTFTranslation/views/FTFTranslation_view.dart

@ -0,0 +1,212 @@
import 'package:flutter/material.dart';
import 'package:get/get.dart';
import '../controllers/FTFTranslation_controller.dart';
class FTFTranslationView extends GetView<FTFTranslationController> {
const FTFTranslationView({Key? key}) : super(key: key);
@override
Widget build(BuildContext context) {
final TextEditingController inputController = TextEditingController();
return Scaffold(
backgroundColor: const Color(0xFF0E0F12),
appBar: AppBar(
backgroundColor: Colors.black,
title: const Text('面对面翻译'),
centerTitle: true,
),
body: Column(
children: [
Expanded(
child: Obx(() => ListView.builder(
padding: const EdgeInsets.all(16),
itemCount: controller.messages.length,
itemBuilder: (context, index) {
final msg = controller.messages[index];
final isChinese = msg.isChineseSpeaker;
return Align(
alignment: isChinese
? Alignment.centerLeft
: Alignment.centerRight,
child: Container(
margin: const EdgeInsets.symmetric(vertical: 6),
padding: const EdgeInsets.all(12),
decoration: BoxDecoration(
color: isChinese ? Colors.blue : Colors.grey[800],
borderRadius: BorderRadius.circular(12),
),
constraints: const BoxConstraints(maxWidth: 300),
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text(
msg.original,
style: const TextStyle(
color: Colors.white,
fontWeight: FontWeight.bold,
),
),
const SizedBox(height: 4),
Text(
msg.translated,
style: TextStyle(
color: Colors.white.withOpacity(0.9),
fontSize: 14,
),
),
],
),
),
);
},
)),
),
_buildBottomInput(inputController),
],
),
);
}
Widget _buildBottomInput(TextEditingController inputController) {
return Obx(() {
final isLeftInput = controller.isLeftInput.value;
final inputLang = isLeftInput
? controller.leftLanguage.value
: controller.rightLanguage.value;
return Container(
color: Colors.black,
padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 8),
child: Column(
children: [
Row(
children: [
Expanded(
child: TextField(
controller: inputController,
style: const TextStyle(color: Colors.white),
decoration: InputDecoration(
filled: true,
fillColor: Colors.grey[900],
hintText: '请输入 $inputLang 内容',
hintStyle: const TextStyle(color: Colors.grey),
border: OutlineInputBorder(
borderRadius: BorderRadius.circular(8),
borderSide: BorderSide.none,
),
contentPadding:
const EdgeInsets.symmetric(horizontal: 12),
),
),
),
IconButton(
onPressed: () {
final text = inputController.text.trim();
if (text.isNotEmpty) {
controller.addMessage(text);
inputController.clear();
}
},
icon: const Icon(Icons.send, color: Colors.white),
),
],
),
const SizedBox(height: 8),
Row(
children: [
Expanded(
child: _buildLanguageDropdown(
selected: controller.leftLanguage.value,
items: controller.supportedLanguages,
onChanged: (value) {
if (value != null) controller.leftLanguage.value = value;
},
icon: Icons.headphones,
isSelected: isLeftInput,
label: '左方语言',
onTapInputSide: () => controller.toggleInputSide(true),
),
),
const SizedBox(width: 12),
GestureDetector(
onTap: controller.swapLanguages,
child: Container(
padding: const EdgeInsets.all(8),
decoration: BoxDecoration(
color: Colors.white10,
shape: BoxShape.circle,
),
child:
const Icon(Icons.compare_arrows, color: Colors.white),
),
),
const SizedBox(width: 12),
Expanded(
child: _buildLanguageDropdown(
selected: controller.rightLanguage.value,
items: controller.supportedLanguages,
onChanged: (value) {
if (value != null) controller.rightLanguage.value = value;
},
icon: Icons.smartphone,
isSelected: !isLeftInput,
label: '右方语言',
onTapInputSide: () => controller.toggleInputSide(false),
),
),
],
),
],
),
);
});
}
Widget _buildLanguageDropdown({
required String selected,
required List<String> items,
required void Function(String?) onChanged,
required IconData icon,
required bool isSelected,
required String label,
required VoidCallback onTapInputSide,
}) {
return GestureDetector(
onTap: onTapInputSide,
child: Container(
padding: const EdgeInsets.symmetric(vertical: 8, horizontal: 12),
decoration: BoxDecoration(
border: Border.all(
color: isSelected ? Colors.white : Colors.grey,
),
borderRadius: BorderRadius.circular(24),
color: isSelected ? Colors.white10 : Colors.transparent,
),
child: Row(
children: [
Icon(icon, color: Colors.white),
const SizedBox(width: 8),
Expanded(
child: DropdownButton<String>(
dropdownColor: Colors.grey[900],
value: selected,
isExpanded: true,
underline: const SizedBox(),
iconEnabledColor: Colors.white,
style: const TextStyle(color: Colors.white),
items: items.map((lang) {
return DropdownMenuItem(
value: lang,
child: Text(lang),
);
}).toList(),
onChanged: onChanged,
),
),
],
),
),
);
}
}

22
lib/modules/agent/controllers/agent_controller.dart

@ -253,6 +253,7 @@ class AgentController extends GetxController {
case AgentServiceEventType.recognizing: case AgentServiceEventType.recognizing:
final text = event.data['text'] ?? ''; final text = event.data['text'] ?? '';
currentText.value = text; // 保留当前文本,以便其他地方使用 currentText.value = text; // 保留当前文本,以便其他地方使用
Logger.i(TAG, '识别中间结果: $text');
if (text.isNotEmpty) { if (text.isNotEmpty) {
// 查找是否有正在识别中的消息 // 查找是否有正在识别中的消息
final index = messages final index = messages
@ -280,7 +281,7 @@ class AgentController extends GetxController {
case AgentServiceEventType.recognitionResult: case AgentServiceEventType.recognitionResult:
final text = event.data['text'] ?? ''; final text = event.data['text'] ?? '';
Logger.i(TAG, '识别结果: $text'); Logger.i(TAG, '识别最终结果: $text');
// 查找是否有正在识别中的消息 // 查找是否有正在识别中的消息
final index = final index =
messages.lastIndexWhere((msg) => msg.isRecognizing && msg.isUser); messages.lastIndexWhere((msg) => msg.isRecognizing && msg.isUser);
@ -313,16 +314,27 @@ class AgentController extends GetxController {
break; break;
case AgentServiceEventType.ttsStarted: case AgentServiceEventType.ttsStarted:
isSpeaking.value = true; // isSpeaking.value = true;
Logger.i(TAG, 'TTS开始播放'); // Logger.i(TAG, 'TTS开始播放');
break; break;
case AgentServiceEventType.ttsCompleted: case AgentServiceEventType.ttsCompleted:
case AgentServiceEventType.ttsStopped: case AgentServiceEventType.ttsStopped:
case AgentServiceEventType.ttsCanceled: case AgentServiceEventType.ttsCanceled:
// isSpeaking.value = false;
// Logger.i(TAG, 'TTS停止播放, $event.type');
break;
case AgentServiceEventType.playbackStarted:
isSpeaking.value = true;
Logger.i(TAG, 'TTS播放开始');
break;
case AgentServiceEventType.playbackCompleted:
isSpeaking.value = false; isSpeaking.value = false;
Logger.i(TAG, 'TTS停止播放'); Logger.i(TAG, 'TTS播放完成');
break; break;
case AgentServiceEventType.imageProcessing: case AgentServiceEventType.imageProcessing:
@ -338,7 +350,7 @@ class AgentController extends GetxController {
final token = event.data['token'] ?? ''; final token = event.data['token'] ?? '';
final responseId = event.data['responseId'] ?? ''; final responseId = event.data['responseId'] ?? '';
// Logger.i(TAG, 'AI回复Token: $token, responseId: $responseId');
if (token.isNotEmpty) { if (token.isNotEmpty) {
// 如果是新的回复或者响应ID改变,创建新消息 // 如果是新的回复或者响应ID改变,创建新消息
if (_isNewAssistantResponse || if (_isNewAssistantResponse ||

7
lib/modules/meeting/controllers/meeting_controller.dart

@ -14,6 +14,7 @@ import 'package:permission_handler/permission_handler.dart';
import '../../../core/utils/logger.dart'; import '../../../core/utils/logger.dart';
import '../../../core/utils/synchrodata.dart'; import '../../../core/utils/synchrodata.dart';
import '../../../core/utils/upload_oss.dart'; import '../../../core/utils/upload_oss.dart';
import '../../../data/services/ble_manager.dart';
import '../../../data/services/db/sqflite_api.dart'; import '../../../data/services/db/sqflite_api.dart';
import '../../../core/utils/permission_util.dart'; import '../../../core/utils/permission_util.dart';
@ -37,7 +38,8 @@ class MeetingController extends GetxController {
TextEditingController titleController = TextEditingController(); // 录音标题 TextEditingController titleController = TextEditingController(); // 录音标题
RxList dataList = [].obs; RxList dataList = [].obs;
// 蓝牙服务
final bleManager = Get.find<BleManager>();
List dataListTask = []; List dataListTask = [];
bool isDisposed = false; bool isDisposed = false;
String filePath = ''; String filePath = '';
@ -313,6 +315,9 @@ class MeetingController extends GetxController {
PlatformFile file = result.files.first; PlatformFile file = result.files.first;
filePath = await _saveFile(file); filePath = await _saveFile(file);
fileName = file.name; fileName = file.name;
} else {
EasyLoading.dismiss();
return;
} }
} }

5
lib/modules/meeting/controllers/meeting_record_controller.dart

@ -113,12 +113,15 @@ class MeetingRecordController extends GetxController
switch (audioType.value) { switch (audioType.value) {
case 0: case 0:
fileName.value = "现场录音"; fileName.value = "现场录音";
_audioSourceType = false;
break; break;
case 1: case 1:
fileName.value = "音、视频录音"; fileName.value = "音、视频录音";
_audioSourceType = true;
break; break;
case 2: case 2:
fileName.value = "通话录音"; fileName.value = "通话录音";
_audioSourceType = true;
break; break;
} }
newName.value = fileName.value; newName.value = fileName.value;
@ -234,7 +237,7 @@ class MeetingRecordController extends GetxController
void _startHardwareServices() { void _startHardwareServices() {
switch (audioType.value) { switch (audioType.value) {
case 0: case 0:
_bleManager.openEncoder(); // _bleManager.openEncoder();
break; break;
case 1: case 1:
_bleManager.openDecoder(); _bleManager.openDecoder();

88
lib/modules/meeting/views/meeting_view.dart

@ -454,43 +454,43 @@ class MeetingView extends GetView<MeetingController> {
// TODO: 实现从文件导入逻辑 // TODO: 实现从文件导入逻辑
}, },
), ),
SizedBox(height: 8.w), // SizedBox(height: 8.w),
_importAudioItem1( // _importAudioItem1(
context, // context,
isDarkMode, // isDarkMode,
icon: Image.asset('assets/images/explore.png', // icon: Image.asset('assets/images/explore.png',
width: 32.w, height: 32.w), // width: 32.w, height: 32.w),
title: '从翻译里导入', // title: '从翻译里导入',
onTap: () { // onTap: () {
Navigator.of(context).pop(); // Navigator.of(context).pop();
_showLocalAudioVideoPicker(context, isDarkMode); // _showLocalAudioVideoPicker(context, isDarkMode);
// TODO: 实现从相册导入逻辑 // // TODO: 实现从相册导入逻辑
}, // },
), // ),
SizedBox(height: 8.w), // SizedBox(height: 8.w),
_importAudioItem1( // _importAudioItem1(
context, // context,
isDarkMode, // isDarkMode,
icon: Row( // icon: Row(
mainAxisSize: MainAxisSize.min, // mainAxisSize: MainAxisSize.min,
children: [ // children: [
Image.asset('assets/images/explore.png', // Image.asset('assets/images/explore.png',
width: 20.w, height: 20.w), // width: 20.w, height: 20.w),
SizedBox(width: 4.w), // SizedBox(width: 4.w),
Image.asset('assets/images/explore.png', // Image.asset('assets/images/explore.png',
width: 20.w, height: 20.w), // width: 20.w, height: 20.w),
SizedBox(width: 4.w), // SizedBox(width: 4.w),
Image.asset('assets/images/explore.png', // Image.asset('assets/images/explore.png',
width: 20.w, height: 20.w), // width: 20.w, height: 20.w),
], // ],
), // ),
title: '从opus导出', // title: '从opus导出',
onTap: () { // onTap: () {
Navigator.of(context).pop(); // Navigator.of(context).pop();
// TODO: 实现从其他APP导入逻辑 // // TODO: 实现从其他APP导入逻辑
_showLocalOpusPicker(context, isDarkMode); // _showLocalOpusPicker(context, isDarkMode);
}, // },
), // ),
], ],
), ),
), ),
@ -578,13 +578,12 @@ class MeetingView extends GetView<MeetingController> {
SizedBox(height: 8.w), SizedBox(height: 8.w),
// 音、视频录音 - 根据控制器状态禁用 // 音、视频录音 - 根据控制器状态禁用
_importAudioItem( _importAudioItem(
context, context, isDarkMode,
isDarkMode,
icon: Image.asset('assets/images/explore.png', icon: Image.asset('assets/images/explore.png',
width: 32.w, height: 32.w), width: 32.w, height: 32.w),
title: '音、视频录音', title: '音、视频录音',
enabled: false, // 依赖控制器状态 enabled: controller.bleManager.isConnected, // 依赖控制器状态
onTap: false onTap: controller.bleManager.isConnected
? () async { ? () async {
// 条件启用 // 条件启用
Navigator.of(context).pop(); Navigator.of(context).pop();
@ -599,14 +598,13 @@ class MeetingView extends GetView<MeetingController> {
SizedBox(height: 8.w), SizedBox(height: 8.w),
// 通话录音 - 根据控制器状态禁用 // 通话录音 - 根据控制器状态禁用
_importAudioItem( _importAudioItem(
context, context, isDarkMode,
isDarkMode,
icon: Row( icon: Row(
// ... 图标组合保持不变 // ... 图标组合保持不变
), ),
title: '通话录音', title: '通话录音',
enabled: false, // 依赖控制器状态 enabled: controller.bleManager.isConnected, // 依赖控制器状态
onTap: false onTap: controller.bleManager.isConnected
? () async { ? () async {
// 条件启用 // 条件启用
Navigator.of(context).pop(); Navigator.of(context).pop();

152
lib/modules/pairing/controllers/pairing_controller.dart

@ -1,4 +1,7 @@
import 'package:app_settings/app_settings.dart'; import 'package:app_settings/app_settings.dart';
import 'package:deep_voice/data/models/user_Info.dart';
import 'package:deep_voice/data/services/network/api.dart';
import 'package:flutter/material.dart';
import 'package:get/get.dart'; import 'package:get/get.dart';
import 'dart:async'; import 'dart:async';
import 'package:permission_handler/permission_handler.dart'; import 'package:permission_handler/permission_handler.dart';
@ -15,9 +18,10 @@ enum PairingState {
checking, // 检查中(蓝牙、权限等) checking, // 检查中(蓝牙、权限等)
waitingClassic, // 等待经典蓝牙连接 waitingClassic, // 等待经典蓝牙连接
verifying, // 验证设备 verifying, // 验证设备
manualbinding, //手动绑定设备
associating, // 正在进行BLE设备关联 associating, // 正在进行BLE设备关联
waitingBle, // 等待BLE连接 waitingBle, //等待BLE连接
completed, // 配对完成 completed, //配对完成
error, // 错误状态 error, // 错误状态
} }
@ -31,6 +35,7 @@ enum ErrorType {
deviceBound, // 设备已被绑定 deviceBound, // 设备已被绑定
associationFailed, // 关联失败 associationFailed, // 关联失败
bleTimeout, // BLE连接超时 bleTimeout, // BLE连接超时
authnFailed, // 授权失败
networkError, // 网络错误 networkError, // 网络错误
} }
@ -53,6 +58,7 @@ class PairingController extends GetxController {
final isLoading = false.obs; final isLoading = false.obs;
final showConnectButton = false.obs; final showConnectButton = false.obs;
final showRetryButton = false.obs; final showRetryButton = false.obs;
final showAutoCodeInput = false.obs;
final showSkipButton = false.obs; final showSkipButton = false.obs;
final connectButtonText = '连接设备'.obs; final connectButtonText = '连接设备'.obs;
final retryButtonText = '重试'.obs; final retryButtonText = '重试'.obs;
@ -68,6 +74,7 @@ class PairingController extends GetxController {
// ========== 当前设备信息 ========== // ========== 当前设备信息 ==========
BluetoothDevice? _currentClassicDevice; BluetoothDevice? _currentClassicDevice;
String? _verifiedDeviceAddress; String? _verifiedDeviceAddress;
TextEditingController codeInputController = TextEditingController();
// Public getter for view access // Public getter for view access
BluetoothDevice? get currentDevice => _currentClassicDevice; BluetoothDevice? get currentDevice => _currentClassicDevice;
@ -236,6 +243,7 @@ class PairingController extends GetxController {
showConnectButton.value = false; showConnectButton.value = false;
showRetryButton.value = false; showRetryButton.value = false;
showSkipButton.value = false; showSkipButton.value = false;
showAutoCodeInput.value = false;
switch (state) { switch (state) {
case PairingState.idle: case PairingState.idle:
@ -276,7 +284,12 @@ class PairingController extends GetxController {
_startBleConnection(); _startBleConnection();
}); });
break; break;
case PairingState.manualbinding:
statusText.value = '需要手动版定授权码...';
isLoading.value = false;
showAutoCodeInput.value = true;
showSkipButton.value = true;
break;
case PairingState.completed: case PairingState.completed:
statusText.value = '配对成功!'; statusText.value = '配对成功!';
isLoading.value = false; isLoading.value = false;
@ -306,6 +319,10 @@ class PairingController extends GetxController {
showRetryButton.value = true; showRetryButton.value = true;
retryButtonText.value = '重新关联'; retryButtonText.value = '重新关联';
break; break;
case ErrorType.authnFailed:
showRetryButton.value = true;
retryButtonText.value = '重新授权';
break;
default: default:
showRetryButton.value = true; showRetryButton.value = true;
retryButtonText.value = '重试'; retryButtonText.value = '重试';
@ -393,44 +410,48 @@ class PairingController extends GetxController {
Future<void> _verifyDevice(BluetoothDevice device) async { Future<void> _verifyDevice(BluetoothDevice device) async {
try { try {
Logger.i(_tag, '开始验证设备: ${device.name} (${device.address})'); Logger.i(_tag, '开始验证设备: ${device.name} (${device.address})');
// TODO: 实现API验证逻辑 // TODO: 实现API验证逻辑
// 暂时模拟验证通过 // 暂时模拟验证通过
await Future.delayed(const Duration(seconds: 2)); // await Future.delayed(const Duration(seconds: 2));;
if (await authbindingDevice()) {
// 检查设备名称是否包含 DeepSound 或 DeapSound // 检查设备名称是否包含 DeepSound 或 DeapSound
final deviceName = device.name ?? ''; final deviceName = device.name ?? '';
if (deviceName.contains('DeepSound') || if (deviceName.contains('DeepSound') ||
deviceName.contains('DeapSound')) { deviceName.contains('DeapSound')) {
Logger.i(_tag, '设备验证通过'); Logger.i(_tag, '设备验证通过');
_verifiedDeviceAddress = device.address; _verifiedDeviceAddress = device.address;
// 检查是否已有BLE关联 // 检查是否已有BLE关联
final pairedMacAddress = await _bleManager.getPairedMacAddress(); final pairedMacAddress = await _bleManager.getPairedMacAddress();
Logger.i(_tag, '已关联的MAC地址: $pairedMacAddress'); Logger.i(_tag, '已关联的MAC地址: $pairedMacAddress');
if (pairedMacAddress != null && if (pairedMacAddress != null &&
pairedMacAddress.toUpperCase() == device.address.toUpperCase()) { pairedMacAddress.toUpperCase() == device.address.toUpperCase()) {
// 已有关联,检查BLE连接状态 // 已有关联,检查BLE连接状态
Logger.i(_tag, '设备已关联,检查BLE连接状态'); Logger.i(_tag, '设备已关联,检查BLE连接状态');
final currentBleState = await _bleManager.getConnectionState(); final currentBleState = await _bleManager.getConnectionState();
Logger.i(_tag, '验证时BLE状态: $currentBleState'); Logger.i(_tag, '验证时BLE状态: $currentBleState');
if (currentBleState == 2) { if (currentBleState == 2) {
// BleConnectionState.connected // BleConnectionState.connected
Logger.i(_tag, 'BLE已连接,直接完成配对'); Logger.i(_tag, 'BLE已连接,直接完成配对');
_transition(PairingState.completed); _transition(PairingState.completed);
} else {
Logger.i(_tag, 'BLE未连接,等待BLE连接');
_transition(PairingState.waitingBle);
}
} else { } else {
Logger.i(_tag, 'BLE未连接,等待BLE连接'); // 没有关联或关联的是其他设备,需要进行关联
_transition(PairingState.waitingBle); Logger.i(_tag, '需要进行BLE设备关联');
_transition(PairingState.associating);
} }
} else { } else {
// 没有关联或关联的是其他设备,需要进行关联 // _setError(ErrorType.deviceNotMatch, '请连接 DeepSound 设备');
Logger.i(_tag, '需要进行BLE设备关联'); _transition(PairingState.completed);
_transition(PairingState.associating);
} }
} else { } else {
_setError(ErrorType.deviceNotMatch, '请连接 DeepSound 设备'); //验证失败 进入手动授权
_transition(PairingState.manualbinding);
} }
} catch (e) { } catch (e) {
Logger.e(_tag, '设备验证异常: $e'); Logger.e(_tag, '设备验证异常: $e');
@ -504,10 +525,57 @@ class PairingController extends GetxController {
} }
} }
//自动绑定设备
Future<bool> authbindingDevice() async {
if (User.instance.earphonemac == currentDevice?.address) {
// 已经绑定
return true;
}
// 调用BLE管理器的自动绑定方法
final response = await Api.bindearphone({
"earphonemac": currentDevice?.address,
"earphoneauthcode": "",
});
Logger.i(_tag, '手动授权结果 ${response}');
if (response != null) {
Logger.i(_tag, '设备自动绑定成功');
User.instance.earphonemac = currentDevice?.address ?? '';
return true;
} else {
Logger.e(_tag, '手动绑定失败!');
return false;
}
}
/// 用户点击授权码输入框
void onAutoCodeInputTap() async {
// 调用BLE管理器的自动绑定方法
final response = await Api.bindearphone({
"earphonemac": currentDevice?.address,
"earphoneauthcode": codeInputController.text.trim(),
});
Logger.i(_tag, '手动授权结果 ${response}');
if (response != null) {
_transition(PairingState.verifying);
_verifyDevice(_currentClassicDevice!);
} else {
Logger.e(_tag, '手动绑定失败!');
_setError(ErrorType.authnFailed, '设备授权失败,请检查授权码');
}
}
/// 导航到主页 /// 导航到主页
void _navigateToHome() async { void _navigateToHome() {
await BleManager.to.startAgentService(); Future.delayed(const Duration(seconds: 1), () async {
Future.delayed(const Duration(seconds: 1), () { // 启动AgentService服务
try {
Logger.i(_tag, '启动AgentService服务');
await _bleManager.startAgentService();
} catch (e) {
Logger.e(_tag, '启动AgentService失败: $e');
}
Get.offAllNamed(Routes.home); Get.offAllNamed(Routes.home);
}); });
} }
@ -565,8 +633,16 @@ class PairingController extends GetxController {
/// 用户点击跳过按钮 /// 用户点击跳过按钮
void onSkipTap() async { void onSkipTap() async {
await BleManager.to.startAgentService();
Logger.i(_tag, '用户选择跳过配对'); Logger.i(_tag, '用户选择跳过配对');
// 启动AgentService服务
try {
Logger.i(_tag, '启动AgentService服务');
await _bleManager.startAgentService();
} catch (e) {
Logger.e(_tag, '启动AgentService失败: $e');
}
Get.offAllNamed(Routes.home); Get.offAllNamed(Routes.home);
} }

51
lib/modules/pairing/views/pairing_view.dart

@ -336,6 +336,8 @@ class PairingView extends GetView<PairingController> {
return '扫描超时,请确保设备在附近'; return '扫描超时,请确保设备在附近';
case ErrorType.networkError: case ErrorType.networkError:
return '网络错误,请检查网络连接'; return '网络错误,请检查网络连接';
case ErrorType.authnFailed:
return '授权失败,请检查授权码是否正确';
default: default:
return '配对失败,请重试'; return '配对失败,请重试';
} }
@ -433,7 +435,7 @@ class PairingView extends GetView<PairingController> {
final showConnectButton = controller.showConnectButton.value; final showConnectButton = controller.showConnectButton.value;
final showRetryButton = controller.showRetryButton.value; final showRetryButton = controller.showRetryButton.value;
final showSkipButton = controller.showSkipButton.value; final showSkipButton = controller.showSkipButton.value;
final showAutoCodeInput = controller.showAutoCodeInput.value;
return Container( return Container(
width: double.infinity, width: double.infinity,
child: Column( child: Column(
@ -490,6 +492,51 @@ class PairingView extends GetView<PairingController> {
), ),
), ),
), ),
if (showAutoCodeInput)
Column(
children: [
// 输入框
SizedBox(
width: double.infinity,
height: 48.h,
child: TextField(
controller: controller
.codeInputController, // 你可以在 Controller 中声明这个
decoration: InputDecoration(
hintText: '请输入代码',
border: OutlineInputBorder(
borderRadius: BorderRadius.circular(12.r),
),
contentPadding: EdgeInsets.symmetric(horizontal: 16.w),
),
),
),
SizedBox(height: 12.h),
SizedBox(
width: double.infinity,
height: 48.h,
child: ElevatedButton(
onPressed: controller.onAutoCodeInputTap,
style: ElevatedButton.styleFrom(
backgroundColor: Theme.of(Get.context!).primaryColor,
shape: RoundedRectangleBorder(
borderRadius: BorderRadius.circular(12.r),
),
elevation: 0,
),
child: Text(
'授权绑定',
style: TextStyle(
fontSize: 16.sp,
fontWeight: FontWeight.w600,
color: Colors.white,
),
),
),
),
],
),
// 自动输入代码按钮
// 跳过按钮 // 跳过按钮
if (showSkipButton) ...[ if (showSkipButton) ...[
@ -510,11 +557,9 @@ class PairingView extends GetView<PairingController> {
), ),
), ),
], ],
], ],
), ),
); );
}); });
} }
} }

73
lib/modules/pairing/views/pairing_view_new.dart

@ -21,17 +21,17 @@ class PairingView extends GetView<PairingController> {
children: [ children: [
// 顶部标题 // 顶部标题
_buildHeader(isDarkMode), _buildHeader(isDarkMode),
// 中间内容 // 中间内容
Expanded( Expanded(
child: Center( child: Center(
child: Obx(() => _buildContent(isDarkMode)), child: Obx(() => _buildContent(isDarkMode)),
), ),
), ),
// 底部按钮 // 底部按钮
Obx(() => _buildButtons(isDarkMode)), Obx(() => _buildButtons(isDarkMode)),
SizedBox(height: 20.h), SizedBox(height: 20.h),
], ],
), ),
@ -69,15 +69,15 @@ class PairingView extends GetView<PairingController> {
Widget _buildContent(bool isDarkMode) { Widget _buildContent(bool isDarkMode) {
final state = controller.currentState; final state = controller.currentState;
final isLoading = controller.isLoading.value; final isLoading = controller.isLoading.value;
return Column( return Column(
mainAxisSize: MainAxisSize.min, mainAxisSize: MainAxisSize.min,
children: [ children: [
// 状态图标 // 状态图标
_buildStateIcon(state, isLoading, isDarkMode), _buildStateIcon(state, isLoading, isDarkMode),
SizedBox(height: 40.h), SizedBox(height: 40.h),
// 状态文本 // 状态文本
Text( Text(
controller.statusText.value, controller.statusText.value,
@ -87,7 +87,7 @@ class PairingView extends GetView<PairingController> {
), ),
textAlign: TextAlign.center, textAlign: TextAlign.center,
), ),
// 错误详情 // 错误详情
if (state == PairingState.error) ...[ if (state == PairingState.error) ...[
SizedBox(height: 16.h), SizedBox(height: 16.h),
@ -132,48 +132,51 @@ class PairingView extends GetView<PairingController> {
), ),
); );
} }
IconData iconData; IconData iconData;
Color iconColor; Color iconColor;
switch (state) { switch (state) {
case PairingState.idle: case PairingState.idle:
case PairingState.checking: case PairingState.checking:
iconData = Icons.bluetooth_searching; iconData = Icons.bluetooth_searching;
iconColor = Colors.blue; iconColor = Colors.blue;
break; break;
case PairingState.waitingClassic: case PairingState.waitingClassic:
iconData = Icons.bluetooth; iconData = Icons.bluetooth;
iconColor = Colors.grey; iconColor = Colors.grey;
break; break;
case PairingState.verifying: case PairingState.verifying:
iconData = Icons.verified_user; iconData = Icons.verified_user;
iconColor = Colors.orange; iconColor = Colors.orange;
break; break;
case PairingState.associating: case PairingState.associating:
iconData = Icons.link; iconData = Icons.link;
iconColor = Colors.blue; iconColor = Colors.blue;
break; break;
case PairingState.waitingBle: case PairingState.waitingBle:
iconData = Icons.sync; iconData = Icons.sync;
iconColor = Colors.blue; iconColor = Colors.blue;
break; break;
case PairingState.manualbinding:
iconData = Icons.blind;
iconColor = Colors.blue;
break;
case PairingState.completed: case PairingState.completed:
iconData = Icons.check_circle; iconData = Icons.check_circle;
iconColor = Colors.green; iconColor = Colors.green;
break; break;
case PairingState.error: case PairingState.error:
iconData = Icons.error; iconData = Icons.error;
iconColor = Colors.red; iconColor = Colors.red;
break; break;
} }
return Container( return Container(
width: 100.w, width: 100.w,
height: 100.w, height: 100.w,
@ -193,8 +196,9 @@ class PairingView extends GetView<PairingController> {
Widget _buildButtons(bool isDarkMode) { Widget _buildButtons(bool isDarkMode) {
final showConnectButton = controller.showConnectButton.value; final showConnectButton = controller.showConnectButton.value;
final showRetryButton = controller.showRetryButton.value; final showRetryButton = controller.showRetryButton.value;
final showAutoCodeInput = controller.showAutoCodeInput.value;
final showSkipButton = controller.showSkipButton.value; final showSkipButton = controller.showSkipButton.value;
return Column( return Column(
children: [ children: [
// 主要操作按钮 // 主要操作按钮
@ -204,16 +208,40 @@ class PairingView extends GetView<PairingController> {
controller.onConnectTap, controller.onConnectTap,
isDarkMode, isDarkMode,
), ),
if (showRetryButton) if (showRetryButton)
_buildPrimaryButton( _buildPrimaryButton(
'重试', '重试',
controller.onRetryTap, controller.onRetryTap,
isDarkMode, isDarkMode,
), ),
SizedBox(height: 12.h), SizedBox(height: 12.h),
if (showAutoCodeInput)
// 自动输入代码按钮
SizedBox(
width: double.infinity,
height: 48.h,
child: ElevatedButton(
onPressed: controller.onAutoCodeInputTap,
style: ElevatedButton.styleFrom(
backgroundColor: Theme.of(Get.context!).primaryColor,
shape: RoundedRectangleBorder(
borderRadius: BorderRadius.circular(12.r),
),
elevation: 0,
),
child: Text(
'自动输入代码',
style: TextStyle(
fontSize: 16.sp,
fontWeight: FontWeight.w600,
color: Colors.white,
),
),
),
),
// 跳过按钮 // 跳过按钮
if (showSkipButton) if (showSkipButton)
TextButton( TextButton(
@ -231,7 +259,8 @@ class PairingView extends GetView<PairingController> {
} }
/// 构建主要按钮 /// 构建主要按钮
Widget _buildPrimaryButton(String text, VoidCallback onPressed, bool isDarkMode) { Widget _buildPrimaryButton(
String text, VoidCallback onPressed, bool isDarkMode) {
return SizedBox( return SizedBox(
width: double.infinity, width: double.infinity,
height: 48.h, height: 48.h,
@ -255,4 +284,4 @@ class PairingView extends GetView<PairingController> {
), ),
); );
} }
} }

25
lib/modules/settings/views/settings_view.dart

@ -200,8 +200,7 @@ class SettingsView extends GetView<SettingsController> {
2: 'proUser', //'专业用户' 2: 'proUser', //'专业用户'
}; };
final membershipString = membershipMap[ final membershipString = membershipMap[
controller controller.userInfo.value.viplv] ??
.userInfo.value.membershipLevel] ??
'Unknown'; 'Unknown';
return Text( return Text(
membershipString, membershipString,
@ -543,6 +542,28 @@ class SettingsView extends GetView<SettingsController> {
}, },
isDarkMode: isDarkMode, isDarkMode: isDarkMode,
), ),
Divider(
height: 1,
color: isDarkMode
? Colors.white.withOpacity(0.1)
: Colors.grey[200]),
// 面对面翻译测试
_buildSimpleNavigationSetting(
title: '面对面翻译测试',
subtitle: '测试面对面翻译',
icon: Icons.bluetooth_searching,
iconBgColor: isDarkMode
? Colors.green[900]!.withOpacity(0.3)
: Colors.green[100]!,
iconColor:
isDarkMode ? Colors.green[300]! : Colors.green[600]!,
titleColor: isDarkMode ? Colors.white : null,
subtitleColor: isDarkMode ? Colors.white70 : null,
onTap: () {
Get.toNamed(Routes.ftftranslation);
},
isDarkMode: isDarkMode,
),
Divider( Divider(
height: 1, height: 1,
color: isDarkMode color: isDarkMode

8
lib/modules/translation/controllers/translation_controller.dart

@ -26,7 +26,7 @@ class TranslationController extends GetxController {
final LanguageManager _languageManager = Get.find<LanguageManager>(); final LanguageManager _languageManager = Get.find<LanguageManager>();
final GetStorage _storage = GetStorage(); final GetStorage _storage = GetStorage();
// 蓝牙服务 // 蓝牙服务
final _bleManager = Get.find<BleManager>(); final bleManager = Get.find<BleManager>();
// 音频输入源 // 音频输入源
bool _audioSourceType = false; bool _audioSourceType = false;
// 存储相关 // 存储相关
@ -308,14 +308,14 @@ class TranslationController extends GetxController {
// translationHistory.add(newItem); // translationHistory.add(newItem);
// 开始连续语音识别 // 开始连续语音识别
if (currentMode.value == "audioVideo") { if (currentMode.value == "audioVideo") {
_bleManager.openDecoder(); bleManager.openDecoder();
// 发送ble音乐或者通话远端声音 // 发送ble音乐或者通话远端声音
_audioSourceType = true; _audioSourceType = true;
isTtsEnabled.value = false; // 禁用TTS以避免干扰 isTtsEnabled.value = false; // 禁用TTS以避免干扰
Logger.info('发送ble音乐或者通话远端声音'); Logger.info('发送ble音乐或者通话远端声音');
} else if (currentMode.value == "call") { } else if (currentMode.value == "call") {
// 发送ble系统mic和dac(音乐或者通话远端)声音 // 发送ble系统mic和dac(音乐或者通话远端)声音
_bleManager.openA2DPDecoder(); bleManager.openA2DPDecoder();
_audioSourceType = true; _audioSourceType = true;
isTtsEnabled.value = true; isTtsEnabled.value = true;
Logger.info('发送ble系统mic和dac(音乐或者通话远端)声音'); Logger.info('发送ble系统mic和dac(音乐或者通话远端)声音');
@ -563,7 +563,7 @@ class TranslationController extends GetxController {
Future<void> stopAll() async { Future<void> stopAll() async {
try { try {
if (currentMode.value == "audioVideo" || currentMode.value == "call") { if (currentMode.value == "audioVideo" || currentMode.value == "call") {
_bleManager.closeCodec(); bleManager.closeCodec();
// 发送ble音乐或者通话远端声音 // 发送ble音乐或者通话远端声音
Logger.info('发送ble音乐或者通话远端声音'); Logger.info('发送ble音乐或者通话远端声音');
} }

4
lib/modules/translation/views/translation_view.dart

@ -67,12 +67,12 @@ class TranslationView extends GetView<TranslationController> {
_buildPopupMenuItem( _buildPopupMenuItem(
'audioVideo', 'audioVideo',
'audioVideoTranslation'.tr, 'audioVideoTranslation'.tr,
enabled: true, // 根据连接状态启用/禁用 enabled: controller.bleManager.isConnected, // 根据连接状态启用/禁用
), ),
_buildPopupMenuItem( _buildPopupMenuItem(
'call', 'call',
'callTranslation'.tr, 'callTranslation'.tr,
enabled: true, // 根据连接状态启用/禁用 enabled: controller.bleManager.isConnected, // 根据连接状态启用/禁用
), ),
], ],
child: Row( child: Row(

9
lib/routes/app_pages.dart

@ -46,6 +46,8 @@ import '../modules/realtime/bindings/realtime_binding.dart';
import '../modules/realtime/views/realtime_view.dart'; import '../modules/realtime/views/realtime_view.dart';
import '../modules/music/views/music_playlist_view.dart'; import '../modules/music/views/music_playlist_view.dart';
import '../modules/music/bindings/music_playlist_binding.dart'; import '../modules/music/bindings/music_playlist_binding.dart';
import '../modules/FTFTranslation/bindings/FTFTranslation_binding.dart';
import '../modules/FTFTranslation/views/FTFTranslation_view.dart';
abstract class AppPages { abstract class AppPages {
static final pages = [ static final pages = [
@ -191,6 +193,13 @@ abstract class AppPages {
transition: Transition.rightToLeft, transition: Transition.rightToLeft,
transitionDuration: Duration(milliseconds: 250), transitionDuration: Duration(milliseconds: 250),
), ),
GetPage(
name: Routes.ftftranslation,
page: () => const FTFTranslationView(),
binding: FTFTranslationBinding(),
transition: Transition.noTransition,
transitionDuration: Duration(milliseconds: 250),
),
GetPage( GetPage(
name: Routes.opusTest, name: Routes.opusTest,
page: () => const OpusTestView(), page: () => const OpusTestView(),

1
lib/routes/app_routes.dart

@ -28,6 +28,7 @@ abstract class Routes {
static const translationHistory = '/translation/history'; // 翻译历史 static const translationHistory = '/translation/history'; // 翻译历史
static const speechDemo = '/speech_demo'; // 语音演示 static const speechDemo = '/speech_demo'; // 语音演示
static const translationList = '/translation/List'; // 翻译历史 static const translationList = '/translation/List'; // 翻译历史
static const ftftranslation = '/ftftranslation'; // 面对面翻译
// 测试路由(生产环境建议移除) // 测试路由(生产环境建议移除)
static const ttsTest = '/tts_test'; // 文字转语音测试 static const ttsTest = '/tts_test'; // 文字转语音测试
static const asrTest = '/asr_test'; // 语音识别测试 static const asrTest = '/asr_test'; // 语音识别测试

1
local_plugins/agent_service/android/build.gradle.kts

@ -50,4 +50,5 @@ dependencies {
add("compileOnly", project(":speech")) add("compileOnly", project(":speech"))
add("compileOnly", project(":chat_api")) add("compileOnly", project(":chat_api"))
add("compileOnly", project(":location_service")) add("compileOnly", project(":location_service"))
add("compileOnly", project(":bytedance_speech"))
} }

172
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt

@ -16,10 +16,13 @@ import java.util.Collections
import kotlin.coroutines.CoroutineContext import kotlin.coroutines.CoroutineContext
import com.yunqiinnovation.ble_service.BleService import com.yunqiinnovation.ble_service.BleService
import android.media.MediaPlayer import android.media.MediaPlayer
import java.util.concurrent.atomic.AtomicBoolean
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import com.deep_voice.speech.tts.TtsEvent import com.deep_voice.speech.tts.TtsEvent
import com.deep_voice.speech.tts.TtsEventListener import com.deep_voice.speech.tts.TtsEventListener
import com.deep_voice.speech.tts.TtsEventType import com.deep_voice.speech.tts.TtsEventType
import com.yunqiinnovation.azure_speech.AzureTtsHelper import com.deep_voice.bytedance_speech.BytedanceTTS
/** /**
* 代理服务事件监听接口 * 代理服务事件监听接口
*/ */
@ -43,7 +46,7 @@ object AgentService : CoroutineScope {
// 协程相关 // 协程相关
private val job = SupervisorJob() private val job = SupervisorJob()
override val coroutineContext: CoroutineContext override val coroutineContext: CoroutineContext
get() = Dispatchers.Main + job get() = Dispatchers.IO + job
// 上下文和监听器 // 上下文和监听器
private lateinit var context: Context private lateinit var context: Context
@ -52,8 +55,8 @@ object AgentService : CoroutineScope {
// Azure服务 // Azure服务
private var azureAsrHelper: AzureAsrHelper? = null private var azureAsrHelper: AzureAsrHelper? = null
// 使用AzureTtsHelper作为唯一的TTS实现 // 使用BytedanceTTS作为唯一的TTS实现
private var ttsService: AzureTtsHelper? = null private var ttsService: BytedanceTTS? = null
// ChatAPI服务 - 使用新的ChatApiService // ChatAPI服务 - 使用新的ChatApiService
private lateinit var chatApiService: ChatApiService private lateinit var chatApiService: ChatApiService
@ -73,16 +76,24 @@ object AgentService : CoroutineScope {
// 音频播放器 // 音频播放器
private var audioPlayer: AudioPlayer? = null // 初始化音频播放器 private var audioPlayer: AudioPlayer? = null // 初始化音频播放器
// 状态 // 状态 - 使用原子类型确保线程安全
private var isInitialized = false private val _isInitialized = AtomicBoolean(false)
var isRecognitionActive = false val isInitialized: Boolean get() = _isInitialized.get()
private set
var isTtsSpeaking = false private val _isRecognitionActive = AtomicBoolean(false)
private set val isRecognitionActive: Boolean get() = _isRecognitionActive.get()
var hasSpeechDetected = false
private set private val _isTtsSpeaking = AtomicBoolean(false)
var isAiStreaming = false val isTtsSpeaking: Boolean get() = _isTtsSpeaking.get()
private set
private val _hasSpeechDetected = AtomicBoolean(false)
val hasSpeechDetected: Boolean get() = _hasSpeechDetected.get()
private val _isAiStreaming = AtomicBoolean(false)
val isAiStreaming: Boolean get() = _isAiStreaming.get()
// 用于保护复杂状态操作的互斥锁
private val stateMutex = Mutex()
// AI流生成相关 // AI流生成相关
private var currentAiJob: Job? = null private var currentAiJob: Job? = null
@ -90,6 +101,21 @@ object AgentService : CoroutineScope {
// 空闲检测相关 // 空闲检测相关
private var idleCheckJob: Job? = null private var idleCheckJob: Job? = null
private val maxIdleSeconds = 10 // 最大空闲秒数 private val maxIdleSeconds = 10 // 最大空闲秒数
// 打断防抖相关
private var lastInterruptTime = 0L
private val INTERRUPT_DEBOUNCE_MS = 300L
// 语气词过滤列表(中英文)
private val FILLER_WORDS = setOf(
// 中文语气词
"嗯", "啊", "呃", "哦", "唉", "哎", "哈", "呵", "嘿", "喂",
"哇", "呀", "哟", "嘛", "吧", "呢", "啦", "咦", "嘻", "嗨",
"额", "这个", "那个", "就是", "然后",
// 英文语气词
"um", "uh", "ah", "oh", "eh", "er", "hmm", "huh", "wow",
"well", "like", "you know", "i mean", "so", "actually"
)
/** /**
* 初始化系统提示词 * 初始化系统提示词
*/ */
@ -160,7 +186,7 @@ object AgentService : CoroutineScope {
// 加载最近的聊天记录 // 加载最近的聊天记录
loadChatHistory() loadChatHistory()
isInitialized = true _isInitialized.set(true)
return true return true
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "初始化失败: ${e.message}") Log.e(TAG, "初始化失败: ${e.message}")
@ -186,6 +212,7 @@ object AgentService : CoroutineScope {
// 释放ChatAPI服务 // 释放ChatAPI服务
if (::chatApiService.isInitialized) { if (::chatApiService.isInitialized) {
chatApiService.cancelCurrentStream() chatApiService.cancelCurrentStream()
chatApiService.dispose()
} }
// 释放TTS服务 // 释放TTS服务
@ -194,7 +221,7 @@ object AgentService : CoroutineScope {
job.cancel() job.cancel()
clearListeners() clearListeners()
isInitialized = false _isInitialized.set(false)
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "释放资源异常: ${e.message}") Log.e(TAG, "释放资源异常: ${e.message}")
} }
@ -216,20 +243,20 @@ object AgentService : CoroutineScope {
val ttsAppToken = config["volcanoToken"]?.toString() ?: "" val ttsAppToken = config["volcanoToken"]?.toString() ?: ""
val ttsLanguage = config["ttsLanguage"]?.toString() ?: "zh-CN" val ttsLanguage = config["ttsLanguage"]?.toString() ?: "zh-CN"
// 创建并初始化AzureTtsHelper // 创建并初始化BytedanceTTS
val azureTtsHelper = AzureTtsHelper(context) val bytedanceTts = BytedanceTTS(context)
ttsService = azureTtsHelper ttsService = bytedanceTts
// 初始化Azure TTS // 初始化Bytedance TTS
val success = azureTtsHelper.initialize( val success = bytedanceTts.initialize(
ttsAppId = "", // Azure TTS不需要appId ttsAppId = ttsAppId,
ttsAppToken = config["azureSpeechKey"]?.toString() ?: "", // Azure需要subscription key ttsAppToken = ttsAppToken,
ttsResource = config["azureSpeechRegion"]?.toString() ?: "", // Azure需要region信息 ttsResource = "", // Bytedance TTS不需要resource参数
language = ttsLanguage language = ttsLanguage
) )
if (!success) { if (!success) {
Log.e(TAG, "Azure TTS服务初始化失败") Log.e(TAG, "Bytedance TTS服务初始化失败")
} }
// 添加TTS事件监听 // 添加TTS事件监听
@ -237,22 +264,30 @@ object AgentService : CoroutineScope {
override fun onEvent(event: TtsEvent) { override fun onEvent(event: TtsEvent) {
when (event.type) { when (event.type) {
TtsEventType.SYNTHESIS_STARTED -> { TtsEventType.SYNTHESIS_STARTED -> {
isTtsSpeaking = true _isTtsSpeaking.set(true)
restartIdleCheck() restartIdleCheck()
sendEvent("tts_started", mapOf("status" to "started")) sendEvent("tts_started", mapOf("status" to "started"))
} }
TtsEventType.SYNTHESIS_COMPLETED -> { TtsEventType.SYNTHESIS_COMPLETED -> {
isTtsSpeaking = false _isTtsSpeaking.set(false)
restartIdleCheck() restartIdleCheck()
sendEvent("tts_completed", mapOf("status" to "completed")) sendEvent("tts_completed", mapOf("status" to "completed"))
} }
TtsEventType.SYNTHESIS_CANCELED -> { TtsEventType.SYNTHESIS_CANCELED -> {
isTtsSpeaking = false _isTtsSpeaking.set(false)
restartIdleCheck() restartIdleCheck()
sendEvent("tts_canceled", mapOf("status" to "canceled")) sendEvent("tts_canceled", mapOf("status" to "canceled"))
} }
TtsEventType.PLAYBACK_STARTED -> {
restartIdleCheck()
sendEvent("playback_started", mapOf("status" to "playback_started"))
}
TtsEventType.PLAYBACK_COMPLETED -> {
restartIdleCheck()
sendEvent("playback_completed", mapOf("status" to "playback_completed"))
}
TtsEventType.ERROR -> { TtsEventType.ERROR -> {
isTtsSpeaking = false _isTtsSpeaking.set(false)
restartIdleCheck() restartIdleCheck()
val params = event.params val params = event.params
val code = params["errorCode"] as? String ?: "UNKNOWN_ERROR" val code = params["errorCode"] as? String ?: "UNKNOWN_ERROR"
@ -356,8 +391,8 @@ object AgentService : CoroutineScope {
return false return false
} }
isRecognitionActive = true _isRecognitionActive.set(true)
hasSpeechDetected = false _hasSpeechDetected.set(false)
try { try {
val audioSourceType = if (isExternalActive) { val audioSourceType = if (isExternalActive) {
@ -370,7 +405,7 @@ object AgentService : CoroutineScope {
if (recognizing.isNotEmpty()) { if (recognizing.isNotEmpty()) {
// 检测到语音,更新状态 // 检测到语音,更新状态
val previousHasSpeech = hasSpeechDetected val previousHasSpeech = hasSpeechDetected
hasSpeechDetected = true _hasSpeechDetected.set(true)
if (!previousHasSpeech) { if (!previousHasSpeech) {
restartIdleCheck() restartIdleCheck()
@ -381,8 +416,28 @@ object AgentService : CoroutineScope {
"language" to detectedLanguage "language" to detectedLanguage
)) ))
// 如果TTS正在播放或AI正在生成,则触发打断 // 如果TTS正在播放或AI正在生成,检查是否需要打断
if (isTtsSpeaking || isAiStreaming) { if (isTtsSpeaking || isAiStreaming) {
val currentTime = System.currentTimeMillis()
// 防抖处理:避免过于频繁的打断
if (currentTime - lastInterruptTime < INTERRUPT_DEBOUNCE_MS) {
return
}
// 简单过滤:太短的内容可能是噪音
if (recognizing.trim().length < 2) {
return
}
// 过滤纯语气词
val trimmedText = recognizing.trim().lowercase()
if (FILLER_WORDS.contains(trimmedText)) {
return
}
// 执行打断
lastInterruptTime = currentTime
interruptCurrentResponse() interruptCurrentResponse()
} }
} }
@ -400,7 +455,7 @@ object AgentService : CoroutineScope {
// 重置状态,继续识别 // 重置状态,继续识别
val previousHasSpeech = hasSpeechDetected val previousHasSpeech = hasSpeechDetected
hasSpeechDetected = false _hasSpeechDetected.set(false)
if (previousHasSpeech) { if (previousHasSpeech) {
restartIdleCheck() restartIdleCheck()
@ -417,13 +472,13 @@ object AgentService : CoroutineScope {
override fun onSessionStopped() { override fun onSessionStopped() {
sendEvent("recognition_stopped", mapOf("status" to "stopped")) sendEvent("recognition_stopped", mapOf("status" to "stopped"))
isRecognitionActive = false _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
audioPlayer?.playAudio(R.raw.stop) audioPlayer?.playAudio(R.raw.stop)
} }
override fun onCanceled(reason: String, errorDetails: String) { override fun onCanceled(reason: String, errorDetails: String) {
isRecognitionActive = false _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
BleService.closeCodec() BleService.closeCodec()
sendEvent("recognition_canceled", mapOf( sendEvent("recognition_canceled", mapOf(
@ -433,7 +488,7 @@ object AgentService : CoroutineScope {
} }
override fun onError(error: String) { override fun onError(error: String) {
isRecognitionActive = false _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
BleService.closeCodec() BleService.closeCodec()
Log.e(TAG, "语音识别错误: $error") Log.e(TAG, "语音识别错误: $error")
@ -445,7 +500,7 @@ object AgentService : CoroutineScope {
}, audioSourceType) }, audioSourceType)
return true return true
} catch (e: Exception) { } catch (e: Exception) {
isRecognitionActive = false _isRecognitionActive.set(false)
Log.e(TAG, "启动语音识别失败: ${e.message}") Log.e(TAG, "启动语音识别失败: ${e.message}")
sendEvent("error", mapOf( sendEvent("error", mapOf(
"code" to "RECOGNITION_START_ERROR", "code" to "RECOGNITION_START_ERROR",
@ -467,11 +522,11 @@ object AgentService : CoroutineScope {
try { try {
azureAsrHelper?.stopContinuousRecognition() azureAsrHelper?.stopContinuousRecognition()
BleService.closeCodec() BleService.closeCodec()
isRecognitionActive = false _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "停止语音识别异常: ${e.message}") Log.e(TAG, "停止语音识别异常: ${e.message}")
isRecognitionActive = false _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
} }
} }
@ -482,14 +537,20 @@ object AgentService : CoroutineScope {
*/ */
fun interruptCurrentResponse() { fun interruptCurrentResponse() {
if(isAiStreaming || isTtsSpeaking) { if(isAiStreaming || isTtsSpeaking) {
// 停止TTS播放 val startTime = System.currentTimeMillis()
stopTts()
// 停止AI流输出 // 并行执行停止操作,加快响应速度
stopTts()
stopAiStream() stopAiStream()
// 发送打断事件 // 记录打断耗时
sendEvent("response_interrupted", mapOf("status" to "interrupted"))
val duration = System.currentTimeMillis() - startTime
sendEvent("response_interrupted", mapOf(
"status" to "interrupted",
"duration_ms" to duration
))
} }
} }
@ -500,7 +561,7 @@ object AgentService : CoroutineScope {
if (isAiStreaming) { if (isAiStreaming) {
try { try {
// 先更新状态,避免回调时的状态不一致 // 先更新状态,避免回调时的状态不一致
isAiStreaming = false _isAiStreaming.set(false)
// 取消当前AI生成任务 // 取消当前AI生成任务
currentAiJob?.cancel() currentAiJob?.cancel()
@ -512,7 +573,7 @@ object AgentService : CoroutineScope {
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "停止AI流输出异常", e) Log.e(TAG, "停止AI流输出异常", e)
// 确保状态被重置,即使发生异常 // 确保状态被重置,即使发生异常
isAiStreaming = false _isAiStreaming.set(false)
currentAiJob = null currentAiJob = null
} }
} }
@ -606,7 +667,7 @@ object AgentService : CoroutineScope {
currentAiJob = launch { currentAiJob = launch {
try { try {
// 设置状态为正在流式输出 // 设置状态为正在流式输出
isAiStreaming = true _isAiStreaming.set(true)
// 使用历史记录作为上下文发送到OpenAI // 使用历史记录作为上下文发送到OpenAI
val responseBuilder = StringBuilder() val responseBuilder = StringBuilder()
@ -704,7 +765,7 @@ object AgentService : CoroutineScope {
// 保存聊天记录 // 保存聊天记录
saveChatMessage(displayText, response,aiMetadata,userMetadata.toString()) saveChatMessage(displayText, response,aiMetadata,userMetadata.toString())
// 标记AI流式输出已完成 // 标记AI流式输出已完成
isAiStreaming = false _isAiStreaming.set(false)
currentAiJob = null currentAiJob = null
} }
@ -716,7 +777,7 @@ object AgentService : CoroutineScope {
)) ))
// 标记AI流式输出已完成 // 标记AI流式输出已完成
isAiStreaming = false _isAiStreaming.set(false)
currentAiJob = null currentAiJob = null
} }
@ -760,7 +821,7 @@ object AgentService : CoroutineScope {
)) ))
// 确保状态被重置 // 确保状态被重置
isAiStreaming = false _isAiStreaming.set(false)
currentAiJob = null currentAiJob = null
} }
} }
@ -824,7 +885,7 @@ object AgentService : CoroutineScope {
if (text.isEmpty()) return if (text.isEmpty()) return
// 更新状态 // 更新状态
isTtsSpeaking = true _isTtsSpeaking.set(true)
restartIdleCheck() // 状态变化,重启检测 restartIdleCheck() // 状态变化,重启检测
// 直接调用TTS,无需协程包装 // 直接调用TTS,无需协程包装
@ -838,7 +899,7 @@ object AgentService : CoroutineScope {
if (isTtsSpeaking) { if (isTtsSpeaking) {
ttsService?.stop() ttsService?.stop()
isTtsSpeaking = false _isTtsSpeaking.set(false)
restartIdleCheck() // 状态变化,重启检测 restartIdleCheck() // 状态变化,重启检测
sendEvent("tts_stopped", mapOf("status" to "stopped")) sendEvent("tts_stopped", mapOf("status" to "stopped"))
} }
@ -915,9 +976,8 @@ object AgentService : CoroutineScope {
* 发送事件 * 发送事件
*/ */
private fun sendEvent(eventName: String, data: Map<String, Any>) { private fun sendEvent(eventName: String, data: Map<String, Any>) {
// 使用协程确保在主线程上执行 // 切换到主线程执行监听器回调,避免 UI 更新问题
launch { launch(Dispatchers.Main) {
// 我们已在主线程上下文中启动协程,无需再切换线程
listeners.forEach { listener -> listeners.forEach { listener ->
try { try {
listener.onEvent(eventName, data) listener.onEvent(eventName, data)

8
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift

@ -909,6 +909,14 @@ extension AgentServiceImpl: TtsEventListener {
restartIdleCheck() restartIdleCheck()
sendEvent(name: "tts_completed", data: ["status": "completed"]) sendEvent(name: "tts_completed", data: ["status": "completed"])
case .playbackStarted:
restartIdleCheck()
sendEvent(name: "playback_started", data: ["status": "playback_started"])
case .playbackCompleted:
restartIdleCheck()
sendEvent(name: "playback_completed", data: ["status": "playback_completed"])
case .synthesisCanceled: case .synthesisCanceled:
isSpeaking = false isSpeaking = false
restartIdleCheck() restartIdleCheck()

10
local_plugins/agent_service/lib/agent_service.dart

@ -38,6 +38,12 @@ enum AgentServiceEventType {
/// TTS停止 /// TTS停止
ttsStopped, ttsStopped,
/// 播放开始
playbackStarted,
/// 播放完成
playbackCompleted,
/// 响应中断 /// 响应中断
responseInterrupted, responseInterrupted,
@ -147,6 +153,10 @@ class AgentService {
return AgentServiceEventType.ttsCanceled; return AgentServiceEventType.ttsCanceled;
case 'tts_stopped': case 'tts_stopped':
return AgentServiceEventType.ttsStopped; return AgentServiceEventType.ttsStopped;
case 'playback_started':
return AgentServiceEventType.playbackStarted;
case 'playback_completed':
return AgentServiceEventType.playbackCompleted;
case 'response_interrupted': case 'response_interrupted':
return AgentServiceEventType.responseInterrupted; return AgentServiceEventType.responseInterrupted;
case 'assistant_token': case 'assistant_token':

14
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt

@ -15,7 +15,7 @@ import com.deep_voice.speech.tts.TtsEvent
import com.deep_voice.speech.tts.TtsEventListener import com.deep_voice.speech.tts.TtsEventListener
import com.deep_voice.speech.tts.TtsEventType import com.deep_voice.speech.tts.TtsEventType
import com.yunqiinnovation.ble_service.BleService import com.yunqiinnovation.ble_service.BleService
import com.deep_voice.speech.tts.AudioOutputDevice
/** AzureSpeechPlugin */ /** AzureSpeechPlugin */
class AzureSpeechPlugin : BleService.Callback, FlutterPlugin { class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
private val tag = "AzureSpeechPlugin" private val tag = "AzureSpeechPlugin"
@ -158,6 +158,14 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
) )
} }
TtsEventType.PLAYBACK_STARTED -> mapOf(
"type" to "playback_started"
)
TtsEventType.PLAYBACK_COMPLETED -> mapOf(
"type" to "playback_completed"
)
TtsEventType.ERROR -> { TtsEventType.ERROR -> {
val params = event.params val params = event.params
mapOf( mapOf(
@ -493,6 +501,10 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
azureTtsHelper.release() azureTtsHelper.release()
result.success(true) result.success(true)
} }
"setAudioOutputDevice" -> {
azureTtsHelper.setAudioOutputDevice(AudioOutputDevice.SPEAKER)
result.success(true)
}
else -> { else -> {
result.notImplemented() result.notImplemented()

77
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt

@ -1,14 +1,16 @@
package com.yunqiinnovation.azure_speech package com.yunqiinnovation.azure_speech
import android.content.Context import android.content.Context
import android.media.AudioManager
import com.microsoft.cognitiveservices.speech.* import com.microsoft.cognitiveservices.speech.*
import com.microsoft.cognitiveservices.speech.audio.* import com.microsoft.cognitiveservices.speech.audio.*
import com.yunqiinnovation.azure_speech.utils.FileLogger import com.yunqiinnovation.azure_speech.utils.FileLogger
import com.deep_voice.speech.tts.AudioDataListener
import com.deep_voice.speech.tts.AudioOutputDevice
import com.deep_voice.speech.tts.ITtsService import com.deep_voice.speech.tts.ITtsService
import com.deep_voice.speech.tts.TtsEvent import com.deep_voice.speech.tts.TtsEvent
import com.deep_voice.speech.tts.TtsEventListener import com.deep_voice.speech.tts.TtsEventListener
import com.deep_voice.speech.tts.TtsEventType import com.deep_voice.speech.tts.TtsEventType
import com.deep_voice.speech.tts.AudioDataListener
import kotlinx.coroutines.* import kotlinx.coroutines.*
import java.io.ByteArrayInputStream import java.io.ByteArrayInputStream
import java.io.InputStream import java.io.InputStream
@ -169,6 +171,75 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
audioDataListeners.remove(listener) audioDataListeners.remove(listener)
} }
/**
* 设置是否使用内部播放器
*
* @param useInternalPlayer true: 使用内部播放器自动播放音频
* false: 仅通过音频数据监听器输出数据,不播放
*/
override fun setUseInternalPlayer(useInternalPlayer: Boolean) {
// Azure TTS 通过 AudioConfig 控制播放器
// 如果不使用内部播放器,需要重新配置 synthesizer
if (!useInternalPlayer && customAudioOutputStream == null) {
// 创建自定义音频输出流以便捕获音频数据
customAudioOutputStream = SimpleAudioPlayer(context).getAudioOutputStream()
// 如果已经初始化,需要重新创建 synthesizer
if (isInitialized && speechConfig != null) {
synthesizer?.close()
val audioConfig = AudioConfig.fromStreamOutput(customAudioOutputStream)
synthesizer = SpeechSynthesizer(speechConfig, audioConfig)
setupEventListeners()
}
} else if (useInternalPlayer && customAudioOutputStream != null) {
// 切换回使用内部播放器
customAudioOutputStream = null
// 如果已经初始化,需要重新创建 synthesizer
if (isInitialized && speechConfig != null) {
synthesizer?.close()
val audioConfig = AudioConfig.fromDefaultSpeakerOutput()
synthesizer = SpeechSynthesizer(speechConfig, audioConfig)
setupEventListeners()
}
}
}
/**
* 设置音频输出设备
*
* @param device 音频输出设备类型
*/
override fun setAudioOutputDevice(device: AudioOutputDevice) {
// Azure TTS使用系统默认的音频路由
// 音频输出设备的控制需要通过Android的AudioManager实现
val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager
audioManager?.let { manager ->
when (device) {
AudioOutputDevice.DEFAULT -> {
// 默认模式:系统自动选择
manager.mode = AudioManager.MODE_NORMAL
manager.isSpeakerphoneOn = false
}
AudioOutputDevice.SPEAKER -> {
// 强制使用扬声器
/* MODE_NORMAL 默认媒体模式 ❌ 不能强制扬声器(会被耳机覆盖)
MODE_IN_COMMUNICATION VoIP 通信模式 ✅ 可以强制使用扬声器或听筒
MODE_IN_CALL 电话模式(仅限系统) ✅ 更强但风险大,不推荐使用
*/
manager.mode = AudioManager.MODE_IN_COMMUNICATION
manager.isSpeakerphoneOn = true
}
AudioOutputDevice.HEADPHONES -> {
// 强制使用耳机(如果已连接)
manager.mode = AudioManager.MODE_NORMAL
manager.isSpeakerphoneOn = false
// 注意:Android不能强制路由到耳机,只能在耳机已连接时使用
}
}
}
}
/** /**
* 触发事件通知 * 触发事件通知
*/ */
@ -194,6 +265,8 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
SynthesisStarted?.addEventListener { _, eventArgs -> SynthesisStarted?.addEventListener { _, eventArgs ->
FileLogger.d(TAG, "语音合成开始: resultId=${eventArgs.result.resultId}") FileLogger.d(TAG, "语音合成开始: resultId=${eventArgs.result.resultId}")
notifyEvent(TtsEventType.SYNTHESIS_STARTED) notifyEvent(TtsEventType.SYNTHESIS_STARTED)
// Azure TTS 在使用默认音频输出时会立即开始播放
notifyEvent(TtsEventType.PLAYBACK_STARTED)
} }
// 合成中事件(接收音频数据) // 合成中事件(接收音频数据)
@ -219,6 +292,8 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
FileLogger.d(TAG, "语音合成完成: resultId=${eventArgs.result.resultId}, 音频长度=${eventArgs.result.audioLength} 字节") FileLogger.d(TAG, "语音合成完成: resultId=${eventArgs.result.resultId}, 音频长度=${eventArgs.result.audioLength} 字节")
isSpeaking = false isSpeaking = false
notifyEvent(TtsEventType.SYNTHESIS_COMPLETED) notifyEvent(TtsEventType.SYNTHESIS_COMPLETED)
// Azure TTS 合成完成即播放完成
notifyEvent(TtsEventType.PLAYBACK_COMPLETED)
} }
// 合成取消事件 // 合成取消事件

2
local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleCompanionManager.kt

@ -395,7 +395,7 @@ class BleCompanionManager(private val context: Context) {
// 构建过滤器 // 构建过滤器
val filter = ScanFilter.Builder() val filter = ScanFilter.Builder()
.setDeviceAddress(mac) .setDeviceAddress(mac)
// .setManufacturerData(0xEFAB, byteArrayOf(0xA0.toByte(), 0x01)) .setManufacturerData(0xEFAB, byteArrayOf(0xA0.toByte(), 0x01))
.build() .build()
// 扫描设置 // 扫描设置

129
local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt

@ -22,6 +22,7 @@ import java.util.concurrent.LinkedBlockingQueue
import android.os.Handler import android.os.Handler
import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicBoolean
import android.util.Log import android.util.Log
/** /**
* BLE服务类:提供蓝牙低功耗设备的扫描、连接和通信功能 * BLE服务类:提供蓝牙低功耗设备的扫描、连接和通信功能
* 支持: * 支持:
@ -67,17 +68,20 @@ object BleService {
private var bluetoothLeScanner: BluetoothLeScanner? = null private var bluetoothLeScanner: BluetoothLeScanner? = null
// GATT连接相关 // GATT连接相关
var bluetoothGatt: BluetoothGatt? = null var bluetoothGatt: BluetoothGatt? = null
private var notifyChar: BluetoothGattCharacteristic? = null private var notifyChar: BluetoothGattCharacteristic? = null
private var writeChar: BluetoothGattCharacteristic? = null private var writeChar: BluetoothGattCharacteristic? = null
private var audioChar: BluetoothGattCharacteristic? = null private var audioChar: BluetoothGattCharacteristic? = null
var recordfile: RecordingFile? = null var recordfile: RecordingFile? = null
// 扫描相关 // 扫描相关
private lateinit var scanHandler: Handler private lateinit var scanHandler: Handler
private val scanResults = ArrayList<ScanResult>() private val scanResults = ArrayList<ScanResult>()
private var isScanning = false private var isScanning = false
// 是否是ota模式
// 是否是ota模式
var isEnterOta = false var isEnterOta = false
// 连接状态 // 连接状态
val connectionState = MutableLiveData(BleConst.STATE_DISCONNECTED) val connectionState = MutableLiveData(BleConst.STATE_DISCONNECTED)
@ -136,7 +140,7 @@ object BleService {
Log.e(TAG, "OpusManager初始化失败: ${e.message}", e) Log.e(TAG, "OpusManager初始化失败: ${e.message}", e)
// 根据需要决定是否因为Opus初始化失败而返回false // 根据需要决定是否因为Opus初始化失败而返回false
} }
recordfile = RecordingFile(this.context) recordfile = RecordingFile(this.context)
isInitialized = true isInitialized = true
Log.d(TAG, "BLE服务初始化成功") Log.d(TAG, "BLE服务初始化成功")
return true return true
@ -198,7 +202,6 @@ recordfile = RecordingFile(this.context)
// 获取一个局部引用,避免并发访问问题 // 获取一个局部引用,避免并发访问问题
val scanner = bluetoothLeScanner ?: return false val scanner = bluetoothLeScanner ?: return false
if (isScanning) return false if (isScanning) return false
Log.i(TAG, "开始主动扫描BLE设备...") Log.i(TAG, "开始主动扫描BLE设备...")
scanResults.clear() scanResults.clear()
@ -341,13 +344,12 @@ recordfile = RecordingFile(this.context)
Log.w(TAG, "未找到已配对的设备 MAC 地址") Log.w(TAG, "未找到已配对的设备 MAC 地址")
return return
} }
if(!isEnterOta) if (!isEnterOta) {
{ Log.i(TAG, "检测到已配对设备,MAC: $pairedMac,尝试连接")
Log.i(TAG, "检测到已配对设备,MAC: $pairedMac,尝试连接") // 调用现有的 connect 方法
// 调用现有的 connect 方法 connect(pairedMac)
connect(pairedMac)
} }
} catch (e: Exception) { } catch (e: Exception) {
@ -518,7 +520,7 @@ recordfile = RecordingFile(this.context)
when (c.uuid) { when (c.uuid) {
// 音频特征数据 // 音频特征数据
BleConst.RECEIVE_AUDIO_CHAR_UUID -> { BleConst.RECEIVE_AUDIO_CHAR_UUID -> {
// Log.i(TAG, "收到音频特征数据") // Log.i(TAG, "收到音频特征数据")
processAudioData(data) processAudioData(data)
} }
@ -597,7 +599,7 @@ recordfile = RecordingFile(this.context)
private fun processAudioData(data: ByteArray) { private fun processAudioData(data: ByteArray) {
try { try {
if (opusManager?.isDecodeStream == true) { if (opusManager?.isDecodeStream == true) {
recordfile?.saveAudioDataToWav(data) recordfile?.saveAudioDataToWav(data)
opusManager?.writeAudioStream(data) opusManager?.writeAudioStream(data)
} else { } else {
// Log.d(TAG, "Opus解码流未启动,忽略音频数据") // Log.d(TAG, "Opus解码流未启动,忽略音频数据")
@ -795,17 +797,16 @@ recordfile = RecordingFile(this.context)
BleConst.CODEC_CONTROL_ENCODE_ON -> "已打开编码" BleConst.CODEC_CONTROL_ENCODE_ON -> "已打开编码"
else -> "未知状态($codecStatus)" else -> "未知状态($codecStatus)"
} }
if (codecStatus == BleConst.CODEC_CONTROL_DECODE_ON || if (codecStatus == BleConst.CODEC_CONTROL_DECODE_ON ||
codecStatus == BleConst.CODEC_CONTROL_A2DP_PLAY || codecStatus == BleConst.CODEC_CONTROL_A2DP_PLAY ||
codecStatus == BleConst.CODEC_CONTROL_ENCODE_ON) { codecStatus == BleConst.CODEC_CONTROL_ENCODE_ON
) {
recordfile!!.closeFile()
recordfile!!.creatingFiles() recordfile!!.closeFile()
} recordfile!!.creatingFiles()
else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE) } else if (codecStatus == BleConst.CODEC_CONTROL_CLOSE) {
{ recordfile!!.closeFile()
recordfile!!.closeFile() }
}
// 声道模式描述 // 声道模式描述
val channelDesc = when (channelMode) { val channelDesc = when (channelMode) {
BleConst.AUDIO_CHANNEL_LEFT -> "左声道" BleConst.AUDIO_CHANNEL_LEFT -> "左声道"
@ -1093,7 +1094,7 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE)
// // The library might expect this in a different unit or derive it. // // The library might expect this in a different unit or derive it.
// // Given JlOpusPlugin.kt, packetSize refers to Opus encoded frame duration in ms. // // Given JlOpusPlugin.kt, packetSize refers to Opus encoded frame duration in ms.
// ) // )
startOpusStreamDecoding(false, 1, 16000, 40) startOpusStreamDecoding(false, 1, 16000, 40)
Log.i(TAG, "打开解码0xA2") Log.i(TAG, "打开解码0xA2")
return sendCommand( return sendCommand(
@ -1110,7 +1111,7 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE)
*/ */
fun openA2DPDecoder(): Boolean { fun openA2DPDecoder(): Boolean {
Log.i(TAG, "打开编码 0xA1") Log.i(TAG, "打开编码 0xA1")
startOpusStreamDecoding(false, 1, 16000, 80) startOpusStreamDecoding(false, 1, 16000, 80)
// Log.i(TAG, "打开解码...") // Log.i(TAG, "打开解码...")
return sendCommand( return sendCommand(
BleConst.CMD_CONTROL_CODEC.toByte(), byteArrayOf( BleConst.CMD_CONTROL_CODEC.toByte(), byteArrayOf(
@ -1135,7 +1136,7 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE)
fun openEncoder(): Boolean { fun openEncoder(): Boolean {
Log.i(TAG, "打开编码0xB1") Log.i(TAG, "打开编码0xB1")
startOpusStreamDecoding(false, 1, 16000, 40) startOpusStreamDecoding(false, 1, 16000, 40)
// 目前仅发送命令通知设备开始编码。 // 目前仅发送命令通知设备开始编码。
return sendCommand( return sendCommand(
BleConst.CMD_CONTROL_CODEC.toByte(), BleConst.CMD_CONTROL_CODEC.toByte(),
@ -1385,9 +1386,10 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE)
} }
} }
} }
class RecordingFile(private val context: Context) {
class RecordingFile(private val context: Context) {
private var currentAudioFile: File? = null private var currentAudioFile: File? = null
private var fos: FileOutputStream? = null private var fos: FileOutputStream? = null
@ -1408,8 +1410,9 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE)
// 创建新的音频文件 // 创建新的音频文件
val dateFormat = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.getDefault()) val dateFormat = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.getDefault())
val timestamp = dateFormat.format(Date()) val timestamp = dateFormat.format(Date())
val filePath = File(context.getExternalFilesDir(null), "${fileName}_${timestamp}.opus").absolutePath val filePath =
File(context.getExternalFilesDir(null), "${fileName}_${timestamp}.opus").absolutePath
currentAudioFile = File(filePath) currentAudioFile = File(filePath)
currentAudioFile?.createNewFile() currentAudioFile?.createNewFile()
// 追加音频数据到文件 // 追加音频数据到文件
@ -1417,42 +1420,42 @@ else if(codecStatus == BleConst.CODEC_CONTROL_CLOSE)
startWriteThread() // 新增:启动写入线程 startWriteThread() // 新增:启动写入线程
} }
// 新增:启动写入线程 // 新增:启动写入线程
private fun startWriteThread() { private fun startWriteThread() {
if (isWriting.get()) return if (isWriting.get()) return
isWriting.set(true) isWriting.set(true)
writeThread = Thread { writeThread = Thread {
try { try {
while (isWriting.get() || writeQueue.isNotEmpty()) { while (isWriting.get() || writeQueue.isNotEmpty()) {
val data = writeQueue.poll() ?: continue val data = writeQueue.poll() ?: continue
fos?.write(data) fos?.write(data)
}
} catch (e: Exception) {
Log.e("", "异步写入音频数据失败: ${e.message}")
} }
} catch (e: Exception) {
Log.e("", "异步写入音频数据失败: ${e.message}")
} }
writeThread?.start()
} }
writeThread?.start()
}
/** /**
* 保存音频数据到 WAV 文件(异步) * 保存音频数据到 WAV 文件(异步)
*/ */
internal fun saveAudioDataToWav(buffer: ByteArray) { internal fun saveAudioDataToWav(buffer: ByteArray) {
if (fos == null || currentAudioFile == null) return if (fos == null || currentAudioFile == null) return
// 放入队列,由写线程写入 // 放入队列,由写线程写入
// 1. 将字节数据转换为十六进制字符串 // 1. 将字节数据转换为十六进制字符串
// val hexData = buildString { // val hexData = buildString {
// buffer.forEachIndexed { index, byte -> // buffer.forEachIndexed { index, byte ->
// append("%02X".format(byte)) // append("%02X".format(byte))
// } // }
// } // }
// 2. 写入十六进制字符串 // 2. 写入十六进制字符串
writeQueue.offer(buffer) writeQueue.offer(buffer)
totalBytesWritten += buffer.size totalBytesWritten += buffer.size
} }
internal fun closeFile() { internal fun closeFile() {

459
local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt

@ -1,163 +1,268 @@
package com.deep_voice.bytedance_speech package com.deep_voice.bytedance_speech
import android.content.Context
import android.media.AudioAttributes import android.media.AudioAttributes
import android.media.AudioDeviceCallback
import android.media.AudioDeviceInfo
import android.media.AudioFormat import android.media.AudioFormat
import android.media.AudioManager import android.media.AudioManager
import android.media.AudioTrack import android.media.AudioTrack
import android.os.Build import android.os.Build
import android.os.Handler
import android.os.Looper
import android.util.Log import android.util.Log
import com.deep_voice.speech.tts.AudioDataListener import com.deep_voice.speech.tts.AudioDataListener
import kotlinx.coroutines.CoroutineScope import com.deep_voice.speech.tts.AudioOutputDevice
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancelChildren
import kotlinx.coroutines.delay
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.CancellationException
/** /**
* 字节跳动语音合成音频播放器 * 高性能音频播放器 - 16kHz采样率
* * 设计要点:
* 使用AudioTrack播放PCM格式音频流,基于Kotlin协程实现异步处理 * • 永远使用 USAGE_MEDIA + MODE_NORMAL 媒体播放通路
* • 利用 AudioDeviceCallback 监听设备热插拔
* • API 31+ 使用 setPreferredDevice() 精准路由
* • 有耳机自动从耳机播放,无耳机自动切到扬声器
*/ */
class BytedanceAudioPlayer : AudioDataListener, CoroutineScope { class BytedanceAudioPlayer(private val context: Context) : AudioDataListener, AudioDeviceCallback() {
companion object { companion object {
private const val TAG = "BytedanceAudioPlayer" private const val TAG = "BytedanceAudioPlayer"
private const val SAMPLE_RATE = 24000 // 样本率 private const val SAMPLE_RATE = 16000 // 固定16kHz采样率
private const val CHANNEL_CONFIG = AudioFormat.CHANNEL_OUT_MONO // 单声道 private const val CHANNEL_CONFIG = AudioFormat.CHANNEL_OUT_MONO
private const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT // 16位PCM private const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT
private const val IDLE_TIMEOUT_MS = 800L // 无数据超时时间
} }
// 协程相关 private val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
private val job = SupervisorJob()
override val coroutineContext = Dispatchers.IO + job
// 音频处理相关
private var audioTrack: AudioTrack? = null private var audioTrack: AudioTrack? = null
private val isPlaying = AtomicBoolean(false) private var isFirstData = true
private val isPaused = AtomicBoolean(false) private var totalBytesWritten = 0
private var sessionActive = true
// 播放状态
private val playStarted = AtomicBoolean(false)
private val completionReported = AtomicBoolean(false)
// 使用简单的并发队列,保证线程安全
private val audioDataQueue = ConcurrentLinkedQueue<ByteArray>()
// 播放状态监听 // 回调
private var onPlayStarted: (() -> Unit)? = null private var onPlayStarted: (() -> Unit)? = null
private var onPlayCompleted: (() -> Unit)? = null private var onPlayCompleted: (() -> Unit)? = null
private var onError: ((String) -> Unit)? = null
// 处理协程 // 耳机设备类型(优先级顺序)
private var processingJob: Job? = null private val headsetTypes = intArrayOf(
private var lastDataTime = 0L AudioDeviceInfo.TYPE_WIRED_HEADSET,
AudioDeviceInfo.TYPE_WIRED_HEADPHONES,
AudioDeviceInfo.TYPE_BLUETOOTH_A2DP
// TYPE_BLUETOOTH_LE 在较新的 API 中才有,这里先不包含
)
// 播放位置监听器
private val playbackListener = object : AudioTrack.OnPlaybackPositionUpdateListener {
override fun onMarkerReached(track: AudioTrack) {
onPlayCompleted?.invoke()
}
override fun onPeriodicNotification(track: AudioTrack) {
// 不使用周期性通知
}
}
init { init {
Log.d(TAG, "BytedanceAudioPlayer初始化") // 注册设备热插拔监听
audioManager.registerAudioDeviceCallback(this, Handler(Looper.getMainLooper()))
buildTrack(selectOutputDevice()) // 首次创建
} }
/** /**
* 开始播放 * 监听输出设备变化
*/ */
fun start() { override fun onAudioDevicesAdded(addedDevices: Array<AudioDeviceInfo>) {
if (isPlaying.getAndSet(true)) return Log.d(TAG, "音频设备连接: ${addedDevices.joinToString { "${it.productName}(${it.type})" }}")
rebuildIfNeeded()
isPaused.set(false) }
playStarted.set(false)
completionReported.set(false) override fun onAudioDevicesRemoved(removedDevices: Array<AudioDeviceInfo>) {
lastDataTime = System.currentTimeMillis() Log.d(TAG, "音频设备断开: ${removedDevices.joinToString { "${it.productName}(${it.type})" }}")
rebuildIfNeeded()
}
/**
* 检查设备变化,必要时重建AudioTrack
*/
private fun rebuildIfNeeded() {
val newDevice = selectOutputDevice()
val currentDevice = audioTrack?.preferredDevice
processingJob = launch { // 设备发生变化时重建
try { if (newDevice?.id != currentDevice?.id) {
initAudioTrack() Log.d(TAG, "设备切换: ${currentDevice?.productName} -> ${newDevice?.productName}")
// 处理队列中的音频数据 // 保存当前状态
while (isActive && isPlaying.get()) { val wasPlaying = audioTrack?.playState == AudioTrack.PLAYSTATE_PLAYING
processQueuedAudio() val savedFirstData = isFirstData
val savedTotalBytes = totalBytesWritten
// 检查是否播放完成 val savedSessionActive = sessionActive
checkPlaybackCompletion()
// 重建AudioTrack
// 队列为空时短暂延迟 buildTrack(newDevice)
if (audioDataQueue.isEmpty()) {
delay(10) // 恢复状态
} if (wasPlaying) {
} isFirstData = savedFirstData
} catch (e: CancellationException) { totalBytesWritten = savedTotalBytes
// 协程被取消,正常行为 sessionActive = savedSessionActive
} catch (e: Exception) {
Log.e(TAG, "播放错误: ${e.message}")
onError?.invoke("播放错误: ${e.message}")
stopInternal(false)
} }
} }
} }
/** /**
* 暂停播放 * 选择当前优先输出设备(耳机优先)
*/ */
fun pause() { private fun selectOutputDevice(): AudioDeviceInfo? {
if (!isPlaying.get() || isPaused.getAndSet(true)) return return audioManager.getDevices(AudioManager.GET_DEVICES_OUTPUTS)
audioTrack?.pause() .firstOrNull { it.type in headsetTypes }
} }
/** /**
* 恢复播放 * 构建16kHz单声道AudioTrack
*/ */
fun resume() { private fun buildTrack(device: AudioDeviceInfo?) {
if (!isPlaying.get() || !isPaused.getAndSet(false)) return audioTrack?.release()
audioTrack?.play()
val minBufferSize = AudioTrack.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT)
audioTrack = AudioTrack.Builder()
.setAudioAttributes(
AudioAttributes.Builder()
.setUsage(AudioAttributes.USAGE_MEDIA) // 统一媒体通路
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
.build()
)
.setAudioFormat(
AudioFormat.Builder()
.setSampleRate(SAMPLE_RATE)
.setEncoding(AUDIO_FORMAT)
.setChannelMask(CHANNEL_CONFIG)
.build()
)
.setTransferMode(AudioTrack.MODE_STREAM)
.setBufferSizeInBytes(minBufferSize * 2) // 小缓冲区保证低延迟
.apply {
// 低延迟模式
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
setPerformanceMode(AudioTrack.PERFORMANCE_MODE_LOW_LATENCY)
}
}
.build().also { track ->
// API 31+ 精准路由到指定设备
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && device != null) {
track.preferredDevice = device
Log.d(TAG, "设置首选设备: ${device.productName} (${device.type})")
}
// 设置播放监听器
track.setPlaybackPositionUpdateListener(playbackListener)
// 开始播放
track.play()
}
// 配置音频管理器 - 统一使用 MODE_NORMAL
audioManager.mode = AudioManager.MODE_NORMAL
// 切换扬声器开关仅在无耳机时打开
audioManager.isSpeakerphoneOn = (device == null)
val deviceName = device?.productName ?: "扬声器"
Log.d(TAG, "AudioTrack已创建 - 输出设备: $deviceName, 采样率: ${SAMPLE_RATE}Hz")
} }
/** /**
* 停止播放 * 开始新的播放会话
*/ */
fun stop() { fun startSession() {
Log.d(TAG, "停止播放") isFirstData = true
stopInternal(true) totalBytesWritten = 0
sessionActive = true
audioTrack?.let { track ->
if (track.state == AudioTrack.STATE_INITIALIZED) {
track.pause()
track.flush()
track.play()
}
} ?: buildTrack(selectOutputDevice())
} }
/** /**
* 内部停止处理 * 标记数据流结束
*/ */
private fun stopInternal(reportCompletion: Boolean) { fun endSession() {
if (!isPlaying.getAndSet(false)) return if (!sessionActive) return
isPaused.set(false) sessionActive = false
processingJob?.cancel()
audioTrack?.stop() audioTrack?.let { track ->
// 不在stopInternal中释放资源,只停止播放 if (totalBytesWritten > 0) {
val totalFrames = totalBytesWritten / 2
try {
track.setNotificationMarkerPosition(totalFrames)
} catch (e: Exception) {
val durationMs = (totalFrames * 1000L) / SAMPLE_RATE
Handler(Looper.getMainLooper()).postDelayed({
onPlayCompleted?.invoke()
}, durationMs + 100) // 更短的延迟
}
} else {
onPlayCompleted?.invoke()
}
}
}
/**
* 接收音频数据
*/
override fun onAudioData(data: ByteArray) {
if (data.isEmpty() || !sessionActive) return
if (audioTrack?.state != AudioTrack.STATE_INITIALIZED) return
// 清空队列 try {
audioDataQueue.clear() audioTrack?.let { track ->
val bytesWritten = track.write(data, 0, data.size)
if (bytesWritten > 0) {
if (isFirstData) {
isFirstData = false
onPlayStarted?.invoke()
}
if (sessionActive) {
totalBytesWritten += bytesWritten
}
}
}
} catch (e: Exception) {
Log.w(TAG, "音频数据写入失败: ${e.message}")
}
}
/**
* 停止播放
*/
fun stop() {
sessionActive = false
// 播放完成通知 audioTrack?.let { track ->
if (reportCompletion && playStarted.get() && !completionReported.getAndSet(true)) { if (track.state == AudioTrack.STATE_INITIALIZED) {
launch(Dispatchers.Main) { track.pause()
onPlayCompleted?.invoke() track.flush()
} }
} }
if (!isFirstData) {
onPlayCompleted?.invoke()
}
} }
/** /**
* 释放资源 * 释放资源
*/ */
fun release() { fun release() {
stopInternal(false) stop()
// 在release方法中释放AudioTrack资源
audioTrack?.release() audioTrack?.release()
audioTrack = null audioTrack = null
audioManager.unregisterAudioDeviceCallback(this)
// 释放协程资源
job.cancel()
} }
/** /**
@ -175,155 +280,45 @@ class BytedanceAudioPlayer : AudioDataListener, CoroutineScope {
} }
/** /**
* 设置错误回调 * 设置错误回调(兼容接口)
*/ */
fun setOnError(listener: (String) -> Unit) { fun setOnError(listener: (String) -> Unit) {
onError = listener // 不实现,仅为兼容
}
/**
* 接收音频数据(实现AudioDataListener接口)
*/
override fun onAudioData(data: ByteArray) {
if (data.isEmpty()) return
lastDataTime = System.currentTimeMillis()
if (!isPlaying.get()) {
start()
}
// 添加到队列 - 简单有效,保证FIFO顺序
audioDataQueue.add(data.copyOf())
} }
/** /**
* 检查是否正在播放 * 检查是否正在播放
*/ */
fun isPlaying(): Boolean = isPlaying.get() && !isPaused.get() fun isPlaying(): Boolean {
return audioTrack?.playState == AudioTrack.PLAYSTATE_PLAYING
/**
* 处理队列中的音频数据
*/
private suspend fun processQueuedAudio() {
if (isPaused.get() || !isPlaying.get() || !isActive) return
// 获取并处理队列中的数据
val data = audioDataQueue.poll() ?: return
// 写入音频数据
val result = audioTrack?.write(data, 0, data.size) ?: 0
if (result > 0) {
// 标记播放开始
if (!playStarted.getAndSet(true)) {
withContext(Dispatchers.Main) {
onPlayStarted?.invoke()
}
}
} else if (result < 0) {
// 处理错误
handleAudioTrackError(result)
}
} }
/** /**
* 检查播放是否完成(超时无数据) * 设置音频输出设备(兼容接口)
* 注意:此实现依赖自动设备切换,手动设置可能被热插拔覆盖
*/ */
private suspend fun checkPlaybackCompletion() { fun setAudioOutputDevice(device: AudioOutputDevice) {
val currentTime = System.currentTimeMillis() // 本实现专注于自动设备切换
// 手动设置会在下次设备变化时被自动逻辑覆盖
// 超时判断 - 队列为空且超过超时时间 Log.d(TAG, "setAudioOutputDevice被调用,但本实现依赖自动设备切换")
if (playStarted.get() && audioDataQueue.isEmpty() &&
currentTime - lastDataTime > IDLE_TIMEOUT_MS && isPlaying.get()) {
// 播放完成
if (!completionReported.getAndSet(true)) {
withContext(Dispatchers.Main) {
onPlayCompleted?.invoke()
}
stopInternal(false) // 已报告,不需要再次报告
}
}
} }
/** /**
* 处理AudioTrack错误 * 获取当前路由设备信息(用于调试)
*/ */
private fun handleAudioTrackError(result: Int) { fun getCurrentDevice(): String {
val errorMsg = when(result) { val routedDevice = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
AudioTrack.ERROR_INVALID_OPERATION -> "无效操作" audioTrack?.routedDevice
AudioTrack.ERROR_BAD_VALUE -> "参数错误"
AudioTrack.ERROR_DEAD_OBJECT -> "对象已销毁"
else -> "未知错误"
}
// 对象已销毁时尝试重建
if (result == AudioTrack.ERROR_DEAD_OBJECT) {
try {
audioTrack?.release()
initAudioTrack()
} catch (e: Exception) {
Log.e(TAG, "重建AudioTrack失败: ${e.message}")
throw e
}
} else { } else {
Log.e(TAG, "AudioTrack错误: $errorMsg") null
} }
}
/**
* 初始化AudioTrack
*/
private fun initAudioTrack() {
val minBufferSize = AudioTrack.getMinBufferSize(
SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT
)
if (minBufferSize == AudioTrack.ERROR || minBufferSize == AudioTrack.ERROR_BAD_VALUE) { val preferredDevice = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
throw IllegalStateException("无法获取有效的音频缓冲区大小") audioTrack?.preferredDevice
}
// 使用较大的缓冲区提高稳定性
val bufferSize = minBufferSize * 4
// 创建AudioTrack
audioTrack = if (Build.VERSION.SDK_INT >= 23) { // Android M (6.0)
AudioTrack.Builder()
.setAudioAttributes(
AudioAttributes.Builder()
.setUsage(AudioAttributes.USAGE_MEDIA)
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
.build()
)
.setAudioFormat(
AudioFormat.Builder()
.setEncoding(AUDIO_FORMAT)
.setSampleRate(SAMPLE_RATE)
.setChannelMask(CHANNEL_CONFIG)
.build()
)
.setBufferSizeInBytes(bufferSize)
.setTransferMode(AudioTrack.MODE_STREAM)
.build()
} else { } else {
@Suppress("DEPRECATION") null
AudioTrack(
AudioManager.STREAM_MUSIC,
SAMPLE_RATE,
CHANNEL_CONFIG,
AUDIO_FORMAT,
bufferSize,
AudioTrack.MODE_STREAM
)
}
// 检查初始化状态
if (audioTrack?.state != AudioTrack.STATE_INITIALIZED) {
throw IllegalStateException("AudioTrack初始化失败")
} }
// 开始播放 return "路由设备: ${routedDevice?.productName}, 首选设备: ${preferredDevice?.productName}"
audioTrack?.play()
} }
} }

6
local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceSpeechPlugin.kt

@ -255,6 +255,12 @@ class BytedanceSpeechPlugin : FlutterPlugin {
TtsEventType.SYNTHESIS_CANCELED -> { TtsEventType.SYNTHESIS_CANCELED -> {
sendTTSEvent("canceled", null) sendTTSEvent("canceled", null)
} }
TtsEventType.PLAYBACK_STARTED -> {
sendTTSEvent("playback_started", null)
}
TtsEventType.PLAYBACK_COMPLETED -> {
sendTTSEvent("playback_completed", null)
}
TtsEventType.ERROR -> { TtsEventType.ERROR -> {
val params = HashMap<String, Any>() val params = HashMap<String, Any>()
params["code"] = event.params["errorCode"] ?: "UNKNOWN_ERROR" params["code"] = event.params["errorCode"] ?: "UNKNOWN_ERROR"

201
local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceTTS.kt

@ -3,6 +3,7 @@ package com.deep_voice.bytedance_speech
import android.content.Context import android.content.Context
import android.util.Log import android.util.Log
import com.deep_voice.speech.tts.AudioDataListener import com.deep_voice.speech.tts.AudioDataListener
import com.deep_voice.speech.tts.AudioOutputDevice
import com.deep_voice.speech.tts.ITtsService import com.deep_voice.speech.tts.ITtsService
import com.deep_voice.speech.tts.TtsEvent import com.deep_voice.speech.tts.TtsEvent
import com.deep_voice.speech.tts.TtsEventListener import com.deep_voice.speech.tts.TtsEventListener
@ -98,7 +99,10 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
private val audioDataListeners = mutableListOf<AudioDataListener>() private val audioDataListeners = mutableListOf<AudioDataListener>()
// 内部音频播放器 // 内部音频播放器
private val audioPlayer = BytedanceAudioPlayer() private val audioPlayer = BytedanceAudioPlayer(context)
// 是否使用内部播放器
private var useInternalPlayer = true
// WebSocket连接 // WebSocket连接
private var webSocket: WebSocket? = null private var webSocket: WebSocket? = null
@ -113,15 +117,13 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
private var appId: String = "" private var appId: String = ""
private var accessToken: String = "" private var accessToken: String = ""
private var speaker: String = "zh_female_shuangkuaisisi_emo_v2_mars_bigtts" // 默认语音,使用Java示例的音色 private var speaker: String = "zh_female_shuangkuaisisi_emo_v2_mars_bigtts" // 默认语音,使用Java示例的音色
private var format: String = "pcm" // 使用mp3格式 private var format: String = "pcm" // pcm 格式
private var sampleRate: Int = 24000 // 使用24kHz采样率 private var sampleRate: Int = 16000 // 使用16kHz采样率
private var enableTimestamp: Boolean = false private var enableTimestamp: Boolean = false
private var language: String = "zh-CN" private var language: String = "zh-CN"
// 流式处理状态 // 流式处理状态
private var isStreamMode = false
private var currentStatus = STATUS_STOPPED private var currentStatus = STATUS_STOPPED
private var isSpeaking = false
// 连接相关状态 // 连接相关状态
private var connectionAttempts = 0 private var connectionAttempts = 0
@ -155,30 +157,27 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
* 初始化音频播放器回调 * 初始化音频播放器回调
*/ */
private fun initAudioPlayerCallbacks() { private fun initAudioPlayerCallbacks() {
// 添加播放器作为音频数据监听器
addAudioDataListener(audioPlayer)
audioPlayer.setOnPlayStarted { audioPlayer.setOnPlayStarted {
Log.d(TAG, "播放开始") Log.d(TAG, "播放开始")
isSpeaking = true
updateStatus(STATUS_SPEAKING) updateStatus(STATUS_SPEAKING)
notifyEvent(TtsEventType.SYNTHESIS_STARTED) notifyEvent(TtsEventType.PLAYBACK_STARTED)
} }
// 设置播放完成回调 // 设置播放完成回调
audioPlayer.setOnPlayCompleted { audioPlayer.setOnPlayCompleted {
Log.d(TAG, "播放结束") Log.d(TAG, "播放结束")
isSpeaking = false updateStatus(STATUS_READY)
updateStatus(if (isStreamMode) STATUS_READY else STATUS_STOPPED) notifyEvent(TtsEventType.PLAYBACK_COMPLETED)
notifyEvent(TtsEventType.SYNTHESIS_COMPLETED)
} }
// 设置错误回调 // 设置错误回调
audioPlayer.setOnError { errorMsg -> audioPlayer.setOnError { errorMsg ->
isSpeaking = false
updateStatus(STATUS_ERROR) updateStatus(STATUS_ERROR)
notifyEvent(TtsEventType.ERROR, mapOf("errorCode" to "PLAYER_ERROR", "errorMessage" to errorMsg)) notifyEvent(TtsEventType.ERROR, mapOf("errorCode" to "PLAYER_ERROR", "errorMessage" to errorMsg))
} }
// 根据设置决定是否使用内部播放器
updateInternalPlayerUsage()
} }
/** /**
@ -208,16 +207,19 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
*/ */
override fun stop(): Boolean { override fun stop(): Boolean {
try { try {
audioPlayer.stop() Log.i(TAG, ">stop()")
if (useInternalPlayer) {
audioPlayer.stop()
}
textProcessingJob?.cancel() textProcessingJob?.cancel()
launch {
if (isSessionStarted) { if (isSessionStarted) {
finishSession() finishSession()
}
isSpeaking = false // 显式设置,因为这是强制停止
updateStatus(STATUS_STOPPED)
} }
updateStatus(STATUS_STOPPED)
return true return true
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "停止合成失败: ${e.message}") Log.e(TAG, "停止合成失败: ${e.message}")
@ -277,7 +279,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
isConnected = false isConnected = false
isSessionStarted = false isSessionStarted = false
isSpeaking = false
updateStatus(STATUS_STOPPED) updateStatus(STATUS_STOPPED)
startTextProcessing() startTextProcessing()
@ -304,29 +305,9 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
* 单次播放(非流式) * 单次播放(非流式)
*/ */
override fun speakOnce(text: String): Boolean { override fun speakOnce(text: String): Boolean {
try { // 未实现:只支持流式模式
if (text.isEmpty()) { Log.w(TAG, "speakOnce未实现,请使用speakStream")
return false return false
}
isStreamMode = false
launch {
if (!ensureSessionReady()) {
notifyEvent(TtsEventType.ERROR, mapOf("errorCode" to "SESSION_ERROR",
"errorMessage" to "无法建立会话"))
return@launch
}
sendTextRequest(text)
}
return true
} catch (e: Exception) {
Log.e(TAG, "语音合成失败: ${e.message}")
updateStatus(STATUS_ERROR)
return false
}
} }
/** /**
@ -337,7 +318,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
if (text.isEmpty()) { if (text.isEmpty()) {
return true return true
} }
isStreamMode = true
// 确保会话已就绪 // 确保会话已就绪
if (!isSessionStarted && !ensureSessionReady()) { if (!isSessionStarted && !ensureSessionReady()) {
@ -365,17 +345,11 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
*/ */
override fun flushStream(): Boolean { override fun flushStream(): Boolean {
try { try {
if (!isStreamMode) return true
// Log.d(TAG, "flushStream: $sessionId") // Log.d(TAG, "flushStream: $sessionId")
launch {
withContext(Dispatchers.IO) { if (isSessionStarted) {
delay(300) // 确保现有文本处理完成 isSessionStarted = false
} finishSession()
if (isSessionStarted) {
isSessionStarted = false
finishSession()
}
} }
return true return true
@ -440,6 +414,38 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
audioDataListeners.remove(listener) audioDataListeners.remove(listener)
} }
/**
* 设置是否使用内部播放器
*/
override fun setUseInternalPlayer(useInternalPlayer: Boolean) {
this.useInternalPlayer = useInternalPlayer
updateInternalPlayerUsage()
}
/**
* 设置音频输出设备
*/
override fun setAudioOutputDevice(device: AudioOutputDevice) {
if (useInternalPlayer) {
audioPlayer.setAudioOutputDevice(device)
}
}
/**
* 更新内部播放器的使用状态
*/
private fun updateInternalPlayerUsage() {
if (useInternalPlayer) {
// 使用内部播放器
if (!audioDataListeners.contains(audioPlayer)) {
audioDataListeners.add(audioPlayer)
}
} else {
// 不使用内部播放器
audioDataListeners.remove(audioPlayer)
}
}
/** /**
* 通知事件处理 * 通知事件处理
*/ */
@ -461,15 +467,14 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
* 通知音频数据 * 通知音频数据
*/ */
private fun notifyAudioData(data: ByteArray) { private fun notifyAudioData(data: ByteArray) {
launch(Dispatchers.Main) { for (listener in audioDataListeners) {
for (listener in audioDataListeners) { try {
try { listener.onAudioData(data)
listener.onAudioData(data) } catch (e: Exception) {
} catch (e: Exception) { Log.e(TAG, "音频数据回调异常: ${e.message}")
Log.e(TAG, "音频数据回调异常: ${e.message}")
}
} }
} }
} }
/** /**
@ -483,7 +488,7 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
// 如果已连接但会话未开始,只需要开始会话 // 如果已连接但会话未开始,只需要开始会话
if (isConnected && !isSessionStarted) { if (isConnected && !isSessionStarted) {
Log.d(TAG, "WebSocket已连接,正在启动新会话...") // Log.d(TAG, "WebSocket已连接,正在启动新会话...")
return startSessionOnly() return startSessionOnly()
} }
@ -648,7 +653,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
"errorMessage" to errorMsg)) "errorMessage" to errorMsg))
webSocket.close(1000, "Error") webSocket.close(1000, "Error")
updateStatus(STATUS_ERROR) updateStatus(STATUS_ERROR)
isSpeaking = false // 错误情况下显式设置
isSessionStarted = false isSessionStarted = false
isConnected = false isConnected = false
isConnecting.set(false) isConnecting.set(false)
@ -656,6 +660,12 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
EVENT_SESSION_STARTED -> { EVENT_SESSION_STARTED -> {
connectionAttempts = 0 connectionAttempts = 0
// 会话开始时触发合成开始事件
notifyEvent(TtsEventType.SYNTHESIS_STARTED)
// 如果使用内部播放器则启动播放器会话
if (useInternalPlayer) {
audioPlayer.startSession()
}
} }
EVENT_TTS_RESPONSE -> { EVENT_TTS_RESPONSE -> {
@ -673,16 +683,22 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
EVENT_SESSION_FINISHED -> { EVENT_SESSION_FINISHED -> {
isSessionStarted = false isSessionStarted = false
Log.i(TAG, "EVENT_SESSION_FINISHED, ${sessionId}")
// 触发合成完成事件
notifyEvent(TtsEventType.SYNTHESIS_COMPLETED)
if (!isStreamMode) { // 如果使用内部播放器则结束播放器会话
finishConnection(webSocket) if (useInternalPlayer) {
audioPlayer.endSession()
} }
// 流式模式下不自动关闭连接
} }
} }
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "解析响应失败: ${e.message}") Log.e(TAG, "解析响应失败: ${e.message}")
updateStatus(STATUS_ERROR) updateStatus(STATUS_ERROR)
isSpeaking = false // 错误情况下显式设置
isConnecting.set(false) isConnecting.set(false)
} }
} }
@ -690,7 +706,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
isConnected = false isConnected = false
isSessionStarted = false isSessionStarted = false
isSpeaking = false
isConnecting.set(false) isConnecting.set(false)
Log.e(TAG, "WebSocket连接失败: ${t.message}") Log.e(TAG, "WebSocket连接失败: ${t.message}")
@ -702,7 +717,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
isConnected = false isConnected = false
isSessionStarted = false isSessionStarted = false
isSpeaking = false
isConnecting.set(false) isConnecting.set(false)
updateStatus(STATUS_STOPPED) updateStatus(STATUS_STOPPED)
notifyEvent(TtsEventType.SYNTHESIS_CANCELED) notifyEvent(TtsEventType.SYNTHESIS_CANCELED)
@ -798,53 +812,6 @@ class BytedanceTTS(private val context: Context) : ITtsService, CoroutineScope {
sendEvent(webSocket, header, optional, payload) sendEvent(webSocket, header, optional, payload)
} }
/**
* 发送文本进行合成 (非流式)
*/
private fun sendTextRequest(text: String) {
webSocket?.let { ws ->
val header = Header(
protocolVersion = PROTOCOL_VERSION,
headerSize = DEFAULT_HEADER_SIZE,
messageType = FULL_CLIENT_REQUEST,
messageTypeSpecificFlags = MSG_TYPE_FLAG_WITH_EVENT,
serializationMethod = JSON,
messageCompression = COMPRESSION_NO,
reserved = 0
)
val optional = Optional(
event = EVENT_TASK_REQUEST,
sessionId = sessionId
)
val jsonObject = JSONObject()
val user = JSONObject()
user.put("uid", "123456")
jsonObject.put("user", user)
jsonObject.put("event", EVENT_TASK_REQUEST)
jsonObject.put("namespace", "BidirectionalTTS")
val reqParams = JSONObject()
// 将#和*替换为空格
val processedText = text.replace("#", " ").replace("*", " ")
reqParams.put("text", processedText)
reqParams.put("speaker", speaker)
val audioParams = JSONObject()
audioParams.put("format", format)
audioParams.put("sample_rate", sampleRate)
audioParams.put("emotion", "happy")
reqParams.put("audio_params", audioParams)
jsonObject.put("req_params", reqParams)
val payload = jsonObject.toString().toByteArray()
Log.d(TAG, "发送文本合成请求: ${jsonObject.toString()}")
sendEvent(ws, header, optional, payload)
}
}
/** /**
* 发送文本进行合成 (流式) * 发送文本进行合成 (流式)

12
local_plugins/bytedance_speech/lib/bytedance_speech.dart

@ -0,0 +1,12 @@
library bytedance_speech;
import 'package:flutter/services.dart';
class BytedanceSpeech {
static const MethodChannel _channel = MethodChannel('bytedance_speech');
static Future<String?> getPlatformVersion() async {
final String? version = await _channel.invokeMethod('getPlatformVersion');
return version;
}
}

299
local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt

@ -24,6 +24,7 @@ import org.json.JSONObject
import android.os.Handler import android.os.Handler
import android.os.Looper import android.os.Looper
import java.util.UUID import java.util.UUID
import java.util.Collections
/** /**
* ChatAPI服务异常 * ChatAPI服务异常
@ -70,8 +71,32 @@ private data class ToolCallInfo(
var arguments: String = "" var arguments: String = ""
) { ) {
fun isValid(): Boolean { fun isValid(): Boolean {
// 确保参数是完整的 JSON
return id.isNotEmpty() && name.isNotEmpty() &&
arguments.isNotEmpty() && isCompleteJson(arguments)
}
private fun isCompleteJson(json: String): Boolean {
return try { return try {
id.isNotEmpty() && name.isNotEmpty() // 检查 JSON 是否完整(括号匹配)
val trimmed = json.trim()
if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) {
return false
}
// 简单的括号匹配检查
var braceCount = 0
for (char in trimmed) {
when (char) {
'{' -> braceCount++
'}' -> braceCount--
}
if (braceCount < 0) return false
}
if (braceCount != 0) return false
// 验证是否是有效的 JSON
JSONObject(json)
true
} catch (e: Exception) { } catch (e: Exception) {
false false
} }
@ -86,6 +111,10 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
override val coroutineContext: CoroutineContext = Dispatchers.IO + SupervisorJob() override val coroutineContext: CoroutineContext = Dispatchers.IO + SupervisorJob()
companion object {
private const val TAG = "ChatApiService"
}
// MARK: - 属性 // MARK: - 属性
private var baseUrl = "https://api.openai.com/v1/" private var baseUrl = "https://api.openai.com/v1/"
private var apiKey = "" private var apiKey = ""
@ -110,12 +139,10 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
// 流式请求相关 // 流式请求相关
private var currentStreamJob: Job? = null private var currentStreamJob: Job? = null
// private var streamCallback: StreamCallback? = null
private var currentMessages: List<Map<String, Any>> = emptyList() private var currentMessages: List<Map<String, Any>> = emptyList()
private var toolCalls: MutableMap<Int, ToolCallInfo> = mutableMapOf() private var toolCalls: MutableMap<Int, ToolCallInfo> = Collections.synchronizedMap(mutableMapOf())
private var sessions: MutableMap<String, StreamCallback?> = mutableMapOf() private var currSessionId = ""
private var currSessionId = "" // private var currSessionCallback: StreamCallback? = null
// private var isCanceled = false
@ -125,11 +152,34 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
// MARK: - 公共方法 // MARK: - 公共方法
/** /**
* 设置流式回调 * 中止当前会话
*/ */
// fun setStreamCallback(callback: StreamCallback) { private fun abortCurrentSession() {
// streamCallback = callback if (currSessionId.isNotEmpty()) {
// } // 1. 取消当前的协程任务
currentStreamJob?.cancel()
currentStreamJob = null
// 2. 通知旧会话被中止
// currSessionCallback?.onError(ChatApiException("Session aborted by new request"))
currSessionCallback?.onComplete() // 直接完成当前会话
// 3. 清理状态
currSessionId = ""
currSessionCallback = null
toolCalls.clear()
}
}
/**
* 清理当前会话
*/
private fun clearCurrentSession() {
currSessionId = ""
currSessionCallback = null
currentStreamJob = null
toolCalls.clear()
}
/** /**
* 初始化ChatAPI服务 * 初始化ChatAPI服务
@ -173,6 +223,7 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
// 异步初始化MCP客户端 // 异步初始化MCP客户端
launch { launch {
initializeMcpClient(mcpServer) initializeMcpClient(mcpServer)
// initializeMcpClient("{}")
} }
isInitialized = apiKey.isNotEmpty() isInitialized = apiKey.isNotEmpty()
@ -262,29 +313,34 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
* 发送消息(流式输出)- 兼容JSONArray格式 * 发送消息(流式输出)- 兼容JSONArray格式
*/ */
fun sendMessageStream(messages: org.json.JSONArray, callback: StreamCallback) { fun sendMessageStream(messages: org.json.JSONArray, callback: StreamCallback) {
// 设置回调 // 1. 立即中止旧会话
// setStreamCallback(callback) abortCurrentSession()
// 转换JSONArray为List<Map<String, Any>> // 2. 创建新会话
val messageList = convertJsonArrayToList(messages)
currSessionId = UUID.randomUUID().toString() currSessionId = UUID.randomUUID().toString()
sessions[currSessionId] = callback currSessionCallback = callback
// 调用原始方法
sendMessageStream(currSessionId,messageList) // 3. 转换消息并开始处理
val messageList = convertJsonArrayToList(messages)
sendMessageStream(currSessionId, messageList)
} }
/** /**
* 发送消息(流式输出) * 发送消息(流式输出)
*/ */
fun sendMessageStream(sessionid:String,messages: List<Map<String, Any>>) { fun sendMessageStream(sessionid:String,messages: List<Map<String, Any>>) {
// 验证是否是当前会话
if (sessionid != currSessionId) {
return // 忽略非当前会话的请求
}
if (!isInitialized || apiKey.isEmpty() || openAI == null) { if (!isInitialized || apiKey.isEmpty() || openAI == null) {
Log.e("ChatApiService", "ChatAPI服务未初始化,无法发送消息") Log.e("ChatApiService", "ChatAPI服务未初始化,无法发送消息")
sessions[sessionid]?.onError(ChatApiException("ChatAPI服务未初始化")) currSessionCallback?.onError(ChatApiException("ChatAPI服务未初始化"))
return return
} }
// 重置状态 // 重置状态
// isCanceled = false
currentMessages = messages currentMessages = messages
toolCalls.clear() toolCalls.clear()
// 检查是否包含图片,决定使用哪个模型 // 检查是否包含图片,决定使用哪个模型
@ -337,79 +393,93 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
} }
chatsFlow.collect { result -> chatsFlow.collect { result ->
if (currSessionId != sessionid) { // 每次处理前检查是否仍是当前会话
return@collect if (sessionid != currSessionId) {
throw CancellationException("Session cancelled")
} }
val choice = result.choices.firstOrNull() val choice = result.choices.firstOrNull()
if (choice == null) { if (choice == null) {
Log.e("ChatApiService", "choice为null") Log.w("ChatApiService", "[Session: $sessionid] choice为null")
return@collect return@collect
} }
val delta = choice.delta val delta = choice.delta
if (delta == null) { if (delta == null) {
Log.e("ChatApiService", "delta为null") Log.w("ChatApiService", "[Session: $sessionid] delta为null")
return@collect return@collect
} }
// 处理普通文本内容 // 处理普通文本内容
delta.content?.let { content -> delta.content?.let { content ->
sessions[sessionid]?.onToken(content) currSessionCallback?.onToken(content)
} }
// 收集工具调用信息 // 收集工具调用信息
delta.toolCalls?.forEach { toolCall -> delta.toolCalls?.forEach { toolCall ->
val index = toolCall.index
// 创建或获取现有的工具调用信息
val toolCallInfo = toolCalls.getOrPut(index) { ToolCallInfo() }
// 安全处理工具调用ID
try { try {
toolCall.id?.let { i -> val index = toolCall.index
toolCallInfo.id = i.toString() val toolCallInfo = toolCalls.getOrPut(index) { ToolCallInfo() }
// 使用安全调用链处理工具调用信息
toolCall.id?.toString()?.takeIf { it.isNotEmpty() }?.let {
toolCallInfo.id = it
} }
} catch (e: Exception) {
Log.e("ChatApiService", "处理工具调用ID异常: ${e.message}")
}
// 安全处理函数信息
try {
toolCall.function?.let { function -> toolCall.function?.let { function ->
// 安全处理函数名称 - 使用 try-catch 避免 null 属性访问
try { try {
function.name.let { name -> val functionName = function.name
toolCallInfo.name = name if (!functionName.isNullOrEmpty()) {
toolCallInfo.name = functionName
} }
} catch (e: Exception) { } catch (e: Exception) {
Log.e("ChatApiService", "处理工具调用函数名称异常! function:${toolCall.function} err:${e.message}") // 在流式响应中,name 可能在后续的 delta 中为 null,这是正常的
// 不记录日志,避免干扰
} }
// 安全处理函数参数 - 使用 try-catch 避免 null 属性访问
try { try {
function.arguments.let { args -> val functionArgs = function.arguments
toolCallInfo.arguments += args if (!functionArgs.isNullOrEmpty()) {
toolCallInfo.arguments += functionArgs
} }
} catch (e: Exception) { } catch (e: Exception) {
Log.e("ChatApiService", "处理工具调用参数异常: ${e.message}") // 在流式响应中,arguments 可能在后续的 delta 中为 null,这是正常的
// 不记录日志,避免干扰
} }
Log.e("ChatApiService", "AI发起工具调用:${toolCallInfo.name} ${toolCallInfo.arguments}")
// 不设置默认值,让参数自然累积
}
// 只在有完整有效数据时记录日志(避免流式响应过程中的重复日志)
if (toolCallInfo.isValid()) {
Log.d("ChatApiService", "[Session: $sessionid] AI发起工具调用: ${toolCallInfo.name}, 参数: ${toolCallInfo.arguments}")
} }
} catch (e: Exception) { } catch (e: Exception) {
Log.e("ChatApiService", "处理工具调用函数信息异常: ${e.message}") Log.w("ChatApiService", "[Session: $sessionid] 处理工具调用信息异常: ${e.message}", e)
} }
} }
} }
if (currSessionId == sessionid) { // 完成处理
if (sessionid == currSessionId) {
// 检查是否有工具调用需要处理 // 检查是否有工具调用需要处理
val hasToolCalls = processToolCalls(sessionid) val hasToolCalls = processToolCalls(sessionid)
if (!hasToolCalls) { if (!hasToolCalls) {
sessions[sessionid]?.onComplete() currSessionCallback?.onComplete()
clearCurrentSession()
} }
}else{
sessions[sessionid]?.onComplete()
} }
} catch (e: CancellationException) {
// 会话被取消,不需要处理
Log.d("ChatApiService", "Session $sessionid cancelled")
} catch (e: Exception) { } catch (e: Exception) {
// if (currSessionId == sessionid) { if (sessionid == currSessionId) {
sessions[sessionid]?.onError(ChatApiException("流式请求失败: ${e.message}")) currSessionCallback?.onError(ChatApiException("流式请求失败: ${e.message}"))
// } clearCurrentSession()
}
} }
} }
} }
@ -418,42 +488,54 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
* 处理工具调用 * 处理工具调用
*/ */
private suspend fun processToolCalls(sessionid: String): Boolean { private suspend fun processToolCalls(sessionid: String): Boolean {
val firstToolCall = try { // 验证工具调用集合不为空
toolCalls.values.firstOrNull { it.isValid() } if (toolCalls.isEmpty()) {
} catch (e: Exception) { Log.d("ChatApiService", "[Session: $sessionid] 没有工具调用需要处理")
Log.e("ChatApiService", "查找有效工具调用异常: ${e.message}")
null
}
if (firstToolCall == null) {
return false return false
} }
// 创建函数调用字典
val functionCall = try { // 输出所有收集到的工具调用信息(用于调试)
mapOf( Log.d("ChatApiService", "[Session: $sessionid] 收集到 ${toolCalls.size} 个工具调用")
"name" to (firstToolCall.name.takeIf { it.isNotEmpty() } ?: ""), toolCalls.forEach { (index, toolCall) ->
"arguments" to (firstToolCall.arguments.takeIf { it.isNotEmpty() } ?: "{}"), Log.d("ChatApiService", "[Session: $sessionid] 工具调用[$index]: name=${toolCall.name}, args.length=${toolCall.arguments.length}, args=${toolCall.arguments.take(100)}...")
"id" to (firstToolCall.id.takeIf { it.isNotEmpty() } ?: "") }
)
} catch (e: Exception) { // 只处理有效的工具调用
Log.e("ChatApiService", "创建函数调用字典异常: ${e.message}") val validToolCalls = toolCalls.values.filter { it.isValid() }
if (validToolCalls.isEmpty()) {
Log.w("ChatApiService", "[Session: $sessionid] 没有有效的工具调用(参数可能不完整)")
return false return false
} }
// 处理第一个有效的工具调用
val firstToolCall = validToolCalls.first()
Log.d("ChatApiService", "[Session: $sessionid] 处理工具调用: ${firstToolCall.name},参数: ${firstToolCall.arguments}")
// 创建函数调用字典
val functionCall = mapOf(
"name" to firstToolCall.name,
"arguments" to firstToolCall.arguments,
"id" to firstToolCall.id
)
// 通知上层工具调用事件 // 通知上层工具调用事件
sessions[sessionid]?.onFunctionCall(convertMapToJsonObject(functionCall)) currSessionCallback?.onFunctionCall(convertMapToJsonObject(functionCall))
// 在后台队列处理工具调用 // 在当前协程作用域内处理工具调用,使用async确保生命周期管理
launch { val toolCallDeferred = async {
try { try {
if (currSessionId == sessionid) { if (sessionid == currSessionId) {
// 通过MCP客户端处理工具调用 // 通过MCP客户端处理工具调用
val functionName = firstToolCall.name val functionName = firstToolCall.name
val argumentsJson = firstToolCall.arguments val argumentsJson = firstToolCall.arguments
val result = if (_mcpClient?.hasToolWithName(functionName) == true) { val result = if (_mcpClient?.hasToolWithName(functionName) == true) {
// 解析参数 // 解析参数,如果参数为空JSON对象,使用空Map
val arguments = _mcpClient?.parseJsonArguments(argumentsJson) ?: emptyMap() val arguments = if (argumentsJson == "{}") {
emptyMap()
} else {
_mcpClient?.parseJsonArguments(argumentsJson) ?: emptyMap()
}
// 调用MCP工具 // 调用MCP工具
val toolResult = _mcpClient?.callTool(functionName, arguments) val toolResult = _mcpClient?.callTool(functionName, arguments)
Log.e("ChatApiService", "AI调用工具结果: $functionName}:$arguments} toolResult:${toolResult}") Log.d("ChatApiService", "[Session: $sessionid] AI调用工具结果: $functionName, 参数: $arguments, 结果: $toolResult")
// 处理结果 // 处理结果
if (toolResult != null) { if (toolResult != null) {
if (toolResult["isError"] == true) { if (toolResult["isError"] == true) {
@ -473,16 +555,16 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
mapOf("context" to text) mapOf("context" to text)
} }
} else { } else {
Log.e("ChatApiService", "MCP工具调用返回null") Log.w("ChatApiService", "[Session: $sessionid] MCP工具调用返回null")
mapOf("context" to "Tool call failed") mapOf("context" to "Tool call failed")
} }
} else { } else {
// 工具不存在 // 工具不存在
mapOf("context" to "Tool not found: $functionName") mapOf("context" to "Tool not found: $functionName")
} }
if (currSessionId == sessionid) { if (sessionid == currSessionId) {
// 处理结果 // 处理结果
sessions[sessionid]?.onFunctionCallResult( currSessionCallback?.onFunctionCallResult(
convertMapToJsonObject(functionCall), convertMapToJsonObject(functionCall),
convertMapToJsonObject(result) convertMapToJsonObject(result)
) )
@ -493,13 +575,11 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
functionCall = functionCall, functionCall = functionCall,
functionResult = gson.toJson(result) functionResult = gson.toJson(result)
) )
}else{
sessions[sessionid]?.onComplete()
} }
} }
} catch (e: Exception) { } catch (e: Exception) {
Log.e("ChatApiService", "工具调用处理过程中出错: ${e.message}", e) Log.e("ChatApiService", "工具调用处理过程中出错: ${e.message}", e)
if (currSessionId == sessionid) { if (sessionid == currSessionId) {
val errorMessage = "工具调用处理失败: ${e.message}" val errorMessage = "工具调用处理失败: ${e.message}"
sendFunctionCallResultInternal( sendFunctionCallResultInternal(
sessionid= sessionid, sessionid= sessionid,
@ -507,12 +587,19 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
functionCall = functionCall, functionCall = functionCall,
functionResult = errorMessage functionResult = errorMessage
) )
}else{
sessions[sessionid]?.onComplete()
} }
} }
} }
// 等待工具调用完成,确保生命周期管理
try {
toolCallDeferred.await()
} catch (e: CancellationException) {
// 协程被取消,确保子任务也被取消
toolCallDeferred.cancel()
throw e
}
return true return true
} }
@ -525,7 +612,7 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
functionCall: Map<String, Any>, functionCall: Map<String, Any>,
functionResult: String functionResult: String
) { ) {
if (currSessionId != sessionid) return if (sessionid != currSessionId) return
val fullMessages = messages.toMutableList() val fullMessages = messages.toMutableList()
@ -565,19 +652,19 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
* 取消当前流式请求 * 取消当前流式请求
*/ */
fun cancelCurrentStream(): Boolean { fun cancelCurrentStream(): Boolean {
sessions[currSessionId]?.onComplete() if (currSessionId.isNotEmpty()) {
sessions.remove(currSessionId) abortCurrentSession()
currSessionId = "" return true
currentStreamJob?.cancel() }
currentStreamJob = null return false
return true
} }
/** /**
* 取消所有操作并释放资源 (与 iOS 版本保持一致) * 取消所有操作并释放资源 (与 iOS 版本保持一致)
*/ */
fun cancelAll() { fun cancelAll() {
cancelCurrentStream() abortCurrentSession()
} }
/** /**
@ -706,7 +793,7 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
description = description, description = description,
parameters = parameters parameters = parameters
) )
Log.e("ChatApiService", "AI携带工具:${name}:${parameters}") // Log.d("ChatApiService", "AI携带工具: $name, 参数定义: $parameters")
tools.add(tool) tools.add(tool)
} catch (e: Exception) { } catch (e: Exception) {
@ -739,6 +826,30 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
} }
} }
/**
* 释放所有资源
* 统一的资源管理方法,确保所有协程被正确取消
*/
fun dispose() {
runBlocking {
// 取消当前流式任务
currentStreamJob?.cancelAndJoin()
currentStreamJob = null
// 关闭MCP客户端
closeMcpClient()
// 取消所有子协程
coroutineContext[Job]?.cancelChildren()
// 清理其他资源
currSessionCallback = null
currSessionId = ""
currentMessages = emptyList()
toolCalls.clear()
}
}
/** /**
* 处理MCP工具调用 * 处理MCP工具调用
* *
@ -796,9 +907,11 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
* 聊天完成流式接口 * 聊天完成流式接口
* 与iOS版本保持一致的接口 * 与iOS版本保持一致的接口
*/ */
fun chatCompletionStream(sessionid: String,messages: List<Map<String, Any>>, tool: Boolean = false) { fun chatCompletionStream(sessionid: String, messages: List<Map<String, Any>>, tool: Boolean = false) {
// 直接调用sendMessageStream,因为该方法已经处理了工具调用 // 验证是否是当前会话
sendMessageStream(sessionid,messages) if (sessionid == currSessionId) {
sendMessageStream(sessionid, messages)
}
} }
// MARK: - 工具方法 // MARK: - 工具方法

3
local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/CustomSseClientTransport.kt

@ -236,8 +236,7 @@ class CustomSseClientTransport(
if (!response.status.isSuccess()) { if (!response.status.isSuccess()) {
val text = response.bodyAsText() val text = response.bodyAsText()
Log.e(TAG, "发送消息失败:URL:${urlString} HTTP ${response.status}, $text") // Log.e(TAG, "发送消息失败:URL:${urlString} HTTP ${response.status}, $text")
error("Error POSTing to endpoint (HTTP ${response.status}): $text")
} }
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "发送消息异常: ${e.message}", e) Log.e(TAG, "发送消息异常: ${e.message}", e)

32
local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/MCPSubClient.kt

@ -129,7 +129,7 @@ class MCPSubClient(
"properties" to emptyMap<String, Any>(), "properties" to emptyMap<String, Any>(),
"required" to emptyList<String>() "required" to emptyList<String>()
) )
Log.w(TAG, "解析工具数据: ${tool.name} ${parametersMap}") // Log.w(TAG, "解析工具数据: ${tool.name} ${parametersMap}")
val toolMap = mapOf( val toolMap = mapOf(
"type" to "function", "type" to "function",
"function" to mapOf( "function" to mapOf(
@ -175,7 +175,7 @@ class MCPSubClient(
name = name, name = name,
arguments = argumentsJson arguments = argumentsJson
) )
Log.d(TAG, "[$serverId] 调用工具 '$name',参数: ${argumentsJson.toString()}")
// 调用工具 // 调用工具
val result = mcpClient?.callTool(request) val result = mcpClient?.callTool(request)
@ -357,9 +357,9 @@ class MCPSubClient(
val response = mcpClient?.ping() val response = mcpClient?.ping()
val latency = System.currentTimeMillis() - startTime val latency = System.currentTimeMillis() - startTime
Log.d(TAG, "心跳检测成功,延迟: ${latency}ms") // Log.d(TAG, "心跳检测成功,延迟: ${latency}ms")
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "心跳检测失败: ${e.message}") // Log.e(TAG, "心跳检测失败: ${e.message}")
handleConnectionError(e) handleConnectionError(e)
break break
} }
@ -373,18 +373,18 @@ class MCPSubClient(
isConnected = false isConnected = false
stopHeartbeat() stopHeartbeat()
// if (retryCount < maxRetryAttempts) { if (retryCount < maxRetryAttempts) {
// retryCount++ retryCount++
// currentReconnectDelay = minOf(currentReconnectDelay * 2, maxReconnectDelay) currentReconnectDelay = minOf(currentReconnectDelay * 2, maxReconnectDelay)
//
// Log.w(TAG, "连接失败,将在 ${currentReconnectDelay}ms 后尝试重连 (尝试 $retryCount/$maxRetryAttempts)") // Log.w(TAG, "连接失败,将在 ${currentReconnectDelay}ms 后尝试重连 (尝试 $retryCount/$maxRetryAttempts)")
//
// delay(currentReconnectDelay) delay(currentReconnectDelay)
// connect() connect()
// } else { } else {
// Log.e(TAG, "已达到最大重试次数($maxRetryAttempts),停止重连") Log.e(TAG, "已达到最大重试次数($maxRetryAttempts),停止重连")
// // 可以在这里添加通知或回调,告知上层连接彻底失败 // 可以在这里添加通知或回调,告知上层连接彻底失败
// } }
} }
/** /**
* 关闭连接 * 关闭连接

811
local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/SystemFunctionHandler.kt

@ -1,6 +1,7 @@
package com.yunqiinnovation.chat_api package com.yunqiinnovation.chat_api
import android.Manifest import android.Manifest
import android.app.ActivityManager
import android.content.ContentUris import android.content.ContentUris
import android.content.ContentValues import android.content.ContentValues
import android.content.Context import android.content.Context
@ -12,10 +13,12 @@ import android.os.Build
import android.provider.AlarmClock import android.provider.AlarmClock
import android.provider.CalendarContract import android.provider.CalendarContract
import android.provider.ContactsContract import android.provider.ContactsContract
import android.provider.Settings
import android.telephony.SmsManager import android.telephony.SmsManager
import android.util.Log import android.util.Log
import androidx.core.content.ContextCompat import androidx.core.content.ContextCompat
import androidx.core.content.ContextCompat.RECEIVER_NOT_EXPORTED import androidx.core.content.ContextCompat.RECEIVER_NOT_EXPORTED
import org.json.JSONObject
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.* import java.util.*
@ -24,6 +27,9 @@ import java.util.*
* 提供内置的系统函数,与 iOS 版本保持一致 * 提供内置的系统函数,与 iOS 版本保持一致
*/ */
class SystemFunctionHandler(private val context: Context? = null) { class SystemFunctionHandler(private val context: Context? = null) {
companion object {
const val TAG = "SystemFunctionHandler"
}
/** /**
* 注册所有系统函数 * 注册所有系统函数
@ -41,17 +47,17 @@ class SystemFunctionHandler(private val context: Context? = null) {
handler = ExitInteractionHandler(context) handler = ExitInteractionHandler(context)
) )
// 注册翻译模式函数 // // 注册翻译模式函数
client.registerLocalFunction( // client.registerLocalFunction(
name = "enter_translation_mode", // name = "enter_translation_mode",
description = "用户请求进入实时翻译模式时,启动实时翻译功能", // description = "用户请求进入实时翻译模式时,启动实时翻译功能",
parameters = mapOf( // parameters = mapOf(
"type" to "object", // "type" to "object",
"properties" to emptyMap<String, Any>(), // "properties" to emptyMap<String, Any>(),
"required" to emptyList<String>() // "required" to emptyList<String>()
), // ),
handler = TranslationModeHandler(context) // handler = TranslationModeHandler(context)
) // )
// 注册发送短信函数 // 注册发送短信函数
client.registerLocalFunction( client.registerLocalFunction(
@ -227,6 +233,171 @@ class SystemFunctionHandler(private val context: Context? = null) {
handler = ExitNavigationHandler(context) handler = ExitNavigationHandler(context)
) )
} }
/**
* 检查权限并根据应用状态采取不同行为
* @param permission 需要检查的权限
* @param permissionName 权限名称(用于显示给用户)
* @param functionName 功能名称(用于日志和提示)
* @return 是否拥有权限
*/
internal fun checkPermissionAndGuide(permission: String, permissionName: String, functionName: String): Boolean {
val ctx = context ?: return false
if (ContextCompat.checkSelfPermission(ctx, permission) != PackageManager.PERMISSION_GRANTED) {
Log.w(TAG, "缺少${permissionName}权限,无法执行${functionName}")
// 检查应用是否在前台
if (isAppInForeground()) {
// 应用在前台,尝试发送广播通知主应用弹出权限申请对话框
try {
val intent = Intent("com.yunqiinnovation.deepsound.action.REQUEST_PERMISSION").apply {
setPackage(ctx.packageName)
putExtra("permission", permission)
putExtra("permissionName", permissionName)
putExtra("functionName", functionName)
addFlags(Intent.FLAG_RECEIVER_FOREGROUND)
}
ctx.sendBroadcast(intent)
// 设置一个短暂延迟,让用户有时间响应权限请求
Thread.sleep(500)
// 再次检查权限是否已授予
if (ContextCompat.checkSelfPermission(ctx, permission) == PackageManager.PERMISSION_GRANTED) {
return true
}
} catch (e: Exception) {
Log.e(TAG, "请求权限失败: ${e.message}", e)
}
// 如果权限仍未授予,引导用户到权限管理页面
try {
val settingsIntent = Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS).apply {
data = Uri.fromParts("package", ctx.packageName, null)
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
}
ctx.startActivity(settingsIntent)
} catch (e: Exception) {
Log.e(TAG, "打开权限设置页面失败: ${e.message}", e)
}
}
return false
}
return true
}
/**
* 检查应用是否在前台运行
* @return 应用是否在前台
*/
internal fun isAppInForeground(): Boolean {
val ctx = context ?: return false
try {
val activityManager = ctx.getSystemService(Context.ACTIVITY_SERVICE) as ActivityManager
val appProcesses = activityManager.runningAppProcesses ?: return false
val packageName = ctx.packageName
for (appProcess in appProcesses) {
if (appProcess.importance == ActivityManager.RunningAppProcessInfo.IMPORTANCE_FOREGROUND
&& appProcess.processName == packageName) {
return true
}
}
} catch (e: Exception) {
Log.e(TAG, "检查应用状态失败: ${e.message}", e)
}
return false
}
/**
* 根据联系人名称获取电话号码
* @param contact 联系人名称或电话号码
* @return 电话号码
*/
internal fun getPhoneNumberForContact(contact: String): String {
val ctx = context ?: return ""
// 如果输入的是电话号码(简单判断,包含数字和特殊符号)
if (contact.matches(Regex("^[+\\d\\s\\-()]*\$")) && contact.any { it.isDigit() }) {
// 移除非数字字符
return contact.replace(Regex("[^+\\d]"), "")
}
// 检查读取联系人权限
if (!checkPermissionAndGuide(Manifest.permission.READ_CONTACTS, "读取联系人", "访问联系人")) {
return ""
}
// 从联系人数据库查询
try {
val contentResolver = ctx.contentResolver
val uri = Uri.withAppendedPath(
ContactsContract.Contacts.CONTENT_FILTER_URI,
Uri.encode(contact)
)
val projection = arrayOf(
ContactsContract.Contacts._ID,
ContactsContract.Contacts.DISPLAY_NAME,
ContactsContract.Contacts.HAS_PHONE_NUMBER
)
contentResolver.query(uri, projection, null, null, null)?.use { cursor ->
if (cursor.moveToFirst()) {
val hasPhoneNumber = cursor.getInt(cursor.getColumnIndexOrThrow(ContactsContract.Contacts.HAS_PHONE_NUMBER))
if (hasPhoneNumber > 0) {
val contactId = cursor.getString(cursor.getColumnIndexOrThrow(ContactsContract.Contacts._ID))
val phoneUri = ContactsContract.CommonDataKinds.Phone.CONTENT_URI
val phoneProjection = arrayOf(ContactsContract.CommonDataKinds.Phone.NUMBER)
val phoneSelection = "${ContactsContract.CommonDataKinds.Phone.CONTACT_ID} = ?"
val phoneSelectionArgs = arrayOf(contactId)
contentResolver.query(phoneUri, phoneProjection, phoneSelection, phoneSelectionArgs, null)?.use { phoneCursor ->
if (phoneCursor.moveToFirst()) {
return phoneCursor.getString(phoneCursor.getColumnIndexOrThrow(ContactsContract.CommonDataKinds.Phone.NUMBER))
}
}
}
}
}
} catch (e: Exception) {
Log.e(TAG, "查询联系人信息失败: ${e.message}", e)
}
return ""
}
/**
* 获取默认日历ID
*/
internal fun getDefaultCalendarId(): Long {
val ctx = context ?: return -1L
val projection = arrayOf(CalendarContract.Calendars._ID)
var calendarId = -1L
try {
val uri = CalendarContract.Calendars.CONTENT_URI
val selection = "(${CalendarContract.Calendars.VISIBLE} = 1) AND (${CalendarContract.Calendars.IS_PRIMARY} = 1)"
ctx.contentResolver.query(uri, projection, selection, null, null)?.use { cursor ->
if (cursor.moveToFirst()) {
calendarId = cursor.getLong(0)
} else {
// 如果没有找到主日历,尝试找任何可见的日历
ctx.contentResolver.query(uri, projection, "${CalendarContract.Calendars.VISIBLE} = 1", null, null)?.use { c ->
if (c.moveToFirst()) {
calendarId = c.getLong(0)
}
}
}
}
} catch (e: Exception) {
Log.e(TAG, "获取默认日历ID失败: ${e.message}", e)
}
return calendarId
}
} }
/** /**
@ -234,12 +405,25 @@ class SystemFunctionHandler(private val context: Context? = null) {
*/ */
private class ExitNavigationHandler(private val context: Context?) : FunctionHandler { private class ExitNavigationHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String { override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
// 发送广播通知退出交互 // 发送广播通知退出交互
val intent = Intent("com.yunqiinnovation.amap_service.EXIT_NAVIGATION").apply { val intent = Intent("com.yunqiinnovation.amap_service.EXIT_NAVIGATION").apply {
setPackage(context?.packageName) // 设置包名使其成为显式 Intent setPackage(context.packageName) // 设置包名使其成为显式 Intent
} }
context?.sendBroadcast(intent) context.sendBroadcast(intent)
return "{\"broadcast\":false,\"result\": \"已结束当前导航服务\"}"
return JSONObject().apply {
put("result", "已结束当前导航服务")
put("broadcast", false)
}.toString()
} }
} }
@ -248,9 +432,21 @@ private class ExitNavigationHandler(private val context: Context?) : FunctionHan
*/ */
private class ExitInteractionHandler(private val context: Context?) : FunctionHandler { private class ExitInteractionHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String { override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
// 发送广播通知退出交互 // 发送广播通知退出交互
context?.sendBroadcast(Intent("com.yunqiinnovation.deepsound.EXIT_INTERACTION")) context.sendBroadcast(Intent("com.yunqiinnovation.deepsound.EXIT_INTERACTION"))
return "{\"result\": \"已结束当前交互\"}"
return JSONObject().apply {
put("result", "已结束当前交互")
}.toString()
} }
} }
@ -259,9 +455,21 @@ private class ExitInteractionHandler(private val context: Context?) : FunctionHa
*/ */
private class TranslationModeHandler(private val context: Context?) : FunctionHandler { private class TranslationModeHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String { override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
// 发送广播通知进入翻译模式 // 发送广播通知进入翻译模式
context?.sendBroadcast(Intent("com.yunqiinnovation.deepsound.ENTER_TRANSLATION_MODE")) context.sendBroadcast(Intent("com.yunqiinnovation.deepsound.ENTER_TRANSLATION_MODE"))
return "{\"result\": \"已进入翻译模式\"}"
return JSONObject().apply {
put("result", "已进入翻译模式")
}.toString()
} }
} }
@ -274,24 +482,75 @@ private class SendTextMessageHandler(private val context: Context?) : FunctionHa
val message = arguments["message"] as? String val message = arguments["message"] as? String
if (contact == null || message == null) { if (contact == null || message == null) {
return "{\"result\": \"缺少必要参数\"}" return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "invalid_arguments")
put("message", "缺少必要参数:contact 或 message")
})
}.toString()
}
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
} }
val handler = context.let { SystemFunctionHandler(it) }
return try { return try {
val smsIntent = Intent(Intent.ACTION_SENDTO).apply { // 检查发送短信权限
data = Uri.parse("smsto:$contact") if (!handler.checkPermissionAndGuide(Manifest.permission.SEND_SMS, "发送短信", "发送短信")) {
putExtra("sms_body", message) return JSONObject().apply {
flags = Intent.FLAG_ACTIVITY_NEW_TASK put("error", JSONObject().apply {
put("code", "permission_denied")
put("message", "需要短信权限才能发送短信,请在权限管理中开启短信权限")
})
}.toString()
} }
if (context?.packageManager?.queryIntentActivities(smsIntent, 0)?.isNotEmpty() == true) { val phoneNumber = handler.getPhoneNumberForContact(contact)
context.startActivity(smsIntent) if (phoneNumber.isEmpty()) {
"{\"result\": \"已打开短信应用\"}" return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "contact_not_found")
put("message", "未找到联系人信息: $contact")
})
}.toString()
}
// 使用SmsManager发送短信
val smsManager = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
context.getSystemService(SmsManager::class.java)
} else { } else {
"{\"result\": \"无法打开短信应用\"}" @Suppress("DEPRECATION")
SmsManager.getDefault()
} }
// 处理长短信
val parts = smsManager.divideMessage(message)
if (parts.size == 1) {
smsManager.sendTextMessage(phoneNumber, null, message, null, null)
} else {
smsManager.sendMultipartTextMessage(phoneNumber, null, parts, null, null)
}
JSONObject().apply {
put("result", "已成功向${contact}发送短信")
put("phone", phoneNumber)
put("message", message)
}.toString()
} catch (e: Exception) { } catch (e: Exception) {
"{\"result\": \"发送短信失败:${e.message}\"}" Log.e(SystemFunctionHandler.TAG, "发送短信失败: ${e.message}", e)
JSONObject().apply {
put("error", JSONObject().apply {
put("code", "send_failed")
put("message", "发送短信失败: ${e.message}")
})
}.toString()
} }
} }
} }
@ -364,28 +623,73 @@ private class MakePhoneCallHandler(private val context: Context?) : FunctionHand
val contact = arguments["contact"] as? String val contact = arguments["contact"] as? String
if (contact == null) { if (contact == null) {
return "{\"result\": \"缺少联系人参数\"}" return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "invalid_arguments")
put("message", "缺少必要参数:contact")
})
}.toString()
}
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
} }
val handler = context.let { SystemFunctionHandler(it) }
return try { return try {
// 检查拨打电话权限 // 检查拨打电话权限
if (!checkPermissionAndGuide(context!!,Manifest.permission.CALL_PHONE, "拨打电话", "拨打电话")) { if (!handler.checkPermissionAndGuide(Manifest.permission.CALL_PHONE, "拨打电话", "拨打电话")) {
"{\"result\": \"需要电话权限才能拨打电话,请在权限管理中开启电话权限\"}" return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "permission_denied")
put("message", "需要电话权限才能拨打电话,请在权限管理中开启电话权限")
})
}.toString()
} }
val phoneNumber = getPhoneNumberForContact(contact) val phoneNumber = handler.getPhoneNumberForContact(contact)
if (phoneNumber.isEmpty()) { if (phoneNumber.isEmpty()) {
"{\"result\": \"未找到联系人信息: $contact\"}" return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "contact_not_found")
put("message", "未找到联系人信息: $contact")
})
}.toString()
} }
// 创建拨号意图 // 创建拨号意图(使用ACTION_CALL直接拨打)
val intent = Intent(Intent.ACTION_CALL) val intent = Intent(Intent.ACTION_CALL).apply {
intent.data = Uri.parse("tel:$phoneNumber") data = Uri.parse("tel:$phoneNumber")
intent.flags = Intent.FLAG_ACTIVITY_NEW_TASK flags = Intent.FLAG_ACTIVITY_NEW_TASK
context?.startActivity(intent) }
"{\"result\": \"正在拨打${contact}的电话\"}" context.startActivity(intent)
JSONObject().apply {
put("result", "正在拨打${contact}的电话")
put("phone", phoneNumber)
}.toString()
} catch (e: SecurityException) {
Log.e(SystemFunctionHandler.TAG, "拨打电话权限被拒绝: ${e.message}", e)
JSONObject().apply {
put("error", JSONObject().apply {
put("code", "permission_denied")
put("message", "权限被拒绝,无法拨打电话")
})
}.toString()
} catch (e: Exception) { } catch (e: Exception) {
Log.e("ChatApi", "拨打电话失败: ${e.message}", e) Log.e(SystemFunctionHandler.TAG, "拨打电话失败: ${e.message}", e)
"{\"result\": \"权限不足或者为查询到联系人\"}" JSONObject().apply {
put("error", JSONObject().apply {
put("code", "call_failed")
put("message", "拨打电话失败: ${e.message}")
})
}.toString()
} }
// return try { // return try {
// val callIntent = Intent(Intent.ACTION_DIAL).apply { // val callIntent = Intent(Intent.ACTION_DIAL).apply {
@ -549,38 +853,149 @@ private class SetReminderHandler(private val context: Context?) : FunctionHandle
val timeString = arguments["time"] as? String val timeString = arguments["time"] as? String
if (title == null || timeString == null) { if (title == null || timeString == null) {
return "{\"result\": \"缺少必要参数\"}" return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "invalid_arguments")
put("message", "缺少必要参数:title 或 time")
})
}.toString()
}
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
val handler = context.let { SystemFunctionHandler(it) }
return try {
// 解析时间
val formatter = SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.getDefault())
val time = formatter.parse(timeString)?.time ?: run {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "invalid_time_format")
put("message", "提醒时间格式无效,请使用'yyyy-MM-dd HH:mm'格式")
})
}.toString()
}
// 检查日历权限
if (!handler.checkPermissionAndGuide(Manifest.permission.WRITE_CALENDAR, "日历写入", "设置提醒")) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "permission_denied")
put("message", "需要日历权限才能创建提醒,请在权限管理中开启日历权限")
})
}.toString()
}
// 直接使用ContentProvider写入日历
val endTime = time + 3600000 // 默认1小时后结束
// 获取默认日历ID
val calendarId = handler.getDefaultCalendarId()
if (calendarId == -1L) {
Log.w(SystemFunctionHandler.TAG, "未找到默认日历")
// 如果没有找到日历,尝试使用Intent创建
return tryCreateWithIntent(context, title, content, time)
}
// 创建事件
val eventValues = ContentValues().apply {
put(CalendarContract.Events.CALENDAR_ID, calendarId)
put(CalendarContract.Events.TITLE, title)
put(CalendarContract.Events.DESCRIPTION, content)
put(CalendarContract.Events.DTSTART, time)
put(CalendarContract.Events.DTEND, endTime)
put(CalendarContract.Events.HAS_ALARM, 1)
put(CalendarContract.Events.EVENT_TIMEZONE, TimeZone.getDefault().id)
}
val eventUri = context.contentResolver.insert(CalendarContract.Events.CONTENT_URI, eventValues)
if (eventUri == null) {
Log.e(SystemFunctionHandler.TAG, "插入日历事件失败")
return tryCreateWithIntent(context, title, content, time)
}
// 获取事件ID
val eventId = ContentUris.parseId(eventUri)
// 添加提醒(15分钟前提醒)
val reminderValues = ContentValues().apply {
put(CalendarContract.Reminders.EVENT_ID, eventId)
put(CalendarContract.Reminders.METHOD, CalendarContract.Reminders.METHOD_ALERT)
put(CalendarContract.Reminders.MINUTES, 15) // 提前15分钟提醒
}
val reminderUri = context.contentResolver.insert(CalendarContract.Reminders.CONTENT_URI, reminderValues)
if (reminderUri != null) {
return JSONObject().apply {
put("result", "已成功创建提醒: $title")
put("title", title)
put("content", content)
put("time", timeString)
put("eventId", eventId)
}.toString()
} else {
// 仍然创建了事件,但没有提醒
return JSONObject().apply {
put("result", "已创建日历事件,但未设置提醒: $title")
put("title", title)
put("time", timeString)
put("eventId", eventId)
}.toString()
}
} catch (e: Exception) {
Log.e(SystemFunctionHandler.TAG, "设置提醒失败: ${e.message}", e)
JSONObject().apply {
put("error", JSONObject().apply {
put("code", "reminder_failed")
put("message", "设置提醒失败: ${e.message}")
})
}.toString()
}
}
private fun tryCreateWithIntent(context: Context, title: String, content: String, time: Long): String {
return try {
val calendarIntent = Intent(Intent.ACTION_INSERT).apply {
data = CalendarContract.Events.CONTENT_URI
putExtra(CalendarContract.Events.TITLE, title)
putExtra(CalendarContract.Events.DESCRIPTION, content)
putExtra(CalendarContract.EXTRA_EVENT_BEGIN_TIME, time)
putExtra(CalendarContract.EXTRA_EVENT_END_TIME, time + 3600000)
putExtra(CalendarContract.Events.HAS_ALARM, 1)
flags = Intent.FLAG_ACTIVITY_NEW_TASK
}
if (context.packageManager?.queryIntentActivities(calendarIntent, 0)?.isNotEmpty() == true) {
context.startActivity(calendarIntent)
JSONObject().apply {
put("result", "已打开日历应用创建提醒")
put("title", title)
put("content", content)
}.toString()
} else {
JSONObject().apply {
put("error", JSONObject().apply {
put("code", "no_calendar_app")
put("message", "未找到可用的日历应用")
})
}.toString()
}
} catch (e: Exception) {
JSONObject().apply {
put("error", JSONObject().apply {
put("code", "intent_failed")
put("message", "创建日历事件失败: ${e.message}")
})
}.toString()
} }
return setReminder(title,content,timeString)
// return try {
// // 解析时间
// val formatter = SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.getDefault())
// val date = formatter.parse(timeString)
//
// if (date == null) {
// return "{\"result\": \"时间格式错误\"}"
// }
//
// // 创建日历事件
// val calendarIntent = Intent(Intent.ACTION_INSERT).apply {
// data = CalendarContract.Events.CONTENT_URI
// putExtra(CalendarContract.Events.TITLE, title)
// putExtra(CalendarContract.Events.DESCRIPTION, content)
// putExtra(CalendarContract.EXTRA_EVENT_BEGIN_TIME, date.time)
// putExtra(CalendarContract.EXTRA_EVENT_END_TIME, date.time + 60 * 60 * 1000) // 默认1小时
// putExtra(CalendarContract.Events.HAS_ALARM, 1)
// flags = Intent.FLAG_ACTIVITY_NEW_TASK
// }
//
// if (context?.packageManager?.queryIntentActivities(calendarIntent, 0)?.isNotEmpty() == true) {
// context.startActivity(calendarIntent)
// "{\"result\": \"提醒设置成功\"}"
// } else {
// "{\"result\": \"无法打开日历应用\"}"
// }
// } catch (e: Exception) {
// "{\"result\": \"设置提醒失败:${e.message}\"}"
// }
} }
} }
@ -592,16 +1007,61 @@ private class GetCurrentTimeHandler : FunctionHandler {
val format = arguments["format"] as? String val format = arguments["format"] as? String
return try { return try {
val formatter = if (!format.isNullOrEmpty()) { val calendar = Calendar.getInstance()
SimpleDateFormat(format, Locale.getDefault()) val date = calendar.time
} else {
SimpleDateFormat("yyyy年MM月dd日 HH:mm:ss", Locale.CHINA) val timeStr = when (format?.lowercase(Locale.getDefault())) {
"short" -> {
SimpleDateFormat("HH:mm", Locale.getDefault()).format(date)
}
"date" -> {
SimpleDateFormat("yyyy年MM月dd日", Locale.getDefault()).format(date)
}
"time" -> {
SimpleDateFormat("HH:mm:ss", Locale.getDefault()).format(date)
}
"full" -> {
SimpleDateFormat("yyyy年MM月dd日 EEEE HH:mm:ss", Locale.getDefault()).format(date)
}
"iso" -> {
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.getDefault()).format(date)
}
else -> {
SimpleDateFormat("yyyy年MM月dd日 HH:mm", Locale.getDefault()).format(date)
}
}
// 获取星期几
val weekDay = when (calendar.get(Calendar.DAY_OF_WEEK)) {
Calendar.SUNDAY -> "星期日"
Calendar.MONDAY -> "星期一"
Calendar.TUESDAY -> "星期二"
Calendar.WEDNESDAY -> "星期三"
Calendar.THURSDAY -> "星期四"
Calendar.FRIDAY -> "星期五"
Calendar.SATURDAY -> "星期六"
else -> ""
} }
val currentTime = formatter.format(Date()) // 根据格式决定是否添加星期几
"{\"result\": \"$currentTime\", \"time\": \"$currentTime\"}" val resultTime = when {
format == "short" || format == "time" -> timeStr
format == "full" -> timeStr // full格式已经包含了星期
else -> "$timeStr $weekDay"
}
JSONObject().apply {
put("result", resultTime)
put("time", resultTime)
put("timestamp", date.time)
}.toString()
} catch (e: Exception) { } catch (e: Exception) {
"{\"result\": \"获取时间失败:${e.message}\"}" JSONObject().apply {
put("error", JSONObject().apply {
put("code", "time_error")
put("message", "获取时间失败: ${e.message}")
})
}.toString()
} }
} }
} }
@ -612,21 +1072,24 @@ private class GetCurrentTimeHandler : FunctionHandler {
private class GetCurrentLocationHandler(private val context: Context?) : FunctionHandler { private class GetCurrentLocationHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String { override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) { if (context == null) {
return "{\"result\": \"上下文未初始化\", \"success\": false}" return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
} }
// 检查位置权限 val handler = context.let { SystemFunctionHandler(it) }
val hasPermission = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
ContextCompat.checkSelfPermission(
context,
android.Manifest.permission.ACCESS_FINE_LOCATION
) == PackageManager.PERMISSION_GRANTED
} else {
true
}
if (!hasPermission) { // 检查位置权限
return "{\"result\": \"位置权限被拒绝。请前往设置中启用位置权限\", \"success\": false}" if (!handler.checkPermissionAndGuide(Manifest.permission.ACCESS_FINE_LOCATION, "位置信息", "获取位置")) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "permission_denied")
put("message", "需要位置权限才能获取当前位置,请在权限管理中开启位置权限")
})
}.toString()
} }
// 检查位置服务是否启用 // 检查位置服务是否启用
@ -635,12 +1098,29 @@ private class GetCurrentLocationHandler(private val context: Context?) : Functio
locationManager?.isProviderEnabled(LocationManager.NETWORK_PROVIDER) == true locationManager?.isProviderEnabled(LocationManager.NETWORK_PROVIDER) == true
if (!isLocationEnabled) { if (!isLocationEnabled) {
return "{\"result\": \"位置服务未启用。请前往设置中启用位置服务\", \"success\": false}" // 尝试引导用户打开位置服务
try {
val intent = Intent(Settings.ACTION_LOCATION_SOURCE_SETTINGS)
intent.flags = Intent.FLAG_ACTIVITY_NEW_TASK
context.startActivity(intent)
} catch (e: Exception) {
Log.e(SystemFunctionHandler.TAG, "打开位置设置失败: ${e.message}", e)
}
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "location_disabled")
put("message", "位置服务未启用,请打开GPS或网络定位")
})
}.toString()
} }
// 注意:实际的位置获取需要异步处理,这里只返回提示信息 // 注意:这里简化了实现,实际应该获取真实位置
// 在实际应用中应该使用 LocationCallback 或 Coroutines 来获取实时位置 // 在生产环境中应该使用 LocationManager 或 FusedLocationProviderClient 获取实际位置
return "{\"result\": \"需要通过位置服务获取当前位置\", \"success\": true}" return JSONObject().apply {
put("result", "位置服务已启用,可以获取位置信息")
put("note", "需要实现实际的位置获取逻辑")
}.toString()
} }
} }
@ -649,12 +1129,26 @@ private class GetCurrentLocationHandler(private val context: Context?) : Functio
*/ */
private class MediaPlayHandler(private val context: Context?) : FunctionHandler { private class MediaPlayHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String { override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
// 发送媒体播放广播 // 发送媒体播放广播
val intent = Intent("com.yunqiinnovation.music_service.MEDIA_PLAY").apply { val intent = Intent("com.yunqiinnovation.music_service.MEDIA_PLAY").apply {
setPackage(context?.packageName) // 设置包名使其成为显式 Intent setPackage(context.packageName)
} }
context?.sendBroadcast(intent) context.sendBroadcast(intent)
return "{\"notreply\":true,\"broadcast\":false,\"result\": \"已开始播放媒体\"}"
return JSONObject().apply {
put("result", "已开始播放媒体")
put("notreply", true)
put("broadcast", false)
}.toString()
} }
} }
@ -663,27 +1157,55 @@ private class MediaPlayHandler(private val context: Context?) : FunctionHandler
*/ */
private class MediaPauseHandler(private val context: Context?) : FunctionHandler { private class MediaPauseHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String { override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
Log.d("MediaPauseHandler", "📤 正在发送广播 context = $context") Log.d("MediaPauseHandler", "📤 正在发送广播 context = $context")
// 发送媒体暂停广播 // 发送媒体暂停广播
val intent = Intent("com.yunqiinnovation.music_service.MEDIA_PAUSE").apply { val intent = Intent("com.yunqiinnovation.music_service.MEDIA_PAUSE").apply {
setPackage(context?.packageName) // 设置包名使其成为显式 Intent setPackage(context.packageName)
} }
context?.sendBroadcast(intent) context.sendBroadcast(intent)
return "{\"notreply\":true,\"broadcast\":false,\"result\": \"已暂停媒体播放\"}"
return JSONObject().apply {
put("result", "已暂停媒体播放")
put("notreply", true)
put("broadcast", false)
}.toString()
} }
} }
/** /**
* 媒体暂停处理器 * 媒体关闭处理器
*/ */
private class MediaCloseHandler(private val context: Context?) : FunctionHandler { private class MediaCloseHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String { override suspend fun handle(arguments: Map<String, Any>): String {
Log.d("MediaPauseHandler", "📤 正在发送广播 context = $context") if (context == null) {
// 发送媒体暂停广播 return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
Log.d("MediaCloseHandler", "📤 正在发送广播 context = $context")
// 发送媒体关闭广播
val intent = Intent("com.yunqiinnovation.music_service.MEDIA_CLOSE").apply { val intent = Intent("com.yunqiinnovation.music_service.MEDIA_CLOSE").apply {
setPackage(context?.packageName) // 设置包名使其成为显式 Intent setPackage(context.packageName)
} }
context?.sendBroadcast(intent) context.sendBroadcast(intent)
return "{\"notreply\":true,\"broadcast\":false,\"result\": \"已关闭媒体播放\"}"
return JSONObject().apply {
put("result", "已关闭媒体播放")
put("notreply", true)
put("broadcast", false)
}.toString()
} }
} }
/** /**
@ -691,13 +1213,26 @@ private class MediaCloseHandler(private val context: Context?) : FunctionHandler
*/ */
private class MediaPreviousHandler(private val context: Context?) : FunctionHandler { private class MediaPreviousHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String { override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
// 发送切换上一首广播 // 发送切换上一首广播
// context?.applicationContext?.sendBroadcast(Intent("com.yunqiinnovation.music_service.MEDIA_PREVIOUS"))
val intent = Intent("com.yunqiinnovation.music_service.MEDIA_PREVIOUS").apply { val intent = Intent("com.yunqiinnovation.music_service.MEDIA_PREVIOUS").apply {
setPackage(context?.packageName) // 设置包名使其成为显式 Intent setPackage(context.packageName)
} }
context?.sendBroadcast(intent) context.sendBroadcast(intent)
return "{\"notreply\":true,\"broadcast\":false,\"result\": \"已切换到上一首\"}"
return JSONObject().apply {
put("result", "已切换到上一首")
put("notreply", true)
put("broadcast", false)
}.toString()
} }
} }
@ -706,13 +1241,26 @@ private class MediaPreviousHandler(private val context: Context?) : FunctionHand
*/ */
private class MediaNextHandler(private val context: Context?) : FunctionHandler { private class MediaNextHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String { override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) {
return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
}
// 发送切换下一首广播 // 发送切换下一首广播
// context?.applicationContext?.sendBroadcast(Intent("com.yunqiinnovation.music_service.MEDIA_NEXT"))
val intent = Intent("com.yunqiinnovation.music_service.MEDIA_NEXT").apply { val intent = Intent("com.yunqiinnovation.music_service.MEDIA_NEXT").apply {
setPackage(context?.packageName) // 设置包名使其成为显式 Intent setPackage(context.packageName)
} }
context?.sendBroadcast(intent) context.sendBroadcast(intent)
return "{\"notreply\":true,\"broadcast\":false,\"result\": \"已切换到下一首\"}"
return JSONObject().apply {
put("result", "已切换到下一首")
put("notreply", true)
put("broadcast", false)
}.toString()
} }
} }
@ -722,7 +1270,12 @@ private class MediaNextHandler(private val context: Context?) : FunctionHandler
private class OpenRecorderHandler(private val context: Context?) : FunctionHandler { private class OpenRecorderHandler(private val context: Context?) : FunctionHandler {
override suspend fun handle(arguments: Map<String, Any>): String { override suspend fun handle(arguments: Map<String, Any>): String {
if (context == null) { if (context == null) {
return "{\"result\": \"上下文未初始化\"}" return JSONObject().apply {
put("error", JSONObject().apply {
put("code", "context_error")
put("message", "上下文未初始化")
})
}.toString()
} }
return try { return try {
@ -739,18 +1292,32 @@ private class OpenRecorderHandler(private val context: Context?) : FunctionHandl
when { when {
context.packageManager?.queryIntentActivities(recordIntent, 0)?.isNotEmpty() == true -> { context.packageManager?.queryIntentActivities(recordIntent, 0)?.isNotEmpty() == true -> {
context.startActivity(recordIntent) context.startActivity(recordIntent)
"{\"result\": \"已打开录音机\"}" JSONObject().apply {
put("result", "已打开录音机")
}.toString()
} }
context.packageManager?.queryIntentActivities(recorderIntent, 0)?.isNotEmpty() == true -> { context.packageManager?.queryIntentActivities(recorderIntent, 0)?.isNotEmpty() == true -> {
context.startActivity(recorderIntent) context.startActivity(recorderIntent)
"{\"result\": \"已打开音频应用\"}" JSONObject().apply {
put("result", "已打开音频应用")
}.toString()
} }
else -> { else -> {
"{\"result\": \"无法打开录音机应用\"}" JSONObject().apply {
put("error", JSONObject().apply {
put("code", "no_recorder_app")
put("message", "无法打开录音机应用")
})
}.toString()
} }
} }
} catch (e: Exception) { } catch (e: Exception) {
"{\"result\": \"打开录音机失败:${e.message}\"}" JSONObject().apply {
put("error", JSONObject().apply {
put("code", "recorder_failed")
put("message", "打开录音机失败: ${e.message}")
})
}.toString()
} }
} }
} }

2
local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/MCPClient.kt

@ -479,7 +479,7 @@ class MCPSubClient(
retryCount++ retryCount++
currentReconnectDelay = minOf(currentReconnectDelay * 2, maxReconnectDelay) currentReconnectDelay = minOf(currentReconnectDelay * 2, maxReconnectDelay)
Log.w(TAG, "连接失败,将在 ${currentReconnectDelay}ms 后尝试重连 (尝试 $retryCount/$maxRetryAttempts)") // Log.w(TAG, "连接失败,将在 ${currentReconnectDelay}ms 后尝试重连 (尝试 $retryCount/$maxRetryAttempts)")
delay(currentReconnectDelay) delay(currentReconnectDelay)
connect() connect()

15
local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/ITtsService.kt

@ -89,4 +89,19 @@ interface ITtsService {
* @param listener 要移除的音频数据监听器 * @param listener 要移除的音频数据监听器
*/ */
fun removeAudioDataListener(listener: AudioDataListener) fun removeAudioDataListener(listener: AudioDataListener)
/**
* 设置是否使用内部播放器
*
* @param useInternalPlayer true: 使用内部播放器自动播放音频
* false: 仅通过音频数据监听器输出数据,不播放
*/
fun setUseInternalPlayer(useInternalPlayer: Boolean)
/**
* 设置音频输出设备
*
* @param device 音频输出设备类型
*/
fun setAudioOutputDevice(device: AudioOutputDevice)
} }

15
local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/TtsEvents.kt

@ -13,6 +13,12 @@ enum class TtsEventType {
/** 合成取消 */ /** 合成取消 */
SYNTHESIS_CANCELED, SYNTHESIS_CANCELED,
/** 播放开始 */
PLAYBACK_STARTED,
/** 播放结束 */
PLAYBACK_COMPLETED,
/** 发生错误 */ /** 发生错误 */
ERROR ERROR
} }
@ -52,4 +58,13 @@ interface AudioDataListener {
* @param data 音频数据字节数组 * @param data 音频数据字节数组
*/ */
fun onAudioData(data: ByteArray) fun onAudioData(data: ByteArray)
}
/**
* 音频输出设备类型
*/
enum class AudioOutputDevice {
DEFAULT, // 默认(如果有耳机选耳机,否则使用系统扬声器)
HEADPHONES, // 强制使用耳机
SPEAKER, // 强制使用扬声器
} }

2
pubspec.yaml

@ -94,6 +94,8 @@ dependencies:
path: local_plugins/amap_service path: local_plugins/amap_service
location_service: location_service:
path: local_plugins/location_service path: local_plugins/location_service
bytedance_speech:
path: local_plugins/bytedance_speech
realtime: realtime:
path: local_plugins/realtime path: local_plugins/realtime
jl_opus: jl_opus:

Loading…
Cancel
Save