Browse Source

Merge branch 'new_dev' of https://github.com/deepcloud2048/deep_voice into new_dev

weicu
lxm 1 year ago
parent
commit
97dfa114ef
  1. 6
      ios/Runner/Info.plist
  2. 4
      lib/core/bindings/initial_binding.dart
  3. 34
      lib/data/services/novice_guide_service.dart
  4. 3
      lib/modules/agent/controllers/agent_controller.dart
  5. 207
      lib/modules/agent/views/agent_view.dart
  6. 1
      lib/modules/devices/controllers/devices_controller.dart
  7. 38
      lib/modules/home/controllers/home_controller.dart
  8. 107
      lib/modules/home/views/home_view.dart
  9. 108
      lib/modules/pairing/controllers/pairing_controller.dart
  10. 14
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  11. 50
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift
  12. 6
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift
  13. 230
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AudioSessionHub.swift
  14. BIN
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/Resources/await.mp3
  15. 186
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift
  16. 214
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/LanguageDetector.swift
  17. 2
      local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleServicePlugin.kt
  18. 2
      local_plugins/ble_service/ios/ble_service/Sources/ble_service/SwiftBleServicePlugin.swift
  19. 2
      local_plugins/ble_service/ios/ble_service/Sources/ble_service/SwiftOpusAudioProcessor.swift
  20. 207
      local_plugins/chat_api/ios/chat_api/Sources/chat_api/SystemFunctionHandler.swift
  21. 2
      local_plugins/location_service/android/src/main/kotlin/com/yunqiinnovation/location_service/LocationServicePlugin.kt
  22. 2
      local_plugins/music_service/android/src/main/kotlin/com/yunqiinnovation/music_service/MusicServicePlugin.kt
  23. 2
      local_plugins/music_service/ios/music_service/Sources/music_service/MusicServicePlugin.swift

6
ios/Runner/Info.plist

@ -59,6 +59,8 @@
<string>193d0a2b1757c41cfa557ea8e8f21c33</string>
<key>LSApplicationQueriesSchemes</key>
<array>
<string>App-Prefs</string>
<string>prefs</string>
<string>weixin</string>
<string>weixinULAPI</string>
<string>weixinURLParamsAPI</string>
@ -119,6 +121,10 @@
<string>audio</string>
<string>location</string>
</array>
<key>UIRequiredDeviceCapabilities</key>
<array>
<string>telephony</string>
</array>
<key>UIDeviceFamily</key>
<array>
<string>1</string>

4
lib/core/bindings/initial_binding.dart

@ -6,6 +6,7 @@ import '../../../data/services/navigation_manager.dart';
import 'package:get/get.dart';
import '../../data/services/meeting/meeting_task_service.dart';
import '../../data/services/meeting/meeting_upload_service.dart';
import '../../data/services/novice_guide_service.dart';
import '../../data/services/speech_factory.dart';
import '../../data/services/volcano_translation_service.dart';
import '../../data/services/volcano_ai_service.dart';
@ -64,6 +65,9 @@ class InitialBinding extends Bindings {
// 会议助手音频上传
Get.lazyPut<UserportraitManager>(() => UserportraitManager(), fenix: true);
// 新手引导
Get.lazyPut<NoviceGuideService>(() => NoviceGuideService(), fenix: true);
// 杰理蓝牙服务
// // 语言管理器(需要最先初始化)

34
lib/data/services/novice_guide_service.dart

@ -0,0 +1,34 @@
import 'package:flutter_intro/flutter_intro.dart';
import 'package:get/get.dart';
import 'package:get_storage/get_storage.dart';
class NoviceGuideService extends GetxService {
final GetStorage _storage = GetStorage();
bool isHomeGuide = false;
bool isAIGuide = false;
@override
void onInit() {
super.onInit();
// isHomeGuide = _storage.read('home_guide') ?? false;
// isAIGuide = _storage.read('ai_guide') ?? false;
}
void startHomeGuide() {
if (!isHomeGuide) {
isHomeGuide = true;
_storage.write('home_guide', true);
Intro.of(Get.context!).start(group: 'home');
}
}
void startAIGuide() async {
if (!isAIGuide) {
await Future.delayed(const Duration(milliseconds: 200));
isAIGuide = true;
_storage.write('ai_guide', true);
Intro.of(Get.context!).start(group: 'ai');
}
}
}

3
lib/modules/agent/controllers/agent_controller.dart

@ -3,6 +3,7 @@ import 'dart:convert';
import 'dart:ffi';
import 'dart:io';
import 'package:amap_service/amap_service.dart';
import '../../../data/services/novice_guide_service.dart';
import '/core/utils/upload_oss.dart';
import '/data/models/user_Info.dart';
import '/data/services/music_manager.dart';
@ -39,6 +40,8 @@ class AgentController extends GetxController {
final TextEditingController textController = TextEditingController();
final ScrollController scrollController = ScrollController();
final noviceGuideService = Get.find<NoviceGuideService>();
//音乐播放管理
final _musiceManager = Get.find<MusicManager>();

207
lib/modules/agent/views/agent_view.dart

@ -2,6 +2,7 @@ import 'dart:async';
import 'dart:io';
import 'dart:math';
import 'package:flutter/material.dart';
import 'package:flutter_intro/flutter_intro.dart';
import 'package:get/get.dart';
import '../controllers/agent_controller.dart';
import '../controllers/message_action_controller.dart';
@ -388,96 +389,110 @@ class _AgentViewState extends State<AgentView> with WidgetsBindingObserver {
if (controller.isTextInputMode.value) {
// 文本输入模式 - 限制最大高度并只向上扩展
return Container(
constraints: BoxConstraints(
minHeight: 44.h,
maxHeight: 120.h,
),
padding: EdgeInsets.symmetric(
horizontal: 16.w),
decoration: BoxDecoration(
color: isDarkMode
? Colors.grey[800]
: Colors.grey[200],
borderRadius:
BorderRadius.circular(22.w),
border: Border.all(
return IntroStepBuilder(
group: 'ai',
order: 1,
text: '对小言说“今天天气怎么样”',
onWidgetLoad: () {
controller.noviceGuideService
.startAIGuide();
},
builder: (BuildContext context,
GlobalKey key) =>
Container(
key: key,
constraints: BoxConstraints(
minHeight: 44.h,
maxHeight: 120.h,
),
padding: EdgeInsets.symmetric(
horizontal: 16.w),
decoration: BoxDecoration(
color: isDarkMode
? Colors.grey[800]!
: Colors.grey.withOpacity(0.1),
width: 1.w,
? Colors.grey[800]
: Colors.grey[200],
borderRadius:
BorderRadius.circular(22.w),
border: Border.all(
color: isDarkMode
? Colors.grey[800]!
: Colors.grey.withOpacity(0.1),
width: 1.w,
),
),
),
child: Row(
crossAxisAlignment: CrossAxisAlignment
.end, // 对齐到底部,实现向上扩展
children: [
Expanded(
child: Obx(() {
// 图片模式下使用更高的输入框
final isImageActive = controller
.isImageInputActive.value;
child: Row(
crossAxisAlignment: CrossAxisAlignment
.end, // 对齐到底部,实现向上扩展
children: [
Expanded(
child: Obx(() {
// 图片模式下使用更高的输入框
final isImageActive = controller
.isImageInputActive.value;
return TextField(
controller:
controller.textController,
maxLines: isImageActive
? 3
: 5, // 设置合理的最大行数
minLines: isImageActive ? 2 : 1,
keyboardType: TextInputType
.multiline, // 启用多行键盘
textInputAction: TextInputAction
.newline, // 支持回车换行
onTap: () {
// 当点击输入框时,如果'+'选项已展开,则收缩
if (controller
.isPlusOptionsExpanded
.value) {
controller
.collapsePlusOptions();
}
},
style: TextStyle(
fontSize: 15.sp,
color: isDarkMode
? Colors.white
: Colors.grey[800],
),
decoration: InputDecoration(
hintText: isImageActive
? 'addQuestionAboutImage'
.tr // 添加对图片的问题或描述...
: 'inputMessage'
.tr, // 输入消息...
hintStyle: TextStyle(
color: Colors.grey[500],
return TextField(
controller:
controller.textController,
maxLines: isImageActive
? 3
: 5, // 设置合理的最大行数
minLines:
isImageActive ? 2 : 1,
keyboardType: TextInputType
.multiline, // 启用多行键盘
textInputAction:
TextInputAction
.newline, // 支持回车换行
onTap: () {
// 当点击输入框时,如果'+'选项已展开,则收缩
if (controller
.isPlusOptionsExpanded
.value) {
controller
.collapsePlusOptions();
}
},
style: TextStyle(
fontSize: 15.sp,
color: isDarkMode
? Colors.white
: Colors.grey[800],
),
filled: false,
border: InputBorder.none,
focusedBorder:
OutlineInputBorder(
borderSide: BorderSide(
color: isDarkMode
? Colors.grey[800]!
: Colors.grey[200]!,
width: 1.w,
decoration: InputDecoration(
hintText: isImageActive
? 'addQuestionAboutImage'
.tr // 添加对图片的问题或描述...
: 'inputMessage'
.tr, // 输入消息...
hintStyle: TextStyle(
color: Colors.grey[500],
fontSize: 15.sp,
),
filled: false,
border: InputBorder.none,
focusedBorder:
OutlineInputBorder(
borderSide: BorderSide(
color: isDarkMode
? Colors.grey[800]!
: Colors.grey[200]!,
width: 1.w,
),
),
contentPadding:
EdgeInsets.symmetric(
vertical:
isImageActive
? 12.h
: 8.h),
),
contentPadding:
EdgeInsets.symmetric(
vertical:
isImageActive
? 12.h
: 8.h),
),
onSubmitted: (_) =>
controller.sendMessage(),
);
}),
),
],
onSubmitted: (_) =>
controller.sendMessage(),
);
}),
),
],
),
),
);
} else {
@ -919,21 +934,19 @@ class _AgentViewState extends State<AgentView> with WidgetsBindingObserver {
Widget _buildPhoneCallOverlay(BuildContext context, bool isDarkMode) {
return Container(
color: Colors.transparent, // 设置整个覆盖层为透明
child: SafeArea(
child: Column(
children: [
// 中间空白区域 - 完全透明
const Expanded(
child: SizedBox(), // 空的占位符,完全透明
),
child: Column(
children: [
// 中间空白区域 - 完全透明
const Expanded(
child: SizedBox(), // 空的占位符,完全透明
),
// 底部控制按钮 - 包含音频波纹
Container(
color: isDarkMode ? Colors.black : Colors.white,
child: _buildPhoneCallControls(isDarkMode),
),
],
),
// 底部控制按钮 - 包含音频波纹
Container(
color: isDarkMode ? Colors.black : Colors.white,
child: _buildPhoneCallControls(isDarkMode),
),
],
),
);
}

1
lib/modules/devices/controllers/devices_controller.dart

@ -57,6 +57,7 @@ class DevicesController extends GetxController {
User.instance.removeUserDevice(device.id);
_bleManager.clearPairedDevices();
_bleManager.disconnect();
await Future.delayed(const Duration(seconds: 1));
Get.toNamed(Routes.pairing);
return;
} else {

38
lib/modules/home/controllers/home_controller.dart

@ -1,5 +1,6 @@
import 'package:get/get.dart';
import 'dart:async';
import '../../../data/services/novice_guide_service.dart';
import '../../../routes/app_routes.dart';
import '../../../core/utils/logger.dart';
import 'package:flutter/material.dart';
@ -21,7 +22,7 @@ class HomeController extends GetxController {
final _isFullyConnected = false.obs;
String? _lastConnectedDeviceName;
bool get isFullyConnected => _isFullyConnected.value;
RxBool get isFullyConnectedRx => _isFullyConnected;
BluetoothDevice? get currentDevice => _bluetoothManager.currentDevice;
@ -50,6 +51,9 @@ class HomeController extends GetxController {
final _bluetoothManager = Get.find<BluetoothManager>();
final _bleManager = Get.find<BleManager>();
final noviceGuideService = Get.find<NoviceGuideService>();
@override
void onInit() {
super.onInit();
@ -77,13 +81,14 @@ class HomeController extends GetxController {
update();
});
_classicBluetoothWorker = ever(_bluetoothManager.currentDeviceRx, (BluetoothDevice? device) {
_classicBluetoothWorker =
ever(_bluetoothManager.currentDeviceRx, (BluetoothDevice? device) {
if (device != null) {
_lastConnectedDeviceName = device.name;
}
_checkConnectionStatus();
});
_bleConnectionWorker = ever(_bleManager.connectionStateRx, (int state) {
_checkConnectionStatus();
});
@ -104,30 +109,30 @@ class HomeController extends GetxController {
void exploreAIFeatures() {
Get.toNamed(Routes.agent);
}
Future<void> _checkConnectionStatus() async {
try {
final currentBleState = await _bleManager.getConnectionState();
bool hasClassicBluetooth = currentDevice != null;
bool hasBleConnection = currentBleState == 2;
bool wasConnected = _isFullyConnected.value;
bool isNowConnected = hasClassicBluetooth && hasBleConnection;
if (isNowConnected && currentDevice == null) {
isNowConnected = false;
}
if (_isFullyConnected.value != isNowConnected) {
_isFullyConnected.value = isNowConnected;
_isFullyConnected.refresh();
update();
if (!wasConnected && isNowConnected) {
await _bleManager.fetchDeviceInfo();
}
}
update();
} catch (e) {
Logger.e(_tag, '${'checkConnectionStatusFailed'.tr}: $e');
@ -148,26 +153,28 @@ class HomeController extends GetxController {
}
bool get shouldShowDeviceInfo => currentDevice != null;
String getDeviceName() {
if (_isFullyConnected.value && currentDevice == null) {
_checkConnectionStatus();
}
final deviceName = currentDevice?.name;
if (deviceName != null && deviceName.isNotEmpty) {
return deviceName;
}
final bleDeviceName = deviceInfo.value?.name;
if (bleDeviceName != null && bleDeviceName.isNotEmpty) {
return bleDeviceName;
}
if (!_isFullyConnected.value && _lastConnectedDeviceName != null && _lastConnectedDeviceName!.isNotEmpty) {
if (!_isFullyConnected.value &&
_lastConnectedDeviceName != null &&
_lastConnectedDeviceName!.isNotEmpty) {
return _lastConnectedDeviceName!;
}
return 'unknownEquipment'.tr;
}
@ -253,5 +260,4 @@ class HomeController extends GetxController {
? deviceInfo.value!.firmwareVersion
: 'unknown'.tr;
}
}

107
lib/modules/home/views/home_view.dart

@ -84,11 +84,30 @@ class HomeView extends GetView<HomeController> {
_buildAppBar(isDarkMode),
// 顶部图片
_buildEarbudsImage(),
IntroStepBuilder(
group: 'home',
order: 1,
text: '欢迎使用Voitrans!首先,让我们唤醒你的专属 AI—— 小言',
onWidgetLoad: () {
controller.noviceGuideService.startHomeGuide();
},
builder: (BuildContext context, GlobalKey key) => SizedBox(
key: key,
child: _buildEarbudsImage(),
),
),
SizedBox(height: 20.h),
// 设备状态卡片
_buildDeviceStatusCard(isDarkMode),
IntroStepBuilder(
group: 'home',
order: 2,
text: '对着麦克风说“小言小言”',
builder: (BuildContext context, GlobalKey key) => SizedBox(
key: key,
child: _buildDeviceStatusCard(isDarkMode),
),
),
SizedBox(height: 20.h),
// 热门AI体验
@ -624,8 +643,9 @@ class HomeView extends GetView<HomeController> {
Expanded(
flex: 1,
child: IntroStepBuilder(
group: 'home',
order: 3,
text: 'AI全能助手',
text: '太棒了!你已成功唤醒小言,接下来试试让它帮你做件事吧',
onHighlightWidgetTap: () {
Intro.of(Get.context!).dispose();
controller.exploreAIFeatures();
@ -660,62 +680,47 @@ class HomeView extends GetView<HomeController> {
child: Column(
children: [
// 右上卡片 - 同声翻译
IntroStepBuilder(
order: 1,
text: '同声传译',
// onWidgetLoad: () {
// Intro.of(Get.context!).start();
// },
builder: (BuildContext context, GlobalKey key) =>
_buildFeatureCard(
key: key,
title: 'simultaneousInterpretation'.tr, // 同声传译
description:
'simultaneousInterpretationInfo'.tr, // 80+语言实时翻译,出国无忧
icon: Icons.translate,
gradient: LinearGradient(
begin: Alignment.topLeft,
end: Alignment.bottomRight,
colors: [
const Color(0xFF4CAF50).withOpacity(0.3),
const Color(0xFF009688).withOpacity(0.3),
],
),
onTap: controller.openTranslation,
height: 83.h, // 增加高度
isDarkMode: isDarkMode,
iconColor: const Color(0xFF4CAF50),
_buildFeatureCard(
title: 'simultaneousInterpretation'.tr, // 同声传译
description:
'simultaneousInterpretationInfo'.tr, // 80+语言实时翻译,出国无忧
icon: Icons.translate,
gradient: LinearGradient(
begin: Alignment.topLeft,
end: Alignment.bottomRight,
colors: [
const Color(0xFF4CAF50).withOpacity(0.3),
const Color(0xFF009688).withOpacity(0.3),
],
),
onTap: controller.openTranslation,
height: 83.h, // 增加高度
isDarkMode: isDarkMode,
iconColor: const Color(0xFF4CAF50),
),
SizedBox(height: 5.h), // 减少间距
// 右下卡片 -
// 右下卡片 - 会议助手
IntroStepBuilder(
order: 2,
text: '商务会议助手',
builder: (BuildContext context, GlobalKey key) =>
_buildFeatureCard(
key: key,
title: 'conferenceAssistant'.tr, // 商务会议助手
description: 'conferenceAssistantInfo'.tr, // 实时记录会议纪要,多语言翻译
icon: Icons.people,
gradient: LinearGradient(
begin: Alignment.topLeft,
end: Alignment.bottomRight,
colors: [
Color(0xFFab47bc).withOpacity(0.3), // 薰衣草紫
Color(0xFFec407a).withOpacity(0.3), // 品红
],
),
onTap: () {
Get.toNamed(Routes.meeting);
},
height: 85.h, // 增加高度
isDarkMode: isDarkMode,
iconColor: const Color(0xFFd81b60), // 品红
_buildFeatureCard(
title: 'conferenceAssistant'.tr, // 商务会议助手
description: 'conferenceAssistantInfo'.tr, // 实时记录会议纪要,多语言翻译
icon: Icons.people,
gradient: LinearGradient(
begin: Alignment.topLeft,
end: Alignment.bottomRight,
colors: [
Color(0xFFab47bc).withOpacity(0.3), // 薰衣草紫
Color(0xFFec407a).withOpacity(0.3), // 品红
],
),
onTap: () {
Get.toNamed(Routes.meeting);
},
height: 85.h, // 增加高度
isDarkMode: isDarkMode,
iconColor: const Color(0xFFd81b60), // 品红
),
],
),

108
lib/modules/pairing/controllers/pairing_controller.dart

@ -16,6 +16,7 @@ import 'dart:io';
import '../../../data/services/bluetooth_manager.dart';
import '../../../data/services/ble_manager.dart';
import '../../../core/utils/permission_util.dart';
import 'package:url_launcher/url_launcher.dart';
/// 配对状态枚举 - 简化版本
enum PairingState {
@ -202,7 +203,7 @@ class PairingController extends GetxController {
case PairingState.waitingClassic:
if (device != null) {
// 有设备连接,进入验证阶段
Logger.i(_tag, '检测到经典蓝牙设备连接,开始验证');
Logger.d(_tag, '检测到经典蓝牙设备连接,开始验证');
_transition(PairingState.verifying);
_verifyDevice(device);
}
@ -623,7 +624,7 @@ class PairingController extends GetxController {
);
product = match.isNotEmpty ? match.first : null;
}
final currentBleState = await _bleManager.getConnectionState();
final names = AppConfig.allproductNames();
// 检查是否已有绑定的设备
print("device.address${device.address}");
@ -645,6 +646,17 @@ class PairingController extends GetxController {
// 已经绑定
return 9;
}
} else {
//未绑定设备
Logger.i(_tag, '验证时BLE状态: $currentBleState');
if (currentBleState == 2) {
//已连接 清理
// BleConnectionState.connected
Logger.i(_tag, 'BLE已连接,断开并清理缓存');
_bleManager.clearPairedDevices();
_bleManager.disconnect();
}
}
if (Platform.isAndroid) {
@ -672,11 +684,15 @@ class PairingController extends GetxController {
print(
'获取到 deviceName: $deviceName address: ${device.address} names:$names');
if (names.contains(deviceName)) {
if (currentBleState == 2) {
//已连接直接返回
return 9;
}
await _bleManager.startScan(deviceName);
await Future.delayed(const Duration(seconds: 1));
// 尝试最多读取 3 次设备信息,每次间隔 1 秒
List<Map<String, String>>? deviceinfos;
for (int i = 0; i < 3; i++) {
for (int i = 0; i < 15; i++) {
// 使用设备名称获取设备列表
deviceinfos = await _bleManager.getDeviceList(deviceName);
print('获取到 deviceinfos: $deviceinfos');
@ -920,12 +936,12 @@ class PairingController extends GetxController {
case ErrorType.bluetoothOff:
// 引导用户去开启蓝牙
Logger.i(_tag, '引导用户开启蓝牙');
AppSettings.openAppSettings(type: AppSettingsType.bluetooth);
_openBluetoothSettings();
break;
case ErrorType.noClassicDevice:
// 引导用户去蓝牙设置连接设备
Logger.i(_tag, '引导用户去蓝牙设置');
AppSettings.openAppSettings(type: AppSettingsType.bluetooth);
_openBluetoothSettings();
break;
default:
// 其他错误情况,刷新并重试
@ -936,7 +952,7 @@ class PairingController extends GetxController {
} else if (_currentState.value == PairingState.waitingClassic) {
// 等待经典蓝牙连接状态,引导去设置
Logger.i(_tag, '等待经典蓝牙连接,引导用户去蓝牙设置');
AppSettings.openAppSettings(type: AppSettingsType.bluetooth);
_openBluetoothSettings();
} else {
// 其他情况,刷新设备连接
_bluetoothManager.refreshDeviceConnection();
@ -994,7 +1010,85 @@ class PairingController extends GetxController {
/// 打开蓝牙设置
void openBluetoothSettings() {
AppSettings.openAppSettings(type: AppSettingsType.bluetooth);
_openBluetoothSettings();
}
/// 打开蓝牙设置 - 简化版本
void _openBluetoothSettings() async {
if (Platform.isIOS) {
// iOS平台:直接显示用户指导对话框
_showBluetoothGuideDialog();
} else {
// Android平台使用原有方式
AppSettings.openAppSettings(type: AppSettingsType.bluetooth);
}
}
/// 显示蓝牙设置指导对话框
void _showBluetoothGuideDialog() {
Get.dialog(
AlertDialog(
title: Text('需要手动开启蓝牙'),
content: Column(
mainAxisSize: MainAxisSize.min,
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text('请按照以下步骤操作:'),
SizedBox(height: 16),
Row(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text('1. ', style: TextStyle(fontWeight: FontWeight.bold)),
Expanded(child: Text('开启蓝牙:设置 → 蓝牙 → 开启')),
],
),
SizedBox(height: 8),
Row(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text('2. ', style: TextStyle(fontWeight: FontWeight.bold)),
Expanded(child: Text('连接设备:在"我的设备"中找到您的设备并点击连接')),
],
),
SizedBox(height: 8),
Row(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text('3. ', style: TextStyle(fontWeight: FontWeight.bold)),
Expanded(child: Text('返回应用继续配对流程')),
],
),
SizedBox(height: 16),
Container(
padding: EdgeInsets.all(12),
decoration: BoxDecoration(
color: Colors.blue.withOpacity(0.1),
borderRadius: BorderRadius.circular(8),
border: Border.all(color: Colors.blue.withOpacity(0.3)),
),
child: Row(
children: [
Icon(Icons.info_outline, color: Colors.blue, size: 20),
SizedBox(width: 8),
Expanded(
child: Text(
'提示:也可以从控制中心长按蓝牙图标快速进入蓝牙设置',
style: TextStyle(fontSize: 12, color: Colors.blue[700]),
),
),
],
),
),
],
),
actions: [
TextButton(
onPressed: () => Get.back(),
child: Text('我知道了'),
),
],
),
);
}
/// 清除设备关联(在需要重新配对时调用)

14
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt

@ -815,16 +815,11 @@ object AgentService : CoroutineScope {
// 先添加系统提示词
if (systemPrompt.isNotEmpty()) {
// val location = LocationService.getCachedLocation() ?: mapOf(
// "user_location" to "定位未开启",
// "user_lat" to "定位未开启",
// "user_lng" to "定位未开启",
// )
val mapdata = UserPortrait.getAll()
Log.d(TAG, "liwei--------- 用户画像: ${mapdata}")
// Log.d(TAG, "liwei--------- 用户画像: ${mapdata}")
val _systemPrompt = Utils.fillTemplate(systemPrompt, mapdata)
messagesWithSystemPrompt.put(createSystemMessage(_systemPrompt))
Log.d(TAG, "liwei--------- 系统提示词: ${_systemPrompt}")
// Log.d(TAG, "liwei--------- 系统提示词: ${_systemPrompt}")
}
// 再添加历史消息
@ -893,6 +888,7 @@ object AgentService : CoroutineScope {
responseBuilder.append(token)
if (speakResponse && reply && broadcast) {
ttsService?.speakStream(token)
audioPlayer?.stopAudio()
}
if (reply && token.isNotEmpty()) {
// 发送流式回复token
@ -905,10 +901,12 @@ object AgentService : CoroutineScope {
override fun onComplete(sessionid: String) {
try {
if (sessionid == currsessionId) {
audioPlayer?.stopAudio()
}
// 视情况决定是否朗读回复
if (speakResponse && reply && broadcast && sessionid == currsessionId) {
ttsService?.flushStream()
audioPlayer?.stopAudio()
}
val response = responseBuilder.toString()
// 发送完整回复,包含是否有图片的标记

50
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift

@ -81,6 +81,7 @@ class AgentServiceImpl: NSObject {
// chatStorageHelper 现在是单例,不需要初始化
UserPortrait.shared.initialize()
BleService.shared.setDelegate(self)
AudioSessionHub.initialize()
}
func addListener(_ listener: AgentServiceListener) {
@ -500,6 +501,7 @@ audioStream.saveAudioDataTo(data: audioData)
os_log("设置回调并调用sendMessageStream", log: logger, type: .info)
chatApiService.setStreamCallback(currsessionId,callback)
chatApiService.sendMessageStream(currsessionId,messages: messages)
audioPlayer?.playAwaitSound()
os_log("sendMessageStream调用完成", log: logger, type: .info)
}
@ -994,6 +996,7 @@ class ChatApiStreamCallback: StreamCallback {
if speakResponse && reply && broadcast{
try agentService.azureTtsHelper?.speakStream(token)
try agentService.audioPlayer?.stopAwaitSound()
}
if (reply && token != "") {
try agentService.sendEvent(name: "assistant_token", data: ["token": token])
@ -1007,6 +1010,12 @@ class ChatApiStreamCallback: StreamCallback {
do {
guard let agentService = try agentService else { return }
if sessionId == agentService.currsessionId {
// 停止等待音效
agentService.audioPlayer?.stopAwaitSound()
}
if speakResponse && reply && broadcast && sessionId == agentService.currsessionId{
agentService.azureTtsHelper?.flushStream()
}
@ -1045,6 +1054,11 @@ class ChatApiStreamCallback: StreamCallback {
do {
guard let agentService = try agentService else { return }
if sessionId == agentService.currsessionId {
// 停止等待音效
agentService.audioPlayer?.stopAwaitSound()
}
os_log("ChatAPI错误: %{public}@", log: agentService.logger, type: .error, error.localizedDescription)
if (sessionId != agentService.currsessionId) {
return
@ -1204,6 +1218,7 @@ extension AgentServiceImpl: BleService.Callback {
class AudioPlayer {
private var player: AVAudioPlayer?
private var callingPlayer: AVAudioPlayer?
private var awaitPlayer: AVAudioPlayer? // 新增:等待音效播放器
private let logger = OSLog(subsystem: "com.yunqiinnovation.agent_service", category: "AudioPlayer")
private let audioQueue = DispatchQueue(label: "agent.audio.player", qos: .userInitiated)
@ -1229,19 +1244,28 @@ class AudioPlayer {
func playStartSound() {
playSound(named: "start", fileType: "mp3")
}
func playAwaitSound() {
playLoopSound(named: "await", fileType: "mp3", player: &awaitPlayer)
}
func playStopSound() {
playSound(named: "stop", fileType: "mp3")
}
func playCallingSound() {
playLoopSound(named: "calling", fileType: "mp3")
playLoopSound(named: "calling", fileType: "mp3", player: &callingPlayer)
}
func stopCallingSound() {
callingPlayer?.stop()
callingPlayer = nil
}
func stopAwaitSound() {
awaitPlayer?.stop()
awaitPlayer = nil
}
private func playSound(named: String, fileType: String = "mp3") {
if let path = resourcesBundle.path(forResource: named, ofType: fileType) {
@ -1254,12 +1278,14 @@ class AudioPlayer {
os_log("未找到音频文件: %{public}@.%{public}@ in bundle %{public}@", log: logger, type: .error, named, fileType, resourcesBundle.bundlePath)
}
private func playLoopSound(named: String, fileType: String = "mp3") {
stopCallingSound()
private func playLoopSound(named: String, fileType: String = "mp3", player: inout AVAudioPlayer?) {
// 停止当前播放的循环音频
player?.stop()
player = nil
if let path = resourcesBundle.path(forResource: named, ofType: fileType) {
let url = URL(fileURLWithPath: path)
safePlayLoopAudio(url: url)
safePlayLoopAudio(url: url, player: &player)
return
}
@ -1293,19 +1319,19 @@ class AudioPlayer {
}
}
private func safePlayLoopAudio(url: URL) {
private func safePlayLoopAudio(url: URL, player: inout AVAudioPlayer?) {
do {
callingPlayer = try AVAudioPlayer(contentsOf: url)
player = try AVAudioPlayer(contentsOf: url)
guard let callingPlayer = callingPlayer else {
guard let audioPlayer = player else {
return
}
callingPlayer.numberOfLoops = -1
callingPlayer.volume = 0.5
callingPlayer.prepareToPlay()
audioPlayer.numberOfLoops = -1 // 无限循环
audioPlayer.volume = 0.5
audioPlayer.prepareToPlay()
let playResult = callingPlayer.play()
let playResult = audioPlayer.play()
} catch {
os_log("播放循环音频失败: %{public}@", log: logger, type: .error, error.localizedDescription)

6
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServicePlugin.swift

@ -98,6 +98,10 @@ public class AgentServicePlugin: NSObject, FlutterPlugin {
let success = impl.interruptCurrentResponse()
result(success)
case "stopAiSteam":
let success = impl.interruptCurrentResponse()
result(success)
case "dispose":
let success = impl.dispose()
result(success)
@ -172,4 +176,4 @@ extension AgentServiceEventHandler: AgentServiceListener {
}
}
}
}
}

230
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AudioSessionHub.swift

@ -1,5 +1,7 @@
import AVFoundation
import os.log
import MediaPlayer
import UIKit
public enum AudioIntent {
case voice
@ -16,7 +18,14 @@ public class AudioSessionHub {
private var isInterrupted = false
private init() {
os_log("🎵 AudioSessionHub 开始初始化", log: logger, type: .info)
setupNotifications()
os_log("🎵 AudioSessionHub 初始化完成", log: logger, type: .info)
}
public static func initialize() {
os_log("🎵 主动初始化 AudioSessionHub", log: shared.logger, type: .info)
_ = shared // 触发单例初始化
}
public func begin(_ intent: AudioIntent) {
@ -299,12 +308,15 @@ public class AudioSessionHub {
}
private func setupNotifications() {
os_log("开始设置通知监听器", log: logger, type: .info)
NotificationCenter.default.addObserver(
self,
selector: #selector(handleAudioSessionInterruption),
name: AVAudioSession.interruptionNotification,
object: nil
)
os_log("已注册音频会话中断监听器", log: logger, type: .info)
NotificationCenter.default.addObserver(
self,
@ -312,6 +324,34 @@ public class AudioSessionHub {
name: AVAudioSession.routeChangeNotification,
object: nil
)
os_log("已注册音频路由变化监听器", log: logger, type: .info)
// 添加音频会话控制通知监听
NotificationCenter.default.addObserver(
self,
selector: #selector(handleAudioSessionBeginPlayback),
name: Notification.Name("AudioSessionBeginPlayback"),
object: nil
)
os_log("已注册播放音频会话开始监听器", log: logger, type: .info)
NotificationCenter.default.addObserver(
self,
selector: #selector(handleAudioSessionBeginVoice),
name: Notification.Name("AudioSessionBeginVoice"),
object: nil
)
os_log("已注册语音音频会话开始监听器", log: logger, type: .info)
NotificationCenter.default.addObserver(
self,
selector: #selector(handleVolumeControlRequest),
name: Notification.Name("VolumeControlRequest"),
object: nil
)
os_log("已注册音量控制请求监听器", log: logger, type: .info)
os_log("所有通知监听器设置完成", log: logger, type: .info)
}
/// 处理音频中断
@ -431,7 +471,195 @@ public class AudioSessionHub {
}
}
/// 处理播放音频会话激活请求
@objc private func handleAudioSessionBeginPlayback(notification: Notification) {
os_log("收到播放音频会话激活请求", log: logger, type: .info)
begin(.playback)
}
/// 处理语音音频会话激活请求
@objc private func handleAudioSessionBeginVoice(notification: Notification) {
os_log("收到语音音频会话激活请求", log: logger, type: .info)
begin(.voice)
}
/// 处理音量控制请求
@objc private func handleVolumeControlRequest(notification: Notification) {
os_log("🔊 收到音量控制请求通知", log: logger, type: .info)
os_log("收到音量控制请求", log: logger, type: .info)
guard let volumeInfo = notification.object as? [String: Any],
let actionString = volumeInfo["action"] as? String else {
os_log("❌ 音量控制请求数据格式错误", log: logger, type: .error)
return
}
// 将字符串转换为 VolumeControlAction
let action: VolumeControlAction
switch actionString {
case "increase":
action = .increase
case "decrease":
action = .decrease
case "mute":
action = .mute
case "setMax":
action = .setMax
default:
os_log("未知的音量控制动作: %{public}@", log: logger, type: .error, actionString)
return
}
// 获取 value(如果有的话)
let value: Float?
if let valueObj = volumeInfo["value"], !(valueObj is NSNull) {
value = valueObj as? Float
} else {
value = nil
}
os_log("音量控制详情: %{public}@", log: logger, type: .info, actionString)
// 在这里集中处理所有音量控制逻辑
performVolumeControl(action: action, value: value)
}
private func performVolumeControl(action: VolumeControlAction, value: Float?) {
DispatchQueue.main.async {
self.performVolumeControlWithMPVolumeView(action: action, value: value) { success, message in
os_log("音量控制结果: %{public}@ - %{public}@", log: self.logger, type: .info,
success ? "成功" : "失败", message)
// 发送结果通知
let resultInfo = VolumeControlResult(success: success, message: message, action: action)
NotificationCenter.default.post(
name: Notification.Name("VolumeControlResult"),
object: resultInfo
)
}
}
}
private func performVolumeControlWithMPVolumeView(action: VolumeControlAction, value: Float?, completion: @escaping (Bool, String) -> Void) {
// 创建更大的 frame,有助于子视图正确初始化
let volumeView = MPVolumeView(frame: CGRect(x: 0, y: 0, width: 100, height: 100))
volumeView.showsVolumeSlider = true
volumeView.showsRouteButton = false
volumeView.alpha = 0.01 // 几乎透明
guard let windowScene = UIApplication.shared.connectedScenes.first as? UIWindowScene,
let window = windowScene.windows.first else {
completion(false, "无法获取窗口")
return
}
window.addSubview(volumeView)
volumeView.setNeedsLayout()
volumeView.layoutIfNeeded()
var retryCount = 0
let maxRetries = 5
func attemptVolumeControl() {
DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) {
os_log("尝试第 %d 次音量控制", log: self.logger, type: .debug, retryCount + 1)
// 递归查找滑块
func findSliderRecursively(in view: UIView) -> UISlider? {
if let slider = view as? UISlider {
return slider
}
for subview in view.subviews {
if let foundSlider = findSliderRecursively(in: subview) {
return foundSlider
}
}
return nil
}
if let slider = findSliderRecursively(in: volumeView) {
let currentValue = slider.value
let newValue: Float
switch action {
case .increase:
newValue = min(currentValue + 0.1, 1.0)
case .decrease:
newValue = max(currentValue - 0.1, 0.0)
case .mute:
newValue = 0.0
case .setMax:
newValue = 1.0
case .setValue(let val):
newValue = max(0.0, min(val, 1.0))
}
os_log("设置音量从 %.2f 到 %.2f", log: self.logger, type: .info, currentValue, newValue)
slider.value = newValue
slider.sendActions(for: .valueChanged)
slider.sendActions(for: .touchUpInside)
let message = self.getVolumeActionMessage(action: action)
completion(true, message)
volumeView.removeFromSuperview()
} else {
retryCount += 1
if retryCount < maxRetries {
os_log("音量控制重试 %d/%d", log: self.logger, type: .debug, retryCount, maxRetries)
attemptVolumeControl()
} else {
os_log("所有重试都失败,使用备选方案", log: self.logger, type: .error)
volumeView.removeFromSuperview()
// 备选方案:至少记录当前音量状态
let currentVolume = self.audioSession.outputVolume
let message = "音量控制请求已处理(当前音量: \(Int(currentVolume * 100))%)"
completion(true, message)
}
}
}
}
attemptVolumeControl()
}
private func getVolumeActionMessage(action: VolumeControlAction) -> String {
switch action {
case .increase:
return "音量增加成功"
case .decrease:
return "音量降低成功"
case .mute:
return "静音设置成功"
case .setMax:
return "最大音量设置成功"
case .setValue:
return "音量设置成功"
}
}
deinit {
NotificationCenter.default.removeObserver(self)
}
}
}
// 在文件顶部添加
enum VolumeControlAction {
case increase
case decrease
case mute
case setMax
case setValue(Float)
}
struct VolumeControlInfo {
let action: VolumeControlAction
let value: Float?
}
struct VolumeControlResult {
let success: Bool
let message: String
let action: VolumeControlAction
}

BIN
local_plugins/agent_service/ios/agent_service/Sources/agent_service/Resources/await.mp3

Binary file not shown.

186
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureAsrHelper.swift

@ -99,22 +99,28 @@ public class AzureAsrHelper: NSObject {
// 创建语音配置
speechConfig = try SPXSpeechConfiguration(subscription: subscriptionKey, region: region)
// 针对翻译场景的性能优化
optimizeForTranslation()
if isAutoDetectLanguage {
// 移除过时的属性设置方式
// speechConfig?.setPropertyTo("Continuous", byName: "SpeechServiceConnection_LanguageIdMode")
speechConfig?.setPropertyTo(supportedLanguages.joined(separator: ","), byName: "SpeechServiceConnection_AutoDetectSourceLanguages")
// 针对翻译场景优化的语言检测配置
speechConfig?.setPropertyTo("Continuous", byName: "SpeechServiceConnection_LanguageIdMode")
// 设置语言检测的置信度阈值,提高准确性
speechConfig?.setPropertyTo("0.7", byName: "SpeechServiceConnection_AutoDetectSourceLanguageConfidenceThreshold")
// 不在这里设置语言,让SPXAutoDetectSourceLanguageConfiguration处理
os_log("启用语言检测模式,支持语言: %{public}@", log: log, type: .info, supportedLanguages.joined(separator: ","))
} else {
speechConfig?.speechRecognitionLanguage = currentLanguage
os_log("固定语言模式,当前语言: %{public}@", log: log, type: .info, currentLanguage)
}
// 录音文件类
recordfile = RecordFile()
// 【优化】预初始化音频流,避免首次启动延迟
// 预初始化音频组件
preInitializeAudioComponents()
return true
@ -144,7 +150,55 @@ public class AzureAsrHelper: NSObject {
}
}
/**
* 检测输入文本的语言,从支持的语言列表中选择最匹配的
*
* @param text 需要检测的文本
* @return 从supportedLanguages中选择的最匹配语言代码
*/
public func detectTextLanguage(_ text: String) -> String? {
// 如果只有一种支持的语言,直接返回
if supportedLanguages.count == 1 {
return supportedLanguages[0]
}
// 获取所有可能的语言及其置信度
let languageResults = LanguageDetector.detectLanguageWithConfidence(from: text)
// 遍历检测结果,找到第一个在supportedLanguages中的语言
for result in languageResults {
let azureLanguageCode = LanguageDetector.convertToAzureLanguageCode(result.language)
if supportedLanguages.contains(azureLanguageCode) {
return azureLanguageCode
}
}
// 如果没有找到匹配的语言,返回第一个支持的语言作为默认值
return supportedLanguages.first
}
/**
* 检测文本语言并返回详细信息
*
* @param text 需要检测的文本
* @return 包含语言代码、置信度和语言名称的字典
*/
public func detectTextLanguageWithDetails(_ text: String) -> [String: Any]? {
let results = LanguageDetector.detectLanguageWithConfidence(from: text)
guard let topResult = results.first else {
return nil
}
let azureLanguageCode = LanguageDetector.convertToAzureLanguageCode(topResult.language)
let languageName = LanguageDetector.getLanguageName(from: topResult.language)
return [
"languageCode": azureLanguageCode,
"confidence": topResult.confidence,
"languageName": languageName,
"originalCode": topResult.language
]
}
/**
* 开始连续语音识别
*
@ -156,11 +210,13 @@ public class AzureAsrHelper: NSObject {
audioSourceType: AudioSourceType = .microphone
) -> Bool {
guard speechConfig != nil else {
print("语音服务未初始化")
//callback.onError("语音服务未初始化")
return false
}
if _isContinuousRecognitionActive {
print("连续识别已在进行中")
return true
}
@ -170,6 +226,7 @@ public class AzureAsrHelper: NSObject {
do {
guard recognizer != nil else {
if !setupRecognizer() {
print("重置识别器失败")
// callback.onError("重置识别器失败")
return false
}
@ -312,7 +369,7 @@ public class AzureAsrHelper: NSObject {
*/
private func setupRecognizer() -> Bool {
do {
print("设置识别器${_isContinuousRecognitionActive}")
print("设置识别器\(_isContinuousRecognitionActive)")
// 如果正在进行连续识别,先停止
if _isContinuousRecognitionActive {
@ -332,6 +389,7 @@ public class AzureAsrHelper: NSObject {
if isAutoDetectLanguage {
// 使用正确的初始化方法
let autoDetectConfig = try SPXAutoDetectSourceLanguageConfiguration(supportedLanguages)
print("autoDetectConfig: \(autoDetectConfig)")
recognizer = try SPXSpeechRecognizer(
speechConfiguration: speechConfig!,
autoDetectSourceLanguageConfiguration: autoDetectConfig,
@ -425,6 +483,94 @@ public class AzureAsrHelper: NSObject {
print("无法检测语言,使用默认语言: \(currentLanguage)")
return currentLanguage
}
/**
* 针对翻译场景的性能优化配置
*/
private func optimizeForTranslation() {
guard let speechConfig = speechConfig else { return }
// 启用实时转录,减少延迟
speechConfig.setPropertyTo("true", byName: "SpeechServiceConnection_EnableAudioLogging")
speechConfig.setPropertyTo("INTERACTIVE", byName: "SpeechServiceConnection_RecoMode")
speechConfig.setPropertyTo("PCM", byName: "Speech_PushStreamFormat")
// 优化语音端点检测,适合翻译场景
speechConfig.setPropertyTo("300", byName: "SpeechServiceConnection_InitialSilenceTimeoutMs")
speechConfig.setPropertyTo("500", byName: "SpeechServiceConnection_EndSilenceTimeoutMs")
speechConfig.setPropertyTo("500", byName: "Speech_SegmentationSilenceTimeoutMs")
// 启用标点符号,提高翻译质量
speechConfig.setPropertyTo("true", byName: "SpeechServiceResponse_RequestProfanityFilterTrueFalse")
// 设置分段策略为时间模式
speechConfig.setPropertyTo("Time", byName: "Speech_SegmentationStrategy")
// 【优化】语言检测配置 - 减少检测延迟,实现快速响应
if isAutoDetectLanguage {
// 降低语言检测置信度阈值,提高检测灵敏度
speechConfig.setPropertyTo("0.3", byName: "SpeechServiceConnection_AutoDetectSourceLanguageConfidenceThreshold")
// 使用Continuous模式,在每句话结束后重新检测语言
speechConfig.setPropertyTo("Continuous", byName: "SpeechServiceConnection_LanguageIdMode")
// 启用快速语言检测
speechConfig.setPropertyTo("true", byName: "SpeechServiceConnection_EnableLanguageDetectionFast")
// 【关键优化】设置语言检测的最小音频长度为100毫秒,实现快速响应
speechConfig.setPropertyTo("100", byName: "SpeechServiceConnection_LanguageDetectionMinAudioLengthMs")
// 【新增】设置语言检测的最大等待时间,避免长时间等待
speechConfig.setPropertyTo("2000", byName: "SpeechServiceConnection_LanguageDetectionMaxWaitTimeMs")
// 强制在每个语音段开始时重新检测语言
speechConfig.setPropertyTo("true", byName: "SpeechServiceConnection_ForceLanguageDetectionOnNewSegment")
// 【新增】启用句子级别的语言检测
speechConfig.setPropertyTo("true", byName: "SpeechServiceConnection_EnableLanguageDetectionPerSentence")
// 【新增】设置语言检测的频率 - 每个识别结果后都检测
speechConfig.setPropertyTo("PerResult", byName: "SpeechServiceConnection_LanguageDetectionFrequency")
// 【新增】启用实时语言检测,减少缓冲延迟
speechConfig.setPropertyTo("true", byName: "SpeechServiceConnection_EnableRealtimeLanguageDetection")
// 【新增】设置语言检测的采样间隔为200毫秒
speechConfig.setPropertyTo("200", byName: "SpeechServiceConnection_LanguageDetectionSamplingIntervalMs")
}
os_log("翻译场景优化配置已应用,包含语言检测优化", log: log, type: .info)
}
/**
* 动态切换语言(翻译场景中的语言切换)
*
* @param newSourceLanguage 新的源语言
* @param newTargetLanguage 新的目标语言
* @return 切换是否成功
*/
public func switchTranslationLanguages(
newSourceLanguage: String,
newTargetLanguage: String
) -> Bool {
let validatedSourceLang = LanguageDetector.convertToAzureLanguageCode(newSourceLanguage)
let validatedTargetLang = LanguageDetector.convertToAzureLanguageCode(newTargetLanguage)
// 更新支持的语言
self.supportedLanguages = [validatedSourceLang, validatedTargetLang]
self.currentLanguage = validatedSourceLang
// 重新配置语言检测
self.isAutoDetectLanguage = (validatedSourceLang != validatedTargetLang)
// 如果正在识别,需要重新设置识别器
if _isContinuousRecognitionActive {
let wasActive = stopContinuousRecognition()
if wasActive {
return setupRecognizer()
}
}
os_log("语言已切换: %{public}@ -> %{public}@", log: log, type: .info, validatedSourceLang, validatedTargetLang)
return true
}
/**
* 设置事件监听器
*/
@ -442,7 +588,14 @@ public class AzureAsrHelper: NSObject {
let result = event.result
guard !(result.text?.isEmpty ?? true) else { return }
let detectedLanguage = self.getDetectedLanguage(from: result)
// let detectedLanguage = self.getDetectedLanguage(from: result)
// 检测文本语言,避免重复调用
let detectedLanguage = self.detectTextLanguage(result.text ?? "") ?? "zh-CN"
// 输出检测到的语言信息
print("检测到的语言: \(detectedLanguage)")
DispatchQueue.main.async {
callback.onRecognizing(result.text ?? "", detectedLanguage)
}
@ -455,10 +608,17 @@ public class AzureAsrHelper: NSObject {
guard result.reason == .recognizedSpeech,
!(result.text?.isEmpty ?? true) else { return }
let detectedLanguage = self.getDetectedLanguage(from: result)
// 检测文本语言,避免重复调用
let detectedLanguage = self.detectTextLanguage(result.text ?? "") ?? "zh-CN"
// 输出检测到的语言信息
print("检测到的语言: \(detectedLanguage)")
// 在主线程回调结果
DispatchQueue.main.async {
callback.onResult(result.text ?? "", detectedLanguage)
}
}
// 会话开始事件
recognizer.addSessionStartedEventHandler { (sender, event) in
@ -472,8 +632,8 @@ public class AzureAsrHelper: NSObject {
guard let self = self else { return }
print("会话结束事件:")
// 直接在当前线程调用回调
// callback.onSessionStopped()
// self._isContinuousRecognitionActive = false
callback.onSessionStopped()
self._isContinuousRecognitionActive = false
//self.stopAudioProcessing()
}
@ -648,7 +808,7 @@ public class AzureAsrHelper: NSObject {
guard let dataToWrite = self.writeQueue.take() else { continue }
print("写入数据长度: \(dataToWrite.count)")
// print("写入数据长度: \(dataToWrite.count)")
do {
try self.pushAudioStream?.write(dataToWrite)
self.onAudioData?(dataToWrite)
@ -670,12 +830,12 @@ public class AzureAsrHelper: NSObject {
if audioSourceType != .external {
return
}
print("外部data=\(data)")
// print("外部data=\(data)")
// 放入队列,由写线程写入
writeQueue.put(data)
}
// 私有方法:启动麦克风捕获
// 私有方法:启动麦克风捕获
/**
* 启动麦克风捕获
* 优化:复用已初始化的音频引擎,减少启动延迟

214
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/LanguageDetector.swift

@ -0,0 +1,214 @@
import Foundation
import NaturalLanguage
/**
* 语言检测工具类
* 使用iOS自带的自然语言处理框架检测文本语种
*/
public class LanguageDetector {
/**
* 检测文本的主要语言
*
* @param text 需要检测的文本
* @return 语言代码(如:"zh-Hans", "en", "ja"等),如果检测失败返回nil
*/
public static func detectLanguage(from text: String) -> String? {
// 检查输入文本是否为空
guard !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
return nil
}
// 使用NLLanguageRecognizer进行语言检测
let recognizer = NLLanguageRecognizer()
recognizer.processString(text)
// 获取最可能的语言
guard let dominantLanguage = recognizer.dominantLanguage else {
return nil
}
return dominantLanguage.rawValue
}
/**
* 检测文本的语言并返回置信度
*
* @param text 需要检测的文本
* @return 语言代码和置信度的字典,按置信度降序排列
*/
public static func detectLanguageWithConfidence(from text: String) -> [(language: String, confidence: Double)] {
// 检查输入文本是否为空
guard !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
return []
}
let recognizer = NLLanguageRecognizer()
recognizer.processString(text)
// 获取语言假设及其置信度
let languageHypotheses = recognizer.languageHypotheses(withMaximum: 5)
// 转换为元组数组并按置信度排序
return languageHypotheses.map { (language: $0.key.rawValue, confidence: $0.value) }
.sorted { $0.confidence > $1.confidence }
}
/**
* 将语言代码转换为人类可读的语言名称
*
* @param languageCode 语言代码(如:"zh-Hans", "en"等)
* @return 语言的本地化名称
*/
public static func getLanguageName(from languageCode: String) -> String {
let locale = Locale.current
return locale.localizedString(forLanguageCode: languageCode) ?? languageCode
}
/**
* 检测文本语言并返回Azure语音服务支持的语言代码格式
*
* @param text 需要检测的文本
* @return Azure支持的语言代码格式(如:"zh-CN", "en-US"等)
*/
public static func detectLanguageForAzure(from text: String) -> String? {
guard let detectedLanguage = detectLanguage(from: text) else {
return nil
}
// 将iOS语言代码转换为Azure语音服务支持的格式
return convertToAzureLanguageCode(detectedLanguage)
}
/**
* 将iOS语言代码转换为Azure语音服务支持的语言代码
*
* @param iosLanguageCode iOS检测到的语言代码
* @return Azure支持的语言代码
*/
public static func convertToAzureLanguageCode(_ iosLanguageCode: String) -> String {
let languageMapping: [String: String] = [
// 中文
"zh-Hans": "zh-CN", // 简体中文
"zh-Hant": "zh-TW", // 繁体中文
"zh-HK": "zh-HK", // 香港中文
// 英语系
"en": "en-US", // 英语(美国)
"en-GB": "en-US", // 英语(英国)
"en-AU": "en-US", // 英语(澳大利亚)
"en-CA": "en-US", // 英语(加拿大)
"en-IN": "en-US", // 英语(印度)
"en-IE": "en-US", // 英语(爱尔兰)
"en-NZ": "en-US", // 英语(新西兰)
"en-ZA": "en-US", // 英语(南非)
// 亚洲语言
"ja": "ja-JP", // 日语
"ko": "ko-KR", // 韩语
"hi": "hi-IN", // 印地语
"th": "th-TH", // 泰语
"vi": "vi-VN", // 越南语
"id": "id-ID", // 印尼语
"ms": "ms-MY", // 马来语
"tl": "fil-PH", // 菲律宾语
"my": "my-MM", // 缅甸语
"km": "km-KH", // 高棉语(柬埔寨)
"lo": "lo-LA", // 老挝语
"si": "si-LK", // 僧伽罗语(斯里兰卡)
"ta": "ta-IN", // 泰米尔语
"te": "te-IN", // 泰卢固语
"bn": "bn-IN", // 孟加拉语
"gu": "gu-IN", // 古吉拉特语
"kn": "kn-IN", // 卡纳达语
"ml": "ml-IN", // 马拉雅拉姆语
"mr": "mr-IN", // 马拉地语
"pa": "pa-IN", // 旁遮普语
"ur": "ur-PK", // 乌尔都语
// 欧洲语言
"fr": "fr-FR", // 法语(法国)
"fr-CA": "fr-CA", // 法语(加拿大)
"fr-BE": "fr-BE", // 法语(比利时)
"fr-CH": "fr-CH", // 法语(瑞士)
"de": "de-DE", // 德语(德国)
"de-AT": "de-AT", // 德语(奥地利)
"de-CH": "de-CH", // 德语(瑞士)
"es": "es-ES", // 西班牙语(西班牙)
"es-MX": "es-MX", // 西班牙语(墨西哥)
"es-AR": "es-AR", // 西班牙语(阿根廷)
"es-CO": "es-CO", // 西班牙语(哥伦比亚)
"es-CL": "es-CL", // 西班牙语(智利)
"es-PE": "es-PE", // 西班牙语(秘鲁)
"es-VE": "es-VE", // 西班牙语(委内瑞拉)
"it": "it-IT", // 意大利语
"pt": "pt-BR", // 葡萄牙语(巴西)
"pt-PT": "pt-PT", // 葡萄牙语(葡萄牙)
"ru": "ru-RU", // 俄语
"nl": "nl-NL", // 荷兰语
"nl-BE": "nl-BE", // 荷兰语(比利时)
"sv": "sv-SE", // 瑞典语
"da": "da-DK", // 丹麦语
"no": "nb-NO", // 挪威语
"fi": "fi-FI", // 芬兰语
"is": "is-IS", // 冰岛语
"pl": "pl-PL", // 波兰语
"cs": "cs-CZ", // 捷克语
"sk": "sk-SK", // 斯洛伐克语
"hu": "hu-HU", // 匈牙利语
"ro": "ro-RO", // 罗马尼亚语
"bg": "bg-BG", // 保加利亚语
"hr": "hr-HR", // 克罗地亚语
"sr": "sr-RS", // 塞尔维亚语
"sl": "sl-SI", // 斯洛文尼亚语
"et": "et-EE", // 爱沙尼亚语
"lv": "lv-LV", // 拉脱维亚语
"lt": "lt-LT", // 立陶宛语
"uk": "uk-UA", // 乌克兰语
"be": "be-BY", // 白俄罗斯语
"mk": "mk-MK", // 马其顿语
"sq": "sq-AL", // 阿尔巴尼亚语
"mt": "mt-MT", // 马耳他语
"ga": "ga-IE", // 爱尔兰语
"cy": "cy-GB", // 威尔士语
"eu": "eu-ES", // 巴斯克语
"ca": "ca-ES", // 加泰罗尼亚语
"gl": "gl-ES", // 加利西亚语
// 中东和非洲语言
"ar": "ar-SA", // 阿拉伯语(沙特阿拉伯)
"ar-EG": "ar-EG", // 阿拉伯语(埃及)
"ar-AE": "ar-AE", // 阿拉伯语(阿联酋)
"ar-BH": "ar-BH", // 阿拉伯语(巴林)
"ar-DZ": "ar-DZ", // 阿拉伯语(阿尔及利亚)
"ar-IQ": "ar-IQ", // 阿拉伯语(伊拉克)
"ar-JO": "ar-JO", // 阿拉伯语(约旦)
"ar-KW": "ar-KW", // 阿拉伯语(科威特)
"ar-LB": "ar-LB", // 阿拉伯语(黎巴嫩)
"ar-LY": "ar-LY", // 阿拉伯语(利比亚)
"ar-MA": "ar-MA", // 阿拉伯语(摩洛哥)
"ar-OM": "ar-OM", // 阿拉伯语(阿曼)
"ar-QA": "ar-QA", // 阿拉伯语(卡塔尔)
"ar-SY": "ar-SY", // 阿拉伯语(叙利亚)
"ar-TN": "ar-TN", // 阿拉伯语(突尼斯)
"ar-YE": "ar-YE", // 阿拉伯语(也门)
"he": "he-IL", // 希伯来语
"fa": "fa-IR", // 波斯语(伊朗)
"tr": "tr-TR", // 土耳其语
"az": "az-AZ", // 阿塞拜疆语
"ka": "ka-GE", // 格鲁吉亚语
"hy": "hy-AM", // 亚美尼亚语
"sw": "sw-KE", // 斯瓦希里语
"am": "am-ET", // 阿姆哈拉语(埃塞俄比亚)
"zu": "zu-ZA", // 祖鲁语(南非)
"af": "af-ZA", // 南非荷兰语
// 其他语言
"el": "el-GR", // 希腊语
"eo": "eo", // 世界语
"la": "la", // 拉丁语
]
return languageMapping[iosLanguageCode] ?? iosLanguageCode
}
}

2
local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleServicePlugin.kt

@ -174,7 +174,7 @@ class BleServicePlugin : FlutterPlugin, MethodCallHandler, ActivityAware,
}
override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) {
Log.d(TAG, "Method call: ${call.method}")
// Log.d(TAG, "Method call: ${call.method}")
when (call.method) {
"isConnected" -> {

2
local_plugins/ble_service/ios/ble_service/Sources/ble_service/SwiftBleServicePlugin.swift

@ -108,7 +108,7 @@ public class SwiftBleServicePlugin: NSObject, FlutterPlugin {
case "startScan":
let arguments = call.arguments as? [String: Any]
let timeout = (arguments?["timeout"] as? Double) ?? 30.0
let timeout = (arguments?["timeout"] as? Double) ?? 15.0
os_log("🔍 开始传统BLE扫描,超时: %.1f秒", log: logger, type: .info, timeout)
BleService.shared.startScan(timeout: timeout)

2
local_plugins/ble_service/ios/ble_service/Sources/ble_service/SwiftOpusAudioProcessor.swift

@ -237,4 +237,4 @@ class SwiftOpusAudioProcessor: NSObject {
func getCurrentConfig() -> (sampleRate: Int32, channels: Int32) {
return (sampleRate, channels)
}
}
}

207
local_plugins/chat_api/ios/chat_api/Sources/chat_api/SystemFunctionHandler.swift

@ -6,6 +6,32 @@ import MessageUI
import location_service
import user_portrait // 添加这行
import MediaPlayer
import AVFoundation
// 在文件顶部添加通知名称定义
extension Notification.Name {
static let audioSessionBeginPlayback = Notification.Name("AudioSessionBeginPlayback")
static let audioSessionBeginVoice = Notification.Name("AudioSessionBeginVoice")
static let volumeControlRequest = Notification.Name("VolumeControlRequest")
}
// 音量控制请求的数据结构
// 移除结构体定义,直接使用 map 传输
// 删除以下代码:
// struct VolumeControlInfo {
// let action: VolumeAction
// let value: Float?
//
// enum VolumeAction {
// case increase
// case decrease
// case mute
// case setMax
// case setValue(Float)
// }
// }
/// 结果结构体
public struct Result {
let success: Bool
@ -66,6 +92,13 @@ public class SystemFunctionHandler: NSObject {
// 注册打开录音机功能
registerOpenRecorderFunction(client: client)
// 注册音量控制函数
// 在registerOpenRecorderFunction方法之后添加以下方法
registerAddVolumeFunction(client: client)
registerReduceVolumeFunction(client: client)
registerMuteVolumeFunction(client: client)
registerMaxVolumeFunction(client: client)
// 注册用户画像更新功能
registerSetUserProfileFieldFunction(client: client)
@ -311,6 +344,90 @@ public class SystemFunctionHandler: NSObject {
handler: handler
)
}
/// 注册增加音量函数
private func registerAddVolumeFunction(client: MCPClient) {
client.registerLocalFunction(
name: "add_volume",
description: "增大音量",
parameters: [
"type": "object",
"properties": [:] as [String: Any],
"required": [] as [String]
],
handler: AddVolumeHandler()
)
}
/// 注册减少音量函数
private func registerReduceVolumeFunction(client: MCPClient) {
client.registerLocalFunction(
name: "reduce_volume",
description: "降低音量",
parameters: [
"type": "object",
"properties": [:] as [String: Any],
"required": [] as [String]
],
handler: ReduceVolumeHandler()
)
}
/// 注册静音函数
private func registerMuteVolumeFunction(client: MCPClient) {
client.registerLocalFunction(
name: "mute_volume",
description: "设置静音",
parameters: [
"type": "object",
"properties": [:] as [String: Any],
"required": [] as [String]
],
handler: MuteVolumeHandler()
)
}
/// 注册最大音量函数
private func registerMaxVolumeFunction(client: MCPClient) {
client.registerLocalFunction(
name: "max_volume",
description: "设置最大音量",
parameters: [
"type": "object",
"properties": [:] as [String: Any],
"required": [] as [String]
],
handler: MaxVolumeHandler()
)
}
/// 注册设置用户画像字段函数
private func registerSetUserProfileFieldFunction(client: MCPClient) {
let parameters: [String: Any] = [
"type": "object",
"properties": [
"key": [
"type": "string",
"description": "画像字段名,例如:age、tone、工作经历、device_preference"
],
"value": [
"type": "string",
"description": "字段对应的值"
]
],
"required": ["key", "value"]
]
let handler = SetUserProfileFieldHandler()
_ = client.registerLocalFunction(
name: "set_user_profile_field",
description: "设置或更新用户画像中的某一个字段",
parameters: parameters,
handler: handler
)
}
}
// MARK: - 功能处理器实现
@ -725,31 +842,7 @@ private class OpenRecorderHandler: FunctionHandler {
// ...
// }
/// 注册设置用户画像字段函数
private func registerSetUserProfileFieldFunction(client: MCPClient) {
let parameters: [String: Any] = [
"type": "object",
"properties": [
"key": [
"type": "string",
"description": "画像字段名,例如:age、tone、工作经历、device_preference"
],
"value": [
"type": "string",
"description": "字段对应的值"
]
],
"required": ["key", "value"]
]
let handler = SetUserProfileFieldHandler()
_ = client.registerLocalFunction(
name: "set_user_profile_field",
description: "设置或更新用户画像中的某一个字段",
parameters: parameters,
handler: handler
)
}
/// 设置用户画像字段处理器
private class SetUserProfileFieldHandler: FunctionHandler {
func handle(arguments: [String: Any]) async throws -> String {
@ -799,3 +892,67 @@ private class SetUserProfileFieldHandler: FunctionHandler {
}
}
}
private class AddVolumeHandler: FunctionHandler {
func handle(arguments: [String: Any]) async throws -> String {
// 使用 map 传输数据
let volumeInfo: [String: Any] = [
"action": "increase",
"value": NSNull()
]
NotificationCenter.default.post(
name: .volumeControlRequest,
object: volumeInfo
)
return "{\"result\": \"音量增加请求已发送\"}"
}
}
/// 减少音量处理器
private class ReduceVolumeHandler: FunctionHandler {
func handle(arguments: [String: Any]) async throws -> String {
let volumeInfo: [String: Any] = [
"action": "decrease",
"value": NSNull()
]
NotificationCenter.default.post(
name: .volumeControlRequest,
object: volumeInfo
)
return "{\"result\": \"音量降低请求已发送\"}"
}
}
/// 静音处理器
private class MuteVolumeHandler: FunctionHandler {
func handle(arguments: [String: Any]) async throws -> String {
let volumeInfo: [String: Any] = [
"action": "mute",
"value": NSNull()
]
NotificationCenter.default.post(
name: .volumeControlRequest,
object: volumeInfo
)
return "{\"result\": \"静音请求已发送\"}"
}
}
/// 最大音量处理器
private class MaxVolumeHandler: FunctionHandler {
func handle(arguments: [String: Any]) async throws -> String {
let volumeInfo: [String: Any] = [
"action": "setMax",
"value": NSNull()
]
NotificationCenter.default.post(
name: .volumeControlRequest,
object: volumeInfo
)
return "{\"result\": \"最大音量请求已发送\"}"
}
}

2
local_plugins/location_service/android/src/main/kotlin/com/yunqiinnovation/location_service/LocationServicePlugin.kt

@ -25,7 +25,7 @@ class LocationServicePlugin: FlutterPlugin, MethodCallHandler, ActivityAware, Pl
}
override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) {
Log.d(TAG, "Method call: ${call.method}")
// Log.d(TAG, "Method call: ${call.method}")
when (call.method) {
"location" -> LocationService.getCurrentLocation(result)
"requestPermission" -> LocationService.requestLocationPermission(activityBinding?.activity, result)

2
local_plugins/music_service/android/src/main/kotlin/com/yunqiinnovation/music_service/MusicServicePlugin.kt

@ -24,7 +24,7 @@ class MusicServicePlugin: FlutterPlugin, MethodCallHandler {
}
override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) {
Log.d(TAG, "Method call: ${call.method}")
// Log.d(TAG, "Method call: ${call.method}")
when (call.method) {
"play" -> {
val song = call.argument<Map<String, Any>>("song")

2
local_plugins/music_service/ios/music_service/Sources/music_service/MusicServicePlugin.swift

@ -13,7 +13,7 @@ public class MusicServicePlugin: NSObject, FlutterPlugin {
}
public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) {
print("Method call: \(call.method)")
// print("Method call: \(call.method)")
switch call.method {
case "play":

Loading…
Cancel
Save