liwei1dao
/
deepvoice_client
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

优化语言识别，优化ble指令发送

weicu
fdp 1 year ago
parent
0d824390ba
commit
09e3e856b8
20 changed files with 1145 additions and 1166 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 11
      lib/data/services/asr_service.dart
  2. 38
      lib/data/services/ble_manager.dart
  3. 57
      lib/data/services/speech_impl/azure_asr_service.dart
  4. 18
      lib/data/services/speech_impl/volcano_asr_api_service.dart
  5. 18
      lib/data/services/speech_impl/volcano_asr_service.dart
  6. 100
      lib/data/services/speech_impl/xunfei_asr_service.dart
  7. 13
      lib/modules/FTFTranslation/controllers/FTFTranslation_controller.dart
  8. 90
      lib/modules/agent/controllers/agent_controller.dart
  9. 31
      lib/modules/meeting/controllers/meeting_record_controller.dart
  10. 44
      lib/modules/settings/views/settings_view.dart
  11. 8
      lib/modules/speech_test/controllers/speech_test_controller.dart
  12. 34
      lib/modules/translation/controllers/translation_controller.dart
  13. 404
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  14. 56
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt
  15. 22
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt
  16. 53
      local_plugins/agent_service/lib/agent_service.dart
  17. 810
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
  18. 128
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt
  19. 1
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/RecordFile.kt
  20. 55
      local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt

11
lib/data/services/asr_service.dart
View File

@ -19,18 +19,17 @@ abstract class AsrService {
/// 开始连续语音识别 /// 开始连续语音识别
/// ///
/// 返回一个包含识别事件的流 /// 返回一个包含识别事件的流
Future<Stream<RecognitionEvent>> startContinuousRecognition( Future<bool> startContinuousRecognition(
bool audioSourceType, bool audioSourceType,
); );
/// 停止连续语音识别 /// 停止连续语音识别
Future<bool> stopContinuousRecognition(); Future<bool> stopContinuousRecognition();
/// 停止音频输入 /// 设置识别回掉
Future<void> setPause(); ///
/// 返回一个包含识别事件的流
/// 继续音频输入 Future<Stream<RecognitionEvent>> recognizeCallback();
Future<void> setContinue();
/// 恢复原始音频状态 /// 恢复原始音频状态
Future<void> restoreOriginalAudioState(); Future<void> restoreOriginalAudioState();

38
lib/data/services/ble_manager.dart
View File

@ -472,37 +472,13 @@ class BleManager extends GetxService {
firmwareProductColor: '', firmwareProductColor: '',
); );
} }
final commands = [
_bleService.getBatteryInfo, // 并行获取所有设备信息，不使用延迟
_bleService.getDeviceVersion, final results = await Future.wait([
_bleService.getProductId, _bleService.getBatteryInfo(),
]; _bleService.getDeviceVersion(),
_bleService.getProductId(),
final delays = [ ]);
Duration.zero, // 第一个命令立即执行
const Duration(milliseconds: 100), // 第二个命令延迟100ms
const Duration(milliseconds: 100), // 第三个命令延迟200ms
];
final futures = commands
.asMap()
.map((index, command) {
final delay = delays[index];
return MapEntry(
index,
Future.delayed(delay, command),
);
})
.values
.toList();
final results = await Future.wait(futures);
// // 并行获取所有设备信息，不使用延迟
// final results = await Future.wait([
// _bleService.getBatteryInfo(),
// _bleService.getDeviceVersion(),
// _bleService.getProductId(),
// ]);
Logger.info( Logger.info(
'设备信息获取结果: 电池=${results[0]}, 版本=${results[1]}, 产品ID=${results[2]}'); '设备信息获取结果: 电池=${results[0]}, 版本=${results[1]}, 产品ID=${results[2]}');

57
lib/data/services/speech_impl/azure_asr_service.dart
View File

@ -150,8 +150,7 @@ class AzureAsrService extends GetxService implements AsrService {
} }
@override @override
Future<Stream<RecognitionEvent>> startContinuousRecognition( Future<bool> startContinuousRecognition(bool audioSourceType) async {
bool audioSourceType) async {
if (!_isInitialized) { if (!_isInitialized) {
await initialize(); await initialize();
} }
@ -161,8 +160,6 @@ class AzureAsrService extends GetxService implements AsrService {
} }
try { try {
_eventStreamController = StreamController<RecognitionEvent>.broadcast();
// 开始连续识别 // 开始连续识别
final bool result = final bool result =
await _channel.invokeMethod('startContinuousRecognition', { await _channel.invokeMethod('startContinuousRecognition', {
@ -170,15 +167,13 @@ class AzureAsrService extends GetxService implements AsrService {
}); });
if (!result) { if (!result) {
_cleanupEventStream();
throw Exception('启动连续识别失败'); throw Exception('启动连续识别失败');
} }
_isContinuousRecognitionActive = true; _isContinuousRecognitionActive = true;
return _eventStreamController!.stream; return result;
} catch (e) { } catch (e) {
Logger.error('开始连续语音识别失败: ${e.toString()}'); Logger.error('开始连续语音识别失败: ${e.toString()}');
_cleanupEventStream();
rethrow; rethrow;
} }
} }
@ -192,11 +187,30 @@ class AzureAsrService extends GetxService implements AsrService {
try { try {
final bool result = final bool result =
await _channel.invokeMethod('stopContinuousRecognition'); await _channel.invokeMethod('stopContinuousRecognition');
_cleanupEventStream();
return result; return result;
} catch (e) { } catch (e) {
Logger.error('停止连续语音识别失败: ${e.toString()}'); Logger.error('停止连续语音识别失败: ${e.toString()}');
_cleanupEventStream(); rethrow;
}
}
@override
Future<Stream<RecognitionEvent>> recognizeCallback() async {
if (!_isInitialized) {
await initialize();
}
try {
_eventStreamController = StreamController<RecognitionEvent>.broadcast();
// 开始连续识别
final bool result = await _channel.invokeMethod('recognizeCallback');
if (!result) {
_cleanupEventStream();
}
return _eventStreamController!.stream;
} catch (e) {
Logger.error('开始连续语音识别失败: ${e.toString()}');
rethrow; rethrow;
} }
} }
@ -415,31 +429,6 @@ class AzureAsrService extends GetxService implements AsrService {
} }
} }
@override
Future<void> setPause() async {
// TODO: implement setPause
try {
final bool result = await _channel.invokeMethod('setPause');
return;
} catch (e) {
Logger.error('开始录音: ${e.toString()}');
rethrow;
}
}
@override
Future<void> setContinue() async {
// TODO: implement setcontinue
try {
final bool result = await _channel.invokeMethod('setcontinue');
return;
} catch (e) {
Logger.error('开始录音: ${e.toString()}');
rethrow;
}
}
@override @override
Future<void> disableBluetoothAudio() async { Future<void> disableBluetoothAudio() async {
// TODO: implement disableBluetoothAudio // TODO: implement disableBluetoothAudio

18
lib/data/services/speech_impl/volcano_asr_api_service.dart
View File

@ -842,18 +842,6 @@ class VolcanoAsrApiService implements AsrService {
throw UnimplementedError(); throw UnimplementedError();
} }
@override
Future<void> setPause() {
// TODO: implement setPause
throw UnimplementedError();
}
@override
Future<void> setContinue() {
// TODO: implement setcontinue
throw UnimplementedError();
}
@override @override
Future<void> disableBluetoothAudio() { Future<void> disableBluetoothAudio() {
// TODO: implement disableBluetoothAudio // TODO: implement disableBluetoothAudio
@ -865,4 +853,10 @@ class VolcanoAsrApiService implements AsrService {
// TODO: implement restoreOriginalAudioState // TODO: implement restoreOriginalAudioState
throw UnimplementedError(); throw UnimplementedError();
} }
@override
Future<Stream<RecognitionEvent>> recognizeCallback() {
// TODO: implement recognizeCallback
throw UnimplementedError();
}
} }

18
lib/data/services/speech_impl/volcano_asr_service.dart
View File

@ -377,18 +377,6 @@ class VolcanoAsrService extends GetxService implements AsrService {
throw UnimplementedError(); throw UnimplementedError();
} }
@override
Future<void> setPause() {
// TODO: implement setPause
throw UnimplementedError();
}
@override
Future<void> setContinue() {
// TODO: implement setcontinue
throw UnimplementedError();
}
@override @override
Future<void> disableBluetoothAudio() { Future<void> disableBluetoothAudio() {
// TODO: implement disableBluetoothAudio // TODO: implement disableBluetoothAudio
@ -400,4 +388,10 @@ class VolcanoAsrService extends GetxService implements AsrService {
// TODO: implement restoreOriginalAudioState // TODO: implement restoreOriginalAudioState
throw UnimplementedError(); throw UnimplementedError();
} }
@override
Future<Stream<RecognitionEvent>> recognizeCallback() {
// TODO: implement recognizeCallback
throw UnimplementedError();
}
} }

100
lib/data/services/speech_impl/xunfei_asr_service.dart
View File

@ -74,48 +74,48 @@ class XunfeiAsrService extends GetxService implements AsrService {
} }
} }
@override // @override
Future<Stream<RecognitionEvent>> startContinuousRecognition( // Future<Stream<RecognitionEvent>> startContinuousRecognition(
bool audioSourceType) async { // bool audioSourceType) async {
if (!_isInitialized) { // if (!_isInitialized) {
await initialize(); // await initialize();
} // }
try { // try {
_eventStreamController = StreamController<RecognitionEvent>.broadcast(); // _eventStreamController = StreamController<RecognitionEvent>.broadcast();
await recorder?.dispose(); // await recorder?.dispose();
recorder = AudioRecorder(); // recorder = AudioRecorder();
Stream<Uint8List>? stream = await recorder?.startStream( // Stream<Uint8List>? stream = await recorder?.startStream(
const RecordConfig( // const RecordConfig(
sampleRate: 16000, // sampleRate: 16000,
encoder: AudioEncoder.pcm16bits, // encoder: AudioEncoder.pcm16bits,
numChannels: 1, // numChannels: 1,
), // ),
); // );
isRecording = true; // isRecording = true;
stream?.listen((data) { // stream?.listen((data) {
if (isRecording) { // if (isRecording) {
_buff = _appendToBuffer(data); // _buff = _appendToBuffer(data);
_originaBytes.add(data); // _originaBytes.add(data);
} // }
}); // });
await _tryConnecting(); // await _tryConnecting();
_timer = Timer.periodic(const Duration(milliseconds: 40), (timer) async { // _timer = Timer.periodic(const Duration(milliseconds: 40), (timer) async {
//每40毫秒读取一次数据 // //每40毫秒读取一次数据
var frame = _getAudioData(); // var frame = _getAudioData();
if (frame.isNotEmpty) { // if (frame.isNotEmpty) {
_channel?.sink.add(frame); // _channel?.sink.add(frame);
} // }
}); // });
_isContinuousRecognitionActive = true; // _isContinuousRecognitionActive = true;
return _eventStreamController!.stream; // return _eventStreamController!.stream;
} catch (e) { // } catch (e) {
Logger.error('开始连续语音识别失败: ${e.toString()}'); // Logger.error('开始连续语音识别失败: ${e.toString()}');
rethrow; // rethrow;
} // }
} // }
//写入音频数据到缓存区中 //写入音频数据到缓存区中
Uint8List _appendToBuffer(Uint8List newData) { Uint8List _appendToBuffer(Uint8List newData) {
@ -302,26 +302,26 @@ class XunfeiAsrService extends GetxService implements AsrService {
} }
@override @override
Future<void> setPause() { Future<void> disableBluetoothAudio() {
// TODO: implement setPause // TODO: implement disableBluetoothAudio
throw UnimplementedError(); throw UnimplementedError();
} }
@override @override
Future<void> setContinue() { Future<void> restoreOriginalAudioState() {
// TODO: implement setcontinue // TODO: implement restoreOriginalAudioState
throw UnimplementedError(); throw UnimplementedError();
} }
@override @override
Future<void> disableBluetoothAudio() { Future<Stream<RecognitionEvent>> recognizeCallback() {
// TODO: implement disableBluetoothAudio // TODO: implement recognizeCallback
throw UnimplementedError(); throw UnimplementedError();
} }
@override @override
Future<void> restoreOriginalAudioState() { Future<bool> startContinuousRecognition(bool audioSourceType) {
// TODO: implement restoreOriginalAudioState // TODO: implement startContinuousRecognition
throw UnimplementedError(); throw UnimplementedError();
} }
} }

13
lib/modules/FTFTranslation/controllers/FTFTranslation_controller.dart
View File

@ -49,6 +49,7 @@ class FTFTranslationController extends GetxController
Get.put(XunfeiAsrService()); Get.put(XunfeiAsrService());
final TtsService _ttsService = Get.find<TtsService>(); final TtsService _ttsService = Get.find<TtsService>();
_asrService = Get.find<XunfeiAsrService>(); _asrService = Get.find<XunfeiAsrService>();
_InfoRecorder();
} }
@override @override
@ -58,15 +59,19 @@ class FTFTranslationController extends GetxController
super.onClose(); super.onClose();
} }
//开始录音 //开始录音
Future<void> _startRecorder() async { Future<void> _InfoRecorder() async {
final recognitionStream = final recognitionStream = await _asrService.recognizeCallback();
await _asrService.startContinuousRecognition(false);
_recognitionSubscription = recognitionStream.listen( _recognitionSubscription = recognitionStream.listen(
_handleRecognitionEvent, _handleRecognitionEvent,
); );
} }
//开始录音
Future<void> _startRecorder() async {
await _asrService.startContinuousRecognition(false);
}
//结束录音 //结束录音
Future<void> _stopRecorder() async { Future<void> _stopRecorder() async {
await _asrService.stopContinuousRecognition(); await _asrService.stopContinuousRecognition();

90
lib/modules/agent/controllers/agent_controller.dart
View File

@ -293,7 +293,8 @@ class AgentController extends GetxController {
currentPage.value = 1; currentPage.value = 1;
hasMoreMessages.value = result.totalPages > 1; hasMoreMessages.value = result.totalPages > 1;
Logger.i(TAG, '成功加载 ${historyMessages.length} 条聊天记录，总页数: ${result.totalPages}'); Logger.i(TAG,
'成功加载 ${historyMessages.length} 条聊天记录，总页数: ${result.totalPages}');
_logMessageDetails(historyMessages); _logMessageDetails(historyMessages);
} else { } else {
// 没有历史记录时添加欢迎消息 // 没有历史记录时添加欢迎消息
@ -316,7 +317,8 @@ class AgentController extends GetxController {
Future<void> loadMoreMessages() async { Future<void> loadMoreMessages() async {
// 防止重复加载 // 防止重复加载
if (isLoadingMore.value || !hasMoreMessages.value) { if (isLoadingMore.value || !hasMoreMessages.value) {
Logger.d(TAG, '跳过加载更多：isLoadingMore=${isLoadingMore.value}, hasMoreMessages=${hasMoreMessages.value}'); Logger.d(TAG,
'跳过加载更多：isLoadingMore=${isLoadingMore.value}, hasMoreMessages=${hasMoreMessages.value}');
return; return;
} }
@ -351,7 +353,8 @@ class AgentController extends GetxController {
currentPage.value = nextPage; currentPage.value = nextPage;
hasMoreMessages.value = nextPage < result.totalPages; hasMoreMessages.value = nextPage < result.totalPages;
Logger.i(TAG, '成功加载 ${newMessages.length} 条历史消息，当前页: $nextPage/${result.totalPages}'); Logger.i(TAG,
'成功加载 ${newMessages.length} 条历史消息，当前页: $nextPage/${result.totalPages}');
} else { } else {
// 没有更多消息 // 没有更多消息
hasMoreMessages.value = false; hasMoreMessages.value = false;
@ -414,7 +417,8 @@ class AgentController extends GetxController {
void _setupScrollListener() { void _setupScrollListener() {
scrollController.addListener(() { scrollController.addListener(() {
// 检查是否滚动到顶部（因为使用了reverse: true，顶部实际是最大滚动位置） // 检查是否滚动到顶部（因为使用了reverse: true，顶部实际是最大滚动位置）
if (scrollController.position.pixels >= scrollController.position.maxScrollExtent - 100) { if (scrollController.position.pixels >=
scrollController.position.maxScrollExtent - 100) {
// 当距离顶部100像素时触发加载更多 // 当距离顶部100像素时触发加载更多
if (hasMoreMessages.value && !isLoadingMore.value) { if (hasMoreMessages.value && !isLoadingMore.value) {
Logger.d(TAG, '触发加载更多历史消息'); Logger.d(TAG, '触发加载更多历史消息');
@ -430,7 +434,8 @@ class AgentController extends GetxController {
Logger.i(TAG, '手动触发加载更多历史消息'); Logger.i(TAG, '手动触发加载更多历史消息');
loadMoreMessages(); loadMoreMessages();
} else { } else {
Logger.w(TAG, '无法加载更多：hasMoreMessages=${hasMoreMessages.value}, isLoadingMore=${isLoadingMore.value}'); Logger.w(TAG,
'无法加载更多：hasMoreMessages=${hasMoreMessages.value}, isLoadingMore=${isLoadingMore.value}');
} }
} }
@ -979,7 +984,8 @@ class AgentController extends GetxController {
permissionType: Permission.microphone, permissionType: Permission.microphone,
permissionName: 'microphonePermission'.tr, // 麦克风 permissionName: 'microphonePermission'.tr, // 麦克风
explanationText: 'voiceRecognitionPermissionText'.tr, // 语音识别功能需要访问麦克风。 explanationText: 'voiceRecognitionPermissionText'.tr, // 语音识别功能需要访问麦克风。
permanentDenialText: 'permissionDeniedText'.tr, // 您已永久拒绝麦克风权限，请前往系统设置开启权限。 permanentDenialText:
'permissionDeniedText'.tr, // 您已永久拒绝麦克风权限，请前往系统设置开启权限。
settingsButtonText: 'goToSettings'.tr, // 去设置 settingsButtonText: 'goToSettings'.tr, // 去设置
cancelButtonText: 'cancel'.tr, // 取消 cancelButtonText: 'cancel'.tr, // 取消
); );
@ -1005,13 +1011,14 @@ class AgentController extends GetxController {
// 开始语音输入 // 开始语音输入
Future<void> startVoiceInput() async { Future<void> startVoiceInput() async {
if (isListening.value) return; if (isListening.value) return;
print('startVoiceInput'.tr); // 开始语音输入 print('开始语音输入'); // 开始语音输入
final hasPermission = await PermissionUtil.instance.requestPermission( final hasPermission = await PermissionUtil.instance.requestPermission(
// context: Get.context!, // context: Get.context!,
permissionType: Permission.microphone, permissionType: Permission.microphone,
permissionName: 'microphonePermission'.tr, // 麦克风 permissionName: 'microphonePermission'.tr, // 麦克风
explanationText: 'voiceTranslationPermissionText'.tr, // 语音识别与翻译功能需要访问麦克风。 explanationText: 'voiceTranslationPermissionText'.tr, // 语音识别与翻译功能需要访问麦克风。
permanentDenialText: 'permissionDeniedText'.tr, // 您已永久拒绝麦克风权限，请前往系统设置开启权限。 permanentDenialText:
'permissionDeniedText'.tr, // 您已永久拒绝麦克风权限，请前往系统设置开启权限。
settingsButtonText: 'goToSettings'.tr, // 去设置 settingsButtonText: 'goToSettings'.tr, // 去设置
cancelButtonText: 'cancel'.tr, // 取消 cancelButtonText: 'cancel'.tr, // 取消
); );
@ -1020,11 +1027,11 @@ class AgentController extends GetxController {
return; return;
} }
try { // try {
await AgentService.startConversation();
} catch (e) { // } catch (e) {
Logger.e(TAG, '启动语音输入失败: $e'); // Logger.e(TAG, '启动语音输入失败: $e');
} // }
} }
// 停止语音输入 // 停止语音输入
@ -1276,32 +1283,32 @@ class AgentController extends GetxController {
isPhoneCallMode.value = true; isPhoneCallMode.value = true;
// 如果当前是文本模式，切换到语音模式???? // 如果当前是文本模式，切换到语音模式????
if (isListening.value) { // if (isListening.value) {
// 检查麦克风权限 // 检查麦克风权限
final hasPermission = await PermissionUtil.instance.requestPermission( final hasPermission = await PermissionUtil.instance.requestPermission(
permissionType: Permission.microphone, permissionType: Permission.microphone,
permissionName: 'microphonePermission'.tr, // 麦克风 permissionName: '麦克风',
explanationText: 'phoneCallPermissionText'.tr, // 打电话功能需要访问麦克风进行语音通话。 explanationText: '打电话功能需要访问麦克风进行语音通话。',
permanentDenialText: 'permissionDeniedText'.tr, // 您已永久拒绝麦克风权限，请前往系统设置开启权限。 permanentDenialText: '您已永久拒绝麦克风权限，请前往系统设置开启权限。',
settingsButtonText: 'goToSettings'.tr, // 去设置 settingsButtonText: '去设置',
cancelButtonText: 'cancel'.tr, // 取消 cancelButtonText: '取消',
); );
if (!hasPermission) {
// 权限未获取，退出打电话模式并恢复之前状态
isPhoneCallMode.value = false;
_previousInputMode = null; // 清除历史记录
return;
}
// 切换到语音模式并开始语音输入 if (!hasPermission) {
isTextInputMode.value = false; // 权限未获取，退出打电话模式并恢复之前状态
// 打电话模式不使用按住说话模式，而是持续语音输入 isPhoneCallMode.value = false;
//isPushToTalkMode.value = false; _previousInputMode = null; // 清除历史记录
// 开始语音识别 return;
print('startPhoneCallMode'.tr); // 开始打电话模式，开始语音输入
await AgentService.audioContinue();
} }
// 切换到语音模式并开始语音输入
isTextInputMode.value = false;
// 打电话模式不使用按住说话模式，而是持续语音输入
//isPushToTalkMode.value = false;
// 开始语音识别
print("开始打电话模式，开始语音输入");
await AgentService.startConversation();
//}
} }
// 结束打电话模式 // 结束打电话模式
@ -1314,7 +1321,7 @@ class AgentController extends GetxController {
// 停止语音输入 // 停止语音输入
//await stopVoiceInput(); //await stopVoiceInput();
// 开始语音识别 // 开始语音识别
await AgentService.audioPause(); await AgentService.stopConversation();
// 恢复之前的输入模式，而不是强制切换到文字模式 // 恢复之前的输入模式，而不是强制切换到文字模式
_restorePreviousInputMode(); _restorePreviousInputMode();
} }
@ -1409,9 +1416,9 @@ class AgentController extends GetxController {
// 设置按住说话状态 // 设置按住说话状态
isPushToTalkActive.value = true; isPushToTalkActive.value = true;
//await AgentService.recognizeCallback();
// 开始语音识别 // 开始语音识别
await AgentService.audioContinue(); await AgentService.startConversation();
// await startVoiceInput(); // await startVoiceInput();
} }
@ -1424,7 +1431,8 @@ class AgentController extends GetxController {
// 清除按住说话状态（但保持按住说话模式） // 清除按住说话状态（但保持按住说话模式）
isPushToTalkActive.value = false; isPushToTalkActive.value = false;
await AgentService.audioPause();
await AgentService.stopConversation();
// 停止语音识别 // 停止语音识别
// await stopVoiceInput(); // await stopVoiceInput();
} }

31
lib/modules/meeting/controllers/meeting_record_controller.dart
View File

@ -66,6 +66,7 @@ class MeetingRecordController extends GetxController
_initializeComponents(); _initializeComponents();
_setupFromArguments(); _setupFromArguments();
generateWaveData(); generateWaveData();
_reinitializeAsrService();
} }
@override @override
@ -74,6 +75,18 @@ class MeetingRecordController extends GetxController
super.onClose(); super.onClose();
} }
// 重新初始化ASR服务以支持新的语言
Future<void> _reinitializeAsrService() async {
try {
// 重新初始化ASR服务
await _asrService.initialize();
Logger.info('ASR服务重新初始化成功');
} catch (e) {
Logger.error('重新初始化ASR服务失败: ${e.toString()}');
}
}
// 请求录音权限 // 请求录音权限
Future<bool> _requestRecordPermission() async { Future<bool> _requestRecordPermission() async {
try { try {
@ -132,6 +145,7 @@ class MeetingRecordController extends GetxController
void _cleanupResources() { void _cleanupResources() {
_timer?.cancel(); _timer?.cancel();
_recognitionSubscription?.cancel(); _recognitionSubscription?.cancel();
_recognitionSubscription = null;
titleEditingController.dispose(); titleEditingController.dispose();
titleFocusNode.dispose(); titleFocusNode.dispose();
if (audioType.value != 0) { if (audioType.value != 0) {
@ -170,7 +184,13 @@ class MeetingRecordController extends GetxController
} }
/// Toggles visibility of transcribed text content /// Toggles visibility of transcribed text content
void toggleTextDisplay() => showTextContent.toggle(); Future<void> toggleTextDisplay() async {
final recognitionStream = await _asrService.recognizeCallback();
_recognitionSubscription?.cancel();
_recognitionSubscription =
recognitionStream.listen(_handleRecognitionEvent);
showTextContent.toggle();
}
/// Toggles important mark status /// Toggles important mark status
void toggleMarked() => isMarked.toggle(); void toggleMarked() => isMarked.toggle();
@ -211,10 +231,7 @@ class MeetingRecordController extends GetxController
// Initialize speech recognition // Initialize speech recognition
final recognitionStream = final recognitionStream =
await _asrService.startContinuousRecognition(_audioSourceType); await _asrService.startContinuousRecognition(_audioSourceType);
_recognitionSubscription?.cancel();
_recognitionSubscription =
recognitionStream.listen(_handleRecognitionEvent);
_asrService.setContinue();
// Prepare storage directory // Prepare storage directory
final appDir = await appDocDir; final appDir = await appDocDir;
final dir = Directory("${appDir.path}/MeetingAudio/"); final dir = Directory("${appDir.path}/MeetingAudio/");
@ -263,9 +280,7 @@ class MeetingRecordController extends GetxController
Future<void> _resumeRecording() async { Future<void> _resumeRecording() async {
final recognitionStream = final recognitionStream =
await _asrService.startContinuousRecognition(_audioSourceType); await _asrService.startContinuousRecognition(_audioSourceType);
_recognitionSubscription?.cancel();
_recognitionSubscription =
recognitionStream.listen(_handleRecognitionEvent);
startTimer(); startTimer();
} }

44
lib/modules/settings/views/settings_view.dart
View File

@ -567,28 +567,28 @@ class SettingsView extends GetView<SettingsController> {
// color: isDarkMode // color: isDarkMode
// ? Colors.white.withOpacity(0.1) // ? Colors.white.withOpacity(0.1)
// : Colors.grey[200]), // : Colors.grey[200]),
// // BLE测试 // BLE测试
// _buildSimpleNavigationSetting( _buildSimpleNavigationSetting(
// title: 'opus解码测试', title: 'opus解码测试',
// subtitle: '测试opus解码', subtitle: '测试opus解码',
// icon: Icons.bluetooth_searching, icon: Icons.bluetooth_searching,
// iconBgColor: isDarkMode iconBgColor: isDarkMode
// ? Colors.green[900]!.withOpacity(0.3) ? Colors.green[900]!.withOpacity(0.3)
// : Colors.green[100]!, : Colors.green[100]!,
// iconColor: iconColor:
// isDarkMode ? Colors.green[300]! : Colors.green[600]!, isDarkMode ? Colors.green[300]! : Colors.green[600]!,
// titleColor: isDarkMode ? Colors.white : null, titleColor: isDarkMode ? Colors.white : null,
// subtitleColor: isDarkMode ? Colors.white70 : null, subtitleColor: isDarkMode ? Colors.white70 : null,
// onTap: () { onTap: () {
// Get.toNamed(Routes.opusTest); Get.toNamed(Routes.opusTest);
// }, },
// isDarkMode: isDarkMode, isDarkMode: isDarkMode,
// ), ),
// Divider( Divider(
// height: 1, height: 1,
// color: isDarkMode color: isDarkMode
// ? Colors.white.withOpacity(0.1) ? Colors.white.withOpacity(0.1)
// : Colors.grey[200]), : Colors.grey[200]),
// BLE测试 // BLE测试
// _buildSimpleNavigationSetting( // _buildSimpleNavigationSetting(

8
lib/modules/speech_test/controllers/speech_test_controller.dart
View File

@ -76,7 +76,10 @@ class SpeechTestController extends GetxController {
// 监听TTS事件 // 监听TTS事件
_setupTtsEventListener(); _setupTtsEventListener();
// 获取识别事件流 - 这启动了异步识别过程
final stream = await _asrService.recognizeCallback();
// 设置事件监听 - 异步处理识别结果
_setupAsrEventListener(stream);
isInitialized.value = true; isInitialized.value = true;
ttsStatus.value = '准备就绪'; ttsStatus.value = '准备就绪';
recognitionStatus.value = '准备就绪'; recognitionStatus.value = '准备就绪';
@ -346,9 +349,6 @@ class SpeechTestController extends GetxController {
// 获取识别事件流 - 这启动了异步识别过程 // 获取识别事件流 - 这启动了异步识别过程
final stream = final stream =
await _asrService.startContinuousRecognition(_audioSourceType); await _asrService.startContinuousRecognition(_audioSourceType);
// 设置事件监听 - 异步处理识别结果
_setupAsrEventListener(stream);
} catch (e) { } catch (e) {
recognitionStatus.value = '启动连续识别异常: $e'; recognitionStatus.value = '启动连续识别异常: $e';
Logger.error('启动连续识别异常: ${e.toString()}'); Logger.error('启动连续识别异常: ${e.toString()}');

34
lib/modules/translation/controllers/translation_controller.dart
View File

@ -122,7 +122,9 @@ class TranslationController extends GetxController {
// 监听识别结果 // 监听识别结果
if (activeSpeaker.value != 0) { if (activeSpeaker.value != 0) {
isPlayback = false; isPlayback = false;
_asrService.setContinue(); final recognitionStream =
await _asrService.startContinuousRecognition(_audioSourceType);
lastActiveSpeaker.value = id; lastActiveSpeaker.value = id;
} else { } else {
if (fTFTranslationResult != '' && !isPlayback) { if (fTFTranslationResult != '' && !isPlayback) {
@ -134,7 +136,7 @@ class TranslationController extends GetxController {
} else if ((translationHistory.isEmpty || } else if ((translationHistory.isEmpty ||
!translationHistory.last.isIntermediate) && !translationHistory.last.isIntermediate) &&
currentMode.value == 'faceToFace') { currentMode.value == 'faceToFace') {
_asrService.setPause(); _asrService.stopContinuousRecognition();
} }
} }
} }
@ -305,6 +307,12 @@ class TranslationController extends GetxController {
await _asrService.initialize( await _asrService.initialize(
supportedLanguages: asrSupportedLanguages, supportedLanguages: asrSupportedLanguages,
); );
// 获取识别事件流 - 这启动了异步识别过程
var recognitionStream = await _asrService.recognizeCallback();
// 监听识别结果
_recognitionSubscription?.cancel();
_recognitionSubscription =
recognitionStream.listen(_handleRecognitionEvent);
final bool storage = await _requestStoragePermission(); final bool storage = await _requestStoragePermission();
if (!storage) { if (!storage) {
Logger.d("Permission", "未授予存储权限，无法提供录音功能"); Logger.d("Permission", "未授予存储权限，无法提供录音功能");
@ -325,6 +333,7 @@ class TranslationController extends GetxController {
void onClose() { void onClose() {
stopRecognition(); stopRecognition();
_recognitionSubscription?.cancel(); _recognitionSubscription?.cancel();
_recognitionSubscription = null;
_voiceInteractionSubscription?.cancel(); _voiceInteractionSubscription?.cancel();
_translationDebounceTimer?.cancel(); _translationDebounceTimer?.cancel();
scrollController.dispose(); scrollController.dispose();
@ -425,22 +434,18 @@ class TranslationController extends GetxController {
_audioSourceType = false; _audioSourceType = false;
isTtsEnabled.value = true; isTtsEnabled.value = true;
} }
final recognitionStream =
await _asrService.startContinuousRecognition(_audioSourceType);
if (currentMode.value == 'faceToFace') { if (currentMode.value == 'faceToFace') {
_asrService.setPause(); _asrService.stopContinuousRecognition();
} else { } else {
await _ttsService.setAudioOutputDevice(0); await _ttsService.setAudioOutputDevice(0);
_asrService.setContinue(); final recognitionStream =
await _asrService.startContinuousRecognition(_audioSourceType);
} }
isRecognizing.value = true; isRecognizing.value = true;
if (isRecording.value && isRecognizing.value) { if (isRecording.value && isRecognizing.value) {
startRecording(); startRecording();
} }
// 监听识别结果
_recognitionSubscription?.cancel();
_recognitionSubscription =
recognitionStream.listen(_handleRecognitionEvent);
} catch (e) { } catch (e) {
isRecognizing.value = false; isRecognizing.value = false;
Logger.error('启动语音识别失败: ${e.toString()}'); Logger.error('启动语音识别失败: ${e.toString()}');
@ -473,10 +478,9 @@ class TranslationController extends GetxController {
if (!isRecognizing.value) return; if (!isRecognizing.value) return;
try { try {
restoreOriginalAudioState();
await _asrService.stopContinuousRecognition(); await _asrService.stopContinuousRecognition();
_recognitionSubscription?.cancel();
_recognitionSubscription = null;
isRecognizing.value = false; isRecognizing.value = false;
// 如果当前有中间结果，将其作为最终结果处理 // 如果当前有中间结果，将其作为最终结果处理
@ -523,7 +527,7 @@ class TranslationController extends GetxController {
); );
// 添加新项后滚动到底部 // 添加新项后滚动到底部
if (currentMode.value == 'faceToFace' && activeSpeaker.value == 0) { if (currentMode.value == 'faceToFace' && activeSpeaker.value == 0) {
_asrService.setPause(); _asrService.stopContinuousRecognition();
} else { } else {
translationHistory.add(newItem); translationHistory.add(newItem);
} }
@ -563,7 +567,7 @@ class TranslationController extends GetxController {
_scrollToBottom(); _scrollToBottom();
saveTranslationHistory(); saveTranslationHistory();
if (activeSpeaker.value == 0 && currentMode.value == 'faceToFace') { if (activeSpeaker.value == 0 && currentMode.value == 'faceToFace') {
_asrService.setPause(); _asrService.stopContinuousRecognition();
} }
// 使用项目的时间戳进行翻译 // 使用项目的时间戳进行翻译
translateText(text, translationHistory.last.timestamp, isFinal: true); translateText(text, translationHistory.last.timestamp, isFinal: true);
@ -582,7 +586,7 @@ class TranslationController extends GetxController {
isIntermediate: false, isIntermediate: false,
); );
if (currentMode.value == 'faceToFace' && activeSpeaker.value == 0) { if (currentMode.value == 'faceToFace' && activeSpeaker.value == 0) {
_asrService.setPause(); _asrService.stopContinuousRecognition();
} //else { } //else {
translationHistory.add(newItem); translationHistory.add(newItem);
translationHistory.refresh(); translationHistory.refresh();

404
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
View File

@ -58,7 +58,7 @@ object AgentService : CoroutineScope {
private val listeners = Collections.synchronizedSet(mutableSetOf<AgentServiceListener>()) private val listeners = Collections.synchronizedSet(mutableSetOf<AgentServiceListener>())
// Azure服务 // Azure服务
var azureAsrHelper: AzureAsrHelper? = null private var azureAsrHelper: AzureAsrHelper? = null
// 使用BytedanceTTS作为唯一的TTS实现 // 使用BytedanceTTS作为唯一的TTS实现
private var ttsService: AzureTtsHelper? = null private var ttsService: AzureTtsHelper? = null
@ -102,7 +102,8 @@ object AgentService : CoroutineScope {
// AI流生成相关 // AI流生成相关
private var currentAiJob: Job? = null private var currentAiJob: Job? = null
private var currsessionId:String = "" private var currsessionId: String = ""
// 空闲检测相关 // 空闲检测相关
private var idleCheckJob: Job? = null private var idleCheckJob: Job? = null
private val maxIdleSeconds = 10 // 最大空闲秒数 private val maxIdleSeconds = 10 // 最大空闲秒数
@ -111,7 +112,8 @@ object AgentService : CoroutineScope {
private var lastInterruptTime = 0L private var lastInterruptTime = 0L
private val INTERRUPT_DEBOUNCE_MS = 300L private val INTERRUPT_DEBOUNCE_MS = 300L
private var audioSourceType= AzureAsrHelper.AudioSourceType.MICROPHONE var audioSourceType = AzureAsrHelper.AudioSourceType.MICROPHONE
// 语气词过滤列表（中英文） // 语气词过滤列表（中英文）
private val FILLER_WORDS = setOf( private val FILLER_WORDS = setOf(
// 中文语气词 // 中文语气词
@ -122,6 +124,7 @@ object AgentService : CoroutineScope {
"um", "uh", "ah", "oh", "eh", "er", "hmm", "huh", "wow", "um", "uh", "ah", "oh", "eh", "er", "hmm", "huh", "wow",
"well", "like", "you know", "i mean", "so", "actually" "well", "like", "you know", "i mean", "so", "actually"
) )
/** /**
* 初始化系统提示词 * 初始化系统提示词
*/ */
@ -142,6 +145,7 @@ object AgentService : CoroutineScope {
你与用户的交流风格应始终温暖、亲切、充满同理心，随时表达关怀与鼓励，让用户感受到真诚的陪伴与支持。 你与用户的交流风格应始终温暖、亲切、充满同理心，随时表达关怀与鼓励，让用户感受到真诚的陪伴与支持。
""".trimIndent() """.trimIndent()
} }
/** /**
* 初始化 * 初始化
* @param appContext 应用上下文 * @param appContext 应用上下文
@ -149,7 +153,11 @@ object AgentService : CoroutineScope {
* @param config 配置参数，包含所需的所有API密钥和设置 * @param config 配置参数，包含所需的所有API密钥和设置
* @return 是否初始化成功 * @return 是否初始化成功
*/ */
fun initialize(appContext: Context, config: Map<String, Any>, isExternalActive: Boolean): Boolean { fun initialize(
appContext: Context,
config: Map<String, Any>,
isExternalActive: Boolean
): Boolean {
if (isInitialized) return true if (isInitialized) return true
try { try {
@ -169,13 +177,24 @@ object AgentService : CoroutineScope {
} }
// 初始化Azure ASR // 初始化Azure ASR
azureAsrHelper = AzureAsrHelper(context).apply { azureAsrHelper = AzureAsrHelper(context).apply {
initialize( // 使用协程处理异步初始化
subscriptionKey = config["azureSpeechKey"]?.toString() ?: "", CoroutineScope(Dispatchers.IO).launch {
region = config["azureSpeechRegion"]?.toString() ?: "", val asrInitSuccess = initialize(
audioSourceType = if (isExternalActive) AzureAsrHelper.AudioSourceType.EXTERNAL subscriptionKey = config["azureSpeechKey"]?.toString() ?: "",
else AzureAsrHelper.AudioSourceType.MICROPHONE region = config["azureSpeechRegion"]?.toString() ?: "",
) audioSourceType = if (isExternalActive) AzureAsrHelper.AudioSourceType.EXTERNAL
else AzureAsrHelper.AudioSourceType.MICROPHONE
)
if (asrInitSuccess) {
withContext(Dispatchers.Main) {
recognizeCallback()
} }
} else {
Log.w(TAG, "Azure ASR初始化失败")
}
}
}
// 初始化TTS服务 // 初始化TTS服务
initializeTtsEngine(config) initializeTtsEngine(config)
@ -189,6 +208,8 @@ object AgentService : CoroutineScope {
config["mcpServer"]?.toString() ?: "" config["mcpServer"]?.toString() ?: ""
) )
// 加载最近的聊天记录 // 加载最近的聊天记录
loadChatHistory() loadChatHistory()
@ -274,45 +295,55 @@ object AgentService : CoroutineScope {
restartIdleCheck() restartIdleCheck()
sendEvent("tts_started", mapOf("status" to "started")) sendEvent("tts_started", mapOf("status" to "started"))
} }
TtsEventType.SYNTHESIS_COMPLETED -> { TtsEventType.SYNTHESIS_COMPLETED -> {
_isTtsSpeaking.set(false) _isTtsSpeaking.set(false)
restartIdleCheck() restartIdleCheck()
sendEvent("tts_completed", mapOf("status" to "completed")) sendEvent("tts_completed", mapOf("status" to "completed"))
} }
TtsEventType.SYNTHESIS_CANCELED -> { TtsEventType.SYNTHESIS_CANCELED -> {
_isTtsSpeaking.set(false) _isTtsSpeaking.set(false)
restartIdleCheck() restartIdleCheck()
sendEvent("tts_canceled", mapOf("status" to "canceled")) sendEvent("tts_canceled", mapOf("status" to "canceled"))
} }
TtsEventType.PLAYBACK_STARTED -> { //开始AI播报内容 TtsEventType.PLAYBACK_STARTED -> { //开始AI播报内容
restartIdleCheck() restartIdleCheck()
sendEvent("playback_started", mapOf("status" to "playback_started")) sendEvent("playback_started", mapOf("status" to "playback_started"))
// 发送AI说话开始广播 // 发送AI说话开始广播
val intent = Intent("com.yunqiinnovation.music_service.AI_SAY_START").apply { val intent =
setPackage(context.packageName) Intent("com.yunqiinnovation.music_service.AI_SAY_START").apply {
} setPackage(context.packageName)
}
context.sendBroadcast(intent) context.sendBroadcast(intent)
} }
TtsEventType.PLAYBACK_COMPLETED -> { //结束AI播报内容 TtsEventType.PLAYBACK_COMPLETED -> { //结束AI播报内容
restartIdleCheck() restartIdleCheck()
sendEvent("playback_completed", mapOf("status" to "playback_completed")) sendEvent("playback_completed", mapOf("status" to "playback_completed"))
// 发送AI说话开始广播 // 发送AI说话开始广播
val intent = Intent("com.yunqiinnovation.music_service.AI_SAY_END").apply { val intent =
setPackage(context.packageName) Intent("com.yunqiinnovation.music_service.AI_SAY_END").apply {
} setPackage(context.packageName)
}
context.sendBroadcast(intent) context.sendBroadcast(intent)
} }
TtsEventType.ERROR -> { TtsEventType.ERROR -> {
_isTtsSpeaking.set(false) _isTtsSpeaking.set(false)
restartIdleCheck() restartIdleCheck()
val params = event.params val params = event.params
val code = params["errorCode"] as? String ?: "UNKNOWN_ERROR" val code = params["errorCode"] as? String ?: "UNKNOWN_ERROR"
val message = params["errorMessage"] as? String ?: "未知错误" val message = params["errorMessage"] as? String ?: "未知错误"
sendEvent("error", mapOf( sendEvent(
"code" to "TTS_ERROR", "error", mapOf(
"message" to "语音合成错误: $code, $message" "code" to "TTS_ERROR",
)) "message" to "语音合成错误: $code, $message"
)
)
} }
else -> { else -> {
// 处理其他类型的事件 // 处理其他类型的事件
} }
@ -359,8 +390,11 @@ object AgentService : CoroutineScope {
delay(maxIdleSeconds * 1000L) delay(maxIdleSeconds * 1000L)
if (isActive && isRecognitionActive && !hasSpeechDetected && !isTtsSpeaking && !isAiStreaming) { if (isActive && isRecognitionActive && !hasSpeechDetected && !isTtsSpeaking && !isAiStreaming) {
//stopRecognition() stopRecognition()
sendEvent("auto_stop", mapOf("reason" to "idle_timeout", "seconds" to maxIdleSeconds)) sendEvent(
"auto_stop",
mapOf("reason" to "idle_timeout", "seconds" to maxIdleSeconds)
)
} }
} catch (e: CancellationException) { } catch (e: CancellationException) {
// 协程被取消，正常行为 // 协程被取消，正常行为
@ -396,14 +430,26 @@ object AgentService : CoroutineScope {
Log.d(TAG, "服务未初始化") Log.d(TAG, "服务未初始化")
return false return false
} }
if(!(azureAsrHelper!!.isPaused)&&isExternalActive) Log.d("TAG", "开始语音识别startRecognition?")
{ stopRecognition()
Log.d(TAG, "麦克风音频未暂停，无法设置") BleService.openEncoder()
BleService.closeCodec() // if (isRecognitionActive&&audioSourceType==AzureAsrHelper.AudioSourceType.MICROPHONE){
return false // Log.d(TAG, "开始语音识别stopRecognition")
} // // BleService.closeCodec()
if (isRecognitionActive) return true // stopRecognition()
// }
// if((isExternalActive&&audioSourceType==AzureAsrHelper.AudioSourceType.MICROPHONE)||!isRecognitionActive)
// {
// Log.d(TAG, "开始语音识别BleService.openEncoder")
// BleService.openEncoder()
// }
// if(isRecognitionActive)
// {
// Log.d(TAG, "开始语音识别azureAsrHelper?.startContinuousRecognition(audioSourceType)")
// return true
// }
// 检查录音权限 // 检查录音权限
if (!isExternalActive && !checkRecordAudioPermission()) { if (!isExternalActive && !checkRecordAudioPermission()) {
Log.e(TAG, "需要录音权限") Log.e(TAG, "需要录音权限")
@ -415,12 +461,33 @@ object AgentService : CoroutineScope {
_hasSpeechDetected.set(false) _hasSpeechDetected.set(false)
try { try {
audioSourceType = if (isExternalActive) { audioSourceType = if (isExternalActive) {
AzureAsrHelper.AudioSourceType.EXTERNAL AzureAsrHelper.AudioSourceType.EXTERNAL
} else { } else {
AzureAsrHelper.AudioSourceType.MICROPHONE AzureAsrHelper.AudioSourceType.MICROPHONE
} }
azureAsrHelper?.startContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { Log.d("TAG", "azureAsrHelper?.startContinuousRecognition")
azureAsrHelper?.startContinuousRecognition(audioSourceType)
return true
} catch (e: Exception) {
_isRecognitionActive.set(false)
Log.e(TAG, "启动语音识别失败: ${e.message}")
sendEvent(
"error", mapOf(
"code" to "RECOGNITION_START_ERROR",
"message" to e.message.toString()
)
)
return false
}
}
fun recognizeCallback(): Boolean {
try {
Log.d(TAG, "recognizeCallback")
val success = azureAsrHelper?.setupEventListeners(object :
AzureAsrHelper.ContinuousRecognizeCallback {
override fun onRecognizing(recognizing: String, detectedLanguage: String) { override fun onRecognizing(recognizing: String, detectedLanguage: String) {
if (recognizing.isNotEmpty()) { if (recognizing.isNotEmpty()) {
// 检测到语音，更新状态 // 检测到语音，更新状态
@ -431,10 +498,12 @@ object AgentService : CoroutineScope {
restartIdleCheck() restartIdleCheck()
} }
sendEvent("recognizing", mapOf( sendEvent(
"text" to recognizing, "recognizing", mapOf(
"language" to detectedLanguage "text" to recognizing,
)) "language" to detectedLanguage
)
)
// 如果TTS正在播放或AI正在生成，检查是否需要打断 // 如果TTS正在播放或AI正在生成，检查是否需要打断
if (isTtsSpeaking || isAiStreaming) { if (isTtsSpeaking || isAiStreaming) {
@ -465,10 +534,12 @@ object AgentService : CoroutineScope {
override fun onResult(result: String, detectedLanguage: String) { override fun onResult(result: String, detectedLanguage: String) {
if (result.isNotEmpty()) { if (result.isNotEmpty()) {
sendEvent("recognition_result", mapOf( sendEvent(
"text" to result, "recognition_result", mapOf(
"language" to detectedLanguage "text" to result,
)) "language" to detectedLanguage
)
)
processWithChatApi(result) processWithChatApi(result)
} }
@ -485,14 +556,12 @@ object AgentService : CoroutineScope {
override fun onSessionStarted() { override fun onSessionStarted() {
sendEvent("recognition_started", mapOf("status" to "started")) sendEvent("recognition_started", mapOf("status" to "started"))
startIdleCheck() startIdleCheck()
Log.d(TAG, "onSessionStarted")
audioPlayer?.playAudio(R.raw.start) audioPlayer?.playAudio(R.raw.start)
} }
override fun onSessionStopped() { override fun onSessionStopped() {
sendEvent("recognition_stopped", mapOf("status" to "stopped")) sendEvent("recognition_stopped", mapOf("status" to "stopped"))
_isRecognitionActive.set(false) _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
audioPlayer?.playAudio(R.raw.stop) audioPlayer?.playAudio(R.raw.stop)
@ -501,41 +570,41 @@ object AgentService : CoroutineScope {
override fun onCanceled(reason: String, errorDetails: String) { override fun onCanceled(reason: String, errorDetails: String) {
_isRecognitionActive.set(false) _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
Log.d(TAG, "recognition_canceled:")
recognizeCallback()
BleService.closeCodec() BleService.closeCodec()
sendEvent("recognition_canceled", mapOf( sendEvent(
"reason" to reason, "recognition_canceled", mapOf(
"details" to errorDetails "reason" to reason,
)) "details" to errorDetails
)
)
} }
override fun onError(error: String) { override fun onError(error: String) {
_isRecognitionActive.set(false) _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
recognizeCallback()
BleService.closeCodec() BleService.closeCodec()
Log.e(TAG, "语音识别错误: $error") Log.d(TAG, "语音识别错误: $error")
sendEvent("error", mapOf( sendEvent(
"code" to "RECOGNITION_ERROR", "error", mapOf(
"message" to error "code" to "RECOGNITION_ERROR",
)) "message" to error
)
)
} }
}, audioSourceType) })
return true return true
} catch (e: Exception) { } catch (e: Exception) {
_isRecognitionActive.set(false)
Log.e(TAG, "启动语音识别失败: ${e.message}")
sendEvent("error", mapOf(
"code" to "RECOGNITION_START_ERROR",
"message" to e.message.toString()
))
return false return false
} }
} }
fun pushAudioData(data: ByteArray) { fun pushAudioData(data: ByteArray) {
azureAsrHelper?.pushAudioData(data) azureAsrHelper?.audioStream?.saveAudioDataTo(data)
} }
/** /**
* 停止语音识别 * 停止语音识别
*/ */
@ -547,67 +616,22 @@ object AgentService : CoroutineScope {
BleService.closeCodec() BleService.closeCodec()
_isRecognitionActive.set(false) _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
return
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "停止语音识别异常: ${e.message}") Log.e(TAG, "停止语音识别异常: ${e.message}")
_isRecognitionActive.set(false) _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
return
} }
} }
/**
* 继续音频输入
*/
fun audioContinue() {
try {
if(audioSourceType== AzureAsrHelper.AudioSourceType.EXTERNAL)
{
BleService.closeCodec()
stopRecognition()
startRecognition(false)
}
// if (!isRecognitionActive) {
// }
azureAsrHelper?.audioContinue()
//BleService.closeCodec()
//_isRecognitionActive.set(false)
//stopIdleCheck()
} catch (e: Exception) {
Log.e(TAG, "暂停音频输入异常: ${e.message}")
// _isRecognitionActive.set(false)
//stopIdleCheck()
}
}
/**
* 暂停音频输入
*/
fun audioPause() {
if (!isRecognitionActive) return
try {
azureAsrHelper?.audioPause()
Log.d(TAG, "暂停音频输入")
BleService.closeCodec()
//_isRecognitionActive.set(false)
//stopIdleCheck()
} catch (e: Exception) {
Log.e(TAG, "暂停音频输入异常: ${e.message}")
// _isRecognitionActive.set(false)
//stopIdleCheck()
}
}
/** /**
* 打断当前响应 * 打断当前响应
* 停止TTS播放和AI流输出 * 停止TTS播放和AI流输出
*/ */
fun interruptCurrentResponse() { fun interruptCurrentResponse() {
if(isAiStreaming || isTtsSpeaking) { if (isAiStreaming || isTtsSpeaking) {
val startTime = System.currentTimeMillis() val startTime = System.currentTimeMillis()
// 并行执行停止操作，加快响应速度 // 并行执行停止操作，加快响应速度
@ -617,10 +641,12 @@ object AgentService : CoroutineScope {
// 记录打断耗时 // 记录打断耗时
val duration = System.currentTimeMillis() - startTime val duration = System.currentTimeMillis() - startTime
sendEvent("response_interrupted", mapOf( sendEvent(
"status" to "interrupted", "response_interrupted", mapOf(
"duration_ms" to duration "status" to "interrupted",
)) "duration_ms" to duration
)
)
} }
} }
@ -628,7 +654,7 @@ object AgentService : CoroutineScope {
/** /**
* 停止AI流输出 * 停止AI流输出
*/ */
fun stopAiStream() { fun stopAiStream() {
if (isAiStreaming) { if (isAiStreaming) {
try { try {
// 先更新状态，避免回调时的状态不一致 // 先更新状态，避免回调时的状态不一致
@ -743,11 +769,11 @@ object AgentService : CoroutineScope {
currsessionId = UUID.randomUUID().toString() currsessionId = UUID.randomUUID().toString()
// 设置状态为正在流式输出 // 设置状态为正在流式输出
_isAiStreaming.set(true) _isAiStreaming.set(true)
Log.d(TAG, "processWithChatApiInternal") Log.d(TAG, "processWithChatApiInternal")
audioPlayer?.playAudio(R.raw.await, true,0.3f) audioPlayer?.playAudio(R.raw.await, true, 0.3f)
// 使用历史记录作为上下文发送到OpenAI // 使用历史记录作为上下文发送到OpenAI
val responseBuilder = StringBuilder() val responseBuilder = StringBuilder()
var aiMetadata:String = "" var aiMetadata: String = ""
var userMetadata = JSONObject() var userMetadata = JSONObject()
var reply = true; //是否回复 var reply = true; //是否回复
var broadcast = true //是否播发 var broadcast = true //是否播发
@ -761,16 +787,16 @@ object AgentService : CoroutineScope {
val messagesWithSystemPrompt = JSONArray() val messagesWithSystemPrompt = JSONArray()
// 先添加系统提示词 // 先添加系统提示词
if (systemPrompt.isNotEmpty()) { if (systemPrompt.isNotEmpty()) {
val location = LocationService.getCachedLocation()?: mapOf( val location = LocationService.getCachedLocation() ?: mapOf(
"user_location" to "定位未开启", "user_location" to "定位未开启",
"user_lat" to "定位未开启", "user_lat" to "定位未开启",
"user_lng" to "定位未开启", "user_lng" to "定位未开启",
) )
val _systemPrompt = Utils.fillTemplate(systemPrompt,location) val _systemPrompt = Utils.fillTemplate(systemPrompt, location)
messagesWithSystemPrompt.put(createSystemMessage(_systemPrompt)) messagesWithSystemPrompt.put(createSystemMessage(_systemPrompt))
// Log.d(TAG, "liwei--------- 系统提示词: ${_systemPrompt}") // Log.d(TAG, "liwei--------- 系统提示词: ${_systemPrompt}")
} }
// 再添加历史消息 // 再添加历史消息
for (i in 0 until historyMessages.length()) { for (i in 0 until historyMessages.length()) {
@ -810,9 +836,9 @@ object AgentService : CoroutineScope {
currsessionId, currsessionId,
messagesWithSystemPrompt, messagesWithSystemPrompt,
object : StreamCallback { object : StreamCallback {
override fun onToken(sessionid:String,token: String) { override fun onToken(sessionid: String, token: String) {
try { try {
if (sessionid != currsessionId){ if (sessionid != currsessionId) {
return return
} }
responseBuilder.append(token) responseBuilder.append(token)
@ -828,7 +854,7 @@ object AgentService : CoroutineScope {
} }
} }
override fun onComplete(sessionid:String) { override fun onComplete(sessionid: String) {
try { try {
// 视情况决定是否朗读回复 // 视情况决定是否朗读回复
if (speakResponse && reply && broadcast && sessionid == currsessionId){ if (speakResponse && reply && broadcast && sessionid == currsessionId){
@ -867,7 +893,7 @@ object AgentService : CoroutineScope {
} }
} }
override fun onError(sessionid:String,e: Exception) { override fun onError(sessionid: String, e: Exception) {
try { try {
Log.e(TAG, "AI处理出错", e) Log.e(TAG, "AI处理出错", e)
sendEvent( sendEvent(
@ -880,20 +906,22 @@ object AgentService : CoroutineScope {
// 标记AI流式输出已完成 // 标记AI流式输出已完成
_isAiStreaming.set(false) _isAiStreaming.set(false)
currentAiJob = null currentAiJob = null
}catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "liwei--------------- AI Call onError 异常", e) Log.e(TAG, "liwei--------------- AI Call onError 异常", e)
} }
} }
override fun onFunctionCall(sessionid:String,call: JSONObject) { override fun onFunctionCall(sessionid: String, call: JSONObject) {
try { try {
// audioPlayer?.playAudio(R.raw.calling, true) // audioPlayer?.playAudio(R.raw.calling, true)
val name = call.getString("name") val name = call.getString("name")
sendEvent("function_call", mapOf( sendEvent(
"name" to name, "function_call", mapOf(
"arguments" to call.toString() "name" to name,
)) "arguments" to call.toString()
if(name == "exit_interaction") { )
)
if (name == "exit_interaction") {
stopRecognition() stopRecognition()
} }
} catch (e: Exception) { } catch (e: Exception) {
@ -901,7 +929,11 @@ object AgentService : CoroutineScope {
} }
} }
override fun onFunctionCallResult(sessionid:String,functionCall: JSONObject, functionCallResult: JSONObject) { override fun onFunctionCallResult(
sessionid: String,
functionCall: JSONObject,
functionCallResult: JSONObject
) {
try { try {
audioPlayer?.stopAudio() audioPlayer?.stopAudio()
val name = functionCall.get("name") as String; val name = functionCall.get("name") as String;
@ -917,7 +949,11 @@ object AgentService : CoroutineScope {
) )
) )
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "liwei--------------- AI Call onFunctionCallResult 异常", e) Log.e(
TAG,
"liwei--------------- AI Call onFunctionCallResult 异常",
e
)
} }
} }
} }
@ -926,10 +962,12 @@ object AgentService : CoroutineScope {
} catch (e: Exception) { } catch (e: Exception) {
val errorType = if (hasImage) "AI_IMAGE_PROCESS_ERROR" else "AI_PROCESS_ERROR" val errorType = if (hasImage) "AI_IMAGE_PROCESS_ERROR" else "AI_PROCESS_ERROR"
Log.e(TAG, "liwei--------------- AI处理出错", e) Log.e(TAG, "liwei--------------- AI处理出错", e)
sendEvent("error", mapOf( sendEvent(
"code" to errorType, "error", mapOf(
"message" to e.message.toString() "code" to errorType,
)) "message" to e.message.toString()
)
)
// 确保状态被重置 // 确保状态被重置
_isAiStreaming.set(false) _isAiStreaming.set(false)
@ -1019,7 +1057,12 @@ object AgentService : CoroutineScope {
/** /**
* 保存聊天消息 * 保存聊天消息
*/ */
private fun saveChatMessage(userMessage: String, assistantMessage: String,aiMetadata : String,userMetadata : String) { private fun saveChatMessage(
userMessage: String,
assistantMessage: String,
aiMetadata: String,
userMetadata: String
) {
// 显式指定IO调度器，数据库操作应在后台线程执行，即使父协程在Main上下文中 // 显式指定IO调度器，数据库操作应在后台线程执行，即使父协程在Main上下文中
launch(Dispatchers.IO) { launch(Dispatchers.IO) {
try { try {
@ -1167,7 +1210,12 @@ object AgentService : CoroutineScope {
/** /**
* 处理图片输入 * 处理图片输入
*/ */
fun processImageInput(imagePath: String, imageUrl: String,text: String = "", speakResponse: Boolean = false): Boolean { fun processImageInput(
imagePath: String,
imageUrl: String,
text: String = "",
speakResponse: Boolean = false
): Boolean {
if (!isInitialized) { if (!isInitialized) {
Log.e(TAG, "服务未初始化") Log.e(TAG, "服务未初始化")
sendEvent("error", mapOf("code" to "NOT_INITIALIZED", "message" to "服务未初始化")) sendEvent("error", mapOf("code" to "NOT_INITIALIZED", "message" to "服务未初始化"))
@ -1181,10 +1229,12 @@ object AgentService : CoroutineScope {
} }
// 通知开始处理图片 // 通知开始处理图片
sendEvent("image_processing", mapOf( sendEvent(
"status" to "processing", "image_processing", mapOf(
"imagePath" to imagePath "status" to "processing",
)) "imagePath" to imagePath
)
)
// 处理图片 // 处理图片
launch { launch {
@ -1204,19 +1254,23 @@ object AgentService : CoroutineScope {
val displayText = text.ifEmpty { "[图片]" } val displayText = text.ifEmpty { "[图片]" }
// 通知图片准备完成 // 通知图片准备完成
sendEvent("image_ready", mapOf( sendEvent(
"status" to "ready", "image_ready", mapOf(
"imagePath" to imagePath "status" to "ready",
)) "imagePath" to imagePath
)
)
Log.d(TAG, "liwei--------- 图像Mesage: ${userMessage}") Log.d(TAG, "liwei--------- 图像Mesage: ${userMessage}")
// 处理消息并传入存储路径 // 处理消息并传入存储路径
processWithChatApiInternal(userMessage, displayText, speakResponse, true, imagePath) processWithChatApiInternal(userMessage, displayText, speakResponse, true, imagePath)
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "处理图片失败: ${e.message}") Log.e(TAG, "处理图片失败: ${e.message}")
sendEvent("error", mapOf( sendEvent(
"code" to "IMAGE_PROCESSING_ERROR", "error", mapOf(
"message" to e.message.toString() "code" to "IMAGE_PROCESSING_ERROR",
)) "message" to e.message.toString()
)
)
} }
} }
return true return true
@ -1254,7 +1308,7 @@ object AgentService : CoroutineScope {
fun playAudio(resId: Int, isLooping: Boolean = false, volume: Float = 1.0f) { fun playAudio(resId: Int, isLooping: Boolean = false, volume: Float = 1.0f) {
synchronized(playbackLock) { synchronized(playbackLock) {
try { try {
Log.d(TAG, "playAudio${isLooping}") Log.d(TAG, "playAudio${isLooping}")
// 强制停止并释放之前的资源 // 强制停止并释放之前的资源
forceStop() forceStop()
@ -1298,7 +1352,7 @@ object AgentService : CoroutineScope {
fun stopAudio() { fun stopAudio() {
synchronized(playbackLock) { synchronized(playbackLock) {
try { try {
Log.d(TAG, "stopAudio") Log.d(TAG, "stopAudio")
mediaPlayer?.apply { mediaPlayer?.apply {
if (isMediaPlayerPlaying()) { if (isMediaPlayerPlaying()) {
stop() stop()
@ -1422,7 +1476,10 @@ object AgentService : CoroutineScope {
/* /*
* 自动播放音乐 * 自动播放音乐
* */ * */
fun autoHandleFcunCallResult(toolname:String,functionCallResult: JSONObject): Map<String,Any>{ fun autoHandleFcunCallResult(
toolname: String,
functionCallResult: JSONObject
): Map<String, Any> {
println("协议工具返回数据 检查是否存在卡片! functionCallResult: ${functionCallResult}") println("协议工具返回数据 检查是否存在卡片! functionCallResult: ${functionCallResult}")
try { try {
val contentStr = functionCallResult.optString("context") val contentStr = functionCallResult.optString("context")
@ -1435,10 +1492,10 @@ object AgentService : CoroutineScope {
var notreply = false; var notreply = false;
if (textStr.isNotEmpty()) { if (textStr.isNotEmpty()) {
val meta = JSONObject(textStr) val meta = JSONObject(textStr)
if (meta.has("broadcast")){ if (meta.has("broadcast")) {
broadcast = meta["broadcast"] as Boolean broadcast = meta["broadcast"] as Boolean
} }
if (meta.has("notreply")){ if (meta.has("notreply")) {
notreply = meta["notreply"] as Boolean notreply = meta["notreply"] as Boolean
} }
metadata[toolname] = meta // 现在可以赋值 metadata[toolname] = meta // 现在可以赋值
@ -1501,13 +1558,13 @@ object AgentService : CoroutineScope {
val end = cardNavigation.optString("end", "") val end = cardNavigation.optString("end", "")
processNavigation(start, end); processNavigation(start, end);
} }
return mapOf("meta" to metaStr, "broadcast" to broadcast,"notreply" to notreply); return mapOf("meta" to metaStr, "broadcast" to broadcast, "notreply" to notreply);
} }
} catch (e: Exception) { } catch (e: Exception) {
// 捕获其他类型的异常（可选） // 捕获其他类型的异常（可选）
println("为解析到卡片数据!") println("为解析到卡片数据!")
} }
return mapOf("meta" to "", "broadcast" to true,"notreply" to false); return mapOf("meta" to "", "broadcast" to true, "notreply" to false);
} }
/** /**
@ -1520,7 +1577,7 @@ object AgentService : CoroutineScope {
* @param singer 歌手名称，默认为空 * @param singer 歌手名称，默认为空
* @return 是否成功开始处理 * @return 是否成功开始处理
*/ */
fun processMusicPlay(song: Map<String,Any>){ fun processMusicPlay(song: Map<String, Any>) {
// 在其他 Service、BroadcastReceiver 或 Application 中调用 // 在其他 Service、BroadcastReceiver 或 Application 中调用
Log.i(TAG, "播放音乐: ${song}") Log.i(TAG, "播放音乐: ${song}")
MusicServiceStarter.startServiceWithCommand(context, command = "play", song = song) MusicServiceStarter.startServiceWithCommand(context, command = "play", song = song)
@ -1535,10 +1592,14 @@ object AgentService : CoroutineScope {
* @param singer 歌手，默认为false * @param singer 歌手，默认为false
* @return 是否成功开始处理 * @return 是否成功开始处理
*/ */
fun processMusicPlayList(songs:List<Map<String,Any>>){ fun processMusicPlayList(songs: List<Map<String, Any>>) {
// 如果 playlist 中有有效的歌曲，开始播放 // 如果 playlist 中有有效的歌曲，开始播放
if (songs.isNotEmpty()) { if (songs.isNotEmpty()) {
MusicServiceStarter.startServiceWithPlaylist(context, command = "playlist", songs = songs) MusicServiceStarter.startServiceWithPlaylist(
context,
command = "playlist",
songs = songs
)
} else { } else {
Log.e(TAG, "音乐列表为空，未启动播放服务") Log.e(TAG, "音乐列表为空，未启动播放服务")
} }
@ -1552,10 +1613,15 @@ object AgentService : CoroutineScope {
* @param singer 歌手，默认为false * @param singer 歌手，默认为false
* @return 是否成功开始处理 * @return 是否成功开始处理
*/ */
fun processNavigation(start:String,end:String){ fun processNavigation(start: String, end: String) {
// 如果 playlist 中有有效的歌曲，开始播放 // 如果 playlist 中有有效的歌曲，开始播放
if (!start.isNullOrEmpty() && !end.isNullOrEmpty()) { if (!start.isNullOrEmpty() && !end.isNullOrEmpty()) {
NavigationServiceHelper.startNavigation(context,"start", startpos = start, endpos = end) NavigationServiceHelper.startNavigation(
context,
"start",
startpos = start,
endpos = end
)
} else { } else {
Log.i(TAG, "启动导航服务失败") Log.i(TAG, "启动导航服务失败")
} }

56
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt
View File

@ -15,7 +15,8 @@ import kotlinx.coroutines.launch
import android.content.Intent import android.content.Intent
import android.os.Build import android.os.Build
class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.StreamHandler, AgentServiceListener { class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.StreamHandler,
AgentServiceListener {
private val TAG = "AgentServicePlugin" private val TAG = "AgentServicePlugin"
private lateinit var context: Context private lateinit var context: Context
private lateinit var methodChannel: MethodChannel private lateinit var methodChannel: MethodChannel
@ -24,10 +25,14 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) {
context = flutterPluginBinding.applicationContext context = flutterPluginBinding.applicationContext
methodChannel = MethodChannel(flutterPluginBinding.binaryMessenger, "com.yunqiinnovation.agent_service") methodChannel =
MethodChannel(flutterPluginBinding.binaryMessenger, "com.yunqiinnovation.agent_service")
methodChannel.setMethodCallHandler(this) methodChannel.setMethodCallHandler(this)
eventChannel = EventChannel(flutterPluginBinding.binaryMessenger, "com.yunqiinnovation.agent_service/events") eventChannel = EventChannel(
flutterPluginBinding.binaryMessenger,
"com.yunqiinnovation.agent_service/events"
)
eventChannel.setStreamHandler(this) eventChannel.setStreamHandler(this)
// 添加监听器 // 添加监听器
@ -52,12 +57,16 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
val intent = Intent(context, BleMediaService::class.java) val intent = Intent(context, BleMediaService::class.java)
// 将参数添加到Intent // 将参数添加到Intent
config["azureSpeechKey"]?.toString()?.let { intent.putExtra("azureSpeechKey", it) } config["azureSpeechKey"]?.toString()
config["azureSpeechRegion"]?.toString()?.let { intent.putExtra("azureSpeechRegion", it) } ?.let { intent.putExtra("azureSpeechKey", it) }
config["azureSpeechRegion"]?.toString()
?.let { intent.putExtra("azureSpeechRegion", it) }
config["openaiApiKey"]?.toString()?.let { intent.putExtra("openaiApiKey", it) } config["openaiApiKey"]?.toString()?.let { intent.putExtra("openaiApiKey", it) }
config["openaiBaseUrl"]?.toString()?.let { intent.putExtra("openaiBaseUrl", it) } config["openaiBaseUrl"]?.toString()
?.let { intent.putExtra("openaiBaseUrl", it) }
config["openaiModel"]?.toString()?.let { intent.putExtra("openaiModel", it) } config["openaiModel"]?.toString()?.let { intent.putExtra("openaiModel", it) }
config["openaivisionModel"]?.toString()?.let { intent.putExtra("openaivisionModel", it) } config["openaivisionModel"]?.toString()
?.let { intent.putExtra("openaivisionModel", it) }
config["systemPrompt"]?.toString()?.let { intent.putExtra("systemPrompt", it) } config["systemPrompt"]?.toString()?.let { intent.putExtra("systemPrompt", it) }
config["mcpServer"]?.toString()?.let { intent.putExtra("mcpServer", it) } config["mcpServer"]?.toString()?.let { intent.putExtra("mcpServer", it) }
config["volcanoAppId"]?.toString()?.let { intent.putExtra("volcanoAppId", it) } config["volcanoAppId"]?.toString()?.let { intent.putExtra("volcanoAppId", it) }
@ -76,6 +85,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("SERVICE_START_ERROR", "启动AgentService失败: ${e.message}", null) result.error("SERVICE_START_ERROR", "启动AgentService失败: ${e.message}", null)
} }
} }
"stopAgentService" -> { "stopAgentService" -> {
try { try {
val intent = Intent(context, BleMediaService::class.java) val intent = Intent(context, BleMediaService::class.java)
@ -86,6 +96,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("SERVICE_STOP_ERROR", "停止AgentService失败: ${e.message}", null) result.error("SERVICE_STOP_ERROR", "停止AgentService失败: ${e.message}", null)
} }
} }
"startConversation" -> { "startConversation" -> {
try { try {
val isStarted = AgentService.startRecognition(false) val isStarted = AgentService.startRecognition(false)
@ -95,6 +106,11 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("START_ERROR", "启动对话失败: ${e.message}", null) result.error("START_ERROR", "启动对话失败: ${e.message}", null)
} }
} }
"recognizeCallback" -> {
val isStarted = AgentService.recognizeCallback()
result.success(isStarted)
}
"stopConversation" -> { "stopConversation" -> {
try { try {
AgentService.stopRecognition() AgentService.stopRecognition()
@ -104,24 +120,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("STOP_ERROR", "停止对话失败: ${e.message}", null) result.error("STOP_ERROR", "停止对话失败: ${e.message}", null)
} }
} }
"audioContinue" -> {
try {
AgentService.audioContinue()
result.success(true)
} catch (e: Exception) {
Log.e(TAG, "audioContinue失败", e)
result.error("STOP_ERROR", "audioContinue失败: ${e.message}", null)
}
}
"audioPause" -> {
try {
AgentService.audioPause()
result.success(true)
} catch (e: Exception) {
Log.e(TAG, "audioContinue失败", e)
result.error("STOP_ERROR", "audioContinue失败: ${e.message}", null)
}
}
"processTextInput" -> { "processTextInput" -> {
val text = call.argument<String>("text") val text = call.argument<String>("text")
if (text.isNullOrEmpty()) { if (text.isNullOrEmpty()) {
@ -143,6 +142,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("TEXT_PROCESS_ERROR", "处理文本输入失败: ${e.message}", null) result.error("TEXT_PROCESS_ERROR", "处理文本输入失败: ${e.message}", null)
} }
} }
"processImageInput" -> { "processImageInput" -> {
val imagePath = call.argument<String>("imagePath") val imagePath = call.argument<String>("imagePath")
if (imagePath.isNullOrEmpty()) { if (imagePath.isNullOrEmpty()) {
@ -171,6 +171,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("IMAGE_PROCESS_ERROR", "处理图片输入失败: ${e.message}", null) result.error("IMAGE_PROCESS_ERROR", "处理图片输入失败: ${e.message}", null)
} }
} }
"speakText" -> { "speakText" -> {
val text = call.argument<String>("text") val text = call.argument<String>("text")
if (text.isNullOrEmpty()) { if (text.isNullOrEmpty()) {
@ -186,6 +187,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("TTS_ERROR", "语音合成失败: ${e.message}", null) result.error("TTS_ERROR", "语音合成失败: ${e.message}", null)
} }
} }
"stopTts" -> { "stopTts" -> {
try { try {
AgentService.stopTts() AgentService.stopTts()
@ -195,6 +197,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("STOP_TTS_ERROR", "停止语音合成失败: ${e.message}", null) result.error("STOP_TTS_ERROR", "停止语音合成失败: ${e.message}", null)
} }
} }
"stopAiSteam" -> { "stopAiSteam" -> {
try { try {
AgentService.stopAiStream() AgentService.stopAiStream()
@ -215,6 +218,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("CLEAR_HISTORY_ERROR", "清除聊天历史失败: ${e.message}", null) result.error("CLEAR_HISTORY_ERROR", "清除聊天历史失败: ${e.message}", null)
} }
} }
"interruptCurrentResponse" -> { "interruptCurrentResponse" -> {
try { try {
AgentService.interruptCurrentResponse() AgentService.interruptCurrentResponse()
@ -224,6 +228,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("INTERRUPT_ERROR", "中断响应失败: ${e.message}", null) result.error("INTERRUPT_ERROR", "中断响应失败: ${e.message}", null)
} }
} }
"dispose" -> { "dispose" -> {
try { try {
AgentService.dispose() AgentService.dispose()
@ -233,6 +238,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("DISPOSE_ERROR", "释放资源失败: ${e.message}", null) result.error("DISPOSE_ERROR", "释放资源失败: ${e.message}", null)
} }
} }
else -> { else -> {
result.notImplemented() result.notImplemented()
} }

22
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt
View File

@ -65,7 +65,7 @@ object BleAgent : BleService.Callback, AgentServiceListener {
Log.e(TAG, "语音代理未初始化") Log.e(TAG, "语音代理未初始化")
return false return false
} }
Log.d(TAG, "手动启动语音识别: ")
return AgentService.startRecognition(true) return AgentService.startRecognition(true)
} }
@ -73,7 +73,12 @@ object BleAgent : BleService.Callback, AgentServiceListener {
* 停止语音识别 * 停止语音识别
*/ */
fun stopListening() { fun stopListening() {
AgentService.stopRecognition() if (!isInitialized) {
Log.e(TAG, "语音代理未初始化")
return
}
AgentService.stopRecognition()
return
} }
/** /**
@ -121,7 +126,7 @@ object BleAgent : BleService.Callback, AgentServiceListener {
} else if (state == BleConst.STATE_DISCONNECTED) { } else if (state == BleConst.STATE_DISCONNECTED) {
Log.d(TAG, "Ble设备已断开，停止语音服务") Log.d(TAG, "Ble设备已断开，停止语音服务")
// 设备断开连接，停止语音服务 // 设备断开连接，停止语音服务
// stopListening() stopListening()
} }
} }
@ -136,18 +141,19 @@ object BleAgent : BleService.Callback, AgentServiceListener {
* 在收到唤醒信号时启动语音识别 * 在收到唤醒信号时启动语音识别
*/ */
override fun onWakeupSignalReceived() { override fun onWakeupSignalReceived() {
Log.d(TAG, "收到唤醒信号，启动语音识别") Log.d(TAG, "收到唤醒信号，启动语音识别${AgentService.isRecognitionActive}")
// 停止当前TTS，避免冲突 // 停止当前TTS，避免冲突
AgentService.interruptCurrentResponse() AgentService.interruptCurrentResponse()
// AgentService.speakText("我在!") // AgentService.speakText("我在!")
// 启动语音识别 // 启动语音识别
if (AgentService.isRecognitionActive&&AgentService.azureAsrHelper!!.isPaused) {
// 如果已经在识别，则重新启动 // 如果已经在识别，则重新启动
stopListening() //stopListening()
}
startListening() //stopListening()
startListening()
} }
override fun onDeviceInfoReceived(infoType: Int, infoData: Map<String, Any>) { override fun onDeviceInfoReceived(infoType: Int, infoData: Map<String, Any>) {

53
local_plugins/agent_service/lib/agent_service.dart
View File

@ -248,16 +248,16 @@ class AgentService {
} }
} }
/// 停止对话 /// 设备识别器
/// ///
/// 停止语音识别 /// 启动语音识别，开始监听用户语音输入
/// 返回是否成功停止对话 /// 返回是否成功开始对话
static Future<bool> stopConversation() async { static Future<bool> recognizeCallback() async {
try { try {
final bool result = await _channel.invokeMethod('stopConversation'); final bool result = await _channel.invokeMethod('recognizeCallback');
return result; return result;
} on PlatformException catch (e) { } on PlatformException catch (e) {
throw AgentServiceException(e.code, e.message ?? '停止对话失败', e.details); throw AgentServiceException(e.code, e.message ?? '设置识别器失败', e.details);
} }
} }
@ -265,27 +265,40 @@ class AgentService {
/// ///
/// 停止语音识别 /// 停止语音识别
/// 返回是否成功停止对话 /// 返回是否成功停止对话
static Future<bool> audioContinue() async { static Future<bool> stopConversation() async {
try { try {
final bool result = await _channel.invokeMethod('audioContinue'); final bool result = await _channel.invokeMethod('stopConversation');
return result; return result;
} on PlatformException catch (e) { } on PlatformException catch (e) {
throw AgentServiceException(e.code, e.message ?? '停止对话失败', e.details); throw AgentServiceException(e.code, e.message ?? '停止对话失败', e.details);
} }
} }
/// 停止对话 // /// 停止对话
/// // ///
/// 停止语音识别 // /// 停止语音识别
/// 返回是否成功停止对话 // /// 返回是否成功停止对话
static Future<bool> audioPause() async { // static Future<bool> audioContinue() async {
try { // try {
final bool result = await _channel.invokeMethod('audioPause'); // final bool result = await _channel.invokeMethod('audioContinue');
return result; // return result;
} on PlatformException catch (e) { // } on PlatformException catch (e) {
throw AgentServiceException(e.code, e.message ?? '停止对话失败', e.details); // throw AgentServiceException(e.code, e.message ?? '停止对话失败', e.details);
} // }
} // }
// /// 停止对话
// ///
// /// 停止语音识别
// /// 返回是否成功停止对话
// static Future<bool> audioPause() async {
// try {
// final bool result = await _channel.invokeMethod('audioPause');
// return result;
// } on PlatformException catch (e) {
// throw AgentServiceException(e.code, e.message ?? '停止对话失败', e.details);
// }
// }
/// 处理文本输入 /// 处理文本输入
/// ///

810
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
View File

File diff suppressed because it is too large

128
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt
View File

@ -255,24 +255,15 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
} }
} }
"setPause" -> {
azureAsrHelper.audioPause();
return
}
"setcontinue" -> {
azureAsrHelper.audioContinue();
return
}
"disableBluetoothAudio" -> { "disableBluetoothAudio" -> {
azureAsrHelper.disableBluetoothAudio(); azureAsrHelper.disableBluetoothAudio();
return return
} }
"restoreOriginalAudioState" -> { "restoreOriginalAudioState" -> {
azureAsrHelper.restoreOriginalAudioState(); azureAsrHelper.restoreOriginalAudioState();
return return
} }
"startContinuousRecognition" -> { "startContinuousRecognition" -> {
@ -296,62 +287,70 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
AzureAsrHelper.AudioSourceType.MICROPHONE AzureAsrHelper.AudioSourceType.MICROPHONE
} }
val success = azureAsrHelper.startContinuousRecognition(object : val success = azureAsrHelper.startContinuousRecognition(audioSourceType)
AzureAsrHelper.ContinuousRecognizeCallback { result.success(true)
override fun onResult(text: String, detectedLanguage: String) { } catch (e: Exception) {
sendAsrEvent( result.error("START_RECOGNITION_ERROR", e.message, null)
mapOf( }
"type" to "result", }
"text" to text,
"detectedLanguage" to detectedLanguage "recognizeCallback" -> {
)
val success = azureAsrHelper.setupEventListeners(object :
AzureAsrHelper.ContinuousRecognizeCallback {
override fun onResult(text: String, detectedLanguage: String) {
sendAsrEvent(
mapOf(
"type" to "result",
"text" to text,
"detectedLanguage" to detectedLanguage
) )
} )
}
override fun onRecognizing( override fun onRecognizing(
recognizing: String, recognizing: String,
detectedLanguage: String detectedLanguage: String
) { ) {
sendAsrEvent( sendAsrEvent(
mapOf( mapOf(
"type" to "recognizing", "type" to "recognizing",
"text" to recognizing, "text" to recognizing,
"detectedLanguage" to detectedLanguage "detectedLanguage" to detectedLanguage
)
) )
} )
}
override fun onSessionStarted() { override fun onSessionStarted() {
sendAsrEvent(mapOf("type" to "sessionStarted")) sendAsrEvent(mapOf("type" to "sessionStarted"))
} }
override fun onSessionStopped() { override fun onSessionStopped() {
sendAsrEvent(mapOf("type" to "sessionStopped")) sendAsrEvent(mapOf("type" to "sessionStopped"))
} }
override fun onCanceled(reason: String, errorDetails: String) { override fun onCanceled(reason: String, errorDetails: String) {
sendAsrEvent(
mapOf( sendAsrEvent(
"type" to "canceled", mapOf(
"reason" to reason, "type" to "canceled",
"errorDetails" to errorDetails "reason" to reason,
) "errorDetails" to errorDetails
) )
} )
}
override fun onError(error: String) { override fun onError(error: String) {
sendAsrEvent( sendAsrEvent(
mapOf( mapOf(
"type" to "error", "type" to "error",
"message" to error "message" to error
)
) )
} )
}, audioSourceType) }
result.success(success) })
} catch (e: Exception) { FileLogger.d(tag, "recognizeCallback: ${success}") //
result.error("START_RECOGNITION_ERROR", e.message, null) result.success(success)
}
} }
"stopContinuousRecognition" -> { "stopContinuousRecognition" -> {
@ -378,7 +377,8 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
"音频数据不能为空", "音频数据不能为空",
null null
) )
azureAsrHelper.pushAudioData(data)
azureAsrHelper.audioStream?.saveAudioDataTo(data)
result.success(true) result.success(true)
} }
@ -525,16 +525,16 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
"setAudioOutputDevice" -> { "setAudioOutputDevice" -> {
val type = call.argument<Int>("type") val type = call.argument<Int>("type")
var success =false var success = false
if (type == 0) { if (type == 0) {
// 默认（如果有耳机选耳机，否则使用系统扬声器） // 默认（如果有耳机选耳机，否则使用系统扬声器）
success=azureTtsHelper.setAudioOutputDevice(AudioOutputDevice.DEFAULT) success = azureTtsHelper.setAudioOutputDevice(AudioOutputDevice.DEFAULT)
} else if (type == 1) { } else if (type == 1) {
// 强制使用声器 // 强制使用声器
success=azureTtsHelper.setAudioOutputDevice(AudioOutputDevice.SPEAKER) success = azureTtsHelper.setAudioOutputDevice(AudioOutputDevice.SPEAKER)
} else if (type == 2) { } else if (type == 2) {
// 强制使用耳机 // 强制使用耳机
success=azureTtsHelper.setAudioOutputDevice(AudioOutputDevice.HEADPHONES) success = azureTtsHelper.setAudioOutputDevice(AudioOutputDevice.HEADPHONES)
} }
@ -583,7 +583,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
override fun onAudioDataReceived(data: ByteArray) { override fun onAudioDataReceived(data: ByteArray) {
if (azureAsrHelper.audioSourceType == AzureAsrHelper.AudioSourceType.EXTERNAL) { if (azureAsrHelper.audioSourceType == AzureAsrHelper.AudioSourceType.EXTERNAL) {
azureAsrHelper.pushAudioData(data) azureAsrHelper.audioStream?.saveAudioDataTo(data)
} }
// 可选：处理音频数据 // 可选：处理音频数据
} }

1
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/RecordFile.kt
View File

@ -248,6 +248,7 @@ object RecordFile {
} finally { } finally {
fos = null fos = null
currentAudioFile = null currentAudioFile = null
writeThread = null
} }
return false return false
} }

55
local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt
View File

@ -1,5 +1,7 @@
package com.yunqiinnovation.ble_service package com.yunqiinnovation.ble_service
import android.Manifest
import android.annotation.SuppressLint
import android.bluetooth.* import android.bluetooth.*
import android.bluetooth.le.* import android.bluetooth.le.*
import android.content.Context import android.content.Context
@ -22,6 +24,7 @@ import java.util.concurrent.LinkedBlockingQueue
import android.os.Handler import android.os.Handler
import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicBoolean
import android.util.Log import android.util.Log
import androidx.annotation.RequiresPermission
/** /**
* BLE服务类：提供蓝牙低功耗设备的扫描、连接和通信功能 * BLE服务类：提供蓝牙低功耗设备的扫描、连接和通信功能
@ -61,7 +64,7 @@ object BleService {
// 上下文 // 上下文
private lateinit var context: Context private lateinit var context: Context
private val commandQueue: Queue<ByteArray> = LinkedList()
// 蓝牙相关组件 // 蓝牙相关组件
private lateinit var bluetoothManager: BluetoothManager private lateinit var bluetoothManager: BluetoothManager
private var bluetoothAdapter: BluetoothAdapter? = null private var bluetoothAdapter: BluetoothAdapter? = null
@ -83,6 +86,10 @@ object BleService {
// 是否是ota模式 // 是否是ota模式
var isEnterOta = false var isEnterOta = false
// 是否回复指令
var isReply = true
// 是否回复指令码
var cmdReplyType =0xff
// 连接状态 // 连接状态
val connectionState = MutableLiveData(BleConst.STATE_DISCONNECTED) val connectionState = MutableLiveData(BleConst.STATE_DISCONNECTED)
@ -93,9 +100,6 @@ object BleService {
private var opusManager: OpusManager? = null private var opusManager: OpusManager? = null
private val mainHandler = Handler(Looper.getMainLooper()) private val mainHandler = Handler(Looper.getMainLooper())
// ota解码器实例
// var otaManager: OTAManager? = null
// 初始化状态 // 初始化状态
private var isInitialized = false private var isInitialized = false
@ -663,6 +667,7 @@ object BleService {
private fun processAudioData(data: ByteArray) { private fun processAudioData(data: ByteArray) {
try { try {
if (opusManager?.isDecodeStream == true) { if (opusManager?.isDecodeStream == true) {
recordfile?.saveAudioDataToWav(data) recordfile?.saveAudioDataToWav(data)
opusManager?.writeAudioStream(data) opusManager?.writeAudioStream(data)
} else { } else {
@ -677,6 +682,8 @@ object BleService {
/** /**
* 处理设备响应数据 - 帧头为0xBB * 处理设备响应数据 - 帧头为0xBB
*/ */
@SuppressLint("StaticFieldLeak")
@RequiresPermission(Manifest.permission.BLUETOOTH_CONNECT)
private fun processDeviceResponse(data: ByteArray) { private fun processDeviceResponse(data: ByteArray) {
Log.d( Log.d(
TAG, TAG,
@ -741,7 +748,21 @@ object BleService {
} }
}" }"
) )
isReply=true;
cmdReplyType=cmdType;
if(!commandQueue.isEmpty())
{
val nextCmd = commandQueue.poll()
writeChar!!.value=nextCmd
var isSeccess = bluetoothGatt?.writeCharacteristic(writeChar)
if(isSeccess==true)
{
Log.d(
TAG,
"收到设备响应:"
)
}
}
// 根据命令类型处理不同的响应 // 根据命令类型处理不同的响应
when (cmdType) { when (cmdType) {
BleConst.CMD_GET_VERSION -> { BleConst.CMD_GET_VERSION -> {
@ -1005,7 +1026,7 @@ object BleService {
BleConst.CMD_WAKE_UP -> { BleConst.CMD_WAKE_UP -> {
Log.i(TAG, "收到唤醒指令") Log.i(TAG, "收到唤醒指令")
// 接收到唤醒信号，打开编码器 (设备侧) // 接收到唤醒信号，打开编码器 (设备侧)
openEncoder()
notifyWakeupSignalReceived() notifyWakeupSignalReceived()
} }
@ -1316,6 +1337,7 @@ object BleService {
* @param cmdData 命令数据 * @param cmdData 命令数据
* @return 是否成功发送 * @return 是否成功发送
*/ */
@RequiresPermission(Manifest.permission.BLUETOOTH_CONNECT)
private fun sendCommand(cmdType: Byte, cmdData: ByteArray = byteArrayOf()): Boolean { private fun sendCommand(cmdType: Byte, cmdData: ByteArray = byteArrayOf()): Boolean {
if (!checkConn()) { if (!checkConn()) {
Log.e(TAG, "发送命令失败: 设备未连接") Log.e(TAG, "发送命令失败: 设备未连接")
@ -1358,7 +1380,26 @@ object BleService {
// 发送数据 // 发送数据
writeChar!!.value = cmdPacket writeChar!!.value = cmdPacket
return bluetoothGatt?.writeCharacteristic(writeChar) ?: false // 加入队列并触发发送
if(isReply)
{
var isSeccess = bluetoothGatt?.writeCharacteristic(writeChar)
if(isSeccess == true)
{
isReply=false;
cmdType;
return true
}
else
return false
}
else
{
commandQueue.offer(cmdPacket)
}
return true
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "发送命令异常: ${e.message}", e) Log.e(TAG, "发送命令异常: ${e.message}", e)
return false return false

Write Preview
Loading…
Cancel
Save