Browse Source

优化语言识别,优化ble指令发送

weicu
fdp 1 year ago
parent
commit
09e3e856b8
  1. 11
      lib/data/services/asr_service.dart
  2. 38
      lib/data/services/ble_manager.dart
  3. 55
      lib/data/services/speech_impl/azure_asr_service.dart
  4. 18
      lib/data/services/speech_impl/volcano_asr_api_service.dart
  5. 18
      lib/data/services/speech_impl/volcano_asr_service.dart
  6. 100
      lib/data/services/speech_impl/xunfei_asr_service.dart
  7. 11
      lib/modules/FTFTranslation/controllers/FTFTranslation_controller.dart
  8. 60
      lib/modules/agent/controllers/agent_controller.dart
  9. 31
      lib/modules/meeting/controllers/meeting_record_controller.dart
  10. 44
      lib/modules/settings/views/settings_view.dart
  11. 8
      lib/modules/speech_test/controllers/speech_test_controller.dart
  12. 34
      lib/modules/translation/controllers/translation_controller.dart
  13. 276
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  14. 56
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt
  15. 18
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt
  16. 53
      local_plugins/agent_service/lib/agent_service.dart
  17. 614
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
  18. 32
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt
  19. 1
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/RecordFile.kt
  20. 55
      local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt

11
lib/data/services/asr_service.dart

@ -19,18 +19,17 @@ abstract class AsrService {
/// 开始连续语音识别 /// 开始连续语音识别
/// ///
/// 返回一个包含识别事件的流 /// 返回一个包含识别事件的流
Future<Stream<RecognitionEvent>> startContinuousRecognition( Future<bool> startContinuousRecognition(
bool audioSourceType, bool audioSourceType,
); );
/// 停止连续语音识别 /// 停止连续语音识别
Future<bool> stopContinuousRecognition(); Future<bool> stopContinuousRecognition();
/// 停止音频输入 /// 设置识别回掉
Future<void> setPause(); ///
/// 返回一个包含识别事件的流
/// 继续音频输入 Future<Stream<RecognitionEvent>> recognizeCallback();
Future<void> setContinue();
/// 恢复原始音频状态 /// 恢复原始音频状态
Future<void> restoreOriginalAudioState(); Future<void> restoreOriginalAudioState();

38
lib/data/services/ble_manager.dart

@ -472,37 +472,13 @@ class BleManager extends GetxService {
firmwareProductColor: '', firmwareProductColor: '',
); );
} }
final commands = [
_bleService.getBatteryInfo, // 并行获取所有设备信息,不使用延迟
_bleService.getDeviceVersion, final results = await Future.wait([
_bleService.getProductId, _bleService.getBatteryInfo(),
]; _bleService.getDeviceVersion(),
_bleService.getProductId(),
final delays = [ ]);
Duration.zero, // 第一个命令立即执行
const Duration(milliseconds: 100), // 第二个命令延迟100ms
const Duration(milliseconds: 100), // 第三个命令延迟200ms
];
final futures = commands
.asMap()
.map((index, command) {
final delay = delays[index];
return MapEntry(
index,
Future.delayed(delay, command),
);
})
.values
.toList();
final results = await Future.wait(futures);
// // 并行获取所有设备信息,不使用延迟
// final results = await Future.wait([
// _bleService.getBatteryInfo(),
// _bleService.getDeviceVersion(),
// _bleService.getProductId(),
// ]);
Logger.info( Logger.info(
'设备信息获取结果: 电池=${results[0]}, 版本=${results[1]}, 产品ID=${results[2]}'); '设备信息获取结果: 电池=${results[0]}, 版本=${results[1]}, 产品ID=${results[2]}');

55
lib/data/services/speech_impl/azure_asr_service.dart

@ -150,8 +150,7 @@ class AzureAsrService extends GetxService implements AsrService {
} }
@override @override
Future<Stream<RecognitionEvent>> startContinuousRecognition( Future<bool> startContinuousRecognition(bool audioSourceType) async {
bool audioSourceType) async {
if (!_isInitialized) { if (!_isInitialized) {
await initialize(); await initialize();
} }
@ -161,8 +160,6 @@ class AzureAsrService extends GetxService implements AsrService {
} }
try { try {
_eventStreamController = StreamController<RecognitionEvent>.broadcast();
// 开始连续识别 // 开始连续识别
final bool result = final bool result =
await _channel.invokeMethod('startContinuousRecognition', { await _channel.invokeMethod('startContinuousRecognition', {
@ -170,15 +167,13 @@ class AzureAsrService extends GetxService implements AsrService {
}); });
if (!result) { if (!result) {
_cleanupEventStream();
throw Exception('启动连续识别失败'); throw Exception('启动连续识别失败');
} }
_isContinuousRecognitionActive = true; _isContinuousRecognitionActive = true;
return _eventStreamController!.stream; return result;
} catch (e) { } catch (e) {
Logger.error('开始连续语音识别失败: ${e.toString()}'); Logger.error('开始连续语音识别失败: ${e.toString()}');
_cleanupEventStream();
rethrow; rethrow;
} }
} }
@ -192,11 +187,30 @@ class AzureAsrService extends GetxService implements AsrService {
try { try {
final bool result = final bool result =
await _channel.invokeMethod('stopContinuousRecognition'); await _channel.invokeMethod('stopContinuousRecognition');
_cleanupEventStream();
return result; return result;
} catch (e) { } catch (e) {
Logger.error('停止连续语音识别失败: ${e.toString()}'); Logger.error('停止连续语音识别失败: ${e.toString()}');
rethrow;
}
}
@override
Future<Stream<RecognitionEvent>> recognizeCallback() async {
if (!_isInitialized) {
await initialize();
}
try {
_eventStreamController = StreamController<RecognitionEvent>.broadcast();
// 开始连续识别
final bool result = await _channel.invokeMethod('recognizeCallback');
if (!result) {
_cleanupEventStream(); _cleanupEventStream();
}
return _eventStreamController!.stream;
} catch (e) {
Logger.error('开始连续语音识别失败: ${e.toString()}');
rethrow; rethrow;
} }
} }
@ -415,31 +429,6 @@ class AzureAsrService extends GetxService implements AsrService {
} }
} }
@override
Future<void> setPause() async {
// TODO: implement setPause
try {
final bool result = await _channel.invokeMethod('setPause');
return;
} catch (e) {
Logger.error('开始录音: ${e.toString()}');
rethrow;
}
}
@override
Future<void> setContinue() async {
// TODO: implement setcontinue
try {
final bool result = await _channel.invokeMethod('setcontinue');
return;
} catch (e) {
Logger.error('开始录音: ${e.toString()}');
rethrow;
}
}
@override @override
Future<void> disableBluetoothAudio() async { Future<void> disableBluetoothAudio() async {
// TODO: implement disableBluetoothAudio // TODO: implement disableBluetoothAudio

18
lib/data/services/speech_impl/volcano_asr_api_service.dart

@ -842,18 +842,6 @@ class VolcanoAsrApiService implements AsrService {
throw UnimplementedError(); throw UnimplementedError();
} }
@override
Future<void> setPause() {
// TODO: implement setPause
throw UnimplementedError();
}
@override
Future<void> setContinue() {
// TODO: implement setcontinue
throw UnimplementedError();
}
@override @override
Future<void> disableBluetoothAudio() { Future<void> disableBluetoothAudio() {
// TODO: implement disableBluetoothAudio // TODO: implement disableBluetoothAudio
@ -865,4 +853,10 @@ class VolcanoAsrApiService implements AsrService {
// TODO: implement restoreOriginalAudioState // TODO: implement restoreOriginalAudioState
throw UnimplementedError(); throw UnimplementedError();
} }
@override
Future<Stream<RecognitionEvent>> recognizeCallback() {
// TODO: implement recognizeCallback
throw UnimplementedError();
}
} }

18
lib/data/services/speech_impl/volcano_asr_service.dart

@ -377,18 +377,6 @@ class VolcanoAsrService extends GetxService implements AsrService {
throw UnimplementedError(); throw UnimplementedError();
} }
@override
Future<void> setPause() {
// TODO: implement setPause
throw UnimplementedError();
}
@override
Future<void> setContinue() {
// TODO: implement setcontinue
throw UnimplementedError();
}
@override @override
Future<void> disableBluetoothAudio() { Future<void> disableBluetoothAudio() {
// TODO: implement disableBluetoothAudio // TODO: implement disableBluetoothAudio
@ -400,4 +388,10 @@ class VolcanoAsrService extends GetxService implements AsrService {
// TODO: implement restoreOriginalAudioState // TODO: implement restoreOriginalAudioState
throw UnimplementedError(); throw UnimplementedError();
} }
@override
Future<Stream<RecognitionEvent>> recognizeCallback() {
// TODO: implement recognizeCallback
throw UnimplementedError();
}
} }

100
lib/data/services/speech_impl/xunfei_asr_service.dart

@ -74,48 +74,48 @@ class XunfeiAsrService extends GetxService implements AsrService {
} }
} }
@override // @override
Future<Stream<RecognitionEvent>> startContinuousRecognition( // Future<Stream<RecognitionEvent>> startContinuousRecognition(
bool audioSourceType) async { // bool audioSourceType) async {
if (!_isInitialized) { // if (!_isInitialized) {
await initialize(); // await initialize();
} // }
try { // try {
_eventStreamController = StreamController<RecognitionEvent>.broadcast(); // _eventStreamController = StreamController<RecognitionEvent>.broadcast();
await recorder?.dispose(); // await recorder?.dispose();
recorder = AudioRecorder(); // recorder = AudioRecorder();
Stream<Uint8List>? stream = await recorder?.startStream( // Stream<Uint8List>? stream = await recorder?.startStream(
const RecordConfig( // const RecordConfig(
sampleRate: 16000, // sampleRate: 16000,
encoder: AudioEncoder.pcm16bits, // encoder: AudioEncoder.pcm16bits,
numChannels: 1, // numChannels: 1,
), // ),
); // );
isRecording = true; // isRecording = true;
stream?.listen((data) { // stream?.listen((data) {
if (isRecording) { // if (isRecording) {
_buff = _appendToBuffer(data); // _buff = _appendToBuffer(data);
_originaBytes.add(data); // _originaBytes.add(data);
} // }
}); // });
await _tryConnecting(); // await _tryConnecting();
_timer = Timer.periodic(const Duration(milliseconds: 40), (timer) async { // _timer = Timer.periodic(const Duration(milliseconds: 40), (timer) async {
//每40毫秒读取一次数据 // //每40毫秒读取一次数据
var frame = _getAudioData(); // var frame = _getAudioData();
if (frame.isNotEmpty) { // if (frame.isNotEmpty) {
_channel?.sink.add(frame); // _channel?.sink.add(frame);
} // }
}); // });
_isContinuousRecognitionActive = true; // _isContinuousRecognitionActive = true;
return _eventStreamController!.stream; // return _eventStreamController!.stream;
} catch (e) { // } catch (e) {
Logger.error('开始连续语音识别失败: ${e.toString()}'); // Logger.error('开始连续语音识别失败: ${e.toString()}');
rethrow; // rethrow;
} // }
} // }
//写入音频数据到缓存区中 //写入音频数据到缓存区中
Uint8List _appendToBuffer(Uint8List newData) { Uint8List _appendToBuffer(Uint8List newData) {
@ -302,26 +302,26 @@ class XunfeiAsrService extends GetxService implements AsrService {
} }
@override @override
Future<void> setPause() { Future<void> disableBluetoothAudio() {
// TODO: implement setPause // TODO: implement disableBluetoothAudio
throw UnimplementedError(); throw UnimplementedError();
} }
@override @override
Future<void> setContinue() { Future<void> restoreOriginalAudioState() {
// TODO: implement setcontinue // TODO: implement restoreOriginalAudioState
throw UnimplementedError(); throw UnimplementedError();
} }
@override @override
Future<void> disableBluetoothAudio() { Future<Stream<RecognitionEvent>> recognizeCallback() {
// TODO: implement disableBluetoothAudio // TODO: implement recognizeCallback
throw UnimplementedError(); throw UnimplementedError();
} }
@override @override
Future<void> restoreOriginalAudioState() { Future<bool> startContinuousRecognition(bool audioSourceType) {
// TODO: implement restoreOriginalAudioState // TODO: implement startContinuousRecognition
throw UnimplementedError(); throw UnimplementedError();
} }
} }

11
lib/modules/FTFTranslation/controllers/FTFTranslation_controller.dart

@ -49,6 +49,7 @@ class FTFTranslationController extends GetxController
Get.put(XunfeiAsrService()); Get.put(XunfeiAsrService());
final TtsService _ttsService = Get.find<TtsService>(); final TtsService _ttsService = Get.find<TtsService>();
_asrService = Get.find<XunfeiAsrService>(); _asrService = Get.find<XunfeiAsrService>();
_InfoRecorder();
} }
@override @override
@ -59,14 +60,18 @@ class FTFTranslationController extends GetxController
} }
//开始录音 //开始录音
Future<void> _startRecorder() async { Future<void> _InfoRecorder() async {
final recognitionStream = final recognitionStream = await _asrService.recognizeCallback();
await _asrService.startContinuousRecognition(false);
_recognitionSubscription = recognitionStream.listen( _recognitionSubscription = recognitionStream.listen(
_handleRecognitionEvent, _handleRecognitionEvent,
); );
} }
//开始录音
Future<void> _startRecorder() async {
await _asrService.startContinuousRecognition(false);
}
//结束录音 //结束录音
Future<void> _stopRecorder() async { Future<void> _stopRecorder() async {
await _asrService.stopContinuousRecognition(); await _asrService.stopContinuousRecognition();

60
lib/modules/agent/controllers/agent_controller.dart

@ -293,7 +293,8 @@ class AgentController extends GetxController {
currentPage.value = 1; currentPage.value = 1;
hasMoreMessages.value = result.totalPages > 1; hasMoreMessages.value = result.totalPages > 1;
Logger.i(TAG, '成功加载 ${historyMessages.length} 条聊天记录,总页数: ${result.totalPages}'); Logger.i(TAG,
'成功加载 ${historyMessages.length} 条聊天记录,总页数: ${result.totalPages}');
_logMessageDetails(historyMessages); _logMessageDetails(historyMessages);
} else { } else {
// 没有历史记录时添加欢迎消息 // 没有历史记录时添加欢迎消息
@ -316,7 +317,8 @@ class AgentController extends GetxController {
Future<void> loadMoreMessages() async { Future<void> loadMoreMessages() async {
// 防止重复加载 // 防止重复加载
if (isLoadingMore.value || !hasMoreMessages.value) { if (isLoadingMore.value || !hasMoreMessages.value) {
Logger.d(TAG, '跳过加载更多:isLoadingMore=${isLoadingMore.value}, hasMoreMessages=${hasMoreMessages.value}'); Logger.d(TAG,
'跳过加载更多:isLoadingMore=${isLoadingMore.value}, hasMoreMessages=${hasMoreMessages.value}');
return; return;
} }
@ -351,7 +353,8 @@ class AgentController extends GetxController {
currentPage.value = nextPage; currentPage.value = nextPage;
hasMoreMessages.value = nextPage < result.totalPages; hasMoreMessages.value = nextPage < result.totalPages;
Logger.i(TAG, '成功加载 ${newMessages.length} 条历史消息,当前页: $nextPage/${result.totalPages}'); Logger.i(TAG,
'成功加载 ${newMessages.length} 条历史消息,当前页: $nextPage/${result.totalPages}');
} else { } else {
// 没有更多消息 // 没有更多消息
hasMoreMessages.value = false; hasMoreMessages.value = false;
@ -414,7 +417,8 @@ class AgentController extends GetxController {
void _setupScrollListener() { void _setupScrollListener() {
scrollController.addListener(() { scrollController.addListener(() {
// 检查是否滚动到顶部(因为使用了reverse: true,顶部实际是最大滚动位置) // 检查是否滚动到顶部(因为使用了reverse: true,顶部实际是最大滚动位置)
if (scrollController.position.pixels >= scrollController.position.maxScrollExtent - 100) { if (scrollController.position.pixels >=
scrollController.position.maxScrollExtent - 100) {
// 当距离顶部100像素时触发加载更多 // 当距离顶部100像素时触发加载更多
if (hasMoreMessages.value && !isLoadingMore.value) { if (hasMoreMessages.value && !isLoadingMore.value) {
Logger.d(TAG, '触发加载更多历史消息'); Logger.d(TAG, '触发加载更多历史消息');
@ -430,7 +434,8 @@ class AgentController extends GetxController {
Logger.i(TAG, '手动触发加载更多历史消息'); Logger.i(TAG, '手动触发加载更多历史消息');
loadMoreMessages(); loadMoreMessages();
} else { } else {
Logger.w(TAG, '无法加载更多:hasMoreMessages=${hasMoreMessages.value}, isLoadingMore=${isLoadingMore.value}'); Logger.w(TAG,
'无法加载更多:hasMoreMessages=${hasMoreMessages.value}, isLoadingMore=${isLoadingMore.value}');
} }
} }
@ -979,7 +984,8 @@ class AgentController extends GetxController {
permissionType: Permission.microphone, permissionType: Permission.microphone,
permissionName: 'microphonePermission'.tr, // 麦克风 permissionName: 'microphonePermission'.tr, // 麦克风
explanationText: 'voiceRecognitionPermissionText'.tr, // 语音识别功能需要访问麦克风。 explanationText: 'voiceRecognitionPermissionText'.tr, // 语音识别功能需要访问麦克风。
permanentDenialText: 'permissionDeniedText'.tr, // 您已永久拒绝麦克风权限,请前往系统设置开启权限。 permanentDenialText:
'permissionDeniedText'.tr, // 您已永久拒绝麦克风权限,请前往系统设置开启权限。
settingsButtonText: 'goToSettings'.tr, // 去设置 settingsButtonText: 'goToSettings'.tr, // 去设置
cancelButtonText: 'cancel'.tr, // 取消 cancelButtonText: 'cancel'.tr, // 取消
); );
@ -1005,13 +1011,14 @@ class AgentController extends GetxController {
// 开始语音输入 // 开始语音输入
Future<void> startVoiceInput() async { Future<void> startVoiceInput() async {
if (isListening.value) return; if (isListening.value) return;
print('startVoiceInput'.tr); // 开始语音输入 print('开始语音输入'); // 开始语音输入
final hasPermission = await PermissionUtil.instance.requestPermission( final hasPermission = await PermissionUtil.instance.requestPermission(
// context: Get.context!, // context: Get.context!,
permissionType: Permission.microphone, permissionType: Permission.microphone,
permissionName: 'microphonePermission'.tr, // 麦克风 permissionName: 'microphonePermission'.tr, // 麦克风
explanationText: 'voiceTranslationPermissionText'.tr, // 语音识别与翻译功能需要访问麦克风。 explanationText: 'voiceTranslationPermissionText'.tr, // 语音识别与翻译功能需要访问麦克风。
permanentDenialText: 'permissionDeniedText'.tr, // 您已永久拒绝麦克风权限,请前往系统设置开启权限。 permanentDenialText:
'permissionDeniedText'.tr, // 您已永久拒绝麦克风权限,请前往系统设置开启权限。
settingsButtonText: 'goToSettings'.tr, // 去设置 settingsButtonText: 'goToSettings'.tr, // 去设置
cancelButtonText: 'cancel'.tr, // 取消 cancelButtonText: 'cancel'.tr, // 取消
); );
@ -1020,11 +1027,11 @@ class AgentController extends GetxController {
return; return;
} }
try { // try {
await AgentService.startConversation();
} catch (e) { // } catch (e) {
Logger.e(TAG, '启动语音输入失败: $e'); // Logger.e(TAG, '启动语音输入失败: $e');
} // }
} }
// 停止语音输入 // 停止语音输入
@ -1276,15 +1283,15 @@ class AgentController extends GetxController {
isPhoneCallMode.value = true; isPhoneCallMode.value = true;
// 如果当前是文本模式,切换到语音模式???? // 如果当前是文本模式,切换到语音模式????
if (isListening.value) { // if (isListening.value) {
// 检查麦克风权限 // 检查麦克风权限
final hasPermission = await PermissionUtil.instance.requestPermission( final hasPermission = await PermissionUtil.instance.requestPermission(
permissionType: Permission.microphone, permissionType: Permission.microphone,
permissionName: 'microphonePermission'.tr, // 麦克风 permissionName: '麦克风',
explanationText: 'phoneCallPermissionText'.tr, // 打电话功能需要访问麦克风进行语音通话。 explanationText: '打电话功能需要访问麦克风进行语音通话。',
permanentDenialText: 'permissionDeniedText'.tr, // 您已永久拒绝麦克风权限,请前往系统设置开启权限。 permanentDenialText: '您已永久拒绝麦克风权限,请前往系统设置开启权限。',
settingsButtonText: 'goToSettings'.tr, // 去设置 settingsButtonText: '去设置',
cancelButtonText: 'cancel'.tr, // 取消 cancelButtonText: '取消',
); );
if (!hasPermission) { if (!hasPermission) {
@ -1299,9 +1306,9 @@ class AgentController extends GetxController {
// 打电话模式不使用按住说话模式,而是持续语音输入 // 打电话模式不使用按住说话模式,而是持续语音输入
//isPushToTalkMode.value = false; //isPushToTalkMode.value = false;
// 开始语音识别 // 开始语音识别
print('startPhoneCallMode'.tr); // 开始打电话模式,开始语音输入 print("开始打电话模式,开始语音输入");
await AgentService.audioContinue(); await AgentService.startConversation();
} //}
} }
// 结束打电话模式 // 结束打电话模式
@ -1314,7 +1321,7 @@ class AgentController extends GetxController {
// 停止语音输入 // 停止语音输入
//await stopVoiceInput(); //await stopVoiceInput();
// 开始语音识别 // 开始语音识别
await AgentService.audioPause(); await AgentService.stopConversation();
// 恢复之前的输入模式,而不是强制切换到文字模式 // 恢复之前的输入模式,而不是强制切换到文字模式
_restorePreviousInputMode(); _restorePreviousInputMode();
} }
@ -1409,9 +1416,9 @@ class AgentController extends GetxController {
// 设置按住说话状态 // 设置按住说话状态
isPushToTalkActive.value = true; isPushToTalkActive.value = true;
//await AgentService.recognizeCallback();
// 开始语音识别 // 开始语音识别
await AgentService.audioContinue(); await AgentService.startConversation();
// await startVoiceInput(); // await startVoiceInput();
} }
@ -1424,7 +1431,8 @@ class AgentController extends GetxController {
// 清除按住说话状态(但保持按住说话模式) // 清除按住说话状态(但保持按住说话模式)
isPushToTalkActive.value = false; isPushToTalkActive.value = false;
await AgentService.audioPause();
await AgentService.stopConversation();
// 停止语音识别 // 停止语音识别
// await stopVoiceInput(); // await stopVoiceInput();
} }

31
lib/modules/meeting/controllers/meeting_record_controller.dart

@ -66,6 +66,7 @@ class MeetingRecordController extends GetxController
_initializeComponents(); _initializeComponents();
_setupFromArguments(); _setupFromArguments();
generateWaveData(); generateWaveData();
_reinitializeAsrService();
} }
@override @override
@ -74,6 +75,18 @@ class MeetingRecordController extends GetxController
super.onClose(); super.onClose();
} }
// 重新初始化ASR服务以支持新的语言
Future<void> _reinitializeAsrService() async {
try {
// 重新初始化ASR服务
await _asrService.initialize();
Logger.info('ASR服务重新初始化成功');
} catch (e) {
Logger.error('重新初始化ASR服务失败: ${e.toString()}');
}
}
// 请求录音权限 // 请求录音权限
Future<bool> _requestRecordPermission() async { Future<bool> _requestRecordPermission() async {
try { try {
@ -132,6 +145,7 @@ class MeetingRecordController extends GetxController
void _cleanupResources() { void _cleanupResources() {
_timer?.cancel(); _timer?.cancel();
_recognitionSubscription?.cancel(); _recognitionSubscription?.cancel();
_recognitionSubscription = null;
titleEditingController.dispose(); titleEditingController.dispose();
titleFocusNode.dispose(); titleFocusNode.dispose();
if (audioType.value != 0) { if (audioType.value != 0) {
@ -170,7 +184,13 @@ class MeetingRecordController extends GetxController
} }
/// Toggles visibility of transcribed text content /// Toggles visibility of transcribed text content
void toggleTextDisplay() => showTextContent.toggle(); Future<void> toggleTextDisplay() async {
final recognitionStream = await _asrService.recognizeCallback();
_recognitionSubscription?.cancel();
_recognitionSubscription =
recognitionStream.listen(_handleRecognitionEvent);
showTextContent.toggle();
}
/// Toggles important mark status /// Toggles important mark status
void toggleMarked() => isMarked.toggle(); void toggleMarked() => isMarked.toggle();
@ -211,10 +231,7 @@ class MeetingRecordController extends GetxController
// Initialize speech recognition // Initialize speech recognition
final recognitionStream = final recognitionStream =
await _asrService.startContinuousRecognition(_audioSourceType); await _asrService.startContinuousRecognition(_audioSourceType);
_recognitionSubscription?.cancel();
_recognitionSubscription =
recognitionStream.listen(_handleRecognitionEvent);
_asrService.setContinue();
// Prepare storage directory // Prepare storage directory
final appDir = await appDocDir; final appDir = await appDocDir;
final dir = Directory("${appDir.path}/MeetingAudio/"); final dir = Directory("${appDir.path}/MeetingAudio/");
@ -263,9 +280,7 @@ class MeetingRecordController extends GetxController
Future<void> _resumeRecording() async { Future<void> _resumeRecording() async {
final recognitionStream = final recognitionStream =
await _asrService.startContinuousRecognition(_audioSourceType); await _asrService.startContinuousRecognition(_audioSourceType);
_recognitionSubscription?.cancel();
_recognitionSubscription =
recognitionStream.listen(_handleRecognitionEvent);
startTimer(); startTimer();
} }

44
lib/modules/settings/views/settings_view.dart

@ -567,28 +567,28 @@ class SettingsView extends GetView<SettingsController> {
// color: isDarkMode // color: isDarkMode
// ? Colors.white.withOpacity(0.1) // ? Colors.white.withOpacity(0.1)
// : Colors.grey[200]), // : Colors.grey[200]),
// // BLE测试 // BLE测试
// _buildSimpleNavigationSetting( _buildSimpleNavigationSetting(
// title: 'opus解码测试', title: 'opus解码测试',
// subtitle: '测试opus解码', subtitle: '测试opus解码',
// icon: Icons.bluetooth_searching, icon: Icons.bluetooth_searching,
// iconBgColor: isDarkMode iconBgColor: isDarkMode
// ? Colors.green[900]!.withOpacity(0.3) ? Colors.green[900]!.withOpacity(0.3)
// : Colors.green[100]!, : Colors.green[100]!,
// iconColor: iconColor:
// isDarkMode ? Colors.green[300]! : Colors.green[600]!, isDarkMode ? Colors.green[300]! : Colors.green[600]!,
// titleColor: isDarkMode ? Colors.white : null, titleColor: isDarkMode ? Colors.white : null,
// subtitleColor: isDarkMode ? Colors.white70 : null, subtitleColor: isDarkMode ? Colors.white70 : null,
// onTap: () { onTap: () {
// Get.toNamed(Routes.opusTest); Get.toNamed(Routes.opusTest);
// }, },
// isDarkMode: isDarkMode, isDarkMode: isDarkMode,
// ), ),
// Divider( Divider(
// height: 1, height: 1,
// color: isDarkMode color: isDarkMode
// ? Colors.white.withOpacity(0.1) ? Colors.white.withOpacity(0.1)
// : Colors.grey[200]), : Colors.grey[200]),
// BLE测试 // BLE测试
// _buildSimpleNavigationSetting( // _buildSimpleNavigationSetting(

8
lib/modules/speech_test/controllers/speech_test_controller.dart

@ -76,7 +76,10 @@ class SpeechTestController extends GetxController {
// 监听TTS事件 // 监听TTS事件
_setupTtsEventListener(); _setupTtsEventListener();
// 获取识别事件流 - 这启动了异步识别过程
final stream = await _asrService.recognizeCallback();
// 设置事件监听 - 异步处理识别结果
_setupAsrEventListener(stream);
isInitialized.value = true; isInitialized.value = true;
ttsStatus.value = '准备就绪'; ttsStatus.value = '准备就绪';
recognitionStatus.value = '准备就绪'; recognitionStatus.value = '准备就绪';
@ -346,9 +349,6 @@ class SpeechTestController extends GetxController {
// 获取识别事件流 - 这启动了异步识别过程 // 获取识别事件流 - 这启动了异步识别过程
final stream = final stream =
await _asrService.startContinuousRecognition(_audioSourceType); await _asrService.startContinuousRecognition(_audioSourceType);
// 设置事件监听 - 异步处理识别结果
_setupAsrEventListener(stream);
} catch (e) { } catch (e) {
recognitionStatus.value = '启动连续识别异常: $e'; recognitionStatus.value = '启动连续识别异常: $e';
Logger.error('启动连续识别异常: ${e.toString()}'); Logger.error('启动连续识别异常: ${e.toString()}');

34
lib/modules/translation/controllers/translation_controller.dart

@ -122,7 +122,9 @@ class TranslationController extends GetxController {
// 监听识别结果 // 监听识别结果
if (activeSpeaker.value != 0) { if (activeSpeaker.value != 0) {
isPlayback = false; isPlayback = false;
_asrService.setContinue(); final recognitionStream =
await _asrService.startContinuousRecognition(_audioSourceType);
lastActiveSpeaker.value = id; lastActiveSpeaker.value = id;
} else { } else {
if (fTFTranslationResult != '' && !isPlayback) { if (fTFTranslationResult != '' && !isPlayback) {
@ -134,7 +136,7 @@ class TranslationController extends GetxController {
} else if ((translationHistory.isEmpty || } else if ((translationHistory.isEmpty ||
!translationHistory.last.isIntermediate) && !translationHistory.last.isIntermediate) &&
currentMode.value == 'faceToFace') { currentMode.value == 'faceToFace') {
_asrService.setPause(); _asrService.stopContinuousRecognition();
} }
} }
} }
@ -305,6 +307,12 @@ class TranslationController extends GetxController {
await _asrService.initialize( await _asrService.initialize(
supportedLanguages: asrSupportedLanguages, supportedLanguages: asrSupportedLanguages,
); );
// 获取识别事件流 - 这启动了异步识别过程
var recognitionStream = await _asrService.recognizeCallback();
// 监听识别结果
_recognitionSubscription?.cancel();
_recognitionSubscription =
recognitionStream.listen(_handleRecognitionEvent);
final bool storage = await _requestStoragePermission(); final bool storage = await _requestStoragePermission();
if (!storage) { if (!storage) {
Logger.d("Permission", "未授予存储权限,无法提供录音功能"); Logger.d("Permission", "未授予存储权限,无法提供录音功能");
@ -325,6 +333,7 @@ class TranslationController extends GetxController {
void onClose() { void onClose() {
stopRecognition(); stopRecognition();
_recognitionSubscription?.cancel(); _recognitionSubscription?.cancel();
_recognitionSubscription = null;
_voiceInteractionSubscription?.cancel(); _voiceInteractionSubscription?.cancel();
_translationDebounceTimer?.cancel(); _translationDebounceTimer?.cancel();
scrollController.dispose(); scrollController.dispose();
@ -425,22 +434,18 @@ class TranslationController extends GetxController {
_audioSourceType = false; _audioSourceType = false;
isTtsEnabled.value = true; isTtsEnabled.value = true;
} }
final recognitionStream =
await _asrService.startContinuousRecognition(_audioSourceType);
if (currentMode.value == 'faceToFace') { if (currentMode.value == 'faceToFace') {
_asrService.setPause(); _asrService.stopContinuousRecognition();
} else { } else {
await _ttsService.setAudioOutputDevice(0); await _ttsService.setAudioOutputDevice(0);
_asrService.setContinue(); final recognitionStream =
await _asrService.startContinuousRecognition(_audioSourceType);
} }
isRecognizing.value = true; isRecognizing.value = true;
if (isRecording.value && isRecognizing.value) { if (isRecording.value && isRecognizing.value) {
startRecording(); startRecording();
} }
// 监听识别结果
_recognitionSubscription?.cancel();
_recognitionSubscription =
recognitionStream.listen(_handleRecognitionEvent);
} catch (e) { } catch (e) {
isRecognizing.value = false; isRecognizing.value = false;
Logger.error('启动语音识别失败: ${e.toString()}'); Logger.error('启动语音识别失败: ${e.toString()}');
@ -473,10 +478,9 @@ class TranslationController extends GetxController {
if (!isRecognizing.value) return; if (!isRecognizing.value) return;
try { try {
restoreOriginalAudioState();
await _asrService.stopContinuousRecognition(); await _asrService.stopContinuousRecognition();
_recognitionSubscription?.cancel();
_recognitionSubscription = null;
isRecognizing.value = false; isRecognizing.value = false;
// 如果当前有中间结果,将其作为最终结果处理 // 如果当前有中间结果,将其作为最终结果处理
@ -523,7 +527,7 @@ class TranslationController extends GetxController {
); );
// 添加新项后滚动到底部 // 添加新项后滚动到底部
if (currentMode.value == 'faceToFace' && activeSpeaker.value == 0) { if (currentMode.value == 'faceToFace' && activeSpeaker.value == 0) {
_asrService.setPause(); _asrService.stopContinuousRecognition();
} else { } else {
translationHistory.add(newItem); translationHistory.add(newItem);
} }
@ -563,7 +567,7 @@ class TranslationController extends GetxController {
_scrollToBottom(); _scrollToBottom();
saveTranslationHistory(); saveTranslationHistory();
if (activeSpeaker.value == 0 && currentMode.value == 'faceToFace') { if (activeSpeaker.value == 0 && currentMode.value == 'faceToFace') {
_asrService.setPause(); _asrService.stopContinuousRecognition();
} }
// 使用项目的时间戳进行翻译 // 使用项目的时间戳进行翻译
translateText(text, translationHistory.last.timestamp, isFinal: true); translateText(text, translationHistory.last.timestamp, isFinal: true);
@ -582,7 +586,7 @@ class TranslationController extends GetxController {
isIntermediate: false, isIntermediate: false,
); );
if (currentMode.value == 'faceToFace' && activeSpeaker.value == 0) { if (currentMode.value == 'faceToFace' && activeSpeaker.value == 0) {
_asrService.setPause(); _asrService.stopContinuousRecognition();
} //else { } //else {
translationHistory.add(newItem); translationHistory.add(newItem);
translationHistory.refresh(); translationHistory.refresh();

276
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt

@ -58,7 +58,7 @@ object AgentService : CoroutineScope {
private val listeners = Collections.synchronizedSet(mutableSetOf<AgentServiceListener>()) private val listeners = Collections.synchronizedSet(mutableSetOf<AgentServiceListener>())
// Azure服务 // Azure服务
var azureAsrHelper: AzureAsrHelper? = null private var azureAsrHelper: AzureAsrHelper? = null
// 使用BytedanceTTS作为唯一的TTS实现 // 使用BytedanceTTS作为唯一的TTS实现
private var ttsService: AzureTtsHelper? = null private var ttsService: AzureTtsHelper? = null
@ -103,6 +103,7 @@ object AgentService : CoroutineScope {
// AI流生成相关 // AI流生成相关
private var currentAiJob: Job? = null private var currentAiJob: Job? = null
private var currsessionId: String = "" private var currsessionId: String = ""
// 空闲检测相关 // 空闲检测相关
private var idleCheckJob: Job? = null private var idleCheckJob: Job? = null
private val maxIdleSeconds = 10 // 最大空闲秒数 private val maxIdleSeconds = 10 // 最大空闲秒数
@ -111,7 +112,8 @@ object AgentService : CoroutineScope {
private var lastInterruptTime = 0L private var lastInterruptTime = 0L
private val INTERRUPT_DEBOUNCE_MS = 300L private val INTERRUPT_DEBOUNCE_MS = 300L
private var audioSourceType= AzureAsrHelper.AudioSourceType.MICROPHONE var audioSourceType = AzureAsrHelper.AudioSourceType.MICROPHONE
// 语气词过滤列表(中英文) // 语气词过滤列表(中英文)
private val FILLER_WORDS = setOf( private val FILLER_WORDS = setOf(
// 中文语气词 // 中文语气词
@ -122,6 +124,7 @@ object AgentService : CoroutineScope {
"um", "uh", "ah", "oh", "eh", "er", "hmm", "huh", "wow", "um", "uh", "ah", "oh", "eh", "er", "hmm", "huh", "wow",
"well", "like", "you know", "i mean", "so", "actually" "well", "like", "you know", "i mean", "so", "actually"
) )
/** /**
* 初始化系统提示词 * 初始化系统提示词
*/ */
@ -142,6 +145,7 @@ object AgentService : CoroutineScope {
你与用户的交流风格应始终温暖、亲切、充满同理心,随时表达关怀与鼓励,让用户感受到真诚的陪伴与支持。 你与用户的交流风格应始终温暖、亲切、充满同理心,随时表达关怀与鼓励,让用户感受到真诚的陪伴与支持。
""".trimIndent() """.trimIndent()
} }
/** /**
* 初始化 * 初始化
* @param appContext 应用上下文 * @param appContext 应用上下文
@ -149,7 +153,11 @@ object AgentService : CoroutineScope {
* @param config 配置参数,包含所需的所有API密钥和设置 * @param config 配置参数,包含所需的所有API密钥和设置
* @return 是否初始化成功 * @return 是否初始化成功
*/ */
fun initialize(appContext: Context, config: Map<String, Any>, isExternalActive: Boolean): Boolean { fun initialize(
appContext: Context,
config: Map<String, Any>,
isExternalActive: Boolean
): Boolean {
if (isInitialized) return true if (isInitialized) return true
try { try {
@ -169,12 +177,23 @@ object AgentService : CoroutineScope {
} }
// 初始化Azure ASR // 初始化Azure ASR
azureAsrHelper = AzureAsrHelper(context).apply { azureAsrHelper = AzureAsrHelper(context).apply {
initialize( // 使用协程处理异步初始化
CoroutineScope(Dispatchers.IO).launch {
val asrInitSuccess = initialize(
subscriptionKey = config["azureSpeechKey"]?.toString() ?: "", subscriptionKey = config["azureSpeechKey"]?.toString() ?: "",
region = config["azureSpeechRegion"]?.toString() ?: "", region = config["azureSpeechRegion"]?.toString() ?: "",
audioSourceType = if (isExternalActive) AzureAsrHelper.AudioSourceType.EXTERNAL audioSourceType = if (isExternalActive) AzureAsrHelper.AudioSourceType.EXTERNAL
else AzureAsrHelper.AudioSourceType.MICROPHONE else AzureAsrHelper.AudioSourceType.MICROPHONE
) )
if (asrInitSuccess) {
withContext(Dispatchers.Main) {
recognizeCallback()
}
} else {
Log.w(TAG, "Azure ASR初始化失败")
}
}
} }
// 初始化TTS服务 // 初始化TTS服务
@ -189,6 +208,8 @@ object AgentService : CoroutineScope {
config["mcpServer"]?.toString() ?: "" config["mcpServer"]?.toString() ?: ""
) )
// 加载最近的聊天记录 // 加载最近的聊天记录
loadChatHistory() loadChatHistory()
@ -274,45 +295,55 @@ object AgentService : CoroutineScope {
restartIdleCheck() restartIdleCheck()
sendEvent("tts_started", mapOf("status" to "started")) sendEvent("tts_started", mapOf("status" to "started"))
} }
TtsEventType.SYNTHESIS_COMPLETED -> { TtsEventType.SYNTHESIS_COMPLETED -> {
_isTtsSpeaking.set(false) _isTtsSpeaking.set(false)
restartIdleCheck() restartIdleCheck()
sendEvent("tts_completed", mapOf("status" to "completed")) sendEvent("tts_completed", mapOf("status" to "completed"))
} }
TtsEventType.SYNTHESIS_CANCELED -> { TtsEventType.SYNTHESIS_CANCELED -> {
_isTtsSpeaking.set(false) _isTtsSpeaking.set(false)
restartIdleCheck() restartIdleCheck()
sendEvent("tts_canceled", mapOf("status" to "canceled")) sendEvent("tts_canceled", mapOf("status" to "canceled"))
} }
TtsEventType.PLAYBACK_STARTED -> { //开始AI播报内容 TtsEventType.PLAYBACK_STARTED -> { //开始AI播报内容
restartIdleCheck() restartIdleCheck()
sendEvent("playback_started", mapOf("status" to "playback_started")) sendEvent("playback_started", mapOf("status" to "playback_started"))
// 发送AI说话开始广播 // 发送AI说话开始广播
val intent = Intent("com.yunqiinnovation.music_service.AI_SAY_START").apply { val intent =
Intent("com.yunqiinnovation.music_service.AI_SAY_START").apply {
setPackage(context.packageName) setPackage(context.packageName)
} }
context.sendBroadcast(intent) context.sendBroadcast(intent)
} }
TtsEventType.PLAYBACK_COMPLETED -> { //结束AI播报内容 TtsEventType.PLAYBACK_COMPLETED -> { //结束AI播报内容
restartIdleCheck() restartIdleCheck()
sendEvent("playback_completed", mapOf("status" to "playback_completed")) sendEvent("playback_completed", mapOf("status" to "playback_completed"))
// 发送AI说话开始广播 // 发送AI说话开始广播
val intent = Intent("com.yunqiinnovation.music_service.AI_SAY_END").apply { val intent =
Intent("com.yunqiinnovation.music_service.AI_SAY_END").apply {
setPackage(context.packageName) setPackage(context.packageName)
} }
context.sendBroadcast(intent) context.sendBroadcast(intent)
} }
TtsEventType.ERROR -> { TtsEventType.ERROR -> {
_isTtsSpeaking.set(false) _isTtsSpeaking.set(false)
restartIdleCheck() restartIdleCheck()
val params = event.params val params = event.params
val code = params["errorCode"] as? String ?: "UNKNOWN_ERROR" val code = params["errorCode"] as? String ?: "UNKNOWN_ERROR"
val message = params["errorMessage"] as? String ?: "未知错误" val message = params["errorMessage"] as? String ?: "未知错误"
sendEvent("error", mapOf( sendEvent(
"error", mapOf(
"code" to "TTS_ERROR", "code" to "TTS_ERROR",
"message" to "语音合成错误: $code, $message" "message" to "语音合成错误: $code, $message"
)) )
)
} }
else -> { else -> {
// 处理其他类型的事件 // 处理其他类型的事件
} }
@ -359,8 +390,11 @@ object AgentService : CoroutineScope {
delay(maxIdleSeconds * 1000L) delay(maxIdleSeconds * 1000L)
if (isActive && isRecognitionActive && !hasSpeechDetected && !isTtsSpeaking && !isAiStreaming) { if (isActive && isRecognitionActive && !hasSpeechDetected && !isTtsSpeaking && !isAiStreaming) {
//stopRecognition() stopRecognition()
sendEvent("auto_stop", mapOf("reason" to "idle_timeout", "seconds" to maxIdleSeconds)) sendEvent(
"auto_stop",
mapOf("reason" to "idle_timeout", "seconds" to maxIdleSeconds)
)
} }
} catch (e: CancellationException) { } catch (e: CancellationException) {
// 协程被取消,正常行为 // 协程被取消,正常行为
@ -396,14 +430,26 @@ object AgentService : CoroutineScope {
Log.d(TAG, "服务未初始化") Log.d(TAG, "服务未初始化")
return false return false
} }
if(!(azureAsrHelper!!.isPaused)&&isExternalActive) Log.d("TAG", "开始语音识别startRecognition?")
{ stopRecognition()
Log.d(TAG, "麦克风音频未暂停,无法设置") BleService.openEncoder()
BleService.closeCodec() // if (isRecognitionActive&&audioSourceType==AzureAsrHelper.AudioSourceType.MICROPHONE){
return false // Log.d(TAG, "开始语音识别stopRecognition")
} // // BleService.closeCodec()
if (isRecognitionActive) return true // stopRecognition()
// }
// if((isExternalActive&&audioSourceType==AzureAsrHelper.AudioSourceType.MICROPHONE)||!isRecognitionActive)
// {
// Log.d(TAG, "开始语音识别BleService.openEncoder")
// BleService.openEncoder()
// }
// if(isRecognitionActive)
// {
// Log.d(TAG, "开始语音识别azureAsrHelper?.startContinuousRecognition(audioSourceType)")
// return true
// }
// 检查录音权限 // 检查录音权限
if (!isExternalActive && !checkRecordAudioPermission()) { if (!isExternalActive && !checkRecordAudioPermission()) {
Log.e(TAG, "需要录音权限") Log.e(TAG, "需要录音权限")
@ -420,7 +466,28 @@ object AgentService : CoroutineScope {
} else { } else {
AzureAsrHelper.AudioSourceType.MICROPHONE AzureAsrHelper.AudioSourceType.MICROPHONE
} }
azureAsrHelper?.startContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { Log.d("TAG", "azureAsrHelper?.startContinuousRecognition")
azureAsrHelper?.startContinuousRecognition(audioSourceType)
return true
} catch (e: Exception) {
_isRecognitionActive.set(false)
Log.e(TAG, "启动语音识别失败: ${e.message}")
sendEvent(
"error", mapOf(
"code" to "RECOGNITION_START_ERROR",
"message" to e.message.toString()
)
)
return false
}
}
fun recognizeCallback(): Boolean {
try {
Log.d(TAG, "recognizeCallback")
val success = azureAsrHelper?.setupEventListeners(object :
AzureAsrHelper.ContinuousRecognizeCallback {
override fun onRecognizing(recognizing: String, detectedLanguage: String) { override fun onRecognizing(recognizing: String, detectedLanguage: String) {
if (recognizing.isNotEmpty()) { if (recognizing.isNotEmpty()) {
// 检测到语音,更新状态 // 检测到语音,更新状态
@ -431,10 +498,12 @@ object AgentService : CoroutineScope {
restartIdleCheck() restartIdleCheck()
} }
sendEvent("recognizing", mapOf( sendEvent(
"recognizing", mapOf(
"text" to recognizing, "text" to recognizing,
"language" to detectedLanguage "language" to detectedLanguage
)) )
)
// 如果TTS正在播放或AI正在生成,检查是否需要打断 // 如果TTS正在播放或AI正在生成,检查是否需要打断
if (isTtsSpeaking || isAiStreaming) { if (isTtsSpeaking || isAiStreaming) {
@ -465,10 +534,12 @@ object AgentService : CoroutineScope {
override fun onResult(result: String, detectedLanguage: String) { override fun onResult(result: String, detectedLanguage: String) {
if (result.isNotEmpty()) { if (result.isNotEmpty()) {
sendEvent("recognition_result", mapOf( sendEvent(
"recognition_result", mapOf(
"text" to result, "text" to result,
"language" to detectedLanguage "language" to detectedLanguage
)) )
)
processWithChatApi(result) processWithChatApi(result)
} }
@ -485,14 +556,12 @@ object AgentService : CoroutineScope {
override fun onSessionStarted() { override fun onSessionStarted() {
sendEvent("recognition_started", mapOf("status" to "started")) sendEvent("recognition_started", mapOf("status" to "started"))
startIdleCheck() startIdleCheck()
Log.d(TAG, "onSessionStarted")
audioPlayer?.playAudio(R.raw.start) audioPlayer?.playAudio(R.raw.start)
} }
override fun onSessionStopped() { override fun onSessionStopped() {
sendEvent("recognition_stopped", mapOf("status" to "stopped")) sendEvent("recognition_stopped", mapOf("status" to "stopped"))
_isRecognitionActive.set(false) _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
audioPlayer?.playAudio(R.raw.stop) audioPlayer?.playAudio(R.raw.stop)
@ -501,41 +570,41 @@ object AgentService : CoroutineScope {
override fun onCanceled(reason: String, errorDetails: String) { override fun onCanceled(reason: String, errorDetails: String) {
_isRecognitionActive.set(false) _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
Log.d(TAG, "recognition_canceled:")
recognizeCallback()
BleService.closeCodec() BleService.closeCodec()
sendEvent("recognition_canceled", mapOf( sendEvent(
"recognition_canceled", mapOf(
"reason" to reason, "reason" to reason,
"details" to errorDetails "details" to errorDetails
)) )
)
} }
override fun onError(error: String) { override fun onError(error: String) {
_isRecognitionActive.set(false) _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
recognizeCallback()
BleService.closeCodec() BleService.closeCodec()
Log.e(TAG, "语音识别错误: $error") Log.d(TAG, "语音识别错误: $error")
sendEvent("error", mapOf( sendEvent(
"error", mapOf(
"code" to "RECOGNITION_ERROR", "code" to "RECOGNITION_ERROR",
"message" to error "message" to error
)) )
)
} }
}, audioSourceType) })
return true return true
} catch (e: Exception) { } catch (e: Exception) {
_isRecognitionActive.set(false)
Log.e(TAG, "启动语音识别失败: ${e.message}")
sendEvent("error", mapOf(
"code" to "RECOGNITION_START_ERROR",
"message" to e.message.toString()
))
return false return false
} }
} }
fun pushAudioData(data: ByteArray) { fun pushAudioData(data: ByteArray) {
azureAsrHelper?.pushAudioData(data) azureAsrHelper?.audioStream?.saveAudioDataTo(data)
} }
/** /**
* 停止语音识别 * 停止语音识别
*/ */
@ -547,61 +616,16 @@ object AgentService : CoroutineScope {
BleService.closeCodec() BleService.closeCodec()
_isRecognitionActive.set(false) _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
return
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "停止语音识别异常: ${e.message}") Log.e(TAG, "停止语音识别异常: ${e.message}")
_isRecognitionActive.set(false) _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
return
} }
} }
/**
* 继续音频输入
*/
fun audioContinue() {
try {
if(audioSourceType== AzureAsrHelper.AudioSourceType.EXTERNAL)
{
BleService.closeCodec()
stopRecognition()
startRecognition(false)
}
// if (!isRecognitionActive) {
// }
azureAsrHelper?.audioContinue()
//BleService.closeCodec()
//_isRecognitionActive.set(false)
//stopIdleCheck()
} catch (e: Exception) {
Log.e(TAG, "暂停音频输入异常: ${e.message}")
// _isRecognitionActive.set(false)
//stopIdleCheck()
}
}
/**
* 暂停音频输入
*/
fun audioPause() {
if (!isRecognitionActive) return
try {
azureAsrHelper?.audioPause()
Log.d(TAG, "暂停音频输入")
BleService.closeCodec()
//_isRecognitionActive.set(false)
//stopIdleCheck()
} catch (e: Exception) {
Log.e(TAG, "暂停音频输入异常: ${e.message}")
// _isRecognitionActive.set(false)
//stopIdleCheck()
}
}
/** /**
* 打断当前响应 * 打断当前响应
* 停止TTS播放和AI流输出 * 停止TTS播放和AI流输出
@ -617,10 +641,12 @@ object AgentService : CoroutineScope {
// 记录打断耗时 // 记录打断耗时
val duration = System.currentTimeMillis() - startTime val duration = System.currentTimeMillis() - startTime
sendEvent("response_interrupted", mapOf( sendEvent(
"response_interrupted", mapOf(
"status" to "interrupted", "status" to "interrupted",
"duration_ms" to duration "duration_ms" to duration
)) )
)
} }
} }
@ -889,10 +915,12 @@ object AgentService : CoroutineScope {
try { try {
// audioPlayer?.playAudio(R.raw.calling, true) // audioPlayer?.playAudio(R.raw.calling, true)
val name = call.getString("name") val name = call.getString("name")
sendEvent("function_call", mapOf( sendEvent(
"function_call", mapOf(
"name" to name, "name" to name,
"arguments" to call.toString() "arguments" to call.toString()
)) )
)
if (name == "exit_interaction") { if (name == "exit_interaction") {
stopRecognition() stopRecognition()
} }
@ -901,7 +929,11 @@ object AgentService : CoroutineScope {
} }
} }
override fun onFunctionCallResult(sessionid:String,functionCall: JSONObject, functionCallResult: JSONObject) { override fun onFunctionCallResult(
sessionid: String,
functionCall: JSONObject,
functionCallResult: JSONObject
) {
try { try {
audioPlayer?.stopAudio() audioPlayer?.stopAudio()
val name = functionCall.get("name") as String; val name = functionCall.get("name") as String;
@ -917,7 +949,11 @@ object AgentService : CoroutineScope {
) )
) )
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "liwei--------------- AI Call onFunctionCallResult 异常", e) Log.e(
TAG,
"liwei--------------- AI Call onFunctionCallResult 异常",
e
)
} }
} }
} }
@ -926,10 +962,12 @@ object AgentService : CoroutineScope {
} catch (e: Exception) { } catch (e: Exception) {
val errorType = if (hasImage) "AI_IMAGE_PROCESS_ERROR" else "AI_PROCESS_ERROR" val errorType = if (hasImage) "AI_IMAGE_PROCESS_ERROR" else "AI_PROCESS_ERROR"
Log.e(TAG, "liwei--------------- AI处理出错", e) Log.e(TAG, "liwei--------------- AI处理出错", e)
sendEvent("error", mapOf( sendEvent(
"error", mapOf(
"code" to errorType, "code" to errorType,
"message" to e.message.toString() "message" to e.message.toString()
)) )
)
// 确保状态被重置 // 确保状态被重置
_isAiStreaming.set(false) _isAiStreaming.set(false)
@ -1019,7 +1057,12 @@ object AgentService : CoroutineScope {
/** /**
* 保存聊天消息 * 保存聊天消息
*/ */
private fun saveChatMessage(userMessage: String, assistantMessage: String,aiMetadata : String,userMetadata : String) { private fun saveChatMessage(
userMessage: String,
assistantMessage: String,
aiMetadata: String,
userMetadata: String
) {
// 显式指定IO调度器,数据库操作应在后台线程执行,即使父协程在Main上下文中 // 显式指定IO调度器,数据库操作应在后台线程执行,即使父协程在Main上下文中
launch(Dispatchers.IO) { launch(Dispatchers.IO) {
try { try {
@ -1167,7 +1210,12 @@ object AgentService : CoroutineScope {
/** /**
* 处理图片输入 * 处理图片输入
*/ */
fun processImageInput(imagePath: String, imageUrl: String,text: String = "", speakResponse: Boolean = false): Boolean { fun processImageInput(
imagePath: String,
imageUrl: String,
text: String = "",
speakResponse: Boolean = false
): Boolean {
if (!isInitialized) { if (!isInitialized) {
Log.e(TAG, "服务未初始化") Log.e(TAG, "服务未初始化")
sendEvent("error", mapOf("code" to "NOT_INITIALIZED", "message" to "服务未初始化")) sendEvent("error", mapOf("code" to "NOT_INITIALIZED", "message" to "服务未初始化"))
@ -1181,10 +1229,12 @@ object AgentService : CoroutineScope {
} }
// 通知开始处理图片 // 通知开始处理图片
sendEvent("image_processing", mapOf( sendEvent(
"image_processing", mapOf(
"status" to "processing", "status" to "processing",
"imagePath" to imagePath "imagePath" to imagePath
)) )
)
// 处理图片 // 处理图片
launch { launch {
@ -1204,19 +1254,23 @@ object AgentService : CoroutineScope {
val displayText = text.ifEmpty { "[图片]" } val displayText = text.ifEmpty { "[图片]" }
// 通知图片准备完成 // 通知图片准备完成
sendEvent("image_ready", mapOf( sendEvent(
"image_ready", mapOf(
"status" to "ready", "status" to "ready",
"imagePath" to imagePath "imagePath" to imagePath
)) )
)
Log.d(TAG, "liwei--------- 图像Mesage: ${userMessage}") Log.d(TAG, "liwei--------- 图像Mesage: ${userMessage}")
// 处理消息并传入存储路径 // 处理消息并传入存储路径
processWithChatApiInternal(userMessage, displayText, speakResponse, true, imagePath) processWithChatApiInternal(userMessage, displayText, speakResponse, true, imagePath)
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "处理图片失败: ${e.message}") Log.e(TAG, "处理图片失败: ${e.message}")
sendEvent("error", mapOf( sendEvent(
"error", mapOf(
"code" to "IMAGE_PROCESSING_ERROR", "code" to "IMAGE_PROCESSING_ERROR",
"message" to e.message.toString() "message" to e.message.toString()
)) )
)
} }
} }
return true return true
@ -1422,7 +1476,10 @@ object AgentService : CoroutineScope {
/* /*
* 自动播放音乐 * 自动播放音乐
* */ * */
fun autoHandleFcunCallResult(toolname:String,functionCallResult: JSONObject): Map<String,Any>{ fun autoHandleFcunCallResult(
toolname: String,
functionCallResult: JSONObject
): Map<String, Any> {
println("协议工具返回数据 检查是否存在卡片! functionCallResult: ${functionCallResult}") println("协议工具返回数据 检查是否存在卡片! functionCallResult: ${functionCallResult}")
try { try {
val contentStr = functionCallResult.optString("context") val contentStr = functionCallResult.optString("context")
@ -1538,7 +1595,11 @@ object AgentService : CoroutineScope {
fun processMusicPlayList(songs: List<Map<String, Any>>) { fun processMusicPlayList(songs: List<Map<String, Any>>) {
// 如果 playlist 中有有效的歌曲,开始播放 // 如果 playlist 中有有效的歌曲,开始播放
if (songs.isNotEmpty()) { if (songs.isNotEmpty()) {
MusicServiceStarter.startServiceWithPlaylist(context, command = "playlist", songs = songs) MusicServiceStarter.startServiceWithPlaylist(
context,
command = "playlist",
songs = songs
)
} else { } else {
Log.e(TAG, "音乐列表为空,未启动播放服务") Log.e(TAG, "音乐列表为空,未启动播放服务")
} }
@ -1555,7 +1616,12 @@ object AgentService : CoroutineScope {
fun processNavigation(start: String, end: String) { fun processNavigation(start: String, end: String) {
// 如果 playlist 中有有效的歌曲,开始播放 // 如果 playlist 中有有效的歌曲,开始播放
if (!start.isNullOrEmpty() && !end.isNullOrEmpty()) { if (!start.isNullOrEmpty() && !end.isNullOrEmpty()) {
NavigationServiceHelper.startNavigation(context,"start", startpos = start, endpos = end) NavigationServiceHelper.startNavigation(
context,
"start",
startpos = start,
endpos = end
)
} else { } else {
Log.i(TAG, "启动导航服务失败") Log.i(TAG, "启动导航服务失败")
} }

56
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentServicePlugin.kt

@ -15,7 +15,8 @@ import kotlinx.coroutines.launch
import android.content.Intent import android.content.Intent
import android.os.Build import android.os.Build
class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.StreamHandler, AgentServiceListener { class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.StreamHandler,
AgentServiceListener {
private val TAG = "AgentServicePlugin" private val TAG = "AgentServicePlugin"
private lateinit var context: Context private lateinit var context: Context
private lateinit var methodChannel: MethodChannel private lateinit var methodChannel: MethodChannel
@ -24,10 +25,14 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) {
context = flutterPluginBinding.applicationContext context = flutterPluginBinding.applicationContext
methodChannel = MethodChannel(flutterPluginBinding.binaryMessenger, "com.yunqiinnovation.agent_service") methodChannel =
MethodChannel(flutterPluginBinding.binaryMessenger, "com.yunqiinnovation.agent_service")
methodChannel.setMethodCallHandler(this) methodChannel.setMethodCallHandler(this)
eventChannel = EventChannel(flutterPluginBinding.binaryMessenger, "com.yunqiinnovation.agent_service/events") eventChannel = EventChannel(
flutterPluginBinding.binaryMessenger,
"com.yunqiinnovation.agent_service/events"
)
eventChannel.setStreamHandler(this) eventChannel.setStreamHandler(this)
// 添加监听器 // 添加监听器
@ -52,12 +57,16 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
val intent = Intent(context, BleMediaService::class.java) val intent = Intent(context, BleMediaService::class.java)
// 将参数添加到Intent // 将参数添加到Intent
config["azureSpeechKey"]?.toString()?.let { intent.putExtra("azureSpeechKey", it) } config["azureSpeechKey"]?.toString()
config["azureSpeechRegion"]?.toString()?.let { intent.putExtra("azureSpeechRegion", it) } ?.let { intent.putExtra("azureSpeechKey", it) }
config["azureSpeechRegion"]?.toString()
?.let { intent.putExtra("azureSpeechRegion", it) }
config["openaiApiKey"]?.toString()?.let { intent.putExtra("openaiApiKey", it) } config["openaiApiKey"]?.toString()?.let { intent.putExtra("openaiApiKey", it) }
config["openaiBaseUrl"]?.toString()?.let { intent.putExtra("openaiBaseUrl", it) } config["openaiBaseUrl"]?.toString()
?.let { intent.putExtra("openaiBaseUrl", it) }
config["openaiModel"]?.toString()?.let { intent.putExtra("openaiModel", it) } config["openaiModel"]?.toString()?.let { intent.putExtra("openaiModel", it) }
config["openaivisionModel"]?.toString()?.let { intent.putExtra("openaivisionModel", it) } config["openaivisionModel"]?.toString()
?.let { intent.putExtra("openaivisionModel", it) }
config["systemPrompt"]?.toString()?.let { intent.putExtra("systemPrompt", it) } config["systemPrompt"]?.toString()?.let { intent.putExtra("systemPrompt", it) }
config["mcpServer"]?.toString()?.let { intent.putExtra("mcpServer", it) } config["mcpServer"]?.toString()?.let { intent.putExtra("mcpServer", it) }
config["volcanoAppId"]?.toString()?.let { intent.putExtra("volcanoAppId", it) } config["volcanoAppId"]?.toString()?.let { intent.putExtra("volcanoAppId", it) }
@ -76,6 +85,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("SERVICE_START_ERROR", "启动AgentService失败: ${e.message}", null) result.error("SERVICE_START_ERROR", "启动AgentService失败: ${e.message}", null)
} }
} }
"stopAgentService" -> { "stopAgentService" -> {
try { try {
val intent = Intent(context, BleMediaService::class.java) val intent = Intent(context, BleMediaService::class.java)
@ -86,6 +96,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("SERVICE_STOP_ERROR", "停止AgentService失败: ${e.message}", null) result.error("SERVICE_STOP_ERROR", "停止AgentService失败: ${e.message}", null)
} }
} }
"startConversation" -> { "startConversation" -> {
try { try {
val isStarted = AgentService.startRecognition(false) val isStarted = AgentService.startRecognition(false)
@ -95,6 +106,11 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("START_ERROR", "启动对话失败: ${e.message}", null) result.error("START_ERROR", "启动对话失败: ${e.message}", null)
} }
} }
"recognizeCallback" -> {
val isStarted = AgentService.recognizeCallback()
result.success(isStarted)
}
"stopConversation" -> { "stopConversation" -> {
try { try {
AgentService.stopRecognition() AgentService.stopRecognition()
@ -104,24 +120,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("STOP_ERROR", "停止对话失败: ${e.message}", null) result.error("STOP_ERROR", "停止对话失败: ${e.message}", null)
} }
} }
"audioContinue" -> {
try {
AgentService.audioContinue()
result.success(true)
} catch (e: Exception) {
Log.e(TAG, "audioContinue失败", e)
result.error("STOP_ERROR", "audioContinue失败: ${e.message}", null)
}
}
"audioPause" -> {
try {
AgentService.audioPause()
result.success(true)
} catch (e: Exception) {
Log.e(TAG, "audioContinue失败", e)
result.error("STOP_ERROR", "audioContinue失败: ${e.message}", null)
}
}
"processTextInput" -> { "processTextInput" -> {
val text = call.argument<String>("text") val text = call.argument<String>("text")
if (text.isNullOrEmpty()) { if (text.isNullOrEmpty()) {
@ -143,6 +142,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("TEXT_PROCESS_ERROR", "处理文本输入失败: ${e.message}", null) result.error("TEXT_PROCESS_ERROR", "处理文本输入失败: ${e.message}", null)
} }
} }
"processImageInput" -> { "processImageInput" -> {
val imagePath = call.argument<String>("imagePath") val imagePath = call.argument<String>("imagePath")
if (imagePath.isNullOrEmpty()) { if (imagePath.isNullOrEmpty()) {
@ -171,6 +171,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("IMAGE_PROCESS_ERROR", "处理图片输入失败: ${e.message}", null) result.error("IMAGE_PROCESS_ERROR", "处理图片输入失败: ${e.message}", null)
} }
} }
"speakText" -> { "speakText" -> {
val text = call.argument<String>("text") val text = call.argument<String>("text")
if (text.isNullOrEmpty()) { if (text.isNullOrEmpty()) {
@ -186,6 +187,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("TTS_ERROR", "语音合成失败: ${e.message}", null) result.error("TTS_ERROR", "语音合成失败: ${e.message}", null)
} }
} }
"stopTts" -> { "stopTts" -> {
try { try {
AgentService.stopTts() AgentService.stopTts()
@ -195,6 +197,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("STOP_TTS_ERROR", "停止语音合成失败: ${e.message}", null) result.error("STOP_TTS_ERROR", "停止语音合成失败: ${e.message}", null)
} }
} }
"stopAiSteam" -> { "stopAiSteam" -> {
try { try {
AgentService.stopAiStream() AgentService.stopAiStream()
@ -215,6 +218,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("CLEAR_HISTORY_ERROR", "清除聊天历史失败: ${e.message}", null) result.error("CLEAR_HISTORY_ERROR", "清除聊天历史失败: ${e.message}", null)
} }
} }
"interruptCurrentResponse" -> { "interruptCurrentResponse" -> {
try { try {
AgentService.interruptCurrentResponse() AgentService.interruptCurrentResponse()
@ -224,6 +228,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("INTERRUPT_ERROR", "中断响应失败: ${e.message}", null) result.error("INTERRUPT_ERROR", "中断响应失败: ${e.message}", null)
} }
} }
"dispose" -> { "dispose" -> {
try { try {
AgentService.dispose() AgentService.dispose()
@ -233,6 +238,7 @@ class AgentServicePlugin : FlutterPlugin, MethodCallHandler, EventChannel.Stream
result.error("DISPOSE_ERROR", "释放资源失败: ${e.message}", null) result.error("DISPOSE_ERROR", "释放资源失败: ${e.message}", null)
} }
} }
else -> { else -> {
result.notImplemented() result.notImplemented()
} }

18
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt

@ -65,7 +65,7 @@ object BleAgent : BleService.Callback, AgentServiceListener {
Log.e(TAG, "语音代理未初始化") Log.e(TAG, "语音代理未初始化")
return false return false
} }
Log.d(TAG, "手动启动语音识别: ")
return AgentService.startRecognition(true) return AgentService.startRecognition(true)
} }
@ -73,7 +73,12 @@ object BleAgent : BleService.Callback, AgentServiceListener {
* 停止语音识别 * 停止语音识别
*/ */
fun stopListening() { fun stopListening() {
if (!isInitialized) {
Log.e(TAG, "语音代理未初始化")
return
}
AgentService.stopRecognition() AgentService.stopRecognition()
return
} }
/** /**
@ -121,7 +126,7 @@ object BleAgent : BleService.Callback, AgentServiceListener {
} else if (state == BleConst.STATE_DISCONNECTED) { } else if (state == BleConst.STATE_DISCONNECTED) {
Log.d(TAG, "Ble设备已断开,停止语音服务") Log.d(TAG, "Ble设备已断开,停止语音服务")
// 设备断开连接,停止语音服务 // 设备断开连接,停止语音服务
// stopListening() stopListening()
} }
} }
@ -136,17 +141,18 @@ object BleAgent : BleService.Callback, AgentServiceListener {
* 在收到唤醒信号时启动语音识别 * 在收到唤醒信号时启动语音识别
*/ */
override fun onWakeupSignalReceived() { override fun onWakeupSignalReceived() {
Log.d(TAG, "收到唤醒信号,启动语音识别") Log.d(TAG, "收到唤醒信号,启动语音识别${AgentService.isRecognitionActive}")
// 停止当前TTS,避免冲突 // 停止当前TTS,避免冲突
AgentService.interruptCurrentResponse() AgentService.interruptCurrentResponse()
// AgentService.speakText("我在!") // AgentService.speakText("我在!")
// 启动语音识别 // 启动语音识别
if (AgentService.isRecognitionActive&&AgentService.azureAsrHelper!!.isPaused) {
// 如果已经在识别,则重新启动 // 如果已经在识别,则重新启动
stopListening() //stopListening()
}
//stopListening()
startListening() startListening()
} }

53
local_plugins/agent_service/lib/agent_service.dart

@ -248,16 +248,16 @@ class AgentService {
} }
} }
/// 停止对话 /// 设备识别器
/// ///
/// 停止语音识别 /// 启动语音识别,开始监听用户语音输入
/// 返回是否成功停止对话 /// 返回是否成功开始对话
static Future<bool> stopConversation() async { static Future<bool> recognizeCallback() async {
try { try {
final bool result = await _channel.invokeMethod('stopConversation'); final bool result = await _channel.invokeMethod('recognizeCallback');
return result; return result;
} on PlatformException catch (e) { } on PlatformException catch (e) {
throw AgentServiceException(e.code, e.message ?? '停止对话失败', e.details); throw AgentServiceException(e.code, e.message ?? '设置识别器失败', e.details);
} }
} }
@ -265,27 +265,40 @@ class AgentService {
/// ///
/// 停止语音识别 /// 停止语音识别
/// 返回是否成功停止对话 /// 返回是否成功停止对话
static Future<bool> audioContinue() async { static Future<bool> stopConversation() async {
try { try {
final bool result = await _channel.invokeMethod('audioContinue'); final bool result = await _channel.invokeMethod('stopConversation');
return result; return result;
} on PlatformException catch (e) { } on PlatformException catch (e) {
throw AgentServiceException(e.code, e.message ?? '停止对话失败', e.details); throw AgentServiceException(e.code, e.message ?? '停止对话失败', e.details);
} }
} }
/// 停止对话 // /// 停止对话
/// // ///
/// 停止语音识别 // /// 停止语音识别
/// 返回是否成功停止对话 // /// 返回是否成功停止对话
static Future<bool> audioPause() async { // static Future<bool> audioContinue() async {
try { // try {
final bool result = await _channel.invokeMethod('audioPause'); // final bool result = await _channel.invokeMethod('audioContinue');
return result; // return result;
} on PlatformException catch (e) { // } on PlatformException catch (e) {
throw AgentServiceException(e.code, e.message ?? '停止对话失败', e.details); // throw AgentServiceException(e.code, e.message ?? '停止对话失败', e.details);
} // }
} // }
// /// 停止对话
// ///
// /// 停止语音识别
// /// 返回是否成功停止对话
// static Future<bool> audioPause() async {
// try {
// final bool result = await _channel.invokeMethod('audioPause');
// return result;
// } on PlatformException catch (e) {
// throw AgentServiceException(e.code, e.message ?? '停止对话失败', e.details);
// }
// }
/// 处理文本输入 /// 处理文本输入
/// ///

614
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt

@ -22,6 +22,7 @@ import com.yunqiinnovation.azure_speech.tools.RecordFile
import com.microsoft.cognitiveservices.speech.audio.AudioConfig import com.microsoft.cognitiveservices.speech.audio.AudioConfig
import com.microsoft.cognitiveservices.speech.audio.AudioInputStream import com.microsoft.cognitiveservices.speech.audio.AudioInputStream
import com.microsoft.cognitiveservices.speech.audio.PushAudioInputStream import com.microsoft.cognitiveservices.speech.audio.PushAudioInputStream
import java.util.concurrent.atomic.AtomicBoolean
/** /**
@ -52,8 +53,7 @@ class AzureAsrHelper(private val context: Context) {
var audioSourceType = AudioSourceType.MICROPHONE var audioSourceType = AudioSourceType.MICROPHONE
// 音频处理 // 音频处理
private var audioStream: AudioStream? = null var audioStream: AudioStream? = null
private var externalAudioStream: ExternalAudioPullStream? = null
// 音频模式管理 // 音频模式管理
private var audioManager: AudioManager? = null private var audioManager: AudioManager? = null
@ -61,7 +61,6 @@ class AzureAsrHelper(private val context: Context) {
// 录音文件处理 // 录音文件处理
var recordfile: RecordFile? = null var recordfile: RecordFile? = null
var isPaused = true
/** /**
* 音频来源类型 * 音频来源类型
@ -74,29 +73,6 @@ class AzureAsrHelper(private val context: Context) {
EXTERNAL EXTERNAL
} }
/**
* 获取设备支持的最佳音频格式
* 优先选择16000Hz,若不支持则降级到8000Hz
*/
private fun getOptimalAudioFormat(): AudioStreamFormat {
// 支持的采样率列表(按优先级排序)
val supportedSampleRates = intArrayOf(16000, 8000, 11025, 22050, 44100)
// 查找设备支持的最佳采样率
val sampleRate = supportedSampleRates.firstOrNull { rate ->
val bufferSize = AudioRecord.getMinBufferSize(
rate,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT
)
bufferSize > 0 // 返回正值表示支持
} ?: 16000 // 默认回退值
Log.i(tag, "使用采样率: ${sampleRate}Hz")
// 创建对应的音频格式
return AudioStreamFormat.getWaveFormatPCM(sampleRate.toLong(), 16, 1)
}
/** /**
* 初始化Azure语音服务 * 初始化Azure语音服务
@ -122,7 +98,6 @@ class AzureAsrHelper(private val context: Context) {
// 释放之前的资源 // 释放之前的资源
dispose() dispose()
// 初始化音频管理器 // 初始化音频管理器
audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
originalAudioMode = audioManager?.mode ?: AudioManager.MODE_NORMAL originalAudioMode = audioManager?.mode ?: AudioManager.MODE_NORMAL
@ -169,47 +144,30 @@ class AzureAsrHelper(private val context: Context) {
} }
// 录音文件类 // 录音文件类
recordfile = RecordFile; recordfile = RecordFile;
//进入界面手机麦克风就被占用,导致其他app无法使用麦克风,在开始录音再去申请
// // 创建识别器
// val setupSuccess = setupRecognizer()
// if (setupSuccess) {
// // 优化:初始化完成后进行预热
// warmupRecognizer()
// }
return true return true
} catch (e: Exception) { } catch (e: Exception) {
Log.e(tag, "初始化失败: ${e.message}") Log.e(tag, "初始化失败: ${e.message}")
return false return false
} }
} }
private fun isRecognizerValid(): Boolean {
return recognizer != null
}
/** /**
* 设置识别器 * 设置识别器
*/ */
private fun setupRecognizer(): Boolean { private fun setupRecognizer(): Boolean {
try { try {
// 如果正在进行连续识别,先停止
// 清理旧的识别器 if (isContinuousRecognitionActive) {
recognizer?.close() // 直接停止,不等待结果
recognizer = null recognizer?.stopContinuousRecognitionAsync()
isContinuousRecognitionActive = false
// 设置音频配置
when (audioSourceType) {
AudioSourceType.MICROPHONE -> {
// 总是使用拉流方式并启用回音消除
Log.i(tag, "使用拉流方式并启用回音消除")
setupMicrophoneStream()
} }
setupMicrophoneStream()
AudioSourceType.EXTERNAL -> {
// 改用拉流方式处理外部音频
Log.d(tag, "改用拉流方式处理外部音频")
setupExternalAudioStream()
}
}
//创建识别器 //创建识别器
recognizer = if (isAutoDetectLanguage) { recognizer = if (isAutoDetectLanguage) {
@ -234,22 +192,13 @@ class AzureAsrHelper(private val context: Context) {
private fun setupMicrophoneStream() { private fun setupMicrophoneStream() {
try { try {
Log.d(tag, "设置麦克风流 - 使用拉流方式: }") Log.d(tag, "设置麦克风流 - 使用拉流方式: }")
restoreOriginalAudioState()
if (audioStream == null) { if (audioStream == null) {
// 创建外部音频拉流对象 // 创建外部音频拉流对象
audioStream = AudioStream() audioStream = AudioStream()
audioStream!!.initAudioRecord() audioStream!!.initAudioRecord()
} }
// else
// {
// audioStream.stopCaptureThread()
// audioStream.startCaptureThread()
// }
// 创建麦克风流
// microphoneStream = MicrophoneStream()
// 创建音频配置 - 正确使用fromStreamInput方法,只传入回调
// audioConfig = AudioConfig.fromStreamInput(microphoneStream)
audioConfig = AudioConfig.fromStreamInput(audioStream!!.pushAudioStream) audioConfig = AudioConfig.fromStreamInput(audioStream!!.pushAudioStream)
@ -259,60 +208,6 @@ class AzureAsrHelper(private val context: Context) {
} }
} }
/**
* 预热识别器(减少首次识别延迟)
*/
private fun warmupRecognizer() {
try {
// 创建极短的音频数据进行预热
val warmupData = ByteArray(320) // 10ms 16kHz 单声道
// 模拟静音数据
warmupData.fill(0)
// 如果是外部音频流,进行预热
if (audioSourceType == AudioSourceType.EXTERNAL) {
externalAudioStream?.pushAudio(warmupData)
}
Log.d(tag, "ASR预热完成")
} catch (e: Exception) {
// 预热失败不影响正常使用
Log.d(tag, "ASR预热失败: ${e.message}")
}
}
/**
* 设置外部音频流 - 使用拉流方式
*/
private fun setupExternalAudioStream() {
try {
Log.d(tag, "设置外部音频流 - 使用拉流方式: }")
//disableBluetoothAudio()
// 创建外部音频拉流对象
externalAudioStream = ExternalAudioPullStream()
// 创建音频配置
audioConfig = AudioConfig.fromStreamInput(externalAudioStream)
} catch (e: Exception) {
Log.e(tag, "设置外部音频流失败: ${e.message}")
externalAudioStream = null
}
}
/**
* 向音频流写入音频数据
* 仅当音频源设置为EXTERNAL时有效
*
* @param data 音频数据字节数组
*/
fun pushAudioData(data: ByteArray) {
if (audioSourceType != AudioSourceType.EXTERNAL) {
return
}
// 使用拉流模式,将数据推入队列
externalAudioStream?.pushAudio(data)
}
/** /**
* 执行一次性语音识别 * 执行一次性语音识别
@ -327,21 +222,18 @@ class AzureAsrHelper(private val context: Context) {
callback.onError("语音服务未初始化") callback.onError("语音服务未初始化")
return return
} }
this.audioSourceType = audioSourceType this.audioSourceType = audioSourceType
// 确保不在连续识别中 // 确保不在连续识别中
if (isContinuousRecognitionActive) { if (isContinuousRecognitionActive) {
stopContinuousRecognition() stopContinuousRecognition()
} }
// 重置识别器
if (!setupRecognizer()) {
callback.onError("重置识别器失败")
return
}
try { try {
// 启动音频处理 // 启动音频处理
startAudioProcessing() audioStream!!.startAudioRecord()
// 执行同步识别 // 执行同步识别
val result = recognizer?.recognizeOnceAsync()?.get() val result = recognizer?.recognizeOnceAsync()?.get()
@ -369,43 +261,43 @@ class AzureAsrHelper(private val context: Context) {
* @return 是否成功开始识别 * @return 是否成功开始识别
*/ */
fun startContinuousRecognition( fun startContinuousRecognition(
callback: ContinuousRecognizeCallback,
audioSourceType: AudioSourceType = AudioSourceType.MICROPHONE audioSourceType: AudioSourceType = AudioSourceType.MICROPHONE
): Boolean { ): Boolean {
if (speechConfig == null) {
callback.onError("语音服务未初始化")
return false
}
Log.d(tag, "startContinuousRecognition:$isContinuousRecognitionActive ")
if (isContinuousRecognitionActive) { if (isContinuousRecognitionActive) {
return true return true
} }
this.audioSourceType = audioSourceType if (!isRecognizerValid()) {
// 重置识别器 Log.w(tag, "识别器已失效,正在重新创建...")
if (!setupRecognizer()) { if (!setupRecognizer()) {
callback.onError("重置识别器失败") Log.e(tag, "重新创建识别器失败")
return false return false
} }
}
this.audioSourceType = audioSourceType
try { try {
// 设置各种事件监听
setupEventListeners(callback)
Log.d(tag, "startContinuousRecognition: ")
// 启动音频处理 // 启动音频处理
//startAudioProcessing() //startAudioProcessing()
// 开始连续识别 // 开始连续识别
// 启动音频处理
audioStream!!.startAudioRecord()
recognizer?.startContinuousRecognitionAsync() recognizer?.startContinuousRecognitionAsync()
audioStream?.startCaptureThread()
isContinuousRecognitionActive = true isContinuousRecognitionActive = true
return true return true
} catch (e: Exception) { } catch (e: Exception) {
stopAudioProcessing() audioStream!!.stopMicrophoneCapture()
isContinuousRecognitionActive = false isContinuousRecognitionActive = false
callback.onError("启动连续识别失败: ${e.message}")
return false return false
} }
} }
@ -413,7 +305,20 @@ class AzureAsrHelper(private val context: Context) {
/** /**
* 设置事件监听器 * 设置事件监听器
*/ */
private fun setupEventListeners(callback: ContinuousRecognizeCallback) { fun setupEventListeners(callback: ContinuousRecognizeCallback): Boolean {
Log.d(tag, "设置ssssss监听器:${speechConfig} ")
if (speechConfig == null) {
callback.onError("语音服务未初始化")
return false
}
// 重设识别器
if (!setupRecognizer()) {
Log.d(tag, "初始化失败: ")
return false
}
try {
Log.d(tag, "设置ssssss监听器: ")
// 优化:识别中事件 - 添加文本长度检查 // 优化:识别中事件 - 添加文本长度检查
recognizer?.recognizing?.addEventListener( recognizer?.recognizing?.addEventListener(
EventHandler<SpeechRecognitionEventArgs> { _, event -> EventHandler<SpeechRecognitionEventArgs> { _, event ->
@ -462,7 +367,7 @@ class AzureAsrHelper(private val context: Context) {
callback.onSessionStopped() callback.onSessionStopped()
isContinuousRecognitionActive = false isContinuousRecognitionActive = false
stopAudioProcessing() // stopAudioProcessing()
} }
} }
) )
@ -473,8 +378,15 @@ class AzureAsrHelper(private val context: Context) {
val errorDetails = event.errorDetails ?: "未知错误" val errorDetails = event.errorDetails ?: "未知错误"
val reason = event.reason.toString() val reason = event.reason.toString()
callback.onCanceled(reason, errorDetails) callback.onCanceled(reason, errorDetails)
} }
) )
} catch (e: Exception) {
callback.onError("启动连续识别失败: ${e.message}")
return false
}
return true
} }
@ -494,36 +406,34 @@ class AzureAsrHelper(private val context: Context) {
} }
try { try {
if (recognizer == null) {
Log.w(tag, "识别器为空,重置状态")
isContinuousRecognitionActive = false isContinuousRecognitionActive = false
return true
}
if (audioSourceType == AudioSourceType.EXTERNAL) { if (audioSourceType == AudioSourceType.EXTERNAL) {
pushAudioData(ByteArray(0)) //pushAudioData(ByteArray(0))
} }
Log.d(tag, "停止连续语音识别: ")
// 直接停止连续识别(SDK内部已是异步操作) // 直接停止连续识别(SDK内部已是异步操作)
recognizer?.stopContinuousRecognitionAsync()?.get(1000, TimeUnit.MILLISECONDS) recognizer?.stopContinuousRecognitionAsync()?.get(1000, TimeUnit.MILLISECONDS)
// 停止音频处理 // 停止音频处理
stopAudioProcessing() audioStream!!.stopMicrophoneCapture()
// 会话结束事件会设置isContinuousRecognitionActive = false // 会话结束事件会设置isContinuousRecognitionActive = false
return true return true
} catch (e: Exception) { } catch (e: Exception) {
// 强制重置状态 // 强制重置状态
isContinuousRecognitionActive = false isContinuousRecognitionActive = false
Log.e(tag, "停止连续识别失败: ${e.message}") Log.e(tag, "停止连续识别失败: ${e.message}")
recognizer?.stopContinuousRecognitionAsync()
// 停止音频处理 // 停止音频处理
stopAudioProcessing() audioStream!!.stopMicrophoneCapture()
// 尝试强制关闭识别器 // 尝试强制关闭识别器
try { // try {
recognizer?.close() // recognizer?.close()
recognizer = null // recognizer = null
} catch (ex: Exception) { // } catch (ex: Exception) {
Log.e(tag, "关闭识别器失败: ${ex.message}") // Log.e(tag, "关闭识别器失败: ${ex.message}")
} // }
return false return false
} }
@ -545,7 +455,7 @@ class AzureAsrHelper(private val context: Context) {
recognizer?.stopContinuousRecognitionAsync() recognizer?.stopContinuousRecognitionAsync()
isContinuousRecognitionActive = false isContinuousRecognitionActive = false
} }
// recognizer?.stopContinuousRecognitionAsync()?.get(1000, TimeUnit.MILLISECONDS)
// 停止音频处理 // 停止音频处理
stopAudioProcessing() stopAudioProcessing()
// 停止录音 // 停止录音
@ -566,7 +476,7 @@ class AzureAsrHelper(private val context: Context) {
// 确保状态被重置 // 确保状态被重置
isContinuousRecognitionActive = false isContinuousRecognitionActive = false
//microphoneStream = null //microphoneStream = null
externalAudioStream = null
audioConfig = null audioConfig = null
recognizer = null recognizer = null
speechConfig = null speechConfig = null
@ -575,7 +485,7 @@ class AzureAsrHelper(private val context: Context) {
isContinuousRecognitionActive = false isContinuousRecognitionActive = false
//microphoneStream = null //microphoneStream = null
externalAudioStream = null
audioConfig = null audioConfig = null
recognizer = null recognizer = null
speechConfig = null speechConfig = null
@ -583,30 +493,13 @@ class AzureAsrHelper(private val context: Context) {
} }
// 音频处理相关方法 // 音频处理相关方法
/**
* 启动音频处理
*/
private fun startAudioProcessing() {
when (audioSourceType) {
AudioSourceType.MICROPHONE -> {
// 拉流模式不需要额外启动,SDK会自动拉取数据
// 无需执行任何操作
}
AudioSourceType.EXTERNAL -> {
// 外部音频数据模式下不需要启动处理,等待外部调用pushAudioData
}
}
}
/** /**
* 停止音频处理 * 停止音频处理
*/ */
private fun stopAudioProcessing() { private fun stopAudioProcessing() {
audioStream?.let { audioStream?.let {
try { try {
it.stopMicrophoneCapture() Log.e(tag, "停止音频处理: }")
it.releaseAudioResources() it.releaseAudioResources()
it.pushAudioStream?.close() it.pushAudioStream?.close()
@ -617,49 +510,9 @@ class AzureAsrHelper(private val context: Context) {
e.printStackTrace() e.printStackTrace()
} }
} // 关闭音频流(根据实际实现可能需要) } // 关闭音频流(根据实际实现可能需要)
externalAudioStream?.let {
try {
it.close()
externalAudioStream = null
} catch (e: Exception) {
Log.e(tag, "关闭外部音频流失败: ${e.message}")
e.printStackTrace()
}
}
}
/*
暂停音频输入
*/
fun audioPause() {
Log.d(tag, "关闭麦克风stopAudioPause: ")
// 清理旧的识别器
isPaused = true
// 如果正在进行连续识别,先停止
if (isContinuousRecognitionActive) {
// 直接停止,不等待结果
recognizer?.stopContinuousRecognitionAsync()
isContinuousRecognitionActive = false
}
}
/*
继续音频输入
*/
fun audioContinue() {
Log.d(tag, "继续麦克风stopAudioPause: ")
// 如果正在进行连续识别,先停止
if (!isContinuousRecognitionActive) {
// 直接停止,不等待结果
recognizer?.startContinuousRecognitionAsync()
isContinuousRecognitionActive = true
} }
// 开始连续识别
isPaused = false
}
/** /**
* 禁用蓝牙音频功能,切换回正常音频模式 * 禁用蓝牙音频功能,切换回正常音频模式
@ -753,21 +606,22 @@ class AzureAsrHelper(private val context: Context) {
recordfile!!.closeFile(isSave) recordfile!!.closeFile(isSave)
} }
/** /**
* 麦克风流 - 拉流模式 * 麦克风流 - 拉流模式
* 实现PullAudioInputStreamCallback,为Azure SDK提供音频数据 * 实现PullAudioInputStreamCallback,为Azure SDK提供音频数据
*/ */
inner class AudioStream { inner class AudioStream {
private val bufferSize = 4096 // 可根据需要调整
private var echoCanceler: AcousticEchoCanceler? = null
private var noiseSuppressor: NoiseSuppressor? = null
private var automaticGainControl: AutomaticGainControl? = null
var audioRecord: AudioRecord? = null var audioRecord: AudioRecord? = null
private var captureThread: Thread? = null
var pushAudioStream: PushAudioInputStream? = null var pushAudioStream: PushAudioInputStream? = null
// 新增:用于异步写入的队列和线程
private val writeQueue = LinkedBlockingQueue<ByteArray>()
private val isRunning = AtomicBoolean(false)// 控制线程是否继续存在
private val isWriting = AtomicBoolean(false) // 控制是否应该写入数据
private var writeThread: Thread? = null
// 音频配置 // 音频配置
@ -775,12 +629,130 @@ class AzureAsrHelper(private val context: Context) {
private val audioFormat = AudioFormat.ENCODING_PCM_16BIT private val audioFormat = AudioFormat.ENCODING_PCM_16BIT
/** /**
* 开始麦克风捕获 * 获取设备支持的最佳音频格式
* 优先选择16000Hz,若不支持则降级到8000Hz
*/
private fun getOptimalAudioFormat(): AudioStreamFormat {
// 支持的采样率列表(按优先级排序)
val supportedSampleRates = intArrayOf(16000, 8000, 11025, 22050, 44100)
// 查找设备支持的最佳采样率
val sampleRate = supportedSampleRates.firstOrNull { rate ->
val bufferSize = AudioRecord.getMinBufferSize(
rate,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT
)
bufferSize > 0 // 返回正值表示支持
} ?: 16000 // 默认回退值
Log.i(tag, "使用采样率: ${sampleRate}Hz")
// 创建对应的音频格式
return AudioStreamFormat.getWaveFormatPCM(sampleRate.toLong(), 16, 1)
}
/**
* 初始化
*/ */
fun initAudioRecord() { fun initAudioRecord() {
val format = getOptimalAudioFormat() val format = getOptimalAudioFormat()
pushAudioStream = AudioInputStream.createPushStream(format) pushAudioStream = AudioInputStream.createPushStream(format)
isRunning.set(true)
startWriteThread() // 再启动数据读取线程
}
private fun startWriteThread() {
writeThread = Thread {
try {
while (isRunning.get()) {
// 等待录音信号
if (!isWriting.get()) {
Thread.sleep(10) // 短暂休眠避免空转
continue
}
var data: ByteArray? = null
var bytesToWrite = 0
// 情况1:正在录制中 -> 直接从AudioRecord读取
if (audioRecord?.recordingState == AudioRecord.RECORDSTATE_RECORDING) {
data = ByteArray(bufferSize)
val bytesRead = audioRecord?.read(data, 0, bufferSize) ?: -1
when {
bytesRead < 0 -> {
Log.e("tag", "读取音频失败,错误码: $bytesRead")
continue
}
bytesRead == 0 -> continue // 无数据可读
else -> bytesToWrite = bytesRead // 有效数据
}
}
// 情况2:不在录制但队列有数据 -> 从队列获取
else if (writeQueue.isNotEmpty()) {
data = writeQueue.poll()
Log.d("tag", "写入数据: ${ data?.size}")
bytesToWrite = data?.size ?: 0
}
// 确保有有效数据再写入
if (data != null && bytesToWrite > 0) {
// 处理实际读取长度 < bufferSize 的情况
val finalData =
if (bytesToWrite < data.size) data.copyOf(bytesToWrite) else data
try {
pushAudioStream?.write(finalData)
recordfile?.saveAudioDataToWav(finalData)
} catch (e: Exception) {
Log.e("tag", "写入失败: ${e.message}")
}
} else {
Thread.yield() // 避免空转消耗CPU
}
}
} catch (e: Exception) {
Log.e("tag", "写入线程异常: ${e.stackTraceToString()}")
} finally {
Log.d("tag", "音频写入线程退出")
writeQueue.clear()
}
}.apply {
name = "AudioWriteThread"
start()
}
}
/**
* 外部音频输入
*/
fun saveAudioDataTo(buffer: ByteArray) {
if (audioSourceType == AudioSourceType.MICROPHONE) return
// 放入队列,由写线程写入
writeQueue.offer(buffer.copyOf())
}
fun startAudioRecord() {
isWriting.set(true)
when (audioSourceType) {
AudioSourceType.MICROPHONE -> runMicrophoneCapture()
AudioSourceType.EXTERNAL -> runExternalCapture()
}
}
private fun runMicrophoneCapture() {
try {
val supportedSampleRates = intArrayOf(16000, 8000, 11025, 22050, 44100) val supportedSampleRates = intArrayOf(16000, 8000, 11025, 22050, 44100)
// 查找设备支持的最佳采样率 // 查找设备支持的最佳采样率
val sampleRate = supportedSampleRates.firstOrNull { rate -> val sampleRate = supportedSampleRates.firstOrNull { rate ->
@ -824,89 +796,38 @@ class AzureAsrHelper(private val context: Context) {
) )
} }
// 检查初始化状态 // 检查初始化状态
if (audioRecord?.state != AudioRecord.STATE_INITIALIZED) { if (audioRecord?.state != AudioRecord.STATE_INITIALIZED) {
throw IllegalStateException("AudioRecord初始化失败") throw IllegalStateException("AudioRecord初始化失败")
} }
startCaptureThread() // 再启动数据读取线程
}
// 3. 单独封装线程启动逻辑
fun startCaptureThread() {
try {
// 确保之前的线程已停止
stopCaptureThread()
// 开始录音 // 4. 开始录音
audioRecord?.startRecording() audioRecord?.startRecording()
Log.d("TAG", "录音开始,采样率: $sampleRate Hz")
captureThread = Thread({
android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_URGENT_AUDIO)
val supportedSampleRates = intArrayOf(16000, 8000, 11025, 22050, 44100)
var bufferSize = 0
val sampleRate = supportedSampleRates.firstOrNull { rate ->
bufferSize = AudioRecord.getMinBufferSize(
rate,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT
)
bufferSize > 0
} ?: 16000
val buffer = ByteArray(bufferSize)
try {
while (!Thread.currentThread().isInterrupted &&
audioRecord?.recordingState == AudioRecord.RECORDSTATE_RECORDING) {
if (!isPaused) {
val bytesRead = audioRecord?.read(buffer, 0, buffer.size) ?: 0
if (bytesRead > 0) {
// 安全地写入Azure流
try {
pushAudioStream?.write(buffer)
recordfile?.saveAudioDataToWav(buffer)
} catch (e: Exception) { } catch (e: Exception) {
Log.w(tag, "写入音频流失败: ${e.message}") Log.e("TAG", "音频捕获异常: ${e.message}")
}
} else if (bytesRead < 0) {
Log.w(tag, "AudioRecord读取错误: $bytesRead")
break
}
} else {
Thread.sleep(10) // 暂停时短暂休眠
}
} }
} catch (e: InterruptedException) {
Log.d(tag, "音频捕获线程被中断")
} catch (e: Exception) {
Log.e(tag, "音频捕获异常: ${e.message}", e)
} }
}, "AudioCaptureThread")
captureThread?.start() private fun runExternalCapture() {
} catch (e: Exception) { Log.d("TAG", "外部音频捕获启动")
Log.e(tag, "启动音频捕获线程失败: ${e.message}", e) try { // TODO: 实现外部音频源捕获逻辑
throw e // 停止录音
} if (audioRecord?.recordingState == AudioRecord.STATE_INITIALIZED) {
Log.d("TAG", "外部音频捕获启动 释放audioRecord")
audioRecord?.stop()
// 释放录音实例
audioRecord?.release()
audioRecord = null
} }
} catch (e: Exception) {
Log.e("TAG", "外部音频捕获异常: ${e.message}")
} finally {
private fun stopCaptureThread() {
captureThread?.let { thread ->
if (thread.isAlive) {
thread.interrupt()
try {
thread.join(500) // 等待最多500ms
} catch (e: InterruptedException) {
Log.w(tag, "等待音频线程结束被中断")
}
}
} }
captureThread = null
} }
/** /**
@ -914,20 +835,25 @@ class AzureAsrHelper(private val context: Context) {
*/ */
fun stopMicrophoneCapture() { fun stopMicrophoneCapture() {
try { try {
if (captureThread != null) { if (!isWriting.get()) return
// 中断并等待捕获线程结束
captureThread?.interrupt()
captureThread?.join(300) // 最多等待300ms
// 停止录音并释放AudioRecord资源
audioRecord?.stop() // 必须先停止再释放 isWriting.set(false)
isPaused= true
// 停止录音
if (audioRecord?.recordingState == AudioRecord.RECORDSTATE_RECORDING) {
audioRecord?.stop()
} }
// 停止录音并释放AudioRecord资源
// 释放录音实例
audioRecord?.release()
audioRecord = null
} catch (e: Exception) { } catch (e: Exception) {
Log.e("AudioConfig", "Error releasing resources: ${e.message}") Log.e("AudioConfig", "Error releasing resources: ${e.message}")
} finally { } finally {
writeQueue.clear()
// 确保恢复原始音频状态 // 确保恢复原始音频状态
// restoreOriginalAudioState() // restoreOriginalAudioState()
} }
@ -939,32 +865,19 @@ class AzureAsrHelper(private val context: Context) {
*/ */
fun releaseAudioResources() { fun releaseAudioResources() {
try { try {
isPaused= true if (!isWriting.get()) return
isWriting.set(false)
isRunning.set(false)
// 停止录音 // 停止录音
if (audioRecord?.recordingState == AudioRecord.RECORDSTATE_RECORDING) { if (audioRecord?.recordingState == AudioRecord.RECORDSTATE_RECORDING) {
audioRecord?.stop() audioRecord?.stop()
} }
// // 释放回音消除器 // 中断并等待捕获线程结束
// echoCanceler?.let { writeThread?.interrupt()
// it.enabled = false writeThread?.join(300) // 最多等待300ms
// it.release()
// echoCanceler = null
// }
// // 释放噪声抑制器
// noiseSuppressor?.let {
// it.enabled = false
// it.release()
// noiseSuppressor = null
// }
// // 释放自动增益控制
// automaticGainControl?.let {
// it.enabled = false
// it.release()
// automaticGainControl = null
// }
// 恢复音频模式 // 恢复音频模式
this@AzureAsrHelper.audioManager?.mode = this@AzureAsrHelper.originalAudioMode this@AzureAsrHelper.audioManager?.mode = this@AzureAsrHelper.originalAudioMode
@ -975,64 +888,13 @@ class AzureAsrHelper(private val context: Context) {
} catch (e: Exception) { } catch (e: Exception) {
Log.e(tag, "释放音频资源失败: ${e.message}") Log.e(tag, "释放音频资源失败: ${e.message}")
e.printStackTrace() e.printStackTrace()
} finally {
writeQueue.clear()
writeThread = null
// 确保恢复原始音频状态
// restoreOriginalAudioState()
} }
} }
}
/**
* 外部音频拉流
* 实现PullAudioInputStreamCallback,将外部推送的音频数据转换为SDK可拉取的形式
*/
private inner class ExternalAudioPullStream : PullAudioInputStreamCallback() {
private val queue: BlockingQueue<ByteArray> = LinkedBlockingQueue()
private var closed = false
/**
* 外部调用:推送音频数据到队列
* @param data 音频数据
*/
fun pushAudio(data: ByteArray) {
// 将音频数据保存成wav格式的音频文件
recordfile?.saveAudioDataToWav(data)
if (!closed) {
queue.offer(data)
}
}
/**
* 修复:外部音频流恢复阻塞读取
* @param buffer SDK提供的缓冲区
* @return 读取的字节数,0表示流结束
*/
override fun read(buffer: ByteArray): Int {
try {
val chunk = queue.take()
// 检查是否是结束标志(空数组)
if (chunk.isEmpty()) {
return 0
}
val toCopy = minOf(chunk.size, buffer.size)
System.arraycopy(chunk, 0, buffer, 0, toCopy)
return toCopy
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
return 0
} catch (e: Exception) {
Log.e(tag, "读取外部音频数据失败: ${e.message}")
return 0
}
}
/**
* SDK调用:关闭流
*/
override fun close() {
closed = true
queue.clear()
}
} }
/** /**

32
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt

@ -255,15 +255,6 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
} }
} }
"setPause" -> {
azureAsrHelper.audioPause();
return
}
"setcontinue" -> {
azureAsrHelper.audioContinue();
return
}
"disableBluetoothAudio" -> { "disableBluetoothAudio" -> {
azureAsrHelper.disableBluetoothAudio(); azureAsrHelper.disableBluetoothAudio();
@ -296,7 +287,16 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
AzureAsrHelper.AudioSourceType.MICROPHONE AzureAsrHelper.AudioSourceType.MICROPHONE
} }
val success = azureAsrHelper.startContinuousRecognition(object : val success = azureAsrHelper.startContinuousRecognition(audioSourceType)
result.success(true)
} catch (e: Exception) {
result.error("START_RECOGNITION_ERROR", e.message, null)
}
}
"recognizeCallback" -> {
val success = azureAsrHelper.setupEventListeners(object :
AzureAsrHelper.ContinuousRecognizeCallback { AzureAsrHelper.ContinuousRecognizeCallback {
override fun onResult(text: String, detectedLanguage: String) { override fun onResult(text: String, detectedLanguage: String) {
sendAsrEvent( sendAsrEvent(
@ -330,6 +330,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
} }
override fun onCanceled(reason: String, errorDetails: String) { override fun onCanceled(reason: String, errorDetails: String) {
sendAsrEvent( sendAsrEvent(
mapOf( mapOf(
"type" to "canceled", "type" to "canceled",
@ -347,11 +348,9 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
) )
) )
} }
}, audioSourceType) })
FileLogger.d(tag, "recognizeCallback: ${success}") //
result.success(success) result.success(success)
} catch (e: Exception) {
result.error("START_RECOGNITION_ERROR", e.message, null)
}
} }
"stopContinuousRecognition" -> { "stopContinuousRecognition" -> {
@ -378,7 +377,8 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
"音频数据不能为空", "音频数据不能为空",
null null
) )
azureAsrHelper.pushAudioData(data)
azureAsrHelper.audioStream?.saveAudioDataTo(data)
result.success(true) result.success(true)
} }
@ -583,7 +583,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
override fun onAudioDataReceived(data: ByteArray) { override fun onAudioDataReceived(data: ByteArray) {
if (azureAsrHelper.audioSourceType == AzureAsrHelper.AudioSourceType.EXTERNAL) { if (azureAsrHelper.audioSourceType == AzureAsrHelper.AudioSourceType.EXTERNAL) {
azureAsrHelper.pushAudioData(data) azureAsrHelper.audioStream?.saveAudioDataTo(data)
} }
// 可选:处理音频数据 // 可选:处理音频数据
} }

1
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/RecordFile.kt

@ -248,6 +248,7 @@ object RecordFile {
} finally { } finally {
fos = null fos = null
currentAudioFile = null currentAudioFile = null
writeThread = null
} }
return false return false
} }

55
local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt

@ -1,5 +1,7 @@
package com.yunqiinnovation.ble_service package com.yunqiinnovation.ble_service
import android.Manifest
import android.annotation.SuppressLint
import android.bluetooth.* import android.bluetooth.*
import android.bluetooth.le.* import android.bluetooth.le.*
import android.content.Context import android.content.Context
@ -22,6 +24,7 @@ import java.util.concurrent.LinkedBlockingQueue
import android.os.Handler import android.os.Handler
import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicBoolean
import android.util.Log import android.util.Log
import androidx.annotation.RequiresPermission
/** /**
* BLE服务类:提供蓝牙低功耗设备的扫描、连接和通信功能 * BLE服务类:提供蓝牙低功耗设备的扫描、连接和通信功能
@ -61,7 +64,7 @@ object BleService {
// 上下文 // 上下文
private lateinit var context: Context private lateinit var context: Context
private val commandQueue: Queue<ByteArray> = LinkedList()
// 蓝牙相关组件 // 蓝牙相关组件
private lateinit var bluetoothManager: BluetoothManager private lateinit var bluetoothManager: BluetoothManager
private var bluetoothAdapter: BluetoothAdapter? = null private var bluetoothAdapter: BluetoothAdapter? = null
@ -83,6 +86,10 @@ object BleService {
// 是否是ota模式 // 是否是ota模式
var isEnterOta = false var isEnterOta = false
// 是否回复指令
var isReply = true
// 是否回复指令码
var cmdReplyType =0xff
// 连接状态 // 连接状态
val connectionState = MutableLiveData(BleConst.STATE_DISCONNECTED) val connectionState = MutableLiveData(BleConst.STATE_DISCONNECTED)
@ -93,9 +100,6 @@ object BleService {
private var opusManager: OpusManager? = null private var opusManager: OpusManager? = null
private val mainHandler = Handler(Looper.getMainLooper()) private val mainHandler = Handler(Looper.getMainLooper())
// ota解码器实例
// var otaManager: OTAManager? = null
// 初始化状态 // 初始化状态
private var isInitialized = false private var isInitialized = false
@ -663,6 +667,7 @@ object BleService {
private fun processAudioData(data: ByteArray) { private fun processAudioData(data: ByteArray) {
try { try {
if (opusManager?.isDecodeStream == true) { if (opusManager?.isDecodeStream == true) {
recordfile?.saveAudioDataToWav(data) recordfile?.saveAudioDataToWav(data)
opusManager?.writeAudioStream(data) opusManager?.writeAudioStream(data)
} else { } else {
@ -677,6 +682,8 @@ object BleService {
/** /**
* 处理设备响应数据 - 帧头为0xBB * 处理设备响应数据 - 帧头为0xBB
*/ */
@SuppressLint("StaticFieldLeak")
@RequiresPermission(Manifest.permission.BLUETOOTH_CONNECT)
private fun processDeviceResponse(data: ByteArray) { private fun processDeviceResponse(data: ByteArray) {
Log.d( Log.d(
TAG, TAG,
@ -741,7 +748,21 @@ object BleService {
} }
}" }"
) )
isReply=true;
cmdReplyType=cmdType;
if(!commandQueue.isEmpty())
{
val nextCmd = commandQueue.poll()
writeChar!!.value=nextCmd
var isSeccess = bluetoothGatt?.writeCharacteristic(writeChar)
if(isSeccess==true)
{
Log.d(
TAG,
"收到设备响应:"
)
}
}
// 根据命令类型处理不同的响应 // 根据命令类型处理不同的响应
when (cmdType) { when (cmdType) {
BleConst.CMD_GET_VERSION -> { BleConst.CMD_GET_VERSION -> {
@ -1005,7 +1026,7 @@ object BleService {
BleConst.CMD_WAKE_UP -> { BleConst.CMD_WAKE_UP -> {
Log.i(TAG, "收到唤醒指令") Log.i(TAG, "收到唤醒指令")
// 接收到唤醒信号,打开编码器 (设备侧) // 接收到唤醒信号,打开编码器 (设备侧)
openEncoder()
notifyWakeupSignalReceived() notifyWakeupSignalReceived()
} }
@ -1316,6 +1337,7 @@ object BleService {
* @param cmdData 命令数据 * @param cmdData 命令数据
* @return 是否成功发送 * @return 是否成功发送
*/ */
@RequiresPermission(Manifest.permission.BLUETOOTH_CONNECT)
private fun sendCommand(cmdType: Byte, cmdData: ByteArray = byteArrayOf()): Boolean { private fun sendCommand(cmdType: Byte, cmdData: ByteArray = byteArrayOf()): Boolean {
if (!checkConn()) { if (!checkConn()) {
Log.e(TAG, "发送命令失败: 设备未连接") Log.e(TAG, "发送命令失败: 设备未连接")
@ -1358,7 +1380,26 @@ object BleService {
// 发送数据 // 发送数据
writeChar!!.value = cmdPacket writeChar!!.value = cmdPacket
return bluetoothGatt?.writeCharacteristic(writeChar) ?: false // 加入队列并触发发送
if(isReply)
{
var isSeccess = bluetoothGatt?.writeCharacteristic(writeChar)
if(isSeccess == true)
{
isReply=false;
cmdType;
return true
}
else
return false
}
else
{
commandQueue.offer(cmdPacket)
}
return true
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "发送命令异常: ${e.message}", e) Log.e(TAG, "发送命令异常: ${e.message}", e)
return false return false

Loading…
Cancel
Save