Browse Source

音频数据抛回flutter层

newdev_shunjiawei
fdp 1 year ago
parent
commit
a42b751ed4
  1. 8
      lib/data/services/asr_service.dart
  2. 72
      lib/data/services/speech_impl/azure_asr_service.dart
  3. 8
      lib/data/services/speech_impl/volcano_asr_api_service.dart
  4. 8
      lib/data/services/speech_impl/volcano_asr_service.dart
  5. 8
      lib/data/services/speech_impl/xunfei_asr_service.dart
  6. 17
      lib/modules/meeting/controllers/meeting_record_controller.dart
  7. 118
      lib/modules/translation/controllers/translation_controller.dart
  8. 17
      lib/modules/translation/views/translation_view.dart
  9. 68
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
  10. 1
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrToAsr.kt
  11. 59
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt
  12. 2
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt
  13. 2
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/SimpleAudioPlayer.kt
  14. 31
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/SimpleAudioReceiver.kt

8
lib/data/services/asr_service.dart

@ -41,7 +41,13 @@ abstract class AsrService {
bool isContinuousRecognitionActive(); bool isContinuousRecognitionActive();
/// 开始录音 /// 开始录音
Future<bool> enableRecord(String filePath); ///
/// [filePath] 录音文件路径
/// [acceptAudioData] 是否接受音频数据回调,默认为 false
Future<bool> enableRecord(String filePath, {bool acceptAudioData = false});
/// 获取音频数据流(如果支持)
Stream<Uint8List>? getAudioDataStream() => null;
/// 暂停录音 /// 暂停录音
Future<bool> pauseRecord(); Future<bool> pauseRecord();

72
lib/data/services/speech_impl/azure_asr_service.dart

@ -21,6 +21,10 @@ class AzureAsrService extends GetxService implements AsrService {
static const MethodChannel _channel = MethodChannel('azure_speech/asr'); static const MethodChannel _channel = MethodChannel('azure_speech/asr');
static const EventChannel _eventChannel = static const EventChannel _eventChannel =
EventChannel('azure_speech/asr_events'); EventChannel('azure_speech/asr_events');
// 音频数据事件通道
static const EventChannel _audioDataEventChannel =
EventChannel('azure_speech/audio_data_events');
// final GetStorage _storage = GetStorage(); // final GetStorage _storage = GetStorage();
bool _isInitialized = false; bool _isInitialized = false;
late final String _subscriptionKey; late final String _subscriptionKey;
@ -43,6 +47,11 @@ class AzureAsrService extends GetxService implements AsrService {
String _latestDetectedLanguage = ''; String _latestDetectedLanguage = '';
String get latestDetectedLanguage => _latestDetectedLanguage; String get latestDetectedLanguage => _latestDetectedLanguage;
// 音频数据事件订阅
StreamSubscription? _audioDataEventSubscription;
// 音频数据流控制器
StreamController<Uint8List>? _audioDataStreamController;
// 当前音频源类型 // 当前音频源类型
AudioSourceType _audioSourceType = AudioSourceType.microphone; AudioSourceType _audioSourceType = AudioSourceType.microphone;
@ -72,6 +81,19 @@ class AzureAsrService extends GetxService implements AsrService {
}, onError: _handleRecognitionError); }, onError: _handleRecognitionError);
} }
/// 设置音频数据事件通道
void _setupAudioDataEventChannel() {
_audioDataEventSubscription?.cancel();
_audioDataEventSubscription =
_audioDataEventChannel.receiveBroadcastStream().listen((event) {
if (event is Map) {
_handleAudioDataEvent(event);
}
}, onError: (error) {
Logger.error('音频数据事件流错误: ${error.toString()}');
});
}
@override @override
Future<bool> initialize({ Future<bool> initialize({
List<String>? supportedLanguages, List<String>? supportedLanguages,
@ -220,6 +242,41 @@ class AzureAsrService extends GetxService implements AsrService {
return _isContinuousRecognitionActive; return _isContinuousRecognitionActive;
} }
/// 处理音频数据事件
void _handleAudioDataEvent(dynamic event) {
if (event is! Map) return;
final Map<dynamic, dynamic> eventMap = event;
final String eventType = eventMap['type'] as String? ?? '';
switch (eventType) {
case 'audioData':
final Uint8List data = eventMap['data'] as Uint8List? ?? Uint8List(0);
final double timestamp =
(eventMap['timestamp'] as num?)?.toDouble() ?? 0.0;
final int size = eventMap['size'] as int? ?? 0;
print('接收到音频数据: 大小=${size}字节, 时间戳=${timestamp}');
Logger.debug('接收到音频数据: 大小=${size}字节, 时间戳=${timestamp}');
// 发送到音频数据流
_audioDataStreamController?.add(data);
// 也可以通过识别事件流发送
_eventStreamController?.add(RecognitionEvent(
type: RecognitionEventType.onAudio,
audio: data,
));
break;
}
}
/// 获取音频数据流
Stream<Uint8List>? getAudioDataStream() {
print("getAudioDataStream");
_audioDataStreamController ??= StreamController<Uint8List>.broadcast();
return _audioDataStreamController?.stream;
}
/// 处理来自原生端的识别事件 /// 处理来自原生端的识别事件
void _handleRecognitionEvent(dynamic event) { void _handleRecognitionEvent(dynamic event) {
if (event is! Map || _eventStreamController == null) return; if (event is! Map || _eventStreamController == null) return;
@ -336,6 +393,14 @@ class AzureAsrService extends GetxService implements AsrService {
await _eventSubscription?.cancel(); await _eventSubscription?.cancel();
_eventSubscription = null; _eventSubscription = null;
// 取消音频数据事件订阅
await _audioDataEventSubscription?.cancel();
_audioDataEventSubscription = null;
// 关闭音频数据流
await _audioDataStreamController?.close();
_audioDataStreamController = null;
// 通知原生端释放资源 // 通知原生端释放资源
await _channel.invokeMethod('dispose'); await _channel.invokeMethod('dispose');
_isInitialized = false; _isInitialized = false;
@ -376,11 +441,16 @@ class AzureAsrService extends GetxService implements AsrService {
} }
@override @override
Future<bool> enableRecord(String filePath) async { Future<bool> enableRecord(String filePath,
{bool acceptAudioData = false}) async {
try { try {
final bool result = await _channel.invokeMethod('enableRecord', { final bool result = await _channel.invokeMethod('enableRecord', {
'filePath': filePath, 'filePath': filePath,
'acceptAudioData': acceptAudioData, // 新增参数
}); });
if (acceptAudioData) {
_setupAudioDataEventChannel();
}
return result; return result;
} catch (e) { } catch (e) {

8
lib/data/services/speech_impl/volcano_asr_api_service.dart

@ -813,7 +813,7 @@ class VolcanoAsrApiService implements AsrService {
} }
@override @override
Future<bool> enableRecord(String filePath) { Future<bool> enableRecord(String filePath, {bool acceptAudioData = false}) {
// TODO: implement enableRecord // TODO: implement enableRecord
throw UnimplementedError(); throw UnimplementedError();
} }
@ -877,4 +877,10 @@ class VolcanoAsrApiService implements AsrService {
// TODO: implement resumeRecord // TODO: implement resumeRecord
throw UnimplementedError(); throw UnimplementedError();
} }
@override
Stream<Uint8List>? getAudioDataStream() {
// TODO: implement getAudioDataStream
throw UnimplementedError();
}
} }

8
lib/data/services/speech_impl/volcano_asr_service.dart

@ -348,7 +348,7 @@ class VolcanoAsrService extends GetxService implements AsrService {
} }
@override @override
Future<bool> enableRecord(String filePath) { Future<bool> enableRecord(String filePath, {bool acceptAudioData = false}) {
// TODO: implement enableRecord // TODO: implement enableRecord
throw UnimplementedError(); throw UnimplementedError();
} }
@ -412,4 +412,10 @@ class VolcanoAsrService extends GetxService implements AsrService {
// TODO: implement resumeRecord // TODO: implement resumeRecord
throw UnimplementedError(); throw UnimplementedError();
} }
@override
Stream<Uint8List>? getAudioDataStream() {
// TODO: implement getAudioDataStream
throw UnimplementedError();
}
} }

8
lib/data/services/speech_impl/xunfei_asr_service.dart

@ -272,7 +272,7 @@ class XunfeiAsrService extends GetxService implements AsrService {
} }
@override @override
Future<bool> enableRecord(String filePath) { Future<bool> enableRecord(String filePath, {bool acceptAudioData = false}) {
// TODO: implement enableRecord // TODO: implement enableRecord
throw UnimplementedError(); throw UnimplementedError();
} }
@ -336,4 +336,10 @@ class XunfeiAsrService extends GetxService implements AsrService {
// TODO: implement resumeRecord // TODO: implement resumeRecord
throw UnimplementedError(); throw UnimplementedError();
} }
@override
Stream<Uint8List>? getAudioDataStream() {
// TODO: implement getAudioDataStream
throw UnimplementedError();
}
} }

17
lib/modules/meeting/controllers/meeting_record_controller.dart

@ -60,6 +60,7 @@ class MeetingRecordController extends GetxController
bool _audioSourceType = false; bool _audioSourceType = false;
late TabController tabController; late TabController tabController;
StreamSubscription<Uint8List>? _audioDataSubscription;
// 添加波形放大倍数变量 // 添加波形放大倍数变量
final double waveAmplifyFactor = 100.0; // 增加波动幅度 final double waveAmplifyFactor = 100.0; // 增加波动幅度
final RxDouble ursorPosition = 0.0.obs; // 当前播放位置 final RxDouble ursorPosition = 0.0.obs; // 当前播放位置
@ -255,7 +256,21 @@ class MeetingRecordController extends GetxController
final fullFileName = "${fileName.value}_$formattedTime"; final fullFileName = "${fileName.value}_$formattedTime";
// Start audio recording // Start audio recording
await _asrService.enableRecord("${dir.path}/$fullFileName.wav"); await _asrService.enableRecord("${dir.path}/$fullFileName.wav",
acceptAudioData: true);
// 监听音频数据流
_audioDataSubscription =
_asrService.getAudioDataStream()?.listen((audioData) {
// 处理音频数据,例如:
// 1. 实时音频可视化
// 2. 音频质量检测
// 3. 发送到其他服务
print('接收到音频数据: ${audioData.length} 字节');
// 处理音频数据的逻辑
_processAudioData(audioData);
});
isRecording.value = true; isRecording.value = true;
// Update state // Update state

118
lib/modules/translation/controllers/translation_controller.dart

@ -65,6 +65,8 @@ class TranslationController extends GetxController {
Timer? _recordTimer; Timer? _recordTimer;
//秒 //秒
int _seconds = 0; int _seconds = 0;
//暂停状态记录计时器是否暂停
bool _isTimerPaused = false;
// 存储相关配置 // 存储相关配置
static const String _historyKey = 'translation_history'; static const String _historyKey = 'translation_history';
static const String _sourceLanguageKey = 'translation_source_language'; static const String _sourceLanguageKey = 'translation_source_language';
@ -92,6 +94,8 @@ class TranslationController extends GetxController {
final isTranslating = false.obs; final isTranslating = false.obs;
final isTtsEnabled = true.obs; final isTtsEnabled = true.obs;
final isRecording = false.obs; final isRecording = false.obs;
final lasyIsRecording = false.obs;
final hasRecordPermission = false.obs;
// 语言相关 // 语言相关
final sourceLanguage = '中文(简体)'.obs; final sourceLanguage = '中文(简体)'.obs;
final targetLanguage = '英语'.obs; final targetLanguage = '英语'.obs;
@ -119,7 +123,6 @@ class TranslationController extends GetxController {
Future<void> changeTranslationMode(String mode) async { Future<void> changeTranslationMode(String mode) async {
currentMode.value = mode; currentMode.value = mode;
currentModeTitle.value = _getModeTitle(mode); currentModeTitle.value = _getModeTitle(mode);
stopAll(); stopAll();
} }
@ -137,12 +140,57 @@ class TranslationController extends GetxController {
} }
} }
/// 暂停计时器
void pauseRecordTimer() {
if (_recordTimer != null && !_isTimerPaused) {
_recordTimer?.cancel();
_recordTimer = null;
_isTimerPaused = true;
Logger.info('录音计时器已暂停');
}
}
/// 恢复计时器
void resumeRecordTimer() {
if (_isTimerPaused) {
_recordTimer = Timer.periodic(Duration(seconds: 1), (_) {
_seconds++;
final minutes = (_seconds ~/ 60).toString().padLeft(2, '0');
final seconds = (_seconds % 60).toString().padLeft(2, '0');
recordDurationText.value = '$minutes:$seconds';
});
_isTimerPaused = false;
Logger.info('录音计时器已恢复');
}
}
/// 重置计时器
void resetRecordTimer() {
_recordTimer?.cancel();
_recordTimer = null;
_seconds = 0;
_isTimerPaused = false;
recordDurationText.value = '00:00';
Logger.info('录音计时器已重置');
}
Future<void> setActiveSpeaker(int id) async { Future<void> setActiveSpeaker(int id) async {
activeSpeaker.value = id; activeSpeaker.value = id;
await _asrService.startContinuousRecognition(_audioSourceType);
// 监听识别结果 // 监听识别结果
if (activeSpeaker.value != 0) { if (activeSpeaker.value != 0) {
isPlayback = false; isPlayback = false;
await _asrService.startContinuousRecognition(_audioSourceType); isRecognizing.value = true;
if (isRecording.value && isRecognizing.value) {
if (lasyIsRecording.value != isRecording.value) {
lasyIsRecording.value = isRecording.value;
await startRecording();
} else {
await _asrService.resumeRecord();
// 恢复计时器
resumeRecordTimer();
}
}
// 开始ASR活跃时长跟踪(面对面模式) // 开始ASR活跃时长跟踪(面对面模式)
_startAsrActiveTracking(); _startAsrActiveTracking();
@ -151,7 +199,11 @@ class TranslationController extends GetxController {
} else { } else {
// 停止ASR活跃时长跟踪(面对面模式) // 停止ASR活跃时长跟踪(面对面模式)
_stopAsrActiveTracking(); _stopAsrActiveTracking();
if (isRecording.value && isRecognizing.value) {
// 暂停计时器
pauseRecordTimer();
await _asrService.pauseRecord();
}
if (fTFTranslationResult != '' && !isPlayback) { if (fTFTranslationResult != '' && !isPlayback) {
isPlayback = true; isPlayback = true;
await _ttsService.setAudioOutputDevice(lastActiveSpeaker.value); await _ttsService.setAudioOutputDevice(lastActiveSpeaker.value);
@ -198,6 +250,9 @@ class TranslationController extends GetxController {
super.onInit(); super.onInit();
//_setupEventListeners(); //_setupEventListeners();
// 预先检查权限,避免在用户交互时阻塞
_preCheckPermissions();
// 初始化统计服务 // 初始化统计服务
_initUsageService(); _initUsageService();
@ -228,6 +283,15 @@ class TranslationController extends GetxController {
// 参考Agent模块的实现模式 // 参考Agent模块的实现模式
} }
/// 预先检查权限
Future<void> _preCheckPermissions() async {
try {
await requestRecordPermission();
} catch (e) {
Logger.e("Permission", "预检查权限失败: $e");
}
}
// 初始化统计服务 // 初始化统计服务
void _initUsageService() { void _initUsageService() {
try { try {
@ -264,16 +328,17 @@ class TranslationController extends GetxController {
_storage.write(_targetLanguageKey, targetLanguage.value); _storage.write(_targetLanguageKey, targetLanguage.value);
} }
Future<bool> _requestRecordPermission() async { Future<bool> requestRecordPermission() async {
try { try {
final bool granted = await PermissionUtil.instance.requestPermission( hasRecordPermission.value =
await PermissionUtil.instance.requestPermission(
permissionType: Permission.microphone, permissionType: Permission.microphone,
permissionName: 'microphonePermission'.tr, permissionName: 'microphonePermission'.tr,
explanationText: 'microphonePermissionExplanation'.tr, explanationText: 'microphonePermissionExplanation'.tr,
permanentDenialText: 'microphonePermissionPermanentDenial'.tr, permanentDenialText: 'microphonePermissionPermanentDenial'.tr,
); );
if (granted) { if (hasRecordPermission.value) {
Logger.d("Permission", "用户授予了录音权限"); Logger.d("Permission", "用户授予了录音权限");
return true; return true;
} else { } else {
@ -539,11 +604,6 @@ class TranslationController extends GetxController {
} }
Future<void> startRecording() async { Future<void> startRecording() async {
final bool storage = await _requestStoragePermission();
if (!storage) {
Logger.d("Permission", "未授予存储权限,无法提供录音功能");
return;
}
//isRecording.value = true; //isRecording.value = true;
_seconds = 0; _seconds = 0;
recordDurationText.value = '00:00'; recordDurationText.value = '00:00';
@ -559,25 +619,29 @@ class TranslationController extends GetxController {
final formattedTime = DateFormat('yyyyMMdd_HHmmss').format(DateTime.now()); final formattedTime = DateFormat('yyyyMMdd_HHmmss').format(DateTime.now());
await _asrService.enableRecord( await _asrService.enableRecord(
"${dir.path}/${currentModeTitle.value.tr}_$formattedTime.wav"); "${dir.path}/${currentModeTitle.value.tr}_$formattedTime.wav");
await _astService.enableRecord( // await _astService.enableRecord(
"${dir.path}/${currentModeTitle.value.tr}_${formattedTime}_mic.wav"); // "${dir.path}/${currentModeTitle.value.tr}_${formattedTime}_mic.wav");
return;
} }
Future<void> stopRecording() async { Future<void> stopRecording() async {
_recordTimer?.cancel(); _recordTimer?.cancel();
_recordTimer = null; _recordTimer = null;
isRecording.value = false; isRecording.value = false;
_isTimerPaused = false;
recordDurationText.value = '00:00'; recordDurationText.value = '00:00';
await _asrService.stopRecord(true); _seconds = 0;
// 停止实际录音逻辑
// ... _asrService.stopRecord(true);
lasyIsRecording.value = false;
Logger.info('录音已停止,计时器已重置');
} }
// 开始语音识别 // 开始语音识别
Future<void> startRecognition() async { Future<void> startRecognition() async {
Logger.info('开始语音识别'); Logger.info('开始语音识别');
// 等待权限请求完成并获取结果 // 等待权限请求完成并获取结果
final bool record = await _requestRecordPermission(); final bool record = await requestRecordPermission();
// 如果权限被拒绝,直接返回 // 如果权限被拒绝,直接返回
if (!record) { if (!record) {
@ -634,18 +698,14 @@ class TranslationController extends GetxController {
isTtsEnabled.value = true; isTtsEnabled.value = true;
} }
if (currentMode.value == 'faceToFace') { if (_bluetoothManager.currentDeviceRx.value != null) {
_asrService.stopContinuousRecognition(); restoreOriginalAudioState();
} else { } else {
if (_bluetoothManager.currentDeviceRx.value != null) { disableBluetoothAudio();
restoreOriginalAudioState();
} else {
disableBluetoothAudio();
}
await _asrService.startContinuousRecognition(_audioSourceType);
// 开始ASR活跃时长计时
_startAsrActiveTracking();
} }
await _asrService.startContinuousRecognition(_audioSourceType);
// 开始ASR活跃时长计时
_startAsrActiveTracking();
isRecognizing.value = true; isRecognizing.value = true;
if (isRecording.value && isRecognizing.value) { if (isRecording.value && isRecognizing.value) {
startRecording(); startRecording();
@ -691,6 +751,7 @@ class TranslationController extends GetxController {
if (!isRecognizing.value) return; if (!isRecognizing.value) return;
try { try {
await stopRecording();
await _asrService.stopContinuousRecognition(); await _asrService.stopContinuousRecognition();
await _astService.stopContinuousTranslation(); await _astService.stopContinuousTranslation();
// 停止ASR活跃时长计时 // 停止ASR活跃时长计时
@ -715,7 +776,7 @@ class TranslationController extends GetxController {
saveTranslationHistory(); saveTranslationHistory();
} }
stopRecording();
currentSessionId = null; currentSessionId = null;
} catch (e) { } catch (e) {
Logger.error('停止语音识别失败: ${e.toString()}'); Logger.error('停止语音识别失败: ${e.toString()}');
@ -993,7 +1054,6 @@ class TranslationController extends GetxController {
} }
// 确保停止ASR活跃时长跟踪 // 确保停止ASR活跃时长跟踪
_stopAsrActiveTracking(); _stopAsrActiveTracking();
stopRecording();
_translationDebounceTimer?.cancel(); _translationDebounceTimer?.cancel();
await _ttsService.stop(); await _ttsService.stop();
} catch (e) { } catch (e) {

17
lib/modules/translation/views/translation_view.dart

@ -472,8 +472,8 @@ class TranslationView extends GetView<TranslationController> {
isDarkMode: isDarkMode, isDarkMode: isDarkMode,
speakerId: 1, speakerId: 1,
activeSpeaker: controller.activeSpeaker.value, activeSpeaker: controller.activeSpeaker.value,
icon: Icons.headset, // 耳机图标 icon: Icons.headset,
label: "headsetUser".tr, // 耳机用户 label: "headsetUser".tr,
), ),
SizedBox(width: 40.w), SizedBox(width: 40.w),
// 手机图标按钮 (说话者2) // 手机图标按钮 (说话者2)
@ -481,8 +481,8 @@ class TranslationView extends GetView<TranslationController> {
isDarkMode: isDarkMode, isDarkMode: isDarkMode,
speakerId: 2, speakerId: 2,
activeSpeaker: controller.activeSpeaker.value, activeSpeaker: controller.activeSpeaker.value,
icon: Icons.phone_android, // 手机图标 icon: Icons.phone_android,
label: "phoneUser".tr, // 手机用户 label: "phoneUser".tr,
), ),
], ],
) )
@ -648,8 +648,13 @@ class TranslationView extends GetView<TranslationController> {
bool isDisabled = activeSpeaker != 0 && activeSpeaker != speakerId; bool isDisabled = activeSpeaker != 0 && activeSpeaker != speakerId;
Timer? _holdTimer; // 增加延迟触发的计时器 Timer? _holdTimer; // 增加延迟触发的计时器
return Listener( return Listener(
onPointerDown: (_) { onPointerDown: (_) async {
// 设置300ms延迟防止误触 // 同步检查权限状态,不使用 await
if (!controller.hasRecordPermission.value) {
Get.snackbar('权限提示', '需要麦克风权限才能使用录音功能');
return;
}
_holdTimer = Timer(const Duration(milliseconds: 200), () { _holdTimer = Timer(const Duration(milliseconds: 200), () {
if (activeSpeaker == 0) { if (activeSpeaker == 0) {
controller.setActiveSpeaker(speakerId); controller.setActiveSpeaker(speakerId);

68
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt

@ -11,7 +11,7 @@ import android.media.audiofx.AutomaticGainControl
import android.util.Log import android.util.Log
import com.microsoft.cognitiveservices.speech.* import com.microsoft.cognitiveservices.speech.*
import com.microsoft.cognitiveservices.speech.util.EventHandler import com.microsoft.cognitiveservices.speech.util.EventHandler
import com.yunqiinnovation.azure_speech.tools.SimpleAudioReceiver
import android.media.AudioManager import android.media.AudioManager
import android.media.AudioAttributes import android.media.AudioAttributes
import android.media.AudioFocusRequest import android.media.AudioFocusRequest
@ -243,14 +243,19 @@ class AzureAsrHelper(private val context: Context) {
stopContinuousRecognition() stopContinuousRecognition()
} }
try { try {
// 启动音频处理 // 启动音频处理(若已启动则跳过)
audioStream!!.startAudioRecord( when(audioSourceType) { if (!audioStream!!.isWriting()) {
AudioSourceType.MICROPHONE -> SimpleAudioReceiver.AudioSourceType.MICROPHONE audioStream!!.startAudioRecord(
AudioSourceType.EXTERNAL -> SimpleAudioReceiver.AudioSourceType.EXTERNAL when (audioSourceType) {
}) AudioSourceType.MICROPHONE -> SimpleAudioReceiver.AudioSourceType.MICROPHONE
AudioSourceType.EXTERNAL -> SimpleAudioReceiver.AudioSourceType.EXTERNAL
},
null
)
} else {
Log.d(tag, "音频已启动,跳过重复启动")
}
// 执行同步识别 // 执行同步识别
val result = recognizer?.recognizeOnceAsync()?.get() val result = recognizer?.recognizeOnceAsync()?.get()
@ -273,13 +278,13 @@ class AzureAsrHelper(private val context: Context) {
/** /**
* 开始连续语音识别 * 开始连续语音识别
* * @param audioSourceType 音频源类型
* @param callback 连续识别结果回调 * @param audioDataCallback 音频数据回调接口
* @return 是否成功开始识别 * @return 是否成功启动
*/ */
fun startContinuousRecognition( fun startContinuousRecognition(
audioSourceType: AudioSourceType = AudioSourceType.MICROPHONE,
audioSourceType: AudioSourceType = AudioSourceType.MICROPHONE audioDataCallback: SimpleAudioReceiver.AudioDataCallback? = null
): Boolean { ): Boolean {
Log.d(tag, "startContinuousRecognition:$isContinuousRecognitionActive ") Log.d(tag, "startContinuousRecognition:$isContinuousRecognitionActive ")
@ -295,31 +300,29 @@ class AzureAsrHelper(private val context: Context) {
} }
this.audioSourceType = audioSourceType this.audioSourceType = audioSourceType
try { try {
Log.d(tag, "startContinuousRecognition: ") Log.d(tag, "startContinuousRecognition: ")
// 启动音频处理 // 启动音频处理(若已启动则跳过)
//startAudioProcessing() if (!audioStream!!.isWriting()) {
// 开始连续识别 audioStream!!.startAudioRecord(
when (audioSourceType) {
AudioSourceType.MICROPHONE -> SimpleAudioReceiver.AudioSourceType.MICROPHONE
AudioSourceType.EXTERNAL -> SimpleAudioReceiver.AudioSourceType.EXTERNAL
},
audioDataCallback
)
} else {
Log.d(tag, "音频已启动,跳过重复启动")
}
// 启动音频处理
audioStream!!.startAudioRecord(
when(audioSourceType) {
AudioSourceType.MICROPHONE -> SimpleAudioReceiver.AudioSourceType.MICROPHONE
AudioSourceType.EXTERNAL -> SimpleAudioReceiver.AudioSourceType.EXTERNAL
}
)
recognizer?.startContinuousRecognitionAsync() recognizer?.startContinuousRecognitionAsync()
isContinuousRecognitionActive = true isContinuousRecognitionActive = true
return true return true
} catch (e: Exception) { } catch (e: Exception) {
audioStream!!.stopMicrophoneCapture() audioStream!!.stopMicrophoneCapture()
isContinuousRecognitionActive = false isContinuousRecognitionActive = false
return false return false
} }
} }
@ -546,7 +549,7 @@ class AzureAsrHelper(private val context: Context) {
/** /**
* 开启录音 * 开启录音
*/ */
fun enableRecord(filePath: String) { fun enableRecord(filePath: String, audioDataCallback: SimpleAudioReceiver.AudioDataCallback? = null) {
Log.i(tag, "开启录音:") Log.i(tag, "开启录音:")
if (audioStream == null) { if (audioStream == null) {
// 创建外部音频拉流对象 // 创建外部音频拉流对象
@ -555,13 +558,13 @@ class AzureAsrHelper(private val context: Context) {
} }
if (audioStream!!.recordfile == null) { if (audioStream!!.recordfile == null) {
audioStream!!.recordfile = RecordFile() audioStream!!.recordfile = RecordFile()
} }
// 启动音频处理 // 启动音频处理
audioStream!!.startAudioRecord( when(audioSourceType) { audioStream!!.startAudioRecord( when(audioSourceType) {
AudioSourceType.MICROPHONE -> SimpleAudioReceiver.AudioSourceType.MICROPHONE AudioSourceType.MICROPHONE -> SimpleAudioReceiver.AudioSourceType.MICROPHONE
AudioSourceType.EXTERNAL -> SimpleAudioReceiver.AudioSourceType.EXTERNAL AudioSourceType.EXTERNAL -> SimpleAudioReceiver.AudioSourceType.EXTERNAL
}) },audioDataCallback)
audioStream!!.recordfile!!.closeFile(true) audioStream!!.recordfile!!.closeFile(true)
@ -618,13 +621,16 @@ class AzureAsrHelper(private val context: Context) {
*/ */
fun stopRecord(isSave: Boolean) { fun stopRecord(isSave: Boolean) {
Log.i(tag, "关闭录音:") Log.i(tag, "关闭录音:")
if (audioStream == null) {
return
}
if (audioStream!!.recordfile == null)return if (audioStream!!.recordfile == null)return
audioStream!!.stopMicrophoneCapture() audioStream!!.stopMicrophoneCapture()
audioStream!!.recordfile!!.closeFile(isSave) audioStream!!.recordfile!!.closeFile(isSave)
} }
/** /**
* 一次性识别回调接口 * 一次性识别回调接口
*/ */

1
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrToAsr.kt

@ -24,6 +24,7 @@ import java.util.*
import javax.crypto.Mac import javax.crypto.Mac
import javax.crypto.spec.SecretKeySpec import javax.crypto.spec.SecretKeySpec
import com.yunqiinnovation.azure_speech.tools.RecordFile import com.yunqiinnovation.azure_speech.tools.RecordFile
import com.yunqiinnovation.azure_speech.tools.SimpleAudioPlayer
/** /**
* 整合的语音翻译服务 * 整合的语音翻译服务
* 集成ASR语音识别、翻译服务和TTS语音合成 * 集成ASR语音识别、翻译服务和TTS语音合成

59
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt

@ -5,6 +5,8 @@ import android.os.Handler
import android.os.Looper import android.os.Looper
import androidx.annotation.NonNull import androidx.annotation.NonNull
import com.yunqiinnovation.azure_speech.utils.FileLogger import com.yunqiinnovation.azure_speech.utils.FileLogger
import com.yunqiinnovation.azure_speech.tools.SimpleAudioReceiver
import com.yunqiinnovation.azure_speech.tools.SimpleAudioPlayer
import io.flutter.embedding.engine.plugins.FlutterPlugin import io.flutter.embedding.engine.plugins.FlutterPlugin
import io.flutter.plugin.common.MethodCall import io.flutter.plugin.common.MethodCall
import io.flutter.plugin.common.MethodChannel import io.flutter.plugin.common.MethodChannel
@ -42,6 +44,12 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
private var astEventSink: EventChannel.EventSink? = null private var astEventSink: EventChannel.EventSink? = null
private lateinit var azureAstHelper: IntegratedSpeechTranslationService private lateinit var azureAstHelper: IntegratedSpeechTranslationService
// 音频数据相关
private lateinit var audioDataChannel: MethodChannel
private lateinit var audioDataEventChannel: EventChannel
private var audioDataEventSink: EventChannel.EventSink? = null
// 是否已添加TTS事件监听器 // 是否已添加TTS事件监听器
private var isTtsListenerAdded = false private var isTtsListenerAdded = false
@ -96,7 +104,21 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
} }
} }
} }
// 音频数据 事件发送方法
private fun sendAudioDataEvent(event: Map<String, Any>) {
if (audioDataEventSink == null) {
FileLogger.w(tag, "无法发送音频数据事件:事件通道未准备好")
return
}
mainHandler.post {
try {
audioDataEventSink?.success(event)
} catch (e: Exception) {
FileLogger.e(tag, "发送音频数据事件失败: ${e.message}")
}
}
}
override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) {
context = flutterPluginBinding.applicationContext context = flutterPluginBinding.applicationContext
@ -149,7 +171,18 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
astEventSink = null astEventSink = null
} }
}) })
// 初始化音频数据事件通道
audioDataEventChannel =
EventChannel(flutterPluginBinding.binaryMessenger, "azure_speech/audio_data_events")
audioDataEventChannel.setStreamHandler(object : EventChannel.StreamHandler {
override fun onListen(arguments: Any?, events: EventChannel.EventSink?) {
audioDataEventSink = events
}
override fun onCancel(arguments: Any?) {
audioDataEventSink = null
}
})
// 初始化Azure语音服务 // 初始化Azure语音服务
azureTtsHelper = AzureTtsHelper(context) azureTtsHelper = AzureTtsHelper(context)
azureAsrHelper = AzureAsrHelper(context) azureAsrHelper = AzureAsrHelper(context)
@ -473,10 +506,30 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin {
"enableRecord" -> { "enableRecord" -> {
val filePath = call.argument<String>("filePath") ?: "" val filePath = call.argument<String>("filePath") ?: ""
// 是否接受音频数据
val acceptAudioData = call.argument<Boolean>("acceptAudioData") ?: false
try { try {
FileLogger.d(tag, "音频文件名称为: ${filePath}") // FileLogger.d(tag, "音频文件名称为: ${filePath}")
azureAsrHelper.enableRecord(filePath) // 启用录音,并根据 acceptAudioData 参数决定是否设置音频数据回调
azureAsrHelper.enableRecord(filePath, if(acceptAudioData) {
// 创建音频数据回调,将音频数据发送到 Flutter 层
object : SimpleAudioReceiver.AudioDataCallback {
override fun onAudio(audioData: ByteArray) {
// 构建音频数据事件映射
val audioEvent = mapOf(
"type" to "audioData",
"data" to audioData,
"timestamp" to System.currentTimeMillis(),
"size" to audioData.size
)
// 发送音频数据事件到 Flutter 层
sendAudioDataEvent(audioEvent)
}
}
} else {
null
})
result.success(true) result.success(true)
} catch (e: Exception) { } catch (e: Exception) {
result.error("ENABLERECORD_ERROR", e.message, null) result.error("ENABLERECORD_ERROR", e.message, null)

2
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt

@ -14,7 +14,7 @@ import com.deep_voice.speech.tts.TtsEventType
import kotlinx.coroutines.* import kotlinx.coroutines.*
import java.io.ByteArrayInputStream import java.io.ByteArrayInputStream
import java.io.InputStream import java.io.InputStream
import com.yunqiinnovation.azure_speech.tools.SimpleAudioPlayer
/** /**
* Azure TTS Helper * Azure TTS Helper
* *

2
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/SimpleAudioPlayer.kt

@ -1,4 +1,4 @@
package com.yunqiinnovation.azure_speech package com.yunqiinnovation.azure_speech.tools
import android.content.Context import android.content.Context
import android.media.AudioAttributes import android.media.AudioAttributes

31
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/SimpleAudioReceiver.kt

@ -1,4 +1,4 @@
package com.yunqiinnovation.azure_speech package com.yunqiinnovation.azure_speech.tools
import android.content.Context import android.content.Context
import android.media.AudioFormat import android.media.AudioFormat
@ -38,6 +38,7 @@ class SimpleAudioReceiver(private val context: Context) {
} }
private val bufferSize = 4096 // 可根据需要调整 private val bufferSize = 4096 // 可根据需要调整
private var audioDataCallback: AudioDataCallback? = null
var audioRecord: AudioRecord? = null var audioRecord: AudioRecord? = null
var pushAudioStream: PushAudioInputStream? = null var pushAudioStream: PushAudioInputStream? = null
// 音频源配置 // 音频源配置
@ -98,7 +99,7 @@ class SimpleAudioReceiver(private val context: Context) {
* 启动写入线程 * 启动写入线程
*/ */
private fun startWriteThread() { private fun startWriteThread() {
writeThread = Thread { writeThread = Thread {
try { try {
while (isRunning.get()) { while (isRunning.get()) {
@ -139,6 +140,9 @@ class SimpleAudioReceiver(private val context: Context) {
try { try {
Log.d(TAG, "写入数据大小: ${finalData.size}") Log.d(TAG, "写入数据大小: ${finalData.size}")
if(audioDataCallback!=null){
audioDataCallback?.onAudio(finalData)
}
pushAudioStream?.write(finalData) pushAudioStream?.write(finalData)
@ -175,12 +179,15 @@ class SimpleAudioReceiver(private val context: Context) {
/** /**
* 开始音频输入 * 开始音频输入
* @param audioSourceType 音频源类型
* @param callback 音频数据回调,可为空
*/ */
fun startAudioRecord(audioSourceType: AudioSourceType) { fun startAudioRecord(audioSourceType: AudioSourceType, callback: AudioDataCallback?) {
this.audioSourceType = audioSourceType this.audioSourceType = audioSourceType
this.audioDataCallback = callback
writeQueue.clear() writeQueue.clear()
isWriting.set(true) isWriting.set(true)
when (audioSourceType) { when (audioSourceType) {
AudioSourceType.MICROPHONE -> runMicrophoneCapture() AudioSourceType.MICROPHONE -> runMicrophoneCapture()
AudioSourceType.EXTERNAL -> runExternalCapture() AudioSourceType.EXTERNAL -> runExternalCapture()
@ -400,4 +407,20 @@ class SimpleAudioReceiver(private val context: Context) {
Log.e("AudioMode", "恢复音频模式失败: ${e.message}") Log.e("AudioMode", "恢复音频模式失败: ${e.message}")
} }
} }
fun isWriting(): Boolean {
return isWriting.get()
}
/**
* 连续识别回调接口
*/
interface AudioDataCallback {
/**
* 音频数据回调
* @param data 音频数据字节数组
*/
fun onAudio(data: ByteArray)
}
} }

Loading…
Cancel
Save