Browse Source

上传服务代码

newdev_shunjiawei
liwei1dao 6 months ago
parent
commit
a7e970d2d7
  1. 15
      lib/data/services/asr_service.dart
  2. 2
      lib/data/services/language_manager.dart
  3. 15
      lib/data/services/speech_impl/azure_asr_service.dart
  4. 68
      lib/data/services/speech_impl/azure_ast_service.dart
  5. 7
      lib/data/services/speech_impl/volcano_asr_api_service.dart
  6. 7
      lib/data/services/speech_impl/volcano_asr_service.dart
  7. 25
      lib/modules/meeting/controllers/meeting_record_controller.dart
  8. 240
      lib/modules/translation/controllers/translation_controller.dart
  9. 8
      lib/modules/translation/models/translation_models.dart
  10. 97
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt
  11. 27
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt
  12. 10
      local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt

15
lib/data/services/asr_service.dart

@ -1,6 +1,14 @@
import 'dart:async';
import 'dart:typed_data';
/// 音频源类型
enum AudioSourceType {
microphone, // 使用设备麦克风
systemAudio, // 使用系统音频(麦克风也启动保证后台也能正常运行)
systemAudioPlusMicrophone, // 使用系统音频+麦克风音频
external // 使用外部提供的音频数据
}
/// 语音识别服务接口
abstract class AsrService {
/// 支持的语言
@ -47,8 +55,11 @@ abstract class AsrService {
/// [filePath] 录音文件路径
/// [audioSourceType] 音频源类型
/// [acceptAudioData] 是否接受音频数据回调,默认为 false
Future<bool> enableRecord(bool audioSourceType, String filePath,
{bool acceptAudioData = false});
Future<bool> enableRecord(
AudioSourceType audioSourceType,
String filePath,
bool acceptAudioData,
);
/// 获取音频数据流(如果支持)
Stream<Uint8List>? getAudioDataStream() => null;

2
lib/data/services/language_manager.dart

@ -319,7 +319,7 @@ class LanguageManager extends GetxService {
chineseName: '土耳其语',
englishName: 'Turkish',
asrCode: 'tr-TR',
ttsVoiceName: 'tr-TR-EmelNeural',
ttsVoiceName: 'tr-TR-SedaNeural',
);
_addLanguage(

15
lib/data/services/speech_impl/azure_asr_service.dart

@ -7,12 +7,6 @@ import '../../../core/utils/logger.dart';
import 'package:get/get.dart';
import '../asr_service.dart';
/// 音频源类型
enum AudioSourceType {
microphone, // 使用设备麦克风
external // 使用外部提供的音频数据
}
/// Azure 语音识别服务
///
/// 该服务提供了通过平台通道与原生 Microsoft Speech SDK 交互的接口
@ -521,11 +515,14 @@ class AzureAsrService extends GetxService implements AsrService {
}
@override
Future<bool> enableRecord(bool audioSourceType, String filePath,
{bool acceptAudioData = false}) async {
Future<bool> enableRecord(
AudioSourceType audioSourceType,
String filePath,
bool acceptAudioData,
) async {
try {
final bool result = await _channel.invokeMethod('enableRecord', {
'audioSourceType': audioSourceType,
'audioSourceType': audioSourceType.index,
'filePath': filePath,
'acceptAudioData': acceptAudioData, // 新增参数
});

68
lib/data/services/speech_impl/azure_ast_service.dart

@ -67,9 +67,16 @@ class AzureAstService extends GetxService implements AstService {
super.onClose();
}
/// 设置事件通道
/// 设置事件通道(只在首次调用时订阅 EventChannel,后续复用)
void _setupEventChannel() {
_eventSubscription?.cancel();
// 确保 _eventStreamController 已就绪,避免丢失早期事件
_eventStreamController ??= StreamController<ASTEvent>.broadcast();
// EventChannel.receiveBroadcastStream() 只能活跃一次,复用已有订阅
if (_eventSubscription != null) {
Logger.info('[STS] _setupEventChannel: 复用已有EventSubscription, streamController=${_eventStreamController.hashCode}');
return;
}
Logger.info('[STS] _setupEventChannel: 首次订阅EventChannel, streamController=${_eventStreamController.hashCode}');
_eventSubscription = _eventChannel.receiveBroadcastStream().listen((event) {
if (event is Map) {
_handleRecognitionEvent(event);
@ -78,20 +85,27 @@ class AzureAstService extends GetxService implements AstService {
}
/// 处理来自原生端的识别事件(AST事件)
/// 将原生侧 AST 事件映射为统一的 ASTEvent,提供给业务层使用。
void _handleRecognitionEvent(dynamic event) {
if (event is! Map || _eventStreamController == null) return;
if (event is! Map) return;
if (_eventStreamController == null) {
Logger.error('[STS] _handleRecognitionEvent: _eventStreamController 为 null,事件被丢弃: ${event['type']}');
return;
}
if (_eventStreamController!.isClosed) {
Logger.error('[STS] _handleRecognitionEvent: _eventStreamController 已关闭,事件被丢弃: ${event['type']}');
return;
}
final Map<dynamic, dynamic> eventMap = event;
final String eventType = eventMap['type'] as String? ?? '';
switch (eventType) {
case 'recognized':
print("Ast处理识别完成事件:${eventMap}");
final String serviceId = eventMap['serviceId'] as String? ?? '';
final String utteranceId = eventMap['utteranceId'] as String? ?? '';
final String text = eventMap['text'] as String? ?? '';
final String detectedLanguage = eventMap['language'] as String? ?? '';
Logger.info('[STS] 识别完成 [$serviceId]: text=$text, uttId=$utteranceId');
_latestRecognizedText = text;
_latestDetectedLanguage = detectedLanguage;
_eventStreamController?.add(ASTEvent(
@ -104,11 +118,11 @@ class AzureAstService extends GetxService implements AstService {
break;
case 'recognizing':
print("Ast处理识别中事件:${eventMap}");
final String serviceId = eventMap['serviceId'] as String? ?? '';
final String utteranceId = eventMap['utteranceId'] as String? ?? '';
final String text = eventMap['text'] as String? ?? '';
final String detectedLanguage = eventMap['language'] as String? ?? '';
Logger.info('[STS] 识别中 [$serviceId]: text=$text');
_eventStreamController?.add(ASTEvent(
type: ASTEventType.intermediateResult,
serviceId: serviceId,
@ -118,12 +132,12 @@ class AzureAstService extends GetxService implements AstService {
));
break;
case 'translatedInterim':
print("Ast处理翻译中事件:${eventMap}");
final String serviceId = eventMap['serviceId'] as String? ?? '';
final String utteranceId = eventMap['utteranceId'] as String? ?? '';
final String text = eventMap['translatedText'] as String? ?? '';
final String detectedLanguage =
eventMap['targetLanguage'] as String? ?? '';
Logger.info('[STS] 翻译中 [$serviceId]: text=$text');
_eventStreamController?.add(ASTEvent(
type: ASTEventType.translationInterim,
serviceId: serviceId,
@ -133,12 +147,12 @@ class AzureAstService extends GetxService implements AstService {
));
break;
case 'translated':
print("Ast处理翻译结果事件:${eventMap}");
final String serviceId = eventMap['serviceId'] as String? ?? '';
final String utteranceId = eventMap['utteranceId'] as String? ?? '';
final String text = eventMap['translatedText'] as String? ?? '';
final String detectedLanguage =
eventMap['targetLanguage'] as String? ?? '';
Logger.info('[STS] 翻译完成 [$serviceId]: text=$text, uttId=$utteranceId');
_eventStreamController?.add(ASTEvent(
type: ASTEventType.translationResult,
serviceId: serviceId,
@ -149,12 +163,14 @@ class AzureAstService extends GetxService implements AstService {
break;
case 'sessionStarted':
Logger.info('[STS] 会话开始');
_eventStreamController?.add(ASTEvent(
type: ASTEventType.sessionStarted,
));
break;
case 'sessionStopped':
Logger.info('[STS] 会话停止');
_isContinuousRecognitionActive = false;
_eventStreamController?.add(ASTEvent(
type: ASTEventType.sessionStopped,
@ -165,11 +181,7 @@ class AzureAstService extends GetxService implements AstService {
_isContinuousRecognitionActive = false;
final String reason = eventMap['reason'] as String? ?? '';
final String errorDetails = eventMap['errorDetails'] as String? ?? '';
if (reason.isNotEmpty || errorDetails.isNotEmpty) {
Logger.error('识别取消: $reason - ${errorDetails.toString()}');
}
Logger.error('[STS] 识别取消: $reason - $errorDetails');
_eventStreamController?.add(ASTEvent(
type: ASTEventType.canceled,
error: '$reason: $errorDetails',
@ -185,12 +197,20 @@ class AzureAstService extends GetxService implements AstService {
if (code.isNotEmpty) {
error = '$error (Code: $code)';
}
Logger.error('识别错误: ${error.toString()}');
Logger.error('[STS] 识别错误: $error');
_eventStreamController?.add(ASTEvent(
type: ASTEventType.error,
error: error,
));
break;
case 'serviceInitialized':
Logger.info('[STS] 服务初始化: ${eventMap['serviceId']} ${eventMap['direction']}');
break;
default:
Logger.info('[STS] 未处理事件: $eventType');
break;
}
}
@ -241,20 +261,23 @@ class AzureAstService extends GetxService implements AstService {
@override
Future<Stream<ASTEvent>> recognizeCallback() async {
if (!_isInitialized) {
Logger.info('[STS] recognizeCallback: 未初始化,先执行initialize');
await initialize();
}
try {
_eventStreamController = StreamController<ASTEvent>.broadcast();
_eventStreamController ??= StreamController<ASTEvent>.broadcast();
Logger.info('[STS] recognizeCallback: streamController=${_eventStreamController.hashCode}, hasListener=${_eventStreamController!.hasListener}');
// 开始连续识别
final bool result = await _channel.invokeMethod('recognizeCallback');
Logger.info('[STS] recognizeCallback: invokeMethod 结果=$result');
if (!result) {
_cleanupEventStream();
}
return _eventStreamController!.stream;
} catch (e) {
Logger.error('开始连续语音识别失败: ${e.toString()}');
Logger.error('[STS] recognizeCallback 失败: ${e.toString()}');
rethrow;
}
}
@ -316,14 +339,15 @@ class AzureAstService extends GetxService implements AstService {
@override
Future<void> dispose() async {
try {
await _eventSubscription?.cancel();
_eventSubscription = null;
Logger.info('[STS] dispose: 开始释放, streamController=${_eventStreamController?.hashCode}, eventSub=${_eventSubscription?.hashCode}');
// 不取消 _eventSubscription(EventChannel 只能订阅一次,保持复用)
// 只关闭 _eventStreamController,下次 initialize 时会重新创建
await _eventStreamController?.close();
_eventStreamController = null;
await _channel.invokeMethod('dispose');
_isInitialized = false;
Logger.info('Azure AST 资源已释放');
Logger.info('[STS] dispose: 完成, streamController=null, eventSub保留');
} catch (e) {
Logger.error('Azure AST 资源释放失败: ${e.toString()}');
_isInitialized = false;
@ -338,11 +362,16 @@ class AzureAstService extends GetxService implements AstService {
try {
final List<String> languages =
supportedLanguages ?? _defaultSupportedLanguages;
Logger.info('[STS] initialize: provider=$provider, isInitialized=$_isInitialized');
// 底层会初始化前释放
if (_isInitialized) {
await dispose();
}
// 先建立事件通道,确保原生端发事件时 astEventSink 已就绪
_setupEventChannel();
Logger.info('[STS] initialize: eventChannel已建立, streamController=${_eventStreamController?.hashCode}');
final bool result = await _channel.invokeMethod('initialize', {
'provider': provider,
'subscriptionKey': _subscriptionKey,
@ -371,7 +400,6 @@ class AzureAstService extends GetxService implements AstService {
});
_isInitialized = result;
_setupEventChannel();
print('Azure 语音识别服务初始化${result ? '成功' : '失败'}');
Logger.info('Azure 语音识别服务初始化${result ? '成功' : '失败'}');
return result;

7
lib/data/services/speech_impl/volcano_asr_api_service.dart

@ -813,8 +813,11 @@ class VolcanoAsrApiService implements AsrService {
}
@override
Future<bool> enableRecord(bool audioSourceType, String filePath,
{bool acceptAudioData = false}) {
Future<bool> enableRecord(
AudioSourceType audioSourceType,
String filePath,
bool acceptAudioData,
) {
// TODO: implement enableRecord
throw UnimplementedError();
}

7
lib/data/services/speech_impl/volcano_asr_service.dart

@ -348,8 +348,11 @@ class VolcanoAsrService extends GetxService implements AsrService {
}
@override
Future<bool> enableRecord(bool audioSourceType, String filePath,
{bool acceptAudioData = false}) {
Future<bool> enableRecord(
AudioSourceType audioSourceType,
String filePath,
bool acceptAudioData,
) {
// TODO: implement enableRecord
throw UnimplementedError();
}

25
lib/modules/meeting/controllers/meeting_record_controller.dart

@ -582,10 +582,31 @@ class MeetingRecordController extends GetxController
final fullFileName = "${safeBase}_$formattedTime";
final filePath = path.join(dir.path, '$fullFileName.wav');
// 根据音频类型选择音频源
AudioSourceType audioSourceType;
bool acceptAudioData = false;
switch (audioType.value) {
case 0: // Live
audioSourceType = AudioSourceType.microphone;
acceptAudioData = true;
break;
case 1: // Media
audioSourceType = AudioSourceType.systemAudio;
acceptAudioData = false;
break;
case 2: // Call
audioSourceType = AudioSourceType.external;
acceptAudioData = false;
break;
default:
audioSourceType = AudioSourceType.microphone;
acceptAudioData = true;
}
await _asrService.enableRecord(
audioType.value != 0,
audioSourceType,
filePath,
acceptAudioData: true,
acceptAudioData,
);
// 通过 asrService 获取音频数据流

240
lib/modules/translation/controllers/translation_controller.dart

@ -834,18 +834,6 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
try {
Logger.info('开始初始化通话模式语音翻译服务');
// 初始化 ASR 服务时,明确指定需要支持的语言
final List<String> asrSupportedLanguages = [targetLanguageCode.value];
Logger.info('1初始化ASR服务,支持语言: $asrSupportedLanguages');
await _asrService.initialize(supportedLanguages: asrSupportedLanguages);
// 获取识别事件流
var recognitionStream = await _asrService.recognizeCallback();
_recognitionSubscription?.cancel();
_recognitionSubscription =
recognitionStream.listen(_handleRecognitionEvent);
final sourceShort = _languageManager.getShortCodeByAsrCode(sourceLanguageCode.value) ?? 'zh';
final targetShort = _languageManager.getShortCodeByAsrCode(targetLanguageCode.value) ?? 'en';
final bestProvider = _languageManager.findBestMatchingProvider(sourceShort, targetShort);
@ -870,10 +858,15 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
await _astService.initialize(supportedLanguages: callModeLanguages, provider: astProvider);
// 获取识别事件流
// 获取识别事件流(端到端服务已包含 ASR+翻译+TTS,不需要单独启动 ASR)
var astStream = await _astService.recognizeCallback();
_astEventSubscription?.cancel();
_astEventSubscription = astStream.listen(_handleAstEvent);
_astEventSubscription = astStream.listen(_handleAstEvent, onError: (e) {
Logger.error('[STS] astStream 错误: $e');
}, onDone: () {
Logger.info('[STS] astStream 已关闭');
});
Logger.info('[STS] 已订阅 astStream, subscription=${_astEventSubscription.hashCode}');
Logger.info('通话模式语音翻译服务初始化完成');
} catch (e) {
@ -948,13 +941,19 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
Future<void> startRecording() async {
_timerManager.startTimer();
final formattedTime = DateFormat('yyyyMMdd_HHmmss').format(DateTime.now());
if (currentMode.value == 'call' || currentMode.value == 'audioVideo') {
await _asrService.enableRecord(
true, "${dir.path}/${currentModeTitle.value.tr}_$formattedTime.wav");
AudioSourceType audioSourceType;
if (currentMode.value == 'call') {
audioSourceType = AudioSourceType.external;
} else if (currentMode.value == 'audioVideo') {
audioSourceType = AudioSourceType.systemAudio;
} else {
await _asrService.enableRecord(
false, "${dir.path}/${currentModeTitle.value.tr}_$formattedTime.wav");
audioSourceType = AudioSourceType.microphone;
}
await _asrService.enableRecord(
audioSourceType,
"${dir.path}/${currentModeTitle.value.tr}_$formattedTime.wav",
false,
);
// if (currentMode.value == 'call') {
// await _astService.enableRecord(
@ -1087,28 +1086,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
}
/// 配置通话模式
/// 配置通话模式(只做 BLE 配置,AST 已在 _initializeCallModeTranslationService 中初始化)
Future<void> _configureCallMode() async {
final sourceShort = _languageManager.getShortCodeByAsrCode(sourceLanguageCode.value) ?? 'zh';
final targetShort = _languageManager.getShortCodeByAsrCode(targetLanguageCode.value) ?? 'en';
final bestProvider = _languageManager.findBestMatchingProvider(sourceShort, targetShort);
final astProvider = bestProvider?.provider ?? 'azure';
final transCode0 = bestProvider?.sourceSpec['translationCode'] ?? sourceLanguageCode.value;
final transCode1 = bestProvider?.targetSpec['translationCode'] ?? targetLanguageCode.value;
final ttsVoice0 = bestProvider?.sourceSpec['ttsCode'] ??
(_languageManager.getTtsVoiceNameByAsrCode(sourceLanguageCode.value) ?? 'zh-CN-XiaoxiaoNeural');
final ttsVoice1 = bestProvider?.targetSpec['ttsCode'] ??
(_languageManager.getTtsVoiceNameByAsrCode(targetLanguageCode.value) ?? 'en-US-AriaNeural');
final List<String> callModeLanguages = [
sourceLanguageCode.value,
targetLanguageCode.value,
transCode0,
transCode1,
ttsVoice0,
ttsVoice1,
];
Logger.info('1初始化AST服务,支持语言: $callModeLanguages, provider: $astProvider');
await _astService.initialize(supportedLanguages: callModeLanguages, provider: astProvider);
isPreparing.value = true;
await bleManager.openA2DPDecoder();
int attempts = 0;
@ -1232,71 +1211,208 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
}
/// 处理 AST(语音识别+翻译一体)事件
/// 按 serviceId 区分双路(A=己方, B=对方),用 utteranceId 匹配同一句话的事件
void _handleAstEvent(ASTEvent event) {
Logger.info('[STS] _handleAstEvent 收到: type=${event.type}, serviceId=${event.serviceId}, text=${event.text}');
// 根据 serviceId 确定语言方向
final String srcCode = event.serviceId == "A"
? sourceLanguageCode.value
: targetLanguageCode.value;
final String tgtCode = event.serviceId == "A"
? targetLanguageCode.value
: sourceLanguageCode.value;
switch (event.type) {
case ASTEventType.intermediateResult:
if (event.text.isEmpty) break;
Logger.info('AST 识别中 [${event.serviceId}]: ${event.text}');
handleIntermediateResult(event.text);
currentSessionId ??= DateTime.now().millisecondsSinceEpoch.toString();
// 按 utteranceId 查找已有项
TranslationItem? target;
int targetIndex = -1;
if (event.utteranceId.isNotEmpty) {
for (int i = translationHistory.length - 1; i >= 0; i--) {
if (translationHistory[i].isIntermediate &&
translationHistory[i].utteranceId == event.utteranceId) {
target = translationHistory[i];
targetIndex = i;
break;
}
}
}
if (target != null) {
target.sourceText = event.text;
target.sourceLanguageCode = srcCode;
target.targetLanguageCode = tgtCode;
target.serviceId = event.serviceId;
translationHistory.refresh();
} else {
_historyManager.addTranslationItem(TranslationItem(
sourceText: event.text,
translatedText: '',
sourceLanguageCode: srcCode,
targetLanguageCode: tgtCode,
timestamp: DateTime.now(),
sessionId: currentSessionId!,
isFirstInSession: translationHistory.isEmpty ||
translationHistory.last.sessionId != currentSessionId,
isIntermediate: true,
utteranceId: event.utteranceId,
serviceId: event.serviceId,
));
}
break;
case ASTEventType.finalResult:
if (event.text.isEmpty) break;
Logger.info('AST 识别完成 [${event.serviceId}]: ${event.text}');
// AST 已内置翻译,直接更新源文本,不再调用 translateText
if (translationHistory.isNotEmpty &&
translationHistory.last.isIntermediate) {
translationHistory.last.sourceText = event.text;
translationHistory.last.isIntermediate = false;
translationHistory.refresh();
_scrollToBottom();
saveTranslationHistory();
} else {
final newItem = TranslationItem(
currentSessionId ??= DateTime.now().millisecondsSinceEpoch.toString();
TranslationItem? target;
if (event.utteranceId.isNotEmpty) {
for (int i = translationHistory.length - 1; i >= 0; i--) {
if (translationHistory[i].isIntermediate &&
translationHistory[i].utteranceId == event.utteranceId) {
target = translationHistory[i];
break;
}
}
}
if (target == null) {
target = TranslationItem(
sourceText: event.text,
translatedText: '',
sourceLanguageCode: sourceLanguageCode.value,
targetLanguageCode: targetLanguageCode.value,
sourceLanguageCode: srcCode,
targetLanguageCode: tgtCode,
timestamp: DateTime.now(),
sessionId: currentSessionId ??
DateTime.now().millisecondsSinceEpoch.toString(),
sessionId: currentSessionId!,
isFirstInSession: translationHistory.isEmpty ||
translationHistory.last.sessionId != currentSessionId,
isIntermediate: false,
isIntermediate: true,
utteranceId: event.utteranceId,
serviceId: event.serviceId,
isFinalResultReceived: true,
);
translationHistory.add(newItem);
_historyManager.addTranslationItem(target);
} else {
target.sourceText = event.text;
target.sourceLanguageCode = srcCode;
target.targetLanguageCode = tgtCode;
target.serviceId = event.serviceId;
target.isFinalResultReceived = true;
}
// 识别+翻译都到齐才标记完成
if (target.isFinalResultReceived && target.isTranslationResultReceived) {
target.isIntermediate = false;
translationHistory.refresh();
_scrollToBottom();
saveTranslationHistory();
} else {
translationHistory.refresh();
}
break;
case ASTEventType.translationInterim:
if (event.text.isEmpty) break;
Logger.info('AST 翻译中 [${event.serviceId}]: ${event.text}');
if (translationHistory.isNotEmpty) {
translationHistory.last.translatedText = event.text;
currentSessionId ??= DateTime.now().millisecondsSinceEpoch.toString();
TranslationItem? target;
if (event.utteranceId.isNotEmpty) {
for (int i = translationHistory.length - 1; i >= 0; i--) {
if (translationHistory[i].isIntermediate &&
translationHistory[i].utteranceId == event.utteranceId) {
target = translationHistory[i];
break;
}
}
}
if (target != null) {
target.translatedText = event.text;
target.sourceLanguageCode = srcCode;
target.targetLanguageCode = tgtCode;
translationHistory.refresh();
} else {
_historyManager.addTranslationItem(TranslationItem(
sourceText: '',
translatedText: event.text,
sourceLanguageCode: srcCode,
targetLanguageCode: tgtCode,
timestamp: DateTime.now(),
sessionId: currentSessionId!,
isFirstInSession: translationHistory.isEmpty ||
translationHistory.last.sessionId != currentSessionId,
isIntermediate: true,
utteranceId: event.utteranceId,
serviceId: event.serviceId,
));
}
break;
case ASTEventType.translationResult:
if (event.text.isEmpty) break;
Logger.info('AST 翻译完成 [${event.serviceId}]: ${event.text}');
if (translationHistory.isNotEmpty) {
translationHistory.last.translatedText = event.text;
currentSessionId ??= DateTime.now().millisecondsSinceEpoch.toString();
TranslationItem? target;
if (event.utteranceId.isNotEmpty) {
for (int i = translationHistory.length - 1; i >= 0; i--) {
if (translationHistory[i].isIntermediate &&
translationHistory[i].utteranceId == event.utteranceId) {
target = translationHistory[i];
break;
}
}
}
if (target == null) {
target = TranslationItem(
sourceText: '',
translatedText: event.text,
sourceLanguageCode: srcCode,
targetLanguageCode: tgtCode,
timestamp: DateTime.now(),
sessionId: currentSessionId!,
isFirstInSession: translationHistory.isEmpty ||
translationHistory.last.sessionId != currentSessionId,
isIntermediate: true,
utteranceId: event.utteranceId,
serviceId: event.serviceId,
isTranslationResultReceived: true,
);
translationHistory.add(target);
translationHistory.refresh();
_scrollToBottom();
} else {
target.translatedText = event.text;
target.sourceLanguageCode = srcCode;
target.targetLanguageCode = tgtCode;
target.isTranslationResultReceived = true;
}
if (target.isFinalResultReceived && target.isTranslationResultReceived) {
target.isIntermediate = false;
translationHistory.refresh();
_scrollToBottom();
saveTranslationHistory();
} else {
translationHistory.refresh();
}
break;
case ASTEventType.error:
Logger.error('AST 错误: ${event.error}');
isRecognizing.value = false;
break;
case ASTEventType.canceled:
Logger.error('AST 取消: ${event.error}');
isRecognizing.value = false;
break;
default:
@ -1898,7 +2014,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
}
if (currentMode.value == "call") {
_initializeCallModeTranslationService();
await _initializeCallModeTranslationService();
} else {
Logger.info('重新初始化ASR服务,支持语言: $asrSupportedLanguages');

8
lib/modules/translation/models/translation_models.dart

@ -12,6 +12,10 @@ class TranslationItem {
String sessionId;
final bool isFirstInSession;
bool isIntermediate; // 是否为中间结果
String utteranceId; // AST 事件的 utteranceId,用于匹配同一句话的识别和翻译
String serviceId; // AST 服务 ID(A=己方, B=对方)
bool isFinalResultReceived; // 是否已收到最终识别结果
bool isTranslationResultReceived; // 是否已收到最终翻译结果
TranslationItem({
required this.sourceText,
@ -22,6 +26,10 @@ class TranslationItem {
required this.sessionId,
this.isFirstInSession = false,
this.isIntermediate = false,
this.utteranceId = '',
this.serviceId = '',
this.isFinalResultReceived = false,
this.isTranslationResultReceived = false,
});
/// 从JSON创建TranslationItem实例

97
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt

@ -130,12 +130,19 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
* 参数:`data` 为单声道 PCM 16kHz 16bit 音频数据
* 返回:无
*/
private var astAudioPushCountA = 0L
private var astAudioPushCountB = 0L
private fun pushAstAudioToA(data: ByteArray) {
astAudioPushCountA++
if (astAudioPushCountA % 100 == 1L) {
FileLogger.d(tag, "pushAstAudioToA: provider=$currentAstProvider, size=${data.size}, count=$astAudioPushCountA")
}
if (currentAstProvider == "iflytek") {
iflytekAstHelperA?.pushAudioData(data)
} else if (currentAstProvider == "azure") {
azureAstHelperA?.pushAudioData(data)
} else if (currentAstProvider == "volcano") {
} else if (currentAstProvider == "volcano") {
doubaoAstHelperA?.pushAudioData(data)
} else if (currentAstProvider == "alibaba") {
bailianAstHelperA?.pushAudioData(data)
@ -149,6 +156,10 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
* 返回:无
*/
private fun pushAstAudioToB(data: ByteArray) {
astAudioPushCountB++
if (astAudioPushCountB % 100 == 1L) {
FileLogger.d(tag, "pushAstAudioToB: provider=$currentAstProvider, size=${data.size}, count=$astAudioPushCountB")
}
if (currentAstProvider == "iflytek" ) {
iflytekAstHelperB?.pushAudioData(data)
} else if (currentAstProvider == "azure") {
@ -1187,7 +1198,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
val azureTranslationKey = call.argument<String>("azureTranslationKey") ?: ""
val azureTranslationRegion =
call.argument<String>("azureTranslationRegion") ?: "cn-north-1"
call.argument<String>("azureTranslationServiceRegion") ?: ""
val wsUrl = call.argument<String>("wsUrl")
?: "wss://openspeech.bytedance.com/api/v4/ast/v2/translate"
@ -1240,8 +1251,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
{ data ->
bleWriteScope.launch {
bleRightMutex.withLock {
// TODO: 双端翻译时启用右声道写入
// BleService.writeExternalRightAudioData(data)
BleService.writeExternalAudioData(data)
}
}
}
@ -1249,15 +1259,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
val callbackB = IflytekAstCallback(
"B", "$lang1->$lang0",
{ sendAstEvent(it) },
{ data ->
bleWriteScope.launch {
bleLeftMutex.withLock {
// TODO: 双端翻译时启用左声道分离写入
// BleService.writeExternalLeftAudioData(data)
BleService.writeExternalAudioData(data)
}
}
}
{ data -> }
)
GlobalScope.launch(Dispatchers.Main) {
@ -1312,8 +1314,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
{ data ->
bleWriteScope.launch {
bleRightMutex.withLock {
// TODO: 双端翻译时启用右声道写入
// BleService.writeExternalRightAudioData(data)
BleService.writeExternalAudioData(data)
}
}
}
@ -1321,15 +1322,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
val callbackB = AzureAstCallback(
"B", "$lang1->$lang0",
{ sendAstEvent(it) },
{ data ->
bleWriteScope.launch {
bleLeftMutex.withLock {
// TODO: 双端翻译时启用左声道分离写入
// BleService.writeExternalLeftAudioData(data)
BleService.writeExternalAudioData(data)
}
}
}
{ data -> }
)
GlobalScope.launch(Dispatchers.Main) {
@ -1366,8 +1359,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
{ data ->
bleWriteScope.launch {
bleRightMutex.withLock {
// TODO: 双端翻译时启用右声道写入
// BleService.writeExternalRightAudioData(data)
BleService.writeExternalAudioData(data)
}
}
}
@ -1375,15 +1367,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
val callbackB = DoubaoAstCallback(
"B", "$translationLang1->$translationLang0", translationLang0,
{ sendAstEvent(it) },
{ data ->
bleWriteScope.launch {
bleLeftMutex.withLock {
// TODO: 双端翻译时启用左声道分离写入
// BleService.writeExternalLeftAudioData(data)
BleService.writeExternalAudioData(data)
}
}
}
{ data -> }
)
Log.d(tag, "initializeIntegrated:lang0= $translationLang0, lang1=$translationLang1")
// 设置会话语言(与 UI 选择一致)
@ -1428,8 +1412,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
{ data ->
bleWriteScope.launch {
bleRightMutex.withLock {
// TODO: 双端翻译时启用右声道写入
// BleService.writeExternalRightAudioData(data)
BleService.writeExternalAudioData(data)
}
}
}
@ -1437,15 +1420,7 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
val callbackB = AliyunAstCallback(
"B", "$translationLang1->$translationLang0", translationLang0,
{ sendAstEvent(it) },
{ data ->
bleWriteScope.launch {
bleLeftMutex.withLock {
// TODO: 双端翻译时启用左声道分离写入
// BleService.writeExternalLeftAudioData(data)
BleService.writeExternalAudioData(data)
}
}
}
{ data -> }
)
GlobalScope.launch(Dispatchers.Main) {
@ -1773,7 +1748,13 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
override fun onConnectionStateChanged(state: Int) {
}
private var bleAudioRecvCount = 0L
override fun onAudioDataReceived(data: ByteArray, channel: Int) {
bleAudioRecvCount++
if (bleAudioRecvCount % 200 == 1L) {
FileLogger.d(tag, "onAudioDataReceived: channel=$channel, size=${data.size}, count=$bleAudioRecvCount, astProvider=$currentAstProvider")
}
if (channel == 0) {
return
@ -1829,12 +1810,30 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware,
rightBuffer[monoIndex + 1] = data[stereoIndex + 3]
}
// 计算左右声道 RMS 音量(每200帧打一次)
if (bleAudioRecvCount % 200 == 1L) {
var leftRms = 0.0
var rightRms = 0.0
for (i in 0 until leftBuffer.size step 2) {
if (i + 1 < leftBuffer.size) {
val lSample = ((leftBuffer[i + 1].toInt() shl 8) or (leftBuffer[i].toInt() and 0xFF)).toShort()
val rSample = ((rightBuffer[i + 1].toInt() shl 8) or (rightBuffer[i].toInt() and 0xFF)).toShort()
leftRms += lSample.toDouble() * lSample.toDouble()
rightRms += rSample.toDouble() * rSample.toDouble()
}
}
val samples = leftBuffer.size / 2
leftRms = Math.sqrt(leftRms / samples)
rightRms = Math.sqrt(rightRms / samples)
FileLogger.d(tag, "[STS] 声道音量: left(己方/麦克风)=${leftRms.toInt()}, right(对方)=${rightRms.toInt()}, threshold=$lowVolumeThreshold")
}
// 过滤低音量音频
val filteredLeftBuffer = filterLowVolumeAudio(leftBuffer, lowVolumeThreshold)
val filteredRightBuffer = filterLowVolumeAudio(rightBuffer, lowVolumeThreshold)
// 左声道是对方的,右声道是麦的
pushAstAudioToA(filteredRightBuffer)
pushAstAudioToB(filteredLeftBuffer)
// 左声道是自己的(麦克风),右声道是对方的
pushAstAudioToA(filteredLeftBuffer)
pushAstAudioToB(filteredRightBuffer)
}

27
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DoubaoE2ETranslateHelper.kt

@ -74,15 +74,22 @@ class DoubaoE2ETranslateHelper(
/**
* 初始化助手,设置配置与回调。
*/
private var instanceTag: String = ""
fun initialize(config: Config, cb: Callback): Boolean {
Log.d(TAG, "initialize: wsUrl=${config.wsUrl}, resourceId=${config.resourceId}")
instanceTag = "${config.sourceLanguage}->${config.targetLanguage}"
Log.d(TAG, "[$instanceTag] initialize: wsUrl=${config.wsUrl}, resourceId=${config.resourceId}")
// 重置状态,确保重新初始化时能正常启动
isStarted.set(false)
try { webSocket?.close(1000, "reinitialize") } catch (_: Exception) {}
webSocket = null
conf = config
callback = cb
client = OkHttpClient.Builder()
.pingInterval(30, TimeUnit.SECONDS)
.readTimeout(0, TimeUnit.SECONDS)
.build()
Log.d(TAG, "initialize: client created")
Log.d(TAG, "[$instanceTag] initialize: client created")
startContinuousTranslation()
return true
}
@ -118,7 +125,7 @@ class DoubaoE2ETranslateHelper(
* 启动会话,建立 WebSocket 并发送 StartSession。
*/
fun startContinuousTranslation(): Boolean {
Log.d(TAG, "startContinuousTranslation: isStarted=${isStarted.get()} clientIsNull=${client==null}")
Log.d(TAG, "[$instanceTag] startContinuousTranslation: isStarted=${isStarted.get()} clientIsNull=${client==null}")
if (client == null) return false
if (isStarted.get()) return true
@ -148,12 +155,12 @@ class DoubaoE2ETranslateHelper(
* @param response 握手响应,包含状态码与头信息
*/
override fun onOpen(ws: WebSocket, response: Response) {
Log.d(TAG, "onOpen: code=${response.code} logid=${response.header("X-Tt-Logid")}")
Log.d(TAG, "[$instanceTag] onOpen: code=${response.code} logid=${response.header("X-Tt-Logid")}")
webSocket = ws
isStarted.set(true)
val startReq = makeStartRequest(sessionId)
ws.send(ByteString.of(*startReq.toByteArray()))
Log.d(TAG, "onOpen: StartSession sent")
isStarted.set(true)
callback?.onSessionStarted(sessionId)
}
@ -169,7 +176,7 @@ class DoubaoE2ETranslateHelper(
* @param bytes 服务端下发的二进制消息
*/
override fun onMessage(ws: WebSocket, bytes: ByteString) {
Log.d(TAG, "onMessage: ${bytes.size} bytes")
Log.d(TAG, "[$instanceTag] onMessage: ${bytes.size} bytes")
val resp = TranslateResponse.parseFrom(bytes.toByteArray())
val event = resp.event
val text = resp.text
@ -322,14 +329,18 @@ class DoubaoE2ETranslateHelper(
/**
* 推送一段 PCM/WAV 音频数据到服务端。
*/
private var pushCount = 0L
fun pushAudioData(data: ByteArray): Boolean {
// Log.d(TAG, "pushAudioData: size=${data.size} isStarted=${isStarted.get()} wsIsNull=${webSocket==null}")
pushCount++
if (pushCount % 100 == 1L) {
Log.d(TAG, "pushAudioData: size=${data.size} isStarted=${isStarted.get()} wsIsNull=${webSocket==null} count=$pushCount")
}
val ws = webSocket ?: return false
if (!isStarted.get()) return false
synchronized(sessionLock) {
val req = makeChunkRequest(sessionId, data)
val ok = ws.send(ByteString.of(*req.toByteArray()))
// Log.d(TAG, "pushAudioData: sent=${ok}")
return ok
}
}

10
local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt

@ -1015,10 +1015,10 @@ object BleService {
override fun onCharacteristicWrite(
g: BluetoothGatt, c: BluetoothGattCharacteristic, status: Int
) {
Log.i(
TAG,
"写入 ${if (status == BluetoothGatt.GATT_SUCCESS) "成功" else "失败"}"
)
// Log.i(
// TAG,
// "写入 ${if (status == BluetoothGatt.GATT_SUCCESS) "成功" else "失败"}"
// )
}
override fun onDescriptorWrite(
@ -1400,7 +1400,7 @@ object BleService {
isWaitingForWriteCallback = false
// recordfile1?.saveAudioDataToWav("发送成功,大小: ${chunk.size} 字节")
bytesSentInCurrentSecond += chunk.size
Log.d(TAG, "成功发送音频数据块,大小: ${chunk.size} 字节,等待写入回调...")
// Log.d(TAG, "成功发送音频数据块,大小: ${chunk.size} 字节,等待写入回调...")
} else {
isWaitingForWriteCallback = false
//recordfile1?.saveAudioDataToWav("发送失败,大小: ${chunk.size} 字节")

Loading…
Cancel
Save