Browse Source

提交端到端翻译逻辑

newdev_shunjiawei
liwei1dao 6 months ago
parent
commit
8cfb318ecb
  1. 2
      .metadata
  2. 1
      android/settings.gradle.kts
  3. 63
      lib/data/services/ast_service.dart
  4. 175
      lib/data/services/language_manager.dart
  5. 12
      lib/data/services/speech_factory.dart
  6. 220
      lib/data/services/speech_impl/azure_ast_service.dart
  7. 8
      lib/modules/speech_test/views/speech_test_view.dart
  8. 128
      lib/modules/translation/controllers/translation_controller.dart
  9. 14
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  10. 12
      local_plugins/azure_speech/.metadata
  11. 38
      local_plugins/azure_speech/android/build.gradle.kts
  12. 162
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
  13. 1038
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrToAsr.kt
  14. 1369
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt
  15. 235
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt
  16. 2
      local_plugins/azure_speech/ios/azure_speech/Package.swift
  17. 405
      local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift
  18. 2
      local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt

2
.metadata

@ -15,7 +15,7 @@ migration:
- platform: root
create_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e
base_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e
- platform: macos
- platform: android
create_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e
base_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e

1
android/settings.gradle.kts

@ -28,6 +28,7 @@ plugins {
id("com.android.application") version "8.9.1" apply false
id("com.android.library") version "8.9.1" apply false
id("org.jetbrains.kotlin.android") version "2.1.10" apply false
id("com.google.protobuf") version "0.9.4" apply false
}
include(":app")

63
lib/data/services/ast_service.dart

@ -7,19 +7,25 @@ abstract class AstService {
List<String> get supportedLanguages;
/// 初始化语音识别服务
Future<bool> initialize({required List<String> supportedLanguages});
/// [supportedLanguages] 支持的语言列表
/// [provider] 提供商:azure / volcano / alibaba / iflytek
Future<bool> initialize(
{required List<String> supportedLanguages, String provider = 'azure'});
/// 设置低音量阈值
void setLowVolumeThreshold(int threshold);
/// 开始录音
Future<bool> enableRecord(String filePath);
///
/// 开始连续翻译
Future<bool> startContinuousTranslation();
///
/// 停止连续翻译
Future<bool> stopContinuousTranslation();
/// 返回一个包含识别事件的流
Future<Stream<RecognitionEvent1>> recognizeCallback();
Future<Stream<ASTEvent>> recognizeCallback();
/// 开始录音
Future<bool> path(String filePath);
@ -29,13 +35,19 @@ abstract class AstService {
}
// 识别事件类型
enum RecognitionEventType1 {
enum ASTEventType {
/// 最终识别结果
finalResult,
/// 中间识别结果(实时反馈)
intermediateResult,
/// 翻译结果
translationResult,
/// 翻译中(实时反馈)
translationInterim,
/// 会话开始
sessionStarted,
@ -50,9 +62,15 @@ enum RecognitionEventType1 {
}
// 识别事件
class RecognitionEvent1 {
class ASTEvent {
/// 事件类型
final RecognitionEventType1 type;
final ASTEventType type;
/// 识别ID
final String utteranceId;
/// 服务ID(A/B)
final String serviceId;
/// 识别文本(仅在 finalResult 和 intermediateResult 类型中有效)
final String text;
@ -60,54 +78,55 @@ class RecognitionEvent1 {
/// 检测到的语言
final String detectedLanguage;
/// 角色
final String role;
/// 原始音频
final Uint8List? audio;
/// 错误信息(仅在 error 和 canceled 类型中有效)
final String error;
RecognitionEvent1({
ASTEvent({
required this.type,
this.utteranceId = '',
this.text = '',
this.detectedLanguage = '',
this.role = '',
this.audio,
this.error = '',
this.serviceId = '',
});
/// 创建最终结果事件的快捷构造函数
factory RecognitionEvent1.finalResult({
factory ASTEvent.finalResult({
required String text,
String utteranceId = '',
String detectedLanguage = '',
String serviceId = '',
}) {
return RecognitionEvent1(
type: RecognitionEventType1.finalResult,
return ASTEvent(
type: ASTEventType.finalResult,
utteranceId: utteranceId,
text: text,
detectedLanguage: detectedLanguage,
serviceId: serviceId,
);
}
/// 创建错误事件的快捷构造函数
factory RecognitionEvent1.error(String errorMessage) {
return RecognitionEvent1(
type: RecognitionEventType1.error,
factory ASTEvent.error(String errorMessage) {
return ASTEvent(
type: ASTEventType.error,
error: errorMessage,
);
}
/// 检查是否为最终结果
bool get isFinalResult => type == RecognitionEventType1.finalResult;
bool get isFinalResult => type == ASTEventType.finalResult;
/// 检查是否为错误
bool get isError =>
type == RecognitionEventType1.error ||
type == RecognitionEventType1.canceled;
type == ASTEventType.error || type == ASTEventType.canceled;
@override
String toString() {
return 'RecognitionEvent{type: $type, text: $text, detectedLanguage: $detectedLanguage, error: $error}';
return 'ASTEvent{type: $type, text: $text, detectedLanguage: $detectedLanguage, error: $error}';
}
}

175
lib/data/services/language_manager.dart

@ -1,5 +1,12 @@
import 'package:get/get.dart';
import '../../core/utils/logger.dart';
import '../services/speech_factory.dart';
import 'language_configs/azure_language_config.dart';
import 'language_configs/google_language_config.dart';
import 'language_configs/iflytek_language_config.dart';
import 'language_configs/volcano_language_config.dart';
import 'language_configs/alibaba_language_config.dart';
import 'language_configs/ast_config.dart';
/// 语言信息类,包含语言的各种表示形式
class LanguageInfo {
@ -529,4 +536,172 @@ class LanguageManager extends GetxService {
Map<String, String> getShortCodeToAsrCodeMap() {
return Map.from(_shortCodeToAsrCodeMap);
}
/// 返回当前所有的AST语言配置(中文名称)
List<String> getAllAstLanguageNames() {
return getAstLanguageConfigs()
.map((e) => e['chineseName'] ?? '')
.where((e) => e.isNotEmpty)
.toList();
}
/// 查找同时支持源语言和目标语言的最佳提供商(通过 shortCode 匹配)
/// 优先级:豆包(Volcano) > 阿里(Alibaba) > 微软(Azure) > 讯飞(Iflytek)
BestProviderResult? findBestMatchingProvider(
String sourceShortCode, String targetShortCode) {
final providers = [
SpeechServiceType.volcano,
SpeechServiceType.alibaba,
SpeechServiceType.azure,
SpeechServiceType.iflytek,
];
for (final provider in providers) {
final transSpecs = _getTranslationSpecs(provider);
Map<String, String>? sourceTransSpec;
Map<String, String>? targetTransSpec;
for (final spec in transSpecs) {
if (spec['shortCode'] == sourceShortCode) {
sourceTransSpec ??= {};
sourceTransSpec["translationCode"] = spec["translationCode"]!;
}
if (spec['shortCode'] == targetShortCode) {
targetTransSpec ??= {};
targetTransSpec["translationCode"] = spec["translationCode"]!;
}
}
if (sourceTransSpec == null || targetTransSpec == null) {
continue;
}
// 豆包特殊限制:源语言或目标语言必须有一个是中文或英语
if (provider == SpeechServiceType.volcano) {
final isSourceZhOrEn =
sourceShortCode == 'zh' || sourceShortCode == 'en';
final isTargetZhOrEn =
targetShortCode == 'zh' || targetShortCode == 'en';
if (!isSourceZhOrEn && !isTargetZhOrEn) {
continue;
}
}
// 对于非端到端服务(Azure, Iflytek),还需要检查 ASR 和 TTS 支持
if (provider != SpeechServiceType.volcano &&
provider != SpeechServiceType.alibaba) {
final asrSpecs = _getAsrSpecs(provider);
final ttsSpecs = _getTtsSpecs(provider);
for (final spec in asrSpecs) {
if (spec['shortCode'] == sourceShortCode) {
sourceTransSpec["asrCode"] = spec["asrCode"]!;
}
if (spec['shortCode'] == targetShortCode) {
targetTransSpec["asrCode"] = spec["asrCode"]!;
}
}
for (final spec in ttsSpecs) {
if (spec['shortCode'] == sourceShortCode) {
sourceTransSpec["ttsCode"] = spec["ttsVoiceName"]!;
}
if (spec['shortCode'] == targetShortCode) {
targetTransSpec["ttsCode"] = spec["ttsVoiceName"]!;
}
}
} else if (provider == SpeechServiceType.alibaba) {
final ttsSpecs = _getTtsSpecs(provider);
for (final spec in ttsSpecs) {
if (spec['shortCode'] == sourceShortCode) {
sourceTransSpec["ttsCode"] = spec["ttsVoiceName"]!;
}
if (spec['shortCode'] == targetShortCode) {
targetTransSpec["ttsCode"] = spec["ttsVoiceName"]!;
}
}
}
final providerString = _providerStringFromType(provider);
Logger.info(
'Found best provider: $providerString for $sourceShortCode -> $targetShortCode');
return BestProviderResult(
providerString, sourceTransSpec, targetTransSpec);
}
return null;
}
String _providerStringFromType(SpeechServiceType type) {
switch (type) {
case SpeechServiceType.iflytek:
return 'iflytek';
case SpeechServiceType.azure:
return 'azure';
case SpeechServiceType.google:
return 'google';
case SpeechServiceType.alibaba:
return 'alibaba';
case SpeechServiceType.volcano:
return 'volcano';
case SpeechServiceType.flutter:
return 'flutter';
}
}
// ===== 私有工具方法:加载不同提供商的语言规格 =====
List<Map<String, String>> _getAsrSpecs(SpeechServiceType provider) {
switch (provider) {
case SpeechServiceType.azure:
return getAzureASRLanguageSpecs();
case SpeechServiceType.google:
return getGoogleASRLanguageSpecs();
case SpeechServiceType.iflytek:
return getIflytekASRLanguageSpecs();
default:
return getAzureASRLanguageSpecs();
}
}
List<Map<String, String>> _getTranslationSpecs(SpeechServiceType provider) {
switch (provider) {
case SpeechServiceType.azure:
return getAzureTranslationLanguageSpecs();
case SpeechServiceType.google:
return getGoogleTranslationLanguageSpecs();
case SpeechServiceType.iflytek:
return getIflytekTranslationLanguageSpecs();
case SpeechServiceType.volcano:
return getVolcanoTranslationLanguageSpecs();
case SpeechServiceType.alibaba:
return getAlibabaTranslationLanguageSpecs();
default:
return getAzureTranslationLanguageSpecs();
}
}
List<Map<String, String>> _getTtsSpecs(SpeechServiceType provider) {
switch (provider) {
case SpeechServiceType.azure:
return getAzureTTSLanguageSpecs();
case SpeechServiceType.google:
return getGoogleTTSLanguageSpecs();
case SpeechServiceType.iflytek:
return getIflytekTTSLanguageSpecs();
case SpeechServiceType.alibaba:
return getAlibabaTranslationLanguageSpecs();
default:
return getAzureTTSLanguageSpecs();
}
}
}
class BestProviderResult {
final String provider;
final Map<String, String> sourceSpec;
final Map<String, String> targetSpec;
BestProviderResult(this.provider, this.sourceSpec, this.targetSpec);
}

12
lib/data/services/speech_factory.dart

@ -11,6 +11,18 @@ enum SpeechServiceType {
/// Azure 语音服务
azure,
/// 科大讯飞语音服务
iflytek,
/// Google 语音服务
google,
/// 阿里巴巴语音服务
alibaba,
/// 火山引擎(豆包)语音服务
volcano,
/// Flutter 本地语音服务
flutter,
}

220
lib/data/services/speech_impl/azure_ast_service.dart

@ -5,12 +5,6 @@ import '../../../core/utils/logger.dart';
import 'package:get/get.dart';
import '../ast_service.dart';
/// 音频源类型
enum AudioSourceType {
microphone, // 使用设备麦克风
external // 使用外部提供的音频数据
}
/// Azure 语音识别服务
///
/// 该服务提供了通过平台通道与原生 Microsoft Speech SDK 交互的接口
@ -19,16 +13,31 @@ class AzureAstService extends GetxService implements AstService {
static const MethodChannel _channel = MethodChannel('azure_speech/ast');
static const EventChannel _eventChannel =
EventChannel('azure_speech/ast_events');
// final GetStorage _storage = GetStorage();
bool _isInitialized = false;
late final String _subscriptionKey;
late final String _serviceRegion;
late String _baseUrl;
final String _endpoint = '/'; // 修改为根路径
late final String _accessKey;
late final String _secretKey;
late final String _region;
late final String _service;
late final String _azureTranslationKey;
late final String _azureTranslationRegion;
late final String _volcanoTranslationAccessKey;
late final String _volcanoTranslationSecretKey;
late final String _volcanoTranslationRegion;
late final String _xunfeiAppId;
late final String _xunfeiAccessKeyId;
late final String _xunfeiAccessKeySecret;
late final String _iflytekHost;
/// 豆包语音识别服务相关配置
late final String _doubaoAppKey;
late final String _doubaoAccessKey;
late final String _doubaoResourceId;
/// 阿里巴巴语音识别服务相关配置
late final String _alibabaAppKey;
late final String _alibabaAppId;
late final String _alibabaAppURL;
final List<String> _defaultSupportedLanguages = ['zh-CN', 'en-US'];
@override
@ -36,7 +45,7 @@ class AzureAstService extends GetxService implements AstService {
//连续识别相关
bool _isContinuousRecognitionActive = false;
StreamController<RecognitionEvent1>? _eventStreamController;
StreamController<ASTEvent>? _eventStreamController;
StreamSubscription? _eventSubscription;
// 最新的识别结果
@ -47,9 +56,6 @@ class AzureAstService extends GetxService implements AstService {
String _latestDetectedLanguage = '';
String get latestDetectedLanguage => _latestDetectedLanguage;
// 当前音频源类型
AudioSourceType _audioSourceType = AudioSourceType.microphone;
AzureAstService() {
_loadConfig();
}
@ -71,7 +77,8 @@ class AzureAstService extends GetxService implements AstService {
}, onError: _handleRecognitionError);
}
/// 处理来自原生端的识别事件
/// 处理来自原生端的识别事件(AST事件)
/// 将原生侧 AST 事件映射为统一的 ASTEvent,提供给业务层使用。
void _handleRecognitionEvent(dynamic event) {
if (event is! Map || _eventStreamController == null) return;
@ -79,40 +86,78 @@ class AzureAstService extends GetxService implements AstService {
final String eventType = eventMap['type'] as String? ?? '';
switch (eventType) {
case 'result':
case 'recognized':
print("Ast处理识别完成事件:${eventMap}");
final String serviceId = eventMap['serviceId'] as String? ?? '';
final String utteranceId = eventMap['utteranceId'] as String? ?? '';
final String text = eventMap['text'] as String? ?? '';
final String detectedLanguage =
eventMap['detectedLanguage'] as String? ?? '';
final String detectedLanguage = eventMap['language'] as String? ?? '';
_latestRecognizedText = text;
_latestDetectedLanguage = detectedLanguage;
_eventStreamController?.add(RecognitionEvent1(
type: RecognitionEventType1.finalResult,
_eventStreamController?.add(ASTEvent(
type: ASTEventType.finalResult,
serviceId: serviceId,
utteranceId: utteranceId,
text: text,
detectedLanguage: detectedLanguage,
));
break;
case 'recognizing':
print("Ast处理识别中事件:${eventMap}");
final String serviceId = eventMap['serviceId'] as String? ?? '';
final String utteranceId = eventMap['utteranceId'] as String? ?? '';
final String text = eventMap['text'] as String? ?? '';
final String detectedLanguage = eventMap['language'] as String? ?? '';
_eventStreamController?.add(ASTEvent(
type: ASTEventType.intermediateResult,
serviceId: serviceId,
utteranceId: utteranceId,
text: text,
detectedLanguage: detectedLanguage,
));
break;
case 'translatedInterim':
print("Ast处理翻译中事件:${eventMap}");
final String serviceId = eventMap['serviceId'] as String? ?? '';
final String utteranceId = eventMap['utteranceId'] as String? ?? '';
final String text = eventMap['translatedText'] as String? ?? '';
final String detectedLanguage =
eventMap['detectedLanguage'] as String? ?? '';
_eventStreamController?.add(RecognitionEvent1(
type: RecognitionEventType1.intermediateResult,
eventMap['targetLanguage'] as String? ?? '';
_eventStreamController?.add(ASTEvent(
type: ASTEventType.translationInterim,
serviceId: serviceId,
utteranceId: utteranceId,
text: text,
detectedLanguage: detectedLanguage,
));
break;
case 'translated':
print("Ast处理翻译结果事件:${eventMap}");
final String serviceId = eventMap['serviceId'] as String? ?? '';
final String utteranceId = eventMap['utteranceId'] as String? ?? '';
final String text = eventMap['translatedText'] as String? ?? '';
final String detectedLanguage =
eventMap['targetLanguage'] as String? ?? '';
_eventStreamController?.add(ASTEvent(
type: ASTEventType.translationResult,
serviceId: serviceId,
utteranceId: utteranceId,
text: text,
detectedLanguage: detectedLanguage,
));
break;
case 'sessionStarted':
Logger.info('AST provider: azure');
_eventStreamController?.add(RecognitionEvent1(
type: RecognitionEventType1.sessionStarted,
_eventStreamController?.add(ASTEvent(
type: ASTEventType.sessionStarted,
));
break;
case 'sessionStopped':
_isContinuousRecognitionActive = false;
_eventStreamController?.add(RecognitionEvent1(
type: RecognitionEventType1.sessionStopped,
_eventStreamController?.add(ASTEvent(
type: ASTEventType.sessionStopped,
));
break;
@ -125,17 +170,24 @@ class AzureAstService extends GetxService implements AstService {
Logger.error('识别取消: $reason - ${errorDetails.toString()}');
}
_eventStreamController?.add(RecognitionEvent1(
type: RecognitionEventType1.canceled,
_eventStreamController?.add(ASTEvent(
type: ASTEventType.canceled,
error: '$reason: $errorDetails',
));
break;
case 'error':
final String error = eventMap['message'] as String? ?? '';
String error = eventMap['message'] as String? ?? '';
if (error.isEmpty) {
error = eventMap['error'] as String? ?? '';
}
final String code = eventMap['code']?.toString() ?? '';
if (code.isNotEmpty) {
error = '$error (Code: $code)';
}
Logger.error('识别错误: ${error.toString()}');
_eventStreamController?.add(RecognitionEvent1(
type: RecognitionEventType1.error,
_eventStreamController?.add(ASTEvent(
type: ASTEventType.error,
error: error,
));
break;
@ -158,14 +210,28 @@ class AzureAstService extends GetxService implements AstService {
/// 从环境变量加载配置
void _loadConfig() {
// final _env = _storage.read("ENV") as Map<String, String>;
_subscriptionKey = AppConfig.env('AZURE_SPEECH_KEY') ?? '';
_serviceRegion = AppConfig.env('AZURE_SPEECH_REGION') ?? '';
_accessKey = AppConfig.env('VOLCANO_TRANSLATION_ACCESS_KEY') ?? '';
_secretKey = AppConfig.env('VOLCANO_TRANSLATION_SECRET_KEY') ?? '';
_region = AppConfig.env('VOLCANO_TRANSLATION_REGION') ?? 'cn-north-1';
_service = 'translate';
_baseUrl = 'https://translate.volcengineapi.com';
_azureTranslationKey = AppConfig.env('AZURE_TRANSLATION_KEY') ?? '';
_azureTranslationRegion = AppConfig.env('AZURE_TRANSLATION_REGION') ?? '';
_volcanoTranslationAccessKey =
AppConfig.env('VOLCANO_TRANSLATION_ACCESS_KEY') ?? '';
_volcanoTranslationSecretKey =
AppConfig.env('VOLCANO_TRANSLATION_SECRET_KEY') ?? '';
_volcanoTranslationRegion =
AppConfig.env('VOLCANO_TRANSLATION_REGION') ?? 'cn-north-1';
_xunfeiAppId = AppConfig.env('XUNFEI_ASR_APP_ID') ?? '';
_xunfeiAccessKeyId = AppConfig.env('XUNFEI_ASR_ACCESS_KEY_ID') ?? '';
_xunfeiAccessKeySecret =
AppConfig.env('XUNFEI_ASR_ACCESS_KEY_SECRET') ?? '';
_iflytekHost = AppConfig.env('IFLYTEK_ASR_HOST') ?? '';
_doubaoAppKey = AppConfig.env('VOLC_OPENSPEECH_APP_ID') ?? '';
_doubaoAccessKey = AppConfig.env('VOLC_OPENSPEECH_ACCESS_TOKEN') ?? '';
_doubaoResourceId =
AppConfig.env('VOLC_OPENSPEECH_TRANSLATION_BIGMODEL') ?? '';
_alibabaAppKey = AppConfig.env('ALIBABA_OPENSPEECH_APP_KEY') ?? '';
_alibabaAppId = AppConfig.env('ALIBABA_OPENSPEECH_APP_ID') ?? '';
_alibabaAppURL = AppConfig.env('ALIBABA_OPENSPEECH_APP_URL') ?? '';
if (_subscriptionKey.isEmpty || _serviceRegion.isEmpty) {
throw Exception(
'未找到 Azure 语音服务配置。请在 .env 文件中设置 AZURE_SPEECH_KEY 和 AZURE_SPEECH_REGION');
@ -173,12 +239,12 @@ class AzureAstService extends GetxService implements AstService {
}
@override
Future<Stream<RecognitionEvent1>> recognizeCallback() async {
Future<Stream<ASTEvent>> recognizeCallback() async {
if (!_isInitialized) {
await initialize();
}
try {
_eventStreamController = StreamController<RecognitionEvent1>.broadcast();
_eventStreamController = StreamController<ASTEvent>.broadcast();
// 开始连续识别
final bool result = await _channel.invokeMethod('recognizeCallback');
@ -250,13 +316,11 @@ class AzureAstService extends GetxService implements AstService {
@override
Future<void> dispose() async {
try {
// 取消事件订阅,关闭事件流,重置状态(补充释放,避免监听残留)
await _eventSubscription?.cancel();
_eventSubscription = null;
await _eventStreamController?.close();
_eventStreamController = null;
// 通知原生端释放资源,并重置初始化标记
await _channel.invokeMethod('dispose');
_isInitialized = false;
Logger.info('Azure AST 资源已释放');
@ -269,46 +333,47 @@ class AzureAstService extends GetxService implements AstService {
@override
Future<bool> initialize({
List<String>? supportedLanguages,
bool useExternalAudio = false,
bool useEchoCancellation = false,
String provider = 'azure',
}) async {
try {
final List<String> languages =
supportedLanguages ?? _defaultSupportedLanguages;
// 底层会初始化前释放
// // 检查是否需要重新初始化
// if (_isInitialized) {
// await dispose();
// }
// 设置音频源类型
_audioSourceType = useExternalAudio
? AudioSourceType.external
: AudioSourceType.microphone;
// Future<bool> initialize({
// required String subscriptionKey,
// required String region,
// required List<String> supportedLanguages,
// required String audioSourceType,
// required String translationAccessKey,
// required String translationSecretKey,
// String translationRegion = 'cn-north-1',
// });
if (_isInitialized) {
await dispose();
}
final bool result = await _channel.invokeMethod('initialize', {
'provider': provider,
'subscriptionKey': _subscriptionKey,
'region': _serviceRegion,
'supportedLanguages': languages,
'audioSourceType': _audioSourceType.toString().split('.').last,
'translationAccessKey': _accessKey,
'translationSecretKey': _secretKey,
'translationRegion': _region,
'volcanoTranslationAccessKey': _volcanoTranslationAccessKey,
'volcanoTranslationSecretKey': _volcanoTranslationSecretKey,
'volcanoTranslationRegion': _volcanoTranslationRegion,
'azureTranslationKey': _azureTranslationKey,
'azureTranslationServiceRegion': _azureTranslationRegion,
'xfyunAppId': _xunfeiAppId,
'xfyunAccessKeyId': _xunfeiAccessKeyId,
'xfyunAccessKeySecret': _xunfeiAccessKeySecret,
'iflytekHost': _iflytekHost,
'appKey': _doubaoAppKey,
'accessKey': _doubaoAccessKey,
'resourceId': _doubaoResourceId,
'alibabaAppKey': _alibabaAppKey,
'alibabaAppId': _alibabaAppId,
'alibabaAppURL': _alibabaAppURL,
});
_isInitialized = result;
_setupEventChannel();
print('Azure 语音识别服务初始化${result ? '成功' : '失败'}');
Logger.info('Azure 语音识别服务初始化${result ? '成功' : '失败'}');
if (result) Logger.info('AST provider: azure');
return result;
} catch (e) {
Logger.error('Azure 语音识别服务初始化失败: ${e.toString()}');
@ -316,4 +381,17 @@ class AzureAstService extends GetxService implements AstService {
rethrow;
}
}
@override
void setLowVolumeThreshold(int threshold) {
try {
_channel.invokeMethod('setLowVolumeThreshold', {
'threshold': threshold,
});
} catch (e) {
Logger.error('设置低音量阈值失败: ${e.toString()}');
rethrow;
}
}
}

8
lib/modules/speech_test/views/speech_test_view.dart

@ -101,6 +101,14 @@ class SpeechTestView extends GetView<SpeechTestController> {
switch (type) {
case SpeechServiceType.azure:
return 'Azure语音';
case SpeechServiceType.iflytek:
return '讯飞语音';
case SpeechServiceType.google:
return 'Google语音';
case SpeechServiceType.alibaba:
return '阿里语音';
case SpeechServiceType.volcano:
return '豆包语音';
case SpeechServiceType.flutter:
return 'Flutter本地';
}

128
lib/modules/translation/controllers/translation_controller.dart

@ -145,6 +145,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
// ==================== 流订阅 ====================
StreamSubscription? _recognitionSubscription;
StreamSubscription? _astEventSubscription;
StreamSubscription? _voiceInteractionSubscription;
Timer? _translationDebounceTimer;
@ -845,12 +846,35 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
_recognitionSubscription =
recognitionStream.listen(_handleRecognitionEvent);
final sourceShort = _languageManager.getShortCodeByAsrCode(sourceLanguageCode.value) ?? 'zh';
final targetShort = _languageManager.getShortCodeByAsrCode(targetLanguageCode.value) ?? 'en';
final bestProvider = _languageManager.findBestMatchingProvider(sourceShort, targetShort);
final astProvider = bestProvider?.provider ?? 'azure';
// 构建完整的语言参数:[asrCode0, asrCode1, transCode0, transCode1, ttsVoice0, ttsVoice1]
final transCode0 = bestProvider?.sourceSpec['translationCode'] ?? sourceLanguageCode.value;
final transCode1 = bestProvider?.targetSpec['translationCode'] ?? targetLanguageCode.value;
final ttsVoice0 = bestProvider?.sourceSpec['ttsCode'] ??
(_languageManager.getTtsVoiceNameByAsrCode(sourceLanguageCode.value) ?? 'zh-CN-XiaoxiaoNeural');
final ttsVoice1 = bestProvider?.targetSpec['ttsCode'] ??
(_languageManager.getTtsVoiceNameByAsrCode(targetLanguageCode.value) ?? 'en-US-AriaNeural');
final List<String> callModeLanguages = [
sourceLanguageCode.value,
targetLanguageCode.value
sourceLanguageCode.value, // [0] ASR source
targetLanguageCode.value, // [1] ASR target
transCode0, // [2] translation source
transCode1, // [3] translation target
ttsVoice0, // [4] TTS voice source
ttsVoice1, // [5] TTS voice target
];
Logger.info('1初始化AST服务,支持语言: $callModeLanguages'); //ast=asr+translate
await _astService.initialize(supportedLanguages: callModeLanguages);
Logger.info('1初始化AST服务,支持语言: $callModeLanguages, provider: $astProvider');
await _astService.initialize(supportedLanguages: callModeLanguages, provider: astProvider);
// 获取识别事件流
var astStream = await _astService.recognizeCallback();
_astEventSubscription?.cancel();
_astEventSubscription = astStream.listen(_handleAstEvent);
Logger.info('通话模式语音翻译服务初始化完成');
} catch (e) {
Logger.error('通话模式语音翻译服务初始化失败: ${e.toString()}');
@ -862,6 +886,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
stopAll();
_recognitionSubscription?.cancel();
_recognitionSubscription = null;
_astEventSubscription?.cancel();
_astEventSubscription = null;
_voiceInteractionSubscription?.cancel();
_translationDebounceTimer?.cancel();
restoreOtherServices();
@ -1062,12 +1088,27 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
/// 配置通话模式
Future<void> _configureCallMode() async {
final sourceShort = _languageManager.getShortCodeByAsrCode(sourceLanguageCode.value) ?? 'zh';
final targetShort = _languageManager.getShortCodeByAsrCode(targetLanguageCode.value) ?? 'en';
final bestProvider = _languageManager.findBestMatchingProvider(sourceShort, targetShort);
final astProvider = bestProvider?.provider ?? 'azure';
final transCode0 = bestProvider?.sourceSpec['translationCode'] ?? sourceLanguageCode.value;
final transCode1 = bestProvider?.targetSpec['translationCode'] ?? targetLanguageCode.value;
final ttsVoice0 = bestProvider?.sourceSpec['ttsCode'] ??
(_languageManager.getTtsVoiceNameByAsrCode(sourceLanguageCode.value) ?? 'zh-CN-XiaoxiaoNeural');
final ttsVoice1 = bestProvider?.targetSpec['ttsCode'] ??
(_languageManager.getTtsVoiceNameByAsrCode(targetLanguageCode.value) ?? 'en-US-AriaNeural');
final List<String> callModeLanguages = [
sourceLanguageCode.value,
targetLanguageCode.value
targetLanguageCode.value,
transCode0,
transCode1,
ttsVoice0,
ttsVoice1,
];
Logger.info('1初始化AST服务,支持语言: $callModeLanguages'); //ast=asr+translate
await _astService.initialize(supportedLanguages: callModeLanguages);
Logger.info('1初始化AST服务,支持语言: $callModeLanguages, provider: $astProvider');
await _astService.initialize(supportedLanguages: callModeLanguages, provider: astProvider);
isPreparing.value = true;
await bleManager.openA2DPDecoder();
int attempts = 0;
@ -1190,6 +1231,79 @@ class TranslationController extends GetxController with WidgetsBindingObserver {
}
}
/// 处理 AST(语音识别+翻译一体)事件
void _handleAstEvent(ASTEvent event) {
switch (event.type) {
case ASTEventType.intermediateResult:
if (event.text.isEmpty) break;
Logger.info('AST 识别中 [${event.serviceId}]: ${event.text}');
handleIntermediateResult(event.text);
break;
case ASTEventType.finalResult:
if (event.text.isEmpty) break;
Logger.info('AST 识别完成 [${event.serviceId}]: ${event.text}');
// AST 已内置翻译,直接更新源文本,不再调用 translateText
if (translationHistory.isNotEmpty &&
translationHistory.last.isIntermediate) {
translationHistory.last.sourceText = event.text;
translationHistory.last.isIntermediate = false;
translationHistory.refresh();
_scrollToBottom();
saveTranslationHistory();
} else {
final newItem = TranslationItem(
sourceText: event.text,
translatedText: '',
sourceLanguageCode: sourceLanguageCode.value,
targetLanguageCode: targetLanguageCode.value,
timestamp: DateTime.now(),
sessionId: currentSessionId ??
DateTime.now().millisecondsSinceEpoch.toString(),
isFirstInSession: translationHistory.isEmpty ||
translationHistory.last.sessionId != currentSessionId,
isIntermediate: false,
);
translationHistory.add(newItem);
translationHistory.refresh();
_scrollToBottom();
saveTranslationHistory();
}
break;
case ASTEventType.translationInterim:
if (event.text.isEmpty) break;
Logger.info('AST 翻译中 [${event.serviceId}]: ${event.text}');
if (translationHistory.isNotEmpty) {
translationHistory.last.translatedText = event.text;
translationHistory.refresh();
}
break;
case ASTEventType.translationResult:
if (event.text.isEmpty) break;
Logger.info('AST 翻译完成 [${event.serviceId}]: ${event.text}');
if (translationHistory.isNotEmpty) {
translationHistory.last.translatedText = event.text;
translationHistory.refresh();
_scrollToBottom();
saveTranslationHistory();
}
break;
case ASTEventType.error:
Logger.error('AST 错误: ${event.error}');
break;
case ASTEventType.canceled:
Logger.error('AST 取消: ${event.error}');
break;
default:
break;
}
}
/// 停止语音识别
Future<void> stopRecognition() async {
if (!isRecognizing.value) return;

14
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt

@ -298,10 +298,7 @@ object AgentService : CoroutineScope {
region = config["azureSpeechRegion"]?.toString() ?: "",
supportedLanguages = supportedLanguages,
audioSourceType = if (isExternalActive) AzureAsrHelper.AudioSourceType.EXTERNAL
else AzureAsrHelper.AudioSourceType.MICROPHONE,
xunfeiAppId = config["xunfeiAppId"]?.toString() ?: "",
xunfeiAccessKeyId = config["xunfeiAccessKeyId"]?.toString() ?: "",
xunfeiAccessKeySecret = config["xunfeiAccessKeySecret"]?.toString() ?: ""
else AzureAsrHelper.AudioSourceType.MICROPHONE
)
if (asrInitSuccess) {
@ -705,7 +702,7 @@ object AgentService : CoroutineScope {
if(isExternalActive){
isKeepResult = true
}
azureAsrHelper?.startContinuousRecognition(audioSourceType, isRemoveFirstPunctuation = false)
azureAsrHelper?.startContinuousRecognition(audioSourceType)
// 根据模式启动相应的空闲检测
startIdleCheckForMode(mode)
@ -1485,10 +1482,7 @@ object AgentService : CoroutineScope {
subscriptionKey = this.azureSpeechKey,
region = this.azureSpeechRegion,
supportedLanguages = languages.toTypedArray(),
audioSourceType = audioSourceType,
xunfeiAppId = this.xunfeiAppId,
xunfeiAccessKeyId = this.xunfeiAccessKeyId,
xunfeiAccessKeySecret = this.xunfeiAccessKeySecret
audioSourceType = audioSourceType
) ?: false
if (asrSuccess) {
recognizeCallback()
@ -1725,7 +1719,7 @@ object AgentService : CoroutineScope {
private fun sendEvent(eventName: String, data: Map<String, Any>) {
val shouldAttachAsrProvider = eventName == "recognizing" || eventName.startsWith("recognition_")
val finalData: Map<String, Any> = if (shouldAttachAsrProvider && !data.containsKey("provider")) {
val provider = azureAsrHelper?.getAsrProvider() ?: "unknown"
val provider = "azure"
data.toMutableMap().apply { put("provider", provider) }
} else {
data

12
local_plugins/azure_speech/.metadata

@ -4,7 +4,7 @@
# This file should be version controlled and should not be manually edited.
version:
revision: "ea121f8859e4b13e47a8f845e4586164519588bc"
revision: "30ade5e937e6afbd2d2c38ec53175b89b556447e"
channel: "stable"
project_type: app
@ -13,11 +13,11 @@ project_type: app
migration:
platforms:
- platform: root
create_revision: ea121f8859e4b13e47a8f845e4586164519588bc
base_revision: ea121f8859e4b13e47a8f845e4586164519588bc
- platform: ios
create_revision: ea121f8859e4b13e47a8f845e4586164519588bc
base_revision: ea121f8859e4b13e47a8f845e4586164519588bc
create_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e
base_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e
- platform: android
create_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e
base_revision: 30ade5e937e6afbd2d2c38ec53175b89b556447e
# User provided section

38
local_plugins/azure_speech/android/build.gradle.kts

@ -1,6 +1,8 @@
import com.google.protobuf.gradle.*
plugins {
id("com.android.library")
id("org.jetbrains.kotlin.android")
id("com.google.protobuf")
}
android {
@ -20,6 +22,9 @@ android {
getByName("main") {
manifest.srcFile("src/main/AndroidManifest.xml")
java.srcDirs("src/main/kotlin")
proto {
srcDir("src/main/protos")
}
}
}
@ -55,6 +60,39 @@ dependencies {
implementation("com.microsoft.cognitiveservices.speech:client-sdk:1.48.1")
implementation(project(":speech"))
implementation("com.squareup.okhttp3:okhttp:4.12.0")
implementation("com.google.protobuf:protobuf-javalite:3.25.1")
add("compileOnly", project(":ble_service"))
// gRPC
val grpcVersion = "1.57.2"
implementation("io.grpc:grpc-android:$grpcVersion")
implementation("io.grpc:grpc-okhttp:$grpcVersion")
implementation("io.grpc:grpc-protobuf-lite:$grpcVersion")
implementation("io.grpc:grpc-stub:$grpcVersion")
implementation("javax.annotation:javax.annotation-api:1.3.2")
}
protobuf {
protoc {
artifact = "com.google.protobuf:protoc:3.25.1"
}
plugins {
id("grpc") {
artifact = "io.grpc:protoc-gen-grpc-java:1.57.2"
}
}
generateProtoTasks {
all().forEach { task ->
task.builtins {
id("java") {
option("lite")
}
}
task.plugins {
id("grpc") {
option("lite")
}
}
}
}
}

162
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt

@ -42,19 +42,6 @@ class AzureAsrHelper(private val context: Context) {
private var subscriptionKey = ""
private var region = ""
private var useXunfei = false
private var xunFeiAsrHelper: XunFeiAsrHelper? = null
/// 原生日志回调,由 Plugin 层设置,通过 EventChannel 回传 Dart
var nativeLogCallback: ((String, String) -> Unit)? = null
private var xunfeiAppId: String = ""
private var xunfeiAccessKeyId: String = ""
private var xunfeiAccessKeySecret: String = ""
fun getAsrProvider(): String {
return if (useXunfei) "xunfei" else "azure"
}
// 音频源配置
var audioSourceType = AudioSourceType.MICROPHONE
@ -102,10 +89,7 @@ class AzureAsrHelper(private val context: Context) {
subscriptionKey: String,
region: String,
supportedLanguages: Array<String> = arrayOf("zh-CN", "en-US"),
audioSourceType: AudioSourceType = AudioSourceType.MICROPHONE,
xunfeiAppId: String = "",
xunfeiAccessKeyId: String = "",
xunfeiAccessKeySecret: String = ""
audioSourceType: AudioSourceType = AudioSourceType.MICROPHONE
): Boolean {
try {
// 检查配置是否为空
@ -121,33 +105,12 @@ class AzureAsrHelper(private val context: Context) {
this.subscriptionKey = subscriptionKey
this.region = region
this.audioSourceType = audioSourceType
this.xunfeiAppId = xunfeiAppId
this.xunfeiAccessKeyId = xunfeiAccessKeyId
this.xunfeiAccessKeySecret = xunfeiAccessKeySecret
// 设置语言
if (supportedLanguages.isNotEmpty()) {
this.supportedLanguages = supportedLanguages
}
val initMsg = "[XunFei_ASR] ASR initialize: supportedLanguages=${supportedLanguages.joinToString(",")}, audioSourceType=$audioSourceType"
Log.i(tag, initMsg)
nativeLogCallback?.invoke("INFO", initMsg)
val xunfeiConfigReady = this.xunfeiAppId.isNotBlank() &&
this.xunfeiAccessKeyId.isNotBlank() &&
this.xunfeiAccessKeySecret.isNotBlank()
val onlySupportedByXunfei = supportedLanguages.all {
it == "zh-CN" || it == "en-US" || it == "zh-HK" || it == "zh-TW"
}
val chineseVariantsCount = supportedLanguages
.map { it.trim().lowercase() }
.filter { it == "zh" || it.startsWith("zh-") }
.distinct()
.size
val hasMultipleChineseVariants = chineseVariantsCount >= 2
this.useXunfei = onlySupportedByXunfei && xunfeiConfigReady && !hasMultipleChineseVariants
// 根据支持的语言数量决定是否启用自动语言检测
this.isAutoDetectLanguage = supportedLanguages.size >= 2
@ -156,10 +119,6 @@ class AzureAsrHelper(private val context: Context) {
this.currentLanguage = supportedLanguages[0]
}
val derivedMsg = "[XunFei_ASR] ASR config: provider=${getAsrProvider()}, xunfeiConfigured=$xunfeiConfigReady, onlyXunfeiLangs=$onlySupportedByXunfei, multiChinese=$hasMultipleChineseVariants, autoDetect=$isAutoDetectLanguage, currentLang=$currentLanguage"
Log.i(tag, derivedMsg)
nativeLogCallback?.invoke("INFO", derivedMsg)
// 创建语音配置
speechConfig = SpeechConfig.fromSubscription(subscriptionKey, region).apply {
if (isAutoDetectLanguage) {
@ -170,16 +129,6 @@ class AzureAsrHelper(private val context: Context) {
speechRecognitionLanguage = currentLanguage
}
setProperty("SpeechServiceConnection_EndSilenceTimeoutMs", "300")
setProperty("Speech_SegmentationSilenceTimeoutMs", "300")
// 优化:添加低延迟连接配置
setProperty("SpeechServiceConnection_InitialSilenceTimeoutMs", "200")
setProperty("SpeechServiceConnection_RecoMode", "INTERACTIVE")
setProperty("Speech_PushStreamFormat", "PCM")
// 设置分段策略为时间模式
setProperty("Speech_SegmentationStrategy", "Time")
}
// 初始化网络监听
networkMonitor.initialize(object : NetworkStateMonitor.NetworkStateListener {
@ -274,13 +223,11 @@ class AzureAsrHelper(private val context: Context) {
* 开始连续语音识别
* @param audioSourceType 音频源类型
* @param audioDataCallback 音频数据回调
* @param isRemoveFirstPunctuation 是否移除第一个标点符号
* @return 是否成功开始识别
*/
fun startContinuousRecognition(
audioSourceType: AudioSourceType = AudioSourceType.MICROPHONE,
audioDataCallback: SimpleAudioReceiver.AudioDataCallback? = null,
isRemoveFirstPunctuation: Boolean = true
audioDataCallback: SimpleAudioReceiver.AudioDataCallback? = null
): Boolean {
Log.d(tag, "startContinuousRecognition:${audioStream.isContinuousRecognitionActive} ")
@ -309,73 +256,6 @@ class AzureAsrHelper(private val context: Context) {
}
}
if (useXunfei) {
Log.d(tag, "Using Xunfei Recognition")
val callback = continuousCallback
if (callback == null) {
Log.e(tag, "无法启动连续识别:回调为空")
continuousCallback?.onError(currsessionid, 1001, "回调为空")
return false
}
if (xunFeiAsrHelper == null) {
xunFeiAsrHelper = XunFeiAsrHelper(
context = context,
appId = xunfeiAppId,
accessKeyId = xunfeiAccessKeyId,
accessKeySecret = xunfeiAccessKeySecret
)
xunFeiAsrHelper?.logCallback = nativeLogCallback
nativeLogCallback?.invoke("INFO", "[XunFei_ASR] Config init (callback ready): appId=${xunfeiAppId.take(6)}***, keyId=${xunfeiAccessKeyId.take(6)}***, secret=${if (xunfeiAccessKeySecret.isNotBlank()) "SET" else "EMPTY"}")
} else {
xunFeiAsrHelper?.updateConfig(
appId = xunfeiAppId,
accessKeyId = xunfeiAccessKeyId,
accessKeySecret = xunfeiAccessKeySecret
)
}
try {
Log.d(tag, "startContinuousRecognition: ${audioStream.isContinuousRecognitionActive}")
xunFeiAsrHelper?.start(
callback,
currentLanguage,
supportedLanguages.toList(),
isAutoDetectLanguage,
isRemoveFirstPunctuation
)
this.audioSourceType = audioSourceType
if (!audioStream.isContinuousRecognitionActive) {
// 确保音频流已初始化
if (!audioStream.isInitialized()) {
audioStream.initAudioRecord()
audioStream.setupStream()
}
audioStream.startAudioRecord(
when (audioSourceType) {
AudioSourceType.MICROPHONE -> SimpleAudioReceiver.AudioSourceType.MICROPHONE
AudioSourceType.EXTERNAL -> SimpleAudioReceiver.AudioSourceType.EXTERNAL
},
object : SimpleAudioReceiver.AudioDataCallback {
override fun onAudio(data: ByteArray) {
audioDataCallback?.onAudio(data)
xunFeiAsrHelper?.sendAudio(data)
}
}
)
}
audioStream.isContinuousRecognitionActive = true
return true
} catch (e: Exception) {
Log.e(tag, "Start Xunfei failed: ${e.message}")
xunFeiAsrHelper?.stop()
audioStream.isContinuousRecognitionActive = false
return false
}
}
if (!isRecognizerValid()) {
Log.d(tag, "重新启动连续识别")
val callback = continuousCallback
@ -479,10 +359,11 @@ class AzureAsrHelper(private val context: Context) {
EventHandler<SessionEventArgs> { _, _ ->
// 直接在当前线程调用回调
Log.d(tag, "会话结束事件")
// if (audioSourceType == AudioSourceType.EXTERNAL) {
callback.onSessionStopped(currsessionid)
if (audioSourceType == AudioSourceType.EXTERNAL) {
audioStream?.isContinuousRecognitionActive = false
// }
}
}
)
@ -510,27 +391,6 @@ class AzureAsrHelper(private val context: Context) {
* @return 是否成功停止
*/
fun stopContinuousRecognition(): Boolean {
if (useXunfei) {
try {
xunFeiAsrHelper?.stop()
stopAudioProcessing()
audioStream.isContinuousRecognitionActive = false
if (isForegroundServiceRunning) {
try {
AudioRecordingForegroundService.stopService(context)
isForegroundServiceRunning = false
} catch (e: Exception) {
Log.e(tag, "停止前台服务失败: ${e.message}")
}
}
return true
} catch (e: Exception) {
Log.e(tag, "Stop Xunfei failed: ${e.message}")
return false
}
}
if (speechConfig == null) {
Log.e(tag, "语音服务未初始化")
return false
@ -595,12 +455,8 @@ class AzureAsrHelper(private val context: Context) {
try {
// 如果正在进行连续识别,先停止
if (audioStream.isContinuousRecognitionActive) {
if (useXunfei) {
xunFeiAsrHelper?.stop()
} else {
// 直接停止,不等待结果
recognizer?.stopContinuousRecognitionAsync()
}
audioStream.isContinuousRecognitionActive = false
}
// 停止音频处理
@ -614,9 +470,6 @@ class AzureAsrHelper(private val context: Context) {
recognizer?.close()
recognizer = null
xunFeiAsrHelper?.stop()
xunFeiAsrHelper = null
// 释放speechConfig
speechConfig?.close()
speechConfig = null
@ -908,7 +761,10 @@ class AzureAsrHelper(private val context: Context) {
// 防抖机制:如果距离上次触发时间小于防抖间隔,则忽略本次触发
if (currentTime - lastNetworkLostTime < networkLostDebounceInterval) {
Log.d(tag, "网络丢失事件被防抖过滤,距离上次触发仅${currentTime - lastNetworkLostTime}ms")
Log.d(
tag,
"网络丢失事件被防抖过滤,距离上次触发仅${currentTime - lastNetworkLostTime}ms"
)
return
}

1038
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrToAsr.kt

File diff suppressed because it is too large

1369
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt

File diff suppressed because it is too large

235
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt

@ -14,8 +14,9 @@ import com.deep_voice.speech.tts.TtsEventType
import kotlinx.coroutines.*
import java.io.ByteArrayInputStream
import java.io.InputStream
import java.util.concurrent.atomic.AtomicInteger
import com.yunqiinnovation.azure_speech.tools.SimpleAudioPlayer
import java.util.concurrent.TimeUnit
/**
* Azure TTS Helper
*
@ -35,7 +36,8 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
private var isInitialized = false
private var isSpeaking = false
// 当前配置
// 最近一次合成的原始文本,用于在合成完成时打印
private var lastSynthesisText: String = ""
private var currentVoice = DEFAULT_VOICE
private var currentRate = "0%"
private var currentPitch = "0%"
@ -52,9 +54,6 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
private val streamBuffer = StringBuilder()
private var lastSpeakTime = 0L
// 待完成的合成任务计数:只有全部完成才触发 PLAYBACK_COMPLETED
private val pendingSpeakCount = AtomicInteger(0)
private var enableVoiceFlocking = false
private var speakerProfileId: String = ""
@ -86,9 +85,9 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
// 确保使用正确的语音输出
speechConfig?.setProperty("SPEECH-AudioOutputFormat", "riff-16khz-16bit-mono-pcm")
// 优化:设置低延迟连接属性
speechConfig?.setProperty("SpeechServiceConnection_InitialSilenceTimeoutMs", "300")
speechConfig?.setProperty("SpeechServiceConnection_EndSilenceTimeoutMs", "300")
// // 优化:设置低延迟连接属性
// speechConfig?.setProperty("SpeechServiceConnection_InitialSilenceTimeoutMs", "300")
// speechConfig?.setProperty("SpeechServiceConnection_EndSilenceTimeoutMs", "300")
speechConfig?.setSpeechSynthesisVoiceName(currentVoice)
@ -96,6 +95,7 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
FileLogger.d(TAG, "语音设置: voice=$currentVoice, format=Riff16Khz16BitMonoPcm")
// 创建音频配置
val audioConfig = if (customAudioOutputStream != null) {
// 使用自定义音频输出流
AudioConfig.fromStreamOutput(customAudioOutputStream)
@ -103,7 +103,6 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
// 使用默认扬声器
AudioConfig.fromDefaultSpeakerOutput()
}
// 创建合成器
synthesizer = SpeechSynthesizer(speechConfig, audioConfig)
@ -278,6 +277,7 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
/**
* 设置事件监听器
*/
// 方法:setupEventListeners(设置事件监听器)
private fun setupEventListeners() {
synthesizer?.apply {
// 合成开始事件
@ -290,36 +290,41 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
// 合成中事件(接收音频数据)
Synthesizing?.addEventListener { _, eventArgs ->
val audioData = eventArgs.result.audioData
FileLogger.d(TAG, "接收到音频数据: ${audioData.size} 字节")
// 通知音频数据监听器
if (audioData.isNotEmpty()) {
val listeners = ArrayList(audioDataListeners)
for (listener in listeners) {
try {
listener.onAudioData(audioData)
} catch (e: Exception) {
FileLogger.e(TAG, "通知音频数据异常: ${e.message}")
}
}
}
// // 打印详细的音频数据信息
// if (audioData.isNotEmpty()) {
// printAudioDataDetails(audioData)
// }
}
// 合成完成事件
SynthesisCompleted?.addEventListener { _, eventArgs ->
FileLogger.d(
TAG,
"语音合成完成: resultId=${eventArgs.result.resultId}, 音频长度=${eventArgs.result.audioLength} 字节"
"语音合成完成: resultId=${eventArgs.result.resultId}, 音频长度=${eventArgs.result.audioLength} 字节, 文本=${lastSynthesisText}"
)
// val audioData = eventArgs.result.audioData
// // 通知音频数据监听器
// if (audioData.isNotEmpty()) {
// val listeners = ArrayList(audioDataListeners)
// for (listener in listeners) {
// try {
// listener.onAudioData(audioData)
// } catch (e: Exception) {
// FileLogger.e(TAG, "通知音频数据异常: ${e.message}")
// }
// }
// }
isSpeaking = false
notifyEvent(TtsEventType.SYNTHESIS_COMPLETED)
// 所有句子都播完后才触发 PLAYBACK_COMPLETED,避免多句流式合成时重复触发
if (pendingSpeakCount.decrementAndGet() <= 0) {
pendingSpeakCount.set(0)
// Azure TTS 合成完成即播放完成
notifyEvent(TtsEventType.PLAYBACK_COMPLETED)
}
}
// 合成取消事件
SynthesisCanceled?.addEventListener { _, eventArgs ->
@ -551,6 +556,7 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
/**
* 单次播放文本(非流式)
*/
// 方法:speakOnce(单次播放文本)
override fun speakOnce(text: String): Boolean {
if (!isInitialized) {
FileLogger.e(TAG, "TTS引擎未初始化")
@ -564,13 +570,15 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
}
try {
// 记录当前合成文本,便于在完成事件打印
lastSynthesisText = text
// 优化:使用简化的SSML生成
val ssml = generateOptimizedSsml(text)
isSpeaking = true
pendingSpeakCount.incrementAndGet()
FileLogger.d(TAG, "开始语音合成,文本长度: ${text.length},当前待完成任务: ${pendingSpeakCount.get()}")
FileLogger.d(TAG, "开始语音合成,文本长度: ${text.length}")
// 直接调用SpeakSsmlAsync,SDK内部已经是异步的
synthesizer?.SpeakSsmlAsync(ssml)
@ -702,14 +710,18 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
* 停止当前语音合成
*/
override fun stop(): Boolean {
/**
* 停止当前语音合成
*
* 说明:避免在停止过程中立即重建合成器,降低与 SDK 异步回调的竞态风险。
*/
if (!isInitialized) return false
try {
synthesizer?.StopSpeakingAsync()
?.get(1500, TimeUnit.MILLISECONDS)
streamBuffer.clear()
isSpeaking = false
pendingSpeakCount.set(0)
recreateSynthesizer()
return true
} catch (e: Exception) {
FileLogger.e(TAG, "停止语音合成失败: ${e.message}")
@ -771,4 +783,169 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
* 是否正在播放
*/
fun isSpeaking(): Boolean = isSpeaking
/**
* 打印音频数据的详细信息
*
* @param audioData 音频数据字节数组
*/
private fun printAudioDataDetails(audioData: ByteArray) {
try {
val dataSize = audioData.size
// 基本信息
FileLogger.d(TAG, "=== 音频数据详情 ===")
FileLogger.d(TAG, "数据大小: $dataSize 字节")
if (dataSize > 0) {
// 计算音频时长(基于16KHz, 16bit, 单声道)
val sampleRate = 16000 // Hz
val bitsPerSample = 16
val channels = 1
val bytesPerSecond = sampleRate * (bitsPerSample / 8) * channels
val durationMs = (dataSize.toDouble() / bytesPerSecond * 1000).toInt()
FileLogger.d(TAG, "预估音频时长: ${durationMs}ms")
// 打印前32字节的十六进制数据
val hexPreview = audioData.take(32).joinToString(" ") {
"%02X".format(it)
}
FileLogger.d(TAG, "前32字节数据(HEX): $hexPreview")
// 音频数据统计
val maxValue = audioData.maxOrNull()?.toInt() ?: 0
val minValue = audioData.minOrNull()?.toInt() ?: 0
val avgValue = audioData.map { it.toInt() }.average().toInt()
FileLogger.d(
TAG,
"数据统计 - 最大值: $maxValue, 最小值: $minValue, 平均值: $avgValue"
)
// 检查是否为有效的PCM数据(非全零)
val nonZeroCount = audioData.count { it != 0.toByte() }
val nonZeroPercentage = (nonZeroCount.toDouble() / dataSize * 100).toInt()
FileLogger.d(TAG, "非零字节占比: $nonZeroPercentage% ($nonZeroCount/$dataSize)")
// 如果数据量较大,还可以打印尾部数据
if (dataSize > 64) {
val hexTail = audioData.takeLast(16).joinToString(" ") {
"%02X".format(it)
}
FileLogger.d(TAG, "后16字节数据(HEX): $hexTail")
}
// 分析音频格式特征
analyzeAudioFormat(audioData)
}
FileLogger.d(TAG, "=== 音频数据详情结束 ===")
} catch (e: Exception) {
FileLogger.e(TAG, "打印音频数据详情失败: ${e.message}")
}
}
/**
* 分析音频格式特征
*
* @param audioData 音频数据字节数组
*/
private fun analyzeAudioFormat(audioData: ByteArray) {
try {
// 检查是否包含WAV文件头
if (audioData.size >= 12) {
val header = audioData.take(12).toByteArray()
val headerStr = String(header, Charsets.US_ASCII)
if (headerStr.startsWith("RIFF") && headerStr.contains("WAVE")) {
FileLogger.d(TAG, "检测到WAV格式文件头")
// 解析WAV文件头信息
if (audioData.size >= 44) {
parseWavHeader(audioData)
}
} else {
FileLogger.d(TAG, "原始PCM数据(无文件头)")
// 分析PCM数据特征
analyzePcmData(audioData)
}
} else {
FileLogger.d(TAG, "数据量太小,无法分析格式")
}
} catch (e: Exception) {
FileLogger.e(TAG, "分析音频格式失败: ${e.message}")
}
}
/**
* 解析WAV文件头信息
*
* @param audioData 包含WAV文件头的音频数据
*/
private fun parseWavHeader(audioData: ByteArray) {
try {
// WAV文件头结构解析(小端序)
val channels =
(audioData[22].toInt() and 0xFF) or ((audioData[23].toInt() and 0xFF) shl 8)
val sampleRate = (audioData[24].toInt() and 0xFF) or
((audioData[25].toInt() and 0xFF) shl 8) or
((audioData[26].toInt() and 0xFF) shl 16) or
((audioData[27].toInt() and 0xFF) shl 24)
val bitsPerSample =
(audioData[34].toInt() and 0xFF) or ((audioData[35].toInt() and 0xFF) shl 8)
FileLogger.d(TAG, "WAV格式信息:")
FileLogger.d(TAG, " 声道数: $channels")
FileLogger.d(TAG, " 采样率: ${sampleRate}Hz")
FileLogger.d(TAG, " 位深度: ${bitsPerSample}bit")
} catch (e: Exception) {
FileLogger.e(TAG, "解析WAV文件头失败: ${e.message}")
}
}
/**
* 分析PCM数据特征
*
* @param audioData PCM音频数据
*/
private fun analyzePcmData(audioData: ByteArray) {
try {
// 分析16位PCM数据的振幅分布
if (audioData.size >= 2) {
val samples = mutableListOf<Short>()
// 将字节数组转换为16位采样点(小端序)
for (i in 0 until audioData.size - 1 step 2) {
val sample = ((audioData[i].toInt() and 0xFF) or
((audioData[i + 1].toInt() and 0xFF) shl 8)).toShort()
samples.add(sample)
}
if (samples.isNotEmpty()) {
val maxAmplitude = samples.maxOrNull() ?: 0
val minAmplitude = samples.minOrNull() ?: 0
val avgAmplitude = samples.map { it.toInt() }.average().toInt()
FileLogger.d(TAG, "PCM数据分析:")
FileLogger.d(TAG, " 采样点数: ${samples.size}")
FileLogger.d(TAG, " 最大振幅: $maxAmplitude")
FileLogger.d(TAG, " 最小振幅: $minAmplitude")
FileLogger.d(TAG, " 平均振幅: $avgAmplitude")
// 计算动态范围
val dynamicRange = maxAmplitude - minAmplitude
FileLogger.d(TAG, " 动态范围: $dynamicRange")
}
}
} catch (e: Exception) {
FileLogger.e(TAG, "分析PCM数据失败: ${e.message}")
}
}
}

2
local_plugins/azure_speech/ios/azure_speech/Package.swift

@ -25,7 +25,7 @@ let package = Package(
"MicrosoftCognitiveServicesSpeech"
],
path: "Sources", // 修改为包含整个 Sources 目录
sources: ["azure_speech/", "tools/"] // 明确指定包含的子目录
sources: ["azure_speech/", "tools/","protos_swift/"] // 明确指定包含的子目录
),
// 使用binaryTarget引用Azure Speech SDK的.xcframework
.binaryTarget(

405
local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift

@ -93,6 +93,19 @@ import ble_service
internal var astEventSink: FlutterEventSink?
private let azureAstHelper = IntegratedSpeechTranslationService()
// 端到端翻译 Helpers(豆包 / 阿里百炼)
private var doubaoAstHelperA = DoubaoE2ETranslateHelper()
private var doubaoAstHelperB = DoubaoE2ETranslateHelper()
private var bailianAstHelperA = AliyunBailianE2EHelper()
private var bailianAstHelperB = AliyunBailianE2EHelper()
private var currentAstProvider: String = "azure"
private var astWsConfig: DoubaoE2ETranslateHelper.Config?
internal var doubaoFinalSourceTextCache: [String: String] = [:]
private var doubaoCallbackA: DoubaoCallbackProxy?
private var doubaoCallbackB: DoubaoCallbackProxy?
private var bailianCallbackA: AliyunCallbackProxy?
private var bailianCallbackB: AliyunCallbackProxy?
// 音频数据相关
private var audioDataEventChannel: FlutterEventChannel?
internal var audioDataEventSink: FlutterEventSink?
@ -747,6 +760,15 @@ private func sendAstEvent(_ event: [String: Any]) {
// }
case "startContinuousTranslation":
if currentAstProvider == "volcano" {
_ = doubaoAstHelperA.startContinuousTranslation()
_ = doubaoAstHelperB.startContinuousTranslation()
result(true)
} else if currentAstProvider == "alibaba" {
_ = bailianAstHelperA.startContinuousConversation()
_ = bailianAstHelperB.startContinuousConversation()
result(true)
} else {
do {
os_log("开启翻译", log: log, type: .info)
try azureAstHelper.startContinuousTranslation()
@ -754,8 +776,18 @@ private func sendAstEvent(_ event: [String: Any]) {
} catch {
result(FlutterError(code: "START_CONTINUOUS_TRANSLATION_ERROR", message: error.localizedDescription, details: nil))
}
}
case "stopContinuousTranslation":
if currentAstProvider == "volcano" {
_ = doubaoAstHelperA.stopContinuousTranslation()
_ = doubaoAstHelperB.stopContinuousTranslation()
result(true)
} else if currentAstProvider == "alibaba" {
_ = bailianAstHelperA.stopContinuousConversation()
_ = bailianAstHelperB.stopContinuousConversation()
result(true)
} else {
do {
os_log("停止翻译", log: log, type: .info)
try azureAstHelper.stopContinuousTranslation()
@ -763,15 +795,16 @@ private func sendAstEvent(_ event: [String: Any]) {
} catch {
result(FlutterError(code: "STOP_CONTINUOUS_TRANSLATION_ERROR", message: error.localizedDescription, details: nil))
}
}
case "dispose":
do {
os_log("释放AST资源", log: log, type: .info)
azureAstHelper.dispose()
doubaoAstHelperA.dispose()
doubaoAstHelperB.dispose()
bailianAstHelperA.dispose()
bailianAstHelperB.dispose()
result(true)
} catch {
result(FlutterError(code: "DISPOSE_ERROR", message: error.localizedDescription, details: nil))
}
case "recognizeCallback":
os_log("recognizeCallback:", log: log, type: .info)
@ -785,66 +818,116 @@ private func sendAstEvent(_ event: [String: Any]) {
}
let fileURL = URL(fileURLWithPath: filePath)
// if FileManager.default.fileExists(atPath: filePath) {
// // 启动异步任务来按播放速度读取音频文件
// Task {
// // await streamAudioFile(fileURL: fileURL)
// }
// result(true)
// } else {
// result(FlutterError(code: "FILE_NOT_FOUND", message: "音频文件不存在: \(filePath)", details: nil))
// }
result(true)
case "initialize":
guard let args = call.arguments as? [String: Any] else {
result(FlutterError(code: "INVALID_ARGUMENTS", message: "Missing arguments", details: nil))
return
}
let subscriptionKey = args["subscriptionKey"] as? String ?? ""
let region = args["region"] as? String ?? ""
let supportedLanguages = args["supportedLanguages"] as? [String] ?? ["zh-CN"]
let useExternalAudio = args["useExternalAudio"] as? Bool ?? false
// 获取翻译服务配置参数
let translationAccessKey = args["translationAccessKey"] as? String ?? ""
let translationSecretKey = args["translationSecretKey"] as? String ?? ""
let translationRegion = args["translationRegion"] as? String ?? "cn-north-1"
os_log("初始化AST服务", log: log, type: .info)
// 创建Azure配置
let azureConfig = AzureConfiguration(
subscriptionKey: subscriptionKey,
region: region
let provider = (args["provider"] as? String ?? "azure").lowercased()
let supported = (args["supportedLanguages"] as? [String]) ?? ["zh-CN"]
let lang0 = supported.count > 0 ? supported[0] : "zh-CN"
let lang1 = supported.count > 1 ? supported[1] : "en-US"
let translationLang0 = supported.count > 2 ? supported[2] : "zh-Hans"
let translationLang1 = supported.count > 3 ? supported[3] : "en-US"
let ttsLang0 = supported.count > 4 ? supported[4] : "zh-CN-XiaoxiaoNeural"
let ttsLang1 = supported.count > 5 ? supported[5] : "en-US-AriaNeural"
currentAstProvider = provider
let subscriptionKeyI = (args["subscriptionKey"] as? String) ?? ""
let regionI = (args["region"] as? String) ?? ""
let volcanoTranslationAccessKey = (args["volcanoTranslationAccessKey"] as? String) ?? ""
let volcanoTranslationSecretKey = (args["volcanoTranslationSecretKey"] as? String) ?? ""
let volcanoTranslationRegion = (args["volcanoTranslationRegion"] as? String) ?? "cn-north-1"
let azureTranslationKey = (args["azureTranslationKey"] as? String) ?? ""
let azureTranslationRegion = (args["azureTranslationServiceRegion"] as? String) ?? ""
// 豆包 AST WS 配置
let wsUrl = (args["wsUrl"] as? String) ?? "wss://openspeech.bytedance.com/api/v4/ast/v2/translate"
let appKey = (args["appKey"] as? String) ?? ""
let accessKeyD = (args["accessKey"] as? String) ?? ""
let resourceId = (args["resourceId"] as? String) ?? "volc.service_type.10053"
// 阿里百炼配置
let alibabaAppKey = (args["alibabaAppKey"] as? String) ?? ""
let alibabaAppId = (args["alibabaAppId"] as? String) ?? ""
let alibabaAppURL = (args["alibabaAppURL"] as? String) ?? ""
os_log("initialize provider=%@ lang0=%@ lang1=%@ trans0=%@ trans1=%@", log: log, type: .info, provider, lang0, lang1, translationLang0, translationLang1)
if currentAstProvider == "azure" {
let svcA = IntegratedSpeechTranslationService.ServiceConfiguration(
sourceLanguage: lang0,
targetLanguage: lang1,
currentVoice: ttsLang1,
translationSourceLanguage: translationLang0,
translationTargetLanguage: translationLang1
)
// 创建翻译配置
let translationConfig = TranslationConfiguration(
accessKey: translationAccessKey,
secretKey: translationSecretKey,
region: translationRegion
let svcB = IntegratedSpeechTranslationService.ServiceConfiguration(
sourceLanguage: lang1,
targetLanguage: lang0,
currentVoice: ttsLang0,
translationSourceLanguage: translationLang1,
translationTargetLanguage: translationLang0
)
let callbackA = AstEventCallback(plugin: self)
let callbackB = AstEventCallback(plugin: self)
astEventCallback = callbackA
// 创建服务配置
let serviceConfig = IntegratedSpeechTranslationService.ServiceConfiguration(
sourceLanguage: supportedLanguages.first ?? "zh-CN",
targetLanguage: supportedLanguages.count > 1 ? supportedLanguages[1] : "en-US"
)
let azureConfigI = AzureConfiguration(subscriptionKey: subscriptionKeyI, region: regionI)
let translationConfigI = TranslationConfiguration(region: azureTranslationRegion, subscriptionKey: azureTranslationKey, location: azureTranslationRegion)
// 创建事件回调
let callback = AstEventCallback(plugin: self)
astEventCallback = callback
Task {
await azureAstHelper.initialize(azureConfig: azureConfigI, translationConfig: translationConfigI, serviceConfig: svcA, callback: callbackA)
}
} else if currentAstProvider == "volcano" {
// 初始化豆包 AST(双路)
let cfg = DoubaoE2ETranslateHelper.Config(wsUrl: wsUrl, appKey: appKey, accessKey: accessKeyD, resourceId: resourceId, sourceLanguage: translationLang0, targetLanguage: translationLang1, clientWaitMs: 60_000)
astWsConfig = cfg
let cbA = DoubaoCallbackProxy(plugin: self, serviceId: "A", direction: "\(translationLang0)->\(translationLang1)", targetLanguage: translationLang1)
let cbB = DoubaoCallbackProxy(plugin: self, serviceId: "B", direction: "\(translationLang1)->\(translationLang0)", targetLanguage: translationLang0)
doubaoCallbackA = cbA
doubaoCallbackB = cbB
var cfgA = cfg
cfgA.sourceLanguage = translationLang0
cfgA.targetLanguage = translationLang1
var cfgB = cfg
cfgB.sourceLanguage = translationLang1
cfgB.targetLanguage = translationLang0
// 使用异步任务调用初始化方法
Task {
await azureAstHelper.initialize(
azureConfig: azureConfig,
translationConfig: translationConfig,
serviceConfig: serviceConfig,
callback: callback
)
_ = await doubaoAstHelperA.initialize(config: cfgA, cb: cbA)
try? await Task.sleep(nanoseconds: 500_000_000)
_ = await doubaoAstHelperB.initialize(config: cfgB, cb: cbB)
}
} else if currentAstProvider == "alibaba" {
// 初始化阿里百炼(双路)
var cfgA = AliyunBailianE2EHelper.Config()
cfgA.wsUrl = alibabaAppURL
cfgA.apiKey = alibabaAppKey
cfgA.appId = "qwen3-livetranslate-flash-realtime"
cfgA.sampleRate = 16000
cfgA.sourceLanguage = translationLang0
cfgA.targetLanguage = translationLang1
cfgA.voice = ttsLang1
var cfgB = AliyunBailianE2EHelper.Config()
cfgB.wsUrl = alibabaAppURL
cfgB.apiKey = alibabaAppKey
cfgB.appId = "qwen3-livetranslate-flash-realtime"
cfgB.sampleRate = 16000
cfgB.sourceLanguage = translationLang1
cfgB.targetLanguage = translationLang0
cfgB.voice = ttsLang0
let cbA = AliyunCallbackProxy(plugin: self, serviceId: "A", direction: "\(translationLang0)->\(translationLang1)", targetLanguage: translationLang1)
let cbB = AliyunCallbackProxy(plugin: self, serviceId: "B", direction: "\(translationLang1)->\(translationLang0)", targetLanguage: translationLang0)
bailianCallbackA = cbA
bailianCallbackB = cbB
Task {
_ = await bailianAstHelperA.initialize(config: cfgA, cb: cbA)
_ = await bailianAstHelperB.initialize(config: cfgB, cb: cbB)
}
}
result(true)
@ -853,6 +936,14 @@ private func sendAstEvent(_ event: [String: Any]) {
}
}
/// 发送AST事件到Flutter层
internal func sendAstEvent(_ event: [String: Any]) {
if astEventSink == nil { return }
DispatchQueue.main.async { [weak self] in
self?.astEventSink?(event)
}
}
}
// MARK: - AST Event Callback
/**
@ -1141,14 +1232,15 @@ extension AzureSpeechPlugin: BleService.Callback {
azureAsrHelper.audioStream?.saveAudioDataTo(data: rightBuffer)
// // 安全解包版本
// guard let audioProcessor = azureAstHelper.audioProcessor else {
// os_log("音频流未初始化", type: .error)
// return
// }
// print("liwei--------------接收到音频数据回调1")
// 根据当前 AST 提供商路由音频数据
if currentAstProvider == "volcano" {
doubaoAstHelperA.pushAudioData(leftBuffer)
} else if currentAstProvider == "alibaba" {
bailianAstHelperA.pushAudioData(leftBuffer)
} else {
azureAstHelper.pushAudioData(audioData: leftBuffer)
}
}
}
@ -1233,6 +1325,203 @@ private class BaseEventStreamHandler: NSObject, FlutterStreamHandler {
// MARK: - Doubao AST 回调桥接
private class DoubaoCallbackProxy: DoubaoE2ETranslateHelper.Callback {
private weak var plugin: AzureSpeechPlugin?
private let serviceId: String
private let direction: String
private let targetLanguage: String
init(plugin: AzureSpeechPlugin, serviceId: String, direction: String, targetLanguage: String) {
self.plugin = plugin
self.serviceId = serviceId
self.direction = direction
self.targetLanguage = targetLanguage
}
func onSessionStarted(sessionId: String) {
plugin?.sendAstEvent([
"type": "serviceInitialized",
"serviceId": serviceId,
"direction": direction
])
}
func onPartialSourceText(sessionId: String, text: String) {
plugin?.sendAstEvent([
"type": "recognizing",
"serviceId": serviceId,
"text": text,
"language": direction,
"utteranceId": sessionId
])
}
func onFinalSourceText(sessionId: String, finalText: String) {
if let p = plugin { p.doubaoFinalSourceTextCache["\(serviceId):\(sessionId)"] = finalText }
plugin?.sendAstEvent([
"type": "recognized",
"serviceId": serviceId,
"text": finalText,
"language": direction,
"utteranceId": sessionId
])
}
func onPartialText(sessionId: String, text: String) {
plugin?.sendAstEvent([
"type": "translatedInterim",
"serviceId": serviceId,
"translatedText": text,
"targetLanguage": targetLanguage,
"utteranceId": sessionId
])
}
func onPartialAudio(sessionId: String, data: Data) {
let isRightChannel = (serviceId == "A")
if data.count > 0 {
if isRightChannel {
BleService.shared.writeExternalRightAudioData(data)
} else {
BleService.shared.writeExternalLeftAudioData(data)
}
}
}
func onSessionFinished(sessionId: String, finalText: String, finalAudio: Data) {
let key = "\(serviceId):\(sessionId)"
let original = plugin?.doubaoFinalSourceTextCache.removeValue(forKey: key) ?? ""
plugin?.sendAstEvent([
"type": "translated",
"serviceId": serviceId,
"translatedText": finalText,
"targetLanguage": targetLanguage,
"utteranceId": sessionId,
"originalText": original
])
}
func onSessionError(sessionId: String, code: Int, message: String) {
plugin?.sendAstEvent([
"type": "error",
"serviceId": serviceId,
"direction": direction,
"component": "doubaoAst",
"error": message,
"code": code
])
}
func onFinalTranslatedText(sessionId: String, finalText: String) {
let key = "\(serviceId):\(sessionId)"
let original = plugin?.doubaoFinalSourceTextCache.removeValue(forKey: key) ?? ""
plugin?.sendAstEvent([
"type": "translated",
"serviceId": serviceId,
"translatedText": finalText,
"targetLanguage": targetLanguage,
"utteranceId": sessionId,
"originalText": original
])
}
}
// MARK: - Aliyun Bailian AST 回调桥接
private class AliyunCallbackProxy: AliyunBailianE2EHelper.Callback {
private weak var plugin: AzureSpeechPlugin?
private let serviceId: String
private let direction: String
private let targetLanguage: String
init(plugin: AzureSpeechPlugin, serviceId: String, direction: String, targetLanguage: String) {
self.plugin = plugin
self.serviceId = serviceId
self.direction = direction
self.targetLanguage = targetLanguage
}
func onSessionStarted(sessionId: String) {
plugin?.sendAstEvent([
"type": "serviceInitialized",
"serviceId": serviceId,
"direction": direction
])
}
func onPartialSourceText(sessionId: String, text: String) {
plugin?.sendAstEvent([
"type": "recognizing",
"serviceId": serviceId,
"text": text,
"language": direction,
"utteranceId": sessionId
])
}
func onFinalSourceText(sessionId: String, finalText: String) {
plugin?.sendAstEvent([
"type": "recognized",
"serviceId": serviceId,
"text": finalText,
"language": direction,
"utteranceId": sessionId
])
}
func onPartialText(sessionId: String, text: String) {
plugin?.sendAstEvent([
"type": "translatedInterim",
"serviceId": serviceId,
"translatedText": text,
"targetLanguage": targetLanguage,
"utteranceId": sessionId
])
}
func onPartialAudio(sessionId: String, data: Data) {
let isRightChannel = (serviceId == "A")
if data.count > 0 {
if isRightChannel {
BleService.shared.writeExternalRightAudioData(data)
} else {
BleService.shared.writeExternalLeftAudioData(data)
}
}
}
func onSessionFinished(sessionId: String, finalText: String, finalAudio: Data) {
plugin?.sendAstEvent([
"type": "translated",
"serviceId": serviceId,
"translatedText": finalText,
"targetLanguage": targetLanguage,
"utteranceId": sessionId
])
}
func onSessionError(sessionId: String, code: Int, message: String) {
plugin?.sendAstEvent([
"type": "error",
"serviceId": serviceId,
"direction": direction,
"component": "aliyunAst",
"error": message,
"code": code
])
}
func onFinalTranslatedText(sessionId: String, finalText: String) {
plugin?.sendAstEvent([
"type": "translated",
"serviceId": serviceId,
"translatedText": finalText,
"targetLanguage": targetLanguage,
"utteranceId": sessionId
])
}
}
// MARK: - TTS 事件监听实现
extension AzureSpeechPlugin: TtsEventListener {
// 使用@objc特性为方法提供一个不同的Objective-C选择器名称

2
local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt

@ -953,7 +953,7 @@ object BleService {
override fun onCharacteristicChanged(g: BluetoothGatt, c: BluetoothGattCharacteristic) {
val data = c.value ?: return
Log.i(TAG, "ble指令----------- 收到通知特征数据(命令和控制 uuid=${c.uuid}")
// Log.i(TAG, "0000abc2-0001-1111-2222-123456789abc(命令和控制 uuid=${c.uuid}")
// 根据特征UUID区分处理
when (c.uuid) {
// 音频特征数据

Loading…
Cancel
Save