diff --git a/apps/client/lib/data/services/ast_service.dart b/apps/client/lib/data/services/ast_service.dart index 22f91c20..16068666 100644 --- a/apps/client/lib/data/services/ast_service.dart +++ b/apps/client/lib/data/services/ast_service.dart @@ -9,8 +9,15 @@ abstract class AstService { /// 初始化语音识别服务 /// [supportedLanguages] 支持的语言列表 /// [provider] 提供商：azure / volcano / alibaba / iflytek - Future initialize( - {required List supportedLanguages, String provider = 'azure'}); + Future initialize({ + required List supportedLanguages, + String provider = 'azure', + bool singleLeg = false, + bool bidirectional = false, + }); + + /// 当前原生会话是不是按这组语言建的。没建过、或语言对变了，返回 false。 + bool acceptsLanguages(List languages); /// 设置低音量阈值 void setLowVolumeThreshold(int threshold); diff --git a/apps/client/lib/data/services/device_ai_session_service.dart b/apps/client/lib/data/services/device_ai_session_service.dart index 52bc22b3..b261b3c0 100644 --- a/apps/client/lib/data/services/device_ai_session_service.dart +++ b/apps/client/lib/data/services/device_ai_session_service.dart @@ -19,8 +19,11 @@ import '../utils/cue_tone.dart'; import '../utils/peer_speech_gate.dart' show rmsOfPcm16; import 'agent_module_defaults.dart'; import 'assistant_directive_service.dart'; +import 'ai_vendor_lock.dart'; import 'bailian_multimodal_service.dart'; import 'bailian_native_dialog.dart'; +import 'emai_iflytek_dialog.dart'; +import 'sparkchain_session.dart'; import 'emai_chat_store.dart'; import 'mcp_token_service.dart'; @@ -59,6 +62,7 @@ class DeviceAiSessionService extends GetxService { /// ⚠️ 两条路**不会同时开**：[_useNative] 是本次会话的唯一开关， /// 两边都起来的话上行会被推两份，服务端 VAD 会把它当成一直有人在说话。 BailianNativeDialog? _native; + EmaiIflytekDialog? _iflytek; bool get _useNative => _native != null; StreamSubscription? _nativeDownSub; StreamSubscription? _nativeMicSub; @@ -301,6 +305,10 @@ class DeviceAiSessionService extends GetxService { /// 起本次对话。两条路径的参数**必须一致**，所以只在这一处写。 Future _startDialog() async { + if (aiOnlyIflytek) { + _iflytek ??= EmaiIflytekDialog(source: 'earphone'); + return _iflytek!.start(); + } final token = await McpTokenService.get(); if (_useNative) { return _native!.start( @@ -395,7 +403,12 @@ class DeviceAiSessionService extends GetxService { } // 门控期间推等长静音而不是断流：上行保持连续，服务端 VAD 收到的是 // "真的没人说话"，而不是一段数据缺口 - _micAccum.add(_uplinkGated ? Uint8List(pcm.length) : pcm); + final gated = _uplinkGated ? Uint8List(pcm.length) : pcm; + if (_iflytek != null) { + _iflytek!.pushAudio(gated); + return; + } + _micAccum.add(gated); if (_micAccum.length >= _micBatchBytes) { _svc.pushAudio(_micAccum.takeBytes()); } @@ -430,7 +443,13 @@ class DeviceAiSessionService extends GetxService { // 起不来就别应答，直接让设备退出 AI 模式——应答之后设备会一直送音频， // 而我们这边没有会话接，它会白等到超时 - if (!_svc.isConfigured) { + if (aiOnlyIflytek) { + if (!await SparkChainSession.ensure()) { + Logger.e(_tag, '讯飞未就绪，拒绝本次唤醒'); + await _sendExit(session); + return; + } + } else if (!_svc.isConfigured) { Logger.e(_tag, '百炼未配置(${_svc.configHint})，拒绝本次唤醒'); await _sendExit(session); return; @@ -487,9 +506,15 @@ class DeviceAiSessionService extends GetxService { // ★ 先试原生直连：上行 PCM 与合成音频都不经过 Dart。 // 插件老 / 设备不是恒玄 → 返回 null，整条链路退回下面的 Dart 路径。 - _native = await _tryOpenNativeDialog(session); + _native = aiOnlyIflytek ? null : await _tryOpenNativeDialog(session); try { - if (_useNative) { + if (aiOnlyIflytek) { + _iflytek ??= EmaiIflytekDialog(source: 'earphone'); + _mic = await session.openMic(route: AudioRoute.ai); + _micSub = _mic!.pcm.listen(_onMicPcm); + _eventSub = _iflytek!.events.listen(_onBailianEvent); + _audioSub = _iflytek!.audioOut.listen(_onDownstreamPcm); + } else if (_useNative) { _eventSub = _native!.events.listen(_onBailianEvent); _nativeDownSub = _native!.downstreamBytes.listen(_onNativeDownstreamTotal); // 设备上行有声只用来续空闲计时，不能当 speechStarted（那会丢弃待播队列） @@ -836,6 +861,10 @@ class DeviceAiSessionService extends GetxService { : (zh ? _greetingsZh : _greetingsEn); final text = list[Random().nextInt(list.length)]; Logger.w(_tag, '唤醒问候: $text'); + if (_iflytek != null) { + unawaited(_iflytek!.speak(text)); + return; + } _ownRequestPending = true; if (_useNative && _native != null) { unawaited(_native!.sendText(text, speakDirectly: true)); @@ -1138,6 +1167,8 @@ class DeviceAiSessionService extends GetxService { } } } + await _iflytek?.stop(); + _iflytek = null; await _svc.stop(); final s = _session; diff --git a/apps/client/lib/data/services/emai_iflytek_dialog.dart b/apps/client/lib/data/services/emai_iflytek_dialog.dart new file mode 100644 index 00000000..af4046d7 --- /dev/null +++ b/apps/client/lib/data/services/emai_iflytek_dialog.dart @@ -0,0 +1,392 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:get/get.dart'; +import 'package:get_storage/get_storage.dart'; +import 'package:http/http.dart' as http; + +import '../../core/utils/logger.dart'; +import '../models/appconfig.dart'; +import '../models/assistant_directive.dart'; +import 'bailian_multimodal_service.dart'; +import 'device_location_service.dart'; +import 'emai_iflytek_turn.dart'; +import 'language_manager.dart'; +import 'mcp_token_service.dart'; +import 'sparkchain_session.dart'; + +/// 耳机唤醒和按住说话共用的一轮编排：讯飞听写 → 文本对话 → 需要时手机调工具 → 合成。 +/// +/// 麦和喇叭不在这里。调用方把 PCM 推进来，把 [audioOut] 播出去。 +class EmaiIflytekDialog { + EmaiIflytekDialog({this.source = 'emai'}); + + static const _tag = 'EmaiIflytek'; + static const upstreamSampleRate = 16000; + static const downstreamSampleRate = 16000; + + final String source; + final state = BailianDialogState.idle.obs; + final _events = StreamController.broadcast(); + final _audioOut = StreamController.broadcast(); + final List _history = []; + + Stream get events => _events.stream; + Stream get audioOut => _audioOut.stream; + + bool _listening = false; + int _gen = 0; + bool _inSpeech = false; + + Future start() async { + if (!await SparkChainSession.ensure()) return false; + SparkChainSession.asrListener = _onAsr; + final ok = await SparkChainSession.startMsftListen(_asrCode()); + if (!ok) { + SparkChainSession.asrListener = null; + return false; + } + _listening = true; + state.value = BailianDialogState.listening; + _events.add(const BailianEvent(BailianEventType.started)); + _events.add(BailianEvent( + BailianEventType.stateChanged, + state: BailianDialogState.listening, + )); + return true; + } + + void pushAudio(Uint8List pcm) { + if (!_listening || pcm.isEmpty) return; + SparkChainSession.write(pcm); + } + + Future stop() async { + _gen++; + _listening = false; + SparkChainSession.asrListener = null; + await SparkChainSession.stopListen(); + await SparkChainSession.stopSpeak(); + state.value = BailianDialogState.idle; + _events.add(const BailianEvent(BailianEventType.closed)); + } + + /// 问候语。不进对话模型，直接合成。 + Future speak(String text) async { + final gen = _gen; + await _say(text, gen); + } + + void _onAsr(Map event) { + if (event['kind'] == 'error') { + Logger.w(_tag, '听写失败 code=${event['code']}'); + return; + } + final text = (event['text'] as String?)?.trim() ?? ''; + final isFinal = event['final'] == true; + if (text.isEmpty) return; + if (!_inSpeech) { + _inSpeech = true; + _events.add(const BailianEvent(BailianEventType.speechStarted)); + } + _events.add(BailianEvent( + BailianEventType.speechContent, + text: text, + isFinal: isFinal, + )); + if (!isFinal) return; + _inSpeech = false; + _events.add(BailianEvent(BailianEventType.speechEnded, text: text)); + unawaited(_answer(text)); + } + + Future _answer(String userText) async { + final gen = ++_gen; + await SparkChainSession.stopSpeak(); + if (EmaiTurn.asksWhere(userText)) { + await DeviceLocationService.maybe?.ensureSpokenPlace(); + } + if (gen != _gen) return; + state.value = BailianDialogState.thinking; + _events.add(BailianEvent( + BailianEventType.stateChanged, + state: BailianDialogState.thinking, + )); + final chinese = _asrCode().toLowerCase().startsWith('zh'); + final overseas = _overseas(); + final tools = EmaiTurn.toolsJson(overseas: overseas); + var query = userText; + var fresh = true; + String spoken = ''; + for (var round = 0; round < 2; round++) { + if (gen != _gen) return; + final SparkLlmReply reply; + try { + reply = await SparkChainSession.chat( + query: query, + system: EmaiTurn.systemPrompt( + now: DateTime.now(), + spokenPlace: DeviceLocationService.maybe?.placeText(), + answerLanguage: chinese ? '中文' : 'English', + history: List.from(_history), + ), + tools: tools, + fresh: fresh, + ); + } catch (e) { + Logger.w(_tag, '对话失败: $e'); + await _say(EmaiTurn.unavailable(chinese), gen); + return; + } + if (gen != _gen) return; + final calls = EmaiTurn.parseCalls(reply.functionCall); + if (calls.isEmpty) { + spoken = reply.text.trim(); + break; + } + final call = calls.first; + if (call.name == 'quit') { + _events.add(BailianEvent( + BailianEventType.respondingContent, + text: reply.text, + directives: [ + AssistantDirective( + id: 'quit-$gen', + name: 'quit', + args: const {}, + rawArguments: '{}', + dialogId: '', + roundId: '$gen', + llmRequestId: '', + source: source, + createdAt: DateTime.now(), + ), + ], + )); + final bye = reply.text.trim().isEmpty + ? (chinese ? '好的' : 'Okay') + : reply.text.trim(); + await _say(bye, gen); + return; + } + final result = await _runTool(call); + if (gen != _gen) return; + if (result == null) { + await _say(EmaiTurn.unavailable(chinese), gen); + return; + } + fresh = false; + query = '工具 ${call.name} 的结果如下，只根据它回答，不要编造：\n$result'; + } + if (spoken.isEmpty) { + await _say(EmaiTurn.unavailable(chinese), gen); + return; + } + _history.add(EmaiMemory(true, userText)); + _history.add(EmaiMemory(false, spoken)); + if (_history.length > 8) { + _history.removeRange(0, _history.length - 8); + } + await _say(spoken, gen); + } + + Future _say(String text, int gen) async { + if (text.isEmpty || gen != _gen) return; + state.value = BailianDialogState.responding; + _events.add(const BailianEvent(BailianEventType.respondingStarted)); + _events.add(BailianEvent( + BailianEventType.respondingContent, + text: text, + isFinal: true, + finishReason: 'stop', + )); + try { + await SparkChainSession.speakNeural( + voice: _voice(), + text: text, + onPcm: (pcm, _) { + if (gen != _gen || pcm.isEmpty) return; + _audioOut.add(pcm); + }, + ); + } catch (e) { + Logger.w(_tag, '合成失败: $e'); + } + if (gen != _gen) return; + _events.add(const BailianEvent(BailianEventType.respondingEnded)); + state.value = BailianDialogState.listening; + _events.add(BailianEvent( + BailianEventType.stateChanged, + state: BailianDialogState.listening, + )); + } + + Future _runTool(EmaiToolCall call) async { + final now = DateTime.now(); + final args = Map.from(call.args); + if (call.name == 'hefeng_weather') { + final city = (args['city'] ?? '').toString().trim(); + if (city.isEmpty) { + final cached = _city(); + if (cached.isEmpty) return '没有位置，请向用户询问要查哪个城市，不要编造天气。'; + args['city'] = cached; + } + if ((args['time'] ?? '').toString().trim().isEmpty) { + args['time'] = EmaiTurn.dayText(now); + } + } + if (call.name == 'tavily_web_search' || call.name == 'bocha_web_search') { + args['query'] = EmaiTurn.searchQuery( + (args['query'] ?? '').toString(), + now, + ); + } + final authed = EmaiTurn.authTools.contains(call.name); + String? token; + if (authed) { + token = await McpTokenService.get(); + if (token == null || token.isEmpty) return null; + } + final url = _mcpUrl(call.name); + if (url == null) return null; + try { + final resp = await http + .post( + url, + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json, text/event-stream', + if (token != null) 'Authorization': 'Bearer $token', + }, + body: jsonEncode({ + 'jsonrpc': '2.0', + 'id': 1, + 'method': 'tools/call', + 'params': {'name': call.name, 'arguments': args}, + }), + ) + .timeout(const Duration(seconds: 12)); + if (resp.statusCode < 200 || resp.statusCode >= 300) return null; + return _toolText(resp.body); + } catch (e) { + Logger.w(_tag, '工具 ${call.name} 失败'); + return null; + } + } + + String? _toolText(String body) { + final payloads = []; + for (final line in body.split('\n')) { + final trimmed = line.trim(); + if (trimmed.startsWith('data:')) { + payloads.add(trimmed.substring(5).trim()); + } + } + if (payloads.isEmpty) payloads.add(body); + for (final payload in payloads.reversed) { + if (payload.isEmpty) continue; + try { + final decoded = jsonDecode(payload); + if (decoded is! Map) continue; + if (decoded['error'] != null) return null; + final result = decoded['result']; + if (result is! Map) continue; + if (result['isError'] == true) return null; + final content = result['content']; + if (content is List) { + final texts = content + .whereType() + .map((m) => (m['text'] ?? '').toString()) + .where((s) => s.isNotEmpty) + .join('\n'); + if (texts.isNotEmpty) return texts; + } + final text = result['text']; + if (text is String && text.isNotEmpty) return text; + } catch (_) {} + } + return null; + } + + Uri? _mcpUrl(String tool) { + if (!AppConfig.isInitialized()) return null; + final raw = AppConfig.mcpConfig(); + final servers = raw is Map ? raw['mcpServers'] : null; + if (servers is! Map) return null; + String? fallback; + for (final value in servers.values) { + if (value is! Map && value.runtimeType.toString() != 'DBMCPServer') { + // generated model + } + final url = _serverUrl(value); + final tools = _serverTools(value); + if (url == null) continue; + fallback ??= url; + if (tools.contains(tool)) return Uri.parse(url); + } + if (fallback == null) return null; + return Uri.parse(fallback); + } + + String? _serverUrl(Object value) { + try { + final url = (value as dynamic).url?.toString().trim() ?? ''; + if (url.isEmpty) return null; + if (url.contains('/mcp')) return url; + return url.endsWith('/') ? '${url}mcp' : '$url/mcp'; + } catch (_) { + return null; + } + } + + String _serverTools(Object value) { + try { + return ((value as dynamic).tools ?? '').toString(); + } catch (_) { + return ''; + } + } + + String _city() { + final place = DeviceLocationService.maybe; + if (place == null) return ''; + final loc = place.clientLocation(); + return (loc?['city_name'] ?? '').trim(); + } + + bool _overseas() { + final raw = AppConfig.cred('sparkchain', 'api_type', 'SPARKCHAIN_API_TYPE'); + final apiType = int.tryParse(raw ?? '') ?? 1; + return apiType != 0; + } + + String _asrCode() { + final saved = GetStorage().read('language') ?? 'system'; + if (saved != 'system') { + final parts = saved.split('_'); + if (parts.length == 2) { + final lang = parts[0].toLowerCase(); + final country = parts[1].toUpperCase(); + if (lang == 'tl' && country == 'PH') return 'fil-PH'; + return '$lang-$country'; + } + } + final locale = Get.locale ?? Get.deviceLocale; + if (locale != null && (locale.countryCode ?? '').isNotEmpty) { + return '${locale.languageCode.toLowerCase()}-${locale.countryCode!.toUpperCase()}'; + } + return 'en-US'; + } + + String _voice() { + final code = _asrCode(); + if (Get.isRegistered()) { + final voice = Get.find().getTtsVoiceNameByAsrCode(code); + if (voice != null && voice.isNotEmpty) return voice; + } + return code.toLowerCase().startsWith('zh') + ? 'zh-CN-XiaoxiaoNeural' + : 'en-US-AriaNeural'; + } +} diff --git a/apps/client/lib/data/services/emai_iflytek_turn.dart b/apps/client/lib/data/services/emai_iflytek_turn.dart new file mode 100644 index 00000000..7f57ecb6 --- /dev/null +++ b/apps/client/lib/data/services/emai_iflytek_turn.dart @@ -0,0 +1,190 @@ +import 'dart:convert'; + +/// 一轮 EMAI 对话里和 SDK 无关的部分：系统提示、工具清单、工具参数。 +class EmaiTurn { + EmaiTurn._(); + + static const authTools = [ + 'search_meeting_notes', + 'get_memory_items', + 'get_memory_stats', + 'get_memory_report', + 'get_user_tasks', + 'add_user_task', + 'cancel_user_task', + ]; + + static String clockText(DateTime t) { + const week = ['星期一', '星期二', '星期三', '星期四', '星期五', '星期六', '星期日']; + String two(int v) => v.toString().padLeft(2, '0'); + final offset = t.timeZoneOffset; + final sign = offset.isNegative ? '-' : '+'; + final hours = offset.inHours.abs(); + final mins = offset.inMinutes.abs() % 60; + final zone = mins == 0 ? 'UTC$sign$hours' : 'UTC$sign$hours:${two(mins)}'; + return '${t.year}年${t.month}月${t.day}日 ${week[t.weekday - 1]} ' + '${two(t.hour)}:${two(t.minute)}，$zone'; + } + + static String dayText(DateTime t) { + String two(int v) => v.toString().padLeft(2, '0'); + return '${t.year}-${two(t.month)}-${two(t.day)}'; + } + + /// 没有能念的地址时用这句。不要写「未知」，模型会把它当成地名。 + static String placeLine(String? spoken) { + final text = (spoken ?? '').trim(); + if (text.isEmpty || text == '未知') return '当前没有位置'; + return text; + } + + static bool asksWhere(String text) { + final s = text.toLowerCase(); + return s.contains('在哪') || + s.contains('我的位置') || + s.contains('where am i') || + s.contains('where i am'); + } + + static String unavailable(bool chinese) => + chinese ? '暂时查不到' : "I can't look that up right now."; + + static String systemPrompt({ + required DateTime now, + required String? spokenPlace, + required String answerLanguage, + required List history, + }) { + final lines = history + .map((m) => '${m.user ? "用户" : "助手"}：${m.text}') + .join('\n'); + return ''' +你是 EAIMAR 的语音助手。用$answerLanguage回答，句子短，适合念出来。 +当前时间：${clockText(now)} +当前位置：${placeLine(spokenPlace)} +时间、星期、日期、地点只用上面这两行，不要调用工具，也不要改口。 +没有位置时说还没有位置，不要编一个城市，不要念经纬度。 +天气用 hefeng_weather，城市用上面的城市名，日期用今天。有城市就不要反问哪个城市。没有位置时才问一次城市。 +新闻和需要联网的问题用搜索工具，搜索词里带上今天的日期 ${dayText(now)}。 +会议、记忆、待办用对应的个人工具。没有就说没有，不要编。 +用户要结束对话时调用 quit。 +${lines.isEmpty ? '' : '最近对话：\n$lines'} +'''.trim(); + } + + static String toolsJson({required bool overseas}) { + final search = overseas ? 'tavily_web_search' : 'bocha_web_search'; + final tools = [ + _fn('hefeng_weather', '按城市名查询天气', { + 'city': _str('城市名，例如北京'), + 'time': _str('日期 yyyy-MM-dd，可省略'), + }, const ['city']), + _fn(search, '联网搜索新闻和实时信息', { + 'query': _str('搜索词，要带上日期'), + 'count': {'type': 'integer', 'description': '条数，1到5'}, + }, const ['query']), + _fn('search_meeting_notes', '查用户自己的会议纪要', { + 'keywords': _str('关键词，可空'), + }, const []), + _fn('get_memory_items', '查用户记下的事项', { + 'category': _str('todo、alarm、idea、expense，可空'), + }, const []), + _fn('get_memory_stats', '查记忆的数量统计', {}, const []), + _fn('get_memory_report', '查记忆的汇总', {}, const []), + _fn('get_user_tasks', '查提醒和待办', {}, const []), + _fn('add_user_task', '添加一条提醒', { + 'task_name': _str('标题'), + 'trigger_time': _str('带时区的 ISO 8601 时间'), + }, const ['task_name', 'trigger_time']), + _fn('cancel_user_task', '取消一条提醒', { + 'task_id': {'type': 'integer', 'description': '任务 id'}, + }, const ['task_id']), + _fn('quit', '用户要结束这次对话', {}, const []), + ]; + return jsonEncode(tools); + } + + static Map _str(String description) => { + 'type': 'string', + 'description': description, + }; + + static Map _fn( + String name, + String description, + Map properties, + List required, + ) { + return { + 'type': 'function', + 'function': { + 'name': name, + 'description': description, + 'parameters': { + 'type': 'object', + 'properties': properties, + 'required': required, + }, + }, + }; + } + + /// SDK 的 functionCall 可能是对象、对象数组，或 arguments 再包一层字符串。 + static List parseCalls(String raw) { + final text = raw.trim(); + if (text.isEmpty) return const []; + try { + final decoded = jsonDecode(text); + if (decoded is List) { + return decoded.map(_one).whereType().toList(); + } + if (decoded is Map) { + final one = _one(decoded); + return one == null ? const [] : [one]; + } + } catch (_) {} + return const []; + } + + static EmaiToolCall? _one(Object? value) { + if (value is! Map) return null; + final map = value.map((k, v) => MapEntry('$k', v)); + final fn = map['function']; + final nested = fn is Map ? fn.map((k, v) => MapEntry('$k', v)) : map; + final name = (nested['name'] ?? map['name'] ?? '').toString().trim(); + if (name.isEmpty) return null; + final args = _args(nested['arguments'] ?? map['arguments']); + args.remove('auth_token'); + args.remove('uid'); + return EmaiToolCall(name, args); + } + + static Map _args(Object? value) { + if (value is Map) return value.map((k, v) => MapEntry('$k', v)); + if (value is String && value.trim().isNotEmpty) { + try { + final decoded = jsonDecode(value); + if (decoded is Map) return decoded.map((k, v) => MapEntry('$k', v)); + } catch (_) {} + } + return {}; + } + + static String searchQuery(String query, DateTime now) { + final day = dayText(now); + if (query.contains(day)) return query; + return '$query $day'.trim(); + } +} + +class EmaiMemory { + EmaiMemory(this.user, this.text); + final bool user; + final String text; +} + +class EmaiToolCall { + EmaiToolCall(this.name, this.args); + final String name; + final Map args; +} diff --git a/apps/client/lib/data/services/language_configs/azure_language_config.dart b/apps/client/lib/data/services/language_configs/azure_language_config.dart index 35e2f4d9..5a846fe1 100644 --- a/apps/client/lib/data/services/language_configs/azure_language_config.dart +++ b/apps/client/lib/data/services/language_configs/azure_language_config.dart @@ -8,6 +8,18 @@ List> getAzureASRLanguageSpecs() { 'englishName': 'Chinese', 'asrCode': 'zh-CN', }, + { + 'shortCode': 'zh-Hant', + 'chineseName': '中文(繁体)', + 'englishName': 'Chinese (traditional)', + 'asrCode': 'zh-TW', + }, + { + 'shortCode': 'zh-Hant-hk', + 'chineseName': '中文(香港繁体)', + 'englishName': 'Chinese (Hongkong traditional)', + 'asrCode': 'zh-HK', + }, { 'shortCode': 'yue', @@ -457,10 +469,10 @@ List> getAzureASRLanguageSpecs() { 'asrCode': 'sr-RS', }, { - 'shortCode': 'sv', + 'shortCode': 'sw', 'chineseName': '斯瓦希里语', - 'englishName': 'Swedish', - 'asrCode': 'sv-SE', + 'englishName': 'Swahili', + 'asrCode': 'sw-KE', }, { 'shortCode': 'ta', @@ -512,6 +524,18 @@ List> getAzureTranslationLanguageSpecs() { 'englishName': 'Chinese', 'translationCode': 'zh-Hans', }, + { + 'shortCode': 'zh-Hant', + 'chineseName': '中文(繁体)', + 'englishName': 'Chinese (traditional)', + 'translationCode': 'zh-Hant', + }, + { + 'shortCode': 'zh-Hant-hk', + 'chineseName': '中文(香港繁体)', + 'englishName': 'Chinese (Hongkong traditional)', + 'translationCode': 'zh-Hant', + }, { 'shortCode': 'yue', @@ -960,10 +984,10 @@ List> getAzureTranslationLanguageSpecs() { 'translationCode': 'sr', }, { - 'shortCode': 'sv', + 'shortCode': 'sw', 'chineseName': '斯瓦希里语', - 'englishName': 'Swedish', - 'translationCode': 'sv', + 'englishName': 'Swahili', + 'translationCode': 'sw', }, { 'shortCode': 'ta', @@ -1014,6 +1038,18 @@ List> getAzureTTSLanguageSpecs() { 'englishName': 'Chinese', 'ttsVoiceName': 'zh-CN-XiaoxiaoNeural', }, + { + 'shortCode': 'zh-Hant', + 'chineseName': '中文(繁体)', + 'englishName': 'Chinese (traditional)', + 'ttsVoiceName': 'zh-TW-HsiaoChenNeural', + }, + { + 'shortCode': 'zh-Hant-hk', + 'chineseName': '中文(香港繁体)', + 'englishName': 'Chinese (Hongkong traditional)', + 'ttsVoiceName': 'zh-HK-HiuMaanNeural', + }, { 'shortCode': 'yue', @@ -1488,11 +1524,10 @@ List> getAzureTTSLanguageSpecs() { 'ttsVoiceName': 'sr-Latn-RS-SophieNeural', }, { - 'shortCode': 'sv', + 'shortCode': 'sw', 'chineseName': '斯瓦希里语', - 'englishName': 'Swedish', - 'ttsVoiceName': - 'sw-KE-ZuriNeural', + 'englishName': 'Swahili', + 'ttsVoiceName': 'sw-KE-ZuriNeural', }, { 'shortCode': 'ta', diff --git a/apps/client/lib/data/services/language_manager.dart b/apps/client/lib/data/services/language_manager.dart index 37136720..6546d1bf 100644 --- a/apps/client/lib/data/services/language_manager.dart +++ b/apps/client/lib/data/services/language_manager.dart @@ -294,7 +294,7 @@ class LanguageManager extends GetxService { ); _addLanguage( - shortCode: 'no', + shortCode: 'nb', chineseName: '挪威语', englishName: 'Norwegian', asrCode: 'nb-NO', @@ -653,7 +653,7 @@ class LanguageManager extends GetxService { return _getVoiceByGender(_byShortCode[shortCode]); } - /// 微软文本翻译要用的语言码。表里没有的（繁体、挪威语）按微软翻译的写法补。 + /// 微软文本翻译要用的语言码。表里没有时，繁体和旧短码 no 仍按微软写法补。 String microsoftTranslateCode(String shortCode) { for (final spec in getAzureTranslationLanguageSpecs()) { final code = spec['translationCode']; @@ -738,13 +738,10 @@ class LanguageManager extends GetxService { BestProviderResult? findBestMatchingProvider( String sourceShortCode, String targetShortCode) { final providers = [ - SpeechServiceType.alibaba35, - SpeechServiceType.alibaba, SpeechServiceType.azure, - SpeechServiceType.iflytek, ]; Logger.info( - '[TRANS-INIT] 开始匹配提供商: $sourceShortCode -> $targetShortCode, 策略=阿里3.5(声音复刻)>阿里旧版>微软>讯飞（豆包已停用）'); + '[TRANS-INIT] 开始匹配提供商: $sourceShortCode -> $targetShortCode, 策略=微软端到端'); final forceAzure = azureCallOnlyShortCodes.contains(sourceShortCode) || azureCallOnlyShortCodes.contains(targetShortCode); @@ -859,7 +856,7 @@ class LanguageManager extends GetxService { } final providerString = _providerStringFromType(provider); - final isEndToEnd = _isAlibaba(provider); + final isEndToEnd = provider == SpeechServiceType.azure || _isAlibaba(provider); // warn 级：这一行决定了「有没有声音复刻」——只有 alibaba35 那一档才有。 // 客户反馈「没被复刻」时，第一件事就是看这里命中的是哪一档。 Logger.warning( diff --git a/apps/client/lib/data/services/microsoft_text_translation.dart b/apps/client/lib/data/services/microsoft_text_translation.dart index 217aeb02..e6a61fff 100644 --- a/apps/client/lib/data/services/microsoft_text_translation.dart +++ b/apps/client/lib/data/services/microsoft_text_translation.dart @@ -1,86 +1,3 @@ -import 'dart:math'; - -import 'package:dio/dio.dart'; - -import '../../core/utils/logger.dart'; -import '../models/appconfig.dart'; -import 'ai_vendor_lock.dart'; - -/// 微软文本翻译。同声 / 面对面 / 音视频里扩展语种走这条，不经服务端阿里。 -/// -/// 密钥与通话翻译同一套：`mt_azure` 的 subscription_key / region。 -/// 请求形状与原生 `MicrosoftTranslationServiceImpl` 一致。 -class MicrosoftTextTranslation { - static const defaultEndpoint = - 'https://api.cognitive.microsofttranslator.com'; - - final Dio _dio = Dio(BaseOptions( - connectTimeout: const Duration(seconds: 10), - sendTimeout: const Duration(seconds: 10), - receiveTimeout: const Duration(seconds: 10), - )); - - Future translate({ - required String text, - required String from, - required String to, - CancelToken? cancelToken, - }) async { - final src = text.trim(); - if (src.isEmpty) return null; - if (aiOnlyIflytek) { - Logger.w('MsTrans', '测试服只允许科大讯飞，不调用微软翻译'); - return null; - } - final key = - AppConfig.cred('mt_azure', 'subscription_key', 'AZURE_TRANSLATION_KEY'); - if (key == null || key.isEmpty) { - Logger.w('MsTrans', '微软翻译密钥未下发，跳过 $from→$to'); - return null; - } - final region = AppConfig.cred('mt_azure', 'region', 'AZURE_TRANSLATION_REGION'); - final endpoint = - AppConfig.cred('mt_azure', 'endpoint', 'AZURE_TRANSLATION_ENDPOINT') ?? - defaultEndpoint; - final headers = { - 'Ocp-Apim-Subscription-Key': key, - 'Content-Type': 'application/json', - 'X-ClientTraceId': _traceId(), - }; - if (region != null && region.isNotEmpty) { - headers['Ocp-Apim-Subscription-Region'] = region; - } - try { - final query = { - 'api-version': '3.0', - 'to': to, - }; - if (from.isNotEmpty) query['from'] = from; - final response = await _dio.post( - '${endpoint.replaceAll(RegExp(r'/+$'), '')}/translate', - queryParameters: query, - data: [ - {'text': src}, - ], - options: Options(headers: headers), - cancelToken: cancelToken, - ); - final out = microsoftTranslationText(response.data); - if (out == null) { - Logger.w('MsTrans', '微软翻译返回空: $from→$to'); - } - return out; - } on DioException catch (e) { - if (e.type == DioExceptionType.cancel) return null; - Logger.w('MsTrans', '微软翻译失败($from→$to): $e'); - return null; - } catch (e) { - Logger.w('MsTrans', '微软翻译失败($from→$to): $e'); - return null; - } - } -} - /// 阿里机器翻译盖不住的语种：繁体会被收成简体，乌克兰语和克罗地亚语直接被拒。 bool microsoftMtRequired(String from, String to) { bool hit(String raw) { @@ -143,10 +60,3 @@ String? microsoftTranslationText(dynamic data) { final out = text.trim(); return out.isEmpty ? null : out; } - -String _traceId() { - final rand = Random(); - String hex(int n) => - List.generate(n, (_) => rand.nextInt(16).toRadixString(16)).join(); - return '${hex(8)}-${hex(4)}-4${hex(3)}-${hex(4)}-${hex(12)}'; -} diff --git a/apps/client/lib/data/services/recording_archive.dart b/apps/client/lib/data/services/recording_archive.dart index eede7db6..44730a04 100644 --- a/apps/client/lib/data/services/recording_archive.dart +++ b/apps/client/lib/data/services/recording_archive.dart @@ -25,9 +25,8 @@ import 'network/api.dart'; /// - `trans_call_*` 通话翻译 /// - `trans_holder_*` 同声翻译 / 面对面翻译 /// -/// 翻译那两条还会带上**现成的转写**（[archive] 的 `transcript`）： -/// 翻译时 ASR 已经逐句出过整句，再让纪要把同一段音频识别一遍既慢又白花算力， -/// 所以纪要那边只做总结（2026-09-18 产品口径）。 +/// 同声 / 面对面 / 通话翻译不走 [archive]：结束时只交已翻译的句子， +/// 见 [archiveTranscript]，不存音频、不上传。 class RecordingArchive { RecordingArchive._(); @@ -160,4 +159,76 @@ class RecordingArchive { return null; } } + + /// 只把已经翻译好的句子送进语音纪要，不存音频、不上传。 + /// + /// 同声 / 面对面 / 通话结束时用。服务端 `rtype=VOICETRANSLAT` 跳过识别， + /// 直接用大模型总结。成功返回记录 id。 + static Future archiveTranscript({ + required String title, + required int seconds, + required List> transcript, + required String summaryLanguage, + String tag = 'RecordingArchive', + }) async { + if (transcript.isEmpty) return null; + try { + final data = { + 'title': title, + 'type': 'LOCAL', + 'seconds': seconds < 1 ? 1 : seconds, + 'filepath': '', + 'audiourl': '', + 'tasktype': 0, + 'creationtime': resolveRecordingCreationTime(null), + }; + final resp = await Api.addRecordEchomeet({ + 'rtype': 'VOICETRANSLAT', + 'size': 0, + 'original': transcript, + ...data, + }); + final meetingId = resp?['record']?['id']; + if (meetingId == null) { + Logger.e(tag, '转写归档失败：服务端未返回记录 id'); + return null; + } + await SqfliteApi.insertMeeting({'id': meetingId, ...data}); + await SqfliteApi.insertMeetingSpeaker([ + for (final seg in transcript) + { + 'meetingid': meetingId, + 'content': seg['content'], + 'starttime': seg['starttime'], + 'endtime': seg['endtime'], + 'speaker': seg['speaker'] ?? '', + } + ]); + if (Get.isRegistered()) { + Get.find().updateListState.value++; + } + final int? id = meetingId is int ? meetingId : int.tryParse('$meetingId'); + Logger.i(tag, '已把翻译转写送入语音纪要 id=$id title=$title 段=${transcript.length}'); + if (id != null && + summaryLanguage.isNotEmpty && + Get.isRegistered()) { + try { + final ok = await Get.find() + .startSummaryForTranscript(id, summaryLanguage); + Logger.i(tag, '自动总结${ok ? '已提交' : '未能提交'} id=$id lang=$summaryLanguage'); + } catch (e) { + Logger.w(tag, '自动总结提交失败(记录已保留) id=$id: $e'); + } + } + try { + Get.snackbar('agentMeeting'.tr, 'recordingSavedDesc'.tr, + snackPosition: SnackPosition.TOP, + duration: const Duration(seconds: 2)); + } catch (_) {} + return id; + } catch (e) { + Logger.e(tag, '转写归档失败: $e'); + return null; + } + } } diff --git a/apps/client/lib/data/services/server_translation_service.dart b/apps/client/lib/data/services/server_translation_service.dart index f0b4629d..9e01bd00 100644 --- a/apps/client/lib/data/services/server_translation_service.dart +++ b/apps/client/lib/data/services/server_translation_service.dart @@ -1,30 +1,15 @@ -import 'package:dio/dio.dart'; import 'package:get/get.dart'; import '../../core/utils/logger.dart'; -import 'microsoft_text_translation.dart'; -import 'network/api.dart'; import 'sparkchain_session.dart'; -import 'ai_vendor_lock.dart'; -/// 文本机器翻译：由**服务端**代调后台「第三方服务配置」里启用的 MT 服务。 -/// -/// 为什么不像以前那样客户端直连服务商： -/// 旧实现直连火山，AK/SK 通过 AppConfig.env **明文下发给每一个客户端**， -/// 抓包或反编译就能拿到云账号凭据。改成服务端代调后凭据不再出网关， -/// 换服务商也只需在后台改配置、不用发版。（火山翻译已于 2026-08-28 整体下线。） -/// -/// 代价是每次翻译多一跳（App → 我们的服务端 → 服务商）。同传是流式高频调用， -/// 所以服务端对翻译客户端做了 60s 缓存，避免每句都查库重建 SDK。 -/// +/// 文本机器翻译走讯飞 SDK 的 gtrans，不再直连微软翻译，也不再打自家网关。 class ServerTranslationService extends GetxService { static ServerTranslationService get to => Get.find(); - final MicrosoftTextTranslation _microsoft = MicrosoftTextTranslation(); - static const _tag = 'ServerTrans'; - /// 与旧实现保持同样的接口形态。服务端无需客户端预初始化，恒为 true。 + /// 与旧实现保持同样的接口形态。讯飞 SDK 在真正翻译时才初始化。 bool get isInitialized => true; /// 保留这个方法只为兼容调用方（translation_controller 会先调 initialize）。 @@ -34,119 +19,38 @@ class ServerTranslationService extends GetxService { final lastProvider = ''.obs; final lastSvcId = ''.obs; - /// 最近一次「自动检测」识别出的源语种短码（zh / en …）。 - /// 只有 sourceLanguageCode 传空串时服务端才会回填。 + /// 讯飞 gtrans 不回填检测语种，自动检测时这里保持空。 final lastDetectedFrom = ''.obs; - /// 翻译一段文本。失败返回 null（调用方按原样保留原文）。 - /// - /// 语言码传 BCP-47（zh-CN / en-US）即可，服务端会归一成服务商要的短码。 - /// [sourceLanguageCode] 传空串 = 自动检测，检测结果读 [lastDetectedFrom]。 - /// [cancelToken] 给同传/面对面的**中间结果**预览翻译用：说话过程中会连发好几次， - /// 后一次发出时把前一次撤掉。被取消返回 null，且不打 warning——那是预期行为， - /// 不是失败。 - /// 实时链路（同传 / 面对面 / 通话）的翻译超时。 - /// - /// ⚠️ 不能用全局那 30s：那是给上传、会议总结这类慢请求留的。同传是流式的， - /// 一句话等 30s 早就没有意义，而且会把后面的句子一起堵在同一条链路上—— - /// 弱网下的「卡住不动」就是这么来的。 - /// - /// 取 12s 是有依据的：服务端 user_translate 内部的上下文只给服务商 10s - /// （api_translate.go 的 context.WithTimeout），10s 拿不到结果它自己就返回失败了， - /// 客户端再多等就是纯白等。留 2s 给网络往返。 static const Duration realtimeTimeout = Duration(seconds: 12); Future translateText({ required String text, required String sourceLanguageCode, required String targetLanguageCode, - CancelToken? cancelToken, + Object? cancelToken, Duration? timeout = realtimeTimeout, bool tryOnDevice = true, }) async { final src = text.trim(); if (src.isEmpty) return null; - if (!tryOnDevice) { - return _translateRemote( - src: src, - sourceLanguageCode: sourceLanguageCode, - targetLanguageCode: targetLanguageCode, - cancelToken: cancelToken, - timeout: timeout, - ); + if (!tryOnDevice) return null; + // 签名留给原来的调用方。翻译本身在 SDK 里，不再用取消令牌和超时打网关。 + if (cancelToken != null || timeout != null) { + Logger.i(_tag, '文本翻译走讯飞 SDK gtrans'); } final onDevice = await SparkChainSession.translate( from: sourceLanguageCode, to: targetLanguageCode, text: src, ); - if (onDevice != null) { - lastProvider.value = 'sparkchain'; - lastSvcId.value = 'sparkchain'; - return onDevice; - } - return _translateRemote( - src: src, - sourceLanguageCode: sourceLanguageCode, - targetLanguageCode: targetLanguageCode, - cancelToken: cancelToken, - timeout: timeout, - ); - } - - Future _translateRemote({ - required String src, - required String sourceLanguageCode, - required String targetLanguageCode, - CancelToken? cancelToken, - Duration? timeout, - }) async { - if (src.isEmpty) return null; - if (!aiOnlyIflytek && microsoftMtRequired(sourceLanguageCode, targetLanguageCode)) { - final from = microsoftMtCode(sourceLanguageCode); - final to = microsoftMtCode(targetLanguageCode); - Logger.i(_tag, '阿里翻译覆盖不了，改走微软: $from→$to'); - final direct = await _microsoft.translate( - text: src, - from: from, - to: to, - cancelToken: cancelToken, - ); - if (direct != null) return direct; - Logger.w(_tag, '微软翻译没有结果，回退服务端: $from→$to'); - } - try { - final data = await Api.userTranslate({ - 'text': src, - 'from': sourceLanguageCode, - 'to': targetLanguageCode, - }, cancelToken: cancelToken, timeout: timeout); - if (data is! Map) return null; - final out = (data['text'] as String?)?.trim() ?? ''; - lastProvider.value = (data['provider'] as String?) ?? ''; - lastSvcId.value = (data['svcid'] as String?) ?? ''; - lastDetectedFrom.value = (data['detected_from'] as String?) ?? ''; - if (out.isEmpty) { - Logger.w(_tag, '翻译返回空: $sourceLanguageCode→$targetLanguageCode'); - return null; - } - return out; - } catch (e) { - // 主动取消的中间结果请求：预期行为，静默返回。 - if (e is DioException && e.type == DioExceptionType.cancel) return null; - // 服务端错误码见 errorcode.proto 5101-5103： - // 5101 未配置 MT 服务 / 5102 参数不合法 / 5103 服务商调用失败 - if (e is DioException && - (e.type == DioExceptionType.receiveTimeout || - e.type == DioExceptionType.sendTimeout || - e.type == DioExceptionType.connectionTimeout)) { - // 弱网：这一句放弃，不阻塞后面的句子。同传宁可丢一句也不能整条卡住。 - Logger.w(_tag, - '翻译超时($sourceLanguageCode→$targetLanguageCode)，跳过这一句'); - return null; - } - Logger.w(_tag, '翻译失败($sourceLanguageCode→$targetLanguageCode): $e'); + if (onDevice == null) { + Logger.w(_tag, '讯飞 SDK 没返回译文: $sourceLanguageCode→$targetLanguageCode'); return null; } + lastProvider.value = 'sparkchain'; + lastSvcId.value = 'sparkchain'; + lastDetectedFrom.value = ''; + return onDevice; } } diff --git a/apps/client/lib/data/services/simul_microsoft.dart b/apps/client/lib/data/services/simul_microsoft.dart new file mode 100644 index 00000000..58c149cf --- /dev/null +++ b/apps/client/lib/data/services/simul_microsoft.dart @@ -0,0 +1,169 @@ +/// 同声翻译选哪条讯飞微软会话。 +/// +/// 两端都在同传表里：一条 AST，发音人用短名。 +/// 源或目标是中文、粤语、爱尔兰语：不启 AST，识别走微软实时转写，合成用区域码音色。 +/// 两边都不在这两类里：不启动。 +class SimulRoute { + const SimulRoute.ast({ + required this.lang, + required this.targetLang, + required this.speaker, + }) : ast = true, + languageName = ''; + + const SimulRoute.splice({ + required this.languageName, + required this.speaker, + }) : ast = false, + lang = '', + targetLang = ''; + + final bool ast; + /// AST 的源语种短码，例如 ja。 + final String lang; + /// AST 的目标语种短码，例如 en。 + final String targetLang; + /// AST 用短名（JennyNeural）。拼接用区域码音色（en-US-JennyNeural）。 + final String speaker; + /// 拼接识别的 language_name，例如 zh-CN。 + final String languageName; +} + +class SimulMicrosoft { + static SimulRoute? plan(String source, String target) { + final from = _norm(source); + final to = _norm(target); + if (_ast.containsKey(from) && _ast.containsKey(to)) { + return SimulRoute.ast( + lang: from, + targetLang: to, + speaker: _ast[to]!.speaker, + ); + } + if (_splice.containsKey(from) || _splice.containsKey(to)) { + final locale = _listenLocale(from); + final voice = _speakVoice(to); + if (locale == null || voice == null) return null; + return SimulRoute.splice(languageName: locale, speaker: voice); + } + return null; + } + + /// 同传能力 ast_msft 没授权时，用同一对语种走微软转写加合成。 + /// 识别用区域码，合成用完整神经音色，不再用 AST 的短名。 + static SimulRoute spliceInstead(SimulRoute route) { + final locale = _listenLocale(route.lang); + final voice = _speakVoice(route.targetLang); + if (locale == null || voice == null) return route; + return SimulRoute.splice(languageName: locale, speaker: voice); + } + + /// 界面短码收成表里的键。香港繁体要在「凡是 zh- 都当普通话」之前认出来。 + static String _norm(String code) { + final raw = code.trim().toLowerCase().replaceAll('_', '-'); + if (raw == 'yue' || raw.startsWith('yue-')) return 'yue'; + if (raw == 'ga' || raw.startsWith('ga-')) return 'ga'; + if (raw == 'zh-hant-hk' || raw == 'zh-hk') return 'zh-HK'; + if (raw == 'zh-hant' || raw == 'zh-tw' || raw.startsWith('zh-hant')) { + return 'zh-Hant'; + } + if (raw == 'zh' || raw.startsWith('zh-')) return 'zh'; + final dash = raw.indexOf('-'); + return dash < 0 ? raw : raw.substring(0, dash); + } + + /// 讯飞 SDK 文本翻译（gtrans）的语种码。中文用 zh-CN，不要收成 zh。 + static String translateCode(String code) { + switch (_norm(code)) { + case 'zh': + return 'zh-CN'; + case 'zh-Hant': + return 'zh-TW'; + case 'zh-HK': + return 'zh-HK'; + case 'yue': + return 'yue'; + case 'ga': + return 'ga'; + default: + return _norm(code); + } + } + + static String? _listenLocale(String key) { + final splice = _splice[key]; + if (splice != null) return splice.locale; + return _ast[key]?.locale; + } + + static String? _speakVoice(String key) { + final splice = _splice[key]; + if (splice != null) return splice.voice; + final ast = _ast[key]; + if (ast == null) return null; + return '${ast.locale}-${ast.speaker}'; + } + + /// 同传表。speaker 是 AST 的短名，locale 是拼接到 OnlineTTS 时用的区域码。 + static const _ast = { + 'ar': _AstLang('ar-EG', 'AbdullahNeural'), + 'bg': _AstLang('bg-BG', 'BorislavNeural'), + 'bn': _AstLang('bn-IN', 'BashkarNeural'), + 'ca': _AstLang('ca-ES', 'AlbaNeural'), + 'cs': _AstLang('cs-CZ', 'AntoninNeural'), + 'da': _AstLang('da-DK', 'ChristelNeural'), + 'de': _AstLang('de-DE', 'AmalaNeural'), + 'el': _AstLang('el-GR', 'AthinaNeural'), + 'en': _AstLang('en-US', 'JennyNeural'), + 'es': _AstLang('es-ES', 'ElviraNeural'), + 'et': _AstLang('et-EE', 'AnuNeural'), + 'fa': _AstLang('fa-IR', 'DilaraNeural'), + 'fi': _AstLang('fi-FI', 'HarriNeural'), + 'fr': _AstLang('fr-FR', 'DeniseNeural'), + 'hi': _AstLang('hi-IN', 'MadhurNeural'), + 'hu': _AstLang('hu-HU', 'NoemiNeural'), + 'id': _AstLang('id-ID', 'ArdiNeural'), + 'it': _AstLang('it-IT', 'ElsaNeural'), + 'ja': _AstLang('ja-JP', 'NanamiNeural'), + 'ko': _AstLang('ko-KR', 'SunHiNeural'), + 'lt': _AstLang('lt-LT', 'LeonasNeural'), + 'lv': _AstLang('lv-LV', 'EveritaNeural'), + 'ms': _AstLang('ms-MY', 'OsmanNeural'), + 'nl': _AstLang('nl-NL', 'ArnaudNeural'), + 'pl': _AstLang('pl-PL', 'AgnieszkaNeural'), + 'pt': _AstLang('pt-BR', 'AntonioNeural'), + 'ro': _AstLang('ro-RO', 'AlinaNeural'), + 'ru': _AstLang('ru-RU', 'DariyaNeural'), + 'sk': _AstLang('sk-SK', 'LukasNeural'), + 'sl': _AstLang('sl-SI', 'PetraNeural'), + 'sv': _AstLang('sv-SE', 'HilleviNeural'), + 'sw': _AstLang('sw-KE', 'DaudiNeural'), + 'ta': _AstLang('ta-IN', 'AnbuNeural'), + 'th': _AstLang('th-TH', 'AcharaNeural'), + 'tr': _AstLang('tr-TR', 'AhmetNeural'), + 'uk': _AstLang('uk-UA', 'OstapNeural'), + 'ur': _AstLang('ur-IN', 'AsadNeural'), + 'vi': _AstLang('vi-VN', 'HoaiMyNeural'), + }; + + /// 微软同传表没有的三语。繁体中文按普通话识别，合成用繁体音色。 + static const _splice = { + 'zh': _SpliceLang('zh-CN', 'zh-CN-XiaoxiaoNeural'), + 'zh-Hant': _SpliceLang('zh-CN', 'zh-TW-HsiaoChenNeural'), + 'yue': _SpliceLang('yue-CN', 'yue-CN-XiaoMinNeural'), + 'zh-HK': _SpliceLang('zh-HK', 'zh-HK-HiuMaanNeural'), + 'ga': _SpliceLang('ga-IE', 'ga-IE-OrlaNeural'), + }; +} + +class _AstLang { + const _AstLang(this.locale, this.speaker); + final String locale; + final String speaker; +} + +class _SpliceLang { + const _SpliceLang(this.locale, this.voice); + final String locale; + final String voice; +} diff --git a/apps/client/lib/data/services/sparkchain_session.dart b/apps/client/lib/data/services/sparkchain_session.dart index ff618e46..00adf1b4 100644 --- a/apps/client/lib/data/services/sparkchain_session.dart +++ b/apps/client/lib/data/services/sparkchain_session.dart @@ -14,14 +14,19 @@ class SparkChainSession { static bool _ready = false; static bool _eventsAttached = false; static bool listening = false; + static int lastStartCode = 0; static int _seq = 0; static final Map>> _waiters = {}; static final Map _ttsHandlers = {}; static final Map> _ttsWaiters = {}; + static final Map event)> _llmHandlers = {}; /// 听写结果。只在 [startListen] 成功之后、[stopListen] 之前有意义。 static void Function(Map event)? asrListener; + /// 同声 AST 的识别、译文和合成音频。 + static void Function(Map event)? astListener; + static Future ensure() async { if (_ready) return true; final appId = AppConfig.cred('sparkchain', 'app_id', 'SPARKCHAIN_APP_ID'); @@ -29,6 +34,12 @@ class SparkChainSession { final apiSecret = AppConfig.cred('sparkchain', 'api_secret', 'SPARKCHAIN_API_SECRET'); if (appId == null || apiKey == null || apiSecret == null) { + final missing = [ + if (appId == null) 'app_id', + if (apiKey == null) 'api_key', + if (apiSecret == null) 'api_secret', + ]; + Logger.w(_tag, '密钥未下发，跳过 init: ${missing.join(",")}'); return false; } if (!_eventsAttached) { @@ -44,11 +55,17 @@ class SparkChainSession { apiSecret: apiSecret, apiType: apiType, ); - Logger.i(_tag, 'init code=$code apiType=$apiType'); + Logger.w(_tag, 'init code=$code apiType=$apiType'); _ready = code == 0; return _ready; } + /// 配置拉下来之后只做 init。不要在这里 startListen 或翻译： + /// 新加坡节点连不上时 SDK 会在关闭 WebSocket 的线程里 SIGSEGV，启动即闪退。 + static Future warmUp() async { + await ensure(); + } + static void _onEvent(Map event) { final id = event['id']?.toString() ?? ''; final kind = event['kind']; @@ -59,9 +76,15 @@ class SparkChainSession { if (kind == 'asr' || (kind == 'error' && event['where'] == 'asr')) { asrListener?.call(event); } + if (kind == 'ast' || (kind == 'error' && event['where'] == 'ast')) { + astListener?.call(event); + } if (kind == 'tts' || (kind == 'error' && event['where'] == 'tts')) { _onTts(id, event); } + if (kind == 'llm' || (kind == 'error' && event['where'] == 'llm')) { + _llmHandlers[id]?.call(event); + } } static void _onTts(String id, Map event) { @@ -83,6 +106,68 @@ class SparkChainSession { } } + /// 同声：一条微软同传。发音人是短名，例如 JennyNeural。 + static Future startAst({ + required String lang, + required String targetLang, + required String speaker, + }) async { + if (!await ensure()) return false; + final appId = AppConfig.cred('sparkchain', 'app_id', 'SPARKCHAIN_APP_ID'); + final apiKey = AppConfig.cred('sparkchain', 'api_key', 'SPARKCHAIN_API_KEY'); + final apiSecret = + AppConfig.cred('sparkchain', 'api_secret', 'SPARKCHAIN_API_SECRET'); + // 同传网关认的是自己的 accessKeyId，不是 SparkChain 应用的 apiKey。 + // 后台「同传单独」三项有值就用那一组，空才回退。 + final ownKey = AppConfig.field('sparkchain', 'ast_api_key') != null; + final astApp = AppConfig.field('sparkchain', 'ast_app_id') ?? appId ?? ''; + final astKey = AppConfig.field('sparkchain', 'ast_api_key') ?? apiKey ?? ''; + final astSecret = + AppConfig.field('sparkchain', 'ast_api_secret') ?? apiSecret ?? ''; + final code = await SparkChain.startAst( + id: 'listen', + lang: lang, + targetLang: targetLang, + speaker: speaker, + astAppId: astApp, + astApiKey: astKey, + astApiSecret: astSecret, + ); + lastStartCode = code; + listening = code == 0; + Logger.w( + _tag, + 'AST 启动 code=$code RegionMSFT lang=$lang target_lang=$targetLang ' + 'tts_target_lang=$targetLang tts_speaker=$speaker use_tts=true ' + '同传密钥=${ownKey ? "单独字段" : "回退应用密钥"}', + ); + return listening; + } + + /// 同声拼接：微软实时转写。四个固定参数加 language_name，不走星火听写。 + static Future startSpliceListen(String locale) async { + if (!await ensure()) return false; + final code = await SparkChain.startSpliceListen(id: 'listen', locale: locale); + lastStartCode = code; + listening = code == 0; + _writeFails = 0; + Logger.w( + _tag, + 'RTASR 启动 code=$code RegionMSFT lang=msft accent=mandarin ' + 'domain=ist_mul_msft dwa=wpgs language_name=$locale', + ); + return listening; + } + + /// 微软区域实时转写，语种用界面语言的区域码。SDK 不自己开麦。 + static Future startMsftListen(String locale) async { + if (!await ensure()) return false; + final code = await SparkChain.startMsftListen(id: 'listen', locale: locale); + listening = code == 0; + if (!listening) Logger.w(_tag, '微软听写启动失败 code=$code locale=$locale'); + return listening; + } + /// 只听 [lang] 这一种。ga 走实时转写，其余走听写。SDK 不自己开麦。 static Future startListen(String lang) async { if (!await ensure()) return false; @@ -94,15 +179,56 @@ class SparkChainSession { return listening; } + static var _writeFails = 0; + + /// 苹果实时转写文档：每 40ms 送 1280 字节。一次把积压全送进去，引擎会出错， + /// 大约 10 秒后就是 18802。 + static const _paceFrame = 1280; + static final List _paceBuf = []; + static Timer? _paceTimer; + static void write(Uint8List pcm) { if (!listening || pcm.isEmpty) return; - unawaited(SparkChain.write(pcm)); + _paceBuf.addAll(pcm); + const maxBytes = 32000 * 2; + if (_paceBuf.length > maxBytes) { + _paceBuf.removeRange(0, _paceBuf.length - maxBytes); + } + _paceTimer ??= Timer.periodic(const Duration(milliseconds: 40), (_) { + _pumpPace(); + }); + } + + static void _pumpPace() { + if (!listening || _paceBuf.isEmpty) { + _paceTimer?.cancel(); + _paceTimer = null; + return; + } + final n = _paceBuf.length < _paceFrame ? _paceBuf.length : _paceFrame; + final frame = Uint8List.fromList(_paceBuf.sublist(0, n)); + _paceBuf.removeRange(0, n); + unawaited(SparkChain.write(frame).then((code) { + if (code != 0 && _writeFails < 3) { + _writeFails++; + Logger.w(_tag, 'write 失败 code=$code 字节=${frame.length}'); + } + })); + } + + static void _stopPace() { + _paceTimer?.cancel(); + _paceTimer = null; + _paceBuf.clear(); } static Future stopListen() async { - if (!listening) return; listening = false; + _stopPace(); asrListener = null; + astListener = null; + // 启动失败时 listening 仍是 false，但原生引擎可能已经占着一条没连上的会话。 + // 不拆掉的话，下一次 start 会在这条会话上一直耗到超时。 try { await SparkChain.stopListen(); } catch (e) { @@ -110,6 +236,81 @@ class SparkChainSession { } } + /// 微软区域神经语音。PCM 16 kHz，按到达顺序交给 [onPcm]。 + static Future speakNeural({ + required String voice, + required String text, + required void Function(Uint8List pcm, bool isFinal) onPcm, + }) async { + if (!await ensure()) throw StateError('sparkchain not ready'); + final id = 'tts-${++_seq}'; + final waiter = Completer(); + _ttsWaiters[id] = waiter; + _ttsHandlers[id] = onPcm; + final code = await SparkChain.speak( + id: id, + lang: 'msft', + text: text, + voice: voice, + msft: true, + ); + if (code != 0) { + _ttsHandlers.remove(id); + _ttsWaiters.remove(id); + throw StateError('speak $code'); + } + await waiter.future.timeout(const Duration(seconds: 30)); + } + + static Future stopSpeak() async { + try { + await SparkChain.stopSpeak(); + } catch (e) { + Logger.w(_tag, 'stopSpeak: $e'); + } + } + + /// 端上文本对话。fresh 为 true 时清掉 SDK 里上一轮的上下文。 + static Future chat({ + required String query, + required String system, + required String tools, + required bool fresh, + }) async { + if (!await ensure()) throw StateError('sparkchain not ready'); + final id = 'llm-${++_seq}'; + final waiter = Completer(); + final text = StringBuffer(); + var functionCall = ''; + _llmHandlers[id] = (event) { + if (event['kind'] == 'error') { + if (!waiter.isCompleted) { + waiter.completeError(StateError('${event['code']}')); + } + return; + } + text.write(event['text'] ?? ''); + final call = (event['functionCall'] as String?) ?? ''; + if (call.isNotEmpty) functionCall = call; + if (event['final'] == true && !waiter.isCompleted) { + waiter.complete(SparkLlmReply(text.toString(), functionCall)); + } + }; + try { + final code = await SparkChain.chat( + id: id, + query: query, + system: system, + tools: tools, + fresh: fresh, + ); + if (code != 0) throw StateError('chat $code'); + return await waiter.future.timeout(const Duration(seconds: 25)); + } finally { + _llmHandlers.remove(id); + } + } + /// 合成一段。PCM 按到达顺序交给 [onPcm]，整段结束或失败才返回。 static Future speak({ required String lang, @@ -137,31 +338,36 @@ class SparkChainSession { } /// 短码 zh / en / ga。失败返回 null，不抛。 + /// [prepared] 为真时语种码已经是 SDK 要的写法，不再收成 zh / en。 static Future translate({ required String from, required String to, required String text, + bool prepared = false, }) async { if (!await ensure()) return null; final id = '${++_seq}'; final waiter = Completer>(); _waiters[id] = waiter; + final src = prepared ? from : sparkChainShortCode(from); + final dst = prepared ? to : sparkChainShortCode(to); try { final code = await SparkChain.translate( id: id, - from: sparkChainShortCode(from), - to: sparkChainShortCode(to), + from: src, + to: dst, text: text, ); if (code != 0) { Logger.w(_tag, 'translate 启动失败 code=$code'); return null; } - final event = await waiter.future.timeout(const Duration(seconds: 12)); + final event = await waiter.future.timeout(const Duration(seconds: 20)); if (event['kind'] != 'its') { Logger.w(_tag, 'translate 失败 code=${event['code']} sid=${event['sid']}'); return null; } + Logger.w(_tag, 'translate engine=${event['engine']}'); final out = (event['text'] as String?)?.trim() ?? ''; return out.isEmpty ? null : out; } on TimeoutException { @@ -172,3 +378,9 @@ class SparkChainSession { } } } + +class SparkLlmReply { + SparkLlmReply(this.text, this.functionCall); + final String text; + final String functionCall; +} diff --git a/apps/client/lib/data/services/speech_impl/azure_asr_service.dart b/apps/client/lib/data/services/speech_impl/azure_asr_service.dart index 1d6c26af..3c80d6fc 100644 --- a/apps/client/lib/data/services/speech_impl/azure_asr_service.dart +++ b/apps/client/lib/data/services/speech_impl/azure_asr_service.dart @@ -4,7 +4,6 @@ import 'dart:typed_data'; import '../../../data/models/appconfig.dart'; import 'package:flutter/services.dart'; import '../../../core/utils/logger.dart'; -import '../ai_vendor_lock.dart'; import 'package:get/get.dart'; import '../asr_service.dart'; @@ -116,10 +115,6 @@ class AzureAsrService extends GetxService implements AsrService { _isInitialized = false; return false; } - if (aiOnlyIflytek) { - Logger.w('AzureASR', '测试服只允许科大讯飞，不初始化微软听写'); - return false; - } try { final List languages = supportedLanguages ?? _defaultSupportedLanguages; diff --git a/apps/client/lib/data/services/speech_impl/azure_ast_service.dart b/apps/client/lib/data/services/speech_impl/azure_ast_service.dart index c392be92..fb92a669 100644 --- a/apps/client/lib/data/services/speech_impl/azure_ast_service.dart +++ b/apps/client/lib/data/services/speech_impl/azure_ast_service.dart @@ -66,6 +66,21 @@ class AzureAstService extends GetxService implements AstService { /// provider=azure + zh-CN/en-US，把刚握手成功的阿里 zh→de 会话整个拆掉。 /// 表现是「德语/意大利语一开就卡死、只有中英正常」——因为默认值恰好就是中英。 List? _lastLanguages; + bool _lastSingleLeg = false; + bool _lastBidirectional = false; + + /// 原生会话是不是按这组语言建的。同声/面对面换语言后如果还用上一对， + /// 英语转日语会继续按上一对（比如日语转英语）把原文念出来。 + bool acceptsLanguages(List languages) { + final last = _lastLanguages; + if (!_isInitialized || last == null || last.length != languages.length) { + return false; + } + for (var i = 0; i < languages.length; i++) { + if (last[i] != languages[i]) return false; + } + return true; + } String _lastProvider = 'azure'; /// 正在进行中的 initialize：后来的调用先等它结束再开始，避免 dispose 撞上在途的原生初始化。 @@ -205,6 +220,15 @@ class AzureAstService extends GetxService implements AstService { )); break; + case 'recognitionStopped': + Logger.w('AzureAST', + '[STS] 识别会话停了 service=${eventMap['serviceId']}'); + _eventStreamController?.add(ASTEvent( + type: ASTEventType.sessionStopped, + serviceId: eventMap['serviceId'] as String? ?? '', + )); + break; + case 'canceled': final String reason = eventMap['reason'] as String? ?? ''; final String errorDetails = eventMap['errorDetails'] as String? ?? ''; @@ -322,7 +346,12 @@ class AzureAstService extends GetxService implements AstService { throw StateError('[STS] $who: AST 尚未初始化，且没有可复用的参数'); } Logger.warning('[STS] $who: 未初始化，按上次参数补初始化 provider=$_lastProvider langs=$langs'); - final ok = await initialize(supportedLanguages: langs, provider: _lastProvider); + final ok = await initialize( + supportedLanguages: langs, + provider: _lastProvider, + singleLeg: _lastSingleLeg, + bidirectional: _lastBidirectional, + ); if (!ok) throw StateError('[STS] $who: 按上次参数补初始化失败'); } } @@ -470,6 +499,8 @@ class AzureAstService extends GetxService implements AstService { Future initialize({ List? supportedLanguages, String provider = 'azure', + bool singleLeg = false, + bool bidirectional = false, }) async { // 串行化：前一条还在原生里跑时就 dispose 会让它返回 false，然后触发一连串误判。 final prev = _initInFlight; @@ -479,7 +510,12 @@ class AzureAstService extends GetxService implements AstService { await prev; } catch (_) {} } - final task = _initializeInner(supportedLanguages: supportedLanguages, provider: provider); + final task = _initializeInner( + supportedLanguages: supportedLanguages, + provider: provider, + singleLeg: singleLeg, + bidirectional: bidirectional, + ); _initInFlight = task; try { return await task; @@ -491,12 +527,12 @@ class AzureAstService extends GetxService implements AstService { Future _initializeInner({ List? supportedLanguages, String provider = 'azure', + bool singleLeg = false, + bool bidirectional = false, }) async { - if (aiOnlyIflytek) { - Logger.w('AzureAST', '测试服只允许科大讯飞，不初始化通话翻译'); - return false; - } _lastProvider = provider; + _lastSingleLeg = singleLeg; + _lastBidirectional = bidirectional; if (supportedLanguages != null) _lastLanguages = List.from(supportedLanguages); try { final List languages = @@ -518,6 +554,7 @@ class AzureAstService extends GetxService implements AstService { // 打开了。写 info 的话正式包里查不到，只能凭猜。 Logger.warning( '[STS] initialize: provider=$provider (native=$nativeProvider), ' + 'singleLeg=$singleLeg, bidirectional=$bidirectional, ' 'alibabaModel=$alibabaModel, voiceClone=$enableVoiceClone, isInitialized=$_isInitialized'); // 底层会初始化前释放 if (_isInitialized) { @@ -553,6 +590,10 @@ class AzureAstService extends GetxService implements AstService { 'alibabaModel': alibabaModel, 'alibabaEnableVoiceClone': enableVoiceClone, 'alibabaVoiceCloneFrequency': kAlibabaVoiceCloneFrequency, + // 同声/面对面只有一路麦。singleLeg 不再建 B 路、不再空等 1.2 秒。 + // bidirectional 只给面对面：这一路按说出的语种决定翻译方向。 + 'singleLeg': singleLeg, + 'bidirectional': bidirectional, }); _isInitialized = result; diff --git a/apps/client/lib/data/services/speech_impl/azure_tts_service.dart b/apps/client/lib/data/services/speech_impl/azure_tts_service.dart index fbd0e916..5b75c10c 100644 --- a/apps/client/lib/data/services/speech_impl/azure_tts_service.dart +++ b/apps/client/lib/data/services/speech_impl/azure_tts_service.dart @@ -3,7 +3,6 @@ import '../../../data/models/appconfig.dart'; import 'package:flutter/services.dart'; import 'package:get/get.dart'; import '../../../core/utils/logger.dart'; -import '../ai_vendor_lock.dart'; import '../tts_service.dart'; /// 微软 Text-to-Speech 服务 @@ -132,10 +131,6 @@ class AzureTtsService extends GetxService implements TtsService { Future initialize({ List? supportedLanguages, }) async { - if (aiOnlyIflytek) { - Logger.w('AzureTTS', '测试服只允许科大讯飞，不初始化微软合成'); - return false; - } // 检查是否需要重新初始化 if (_isInitialized) { await dispose(); diff --git a/apps/client/lib/data/utils/background_keepalive.dart b/apps/client/lib/data/utils/background_keepalive.dart index 3321e346..956886a1 100644 --- a/apps/client/lib/data/utils/background_keepalive.dart +++ b/apps/client/lib/data/utils/background_keepalive.dart @@ -41,17 +41,21 @@ class BackgroundKeepalive { if (_refs > 1) return; try { final session = await AudioSession.instance; - // ⚠️ **只在当前类别撑不住后台音频时才改它**。 - // - // 无条件 configure 成 .playback 会把通话翻译/多媒体翻译正在跑的 - // .playAndRecord 录音链路踩掉（同 SubtitlePip 的 prepareAudioSessionForPip， - // 那边栽过一次）。而 .playAndRecord / .multiRoute 本身就满足 - // UIBackgroundModes=audio 的要求，不需要降级——这时只要把静音循环播起来， - // 让「确实有音频在播」这个条件成立就够了。 - final cur = session.configuration?.avAudioSessionCategory; - final alreadyOk = cur == AVAudioSessionCategory.playback || - cur == AVAudioSessionCategory.playAndRecord || + // 读系统此刻的类别，不要读 audio_session 自己记住的那份。 + // 同声翻译开麦之后系统已经是 playAndRecord，缓存却可能还是上一次的 + // playback / 空。按缓存再配成 playback，输入声道立刻变成 0，麦流静默， + // 看门狗重启也探不到输入。 + AVAudioSessionCategory? live; + try { + live = await AVAudioSession().category; + } catch (e) { + Logger.w(_tag, '读取音频类别失败: $e'); + } + final cur = live ?? session.configuration?.avAudioSessionCategory; + final recording = cur == AVAudioSessionCategory.playAndRecord || cur == AVAudioSessionCategory.multiRoute; + final alreadyOk = recording || cur == AVAudioSessionCategory.playback; + Logger.w(_tag, '保活前类别 live=$live 采用=$cur'); if (!alreadyOk) { await session.configure(const AudioSessionConfiguration( avAudioSessionCategory: AVAudioSessionCategory.playback, @@ -59,15 +63,18 @@ class BackgroundKeepalive { AVAudioSessionCategoryOptions.mixWithOthers, avAudioSessionMode: AVAudioSessionMode.defaultMode, )); - } else { - Logger.i(_tag, '当前音频类别 $cur 已支持后台音频，不改类别'); } - await session.setActive(true); + // 录音会话已经激活。再 setActive 会让正在跑的输入 tap 停回调。 + if (!recording) { + await session.setActive(true); + } // 播放器自己再 setActive 会盖掉后面独占会话的时机，结束时音乐就续不上。 final p = AudioPlayer(handleAudioSessionActivation: false); await p.setAsset(_asset); await p.setLoopMode(LoopMode.one); await p.setVolume(0); + // 不要 await。playAndRecord 已经由麦克风占着时，play() 会一直不返回， + // 同声翻译的开始按钮就停在「准备中」。 unawaited(p.play()); _player = p; // warning 级别：release 包的文件日志只收 warning 以上，切后台排查要看得到它 diff --git a/apps/client/lib/data/utils/emai_audio_session.dart b/apps/client/lib/data/utils/emai_audio_session.dart index d56489d0..f4b8fad5 100644 --- a/apps/client/lib/data/utils/emai_audio_session.dart +++ b/apps/client/lib/data/utils/emai_audio_session.dart @@ -203,6 +203,7 @@ class EmaiAudioSession { Logger.i(_tag, '音频会话档位 ${_configured?.name ?? '-'} → ${profile.name}'); } _configured = profile; + _speakerPinned = false; if (profile == EmaiAudioProfile.playbackOnly) { // 设备对话已经用独占焦点停过音乐。这里再 setActive 会向 audio_session // 申请更弱的 GAIN_TRANSIENT，华为上音乐就跟着响回来。 @@ -231,6 +232,66 @@ class EmaiAudioSession { } } + /// 同声翻译的手机麦。不要用 voiceChat。 + /// + /// voiceChat 是给语音处理单元做回声消除的。同声这条用的是 AudioQueue， + /// 进了 voiceChat 之后采到的样本会被压到噪声底（峰值几十）， + /// 云端 10 秒听不到人声就关连接，SDK 报 18802。 + static Future ensureSimulMic() async { + if (!_apply) return; + try { + final session = await AudioSession.instance; + await session.configure(_phoneMicConfig(a2dpOutput: true)); + _configured = EmaiAudioProfile.phoneMic; + _speakerPinned = false; + await session.setActive(true); + } catch (e) { + Logger.w(_tag, '同声麦克风会话配置失败: $e'); + } + } + + /// 面对面测试：输入留在手机麦，输出钉在内置扬声器。 + /// 耳机连着时普通手机麦档会把译文送到 A2DP，这里不要。 + static bool _speakerPinned = false; + + static Future pinBuiltInSpeaker() async { + if (!_apply) return; + try { + if (Platform.isIOS) { + await AVAudioSession() + .overrideOutputAudioPort(AVAudioSessionPortOverride.speaker); + } + // 已经是语音处理档就不要再 setCategory。麦开着的时候再改类别， + // 输入 tap 会在两三秒后停住，看门狗重启会把音频线程打崩。 + if (_speakerPinned && _configured == EmaiAudioProfile.phoneMic) { + return; + } + final session = await AudioSession.instance; + await session.configure(_phoneMicConfig(a2dpOutput: false)); + _configured = EmaiAudioProfile.phoneMic; + _speakerPinned = true; + await session.setActive(true); + if (Platform.isIOS) { + await AVAudioSession() + .overrideOutputAudioPort(AVAudioSessionPortOverride.speaker); + } + Logger.w(_tag, '输出钉在手机扬声器'); + } catch (e) { + Logger.w(_tag, '钉手机扬声器失败: $e'); + } + } + + static Future releaseBuiltInSpeaker() async { + _speakerPinned = false; + if (!_apply || !Platform.isIOS) return; + try { + await AVAudioSession() + .overrideOutputAudioPort(AVAudioSessionPortOverride.none); + } catch (e) { + Logger.w(_tag, '放开扬声器锁定失败: $e'); + } + } + /// 当前会话的输出里有没有蓝牙 A2DP 口。只有 iOS 有这个查询；Android 的 /// 媒体流本来就由系统自动路到 A2DP，不需要客户端判。 static Future _outputIsA2dp() async { diff --git a/apps/client/lib/data/utils/mic_pcm_streamer.dart b/apps/client/lib/data/utils/mic_pcm_streamer.dart index 93d29758..7226e181 100644 --- a/apps/client/lib/data/utils/mic_pcm_streamer.dart +++ b/apps/client/lib/data/utils/mic_pcm_streamer.dart @@ -27,6 +27,7 @@ class MicPcmStreamer { final AudioRecorder _recorder = AudioRecorder(); StreamSubscription? _sub; + StreamSubscription? _iosSub; final _buffer = BytesBuilder(copy: false); final _packets = StreamController.broadcast(); @@ -42,6 +43,14 @@ class MicPcmStreamer { bool _running = false; bool get isRunning => _running; + /// 面对面测试要把输出钉在手机扬声器。必须在开麦之前配好， + /// 开麦后再改会话类别会把输入 tap 打断，看门狗重启时容易把音频线程打崩。 + bool _pinSpeaker = false; + bool _plainMic = false; + bool _restarting = false; + int _kickAttempts = 0; + DateTime? _kickedAt; + Future hasPermission() => _recorder.hasPermission(); /// 正在进行中的 [start]。并发调用共用同一个 future。 @@ -53,8 +62,10 @@ class MicPcmStreamer { /// `ClientAudioTimeout`，客户端不报任何错。 Future? _startFuture; - Future start() { + Future start({bool pinSpeaker = false, bool plainMic = false}) { if (_running) return Future.value(true); + _pinSpeaker = pinSpeaker; + _plainMic = plainMic; return _startFuture ??= _start().whenComplete(() => _startFuture = null); } @@ -64,8 +75,19 @@ class MicPcmStreamer { // startStream 会成功返回、引擎也起得来，但输入 tap 一个 buffer 都不回调。 // 返回值是「输出此刻走不走 A2DP」：走的话 iOS 不能开 voice-processing， // 否则系统把 A2DP 从输出里剔掉、回复又回到扬声器（见 EmaiAudioSession.ensure）。 - final a2dpOutput = await EmaiAudioSession.ensure(); - final bool voiceProcessing = !(Platform.isIOS && a2dpOutput); + // pinSpeaker 时直接钉内置扬声器，不再走 A2DP 探测——探测完再改类别 + // 会把已经装上的输入 tap 弄哑。 + final bool voiceProcessing; + if (_plainMic) { + await EmaiAudioSession.ensureSimulMic(); + voiceProcessing = false; + } else if (_pinSpeaker) { + await EmaiAudioSession.pinBuiltInSpeaker(); + voiceProcessing = true; + } else { + final a2dpOutput = await EmaiAudioSession.ensure(); + voiceProcessing = !(Platform.isIOS && a2dpOutput); + } if (!await _recorder.hasPermission()) { Logger.w(_tag, '没有麦克风权限'); return false; @@ -73,45 +95,69 @@ class MicPcmStreamer { // ⚠️ iOS：record 拿 inputNode.inputFormat 去 installTap，格式是 0Hz（会话刚 // 切类别 / 路由正在切换）时那是 ObjC 断言，这里的 try/catch 接不住，直接崩。 // 先问原生一声输入格式是否就绪，不就绪等一小会儿，还不行就报失败而不是崩。 + if (!kIsWeb && Platform.isIOS) { + final ios = await _startIosEngine(voiceProcessing); + if (ios != null) return ios; + } if (!await _waitIosInputReady()) { Logger.e(_tag, '输入音频格式无效（会话/路由切换中），本次不开麦'); return false; } - final stream = await _recorder.startStream( - RecordConfig( - encoder: AudioEncoder.pcm16bits, - sampleRate: sampleRate, - numChannels: 1, - // 语音助手场景：开回声消除和降噪，否则外放播报会被自己的麦克风收回去， - // duplex 模式下会直接把服务端 VAD 打乱。 - // ⚠️ iOS 上回复走 A2DP 耳机时**必须关**：这两项会开 voice-processing， - // 而语音处理模式下 A2DP 不可作输出。耳机里放的声音手机麦本来就收不到多少。 - echoCancel: voiceProcessing, - noiseSuppress: voiceProcessing, - // ⚠️ iOS 上不让 record 碰 AVAudioSession（会话由 EmaiAudioSession 统一配）。 - // 它自己那套是 `setCategory(.playAndRecord, options:)` → `setActive(true)`， - // 而 setCategory 会把 mode 重置成 default；紧接着 echoCancel 触发的 - // `setVoiceProcessingEnabled(true)` 又把 mode 拽回 voiceChat。这一来一回 - // 是一次 AVAudioEngine 配置变更，而 record 没有监听 - // `AVAudioEngineConfigurationChange` —— 引擎「还在运行」，输入 tap 却 - // 从此不再回调。真机表现：上行送了几十包（两三秒）后彻底停住， - // 没有任何报错，服务端随后回 ClientAudioTimeout。 - // 我们先把会话配成 playAndRecord + voiceChat 并激活，voice-processing - // 打开时 mode 已经就位，就不会再触发那次变更。 - // （Android 侧忽略 iosConfig，这里不影响。） - // ignore: deprecated_member_use - iosConfig: const IosRecordConfig(manageAudioSession: false), - ), + final config = RecordConfig( + encoder: AudioEncoder.pcm16bits, + sampleRate: sampleRate, + numChannels: 1, + // 语音助手场景：开回声消除和降噪，否则外放播报会被自己的麦克风收回去， + // duplex 模式下会直接把服务端 VAD 打乱。 + // ⚠️ iOS 上回复走 A2DP 耳机时**必须关**：这两项会开 voice-processing， + // 而语音处理模式下 A2DP 不可作输出。耳机里放的声音手机麦本来就收不到多少。 + echoCancel: voiceProcessing, + noiseSuppress: voiceProcessing, + // ⚠️ iOS 上不让 record 碰 AVAudioSession（会话由 EmaiAudioSession 统一配）。 + // 它自己那套是 `setCategory(.playAndRecord, options:)` → `setActive(true)`， + // 而 setCategory 会把 mode 重置成 default；紧接着 echoCancel 触发的 + // `setVoiceProcessingEnabled(true)` 又把 mode 拽回 voiceChat。这一来一回 + // 是一次 AVAudioEngine 配置变更，而 record 没有监听 + // `AVAudioEngineConfigurationChange` —— 引擎「还在运行」，输入 tap 却 + // 从此不再回调。真机表现：上行送了几十包（两三秒）后彻底停住， + // 没有任何报错，服务端随后回 ClientAudioTimeout。 + // 我们先把会话配成 playAndRecord + voiceChat 并激活，voice-processing + // 打开时 mode 已经就位，就不会再触发那次变更。 + // （Android 侧忽略 iosConfig，这里不影响。） + // ignore: deprecated_member_use + iosConfig: const IosRecordConfig(manageAudioSession: false), ); - _running = true; - _buffer.clear(); - _startWatchdog(); - _sub = stream.listen( - _onChunk, - onError: (e) => Logger.e(_tag, '录音流错误: $e'), - ); - Logger.i(_tag, '麦克风采集启动 ${sampleRate}Hz'); - return true; + // 会话刚切到 voiceChat 时，第一次起引擎会抛 2003329396（IO 还没让出来）。 + // 直接失败的话同声翻译第一次点开始就像没反应。等一下再试。 + Object? lastError; + for (var attempt = 0; attempt < 3; attempt++) { + try { + if (attempt > 0) { + Logger.w(_tag, '麦克风引擎未就绪，重试 #${attempt + 1}'); + await Future.delayed(Duration(milliseconds: 200 * attempt)); + try { + await _recorder.stop(); + } catch (_) {} + } + final stream = await _recorder.startStream(config); + _running = true; + _buffer.clear(); + _startWatchdog(); + _sub = stream.listen( + _onChunk, + onError: (e) => Logger.e(_tag, '录音流错误: $e'), + ); + Logger.i(_tag, '麦克风采集启动 ${sampleRate}Hz'); + return true; + } catch (e) { + lastError = e; + final retry = e.toString().contains('2003329396'); + if (!retry || attempt == 2) break; + } + } + Logger.e(_tag, '麦克风采集启动失败: $lastError'); + _running = false; + return false; } catch (e) { Logger.e(_tag, '麦克风采集启动失败: $e'); _running = false; @@ -119,6 +165,84 @@ class MicPcmStreamer { } } + /// iOS 同声/面对面不走 record。record 在输入格式 0Hz 时 installTap + /// 是进程级断言；另外它和译文播放器各有一台引擎，第一句译文一播麦就断。 + static const EventChannel _iosMicEvents = EventChannel('azure_speech/phone_mic'); + + /// 成功 / 失败都是确定结果。返回 null 表示插件没有这条通道，调用方退回 record。 + Future _startIosEngine(bool voiceProcessing) async { + try { + await _iosSub?.cancel(); + _iosSub = _iosMicEvents.receiveBroadcastStream().listen( + _onIosMicEvent, + onError: (Object e) => Logger.e(_tag, '录音流错误: $e'), + ); + // 先挂上事件通道，再开麦，否则开头几包没人接。 + await Future.delayed(const Duration(milliseconds: 30)); + Object? lastError; + for (var attempt = 0; attempt < 3; attempt++) { + if (attempt > 0) { + Logger.w(_tag, '麦克风引擎未就绪，重试 #${attempt + 1}'); + try { + await _iosProbe.invokeMethod('phoneMicStop'); + } catch (_) {} + if (_plainMic) { + await EmaiAudioSession.ensureSimulMic(); + } else { + await EmaiAudioSession.ensure(); + } + await Future.delayed(const Duration(milliseconds: 600)); + } + final err = await _iosProbe.invokeMethod('phoneMicStart', { + 'voiceProcessing': voiceProcessing, + }); + if (err == null || err.isEmpty) { + _running = true; + _buffer.clear(); + _startWatchdog(); + Logger.i(_tag, '麦克风采集启动 ${sampleRate}Hz'); + return true; + } + lastError = err; + Logger.w(_tag, '麦克风引擎未就绪: $err'); + } + Logger.e(_tag, '麦克风采集启动失败: $lastError'); + await _iosSub?.cancel(); + _iosSub = null; + return false; + } on MissingPluginException { + Logger.w(_tag, '手机麦通道不存在，退回 record'); + await _iosSub?.cancel(); + _iosSub = null; + return null; + } catch (e) { + Logger.e(_tag, '麦克风采集启动失败: $e'); + await _iosSub?.cancel(); + _iosSub = null; + return false; + } + } + + void _onIosMicEvent(dynamic event) { + if (event is Uint8List) { + _onChunk(event); + return; + } + if (event is Map && event['type'] == 'opened') { + Logger.w( + _tag, + '麦克风已开 mode=${event['mode']} rate=${event['rate']} ' + 'in=${event['input']} out=${event['output']}', + ); + return; + } + if (event is Map && event['type'] == 'reinstalled') { + Logger.w(_tag, '麦克风输入被会话打断，已重新接上'); + } else if (event is Map && event['type'] == 'reinstallFailed') { + Logger.e(_tag, '麦克风输入被会话打断后没能接上'); + } + } + /// 见 [_start] 里的说明。非 iOS 直接放行；原生不认这个方法（老插件）也放行。 static const MethodChannel _iosProbe = MethodChannel('azure_speech/pcm_player'); @@ -149,6 +273,9 @@ class MicPcmStreamer { static const Duration _silenceLimit = Duration(seconds: 3); + /// 为 true 时，静默只续计时，不重启采集。 + bool holdRestart = false; + void _startWatchdog() { _watchdog?.cancel(); _lastChunkAt = DateTime.now(); @@ -157,15 +284,53 @@ class MicPcmStreamer { final last = _lastChunkAt; if (last == null) return; if (DateTime.now().difference(last) < _silenceLimit) return; - Logger.e(_tag, '录音流已静默 ${_silenceLimit.inSeconds}s，判定断流，尝试重启'); + // 云端还在握手时不要重启麦。重启会改音频会话，握手会被拖成 18801。 + if (holdRestart) { + _lastChunkAt = DateTime.now(); + return; + } _lastChunkAt = DateTime.now(); + if (_kickAttempts < 2) { + _kickAttempts++; + Logger.w(_tag, '录音流已静默 ${_silenceLimit.inSeconds}s，重新接上输入'); + unawaited(kick()); + return; + } + _kickAttempts = 0; + Logger.e(_tag, '录音流重新接上后仍然没有音频，重启采集'); unawaited(_restart()); }); } + /// 输入 tap 断了，但事件通道还在。先重装 tap，避免把整路采集拆掉造成一段空窗。 + Future kick() async { + _kickedAt = DateTime.now(); + if (!_running || kIsWeb || !Platform.isIOS) { + await _restart(); + return; + } + try { + await _iosProbe.invokeMethod('phoneMicKick'); + } catch (e) { + Logger.w(_tag, '重新接上麦克风失败，改为重启: $e'); + await _restart(); + } + } + Future _restart() async { - await stop(); - await start(); + if (_restarting) return; + _restarting = true; + final pin = _pinSpeaker; + final plain = _plainMic; + try { + await stop(); + // 不要在 start 之前探输入。断流时会话可能刚被改成没有输入的类别， + // 这时探到的声道数是 0。start 会先把会话配回 playAndRecord 再等输入就绪。 + final ok = await start(pinSpeaker: pin, plainMic: plain); + if (!ok) Logger.e(_tag, '录音流重启失败'); + } finally { + _restarting = false; + } } /// 音浪发布间隔与窗口峰值。原始块由平台决定大小（Android 几百字节~2KB @@ -190,7 +355,16 @@ class MicPcmStreamer { void _onChunk(Uint8List chunk) { if (chunk.isEmpty) return; - _lastChunkAt = DateTime.now(); + final now = DateTime.now(); + _lastChunkAt = now; + // 重接之后若只来了一包又停，不能把次数清掉，否则永远走不到整路重启。 + final kicked = _kickedAt; + if (_kickAttempts > 0 && + kicked != null && + now.difference(kicked) > const Duration(seconds: 1)) { + _kickAttempts = 0; + _kickedAt = null; + } _emitLevel(chunk); _buffer.add(chunk); @@ -221,6 +395,13 @@ class MicPcmStreamer { _watchdog = null; await _sub?.cancel(); _sub = null; + await _iosSub?.cancel(); + _iosSub = null; + if (!kIsWeb && Platform.isIOS) { + try { + await _iosProbe.invokeMethod('phoneMicStop'); + } catch (_) {} + } try { await _recorder.stop(); } catch (e) { diff --git a/apps/client/lib/data/utils/pcm_stream_player.dart b/apps/client/lib/data/utils/pcm_stream_player.dart index af7d6086..2582ce00 100644 --- a/apps/client/lib/data/utils/pcm_stream_player.dart +++ b/apps/client/lib/data/utils/pcm_stream_player.dart @@ -33,6 +33,7 @@ class PcmStreamPlayer { this.sampleRate = 24000, this.channels = 1, this.playbackOnly = false, + this.pinSpeaker = false, }); static const String _tag = 'PcmStreamPlayer'; @@ -50,6 +51,9 @@ class PcmStreamPlayer { /// 必须 playAndRecord，不能改。 final bool playbackOnly; + /// 译文从手机扬声器出，不跟到已连接的蓝牙耳机。 + final bool pinSpeaker; + /// 原生 PCM 播放通道（两端同名同方法，见 [_useNative] 的说明） static const MethodChannel _nativeChannel = MethodChannel('azure_speech/pcm_player'); @@ -118,11 +122,14 @@ class PcmStreamPlayer { /// /// 显式 setActive(true) 是把会话按本方的配置重新激活一次， /// 相当于「录音刚放开，播放把它拿回来」。 - Future _ensureSession() => EmaiAudioSession.ensure( - profile: playbackOnly - ? EmaiAudioProfile.playbackOnly - : EmaiAudioProfile.phoneMic, - ); + Future _ensureSession() { + if (pinSpeaker) return EmaiAudioSession.pinBuiltInSpeaker(); + return EmaiAudioSession.ensure( + profile: playbackOnly + ? EmaiAudioProfile.playbackOnly + : EmaiAudioProfile.phoneMic, + ); + } /// 正在进行中的 [begin]。[end] 要等它落地才能判断这一轮有没有音频—— /// 回复很短时（音频首包和 RespondingEnded 几乎同时到）`_active` 还是 false， @@ -243,6 +250,7 @@ class PcmStreamPlayer { // 只有车载香薰的设备会话用 playbackOnly（声音走 A2DP）。Android 原生据此在 // 回复之间垫静音，别让 A2DP 流被挂起吞掉下一句的开头；其它调用方不受影响。 'keepAlive': playbackOnly, + 'pinSpeaker': pinSpeaker, }); if (ok != true) { Logger.e(_tag, '原生播放器启动失败'); diff --git a/apps/client/lib/devices/bes/bes_bluetooth_service.dart b/apps/client/lib/devices/bes/bes_bluetooth_service.dart index 48dd7588..0e5debf5 100644 --- a/apps/client/lib/devices/bes/bes_bluetooth_service.dart +++ b/apps/client/lib/devices/bes/bes_bluetooth_service.dart @@ -41,6 +41,9 @@ class BesBluetoothService extends GetxService { /// 连接后补查电量的重试定时器，断开时取消（见 [_scheduleBatteryRetry]） Timer? _batteryRetryTimer; + /// 通话列表帧来了之后等 `BB 8E` 定论。3 秒还没回，才按结束处理。 + Timer? _callListConfirmTimer; + /// 0=未连接 1=连接中 2=已连接（与原生侧约定一致） final RxInt deviceStatus = 0.obs; final RxString connectedDeviceName = ''.obs; @@ -178,6 +181,7 @@ class BesBluetoothService extends GetxService { _statusSubscription?.cancel(); _cmdEventSubscription?.cancel(); _batteryRetryTimer?.cancel(); + _callListConfirmTimer?.cancel(); super.onClose(); } @@ -275,6 +279,8 @@ class BesBluetoothService extends GetxService { } else if (event == 0) { _batteryRetryTimer?.cancel(); _batteryRetryTimer = null; + _callListConfirmTimer?.cancel(); + _callListConfirmTimer = null; connectedDeviceId.value = ''; connectedDeviceName.value = ''; deviceVersion.value = ''; @@ -303,8 +309,8 @@ class BesBluetoothService extends GetxService { final type = event['type']; if (type is String && type.isNotEmpty) lastCmdType.value = type; switch (type) { - // 0x02/0x03 是去电起止 —— **只有它们**代表真实通话状态， - // 通话录音和通话翻译的准入都以此为准。 + // 通话状态以 `BB 8E` 为准。`BB 02` 可以提前置成通话中； + // `BB 03`/`BB 04` 是带号码的通话列表，不当成挂断。 // // ⚠️ **0xD1/0xD2 不在此列**（2026-09-19 真机定性）：它们是 // `callModeStart(0x51)` / `callModeStop(0x52)` 的应答（`| 0x80`）， @@ -319,14 +325,22 @@ class BesBluetoothService extends GetxService { if (!isCallOngoing.value) { Logger.w(_tag, '[通话状态] false → true，触发帧 raw=[$hex]'); } - // ⚠️ 用 info 而不是 debug：这是排查「刚连上耳机就被判成通话中、 - // 同传点不动」的关键线索——原生侧 `case 0x02` 对任何第二字节为 0x02 - // 的帧都会上报 startMakeCall，且不做任何长度/上下文校验， - // 握手期间的推送帧有可能被误判。要看清是哪一帧干的就得有原始字节。 + // `BB 8E` 说还在通话，刚才那帧列表不算挂断。 + if (BesCmd.frameCmd(_asBytes(event['data'])) == 0x8E) { + _callListConfirmTimer?.cancel(); + _callListConfirmTimer = null; + } Logger.i(_tag, 'isCallOngoing → true (type=$type, raw=[$hex])'); isCallOngoing.value = true; break; case 'stopMakeCall': + final bytes = _asBytes(event['data']); + if (BesCmd.isCallListReport(bytes)) { + _confirmCallStillActive(hex); + break; + } + _callListConfirmTimer?.cancel(); + _callListConfirmTimer = null; Logger.i(_tag, 'isCallOngoing → false (type=$type, raw=[$hex])'); isCallOngoing.value = false; break; @@ -525,6 +539,25 @@ class BesBluetoothService extends GetxService { next(); } + List _asBytes(dynamic raw) { + if (raw is! List) return const []; + return raw.map((b) => (b as num).toInt()).toList(); + } + + /// `BB 03`/`BB 04` 来了先问一次 `AA 0E`。`BB 8E` 说还在通话就作废； + /// 3 秒都没回，才按结束处理，避免列表帧把正在进行的通话翻译踢掉。 + void _confirmCallStillActive(String hex) { + if (_callListConfirmTimer?.isActive ?? false) return; + Logger.w(_tag, '通话列表帧不当成挂断，改问 BB 8E raw=[$hex]'); + unawaited(queryCallState()); + _callListConfirmTimer = Timer(const Duration(seconds: 3), () { + _callListConfirmTimer = null; + if (!isCallOngoing.value) return; + Logger.w(_tag, 'BB 8E 3 秒未回，按通话结束处理'); + isCallOngoing.value = false; + }); + } + /// 请求上报通话状态（`AA 0E`），耳机回 `0x8E`： /// data[6] 0x01=来电 / 0x02=通话中 / 0x03=通话结束。 /// 原生侧已把 0x02/0x03 解析成 startMakeCall/stopMakeCall 事件，与 diff --git a/apps/client/lib/devices/bes/bes_protocol.dart b/apps/client/lib/devices/bes/bes_protocol.dart index 2b5e799b..25c2e2d0 100644 --- a/apps/client/lib/devices/bes/bes_protocol.dart +++ b/apps/client/lib/devices/bes/bes_protocol.dart @@ -78,6 +78,17 @@ class BesCmd { static bool switchesMode(int cmd) => cmd >= 0x51 && cmd <= 0x6C; static List frame(int cmd) => [sof, cmd & 0xFF]; + + static int? frameCmd(List data) { + if (data.length < 2) return null; + return data[1] & 0xFF; + } + + /// `BB 03` / `BB 04` 是通话列表，带的是电话号码，不是挂断。 + static bool isCallListReport(List data) { + final cmd = frameCmd(data); + return cmd == 0x03 || cmd == 0x04; + } } /// 耳机 → App 的响应/上报（`BB xx`）。 diff --git a/apps/client/lib/modules/emai/controllers/emai_controller.dart b/apps/client/lib/modules/emai/controllers/emai_controller.dart index e5438dba..4ff5d9b1 100644 --- a/apps/client/lib/modules/emai/controllers/emai_controller.dart +++ b/apps/client/lib/modules/emai/controllers/emai_controller.dart @@ -15,7 +15,10 @@ import '../../../data/models/emai_message.dart'; import '../../../data/services/device_ai_session_service.dart'; import '../../../data/services/emai_chat_store.dart'; import '../../../data/utils/mic_pcm_streamer.dart'; +import '../../../data/services/ai_vendor_lock.dart'; +import '../../../data/services/emai_iflytek_dialog.dart'; import '../../../data/services/mcp_token_service.dart'; +import '../../../data/services/sparkchain_session.dart'; import '../../../data/utils/pcm_stream_player.dart'; /// 会话链路状态（跟 [BailianDialogState] 是两回事：这个说的是"连上了没"， @@ -60,10 +63,14 @@ class EmaiController extends GetxController static const String _tag = 'EmaiController'; final BailianMultimodalService _svc; + final EmaiIflytekDialog? _iflytek = + aiOnlyIflytek ? EmaiIflytekDialog(source: 'emai') : null; final MicPcmStreamer _mic = MicPcmStreamer(sampleRate: BailianMultimodalService.upstreamSampleRate); final PcmStreamPlayer _player = PcmStreamPlayer( - sampleRate: BailianMultimodalService.downstreamSampleRate); + sampleRate: aiOnlyIflytek + ? EmaiIflytekDialog.downstreamSampleRate + : BailianMultimodalService.downstreamSampleRate); // ---------------- UI 状态 ---------------- @@ -74,7 +81,7 @@ class EmaiController extends GetxController final linkState = EmaiLinkState.idle.obs; /// 服务端对话状态：聆听 / 思考 / 回答 - Rx get dialogState => _svc.state; + Rx get dialogState => _iflytek?.state ?? _svc.state; /// 是否处于"按住说话"中 final isTalking = false.obs; @@ -277,8 +284,10 @@ class EmaiController extends GetxController DeviceAiSessionService.to.attachPhonePage(this); } - _subs.add(_svc.events.listen(_onEvent)); - _subs.add(_svc.audioOut.listen((pcm) { + final events = _iflytek?.events ?? _svc.events; + final audio = _iflytek?.audioOut ?? _svc.audioOut; + _subs.add(events.listen(_onEvent)); + _subs.add(audio.listen((pcm) { if (halfDuplexMic) { _closeMicGate(); _onDownlinkAudio(pcm.length); @@ -287,7 +296,12 @@ class EmaiController extends GetxController })); _subs.add(_mic.packets.listen((pcm) { // 门关着就推等长静音，不停推流：服务端的 VAD / 断句按时间走，断流会被当成网络问题 - _svc.pushAudio(_micGated ? Uint8List(pcm.length) : pcm); + final gated = _micGated ? Uint8List(pcm.length) : pcm; + if (_iflytek != null) { + _iflytek.pushAudio(gated); + } else { + _svc.pushAudio(gated); + } })); // ⚠️ 音浪的门控不能只看 isTalking：连续对话下它恒为 false， // 麦克风明明开着，界面上却一点动静都没有，用户完全看不出在不在收音。 @@ -347,6 +361,7 @@ class EmaiController extends GetxController _mic.dispose(); _player.dispose(); _svc.stop(); + unawaited(_iflytek?.stop()); _storeWorker?.dispose(); textController.dispose(); scrollController.removeListener(_onScroll); @@ -394,13 +409,27 @@ class EmaiController extends GetxController return; } - if (!AppConfig.isInitialized() || !_svc.isConfigured) { + if (_iflytek != null) { + if (!await SparkChainSession.ensure()) { + linkState.value = EmaiLinkState.unconfigured; + Logger.w(_tag, '讯飞未就绪，EMAI 起不来'); + return; + } + } else if (!AppConfig.isInitialized() || !_svc.isConfigured) { linkState.value = EmaiLinkState.unconfigured; Logger.w(_tag, 'EMAI 助手未配置：${_svc.configHint}'); return; } linkState.value = EmaiLinkState.connecting; + if (_iflytek != null) { + final ok = await _iflytek.start(); + if (!ok) { + linkState.value = EmaiLinkState.failed; + _pushError('emaiConnectFailed'.tr); + } + return; + } // MCP 令牌要在 Start 之前拿到：biz_params 是握手时定死的， // 会话建起来之后再补就没地方放了。 final mcpToken = await McpTokenService.get(); @@ -427,8 +456,9 @@ class EmaiController extends GetxController _handshakeTimer?.cancel(); _autoRetried = false; await _abortTalk(); - await _svc.stop(); linkState.value = EmaiLinkState.idle; + await _svc.stop(); + await _iflytek?.stop(); await connect(); } @@ -442,8 +472,9 @@ class EmaiController extends GetxController await _abortTalk(); await _player.stopNow(); if (halfDuplexMic) _openMicGate(); - await _svc.stop(); linkState.value = EmaiLinkState.onEarphone; + await _svc.stop(); + await _iflytek?.stop(); } /// 手机页面主动把控制权要回来：让耳机退出 AI 模式。 diff --git a/apps/client/lib/modules/login/controllers/login_controller.dart b/apps/client/lib/modules/login/controllers/login_controller.dart index 38057b6f..fe1b7358 100644 --- a/apps/client/lib/modules/login/controllers/login_controller.dart +++ b/apps/client/lib/modules/login/controllers/login_controller.dart @@ -19,6 +19,7 @@ import '../../../routes/app_routes.dart'; import '../../../data/models/appconfig.dart'; import '../../../data/models/appconfig_model.dart'; import '../../../data/services/network/api.dart'; +import '../../../data/services/sparkchain_session.dart'; import '../../../../data/models/user_Info.dart'; import '../../../data/models/channel_app_model.dart'; import '../../../core/utils/device_brand.dart'; @@ -845,6 +846,7 @@ class LoginController extends GetxController { } AppConfig.initialize(resp); + await SparkChainSession.warmUp(); Logger().d( 'getAppConfig: AppConfig初始化完成，产品数量验证 = ${AppConfig.config.products.length}'); diff --git a/apps/client/lib/modules/meeting/controllers/meeting_details_controller.dart b/apps/client/lib/modules/meeting/controllers/meeting_details_controller.dart index 39052576..09c2a4a4 100644 --- a/apps/client/lib/modules/meeting/controllers/meeting_details_controller.dart +++ b/apps/client/lib/modules/meeting/controllers/meeting_details_controller.dart @@ -452,6 +452,7 @@ class MeetingDetailsController extends GetxController void downloadAudio() async { if (isDownload.value) return; + if (meetingData.value.audiourl.isEmpty) return; isDownload.value = true; final directory = await getApplicationDocumentsDirectory(); final directoryDir = Directory('${directory.path}/DownloadAudio'); diff --git a/apps/client/lib/modules/picture_translation/controllers/picture_translation_controller.dart b/apps/client/lib/modules/picture_translation/controllers/picture_translation_controller.dart index e48c9639..c0abbc9e 100644 --- a/apps/client/lib/modules/picture_translation/controllers/picture_translation_controller.dart +++ b/apps/client/lib/modules/picture_translation/controllers/picture_translation_controller.dart @@ -559,12 +559,7 @@ class PictureTranslationController extends GetxController { return languageManager.getChineseNameByAsrCode('zh-CN'); } - final normalizedShortCode = switch (languageCode) { - 'nb' => 'no', - _ => languageCode, - }; - - return languageManager.getChineseNameByShortCode(normalizedShortCode); + return languageManager.getChineseNameByShortCode(languageCode); } String? _resolveDetectedSourceLanguageAsrCode(ImageTranslationResult result) { diff --git a/apps/client/lib/modules/splash/controllers/splash_controller.dart b/apps/client/lib/modules/splash/controllers/splash_controller.dart index e882b1da..46891a7b 100644 --- a/apps/client/lib/modules/splash/controllers/splash_controller.dart +++ b/apps/client/lib/modules/splash/controllers/splash_controller.dart @@ -18,6 +18,7 @@ import '../../../modules/common/widgets/version_update_dialog.dart'; import '../../../modules/common/widgets/privacy_policy_dialog.dart'; import '../../../core/utils/logger.dart'; import '../../../data/services/db/sqflite_api.dart'; +import '../../../data/services/sparkchain_session.dart'; class SplashController extends GetxController { final GetStorage _storage = GetStorage(); @@ -214,21 +215,18 @@ class SplashController extends GetxController { Future _validateTokenAsync() async { try { final token = _storage.read('logintoken'); + var loggedIn = false; if (token != null && token.isNotEmpty) { - final isLoggedIn = await getLoginToken(); - if (isLoggedIn) { - // 只有登录成功，才获取App配置 - await getAppConfig(); - // ⚠️ 会议模板**不放在启动关键路径上**：它只被会议模板页读取 - // （落 sqlite，由 SqfliteApi.getMeetingTemplateList 消费）， - // 启动时没有任何地方要用。原来 await 它等于让每次冷启动的白屏 - // 多背一次网络往返——而 _initializeApp 要等这一整串跑完才允许导航。 - // 改成后台拉：模板照常更新，启动不再等它。 - unawaited(_initMeetingTemplate()); - } else { - await User.clearUserSession(); // + loggedIn = await getLoginToken(); + if (!loggedIn) { + await User.clearUserSession(); } } + // 讯飞密钥在这份配置里。未登录也拉：v3 在白名单里，否则停在登录页看不出密钥有没有下发。 + await getAppConfig(); + if (loggedIn) { + unawaited(_initMeetingTemplate()); + } } catch (e) { Logger.e(_tag, 'Token验证或配置加载失败: $e'); await User.clearUserSession(); @@ -309,6 +307,7 @@ class SplashController extends GetxController { if (response == null) return false; final resp = UserGetAppConfigResp.fromJson(response); AppConfig.initialize(resp); + await SparkChainSession.warmUp(); return true; } catch (e) { Logger.e(_tag, 'getAppConfig 获取失败: $e'); diff --git a/apps/client/lib/modules/text_translation/controllers/text_translation_controller.dart b/apps/client/lib/modules/text_translation/controllers/text_translation_controller.dart index 7c4f9442..ef5812ed 100644 --- a/apps/client/lib/modules/text_translation/controllers/text_translation_controller.dart +++ b/apps/client/lib/modules/text_translation/controllers/text_translation_controller.dart @@ -421,11 +421,6 @@ class TextTranslationController extends GetxController { return _languageManager.getChineseNameByAsrCode('zh-CN'); } - final normalizedShortCode = switch (languageCode) { - 'nb' => 'no', - _ => languageCode, - }; - - return _languageManager.getChineseNameByShortCode(normalizedShortCode); + return _languageManager.getChineseNameByShortCode(languageCode); } } diff --git a/apps/client/lib/modules/translation/controllers/translation_controller.dart b/apps/client/lib/modules/translation/controllers/translation_controller.dart index 132d0f59..77b446ee 100644 --- a/apps/client/lib/modules/translation/controllers/translation_controller.dart +++ b/apps/client/lib/modules/translation/controllers/translation_controller.dart @@ -16,7 +16,6 @@ import 'package:path_provider/path_provider.dart'; import 'package:permission_handler/permission_handler.dart'; import 'package:device_info_plus/device_info_plus.dart'; import '../../../data/services/ast_service.dart'; -import '../../../data/services/microsoft_text_translation.dart'; import '../../../data/services/server_translation_service.dart'; import '../../../data/services/music_manager.dart'; import '../../../data/services/tts_service.dart'; @@ -27,8 +26,9 @@ import '../../../core/utils/toast_throttle.dart'; import '../../../core/services/floating_window_manager.dart'; import '../../../data/services/asr_service.dart'; +import '../../../data/services/simul_microsoft.dart'; import '../../../data/services/sparkchain_session.dart'; -import '../../../data/services/ai_vendor_lock.dart'; +import '../../../data/utils/emai_audio_session.dart'; import '../../../data/utils/mic_pcm_streamer.dart'; import '../../../data/utils/pcm_stream_player.dart'; import '../../../data/services/usage_stats_service.dart'; @@ -68,6 +68,10 @@ enum AudioOutputDevice { /// [device]，业务逻辑无需再改。 enum FaceToFaceTrigger { client, device } +/// 设备双耳面对面还没做好。为 true 时不发拆双耳的指令， +/// 手机麦收音，译文从手机扬声器播。固件能双耳翻译后再改回 false。 +const faceToFaceUsePhoneForTest = true; + /// 翻译主控制器 - 负责管理语音识别、翻译和TTS功能(主控制器) class TranslationController extends GetxController with WidgetsBindingObserver { static const String TAG = 'TranslationController'; @@ -80,8 +84,6 @@ class TranslationController extends GetxController with WidgetsBindingObserver { // 以前直连火山，AK/SK 要明文下发给每个客户端，换服务商还得发版。 final ServerTranslationService _translationService = Get.find(); - final MicrosoftTextTranslation _microsoftTranslation = - MicrosoftTextTranslation(); final TtsService _ttsService = Get.find(); final AstService _astService = Get.find(); final LanguageManager _languageManager = Get.find(); @@ -97,7 +99,6 @@ class TranslationController extends GetxController with WidgetsBindingObserver { /// "AST 已经被 initialize() 自动拉起，不用再调一次"，startRecognition 消费一次 /// 后清掉；真正停止过之后（stopRecognition/_terminateCallOnError）也要清掉， /// 否则下次点开始会以为不需要再调，实际原生那边已经停了。 - bool _astAutoStartPending = false; // AST 掉线自动重连：切后台被 ROM 冻结、弱网抖动都会让 WebSocket 断开， // 那不是「功能该结束了」，用户还在通话里。见 _handleAstDrop。 @@ -139,6 +140,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver { // ==================== ASR统计相关 ==================== DateTime? _asrSessionStartTime; // ASR会话开始时间 + DateTime? _asrStatsCursor; + bool _handoffInFlight = false; DateTime? _asrActiveStartTime; // ASR活跃开始时间（用户实际开始语音识别的时间） Duration _asrActiveDuration = Duration.zero; // ASR累计活跃时长 bool _isAsrActive = false; // ASR是否处于活跃状态 @@ -171,6 +174,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { DeviceSession? _f2fSession; final isRecognizing = false.obs; final isPreparing = false.obs; + bool _startInFlight = false; final isTranslating = false.obs; final isTtsEnabled = true.obs; final isRecording = false.obs; @@ -196,6 +200,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver { final detectedLanguageCode1 = ''.obs; final latestRecognitionText = ''.obs; final faceToFaceRecognitionText = ''.obs; + /// 面对面这一句听出来的是目标语，气泡方向要反过来。 + final Set _astReversedUtterances = {}; // ==================== 翻译相关 ==================== var currsessionid = ''; final currentModeTitle = 'simultaneousTranslation'.obs; @@ -237,9 +243,27 @@ class TranslationController extends GetxController with WidgetsBindingObserver { StreamSubscription? _simHeadsetMicSub; /// 本轮同声翻译的识别、翻译、合成走讯飞。密钥没配好时为 false，沿用原来的识别。 bool _sparkListen = false; + /// 同声走一条 AST。译文和音频都从这条会话来，不再另翻、另合成。 + bool _sparkAst = false; + /// 拼接路径的微软神经音色，区域码全名。 + String _sparkVoice = ''; + Future _sparkSpeakChain = Future.value(); MicPcmStreamer? _sparkMic; StreamSubscription? _sparkMicSub; PcmStreamPlayer? _sparkPlayer; + /// 同声 / 面对面走微软端到端时，麦克风要自己推进去。 + /// 原生 `startContinuousTranslation` 只开外部音频口，不打开手机麦。 + bool _e2eMicFeed = false; + bool _e2ePinSpeaker = false; + DateTime? _e2eRestartAt; + MicPcmStreamer? _e2eMic; + StreamSubscription? _e2eMicSub; + PcmStreamPlayer? _e2ePlayer; + StreamSubscription? _e2eTtsSub; + /// 译文还在扬声器里的截止时间。这段时间麦克风改送静音，避免把播报又识别进去。 + DateTime? _e2ePlaybackUntil; + bool _e2eMicMuted = false; + Uint8List _e2eSilence = Uint8List(0); DeviceFileRecorder? _simHeadsetRec; DeviceAudioSource? _callLocalSrc; DeviceAudioSource? _callPeerSrc; @@ -563,10 +587,11 @@ class TranslationController extends GetxController with WidgetsBindingObserver { await _disableFloatingWindow(); } - if (mode == 'call') { - await _initializeCallModeTranslationService(); - _astAutoStartPending = true; - } else { + // 通话、同声、面对面、音视频的识别、翻译、合成都走讯飞 SDK，不在进页面时连微软。 + if (mode != 'call' && + mode != 'simultaneous' && + mode != 'faceToFace' && + mode != 'audioVideo') { await _reinitializeAsrService(); } } @@ -651,11 +676,11 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } // 按触发来源定收音路径（设备端触发 → 设备端收音） await _configureAudioMode(); - await _asrService.startContinuousRecognition( - _audioSourceType, - isRemoveFirstPunctuation: true, - mode: 'push_to_talk', - ); //开启识别 + final started = await _startSparkMicrosoft(); + if (!started) { + activeSpeaker.value = 0; + return; + } _startAsrActiveTracking(); //开启识别活动跟踪 if (isRecording.value) { if (isCreateRecord.value) { @@ -683,10 +708,10 @@ class TranslationController extends GetxController with WidgetsBindingObserver { faceToFaceRecognitionText.value = ''; } await _asrService.stopContinuousRecognition(); - _stopAsrActiveTracking(); //停止识别活动跟踪 - _recordAsrUsageStats(sourceLanguage.value); + _stopAsrActiveTracking(); + // 松开只是一句结束，整场还没交到纪要。会话时钟留着，下一句接着用。 + _recordAsrUsageStats(sourceLanguage.value, endSession: false); isRecognizing.value = false; - currentSessionId = null; Logger.info('面对面翻译：停止语音识别'); } catch (e) { @@ -928,12 +953,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { return _languageManager.getChineseNameByAsrCode('zh-CN'); } - final normalizedShortCode = switch (languageCode) { - 'nb' => 'no', - _ => languageCode, - }; - - return _languageManager.getChineseNameByShortCode(normalizedShortCode); + return _languageManager.getChineseNameByShortCode(languageCode); } /// 请求录音权限 @@ -1081,6 +1101,56 @@ class TranslationController extends GetxController with WidgetsBindingObserver { /// 这里让后来的等前一条整个跑完再开始；服务层 AzureAstService 还有一道同样的闸。 Future _callInitChain = Future.value(); + /// 这一对语言在微软语音翻译表里，同声 / 面对面 / 通话走端到端。 + /// 表里没有的返回 false，调用方改走微软听写 + 机器翻译 + 合成。 + bool _microsoftE2eAvailable() { + final sourceShort = _languageManager + .getLanguageByChineseName(sourceLanguage.value) + ?.shortCode ?? + 'zh'; + final targetShort = _languageManager + .getLanguageByChineseName(targetLanguage.value) + ?.shortCode ?? + 'en'; + final best = + _languageManager.findBestMatchingProvider(sourceShort, targetShort); + return best != null && best.provider == 'azure'; + } + + /// 端到端会话的语言表：[识别源, 识别目标, 翻译源, 翻译目标, 源音色, 目标音色]。 + List _currentCallLanguages() { + final sourceInfo = + _languageManager.getLanguageByChineseName(sourceLanguage.value); + final targetInfo = + _languageManager.getLanguageByChineseName(targetLanguage.value); + final sourceShort = sourceInfo?.shortCode ?? + _languageManager.getShortCodeByAsrCode(sourceLanguageCode.value) ?? + 'zh'; + final targetShort = targetInfo?.shortCode ?? + _languageManager.getShortCodeByAsrCode(targetLanguageCode.value) ?? + 'en'; + final bestProvider = + _languageManager.findBestMatchingProvider(sourceShort, targetShort); + final transCode0 = bestProvider?.sourceSpec['translationCode'] ?? + _languageManager.microsoftTranslateCode(sourceShort); + final transCode1 = bestProvider?.targetSpec['translationCode'] ?? + _languageManager.microsoftTranslateCode(targetShort); + final ttsVoice0 = bestProvider?.sourceSpec['ttsCode'] ?? + (_languageManager.getTtsVoiceNameByAsrCode(sourceLanguageCode.value) ?? + 'zh-CN-XiaoxiaoNeural'); + final ttsVoice1 = bestProvider?.targetSpec['ttsCode'] ?? + (_languageManager.getTtsVoiceNameByAsrCode(targetLanguageCode.value) ?? + 'en-US-AriaNeural'); + return [ + sourceLanguageCode.value, + targetLanguageCode.value, + transCode0, + transCode1, + ttsVoice0, + ttsVoice1, + ]; + } + Future _initializeCallModeTranslationService({bool isRecovery = false}) { final next = _callInitChain.then( (_) => _initializeCallModeTranslationServiceInner(isRecovery: isRecovery)); @@ -1115,29 +1185,20 @@ class TranslationController extends GetxController with WidgetsBindingObserver { '[TRANS-INIT] 未找到匹配的提供商，回退到默认 ${LanguageManager.providerDisplayName('azure')}[azure]（asrCode=${sourceLanguageCode.value} -> ${targetLanguageCode.value}）'); } - // 构建完整的语言参数：[asrCode0, asrCode1, transCode0, transCode1, ttsVoice0, ttsVoice1] - // 没匹配上提供商时按语言名取微软翻译码。繁体的识别码是 zh-TW / zh-HK， - // 直接拿去翻译会被当成简体。 - final transCode0 = bestProvider?.sourceSpec['translationCode'] ?? - _languageManager.microsoftTranslateCode(sourceShort); - final transCode1 = bestProvider?.targetSpec['translationCode'] ?? - _languageManager.microsoftTranslateCode(targetShort); - final ttsVoice0 = bestProvider?.sourceSpec['ttsCode'] ?? - (_languageManager.getTtsVoiceNameByAsrCode(sourceLanguageCode.value) ?? 'zh-CN-XiaoxiaoNeural'); - final ttsVoice1 = bestProvider?.targetSpec['ttsCode'] ?? - (_languageManager.getTtsVoiceNameByAsrCode(targetLanguageCode.value) ?? 'en-US-AriaNeural'); - final List callModeLanguages = [ - sourceLanguageCode.value, // [0] ASR source - targetLanguageCode.value, // [1] ASR target - transCode0, // [2] translation source - transCode1, // [3] translation target - ttsVoice0, // [4] TTS voice source - ttsVoice1, // [5] TTS voice target - ]; + final List callModeLanguages = _currentCallLanguages(); Logger.info( '[TRANS-INIT] 最终启动服务: ${LanguageManager.providerDisplayName(astProvider)}[$astProvider], asrSrc=${callModeLanguages[0]}, asrTgt=${callModeLanguages[1]}, transSrc=${callModeLanguages[2]}, transTgt=${callModeLanguages[3]}, ttsSrc=${callModeLanguages[4]}, ttsTgt=${callModeLanguages[5]}'); - await _astService.initialize(supportedLanguages: callModeLanguages, provider: astProvider); + // 通话是两路（本端麦 + 对端）。同声和面对面只有手机/耳机这一路麦， + // 再建 B 路只会空等 1.2 秒，冷启动时还和保活抢音频会话。 + final bool singleMic = currentMode.value == 'simultaneous' || + currentMode.value == 'faceToFace'; + await _astService.initialize( + supportedLanguages: callModeLanguages, + provider: astProvider, + singleLeg: singleMic, + bidirectional: currentMode.value == 'faceToFace', + ); // ⚠️ 新会话 = 重新刻一次音色，门限必须跟着重新关上，否则重连后照样刻到串音。 _peerGate.reset(); @@ -1325,6 +1386,10 @@ class TranslationController extends GetxController with WidgetsBindingObserver { _simHeadsetRec != null ? '同声翻译录音走耳机麦' : '同声翻译耳机麦落盘未开，识别仍走耳机麦'); + } else if (_e2eMicFeed) { + // 端到端已经占着手机麦。再开一路 ASR 录音会把麦抢走，识别一个字都进不去。 + _currentRecordingPath = null; + Logger.w('Translation', '端到端收音中，不再另开录音麦'); } else { AudioSourceType audioSourceType; if (currentMode.value == 'audioVideo') { @@ -1341,24 +1406,17 @@ class TranslationController extends GetxController with WidgetsBindingObserver { _pauseStartedAt = null; } - /// 停止录音 + /// 停止录音。 /// - /// 通话 / 同声 / 面对面翻译的录音停止后归档进「语音纪要」，**连同这段 - /// 时间里翻译出的整句一起送过去**：纪要那边不再重新识别，直接用智能模板总结 - /// （2026-09-18 产品口径，见 [RecordingArchive.archive] 的 `transcript`）。 - /// 录音在这三个模式下默认开（见 [_initializeSession]），用户本轮可随时关。 + /// 同声 / 面对面 / 通话结束时不靠这里进纪要：那三条只交已翻译的文字， + /// 见 [_handoffSessionToMeeting]。这里若落过文件，直接删掉，不上传。 Future stopRecording() async { - // stopTimer() 会把 _seconds 清零，归档要用的时长必须在它之前取 - final int seconds = _timerManager.currentSeconds; final String? path = _currentRecordingPath; _currentRecordingPath = null; - // 面对面翻译的会话是在「暂停」状态下结束的（最后一句松手就暂停了）， - // 把这段收尾进暂停区间，最后一句之后的空档不算进偏移。 if (_pauseStartedAt != null) { _recordingPauses.add((_pauseStartedAt!, DateTime.now())); _pauseStartedAt = null; } - final DateTime? startedAt = _recordingStartedAt; _recordingStartedAt = null; _timerManager.stopTimer(); @@ -1371,7 +1429,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { _simHeadsetRec = null; } else if (currentMode.value == 'call') { await _astService.stopRecord(true); - } else { + } else if (!_e2eMicFeed) { await _asrService.stopRecord(true); } isCreateRecord.value = false; @@ -1381,55 +1439,75 @@ class TranslationController extends GetxController with WidgetsBindingObserver { // 用户以为翻译还活着。收口到这里，所有停止路径都覆盖得到。 isRecording.value = false; Logger.info('录音已停止，计时器已重置'); - - if (_shouldArchiveRecording && path != null && seconds > 0) { - // 转写在归档**之前**就地算好：列表和暂停区间都是 controller 的状态， - // 归档是异步的，等它跑到的时候页面可能已经退了、下一轮也可能已经开始。 - final transcript = startedAt == null - ? const >[] - : buildMeetingTranscript( - items: translationHistory.toList(), - recordingStartedAt: startedAt, - mode: currentMode.value, - ownLanguageCode: sourceLanguageCode.value, - pauses: List.of(_recordingPauses), - ); - Logger.i('Translation', - '归档翻译录音：${seconds}s 转写 ${transcript.length} 段 模式=${currentMode.value}'); - // ⚠️ 不 await：归档里有一次 echomeet_addrecord 网络请求（Dio 30s - // connect + 30s receive），await 会把「点结束」一路堵到超时。 - // 归档本身不依赖 controller（RecordingArchive 是静态的）， - // 页面退了也能跑完。 - unawaited(RecordingArchive.archive( - path: path, - seconds: seconds, - tag: 'TranslationController', - transcript: transcript, - // 总结用用户自己的语言（源语言）；对方说的外语那几句原文+译文都在转写里 - summaryLanguage: sourceLanguageCode.value, - )); + if (_shouldArchiveRecording && path != null) { + try { + final file = File(path); + if (await file.exists()) await file.delete(); + } catch (e) { + Logger.w('Translation', '删除翻译临时录音失败: $e'); + } } _recordingPauses.clear(); } - /// 这个模式的录音要不要进「语音纪要」。 - /// - /// 通话 / 同声 / 面对面翻译都要（2026-09-18 起面对面也归档：它逐句按轮次 - /// 暂停/继续，偏移由 [_recordingPauses] 扣掉）。音视频翻译录的是系统播放声， - /// 归档语义还没定，先不动。 + /// 同声 / 面对面 / 通话结束时把已翻译的句子交给语音纪要。音视频不走这条。 bool get _shouldArchiveRecording => currentMode.value == 'call' || currentMode.value == 'simultaneous' || currentMode.value == 'faceToFace'; - /// 这三个模式默认开录音：不录就没有文件，也就没有东西进语音纪要， - /// 而产品口径是「翻译完在纪要里能看到音频 + 转写 + 总结」。 - bool get _recordsByDefault => _shouldArchiveRecording; + /// 这三条不再默认录音：纪要只要文字，不存音频。 + bool get _recordsByDefault => false; /// 归档文件名前缀，取值见 RecordingArchive 的约定。 String get _archivePrefix => currentMode.value == 'call' ? 'trans_call' : 'trans_holder'; + /// 把本场已翻译的整句交给语音纪要，成功后清掉翻译页这场记录。 + /// + /// 只在会话真正结束时调。中途关录音不走这里。网络失败时翻译页内容留着。 + void _handoffSessionToMeeting() { + if (!_shouldArchiveRecording || _handoffInFlight) return; + final sessionId = currentSessionId; + final startedAt = _asrSessionStartTime ?? _recordingStartedAt; + if (sessionId == null || startedAt == null) return; + final pauses = List<(DateTime, DateTime)>.of(_recordingPauses); + if (_pauseStartedAt != null) { + pauses.add((_pauseStartedAt!, DateTime.now())); + } + final transcript = buildMeetingTranscript( + items: translationHistory.toList(), + recordingStartedAt: startedAt, + mode: currentMode.value, + ownLanguageCode: sourceLanguageCode.value, + pauses: pauses, + ); + if (transcript.isEmpty) return; + var seconds = _timerManager.currentSeconds; + if (seconds < 1) { + seconds = DateTime.now().difference(startedAt).inSeconds; + } + if (seconds < 1) seconds = 1; + final title = + '${_archivePrefix}_${DateFormat('yyyyMMdd_HHmmss').format(DateTime.now())}'; + final language = sourceLanguageCode.value; + _handoffInFlight = true; + unawaited(() async { + try { + final id = await RecordingArchive.archiveTranscript( + title: title, + seconds: seconds, + transcript: transcript, + summaryLanguage: language, + tag: 'TranslationController', + ); + if (id != null) deleteSessionHistory(sessionId); + } finally { + _handoffInFlight = false; + } + }()); + } + /// 暂停录音 Future pauseRecording() async { _pauseStartedAt ??= DateTime.now(); @@ -1496,7 +1574,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver { if (!await _checkPermissions()) return; if (isRecognizing.value) return; // 上一次点击还在准备中（等初始化 / 开桥），再点一次会起第二条初始化与第一条并发。 - if (isPreparing.value) { + // stopAll 会把 isPreparing 清掉，但原生 start 可能还堵在队列里，所以另用一把锁。 + if (isPreparing.value || _startInFlight) { Logger.w('Translation', '识别正在准备中，忽略重复的开始请求'); return; } @@ -1506,6 +1585,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { // 用户看到的就是「点了没反应」。ASR 首次初始化要连 Azure，本来就有秒级耗时， // 没有反馈时这段等待格外难受。下面的 finally 负责清，所有 return 路径都覆盖。 isPreparing.value = true; + _startInFlight = true; final startedAt = DateTime.now(); try { // 等 onInit 里那次按当前语言对做的 ASR 初始化落地，别让它被兜底的 @@ -1597,56 +1677,55 @@ class TranslationController extends GetxController with WidgetsBindingObserver { return; } if (currentMode.value == 'simultaneous') { - // 耳机麦不经过手机录音，只挂麦克风型前台服务时华为仍会掐后台网络。 - // 保活必须在还处于前台时拉起。 - await AudioForegroundService.start(keepalive: true, pinSpeaker: false); - // iOS 没有安卓那条前台服务。同声翻译的收音走耳机、译文走系统播放， - // 切后台要靠静音循环把 audio 后台模式撑住。 - if (Platform.isIOS) { - await BackgroundKeepalive.acquire('simultaneous'); + final directed = _directedShortCodes(); + final route = SimulMicrosoft.plan(directed.source, directed.target); + if (route == null) { + Logger.w( + 'Translation', + '同声翻译 ${directed.source}->${directed.target} 不在微软同传表，' + '也不是中文、粤语、爱尔兰语，不启动', + ); + Get.snackbar('tip'.tr, '这一对语言没有微软同声'); + return; } await _armSimultaneousInput(); - Logger.w( - 'Translation', - '开始语音识别，模式：同声传译，收音=${_simHeadsetSession != null ? "耳机麦" : "手机麦克风"}'); - final spark = await _startSparkListen(); - if (!spark) { - if (aiOnlyIflytek) { - Logger.w('Translation', '测试服只允许科大讯飞，同声翻译不改走原识别'); - } else { - await _startAsrService('phone_call'); - } + final started = await _startSimul(route); + if (!started) { + Get.snackbar( + 'tip'.tr, + '同声翻译没启动(${SparkChainSession.lastStartCode})', + ); + throw StateError('同声翻译没启动'); } - await _attachSimultaneousHeadsetMic(); + // iOS 不再播静音保活。手机麦已经占着 playAndRecord，进程退后台也活着。 + // 静音播放器再起一个引擎，会把输入 tap 停掉，第一句空几秒； + // 等它开播还会卡住，开始按钮一直停在准备中。 + await AudioForegroundService.start(keepalive: true, pinSpeaker: false); } else if (currentMode.value == 'faceToFace') { - final bes = _besFaceToFaceSession(); + final bes = faceToFaceUsePhoneForTest ? null : _besFaceToFaceSession(); if (bes != null) { await _startBesFaceToFace(bes); } else { - // 没有 Echo-one 时仍走手机麦克风，其它设备的面对面不受影响。 - Logger.i('Translation', '开始语音识别，模式：面对面(客户端触发/手机收音)，ASR normal'); - await _startAsrService('normal'); + if (faceToFaceUsePhoneForTest) { + await EmaiAudioSession.pinBuiltInSpeaker(); + } + final started = await _startSparkMicrosoft(); + if (!started) { + throw StateError('面对面翻译没启动'); + } } } else if (currentMode.value == 'call') { - if (_astAutoStartPending) { - // initialize() 已经自动拉起过一次了，见 _astAutoStartPending 注释—— - // 这里不能再显式调用，否则会跟那次握手中的连接抢 generation。 - Logger.i('Translation', '通话模式：AST 已由 initialize() 自动启动，跳过重复调用'); - _astAutoStartPending = false; - } else { - Logger.i('Translation', '通话模式：启动 AST 端到端服务'); - // 出错终止过的页面订阅已经没了，不补上这一轮就是「聋」的 - if (_astEventSubscription == null) { - await _subscribeAstEvents(); - } - await _astService.startContinuousTranslation(); + Logger.w('Translation', '通话翻译走讯飞 SDK，本端这一路识别、翻译、合成'); + final started = await _startSparkMicrosoft(openMic: false); + if (!started) { + throw StateError('通话翻译没启动'); + } + } else if (isIosAudioVideo || currentMode.value == 'audioVideo') { + Logger.w('Translation', '音视频翻译走讯飞 SDK'); + final started = await _startSparkMicrosoft(); + if (!started) { + throw StateError('音视频翻译没启动'); } - } else if (isIosAudioVideo) { - // 专用的 media 模式：原生据此把 AVAudioSession 配成 - // .playAndRecord + .mixWithOthers + .default —— 既不打断正在播放的 - // 那个 App，又不开回声消除（AEC 会把本机扬声器的声音当回声消掉）。 - Logger.i('Translation', '开始语音识别，模式：音视频(iOS/手机麦克风外放收音)，ASR media'); - await _startAsrService('media'); } else { Logger.i('Translation', '开始语音识别，模式：${currentMode.value}，ASR 模式：normal'); @@ -1656,7 +1735,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { // Android 音视频翻译：识别就绪后才弹"录屏"授权 — 用户同意的瞬间 // 系统音频开始流入已经准备好的 ASR 通路，可以立即切到目标 App。 - if (isAndroidAudioVideo || isIosAudioVideo) { + if ((isAndroidAudioVideo || isIosAudioVideo) && !_sparkListen) { // iOS：起 socket 接收端 + 弹系统「开始广播」面板。 // ⚠️ 返回 true 只代表面板弹出来了，**不代表用户真的开始了广播**—— // 系统没有可查询的回调。所以下面的回滚分支在 iOS 上只会在 @@ -1679,6 +1758,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } catch (e) { await _abortRecognition('启动语音识别失败: ${e.toString()}'); } finally { + _startInFlight = false; isPreparing.value = false; final totalMs = DateTime.now().difference(startedAt).inMilliseconds; // 只在明显偏慢时用 warning 记一笔：release 的日志级别是 warning， @@ -1692,10 +1772,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { /// 识别异常终止时的统一回滚。 /// - /// ⚠️ **必须把录音一起停掉**：同传的录音是 [_finalizeRecognitionStart] 跟着识别 - /// 一起拉起来的（「同声翻译默认开录音」），识别挂了而录音还在跑，表现就是 - /// 右上角还在计时、用户以为翻译活着，实际早停了。 - /// [stopRecording] 内部会把已录的部分正常落盘并归档，不会丢内容。 + /// 识别挂了也要把已经翻出来的句子交给纪要，再停掉还开着的录音。 Future _abortRecognition(String reason) async { Logger.error('识别异常终止: $reason'); isPreparing.value = false; @@ -1703,6 +1780,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { if (_f2fBleActive || _f2fSession != null) { await _stopBesFaceToFace(); } + _handoffSessionToMeeting(); if (isRecording.value) { try { await stopRecording(); @@ -1710,12 +1788,11 @@ class TranslationController extends GetxController with WidgetsBindingObserver { Logger.error('异常终止时停止录音失败(忽略): $e'); } } + await _stopSparkInput(); + await _stopE2eInput(); await _stopSimultaneousHeadsetMic(); if (currentMode.value == 'simultaneous') { await AudioForegroundService.stop(); - if (Platform.isIOS) { - await BackgroundKeepalive.release('simultaneous'); - } } // 音视频/通话模式：识别挂了，设备侧/录屏广播那条链路也得收掉， // 否则 socket 接收端还开着、没人消费（正常 stopAll 走的是同一个收尾，幂等）。 @@ -1741,11 +1818,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { /// 初始化会话 Future _initializeSession() async { currentSessionId = DateTime.now().millisecondsSinceEpoch.toString(); - // 通话 / 同声 / 面对面默认开录音，每轮开始回到默认开（本轮关掉只管本轮）。 - // 放在这里而不是 _finalizeRecognitionStart：设备端触发的面对面（按住说话） - // 不经过后者，但第一句一定经过这里。 - // ⚠️ 通话翻译录的是通话双方的声音。默认录是 2026-09-18 的产品决定 - // （翻译完要能在纪要里回听），上线前请确认录音告知/合规口径。 + _astReversedUtterances.clear(); if (_recordsByDefault && !isRecording.value) { isRecording.value = true; } @@ -1753,6 +1826,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { try { _usageService.startNewSession(); _asrSessionStartTime = DateTime.now(); + _asrStatsCursor = null; _asrActiveDuration = Duration.zero; _isAsrActive = false; } catch (e) { @@ -1786,7 +1860,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver { /// 设备端那条链路还在等固件（见 docs/face_to_face_ble_protocol.md §14）， /// 所以当前实际生效的是手机麦克风这一路。 Future _configureFaceToFaceMode() async { - _audioSourceType = _isFaceToFaceDeviceMode; + _audioSourceType = + faceToFaceUsePhoneForTest ? false : _isFaceToFaceDeviceMode; isTtsEnabled.value = true; Logger.info( '面对面翻译收音来源：${_audioSourceType ? "设备端(BLE)" : "手机麦克风"}' @@ -2043,108 +2118,16 @@ class TranslationController extends GetxController with WidgetsBindingObserver { final session = _callSession; if (session == null) return; - // ---- 首选：原生直连桥（两个原生插件之间直接传 PCM，Dart 一帧都不碰）---- - // - // 原来这条链路是 `恒玄插件 → EventChannel → Dart → MethodChannel → azure_speech`， - // 20ms 一帧、双路，每秒约 200 次跨界。长句一多 event loop 就被拖住，原生那边 - // 报 AudioSendSlow 1011——2026-09-20 苹果测试「WiFi 下断断续续、长句基本不行」 - // 就是它。桥开着时门限（PeerSpeechGate）也在原生做，重连期间的帧由原生缓存补推。 - // 任一端不支持（老包 / 别的厂商）就退回下面的 Dart 流路径，功能不受影响。 - if (await _tryEnableNativeCallBridge(session)) { - _callBridgeNative = true; - // iOS 后台保活。通话翻译的两路音频**全走耳机 GATT**，手机侧没有任何 - // AVAudioSession 的实际 I/O（译文也是发给耳机、不经扬声器），于是 - // UIBackgroundModes=audio 没有成立的理由，切后台几十秒后上行就被系统掐掉： - // WebSocket 还连着（所以译文照出），但服务端收不到可识别的语音，只能靠 - // 默认 VAD 每 30 秒强切一句，吐出空译文或幻觉 —— 真机表现就是 - // 「切后台一会儿，只剩译文、原文全没了」（2026-09-21 实测，23:02:33 起 - // 连流式的「识别中」都消失，之后每 30.0/29.5/30.4/31.0 秒一条空翻译）。 - // ⚠️ acquire 内部只在当前类别撑不住后台音频时才改类别，不会踩掉 .playAndRecord。 - unawaited(BackgroundKeepalive.acquire('callTranslation')); - int finals = 0; - _besCallAstTtsSub = _astService.astTtsFrames.listen( - (frame) { - // 桥模式下原生只回 final 标记（PCM 已直接下发耳机），这里只记账 - if (frame.isFinal) { - finals++; - Logger.info('[CALL-BRIDGE] TTS 段尾 leg=${frame.leg} (#$finals, 原生直发)'); - } - }, - onError: (e) => Logger.error('[CALL-BRIDGE] ast tts stream error: $e'), - ); - Logger.w('Translation', '[CALL-BRIDGE] 已连通 设备双路 ↔ AST（原生直连，不经 Dart）'); - return; - } - - // ---- 兜底：Dart 流路径（原样保留）---- - Logger.w('Translation', '[CALL-BRIDGE] 原生直连桥不可用，退回 Dart 流路径'); + // 通话翻译不再把双路 PCM 送进微软语音 SDK。本端麦交给讯飞 SDK， + // 译文从合成口写回这一路耳机。对端那一路要第二条会话，这一版先不送。 + Logger.w('Translation', '[CALL-BRIDGE] 本端麦送讯飞 SDK，不连微软'); _callLocalSrc = await session.openMic(route: AudioRoute.callLocal); _callPeerSrc = await session.openMic(route: AudioRoute.callPeer); _besCallMicSub = _callLocalSrc!.pcm.listen( - (pcm) => _astService.pushExternalAudio(leg: 'A', pcm: pcm), + (pcm) => _holdOrWriteSimul(pcm), onError: (e) => Logger.error('[CALL-BRIDGE] local stream error: $e'), ); - // ⚠️ B 路**不能拿到帧就推**：阿里的声音复刻是 frequency=once，只在会话开头刻一次 - // 音色，而那时对端往往还没开口，先进来的是机主声音的串音（2026-09-19 真机实锤： - // B 路配的是 language:en，却识别出机主刚说的中文）。刻错的结果是用户听到的对端 - // 译文用的是自己的声音——两个人音色一模一样。门限保证第一段来自对端本人， - // 未开门期间的音频进环形缓冲、开门时整段补发，所以对端先开口也不会被吃掉开头。 - _peerGate.reset(); - _besCallSpkSub = _callPeerSrc!.pcm.listen( - (pcm) { - final wasOpen = _peerGate.isOpen; - for (final f in _peerGate.accept(pcm)) { - _astService.pushExternalAudio(leg: 'B', pcm: f); - } - if (!wasOpen && _peerGate.isOpen) { - // 真机调 rmsThreshold 的唯一依据，必须留在正式包（release 级别是 warning） - Logger.w('Translation', - '[CALL-BRIDGE] B 路门限开启：${_peerGate.openedByFallback ? "兜底强开(阈值可能偏高/全程只有串音)" : "检测到对端语音"} rms=${_peerGate.openRms}'); - } - }, - onError: (e) => Logger.error('[CALL-BRIDGE] peer stream error: $e'), - ); - // 诊断：计数 + isFinal 必打，用于定位 "TTS 到底有没有被转发到 native"。 - int forwardCnt = 0; - _besCallAstTtsSub = _astService.astTtsFrames.listen( - (frame) { - forwardCnt++; - if (frame.isFinal || forwardCnt == 1 || forwardCnt % 50 == 0) { - Logger.info( - '[CALL-BRIDGE] forward TTS leg=${frame.leg} pcm=${frame.pcm.length}B isFinal=${frame.isFinal} (#$forwardCnt)', - ); - } - final sink = frame.leg == 'A' ? _callSinkA : _callSinkB; - sink?.write(frame.pcm); - }, - onError: (e) => Logger.error('[CALL-BRIDGE] ast tts stream error: $e'), - ); - Logger.info('[CALL-BRIDGE] 已连通 设备双路 ↔ AST'); - } - - /// 试着把两端的原生直连桥都打开；任一端失败就把已开的那端关回去，返回 false。 - /// - /// 顺序：先开翻译插件那端（它要先挂上收帧的槽/观察者），再开设备那端 - /// （它一开就开始往外推帧）。反过来会有帧无人接收。 - Future _tryEnableNativeCallBridge(DeviceSession session) async { - bool astOn = false; - try { - astOn = await _astService.setNativeCallBridge(true); - if (!astOn) return false; - final r = await session - .invokeFeature(DeviceFeatures.callPcmNativeBridge, {'enabled': true}); - if (r['enabled'] == true) return true; - } on DeviceException catch (e) { - Logger.w('Translation', '[CALL-BRIDGE] 设备端不支持原生直连桥: ${e.code}'); - } catch (e) { - Logger.w('Translation', '[CALL-BRIDGE] 开启原生直连桥失败: $e'); - } - if (astOn) { - try { - await _astService.setNativeCallBridge(false); - } catch (_) {} - } - return false; + Logger.info('[CALL-BRIDGE] 已连通 本端麦 → 讯飞'); } /// 收到终态结果：链路健康，重连预算与时间窗都归零。 @@ -2252,6 +2235,101 @@ class TranslationController extends GetxController with WidgetsBindingObserver { return null; } + /// 译文是 16k PCM，从端到端合成口出来。只播己方这一路。 + Future _attachE2ePlayback() async { + await _e2eTtsSub?.cancel(); + final player = _e2ePlayer ??= PcmStreamPlayer( + sampleRate: 16000, + playbackOnly: !_e2ePinSpeaker && _simHeadsetSession != null, + pinSpeaker: _e2ePinSpeaker, + ); + var logged = false; + _e2eTtsSub = _astService.astTtsFrames.listen((frame) { + if (frame.leg != 'A' || frame.pcm.isEmpty) return; + if (!logged) { + logged = true; + Logger.w('Translation', '端到端译文首包 ${frame.pcm.length} 字节'); + } + _noteE2ePlayback(frame.pcm.length); + player.feed(frame.pcm); + }); + } + + /// 16k 单声道 16bit。播报时长按字节数累加，末尾留一段让扬声器余音过去。 + void _noteE2ePlayback(int byteCount) { + if (byteCount <= 0) return; + const tail = Duration(milliseconds: 400); + final chunk = Duration(microseconds: byteCount * 1000000 ~/ 32000); + final now = DateTime.now(); + final until = _e2ePlaybackUntil; + if (until == null || !until.isAfter(now)) { + _e2ePlaybackUntil = now.add(chunk).add(tail); + Logger.w('Translation', '端到端译文开播 $byteCount 字节'); + } else { + _e2ePlaybackUntil = until.add(chunk); + } + } + + /// 播报还没结束时送等长静音。麦本身不停，识别会话也不会因为断流被关掉。 + Uint8List _pcmForRecognizer(Uint8List pcm) { + final until = _e2ePlaybackUntil; + if (until != null && DateTime.now().isBefore(until)) { + if (!_e2eMicMuted) { + _e2eMicMuted = true; + Logger.w('Translation', '译文播报中，麦克风改送静音，避免录进回声'); + } + if (_e2eSilence.length < pcm.length) { + _e2eSilence = Uint8List(pcm.length); + } + return Uint8List.sublistView(_e2eSilence, 0, pcm.length); + } + if (_e2eMicMuted) { + _e2eMicMuted = false; + Logger.w('Translation', '译文播完，麦克风恢复收音'); + } + return pcm; + } + + Future _stopE2eInput() async { + final feed = _e2eMicFeed; + await _e2eMicSub?.cancel(); + _e2eMicSub = null; + final mic = _e2eMic; + _e2eMic = null; + if (mic != null) { + try { + await mic.stop(); + } catch (e) { + Logger.w('Translation', '关闭端到端手机麦失败: $e'); + } + } + await _e2eTtsSub?.cancel(); + _e2eTtsSub = null; + final player = _e2ePlayer; + _e2ePlayer = null; + if (player != null) { + try { + await player.stopNow(); + } catch (e) { + Logger.w('Translation', '停止端到端播报失败: $e'); + } + } + if (_e2ePinSpeaker) { + await EmaiAudioSession.releaseBuiltInSpeaker(); + } + _e2ePinSpeaker = false; + _e2ePlaybackUntil = null; + _e2eMicMuted = false; + if (feed && currentMode.value != 'call') { + try { + await _astService.stopContinuousTranslation(); + } catch (e) { + Logger.w('Translation', '停止端到端识别失败: $e'); + } + } + _e2eMicFeed = false; + } + Future _attachSimultaneousHeadsetMic() async { final session = _simHeadsetSession; if (session == null) return; @@ -2263,8 +2341,12 @@ class TranslationController extends GetxController with WidgetsBindingObserver { logged = true; Logger.w('Translation', '同声翻译耳机麦首帧 ${pcm.length} 字节'); } - if (_sparkListen) { - SparkChainSession.write(pcm); + if (_simulPcm != null) { + _simulPcm!(pcm); + } else if (_sparkListen) { + SparkChainSession.write(_pcmForRecognizer(pcm)); + } else if (_e2eMicFeed) { + unawaited(_astService.pushExternalAudio(leg: 'A', pcm: _pcmForRecognizer(pcm))); } else { unawaited(_asrService.pushAudioData(pcm)); } @@ -2291,38 +2373,260 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } } - /// 同声翻译改走讯飞听写。密钥没有配好、或手机麦采集失败时返回 false， - /// 调用方继续用原来的识别，不影响没配讯飞的包。 - Future _startSparkListen() async { + void Function(Uint8List pcm)? _simulPcm; + final List _simulPending = []; + bool _simulSent = false; + bool _simulHeard = false; + bool _simulStatus0Logged = false; + int _simulPeak = 0; + Timer? _simulPeakTimer; + + /// 听写还没就绪时先攒着。write 在 listening 为 false 时直接丢掉， + /// 开头那一包若没送进去，云端就会空等然后报 18802。 + void _holdOrWriteSimul(Uint8List pcm) { + if (!SparkChainSession.listening) { + _simulPending.add(Uint8List.fromList(pcm)); + if (_simulPending.length > 25) _simulPending.removeAt(0); + return; + } + _flushSimulPending(); + _writeSimul(pcm); + } + + int _pcmPeak(Uint8List pcm) { + var peak = 0; + for (var i = 0; i + 1 < pcm.length; i += 2) { + final sample = pcm[i] | (pcm[i + 1] << 8); + final value = sample > 32767 ? sample - 65536 : sample; + final amp = value < 0 ? -value : value; + if (amp > peak) peak = amp; + } + return peak; + } + + void _writeSimul(Uint8List pcm) { + final peak = _pcmPeak(pcm); + if (peak > _simulPeak) _simulPeak = peak; + SparkChainSession.write(_pcmForRecognizer(pcm)); + if (_simulSent) return; + _simulSent = true; + Logger.w('Translation', '同声翻译音频开始送入识别 ${pcm.length} 字节 peak=$peak'); + _simulPeakTimer?.cancel(); + _simulPeakTimer = Timer(const Duration(seconds: 2), () { + Logger.w('Translation', '同声翻译收音 2 秒峰值 $_simulPeak'); + }); + } + + void _flushSimulPending() { + if (!SparkChainSession.listening || _simulPending.isEmpty) return; + final batch = List.of(_simulPending); + _simulPending.clear(); + var bytes = 0; + for (final pcm in batch) { + bytes += pcm.length; + SparkChainSession.write(_pcmForRecognizer(pcm)); + } + if (_simulSent) return; + _simulSent = true; + Logger.w('Translation', '同声翻译音频开始送入识别 $bytes 字节'); + } + + /// 先把麦接上并等到第一包，再开云端会话。空会话会在十几秒后被讯飞判成 18802。 + Future _openSimulCapture() async { + final first = Completer(); + _simulSent = false; + _simulHeard = false; + _simulStatus0Logged = false; + _simulPending.clear(); + _simulPcm = (pcm) { + if (pcm.isEmpty) return; + if (!first.isCompleted) { + Logger.w('Translation', '同声翻译收音首帧 ${pcm.length} 字节'); + first.complete(); + } + _holdOrWriteSimul(pcm); + }; + if (_simHeadsetSession != null) { + await _attachSimultaneousHeadsetMic(); + } else { + final mic = MicPcmStreamer(); + // 会话还活着时不要重启麦。重启会改音频会话，把已经连上的转写拆成 18701。 + mic.holdRestart = true; + final started = await mic.start(plainMic: true); + if (!started) { + Logger.w('Translation', '同声翻译手机麦没打开'); + return false; + } + _sparkMic = mic; + _sparkMicSub = mic.packets.listen(_simulPcm!); + } + try { + await first.future.timeout(const Duration(seconds: 2)); + return true; + } on TimeoutException { + Logger.w('Translation', '同声翻译 2 秒没有音频，重新接上麦克风'); + } + await _sparkMic?.kick(); + try { + await first.future.timeout(const Duration(seconds: 2)); + return true; + } on TimeoutException { + Logger.w('Translation', '同声翻译麦克风没有音频，不启动识别'); + return false; + } + } + + /// 通话、面对面、音视频和同声共用这一条：讯飞 SDK 里的微软识别、翻译、合成。 + Future _startSparkMicrosoft({bool openMic = true}) async { final directed = _directedShortCodes(); - _sparkListen = true; - SparkChainSession.asrListener = _onSparkAsr; - final ok = await SparkChainSession.startListen(directed.source); - if (!ok) { - _sparkListen = false; - SparkChainSession.asrListener = null; - Logger.w('Translation', '同声翻译识别：SparkChain 未就绪，沿用原识别'); + final route = SimulMicrosoft.plan(directed.source, directed.target); + if (route == null) { + Logger.w( + 'Translation', + '${currentMode.value} ${directed.source}->${directed.target} 不在讯飞微软语种表', + ); + Get.snackbar('tip'.tr, '这一对语言没有微软同声'); return false; } - Logger.w('Translation', '同声翻译识别：SparkChain lang=${directed.source}'); - if (_simHeadsetSession != null) return true; - final mic = MicPcmStreamer(); - final started = await mic.start(); + final started = await _startSimul(route, openMic: openMic); if (!started) { - Logger.w('Translation', '同声翻译手机麦采集失败，改回原识别'); - await _stopSparkInput(); - return false; + Get.snackbar( + 'tip'.tr, + '翻译没启动(${SparkChainSession.lastStartCode})', + ); } - _sparkMic = mic; - _sparkMicSub = mic.packets.listen(SparkChainSession.write); + return started; + } + + /// 同声只走讯飞微软。表内一条 AST，表外三语走转写加合成。不开星火听写，也不退回原来的识别。 + /// + /// 苹果 demo 先把播放录音会话配好，再 start，最后才开录音。 + /// 会话还是别的类别时，start 会空等大约 12 秒后返回 18801，按钮看起来像没点到。 + /// 麦先开着再连，连上后队列一停再重启，又会把连接拆成 18701。 + /// [openMic] 为 false 时不自己开麦，通话翻译由耳机的本端麦把 PCM 送进来。 + Future _startSimul(SimulRoute route, {bool openMic = true}) async { + _sparkAst = route.ast; + _sparkVoice = route.speaker; + _sparkListen = true; + if (openMic) { + // 同声不要进 voiceChat。那一档会把 AudioQueue 采到的声音压到噪声底， + // 第一包峰值只有几十，云端 10 秒没有识别结果就关连接，报 18802。 + await EmaiAudioSession.ensureSimulMic(); + } + var active = route; + if (active.ast) { + SparkChainSession.astListener = _onSparkAst; + Logger.w( + 'Translation', + '同声翻译走讯飞 AST RegionMSFT ${active.lang} -> ${active.targetLang} ' + 'tts_speaker=${active.speaker}', + ); + final ok = await SparkChainSession.startAst( + lang: active.lang, + targetLang: active.targetLang, + speaker: active.speaker, + ); + if (!ok) { + await _stopSparkInput(); + return false; + } + _sparkPlayer = PcmStreamPlayer( + sampleRate: 16000, + playbackOnly: _simHeadsetSession != null, + ); + await _sparkPlayer!.begin(); + } + if (!active.ast) { + SparkChainSession.asrListener = _onSparkAsr; + Logger.w( + 'Translation', + '${currentMode.value} 走讯飞 RTASR RegionMSFT ' + 'language_name=${active.languageName} voice=${active.speaker}', + ); + final ok = await SparkChainSession.startSpliceListen(active.languageName); + if (!ok) { + await _stopSparkInput(); + return false; + } + } + if (openMic) { + if (!await _openSimulCapture()) { + await _stopSparkInput(); + return false; + } + } + _flushSimulPending(); return true; } + void _onSparkAst(Map event) { + if (!_sparkAst) return; + if (event['kind'] == 'error') { + Logger.w('Translation', '同声 AST 失败 code=${event['code']} sid=${event['sid']}'); + return; + } + final kind = event['resType'] as String? ?? ''; + if (kind == 'asr' || kind == 'trans') { + _showAstText( + source: (event['text'] as String?) ?? '', + translation: (event['dst'] as String?) ?? '', + isFinal: event['final'] == true && kind == 'trans', + ); + return; + } + if (kind != 'tts') return; + final raw = event['pcm']; + final pcm = raw is Uint8List ? raw : Uint8List(0); + if (pcm.isEmpty) return; + _noteE2ePlayback(pcm.length); + _sparkPlayer?.feed(pcm); + } + + /// AST 的原文和译文直接上屏。不再发翻译请求，也不在这里合成。 + void _showAstText({ + required String source, + required String translation, + required bool isFinal, + }) { + if (source.isEmpty && translation.isEmpty) return; + currentSessionId ??= DateTime.now().millisecondsSinceEpoch.toString(); + final last = translationHistory.isNotEmpty ? translationHistory.last : null; + final sameSentence = last != null && + source.isNotEmpty && + last.sourceText == source; + final open = last != null && (last.isIntermediate || sameSentence); + if (!open) { + final languageCodes = _determineLanguageCodes(); + _historyManager.addTranslationItem(TranslationItem( + sourceText: source, + translatedText: translation, + sourceLanguageCode: languageCodes['source']!, + targetLanguageCode: languageCodes['target']!, + timestamp: DateTime.now(), + sessionId: currentSessionId!, + isFirstInSession: translationHistory.isEmpty || + translationHistory.last.sessionId != currentSessionId, + isIntermediate: !isFinal, + )); + } else { + final item = translationHistory.last; + if (source.isNotEmpty) item.sourceText = source; + if (translation.isNotEmpty) item.translatedText = translation; + if (isFinal) item.isIntermediate = false; + translationHistory.refresh(); + } + if (isFinal) saveTranslationHistory(); + } + void _onSparkAsr(Map event) { if (!_sparkListen) return; if (event['kind'] == 'error') { - Logger.w('Translation', - 'SparkChain 听写失败 code=${event['code']} sid=${event['sid']}'); + Logger.w( + 'Translation', + '同声转写失败 code=${event['code']} ${event['message']} sid=${event['sid']}', + ); + // 上一条会话的超时会晚到。这时新的一轮还没进入识别，拆掉它会把刚启动的会话一起停掉。 + if (!isRecognizing.value) return; _handleRecognitionEvent(RecognitionEvent( type: RecognitionEventType.error, error: '${event['code']}', @@ -2331,6 +2635,24 @@ class TranslationController extends GetxController with WidgetsBindingObserver { return; } final text = (event['text'] as String?) ?? ''; + // 苹果 demo 把 status 0 当成翻译结果丢掉。空文本确实不是识别， + // 有字就照样上屏，否则服务端把整句放在 status 0 时界面一直是空的。 + if (event['status'] == 0) { + if (!_simulStatus0Logged) { + _simulStatus0Logged = true; + Logger.w( + 'Translation', + '同声转写回调 status=0 sid=${event['sid']} text=$text', + ); + } + if (text.isEmpty) return; + } + final isFinal = event['final'] == true; + if (text.isNotEmpty && (isFinal || !_simulHeard)) { + _simulHeard = true; + final shown = text.length > 40 ? text.substring(0, 40) : text; + Logger.w('Translation', '同声转写出字 ${isFinal ? '整句' : '中间'} $shown'); + } _handleRecognitionEvent(RecognitionEvent( type: event['final'] == true ? RecognitionEventType.finalResult @@ -2355,7 +2677,18 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } } _sparkListen = false; + _sparkAst = false; + _sparkVoice = ''; + _simulPcm = null; + _simulPending.clear(); + _simulSent = false; + _simulHeard = false; + _simulStatus0Logged = false; + _simulPeakTimer?.cancel(); + _simulPeakTimer = null; + _simulPeak = 0; SparkChainSession.asrListener = null; + SparkChainSession.astListener = null; final player = _sparkPlayer; _sparkPlayer = null; try { @@ -2363,25 +2696,40 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } catch (e) { Logger.w('Translation', '停止讯飞播报失败: $e'); } + await SparkChainSession.stopSpeak(); await SparkChainSession.stopListen(); } - /// 译文 PCM 交给现有播放器。连着耳机时只放音，避免把声音钉在手机扬声器。 - Future _playSparkTts(String lang, String text) async { - final player = _sparkPlayer ??= PcmStreamPlayer( - sampleRate: 16000, - playbackOnly: _simHeadsetSession != null, - ); - await player.begin(); - try { - await SparkChainSession.speak( - lang: lang, - text: text, - onPcm: (pcm, _) => player.feed(pcm), + /// 拼接路径的整句合成。音色是微软神经音色，按句排队，不打断上一句。 + Future _playSparkTts(String text) async { + final voice = _sparkVoice; + if (voice.isEmpty || text.isEmpty) return; + final run = () async { + final player = _sparkPlayer ??= PcmStreamPlayer( + sampleRate: 16000, + playbackOnly: _simHeadsetSession != null, ); - } finally { - await player.end(); - } + await player.begin(); + try { + await SparkChainSession.speakNeural( + voice: voice, + text: text, + onPcm: (pcm, _) { + _noteE2ePlayback(pcm.length); + if (currentMode.value == 'call') { + _callSinkA?.write(pcm); + } else { + player.feed(pcm); + } + }, + ); + } finally { + await player.end(); + } + }; + final done = _sparkSpeakChain.then((_) => run()); + _sparkSpeakChain = done.catchError((_) {}); + await done; } /// 启动语音识别 @@ -2533,6 +2881,12 @@ class TranslationController extends GetxController with WidgetsBindingObserver { /// 原文被后一句覆盖、译文却是前一句的，字幕越说越乱（2026-09-19 修）； /// - `serviceId` 是 A/B 两条腿（己方 / 对方）。两腿 id 本就不同，带上它只是 /// 多一道保险：万一哪天 id 撞了，宁可多起一条，也不能把两个人的话并成一句。 + bool _sameLanguage(String a, String b) { + final sa = a.split('-').first.toLowerCase(); + final sb = b.split('-').first.toLowerCase(); + return sa.isNotEmpty && sa == sb; + } + TranslationItem? _findOpenAstItem(ASTEvent event) { if (event.utteranceId.isEmpty) return null; for (int i = translationHistory.length - 1; i >= 0; i--) { @@ -2565,13 +2919,25 @@ class TranslationController extends GetxController with WidgetsBindingObserver { Logger.info( '[STS] _handleAstEvent 收到: type=${event.type}, serviceId=${event.serviceId}, text=${event.text}'); } - // 根据 serviceId 确定语言方向 - final String srcCode = event.serviceId == "A" - ? sourceLanguageCode.value - : targetLanguageCode.value; - final String tgtCode = event.serviceId == "A" + // 通话两路靠 serviceId 分方向。面对面只有一路，听出目标语就把方向反过来。 + if (currentMode.value == 'faceToFace' && + (event.type == ASTEventType.intermediateResult || + event.type == ASTEventType.finalResult) && + _sameLanguage(event.detectedLanguage, targetLanguageCode.value)) { + _astReversedUtterances.add(event.utteranceId); + } + final bool reversed = currentMode.value == 'faceToFace' && + _astReversedUtterances.contains(event.utteranceId); + final String srcCode = reversed ? targetLanguageCode.value - : sourceLanguageCode.value; + : (event.serviceId == "A" + ? sourceLanguageCode.value + : targetLanguageCode.value); + final String tgtCode = reversed + ? sourceLanguageCode.value + : (event.serviceId == "A" + ? targetLanguageCode.value + : sourceLanguageCode.value); switch (event.type) { case ASTEventType.intermediateResult: @@ -2750,6 +3116,21 @@ class TranslationController extends GetxController with WidgetsBindingObserver { unawaited(_handleAstDrop('AST 取消: ${event.error}')); break; + case ASTEventType.sessionStopped: + if (currentMode.value == 'simultaneous' && + isRecognizing.value && + _e2eMicFeed) { + final now = DateTime.now(); + final last = _e2eRestartAt; + if (last != null && now.difference(last) < const Duration(seconds: 2)) { + break; + } + _e2eRestartAt = now; + Logger.w('Translation', '同声识别会话中途停了，重新拉起'); + unawaited(_astService.startContinuousTranslation()); + } + break; + default: break; } @@ -2775,6 +3156,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { if (_f2fBleActive || _f2fSession != null) { await _stopBesFaceToFace(); } + _handoffSessionToMeeting(); try { await _stopSparkInput(); @@ -2786,7 +3168,6 @@ class TranslationController extends GetxController with WidgetsBindingObserver { _astReconnecting = false; _markAstLinkHealthy(); await _astService.stopContinuousTranslation(); - _astAutoStartPending = false; } _stopAsrActiveTracking(); final langName = _languageManager.getChineseNameByAsrCode( @@ -2799,12 +3180,10 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } catch (e) { Logger.error('停止语音识别失败: ${e.toString()}'); } finally { + await _stopE2eInput(); await _stopSimultaneousHeadsetMic(); if (currentMode.value == 'simultaneous') { await AudioForegroundService.stop(); - if (Platform.isIOS) { - await BackgroundKeepalive.release('simultaneous'); - } } } } @@ -2852,6 +3231,9 @@ class TranslationController extends GetxController with WidgetsBindingObserver { _historyManager.scrollToBottom(); } + // 同声拼接只在整句上翻译和合成。中间结果留在屏幕上，不再发翻译请求。 + if (currentMode.value == 'simultaneous' && _sparkListen) return; + // 中间结果的翻译只是屏幕上的预览：它不播 TTS，也不写历史、不记统计。 // 原来是「每 300ms 无条件发一次」，一句 5 秒的话就是十几次真实的服务商调用； // 这些在途请求还会跟说完之后那一次 final 翻译抢同一条链路和同一份配额， @@ -3064,37 +3446,19 @@ class TranslationController extends GetxController with WidgetsBindingObserver { Future _performTranslation(String sourceText, String sourceCode, String targetCode, CancelToken? cancelToken) async { final directed = _directedShortCodes(); - if (_sparkListen) { - final onDevice = await SparkChainSession.translate( - from: directed.source, - to: directed.target, - text: sourceText, - ); - if (onDevice != null) return onDevice; - if (!aiOnlyIflytek) { - Logger.w('Translation', '同声 SparkChain 翻译没有结果，改走原路径'); - } - } - if (!aiOnlyIflytek && - LanguageManager.microsoftTextPipeline( - currentMode.value, directed.source, directed.target)) { - final from = _languageManager.microsoftTranslateCode(directed.source); - final to = _languageManager.microsoftTranslateCode(directed.target); - Logger.info('[MT] 扩展语种走微软翻译: $from → $to'); - return _microsoftTranslation.translate( - text: sourceText, - from: from, - to: to, - cancelToken: cancelToken, - ); - } - return await _translationService.translateText( + if (_sparkAst) return null; + final from = SimulMicrosoft.translateCode(directed.source); + final to = SimulMicrosoft.translateCode(directed.target); + Logger.w('Translation', '文本翻译走讯飞 SDK gtrans $from -> $to'); + final onDevice = await SparkChainSession.translate( + from: from, + to: to, text: sourceText, - sourceLanguageCode: sourceCode, - targetLanguageCode: targetCode, - cancelToken: cancelToken, - tryOnDevice: !_sparkListen, + prepared: true, ); + if (onDevice != null) return onDevice; + Logger.w('Translation', '讯飞 SDK 没返回译文'); + return null; } /// 这一句实际的翻译方向，用语言短码而不是识别码。 @@ -3267,12 +3631,11 @@ class TranslationController extends GetxController with WidgetsBindingObserver { Logger.info("voiceName=$voiceName,text=$text"); final startTime = DateTime.now(); - if (_sparkListen) { - await _playSparkTts(directed.target, text); - } else { - await _ttsService.setVoice(voiceName); - await _ttsService.speakOnce(currsessionid, text); + if (_sparkAst) { + return; } + if (_sparkVoice.isEmpty) _sparkVoice = voiceName; + await _playSparkTts(text); final endTime = DateTime.now(); // 记录TTS API调用统计 @@ -3309,11 +3672,16 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } if (isRecognizing.value) { await stopRecognition(); - } else if (isCreateRecord.value) { - // 面对面翻译每句松手就把识别停了（isRecognizing=false），录音只是暂停、 - // 文件还开着——原来 stopAll 一看识别没在跑就什么都不做，文件永远收不了尾， - // 自然也从来没进过纪要。 - await stopRecording(); + } else { + // 面对面每句松手就把识别停了，会话还在。结束时照样把已翻译的句子交出去。 + _handoffSessionToMeeting(); + if (_asrSessionStartTime != null) { + _recordAsrUsageStats(sourceLanguage.value); + } + currentSessionId = null; + if (isCreateRecord.value || isRecording.value) { + await stopRecording(); + } } _stopAsrActiveTracking(); @@ -3423,14 +3791,10 @@ class TranslationController extends GetxController with WidgetsBindingObserver { try { await _astService.stopContinuousTranslation(); } catch (_) {} - _astAutoStartPending = false; isRecognizing.value = false; isPreparing.value = false; - // 录音跟着识别一起收：原来这条路径不管录音，AST 挂了以后识别停了、 - // 原生录音还在往文件里写，右上角一直显示在录，直到用户退页面才被 onClose - // 的 stopAll 收掉——而且那时 isRecognizing 已经是 false，老的 stopAll 根本 - // 不会去停录音。已录到的部分照常归档（转写有多少算多少）。 - if (isCreateRecord.value) { + _handoffSessionToMeeting(); + if (isCreateRecord.value || isRecording.value) { try { await stopRecording(); } catch (e) { @@ -3552,8 +3916,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver { _lastPlayTime = currentTime; final voiceName = _voiceForStoredAsr(item.targetLanguageCode); - await _ttsService.setVoice(voiceName); - await _ttsService.speakOnce(currsessionid, item.translatedText); + if (_sparkVoice.isEmpty) _sparkVoice = voiceName; + await _playSparkTts(item.translatedText); } catch (e) { Logger.error('播放翻译项失败: ${e.toString()}'); } @@ -3614,8 +3978,8 @@ class TranslationController extends GetxController with WidgetsBindingObserver { .getTtsVoiceNameByAsrCode(targetLanguageCode.value) ?? 'en-US-AriaNeural'; - await _ttsService.setVoice(voiceName); - await _ttsService.speakOnce(currsessionid, text); + if (_sparkVoice.isEmpty) _sparkVoice = voiceName; + await _playSparkTts(text); } catch (e) { Logger.error('播放文本失败: ${e.toString()}'); } @@ -3822,9 +4186,28 @@ class TranslationController extends GetxController with WidgetsBindingObserver { Future _reinitializeAsrService({bool restartRecognition = false}) async { _isReinitializing = true; try { - // 通话模式：无论 isRecognizing 状态如何，都强制停止 AST 服务 + // 通话、同声、面对面、音视频换语言不连微软。下次点开始会按新语种走讯飞 SDK。 + final sparkMode = currentMode.value == 'call' || + currentMode.value == 'simultaneous' || + currentMode.value == 'faceToFace' || + currentMode.value == 'audioVideo'; + if (sparkMode) { + if (isRecognizing.value) { + await stopRecognition(); + } + isRecognizing.value = false; + if (restartRecognition) { + await startRecognition(); + } + return; + } + // 通话、同声、面对面走的是同一条端到端会话。换语言只重建通话那条的话， + // 同声还拿着上一对（比如日→英）开识别，界面已经是英→日，说英语听回去还是英语。 + final e2eMode = currentMode.value == 'call' || + (currentMode.value == 'faceToFace' && _microsoftE2eAvailable()); + // 无论 isRecognizing 状态如何，都强制停止 AST 服务 // 防止旧会话未完全关闭就创建新会话（Socket 1011 / Session 1012） - if (currentMode.value == "call") { + if (e2eMode) { _astEventSubscription?.cancel(); _astEventSubscription = null; try { @@ -3832,7 +4215,6 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } catch (e) { Logger.info('[STS] 停止旧AST服务(忽略): $e'); } - _astAutoStartPending = false; } //1.统一先停止旧识别 @@ -3843,12 +4225,11 @@ class TranslationController extends GetxController with WidgetsBindingObserver { final List asrSupportedLanguages = _asrLanguagesForCurrentMode(); - if (currentMode.value == "call") { + if (e2eMode) { await _initializeCallModeTranslationService(); // _astService.initialize 的原生实现会自动启动 AST（AzureAsrToAsr / DoubaoE2E // 等在 initialize 中都调用了 startContinuousTranslation）。若切换语言前未处于 // 识别状态，这里需要立即停止原生 AST，避免 UI 显示未启动但原生仍在识别。 - _astAutoStartPending = true; if (!restartRecognition) { try { await _astService.stopContinuousTranslation(); @@ -3856,7 +4237,6 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } catch (e) { Logger.info('[STS] 停止新AST(忽略): $e'); } - _astAutoStartPending = false; } // restartRecognition=true 时下面会调 startRecognition()，它会消费 // _astAutoStartPending，跳过重复的 startContinuousTranslation 调用。 @@ -3948,21 +4328,23 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } /// 记录ASR使用统计 - 基于精确时长跟踪 - /// [language] 语言名称 - void _recordAsrUsageStats(String language) { + /// [endSession] 为 false 时只记这一段，会话起点留着给下一句和纪要交接。 + void _recordAsrUsageStats(String language, {bool endSession = true}) { try { if (_asrSessionStartTime == null) return; _stopAsrActiveTracking(); final now = DateTime.now(); - final totalDurationSeconds = - now.difference(_asrSessionStartTime!).inSeconds; - final effectiveDurationSeconds = _asrActiveDuration.inSeconds; + final from = _asrStatsCursor ?? _asrSessionStartTime!; + final sliceSeconds = now.difference(from).inSeconds; + final effectiveDurationSeconds = endSession + ? _asrActiveDuration.inSeconds + : sliceSeconds; _usageService.recordAsrApiCall( effectiveDurationSeconds: effectiveDurationSeconds, - totalDurationSeconds: totalDurationSeconds, + totalDurationSeconds: sliceSeconds, language: language, mode: currentMode.value, audioFormat: 'pcm', @@ -3970,8 +4352,11 @@ class TranslationController extends GetxController with WidgetsBindingObserver { ); Logger.info( - 'ASR会话统计: 总时长${totalDurationSeconds}秒, 实际活跃时长${effectiveDurationSeconds}秒, 语言$language'); + 'ASR会话统计: 本段${sliceSeconds}秒, 语言$language, 结束=$endSession'); + _asrStatsCursor = now; + if (!endSession) return; _asrSessionStartTime = null; + _asrStatsCursor = null; _asrActiveDuration = Duration.zero; _isAsrActive = false; _asrActiveStartTime = null; diff --git a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrToAsr.kt b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrToAsr.kt index 9a837b13..d2d7c389 100644 --- a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrToAsr.kt +++ b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrToAsr.kt @@ -74,6 +74,8 @@ class IntegratedSpeechTranslationService( // 配置管理 private var serviceConfig = ServiceConfiguration() + private var turnXmlLang: String = "" + private var turnVoice: String = "" // 状态管理 private val serviceState = ServiceState() @@ -167,6 +169,7 @@ class IntegratedSpeechTranslationService( var currentVoice: String = "en-US-AriaNeural", var translationSourceLanguage: String = "", var translationTargetLanguage: String = "", + var reverseVoice: String = "", var speechRate: String = "0%", var speechPitch: String = "0%", var speechVolume: String = "100%", @@ -331,14 +334,17 @@ class IntegratedSpeechTranslationService( Log.d(TAG, "开始初始化Azure服务${config.subscriptionKey},${config.region}") speechConfig = SpeechConfig.fromSubscription(config.subscriptionKey, config.region).apply { - speechRecognitionLanguage = serviceConfig.sourceLanguage val voice = serviceConfig.currentVoice.ifEmpty { serviceConfig.targetLanguage } setSpeechSynthesisVoiceName(voice) Log.d(TAG, "设置语音合成器语言: ${serviceConfig.targetLanguage}") setSpeechSynthesisOutputFormat(SpeechSynthesisOutputFormat.Raw16Khz16BitMonoPcm) - // 自动语言检测 + // 说完到出句的静音等待。iOS 同一条链路也是 300ms。 + setProperty("Speech_SegmentationSilenceTimeoutMs", SEGMENTATION_SILENCE_TIMEOUT) + // 单向钉死识别语种。面对面才开自动检测，两种都能听。 if (serviceConfig.enableAutoLanguageDetection) { setProperty("SpeechServiceConnection_LanguageIdMode", "Continuous") + } else { + speechRecognitionLanguage = serviceConfig.sourceLanguage } } @@ -382,12 +388,22 @@ class IntegratedSpeechTranslationService( */ private fun setupSpeechRecognizer() { recognizer?.close() - recognizer = SpeechRecognizer(speechConfig, audioConfig).apply { + val created = if (serviceConfig.enableAutoLanguageDetection) { + val auto = com.microsoft.cognitiveservices.speech.AutoDetectSourceLanguageConfig.fromLanguages( + listOf(serviceConfig.sourceLanguage, serviceConfig.targetLanguage) + ) + Log.i(TAG, "识别器双向: ${serviceConfig.sourceLanguage} / ${serviceConfig.targetLanguage}") + SpeechRecognizer(speechConfig, auto, audioConfig) + } else { + SpeechRecognizer(speechConfig, audioConfig) + } + recognizer = created.apply { // 识别中事件 recognizing.addEventListener { _, event -> if (event.result.text.isNotEmpty()) { val confidence = extractConfidence(event.result) + val detected = detectedLanguage(event.result) // 开始新的话段时生成并记录 utteranceId if (!isInUtterance.get()) { currentUtteranceId = nextUtteranceId() @@ -397,7 +413,7 @@ class IntegratedSpeechTranslationService( eventCallback?.onRecognizing( uttId, event.result.text, - serviceConfig.sourceLanguage, + detected, confidence ) @@ -406,7 +422,7 @@ class IntegratedSpeechTranslationService( // 取消上一次“识别中翻译”任务，避免旧翻译结果覆盖新结果 recognizingTranslationJob?.cancel() recognizingTranslationJob = launch { - processTranslationOnly(uttId, event.result.text) + processTranslationOnly(uttId, event.result.text, detected) } } } @@ -418,21 +434,22 @@ class IntegratedSpeechTranslationService( ResultReason.RecognizedSpeech -> { if (event.result.text.isNotEmpty()) { val confidence = extractConfidence(event.result) + val detected = detectedLanguage(event.result) val uttId = currentUtteranceId ?: nextUtteranceId() eventCallback?.onRecognized( uttId, event.result.text, - serviceConfig.sourceLanguage, + detected, confidence ) - Log.i(TAG, "识别完成[${uttId}]: ${event.result.text}") + Log.i(TAG, "识别完成[${uttId}]: ${event.result.text} lang=$detected") // 只有在服务仍在运行时才触发翻译流程 if (serviceState.isRecognizing.get()) { isInUtterance.set(false) currentUtteranceId = null launch { cancelRecognizingTranslationJobSafely() - processTranslationAndSynthesis(uttId, event.result.text) + processTranslationAndSynthesis(uttId, event.result.text, detected) } } } @@ -700,7 +717,55 @@ class IntegratedSpeechTranslationService( * - utteranceId: 该句话的唯一ID，用于绑定识别/翻译/合成事件 * - text: 识别完成的最终文本 */ - private suspend fun processTranslationAndSynthesis(utteranceId: String, text: String) { + private fun sameLanguage(a: String, b: String): Boolean { + val sa = a.substringBefore('-').lowercase() + val sb = b.substringBefore('-').lowercase() + return sa.isNotEmpty() && sa == sb + } + + private fun detectedLanguage(result: com.microsoft.cognitiveservices.speech.SpeechRecognitionResult): String { + if (!serviceConfig.enableAutoLanguageDetection) return serviceConfig.sourceLanguage + val fromProp = try { + result.properties?.getProperty("speechServiceResponse_RecognitionLanguage") + } catch (_: Exception) { + null + } + if (!fromProp.isNullOrEmpty()) return fromProp + val lang = try { + com.microsoft.cognitiveservices.speech.AutoDetectSourceLanguageResult.fromResult(result).language + } catch (_: Exception) { + "" + } + return if (lang.isNullOrEmpty()) serviceConfig.sourceLanguage else lang + } + + private data class TurnDirection( + val source: String, + val target: String, + val voice: String, + val xmlLang: String + ) + + private fun direction(detected: String): TurnDirection { + val reverse = serviceConfig.enableAutoLanguageDetection && sameLanguage(detected, serviceConfig.targetLanguage) + return if (reverse) { + TurnDirection( + source = serviceConfig.translationTargetLanguage.ifEmpty { serviceConfig.targetLanguage }, + target = serviceConfig.translationSourceLanguage.ifEmpty { serviceConfig.sourceLanguage }, + voice = serviceConfig.reverseVoice.ifEmpty { serviceConfig.currentVoice }, + xmlLang = serviceConfig.sourceLanguage + ) + } else { + TurnDirection( + source = serviceConfig.translationSourceLanguage.ifEmpty { serviceConfig.sourceLanguage }, + target = serviceConfig.translationTargetLanguage.ifEmpty { serviceConfig.targetLanguage }, + voice = serviceConfig.currentVoice, + xmlLang = serviceConfig.targetLanguage + ) + } + } + + private suspend fun processTranslationAndSynthesis(utteranceId: String, text: String, detectedLanguage: String = "") { // 检查服务是否仍在运行 if (!serviceState.isRecognizing.get()) { Log.d(TAG, "服务已停止，取消翻译流程: $text") @@ -725,8 +790,12 @@ class IntegratedSpeechTranslationService( eventCallback?.onTranslationStarted(targetUtteranceId, targetText) eventCallback?.onStateChanged("Translation", true) try { - val srcLang = if (serviceConfig.translationSourceLanguage.isNotEmpty()) serviceConfig.translationSourceLanguage else serviceConfig.sourceLanguage - val tgtLang = if (serviceConfig.translationTargetLanguage.isNotEmpty()) serviceConfig.translationTargetLanguage else serviceConfig.targetLanguage + val turn = direction(detectedLanguage) + turnXmlLang = turn.xmlLang + turnVoice = turn.voice + val srcLang = turn.source + val tgtLang = turn.target + Log.d(TAG, "翻译语言对: $srcLang -> $tgtLang (heard=$detectedLanguage)") val translationResult = withTimeout(serviceConfig.translationTimeout) { translationService?.translateText( text = targetText, @@ -844,7 +913,7 @@ class IntegratedSpeechTranslationService( * - utteranceId: 句子唯一ID * - text: 待翻译的源文本 */ - private suspend fun processTranslationOnly(utteranceId: String, text: String) { + private suspend fun processTranslationOnly(utteranceId: String, text: String, detectedLanguage: String = "") { if (!serviceState.isRecognizing.get()) return if (serviceState.isTranslating.get()) return @@ -853,8 +922,9 @@ class IntegratedSpeechTranslationService( eventCallback?.onStateChanged("Translation", true) try { - val srcLang = if (serviceConfig.translationSourceLanguage.isNotEmpty()) serviceConfig.translationSourceLanguage else serviceConfig.sourceLanguage - val tgtLang = if (serviceConfig.translationTargetLanguage.isNotEmpty()) serviceConfig.translationTargetLanguage else serviceConfig.targetLanguage + val turn = direction(detectedLanguage) + val srcLang = turn.source + val tgtLang = turn.target val translationResult = withTimeout(serviceConfig.translationTimeout) { translationService?.translateText( text = text, @@ -902,9 +972,11 @@ class IntegratedSpeechTranslationService( val cleanText = cleanTextForSynthesis(text) val escapedText = escapeXmlText(cleanText) + val xmlLang = turnXmlLang.ifEmpty { serviceConfig.targetLanguage } + val voice = turnVoice.ifEmpty { serviceConfig.currentVoice } return """ - - + + $escapedText diff --git a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt index b23f19f2..97e9d510 100644 --- a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt +++ b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt @@ -1,7 +1,9 @@ package com.yunqiinnovation.azure_speech import android.content.Context +import android.media.AudioDeviceInfo import android.media.AudioManager +import android.os.Build import android.os.Handler import android.os.Looper import android.app.Activity @@ -247,7 +249,15 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, "start" -> { val rate = (call.argument("sampleRate"))?.toInt() ?: 24000 val keepAlive = call.argument("keepAlive") ?: false - result.success(pcmSpeakerPlayer.start(rate, keepAlive)) + val pin = call.argument("pinSpeaker") ?: false + val speaker = if (pin && Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + val am = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager + am.getDevices(AudioManager.GET_DEVICES_OUTPUTS) + .firstOrNull { it.type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER } + } else { + null + } + result.success(pcmSpeakerPlayer.start(rate, keepAlive, speaker)) } "feed" -> { call.argument("data")?.let { pcmSpeakerPlayer.feed(it) } @@ -1677,6 +1687,8 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, val translationLang1 = supportedLanguages.getOrElse(3) { "en-US" } val ttsLang0 = supportedLanguages.getOrElse(4) { "zh-CN-XiaoxiaoNeural" } val ttsLang1 = supportedLanguages.getOrElse(5) { "en-US-AriaNeural" } + val singleLeg = call.argument("singleLeg") ?: false + val bidirectional = call.argument("bidirectional") ?: false Log.d(tag, "initializeIntegrated: supportedLanguages=$supportedLanguages") currentAstProvider = provider // 原生直连桥：新会话重新刻音色 → 门限复位；helper 起来之前的帧先攒着（见 BesCallPcmBridge） @@ -1803,7 +1815,9 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, targetLanguage = lang1, currentVoice = ttsLang1, translationSourceLanguage = translationLang0, - translationTargetLanguage = translationLang1 + translationTargetLanguage = translationLang1, + enableAutoLanguageDetection = bidirectional, + reverseVoice = ttsLang0 ) val serviceConfigB = IntegratedSpeechTranslationService.ServiceConfiguration( @@ -1835,6 +1849,12 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, serviceConfig = serviceConfigA, callback = callbackA ) + if (singleLeg) { + Log.i(tag, "initializeIntegrated: Azure single leg bidirectional=$bidirectional, skip B") + markAstBridgeReady() + result.success(true) + return@launch + } delay(1200L) if (!isAstInitGenerationCurrent(initGeneration)) { result.success(false); return@launch diff --git a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/PcmSpeakerPlayer.kt b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/PcmSpeakerPlayer.kt index 335bd7e4..1ab2d829 100644 --- a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/PcmSpeakerPlayer.kt +++ b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/PcmSpeakerPlayer.kt @@ -1,6 +1,7 @@ package com.yunqiinnovation.azure_speech import android.media.AudioAttributes +import android.media.AudioDeviceInfo import android.media.AudioFormat import android.media.AudioManager import android.media.AudioTrack @@ -79,9 +80,17 @@ class PcmSpeakerPlayer(private val onCompleted: () -> Unit) { /** 本轮是否在空闲时垫静音（只有车载香薰的设备会话传 true，见 [keepAlive]） */ @Volatile private var keepAliveEnabled = false - fun start(sampleRate: Int, keepAlive: Boolean = false): Boolean { + /** 面对面测试：译文钉在内置扬声器，不跟到蓝牙耳机。 */ + @Volatile private var speakerDevice: AudioDeviceInfo? = null + + fun start( + sampleRate: Int, + keepAlive: Boolean = false, + speaker: AudioDeviceInfo? = null, + ): Boolean { synchronized(lock) { keepAliveEnabled = keepAlive + speakerDevice = speaker generation++ val gen = generation stopWriterLocked() @@ -297,6 +306,7 @@ class PcmSpeakerPlayer(private val onCompleted: () -> Unit) { t.flush() // 上一轮的残留不能带到这一轮 } catch (_: Exception) { } + pinSpeaker(t) return true } try { @@ -357,6 +367,14 @@ class PcmSpeakerPlayer(private val onCompleted: () -> Unit) { } track = newTrack trackRate = sampleRate + pinSpeaker(newTrack) return true } + + private fun pinSpeaker(track: AudioTrack) { + val device = speakerDevice ?: return + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) return + val ok = track.setPreferredDevice(device) + Log.i(TAG, "译文钉到手机扬声器=$ok") + } } diff --git a/apps/client/local_plugins/azure_speech/ios/azure_speech/AudioCatch/AudioCatch.m b/apps/client/local_plugins/azure_speech/ios/azure_speech/AudioCatch/AudioCatch.m new file mode 100644 index 00000000..457c436b --- /dev/null +++ b/apps/client/local_plugins/azure_speech/ios/azure_speech/AudioCatch/AudioCatch.m @@ -0,0 +1,50 @@ +#import "AudioCatch.h" + +NSString *AudioEngineStart(AVAudioEngine *engine) { + @try { + [engine prepare]; + NSError *error = nil; + if (![engine startAndReturnError:&error]) { + return error.localizedDescription ?: @"麦克风引擎启动失败"; + } + return nil; + } @catch (NSException *e) { + return e.reason ?: e.name; + } +} + +NSString *AudioEngineInstallTap(AVAudioEngine *engine, AVAudioFormat *format, void *ctx, PhoneMicTap tap) { + @try { + if (format.sampleRate <= 0 || format.channelCount == 0) { + return @"输入格式未就绪"; + } + AVAudioInputNode *input = engine.inputNode; + [input installTapOnBus:0 bufferSize:4096 format:format block:^(AVAudioPCMBuffer *buffer, AVAudioTime *when) { + if (tap == NULL || buffer.frameLength == 0) return; + float *const *channels = buffer.floatChannelData; + if (channels == NULL || channels[0] == NULL) return; + tap(ctx, channels[0], (int)buffer.frameLength, buffer.format.sampleRate); + }]; + return nil; + } @catch (NSException *e) { + return e.reason ?: e.name; + } +} + +NSString *AudioEngineConnect(AVAudioEngine *engine, AVAudioPlayerNode *node, AVAudioFormat *format) { + @try { + if (format.sampleRate <= 0 || format.channelCount == 0) { + return @"播放格式无效"; + } + if (![engine.attachedNodes containsObject:node]) { + [engine attachNode:node]; + } + if ([engine outputConnectionPointsForNode:node outputBus:0].count == 0) { + [engine connect:node to:engine.mainMixerNode format:format]; + } + engine.mainMixerNode.outputVolume = 1; + return nil; + } @catch (NSException *e) { + return e.reason ?: e.name; + } +} diff --git a/apps/client/local_plugins/azure_speech/ios/azure_speech/AudioCatch/include/AudioCatch.h b/apps/client/local_plugins/azure_speech/ios/azure_speech/AudioCatch/include/AudioCatch.h new file mode 100644 index 00000000..5f1df408 --- /dev/null +++ b/apps/client/local_plugins/azure_speech/ios/azure_speech/AudioCatch/include/AudioCatch.h @@ -0,0 +1,12 @@ +#import + +typedef void (*PhoneMicTap)(void *_Nonnull ctx, const float *_Nonnull samples, int frames, double sampleRate); + +NSString *_Nullable AudioEngineStart(AVAudioEngine *_Nonnull engine); +NSString *_Nullable AudioEngineInstallTap(AVAudioEngine *_Nonnull engine, + AVAudioFormat *_Nonnull format, + void *_Nonnull ctx, + PhoneMicTap _Nonnull tap); +NSString *_Nullable AudioEngineConnect(AVAudioEngine *_Nonnull engine, + AVAudioPlayerNode *_Nonnull node, + AVAudioFormat *_Nonnull format); diff --git a/apps/client/local_plugins/azure_speech/ios/azure_speech/Package.swift b/apps/client/local_plugins/azure_speech/ios/azure_speech/Package.swift index a389be1d..36a3b21f 100644 --- a/apps/client/local_plugins/azure_speech/ios/azure_speech/Package.swift +++ b/apps/client/local_plugins/azure_speech/ios/azure_speech/Package.swift @@ -18,9 +18,16 @@ let package = Package( .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0") ], targets: [ + .target( + name: "AudioCatch", + path: "AudioCatch", + publicHeadersPath: "include", + linkerSettings: [.linkedFramework("AVFoundation")] + ), .target( name: "azure_speech", dependencies: [ + "AudioCatch", .product(name: "speech", package: "speech"), .product(name: "ble-service", package: "ble_service"), .product(name: "classic-bluetooth", package: "classic_bluetooth"), diff --git a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift index 4c4a4308..4a971908 100644 --- a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift +++ b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift @@ -316,6 +316,8 @@ import ble_service /// EMAI 助手下行 PCM 的外放播放器（iOS 专用，见 PcmSpeakerPlayer 的说明） private let pcmSpeakerPlayer = PcmSpeakerPlayer() private var pcmPlayerChannel: FlutterMethodChannel? + private let phoneMic = PhoneMicCapture() + private var phoneMicEventChannel: FlutterEventChannel? internal var audioDataEventSink: FlutterEventSink? // 通话翻译音频统计（日志用） @@ -484,6 +486,11 @@ private func sendEvent(_ event: [String: Any], type: EventType) { instance.pcmSpeakerPlayer.onCompleted = { [weak instance] in instance?.pcmPlayerChannel?.invokeMethod("onCompleted", arguments: nil) } + PhoneMicCapture.shared = instance.phoneMic + let phoneMicEvents = FlutterEventChannel( + name: "azure_speech/phone_mic", binaryMessenger: registrar.messenger()) + phoneMicEvents.setStreamHandler(instance.phoneMic) + instance.phoneMicEventChannel = phoneMicEvents // 设置讯飞原生日志回调，通过 EventChannel 回传 Dart instance.azureAsrHelper.nativeLogCallback = { [weak instance] level, message in @@ -592,7 +599,8 @@ internal func handlePcmPlayerMethodCall(_ call: FlutterMethodCall, result: @esca case "start": let args = call.arguments as? [String: Any] let rate = (args?["sampleRate"] as? NSNumber)?.doubleValue ?? 24000 - result(pcmSpeakerPlayer.start(sampleRate: rate)) + let pin = args?["pinSpeaker"] as? Bool ?? false + result(pcmSpeakerPlayer.start(sampleRate: rate, pinSpeaker: pin)) case "feed": if let args = call.arguments as? [String: Any], let data = args["data"] as? FlutterStandardTypedData { @@ -615,6 +623,17 @@ internal func handlePcmPlayerMethodCall(_ call: FlutterMethodCall, result: @esca let active = args?["active"] as? Bool ?? false setDeviceDialogNowPlaying(active) result(nil) + case "phoneMicStart": + let micArgs = call.arguments as? [String: Any] + let vp = micArgs?["voiceProcessing"] as? Bool ?? true + phoneMic.start(voiceProcessing: vp) { err in + result(err) + } + case "phoneMicStop": + phoneMic.stop { result(nil) } + case "phoneMicKick": + phoneMic.kick() + result(nil) case "inputFormat": // 给 Dart 侧 MicPcmStreamer 探路：record 插件会拿输入格式去 installTap， // 0Hz 时那是 ObjC 断言必崩，所以开麦前先在这里看一眼。 @@ -1438,6 +1457,8 @@ private func sendAudioDataEvent(_ event: [String: Any]) { let translationLang1 = supported.count > 3 ? supported[3] : "en-US" let ttsLang0 = supported.count > 4 ? supported[4] : "zh-CN-XiaoxiaoNeural" let ttsLang1 = supported.count > 5 ? supported[5] : "en-US-AriaNeural" + let singleLeg = args["singleLeg"] as? Bool ?? false + let bidirectional = args["bidirectional"] as? Bool ?? false currentAstProvider = provider // 原生直连桥：新会话重新刻音色 → 门限复位；helper 起来之前的帧先攒着（见 BesCallPcmBridge） setAstSessionReady(false) @@ -1488,8 +1509,10 @@ private func sendAudioDataEvent(_ event: [String: Any]) { sourceLanguage: lang0, targetLanguage: lang1, currentVoice: ttsLang1, + enableAutoLanguageDetection: bidirectional, translationSourceLanguage: translationLang0, - translationTargetLanguage: translationLang1 + translationTargetLanguage: translationLang1, + reverseVoice: ttsLang0 ) let svcB = IntegratedSpeechTranslationService.ServiceConfiguration( sourceLanguage: lang1, @@ -1515,6 +1538,12 @@ private func sendAudioDataEvent(_ event: [String: Any]) { } os_log("[AST-INIT] Azure: initializing A service...", log: self.ctLog, type: .info) await azureAstHelperA.initialize(azureConfig: azureConfigI, translationConfig: translationConfigI, serviceConfig: svcA, callback: callbackA) + if singleLeg { + os_log("[AST-INIT] Azure: single leg bidirectional=%d, skip B", log: self.ctLog, type: .info, bidirectional ? 1 : 0) + self.markAstBridgeReady() + DispatchQueue.main.async { result(true) } + return + } os_log("[AST-INIT] Azure: A initialized, waiting 1.2s before B...", log: self.ctLog, type: .info) try? await Task.sleep(nanoseconds: 1_200_000_000) if !self.isAstInitGenerationCurrent(initGeneration) { diff --git a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/IntegratedSpeechTranslationService.swift b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/IntegratedSpeechTranslationService.swift index 2888388f..e3b0aa68 100644 --- a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/IntegratedSpeechTranslationService.swift +++ b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/IntegratedSpeechTranslationService.swift @@ -44,6 +44,21 @@ import os.log private var speechConfig: SPXSpeechConfiguration? private var recognizer: SPXSpeechRecognizer? private var synthesizer: SPXSpeechSynthesizer? + /// 合成器不走扬声器时挂的输出流。不留着的话流会被释放，合成器又会去占系统播放引擎。 + private var synthDiscard: SPXPushAudioOutputStream? + /// 合成器同一时间只能说一句。上一句还在出音频时再 startSpeaking，下一句会被拒， + /// 屏幕上有译文，扬声器却没声音。这里按句排队，说完再起下一句。 + private struct PendingSpeak { + let id: String + let text: String + let ssml: String + } + private let speakLock = NSLock() + private let speakChain = DispatchQueue(label: "azure_speech.speak_chain") + private var speakQueue: [PendingSpeak] = [] + private var speakBusy = false + private var speakClosed = false + private var speakToken = 0 // 翻译服务 private var translationService: TranslationServiceInterface? @@ -58,6 +73,9 @@ import os.log // 配置管理 private var serviceConfig = ServiceConfiguration() + /// 这一句译文的合成语种和音色。面对面听出目标语时改成反向。 + private var turnXmlLang: String = "" + private var turnVoice: String = "" // 状态管理 private let serviceState = ServiceState() @@ -66,6 +84,9 @@ import os.log private var utteranceSeq: Int64 = 0 private var currentUtteranceId: String? private var isInUtterance: Bool = false + /// 上一句还在翻译时，后面的整句先排在这里。直接丢掉的话，第二句永远没有译文。 + private var queuedFinals: [(id: String, text: String, detected: String)] = [] + private let finalLock = NSLock() private var recognizingTranslationTask: Task? private var currentSynthesisId: String? private var currentSynthesisText: String? @@ -112,6 +133,8 @@ import os.log // 新增：可选翻译源/目标语言（优先级高于识别语言对） var translationSourceLanguage: String = "" var translationTargetLanguage: String = "" + /// 面对面反向时的音色：用户说的是目标语，译文用源语种的声音。 + var reverseVoice: String = "" public init( sourceLanguage: String = "zh-CN", @@ -126,7 +149,8 @@ import os.log translationTimeout: TimeInterval = 10.0, enableAudioPlayback: Bool = false, translationSourceLanguage: String = "", - translationTargetLanguage: String = "" + translationTargetLanguage: String = "", + reverseVoice: String = "" ) { self.sourceLanguage = sourceLanguage self.targetLanguage = targetLanguage @@ -141,6 +165,7 @@ import os.log self.enableAudioPlayback = enableAudioPlayback self.translationSourceLanguage = translationSourceLanguage self.translationTargetLanguage = translationTargetLanguage + self.reverseVoice = reverseVoice } } @@ -321,15 +346,14 @@ import os.log return false } - // 识别配置上只钉这一路要听的语种。远端中→爱尔兰就是 ga-IE， + // 单向（通话、同声）只钉这一路要听的语种。远端中→爱尔兰就是 ga-IE， // 不能再混进中文，否则开头的串音会把整场锁成中文，爱尔兰语永远无字。 - // 合成音色另建一份配置，避免和识别语种抢同一份 SpeechConfig。 - config.speechRecognitionLanguage = serviceConfig.sourceLanguage + // 面对面才开自动检测，两种语言都能听，再按听出来的语种决定翻译方向。 config.setPropertyTo("300", by: SPXPropertyId.speechSegmentationSilenceTimeoutMs) - - // 自动语言检测 if serviceConfig.enableAutoLanguageDetection { config.setPropertyTo("Continuous", by: SPXPropertyId.speechServiceConnectionLanguageIdMode) + } else { + config.speechRecognitionLanguage = serviceConfig.sourceLanguage } os_log("Azure语音服务配置完成", log: log, type: .info) @@ -386,9 +410,20 @@ import os.log } do { - speechConfig.speechRecognitionLanguage = serviceConfig.sourceLanguage - os_log("识别器语种: %@ service=%@", log: log, type: .info, serviceConfig.sourceLanguage, serviceId) - recognizer = try SPXSpeechRecognizer(speechConfiguration: speechConfig, audioConfiguration: audioConfig) + if serviceConfig.enableAutoLanguageDetection { + let candidates = [serviceConfig.sourceLanguage, serviceConfig.targetLanguage] + let auto = try SPXAutoDetectSourceLanguageConfiguration(candidates) + os_log("识别器双向: %@ / %@ service=%@", log: log, type: .info, serviceConfig.sourceLanguage, serviceConfig.targetLanguage, serviceId) + recognizer = try SPXSpeechRecognizer( + speechConfiguration: speechConfig, + autoDetectSourceLanguageConfiguration: auto, + audioConfiguration: audioConfig + ) + } else { + speechConfig.speechRecognitionLanguage = serviceConfig.sourceLanguage + os_log("识别器语种: %@ service=%@", log: log, type: .info, serviceConfig.sourceLanguage, serviceId) + recognizer = try SPXSpeechRecognizer(speechConfiguration: speechConfig, audioConfiguration: audioConfig) + } // 识别中事件 recognizer?.addRecognizingEventHandler { [weak self] (_, event) in @@ -396,7 +431,8 @@ import os.log let result = event.result guard let text = result.text, !text.isEmpty else { return } let confidence = self.extractConfidence(from: result) - os_log("识别中事件: %@", log: self.log, type: .debug, text) + let detected = self.detectedLanguage(from: result) + os_log("识别中事件: %@ lang=%@", log: self.log, type: .debug, text, detected) // 开始新的话段时生成并记录 utteranceId if !self.isInUtterance { self.currentUtteranceId = self.nextUtteranceId() @@ -407,7 +443,7 @@ import os.log self.eventCallback?.onRecognizing( utteranceId: uttId, text: text, - language: self.serviceConfig.sourceLanguage, + language: detected, confidence: confidence ) } @@ -416,7 +452,7 @@ import os.log self.recognizingTranslationTask?.cancel() self.recognizingTranslationTask = Task { [weak self] in guard let self = self else { return } - await self.processTranslationOnly(utteranceId: uttId, text: text) + await self.processTranslationOnly(utteranceId: uttId, text: text, detectedLanguage: detected) } } } @@ -430,17 +466,18 @@ import os.log case .recognizedSpeech: if let text = result.text, !text.isEmpty { let confidence = self.extractConfidence(from: result) + let detected = self.detectedLanguage(from: result) let uttId = self.currentUtteranceId ?? self.nextUtteranceId() DispatchQueue.main.async { self.eventCallback?.onRecognized( utteranceId: uttId, text: text, - language: self.serviceConfig.sourceLanguage, + language: detected, confidence: confidence ) } - os_log("识别完成事件: %@", log: self.log, type: .info, text) + os_log("识别完成事件: %@ lang=%@", log: self.log, type: .info, text, detected) // 检查服务状态，只有在识别仍然活跃时才触发翻译流程 if self.serviceState.isRecognizing { @@ -450,7 +487,7 @@ import os.log } Task { await self.cancelRecognizingTranslationTaskSafely() - await self.processTranslationAndSynthesis(utteranceId: uttId, text: text) + await self.processTranslationAndSynthesis(utteranceId: uttId, text: text, detectedLanguage: detected) } } else { os_log("服务已停止，跳过翻译流程", log: self.log, type: .info) @@ -528,8 +565,16 @@ import os.log if serviceConfig.enableAudioPlayback { audioConfig = SPXAudioConfiguration() } else { - // 不输出到扬声器，只生成音频数据 - audioConfig = nil + // nil 会让 SDK 自己打开扬声器引擎。同声翻译的手机麦接着要开 + // voice-processing，两个引擎抢 IO，第一次 start 必抛 2003329396， + // 按钮看起来就像点了没反应。这里把 PCM 丢进空流，播放仍走合成事件。 + guard let discard = SPXPushAudioOutputStream(writeHandler: { data in + UInt(data.count) + }, closeHandler: {}) else { + throw NSError(domain: "azure_speech", code: -1) + } + synthDiscard = discard + audioConfig = try SPXAudioConfiguration(streamOutput: discard) } let synthConfig: SPXSpeechConfiguration @@ -582,23 +627,25 @@ import os.log guard let self = self else { return } os_log("合成完成事件", log: self.log, type: .info) - // 安全解包audioData - guard let audioData = event.result.audioData else { + let audioCount = event.result.audioData?.count ?? 0 + if audioCount == 0 { os_log("合成完成事件，音频数据为空", log: self.log, type: .debug) - return + } else { + os_log("合成完成事件，事件数据长度: %d bytes", log: self.log, type: .debug, audioCount) } - os_log("合成完成事件，事件数据长度: %d bytes, 缓冲数据长度: %d bytes", log: self.log, type: .debug, audioData.count, 0) - let audioCopy = Data(audioData) + self.speakLock.lock() + let token = self.speakToken + self.speakLock.unlock() DispatchQueue.main.async { self.serviceState.isSynthesizing = false - // 立即返回当前生成的音频数据 let uttId = self.currentSynthesisId ?? "" let text = self.currentSynthesisText ?? "" - //self.eventCallback?.onSynthesisAudioGenerated(utteranceId: uttId, text: text, audioData: audioCopy) + // 完成事件里的音频是整段，进行中事件已经逐块送出去了，这里再送会播两遍。 self.eventCallback?.onSynthesisCompleted(utteranceId: uttId, text: text) self.currentSynthesisId = nil self.currentSynthesisText = nil self.eventCallback?.onStateChanged(component: "Synthesis", isActive: false) + self.speakFinished(token) } } @@ -612,6 +659,9 @@ import os.log // 修复：移除可选链，因为event.result不是可选类型 let reason = event.result.reason + self.speakLock.lock() + let token = self.speakToken + self.speakLock.unlock() DispatchQueue.main.async { let uttId = self.currentSynthesisId ?? "" @@ -620,6 +670,7 @@ import os.log self.currentSynthesisId = nil self.currentSynthesisText = nil self.eventCallback?.onStateChanged(component: "Synthesis", isActive: false) + self.speakFinished(token) } } @@ -636,7 +687,10 @@ import os.log * 开始连续语音翻译 */ public func startContinuousTranslation() -> Bool { - audioProcessor?.isContinuousRecognitionActive = true + speakLock.lock() + speakClosed = false + speakLock.unlock() + audioProcessor?.isContinuousRecognitionActive = true os_log("尝试启动连续翻译，当前状态：初始化=%@, 识别中=%@", log: log, type: .info, serviceState.isInitialized ? "true" : "false", serviceState.isRecognizing ? "true" : "false") @@ -686,6 +740,10 @@ import os.log * 停止连续语音翻译 */ public func stopContinuousTranslation() { + speakLock.lock() + speakClosed = true + speakQueue.removeAll() + speakLock.unlock() do { audioProcessor?.isContinuousRecognitionActive = false self.serviceState.isRecognizing = false @@ -738,23 +796,78 @@ import os.log * - utteranceId: 该句话的唯一ID * - text: 识别完成的最终文本 */ + private func sameLanguage(_ a: String, _ b: String) -> Bool { + let sa = a.split(separator: "-").first?.lowercased() ?? "" + let sb = b.split(separator: "-").first?.lowercased() ?? "" + return !sa.isEmpty && sa == sb + } + + private func detectedLanguage(from result: SPXSpeechRecognitionResult) -> String { + guard serviceConfig.enableAutoLanguageDetection else { + return serviceConfig.sourceLanguage + } + if let properties = result.properties, + let language = properties.getPropertyByName("speechServiceResponse_RecognitionLanguage"), + !language.isEmpty { + return language + } + if let langResult = try? SPXAutoDetectSourceLanguageResult(result), + let language = langResult.language, !language.isEmpty { + return language + } + return serviceConfig.sourceLanguage + } + + private struct TurnDirection { + let source: String + let target: String + let voice: String + let xmlLang: String + } + + /// 面对面听出目标语（比如选了中→日却说了日语）就反过来翻。单向模式永远按配置的方向。 + private func direction(for detected: String) -> TurnDirection { + let reverse = serviceConfig.enableAutoLanguageDetection && sameLanguage(detected, serviceConfig.targetLanguage) + if reverse { + let src = serviceConfig.translationTargetLanguage.isEmpty ? serviceConfig.targetLanguage : serviceConfig.translationTargetLanguage + let tgt = serviceConfig.translationSourceLanguage.isEmpty ? serviceConfig.sourceLanguage : serviceConfig.translationSourceLanguage + let voice = serviceConfig.reverseVoice.isEmpty ? serviceConfig.currentVoice : serviceConfig.reverseVoice + return TurnDirection(source: src, target: tgt, voice: voice, xmlLang: serviceConfig.sourceLanguage) + } + let src = serviceConfig.translationSourceLanguage.isEmpty ? serviceConfig.sourceLanguage : serviceConfig.translationSourceLanguage + let tgt = serviceConfig.translationTargetLanguage.isEmpty ? serviceConfig.targetLanguage : serviceConfig.translationTargetLanguage + return TurnDirection(source: src, target: tgt, voice: serviceConfig.currentVoice, xmlLang: serviceConfig.targetLanguage) + } + /// 处理最终翻译并进行语音合成（优先使用配置的翻译语言对） - private func processTranslationAndSynthesis(utteranceId: String, text: String) async { + private func processTranslationAndSynthesis(utteranceId: String, text: String, detectedLanguage: String = "", owned: Bool = false) async { // 添加服务状态检查 guard serviceState.isRecognizing else { os_log("服务已停止，取消翻译流程", log: log, type: .info) + if owned { + finalLock.lock() + queuedFinals.removeAll() + serviceState.isTranslating = false + finalLock.unlock() + } return } os_log("翻译状态: %@", log: log, type: .debug, serviceState.isTranslating ? "进行中" : "空闲") - - if serviceState.isTranslating { - os_log("翻译正在进行中，跳过当前请求", log: log, type: .info) - return + + if !owned { + finalLock.lock() + if serviceState.isTranslating { + queuedFinals.append((id: utteranceId, text: text, detected: detectedLanguage)) + let queued = queuedFinals.count + finalLock.unlock() + os_log("上一句还在翻译，本句排队 #%d: %@", log: log, type: .info, queued, text) + return + } + serviceState.isTranslating = true + finalLock.unlock() } - serviceState.isTranslating = true - await MainActor.run { eventCallback?.onTranslationStarted(utteranceId: utteranceId, text: text) eventCallback?.onStateChanged(component: "Translation", isActive: true) @@ -763,9 +876,12 @@ import os.log os_log("翻译开始: %@", log: log, type: .info, text) do { - let srcLang = self.serviceConfig.translationSourceLanguage.isEmpty ? self.serviceConfig.sourceLanguage : self.serviceConfig.translationSourceLanguage - let tgtLang = self.serviceConfig.translationTargetLanguage.isEmpty ? self.serviceConfig.targetLanguage : self.serviceConfig.translationTargetLanguage - os_log("翻译语言对: %@ -> %@", log: log, type: .info, srcLang, tgtLang) + let turn = self.direction(for: detectedLanguage) + self.turnXmlLang = turn.xmlLang + self.turnVoice = turn.voice + let srcLang = turn.source + let tgtLang = turn.target + os_log("翻译语言对: %@ -> %@ (heard=%@)", log: log, type: .info, srcLang, tgtLang, detectedLanguage) let translationResult = try await withTimeout(self.serviceConfig.translationTimeout) { os_log("开始翻译文本: %@", log: self.log, type: .debug, text) return await self.translationService?.translateText( @@ -775,8 +891,6 @@ import os.log ) } - serviceState.isTranslating = false - await MainActor.run { eventCallback?.onStateChanged(component: "Translation", isActive: false) } @@ -807,8 +921,6 @@ import os.log } } catch { - serviceState.isTranslating = false - await MainActor.run { eventCallback?.onStateChanged(component: "Translation", isActive: false) eventCallback?.onTranslationFailed(utteranceId: utteranceId, text: text, error: "翻译超时或异常: \(error.localizedDescription)") @@ -816,6 +928,29 @@ import os.log os_log("翻译异常: %@", log: log, type: .error, error.localizedDescription) } + await drainQueuedFinals() + } + + /// 本句翻译结束。队列里还有下一句就接着翻，没有才放开翻译占位。 + private func drainQueuedFinals() async { + let next: (id: String, text: String, detected: String)? + finalLock.lock() + if serviceState.isRecognizing, !queuedFinals.isEmpty { + next = queuedFinals.removeFirst() + } else { + next = nil + queuedFinals.removeAll() + serviceState.isTranslating = false + } + finalLock.unlock() + if let next { + await processTranslationAndSynthesis( + utteranceId: next.id, + text: next.text, + detectedLanguage: next.detected, + owned: true + ) + } } /** @@ -825,11 +960,12 @@ import os.log * - text: 识别中的临时文本 */ /// 处理识别中临时翻译（优先使用配置的翻译语言对） - private func processTranslationOnly(utteranceId: String, text: String) async { + private func processTranslationOnly(utteranceId: String, text: String, detectedLanguage: String = "") async { guard serviceState.isRecognizing else { return } do { - let srcLang = self.serviceConfig.translationSourceLanguage.isEmpty ? self.serviceConfig.sourceLanguage : self.serviceConfig.translationSourceLanguage - let tgtLang = self.serviceConfig.translationTargetLanguage.isEmpty ? self.serviceConfig.targetLanguage : self.serviceConfig.translationTargetLanguage + let turn = self.direction(for: detectedLanguage) + let srcLang = turn.source + let tgtLang = turn.target let translationResult = try await withTimeout(self.serviceConfig.translationTimeout) { return await self.translationService?.translateText( text: text, @@ -897,42 +1033,99 @@ import os.log os_log("合成文本为空或仅空白，跳过合成", log: log, type: .info) return false } - guard let synthesizer = synthesizer else { + guard synthesizer != nil else { os_log("语音合成器未初始化", log: log, type: .error) return false } - + let sanitized = cleanTextForTTS(trimmed) + // SSML 在入队时就定下来。等轮到它再读音色，下一句可能已经把音色改掉。 + let item = PendingSpeak(id: utteranceId, text: trimmed, ssml: generateOptimizedSsml(sanitized)) + speakLock.lock() + if speakClosed { + speakLock.unlock() + os_log("合成已关闭，跳过: %@", log: log, type: .info, trimmed) + return false + } + speakQueue.append(item) + let start = !speakBusy + if start { speakBusy = true } + let waiting = speakQueue.count + speakLock.unlock() + if start { + scheduleNextSpeak() + } else { + os_log("上一句还在合成，本句排队 #%d: %@", log: log, type: .info, waiting, trimmed) + } + return true + } + + private func scheduleNextSpeak() { + speakChain.async { [weak self] in + self?.startNextSpeak() + } + } + + private func startNextSpeak() { + let item: PendingSpeak? + speakLock.lock() + if speakClosed || speakQueue.isEmpty { + speakQueue.removeAll() + speakBusy = false + item = nil + } else { + item = speakQueue.removeFirst() + speakBusy = true + } + speakLock.unlock() + guard let item else { return } + beginSpeak(item) + } + + /// 本句合成结束（成功、取消、启动失败）才放行下一句。完成和取消可能都来一次，用 token 只放行一次。 + private func speakFinished(_ token: Int) { + speakLock.lock() + // 完成和取消可能对同一句各来一次，只放行下一句一次。 + guard token != 0, token == speakToken else { + speakLock.unlock() + return + } + speakToken = 0 + speakLock.unlock() + scheduleNextSpeak() + } + + private func beginSpeak(_ item: PendingSpeak) { + guard let synthesizer else { + os_log("语音合成器未初始化", log: log, type: .error) + speakLock.lock() + speakBusy = false + speakLock.unlock() + scheduleNextSpeak() + return + } + speakLock.lock() + speakToken += 1 + let token = speakToken + speakLock.unlock() do { - let sanitized = cleanTextForTTS(trimmed) - let ssml = generateOptimizedSsml(sanitized) - os_log("开始流式合成语音: %@", log: log, type: .info, text) - + os_log("开始流式合成语音: %@", log: log, type: .info, item.text) serviceState.isSynthesizing = true - currentSynthesisId = utteranceId - currentSynthesisText = text - + currentSynthesisId = item.id + currentSynthesisText = item.text DispatchQueue.main.async { - self.eventCallback?.onSynthesisStarted(utteranceId: utteranceId, text: text) + self.eventCallback?.onSynthesisStarted(utteranceId: item.id, text: item.text) self.eventCallback?.onStateChanged(component: "Synthesis", isActive: true) } - // 使用异步启动，提升并发兼容性 - try synthesizer.startSpeakingSsml(ssml) - - - return true - + try synthesizer.startSpeakingSsml(item.ssml) } catch { serviceState.isSynthesizing = false - let errorMessage = "合成启动异常: \(error.localizedDescription)" os_log("%@", log: log, type: .error, errorMessage) - DispatchQueue.main.async { - self.eventCallback?.onSynthesisFailed(utteranceId: utteranceId, text: text, error: errorMessage) + self.eventCallback?.onSynthesisFailed(utteranceId: item.id, text: item.text, error: errorMessage) self.eventCallback?.onStateChanged(component: "Synthesis", isActive: false) } - - return false + speakFinished(token) } } @@ -982,11 +1175,11 @@ import os.log .replacingOccurrences(of: "&", with: "&") .replacingOccurrences(of: "<", with: "<") .replacingOccurrences(of: ">", with: ">") - print("serviceConfig.targetLanguage: \(serviceConfig.targetLanguage)") - let voice = getVoiceForLanguage(serviceConfig.targetLanguage) + let xmlLang = turnXmlLang.isEmpty ? serviceConfig.targetLanguage : turnXmlLang + let voice = turnVoice.isEmpty ? getVoiceForLanguage(xmlLang) : turnVoice // 简化SSML结构 return """ - + \(escapedText) @@ -1184,6 +1377,7 @@ import os.log // 释放Azure组件 recognizer = nil synthesizer = nil + synthDiscard = nil speechConfig = nil audioConfig = nil diff --git a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/PcmSpeakerPlayer.swift b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/PcmSpeakerPlayer.swift index 4aea8f02..a131b4ba 100644 --- a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/PcmSpeakerPlayer.swift +++ b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/PcmSpeakerPlayer.swift @@ -1,187 +1,249 @@ +import AudioToolbox import AVFoundation import Foundation -/// 流式 PCM 外放播放器（16bit 小端、单声道）。 +/// 流式 PCM 外放（16bit 小端、单声道）。 /// -/// 存在的理由：iOS 上 **just_audio 放不了「长度未知的直播 WAV 流」**。 -/// 真机实测（iPhone 12 / iOS 26.6）AVPlayer 拿到 just_audio 代理服务给的 -/// chunked 响应后，只发一次 `Range: bytes=0-1` 的嗅探请求就判定资源不可播， -/// 立刻回 `-11850 Operation Stopped` 且 processingState 直接落到 idle， -/// **一次都没真正起播**。声明长度、支持 range、把收到的字节全缓存下来允许 -/// 重放，三种都试过，结论一样。ExoPlayer 不发这个嗅探请求，所以 Android -/// 那条路一直是好的 —— 这就是「安卓有声、苹果没声」的全部原因。 +/// 用 AudioQueue，不用 AVAudioEngine。采集也是一条独立的 AudioQueue。 +/// 把播放节点挂到引擎上，节点没连上就 play()，系统会以 +/// "player started when in a disconnected state" 直接杀进程 +/// （2026-10-18 09:13 起每次点开始都是这个）。 /// -/// 这里绕开 AVPlayer + HTTP 代理，直接 AVAudioEngine + AVAudioPlayerNode -/// 排队播放，和 `AzureTtsHelper` 里那套流式播放同源。 -/// -/// ⚠️ **不碰 AVAudioSession**：会话由 Dart 侧的 audio_session 统一配置 -/// （连续对话时录音的 voice-processing 也在同一个会话上）。这里再去 -/// setCategory/setActive 就会和录音互相掀桌子。 +/// 播放队列和麦克风队列各走各的，互不 start / connect。 +/// 不改 AVAudioSession：会话在开麦前已经配成扬声器。 class PcmSpeakerPlayer { - private let queue = DispatchQueue(label: "azure_speech.pcm_speaker") - - private var engine: AVAudioEngine? - private var node: AVAudioPlayerNode? - private var format: AVAudioFormat? + private let lock = NSRecursiveLock() - /// 已排队但还没播完的缓冲数 - private var pending = 0 - /// 上层已经调过 end()：pending 归零就算这一轮播完 + private var output: AudioQueueRef? + private var owned: [AudioQueueBufferRef] = [] + private var freeBuffers: [AudioQueueBufferRef] = [] + private var stash = Data() + private var rate: Double = 0 private var ended = false - /// 本轮代际，stop() 之后迟到的完成回调按代际丢弃 + private var inFlight = 0 private var generation = 0 - /// 一轮音频播完（或本轮压根没有音频） var onCompleted: (() -> Void)? - // MARK: - 对外接口 + private static let outputCallback: AudioQueueOutputCallback = { userData, _, buffer in + guard let userData else { return } + Unmanaged.fromOpaque(userData).takeUnretainedValue().bufferReturned(buffer) + } - func start(sampleRate: Double) -> Bool { - var ok = false - queue.sync { - generation += 1 - pending = 0 - ended = false - ok = ensureEngine(sampleRate: sampleRate) + func start(sampleRate: Double, pinSpeaker: Bool = false) -> Bool { + lock.lock() + generation += 1 + ended = false + let ok = ensureQueue(sampleRate: sampleRate) + lock.unlock() + // 只有明确要求钉扬声器时才改路由。同声翻译不钉： + // 开着麦再改路由，采集会断，第一句话就没了。 + if ok && pinSpeaker { + try? AVAudioSession.sharedInstance().overrideOutputAudioPort(.speaker) } return ok } func feed(_ data: Data) { - queue.async { [weak self] in - guard let self = self else { return } - guard let node = self.node, let engine = self.engine, engine.isRunning else { return } - guard let buffer = self.makeBuffer(from: data) else { return } - let gen = self.generation - self.pending += 1 - // ⚠️ 必须用 .dataPlayedBack 而不是默认的 .dataRendered： - // 播完这一轮要顺手停引擎（见 finishIfDone），用 dataRendered 会在 - // 声音真正出完之前就回调，尾音被切掉。 - node.scheduleBuffer(buffer, completionCallbackType: .dataPlayedBack) { [weak self] _ in - self?.queue.async { - guard let self = self, gen == self.generation else { return } - self.pending -= 1 - self.finishIfDone() - } - } - if !node.isPlaying { node.play() } - } + guard !data.isEmpty else { return } + lock.lock() + stash.append(data) + let started = pump() + let q = output + let gen = generation + lock.unlock() + // 一句播完队列会自己停。停着的时候只入队不会出声，下一句必须再 Start。 + // 不在持锁时调用，回调线程里再 Start 会和队列互相等。 + if started { kick(q, generation: gen) } } - /// 上层不再喂数据了：已排队的播完就报完成 func end() { - queue.async { [weak self] in - guard let self = self else { return } - self.ended = true - self.finishIfDone() + lock.lock() + ended = true + pump() + let fire = stash.isEmpty && inFlight == 0 + if fire { ended = false } + lock.unlock() + if fire { + let cb = onCompleted + DispatchQueue.main.async { cb?() } } } - /// 打断：立刻停声，不报完成 func stop() { - queue.async { [weak self] in - guard let self = self else { return } - self.generation += 1 - self.pending = 0 - self.ended = false - self.node?.stop() - self.engine?.stop() + lock.lock() + generation += 1 + ended = false + stash.removeAll() + inFlight = 0 + let q = output + lock.unlock() + if let q { + // true：等回调都回来再往下走。回调里不再把缓冲区丢到别的队列。 + AudioQueueStop(q, true) + } + lock.lock() + if output != nil { + freeBuffers = owned + inFlight = 0 } + lock.unlock() } func dispose() { - queue.async { [weak self] in - guard let self = self else { return } - self.generation += 1 - self.node?.stop() - self.engine?.stop() - self.node = nil - self.engine = nil - self.format = nil + lock.lock() + generation += 1 + ended = false + stash.removeAll() + inFlight = 0 + let q = output + output = nil + owned.removeAll() + freeBuffers.removeAll() + rate = 0 + lock.unlock() + if let q { + AudioQueueStop(q, true) + AudioQueueDispose(q, true) } } - // MARK: - 内部 - - private func finishIfDone() { - guard ended, pending <= 0 else { return } - ended = false - // ⚠️ 一轮播完必须把引擎停掉，不能让它空跑着占住音频 IO。 - // 连续对话下 record 要在同一个会话上开 voice-processing， - // 我们这台引擎还占着的话它会拿不到 IO，报 - // `avfaudio error 2003329396`（FourCC 'what'）—— 表现就是 - // 「助手说完话之后麦克风再也开不起来」。真机上间歇复现过。 - // 对象留着不销毁，下一轮 ensureEngine 直接 start() 就好。 - node?.stop() - engine?.stop() - let cb = onCompleted - DispatchQueue.main.async { cb?() } + private func bufferReturned(_ buffer: AudioQueueBufferRef) { + lock.lock() + // 队列已经拆掉：这块内存马上要被 Dispose，不能再收进空闲表。 + guard output != nil else { + lock.unlock() + return + } + freeBuffers.append(buffer) + if inFlight > 0 { inFlight -= 1 } + let started = pump() + let q = output + let gen = generation + let fire = ended && stash.isEmpty && inFlight == 0 + if fire { ended = false } + lock.unlock() + if started { kick(q, generation: gen) } + if fire { + let cb = onCompleted + DispatchQueue.main.async { cb?() } + } } - /// 采样率变了要整条重建：AVAudioEngine 的连接格式是建链路时定死的 - private func ensureEngine(sampleRate: Double) -> Bool { - if let engine = engine, let node = node, let fmt = format, - fmt.sampleRate == sampleRate { - if !engine.isRunning { - do { try engine.start() } catch { tearDown(); return buildEngine(sampleRate: sampleRate) } - } - if !node.isPlaying { node.play() } + private func ensureQueue(sampleRate: Double) -> Bool { + if let output, rate == sampleRate { + AudioQueueStart(output, nil) return true } tearDown() - return buildEngine(sampleRate: sampleRate) + return buildQueue(sampleRate: sampleRate) } - private func buildEngine(sampleRate: Double) -> Bool { - guard let fmt = AVAudioFormat(standardFormatWithSampleRate: sampleRate, channels: 1) else { - return false + private func buildQueue(sampleRate: Double) -> Bool { + var format = AudioStreamBasicDescription( + mSampleRate: sampleRate, + mFormatID: kAudioFormatLinearPCM, + mFormatFlags: kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked, + mBytesPerPacket: 2, + mFramesPerPacket: 1, + mBytesPerFrame: 2, + mChannelsPerFrame: 1, + mBitsPerChannel: 16, + mReserved: 0 + ) + var queue: AudioQueueRef? + let status = AudioQueueNewOutput( + &format, + PcmSpeakerPlayer.outputCallback, + Unmanaged.passUnretained(self).toOpaque(), + nil, + nil, + 0, + &queue + ) + guard status == noErr, let queue else { return false } + var allocated: [AudioQueueBufferRef] = [] + for _ in 0..<6 { + var buffer: AudioQueueBufferRef? + let alloc = AudioQueueAllocateBuffer(queue, 4096, &buffer) + guard alloc == noErr, let buffer else { + AudioQueueDispose(queue, true) + return false + } + allocated.append(buffer) } - let e = AVAudioEngine() - let n = AVAudioPlayerNode() - e.attach(n) - e.connect(n, to: e.mainMixerNode, format: fmt) - do { - try e.start() - } catch { - NSLog("[PcmSpeakerPlayer] 引擎启动失败: \(error.localizedDescription)") + let start = AudioQueueStart(queue, nil) + guard start == noErr else { + AudioQueueDispose(queue, true) return false } - n.play() - engine = e - node = n - format = fmt + output = queue + owned = allocated + freeBuffers = allocated + rate = sampleRate + inFlight = 0 return true } private func tearDown() { - node?.stop() - engine?.stop() - node = nil - engine = nil - format = nil + let q = output + output = nil + owned.removeAll() + freeBuffers.removeAll() + rate = 0 + inFlight = 0 + stash.removeAll() + if let q { + // 先放开锁再停，避免回调里再来抢这把锁时和 Stop 互相等。 + lock.unlock() + AudioQueueStop(q, true) + AudioQueueDispose(q, true) + lock.lock() + } } - /// Int16 小端裸 PCM → Float32 单声道 buffer - private func makeBuffer(from data: Data) -> AVAudioPCMBuffer? { - guard let fmt = format else { return nil } - let frames = data.count / 2 - guard frames > 0, - let buffer = AVAudioPCMBuffer(pcmFormat: fmt, - frameCapacity: AVAudioFrameCount(frames)), - let dst = buffer.floatChannelData else { return nil } - buffer.frameLength = AVAudioFrameCount(frames) - data.withUnsafeBytes { raw in - guard let base = raw.baseAddress else { return } - // ⚠️ 不能 bindMemory(to: Int16.self)：Data 的起始地址不保证 2 字节对齐， - // 未对齐访问在 arm64 上是未定义行为。逐字节取再拼。 - let bytes = base.assumingMemoryBound(to: UInt8.self) - for i in 0.. Bool { + guard let output else { return false } + var started = false + while !freeBuffers.isEmpty, stash.count >= 2 { + let buffer = freeBuffers.removeFirst() + let capacity = Int(buffer.pointee.mAudioDataBytesCapacity) + var count = min(capacity, stash.count) + count -= count % 2 + guard count >= 2 else { + freeBuffers.append(buffer) + break + } + let dest = buffer.pointee.mAudioData + stash.withUnsafeBytes { raw in + if let base = raw.baseAddress { + memcpy(dest, base, count) + } + } + stash.removeFirst(count) + buffer.pointee.mAudioDataByteSize = UInt32(count) + let status = AudioQueueEnqueueBuffer(output, buffer, 0, nil) + if status != noErr { + freeBuffers.append(buffer) + break } + inFlight += 1 + started = true } - return buffer + return started } } diff --git a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/PhoneMicCapture.swift b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/PhoneMicCapture.swift new file mode 100644 index 00000000..4ae91828 --- /dev/null +++ b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/PhoneMicCapture.swift @@ -0,0 +1,313 @@ +import AudioToolbox +import AVFoundation +import Flutter +import Foundation + +enum AudioGraph { + static let queue = DispatchQueue(label: "azure_speech.audio_graph") +} + +/// 同声 / 面对面的手机麦。只采集，不播放。 +/// +/// 用 AudioQueue 输入，不用 AVAudioEngine。引擎上的输入 tap 在这台系统上 +/// 往往只回调一次就停，云端等不到声音，十几秒后报 18802。 +/// 播放是另一条 AudioQueue，不要把播放节点挂到采集上。 +final class PhoneMicCapture: NSObject, FlutterStreamHandler { + static weak var shared: PhoneMicCapture? + + private let lock = NSLock() + private var input: AudioQueueRef? + private var sink: FlutterEventSink? + private var running = false + private var life = 0 + private var sourceRate: Double = 16000 + private var resamplePos: Double = 0 + private var routeObserver: NSObjectProtocol? + private var interruptionObserver: NSObjectProtocol? + + func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { + sink = events + return nil + } + + func onCancel(withArguments arguments: Any?) -> FlutterError? { + sink = nil + return nil + } + + func start(voiceProcessing: Bool, done: @escaping (String?) -> Void) { + _ = voiceProcessing + // 队列必须在主线程启动。放在后台队列时，这台系统只回调一次就停。 + DispatchQueue.main.async { [weak self] in + guard let self else { + done("麦克风已释放") + return + } + self.life += 1 + self.attempt(life: self.life, left: 4, done: done) + } + } + + /// AudioQueueStart 在会话刚激活时会回 'what'(2003329396)。等一下再激活再试。 + private func attempt(life: Int, left: Int, done: @escaping (String?) -> Void) { + if self.life != life { + tearDownQueue() + done(nil) + return + } + let err = startLocked() + if err == nil || left <= 1 { + done(err) + return + } + tearDownQueue() + DispatchQueue.main.asyncAfter(deadline: .now() + 0.45) { [weak self] in + guard let self else { + done("麦克风已释放") + return + } + let session = AVAudioSession.sharedInstance() + if session.category != .playAndRecord { + try? session.setCategory( + .playAndRecord, + mode: session.mode == .voiceChat ? .voiceChat : .default, + options: [.defaultToSpeaker, .allowBluetoothA2DP] + ) + } + try? session.setActive(true) + self.attempt(life: life, left: left - 1, done: done) + } + } + + func stop(done: @escaping () -> Void) { + DispatchQueue.main.async { [weak self] in + guard let self else { + done() + return + } + self.life += 1 + self.stopLocked() + done() + } + } + + /// 输入停了但事件通道还在。拆掉队列再起一条。 + func kick() { + DispatchQueue.main.async { [weak self] in + guard let self, self.running else { return } + let life = self.life + self.tearDownQueue() + if self.life != life { return } + let err = self.startLocked() + if self.life != life { + self.tearDownQueue() + return + } + if err != nil { + self.sink?(["type": "reinstallFailed"]) + } else { + self.sink?(["type": "reinstalled"]) + } + } + } + + private func startLocked() -> String? { + if running, input != nil { return nil } + let session = AVAudioSession.sharedInstance() + do { + try session.setActive(true) + } catch { + return "音频会话未激活 \(error.localizedDescription)" + } + // 这台手机把采集写成 16 kHz 时，队列只吐一块就停。停掉的瞬间如果正在连讯飞， + // 重启麦会把握手打断，start 返回 18801。按硬件采样率采，送出前再降到 16 kHz。 + var rates: [Double] = [] + let hw = session.sampleRate + if hw > 1 { rates.append(hw) } + for rate in [48000.0, 44100.0, 16000.0] where !rates.contains(where: { abs($0 - rate) < 1 }) { + rates.append(rate) + } + var last = "麦克风队列启动失败" + for rate in rates { + if let err = openQueue(rate: rate) { + last = err + continue + } + return nil + } + let inputs = session.currentRoute.inputs.map(\.portType.rawValue).joined(separator: ",") + return "\(last) rate=\(hw) cat=\(session.category.rawValue) mode=\(session.mode.rawValue) in=\(inputs)" + } + + /// 按硬件采样率要数据。这个采样率起不来就换下一个，不在同一次失败里空转。 + private func openQueue(rate: Double) -> String? { + sourceRate = rate + resamplePos = 0 + var format = AudioStreamBasicDescription( + mSampleRate: rate, + mFormatID: kAudioFormatLinearPCM, + mFormatFlags: kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked, + mBytesPerPacket: 2, + mFramesPerPacket: 1, + mBytesPerFrame: 2, + mChannelsPerFrame: 1, + mBitsPerChannel: 16, + mReserved: 0 + ) + var queue: AudioQueueRef? + let ctx = Unmanaged.passUnretained(self).toOpaque() + let created = AudioQueueNewInput(&format, PhoneMicCapture.inputCallback, ctx, nil, nil, 0, &queue) + guard created == noErr, let queue else { + return "麦克风队列创建失败 \(created) rate=\(rate)" + } + let bytesPerBuffer = UInt32(max(3200, Int(rate) * 2 / 10)) + for _ in 0..<3 { + var buffer: AudioQueueBufferRef? + let allocated = AudioQueueAllocateBuffer(queue, bytesPerBuffer, &buffer) + if allocated != noErr || buffer == nil { + AudioQueueDispose(queue, true) + return "麦克风缓冲创建失败 \(allocated) rate=\(rate)" + } + let queued = AudioQueueEnqueueBuffer(queue, buffer!, 0, nil) + if queued != noErr { + AudioQueueDispose(queue, true) + return "麦克风缓冲入队失败 \(queued) rate=\(rate)" + } + } + lock.lock() + input = queue + running = true + lock.unlock() + let started = AudioQueueStart(queue, nil) + if started != noErr { + tearDownQueue() + return "麦克风队列启动失败 \(started) rate=\(rate)" + } + watchSession() + reportOpened() + return nil + } + + private func reportOpened() { + let session = AVAudioSession.sharedInstance() + let inputs = session.currentRoute.inputs.map(\.portType.rawValue).joined(separator: ",") + let outputs = session.currentRoute.outputs.map(\.portType.rawValue).joined(separator: ",") + sink?([ + "type": "opened", + "mode": session.mode.rawValue, + "rate": sourceRate, + "input": inputs, + "output": outputs, + ]) + } + + private func watchSession() { + if routeObserver == nil { + routeObserver = NotificationCenter.default.addObserver( + forName: AVAudioSession.routeChangeNotification, + object: nil, + queue: nil + ) { [weak self] _ in + DispatchQueue.main.async { self?.restartIfStopped() } + } + } + if interruptionObserver == nil { + interruptionObserver = NotificationCenter.default.addObserver( + forName: AVAudioSession.interruptionNotification, + object: nil, + queue: nil + ) { [weak self] note in + let raw = note.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt + let type = raw.flatMap { AVAudioSession.InterruptionType(rawValue: $0) } + guard type == .ended else { return } + DispatchQueue.main.async { self?.restartIfStopped() } + } + } + } + + /// 路由或中断把队列停掉时接上。队列还在跑就什么都不做，避免自己把自己重启个没完。 + private func restartIfStopped() { + guard running else { return } + lock.lock() + let queue = input + lock.unlock() + guard let queue else { return } + var flag: UInt32 = 0 + var size = UInt32(MemoryLayout.size) + let status = AudioQueueGetProperty(queue, kAudioQueueProperty_IsRunning, &flag, &size) + if status == noErr, flag != 0 { return } + tearDownQueue() + let err = startLocked() + DispatchQueue.main.async { [weak self] in + if err != nil { + self?.sink?(["type": "reinstallFailed"]) + } else { + self?.sink?(["type": "reinstalled"]) + } + } + } + + private func stopLocked() { + lock.lock() + running = false + lock.unlock() + tearDownQueue() + if let routeObserver { + NotificationCenter.default.removeObserver(routeObserver) + self.routeObserver = nil + } + if let interruptionObserver { + NotificationCenter.default.removeObserver(interruptionObserver) + self.interruptionObserver = nil + } + } + + private func tearDownQueue() { + lock.lock() + running = false + let queue = input + input = nil + lock.unlock() + guard let queue else { return } + AudioQueueStop(queue, true) + AudioQueueDispose(queue, true) + } + + private static let inputCallback: AudioQueueInputCallback = { ctx, queue, buffer, _, _, _ in + guard let ctx else { return } + let mic = Unmanaged.fromOpaque(ctx).takeUnretainedValue() + let size = Int(buffer.pointee.mAudioDataByteSize) + var copied: Data? + if size > 1 { + let frames = size / 2 + copied = mic.downsample(buffer.pointee.mAudioData.assumingMemoryBound(to: Int16.self), frames: frames) + } + mic.lock.lock() + let live = mic.running + mic.lock.unlock() + if live { + AudioQueueEnqueueBuffer(queue, buffer, 0, nil) + } + guard let copied, !copied.isEmpty else { return } + DispatchQueue.main.async { [weak mic] in + mic?.sink?(FlutterStandardTypedData(bytes: copied)) + } + } + + /// 硬件采样率收进来，送出去的一律是 16 kHz。 + private func downsample(_ samples: UnsafePointer, frames: Int) -> Data { + if sourceRate <= 16000 + 1 { + return Data(bytes: samples, count: frames * 2) + } + let ratio = sourceRate / 16000 + var out: [Int16] = [] + out.reserveCapacity(Int(Double(frames) / ratio) + 2) + var cursor = resamplePos + while cursor < Double(frames) { + out.append(samples[Int(cursor)]) + cursor += ratio + } + resamplePos = cursor - Double(frames) + return out.withUnsafeBytes { Data($0) } + } +} diff --git a/apps/client/local_plugins/bluetooth_manager/android/src/main/kotlin/com/example/bluetooth_manager/BluetoothManager.kt b/apps/client/local_plugins/bluetooth_manager/android/src/main/kotlin/com/example/bluetooth_manager/BluetoothManager.kt index eb779422..57820612 100644 --- a/apps/client/local_plugins/bluetooth_manager/android/src/main/kotlin/com/example/bluetooth_manager/BluetoothManager.kt +++ b/apps/client/local_plugins/bluetooth_manager/android/src/main/kotlin/com/example/bluetooth_manager/BluetoothManager.kt @@ -273,7 +273,7 @@ object BluetoothManager : PluginRegistry.RequestPermissionsResultListener { 0x02.toByte() -> BluetoothManager.publishBluetoothCmd(mapOf("type" to "startMakeCall", "data" to data)) 0x03.toByte(), 0x04.toByte() -> { - publishTranslationData(mapOf("type" to "stopCall", "data" to data)) + // 通话列表，不是挂断。不要拆解码器，也不要报 stopCall。 BluetoothManager.publishBluetoothCmd(mapOf("type" to "stopMakeCall", "data" to data)) } diff --git a/apps/client/local_plugins/bluetooth_manager/ios/bluetooth_manager/Sources/bluetooth_manager/BluetoothManager.swift b/apps/client/local_plugins/bluetooth_manager/ios/bluetooth_manager/Sources/bluetooth_manager/BluetoothManager.swift index 08d8ead0..ea0cd544 100644 --- a/apps/client/local_plugins/bluetooth_manager/ios/bluetooth_manager/Sources/bluetooth_manager/BluetoothManager.swift +++ b/apps/client/local_plugins/bluetooth_manager/ios/bluetooth_manager/Sources/bluetooth_manager/BluetoothManager.swift @@ -480,8 +480,8 @@ public final class BluetoothManager: NSObject { case 0x02: publishCmd(["type": "startMakeCall", "data": frame]) case 0x03, 0x04: - releaseCodecs() - publishCallTranslation(["type": "stopCall", "data": frame]) + // 通话列表（带电话号码），不是挂断。拆解码器或报 stopCall 会把 + // 刚进的通话翻译踢出去，耳机听感像重启。状态只交给 BB 8E。 publishCmd(["type": "stopMakeCall", "data": frame]) case 0xD6, 0xE9: // 调速命令：翻译与 AI 两条链路共用（两回调互斥，激活的处理、另一条为 nil 忽略） diff --git a/apps/client/local_plugins/sparkchain/android/src/main/kotlin/com/yunqiinnovation/sparkchain/SparkChainPlugin.kt b/apps/client/local_plugins/sparkchain/android/src/main/kotlin/com/yunqiinnovation/sparkchain/SparkChainPlugin.kt index fd20bab1..5db6880f 100644 --- a/apps/client/local_plugins/sparkchain/android/src/main/kotlin/com/yunqiinnovation/sparkchain/SparkChainPlugin.kt +++ b/apps/client/local_plugins/sparkchain/android/src/main/kotlin/com/yunqiinnovation/sparkchain/SparkChainPlugin.kt @@ -13,8 +13,18 @@ import com.iflytek.sparkchain.core.common.ApiType import com.iflytek.sparkchain.core.its.ITS import com.iflytek.sparkchain.core.its.ITSCallbacks import com.iflytek.sparkchain.core.its.TransType +import com.iflytek.sparkchain.core.ast.AST +import com.iflytek.sparkchain.core.ast.ASTCallbacks import com.iflytek.sparkchain.core.rtasr.RTASR import com.iflytek.sparkchain.core.rtasr.RTASRCallbacks +import com.iflytek.sparkchain.core.LLM +import com.iflytek.sparkchain.core.LLMCallbacks +import com.iflytek.sparkchain.core.LLMConfig +import com.iflytek.sparkchain.core.LLMError +import com.iflytek.sparkchain.core.LLMEvent +import com.iflytek.sparkchain.core.LLMFactory +import com.iflytek.sparkchain.core.LLMResult +import com.iflytek.sparkchain.core.LLMTools import com.iflytek.sparkchain.core.tts.OnlineTTS import com.iflytek.sparkchain.core.tts.TTS import com.iflytek.sparkchain.core.tts.TTSCallbacks @@ -38,8 +48,10 @@ class SparkChainPlugin : FlutterPlugin, MethodChannel.MethodCallHandler, EventCh private var asr: ASR? = null private var rtasr: RTASR? = null + private var ast: AST? = null private var its: ITS? = null private var tts: OnlineTTS? = null + private var llm: LLM? = null private val asrCallbacks = object : AsrCallbacks { override fun onResult(result: ASR.ASRResult, usrTag: Any?) { @@ -106,6 +118,7 @@ class SparkChainPlugin : FlutterPlugin, MethodChannel.MethodCallHandler, EventCh "id" to (usrTag?.toString() ?: ""), "text" to dst, "sid" to (result.sid ?: ""), + "engine" to "gtrans", ), ) } @@ -192,24 +205,38 @@ class SparkChainPlugin : FlutterPlugin, MethodChannel.MethodCallHandler, EventCh val code = doSpeak(call) main.post { result.success(code) } } + "stopSpeak" -> cloud.execute { + tts?.stop() + main.post { result.success(0) } + } + "chat" -> cloud.execute { + val code = doChat(call) + main.post { result.success(code) } + } "startListen" -> audio.execute { val code = doStartListen(call) main.post { result.success(code) } } + "startSpliceListen" -> audio.execute { + val code = doStartSpliceListen(call) + main.post { result.success(code) } + } + "startAst" -> audio.execute { + val code = doStartAst(call) + main.post { result.success(code) } + } "write" -> audio.execute { val pcm = call.argument("pcm") val code = when { - rtasr != null && pcm != null -> rtasr!!.write(pcm) - asr != null && pcm != null -> asr!!.write(pcm) + ast != null && pcm != null -> writeFrames(pcm) { ast!!.write(it) } + rtasr != null && pcm != null -> writeFrames(pcm) { rtasr!!.write(it) } + asr != null && pcm != null -> writeFrames(pcm) { asr!!.write(it) } else -> -1 } main.post { result.success(code) } } "stopListen" -> audio.execute { - rtasr?.stop(false) - asr?.stop(false) - rtasr = null - asr = null + releaseListeners(false) main.post { result.success(0) } } "uninit" -> cloud.execute { @@ -283,6 +310,13 @@ class SparkChainPlugin : FlutterPlugin, MethodChannel.MethodCallHandler, EventCh val id = call.argument("id") ?: "" val lang = call.argument("lang") ?: "zh" val voice = call.argument("voice") + if (call.argument("msft") == true) { + val engine = OnlineTTS(RegionType.MSFT_TYPE, voice ?: "en-US-AriaNeural") + engine.aue("raw") + engine.registerCallbacks(ttsCallbacks) + tts = engine + return engine.aRun(text, id) + } val (region, vcn) = when (lang) { "ga" -> RegionType.MSFT_TYPE to (voice ?: "ga-IE-OrlaNeural") "en" -> null to (voice ?: "x_Catherine") @@ -295,14 +329,126 @@ class SparkChainPlugin : FlutterPlugin, MethodChannel.MethodCallHandler, EventCh return engine.aRun(text, id) } - private fun doStartListen(call: MethodCall): Int { - if (!ready) return -1 - rtasr?.stop(true) - asr?.stop(true) + private fun releaseListeners(immediate: Boolean) { + rtasr?.stop(immediate) + asr?.stop(immediate) + ast?.stop(immediate) rtasr = null asr = null + ast = null + } + + private fun writeFrames(pcm: ByteArray, write: (ByteArray) -> Int): Int { + val frame = 1280 + var offset = 0 + while (offset < pcm.size) { + val end = minOf(offset + frame, pcm.size) + val code = write(pcm.copyOfRange(offset, end)) + if (code != 0) return code + offset = end + } + return 0 + } + + private fun doStartSpliceListen(call: MethodCall): Int { + if (!ready) return -1 + releaseListeners(true) + val locale = call.argument("locale") ?: "en-US" + val id = call.argument("id") ?: "" + val engine = RTASR(null, RegionType.MSFT_TYPE) + engine.registerCallbacks(rtasrCallbacks) + engine.lang("msft") + engine.OstAccent("mandarin") + engine.OstDomain("ist_mul_msft") + engine.OstEncoding("raw") + engine.OstDwa("wpgs") + engine.setParams("language_name", locale) + rtasr = engine + return engine.start(id) + } + + private val astCallbacks = object : ASTCallbacks { + override fun onResult(result: AST.ASTResult, usrTag: Any?) { + val kind = (result.resType ?: "").lowercase() + val event = mutableMapOf( + "kind" to "ast", + "id" to (usrTag?.toString() ?: ""), + "resType" to kind, + "status" to result.status, + ) + when (kind) { + "asr" -> { + event["text"] = result.data ?: "" + event["final"] = result.status == 2 || result.isEnd == 1 + } + "trans" -> { + val trans = result.transResult + event["text"] = trans?.src ?: "" + event["dst"] = trans?.dst ?: "" + val flag = (trans?.isFinal ?: "").lowercase() + event["final"] = (trans?.isEnd == true) || flag == "1" || flag == "true" + } + "tts" -> { + val audio = result.ttsResult?.audio ?: ByteArray(0) + event["pcm"] = audio + event["final"] = result.ttsResult?.status == 2 + } + } + emit(event) + } + + override fun onError(error: AST.ASTError, usrTag: Any?) { + emit( + mapOf( + "kind" to "error", + "where" to "ast", + "id" to (usrTag?.toString() ?: ""), + "code" to (error.code?.toIntOrNull() ?: 0), + "message" to (error.desc ?: ""), + "sid" to (error.sid ?: ""), + ), + ) + } + } + + private fun doStartAst(call: MethodCall): Int { + if (!ready) return -1 + releaseListeners(true) + val lang = call.argument("lang") ?: "en" + val target = call.argument("targetLang") ?: "en" + val speaker = call.argument("speaker") ?: "JennyNeural" + val id = call.argument("id") ?: "" + val engine = AST(RegionType.MSFT_TYPE) + engine.registerCallbacks(astCallbacks) + val astApp = call.argument("astAppId").orEmpty() + val astKey = call.argument("astApiKey").orEmpty() + val astSecret = call.argument("astApiSecret").orEmpty() + engine.setParams("ast_appid", astApp) + engine.setParams("ast_apikey", astKey) + engine.setParams("ast_apisecret", astSecret) + engine.setParams("target_lang", target) + engine.setParams("use_tts", true) + engine.setParams("tts_target_lang", target) + engine.setParams("tts_speaker", speaker) + ast = engine + return engine.start(lang, "pcm_s16le", 16000, id) + } + + private fun doStartListen(call: MethodCall): Int { + if (!ready) return -1 + releaseListeners(true) val lang = call.argument("lang") ?: "zh" val id = call.argument("id") ?: "" + if (call.argument("msft") == true) { + val locale = call.argument("locale") ?: lang + val engine = RTASR(null, RegionType.MSFT_TYPE) + engine.registerCallbacks(rtasrCallbacks) + engine.lang(locale) + engine.setParams("language_name", locale) + engine.OstEncoding("raw") + rtasr = engine + return engine.start(id) + } if (lang == "ga") { val engine = RTASR(null, RegionType.MSFT_TYPE) engine.registerCallbacks(rtasrCallbacks) @@ -327,6 +473,61 @@ class SparkChainPlugin : FlutterPlugin, MethodChannel.MethodCallHandler, EventCh return engine.start(attr, id) } + private val llmCallbacks = object : LLMCallbacks { + override fun onLLMResult(result: LLMResult, usrContext: Any?) { + emit( + mapOf( + "kind" to "llm", + "id" to (usrContext?.toString() ?: ""), + "text" to (result.content ?: ""), + "functionCall" to (result.functionCall ?: ""), + "final" to (result.status == 2), + ) + ) + } + + override fun onLLMEvent(event: LLMEvent, usrContext: Any?) {} + + override fun onLLMError(error: LLMError, usrContext: Any?) { + emit( + mapOf( + "kind" to "error", + "where" to "llm", + "id" to (usrContext?.toString() ?: ""), + "code" to error.errCode, + "message" to (error.errMsg ?: ""), + "sid" to (error.sid ?: ""), + ) + ) + } + } + + private fun doChat(call: MethodCall): Int { + if (!ready) return -1 + val id = call.argument("id") ?: "" + val query = call.argument("query") ?: return -2 + val system = call.argument("system") ?: "" + val tools = call.argument("tools") ?: "" + val fresh = call.argument("fresh") ?: false + if (llm == null || fresh) { + val config = LLMConfig.builder().domain("openai").param("model", "gpt-5.4") + if (tools.isNotEmpty()) config.tools(tools) + llm = LLMFactory.textGeneration(config) + llm?.registerLLMCallbacks(llmCallbacks) + } + if (fresh) llm?.clearHistory() + if (system.isNotEmpty()) llm?.addSystemPrompt(system) + val tool = if (tools.isEmpty()) { + null + } else { + LLMTools().apply { + type = "function" + description = tools + } + } + return if (tool != null) llm!!.arun(query, tool, id) else llm!!.arun(query, id) + } + private fun emit(event: Map) { val current = sink ?: return main.post { current.success(event) } diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/Sources/sparkchain/SparkChainPlugin.swift b/apps/client/local_plugins/sparkchain/ios/sparkchain/Sources/sparkchain/SparkChainPlugin.swift index a51e2a23..31343182 100644 --- a/apps/client/local_plugins/sparkchain/ios/sparkchain/Sources/sparkchain/SparkChainPlugin.swift +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/Sources/sparkchain/SparkChainPlugin.swift @@ -1,11 +1,13 @@ +import CryptoKit import Flutter +import Foundation import SparkChain import SparkChainGlue /// 回调对象必须由插件强引用。SDK 里的 delegate 是 weak，局部变量会在 start 返回后被释放。 @objc(SparkChainPlugin) public final class SparkChainPlugin: NSObject, FlutterPlugin, FlutterStreamHandler, - ASRCallbacks, RTASRCallback, ITSCallbacks, TTSCallbacks { + ASRCallbacks, RTASRCallback, ITSCallbacks, TTSCallbacks, LLMCallback, ASTDelegate { private let channelName = "sparkchain" private var events: FlutterEventSink? @@ -13,12 +15,27 @@ public final class SparkChainPlugin: NSObject, FlutterPlugin, FlutterStreamHandl private var asr: ASR? private var rtasr: RTASR? + /// start() 还没返回时服务端已经回了会话号。这时再等下去会被判成 18801。 + private var spliceConnected = false + private var ast: AST? private var its: ITS? private var tts: OnlineTTS? + private var llm: LLM? + + private var appId = "" + private var apiKey = "" + private var apiSecret = "" + private var iat: ClassicIat? + private var classicTts: ClassicTts? private let transOrder: [TransType] = [GOOGLETRANS, NIUTRANS, ITRANS_SG, ITRANS] + private var itsEngine = "gtrans" private var transIndex = 0 private var pendingIts: [String: String] = [:] + /// 某一档翻译迟迟不回调时，换下一档。挂住不换的话，后面三档永远走不到。 + private var itsGen = 0 + /// SDK 的 start 会自己去连服务器。堵在主线程上，页面转场整段画不出来。 + private let work = DispatchQueue(label: "sparkchain.session") public static func register(with registrar: FlutterPluginRegistrar) { let instance = SparkChainPlugin() @@ -40,36 +57,56 @@ public final class SparkChainPlugin: NSObject, FlutterPlugin, FlutterStreamHandl public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { let args = call.arguments as? [String: Any] ?? [:] - switch call.method { - case "init": - result(doInit(args)) - case "translate": - result(doTranslate(args)) - case "speak": - result(doSpeak(args)) - case "startListen": - result(doStartListen(args)) - case "write": - let data = args["pcm"] as? FlutterStandardTypedData - if let rtasr, let data { - result(rtasr.write(data.data)) - } else if let asr, let data { - result(asr.write(data.data)) - } else { - result(-1) + work.async { [weak self] in + guard let self else { + DispatchQueue.main.async { result(-1) } + return } - case "stopListen": - rtasr?.stop(false) - asr?.stop(false) - rtasr = nil - asr = nil - result(0) - case "uninit": - let code = ready ? SparkChainGlueUnInit() : 0 - ready = false - result(code) - default: - result(FlutterMethodNotImplemented) + let value: Any? + switch call.method { + case "init": + value = self.doInit(args) + case "translate": + value = self.doTranslate(args) + case "speak": + value = self.doSpeak(args) + case "stopSpeak": + self.tts?.stop() + self.classicTts?.cancel() + value = 0 + case "chat": + value = self.doChat(args) + case "startListen": + value = self.doStartListen(args) + case "startSpliceListen": + value = self.doStartSpliceListen(args) + case "startAst": + value = self.doStartAst(args) + case "write": + let data = args["pcm"] as? FlutterStandardTypedData + if let ast = self.ast, let data { + value = self.writeFrames(data.data) { Int(ast.write($0)) } + } else if let iat = self.iat, let data { + value = self.writeFrames(data.data) { iat.write($0) } + } else if let rtasr = self.rtasr, let data { + value = self.writeFrames(data.data) { Int(rtasr.write($0)) } + } else if let asr = self.asr, let data { + value = self.writeFrames(data.data) { Int(asr.write($0)) } + } else { + value = -1 + } + case "stopListen": + self.releaseListeners(immediate: false) + value = 0 + case "uninit": + let code = self.ready ? SparkChainGlueUnInit() : 0 + self.ready = false + value = code + default: + DispatchQueue.main.async { result(FlutterMethodNotImplemented) } + return + } + DispatchQueue.main.async { result(value) } } } @@ -79,6 +116,9 @@ public final class SparkChainPlugin: NSObject, FlutterPlugin, FlutterStreamHandl let apiSecret = args["apiSecret"] as? String else { return -2 } + self.appId = appId + self.apiKey = apiKey + self.apiSecret = apiSecret if ready { SparkChainGlueUnInit() ready = false @@ -102,13 +142,37 @@ public final class SparkChainPlugin: NSObject, FlutterPlugin, FlutterStreamHandl } } + /// 同声拼接的文本翻译只打 SDK 的 gtrans。iOS 枚举名是 GOOGLETRANS，和安卓 GTRANS 是同一个值。 + /// 不再轮询国内 ITRANS，那个节点会把一句拖住，失败后也不改走原来的微软翻译。 private func doTranslate(_ args: [String: Any]) -> Int { - guard ready, let text = args["text"] as? String else { return ready ? -2 : -1 } + guard !appId.isEmpty, !apiKey.isEmpty, !apiSecret.isEmpty else { return -1 } + guard let text = args["text"] as? String, !text.isEmpty else { return -2 } let id = args["id"] as? String ?? "" - let from = args["from"] as? String ?? "zh" - let to = args["to"] as? String ?? "en" - pendingIts[id] = "\(from)|\(to)|\(text)" - return runIts(id: id, from: from, to: to, text: text) + let from = gtransCode(args["from"] as? String ?? "zh-CN") + let to = gtransCode(args["to"] as? String ?? "en") + itsEngine = "gtrans" + itsGen += 1 + let engine = ITS(formLanguage: from, toLanguage: to, type: GOOGLETRANS) + engine.callbacks = self + its = engine + return Int(engine.arun(text, userInfo: id)) + } + + private func gtransCode(_ code: String) -> String { + switch code { + case "zh", "cn": + return "zh-CN" + default: + return code + } + } + + private func transName(_ type: TransType) -> String { + if type == GOOGLETRANS { return "GOOGLETRANS" } + if type == NIUTRANS { return "NIUTRANS" } + if type == ITRANS_SG { return "ITRANS_SG" } + if type == ITRANS { return "ITRANS" } + return "unknown" } private func runIts(id: String, from: String, to: String, text: String) -> Int { @@ -117,57 +181,289 @@ public final class SparkChainPlugin: NSObject, FlutterPlugin, FlutterStreamHandl let engine = ITS(formLanguage: itsCode(from, smallTable: small), toLanguage: itsCode(to, smallTable: small), type: type) engine.callbacks = self its = engine + itsGen += 1 + let gen = itsGen + DispatchQueue.main.asyncAfter(deadline: .now() + 4) { [weak self] in + guard let self, self.itsGen == gen else { return } + self.advanceIts(id: id) + } return Int(engine.arun(text, userInfo: id)) } + /// 当前这一档失败或超时，换下一档。四档都没有结果才把错误交回 Dart。 + private func advanceIts(id: String) { + itsGen += 1 + guard transIndex + 1 < transOrder.count, let packed = pendingIts[id] else { + emit([ + "kind": "error", + "where": "its", + "id": id, + "code": -1, + "message": "its exhausted", + "sid": "", + ]) + return + } + transIndex += 1 + let parts = packed.split(separator: "|", maxSplits: 2).map(String.init) + guard parts.count == 3 else { return } + let code = runIts(id: id, from: parts[0], to: parts[1], text: parts[2]) + if code != 0 { + advanceIts(id: id) + } + } + private func doSpeak(_ args: [String: Any]) -> Int { - guard ready, let text = args["text"] as? String else { return ready ? -2 : -1 } + guard !appId.isEmpty, !apiKey.isEmpty, !apiSecret.isEmpty else { return -1 } + guard let text = args["text"] as? String, !text.isEmpty else { return -2 } let id = args["id"] as? String ?? "" let lang = args["lang"] as? String ?? "zh" - let voice = args["voice"] as? String - let engine: OnlineTTS - if lang == "ga" { - engine = OnlineTTS(region: RegionMSFT, vcn: voice ?? "ga-IE-OrlaNeural") - } else if lang == "en" { - engine = OnlineTTS(vcn: voice ?? "x_Catherine") - } else { - engine = OnlineTTS(vcn: voice ?? "x_xiaoyan") - } - engine.aue = "raw" - engine.cbs = self - tts = engine - return Int(engine.arun(text, userInfo: id)) + if args["msft"] as? Bool == true || lang == "ga" { + classicTts?.cancel() + let voice = args["voice"] as? String + ?? (lang == "ga" ? "ga-IE-OrlaNeural" : "en-US-AriaNeural") + let speaker = OnlineTTS(region: RegionMSFT, vcn: voice) + speaker.aue = "raw" + speaker.cbs = self + tts = speaker + return Int(speaker.arun(text, userInfo: id)) + } + classicTts?.cancel() + let speaker = ClassicTts( + appId: appId, + apiKey: apiKey, + apiSecret: apiSecret, + emit: { [weak self] event in self?.emit(event) } + ) + classicTts = speaker + return speaker.start(id: id, lang: lang, text: text) } - private func doStartListen(_ args: [String: Any]) -> Int { + private func doChat(_ args: [String: Any]) -> Int { guard ready else { return -1 } - rtasr?.stop(true) - asr?.stop(true) + let id = args["id"] as? String ?? "" + let query = args["query"] as? String ?? "" + let system = args["system"] as? String ?? "" + let tools = args["tools"] as? String ?? "" + let fresh = args["fresh"] as? Bool ?? false + if llm == nil || fresh { + let config = LLMConfig() + _ = config.domain("openai") + _ = config.addString("model", "gpt-5.4") + if !tools.isEmpty { _ = config.tools(tools) } + llm = LLM(config: config, callback: self) + } + if fresh { llm?.clearHistory() } + if !system.isEmpty { _ = llm?.addSystemPrompt(system) } + if !tools.isEmpty { + let tool = LLMTools(type: "function", description: tools) + return Int(llm?.arun(query, tools: tool, usrContext: id) ?? -1) + } + return Int(llm?.arun(query, usrContext: id) ?? -1) + } + + public func llm(_ llm: LLM, on result: LLMResult?, usrContext: Any?) { + guard let result else { return } + emit([ + "kind": "llm", + "id": (usrContext as? String) ?? "", + "text": result.content ?? "", + "functionCall": result.functionCall, + "final": result.status == 2, + ]) + } + + public func llm(_ llm: LLM, on event: LLMEvent?, usrContext: Any?) {} + + public func llm(_ llm: LLM, onError error: LLMError?, usrContext: Any?) { + emit([ + "kind": "error", + "where": "llm", + "id": (usrContext as? String) ?? "", + "code": Int(error?.errCode ?? 0), + "message": error?.errMsg ?? "", + "sid": error?.sid ?? "", + ]) + } + + /// 停掉上一条听写或同传。同声结束用 immediate=false，对应 AST.stop(NO)。 + private func releaseListeners(immediate: Bool) { + iat?.stop() + iat = nil + rtasr?.stop(immediate) + asr?.stop(immediate) + ast?.stop(immediate) rtasr = nil asr = nil - let lang = args["lang"] as? String ?? "zh" + ast = nil + } + + /// 官方实时转写和同传 demo 都按 40ms、1280 字节写入。 + private func writeFrames(_ data: Data, write: (Data) -> Int) -> Int { + if data.isEmpty { return -1 } + let frame = 1280 + var offset = 0 + while offset < data.count { + let end = min(offset + frame, data.count) + let code = write(data.subdata(in: offset.. Int { + guard !appId.isEmpty, !apiKey.isEmpty, !apiSecret.isEmpty else { return -1 } + releaseListeners(immediate: true) + let locale = args["locale"] as? String ?? "en-US" let id = args["id"] as? String ?? "" - if lang == "ga" { + // 这台应用只开通了 rtasr_msft，没有 ast_msft。RegionSG 握手是 401。 + // 口音、领域的属性不会进握手参数，要走 addStringParam。 + // 不要再写键 language：它和 lang 撞在一起时，start 会空等 10 秒返回 18801。 + spliceConnected = false + let engine = RTASR(region: RegionMSFT) + engine.delegate = self + engine.lang = "msft" + engine.ostDwa = "wpgs" + engine.addStringParam("accent", value: "mandarin") + engine.addStringParam("domain", value: "ist_mul_msft") + engine.addStringParam("language_name", value: locale) + rtasr = engine + let code = Int(engine.start(withUserInfo: id)) + if code == 18801 && spliceConnected { + return 0 + } + if code != 0 { + engine.stop(true) + rtasr = nil + } + return code + } + + /// 微软同传。一条会话同时出识别、译文和合成音频。 + private func doStartAst(_ args: [String: Any]) -> Int { + guard !appId.isEmpty, !apiKey.isEmpty, !apiSecret.isEmpty else { return -1 } + releaseListeners(immediate: true) + let lang = args["lang"] as? String ?? "en" + let target = args["targetLang"] as? String ?? "en" + let speaker = args["speaker"] as? String ?? "JennyNeural" + let id = args["id"] as? String ?? "" + let engine = AST(region: RegionMSFT, delegate: self) + // 全局 apiType 是新加坡时，SDK 不看机房，启动前先读这三项。 + // 缺任意一项 mInit 返回 18304，start 再改写成 18302。 + // 这三项是同传自己的 accessKeyId，不要用 SparkChain 应用密钥顶上。 + let astApp = (args["astAppId"] as? String).flatMap { $0.isEmpty ? nil : $0 } ?? appId + let astKey = (args["astApiKey"] as? String).flatMap { $0.isEmpty ? nil : $0 } ?? apiKey + let astSecret = (args["astApiSecret"] as? String).flatMap { $0.isEmpty ? nil : $0 } ?? apiSecret + engine.addStringParam("ast_appid", value: astApp) + engine.addStringParam("ast_apikey", value: astKey) + engine.addStringParam("ast_apisecret", value: astSecret) + engine.addStringParam("target_lang", value: target) + engine.addBoolParam("use_tts", value: true) + engine.addStringParam("tts_target_lang", value: target) + engine.addStringParam("tts_speaker", value: speaker) + ast = engine + let code = Int(engine.start(lang, audioEncoding: "pcm_s16le", sampleRate: 16000, userInfo: id)) + if code != 0 { + engine.stop(true) + ast = nil + } + return code + } + + public func ast(_ ast: AST, on result: ASTResult, userInfo: Any?) { + let kind = result.resType.lowercased() + var event: [String: Any] = [ + "kind": "ast", + "id": (userInfo as? String) ?? "", + "resType": kind, + "status": result.status, + ] + if kind == "asr" { + event["text"] = result.data + event["final"] = result.status == 2 || result.isEnd == 1 + } else if kind == "trans" { + let trans = result.transResult + event["text"] = trans.src + event["dst"] = trans.dst + let flag = trans.isFinal.lowercased() + event["final"] = trans.end || flag == "1" || flag == "true" + } else if kind == "tts" { + event["pcm"] = FlutterStandardTypedData(bytes: result.ttsResult.audio) + event["final"] = result.ttsResult.status == 2 + } + emit(event) + } + + public func ast(_ ast: AST, onError error: ASTError, userInfo: Any?) { + emit([ + "kind": "error", + "where": "ast", + "id": (userInfo as? String) ?? "", + "code": Int(error.code) ?? 0, + "message": error.desc, + "sid": error.sid, + ]) + } + + private func doStartListen(_ args: [String: Any]) -> Int { + // 不要调用 SDK 的 ASR.start。新加坡听写会 401，SDK 在关 WebSocket 的线程里 SIGSEGV。 + guard !appId.isEmpty, !apiKey.isEmpty, !apiSecret.isEmpty else { return -1 } + if args["msft"] as? Bool == true { + releaseListeners(immediate: true) + let locale = args["locale"] as? String ?? "en-US" + let id = args["id"] as? String ?? "" + let engine = RTASR(region: RegionMSFT) + engine.delegate = self + engine.lang = locale + engine.ostEncoding = "raw" + engine.addStringParam("language_name", value: locale) + rtasr = engine + let code = Int(engine.start(withUserInfo: id)) + if code != 0 { + engine.stop(true) + rtasr = nil + } + return code + } + if (args["lang"] as? String ?? "zh") == "ga" { + releaseListeners(immediate: true) + let id = args["id"] as? String ?? "" let engine = RTASR(region: RegionMSFT) engine.delegate = self + engine.ostEncoding = "raw" engine.addStringParam("language_name", value: "ga-IE") rtasr = engine - return Int(engine.start(withUserInfo: id)) - } - let engine = ASR() - engine.cbs = self - let code = lang == "en" ? "en_us" : "zh_cn" - engine.language(code) - engine.domain("iat") - engine.accent("mandarin") - if lang == "zh" { engine.dwa("wpgs") } - let attr = AudioAttributes() - attr.sampleRate = 16000 - attr.encoding = "raw" - attr.channels = 1 - attr.bitdepth = 16 - asr = engine - return Int(engine.start(attr, usrTag: id)) + let code = Int(engine.start(withUserInfo: id)) + if code != 0 { + engine.stop(true) + rtasr = nil + } + return code + } + releaseListeners(immediate: true) + let lang = args["lang"] as? String ?? "zh" + let id = args["id"] as? String ?? "" + if lang != "zh" && lang != "en" { + emit([ + "kind": "error", + "where": "asr", + "id": id, + "code": -1, + "message": "lang \(lang) unsupported", + "sid": "", + ]) + return -1 + } + let session = ClassicIat( + appId: appId, + apiKey: apiKey, + apiSecret: apiSecret, + emit: { [weak self] event in self?.emit(event) } + ) + iat = session + return session.start(id: id, lang: lang) } private func emit(_ event: [String: Any]) { @@ -197,11 +493,13 @@ public final class SparkChainPlugin: NSObject, FlutterPlugin, FlutterStreamHandl } public func onResult(_ result: RTASRResult, useInfo userInfo: Any?) { + spliceConnected = true let status = result.status emit([ "kind": "asr", "id": (userInfo as? String) ?? "", "text": result.data, + "status": status, "final": status == 2 || status == 3, "sid": result.sid, ]) @@ -219,24 +517,23 @@ public final class SparkChainPlugin: NSObject, FlutterPlugin, FlutterStreamHandl } public func its(_ its: ITS, on result: ITSResult, userInfo: Any) { + itsGen += 1 emit([ "kind": "its", "id": (userInfo as? String) ?? "", "text": result.trans.dst, "sid": result.sid, + "engine": itsEngine, ]) } public func its(_ its: ITS, onError error: ITSError, userInfo: Any) { let id = (userInfo as? String) ?? "" - if transIndex + 1 < transOrder.count, let packed = pendingIts[id] { - transIndex += 1 - let parts = packed.split(separator: "|", maxSplits: 2).map(String.init) - if parts.count == 3 { - _ = runIts(id: id, from: parts[0], to: parts[1], text: parts[2]) - return - } + if transIndex + 1 < transOrder.count, pendingIts[id] != nil { + advanceIts(id: id) + return } + itsGen += 1 emit([ "kind": "error", "where": "its", @@ -268,3 +565,396 @@ public final class SparkChainPlugin: NSObject, FlutterPlugin, FlutterStreamHandl ]) } } + +/// 国内开放平台听写。密钥能在这里握手；海外 SDK 的听写失败关闭会把进程打崩。 +private final class ClassicIat { + private let appId: String + private let apiKey: String + private let apiSecret: String + private let emit: ([String: Any]) -> Void + private let lock = NSLock() + private var task: URLSessionWebSocketTask? + private var id = "" + private var lang = "zh" + private var alive = false + private var opened = false + private var sentHead = false + private var buf = Data() + private var pieces: [Int: String] = [:] + private var failures = 0 + + init(appId: String, apiKey: String, apiSecret: String, emit: @escaping ([String: Any]) -> Void) { + self.appId = appId + self.apiKey = apiKey + self.apiSecret = apiSecret + self.emit = emit + } + + func start(id: String, lang: String) -> Int { + self.id = id + self.lang = lang + alive = true + failures = 0 + return open() ? 0 : -1 + } + + func write(_ pcm: Data) -> Int { + lock.lock() + buf.append(pcm) + let task = self.task + let opened = self.opened + lock.unlock() + if opened, let task { flush(task) } + return 0 + } + + func stop() { + lock.lock() + alive = false + let task = self.task + let rest = buf + buf.removeAll() + let opened = self.opened + let sentHead = self.sentHead + self.task = nil + lock.unlock() + guard let task, opened else { + task?.cancel(with: .goingAway, reason: nil) + return + } + if !sentHead { + send(status: 0, audio: rest, task: task) + } + send(status: 2, audio: sentHead ? rest : Data(), task: task) { [weak task] in + task?.cancel(with: .goingAway, reason: nil) + } + } + + private func open() -> Bool { + guard let url = XfyunSign.url(host: "iat-api.xfyun.cn", path: "/v2/iat", apiKey: apiKey, apiSecret: apiSecret) else { + return false + } + let task = URLSession.shared.webSocketTask(with: url) + lock.lock() + let previous = self.task + self.task = task + opened = false + sentHead = false + pieces.removeAll() + lock.unlock() + if let previous, previous !== task { + previous.cancel(with: .goingAway, reason: nil) + } + task.resume() + receive(task) + lock.lock() + opened = true + lock.unlock() + flush(task) + return true + } + + private func flush(_ task: URLSessionWebSocketTask) { + while true { + lock.lock() + guard buf.count >= 1280, self.task === task else { + lock.unlock() + return + } + let chunk = buf.prefix(1280) + buf.removeFirst(1280) + let status = sentHead ? 1 : 0 + sentHead = true + lock.unlock() + send(status: status, audio: Data(chunk), task: task) + } + } + + private func send(status: Int, audio: Data, task: URLSessionWebSocketTask, done: (() -> Void)? = nil) { + var body: [String: Any] = [ + "data": [ + "status": status, + "format": "audio/L16;rate=16000", + "encoding": "raw", + "audio": audio.base64EncodedString(), + ], + ] + if status == 0 { + body["common"] = ["app_id": appId] + var business: [String: Any] = [ + "language": lang == "en" ? "en_us" : "zh_cn", + "domain": "iat", + "accent": "mandarin", + "vad_eos": 2000, + ] + if lang == "zh" { business["dwa"] = "wpgs" } + body["business"] = business + } + guard let data = try? JSONSerialization.data(withJSONObject: body), + let text = String(data: data, encoding: .utf8) else { + done?() + return + } + task.send(.string(text)) { _ in done?() } + } + + private func receive(_ task: URLSessionWebSocketTask) { + task.receive { [weak self] result in + guard let self else { return } + switch result { + case .success(let message): + let text: String + switch message { + case .string(let value): text = value + case .data(let value): text = String(data: value, encoding: .utf8) ?? "" + @unknown default: text = "" + } + let ended = self.onMessage(text, task: task) + if !ended { self.receive(task) } + case .failure: + self.onSocketClosed(task) + } + } + } + + /// 服务端宣告本句结束时返回 true，调用方不要再在这条已经结束的连接上收包。 + private func onMessage(_ text: String, task: URLSessionWebSocketTask) -> Bool { + guard let data = text.data(using: .utf8), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + return false + } + let code = json["code"] as? Int ?? 0 + let sid = (json["sid"] as? String) ?? "" + if code != 0 { + failures += 1 + emit([ + "kind": "error", + "where": "asr", + "id": id, + "code": code, + "message": (json["message"] as? String) ?? "", + "sid": sid, + ]) + alive = false + return true + } + failures = 0 + let payload = json["data"] as? [String: Any] + let status = payload?["status"] as? Int ?? 0 + if let result = payload?["result"] as? [String: Any] { + let spoken = apply(result) + if !spoken.isEmpty { + emit([ + "kind": "asr", + "id": id, + "text": spoken, + "final": status == 2, + "sid": sid, + ]) + } + } else if status == 2 { + let spoken = pieces.keys.sorted().map { pieces[$0] ?? "" }.joined() + if !spoken.isEmpty { + emit([ + "kind": "asr", + "id": id, + "text": spoken, + "final": true, + "sid": sid, + ]) + } + } + if status == 2 { + reopen(task) + return true + } + return false + } + + private func onSocketClosed(_ task: URLSessionWebSocketTask) { + lock.lock() + let current = self.task === task + let keep = alive + lock.unlock() + guard current, keep else { return } + failures += 1 + if failures >= 3 { + alive = false + emit([ + "kind": "error", + "where": "asr", + "id": id, + "code": -1, + "message": "iat closed", + "sid": "", + ]) + return + } + reopen(task) + } + + private func reopen(_ task: URLSessionWebSocketTask) { + lock.lock() + let current = self.task === task && alive + lock.unlock() + guard current else { return } + _ = open() + } + + private func apply(_ result: [String: Any]) -> String { + let sn = result["sn"] as? Int ?? pieces.count + 1 + var word = "" + if let ws = result["ws"] as? [[String: Any]] { + for item in ws { + guard let cw = item["cw"] as? [[String: Any]] else { continue } + for one in cw { + word += (one["w"] as? String) ?? "" + } + } + } + if (result["pgs"] as? String) == "rpl", + let rg = result["rg"] as? [Int], rg.count == 2, rg[0] <= rg[1] { + for i in rg[0]...rg[1] { pieces.removeValue(forKey: i) } + } + pieces[sn] = word + return pieces.keys.sorted().map { pieces[$0] ?? "" }.joined() + } +} + +/// 国内开放平台合成。海外 SDK 的合成连不上时同样会在关连接时崩溃。 +private final class ClassicTts { + private let appId: String + private let apiKey: String + private let apiSecret: String + private let emit: ([String: Any]) -> Void + private var task: URLSessionWebSocketTask? + private var id = "" + + init(appId: String, apiKey: String, apiSecret: String, emit: @escaping ([String: Any]) -> Void) { + self.appId = appId + self.apiKey = apiKey + self.apiSecret = apiSecret + self.emit = emit + } + + func start(id: String, lang: String, text: String) -> Int { + self.id = id + guard let url = XfyunSign.url(host: "tts-api.xfyun.cn", path: "/v2/tts", apiKey: apiKey, apiSecret: apiSecret) else { + return -1 + } + let task = URLSession.shared.webSocketTask(with: url) + self.task = task + task.resume() + let body: [String: Any] = [ + "common": ["app_id": appId], + "business": [ + "aue": "raw", + "auf": "audio/L16;rate=16000", + "vcn": lang == "en" ? "x2_catherine" : "xiaoyan", + "tte": "UTF8", + "speed": 50, + ], + "data": [ + "status": 2, + "text": Data(text.utf8).base64EncodedString(), + ], + ] + guard let data = try? JSONSerialization.data(withJSONObject: body), + let payload = String(data: data, encoding: .utf8) else { + return -1 + } + task.send(.string(payload)) { [weak self] error in + if error != nil { self?.fail(code: -1, message: "tts send") } + } + receive(task) + return 0 + } + + func cancel() { + task?.cancel(with: .goingAway, reason: nil) + task = nil + } + + private func receive(_ task: URLSessionWebSocketTask) { + task.receive { [weak self] result in + guard let self else { return } + switch result { + case .success(let message): + let text: String + switch message { + case .string(let value): text = value + case .data(let value): text = String(data: value, encoding: .utf8) ?? "" + @unknown default: text = "" + } + if !self.onMessage(text) { self.receive(task) } + case .failure: + self.fail(code: -1, message: "tts closed") + } + } + } + + private func onMessage(_ text: String) -> Bool { + guard let data = text.data(using: .utf8), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + return false + } + let code = json["code"] as? Int ?? 0 + let sid = (json["sid"] as? String) ?? "" + if code != 0 { + fail(code: code, message: (json["message"] as? String) ?? "", sid: sid) + return true + } + let payload = json["data"] as? [String: Any] + let status = payload?["status"] as? Int ?? 1 + let audio = Data(base64Encoded: (payload?["audio"] as? String) ?? "") ?? Data() + emit([ + "kind": "tts", + "id": id, + "pcm": FlutterStandardTypedData(bytes: audio), + "final": status == 2, + "sid": sid, + ]) + if status == 2 { + task?.cancel(with: .normalClosure, reason: nil) + return true + } + return false + } + + private func fail(code: Int, message: String, sid: String = "") { + emit([ + "kind": "error", + "where": "tts", + "id": id, + "code": code, + "message": message, + "sid": sid, + ]) + } +} + +private enum XfyunSign { + static func url(host: String, path: String, apiKey: String, apiSecret: String) -> URL? { + let formatter = DateFormatter() + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.timeZone = TimeZone(secondsFromGMT: 0) + formatter.dateFormat = "EEE, dd MMM yyyy HH:mm:ss 'GMT'" + let date = formatter.string(from: Date()) + let origin = "host: \(host)\ndate: \(date)\nGET \(path) HTTP/1.1" + let key = SymmetricKey(data: Data(apiSecret.utf8)) + let mac = HMAC.authenticationCode(for: Data(origin.utf8), using: key) + let signature = Data(mac).base64EncodedString() + let raw = "api_key=\"\(apiKey)\", algorithm=\"hmac-sha256\", headers=\"host date request-line\", signature=\"\(signature)\"" + let authorization = Data(raw.utf8).base64EncodedString() + var components = URLComponents() + components.scheme = "wss" + components.host = host + components.path = path + components.queryItems = [ + URLQueryItem(name: "authorization", value: authorization), + URLQueryItem(name: "date", value: date), + URLQueryItem(name: "host", value: host), + ] + return components.url + } +} diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SparkChain.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SparkChain.h index ac80b1ea..03ce4592 100644 --- a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SparkChain.h +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SparkChain.h @@ -61,6 +61,11 @@ FOUNDATION_EXPORT const unsigned char SparkChainVersionString[]; #import #import +// 同声传译 +#import +#import +#import + diff --git a/apps/client/local_plugins/sparkchain/lib/sparkchain.dart b/apps/client/local_plugins/sparkchain/lib/sparkchain.dart index 02923c95..1788d58e 100644 --- a/apps/client/local_plugins/sparkchain/lib/sparkchain.dart +++ b/apps/client/local_plugins/sparkchain/lib/sparkchain.dart @@ -53,12 +53,36 @@ class SparkChain { required String lang, required String text, String? voice, + bool msft = false, }) async { final code = await _method.invokeMethod('speak', { 'id': id, 'lang': lang, 'text': text, if (voice != null) 'voice': voice, + 'msft': msft, + }); + return code ?? -1; + } + + static Future stopSpeak() async { + await _method.invokeMethod('stopSpeak'); + } + + /// 文本对话。结果从事件 `kind=llm` 回来，这里只返回启动码。 + static Future chat({ + required String id, + required String query, + required String system, + required String tools, + required bool fresh, + }) async { + final code = await _method.invokeMethod('chat', { + 'id': id, + 'query': query, + 'system': system, + 'tools': tools, + 'fresh': fresh, }); return code ?? -1; } @@ -75,6 +99,55 @@ class SparkChain { return code ?? -1; } + /// 微软区域实时转写。[locale] 是界面语言的区域码，例如 zh-CN。 + static Future startMsftListen({ + required String id, + required String locale, + }) async { + final code = await _method.invokeMethod('startListen', { + 'id': id, + 'lang': locale, + 'msft': true, + 'locale': locale, + }); + return code ?? -1; + } + + /// 同声拼接用的微软实时转写。lang / accent / domain 由原生写成固定值， + /// [locale] 只进 language_name。 + static Future startSpliceListen({ + required String id, + required String locale, + }) async { + final code = await _method.invokeMethod('startSpliceListen', { + 'id': id, + 'locale': locale, + }); + return code ?? -1; + } + + /// 微软同传。识别、翻译、合成在这一条会话里完成。SDK 不自己开麦。 + static Future startAst({ + required String id, + required String lang, + required String targetLang, + required String speaker, + required String astAppId, + required String astApiKey, + required String astApiSecret, + }) async { + final code = await _method.invokeMethod('startAst', { + 'id': id, + 'lang': lang, + 'targetLang': targetLang, + 'speaker': speaker, + 'astAppId': astAppId, + 'astApiKey': astApiKey, + 'astApiSecret': astApiSecret, + }); + return code ?? -1; + } + static Future write(Uint8List pcm) async { final code = await _method.invokeMethod('write', {'pcm': pcm}); return code ?? -1; diff --git a/apps/client/pubspec.yaml b/apps/client/pubspec.yaml index 45c421be..5028427f 100644 --- a/apps/client/pubspec.yaml +++ b/apps/client/pubspec.yaml @@ -1,7 +1,7 @@ name: eaimar description: "Eaimar - AI Voice Assistant." publish_to: "none" -version: 1.0.3+48 +version: 1.0.3+94 environment: sdk: ">=3.3.0 <4.0.0" diff --git a/apps/client/test/call_extended_languages_test.dart b/apps/client/test/call_extended_languages_test.dart index 71dc9a96..84e40297 100644 --- a/apps/client/test/call_extended_languages_test.dart +++ b/apps/client/test/call_extended_languages_test.dart @@ -35,12 +35,24 @@ void main() { expect(languages.getChineseNameByShortCode('yue'), '粤语'); }); - test('原有通话档位不变，扩展语种整对走微软', () { - expect(languages.findBestMatchingProvider('zh', 'en')?.provider, 'alibaba35'); - expect(languages.findBestMatchingProvider('zh', 'el')?.provider, 'alibaba'); + test('界面上的语言对都走微软端到端', () { + expect(languages.findBestMatchingProvider('zh', 'en')?.provider, 'azure'); + expect(languages.findBestMatchingProvider('zh', 'el')?.provider, 'azure'); expect(languages.findBestMatchingProvider('zh', 'ro')?.provider, 'azure'); + expect(languages.getLanguageByChineseName('挪威语')?.shortCode, 'nb'); - for (final code in ['fil', 'he', 'is', 'yue', 'cy', 'jv']) { + for (final code in [ + 'fil', + 'he', + 'is', + 'yue', + 'cy', + 'jv', + 'nb', + 'zh-Hant', + 'zh-Hant-hk', + 'sw', + ]) { expect(languages.findBestMatchingProvider('zh', code)?.provider, 'azure', reason: 'zh -> $code'); expect(languages.findBestMatchingProvider(code, 'en')?.provider, 'azure', diff --git a/apps/client/test/devices/bes_protocol_test.dart b/apps/client/test/devices/bes_protocol_test.dart index 0908e22a..13b21345 100644 --- a/apps/client/test/devices/bes_protocol_test.dart +++ b/apps/client/test/devices/bes_protocol_test.dart @@ -107,5 +107,25 @@ void main() { expect(BesCmd.switchesMode(0x06), isFalse); expect(BesCmd.switchesMode(0x6D), isFalse); }); + + test('BB 03 和 BB 04 是通话列表，BB 8E 才是通话状态', () { + expect( + BesCmd.isCallListReport([ + 0xBB, 0x03, 0x13, 0x0F, 0x01, 0x2B, 0x38, 0x36, + ]), + isTrue, + ); + expect(BesCmd.isCallListReport([0xBB, 0x04, 0x03]), isTrue); + expect(BesCmd.isCallListReport([0xBB, 0x8E, 0x07, 0x00, 0x02, 0x01, 0x03]), + isFalse); + expect( + BesProtocol.parseCallState([0xBB, 0x8E, 0x07, 0x00, 0x02, 0x01, 0x02]), + 0x02, + ); + expect( + BesProtocol.parseCallState([0xBB, 0x8E, 0x07, 0x00, 0x02, 0x01, 0x03]), + 0x03, + ); + }); }); } diff --git a/apps/client/test/emai_iflytek_turn_test.dart b/apps/client/test/emai_iflytek_turn_test.dart new file mode 100644 index 00000000..61647a9b --- /dev/null +++ b/apps/client/test/emai_iflytek_turn_test.dart @@ -0,0 +1,53 @@ +import 'package:eaimar/data/services/emai_iflytek_turn.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('时间带星期和时区，提示里不出现经纬度', () { + final now = DateTime(2026, 10, 18, 0, 59); + final prompt = EmaiTurn.systemPrompt( + now: now, + spokenPlace: '上海市浦东新区世纪大道1号', + answerLanguage: '中文', + history: [EmaiMemory(true, '你好')], + ); + expect(prompt, contains('2026年10月18日')); + expect(prompt, contains('星期日')); + expect(prompt, contains('00:59')); + expect(prompt, contains('UTC')); + expect(prompt, contains('当前位置：上海市浦东新区世纪大道1号')); + expect(prompt, isNot(contains('31.2'))); + }); + + test('没有地址时写当前没有位置', () { + final prompt = EmaiTurn.systemPrompt( + now: DateTime(2026, 10, 18), + spokenPlace: '未知', + answerLanguage: '中文', + history: const [], + ); + expect(prompt, contains('当前没有位置')); + expect(prompt, isNot(contains('当前位置：未知'))); + }); + + test('搜索词补上当天日期', () { + final day = DateTime(2026, 10, 18); + expect(EmaiTurn.searchQuery('今天新闻', day), '今天新闻 2026-10-18'); + expect(EmaiTurn.searchQuery('新闻 2026-10-18', day), '新闻 2026-10-18'); + }); + + test('解析工具调用并丢掉令牌字段', () { + final calls = EmaiTurn.parseCalls( + '{"name":"hefeng_weather","arguments":"{\\"city\\":\\"上海\\",\\"auth_token\\":\\"secret\\"}"}', + ); + expect(calls, hasLength(1)); + expect(calls.single.name, 'hefeng_weather'); + expect(calls.single.args['city'], '上海'); + expect(calls.single.args.containsKey('auth_token'), isFalse); + }); + + test('问我在哪才等待定位', () { + expect(EmaiTurn.asksWhere('我现在在哪里'), isTrue); + expect(EmaiTurn.asksWhere('where am I'), isTrue); + expect(EmaiTurn.asksWhere('今天天气怎么样'), isFalse); + }); +} diff --git a/apps/client/test/simul_microsoft_test.dart b/apps/client/test/simul_microsoft_test.dart new file mode 100644 index 00000000..9baecb8b --- /dev/null +++ b/apps/client/test/simul_microsoft_test.dart @@ -0,0 +1,74 @@ +import 'package:eaimar/data/services/simul_microsoft.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('日语到英语走 AST，发音人是 JennyNeural', () { + final route = SimulMicrosoft.plan('ja', 'en'); + expect(route, isNotNull); + expect(route!.ast, isTrue); + expect(route.lang, 'ja'); + expect(route.targetLang, 'en'); + expect(route.speaker, 'JennyNeural'); + }); + + test('英语到日语同传没授权时改走转写，识别 en-US，合成 Nanami', () { + final route = SimulMicrosoft.plan('en', 'ja')!; + final splice = SimulMicrosoft.spliceInstead(route); + expect(splice.ast, isFalse); + expect(splice.languageName, 'en-US'); + expect(splice.speaker, 'ja-JP-NanamiNeural'); + }); + + test('英语到日语走 AST，发音人是 NanamiNeural', () { + final route = SimulMicrosoft.plan('en-US', 'ja-JP'); + expect(route!.ast, isTrue); + expect(route.lang, 'en'); + expect(route.targetLang, 'ja'); + expect(route.speaker, 'NanamiNeural'); + }); + + test('普通话到英语走拼接，识别 zh-CN，合成 en-US-JennyNeural', () { + final route = SimulMicrosoft.plan('zh', 'en'); + expect(route!.ast, isFalse); + expect(route.languageName, 'zh-CN'); + expect(route.speaker, 'en-US-JennyNeural'); + }); + + test('英语到普通话走拼接，识别 en-US，合成晓晓', () { + final route = SimulMicrosoft.plan('en', 'zh-CN'); + expect(route!.ast, isFalse); + expect(route.languageName, 'en-US'); + expect(route.speaker, 'zh-CN-XiaoxiaoNeural'); + }); + + test('普通话到爱尔兰语识别 zh-CN，合成 Orla', () { + final route = SimulMicrosoft.plan('zh', 'ga'); + expect(route!.ast, isFalse); + expect(route.languageName, 'zh-CN'); + expect(route.speaker, 'ga-IE-OrlaNeural'); + }); + + test('粤语简体识别 yue-CN，香港繁体识别 zh-HK', () { + final yue = SimulMicrosoft.plan('yue', 'en'); + expect(yue!.languageName, 'yue-CN'); + expect(yue.speaker, 'en-US-JennyNeural'); + + final hk = SimulMicrosoft.plan('zh-Hant-hk', 'en'); + expect(hk!.languageName, 'zh-HK'); + expect(hk.speaker, 'en-US-JennyNeural'); + }); + + test('威尔士语到冰岛语不启动', () { + expect(SimulMicrosoft.plan('cy', 'is'), isNull); + }); + + test('拼接翻译语种码给讯飞 SDK，中文保持 zh-CN', () { + expect(SimulMicrosoft.translateCode('zh'), 'zh-CN'); + expect(SimulMicrosoft.translateCode('zh-Hant'), 'zh-TW'); + expect(SimulMicrosoft.translateCode('zh-Hant-hk'), 'zh-HK'); + expect(SimulMicrosoft.translateCode('yue'), 'yue'); + expect(SimulMicrosoft.translateCode('ga'), 'ga'); + expect(SimulMicrosoft.translateCode('ja-JP'), 'ja'); + expect(SimulMicrosoft.translateCode('en'), 'en'); + }); +} diff --git a/apps/services/modules/console/api_svctemplate.go b/apps/services/modules/console/api_svctemplate.go index cfcd9777..ab3a4225 100644 --- a/apps/services/modules/console/api_svctemplate.go +++ b/apps/services/modules/console/api_svctemplate.go @@ -299,6 +299,14 @@ func builtinSvcTemplates() []ThirdSvcTemplate { sf("json", "服务账号 JSON(内容或路径)", true)), tpl("asrfile_azure", "azure", "微软 Azure 语音", svcCatASRFile, 3, sf("subscription_key", "订阅密钥", true), sf("region", "区域，如 eastasia", false)), + // 科大讯飞 SDK 里 RegionMSFT 的录音文件转写。引擎名写在 SDK 二进制里， + // 服务端按同一套 OST 协议调用，音频仍用客户端已经传到 OSS 的公网地址。 + tpl("asrfile_iflytek", "iflytek", "科大讯飞 微软录音文件转写", svcCatASRFile, 4, + sf("app_id", "SparkChain appId", false), + sf("api_key", "SparkChain apiKey", true), + sf("api_secret", "SparkChain apiSecret", true), + sfd("host", "OST 节点，新加坡账号用 ost-api-sg.xf-yun.com", false, "ost-api-sg.xf-yun.com"), + sfd("task_type", "SDK RegionMSFT 的引擎名", false, "iflyrec_voice_mul_cbm_msft_24h")), // 多媒体大模型（能识别图像，用于带图会议总结；一个全能模型服务可同时勾选「文本大模型」） tpl("llmv_openai", "openai", "OpenAI(GPT-4o 视觉)", svcCatLLMVision, 0, @@ -316,6 +324,12 @@ func builtinSvcTemplates() []ThirdSvcTemplate { // 加上 languages/default_lang 两个对它没意义的字段、还被巡检当成「没配语言」。 // agent_type 是 AI 球的跳转开关：值为 yidong 时进移动精灵，其它一律进 EMAI； // 停用本服务 = 不下发这些键 = 客户端按默认进 EMAI。 + tpl("llm_iflytek", "iflytek", "gpt-5.3-chat", svcCatLLM, 8, + sf("app_id", "SparkChain appId", false), + sf("api_key", "SparkChain apiKey", true), + sf("api_secret", "SparkChain apiSecret", true), + sfd("model", "SDK 文本模型，默认 gpt-5.3-chat", false, "gpt-5.3-chat"), + sfd("host", "SDK 预置 openai 主机", false, "cn-huadong-1.xf-yun.com")), tpl("llm_mobile_elf", "chinamobile", "中国移动 灵犀（移动精灵）", svcCatLLM, 9, sfe("agent_type", "AI 球跳转：yidong=移动精灵，其它=EMAI 助手", false, "AGENT_TYPE", ""), sfe("agent_id", "灵犀 AgentId", false, "MOBILE_ELF_AGENT_ID", ""), diff --git a/apps/services/modules/echomeet/provider_iflytek.go b/apps/services/modules/echomeet/provider_iflytek.go new file mode 100644 index 00000000..390e149e --- /dev/null +++ b/apps/services/modules/echomeet/provider_iflytek.go @@ -0,0 +1,112 @@ +package echomeet + +import ( + "context" + "fmt" + + "yunyan/pb" + "yunyan/sys/iflytek/openaichat" + "yunyan/sys/iflytek/ost" +) + +// 科大讯飞 SparkChain 的两条服务端能力： +// 转写用 SDK 里 RegionMSFT 的录音文件引擎（音频地址仍是客户端 OSS 公网 URL）； +// 总结用同一套凭据的海外文本模型，domain=openai（gpt-5.3-chat），不走星火。 + +type iflytekTranscriber struct { + sys *ost.Client +} + +func newIflytekTranscriber(f map[string]string) (Transcriber, error) { + appID, err := need(f, "iflytek", "app_id") + if err != nil { + return nil, err + } + key, err := need(f, "iflytek", "api_key") + if err != nil { + return nil, err + } + secret, err := need(f, "iflytek", "api_secret") + if err != nil { + return nil, err + } + sys, err := ost.New(appID, key, secret, f["host"], f["task_type"]) + if err != nil { + return nil, err + } + return &iflytekTranscriber{sys: sys}, nil +} + +func (this *iflytekTranscriber) Submit(_ context.Context, req SubmitRequest) (*SubmitResult, error) { + taskID, err := this.sys.Create(req.AudioURL, req.EnableSpeaker, req.Language) + if err != nil { + return nil, err + } + return &SubmitResult{TaskID: taskID}, nil +} + +func (this *iflytekTranscriber) Query(_ context.Context, taskID, _ string) (*QueryResult, error) { + status, segs, err := this.sys.Query(taskID) + switch status { + case ost.StatusSuccess: + out := make([]*pb.ContextStruct, 0, len(segs)) + for _, s := range segs { + out = append(out, &pb.ContextStruct{ + Content: s.Text, + Starttime: s.StartMs, + Endtime: s.EndMs, + Speaker: s.Speaker, + }) + } + return &QueryResult{Status: StatusSuccess, Contexts: out}, nil + case ost.StatusRunning: + return &QueryResult{Status: StatusRunning}, nil + default: + if err == nil { + err = fmt.Errorf("iflytek ost query failed") + } + return &QueryResult{Status: StatusFailed}, err + } +} + +type iflytekSummarizer struct { + sys *openaichat.Client +} + +func newIflytekSummarizer(f map[string]string) (Summarizer, error) { + appID, err := need(f, "iflytek", "app_id") + if err != nil { + return nil, err + } + key, err := need(f, "iflytek", "api_key") + if err != nil { + return nil, err + } + secret, err := need(f, "iflytek", "api_secret") + if err != nil { + return nil, err + } + // 库里若还留着星火的 host/domain，New 会丢掉，改走 SDK 预置的 openai 地址和 gpt-5.3-chat。 + sys, err := openaichat.New(appID, key, secret, f["host"], f["model"]) + if err != nil { + return nil, err + } + return &iflytekSummarizer{sys: sys}, nil +} + +// iflytekSummaryDisplayName 生成弹窗里显示的模型名，跟真正发出去的模型一致。 +func iflytekSummaryDisplayName(fields map[string]string) string { + model := "" + if fields != nil { + model = fields["model"] + } + return openaichat.EffectiveModel(model) +} + +func (this *iflytekSummarizer) Chat(_ context.Context, messages []ChatMessage) (string, error) { + in := make([]openaichat.Message, 0, len(messages)) + for _, m := range messages { + in = append(in, openaichat.Message{Role: m.Role, Content: m.Content}) + } + return this.sys.Chat(in) +} diff --git a/apps/services/modules/echomeet/provider_registry.go b/apps/services/modules/echomeet/provider_registry.go index 2943ea60..899d0121 100644 --- a/apps/services/modules/echomeet/provider_registry.go +++ b/apps/services/modules/echomeet/provider_registry.go @@ -92,6 +92,8 @@ func buildTranscriber(svc *resolvedSvc) (Transcriber, error) { return nil, err } return µsoftTranscriber{sys: sys}, nil + case "iflytek": + return newIflytekTranscriber(f) default: return nil, fmt.Errorf("未支持的识别服务商: %s", svc.Provider) } @@ -193,6 +195,8 @@ func buildSummarizer(svc *resolvedSvc) (Summarizer, error) { f := svc.Fields provider := strings.ToLower(svc.Provider) switch provider { + case "iflytek": + return newIflytekSummarizer(f) case "doubao", "volcengine": key, err := need(f, "doubao", "api_key") if err != nil { diff --git a/apps/services/modules/echomeet/providers.go b/apps/services/modules/echomeet/providers.go index bd19c7b2..eb8777eb 100644 --- a/apps/services/modules/echomeet/providers.go +++ b/apps/services/modules/echomeet/providers.go @@ -382,9 +382,14 @@ func (this *providersComp) SummaryServices() (clientChoice bool, items []*pb.Ech if !r.IsDefault && (!clientChoice || !r.ClientVisible) { continue } + name := svc.Name + if svc.Provider == "iflytek" { + // 配置名还可能写着「科大讯飞文本总结」。生成弹窗要显示真正在跑的模型。 + name = iflytekSummaryDisplayName(svc.Fields) + } items = append(items, &pb.EchomeetSummaryServiceItem{ SvcId: r.SvcId, - Name: svc.Name, + Name: name, IsDefault: r.IsDefault, }) } diff --git a/apps/services/modules/user/model_translate.go b/apps/services/modules/user/model_translate.go index 610bb1a7..9cfee65f 100644 --- a/apps/services/modules/user/model_translate.go +++ b/apps/services/modules/user/model_translate.go @@ -1,6 +1,7 @@ package user import ( + "context" "os" "strings" "sync" @@ -11,7 +12,7 @@ import ( "yunyan/lego/core/cbase" "yunyan/lego/sys/log" "yunyan/lego/sys/postgres" - alitranslate "yunyan/sys/aliyun/translate" + mstranslate "yunyan/sys/microsoft/translate" ) // 实时机器翻译（同声传译 / 面对面翻译）的服务解析。 @@ -45,12 +46,17 @@ const mtCacheTTL = 60 * time.Second // 但只有阿里实现了自动检测源语种的 TranslateDetect），这里加一行， // 并在 resolveTranslator 下半段按 provider 加一个构造分支。两处都改才算接上。 var mtProviderPriority = map[string]int{ - "alibaba": 0, + "azure": 0, } // mtNormProvider 统一 provider 的大小写与空白，库里的值是人手填的。 func mtNormProvider(p string) string { return strings.TrimSpace(strings.ToLower(p)) } +// sentenceTranslator 单句机器翻译。from 为空时由服务商检测源语种。 +type sentenceTranslator interface { + TranslateDetect(ctx context.Context, from, to, text string) (translated, detected string, err error) +} + // pickMTSvc 从候选里挑一个服务端接得上的机器翻译服务。 // // 排序口径：先按作用域（本应用的行优先于全局行），再按 mtProviderPriority。 @@ -90,7 +96,7 @@ type modelTranslateComp struct { module *User mu sync.RWMutex - cached alitranslate.ISys + cached sentenceTranslator cachedId string cachedProvider string cachedAt time.Time @@ -111,7 +117,7 @@ func (this *modelTranslateComp) Init(service core.IService, module core.IModule, // 候选须同时满足三条：启用、类别含 comm.SvcCatMT、**provider 在 mtProviderPriority 里** // （即服务端真的接了这家）。挑选见 pickMTSvc。 // 返回 svcId/provider 供落日志与响应回带，便于排查"到底用的哪家"。 -func (this *modelTranslateComp) resolveTranslator() (svcId, provider string, t alitranslate.ISys, err error) { +func (this *modelTranslateComp) resolveTranslator() (svcId, provider string, t sentenceTranslator, err error) { this.mu.RLock() if this.cached != nil && time.Since(this.cachedAt) < mtCacheTTL { svcId, provider, t = this.cachedId, this.cachedProvider, this.cached @@ -150,21 +156,19 @@ func (this *modelTranslateComp) resolveTranslator() (svcId, provider string, t a return picked.Id, picked.Provider, nil, ferr } - // 目前只实现了阿里云。接别家时在这里加分支，并同步 mtProviderPriority—— - // 只加一处的话：只加表 → 走到这里返回 nil 仍报「未配置」；只加分支 → 上面过滤时 - // 就被当成未接入跳过了，分支根本执行不到。 - if mtNormProvider(picked.Provider) != "alibaba" { + if mtNormProvider(picked.Provider) != "azure" { return picked.Id, picked.Provider, nil, nil } - - opts := []alitranslate.Option{ - alitranslate.SetAccessKeyId(strings.TrimSpace(fields["access_key_id"])), - alitranslate.SetAccessKeySecret(strings.TrimSpace(fields["access_key_secret"])), + opts := []mstranslate.Option{ + mstranslate.SetKey(strings.TrimSpace(fields["subscription_key"])), } if r := strings.TrimSpace(fields["region"]); r != "" { - opts = append(opts, alitranslate.SetRegion(r)) + opts = append(opts, mstranslate.SetRegion(r)) + } + if ep := strings.TrimSpace(fields["endpoint"]); ep != "" { + opts = append(opts, mstranslate.SetEndpoint(ep)) } - sys, serr := alitranslate.NewSys(opts...) + sys, serr := mstranslate.NewSys(opts...) if serr != nil { return picked.Id, picked.Provider, nil, serr } diff --git a/apps/services/modules/user/model_translate_test.go b/apps/services/modules/user/model_translate_test.go index a5ec1eb9..8b03a834 100644 --- a/apps/services/modules/user/model_translate_test.go +++ b/apps/services/modules/user/model_translate_test.go @@ -17,22 +17,20 @@ func mtSvc(app, id, provider string, enable bool) *comm.ThirdSvcConfig { } } -// 后台同时启用了三家 MT，只有阿里是服务端真接了的：必须挑阿里， -// 而不是看 postgres 先返回谁。2026-09-14 线上翻译整体报「未配置机器翻译服务」 -// 就是因为没这条规则、随手挑中了 mt_volcano。 +// 机器翻译只接微软。阿里即使启用也不再被挑中。 func TestPickMTSvcSkipsUnimplementedProviders(t *testing.T) { - // 顺序刻意把没接入的排在最前，模拟库的返回顺序不利的情况 svcs := []*comm.ThirdSvcConfig{ mtSvc("EAIMAR", "mt_volcano", "volcano", true), - mtSvc("EAIMAR", "mt_azure", "azure", true), mtSvc("EAIMAR", "mt_alibaba", "alibaba", true), + mtSvc("EAIMAR", "mt_azure", "azure", true), } picked, skipped := pickMTSvc(svcs, "EAIMAR") - if picked == nil || picked.Id != "mt_alibaba" { - t.Fatalf("应挑中 mt_alibaba，实际 %v", pickedIdOf(picked)) + if picked == nil || picked.Id != "mt_azure" { + t.Fatalf("应挑中 mt_azure，实际 %v", pickedIdOf(picked)) } - if len(skipped) != 2 || !strings.Contains(strings.Join(skipped, ","), "mt_volcano") { - t.Fatalf("未接入的两家应记进 skipped 供日志排查，实际 %v", skipped) + joined := strings.Join(skipped, ",") + if !strings.Contains(joined, "mt_alibaba") || !strings.Contains(joined, "mt_volcano") { + t.Fatalf("阿里和未接入的服务应记进 skipped，实际 %v", skipped) } } @@ -41,7 +39,7 @@ func TestPickMTSvcSkipsUnimplementedProviders(t *testing.T) { func TestPickMTSvcAllUnimplemented(t *testing.T) { svcs := []*comm.ThirdSvcConfig{ mtSvc("EAIMAR", "mt_volcano", "volcano", true), - mtSvc("EAIMAR", "mt_azure", "azure", true), + mtSvc("EAIMAR", "mt_alibaba", "alibaba", true), } picked, skipped := pickMTSvc(svcs, "EAIMAR") if picked != nil { @@ -68,8 +66,8 @@ func TestPickMTSvcIgnoresDisabledAndOtherCategories(t *testing.T) { // 本应用的行优先于全局行（作用域归一后全局行已不再产生，这条守存量数据）。 func TestPickMTSvcPrefersAppScope(t *testing.T) { svcs := []*comm.ThirdSvcConfig{ - mtSvc("", "mt_alibaba", "alibaba", true), - mtSvc("EAIMAR", "mt_alibaba", "alibaba", true), + mtSvc("", "mt_azure", "azure", true), + mtSvc("EAIMAR", "mt_azure", "azure", true), } picked, _ := pickMTSvc(svcs, "EAIMAR") if picked == nil || picked.AppName != "EAIMAR" { @@ -79,7 +77,7 @@ func TestPickMTSvcPrefersAppScope(t *testing.T) { // provider 是人手填的，大小写/空格不该影响匹配。 func TestPickMTSvcNormalizesProvider(t *testing.T) { - svcs := []*comm.ThirdSvcConfig{mtSvc("EAIMAR", "mt_alibaba", " Alibaba ", true)} + svcs := []*comm.ThirdSvcConfig{mtSvc("EAIMAR", "mt_azure", " Azure ", true)} if picked, _ := pickMTSvc(svcs, "EAIMAR"); picked == nil { t.Fatal("provider 大小写/空白不应影响匹配") } diff --git a/apps/services/sys/iflytek/openaichat/openaichat.go b/apps/services/sys/iflytek/openaichat/openaichat.go new file mode 100644 index 00000000..a3a2a2ec --- /dev/null +++ b/apps/services/sys/iflytek/openaichat/openaichat.go @@ -0,0 +1,298 @@ +// Package openaichat 调用 SparkChain 海外 SDK 里 domain=openai 的文本模型。 +// +// 会议总结走这条，不走星火 websocket。SDK 二进制里的形状是： +// +// POST https://cn-huadong-1.xf-yun.com/v1/private/s56f331b2 +// 鉴权挂在 URL 上（host date request-line，没有 digest） +// 正文是讯飞信封，payload.input.text = base64(OpenAI chat JSON) +// 响应 payload.output.text 再 base64 解出来，里面是 chat.completion 或 responses +package openaichat + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +const ( + defaultHost = "cn-huadong-1.xf-yun.com" + defaultPath = "/v1/private/s56f331b2" + defaultModel = "gpt-5.3-chat" +) + +// Message 一轮对话。Role 取 system / user / assistant。 +type Message struct { + Role string `json:"role"` + Content string `json:"content"` +} + +// Client 讯飞 OpenAI 文本客户端。 +type Client struct { + AppID string + APIKey string + Secret string + Host string + Model string + HTTP *http.Client +} + +// New 用 SparkChain 的 appId / apiKey / apiSecret 建客户端。 +// model 空则用文档里的对话模型 gpt-5.3-chat。host 空则用 SDK 预置地址。 +func New(appID, apiKey, secret, host, model string) (*Client, error) { + if strings.TrimSpace(appID) == "" || strings.TrimSpace(apiKey) == "" || strings.TrimSpace(secret) == "" { + return nil, fmt.Errorf("iflytek openai: app_id/api_key/api_secret 都不能空") + } + host = strings.TrimSpace(host) + host = strings.TrimPrefix(strings.TrimPrefix(host, "https://"), "http://") + host = strings.Trim(host, "/") + if host == "" || strings.Contains(host, "spark-api") { + host = defaultHost + } + model = EffectiveModel(model) + return &Client{ + AppID: appID, + APIKey: apiKey, + Secret: secret, + Host: host, + Model: model, + HTTP: &http.Client{Timeout: 120 * time.Second, CheckRedirect: refuseRedirect}, + }, nil +} + +// EffectiveModel 空模型或星火的 general* 都改成文档里的对话模型。 +func EffectiveModel(model string) string { + model = strings.TrimSpace(model) + if model == "" || strings.HasPrefix(model, "general") { + return defaultModel + } + return model +} + +func refuseRedirect(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse +} + +// Chat 同步要一段回复。 +func (c *Client) Chat(messages []Message) (string, error) { + if len(messages) == 0 { + return "", fmt.Errorf("iflytek openai: 没有输入") + } + body, err := c.envelope(messages) + if err != nil { + return "", err + } + endpoint, err := c.authURL() + if err != nil { + return "", err + } + req, err := http.NewRequest(http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return "", err + } + req.Host = c.Host + req.Header.Set("Content-Type", "application/json") + resp, err := c.HTTP.Do(req) + if err != nil { + return "", fmt.Errorf("iflytek openai: %w", err) + } + defer resp.Body.Close() + raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return "", err + } + text, err := parseOuter(raw) + if err != nil { + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("iflytek openai: http %d: %w", resp.StatusCode, err) + } + return "", err + } + return text, nil +} + +// envelope 讯飞外层。内层是 OpenAI chat JSON，text 字段 base64。 +func (c *Client) envelope(messages []Message) ([]byte, error) { + inner := map[string]any{ + "model": c.Model, + "max_completion_tokens": 4096, + "stream": false, + "messages": messages, + } + raw, err := json.Marshal(inner) + if err != nil { + return nil, err + } + outer := map[string]any{ + "header": map[string]any{ + "app_id": c.AppID, + "status": 3, + }, + "parameter": map[string]any{ + "chat": map[string]any{ + "output": map[string]any{ + "encoding": "utf8", + "compress": "raw", + "format": "json", + }, + }, + }, + "payload": map[string]any{ + "input": map[string]any{ + "encoding": "utf8", + "compress": "raw", + "format": "json", + "status": 3, + "text": base64.StdEncoding.EncodeToString(raw), + }, + }, + } + return json.Marshal(outer) +} + +func (c *Client) authURL() (string, error) { + date := time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT") + origin := "host: " + c.Host + "\n" + "date: " + date + "\n" + "POST " + defaultPath + " HTTP/1.1" + mac := hmac.New(sha256.New, []byte(c.Secret)) + _, _ = mac.Write([]byte(origin)) + signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) + authOrigin := fmt.Sprintf(`api_key="%s", algorithm="hmac-sha256", headers="host date request-line", signature="%s"`, + c.APIKey, signature) + q := url.Values{} + q.Set("authorization", base64.StdEncoding.EncodeToString([]byte(authOrigin))) + q.Set("date", date) + q.Set("host", c.Host) + return "https://" + c.Host + defaultPath + "?" + q.Encode(), nil +} + +func parseOuter(raw []byte) (string, error) { + var outer struct { + Header struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"header"` + Payload struct { + Output struct { + Text string `json:"text"` + } `json:"output"` + } `json:"payload"` + } + if err := json.Unmarshal(raw, &outer); err != nil { + return "", fmt.Errorf("iflytek openai: 响应不是 JSON") + } + if outer.Header.Code != 0 { + msg := strings.TrimSpace(outer.Header.Message) + if msg == "" { + msg = "gateway request failed" + } + return "", fmt.Errorf("iflytek openai: code=%d %s", outer.Header.Code, msg) + } + decoded, err := base64.StdEncoding.DecodeString(outer.Payload.Output.Text) + if err != nil { + decoded, err = base64.RawStdEncoding.DecodeString(outer.Payload.Output.Text) + } + if err != nil { + return "", fmt.Errorf("iflytek openai: 输出不是 base64") + } + text := extractContent(decoded) + text = strings.TrimSpace(text) + if text == "" { + return "", fmt.Errorf("iflytek openai: 空回复") + } + return text, nil +} + +func extractContent(decoded []byte) string { + s := strings.TrimSpace(string(decoded)) + if strings.Contains(s, "data:") && !json.Valid(decoded) { + return sseContent(s) + } + var body map[string]any + if json.Unmarshal(decoded, &body) != nil { + return s + } + if fmt.Sprint(body["object"]) == "response" { + if t := responseText(body); t != "" { + return t + } + } + if t := chatCompletionText(body); t != "" { + return t + } + if t := responseText(body); t != "" { + return t + } + return "" +} + +func chatCompletionText(body map[string]any) string { + choices, _ := body["choices"].([]any) + var b strings.Builder + for _, item := range choices { + m, _ := item.(map[string]any) + if msg, ok := m["message"].(map[string]any); ok { + if c, ok := msg["content"].(string); ok { + b.WriteString(c) + } + } + if delta, ok := m["delta"].(map[string]any); ok { + if c, ok := delta["content"].(string); ok { + b.WriteString(c) + } + } + if t, ok := m["text"].(string); ok { + b.WriteString(t) + } + } + return b.String() +} + +func responseText(body map[string]any) string { + var b strings.Builder + switch out := body["output"].(type) { + case string: + return out + case []any: + for _, item := range out { + m, _ := item.(map[string]any) + if t, ok := m["text"].(string); ok { + b.WriteString(t) + } + content, _ := m["content"].([]any) + for _, part := range content { + pm, _ := part.(map[string]any) + if t, ok := pm["text"].(string); ok { + b.WriteString(t) + } + } + } + } + return b.String() +} + +func sseContent(s string) string { + var b strings.Builder + for _, line := range strings.Split(s, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "data:") { + continue + } + payload := strings.TrimSpace(strings.TrimPrefix(line, "data:")) + if payload == "" || payload == "[DONE]" { + continue + } + var body map[string]any + if json.Unmarshal([]byte(payload), &body) != nil { + continue + } + b.WriteString(chatCompletionText(body)) + } + return b.String() +} diff --git a/apps/services/sys/iflytek/openaichat/openaichat_test.go b/apps/services/sys/iflytek/openaichat/openaichat_test.go new file mode 100644 index 00000000..67985066 --- /dev/null +++ b/apps/services/sys/iflytek/openaichat/openaichat_test.go @@ -0,0 +1,99 @@ +package openaichat + +import ( + "encoding/base64" + "encoding/json" + "strings" + "testing" +) + +func TestEffectiveModelDropsSpark(t *testing.T) { + if EffectiveModel("") != "gpt-5.3-chat" || EffectiveModel("generalv3.5") != "gpt-5.3-chat" { + t.Fatal(EffectiveModel("generalv3.5")) + } + if EffectiveModel("gpt-5.4") != "gpt-5.4" { + t.Fatal(EffectiveModel("gpt-5.4")) + } +} + +func TestEnvelopeUsesOpenAIModelNotSpark(t *testing.T) { + c, err := New("app", "key", "secret", "spark-api.xf-yun.com", "generalv3.5") + if err != nil { + t.Fatal(err) + } + if c.Host != defaultHost { + t.Fatalf("星火主机必须丢掉，得到 %s", c.Host) + } + if c.Model != defaultModel { + t.Fatalf("星火 domain 不能当模型，得到 %s", c.Model) + } + raw, err := c.envelope([]Message{{Role: "user", Content: "1+1等于几"}}) + if err != nil { + t.Fatal(err) + } + var outer struct { + Header struct { + AppID string `json:"app_id"` + Status int `json:"status"` + } `json:"header"` + Payload struct { + Input struct { + Status int `json:"status"` + Text string `json:"text"` + } `json:"input"` + } `json:"payload"` + } + if err := json.Unmarshal(raw, &outer); err != nil { + t.Fatal(err) + } + if outer.Header.AppID != "app" || outer.Header.Status != 3 || outer.Payload.Input.Status != 3 { + t.Fatalf("信封不对: %+v", outer.Header) + } + inner, err := base64.StdEncoding.DecodeString(outer.Payload.Input.Text) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(inner), "generalv3") || strings.Contains(string(inner), "spark") { + t.Fatalf("内层还带着星火: %s", inner) + } + var chat struct { + Model string `json:"model"` + Stream bool `json:"stream"` + Messages []Message `json:"messages"` + } + if err := json.Unmarshal(inner, &chat); err != nil { + t.Fatal(err) + } + if chat.Model != "gpt-5.3-chat" || chat.Stream || chat.Messages[0].Content != "1+1等于几" { + t.Fatalf("内层不对: %+v", chat) + } +} + +func TestParseChatCompletion(t *testing.T) { + inner, _ := json.Marshal(map[string]any{ + "object": "chat.completion", + "choices": []any{ + map[string]any{"message": map[string]any{"content": "1+1等于2。"}}, + }, + }) + outer, _ := json.Marshal(map[string]any{ + "header": map[string]any{"code": 0, "message": "success"}, + "payload": map[string]any{ + "output": map[string]any{"text": base64.StdEncoding.EncodeToString(inner)}, + }, + }) + got, err := parseOuter(outer) + if err != nil || got != "1+1等于2。" { + t.Fatalf("got %q err %v", got, err) + } +} + +func TestParseGatewayError(t *testing.T) { + outer, _ := json.Marshal(map[string]any{ + "header": map[string]any{"code": 11200, "message": "授权错误"}, + }) + _, err := parseOuter(outer) + if err == nil || !strings.Contains(err.Error(), "11200") { + t.Fatalf("err=%v", err) + } +} diff --git a/apps/services/sys/iflytek/ost/ost.go b/apps/services/sys/iflytek/ost/ost.go new file mode 100644 index 00000000..97cbf1e9 --- /dev/null +++ b/apps/services/sys/iflytek/ost/ost.go @@ -0,0 +1,381 @@ +// Package ost 调用科大讯飞 SparkChain SDK 里 RegionMSFT 那条录音文件转写。 +// +// SDK 二进制里的引擎名是 iflyrec_voice_mul_cbm_msft_24h，走海外 OST： +// 创建 https:///v2/ost/pro_create ，查询 https:///v2/ost/query 。 +// 音频不经过讯飞上传接口，直接用客户端已经放到 OSS 上的公网地址。 +package ost + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/google/uuid" +) + +const ( + defaultHost = "ost-api-sg.xf-yun.com" + defaultTaskType = "iflyrec_voice_mul_cbm_msft_24h" + createPath = "/v2/ost/create" + queryPath = "/v2/ost/query" +) + +// Segment 一句转写。StartMs/EndMs 是相对音频开头的毫秒。 +type Segment struct { + Text string + StartMs int64 + EndMs int64 + Speaker string +} + +// Client 讯飞 OST 客户端。 +type Client struct { + AppID string + APIKey string + Secret string + Host string + TaskType string + HTTP *http.Client +} + +// New 用 SparkChain 的 appId / apiKey / apiSecret 建客户端。 +func New(appID, apiKey, secret, host, taskType string) (*Client, error) { + if strings.TrimSpace(appID) == "" || strings.TrimSpace(apiKey) == "" || strings.TrimSpace(secret) == "" { + return nil, fmt.Errorf("iflytek ost: app_id/api_key/api_secret 都不能空") + } + host = strings.TrimSpace(host) + if host == "" { + host = defaultHost + } + host = strings.TrimPrefix(strings.TrimPrefix(host, "https://"), "http://") + host = strings.TrimRight(host, "/") + taskType = strings.TrimSpace(taskType) + if taskType == "" { + taskType = defaultTaskType + } + return &Client{ + AppID: appID, + APIKey: apiKey, + Secret: secret, + Host: host, + TaskType: taskType, + HTTP: &http.Client{Timeout: 45 * time.Second}, + }, nil +} + +// Create 提交一条公网音频。language 用 BCP-47（zh-CN / en-US），空则按中文。 +func (c *Client) Create(audioURL string, speaker bool, language string) (string, error) { + enc, format := audioEncoding(audioURL) + business := map[string]any{ + "request_id": strings.ReplaceAll(uuid.NewString(), "-", ""), + "task_type": c.TaskType, + "language": ostLanguage(language), + } + applySpeaker(business, c.TaskType, speaker, enc) + body := map[string]any{ + "common": map[string]string{"app_id": c.AppID}, + "business": business, + "data": map[string]any{ + "audio_url": audioURL, + "audio_src": "http", + "format": format, + "encoding": enc, + }, + } + var resp struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + TaskID string `json:"task_id"` + } `json:"data"` + } + if err := c.post(createPath, body, &resp); err != nil { + return "", err + } + if resp.Code != 0 { + return "", fmt.Errorf("iflytek ost create: code=%d %s", resp.Code, resp.Message) + } + if resp.Data.TaskID == "" { + return "", fmt.Errorf("iflytek ost create: 没有 task_id") + } + return resp.Data.TaskID, nil +} + +// msftSpeakerCap 微软文件转写的说话人数上限。speaker_num 合法范围是 1–35， +// 不传时文档默认 1，等于所有人合成一个角色。会议人数事先不知道，按 10 交给引擎去分。 +const msftSpeakerCap = 10 + +// applySpeaker 按引擎写角色分离参数。 +// 微软通道（task_type 含 msft）认 spkdia，不认讯飞自研的 vspp_on。 +// 自研通道的 mp3 官方写明不支持角色分离，硬开会失败。 +func applySpeaker(business map[string]any, taskType string, speaker bool, enc string) { + if !speaker { + return + } + if strings.Contains(taskType, "msft") { + business["spkdia"] = 1 + business["speaker_num"] = msftSpeakerCap + return + } + if enc == "lame" { + return + } + business["vspp_on"] = 1 + business["speaker_num"] = 0 +} + +// QueryStatus 任务状态。running 表示还要再查，failed 时 err 带原因。 +type QueryStatus int + +const ( + StatusRunning QueryStatus = iota + StatusSuccess + StatusFailed +) + +// Query 查一次。完成时 segments 是按句切开的文本。 +func (c *Client) Query(taskID string) (QueryStatus, []Segment, error) { + body := map[string]any{ + "common": map[string]string{"app_id": c.AppID}, + "business": map[string]string{"task_id": taskID}, + } + var raw map[string]any + if err := c.post(queryPath, body, &raw); err != nil { + return StatusFailed, nil, err + } + code := intFrom(raw["code"]) + if code != 0 { + return StatusFailed, nil, fmt.Errorf("iflytek ost query: code=%d %v", code, raw["message"]) + } + data, _ := raw["data"].(map[string]any) + if data == nil { + return StatusRunning, nil, nil + } + switch fmt.Sprint(data["task_status"]) { + case "1", "2", "": + return StatusRunning, nil, nil + case "3", "4": + return StatusSuccess, parseLattice(data["result"]), nil + default: + return StatusFailed, nil, fmt.Errorf("iflytek ost query: task_status=%v %v", data["task_status"], raw["message"]) + } +} + +func (c *Client) post(path string, payload any, out any) error { + raw, err := json.Marshal(payload) + if err != nil { + return err + } + endpoint := "https://" + c.Host + path + req, err := http.NewRequest(http.MethodPost, endpoint, bytes.NewReader(raw)) + if err != nil { + return err + } + req.Host = c.Host + for k, v := range c.authHeader(path, raw) { + if k == "Host" { + continue + } + req.Header.Set(k, v) + } + client := c.HTTP + if client.CheckRedirect == nil { + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + } + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("iflytek ost %s: %w", path, err) + } + defer resp.Body.Close() + buf, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + if json.Unmarshal(buf, out) == nil { + return nil + } + return fmt.Errorf("iflytek ost %s: http %d %s", endpoint, resp.StatusCode, truncate(string(buf), 300)) + } + if err := json.Unmarshal(buf, out); err != nil { + return fmt.Errorf("iflytek ost %s: 响应不是 JSON: %s", path, truncate(string(buf), 200)) + } + return nil +} + +func (c *Client) authHeader(path string, body []byte) map[string]string { + sum := sha256.Sum256(body) + digest := "SHA-256=" + base64.StdEncoding.EncodeToString(sum[:]) + date := time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT") + origin := "host: " + c.Host + "\n" + + "date: " + date + "\n" + + "POST " + path + " HTTP/1.1\n" + + "digest: " + digest + mac := hmac.New(sha256.New, []byte(c.Secret)) + _, _ = mac.Write([]byte(origin)) + signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) + auth := fmt.Sprintf(`api_key="%s", algorithm="hmac-sha256", headers="host date request-line digest", signature="%s"`, + c.APIKey, signature) + return map[string]string{ + "Content-Type": "application/json", + "Accept": "application/json", + "Host": c.Host, + "Date": date, + "Digest": digest, + "Authorization": auth, + } +} + +func ostLanguage(code string) string { + c := strings.TrimSpace(code) + if c == "" { + return "zh-CN" + } + c = strings.ReplaceAll(c, "_", "-") + if !strings.Contains(c, "-") { + switch strings.ToLower(c) { + case "zh": + return "zh-CN" + case "en": + return "en-US" + case "ja": + return "ja-JP" + case "ko": + return "ko-KR" + } + } + return c +} + +func audioEncoding(audioURL string) (encoding, format string) { + u, err := url.Parse(audioURL) + path := audioURL + if err == nil && u.Path != "" { + path = u.Path + } + lower := strings.ToLower(path) + switch { + case strings.HasSuffix(lower, ".mp3"): + return "lame", "audio/mpeg" + case strings.HasSuffix(lower, ".wav"), strings.HasSuffix(lower, ".pcm"): + return "raw", "audio/L16;rate=16000" + default: + return "raw", "audio/L16;rate=16000" + } +} + +// parseLattice 把 OST 的 lattice 收成一句一条。json_1best 有时是对象，有时是字符串。 +func parseLattice(result any) []Segment { + obj, _ := result.(map[string]any) + if obj == nil { + if s, ok := result.(string); ok && s != "" { + _ = json.Unmarshal([]byte(s), &obj) + } + } + if obj == nil { + return nil + } + lattice, _ := obj["lattice"].([]any) + if len(lattice) == 0 { + lattice, _ = obj["lattice2"].([]any) + } + out := make([]Segment, 0, len(lattice)) + for _, item := range lattice { + m, _ := item.(map[string]any) + if m == nil { + continue + } + best := unwrapBest(m["json_1best"]) + st, _ := best["st"].(map[string]any) + if st == nil { + continue + } + text := wordsOf(st) + if text == "" { + continue + } + spk := strings.TrimSpace(fmt.Sprint(m["spk"])) + if spk == "" || spk == "" { + spk = strings.TrimSpace(fmt.Sprint(st["rl"])) + } + out = append(out, Segment{ + Text: text, + StartMs: intFrom(st["bg"]), + EndMs: intFrom(st["ed"]), + Speaker: spk, + }) + } + return out +} + +func unwrapBest(v any) map[string]any { + switch t := v.(type) { + case map[string]any: + return t + case string: + var m map[string]any + if json.Unmarshal([]byte(t), &m) == nil { + return m + } + } + return map[string]any{} +} + +func wordsOf(st map[string]any) string { + rts, _ := st["rt"].([]any) + var b strings.Builder + for _, rt := range rts { + rtm, _ := rt.(map[string]any) + wss, _ := rtm["ws"].([]any) + for _, ws := range wss { + wsm, _ := ws.(map[string]any) + cws, _ := wsm["cw"].([]any) + for _, cw := range cws { + cwm, _ := cw.(map[string]any) + if fmt.Sprint(cwm["wp"]) == "g" { + continue + } + w, _ := cwm["w"].(string) + b.WriteString(w) + } + } + } + return strings.TrimSpace(b.String()) +} + +func intFrom(v any) int64 { + switch t := v.(type) { + case float64: + return int64(t) + case json.Number: + n, _ := t.Int64() + return n + case string: + var n int64 + fmt.Sscan(t, &n) + return n + case int: + return int64(t) + case int64: + return t + default: + return 0 + } +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} diff --git a/apps/services/sys/iflytek/ost/ost_test.go b/apps/services/sys/iflytek/ost/ost_test.go new file mode 100644 index 00000000..7cdda3d6 --- /dev/null +++ b/apps/services/sys/iflytek/ost/ost_test.go @@ -0,0 +1,72 @@ +package ost + +import "testing" + +func TestParseLattice(t *testing.T) { + raw := map[string]any{ + "lattice": []any{ + map[string]any{ + "spk": "段落-0", + "json_1best": map[string]any{ + "st": map[string]any{ + "bg": "0", + "ed": "470", + "rt": []any{ + map[string]any{ + "ws": []any{ + map[string]any{"cw": []any{map[string]any{"w": "听说", "wp": "n"}}}, + map[string]any{"cw": []any{map[string]any{"w": "。", "wp": "p"}}}, + map[string]any{"cw": []any{map[string]any{"w": "", "wp": "g"}}}, + }, + }, + }, + }, + }, + }, + }, + } + segs := parseLattice(raw) + if len(segs) != 1 { + t.Fatalf("segs=%d", len(segs)) + } + if segs[0].Text != "听说。" || segs[0].StartMs != 0 || segs[0].EndMs != 470 || segs[0].Speaker != "段落-0" { + t.Fatalf("%+v", segs[0]) + } +} + +func TestApplySpeaker(t *testing.T) { + msft := map[string]any{} + applySpeaker(msft, "iflyrec_voice_mul_cbm_msft_24h", true, "raw") + if msft["spkdia"] != 1 || msft["speaker_num"] != msftSpeakerCap { + t.Fatalf("msft %+v", msft) + } + if _, ok := msft["vspp_on"]; ok { + t.Fatal("微软通道不该带 vspp_on") + } + off := map[string]any{} + applySpeaker(off, "iflyrec_voice_mul_cbm_msft_24h", false, "raw") + if len(off) != 0 { + t.Fatalf("off %+v", off) + } + self := map[string]any{} + applySpeaker(self, "iflyrec_voice_cn_10m_ed", true, "raw") + if self["vspp_on"] != 1 || self["speaker_num"] != 0 { + t.Fatalf("self %+v", self) + } + mp3 := map[string]any{} + applySpeaker(mp3, "iflyrec_voice_cn_10m_ed", true, "lame") + if len(mp3) != 0 { + t.Fatalf("mp3 %+v", mp3) + } +} + +func TestAudioEncoding(t *testing.T) { + enc, _ := audioEncoding("https://oss.example.com/a/b.MP3?x=1") + if enc != "lame" { + t.Fatal(enc) + } + enc, format := audioEncoding("https://oss.example.com/a/b.wav") + if enc != "raw" || format != "audio/L16;rate=16000" { + t.Fatal(enc, format) + } +} diff --git a/apps/services/sys/iflytek/sparkchat/sparkchat.go b/apps/services/sys/iflytek/sparkchat/sparkchat.go new file mode 100644 index 00000000..2b24b854 --- /dev/null +++ b/apps/services/sys/iflytek/sparkchat/sparkchat.go @@ -0,0 +1,294 @@ +// Package sparkchat 是星火文本 websocket。会议总结不要用它。 +// +// 海外 SDK 的文本模型是 domain=openai，见 sys/iflytek/openaichat。 +// 这个包留着只因为星火协议和 openai 那条不是同一个地址。 +package sparkchat + +import ( + "bufio" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net" + "net/url" + "strings" + "time" +) + +const ( + defaultHost = "spark-api.xf-yun.com" + defaultDomain = "generalv3.5" +) + +// Message 一轮对话。Role 取 system / user / assistant。 +type Message struct { + Role string + Content string +} + +// Client 星火文本客户端。 +type Client struct { + AppID string + APIKey string + Secret string + Host string + Domain string +} + +// New 用 SparkChain 凭据建客户端。 +func New(appID, apiKey, secret, host, domain string) (*Client, error) { + if strings.TrimSpace(appID) == "" || strings.TrimSpace(apiKey) == "" || strings.TrimSpace(secret) == "" { + return nil, fmt.Errorf("iflytek spark: app_id/api_key/api_secret 都不能空") + } + host = strings.TrimSpace(host) + if host == "" { + host = defaultHost + } + domain = strings.TrimSpace(domain) + if domain == "" { + domain = defaultDomain + } + return &Client{AppID: appID, APIKey: apiKey, Secret: secret, Host: host, Domain: domain}, nil +} + +// Chat 同步要一段回复。多帧内容拼在一起返回。 +func (c *Client) Chat(messages []Message) (string, error) { + if len(messages) == 0 { + return "", fmt.Errorf("iflytek spark: 没有输入") + } + path := domainPath(c.Domain) + conn, err := c.dial(path) + if err != nil { + return "", err + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(90 * time.Second)) + + text := make([]map[string]string, 0, len(messages)) + for _, m := range messages { + role := m.Role + if role == "" { + role = "user" + } + text = append(text, map[string]string{"role": role, "content": m.Content}) + } + payload := map[string]any{ + "header": map[string]string{"app_id": c.AppID, "uid": "echomeet"}, + "parameter": map[string]any{ + "chat": map[string]any{ + "domain": c.Domain, + "temperature": 0.5, + "max_tokens": 4096, + }, + }, + "payload": map[string]any{ + "message": map[string]any{"text": text}, + }, + } + raw, err := json.Marshal(payload) + if err != nil { + return "", err + } + if err := writeText(conn, raw); err != nil { + return "", err + } + var b strings.Builder + for { + frame, err := readFrame(conn) + if err != nil { + if b.Len() > 0 && err == io.EOF { + break + } + return "", err + } + if frame == nil { + continue + } + var resp struct { + Header struct { + Code int `json:"code"` + Message string `json:"message"` + Status int `json:"status"` + } `json:"header"` + Payload struct { + Choices struct { + Status int `json:"status"` + Text []struct { + Content string `json:"content"` + } `json:"text"` + } `json:"choices"` + } `json:"payload"` + } + if json.Unmarshal(frame, &resp) != nil { + continue + } + if resp.Header.Code != 0 { + return "", fmt.Errorf("iflytek spark: code=%d %s", resp.Header.Code, resp.Header.Message) + } + for _, t := range resp.Payload.Choices.Text { + b.WriteString(t.Content) + } + if resp.Header.Status == 2 || resp.Payload.Choices.Status == 2 { + break + } + } + out := strings.TrimSpace(b.String()) + if out == "" { + return "", fmt.Errorf("iflytek spark: 空回复") + } + return out, nil +} + +func domainPath(domain string) string { + switch domain { + case "general": + return "/v1.1/chat" + case "generalv2": + return "/v2.1/chat" + case "generalv3": + return "/v3.1/chat" + case "4.0Ultra": + return "/v4.0/chat" + default: + return "/v3.5/chat" + } +} + +func (c *Client) dial(path string) (net.Conn, error) { + date := time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT") + origin := "host: " + c.Host + "\n" + "date: " + date + "\n" + "GET " + path + " HTTP/1.1" + mac := hmac.New(sha256.New, []byte(c.Secret)) + _, _ = mac.Write([]byte(origin)) + signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) + authOrigin := fmt.Sprintf(`api_key="%s", algorithm="hmac-sha256", headers="host date request-line", signature="%s"`, + c.APIKey, signature) + auth := base64.StdEncoding.EncodeToString([]byte(authOrigin)) + q := url.Values{} + q.Set("authorization", auth) + q.Set("date", date) + q.Set("host", c.Host) + conn, err := tls.Dial("tcp", c.Host+":443", &tls.Config{ServerName: c.Host}) + if err != nil { + return nil, fmt.Errorf("iflytek spark dial: %w", err) + } + key := make([]byte, 16) + _, _ = rand.Read(key) + secKey := base64.StdEncoding.EncodeToString(key) + req := "GET " + path + "?" + q.Encode() + " HTTP/1.1\r\n" + + "Host: " + c.Host + "\r\n" + + "Upgrade: websocket\r\n" + + "Connection: Upgrade\r\n" + + "Sec-WebSocket-Key: " + secKey + "\r\n" + + "Sec-WebSocket-Version: 13\r\n\r\n" + if _, err := io.WriteString(conn, req); err != nil { + conn.Close() + return nil, err + } + br := bufio.NewReader(conn) + status, err := br.ReadString('\n') + if err != nil { + conn.Close() + return nil, err + } + if !strings.Contains(status, "101") { + conn.Close() + return nil, fmt.Errorf("iflytek spark handshake: %s", strings.TrimSpace(status)) + } + for { + line, err := br.ReadString('\n') + if err != nil { + conn.Close() + return nil, err + } + if line == "\r\n" || line == "\n" { + break + } + } + return &bufferedConn{Conn: conn, r: br}, nil +} + +// bufferedConn 把握手时多读进 bufio 的字节交还给后续读帧。 +type bufferedConn struct { + net.Conn + r *bufio.Reader +} + +func (c *bufferedConn) Read(p []byte) (int, error) { return c.r.Read(p) } + +func writeText(conn net.Conn, payload []byte) error { + mask := make([]byte, 4) + _, _ = rand.Read(mask) + var hdr []byte + n := len(payload) + switch { + case n < 126: + hdr = []byte{0x81, byte(n) | 0x80} + case n < 65536: + hdr = []byte{0x81, 126 | 0x80, byte(n >> 8), byte(n)} + default: + return fmt.Errorf("iflytek spark: 请求过长") + } + frame := append(hdr, mask...) + masked := make([]byte, n) + for i := 0; i < n; i++ { + masked[i] = payload[i] ^ mask[i%4] + } + frame = append(frame, masked...) + _, err := conn.Write(frame) + return err +} + +func readFrame(conn net.Conn) ([]byte, error) { + for { + hdr := make([]byte, 2) + if _, err := io.ReadFull(conn, hdr); err != nil { + return nil, err + } + opcode := hdr[0] & 0x0f + masked := hdr[1]&0x80 != 0 + n := int(hdr[1] & 0x7f) + switch n { + case 126: + ext := make([]byte, 2) + if _, err := io.ReadFull(conn, ext); err != nil { + return nil, err + } + n = int(ext[0])<<8 | int(ext[1]) + case 127: + return nil, fmt.Errorf("iflytek spark: 帧过长") + } + var mask []byte + if masked { + mask = make([]byte, 4) + if _, err := io.ReadFull(conn, mask); err != nil { + return nil, err + } + } + payload := make([]byte, n) + if _, err := io.ReadFull(conn, payload); err != nil { + return nil, err + } + if masked { + for i := 0; i < n; i++ { + payload[i] ^= mask[i%4] + } + } + switch opcode { + case 0x1, 0x0: + return payload, nil + case 0x8: + return nil, io.EOF + case 0x9: // ping + pong := []byte{0x8A, 0} + _, _ = conn.Write(pong) + continue + default: + continue + } + } +} diff --git a/apps/services/sys/microsoft/translate/core.go b/apps/services/sys/microsoft/translate/core.go index 06d4828d..33bf6fc5 100644 --- a/apps/services/sys/microsoft/translate/core.go +++ b/apps/services/sys/microsoft/translate/core.go @@ -5,6 +5,7 @@ import "context" type ( ISys interface { Translate(ctx context.Context, from string, to string, text []string) (results []string, err error) + TranslateDetect(ctx context.Context, from, to, text string) (translated, detected string, err error) } ) diff --git a/apps/services/sys/microsoft/translate/translate.go b/apps/services/sys/microsoft/translate/translate.go index 4b937b15..e13ef73c 100644 --- a/apps/services/sys/microsoft/translate/translate.go +++ b/apps/services/sys/microsoft/translate/translate.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "strings" "time" ) @@ -42,12 +43,47 @@ type azureInput struct { } type azureOutput struct { + DetectedLanguage *struct { + Language string `json:"language"` + } `json:"detectedLanguage"` Translations []struct { Text string `json:"text"` To string `json:"to"` } `json:"translations"` } +// TranslateDetect 翻一句。from 为空时不传 from，让微软检测源语种。 +func (this *TranslateSys) TranslateDetect(ctx context.Context, from, to, text string) (translated, detected string, err error) { + out, err := this.translateBatch(ctx, azureLang(from), azureLang(to), []string{text}) + if err != nil { + return "", "", err + } + if len(out) == 0 { + return "", "", nil + } + return out[0].text, out[0].detected, nil +} + +type azureLine struct { + text string + detected string +} + +func azureLang(code string) string { + switch strings.ToLower(strings.TrimSpace(code)) { + case "", "auto": + return "" + case "zh", "zh-cn", "zh-hans": + return "zh-Hans" + case "zh-tw", "zh-hk", "zh-hant": + return "zh-Hant" + case "no", "nb", "nn": + return "nb" + default: + return code + } +} + func (this *TranslateSys) Translate(ctx context.Context, from string, to string, texts []string) (results []string, err error) { results = make([]string, 0, len(texts)) batch := make([]string, 0, maxTextsPerBatch) @@ -60,7 +96,9 @@ func (this *TranslateSys) Translate(ctx context.Context, from string, to string, if e != nil { return e } - results = append(results, part...) + for _, line := range part { + results = append(results, line.text) + } batch = batch[:0] batchChars = 0 return nil @@ -77,7 +115,9 @@ func (this *TranslateSys) Translate(ctx context.Context, from string, to string, if e != nil { return nil, e } - results = append(results, part...) + for _, line := range part { + results = append(results, line.text) + } continue } if (batchChars+tl > maxCharsPerBatch || len(batch) >= maxTextsPerBatch) && len(batch) > 0 { @@ -94,13 +134,13 @@ func (this *TranslateSys) Translate(ctx context.Context, from string, to string, return } -func (this *TranslateSys) translateBatch(ctx context.Context, from, to string, texts []string) ([]string, error) { +func (this *TranslateSys) translateBatch(ctx context.Context, from, to string, texts []string) ([]azureLine, error) { q := url.Values{} q.Set("api-version", "3.0") - if from != "" && from != "auto" { - q.Set("from", from) + if mapped := azureLang(from); mapped != "" { + q.Set("from", mapped) } - q.Set("to", to) + q.Set("to", azureLang(to)) endpoint := this.options.Endpoint + "/translate?" + q.Encode() inputs := make([]azureInput, 0, len(texts)) @@ -136,13 +176,16 @@ func (this *TranslateSys) translateBatch(ctx context.Context, from, to string, t if err := json.Unmarshal(data, &out); err != nil { return nil, fmt.Errorf("azure parse resp: %w body=%s", err, string(data)) } - results := make([]string, 0, len(out)) + results := make([]azureLine, 0, len(out)) for _, o := range out { + line := azureLine{} + if o.DetectedLanguage != nil { + line.detected = o.DetectedLanguage.Language + } if len(o.Translations) > 0 { - results = append(results, o.Translations[0].Text) - } else { - results = append(results, "") + line.text = o.Translations[0].Text } + results = append(results, line) } return results, nil } diff --git a/docs/EMAI助手-讯飞对话-设计.md b/docs/EMAI助手-讯飞对话-设计.md new file mode 100644 index 00000000..7ee7cbab --- /dev/null +++ b/docs/EMAI助手-讯飞对话-设计.md @@ -0,0 +1,142 @@ +# EMAI 助手：端上听写 + 对话 + 合成 —— 设计 + +> 版本：v0.2　　日期：2026-10-18 +> 状态：**设计稿，未写代码。** +> 范围：耳机唤醒和 App 里按住说话，两条入口共用这一套。通话 / 同声 / 面对面 / 音视频不在本文。 +> 对照：[科大讯飞SparkChain-全面替换第三方-落地文档](科大讯飞SparkChain-全面替换第三方-落地文档.md) 第 5.5 节是「三段接上、先问通」；本文把时间、地点、天气、新闻、个人工具的分工定死。 + +--- + +## 零、结论 + +每轮对话都带上**手机本地时间**和**缓存地址**，只解决「现在几点、今天星期几、我在哪」。 + +天气、新闻和其它要联网才知道的问题，模型手里只有这两样事实，答不出来。这两类仍由手机执行工具，把结果喂回模型，再合成。 + +个人数据（纪要、拾忆、待办）同样是手机拿登录身份去调已有 MCP。对话本身不经过业务接口。 + +--- + +## 一、时间地址怎么带（相对「原样塞进每句话」改过的地方） + +| | 做法 | 原因 | +|---|---|---| +| 时间 | 每一轮用户说完，用手机本地钟**重新拼一次** | 会话可能持续十几分钟。「现在几点」不能用开场那一刻 | +| 地址 | 只读 `DeviceLocationService` 的缓存，**这一轮不等 GPS** | 定位会拖住开口。缓存规则维持现状：回前台刷新，10 分钟内不重复 | +| 放哪 | 放进**本轮系统提示**，不拼进用户原话，不写入聊天历史 | 写进原话，模型会当成用户自己说的；写进历史，旧时间会把「现在」答错 | +| 经纬度 | **不进模型**。只有天气工具要用时，手机自己附上 | 模型念出坐标没有用，还容易被读出来 | +| 没有位置 | 系统提示写「当前没有位置」，不要写「未知」 | 「未知」会被当成地名 | +| 专问「我在哪」且缓存空 | 只这一问允许再等一次定位，上限 4 秒，到点仍没有就按「当前没有位置」答 | 别的问题（含天气）不为这一问阻塞 | + +时间文案用手机时区，带星期和偏移，例如：`2026年10月18日 星期日 00:59，东八区（UTC+8）`。 + +地址文案沿用现在能念出来的那串（完整地址，没有再退省市区街道）。念「我在哪」用这串。查天气只用其中的**城市名**。 + +--- + +## 二、整轮怎么走 + +SDK 没有百炼那种边听边说。一轮是说完再答。耳机协议、问候语、打断、后台保活维持第 5.5 节，本文只换「听完之后谁来答」。 + +``` +麦克风 PCM + → 讯飞 SDK 实时转写，区域 RegionMSFT，语种 = 下面第三节 + → 一句最终文本 + → 本轮系统提示（新时间 + 缓存地址 + 回答规矩）+ 最近对话 + 工具清单 + → 讯飞 SDK 文本对话 LLM（端上，不经业务服务器） + ├─ 直接有文字 → 讯飞 SDK 在线合成，区域 RegionMSFT，音色 = 第三节 + └─ 返回 functionCall → 手机执行工具（最多两轮）→ 结果喂回 LLM → 再合成 +``` + +两条入口（`DeviceAiSessionService` 耳机唤醒、`EmaiController` 按住说话）共用这一个编排。PCM 从哪来、喇叭往哪放，仍由各自的会话服务决定。 + +空识别不调用对话模型。工具失败时合成「暂时查不到」，不让模型编一个天气或一条新闻。工具来回超过两轮就停，用已经拿到的结果回答；还什么都没有就说查不到。 + +对话模型用 SDK 的文本 `LLM`（海外 `domain=openai`，文档点名 `gpt-5.3-chat` / `gpt-5.4`）。听写和合成才是 SDK 里的微软区域（`RegionMSFT`）。会议总结走同一套 OpenAI 文本模型，不走星火，也不拿来做 EMAI 的实时对话。 + +--- + +## 三、识别语种和合成音色 + +跟设置里的**界面语言**走，不跟用户刚说的那一句走。 + +设置页切换语言时，`settings_controller` 已经把该语言收成一条 ASR 区域码（跟随系统则用匹配后的系统语言，兜底 `en-US`）。EMAI 直接用这一条： + +| | 取值 | +|---|---| +| 实时转写 `lang` | 上面那条区域码，例如界面中文 → `zh-CN` | +| 合成发音人 | 同一条语言在音色表里的神经语音。性别只有 `1` 用男声；未设置 `0` 和女 `2` 用女声。用户对象还没就绪时用女声 | +| 回答语言 | 与这条界面语言一致，写进系统提示 | + +用户用别的语言插一句，本轮仍按界面语言识别和回答。要换语种，去设置里改，不在对话中途猜。 + +--- + +## 四、问题分成四类 + +系统提示里写明：时间、星期、日期、地点只用本轮给出的事实，不要调用工具，也不要改口。 + +| 用户在问 | 谁处理 | 落地 | +|---|---|---| +| 几点、星期几、今天几号 | 模型直接答 | 只用本轮时间。不调工具 | +| 我在哪 | 模型直接答 | 有地址就念地址。没有就说还没有位置，不要编一个城市 | +| 天气 | 手机执行 `hefeng_weather` | 城市用缓存里的城市名，日期用本轮日期。有城市就不要反问「哪个城市」。没有位置时才问一次城市，问到了再查 | +| 新闻、实时信息 | 手机执行联网搜索 | 搜索词带上本轮日期，避免「今天」落到模型的训练截止日期。海外节点用 `tavily_web_search`，国内节点用 `bocha_web_search`，和现有 MCP 分组一致 | +| 我的会议、记忆、待办 | 手机执行个人 MCP | 见第五节。没有的就说没有，不要编 | + +天气和搜索的密钥留在现有 MCP 进程里。手机只发起这一次工具调用，不把天气接口嵌进对话链路，也不把密钥放进安装包。 + +和风天气的入参是城市名加可选日期，不是经纬度。所以坐标只在缓存里，给不了城市名时不能拿坐标假装能查。 + +--- + +## 五、个人 MCP + +手机调用，数据仍在用户自己的服务器上。这一跳是工具，不是把整段对话交给业务接口。 + +沿用现在这 7 个、且要身份的工具： + +`search_meeting_notes`、`get_memory_items`、`get_memory_stats`、`get_memory_report`、`get_user_tasks`、`add_user_task`、`cancel_user_task`。 + +身份仍用 `user_getmcptoken` 的短时令牌，按工具名附在调用上。令牌不写进系统提示，不进历史，不出现在给模型的工具参数里。模型只看到工具名和参数说明。 + +百炼时期令牌要按工具名嵌进 `user_defined_params`，是因为第三方云在代调。换成手机自己调之后，手机在发 MCP 请求时带上令牌即可，形状不再跟百炼的嵌套走。 + +`hefeng_weather` 不在这 7 个里面。它和联网搜索不需要用户身份，手机调用时不要附登录令牌。 + +--- + +## 六、和现在百炼的对应 + +| 百炼现在 | 换成 | +|---|---| +| 会话开始时把日期、地址放进 `user_prompt_params`，整场不变 | 每轮重算时间；地址仍只读缓存 | +| `client_info.location` 把经纬度交给云端 | 经纬度不进模型 | +| 云端插件自己搜网页、自己查天气 | 模型只返回要调哪个工具；手机执行，再把结果喂回去 | +| 个人 MCP 由云端代调，令牌按工具名塞进会话 | 手机自己调，令牌只附在那一次请求上 | +| 边听边说 | 做不到。说完一句再答 | + +测试服 `aiOnlyIflytek` 已经让百炼 `start` 直接返回失败。本文落地前 EMAI 在测试服起不来。不要为了对照把百炼重新打开。 + +--- + +## 七、明确不做 + +- 不把对话转发到 `user_*` 业务接口，也不在服务端再包一层「代答」。 +- 不把会议总结的星火接口拿来当 EMAI 的对话模型。 +- 不在每一轮阻塞等 GPS。 +- 不把经纬度、令牌、天气密钥写进提示词。 +- 不改耳机协议字节，不改通话 / 同声 / 面对面 / 音视频的选路。 +- 第一版不恢复百炼的全双工。工具调用失败不影响「问一句有回答」。 + +--- + +## 八、验收 + +1. 问「现在几点」「今天星期几」：答的是手机本地时间，和会话开始时的时间可以不同。 +2. 定位开着时问「我在哪」：念缓存地址，不念经纬度。关掉定位再问：说没有位置，不编城市。 +3. 有城市缓存时问天气：不反问城市，答案来自工具。把工具结果改成失败时，嘴里是「暂时查不到」，不是编出来的温度。 +4. 问「今天新闻」：搜索词里带当天日期，回答来自搜索结果。 +5. 问「我最近的会议 / 待办」：走个人 MCP，换一个账号看不到上一个账号的数据。抓对话请求里看不到令牌。 +6. 设置改成英语后，识别和回答都是英语，音色是该语言的女声；性别设为男之后换成男声。 +7. 耳机按键和 App 按住说话，问同一句，走的是同一套编排。 diff --git a/docs/同声翻译-微软端到端-优化.md b/docs/同声翻译-微软端到端-优化.md new file mode 100644 index 00000000..2bf91b19 --- /dev/null +++ b/docs/同声翻译-微软端到端-优化.md @@ -0,0 +1,184 @@ +# 同声翻译改走微软端到端 + +> 2026-10-18。只覆盖翻译页的同声模式（`simultaneous`）。本文只做设计，不改代码。 +> 端到端依据 iOS 集成文档《同声传译简介》第 9.3、9.6 节。拼接时的识别依据安卓海外文档《实时语音转写》第 8.2、9.3 节（iOS《实时语音转写简介》只写了微软模式必填 `language_name`，没有语种表）。合成依据安卓海外文档《在线语音合成》的微软发音人表。 +> 同声这一条覆盖 [科大讯飞SparkChain-全面替换第三方-落地文档.md](科大讯飞SparkChain-全面替换第三方-落地文档.md) 第 5.1 节里「表外拆成星火听写」的写法。面对面、通话、音视频、EMAI 不在本文范围。 + +## 结论 + +两端语种都在微软同传表里：只开一条 `AST`，机房 `RegionMSFT`（安卓 `RegionType.MSFT_TYPE`）。识别、翻译、合成由这一条会话完成。 + +源或目标是中文、粤语、爱尔兰语：微软同传表没有这三语，改走拼接。拼接的识别必须是微软实时转写 `RTASR(RegionMSFT)`，用 `language_name` 指定语种。不要用星火听写 `ASR`（`zh_cn` / `cn_cbm` / `iat`），也不要用新加坡、欧洲机房那两张转写表（`ist_open`、`ist_mul_sp`、`ga_IE`）。 + +拼接的合成用 `OnlineTTS(RegionMSFT)`，发音人用微软神经音色。不要用 `xiaoyan`、`x_xiaoyan`、`x_Catherine`，也不要打 `tts-api.xfyun.cn`。 + +## 现在为什么一种声音念所有语言 + +同声在讯飞听写成功之后，是三段接起来的，没有用到 `AST`。 + +1. `_startSparkListen` 调 `SparkChainSession.startListen`，只做听写。 +2. 整句文本再走原来的翻译。 +3. `playTranslatedText` 算出的 `voiceName` 在 `_sparkListen` 分支被丢掉，`_playSparkTts` 只把目标语种短码传给 `SparkChainSession.speak`。 + +原生合成不看音色表： + +| 端 | 英语 | 其余语种 | +|---|---|---| +| iOS `ClassicTts`，打 `tts-api.xfyun.cn` | `x2_catherine` | `xiaoyan` | +| Android `OnlineTTS` | `x_Catherine` | `x_xiaoyan`（爱尔兰语例外，用 `ga-IE-OrlaNeural`） | + +日语、韩语、法语都会用中文晓燕念出来。`getIflytekTTSLanguageSpecs()` 里按语种分好的发音人没有接到这条播报上。那张表是讯飞发音人，本方案也不再用。 + +## 目标链路 + +麦克风 PCM 仍由现在的耳机麦或手机麦采集，SDK 不自己开麦。采集、保活、播放路由保持现有调用点。 + +``` +耳机麦 / 手机麦 PCM + │ 16 kHz / 16 bit / 单声道，每 40ms 1280 字节 + ▼ +AST(RegionMSFT) + start(源语种, "pcm_s16le", 16000) + use_tts = true + target_lang = 目标语种 + tts_target_lang = 目标语种 + tts_speaker = 该目标语种的默认发音人 + │ + ├─ resType=asr / trans → 原文、译文上屏（流式，不再发翻译请求） + └─ resType=tts → ttsResult.audio 交给现有 PcmStreamPlayer +``` + +iOS： + +```objc +AST *ast = [[AST alloc] initWithregion:RegionMSFT delegate:self]; +[ast addStringParam:@"target_lang" value:@"en"]; +[ast addBoolParam:@"use_tts" value:YES]; +[ast addStringParam:@"tts_target_lang" value:@"en"]; +[ast addStringParam:@"tts_speaker" value:@"JennyNeural"]; +[ast start:@"ja" audioEncoding:@"pcm_s16le" sampleRate:16000 userInfo:tag]; +``` + +安卓用同一个机房枚举和同一组参数名。这条端到端会话的 `tts_speaker` 用第 9.6 节的短名（`JennyNeural`）。`en-US-JennyNeural` 是拼接路径里 `OnlineTTS` 的写法，不要填进 `AST`。 + +方向固定为用户选的源 → 目标。不要用识别回来的语种对调方向。 + +停的顺序不变：先停推流和 `AST.stop(NO)`，再关耳机麦，再放保活。 + +## 语种和音色 + +源语种、目标语种都必须在下表里。默认发音人取该行的第一个；同行其余名字是可换的音色，换的时候整段会话重开，不要在 `write` 过程中改 `tts_speaker`。 + +界面短码和表里的 `language` 一致的直接用。应用里的 `zh` 不要写成 `cn` 再送进微软会话：微软这张表没有中文。 + +| 语种 | language | 默认发音人 | 还可选 | +|---|---|---|---| +| 阿拉伯语 | ar | AbdullahNeural | AliNeural、AmalNeural、AmanyNeural、AminaNeural、AyshaNeural、BasselNeural、FahedNeural、FatimaNeural、HamdanNeural、HamedNeural、HediNeural、ImanNeural、IsmaelNeural、JamalNeural、LailaNeural、LaithNeural、LaylaNeural、MaryamNeural、MoazNeural、MounaNeural、NouraNeural、OmarNeural、RamiNeural、RanaNeural、ReemNeural、SalehNeural、SalmaNeural、SanaNeural、ShakirNeural、TaimNeural、ZariyahNeural | +| 保加利亚语 | bg | BorislavNeural | KalinaNeural | +| 孟加拉语 | bn | BashkarNeural | NabanitaNeural、PradeepNeural、TanishaaNeural | +| 加泰罗尼亚语 | ca | AlbaNeural | EnricNeural、JoanaNeural | +| 捷克语 | cs | AntoninNeural | VlastaNeural | +| 丹麦语 | da | ChristelNeural | JeppeNeural | +| 德语 | de | AmalaNeural | BerndNeural、ChristophNeural、ConradNeural、ElkeNeural、GiselaNeural、IngridNeural、JanNeural、JonasNeural、KasperNeural、KatjaNeural、KillianNeural、KlarissaNeural、KlausNeural、LeniNeural、LouisaNeural、MajaNeural、RalfNeural、TanjaNeural | +| 希腊语 | el | AthinaNeural | NestorasNeural | +| 英语 | en | JennyNeural | GuyNeural、AriaNeural、DavisNeural、JaneNeural、JasonNeural、NancyNeural、TonyNeural、WayneNeural、WilliamNeural、RyanNeural、SoniaNeural、LibbyNeural、NatashaNeural、ClaraNeural、LiamNeural、NeerjaNeural、PrabhatNeural、LunaNeural | +| 西班牙语 | es | ElviraNeural | AlvaroNeural、AbrilNeural、ArnauNeural、DarioNeural、ElenaNeural、TomasNeural、DaliaNeural、JorgeNeural、PalomaNeural、AlonsoNeural | +| 爱沙尼亚语 | et | AnuNeural | KertNeural | +| 波斯语 | fa | DilaraNeural | FaridNeural | +| 芬兰语 | fi | HarriNeural | NooraNeural、SelmaNeural | +| 法语 | fr | DeniseNeural | HenriNeural、AlainNeural、BrigitteNeural、ClaudeNeural、SylvieNeural、JeanNeural、AntoineNeural | +| 印地语 | hi | MadhurNeural | SwaraNeural | +| 匈牙利语 | hu | NoemiNeural | TamasNeural | +| 印尼语 | id | ArdiNeural | GadisNeural | +| 意大利语 | it | ElsaNeural | IsabellaNeural、DiegoNeural、BenignoNeural、FabiolaNeural、GianniNeural | +| 日语 | ja | NanamiNeural | KeitaNeural、AoiNeural、DaichiNeural、MayuNeural、NaokiNeural、ShioriNeural | +| 韩语 | ko | SunHiNeural | InJoonNeural、BongJinNeural、HyunsuNeural、JiMinNeural、SeoHyeonNeural | +| 立陶宛语 | lt | LeonasNeural | OnaNeural | +| 拉脱维亚语 | lv | EveritaNeural | NilsNeural | +| 马来语 | ms | OsmanNeural | YasminNeural | +| 荷兰语 | nl | ArnaudNeural | ColetteNeural、FennaNeural、MaartenNeural | +| 波兰语 | pl | AgnieszkaNeural | MarekNeural、ZofiaNeural | +| 葡萄牙语 | pt | AntonioNeural | BrendaNeural、DuarteNeural、FranciscaNeural、RaquelNeural、YaraNeural | +| 罗马尼亚语 | ro | AlinaNeural | EmilNeural | +| 俄语 | ru | DariyaNeural | DmitryNeural、SvetlanaNeural | +| 斯洛伐克语 | sk | LukasNeural | ViktoriaNeural | +| 斯洛文尼亚语 | sl | PetraNeural | RokNeural | +| 瑞典语 | sv | HilleviNeural | MattiasNeural、SofieNeural | +| 斯瓦希里语 | sw | DaudiNeural | RafikiNeural、RehemaNeural、ZuriNeural | +| 泰米尔语 | ta | AnbuNeural | KaniNeural、KumarNeural、PallaviNeural | +| 泰语 | th | AcharaNeural | NiwatNeural、PremwadeeNeural | +| 土耳其语 | tr | AhmetNeural | EmelNeural | +| 乌克兰语 | uk | OstapNeural | PolinaNeural | +| 乌尔都语 | ur | AsadNeural | GulNeural、SalmanNeural、UzmaNeural | +| 越南语 | vi | HoaiMyNeural | NamMinhNeural | + +文档没有给这些发音人标性别。默认列只是每行的第一个名字，用来保证换语言就换音色。 + +## 中文、粤语、爱尔兰语：拼接，但识别走微软 + +微软同传表没有这三语。源或目标有其中一种时，不启 `AST`，改成三段： + +``` +耳机麦 / 手机麦 PCM + ▼ +RTASR(RegionMSFT) 只识别源语种 + lang = "msft" 固定值，不要把 zh-CN 填进 lang + accent = "mandarin" 固定值 + domain = "ist_mul_msft" 固定值 + language_name = 下表 真正的语种 + ▼ +整句文本翻译 不再次开识别 + ▼ +OnlineTTS(RegionMSFT) 按目标语种选微软神经音色 + ▼ +现有 PcmStreamPlayer +``` + +安卓文档的写法： + +```java +mRTASR.lang("msft"); +mRTASR.OstAccent("mandarin"); +mRTASR.OstDomain("ist_mul_msft"); +mRTASR.setParams("language_name", "zh-CN"); +``` + +iOS 同样用 `RTASR(region: RegionMSFT)`，`language_name` 用下表。现在插件把界面语种直接写进 `engine.lang`，和上面的固定值不符，拼接落地时要改过来。 + +识别语种和这条路径的默认音色： + +| 界面语种 | 识别 `language_name` | 作为译文时的默认音色 | +|---|---|---| +| 普通话 `zh` | `zh-CN` | `zh-CN-XiaoxiaoNeural` | +| 粤语简体 `yue` | `yue-CN` | `yue-CN-XiaoMinNeural` | +| 粤语繁体 / 香港 `zh-HK` | `zh-HK` | `zh-HK-HiuMaanNeural` | +| 爱尔兰语 `ga` | `ga-IE` | `ga-IE-OrlaNeural` | + +另一端如果是微软同传表里的语言，识别用该语言的区域码（英语 `en-US`、日语 `ja-JP`），合成用 `OnlineTTS` 的区域码音色（英语 `en-US-JennyNeural`，对应同传表里的 `JennyNeural`）。不要用星火的 `x_John`、`x_Catherine`、`xiaoyan`。 + +现在同声的 `startListen` 在非爱尔兰语时走进 `ASR()`，`language=zh_cn`、`domain=iat`。那是星火听写。中文、粤语、爱尔兰语的拼接不准再进这个分支。 + +翻译是整句文本，方向仍固定为源 → 目标。中间结果只上屏，整句再翻译、再合成。 + +两端都不在微软同传表里、又不是中文、粤语、爱尔兰语：同声不启动，也不退回星火听写。 + +## 改哪些地方 + +| 位置 | 改动 | +|---|---| +| `sparkchain` 插件 iOS / Android | 同传表内：新增 `AST(RegionMSFT)`，`use_tts` 和 `tts_speaker` 按上一节。表外三语：`RTASR` 按上面的四个固定参数加 `language_name` 启动；`OnlineTTS(RegionMSFT)` 用微软神经音色。同声路径不再创建 `ASR`，不再走 `ClassicTts` | +| `sparkchain_session.dart` | 同声按语种对选一条：`startAst` 或 `startMsftListen`。`startListen` 里现在的星火 `ASR` 不给同声用 | +| `translation_controller.dart` 同声分支 | 表内：PCM 写入 `AST`，译文用 `trans`，音频用 `tts`。表外三语：PCM 写入微软 `RTASR`，整句再翻译，再用 `OnlineTTS` 播目标语种。播放仍进现在的 `PcmStreamPlayer` | +| 发音人 | 表内用同传短名；拼接用上表的 `xx-XX-NameNeural`。不要读 `getIflytekTTSLanguageSpecs` | + +`SparkChain.init` 的 `apiType` 仍按现有配置。`RegionMSFT` 是这一次会话的机房，不要为了同声把全局节点改掉。 + +## 验收 + +- 日语 → 英语：一条 `AST`，`tts_speaker=JennyNeural`。没有 `ASR`，没有 `xiaoyan` / `tts-api.xfyun.cn`。 +- 英语 → 日语：`tts_speaker=NanamiNeural`。两种方向的声音不同。 +- 中文 → 英语：日志是 `RTASR`、`language_name=zh-CN`、`lang=msft`。译文用 `en-US-JennyNeural`。没有 `ASR`、没有 `zh_cn`、没有晓燕。 +- 英语 → 中文：识别 `language_name=en-US`，播报 `zh-CN-XiaoxiaoNeural`。 +- 中文 → 爱尔兰语：识别 `zh-CN`，播报 `ga-IE-OrlaNeural`。 +- 粤语 → 英语：识别 `yue-CN` 或 `zh-HK`，对上用户选的简体或香港繁体。 +- 连耳机时收音日志仍是「同声翻译收音：耳机麦」，译文从耳机出。