diff --git a/apps/client/lib/core/translations/language/en_us.dart b/apps/client/lib/core/translations/language/en_us.dart index 5ec866c9..ab573540 100644 --- a/apps/client/lib/core/translations/language/en_us.dart +++ b/apps/client/lib/core/translations/language/en_us.dart @@ -74,6 +74,8 @@ const Map enUS = { "unknownError": "Unknown error", "asrConnectionFailed": "Speech recognition service connection failed. Please check network and try again.", + "azureSpeechMissing": + "Microsoft speech settings were not received. Speech recognition is unavailable.", "justNow": "Just now", "minutesAgo": "@minutes minutes ago", "yesterday": "Yesterday", diff --git a/apps/client/lib/core/translations/language/zh_cn.dart b/apps/client/lib/core/translations/language/zh_cn.dart index 22be5902..78a1cdf6 100644 --- a/apps/client/lib/core/translations/language/zh_cn.dart +++ b/apps/client/lib/core/translations/language/zh_cn.dart @@ -73,6 +73,7 @@ const Map zhCN = { "responseTimeout": "响应超时", "unknownError": "未知错误", "asrConnectionFailed": "语音识别服务连接失败，请检查网络或稍后重试", + "azureSpeechMissing": "未获取到微软语音配置，语音识别暂不可用", "justNow": "刚刚", "minutesAgo": "@minutes分钟前", "yesterday": "昨天", diff --git a/apps/client/lib/core/utils/app_lifecycle_manager.dart b/apps/client/lib/core/utils/app_lifecycle_manager.dart index 56090710..abeaaf7d 100644 --- a/apps/client/lib/core/utils/app_lifecycle_manager.dart +++ b/apps/client/lib/core/utils/app_lifecycle_manager.dart @@ -8,6 +8,7 @@ import '../../modules/meeting/controllers/meeting_record_controller.dart'; import '../../routes/app_routes.dart'; import 'logger.dart'; import '../../data/models/user_Info.dart'; +import '../../data/services/ai_vendor_lock.dart'; import '../../data/services/memory_reminder_scheduler.dart'; import '../../data/services/memory_service.dart'; import '../../modules/main_tab/views/widgets/memory_daily_sheet.dart'; @@ -79,7 +80,8 @@ class AppLifecycleManager extends GetxService with WidgetsBindingObserver { _isMeetingRoute(currentRoute) || _isTranslationRoute(currentRoute); - final bool canWakeUp = !shouldDisableWakeup; + // 测试服禁止唤醒移动精灵，避免按键把会话打到灵犀。 + final bool canWakeUp = !aiOnlyIflytek && !shouldDisableWakeup; if (_lastCanWakeUp == canWakeUp) return; _lastCanWakeUp = canWakeUp; diff --git a/apps/client/lib/core/utils/audio_foreground_service.dart b/apps/client/lib/core/utils/audio_foreground_service.dart index 0c05c3d1..0f6182f5 100644 --- a/apps/client/lib/core/utils/audio_foreground_service.dart +++ b/apps/client/lib/core/utils/audio_foreground_service.dart @@ -35,15 +35,21 @@ class AudioForegroundService { /// /// [pinSpeaker] 为真时静音钉在手机扬声器，避免把耳机 A2DP 抢走。 /// 同声翻译的译文本身就走这条媒体通道，钉住会让译文改从扬声器出来，这时要传 false。 + /// 车载香薰的回复也走 A2DP，同样不能钉。 + /// + /// [holdFocus] 为真时，服务先进入前台，再决定要不要暂停外面的音乐，最后才开静音。 + /// 这一次调用会等到判断做完才返回。后台直接抢焦点会被系统拒绝。 static Future start({ bool keepalive = false, bool pinSpeaker = true, + bool holdFocus = false, }) async { if (!Platform.isAndroid) return; try { await _channel.invokeMethod('startAudioForegroundService', { 'keepalive': keepalive, 'pinSpeaker': pinSpeaker, + 'holdFocus': holdFocus, }); } catch (e) { // 起不来不该阻断录音本身：前台是为了「切后台还能录」， diff --git a/apps/client/lib/data/services/ai_vendor_lock.dart b/apps/client/lib/data/services/ai_vendor_lock.dart new file mode 100644 index 00000000..c00b16cc --- /dev/null +++ b/apps/client/lib/data/services/ai_vendor_lock.dart @@ -0,0 +1,18 @@ +import 'package:flutter_dotenv/flutter_dotenv.dart'; + +import 'network/dio_manager.dart'; + +/// 连着测试服时，AI 只走科大讯飞。微软、阿里、字节、谷歌、灵犀都不要发请求。 +/// +/// 正式域名不进这把锁。密钥还在本地缓存里也不许拿去调别家。 +bool get aiOnlyIflytek { + final hosts = [ + DioManager().baseUrl, + dotenv.env['SERVER_URL'] ?? '', + dotenv.env['HW_SERVER_URL'] ?? '', + ]; + for (final host in hosts) { + if (host.contains('ym-dev.ymaikj.com')) return true; + } + return false; +} diff --git a/apps/client/lib/data/services/alibaba_image_translation_service.dart b/apps/client/lib/data/services/alibaba_image_translation_service.dart index c5d8198d..94c9e068 100644 --- a/apps/client/lib/data/services/alibaba_image_translation_service.dart +++ b/apps/client/lib/data/services/alibaba_image_translation_service.dart @@ -6,6 +6,7 @@ import 'package:http/http.dart' as http; import '../../core/utils/logger.dart'; import '../../data/models/appconfig.dart'; +import 'ai_vendor_lock.dart'; import '../../modules/picture_translation/models/picture_translation_models.dart'; import 'language_manager.dart'; @@ -72,6 +73,10 @@ class AlibabaImageTranslationService extends GetxService { required Uint8List imageBytes, required String targetLanguageCode, }) async { + if (aiOnlyIflytek) { + Logger.w(_tag, '测试服只允许科大讯飞，不调用通义图片理解'); + return null; + } if (!_isInitialized && !await initialize()) return null; try { diff --git a/apps/client/lib/data/services/asr_service.dart b/apps/client/lib/data/services/asr_service.dart index 9e9f95c4..96069055 100644 --- a/apps/client/lib/data/services/asr_service.dart +++ b/apps/client/lib/data/services/asr_service.dart @@ -14,6 +14,9 @@ abstract class AsrService { /// 支持的语言 List get supportedLanguages; + /// 微软订阅密钥和区域是否已经下发。没有时调用方提示即可，不要再调原生。 + bool get configured; + /// 初始化语音识别服务 Future initialize({ List? supportedLanguages, diff --git a/apps/client/lib/data/services/bailian_multimodal_service.dart b/apps/client/lib/data/services/bailian_multimodal_service.dart index fe2d7553..75041eb6 100644 --- a/apps/client/lib/data/services/bailian_multimodal_service.dart +++ b/apps/client/lib/data/services/bailian_multimodal_service.dart @@ -7,6 +7,7 @@ import 'package:get/get.dart'; import 'package:web_socket_channel/io.dart'; import '../../core/utils/logger.dart'; +import 'ai_vendor_lock.dart'; import '../models/appconfig.dart'; import '../models/assistant_directive.dart'; import 'device_location_service.dart'; @@ -254,6 +255,10 @@ class BailianMultimodalService extends GetxService { String source = 'emai', String? mcpToken, }) async { + if (aiOnlyIflytek) { + Logger.w(_tag, '测试服只允许科大讯飞，不连接百炼'); + return false; + } if (!isConfigured) { Logger.e(_tag, '配置不全，无法启动会话：$configHint'); _events.add(BailianEvent(BailianEventType.error, diff --git a/apps/client/lib/data/services/bailian_native_dialog.dart b/apps/client/lib/data/services/bailian_native_dialog.dart index 10a9f17f..d7837876 100644 --- a/apps/client/lib/data/services/bailian_native_dialog.dart +++ b/apps/client/lib/data/services/bailian_native_dialog.dart @@ -7,6 +7,7 @@ import 'package:flutter/services.dart'; import '../../core/utils/logger.dart'; import '../models/assistant_directive.dart'; +import 'ai_vendor_lock.dart'; import 'bailian_multimodal_service.dart'; /// 耳机唤醒那条 EMAI 链路的**原生**百炼客户端句柄。 @@ -102,6 +103,10 @@ class BailianNativeDialog { String source = 'earphone', String? mcpToken, }) async { + if (aiOnlyIflytek) { + Logger.w(_tag, '测试服只允许科大讯飞，不连接百炼'); + return false; + } if (!isConfigured) { Logger.e(_tag, '配置不全，无法启动会话：$configHint'); _eventCtl.add(BailianEvent(BailianEventType.error, diff --git a/apps/client/lib/data/services/deapsound_ai_service.dart b/apps/client/lib/data/services/deapsound_ai_service.dart index c31fcf67..4870446d 100644 --- a/apps/client/lib/data/services/deapsound_ai_service.dart +++ b/apps/client/lib/data/services/deapsound_ai_service.dart @@ -5,6 +5,7 @@ import 'package:http/http.dart' as http; import '../models/appconfig.dart'; import 'ai_service.dart'; +import 'ai_vendor_lock.dart'; /// 语音纪要「Ask AI」用的文本大模型（OpenAI 兼容接口，客户端直连）。 /// @@ -49,6 +50,9 @@ class DeapsoundAIService implements AiService { String? systemPrompt, String? userProperties, }) async { + if (aiOnlyIflytek) { + throw StateError('测试服只允许科大讯飞'); + } final url = chatUrl; final headers = { @@ -85,6 +89,9 @@ class DeapsoundAIService implements AiService { String? systemPrompt, String? userProperties, }) async* { + if (aiOnlyIflytek) { + throw StateError('测试服只允许科大讯飞'); + } final client = http.Client(); final url = chatUrl; diff --git a/apps/client/lib/data/services/device_ai_session_service.dart b/apps/client/lib/data/services/device_ai_session_service.dart index a27dbe0f..52bc22b3 100644 --- a/apps/client/lib/data/services/device_ai_session_service.dart +++ b/apps/client/lib/data/services/device_ai_session_service.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:io' show Platform; import 'dart:math'; import 'dart:typed_data'; @@ -464,14 +465,24 @@ class DeviceAiSessionService extends GetxService { // iOS：会话期间保持一个静音的后台音频播放，否则锁屏几秒后进程被挂起， // BLE 上行与 WebSocket 一起停摆（见 BackgroundKeepalive）。Android 空操作。 await BackgroundKeepalive.acquire(_tag); - // 先看外面有没有音乐在播，再开静音保活。顺序反了的话，自己的音轨会让 - // isMusicActive 变成 true，结束时交还焦点就把没在播的音乐拉起来。 - await EmaiAudioSession.holdForDeviceDialog(); - // 华为把退到后台的进程网络掐掉，日志是 Software caused connection abort。 - // 用户那句已经写进记录，回复还在这条 WebSocket 上，连接一断就没有回答。 - // 前台服务把进程钉住。香薰和耳机都要，不能只给走手机播放器的香薰。 - await AudioForegroundService.start(keepalive: true); - _holdsForeground = true; + // 安卓：先成为前台服务再抢焦点，最后才开静音。顺序反了的话， + // 切到音乐 App 之后再唤醒，焦点申请会被拒绝，音乐不暂停，香薰麦一直听到音乐， + // 机体就停在聆听。香薰回复走 A2DP，静音不能钉在手机扬声器上，否则音乐会停、 + // 结束却不在香薰上续上，第二次唤醒音量也不降。 + // iOS 没有这道前台服务，仍在静音循环起来之后再独占会话。 + if (Platform.isAndroid) { + await AudioForegroundService.start( + keepalive: true, + pinSpeaker: !_isEaiCar, + holdFocus: true, + ); + _holdsForeground = true; + await EmaiAudioSession.holdForDeviceDialog(focusAlreadyTaken: true); + } else { + await EmaiAudioSession.holdForDeviceDialog(); + await AudioForegroundService.start(keepalive: true); + _holdsForeground = true; + } Logger.w(_tag, '设备对话已开前台服务'); // ★ 先试原生直连：上行 PCM 与合成音频都不经过 Dart。 @@ -1140,8 +1151,9 @@ class DeviceAiSessionService extends GetxService { } if (wasActive && notifyDevice && s != null) await _sendExit(s); _session = null; - // 先停静音保活，再交还会话。保活还在播时 deactivate 会被系统拒绝， - // 外部音乐收不到继续播放的通知。 + // 先停 iOS 静音循环，再交还会话。循环还在播时 deactivate 会被系统拒绝， + // 外部音乐收不到继续播放的通知。安卓的静音轨在交还焦点时先停，播放键 + // 才能送到音乐 App；前台服务留到键发出去再停。 await BackgroundKeepalive.release(_tag); await EmaiAudioSession.releaseDeviceDialog(); if (_holdsForeground) { diff --git a/apps/client/lib/data/services/microsoft_text_translation.dart b/apps/client/lib/data/services/microsoft_text_translation.dart index ee09466f..217aeb02 100644 --- a/apps/client/lib/data/services/microsoft_text_translation.dart +++ b/apps/client/lib/data/services/microsoft_text_translation.dart @@ -4,6 +4,7 @@ import 'package:dio/dio.dart'; import '../../core/utils/logger.dart'; import '../models/appconfig.dart'; +import 'ai_vendor_lock.dart'; /// 微软文本翻译。同声 / 面对面 / 音视频里扩展语种走这条，不经服务端阿里。 /// @@ -27,6 +28,10 @@ class MicrosoftTextTranslation { }) async { final src = text.trim(); if (src.isEmpty) return null; + if (aiOnlyIflytek) { + Logger.w('MsTrans', '测试服只允许科大讯飞，不调用微软翻译'); + return null; + } final key = AppConfig.cred('mt_azure', 'subscription_key', 'AZURE_TRANSLATION_KEY'); if (key == null || key.isEmpty) { diff --git a/apps/client/lib/data/services/server_translation_service.dart b/apps/client/lib/data/services/server_translation_service.dart index 33478406..f0b4629d 100644 --- a/apps/client/lib/data/services/server_translation_service.dart +++ b/apps/client/lib/data/services/server_translation_service.dart @@ -4,6 +4,8 @@ import 'package:get/get.dart'; import '../../core/utils/logger.dart'; import 'microsoft_text_translation.dart'; import 'network/api.dart'; +import 'sparkchain_session.dart'; +import 'ai_vendor_lock.dart'; /// 文本机器翻译：由**服务端**代调后台「第三方服务配置」里启用的 MT 服务。 /// @@ -60,10 +62,47 @@ class ServerTranslationService extends GetxService { required String targetLanguageCode, CancelToken? cancelToken, Duration? timeout = realtimeTimeout, + bool tryOnDevice = true, }) async { final src = text.trim(); if (src.isEmpty) return null; - if (microsoftMtRequired(sourceLanguageCode, targetLanguageCode)) { + if (!tryOnDevice) { + return _translateRemote( + src: src, + sourceLanguageCode: sourceLanguageCode, + targetLanguageCode: targetLanguageCode, + cancelToken: cancelToken, + timeout: timeout, + ); + } + final onDevice = await SparkChainSession.translate( + from: sourceLanguageCode, + to: targetLanguageCode, + text: src, + ); + if (onDevice != null) { + lastProvider.value = 'sparkchain'; + lastSvcId.value = 'sparkchain'; + return onDevice; + } + return _translateRemote( + src: src, + sourceLanguageCode: sourceLanguageCode, + targetLanguageCode: targetLanguageCode, + cancelToken: cancelToken, + timeout: timeout, + ); + } + + Future _translateRemote({ + required String src, + required String sourceLanguageCode, + required String targetLanguageCode, + CancelToken? cancelToken, + Duration? timeout, + }) async { + if (src.isEmpty) return null; + if (!aiOnlyIflytek && microsoftMtRequired(sourceLanguageCode, targetLanguageCode)) { final from = microsoftMtCode(sourceLanguageCode); final to = microsoftMtCode(targetLanguageCode); Logger.i(_tag, '阿里翻译覆盖不了，改走微软: $from→$to'); diff --git a/apps/client/lib/data/services/sparkchain_session.dart b/apps/client/lib/data/services/sparkchain_session.dart new file mode 100644 index 00000000..ff618e46 --- /dev/null +++ b/apps/client/lib/data/services/sparkchain_session.dart @@ -0,0 +1,174 @@ +import 'dart:async'; +import 'dart:typed_data'; + +import 'package:sparkchain/sparkchain.dart'; + +import '../../core/utils/logger.dart'; +import '../models/appconfig.dart'; + +/// 端上讯飞翻译。没配密钥就返回 null，调用方继续走原来的服务端翻译。 +class SparkChainSession { + SparkChainSession._(); + + static const _tag = 'SparkChain'; + static bool _ready = false; + static bool _eventsAttached = false; + static bool listening = false; + static int _seq = 0; + static final Map>> _waiters = {}; + static final Map _ttsHandlers = {}; + static final Map> _ttsWaiters = {}; + + /// 听写结果。只在 [startListen] 成功之后、[stopListen] 之前有意义。 + static void Function(Map event)? asrListener; + + static Future ensure() async { + if (_ready) return true; + final appId = AppConfig.cred('sparkchain', 'app_id', 'SPARKCHAIN_APP_ID'); + final apiKey = AppConfig.cred('sparkchain', 'api_key', 'SPARKCHAIN_API_KEY'); + final apiSecret = + AppConfig.cred('sparkchain', 'api_secret', 'SPARKCHAIN_API_SECRET'); + if (appId == null || apiKey == null || apiSecret == null) { + return false; + } + if (!_eventsAttached) { + _eventsAttached = true; + SparkChain.events.listen(_onEvent); + } + final apiType = + int.tryParse(AppConfig.cred('sparkchain', 'api_type', 'SPARKCHAIN_API_TYPE') ?? '') ?? + 1; + final code = await SparkChain.init( + appId: appId, + apiKey: apiKey, + apiSecret: apiSecret, + apiType: apiType, + ); + Logger.i(_tag, 'init code=$code apiType=$apiType'); + _ready = code == 0; + return _ready; + } + + static void _onEvent(Map event) { + final id = event['id']?.toString() ?? ''; + final kind = event['kind']; + if (kind == 'its' || (kind == 'error' && event['where'] == 'its')) { + final waiter = _waiters[id]; + if (waiter != null && !waiter.isCompleted) waiter.complete(event); + } + if (kind == 'asr' || (kind == 'error' && event['where'] == 'asr')) { + asrListener?.call(event); + } + if (kind == 'tts' || (kind == 'error' && event['where'] == 'tts')) { + _onTts(id, event); + } + } + + static void _onTts(String id, Map event) { + if (event['kind'] == 'error') { + _ttsHandlers.remove(id); + final waiter = _ttsWaiters.remove(id); + if (waiter != null && !waiter.isCompleted) { + waiter.completeError(StateError('${event['code']}')); + } + return; + } + final raw = event['pcm']; + final pcm = raw is Uint8List ? raw : Uint8List(0); + _ttsHandlers[id]?.call(pcm, event['final'] == true); + if (event['final'] == true) { + _ttsHandlers.remove(id); + final waiter = _ttsWaiters.remove(id); + if (waiter != null && !waiter.isCompleted) waiter.complete(); + } + } + + /// 只听 [lang] 这一种。ga 走实时转写，其余走听写。SDK 不自己开麦。 + static Future startListen(String lang) async { + if (!await ensure()) return false; + final code = await SparkChain.startListen(id: 'listen', lang: sparkChainShortCode(lang)); + listening = code == 0; + if (!listening) { + Logger.w(_tag, 'startListen 失败 code=$code'); + } + return listening; + } + + static void write(Uint8List pcm) { + if (!listening || pcm.isEmpty) return; + unawaited(SparkChain.write(pcm)); + } + + static Future stopListen() async { + if (!listening) return; + listening = false; + asrListener = null; + try { + await SparkChain.stopListen(); + } catch (e) { + Logger.w(_tag, 'stopListen: $e'); + } + } + + /// 合成一段。PCM 按到达顺序交给 [onPcm]，整段结束或失败才返回。 + static Future speak({ + required String lang, + required String text, + required void Function(Uint8List pcm, bool isFinal) onPcm, + }) async { + if (!await ensure()) { + throw StateError('sparkchain not ready'); + } + final id = 'tts-${++_seq}'; + final waiter = Completer(); + _ttsWaiters[id] = waiter; + _ttsHandlers[id] = onPcm; + final code = await SparkChain.speak( + id: id, + lang: sparkChainShortCode(lang), + text: text, + ); + if (code != 0) { + _ttsHandlers.remove(id); + _ttsWaiters.remove(id); + throw StateError('speak $code'); + } + await waiter.future.timeout(const Duration(seconds: 20)); + } + + /// 短码 zh / en / ga。失败返回 null，不抛。 + static Future translate({ + required String from, + required String to, + required String text, + }) async { + if (!await ensure()) return null; + final id = '${++_seq}'; + final waiter = Completer>(); + _waiters[id] = waiter; + try { + final code = await SparkChain.translate( + id: id, + from: sparkChainShortCode(from), + to: sparkChainShortCode(to), + text: text, + ); + if (code != 0) { + Logger.w(_tag, 'translate 启动失败 code=$code'); + return null; + } + final event = await waiter.future.timeout(const Duration(seconds: 12)); + if (event['kind'] != 'its') { + Logger.w(_tag, 'translate 失败 code=${event['code']} sid=${event['sid']}'); + return null; + } + final out = (event['text'] as String?)?.trim() ?? ''; + return out.isEmpty ? null : out; + } on TimeoutException { + Logger.w(_tag, 'translate 超时'); + return null; + } finally { + _waiters.remove(id); + } + } +} diff --git a/apps/client/lib/data/services/speech_impl/azure_asr_service.dart b/apps/client/lib/data/services/speech_impl/azure_asr_service.dart index 7d52c42a..1d6c26af 100644 --- a/apps/client/lib/data/services/speech_impl/azure_asr_service.dart +++ b/apps/client/lib/data/services/speech_impl/azure_asr_service.dart @@ -4,6 +4,7 @@ import 'dart:typed_data'; import '../../../data/models/appconfig.dart'; import 'package:flutter/services.dart'; import '../../../core/utils/logger.dart'; +import '../ai_vendor_lock.dart'; import 'package:get/get.dart'; import '../asr_service.dart'; @@ -73,12 +74,14 @@ class AzureAsrService extends GetxService implements AsrService { _xunfeiAccessKeySecret = AppConfig.cred('stt_iflytek', 'access_key_secret', 'XUNFEI_ASR_ACCESS_KEY_SECRET') ?? ''; - if (_subscriptionKey.isEmpty || _serviceRegion.isEmpty) { - throw Exception( - '未找到 Azure 语音服务配置。请在 .env 文件中设置 AZURE_SPEECH_KEY 和 AZURE_SPEECH_REGION'); - } + // 密钥没下发时留空。构造函数里抛异常会让翻译页直接起不来。 + if (_subscriptionKey.isEmpty || _serviceRegion.isEmpty) return; } + @override + bool get configured => + _subscriptionKey.isNotEmpty && _serviceRegion.isNotEmpty; + /// 设置事件通道 void _setupEventChannel() { _eventSubscription?.cancel(); @@ -108,6 +111,15 @@ class AzureAsrService extends GetxService implements AsrService { bool useExternalAudio = false, bool useEchoCancellation = false, }) async { + if (!configured) { + Logger.w('AzureASR', '未获取到微软语音配置'); + _isInitialized = false; + return false; + } + if (aiOnlyIflytek) { + Logger.w('AzureASR', '测试服只允许科大讯飞，不初始化微软听写'); + return false; + } try { final List languages = supportedLanguages ?? _defaultSupportedLanguages; @@ -141,7 +153,7 @@ class AzureAsrService extends GetxService implements AsrService { } catch (e) { Logger.error('Azure 语音识别服务初始化失败: ${e.toString()}'); _isInitialized = false; - rethrow; + return false; } } @@ -211,7 +223,11 @@ class AzureAsrService extends GetxService implements AsrService { _lastStartAt = now; if (!_isInitialized) { - await initialize(); + final ok = await initialize(); + if (!ok || !_isInitialized) { + Logger.w('AzureASR', '听写未就绪，不启动微软连续识别'); + return false; + } } if (_isContinuousRecognitionActive) { @@ -235,14 +251,15 @@ class AzureAsrService extends GetxService implements AsrService { }); if (!result) { - throw Exception('启动连续识别失败'); + Logger.w('AzureASR', '启动连续识别失败'); + return false; } _isContinuousRecognitionActive = true; return result; } catch (e) { Logger.error('开始连续语音识别失败: ${e.toString()}'); - rethrow; + return false; } } @@ -278,7 +295,11 @@ class AzureAsrService extends GetxService implements AsrService { @override Future> recognizeCallback() async { if (!_isInitialized) { - await initialize(); + final ok = await initialize(); + if (!ok || !_isInitialized) { + Logger.w('AzureASR', '听写未就绪，不挂微软回调'); + return const Stream.empty(); + } } try { _eventStreamController = StreamController.broadcast(); diff --git a/apps/client/lib/data/services/speech_impl/azure_ast_service.dart b/apps/client/lib/data/services/speech_impl/azure_ast_service.dart index 0c042cc8..c392be92 100644 --- a/apps/client/lib/data/services/speech_impl/azure_ast_service.dart +++ b/apps/client/lib/data/services/speech_impl/azure_ast_service.dart @@ -2,6 +2,7 @@ import 'dart:async'; import '../../../data/models/appconfig.dart'; import 'package:flutter/services.dart'; import '../../../core/utils/logger.dart'; +import '../ai_vendor_lock.dart'; import 'package:get/get.dart'; import '../ast_service.dart'; @@ -293,6 +294,8 @@ class AzureAstService extends GetxService implements AstService { ? 'wss://dashscope.aliyuncs.com/api-ws/v1/realtime' : aliUrl; if (_subscriptionKey.isEmpty || _serviceRegion.isEmpty) { + // 测试服停用微软后，这里再抛会让翻译页在画出第一帧之前失败，发布包就是一整块灰。 + if (aiOnlyIflytek) return; throw Exception( '未找到 Azure 语音服务配置。请在 .env 文件中设置 AZURE_SPEECH_KEY 和 AZURE_SPEECH_REGION'); } @@ -489,6 +492,10 @@ class AzureAstService extends GetxService implements AstService { List? supportedLanguages, String provider = 'azure', }) async { + if (aiOnlyIflytek) { + Logger.w('AzureAST', '测试服只允许科大讯飞，不初始化通话翻译'); + return false; + } _lastProvider = provider; if (supportedLanguages != null) _lastLanguages = List.from(supportedLanguages); try { diff --git a/apps/client/lib/data/services/speech_impl/azure_tts_service.dart b/apps/client/lib/data/services/speech_impl/azure_tts_service.dart index 5b75c10c..fbd0e916 100644 --- a/apps/client/lib/data/services/speech_impl/azure_tts_service.dart +++ b/apps/client/lib/data/services/speech_impl/azure_tts_service.dart @@ -3,6 +3,7 @@ import '../../../data/models/appconfig.dart'; import 'package:flutter/services.dart'; import 'package:get/get.dart'; import '../../../core/utils/logger.dart'; +import '../ai_vendor_lock.dart'; import '../tts_service.dart'; /// 微软 Text-to-Speech 服务 @@ -131,6 +132,10 @@ class AzureTtsService extends GetxService implements TtsService { Future initialize({ List? supportedLanguages, }) async { + if (aiOnlyIflytek) { + Logger.w('AzureTTS', '测试服只允许科大讯飞，不初始化微软合成'); + return false; + } // 检查是否需要重新初始化 if (_isInitialized) { await dispose(); diff --git a/apps/client/lib/data/utils/background_keepalive.dart b/apps/client/lib/data/utils/background_keepalive.dart index d454dfea..3321e346 100644 --- a/apps/client/lib/data/utils/background_keepalive.dart +++ b/apps/client/lib/data/utils/background_keepalive.dart @@ -63,7 +63,8 @@ class BackgroundKeepalive { Logger.i(_tag, '当前音频类别 $cur 已支持后台音频，不改类别'); } await session.setActive(true); - final p = AudioPlayer(); + // 播放器自己再 setActive 会盖掉后面独占会话的时机，结束时音乐就续不上。 + final p = AudioPlayer(handleAudioSessionActivation: false); await p.setAsset(_asset); await p.setLoopMode(LoopMode.one); await p.setVolume(0); diff --git a/apps/client/lib/data/utils/emai_audio_session.dart b/apps/client/lib/data/utils/emai_audio_session.dart index 82ddc760..d56489d0 100644 --- a/apps/client/lib/data/utils/emai_audio_session.dart +++ b/apps/client/lib/data/utils/emai_audio_session.dart @@ -63,10 +63,18 @@ class EmaiAudioSession { /// 设备按键唤醒的对话（EaiCar 与耳机 EMAI）一开始就占住手机上的媒体播放。 /// /// 已经占住时直接返回，同一次唤醒不重复激活会话。失败只记日志，对话照常进行。 - static Future holdForDeviceDialog() async { + /// + /// [focusAlreadyTaken]：Android 上焦点是前台服务进前台之后抢的，这里只登记 + /// 「这场对话占着」，避免后面的播放器再去申请更弱的焦点把音乐放出来。 + static Future holdForDeviceDialog({bool focusAlreadyTaken = false}) async { if (!_apply || _deviceDialogHeld) return; _deviceDialogHeld = true; try { + if (focusAlreadyTaken) { + _configured = EmaiAudioProfile.playbackOnly; + Logger.w(_tag, '媒体播放已由前台服务占住'); + return; + } if (Platform.isAndroid) { // 不走 audio_session.setActive：它在已有焦点对象时直接返回 true， // 并且申请的是 GAIN_TRANSIENT，华为上音乐可以继续播。 @@ -80,14 +88,20 @@ class EmaiAudioSession { Logger.w(_tag, '系统没在播音乐，不独占会话 before=$before'); return; } - await _setNowPlaying(true); final session = await AudioSession.instance; await session.configure(_playbackOnlyConfig()); _configured = EmaiAudioProfile.playbackOnly; - final ok = await session.setActive(true); + var ok = await session.setActive(true); + var after = await _externalPlaying(); + if (after == true) { + Logger.w(_tag, '独占后外部还在播，重新激活一次'); + await session.setActive(false); + ok = await session.setActive(true); + after = await _externalPlaying(); + } _iosInterruptedOthers = ok == true; + await _setNowPlaying(true); await BackgroundKeepalive.keepPlaying(); - final after = await _externalPlaying(); Logger.w(_tag, '占住媒体播放 ok=$ok 外部播放 $before → $after'); } catch (e) { Logger.w(_tag, '占住媒体播放失败（对话继续）: $e'); @@ -116,6 +130,9 @@ class EmaiAudioSession { _iosInterruptedOthers = false; final before = await _externalPlaying(); final session = await AudioSession.instance; + // 先摘掉自己的 Now Playing。还挂着的话，系统会把「继续播放」发给我们， + // 音乐 App 就一直停着，直到用户切进去。 + await _setNowPlaying(false); final ok = await session.setActive( false, avAudioSessionSetActiveOptions: @@ -125,7 +142,6 @@ class EmaiAudioSession { // 这里不再 setActive，否则外部音乐刚被通知继续又会被打断。 await session.configure(_mixPlaybackConfig()); _configured = null; - await _setNowPlaying(false); final after = await _externalPlaying(); Logger.i(_tag, '交还媒体播放 ok=$ok 外部播放 $before → $after'); } catch (e) { diff --git a/apps/client/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart b/apps/client/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart index 7f44df8e..b909537d 100644 --- a/apps/client/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart +++ b/apps/client/lib/modules/mobile_elf/controllers/mobile_elf_controller.dart @@ -7,6 +7,7 @@ import 'package:get_storage/get_storage.dart'; import 'package:permission_handler/permission_handler.dart'; import '../../../data/models/appconfig.dart'; +import '../../../data/services/ai_vendor_lock.dart'; import '../../../core/utils/logger.dart'; import '../../../core/utils/permission_util.dart'; @@ -80,8 +81,10 @@ class MobileElfController extends GetxController { ever(ttsEnabled, (bool enabled) { AgentService.setTtsMuted(!enabled); }); - // 主动触发认证检查（页面打开时，确保 credential_failed 事件不被错过） - AgentService.fetchCredentials(); + // 测试服不走移动精灵（灵犀）。凭据拉取会打到中国移动。 + if (!aiOnlyIflytek) { + AgentService.fetchCredentials(); + } } Future _loadChatHistory() async { @@ -252,6 +255,11 @@ class MobileElfController extends GetxController { // ═══════════════════════════════════════════════════════════════════════════ Future startCall() async { + if (aiOnlyIflytek) { + status.value = MobileElfStatus.error; + errorMsg.value = '测试服只允许科大讯飞'; + return; + } if (inCall.value || _callStarting) return; _callStarting = true; _pttAborted = false; @@ -320,6 +328,11 @@ class MobileElfController extends GetxController { } Future startPtt() async { + if (aiOnlyIflytek) { + status.value = MobileElfStatus.error; + errorMsg.value = '测试服只允许科大讯飞'; + return; + } if (pttRecording.value || _pttStarting) return; _pttStarting = true; _pttAborted = false; diff --git a/apps/client/lib/modules/translation/controllers/translation_controller.dart b/apps/client/lib/modules/translation/controllers/translation_controller.dart index e77de5d2..132d0f59 100644 --- a/apps/client/lib/modules/translation/controllers/translation_controller.dart +++ b/apps/client/lib/modules/translation/controllers/translation_controller.dart @@ -27,6 +27,10 @@ import '../../../core/utils/toast_throttle.dart'; import '../../../core/services/floating_window_manager.dart'; import '../../../data/services/asr_service.dart'; +import '../../../data/services/sparkchain_session.dart'; +import '../../../data/services/ai_vendor_lock.dart'; +import '../../../data/utils/mic_pcm_streamer.dart'; +import '../../../data/utils/pcm_stream_player.dart'; import '../../../data/services/usage_stats_service.dart'; import 'package:device_plugin_interface/device_plugin_interface.dart'; import '../../../devices/device_hub.dart'; @@ -231,6 +235,11 @@ class TranslationController extends GetxController with WidgetsBindingObserver { DeviceSession? _simHeadsetSession; DeviceAudioSource? _simHeadsetMic; StreamSubscription? _simHeadsetMicSub; + /// 本轮同声翻译的识别、翻译、合成走讯飞。密钥没配好时为 false，沿用原来的识别。 + bool _sparkListen = false; + MicPcmStreamer? _sparkMic; + StreamSubscription? _sparkMicSub; + PcmStreamPlayer? _sparkPlayer; DeviceFileRecorder? _simHeadsetRec; DeviceAudioSource? _callLocalSrc; DeviceAudioSource? _callPeerSrc; @@ -1600,7 +1609,14 @@ class TranslationController extends GetxController with WidgetsBindingObserver { Logger.w( 'Translation', '开始语音识别，模式：同声传译，收音=${_simHeadsetSession != null ? "耳机麦" : "手机麦克风"}'); - await _startAsrService('phone_call'); + final spark = await _startSparkListen(); + if (!spark) { + if (aiOnlyIflytek) { + Logger.w('Translation', '测试服只允许科大讯飞，同声翻译不改走原识别'); + } else { + await _startAsrService('phone_call'); + } + } await _attachSimultaneousHeadsetMic(); } else if (currentMode.value == 'faceToFace') { final bes = _besFaceToFaceSession(); @@ -1921,7 +1937,9 @@ class TranslationController extends GetxController with WidgetsBindingObserver { // `Pged-Freezer: Freeze process` 冻结进程 → 两条 WebSocket 当场 // `Software caused connection abort` → Dart 当成 AST 故障 → // _terminateCallOnError 把整个通话翻译关掉。用户看到的是「切个后台功能就没了」。 - await AudioForegroundService.start(keepalive: true); + // 静音不能钉在手机扬声器上。华为上 setPreferredDevice(扬声器) 会把整路 + // 输出从耳机拽走，通话和对译文都改从手机出来。跟同声翻译一样，跟着当前输出。 + await AudioForegroundService.start(keepalive: true, pinSpeaker: false); _callSession = await _awaitCallAudioSession(); _callModeUsingBes = _callSession != null; _callAudioTornDown = false; @@ -2245,7 +2263,11 @@ class TranslationController extends GetxController with WidgetsBindingObserver { logged = true; Logger.w('Translation', '同声翻译耳机麦首帧 ${pcm.length} 字节'); } - unawaited(_asrService.pushAudioData(pcm)); + if (_sparkListen) { + SparkChainSession.write(pcm); + } else { + unawaited(_asrService.pushAudioData(pcm)); + } }); } @@ -2269,6 +2291,99 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } } + /// 同声翻译改走讯飞听写。密钥没有配好、或手机麦采集失败时返回 false， + /// 调用方继续用原来的识别，不影响没配讯飞的包。 + Future _startSparkListen() async { + final directed = _directedShortCodes(); + _sparkListen = true; + SparkChainSession.asrListener = _onSparkAsr; + final ok = await SparkChainSession.startListen(directed.source); + if (!ok) { + _sparkListen = false; + SparkChainSession.asrListener = null; + Logger.w('Translation', '同声翻译识别：SparkChain 未就绪，沿用原识别'); + return false; + } + Logger.w('Translation', '同声翻译识别：SparkChain lang=${directed.source}'); + if (_simHeadsetSession != null) return true; + final mic = MicPcmStreamer(); + final started = await mic.start(); + if (!started) { + Logger.w('Translation', '同声翻译手机麦采集失败，改回原识别'); + await _stopSparkInput(); + return false; + } + _sparkMic = mic; + _sparkMicSub = mic.packets.listen(SparkChainSession.write); + return true; + } + + void _onSparkAsr(Map event) { + if (!_sparkListen) return; + if (event['kind'] == 'error') { + Logger.w('Translation', + 'SparkChain 听写失败 code=${event['code']} sid=${event['sid']}'); + _handleRecognitionEvent(RecognitionEvent( + type: RecognitionEventType.error, + error: '${event['code']}', + provider: 'sparkchain', + )); + return; + } + final text = (event['text'] as String?) ?? ''; + _handleRecognitionEvent(RecognitionEvent( + type: event['final'] == true + ? RecognitionEventType.finalResult + : RecognitionEventType.intermediateResult, + text: text, + detectedLanguage: sourceLanguageCode.value, + provider: 'sparkchain', + )); + } + + /// 先停推流和听写，再由调用方关耳机麦、释放保活。 + Future _stopSparkInput() async { + await _sparkMicSub?.cancel(); + _sparkMicSub = null; + final mic = _sparkMic; + _sparkMic = null; + if (mic != null) { + try { + await mic.stop(); + } catch (e) { + Logger.w('Translation', '关闭讯飞手机麦失败: $e'); + } + } + _sparkListen = false; + SparkChainSession.asrListener = null; + final player = _sparkPlayer; + _sparkPlayer = null; + try { + await player?.dispose(); + } catch (e) { + Logger.w('Translation', '停止讯飞播报失败: $e'); + } + await SparkChainSession.stopListen(); + } + + /// 译文 PCM 交给现有播放器。连着耳机时只放音，避免把声音钉在手机扬声器。 + Future _playSparkTts(String lang, String text) async { + final player = _sparkPlayer ??= PcmStreamPlayer( + sampleRate: 16000, + playbackOnly: _simHeadsetSession != null, + ); + await player.begin(); + try { + await SparkChainSession.speak( + lang: lang, + text: text, + onPcm: (pcm, _) => player.feed(pcm), + ); + } finally { + await player.end(); + } + } + /// 启动语音识别 /// /// 这一段是「首次进同传拉不起识别」的另一半嫌疑：原生 @@ -2288,19 +2403,34 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } Future _startAsrServiceInner(String mode) async { + final bool started; if (mode == 'push_to_talk') { - await _asrService.startContinuousRecognition( + started = await _asrService.startContinuousRecognition( _audioSourceType, isRemoveFirstPunctuation: false, mode: mode, ); } else { - await _asrService.startContinuousRecognition( + started = await _asrService.startContinuousRecognition( _audioSourceType, isRemoveFirstPunctuation: true, mode: mode, ); } + if (!started) _showAzureSpeechMissing(); + } + + /// 微软配置没下发时给一条提示。页面还能留着，不能因为这个把进程打崩。 + void _showAzureSpeechMissing() { + if (_asrService.configured) return; + const key = 'azureSpeechMissing'; + final translated = key.tr; + ToastThrottle.show( + 'asr.azure_missing', + title: 'error'.tr, + message: translated == key ? '未获取到微软语音配置，语音识别暂不可用' : translated, + window: ToastThrottle.gateWindow, + ); } /// 完成识别启动 @@ -2647,6 +2777,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } try { + await _stopSparkInput(); await stopRecording(); await _asrService.stopContinuousRecognition(); if (currentMode.value == "call") { @@ -2933,7 +3064,19 @@ class TranslationController extends GetxController with WidgetsBindingObserver { Future _performTranslation(String sourceText, String sourceCode, String targetCode, CancelToken? cancelToken) async { final directed = _directedShortCodes(); - if (LanguageManager.microsoftTextPipeline( + if (_sparkListen) { + final onDevice = await SparkChainSession.translate( + from: directed.source, + to: directed.target, + text: sourceText, + ); + if (onDevice != null) return onDevice; + if (!aiOnlyIflytek) { + Logger.w('Translation', '同声 SparkChain 翻译没有结果，改走原路径'); + } + } + if (!aiOnlyIflytek && + LanguageManager.microsoftTextPipeline( currentMode.value, directed.source, directed.target)) { final from = _languageManager.microsoftTranslateCode(directed.source); final to = _languageManager.microsoftTranslateCode(directed.target); @@ -2950,6 +3093,7 @@ class TranslationController extends GetxController with WidgetsBindingObserver { sourceLanguageCode: sourceCode, targetLanguageCode: targetCode, cancelToken: cancelToken, + tryOnDevice: !_sparkListen, ); } @@ -3123,8 +3267,12 @@ class TranslationController extends GetxController with WidgetsBindingObserver { Logger.info("voiceName=$voiceName,text=$text"); final startTime = DateTime.now(); - await _ttsService.setVoice(voiceName); - await _ttsService.speakOnce(currsessionid, text); + if (_sparkListen) { + await _playSparkTts(directed.target, text); + } else { + await _ttsService.setVoice(voiceName); + await _ttsService.speakOnce(currsessionid, text); + } final endTime = DateTime.now(); // 记录TTS API调用统计 @@ -3715,12 +3863,16 @@ class TranslationController extends GetxController with WidgetsBindingObserver { } else { Logger.info('重新初始化ASR服务，支持语言: $asrSupportedLanguages'); - await _asrService.initialize(supportedLanguages: asrSupportedLanguages); - - var recognitionStream = await _asrService.recognizeCallback(); - _recognitionSubscription?.cancel(); - _recognitionSubscription = - recognitionStream.listen(_handleRecognitionEvent); + final asrReady = + await _asrService.initialize(supportedLanguages: asrSupportedLanguages); + if (!asrReady) { + _showAzureSpeechMissing(); + } else { + var recognitionStream = await _asrService.recognizeCallback(); + _recognitionSubscription?.cancel(); + _recognitionSubscription = + recognitionStream.listen(_handleRecognitionEvent); + } } Logger.info('ASR服务重新初始化成功'); diff --git a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt index 624f59be..b23b247a 100644 --- a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt +++ b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt @@ -317,7 +317,9 @@ class AzureAsrHelper(private val context: Context) { fun setupEventListeners(callback: ContinuousRecognizeCallback): Boolean { Log.d(tag, "设置ssssss监听器:${speechConfig ?: "null"} ") if (speechConfig == null) { - callback.onError(currsessionid, 1002, "语音服务未初始化") + // 未初始化时不能走 onError：插件的 onError 会立刻再调 recognizeCallback， + // 而这次 setup 还在栈上，等于自己套自己，一点同声翻译就栈溢出崩溃。 + Log.e(tag, "语音服务未初始化，跳过挂监听") return false } diff --git a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt index 78c7a77b..b23f19f2 100644 --- a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt +++ b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureSpeechPlugin.kt @@ -759,9 +759,13 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, } //设置识别器 + private var recognizeCallbackBusy = false + private fun recognizeCallback(): Boolean { + if (recognizeCallbackBusy) return false + recognizeCallbackBusy = true try { - azureAsrHelper.setupEventListeners(object : + return azureAsrHelper.setupEventListeners(object : AzureAsrHelper.ContinuousRecognizeCallback { override fun onResult(sessionid: String, text: String, detectedLanguage: String) { @@ -831,9 +835,10 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, recognizeCallback() } }) - return true } catch (e: Exception) { return false + } finally { + recognizeCallbackBusy = false } } @@ -1412,8 +1417,19 @@ class AzureSpeechPlugin : BleService.Callback, FlutterPlugin, ActivityAware, try { val keepalive = call.argument("keepalive") ?: false val pinSpeaker = call.argument("pinSpeaker") ?: true - AudioRecordingForegroundService.startService(context, keepalive, pinSpeaker) - result.success(true) + val holdFocus = call.argument("holdFocus") ?: false + if (holdFocus) { + // 结果要等服务进了前台、并且已经决定过音乐之后再回。 + // 这里立刻 success 的话，后面的静音轨会赶在焦点判断之前响起来。 + AudioRecordingForegroundService.startServiceHoldingFocus( + context, + pinSpeaker, + result, + ) + } else { + AudioRecordingForegroundService.startService(context, keepalive, pinSpeaker) + result.success(true) + } } catch (e: Exception) { result.error("START_FGS_ERROR", e.message, null) } diff --git a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DeviceDialogFocus.kt b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DeviceDialogFocus.kt index 2f04234c..dc9e6aa9 100644 --- a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DeviceDialogFocus.kt +++ b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/DeviceDialogFocus.kt @@ -8,10 +8,12 @@ import android.media.AudioManager import android.media.AudioPlaybackConfiguration import android.os.Build import android.os.Handler +import android.os.Looper import android.os.Process import android.os.SystemClock import android.util.Log import android.view.KeyEvent +import com.yunqiinnovation.azure_speech.tools.AudioRecordingForegroundService import io.flutter.plugin.common.MethodChannel /** @@ -29,6 +31,9 @@ import io.flutter.plugin.common.MethodChannel * 所以整段对话只持有一次 MEDIA 独占焦点。进来时**别的应用**正在播，才暂停， * 结束时再补一次播放键。系统本来没在播，就不抢焦点、也不发播放键—— * 交还焦点或 MEDIA_PLAY 都会把华为音乐从暂停状态拉起来。 + * + * 播放键必须在静音保活停掉之后发。音轨还在写的时候，键会落到我们自己身上， + * 音乐 App 收不到，用户只能切进那个 App 才会再响。 */ object DeviceDialogFocus { private const val TAG = "EmaiFocus" @@ -50,12 +55,20 @@ object DeviceDialogFocus { } fun hold(context: Context, handler: Handler, result: MethodChannel.Result) { + result.success(holdNow(context, handler)) + } + + /** + * 必须在本进程的静音音轨起来之前调用，并且调用时进程已经是前台服务。 + * 后台直接申请焦点，Android 12 会拒绝，音乐继续响，香薰麦把音乐当成用户在说， + * 设备就一直停在聆听。 + */ + fun holdNow(context: Context, handler: Handler = Handler(Looper.getMainLooper())): Map { val am = try { manager(context) } catch (e: Throwable) { Log.e(TAG, "拿不到 AudioManager: ${e.message}") - result.success(mapOf("error" to (e.message ?: "no AudioManager"))) - return + return mapOf("error" to (e.message ?: "no AudioManager")) } val before = music(am) val external = externalStarted(am) @@ -72,21 +85,35 @@ object DeviceDialogFocus { "系统没在播音乐，不抢焦点、不发媒体键 isMusicActive=$before " + "importance=$importance ${Build.BRAND}/${Build.MODEL}" ) - result.success( - mapOf( - "gain" to "NONE", - "usage" to "MEDIA", - "result" to "SKIPPED", - "mediaPause" to false, - "externalPlaying" to false, - "musicBefore" to before, - "importance" to importance, - ) + return mapOf( + "gain" to "NONE", + "usage" to "MEDIA", + "result" to "SKIPPED", + "mediaPause" to false, + "externalPlaying" to false, + "musicBefore" to before, + "importance" to importance, ) - return } abandon(am, "重抢前") val outcome = request(am, AudioAttributes.USAGE_MEDIA, "MEDIA") + val granted = outcome.code == AudioManager.AUDIOFOCUS_REQUEST_GRANTED + if (!granted) { + Log.w( + TAG, + "申请 EXCLUSIVE/MEDIA 未拿到，不发暂停键 结果=${outcome.name} " + + "importance=$importance ${Build.BRAND}/${Build.MODEL}" + ) + return mapOf( + "gain" to "EXCLUSIVE", + "usage" to "MEDIA", + "result" to outcome.name, + "mediaPause" to false, + "externalPlaying" to true, + "musicBefore" to before, + "importance" to importance, + ) + } mediaKey(am, KeyEvent.KEYCODE_MEDIA_PAUSE) pausedExternal = true val after = music(am) @@ -96,24 +123,22 @@ object DeviceDialogFocus { "mode=$mode importance=$importance 音乐 $before → $after " + "sdk=${Build.VERSION.SDK_INT} ${Build.BRAND}/${Build.MODEL}" ) - result.success( - mapOf( - "gain" to "EXCLUSIVE", - "usage" to "MEDIA", - "result" to outcome.name, - "mediaPause" to pausedExternal, - "musicBefore" to before, - "musicAfter" to after, - "mode" to mode, - "importance" to importance, - "brand" to Build.BRAND, - "model" to Build.MODEL, - "sdk" to Build.VERSION.SDK_INT, - ) - ) handler.postDelayed({ Log.i(TAG, "300ms 复查 音乐=${music(am)} 已发暂停键=$pausedExternal") }, 300) + return mapOf( + "gain" to "EXCLUSIVE", + "usage" to "MEDIA", + "result" to outcome.name, + "mediaPause" to pausedExternal, + "musicBefore" to before, + "musicAfter" to after, + "mode" to mode, + "importance" to importance, + "brand" to Build.BRAND, + "model" to Build.MODEL, + "sdk" to Build.VERSION.SDK_INT, + ) } fun release(context: Context): Map { @@ -130,12 +155,22 @@ object DeviceDialogFocus { "musicAfter" to before, ) } + // 先停自己的静音轨，前台服务先留着。键要在我们不再占着媒体输出时发， + // 才会进到音乐 App；服务还在前台，后台发键会被华为丢掉。 + AudioRecordingForegroundService.stopSilenceKeepalive() + SystemClock.sleep(120) abandon(am, "交还") pausedExternal = false val afterAbandon = music(am) mediaKey(am, KeyEvent.KEYCODE_MEDIA_PLAY) + var after = music(am) + if (!after) { + SystemClock.sleep(280) + mediaKey(am, KeyEvent.KEYCODE_MEDIA_PLAY) + after = music(am) + Log.i(TAG, "补发播放键 音乐=$after") + } val sentPlay = true - val after = music(am) Log.i( TAG, "交还 had=$had 播放键=$sentPlay 音乐 $before → $afterAbandon → $after" diff --git a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/AudioRecordingForegroundService.kt b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/AudioRecordingForegroundService.kt index 49a183ef..8928e346 100644 --- a/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/AudioRecordingForegroundService.kt +++ b/apps/client/local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/tools/AudioRecordingForegroundService.kt @@ -14,11 +14,17 @@ import android.media.AudioFormat import android.media.AudioManager import android.media.AudioTrack import android.os.Build +import android.os.Handler import android.os.IBinder +import android.os.Looper import android.os.PowerManager import androidx.core.app.NotificationCompat import android.util.Log +import com.yunqiinnovation.azure_speech.DeviceDialogFocus +import io.flutter.plugin.common.MethodChannel import org.json.JSONObject +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference import kotlin.math.max /** @@ -44,6 +50,7 @@ class AudioRecordingForegroundService : Service() { private const val KEEPALIVE_CHANNEL_ID = "session_keepalive" private const val EXTRA_KEEPALIVE = "keepalive" private const val EXTRA_PIN_SPEAKER = "pinSpeaker" + private const val EXTRA_HOLD_FOCUS = "holdFocus" private const val ACTION_STOP = "com.yunqiinnovation.azure_speech.tools.STOP_SERVICE" private const val NATIVE_LOCALIZATION_PREF_NAME = "native_localization" private const val NATIVE_LOCALIZATION_PREF_KEY_STRINGS_JSON = "strings_json" @@ -63,11 +70,67 @@ class AudioRecordingForegroundService : Service() { } Log.i("EmaiFocus", "前台服务启动请求 keepalive=$keepalive") } - + + @Volatile + private var running: AudioRecordingForegroundService? = null + + private val pendingHold = AtomicReference(null) + private val holdGen = AtomicInteger(0) + /** - * 停止前台服务 - * @param context 上下文 + * 先进入前台（含媒体播放类型），再决定要不要暂停外面的音乐，最后才开静音轨。 + * 结果在这一串做完之后才回调。后台直接抢焦点会被 Android 12 拒绝。 */ + fun startServiceHoldingFocus( + context: Context, + pinSpeaker: Boolean, + result: MethodChannel.Result, + ) { + val gen = holdGen.incrementAndGet() + val intent = Intent(context, AudioRecordingForegroundService::class.java) + intent.putExtra(EXTRA_KEEPALIVE, true) + intent.putExtra(EXTRA_PIN_SPEAKER, pinSpeaker) + intent.putExtra(EXTRA_HOLD_FOCUS, true) + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + context.startForegroundService(intent) + } else { + context.startService(intent) + } + } catch (e: Exception) { + Log.e("EmaiFocus", "前台服务启动失败: ${e.message}") + result.error("START_FGS_ERROR", e.message, null) + return + } + val previous = pendingHold.getAndSet(result) + previous?.success(false) + Log.i("EmaiFocus", "前台服务启动请求 keepalive=true holdFocus pinSpeaker=$pinSpeaker") + Handler(Looper.getMainLooper()).postDelayed({ + if (holdGen.get() != gen) return@postDelayed + val stuck = pendingHold.getAndSet(null) + if (stuck != null) { + Log.e("EmaiFocus", "前台服务未在时限内起来，焦点未抢") + stuck.success(false) + } + }, 2000) + } + + fun stopSilenceKeepalive() { + val svc = running + if (svc == null) { + Log.i("EmaiFocus", "交还前没有静音保活可停") + return + } + svc.stopDialogKeepalive() + Log.i("EmaiFocus", "交还前已停静音保活") + } + + private fun completeHold(ok: Boolean) { + holdGen.incrementAndGet() + val waiting = pendingHold.getAndSet(null) + waiting?.success(ok) + } + fun stopService(context: Context) { val intent = Intent(context, AudioRecordingForegroundService::class.java) // ⚠️ 用 stopService 而不是「startService + ACTION_STOP」。 @@ -86,6 +149,7 @@ class AudioRecordingForegroundService : Service() { override fun onCreate() { super.onCreate() + running = this Log.d(TAG, "AudioRecordingForegroundService onCreate") createNotificationChannel() } @@ -123,18 +187,33 @@ class AudioRecordingForegroundService : Service() { Log.d(TAG, "AudioRecordingForegroundService onStartCommand action=${intent?.action}") if (intent?.getBooleanExtra(EXTRA_KEEPALIVE, false) == true) { keepaliveLatched = true + val wantPin = intent?.getBooleanExtra(EXTRA_PIN_SPEAKER, true) ?: true + // 上一场钉在扬声器时，音轨还在的话新的 pin 不会生效，通话翻译就会继续 + // 把耳机路由拽走。钉法变了就先拆掉，下面按新的钉法重开。 + if (silenceTrack != null && pinSilenceToSpeaker != wantPin) { + stopDialogKeepalive() + } if (silenceTrack == null) { - pinSilenceToSpeaker = - intent?.getBooleanExtra(EXTRA_PIN_SPEAKER, true) ?: true + pinSilenceToSpeaker = wantPin } } val keepalive = keepaliveLatched + val holdFocus = intent?.getBooleanExtra(EXTRA_HOLD_FOCUS, false) == true // 创建前台通知 val notification = createNotification(keepalive) // 必须立即调用 startForeground 以满足 Android 8.0+ 的要求 startForegroundCompat(notification, keepalive) + if (holdFocus) { + // 上一场若没停干净，自己的音轨会让 isMusicActive 变成 true， + // 这场就会误判「外面在播」或者反过来把外面的音乐漏掉。 + if (silenceTrack != null) stopDialogKeepalive() + pinSilenceToSpeaker = intent?.getBooleanExtra(EXTRA_PIN_SPEAKER, true) ?: true + val snap = DeviceDialogFocus.holdNow(this) + Log.i("EmaiFocus", "设备对话焦点 $snap") + } if (keepalive) startDialogKeepalive() + if (holdFocus) completeHold(true) // 检查是否是停止指令 if (intent?.action == ACTION_STOP) { @@ -153,6 +232,7 @@ class AudioRecordingForegroundService : Service() { } override fun onDestroy() { + if (running === this) running = null keepaliveLatched = false pinSilenceToSpeaker = true stopDialogKeepalive() diff --git a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift index 6a64eea5..4c4a4308 100644 --- a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift +++ b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/AzureSpeechPlugin.swift @@ -324,8 +324,10 @@ import ble_service // 录音相关 public var recordfile: RecordFile? private var isRecord = false - // 创建事件回调 + // 创建事件回调。A/B 都要留着：服务内部以前是 weak，只留 A 时 + // B 路（远端）的识别回调会在初始化结束时被释放，表现为远端有声音但一个字都没有。 private var astEventCallback: AstEventCallback? + private var astEventCallbackB: AstEventCallback? // 是否已添加AST事件监听器 private var isAstListenerAdded = false @@ -632,21 +634,14 @@ internal func handlePcmPlayerMethodCall(_ call: FlutterMethodCall, result: @esca } } -private static var deviceDialogNowPlayingReady = false private static let deviceDialogNowPlayingTitle = "AI 助手" /// 让系统把这次设备对话认成正在播放。不改会话类别。 /// 只清自己写上去的那条，避免抹掉别的模块的 Now Playing。 private func setDeviceDialogNowPlaying(_ active: Bool) { if active { - if !Self.deviceDialogNowPlayingReady { - let commands = MPRemoteCommandCenter.shared() - commands.playCommand.isEnabled = true - commands.playCommand.addTarget { _ in .success } - commands.pauseCommand.isEnabled = true - commands.pauseCommand.addTarget { _ in .success } - Self.deviceDialogNowPlayingReady = true - } + // 不注册 play/pause。返回 .success 会让系统以为播放键已经有人处理， + // 对话结束时音乐 App 收不到继续播放，要用户切进那个 App 才会再响。 MPNowPlayingInfoCenter.default().nowPlayingInfo = [ MPMediaItemPropertyTitle: Self.deviceDialogNowPlayingTitle, MPNowPlayingInfoPropertyPlaybackRate: 1.0, @@ -1370,7 +1365,11 @@ private func sendAudioDataEvent(_ event: [String: Any]) { let args = call.arguments as? [String: Any] let enabled = (args?["enabled"] as? Bool) ?? false if enabled { - ensureCallPcmBridge().enable() + let b = ensureCallPcmBridge() + b.enable() + // 同页面第二场：helper 早就 initialize 过、不会再发 markReady。 + // 按插件上的就绪状态补一次，否则上行全扣在缓冲里，界面没动静。 + if isAstSessionReady { b.markReady() } } else { callPcmBridge?.disable() } @@ -1399,6 +1398,8 @@ private func sendAudioDataEvent(_ event: [String: Any]) { os_log("[AST-DISPOSE] Releasing all AST resources, provider=%{public}@", log: ctLog, type: .info, currentAstProvider) azureAstHelperA.dispose() azureAstHelperB.dispose() + astEventCallback = nil + astEventCallbackB = nil doubaoAstHelperA.dispose() doubaoAstHelperB.dispose() bailianAstHelperA.dispose() @@ -1500,6 +1501,7 @@ private func sendAudioDataEvent(_ event: [String: Any]) { let callbackA = AstEventCallback(plugin: self, serviceId: "A") let callbackB = AstEventCallback(plugin: self, serviceId: "B") astEventCallback = callbackA + astEventCallbackB = callbackB azureAstHelperA.serviceId = "A" azureAstHelperB.serviceId = "B" @@ -1798,6 +1800,8 @@ private class AstEventCallback: IntegratedSpeechTranslationService.ServiceEventC "utteranceId": utteranceId, "text": text ]) + // 段尾空帧，和阿里回调、安卓 markEnd 同一份合约，下行才能把这一句收完。 + plugin?.pushAstTtsFrame(leg: serviceId, pcm: Data(), isFinal: true) } public func onSynthesisFailed(utteranceId: String, text: String, error: String) { @@ -1854,9 +1858,11 @@ private class AstEventCallback: IntegratedSpeechTranslationService.ServiceEventC public func onSynthesisAudioGenerated(utteranceId: String, text: String, audioData: Data) { let pcmData = audioData - print("[\(serviceId)] 合成音频生成，PCM大小=\(pcmData.count)，合成ID=\(utteranceId)") + os_log("[AST-TTS] 微软合成 leg=%{public}@ pcm=%d", log: OSLog(subsystem: "com.azure.speech", category: "CallTranslation"), type: .info, serviceId, pcmData.count) if pcmData.count > 0 { - BleService.shared.writeExternalAudioData(data: pcmData) + // 爱尔兰语等扩展语种走微软三段拼装，合成音必须进耳机下行。 + // 旧的 BleService 写入对恒玄是空操作，耳机里就没有译文。 + plugin?.pushAstTtsFrame(leg: serviceId, pcm: pcmData, isFinal: false) } } } diff --git a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/BesCallPcmBridge.swift b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/BesCallPcmBridge.swift index 0dc6bfd1..f25314ea 100644 --- a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/BesCallPcmBridge.swift +++ b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/BesCallPcmBridge.swift @@ -91,7 +91,9 @@ final class BesCallPcmBridge { ingress.async { NSLog("[BesCallPcmBridge] 已关闭 A=\(self.framesA) B=\(self.framesB) 丢帧=\(self.dropped) B门限前=\(self.gatedB) 下行=\(self.downlinkFrames) 峰值积压=\(self.maxPendingSeen)") self.holdA.removeAll(); self.holdB.removeAll() - self.ready = false + // 不把 ready 打回 false。它表示 AST helper 起来了没，只由 + // onSessionInit / onSessionDisposed 翻假。在这里清掉的话，同页面第二场 + // 不会再 initialize，也就没人 markReady，之后每一场上行都扣在缓冲里。 } } diff --git a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/IntegratedSpeechTranslationService.swift b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/IntegratedSpeechTranslationService.swift index 5a3c8b0b..2888388f 100644 --- a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/IntegratedSpeechTranslationService.swift +++ b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/azure_speech/IntegratedSpeechTranslationService.swift @@ -69,6 +69,8 @@ import os.log private var recognizingTranslationTask: Task? private var currentSynthesisId: String? private var currentSynthesisText: String? + private var azureKey: String = "" + private var azureRegion: String = "" /** * 生成新的 utteranceId @@ -79,8 +81,9 @@ import os.log return "utt-\(serviceId)-\(ts)-\(utteranceSeq)" } - // 事件回调 - private weak var eventCallback: ServiceEventCallback? + // 事件回调。必须强引用：调用方（插件）如果只把 B 路回调留在局部变量里， + // weak 会在 initialize 返回后立刻变 nil，远端识别结果一个都送不出去。 + private var eventCallback: ServiceEventCallback? // 语音映射缓存 private var voiceCache: [String: String] = [:] @@ -310,15 +313,19 @@ import os.log do { os_log("开始初始化Azure服务", log: log, type: .info) + azureKey = azureConfig.subscriptionKey + azureRegion = azureConfig.region speechConfig = try SPXSpeechConfiguration(subscription: azureConfig.subscriptionKey, region: azureConfig.region) guard let config = speechConfig else { return false } + // 识别配置上只钉这一路要听的语种。远端中→爱尔兰就是 ga-IE， + // 不能再混进中文，否则开头的串音会把整场锁成中文，爱尔兰语永远无字。 + // 合成音色另建一份配置，避免和识别语种抢同一份 SpeechConfig。 config.speechRecognitionLanguage = serviceConfig.sourceLanguage - config.speechSynthesisVoiceName = getVoiceForLanguage(serviceConfig.targetLanguage) - config.setSpeechSynthesisOutputFormat(.raw16Khz16BitMonoPcm) + config.setPropertyTo("300", by: SPXPropertyId.speechSegmentationSilenceTimeoutMs) // 自动语言检测 if serviceConfig.enableAutoLanguageDetection { @@ -379,6 +386,8 @@ import os.log } do { + speechConfig.speechRecognitionLanguage = serviceConfig.sourceLanguage + os_log("识别器语种: %@ service=%@", log: log, type: .info, serviceConfig.sourceLanguage, serviceId) recognizer = try SPXSpeechRecognizer(speechConfiguration: speechConfig, audioConfiguration: audioConfig) // 识别中事件 @@ -449,7 +458,7 @@ import os.log } case .noMatch: - os_log("未识别到语音", log: self.log, type: .debug) + os_log("未识别到语音 lang=%@", log: self.log, type: .info, self.serviceConfig.sourceLanguage) default: os_log("识别结果: %@", log: self.log, type: .info, String(describing: result.reason)) @@ -501,6 +510,7 @@ import os.log } catch { os_log("语音识别器设置失败: %@", log: log, type: .error, error.localizedDescription) + eventCallback?.onError(component: "Recognition", error: "识别器创建失败[\(serviceId)] \(serviceConfig.sourceLanguage): \(error.localizedDescription)") } } @@ -521,8 +531,16 @@ import os.log // 不输出到扬声器，只生成音频数据 audioConfig = nil } - - synthesizer = try SPXSpeechSynthesizer(speechConfiguration: speechConfig, audioConfiguration: audioConfig) + + let synthConfig: SPXSpeechConfiguration + if !azureKey.isEmpty && !azureRegion.isEmpty { + synthConfig = try SPXSpeechConfiguration(subscription: azureKey, region: azureRegion) + } else { + synthConfig = speechConfig + } + synthConfig.speechSynthesisVoiceName = getVoiceForLanguage(serviceConfig.targetLanguage) + synthConfig.setSpeechSynthesisOutputFormat(.raw16Khz16BitMonoPcm) + synthesizer = try SPXSpeechSynthesizer(speechConfiguration: synthConfig, audioConfiguration: audioConfig) // 合成开始事件 synthesizer?.addSynthesisStartedEventHandler { [weak self] (synthesizer, event) in @@ -747,8 +765,7 @@ import os.log do { let srcLang = self.serviceConfig.translationSourceLanguage.isEmpty ? self.serviceConfig.sourceLanguage : self.serviceConfig.translationSourceLanguage let tgtLang = self.serviceConfig.translationTargetLanguage.isEmpty ? self.serviceConfig.targetLanguage : self.serviceConfig.translationTargetLanguage - os_log("翻译语言对: %@ -> %@", log: log, type: .debug, srcLang, tgtLang) - + os_log("翻译语言对: %@ -> %@", log: log, type: .info, srcLang, tgtLang) let translationResult = try await withTimeout(self.serviceConfig.translationTimeout) { os_log("开始翻译文本: %@", log: self.log, type: .debug, text) return await self.translationService?.translateText( @@ -1179,6 +1196,7 @@ import os.log recordFile?.closeFile(isSave: false) recordFile = nil + eventCallback = nil // 清理缓存 voiceCache.removeAll() diff --git a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift index f4e7a1c2..236a0f37 100644 --- a/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift +++ b/apps/client/local_plugins/azure_speech/ios/azure_speech/Sources/tools/SimpleAudioReceiver.swift @@ -137,7 +137,13 @@ public class SimpleAudioReceiver: NSObject { _isWriting = true - print("liwei---------- startAudioRecord=mod:\(mod), audioSourceType:\(audioSourceType), isHeadphones:\(isHeadphones)") + // 外部 PCM（耳机通话的双路）不改音频会话。两路识别器都会走到这里， + // 第二路再 setCategory/setActive 会把刚起来的识别会话打断， + // 表现就是近端中文正常、远端爱尔兰语一句都没有。 + if audioSourceType == .external { + runExternalCapture() + return + } do { try configureAudioSession() @@ -375,12 +381,17 @@ public class SimpleAudioReceiver: NSObject { * @param data 音频数据字节数组 */ public func saveAudioDataTo(data: Data) { - // 通过父类引用调用方法 if audioSourceType != .external { return } - // 放入队列，由写线程写入 - self.writeQueue.put(data) + guard isContinuousRecognitionActive, let stream = pushAudioStream else { + return + } + do { + try stream.write(data) + } catch { + os_log("写入识别器失败: %@", log: log, type: .error, error.localizedDescription) + } } /** diff --git a/apps/client/local_plugins/sparkchain/android/build.gradle.kts b/apps/client/local_plugins/sparkchain/android/build.gradle.kts new file mode 100644 index 00000000..871d1332 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/android/build.gradle.kts @@ -0,0 +1,31 @@ +plugins { + id("com.android.library") + id("org.jetbrains.kotlin.android") +} + +android { + namespace = "com.yunqiinnovation.sparkchain" + compileSdk = 35 + + defaultConfig { + minSdk = 21 + consumerProguardFiles("consumer-rules.pro") + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 + } + + kotlinOptions { + jvmTarget = "11" + } + + sourceSets.getByName("main").jniLibs.srcDirs("src/main/jniLibs") +} + +// 插件自己会打成 AAR，不能再 implementation 本地 AAR，否则 release 打包直接失败。 +// SparkChain.aar / Codec.aar 已拆成 libs/*.jar、jniLibs 和 assets。 +dependencies { + implementation(fileTree(mapOf("dir" to "libs", "include" to listOf("*.jar")))) +} diff --git a/apps/client/local_plugins/sparkchain/android/consumer-rules.pro b/apps/client/local_plugins/sparkchain/android/consumer-rules.pro new file mode 100644 index 00000000..cd0664d8 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/android/consumer-rules.pro @@ -0,0 +1 @@ +-keep class com.iflytek.sparkchain.** { *; } diff --git a/apps/client/local_plugins/sparkchain/android/libs/Codec.aar b/apps/client/local_plugins/sparkchain/android/libs/Codec.aar new file mode 100644 index 00000000..17350c31 Binary files /dev/null and b/apps/client/local_plugins/sparkchain/android/libs/Codec.aar differ diff --git a/apps/client/local_plugins/sparkchain/android/libs/Codec.jar b/apps/client/local_plugins/sparkchain/android/libs/Codec.jar new file mode 100644 index 00000000..ed16f474 Binary files /dev/null and b/apps/client/local_plugins/sparkchain/android/libs/Codec.jar differ diff --git a/apps/client/local_plugins/sparkchain/android/libs/SparkChain.aar b/apps/client/local_plugins/sparkchain/android/libs/SparkChain.aar new file mode 100644 index 00000000..f9d2f232 Binary files /dev/null and b/apps/client/local_plugins/sparkchain/android/libs/SparkChain.aar differ diff --git a/apps/client/local_plugins/sparkchain/android/libs/SparkChain.jar b/apps/client/local_plugins/sparkchain/android/libs/SparkChain.jar new file mode 100644 index 00000000..c0267e78 Binary files /dev/null and b/apps/client/local_plugins/sparkchain/android/libs/SparkChain.jar differ diff --git a/apps/client/local_plugins/sparkchain/android/src/main/AndroidManifest.xml b/apps/client/local_plugins/sparkchain/android/src/main/AndroidManifest.xml new file mode 100644 index 00000000..fbb2ef83 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/android/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + + diff --git a/apps/client/local_plugins/sparkchain/android/src/main/assets/supplierconfig.json b/apps/client/local_plugins/sparkchain/android/src/main/assets/supplierconfig.json new file mode 100644 index 00000000..1015fe3b --- /dev/null +++ b/apps/client/local_plugins/sparkchain/android/src/main/assets/supplierconfig.json @@ -0,0 +1,17 @@ +{ + "supplier":{ + "vivo":{ + "appid":"" + }, + "xiaomi":{ + "appid":"" + }, + "huawei":{ + "appid":"103256863" + }, + "oppo":{ + + } + } + +} diff --git a/apps/client/local_plugins/sparkchain/android/src/main/kotlin/com/yunqiinnovation/sparkchain/SparkChainPlugin.kt b/apps/client/local_plugins/sparkchain/android/src/main/kotlin/com/yunqiinnovation/sparkchain/SparkChainPlugin.kt new file mode 100644 index 00000000..fd20bab1 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/android/src/main/kotlin/com/yunqiinnovation/sparkchain/SparkChainPlugin.kt @@ -0,0 +1,334 @@ +package com.yunqiinnovation.sparkchain + +import android.content.Context +import android.os.Handler +import android.os.Looper +import com.iflytek.sparkchain.core.SparkChain +import com.iflytek.sparkchain.core.SparkChainConfig +import com.iflytek.sparkchain.core.asr.ASR +import com.iflytek.sparkchain.core.asr.AsrCallbacks +import com.iflytek.sparkchain.core.asr.AudioAttributes +import com.iflytek.sparkchain.core.asr.RegionType +import com.iflytek.sparkchain.core.common.ApiType +import com.iflytek.sparkchain.core.its.ITS +import com.iflytek.sparkchain.core.its.ITSCallbacks +import com.iflytek.sparkchain.core.its.TransType +import com.iflytek.sparkchain.core.rtasr.RTASR +import com.iflytek.sparkchain.core.rtasr.RTASRCallbacks +import com.iflytek.sparkchain.core.tts.OnlineTTS +import com.iflytek.sparkchain.core.tts.TTS +import com.iflytek.sparkchain.core.tts.TTSCallbacks +import io.flutter.embedding.engine.plugins.FlutterPlugin +import io.flutter.plugin.common.EventChannel +import io.flutter.plugin.common.MethodCall +import io.flutter.plugin.common.MethodChannel +import java.util.concurrent.Executors + +/// 讯飞海外 SDK。不自己开麦，PCM 由调用方 write 进来。 +class SparkChainPlugin : FlutterPlugin, MethodChannel.MethodCallHandler, EventChannel.StreamHandler { + private val main = Handler(Looper.getMainLooper()) + // 听写的 write 不能排在翻译后面：arun 若卡住，耳机麦帧会在队列里过期。 + private val audio = Executors.newSingleThreadExecutor() + private val cloud = Executors.newSingleThreadExecutor() + private var channel: MethodChannel? = null + private var events: EventChannel? = null + private var sink: EventChannel.EventSink? = null + private var appContext: Context? = null + private var ready = false + + private var asr: ASR? = null + private var rtasr: RTASR? = null + private var its: ITS? = null + private var tts: OnlineTTS? = null + + private val asrCallbacks = object : AsrCallbacks { + override fun onResult(result: ASR.ASRResult, usrTag: Any?) { + val text = result.bestMatchText ?: "" + emit( + mapOf( + "kind" to "asr", + "id" to (usrTag?.toString() ?: ""), + "text" to text, + "final" to (result.status == 2), + "sid" to (result.sid ?: ""), + ), + ) + } + + override fun onError(error: ASR.ASRError, usrTag: Any?) { + emit( + mapOf( + "kind" to "error", + "where" to "asr", + "id" to (usrTag?.toString() ?: ""), + "code" to error.code, + "message" to (error.errMsg ?: ""), + "sid" to (error.sid ?: ""), + ), + ) + } + } + + private val rtasrCallbacks = object : RTASRCallbacks { + override fun onResult(result: RTASR.RtAsrResult, usrTag: Any?) { + val status = result.status + emit( + mapOf( + "kind" to "asr", + "id" to (usrTag?.toString() ?: ""), + "text" to (result.data ?: ""), + "final" to (status == 2 || status == 3), + "sid" to (result.sid ?: ""), + ), + ) + } + + override fun onError(error: RTASR.RtAsrError, usrTag: Any?) { + emit( + mapOf( + "kind" to "error", + "where" to "asr", + "id" to (usrTag?.toString() ?: ""), + "code" to error.code, + "message" to (error.errMsg ?: ""), + "sid" to (error.sid ?: ""), + ), + ) + } + } + + private val itsCallbacks = object : ITSCallbacks { + override fun onResult(result: ITS.ITSResult, usrTag: Any?) { + val dst = result.transResult?.dst ?: "" + emit( + mapOf( + "kind" to "its", + "id" to (usrTag?.toString() ?: ""), + "text" to dst, + "sid" to (result.sid ?: ""), + ), + ) + } + + override fun onError(error: ITS.ITSError, usrTag: Any?) { + emit( + mapOf( + "kind" to "error", + "where" to "its", + "id" to (usrTag?.toString() ?: ""), + "code" to error.code, + "message" to (error.errMsg ?: ""), + "sid" to (error.sid ?: ""), + ), + ) + } + } + + private val ttsCallbacks = object : TTSCallbacks { + override fun onResult(result: TTS.TTSResult, usrTag: Any?) { + val audio = result.data + emit( + mapOf( + "kind" to "tts", + "id" to (usrTag?.toString() ?: ""), + "pcm" to (audio ?: ByteArray(0)), + "final" to (result.status == 2), + "sid" to (result.sid ?: ""), + ), + ) + } + + override fun onError(error: TTS.TTSError, usrTag: Any?) { + emit( + mapOf( + "kind" to "error", + "where" to "tts", + "id" to (usrTag?.toString() ?: ""), + "code" to error.code, + "message" to (error.errMsg ?: ""), + "sid" to (error.sid ?: ""), + ), + ) + } + } + + override fun onAttachedToEngine(binding: FlutterPlugin.FlutterPluginBinding) { + appContext = binding.applicationContext + channel = MethodChannel(binding.binaryMessenger, "sparkchain").also { + it.setMethodCallHandler(this) + } + events = EventChannel(binding.binaryMessenger, "sparkchain/events").also { + it.setStreamHandler(this) + } + } + + override fun onDetachedFromEngine(binding: FlutterPlugin.FlutterPluginBinding) { + channel?.setMethodCallHandler(null) + events?.setStreamHandler(null) + channel = null + events = null + sink = null + } + + override fun onListen(arguments: Any?, eventSink: EventChannel.EventSink) { + sink = eventSink + } + + override fun onCancel(arguments: Any?) { + sink = null + } + + override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) { + when (call.method) { + "init" -> cloud.execute { + val code = doInit(call) + main.post { result.success(code) } + } + "translate" -> cloud.execute { + val code = doTranslate(call) + main.post { result.success(code) } + } + "speak" -> cloud.execute { + val code = doSpeak(call) + main.post { result.success(code) } + } + "startListen" -> audio.execute { + val code = doStartListen(call) + main.post { result.success(code) } + } + "write" -> audio.execute { + val pcm = call.argument("pcm") + val code = when { + rtasr != null && pcm != null -> rtasr!!.write(pcm) + asr != null && pcm != null -> asr!!.write(pcm) + else -> -1 + } + main.post { result.success(code) } + } + "stopListen" -> audio.execute { + rtasr?.stop(false) + asr?.stop(false) + rtasr = null + asr = null + main.post { result.success(0) } + } + "uninit" -> cloud.execute { + val code = if (ready) SparkChain.getInst().unInit() else 0 + ready = false + main.post { result.success(code) } + } + else -> result.notImplemented() + } + } + + private fun doInit(call: MethodCall): Int { + val ctx = appContext ?: return -1 + val appId = call.argument("appId") ?: return -2 + val apiKey = call.argument("apiKey") ?: return -2 + val apiSecret = call.argument("apiSecret") ?: return -2 + if (ready) { + SparkChain.getInst().unInit() + ready = false + } + val apiType = when (call.argument("apiType") ?: 1) { + 0 -> ApiType.TYPE_CH + 2 -> ApiType.TYPE_EU + else -> ApiType.TYPE_SG + } + val work = java.io.File(ctx.filesDir, "sparkchain") + work.mkdirs() + val config = SparkChainConfig.builder() + .appID(appId) + .apiKey(apiKey) + .apiSecret(apiSecret) + .workDir(work.absolutePath) + .logLevel(3) + .apiType(apiType) + val code = SparkChain.getInst().init(ctx, config) + ready = code == 0 + return code + } + + /// GTRANS 的中文码是 zh-CN，爱尔兰语是 ga。听写中文码是 zh_cn，两套不要混。 + private fun itsCode(short: String): String = when (short) { + "zh" -> "zh-CN" + "en" -> "en" + "ga" -> "ga" + else -> short + } + + private fun asrCode(short: String): String = when (short) { + "zh" -> "zh_cn" + "en" -> "en_us" + else -> short + } + + private fun doTranslate(call: MethodCall): Int { + if (!ready) return -1 + val text = call.argument("text") ?: return -2 + val id = call.argument("id") ?: "" + val from = itsCode(call.argument("from") ?: "zh") + val to = itsCode(call.argument("to") ?: "en") + val engine = ITS(TransType.GTRANS) + engine.registerCallbacks(itsCallbacks) + engine.fromlanguage(from) + engine.tolanguage(to) + its = engine + return engine.arun(text, id) + } + + private fun doSpeak(call: MethodCall): Int { + if (!ready) return -1 + val text = call.argument("text") ?: return -2 + val id = call.argument("id") ?: "" + val lang = call.argument("lang") ?: "zh" + val voice = call.argument("voice") + val (region, vcn) = when (lang) { + "ga" -> RegionType.MSFT_TYPE to (voice ?: "ga-IE-OrlaNeural") + "en" -> null to (voice ?: "x_Catherine") + else -> null to (voice ?: "x_xiaoyan") + } + val engine = if (region != null) OnlineTTS(region, vcn) else OnlineTTS(vcn) + engine.aue("raw") + engine.registerCallbacks(ttsCallbacks) + tts = engine + return engine.aRun(text, id) + } + + private fun doStartListen(call: MethodCall): Int { + if (!ready) return -1 + rtasr?.stop(true) + asr?.stop(true) + rtasr = null + asr = null + val lang = call.argument("lang") ?: "zh" + val id = call.argument("id") ?: "" + if (lang == "ga") { + val engine = RTASR(null, RegionType.MSFT_TYPE) + engine.registerCallbacks(rtasrCallbacks) + engine.setParams("language_name", "ga-IE") + engine.OstEncoding("raw") + rtasr = engine + return engine.start(id) + } + val engine = ASR() + engine.registerCallbacks(asrCallbacks) + engine.language(asrCode(lang)) + engine.domain("iat") + engine.accent("mandarin") + if (lang == "zh") engine.dwa("wpgs") + val attr = AudioAttributes() + attr.sampleRate = 16000 + attr.encoding = "raw" + attr.channels = 1 + attr.bitdepth = 16 + asr = engine + // 带 AudioAttributes 的 start 只收 write 进来的 PCM，不让 SDK 自己开麦。 + return engine.start(attr, id) + } + + private fun emit(event: Map) { + val current = sink ?: return + main.post { current.success(event) } + } +} diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/Package.swift b/apps/client/local_plugins/sparkchain/ios/sparkchain/Package.swift new file mode 100644 index 00000000..5db9090c --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/Package.swift @@ -0,0 +1,27 @@ +// swift-tools-version: 5.9 +import PackageDescription + +let package = Package( + name: "sparkchain", + platforms: [.iOS("13.0")], + products: [ + .library(name: "sparkchain", targets: ["sparkchain"]) + ], + targets: [ + .target( + name: "SparkChainGlue", + dependencies: ["SparkChain"], + path: "Sources/glue", + publicHeadersPath: "include" + ), + .target( + name: "sparkchain", + dependencies: ["SparkChain", "SparkChainGlue"], + path: "Sources/sparkchain" + ), + .binaryTarget( + name: "SparkChain", + path: "SparkChain.xcframework" + ) + ] +) diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/Sources/glue/SparkChainGlue.m b/apps/client/local_plugins/sparkchain/ios/sparkchain/Sources/glue/SparkChainGlue.m new file mode 100644 index 00000000..60993e25 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/Sources/glue/SparkChainGlue.m @@ -0,0 +1,21 @@ +#import "SparkChainGlue.h" +#import + +int SparkChainGlueInit(NSString *appId, + NSString *apiKey, + NSString *apiSecret, + NSString *workDir, + int apiType) { + SparkChainConfig *config = [[SparkChainConfig alloc] init]; + config.appID(appId) + .apiKey(apiKey) + .apiSecret(apiSecret) + .workDir(workDir) + .logLevel(3) + .apiType(apiType); + return [[SparkChain getInst] init:config]; +} + +int SparkChainGlueUnInit(void) { + return [[SparkChain getInst] unInit]; +} diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/Sources/glue/include/SparkChainGlue.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/Sources/glue/include/SparkChainGlue.h new file mode 100644 index 00000000..2bdceb83 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/Sources/glue/include/SparkChainGlue.h @@ -0,0 +1,9 @@ +#import + +/// SDK 的 init: 和方法名冲突，放在 ObjC 里调用。返回 0 表示成功。 +int SparkChainGlueInit(NSString *_Nonnull appId, + NSString *_Nonnull apiKey, + NSString *_Nonnull apiSecret, + NSString *_Nonnull workDir, + int apiType); +int SparkChainGlueUnInit(void); diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/Sources/sparkchain/SparkChainPlugin.swift b/apps/client/local_plugins/sparkchain/ios/sparkchain/Sources/sparkchain/SparkChainPlugin.swift new file mode 100644 index 00000000..a51e2a23 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/Sources/sparkchain/SparkChainPlugin.swift @@ -0,0 +1,270 @@ +import Flutter +import SparkChain +import SparkChainGlue + +/// 回调对象必须由插件强引用。SDK 里的 delegate 是 weak，局部变量会在 start 返回后被释放。 +@objc(SparkChainPlugin) +public final class SparkChainPlugin: NSObject, FlutterPlugin, FlutterStreamHandler, + ASRCallbacks, RTASRCallback, ITSCallbacks, TTSCallbacks { + + private let channelName = "sparkchain" + private var events: FlutterEventSink? + private var ready = false + + private var asr: ASR? + private var rtasr: RTASR? + private var its: ITS? + private var tts: OnlineTTS? + + private let transOrder: [TransType] = [GOOGLETRANS, NIUTRANS, ITRANS_SG, ITRANS] + private var transIndex = 0 + private var pendingIts: [String: String] = [:] + + public static func register(with registrar: FlutterPluginRegistrar) { + let instance = SparkChainPlugin() + let channel = FlutterMethodChannel(name: "sparkchain", binaryMessenger: registrar.messenger()) + registrar.addMethodCallDelegate(instance, channel: channel) + let events = FlutterEventChannel(name: "sparkchain/events", binaryMessenger: registrar.messenger()) + events.setStreamHandler(instance) + } + + public func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { + self.events = events + return nil + } + + public func onCancel(withArguments arguments: Any?) -> FlutterError? { + events = nil + return nil + } + + public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { + let args = call.arguments as? [String: Any] ?? [:] + switch call.method { + case "init": + result(doInit(args)) + case "translate": + result(doTranslate(args)) + case "speak": + result(doSpeak(args)) + case "startListen": + result(doStartListen(args)) + case "write": + let data = args["pcm"] as? FlutterStandardTypedData + if let rtasr, let data { + result(rtasr.write(data.data)) + } else if let asr, let data { + result(asr.write(data.data)) + } else { + result(-1) + } + case "stopListen": + rtasr?.stop(false) + asr?.stop(false) + rtasr = nil + asr = nil + result(0) + case "uninit": + let code = ready ? SparkChainGlueUnInit() : 0 + ready = false + result(code) + default: + result(FlutterMethodNotImplemented) + } + } + + private func doInit(_ args: [String: Any]) -> Int { + guard let appId = args["appId"] as? String, + let apiKey = args["apiKey"] as? String, + let apiSecret = args["apiSecret"] as? String else { + return -2 + } + if ready { + SparkChainGlueUnInit() + ready = false + } + let apiType = args["apiType"] as? Int ?? 1 + let dir = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0] + .appendingPathComponent("sparkchain", isDirectory: true) + try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + let code = SparkChainGlueInit(appId, apiKey, apiSecret, dir.path, Int32(apiType)) + ready = code == 0 + return Int(code) + } + + /// 大语种表用 zh-CN / ga。ITRANS 那张小表的中文是 cn，重试到 ITRANS 时再换。 + private func itsCode(_ short: String, smallTable: Bool) -> String { + switch short { + case "zh": return smallTable ? "cn" : "zh-CN" + case "en": return "en" + case "ga": return "ga" + default: return short + } + } + + private func doTranslate(_ args: [String: Any]) -> Int { + guard ready, let text = args["text"] as? String else { return ready ? -2 : -1 } + let id = args["id"] as? String ?? "" + let from = args["from"] as? String ?? "zh" + let to = args["to"] as? String ?? "en" + pendingIts[id] = "\(from)|\(to)|\(text)" + return runIts(id: id, from: from, to: to, text: text) + } + + private func runIts(id: String, from: String, to: String, text: String) -> Int { + let type = transOrder[transIndex] + let small = type == ITRANS || type == ITRANS_SG + let engine = ITS(formLanguage: itsCode(from, smallTable: small), toLanguage: itsCode(to, smallTable: small), type: type) + engine.callbacks = self + its = engine + return Int(engine.arun(text, userInfo: id)) + } + + private func doSpeak(_ args: [String: Any]) -> Int { + guard ready, let text = args["text"] as? String else { return ready ? -2 : -1 } + let id = args["id"] as? String ?? "" + let lang = args["lang"] as? String ?? "zh" + let voice = args["voice"] as? String + let engine: OnlineTTS + if lang == "ga" { + engine = OnlineTTS(region: RegionMSFT, vcn: voice ?? "ga-IE-OrlaNeural") + } else if lang == "en" { + engine = OnlineTTS(vcn: voice ?? "x_Catherine") + } else { + engine = OnlineTTS(vcn: voice ?? "x_xiaoyan") + } + engine.aue = "raw" + engine.cbs = self + tts = engine + return Int(engine.arun(text, userInfo: id)) + } + + private func doStartListen(_ args: [String: Any]) -> Int { + guard ready else { return -1 } + rtasr?.stop(true) + asr?.stop(true) + rtasr = nil + asr = nil + let lang = args["lang"] as? String ?? "zh" + let id = args["id"] as? String ?? "" + if lang == "ga" { + let engine = RTASR(region: RegionMSFT) + engine.delegate = self + engine.addStringParam("language_name", value: "ga-IE") + rtasr = engine + return Int(engine.start(withUserInfo: id)) + } + let engine = ASR() + engine.cbs = self + let code = lang == "en" ? "en_us" : "zh_cn" + engine.language(code) + engine.domain("iat") + engine.accent("mandarin") + if lang == "zh" { engine.dwa("wpgs") } + let attr = AudioAttributes() + attr.sampleRate = 16000 + attr.encoding = "raw" + attr.channels = 1 + attr.bitdepth = 16 + asr = engine + return Int(engine.start(attr, usrTag: id)) + } + + private func emit(_ event: [String: Any]) { + let sink = events + DispatchQueue.main.async { sink?(event) } + } + + public func onResult(_ result: ASRResult, usrTag: Any) { + emit([ + "kind": "asr", + "id": (usrTag as? String) ?? "", + "text": result.bestMatchText, + "final": result.status == 2, + "sid": result.sid, + ]) + } + + public func onError(_ error: ASRError, usrTag: Any) { + emit([ + "kind": "error", + "where": "asr", + "id": (usrTag as? String) ?? "", + "code": error.code, + "message": error.errMsg, + "sid": error.sid, + ]) + } + + public func onResult(_ result: RTASRResult, useInfo userInfo: Any?) { + let status = result.status + emit([ + "kind": "asr", + "id": (userInfo as? String) ?? "", + "text": result.data, + "final": status == 2 || status == 3, + "sid": result.sid, + ]) + } + + public func onError(_ error: RTASRError, useInfo userInfo: Any?) { + emit([ + "kind": "error", + "where": "asr", + "id": (userInfo as? String) ?? "", + "code": error.code, + "message": error.errMsg, + "sid": error.sid, + ]) + } + + public func its(_ its: ITS, on result: ITSResult, userInfo: Any) { + emit([ + "kind": "its", + "id": (userInfo as? String) ?? "", + "text": result.trans.dst, + "sid": result.sid, + ]) + } + + public func its(_ its: ITS, onError error: ITSError, userInfo: Any) { + let id = (userInfo as? String) ?? "" + if transIndex + 1 < transOrder.count, let packed = pendingIts[id] { + transIndex += 1 + let parts = packed.split(separator: "|", maxSplits: 2).map(String.init) + if parts.count == 3 { + _ = runIts(id: id, from: parts[0], to: parts[1], text: parts[2]) + return + } + } + emit([ + "kind": "error", + "where": "its", + "id": id, + "code": error.code, + "message": error.errMsg, + "sid": error.sid, + ]) + } + + public func tts(_ tts: TTS, on result: TTSResult, usrContext: Any) { + emit([ + "kind": "tts", + "id": (usrContext as? String) ?? "", + "pcm": FlutterStandardTypedData(bytes: result.data), + "final": result.status == 2, + "sid": result.sid, + ]) + } + + public func tts(_ tts: TTS, onError error: TTSError, usrContext: Any) { + emit([ + "kind": "error", + "where": "tts", + "id": (usrContext as? String) ?? "", + "code": error.code, + "message": error.errMsg, + "sid": error.sid, + ]) + } +} diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/Info.plist b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/Info.plist new file mode 100644 index 00000000..de3c190f --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/Info.plist @@ -0,0 +1,27 @@ + + + + + AvailableLibraries + + + BinaryPath + SparkChain.framework/SparkChain + LibraryIdentifier + ios-arm64 + LibraryPath + SparkChain.framework + SupportedArchitectures + + arm64 + + SupportedPlatform + ios + + + CFBundlePackageType + XFWK + XCFrameworkFormatVersion + 1.0 + + diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/AIPPT.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/AIPPT.h new file mode 100644 index 00000000..df7cb558 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/AIPPT.h @@ -0,0 +1,64 @@ +// +// AIPPT.h +// SparkChain_Mac +// +// Created by pcfang on 21.5.24. +// + +#import +@class PPTResult; +@class PPTError; +@class AIPPT; +NS_ASSUME_NONNULL_BEGIN +@protocol PPTCallback + +- (void)ppt:(AIPPT *)ppt onResult:(PPTResult *)result userInfo:(id)userInfo; + +- (void)ppt:(AIPPT *)ppt onError:(PPTError *)error userInfo:(id)userInfo; + +@end + +@interface AIPPT : NSObject + +@property (nonatomic, weak) id callback; + +- (instancetype)init; + +- (void)createModel:(NSString *)createModel; + +- (void)theme:(NSString *)theme; + +- (void)businessId:(NSString *)businessId; + +- (void)author:(NSString *)author; + +- (void)isCardNote:(BOOL)isCardNote; + +- (int)asyncPPTFromText:(NSString *)text; + +- (int)asyncPPTFromText:(NSString *)text userInfo:(id _Nullable)userInfo; + +- (int)asyncPPTOutlineCreat:(NSString *)text; + +- (int)asyncPPTOutlineCreat:(NSString *)text userInfo:(id _Nullable)userInfo; + +- (int)asyncPPTFromSid:(NSString *)sid outline:(NSString * _Nullable)outline; + +- (int)asyncPPTFromSid:(NSString *)sid outline:(NSString * _Nullable)outline userInfo:(id _Nullable)userInfo; + +- (int)asyncPPTFromOutline:(NSString *)outline query:(NSString *)query; + +- (int)asyncPPTFromOutline:(NSString *)outline query:(NSString *)query userInfo:(id _Nullable)userInfo; + +- (int)asyncPPTGetTopicList; + +- (int)asyncPPTGetTopicListWithUserInfo:(id _Nullable)userInfo; + +- (int)asyncPPTGetResult:(NSString *)sid; + +- (int)asyncPPTGetResult:(NSString *)sid userInfo:(id _Nullable)userInfo; + + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASR.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASR.h new file mode 100644 index 00000000..7008ce5f --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASR.h @@ -0,0 +1,83 @@ +// +// ASR.h +// SparkChain_Mac +// +// Created by XX on 2024/5/10. +// + +#ifndef ASR_h +#define ASR_h + + +#import +#import +#import "AudioAttributes.h" +#import "ASRCallbacks.h" +#import "ASRHotwordsResult.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface ASR : NSObject + +@property (nonatomic, weak) id cbs; + +- (instancetype)init; + +- (instancetype)initWithRegion:(RegionType)region; + +- (instancetype)initWithLanguage:(NSString *)language domain:(NSString *)domain accent:(NSString *)accent; + +- (instancetype)initWithLanguage:(NSString *)language domain:(NSString *)domain accent:(NSString *)accent region:(RegionType)region; + +- (int)start:(AudioAttributes *)attributes; +- (int)start:(AudioAttributes *)attributes usrTag:(id)usrTag; +- (int)write:(NSData *)data; +- (int)stop:(BOOL)immediate; + +- (void)language:(NSString *)language; +- (void)domain:(NSString *)domain; +- (void)accent:(NSString *)accent; +- (void)vadEos:(int)vadEos; +- (void)dwa:(NSString *)dwa; +- (void)pd:(NSString *)pd; +- (void)ptt:(BOOL)enable; +- (void)rlang:(NSString *)rlang; +- (void)vinfo:(BOOL)vinfo; +- (void)nunum:(BOOL)enable; +- (void)speexSize:(int)speexSize; +- (void)nbest:(int)nbest; +- (void)wbest:(int)wbest; +- (void)vgap:(int)vgap; +- (void)proc:(BOOL)proc; +- (void)smth:(BOOL)smth; +- (void)ln:(NSString *)ln; +- (void)opt:(int)opt; +- (void)dhw:(NSString *)dhw; +- (void)ltc:(int)ltc; + +- (void)svad:(int)svad; +- (void)vadEnable:(BOOL)enable; +- (void)targetLanuage:(NSString *)target; +- (void)region:(RegionType)region; + + +- (void)addIntParam:(NSString *)key value:(int)value; +- (void)addBoolParam:(NSString *)key value:(BOOL)value; +- (void)addStringParam:(NSString *)key value:(NSString *)value; + +- (ASRHotwordsResult *)uploadHotwordsWithResId:(NSString *)resId + hotwordsFile:(NSString *)hotwordsFile + namexp:(NSString *)namexp + wordType:(NSString *)wordType; +- (ASRHotwordsResult *)uploadHotwordsWithResId:(NSString *)resId + hotwords:(NSData *)hotwords + namexp:(NSString *)namexp + wordType:(NSString *)wordType; +- (ASRHotwordsResult *)queryHotwordsWithResId:(NSString *)resId; + +@end + +NS_ASSUME_NONNULL_END + + +#endif /* ASR_h */ diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASRCallbacks.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASRCallbacks.h new file mode 100644 index 00000000..355fe4ac --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASRCallbacks.h @@ -0,0 +1,23 @@ +// +// AsrCallbacks.h +// SparkChain_Mac +// +// Created by XX on 2024/5/11. +// + +#import +#import "ASRResult.h" +#import "ASRError.h" + +NS_ASSUME_NONNULL_BEGIN + +@protocol ASRCallbacks + +- (void)onResult:(ASRResult *)result usrTag:(id)usrTag; + +- (void)onError:(ASRError *)error usrTag:(id)usrTag; + +@end + + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASRError.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASRError.h new file mode 100644 index 00000000..5e2c7531 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASRError.h @@ -0,0 +1,25 @@ +// +// ASRError.h +// SparkChain_Mac +// +// Created by XX on 2024/5/11. +// + +#ifndef ASRError_h +#define ASRError_h + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface ASRError : NSObject + +@property (nonatomic, assign) int code; +@property (nonatomic, copy) NSString *errMsg; +@property (nonatomic, copy) NSString *sid; + +@end + +NS_ASSUME_NONNULL_END + +#endif /* ASRError_h */ diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASRHotwordsResult.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASRHotwordsResult.h new file mode 100644 index 00000000..3140f531 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASRHotwordsResult.h @@ -0,0 +1,22 @@ +// +// ASRHotwordsResult.h +// SparkChain +// + +#ifndef ASRHotwordsResult_h +#define ASRHotwordsResult_h + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface ASRHotwordsResult : NSObject + +@property (nonatomic, assign) int code; +@property (nonatomic, copy) NSString *response; + +@end + +NS_ASSUME_NONNULL_END + +#endif /* ASRHotwordsResult_h */ diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASRResult.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASRResult.h new file mode 100644 index 00000000..f9c6836f --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASRResult.h @@ -0,0 +1,31 @@ +// +// ASRResult.h +// SparkChain_Mac +// +// Created by XX on 2024/5/11. +// + +#ifndef ASRResult_h +#define ASRResult_h + +#import +#import "Transcription.h" +#import "Vad.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface ASRResult : NSObject + +@property (nonatomic, strong) NSArray *transcriptions; +@property (nonatomic, strong) NSArray *vads; +@property (nonatomic) int begin; +@property (nonatomic) int end; +@property (nonatomic, copy) NSString *sid; +@property (nonatomic, copy) NSString *bestMatchText; +@property (nonatomic) int status; + +@end + +NS_ASSUME_NONNULL_END + +#endif /* ASRResult_h */ diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/AST.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/AST.h new file mode 100644 index 00000000..a10d0119 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/AST.h @@ -0,0 +1,81 @@ +// +// AST.h +// SparkChain_iOS +// +// Created by pcfang on 2025/11/14. +// + +#import +#import +@class AST; +@class ASTResult; +@class ASTError; + +NS_ASSUME_NONNULL_BEGIN + +// 回调协议 +@protocol ASTDelegate + +- (void)ast:(AST *)ast onResult:(ASTResult *)result userInfo:(_Nullable id)userInfo; + +- (void)ast:(AST *)ast onError:(ASTError *)error userInfo:(_Nullable id)userInfo; + +@end + +@interface AST : NSObject + +@property (nonatomic, weak, readonly) id delegate; + +- (instancetype)init NS_UNAVAILABLE; + +- (instancetype)initWithDelegate:(id) delegate; + +- (instancetype)initWithregion:(RegionType)region delegate:(id)delegate; + +- (int)start:(NSString *)lang audioEncoding:(NSString *)encoding sampleRate:(int)sampleRate userInfo:(_Nullable id)userInfo; + +- (int)write:(NSData *)data; + +- (int)stop:(BOOL)immediate; + + +#pragma mark - 热词管理 + +- (int)setHotwords:(NSString *)hotwords; + +- (int)uploadHotwords:(NSString *)hotwordsFile userInfo:(_Nullable id)userInfo; + +- (int)updateHotwords:(NSString *)hotwordsFile hotwordId:(int)hotwordId userInfo:(_Nullable id)userInfo; + +- (int)queryHotWords:(int)hotwordId userInfo:(_Nullable id)userInfo; + +#pragma mark - 翻译列表管理 + +- (int)setTransLists:(NSDictionary *)transLists; + +- (int)uploadTransLists:(NSString *)from to:(NSString *)to transLists:(NSDictionary *)transLists userInfo:(_Nullable id)userInfo; + +#pragma mark - 替换词管理 +- (int)setReplaceWords:(NSDictionary *)replaceWords; + +- (int)uploadReplaceList:(NSDictionary *)replaceList userInfo:(_Nullable id)userInfo; + +- (int)deleteReplaceList:(NSString *)replaceListId userInfo:(_Nullable id)userInfo; + +#pragma mark - 声纹管理 +- (int)registerVoicePrint:(NSData *)audioData audioType:(NSString *)audioType userInfo:(_Nullable id)userInfo; + +- (int)updateVoicePrint:(NSData *)audioData audioType:(NSString *)audioType featureId:(NSString *)featureId userInfo:(_Nullable id)userInfo; + +- (int)deleteVoicePrint:(NSArray *)featureIds userInfo:(_Nullable id)userInfo; + +#pragma mark - 参数设置 +- (void)addIntParam:(NSString *)key value:(int)value; + +- (void)addBoolParam:(NSString *)key value:(BOOL)value; + +- (void)addStringParam:(NSString *)key value:(NSString *)value; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASTError.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASTError.h new file mode 100644 index 00000000..ce7e51e3 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASTError.h @@ -0,0 +1,30 @@ +// +// ASTError.h +// SparkChain_iOS +// +// Created by pcfang on 2025/11/19. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface ASTError : NSObject + +@property (nonatomic, readonly) NSString *code; + +@property (nonatomic, readonly) NSString *sid; + +@property (nonatomic, readonly) NSString *fnType; + +@property (nonatomic, readonly) NSString *detail; + +@property (nonatomic, readonly) NSString *desc; + +@property (nonatomic, readonly) NSString *normal; + +@property (nonatomic, readonly) NSString *resType; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASTResult.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASTResult.h new file mode 100644 index 00000000..56faa283 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ASTResult.h @@ -0,0 +1,80 @@ +// +// ASTTransResult.h +// SparkChain_iOS +// +// Created by pcfang on 2025/11/14. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +// AST 声纹结果 +@interface ASTVoicePrintResult : NSObject + +@property (nonatomic, readonly) NSString *sid; +@property (nonatomic, readonly) NSString *featureId; +@property (nonatomic, readonly) NSInteger status; +@property (nonatomic, readonly) NSString *delFailIds; + +@end + +// AST TTS结果 +@interface ASTTtsResult : NSObject + +@property (nonatomic, readonly) NSInteger status; +@property (nonatomic, readonly) NSData * audio; +@property (nonatomic, readonly) NSString *engine; +@property (nonatomic, readonly) NSString *lang; +@property (nonatomic, readonly) NSString *rawResult; + +@end + +// AST 翻译结果 +@interface ASTTransResult : NSObject + +@property (nonatomic, readonly) NSString *src; +@property (nonatomic, readonly) NSString *dst; +@property (nonatomic, readonly) NSString *from; +@property (nonatomic, readonly) NSString *to; +@property (nonatomic, readonly) NSInteger type; +@property (nonatomic, readonly) BOOL end; +@property (nonatomic, readonly) NSString *isOver; +@property (nonatomic, readonly) NSString *isFinal; +@property (nonatomic, readonly) NSString *dstFixedPart; + +@end + +// AST热词结果 +@interface ASTHotwordsResult : NSObject + +@property (nonatomic, readonly) NSInteger hotWordId; +@property (nonatomic, readonly) NSString *hotWord; +@property (nonatomic, readonly) NSString *sid; +@property (nonatomic, readonly) NSString *engineUid; +@property (nonatomic, readonly) NSString *resId; +@property (nonatomic, readonly) NSString *lang; + +@end + +@interface ASTResult : NSObject + +@property (nonatomic, readonly) NSString *contextId; +@property (nonatomic, readonly) NSString *sessionId; +@property (nonatomic, readonly) NSString *resType; +@property (nonatomic, readonly) NSInteger status; +@property (nonatomic, readonly) NSInteger isEnd; +@property (nonatomic, readonly) NSString *data; +@property (nonatomic, readonly) NSString *rawResult; + +@property (nonatomic, readonly) ASTTransResult *transResult; +@property (nonatomic, readonly) ASTTtsResult *ttsResult; +@property (nonatomic, readonly) ASTHotwordsResult *hotwordsResult; +@property (nonatomic, readonly) ASTVoicePrintResult *voicePrintResult; +@property (nonatomic, readonly) NSString *transReplaceWordId; +@property (nonatomic, readonly) NSString *replaceListId; +@property (nonatomic, readonly) NSString *replaceListSid; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/AudioAttributes.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/AudioAttributes.h new file mode 100644 index 00000000..2694dc08 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/AudioAttributes.h @@ -0,0 +1,29 @@ +// +// AudioAttributes.h +// SparkChain_Mac +// +// Created by XX on 2024/5/11. +// + +#ifndef AudioAttributes_h +#define AudioAttributes_h + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface AudioAttributes : NSObject + +@property (nonatomic, assign) int sampleRate; +@property (nonatomic, copy) NSString *encoding; +@property (nonatomic, assign) int channels; +@property (nonatomic, assign) int bitdepth; +@property (nonatomic, assign) int frameSize; + +- (instancetype)init; + +@end + +NS_ASSUME_NONNULL_END + +#endif /* AudioAttributes_h */ diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Embedding.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Embedding.h new file mode 100644 index 00000000..f0077b25 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Embedding.h @@ -0,0 +1,20 @@ +// +// Embedding.h +// SparkChain +// +// Created by pcfang on 2024/11/28. +// + +#import +@class EmbeddingOutput; +NS_ASSUME_NONNULL_BEGIN + +@interface Embedding : NSObject + ++ (EmbeddingOutput *)embedding:(NSString *)input domain:(NSString *)domain; + ++ (EmbeddingOutput *)embedding:(NSString *)input domain:(NSString *)domain uid:(NSString * __nullable)uid; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/EmbeddingOutput.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/EmbeddingOutput.h new file mode 100644 index 00000000..b0fed205 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/EmbeddingOutput.h @@ -0,0 +1,34 @@ +// +// EmbeddingOutput.h +// SparkChain +// +// Created by pcfang on 2024/11/28. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface EmbeddingData : NSObject + +@property (readonly) float * byte NS_RETURNS_INNER_POINTER; + +@property (nonatomic, assign) NSUInteger length; + +@end + +@interface EmbeddingOutput : NSObject + +@property (nonatomic,copy,readonly) NSString * sid; + +@property (nonatomic,strong,readonly) EmbeddingData * data; + +@property (nonatomic,copy,readonly) NSString * raw; + +@property (nonatomic,copy,readonly) NSString * errMsg; + +@property (nonatomic,assign,readonly) int errCode; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Feature.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Feature.h new file mode 100644 index 00000000..68c35e2d --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Feature.h @@ -0,0 +1,57 @@ +// +// Feature.h +// SparkChain_iOS +// + +#import + +@class FeatureResult; +@class FeatureError; + +NS_ASSUME_NONNULL_BEGIN + +@protocol FeatureCallback + +- (void)onResult:(FeatureResult *)result userInfo:(id _Nullable)userInfo; +- (void)onError:(FeatureError *)error userInfo:(id _Nullable)userInfo; + +@end + +@interface Feature : NSObject + +@property (nonatomic, weak) id delegate; + +- (instancetype)init; + +/// 声纹注册 +/// - Parameters: +/// - audioData: 音频数据 +/// - audioType: 音频类型 +/// - uid: 用户 id +/// - userInfo: 用户自定义标识，可为 nil；非 nil 时会在回调中回传 +- (int)featureRegisterWithAudioData:(NSData *)audioData + audioType:(NSString *)audioType + uid:(NSString *)uid + userInfo:(id _Nullable)userInfo; + +/// 声纹更新 +/// - Parameters: +/// - audioData: 音频数据 +/// - audioType: 音频类型 +/// - featureId: 声纹 id +/// - userInfo: 用户自定义标识 +- (int)featureUpdateWithAudioData:(NSData *)audioData + audioType:(NSString *)audioType + featureId:(NSString *)featureId + userInfo:(id _Nullable)userInfo; + +/// 声纹删除（多个声纹 id 以逗号隔开） +/// - Parameters: +/// - featureIds: 声纹 id，多个以逗号分隔 +/// - userInfo: 用户自定义标识 +- (int)featureDeleteWithFeatureIds:(NSString *)featureIds + userInfo:(id _Nullable)userInfo; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/FeatureError.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/FeatureError.h new file mode 100644 index 00000000..cffb1d3e --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/FeatureError.h @@ -0,0 +1,17 @@ +// +// FeatureError.h +// SparkChain_iOS +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface FeatureError : NSObject + +@property (nonatomic, copy, readonly) NSString *code; +@property (nonatomic, copy, readonly) NSString *message; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/FeatureResult.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/FeatureResult.h new file mode 100644 index 00000000..569e223d --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/FeatureResult.h @@ -0,0 +1,16 @@ +// +// FeatureResult.h +// SparkChain_iOS +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface FeatureResult : NSObject + +@property (nonatomic, copy, readonly) NSString *featureId; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/IST.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/IST.h new file mode 100644 index 00000000..0be61aad --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/IST.h @@ -0,0 +1,112 @@ +// +// IST.h +// SparkChain_iOS +// 极速语音转写 +// Created by pcfang on 2025/8/7. +// + +#import + +NS_ASSUME_NONNULL_BEGIN +@class IST; +@class ISTResult; +@class ISTError; + +@protocol ISTDelegate +@optional +- (void)ist:(IST *)ist onResult:(ISTResult *)result userInfo:(id)userInfo; + +- (void)ist:(IST *)ist onProcess:(NSString *)process userInfo:(id)userInfo; + +- (void)ist:(IST *)ist onError:(ISTError *)error userInfo:(id)userInfo; + +@end + +@interface IST : NSObject + +@property (nonatomic, copy) NSString * requestId; + +@property (nonatomic, copy) NSString * language; + +@property (nonatomic, copy) NSString * domain; + +@property (nonatomic, copy) NSString * accent; + +@property (nonatomic, copy) NSString * callbackUrl; + +@property (nonatomic, assign) int vsppOn; + +@property (nonatomic, assign) int speakerNum; + +@property (nonatomic, assign) int outputType; + +@property (nonatomic, assign) int postprocON; + +@property (nonatomic, copy) NSString * pd; + +@property (nonatomic, assign) int duration; + +@property (nonatomic, assign) int enableSubtitle; + +@property (nonatomic, assign) BOOL smoothproc; + +@property (nonatomic, assign) BOOL colloqproc; + +@property (nonatomic, assign) BOOL languageType; + +@property (nonatomic, assign) int vto; + +@property (nonatomic, copy) NSString * dhw; + +@property (nonatomic, weak) id delegate; + +- (void)addIntParam:(NSString *)key value:(int)value; +- (void)addBoolParam:(NSString *)key value:(BOOL)value; +- (void)addStringParam:(NSString *)key value:(NSString *)value; + + +- (instancetype)initDelegate:(id) delegate; + + +/// 上传音频文件,返回0成功，其他失败 +/// - Parameters: +/// - audioPath: 音频文件路径 +/// - requestId: 请求ID +/// - userInfo: 用户自定义数据 +- (int)upload:(NSString *) audioPath requestId:(NSString *)requestId userInfo:(__nullable id)userInfo; + +/// 分片上传音频文件，0表示成功，其他表示失败 +/// - Parameters: +/// - audioPath: 音频文件路径 +/// - requestId: 请求ID +/// - partSize: 分片大小 +/// - userInfo: 用户自定义数据 +- (int)mpUpload:(NSString *)audioPath + requestId:(NSString *)requestId + partSize:(NSUInteger)partSize + userInfo:(__nullable id)userInfo; + +/// 创建任务，0表示成功，其他表示失败 +/// - Parameters: +/// - requestId: 请求ID +/// - path: 音频文件URL +/// - format: 音频的采样率支持16k,audio/L16;rate=16000 +/// - encoding: 音频数据格式：wav、pcm（传参raw），mp3（传参lame） +/// - userInfo: 用户自定义数据 +- (int)createTask:(NSString *)requestId + audioURLPath:(NSString *)path + format:(NSString *)format + encoding:(NSString *)encoding + userInfo:(__nullable id)userInfo; + +/// 查询任务，0表示成功，其他表示失败 +/// - Parameters: +/// - taskId: 任务ID +/// - userInfo: 用户自定义数据 +- (int)queryTask:(NSString *)taskId userInfo:(__nullable id)userInfo; + +/// 停止识别 +- (int)stop; +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ISTError.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ISTError.h new file mode 100644 index 00000000..a8c2ee04 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ISTError.h @@ -0,0 +1,22 @@ +// +// ISTError.h +// SparkChain_iOS +// +// Created by pcfang on 2025/8/7. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface ISTError : NSObject + +@property(nonatomic, copy, readonly) NSString * sid; + +@property(nonatomic, assign, readonly) int code; + +@property(nonatomic, copy, readonly) NSString * errMsg; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ISTResult.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ISTResult.h new file mode 100644 index 00000000..48721260 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ISTResult.h @@ -0,0 +1,34 @@ +// +// ISTResult.h +// SparkChain_iOS +// +// Created by pcfang on 2025/8/7. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface ISTResult : NSObject + +@property (nonatomic, assign, readonly) int code; + +@property (nonatomic, copy, readonly) NSString * key; + +@property (nonatomic, copy, readonly) NSString * sid; + +@property (nonatomic, copy, readonly) NSString * url; + +@property (nonatomic, copy, readonly) NSString * taskId; + +@property (nonatomic, copy, readonly) NSString * taskStatus; + +@property (nonatomic, copy, readonly) NSString * taskType; + +@property (nonatomic, copy, readonly) NSString * result; + +@property (nonatomic, copy, readonly) NSString * rawResult; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ITS.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ITS.h new file mode 100644 index 00000000..66c72b89 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ITS.h @@ -0,0 +1,53 @@ +// +// ITS.h +// SparkChain_Mac +// +// Created by pcfang on 15.5.24. +// + +#import +@class ITS; +@class ITSResult; +@class ITSError; +NS_ASSUME_NONNULL_BEGIN + +typedef enum { + ITRANS = 0, //讯飞翻译 + NIUTRANS, //小牛翻译 + ITRANS_SG, //讯飞翻译(新加坡) + GOOGLETRANS //谷歌翻译 +} TransType; + +@protocol ITSCallbacks + +- (void)its:(ITS *)its onResult:(ITSResult *)result userInfo:(id)userInfo; + +- (void)its:(ITS *)its onError:(ITSError *)error userInfo:(id)userInfo; + +@end + +@interface ITS : NSObject + +- (instancetype) init; + +- (instancetype) initWithType:(TransType)type; + +- (instancetype) initWithFormLanguage:(NSString *)from toLanguage:(NSString *)to; + +- (instancetype)initWithFormLanguage:(NSString *)from toLanguage:(NSString *)to type:(TransType)type; + +@property (nonatomic, weak) id callbacks; + +@property (nonatomic, copy) NSString * fromLanguage; + +@property (nonatomic, copy) NSString * toLanguage; + +@property (nonatomic, copy) NSString * resId; + +- (int)arun:(NSString *)text; + +- (int)arun:(NSString *)text userInfo:(_Nullable id)userInfo; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ITSError.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ITSError.h new file mode 100644 index 00000000..5221086d --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ITSError.h @@ -0,0 +1,22 @@ +// +// ITSError.h +// SparkChain_Mac +// +// Created by pcfang on 15.5.24. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface ITSError : NSObject + +@property(nonatomic, copy, readonly) NSString * sid; + +@property(nonatomic, assign, readonly) int code; + +@property(nonatomic, copy, readonly) NSString * errMsg; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ITSResult.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ITSResult.h new file mode 100644 index 00000000..559f019f --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ITSResult.h @@ -0,0 +1,34 @@ +// +// ITSResult.h +// SparkChain_Mac +// +// Created by pcfang on 15.5.24. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface Trans : NSObject + +@property (nonatomic, copy, readonly) NSString * src; + +@property (nonatomic, copy, readonly) NSString * dst; + +@end + +@interface ITSResult : NSObject + +@property (nonatomic, copy, readonly) NSString * sid; + +@property (nonatomic, assign, readonly) int status; + +@property (nonatomic, copy, readonly) NSString * from; + +@property (nonatomic, copy, readonly) NSString * to; + +@property (nonatomic, strong, readonly) Trans * trans; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ImageTrans.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ImageTrans.h new file mode 100644 index 00000000..5746490d --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ImageTrans.h @@ -0,0 +1,40 @@ +// +// ImageTrans.h +// IMAGE_TRANS +// +// Created by xyzhang on 2024/5/28. +// + +#import + +@class ImageTrans; +@class ImageTransResult; +@class ImageTransError; + +NS_ASSUME_NONNULL_BEGIN + +@protocol ImageTransDelegate + +- (void)imgTrans:(ImageTrans *)imgTrans onResult:(ImageTransResult *)result usrContext:(id)usrContext; + +- (void)imgTrans:(ImageTrans *)imgTrans onError:(ImageTransError *)error usrContext:(id)usrContext; + +@end + +@interface ImageTrans : NSObject + +@property (nonatomic, weak) id delegate; +//@property (nonatomic, copy) NSString *imagePath; +@property (nonatomic, copy) NSString *ocrLanguage; +@property (nonatomic, copy) NSString *fromLanguage; +@property (nonatomic, copy) NSString *toLanguage; +@property (nonatomic, assign) int returnType; // ??? + +- (instancetype)init; +- (instancetype)initWithOcrLanguage:(NSString *)ocrLanguage fromLanguage:(NSString *)fromLanguage toLanguage:(NSString *)toLanguage returnType:(int)returnType; + +- (int)arun:(NSData *)imageData encoding:(NSString *)encoding userInfo:(id)userInfo; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ImageTransError.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ImageTransError.h new file mode 100644 index 00000000..f4d8340a --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ImageTransError.h @@ -0,0 +1,20 @@ +// +// ImageTransError.h +// IMAGE_TRANS +// +// Created by xyzhang on 2024/5/28. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface ImageTransError : NSObject + +@property (nonatomic, copy, readonly) NSString *sid; +@property (nonatomic, copy, readonly) NSString *errMsg; +@property (nonatomic, assign, readonly) int code; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ImageTransResult.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ImageTransResult.h new file mode 100644 index 00000000..d7a907d3 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/ImageTransResult.h @@ -0,0 +1,32 @@ +// +// ImageTransResult.h +// IMAGE_TRANS +// +// Created by xyzhang on 2024/5/28. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface BlockText : NSObject + +@property (nonatomic, copy) NSString *srcText; +@property (nonatomic, copy) NSString *dstText; + +@end + +@interface ImageTransResult : NSObject + +@property (nonatomic, copy, readonly) NSString *sid; +//@property (nonatomic, copy, readonly) NSString *itsImage; +//@property (nonatomic, copy, readonly) NSString *itsOutput; +@property (nonatomic, copy, readonly) NSData *itsImageData; //图片翻译 二进制数据 +@property (nonatomic, copy, readonly) NSString *itsOutputJsStr; // 整体 + +@property (nonatomic, assign, readonly) int status; +@property (nonatomic, copy, readonly) NSArray *blockTexts; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLM.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLM.h new file mode 100644 index 00000000..55b912c1 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLM.h @@ -0,0 +1,168 @@ +// +// LLM.h +// SparkChain +// +// Created by pcfang on 12.9.23. +// + +#import +@class LLM; +@class LLMEvent; +@class LLMError; +@class LLMResult; +@class LLMConfig; +@class LLMOutput; +@class LLMTools; +@class Memory; + +NS_ASSUME_NONNULL_BEGIN + +typedef enum{ + TEXT_GENERATION = 0, + IMAGE_GENERATION, + IMAGE_UNDERSTANDING +} ModelType; + +@protocol LLMCallback + +- (void)llm:(LLM *)llm onResult:(LLMResult * _Nullable)result usrContext:(id _Nullable)usrContext; + +- (void)llm:(LLM *)llm onEvent:(LLMEvent * _Nullable)event usrContext:(id _Nullable)usrContext; + +- (void)llm:(LLM *)llm onError:(LLMError * _Nullable)error usrContext:(id _Nullable)usrContext; + +@end + + +@interface LLM : NSObject + +@property(nonatomic, weak) id callback; + +@property(nonatomic, strong, readonly) LLMConfig * config; + +#pragma mark - init +- (instancetype)init; + +- (instancetype)initWithConfig:(LLMConfig * _Nullable)config; + +- (instancetype)initWithCallback:(id _Nullable)callback; + +- (instancetype)initWithConfig:(LLMConfig * _Nullable)config callback:(id _Nullable)callback; + +- (instancetype)initWithModelType:(ModelType)modelType config:(LLMConfig *)config; + +- (instancetype)initWithModelType:(ModelType)modelType config:(LLMConfig * _Nullable)config memory:(Memory * _Nullable)memory; + +#pragma mark - 同步接口 +/// 同步获取LLM结果 +/// - Parameter query: 用户问题 +- (LLMOutput *)run:(NSString *)query; + +- (LLMOutput *)run:(NSString *)query image:(NSData *)image; + +/// 同步获取LLM结果 +/// - Parameters: +/// - query: 用户问题 +/// - ttl: 超时时间 +- (LLMOutput *)run:(NSString *)query ttl:(int)ttl; + +/// 同步获取LLM结果 +/// - Parameters: +/// - query: 用户问题 +/// - tools: 工具（如function call） +- (LLMOutput *)run:(NSString *)query tools:(LLMTools *)tools; + +/// 同步获取LLM结果 +/// - Parameters: +/// - query: 用户问题 +/// - tools: 工具（如function call） +/// - ttl: 超时时间 +- (LLMOutput *)run:(NSString *)query tools:(LLMTools *)tools ttl:(int)ttl; + +/// 同步获取图片理解结果 +/// - Parameters: +/// - query: 用户问题 +/// - image: 图片 +- (LLMOutput *)run:(NSString *)query image:(NSData *)image; + +/// 使用raw json同步调用大模型 +/// - Parameters: +/// - json: 原始JSON +/// - ttl: 超时时间 +- (LLMOutput *)runWithJSON:(NSString *)json ttl:(int)ttl; + +#pragma mark - 异步接口 +/// 异步获取LLM结果 +/// - Parameter query: 用户问题 +- (int)arun:(NSString *)query; + +/// 异步获取LLM结果 +/// - Parameters: +/// - query: 用户问题 +/// - usrContext: 用户自定义数据 +- (int)arun:(NSString *)query usrContext:(id _Nullable)usrContext; + +- (int)arun:(NSString *)query image:(NSData *)image usrContext:(id _Nullable)usrContext; + +/// 异步获取LLM结果 +/// - Parameters: +/// - query: 用户问题 +/// - tools: 工具（如function call） +- (int)arun:(NSString *)query tools:(LLMTools *)tools; + +/// 异步获取LLM结果 +/// - Parameters: +/// - query: 用户问题 +/// - tools: 工具（如function call） +/// - usrContext: 用户自定义数据 +- (int)arun:(NSString *)query tools:(LLMTools *)tools usrContext:(id _Nullable)usrContext; + +/// 异步获取图片理解结果 +/// - Parameters: +/// - query: 用户问题 +/// - image: 图片 +- (int)arun:(NSString *)query image:(NSData *)image; + +/// 异步获取图片理解结果 +/// - Parameters: +/// - query: 用户问题 +/// - image: 图片 +/// - usrContext: 用户自定义数据 +- (int)arun:(NSString *)query image:(NSData *)image usrContext:(id _Nullable)usrContext; + +/// 使用raw json异步调用大模型 +/// - Parameters: +/// - json: 原始JSON +/// - usrContext: 用户自定义数据 +- (int)arunWithJSON:(NSString *)json usrContext:(id _Nullable)usrContext; + +#pragma mark - 其他 +/// 添加system角色（如：你是一个智能旅游助手，推荐旅游地点） +/// - Parameter prompt: system内容 +- (int)addSystemPrompt:(NSString *)prompt; + +/// 清除上下文 +- (void)clearHistory; + +/// 停止输出 +- (int)stop; + +@end + + +@interface LLMFactory : NSObject + ++ (LLM *)textGeneration:(LLMConfig * _Nullable)config; + ++ (LLM *)textGeneration:(LLMConfig * _Nullable)config memory:(Memory * _Nullable)memory; + ++ (LLM *)imageGeneration:(LLMConfig * _Nullable)config; + ++ (LLM *)imageGeneration:(LLMConfig * _Nullable)config width:(int)width height:(int)height; + ++ (LLM *)imageUnderstanding:(LLMConfig * _Nullable)config; + ++ (LLM *)imageUnderstanding:(LLMConfig * _Nullable)config memory:(Memory * _Nullable)memory; +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMBaseOutput.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMBaseOutput.h new file mode 100644 index 00000000..ad670f1c --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMBaseOutput.h @@ -0,0 +1,20 @@ +// +// LLMBaseOutput.h +// SparkChain +// +// Created by pcfang on 15.9.23. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface LLMBaseOutput : NSObject + +@property (nonatomic, copy, readonly) NSString * sid; + + +- (instancetype)initWithSid:(NSString *)sid; +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMConfig.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMConfig.h new file mode 100644 index 00000000..444c336c --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMConfig.h @@ -0,0 +1,46 @@ +// +// LLMConfig.h +// SparkChain +// +// Created by pcfang on 12.9.23. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +#define chainProperty(returnType,name) property(nonatomic, readonly, copy) returnType (^name) + +@interface LLMConfig : NSObject + +@chainProperty(LLMConfig *,uid)(NSString * uid); + +@chainProperty(LLMConfig *,domain)(NSString * domain); + +@chainProperty(LLMConfig *,auditing)(NSString * auditing); + +@chainProperty(LLMConfig *,chatID)(NSString * chatID); + +@chainProperty(LLMConfig *,temperature)(float temperature); + +@chainProperty(LLMConfig *,topK)(int topK); + +@chainProperty(LLMConfig *,maxToken)(int maxToken); + +@chainProperty(LLMConfig *,url)(NSString * url); + +@chainProperty(LLMConfig *,showRefLabel)(BOOL isShow); + +@chainProperty(LLMConfig *,tools)(NSString * toolList); + +@chainProperty(LLMConfig *,addString)(NSString * key, NSString * param); + +@chainProperty(LLMConfig *,addDouble)(NSString * key,double param); + +@chainProperty(LLMConfig *,addInt)(NSString * key,int param); + +@chainProperty(LLMConfig *,addBool)(NSString * key,BOOL param); + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMError.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMError.h new file mode 100644 index 00000000..e625429d --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMError.h @@ -0,0 +1,21 @@ +// +// LLMError.h +// SparkChain +// +// Created by pcfang on 12.9.23. +// + +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface LLMError : LLMBaseOutput + +@property (nonatomic, assign, readonly) int errCode; + +@property (nonatomic, copy, readonly) NSString * errMsg; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMEvent.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMEvent.h new file mode 100644 index 00000000..f38243fb --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMEvent.h @@ -0,0 +1,21 @@ +// +// LLMEvent.h +// SparkChain +// +// Created by pcfang on 12.9.23. +// + +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface LLMEvent : LLMBaseOutput + +@property (nonatomic, assign, readonly) int eventID; + +@property (nonatomic, copy, readonly) NSString * eventMsg; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMResult.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMResult.h new file mode 100644 index 00000000..30799654 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMResult.h @@ -0,0 +1,65 @@ +// +// LLMResult.h +// SparkChain +// +// Created by pcfang on 12.9.23. +// + +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface LLMResult : LLMBaseOutput + +@property (nonatomic, copy, readonly) NSString * _Nullable raw; + +@property (nonatomic, copy, readonly) NSString * _Nullable role; + +@property (nonatomic, copy, readonly) NSString * _Nullable content; + +@property (nonatomic, copy, readonly) NSString * _Nullable reasoningContent; + +@property (nonatomic, copy, readonly) NSData * _Nullable image; + +@property (nonatomic, assign, readonly) int status; + +@property (nonatomic, assign, readonly) int completionTokens; + +@property (nonatomic, assign, readonly) int promptTokens; + +@property (nonatomic, assign, readonly) int totalTokens; + +@property (nonatomic, copy, readonly) NSString * functionCall; +@end + + +@interface LLMOutput : LLMBaseOutput + +@property (nonatomic, copy, readonly) NSString * _Nullable raw; + +@property (nonatomic, copy, readonly) NSString * _Nullable role; + +@property (nonatomic, copy, readonly) NSString * _Nullable content; + +@property (nonatomic, copy, readonly) NSString * _Nullable reasoningContent; + +@property (nonatomic, copy, readonly) NSData * _Nullable image; + +@property (nonatomic, assign, readonly) int status; + +@property (nonatomic, assign, readonly) int completionTokens; + +@property (nonatomic, assign, readonly) int promptTokens; + +@property (nonatomic, assign, readonly) int totalTokens; + +@property (nonatomic, assign, readonly) int errCode; + +@property (nonatomic, copy, readonly) NSString * _Nullable errMsg; + +@property (nonatomic, copy, readonly) NSString * functionCall; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMTools.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMTools.h new file mode 100644 index 00000000..ba6c9466 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/LLMTools.h @@ -0,0 +1,22 @@ +// +// LLMTools.h +// SparkChain +// +// Created by pcfang on 2024/11/21. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface LLMTools : NSObject + +@property(nonatomic, copy) NSString * toolType; + +@property(nonatomic, copy) NSString * toolDescription; + +- (instancetype)initWithType:(NSString *)type description:(NSString *)description; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Memory.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Memory.h new file mode 100644 index 00000000..ad945508 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Memory.h @@ -0,0 +1,28 @@ +// +// Memory.h +// SparkChain +// +// Created by pcfang on 2024/11/22. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface Memory : NSObject + +- (instancetype) init NS_UNAVAILABLE; + +/// 按照轮数进行大模型交互上下文记忆，默认5轮,超出轮数自动移除最早的会话 ++ (instancetype)windowMemory; +/// 按照轮数进行大模型交互上下文记忆，用户自定义轮数，超出轮数自动移除最早的会话 ++ (instancetype)windowMemory:(int)maxCache; + +/// 按照token进行大模型交互上下文记忆，默认4096 ++ (instancetype)tokenMemory; +/// 按照token进行大模型交互上下文记忆，用户自定义token ++ (instancetype)tokenMemory:(int)maxToken; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/OCR.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/OCR.h new file mode 100644 index 00000000..5de94ab3 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/OCR.h @@ -0,0 +1,37 @@ +// +// OCR.h +// SparkChain_Mac +// +// Created by pcfang on 16.5.24. +// + +#import +@class OCR; +@class OCRResult; +@class OCRError; + +NS_ASSUME_NONNULL_BEGIN + +@protocol OCRCallbacks + +- (void)ocr:(OCR *)ocr onResult:(OCRResult *)result userInfo:(id)userInfo; + +- (void)ocr:(OCR *)ocr onError:(OCRError *)error userInfo:(id)userInfo; + +@end + +@interface OCR : NSObject + +@property (nonatomic, weak) id callbacks; + +- (int)arunWithImagePath:(NSString *)imagePath; + +- (int)arunWithImagePath:(NSString *)imagePath userInfo:(id)userInfo; + +- (int)arunWithImageData:(NSData *)imageData; + +- (int)arunWithImageData:(NSData *)imageData userInfo:(id)userInfo; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/OCRError.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/OCRError.h new file mode 100644 index 00000000..da293583 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/OCRError.h @@ -0,0 +1,22 @@ +// +// OCRError.h +// SparkChain_Mac +// +// Created by pcfang on 17.5.24. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface OCRError : NSObject + +@property(nonatomic, copy, readonly) NSString * sid; + +@property(nonatomic, assign, readonly) int code; + +@property(nonatomic, copy, readonly) NSString * errMsg; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/OCRExtension.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/OCRExtension.h new file mode 100644 index 00000000..bae48ceb --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/OCRExtension.h @@ -0,0 +1,27 @@ +// +// OCRExtension.h +// SparkChain_Mac +// +// Created by pcfang on 17.5.24. +// + +#import "OCRResult.h" +#import "OCRError.h" +#include + +@interface OCRLine () +- (instancetype)initWithLine:(SparkChain::Line *)line; +@end + +@interface OCRResult () + +- (instancetype)initWithResult:(SparkChain::OCRResult *)result; + +@end + + +@interface OCRError () + +- (instancetype)initWithError:(SparkChain::OCRError *)error; + +@end diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/OCRResult.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/OCRResult.h new file mode 100644 index 00000000..2a2ac94c --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/OCRResult.h @@ -0,0 +1,52 @@ +// +// OCRResult.h +// SparkChain_Mac +// +// Created by pcfang on 17.5.24. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface OCRLine : NSObject +/// 文本块被旋转的角度 +@property (nonatomic, assign, readonly) int angle; +///字符中心坐标，Int类型 +@property (nonatomic, copy, readonly) NSArray *> * charCenters; +/// 字符块坐标（每个字符左上角起，顺时针一周四角坐标形成的集合）, int类型 +@property (nonatomic, copy, readonly) NSArray *> * charPolygons; +/// 字符置信度，值域范围:0-1，double类型 +@property (nonatomic, copy, readonly) NSArray * charScore; +/// 文本块坐标(文本阅读方向的左上角起，顺时针一周四角坐标形成的集合),int类型 +@property (nonatomic, copy, readonly) NSArray * position; +/// 文本块所属属性表下标，从0开始 +@property (nonatomic, assign, readonly) int property; +/// 文本块的整体置信度 +@property (nonatomic, assign, readonly) double score; +/// 文本块内容 +@property (nonatomic, copy, readonly) NSString * text; +@end + +@interface OCRResult : NSObject +/// 请求的sid +@property (nonatomic, copy, readonly) NSString * sid; +/// 返回结果的状态 （0：首结果，1：中间结果，2：尾结果, 3：一次性结果） +@property (nonatomic, assign, readonly) int status; +/// 图片旋转角度 +@property (nonatomic, assign, readonly) int imageAngle; +/// 文本块识别信息集合 +@property (nonatomic, copy, readonly) NSArray * lines; +/// 文本块属性表，可选值：text（文本）、stamp（印章）、formula（公示）； +/// 与lines里的每个元素的property搭配使用， +/// property：0文本、1印章、2公式 +@property (nonatomic, copy, readonly) NSArray * properties; +/// 旋转后图片宽度 +@property (nonatomic, assign, readonly) int rotatedImageWidth; +/// 旋转后图片高度 +@property (nonatomic, assign, readonly) int rotatedImageHeight; +/// 全文识别的所有内容（同行内容整合为一行输出) +@property (nonatomic, copy, readonly) NSString * wholeText; +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/OnlineTTS.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/OnlineTTS.h new file mode 100644 index 00000000..4d51752a --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/OnlineTTS.h @@ -0,0 +1,28 @@ +// +// OnlineTTS.h +// SparkChain_Mac +// +// Created by XX on 2024/5/10. +// + +#import +#import +#import "TTS.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface OnlineTTS : TTS + +@property (nonatomic, copy) NSString *aue; +@property (nonatomic, assign) int sfl; +@property (nonatomic, copy) NSString *auf; +@property (nonatomic, copy) NSString *tte; + +- (instancetype)initWithVcn:(NSString *)vcn; + +/// 指定接入区域与发音人（与底层 `OnlineTTS(REGION_TYPE, string)` 一致）。 +- (instancetype)initWithRegion:(RegionType)region vcn:(NSString *)vcn; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/PPTError.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/PPTError.h new file mode 100644 index 00000000..338b35c0 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/PPTError.h @@ -0,0 +1,22 @@ +// +// PPTError.h +// SparkChain +// +// Created by pcfang on 2025/5/14. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface PPTError : NSObject + +@property (nonatomic, assign) int code; + +@property (nonatomic, copy) NSString *errMsg; + +@property (nonatomic, copy) NSString *sid; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/PPTResult.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/PPTResult.h new file mode 100644 index 00000000..ddadd224 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/PPTResult.h @@ -0,0 +1,52 @@ +// +// PPTInfo.h +// SparkChain_Mac +// +// Created by pcfang on 22.5.24. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface PPTTheme : NSObject + +@property (nonatomic, copy, readonly) NSString * key; + +@property (nonatomic, copy, readonly) NSString * name; + +@property (nonatomic, copy, readonly) NSString * thumbnail; + +@end + +@interface PPTInfo : NSObject + +@property (nonatomic, assign, readonly) int code; + +@property (nonatomic, copy, readonly) NSString * coverImgSrc; + +@property (nonatomic, copy, readonly) NSString * title; + +@property (nonatomic, copy, readonly) NSString * subtitle; + +@property (nonatomic, copy, readonly) NSString * pptUrl; + +@property (nonatomic, copy, readonly) NSString * pptId; + +@property (nonatomic, copy, readonly) NSString * outline; + +@property (nonatomic, strong, readonly) NSArray * themes; + +@end + +@interface PPTResult : NSObject + +@property (nonatomic, copy, readonly) NSString * sid; + +@property (nonatomic, strong, readonly) PPTInfo * info; + +@property (nonatomic, assign, readonly) int progress; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/PersonateTTS.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/PersonateTTS.h new file mode 100644 index 00000000..fc6dcba9 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/PersonateTTS.h @@ -0,0 +1,25 @@ +// +// PersonateTTS.h +// SparkChain +// +// Created by pcfang on 23.2.24. +// + +#import +#import "TTS.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface PersonateTTS : TTS +@property(nonatomic, assign) int topK; + +@property(nonatomic, assign) int maxTokens; + +@property(nonatomic, copy) NSString * oralLevel; + +@property(nonatomic, assign) bool sparkAssist; + +- (instancetype)initWithVcn:(NSString *)vcn; +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RAASR.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RAASR.h new file mode 100644 index 00000000..8aca3149 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RAASR.h @@ -0,0 +1,176 @@ +// +// RAASR.h +// SparkChain_iOS +// +// Created by pcfang on 2024/8/8. +// + +#import +#import +@class RAASRResult; +@class RAASRError; + +NS_ASSUME_NONNULL_BEGIN + +@protocol RAASRCallback + +- (void)onResult:(RAASRResult *)result userInfo:(id _Nullable)userInfo; + +- (void)onError:(RAASRError *)error userInfo:(id _Nullable)userInfo; + +- (void)onProcess:(double)dltotal dlnow:(double)dlnow utotal:(double)utotal ulnow:(double)ulnow userInfo:(id _Nullable)userInfo; + +@end + +@interface RAASR : NSObject + +//common +@property (nonatomic, copy) NSString * pd; + +@property (nonatomic, copy) NSString * audioURL; + +@property (nonatomic, assign) int languageType; + +@property (nonatomic, assign) int engSegMax; + +@property (nonatomic, assign) int engSegMin; + +@property (nonatomic, assign) float engSegWeight; + +@property (nonatomic, assign) bool engSmoothproc; + +@property (nonatomic, assign) bool engColloqproc; + +@property (nonatomic, assign) int engVadMdn; + +@property (nonatomic, assign) int engVadMargin; + +@property (nonatomic, assign) int engRlang; + +//国内 +@property (nonatomic, copy) NSString * language; + +@property (nonatomic, copy) NSString * resultType; + +@property (nonatomic, copy) NSString * hotWord; + +@property (nonatomic, assign) int candidate; + +@property (nonatomic, assign) int roleType; + +@property (nonatomic, assign) int roleNum; + +@property (nonatomic, copy) NSString * audioMode; + +@property (nonatomic, assign) int standardWav; + +@property (nonatomic, assign) int trackMode; + +@property (nonatomic, copy) NSString * transLanguage; + +@property (nonatomic, assign) int transMode; + +//海外 +// common params +@property (nonatomic, copy) NSString * ostUid; +@property (nonatomic, copy) NSString * ostDeviceId; +//@property (nonatomic, copy) NSString * ostRequestId; +@property (nonatomic, copy) NSString * ostDeviceImei; +@property (nonatomic, copy) NSString * ostDeviceImsi; +@property (nonatomic, copy) NSString * ostDeviceMac; +@property (nonatomic, copy) NSString * ostDeviceOther; +@property (nonatomic, copy) NSString * ostNetType; +@property (nonatomic, copy) NSString * ostNetIsp; +@property (nonatomic, copy) NSString * ostAppVer; + +// business params +@property (nonatomic, copy) NSString * ostTaskType; +@property (nonatomic, copy) NSString * ostFormat; +@property (nonatomic, copy) NSString * ostEncoding; +@property (nonatomic, assign) int ostAudioSize; +@property (nonatomic, copy) NSString * ostAudioSrc; +@property (nonatomic, copy) NSString * ostCallbackKey; +@property (nonatomic, copy) NSString * ostCallbackSecret; +@property (nonatomic, copy) NSArray * ostFeatureList; +@property (nonatomic, assign) int ostProcinfoOn; +@property (nonatomic, assign) int ostVsppOn; +@property (nonatomic, assign) int ostSpeakerNum; +@property (nonatomic, assign) int ostOutputType; +@property (nonatomic, assign) int ostPostprocOn; +@property (nonatomic, assign) int ostResegment; +@property (nonatomic, copy) NSString * ostResId; +@property (nonatomic, assign) int ostDuration; +@property (nonatomic, copy) NSString * ostResUrl; +@property (nonatomic, assign) int ostEnableSubtitle; +@property (nonatomic, assign) int ostVto; +@property (nonatomic, copy) NSString * ostDhw; +@property (nonatomic, assign) int ostNbest; +@property (nonatomic, assign) int ostCombineMax; +@property (nonatomic, assign) int ostNunum; + + +- (void)addIntParam:(NSString *)key value:(int)value; +- (void)addBoolParam:(NSString *)key value:(BOOL)value; +- (void)addStringParam:(NSString *)key value:(NSString *)value; + +@property (nonatomic, weak) id delegate; + +- (instancetype)init; + +- (instancetype)initWithRegion:(RegionType)regoin; + +- (instancetype)initWithSecretKey:(NSString *)apiKey; + +- (instancetype)initWithSecretKey:(NSString *)apiKey regoin:(RegionType)regoin; + +- (void)setOstPersonalizationKey:(NSString *)key value:(NSString *)value; + +/// 上传音频文件 +/// - Parameters: +/// - fileName: 音频文件路径(仅国内支持)，或者音频链接，国内需设置audioMode为urlLink +/// - requestId: 仅海外支持，表示请求id，必传；国内可传空值 +/// - userInfo: 用户自定义标识 +- (int)uploadAsync:(NSString *)fileName requestId:(NSString *)requestId userInfo:(id _Nullable)userInfo; + +/// 上传音频文件 +/// - Parameter fileName: 音频文件路径(仅国内支持)，或者音频链接，国内需设置audioMode为urlLink +- (int) uploadAsync:(NSString *)fileName; + +/// 查询订单状态 +/// - Parameters: +/// - orderId: 订单id +/// - userInfo: 用户自定义标识 +- (int)queryResult:(NSString *)orderId userInfo:(id _Nullable)userInfo; + +/// 查询订单状态 +/// - Parameter orderId: 订单id +- (int)queryResult:(NSString *)orderId; + +//仅国内 +- (int)stop; + +#pragma mark - overseas +//海外 + +/// 取消任务，仅海外支持 +/// - Parameters: +/// - orderId: 订单id +/// - userInfo: 用户自定义标识 +- (int)ostRetryTaskWithOrderId:( NSString * _Nonnull )orderId userInfo:(id _Nullable)userInfo; + +/// 删除任务，仅海外支持 +/// - Parameters: +/// - orderId: 订单id +/// - requestId: 客户端用于标记任务的唯一id，最大长度64字符，由客户端保证唯一性，服务回调结果时会会包含此参数 +/// - userInfo: 用户自定义标识 +- (int)ostCancelTaskWithOrderId:( NSString * _Nonnull )orderId requestId:( NSString * _Nonnull )requestId userInfo:(id _Nullable)userInfo; + +/// 删除任务，仅海外支持 +/// - Parameters: +/// - orderId: 订单id +/// - requestId: 客户端用于标记任务的唯一id，最大长度64字符，由客户端保证唯一性，服务回调结果时会会包含此参数 +/// - userInfo: 用户自定义标识 +- (int)ostDeleteTaskWithOrderId:( NSString * _Nonnull )orderId requestId:( NSString * _Nonnull )requestId userInfo:(id _Nullable)userInfo; +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RAASRError.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RAASRError.h new file mode 100644 index 00000000..5cc81cc6 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RAASRError.h @@ -0,0 +1,35 @@ +// +// RAASRError.h +// SparkChain_iOS +// +// Created by pcfang on 2024/8/9. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface RAASRError : NSObject +//common +@property (nonatomic, assign) int code; + +@property (nonatomic, copy) NSString *errMsg; + +//国内 +@property (nonatomic, copy) NSString *orderId; + +@property (nonatomic, assign) int failType; + +//海外 +//海外 +@property (nonatomic, copy, readonly) NSString * ostTaskStatus; + +@property (nonatomic, copy, readonly) NSString * ostSid; + +@property (nonatomic, copy, readonly) NSString * ostTaskId; + +@property (nonatomic, copy, readonly) NSString * ostTaskType; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RAASRResult.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RAASRResult.h new file mode 100644 index 00000000..0cf54822 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RAASRResult.h @@ -0,0 +1,59 @@ +// +// RAASRResult.h +// SparkChain_iOS +// +// Created by pcfang on 2024/8/9. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + + +@interface RAASRTransResult : NSObject + +@property (nonatomic, copy, readonly) NSString * segId; + +@property (nonatomic, copy, readonly) NSString * dst; + +@property (nonatomic, assign, readonly) int bg; + +@property (nonatomic, assign, readonly) int ed; + +@property (nonatomic, copy, readonly) NSArray * tags; + +@property (nonatomic, copy, readonly) NSArray * roles; + +@end + +@interface RAASRResult : NSObject + +//common +@property (nonatomic, assign, readonly) int code; +@property (nonatomic, copy, readonly) NSString * orderResult; + +//国内 +@property (nonatomic, assign, readonly) int status; +@property (nonatomic, copy, readonly) NSString * orderId; + +@property (nonatomic, assign, readonly) long originalDuration; + +@property (nonatomic, assign, readonly) long realDuration; + +@property (nonatomic, assign, readonly) long expireTime; + +@property (nonatomic, assign, readonly) int taskEstimateTime; + +@property (nonatomic, copy, readonly) NSArray * transResult; + +//海外 +@property (nonatomic, copy, readonly) NSString * ostTaskStatus; + +@property (nonatomic, copy, readonly) NSString * ostSid; + +@property (nonatomic, copy, readonly) NSString * ostTaskId; + +@property (nonatomic, copy, readonly) NSString * ostTaskType; +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RTASR.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RTASR.h new file mode 100644 index 00000000..00d71944 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RTASR.h @@ -0,0 +1,149 @@ +// +// RTASR.h +// SparkChain_iOS +// 实时语音转写 +// Created by pcfang on 2024/8/8. +// + +#import +#import +@class RTASRResult; +@class RTASRError; +NS_ASSUME_NONNULL_BEGIN + +@protocol RTASRCallback + +- (void)onResult:(RTASRResult *) result useInfo:(id _Nullable)userInfo; + +- (void)onError:(RTASRError *) error useInfo:(id _Nullable)userInfo; +@end + +@interface RTASR : NSObject + +//common +@property (nonatomic, copy) NSString * lang; + +@property (nonatomic, copy) NSString * punc; + +@property (nonatomic, copy) NSString * pd; + +@property (nonatomic, assign) int vadMdn; + +//国内 +@property (nonatomic, copy) NSString * transType; + +@property (nonatomic, assign) int transStrategy; + +@property (nonatomic, copy) NSString * targetLanguage; + +@property (nonatomic, assign) int roleType; + +@property (nonatomic, assign) int engLangType; + +#pragma mark - region overseas params +@property (nonatomic, copy) NSString * ostContextId; + +@property (nonatomic, copy) NSString * ostUid; + +@property (nonatomic, copy) NSString * ostDeviceId; + +@property (nonatomic, copy) NSString * ostDeviceImei; + +@property (nonatomic, copy) NSString * ostDeviceMac; + +@property (nonatomic, copy) NSString * ostDeviceImsi; + +@property (nonatomic, copy) NSString * ostDeviceOther; + +@property (nonatomic, copy) NSString * ostNetType; + +@property (nonatomic, copy) NSString * ostNetIsp; + +@property (nonatomic, copy) NSString * ostAppVer; + +@property (nonatomic, copy) NSString * ostDid; + +//business params +@property (nonatomic, copy) NSString * ostFormat; + +@property (nonatomic, copy) NSString * ostEncoding; + +@property (nonatomic, copy) NSString * ostDomain; + +@property (nonatomic, copy) NSString * ostAccent; + +@property (nonatomic, copy) NSString * ostRequestId; + +@property (nonatomic, copy) NSString * ostDwa; + +@property (nonatomic, assign) int ostEos; + +@property (nonatomic, copy) NSString * ostRf; + +@property (nonatomic, copy) NSString * ostResId; + +@property (nonatomic, assign) int ostVto; + +@property (nonatomic, assign) int ostNunum; + +@property (nonatomic, assign) int ostPptaw; + +@property (nonatomic, assign) int ostDyhotws; + +@property (nonatomic, assign) int ostSegMax; + +@property (nonatomic, assign) int ostSegMin; + +@property (nonatomic, assign) float ostSegWeight; + +@property (nonatomic, assign) int ostSpeexSize; + +@property (nonatomic, assign) int ostSpkdia; + +@property (nonatomic, assign) int ostPgsnum; + +@property (nonatomic, assign) int ostLanguageType; + +@property (nonatomic, copy) NSString * ostDhw; + +@property (nonatomic, assign) int ostDhwMod; + +@property (nonatomic, copy) NSArray * ostFeatureList; + +@property (nonatomic, assign) int ostRsgid; + +@property (nonatomic, assign) int ostRlang; + +@property (nonatomic, assign) int ostPgsFlashFreq; + +@property (nonatomic, copy) NSString * ostFrameId; + +- (void)addIntParam:(NSString *)key value:(int)value; +- (void)addBoolParam:(NSString *)key value:(BOOL)value; +- (void)addStringParam:(NSString *)key value:(NSString *)value; + +@property (nonatomic, weak) id delegate; + +- (instancetype)init; + +- (instancetype)initWithRegion:(RegionType)region; + +- (instancetype)initWithSecretKey:(NSString *)secretKey; + +- (instancetype)initWithSecretKey:(NSString *)secretKey region:(RegionType)region; + +- (void)setOstPersonalization:(NSString *)key value:(NSString *)value; + +- (int)start; + +- (int)startWithUserInfo:(id _Nullable)userInfo; + +- (int)write:(NSData *)audioData; + +- (int)stop; + +- (int)stop:(BOOL)immediate; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RTASRError.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RTASRError.h new file mode 100644 index 00000000..74df4e13 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RTASRError.h @@ -0,0 +1,25 @@ +// +// RTASRError.h +// SparkChain_iOS +// +// Created by pcfang on 2024/8/9. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface RTASRError : NSObject + +@property (nonatomic, assign) int code; + +@property (nonatomic, copy) NSString *errMsg; + +@property (nonatomic, copy) NSString *sid; + +//海外 +@property (nonatomic, copy) NSString *ostContextId; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RTASRResult.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RTASRResult.h new file mode 100644 index 00000000..10a902ee --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/RTASRResult.h @@ -0,0 +1,38 @@ +// +// RTASRResult.h +// SparkChain_iOS +// +// Created by pcfang on 2024/8/9. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface RTASRTransResult : NSObject + +@property(nonatomic, assign, readonly) int status; + +@property(nonatomic, copy, readonly) NSString * src; + +@property(nonatomic, copy, readonly) NSString * dst; + +@end + +@interface RTASRResult : NSObject + +@property(nonatomic, assign, readonly) int status; + +@property(nonatomic, copy, readonly) NSString * sid; + +@property(nonatomic, copy, readonly) NSString * data; + +@property(nonatomic, copy, readonly) NSString * raw; + +@property(nonatomic, copy, readonly) RTASRTransResult * transResult; + +@property(nonatomic, copy, readonly) NSString * ostContextId; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SCEnums.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SCEnums.h new file mode 100644 index 00000000..8b407903 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SCEnums.h @@ -0,0 +1,20 @@ +// +// SCEnums.h +// SparkChain_iOS +// +// Created by pcfang on 2025/5/8. +// + +#ifndef SCEnums_h +#define SCEnums_h + +typedef enum : NSUInteger { + RegionDefault, + RegionSG, + RegionEU, + RegionCNLLM, + RegionMSFT +} RegionType; + + +#endif /* SCEnums_h */ diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Segment.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Segment.h new file mode 100644 index 00000000..cbb4e934 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Segment.h @@ -0,0 +1,20 @@ +// +// Segment.h +// SparkChain_Mac +// +// Created by XX on 2024/5/11. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface Segment : NSObject + +@property (nonatomic) int score; +@property (nonatomic, copy) NSString *text; +@property (nonatomic, copy) NSString *lg; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SparkChain.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SparkChain.h new file mode 100644 index 00000000..ac80b1ea --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SparkChain.h @@ -0,0 +1,67 @@ +// +// SparkChain.h +// SparkChain +// +// Created by pcfang on 12.9.23. +// + +#import + +//! Project version number for SparkChain. +FOUNDATION_EXPORT double SparkChainVersionNumber; + +//! Project version string for SparkChain. +FOUNDATION_EXPORT const unsigned char SparkChainVersionString[]; + +#import +#import + +#import +#import +#import +#import +#import +#import +#import + +#import +#import +//合成 +#import +#import +#import + +// 实时转写 +#import +#import +#import + +// 录音文件转写 +#import +#import +#import + +// Feature / 虚拟人 VC +#import +#import +#import +#import +#import +#import +#import + +// 翻译 +#import +#import +#import + +//识别 +#import +#import +#import +#import + + + + + diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SparkChainConfig.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SparkChainConfig.h new file mode 100644 index 00000000..69af151a --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SparkChainConfig.h @@ -0,0 +1,33 @@ +// +// SparkChainConfig.h +// SparkChain +// +// Created by pcfang on 12.9.23. +// + +#import + +NS_ASSUME_NONNULL_BEGIN +#define chainProperty(returnType,name) property(nonatomic, readonly, copy) returnType (^name) + +@interface SparkChainConfig : NSObject + +@chainProperty(SparkChainConfig *,appID)(NSString * appID); + +@chainProperty(SparkChainConfig *,apiKey)(NSString * apiKey); + +@chainProperty(SparkChainConfig *,apiSecret)(NSString * apiSecret); + +@chainProperty(SparkChainConfig *,workDir)(NSString * workDir); + +@chainProperty(SparkChainConfig *,uid)(NSString * uid); + +@chainProperty(SparkChainConfig *,logLevel)(int logLevel); + +@chainProperty(SparkChainConfig *,logPath)(NSString * logPath); + +@chainProperty(SparkChainConfig *,apiType)(int apiType); + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SparkChainEntrance.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SparkChainEntrance.h new file mode 100644 index 00000000..671c2e02 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/SparkChainEntrance.h @@ -0,0 +1,24 @@ +// +// SparkChain.h +// SparkChain +// +// Created by pcfang on 12.9.23. +// + +#import +@class SparkChainConfig; + +NS_ASSUME_NONNULL_BEGIN + +@interface SparkChain : NSObject + ++ (SparkChain *)getInst; + +- (int)init:(SparkChainConfig *)config; + +- (int)unInit; + +- (int)setConfig:(NSString *)key value:(void *)value; +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TTS.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TTS.h new file mode 100644 index 00000000..f671db7b --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TTS.h @@ -0,0 +1,90 @@ +// +// TTS.h +// SparkChain +// +// Created by pcfang on 22.2.24. +// + +#import +@class TTS; +@class TTSResult; +@class TTSError; + +NS_ASSUME_NONNULL_BEGIN + +typedef enum : NSUInteger { + Begin, + Continue, + End, +} Status; + +@protocol TTSCallbacks + +- (void)tts:(TTS *)tts onResult:(TTSResult *)result usrContext:(id)usrContext; + +- (void)tts:(TTS *)tts onError:(TTSError *)error usrContext:(id)usrContext; +@end + +@interface TTS : NSObject + +@property(nonatomic, weak) id cbs; + +@property(nonatomic, copy) NSString * vcn; + +@property(nonatomic, assign) int speed; + +@property(nonatomic, assign) int pitch; + +@property(nonatomic, assign) int volume; + +@property(nonatomic, assign) int reg; + +@property(nonatomic, assign) int rdn; + +// 以下海外不支持 +@property(nonatomic, assign) int sampleRate; + +@property(nonatomic, copy) NSString * encoding; + +@property(nonatomic, assign) int bitDepth; + +@property(nonatomic, assign) int channels; + +@property(nonatomic, assign) int frameSize; + +@property(nonatomic, assign) int bgs; + +@property(nonatomic, assign) BOOL rhy; + +@property(nonatomic, assign) int scn; + +@property(nonatomic, assign) BOOL version; + +@property(nonatomic, assign) int l5SilLen; + +@property(nonatomic, assign) int paragraphSilLen; + +- (instancetype)init; + +- (instancetype)initWithVcn:(NSString *)vcn; + +- (int)arun:(NSString *)text; + +- (int)arun:(NSString *)text userInfo:(id)userInfo; + +- (int)arun:(NSString *)text status:(Status)status; + +- (int)arun:(NSString *)text status:(Status)status userInfo:(id)userInfo; + +- (void)stop; + + +- (void)addIntParam:(NSString *)key value:(int)value; + +- (void)addBoolParam:(NSString *)key value:(BOOL)value; + +- (void)addStringParam:(NSString *)key value:(NSString *)value; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TTSError.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TTSError.h new file mode 100644 index 00000000..9351ea50 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TTSError.h @@ -0,0 +1,22 @@ +// +// TTSError.h +// SparkChain +// +// Created by pcfang on 22.2.24. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface TTSError : NSObject + +@property(nonatomic, copy, readonly) NSString * sid; + +@property(nonatomic, assign, readonly) int code; + +@property(nonatomic, copy, readonly) NSString * errMsg; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TTSResult.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TTSResult.h new file mode 100644 index 00000000..604df4d1 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TTSResult.h @@ -0,0 +1,32 @@ +// +// TTSResult.h +// SparkChain +// +// Created by pcfang on 22.2.24. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface TTSResult : NSObject + +@property(nonatomic, copy, readonly) NSString * sid; + +@property(nonatomic, assign, readonly) int seq; + +@property(nonatomic, copy, readonly) NSData * data; + +@property(nonatomic, copy, readonly) NSString * ced; + +@property(nonatomic, copy, readonly) NSString * pybuf; + +@property(nonatomic, copy, readonly) NSString * spell; + +@property(nonatomic, copy, readonly) NSString * version; + +@property(nonatomic, assign, readonly) int status; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TableOcr.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TableOcr.h new file mode 100644 index 00000000..c2047261 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TableOcr.h @@ -0,0 +1,36 @@ +// +// OCTableOcr.h +// TABLE_OCR +// +// Created by xyzhang on 2024/6/4. +// + +#import + +@class TableOcr; +@class TableOcrResult; +@class TableOcrError; + +NS_ASSUME_NONNULL_BEGIN + +@protocol TableOcrDelegate + +- (void)tableOcr:(TableOcr *)tableOcr onResult:(TableOcrResult *)result usrContext:(id)usrContext; + +- (void)tableOcr:(TableOcr *)tableOcr onError:(TableOcrError *)error usrContext:(id)usrContext; + +@end + +@interface TableOcr : NSObject + +@property (nonatomic, weak) id delegate; +@property (nonatomic, copy) NSString *resultEncoding; +@property (nonatomic, copy) NSString *resultCompress; +@property (nonatomic, copy) NSString *resultFormat; + +- (instancetype)init; +- (int)arun:(NSData *)imageData encoding:(NSString *)encoding userInfo:(id)userInfo; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TableOcrError.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TableOcrError.h new file mode 100644 index 00000000..35b58e38 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TableOcrError.h @@ -0,0 +1,20 @@ +// +// OCTableOcrError.h +// TABLE_OCR +// +// Created by xyzhang on 2024/6/4. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface TableOcrError : NSObject + +@property (nonatomic, copy, readonly) NSString *sid; +@property (nonatomic, copy, readonly) NSString *errMsg; +@property (nonatomic, assign, readonly) int code; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TableOcrExtension.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TableOcrExtension.h new file mode 100644 index 00000000..ae163774 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TableOcrExtension.h @@ -0,0 +1,24 @@ +// +// OCTableOcrExtension.h +// TABLE_OCR +// +// Created by xyzhang on 2024/6/4. +// + +#import "TableOcr.h" +#import "TableOcrResult.h" +#import "TableOcrError.h" +#include "sc_tableOCR.h" + +@interface TableOcr () +- (instancetype)initWithTableOcr:(SparkChain::TableOCR *)tableOcr; +- (SparkChain::TableOCR *)tableOcrRef; +@end + +@interface TableOcrResult () +- (instancetype)initWithResult:(SparkChain::TableOCRResult *)result; +@end + +@interface TableOcrError () +- (instancetype)initWithError:(SparkChain::TableOCRError *)error; +@end diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TableOcrResult.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TableOcrResult.h new file mode 100644 index 00000000..963a9db9 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/TableOcrResult.h @@ -0,0 +1,20 @@ +// +// OCTableOcrResult.h +// TABLE_OCR +// +// Created by xyzhang on 2024/6/4. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface TableOcrResult : NSObject + +@property (nonatomic, copy, readonly) NSString *sid; +@property (nonatomic, copy, readonly) NSString *results; +@property (nonatomic, copy, readonly) NSArray *tables; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Transcription.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Transcription.h new file mode 100644 index 00000000..8224a031 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Transcription.h @@ -0,0 +1,25 @@ +// +// Transcription.h +// SparkChain_Mac +// +// Created by XX on 2024/5/11. +// + +#ifndef Transcription_h +#define Transcription_h + +#import +#import "Segment.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface Transcription : NSObject + +@property (nonatomic) int index; +@property (nonatomic, strong) NSArray *segments; + +@end + +NS_ASSUME_NONNULL_END + +#endif /* Transcription_h */ diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/VC.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/VC.h new file mode 100644 index 00000000..cf444e3f --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/VC.h @@ -0,0 +1,46 @@ +// +// VC.h +// SparkChain_iOS +// + +#import + +@class VCResult; +@class VCError; + +NS_ASSUME_NONNULL_BEGIN + +@protocol VCCallback + +- (void)onResult:(VCResult *)result userInfo:(id _Nullable)userInfo; +- (void)onError:(VCError *)error userInfo:(id _Nullable)userInfo; +- (void)onProcess:(double)dltotal dlnow:(double)dlnow utotal:(double)utotal ulnow:(double)ulnow userInfo:(id _Nullable)userInfo; + +@end + +@interface VC : NSObject + +@property (nonatomic, weak) id delegate; + +- (instancetype)init; + +/// mode: 0=voice clone, 1=voice clone tts +- (void)setParamsWithMode:(int)mode key:(NSString *)key stringValue:(NSString *)value; +- (void)setParamsWithMode:(int)mode key:(NSString *)key intValue:(int)value; +- (void)setParamsWithMode:(int)mode key:(NSString *)key boolValue:(BOOL)value; + +- (void)mosRatio:(float)ratio; + +- (int)vcGetTokenWithUserInfo:(id _Nullable)userInfo; +- (int)vcGetTraninTextWithToken:(NSString *)token textId:(long)textId userInfo:(id _Nullable)userInfo; +- (int)vcTaskAddWithToken:(NSString *)token userInfo:(id _Nullable)userInfo; +- (int)vcSubmitWithAudioToken:(NSString *)token audioPath:(NSString *)audioPath taskId:(NSString *)taskId textSegId:(int)textSegId textId:(long)textId userInfo:(id _Nullable)userInfo; +- (int)vcAddAudioUrlWithToken:(NSString *)token audioUrl:(NSString *)audioUrl taskId:(NSString *)taskId textSegId:(int)textSegId textId:(long)textId userInfo:(id _Nullable)userInfo; +- (int)vcSubmitTaskWithToken:(NSString *)token taskId:(NSString *)taskId userInfo:(id _Nullable)userInfo; +- (int)vcGetProcessWithToken:(NSString *)token taskId:(NSString *)taskId userInfo:(id _Nullable)userInfo; + +- (int)vcTTSWithResourceId:(NSString *)resourceId text:(NSString *)text audioEncoding:(NSString *)audioEncoding status:(int)status seq:(int)seq userInfo:(id _Nullable)userInfo; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/VCError.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/VCError.h new file mode 100644 index 00000000..6281435d --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/VCError.h @@ -0,0 +1,19 @@ +// +// VCError.h +// SparkChain_iOS +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface VCError : NSObject + +@property (nonatomic, assign, readonly) int code; +@property (nonatomic, copy, readonly) NSString *key; +@property (nonatomic, copy, readonly) NSString *message; +@property (nonatomic, copy, readonly) NSString *sid; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/VCResult.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/VCResult.h new file mode 100644 index 00000000..9a5e9c7f --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/VCResult.h @@ -0,0 +1,25 @@ +// +// VCResult.h +// SparkChain_iOS +// + +#import + +@class VCTextSeg; + +NS_ASSUME_NONNULL_BEGIN + +@interface VCResult : NSObject + +@property (nonatomic, copy, readonly) NSString *key; +@property (nonatomic, copy, readonly) NSString *rawResult; +@property (nonatomic, copy, readonly) NSString *token; +@property (nonatomic, copy, readonly) NSString *taskId; +@property (nonatomic, copy, readonly) NSArray *textSegs; +@property (nonatomic, assign, readonly) int status; +@property (nonatomic, copy, readonly) NSString *assetId; +@property (nonatomic, copy, readonly) NSData *audio; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/VCTextSeg.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/VCTextSeg.h new file mode 100644 index 00000000..cab1cdb3 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/VCTextSeg.h @@ -0,0 +1,19 @@ +// +// VCTextSeg.h +// SparkChain_iOS +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface VCTextSeg : NSObject + +@property (nonatomic, assign, readonly) int segId; +@property (nonatomic, copy, readonly) NSString *segText; +@property (nonatomic, copy, readonly) NSString *segTextLan; +@property (nonatomic, assign, readonly) int segSize; + +@end + +NS_ASSUME_NONNULL_END diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Vad.h b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Vad.h new file mode 100644 index 00000000..8559e7f6 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Headers/Vad.h @@ -0,0 +1,25 @@ +// +// Vad.h +// SparkChain_Mac +// +// Created by XX on 2024/5/11. +// + +#ifndef Vad_h +#define Vad_h + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface Vad : NSObject + +@property (nonatomic) int begin; +@property (nonatomic) int end; + + +@end + +NS_ASSUME_NONNULL_END + +#endif /* Vad_h */ diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Info.plist b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Info.plist new file mode 100644 index 00000000..9cbdb1a5 Binary files /dev/null and b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Info.plist differ diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Modules/module.modulemap b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Modules/module.modulemap new file mode 100644 index 00000000..c84503e0 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/Modules/module.modulemap @@ -0,0 +1,6 @@ +framework module SparkChain { + umbrella header "SparkChain.h" + export * + + module * { export * } +} diff --git a/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/SparkChain b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/SparkChain new file mode 100755 index 00000000..47141d8e Binary files /dev/null and b/apps/client/local_plugins/sparkchain/ios/sparkchain/SparkChain.xcframework/ios-arm64/SparkChain.framework/SparkChain differ diff --git a/apps/client/local_plugins/sparkchain/lib/sparkchain.dart b/apps/client/local_plugins/sparkchain/lib/sparkchain.dart new file mode 100644 index 00000000..02923c95 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/lib/sparkchain.dart @@ -0,0 +1,97 @@ +import 'dart:async'; + +import 'package:flutter/services.dart'; + +/// 讯飞 SparkChain。事件从 [events] 来，方法调用只返回启动码。 +class SparkChain { + static const _method = MethodChannel('sparkchain'); + static const _events = EventChannel('sparkchain/events'); + + static Stream>? _stream; + + static Stream> get events { + _stream ??= _events.receiveBroadcastStream().map((event) { + final raw = Map.from(event as Map); + return raw.map((key, value) => MapEntry(key.toString(), value)); + }); + return _stream!; + } + + /// 0 成功。不打印密钥。 + static Future init({ + required String appId, + required String apiKey, + required String apiSecret, + int apiType = 1, + }) async { + final code = await _method.invokeMethod('init', { + 'appId': appId, + 'apiKey': apiKey, + 'apiSecret': apiSecret, + 'apiType': apiType, + }); + return code ?? -1; + } + + static Future translate({ + required String id, + required String from, + required String to, + required String text, + }) async { + final code = await _method.invokeMethod('translate', { + 'id': id, + 'from': from, + 'to': to, + 'text': text, + }); + return code ?? -1; + } + + static Future speak({ + required String id, + required String lang, + required String text, + String? voice, + }) async { + final code = await _method.invokeMethod('speak', { + 'id': id, + 'lang': lang, + 'text': text, + if (voice != null) 'voice': voice, + }); + return code ?? -1; + } + + /// [lang] 用短码 zh / en / ga。ga 走实时转写，其余走听写。 + static Future startListen({ + required String id, + required String lang, + }) async { + final code = await _method.invokeMethod('startListen', { + 'id': id, + 'lang': lang, + }); + return code ?? -1; + } + + static Future write(Uint8List pcm) async { + final code = await _method.invokeMethod('write', {'pcm': pcm}); + return code ?? -1; + } + + static Future stopListen() async { + await _method.invokeMethod('stopListen'); + } +} + +/// 把界面上的 BCP-47 收成插件认的短码。不认识的原样返回，让原生按失败处理。 +String sparkChainShortCode(String code) { + final raw = code.trim().toLowerCase().replaceAll('_', '-'); + if (raw.isEmpty) return raw; + if (raw == 'ga' || raw.startsWith('ga-')) return 'ga'; + if (raw == 'zh' || raw.startsWith('zh-')) return 'zh'; + if (raw == 'en' || raw.startsWith('en-')) return 'en'; + final dash = raw.indexOf('-'); + return dash < 0 ? raw : raw.substring(0, dash); +} diff --git a/apps/client/local_plugins/sparkchain/pubspec.yaml b/apps/client/local_plugins/sparkchain/pubspec.yaml new file mode 100644 index 00000000..22f0f044 --- /dev/null +++ b/apps/client/local_plugins/sparkchain/pubspec.yaml @@ -0,0 +1,23 @@ +name: sparkchain +description: 科大讯飞 SparkChain 海外 SDK。只收已有 PCM，不自己开麦克风。 +version: 0.0.1 +publish_to: 'none' + +environment: + sdk: '>=3.3.0 <4.0.0' + flutter: ">=3.3.0" + +dependencies: + flutter: + sdk: flutter + +flutter: + plugin: + platforms: + android: + package: com.yunqiinnovation.sparkchain + pluginClass: SparkChainPlugin + ios: + pluginClass: SparkChainPlugin + swiftPackage: + path: ios/sparkchain diff --git a/apps/client/pubspec.lock b/apps/client/pubspec.lock index 5f53fc61..4049bcb7 100644 --- a/apps/client/pubspec.lock +++ b/apps/client/pubspec.lock @@ -1737,6 +1737,13 @@ packages: url: "https://pub.dev" source: hosted version: "1.10.2" + sparkchain: + dependency: "direct main" + description: + path: "local_plugins/sparkchain" + relative: true + source: path + version: "0.0.1" speech: dependency: "direct main" description: diff --git a/apps/client/pubspec.yaml b/apps/client/pubspec.yaml index 42e9a11d..45c421be 100644 --- a/apps/client/pubspec.yaml +++ b/apps/client/pubspec.yaml @@ -1,7 +1,7 @@ name: eaimar description: "Eaimar - AI Voice Assistant." publish_to: "none" -version: 1.0.3+37 +version: 1.0.3+48 environment: sdk: ">=3.3.0 <4.0.0" @@ -103,6 +103,8 @@ dependencies: path: local_plugins/speech azure_speech: path: local_plugins/azure_speech + sparkchain: + path: local_plugins/sparkchain chat_storage: path: local_plugins/chat_storage # ⚠️ 下面两个 Dart 侧已无引用，但**原生侧仍被别的插件依赖**，不能删： diff --git a/apps/services/comm/ai_vendor.go b/apps/services/comm/ai_vendor.go new file mode 100644 index 00000000..2bf1632f --- /dev/null +++ b/apps/services/comm/ai_vendor.go @@ -0,0 +1,80 @@ +package comm + +import ( + "os" + "strconv" + "strings" +) + +// 测试服锁：AI 能力只许走科大讯飞 SparkChain 这一套，不许再去调微软、阿里、字节、谷歌、灵犀。 +// +// 开关只认环境变量 AI_ONLY_IFLYTEK=1。正式服不设这个变量，解析和调用都跟原来一样。 +// 存储、短信、实名、支付、MCP、应用参数不在这把锁里。 +// +// 允许的服务 id 只有 SparkChain 这一族。同是讯飞的 AIUI、老的 ntrans 听写不在里面。 + +const ( + svcCatSTT int32 = 1 + svcCatTTS int32 = 2 + svcCatLLM int32 = 5 + svcCatSTS int32 = 7 + svcCatASRFile int32 = 8 + svcCatLLMVision int32 = 9 +) + +var aiVendorCats = map[int32]bool{ + svcCatSTT: true, + svcCatTTS: true, + SvcCatMT: true, + SvcCatAST: true, + svcCatLLM: true, + svcCatSTS: true, + svcCatASRFile: true, + svcCatLLMVision: true, +} + +var iflytekAiSvcIDs = map[string]bool{ + "sparkchain": true, + "mt_iflytek": true, + "asrfile_iflytek": true, + "llm_iflytek": true, +} + +// AiOnlyIflytek 当前进程是不是「AI 只许讯飞」的测试服。 +func AiOnlyIflytek() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv("AI_ONLY_IFLYTEK"))) { + case "1", "true", "yes", "on": + return true + default: + return false + } +} + +// IflytekAiSvc 是不是这次允许的科大讯飞服务。 +func IflytekAiSvc(id, provider string) bool { + if strings.ToLower(strings.TrimSpace(provider)) != "iflytek" { + return false + } + return iflytekAiSvcIDs[strings.TrimSpace(id)] +} + +// AiVendorBlocked 锁开着、而且这是一家别的 AI 服务时返回 true。调用方应整条跳过，不要发请求。 +func AiVendorBlocked(id, provider, categories string) bool { + if !AiOnlyIflytek() || !categoriesHaveAI(categories) { + return false + } + return !IflytekAiSvc(id, provider) +} + +func categoriesHaveAI(categories string) bool { + for _, p := range strings.Split(categories, ",") { + v, err := strconv.Atoi(strings.TrimSpace(p)) + if err != nil { + continue + } + if aiVendorCats[int32(v)] { + return true + } + } + return false +} diff --git a/apps/services/comm/ai_vendor_test.go b/apps/services/comm/ai_vendor_test.go new file mode 100644 index 00000000..df98173c --- /dev/null +++ b/apps/services/comm/ai_vendor_test.go @@ -0,0 +1,32 @@ +package comm + +import "testing" + +func TestAiVendorBlockedOnlyWhenLocked(t *testing.T) { + t.Setenv("AI_ONLY_IFLYTEK", "") + if AiVendorBlocked("stt_azure", "azure", "1") { + t.Fatal("锁关着时不应拦截微软听写") + } + t.Setenv("AI_ONLY_IFLYTEK", "1") + if !AiOnlyIflytek() { + t.Fatal("AI_ONLY_IFLYTEK=1 应开启") + } + if !AiVendorBlocked("mt_alibaba", "alibaba", "3") { + t.Fatal("锁开着应拦截阿里翻译") + } + if !AiVendorBlocked("llm_mobile_elf", "chinamobile", "5") { + t.Fatal("锁开着应拦截灵犀") + } + if !AiVendorBlocked("sts_iflytek_aiui", "iflytek", "7") { + t.Fatal("AIUI 不是这次的 SparkChain，应拦截") + } + if AiVendorBlocked("sparkchain", "iflytek", "12") { + t.Fatal("应用参数里的 SparkChain 不是 AI 类别，应继续下发") + } + if AiVendorBlocked("mt_iflytek", "iflytek", "3") { + t.Fatal("讯飞机器翻译应放行") + } + if AiVendorBlocked("oss_aliyun", "alibaba", "6") { + t.Fatal("对象存储不在这把锁里") + } +} diff --git a/apps/services/modules/allhelp/summary.go b/apps/services/modules/allhelp/summary.go index d333d4ca..21cf3800 100644 --- a/apps/services/modules/allhelp/summary.go +++ b/apps/services/modules/allhelp/summary.go @@ -2,15 +2,17 @@ package allhelp import ( "context" + "fmt" + "strconv" + "time" + + "yunyan/comm" "yunyan/lego/core" "yunyan/lego/core/cbase" "yunyan/lego/sys/log" redissys "yunyan/lego/sys/redis" "yunyan/pb" "yunyan/sys/doubao" - "fmt" - "strconv" - "time" ) /* @@ -112,6 +114,14 @@ func (this *summaryComp) process(rec *pb.DBChatSummary) { rec.State = pb.ChatSummaryState_ChatSummaryState_Processing _ = this.module.model.saveChatSummary(rec) + if comm.AiOnlyIflytek() { + rec.State = pb.ChatSummaryState_ChatSummaryState_Failed + rec.ErrorMsg = "测试服只允许科大讯飞" + rec.FinishTime = time.Now().Unix() + _ = this.module.model.saveChatSummary(rec) + return + } + resp, err := doubao.Chat(this.ctx, []doubao.Message{ {Role: "system", Content: this.module.options.ChatSummaryPrompt}, {Role: "user", Content: rec.ChatContent}, diff --git a/apps/services/modules/api/api_translate.go b/apps/services/modules/api/api_translate.go index ea244bba..6afd25fd 100644 --- a/apps/services/modules/api/api_translate.go +++ b/apps/services/modules/api/api_translate.go @@ -9,6 +9,13 @@ import ( // 翻译接口（后台管理用） func (this *apiComp) Translate(session comm.IUserSession, req *pb.ApiTranslateReq) (resp *pb.ApiTranslateResp, errdata *pb.ErrorData) { + if comm.AiOnlyIflytek() { + errdata = &pb.ErrorData{ + Code: pb.ErrorCode_SystemError, + Message: "测试服只允许科大讯飞", + } + return + } results, err := translate.Translate(context.Background(), req.From, req.To, req.Texts) if err != nil { errdata = &pb.ErrorData{ diff --git a/apps/services/modules/console/api_svctemplate.go b/apps/services/modules/console/api_svctemplate.go index dfdc3c18..cfcd9777 100644 --- a/apps/services/modules/console/api_svctemplate.go +++ b/apps/services/modules/console/api_svctemplate.go @@ -195,6 +195,18 @@ var asrFileDefaultLangs = map[string]string{ // builtinSvcTemplates 内置服务商模板清单（参考 polychat-agent 的服务商与字段归纳，可后续增改）。 func builtinSvcTemplates() []ThirdSvcTemplate { list := []ThirdSvcTemplate{ + // 客户端 SparkChain。类别用应用参数，不要用端到端（4）： + // filterSvcsByVip 会把类别 4 整份从下发列表拿掉，文本翻译会一起没有密钥。 + // 也不进语音识别类别，避免被会议转写选路当成一家服务商。 + tpl("sparkchain", "iflytek", "科大讯飞 SparkChain", svcCatAppParams, 0, + sf("app_id", "海外应用 appId", false), + sf("api_key", "apiKey", true), + sf("api_secret", "apiSecret", true), + sfd("api_type", "节点兜底：0 国内不用，1 新加坡，2 欧洲。IP 查不到时用", false, "1"), + sf("ast_app_id", "同传单独 appId，空则用上面的 appId", false), + sf("ast_api_key", "同传单独 apiKey，空则用上面的 apiKey", true), + sf("ast_api_secret", "同传单独 apiSecret，空则用上面的 apiSecret", true)), + // STT 语音识别 tpl("stt_azure", "azure", "微软 Azure STT", svcCatSTT, 0, sf("subscription_key", "订阅密钥", true), sf("region", "区域/终结点，如 eastus", false), sf("language", "语言代码，如 zh-CN", false)), diff --git a/apps/services/modules/echomeet/provider_registry.go b/apps/services/modules/echomeet/provider_registry.go index 5ec28b3d..2943ea60 100644 --- a/apps/services/modules/echomeet/provider_registry.go +++ b/apps/services/modules/echomeet/provider_registry.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "yunyan/comm" alitranslate "yunyan/sys/aliyun/translate" "yunyan/sys/aliyun/filetrans" @@ -22,6 +23,14 @@ import ( // 字段 key 与 console 服务商模板 seed（modules/console/api_svctemplate.go）严格对齐； // 关键凭据缺失直接报错（fail-closed），不让空凭据实例静默失败在业务深处。 +// rejectNonIflytek 测试服锁开着时，非 SparkChain 的识别/翻译/总结一律不建客户端。 +func rejectNonIflytek(svc *resolvedSvc) error { + if svc == nil || !comm.AiOnlyIflytek() || comm.IflytekAiSvc(svc.SvcId, svc.Provider) { + return nil + } + return fmt.Errorf("测试服只允许科大讯飞，已拒绝 %s/%s", svc.SvcId, svc.Provider) +} + // need 取必填字段，缺失报错。 func need(f map[string]string, provider, key string) (string, error) { v := strings.TrimSpace(f[key]) @@ -33,6 +42,9 @@ func need(f map[string]string, provider, key string) (string, error) { // buildTranscriber 构建识别（录音文件转写）实例。 func buildTranscriber(svc *resolvedSvc) (Transcriber, error) { + if err := rejectNonIflytek(svc); err != nil { + return nil, err + } f := svc.Fields switch strings.ToLower(svc.Provider) { case "bytedance": @@ -99,6 +111,9 @@ func newGoogleSpeech(f map[string]string) (gspeech.ISys, error) { // buildTranslator 构建翻译实例。 func buildTranslator(svc *resolvedSvc) (Translator, error) { + if err := rejectNonIflytek(svc); err != nil { + return nil, err + } f := svc.Fields switch strings.ToLower(svc.Provider) { case "doubao", "volcengine", "bytedance": // 火山翻译 @@ -172,6 +187,9 @@ var openAICompatBase = map[string]string{ // buildSummarizer 构建总结（大模型）实例。 // doubao 走豆包 SDK、google 走 Gemini，其余（openai/qwen/deepseek/azure 兼容端点…）统一走 OpenAI 兼容协议。 func buildSummarizer(svc *resolvedSvc) (Summarizer, error) { + if err := rejectNonIflytek(svc); err != nil { + return nil, err + } f := svc.Fields provider := strings.ToLower(svc.Provider) switch provider { diff --git a/apps/services/modules/mcp/tool_migu_music.go b/apps/services/modules/mcp/tool_migu_music.go index 8f8a877f..e8246e65 100644 --- a/apps/services/modules/mcp/tool_migu_music.go +++ b/apps/services/modules/mcp/tool_migu_music.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" + "yunyan/comm" "yunyan/lego/core" "yunyan/lego/core/cbase" "yunyan/lego/sys/log" @@ -54,6 +55,16 @@ type MiguMusicCard struct { } func (this *tool_migu_music) Handl(ctx context.Context, request mcp.CallToolRequest) (result *mcp.CallToolResult, err error) { + if comm.AiOnlyIflytek() { + result = &mcp.CallToolResult{ + Content: []mcp.Content{ + mcp.TextContent{Type: "text", Text: `{"result":"blocked","message":"测试服只允许科大讯飞"}`}, + }, + IsError: true, + } + return + } + query, err := request.RequireString("query") if err != nil { err = fmt.Errorf("query is required") diff --git a/apps/services/modules/user/model_translate.go b/apps/services/modules/user/model_translate.go index d0b8d4b2..610bb1a7 100644 --- a/apps/services/modules/user/model_translate.go +++ b/apps/services/modules/user/model_translate.go @@ -64,6 +64,10 @@ func pickMTSvc(svcs []*comm.ThirdSvcConfig, app string) (picked *comm.ThirdSvcCo if s == nil || !s.Enable || !comm.CategoriesHas(s.Categories, comm.SvcCatMT) { continue } + if comm.AiVendorBlocked(s.Id, s.Provider, s.Categories) { + skipped = append(skipped, s.Id+"/"+s.Provider) + continue + } rank, ok := mtProviderPriority[mtNormProvider(s.Provider)] if !ok { skipped = append(skipped, s.Id+"/"+s.Provider) diff --git a/apps/services/modules/user/svcresolve.go b/apps/services/modules/user/svcresolve.go index ef6701d9..12ef388f 100644 --- a/apps/services/modules/user/svcresolve.go +++ b/apps/services/modules/user/svcresolve.go @@ -66,6 +66,10 @@ func resolveThirdSvcsWithEnv(app string, region pb.Region, ids []string) ([]*pb. if comm.IsServerOnlySvc(s.Categories) { continue } + // 测试服 AI 只下发科大讯飞。别的语音/翻译/大模型凭据留在库里，但不发给客户端。 + if comm.AiVendorBlocked(s.Id, s.Provider, s.Categories) { + continue + } if exist, ok := picked[s.Id]; !ok || (exist.AppName == "" && s.AppName == app && app != "") { if _, ok := picked[s.Id]; !ok { svcIds = append(svcIds, s.Id) diff --git a/docs/科大讯飞SparkChain-全面替换第三方-落地文档.md b/docs/科大讯飞SparkChain-全面替换第三方-落地文档.md new file mode 100644 index 00000000..5e8f72ea --- /dev/null +++ b/docs/科大讯飞SparkChain-全面替换第三方-落地文档.md @@ -0,0 +1,492 @@ +# 科大讯飞 SparkChain —— 第三方语音与模型全面替换落地文档 + +> 2026-10-15 起草，同日按功能、测试服边界、两端 SDK 差异和项目入口再补过。**本文只做设计，未改任何代码。** +> 真机怎么测、失败了改哪一行，见 [科大讯飞SparkChain-真机自测与回改.md](科大讯飞SparkChain-真机自测与回改.md)。 +> SDK 版本：`SparkChain_Android_SDK_overseas_2.1.4_rc4`、`SparkChain_IOS_SDK_overseas_2.1.4_rc4`。 +> 目标：语音识别、合成、机器翻译、同传、文本大模型、会议转写改走讯飞。微软语种（含爱尔兰语）走 SDK 里的微软引擎。 +> +> **服务器只动测试服。** 正式机 `47.116.104.181`、正式接口 `https://ym.ymaikj.com`、正式后台库一律不改。 +> 测试服是阿龙 `8.133.166.29`，业务域名 `https://ym-dev.ymaikj.com`。发布只用各目录的 `./dev-deploy.sh along`。 +> 禁止 `prod-build.sh` / `prod-deploy.sh`。客户端联调 `.env` 保持 `SERVER_URL=https://ym-dev.ymaikj.com`，不要改去正式域名。 + +--- + +## 0. 一句话结论 + +SparkChain 是**装在手机里的原生 SDK**（安卓 AAR、iOS framework），不是 Go 能链接的库。 +所以「全面换成它」要拆成两截，缺一截都不算换完： + +1. **手机上的实时链路**（通话翻译、同声传译、面对面、文本翻译、播报）直接调 SDK。 +2. **服务器上的离线链路**（`user_translate` 兜底、会议转写、会议总结）留在 Go 里，调讯飞开放平台 HTTP。 + 这套代码和后台配置**只发布、只改测试服**。正式服继续跑现在的阿里/微软/字节，直到以后单独决定上线。 + +现在工程里那套讯飞（`ntrans.xfyun.cn`、`IflytekIntegratedSpeechService`）是旧的听写/翻译接口， +**不是**这份 SparkChain。不要在旧类上打补丁，新开一个原生插件。 + +有三块这份 SDK **不能原样替换**，落地时按第 5 节的写法处理，不要假装是同一条接口： + +- 耳机 EMAI。海外 SDK 没有百炼那种「持续送 PCM、边说边回」的全双工对话。落地是听写 + 文本 LLM + 合成三段接进现有会话服务，工具调用和设备协议留在原处。 +- 中国移动灵犀、对象存储、短信、实名、微信、地图。它们不是语音模型。 +- 声音复刻、超拟人合成、图片 OCR 翻译。只在安卓**国内**文档里，海外包没有。 + +--- + +## 1. SDK 本身怎么用 + +### 1.1 初始化只做一次 + +两端都是 `appId` + `apiKey` + `apiSecret`，全局 `init` 一次，退出时 `unInit`。 + +| 节点 | 安卓 `apiType` | iOS `apiType` | 什么时候用 | +|---|---|---|---| +| 国内 | `ApiType.TPYE_CH` | `0` | 不用。国内文档的星火/DeepSeek 和海外这套不是同一条 | +| 新加坡 | `ApiType.TPYE_SG` | `1` | **默认** | +| 欧洲 | `ApiType.TPYE_EU` | `2` | 2026-09-02 起是白名单节点。账号没加白之前不要切 | + +枚举名里的 `TPYE` 是 SDK 自己的拼写，照抄，不要「改对」。 + +单次能力还可以再选引擎，和全局节点是两层： + +- 讯飞自研：听写、大模型识别、同声传译新加坡语种表、新加坡发音人。 +- 微软（文档里的 `msft` / iOS `RegionMSFT`）：实时转写、录音文件转写、在线合成、同声传译的另一张语种表。 + **爱尔兰语走这一层，不要为了它把全局节点改成欧洲。** + +兼容范围：安卓 5.0+，armeabi-v7a / arm64-v8a；iOS 9.0+，arm64。 +音频入口统一是 **16 kHz、16 bit、单声道 PCM**，和现在耳机插件出口一致，中间不用再转格式。 + +### 1.2 海外包里两端都有的 10 项 + +| SDK 能力 | 类 | 对应我们现在的事 | +|---|---|---| +| 语音听写 | `ASR` | 短句识别（1 分钟内）。同传/面对面的识别段可以先用它 | +| 大模型识别 | `ASR`（语种码 `*_cbm`） | 识别效果更好的听写。新加坡约 27 语，含自动切语 `mq_cbm`。欧洲大模型文档写了「暂不提供」 | +| 实时转写 | `RTASR` | 长语音流。会议进行中的实时字幕、以及爱尔兰语识别（欧洲 `ga_IE`，微软 `ga-IE`） | +| 录音文件转写 | `RAASR` | 已录好的长音频，最长 5 小时，创建任务再查询。微软任务类型 `iflyrec_voice_mul_cbm_msft_24h` 含 `ga-IE` | +| 在线合成 | `OnlineTTS` | 译文播报，微软引擎只出 raw PCM。安卓文档有 `ga-IE-OrlaNeural`；iOS 发音人表没有爱尔兰语，见第 4.5 节 | +| 在线翻译 | `ITS` | 文本翻译。小语种表两边都有爱尔兰语 `ga`，但引擎枚举不同：安卓是 `GTRANS`，iOS 没有这个名字 | +| 同声传译 | `AST` | 识别 + 翻译 + 可选合成，一条链路。给中英这类已覆盖的通话/同传 | +| 文本对话 | `LLM` | 海外走 OpenAI 兼容接口，经讯飞私有地址。文档点名 `gpt-5.3-*`、`gpt-5.4`、`gpt-5.4-pro` | +| 图片理解 | `LLM` | 一张图加一段文字提问。不是「图中文字 OCR 再贴回原图」 | +| 图片生成 | `LLM`（`domain=nrt`） | 文生图，`gpt-image-1.5`。本产品现在没有这个功能，不接 | + +### 1.3 只在安卓国内文档里、这次不接 + +声纹特征、声音复刻、超拟人合成、图片 OCR 翻译、Embedding、实时转写大模型、录音文件转写大模型、极速转写。 +iOS 这包没有对应文档。产品若以后要「译文用说话人自己的声音」，那是国内声音复刻，和现在阿里 3.5 的实时声音复刻不是同一个东西，本次范围外。 + +--- + +## 2. 现在第三方实际落在哪 + +后台模板在 `apps/services/modules/console/api_svctemplate.go`。下表是**测试服当前会走到的**，换成一列是测试服上的去向。正式服配置不动，所以正式用户仍走旧服务商。 + +### 2.1 客户端实时 + +| 功能 | 现在 | 代码入口 | 换成 | +|---|---|---|---| +| 通话翻译选路 | 阿里 3.5 声音复刻 > 阿里旧版 > 微软三段 > 讯飞旧接口 | `language_manager.dart` 的 `findBestMatchingProvider` | 语种在 AST 表里走 SparkChain `AST`；不在表里走三段（`RTASR`/`ASR` + `ITS` + `OnlineTTS`） | +| 通话双路音频 | 耳机 PCM 不经 Dart，原生直连 `azure_speech` | `BesCallPcmBridge`（iOS 通知 / Android 静态槽） | 桥的契约不动，消费方从 `azure_speech` 改到新插件 | +| 同声传译 / 面对面 | 微软听写 + `user_translate` + 合成 | `translation_controller.dart` | 识别和合成改 SDK；翻译改端上 `ITS`，不再打 `user_translate` | +| 文本翻译页 | `ServerTranslationService` → `user_translate`，阿里覆盖不了再直连微软翻译 | `server_translation_service.dart` | 端上 `ITS`。安卓 `GTRANS`，iOS 用第 4.5 节确定的类型 | +| 图片翻译 | 百炼视觉模型，只出原文/译文，不回贴图 | `alibaba_image_translation_service.dart` | 端上图片理解，提示词要求抽出图中文字并翻译。坐标和回贴图本来就没有，不用补 | +| 耳机 EMAI | 百炼多模态，原生 WebSocket，工具调用留在 Dart | `BailianDialogClient` / `DeviceAiSessionService` | 听写 + 文本 LLM + 合成接进同一条唤醒，见第 5.5 节。百炼密钥在测试服停用前保留，便于切回 | +| 旧讯飞 | `stt_iflytek` 的 appId / accessKey，host 默认 `ntrans.xfyun.cn` | `azure_speech` 里 `IflytekIntegratedSpeechService` | 废弃。SparkChain 用另一套 appId / apiKey / apiSecret | + +通话翻译现在是端到端直连，会员拦不住那一跳，靠「不下发凭据」关。换 SDK 之后凭据在端上，会员闸门要在**开桥之前**的 Dart 里拦，不能再指望密钥不下发。 + +### 2.2 服务端 + +| 功能 | 现在 | 代码入口 | 换成 | +|---|---|---|---| +| 实时机器翻译 | 只实现了阿里翻译。客户端对阿里没有的语种改直连微软 | `modules/user/api_translate.go`、`model_translate.go` | 端上改走 `ITS` 之后，这个接口变成兜底。兜底也改讯飞机器翻译 HTTP，不再留阿里/微软两套 | +| 会议转写 | 字节 / 阿里百炼 / Google / Azure 四选一 | `modules/echomeet/provider_registry.go` | 讯飞录音文件转写 HTTP（和 SDK 的 `RAASR` 是同一个云能力） | +| 会议总结 | 豆包 / 通义 / Gemini / Azure OpenAI | 同上，`Summarizer` | 讯飞文本大模型 HTTP。海外 SDK 的 LLM 是 OpenAI 兼容私有地址，服务端按讯飞开放平台文档接，不要自己猜 SDK 内部签名 | +| 会议翻译 | 2026-09-26 起转写不再翻译，翻译只决定总结用什么语言写 | `tasks.go` 的 `TranslateProcess` | 不恢复「转写再翻译一遍」。总结语言仍由提示词决定 | + +### 2.3 明确不动 + +| 东西 | 原因 | +|---|---| +| 阿里云 OSS、短信 | 存储和短信，SDK 没有 | +| 实名（阿里 / 腾讯 / 创蓝） | 身份核验，SDK 没有 | +| 微信登录、地图、内购 | 与语音无关 | +| 中国移动灵犀 `llm_mobile_elf` | 另一份合同和原生 SDK，`AGENT_TYPE=yidong` 才走进去 | +| 耳机协议、通话下行、后台保活 | 音频通路已定，这次只换「PCM 送去哪家云」 | + +--- + +## 3. 语种怎么选引擎 + +原则：**能一条链路走完的用 `AST`，表里没有的拆成三段。** 不要再按「阿里 > 微软 > 讯飞」排四家。 + +| 场景 | 引擎 | 说明 | +|---|---|---| +| 中英日韩和 AST 新加坡表里的约 30 个语种 | `AST`，讯飞自研 | 识别、翻译、合成一次会话。合成默认关，要播报再开 `use_tts` | +| AST 新加坡没有、微软同传表里有的 | `AST`，`msft` | 音色是 `*Neural`。同传表里**没有爱尔兰语** | +| 爱尔兰语，以及两张 AST 表都没有的语种 | 三段 | 识别、翻译、合成的**类和语种码按第 4.5 节分端写**。安卓文档写明了微软 `ga-IE` 和 `ga-IE-OrlaNeural`；iOS 文档没有把这两项列全，不能把安卓参数原样抄过去 | +| 同传/面对面只需要文字、句子短 | 听写 `ASR` + `ITS` | 听写限制约 1 分钟。超时或长会改 `RTASR` | +| 会议文件 | 服务端录音文件转写 HTTP | 语种用微软 24 小时任务时与 `RAASR` 的 `ga-IE` 对齐 | + +爱尔兰语是这次换厂商的验收样本，因为现在中文→爱尔兰语的通话翻译，远端爱尔兰语识别走的是微软三段，而阿里端到端覆盖不了它。换完之后这一对必须仍然是：近端中文能识别、译成爱尔兰语、远端听得到；远端说爱尔兰语，本端能出原文、中文译文和中文声音。 + +声音复刻（对方听到的是说话人自己的声音）这次会丢掉。那是阿里 3.5 的能力，海外 SparkChain 没有。中英语种改走 `AST` 之后，音色是发音人表里的固定音色，按路选，不再按说话人现场刻。 + +--- + +## 4. 客户端怎么接 + +### 4.1 新插件，不塞进 `azure_speech` + +新建 `apps/client/local_plugins/sparkchain/`： + +- 安卓依赖 `SparkChain.aar`。编解码需要的话再带 `Codec.aar`（SDK 包里有）。 +- iOS 用 `sdk/overseas-uid/SparkChain.framework`。不要链 `domestic-uid`。 +- Dart 侧只暴露和现有服务同形的方法：初始化、听写开始/送 PCM/结束、同传开始/送 PCM/结束、翻译一句、合成一段、释放。 +- 回调进现在的 `AstService` / 翻译控制器已经在听的事件：识别中间结果、最终结果、译文、TTS PCM、错误。 + +`azure_speech` 先留着，用开关切。新插件稳定之前，旧的阿里/微软路径还能回退。确认全量切走之后再删旧依赖，不要在同一次改动里既接新 SDK 又拆旧插件。 + +iOS 本地插件的 Package.swift 若引用兄弟插件，按仓库里已有的做法：`flutter pub get` 之后跑 `apps/client/ios/sync_spm_platform.sh`。 + +### 4.2 通话双路 PCM 的契约不变 + +现在的约定（改一处必须同改另一处，改漏了编译能过、但一帧都到不了）： + +- iOS 通知 `com.eaimar.bes.callpcm.uplink` / `downlink`，键 `leg`、`pcm`。A = 本端麦，B = 对端通话音。 +- 安卓 `bluetooth_manager` 的静态槽，`azure_speech` 现在来接。 + +三条桥不要接错。通话翻译、面对面、耳机 EMAI 各有一条，通知名和槽名改漏了编译能过，但一帧都到不了： + +| 功能 | iOS 通知 | 安卓槽 | 现在谁在听 | +|---|---|---|---| +| 通话翻译 | `com.eaimar.bes.callpcm.uplink` / `downlink`，键 `leg`、`pcm`。A = 本端麦，B = 对端 | `BluetoothManager.callPcmTap` | `BesCallPcmBridge` | +| 面对面 | `com.eaimar.bes.f2fpcm.uplink` / `downlink`，同样的键。A/B 是左右耳 | `BluetoothManager.f2fPcmTap` | `BesF2fPcmBridge` | +| 耳机 EMAI | `AiPcmBridge` | `BluetoothManager` 的 AI 上行槽（与 `callPcmTap` 分开） | `BesAiPcmBridge` | + +SparkChain 插件去接现在这条桥，门限、积压丢帧、`ready` 留在桥上。不要让 SDK 自己开麦克风。手机麦、录屏、广播扩展已经在现有代码里产出 PCM，继续推进插件的 `write`。SDK 再开一次麦，会和前台服务、`AVAudioSession`、耳机麦抢设备，表现为没声音或声音跑到扬声器。 + +`AST` 要两路各一个实例：A 路源语种是用户语言、目标是对方语言；B 路相反。和现在阿里/微软双路的方向一致。 + +下行译文 PCM 仍走现有 `CallTranslationDownlink`（16 kHz PCM → G.722 → 耳机）。SDK 合成如果给出别的采样率，在插件里重采样到 16 kHz 再交出，耳机侧不改。 + +### 4.3 初始化与线程 + +- App 启动、配置拉下来之后初始化一次。重复 `init` 先 `unInit`。`unInit` 只在退出登录或密钥变了的时候做，不要在每句结束时做。 +- `workDir` 文档标了必填，且必须可写。用应用私有目录（安卓 `filesDir`，iOS Documents 下单独一个子目录）。不要写 `/sdcard`，也不要为此申请 `MANAGE_EXTERNAL_STORAGE`。测试机华为是 Android 12，`requestLegacyExternalStorage` 已经不起作用。`logPath` 不设，日志走 logcat / 控制台。release 的 `logLevel` 用 WARN，避免 SDK 把音频路径打进日志。 +- 听写/转写/同传的 `write` 不要在主线程上堵。桥上的 PCM 已经在串行队列里，插件里直接 `write`。 +- 同一时刻只跑一条流式会话（一路 `AST`，或通话/面对面的两路 `AST`）。翻译四个模式和 EMAI 本来就会 `suspendOtherServices`，保持这个互斥。不要通话翻译开着再把 EMAI 的听写叠上去。 +- 两路同时 `start` 时按 SDK 文档错开（现有微软实现是 A 起来再等 1.2 秒再起 B）。SparkChain 是否必须错开，以第一版真机日志为准；若两路并行会让第二路无回调，就沿用这个间隔。 +- 回调对象在插件里强引用，持有到 `stop`。不要只留在局部变量里——iOS 上弱引用会在初始化返回后被释放，表现就是一路完全没有识别结果。 +- 合成采样率按 16 kHz 交给现有播放器。耳机 EMAI 的 `_deviceRate` 就是 16000。百炼下行是 24000，那是百炼自己的出口，不要套到 SparkChain 上。SDK 若给出别的采样率，在插件里重采样到 16 kHz 再交。 + +### 4.4 凭据 + +SDK 要求密钥在手机上。这和 2026-08 把机器翻译改成服务端代调、避免密钥进包的决定相反，是用这家 SDK 的代价，单独接受： + +- 后台新增一个第三方服务 `sparkchain`。全局 `init` 用 `app_id`、`api_key`、`api_secret`、`api_type`（默认新加坡）。 +- 新加坡同传文档要求再传一套 `ast_appid` / `ast_apikey` / `ast_apisecret`。模板里单独三个字段。和上面相同就填同一份，不要在客户端写死「肯定相同」。 +- 经现有 `user_getappconfig_v3` 下发。密钥字段标加密。 +- 客户端用 `AppConfig.cred` 读取，空则该能力不可用，不要回退到旧的 Azure/阿里密钥。 +- release 日志禁止打密钥。可以打 init 返回码和会话 `sid`，方便真机对账。 +- **类别不要标成端到端（类别 4）。** `filterSvcsByVip` 会把类别 4 整份从下发列表拿掉。`sparkchain` 一份密钥同时给文本翻译和 EMAI 用，标成 4 之后，非会员连这两项都初始化不了。 + +会员：通话、同声、面对面、音视频在点「开始」时拦。现在 `translation_controller.dart` 的 `_hasTranslationQuota()` 恒为 `true`，不能当成已经有闸门。落地时改成读现有会员状态。EMAI 不新加会员限制。`user_translate` 兜底仍走服务端原来的会员判断。 + +### 4.6 什么时候选讯飞节点 + +有两套「节点」，不要混成一件事。 + +**我们自己的接口**（`ym-dev` / 正式域名）不在这次按地区新开机器。`DioManager.baseUrl` 已经在进程启动时用手机系统地区选 `SERVER_URL` 或 `HW_SERVER_URL`。现在这两个值都是 `https://ym-dev.ymaikj.com`。登录页上选的国家是短信区号，不是人所在的地方，不能拿来切服务器。这次也不要为欧洲用户再部署一台业务服务器。 + +**讯飞云节点**才按用户所在地选。选的是 SDK 的 `apiType`，不是我们 Go 服务的地址。 + +时机不要等到进翻译页，也不要只绑在登录按钮上： + +1. 启动拉 `user_getappconfig` 时，服务端已经用请求 IP 做过 `ipinfo` → `ToRegion`（`modules/user/core.go`）。这个结果今天只用于 MCP 和配置作用域，**没有回给客户端**。测试服在这份响应里多带一个讯飞节点建议。 +2. 客户端拿到之后再 `init`。同一进程里节点不变就不 `unInit`。 +3. 登录（`api_sgin`）也会按 IP 写下 `CountryCode`。若登录时的地区算出的节点和启动时不同（人换了网络），登录成功后 `unInit` 再 `init` 一次。一场翻译或 EMAI 进行中不要换节点。 +4. IP 查不到时用手机系统地区兜底，再不行用新加坡。不要用登录页的短信区号。 + +节点对照： + +| 服务端 `ToRegion` | 讯飞全局节点 | 说明 | +|---|---|---| +| 欧洲 `RegionEurope` | 欧洲 `TPYE_EU` / iOS `2` | 只有账号已加白才切。没加白之前欧洲用户也用新加坡，否则 init 失败，整份语音都不可用 | +| 其他地区，含中国、东南亚、美洲 | 新加坡 `TPYE_SG` / iOS `1` | 中国用户也走海外这套 SDK 的新加坡节点。不要切 `TPYE_CH`，那是另一份国内文档，星火和 DeepSeek 不在这次的包里 | +| 未知 | 新加坡 | 失败方向是能用，不是切到一个没加白的节点 | + +人在哪，只决定上面这个全局节点。一句爱尔兰语仍然走第 4.5 节的微软引擎，不因为人在新加坡就把 `ga` 送进新加坡同传表。人在欧洲，也是先加白再用欧洲节点，识别码仍按第 4.5 节，不把两层合成一个 if。 + +测试服改的是「响应里多一个节点字段」。正式服的 `user_getappconfig` 不动，正式包也还不会读这个字段。 + +### 4.5 安卓和 iOS 不是同一份 SDK + +Dart 只保留一套方法。Kotlin 和 Swift 各写一份，不要共用一份 JNI，也不要把一边的枚举名写到另一边。插件对外只收我们自己的语种短码（`zh`、`en`、`ga`），**各端自己换成该能力在该平台文档里的码**。 + +| | 安卓 | iOS | +|---|---|---| +| 包 | `SparkChain.aar`，需要时再加 `Codec.aar`。混淆保留 `com.iflytek.sparkchain.**`。只加 `INTERNET`。`workDir` 用应用私有目录，不申请存储权限 | `sdk/overseas-uid/SparkChain.framework`，Embed & Sign。不要链 `domestic-uid`。`workDir` 用 Documents 子目录 | +| 全局节点 | `ApiType.TPYE_SG`（拼写照抄） | `apiType = 1` | +| 选微软引擎 | 各能力自己的 region / 参数，见下表。不要传 iOS 的 `RegionMSFT` 这个名字 | `RegionMSFT` | +| 参数写法 | `setParams(key, value)`，`registerCallbacks` | `addStringParam` / `addIntParam` / `addBoolParam`，回调是构造时传入的 delegate，必须强引用到 `stop` | +| 同传 `AST` | `new AST(RegionType.SG_TYPE)`，新加坡要再设 `ast_appid` / `ast_apikey` / `ast_apisecret`。`start(lang, "pcm_s16le", 16000, tag)`，`write(byte[])` | `initWithregion:RegionSG delegate:`，同样再设那三个 `ast_*`。`start:audioEncoding:sampleRate:userInfo:`，`write:` 收 `NSData`。文档写 pcm 建议每 40ms 送 1280 字节，送太快会报错 | +| 同传语种码 | 中文是 `cn`，英文是 `en`，不是我们现在的 `zh-CN` / `zh-Hans`。微软同传表两端都没有爱尔兰语 | 同左。`tts_target_lang` 传字符串 | +| 在线翻译 `ITS` | `TransType.ITRANS` 只有中英日韩等约 9 个，中文码 `cn`。一百多种（含爱尔兰语 `ga`，中文码 `zh-CN` 或 `zh`）走 `TransType.GTRANS` | **没有 `GTRANS` 这个枚举。** 类型是 `ITRANS`、`NIUTRANS`、`ITRANS_SG`、`GOOGLETRANS`。`ITRANS` 对上那张 9 语表。含 `ga` 的那张大表标题是「讯飞机器翻译」，文档没写它对应哪个 `TransType`。接的时候用爱尔兰语 `ga` 把四个类型各打一次，用能返回译文的那个，不要在 Swift 里写一个不存在的 `GTRANS` | +| 爱尔兰语识别 `RTASR` | 欧洲节点 `ga_IE`（`ist_mul_sp`）。微软引擎语种表有 `ga-IE`（`ist_mul_msft`），不依赖欧洲加白 | 欧洲节点表有 `ga_IE`。微软一节只给了 `language_name` 参数，**没有语种表，没有 `ga-IE` 这一行**。先用 `RegionMSFT` + `language_name=ga-IE` 在真机打一次；没有识别结果就不要假装和安卓一样，爱尔兰语识别改走已加白的欧洲 `ga_IE`，或向讯飞要 iOS 微软语种表后再接 | +| 爱尔兰语合成 | `OnlineTTS` 微软表写了 `ga-IE-OrlaNeural`、`ga-IE-ColmNeural`。微软只出 `aue=raw` 的 PCM | 发音人表是讯飞的 `x_xiaoyan` 这一套，没有爱尔兰语。文档示例是 `initWithRegion:RegionMSFT vcn:@"en-US-JennyNeural"`。爱尔兰语先按安卓那个音色名、用 `RegionMSFT` 打一次；不出 PCM 就不要在 iOS 上报「已支持爱尔兰语播报」 | +| 听写推流 | `write` 在 SDK 回调线程之外调用，不要堵主线程 | 外部 PCM 不要让 SDK 再配 `AVAudioSession`。会话类别仍由现在的 `EmaiAudioSession` / 翻译页负责，否则会把耳机路由或后台音频踩掉 | + +合成结果两边都只取 PCM 字节，交给现在的播放器或 `CallTranslationDownlink`。插件里不要再 `new AudioTrack`，也不要让 SDK 自己播。 + +验收按端分开记，不能只测一台： + +- 安卓：中英 `AST` 有字有声；中文→爱尔兰语识别用微软 `ga-IE`、翻译用 `GTRANS` 的 `ga`、合成用 `ga-IE-OrlaNeural`。 +- iOS：中英 `AST` 同样有字有声。爱尔兰语三段以真机结果为准，写明实际用的 `TransType`、识别 region 和音色名。和安卓不一致就在插件里分端，不要改 Dart 的语种短码。 + +--- + +## 5. 分功能落地 + +六个功能都挂在现有页面和设备协议上，只换「PCM / 文本送去哪家云」。 +模式字符串不要改：`simultaneous`、`faceToFace`、`call`、`audioVideo`。 +`translation_controller.dart` 的 `changeTranslationMode` 仍是翻译四个模式的总入口。 + +共用约定： + +- 新提供商字符串用 `sparkchain`，不要复用已经绑在 `ntrans.xfyun.cn` 上的 `iflytek`。 +- 语种对在 `AST` 表里就走一条同传会话；不在表里（爱尔兰语）走三段。三段的引擎枚举和语种码按第 4.5 节分端，不要两端都写 `GTRANS`。 +- 端上 `ITS` 不能取消在途请求时，用序号丢掉过期的中间译文。最终那一次不能丢。 +- 会员在点「开始」时拦，不要只留在 `user_translate`。 +- 切后台的前台服务 / iOS `BackgroundKeepalive`、译文路由（`pinSpeaker: false`）保持现有调用点，不要为了换 SDK 重写。 + +### 5.1 同声翻译（`simultaneous`） + +现在的路径：单向。连上 Echo-one 就 `DeviceFeatures.recordStart` + `openMic(AudioRoute.mic)`，PCM 推进现在的 ASR；没连耳机就用手机麦。识别出整句后走 `ServerTranslationService`（阿里，覆盖不了再直连微软），再 TTS。安卓进页就 `AudioForegroundService.start(keepalive: true, pinSpeaker: false)`，iOS 再 `BackgroundKeepalive.acquire('simultaneous')`。默认录音，结束经 `RecordingArchive` 进语音纪要。 + +换成 SparkChain 时只动识别、翻译、合成三步： + +1. `changeTranslationMode('simultaneous')` 里现在的 `_reinitializeAsrService` 改为按第 3 节选引擎。在 `AST` 表内就初始化一条 `AST`（源 = 用户选的源语言，目标 = 目标语言，`use_tts=true`）。不在表内就初始化听写（只传源语言一种，保持现在的单向，不要开连续语种识别）加端上 `ITS` 加 `OnlineTTS`。 +2. `_armSimultaneousInput` / `_attachSimultaneousHeadsetMic` 不动。耳机麦和手机麦的 PCM 都改送到新插件的 `write`。SDK 不自己开麦。手机麦类别保持现在的 `phone_call`。 +3. 译文 PCM 继续交给现在的播放器。不要在插件里另开一路 `AudioTrack` 去抢路由，否则又会把声音从耳机拽到手机扬声器。 +4. 中间结果：现在 700ms 节流、增长不足 4 字不发、可取消。`AST` 自带流式译文时直接上屏、不播报。三段路径仍按这个节流去调 `ITS`。 +5. 停：先停推流和 SDK 会话，再 `recordStop` 关耳机麦，再释放保活。顺序反了，耳机会继续推麦。 +6. 方向固定为源 → 目标。不要用识别回来的语种去对调方向，现在单向模式就是为了避开这个坑。 + +验收：连耳机时日志仍是「同声翻译收音：耳机麦」，首帧到达；译文从耳机出，不从手机扬声器出。切后台仍能出字。中英走 `AST`。中文→爱尔兰语走三段，译文是爱尔兰语文字和声音。 + +### 5.2 面对面翻译（`faceToFace`） + +现在有两条收音，不要合成一条： + +| 条件 | 现在 | 换 SDK 之后 | +|---|---|---| +| Echo-one 已连接且有 `faceToFace` 能力 | `_startBesFaceToFace`：`f2fBleStart`，双耳 PCM 经 `setNativeF2fBridge` 进端到端。左右耳按键走 `setActiveSpeaker(1/2/0)`。失败不退回手机麦 | 桥的通知/槽改由 SparkChain 插件接收，协议帧（`AA 6B` 等）一个字节不改。语种在 `AST` 表内：左耳、右耳各一条 `AST`，方向相反（左耳源→目标，右耳目标→源）。不在表内：两路各一个听写，方向由 `activeSpeaker` 决定，翻译走 `ITS` | +| 没连这台耳机 | 手机麦，连续识别，`ASR normal`，TTS 模式 `push_to_talk` | 同样手机麦。双向仍靠现在的说话人判断，不要改成单向 | + +按住说话只在设备触发时成立（`FaceToFaceTrigger.device` 且松手 `activeSpeaker==0` 停识别）。App 里点开始是连续识别，松手逻辑不能套到这条上。 + +保活：安卓仍是 `AudioForegroundService.start()`（面对面这条今天没有 `keepalive` 参数，不要顺手改成和同传一样，除非单独验证双耳 BLE 不被静音轨抢走）。iOS 仍是 `BackgroundKeepalive.acquire('faceToFace')`。 + +停：`f2fBleStop`、关桥、`stopContinuousTranslation`、释放 iOS 保活。现在的 `_stopBesFaceToFace` 顺序保留。 + +验收：双耳都戴上才能开始；左耳说中文、右耳说英文（或爱尔兰语）方向相反且都有声音；来电退出仍弹现有文案并停会话；不连耳机时手机麦仍能翻。 + +### 5.3 通话翻译（`call`） + +现在：必须有 `DeviceCapability.callAudioTap`。`callStart` + 两路扬声器 + `asrStarted`，PCM 经 `BesCallPcmBridge` 直连，不经 Dart。A = 本端麦 → 对方语种，声音给对方；B = 对端通话音 → 用户语种，声音给用户。`AudioForegroundService.start(keepalive: true, pinSpeaker: false)`。iOS 再保活 `callTranslation`。`initialize` 已经拉起会话时，点开始不要再 `start` 一次。 + +换成 SparkChain： + +1. 选路改 `findBestMatchingProvider`。中英等在 `AST` 表内的，A/B 各一个 `AST` 实例，打开合成。爱尔兰语 A/B 各走三段：B 路识别对方的爱尔兰语、译成中文、合成中文；A 路相反。识别码、翻译引擎、音色按第 4.5 节分端，安卓用微软 `ga-IE` 和 `ga-IE-OrlaNeural`。不要做中文/爱尔兰语自动检测。 +2. 桥的契约不变（iOS 通知名和键、安卓静态槽）。`ready` 只在会话销毁时清掉，`disable` 不清，否则同页第二场没有声音。 +3. 两路 `start` 若并行导致第二路没有回调，沿用现在的间隔：A 起来后约 1.2 秒再起 B。回调对象在插件里强引用到 `stop`。 +4. 译文 PCM 仍进 `CallTranslationDownlink`，16 kHz。SDK 若不是 16 kHz，在插件里重采样后再交。耳机协议和 G.722 打包不改。 +5. `broadcastpeertranslate` 仍只决定 B 路要不要在手机上再播一遍，默认 false。 +6. 点开始时若 `initialize` 已拉起，保持 `_astAutoStartPending`，不要再开一路把前一路 generation 冲掉。 + +验收（不退出页面）：中英双方都有字和声音；停掉再开一场仍有；中文→爱尔兰语时近端中文能识别、远端听到爱尔兰语，远端说爱尔兰语时本端有原文、中文译文和耳机里的中文。 + +### 5.4 音视频翻译（`audioVideo`） + +现在是单向，和同声一样「听到什么就翻成目标语言」，但音源不是麦： + +- 安卓：`MediaProjection` 抓系统播放。先开悬浮窗，识别就绪后再弹录屏授权。浮窗被拒绝就不起识别。 +- iOS：ReplayKit 广播扩展抓系统音频，同时默认拉字幕小窗。小窗起不来不中断翻译。不要再走杰理 `openDecoder()`，那条已经没有 PCM。 +- 两端都把 `_audioSourceType` 设为外部 PCM。TTS 默认关（`isTtsEnabled=false`），这个模式是看字幕，不是把译文再播出来盖住原视频。 +- 小窗只在 `call` 和 `audioVideo` 显示。切到同声或面对面必须关掉，否则会盖住页面。 +- 录音不进语音纪要。这条先保持。 + +换成 SparkChain： + +1. 捕获链路（录屏、广播扩展、悬浮窗、画中画）不动。外部 PCM 改推进新插件。 +2. 引擎与同声相同：表内 `AST` 但 **不要开合成**（`use_tts=false`），只要识别和翻译文字。表外用听写 + `ITS`，不调 `OnlineTTS`。 +3. 识别只传源语言一种。方向用现在的 `_isOneWayMode`，不要按检测到的语种对调。 +4. 字幕小窗继续吃现在的历史列表。不要为 SDK 再做一套浮窗数据。 + +验收：安卓授权录屏后切到别的 App，悬浮窗里有译文；拒绝悬浮窗权限则不开始。iOS 广播开始后切后台，小窗能出字，原视频声音不被译文盖住。 + +### 5.5 EMAI（耳机唤醒对话） + +现在：`DeviceAiSessionService._handleStart`。3 秒内先 `aiAccept`。耳机走原生百炼（`BailianNativeDialog` + 设备 PCM 桥），车载香薰走 Dart 百炼、回复用手机 A2DP。问候语、工具调用解析、切后台保活、音乐焦点都在这条服务里。手机里按住说话的那一页不在这条链路里，本次不动。 + +海外 SDK 没有对等的全双工对话。落地用三段接进**同一个** `_handleStart`，不新做一套唤醒： + +1. 设备协议不动：`aiAccept`、退出命令、PCM 进出耳机的桥。桥的消费方从百炼客户端改成 SparkChain 听写推流。 +2. 会话建立后先播现在的随机问候（「我在 / 回来了 / 有什么可以帮到你」），用 `OnlineTTS` 合成，走现在的喇叭（耳机私有下行，或香薰的 A2DP 播放器）。不要等用户先说话。 +3. 上行 PCM 进听写。一句最终结果出来后，调文本 `LLM`。location、date 仍在 Dart 里拼进提示词（地址是「未知」时不要把这两个字写进去），再把整段提示词交给 SDK。回复 PCM 按 16 kHz 进现在的喇叭。百炼那路 24000 不要照搬。 +4. 工具调用：安卓 `LLM` 有 `LLMTools`，iOS 用同一份工具 JSON 的对应写法。返回仍交给 `assistant_directive_service.dart`，金测试不挪到原生。第一版先打通「问一句有回答」；工具不通时记下来，不要为了工具去改待办的解析格式。`quit` 仍走现在的退出命令。SDK 不支持工具时，定闹钟会失败，问答不能因此被拿掉。 +5. 打断：香薰在会话中再按唤醒键，保持现在的「扔掉待播、听新的一句」。耳机原生路径保持现在的门控（播报时上行换静音），开关时机仍由 `DeviceAiSessionService` 决定，不要搬进 SDK。 +6. 保活和音乐焦点调用点不动：安卓先前台服务再抢焦点再静音；耳机静音钉扬声器，香薰不钉。iOS 仍是 `BackgroundKeepalive`。 +7. 百炼配置缺失时现在会拒唤醒。改完后改为 SparkChain 未初始化就拒，仍要 `ai` 退出，避免设备空等。 + +做不到的要写进验收说明：没有百炼那种「模型边听边说」的同传式回复，是说完一句再答。延迟会比现在大。这是 SDK 能力边界，不是实现遗漏。 + +验收：耳机按键能听到随机问候；问一句有文字和声音；定闹钟仍进待办；切后台不断；没在播的音乐不会被拉起来；香薰有音乐在播时，结束对话后音乐恢复。 + +### 5.6 语音纪要 + +两条入口，换的地方不同。 + +**翻译页顺带归档**（通话、同声、面对面；音视频不归档）： + +停录音时 `RecordingArchive.archive` 把文件和**已经翻译好的句子**交给 `echomeet_addrecord`。服务端不再对这段转写做识别。所以这三条换 SDK 之后，纪要里的字来自端上识别结果，服务端只负责存和总结。总结模型改讯飞 HTTP 时，这条归档不用再传音频给转写服务商。时间轴、暂停区间、文件名前缀（`trans_call` / `trans_holder`）保持。 + +**纪要首页自己录或从设备导入**： + +`meeting_home_controller` 调 `echomeet_addrecord`，音频在服务端排队转写再总结。这里换的是 `echomeet` 的服务商，不是手机 SDK： + +- 新增 `ServiceType`（例如 `Iflytek`），实现现有 `Transcriber` 和 `Summarizer`。状态机、轮询、卡住扫描、失败退算力都不改。 +- 转写走讯飞开放平台「录音文件转写」HTTP，与 SDK 的 `RAASR` 是同一类云能力，但是 HTTP，不要在容器里跑 AAR。空语言仍是自动识别，不要退回中文。转写结果不再翻译；总结用什么语言仍由现有提示词决定。 +- 总结走讯飞文本大模型 HTTP。`` / `` 的包法不动。 +- 支架 TF 卡导入只负责把文件送进同一条 `addrecord`，导入逻辑不改。 +- 客户端能力接口不改。语言列表跟着新服务的 `languages`。 + +开放平台 appId 若和 SparkChain 不是同一个，后台配两套字段，对用户仍是一个「讯飞」。 + +验收：同声或通话翻完，纪要里能看到音频、端上那份转写、以及总结。首页新录一条中文、一条非中文，转写和总结都能完成；失败退算力只退一次。 + +### 5.7 文本翻译页和图片翻译 + +文本页走 `ServerTranslationService`。改为优先端上 `ITS`，爱尔兰语码 `ga`。安卓用 `GTRANS`；iOS 用第 4.5 节试出来的那个 `TransType`。失败再调 `user_translate`，服务端内部改讯飞 HTTP，路径名不变，给没升级的包兜底。 + +图片翻译继续只出原文和译文，不回贴图。用图片理解加提示词抽出图中文字并翻译。国内「在线图片翻译」不接。 + +--- + +## 6. 只改测试服：服务器接口和管理后台 + +测试服和正式服是两套机器、两套库、两个域名。改测试服的 `svc_config` / `echomeet_orch` **不会**同步到正式服。反过来，对正式后台点保存会立刻影响正式用户。本次所有后台操作都在测试后台里、应用选 `ym`，看请求是不是打到 `ym-dev`。 + +| | 测试服（本次唯一允许动的） | 正式服（禁止） | +|---|---|---| +| 机器 | `8.133.166.29` | `47.116.104.181` | +| 业务接口 | `https://ym-dev.ymaikj.com` | `https://ym.ymaikj.com` | +| 发布 | `deploy/app`、`deploy/console`、`deploy/admin` 下 `./dev-deploy.sh along` | `prod-build.sh`、`prod-deploy.sh` | +| 客户端 | `apps/client/.env` 的 `SERVER_URL` 保持 `ym-dev` | 不要把联调包的服务器改成正式域名，也不要把这版上架 | +| 密钥 | 本机 `DEV_KEY=$HOME/Documents/keys/loginscre.pem`，不改脚本里的默认路径 | 同一把钥匙也能登上正式机，所以上面这条禁止要靠人不执行正式脚本，不是网络隔离 | + +`dev-deploy.sh` 只换镜像，不上传服务器上的 `.env` 和 `confs/*.yaml`。测试机上已有的网关白名单、数据库不要用正式机文件覆盖。新接口若要进网关白名单，只改测试机那份 `confs/gateway.yaml` 然后重启测试容器。 + +### 6.1 后台改哪些页 + +管理后台登录测试环境，右上角应用固定为 `ym`。凭据按应用隔离，改错应用等于改了另一套产品。页面只动这两个： + +**第三方服务配置**（`/serviceconfig`） + +模板写在 `api_svctemplate.go`，要先把 console 镜像发到测试服，页面上才出新行。然后只在 `ym` 上启用并填值，不要点到别的应用。 + +| 新服务 id | 类别 | 谁读 | 字段 | +|---|---|---|---| +| `sparkchain` | 不标类别 4。随 `user_getappconfig_v3` 下发 | 客户端 `AppConfig.cred` | `app_id`、`api_key`、`api_secret`（后两个加密）。`api_type` 默认新加坡，只是 IP 查不到时的兜底，真正节点见第 4.6 节。另三个可选：`ast_app_id`、`ast_api_key`、`ast_api_secret`，空则原生用上面那一套 | +| `asrfile_iflytek` | 录音文件识别 | 会议配置 → 转写 | 开放平台的 appId / secret，以及任务用的引擎（微软长音频含 `ga-IE` 时写在这里，不要写进客户端） | +| `llm_iflytek` | 文本大模型 | 会议配置 → 总结，以及 EMAI 若改走服务端代答时 | 模型名、接口地址。地址以讯飞开放平台文档为准，不要抄 SDK 里那个私有 path 自己猜签名 | +| `mt_iflytek` | 机器翻译 | `user_translate` 兜底 | 这是服务端 HTTP，不是手机 SDK 的枚举。爱尔兰语短码 `ga`。中文用开放平台要求的码，不要写成安卓 `GTRANS` 或 iOS `GOOGLETRANS` | + +`api_key` / `api_secret` 必须标加密。下发仍走已有的 v3，测试机网关白名单里已经要有 `user_getappconfig_v3`。缺这一行时未登录拉配置会 `code:18`，客户端静默拿不到密钥，表现是「后台填了手机却初始化失败」。 + +旧服务在测试服由 `AI_ONLY_IFLYTEK=1` 整段拦住：识别、合成、翻译、端到端、大模型、录音转写、灵犀都不许再打到微软、阿里、字节、谷歌。库里的行先留着，但接口不会去调，凭据也不再下发给连着测试服的包。正式服不设这个变量。允许的只有 `sparkchain`、`mt_iflytek`、`asrfile_iflytek`、`llm_iflytek`。 + +**会议配置**（`/meetconfig`） + +这一页不存密钥，只引用上面配好的服务 id，写在该应用的 `echomeet_orch`（`region=0`）。 + +1. 转写：把 `asrfile_iflytek` 设为优先级最高。旧的录音识别先留在后面当兜底，测试通过后再在**测试服**停用。选路是「已启用且支持该语言的第一个」，所以旧服务如果优先级更高，讯飞永远不会被叫到。 +2. 总结：默认模型改成 `llm_iflytek`。允许客户端选模型时，列表里不要再出现正式环境才有的模型名，避免测试包选了一个测试库里没有的 id。 +3. 不要改会议模板正文，也不要改「转写不翻译、翻译只决定总结语言」的开关语义。 +4. 保存后看测试服 home 日志有没有 `ConfigChanged`。没有就等最多 10 分钟的定时重载，或重启测试 home。不要为了生效去重启正式机。 + +### 6.2 服务器接口怎么改（代码进仓库，镜像只上测试服） + +| 接口 | 现在 | 测试服上的新行为 | 正式服 | +|---|---|---|---| +| `POST /api/home/user_translate` | `model_translate.go` 只构造阿里翻译。阿里没有的语种，客户端自己直连微软 | 增加 `mt_iflytek` 分支。测试服启用它、停用阿里 MT 后，这个接口只打讯飞。路径、请求字段、VIP 判断不改，老包还能调 | 代码不到正式机，正式仍是阿里 | +| `user_getappconfig_v3` | 下发 `thirdsvcs` | 多一个 `sparkchain`。客户端 `cred()` 读它。空则该能力不可用，不回退旧密钥 | 正式不下发这套密钥 | +| `echomeet_addrecord` / 转写轮询 / 总结 | 四家服务商 | `provider_registry.go` 增加讯飞 `Transcriber` + `Summarizer`。状态机、退算力、空语言自动识别不改。翻译页归档带来的句子仍然不重转写 | 正式仍是原来的四家 | +| `echomeet_getcapabilities` | 语言列表来自编排里启用的服务 | 协议不改。测试服换成讯飞后，列表跟着新服务的 `languages` | 正式列表不变 | + +`resolveTranslator` 今天认不出的 MT 会打日志并跳过。接 `mt_iflytek` 时要在构造分支里真正返回翻译客户端，不能只在模板里加一行却让运行时跳过。跳过的表现是测试服 `user_translate` 返回「未配置机器翻译服务」，客户端再去直连微软，等于没换。 + +会议转写 HTTP 与手机 SDK 的 `init` 不是同一段鉴权。开放平台 appId 若和 SparkChain 不是同一个，测试后台配两套字段，对测试用户仍显示一个「讯飞」。 + +部署顺序（都在测试服）： + +1. `deploy/console && DEV_KEY=$HOME/Documents/keys/loginscre.pem ./dev-deploy.sh along`，让新模板出现。 +2. 测试后台按 6.1 填 `ym` 的服务，先不要停旧的。 +3. `deploy/app` 同样 `./dev-deploy.sh along`，home 里的讯飞分支才在 `ym-dev` 生效。 +4. 用测试包打 `user_translate` 和一条纪要，确认日志里的服务 id 是讯飞。 +5. 再在测试后台停用旧的语音服务。 +6. 管理后台本身若改了页面，`deploy/admin` 也只 `./dev-deploy.sh along`。 + +回退只回测试服：测试后台把旧服务重新启用、讯飞编排优先级放后；必要时把测试机镜像换回上一版 `dev-latest` 之前的镜像。正式服没有这次的镜像和配置，不需要回退。 + +版本号：实施时再升客户端 `pubspec` 的构建号。本文不改版本。联调包不要提交商店。 + +--- + +## 7. 实施顺序 + +| 步 | 内容 | 完成的标志 | +|---|---|---| +| 1 | 原生插件：init、听写一句、翻译一句、合成一句 | 安卓、iOS 各通一次。爱尔兰语按第 4.5 节各用各的枚举，日志无密钥 | +| 2 | 文本翻译页改 `ITS` | 中英、中文→爱尔兰语出字 | +| 3 | 同声翻译 | 第 5.1 节：耳机麦、译文走耳机、切后台、爱尔兰语三段 | +| 4 | 面对面 | 第 5.2 节：双耳方向相反；无耳机时手机麦仍可用 | +| 5 | 音视频 | 第 5.4 节：字幕出来，不播译文盖住原视频 | +| 6 | 通话翻译 | 第 5.3 节三条验收 | +| 7 | EMAI | 第 5.5 节：问候、问答、待办、切后台、音乐 | +| 8 | 语音纪要 | 第 5.6 节。服务端只部署测试服；首页新录音在 `ym-dev` 上走讯飞 | +| 9 | 测试后台停用旧语音服务 | 只停测试库 `ym`。正式库不停。旧密钥不再下发给连着 `ym-dev` 的包 | + +灵犀、存储、短信、实名不在表里。 + +--- + +## 8. 不要一起改的东西 + +- 正式服：机器 `47.116.104.181`、域名 `ym.ymaikj.com`、正式后台库、`prod-deploy.sh`。 +- Echo-one 协议字节、通话下行的 G.722 打包、`pinSpeaker` 的策略（同传和通话翻译保持不钉死手机扬声器）。 +- 面对面的双耳 BLE。 +- 设备 AI 对话的音乐焦点和后台保活。 +- `user_translate` 的路由名。老包还在调。新实现可以换内部服务商，不能改路径。 +- 会议「转写不翻译、翻译只影响总结语言」的口径。 + +--- + +## 9. 风险 + +| 风险 | 处理 | +|---|---| +| 密钥重新进客户端 | 只经测试服 v3 下发这一套讯飞密钥；日志不打；混淆沿用现有 release 包 | +| 误发到正式服 | 发布命令只允许 `dev-deploy.sh along`。正式机和正式后台本次不登录去改配置 | +| 同传微软表没有爱尔兰语 | 该语种强制三段，不走 `AST` | +| 两端 SDK 接口不一致 | 按第 4.5 节分端实现。iOS 没有 `GTRANS`，微软爱尔兰语识别和 `ga-IE-OrlaNeural` 只在安卓文档里写全。iOS 用真机打通后再写死参数，不要抄安卓枚举 | +| `workDir` 指向外部存储 | init 在华为 Android 12 上直接失败。用应用私有目录，不申请存储权限 | +| `sparkchain` 标成类别 4 | 非会员整份密钥被拿掉，文本翻译和 EMAI 一起不能用。类别不要用 4，翻译会员在点开始时拦 | +| SDK 自己开麦 | 和耳机麦、前台服务抢设备。PCM 一律由现有采集推进 `write` | +| 语种不在 AST 表里却仍走 `AST` | 按我们的语种表显式映射。表里没有的走三段；三段也没有的，沿用现在的不支持提示，不要猜一个相近语种码 | +| 按手机系统语言切到欧洲节点 | 系统地区不是人所在地。节点用测试服 IP 区域，欧洲未加白就保持新加坡。中国用户也不切国内节点 | +| 欧洲节点未加白 | 识别用微软引擎，全局节点保持新加坡 | +| 两路会话第二路无回调 | 强引用回调；必要时两路启动间隔 1.2 秒 | +| 会议 HTTP 与 SDK 不是同一个 appId | 实施前向讯飞确认；不能共用就后台两套字段 | +| 声音复刻消失 | 对用户说明中英通话不再用说话人本人音色，改固定发音人。不在本次用国内声音复刻补 | +| EMAI 没有全双工 | 按第 5.5 节接成说完再答；工具调用和设备协议不搬进 SDK | diff --git a/docs/科大讯飞SparkChain-真机自测与回改.md b/docs/科大讯飞SparkChain-真机自测与回改.md new file mode 100644 index 00000000..cb7a9a9e --- /dev/null +++ b/docs/科大讯飞SparkChain-真机自测与回改.md @@ -0,0 +1,210 @@ +# 科大讯飞 SparkChain —— 真机自测与回改 + +配合 [科大讯飞SparkChain-全面替换第三方-落地文档.md](科大讯飞SparkChain-全面替换第三方-落地文档.md)。 +客户端 `1.0.3+45` 起，连着 `ym-dev.ymaikj.com` 时 AI 不再调用微软、阿里、字节、谷歌、灵犀：文本和同声优先走端上 SparkChain，密钥还没配时这些能力直接失败，不再退回旧服务。面对面、通话、音视频、EMAI、会议转写还没接到讯飞，所以在这把锁下也不可用。测试后台还没填 `sparkchain` 密钥时，init 不会被调用。测试服容器要有 `AI_ONLY_IFLYTEK=1`，正式服不要设。 + +2026-10-15 电脑上已经连着的两台，下面的命令都用这两台，不要换别的机器对日志： + +| | 设备 | 标识 | +|---|---|---| +| 安卓 | 华为 ALN-AL80，Android 12 | 序列号 `2MM0223B09017126` | +| 苹果 | 李蒙蒙的 iPhone，iPhone 12 | UDID `00008101-001C684A2651003A` | + +服务器只连测试服 `https://ym-dev.ymaikj.com`。`.env` 的 `SERVER_URL` 保持这个。不要装到连着 `ym.ymaikj.com` 的包上测，也不要上架。 + +耳机用 Echo-one。没戴耳机的用例用手机麦，两套都要测，不要只测一种。 + +--- + +## 1. 每一轮怎么装 + +改完代码再升 `apps/client/pubspec.yaml` 的构建号，日志里要能对上是哪一轮。 + +安卓（在 `apps/client` 下）： + +```bash +export GRADLE_USER_HOME="$HOME/.gradle" +~/flutter/bin/flutter build apk --release --no-pub --obfuscate --split-debug-info=./build/app/outputs/symbols +adb -s 2MM0223B09017126 install -r build/app/outputs/flutter-apk/app-release.apk +``` + +苹果： + +```bash +./ios/sync_spm_platform.sh +~/flutter/bin/flutter build ipa --release --no-pub --obfuscate --split-debug-info=./build/ios/symbols --export-method development +xcrun devicectl device install app --device 00008101-001C684A2651003A build/ios/ipa/EAIMAR.ipa +``` + +`flutter build ipa` 会改 `ios/Runner.xcodeproj/project.pbxproj`，那是本地噪声，不要提交。 + +装完先看关于页或日志里的构建号，对不上就不要开始用例。 + +--- + +## 2. 日志怎么拿 + +安卓。`*:S` 在 zsh 里必须加引号，否则会被当成通配符： + +```bash +adb -s 2MM0223B09017126 logcat -c +adb -s 2MM0223B09017126 logcat -v time 'EmaiFocus:I' 'flutter:I' '*:S' +``` + +苹果不靠 syslog。App 自己的文件在 Documents/logs/： + +```bash +xcrun devicectl device info files --device 00008101-001C684A2651003A \ + --domain-type appDataContainer --domain-identifier com.yimai.eaimar | grep logs +xcrun devicectl device copy from --device 00008101-001C684A2651003A \ + --domain-type appDataContainer --domain-identifier com.yimai.eaimar \ + --source Documents/logs/<刚列出来的最新文件> --destination /tmp/eaimar.log +``` + +`copy from` 不要加 `--username`。 + +日志里允许出现 init 返回码和 `sid`。出现 `apiKey`、`apiSecret`、`ast_apisecret` 的原文，这一轮算失败，先把打印删掉再测功能。 + +--- + +## 3. 测之前确认 + +1. 手机网络正常，打开 App 后请求的是 `ym-dev`，不是 `ym.ymaikj.com`。 +2. 测试后台应用是 `ym`，`sparkchain` 已启用，三个密钥非空。`sparkchain` 的类别不是端到端（类别 4）。 +3. 旧的微软/阿里语音服务先还留着，等第 9 步全部用例过了再在测试服停用。 +4. 杀进程冷启动一次再测。热重载看不出原生 SDK 有没有 init。 + +init 失败时先看这三行，不要改业务页面： + +| 日志 | 含义 | 改哪里 | +|---|---|---| +| 返回码 `18400` 或 `18105` | 工作目录不可写 | `workDir` 改到应用私有目录 | +| 配置为空、init 没被调用 | 没拿到 v3 密钥 | 测试后台、白名单 `user_getappconfig_v3`、客户端是否连的 ym-dev | +| 只有一路有识别 | 第二路回调被释放，或两路同时 start | 回调改强引用；A 起来约 1.2 秒再起 B | + +--- + +## 4. 用例 + +每一条都要在华为和 iPhone 上各做一次。结果记在第 6 节的表里。没写「通过」的不要进入下一条的代码修改，先把失败的这一条收掉。 + +耳机相关的条目前提是 Echo-one 已连接。日志里要能看到对应的麦，而不是手机麦。 + +### 4.1 插件冒烟（不进业务页也能看日志） + +| 编号 | 做什么 | 通过 | +|---|---|---| +| S1 | 冷启动 | 日志有 init 成功，节点是新加坡（欧洲账号未加白时，人在欧洲也是新加坡）。无私钥 | +| S2 | 中文说一句短句 | 听写出中文 | +| S3 | 文本「你好」译成英文 | 安卓走 `GTRANS`。iOS 四个翻译类型里，把能返回英文的那个类型写进第 6 节 | +| S4 | 文本「你好」译成爱尔兰语 | 译文是爱尔兰语，语种码 `ga`。iOS 实际用的 `TransType` 写进第 6 节 | +| S5 | 把 S4 的爱尔兰语再合成 | 安卓出声，音色是 `ga-IE-OrlaNeural`。iOS 不出声就停在这一条，不要装进同声翻译冒充已支持 | + +S3 到 S5 没过，不要测同声和通话。后面全靠这三步。 + +### 4.2 同声翻译 + +| 编号 | 做什么 | 通过 | 失败先看 | +|---|---|---|---| +| M1 | 连耳机，中文→英文 | 日志是「同声翻译收音：耳机麦」，有首帧；译文从耳机出 | 麦走错了看 `_attachSimultaneousHeadsetMic`。声音在手机扬声器看 `pinSpeaker` 是不是被改成 true，以及 SDK 有没有自己开麦 | +| M2 | 不连耳机，中文→英文 | 手机麦能出字、出声 | SDK 自己开麦会和现有采集抢，改成只 `write` 现有 PCM | +| M3 | M1 进行中按 Home 切后台，继续说 | 还能出字 | 前台服务 `keepalive: true, pinSpeaker: false`，iOS `BackgroundKeepalive('simultaneous')` | +| M4 | 连耳机，中文→爱尔兰语 | 文字和声音都是爱尔兰语，不走 `AST` | 安卓：识别 `ga-IE`，翻译 `GTRANS`/`ga`，合成 `ga-IE-OrlaNeural`。iOS 用 S4/S5 记下来的参数 | + +### 4.3 面对面 + +| 编号 | 做什么 | 通过 | +|---|---|---| +| F1 | 双耳戴上，左中文、右英文 | 方向相反，两边都有声音。协议帧没有改 | +| F2 | 右耳改说爱尔兰语 | 本端有中文译文。iOS 若 S5 没过，这条只要求出字，出声标成「等音色」 | +| F3 | 摘掉耳机再开 | 手机麦仍能翻，不要退回成失败 | +| F4 | 翻译中来电 | 现有文案还在，会话停 | + +安卓面对面不要顺手加上同声那种静音保活。 + +### 4.4 音视频 + +| 编号 | 做什么 | 通过 | +|---|---|---| +| V1 | 安卓：允许悬浮窗和录屏，切到别的 App 播一段中文 | 悬浮窗有英文译文，原视频声音还在，没有被译文盖住 | +| V2 | 安卓：拒绝悬浮窗 | 不开始识别 | +| V3 | iPhone：开始广播，切后台 | 小窗有字，原视频声音还在 | +| V4 | 从音视频切到同声 | 小窗关掉 | + +### 4.5 通话翻译 + +不退出通话翻译页，按顺序做。A 是近端（自己），B 是远端（对方）。 + +| 编号 | 做什么 | 通过 | 失败先看 | +|---|---|---|---| +| C1 | 中文↔英文，双方各说一句 | 两边都有字，声音在耳机里 | `pinSpeaker: false`。PCM 走 `callPcmTap` / `com.eaimar.bes.callpcm.uplink`，不要接成面对面那条 | +| C2 | 停掉，不退出页面，再开一场中英 | 仍有识别和声音 | `ready` 被 `disable` 清掉了。只在会话销毁时清 | +| C3 | 中文→爱尔兰语。自己说中文 | 自己这边有爱尔兰语文字，对方耳机里听到爱尔兰语 | A 路合成。iOS 依赖 S5 | +| C4 | 接着对方说爱尔兰语 | 自己这边有爱尔兰语原文、中文译文、耳机里的中文。不要做中文/爱尔兰语自动检测 | B 路回调是不是强引用。日志里 B 路有帧但没有识别，就是回调被释放 | + +C1 里如果只有近端有字、远端完全没有，先看 B 路回调和 1.2 秒间隔，不要改语种码。 + +### 4.6 EMAI + +手机里按住说话的那一页不测。 + +| 编号 | 做什么 | 通过 | +|---|---|---| +| E1 | 耳机按键唤醒 | 听到「我在 / 回来了 / 有什么可以帮到你」里的一句，不等用户先说话 | +| E2 | 问一句 | 有文字和声音。说完再答，不要按百炼那种边听边说来判失败 | +| E3 | 「明天下午三点提醒我开会」 | 能进待办。进不去就只记「工具未通」，不要改待办解析 | +| E4 | 对话中切后台 | 不断开 | +| E5 | 没在播音乐时唤醒再结束 | 不要自己把音乐播出来 | +| E6 | 有音乐在播时唤醒 | 音乐变小或暂停；结束后恢复。香薰也做这一条 | +| E7 | 密钥配错或 init 失败时按唤醒 | 设备退出聆听，不空等 | + +声音发飘、像慢放或快放：合成没有按 16 kHz 送进喇叭，百炼的 24000 被套过来了。 + +### 4.7 文本页、图片、纪要 + +| 编号 | 做什么 | 通过 | +|---|---|---| +| T1 | 文本页中英、中文→爱尔兰语 | 出字。断网或端上失败时，请求打到测试服 `user_translate`，不是微软直连 | +| P1 | 图片翻译一张带字的图 | 有原文和译文。没有把译文贴回原图，这是现状 | +| R1 | 同声或通话翻完进语音纪要 | 有音频、端上那份转写、总结。服务端没有把这段再转写一遍 | +| R2 | 纪要首页新录一条中文、一条英文 | 测试服日志里的转写服务是 `asrfile_iflytek`，总结是 `llm_iflytek`。失败退算力只退一次 | + +R2 依赖测试服已经按落地文档第 6 节发过 home，并且会议配置里讯飞的优先级高于旧服务。正式服不会有这条数据。 + +--- + +## 5. 一轮失败了怎么改 + +一次只改失败表里指出的那一处，然后两台都重装，重测失败的那一条，再加一条已经通过的相邻用例，防止改坏。 + +| 现象 | 只改 | +|---|---| +| 安卓有、iOS 没有，或相反 | 该端原生枚举。不要改 Dart 的 `zh` / `en` / `ga`，也不要把另一端的枚举抄过来 | +| 有识别文字、没有声音 | 合成回调有没有把 PCM 交给现有播放器或 `CallTranslationDownlink`。不要在插件里新开 `AudioTrack` | +| 声音从耳机跑到手机喇叭 | `pinSpeaker` 和 SDK 是否自己开了麦、改了音频会话 | +| 同页第二场通话没声音 | `ready` | +| 切后台没反应 | 保活调用点。不要改音乐焦点那一套 | +| 没播的音乐被拉起来 | 设备对话的焦点。没在播就不要抢焦点、不要发播放键 | +| 爱尔兰语走成了中英同传 | 选路把 `ga` 送进了 `AST`。`ga` 强制三段 | +| 纪要仍是字节/阿里/微软 | 测试服会议配置的优先级，或 home 镜像还是旧的。不要去正式服改 | + +iOS 爱尔兰语翻译类型、识别 region、音色名，一旦真机确定，写回落地文档第 4.5 节，把「先试四个类型」改成确定值。没确定之前不要在文档里写成已经支持。 + +--- + +## 6. 记录 + +每轮测完填一行。构建号和日志对不上的结果不算。 + +| 日期 | 构建号 | 端 | 用例 | 结果 | 日志里的一句 | 下一轮只改 | +|---|---|---|---|---|---|---| +| | | 华为 / iPhone | | 过 / 不过 | | | + +iOS 爱尔兰语定下来之后补这一行，后面的用例都用它： + +| 项 | 真机确定的值 | +|---|---| +| 翻译 `TransType` | | +| 识别 region 和语种码 | | +| 合成 region 和音色 | | +| 是否出声 | |