Rodger-Wang
9 hours ago
63 changed files with 6265 additions and 1051 deletions
@ -0,0 +1,392 @@ |
|||
import 'dart:async'; |
|||
import 'dart:convert'; |
|||
import 'dart:typed_data'; |
|||
|
|||
import 'package:get/get.dart'; |
|||
import 'package:get_storage/get_storage.dart'; |
|||
import 'package:http/http.dart' as http; |
|||
|
|||
import '../../core/utils/logger.dart'; |
|||
import '../models/appconfig.dart'; |
|||
import '../models/assistant_directive.dart'; |
|||
import 'bailian_multimodal_service.dart'; |
|||
import 'device_location_service.dart'; |
|||
import 'emai_iflytek_turn.dart'; |
|||
import 'language_manager.dart'; |
|||
import 'mcp_token_service.dart'; |
|||
import 'sparkchain_session.dart'; |
|||
|
|||
/// 耳机唤醒和按住说话共用的一轮编排：讯飞听写 → 文本对话 → 需要时手机调工具 → 合成。 |
|||
/// |
|||
/// 麦和喇叭不在这里。调用方把 PCM 推进来，把 [audioOut] 播出去。 |
|||
class EmaiIflytekDialog { |
|||
EmaiIflytekDialog({this.source = 'emai'}); |
|||
|
|||
static const _tag = 'EmaiIflytek'; |
|||
static const upstreamSampleRate = 16000; |
|||
static const downstreamSampleRate = 16000; |
|||
|
|||
final String source; |
|||
final state = BailianDialogState.idle.obs; |
|||
final _events = StreamController<BailianEvent>.broadcast(); |
|||
final _audioOut = StreamController<Uint8List>.broadcast(); |
|||
final List<EmaiMemory> _history = []; |
|||
|
|||
Stream<BailianEvent> get events => _events.stream; |
|||
Stream<Uint8List> get audioOut => _audioOut.stream; |
|||
|
|||
bool _listening = false; |
|||
int _gen = 0; |
|||
bool _inSpeech = false; |
|||
|
|||
Future<bool> start() async { |
|||
if (!await SparkChainSession.ensure()) return false; |
|||
SparkChainSession.asrListener = _onAsr; |
|||
final ok = await SparkChainSession.startMsftListen(_asrCode()); |
|||
if (!ok) { |
|||
SparkChainSession.asrListener = null; |
|||
return false; |
|||
} |
|||
_listening = true; |
|||
state.value = BailianDialogState.listening; |
|||
_events.add(const BailianEvent(BailianEventType.started)); |
|||
_events.add(BailianEvent( |
|||
BailianEventType.stateChanged, |
|||
state: BailianDialogState.listening, |
|||
)); |
|||
return true; |
|||
} |
|||
|
|||
void pushAudio(Uint8List pcm) { |
|||
if (!_listening || pcm.isEmpty) return; |
|||
SparkChainSession.write(pcm); |
|||
} |
|||
|
|||
Future<void> stop() async { |
|||
_gen++; |
|||
_listening = false; |
|||
SparkChainSession.asrListener = null; |
|||
await SparkChainSession.stopListen(); |
|||
await SparkChainSession.stopSpeak(); |
|||
state.value = BailianDialogState.idle; |
|||
_events.add(const BailianEvent(BailianEventType.closed)); |
|||
} |
|||
|
|||
/// 问候语。不进对话模型，直接合成。 |
|||
Future<void> speak(String text) async { |
|||
final gen = _gen; |
|||
await _say(text, gen); |
|||
} |
|||
|
|||
void _onAsr(Map<String, dynamic> event) { |
|||
if (event['kind'] == 'error') { |
|||
Logger.w(_tag, '听写失败 code=${event['code']}'); |
|||
return; |
|||
} |
|||
final text = (event['text'] as String?)?.trim() ?? ''; |
|||
final isFinal = event['final'] == true; |
|||
if (text.isEmpty) return; |
|||
if (!_inSpeech) { |
|||
_inSpeech = true; |
|||
_events.add(const BailianEvent(BailianEventType.speechStarted)); |
|||
} |
|||
_events.add(BailianEvent( |
|||
BailianEventType.speechContent, |
|||
text: text, |
|||
isFinal: isFinal, |
|||
)); |
|||
if (!isFinal) return; |
|||
_inSpeech = false; |
|||
_events.add(BailianEvent(BailianEventType.speechEnded, text: text)); |
|||
unawaited(_answer(text)); |
|||
} |
|||
|
|||
Future<void> _answer(String userText) async { |
|||
final gen = ++_gen; |
|||
await SparkChainSession.stopSpeak(); |
|||
if (EmaiTurn.asksWhere(userText)) { |
|||
await DeviceLocationService.maybe?.ensureSpokenPlace(); |
|||
} |
|||
if (gen != _gen) return; |
|||
state.value = BailianDialogState.thinking; |
|||
_events.add(BailianEvent( |
|||
BailianEventType.stateChanged, |
|||
state: BailianDialogState.thinking, |
|||
)); |
|||
final chinese = _asrCode().toLowerCase().startsWith('zh'); |
|||
final overseas = _overseas(); |
|||
final tools = EmaiTurn.toolsJson(overseas: overseas); |
|||
var query = userText; |
|||
var fresh = true; |
|||
String spoken = ''; |
|||
for (var round = 0; round < 2; round++) { |
|||
if (gen != _gen) return; |
|||
final SparkLlmReply reply; |
|||
try { |
|||
reply = await SparkChainSession.chat( |
|||
query: query, |
|||
system: EmaiTurn.systemPrompt( |
|||
now: DateTime.now(), |
|||
spokenPlace: DeviceLocationService.maybe?.placeText(), |
|||
answerLanguage: chinese ? '中文' : 'English', |
|||
history: List<EmaiMemory>.from(_history), |
|||
), |
|||
tools: tools, |
|||
fresh: fresh, |
|||
); |
|||
} catch (e) { |
|||
Logger.w(_tag, '对话失败: $e'); |
|||
await _say(EmaiTurn.unavailable(chinese), gen); |
|||
return; |
|||
} |
|||
if (gen != _gen) return; |
|||
final calls = EmaiTurn.parseCalls(reply.functionCall); |
|||
if (calls.isEmpty) { |
|||
spoken = reply.text.trim(); |
|||
break; |
|||
} |
|||
final call = calls.first; |
|||
if (call.name == 'quit') { |
|||
_events.add(BailianEvent( |
|||
BailianEventType.respondingContent, |
|||
text: reply.text, |
|||
directives: [ |
|||
AssistantDirective( |
|||
id: 'quit-$gen', |
|||
name: 'quit', |
|||
args: const {}, |
|||
rawArguments: '{}', |
|||
dialogId: '', |
|||
roundId: '$gen', |
|||
llmRequestId: '', |
|||
source: source, |
|||
createdAt: DateTime.now(), |
|||
), |
|||
], |
|||
)); |
|||
final bye = reply.text.trim().isEmpty |
|||
? (chinese ? '好的' : 'Okay') |
|||
: reply.text.trim(); |
|||
await _say(bye, gen); |
|||
return; |
|||
} |
|||
final result = await _runTool(call); |
|||
if (gen != _gen) return; |
|||
if (result == null) { |
|||
await _say(EmaiTurn.unavailable(chinese), gen); |
|||
return; |
|||
} |
|||
fresh = false; |
|||
query = '工具 ${call.name} 的结果如下，只根据它回答，不要编造：\n$result'; |
|||
} |
|||
if (spoken.isEmpty) { |
|||
await _say(EmaiTurn.unavailable(chinese), gen); |
|||
return; |
|||
} |
|||
_history.add(EmaiMemory(true, userText)); |
|||
_history.add(EmaiMemory(false, spoken)); |
|||
if (_history.length > 8) { |
|||
_history.removeRange(0, _history.length - 8); |
|||
} |
|||
await _say(spoken, gen); |
|||
} |
|||
|
|||
Future<void> _say(String text, int gen) async { |
|||
if (text.isEmpty || gen != _gen) return; |
|||
state.value = BailianDialogState.responding; |
|||
_events.add(const BailianEvent(BailianEventType.respondingStarted)); |
|||
_events.add(BailianEvent( |
|||
BailianEventType.respondingContent, |
|||
text: text, |
|||
isFinal: true, |
|||
finishReason: 'stop', |
|||
)); |
|||
try { |
|||
await SparkChainSession.speakNeural( |
|||
voice: _voice(), |
|||
text: text, |
|||
onPcm: (pcm, _) { |
|||
if (gen != _gen || pcm.isEmpty) return; |
|||
_audioOut.add(pcm); |
|||
}, |
|||
); |
|||
} catch (e) { |
|||
Logger.w(_tag, '合成失败: $e'); |
|||
} |
|||
if (gen != _gen) return; |
|||
_events.add(const BailianEvent(BailianEventType.respondingEnded)); |
|||
state.value = BailianDialogState.listening; |
|||
_events.add(BailianEvent( |
|||
BailianEventType.stateChanged, |
|||
state: BailianDialogState.listening, |
|||
)); |
|||
} |
|||
|
|||
Future<String?> _runTool(EmaiToolCall call) async { |
|||
final now = DateTime.now(); |
|||
final args = Map<String, dynamic>.from(call.args); |
|||
if (call.name == 'hefeng_weather') { |
|||
final city = (args['city'] ?? '').toString().trim(); |
|||
if (city.isEmpty) { |
|||
final cached = _city(); |
|||
if (cached.isEmpty) return '没有位置，请向用户询问要查哪个城市，不要编造天气。'; |
|||
args['city'] = cached; |
|||
} |
|||
if ((args['time'] ?? '').toString().trim().isEmpty) { |
|||
args['time'] = EmaiTurn.dayText(now); |
|||
} |
|||
} |
|||
if (call.name == 'tavily_web_search' || call.name == 'bocha_web_search') { |
|||
args['query'] = EmaiTurn.searchQuery( |
|||
(args['query'] ?? '').toString(), |
|||
now, |
|||
); |
|||
} |
|||
final authed = EmaiTurn.authTools.contains(call.name); |
|||
String? token; |
|||
if (authed) { |
|||
token = await McpTokenService.get(); |
|||
if (token == null || token.isEmpty) return null; |
|||
} |
|||
final url = _mcpUrl(call.name); |
|||
if (url == null) return null; |
|||
try { |
|||
final resp = await http |
|||
.post( |
|||
url, |
|||
headers: { |
|||
'Content-Type': 'application/json', |
|||
'Accept': 'application/json, text/event-stream', |
|||
if (token != null) 'Authorization': 'Bearer $token', |
|||
}, |
|||
body: jsonEncode({ |
|||
'jsonrpc': '2.0', |
|||
'id': 1, |
|||
'method': 'tools/call', |
|||
'params': {'name': call.name, 'arguments': args}, |
|||
}), |
|||
) |
|||
.timeout(const Duration(seconds: 12)); |
|||
if (resp.statusCode < 200 || resp.statusCode >= 300) return null; |
|||
return _toolText(resp.body); |
|||
} catch (e) { |
|||
Logger.w(_tag, '工具 ${call.name} 失败'); |
|||
return null; |
|||
} |
|||
} |
|||
|
|||
String? _toolText(String body) { |
|||
final payloads = <String>[]; |
|||
for (final line in body.split('\n')) { |
|||
final trimmed = line.trim(); |
|||
if (trimmed.startsWith('data:')) { |
|||
payloads.add(trimmed.substring(5).trim()); |
|||
} |
|||
} |
|||
if (payloads.isEmpty) payloads.add(body); |
|||
for (final payload in payloads.reversed) { |
|||
if (payload.isEmpty) continue; |
|||
try { |
|||
final decoded = jsonDecode(payload); |
|||
if (decoded is! Map) continue; |
|||
if (decoded['error'] != null) return null; |
|||
final result = decoded['result']; |
|||
if (result is! Map) continue; |
|||
if (result['isError'] == true) return null; |
|||
final content = result['content']; |
|||
if (content is List) { |
|||
final texts = content |
|||
.whereType<Map>() |
|||
.map((m) => (m['text'] ?? '').toString()) |
|||
.where((s) => s.isNotEmpty) |
|||
.join('\n'); |
|||
if (texts.isNotEmpty) return texts; |
|||
} |
|||
final text = result['text']; |
|||
if (text is String && text.isNotEmpty) return text; |
|||
} catch (_) {} |
|||
} |
|||
return null; |
|||
} |
|||
|
|||
Uri? _mcpUrl(String tool) { |
|||
if (!AppConfig.isInitialized()) return null; |
|||
final raw = AppConfig.mcpConfig(); |
|||
final servers = raw is Map ? raw['mcpServers'] : null; |
|||
if (servers is! Map) return null; |
|||
String? fallback; |
|||
for (final value in servers.values) { |
|||
if (value is! Map && value.runtimeType.toString() != 'DBMCPServer') { |
|||
// generated model |
|||
} |
|||
final url = _serverUrl(value); |
|||
final tools = _serverTools(value); |
|||
if (url == null) continue; |
|||
fallback ??= url; |
|||
if (tools.contains(tool)) return Uri.parse(url); |
|||
} |
|||
if (fallback == null) return null; |
|||
return Uri.parse(fallback); |
|||
} |
|||
|
|||
String? _serverUrl(Object value) { |
|||
try { |
|||
final url = (value as dynamic).url?.toString().trim() ?? ''; |
|||
if (url.isEmpty) return null; |
|||
if (url.contains('/mcp')) return url; |
|||
return url.endsWith('/') ? '${url}mcp' : '$url/mcp'; |
|||
} catch (_) { |
|||
return null; |
|||
} |
|||
} |
|||
|
|||
String _serverTools(Object value) { |
|||
try { |
|||
return ((value as dynamic).tools ?? '').toString(); |
|||
} catch (_) { |
|||
return ''; |
|||
} |
|||
} |
|||
|
|||
String _city() { |
|||
final place = DeviceLocationService.maybe; |
|||
if (place == null) return ''; |
|||
final loc = place.clientLocation(); |
|||
return (loc?['city_name'] ?? '').trim(); |
|||
} |
|||
|
|||
bool _overseas() { |
|||
final raw = AppConfig.cred('sparkchain', 'api_type', 'SPARKCHAIN_API_TYPE'); |
|||
final apiType = int.tryParse(raw ?? '') ?? 1; |
|||
return apiType != 0; |
|||
} |
|||
|
|||
String _asrCode() { |
|||
final saved = GetStorage().read<String>('language') ?? 'system'; |
|||
if (saved != 'system') { |
|||
final parts = saved.split('_'); |
|||
if (parts.length == 2) { |
|||
final lang = parts[0].toLowerCase(); |
|||
final country = parts[1].toUpperCase(); |
|||
if (lang == 'tl' && country == 'PH') return 'fil-PH'; |
|||
return '$lang-$country'; |
|||
} |
|||
} |
|||
final locale = Get.locale ?? Get.deviceLocale; |
|||
if (locale != null && (locale.countryCode ?? '').isNotEmpty) { |
|||
return '${locale.languageCode.toLowerCase()}-${locale.countryCode!.toUpperCase()}'; |
|||
} |
|||
return 'en-US'; |
|||
} |
|||
|
|||
String _voice() { |
|||
final code = _asrCode(); |
|||
if (Get.isRegistered<LanguageManager>()) { |
|||
final voice = Get.find<LanguageManager>().getTtsVoiceNameByAsrCode(code); |
|||
if (voice != null && voice.isNotEmpty) return voice; |
|||
} |
|||
return code.toLowerCase().startsWith('zh') |
|||
? 'zh-CN-XiaoxiaoNeural' |
|||
: 'en-US-AriaNeural'; |
|||
} |
|||
} |
|||
@ -0,0 +1,190 @@ |
|||
import 'dart:convert'; |
|||
|
|||
/// 一轮 EMAI 对话里和 SDK 无关的部分：系统提示、工具清单、工具参数。 |
|||
class EmaiTurn { |
|||
EmaiTurn._(); |
|||
|
|||
static const authTools = [ |
|||
'search_meeting_notes', |
|||
'get_memory_items', |
|||
'get_memory_stats', |
|||
'get_memory_report', |
|||
'get_user_tasks', |
|||
'add_user_task', |
|||
'cancel_user_task', |
|||
]; |
|||
|
|||
static String clockText(DateTime t) { |
|||
const week = ['星期一', '星期二', '星期三', '星期四', '星期五', '星期六', '星期日']; |
|||
String two(int v) => v.toString().padLeft(2, '0'); |
|||
final offset = t.timeZoneOffset; |
|||
final sign = offset.isNegative ? '-' : '+'; |
|||
final hours = offset.inHours.abs(); |
|||
final mins = offset.inMinutes.abs() % 60; |
|||
final zone = mins == 0 ? 'UTC$sign$hours' : 'UTC$sign$hours:${two(mins)}'; |
|||
return '${t.year}年${t.month}月${t.day}日 ${week[t.weekday - 1]} ' |
|||
'${two(t.hour)}:${two(t.minute)}，$zone'; |
|||
} |
|||
|
|||
static String dayText(DateTime t) { |
|||
String two(int v) => v.toString().padLeft(2, '0'); |
|||
return '${t.year}-${two(t.month)}-${two(t.day)}'; |
|||
} |
|||
|
|||
/// 没有能念的地址时用这句。不要写「未知」，模型会把它当成地名。 |
|||
static String placeLine(String? spoken) { |
|||
final text = (spoken ?? '').trim(); |
|||
if (text.isEmpty || text == '未知') return '当前没有位置'; |
|||
return text; |
|||
} |
|||
|
|||
static bool asksWhere(String text) { |
|||
final s = text.toLowerCase(); |
|||
return s.contains('在哪') || |
|||
s.contains('我的位置') || |
|||
s.contains('where am i') || |
|||
s.contains('where i am'); |
|||
} |
|||
|
|||
static String unavailable(bool chinese) => |
|||
chinese ? '暂时查不到' : "I can't look that up right now."; |
|||
|
|||
static String systemPrompt({ |
|||
required DateTime now, |
|||
required String? spokenPlace, |
|||
required String answerLanguage, |
|||
required List<EmaiMemory> history, |
|||
}) { |
|||
final lines = history |
|||
.map((m) => '${m.user ? "用户" : "助手"}：${m.text}') |
|||
.join('\n'); |
|||
return ''' |
|||
你是 EAIMAR 的语音助手。用$answerLanguage回答，句子短，适合念出来。 |
|||
当前时间：${clockText(now)} |
|||
当前位置：${placeLine(spokenPlace)} |
|||
时间、星期、日期、地点只用上面这两行，不要调用工具，也不要改口。 |
|||
没有位置时说还没有位置，不要编一个城市，不要念经纬度。 |
|||
天气用 hefeng_weather，城市用上面的城市名，日期用今天。有城市就不要反问哪个城市。没有位置时才问一次城市。 |
|||
新闻和需要联网的问题用搜索工具，搜索词里带上今天的日期 ${dayText(now)}。 |
|||
会议、记忆、待办用对应的个人工具。没有就说没有，不要编。 |
|||
用户要结束对话时调用 quit。 |
|||
${lines.isEmpty ? '' : '最近对话：\n$lines'} |
|||
'''.trim(); |
|||
} |
|||
|
|||
static String toolsJson({required bool overseas}) { |
|||
final search = overseas ? 'tavily_web_search' : 'bocha_web_search'; |
|||
final tools = [ |
|||
_fn('hefeng_weather', '按城市名查询天气', { |
|||
'city': _str('城市名，例如北京'), |
|||
'time': _str('日期 yyyy-MM-dd，可省略'), |
|||
}, const ['city']), |
|||
_fn(search, '联网搜索新闻和实时信息', { |
|||
'query': _str('搜索词，要带上日期'), |
|||
'count': {'type': 'integer', 'description': '条数，1到5'}, |
|||
}, const ['query']), |
|||
_fn('search_meeting_notes', '查用户自己的会议纪要', { |
|||
'keywords': _str('关键词，可空'), |
|||
}, const []), |
|||
_fn('get_memory_items', '查用户记下的事项', { |
|||
'category': _str('todo、alarm、idea、expense，可空'), |
|||
}, const []), |
|||
_fn('get_memory_stats', '查记忆的数量统计', {}, const []), |
|||
_fn('get_memory_report', '查记忆的汇总', {}, const []), |
|||
_fn('get_user_tasks', '查提醒和待办', {}, const []), |
|||
_fn('add_user_task', '添加一条提醒', { |
|||
'task_name': _str('标题'), |
|||
'trigger_time': _str('带时区的 ISO 8601 时间'), |
|||
}, const ['task_name', 'trigger_time']), |
|||
_fn('cancel_user_task', '取消一条提醒', { |
|||
'task_id': {'type': 'integer', 'description': '任务 id'}, |
|||
}, const ['task_id']), |
|||
_fn('quit', '用户要结束这次对话', {}, const []), |
|||
]; |
|||
return jsonEncode(tools); |
|||
} |
|||
|
|||
static Map<String, dynamic> _str(String description) => { |
|||
'type': 'string', |
|||
'description': description, |
|||
}; |
|||
|
|||
static Map<String, dynamic> _fn( |
|||
String name, |
|||
String description, |
|||
Map<String, dynamic> properties, |
|||
List<String> required, |
|||
) { |
|||
return { |
|||
'type': 'function', |
|||
'function': { |
|||
'name': name, |
|||
'description': description, |
|||
'parameters': { |
|||
'type': 'object', |
|||
'properties': properties, |
|||
'required': required, |
|||
}, |
|||
}, |
|||
}; |
|||
} |
|||
|
|||
/// SDK 的 functionCall 可能是对象、对象数组，或 arguments 再包一层字符串。 |
|||
static List<EmaiToolCall> parseCalls(String raw) { |
|||
final text = raw.trim(); |
|||
if (text.isEmpty) return const []; |
|||
try { |
|||
final decoded = jsonDecode(text); |
|||
if (decoded is List) { |
|||
return decoded.map(_one).whereType<EmaiToolCall>().toList(); |
|||
} |
|||
if (decoded is Map) { |
|||
final one = _one(decoded); |
|||
return one == null ? const [] : [one]; |
|||
} |
|||
} catch (_) {} |
|||
return const []; |
|||
} |
|||
|
|||
static EmaiToolCall? _one(Object? value) { |
|||
if (value is! Map) return null; |
|||
final map = value.map((k, v) => MapEntry('$k', v)); |
|||
final fn = map['function']; |
|||
final nested = fn is Map ? fn.map((k, v) => MapEntry('$k', v)) : map; |
|||
final name = (nested['name'] ?? map['name'] ?? '').toString().trim(); |
|||
if (name.isEmpty) return null; |
|||
final args = _args(nested['arguments'] ?? map['arguments']); |
|||
args.remove('auth_token'); |
|||
args.remove('uid'); |
|||
return EmaiToolCall(name, args); |
|||
} |
|||
|
|||
static Map<String, dynamic> _args(Object? value) { |
|||
if (value is Map) return value.map((k, v) => MapEntry('$k', v)); |
|||
if (value is String && value.trim().isNotEmpty) { |
|||
try { |
|||
final decoded = jsonDecode(value); |
|||
if (decoded is Map) return decoded.map((k, v) => MapEntry('$k', v)); |
|||
} catch (_) {} |
|||
} |
|||
return {}; |
|||
} |
|||
|
|||
static String searchQuery(String query, DateTime now) { |
|||
final day = dayText(now); |
|||
if (query.contains(day)) return query; |
|||
return '$query $day'.trim(); |
|||
} |
|||
} |
|||
|
|||
class EmaiMemory { |
|||
EmaiMemory(this.user, this.text); |
|||
final bool user; |
|||
final String text; |
|||
} |
|||
|
|||
class EmaiToolCall { |
|||
EmaiToolCall(this.name, this.args); |
|||
final String name; |
|||
final Map<String, dynamic> args; |
|||
} |
|||
@ -0,0 +1,169 @@ |
|||
/// 同声翻译选哪条讯飞微软会话。 |
|||
/// |
|||
/// 两端都在同传表里：一条 AST，发音人用短名。 |
|||
/// 源或目标是中文、粤语、爱尔兰语：不启 AST，识别走微软实时转写，合成用区域码音色。 |
|||
/// 两边都不在这两类里：不启动。 |
|||
class SimulRoute { |
|||
const SimulRoute.ast({ |
|||
required this.lang, |
|||
required this.targetLang, |
|||
required this.speaker, |
|||
}) : ast = true, |
|||
languageName = ''; |
|||
|
|||
const SimulRoute.splice({ |
|||
required this.languageName, |
|||
required this.speaker, |
|||
}) : ast = false, |
|||
lang = '', |
|||
targetLang = ''; |
|||
|
|||
final bool ast; |
|||
/// AST 的源语种短码，例如 ja。 |
|||
final String lang; |
|||
/// AST 的目标语种短码，例如 en。 |
|||
final String targetLang; |
|||
/// AST 用短名（JennyNeural）。拼接用区域码音色（en-US-JennyNeural）。 |
|||
final String speaker; |
|||
/// 拼接识别的 language_name，例如 zh-CN。 |
|||
final String languageName; |
|||
} |
|||
|
|||
class SimulMicrosoft { |
|||
static SimulRoute? plan(String source, String target) { |
|||
final from = _norm(source); |
|||
final to = _norm(target); |
|||
if (_ast.containsKey(from) && _ast.containsKey(to)) { |
|||
return SimulRoute.ast( |
|||
lang: from, |
|||
targetLang: to, |
|||
speaker: _ast[to]!.speaker, |
|||
); |
|||
} |
|||
if (_splice.containsKey(from) || _splice.containsKey(to)) { |
|||
final locale = _listenLocale(from); |
|||
final voice = _speakVoice(to); |
|||
if (locale == null || voice == null) return null; |
|||
return SimulRoute.splice(languageName: locale, speaker: voice); |
|||
} |
|||
return null; |
|||
} |
|||
|
|||
/// 同传能力 ast_msft 没授权时，用同一对语种走微软转写加合成。 |
|||
/// 识别用区域码，合成用完整神经音色，不再用 AST 的短名。 |
|||
static SimulRoute spliceInstead(SimulRoute route) { |
|||
final locale = _listenLocale(route.lang); |
|||
final voice = _speakVoice(route.targetLang); |
|||
if (locale == null || voice == null) return route; |
|||
return SimulRoute.splice(languageName: locale, speaker: voice); |
|||
} |
|||
|
|||
/// 界面短码收成表里的键。香港繁体要在「凡是 zh- 都当普通话」之前认出来。 |
|||
static String _norm(String code) { |
|||
final raw = code.trim().toLowerCase().replaceAll('_', '-'); |
|||
if (raw == 'yue' || raw.startsWith('yue-')) return 'yue'; |
|||
if (raw == 'ga' || raw.startsWith('ga-')) return 'ga'; |
|||
if (raw == 'zh-hant-hk' || raw == 'zh-hk') return 'zh-HK'; |
|||
if (raw == 'zh-hant' || raw == 'zh-tw' || raw.startsWith('zh-hant')) { |
|||
return 'zh-Hant'; |
|||
} |
|||
if (raw == 'zh' || raw.startsWith('zh-')) return 'zh'; |
|||
final dash = raw.indexOf('-'); |
|||
return dash < 0 ? raw : raw.substring(0, dash); |
|||
} |
|||
|
|||
/// 讯飞 SDK 文本翻译（gtrans）的语种码。中文用 zh-CN，不要收成 zh。 |
|||
static String translateCode(String code) { |
|||
switch (_norm(code)) { |
|||
case 'zh': |
|||
return 'zh-CN'; |
|||
case 'zh-Hant': |
|||
return 'zh-TW'; |
|||
case 'zh-HK': |
|||
return 'zh-HK'; |
|||
case 'yue': |
|||
return 'yue'; |
|||
case 'ga': |
|||
return 'ga'; |
|||
default: |
|||
return _norm(code); |
|||
} |
|||
} |
|||
|
|||
static String? _listenLocale(String key) { |
|||
final splice = _splice[key]; |
|||
if (splice != null) return splice.locale; |
|||
return _ast[key]?.locale; |
|||
} |
|||
|
|||
static String? _speakVoice(String key) { |
|||
final splice = _splice[key]; |
|||
if (splice != null) return splice.voice; |
|||
final ast = _ast[key]; |
|||
if (ast == null) return null; |
|||
return '${ast.locale}-${ast.speaker}'; |
|||
} |
|||
|
|||
/// 同传表。speaker 是 AST 的短名，locale 是拼接到 OnlineTTS 时用的区域码。 |
|||
static const _ast = <String, _AstLang>{ |
|||
'ar': _AstLang('ar-EG', 'AbdullahNeural'), |
|||
'bg': _AstLang('bg-BG', 'BorislavNeural'), |
|||
'bn': _AstLang('bn-IN', 'BashkarNeural'), |
|||
'ca': _AstLang('ca-ES', 'AlbaNeural'), |
|||
'cs': _AstLang('cs-CZ', 'AntoninNeural'), |
|||
'da': _AstLang('da-DK', 'ChristelNeural'), |
|||
'de': _AstLang('de-DE', 'AmalaNeural'), |
|||
'el': _AstLang('el-GR', 'AthinaNeural'), |
|||
'en': _AstLang('en-US', 'JennyNeural'), |
|||
'es': _AstLang('es-ES', 'ElviraNeural'), |
|||
'et': _AstLang('et-EE', 'AnuNeural'), |
|||
'fa': _AstLang('fa-IR', 'DilaraNeural'), |
|||
'fi': _AstLang('fi-FI', 'HarriNeural'), |
|||
'fr': _AstLang('fr-FR', 'DeniseNeural'), |
|||
'hi': _AstLang('hi-IN', 'MadhurNeural'), |
|||
'hu': _AstLang('hu-HU', 'NoemiNeural'), |
|||
'id': _AstLang('id-ID', 'ArdiNeural'), |
|||
'it': _AstLang('it-IT', 'ElsaNeural'), |
|||
'ja': _AstLang('ja-JP', 'NanamiNeural'), |
|||
'ko': _AstLang('ko-KR', 'SunHiNeural'), |
|||
'lt': _AstLang('lt-LT', 'LeonasNeural'), |
|||
'lv': _AstLang('lv-LV', 'EveritaNeural'), |
|||
'ms': _AstLang('ms-MY', 'OsmanNeural'), |
|||
'nl': _AstLang('nl-NL', 'ArnaudNeural'), |
|||
'pl': _AstLang('pl-PL', 'AgnieszkaNeural'), |
|||
'pt': _AstLang('pt-BR', 'AntonioNeural'), |
|||
'ro': _AstLang('ro-RO', 'AlinaNeural'), |
|||
'ru': _AstLang('ru-RU', 'DariyaNeural'), |
|||
'sk': _AstLang('sk-SK', 'LukasNeural'), |
|||
'sl': _AstLang('sl-SI', 'PetraNeural'), |
|||
'sv': _AstLang('sv-SE', 'HilleviNeural'), |
|||
'sw': _AstLang('sw-KE', 'DaudiNeural'), |
|||
'ta': _AstLang('ta-IN', 'AnbuNeural'), |
|||
'th': _AstLang('th-TH', 'AcharaNeural'), |
|||
'tr': _AstLang('tr-TR', 'AhmetNeural'), |
|||
'uk': _AstLang('uk-UA', 'OstapNeural'), |
|||
'ur': _AstLang('ur-IN', 'AsadNeural'), |
|||
'vi': _AstLang('vi-VN', 'HoaiMyNeural'), |
|||
}; |
|||
|
|||
/// 微软同传表没有的三语。繁体中文按普通话识别，合成用繁体音色。 |
|||
static const _splice = <String, _SpliceLang>{ |
|||
'zh': _SpliceLang('zh-CN', 'zh-CN-XiaoxiaoNeural'), |
|||
'zh-Hant': _SpliceLang('zh-CN', 'zh-TW-HsiaoChenNeural'), |
|||
'yue': _SpliceLang('yue-CN', 'yue-CN-XiaoMinNeural'), |
|||
'zh-HK': _SpliceLang('zh-HK', 'zh-HK-HiuMaanNeural'), |
|||
'ga': _SpliceLang('ga-IE', 'ga-IE-OrlaNeural'), |
|||
}; |
|||
} |
|||
|
|||
class _AstLang { |
|||
const _AstLang(this.locale, this.speaker); |
|||
final String locale; |
|||
final String speaker; |
|||
} |
|||
|
|||
class _SpliceLang { |
|||
const _SpliceLang(this.locale, this.voice); |
|||
final String locale; |
|||
final String voice; |
|||
} |
|||
File diff suppressed because it is too large
@ -0,0 +1,50 @@ |
|||
#import "AudioCatch.h" |
|||
|
|||
NSString *AudioEngineStart(AVAudioEngine *engine) { |
|||
@try { |
|||
[engine prepare]; |
|||
NSError *error = nil; |
|||
if (![engine startAndReturnError:&error]) { |
|||
return error.localizedDescription ?: @"麦克风引擎启动失败"; |
|||
} |
|||
return nil; |
|||
} @catch (NSException *e) { |
|||
return e.reason ?: e.name; |
|||
} |
|||
} |
|||
|
|||
NSString *AudioEngineInstallTap(AVAudioEngine *engine, AVAudioFormat *format, void *ctx, PhoneMicTap tap) { |
|||
@try { |
|||
if (format.sampleRate <= 0 || format.channelCount == 0) { |
|||
return @"输入格式未就绪"; |
|||
} |
|||
AVAudioInputNode *input = engine.inputNode; |
|||
[input installTapOnBus:0 bufferSize:4096 format:format block:^(AVAudioPCMBuffer *buffer, AVAudioTime *when) { |
|||
if (tap == NULL || buffer.frameLength == 0) return; |
|||
float *const *channels = buffer.floatChannelData; |
|||
if (channels == NULL || channels[0] == NULL) return; |
|||
tap(ctx, channels[0], (int)buffer.frameLength, buffer.format.sampleRate); |
|||
}]; |
|||
return nil; |
|||
} @catch (NSException *e) { |
|||
return e.reason ?: e.name; |
|||
} |
|||
} |
|||
|
|||
NSString *AudioEngineConnect(AVAudioEngine *engine, AVAudioPlayerNode *node, AVAudioFormat *format) { |
|||
@try { |
|||
if (format.sampleRate <= 0 || format.channelCount == 0) { |
|||
return @"播放格式无效"; |
|||
} |
|||
if (![engine.attachedNodes containsObject:node]) { |
|||
[engine attachNode:node]; |
|||
} |
|||
if ([engine outputConnectionPointsForNode:node outputBus:0].count == 0) { |
|||
[engine connect:node to:engine.mainMixerNode format:format]; |
|||
} |
|||
engine.mainMixerNode.outputVolume = 1; |
|||
return nil; |
|||
} @catch (NSException *e) { |
|||
return e.reason ?: e.name; |
|||
} |
|||
} |
|||
@ -0,0 +1,12 @@ |
|||
#import <AVFoundation/AVFoundation.h> |
|||
|
|||
typedef void (*PhoneMicTap)(void *_Nonnull ctx, const float *_Nonnull samples, int frames, double sampleRate); |
|||
|
|||
NSString *_Nullable AudioEngineStart(AVAudioEngine *_Nonnull engine); |
|||
NSString *_Nullable AudioEngineInstallTap(AVAudioEngine *_Nonnull engine, |
|||
AVAudioFormat *_Nonnull format, |
|||
void *_Nonnull ctx, |
|||
PhoneMicTap _Nonnull tap); |
|||
NSString *_Nullable AudioEngineConnect(AVAudioEngine *_Nonnull engine, |
|||
AVAudioPlayerNode *_Nonnull node, |
|||
AVAudioFormat *_Nonnull format); |
|||
@ -1,187 +1,249 @@ |
|||
import AudioToolbox |
|||
import AVFoundation |
|||
import Foundation |
|||
|
|||
/// 流式 PCM 外放播放器（16bit 小端、单声道）。 |
|||
/// 流式 PCM 外放（16bit 小端、单声道）。 |
|||
/// |
|||
/// 存在的理由：iOS 上 **just_audio 放不了「长度未知的直播 WAV 流」**。 |
|||
/// 真机实测（iPhone 12 / iOS 26.6）AVPlayer 拿到 just_audio 代理服务给的 |
|||
/// chunked 响应后，只发一次 `Range: bytes=0-1` 的嗅探请求就判定资源不可播， |
|||
/// 立刻回 `-11850 Operation Stopped` 且 processingState 直接落到 idle， |
|||
/// **一次都没真正起播**。声明长度、支持 range、把收到的字节全缓存下来允许 |
|||
/// 重放，三种都试过，结论一样。ExoPlayer 不发这个嗅探请求，所以 Android |
|||
/// 那条路一直是好的 —— 这就是「安卓有声、苹果没声」的全部原因。 |
|||
/// 用 AudioQueue，不用 AVAudioEngine。采集也是一条独立的 AudioQueue。 |
|||
/// 把播放节点挂到引擎上，节点没连上就 play()，系统会以 |
|||
/// "player started when in a disconnected state" 直接杀进程 |
|||
/// （2026-10-18 09:13 起每次点开始都是这个）。 |
|||
/// |
|||
/// 这里绕开 AVPlayer + HTTP 代理，直接 AVAudioEngine + AVAudioPlayerNode |
|||
/// 排队播放，和 `AzureTtsHelper` 里那套流式播放同源。 |
|||
/// |
|||
/// ⚠️ **不碰 AVAudioSession**：会话由 Dart 侧的 audio_session 统一配置 |
|||
/// （连续对话时录音的 voice-processing 也在同一个会话上）。这里再去 |
|||
/// setCategory/setActive 就会和录音互相掀桌子。 |
|||
/// 播放队列和麦克风队列各走各的，互不 start / connect。 |
|||
/// 不改 AVAudioSession：会话在开麦前已经配成扬声器。 |
|||
class PcmSpeakerPlayer { |
|||
private let queue = DispatchQueue(label: "azure_speech.pcm_speaker") |
|||
|
|||
private var engine: AVAudioEngine? |
|||
private var node: AVAudioPlayerNode? |
|||
private var format: AVAudioFormat? |
|||
private let lock = NSRecursiveLock() |
|||
|
|||
/// 已排队但还没播完的缓冲数 |
|||
private var pending = 0 |
|||
/// 上层已经调过 end()：pending 归零就算这一轮播完 |
|||
private var output: AudioQueueRef? |
|||
private var owned: [AudioQueueBufferRef] = [] |
|||
private var freeBuffers: [AudioQueueBufferRef] = [] |
|||
private var stash = Data() |
|||
private var rate: Double = 0 |
|||
private var ended = false |
|||
/// 本轮代际，stop() 之后迟到的完成回调按代际丢弃 |
|||
private var inFlight = 0 |
|||
private var generation = 0 |
|||
|
|||
/// 一轮音频播完（或本轮压根没有音频） |
|||
var onCompleted: (() -> Void)? |
|||
|
|||
// MARK: - 对外接口 |
|||
private static let outputCallback: AudioQueueOutputCallback = { userData, _, buffer in |
|||
guard let userData else { return } |
|||
Unmanaged<PcmSpeakerPlayer>.fromOpaque(userData).takeUnretainedValue().bufferReturned(buffer) |
|||
} |
|||
|
|||
func start(sampleRate: Double) -> Bool { |
|||
var ok = false |
|||
queue.sync { |
|||
generation += 1 |
|||
pending = 0 |
|||
ended = false |
|||
ok = ensureEngine(sampleRate: sampleRate) |
|||
func start(sampleRate: Double, pinSpeaker: Bool = false) -> Bool { |
|||
lock.lock() |
|||
generation += 1 |
|||
ended = false |
|||
let ok = ensureQueue(sampleRate: sampleRate) |
|||
lock.unlock() |
|||
// 只有明确要求钉扬声器时才改路由。同声翻译不钉： |
|||
// 开着麦再改路由，采集会断，第一句话就没了。 |
|||
if ok && pinSpeaker { |
|||
try? AVAudioSession.sharedInstance().overrideOutputAudioPort(.speaker) |
|||
} |
|||
return ok |
|||
} |
|||
|
|||
func feed(_ data: Data) { |
|||
queue.async { [weak self] in |
|||
guard let self = self else { return } |
|||
guard let node = self.node, let engine = self.engine, engine.isRunning else { return } |
|||
guard let buffer = self.makeBuffer(from: data) else { return } |
|||
let gen = self.generation |
|||
self.pending += 1 |
|||
// ⚠️ 必须用 .dataPlayedBack 而不是默认的 .dataRendered： |
|||
// 播完这一轮要顺手停引擎（见 finishIfDone），用 dataRendered 会在 |
|||
// 声音真正出完之前就回调，尾音被切掉。 |
|||
node.scheduleBuffer(buffer, completionCallbackType: .dataPlayedBack) { [weak self] _ in |
|||
self?.queue.async { |
|||
guard let self = self, gen == self.generation else { return } |
|||
self.pending -= 1 |
|||
self.finishIfDone() |
|||
} |
|||
} |
|||
if !node.isPlaying { node.play() } |
|||
} |
|||
guard !data.isEmpty else { return } |
|||
lock.lock() |
|||
stash.append(data) |
|||
let started = pump() |
|||
let q = output |
|||
let gen = generation |
|||
lock.unlock() |
|||
// 一句播完队列会自己停。停着的时候只入队不会出声，下一句必须再 Start。 |
|||
// 不在持锁时调用，回调线程里再 Start 会和队列互相等。 |
|||
if started { kick(q, generation: gen) } |
|||
} |
|||
|
|||
/// 上层不再喂数据了：已排队的播完就报完成 |
|||
func end() { |
|||
queue.async { [weak self] in |
|||
guard let self = self else { return } |
|||
self.ended = true |
|||
self.finishIfDone() |
|||
lock.lock() |
|||
ended = true |
|||
pump() |
|||
let fire = stash.isEmpty && inFlight == 0 |
|||
if fire { ended = false } |
|||
lock.unlock() |
|||
if fire { |
|||
let cb = onCompleted |
|||
DispatchQueue.main.async { cb?() } |
|||
} |
|||
} |
|||
|
|||
/// 打断：立刻停声，不报完成 |
|||
func stop() { |
|||
queue.async { [weak self] in |
|||
guard let self = self else { return } |
|||
self.generation += 1 |
|||
self.pending = 0 |
|||
self.ended = false |
|||
self.node?.stop() |
|||
self.engine?.stop() |
|||
lock.lock() |
|||
generation += 1 |
|||
ended = false |
|||
stash.removeAll() |
|||
inFlight = 0 |
|||
let q = output |
|||
lock.unlock() |
|||
if let q { |
|||
// true：等回调都回来再往下走。回调里不再把缓冲区丢到别的队列。 |
|||
AudioQueueStop(q, true) |
|||
} |
|||
lock.lock() |
|||
if output != nil { |
|||
freeBuffers = owned |
|||
inFlight = 0 |
|||
} |
|||
lock.unlock() |
|||
} |
|||
|
|||
func dispose() { |
|||
queue.async { [weak self] in |
|||
guard let self = self else { return } |
|||
self.generation += 1 |
|||
self.node?.stop() |
|||
self.engine?.stop() |
|||
self.node = nil |
|||
self.engine = nil |
|||
self.format = nil |
|||
lock.lock() |
|||
generation += 1 |
|||
ended = false |
|||
stash.removeAll() |
|||
inFlight = 0 |
|||
let q = output |
|||
output = nil |
|||
owned.removeAll() |
|||
freeBuffers.removeAll() |
|||
rate = 0 |
|||
lock.unlock() |
|||
if let q { |
|||
AudioQueueStop(q, true) |
|||
AudioQueueDispose(q, true) |
|||
} |
|||
} |
|||
|
|||
// MARK: - 内部 |
|||
|
|||
private func finishIfDone() { |
|||
guard ended, pending <= 0 else { return } |
|||
ended = false |
|||
// ⚠️ 一轮播完必须把引擎停掉，不能让它空跑着占住音频 IO。 |
|||
// 连续对话下 record 要在同一个会话上开 voice-processing， |
|||
// 我们这台引擎还占着的话它会拿不到 IO，报 |
|||
// `avfaudio error 2003329396`（FourCC 'what'）—— 表现就是 |
|||
// 「助手说完话之后麦克风再也开不起来」。真机上间歇复现过。 |
|||
// 对象留着不销毁，下一轮 ensureEngine 直接 start() 就好。 |
|||
node?.stop() |
|||
engine?.stop() |
|||
let cb = onCompleted |
|||
DispatchQueue.main.async { cb?() } |
|||
private func bufferReturned(_ buffer: AudioQueueBufferRef) { |
|||
lock.lock() |
|||
// 队列已经拆掉：这块内存马上要被 Dispose，不能再收进空闲表。 |
|||
guard output != nil else { |
|||
lock.unlock() |
|||
return |
|||
} |
|||
freeBuffers.append(buffer) |
|||
if inFlight > 0 { inFlight -= 1 } |
|||
let started = pump() |
|||
let q = output |
|||
let gen = generation |
|||
let fire = ended && stash.isEmpty && inFlight == 0 |
|||
if fire { ended = false } |
|||
lock.unlock() |
|||
if started { kick(q, generation: gen) } |
|||
if fire { |
|||
let cb = onCompleted |
|||
DispatchQueue.main.async { cb?() } |
|||
} |
|||
} |
|||
|
|||
/// 采样率变了要整条重建：AVAudioEngine 的连接格式是建链路时定死的 |
|||
private func ensureEngine(sampleRate: Double) -> Bool { |
|||
if let engine = engine, let node = node, let fmt = format, |
|||
fmt.sampleRate == sampleRate { |
|||
if !engine.isRunning { |
|||
do { try engine.start() } catch { tearDown(); return buildEngine(sampleRate: sampleRate) } |
|||
} |
|||
if !node.isPlaying { node.play() } |
|||
private func ensureQueue(sampleRate: Double) -> Bool { |
|||
if let output, rate == sampleRate { |
|||
AudioQueueStart(output, nil) |
|||
return true |
|||
} |
|||
tearDown() |
|||
return buildEngine(sampleRate: sampleRate) |
|||
return buildQueue(sampleRate: sampleRate) |
|||
} |
|||
|
|||
private func buildEngine(sampleRate: Double) -> Bool { |
|||
guard let fmt = AVAudioFormat(standardFormatWithSampleRate: sampleRate, channels: 1) else { |
|||
return false |
|||
private func buildQueue(sampleRate: Double) -> Bool { |
|||
var format = AudioStreamBasicDescription( |
|||
mSampleRate: sampleRate, |
|||
mFormatID: kAudioFormatLinearPCM, |
|||
mFormatFlags: kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked, |
|||
mBytesPerPacket: 2, |
|||
mFramesPerPacket: 1, |
|||
mBytesPerFrame: 2, |
|||
mChannelsPerFrame: 1, |
|||
mBitsPerChannel: 16, |
|||
mReserved: 0 |
|||
) |
|||
var queue: AudioQueueRef? |
|||
let status = AudioQueueNewOutput( |
|||
&format, |
|||
PcmSpeakerPlayer.outputCallback, |
|||
Unmanaged.passUnretained(self).toOpaque(), |
|||
nil, |
|||
nil, |
|||
0, |
|||
&queue |
|||
) |
|||
guard status == noErr, let queue else { return false } |
|||
var allocated: [AudioQueueBufferRef] = [] |
|||
for _ in 0..<6 { |
|||
var buffer: AudioQueueBufferRef? |
|||
let alloc = AudioQueueAllocateBuffer(queue, 4096, &buffer) |
|||
guard alloc == noErr, let buffer else { |
|||
AudioQueueDispose(queue, true) |
|||
return false |
|||
} |
|||
allocated.append(buffer) |
|||
} |
|||
let e = AVAudioEngine() |
|||
let n = AVAudioPlayerNode() |
|||
e.attach(n) |
|||
e.connect(n, to: e.mainMixerNode, format: fmt) |
|||
do { |
|||
try e.start() |
|||
} catch { |
|||
NSLog("[PcmSpeakerPlayer] 引擎启动失败: \(error.localizedDescription)") |
|||
let start = AudioQueueStart(queue, nil) |
|||
guard start == noErr else { |
|||
AudioQueueDispose(queue, true) |
|||
return false |
|||
} |
|||
n.play() |
|||
engine = e |
|||
node = n |
|||
format = fmt |
|||
output = queue |
|||
owned = allocated |
|||
freeBuffers = allocated |
|||
rate = sampleRate |
|||
inFlight = 0 |
|||
return true |
|||
} |
|||
|
|||
private func tearDown() { |
|||
node?.stop() |
|||
engine?.stop() |
|||
node = nil |
|||
engine = nil |
|||
format = nil |
|||
let q = output |
|||
output = nil |
|||
owned.removeAll() |
|||
freeBuffers.removeAll() |
|||
rate = 0 |
|||
inFlight = 0 |
|||
stash.removeAll() |
|||
if let q { |
|||
// 先放开锁再停，避免回调里再来抢这把锁时和 Stop 互相等。 |
|||
lock.unlock() |
|||
AudioQueueStop(q, true) |
|||
AudioQueueDispose(q, true) |
|||
lock.lock() |
|||
} |
|||
} |
|||
|
|||
/// Int16 小端裸 PCM → Float32 单声道 buffer |
|||
private func makeBuffer(from data: Data) -> AVAudioPCMBuffer? { |
|||
guard let fmt = format else { return nil } |
|||
let frames = data.count / 2 |
|||
guard frames > 0, |
|||
let buffer = AVAudioPCMBuffer(pcmFormat: fmt, |
|||
frameCapacity: AVAudioFrameCount(frames)), |
|||
let dst = buffer.floatChannelData else { return nil } |
|||
buffer.frameLength = AVAudioFrameCount(frames) |
|||
data.withUnsafeBytes { raw in |
|||
guard let base = raw.baseAddress else { return } |
|||
// ⚠️ 不能 bindMemory(to: Int16.self)：Data 的起始地址不保证 2 字节对齐， |
|||
// 未对齐访问在 arm64 上是未定义行为。逐字节取再拼。 |
|||
let bytes = base.assumingMemoryBound(to: UInt8.self) |
|||
for i in 0..<frames { |
|||
let lo = UInt16(bytes[i * 2]) |
|||
let hi = UInt16(bytes[i * 2 + 1]) |
|||
let sample = Int16(bitPattern: lo | (hi << 8)) |
|||
dst[0][i] = Float(sample) / 32768.0 |
|||
/// 队列停过之后，入队不会自动接着播。换到别的线程再 Start。 |
|||
private func kick(_ queue: AudioQueueRef?, generation gen: Int) { |
|||
guard let queue else { return } |
|||
DispatchQueue.global(qos: .userInteractive).async { [weak self] in |
|||
guard let self else { return } |
|||
self.lock.lock() |
|||
let current = self.generation == gen ? self.output : nil |
|||
self.lock.unlock() |
|||
guard let current else { return } |
|||
AudioQueueStart(current, nil) |
|||
} |
|||
} |
|||
|
|||
/// 有数据入队时返回 true，调用方据此把停掉的队列重新拉起来。 |
|||
private func pump() -> Bool { |
|||
guard let output else { return false } |
|||
var started = false |
|||
while !freeBuffers.isEmpty, stash.count >= 2 { |
|||
let buffer = freeBuffers.removeFirst() |
|||
let capacity = Int(buffer.pointee.mAudioDataBytesCapacity) |
|||
var count = min(capacity, stash.count) |
|||
count -= count % 2 |
|||
guard count >= 2 else { |
|||
freeBuffers.append(buffer) |
|||
break |
|||
} |
|||
let dest = buffer.pointee.mAudioData |
|||
stash.withUnsafeBytes { raw in |
|||
if let base = raw.baseAddress { |
|||
memcpy(dest, base, count) |
|||
} |
|||
} |
|||
stash.removeFirst(count) |
|||
buffer.pointee.mAudioDataByteSize = UInt32(count) |
|||
let status = AudioQueueEnqueueBuffer(output, buffer, 0, nil) |
|||
if status != noErr { |
|||
freeBuffers.append(buffer) |
|||
break |
|||
} |
|||
inFlight += 1 |
|||
started = true |
|||
} |
|||
return buffer |
|||
return started |
|||
} |
|||
} |
|||
|
|||
@ -0,0 +1,313 @@ |
|||
import AudioToolbox |
|||
import AVFoundation |
|||
import Flutter |
|||
import Foundation |
|||
|
|||
enum AudioGraph { |
|||
static let queue = DispatchQueue(label: "azure_speech.audio_graph") |
|||
} |
|||
|
|||
/// 同声 / 面对面的手机麦。只采集，不播放。 |
|||
/// |
|||
/// 用 AudioQueue 输入，不用 AVAudioEngine。引擎上的输入 tap 在这台系统上 |
|||
/// 往往只回调一次就停，云端等不到声音，十几秒后报 18802。 |
|||
/// 播放是另一条 AudioQueue，不要把播放节点挂到采集上。 |
|||
final class PhoneMicCapture: NSObject, FlutterStreamHandler { |
|||
static weak var shared: PhoneMicCapture? |
|||
|
|||
private let lock = NSLock() |
|||
private var input: AudioQueueRef? |
|||
private var sink: FlutterEventSink? |
|||
private var running = false |
|||
private var life = 0 |
|||
private var sourceRate: Double = 16000 |
|||
private var resamplePos: Double = 0 |
|||
private var routeObserver: NSObjectProtocol? |
|||
private var interruptionObserver: NSObjectProtocol? |
|||
|
|||
func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { |
|||
sink = events |
|||
return nil |
|||
} |
|||
|
|||
func onCancel(withArguments arguments: Any?) -> FlutterError? { |
|||
sink = nil |
|||
return nil |
|||
} |
|||
|
|||
func start(voiceProcessing: Bool, done: @escaping (String?) -> Void) { |
|||
_ = voiceProcessing |
|||
// 队列必须在主线程启动。放在后台队列时，这台系统只回调一次就停。 |
|||
DispatchQueue.main.async { [weak self] in |
|||
guard let self else { |
|||
done("麦克风已释放") |
|||
return |
|||
} |
|||
self.life += 1 |
|||
self.attempt(life: self.life, left: 4, done: done) |
|||
} |
|||
} |
|||
|
|||
/// AudioQueueStart 在会话刚激活时会回 'what'(2003329396)。等一下再激活再试。 |
|||
private func attempt(life: Int, left: Int, done: @escaping (String?) -> Void) { |
|||
if self.life != life { |
|||
tearDownQueue() |
|||
done(nil) |
|||
return |
|||
} |
|||
let err = startLocked() |
|||
if err == nil || left <= 1 { |
|||
done(err) |
|||
return |
|||
} |
|||
tearDownQueue() |
|||
DispatchQueue.main.asyncAfter(deadline: .now() + 0.45) { [weak self] in |
|||
guard let self else { |
|||
done("麦克风已释放") |
|||
return |
|||
} |
|||
let session = AVAudioSession.sharedInstance() |
|||
if session.category != .playAndRecord { |
|||
try? session.setCategory( |
|||
.playAndRecord, |
|||
mode: session.mode == .voiceChat ? .voiceChat : .default, |
|||
options: [.defaultToSpeaker, .allowBluetoothA2DP] |
|||
) |
|||
} |
|||
try? session.setActive(true) |
|||
self.attempt(life: life, left: left - 1, done: done) |
|||
} |
|||
} |
|||
|
|||
func stop(done: @escaping () -> Void) { |
|||
DispatchQueue.main.async { [weak self] in |
|||
guard let self else { |
|||
done() |
|||
return |
|||
} |
|||
self.life += 1 |
|||
self.stopLocked() |
|||
done() |
|||
} |
|||
} |
|||
|
|||
/// 输入停了但事件通道还在。拆掉队列再起一条。 |
|||
func kick() { |
|||
DispatchQueue.main.async { [weak self] in |
|||
guard let self, self.running else { return } |
|||
let life = self.life |
|||
self.tearDownQueue() |
|||
if self.life != life { return } |
|||
let err = self.startLocked() |
|||
if self.life != life { |
|||
self.tearDownQueue() |
|||
return |
|||
} |
|||
if err != nil { |
|||
self.sink?(["type": "reinstallFailed"]) |
|||
} else { |
|||
self.sink?(["type": "reinstalled"]) |
|||
} |
|||
} |
|||
} |
|||
|
|||
private func startLocked() -> String? { |
|||
if running, input != nil { return nil } |
|||
let session = AVAudioSession.sharedInstance() |
|||
do { |
|||
try session.setActive(true) |
|||
} catch { |
|||
return "音频会话未激活 \(error.localizedDescription)" |
|||
} |
|||
// 这台手机把采集写成 16 kHz 时，队列只吐一块就停。停掉的瞬间如果正在连讯飞， |
|||
// 重启麦会把握手打断，start 返回 18801。按硬件采样率采，送出前再降到 16 kHz。 |
|||
var rates: [Double] = [] |
|||
let hw = session.sampleRate |
|||
if hw > 1 { rates.append(hw) } |
|||
for rate in [48000.0, 44100.0, 16000.0] where !rates.contains(where: { abs($0 - rate) < 1 }) { |
|||
rates.append(rate) |
|||
} |
|||
var last = "麦克风队列启动失败" |
|||
for rate in rates { |
|||
if let err = openQueue(rate: rate) { |
|||
last = err |
|||
continue |
|||
} |
|||
return nil |
|||
} |
|||
let inputs = session.currentRoute.inputs.map(\.portType.rawValue).joined(separator: ",") |
|||
return "\(last) rate=\(hw) cat=\(session.category.rawValue) mode=\(session.mode.rawValue) in=\(inputs)" |
|||
} |
|||
|
|||
/// 按硬件采样率要数据。这个采样率起不来就换下一个，不在同一次失败里空转。 |
|||
private func openQueue(rate: Double) -> String? { |
|||
sourceRate = rate |
|||
resamplePos = 0 |
|||
var format = AudioStreamBasicDescription( |
|||
mSampleRate: rate, |
|||
mFormatID: kAudioFormatLinearPCM, |
|||
mFormatFlags: kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked, |
|||
mBytesPerPacket: 2, |
|||
mFramesPerPacket: 1, |
|||
mBytesPerFrame: 2, |
|||
mChannelsPerFrame: 1, |
|||
mBitsPerChannel: 16, |
|||
mReserved: 0 |
|||
) |
|||
var queue: AudioQueueRef? |
|||
let ctx = Unmanaged.passUnretained(self).toOpaque() |
|||
let created = AudioQueueNewInput(&format, PhoneMicCapture.inputCallback, ctx, nil, nil, 0, &queue) |
|||
guard created == noErr, let queue else { |
|||
return "麦克风队列创建失败 \(created) rate=\(rate)" |
|||
} |
|||
let bytesPerBuffer = UInt32(max(3200, Int(rate) * 2 / 10)) |
|||
for _ in 0..<3 { |
|||
var buffer: AudioQueueBufferRef? |
|||
let allocated = AudioQueueAllocateBuffer(queue, bytesPerBuffer, &buffer) |
|||
if allocated != noErr || buffer == nil { |
|||
AudioQueueDispose(queue, true) |
|||
return "麦克风缓冲创建失败 \(allocated) rate=\(rate)" |
|||
} |
|||
let queued = AudioQueueEnqueueBuffer(queue, buffer!, 0, nil) |
|||
if queued != noErr { |
|||
AudioQueueDispose(queue, true) |
|||
return "麦克风缓冲入队失败 \(queued) rate=\(rate)" |
|||
} |
|||
} |
|||
lock.lock() |
|||
input = queue |
|||
running = true |
|||
lock.unlock() |
|||
let started = AudioQueueStart(queue, nil) |
|||
if started != noErr { |
|||
tearDownQueue() |
|||
return "麦克风队列启动失败 \(started) rate=\(rate)" |
|||
} |
|||
watchSession() |
|||
reportOpened() |
|||
return nil |
|||
} |
|||
|
|||
private func reportOpened() { |
|||
let session = AVAudioSession.sharedInstance() |
|||
let inputs = session.currentRoute.inputs.map(\.portType.rawValue).joined(separator: ",") |
|||
let outputs = session.currentRoute.outputs.map(\.portType.rawValue).joined(separator: ",") |
|||
sink?([ |
|||
"type": "opened", |
|||
"mode": session.mode.rawValue, |
|||
"rate": sourceRate, |
|||
"input": inputs, |
|||
"output": outputs, |
|||
]) |
|||
} |
|||
|
|||
private func watchSession() { |
|||
if routeObserver == nil { |
|||
routeObserver = NotificationCenter.default.addObserver( |
|||
forName: AVAudioSession.routeChangeNotification, |
|||
object: nil, |
|||
queue: nil |
|||
) { [weak self] _ in |
|||
DispatchQueue.main.async { self?.restartIfStopped() } |
|||
} |
|||
} |
|||
if interruptionObserver == nil { |
|||
interruptionObserver = NotificationCenter.default.addObserver( |
|||
forName: AVAudioSession.interruptionNotification, |
|||
object: nil, |
|||
queue: nil |
|||
) { [weak self] note in |
|||
let raw = note.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt |
|||
let type = raw.flatMap { AVAudioSession.InterruptionType(rawValue: $0) } |
|||
guard type == .ended else { return } |
|||
DispatchQueue.main.async { self?.restartIfStopped() } |
|||
} |
|||
} |
|||
} |
|||
|
|||
/// 路由或中断把队列停掉时接上。队列还在跑就什么都不做，避免自己把自己重启个没完。 |
|||
private func restartIfStopped() { |
|||
guard running else { return } |
|||
lock.lock() |
|||
let queue = input |
|||
lock.unlock() |
|||
guard let queue else { return } |
|||
var flag: UInt32 = 0 |
|||
var size = UInt32(MemoryLayout<UInt32>.size) |
|||
let status = AudioQueueGetProperty(queue, kAudioQueueProperty_IsRunning, &flag, &size) |
|||
if status == noErr, flag != 0 { return } |
|||
tearDownQueue() |
|||
let err = startLocked() |
|||
DispatchQueue.main.async { [weak self] in |
|||
if err != nil { |
|||
self?.sink?(["type": "reinstallFailed"]) |
|||
} else { |
|||
self?.sink?(["type": "reinstalled"]) |
|||
} |
|||
} |
|||
} |
|||
|
|||
private func stopLocked() { |
|||
lock.lock() |
|||
running = false |
|||
lock.unlock() |
|||
tearDownQueue() |
|||
if let routeObserver { |
|||
NotificationCenter.default.removeObserver(routeObserver) |
|||
self.routeObserver = nil |
|||
} |
|||
if let interruptionObserver { |
|||
NotificationCenter.default.removeObserver(interruptionObserver) |
|||
self.interruptionObserver = nil |
|||
} |
|||
} |
|||
|
|||
private func tearDownQueue() { |
|||
lock.lock() |
|||
running = false |
|||
let queue = input |
|||
input = nil |
|||
lock.unlock() |
|||
guard let queue else { return } |
|||
AudioQueueStop(queue, true) |
|||
AudioQueueDispose(queue, true) |
|||
} |
|||
|
|||
private static let inputCallback: AudioQueueInputCallback = { ctx, queue, buffer, _, _, _ in |
|||
guard let ctx else { return } |
|||
let mic = Unmanaged<PhoneMicCapture>.fromOpaque(ctx).takeUnretainedValue() |
|||
let size = Int(buffer.pointee.mAudioDataByteSize) |
|||
var copied: Data? |
|||
if size > 1 { |
|||
let frames = size / 2 |
|||
copied = mic.downsample(buffer.pointee.mAudioData.assumingMemoryBound(to: Int16.self), frames: frames) |
|||
} |
|||
mic.lock.lock() |
|||
let live = mic.running |
|||
mic.lock.unlock() |
|||
if live { |
|||
AudioQueueEnqueueBuffer(queue, buffer, 0, nil) |
|||
} |
|||
guard let copied, !copied.isEmpty else { return } |
|||
DispatchQueue.main.async { [weak mic] in |
|||
mic?.sink?(FlutterStandardTypedData(bytes: copied)) |
|||
} |
|||
} |
|||
|
|||
/// 硬件采样率收进来，送出去的一律是 16 kHz。 |
|||
private func downsample(_ samples: UnsafePointer<Int16>, frames: Int) -> Data { |
|||
if sourceRate <= 16000 + 1 { |
|||
return Data(bytes: samples, count: frames * 2) |
|||
} |
|||
let ratio = sourceRate / 16000 |
|||
var out: [Int16] = [] |
|||
out.reserveCapacity(Int(Double(frames) / ratio) + 2) |
|||
var cursor = resamplePos |
|||
while cursor < Double(frames) { |
|||
out.append(samples[Int(cursor)]) |
|||
cursor += ratio |
|||
} |
|||
resamplePos = cursor - Double(frames) |
|||
return out.withUnsafeBytes { Data($0) } |
|||
} |
|||
} |
|||
@ -0,0 +1,53 @@ |
|||
import 'package:eaimar/data/services/emai_iflytek_turn.dart'; |
|||
import 'package:flutter_test/flutter_test.dart'; |
|||
|
|||
void main() { |
|||
test('时间带星期和时区，提示里不出现经纬度', () { |
|||
final now = DateTime(2026, 10, 18, 0, 59); |
|||
final prompt = EmaiTurn.systemPrompt( |
|||
now: now, |
|||
spokenPlace: '上海市浦东新区世纪大道1号', |
|||
answerLanguage: '中文', |
|||
history: [EmaiMemory(true, '你好')], |
|||
); |
|||
expect(prompt, contains('2026年10月18日')); |
|||
expect(prompt, contains('星期日')); |
|||
expect(prompt, contains('00:59')); |
|||
expect(prompt, contains('UTC')); |
|||
expect(prompt, contains('当前位置：上海市浦东新区世纪大道1号')); |
|||
expect(prompt, isNot(contains('31.2'))); |
|||
}); |
|||
|
|||
test('没有地址时写当前没有位置', () { |
|||
final prompt = EmaiTurn.systemPrompt( |
|||
now: DateTime(2026, 10, 18), |
|||
spokenPlace: '未知', |
|||
answerLanguage: '中文', |
|||
history: const [], |
|||
); |
|||
expect(prompt, contains('当前没有位置')); |
|||
expect(prompt, isNot(contains('当前位置：未知'))); |
|||
}); |
|||
|
|||
test('搜索词补上当天日期', () { |
|||
final day = DateTime(2026, 10, 18); |
|||
expect(EmaiTurn.searchQuery('今天新闻', day), '今天新闻 2026-10-18'); |
|||
expect(EmaiTurn.searchQuery('新闻 2026-10-18', day), '新闻 2026-10-18'); |
|||
}); |
|||
|
|||
test('解析工具调用并丢掉令牌字段', () { |
|||
final calls = EmaiTurn.parseCalls( |
|||
'{"name":"hefeng_weather","arguments":"{\\"city\\":\\"上海\\",\\"auth_token\\":\\"secret\\"}"}', |
|||
); |
|||
expect(calls, hasLength(1)); |
|||
expect(calls.single.name, 'hefeng_weather'); |
|||
expect(calls.single.args['city'], '上海'); |
|||
expect(calls.single.args.containsKey('auth_token'), isFalse); |
|||
}); |
|||
|
|||
test('问我在哪才等待定位', () { |
|||
expect(EmaiTurn.asksWhere('我现在在哪里'), isTrue); |
|||
expect(EmaiTurn.asksWhere('where am I'), isTrue); |
|||
expect(EmaiTurn.asksWhere('今天天气怎么样'), isFalse); |
|||
}); |
|||
} |
|||
@ -0,0 +1,74 @@ |
|||
import 'package:eaimar/data/services/simul_microsoft.dart'; |
|||
import 'package:flutter_test/flutter_test.dart'; |
|||
|
|||
void main() { |
|||
test('日语到英语走 AST，发音人是 JennyNeural', () { |
|||
final route = SimulMicrosoft.plan('ja', 'en'); |
|||
expect(route, isNotNull); |
|||
expect(route!.ast, isTrue); |
|||
expect(route.lang, 'ja'); |
|||
expect(route.targetLang, 'en'); |
|||
expect(route.speaker, 'JennyNeural'); |
|||
}); |
|||
|
|||
test('英语到日语同传没授权时改走转写，识别 en-US，合成 Nanami', () { |
|||
final route = SimulMicrosoft.plan('en', 'ja')!; |
|||
final splice = SimulMicrosoft.spliceInstead(route); |
|||
expect(splice.ast, isFalse); |
|||
expect(splice.languageName, 'en-US'); |
|||
expect(splice.speaker, 'ja-JP-NanamiNeural'); |
|||
}); |
|||
|
|||
test('英语到日语走 AST，发音人是 NanamiNeural', () { |
|||
final route = SimulMicrosoft.plan('en-US', 'ja-JP'); |
|||
expect(route!.ast, isTrue); |
|||
expect(route.lang, 'en'); |
|||
expect(route.targetLang, 'ja'); |
|||
expect(route.speaker, 'NanamiNeural'); |
|||
}); |
|||
|
|||
test('普通话到英语走拼接，识别 zh-CN，合成 en-US-JennyNeural', () { |
|||
final route = SimulMicrosoft.plan('zh', 'en'); |
|||
expect(route!.ast, isFalse); |
|||
expect(route.languageName, 'zh-CN'); |
|||
expect(route.speaker, 'en-US-JennyNeural'); |
|||
}); |
|||
|
|||
test('英语到普通话走拼接，识别 en-US，合成晓晓', () { |
|||
final route = SimulMicrosoft.plan('en', 'zh-CN'); |
|||
expect(route!.ast, isFalse); |
|||
expect(route.languageName, 'en-US'); |
|||
expect(route.speaker, 'zh-CN-XiaoxiaoNeural'); |
|||
}); |
|||
|
|||
test('普通话到爱尔兰语识别 zh-CN，合成 Orla', () { |
|||
final route = SimulMicrosoft.plan('zh', 'ga'); |
|||
expect(route!.ast, isFalse); |
|||
expect(route.languageName, 'zh-CN'); |
|||
expect(route.speaker, 'ga-IE-OrlaNeural'); |
|||
}); |
|||
|
|||
test('粤语简体识别 yue-CN，香港繁体识别 zh-HK', () { |
|||
final yue = SimulMicrosoft.plan('yue', 'en'); |
|||
expect(yue!.languageName, 'yue-CN'); |
|||
expect(yue.speaker, 'en-US-JennyNeural'); |
|||
|
|||
final hk = SimulMicrosoft.plan('zh-Hant-hk', 'en'); |
|||
expect(hk!.languageName, 'zh-HK'); |
|||
expect(hk.speaker, 'en-US-JennyNeural'); |
|||
}); |
|||
|
|||
test('威尔士语到冰岛语不启动', () { |
|||
expect(SimulMicrosoft.plan('cy', 'is'), isNull); |
|||
}); |
|||
|
|||
test('拼接翻译语种码给讯飞 SDK，中文保持 zh-CN', () { |
|||
expect(SimulMicrosoft.translateCode('zh'), 'zh-CN'); |
|||
expect(SimulMicrosoft.translateCode('zh-Hant'), 'zh-TW'); |
|||
expect(SimulMicrosoft.translateCode('zh-Hant-hk'), 'zh-HK'); |
|||
expect(SimulMicrosoft.translateCode('yue'), 'yue'); |
|||
expect(SimulMicrosoft.translateCode('ga'), 'ga'); |
|||
expect(SimulMicrosoft.translateCode('ja-JP'), 'ja'); |
|||
expect(SimulMicrosoft.translateCode('en'), 'en'); |
|||
}); |
|||
} |
|||
@ -0,0 +1,112 @@ |
|||
package echomeet |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
|
|||
"yunyan/pb" |
|||
"yunyan/sys/iflytek/openaichat" |
|||
"yunyan/sys/iflytek/ost" |
|||
) |
|||
|
|||
// 科大讯飞 SparkChain 的两条服务端能力：
|
|||
// 转写用 SDK 里 RegionMSFT 的录音文件引擎（音频地址仍是客户端 OSS 公网 URL）；
|
|||
// 总结用同一套凭据的海外文本模型，domain=openai（gpt-5.3-chat），不走星火。
|
|||
|
|||
type iflytekTranscriber struct { |
|||
sys *ost.Client |
|||
} |
|||
|
|||
func newIflytekTranscriber(f map[string]string) (Transcriber, error) { |
|||
appID, err := need(f, "iflytek", "app_id") |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
key, err := need(f, "iflytek", "api_key") |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
secret, err := need(f, "iflytek", "api_secret") |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
sys, err := ost.New(appID, key, secret, f["host"], f["task_type"]) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
return &iflytekTranscriber{sys: sys}, nil |
|||
} |
|||
|
|||
func (this *iflytekTranscriber) Submit(_ context.Context, req SubmitRequest) (*SubmitResult, error) { |
|||
taskID, err := this.sys.Create(req.AudioURL, req.EnableSpeaker, req.Language) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
return &SubmitResult{TaskID: taskID}, nil |
|||
} |
|||
|
|||
func (this *iflytekTranscriber) Query(_ context.Context, taskID, _ string) (*QueryResult, error) { |
|||
status, segs, err := this.sys.Query(taskID) |
|||
switch status { |
|||
case ost.StatusSuccess: |
|||
out := make([]*pb.ContextStruct, 0, len(segs)) |
|||
for _, s := range segs { |
|||
out = append(out, &pb.ContextStruct{ |
|||
Content: s.Text, |
|||
Starttime: s.StartMs, |
|||
Endtime: s.EndMs, |
|||
Speaker: s.Speaker, |
|||
}) |
|||
} |
|||
return &QueryResult{Status: StatusSuccess, Contexts: out}, nil |
|||
case ost.StatusRunning: |
|||
return &QueryResult{Status: StatusRunning}, nil |
|||
default: |
|||
if err == nil { |
|||
err = fmt.Errorf("iflytek ost query failed") |
|||
} |
|||
return &QueryResult{Status: StatusFailed}, err |
|||
} |
|||
} |
|||
|
|||
type iflytekSummarizer struct { |
|||
sys *openaichat.Client |
|||
} |
|||
|
|||
func newIflytekSummarizer(f map[string]string) (Summarizer, error) { |
|||
appID, err := need(f, "iflytek", "app_id") |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
key, err := need(f, "iflytek", "api_key") |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
secret, err := need(f, "iflytek", "api_secret") |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
// 库里若还留着星火的 host/domain，New 会丢掉，改走 SDK 预置的 openai 地址和 gpt-5.3-chat。
|
|||
sys, err := openaichat.New(appID, key, secret, f["host"], f["model"]) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
return &iflytekSummarizer{sys: sys}, nil |
|||
} |
|||
|
|||
// iflytekSummaryDisplayName 生成弹窗里显示的模型名，跟真正发出去的模型一致。
|
|||
func iflytekSummaryDisplayName(fields map[string]string) string { |
|||
model := "" |
|||
if fields != nil { |
|||
model = fields["model"] |
|||
} |
|||
return openaichat.EffectiveModel(model) |
|||
} |
|||
|
|||
func (this *iflytekSummarizer) Chat(_ context.Context, messages []ChatMessage) (string, error) { |
|||
in := make([]openaichat.Message, 0, len(messages)) |
|||
for _, m := range messages { |
|||
in = append(in, openaichat.Message{Role: m.Role, Content: m.Content}) |
|||
} |
|||
return this.sys.Chat(in) |
|||
} |
|||
@ -0,0 +1,298 @@ |
|||
// Package openaichat 调用 SparkChain 海外 SDK 里 domain=openai 的文本模型。
|
|||
//
|
|||
// 会议总结走这条，不走星火 websocket。SDK 二进制里的形状是：
|
|||
//
|
|||
// POST https://cn-huadong-1.xf-yun.com/v1/private/s56f331b2
|
|||
// 鉴权挂在 URL 上（host date request-line，没有 digest）
|
|||
// 正文是讯飞信封，payload.input.text = base64(OpenAI chat JSON)
|
|||
// 响应 payload.output.text 再 base64 解出来，里面是 chat.completion 或 responses
|
|||
package openaichat |
|||
|
|||
import ( |
|||
"bytes" |
|||
"crypto/hmac" |
|||
"crypto/sha256" |
|||
"encoding/base64" |
|||
"encoding/json" |
|||
"fmt" |
|||
"io" |
|||
"net/http" |
|||
"net/url" |
|||
"strings" |
|||
"time" |
|||
) |
|||
|
|||
const ( |
|||
defaultHost = "cn-huadong-1.xf-yun.com" |
|||
defaultPath = "/v1/private/s56f331b2" |
|||
defaultModel = "gpt-5.3-chat" |
|||
) |
|||
|
|||
// Message 一轮对话。Role 取 system / user / assistant。
|
|||
type Message struct { |
|||
Role string `json:"role"` |
|||
Content string `json:"content"` |
|||
} |
|||
|
|||
// Client 讯飞 OpenAI 文本客户端。
|
|||
type Client struct { |
|||
AppID string |
|||
APIKey string |
|||
Secret string |
|||
Host string |
|||
Model string |
|||
HTTP *http.Client |
|||
} |
|||
|
|||
// New 用 SparkChain 的 appId / apiKey / apiSecret 建客户端。
|
|||
// model 空则用文档里的对话模型 gpt-5.3-chat。host 空则用 SDK 预置地址。
|
|||
func New(appID, apiKey, secret, host, model string) (*Client, error) { |
|||
if strings.TrimSpace(appID) == "" || strings.TrimSpace(apiKey) == "" || strings.TrimSpace(secret) == "" { |
|||
return nil, fmt.Errorf("iflytek openai: app_id/api_key/api_secret 都不能空") |
|||
} |
|||
host = strings.TrimSpace(host) |
|||
host = strings.TrimPrefix(strings.TrimPrefix(host, "https://"), "http://") |
|||
host = strings.Trim(host, "/") |
|||
if host == "" || strings.Contains(host, "spark-api") { |
|||
host = defaultHost |
|||
} |
|||
model = EffectiveModel(model) |
|||
return &Client{ |
|||
AppID: appID, |
|||
APIKey: apiKey, |
|||
Secret: secret, |
|||
Host: host, |
|||
Model: model, |
|||
HTTP: &http.Client{Timeout: 120 * time.Second, CheckRedirect: refuseRedirect}, |
|||
}, nil |
|||
} |
|||
|
|||
// EffectiveModel 空模型或星火的 general* 都改成文档里的对话模型。
|
|||
func EffectiveModel(model string) string { |
|||
model = strings.TrimSpace(model) |
|||
if model == "" || strings.HasPrefix(model, "general") { |
|||
return defaultModel |
|||
} |
|||
return model |
|||
} |
|||
|
|||
func refuseRedirect(*http.Request, []*http.Request) error { |
|||
return http.ErrUseLastResponse |
|||
} |
|||
|
|||
// Chat 同步要一段回复。
|
|||
func (c *Client) Chat(messages []Message) (string, error) { |
|||
if len(messages) == 0 { |
|||
return "", fmt.Errorf("iflytek openai: 没有输入") |
|||
} |
|||
body, err := c.envelope(messages) |
|||
if err != nil { |
|||
return "", err |
|||
} |
|||
endpoint, err := c.authURL() |
|||
if err != nil { |
|||
return "", err |
|||
} |
|||
req, err := http.NewRequest(http.MethodPost, endpoint, bytes.NewReader(body)) |
|||
if err != nil { |
|||
return "", err |
|||
} |
|||
req.Host = c.Host |
|||
req.Header.Set("Content-Type", "application/json") |
|||
resp, err := c.HTTP.Do(req) |
|||
if err != nil { |
|||
return "", fmt.Errorf("iflytek openai: %w", err) |
|||
} |
|||
defer resp.Body.Close() |
|||
raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) |
|||
if err != nil { |
|||
return "", err |
|||
} |
|||
text, err := parseOuter(raw) |
|||
if err != nil { |
|||
if resp.StatusCode != http.StatusOK { |
|||
return "", fmt.Errorf("iflytek openai: http %d: %w", resp.StatusCode, err) |
|||
} |
|||
return "", err |
|||
} |
|||
return text, nil |
|||
} |
|||
|
|||
// envelope 讯飞外层。内层是 OpenAI chat JSON，text 字段 base64。
|
|||
func (c *Client) envelope(messages []Message) ([]byte, error) { |
|||
inner := map[string]any{ |
|||
"model": c.Model, |
|||
"max_completion_tokens": 4096, |
|||
"stream": false, |
|||
"messages": messages, |
|||
} |
|||
raw, err := json.Marshal(inner) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
outer := map[string]any{ |
|||
"header": map[string]any{ |
|||
"app_id": c.AppID, |
|||
"status": 3, |
|||
}, |
|||
"parameter": map[string]any{ |
|||
"chat": map[string]any{ |
|||
"output": map[string]any{ |
|||
"encoding": "utf8", |
|||
"compress": "raw", |
|||
"format": "json", |
|||
}, |
|||
}, |
|||
}, |
|||
"payload": map[string]any{ |
|||
"input": map[string]any{ |
|||
"encoding": "utf8", |
|||
"compress": "raw", |
|||
"format": "json", |
|||
"status": 3, |
|||
"text": base64.StdEncoding.EncodeToString(raw), |
|||
}, |
|||
}, |
|||
} |
|||
return json.Marshal(outer) |
|||
} |
|||
|
|||
func (c *Client) authURL() (string, error) { |
|||
date := time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT") |
|||
origin := "host: " + c.Host + "\n" + "date: " + date + "\n" + "POST " + defaultPath + " HTTP/1.1" |
|||
mac := hmac.New(sha256.New, []byte(c.Secret)) |
|||
_, _ = mac.Write([]byte(origin)) |
|||
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) |
|||
authOrigin := fmt.Sprintf(`api_key="%s", algorithm="hmac-sha256", headers="host date request-line", signature="%s"`, |
|||
c.APIKey, signature) |
|||
q := url.Values{} |
|||
q.Set("authorization", base64.StdEncoding.EncodeToString([]byte(authOrigin))) |
|||
q.Set("date", date) |
|||
q.Set("host", c.Host) |
|||
return "https://" + c.Host + defaultPath + "?" + q.Encode(), nil |
|||
} |
|||
|
|||
func parseOuter(raw []byte) (string, error) { |
|||
var outer struct { |
|||
Header struct { |
|||
Code int `json:"code"` |
|||
Message string `json:"message"` |
|||
} `json:"header"` |
|||
Payload struct { |
|||
Output struct { |
|||
Text string `json:"text"` |
|||
} `json:"output"` |
|||
} `json:"payload"` |
|||
} |
|||
if err := json.Unmarshal(raw, &outer); err != nil { |
|||
return "", fmt.Errorf("iflytek openai: 响应不是 JSON") |
|||
} |
|||
if outer.Header.Code != 0 { |
|||
msg := strings.TrimSpace(outer.Header.Message) |
|||
if msg == "" { |
|||
msg = "gateway request failed" |
|||
} |
|||
return "", fmt.Errorf("iflytek openai: code=%d %s", outer.Header.Code, msg) |
|||
} |
|||
decoded, err := base64.StdEncoding.DecodeString(outer.Payload.Output.Text) |
|||
if err != nil { |
|||
decoded, err = base64.RawStdEncoding.DecodeString(outer.Payload.Output.Text) |
|||
} |
|||
if err != nil { |
|||
return "", fmt.Errorf("iflytek openai: 输出不是 base64") |
|||
} |
|||
text := extractContent(decoded) |
|||
text = strings.TrimSpace(text) |
|||
if text == "" { |
|||
return "", fmt.Errorf("iflytek openai: 空回复") |
|||
} |
|||
return text, nil |
|||
} |
|||
|
|||
func extractContent(decoded []byte) string { |
|||
s := strings.TrimSpace(string(decoded)) |
|||
if strings.Contains(s, "data:") && !json.Valid(decoded) { |
|||
return sseContent(s) |
|||
} |
|||
var body map[string]any |
|||
if json.Unmarshal(decoded, &body) != nil { |
|||
return s |
|||
} |
|||
if fmt.Sprint(body["object"]) == "response" { |
|||
if t := responseText(body); t != "" { |
|||
return t |
|||
} |
|||
} |
|||
if t := chatCompletionText(body); t != "" { |
|||
return t |
|||
} |
|||
if t := responseText(body); t != "" { |
|||
return t |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func chatCompletionText(body map[string]any) string { |
|||
choices, _ := body["choices"].([]any) |
|||
var b strings.Builder |
|||
for _, item := range choices { |
|||
m, _ := item.(map[string]any) |
|||
if msg, ok := m["message"].(map[string]any); ok { |
|||
if c, ok := msg["content"].(string); ok { |
|||
b.WriteString(c) |
|||
} |
|||
} |
|||
if delta, ok := m["delta"].(map[string]any); ok { |
|||
if c, ok := delta["content"].(string); ok { |
|||
b.WriteString(c) |
|||
} |
|||
} |
|||
if t, ok := m["text"].(string); ok { |
|||
b.WriteString(t) |
|||
} |
|||
} |
|||
return b.String() |
|||
} |
|||
|
|||
func responseText(body map[string]any) string { |
|||
var b strings.Builder |
|||
switch out := body["output"].(type) { |
|||
case string: |
|||
return out |
|||
case []any: |
|||
for _, item := range out { |
|||
m, _ := item.(map[string]any) |
|||
if t, ok := m["text"].(string); ok { |
|||
b.WriteString(t) |
|||
} |
|||
content, _ := m["content"].([]any) |
|||
for _, part := range content { |
|||
pm, _ := part.(map[string]any) |
|||
if t, ok := pm["text"].(string); ok { |
|||
b.WriteString(t) |
|||
} |
|||
} |
|||
} |
|||
} |
|||
return b.String() |
|||
} |
|||
|
|||
func sseContent(s string) string { |
|||
var b strings.Builder |
|||
for _, line := range strings.Split(s, "\n") { |
|||
line = strings.TrimSpace(line) |
|||
if !strings.HasPrefix(line, "data:") { |
|||
continue |
|||
} |
|||
payload := strings.TrimSpace(strings.TrimPrefix(line, "data:")) |
|||
if payload == "" || payload == "[DONE]" { |
|||
continue |
|||
} |
|||
var body map[string]any |
|||
if json.Unmarshal([]byte(payload), &body) != nil { |
|||
continue |
|||
} |
|||
b.WriteString(chatCompletionText(body)) |
|||
} |
|||
return b.String() |
|||
} |
|||
@ -0,0 +1,99 @@ |
|||
package openaichat |
|||
|
|||
import ( |
|||
"encoding/base64" |
|||
"encoding/json" |
|||
"strings" |
|||
"testing" |
|||
) |
|||
|
|||
func TestEffectiveModelDropsSpark(t *testing.T) { |
|||
if EffectiveModel("") != "gpt-5.3-chat" || EffectiveModel("generalv3.5") != "gpt-5.3-chat" { |
|||
t.Fatal(EffectiveModel("generalv3.5")) |
|||
} |
|||
if EffectiveModel("gpt-5.4") != "gpt-5.4" { |
|||
t.Fatal(EffectiveModel("gpt-5.4")) |
|||
} |
|||
} |
|||
|
|||
func TestEnvelopeUsesOpenAIModelNotSpark(t *testing.T) { |
|||
c, err := New("app", "key", "secret", "spark-api.xf-yun.com", "generalv3.5") |
|||
if err != nil { |
|||
t.Fatal(err) |
|||
} |
|||
if c.Host != defaultHost { |
|||
t.Fatalf("星火主机必须丢掉，得到 %s", c.Host) |
|||
} |
|||
if c.Model != defaultModel { |
|||
t.Fatalf("星火 domain 不能当模型，得到 %s", c.Model) |
|||
} |
|||
raw, err := c.envelope([]Message{{Role: "user", Content: "1+1等于几"}}) |
|||
if err != nil { |
|||
t.Fatal(err) |
|||
} |
|||
var outer struct { |
|||
Header struct { |
|||
AppID string `json:"app_id"` |
|||
Status int `json:"status"` |
|||
} `json:"header"` |
|||
Payload struct { |
|||
Input struct { |
|||
Status int `json:"status"` |
|||
Text string `json:"text"` |
|||
} `json:"input"` |
|||
} `json:"payload"` |
|||
} |
|||
if err := json.Unmarshal(raw, &outer); err != nil { |
|||
t.Fatal(err) |
|||
} |
|||
if outer.Header.AppID != "app" || outer.Header.Status != 3 || outer.Payload.Input.Status != 3 { |
|||
t.Fatalf("信封不对: %+v", outer.Header) |
|||
} |
|||
inner, err := base64.StdEncoding.DecodeString(outer.Payload.Input.Text) |
|||
if err != nil { |
|||
t.Fatal(err) |
|||
} |
|||
if strings.Contains(string(inner), "generalv3") || strings.Contains(string(inner), "spark") { |
|||
t.Fatalf("内层还带着星火: %s", inner) |
|||
} |
|||
var chat struct { |
|||
Model string `json:"model"` |
|||
Stream bool `json:"stream"` |
|||
Messages []Message `json:"messages"` |
|||
} |
|||
if err := json.Unmarshal(inner, &chat); err != nil { |
|||
t.Fatal(err) |
|||
} |
|||
if chat.Model != "gpt-5.3-chat" || chat.Stream || chat.Messages[0].Content != "1+1等于几" { |
|||
t.Fatalf("内层不对: %+v", chat) |
|||
} |
|||
} |
|||
|
|||
func TestParseChatCompletion(t *testing.T) { |
|||
inner, _ := json.Marshal(map[string]any{ |
|||
"object": "chat.completion", |
|||
"choices": []any{ |
|||
map[string]any{"message": map[string]any{"content": "1+1等于2。"}}, |
|||
}, |
|||
}) |
|||
outer, _ := json.Marshal(map[string]any{ |
|||
"header": map[string]any{"code": 0, "message": "success"}, |
|||
"payload": map[string]any{ |
|||
"output": map[string]any{"text": base64.StdEncoding.EncodeToString(inner)}, |
|||
}, |
|||
}) |
|||
got, err := parseOuter(outer) |
|||
if err != nil || got != "1+1等于2。" { |
|||
t.Fatalf("got %q err %v", got, err) |
|||
} |
|||
} |
|||
|
|||
func TestParseGatewayError(t *testing.T) { |
|||
outer, _ := json.Marshal(map[string]any{ |
|||
"header": map[string]any{"code": 11200, "message": "授权错误"}, |
|||
}) |
|||
_, err := parseOuter(outer) |
|||
if err == nil || !strings.Contains(err.Error(), "11200") { |
|||
t.Fatalf("err=%v", err) |
|||
} |
|||
} |
|||
@ -0,0 +1,381 @@ |
|||
// Package ost 调用科大讯飞 SparkChain SDK 里 RegionMSFT 那条录音文件转写。
|
|||
//
|
|||
// SDK 二进制里的引擎名是 iflyrec_voice_mul_cbm_msft_24h，走海外 OST：
|
|||
// 创建 https://<host>/v2/ost/pro_create ，查询 https://<host>/v2/ost/query 。
|
|||
// 音频不经过讯飞上传接口，直接用客户端已经放到 OSS 上的公网地址。
|
|||
package ost |
|||
|
|||
import ( |
|||
"bytes" |
|||
"crypto/hmac" |
|||
"crypto/sha256" |
|||
"encoding/base64" |
|||
"encoding/json" |
|||
"fmt" |
|||
"io" |
|||
"net/http" |
|||
"net/url" |
|||
"strings" |
|||
"time" |
|||
|
|||
"github.com/google/uuid" |
|||
) |
|||
|
|||
const ( |
|||
defaultHost = "ost-api-sg.xf-yun.com" |
|||
defaultTaskType = "iflyrec_voice_mul_cbm_msft_24h" |
|||
createPath = "/v2/ost/create" |
|||
queryPath = "/v2/ost/query" |
|||
) |
|||
|
|||
// Segment 一句转写。StartMs/EndMs 是相对音频开头的毫秒。
|
|||
type Segment struct { |
|||
Text string |
|||
StartMs int64 |
|||
EndMs int64 |
|||
Speaker string |
|||
} |
|||
|
|||
// Client 讯飞 OST 客户端。
|
|||
type Client struct { |
|||
AppID string |
|||
APIKey string |
|||
Secret string |
|||
Host string |
|||
TaskType string |
|||
HTTP *http.Client |
|||
} |
|||
|
|||
// New 用 SparkChain 的 appId / apiKey / apiSecret 建客户端。
|
|||
func New(appID, apiKey, secret, host, taskType string) (*Client, error) { |
|||
if strings.TrimSpace(appID) == "" || strings.TrimSpace(apiKey) == "" || strings.TrimSpace(secret) == "" { |
|||
return nil, fmt.Errorf("iflytek ost: app_id/api_key/api_secret 都不能空") |
|||
} |
|||
host = strings.TrimSpace(host) |
|||
if host == "" { |
|||
host = defaultHost |
|||
} |
|||
host = strings.TrimPrefix(strings.TrimPrefix(host, "https://"), "http://") |
|||
host = strings.TrimRight(host, "/") |
|||
taskType = strings.TrimSpace(taskType) |
|||
if taskType == "" { |
|||
taskType = defaultTaskType |
|||
} |
|||
return &Client{ |
|||
AppID: appID, |
|||
APIKey: apiKey, |
|||
Secret: secret, |
|||
Host: host, |
|||
TaskType: taskType, |
|||
HTTP: &http.Client{Timeout: 45 * time.Second}, |
|||
}, nil |
|||
} |
|||
|
|||
// Create 提交一条公网音频。language 用 BCP-47（zh-CN / en-US），空则按中文。
|
|||
func (c *Client) Create(audioURL string, speaker bool, language string) (string, error) { |
|||
enc, format := audioEncoding(audioURL) |
|||
business := map[string]any{ |
|||
"request_id": strings.ReplaceAll(uuid.NewString(), "-", ""), |
|||
"task_type": c.TaskType, |
|||
"language": ostLanguage(language), |
|||
} |
|||
applySpeaker(business, c.TaskType, speaker, enc) |
|||
body := map[string]any{ |
|||
"common": map[string]string{"app_id": c.AppID}, |
|||
"business": business, |
|||
"data": map[string]any{ |
|||
"audio_url": audioURL, |
|||
"audio_src": "http", |
|||
"format": format, |
|||
"encoding": enc, |
|||
}, |
|||
} |
|||
var resp struct { |
|||
Code int `json:"code"` |
|||
Message string `json:"message"` |
|||
Data struct { |
|||
TaskID string `json:"task_id"` |
|||
} `json:"data"` |
|||
} |
|||
if err := c.post(createPath, body, &resp); err != nil { |
|||
return "", err |
|||
} |
|||
if resp.Code != 0 { |
|||
return "", fmt.Errorf("iflytek ost create: code=%d %s", resp.Code, resp.Message) |
|||
} |
|||
if resp.Data.TaskID == "" { |
|||
return "", fmt.Errorf("iflytek ost create: 没有 task_id") |
|||
} |
|||
return resp.Data.TaskID, nil |
|||
} |
|||
|
|||
// msftSpeakerCap 微软文件转写的说话人数上限。speaker_num 合法范围是 1–35，
|
|||
// 不传时文档默认 1，等于所有人合成一个角色。会议人数事先不知道，按 10 交给引擎去分。
|
|||
const msftSpeakerCap = 10 |
|||
|
|||
// applySpeaker 按引擎写角色分离参数。
|
|||
// 微软通道（task_type 含 msft）认 spkdia，不认讯飞自研的 vspp_on。
|
|||
// 自研通道的 mp3 官方写明不支持角色分离，硬开会失败。
|
|||
func applySpeaker(business map[string]any, taskType string, speaker bool, enc string) { |
|||
if !speaker { |
|||
return |
|||
} |
|||
if strings.Contains(taskType, "msft") { |
|||
business["spkdia"] = 1 |
|||
business["speaker_num"] = msftSpeakerCap |
|||
return |
|||
} |
|||
if enc == "lame" { |
|||
return |
|||
} |
|||
business["vspp_on"] = 1 |
|||
business["speaker_num"] = 0 |
|||
} |
|||
|
|||
// QueryStatus 任务状态。running 表示还要再查，failed 时 err 带原因。
|
|||
type QueryStatus int |
|||
|
|||
const ( |
|||
StatusRunning QueryStatus = iota |
|||
StatusSuccess |
|||
StatusFailed |
|||
) |
|||
|
|||
// Query 查一次。完成时 segments 是按句切开的文本。
|
|||
func (c *Client) Query(taskID string) (QueryStatus, []Segment, error) { |
|||
body := map[string]any{ |
|||
"common": map[string]string{"app_id": c.AppID}, |
|||
"business": map[string]string{"task_id": taskID}, |
|||
} |
|||
var raw map[string]any |
|||
if err := c.post(queryPath, body, &raw); err != nil { |
|||
return StatusFailed, nil, err |
|||
} |
|||
code := intFrom(raw["code"]) |
|||
if code != 0 { |
|||
return StatusFailed, nil, fmt.Errorf("iflytek ost query: code=%d %v", code, raw["message"]) |
|||
} |
|||
data, _ := raw["data"].(map[string]any) |
|||
if data == nil { |
|||
return StatusRunning, nil, nil |
|||
} |
|||
switch fmt.Sprint(data["task_status"]) { |
|||
case "1", "2", "": |
|||
return StatusRunning, nil, nil |
|||
case "3", "4": |
|||
return StatusSuccess, parseLattice(data["result"]), nil |
|||
default: |
|||
return StatusFailed, nil, fmt.Errorf("iflytek ost query: task_status=%v %v", data["task_status"], raw["message"]) |
|||
} |
|||
} |
|||
|
|||
func (c *Client) post(path string, payload any, out any) error { |
|||
raw, err := json.Marshal(payload) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
endpoint := "https://" + c.Host + path |
|||
req, err := http.NewRequest(http.MethodPost, endpoint, bytes.NewReader(raw)) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
req.Host = c.Host |
|||
for k, v := range c.authHeader(path, raw) { |
|||
if k == "Host" { |
|||
continue |
|||
} |
|||
req.Header.Set(k, v) |
|||
} |
|||
client := c.HTTP |
|||
if client.CheckRedirect == nil { |
|||
client.CheckRedirect = func(req *http.Request, via []*http.Request) error { |
|||
return http.ErrUseLastResponse |
|||
} |
|||
} |
|||
resp, err := client.Do(req) |
|||
if err != nil { |
|||
return fmt.Errorf("iflytek ost %s: %w", path, err) |
|||
} |
|||
defer resp.Body.Close() |
|||
buf, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
if resp.StatusCode != http.StatusOK { |
|||
if json.Unmarshal(buf, out) == nil { |
|||
return nil |
|||
} |
|||
return fmt.Errorf("iflytek ost %s: http %d %s", endpoint, resp.StatusCode, truncate(string(buf), 300)) |
|||
} |
|||
if err := json.Unmarshal(buf, out); err != nil { |
|||
return fmt.Errorf("iflytek ost %s: 响应不是 JSON: %s", path, truncate(string(buf), 200)) |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (c *Client) authHeader(path string, body []byte) map[string]string { |
|||
sum := sha256.Sum256(body) |
|||
digest := "SHA-256=" + base64.StdEncoding.EncodeToString(sum[:]) |
|||
date := time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT") |
|||
origin := "host: " + c.Host + "\n" + |
|||
"date: " + date + "\n" + |
|||
"POST " + path + " HTTP/1.1\n" + |
|||
"digest: " + digest |
|||
mac := hmac.New(sha256.New, []byte(c.Secret)) |
|||
_, _ = mac.Write([]byte(origin)) |
|||
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) |
|||
auth := fmt.Sprintf(`api_key="%s", algorithm="hmac-sha256", headers="host date request-line digest", signature="%s"`, |
|||
c.APIKey, signature) |
|||
return map[string]string{ |
|||
"Content-Type": "application/json", |
|||
"Accept": "application/json", |
|||
"Host": c.Host, |
|||
"Date": date, |
|||
"Digest": digest, |
|||
"Authorization": auth, |
|||
} |
|||
} |
|||
|
|||
func ostLanguage(code string) string { |
|||
c := strings.TrimSpace(code) |
|||
if c == "" { |
|||
return "zh-CN" |
|||
} |
|||
c = strings.ReplaceAll(c, "_", "-") |
|||
if !strings.Contains(c, "-") { |
|||
switch strings.ToLower(c) { |
|||
case "zh": |
|||
return "zh-CN" |
|||
case "en": |
|||
return "en-US" |
|||
case "ja": |
|||
return "ja-JP" |
|||
case "ko": |
|||
return "ko-KR" |
|||
} |
|||
} |
|||
return c |
|||
} |
|||
|
|||
func audioEncoding(audioURL string) (encoding, format string) { |
|||
u, err := url.Parse(audioURL) |
|||
path := audioURL |
|||
if err == nil && u.Path != "" { |
|||
path = u.Path |
|||
} |
|||
lower := strings.ToLower(path) |
|||
switch { |
|||
case strings.HasSuffix(lower, ".mp3"): |
|||
return "lame", "audio/mpeg" |
|||
case strings.HasSuffix(lower, ".wav"), strings.HasSuffix(lower, ".pcm"): |
|||
return "raw", "audio/L16;rate=16000" |
|||
default: |
|||
return "raw", "audio/L16;rate=16000" |
|||
} |
|||
} |
|||
|
|||
// parseLattice 把 OST 的 lattice 收成一句一条。json_1best 有时是对象，有时是字符串。
|
|||
func parseLattice(result any) []Segment { |
|||
obj, _ := result.(map[string]any) |
|||
if obj == nil { |
|||
if s, ok := result.(string); ok && s != "" { |
|||
_ = json.Unmarshal([]byte(s), &obj) |
|||
} |
|||
} |
|||
if obj == nil { |
|||
return nil |
|||
} |
|||
lattice, _ := obj["lattice"].([]any) |
|||
if len(lattice) == 0 { |
|||
lattice, _ = obj["lattice2"].([]any) |
|||
} |
|||
out := make([]Segment, 0, len(lattice)) |
|||
for _, item := range lattice { |
|||
m, _ := item.(map[string]any) |
|||
if m == nil { |
|||
continue |
|||
} |
|||
best := unwrapBest(m["json_1best"]) |
|||
st, _ := best["st"].(map[string]any) |
|||
if st == nil { |
|||
continue |
|||
} |
|||
text := wordsOf(st) |
|||
if text == "" { |
|||
continue |
|||
} |
|||
spk := strings.TrimSpace(fmt.Sprint(m["spk"])) |
|||
if spk == "" || spk == "<nil>" { |
|||
spk = strings.TrimSpace(fmt.Sprint(st["rl"])) |
|||
} |
|||
out = append(out, Segment{ |
|||
Text: text, |
|||
StartMs: intFrom(st["bg"]), |
|||
EndMs: intFrom(st["ed"]), |
|||
Speaker: spk, |
|||
}) |
|||
} |
|||
return out |
|||
} |
|||
|
|||
func unwrapBest(v any) map[string]any { |
|||
switch t := v.(type) { |
|||
case map[string]any: |
|||
return t |
|||
case string: |
|||
var m map[string]any |
|||
if json.Unmarshal([]byte(t), &m) == nil { |
|||
return m |
|||
} |
|||
} |
|||
return map[string]any{} |
|||
} |
|||
|
|||
func wordsOf(st map[string]any) string { |
|||
rts, _ := st["rt"].([]any) |
|||
var b strings.Builder |
|||
for _, rt := range rts { |
|||
rtm, _ := rt.(map[string]any) |
|||
wss, _ := rtm["ws"].([]any) |
|||
for _, ws := range wss { |
|||
wsm, _ := ws.(map[string]any) |
|||
cws, _ := wsm["cw"].([]any) |
|||
for _, cw := range cws { |
|||
cwm, _ := cw.(map[string]any) |
|||
if fmt.Sprint(cwm["wp"]) == "g" { |
|||
continue |
|||
} |
|||
w, _ := cwm["w"].(string) |
|||
b.WriteString(w) |
|||
} |
|||
} |
|||
} |
|||
return strings.TrimSpace(b.String()) |
|||
} |
|||
|
|||
func intFrom(v any) int64 { |
|||
switch t := v.(type) { |
|||
case float64: |
|||
return int64(t) |
|||
case json.Number: |
|||
n, _ := t.Int64() |
|||
return n |
|||
case string: |
|||
var n int64 |
|||
fmt.Sscan(t, &n) |
|||
return n |
|||
case int: |
|||
return int64(t) |
|||
case int64: |
|||
return t |
|||
default: |
|||
return 0 |
|||
} |
|||
} |
|||
|
|||
func truncate(s string, n int) string { |
|||
if len(s) <= n { |
|||
return s |
|||
} |
|||
return s[:n] |
|||
} |
|||
@ -0,0 +1,72 @@ |
|||
package ost |
|||
|
|||
import "testing" |
|||
|
|||
func TestParseLattice(t *testing.T) { |
|||
raw := map[string]any{ |
|||
"lattice": []any{ |
|||
map[string]any{ |
|||
"spk": "段落-0", |
|||
"json_1best": map[string]any{ |
|||
"st": map[string]any{ |
|||
"bg": "0", |
|||
"ed": "470", |
|||
"rt": []any{ |
|||
map[string]any{ |
|||
"ws": []any{ |
|||
map[string]any{"cw": []any{map[string]any{"w": "听说", "wp": "n"}}}, |
|||
map[string]any{"cw": []any{map[string]any{"w": "。", "wp": "p"}}}, |
|||
map[string]any{"cw": []any{map[string]any{"w": "", "wp": "g"}}}, |
|||
}, |
|||
}, |
|||
}, |
|||
}, |
|||
}, |
|||
}, |
|||
}, |
|||
} |
|||
segs := parseLattice(raw) |
|||
if len(segs) != 1 { |
|||
t.Fatalf("segs=%d", len(segs)) |
|||
} |
|||
if segs[0].Text != "听说。" || segs[0].StartMs != 0 || segs[0].EndMs != 470 || segs[0].Speaker != "段落-0" { |
|||
t.Fatalf("%+v", segs[0]) |
|||
} |
|||
} |
|||
|
|||
func TestApplySpeaker(t *testing.T) { |
|||
msft := map[string]any{} |
|||
applySpeaker(msft, "iflyrec_voice_mul_cbm_msft_24h", true, "raw") |
|||
if msft["spkdia"] != 1 || msft["speaker_num"] != msftSpeakerCap { |
|||
t.Fatalf("msft %+v", msft) |
|||
} |
|||
if _, ok := msft["vspp_on"]; ok { |
|||
t.Fatal("微软通道不该带 vspp_on") |
|||
} |
|||
off := map[string]any{} |
|||
applySpeaker(off, "iflyrec_voice_mul_cbm_msft_24h", false, "raw") |
|||
if len(off) != 0 { |
|||
t.Fatalf("off %+v", off) |
|||
} |
|||
self := map[string]any{} |
|||
applySpeaker(self, "iflyrec_voice_cn_10m_ed", true, "raw") |
|||
if self["vspp_on"] != 1 || self["speaker_num"] != 0 { |
|||
t.Fatalf("self %+v", self) |
|||
} |
|||
mp3 := map[string]any{} |
|||
applySpeaker(mp3, "iflyrec_voice_cn_10m_ed", true, "lame") |
|||
if len(mp3) != 0 { |
|||
t.Fatalf("mp3 %+v", mp3) |
|||
} |
|||
} |
|||
|
|||
func TestAudioEncoding(t *testing.T) { |
|||
enc, _ := audioEncoding("https://oss.example.com/a/b.MP3?x=1") |
|||
if enc != "lame" { |
|||
t.Fatal(enc) |
|||
} |
|||
enc, format := audioEncoding("https://oss.example.com/a/b.wav") |
|||
if enc != "raw" || format != "audio/L16;rate=16000" { |
|||
t.Fatal(enc, format) |
|||
} |
|||
} |
|||
@ -0,0 +1,294 @@ |
|||
// Package sparkchat 是星火文本 websocket。会议总结不要用它。
|
|||
//
|
|||
// 海外 SDK 的文本模型是 domain=openai，见 sys/iflytek/openaichat。
|
|||
// 这个包留着只因为星火协议和 openai 那条不是同一个地址。
|
|||
package sparkchat |
|||
|
|||
import ( |
|||
"bufio" |
|||
"crypto/hmac" |
|||
"crypto/rand" |
|||
"crypto/sha256" |
|||
"crypto/tls" |
|||
"encoding/base64" |
|||
"encoding/json" |
|||
"fmt" |
|||
"io" |
|||
"net" |
|||
"net/url" |
|||
"strings" |
|||
"time" |
|||
) |
|||
|
|||
const ( |
|||
defaultHost = "spark-api.xf-yun.com" |
|||
defaultDomain = "generalv3.5" |
|||
) |
|||
|
|||
// Message 一轮对话。Role 取 system / user / assistant。
|
|||
type Message struct { |
|||
Role string |
|||
Content string |
|||
} |
|||
|
|||
// Client 星火文本客户端。
|
|||
type Client struct { |
|||
AppID string |
|||
APIKey string |
|||
Secret string |
|||
Host string |
|||
Domain string |
|||
} |
|||
|
|||
// New 用 SparkChain 凭据建客户端。
|
|||
func New(appID, apiKey, secret, host, domain string) (*Client, error) { |
|||
if strings.TrimSpace(appID) == "" || strings.TrimSpace(apiKey) == "" || strings.TrimSpace(secret) == "" { |
|||
return nil, fmt.Errorf("iflytek spark: app_id/api_key/api_secret 都不能空") |
|||
} |
|||
host = strings.TrimSpace(host) |
|||
if host == "" { |
|||
host = defaultHost |
|||
} |
|||
domain = strings.TrimSpace(domain) |
|||
if domain == "" { |
|||
domain = defaultDomain |
|||
} |
|||
return &Client{AppID: appID, APIKey: apiKey, Secret: secret, Host: host, Domain: domain}, nil |
|||
} |
|||
|
|||
// Chat 同步要一段回复。多帧内容拼在一起返回。
|
|||
func (c *Client) Chat(messages []Message) (string, error) { |
|||
if len(messages) == 0 { |
|||
return "", fmt.Errorf("iflytek spark: 没有输入") |
|||
} |
|||
path := domainPath(c.Domain) |
|||
conn, err := c.dial(path) |
|||
if err != nil { |
|||
return "", err |
|||
} |
|||
defer conn.Close() |
|||
_ = conn.SetDeadline(time.Now().Add(90 * time.Second)) |
|||
|
|||
text := make([]map[string]string, 0, len(messages)) |
|||
for _, m := range messages { |
|||
role := m.Role |
|||
if role == "" { |
|||
role = "user" |
|||
} |
|||
text = append(text, map[string]string{"role": role, "content": m.Content}) |
|||
} |
|||
payload := map[string]any{ |
|||
"header": map[string]string{"app_id": c.AppID, "uid": "echomeet"}, |
|||
"parameter": map[string]any{ |
|||
"chat": map[string]any{ |
|||
"domain": c.Domain, |
|||
"temperature": 0.5, |
|||
"max_tokens": 4096, |
|||
}, |
|||
}, |
|||
"payload": map[string]any{ |
|||
"message": map[string]any{"text": text}, |
|||
}, |
|||
} |
|||
raw, err := json.Marshal(payload) |
|||
if err != nil { |
|||
return "", err |
|||
} |
|||
if err := writeText(conn, raw); err != nil { |
|||
return "", err |
|||
} |
|||
var b strings.Builder |
|||
for { |
|||
frame, err := readFrame(conn) |
|||
if err != nil { |
|||
if b.Len() > 0 && err == io.EOF { |
|||
break |
|||
} |
|||
return "", err |
|||
} |
|||
if frame == nil { |
|||
continue |
|||
} |
|||
var resp struct { |
|||
Header struct { |
|||
Code int `json:"code"` |
|||
Message string `json:"message"` |
|||
Status int `json:"status"` |
|||
} `json:"header"` |
|||
Payload struct { |
|||
Choices struct { |
|||
Status int `json:"status"` |
|||
Text []struct { |
|||
Content string `json:"content"` |
|||
} `json:"text"` |
|||
} `json:"choices"` |
|||
} `json:"payload"` |
|||
} |
|||
if json.Unmarshal(frame, &resp) != nil { |
|||
continue |
|||
} |
|||
if resp.Header.Code != 0 { |
|||
return "", fmt.Errorf("iflytek spark: code=%d %s", resp.Header.Code, resp.Header.Message) |
|||
} |
|||
for _, t := range resp.Payload.Choices.Text { |
|||
b.WriteString(t.Content) |
|||
} |
|||
if resp.Header.Status == 2 || resp.Payload.Choices.Status == 2 { |
|||
break |
|||
} |
|||
} |
|||
out := strings.TrimSpace(b.String()) |
|||
if out == "" { |
|||
return "", fmt.Errorf("iflytek spark: 空回复") |
|||
} |
|||
return out, nil |
|||
} |
|||
|
|||
func domainPath(domain string) string { |
|||
switch domain { |
|||
case "general": |
|||
return "/v1.1/chat" |
|||
case "generalv2": |
|||
return "/v2.1/chat" |
|||
case "generalv3": |
|||
return "/v3.1/chat" |
|||
case "4.0Ultra": |
|||
return "/v4.0/chat" |
|||
default: |
|||
return "/v3.5/chat" |
|||
} |
|||
} |
|||
|
|||
func (c *Client) dial(path string) (net.Conn, error) { |
|||
date := time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT") |
|||
origin := "host: " + c.Host + "\n" + "date: " + date + "\n" + "GET " + path + " HTTP/1.1" |
|||
mac := hmac.New(sha256.New, []byte(c.Secret)) |
|||
_, _ = mac.Write([]byte(origin)) |
|||
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) |
|||
authOrigin := fmt.Sprintf(`api_key="%s", algorithm="hmac-sha256", headers="host date request-line", signature="%s"`, |
|||
c.APIKey, signature) |
|||
auth := base64.StdEncoding.EncodeToString([]byte(authOrigin)) |
|||
q := url.Values{} |
|||
q.Set("authorization", auth) |
|||
q.Set("date", date) |
|||
q.Set("host", c.Host) |
|||
conn, err := tls.Dial("tcp", c.Host+":443", &tls.Config{ServerName: c.Host}) |
|||
if err != nil { |
|||
return nil, fmt.Errorf("iflytek spark dial: %w", err) |
|||
} |
|||
key := make([]byte, 16) |
|||
_, _ = rand.Read(key) |
|||
secKey := base64.StdEncoding.EncodeToString(key) |
|||
req := "GET " + path + "?" + q.Encode() + " HTTP/1.1\r\n" + |
|||
"Host: " + c.Host + "\r\n" + |
|||
"Upgrade: websocket\r\n" + |
|||
"Connection: Upgrade\r\n" + |
|||
"Sec-WebSocket-Key: " + secKey + "\r\n" + |
|||
"Sec-WebSocket-Version: 13\r\n\r\n" |
|||
if _, err := io.WriteString(conn, req); err != nil { |
|||
conn.Close() |
|||
return nil, err |
|||
} |
|||
br := bufio.NewReader(conn) |
|||
status, err := br.ReadString('\n') |
|||
if err != nil { |
|||
conn.Close() |
|||
return nil, err |
|||
} |
|||
if !strings.Contains(status, "101") { |
|||
conn.Close() |
|||
return nil, fmt.Errorf("iflytek spark handshake: %s", strings.TrimSpace(status)) |
|||
} |
|||
for { |
|||
line, err := br.ReadString('\n') |
|||
if err != nil { |
|||
conn.Close() |
|||
return nil, err |
|||
} |
|||
if line == "\r\n" || line == "\n" { |
|||
break |
|||
} |
|||
} |
|||
return &bufferedConn{Conn: conn, r: br}, nil |
|||
} |
|||
|
|||
// bufferedConn 把握手时多读进 bufio 的字节交还给后续读帧。
|
|||
type bufferedConn struct { |
|||
net.Conn |
|||
r *bufio.Reader |
|||
} |
|||
|
|||
func (c *bufferedConn) Read(p []byte) (int, error) { return c.r.Read(p) } |
|||
|
|||
func writeText(conn net.Conn, payload []byte) error { |
|||
mask := make([]byte, 4) |
|||
_, _ = rand.Read(mask) |
|||
var hdr []byte |
|||
n := len(payload) |
|||
switch { |
|||
case n < 126: |
|||
hdr = []byte{0x81, byte(n) | 0x80} |
|||
case n < 65536: |
|||
hdr = []byte{0x81, 126 | 0x80, byte(n >> 8), byte(n)} |
|||
default: |
|||
return fmt.Errorf("iflytek spark: 请求过长") |
|||
} |
|||
frame := append(hdr, mask...) |
|||
masked := make([]byte, n) |
|||
for i := 0; i < n; i++ { |
|||
masked[i] = payload[i] ^ mask[i%4] |
|||
} |
|||
frame = append(frame, masked...) |
|||
_, err := conn.Write(frame) |
|||
return err |
|||
} |
|||
|
|||
func readFrame(conn net.Conn) ([]byte, error) { |
|||
for { |
|||
hdr := make([]byte, 2) |
|||
if _, err := io.ReadFull(conn, hdr); err != nil { |
|||
return nil, err |
|||
} |
|||
opcode := hdr[0] & 0x0f |
|||
masked := hdr[1]&0x80 != 0 |
|||
n := int(hdr[1] & 0x7f) |
|||
switch n { |
|||
case 126: |
|||
ext := make([]byte, 2) |
|||
if _, err := io.ReadFull(conn, ext); err != nil { |
|||
return nil, err |
|||
} |
|||
n = int(ext[0])<<8 | int(ext[1]) |
|||
case 127: |
|||
return nil, fmt.Errorf("iflytek spark: 帧过长") |
|||
} |
|||
var mask []byte |
|||
if masked { |
|||
mask = make([]byte, 4) |
|||
if _, err := io.ReadFull(conn, mask); err != nil { |
|||
return nil, err |
|||
} |
|||
} |
|||
payload := make([]byte, n) |
|||
if _, err := io.ReadFull(conn, payload); err != nil { |
|||
return nil, err |
|||
} |
|||
if masked { |
|||
for i := 0; i < n; i++ { |
|||
payload[i] ^= mask[i%4] |
|||
} |
|||
} |
|||
switch opcode { |
|||
case 0x1, 0x0: |
|||
return payload, nil |
|||
case 0x8: |
|||
return nil, io.EOF |
|||
case 0x9: // ping
|
|||
pong := []byte{0x8A, 0} |
|||
_, _ = conn.Write(pong) |
|||
continue |
|||
default: |
|||
continue |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,142 @@ |
|||
# EMAI 助手：端上听写 + 对话 + 合成 —— 设计 |
|||
|
|||
> 版本：v0.2 日期：2026-10-18 |
|||
> 状态：**设计稿，未写代码。** |
|||
> 范围：耳机唤醒和 App 里按住说话，两条入口共用这一套。通话 / 同声 / 面对面 / 音视频不在本文。 |
|||
> 对照：[科大讯飞SparkChain-全面替换第三方-落地文档](科大讯飞SparkChain-全面替换第三方-落地文档.md) 第 5.5 节是「三段接上、先问通」；本文把时间、地点、天气、新闻、个人工具的分工定死。 |
|||
|
|||
--- |
|||
|
|||
## 零、结论 |
|||
|
|||
每轮对话都带上**手机本地时间**和**缓存地址**，只解决「现在几点、今天星期几、我在哪」。 |
|||
|
|||
天气、新闻和其它要联网才知道的问题，模型手里只有这两样事实，答不出来。这两类仍由手机执行工具，把结果喂回模型，再合成。 |
|||
|
|||
个人数据（纪要、拾忆、待办）同样是手机拿登录身份去调已有 MCP。对话本身不经过业务接口。 |
|||
|
|||
--- |
|||
|
|||
## 一、时间地址怎么带（相对「原样塞进每句话」改过的地方） |
|||
|
|||
| | 做法 | 原因 | |
|||
|---|---|---| |
|||
| 时间 | 每一轮用户说完，用手机本地钟**重新拼一次** | 会话可能持续十几分钟。「现在几点」不能用开场那一刻 | |
|||
| 地址 | 只读 `DeviceLocationService` 的缓存，**这一轮不等 GPS** | 定位会拖住开口。缓存规则维持现状：回前台刷新，10 分钟内不重复 | |
|||
| 放哪 | 放进**本轮系统提示**，不拼进用户原话，不写入聊天历史 | 写进原话，模型会当成用户自己说的；写进历史，旧时间会把「现在」答错 | |
|||
| 经纬度 | **不进模型**。只有天气工具要用时，手机自己附上 | 模型念出坐标没有用，还容易被读出来 | |
|||
| 没有位置 | 系统提示写「当前没有位置」，不要写「未知」 | 「未知」会被当成地名 | |
|||
| 专问「我在哪」且缓存空 | 只这一问允许再等一次定位，上限 4 秒，到点仍没有就按「当前没有位置」答 | 别的问题（含天气）不为这一问阻塞 | |
|||
|
|||
时间文案用手机时区，带星期和偏移，例如：`2026年10月18日 星期日 00:59，东八区（UTC+8）`。 |
|||
|
|||
地址文案沿用现在能念出来的那串（完整地址，没有再退省市区街道）。念「我在哪」用这串。查天气只用其中的**城市名**。 |
|||
|
|||
--- |
|||
|
|||
## 二、整轮怎么走 |
|||
|
|||
SDK 没有百炼那种边听边说。一轮是说完再答。耳机协议、问候语、打断、后台保活维持第 5.5 节，本文只换「听完之后谁来答」。 |
|||
|
|||
``` |
|||
麦克风 PCM |
|||
→ 讯飞 SDK 实时转写，区域 RegionMSFT，语种 = 下面第三节 |
|||
→ 一句最终文本 |
|||
→ 本轮系统提示（新时间 + 缓存地址 + 回答规矩）+ 最近对话 + 工具清单 |
|||
→ 讯飞 SDK 文本对话 LLM（端上，不经业务服务器） |
|||
├─ 直接有文字 → 讯飞 SDK 在线合成，区域 RegionMSFT，音色 = 第三节 |
|||
└─ 返回 functionCall → 手机执行工具（最多两轮）→ 结果喂回 LLM → 再合成 |
|||
``` |
|||
|
|||
两条入口（`DeviceAiSessionService` 耳机唤醒、`EmaiController` 按住说话）共用这一个编排。PCM 从哪来、喇叭往哪放，仍由各自的会话服务决定。 |
|||
|
|||
空识别不调用对话模型。工具失败时合成「暂时查不到」，不让模型编一个天气或一条新闻。工具来回超过两轮就停，用已经拿到的结果回答；还什么都没有就说查不到。 |
|||
|
|||
对话模型用 SDK 的文本 `LLM`（海外 `domain=openai`，文档点名 `gpt-5.3-chat` / `gpt-5.4`）。听写和合成才是 SDK 里的微软区域（`RegionMSFT`）。会议总结走同一套 OpenAI 文本模型，不走星火，也不拿来做 EMAI 的实时对话。 |
|||
|
|||
--- |
|||
|
|||
## 三、识别语种和合成音色 |
|||
|
|||
跟设置里的**界面语言**走，不跟用户刚说的那一句走。 |
|||
|
|||
设置页切换语言时，`settings_controller` 已经把该语言收成一条 ASR 区域码（跟随系统则用匹配后的系统语言，兜底 `en-US`）。EMAI 直接用这一条： |
|||
|
|||
| | 取值 | |
|||
|---|---| |
|||
| 实时转写 `lang` | 上面那条区域码，例如界面中文 → `zh-CN` | |
|||
| 合成发音人 | 同一条语言在音色表里的神经语音。性别只有 `1` 用男声；未设置 `0` 和女 `2` 用女声。用户对象还没就绪时用女声 | |
|||
| 回答语言 | 与这条界面语言一致，写进系统提示 | |
|||
|
|||
用户用别的语言插一句，本轮仍按界面语言识别和回答。要换语种，去设置里改，不在对话中途猜。 |
|||
|
|||
--- |
|||
|
|||
## 四、问题分成四类 |
|||
|
|||
系统提示里写明：时间、星期、日期、地点只用本轮给出的事实，不要调用工具，也不要改口。 |
|||
|
|||
| 用户在问 | 谁处理 | 落地 | |
|||
|---|---|---| |
|||
| 几点、星期几、今天几号 | 模型直接答 | 只用本轮时间。不调工具 | |
|||
| 我在哪 | 模型直接答 | 有地址就念地址。没有就说还没有位置，不要编一个城市 | |
|||
| 天气 | 手机执行 `hefeng_weather` | 城市用缓存里的城市名，日期用本轮日期。有城市就不要反问「哪个城市」。没有位置时才问一次城市，问到了再查 | |
|||
| 新闻、实时信息 | 手机执行联网搜索 | 搜索词带上本轮日期，避免「今天」落到模型的训练截止日期。海外节点用 `tavily_web_search`，国内节点用 `bocha_web_search`，和现有 MCP 分组一致 | |
|||
| 我的会议、记忆、待办 | 手机执行个人 MCP | 见第五节。没有的就说没有，不要编 | |
|||
|
|||
天气和搜索的密钥留在现有 MCP 进程里。手机只发起这一次工具调用，不把天气接口嵌进对话链路，也不把密钥放进安装包。 |
|||
|
|||
和风天气的入参是城市名加可选日期，不是经纬度。所以坐标只在缓存里，给不了城市名时不能拿坐标假装能查。 |
|||
|
|||
--- |
|||
|
|||
## 五、个人 MCP |
|||
|
|||
手机调用，数据仍在用户自己的服务器上。这一跳是工具，不是把整段对话交给业务接口。 |
|||
|
|||
沿用现在这 7 个、且要身份的工具： |
|||
|
|||
`search_meeting_notes`、`get_memory_items`、`get_memory_stats`、`get_memory_report`、`get_user_tasks`、`add_user_task`、`cancel_user_task`。 |
|||
|
|||
身份仍用 `user_getmcptoken` 的短时令牌，按工具名附在调用上。令牌不写进系统提示，不进历史，不出现在给模型的工具参数里。模型只看到工具名和参数说明。 |
|||
|
|||
百炼时期令牌要按工具名嵌进 `user_defined_params`，是因为第三方云在代调。换成手机自己调之后，手机在发 MCP 请求时带上令牌即可，形状不再跟百炼的嵌套走。 |
|||
|
|||
`hefeng_weather` 不在这 7 个里面。它和联网搜索不需要用户身份，手机调用时不要附登录令牌。 |
|||
|
|||
--- |
|||
|
|||
## 六、和现在百炼的对应 |
|||
|
|||
| 百炼现在 | 换成 | |
|||
|---|---| |
|||
| 会话开始时把日期、地址放进 `user_prompt_params`，整场不变 | 每轮重算时间；地址仍只读缓存 | |
|||
| `client_info.location` 把经纬度交给云端 | 经纬度不进模型 | |
|||
| 云端插件自己搜网页、自己查天气 | 模型只返回要调哪个工具；手机执行，再把结果喂回去 | |
|||
| 个人 MCP 由云端代调，令牌按工具名塞进会话 | 手机自己调，令牌只附在那一次请求上 | |
|||
| 边听边说 | 做不到。说完一句再答 | |
|||
|
|||
测试服 `aiOnlyIflytek` 已经让百炼 `start` 直接返回失败。本文落地前 EMAI 在测试服起不来。不要为了对照把百炼重新打开。 |
|||
|
|||
--- |
|||
|
|||
## 七、明确不做 |
|||
|
|||
- 不把对话转发到 `user_*` 业务接口，也不在服务端再包一层「代答」。 |
|||
- 不把会议总结的星火接口拿来当 EMAI 的对话模型。 |
|||
- 不在每一轮阻塞等 GPS。 |
|||
- 不把经纬度、令牌、天气密钥写进提示词。 |
|||
- 不改耳机协议字节，不改通话 / 同声 / 面对面 / 音视频的选路。 |
|||
- 第一版不恢复百炼的全双工。工具调用失败不影响「问一句有回答」。 |
|||
|
|||
--- |
|||
|
|||
## 八、验收 |
|||
|
|||
1. 问「现在几点」「今天星期几」：答的是手机本地时间，和会话开始时的时间可以不同。 |
|||
2. 定位开着时问「我在哪」：念缓存地址，不念经纬度。关掉定位再问：说没有位置，不编城市。 |
|||
3. 有城市缓存时问天气：不反问城市，答案来自工具。把工具结果改成失败时，嘴里是「暂时查不到」，不是编出来的温度。 |
|||
4. 问「今天新闻」：搜索词里带当天日期，回答来自搜索结果。 |
|||
5. 问「我最近的会议 / 待办」：走个人 MCP，换一个账号看不到上一个账号的数据。抓对话请求里看不到令牌。 |
|||
6. 设置改成英语后，识别和回答都是英语，音色是该语言的女声；性别设为男之后换成男声。 |
|||
7. 耳机按键和 App 按住说话，问同一句，走的是同一套编排。 |
|||
@ -0,0 +1,184 @@ |
|||
# 同声翻译改走微软端到端 |
|||
|
|||
> 2026-10-18。只覆盖翻译页的同声模式（`simultaneous`）。本文只做设计，不改代码。 |
|||
> 端到端依据 iOS 集成文档《同声传译简介》第 9.3、9.6 节。拼接时的识别依据安卓海外文档《实时语音转写》第 8.2、9.3 节（iOS《实时语音转写简介》只写了微软模式必填 `language_name`，没有语种表）。合成依据安卓海外文档《在线语音合成》的微软发音人表。 |
|||
> 同声这一条覆盖 [科大讯飞SparkChain-全面替换第三方-落地文档.md](科大讯飞SparkChain-全面替换第三方-落地文档.md) 第 5.1 节里「表外拆成星火听写」的写法。面对面、通话、音视频、EMAI 不在本文范围。 |
|||
|
|||
## 结论 |
|||
|
|||
两端语种都在微软同传表里：只开一条 `AST`，机房 `RegionMSFT`（安卓 `RegionType.MSFT_TYPE`）。识别、翻译、合成由这一条会话完成。 |
|||
|
|||
源或目标是中文、粤语、爱尔兰语：微软同传表没有这三语，改走拼接。拼接的识别必须是微软实时转写 `RTASR(RegionMSFT)`，用 `language_name` 指定语种。不要用星火听写 `ASR`（`zh_cn` / `cn_cbm` / `iat`），也不要用新加坡、欧洲机房那两张转写表（`ist_open`、`ist_mul_sp`、`ga_IE`）。 |
|||
|
|||
拼接的合成用 `OnlineTTS(RegionMSFT)`，发音人用微软神经音色。不要用 `xiaoyan`、`x_xiaoyan`、`x_Catherine`，也不要打 `tts-api.xfyun.cn`。 |
|||
|
|||
## 现在为什么一种声音念所有语言 |
|||
|
|||
同声在讯飞听写成功之后，是三段接起来的，没有用到 `AST`。 |
|||
|
|||
1. `_startSparkListen` 调 `SparkChainSession.startListen`，只做听写。 |
|||
2. 整句文本再走原来的翻译。 |
|||
3. `playTranslatedText` 算出的 `voiceName` 在 `_sparkListen` 分支被丢掉，`_playSparkTts` 只把目标语种短码传给 `SparkChainSession.speak`。 |
|||
|
|||
原生合成不看音色表： |
|||
|
|||
| 端 | 英语 | 其余语种 | |
|||
|---|---|---| |
|||
| iOS `ClassicTts`，打 `tts-api.xfyun.cn` | `x2_catherine` | `xiaoyan` | |
|||
| Android `OnlineTTS` | `x_Catherine` | `x_xiaoyan`（爱尔兰语例外，用 `ga-IE-OrlaNeural`） | |
|||
|
|||
日语、韩语、法语都会用中文晓燕念出来。`getIflytekTTSLanguageSpecs()` 里按语种分好的发音人没有接到这条播报上。那张表是讯飞发音人，本方案也不再用。 |
|||
|
|||
## 目标链路 |
|||
|
|||
麦克风 PCM 仍由现在的耳机麦或手机麦采集，SDK 不自己开麦。采集、保活、播放路由保持现有调用点。 |
|||
|
|||
``` |
|||
耳机麦 / 手机麦 PCM |
|||
│ 16 kHz / 16 bit / 单声道，每 40ms 1280 字节 |
|||
▼ |
|||
AST(RegionMSFT) |
|||
start(源语种, "pcm_s16le", 16000) |
|||
use_tts = true |
|||
target_lang = 目标语种 |
|||
tts_target_lang = 目标语种 |
|||
tts_speaker = 该目标语种的默认发音人 |
|||
│ |
|||
├─ resType=asr / trans → 原文、译文上屏（流式，不再发翻译请求） |
|||
└─ resType=tts → ttsResult.audio 交给现有 PcmStreamPlayer |
|||
``` |
|||
|
|||
iOS： |
|||
|
|||
```objc |
|||
AST *ast = [[AST alloc] initWithregion:RegionMSFT delegate:self]; |
|||
[ast addStringParam:@"target_lang" value:@"en"]; |
|||
[ast addBoolParam:@"use_tts" value:YES]; |
|||
[ast addStringParam:@"tts_target_lang" value:@"en"]; |
|||
[ast addStringParam:@"tts_speaker" value:@"JennyNeural"]; |
|||
[ast start:@"ja" audioEncoding:@"pcm_s16le" sampleRate:16000 userInfo:tag]; |
|||
``` |
|||
|
|||
安卓用同一个机房枚举和同一组参数名。这条端到端会话的 `tts_speaker` 用第 9.6 节的短名（`JennyNeural`）。`en-US-JennyNeural` 是拼接路径里 `OnlineTTS` 的写法，不要填进 `AST`。 |
|||
|
|||
方向固定为用户选的源 → 目标。不要用识别回来的语种对调方向。 |
|||
|
|||
停的顺序不变：先停推流和 `AST.stop(NO)`，再关耳机麦，再放保活。 |
|||
|
|||
## 语种和音色 |
|||
|
|||
源语种、目标语种都必须在下表里。默认发音人取该行的第一个；同行其余名字是可换的音色，换的时候整段会话重开，不要在 `write` 过程中改 `tts_speaker`。 |
|||
|
|||
界面短码和表里的 `language` 一致的直接用。应用里的 `zh` 不要写成 `cn` 再送进微软会话：微软这张表没有中文。 |
|||
|
|||
| 语种 | language | 默认发音人 | 还可选 | |
|||
|---|---|---|---| |
|||
| 阿拉伯语 | ar | AbdullahNeural | AliNeural、AmalNeural、AmanyNeural、AminaNeural、AyshaNeural、BasselNeural、FahedNeural、FatimaNeural、HamdanNeural、HamedNeural、HediNeural、ImanNeural、IsmaelNeural、JamalNeural、LailaNeural、LaithNeural、LaylaNeural、MaryamNeural、MoazNeural、MounaNeural、NouraNeural、OmarNeural、RamiNeural、RanaNeural、ReemNeural、SalehNeural、SalmaNeural、SanaNeural、ShakirNeural、TaimNeural、ZariyahNeural | |
|||
| 保加利亚语 | bg | BorislavNeural | KalinaNeural | |
|||
| 孟加拉语 | bn | BashkarNeural | NabanitaNeural、PradeepNeural、TanishaaNeural | |
|||
| 加泰罗尼亚语 | ca | AlbaNeural | EnricNeural、JoanaNeural | |
|||
| 捷克语 | cs | AntoninNeural | VlastaNeural | |
|||
| 丹麦语 | da | ChristelNeural | JeppeNeural | |
|||
| 德语 | de | AmalaNeural | BerndNeural、ChristophNeural、ConradNeural、ElkeNeural、GiselaNeural、IngridNeural、JanNeural、JonasNeural、KasperNeural、KatjaNeural、KillianNeural、KlarissaNeural、KlausNeural、LeniNeural、LouisaNeural、MajaNeural、RalfNeural、TanjaNeural | |
|||
| 希腊语 | el | AthinaNeural | NestorasNeural | |
|||
| 英语 | en | JennyNeural | GuyNeural、AriaNeural、DavisNeural、JaneNeural、JasonNeural、NancyNeural、TonyNeural、WayneNeural、WilliamNeural、RyanNeural、SoniaNeural、LibbyNeural、NatashaNeural、ClaraNeural、LiamNeural、NeerjaNeural、PrabhatNeural、LunaNeural | |
|||
| 西班牙语 | es | ElviraNeural | AlvaroNeural、AbrilNeural、ArnauNeural、DarioNeural、ElenaNeural、TomasNeural、DaliaNeural、JorgeNeural、PalomaNeural、AlonsoNeural | |
|||
| 爱沙尼亚语 | et | AnuNeural | KertNeural | |
|||
| 波斯语 | fa | DilaraNeural | FaridNeural | |
|||
| 芬兰语 | fi | HarriNeural | NooraNeural、SelmaNeural | |
|||
| 法语 | fr | DeniseNeural | HenriNeural、AlainNeural、BrigitteNeural、ClaudeNeural、SylvieNeural、JeanNeural、AntoineNeural | |
|||
| 印地语 | hi | MadhurNeural | SwaraNeural | |
|||
| 匈牙利语 | hu | NoemiNeural | TamasNeural | |
|||
| 印尼语 | id | ArdiNeural | GadisNeural | |
|||
| 意大利语 | it | ElsaNeural | IsabellaNeural、DiegoNeural、BenignoNeural、FabiolaNeural、GianniNeural | |
|||
| 日语 | ja | NanamiNeural | KeitaNeural、AoiNeural、DaichiNeural、MayuNeural、NaokiNeural、ShioriNeural | |
|||
| 韩语 | ko | SunHiNeural | InJoonNeural、BongJinNeural、HyunsuNeural、JiMinNeural、SeoHyeonNeural | |
|||
| 立陶宛语 | lt | LeonasNeural | OnaNeural | |
|||
| 拉脱维亚语 | lv | EveritaNeural | NilsNeural | |
|||
| 马来语 | ms | OsmanNeural | YasminNeural | |
|||
| 荷兰语 | nl | ArnaudNeural | ColetteNeural、FennaNeural、MaartenNeural | |
|||
| 波兰语 | pl | AgnieszkaNeural | MarekNeural、ZofiaNeural | |
|||
| 葡萄牙语 | pt | AntonioNeural | BrendaNeural、DuarteNeural、FranciscaNeural、RaquelNeural、YaraNeural | |
|||
| 罗马尼亚语 | ro | AlinaNeural | EmilNeural | |
|||
| 俄语 | ru | DariyaNeural | DmitryNeural、SvetlanaNeural | |
|||
| 斯洛伐克语 | sk | LukasNeural | ViktoriaNeural | |
|||
| 斯洛文尼亚语 | sl | PetraNeural | RokNeural | |
|||
| 瑞典语 | sv | HilleviNeural | MattiasNeural、SofieNeural | |
|||
| 斯瓦希里语 | sw | DaudiNeural | RafikiNeural、RehemaNeural、ZuriNeural | |
|||
| 泰米尔语 | ta | AnbuNeural | KaniNeural、KumarNeural、PallaviNeural | |
|||
| 泰语 | th | AcharaNeural | NiwatNeural、PremwadeeNeural | |
|||
| 土耳其语 | tr | AhmetNeural | EmelNeural | |
|||
| 乌克兰语 | uk | OstapNeural | PolinaNeural | |
|||
| 乌尔都语 | ur | AsadNeural | GulNeural、SalmanNeural、UzmaNeural | |
|||
| 越南语 | vi | HoaiMyNeural | NamMinhNeural | |
|||
|
|||
文档没有给这些发音人标性别。默认列只是每行的第一个名字，用来保证换语言就换音色。 |
|||
|
|||
## 中文、粤语、爱尔兰语：拼接，但识别走微软 |
|||
|
|||
微软同传表没有这三语。源或目标有其中一种时，不启 `AST`，改成三段： |
|||
|
|||
``` |
|||
耳机麦 / 手机麦 PCM |
|||
▼ |
|||
RTASR(RegionMSFT) 只识别源语种 |
|||
lang = "msft" 固定值，不要把 zh-CN 填进 lang |
|||
accent = "mandarin" 固定值 |
|||
domain = "ist_mul_msft" 固定值 |
|||
language_name = 下表 真正的语种 |
|||
▼ |
|||
整句文本翻译 不再次开识别 |
|||
▼ |
|||
OnlineTTS(RegionMSFT) 按目标语种选微软神经音色 |
|||
▼ |
|||
现有 PcmStreamPlayer |
|||
``` |
|||
|
|||
安卓文档的写法： |
|||
|
|||
```java |
|||
mRTASR.lang("msft"); |
|||
mRTASR.OstAccent("mandarin"); |
|||
mRTASR.OstDomain("ist_mul_msft"); |
|||
mRTASR.setParams("language_name", "zh-CN"); |
|||
``` |
|||
|
|||
iOS 同样用 `RTASR(region: RegionMSFT)`，`language_name` 用下表。现在插件把界面语种直接写进 `engine.lang`，和上面的固定值不符，拼接落地时要改过来。 |
|||
|
|||
识别语种和这条路径的默认音色： |
|||
|
|||
| 界面语种 | 识别 `language_name` | 作为译文时的默认音色 | |
|||
|---|---|---| |
|||
| 普通话 `zh` | `zh-CN` | `zh-CN-XiaoxiaoNeural` | |
|||
| 粤语简体 `yue` | `yue-CN` | `yue-CN-XiaoMinNeural` | |
|||
| 粤语繁体 / 香港 `zh-HK` | `zh-HK` | `zh-HK-HiuMaanNeural` | |
|||
| 爱尔兰语 `ga` | `ga-IE` | `ga-IE-OrlaNeural` | |
|||
|
|||
另一端如果是微软同传表里的语言，识别用该语言的区域码（英语 `en-US`、日语 `ja-JP`），合成用 `OnlineTTS` 的区域码音色（英语 `en-US-JennyNeural`，对应同传表里的 `JennyNeural`）。不要用星火的 `x_John`、`x_Catherine`、`xiaoyan`。 |
|||
|
|||
现在同声的 `startListen` 在非爱尔兰语时走进 `ASR()`，`language=zh_cn`、`domain=iat`。那是星火听写。中文、粤语、爱尔兰语的拼接不准再进这个分支。 |
|||
|
|||
翻译是整句文本，方向仍固定为源 → 目标。中间结果只上屏，整句再翻译、再合成。 |
|||
|
|||
两端都不在微软同传表里、又不是中文、粤语、爱尔兰语：同声不启动，也不退回星火听写。 |
|||
|
|||
## 改哪些地方 |
|||
|
|||
| 位置 | 改动 | |
|||
|---|---| |
|||
| `sparkchain` 插件 iOS / Android | 同传表内：新增 `AST(RegionMSFT)`，`use_tts` 和 `tts_speaker` 按上一节。表外三语：`RTASR` 按上面的四个固定参数加 `language_name` 启动；`OnlineTTS(RegionMSFT)` 用微软神经音色。同声路径不再创建 `ASR`，不再走 `ClassicTts` | |
|||
| `sparkchain_session.dart` | 同声按语种对选一条：`startAst` 或 `startMsftListen`。`startListen` 里现在的星火 `ASR` 不给同声用 | |
|||
| `translation_controller.dart` 同声分支 | 表内：PCM 写入 `AST`，译文用 `trans`，音频用 `tts`。表外三语：PCM 写入微软 `RTASR`，整句再翻译，再用 `OnlineTTS` 播目标语种。播放仍进现在的 `PcmStreamPlayer` | |
|||
| 发音人 | 表内用同传短名；拼接用上表的 `xx-XX-NameNeural`。不要读 `getIflytekTTSLanguageSpecs` | |
|||
|
|||
`SparkChain.init` 的 `apiType` 仍按现有配置。`RegionMSFT` 是这一次会话的机房，不要为了同声把全局节点改掉。 |
|||
|
|||
## 验收 |
|||
|
|||
- 日语 → 英语：一条 `AST`，`tts_speaker=JennyNeural`。没有 `ASR`，没有 `xiaoyan` / `tts-api.xfyun.cn`。 |
|||
- 英语 → 日语：`tts_speaker=NanamiNeural`。两种方向的声音不同。 |
|||
- 中文 → 英语：日志是 `RTASR`、`language_name=zh-CN`、`lang=msft`。译文用 `en-US-JennyNeural`。没有 `ASR`、没有 `zh_cn`、没有晓燕。 |
|||
- 英语 → 中文：识别 `language_name=en-US`，播报 `zh-CN-XiaoxiaoNeural`。 |
|||
- 中文 → 爱尔兰语：识别 `zh-CN`，播报 `ga-IE-OrlaNeural`。 |
|||
- 粤语 → 英语：识别 `yue-CN` 或 `zh-HK`，对上用户选的简体或香港繁体。 |
|||
- 连耳机时收音日志仍是「同声翻译收音：耳机麦」，译文从耳机出。 |
|||
Loading…
Reference in new issue