63 changed files with 6265 additions and 1051 deletions
@ -0,0 +1,392 @@ |
|||||
|
import 'dart:async'; |
||||
|
import 'dart:convert'; |
||||
|
import 'dart:typed_data'; |
||||
|
|
||||
|
import 'package:get/get.dart'; |
||||
|
import 'package:get_storage/get_storage.dart'; |
||||
|
import 'package:http/http.dart' as http; |
||||
|
|
||||
|
import '../../core/utils/logger.dart'; |
||||
|
import '../models/appconfig.dart'; |
||||
|
import '../models/assistant_directive.dart'; |
||||
|
import 'bailian_multimodal_service.dart'; |
||||
|
import 'device_location_service.dart'; |
||||
|
import 'emai_iflytek_turn.dart'; |
||||
|
import 'language_manager.dart'; |
||||
|
import 'mcp_token_service.dart'; |
||||
|
import 'sparkchain_session.dart'; |
||||
|
|
||||
|
/// 耳机唤醒和按住说话共用的一轮编排:讯飞听写 → 文本对话 → 需要时手机调工具 → 合成。 |
||||
|
/// |
||||
|
/// 麦和喇叭不在这里。调用方把 PCM 推进来,把 [audioOut] 播出去。 |
||||
|
class EmaiIflytekDialog { |
||||
|
EmaiIflytekDialog({this.source = 'emai'}); |
||||
|
|
||||
|
static const _tag = 'EmaiIflytek'; |
||||
|
static const upstreamSampleRate = 16000; |
||||
|
static const downstreamSampleRate = 16000; |
||||
|
|
||||
|
final String source; |
||||
|
final state = BailianDialogState.idle.obs; |
||||
|
final _events = StreamController<BailianEvent>.broadcast(); |
||||
|
final _audioOut = StreamController<Uint8List>.broadcast(); |
||||
|
final List<EmaiMemory> _history = []; |
||||
|
|
||||
|
Stream<BailianEvent> get events => _events.stream; |
||||
|
Stream<Uint8List> get audioOut => _audioOut.stream; |
||||
|
|
||||
|
bool _listening = false; |
||||
|
int _gen = 0; |
||||
|
bool _inSpeech = false; |
||||
|
|
||||
|
Future<bool> start() async { |
||||
|
if (!await SparkChainSession.ensure()) return false; |
||||
|
SparkChainSession.asrListener = _onAsr; |
||||
|
final ok = await SparkChainSession.startMsftListen(_asrCode()); |
||||
|
if (!ok) { |
||||
|
SparkChainSession.asrListener = null; |
||||
|
return false; |
||||
|
} |
||||
|
_listening = true; |
||||
|
state.value = BailianDialogState.listening; |
||||
|
_events.add(const BailianEvent(BailianEventType.started)); |
||||
|
_events.add(BailianEvent( |
||||
|
BailianEventType.stateChanged, |
||||
|
state: BailianDialogState.listening, |
||||
|
)); |
||||
|
return true; |
||||
|
} |
||||
|
|
||||
|
void pushAudio(Uint8List pcm) { |
||||
|
if (!_listening || pcm.isEmpty) return; |
||||
|
SparkChainSession.write(pcm); |
||||
|
} |
||||
|
|
||||
|
Future<void> stop() async { |
||||
|
_gen++; |
||||
|
_listening = false; |
||||
|
SparkChainSession.asrListener = null; |
||||
|
await SparkChainSession.stopListen(); |
||||
|
await SparkChainSession.stopSpeak(); |
||||
|
state.value = BailianDialogState.idle; |
||||
|
_events.add(const BailianEvent(BailianEventType.closed)); |
||||
|
} |
||||
|
|
||||
|
/// 问候语。不进对话模型,直接合成。 |
||||
|
Future<void> speak(String text) async { |
||||
|
final gen = _gen; |
||||
|
await _say(text, gen); |
||||
|
} |
||||
|
|
||||
|
void _onAsr(Map<String, dynamic> event) { |
||||
|
if (event['kind'] == 'error') { |
||||
|
Logger.w(_tag, '听写失败 code=${event['code']}'); |
||||
|
return; |
||||
|
} |
||||
|
final text = (event['text'] as String?)?.trim() ?? ''; |
||||
|
final isFinal = event['final'] == true; |
||||
|
if (text.isEmpty) return; |
||||
|
if (!_inSpeech) { |
||||
|
_inSpeech = true; |
||||
|
_events.add(const BailianEvent(BailianEventType.speechStarted)); |
||||
|
} |
||||
|
_events.add(BailianEvent( |
||||
|
BailianEventType.speechContent, |
||||
|
text: text, |
||||
|
isFinal: isFinal, |
||||
|
)); |
||||
|
if (!isFinal) return; |
||||
|
_inSpeech = false; |
||||
|
_events.add(BailianEvent(BailianEventType.speechEnded, text: text)); |
||||
|
unawaited(_answer(text)); |
||||
|
} |
||||
|
|
||||
|
Future<void> _answer(String userText) async { |
||||
|
final gen = ++_gen; |
||||
|
await SparkChainSession.stopSpeak(); |
||||
|
if (EmaiTurn.asksWhere(userText)) { |
||||
|
await DeviceLocationService.maybe?.ensureSpokenPlace(); |
||||
|
} |
||||
|
if (gen != _gen) return; |
||||
|
state.value = BailianDialogState.thinking; |
||||
|
_events.add(BailianEvent( |
||||
|
BailianEventType.stateChanged, |
||||
|
state: BailianDialogState.thinking, |
||||
|
)); |
||||
|
final chinese = _asrCode().toLowerCase().startsWith('zh'); |
||||
|
final overseas = _overseas(); |
||||
|
final tools = EmaiTurn.toolsJson(overseas: overseas); |
||||
|
var query = userText; |
||||
|
var fresh = true; |
||||
|
String spoken = ''; |
||||
|
for (var round = 0; round < 2; round++) { |
||||
|
if (gen != _gen) return; |
||||
|
final SparkLlmReply reply; |
||||
|
try { |
||||
|
reply = await SparkChainSession.chat( |
||||
|
query: query, |
||||
|
system: EmaiTurn.systemPrompt( |
||||
|
now: DateTime.now(), |
||||
|
spokenPlace: DeviceLocationService.maybe?.placeText(), |
||||
|
answerLanguage: chinese ? '中文' : 'English', |
||||
|
history: List<EmaiMemory>.from(_history), |
||||
|
), |
||||
|
tools: tools, |
||||
|
fresh: fresh, |
||||
|
); |
||||
|
} catch (e) { |
||||
|
Logger.w(_tag, '对话失败: $e'); |
||||
|
await _say(EmaiTurn.unavailable(chinese), gen); |
||||
|
return; |
||||
|
} |
||||
|
if (gen != _gen) return; |
||||
|
final calls = EmaiTurn.parseCalls(reply.functionCall); |
||||
|
if (calls.isEmpty) { |
||||
|
spoken = reply.text.trim(); |
||||
|
break; |
||||
|
} |
||||
|
final call = calls.first; |
||||
|
if (call.name == 'quit') { |
||||
|
_events.add(BailianEvent( |
||||
|
BailianEventType.respondingContent, |
||||
|
text: reply.text, |
||||
|
directives: [ |
||||
|
AssistantDirective( |
||||
|
id: 'quit-$gen', |
||||
|
name: 'quit', |
||||
|
args: const {}, |
||||
|
rawArguments: '{}', |
||||
|
dialogId: '', |
||||
|
roundId: '$gen', |
||||
|
llmRequestId: '', |
||||
|
source: source, |
||||
|
createdAt: DateTime.now(), |
||||
|
), |
||||
|
], |
||||
|
)); |
||||
|
final bye = reply.text.trim().isEmpty |
||||
|
? (chinese ? '好的' : 'Okay') |
||||
|
: reply.text.trim(); |
||||
|
await _say(bye, gen); |
||||
|
return; |
||||
|
} |
||||
|
final result = await _runTool(call); |
||||
|
if (gen != _gen) return; |
||||
|
if (result == null) { |
||||
|
await _say(EmaiTurn.unavailable(chinese), gen); |
||||
|
return; |
||||
|
} |
||||
|
fresh = false; |
||||
|
query = '工具 ${call.name} 的结果如下,只根据它回答,不要编造:\n$result'; |
||||
|
} |
||||
|
if (spoken.isEmpty) { |
||||
|
await _say(EmaiTurn.unavailable(chinese), gen); |
||||
|
return; |
||||
|
} |
||||
|
_history.add(EmaiMemory(true, userText)); |
||||
|
_history.add(EmaiMemory(false, spoken)); |
||||
|
if (_history.length > 8) { |
||||
|
_history.removeRange(0, _history.length - 8); |
||||
|
} |
||||
|
await _say(spoken, gen); |
||||
|
} |
||||
|
|
||||
|
Future<void> _say(String text, int gen) async { |
||||
|
if (text.isEmpty || gen != _gen) return; |
||||
|
state.value = BailianDialogState.responding; |
||||
|
_events.add(const BailianEvent(BailianEventType.respondingStarted)); |
||||
|
_events.add(BailianEvent( |
||||
|
BailianEventType.respondingContent, |
||||
|
text: text, |
||||
|
isFinal: true, |
||||
|
finishReason: 'stop', |
||||
|
)); |
||||
|
try { |
||||
|
await SparkChainSession.speakNeural( |
||||
|
voice: _voice(), |
||||
|
text: text, |
||||
|
onPcm: (pcm, _) { |
||||
|
if (gen != _gen || pcm.isEmpty) return; |
||||
|
_audioOut.add(pcm); |
||||
|
}, |
||||
|
); |
||||
|
} catch (e) { |
||||
|
Logger.w(_tag, '合成失败: $e'); |
||||
|
} |
||||
|
if (gen != _gen) return; |
||||
|
_events.add(const BailianEvent(BailianEventType.respondingEnded)); |
||||
|
state.value = BailianDialogState.listening; |
||||
|
_events.add(BailianEvent( |
||||
|
BailianEventType.stateChanged, |
||||
|
state: BailianDialogState.listening, |
||||
|
)); |
||||
|
} |
||||
|
|
||||
|
Future<String?> _runTool(EmaiToolCall call) async { |
||||
|
final now = DateTime.now(); |
||||
|
final args = Map<String, dynamic>.from(call.args); |
||||
|
if (call.name == 'hefeng_weather') { |
||||
|
final city = (args['city'] ?? '').toString().trim(); |
||||
|
if (city.isEmpty) { |
||||
|
final cached = _city(); |
||||
|
if (cached.isEmpty) return '没有位置,请向用户询问要查哪个城市,不要编造天气。'; |
||||
|
args['city'] = cached; |
||||
|
} |
||||
|
if ((args['time'] ?? '').toString().trim().isEmpty) { |
||||
|
args['time'] = EmaiTurn.dayText(now); |
||||
|
} |
||||
|
} |
||||
|
if (call.name == 'tavily_web_search' || call.name == 'bocha_web_search') { |
||||
|
args['query'] = EmaiTurn.searchQuery( |
||||
|
(args['query'] ?? '').toString(), |
||||
|
now, |
||||
|
); |
||||
|
} |
||||
|
final authed = EmaiTurn.authTools.contains(call.name); |
||||
|
String? token; |
||||
|
if (authed) { |
||||
|
token = await McpTokenService.get(); |
||||
|
if (token == null || token.isEmpty) return null; |
||||
|
} |
||||
|
final url = _mcpUrl(call.name); |
||||
|
if (url == null) return null; |
||||
|
try { |
||||
|
final resp = await http |
||||
|
.post( |
||||
|
url, |
||||
|
headers: { |
||||
|
'Content-Type': 'application/json', |
||||
|
'Accept': 'application/json, text/event-stream', |
||||
|
if (token != null) 'Authorization': 'Bearer $token', |
||||
|
}, |
||||
|
body: jsonEncode({ |
||||
|
'jsonrpc': '2.0', |
||||
|
'id': 1, |
||||
|
'method': 'tools/call', |
||||
|
'params': {'name': call.name, 'arguments': args}, |
||||
|
}), |
||||
|
) |
||||
|
.timeout(const Duration(seconds: 12)); |
||||
|
if (resp.statusCode < 200 || resp.statusCode >= 300) return null; |
||||
|
return _toolText(resp.body); |
||||
|
} catch (e) { |
||||
|
Logger.w(_tag, '工具 ${call.name} 失败'); |
||||
|
return null; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
String? _toolText(String body) { |
||||
|
final payloads = <String>[]; |
||||
|
for (final line in body.split('\n')) { |
||||
|
final trimmed = line.trim(); |
||||
|
if (trimmed.startsWith('data:')) { |
||||
|
payloads.add(trimmed.substring(5).trim()); |
||||
|
} |
||||
|
} |
||||
|
if (payloads.isEmpty) payloads.add(body); |
||||
|
for (final payload in payloads.reversed) { |
||||
|
if (payload.isEmpty) continue; |
||||
|
try { |
||||
|
final decoded = jsonDecode(payload); |
||||
|
if (decoded is! Map) continue; |
||||
|
if (decoded['error'] != null) return null; |
||||
|
final result = decoded['result']; |
||||
|
if (result is! Map) continue; |
||||
|
if (result['isError'] == true) return null; |
||||
|
final content = result['content']; |
||||
|
if (content is List) { |
||||
|
final texts = content |
||||
|
.whereType<Map>() |
||||
|
.map((m) => (m['text'] ?? '').toString()) |
||||
|
.where((s) => s.isNotEmpty) |
||||
|
.join('\n'); |
||||
|
if (texts.isNotEmpty) return texts; |
||||
|
} |
||||
|
final text = result['text']; |
||||
|
if (text is String && text.isNotEmpty) return text; |
||||
|
} catch (_) {} |
||||
|
} |
||||
|
return null; |
||||
|
} |
||||
|
|
||||
|
Uri? _mcpUrl(String tool) { |
||||
|
if (!AppConfig.isInitialized()) return null; |
||||
|
final raw = AppConfig.mcpConfig(); |
||||
|
final servers = raw is Map ? raw['mcpServers'] : null; |
||||
|
if (servers is! Map) return null; |
||||
|
String? fallback; |
||||
|
for (final value in servers.values) { |
||||
|
if (value is! Map && value.runtimeType.toString() != 'DBMCPServer') { |
||||
|
// generated model |
||||
|
} |
||||
|
final url = _serverUrl(value); |
||||
|
final tools = _serverTools(value); |
||||
|
if (url == null) continue; |
||||
|
fallback ??= url; |
||||
|
if (tools.contains(tool)) return Uri.parse(url); |
||||
|
} |
||||
|
if (fallback == null) return null; |
||||
|
return Uri.parse(fallback); |
||||
|
} |
||||
|
|
||||
|
String? _serverUrl(Object value) { |
||||
|
try { |
||||
|
final url = (value as dynamic).url?.toString().trim() ?? ''; |
||||
|
if (url.isEmpty) return null; |
||||
|
if (url.contains('/mcp')) return url; |
||||
|
return url.endsWith('/') ? '${url}mcp' : '$url/mcp'; |
||||
|
} catch (_) { |
||||
|
return null; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
String _serverTools(Object value) { |
||||
|
try { |
||||
|
return ((value as dynamic).tools ?? '').toString(); |
||||
|
} catch (_) { |
||||
|
return ''; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
String _city() { |
||||
|
final place = DeviceLocationService.maybe; |
||||
|
if (place == null) return ''; |
||||
|
final loc = place.clientLocation(); |
||||
|
return (loc?['city_name'] ?? '').trim(); |
||||
|
} |
||||
|
|
||||
|
bool _overseas() { |
||||
|
final raw = AppConfig.cred('sparkchain', 'api_type', 'SPARKCHAIN_API_TYPE'); |
||||
|
final apiType = int.tryParse(raw ?? '') ?? 1; |
||||
|
return apiType != 0; |
||||
|
} |
||||
|
|
||||
|
String _asrCode() { |
||||
|
final saved = GetStorage().read<String>('language') ?? 'system'; |
||||
|
if (saved != 'system') { |
||||
|
final parts = saved.split('_'); |
||||
|
if (parts.length == 2) { |
||||
|
final lang = parts[0].toLowerCase(); |
||||
|
final country = parts[1].toUpperCase(); |
||||
|
if (lang == 'tl' && country == 'PH') return 'fil-PH'; |
||||
|
return '$lang-$country'; |
||||
|
} |
||||
|
} |
||||
|
final locale = Get.locale ?? Get.deviceLocale; |
||||
|
if (locale != null && (locale.countryCode ?? '').isNotEmpty) { |
||||
|
return '${locale.languageCode.toLowerCase()}-${locale.countryCode!.toUpperCase()}'; |
||||
|
} |
||||
|
return 'en-US'; |
||||
|
} |
||||
|
|
||||
|
String _voice() { |
||||
|
final code = _asrCode(); |
||||
|
if (Get.isRegistered<LanguageManager>()) { |
||||
|
final voice = Get.find<LanguageManager>().getTtsVoiceNameByAsrCode(code); |
||||
|
if (voice != null && voice.isNotEmpty) return voice; |
||||
|
} |
||||
|
return code.toLowerCase().startsWith('zh') |
||||
|
? 'zh-CN-XiaoxiaoNeural' |
||||
|
: 'en-US-AriaNeural'; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,190 @@ |
|||||
|
import 'dart:convert'; |
||||
|
|
||||
|
/// 一轮 EMAI 对话里和 SDK 无关的部分:系统提示、工具清单、工具参数。 |
||||
|
class EmaiTurn { |
||||
|
EmaiTurn._(); |
||||
|
|
||||
|
static const authTools = [ |
||||
|
'search_meeting_notes', |
||||
|
'get_memory_items', |
||||
|
'get_memory_stats', |
||||
|
'get_memory_report', |
||||
|
'get_user_tasks', |
||||
|
'add_user_task', |
||||
|
'cancel_user_task', |
||||
|
]; |
||||
|
|
||||
|
static String clockText(DateTime t) { |
||||
|
const week = ['星期一', '星期二', '星期三', '星期四', '星期五', '星期六', '星期日']; |
||||
|
String two(int v) => v.toString().padLeft(2, '0'); |
||||
|
final offset = t.timeZoneOffset; |
||||
|
final sign = offset.isNegative ? '-' : '+'; |
||||
|
final hours = offset.inHours.abs(); |
||||
|
final mins = offset.inMinutes.abs() % 60; |
||||
|
final zone = mins == 0 ? 'UTC$sign$hours' : 'UTC$sign$hours:${two(mins)}'; |
||||
|
return '${t.year}年${t.month}月${t.day}日 ${week[t.weekday - 1]} ' |
||||
|
'${two(t.hour)}:${two(t.minute)},$zone'; |
||||
|
} |
||||
|
|
||||
|
static String dayText(DateTime t) { |
||||
|
String two(int v) => v.toString().padLeft(2, '0'); |
||||
|
return '${t.year}-${two(t.month)}-${two(t.day)}'; |
||||
|
} |
||||
|
|
||||
|
/// 没有能念的地址时用这句。不要写「未知」,模型会把它当成地名。 |
||||
|
static String placeLine(String? spoken) { |
||||
|
final text = (spoken ?? '').trim(); |
||||
|
if (text.isEmpty || text == '未知') return '当前没有位置'; |
||||
|
return text; |
||||
|
} |
||||
|
|
||||
|
static bool asksWhere(String text) { |
||||
|
final s = text.toLowerCase(); |
||||
|
return s.contains('在哪') || |
||||
|
s.contains('我的位置') || |
||||
|
s.contains('where am i') || |
||||
|
s.contains('where i am'); |
||||
|
} |
||||
|
|
||||
|
static String unavailable(bool chinese) => |
||||
|
chinese ? '暂时查不到' : "I can't look that up right now."; |
||||
|
|
||||
|
static String systemPrompt({ |
||||
|
required DateTime now, |
||||
|
required String? spokenPlace, |
||||
|
required String answerLanguage, |
||||
|
required List<EmaiMemory> history, |
||||
|
}) { |
||||
|
final lines = history |
||||
|
.map((m) => '${m.user ? "用户" : "助手"}:${m.text}') |
||||
|
.join('\n'); |
||||
|
return ''' |
||||
|
你是 EAIMAR 的语音助手。用$answerLanguage回答,句子短,适合念出来。 |
||||
|
当前时间:${clockText(now)} |
||||
|
当前位置:${placeLine(spokenPlace)} |
||||
|
时间、星期、日期、地点只用上面这两行,不要调用工具,也不要改口。 |
||||
|
没有位置时说还没有位置,不要编一个城市,不要念经纬度。 |
||||
|
天气用 hefeng_weather,城市用上面的城市名,日期用今天。有城市就不要反问哪个城市。没有位置时才问一次城市。 |
||||
|
新闻和需要联网的问题用搜索工具,搜索词里带上今天的日期 ${dayText(now)}。 |
||||
|
会议、记忆、待办用对应的个人工具。没有就说没有,不要编。 |
||||
|
用户要结束对话时调用 quit。 |
||||
|
${lines.isEmpty ? '' : '最近对话:\n$lines'} |
||||
|
'''.trim(); |
||||
|
} |
||||
|
|
||||
|
static String toolsJson({required bool overseas}) { |
||||
|
final search = overseas ? 'tavily_web_search' : 'bocha_web_search'; |
||||
|
final tools = [ |
||||
|
_fn('hefeng_weather', '按城市名查询天气', { |
||||
|
'city': _str('城市名,例如北京'), |
||||
|
'time': _str('日期 yyyy-MM-dd,可省略'), |
||||
|
}, const ['city']), |
||||
|
_fn(search, '联网搜索新闻和实时信息', { |
||||
|
'query': _str('搜索词,要带上日期'), |
||||
|
'count': {'type': 'integer', 'description': '条数,1到5'}, |
||||
|
}, const ['query']), |
||||
|
_fn('search_meeting_notes', '查用户自己的会议纪要', { |
||||
|
'keywords': _str('关键词,可空'), |
||||
|
}, const []), |
||||
|
_fn('get_memory_items', '查用户记下的事项', { |
||||
|
'category': _str('todo、alarm、idea、expense,可空'), |
||||
|
}, const []), |
||||
|
_fn('get_memory_stats', '查记忆的数量统计', {}, const []), |
||||
|
_fn('get_memory_report', '查记忆的汇总', {}, const []), |
||||
|
_fn('get_user_tasks', '查提醒和待办', {}, const []), |
||||
|
_fn('add_user_task', '添加一条提醒', { |
||||
|
'task_name': _str('标题'), |
||||
|
'trigger_time': _str('带时区的 ISO 8601 时间'), |
||||
|
}, const ['task_name', 'trigger_time']), |
||||
|
_fn('cancel_user_task', '取消一条提醒', { |
||||
|
'task_id': {'type': 'integer', 'description': '任务 id'}, |
||||
|
}, const ['task_id']), |
||||
|
_fn('quit', '用户要结束这次对话', {}, const []), |
||||
|
]; |
||||
|
return jsonEncode(tools); |
||||
|
} |
||||
|
|
||||
|
static Map<String, dynamic> _str(String description) => { |
||||
|
'type': 'string', |
||||
|
'description': description, |
||||
|
}; |
||||
|
|
||||
|
static Map<String, dynamic> _fn( |
||||
|
String name, |
||||
|
String description, |
||||
|
Map<String, dynamic> properties, |
||||
|
List<String> required, |
||||
|
) { |
||||
|
return { |
||||
|
'type': 'function', |
||||
|
'function': { |
||||
|
'name': name, |
||||
|
'description': description, |
||||
|
'parameters': { |
||||
|
'type': 'object', |
||||
|
'properties': properties, |
||||
|
'required': required, |
||||
|
}, |
||||
|
}, |
||||
|
}; |
||||
|
} |
||||
|
|
||||
|
/// SDK 的 functionCall 可能是对象、对象数组,或 arguments 再包一层字符串。 |
||||
|
static List<EmaiToolCall> parseCalls(String raw) { |
||||
|
final text = raw.trim(); |
||||
|
if (text.isEmpty) return const []; |
||||
|
try { |
||||
|
final decoded = jsonDecode(text); |
||||
|
if (decoded is List) { |
||||
|
return decoded.map(_one).whereType<EmaiToolCall>().toList(); |
||||
|
} |
||||
|
if (decoded is Map) { |
||||
|
final one = _one(decoded); |
||||
|
return one == null ? const [] : [one]; |
||||
|
} |
||||
|
} catch (_) {} |
||||
|
return const []; |
||||
|
} |
||||
|
|
||||
|
static EmaiToolCall? _one(Object? value) { |
||||
|
if (value is! Map) return null; |
||||
|
final map = value.map((k, v) => MapEntry('$k', v)); |
||||
|
final fn = map['function']; |
||||
|
final nested = fn is Map ? fn.map((k, v) => MapEntry('$k', v)) : map; |
||||
|
final name = (nested['name'] ?? map['name'] ?? '').toString().trim(); |
||||
|
if (name.isEmpty) return null; |
||||
|
final args = _args(nested['arguments'] ?? map['arguments']); |
||||
|
args.remove('auth_token'); |
||||
|
args.remove('uid'); |
||||
|
return EmaiToolCall(name, args); |
||||
|
} |
||||
|
|
||||
|
static Map<String, dynamic> _args(Object? value) { |
||||
|
if (value is Map) return value.map((k, v) => MapEntry('$k', v)); |
||||
|
if (value is String && value.trim().isNotEmpty) { |
||||
|
try { |
||||
|
final decoded = jsonDecode(value); |
||||
|
if (decoded is Map) return decoded.map((k, v) => MapEntry('$k', v)); |
||||
|
} catch (_) {} |
||||
|
} |
||||
|
return {}; |
||||
|
} |
||||
|
|
||||
|
static String searchQuery(String query, DateTime now) { |
||||
|
final day = dayText(now); |
||||
|
if (query.contains(day)) return query; |
||||
|
return '$query $day'.trim(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
class EmaiMemory { |
||||
|
EmaiMemory(this.user, this.text); |
||||
|
final bool user; |
||||
|
final String text; |
||||
|
} |
||||
|
|
||||
|
class EmaiToolCall { |
||||
|
EmaiToolCall(this.name, this.args); |
||||
|
final String name; |
||||
|
final Map<String, dynamic> args; |
||||
|
} |
||||
@ -0,0 +1,169 @@ |
|||||
|
/// 同声翻译选哪条讯飞微软会话。 |
||||
|
/// |
||||
|
/// 两端都在同传表里:一条 AST,发音人用短名。 |
||||
|
/// 源或目标是中文、粤语、爱尔兰语:不启 AST,识别走微软实时转写,合成用区域码音色。 |
||||
|
/// 两边都不在这两类里:不启动。 |
||||
|
class SimulRoute { |
||||
|
const SimulRoute.ast({ |
||||
|
required this.lang, |
||||
|
required this.targetLang, |
||||
|
required this.speaker, |
||||
|
}) : ast = true, |
||||
|
languageName = ''; |
||||
|
|
||||
|
const SimulRoute.splice({ |
||||
|
required this.languageName, |
||||
|
required this.speaker, |
||||
|
}) : ast = false, |
||||
|
lang = '', |
||||
|
targetLang = ''; |
||||
|
|
||||
|
final bool ast; |
||||
|
/// AST 的源语种短码,例如 ja。 |
||||
|
final String lang; |
||||
|
/// AST 的目标语种短码,例如 en。 |
||||
|
final String targetLang; |
||||
|
/// AST 用短名(JennyNeural)。拼接用区域码音色(en-US-JennyNeural)。 |
||||
|
final String speaker; |
||||
|
/// 拼接识别的 language_name,例如 zh-CN。 |
||||
|
final String languageName; |
||||
|
} |
||||
|
|
||||
|
class SimulMicrosoft { |
||||
|
static SimulRoute? plan(String source, String target) { |
||||
|
final from = _norm(source); |
||||
|
final to = _norm(target); |
||||
|
if (_ast.containsKey(from) && _ast.containsKey(to)) { |
||||
|
return SimulRoute.ast( |
||||
|
lang: from, |
||||
|
targetLang: to, |
||||
|
speaker: _ast[to]!.speaker, |
||||
|
); |
||||
|
} |
||||
|
if (_splice.containsKey(from) || _splice.containsKey(to)) { |
||||
|
final locale = _listenLocale(from); |
||||
|
final voice = _speakVoice(to); |
||||
|
if (locale == null || voice == null) return null; |
||||
|
return SimulRoute.splice(languageName: locale, speaker: voice); |
||||
|
} |
||||
|
return null; |
||||
|
} |
||||
|
|
||||
|
/// 同传能力 ast_msft 没授权时,用同一对语种走微软转写加合成。 |
||||
|
/// 识别用区域码,合成用完整神经音色,不再用 AST 的短名。 |
||||
|
static SimulRoute spliceInstead(SimulRoute route) { |
||||
|
final locale = _listenLocale(route.lang); |
||||
|
final voice = _speakVoice(route.targetLang); |
||||
|
if (locale == null || voice == null) return route; |
||||
|
return SimulRoute.splice(languageName: locale, speaker: voice); |
||||
|
} |
||||
|
|
||||
|
/// 界面短码收成表里的键。香港繁体要在「凡是 zh- 都当普通话」之前认出来。 |
||||
|
static String _norm(String code) { |
||||
|
final raw = code.trim().toLowerCase().replaceAll('_', '-'); |
||||
|
if (raw == 'yue' || raw.startsWith('yue-')) return 'yue'; |
||||
|
if (raw == 'ga' || raw.startsWith('ga-')) return 'ga'; |
||||
|
if (raw == 'zh-hant-hk' || raw == 'zh-hk') return 'zh-HK'; |
||||
|
if (raw == 'zh-hant' || raw == 'zh-tw' || raw.startsWith('zh-hant')) { |
||||
|
return 'zh-Hant'; |
||||
|
} |
||||
|
if (raw == 'zh' || raw.startsWith('zh-')) return 'zh'; |
||||
|
final dash = raw.indexOf('-'); |
||||
|
return dash < 0 ? raw : raw.substring(0, dash); |
||||
|
} |
||||
|
|
||||
|
/// 讯飞 SDK 文本翻译(gtrans)的语种码。中文用 zh-CN,不要收成 zh。 |
||||
|
static String translateCode(String code) { |
||||
|
switch (_norm(code)) { |
||||
|
case 'zh': |
||||
|
return 'zh-CN'; |
||||
|
case 'zh-Hant': |
||||
|
return 'zh-TW'; |
||||
|
case 'zh-HK': |
||||
|
return 'zh-HK'; |
||||
|
case 'yue': |
||||
|
return 'yue'; |
||||
|
case 'ga': |
||||
|
return 'ga'; |
||||
|
default: |
||||
|
return _norm(code); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
static String? _listenLocale(String key) { |
||||
|
final splice = _splice[key]; |
||||
|
if (splice != null) return splice.locale; |
||||
|
return _ast[key]?.locale; |
||||
|
} |
||||
|
|
||||
|
static String? _speakVoice(String key) { |
||||
|
final splice = _splice[key]; |
||||
|
if (splice != null) return splice.voice; |
||||
|
final ast = _ast[key]; |
||||
|
if (ast == null) return null; |
||||
|
return '${ast.locale}-${ast.speaker}'; |
||||
|
} |
||||
|
|
||||
|
/// 同传表。speaker 是 AST 的短名,locale 是拼接到 OnlineTTS 时用的区域码。 |
||||
|
static const _ast = <String, _AstLang>{ |
||||
|
'ar': _AstLang('ar-EG', 'AbdullahNeural'), |
||||
|
'bg': _AstLang('bg-BG', 'BorislavNeural'), |
||||
|
'bn': _AstLang('bn-IN', 'BashkarNeural'), |
||||
|
'ca': _AstLang('ca-ES', 'AlbaNeural'), |
||||
|
'cs': _AstLang('cs-CZ', 'AntoninNeural'), |
||||
|
'da': _AstLang('da-DK', 'ChristelNeural'), |
||||
|
'de': _AstLang('de-DE', 'AmalaNeural'), |
||||
|
'el': _AstLang('el-GR', 'AthinaNeural'), |
||||
|
'en': _AstLang('en-US', 'JennyNeural'), |
||||
|
'es': _AstLang('es-ES', 'ElviraNeural'), |
||||
|
'et': _AstLang('et-EE', 'AnuNeural'), |
||||
|
'fa': _AstLang('fa-IR', 'DilaraNeural'), |
||||
|
'fi': _AstLang('fi-FI', 'HarriNeural'), |
||||
|
'fr': _AstLang('fr-FR', 'DeniseNeural'), |
||||
|
'hi': _AstLang('hi-IN', 'MadhurNeural'), |
||||
|
'hu': _AstLang('hu-HU', 'NoemiNeural'), |
||||
|
'id': _AstLang('id-ID', 'ArdiNeural'), |
||||
|
'it': _AstLang('it-IT', 'ElsaNeural'), |
||||
|
'ja': _AstLang('ja-JP', 'NanamiNeural'), |
||||
|
'ko': _AstLang('ko-KR', 'SunHiNeural'), |
||||
|
'lt': _AstLang('lt-LT', 'LeonasNeural'), |
||||
|
'lv': _AstLang('lv-LV', 'EveritaNeural'), |
||||
|
'ms': _AstLang('ms-MY', 'OsmanNeural'), |
||||
|
'nl': _AstLang('nl-NL', 'ArnaudNeural'), |
||||
|
'pl': _AstLang('pl-PL', 'AgnieszkaNeural'), |
||||
|
'pt': _AstLang('pt-BR', 'AntonioNeural'), |
||||
|
'ro': _AstLang('ro-RO', 'AlinaNeural'), |
||||
|
'ru': _AstLang('ru-RU', 'DariyaNeural'), |
||||
|
'sk': _AstLang('sk-SK', 'LukasNeural'), |
||||
|
'sl': _AstLang('sl-SI', 'PetraNeural'), |
||||
|
'sv': _AstLang('sv-SE', 'HilleviNeural'), |
||||
|
'sw': _AstLang('sw-KE', 'DaudiNeural'), |
||||
|
'ta': _AstLang('ta-IN', 'AnbuNeural'), |
||||
|
'th': _AstLang('th-TH', 'AcharaNeural'), |
||||
|
'tr': _AstLang('tr-TR', 'AhmetNeural'), |
||||
|
'uk': _AstLang('uk-UA', 'OstapNeural'), |
||||
|
'ur': _AstLang('ur-IN', 'AsadNeural'), |
||||
|
'vi': _AstLang('vi-VN', 'HoaiMyNeural'), |
||||
|
}; |
||||
|
|
||||
|
/// 微软同传表没有的三语。繁体中文按普通话识别,合成用繁体音色。 |
||||
|
static const _splice = <String, _SpliceLang>{ |
||||
|
'zh': _SpliceLang('zh-CN', 'zh-CN-XiaoxiaoNeural'), |
||||
|
'zh-Hant': _SpliceLang('zh-CN', 'zh-TW-HsiaoChenNeural'), |
||||
|
'yue': _SpliceLang('yue-CN', 'yue-CN-XiaoMinNeural'), |
||||
|
'zh-HK': _SpliceLang('zh-HK', 'zh-HK-HiuMaanNeural'), |
||||
|
'ga': _SpliceLang('ga-IE', 'ga-IE-OrlaNeural'), |
||||
|
}; |
||||
|
} |
||||
|
|
||||
|
class _AstLang { |
||||
|
const _AstLang(this.locale, this.speaker); |
||||
|
final String locale; |
||||
|
final String speaker; |
||||
|
} |
||||
|
|
||||
|
class _SpliceLang { |
||||
|
const _SpliceLang(this.locale, this.voice); |
||||
|
final String locale; |
||||
|
final String voice; |
||||
|
} |
||||
File diff suppressed because it is too large
@ -0,0 +1,50 @@ |
|||||
|
#import "AudioCatch.h" |
||||
|
|
||||
|
NSString *AudioEngineStart(AVAudioEngine *engine) { |
||||
|
@try { |
||||
|
[engine prepare]; |
||||
|
NSError *error = nil; |
||||
|
if (![engine startAndReturnError:&error]) { |
||||
|
return error.localizedDescription ?: @"麦克风引擎启动失败"; |
||||
|
} |
||||
|
return nil; |
||||
|
} @catch (NSException *e) { |
||||
|
return e.reason ?: e.name; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
NSString *AudioEngineInstallTap(AVAudioEngine *engine, AVAudioFormat *format, void *ctx, PhoneMicTap tap) { |
||||
|
@try { |
||||
|
if (format.sampleRate <= 0 || format.channelCount == 0) { |
||||
|
return @"输入格式未就绪"; |
||||
|
} |
||||
|
AVAudioInputNode *input = engine.inputNode; |
||||
|
[input installTapOnBus:0 bufferSize:4096 format:format block:^(AVAudioPCMBuffer *buffer, AVAudioTime *when) { |
||||
|
if (tap == NULL || buffer.frameLength == 0) return; |
||||
|
float *const *channels = buffer.floatChannelData; |
||||
|
if (channels == NULL || channels[0] == NULL) return; |
||||
|
tap(ctx, channels[0], (int)buffer.frameLength, buffer.format.sampleRate); |
||||
|
}]; |
||||
|
return nil; |
||||
|
} @catch (NSException *e) { |
||||
|
return e.reason ?: e.name; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
NSString *AudioEngineConnect(AVAudioEngine *engine, AVAudioPlayerNode *node, AVAudioFormat *format) { |
||||
|
@try { |
||||
|
if (format.sampleRate <= 0 || format.channelCount == 0) { |
||||
|
return @"播放格式无效"; |
||||
|
} |
||||
|
if (![engine.attachedNodes containsObject:node]) { |
||||
|
[engine attachNode:node]; |
||||
|
} |
||||
|
if ([engine outputConnectionPointsForNode:node outputBus:0].count == 0) { |
||||
|
[engine connect:node to:engine.mainMixerNode format:format]; |
||||
|
} |
||||
|
engine.mainMixerNode.outputVolume = 1; |
||||
|
return nil; |
||||
|
} @catch (NSException *e) { |
||||
|
return e.reason ?: e.name; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,12 @@ |
|||||
|
#import <AVFoundation/AVFoundation.h> |
||||
|
|
||||
|
typedef void (*PhoneMicTap)(void *_Nonnull ctx, const float *_Nonnull samples, int frames, double sampleRate); |
||||
|
|
||||
|
NSString *_Nullable AudioEngineStart(AVAudioEngine *_Nonnull engine); |
||||
|
NSString *_Nullable AudioEngineInstallTap(AVAudioEngine *_Nonnull engine, |
||||
|
AVAudioFormat *_Nonnull format, |
||||
|
void *_Nonnull ctx, |
||||
|
PhoneMicTap _Nonnull tap); |
||||
|
NSString *_Nullable AudioEngineConnect(AVAudioEngine *_Nonnull engine, |
||||
|
AVAudioPlayerNode *_Nonnull node, |
||||
|
AVAudioFormat *_Nonnull format); |
||||
@ -1,187 +1,249 @@ |
|||||
|
import AudioToolbox |
||||
import AVFoundation |
import AVFoundation |
||||
import Foundation |
import Foundation |
||||
|
|
||||
/// 流式 PCM 外放播放器(16bit 小端、单声道)。 |
/// 流式 PCM 外放(16bit 小端、单声道)。 |
||||
/// |
/// |
||||
/// 存在的理由:iOS 上 **just_audio 放不了「长度未知的直播 WAV 流」**。 |
/// 用 AudioQueue,不用 AVAudioEngine。采集也是一条独立的 AudioQueue。 |
||||
/// 真机实测(iPhone 12 / iOS 26.6)AVPlayer 拿到 just_audio 代理服务给的 |
/// 把播放节点挂到引擎上,节点没连上就 play(),系统会以 |
||||
/// chunked 响应后,只发一次 `Range: bytes=0-1` 的嗅探请求就判定资源不可播, |
/// "player started when in a disconnected state" 直接杀进程 |
||||
/// 立刻回 `-11850 Operation Stopped` 且 processingState 直接落到 idle, |
/// (2026-10-18 09:13 起每次点开始都是这个)。 |
||||
/// **一次都没真正起播**。声明长度、支持 range、把收到的字节全缓存下来允许 |
|
||||
/// 重放,三种都试过,结论一样。ExoPlayer 不发这个嗅探请求,所以 Android |
|
||||
/// 那条路一直是好的 —— 这就是「安卓有声、苹果没声」的全部原因。 |
|
||||
/// |
/// |
||||
/// 这里绕开 AVPlayer + HTTP 代理,直接 AVAudioEngine + AVAudioPlayerNode |
/// 播放队列和麦克风队列各走各的,互不 start / connect。 |
||||
/// 排队播放,和 `AzureTtsHelper` 里那套流式播放同源。 |
/// 不改 AVAudioSession:会话在开麦前已经配成扬声器。 |
||||
/// |
|
||||
/// ⚠️ **不碰 AVAudioSession**:会话由 Dart 侧的 audio_session 统一配置 |
|
||||
/// (连续对话时录音的 voice-processing 也在同一个会话上)。这里再去 |
|
||||
/// setCategory/setActive 就会和录音互相掀桌子。 |
|
||||
class PcmSpeakerPlayer { |
class PcmSpeakerPlayer { |
||||
private let queue = DispatchQueue(label: "azure_speech.pcm_speaker") |
private let lock = NSRecursiveLock() |
||||
|
|
||||
private var engine: AVAudioEngine? |
|
||||
private var node: AVAudioPlayerNode? |
|
||||
private var format: AVAudioFormat? |
|
||||
|
|
||||
/// 已排队但还没播完的缓冲数 |
private var output: AudioQueueRef? |
||||
private var pending = 0 |
private var owned: [AudioQueueBufferRef] = [] |
||||
/// 上层已经调过 end():pending 归零就算这一轮播完 |
private var freeBuffers: [AudioQueueBufferRef] = [] |
||||
|
private var stash = Data() |
||||
|
private var rate: Double = 0 |
||||
private var ended = false |
private var ended = false |
||||
/// 本轮代际,stop() 之后迟到的完成回调按代际丢弃 |
private var inFlight = 0 |
||||
private var generation = 0 |
private var generation = 0 |
||||
|
|
||||
/// 一轮音频播完(或本轮压根没有音频) |
|
||||
var onCompleted: (() -> Void)? |
var onCompleted: (() -> Void)? |
||||
|
|
||||
// MARK: - 对外接口 |
private static let outputCallback: AudioQueueOutputCallback = { userData, _, buffer in |
||||
|
guard let userData else { return } |
||||
|
Unmanaged<PcmSpeakerPlayer>.fromOpaque(userData).takeUnretainedValue().bufferReturned(buffer) |
||||
|
} |
||||
|
|
||||
func start(sampleRate: Double) -> Bool { |
func start(sampleRate: Double, pinSpeaker: Bool = false) -> Bool { |
||||
var ok = false |
lock.lock() |
||||
queue.sync { |
generation += 1 |
||||
generation += 1 |
ended = false |
||||
pending = 0 |
let ok = ensureQueue(sampleRate: sampleRate) |
||||
ended = false |
lock.unlock() |
||||
ok = ensureEngine(sampleRate: sampleRate) |
// 只有明确要求钉扬声器时才改路由。同声翻译不钉: |
||||
|
// 开着麦再改路由,采集会断,第一句话就没了。 |
||||
|
if ok && pinSpeaker { |
||||
|
try? AVAudioSession.sharedInstance().overrideOutputAudioPort(.speaker) |
||||
} |
} |
||||
return ok |
return ok |
||||
} |
} |
||||
|
|
||||
func feed(_ data: Data) { |
func feed(_ data: Data) { |
||||
queue.async { [weak self] in |
guard !data.isEmpty else { return } |
||||
guard let self = self else { return } |
lock.lock() |
||||
guard let node = self.node, let engine = self.engine, engine.isRunning else { return } |
stash.append(data) |
||||
guard let buffer = self.makeBuffer(from: data) else { return } |
let started = pump() |
||||
let gen = self.generation |
let q = output |
||||
self.pending += 1 |
let gen = generation |
||||
// ⚠️ 必须用 .dataPlayedBack 而不是默认的 .dataRendered: |
lock.unlock() |
||||
// 播完这一轮要顺手停引擎(见 finishIfDone),用 dataRendered 会在 |
// 一句播完队列会自己停。停着的时候只入队不会出声,下一句必须再 Start。 |
||||
// 声音真正出完之前就回调,尾音被切掉。 |
// 不在持锁时调用,回调线程里再 Start 会和队列互相等。 |
||||
node.scheduleBuffer(buffer, completionCallbackType: .dataPlayedBack) { [weak self] _ in |
if started { kick(q, generation: gen) } |
||||
self?.queue.async { |
|
||||
guard let self = self, gen == self.generation else { return } |
|
||||
self.pending -= 1 |
|
||||
self.finishIfDone() |
|
||||
} |
|
||||
} |
|
||||
if !node.isPlaying { node.play() } |
|
||||
} |
|
||||
} |
} |
||||
|
|
||||
/// 上层不再喂数据了:已排队的播完就报完成 |
|
||||
func end() { |
func end() { |
||||
queue.async { [weak self] in |
lock.lock() |
||||
guard let self = self else { return } |
ended = true |
||||
self.ended = true |
pump() |
||||
self.finishIfDone() |
let fire = stash.isEmpty && inFlight == 0 |
||||
|
if fire { ended = false } |
||||
|
lock.unlock() |
||||
|
if fire { |
||||
|
let cb = onCompleted |
||||
|
DispatchQueue.main.async { cb?() } |
||||
} |
} |
||||
} |
} |
||||
|
|
||||
/// 打断:立刻停声,不报完成 |
|
||||
func stop() { |
func stop() { |
||||
queue.async { [weak self] in |
lock.lock() |
||||
guard let self = self else { return } |
generation += 1 |
||||
self.generation += 1 |
ended = false |
||||
self.pending = 0 |
stash.removeAll() |
||||
self.ended = false |
inFlight = 0 |
||||
self.node?.stop() |
let q = output |
||||
self.engine?.stop() |
lock.unlock() |
||||
|
if let q { |
||||
|
// true:等回调都回来再往下走。回调里不再把缓冲区丢到别的队列。 |
||||
|
AudioQueueStop(q, true) |
||||
|
} |
||||
|
lock.lock() |
||||
|
if output != nil { |
||||
|
freeBuffers = owned |
||||
|
inFlight = 0 |
||||
} |
} |
||||
|
lock.unlock() |
||||
} |
} |
||||
|
|
||||
func dispose() { |
func dispose() { |
||||
queue.async { [weak self] in |
lock.lock() |
||||
guard let self = self else { return } |
generation += 1 |
||||
self.generation += 1 |
ended = false |
||||
self.node?.stop() |
stash.removeAll() |
||||
self.engine?.stop() |
inFlight = 0 |
||||
self.node = nil |
let q = output |
||||
self.engine = nil |
output = nil |
||||
self.format = nil |
owned.removeAll() |
||||
|
freeBuffers.removeAll() |
||||
|
rate = 0 |
||||
|
lock.unlock() |
||||
|
if let q { |
||||
|
AudioQueueStop(q, true) |
||||
|
AudioQueueDispose(q, true) |
||||
} |
} |
||||
} |
} |
||||
|
|
||||
// MARK: - 内部 |
private func bufferReturned(_ buffer: AudioQueueBufferRef) { |
||||
|
lock.lock() |
||||
private func finishIfDone() { |
// 队列已经拆掉:这块内存马上要被 Dispose,不能再收进空闲表。 |
||||
guard ended, pending <= 0 else { return } |
guard output != nil else { |
||||
ended = false |
lock.unlock() |
||||
// ⚠️ 一轮播完必须把引擎停掉,不能让它空跑着占住音频 IO。 |
return |
||||
// 连续对话下 record 要在同一个会话上开 voice-processing, |
} |
||||
// 我们这台引擎还占着的话它会拿不到 IO,报 |
freeBuffers.append(buffer) |
||||
// `avfaudio error 2003329396`(FourCC 'what')—— 表现就是 |
if inFlight > 0 { inFlight -= 1 } |
||||
// 「助手说完话之后麦克风再也开不起来」。真机上间歇复现过。 |
let started = pump() |
||||
// 对象留着不销毁,下一轮 ensureEngine 直接 start() 就好。 |
let q = output |
||||
node?.stop() |
let gen = generation |
||||
engine?.stop() |
let fire = ended && stash.isEmpty && inFlight == 0 |
||||
let cb = onCompleted |
if fire { ended = false } |
||||
DispatchQueue.main.async { cb?() } |
lock.unlock() |
||||
|
if started { kick(q, generation: gen) } |
||||
|
if fire { |
||||
|
let cb = onCompleted |
||||
|
DispatchQueue.main.async { cb?() } |
||||
|
} |
||||
} |
} |
||||
|
|
||||
/// 采样率变了要整条重建:AVAudioEngine 的连接格式是建链路时定死的 |
private func ensureQueue(sampleRate: Double) -> Bool { |
||||
private func ensureEngine(sampleRate: Double) -> Bool { |
if let output, rate == sampleRate { |
||||
if let engine = engine, let node = node, let fmt = format, |
AudioQueueStart(output, nil) |
||||
fmt.sampleRate == sampleRate { |
|
||||
if !engine.isRunning { |
|
||||
do { try engine.start() } catch { tearDown(); return buildEngine(sampleRate: sampleRate) } |
|
||||
} |
|
||||
if !node.isPlaying { node.play() } |
|
||||
return true |
return true |
||||
} |
} |
||||
tearDown() |
tearDown() |
||||
return buildEngine(sampleRate: sampleRate) |
return buildQueue(sampleRate: sampleRate) |
||||
} |
} |
||||
|
|
||||
private func buildEngine(sampleRate: Double) -> Bool { |
private func buildQueue(sampleRate: Double) -> Bool { |
||||
guard let fmt = AVAudioFormat(standardFormatWithSampleRate: sampleRate, channels: 1) else { |
var format = AudioStreamBasicDescription( |
||||
return false |
mSampleRate: sampleRate, |
||||
|
mFormatID: kAudioFormatLinearPCM, |
||||
|
mFormatFlags: kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked, |
||||
|
mBytesPerPacket: 2, |
||||
|
mFramesPerPacket: 1, |
||||
|
mBytesPerFrame: 2, |
||||
|
mChannelsPerFrame: 1, |
||||
|
mBitsPerChannel: 16, |
||||
|
mReserved: 0 |
||||
|
) |
||||
|
var queue: AudioQueueRef? |
||||
|
let status = AudioQueueNewOutput( |
||||
|
&format, |
||||
|
PcmSpeakerPlayer.outputCallback, |
||||
|
Unmanaged.passUnretained(self).toOpaque(), |
||||
|
nil, |
||||
|
nil, |
||||
|
0, |
||||
|
&queue |
||||
|
) |
||||
|
guard status == noErr, let queue else { return false } |
||||
|
var allocated: [AudioQueueBufferRef] = [] |
||||
|
for _ in 0..<6 { |
||||
|
var buffer: AudioQueueBufferRef? |
||||
|
let alloc = AudioQueueAllocateBuffer(queue, 4096, &buffer) |
||||
|
guard alloc == noErr, let buffer else { |
||||
|
AudioQueueDispose(queue, true) |
||||
|
return false |
||||
|
} |
||||
|
allocated.append(buffer) |
||||
} |
} |
||||
let e = AVAudioEngine() |
let start = AudioQueueStart(queue, nil) |
||||
let n = AVAudioPlayerNode() |
guard start == noErr else { |
||||
e.attach(n) |
AudioQueueDispose(queue, true) |
||||
e.connect(n, to: e.mainMixerNode, format: fmt) |
|
||||
do { |
|
||||
try e.start() |
|
||||
} catch { |
|
||||
NSLog("[PcmSpeakerPlayer] 引擎启动失败: \(error.localizedDescription)") |
|
||||
return false |
return false |
||||
} |
} |
||||
n.play() |
output = queue |
||||
engine = e |
owned = allocated |
||||
node = n |
freeBuffers = allocated |
||||
format = fmt |
rate = sampleRate |
||||
|
inFlight = 0 |
||||
return true |
return true |
||||
} |
} |
||||
|
|
||||
private func tearDown() { |
private func tearDown() { |
||||
node?.stop() |
let q = output |
||||
engine?.stop() |
output = nil |
||||
node = nil |
owned.removeAll() |
||||
engine = nil |
freeBuffers.removeAll() |
||||
format = nil |
rate = 0 |
||||
|
inFlight = 0 |
||||
|
stash.removeAll() |
||||
|
if let q { |
||||
|
// 先放开锁再停,避免回调里再来抢这把锁时和 Stop 互相等。 |
||||
|
lock.unlock() |
||||
|
AudioQueueStop(q, true) |
||||
|
AudioQueueDispose(q, true) |
||||
|
lock.lock() |
||||
|
} |
||||
} |
} |
||||
|
|
||||
/// Int16 小端裸 PCM → Float32 单声道 buffer |
/// 队列停过之后,入队不会自动接着播。换到别的线程再 Start。 |
||||
private func makeBuffer(from data: Data) -> AVAudioPCMBuffer? { |
private func kick(_ queue: AudioQueueRef?, generation gen: Int) { |
||||
guard let fmt = format else { return nil } |
guard let queue else { return } |
||||
let frames = data.count / 2 |
DispatchQueue.global(qos: .userInteractive).async { [weak self] in |
||||
guard frames > 0, |
guard let self else { return } |
||||
let buffer = AVAudioPCMBuffer(pcmFormat: fmt, |
self.lock.lock() |
||||
frameCapacity: AVAudioFrameCount(frames)), |
let current = self.generation == gen ? self.output : nil |
||||
let dst = buffer.floatChannelData else { return nil } |
self.lock.unlock() |
||||
buffer.frameLength = AVAudioFrameCount(frames) |
guard let current else { return } |
||||
data.withUnsafeBytes { raw in |
AudioQueueStart(current, nil) |
||||
guard let base = raw.baseAddress else { return } |
} |
||||
// ⚠️ 不能 bindMemory(to: Int16.self):Data 的起始地址不保证 2 字节对齐, |
} |
||||
// 未对齐访问在 arm64 上是未定义行为。逐字节取再拼。 |
|
||||
let bytes = base.assumingMemoryBound(to: UInt8.self) |
/// 有数据入队时返回 true,调用方据此把停掉的队列重新拉起来。 |
||||
for i in 0..<frames { |
private func pump() -> Bool { |
||||
let lo = UInt16(bytes[i * 2]) |
guard let output else { return false } |
||||
let hi = UInt16(bytes[i * 2 + 1]) |
var started = false |
||||
let sample = Int16(bitPattern: lo | (hi << 8)) |
while !freeBuffers.isEmpty, stash.count >= 2 { |
||||
dst[0][i] = Float(sample) / 32768.0 |
let buffer = freeBuffers.removeFirst() |
||||
|
let capacity = Int(buffer.pointee.mAudioDataBytesCapacity) |
||||
|
var count = min(capacity, stash.count) |
||||
|
count -= count % 2 |
||||
|
guard count >= 2 else { |
||||
|
freeBuffers.append(buffer) |
||||
|
break |
||||
|
} |
||||
|
let dest = buffer.pointee.mAudioData |
||||
|
stash.withUnsafeBytes { raw in |
||||
|
if let base = raw.baseAddress { |
||||
|
memcpy(dest, base, count) |
||||
|
} |
||||
|
} |
||||
|
stash.removeFirst(count) |
||||
|
buffer.pointee.mAudioDataByteSize = UInt32(count) |
||||
|
let status = AudioQueueEnqueueBuffer(output, buffer, 0, nil) |
||||
|
if status != noErr { |
||||
|
freeBuffers.append(buffer) |
||||
|
break |
||||
} |
} |
||||
|
inFlight += 1 |
||||
|
started = true |
||||
} |
} |
||||
return buffer |
return started |
||||
} |
} |
||||
} |
} |
||||
|
|||||
@ -0,0 +1,313 @@ |
|||||
|
import AudioToolbox |
||||
|
import AVFoundation |
||||
|
import Flutter |
||||
|
import Foundation |
||||
|
|
||||
|
enum AudioGraph { |
||||
|
static let queue = DispatchQueue(label: "azure_speech.audio_graph") |
||||
|
} |
||||
|
|
||||
|
/// 同声 / 面对面的手机麦。只采集,不播放。 |
||||
|
/// |
||||
|
/// 用 AudioQueue 输入,不用 AVAudioEngine。引擎上的输入 tap 在这台系统上 |
||||
|
/// 往往只回调一次就停,云端等不到声音,十几秒后报 18802。 |
||||
|
/// 播放是另一条 AudioQueue,不要把播放节点挂到采集上。 |
||||
|
final class PhoneMicCapture: NSObject, FlutterStreamHandler { |
||||
|
static weak var shared: PhoneMicCapture? |
||||
|
|
||||
|
private let lock = NSLock() |
||||
|
private var input: AudioQueueRef? |
||||
|
private var sink: FlutterEventSink? |
||||
|
private var running = false |
||||
|
private var life = 0 |
||||
|
private var sourceRate: Double = 16000 |
||||
|
private var resamplePos: Double = 0 |
||||
|
private var routeObserver: NSObjectProtocol? |
||||
|
private var interruptionObserver: NSObjectProtocol? |
||||
|
|
||||
|
func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { |
||||
|
sink = events |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func onCancel(withArguments arguments: Any?) -> FlutterError? { |
||||
|
sink = nil |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func start(voiceProcessing: Bool, done: @escaping (String?) -> Void) { |
||||
|
_ = voiceProcessing |
||||
|
// 队列必须在主线程启动。放在后台队列时,这台系统只回调一次就停。 |
||||
|
DispatchQueue.main.async { [weak self] in |
||||
|
guard let self else { |
||||
|
done("麦克风已释放") |
||||
|
return |
||||
|
} |
||||
|
self.life += 1 |
||||
|
self.attempt(life: self.life, left: 4, done: done) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/// AudioQueueStart 在会话刚激活时会回 'what'(2003329396)。等一下再激活再试。 |
||||
|
private func attempt(life: Int, left: Int, done: @escaping (String?) -> Void) { |
||||
|
if self.life != life { |
||||
|
tearDownQueue() |
||||
|
done(nil) |
||||
|
return |
||||
|
} |
||||
|
let err = startLocked() |
||||
|
if err == nil || left <= 1 { |
||||
|
done(err) |
||||
|
return |
||||
|
} |
||||
|
tearDownQueue() |
||||
|
DispatchQueue.main.asyncAfter(deadline: .now() + 0.45) { [weak self] in |
||||
|
guard let self else { |
||||
|
done("麦克风已释放") |
||||
|
return |
||||
|
} |
||||
|
let session = AVAudioSession.sharedInstance() |
||||
|
if session.category != .playAndRecord { |
||||
|
try? session.setCategory( |
||||
|
.playAndRecord, |
||||
|
mode: session.mode == .voiceChat ? .voiceChat : .default, |
||||
|
options: [.defaultToSpeaker, .allowBluetoothA2DP] |
||||
|
) |
||||
|
} |
||||
|
try? session.setActive(true) |
||||
|
self.attempt(life: life, left: left - 1, done: done) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func stop(done: @escaping () -> Void) { |
||||
|
DispatchQueue.main.async { [weak self] in |
||||
|
guard let self else { |
||||
|
done() |
||||
|
return |
||||
|
} |
||||
|
self.life += 1 |
||||
|
self.stopLocked() |
||||
|
done() |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/// 输入停了但事件通道还在。拆掉队列再起一条。 |
||||
|
func kick() { |
||||
|
DispatchQueue.main.async { [weak self] in |
||||
|
guard let self, self.running else { return } |
||||
|
let life = self.life |
||||
|
self.tearDownQueue() |
||||
|
if self.life != life { return } |
||||
|
let err = self.startLocked() |
||||
|
if self.life != life { |
||||
|
self.tearDownQueue() |
||||
|
return |
||||
|
} |
||||
|
if err != nil { |
||||
|
self.sink?(["type": "reinstallFailed"]) |
||||
|
} else { |
||||
|
self.sink?(["type": "reinstalled"]) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private func startLocked() -> String? { |
||||
|
if running, input != nil { return nil } |
||||
|
let session = AVAudioSession.sharedInstance() |
||||
|
do { |
||||
|
try session.setActive(true) |
||||
|
} catch { |
||||
|
return "音频会话未激活 \(error.localizedDescription)" |
||||
|
} |
||||
|
// 这台手机把采集写成 16 kHz 时,队列只吐一块就停。停掉的瞬间如果正在连讯飞, |
||||
|
// 重启麦会把握手打断,start 返回 18801。按硬件采样率采,送出前再降到 16 kHz。 |
||||
|
var rates: [Double] = [] |
||||
|
let hw = session.sampleRate |
||||
|
if hw > 1 { rates.append(hw) } |
||||
|
for rate in [48000.0, 44100.0, 16000.0] where !rates.contains(where: { abs($0 - rate) < 1 }) { |
||||
|
rates.append(rate) |
||||
|
} |
||||
|
var last = "麦克风队列启动失败" |
||||
|
for rate in rates { |
||||
|
if let err = openQueue(rate: rate) { |
||||
|
last = err |
||||
|
continue |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
let inputs = session.currentRoute.inputs.map(\.portType.rawValue).joined(separator: ",") |
||||
|
return "\(last) rate=\(hw) cat=\(session.category.rawValue) mode=\(session.mode.rawValue) in=\(inputs)" |
||||
|
} |
||||
|
|
||||
|
/// 按硬件采样率要数据。这个采样率起不来就换下一个,不在同一次失败里空转。 |
||||
|
private func openQueue(rate: Double) -> String? { |
||||
|
sourceRate = rate |
||||
|
resamplePos = 0 |
||||
|
var format = AudioStreamBasicDescription( |
||||
|
mSampleRate: rate, |
||||
|
mFormatID: kAudioFormatLinearPCM, |
||||
|
mFormatFlags: kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked, |
||||
|
mBytesPerPacket: 2, |
||||
|
mFramesPerPacket: 1, |
||||
|
mBytesPerFrame: 2, |
||||
|
mChannelsPerFrame: 1, |
||||
|
mBitsPerChannel: 16, |
||||
|
mReserved: 0 |
||||
|
) |
||||
|
var queue: AudioQueueRef? |
||||
|
let ctx = Unmanaged.passUnretained(self).toOpaque() |
||||
|
let created = AudioQueueNewInput(&format, PhoneMicCapture.inputCallback, ctx, nil, nil, 0, &queue) |
||||
|
guard created == noErr, let queue else { |
||||
|
return "麦克风队列创建失败 \(created) rate=\(rate)" |
||||
|
} |
||||
|
let bytesPerBuffer = UInt32(max(3200, Int(rate) * 2 / 10)) |
||||
|
for _ in 0..<3 { |
||||
|
var buffer: AudioQueueBufferRef? |
||||
|
let allocated = AudioQueueAllocateBuffer(queue, bytesPerBuffer, &buffer) |
||||
|
if allocated != noErr || buffer == nil { |
||||
|
AudioQueueDispose(queue, true) |
||||
|
return "麦克风缓冲创建失败 \(allocated) rate=\(rate)" |
||||
|
} |
||||
|
let queued = AudioQueueEnqueueBuffer(queue, buffer!, 0, nil) |
||||
|
if queued != noErr { |
||||
|
AudioQueueDispose(queue, true) |
||||
|
return "麦克风缓冲入队失败 \(queued) rate=\(rate)" |
||||
|
} |
||||
|
} |
||||
|
lock.lock() |
||||
|
input = queue |
||||
|
running = true |
||||
|
lock.unlock() |
||||
|
let started = AudioQueueStart(queue, nil) |
||||
|
if started != noErr { |
||||
|
tearDownQueue() |
||||
|
return "麦克风队列启动失败 \(started) rate=\(rate)" |
||||
|
} |
||||
|
watchSession() |
||||
|
reportOpened() |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
private func reportOpened() { |
||||
|
let session = AVAudioSession.sharedInstance() |
||||
|
let inputs = session.currentRoute.inputs.map(\.portType.rawValue).joined(separator: ",") |
||||
|
let outputs = session.currentRoute.outputs.map(\.portType.rawValue).joined(separator: ",") |
||||
|
sink?([ |
||||
|
"type": "opened", |
||||
|
"mode": session.mode.rawValue, |
||||
|
"rate": sourceRate, |
||||
|
"input": inputs, |
||||
|
"output": outputs, |
||||
|
]) |
||||
|
} |
||||
|
|
||||
|
private func watchSession() { |
||||
|
if routeObserver == nil { |
||||
|
routeObserver = NotificationCenter.default.addObserver( |
||||
|
forName: AVAudioSession.routeChangeNotification, |
||||
|
object: nil, |
||||
|
queue: nil |
||||
|
) { [weak self] _ in |
||||
|
DispatchQueue.main.async { self?.restartIfStopped() } |
||||
|
} |
||||
|
} |
||||
|
if interruptionObserver == nil { |
||||
|
interruptionObserver = NotificationCenter.default.addObserver( |
||||
|
forName: AVAudioSession.interruptionNotification, |
||||
|
object: nil, |
||||
|
queue: nil |
||||
|
) { [weak self] note in |
||||
|
let raw = note.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt |
||||
|
let type = raw.flatMap { AVAudioSession.InterruptionType(rawValue: $0) } |
||||
|
guard type == .ended else { return } |
||||
|
DispatchQueue.main.async { self?.restartIfStopped() } |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/// 路由或中断把队列停掉时接上。队列还在跑就什么都不做,避免自己把自己重启个没完。 |
||||
|
private func restartIfStopped() { |
||||
|
guard running else { return } |
||||
|
lock.lock() |
||||
|
let queue = input |
||||
|
lock.unlock() |
||||
|
guard let queue else { return } |
||||
|
var flag: UInt32 = 0 |
||||
|
var size = UInt32(MemoryLayout<UInt32>.size) |
||||
|
let status = AudioQueueGetProperty(queue, kAudioQueueProperty_IsRunning, &flag, &size) |
||||
|
if status == noErr, flag != 0 { return } |
||||
|
tearDownQueue() |
||||
|
let err = startLocked() |
||||
|
DispatchQueue.main.async { [weak self] in |
||||
|
if err != nil { |
||||
|
self?.sink?(["type": "reinstallFailed"]) |
||||
|
} else { |
||||
|
self?.sink?(["type": "reinstalled"]) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private func stopLocked() { |
||||
|
lock.lock() |
||||
|
running = false |
||||
|
lock.unlock() |
||||
|
tearDownQueue() |
||||
|
if let routeObserver { |
||||
|
NotificationCenter.default.removeObserver(routeObserver) |
||||
|
self.routeObserver = nil |
||||
|
} |
||||
|
if let interruptionObserver { |
||||
|
NotificationCenter.default.removeObserver(interruptionObserver) |
||||
|
self.interruptionObserver = nil |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private func tearDownQueue() { |
||||
|
lock.lock() |
||||
|
running = false |
||||
|
let queue = input |
||||
|
input = nil |
||||
|
lock.unlock() |
||||
|
guard let queue else { return } |
||||
|
AudioQueueStop(queue, true) |
||||
|
AudioQueueDispose(queue, true) |
||||
|
} |
||||
|
|
||||
|
private static let inputCallback: AudioQueueInputCallback = { ctx, queue, buffer, _, _, _ in |
||||
|
guard let ctx else { return } |
||||
|
let mic = Unmanaged<PhoneMicCapture>.fromOpaque(ctx).takeUnretainedValue() |
||||
|
let size = Int(buffer.pointee.mAudioDataByteSize) |
||||
|
var copied: Data? |
||||
|
if size > 1 { |
||||
|
let frames = size / 2 |
||||
|
copied = mic.downsample(buffer.pointee.mAudioData.assumingMemoryBound(to: Int16.self), frames: frames) |
||||
|
} |
||||
|
mic.lock.lock() |
||||
|
let live = mic.running |
||||
|
mic.lock.unlock() |
||||
|
if live { |
||||
|
AudioQueueEnqueueBuffer(queue, buffer, 0, nil) |
||||
|
} |
||||
|
guard let copied, !copied.isEmpty else { return } |
||||
|
DispatchQueue.main.async { [weak mic] in |
||||
|
mic?.sink?(FlutterStandardTypedData(bytes: copied)) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/// 硬件采样率收进来,送出去的一律是 16 kHz。 |
||||
|
private func downsample(_ samples: UnsafePointer<Int16>, frames: Int) -> Data { |
||||
|
if sourceRate <= 16000 + 1 { |
||||
|
return Data(bytes: samples, count: frames * 2) |
||||
|
} |
||||
|
let ratio = sourceRate / 16000 |
||||
|
var out: [Int16] = [] |
||||
|
out.reserveCapacity(Int(Double(frames) / ratio) + 2) |
||||
|
var cursor = resamplePos |
||||
|
while cursor < Double(frames) { |
||||
|
out.append(samples[Int(cursor)]) |
||||
|
cursor += ratio |
||||
|
} |
||||
|
resamplePos = cursor - Double(frames) |
||||
|
return out.withUnsafeBytes { Data($0) } |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,53 @@ |
|||||
|
import 'package:eaimar/data/services/emai_iflytek_turn.dart'; |
||||
|
import 'package:flutter_test/flutter_test.dart'; |
||||
|
|
||||
|
void main() { |
||||
|
test('时间带星期和时区,提示里不出现经纬度', () { |
||||
|
final now = DateTime(2026, 10, 18, 0, 59); |
||||
|
final prompt = EmaiTurn.systemPrompt( |
||||
|
now: now, |
||||
|
spokenPlace: '上海市浦东新区世纪大道1号', |
||||
|
answerLanguage: '中文', |
||||
|
history: [EmaiMemory(true, '你好')], |
||||
|
); |
||||
|
expect(prompt, contains('2026年10月18日')); |
||||
|
expect(prompt, contains('星期日')); |
||||
|
expect(prompt, contains('00:59')); |
||||
|
expect(prompt, contains('UTC')); |
||||
|
expect(prompt, contains('当前位置:上海市浦东新区世纪大道1号')); |
||||
|
expect(prompt, isNot(contains('31.2'))); |
||||
|
}); |
||||
|
|
||||
|
test('没有地址时写当前没有位置', () { |
||||
|
final prompt = EmaiTurn.systemPrompt( |
||||
|
now: DateTime(2026, 10, 18), |
||||
|
spokenPlace: '未知', |
||||
|
answerLanguage: '中文', |
||||
|
history: const [], |
||||
|
); |
||||
|
expect(prompt, contains('当前没有位置')); |
||||
|
expect(prompt, isNot(contains('当前位置:未知'))); |
||||
|
}); |
||||
|
|
||||
|
test('搜索词补上当天日期', () { |
||||
|
final day = DateTime(2026, 10, 18); |
||||
|
expect(EmaiTurn.searchQuery('今天新闻', day), '今天新闻 2026-10-18'); |
||||
|
expect(EmaiTurn.searchQuery('新闻 2026-10-18', day), '新闻 2026-10-18'); |
||||
|
}); |
||||
|
|
||||
|
test('解析工具调用并丢掉令牌字段', () { |
||||
|
final calls = EmaiTurn.parseCalls( |
||||
|
'{"name":"hefeng_weather","arguments":"{\\"city\\":\\"上海\\",\\"auth_token\\":\\"secret\\"}"}', |
||||
|
); |
||||
|
expect(calls, hasLength(1)); |
||||
|
expect(calls.single.name, 'hefeng_weather'); |
||||
|
expect(calls.single.args['city'], '上海'); |
||||
|
expect(calls.single.args.containsKey('auth_token'), isFalse); |
||||
|
}); |
||||
|
|
||||
|
test('问我在哪才等待定位', () { |
||||
|
expect(EmaiTurn.asksWhere('我现在在哪里'), isTrue); |
||||
|
expect(EmaiTurn.asksWhere('where am I'), isTrue); |
||||
|
expect(EmaiTurn.asksWhere('今天天气怎么样'), isFalse); |
||||
|
}); |
||||
|
} |
||||
@ -0,0 +1,74 @@ |
|||||
|
import 'package:eaimar/data/services/simul_microsoft.dart'; |
||||
|
import 'package:flutter_test/flutter_test.dart'; |
||||
|
|
||||
|
void main() { |
||||
|
test('日语到英语走 AST,发音人是 JennyNeural', () { |
||||
|
final route = SimulMicrosoft.plan('ja', 'en'); |
||||
|
expect(route, isNotNull); |
||||
|
expect(route!.ast, isTrue); |
||||
|
expect(route.lang, 'ja'); |
||||
|
expect(route.targetLang, 'en'); |
||||
|
expect(route.speaker, 'JennyNeural'); |
||||
|
}); |
||||
|
|
||||
|
test('英语到日语同传没授权时改走转写,识别 en-US,合成 Nanami', () { |
||||
|
final route = SimulMicrosoft.plan('en', 'ja')!; |
||||
|
final splice = SimulMicrosoft.spliceInstead(route); |
||||
|
expect(splice.ast, isFalse); |
||||
|
expect(splice.languageName, 'en-US'); |
||||
|
expect(splice.speaker, 'ja-JP-NanamiNeural'); |
||||
|
}); |
||||
|
|
||||
|
test('英语到日语走 AST,发音人是 NanamiNeural', () { |
||||
|
final route = SimulMicrosoft.plan('en-US', 'ja-JP'); |
||||
|
expect(route!.ast, isTrue); |
||||
|
expect(route.lang, 'en'); |
||||
|
expect(route.targetLang, 'ja'); |
||||
|
expect(route.speaker, 'NanamiNeural'); |
||||
|
}); |
||||
|
|
||||
|
test('普通话到英语走拼接,识别 zh-CN,合成 en-US-JennyNeural', () { |
||||
|
final route = SimulMicrosoft.plan('zh', 'en'); |
||||
|
expect(route!.ast, isFalse); |
||||
|
expect(route.languageName, 'zh-CN'); |
||||
|
expect(route.speaker, 'en-US-JennyNeural'); |
||||
|
}); |
||||
|
|
||||
|
test('英语到普通话走拼接,识别 en-US,合成晓晓', () { |
||||
|
final route = SimulMicrosoft.plan('en', 'zh-CN'); |
||||
|
expect(route!.ast, isFalse); |
||||
|
expect(route.languageName, 'en-US'); |
||||
|
expect(route.speaker, 'zh-CN-XiaoxiaoNeural'); |
||||
|
}); |
||||
|
|
||||
|
test('普通话到爱尔兰语识别 zh-CN,合成 Orla', () { |
||||
|
final route = SimulMicrosoft.plan('zh', 'ga'); |
||||
|
expect(route!.ast, isFalse); |
||||
|
expect(route.languageName, 'zh-CN'); |
||||
|
expect(route.speaker, 'ga-IE-OrlaNeural'); |
||||
|
}); |
||||
|
|
||||
|
test('粤语简体识别 yue-CN,香港繁体识别 zh-HK', () { |
||||
|
final yue = SimulMicrosoft.plan('yue', 'en'); |
||||
|
expect(yue!.languageName, 'yue-CN'); |
||||
|
expect(yue.speaker, 'en-US-JennyNeural'); |
||||
|
|
||||
|
final hk = SimulMicrosoft.plan('zh-Hant-hk', 'en'); |
||||
|
expect(hk!.languageName, 'zh-HK'); |
||||
|
expect(hk.speaker, 'en-US-JennyNeural'); |
||||
|
}); |
||||
|
|
||||
|
test('威尔士语到冰岛语不启动', () { |
||||
|
expect(SimulMicrosoft.plan('cy', 'is'), isNull); |
||||
|
}); |
||||
|
|
||||
|
test('拼接翻译语种码给讯飞 SDK,中文保持 zh-CN', () { |
||||
|
expect(SimulMicrosoft.translateCode('zh'), 'zh-CN'); |
||||
|
expect(SimulMicrosoft.translateCode('zh-Hant'), 'zh-TW'); |
||||
|
expect(SimulMicrosoft.translateCode('zh-Hant-hk'), 'zh-HK'); |
||||
|
expect(SimulMicrosoft.translateCode('yue'), 'yue'); |
||||
|
expect(SimulMicrosoft.translateCode('ga'), 'ga'); |
||||
|
expect(SimulMicrosoft.translateCode('ja-JP'), 'ja'); |
||||
|
expect(SimulMicrosoft.translateCode('en'), 'en'); |
||||
|
}); |
||||
|
} |
||||
@ -0,0 +1,112 @@ |
|||||
|
package echomeet |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"fmt" |
||||
|
|
||||
|
"yunyan/pb" |
||||
|
"yunyan/sys/iflytek/openaichat" |
||||
|
"yunyan/sys/iflytek/ost" |
||||
|
) |
||||
|
|
||||
|
// 科大讯飞 SparkChain 的两条服务端能力:
|
||||
|
// 转写用 SDK 里 RegionMSFT 的录音文件引擎(音频地址仍是客户端 OSS 公网 URL);
|
||||
|
// 总结用同一套凭据的海外文本模型,domain=openai(gpt-5.3-chat),不走星火。
|
||||
|
|
||||
|
type iflytekTranscriber struct { |
||||
|
sys *ost.Client |
||||
|
} |
||||
|
|
||||
|
func newIflytekTranscriber(f map[string]string) (Transcriber, error) { |
||||
|
appID, err := need(f, "iflytek", "app_id") |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
key, err := need(f, "iflytek", "api_key") |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
secret, err := need(f, "iflytek", "api_secret") |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
sys, err := ost.New(appID, key, secret, f["host"], f["task_type"]) |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
return &iflytekTranscriber{sys: sys}, nil |
||||
|
} |
||||
|
|
||||
|
func (this *iflytekTranscriber) Submit(_ context.Context, req SubmitRequest) (*SubmitResult, error) { |
||||
|
taskID, err := this.sys.Create(req.AudioURL, req.EnableSpeaker, req.Language) |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
return &SubmitResult{TaskID: taskID}, nil |
||||
|
} |
||||
|
|
||||
|
func (this *iflytekTranscriber) Query(_ context.Context, taskID, _ string) (*QueryResult, error) { |
||||
|
status, segs, err := this.sys.Query(taskID) |
||||
|
switch status { |
||||
|
case ost.StatusSuccess: |
||||
|
out := make([]*pb.ContextStruct, 0, len(segs)) |
||||
|
for _, s := range segs { |
||||
|
out = append(out, &pb.ContextStruct{ |
||||
|
Content: s.Text, |
||||
|
Starttime: s.StartMs, |
||||
|
Endtime: s.EndMs, |
||||
|
Speaker: s.Speaker, |
||||
|
}) |
||||
|
} |
||||
|
return &QueryResult{Status: StatusSuccess, Contexts: out}, nil |
||||
|
case ost.StatusRunning: |
||||
|
return &QueryResult{Status: StatusRunning}, nil |
||||
|
default: |
||||
|
if err == nil { |
||||
|
err = fmt.Errorf("iflytek ost query failed") |
||||
|
} |
||||
|
return &QueryResult{Status: StatusFailed}, err |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
type iflytekSummarizer struct { |
||||
|
sys *openaichat.Client |
||||
|
} |
||||
|
|
||||
|
func newIflytekSummarizer(f map[string]string) (Summarizer, error) { |
||||
|
appID, err := need(f, "iflytek", "app_id") |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
key, err := need(f, "iflytek", "api_key") |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
secret, err := need(f, "iflytek", "api_secret") |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
// 库里若还留着星火的 host/domain,New 会丢掉,改走 SDK 预置的 openai 地址和 gpt-5.3-chat。
|
||||
|
sys, err := openaichat.New(appID, key, secret, f["host"], f["model"]) |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
return &iflytekSummarizer{sys: sys}, nil |
||||
|
} |
||||
|
|
||||
|
// iflytekSummaryDisplayName 生成弹窗里显示的模型名,跟真正发出去的模型一致。
|
||||
|
func iflytekSummaryDisplayName(fields map[string]string) string { |
||||
|
model := "" |
||||
|
if fields != nil { |
||||
|
model = fields["model"] |
||||
|
} |
||||
|
return openaichat.EffectiveModel(model) |
||||
|
} |
||||
|
|
||||
|
func (this *iflytekSummarizer) Chat(_ context.Context, messages []ChatMessage) (string, error) { |
||||
|
in := make([]openaichat.Message, 0, len(messages)) |
||||
|
for _, m := range messages { |
||||
|
in = append(in, openaichat.Message{Role: m.Role, Content: m.Content}) |
||||
|
} |
||||
|
return this.sys.Chat(in) |
||||
|
} |
||||
@ -0,0 +1,298 @@ |
|||||
|
// Package openaichat 调用 SparkChain 海外 SDK 里 domain=openai 的文本模型。
|
||||
|
//
|
||||
|
// 会议总结走这条,不走星火 websocket。SDK 二进制里的形状是:
|
||||
|
//
|
||||
|
// POST https://cn-huadong-1.xf-yun.com/v1/private/s56f331b2
|
||||
|
// 鉴权挂在 URL 上(host date request-line,没有 digest)
|
||||
|
// 正文是讯飞信封,payload.input.text = base64(OpenAI chat JSON)
|
||||
|
// 响应 payload.output.text 再 base64 解出来,里面是 chat.completion 或 responses
|
||||
|
package openaichat |
||||
|
|
||||
|
import ( |
||||
|
"bytes" |
||||
|
"crypto/hmac" |
||||
|
"crypto/sha256" |
||||
|
"encoding/base64" |
||||
|
"encoding/json" |
||||
|
"fmt" |
||||
|
"io" |
||||
|
"net/http" |
||||
|
"net/url" |
||||
|
"strings" |
||||
|
"time" |
||||
|
) |
||||
|
|
||||
|
const ( |
||||
|
defaultHost = "cn-huadong-1.xf-yun.com" |
||||
|
defaultPath = "/v1/private/s56f331b2" |
||||
|
defaultModel = "gpt-5.3-chat" |
||||
|
) |
||||
|
|
||||
|
// Message 一轮对话。Role 取 system / user / assistant。
|
||||
|
type Message struct { |
||||
|
Role string `json:"role"` |
||||
|
Content string `json:"content"` |
||||
|
} |
||||
|
|
||||
|
// Client 讯飞 OpenAI 文本客户端。
|
||||
|
type Client struct { |
||||
|
AppID string |
||||
|
APIKey string |
||||
|
Secret string |
||||
|
Host string |
||||
|
Model string |
||||
|
HTTP *http.Client |
||||
|
} |
||||
|
|
||||
|
// New 用 SparkChain 的 appId / apiKey / apiSecret 建客户端。
|
||||
|
// model 空则用文档里的对话模型 gpt-5.3-chat。host 空则用 SDK 预置地址。
|
||||
|
func New(appID, apiKey, secret, host, model string) (*Client, error) { |
||||
|
if strings.TrimSpace(appID) == "" || strings.TrimSpace(apiKey) == "" || strings.TrimSpace(secret) == "" { |
||||
|
return nil, fmt.Errorf("iflytek openai: app_id/api_key/api_secret 都不能空") |
||||
|
} |
||||
|
host = strings.TrimSpace(host) |
||||
|
host = strings.TrimPrefix(strings.TrimPrefix(host, "https://"), "http://") |
||||
|
host = strings.Trim(host, "/") |
||||
|
if host == "" || strings.Contains(host, "spark-api") { |
||||
|
host = defaultHost |
||||
|
} |
||||
|
model = EffectiveModel(model) |
||||
|
return &Client{ |
||||
|
AppID: appID, |
||||
|
APIKey: apiKey, |
||||
|
Secret: secret, |
||||
|
Host: host, |
||||
|
Model: model, |
||||
|
HTTP: &http.Client{Timeout: 120 * time.Second, CheckRedirect: refuseRedirect}, |
||||
|
}, nil |
||||
|
} |
||||
|
|
||||
|
// EffectiveModel 空模型或星火的 general* 都改成文档里的对话模型。
|
||||
|
func EffectiveModel(model string) string { |
||||
|
model = strings.TrimSpace(model) |
||||
|
if model == "" || strings.HasPrefix(model, "general") { |
||||
|
return defaultModel |
||||
|
} |
||||
|
return model |
||||
|
} |
||||
|
|
||||
|
func refuseRedirect(*http.Request, []*http.Request) error { |
||||
|
return http.ErrUseLastResponse |
||||
|
} |
||||
|
|
||||
|
// Chat 同步要一段回复。
|
||||
|
func (c *Client) Chat(messages []Message) (string, error) { |
||||
|
if len(messages) == 0 { |
||||
|
return "", fmt.Errorf("iflytek openai: 没有输入") |
||||
|
} |
||||
|
body, err := c.envelope(messages) |
||||
|
if err != nil { |
||||
|
return "", err |
||||
|
} |
||||
|
endpoint, err := c.authURL() |
||||
|
if err != nil { |
||||
|
return "", err |
||||
|
} |
||||
|
req, err := http.NewRequest(http.MethodPost, endpoint, bytes.NewReader(body)) |
||||
|
if err != nil { |
||||
|
return "", err |
||||
|
} |
||||
|
req.Host = c.Host |
||||
|
req.Header.Set("Content-Type", "application/json") |
||||
|
resp, err := c.HTTP.Do(req) |
||||
|
if err != nil { |
||||
|
return "", fmt.Errorf("iflytek openai: %w", err) |
||||
|
} |
||||
|
defer resp.Body.Close() |
||||
|
raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) |
||||
|
if err != nil { |
||||
|
return "", err |
||||
|
} |
||||
|
text, err := parseOuter(raw) |
||||
|
if err != nil { |
||||
|
if resp.StatusCode != http.StatusOK { |
||||
|
return "", fmt.Errorf("iflytek openai: http %d: %w", resp.StatusCode, err) |
||||
|
} |
||||
|
return "", err |
||||
|
} |
||||
|
return text, nil |
||||
|
} |
||||
|
|
||||
|
// envelope 讯飞外层。内层是 OpenAI chat JSON,text 字段 base64。
|
||||
|
func (c *Client) envelope(messages []Message) ([]byte, error) { |
||||
|
inner := map[string]any{ |
||||
|
"model": c.Model, |
||||
|
"max_completion_tokens": 4096, |
||||
|
"stream": false, |
||||
|
"messages": messages, |
||||
|
} |
||||
|
raw, err := json.Marshal(inner) |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
outer := map[string]any{ |
||||
|
"header": map[string]any{ |
||||
|
"app_id": c.AppID, |
||||
|
"status": 3, |
||||
|
}, |
||||
|
"parameter": map[string]any{ |
||||
|
"chat": map[string]any{ |
||||
|
"output": map[string]any{ |
||||
|
"encoding": "utf8", |
||||
|
"compress": "raw", |
||||
|
"format": "json", |
||||
|
}, |
||||
|
}, |
||||
|
}, |
||||
|
"payload": map[string]any{ |
||||
|
"input": map[string]any{ |
||||
|
"encoding": "utf8", |
||||
|
"compress": "raw", |
||||
|
"format": "json", |
||||
|
"status": 3, |
||||
|
"text": base64.StdEncoding.EncodeToString(raw), |
||||
|
}, |
||||
|
}, |
||||
|
} |
||||
|
return json.Marshal(outer) |
||||
|
} |
||||
|
|
||||
|
func (c *Client) authURL() (string, error) { |
||||
|
date := time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT") |
||||
|
origin := "host: " + c.Host + "\n" + "date: " + date + "\n" + "POST " + defaultPath + " HTTP/1.1" |
||||
|
mac := hmac.New(sha256.New, []byte(c.Secret)) |
||||
|
_, _ = mac.Write([]byte(origin)) |
||||
|
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) |
||||
|
authOrigin := fmt.Sprintf(`api_key="%s", algorithm="hmac-sha256", headers="host date request-line", signature="%s"`, |
||||
|
c.APIKey, signature) |
||||
|
q := url.Values{} |
||||
|
q.Set("authorization", base64.StdEncoding.EncodeToString([]byte(authOrigin))) |
||||
|
q.Set("date", date) |
||||
|
q.Set("host", c.Host) |
||||
|
return "https://" + c.Host + defaultPath + "?" + q.Encode(), nil |
||||
|
} |
||||
|
|
||||
|
func parseOuter(raw []byte) (string, error) { |
||||
|
var outer struct { |
||||
|
Header struct { |
||||
|
Code int `json:"code"` |
||||
|
Message string `json:"message"` |
||||
|
} `json:"header"` |
||||
|
Payload struct { |
||||
|
Output struct { |
||||
|
Text string `json:"text"` |
||||
|
} `json:"output"` |
||||
|
} `json:"payload"` |
||||
|
} |
||||
|
if err := json.Unmarshal(raw, &outer); err != nil { |
||||
|
return "", fmt.Errorf("iflytek openai: 响应不是 JSON") |
||||
|
} |
||||
|
if outer.Header.Code != 0 { |
||||
|
msg := strings.TrimSpace(outer.Header.Message) |
||||
|
if msg == "" { |
||||
|
msg = "gateway request failed" |
||||
|
} |
||||
|
return "", fmt.Errorf("iflytek openai: code=%d %s", outer.Header.Code, msg) |
||||
|
} |
||||
|
decoded, err := base64.StdEncoding.DecodeString(outer.Payload.Output.Text) |
||||
|
if err != nil { |
||||
|
decoded, err = base64.RawStdEncoding.DecodeString(outer.Payload.Output.Text) |
||||
|
} |
||||
|
if err != nil { |
||||
|
return "", fmt.Errorf("iflytek openai: 输出不是 base64") |
||||
|
} |
||||
|
text := extractContent(decoded) |
||||
|
text = strings.TrimSpace(text) |
||||
|
if text == "" { |
||||
|
return "", fmt.Errorf("iflytek openai: 空回复") |
||||
|
} |
||||
|
return text, nil |
||||
|
} |
||||
|
|
||||
|
func extractContent(decoded []byte) string { |
||||
|
s := strings.TrimSpace(string(decoded)) |
||||
|
if strings.Contains(s, "data:") && !json.Valid(decoded) { |
||||
|
return sseContent(s) |
||||
|
} |
||||
|
var body map[string]any |
||||
|
if json.Unmarshal(decoded, &body) != nil { |
||||
|
return s |
||||
|
} |
||||
|
if fmt.Sprint(body["object"]) == "response" { |
||||
|
if t := responseText(body); t != "" { |
||||
|
return t |
||||
|
} |
||||
|
} |
||||
|
if t := chatCompletionText(body); t != "" { |
||||
|
return t |
||||
|
} |
||||
|
if t := responseText(body); t != "" { |
||||
|
return t |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
func chatCompletionText(body map[string]any) string { |
||||
|
choices, _ := body["choices"].([]any) |
||||
|
var b strings.Builder |
||||
|
for _, item := range choices { |
||||
|
m, _ := item.(map[string]any) |
||||
|
if msg, ok := m["message"].(map[string]any); ok { |
||||
|
if c, ok := msg["content"].(string); ok { |
||||
|
b.WriteString(c) |
||||
|
} |
||||
|
} |
||||
|
if delta, ok := m["delta"].(map[string]any); ok { |
||||
|
if c, ok := delta["content"].(string); ok { |
||||
|
b.WriteString(c) |
||||
|
} |
||||
|
} |
||||
|
if t, ok := m["text"].(string); ok { |
||||
|
b.WriteString(t) |
||||
|
} |
||||
|
} |
||||
|
return b.String() |
||||
|
} |
||||
|
|
||||
|
func responseText(body map[string]any) string { |
||||
|
var b strings.Builder |
||||
|
switch out := body["output"].(type) { |
||||
|
case string: |
||||
|
return out |
||||
|
case []any: |
||||
|
for _, item := range out { |
||||
|
m, _ := item.(map[string]any) |
||||
|
if t, ok := m["text"].(string); ok { |
||||
|
b.WriteString(t) |
||||
|
} |
||||
|
content, _ := m["content"].([]any) |
||||
|
for _, part := range content { |
||||
|
pm, _ := part.(map[string]any) |
||||
|
if t, ok := pm["text"].(string); ok { |
||||
|
b.WriteString(t) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
return b.String() |
||||
|
} |
||||
|
|
||||
|
func sseContent(s string) string { |
||||
|
var b strings.Builder |
||||
|
for _, line := range strings.Split(s, "\n") { |
||||
|
line = strings.TrimSpace(line) |
||||
|
if !strings.HasPrefix(line, "data:") { |
||||
|
continue |
||||
|
} |
||||
|
payload := strings.TrimSpace(strings.TrimPrefix(line, "data:")) |
||||
|
if payload == "" || payload == "[DONE]" { |
||||
|
continue |
||||
|
} |
||||
|
var body map[string]any |
||||
|
if json.Unmarshal([]byte(payload), &body) != nil { |
||||
|
continue |
||||
|
} |
||||
|
b.WriteString(chatCompletionText(body)) |
||||
|
} |
||||
|
return b.String() |
||||
|
} |
||||
@ -0,0 +1,99 @@ |
|||||
|
package openaichat |
||||
|
|
||||
|
import ( |
||||
|
"encoding/base64" |
||||
|
"encoding/json" |
||||
|
"strings" |
||||
|
"testing" |
||||
|
) |
||||
|
|
||||
|
func TestEffectiveModelDropsSpark(t *testing.T) { |
||||
|
if EffectiveModel("") != "gpt-5.3-chat" || EffectiveModel("generalv3.5") != "gpt-5.3-chat" { |
||||
|
t.Fatal(EffectiveModel("generalv3.5")) |
||||
|
} |
||||
|
if EffectiveModel("gpt-5.4") != "gpt-5.4" { |
||||
|
t.Fatal(EffectiveModel("gpt-5.4")) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestEnvelopeUsesOpenAIModelNotSpark(t *testing.T) { |
||||
|
c, err := New("app", "key", "secret", "spark-api.xf-yun.com", "generalv3.5") |
||||
|
if err != nil { |
||||
|
t.Fatal(err) |
||||
|
} |
||||
|
if c.Host != defaultHost { |
||||
|
t.Fatalf("星火主机必须丢掉,得到 %s", c.Host) |
||||
|
} |
||||
|
if c.Model != defaultModel { |
||||
|
t.Fatalf("星火 domain 不能当模型,得到 %s", c.Model) |
||||
|
} |
||||
|
raw, err := c.envelope([]Message{{Role: "user", Content: "1+1等于几"}}) |
||||
|
if err != nil { |
||||
|
t.Fatal(err) |
||||
|
} |
||||
|
var outer struct { |
||||
|
Header struct { |
||||
|
AppID string `json:"app_id"` |
||||
|
Status int `json:"status"` |
||||
|
} `json:"header"` |
||||
|
Payload struct { |
||||
|
Input struct { |
||||
|
Status int `json:"status"` |
||||
|
Text string `json:"text"` |
||||
|
} `json:"input"` |
||||
|
} `json:"payload"` |
||||
|
} |
||||
|
if err := json.Unmarshal(raw, &outer); err != nil { |
||||
|
t.Fatal(err) |
||||
|
} |
||||
|
if outer.Header.AppID != "app" || outer.Header.Status != 3 || outer.Payload.Input.Status != 3 { |
||||
|
t.Fatalf("信封不对: %+v", outer.Header) |
||||
|
} |
||||
|
inner, err := base64.StdEncoding.DecodeString(outer.Payload.Input.Text) |
||||
|
if err != nil { |
||||
|
t.Fatal(err) |
||||
|
} |
||||
|
if strings.Contains(string(inner), "generalv3") || strings.Contains(string(inner), "spark") { |
||||
|
t.Fatalf("内层还带着星火: %s", inner) |
||||
|
} |
||||
|
var chat struct { |
||||
|
Model string `json:"model"` |
||||
|
Stream bool `json:"stream"` |
||||
|
Messages []Message `json:"messages"` |
||||
|
} |
||||
|
if err := json.Unmarshal(inner, &chat); err != nil { |
||||
|
t.Fatal(err) |
||||
|
} |
||||
|
if chat.Model != "gpt-5.3-chat" || chat.Stream || chat.Messages[0].Content != "1+1等于几" { |
||||
|
t.Fatalf("内层不对: %+v", chat) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestParseChatCompletion(t *testing.T) { |
||||
|
inner, _ := json.Marshal(map[string]any{ |
||||
|
"object": "chat.completion", |
||||
|
"choices": []any{ |
||||
|
map[string]any{"message": map[string]any{"content": "1+1等于2。"}}, |
||||
|
}, |
||||
|
}) |
||||
|
outer, _ := json.Marshal(map[string]any{ |
||||
|
"header": map[string]any{"code": 0, "message": "success"}, |
||||
|
"payload": map[string]any{ |
||||
|
"output": map[string]any{"text": base64.StdEncoding.EncodeToString(inner)}, |
||||
|
}, |
||||
|
}) |
||||
|
got, err := parseOuter(outer) |
||||
|
if err != nil || got != "1+1等于2。" { |
||||
|
t.Fatalf("got %q err %v", got, err) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestParseGatewayError(t *testing.T) { |
||||
|
outer, _ := json.Marshal(map[string]any{ |
||||
|
"header": map[string]any{"code": 11200, "message": "授权错误"}, |
||||
|
}) |
||||
|
_, err := parseOuter(outer) |
||||
|
if err == nil || !strings.Contains(err.Error(), "11200") { |
||||
|
t.Fatalf("err=%v", err) |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,381 @@ |
|||||
|
// Package ost 调用科大讯飞 SparkChain SDK 里 RegionMSFT 那条录音文件转写。
|
||||
|
//
|
||||
|
// SDK 二进制里的引擎名是 iflyrec_voice_mul_cbm_msft_24h,走海外 OST:
|
||||
|
// 创建 https://<host>/v2/ost/pro_create ,查询 https://<host>/v2/ost/query 。
|
||||
|
// 音频不经过讯飞上传接口,直接用客户端已经放到 OSS 上的公网地址。
|
||||
|
package ost |
||||
|
|
||||
|
import ( |
||||
|
"bytes" |
||||
|
"crypto/hmac" |
||||
|
"crypto/sha256" |
||||
|
"encoding/base64" |
||||
|
"encoding/json" |
||||
|
"fmt" |
||||
|
"io" |
||||
|
"net/http" |
||||
|
"net/url" |
||||
|
"strings" |
||||
|
"time" |
||||
|
|
||||
|
"github.com/google/uuid" |
||||
|
) |
||||
|
|
||||
|
const ( |
||||
|
defaultHost = "ost-api-sg.xf-yun.com" |
||||
|
defaultTaskType = "iflyrec_voice_mul_cbm_msft_24h" |
||||
|
createPath = "/v2/ost/create" |
||||
|
queryPath = "/v2/ost/query" |
||||
|
) |
||||
|
|
||||
|
// Segment 一句转写。StartMs/EndMs 是相对音频开头的毫秒。
|
||||
|
type Segment struct { |
||||
|
Text string |
||||
|
StartMs int64 |
||||
|
EndMs int64 |
||||
|
Speaker string |
||||
|
} |
||||
|
|
||||
|
// Client 讯飞 OST 客户端。
|
||||
|
type Client struct { |
||||
|
AppID string |
||||
|
APIKey string |
||||
|
Secret string |
||||
|
Host string |
||||
|
TaskType string |
||||
|
HTTP *http.Client |
||||
|
} |
||||
|
|
||||
|
// New 用 SparkChain 的 appId / apiKey / apiSecret 建客户端。
|
||||
|
func New(appID, apiKey, secret, host, taskType string) (*Client, error) { |
||||
|
if strings.TrimSpace(appID) == "" || strings.TrimSpace(apiKey) == "" || strings.TrimSpace(secret) == "" { |
||||
|
return nil, fmt.Errorf("iflytek ost: app_id/api_key/api_secret 都不能空") |
||||
|
} |
||||
|
host = strings.TrimSpace(host) |
||||
|
if host == "" { |
||||
|
host = defaultHost |
||||
|
} |
||||
|
host = strings.TrimPrefix(strings.TrimPrefix(host, "https://"), "http://") |
||||
|
host = strings.TrimRight(host, "/") |
||||
|
taskType = strings.TrimSpace(taskType) |
||||
|
if taskType == "" { |
||||
|
taskType = defaultTaskType |
||||
|
} |
||||
|
return &Client{ |
||||
|
AppID: appID, |
||||
|
APIKey: apiKey, |
||||
|
Secret: secret, |
||||
|
Host: host, |
||||
|
TaskType: taskType, |
||||
|
HTTP: &http.Client{Timeout: 45 * time.Second}, |
||||
|
}, nil |
||||
|
} |
||||
|
|
||||
|
// Create 提交一条公网音频。language 用 BCP-47(zh-CN / en-US),空则按中文。
|
||||
|
func (c *Client) Create(audioURL string, speaker bool, language string) (string, error) { |
||||
|
enc, format := audioEncoding(audioURL) |
||||
|
business := map[string]any{ |
||||
|
"request_id": strings.ReplaceAll(uuid.NewString(), "-", ""), |
||||
|
"task_type": c.TaskType, |
||||
|
"language": ostLanguage(language), |
||||
|
} |
||||
|
applySpeaker(business, c.TaskType, speaker, enc) |
||||
|
body := map[string]any{ |
||||
|
"common": map[string]string{"app_id": c.AppID}, |
||||
|
"business": business, |
||||
|
"data": map[string]any{ |
||||
|
"audio_url": audioURL, |
||||
|
"audio_src": "http", |
||||
|
"format": format, |
||||
|
"encoding": enc, |
||||
|
}, |
||||
|
} |
||||
|
var resp struct { |
||||
|
Code int `json:"code"` |
||||
|
Message string `json:"message"` |
||||
|
Data struct { |
||||
|
TaskID string `json:"task_id"` |
||||
|
} `json:"data"` |
||||
|
} |
||||
|
if err := c.post(createPath, body, &resp); err != nil { |
||||
|
return "", err |
||||
|
} |
||||
|
if resp.Code != 0 { |
||||
|
return "", fmt.Errorf("iflytek ost create: code=%d %s", resp.Code, resp.Message) |
||||
|
} |
||||
|
if resp.Data.TaskID == "" { |
||||
|
return "", fmt.Errorf("iflytek ost create: 没有 task_id") |
||||
|
} |
||||
|
return resp.Data.TaskID, nil |
||||
|
} |
||||
|
|
||||
|
// msftSpeakerCap 微软文件转写的说话人数上限。speaker_num 合法范围是 1–35,
|
||||
|
// 不传时文档默认 1,等于所有人合成一个角色。会议人数事先不知道,按 10 交给引擎去分。
|
||||
|
const msftSpeakerCap = 10 |
||||
|
|
||||
|
// applySpeaker 按引擎写角色分离参数。
|
||||
|
// 微软通道(task_type 含 msft)认 spkdia,不认讯飞自研的 vspp_on。
|
||||
|
// 自研通道的 mp3 官方写明不支持角色分离,硬开会失败。
|
||||
|
func applySpeaker(business map[string]any, taskType string, speaker bool, enc string) { |
||||
|
if !speaker { |
||||
|
return |
||||
|
} |
||||
|
if strings.Contains(taskType, "msft") { |
||||
|
business["spkdia"] = 1 |
||||
|
business["speaker_num"] = msftSpeakerCap |
||||
|
return |
||||
|
} |
||||
|
if enc == "lame" { |
||||
|
return |
||||
|
} |
||||
|
business["vspp_on"] = 1 |
||||
|
business["speaker_num"] = 0 |
||||
|
} |
||||
|
|
||||
|
// QueryStatus 任务状态。running 表示还要再查,failed 时 err 带原因。
|
||||
|
type QueryStatus int |
||||
|
|
||||
|
const ( |
||||
|
StatusRunning QueryStatus = iota |
||||
|
StatusSuccess |
||||
|
StatusFailed |
||||
|
) |
||||
|
|
||||
|
// Query 查一次。完成时 segments 是按句切开的文本。
|
||||
|
func (c *Client) Query(taskID string) (QueryStatus, []Segment, error) { |
||||
|
body := map[string]any{ |
||||
|
"common": map[string]string{"app_id": c.AppID}, |
||||
|
"business": map[string]string{"task_id": taskID}, |
||||
|
} |
||||
|
var raw map[string]any |
||||
|
if err := c.post(queryPath, body, &raw); err != nil { |
||||
|
return StatusFailed, nil, err |
||||
|
} |
||||
|
code := intFrom(raw["code"]) |
||||
|
if code != 0 { |
||||
|
return StatusFailed, nil, fmt.Errorf("iflytek ost query: code=%d %v", code, raw["message"]) |
||||
|
} |
||||
|
data, _ := raw["data"].(map[string]any) |
||||
|
if data == nil { |
||||
|
return StatusRunning, nil, nil |
||||
|
} |
||||
|
switch fmt.Sprint(data["task_status"]) { |
||||
|
case "1", "2", "": |
||||
|
return StatusRunning, nil, nil |
||||
|
case "3", "4": |
||||
|
return StatusSuccess, parseLattice(data["result"]), nil |
||||
|
default: |
||||
|
return StatusFailed, nil, fmt.Errorf("iflytek ost query: task_status=%v %v", data["task_status"], raw["message"]) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (c *Client) post(path string, payload any, out any) error { |
||||
|
raw, err := json.Marshal(payload) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
endpoint := "https://" + c.Host + path |
||||
|
req, err := http.NewRequest(http.MethodPost, endpoint, bytes.NewReader(raw)) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
req.Host = c.Host |
||||
|
for k, v := range c.authHeader(path, raw) { |
||||
|
if k == "Host" { |
||||
|
continue |
||||
|
} |
||||
|
req.Header.Set(k, v) |
||||
|
} |
||||
|
client := c.HTTP |
||||
|
if client.CheckRedirect == nil { |
||||
|
client.CheckRedirect = func(req *http.Request, via []*http.Request) error { |
||||
|
return http.ErrUseLastResponse |
||||
|
} |
||||
|
} |
||||
|
resp, err := client.Do(req) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("iflytek ost %s: %w", path, err) |
||||
|
} |
||||
|
defer resp.Body.Close() |
||||
|
buf, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
if resp.StatusCode != http.StatusOK { |
||||
|
if json.Unmarshal(buf, out) == nil { |
||||
|
return nil |
||||
|
} |
||||
|
return fmt.Errorf("iflytek ost %s: http %d %s", endpoint, resp.StatusCode, truncate(string(buf), 300)) |
||||
|
} |
||||
|
if err := json.Unmarshal(buf, out); err != nil { |
||||
|
return fmt.Errorf("iflytek ost %s: 响应不是 JSON: %s", path, truncate(string(buf), 200)) |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (c *Client) authHeader(path string, body []byte) map[string]string { |
||||
|
sum := sha256.Sum256(body) |
||||
|
digest := "SHA-256=" + base64.StdEncoding.EncodeToString(sum[:]) |
||||
|
date := time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT") |
||||
|
origin := "host: " + c.Host + "\n" + |
||||
|
"date: " + date + "\n" + |
||||
|
"POST " + path + " HTTP/1.1\n" + |
||||
|
"digest: " + digest |
||||
|
mac := hmac.New(sha256.New, []byte(c.Secret)) |
||||
|
_, _ = mac.Write([]byte(origin)) |
||||
|
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) |
||||
|
auth := fmt.Sprintf(`api_key="%s", algorithm="hmac-sha256", headers="host date request-line digest", signature="%s"`, |
||||
|
c.APIKey, signature) |
||||
|
return map[string]string{ |
||||
|
"Content-Type": "application/json", |
||||
|
"Accept": "application/json", |
||||
|
"Host": c.Host, |
||||
|
"Date": date, |
||||
|
"Digest": digest, |
||||
|
"Authorization": auth, |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func ostLanguage(code string) string { |
||||
|
c := strings.TrimSpace(code) |
||||
|
if c == "" { |
||||
|
return "zh-CN" |
||||
|
} |
||||
|
c = strings.ReplaceAll(c, "_", "-") |
||||
|
if !strings.Contains(c, "-") { |
||||
|
switch strings.ToLower(c) { |
||||
|
case "zh": |
||||
|
return "zh-CN" |
||||
|
case "en": |
||||
|
return "en-US" |
||||
|
case "ja": |
||||
|
return "ja-JP" |
||||
|
case "ko": |
||||
|
return "ko-KR" |
||||
|
} |
||||
|
} |
||||
|
return c |
||||
|
} |
||||
|
|
||||
|
func audioEncoding(audioURL string) (encoding, format string) { |
||||
|
u, err := url.Parse(audioURL) |
||||
|
path := audioURL |
||||
|
if err == nil && u.Path != "" { |
||||
|
path = u.Path |
||||
|
} |
||||
|
lower := strings.ToLower(path) |
||||
|
switch { |
||||
|
case strings.HasSuffix(lower, ".mp3"): |
||||
|
return "lame", "audio/mpeg" |
||||
|
case strings.HasSuffix(lower, ".wav"), strings.HasSuffix(lower, ".pcm"): |
||||
|
return "raw", "audio/L16;rate=16000" |
||||
|
default: |
||||
|
return "raw", "audio/L16;rate=16000" |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// parseLattice 把 OST 的 lattice 收成一句一条。json_1best 有时是对象,有时是字符串。
|
||||
|
func parseLattice(result any) []Segment { |
||||
|
obj, _ := result.(map[string]any) |
||||
|
if obj == nil { |
||||
|
if s, ok := result.(string); ok && s != "" { |
||||
|
_ = json.Unmarshal([]byte(s), &obj) |
||||
|
} |
||||
|
} |
||||
|
if obj == nil { |
||||
|
return nil |
||||
|
} |
||||
|
lattice, _ := obj["lattice"].([]any) |
||||
|
if len(lattice) == 0 { |
||||
|
lattice, _ = obj["lattice2"].([]any) |
||||
|
} |
||||
|
out := make([]Segment, 0, len(lattice)) |
||||
|
for _, item := range lattice { |
||||
|
m, _ := item.(map[string]any) |
||||
|
if m == nil { |
||||
|
continue |
||||
|
} |
||||
|
best := unwrapBest(m["json_1best"]) |
||||
|
st, _ := best["st"].(map[string]any) |
||||
|
if st == nil { |
||||
|
continue |
||||
|
} |
||||
|
text := wordsOf(st) |
||||
|
if text == "" { |
||||
|
continue |
||||
|
} |
||||
|
spk := strings.TrimSpace(fmt.Sprint(m["spk"])) |
||||
|
if spk == "" || spk == "<nil>" { |
||||
|
spk = strings.TrimSpace(fmt.Sprint(st["rl"])) |
||||
|
} |
||||
|
out = append(out, Segment{ |
||||
|
Text: text, |
||||
|
StartMs: intFrom(st["bg"]), |
||||
|
EndMs: intFrom(st["ed"]), |
||||
|
Speaker: spk, |
||||
|
}) |
||||
|
} |
||||
|
return out |
||||
|
} |
||||
|
|
||||
|
func unwrapBest(v any) map[string]any { |
||||
|
switch t := v.(type) { |
||||
|
case map[string]any: |
||||
|
return t |
||||
|
case string: |
||||
|
var m map[string]any |
||||
|
if json.Unmarshal([]byte(t), &m) == nil { |
||||
|
return m |
||||
|
} |
||||
|
} |
||||
|
return map[string]any{} |
||||
|
} |
||||
|
|
||||
|
func wordsOf(st map[string]any) string { |
||||
|
rts, _ := st["rt"].([]any) |
||||
|
var b strings.Builder |
||||
|
for _, rt := range rts { |
||||
|
rtm, _ := rt.(map[string]any) |
||||
|
wss, _ := rtm["ws"].([]any) |
||||
|
for _, ws := range wss { |
||||
|
wsm, _ := ws.(map[string]any) |
||||
|
cws, _ := wsm["cw"].([]any) |
||||
|
for _, cw := range cws { |
||||
|
cwm, _ := cw.(map[string]any) |
||||
|
if fmt.Sprint(cwm["wp"]) == "g" { |
||||
|
continue |
||||
|
} |
||||
|
w, _ := cwm["w"].(string) |
||||
|
b.WriteString(w) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
return strings.TrimSpace(b.String()) |
||||
|
} |
||||
|
|
||||
|
func intFrom(v any) int64 { |
||||
|
switch t := v.(type) { |
||||
|
case float64: |
||||
|
return int64(t) |
||||
|
case json.Number: |
||||
|
n, _ := t.Int64() |
||||
|
return n |
||||
|
case string: |
||||
|
var n int64 |
||||
|
fmt.Sscan(t, &n) |
||||
|
return n |
||||
|
case int: |
||||
|
return int64(t) |
||||
|
case int64: |
||||
|
return t |
||||
|
default: |
||||
|
return 0 |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func truncate(s string, n int) string { |
||||
|
if len(s) <= n { |
||||
|
return s |
||||
|
} |
||||
|
return s[:n] |
||||
|
} |
||||
@ -0,0 +1,72 @@ |
|||||
|
package ost |
||||
|
|
||||
|
import "testing" |
||||
|
|
||||
|
func TestParseLattice(t *testing.T) { |
||||
|
raw := map[string]any{ |
||||
|
"lattice": []any{ |
||||
|
map[string]any{ |
||||
|
"spk": "段落-0", |
||||
|
"json_1best": map[string]any{ |
||||
|
"st": map[string]any{ |
||||
|
"bg": "0", |
||||
|
"ed": "470", |
||||
|
"rt": []any{ |
||||
|
map[string]any{ |
||||
|
"ws": []any{ |
||||
|
map[string]any{"cw": []any{map[string]any{"w": "听说", "wp": "n"}}}, |
||||
|
map[string]any{"cw": []any{map[string]any{"w": "。", "wp": "p"}}}, |
||||
|
map[string]any{"cw": []any{map[string]any{"w": "", "wp": "g"}}}, |
||||
|
}, |
||||
|
}, |
||||
|
}, |
||||
|
}, |
||||
|
}, |
||||
|
}, |
||||
|
}, |
||||
|
} |
||||
|
segs := parseLattice(raw) |
||||
|
if len(segs) != 1 { |
||||
|
t.Fatalf("segs=%d", len(segs)) |
||||
|
} |
||||
|
if segs[0].Text != "听说。" || segs[0].StartMs != 0 || segs[0].EndMs != 470 || segs[0].Speaker != "段落-0" { |
||||
|
t.Fatalf("%+v", segs[0]) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestApplySpeaker(t *testing.T) { |
||||
|
msft := map[string]any{} |
||||
|
applySpeaker(msft, "iflyrec_voice_mul_cbm_msft_24h", true, "raw") |
||||
|
if msft["spkdia"] != 1 || msft["speaker_num"] != msftSpeakerCap { |
||||
|
t.Fatalf("msft %+v", msft) |
||||
|
} |
||||
|
if _, ok := msft["vspp_on"]; ok { |
||||
|
t.Fatal("微软通道不该带 vspp_on") |
||||
|
} |
||||
|
off := map[string]any{} |
||||
|
applySpeaker(off, "iflyrec_voice_mul_cbm_msft_24h", false, "raw") |
||||
|
if len(off) != 0 { |
||||
|
t.Fatalf("off %+v", off) |
||||
|
} |
||||
|
self := map[string]any{} |
||||
|
applySpeaker(self, "iflyrec_voice_cn_10m_ed", true, "raw") |
||||
|
if self["vspp_on"] != 1 || self["speaker_num"] != 0 { |
||||
|
t.Fatalf("self %+v", self) |
||||
|
} |
||||
|
mp3 := map[string]any{} |
||||
|
applySpeaker(mp3, "iflyrec_voice_cn_10m_ed", true, "lame") |
||||
|
if len(mp3) != 0 { |
||||
|
t.Fatalf("mp3 %+v", mp3) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestAudioEncoding(t *testing.T) { |
||||
|
enc, _ := audioEncoding("https://oss.example.com/a/b.MP3?x=1") |
||||
|
if enc != "lame" { |
||||
|
t.Fatal(enc) |
||||
|
} |
||||
|
enc, format := audioEncoding("https://oss.example.com/a/b.wav") |
||||
|
if enc != "raw" || format != "audio/L16;rate=16000" { |
||||
|
t.Fatal(enc, format) |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,294 @@ |
|||||
|
// Package sparkchat 是星火文本 websocket。会议总结不要用它。
|
||||
|
//
|
||||
|
// 海外 SDK 的文本模型是 domain=openai,见 sys/iflytek/openaichat。
|
||||
|
// 这个包留着只因为星火协议和 openai 那条不是同一个地址。
|
||||
|
package sparkchat |
||||
|
|
||||
|
import ( |
||||
|
"bufio" |
||||
|
"crypto/hmac" |
||||
|
"crypto/rand" |
||||
|
"crypto/sha256" |
||||
|
"crypto/tls" |
||||
|
"encoding/base64" |
||||
|
"encoding/json" |
||||
|
"fmt" |
||||
|
"io" |
||||
|
"net" |
||||
|
"net/url" |
||||
|
"strings" |
||||
|
"time" |
||||
|
) |
||||
|
|
||||
|
const ( |
||||
|
defaultHost = "spark-api.xf-yun.com" |
||||
|
defaultDomain = "generalv3.5" |
||||
|
) |
||||
|
|
||||
|
// Message 一轮对话。Role 取 system / user / assistant。
|
||||
|
type Message struct { |
||||
|
Role string |
||||
|
Content string |
||||
|
} |
||||
|
|
||||
|
// Client 星火文本客户端。
|
||||
|
type Client struct { |
||||
|
AppID string |
||||
|
APIKey string |
||||
|
Secret string |
||||
|
Host string |
||||
|
Domain string |
||||
|
} |
||||
|
|
||||
|
// New 用 SparkChain 凭据建客户端。
|
||||
|
func New(appID, apiKey, secret, host, domain string) (*Client, error) { |
||||
|
if strings.TrimSpace(appID) == "" || strings.TrimSpace(apiKey) == "" || strings.TrimSpace(secret) == "" { |
||||
|
return nil, fmt.Errorf("iflytek spark: app_id/api_key/api_secret 都不能空") |
||||
|
} |
||||
|
host = strings.TrimSpace(host) |
||||
|
if host == "" { |
||||
|
host = defaultHost |
||||
|
} |
||||
|
domain = strings.TrimSpace(domain) |
||||
|
if domain == "" { |
||||
|
domain = defaultDomain |
||||
|
} |
||||
|
return &Client{AppID: appID, APIKey: apiKey, Secret: secret, Host: host, Domain: domain}, nil |
||||
|
} |
||||
|
|
||||
|
// Chat 同步要一段回复。多帧内容拼在一起返回。
|
||||
|
func (c *Client) Chat(messages []Message) (string, error) { |
||||
|
if len(messages) == 0 { |
||||
|
return "", fmt.Errorf("iflytek spark: 没有输入") |
||||
|
} |
||||
|
path := domainPath(c.Domain) |
||||
|
conn, err := c.dial(path) |
||||
|
if err != nil { |
||||
|
return "", err |
||||
|
} |
||||
|
defer conn.Close() |
||||
|
_ = conn.SetDeadline(time.Now().Add(90 * time.Second)) |
||||
|
|
||||
|
text := make([]map[string]string, 0, len(messages)) |
||||
|
for _, m := range messages { |
||||
|
role := m.Role |
||||
|
if role == "" { |
||||
|
role = "user" |
||||
|
} |
||||
|
text = append(text, map[string]string{"role": role, "content": m.Content}) |
||||
|
} |
||||
|
payload := map[string]any{ |
||||
|
"header": map[string]string{"app_id": c.AppID, "uid": "echomeet"}, |
||||
|
"parameter": map[string]any{ |
||||
|
"chat": map[string]any{ |
||||
|
"domain": c.Domain, |
||||
|
"temperature": 0.5, |
||||
|
"max_tokens": 4096, |
||||
|
}, |
||||
|
}, |
||||
|
"payload": map[string]any{ |
||||
|
"message": map[string]any{"text": text}, |
||||
|
}, |
||||
|
} |
||||
|
raw, err := json.Marshal(payload) |
||||
|
if err != nil { |
||||
|
return "", err |
||||
|
} |
||||
|
if err := writeText(conn, raw); err != nil { |
||||
|
return "", err |
||||
|
} |
||||
|
var b strings.Builder |
||||
|
for { |
||||
|
frame, err := readFrame(conn) |
||||
|
if err != nil { |
||||
|
if b.Len() > 0 && err == io.EOF { |
||||
|
break |
||||
|
} |
||||
|
return "", err |
||||
|
} |
||||
|
if frame == nil { |
||||
|
continue |
||||
|
} |
||||
|
var resp struct { |
||||
|
Header struct { |
||||
|
Code int `json:"code"` |
||||
|
Message string `json:"message"` |
||||
|
Status int `json:"status"` |
||||
|
} `json:"header"` |
||||
|
Payload struct { |
||||
|
Choices struct { |
||||
|
Status int `json:"status"` |
||||
|
Text []struct { |
||||
|
Content string `json:"content"` |
||||
|
} `json:"text"` |
||||
|
} `json:"choices"` |
||||
|
} `json:"payload"` |
||||
|
} |
||||
|
if json.Unmarshal(frame, &resp) != nil { |
||||
|
continue |
||||
|
} |
||||
|
if resp.Header.Code != 0 { |
||||
|
return "", fmt.Errorf("iflytek spark: code=%d %s", resp.Header.Code, resp.Header.Message) |
||||
|
} |
||||
|
for _, t := range resp.Payload.Choices.Text { |
||||
|
b.WriteString(t.Content) |
||||
|
} |
||||
|
if resp.Header.Status == 2 || resp.Payload.Choices.Status == 2 { |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
out := strings.TrimSpace(b.String()) |
||||
|
if out == "" { |
||||
|
return "", fmt.Errorf("iflytek spark: 空回复") |
||||
|
} |
||||
|
return out, nil |
||||
|
} |
||||
|
|
||||
|
func domainPath(domain string) string { |
||||
|
switch domain { |
||||
|
case "general": |
||||
|
return "/v1.1/chat" |
||||
|
case "generalv2": |
||||
|
return "/v2.1/chat" |
||||
|
case "generalv3": |
||||
|
return "/v3.1/chat" |
||||
|
case "4.0Ultra": |
||||
|
return "/v4.0/chat" |
||||
|
default: |
||||
|
return "/v3.5/chat" |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (c *Client) dial(path string) (net.Conn, error) { |
||||
|
date := time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT") |
||||
|
origin := "host: " + c.Host + "\n" + "date: " + date + "\n" + "GET " + path + " HTTP/1.1" |
||||
|
mac := hmac.New(sha256.New, []byte(c.Secret)) |
||||
|
_, _ = mac.Write([]byte(origin)) |
||||
|
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) |
||||
|
authOrigin := fmt.Sprintf(`api_key="%s", algorithm="hmac-sha256", headers="host date request-line", signature="%s"`, |
||||
|
c.APIKey, signature) |
||||
|
auth := base64.StdEncoding.EncodeToString([]byte(authOrigin)) |
||||
|
q := url.Values{} |
||||
|
q.Set("authorization", auth) |
||||
|
q.Set("date", date) |
||||
|
q.Set("host", c.Host) |
||||
|
conn, err := tls.Dial("tcp", c.Host+":443", &tls.Config{ServerName: c.Host}) |
||||
|
if err != nil { |
||||
|
return nil, fmt.Errorf("iflytek spark dial: %w", err) |
||||
|
} |
||||
|
key := make([]byte, 16) |
||||
|
_, _ = rand.Read(key) |
||||
|
secKey := base64.StdEncoding.EncodeToString(key) |
||||
|
req := "GET " + path + "?" + q.Encode() + " HTTP/1.1\r\n" + |
||||
|
"Host: " + c.Host + "\r\n" + |
||||
|
"Upgrade: websocket\r\n" + |
||||
|
"Connection: Upgrade\r\n" + |
||||
|
"Sec-WebSocket-Key: " + secKey + "\r\n" + |
||||
|
"Sec-WebSocket-Version: 13\r\n\r\n" |
||||
|
if _, err := io.WriteString(conn, req); err != nil { |
||||
|
conn.Close() |
||||
|
return nil, err |
||||
|
} |
||||
|
br := bufio.NewReader(conn) |
||||
|
status, err := br.ReadString('\n') |
||||
|
if err != nil { |
||||
|
conn.Close() |
||||
|
return nil, err |
||||
|
} |
||||
|
if !strings.Contains(status, "101") { |
||||
|
conn.Close() |
||||
|
return nil, fmt.Errorf("iflytek spark handshake: %s", strings.TrimSpace(status)) |
||||
|
} |
||||
|
for { |
||||
|
line, err := br.ReadString('\n') |
||||
|
if err != nil { |
||||
|
conn.Close() |
||||
|
return nil, err |
||||
|
} |
||||
|
if line == "\r\n" || line == "\n" { |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
return &bufferedConn{Conn: conn, r: br}, nil |
||||
|
} |
||||
|
|
||||
|
// bufferedConn 把握手时多读进 bufio 的字节交还给后续读帧。
|
||||
|
type bufferedConn struct { |
||||
|
net.Conn |
||||
|
r *bufio.Reader |
||||
|
} |
||||
|
|
||||
|
func (c *bufferedConn) Read(p []byte) (int, error) { return c.r.Read(p) } |
||||
|
|
||||
|
func writeText(conn net.Conn, payload []byte) error { |
||||
|
mask := make([]byte, 4) |
||||
|
_, _ = rand.Read(mask) |
||||
|
var hdr []byte |
||||
|
n := len(payload) |
||||
|
switch { |
||||
|
case n < 126: |
||||
|
hdr = []byte{0x81, byte(n) | 0x80} |
||||
|
case n < 65536: |
||||
|
hdr = []byte{0x81, 126 | 0x80, byte(n >> 8), byte(n)} |
||||
|
default: |
||||
|
return fmt.Errorf("iflytek spark: 请求过长") |
||||
|
} |
||||
|
frame := append(hdr, mask...) |
||||
|
masked := make([]byte, n) |
||||
|
for i := 0; i < n; i++ { |
||||
|
masked[i] = payload[i] ^ mask[i%4] |
||||
|
} |
||||
|
frame = append(frame, masked...) |
||||
|
_, err := conn.Write(frame) |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
func readFrame(conn net.Conn) ([]byte, error) { |
||||
|
for { |
||||
|
hdr := make([]byte, 2) |
||||
|
if _, err := io.ReadFull(conn, hdr); err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
opcode := hdr[0] & 0x0f |
||||
|
masked := hdr[1]&0x80 != 0 |
||||
|
n := int(hdr[1] & 0x7f) |
||||
|
switch n { |
||||
|
case 126: |
||||
|
ext := make([]byte, 2) |
||||
|
if _, err := io.ReadFull(conn, ext); err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
n = int(ext[0])<<8 | int(ext[1]) |
||||
|
case 127: |
||||
|
return nil, fmt.Errorf("iflytek spark: 帧过长") |
||||
|
} |
||||
|
var mask []byte |
||||
|
if masked { |
||||
|
mask = make([]byte, 4) |
||||
|
if _, err := io.ReadFull(conn, mask); err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
} |
||||
|
payload := make([]byte, n) |
||||
|
if _, err := io.ReadFull(conn, payload); err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
if masked { |
||||
|
for i := 0; i < n; i++ { |
||||
|
payload[i] ^= mask[i%4] |
||||
|
} |
||||
|
} |
||||
|
switch opcode { |
||||
|
case 0x1, 0x0: |
||||
|
return payload, nil |
||||
|
case 0x8: |
||||
|
return nil, io.EOF |
||||
|
case 0x9: // ping
|
||||
|
pong := []byte{0x8A, 0} |
||||
|
_, _ = conn.Write(pong) |
||||
|
continue |
||||
|
default: |
||||
|
continue |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,142 @@ |
|||||
|
# EMAI 助手:端上听写 + 对话 + 合成 —— 设计 |
||||
|
|
||||
|
> 版本:v0.2 日期:2026-10-18 |
||||
|
> 状态:**设计稿,未写代码。** |
||||
|
> 范围:耳机唤醒和 App 里按住说话,两条入口共用这一套。通话 / 同声 / 面对面 / 音视频不在本文。 |
||||
|
> 对照:[科大讯飞SparkChain-全面替换第三方-落地文档](科大讯飞SparkChain-全面替换第三方-落地文档.md) 第 5.5 节是「三段接上、先问通」;本文把时间、地点、天气、新闻、个人工具的分工定死。 |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
## 零、结论 |
||||
|
|
||||
|
每轮对话都带上**手机本地时间**和**缓存地址**,只解决「现在几点、今天星期几、我在哪」。 |
||||
|
|
||||
|
天气、新闻和其它要联网才知道的问题,模型手里只有这两样事实,答不出来。这两类仍由手机执行工具,把结果喂回模型,再合成。 |
||||
|
|
||||
|
个人数据(纪要、拾忆、待办)同样是手机拿登录身份去调已有 MCP。对话本身不经过业务接口。 |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
## 一、时间地址怎么带(相对「原样塞进每句话」改过的地方) |
||||
|
|
||||
|
| | 做法 | 原因 | |
||||
|
|---|---|---| |
||||
|
| 时间 | 每一轮用户说完,用手机本地钟**重新拼一次** | 会话可能持续十几分钟。「现在几点」不能用开场那一刻 | |
||||
|
| 地址 | 只读 `DeviceLocationService` 的缓存,**这一轮不等 GPS** | 定位会拖住开口。缓存规则维持现状:回前台刷新,10 分钟内不重复 | |
||||
|
| 放哪 | 放进**本轮系统提示**,不拼进用户原话,不写入聊天历史 | 写进原话,模型会当成用户自己说的;写进历史,旧时间会把「现在」答错 | |
||||
|
| 经纬度 | **不进模型**。只有天气工具要用时,手机自己附上 | 模型念出坐标没有用,还容易被读出来 | |
||||
|
| 没有位置 | 系统提示写「当前没有位置」,不要写「未知」 | 「未知」会被当成地名 | |
||||
|
| 专问「我在哪」且缓存空 | 只这一问允许再等一次定位,上限 4 秒,到点仍没有就按「当前没有位置」答 | 别的问题(含天气)不为这一问阻塞 | |
||||
|
|
||||
|
时间文案用手机时区,带星期和偏移,例如:`2026年10月18日 星期日 00:59,东八区(UTC+8)`。 |
||||
|
|
||||
|
地址文案沿用现在能念出来的那串(完整地址,没有再退省市区街道)。念「我在哪」用这串。查天气只用其中的**城市名**。 |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
## 二、整轮怎么走 |
||||
|
|
||||
|
SDK 没有百炼那种边听边说。一轮是说完再答。耳机协议、问候语、打断、后台保活维持第 5.5 节,本文只换「听完之后谁来答」。 |
||||
|
|
||||
|
``` |
||||
|
麦克风 PCM |
||||
|
→ 讯飞 SDK 实时转写,区域 RegionMSFT,语种 = 下面第三节 |
||||
|
→ 一句最终文本 |
||||
|
→ 本轮系统提示(新时间 + 缓存地址 + 回答规矩)+ 最近对话 + 工具清单 |
||||
|
→ 讯飞 SDK 文本对话 LLM(端上,不经业务服务器) |
||||
|
├─ 直接有文字 → 讯飞 SDK 在线合成,区域 RegionMSFT,音色 = 第三节 |
||||
|
└─ 返回 functionCall → 手机执行工具(最多两轮)→ 结果喂回 LLM → 再合成 |
||||
|
``` |
||||
|
|
||||
|
两条入口(`DeviceAiSessionService` 耳机唤醒、`EmaiController` 按住说话)共用这一个编排。PCM 从哪来、喇叭往哪放,仍由各自的会话服务决定。 |
||||
|
|
||||
|
空识别不调用对话模型。工具失败时合成「暂时查不到」,不让模型编一个天气或一条新闻。工具来回超过两轮就停,用已经拿到的结果回答;还什么都没有就说查不到。 |
||||
|
|
||||
|
对话模型用 SDK 的文本 `LLM`(海外 `domain=openai`,文档点名 `gpt-5.3-chat` / `gpt-5.4`)。听写和合成才是 SDK 里的微软区域(`RegionMSFT`)。会议总结走同一套 OpenAI 文本模型,不走星火,也不拿来做 EMAI 的实时对话。 |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
## 三、识别语种和合成音色 |
||||
|
|
||||
|
跟设置里的**界面语言**走,不跟用户刚说的那一句走。 |
||||
|
|
||||
|
设置页切换语言时,`settings_controller` 已经把该语言收成一条 ASR 区域码(跟随系统则用匹配后的系统语言,兜底 `en-US`)。EMAI 直接用这一条: |
||||
|
|
||||
|
| | 取值 | |
||||
|
|---|---| |
||||
|
| 实时转写 `lang` | 上面那条区域码,例如界面中文 → `zh-CN` | |
||||
|
| 合成发音人 | 同一条语言在音色表里的神经语音。性别只有 `1` 用男声;未设置 `0` 和女 `2` 用女声。用户对象还没就绪时用女声 | |
||||
|
| 回答语言 | 与这条界面语言一致,写进系统提示 | |
||||
|
|
||||
|
用户用别的语言插一句,本轮仍按界面语言识别和回答。要换语种,去设置里改,不在对话中途猜。 |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
## 四、问题分成四类 |
||||
|
|
||||
|
系统提示里写明:时间、星期、日期、地点只用本轮给出的事实,不要调用工具,也不要改口。 |
||||
|
|
||||
|
| 用户在问 | 谁处理 | 落地 | |
||||
|
|---|---|---| |
||||
|
| 几点、星期几、今天几号 | 模型直接答 | 只用本轮时间。不调工具 | |
||||
|
| 我在哪 | 模型直接答 | 有地址就念地址。没有就说还没有位置,不要编一个城市 | |
||||
|
| 天气 | 手机执行 `hefeng_weather` | 城市用缓存里的城市名,日期用本轮日期。有城市就不要反问「哪个城市」。没有位置时才问一次城市,问到了再查 | |
||||
|
| 新闻、实时信息 | 手机执行联网搜索 | 搜索词带上本轮日期,避免「今天」落到模型的训练截止日期。海外节点用 `tavily_web_search`,国内节点用 `bocha_web_search`,和现有 MCP 分组一致 | |
||||
|
| 我的会议、记忆、待办 | 手机执行个人 MCP | 见第五节。没有的就说没有,不要编 | |
||||
|
|
||||
|
天气和搜索的密钥留在现有 MCP 进程里。手机只发起这一次工具调用,不把天气接口嵌进对话链路,也不把密钥放进安装包。 |
||||
|
|
||||
|
和风天气的入参是城市名加可选日期,不是经纬度。所以坐标只在缓存里,给不了城市名时不能拿坐标假装能查。 |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
## 五、个人 MCP |
||||
|
|
||||
|
手机调用,数据仍在用户自己的服务器上。这一跳是工具,不是把整段对话交给业务接口。 |
||||
|
|
||||
|
沿用现在这 7 个、且要身份的工具: |
||||
|
|
||||
|
`search_meeting_notes`、`get_memory_items`、`get_memory_stats`、`get_memory_report`、`get_user_tasks`、`add_user_task`、`cancel_user_task`。 |
||||
|
|
||||
|
身份仍用 `user_getmcptoken` 的短时令牌,按工具名附在调用上。令牌不写进系统提示,不进历史,不出现在给模型的工具参数里。模型只看到工具名和参数说明。 |
||||
|
|
||||
|
百炼时期令牌要按工具名嵌进 `user_defined_params`,是因为第三方云在代调。换成手机自己调之后,手机在发 MCP 请求时带上令牌即可,形状不再跟百炼的嵌套走。 |
||||
|
|
||||
|
`hefeng_weather` 不在这 7 个里面。它和联网搜索不需要用户身份,手机调用时不要附登录令牌。 |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
## 六、和现在百炼的对应 |
||||
|
|
||||
|
| 百炼现在 | 换成 | |
||||
|
|---|---| |
||||
|
| 会话开始时把日期、地址放进 `user_prompt_params`,整场不变 | 每轮重算时间;地址仍只读缓存 | |
||||
|
| `client_info.location` 把经纬度交给云端 | 经纬度不进模型 | |
||||
|
| 云端插件自己搜网页、自己查天气 | 模型只返回要调哪个工具;手机执行,再把结果喂回去 | |
||||
|
| 个人 MCP 由云端代调,令牌按工具名塞进会话 | 手机自己调,令牌只附在那一次请求上 | |
||||
|
| 边听边说 | 做不到。说完一句再答 | |
||||
|
|
||||
|
测试服 `aiOnlyIflytek` 已经让百炼 `start` 直接返回失败。本文落地前 EMAI 在测试服起不来。不要为了对照把百炼重新打开。 |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
## 七、明确不做 |
||||
|
|
||||
|
- 不把对话转发到 `user_*` 业务接口,也不在服务端再包一层「代答」。 |
||||
|
- 不把会议总结的星火接口拿来当 EMAI 的对话模型。 |
||||
|
- 不在每一轮阻塞等 GPS。 |
||||
|
- 不把经纬度、令牌、天气密钥写进提示词。 |
||||
|
- 不改耳机协议字节,不改通话 / 同声 / 面对面 / 音视频的选路。 |
||||
|
- 第一版不恢复百炼的全双工。工具调用失败不影响「问一句有回答」。 |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
## 八、验收 |
||||
|
|
||||
|
1. 问「现在几点」「今天星期几」:答的是手机本地时间,和会话开始时的时间可以不同。 |
||||
|
2. 定位开着时问「我在哪」:念缓存地址,不念经纬度。关掉定位再问:说没有位置,不编城市。 |
||||
|
3. 有城市缓存时问天气:不反问城市,答案来自工具。把工具结果改成失败时,嘴里是「暂时查不到」,不是编出来的温度。 |
||||
|
4. 问「今天新闻」:搜索词里带当天日期,回答来自搜索结果。 |
||||
|
5. 问「我最近的会议 / 待办」:走个人 MCP,换一个账号看不到上一个账号的数据。抓对话请求里看不到令牌。 |
||||
|
6. 设置改成英语后,识别和回答都是英语,音色是该语言的女声;性别设为男之后换成男声。 |
||||
|
7. 耳机按键和 App 按住说话,问同一句,走的是同一套编排。 |
||||
@ -0,0 +1,184 @@ |
|||||
|
# 同声翻译改走微软端到端 |
||||
|
|
||||
|
> 2026-10-18。只覆盖翻译页的同声模式(`simultaneous`)。本文只做设计,不改代码。 |
||||
|
> 端到端依据 iOS 集成文档《同声传译简介》第 9.3、9.6 节。拼接时的识别依据安卓海外文档《实时语音转写》第 8.2、9.3 节(iOS《实时语音转写简介》只写了微软模式必填 `language_name`,没有语种表)。合成依据安卓海外文档《在线语音合成》的微软发音人表。 |
||||
|
> 同声这一条覆盖 [科大讯飞SparkChain-全面替换第三方-落地文档.md](科大讯飞SparkChain-全面替换第三方-落地文档.md) 第 5.1 节里「表外拆成星火听写」的写法。面对面、通话、音视频、EMAI 不在本文范围。 |
||||
|
|
||||
|
## 结论 |
||||
|
|
||||
|
两端语种都在微软同传表里:只开一条 `AST`,机房 `RegionMSFT`(安卓 `RegionType.MSFT_TYPE`)。识别、翻译、合成由这一条会话完成。 |
||||
|
|
||||
|
源或目标是中文、粤语、爱尔兰语:微软同传表没有这三语,改走拼接。拼接的识别必须是微软实时转写 `RTASR(RegionMSFT)`,用 `language_name` 指定语种。不要用星火听写 `ASR`(`zh_cn` / `cn_cbm` / `iat`),也不要用新加坡、欧洲机房那两张转写表(`ist_open`、`ist_mul_sp`、`ga_IE`)。 |
||||
|
|
||||
|
拼接的合成用 `OnlineTTS(RegionMSFT)`,发音人用微软神经音色。不要用 `xiaoyan`、`x_xiaoyan`、`x_Catherine`,也不要打 `tts-api.xfyun.cn`。 |
||||
|
|
||||
|
## 现在为什么一种声音念所有语言 |
||||
|
|
||||
|
同声在讯飞听写成功之后,是三段接起来的,没有用到 `AST`。 |
||||
|
|
||||
|
1. `_startSparkListen` 调 `SparkChainSession.startListen`,只做听写。 |
||||
|
2. 整句文本再走原来的翻译。 |
||||
|
3. `playTranslatedText` 算出的 `voiceName` 在 `_sparkListen` 分支被丢掉,`_playSparkTts` 只把目标语种短码传给 `SparkChainSession.speak`。 |
||||
|
|
||||
|
原生合成不看音色表: |
||||
|
|
||||
|
| 端 | 英语 | 其余语种 | |
||||
|
|---|---|---| |
||||
|
| iOS `ClassicTts`,打 `tts-api.xfyun.cn` | `x2_catherine` | `xiaoyan` | |
||||
|
| Android `OnlineTTS` | `x_Catherine` | `x_xiaoyan`(爱尔兰语例外,用 `ga-IE-OrlaNeural`) | |
||||
|
|
||||
|
日语、韩语、法语都会用中文晓燕念出来。`getIflytekTTSLanguageSpecs()` 里按语种分好的发音人没有接到这条播报上。那张表是讯飞发音人,本方案也不再用。 |
||||
|
|
||||
|
## 目标链路 |
||||
|
|
||||
|
麦克风 PCM 仍由现在的耳机麦或手机麦采集,SDK 不自己开麦。采集、保活、播放路由保持现有调用点。 |
||||
|
|
||||
|
``` |
||||
|
耳机麦 / 手机麦 PCM |
||||
|
│ 16 kHz / 16 bit / 单声道,每 40ms 1280 字节 |
||||
|
▼ |
||||
|
AST(RegionMSFT) |
||||
|
start(源语种, "pcm_s16le", 16000) |
||||
|
use_tts = true |
||||
|
target_lang = 目标语种 |
||||
|
tts_target_lang = 目标语种 |
||||
|
tts_speaker = 该目标语种的默认发音人 |
||||
|
│ |
||||
|
├─ resType=asr / trans → 原文、译文上屏(流式,不再发翻译请求) |
||||
|
└─ resType=tts → ttsResult.audio 交给现有 PcmStreamPlayer |
||||
|
``` |
||||
|
|
||||
|
iOS: |
||||
|
|
||||
|
```objc |
||||
|
AST *ast = [[AST alloc] initWithregion:RegionMSFT delegate:self]; |
||||
|
[ast addStringParam:@"target_lang" value:@"en"]; |
||||
|
[ast addBoolParam:@"use_tts" value:YES]; |
||||
|
[ast addStringParam:@"tts_target_lang" value:@"en"]; |
||||
|
[ast addStringParam:@"tts_speaker" value:@"JennyNeural"]; |
||||
|
[ast start:@"ja" audioEncoding:@"pcm_s16le" sampleRate:16000 userInfo:tag]; |
||||
|
``` |
||||
|
|
||||
|
安卓用同一个机房枚举和同一组参数名。这条端到端会话的 `tts_speaker` 用第 9.6 节的短名(`JennyNeural`)。`en-US-JennyNeural` 是拼接路径里 `OnlineTTS` 的写法,不要填进 `AST`。 |
||||
|
|
||||
|
方向固定为用户选的源 → 目标。不要用识别回来的语种对调方向。 |
||||
|
|
||||
|
停的顺序不变:先停推流和 `AST.stop(NO)`,再关耳机麦,再放保活。 |
||||
|
|
||||
|
## 语种和音色 |
||||
|
|
||||
|
源语种、目标语种都必须在下表里。默认发音人取该行的第一个;同行其余名字是可换的音色,换的时候整段会话重开,不要在 `write` 过程中改 `tts_speaker`。 |
||||
|
|
||||
|
界面短码和表里的 `language` 一致的直接用。应用里的 `zh` 不要写成 `cn` 再送进微软会话:微软这张表没有中文。 |
||||
|
|
||||
|
| 语种 | language | 默认发音人 | 还可选 | |
||||
|
|---|---|---|---| |
||||
|
| 阿拉伯语 | ar | AbdullahNeural | AliNeural、AmalNeural、AmanyNeural、AminaNeural、AyshaNeural、BasselNeural、FahedNeural、FatimaNeural、HamdanNeural、HamedNeural、HediNeural、ImanNeural、IsmaelNeural、JamalNeural、LailaNeural、LaithNeural、LaylaNeural、MaryamNeural、MoazNeural、MounaNeural、NouraNeural、OmarNeural、RamiNeural、RanaNeural、ReemNeural、SalehNeural、SalmaNeural、SanaNeural、ShakirNeural、TaimNeural、ZariyahNeural | |
||||
|
| 保加利亚语 | bg | BorislavNeural | KalinaNeural | |
||||
|
| 孟加拉语 | bn | BashkarNeural | NabanitaNeural、PradeepNeural、TanishaaNeural | |
||||
|
| 加泰罗尼亚语 | ca | AlbaNeural | EnricNeural、JoanaNeural | |
||||
|
| 捷克语 | cs | AntoninNeural | VlastaNeural | |
||||
|
| 丹麦语 | da | ChristelNeural | JeppeNeural | |
||||
|
| 德语 | de | AmalaNeural | BerndNeural、ChristophNeural、ConradNeural、ElkeNeural、GiselaNeural、IngridNeural、JanNeural、JonasNeural、KasperNeural、KatjaNeural、KillianNeural、KlarissaNeural、KlausNeural、LeniNeural、LouisaNeural、MajaNeural、RalfNeural、TanjaNeural | |
||||
|
| 希腊语 | el | AthinaNeural | NestorasNeural | |
||||
|
| 英语 | en | JennyNeural | GuyNeural、AriaNeural、DavisNeural、JaneNeural、JasonNeural、NancyNeural、TonyNeural、WayneNeural、WilliamNeural、RyanNeural、SoniaNeural、LibbyNeural、NatashaNeural、ClaraNeural、LiamNeural、NeerjaNeural、PrabhatNeural、LunaNeural | |
||||
|
| 西班牙语 | es | ElviraNeural | AlvaroNeural、AbrilNeural、ArnauNeural、DarioNeural、ElenaNeural、TomasNeural、DaliaNeural、JorgeNeural、PalomaNeural、AlonsoNeural | |
||||
|
| 爱沙尼亚语 | et | AnuNeural | KertNeural | |
||||
|
| 波斯语 | fa | DilaraNeural | FaridNeural | |
||||
|
| 芬兰语 | fi | HarriNeural | NooraNeural、SelmaNeural | |
||||
|
| 法语 | fr | DeniseNeural | HenriNeural、AlainNeural、BrigitteNeural、ClaudeNeural、SylvieNeural、JeanNeural、AntoineNeural | |
||||
|
| 印地语 | hi | MadhurNeural | SwaraNeural | |
||||
|
| 匈牙利语 | hu | NoemiNeural | TamasNeural | |
||||
|
| 印尼语 | id | ArdiNeural | GadisNeural | |
||||
|
| 意大利语 | it | ElsaNeural | IsabellaNeural、DiegoNeural、BenignoNeural、FabiolaNeural、GianniNeural | |
||||
|
| 日语 | ja | NanamiNeural | KeitaNeural、AoiNeural、DaichiNeural、MayuNeural、NaokiNeural、ShioriNeural | |
||||
|
| 韩语 | ko | SunHiNeural | InJoonNeural、BongJinNeural、HyunsuNeural、JiMinNeural、SeoHyeonNeural | |
||||
|
| 立陶宛语 | lt | LeonasNeural | OnaNeural | |
||||
|
| 拉脱维亚语 | lv | EveritaNeural | NilsNeural | |
||||
|
| 马来语 | ms | OsmanNeural | YasminNeural | |
||||
|
| 荷兰语 | nl | ArnaudNeural | ColetteNeural、FennaNeural、MaartenNeural | |
||||
|
| 波兰语 | pl | AgnieszkaNeural | MarekNeural、ZofiaNeural | |
||||
|
| 葡萄牙语 | pt | AntonioNeural | BrendaNeural、DuarteNeural、FranciscaNeural、RaquelNeural、YaraNeural | |
||||
|
| 罗马尼亚语 | ro | AlinaNeural | EmilNeural | |
||||
|
| 俄语 | ru | DariyaNeural | DmitryNeural、SvetlanaNeural | |
||||
|
| 斯洛伐克语 | sk | LukasNeural | ViktoriaNeural | |
||||
|
| 斯洛文尼亚语 | sl | PetraNeural | RokNeural | |
||||
|
| 瑞典语 | sv | HilleviNeural | MattiasNeural、SofieNeural | |
||||
|
| 斯瓦希里语 | sw | DaudiNeural | RafikiNeural、RehemaNeural、ZuriNeural | |
||||
|
| 泰米尔语 | ta | AnbuNeural | KaniNeural、KumarNeural、PallaviNeural | |
||||
|
| 泰语 | th | AcharaNeural | NiwatNeural、PremwadeeNeural | |
||||
|
| 土耳其语 | tr | AhmetNeural | EmelNeural | |
||||
|
| 乌克兰语 | uk | OstapNeural | PolinaNeural | |
||||
|
| 乌尔都语 | ur | AsadNeural | GulNeural、SalmanNeural、UzmaNeural | |
||||
|
| 越南语 | vi | HoaiMyNeural | NamMinhNeural | |
||||
|
|
||||
|
文档没有给这些发音人标性别。默认列只是每行的第一个名字,用来保证换语言就换音色。 |
||||
|
|
||||
|
## 中文、粤语、爱尔兰语:拼接,但识别走微软 |
||||
|
|
||||
|
微软同传表没有这三语。源或目标有其中一种时,不启 `AST`,改成三段: |
||||
|
|
||||
|
``` |
||||
|
耳机麦 / 手机麦 PCM |
||||
|
▼ |
||||
|
RTASR(RegionMSFT) 只识别源语种 |
||||
|
lang = "msft" 固定值,不要把 zh-CN 填进 lang |
||||
|
accent = "mandarin" 固定值 |
||||
|
domain = "ist_mul_msft" 固定值 |
||||
|
language_name = 下表 真正的语种 |
||||
|
▼ |
||||
|
整句文本翻译 不再次开识别 |
||||
|
▼ |
||||
|
OnlineTTS(RegionMSFT) 按目标语种选微软神经音色 |
||||
|
▼ |
||||
|
现有 PcmStreamPlayer |
||||
|
``` |
||||
|
|
||||
|
安卓文档的写法: |
||||
|
|
||||
|
```java |
||||
|
mRTASR.lang("msft"); |
||||
|
mRTASR.OstAccent("mandarin"); |
||||
|
mRTASR.OstDomain("ist_mul_msft"); |
||||
|
mRTASR.setParams("language_name", "zh-CN"); |
||||
|
``` |
||||
|
|
||||
|
iOS 同样用 `RTASR(region: RegionMSFT)`,`language_name` 用下表。现在插件把界面语种直接写进 `engine.lang`,和上面的固定值不符,拼接落地时要改过来。 |
||||
|
|
||||
|
识别语种和这条路径的默认音色: |
||||
|
|
||||
|
| 界面语种 | 识别 `language_name` | 作为译文时的默认音色 | |
||||
|
|---|---|---| |
||||
|
| 普通话 `zh` | `zh-CN` | `zh-CN-XiaoxiaoNeural` | |
||||
|
| 粤语简体 `yue` | `yue-CN` | `yue-CN-XiaoMinNeural` | |
||||
|
| 粤语繁体 / 香港 `zh-HK` | `zh-HK` | `zh-HK-HiuMaanNeural` | |
||||
|
| 爱尔兰语 `ga` | `ga-IE` | `ga-IE-OrlaNeural` | |
||||
|
|
||||
|
另一端如果是微软同传表里的语言,识别用该语言的区域码(英语 `en-US`、日语 `ja-JP`),合成用 `OnlineTTS` 的区域码音色(英语 `en-US-JennyNeural`,对应同传表里的 `JennyNeural`)。不要用星火的 `x_John`、`x_Catherine`、`xiaoyan`。 |
||||
|
|
||||
|
现在同声的 `startListen` 在非爱尔兰语时走进 `ASR()`,`language=zh_cn`、`domain=iat`。那是星火听写。中文、粤语、爱尔兰语的拼接不准再进这个分支。 |
||||
|
|
||||
|
翻译是整句文本,方向仍固定为源 → 目标。中间结果只上屏,整句再翻译、再合成。 |
||||
|
|
||||
|
两端都不在微软同传表里、又不是中文、粤语、爱尔兰语:同声不启动,也不退回星火听写。 |
||||
|
|
||||
|
## 改哪些地方 |
||||
|
|
||||
|
| 位置 | 改动 | |
||||
|
|---|---| |
||||
|
| `sparkchain` 插件 iOS / Android | 同传表内:新增 `AST(RegionMSFT)`,`use_tts` 和 `tts_speaker` 按上一节。表外三语:`RTASR` 按上面的四个固定参数加 `language_name` 启动;`OnlineTTS(RegionMSFT)` 用微软神经音色。同声路径不再创建 `ASR`,不再走 `ClassicTts` | |
||||
|
| `sparkchain_session.dart` | 同声按语种对选一条:`startAst` 或 `startMsftListen`。`startListen` 里现在的星火 `ASR` 不给同声用 | |
||||
|
| `translation_controller.dart` 同声分支 | 表内:PCM 写入 `AST`,译文用 `trans`,音频用 `tts`。表外三语:PCM 写入微软 `RTASR`,整句再翻译,再用 `OnlineTTS` 播目标语种。播放仍进现在的 `PcmStreamPlayer` | |
||||
|
| 发音人 | 表内用同传短名;拼接用上表的 `xx-XX-NameNeural`。不要读 `getIflytekTTSLanguageSpecs` | |
||||
|
|
||||
|
`SparkChain.init` 的 `apiType` 仍按现有配置。`RegionMSFT` 是这一次会话的机房,不要为了同声把全局节点改掉。 |
||||
|
|
||||
|
## 验收 |
||||
|
|
||||
|
- 日语 → 英语:一条 `AST`,`tts_speaker=JennyNeural`。没有 `ASR`,没有 `xiaoyan` / `tts-api.xfyun.cn`。 |
||||
|
- 英语 → 日语:`tts_speaker=NanamiNeural`。两种方向的声音不同。 |
||||
|
- 中文 → 英语:日志是 `RTASR`、`language_name=zh-CN`、`lang=msft`。译文用 `en-US-JennyNeural`。没有 `ASR`、没有 `zh_cn`、没有晓燕。 |
||||
|
- 英语 → 中文:识别 `language_name=en-US`,播报 `zh-CN-XiaoxiaoNeural`。 |
||||
|
- 中文 → 爱尔兰语:识别 `zh-CN`,播报 `ga-IE-OrlaNeural`。 |
||||
|
- 粤语 → 英语:识别 `yue-CN` 或 `zh-HK`,对上用户选的简体或香港繁体。 |
||||
|
- 连耳机时收音日志仍是「同声翻译收音:耳机麦」,译文从耳机出。 |
||||
Loading…
Reference in new issue