13 changed files with 1885 additions and 959 deletions
File diff suppressed because it is too large
@ -0,0 +1,783 @@ |
|||||
|
package com.yunqiinnovation.agent_service |
||||
|
|
||||
|
import android.content.Context |
||||
|
import android.media.AudioFormat |
||||
|
import android.media.AudioRecord |
||||
|
import android.media.MediaPlayer |
||||
|
import android.media.MediaRecorder |
||||
|
import android.os.Handler |
||||
|
import android.os.Looper |
||||
|
import android.util.Log |
||||
|
import com.yunqiinnovation.chat_storage.ChatStorageHelper |
||||
|
import kotlinx.coroutines.* |
||||
|
import okhttp3.* |
||||
|
import okhttp3.MediaType.Companion.toMediaType |
||||
|
import okhttp3.RequestBody.Companion.toRequestBody |
||||
|
import okio.ByteString.Companion.toByteString |
||||
|
import org.json.JSONArray |
||||
|
import org.json.JSONObject |
||||
|
import java.io.IOException |
||||
|
import java.security.MessageDigest |
||||
|
import java.security.SecureRandom |
||||
|
import java.security.cert.X509Certificate |
||||
|
import java.util.UUID |
||||
|
import java.util.concurrent.TimeUnit |
||||
|
import javax.net.ssl.SSLContext |
||||
|
import javax.net.ssl.TrustManager |
||||
|
import javax.net.ssl.X509TrustManager |
||||
|
import kotlin.coroutines.resumeWithException |
||||
|
import kotlin.math.sqrt |
||||
|
|
||||
|
/** |
||||
|
* STS 配置参数（移动精灵 A80 平台） |
||||
|
*/ |
||||
|
data class StsConfig( |
||||
|
val productId: String, |
||||
|
val productKey: String, |
||||
|
var deviceNo: String, |
||||
|
var deviceId: String, |
||||
|
var deviceSecret: String, |
||||
|
val agentId: String, |
||||
|
) |
||||
|
|
||||
|
/** |
||||
|
* StsAgent — 中国移动 A80 WS 一体化语音对话实现 |
||||
|
* |
||||
|
* 通过一条 WSS 连接完成 ASR + AI + TTS，事件以与标准 stt_llm_tts 相同的格式 |
||||
|
* 通过 [onEvent] 回调发出，供 AgentService 转发到 Flutter EventChannel。 |
||||
|
*/ |
||||
|
class StsAgent( |
||||
|
private val context: Context, |
||||
|
private var config: StsConfig, |
||||
|
private val onEvent: (eventName: String, data: Map<String, Any>) -> Unit, |
||||
|
private val onCredentialsFetched: ((deviceId: String, deviceSecret: String, deviceNo: String) -> Unit)? = null, |
||||
|
) : CoroutineScope { |
||||
|
|
||||
|
private val TAG = "StsAgent" |
||||
|
private val job = SupervisorJob() |
||||
|
override val coroutineContext = Dispatchers.IO + job |
||||
|
|
||||
|
init { |
||||
|
fetchCredentials() |
||||
|
} |
||||
|
|
||||
|
// ── WebSocket ───────────────────────────────────────────────────────────── |
||||
|
private val okHttpClient: OkHttpClient = buildTrustAllClient() |
||||
|
private var webSocket: WebSocket? = null |
||||
|
private var readyReceived = false |
||||
|
private var sessionEnded = true |
||||
|
private var dialogRequestId = "" |
||||
|
|
||||
|
// ── 录音 ────────────────────────────────────────────────────────────────── |
||||
|
private val SAMPLE_RATE = 16000 |
||||
|
private val FRAME_SIZE = 5120 // 160ms @ 16kHz 16bit mono |
||||
|
private val RECORD_CHUNK = FRAME_SIZE / 2 // 读取块大小 |
||||
|
private val RECORD_BUFFER = maxOf( |
||||
|
AudioRecord.getMinBufferSize( |
||||
|
SAMPLE_RATE, |
||||
|
AudioFormat.CHANNEL_IN_MONO, |
||||
|
AudioFormat.ENCODING_PCM_16BIT |
||||
|
), FRAME_SIZE * 2 |
||||
|
) |
||||
|
private var audioRecord: AudioRecord? = null |
||||
|
private var recordingJob: Job? = null |
||||
|
private var micChunkCount = 0 |
||||
|
|
||||
|
// ── 发送缓冲 ────────────────────────────────────────────────────────────── |
||||
|
private val sendBuffer = ArrayDeque<Byte>() |
||||
|
private var sendBufferOffset = 0 |
||||
|
|
||||
|
// ── 预缓冲（VAD 触发前 ~1.5s）──────────────────────────────────────────── |
||||
|
private val preBuffer = ArrayDeque<ByteArray>() |
||||
|
private val PRE_BUFFER_MAX_CHUNKS = 75 |
||||
|
|
||||
|
// ── 本地 VAD ────────────────────────────────────────────────────────────── |
||||
|
private var speechDetected = false |
||||
|
private val VAD_THRESHOLD = 800.0 |
||||
|
private val VAD_ONSET_CHUNKS = 3 |
||||
|
private var vadOnsetCount = 0 |
||||
|
|
||||
|
// ── 状态标记 ────────────────────────────────────────────────────────────── |
||||
|
private var forwardingAudio = false |
||||
|
private var wsConnecting = false |
||||
|
private var inCall = false |
||||
|
private var pttRecording = false |
||||
|
|
||||
|
// ── TTS 播放队列 ────────────────────────────────────────────────────────── |
||||
|
private val ttsQueue = ArrayDeque<String>() |
||||
|
private var ttsPlaying = false |
||||
|
private var ttsInterrupted = false |
||||
|
private var currentPlayer: MediaPlayer? = null |
||||
|
private val mainHandler = Handler(Looper.getMainLooper()) |
||||
|
/** 静音开关：设为 true 时停止当前播放、跳过后续 TTS，文本接收完后主动关闭连接 */ |
||||
|
var ttsMuted = false |
||||
|
set(value) { |
||||
|
field = value |
||||
|
if (value) { |
||||
|
// 立即停止当前播放 |
||||
|
ttsQueue.clear() |
||||
|
ttsPlaying = false |
||||
|
mainHandler.post { |
||||
|
try { currentPlayer?.stop(); currentPlayer?.release() } catch (_: Exception) {} |
||||
|
currentPlayer = null |
||||
|
} |
||||
|
emit("tts_canceled", emptyMap()) |
||||
|
// 如果会话已结束（文本已全部接收），主动关闭连接 |
||||
|
if (sessionEnded) { |
||||
|
closeWs() |
||||
|
if (inCall) { resetVad(); startIdleTimer() } |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// ── 消息历史（WS 上下文） ────────────────────────────────────────────────── |
||||
|
private var messages = JSONArray() |
||||
|
private val chatStorageHelper = ChatStorageHelper(context) |
||||
|
private var currentSessionId = "" |
||||
|
private var pendingAnswer = "" // 累积最完整的回复文本，会话结束时统一发送 |
||||
|
|
||||
|
// ── 计时器 ──────────────────────────────────────────────────────────────── |
||||
|
private var micWatchdogJob: Job? = null |
||||
|
private var lastWatchdogChunkCount = 0 |
||||
|
private var idleTimerJob: Job? = null |
||||
|
private val IDLE_TIMEOUT_MS = 15_000L |
||||
|
|
||||
|
// ── 凭证就绪 ────────────────────────────────────────────────────────────── |
||||
|
var credentialReady = false |
||||
|
private set |
||||
|
|
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
// 凭证获取 |
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
|
||||
|
fun fetchCredentials() { |
||||
|
if (config.deviceSecret.isNotEmpty()) { |
||||
|
credentialReady = true |
||||
|
return |
||||
|
} |
||||
|
launch { |
||||
|
try { |
||||
|
val body = JSONObject().apply { |
||||
|
put("deviceNoType", "SN") |
||||
|
put("deviceNo", config.deviceNo) |
||||
|
put("productId", config.productId) |
||||
|
put("productKey", config.productKey) |
||||
|
}.toString().toRequestBody("application/json".toMediaType()) |
||||
|
|
||||
|
val req = Request.Builder() |
||||
|
.url("https://ivs.chinamobiledevice.com:11443/v2/customer/device/secret/info") |
||||
|
.post(body) |
||||
|
.build() |
||||
|
|
||||
|
okHttpClient.newCall(req).execute().use { resp -> |
||||
|
val json = JSONObject(resp.body!!.string()) |
||||
|
if (json.getString("code") == "200" && json.getBoolean("success")) { |
||||
|
val d = json.getJSONObject("data") |
||||
|
config = config.copy( |
||||
|
deviceNo = d.optString("deviceNo", config.deviceNo), |
||||
|
deviceId = d.optString("deviceId", ""), |
||||
|
deviceSecret = d.optString("deviceSecret", ""), |
||||
|
) |
||||
|
credentialReady = config.deviceSecret.isNotEmpty() |
||||
|
if (credentialReady) { |
||||
|
onCredentialsFetched?.invoke(config.deviceId, config.deviceSecret, config.deviceNo) |
||||
|
} |
||||
|
} else { |
||||
|
Log.e(TAG, "凭证获取失败: ${json.optString("message")}") |
||||
|
} |
||||
|
} |
||||
|
} catch (e: Exception) { |
||||
|
Log.e(TAG, "凭证获取异常: ${e.message}") |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
// 公开接口 |
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
|
||||
|
/** 通话模式：开启麦克风，VAD 自动触发 WS */ |
||||
|
fun startCall() { |
||||
|
if (inCall) return |
||||
|
if (ttsPlaying) interruptTts() |
||||
|
if (pttRecording) stopPtt(cancel = true) |
||||
|
if (!credentialReady) { emitError("移动精灵凭证未就绪，请稍后重试"); return } |
||||
|
openMic() |
||||
|
startMicWatchdog() |
||||
|
inCall = true |
||||
|
startIdleTimer() |
||||
|
} |
||||
|
|
||||
|
/** PTT 模式：立即连 WS 并开始录音 */ |
||||
|
fun startPtt() { |
||||
|
if (pttRecording) return |
||||
|
if (ttsPlaying) interruptTts() |
||||
|
if (!credentialReady) { emitError("移动精灵凭证未就绪"); return } |
||||
|
|
||||
|
pttRecording = true |
||||
|
openMic() |
||||
|
startMicWatchdog() |
||||
|
ttsInterrupted = false |
||||
|
closeWs() |
||||
|
sessionEnded = false |
||||
|
wsConnecting = true |
||||
|
dialogRequestId = UUID.randomUUID().toString() |
||||
|
clearSendBuffer() |
||||
|
pendingAnswer = "" |
||||
|
|
||||
|
launch { |
||||
|
try { |
||||
|
connectWs() |
||||
|
forwardingAudio = true |
||||
|
wsConnecting = false |
||||
|
emit("recognition_started", emptyMap()) |
||||
|
} catch (e: Exception) { |
||||
|
wsConnecting = false |
||||
|
pttRecording = false |
||||
|
stopMicWatchdog() |
||||
|
emitError("连接失败: ${e.message}") |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/** PTT 松开：发送剩余音频 + finish */ |
||||
|
fun stopPtt(cancel: Boolean = false) { |
||||
|
pttRecording = false |
||||
|
stopMicWatchdog() |
||||
|
if (cancel) { |
||||
|
forwardingAudio = false |
||||
|
closeWs() |
||||
|
stopMicHard() |
||||
|
return |
||||
|
} |
||||
|
if (forwardingAudio) { |
||||
|
forwardingAudio = false |
||||
|
synchronized(sendBufferLock) { |
||||
|
val remain = sendBuffer.size - sendBufferOffset |
||||
|
if (remain > 0) { |
||||
|
sendAudioChunk(sendBuffer.toMutableList().subList(sendBufferOffset, sendBuffer.size).toByteArray()) |
||||
|
} |
||||
|
sendBuffer.clear(); sendBufferOffset = 0 |
||||
|
} |
||||
|
sendFinish() |
||||
|
emit("recognition_stopped", emptyMap()) |
||||
|
} |
||||
|
stopMicHard() |
||||
|
} |
||||
|
|
||||
|
/** 统一停止：根据当前状态自动选择 stopPtt 或 disconnect */ |
||||
|
fun stop() { |
||||
|
if (pttRecording) stopPtt() |
||||
|
else disconnect() |
||||
|
} |
||||
|
|
||||
|
/** 挂断通话 */ |
||||
|
fun disconnect() { |
||||
|
stopMicWatchdog() |
||||
|
cancelIdleTimer() |
||||
|
forwardingAudio = false |
||||
|
wsConnecting = false |
||||
|
speechDetected = false |
||||
|
stopMicHard() |
||||
|
ttsInterrupted = true |
||||
|
ttsQueue.clear() |
||||
|
ttsPlaying = false |
||||
|
mainHandler.post { |
||||
|
try { currentPlayer?.stop(); currentPlayer?.release() } catch (_: Exception) {} |
||||
|
currentPlayer = null |
||||
|
} |
||||
|
closeWs() |
||||
|
preBuffer.clear() |
||||
|
clearSendBuffer() |
||||
|
flushPendingAnswer() |
||||
|
inCall = false |
||||
|
pttRecording = false |
||||
|
} |
||||
|
|
||||
|
/** 打断 AI 播报 */ |
||||
|
fun interruptSpeaking() { |
||||
|
interruptTts() |
||||
|
ttsInterrupted = false |
||||
|
if (inCall) resetVad() |
||||
|
} |
||||
|
|
||||
|
/** 清空消息历史 */ |
||||
|
fun clearHistory() { |
||||
|
messages = JSONArray() |
||||
|
} |
||||
|
|
||||
|
/** 释放资源 */ |
||||
|
fun dispose() { |
||||
|
disconnect() |
||||
|
job.cancel() |
||||
|
} |
||||
|
|
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
// 麦克风 & 本地 VAD |
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
|
||||
|
private fun openMic() { |
||||
|
stopMicHard() |
||||
|
speechDetected = false; vadOnsetCount = 0; micChunkCount = 0 |
||||
|
try { |
||||
|
audioRecord = AudioRecord( |
||||
|
MediaRecorder.AudioSource.VOICE_COMMUNICATION, |
||||
|
SAMPLE_RATE, |
||||
|
AudioFormat.CHANNEL_IN_MONO, |
||||
|
AudioFormat.ENCODING_PCM_16BIT, |
||||
|
RECORD_BUFFER |
||||
|
) |
||||
|
audioRecord?.startRecording() |
||||
|
recordingJob = launch { |
||||
|
val buf = ByteArray(RECORD_CHUNK) |
||||
|
while (isActive && (inCall || pttRecording)) { |
||||
|
val read = audioRecord?.read(buf, 0, buf.size) ?: break |
||||
|
if (read > 0) { micChunkCount++; onMicAudio(buf.copyOf(read)) } |
||||
|
} |
||||
|
} |
||||
|
} catch (e: Exception) { |
||||
|
Log.e(TAG, "打开麦克风失败: ${e.message}") |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private fun stopMicHard() { |
||||
|
recordingJob?.cancel(); recordingJob = null |
||||
|
try { audioRecord?.stop(); audioRecord?.release() } catch (_: Exception) {} |
||||
|
audioRecord = null |
||||
|
} |
||||
|
|
||||
|
private fun startMicWatchdog() { |
||||
|
stopMicWatchdog() |
||||
|
lastWatchdogChunkCount = micChunkCount |
||||
|
micWatchdogJob = launch { |
||||
|
while (isActive && (inCall || pttRecording)) { |
||||
|
delay(1500) |
||||
|
if (micChunkCount == lastWatchdogChunkCount && !ttsPlaying) openMic() |
||||
|
lastWatchdogChunkCount = micChunkCount |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private fun stopMicWatchdog() { micWatchdogJob?.cancel(); micWatchdogJob = null } |
||||
|
|
||||
|
private fun onMicAudio(chunk: ByteArray) { |
||||
|
if (!inCall && !pttRecording) return |
||||
|
if (forwardingAudio && webSocket != null) { |
||||
|
if (micChunkCount % 3 == 0) emit("mic_rms", mapOf("rms" to calcRms(chunk))) |
||||
|
feedSendBuffer(chunk); return |
||||
|
} |
||||
|
if (wsConnecting) { addToPreBuffer(chunk); return } |
||||
|
if (ttsPlaying) return |
||||
|
|
||||
|
addToPreBuffer(chunk) |
||||
|
val rms = calcRms(chunk) |
||||
|
// 发送麦克风音量给 Dart 层 |
||||
|
if (micChunkCount % 3 == 0) { |
||||
|
emit("mic_rms", mapOf("rms" to rms)) |
||||
|
} |
||||
|
if (rms >= VAD_THRESHOLD) { |
||||
|
vadOnsetCount++ |
||||
|
if (vadOnsetCount >= VAD_ONSET_CHUNKS && !speechDetected) { |
||||
|
speechDetected = true |
||||
|
cancelIdleTimer() |
||||
|
launch { onSpeechDetected() } |
||||
|
} |
||||
|
} else { |
||||
|
vadOnsetCount = 0 |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private fun calcRms(pcm: ByteArray): Double { |
||||
|
if (pcm.size < 2) return 0.0 |
||||
|
val samples = pcm.size / 2 |
||||
|
var sum = 0.0 |
||||
|
for (i in 0 until pcm.size - 1 step 2) { |
||||
|
var s = (pcm[i].toInt() and 0xFF) or (pcm[i + 1].toInt() shl 8) |
||||
|
if (s >= 32768) s -= 65536 |
||||
|
sum += s.toLong() * s.toLong() |
||||
|
} |
||||
|
return sqrt(sum / samples) |
||||
|
} |
||||
|
|
||||
|
private fun addToPreBuffer(chunk: ByteArray) { |
||||
|
preBuffer.addLast(chunk.copyOf()) |
||||
|
while (preBuffer.size > PRE_BUFFER_MAX_CHUNKS) preBuffer.removeFirst() |
||||
|
} |
||||
|
|
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
// VAD 触发 → WS 连接 |
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
|
||||
|
private suspend fun onSpeechDetected() { |
||||
|
if (wsConnecting) return |
||||
|
ttsInterrupted = false |
||||
|
closeWs() |
||||
|
sessionEnded = false |
||||
|
wsConnecting = true |
||||
|
dialogRequestId = UUID.randomUUID().toString() |
||||
|
clearSendBuffer() |
||||
|
pendingAnswer = "" |
||||
|
|
||||
|
try { |
||||
|
connectWs() |
||||
|
} catch (e: Exception) { |
||||
|
wsConnecting = false; speechDetected = false |
||||
|
emitError("连接失败: ${e.message}") |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
forwardingAudio = true; wsConnecting = false |
||||
|
emit("recognition_started", emptyMap()) |
||||
|
|
||||
|
val cached = preBuffer.toList(); preBuffer.clear() |
||||
|
for (c in cached) feedSendBuffer(c) |
||||
|
} |
||||
|
|
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
// WebSocket |
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
|
||||
|
private suspend fun connectWs(): Unit = suspendCancellableCoroutine { cont -> |
||||
|
val ts = System.currentTimeMillis() |
||||
|
val sign = calcSign(config.deviceSecret, ts.toString()) |
||||
|
val sn = UUID.randomUUID().toString() |
||||
|
val url = "wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr" + |
||||
|
"?deviceNo=${config.deviceNo}&deviceId=${config.deviceId}" + |
||||
|
"&productId=${config.productId}&productKey=${config.productKey}" + |
||||
|
"&sign=$sign&sn=$sn&ts=$ts" |
||||
|
|
||||
|
val req = Request.Builder().url(url) |
||||
|
.header("Origin", "https://ivs.chinamobiledevice.com:11443") |
||||
|
.header("Host", "ivs.chinamobiledevice.com:11443") |
||||
|
.build() |
||||
|
|
||||
|
var resolved = false |
||||
|
|
||||
|
webSocket = okHttpClient.newWebSocket(req, object : WebSocketListener() { |
||||
|
override fun onOpen(ws: WebSocket, response: Response) { |
||||
|
sendStartFrame() |
||||
|
} |
||||
|
|
||||
|
override fun onMessage(ws: WebSocket, text: String) { |
||||
|
try { |
||||
|
val msg = JSONObject(text) |
||||
|
if (msg.optString("type") == "ready") { |
||||
|
if (!resolved) { resolved = true; cont.resume(Unit) {} } |
||||
|
} else { |
||||
|
launch { handleMessage(msg) } |
||||
|
} |
||||
|
} catch (_: Exception) {} |
||||
|
} |
||||
|
|
||||
|
override fun onMessage(ws: WebSocket, bytes: okio.ByteString) {} |
||||
|
|
||||
|
override fun onFailure(ws: WebSocket, t: Throwable, response: Response?) { |
||||
|
webSocket = null |
||||
|
if (!resolved) { resolved = true; cont.resumeWithException(t) } |
||||
|
else launch { handleWsDone() } |
||||
|
} |
||||
|
|
||||
|
override fun onClosing(ws: WebSocket, code: Int, reason: String) { ws.close(1000, null) } |
||||
|
|
||||
|
override fun onClosed(ws: WebSocket, code: Int, reason: String) { |
||||
|
webSocket = null |
||||
|
if (!resolved) { resolved = true; cont.resumeWithException(IOException("WS closed")) } |
||||
|
else launch { handleWsDone() } |
||||
|
} |
||||
|
}) |
||||
|
|
||||
|
cont.invokeOnCancellation { closeWs() } |
||||
|
|
||||
|
// 15s 超时 |
||||
|
launch { |
||||
|
delay(15_000) |
||||
|
if (!resolved) { |
||||
|
resolved = true; closeWs() |
||||
|
cont.resumeWithException(IOException("等待 ready 超时")) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private fun handleWsDone() { |
||||
|
webSocket = null; forwardingAudio = false; wsConnecting = false |
||||
|
when { |
||||
|
sessionEnded && inCall -> if (!ttsPlaying) { resetVad(); startIdleTimer() } |
||||
|
sessionEnded -> { /* PTT 结束，等 TTS 播完 */ } |
||||
|
inCall -> resetVad() |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private fun resetVad() { |
||||
|
speechDetected = false; vadOnsetCount = 0; preBuffer.clear(); openMic() |
||||
|
} |
||||
|
|
||||
|
private fun closeWs() { |
||||
|
try { webSocket?.cancel() } catch (_: Exception) {} |
||||
|
webSocket = null |
||||
|
} |
||||
|
|
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
// 发送帧 |
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
|
||||
|
private fun sendStartFrame() { |
||||
|
currentSessionId = UUID.randomUUID().toString() |
||||
|
val payload = JSONObject().apply { |
||||
|
put("type", "start") |
||||
|
put("data", JSONObject().apply { |
||||
|
put("cuid", config.deviceId) |
||||
|
put("format", "pcm") |
||||
|
put("sample", 16000) |
||||
|
put("support_dcs", 2) |
||||
|
put("chunk_size", 10240) |
||||
|
put("support_tts", true) |
||||
|
put("support_text2dcs", true) |
||||
|
put("user_agent", "voitrans/1.0") |
||||
|
put("dialog_request_id", dialogRequestId) |
||||
|
put("access_rc", true) |
||||
|
put("support_part_tts", true) |
||||
|
put("need_stoplisten", true) |
||||
|
put("need_dialogue_finish", true) |
||||
|
put("result_trans2directive", false) |
||||
|
put("rc_version", "5") |
||||
|
put("messages", messages) |
||||
|
put("client_context", JSONArray()) |
||||
|
}) |
||||
|
} |
||||
|
webSocket?.send(payload.toString()) |
||||
|
} |
||||
|
|
||||
|
private val sendBufferLock = Any() |
||||
|
|
||||
|
private fun clearSendBuffer() { |
||||
|
synchronized(sendBufferLock) { |
||||
|
sendBuffer.clear(); sendBufferOffset = 0 |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private fun feedSendBuffer(chunk: ByteArray) { |
||||
|
synchronized(sendBufferLock) { |
||||
|
sendBuffer.addAll(chunk.toList()) |
||||
|
while (sendBuffer.size - sendBufferOffset >= FRAME_SIZE) { |
||||
|
val frame = ByteArray(FRAME_SIZE) { i -> sendBuffer[sendBufferOffset + i] } |
||||
|
sendBufferOffset += FRAME_SIZE |
||||
|
sendAudioChunk(frame) |
||||
|
} |
||||
|
// 收缩缓冲 |
||||
|
if (sendBufferOffset >= FRAME_SIZE * 8) { |
||||
|
val toRemove = sendBufferOffset |
||||
|
repeat(toRemove) { if (sendBuffer.isNotEmpty()) sendBuffer.removeFirst() } |
||||
|
sendBufferOffset = 0 |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private fun sendAudioChunk(data: ByteArray) { |
||||
|
try { webSocket?.send(data.toByteString()) } catch (_: Exception) {} |
||||
|
} |
||||
|
|
||||
|
private fun sendFinish() { |
||||
|
try { webSocket?.send("""{"type":"finish"}""") } catch (_: Exception) {} |
||||
|
} |
||||
|
|
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
// 消息处理 |
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
|
||||
|
private suspend fun handleMessage(msg: JSONObject) { |
||||
|
when (msg.optString("type")) { |
||||
|
"stoplisten" -> { |
||||
|
forwardingAudio = false; speechDetected = false; vadOnsetCount = 0 |
||||
|
synchronized(sendBufferLock) { |
||||
|
val remain = sendBuffer.size - sendBufferOffset |
||||
|
if (remain > 0) { |
||||
|
sendAudioChunk(ByteArray(remain) { i -> sendBuffer[sendBufferOffset + i] }) |
||||
|
} |
||||
|
sendBuffer.clear(); sendBufferOffset = 0 |
||||
|
} |
||||
|
sendFinish() |
||||
|
emit("recognition_stopped", emptyMap()) |
||||
|
} |
||||
|
"mid_result" -> { |
||||
|
val text = msg.optString("result", "") |
||||
|
if (text.isNotEmpty()) emit("recognizing", mapOf("text" to text)) |
||||
|
} |
||||
|
"fin_result" -> { |
||||
|
val result = msg.optString("result", "") |
||||
|
if (result.isNotEmpty()) { |
||||
|
emit("recognition_result", mapOf("text" to result)) |
||||
|
messages.put(JSONObject(mapOf("role" to "user", "content" to result))) |
||||
|
withContext(Dispatchers.IO) { |
||||
|
try { |
||||
|
chatStorageHelper.saveMessage(config.agentId, currentSessionId, result, "user") |
||||
|
} catch (_: Exception) {} |
||||
|
} |
||||
|
} |
||||
|
if (msg.optInt("end", 0) == 1) markSessionEnded() |
||||
|
} |
||||
|
"inside_rc" -> handleInsideRc(msg) |
||||
|
"dcs_decide" -> if (msg.optInt("end", 0) == 1) markSessionEnded() |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private suspend fun handleInsideRc(msg: JSONObject) { |
||||
|
val data = msg.optJSONObject("data") ?: return |
||||
|
if (data.optInt("code", -1) != 0) return |
||||
|
|
||||
|
val rawAnswer = data.optString("assistant_answer", "") |
||||
|
val cleanAnswer = try { |
||||
|
JSONObject(rawAnswer).optString("content", rawAnswer) |
||||
|
} catch (_: Exception) { rawAnswer } |
||||
|
|
||||
|
var ttsUrl: String? = null |
||||
|
val items = data.optJSONArray("data") |
||||
|
if (items != null) { |
||||
|
for (i in 0 until items.length()) { |
||||
|
val item = items.optJSONObject(i) ?: continue |
||||
|
val h = item.optJSONObject("header") ?: continue |
||||
|
if (h.optString("namespace") == "ai.fxzsos.device_interface.voice_output" && |
||||
|
h.optString("name") == "Speak" |
||||
|
) { |
||||
|
ttsUrl = item.optJSONObject("payload")?.optString("url") |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// 每次取更完整的回复文本，立即发送给 UI |
||||
|
if (cleanAnswer.isNotEmpty() && cleanAnswer.length > pendingAnswer.length) { |
||||
|
pendingAnswer = cleanAnswer |
||||
|
emit("assistant_response", mapOf("response" to cleanAnswer, "sessionid" to currentSessionId)) |
||||
|
} |
||||
|
ttsUrl?.takeIf { it.isNotEmpty() }?.let { enqueueTts(it) } |
||||
|
if (msg.optInt("end", 0) == 1) markSessionEnded() |
||||
|
} |
||||
|
|
||||
|
/** 将 pendingAnswer 保存到历史和存储，并清空 */ |
||||
|
private fun flushPendingAnswer() { |
||||
|
if (pendingAnswer.isEmpty()) return |
||||
|
val answer = pendingAnswer |
||||
|
val sid = currentSessionId |
||||
|
messages.put(JSONObject(mapOf("role" to "assistant", "content" to answer))) |
||||
|
launch(Dispatchers.IO) { |
||||
|
try { |
||||
|
chatStorageHelper.saveMessage(config.agentId, sid, answer, "assistant") |
||||
|
} catch (_: Exception) {} |
||||
|
} |
||||
|
pendingAnswer = "" |
||||
|
} |
||||
|
|
||||
|
private fun markSessionEnded() { |
||||
|
if (sessionEnded) return |
||||
|
sessionEnded = true; forwardingAudio = false; speechDetected = false; vadOnsetCount = 0 |
||||
|
clearSendBuffer() |
||||
|
flushPendingAnswer() |
||||
|
// 静音模式下文本接收完毕，发送完成事件并关闭连接 |
||||
|
if (ttsMuted) { |
||||
|
emit("tts_completed", emptyMap()) |
||||
|
closeWs() |
||||
|
if (inCall) { resetVad(); startIdleTimer() } |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
// TTS 播放队列 |
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
|
||||
|
private fun enqueueTts(url: String) { |
||||
|
if (ttsMuted) return // 静音模式跳过 TTS |
||||
|
ttsQueue.addLast(url) |
||||
|
if (!ttsPlaying) { |
||||
|
stopMicHard() |
||||
|
mainHandler.post { playNextTts() } |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private fun playNextTts() { |
||||
|
if (ttsQueue.isEmpty() || ttsInterrupted) { |
||||
|
ttsPlaying = false |
||||
|
if (!ttsInterrupted) { |
||||
|
emit("tts_completed", emptyMap()) |
||||
|
if (inCall) { resetVad(); startIdleTimer() } |
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
ttsPlaying = true |
||||
|
emit("tts_started", emptyMap()) |
||||
|
val url = ttsQueue.removeFirst() |
||||
|
try { |
||||
|
currentPlayer?.stop(); currentPlayer?.release() |
||||
|
currentPlayer = MediaPlayer().apply { |
||||
|
setDataSource(url) |
||||
|
setOnCompletionListener { |
||||
|
if (!ttsInterrupted) mainHandler.post { playNextTts() } |
||||
|
} |
||||
|
setOnErrorListener { _, _, _ -> |
||||
|
if (!ttsInterrupted) mainHandler.post { playNextTts() } |
||||
|
true |
||||
|
} |
||||
|
prepare() |
||||
|
start() |
||||
|
} |
||||
|
} catch (e: Exception) { |
||||
|
Log.e(TAG, "TTS 播放失败: ${e.message}") |
||||
|
if (!ttsInterrupted) mainHandler.post { playNextTts() } |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private fun interruptTts() { |
||||
|
ttsInterrupted = true; ttsQueue.clear(); ttsPlaying = false |
||||
|
mainHandler.post { |
||||
|
try { currentPlayer?.stop(); currentPlayer?.release() } catch (_: Exception) {} |
||||
|
currentPlayer = null |
||||
|
} |
||||
|
emit("tts_canceled", emptyMap()) |
||||
|
} |
||||
|
|
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
// 超时 & 工具 |
||||
|
// ═════════════════════════════════════════════════════════════════════════ |
||||
|
|
||||
|
private fun startIdleTimer() { |
||||
|
cancelIdleTimer() |
||||
|
idleTimerJob = launch { |
||||
|
delay(IDLE_TIMEOUT_MS) |
||||
|
if (inCall && !speechDetected) disconnect() |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private fun cancelIdleTimer() { idleTimerJob?.cancel(); idleTimerJob = null } |
||||
|
|
||||
|
private fun calcSign(secret: String, ts: String): String { |
||||
|
val digest = MessageDigest.getInstance("MD5").digest("$secret$ts".toByteArray()) |
||||
|
return digest.joinToString("") { "%02x".format(it) } |
||||
|
} |
||||
|
|
||||
|
private fun emit(eventName: String, data: Map<String, Any>) { |
||||
|
launch(Dispatchers.Main) { onEvent(eventName, data) } |
||||
|
} |
||||
|
|
||||
|
private fun emitError(msg: String) = emit("error", mapOf("message" to msg)) |
||||
|
|
||||
|
private fun buildTrustAllClient(): OkHttpClient { |
||||
|
return try { |
||||
|
val trustAll = arrayOf<TrustManager>(object : X509TrustManager { |
||||
|
override fun checkClientTrusted(c: Array<X509Certificate>, t: String) {} |
||||
|
override fun checkServerTrusted(c: Array<X509Certificate>, t: String) {} |
||||
|
override fun getAcceptedIssuers(): Array<X509Certificate> = emptyArray() |
||||
|
}) |
||||
|
val sc = SSLContext.getInstance("TLS") |
||||
|
sc.init(null, trustAll, SecureRandom()) |
||||
|
OkHttpClient.Builder() |
||||
|
.sslSocketFactory(sc.socketFactory, trustAll[0] as X509TrustManager) |
||||
|
.hostnameVerifier { _, _ -> true } |
||||
|
.connectTimeout(15, TimeUnit.SECONDS) |
||||
|
.readTimeout(90, TimeUnit.SECONDS) |
||||
|
.writeTimeout(90, TimeUnit.SECONDS) |
||||
|
.build() |
||||
|
} catch (e: Exception) { |
||||
|
Log.e(TAG, "buildTrustAllClient 失败，使用默认客户端", e) |
||||
|
OkHttpClient() |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,495 @@ |
|||||
|
import Foundation |
||||
|
import AVFoundation |
||||
|
import CryptoKit |
||||
|
import os.log |
||||
|
import chat_storage |
||||
|
|
||||
|
// MARK: - StsConfig |
||||
|
|
||||
|
struct StsConfig { |
||||
|
let productId: String |
||||
|
let productKey: String |
||||
|
var deviceNo: String |
||||
|
var deviceId: String |
||||
|
var deviceSecret: String |
||||
|
let agentId: String |
||||
|
} |
||||
|
|
||||
|
// MARK: - StsAgent |
||||
|
|
||||
|
/// StsAgent — 中国移动 A80 WS 一体化语音对话实现（iOS） |
||||
|
/// |
||||
|
/// 通过一条 WSS 连接完成 ASR + AI + TTS，事件格式与标准 stt_llm_tts 完全一致， |
||||
|
/// 通过 onEvent 回调发出，供 AgentServiceImpl 转发到 Flutter EventChannel。 |
||||
|
class StsAgent: NSObject { |
||||
|
|
||||
|
private let TAG = "StsAgent" |
||||
|
private let logger = OSLog(subsystem: "com.yunqiinnovation.agent_service", category: "StsAgent") |
||||
|
|
||||
|
private var config: StsConfig |
||||
|
private let onEvent: (String, [String: Any]) -> Void |
||||
|
private let onCredentialsFetched: ((String, String, String) -> Void)? |
||||
|
|
||||
|
// MARK: WebSocket |
||||
|
private var webSocketTask: URLSessionWebSocketTask? |
||||
|
private var urlSession: URLSession! |
||||
|
private var readyReceived = false |
||||
|
private var sessionEnded = true |
||||
|
private var dialogRequestId = "" |
||||
|
|
||||
|
// MARK: 录音 |
||||
|
private let SAMPLE_RATE: Double = 16000 |
||||
|
private let FRAME_SIZE = 5120 // 160ms @ 16kHz 16bit mono |
||||
|
private var audioEngine: AVAudioEngine? |
||||
|
private var inputNode: AVAudioInputNode? |
||||
|
private var recordingActive = false |
||||
|
private var sendBuffer = Data() |
||||
|
private var micChunkCount = 0 |
||||
|
|
||||
|
// MARK: VAD |
||||
|
private let VAD_THRESHOLD: Float = 800 |
||||
|
private let VAD_ONSET_CHUNKS = 3 |
||||
|
private var vadOnsetCount = 0 |
||||
|
private var vadActive = false |
||||
|
private var preBuffer: [[UInt8]] = [] |
||||
|
private let PRE_BUFFER_MAX = 75 |
||||
|
|
||||
|
// MARK: TTS |
||||
|
private var ttsQueue: [String] = [] |
||||
|
private var ttsPlayer: AVPlayer? |
||||
|
private var ttsPlaying = false |
||||
|
private var ttsPlayerObserver: Any? |
||||
|
|
||||
|
// MARK: 历史 |
||||
|
private var messages: [[String: String]] = [] |
||||
|
private var currentSessionId = "" |
||||
|
private let chatStorage = ChatStorageHelper.shared |
||||
|
|
||||
|
// MARK: 模式 |
||||
|
private var isPttMode = false |
||||
|
|
||||
|
// MARK: 凭证请求 |
||||
|
private var credentialsFetched = false |
||||
|
|
||||
|
init(config: StsConfig, |
||||
|
onEvent: @escaping (String, [String: Any]) -> Void, |
||||
|
onCredentialsFetched: ((String, String, String) -> Void)? = nil) { |
||||
|
self.config = config |
||||
|
self.onEvent = onEvent |
||||
|
self.onCredentialsFetched = onCredentialsFetched |
||||
|
super.init() |
||||
|
let sessionConfig = URLSessionConfiguration.default |
||||
|
urlSession = URLSession(configuration: sessionConfig, delegate: nil, delegateQueue: OperationQueue()) |
||||
|
} |
||||
|
|
||||
|
// MARK: - 公共接口 |
||||
|
|
||||
|
func startCall() { |
||||
|
isPttMode = false |
||||
|
os_log("StsAgent startCall", log: logger, type: .info) |
||||
|
fetchCredentialsIfNeeded { [weak self] in |
||||
|
self?.connect() |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func startPtt() { |
||||
|
isPttMode = true |
||||
|
os_log("StsAgent startPtt", log: logger, type: .info) |
||||
|
fetchCredentialsIfNeeded { [weak self] in |
||||
|
self?.connect() |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func stopPtt() { |
||||
|
os_log("StsAgent stopPtt", log: logger, type: .info) |
||||
|
guard isPttMode else { return } |
||||
|
sendFinishFrame() |
||||
|
} |
||||
|
|
||||
|
/// 统一停止：根据当前模式自动选择 stopPtt 或 dispose |
||||
|
func stop() { |
||||
|
if isPttMode { |
||||
|
stopPtt() |
||||
|
} else { |
||||
|
dispose() |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func interruptSpeaking() { |
||||
|
os_log("StsAgent interruptSpeaking", log: logger, type: .info) |
||||
|
stopTts() |
||||
|
emit("tts_canceled") |
||||
|
} |
||||
|
|
||||
|
func clearHistory() { |
||||
|
messages.removeAll() |
||||
|
DispatchQueue.global().async { [weak self] in |
||||
|
guard let self = self else { return } |
||||
|
_ = self.chatStorage.deleteMessages(agentId: self.config.agentId, messageIds: nil as [Int]?) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func dispose() { |
||||
|
os_log("StsAgent dispose", log: logger, type: .info) |
||||
|
stopRecording() |
||||
|
webSocketTask?.cancel(with: .normalClosure, reason: nil) |
||||
|
webSocketTask = nil |
||||
|
stopTts() |
||||
|
} |
||||
|
|
||||
|
// MARK: - 凭证获取 |
||||
|
|
||||
|
private func fetchCredentialsIfNeeded(completion: @escaping () -> Void) { |
||||
|
if credentialsFetched && !config.deviceId.isEmpty && !config.deviceSecret.isEmpty { |
||||
|
completion() |
||||
|
return |
||||
|
} |
||||
|
let ts = Int64(Date().timeIntervalSince1970 * 1000) |
||||
|
let sign = calcSign(secret: config.productKey, ts: ts) |
||||
|
let urlStr = "https://ivs.chinamobiledevice.com:11443/v2/customer/device/secret/info" |
||||
|
guard var components = URLComponents(string: urlStr) else { completion(); return } |
||||
|
components.queryItems = [ |
||||
|
URLQueryItem(name: "productId", value: config.productId), |
||||
|
URLQueryItem(name: "deviceNo", value: config.deviceNo), |
||||
|
URLQueryItem(name: "ts", value: "\(ts)"), |
||||
|
URLQueryItem(name: "sign", value: sign), |
||||
|
] |
||||
|
guard let url = components.url else { completion(); return } |
||||
|
var req = URLRequest(url: url) |
||||
|
req.timeoutInterval = 10 |
||||
|
URLSession.shared.dataTask(with: req) { [weak self] data, _, error in |
||||
|
guard let self = self else { return } |
||||
|
if let error = error { |
||||
|
os_log("凭证请求失败: %{public}@", log: self.logger, type: .error, error.localizedDescription) |
||||
|
completion() |
||||
|
return |
||||
|
} |
||||
|
guard let data = data, |
||||
|
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], |
||||
|
let body = json["body"] as? [String: Any] else { |
||||
|
completion() |
||||
|
return |
||||
|
} |
||||
|
if let deviceId = body["deviceId"] as? String { self.config.deviceId = deviceId } |
||||
|
if let deviceSecret = body["deviceSecret"] as? String { self.config.deviceSecret = deviceSecret } |
||||
|
if let deviceNo = body["deviceNo"] as? String { self.config.deviceNo = deviceNo } |
||||
|
self.credentialsFetched = true |
||||
|
self.onCredentialsFetched?(self.config.deviceId, self.config.deviceSecret, self.config.deviceNo) |
||||
|
completion() |
||||
|
}.resume() |
||||
|
} |
||||
|
|
||||
|
// MARK: - WebSocket 连接 |
||||
|
|
||||
|
private func connect() { |
||||
|
let ts = Int64(Date().timeIntervalSince1970 * 1000) |
||||
|
let sign = calcSign(secret: config.deviceSecret, ts: ts) |
||||
|
let urlStr = "wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr" |
||||
|
guard var components = URLComponents(string: urlStr) else { return } |
||||
|
components.queryItems = [ |
||||
|
URLQueryItem(name: "productId", value: config.productId), |
||||
|
URLQueryItem(name: "deviceId", value: config.deviceId), |
||||
|
URLQueryItem(name: "ts", value: "\(ts)"), |
||||
|
URLQueryItem(name: "sign", value: sign), |
||||
|
] |
||||
|
guard let url = components.url else { return } |
||||
|
webSocketTask = urlSession.webSocketTask(with: url) |
||||
|
webSocketTask?.resume() |
||||
|
receiveMessages() |
||||
|
|
||||
|
// 发送 start 帧 |
||||
|
sendStartFrame() |
||||
|
} |
||||
|
|
||||
|
private func sendStartFrame() { |
||||
|
dialogRequestId = UUID().uuidString.replacingOccurrences(of: "-", with: "").lowercased() |
||||
|
currentSessionId = dialogRequestId |
||||
|
let payload: [String: Any] = [ |
||||
|
"msgType": "start", |
||||
|
"dialogRequestId": dialogRequestId, |
||||
|
"params": [ |
||||
|
"asrParams": [ |
||||
|
"format": "pcm", |
||||
|
"sampleRate": 16000, |
||||
|
"lang": "zh-CN", |
||||
|
"vadEnable": false, |
||||
|
], |
||||
|
"contextParams": [ |
||||
|
"history": messages.map { ["role": $0["role"] ?? "", "content": $0["content"] ?? ""] } |
||||
|
] |
||||
|
] |
||||
|
] |
||||
|
sendJson(payload) |
||||
|
} |
||||
|
|
||||
|
private func receiveMessages() { |
||||
|
webSocketTask?.receive { [weak self] result in |
||||
|
guard let self = self else { return } |
||||
|
switch result { |
||||
|
case .success(let msg): |
||||
|
switch msg { |
||||
|
case .string(let text): |
||||
|
self.handleMessage(text) |
||||
|
default: |
||||
|
break |
||||
|
} |
||||
|
self.receiveMessages() |
||||
|
case .failure(let err): |
||||
|
os_log("WS error: %{public}@", log: self.logger, type: .error, err.localizedDescription) |
||||
|
self.emit("error", data: ["message": err.localizedDescription]) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// MARK: - 消息处理 |
||||
|
|
||||
|
private func handleMessage(_ text: String) { |
||||
|
guard let data = text.data(using: .utf8), |
||||
|
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return } |
||||
|
let msgType = json["msgType"] as? String ?? "" |
||||
|
os_log("WS recv: %{public}@", log: logger, type: .debug, msgType) |
||||
|
|
||||
|
switch msgType { |
||||
|
case "ready": |
||||
|
readyReceived = true |
||||
|
sessionEnded = false |
||||
|
emit("recognition_started") |
||||
|
startRecording() |
||||
|
if isPttMode { |
||||
|
// PTT 模式等待 stopPtt() 调用发送 finish |
||||
|
} |
||||
|
|
||||
|
case "mid_result": |
||||
|
let result = (json["data"] as? [String: Any])?["result"] as? String ?? "" |
||||
|
if !result.isEmpty { emit("recognizing", data: ["text": result]) } |
||||
|
|
||||
|
case "fin_result": |
||||
|
let result = (json["data"] as? [String: Any])?["result"] as? String ?? "" |
||||
|
emit("recognition_result", data: ["text": result]) |
||||
|
emit("recognition_stopped") |
||||
|
stopRecording() |
||||
|
if !result.isEmpty { |
||||
|
messages.append(["role": "user", "content": result]) |
||||
|
saveToChatStorage(text: result, role: "user") |
||||
|
} |
||||
|
|
||||
|
case "stoplisten": |
||||
|
stopRecording() |
||||
|
emit("recognition_stopped") |
||||
|
|
||||
|
case "inside_rc": |
||||
|
handleInsideRc(json["data"] as? [String: Any]) |
||||
|
|
||||
|
case "dcs_decide": |
||||
|
// 语义决策，忽略 |
||||
|
break |
||||
|
|
||||
|
default: |
||||
|
os_log("未处理的消息类型: %{public}@", log: logger, type: .info, msgType) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private func handleInsideRc(_ data: [String: Any]?) { |
||||
|
guard let data = data else { return } |
||||
|
guard let rcType = data["rcType"] as? String else { return } |
||||
|
if rcType == "assistant_answer" { |
||||
|
let answer = (data["answer"] as? [String: Any])?["text"] as? String ?? "" |
||||
|
let cleanAnswer = answer.trimmingCharacters(in: .whitespacesAndNewlines) |
||||
|
let sid = data["sessionId"] as? String ?? currentSessionId |
||||
|
if !cleanAnswer.isEmpty { |
||||
|
messages.append(["role": "assistant", "content": cleanAnswer]) |
||||
|
saveToChatStorage(text: cleanAnswer, role: "assistant") |
||||
|
emit("assistant_response", data: ["response": cleanAnswer, "sessionid": sid]) |
||||
|
} |
||||
|
// TTS URLs |
||||
|
if let ttsData = data["tts"] as? [[String: Any]] { |
||||
|
for item in ttsData { |
||||
|
if let url = item["url"] as? String, !url.isEmpty { |
||||
|
ttsQueue.append(url) |
||||
|
} |
||||
|
} |
||||
|
if !ttsPlaying { playNextTts() } |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// MARK: - TTS 播放 |
||||
|
|
||||
|
private func playNextTts() { |
||||
|
guard !ttsQueue.isEmpty else { |
||||
|
ttsPlaying = false |
||||
|
emit("tts_completed") |
||||
|
return |
||||
|
} |
||||
|
let urlStr = ttsQueue.removeFirst() |
||||
|
guard let url = URL(string: urlStr) else { playNextTts(); return } |
||||
|
|
||||
|
ttsPlaying = true |
||||
|
if ttsQueue.isEmpty && !ttsPlaying { emit("tts_started") } |
||||
|
emit("tts_started") |
||||
|
|
||||
|
let item = AVPlayerItem(url: url) |
||||
|
ttsPlayer = AVPlayer(playerItem: item) |
||||
|
if let observer = ttsPlayerObserver { |
||||
|
NotificationCenter.default.removeObserver(observer) |
||||
|
} |
||||
|
ttsPlayerObserver = NotificationCenter.default.addObserver( |
||||
|
forName: .AVPlayerItemDidPlayToEndTime, object: item, queue: .main |
||||
|
) { [weak self] _ in |
||||
|
self?.playNextTts() |
||||
|
} |
||||
|
ttsPlayer?.play() |
||||
|
} |
||||
|
|
||||
|
private func stopTts() { |
||||
|
ttsPlayer?.pause() |
||||
|
ttsPlayer = nil |
||||
|
ttsQueue.removeAll() |
||||
|
ttsPlaying = false |
||||
|
if let observer = ttsPlayerObserver { |
||||
|
NotificationCenter.default.removeObserver(observer) |
||||
|
ttsPlayerObserver = nil |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// MARK: - 录音 |
||||
|
|
||||
|
private func startRecording() { |
||||
|
guard !recordingActive else { return } |
||||
|
let engine = AVAudioEngine() |
||||
|
audioEngine = engine |
||||
|
inputNode = engine.inputNode |
||||
|
let fmt = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: SAMPLE_RATE, channels: 1, interleaved: true)! |
||||
|
let hwFmt = inputNode!.outputFormat(forBus: 0) |
||||
|
let converterFmt = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: SAMPLE_RATE, channels: 1, interleaved: true)! |
||||
|
|
||||
|
inputNode!.installTap(onBus: 0, bufferSize: 4096, format: hwFmt) { [weak self] buffer, _ in |
||||
|
guard let self = self, self.recordingActive else { return } |
||||
|
// 转换为 16kHz PCM16 |
||||
|
guard let cvt = AVAudioConverter(from: hwFmt, to: converterFmt) else { return } |
||||
|
let frameCount = AVAudioFrameCount(Double(buffer.frameLength) * self.SAMPLE_RATE / hwFmt.sampleRate) |
||||
|
guard let outBuf = AVAudioPCMBuffer(pcmFormat: converterFmt, frameCapacity: frameCount) else { return } |
||||
|
var error: NSError? |
||||
|
cvt.convert(to: outBuf, error: &error) { _, outStatus in |
||||
|
outStatus.pointee = .haveData |
||||
|
return buffer |
||||
|
} |
||||
|
if error != nil { return } |
||||
|
let byteCount = Int(outBuf.frameLength) * 2 |
||||
|
let ptr = outBuf.int16ChannelData![0] |
||||
|
let data = Data(bytes: ptr, count: byteCount) |
||||
|
self.onMicAudio(data) |
||||
|
} |
||||
|
|
||||
|
do { |
||||
|
try AVAudioSession.sharedInstance().setCategory(.playAndRecord, mode: .voiceChat, options: [.defaultToSpeaker]) |
||||
|
try AVAudioSession.sharedInstance().setActive(true) |
||||
|
try engine.start() |
||||
|
recordingActive = true |
||||
|
} catch { |
||||
|
os_log("录音启动失败: %{public}@", log: logger, type: .error, error.localizedDescription) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private func stopRecording() { |
||||
|
guard recordingActive else { return } |
||||
|
recordingActive = false |
||||
|
inputNode?.removeTap(onBus: 0) |
||||
|
audioEngine?.stop() |
||||
|
audioEngine = nil |
||||
|
inputNode = nil |
||||
|
sendBuffer.removeAll() |
||||
|
micChunkCount = 0 |
||||
|
vadActive = false |
||||
|
vadOnsetCount = 0 |
||||
|
preBuffer.removeAll() |
||||
|
} |
||||
|
|
||||
|
private func onMicAudio(_ pcm: Data) { |
||||
|
micChunkCount += 1 |
||||
|
let rms = calcRms(pcm) |
||||
|
let chunk = [UInt8](pcm) |
||||
|
|
||||
|
if !vadActive { |
||||
|
if preBuffer.count >= PRE_BUFFER_MAX { preBuffer.removeFirst() } |
||||
|
preBuffer.append(chunk) |
||||
|
} |
||||
|
|
||||
|
if !vadActive && rms > VAD_THRESHOLD { |
||||
|
vadOnsetCount += 1 |
||||
|
if vadOnsetCount >= VAD_ONSET_CHUNKS { |
||||
|
vadActive = true |
||||
|
// 先发预缓存 |
||||
|
for c in preBuffer { feedSendBuffer(Data(c)) } |
||||
|
preBuffer.removeAll() |
||||
|
} |
||||
|
} else if vadActive { |
||||
|
feedSendBuffer(pcm) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private func calcRms(_ pcm: Data) -> Float { |
||||
|
let count = pcm.count / 2 |
||||
|
guard count > 0 else { return 0 } |
||||
|
var sum: Float = 0 |
||||
|
pcm.withUnsafeBytes { ptr in |
||||
|
let samples = ptr.bindMemory(to: Int16.self) |
||||
|
for i in 0..<count { |
||||
|
let s = Float(samples[i]) |
||||
|
sum += s * s |
||||
|
} |
||||
|
} |
||||
|
return sqrtf(sum / Float(count)) |
||||
|
} |
||||
|
|
||||
|
private func feedSendBuffer(_ chunk: Data) { |
||||
|
sendBuffer.append(chunk) |
||||
|
while sendBuffer.count >= FRAME_SIZE { |
||||
|
let frame = sendBuffer.prefix(FRAME_SIZE) |
||||
|
sendBuffer.removeFirst(FRAME_SIZE) |
||||
|
webSocketTask?.send(.data(Data(frame))) { _ in } |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private func sendFinishFrame() { |
||||
|
let payload: [String: Any] = [ |
||||
|
"msgType": "finish", |
||||
|
"dialogRequestId": dialogRequestId, |
||||
|
] |
||||
|
sendJson(payload) |
||||
|
} |
||||
|
|
||||
|
// MARK: - 工具 |
||||
|
|
||||
|
private func sendJson(_ dict: [String: Any]) { |
||||
|
guard let data = try? JSONSerialization.data(withJSONObject: dict), |
||||
|
let str = String(data: data, encoding: .utf8) else { return } |
||||
|
webSocketTask?.send(.string(str)) { _ in } |
||||
|
} |
||||
|
|
||||
|
private func calcSign(secret: String, ts: Int64) -> String { |
||||
|
let raw = "\(secret)\(ts)" |
||||
|
let data = Data(raw.utf8) |
||||
|
let digest = Insecure.MD5.hash(data: data) |
||||
|
return digest.map { String(format: "%02x", $0) }.joined() |
||||
|
} |
||||
|
|
||||
|
private func emit(_ name: String, data: [String: Any] = [:]) { |
||||
|
DispatchQueue.main.async { [weak self] in |
||||
|
self?.onEvent(name, data) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private func saveToChatStorage(text: String, role: String) { |
||||
|
DispatchQueue.global().async { [weak self] in |
||||
|
guard let self = self else { return } |
||||
|
_ = self.chatStorage.saveMessage( |
||||
|
agentId: self.config.agentId, |
||||
|
sessionId: self.currentSessionId, |
||||
|
message: text, |
||||
|
sender: role, |
||||
|
metadata: nil |
||||
|
) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
Loading…
Reference in new issue