13 changed files with 1885 additions and 959 deletions
File diff suppressed because it is too large
@ -0,0 +1,783 @@ |
|||
package com.yunqiinnovation.agent_service |
|||
|
|||
import android.content.Context |
|||
import android.media.AudioFormat |
|||
import android.media.AudioRecord |
|||
import android.media.MediaPlayer |
|||
import android.media.MediaRecorder |
|||
import android.os.Handler |
|||
import android.os.Looper |
|||
import android.util.Log |
|||
import com.yunqiinnovation.chat_storage.ChatStorageHelper |
|||
import kotlinx.coroutines.* |
|||
import okhttp3.* |
|||
import okhttp3.MediaType.Companion.toMediaType |
|||
import okhttp3.RequestBody.Companion.toRequestBody |
|||
import okio.ByteString.Companion.toByteString |
|||
import org.json.JSONArray |
|||
import org.json.JSONObject |
|||
import java.io.IOException |
|||
import java.security.MessageDigest |
|||
import java.security.SecureRandom |
|||
import java.security.cert.X509Certificate |
|||
import java.util.UUID |
|||
import java.util.concurrent.TimeUnit |
|||
import javax.net.ssl.SSLContext |
|||
import javax.net.ssl.TrustManager |
|||
import javax.net.ssl.X509TrustManager |
|||
import kotlin.coroutines.resumeWithException |
|||
import kotlin.math.sqrt |
|||
|
|||
/** |
|||
* STS 配置参数（移动精灵 A80 平台） |
|||
*/ |
|||
data class StsConfig( |
|||
val productId: String, |
|||
val productKey: String, |
|||
var deviceNo: String, |
|||
var deviceId: String, |
|||
var deviceSecret: String, |
|||
val agentId: String, |
|||
) |
|||
|
|||
/** |
|||
* StsAgent — 中国移动 A80 WS 一体化语音对话实现 |
|||
* |
|||
* 通过一条 WSS 连接完成 ASR + AI + TTS，事件以与标准 stt_llm_tts 相同的格式 |
|||
* 通过 [onEvent] 回调发出，供 AgentService 转发到 Flutter EventChannel。 |
|||
*/ |
|||
class StsAgent( |
|||
private val context: Context, |
|||
private var config: StsConfig, |
|||
private val onEvent: (eventName: String, data: Map<String, Any>) -> Unit, |
|||
private val onCredentialsFetched: ((deviceId: String, deviceSecret: String, deviceNo: String) -> Unit)? = null, |
|||
) : CoroutineScope { |
|||
|
|||
private val TAG = "StsAgent" |
|||
private val job = SupervisorJob() |
|||
override val coroutineContext = Dispatchers.IO + job |
|||
|
|||
init { |
|||
fetchCredentials() |
|||
} |
|||
|
|||
// ── WebSocket ───────────────────────────────────────────────────────────── |
|||
private val okHttpClient: OkHttpClient = buildTrustAllClient() |
|||
private var webSocket: WebSocket? = null |
|||
private var readyReceived = false |
|||
private var sessionEnded = true |
|||
private var dialogRequestId = "" |
|||
|
|||
// ── 录音 ────────────────────────────────────────────────────────────────── |
|||
private val SAMPLE_RATE = 16000 |
|||
private val FRAME_SIZE = 5120 // 160ms @ 16kHz 16bit mono |
|||
private val RECORD_CHUNK = FRAME_SIZE / 2 // 读取块大小 |
|||
private val RECORD_BUFFER = maxOf( |
|||
AudioRecord.getMinBufferSize( |
|||
SAMPLE_RATE, |
|||
AudioFormat.CHANNEL_IN_MONO, |
|||
AudioFormat.ENCODING_PCM_16BIT |
|||
), FRAME_SIZE * 2 |
|||
) |
|||
private var audioRecord: AudioRecord? = null |
|||
private var recordingJob: Job? = null |
|||
private var micChunkCount = 0 |
|||
|
|||
// ── 发送缓冲 ────────────────────────────────────────────────────────────── |
|||
private val sendBuffer = ArrayDeque<Byte>() |
|||
private var sendBufferOffset = 0 |
|||
|
|||
// ── 预缓冲（VAD 触发前 ~1.5s）──────────────────────────────────────────── |
|||
private val preBuffer = ArrayDeque<ByteArray>() |
|||
private val PRE_BUFFER_MAX_CHUNKS = 75 |
|||
|
|||
// ── 本地 VAD ────────────────────────────────────────────────────────────── |
|||
private var speechDetected = false |
|||
private val VAD_THRESHOLD = 800.0 |
|||
private val VAD_ONSET_CHUNKS = 3 |
|||
private var vadOnsetCount = 0 |
|||
|
|||
// ── 状态标记 ────────────────────────────────────────────────────────────── |
|||
private var forwardingAudio = false |
|||
private var wsConnecting = false |
|||
private var inCall = false |
|||
private var pttRecording = false |
|||
|
|||
// ── TTS 播放队列 ────────────────────────────────────────────────────────── |
|||
private val ttsQueue = ArrayDeque<String>() |
|||
private var ttsPlaying = false |
|||
private var ttsInterrupted = false |
|||
private var currentPlayer: MediaPlayer? = null |
|||
private val mainHandler = Handler(Looper.getMainLooper()) |
|||
/** 静音开关：设为 true 时停止当前播放、跳过后续 TTS，文本接收完后主动关闭连接 */ |
|||
var ttsMuted = false |
|||
set(value) { |
|||
field = value |
|||
if (value) { |
|||
// 立即停止当前播放 |
|||
ttsQueue.clear() |
|||
ttsPlaying = false |
|||
mainHandler.post { |
|||
try { currentPlayer?.stop(); currentPlayer?.release() } catch (_: Exception) {} |
|||
currentPlayer = null |
|||
} |
|||
emit("tts_canceled", emptyMap()) |
|||
// 如果会话已结束（文本已全部接收），主动关闭连接 |
|||
if (sessionEnded) { |
|||
closeWs() |
|||
if (inCall) { resetVad(); startIdleTimer() } |
|||
} |
|||
} |
|||
} |
|||
|
|||
// ── 消息历史（WS 上下文） ────────────────────────────────────────────────── |
|||
private var messages = JSONArray() |
|||
private val chatStorageHelper = ChatStorageHelper(context) |
|||
private var currentSessionId = "" |
|||
private var pendingAnswer = "" // 累积最完整的回复文本，会话结束时统一发送 |
|||
|
|||
// ── 计时器 ──────────────────────────────────────────────────────────────── |
|||
private var micWatchdogJob: Job? = null |
|||
private var lastWatchdogChunkCount = 0 |
|||
private var idleTimerJob: Job? = null |
|||
private val IDLE_TIMEOUT_MS = 15_000L |
|||
|
|||
// ── 凭证就绪 ────────────────────────────────────────────────────────────── |
|||
var credentialReady = false |
|||
private set |
|||
|
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
// 凭证获取 |
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
|
|||
fun fetchCredentials() { |
|||
if (config.deviceSecret.isNotEmpty()) { |
|||
credentialReady = true |
|||
return |
|||
} |
|||
launch { |
|||
try { |
|||
val body = JSONObject().apply { |
|||
put("deviceNoType", "SN") |
|||
put("deviceNo", config.deviceNo) |
|||
put("productId", config.productId) |
|||
put("productKey", config.productKey) |
|||
}.toString().toRequestBody("application/json".toMediaType()) |
|||
|
|||
val req = Request.Builder() |
|||
.url("https://ivs.chinamobiledevice.com:11443/v2/customer/device/secret/info") |
|||
.post(body) |
|||
.build() |
|||
|
|||
okHttpClient.newCall(req).execute().use { resp -> |
|||
val json = JSONObject(resp.body!!.string()) |
|||
if (json.getString("code") == "200" && json.getBoolean("success")) { |
|||
val d = json.getJSONObject("data") |
|||
config = config.copy( |
|||
deviceNo = d.optString("deviceNo", config.deviceNo), |
|||
deviceId = d.optString("deviceId", ""), |
|||
deviceSecret = d.optString("deviceSecret", ""), |
|||
) |
|||
credentialReady = config.deviceSecret.isNotEmpty() |
|||
if (credentialReady) { |
|||
onCredentialsFetched?.invoke(config.deviceId, config.deviceSecret, config.deviceNo) |
|||
} |
|||
} else { |
|||
Log.e(TAG, "凭证获取失败: ${json.optString("message")}") |
|||
} |
|||
} |
|||
} catch (e: Exception) { |
|||
Log.e(TAG, "凭证获取异常: ${e.message}") |
|||
} |
|||
} |
|||
} |
|||
|
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
// 公开接口 |
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
|
|||
/** 通话模式：开启麦克风，VAD 自动触发 WS */ |
|||
fun startCall() { |
|||
if (inCall) return |
|||
if (ttsPlaying) interruptTts() |
|||
if (pttRecording) stopPtt(cancel = true) |
|||
if (!credentialReady) { emitError("移动精灵凭证未就绪，请稍后重试"); return } |
|||
openMic() |
|||
startMicWatchdog() |
|||
inCall = true |
|||
startIdleTimer() |
|||
} |
|||
|
|||
/** PTT 模式：立即连 WS 并开始录音 */ |
|||
fun startPtt() { |
|||
if (pttRecording) return |
|||
if (ttsPlaying) interruptTts() |
|||
if (!credentialReady) { emitError("移动精灵凭证未就绪"); return } |
|||
|
|||
pttRecording = true |
|||
openMic() |
|||
startMicWatchdog() |
|||
ttsInterrupted = false |
|||
closeWs() |
|||
sessionEnded = false |
|||
wsConnecting = true |
|||
dialogRequestId = UUID.randomUUID().toString() |
|||
clearSendBuffer() |
|||
pendingAnswer = "" |
|||
|
|||
launch { |
|||
try { |
|||
connectWs() |
|||
forwardingAudio = true |
|||
wsConnecting = false |
|||
emit("recognition_started", emptyMap()) |
|||
} catch (e: Exception) { |
|||
wsConnecting = false |
|||
pttRecording = false |
|||
stopMicWatchdog() |
|||
emitError("连接失败: ${e.message}") |
|||
} |
|||
} |
|||
} |
|||
|
|||
/** PTT 松开：发送剩余音频 + finish */ |
|||
fun stopPtt(cancel: Boolean = false) { |
|||
pttRecording = false |
|||
stopMicWatchdog() |
|||
if (cancel) { |
|||
forwardingAudio = false |
|||
closeWs() |
|||
stopMicHard() |
|||
return |
|||
} |
|||
if (forwardingAudio) { |
|||
forwardingAudio = false |
|||
synchronized(sendBufferLock) { |
|||
val remain = sendBuffer.size - sendBufferOffset |
|||
if (remain > 0) { |
|||
sendAudioChunk(sendBuffer.toMutableList().subList(sendBufferOffset, sendBuffer.size).toByteArray()) |
|||
} |
|||
sendBuffer.clear(); sendBufferOffset = 0 |
|||
} |
|||
sendFinish() |
|||
emit("recognition_stopped", emptyMap()) |
|||
} |
|||
stopMicHard() |
|||
} |
|||
|
|||
/** 统一停止：根据当前状态自动选择 stopPtt 或 disconnect */ |
|||
fun stop() { |
|||
if (pttRecording) stopPtt() |
|||
else disconnect() |
|||
} |
|||
|
|||
/** 挂断通话 */ |
|||
fun disconnect() { |
|||
stopMicWatchdog() |
|||
cancelIdleTimer() |
|||
forwardingAudio = false |
|||
wsConnecting = false |
|||
speechDetected = false |
|||
stopMicHard() |
|||
ttsInterrupted = true |
|||
ttsQueue.clear() |
|||
ttsPlaying = false |
|||
mainHandler.post { |
|||
try { currentPlayer?.stop(); currentPlayer?.release() } catch (_: Exception) {} |
|||
currentPlayer = null |
|||
} |
|||
closeWs() |
|||
preBuffer.clear() |
|||
clearSendBuffer() |
|||
flushPendingAnswer() |
|||
inCall = false |
|||
pttRecording = false |
|||
} |
|||
|
|||
/** 打断 AI 播报 */ |
|||
fun interruptSpeaking() { |
|||
interruptTts() |
|||
ttsInterrupted = false |
|||
if (inCall) resetVad() |
|||
} |
|||
|
|||
/** 清空消息历史 */ |
|||
fun clearHistory() { |
|||
messages = JSONArray() |
|||
} |
|||
|
|||
/** 释放资源 */ |
|||
fun dispose() { |
|||
disconnect() |
|||
job.cancel() |
|||
} |
|||
|
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
// 麦克风 & 本地 VAD |
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
|
|||
private fun openMic() { |
|||
stopMicHard() |
|||
speechDetected = false; vadOnsetCount = 0; micChunkCount = 0 |
|||
try { |
|||
audioRecord = AudioRecord( |
|||
MediaRecorder.AudioSource.VOICE_COMMUNICATION, |
|||
SAMPLE_RATE, |
|||
AudioFormat.CHANNEL_IN_MONO, |
|||
AudioFormat.ENCODING_PCM_16BIT, |
|||
RECORD_BUFFER |
|||
) |
|||
audioRecord?.startRecording() |
|||
recordingJob = launch { |
|||
val buf = ByteArray(RECORD_CHUNK) |
|||
while (isActive && (inCall || pttRecording)) { |
|||
val read = audioRecord?.read(buf, 0, buf.size) ?: break |
|||
if (read > 0) { micChunkCount++; onMicAudio(buf.copyOf(read)) } |
|||
} |
|||
} |
|||
} catch (e: Exception) { |
|||
Log.e(TAG, "打开麦克风失败: ${e.message}") |
|||
} |
|||
} |
|||
|
|||
private fun stopMicHard() { |
|||
recordingJob?.cancel(); recordingJob = null |
|||
try { audioRecord?.stop(); audioRecord?.release() } catch (_: Exception) {} |
|||
audioRecord = null |
|||
} |
|||
|
|||
private fun startMicWatchdog() { |
|||
stopMicWatchdog() |
|||
lastWatchdogChunkCount = micChunkCount |
|||
micWatchdogJob = launch { |
|||
while (isActive && (inCall || pttRecording)) { |
|||
delay(1500) |
|||
if (micChunkCount == lastWatchdogChunkCount && !ttsPlaying) openMic() |
|||
lastWatchdogChunkCount = micChunkCount |
|||
} |
|||
} |
|||
} |
|||
|
|||
private fun stopMicWatchdog() { micWatchdogJob?.cancel(); micWatchdogJob = null } |
|||
|
|||
private fun onMicAudio(chunk: ByteArray) { |
|||
if (!inCall && !pttRecording) return |
|||
if (forwardingAudio && webSocket != null) { |
|||
if (micChunkCount % 3 == 0) emit("mic_rms", mapOf("rms" to calcRms(chunk))) |
|||
feedSendBuffer(chunk); return |
|||
} |
|||
if (wsConnecting) { addToPreBuffer(chunk); return } |
|||
if (ttsPlaying) return |
|||
|
|||
addToPreBuffer(chunk) |
|||
val rms = calcRms(chunk) |
|||
// 发送麦克风音量给 Dart 层 |
|||
if (micChunkCount % 3 == 0) { |
|||
emit("mic_rms", mapOf("rms" to rms)) |
|||
} |
|||
if (rms >= VAD_THRESHOLD) { |
|||
vadOnsetCount++ |
|||
if (vadOnsetCount >= VAD_ONSET_CHUNKS && !speechDetected) { |
|||
speechDetected = true |
|||
cancelIdleTimer() |
|||
launch { onSpeechDetected() } |
|||
} |
|||
} else { |
|||
vadOnsetCount = 0 |
|||
} |
|||
} |
|||
|
|||
private fun calcRms(pcm: ByteArray): Double { |
|||
if (pcm.size < 2) return 0.0 |
|||
val samples = pcm.size / 2 |
|||
var sum = 0.0 |
|||
for (i in 0 until pcm.size - 1 step 2) { |
|||
var s = (pcm[i].toInt() and 0xFF) or (pcm[i + 1].toInt() shl 8) |
|||
if (s >= 32768) s -= 65536 |
|||
sum += s.toLong() * s.toLong() |
|||
} |
|||
return sqrt(sum / samples) |
|||
} |
|||
|
|||
private fun addToPreBuffer(chunk: ByteArray) { |
|||
preBuffer.addLast(chunk.copyOf()) |
|||
while (preBuffer.size > PRE_BUFFER_MAX_CHUNKS) preBuffer.removeFirst() |
|||
} |
|||
|
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
// VAD 触发 → WS 连接 |
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
|
|||
private suspend fun onSpeechDetected() { |
|||
if (wsConnecting) return |
|||
ttsInterrupted = false |
|||
closeWs() |
|||
sessionEnded = false |
|||
wsConnecting = true |
|||
dialogRequestId = UUID.randomUUID().toString() |
|||
clearSendBuffer() |
|||
pendingAnswer = "" |
|||
|
|||
try { |
|||
connectWs() |
|||
} catch (e: Exception) { |
|||
wsConnecting = false; speechDetected = false |
|||
emitError("连接失败: ${e.message}") |
|||
return |
|||
} |
|||
|
|||
forwardingAudio = true; wsConnecting = false |
|||
emit("recognition_started", emptyMap()) |
|||
|
|||
val cached = preBuffer.toList(); preBuffer.clear() |
|||
for (c in cached) feedSendBuffer(c) |
|||
} |
|||
|
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
// WebSocket |
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
|
|||
private suspend fun connectWs(): Unit = suspendCancellableCoroutine { cont -> |
|||
val ts = System.currentTimeMillis() |
|||
val sign = calcSign(config.deviceSecret, ts.toString()) |
|||
val sn = UUID.randomUUID().toString() |
|||
val url = "wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr" + |
|||
"?deviceNo=${config.deviceNo}&deviceId=${config.deviceId}" + |
|||
"&productId=${config.productId}&productKey=${config.productKey}" + |
|||
"&sign=$sign&sn=$sn&ts=$ts" |
|||
|
|||
val req = Request.Builder().url(url) |
|||
.header("Origin", "https://ivs.chinamobiledevice.com:11443") |
|||
.header("Host", "ivs.chinamobiledevice.com:11443") |
|||
.build() |
|||
|
|||
var resolved = false |
|||
|
|||
webSocket = okHttpClient.newWebSocket(req, object : WebSocketListener() { |
|||
override fun onOpen(ws: WebSocket, response: Response) { |
|||
sendStartFrame() |
|||
} |
|||
|
|||
override fun onMessage(ws: WebSocket, text: String) { |
|||
try { |
|||
val msg = JSONObject(text) |
|||
if (msg.optString("type") == "ready") { |
|||
if (!resolved) { resolved = true; cont.resume(Unit) {} } |
|||
} else { |
|||
launch { handleMessage(msg) } |
|||
} |
|||
} catch (_: Exception) {} |
|||
} |
|||
|
|||
override fun onMessage(ws: WebSocket, bytes: okio.ByteString) {} |
|||
|
|||
override fun onFailure(ws: WebSocket, t: Throwable, response: Response?) { |
|||
webSocket = null |
|||
if (!resolved) { resolved = true; cont.resumeWithException(t) } |
|||
else launch { handleWsDone() } |
|||
} |
|||
|
|||
override fun onClosing(ws: WebSocket, code: Int, reason: String) { ws.close(1000, null) } |
|||
|
|||
override fun onClosed(ws: WebSocket, code: Int, reason: String) { |
|||
webSocket = null |
|||
if (!resolved) { resolved = true; cont.resumeWithException(IOException("WS closed")) } |
|||
else launch { handleWsDone() } |
|||
} |
|||
}) |
|||
|
|||
cont.invokeOnCancellation { closeWs() } |
|||
|
|||
// 15s 超时 |
|||
launch { |
|||
delay(15_000) |
|||
if (!resolved) { |
|||
resolved = true; closeWs() |
|||
cont.resumeWithException(IOException("等待 ready 超时")) |
|||
} |
|||
} |
|||
} |
|||
|
|||
private fun handleWsDone() { |
|||
webSocket = null; forwardingAudio = false; wsConnecting = false |
|||
when { |
|||
sessionEnded && inCall -> if (!ttsPlaying) { resetVad(); startIdleTimer() } |
|||
sessionEnded -> { /* PTT 结束，等 TTS 播完 */ } |
|||
inCall -> resetVad() |
|||
} |
|||
} |
|||
|
|||
private fun resetVad() { |
|||
speechDetected = false; vadOnsetCount = 0; preBuffer.clear(); openMic() |
|||
} |
|||
|
|||
private fun closeWs() { |
|||
try { webSocket?.cancel() } catch (_: Exception) {} |
|||
webSocket = null |
|||
} |
|||
|
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
// 发送帧 |
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
|
|||
private fun sendStartFrame() { |
|||
currentSessionId = UUID.randomUUID().toString() |
|||
val payload = JSONObject().apply { |
|||
put("type", "start") |
|||
put("data", JSONObject().apply { |
|||
put("cuid", config.deviceId) |
|||
put("format", "pcm") |
|||
put("sample", 16000) |
|||
put("support_dcs", 2) |
|||
put("chunk_size", 10240) |
|||
put("support_tts", true) |
|||
put("support_text2dcs", true) |
|||
put("user_agent", "voitrans/1.0") |
|||
put("dialog_request_id", dialogRequestId) |
|||
put("access_rc", true) |
|||
put("support_part_tts", true) |
|||
put("need_stoplisten", true) |
|||
put("need_dialogue_finish", true) |
|||
put("result_trans2directive", false) |
|||
put("rc_version", "5") |
|||
put("messages", messages) |
|||
put("client_context", JSONArray()) |
|||
}) |
|||
} |
|||
webSocket?.send(payload.toString()) |
|||
} |
|||
|
|||
private val sendBufferLock = Any() |
|||
|
|||
private fun clearSendBuffer() { |
|||
synchronized(sendBufferLock) { |
|||
sendBuffer.clear(); sendBufferOffset = 0 |
|||
} |
|||
} |
|||
|
|||
private fun feedSendBuffer(chunk: ByteArray) { |
|||
synchronized(sendBufferLock) { |
|||
sendBuffer.addAll(chunk.toList()) |
|||
while (sendBuffer.size - sendBufferOffset >= FRAME_SIZE) { |
|||
val frame = ByteArray(FRAME_SIZE) { i -> sendBuffer[sendBufferOffset + i] } |
|||
sendBufferOffset += FRAME_SIZE |
|||
sendAudioChunk(frame) |
|||
} |
|||
// 收缩缓冲 |
|||
if (sendBufferOffset >= FRAME_SIZE * 8) { |
|||
val toRemove = sendBufferOffset |
|||
repeat(toRemove) { if (sendBuffer.isNotEmpty()) sendBuffer.removeFirst() } |
|||
sendBufferOffset = 0 |
|||
} |
|||
} |
|||
} |
|||
|
|||
private fun sendAudioChunk(data: ByteArray) { |
|||
try { webSocket?.send(data.toByteString()) } catch (_: Exception) {} |
|||
} |
|||
|
|||
private fun sendFinish() { |
|||
try { webSocket?.send("""{"type":"finish"}""") } catch (_: Exception) {} |
|||
} |
|||
|
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
// 消息处理 |
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
|
|||
private suspend fun handleMessage(msg: JSONObject) { |
|||
when (msg.optString("type")) { |
|||
"stoplisten" -> { |
|||
forwardingAudio = false; speechDetected = false; vadOnsetCount = 0 |
|||
synchronized(sendBufferLock) { |
|||
val remain = sendBuffer.size - sendBufferOffset |
|||
if (remain > 0) { |
|||
sendAudioChunk(ByteArray(remain) { i -> sendBuffer[sendBufferOffset + i] }) |
|||
} |
|||
sendBuffer.clear(); sendBufferOffset = 0 |
|||
} |
|||
sendFinish() |
|||
emit("recognition_stopped", emptyMap()) |
|||
} |
|||
"mid_result" -> { |
|||
val text = msg.optString("result", "") |
|||
if (text.isNotEmpty()) emit("recognizing", mapOf("text" to text)) |
|||
} |
|||
"fin_result" -> { |
|||
val result = msg.optString("result", "") |
|||
if (result.isNotEmpty()) { |
|||
emit("recognition_result", mapOf("text" to result)) |
|||
messages.put(JSONObject(mapOf("role" to "user", "content" to result))) |
|||
withContext(Dispatchers.IO) { |
|||
try { |
|||
chatStorageHelper.saveMessage(config.agentId, currentSessionId, result, "user") |
|||
} catch (_: Exception) {} |
|||
} |
|||
} |
|||
if (msg.optInt("end", 0) == 1) markSessionEnded() |
|||
} |
|||
"inside_rc" -> handleInsideRc(msg) |
|||
"dcs_decide" -> if (msg.optInt("end", 0) == 1) markSessionEnded() |
|||
} |
|||
} |
|||
|
|||
private suspend fun handleInsideRc(msg: JSONObject) { |
|||
val data = msg.optJSONObject("data") ?: return |
|||
if (data.optInt("code", -1) != 0) return |
|||
|
|||
val rawAnswer = data.optString("assistant_answer", "") |
|||
val cleanAnswer = try { |
|||
JSONObject(rawAnswer).optString("content", rawAnswer) |
|||
} catch (_: Exception) { rawAnswer } |
|||
|
|||
var ttsUrl: String? = null |
|||
val items = data.optJSONArray("data") |
|||
if (items != null) { |
|||
for (i in 0 until items.length()) { |
|||
val item = items.optJSONObject(i) ?: continue |
|||
val h = item.optJSONObject("header") ?: continue |
|||
if (h.optString("namespace") == "ai.fxzsos.device_interface.voice_output" && |
|||
h.optString("name") == "Speak" |
|||
) { |
|||
ttsUrl = item.optJSONObject("payload")?.optString("url") |
|||
break |
|||
} |
|||
} |
|||
} |
|||
|
|||
// 每次取更完整的回复文本，立即发送给 UI |
|||
if (cleanAnswer.isNotEmpty() && cleanAnswer.length > pendingAnswer.length) { |
|||
pendingAnswer = cleanAnswer |
|||
emit("assistant_response", mapOf("response" to cleanAnswer, "sessionid" to currentSessionId)) |
|||
} |
|||
ttsUrl?.takeIf { it.isNotEmpty() }?.let { enqueueTts(it) } |
|||
if (msg.optInt("end", 0) == 1) markSessionEnded() |
|||
} |
|||
|
|||
/** 将 pendingAnswer 保存到历史和存储，并清空 */ |
|||
private fun flushPendingAnswer() { |
|||
if (pendingAnswer.isEmpty()) return |
|||
val answer = pendingAnswer |
|||
val sid = currentSessionId |
|||
messages.put(JSONObject(mapOf("role" to "assistant", "content" to answer))) |
|||
launch(Dispatchers.IO) { |
|||
try { |
|||
chatStorageHelper.saveMessage(config.agentId, sid, answer, "assistant") |
|||
} catch (_: Exception) {} |
|||
} |
|||
pendingAnswer = "" |
|||
} |
|||
|
|||
private fun markSessionEnded() { |
|||
if (sessionEnded) return |
|||
sessionEnded = true; forwardingAudio = false; speechDetected = false; vadOnsetCount = 0 |
|||
clearSendBuffer() |
|||
flushPendingAnswer() |
|||
// 静音模式下文本接收完毕，发送完成事件并关闭连接 |
|||
if (ttsMuted) { |
|||
emit("tts_completed", emptyMap()) |
|||
closeWs() |
|||
if (inCall) { resetVad(); startIdleTimer() } |
|||
} |
|||
} |
|||
|
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
// TTS 播放队列 |
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
|
|||
private fun enqueueTts(url: String) { |
|||
if (ttsMuted) return // 静音模式跳过 TTS |
|||
ttsQueue.addLast(url) |
|||
if (!ttsPlaying) { |
|||
stopMicHard() |
|||
mainHandler.post { playNextTts() } |
|||
} |
|||
} |
|||
|
|||
private fun playNextTts() { |
|||
if (ttsQueue.isEmpty() || ttsInterrupted) { |
|||
ttsPlaying = false |
|||
if (!ttsInterrupted) { |
|||
emit("tts_completed", emptyMap()) |
|||
if (inCall) { resetVad(); startIdleTimer() } |
|||
} |
|||
return |
|||
} |
|||
ttsPlaying = true |
|||
emit("tts_started", emptyMap()) |
|||
val url = ttsQueue.removeFirst() |
|||
try { |
|||
currentPlayer?.stop(); currentPlayer?.release() |
|||
currentPlayer = MediaPlayer().apply { |
|||
setDataSource(url) |
|||
setOnCompletionListener { |
|||
if (!ttsInterrupted) mainHandler.post { playNextTts() } |
|||
} |
|||
setOnErrorListener { _, _, _ -> |
|||
if (!ttsInterrupted) mainHandler.post { playNextTts() } |
|||
true |
|||
} |
|||
prepare() |
|||
start() |
|||
} |
|||
} catch (e: Exception) { |
|||
Log.e(TAG, "TTS 播放失败: ${e.message}") |
|||
if (!ttsInterrupted) mainHandler.post { playNextTts() } |
|||
} |
|||
} |
|||
|
|||
private fun interruptTts() { |
|||
ttsInterrupted = true; ttsQueue.clear(); ttsPlaying = false |
|||
mainHandler.post { |
|||
try { currentPlayer?.stop(); currentPlayer?.release() } catch (_: Exception) {} |
|||
currentPlayer = null |
|||
} |
|||
emit("tts_canceled", emptyMap()) |
|||
} |
|||
|
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
// 超时 & 工具 |
|||
// ═════════════════════════════════════════════════════════════════════════ |
|||
|
|||
private fun startIdleTimer() { |
|||
cancelIdleTimer() |
|||
idleTimerJob = launch { |
|||
delay(IDLE_TIMEOUT_MS) |
|||
if (inCall && !speechDetected) disconnect() |
|||
} |
|||
} |
|||
|
|||
private fun cancelIdleTimer() { idleTimerJob?.cancel(); idleTimerJob = null } |
|||
|
|||
private fun calcSign(secret: String, ts: String): String { |
|||
val digest = MessageDigest.getInstance("MD5").digest("$secret$ts".toByteArray()) |
|||
return digest.joinToString("") { "%02x".format(it) } |
|||
} |
|||
|
|||
private fun emit(eventName: String, data: Map<String, Any>) { |
|||
launch(Dispatchers.Main) { onEvent(eventName, data) } |
|||
} |
|||
|
|||
private fun emitError(msg: String) = emit("error", mapOf("message" to msg)) |
|||
|
|||
private fun buildTrustAllClient(): OkHttpClient { |
|||
return try { |
|||
val trustAll = arrayOf<TrustManager>(object : X509TrustManager { |
|||
override fun checkClientTrusted(c: Array<X509Certificate>, t: String) {} |
|||
override fun checkServerTrusted(c: Array<X509Certificate>, t: String) {} |
|||
override fun getAcceptedIssuers(): Array<X509Certificate> = emptyArray() |
|||
}) |
|||
val sc = SSLContext.getInstance("TLS") |
|||
sc.init(null, trustAll, SecureRandom()) |
|||
OkHttpClient.Builder() |
|||
.sslSocketFactory(sc.socketFactory, trustAll[0] as X509TrustManager) |
|||
.hostnameVerifier { _, _ -> true } |
|||
.connectTimeout(15, TimeUnit.SECONDS) |
|||
.readTimeout(90, TimeUnit.SECONDS) |
|||
.writeTimeout(90, TimeUnit.SECONDS) |
|||
.build() |
|||
} catch (e: Exception) { |
|||
Log.e(TAG, "buildTrustAllClient 失败，使用默认客户端", e) |
|||
OkHttpClient() |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,495 @@ |
|||
import Foundation |
|||
import AVFoundation |
|||
import CryptoKit |
|||
import os.log |
|||
import chat_storage |
|||
|
|||
// MARK: - StsConfig |
|||
|
|||
struct StsConfig { |
|||
let productId: String |
|||
let productKey: String |
|||
var deviceNo: String |
|||
var deviceId: String |
|||
var deviceSecret: String |
|||
let agentId: String |
|||
} |
|||
|
|||
// MARK: - StsAgent |
|||
|
|||
/// StsAgent — 中国移动 A80 WS 一体化语音对话实现（iOS） |
|||
/// |
|||
/// 通过一条 WSS 连接完成 ASR + AI + TTS，事件格式与标准 stt_llm_tts 完全一致， |
|||
/// 通过 onEvent 回调发出，供 AgentServiceImpl 转发到 Flutter EventChannel。 |
|||
class StsAgent: NSObject { |
|||
|
|||
private let TAG = "StsAgent" |
|||
private let logger = OSLog(subsystem: "com.yunqiinnovation.agent_service", category: "StsAgent") |
|||
|
|||
private var config: StsConfig |
|||
private let onEvent: (String, [String: Any]) -> Void |
|||
private let onCredentialsFetched: ((String, String, String) -> Void)? |
|||
|
|||
// MARK: WebSocket |
|||
private var webSocketTask: URLSessionWebSocketTask? |
|||
private var urlSession: URLSession! |
|||
private var readyReceived = false |
|||
private var sessionEnded = true |
|||
private var dialogRequestId = "" |
|||
|
|||
// MARK: 录音 |
|||
private let SAMPLE_RATE: Double = 16000 |
|||
private let FRAME_SIZE = 5120 // 160ms @ 16kHz 16bit mono |
|||
private var audioEngine: AVAudioEngine? |
|||
private var inputNode: AVAudioInputNode? |
|||
private var recordingActive = false |
|||
private var sendBuffer = Data() |
|||
private var micChunkCount = 0 |
|||
|
|||
// MARK: VAD |
|||
private let VAD_THRESHOLD: Float = 800 |
|||
private let VAD_ONSET_CHUNKS = 3 |
|||
private var vadOnsetCount = 0 |
|||
private var vadActive = false |
|||
private var preBuffer: [[UInt8]] = [] |
|||
private let PRE_BUFFER_MAX = 75 |
|||
|
|||
// MARK: TTS |
|||
private var ttsQueue: [String] = [] |
|||
private var ttsPlayer: AVPlayer? |
|||
private var ttsPlaying = false |
|||
private var ttsPlayerObserver: Any? |
|||
|
|||
// MARK: 历史 |
|||
private var messages: [[String: String]] = [] |
|||
private var currentSessionId = "" |
|||
private let chatStorage = ChatStorageHelper.shared |
|||
|
|||
// MARK: 模式 |
|||
private var isPttMode = false |
|||
|
|||
// MARK: 凭证请求 |
|||
private var credentialsFetched = false |
|||
|
|||
init(config: StsConfig, |
|||
onEvent: @escaping (String, [String: Any]) -> Void, |
|||
onCredentialsFetched: ((String, String, String) -> Void)? = nil) { |
|||
self.config = config |
|||
self.onEvent = onEvent |
|||
self.onCredentialsFetched = onCredentialsFetched |
|||
super.init() |
|||
let sessionConfig = URLSessionConfiguration.default |
|||
urlSession = URLSession(configuration: sessionConfig, delegate: nil, delegateQueue: OperationQueue()) |
|||
} |
|||
|
|||
// MARK: - 公共接口 |
|||
|
|||
func startCall() { |
|||
isPttMode = false |
|||
os_log("StsAgent startCall", log: logger, type: .info) |
|||
fetchCredentialsIfNeeded { [weak self] in |
|||
self?.connect() |
|||
} |
|||
} |
|||
|
|||
func startPtt() { |
|||
isPttMode = true |
|||
os_log("StsAgent startPtt", log: logger, type: .info) |
|||
fetchCredentialsIfNeeded { [weak self] in |
|||
self?.connect() |
|||
} |
|||
} |
|||
|
|||
func stopPtt() { |
|||
os_log("StsAgent stopPtt", log: logger, type: .info) |
|||
guard isPttMode else { return } |
|||
sendFinishFrame() |
|||
} |
|||
|
|||
/// 统一停止：根据当前模式自动选择 stopPtt 或 dispose |
|||
func stop() { |
|||
if isPttMode { |
|||
stopPtt() |
|||
} else { |
|||
dispose() |
|||
} |
|||
} |
|||
|
|||
func interruptSpeaking() { |
|||
os_log("StsAgent interruptSpeaking", log: logger, type: .info) |
|||
stopTts() |
|||
emit("tts_canceled") |
|||
} |
|||
|
|||
func clearHistory() { |
|||
messages.removeAll() |
|||
DispatchQueue.global().async { [weak self] in |
|||
guard let self = self else { return } |
|||
_ = self.chatStorage.deleteMessages(agentId: self.config.agentId, messageIds: nil as [Int]?) |
|||
} |
|||
} |
|||
|
|||
func dispose() { |
|||
os_log("StsAgent dispose", log: logger, type: .info) |
|||
stopRecording() |
|||
webSocketTask?.cancel(with: .normalClosure, reason: nil) |
|||
webSocketTask = nil |
|||
stopTts() |
|||
} |
|||
|
|||
// MARK: - 凭证获取 |
|||
|
|||
private func fetchCredentialsIfNeeded(completion: @escaping () -> Void) { |
|||
if credentialsFetched && !config.deviceId.isEmpty && !config.deviceSecret.isEmpty { |
|||
completion() |
|||
return |
|||
} |
|||
let ts = Int64(Date().timeIntervalSince1970 * 1000) |
|||
let sign = calcSign(secret: config.productKey, ts: ts) |
|||
let urlStr = "https://ivs.chinamobiledevice.com:11443/v2/customer/device/secret/info" |
|||
guard var components = URLComponents(string: urlStr) else { completion(); return } |
|||
components.queryItems = [ |
|||
URLQueryItem(name: "productId", value: config.productId), |
|||
URLQueryItem(name: "deviceNo", value: config.deviceNo), |
|||
URLQueryItem(name: "ts", value: "\(ts)"), |
|||
URLQueryItem(name: "sign", value: sign), |
|||
] |
|||
guard let url = components.url else { completion(); return } |
|||
var req = URLRequest(url: url) |
|||
req.timeoutInterval = 10 |
|||
URLSession.shared.dataTask(with: req) { [weak self] data, _, error in |
|||
guard let self = self else { return } |
|||
if let error = error { |
|||
os_log("凭证请求失败: %{public}@", log: self.logger, type: .error, error.localizedDescription) |
|||
completion() |
|||
return |
|||
} |
|||
guard let data = data, |
|||
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], |
|||
let body = json["body"] as? [String: Any] else { |
|||
completion() |
|||
return |
|||
} |
|||
if let deviceId = body["deviceId"] as? String { self.config.deviceId = deviceId } |
|||
if let deviceSecret = body["deviceSecret"] as? String { self.config.deviceSecret = deviceSecret } |
|||
if let deviceNo = body["deviceNo"] as? String { self.config.deviceNo = deviceNo } |
|||
self.credentialsFetched = true |
|||
self.onCredentialsFetched?(self.config.deviceId, self.config.deviceSecret, self.config.deviceNo) |
|||
completion() |
|||
}.resume() |
|||
} |
|||
|
|||
// MARK: - WebSocket 连接 |
|||
|
|||
private func connect() { |
|||
let ts = Int64(Date().timeIntervalSince1970 * 1000) |
|||
let sign = calcSign(secret: config.deviceSecret, ts: ts) |
|||
let urlStr = "wss://ivs.chinamobiledevice.com:11443/app-ws/v2/asr" |
|||
guard var components = URLComponents(string: urlStr) else { return } |
|||
components.queryItems = [ |
|||
URLQueryItem(name: "productId", value: config.productId), |
|||
URLQueryItem(name: "deviceId", value: config.deviceId), |
|||
URLQueryItem(name: "ts", value: "\(ts)"), |
|||
URLQueryItem(name: "sign", value: sign), |
|||
] |
|||
guard let url = components.url else { return } |
|||
webSocketTask = urlSession.webSocketTask(with: url) |
|||
webSocketTask?.resume() |
|||
receiveMessages() |
|||
|
|||
// 发送 start 帧 |
|||
sendStartFrame() |
|||
} |
|||
|
|||
private func sendStartFrame() { |
|||
dialogRequestId = UUID().uuidString.replacingOccurrences(of: "-", with: "").lowercased() |
|||
currentSessionId = dialogRequestId |
|||
let payload: [String: Any] = [ |
|||
"msgType": "start", |
|||
"dialogRequestId": dialogRequestId, |
|||
"params": [ |
|||
"asrParams": [ |
|||
"format": "pcm", |
|||
"sampleRate": 16000, |
|||
"lang": "zh-CN", |
|||
"vadEnable": false, |
|||
], |
|||
"contextParams": [ |
|||
"history": messages.map { ["role": $0["role"] ?? "", "content": $0["content"] ?? ""] } |
|||
] |
|||
] |
|||
] |
|||
sendJson(payload) |
|||
} |
|||
|
|||
private func receiveMessages() { |
|||
webSocketTask?.receive { [weak self] result in |
|||
guard let self = self else { return } |
|||
switch result { |
|||
case .success(let msg): |
|||
switch msg { |
|||
case .string(let text): |
|||
self.handleMessage(text) |
|||
default: |
|||
break |
|||
} |
|||
self.receiveMessages() |
|||
case .failure(let err): |
|||
os_log("WS error: %{public}@", log: self.logger, type: .error, err.localizedDescription) |
|||
self.emit("error", data: ["message": err.localizedDescription]) |
|||
} |
|||
} |
|||
} |
|||
|
|||
// MARK: - 消息处理 |
|||
|
|||
private func handleMessage(_ text: String) { |
|||
guard let data = text.data(using: .utf8), |
|||
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return } |
|||
let msgType = json["msgType"] as? String ?? "" |
|||
os_log("WS recv: %{public}@", log: logger, type: .debug, msgType) |
|||
|
|||
switch msgType { |
|||
case "ready": |
|||
readyReceived = true |
|||
sessionEnded = false |
|||
emit("recognition_started") |
|||
startRecording() |
|||
if isPttMode { |
|||
// PTT 模式等待 stopPtt() 调用发送 finish |
|||
} |
|||
|
|||
case "mid_result": |
|||
let result = (json["data"] as? [String: Any])?["result"] as? String ?? "" |
|||
if !result.isEmpty { emit("recognizing", data: ["text": result]) } |
|||
|
|||
case "fin_result": |
|||
let result = (json["data"] as? [String: Any])?["result"] as? String ?? "" |
|||
emit("recognition_result", data: ["text": result]) |
|||
emit("recognition_stopped") |
|||
stopRecording() |
|||
if !result.isEmpty { |
|||
messages.append(["role": "user", "content": result]) |
|||
saveToChatStorage(text: result, role: "user") |
|||
} |
|||
|
|||
case "stoplisten": |
|||
stopRecording() |
|||
emit("recognition_stopped") |
|||
|
|||
case "inside_rc": |
|||
handleInsideRc(json["data"] as? [String: Any]) |
|||
|
|||
case "dcs_decide": |
|||
// 语义决策，忽略 |
|||
break |
|||
|
|||
default: |
|||
os_log("未处理的消息类型: %{public}@", log: logger, type: .info, msgType) |
|||
} |
|||
} |
|||
|
|||
private func handleInsideRc(_ data: [String: Any]?) { |
|||
guard let data = data else { return } |
|||
guard let rcType = data["rcType"] as? String else { return } |
|||
if rcType == "assistant_answer" { |
|||
let answer = (data["answer"] as? [String: Any])?["text"] as? String ?? "" |
|||
let cleanAnswer = answer.trimmingCharacters(in: .whitespacesAndNewlines) |
|||
let sid = data["sessionId"] as? String ?? currentSessionId |
|||
if !cleanAnswer.isEmpty { |
|||
messages.append(["role": "assistant", "content": cleanAnswer]) |
|||
saveToChatStorage(text: cleanAnswer, role: "assistant") |
|||
emit("assistant_response", data: ["response": cleanAnswer, "sessionid": sid]) |
|||
} |
|||
// TTS URLs |
|||
if let ttsData = data["tts"] as? [[String: Any]] { |
|||
for item in ttsData { |
|||
if let url = item["url"] as? String, !url.isEmpty { |
|||
ttsQueue.append(url) |
|||
} |
|||
} |
|||
if !ttsPlaying { playNextTts() } |
|||
} |
|||
} |
|||
} |
|||
|
|||
// MARK: - TTS 播放 |
|||
|
|||
private func playNextTts() { |
|||
guard !ttsQueue.isEmpty else { |
|||
ttsPlaying = false |
|||
emit("tts_completed") |
|||
return |
|||
} |
|||
let urlStr = ttsQueue.removeFirst() |
|||
guard let url = URL(string: urlStr) else { playNextTts(); return } |
|||
|
|||
ttsPlaying = true |
|||
if ttsQueue.isEmpty && !ttsPlaying { emit("tts_started") } |
|||
emit("tts_started") |
|||
|
|||
let item = AVPlayerItem(url: url) |
|||
ttsPlayer = AVPlayer(playerItem: item) |
|||
if let observer = ttsPlayerObserver { |
|||
NotificationCenter.default.removeObserver(observer) |
|||
} |
|||
ttsPlayerObserver = NotificationCenter.default.addObserver( |
|||
forName: .AVPlayerItemDidPlayToEndTime, object: item, queue: .main |
|||
) { [weak self] _ in |
|||
self?.playNextTts() |
|||
} |
|||
ttsPlayer?.play() |
|||
} |
|||
|
|||
private func stopTts() { |
|||
ttsPlayer?.pause() |
|||
ttsPlayer = nil |
|||
ttsQueue.removeAll() |
|||
ttsPlaying = false |
|||
if let observer = ttsPlayerObserver { |
|||
NotificationCenter.default.removeObserver(observer) |
|||
ttsPlayerObserver = nil |
|||
} |
|||
} |
|||
|
|||
// MARK: - 录音 |
|||
|
|||
private func startRecording() { |
|||
guard !recordingActive else { return } |
|||
let engine = AVAudioEngine() |
|||
audioEngine = engine |
|||
inputNode = engine.inputNode |
|||
let fmt = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: SAMPLE_RATE, channels: 1, interleaved: true)! |
|||
let hwFmt = inputNode!.outputFormat(forBus: 0) |
|||
let converterFmt = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: SAMPLE_RATE, channels: 1, interleaved: true)! |
|||
|
|||
inputNode!.installTap(onBus: 0, bufferSize: 4096, format: hwFmt) { [weak self] buffer, _ in |
|||
guard let self = self, self.recordingActive else { return } |
|||
// 转换为 16kHz PCM16 |
|||
guard let cvt = AVAudioConverter(from: hwFmt, to: converterFmt) else { return } |
|||
let frameCount = AVAudioFrameCount(Double(buffer.frameLength) * self.SAMPLE_RATE / hwFmt.sampleRate) |
|||
guard let outBuf = AVAudioPCMBuffer(pcmFormat: converterFmt, frameCapacity: frameCount) else { return } |
|||
var error: NSError? |
|||
cvt.convert(to: outBuf, error: &error) { _, outStatus in |
|||
outStatus.pointee = .haveData |
|||
return buffer |
|||
} |
|||
if error != nil { return } |
|||
let byteCount = Int(outBuf.frameLength) * 2 |
|||
let ptr = outBuf.int16ChannelData![0] |
|||
let data = Data(bytes: ptr, count: byteCount) |
|||
self.onMicAudio(data) |
|||
} |
|||
|
|||
do { |
|||
try AVAudioSession.sharedInstance().setCategory(.playAndRecord, mode: .voiceChat, options: [.defaultToSpeaker]) |
|||
try AVAudioSession.sharedInstance().setActive(true) |
|||
try engine.start() |
|||
recordingActive = true |
|||
} catch { |
|||
os_log("录音启动失败: %{public}@", log: logger, type: .error, error.localizedDescription) |
|||
} |
|||
} |
|||
|
|||
private func stopRecording() { |
|||
guard recordingActive else { return } |
|||
recordingActive = false |
|||
inputNode?.removeTap(onBus: 0) |
|||
audioEngine?.stop() |
|||
audioEngine = nil |
|||
inputNode = nil |
|||
sendBuffer.removeAll() |
|||
micChunkCount = 0 |
|||
vadActive = false |
|||
vadOnsetCount = 0 |
|||
preBuffer.removeAll() |
|||
} |
|||
|
|||
private func onMicAudio(_ pcm: Data) { |
|||
micChunkCount += 1 |
|||
let rms = calcRms(pcm) |
|||
let chunk = [UInt8](pcm) |
|||
|
|||
if !vadActive { |
|||
if preBuffer.count >= PRE_BUFFER_MAX { preBuffer.removeFirst() } |
|||
preBuffer.append(chunk) |
|||
} |
|||
|
|||
if !vadActive && rms > VAD_THRESHOLD { |
|||
vadOnsetCount += 1 |
|||
if vadOnsetCount >= VAD_ONSET_CHUNKS { |
|||
vadActive = true |
|||
// 先发预缓存 |
|||
for c in preBuffer { feedSendBuffer(Data(c)) } |
|||
preBuffer.removeAll() |
|||
} |
|||
} else if vadActive { |
|||
feedSendBuffer(pcm) |
|||
} |
|||
} |
|||
|
|||
private func calcRms(_ pcm: Data) -> Float { |
|||
let count = pcm.count / 2 |
|||
guard count > 0 else { return 0 } |
|||
var sum: Float = 0 |
|||
pcm.withUnsafeBytes { ptr in |
|||
let samples = ptr.bindMemory(to: Int16.self) |
|||
for i in 0..<count { |
|||
let s = Float(samples[i]) |
|||
sum += s * s |
|||
} |
|||
} |
|||
return sqrtf(sum / Float(count)) |
|||
} |
|||
|
|||
private func feedSendBuffer(_ chunk: Data) { |
|||
sendBuffer.append(chunk) |
|||
while sendBuffer.count >= FRAME_SIZE { |
|||
let frame = sendBuffer.prefix(FRAME_SIZE) |
|||
sendBuffer.removeFirst(FRAME_SIZE) |
|||
webSocketTask?.send(.data(Data(frame))) { _ in } |
|||
} |
|||
} |
|||
|
|||
private func sendFinishFrame() { |
|||
let payload: [String: Any] = [ |
|||
"msgType": "finish", |
|||
"dialogRequestId": dialogRequestId, |
|||
] |
|||
sendJson(payload) |
|||
} |
|||
|
|||
// MARK: - 工具 |
|||
|
|||
private func sendJson(_ dict: [String: Any]) { |
|||
guard let data = try? JSONSerialization.data(withJSONObject: dict), |
|||
let str = String(data: data, encoding: .utf8) else { return } |
|||
webSocketTask?.send(.string(str)) { _ in } |
|||
} |
|||
|
|||
private func calcSign(secret: String, ts: Int64) -> String { |
|||
let raw = "\(secret)\(ts)" |
|||
let data = Data(raw.utf8) |
|||
let digest = Insecure.MD5.hash(data: data) |
|||
return digest.map { String(format: "%02x", $0) }.joined() |
|||
} |
|||
|
|||
private func emit(_ name: String, data: [String: Any] = [:]) { |
|||
DispatchQueue.main.async { [weak self] in |
|||
self?.onEvent(name, data) |
|||
} |
|||
} |
|||
|
|||
private func saveToChatStorage(text: String, role: String) { |
|||
DispatchQueue.global().async { [weak self] in |
|||
guard let self = self else { return } |
|||
_ = self.chatStorage.saveMessage( |
|||
agentId: self.config.agentId, |
|||
sessionId: self.currentSessionId, |
|||
message: text, |
|||
sender: role, |
|||
metadata: nil |
|||
) |
|||
} |
|||
} |
|||
} |
|||
Loading…
Reference in new issue