Browse Source

Merge branch 'new_dev' of https://github.com/deepcloud2048/deep_voice into new_dev

newdev_shunjiawei
lxm 1 year ago
parent
commit
9c0235ad2f
  1. 22
      lib/modules/agent/controllers/agent_controller.dart
  2. 114
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  3. 8
      local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift
  4. 10
      local_plugins/agent_service/lib/agent_service.dart
  5. 5
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt
  6. 351
      local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt
  7. 44
      local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt
  8. 4
      local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/MCPSubClient.kt
  9. 22
      local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/SystemFunctionHandler.kt
  10. 1
      local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/TtsEvents.kt

22
lib/modules/agent/controllers/agent_controller.dart

@ -263,6 +263,7 @@ class AgentController extends GetxController {
case AgentServiceEventType.recognizing: case AgentServiceEventType.recognizing:
final text = event.data['text'] ?? ''; final text = event.data['text'] ?? '';
currentText.value = text; // 保留当前文本,以便其他地方使用 currentText.value = text; // 保留当前文本,以便其他地方使用
Logger.i(TAG, '识别中间结果: $text');
if (text.isNotEmpty) { if (text.isNotEmpty) {
// 查找是否有正在识别中的消息 // 查找是否有正在识别中的消息
final index = messages final index = messages
@ -290,7 +291,7 @@ class AgentController extends GetxController {
case AgentServiceEventType.recognitionResult: case AgentServiceEventType.recognitionResult:
final text = event.data['text'] ?? ''; final text = event.data['text'] ?? '';
Logger.i(TAG, '识别结果: $text'); Logger.i(TAG, '识别最终结果: $text');
// 查找是否有正在识别中的消息 // 查找是否有正在识别中的消息
final index = final index =
messages.lastIndexWhere((msg) => msg.isRecognizing && msg.isUser); messages.lastIndexWhere((msg) => msg.isRecognizing && msg.isUser);
@ -323,16 +324,27 @@ class AgentController extends GetxController {
break; break;
case AgentServiceEventType.ttsStarted: case AgentServiceEventType.ttsStarted:
isSpeaking.value = true; // isSpeaking.value = true;
Logger.i(TAG, 'TTS开始播放'); // Logger.i(TAG, 'TTS开始播放');
break; break;
case AgentServiceEventType.ttsCompleted: case AgentServiceEventType.ttsCompleted:
case AgentServiceEventType.ttsStopped: case AgentServiceEventType.ttsStopped:
case AgentServiceEventType.ttsCanceled: case AgentServiceEventType.ttsCanceled:
// isSpeaking.value = false;
// Logger.i(TAG, 'TTS停止播放, $event.type');
break;
case AgentServiceEventType.playbackStarted:
isSpeaking.value = true;
Logger.i(TAG, 'TTS播放开始');
break;
case AgentServiceEventType.playbackCompleted:
isSpeaking.value = false; isSpeaking.value = false;
Logger.i(TAG, 'TTS停止播放, $event.type'); Logger.i(TAG, 'TTS播放完成');
break; break;
case AgentServiceEventType.imageProcessing: case AgentServiceEventType.imageProcessing:
@ -348,7 +360,7 @@ class AgentController extends GetxController {
final token = event.data['token'] ?? ''; final token = event.data['token'] ?? '';
final responseId = event.data['responseId'] ?? ''; final responseId = event.data['responseId'] ?? '';
// Logger.i(TAG, 'AI回复Token: $token, responseId: $responseId');
if (token.isNotEmpty) { if (token.isNotEmpty) {
// 如果是新的回复或者响应ID改变,创建新消息 // 如果是新的回复或者响应ID改变,创建新消息
if (_isNewAssistantResponse || if (_isNewAssistantResponse ||

114
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt

@ -16,6 +16,9 @@ import java.util.Collections
import kotlin.coroutines.CoroutineContext import kotlin.coroutines.CoroutineContext
import com.yunqiinnovation.ble_service.BleService import com.yunqiinnovation.ble_service.BleService
import android.media.MediaPlayer import android.media.MediaPlayer
import java.util.concurrent.atomic.AtomicBoolean
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import com.deep_voice.speech.tts.TtsEvent import com.deep_voice.speech.tts.TtsEvent
import com.deep_voice.speech.tts.TtsEventListener import com.deep_voice.speech.tts.TtsEventListener
import com.deep_voice.speech.tts.TtsEventType import com.deep_voice.speech.tts.TtsEventType
@ -43,7 +46,7 @@ object AgentService : CoroutineScope {
// 协程相关 // 协程相关
private val job = SupervisorJob() private val job = SupervisorJob()
override val coroutineContext: CoroutineContext override val coroutineContext: CoroutineContext
get() = Dispatchers.Main + job get() = Dispatchers.IO + job
// 上下文和监听器 // 上下文和监听器
private lateinit var context: Context private lateinit var context: Context
@ -73,16 +76,24 @@ object AgentService : CoroutineScope {
// 音频播放器 // 音频播放器
private var audioPlayer: AudioPlayer? = null // 初始化音频播放器 private var audioPlayer: AudioPlayer? = null // 初始化音频播放器
// 状态 // 状态 - 使用原子类型确保线程安全
private var isInitialized = false private val _isInitialized = AtomicBoolean(false)
var isRecognitionActive = false val isInitialized: Boolean get() = _isInitialized.get()
private set
var isTtsSpeaking = false private val _isRecognitionActive = AtomicBoolean(false)
private set val isRecognitionActive: Boolean get() = _isRecognitionActive.get()
var hasSpeechDetected = false
private set private val _isTtsSpeaking = AtomicBoolean(false)
var isAiStreaming = false val isTtsSpeaking: Boolean get() = _isTtsSpeaking.get()
private set
private val _hasSpeechDetected = AtomicBoolean(false)
val hasSpeechDetected: Boolean get() = _hasSpeechDetected.get()
private val _isAiStreaming = AtomicBoolean(false)
val isAiStreaming: Boolean get() = _isAiStreaming.get()
// 用于保护复杂状态操作的互斥锁
private val stateMutex = Mutex()
// AI流生成相关 // AI流生成相关
private var currentAiJob: Job? = null private var currentAiJob: Job? = null
@ -175,7 +186,7 @@ object AgentService : CoroutineScope {
// 加载最近的聊天记录 // 加载最近的聊天记录
loadChatHistory() loadChatHistory()
isInitialized = true _isInitialized.set(true)
return true return true
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "初始化失败: ${e.message}") Log.e(TAG, "初始化失败: ${e.message}")
@ -201,6 +212,7 @@ object AgentService : CoroutineScope {
// 释放ChatAPI服务 // 释放ChatAPI服务
if (::chatApiService.isInitialized) { if (::chatApiService.isInitialized) {
chatApiService.cancelCurrentStream() chatApiService.cancelCurrentStream()
chatApiService.dispose()
} }
// 释放TTS服务 // 释放TTS服务
@ -209,7 +221,7 @@ object AgentService : CoroutineScope {
job.cancel() job.cancel()
clearListeners() clearListeners()
isInitialized = false _isInitialized.set(false)
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "释放资源异常: ${e.message}") Log.e(TAG, "释放资源异常: ${e.message}")
} }
@ -252,22 +264,30 @@ object AgentService : CoroutineScope {
override fun onEvent(event: TtsEvent) { override fun onEvent(event: TtsEvent) {
when (event.type) { when (event.type) {
TtsEventType.SYNTHESIS_STARTED -> { TtsEventType.SYNTHESIS_STARTED -> {
isTtsSpeaking = true _isTtsSpeaking.set(true)
restartIdleCheck() restartIdleCheck()
sendEvent("tts_started", mapOf("status" to "started")) sendEvent("tts_started", mapOf("status" to "started"))
} }
TtsEventType.SYNTHESIS_COMPLETED -> { TtsEventType.SYNTHESIS_COMPLETED -> {
isTtsSpeaking = false _isTtsSpeaking.set(false)
restartIdleCheck() restartIdleCheck()
sendEvent("tts_completed", mapOf("status" to "completed")) sendEvent("tts_completed", mapOf("status" to "completed"))
} }
TtsEventType.SYNTHESIS_CANCELED -> { TtsEventType.SYNTHESIS_CANCELED -> {
isTtsSpeaking = false _isTtsSpeaking.set(false)
restartIdleCheck() restartIdleCheck()
sendEvent("tts_canceled", mapOf("status" to "canceled")) sendEvent("tts_canceled", mapOf("status" to "canceled"))
} }
TtsEventType.PLAYBACK_STARTED -> {
restartIdleCheck()
sendEvent("playback_started", mapOf("status" to "playback_started"))
}
TtsEventType.PLAYBACK_COMPLETED -> {
restartIdleCheck()
sendEvent("playback_completed", mapOf("status" to "playback_completed"))
}
TtsEventType.ERROR -> { TtsEventType.ERROR -> {
isTtsSpeaking = false _isTtsSpeaking.set(false)
restartIdleCheck() restartIdleCheck()
val params = event.params val params = event.params
val code = params["errorCode"] as? String ?: "UNKNOWN_ERROR" val code = params["errorCode"] as? String ?: "UNKNOWN_ERROR"
@ -371,8 +391,8 @@ object AgentService : CoroutineScope {
return false return false
} }
isRecognitionActive = true _isRecognitionActive.set(true)
hasSpeechDetected = false _hasSpeechDetected.set(false)
try { try {
val audioSourceType = if (isExternalActive) { val audioSourceType = if (isExternalActive) {
@ -385,7 +405,7 @@ object AgentService : CoroutineScope {
if (recognizing.isNotEmpty()) { if (recognizing.isNotEmpty()) {
// 检测到语音,更新状态 // 检测到语音,更新状态
val previousHasSpeech = hasSpeechDetected val previousHasSpeech = hasSpeechDetected
hasSpeechDetected = true _hasSpeechDetected.set(true)
if (!previousHasSpeech) { if (!previousHasSpeech) {
restartIdleCheck() restartIdleCheck()
@ -435,7 +455,7 @@ object AgentService : CoroutineScope {
// 重置状态,继续识别 // 重置状态,继续识别
val previousHasSpeech = hasSpeechDetected val previousHasSpeech = hasSpeechDetected
hasSpeechDetected = false _hasSpeechDetected.set(false)
if (previousHasSpeech) { if (previousHasSpeech) {
restartIdleCheck() restartIdleCheck()
@ -452,13 +472,13 @@ object AgentService : CoroutineScope {
override fun onSessionStopped() { override fun onSessionStopped() {
sendEvent("recognition_stopped", mapOf("status" to "stopped")) sendEvent("recognition_stopped", mapOf("status" to "stopped"))
isRecognitionActive = false _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
audioPlayer?.playAudio(R.raw.stop) audioPlayer?.playAudio(R.raw.stop)
} }
override fun onCanceled(reason: String, errorDetails: String) { override fun onCanceled(reason: String, errorDetails: String) {
isRecognitionActive = false _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
BleService.closeCodec() BleService.closeCodec()
sendEvent("recognition_canceled", mapOf( sendEvent("recognition_canceled", mapOf(
@ -468,7 +488,7 @@ object AgentService : CoroutineScope {
} }
override fun onError(error: String) { override fun onError(error: String) {
isRecognitionActive = false _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
BleService.closeCodec() BleService.closeCodec()
Log.e(TAG, "语音识别错误: $error") Log.e(TAG, "语音识别错误: $error")
@ -480,7 +500,7 @@ object AgentService : CoroutineScope {
}, audioSourceType) }, audioSourceType)
return true return true
} catch (e: Exception) { } catch (e: Exception) {
isRecognitionActive = false _isRecognitionActive.set(false)
Log.e(TAG, "启动语音识别失败: ${e.message}") Log.e(TAG, "启动语音识别失败: ${e.message}")
sendEvent("error", mapOf( sendEvent("error", mapOf(
"code" to "RECOGNITION_START_ERROR", "code" to "RECOGNITION_START_ERROR",
@ -502,11 +522,11 @@ object AgentService : CoroutineScope {
try { try {
azureAsrHelper?.stopContinuousRecognition() azureAsrHelper?.stopContinuousRecognition()
BleService.closeCodec() BleService.closeCodec()
isRecognitionActive = false _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "停止语音识别异常: ${e.message}") Log.e(TAG, "停止语音识别异常: ${e.message}")
isRecognitionActive = false _isRecognitionActive.set(false)
stopIdleCheck() stopIdleCheck()
} }
} }
@ -520,18 +540,17 @@ object AgentService : CoroutineScope {
val startTime = System.currentTimeMillis() val startTime = System.currentTimeMillis()
// 并行执行停止操作,加快响应速度 // 并行执行停止操作,加快响应速度
launch { stopTts() } stopTts()
launch { stopAiStream() } stopAiStream()
// 记录打断耗时 // 记录打断耗时
launch {
delay(100) // 短暂延迟后计算耗时 val duration = System.currentTimeMillis() - startTime
val duration = System.currentTimeMillis() - startTime sendEvent("response_interrupted", mapOf(
sendEvent("response_interrupted", mapOf( "status" to "interrupted",
"status" to "interrupted", "duration_ms" to duration
"duration_ms" to duration ))
))
}
} }
} }
@ -542,7 +561,7 @@ object AgentService : CoroutineScope {
if (isAiStreaming) { if (isAiStreaming) {
try { try {
// 先更新状态,避免回调时的状态不一致 // 先更新状态,避免回调时的状态不一致
isAiStreaming = false _isAiStreaming.set(false)
// 取消当前AI生成任务 // 取消当前AI生成任务
currentAiJob?.cancel() currentAiJob?.cancel()
@ -554,7 +573,7 @@ object AgentService : CoroutineScope {
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "停止AI流输出异常", e) Log.e(TAG, "停止AI流输出异常", e)
// 确保状态被重置,即使发生异常 // 确保状态被重置,即使发生异常
isAiStreaming = false _isAiStreaming.set(false)
currentAiJob = null currentAiJob = null
} }
} }
@ -648,7 +667,7 @@ object AgentService : CoroutineScope {
currentAiJob = launch { currentAiJob = launch {
try { try {
// 设置状态为正在流式输出 // 设置状态为正在流式输出
isAiStreaming = true _isAiStreaming.set(true)
// 使用历史记录作为上下文发送到OpenAI // 使用历史记录作为上下文发送到OpenAI
val responseBuilder = StringBuilder() val responseBuilder = StringBuilder()
@ -746,7 +765,7 @@ object AgentService : CoroutineScope {
// 保存聊天记录 // 保存聊天记录
saveChatMessage(displayText, response,aiMetadata,userMetadata.toString()) saveChatMessage(displayText, response,aiMetadata,userMetadata.toString())
// 标记AI流式输出已完成 // 标记AI流式输出已完成
isAiStreaming = false _isAiStreaming.set(false)
currentAiJob = null currentAiJob = null
} }
@ -758,7 +777,7 @@ object AgentService : CoroutineScope {
)) ))
// 标记AI流式输出已完成 // 标记AI流式输出已完成
isAiStreaming = false _isAiStreaming.set(false)
currentAiJob = null currentAiJob = null
} }
@ -802,7 +821,7 @@ object AgentService : CoroutineScope {
)) ))
// 确保状态被重置 // 确保状态被重置
isAiStreaming = false _isAiStreaming.set(false)
currentAiJob = null currentAiJob = null
} }
} }
@ -866,7 +885,7 @@ object AgentService : CoroutineScope {
if (text.isEmpty()) return if (text.isEmpty()) return
// 更新状态 // 更新状态
isTtsSpeaking = true _isTtsSpeaking.set(true)
restartIdleCheck() // 状态变化,重启检测 restartIdleCheck() // 状态变化,重启检测
// 直接调用TTS,无需协程包装 // 直接调用TTS,无需协程包装
@ -880,7 +899,7 @@ object AgentService : CoroutineScope {
if (isTtsSpeaking) { if (isTtsSpeaking) {
ttsService?.stop() ttsService?.stop()
isTtsSpeaking = false _isTtsSpeaking.set(false)
restartIdleCheck() // 状态变化,重启检测 restartIdleCheck() // 状态变化,重启检测
sendEvent("tts_stopped", mapOf("status" to "stopped")) sendEvent("tts_stopped", mapOf("status" to "stopped"))
} }
@ -957,9 +976,8 @@ object AgentService : CoroutineScope {
* 发送事件 * 发送事件
*/ */
private fun sendEvent(eventName: String, data: Map<String, Any>) { private fun sendEvent(eventName: String, data: Map<String, Any>) {
// 使用协程确保在主线程上执行 // 切换到主线程执行监听器回调,避免 UI 更新问题
launch { launch(Dispatchers.Main) {
// 我们已在主线程上下文中启动协程,无需再切换线程
listeners.forEach { listener -> listeners.forEach { listener ->
try { try {
listener.onEvent(eventName, data) listener.onEvent(eventName, data)

8
local_plugins/agent_service/ios/agent_service/Sources/agent_service/AgentServiceImpl.swift

@ -909,6 +909,14 @@ extension AgentServiceImpl: TtsEventListener {
restartIdleCheck() restartIdleCheck()
sendEvent(name: "tts_completed", data: ["status": "completed"]) sendEvent(name: "tts_completed", data: ["status": "completed"])
case .playbackStarted:
restartIdleCheck()
sendEvent(name: "playback_started", data: ["status": "playback_started"])
case .playbackCompleted:
restartIdleCheck()
sendEvent(name: "playback_completed", data: ["status": "playback_completed"])
case .synthesisCanceled: case .synthesisCanceled:
isSpeaking = false isSpeaking = false
restartIdleCheck() restartIdleCheck()

10
local_plugins/agent_service/lib/agent_service.dart

@ -38,6 +38,12 @@ enum AgentServiceEventType {
/// TTS停止 /// TTS停止
ttsStopped, ttsStopped,
/// 播放开始
playbackStarted,
/// 播放完成
playbackCompleted,
/// 响应中断 /// 响应中断
responseInterrupted, responseInterrupted,
@ -147,6 +153,10 @@ class AgentService {
return AgentServiceEventType.ttsCanceled; return AgentServiceEventType.ttsCanceled;
case 'tts_stopped': case 'tts_stopped':
return AgentServiceEventType.ttsStopped; return AgentServiceEventType.ttsStopped;
case 'playback_started':
return AgentServiceEventType.playbackStarted;
case 'playback_completed':
return AgentServiceEventType.playbackCompleted;
case 'response_interrupted': case 'response_interrupted':
return AgentServiceEventType.responseInterrupted; return AgentServiceEventType.responseInterrupted;
case 'assistant_token': case 'assistant_token':

5
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureTtsHelper.kt

@ -232,11 +232,6 @@ class AzureTtsHelper(private val context: Context) : ITtsService {
manager.isSpeakerphoneOn = false manager.isSpeakerphoneOn = false
// 注意:Android不能强制路由到耳机,只能在耳机已连接时使用 // 注意:Android不能强制路由到耳机,只能在耳机已连接时使用
} }
AudioOutputDevice.EARPIECE -> {
// 强制使用听筒
manager.mode = AudioManager.MODE_IN_COMMUNICATION
manager.isSpeakerphoneOn = false
}
} }
} }
} }

351
local_plugins/bytedance_speech/android/src/main/kotlin/com/deep_voice/bytedance_speech/BytedanceAudioPlayer.kt

@ -1,12 +1,16 @@
package com.deep_voice.bytedance_speech package com.deep_voice.bytedance_speech
import android.content.BroadcastReceiver
import android.content.Context import android.content.Context
import android.content.Intent
import android.content.IntentFilter
import android.media.AudioAttributes import android.media.AudioAttributes
import android.media.AudioDeviceCallback
import android.media.AudioDeviceInfo
import android.media.AudioFormat import android.media.AudioFormat
import android.media.AudioManager import android.media.AudioManager
import android.media.AudioTrack import android.media.AudioTrack
import android.os.Build import android.os.Build
import android.util.Log
import com.deep_voice.speech.tts.AudioDataListener import com.deep_voice.speech.tts.AudioDataListener
import com.deep_voice.speech.tts.AudioOutputDevice import com.deep_voice.speech.tts.AudioOutputDevice
@ -20,6 +24,7 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener {
private const val SAMPLE_RATE = 24000 private const val SAMPLE_RATE = 24000
private const val CHANNEL_CONFIG = AudioFormat.CHANNEL_OUT_MONO private const val CHANNEL_CONFIG = AudioFormat.CHANNEL_OUT_MONO
private const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT private const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT
private const val VOLUME_REDUCTION = 0.7f // 降低音量以减少回音
} }
private var audioTrack: AudioTrack? = null private var audioTrack: AudioTrack? = null
@ -29,6 +34,11 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener {
private var audioOutputDevice = AudioOutputDevice.DEFAULT // 音频输出设备 private var audioOutputDevice = AudioOutputDevice.DEFAULT // 音频输出设备
private var audioManager: AudioManager? = null private var audioManager: AudioManager? = null
// 设备监听相关
private var deviceCallback: AudioDeviceCallback? = null
private var noisyReceiver: BroadcastReceiver? = null
private var isMonitoringDevices = false
// 回调 // 回调
private var onPlayStarted: (() -> Unit)? = null private var onPlayStarted: (() -> Unit)? = null
private var onPlayCompleted: (() -> Unit)? = null private var onPlayCompleted: (() -> Unit)? = null
@ -36,7 +46,6 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener {
// 播放位置监听器 // 播放位置监听器
private val playbackListener = object : AudioTrack.OnPlaybackPositionUpdateListener { private val playbackListener = object : AudioTrack.OnPlaybackPositionUpdateListener {
override fun onMarkerReached(track: AudioTrack) { override fun onMarkerReached(track: AudioTrack) {
Log.d(TAG, "播放到达标记位置: ${track.playbackHeadPosition}")
onPlayCompleted?.invoke() onPlayCompleted?.invoke()
} }
@ -46,7 +55,6 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener {
} }
init { init {
Log.d(TAG, "BytedanceAudioPlayer 初始化")
audioManager = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager audioManager = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager
initAudioTrack() initAudioTrack()
} }
@ -56,24 +64,19 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener {
* 开始新的播放会话 * 开始新的播放会话
*/ */
fun startSession() { fun startSession() {
Log.d(TAG, "开始新会话")
// 重置状态
isFirstData = true isFirstData = true
totalBytesWritten = 0 totalBytesWritten = 0
sessionActive = true sessionActive = true
// 重置 AudioTrack
audioTrack?.let { track -> audioTrack?.let { track ->
if (track.state == AudioTrack.STATE_INITIALIZED) { if (track.state == AudioTrack.STATE_INITIALIZED) {
// 停止并清空缓冲区
track.pause() track.pause()
track.flush() track.flush()
// 重新开始播放
track.play() track.play()
Log.d(TAG, "AudioTrack 已重置")
} }
} ?: initAudioTrack() // 如果没有初始化,则初始化 } ?: initAudioTrack()
startDeviceMonitoring()
} }
/** /**
@ -82,28 +85,20 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener {
fun endSession() { fun endSession() {
if (!sessionActive) return if (!sessionActive) return
Log.d(TAG, "数据流结束,总共写入字节数: $totalBytesWritten")
sessionActive = false sessionActive = false
audioTrack?.let { track -> audioTrack?.let { track ->
if (totalBytesWritten > 0) { if (totalBytesWritten > 0) {
// 计算总帧数(16-bit 单声道,每帧2字节)
val totalFrames = totalBytesWritten / 2 val totalFrames = totalBytesWritten / 2
// 设置标记位置
try { try {
track.setNotificationMarkerPosition(totalFrames) track.setNotificationMarkerPosition(totalFrames)
Log.d(TAG, "设置播放完成标记位置: $totalFrames")
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "设置标记失败: ${e.message}")
// 设置失败时,使用延迟触发作为后备
val durationMs = (totalFrames * 1000L) / SAMPLE_RATE val durationMs = (totalFrames * 1000L) / SAMPLE_RATE
android.os.Handler(android.os.Looper.getMainLooper()).postDelayed({ android.os.Handler(android.os.Looper.getMainLooper()).postDelayed({
onPlayCompleted?.invoke() onPlayCompleted?.invoke()
}, durationMs + 500) }, durationMs + 500)
} }
} else { } else {
// 没有数据,直接触发完成
onPlayCompleted?.invoke() onPlayCompleted?.invoke()
} }
} }
@ -113,37 +108,26 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener {
* 接收音频数据 * 接收音频数据
*/ */
override fun onAudioData(data: ByteArray) { override fun onAudioData(data: ByteArray) {
if (data.isEmpty()) return if (data.isEmpty() || !sessionActive) return
if (!sessionActive) { if (audioTrack?.state != AudioTrack.STATE_INITIALIZED) return
Log.w(TAG, "收到音频数据但会话未激活,忽略数据")
return
}
if (audioTrack == null || audioTrack?.state != AudioTrack.STATE_INITIALIZED) {
Log.w(TAG, "收到音频数据但 AudioTrack 未初始化或状态不正确,忽略数据")
return
}
try { try {
audioTrack?.let { track -> audioTrack?.let { track ->
if (track.state == AudioTrack.STATE_INITIALIZED) { val bytesWritten = track.write(data, 0, data.size)
val bytesWritten = track.write(data, 0, data.size)
if (bytesWritten > 0) {
if (isFirstData) {
isFirstData = false
onPlayStarted?.invoke()
}
if (bytesWritten > 0) { if (sessionActive) {
// 第一次写入数据时自动触发开始回调 totalBytesWritten += bytesWritten
if (isFirstData) {
isFirstData = false
onPlayStarted?.invoke()
Log.d(TAG, "播放开始")
}
// 累计写入字节数
if (sessionActive) {
totalBytesWritten += bytesWritten
}
} }
} }
} }
} catch (e: Exception) { } catch (e: Exception) {
Log.e(TAG, "写入音频数据失败: ${e.message}") // 忽略写入失败
} }
} }
@ -151,8 +135,6 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener {
* 停止播放 * 停止播放
*/ */
fun stop() { fun stop() {
Log.d(TAG, "停止播放")
sessionActive = false sessionActive = false
audioTrack?.let { track -> audioTrack?.let { track ->
@ -162,7 +144,8 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener {
} }
} }
// 如果已经开始播放,触发完成回调 stopDeviceMonitoring()
if (!isFirstData) { if (!isFirstData) {
onPlayCompleted?.invoke() onPlayCompleted?.invoke()
} }
@ -172,12 +155,10 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener {
* 释放资源 * 释放资源
*/ */
fun release() { fun release() {
Log.d(TAG, "释放资源")
stop() stop()
audioTrack?.release() audioTrack?.release()
audioTrack = null audioTrack = null
stopDeviceMonitoring()
} }
/** /**
@ -214,58 +195,41 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener {
fun setAudioOutputDevice(device: AudioOutputDevice) { fun setAudioOutputDevice(device: AudioOutputDevice) {
if (audioOutputDevice != device) { if (audioOutputDevice != device) {
audioOutputDevice = device audioOutputDevice = device
// 如果AudioTrack已初始化,需要重新创建以应用新的输出设备设置
if (audioTrack != null) { // 应用新的音频路由设置
val wasPlaying = isPlaying() if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
initAudioTrack() // API 31+ 可以动态更改设备
if (wasPlaying) { setPreferredDeviceForTrack()
audioTrack?.play() } else {
} // API 30 及以下需要重新配置
configureAudioRouting()
} }
} }
} }
/** /**
* 初始化 AudioTrack * 初始化 AudioTrack
*/ */
private fun initAudioTrack() { private fun initAudioTrack() {
// 释放旧实例
audioTrack?.release() audioTrack?.release()
// 计算缓冲区大小
val minBufferSize = AudioTrack.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT) val minBufferSize = AudioTrack.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT)
val bufferSize = minBufferSize * 2 val bufferSize = minBufferSize * 2 // 使用较小的缓冲区以降低延迟
// 根据输出设备配置AudioAttributes audioTrack = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
val audioAttributes = when (audioOutputDevice) { val builder = AudioTrack.Builder()
AudioOutputDevice.EARPIECE -> { .setAudioAttributes(
// 听筒模式
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
AudioAttributes.Builder()
.setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION)
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
.build()
} else {
null
}
}
else -> {
// 默认、耳机、扬声器模式
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
AudioAttributes.Builder() AudioAttributes.Builder()
.setUsage(AudioAttributes.USAGE_MEDIA) .setUsage(
// 扬声器模式下默认使用语音通信模式以启用回声消除
if (audioOutputDevice == AudioOutputDevice.SPEAKER)
AudioAttributes.USAGE_VOICE_COMMUNICATION
else AudioAttributes.USAGE_MEDIA
)
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
.build() .build()
} else { )
null
}
}
}
// 创建 AudioTrack
audioTrack = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && audioAttributes != null) {
AudioTrack.Builder()
.setAudioAttributes(audioAttributes)
.setAudioFormat( .setAudioFormat(
AudioFormat.Builder() AudioFormat.Builder()
.setEncoding(AUDIO_FORMAT) .setEncoding(AUDIO_FORMAT)
@ -275,15 +239,17 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener {
) )
.setBufferSizeInBytes(bufferSize) .setBufferSizeInBytes(bufferSize)
.setTransferMode(AudioTrack.MODE_STREAM) .setTransferMode(AudioTrack.MODE_STREAM)
.build()
// API 26+ 设置低延迟模式
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
builder.setPerformanceMode(AudioTrack.PERFORMANCE_MODE_LOW_LATENCY)
}
builder.build()
} else { } else {
@Suppress("DEPRECATION") @Suppress("DEPRECATION")
val streamType = when (audioOutputDevice) {
AudioOutputDevice.EARPIECE -> AudioManager.STREAM_VOICE_CALL
else -> AudioManager.STREAM_MUSIC
}
AudioTrack( AudioTrack(
streamType, AudioManager.STREAM_MUSIC,
SAMPLE_RATE, SAMPLE_RATE,
CHANNEL_CONFIG, CHANNEL_CONFIG,
AUDIO_FORMAT, AUDIO_FORMAT,
@ -292,46 +258,215 @@ class BytedanceAudioPlayer(private val context: Context) : AudioDataListener {
) )
} }
// 设置播放位置监听器
audioTrack?.setPlaybackPositionUpdateListener(playbackListener) audioTrack?.setPlaybackPositionUpdateListener(playbackListener)
// 配置音频路由 // 设置音量以减少回音
configureAudioRouting() audioTrack?.setVolume(VOLUME_REDUCTION)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
setPreferredDeviceForTrack()
} else {
configureAudioRouting()
}
// 开始播放
audioTrack?.play() audioTrack?.play()
}
/**
* API 31+ 使用 setPreferredDevice 设置音频输出设备
*/
private fun setPreferredDeviceForTrack() {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) return
Log.d(TAG, "AudioTrack 初始化成功") audioManager?.let { manager ->
audioTrack?.let { track ->
when (audioOutputDevice) {
AudioOutputDevice.DEFAULT -> {
track.setPreferredDevice(null)
manager.isSpeakerphoneOn = false
// 默认模式下使用通信模式以获得更好的音频处理
manager.mode = AudioManager.MODE_IN_COMMUNICATION
}
AudioOutputDevice.SPEAKER -> {
val speaker = manager.getDevices(AudioManager.GET_DEVICES_OUTPUTS)
.firstOrNull { it.type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER }
speaker?.let { track.setPreferredDevice(it) }
// 扬声器模式下始终启用通信模式以获得回声消除
manager.mode = AudioManager.MODE_IN_COMMUNICATION
manager.isSpeakerphoneOn = true
}
AudioOutputDevice.HEADPHONES -> {
val headphones = manager.getDevices(AudioManager.GET_DEVICES_OUTPUTS)
.firstOrNull { device ->
device.type == AudioDeviceInfo.TYPE_WIRED_HEADSET ||
device.type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES ||
device.type == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP ||
device.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO
}
headphones?.let { track.setPreferredDevice(it) }
?: track.setPreferredDevice(null)
// 耳机模式下可以使用普通模式
manager.mode = AudioManager.MODE_NORMAL
manager.isSpeakerphoneOn = false
}
}
}
}
} }
/** /**
* 配置音频路由 * 配置音频路由(API 30 及以下)
*/ */
private fun configureAudioRouting() { private fun configureAudioRouting() {
audioManager?.let { manager -> audioManager?.let { manager ->
when (audioOutputDevice) { when (audioOutputDevice) {
AudioOutputDevice.DEFAULT -> { AudioOutputDevice.DEFAULT -> {
// 默认模式:系统自动选择 // 默认模式下使用通信模式以获得更好的音频处理
manager.mode = AudioManager.MODE_NORMAL manager.mode = AudioManager.MODE_IN_COMMUNICATION
manager.isSpeakerphoneOn = false manager.isSpeakerphoneOn = false
} }
AudioOutputDevice.SPEAKER -> { AudioOutputDevice.SPEAKER -> {
// 强制使用扬声器 // 扬声器模式下始终启用通信模式以获得回声消除
manager.mode = AudioManager.MODE_NORMAL manager.mode = AudioManager.MODE_IN_COMMUNICATION
manager.isSpeakerphoneOn = true manager.isSpeakerphoneOn = true
} }
AudioOutputDevice.HEADPHONES -> { AudioOutputDevice.HEADPHONES -> {
// 强制使用耳机(如果已连接) // 耳机模式下可以使用普通模式
manager.mode = AudioManager.MODE_NORMAL manager.mode = AudioManager.MODE_NORMAL
manager.isSpeakerphoneOn = false manager.isSpeakerphoneOn = false
// 注意:Android不能强制路由到耳机,只能在耳机已连接时使用
} }
AudioOutputDevice.EARPIECE -> { }
// 强制使用听筒 }
manager.mode = AudioManager.MODE_IN_COMMUNICATION }
manager.isSpeakerphoneOn = false
/**
* 开始监听设备变化
*/
private fun startDeviceMonitoring() {
if (isMonitoringDevices) return
// API 23+ 使用 AudioDeviceCallback
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
deviceCallback = object : AudioDeviceCallback() {
override fun onAudioDevicesAdded(addedDevices: Array<AudioDeviceInfo>) {
super.onAudioDevicesAdded(addedDevices)
handleDeviceAdded(addedDevices)
}
override fun onAudioDevicesRemoved(removedDevices: Array<AudioDeviceInfo>) {
super.onAudioDevicesRemoved(removedDevices)
handleDeviceRemoved(removedDevices)
} }
} }
audioManager?.registerAudioDeviceCallback(deviceCallback, null)
}
// 所有版本都监听 NOISY 广播(耳机拔出)
noisyReceiver = object : BroadcastReceiver() {
override fun onReceive(context: Context, intent: Intent) {
if (intent.action == AudioManager.ACTION_AUDIO_BECOMING_NOISY) {
handleNoisyAudioEvent()
}
}
}
context.registerReceiver(noisyReceiver, IntentFilter(AudioManager.ACTION_AUDIO_BECOMING_NOISY))
isMonitoringDevices = true
}
/**
* 停止监听设备变化
*/
private fun stopDeviceMonitoring() {
if (!isMonitoringDevices) return
// 注销 AudioDeviceCallback
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
deviceCallback?.let {
audioManager?.unregisterAudioDeviceCallback(it)
}
deviceCallback = null
}
// 注销广播接收器
noisyReceiver?.let {
try {
context.unregisterReceiver(it)
} catch (e: Exception) {
// 忽略已注销的异常
}
}
noisyReceiver = null
isMonitoringDevices = false
}
/**
* 处理设备添加
*/
private fun handleDeviceAdded(devices: Array<AudioDeviceInfo>) {
val hasHeadphones = devices.any { device ->
device.type == AudioDeviceInfo.TYPE_WIRED_HEADSET ||
device.type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES ||
device.type == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP ||
device.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO
}
if (hasHeadphones) {
switchToDefaultMode()
}
}
/**
* 处理设备移除
*/
private fun handleDeviceRemoved(devices: Array<AudioDeviceInfo>) {
val hasHeadphones = devices.any { device ->
device.type == AudioDeviceInfo.TYPE_WIRED_HEADSET ||
device.type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES ||
device.type == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP ||
device.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO
}
if (hasHeadphones) {
switchToSpeakerMode()
}
}
/**
* 处理音频变得嘈杂事件(通常是耳机拔出)
*/
private fun handleNoisyAudioEvent() {
switchToSpeakerMode()
}
/**
* 切换到 DEFAULT 模式
*/
private fun switchToDefaultMode() {
audioOutputDevice = AudioOutputDevice.DEFAULT
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
setPreferredDeviceForTrack()
} else {
configureAudioRouting()
}
}
/**
* 切换到扬声器模式
*/
private fun switchToSpeakerMode() {
audioOutputDevice = AudioOutputDevice.SPEAKER
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
setPreferredDeviceForTrack()
} else {
configureAudioRouting()
} }
} }
} }

44
local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/ChatApiService.kt

@ -111,6 +111,10 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
override val coroutineContext: CoroutineContext = Dispatchers.IO + SupervisorJob() override val coroutineContext: CoroutineContext = Dispatchers.IO + SupervisorJob()
companion object {
private const val TAG = "ChatApiService"
}
// MARK: - 属性 // MARK: - 属性
private var baseUrl = "https://api.openai.com/v1/" private var baseUrl = "https://api.openai.com/v1/"
private var apiKey = "" private var apiKey = ""
@ -219,6 +223,7 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
// 异步初始化MCP客户端 // 异步初始化MCP客户端
launch { launch {
initializeMcpClient(mcpServer) initializeMcpClient(mcpServer)
// initializeMcpClient("{}")
} }
isInitialized = apiKey.isNotEmpty() isInitialized = apiKey.isNotEmpty()
@ -513,8 +518,8 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
) )
// 通知上层工具调用事件 // 通知上层工具调用事件
currSessionCallback?.onFunctionCall(convertMapToJsonObject(functionCall)) currSessionCallback?.onFunctionCall(convertMapToJsonObject(functionCall))
// 在后台队列处理工具调用 // 在当前协程作用域内处理工具调用,使用async确保生命周期管理
launch { val toolCallDeferred = async {
try { try {
if (sessionid == currSessionId) { if (sessionid == currSessionId) {
// 通过MCP客户端处理工具调用 // 通过MCP客户端处理工具调用
@ -586,6 +591,15 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
} }
} }
// 等待工具调用完成,确保生命周期管理
try {
toolCallDeferred.await()
} catch (e: CancellationException) {
// 协程被取消,确保子任务也被取消
toolCallDeferred.cancel()
throw e
}
return true return true
} }
@ -779,7 +793,7 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
description = description, description = description,
parameters = parameters parameters = parameters
) )
Log.d("ChatApiService", "AI携带工具: $name, 参数定义: $parameters") // Log.d("ChatApiService", "AI携带工具: $name, 参数定义: $parameters")
tools.add(tool) tools.add(tool)
} catch (e: Exception) { } catch (e: Exception) {
@ -812,6 +826,30 @@ class ChatApiService(private val context: android.content.Context? = null) : Cor
} }
} }
/**
* 释放所有资源
* 统一的资源管理方法,确保所有协程被正确取消
*/
fun dispose() {
runBlocking {
// 取消当前流式任务
currentStreamJob?.cancelAndJoin()
currentStreamJob = null
// 关闭MCP客户端
closeMcpClient()
// 取消所有子协程
coroutineContext[Job]?.cancelChildren()
// 清理其他资源
currSessionCallback = null
currSessionId = ""
currentMessages = emptyList()
toolCalls.clear()
}
}
/** /**
* 处理MCP工具调用 * 处理MCP工具调用
* *

4
local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/MCPSubClient.kt

@ -129,7 +129,7 @@ class MCPSubClient(
"properties" to emptyMap<String, Any>(), "properties" to emptyMap<String, Any>(),
"required" to emptyList<String>() "required" to emptyList<String>()
) )
Log.w(TAG, "解析工具数据: ${tool.name} ${parametersMap}") // Log.w(TAG, "解析工具数据: ${tool.name} ${parametersMap}")
val toolMap = mapOf( val toolMap = mapOf(
"type" to "function", "type" to "function",
"function" to mapOf( "function" to mapOf(
@ -175,7 +175,7 @@ class MCPSubClient(
name = name, name = name,
arguments = argumentsJson arguments = argumentsJson
) )
Log.d(TAG, "[$serverId] 调用工具 '$name',参数: ${argumentsJson.toString()}")
// 调用工具 // 调用工具
val result = mcpClient?.callTool(request) val result = mcpClient?.callTool(request)

22
local_plugins/chat_api/android/src/main/kotlin/com/yunqiinnovation/chat_api/SystemFunctionHandler.kt

@ -47,17 +47,17 @@ class SystemFunctionHandler(private val context: Context? = null) {
handler = ExitInteractionHandler(context) handler = ExitInteractionHandler(context)
) )
// 注册翻译模式函数 // // 注册翻译模式函数
client.registerLocalFunction( // client.registerLocalFunction(
name = "enter_translation_mode", // name = "enter_translation_mode",
description = "用户请求进入实时翻译模式时,启动实时翻译功能", // description = "用户请求进入实时翻译模式时,启动实时翻译功能",
parameters = mapOf( // parameters = mapOf(
"type" to "object", // "type" to "object",
"properties" to emptyMap<String, Any>(), // "properties" to emptyMap<String, Any>(),
"required" to emptyList<String>() // "required" to emptyList<String>()
), // ),
handler = TranslationModeHandler(context) // handler = TranslationModeHandler(context)
) // )
// 注册发送短信函数 // 注册发送短信函数
client.registerLocalFunction( client.registerLocalFunction(

1
local_plugins/speech/android/src/main/kotlin/com/deep_voice/speech/TtsEvents.kt

@ -67,5 +67,4 @@ enum class AudioOutputDevice {
DEFAULT, // 默认(如果有耳机选耳机,否则使用系统扬声器) DEFAULT, // 默认(如果有耳机选耳机,否则使用系统扬声器)
HEADPHONES, // 强制使用耳机 HEADPHONES, // 强制使用耳机
SPEAKER, // 强制使用扬声器 SPEAKER, // 强制使用扬声器
EARPIECE // 强制使用听筒
} }
Loading…
Cancel
Save