Browse Source

Merge commit '7599cdb573aab513017cd98ca6802b9d1db12d01' into lxm

weicu
lxm 1 year ago
parent
commit
93bca955c0
  1. 17
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt
  2. 3
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt
  3. 18
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt
  4. 2
      local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleWakeReceiver.kt
  5. 16
      local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt
  6. 3
      local_plugins/ble_service/android/build.gradle.kts
  7. 154
      local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt
  8. 2
      local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt
  9. BIN
      local_plugins/opus/.gradle/8.10/fileHashes/fileHashes.lock
  10. BIN
      local_plugins/opus/.gradle/buildOutputCleanup/buildOutputCleanup.lock
  11. 2
      local_plugins/opus/.gradle/buildOutputCleanup/cache.properties

17
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/AgentService.kt

@ -50,7 +50,7 @@ object AgentService : CoroutineScope {
private var openaiModel: String = "gpt-3.5-turbo"
// Azure服务
var azureAsrHelper: AzureAsrHelper? = null
private var azureAsrHelper: AzureAsrHelper? = null
private var azureTtsHelper: AzureTtsHelper? = null
// OpenAI服务
@ -288,11 +288,11 @@ object AgentService : CoroutineScope {
// 检查录音权限
if (!isExternalActive||!checkRecordAudioPermission()) {
FileLogger.e(TAG, "需要录音权限,请在设置中授予权限")
sendEvent("error", mapOf("code" to "PERMISSION_DENIED", "message" to "需要录音权限"))
return false
}
// if (!isExternalActive||!checkRecordAudioPermission()) {
// FileLogger.e(TAG, "需要录音权限,请在设置中授予权限")
// sendEvent("error", mapOf("code" to "PERMISSION_DENIED", "message" to "需要录音权限"))
// return false
// }
isRecognitionActive = true
hasSpeechDetected = false
@ -392,6 +392,9 @@ object AgentService : CoroutineScope {
}
}
fun pushAudioData(data: ByteArray) {
azureAsrHelper?.pushAudioData(data)
}
/**
* 停止语音识别
*/
@ -401,8 +404,8 @@ object AgentService : CoroutineScope {
FileLogger.d(TAG, "停止语音识别")
try {
BleService.closeCodec()
azureAsrHelper?.stopContinuousRecognition()
BleService.closeCodec()
isRecognitionActive = false
stopIdleCheck()
FileLogger.d(TAG, "语音识别已停止")

3
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleAgent.kt

@ -128,7 +128,8 @@ object BleAgent : BleService.Callback, AgentServiceListener {
}
override fun onAudioDataReceived(data: ByteArray) {
AgentService.azureAsrHelper?.pushAudioData(data)
// FileLogger.d(TAG, "onAudioDataReceived, data: ${data.size}")
AgentService.pushAudioData(data)
// 可选:处理音频数据
}

18
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleMediaService.kt

@ -353,15 +353,15 @@ class BleMediaService : Service(), CoroutineScope {
* 包含防抖逻辑,避免短时间内重复触发
*/
private fun handleMediaAction() {
FileLogger.i(TAG, "handleMediaAction")
// 防抖处理逻辑
val currentTime = System.currentTimeMillis()
if (currentTime - lastActionTime >= ACTION_DEBOUNCE_MS) {
lastActionTime = currentTime
BleAgent.onWakeupSignalReceived()
} else {
FileLogger.i(TAG, "媒体操作防抖: 忽略${ACTION_DEBOUNCE_MS}ms内的重复请求")
}
// FileLogger.i(TAG, "handleMediaAction")
// // 防抖处理逻辑
// val currentTime = System.currentTimeMillis()
// if (currentTime - lastActionTime >= ACTION_DEBOUNCE_MS) {
// lastActionTime = currentTime
// BleAgent.onWakeupSignalReceived()
// } else {
// FileLogger.i(TAG, "媒体操作防抖: 忽略${ACTION_DEBOUNCE_MS}ms内的重复请求")
// }
}

2
local_plugins/agent_service/android/src/main/kotlin/com/yunqiinnovation/agent_service/BleWakeReceiver.kt

@ -47,7 +47,7 @@ class BleWakeReceiver : BroadcastReceiver() {
if(!BleMediaService.isServiceRunning()) {
val am = ctx.getSystemService(Context.AUDIO_SERVICE) as AudioManager
val keyEvent = KeyEvent(KeyEvent.ACTION_DOWN, KeyEvent.KEYCODE_MEDIA_PLAY_PAUSE)
val keyEvent = KeyEvent(KeyEvent.ACTION_DOWN, KeyEvent.KEYCODE_MEDIA_AUDIO_TRACK)
am.dispatchMediaKeyEvent(keyEvent)
// val mediaIntent = Intent(Intent.ACTION_MEDIA_BUTTON)
// mediaIntent.putExtra(Intent.EXTRA_KEY_EVENT, keyEvent)

16
local_plugins/azure_speech/android/src/main/kotlin/com/yunqiinnovation/azure_speech/AzureAsrHelper.kt

@ -23,7 +23,7 @@ import com.microsoft.cognitiveservices.speech.audio.PullAudioInputStreamCallback
import com.microsoft.cognitiveservices.speech.audio.AudioStreamFormat
import java.util.concurrent.BlockingQueue
import java.util.concurrent.LinkedBlockingQueue
import java.util.concurrent.TimeUnit
/**
* Azure语音识别辅助类,支持麦克风和外部音频源的单次和连续语音识别
*
@ -122,7 +122,7 @@ class AzureAsrHelper(private val context: Context) {
}
// SetProperty(PropertyId.Speech_SegmentationStrategy, "Semantic");
}
// 创建识别器
return setupRecognizer()
} catch (e: Exception) {
@ -308,7 +308,7 @@ class AzureAsrHelper(private val context: Context) {
if (isContinuousRecognitionActive) {
return true
}
this.audioSourceType = audioSourceType
this.audioSourceType = audioSourceType
// 重置识别器
if (!setupRecognizer()) {
callback.onError("重置识别器失败")
@ -377,11 +377,12 @@ class AzureAsrHelper(private val context: Context) {
recognizer?.sessionStopped?.addEventListener(
EventHandler<SessionEventArgs> { _, _ ->
FileLogger.d(tag, "识别会话已结束")
// 直接在当前线程调用回调
callback.onSessionStopped()
isContinuousRecognitionActive = false
stopAudioProcessing()
// 直接在当前线程调用回调
callback.onSessionStopped()
}
)
@ -430,9 +431,12 @@ class AzureAsrHelper(private val context: Context) {
isContinuousRecognitionActive = false
return true
}
if (audioSourceType == AudioSourceType.EXTERNAL) {
pushAudioData(ByteArray(0))
}
// 直接停止连续识别(SDK内部已是异步操作)
recognizer?.stopContinuousRecognitionAsync()
recognizer?.stopContinuousRecognitionAsync()?.get(1000, TimeUnit.MILLISECONDS)
// 停止音频处理
stopAudioProcessing()

3
local_plugins/ble_service/android/build.gradle.kts

@ -28,7 +28,8 @@ android {
dependencies {
implementation("androidx.core:core-ktx:1.13.1")
// 使用从本地Maven仓库引用的AAR依赖
implementation("com.jieli.audio:jl_audio_decode:2.0.0")
implementation(project(":deepvoice_utils"))
implementation(project(":azure_speech"))
implementation(project(":jl_opus"))
}

154
local_plugins/ble_service/android/src/main/kotlin/com/yunqiinnovation/ble_service/BleService.kt

@ -23,7 +23,12 @@ import androidx.core.content.ContextCompat
import java.util.concurrent.ConcurrentHashMap
import android.content.pm.ServiceInfo
import java.util.concurrent.CopyOnWriteArrayList
import com.yunqiinnovation.jl_opus.JlOpusPlugin
// Import Jieli Opus SDK classes
import com.jieli.jl_audio_decode.opus.model.OpusOption
import com.jieli.jl_audio_decode.callback.OnDecodeStreamCallback
import com.jieli.jl_audio_decode.callback.OnStateCallback
import com.jieli.jl_audio_decode.opus.OpusManager
import com.jieli.jl_audio_decode.exceptions.OpusException
/**
* BLE服务类:提供蓝牙低功耗设备的扫描、连接和通信功能
@ -86,8 +91,10 @@ object BleService {
// 回调集合 - 使用线程安全的集合存储所有回调
private val callbacks = CopyOnWriteArrayList<Callback>()
// 获取插件实例
private val opusPlugin = JlOpusPlugin()
// Opus解码器实例
private var opusManager: OpusManager? = null
private val mainHandler = Handler(Looper.getMainLooper())
// 初始化状态
private var isInitialized = false
@ -112,6 +119,16 @@ object BleService {
// 初始化Handler
scanHandler = Handler(Looper.getMainLooper())
// 初始化OpusManager
try {
opusManager = OpusManager()
FileLogger.d(TAG, "OpusManager初始化成功")
} catch (e: OpusException) {
FileLogger.e(TAG, "OpusManager初始化失败: ${e.message}", e)
// 根据需要决定是否因为Opus初始化失败而返回false
}
isInitialized = true
FileLogger.d(TAG, "BLE服务初始化成功")
return true
@ -424,13 +441,7 @@ object BleService {
// 设置主服务的通知
setupNotifications(g, notifyChar)
//开启解码
opusPlugin?.startOpusDecodeStream(false, 1, 16000, 40, object :
JlOpusPlugin.CallbackOpus {
override fun onAudioDataReceivedOpus(pcmData: ByteArray) {
// 回调给上层
notifyAudioDataReceived(pcmData)
}
})
}
override fun onCharacteristicChanged(g: BluetoothGatt, c: BluetoothGattCharacteristic) {
@ -512,18 +523,21 @@ object BleService {
/**
* 处理音频数据
* @param data 音频数据
* @param data 音频数据 (Opus)
*/
private fun processAudioData(data: ByteArray) {
try {
opusPlugin?.handleDecodeRequest(data)
if (opusManager?.isDecodeStream == true) {
opusManager?.writeAudioStream(data)
} else {
// FileLogger.d(TAG, "Opus解码流未启动,忽略音频数据")
}
} catch (e: Exception) {
FileLogger.e(TAG, "处理音频数据异常: ${e.message}", e)
}
}
/**
* 处理设备响应数据 - 帧头为0xBB
*/
@ -841,7 +855,7 @@ object BleService {
BleConst.CMD_WAKE_UP -> {
FileLogger.i(TAG, "收到唤醒指令")
// 接收到唤醒信号,打开编码器
// 接收到唤醒信号,打开编码器 (设备侧)
openEncoder()
notifyWakeupSignalReceived()
@ -862,6 +876,75 @@ object BleService {
}
}
// ======================================================================================================
// Opus 解码相关
// ======================================================================================================
private fun startOpusStreamDecoding(
hasHeader: Boolean = false, // 通常BLE传输的Opus没有文件头
channel: Int = 1,
sampleRate: Int = 16000, // 确认设备端Opus编码采样率
packetSize: Int = 40 // 确认设备端Opus编码帧长
): Boolean {
if (opusManager == null) {
FileLogger.e(TAG, "OpusManager未初始化,无法开始解码")
return false
}
// 如果已经在解码流,先停止
if (opusManager?.isDecodeStream == true) {
opusManager?.stopDecodeStream()
FileLogger.d(TAG, "已停止正在进行的Opus解码流")
}
val option = OpusOption()
.setHasHead(hasHeader)
.setChannel(channel)
.setSampleRate(sampleRate)
.setPacketSize(packetSize)
FileLogger.d(TAG, "准备开始Opus数据流解码, 参数: $option")
var streamStartedSuccessfully = false
opusManager?.startDecodeStream(option, object : OnDecodeStreamCallback {
override fun onDecodeStream(data: ByteArray?) {
if (data != null) {
// FileLogger.d(TAG, "Opus解码数据: ${data.size} bytes")
mainHandler.post { notifyAudioDataReceived(data) }
}
}
override fun onStart() {
streamStartedSuccessfully = true
FileLogger.i(TAG, "Opus数据流解码已开始")
// 可以通过回调通知上层解码已开始
}
override fun onComplete(outPath: String?) {
FileLogger.i(TAG, "Opus数据流解码完成: $outPath (通常流式解码不会调用此方法)")
}
override fun onError(code: Int, message: String?) {
FileLogger.e(TAG, "Opus数据流解码错误: [$code] $message")
// 可以通过回调通知上层解码错误
}
})
// 注意:startDecodeStream是异步启动的,这里的返回值可能不准确反映是否真的成功启动。
// onStart回调才是真正启动成功的标志。为了简单,我们暂时认为调用即尝试启动。
// 后续可以改造为返回Future或通过回调传递启动结果。
return true // 暂定为调用即成功
}
private fun stopOpusStreamDecoding(): Boolean {
if (opusManager?.isDecodeStream == true) {
opusManager?.stopDecodeStream()
FileLogger.i(TAG, "已停止Opus数据流解码")
return true
}
FileLogger.d(TAG, "Opus数据流未在解码或OpusManager未初始化")
return false
}
// ======================================================================================================
// 公开的命令接口
// ======================================================================================================
@ -902,11 +985,13 @@ object BleService {
*/
fun closeCodec(): Boolean {
FileLogger.i(TAG, "关闭编解码...")
stopOpusStreamDecoding() // 停止Opus解码流
return sendCommand(
BleConst.CMD_CONTROL_CODEC.toByte(),
byteArrayOf(BleConst.CODEC_CONTROL_CLOSE.toByte(),
BleConst.AUDIO_CHANNEL_RIGHT.toByte())
BleConst.AUDIO_CHANNEL_RIGHT.toByte()) // 或其他默认声道
)
}
@ -915,10 +1000,7 @@ object BleService {
*/
fun openDecoder(): Boolean {
FileLogger.i(TAG, "打开解码...")
return sendCommand(
BleConst.CMD_CONTROL_CODEC.toByte(),
byteArrayOf(BleConst.CODEC_CONTROL_DECODE_ON.toByte())
)
return sendCommand(BleConst.CMD_CONTROL_CODEC.toByte(), byteArrayOf(BleConst.CODEC_CONTROL_DECODE_ON.toByte()))
}
/**
@ -933,16 +1015,33 @@ object BleService {
* 控制编解码 - 打开编码
*/
fun openEncoder(): Boolean {
FileLogger.i(TAG, "打开编码...")
FileLogger.i(TAG, "打开编码 (设备侧)...")
val opusStarted = startOpusStreamDecoding(
hasHeader = false, // BLE传输的Opus数据通常无文件头
channel = 1, // 单声道
sampleRate = 16000,// Opus推荐16kHz或更高
packetSize = 40 // Opus常用帧长,如20ms, 40ms, 60ms。 (16000Hz * 1channel * 2bytes/sample * 0.04s = 1280 bytes/frame if PCM)
// packetSize for Opus is typically in ms, e.g. 20, 40, 60.
// The library might expect this in a different unit or derive it.
// Given JlOpusPlugin.kt, packetSize refers to Opus encoded frame duration in ms.
)
// 注意:此方法通常是控制设备端开始编码并发送数据。
// App侧通常是打开解码器(openDecoder)来接收和处理这些数据。
// 如果App也需要编码(例如发送语音到设备),则需要实现Opus编码逻辑。
// 目前仅发送命令通知设备开始编码。
return sendCommand(
BleConst.CMD_CONTROL_CODEC.toByte(),
byteArrayOf(
BleConst.CODEC_CONTROL_ENCODE_ON.toByte(),
BleConst.AUDIO_CHANNEL_RIGHT.toByte()
BleConst.AUDIO_CHANNEL_RIGHT.toByte() // 或设备期望的声道
)
)
}
/**
* 设置声道模式
* @param channelMode 声道模式:AUDIO_CHANNEL_LEFT/AUDIO_CHANNEL_RIGHT/AUDIO_CHANNEL_STEREO
@ -1026,8 +1125,19 @@ object BleService {
stopScan()
scanHandler.removeCallbacksAndMessages(null)
disconnectGatt()
// 释放OpusManager
opusManager?.let {
if (it.isDecodeStream) {
it.stopDecodeStream()
}
it.release()
}
opusManager = null
FileLogger.d(TAG, "OpusManager已释放")
isInitialized = false // 标记为未初始化
}
// ======================================================================================================
// 协议命令实现
// ======================================================================================================

2
local_plugins/open_ai_service/android/src/main/kotlin/com/yunqiinnovation/open_ai_service/OpenAIService.kt

@ -609,7 +609,7 @@ class OpenAIService(private val context: Context? = null) {
for (i in 0 until toolCalls.length()) {
val toolCall = toolCalls.getJSONObject(i)
val index = toolCall.getInt("index")
Log.e(TAG, "解析道AI需要调用工具: $toolCall")
// Log.e(TAG, "解析道AI需要调用工具: $toolCall")
// 创建或获取现有的工具调用信息
val toolCallInfo = finalToolCalls.getOrPut(index) { ToolCallInfo() }

BIN
local_plugins/opus/.gradle/8.10/fileHashes/fileHashes.lock

Binary file not shown.

BIN
local_plugins/opus/.gradle/buildOutputCleanup/buildOutputCleanup.lock

Binary file not shown.

2
local_plugins/opus/.gradle/buildOutputCleanup/cache.properties

@ -1,2 +1,2 @@
#Fri May 02 14:33:07 IST 2025
#Fri May 09 15:56:53 IST 2025
gradle.version=8.10

Loading…
Cancel
Save