From 28b666cd33faa95314c45bc70302b8db384e5bdb Mon Sep 17 00:00:00 2001 From: wolfplus Date: Mon, 7 Apr 2025 21:18:12 +0100 Subject: [PATCH] mcp --- android/app/build.gradle.kts | 14 + .../yunqiinnovation/deepsound/MainActivity.kt | 31 +- .../deepsound/SystemFunctionHandler.kt | 225 -------- .../deepsound/VoiceFunctionHandler.kt | 510 ------------------ .../VoiceInteractionHandler copy.kt.bak | 456 ++++++++++++++++ .../deepsound/VoiceInteractionHandler.kt | 107 ++-- .../deepsound/VoiceInteractionService.kt | 14 +- android/build.gradle.kts | 11 + android/settings.gradle.kts | 2 +- 9 files changed, 551 insertions(+), 819 deletions(-) delete mode 100644 android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt delete mode 100644 android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt create mode 100644 android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler copy.kt.bak diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 026060648..8c371408e 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -39,6 +39,20 @@ android { isCheckReleaseBuilds = false } + // 添加packaging配置，排除冲突文件 + packaging { + resources { + excludes.add("META-INF/DEPENDENCIES") + excludes.add("META-INF/LICENSE") + excludes.add("META-INF/LICENSE.txt") + excludes.add("META-INF/license.txt") + excludes.add("META-INF/NOTICE") + excludes.add("META-INF/NOTICE.txt") + excludes.add("META-INF/notice.txt") + excludes.add("META-INF/*.kotlin_module") + } + } + defaultConfig { // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html). applicationId = "com.yunqiinnovation.deepsound" diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt index 9f06844db..1c4717f11 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/MainActivity.kt @@ -170,6 +170,8 @@ class MainActivity: FlutterActivity() { var openaiApiKey: String = "" var openaiBaseUrl: String? = null var openaiModel: String? = null + var volcanoSpeechAppId: String = "" + var volcanoSpeechAppToken: String = "" // 安全存储相关常量 private const val SECURE_PREFS_FILENAME = "deep_voice_secure_prefs" @@ -180,6 +182,8 @@ class MainActivity: FlutterActivity() { private const val KEY_OPENAI_BASE_URL = "openai_base_url" private const val KEY_OPENAI_MODEL = "openai_model" private const val KEY_MCP_SERVER_ENDPOINT = "mcp_server_endpoint" + private const val KEY_VOLCANO_SPEECH_APP_ID = "volcano_speech_app_id" + private const val KEY_VOLCANO_SPEECH_APP_TOKEN = "volcano_speech_app_token" // 会话管理 private const val KEY_SESSION_ID = "session_id" @@ -218,6 +222,8 @@ class MainActivity: FlutterActivity() { .putString(KEY_OPENAI_BASE_URL, openaiBaseUrl) .putString(KEY_SESSION_ID, currentSessionId) .putString(KEY_OPENAI_MODEL, openaiModel) + .putString(KEY_VOLCANO_SPEECH_APP_ID, volcanoSpeechAppId) + .putString(KEY_VOLCANO_SPEECH_APP_TOKEN, volcanoSpeechAppToken) .apply() FileLogger.d("MainActivity", "密钥已安全保存到加密存储中") @@ -264,6 +270,8 @@ class MainActivity: FlutterActivity() { openaiApiKey = sharedPreferences.getString(KEY_OPENAI_API_KEY, "") ?: "" openaiBaseUrl = sharedPreferences.getString(KEY_OPENAI_BASE_URL, null) openaiModel = sharedPreferences.getString(KEY_OPENAI_MODEL, null) + volcanoSpeechAppId = sharedPreferences.getString(KEY_VOLCANO_SPEECH_APP_ID, "") ?: "" + volcanoSpeechAppToken = sharedPreferences.getString(KEY_VOLCANO_SPEECH_APP_TOKEN, "") ?: "" FileLogger.d("MainActivity", "已从加密存储加载密钥") // 检查是否成功获取所有必要密钥 @@ -505,21 +513,14 @@ class MainActivity: FlutterActivity() { FileLogger.d(TAG, "启动语音交互服务") // 获取配置参数 - val key = call.argument("azure_speech_key") ?: "" - val region = call.argument("azure_speech_region") ?: "" - val openaiKey = call.argument("openai_api_key") ?: "" - val baseUrl = call.argument("openai_base_url") - val model = call.argument("openai_model") - // 设置 Azure Speech 和 AI 配置 - azureSpeechKey = key - azureSpeechRegion = region - openaiApiKey = openaiKey - if (baseUrl != null) { - openaiBaseUrl = baseUrl - } - if (model != null) { - openaiModel = model - } + azureSpeechKey = call.argument("azure_speech_key") ?: "" + azureSpeechRegion = call.argument("azure_speech_region") ?: "" + openaiApiKey = call.argument("openai_api_key") ?: "" + openaiBaseUrl = call.argument("openai_base_url") + openaiModel = call.argument("openai_model") + volcanoSpeechAppId = call.argument("volcano_speech_app_id") ?: "" + volcanoSpeechAppToken = call.argument("volcano_speech_app_token") ?: "" + // 保存密钥到安全存储 saveKeysToSecureStorage(applicationContext) diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt deleted file mode 100644 index 4e28f26aa..000000000 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/SystemFunctionHandler.kt +++ /dev/null @@ -1,225 +0,0 @@ -package com.yunqiinnovation.deepsound - -import android.content.Context -import android.content.Intent -import android.content.pm.PackageManager -import android.net.Uri -import android.provider.ContactsContract -import android.telephony.SmsManager -import com.yunqiinnovation.deepsound.core.utils.FileLogger - -/** - * 系统功能处理器 - 负责调用系统API执行各种功能 - */ -class SystemFunctionHandler(private val context: Context) { - companion object { - private const val TAG = "SystemFunctionHandler" - } - - /** - * 启动翻译模式 - * @return 操作结果 - */ - fun enterTranslationMode(): Result { - FileLogger.d(TAG, "进入翻译模式") - // 此处应添加实际的翻译模式实现 - // 可能需要启动一个专门的翻译服务或活动 - - // 广播通知APP的其他部分进入翻译模式 - // 使用VoiceInteractionService中定义的常量 - val intent = Intent(VoiceInteractionService.ACTION_ENTER_TRANSLATION_MODE).apply { - - putExtra("timestamp", System.currentTimeMillis()) - } - - // 发送广播 - context.sendBroadcast(intent) - FileLogger.d(TAG, "已发送广播: ${VoiceInteractionService.ACTION_ENTER_TRANSLATION_MODE}") - - return Result( - success = true, - message = "已进入翻译模式" - ) - } - - /** - * 发送短信 - * @param contact 联系人（名称或电话号码） - * @param message 短信内容 - * @return 操作结果 - */ - fun sendTextMessage(contact: String, message: String): Result { - FileLogger.d(TAG, "发送短信给 $contact: $message") - - try { - val phoneNumber = getPhoneNumberForContact(contact) - if (phoneNumber.isEmpty()) { - return Result( - success = false, - message = "未找到联系人信息: $contact" - ) - } - - // 使用SmsManager发送短信 - val smsManager = SmsManager.getDefault() - smsManager.sendTextMessage(phoneNumber, null, message, null, null) - - return Result( - success = true, - message = "已向${contact}发送短信: $message" - ) - } catch (e: Exception) { - FileLogger.e(TAG, "发送短信失败: ${e.message}", e) - return Result( - success = false, - message = "发送短信失败: ${e.message}" - ) - } - } - - /** - * 拨打电话 - * @param contact 联系人（名称或电话号码） - * @return 操作结果 - */ - fun makePhoneCall(contact: String): Result { - FileLogger.d(TAG, "拨打电话给 $contact") - - try { - val phoneNumber = getPhoneNumberForContact(contact) - if (phoneNumber.isEmpty()) { - return Result( - success = false, - message = "未找到联系人信息: $contact" - ) - } - - // 创建拨号意图 - val intent = Intent(Intent.ACTION_CALL) - intent.data = Uri.parse("tel:$phoneNumber") - intent.flags = Intent.FLAG_ACTIVITY_NEW_TASK - context.startActivity(intent) - - return Result( - success = true, - message = "正在拨打${contact}的电话" - ) - } catch (e: Exception) { - FileLogger.e(TAG, "拨打电话失败: ${e.message}", e) - return Result( - success = false, - message = "拨打电话失败: ${e.message}" - ) - } - } - - /** - * 打开应用程序 - * @param appName 应用名称 - * @return 操作结果 - */ - fun openApp(appName: String): Result { - FileLogger.d(TAG, "打开应用: $appName") - - try { - // 查找应用的包名 - val packageManager = context.packageManager - val intent = Intent(Intent.ACTION_MAIN, null) - intent.addCategory(Intent.CATEGORY_LAUNCHER) - - val apps = packageManager.queryIntentActivities(intent, 0) - - for (resolveInfo in apps) { - val appLabel = resolveInfo.loadLabel(packageManager).toString() - if (appLabel.equals(appName, ignoreCase = true) || - appLabel.contains(appName, ignoreCase = true)) { - - // 找到匹配的应用 - val packageName = resolveInfo.activityInfo.packageName - val launchIntent = packageManager.getLaunchIntentForPackage(packageName) - - if (launchIntent != null) { - launchIntent.flags = Intent.FLAG_ACTIVITY_NEW_TASK - context.startActivity(launchIntent) - - return Result( - success = true, - message = "正在打开${appName}应用" - ) - } - } - } - - return Result( - success = false, - message = "未找到应用: $appName" - ) - } catch (e: Exception) { - FileLogger.e(TAG, "打开应用失败: ${e.message}", e) - return Result( - success = false, - message = "打开应用失败: ${e.message}" - ) - } - } - - /** - * 根据联系人名称获取电话号码 - * @param contact 联系人名称或电话号码 - * @return 电话号码 - */ - private fun getPhoneNumberForContact(contact: String): String { - // 如果输入的是电话号码（简单判断，包含数字和特殊符号） - if (contact.matches(Regex("^[+\\d\\s\\-()]*\$")) && contact.any { it.isDigit() }) { - // 移除非数字字符 - return contact.replace(Regex("[^+\\d]"), "") - } - - // 从联系人数据库查询 - try { - val contentResolver = context.contentResolver - val uri = Uri.withAppendedPath( - ContactsContract.Contacts.CONTENT_FILTER_URI, - Uri.encode(contact) - ) - - val projection = arrayOf( - ContactsContract.Contacts._ID, - ContactsContract.Contacts.DISPLAY_NAME, - ContactsContract.Contacts.HAS_PHONE_NUMBER - ) - - contentResolver.query(uri, projection, null, null, null)?.use { cursor -> - if (cursor.moveToFirst()) { - val hasPhoneNumber = cursor.getInt(cursor.getColumnIndexOrThrow(ContactsContract.Contacts.HAS_PHONE_NUMBER)) - - if (hasPhoneNumber > 0) { - val contactId = cursor.getString(cursor.getColumnIndexOrThrow(ContactsContract.Contacts._ID)) - val phoneUri = ContactsContract.CommonDataKinds.Phone.CONTENT_URI - val phoneProjection = arrayOf(ContactsContract.CommonDataKinds.Phone.NUMBER) - val phoneSelection = "${ContactsContract.CommonDataKinds.Phone.CONTACT_ID} = ?" - val phoneSelectionArgs = arrayOf(contactId) - - contentResolver.query(phoneUri, phoneProjection, phoneSelection, phoneSelectionArgs, null)?.use { phoneCursor -> - if (phoneCursor.moveToFirst()) { - return phoneCursor.getString(phoneCursor.getColumnIndexOrThrow(ContactsContract.CommonDataKinds.Phone.NUMBER)) - } - } - } - } - } - } catch (e: Exception) { - FileLogger.e(TAG, "查询联系人信息失败: ${e.message}", e) - } - - return "" - } - - /** - * 操作结果数据类 - */ - data class Result( - val success: Boolean, - val message: String - ) -} \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt deleted file mode 100644 index c15ca7e17..000000000 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceFunctionHandler.kt +++ /dev/null @@ -1,510 +0,0 @@ -package com.yunqiinnovation.deepsound - -import org.json.JSONArray -import org.json.JSONObject -import com.yunqiinnovation.open_ai_service.OpenAIService -import com.yunqiinnovation.deepsound.core.utils.FileLogger -import android.content.Context - -/** - * 语音功能处理器 - 处理AI函数调用 - */ -class VoiceFunctionHandler( - private val openAIService: OpenAIService, - private val context: Context -) { - companion object { - private const val TAG = "VoiceFunctionHandler" - } - - // 添加系统功能处理器 - private val systemFunctionHandler = SystemFunctionHandler(context) - - /** - * 初始化并注册所有可用的函数 - */ - fun initialize() { - try { - // 注册退出交互函数 - registerExitInteractionFunction() - - // 注册翻译模式函数 - registerTranslationModeFunction() - - // 注册发送短信函数 - registerSendTextMessageFunction() - - // 注册拨打电话函数 - registerMakePhoneCallFunction() - - // 注册打开应用函数 - registerOpenAppFunction() - - // 在这里可以注册更多函数 - - } catch (e: Exception) { - FileLogger.e(TAG, "初始化函数处理器失败: ${e.message}", e) - } - } - - /** - * 注册退出交互函数 - */ - private fun registerExitInteractionFunction() { - try { - openAIService.registerFunction( - "exit_interaction", - "退出当前语音交互", - JSONObject(""" - { - "type": "object", - "properties": {}, - "required": [] - } - """) - ) - FileLogger.d(TAG, "退出交互功能已注册") - } catch (e: Exception) { - FileLogger.e(TAG, "注册退出交互函数失败: ${e.message}", e) - } - } - - /** - * 注册翻译模式函数 - */ - private fun registerTranslationModeFunction() { - try { - openAIService.registerFunction( - "enter_translation_mode", - "进入翻译模式", - JSONObject(""" - { - "type": "object", - "properties": {}, - "required": [] - } - """) - ) - FileLogger.d(TAG, "翻译模式功能已注册") - } catch (e: Exception) { - FileLogger.e(TAG, "注册翻译模式函数失败: ${e.message}", e) - } - } - - /** - * 注册发送短信函数 - */ - private fun registerSendTextMessageFunction() { - try { - openAIService.registerFunction( - "send_text_message", - "发送短信", - JSONObject(""" - { - "type": "object", - "properties": { - "contact": { - "type": "string", - "description": "联系人姓名或电话号码" - }, - "message": { - "type": "string", - "description": "短信内容" - } - }, - "required": ["contact", "message"] - } - """) - ) - FileLogger.d(TAG, "发送短信功能已注册") - } catch (e: Exception) { - FileLogger.e(TAG, "注册发送短信函数失败: ${e.message}", e) - } - } - - /** - * 注册拨打电话函数 - */ - private fun registerMakePhoneCallFunction() { - try { - openAIService.registerFunction( - "make_phone_call", - "拨打电话", - JSONObject(""" - { - "type": "object", - "properties": { - "contact": { - "type": "string", - "description": "联系人姓名或电话号码" - } - }, - "required": ["contact"] - } - """) - ) - FileLogger.d(TAG, "拨打电话功能已注册") - } catch (e: Exception) { - FileLogger.e(TAG, "注册拨打电话函数失败: ${e.message}", e) - } - } - - /** - * 注册打开应用函数 - */ - private fun registerOpenAppFunction() { - try { - openAIService.registerFunction( - "open_app", - "打开应用", - JSONObject(""" - { - "type": "object", - "properties": { - "app_name": { - "type": "string", - "description": "应用名称" - } - }, - "required": ["app_name"] - } - """) - ) - FileLogger.d(TAG, "打开应用功能已注册") - } catch (e: Exception) { - FileLogger.e(TAG, "注册打开应用函数失败: ${e.message}", e) - } - } - - /** - * 处理函数调用 - * - * @param functionCall 函数调用信息 - * @param messages 消息历史 - * @param callback 回调，处理退出等操作 - * @return 是否已处理函数调用 - */ - fun handleFunctionCall( - functionCall: JSONObject, - messages: JSONArray, - callback: FunctionCallCallback - ): Boolean { - val functionName = functionCall.getString("name") - FileLogger.d(TAG, "处理函数调用: $functionName") - - return when (functionName) { - "exit_interaction" -> { - handleExitInteraction(functionCall, messages, callback) - true - } - "enter_translation_mode" -> { - handleEnterTranslationMode(functionCall, messages, callback) - true - } - "send_text_message" -> { - handleSendTextMessage(functionCall, messages, callback) - true - } - "make_phone_call" -> { - handleMakePhoneCall(functionCall, messages, callback) - true - } - "open_app" -> { - handleOpenApp(functionCall, messages, callback) - true - } - else -> { - // 未知函数，返回默认结果 - handleUnknownFunction(functionCall, messages, callback) - false - } - } - } - - /** - * 处理退出交互函数 - */ - private fun handleExitInteraction( - functionCall: JSONObject, - messages: JSONArray, - callback: FunctionCallCallback - ) { - FileLogger.d(TAG, "处理退出交互函数") - - val responseBuilder = StringBuilder() - - openAIService.sendFunctionCallResult( - messages = messages, - functionCall = functionCall, - functionResult = "{\"result\": \"已退出语音交互\"}", - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - responseBuilder.append(token) - } - - override fun onComplete() { - FileLogger.d(TAG, "handleExitInteraction onComplete: ${responseBuilder.toString()}") - val response = responseBuilder.toString() - if (response.isNotEmpty()) { - callback.onExitWithMessage(response) - } else { - callback.onExitWithMessage("已退出语音交互") - } - } - - override fun onError(e: Exception) { - FileLogger.e(TAG, "处理退出交互函数调用出错: ${e.message}") - callback.onError("退出交互时出错") - } - - override fun onFunctionCall(nestedCall: JSONObject) { - FileLogger.e(TAG, "意外收到嵌套函数调用: ${nestedCall.getString("name")}") - } - } - ) - } - - /** - * 处理进入翻译模式函数 - */ - private fun handleEnterTranslationMode( - functionCall: JSONObject, - messages: JSONArray, - callback: FunctionCallCallback - ) { - FileLogger.d(TAG, "处理进入翻译模式函数") - - // 调用系统功能处理器执行翻译模式操作 - val result = systemFunctionHandler.enterTranslationMode() - - val responseBuilder = StringBuilder() - - openAIService.sendFunctionCallResult( - messages = messages, - functionCall = functionCall, - functionResult = "{\"result\": \"${result.message}\"}", - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - responseBuilder.append(token) - callback.onTokenReceived(token) - } - - override fun onComplete() { - FileLogger.d(TAG, "handleEnterTranslationMode onComplete: ${responseBuilder.toString()}") - callback.onComplete() - } - - override fun onError(e: Exception) { - FileLogger.e(TAG, "处理进入翻译模式函数调用出错: ${e.message}") - callback.onError("进入翻译模式时出错") - } - - override fun onFunctionCall(nestedCall: JSONObject) { - callback.onFunctionCall(nestedCall) - } - } - ) - } - - /** - * 处理发送短信函数 - */ - private fun handleSendTextMessage( - functionCall: JSONObject, - messages: JSONArray, - callback: FunctionCallCallback - ) { - FileLogger.d(TAG, "处理发送短信函数") - - try { - val arguments = JSONObject(functionCall.getString("arguments")) - val contact = arguments.getString("contact") - val message = arguments.getString("message") - - // 调用系统功能处理器发送短信 - val result = systemFunctionHandler.sendTextMessage(contact, message) - - val responseBuilder = StringBuilder() - - openAIService.sendFunctionCallResult( - messages = messages, - functionCall = functionCall, - functionResult = "{\"result\": \"${result.message}\"}", - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - responseBuilder.append(token) - callback.onTokenReceived(token) - } - - override fun onComplete() { - FileLogger.d(TAG, "handleSendTextMessage onComplete: ${responseBuilder.toString()}") - callback.onComplete() - } - - override fun onError(e: Exception) { - FileLogger.e(TAG, "处理发送短信函数调用出错: ${e.message}") - callback.onError("发送短信时出错") - } - - override fun onFunctionCall(nestedCall: JSONObject) { - callback.onFunctionCall(nestedCall) - } - } - ) - } catch (e: Exception) { - FileLogger.e(TAG, "解析发送短信参数失败: ${e.message}") - callback.onError("发送短信参数错误") - } - } - - /** - * 处理拨打电话函数 - */ - private fun handleMakePhoneCall( - functionCall: JSONObject, - messages: JSONArray, - callback: FunctionCallCallback - ) { - FileLogger.d(TAG, "处理拨打电话函数") - - try { - val arguments = JSONObject(functionCall.getString("arguments")) - val contact = arguments.getString("contact") - - // 调用系统功能处理器拨打电话 - val result = systemFunctionHandler.makePhoneCall(contact) - - val responseBuilder = StringBuilder() - - openAIService.sendFunctionCallResult( - messages = messages, - functionCall = functionCall, - functionResult = "{\"result\": \"${result.message}\"}", - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - responseBuilder.append(token) - callback.onTokenReceived(token) - } - - override fun onComplete() { - FileLogger.d(TAG, "handleMakePhoneCall onComplete: ${responseBuilder.toString()}") - callback.onComplete() - } - - override fun onError(e: Exception) { - FileLogger.e(TAG, "处理拨打电话函数调用出错: ${e.message}") - callback.onError("拨打电话时出错") - } - - override fun onFunctionCall(nestedCall: JSONObject) { - callback.onFunctionCall(nestedCall) - } - } - ) - } catch (e: Exception) { - FileLogger.e(TAG, "解析拨打电话参数失败: ${e.message}") - callback.onError("拨打电话参数错误") - } - } - - /** - * 处理打开应用函数 - */ - private fun handleOpenApp( - functionCall: JSONObject, - messages: JSONArray, - callback: FunctionCallCallback - ) { - FileLogger.d(TAG, "处理打开应用函数") - - try { - val arguments = JSONObject(functionCall.getString("arguments")) - val appName = arguments.getString("app_name") - - // 调用系统功能处理器打开应用 - val result = systemFunctionHandler.openApp(appName) - - val responseBuilder = StringBuilder() - - openAIService.sendFunctionCallResult( - messages = messages, - functionCall = functionCall, - functionResult = "{\"result\": \"${result.message}\"}", - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - responseBuilder.append(token) - callback.onTokenReceived(token) - } - - override fun onComplete() { - FileLogger.d(TAG, "handleOpenApp onComplete: ${responseBuilder.toString()}") - callback.onComplete() - } - - override fun onError(e: Exception) { - FileLogger.e(TAG, "处理打开应用函数调用出错: ${e.message}") - callback.onError("打开应用时出错") - } - - override fun onFunctionCall(nestedCall: JSONObject) { - callback.onFunctionCall(nestedCall) - } - } - ) - } catch (e: Exception) { - FileLogger.e(TAG, "解析打开应用参数失败: ${e.message}") - callback.onError("打开应用参数错误") - } - } - - /** - * 处理未知函数调用 - */ - private fun handleUnknownFunction( - functionCall: JSONObject, - messages: JSONArray, - callback: FunctionCallCallback - ) { - FileLogger.d(TAG, "处理未知函数: ${functionCall.getString("name")}") - - try { - openAIService.sendFunctionCallResult( - messages = messages, - functionCall = functionCall, - functionResult = "{\"result\": \"处理函数调用中\"}", - callback = object : OpenAIService.StreamCallback { - override fun onToken(token: String) { - callback.onTokenReceived(token) - } - - override fun onComplete() { - callback.onComplete() - } - - override fun onError(e: Exception) { - FileLogger.e(TAG, "处理函数调用失败: ${e.message}") - callback.onError("处理函数调用失败: ${e.message}") - } - - override fun onFunctionCall(nestedCall: JSONObject) { - callback.onFunctionCall(nestedCall) - } - } - ) - } catch (e: Exception) { - FileLogger.e(TAG, "处理函数调用失败: ${e.message}") - callback.onError("处理函数调用失败: ${e.message}") - } - } - - /** - * 函数调用回调接口 - */ - interface FunctionCallCallback { - fun onTokenReceived(token: String) - fun onComplete() - fun onError(message: String) - fun onFunctionCall(functionCall: JSONObject) - fun onExitWithMessage(farewell: String) - } -} \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler copy.kt.bak b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler copy.kt.bak new file mode 100644 index 000000000..1a9e2df13 --- /dev/null +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler copy.kt.bak @@ -0,0 +1,456 @@ +package com.yunqiinnovation.deepsound + +import android.content.Context +import org.json.JSONArray +import org.json.JSONObject +import android.util.Log +import com.yunqiinnovation.deepsound.core.utils.FileLogger +import com.yunqiinnovation.azure_speech.AzureAsrHelper +import com.yunqiinnovation.volcano_speech.VolcanoTtsHelper +import com.yunqiinnovation.open_ai_service.OpenAIService + + +/** + * 语音交互处理器 - 处理语音识别、TTS和AI对话相关逻辑 + */ +class VoiceInteractionHandler( + private val context: Context, + private val azureSpeechKey: String, + private val azureSpeechRegion: String, + private val openaiApiKey: String, + private val openaiBaseUrl: String = "", + private val openaiModel: String = "", + private val volcanoSpeechAppId: String, + private val volcanoSpeechAppToken: String +) { + private val TAG = "VoiceInteractionHandler" + + // Azure服务 + private var azureAsrHelper: AzureAsrHelper? = null + private var volcanoTtsHelper: VolcanoTtsHelper? = null + + // OpenAI服务 + private val openAIService = OpenAIService() + + + // 语音功能处理 + private val voiceFunctionHandler = VoiceFunctionHandler(openAIService, context) + + // 当前用户输入 + private var currentUserInput = "" + + // 状态 + private var isInitialized = false + var isRecognitionActive = false + private set + var isTtsSpeaking = false + private set + var hasSpeechDetected = false + private set + + // 回调 + private var callback: InteractionCallback? = null + + /** + * 初始化 + */ + fun initialize(): Boolean { + if (isInitialized) return true + + try { + // 初始化Azure ASR + azureAsrHelper = AzureAsrHelper(context).apply { + initialize(azureSpeechKey, azureSpeechRegion) + } + + // 初始化Volcano TTS (使用大模型TTS) + volcanoTtsHelper = VolcanoTtsHelper(context).apply { + // 这里需要替换为实际的Volcano SDK初始化参数 + // 暂时使用假参数，实际使用时需要替换为真实值 + initialize(volcanoSpeechAppId, volcanoSpeechAppToken, "volc.bigasr.sauc.duration") + + } + + // 初始化OpenAI服务 + openAIService.initialize(openaiApiKey, openaiBaseUrl, openaiModel) + + // 初始化语音功能处理器 + voiceFunctionHandler.initialize() + + isInitialized = true + return true + } catch (e: Exception) { + FileLogger.e(TAG, "初始化失败: ${e.message}", e) + return false + } + } + + /** + * 设置回调 + */ + fun setCallback(callback: InteractionCallback) { + this.callback = callback + } + + /** + * 开始语音识别 + */ + fun startRecognition() { + if (isRecognitionActive) return + + // 检查录音权限 + if (!checkRecordAudioPermission()) { + callback?.onError("需要录音权限，请在设置中授予权限") + return + } + + isRecognitionActive = true + hasSpeechDetected = false + notifyStateChanged() + + try { + azureAsrHelper?.startContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { + override fun onRecognizing(recognizing: String, detectedLanguage: String) { + if (recognizing.isNotEmpty()) { + hasSpeechDetected = true + stopTts() + notifyStateChanged() + } + } + + override fun onResult(result: String, detectedLanguage: String) { + if (result.isNotEmpty()) { + notifyStateChanged() + + processWithOpenAI(result) + + } + + // 重置状态，继续识别 + hasSpeechDetected = false + } + + override fun onSessionStarted() { + notifyStateChanged() + } + + override fun onSessionStopped() { + isRecognitionActive = false + notifyStateChanged() + } + + override fun onCanceled(reason: String, errorDetails: String) { + isRecognitionActive = false + notifyStateChanged() + } + + override fun onError(error: String) { + isRecognitionActive = false + callback?.onError("语音识别出错") + notifyStateChanged() + } + + override fun onSuccess(message: String) { + // 处理成功事件 + } + }) + } catch (e: Exception) { + isRecognitionActive = false + FileLogger.e(TAG, "启动语音识别失败: ${e.message}", e) + callback?.onError("启动语音识别失败") + notifyStateChanged() + } + } + + /** + * 停止语音识别 + */ + fun stopRecognition() { + if (!isRecognitionActive) return + + FileLogger.d(TAG, "停止语音识别") + + try { + azureAsrHelper?.stopContinuousRecognition(object : AzureAsrHelper.ContinuousRecognizeCallback { + override fun onResult(result: String, detectedLanguage: String) {} + override fun onRecognizing(recognizing: String, detectedLanguage: String) {} + override fun onSessionStarted() {} + override fun onSessionStopped() { + isRecognitionActive = false + FileLogger.d(TAG, "语音识别会话已停止") + notifyStateChanged() + } + override fun onCanceled(reason: String, errorDetails: String) { + isRecognitionActive = false + FileLogger.d(TAG, "语音识别已取消: $reason") + notifyStateChanged() + } + override fun onError(error: String) { + isRecognitionActive = false + FileLogger.e(TAG, "停止语音识别时出错: $error") + notifyStateChanged() + } + override fun onSuccess(message: String) { + isRecognitionActive = false + FileLogger.d(TAG, "语音识别已停止: $message") + notifyStateChanged() + } + }) + } catch (e: Exception) { + FileLogger.e(TAG, "停止语音识别异常: ${e.message}", e) + // 确保状态一致性 + isRecognitionActive = false + notifyStateChanged() + } + } + + /** + * 使用OpenAI处理语音识别结果 + */ + private fun processWithOpenAI(text: String) { + // 保存当前用户输入，用于后续同步聊天记录 + currentUserInput = text + + Thread { + try { + val messages = JSONArray().apply { + put(openAIService.createUserMessage(text)) + } + + // 创建响应构建器 + val responseBuilder = StringBuilder() + + openAIService.sendMessageStream( + messages = messages, + callback = object : OpenAIService.StreamCallback { + override fun onToken(token: String) { + // 累加响应内容 + responseBuilder.append(token) + } + + override fun onComplete() { + // 处理完整响应 + val response = responseBuilder.toString() + if (response.isNotEmpty()) { + // 播放AI回复 + Log.d(TAG, "AI 回复: $response") + + speakAIResponse(response) + + // 同步聊天记录到Flutter端 + sendChatHistoryUpdate("personal_assistant", text, response) + } + } + + override fun onError(e: Exception) { + FileLogger.e(TAG, "AI流式处理出错: ${e.message}", e) + callback?.onError("AI处理出错") + } + + override fun onFunctionCall(call: JSONObject) { + FileLogger.d(TAG, "收到函数调用请求: ${call.getString("name")}") + + // 使用函数处理器处理函数调用 + val handled = voiceFunctionHandler.handleFunctionCall( + functionCall = call, + messages = messages, + callback = object : VoiceFunctionHandler.FunctionCallCallback { + override fun onTokenReceived(token: String) { + responseBuilder.append(token) + } + + override fun onComplete() { + val response = responseBuilder.toString() + if (response.isNotEmpty()) { + // 播放AI回复 + Log.d(TAG, "AI Function Call 回复: $response") + speakAIResponse(response) + + // 同步聊天记录到Flutter端 + sendChatHistoryUpdate("personal_assistant", text, response) + } + notifyStateChanged() + } + + override fun onError(message: String) { + FileLogger.e(TAG, "函数处理出错: $message") + callback?.onError(message) + } + + override fun onFunctionCall(nestedCall: JSONObject) { + FileLogger.d(TAG, "收到嵌套函数调用: ${nestedCall.getString("name")}") + // 不处理嵌套函数调用，直接返回错误提示 + // speakAIResponse("抱歉，暂不支持嵌套函数调用") + } + + override fun onExitWithMessage(farewell: String) { + // 播放退出消息 + speakAIResponse(farewell) + + // 同步聊天记录 + sendChatHistoryUpdate("personal_assistant", text, farewell) + + // 停止语音识别 + stopRecognition() + } + } + ) + + if (!handled) { + // 如果函数没有被处理，作为普通文本处理 + FileLogger.d(TAG, "函数未处理，作为普通文本处理") + speakAIResponse("我无法处理这个请求") + sendChatHistoryUpdate("personal_assistant", text, "我无法处理这个请求") + } + } + } + ) + + } catch (e: Exception) { + FileLogger.e(TAG, "AI处理出错: ${e.message}", e) + callback?.onError("AI处理出错") + } + }.start() + } + + /** + * 播放TTS + */ + fun playTts(text: String, callback: TtsCallback? = null) { + isTtsSpeaking = true + notifyStateChanged() + + volcanoTtsHelper?.speak(text, object : VolcanoTtsHelper.TTSCallback { + override fun onStart(reqId: String) { + // TTS开始播放事件 + } + + override fun onProgress(reqId: String, progress: Double) { + // TTS播放进度事件 + } + + override fun onComplete(reqId: String) { + isTtsSpeaking = false + notifyStateChanged() + callback?.onComplete() + } + + override fun onError(reqId: String, errorCode: Int, errorMsg: String) { + isTtsSpeaking = false + notifyStateChanged() + callback?.onError(errorMsg) + } + }) + } + + /** + * 停止TTS播放 + */ + fun stopTts() { + if (isTtsSpeaking) { + volcanoTtsHelper?.stop() + isTtsSpeaking = false + notifyStateChanged() + } + } + + /** + * 播放AI回复 + */ + private fun speakAIResponse(text: String) { + isTtsSpeaking = true + notifyStateChanged() + + volcanoTtsHelper?.speak(text, object : VolcanoTtsHelper.TTSCallback { + override fun onStart(reqId: String) { + // TTS开始播放事件 + } + + override fun onProgress(reqId: String, progress: Double) { + // TTS播放进度事件 + } + + override fun onComplete(reqId: String) { + isTtsSpeaking = false + notifyStateChanged() + } + + override fun onError(reqId: String, errorCode: Int, errorMsg: String) { + isTtsSpeaking = false + notifyStateChanged() + } + }) + } + + /** + * 释放资源 + */ + fun dispose() { + // 停止语音识别 + stopRecognition() + + // 停止TTS播放 + stopTts() + + // 释放Azure资源 + azureAsrHelper?.let { + FileLogger.d(TAG, "关闭Azure ASR服务") + it.dispose() + } + + volcanoTtsHelper?.let { + FileLogger.d(TAG, "关闭Volcano TTS服务") + it.release() + } + + FileLogger.d(TAG, "语音交互处理器资源已释放") + } + + /** + * 检查录音权限 + */ + private fun checkRecordAudioPermission(): Boolean { + val permission = android.Manifest.permission.RECORD_AUDIO + val result = context.checkCallingOrSelfPermission(permission) + return result == android.content.pm.PackageManager.PERMISSION_GRANTED + } + + /** + * 通知状态变化 + */ + private fun notifyStateChanged() { + callback?.onStateChanged(isRecognitionActive, isTtsSpeaking, hasSpeechDetected) + } + + /** + * 发送聊天历史更新 + */ + private fun sendChatHistoryUpdate(agentId: String, userMessage: String, assistantMessage: String) { + val intent = android.content.Intent(VoiceInteractionService.ACTION_CHAT_HISTORY_UPDATED).apply { + putExtra("agentId", agentId) + putExtra("userMessage", userMessage) + putExtra("assistantMessage", assistantMessage) + putExtra("timestamp", System.currentTimeMillis()) + } + + // 发送广播 + context.sendBroadcast(intent) + } + + /** + * 交互回调接口 + */ + interface InteractionCallback { + fun onStateChanged(isRecognitionActive: Boolean, isTtsSpeaking: Boolean, hasSpeechDetected: Boolean) + fun onError(message: String) + fun onPromptRequest(message: String) + } + + /** + * TTS回调接口 + */ + interface TtsCallback { + fun onComplete() + fun onError(error: String) + } +} \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt index 1d6a528b3..b885117c0 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionHandler.kt @@ -1,13 +1,17 @@ package com.yunqiinnovation.deepsound +import android.content.BroadcastReceiver import android.content.Context +import android.content.Intent +import android.content.IntentFilter import org.json.JSONArray import org.json.JSONObject import android.util.Log import com.yunqiinnovation.deepsound.core.utils.FileLogger import com.yunqiinnovation.azure_speech.AzureAsrHelper import com.yunqiinnovation.azure_speech.AzureTtsHelper -import com.yunqiinnovation.open_ai_service.OpenAIService +import com.yunqiinnovation.open_ai_service.OpenAIService +import com.yunqiinnovation.open_ai_service.SystemFunctionHandler /** @@ -19,7 +23,9 @@ class VoiceInteractionHandler( private val azureSpeechRegion: String, private val openaiApiKey: String, private val openaiBaseUrl: String = "", - private val openaiModel: String = "" + private val openaiModel: String = "", + private val volcanoSpeechAppId: String, + private val volcanoSpeechAppToken: String ) { private val TAG = "VoiceInteractionHandler" @@ -28,11 +34,7 @@ class VoiceInteractionHandler( private var azureTtsHelper: AzureTtsHelper? = null // OpenAI服务 - private val openAIService = OpenAIService() - - - // 语音功能处理 - private val voiceFunctionHandler = VoiceFunctionHandler(openAIService, context) + private val openAIService = OpenAIService(context.applicationContext) // 当前用户输入 private var currentUserInput = "" @@ -49,6 +51,17 @@ class VoiceInteractionHandler( // 回调 private var callback: InteractionCallback? = null + // 广播接收器 + private val exitInteractionReceiver = object : BroadcastReceiver() { + override fun onReceive(context: Context, intent: Intent) { + if (intent.action == SystemFunctionHandler.ACTION_EXIT_INTERACTION) { + Log.d(TAG, "收到退出交互广播") + stopRecognition() + + } + } + } + /** * 初始化 */ @@ -69,8 +82,19 @@ class VoiceInteractionHandler( // 初始化OpenAI服务 openAIService.initialize(openaiApiKey, openaiBaseUrl, openaiModel) - // 初始化语音功能处理器 - voiceFunctionHandler.initialize() + // 注册广播接收器 + try { + Log.d(TAG, "注册退出交互广播接收器，包名=${context.packageName}, action=${SystemFunctionHandler.ACTION_EXIT_INTERACTION}") + context.registerReceiver( + exitInteractionReceiver, + IntentFilter(SystemFunctionHandler.ACTION_EXIT_INTERACTION), + Context.RECEIVER_NOT_EXPORTED + ) + Log.d(TAG, "退出交互广播接收器注册成功") + } catch (e: Exception) { + // 广播注册失败不应该影响整个应用初始化 + Log.e(TAG, "注册退出交互广播接收器失败: ${e.message}", e) + } isInitialized = true return true @@ -243,64 +267,9 @@ class VoiceInteractionHandler( } override fun onFunctionCall(call: JSONObject) { - FileLogger.d(TAG, "收到函数调用请求: ${call.getString("name")}") + FileLogger.d(TAG, "processWithOpenAI 收到函数调用请求: ${call.getString("name")}") - // 使用函数处理器处理函数调用 - val handled = voiceFunctionHandler.handleFunctionCall( - functionCall = call, - messages = messages, - callback = object : VoiceFunctionHandler.FunctionCallCallback { - override fun onTokenReceived(token: String) { - responseBuilder.append(token) - } - - override fun onComplete() { - val response = responseBuilder.toString() - if (response.isNotEmpty()) { - // 播放AI回复 - Log.d(TAG, "AI Function Call 回复: $response") - speakAIResponse(response) - - // 同步聊天记录到Flutter端 - sendChatHistoryUpdate("personal_assistant", text, response) - } - notifyStateChanged() - } - - override fun onError(message: String) { - FileLogger.e(TAG, "函数处理出错: $message") - callback?.onError(message) - } - - override fun onFunctionCall(nestedCall: JSONObject) { - FileLogger.d(TAG, "收到嵌套函数调用: ${nestedCall.getString("name")}") - // 递归处理嵌套函数调用 - voiceFunctionHandler.handleFunctionCall( - functionCall = nestedCall, - messages = messages, - callback = this - ) - } - - override fun onExitWithMessage(farewell: String) { - // 播放退出消息 - speakAIResponse(farewell) - - // 同步聊天记录 - sendChatHistoryUpdate("personal_assistant", text, farewell) - - // 停止语音识别 - stopRecognition() - } - } - ) - if (!handled) { - // 如果函数没有被处理，作为普通文本处理 - FileLogger.d(TAG, "函数未处理，作为普通文本处理") - speakAIResponse("我无法处理这个请求") - sendChatHistoryUpdate("personal_assistant", text, "我无法处理这个请求") - } } } ) @@ -375,6 +344,13 @@ class VoiceInteractionHandler( // 停止TTS播放 stopTts() + // 注销广播接收器 + try { + context.unregisterReceiver(exitInteractionReceiver) + } catch (e: Exception) { + FileLogger.e(TAG, "注销广播接收器失败: ${e.message}", e) + } + // 释放Azure资源 azureAsrHelper?.let { FileLogger.d(TAG, "关闭Azure ASR服务") @@ -414,6 +390,7 @@ class VoiceInteractionHandler( putExtra("userMessage", userMessage) putExtra("assistantMessage", assistantMessage) putExtra("timestamp", System.currentTimeMillis()) + setPackage(context.packageName) } // 发送广播 diff --git a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt index 1cb132e1a..dbe64c79e 100644 --- a/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt +++ b/android/app/src/main/kotlin/com/yunqiinnovation/deepsound/VoiceInteractionService.kt @@ -28,7 +28,7 @@ import android.media.AudioAttributes import android.net.Uri import com.yunqiinnovation.deepsound.core.utils.FileLogger import com.yunqiinnovation.azure_speech.AzureAsrHelper -import com.yunqiinnovation.azure_speech.AzureTtsHelper +import com.yunqiinnovation.volcano_speech.VolcanoTtsHelper import com.yunqiinnovation.open_ai_service.OpenAIService @@ -145,6 +145,8 @@ class VoiceInteractionService : Service() { var openaiKey = MainActivity.openaiApiKey var openaiBaseUrl = MainActivity.openaiBaseUrl ?: "" // OpenAI API基本URL var openaiModel = MainActivity.openaiModel ?: "" // OpenAI模型 + var volcanoSpeechAppId = MainActivity.volcanoSpeechAppId ?: "" + var volcanoSpeechAppToken = MainActivity.volcanoSpeechAppToken ?: "" // 如果静态变量中没有配置，尝试从加密存储中加载 if (subscriptionKey.isEmpty() || serviceRegion.isEmpty() || openaiKey.isEmpty()) { @@ -160,6 +162,8 @@ class VoiceInteractionService : Service() { openaiKey = MainActivity.openaiApiKey openaiBaseUrl = MainActivity.openaiBaseUrl ?: "" openaiModel = MainActivity.openaiModel ?: "" + volcanoSpeechAppId = MainActivity.volcanoSpeechAppId ?: "" + volcanoSpeechAppToken = MainActivity.volcanoSpeechAppToken ?: "" FileLogger.d(TAG, "已从加密存储加载配置信息") } else { @@ -168,7 +172,10 @@ class VoiceInteractionService : Service() { } // 初始化语音交互处理器 - voiceInteractionHandler = VoiceInteractionHandler(applicationContext, subscriptionKey, serviceRegion, openaiKey, openaiBaseUrl, openaiModel) + voiceInteractionHandler = VoiceInteractionHandler(applicationContext, + subscriptionKey, serviceRegion, + openaiKey, openaiBaseUrl, openaiModel, + volcanoSpeechAppId, volcanoSpeechAppToken) // 初始化回调 voiceInteractionHandler.setCallback(object : VoiceInteractionHandler.InteractionCallback { @@ -299,7 +306,7 @@ class VoiceInteractionService : Service() { if (voiceInteractionHandler.isRecognitionActive) { val currentTime = System.currentTimeMillis() val elapsedTime = currentTime - lastActivityTime - FileLogger.d(TAG, "hasSpeechDetected=${voiceInteractionHandler.hasSpeechDetected}, isTtsSpeaking=${voiceInteractionHandler.isTtsSpeaking}, elapsedTime=${elapsedTime}") + // FileLogger.d(TAG, "hasSpeechDetected=${voiceInteractionHandler.hasSpeechDetected}, isTtsSpeaking=${voiceInteractionHandler.isTtsSpeaking}, elapsedTime=${elapsedTime}") // 如果超过指定时间没有检测到语音，且不在TTS播放中，暂停语音识别 if (!voiceInteractionHandler.hasSpeechDetected && !voiceInteractionHandler.isTtsSpeaking && @@ -627,6 +634,7 @@ class VoiceInteractionService : Service() { val intent = Intent(ACTION_RECOGNITION_STARTED).apply { // 可以添加额外数据 putExtra("timestamp", System.currentTimeMillis()) + putExtra("packageName", applicationContext.packageName) } // 发送广播 diff --git a/android/build.gradle.kts b/android/build.gradle.kts index 67f214248..5421f73bd 100644 --- a/android/build.gradle.kts +++ b/android/build.gradle.kts @@ -25,3 +25,14 @@ subprojects { tasks.register("clean") { delete(rootProject.layout.buildDirectory) } + +buildscript { + repositories { + google() + mavenCentral() + } + dependencies { + classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:2.1.10") + // 其他依赖... + } +} diff --git a/android/settings.gradle.kts b/android/settings.gradle.kts index c0eebbf43..e332da5f9 100644 --- a/android/settings.gradle.kts +++ b/android/settings.gradle.kts @@ -25,7 +25,7 @@ pluginManagement { plugins { id("dev.flutter.flutter-plugin-loader") id("com.android.application") version "8.7.0" apply false - id("org.jetbrains.kotlin.android") version "1.8.22" apply false + id("org.jetbrains.kotlin.android") version "2.1.10" apply false } include(":app")